pi-mega-compact 0.16.3 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/vector-cortex.js +91 -0
- package/dist/dedup/digest.js +30 -0
- package/dist/dedup/l1-lsh-v2.js +60 -0
- package/dist/dedup/l1-lsh.js +52 -0
- package/dist/dedup/l1-minhash-v2.js +193 -0
- package/dist/dedup/l1-minhash.js +91 -0
- package/dist/dedup/l1-verify.js +54 -0
- package/dist/dedup/mmr.js +45 -0
- package/dist/dedup/normalize.js +39 -0
- package/dist/dedup/raptor/buildHistory.js +164 -0
- package/dist/dedup/raptor/guardrails.js +83 -0
- package/dist/dedup/raptor/incremental.js +304 -0
- package/dist/dedup/raptor/index.js +190 -0
- package/dist/dedup/raptor/kmeans.js +152 -0
- package/dist/dedup/raptor/multilevel.js +184 -0
- package/dist/dedup/raptor/retrieval.js +93 -0
- package/dist/dedup/raptor/summarizer.js +86 -0
- package/dist/dedup/raptor/tree.js +202 -0
- package/dist/dedup/topk.js +60 -0
- package/dist/extensions/dashboard-server/api-contracts/endpoints/registry-ext.js +30 -0
- package/dist/extensions/dashboard-server/api-contracts/endpoints/registry.js +4 -14
- package/dist/extensions/dashboard-server/api-contracts/vector-cortex.js +12 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +11 -0
- package/dist/extensions/dashboard-server/routes-vector-cortex.js +292 -0
- package/dist/extensions/dashboard-server/routes.js +1 -0
- package/dist/extensions/dashboard-server/server.js +9 -1
- package/dist/extensions/mega-events/context-handler.js +11 -0
- package/dist/extensions/mega-runtime/vector-cortex-ledger.js +131 -0
- package/dist/extensions/mega-runtime/vector-cortex-safety.js +100 -0
- package/dist/src/config/vector-cortex.js +91 -0
- package/dist/src/config.js +6 -0
- package/dist/src/dedup/l1-lsh-v2.js +60 -0
- package/dist/src/dedup/l1-minhash-v2.js +193 -0
- package/dist/src/dedup/normalize.js +1 -1
- package/dist/src/vector-cortex/conformance/emit.js +46 -0
- package/dist/src/vector-cortex/conformance/manifest.js +235 -0
- package/dist/src/vector-cortex/conformance/runner.js +111 -0
- package/dist/src/vector-cortex/conformance/triadB-reader.js +95 -0
- package/dist/src/vector-cortex/eval/annotations.js +41 -0
- package/dist/src/vector-cortex/eval/metrics.js +86 -0
- package/dist/src/vector-cortex/eval/observer.js +61 -0
- package/dist/src/vector-cortex/eval/persist.js +67 -0
- package/dist/src/vector-cortex/eval/reader.js +29 -0
- package/dist/src/vector-cortex/eval/types.js +33 -0
- package/dist/src/vector-cortex/ledger/adapter.js +55 -0
- package/dist/src/vector-cortex/ledger/compat-journal.js +206 -0
- package/dist/src/vector-cortex/ledger/emit.js +53 -0
- package/dist/src/vector-cortex/ledger/event-codec.js +72 -0
- package/dist/src/vector-cortex/ledger/event-codecB.js +76 -0
- package/dist/src/vector-cortex/ledger/sqlite.js +200 -0
- package/dist/src/vector-cortex/ledger/store.js +184 -0
- package/dist/src/vector-cortex/ledger/types.js +52 -0
- package/dist/src/vector-cortex/ledger/validator.js +117 -0
- package/dist/src/vector-cortex/migrations/effective-cut-v2.js +103 -0
- package/dist/src/vector-cortex/migrations/minhash-v2.js +209 -0
- package/dist/src/vector-cortex/migrations/occurrence-v2.js +139 -0
- package/dist/src/vector-cortex/replay/cut.js +123 -0
- package/dist/src/vector-cortex/replay/emit.js +58 -0
- package/dist/src/vector-cortex/replay/replay.js +260 -0
- package/dist/src/vector-cortex/replay/replayB.js +73 -0
- package/dist/src/vector-cortex/replay/types.js +52 -0
- package/dist/src/vector-cortex/resilience/breaker-core.js +359 -0
- package/dist/src/vector-cortex/resilience/breaker.js +12 -0
- package/dist/src/vector-cortex/resilience/emit.js +56 -0
- package/dist/src/vector-cortex/resilience/spool-core.js +347 -0
- package/dist/src/vector-cortex/resilience/spool.js +12 -0
- package/dist/src/vector-cortex/resilience/types.js +32 -0
- package/dist/vector-cortex/conformance/emit.js +46 -0
- package/dist/vector-cortex/conformance/manifest.js +235 -0
- package/dist/vector-cortex/conformance/runner.js +111 -0
- package/dist/vector-cortex/conformance/triadB-reader.js +95 -0
- package/dist/vector-cortex/eval/annotations.js +41 -0
- package/dist/vector-cortex/eval/metrics.js +86 -0
- package/dist/vector-cortex/eval/observer.js +61 -0
- package/dist/vector-cortex/eval/persist.js +67 -0
- package/dist/vector-cortex/eval/reader.js +29 -0
- package/dist/vector-cortex/eval/types.js +33 -0
- package/dist/vector-cortex/ledger/adapter.js +55 -0
- package/dist/vector-cortex/ledger/compat-journal.js +206 -0
- package/dist/vector-cortex/ledger/emit.js +53 -0
- package/dist/vector-cortex/ledger/event-codec.js +72 -0
- package/dist/vector-cortex/ledger/event-codecB.js +76 -0
- package/dist/vector-cortex/ledger/sqlite.js +200 -0
- package/dist/vector-cortex/ledger/store.js +184 -0
- package/dist/vector-cortex/ledger/types.js +52 -0
- package/dist/vector-cortex/ledger/validator.js +117 -0
- package/dist/vector-cortex/migrations/effective-cut-v2.js +103 -0
- package/dist/vector-cortex/migrations/minhash-v2.js +209 -0
- package/dist/vector-cortex/migrations/occurrence-v2.js +139 -0
- package/dist/vector-cortex/replay/cut.js +123 -0
- package/dist/vector-cortex/replay/emit.js +58 -0
- package/dist/vector-cortex/replay/replay.js +260 -0
- package/dist/vector-cortex/replay/replayB.js +73 -0
- package/dist/vector-cortex/replay/types.js +52 -0
- package/dist/vector-cortex/resilience/breaker-core.js +359 -0
- package/dist/vector-cortex/resilience/breaker.js +12 -0
- package/dist/vector-cortex/resilience/emit.js +56 -0
- package/dist/vector-cortex/resilience/spool-core.js +347 -0
- package/dist/vector-cortex/resilience/spool.js +12 -0
- package/dist/vector-cortex/resilience/types.js +32 -0
- package/extensions/dashboard-client/dist/assets/{AreaChart-Y_X0Bbjd.js → AreaChart-BQXFF7UI.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{AreaChart-Y_X0Bbjd.js.map → AreaChart-BQXFF7UI.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{BarChart-B9XDb-Ob.js → BarChart-BVtoz9Eg.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{BarChart-B9XDb-Ob.js.map → BarChart-BVtoz9Eg.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{CacheTab-BUGVb1G1.js → CacheTab-trI2TIGy.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{CacheTab-BUGVb1G1.js.map → CacheTab-trI2TIGy.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{EventsTab-dOqLYTsy.js → EventsTab-CNUbk9__.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{EventsTab-dOqLYTsy.js.map → EventsTab-CNUbk9__.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{HealthTab-c2vezKg1.js → HealthTab-K-bd_CM-.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{HealthTab-c2vezKg1.js.map → HealthTab-K-bd_CM-.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-bQYHIcdx.js → MaintenanceTab-CoEioKzx.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-bQYHIcdx.js.map → MaintenanceTab-CoEioKzx.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MemoryMapTab-2L8pH6y2.js → MemoryMapTab-CJo24ulv.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MemoryMapTab-2L8pH6y2.js.map → MemoryMapTab-CJo24ulv.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MetricsTab-CzUx1jpo.js → MetricsTab-DCNlQWWP.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MetricsTab-CzUx1jpo.js.map → MetricsTab-DCNlQWWP.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{OverviewTab-eCNoaclv.js → OverviewTab-CQQlyIQ6.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{OverviewTab-eCNoaclv.js.map → OverviewTab-CQQlyIQ6.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{ReposTab-tb0IW3NF.js → ReposTab-Cqm82_yn.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{ReposTab-tb0IW3NF.js.map → ReposTab-Cqm82_yn.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SessionsTab-C-IoLw5h.js → SessionsTab-A8xTAdET.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SessionsTab-C-IoLw5h.js.map → SessionsTab-A8xTAdET.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SetupTab-BvmFi8gf.js → SetupTab-CqaSgaZl.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SetupTab-BvmFi8gf.js.map → SetupTab-CqaSgaZl.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-DBH-cOlA.js → TimeSavedCard-C_f3q8GR.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-DBH-cOlA.js.map → TimeSavedCard-C_f3q8GR.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TurnsTab-DBMgmvnX.js → TurnsTab-BEsafACo.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TurnsTab-DBMgmvnX.js.map → TurnsTab-BEsafACo.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/VectorCortexTab-DD1CMK80.js +2 -0
- package/extensions/dashboard-client/dist/assets/VectorCortexTab-DD1CMK80.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{WikiTab-1LcQrXk3.js → WikiTab-Bf01M2_Z.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{WikiTab-1LcQrXk3.js.map → WikiTab-Bf01M2_Z.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{button-DtEsZTUE.js → button-BEJPTk_E.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{button-DtEsZTUE.js.map → button-BEJPTk_E.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{card-DcfTmw-4.js → card-DK1V382s.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{card-DcfTmw-4.js.map → card-DK1V382s.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-BsIxUDyw.js → generateCategoricalChart-CRtaPphL.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-BsIxUDyw.js.map → generateCategoricalChart-CRtaPphL.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{index-B93cewiX.js → index-CB6BslE4.js} +34 -29
- package/extensions/dashboard-client/dist/assets/index-CB6BslE4.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/index-CfvziyBi.css +1 -0
- package/extensions/dashboard-client/dist/assets/{switch-Bowd7jCD.js → switch-C4FV6B9k.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{switch-Bowd7jCD.js.map → switch-C4FV6B9k.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{toggle-Bo7kJZ4u.js → toggle-GBmeENMd.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{toggle-Bo7kJZ4u.js.map → toggle-GBmeENMd.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{useSSE-zNhb0c3u.js → useSSE-BOKZGSn1.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{useSSE-zNhb0c3u.js.map → useSSE-BOKZGSn1.js.map} +1 -1
- package/extensions/dashboard-client/dist/index.html +2 -2
- package/extensions/dashboard-client/src/App.tsx +2 -0
- package/extensions/dashboard-client/src/api/vector-cortex.ts +54 -0
- package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +272 -0
- package/extensions/dashboard-client/src/tabs/registry.ts +4 -1
- package/extensions/dashboard-client/src/types/vector-cortex.ts +84 -0
- package/extensions/dashboard-client/tsconfig.json +1 -1
- package/extensions/dashboard-server/api-contracts/endpoints/registry-ext.ts +67 -0
- package/extensions/dashboard-server/api-contracts/endpoints/registry.ts +4 -21
- package/extensions/dashboard-server/api-contracts/index.ts +3 -0
- package/extensions/dashboard-server/api-contracts/vector-cortex.ts +136 -0
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +41 -0
- package/extensions/dashboard-server/routes-vector-cortex.ts +332 -0
- package/extensions/dashboard-server/routes.ts +6 -0
- package/extensions/dashboard-server/server.ts +8 -0
- package/extensions/mega-events/context-handler.ts +16 -0
- package/extensions/mega-runtime/vector-cortex-ledger.ts +165 -0
- package/extensions/mega-runtime/vector-cortex-safety.ts +181 -0
- package/package.json +2 -1
- package/src/config/vector-cortex.ts +98 -0
- package/src/config.ts +28 -0
- package/src/dedup/l1-lsh-v2.ts +78 -0
- package/src/dedup/l1-minhash-v2.ts +221 -0
- package/src/dedup/normalize.ts +1 -1
- package/src/store/sqlite/game-scores.ts +2 -1
- package/src/vector-cortex/conformance/emit.ts +56 -0
- package/src/vector-cortex/conformance/manifest.ts +298 -0
- package/src/vector-cortex/conformance/runner.ts +165 -0
- package/src/vector-cortex/conformance/triadB-reader.ts +110 -0
- package/src/vector-cortex/eval/annotations.ts +70 -0
- package/src/vector-cortex/eval/metrics.ts +113 -0
- package/src/vector-cortex/eval/observer.ts +86 -0
- package/src/vector-cortex/eval/persist.ts +73 -0
- package/src/vector-cortex/eval/reader.ts +45 -0
- package/src/vector-cortex/eval/types.ts +89 -0
- package/src/vector-cortex/ledger/adapter.ts +71 -0
- package/src/vector-cortex/ledger/compat-journal.ts +234 -0
- package/src/vector-cortex/ledger/emit.ts +86 -0
- package/src/vector-cortex/ledger/event-codec.ts +79 -0
- package/src/vector-cortex/ledger/event-codecB.ts +87 -0
- package/src/vector-cortex/ledger/sqlite.ts +265 -0
- package/src/vector-cortex/ledger/store.ts +380 -0
- package/src/vector-cortex/ledger/types.ts +141 -0
- package/src/vector-cortex/ledger/validator.ts +124 -0
- package/src/vector-cortex/migrations/effective-cut-v2.ts +149 -0
- package/src/vector-cortex/migrations/minhash-v2.ts +263 -0
- package/src/vector-cortex/migrations/occurrence-v2.ts +192 -0
- package/src/vector-cortex/replay/cut.ts +169 -0
- package/src/vector-cortex/replay/emit.ts +89 -0
- package/src/vector-cortex/replay/replay.ts +308 -0
- package/src/vector-cortex/replay/replayB.ts +97 -0
- package/src/vector-cortex/replay/types.ts +145 -0
- package/src/vector-cortex/resilience/breaker-core.ts +444 -0
- package/src/vector-cortex/resilience/breaker.ts +22 -0
- package/src/vector-cortex/resilience/emit.ts +99 -0
- package/src/vector-cortex/resilience/spool-core.ts +411 -0
- package/src/vector-cortex/resilience/spool.ts +22 -0
- package/src/vector-cortex/resilience/types.ts +158 -0
- package/extensions/dashboard-client/dist/assets/index-B93cewiX.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/index-C_tCbOI0.css +0 -1
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/resilience/spool-core.ts — VC0C durable spool implementation (VC0C).
|
|
3
|
+
*
|
|
4
|
+
* Implementation file behind the `spool.ts` factory barrel. See spool.ts for the
|
|
5
|
+
* public surface; this file owns the TRIAD_RESILIENCE §spool append/fsync/ack
|
|
6
|
+
* protocol.
|
|
7
|
+
*
|
|
8
|
+
* Implements TRIAD_RESILIENCE.md §Spool protocol as the mode-B independent,
|
|
9
|
+
* deterministic, local disk spool that derives directly from authority:
|
|
10
|
+
*
|
|
11
|
+
* - One spool file per session: `spool-<session>.spool` under a state dir.
|
|
12
|
+
* - Header records schema, session, first sequence, and the prior durable
|
|
13
|
+
* contiguous high-water.
|
|
14
|
+
* - Frames are length-prefixed binary records carrying seq + eventId +
|
|
15
|
+
* original bytes + SHA-256 + CRC32C (Castagnoli). A frame is FSYNC'd to disk
|
|
16
|
+
* before the writer acknowledges `SPOOLED`; only a durable ledger commit
|
|
17
|
+
* (after `drain` inserts into authority and the ledger is fsynced) returns
|
|
18
|
+
* `SPOOL_COMMITTED` and advances the contiguous high-water.
|
|
19
|
+
* - `drain` strictly sorts frames by `(seq, eventId)`, rejects gaps/conflicts,
|
|
20
|
+
* inserts idempotently by `(eventId, digest)`, appends an fsynced ack frame,
|
|
21
|
+
* then advances contiguous high-water. A duplicate with the same id+digest is
|
|
22
|
+
* acknowledged (`SPOOL_IDEMPOTENT_ACK`); the same id with a DIFFERENT digest
|
|
23
|
+
* is `SPOOL_MANUAL_HALT`.
|
|
24
|
+
* - Crash before an ack safely replays: on reopen, frames after the last ack
|
|
25
|
+
* are re-drained (the ack frame carries the committed seq).
|
|
26
|
+
* - A derived builder may never read beyond the durable contiguous authority
|
|
27
|
+
* high-water; during an authority outage the high-water freezes (drain cannot
|
|
28
|
+
* commit) even while the spool keeps accepting frames.
|
|
29
|
+
*
|
|
30
|
+
* LOCAL ONLY: filesystem + node:crypto, zero network (PREVENT-PI-004); no `any`
|
|
31
|
+
* (PREVENT-011); synchronous durable writes (fsyncSync) per the spec.
|
|
32
|
+
*/
|
|
33
|
+
import { createHash } from "node:crypto";
|
|
34
|
+
import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, writeSync } from "node:fs";
|
|
35
|
+
import { basename, join } from "node:path";
|
|
36
|
+
export const SPOOL_SCHEMA = "spool-v1";
|
|
37
|
+
/** Sentinel eventId of an acknowledgement frame (carries the committed seq). */
|
|
38
|
+
export const ACK_EVENT_ID = "__ack__";
|
|
39
|
+
/** Fixed on-disk width of the digest field: "sha256:" + 64 hex chars = 71 bytes. */
|
|
40
|
+
export const DIGEST_FIELD_LEN = "sha256:".length + 64;
|
|
41
|
+
/** CRC32C (Castagnoli) table — seeded once. */
|
|
42
|
+
const CRC_TABLE = (() => {
|
|
43
|
+
const t = new Array(256);
|
|
44
|
+
for (let n = 0; n < 256; n++) {
|
|
45
|
+
let c = n;
|
|
46
|
+
for (let k = 0; k < 8; k++)
|
|
47
|
+
c = c & 1 ? 0x82f63b78 ^ (c >>> 1) : c >>> 1;
|
|
48
|
+
t[n] = c >>> 0;
|
|
49
|
+
}
|
|
50
|
+
return t;
|
|
51
|
+
})();
|
|
52
|
+
export function crc32c(bytes) {
|
|
53
|
+
let c = 0xffffffff;
|
|
54
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
55
|
+
c = CRC_TABLE[(c ^ bytes[i]) & 0xff] ^ (c >>> 8);
|
|
56
|
+
}
|
|
57
|
+
return (c ^ 0xffffffff) >>> 0;
|
|
58
|
+
}
|
|
59
|
+
export function sha256Hex(bytes) {
|
|
60
|
+
return createHash("sha256").update(bytes).digest("hex");
|
|
61
|
+
}
|
|
62
|
+
export function createSpool(opts) {
|
|
63
|
+
mkdirSync(opts.dir, { recursive: true });
|
|
64
|
+
const bySession = new Map();
|
|
65
|
+
return {
|
|
66
|
+
session(session) {
|
|
67
|
+
let s = bySession.get(session);
|
|
68
|
+
if (!s) {
|
|
69
|
+
s = new SessionSpoolImpl(join(opts.dir, `spool-${basename(session)}.spool`), session, opts);
|
|
70
|
+
bySession.set(session, s);
|
|
71
|
+
}
|
|
72
|
+
return s;
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Durably manages one session's spool file. Fully synchronous: every append is
|
|
78
|
+
* fsynced before acknowledging; every committed drain appends+fsyncs an ack
|
|
79
|
+
* frame. Reopen parses the header + frames, stops at a torn trailing frame
|
|
80
|
+
* (crash mid-write => unacknowledged tail), and drains only frames strictly
|
|
81
|
+
* beyond the recovered contiguous high-water — crash before ack safely replays.
|
|
82
|
+
*/
|
|
83
|
+
class SessionSpoolImpl {
|
|
84
|
+
session;
|
|
85
|
+
file;
|
|
86
|
+
outage;
|
|
87
|
+
reporter;
|
|
88
|
+
header;
|
|
89
|
+
frames = [];
|
|
90
|
+
ackedSeq;
|
|
91
|
+
frontierFrozen = false;
|
|
92
|
+
constructor(file, session, opts) {
|
|
93
|
+
this.file = file;
|
|
94
|
+
this.session = session;
|
|
95
|
+
this.outage = opts.authorityOutage ?? (() => false);
|
|
96
|
+
this.reporter = opts.reporter;
|
|
97
|
+
mkdirSync(join(file, ".."), { recursive: true });
|
|
98
|
+
if (existsSync(file)) {
|
|
99
|
+
const parsed = this.readExisting();
|
|
100
|
+
this.header = parsed.header;
|
|
101
|
+
this.frames = parsed.frames;
|
|
102
|
+
this.ackedSeq = parsed.highWater;
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
this.header = { schema: SPOOL_SCHEMA, session, firstSeq: 1, priorHighWater: "0" };
|
|
106
|
+
this.ackedSeq = 0n;
|
|
107
|
+
this.writeHeader();
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
writeHeader() {
|
|
111
|
+
const fd = openSync(this.file, "a");
|
|
112
|
+
try {
|
|
113
|
+
writeSync(fd, Buffer.from(encodeHeader(this.header), "utf8"));
|
|
114
|
+
fsyncSync(fd);
|
|
115
|
+
}
|
|
116
|
+
finally {
|
|
117
|
+
closeSync(fd);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
/** Reopen: parse header + frames; recover the last contiguous ack high-water. */
|
|
121
|
+
readExisting() {
|
|
122
|
+
const raw = readFileSync(this.file);
|
|
123
|
+
const nl = raw.indexOf(10);
|
|
124
|
+
if (nl < 0) {
|
|
125
|
+
return { header: { schema: SPOOL_SCHEMA, session: this.session, firstSeq: 1, priorHighWater: "0" }, frames: [], highWater: 0n };
|
|
126
|
+
}
|
|
127
|
+
const header = parseHeader(raw.subarray(0, nl).toString("utf8"));
|
|
128
|
+
let highWater = BigInt(header.priorHighWater);
|
|
129
|
+
const frames = [];
|
|
130
|
+
let off = nl + 1;
|
|
131
|
+
for (;;) {
|
|
132
|
+
if (off >= raw.length)
|
|
133
|
+
break;
|
|
134
|
+
const f = tryParseFrame(raw.subarray(off));
|
|
135
|
+
if (!f)
|
|
136
|
+
break; // torn/unacknowledged tail — stop, replay from last ack
|
|
137
|
+
const frame = { ...f, session: this.session };
|
|
138
|
+
if (frame.eventId === ACK_EVENT_ID) {
|
|
139
|
+
highWater = frame.seq; // ack frame carries the committed seq
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
frames.push(frame);
|
|
143
|
+
}
|
|
144
|
+
const len = raw.readUInt32LE(off);
|
|
145
|
+
off += 4 + len;
|
|
146
|
+
}
|
|
147
|
+
// Only frames strictly beyond the recovered high-water need re-drain.
|
|
148
|
+
const unacked = frames.filter((f) => f.seq > highWater);
|
|
149
|
+
return { header, frames: unacked, highWater };
|
|
150
|
+
}
|
|
151
|
+
append(input) {
|
|
152
|
+
const digest = input.digest ?? `sha256:${sha256Hex(input.bytes)}`;
|
|
153
|
+
const frame = {
|
|
154
|
+
session: this.session,
|
|
155
|
+
seq: input.seq,
|
|
156
|
+
eventId: input.eventId,
|
|
157
|
+
digest,
|
|
158
|
+
bytes: input.bytes,
|
|
159
|
+
crc32c: crc32c(input.bytes),
|
|
160
|
+
};
|
|
161
|
+
const framed = encodeFrame(frame);
|
|
162
|
+
// Fsync BEFORE acknowledging SPOOLED.
|
|
163
|
+
const fd = openSync(this.file, "a");
|
|
164
|
+
try {
|
|
165
|
+
writeSync(fd, framed, 0, framed.length);
|
|
166
|
+
fsyncSync(fd);
|
|
167
|
+
}
|
|
168
|
+
finally {
|
|
169
|
+
closeSync(fd);
|
|
170
|
+
}
|
|
171
|
+
this.frames.push(frame);
|
|
172
|
+
return { verdict: "SPOOLED", seq: input.seq };
|
|
173
|
+
}
|
|
174
|
+
drain(insert) {
|
|
175
|
+
const tail = this.frames;
|
|
176
|
+
if (tail.length === 0)
|
|
177
|
+
return { verdict: "SPOOL_COMMITTED", committedSeq: this.ackedSeq };
|
|
178
|
+
// Strictly sort by (seq, eventId). NOTE: we do NOT clear this.frames up front
|
|
179
|
+
// (VC0C-Q04) — the queue is only cleared once every insert has succeeded and
|
|
180
|
+
// the ack is durably appended. If any insert throws (rather than returning a
|
|
181
|
+
// conflict), the unacked tail must be retained for a later retry, not dropped.
|
|
182
|
+
const sorted = [...tail].sort(compareFrames);
|
|
183
|
+
let committed = this.ackedSeq;
|
|
184
|
+
for (const f of sorted) {
|
|
185
|
+
if (f.seq !== committed + 1n) {
|
|
186
|
+
// Gap: requeue everything beyond the last committed seq and halt.
|
|
187
|
+
this.frames = sorted.filter((x) => x.seq > committed);
|
|
188
|
+
return { verdict: "SPOOL_MANUAL_HALT", committedSeq: committed, reason: `TRI_SPOOL_GAP expected ${committed + 1n} got ${f.seq}` };
|
|
189
|
+
}
|
|
190
|
+
let outcome;
|
|
191
|
+
try {
|
|
192
|
+
outcome = insert(this.session, f.seq, f.eventId, f.digest, f.bytes);
|
|
193
|
+
}
|
|
194
|
+
catch {
|
|
195
|
+
// Authority insert THREW (e.g. a transient authority write error). This is
|
|
196
|
+
// NON-FATAL: requeue the unacked tail and halt, mirroring the manual-halt
|
|
197
|
+
// path, instead of propagating (a propagating throw would lose the frames
|
|
198
|
+
// and leave a retry seeing SPOOL_COMMITTED-with-nothing-processed).
|
|
199
|
+
this.frames = sorted.filter((x) => x.seq > committed);
|
|
200
|
+
return { verdict: "SPOOL_MANUAL_HALT", committedSeq: committed, reason: "TRI_SPOOL_INSERT_THROW" };
|
|
201
|
+
}
|
|
202
|
+
if (outcome === "committed" || outcome === "idempotent") {
|
|
203
|
+
committed = f.seq;
|
|
204
|
+
}
|
|
205
|
+
else {
|
|
206
|
+
this.frames = sorted.filter((x) => x.seq > committed);
|
|
207
|
+
return { verdict: "SPOOL_MANUAL_HALT", committedSeq: committed, reason: "TRI_SPOOL_CONFLICT conflicting digest" };
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
// All frames committed: clear the in-memory queue, then durably commit by
|
|
211
|
+
// appending+fsyncing an ack frame and advancing the high-water.
|
|
212
|
+
this.frames = [];
|
|
213
|
+
this.appendAck(committed);
|
|
214
|
+
this.ackedSeq = committed;
|
|
215
|
+
return { verdict: "SPOOL_COMMITTED", committedSeq: committed };
|
|
216
|
+
}
|
|
217
|
+
appendAck(seq) {
|
|
218
|
+
const frame = {
|
|
219
|
+
session: this.session,
|
|
220
|
+
seq,
|
|
221
|
+
eventId: ACK_EVENT_ID,
|
|
222
|
+
digest: `sha256:${sha256Hex(new Uint8Array(0))}`,
|
|
223
|
+
bytes: new Uint8Array(0),
|
|
224
|
+
crc32c: crc32c(new Uint8Array(0)),
|
|
225
|
+
};
|
|
226
|
+
const framed = encodeFrame(frame);
|
|
227
|
+
const fd = openSync(this.file, "a");
|
|
228
|
+
try {
|
|
229
|
+
writeSync(fd, framed, 0, framed.length);
|
|
230
|
+
fsyncSync(fd);
|
|
231
|
+
}
|
|
232
|
+
finally {
|
|
233
|
+
closeSync(fd);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
highWater() {
|
|
237
|
+
// The contiguous authority high-water advances ONLY on a durable ack
|
|
238
|
+
// (appendAck after a successful drain commit). During authority outage drain
|
|
239
|
+
// cannot commit, so ackedSeq naturally freezes here — an explicit outage
|
|
240
|
+
// branch would return the same value and was dead code (VC0C-Q05). The
|
|
241
|
+
// freeze/outage SIGNAL is the `frozen()`/`frontierFrozen` flag, not this
|
|
242
|
+
// scalar; frames appended during outage remain in the unacked tail.
|
|
243
|
+
return this.ackedSeq;
|
|
244
|
+
}
|
|
245
|
+
frozen() {
|
|
246
|
+
return this.frontierFrozen || this.outage();
|
|
247
|
+
}
|
|
248
|
+
freezeFrontier() {
|
|
249
|
+
const firstFreeze = !this.frontierFrozen && !this.outage();
|
|
250
|
+
this.frontierFrozen = true;
|
|
251
|
+
// Emit vector_cortex_frontier_frozen only on an actual freeze transition
|
|
252
|
+
// (once per outage; flag-gated + non-fatal inside the reporter). No event on
|
|
253
|
+
// an already-frozen or authority-outage-covered frontier. flag OFF => no-op.
|
|
254
|
+
if (firstFreeze) {
|
|
255
|
+
this.reporter?.frontierFrozen({
|
|
256
|
+
session: this.session,
|
|
257
|
+
committedSeq: String(this.ackedSeq),
|
|
258
|
+
frozenHighWater: String(this.ackedSeq),
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
return this.ackedSeq;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
function compareFrames(a, b) {
|
|
265
|
+
if (a.seq !== b.seq)
|
|
266
|
+
return a.seq < b.seq ? -1 : 1;
|
|
267
|
+
return a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0;
|
|
268
|
+
}
|
|
269
|
+
function encodeHeader(h) {
|
|
270
|
+
return `${JSON.stringify(h)}\n`;
|
|
271
|
+
}
|
|
272
|
+
function parseHeader(line) {
|
|
273
|
+
const raw = JSON.parse(line);
|
|
274
|
+
if (raw.schema !== SPOOL_SCHEMA)
|
|
275
|
+
throw new Error("TRI_SPOOL_SCHEMA");
|
|
276
|
+
return raw;
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Encode one binary frame: [u32 len][seq i64][eventId UTF-8 NUL][bytes][NUL][digest][u32 crc32c].
|
|
280
|
+
* The digest field is DIGEST_FIELD_LEN bytes ("sha256:" + 64 hex = 71) — sizing it
|
|
281
|
+
* to fit the FULL digest string is load-bearing for read-side verification
|
|
282
|
+
* (VC0C-Q03): a 64-byte cap silently truncated Buffer.write, so the stored digest
|
|
283
|
+
* could never match the recomputed value and every frame would be rejected on read.
|
|
284
|
+
* body length = 8 + eventId.length + 1 + bytes.length + 1 + DIGEST_FIELD_LEN + 4.
|
|
285
|
+
*/
|
|
286
|
+
function encodeFrame(f) {
|
|
287
|
+
const idBuf = Buffer.from(f.eventId, "utf8");
|
|
288
|
+
const bodyLen = 8 + idBuf.length + 1 + f.bytes.length + 1 + DIGEST_FIELD_LEN + 4;
|
|
289
|
+
const body = Buffer.alloc(bodyLen);
|
|
290
|
+
let o = 0;
|
|
291
|
+
body.writeBigInt64LE(BigInt(f.seq), o);
|
|
292
|
+
o += 8;
|
|
293
|
+
idBuf.copy(body, o);
|
|
294
|
+
o += idBuf.length;
|
|
295
|
+
body[o] = 0;
|
|
296
|
+
o += 1;
|
|
297
|
+
Buffer.from(f.bytes).copy(body, o);
|
|
298
|
+
o += f.bytes.length;
|
|
299
|
+
body[o] = 0;
|
|
300
|
+
o += 1;
|
|
301
|
+
body.write(f.digest, o, "utf8");
|
|
302
|
+
o += DIGEST_FIELD_LEN;
|
|
303
|
+
body.writeUInt32LE(f.crc32c, o);
|
|
304
|
+
o += 4;
|
|
305
|
+
const lenBuf = Buffer.alloc(4);
|
|
306
|
+
lenBuf.writeUInt32LE(bodyLen, 0);
|
|
307
|
+
return Buffer.concat([lenBuf, body]);
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Parse a length-prefixed frame; returns null for a torn/truncated tail OR a
|
|
311
|
+
* CRYPTOGRAPHIC INTEGRITY MISMATCH (VC0C-Q03). The stored SHA-256 digest and
|
|
312
|
+
* CRC32C are recomputed over the parsed bytes and compared — a same-length bit
|
|
313
|
+
* flip must not survive reopen and be drained/committed. A mismatch is treated
|
|
314
|
+
* like a torn tail: `readExisting` stops there and the frame is NOT accepted —
|
|
315
|
+
* it remains unacknowledged, so replay/requeue preserves it rather than
|
|
316
|
+
* silently committing corrupted bytes.
|
|
317
|
+
*/
|
|
318
|
+
function tryParseFrame(buf) {
|
|
319
|
+
if (buf.length < 4)
|
|
320
|
+
return null;
|
|
321
|
+
const len = buf.readUInt32LE(0);
|
|
322
|
+
if (buf.length < 4 + len)
|
|
323
|
+
return null; // torn
|
|
324
|
+
const body = buf.subarray(4, 4 + len);
|
|
325
|
+
let o = 0;
|
|
326
|
+
const seq = body.readBigInt64LE(o);
|
|
327
|
+
o += 8;
|
|
328
|
+
const idEnd = body.indexOf(0, o);
|
|
329
|
+
if (idEnd < 0)
|
|
330
|
+
return null;
|
|
331
|
+
const eventId = body.subarray(o, idEnd).toString("utf8");
|
|
332
|
+
o = idEnd + 1;
|
|
333
|
+
const bytesStart = o;
|
|
334
|
+
const nul2 = body.indexOf(0, o);
|
|
335
|
+
if (nul2 < 0)
|
|
336
|
+
return null;
|
|
337
|
+
const bytes = new Uint8Array(body.subarray(bytesStart, nul2));
|
|
338
|
+
o = nul2 + 1;
|
|
339
|
+
const digest = body.subarray(o, o + DIGEST_FIELD_LEN).toString("utf8");
|
|
340
|
+
const storedCrc = body.readUInt32LE(o + DIGEST_FIELD_LEN);
|
|
341
|
+
// Recompute and verify — write-only integrity would let a same-length bit flip
|
|
342
|
+
// drain/commit silently. Return null (treat as torn/corrupt) on mismatch.
|
|
343
|
+
if (digest !== `sha256:${sha256Hex(bytes)}` || storedCrc !== crc32c(bytes)) {
|
|
344
|
+
return null;
|
|
345
|
+
}
|
|
346
|
+
return { session: "", seq, eventId, digest, bytes, crc32c: storedCrc };
|
|
347
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/resilience/spool.ts — VC0C durable spool barrel (VC0C).
|
|
3
|
+
*
|
|
4
|
+
* Thin barrel over the append/fsync/ack spool implementation in `spool-core.ts`
|
|
5
|
+
* (delegate-shell + impl-file split keeps the frequently-touched file small).
|
|
6
|
+
* Re-exports the public `createSpool` factory, the mode-B spool/session types,
|
|
7
|
+
* and the SHA-256 / CRC32C helpers for the safety adapter and tests.
|
|
8
|
+
*
|
|
9
|
+
* Full protocol semantics live in spool-core.ts (TRIAD_RESILIENCE §spool).
|
|
10
|
+
* Local-only, no network (PREVENT-PI-004), no `any` (PREVENT-011).
|
|
11
|
+
*/
|
|
12
|
+
export { createSpool, SPOOL_SCHEMA, ACK_EVENT_ID, crc32c, sha256Hex } from "./spool-core.js";
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/resilience/types.ts — VC0C breaker/triad contracts (VC0C).
|
|
3
|
+
*
|
|
4
|
+
* Owns `Mode`, `TriadResult<T>`, `BreakerRecord`, `KillDecision`, and the
|
|
5
|
+
* `Breaker` circuit-breaker seam from CONTRACTS.md §TriadResult and breaker
|
|
6
|
+
* seam, plus the durable spool record shapes. Pure type/schema definitions +
|
|
7
|
+
* small pure predicates (no storage, no console, no network, no side effects —
|
|
8
|
+
* PREVENT-PI-004 / PREVENT-011). The live state machine lives in
|
|
9
|
+
* `breaker.ts`/`spool.ts`; the safety adapter selects A→B→C in
|
|
10
|
+
* `extensions/mega-runtime/vector-cortex-safety.ts`.
|
|
11
|
+
*
|
|
12
|
+
* Triad semantics (TRIAD_RESILIENCE.md): A is optimized/learned; B is
|
|
13
|
+
* deterministic/local and derives directly from authority; C is continuity, not
|
|
14
|
+
* semantic completeness — it uses only the exact current transcript and reports
|
|
15
|
+
* that limitation.
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* Registered TRI conformance ID range (TRI-001..030). The acceptance test reads
|
|
19
|
+
* these rows from the v2 `resilience/` domain and asserts each returns its
|
|
20
|
+
* manifest bytes or exactly its listed failure code. TRI-001..015 are owned by
|
|
21
|
+
* VC0C (window/probe/hysteresis/cooldown transitions); TRI-016..030 span the
|
|
22
|
+
* spool protocol (frame, drain, gap, digest conflict, ack crash, frozen
|
|
23
|
+
* frontier). Mirrors EVT_IDS / CUT_IDS / M3_IDS.
|
|
24
|
+
*/
|
|
25
|
+
export const TRI_IDS = [
|
|
26
|
+
"TRI-001", "TRI-002", "TRI-003", "TRI-004", "TRI-005",
|
|
27
|
+
"TRI-006", "TRI-007", "TRI-008", "TRI-009", "TRI-010",
|
|
28
|
+
"TRI-011", "TRI-012", "TRI-013", "TRI-014", "TRI-015",
|
|
29
|
+
"TRI-016", "TRI-017", "TRI-018", "TRI-019", "TRI-020",
|
|
30
|
+
"TRI-021", "TRI-022", "TRI-023", "TRI-024", "TRI-025",
|
|
31
|
+
"TRI-026", "TRI-027", "TRI-028", "TRI-029", "TRI-030",
|
|
32
|
+
];
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/conformance/emit.ts — VC1C observability seam.
|
|
3
|
+
*
|
|
4
|
+
* Emits the three VC1C structured events, all gated on `MEGACOMPACT_VC1C`
|
|
5
|
+
* (mode C parity: flag OFF => zero emissions). Every event is a JSON line with
|
|
6
|
+
* `ts` + `event` (ENGINEERING_PRACTICES §8); the emitters below capture the
|
|
7
|
+
* event name + fields and are never fatal on consumer failure (non-fatal
|
|
8
|
+
* observability, never breaks the agent loop).
|
|
9
|
+
*
|
|
10
|
+
* vector_cortex_minhash_v2_backfilled — M4 backfill wrote v2 rows
|
|
11
|
+
* vector_cortex_conformance_case_checked — a v2 conformance case dispatched
|
|
12
|
+
* vector_cortex_downgrade_copy_written — a downgrade legacy copy written
|
|
13
|
+
*
|
|
14
|
+
* No network, no side effects beyond the supplied emit callback
|
|
15
|
+
* (PREVENT-PI-004 / PREVENT-011).
|
|
16
|
+
*/
|
|
17
|
+
import { VC1C_ENABLED } from "../../config/vector-cortex.js";
|
|
18
|
+
/** A flag-gated no-op reporter (zero emissions, default when none injected). */
|
|
19
|
+
export const NOOP_CONFORMANCE_REPORTER = {
|
|
20
|
+
backfilled: () => { },
|
|
21
|
+
caseChecked: () => { },
|
|
22
|
+
downgradeWritten: () => { },
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Flag-gated emit: no-op when VC1C is off or no emitter is supplied. The
|
|
26
|
+
* returned reporter is itself flag-gated (`VC1C_ENABLED`), so wiring it into a
|
|
27
|
+
* runtime seam yields zero emissions when `MEGACOMPACT_VC1C=0` (byte-identical
|
|
28
|
+
* to the predecessor).
|
|
29
|
+
*/
|
|
30
|
+
export function createConformanceReporter(emit) {
|
|
31
|
+
const fire = (event, fields) => {
|
|
32
|
+
if (!VC1C_ENABLED())
|
|
33
|
+
return;
|
|
34
|
+
try {
|
|
35
|
+
emit?.(event, { ...fields, ts: new Date().toISOString() });
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
/* non-fatal observability */
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
return {
|
|
42
|
+
backfilled: (fields) => fire("vector_cortex_minhash_v2_backfilled", fields),
|
|
43
|
+
caseChecked: (fields) => fire("vector_cortex_conformance_case_checked", fields),
|
|
44
|
+
downgradeWritten: (fields) => fire("vector_cortex_downgrade_copy_written", fields),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/conformance/manifest.ts — FixtureManifestV2 (VC1C).
|
|
3
|
+
*
|
|
4
|
+
* Owns the v2 conformance manifest: reads the authoritative
|
|
5
|
+
* `conformance/vector-cortex/v2/manifest.json` plus the on-disk fixture tree
|
|
6
|
+
* and validates it canonically. Each FixtureManifestV2 entry carries:
|
|
7
|
+
* id, domain, inputDigest, expectedDigest, failureCode, algorithmTuple.
|
|
8
|
+
*
|
|
9
|
+
* Canonical validation rejects a mismatched corpus with the frozen codes:
|
|
10
|
+
* CONF_EXTRA_FIXTURE — a file on disk that the manifest does not list;
|
|
11
|
+
* CONF_MISSING_FIXTURE — a manifest entry with no file on disk;
|
|
12
|
+
* CONF_DIGEST_DRIFT — on-disk bytes do not match the manifest SHA-256.
|
|
13
|
+
*
|
|
14
|
+
* Key ordering rule (canonical JSON): object keys are sorted by UTF-8 bytes;
|
|
15
|
+
* a manifest that is not canonical (or a fixture whose canonical re-serialization
|
|
16
|
+
* differs) is itself a digest-drift / noncanonical condition. The invariant:
|
|
17
|
+
* canonical valid manifests converge to ONE digest (the corpus is a single
|
|
18
|
+
* reproducible byte image).
|
|
19
|
+
*
|
|
20
|
+
* Pure FS reads + pure predicates; no network, no side effects on authority
|
|
21
|
+
* data (PREVENT-PI-004 / PREVENT-011).
|
|
22
|
+
*/
|
|
23
|
+
import { readFileSync, readdirSync, statSync } from "node:fs";
|
|
24
|
+
import { createHash as sha256 } from "node:crypto";
|
|
25
|
+
import { join, relative, sep } from "node:path";
|
|
26
|
+
/** Frozen VC1C conformance failure codes (the "reject unknown / extra / drifted"). */
|
|
27
|
+
export const CONF_FAIL = {
|
|
28
|
+
EXTRA_FIXTURE: "CONF_EXTRA_FIXTURE",
|
|
29
|
+
MISSING_FIXTURE: "CONF_MISSING_FIXTURE",
|
|
30
|
+
DIGEST_DRIFT: "CONF_DIGEST_DRIFT",
|
|
31
|
+
NONCANONICAL: "CONF_NONCANONICAL",
|
|
32
|
+
UNKNOWN_DOMAIN: "CONF_UNKNOWN_DOMAIN",
|
|
33
|
+
};
|
|
34
|
+
/** Walk a directory tree returning relative POSIX paths (sorted). */
|
|
35
|
+
function walk(dir, base, acc = []) {
|
|
36
|
+
for (const name of readdirSync(dir)) {
|
|
37
|
+
const p = join(dir, name);
|
|
38
|
+
const st = statSync(p);
|
|
39
|
+
if (st.isDirectory())
|
|
40
|
+
walk(p, base, acc);
|
|
41
|
+
else
|
|
42
|
+
acc.push(relative(base, p).split(sep).join("/"));
|
|
43
|
+
}
|
|
44
|
+
return acc.sort();
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Canonical JSON bytes for a value (UTF-8, NFC keys, sorted, shortest numbers).
|
|
48
|
+
* Matches the stricter `canonicalNumber` in scripts/vector-cortex-conformance.mjs:
|
|
49
|
+
* a non-finite number or -0 has no canonical shortest representation and is
|
|
50
|
+
* rejected, so the two "canonical" serializers can never diverge for the same
|
|
51
|
+
* value (thereby preserving the "converge to one digest" invariant).
|
|
52
|
+
*/
|
|
53
|
+
function canonicalValue(value) {
|
|
54
|
+
if (value === null || typeof value !== "object") {
|
|
55
|
+
if (typeof value === "number") {
|
|
56
|
+
if (!Number.isFinite(value) || Object.is(value, -0)) {
|
|
57
|
+
throw new Error(`non-canonical number: ${value}`);
|
|
58
|
+
}
|
|
59
|
+
if (Number.isInteger(value) && Number.isSafeInteger(value))
|
|
60
|
+
return String(value);
|
|
61
|
+
return JSON.stringify(value);
|
|
62
|
+
}
|
|
63
|
+
return JSON.stringify(value);
|
|
64
|
+
}
|
|
65
|
+
if (Array.isArray(value))
|
|
66
|
+
return `[${value.map((v) => canonicalValue(v)).join(",")}]`;
|
|
67
|
+
const keys = Object.keys(value)
|
|
68
|
+
.map((k) => k.normalize("NFC"))
|
|
69
|
+
.sort();
|
|
70
|
+
const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalValue(value[k])}`);
|
|
71
|
+
return `{${parts.join(",")}}`;
|
|
72
|
+
}
|
|
73
|
+
function canonicalJson(value) {
|
|
74
|
+
return Buffer.from(`${canonicalValue(value)}\n`, "utf8");
|
|
75
|
+
}
|
|
76
|
+
function sha256Hex(bytes) {
|
|
77
|
+
return sha256("sha256").update(bytes).digest("hex");
|
|
78
|
+
}
|
|
79
|
+
/** parse a manifest.json (must be a valid v2 manifest object). */
|
|
80
|
+
function parseManifest(raw) {
|
|
81
|
+
const obj = JSON.parse(raw);
|
|
82
|
+
if (obj === null || typeof obj !== "object" || !Array.isArray(obj.fixtures)) {
|
|
83
|
+
throw new Error("CONF_MANIFEST_INVALID: manifest.fixtures must be an array");
|
|
84
|
+
}
|
|
85
|
+
return {
|
|
86
|
+
version: typeof obj.version === "string" ? obj.version : "2",
|
|
87
|
+
owner: typeof obj.owner === "string" ? obj.owner : "",
|
|
88
|
+
fixtures: obj.fixtures,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Read + normalize the v2 manifest and on-disk tree into a FixtureManifestV2.
|
|
93
|
+
* `fixtureRoot` is the conformance v2 root directory (injected by callers so
|
|
94
|
+
* `src/` stays runtime-independent of a hardcoded path).
|
|
95
|
+
*/
|
|
96
|
+
export function readFixtureManifestV2(fixtureRoot) {
|
|
97
|
+
const manifestRaw = readFileSync(join(fixtureRoot, "manifest.json"), "utf8");
|
|
98
|
+
const manifest = parseManifest(manifestRaw);
|
|
99
|
+
const byId = new Map();
|
|
100
|
+
const entries = [];
|
|
101
|
+
for (const fx of manifest.fixtures) {
|
|
102
|
+
const expectedDigest = String(fx.sha256 ?? "");
|
|
103
|
+
const domain = fx.domain ?? domainOf(fx.path);
|
|
104
|
+
const inputDigest = canonicalFileDigest(join(fixtureRoot, fx.path));
|
|
105
|
+
const entry = {
|
|
106
|
+
id: fx.id,
|
|
107
|
+
domain,
|
|
108
|
+
path: fx.path,
|
|
109
|
+
algorithm: fx.algorithm,
|
|
110
|
+
expected: fx.expected,
|
|
111
|
+
inputDigest,
|
|
112
|
+
expectedDigest,
|
|
113
|
+
expectedOutputDigest: typeof fx.outputDigest === "string" && fx.outputDigest.length > 0
|
|
114
|
+
? fx.outputDigest
|
|
115
|
+
: undefined,
|
|
116
|
+
failureCode: fx.expected === "ok" ? undefined : fx.expected,
|
|
117
|
+
algorithmTuple: String(fx.algorithm).split(";").filter(Boolean).map((s) => s.trim()),
|
|
118
|
+
};
|
|
119
|
+
entries.push(entry);
|
|
120
|
+
byId.set(fx.id, entry);
|
|
121
|
+
}
|
|
122
|
+
return {
|
|
123
|
+
version: manifest.version,
|
|
124
|
+
owner: manifest.owner.split(",").map((s) => s.trim()).filter(Boolean),
|
|
125
|
+
fixtureEntries: entries,
|
|
126
|
+
byId,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/** The v2 domain a fixture path belongs to (first path segment). */
|
|
130
|
+
export function domainOf(path) {
|
|
131
|
+
return path.split("/")[0] ?? "";
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* SHA-256 over the canonical JSON bytes of a fixture file. A missing file
|
|
135
|
+
* returns "" so `validateCanonicalV2` can report CONF_MISSING_FIXTURE instead of
|
|
136
|
+
* throwing ENOENT part-way through manifest normalization.
|
|
137
|
+
*/
|
|
138
|
+
function canonicalFileDigest(absPath) {
|
|
139
|
+
let raw;
|
|
140
|
+
try {
|
|
141
|
+
raw = readFileSync(absPath, "utf8");
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
return "";
|
|
145
|
+
}
|
|
146
|
+
let parsed;
|
|
147
|
+
try {
|
|
148
|
+
parsed = JSON.parse(raw);
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
parsed = raw; // non-JSON (e.g. seeds as bytes) — digest raw
|
|
152
|
+
}
|
|
153
|
+
return sha256Hex(canonicalJson(parsed));
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Validate the on-disk v2 corpus against the manifest. Rejects an extra,
|
|
157
|
+
* missing, or digest-drifted fixture file with the frozen codes. Also verifies
|
|
158
|
+
* the manifest itself is canonical (sorted keys, shortest numbers) — a
|
|
159
|
+
* noncanonical manifest is reported as NONCANONICAL.
|
|
160
|
+
*/
|
|
161
|
+
export function validateCanonicalV2(fixtureRoot) {
|
|
162
|
+
const codes = [];
|
|
163
|
+
const issues = [];
|
|
164
|
+
const manifestRaw = readFileSync(join(fixtureRoot, "manifest.json"), "utf8");
|
|
165
|
+
try {
|
|
166
|
+
const canonicalBytes = canonicalJson(JSON.parse(manifestRaw));
|
|
167
|
+
if (Buffer.from(manifestRaw, "utf8").toString("hex") !== canonicalBytes.toString("hex")) {
|
|
168
|
+
codes.push(CONF_FAIL.NONCANONICAL);
|
|
169
|
+
issues.push("manifest.json is not canonical");
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
codes.push(CONF_FAIL.NONCANONICAL);
|
|
174
|
+
issues.push("manifest.json is not valid JSON");
|
|
175
|
+
}
|
|
176
|
+
const manifest = readFixtureManifestV2(fixtureRoot);
|
|
177
|
+
const listedPaths = new Set(manifest.fixtureEntries.map((e) => e.path));
|
|
178
|
+
// Extra-file detection: every file under the root (except manifest) listed.
|
|
179
|
+
const onDisk = walk(fixtureRoot, fixtureRoot).filter((p) => p !== "manifest.json");
|
|
180
|
+
for (const p of onDisk) {
|
|
181
|
+
if (!listedPaths.has(p)) {
|
|
182
|
+
codes.push(CONF_FAIL.EXTRA_FIXTURE);
|
|
183
|
+
issues.push(`extra fixture: ${p}`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
// Missing + digest-drift for every listed fixture.
|
|
187
|
+
for (const entry of manifest.fixtureEntries) {
|
|
188
|
+
const abs = join(fixtureRoot, entry.path);
|
|
189
|
+
if (!exists(abs)) {
|
|
190
|
+
codes.push(CONF_FAIL.MISSING_FIXTURE);
|
|
191
|
+
issues.push(`missing fixture: ${entry.path}`);
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
const raw = readFileSync(abs, "utf8");
|
|
195
|
+
let canonicalHex;
|
|
196
|
+
try {
|
|
197
|
+
canonicalHex = sha256Hex(canonicalJson(JSON.parse(raw)));
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
canonicalHex = sha256Hex(Buffer.from(raw, "utf8"));
|
|
201
|
+
}
|
|
202
|
+
if (canonicalHex !== entry.expectedDigest) {
|
|
203
|
+
codes.push(CONF_FAIL.DIGEST_DRIFT);
|
|
204
|
+
issues.push(`digest drift for ${entry.path}: expected ${entry.expectedDigest} got ${canonicalHex}`);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
// The canonical-key ordering invariant: a canonical valid manifest is a single
|
|
208
|
+
// reproducible digest. (Extra-file presence also breaks purity but is the
|
|
209
|
+
// extra/missing covers it.) Noncanonical keys are caught above.
|
|
210
|
+
if (codes.length === 0) {
|
|
211
|
+
return { ok: true, entryCount: manifest.fixtureEntries.length };
|
|
212
|
+
}
|
|
213
|
+
return { ok: false, codes: dedupeCodes(codes), issues };
|
|
214
|
+
}
|
|
215
|
+
/** True if the manifest+corpus is canonical (converges to one digest). */
|
|
216
|
+
export function canonicalManifestsConverge(fixtureRoot) {
|
|
217
|
+
const v = validateCanonicalV2(fixtureRoot);
|
|
218
|
+
return v.ok;
|
|
219
|
+
}
|
|
220
|
+
function exists(p) {
|
|
221
|
+
try {
|
|
222
|
+
statSync(p);
|
|
223
|
+
return true;
|
|
224
|
+
}
|
|
225
|
+
catch {
|
|
226
|
+
return false;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
function dedupeCodes(codes) {
|
|
230
|
+
const out = [];
|
|
231
|
+
for (const c of codes)
|
|
232
|
+
if (!out.includes(c))
|
|
233
|
+
out.push(c);
|
|
234
|
+
return out;
|
|
235
|
+
}
|