pi-mega-compact 0.19.2 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/dist/config/vector-cortex.js +24 -0
  2. package/dist/config.js +1 -1
  3. package/dist/extensions/dashboard-server/routes-cache.test/_helpers.js +2 -0
  4. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +2 -0
  5. package/dist/extensions/dashboard-server/routes-vector-cortex-helpers.js +100 -0
  6. package/dist/extensions/dashboard-server/routes-vector-cortex-query.js +46 -0
  7. package/dist/extensions/dashboard-server/routes-vector-cortex-shards.js +48 -0
  8. package/dist/extensions/dashboard-server/routes-vector-cortex.js +2 -0
  9. package/dist/extensions/dashboard-server/routes.js +1 -1
  10. package/dist/extensions/dashboard-server/server.js +41 -16
  11. package/dist/extensions/dashboard-server/tailscale.js +32 -0
  12. package/dist/src/config/vector-cortex.js +24 -0
  13. package/dist/src/config.js +1 -1
  14. package/dist/src/tieredRouter.js +8 -0
  15. package/dist/src/vector-cortex/migrations/router-generation-v2.js +252 -0
  16. package/dist/src/vector-cortex/shards/exact.js +134 -0
  17. package/dist/src/vector-cortex/shards/manifest.js +237 -0
  18. package/dist/src/vector-cortex/shards/semantic.js +137 -0
  19. package/dist/src/vector-cortex/shards/types.js +51 -0
  20. package/dist/src/vector-cortex/topology/query.js +342 -0
  21. package/dist/vector-cortex/migrations/router-generation-v2.js +252 -0
  22. package/dist/vector-cortex/shards/exact.js +134 -0
  23. package/dist/vector-cortex/shards/manifest.js +237 -0
  24. package/dist/vector-cortex/shards/semantic.js +137 -0
  25. package/dist/vector-cortex/shards/types.js +51 -0
  26. package/dist/vector-cortex/topology/query.js +342 -0
  27. package/extensions/dashboard-client/dist/assets/{AreaChart-CSaJZXru.js → AreaChart-n4z5w9Ox.js} +2 -2
  28. package/extensions/dashboard-client/dist/assets/{AreaChart-CSaJZXru.js.map → AreaChart-n4z5w9Ox.js.map} +1 -1
  29. package/extensions/dashboard-client/dist/assets/{BarChart-CQouNj1g.js → BarChart-BLGjWWBr.js} +2 -2
  30. package/extensions/dashboard-client/dist/assets/{BarChart-CQouNj1g.js.map → BarChart-BLGjWWBr.js.map} +1 -1
  31. package/extensions/dashboard-client/dist/assets/{CacheTab-m_eiCYoL.js → CacheTab-CAnrKKVK.js} +2 -2
  32. package/extensions/dashboard-client/dist/assets/{CacheTab-m_eiCYoL.js.map → CacheTab-CAnrKKVK.js.map} +1 -1
  33. package/extensions/dashboard-client/dist/assets/{EventsTab-DRyOwjYN.js → EventsTab-D-GL7Ngq.js} +2 -2
  34. package/extensions/dashboard-client/dist/assets/{EventsTab-DRyOwjYN.js.map → EventsTab-D-GL7Ngq.js.map} +1 -1
  35. package/extensions/dashboard-client/dist/assets/{HealthTab-Dj9k40TJ.js → HealthTab-C4x5qQ8s.js} +2 -2
  36. package/extensions/dashboard-client/dist/assets/{HealthTab-Dj9k40TJ.js.map → HealthTab-C4x5qQ8s.js.map} +1 -1
  37. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-Bk9mhYfD.js → MaintenanceTab-BSPs0eq5.js} +2 -2
  38. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-Bk9mhYfD.js.map → MaintenanceTab-BSPs0eq5.js.map} +1 -1
  39. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-C7ve49VG.js → MemoryMapTab-BWEUtTSu.js} +2 -2
  40. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-C7ve49VG.js.map → MemoryMapTab-BWEUtTSu.js.map} +1 -1
  41. package/extensions/dashboard-client/dist/assets/{MetricsTab-DzjhqZA9.js → MetricsTab-Dhm1c3Cx.js} +2 -2
  42. package/extensions/dashboard-client/dist/assets/{MetricsTab-DzjhqZA9.js.map → MetricsTab-Dhm1c3Cx.js.map} +1 -1
  43. package/extensions/dashboard-client/dist/assets/{OverviewTab-CFQdfIm_.js → OverviewTab-Cm6cySiQ.js} +2 -2
  44. package/extensions/dashboard-client/dist/assets/{OverviewTab-CFQdfIm_.js.map → OverviewTab-Cm6cySiQ.js.map} +1 -1
  45. package/extensions/dashboard-client/dist/assets/{ReposTab-DyjwEFUr.js → ReposTab-DxLN1N4K.js} +2 -2
  46. package/extensions/dashboard-client/dist/assets/{ReposTab-DyjwEFUr.js.map → ReposTab-DxLN1N4K.js.map} +1 -1
  47. package/extensions/dashboard-client/dist/assets/{SessionsTab-njLIy7ab.js → SessionsTab-CZuI4ISW.js} +2 -2
  48. package/extensions/dashboard-client/dist/assets/{SessionsTab-njLIy7ab.js.map → SessionsTab-CZuI4ISW.js.map} +1 -1
  49. package/extensions/dashboard-client/dist/assets/{SetupTab-UHIs8-Lh.js → SetupTab-C_0VOomm.js} +2 -2
  50. package/extensions/dashboard-client/dist/assets/{SetupTab-UHIs8-Lh.js.map → SetupTab-C_0VOomm.js.map} +1 -1
  51. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-inWHjCxg.js → TimeSavedCard-BhbaRLmi.js} +2 -2
  52. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-inWHjCxg.js.map → TimeSavedCard-BhbaRLmi.js.map} +1 -1
  53. package/extensions/dashboard-client/dist/assets/{TurnsTab-CXJYugBu.js → TurnsTab-CXJmbjje.js} +2 -2
  54. package/extensions/dashboard-client/dist/assets/{TurnsTab-CXJYugBu.js.map → TurnsTab-CXJmbjje.js.map} +1 -1
  55. package/extensions/dashboard-client/dist/assets/VectorCortexTab-BGe9hu3r.js +2 -0
  56. package/extensions/dashboard-client/dist/assets/VectorCortexTab-BGe9hu3r.js.map +1 -0
  57. package/extensions/dashboard-client/dist/assets/{WikiTab-BJ4A2oi9.js → WikiTab-C5uvMCzp.js} +2 -2
  58. package/extensions/dashboard-client/dist/assets/{WikiTab-BJ4A2oi9.js.map → WikiTab-C5uvMCzp.js.map} +1 -1
  59. package/extensions/dashboard-client/dist/assets/{button-BWl-moyn.js → button-Cl_fxJcg.js} +2 -2
  60. package/extensions/dashboard-client/dist/assets/{button-BWl-moyn.js.map → button-Cl_fxJcg.js.map} +1 -1
  61. package/extensions/dashboard-client/dist/assets/{card-C58RVPOk.js → card-BH8Le5YM.js} +2 -2
  62. package/extensions/dashboard-client/dist/assets/{card-C58RVPOk.js.map → card-BH8Le5YM.js.map} +1 -1
  63. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-vgeOQU3L.js → generateCategoricalChart-sZ6ZNtBi.js} +2 -2
  64. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-vgeOQU3L.js.map → generateCategoricalChart-sZ6ZNtBi.js.map} +1 -1
  65. package/extensions/dashboard-client/dist/assets/{index-lVrWQjpu.js → index-CX9zv8cJ.js} +3 -3
  66. package/extensions/dashboard-client/dist/assets/{index-lVrWQjpu.js.map → index-CX9zv8cJ.js.map} +1 -1
  67. package/extensions/dashboard-client/dist/assets/{switch-CSeplFQT.js → switch-CXJhcjhM.js} +2 -2
  68. package/extensions/dashboard-client/dist/assets/{switch-CSeplFQT.js.map → switch-CXJhcjhM.js.map} +1 -1
  69. package/extensions/dashboard-client/dist/assets/{toggle-CQRxgVRi.js → toggle-CoLtIc6y.js} +2 -2
  70. package/extensions/dashboard-client/dist/assets/{toggle-CQRxgVRi.js.map → toggle-CoLtIc6y.js.map} +1 -1
  71. package/extensions/dashboard-client/dist/assets/{useSSE-BUz6IOI-.js → useSSE-fSXho606.js} +2 -2
  72. package/extensions/dashboard-client/dist/assets/{useSSE-BUz6IOI-.js.map → useSSE-fSXho606.js.map} +1 -1
  73. package/extensions/dashboard-client/dist/index.html +1 -1
  74. package/extensions/dashboard-client/src/api/vector-cortex.ts +18 -0
  75. package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +62 -0
  76. package/extensions/dashboard-client/src/types/vector-cortex.ts +25 -0
  77. package/extensions/dashboard-server/api-contracts/vector-cortex.ts +41 -0
  78. package/extensions/dashboard-server/routes-cache.test/_helpers.ts +2 -0
  79. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +12 -0
  80. package/extensions/dashboard-server/routes-vector-cortex-helpers.ts +110 -0
  81. package/extensions/dashboard-server/routes-vector-cortex-query.ts +54 -0
  82. package/extensions/dashboard-server/routes-vector-cortex-shards.ts +57 -0
  83. package/extensions/dashboard-server/routes-vector-cortex.ts +2 -0
  84. package/extensions/dashboard-server/routes.ts +2 -0
  85. package/extensions/dashboard-server/server.ts +46 -17
  86. package/extensions/dashboard-server/tailscale.ts +32 -0
  87. package/package.json +1 -1
  88. package/src/config/vector-cortex.ts +26 -0
  89. package/src/config.ts +2 -0
  90. package/src/tieredRouter.ts +9 -0
  91. package/src/vector-cortex/migrations/router-generation-v2.ts +317 -0
  92. package/src/vector-cortex/shards/exact.ts +141 -0
  93. package/src/vector-cortex/shards/manifest.ts +276 -0
  94. package/src/vector-cortex/shards/semantic.ts +156 -0
  95. package/src/vector-cortex/shards/types.ts +212 -0
  96. package/src/vector-cortex/topology/query.ts +485 -0
  97. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DkS0Azx9.js +0 -2
  98. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DkS0Azx9.js.map +0 -1
@@ -0,0 +1,252 @@
1
+ /**
2
+ * vector-cortex/migrations/router-generation-v2.ts — M6 copy/validate/switch
3
+ * migration (VC3C).
4
+ *
5
+ * M6 versions the tiered router's query set: the predecessor routed on ad-hoc
6
+ * per-session keys (no generation identity); VC3C routes on the structured,
7
+ * length-delimited `RouterKeyV2`. M6 copies the old per-session query set into
8
+ * v2 `RouterGenV2Row`s BESIDE the legacy state, validates, and only then
9
+ * atomically switches the active version pointer — the same copy/validate/
10
+ * switch + resume contract as M4.
11
+ *
12
+ * - copy: resumable per (session, old key) — an interrupted run resumes
13
+ * without duplicate rows or active-pointer drift.
14
+ * - validate: every old query key has exactly one v2 row, every row's key is
15
+ * canonical + re-hashes, and — the VC3C-specific invariant — the new query
16
+ * set is partition-preserving: a row derived from an old key of session S
17
+ * MUST carry session S. A row whose session differs is rejected as
18
+ * cross-session eviction (`M6_CROSS_SESSION_EVICTION`): one session may
19
+ * never lose or steal another session's query identity.
20
+ * - switch: atomically flips the ACTIVE VERSION pointer from 1 (legacy) to 2
21
+ * (router-generation-v2) only after a verified copy; interruption keeps the
22
+ * old active pointer.
23
+ *
24
+ * Pure logic over an injected M6Host (deterministic + testable; no console).
25
+ * PREVENT-002/011/PI-004 honored.
26
+ */
27
+ import { createHash } from "node:crypto";
28
+ import { encodeRouterKeyV2, decodeRouterKeyV2, } from "../topology/query.js";
29
+ /** The active-version value the v2 router generation pointer is switched to. */
30
+ export const ROUTER_GEN_V2_VERSION = 2;
31
+ /** The legacy (predecessor) active-version value before any verified switch. */
32
+ export const ROUTER_GEN_LEGACY_VERSION = 1;
33
+ /** M6 failure codes. */
34
+ export const M6_FAIL = {
35
+ VERSION_MISMATCH: "M6_VERSION_MISMATCH",
36
+ COPY_PARTIAL: "M6_COPY_PARTIAL",
37
+ COUNT_MISMATCH: "M6_COUNT_MISMATCH",
38
+ DIGEST_MISMATCH: "M6_DIGEST_MISMATCH",
39
+ BAD_OLD_KEY: "M6_BAD_OLD_KEY",
40
+ CROSS_SESSION_EVICTION: "M6_CROSS_SESSION_EVICTION",
41
+ SWITCH_PRECONDITION: "M6_SWITCH_PRECONDITION",
42
+ };
43
+ /** Derive one v2 row from an old key (authoritative bytes). */
44
+ export function deriveRouterGenRow(host, session, oldKey) {
45
+ const parsed = host.parseOldKey(session, oldKey);
46
+ if (!parsed.ok)
47
+ return { ok: false, session, oldKey };
48
+ const key = parsed.key;
49
+ // Cross-session guard at COPY time: the structured key's session must equal
50
+ // the slot session. A key that claims a different session is never copied.
51
+ if (key.session !== session)
52
+ return { ok: false, session, oldKey };
53
+ const encoded = encodeRouterKeyV2(key);
54
+ return {
55
+ ok: true,
56
+ row: {
57
+ session,
58
+ generation: key.generation,
59
+ sourceStart: key.sourceStart,
60
+ sourceEnd: key.sourceEnd,
61
+ algorithm: key.algorithm,
62
+ key: encoded,
63
+ digest: sha256Hex(Buffer.from(encoded.slice(4), "hex")),
64
+ },
65
+ };
66
+ }
67
+ function sha256Hex(bytes) {
68
+ return createHash("sha256").update(bytes).digest("hex");
69
+ }
70
+ /**
71
+ * Copy the full old query set into v2 rows (resumable): rows already persisted
72
+ * with a matching canonical key + digest are left untouched (no duplicates);
73
+ * absent or corrupt (wrong session / wrong key / stale digest / bad old key)
74
+ * slots are (re)written from the authoritative derivation. Returns the delta
75
+ * written this call. The active pointer stays at the legacy version until a
76
+ * verified switch.
77
+ */
78
+ export function m6Copy(host) {
79
+ const existing = new Map(host.existingV2().map((r) => [`${r.session}::${r.key}`, r]));
80
+ const wanted = [];
81
+ for (const session of host.sessions()) {
82
+ for (const oldKey of host.oldKeysOf(session)) {
83
+ const derived = deriveRouterGenRow(host, session, oldKey);
84
+ if (!derived.ok)
85
+ continue; // bad old key surfaced by verify, not lost silently
86
+ const stored = existing.get(`${session}::${derived.row.key}`);
87
+ if (stored !== undefined && rowMatches(stored, derived.row))
88
+ continue; // healthy
89
+ wanted.push(derived.row);
90
+ }
91
+ }
92
+ if (wanted.length > 0)
93
+ host.putV2(wanted);
94
+ return { written: wanted, all: [...host.existingV2()] };
95
+ }
96
+ /** True when a persisted row already matches the authoritative derivation. */
97
+ function rowMatches(stored, fresh) {
98
+ return (stored.session === fresh.session &&
99
+ stored.generation === fresh.generation &&
100
+ stored.sourceStart === fresh.sourceStart &&
101
+ stored.sourceEnd === fresh.sourceEnd &&
102
+ stored.algorithm === fresh.algorithm &&
103
+ stored.key === fresh.key &&
104
+ stored.digest === fresh.digest);
105
+ }
106
+ /**
107
+ * Verify the migrated query set: every old query key across every session has a
108
+ * v2 row (exact-once), every row's key is canonical + re-hashes, and the new set
109
+ * is partition-preserving — every row's session is exactly the session of the
110
+ * old key it was derived from (rejects cross-session eviction). Never mutates.
111
+ *
112
+ * Comparison of old/new query sets is exact-by-identity (structured key bytes),
113
+ * never by string prefix.
114
+ */
115
+ export function m6Verify(host) {
116
+ const codes = [];
117
+ const stored = host.existingV2();
118
+ // Expected slot inventory: (session, canonical key) for every derivable old key.
119
+ const expectedSlots = [];
120
+ const badOldKeys = [];
121
+ for (const session of host.sessions()) {
122
+ for (const oldKey of host.oldKeysOf(session)) {
123
+ const parsed = host.parseOldKey(session, oldKey);
124
+ if (!parsed.ok) {
125
+ badOldKeys.push({ session, oldKey });
126
+ continue;
127
+ }
128
+ if (parsed.key.session !== session) {
129
+ // A structured key that claims a different session is a cross-session
130
+ // eviction attempt — never migrated, never evicted.
131
+ codes.push(M6_FAIL.CROSS_SESSION_EVICTION);
132
+ continue;
133
+ }
134
+ expectedSlots.push({ session, oldKey });
135
+ }
136
+ }
137
+ if (badOldKeys.length > 0)
138
+ codes.push(M6_FAIL.BAD_OLD_KEY);
139
+ // Map stored rows by their (session,key) identity.
140
+ const byIdentity = new Map();
141
+ const duplicateCheck = new Map();
142
+ for (const r of stored) {
143
+ const ident = `${r.session}::${r.key}`;
144
+ byIdentity.set(ident, r);
145
+ duplicateCheck.set(ident, (duplicateCheck.get(ident) ?? 0) + 1);
146
+ }
147
+ // Each expected slot must be present exactly once, with a healthy row.
148
+ for (const slot of expectedSlots) {
149
+ const parsed = host.parseOldKey(slot.session, slot.oldKey);
150
+ if (!parsed.ok)
151
+ continue;
152
+ const expectedKey = encodeRouterKeyV2(parsed.key);
153
+ const ident = `${slot.session}::${expectedKey}`;
154
+ const count = duplicateCheck.get(ident) ?? 0;
155
+ if (count === 0)
156
+ codes.push(M6_FAIL.COPY_PARTIAL);
157
+ if (count > 1)
158
+ codes.push(M6_FAIL.COUNT_MISMATCH);
159
+ }
160
+ // Row health: digest re-hashes + session partition-preserving.
161
+ for (const r of stored) {
162
+ if (!r.key.startsWith("rk2:"))
163
+ codes.push(M6_FAIL.DIGEST_MISMATCH);
164
+ const expectedDigest = sha256Hex(Buffer.from(r.key.slice(4), "hex"));
165
+ if (r.digest !== expectedDigest)
166
+ codes.push(M6_FAIL.DIGEST_MISMATCH);
167
+ // The row's own structured key must decode and agree with its declared fields.
168
+ const dec = decodeRow(r);
169
+ if (!dec)
170
+ codes.push(M6_FAIL.DIGEST_MISMATCH);
171
+ }
172
+ // No stored row may be an orphan (no corresponding old key anywhere).
173
+ const expectedIdentities = new Set();
174
+ for (const slot of expectedSlots) {
175
+ const parsed = host.parseOldKey(slot.session, slot.oldKey);
176
+ if (!parsed.ok)
177
+ continue;
178
+ expectedIdentities.add(`${slot.session}::${encodeRouterKeyV2(parsed.key)}`);
179
+ }
180
+ for (const ident of byIdentity.keys()) {
181
+ if (!expectedIdentities.has(ident))
182
+ codes.push(M6_FAIL.COUNT_MISMATCH);
183
+ }
184
+ const ok = codes.length === 0;
185
+ return { ok, codes: dedupe(codes) };
186
+ }
187
+ /**
188
+ * Self-consistent decode of a stored row: its canonical key decodes to a
189
+ * RouterKeyV2 whose five fields equal the row's declared fields. A row whose key
190
+ * doesn't round-trip against its own declared session/generation/range/algorithm
191
+ * is corrupt (cannot be trusted as the query set's identity).
192
+ */
193
+ function decodeRow(r) {
194
+ if (!r.key.startsWith("rk2:"))
195
+ return false;
196
+ const hex = r.key.slice(4);
197
+ if (!/^[0-9a-f]+$/.test(hex) || hex.length % 2 !== 0)
198
+ return false;
199
+ const dec = decodeRouterKeyV2(r.key);
200
+ if (!dec.ok)
201
+ return false;
202
+ return (dec.key.session === r.session &&
203
+ dec.key.generation === r.generation &&
204
+ dec.key.sourceStart === r.sourceStart &&
205
+ dec.key.sourceEnd === r.sourceEnd &&
206
+ dec.key.algorithm === r.algorithm);
207
+ }
208
+ /**
209
+ * Switch the active version to 2. Only call after `m6Verify` reports ok; a
210
+ * crash/return before switch leaves the legacy pointer active (interruption
211
+ * keeps old authority, resumable idempotently).
212
+ */
213
+ export function m6Switch(host) {
214
+ if (host.activeVersion() !== ROUTER_GEN_LEGACY_VERSION)
215
+ return;
216
+ host.switchToV2();
217
+ }
218
+ /** Full M6 lifecycle: copy -> verify -> switch; returns the verify result. */
219
+ export function migrateRouterGenerationV2(host) {
220
+ m6Copy(host);
221
+ const v = m6Verify(host);
222
+ if (v.ok)
223
+ m6Switch(host);
224
+ return v;
225
+ }
226
+ /** Registered M6 conformance ID range (M6-001..012). */
227
+ export const M6_IDS = [
228
+ "M6-001",
229
+ "M6-002",
230
+ "M6-003",
231
+ "M6-004",
232
+ "M6-005",
233
+ "M6-006",
234
+ "M6-007",
235
+ "M6-008",
236
+ "M6-009",
237
+ "M6-010",
238
+ "M6-011",
239
+ "M6-012",
240
+ ];
241
+ /** Registered named M6 conformance IDs. */
242
+ export const M6_NAMED_IDS = [
243
+ "M6-KEY-001",
244
+ "M6-STALE-002",
245
+ ];
246
+ function dedupe(codes) {
247
+ const out = [];
248
+ for (const c of codes)
249
+ if (!out.includes(c))
250
+ out.push(c);
251
+ return out;
252
+ }
@@ -0,0 +1,134 @@
1
+ /**
2
+ * vector-cortex/shards/exact.ts — exact tier partition (VC4A).
3
+ *
4
+ * Carves the session's PROTECTED spans — every tool call/result pair, every
5
+ * anchor, every invalid UTF-8 event — into `ExactShardV1` payloads of the
6
+ * ORIGINAL bytes, verbatim and never normalized (SHD-UTF8-002: invalid bytes
7
+ * are exact-only and preserved unchanged — the codec's strict-fatal classifier
8
+ * means an invalid event has no lossy text to re-encode; we copy `originalBytes`
9
+ * directly). Each protected span is ATOMIC: a tool call/result pair is one
10
+ * `ProtectedSpan`, so no exact shard EVER splits a pair, even when the pair
11
+ * straddles the target-size boundary (SHD-PAIR-001).
12
+ *
13
+ * The partition walks the protected spans in canonical stream order (ascending
14
+ * first-seq), accrues each span's bytes, and closes an exact shard the moment
15
+ * adding the next atomic span would exceed `targetSize`. A single span over the
16
+ * budget still occupies exactly one shard — the pair case never splits.
17
+ *
18
+ * Triad: A = used with the semantic tier; B = the same exact spans preserved
19
+ * alongside extractive-derived semantic content; C = exact anchors/current
20
+ * transcript ONLY (this partition alone, with zero semantic shards).
21
+ *
22
+ * Pure/deterministic: hashes + array walking only, no storage, no console, no
23
+ * network (PREVENT-PI-004 / PREVENT-011).
24
+ */
25
+ import { createHash } from "node:crypto";
26
+ import { cumulativeOffsets } from "./semantic.js";
27
+ /** Deterministic exact-shard digest over the verbatim original bytes. */
28
+ function sealExactDigest(sessionId, bytes) {
29
+ const h = createHash("sha256");
30
+ h.update(`exact-shard-v1|${sessionId}|`);
31
+ h.update(bytes);
32
+ return `sha256:${h.digest("hex")}`;
33
+ }
34
+ /** Combine per-span cases into one shard case (invalid dominates, then tool-pair). */
35
+ function combineCase(span, acc) {
36
+ if (span.case === "invalid-utf8" || acc === "invalid-utf8")
37
+ return "invalid-utf8";
38
+ if (span.case === "tool-pair" || acc === "tool-pair")
39
+ return "tool-pair";
40
+ return "anchor";
41
+ }
42
+ /** Concatenate an ordered group of events' original bytes, verbatim. */
43
+ function concatBytes(events) {
44
+ let len = 0;
45
+ for (const e of events)
46
+ len += e.originalBytes.length;
47
+ const out = new Uint8Array(len);
48
+ let off = 0;
49
+ for (const e of events) {
50
+ out.set(e.originalBytes, off);
51
+ off += e.originalBytes.length;
52
+ }
53
+ return out;
54
+ }
55
+ /**
56
+ * Partition the exact tier (task 3). Returns `ok:true` with a list of
57
+ * `ExactShardV1` whose (sorted) ranges are contiguous, non-overlapping and
58
+ * jointly cover every protected span exactly once, each preserving the original
59
+ * bytes verbatim. Defensive failures mirror the semantic tier: a bad target size
60
+ * rejects with `SHD_INVALID_TARGET_SIZE`; a protected span referencing an event
61
+ * outside the session's stream, or a mixed-session span, rejects with
62
+ * `SHD_CROSS_SESSION`.
63
+ */
64
+ export function partitionExact(input) {
65
+ const { sessionId, events, protectedSpans, targetSize } = input;
66
+ if (!Number.isFinite(targetSize) || targetSize <= 0) {
67
+ return { ok: false, code: "SHD_INVALID_TARGET_SIZE" };
68
+ }
69
+ const stream = new Set(events.map((e) => e.seq.toString()));
70
+ for (const span of protectedSpans) {
71
+ if (span.events.some((e) => e.sessionId !== sessionId || !stream.has(e.seq.toString()))) {
72
+ return { ok: false, code: "SHD_CROSS_SESSION" };
73
+ }
74
+ }
75
+ const offsets = cumulativeOffsets(events);
76
+ if (protectedSpans.length === 0)
77
+ return { ok: true, shards: [] };
78
+ // Process protected spans in canonical stream order (ascending first-seq;
79
+ // equal first-seq preserves input order for determinism).
80
+ const ordered = [...protectedSpans].sort((a, b) => {
81
+ const as = a.events.length ? a.events[0].seq : 0n;
82
+ const bs = b.events.length ? b.events[0].seq : 0n;
83
+ return as < bs ? -1 : as > bs ? 1 : 0;
84
+ });
85
+ const shards = [];
86
+ let batch = [];
87
+ let batchBytes = 0;
88
+ let batchCase = "anchor";
89
+ const spanBytes = (s) => s.events.reduce((n, e) => n + e.originalBytes.length, 0);
90
+ const flush = () => {
91
+ if (batch.length === 0)
92
+ return;
93
+ const first = batch[0];
94
+ const last = batch[batch.length - 1];
95
+ const firstEv = first.events[0];
96
+ const lastEv = last.events[last.events.length - 1];
97
+ const start = offsets.of(firstEv.seq);
98
+ const end = offsets.of(lastEv.seq);
99
+ const byteStart = start ? start.byteStart : 0;
100
+ const byteEnd = end ? end.byteEnd : byteStart;
101
+ const bytes = concatBytes(batch.flatMap((s) => s.events));
102
+ shards.push({
103
+ schema: "exact-shard-v1",
104
+ sessionId,
105
+ range: {
106
+ sessionId,
107
+ seqStart: firstEv.seq,
108
+ seqEnd: lastEv.seq,
109
+ byteStart,
110
+ byteEnd,
111
+ },
112
+ kind: "exact",
113
+ originalBytes: bytes,
114
+ digest: sealExactDigest(sessionId, bytes),
115
+ byteCount: batchBytes,
116
+ case: batchCase,
117
+ });
118
+ batch = [];
119
+ batchBytes = 0;
120
+ batchCase = "anchor";
121
+ };
122
+ for (const span of ordered) {
123
+ const sz = spanBytes(span);
124
+ // Close at a complete-span boundary when adding this atomic span would
125
+ // exceed the budget and we already hold at least one complete span.
126
+ if (batch.length > 0 && batchBytes + sz > targetSize)
127
+ flush();
128
+ batch.push(span);
129
+ batchBytes += sz;
130
+ batchCase = combineCase(span, batchCase);
131
+ }
132
+ flush();
133
+ return { ok: true, shards };
134
+ }
@@ -0,0 +1,237 @@
1
+ /**
2
+ * vector-cortex/shards/manifest.ts — shard manifest assembly + validation (VC4A).
3
+ *
4
+ * Owns `assembleShardManifest` (the deterministic build) and
5
+ * `validateShardManifest` (task 4). The builder takes the already-partitioned
6
+ * semantic + exact tiers plus the protected spans and emits a `ShardManifestV1`
7
+ * whose ranges are disjoint and sorted by `(seqStart, byteStart)`. The validator
8
+ * enforces the two structural rules the sprint acceptance pins:
9
+ *
10
+ * - SHD_RANGE_OVERLAP — any two manifest shard ranges (semantic or exact)
11
+ * overlap in the session byte stream;
12
+ * - SHD_PROTECTED_GAP — the exact shards do not cover every protected byte
13
+ * exactly once (a protected span is unterminated by an exact shard, or an
14
+ * exact shard covers a byte no protected span claims).
15
+ *
16
+ * Emits `vector_cortex_shard_manifest_built` on a successful assembly and
17
+ * `vector_cortex_protected_span_rejected` when validation fails, via the same
18
+ * flag-gated reporter pattern VC3A/VC3B/VC3C use (`MEGACOMPACT_VC4A=0` emits
19
+ * nothing — byte-identical predecessor).
20
+ *
21
+ * Pure/deterministic: hashes + range math only, no storage, no console, no
22
+ * network (PREVENT-PI-004 / PREVENT-011).
23
+ */
24
+ import { createHash } from "node:crypto";
25
+ import { VC4A_ENABLED } from "../../config/vector-cortex.js";
26
+ /** A manifest shard's range, normalized for sorting / digest purposes. */
27
+ function rangeKey(r) {
28
+ return `${r.seqStart.toString()},${r.byteStart}`;
29
+ }
30
+ /**
31
+ * Merge byte intervals into disjoint, sorted intervals (half-open). Overlapping
32
+ * AND touching intervals merge into one — this produces the "union" used for
33
+ * coverage math, never used to hide an overlap (overlap is checked separately in
34
+ * `hasOverlap` before any merging).
35
+ */
36
+ function mergeIntervals(ranges) {
37
+ const sorted = [...ranges]
38
+ .map((r) => ({ start: r.byteStart, end: r.byteEnd }))
39
+ .sort((a, b) => (a.start !== b.start ? a.start - b.start : a.end - b.end));
40
+ const out = [];
41
+ for (const iv of sorted) {
42
+ const last = out[out.length - 1];
43
+ if (last !== undefined && iv.start <= last.end) {
44
+ out[out.length - 1] = { start: last.start, end: Math.max(last.end, iv.end) };
45
+ continue;
46
+ }
47
+ out.push(iv);
48
+ }
49
+ return out;
50
+ }
51
+ /** Difference: the bytes of `base` intervals NOT covered by `cover` intervals. */
52
+ function subtractCover(base, cover) {
53
+ const ordered = cover
54
+ .slice()
55
+ .sort((a, b) => (a.start !== b.start ? a.start - b.start : a.end - b.end));
56
+ const gaps = [];
57
+ for (const iv of base) {
58
+ let cursor = iv.start;
59
+ for (const c of ordered) {
60
+ if (c.end <= cursor)
61
+ continue;
62
+ if (c.start > cursor) {
63
+ gaps.push({ start: cursor, end: Math.min(c.start, iv.end) });
64
+ if (c.start >= iv.end)
65
+ break;
66
+ }
67
+ cursor = Math.max(cursor, c.end);
68
+ if (cursor >= iv.end)
69
+ break;
70
+ }
71
+ if (cursor < iv.end)
72
+ gaps.push({ start: cursor, end: iv.end });
73
+ }
74
+ return gaps;
75
+ }
76
+ /** True when any two ranges overlap (share at least one byte). */
77
+ function hasOverlap(ranges) {
78
+ const sorted = [...ranges].sort((a, b) => (a.byteStart !== b.byteStart ? a.byteStart - b.byteStart : a.byteEnd - b.byteEnd));
79
+ for (let i = 1; i < sorted.length; i++) {
80
+ const prev = sorted[i - 1];
81
+ const cur = sorted[i];
82
+ if (cur.byteStart < prev.byteEnd)
83
+ return true;
84
+ }
85
+ return false;
86
+ }
87
+ /**
88
+ * Validate a manifest (task 4). Disjointness is a PER-TIER property: semantic
89
+ * shard ranges must be pairwise disjoint among themselves, and exact shard
90
+ * ranges pairwise disjoint among themselves (else `SHD_RANGE_OVERLAP`). The two
91
+ * tiers are NOT disjoint from each other — exact shards are a verbatim SUBSET of
92
+ * the semantic stream's bytes (a protected tool-pair's bytes are part of the
93
+ * semantic content), so a semantic range and an exact range overlapping is
94
+ * expected and correct. The cross-tier property is complete protected-span
95
+ * coverage: the exact shards must tile every protected span byte exactly once,
96
+ * no gap and no extra coverage (else `SHD_PROTECTED_GAP`). A manifest missing
97
+ * protected spans is malformed (`SHD_PROTECTED_GAP`). `generationDigest` is NOT
98
+ * verified here (it is a deterministic projection; the caller verifies it
99
+ * against the rebuilt digest).
100
+ */
101
+ export function validateShardManifest(manifest) {
102
+ // Disjointness is enforced within each tier separately. An exact shard sitting
103
+ // inside a semantic shard is legitimate (exact ⊂ semantic).
104
+ if (hasOverlap(manifest.semantic.map((s) => s.range))) {
105
+ return { ok: false, code: "SHD_RANGE_OVERLAP" };
106
+ }
107
+ if (hasOverlap(manifest.exact.map((s) => s.range))) {
108
+ return { ok: false, code: "SHD_RANGE_OVERLAP" };
109
+ }
110
+ const protectedUnion = mergeIntervals(manifest.protectedSpans);
111
+ const exactUnion = mergeIntervals(manifest.exact.map((s) => s.range));
112
+ // Every protected byte must be covered by at least one exact shard (no gap),
113
+ // and no exact byte may fall outside the protected union (no extra coverage).
114
+ const gaps = subtractCover(protectedUnion, exactUnion);
115
+ const extras = subtractCover(exactUnion, protectedUnion);
116
+ if (gaps.length > 0 || extras.length > 0) {
117
+ return { ok: false, code: "SHD_PROTECTED_GAP" };
118
+ }
119
+ return { ok: true };
120
+ }
121
+ /**
122
+ * Deterministic manifest digest over a shard set — order-independent because it
123
+ * hashes the shards sorted by `(kind, seqStart, byteStart, digest)` (never input
124
+ * order). One SHA-256 over the canonical parts; the caller compares it against
125
+ * the stored `generationDigest`.
126
+ */
127
+ export function shardManifestDigest(sessionId, sourceHighWater, semantic, exact) {
128
+ const rows = [
129
+ ...semantic.map((s) => ["semantic", s.range, s.digest]),
130
+ ...exact.map((s) => ["exact", s.range, s.digest]),
131
+ ].sort((a, b) => {
132
+ if (a[0] !== b[0])
133
+ return a[0] < b[0] ? -1 : 1;
134
+ const rk = rangeKey(a[1]);
135
+ const rk2 = rangeKey(b[1]);
136
+ return rk !== rk2 ? (rk < rk2 ? -1 : 1) : a[2] < b[2] ? -1 : a[2] > b[2] ? 1 : 0;
137
+ });
138
+ const h = createHash("sha256");
139
+ h.update(`shard-manifest-v1|${sessionId}|${sourceHighWater.toString()}|`);
140
+ for (const [kind, r, digest] of rows) {
141
+ h.update(`${kind}|${r.seqStart.toString()}|${r.seqEnd.toString()}|${r.byteStart}|${r.byteEnd}|${digest}~`);
142
+ }
143
+ return `sha256:${h.digest("hex")}`;
144
+ }
145
+ /**
146
+ * Assemble a `ShardManifestV1` from already-partitioned tiers (deterministic).
147
+ * Shards are sorted by `(seqStart, byteStart)`; `generationDigest`, `byteTotal`
148
+ * and `shardCount` are computed from the sorted set. The caller typically runs
149
+ * this through `assembleAndValidate` so the structural rules are enforced too.
150
+ */
151
+ export function buildShardManifest(input) {
152
+ const cmp = (a, b) => {
153
+ if (a.range.seqStart !== b.range.seqStart) {
154
+ return a.range.seqStart < b.range.seqStart ? -1 : 1;
155
+ }
156
+ return a.range.byteStart !== b.range.byteStart
157
+ ? a.range.byteStart < b.range.byteStart ? -1 : 1
158
+ : 0;
159
+ };
160
+ const semantic = [...input.semantic].sort(cmp);
161
+ const exact = [...input.exact].sort(cmp);
162
+ const byteTotal = semantic.reduce((n, s) => n + s.byteCount, 0) +
163
+ exact.reduce((n, s) => n + s.byteCount, 0);
164
+ const generationDigest = shardManifestDigest(input.sessionId, input.sourceHighWater, semantic, exact);
165
+ return {
166
+ schema: "shard-manifest-v1",
167
+ sessionId: input.sessionId,
168
+ sourceHighWater: input.sourceHighWater,
169
+ semantic,
170
+ exact,
171
+ protectedSpans: [...input.protectedSpans],
172
+ byteTotal,
173
+ shardCount: semantic.length + exact.length,
174
+ generationDigest,
175
+ };
176
+ }
177
+ /**
178
+ * Assembly + validation seam (task 5). Builds the manifest, validates it, and
179
+ * emits `vector_cortex_shard_manifest_built` on success or
180
+ * `vector_cortex_protected_span_rejected` on failure (flag-gated). Non-fatal:
181
+ * a validation failure returns `{ok:false, code}` and NEVER throws into the host.
182
+ */
183
+ export function assembleAndValidate(input, emit) {
184
+ const manifest = buildShardManifest(input);
185
+ const validation = validateShardManifest(manifest);
186
+ const reporter = createShardReporter(emit);
187
+ if (!validation.ok) {
188
+ reporter.protectedSpanRejected({
189
+ sessionId: input.sessionId,
190
+ code: validation.code,
191
+ });
192
+ return { ok: false, code: validation.code };
193
+ }
194
+ reporter.manifestBuilt({
195
+ sessionId: input.sessionId,
196
+ sourceHighWater: input.sourceHighWater.toString(),
197
+ shardCount: manifest.shardCount,
198
+ byteTotal: manifest.byteTotal,
199
+ generationDigest: manifest.generationDigest,
200
+ });
201
+ return { ok: true, manifest };
202
+ }
203
+ /**
204
+ * Whether each tier's ranges are internally sorted by (seqStart, byteStart).
205
+ * The two tiers are NOT globally ordered against each other (exact shards are a
206
+ * subset of the semantic stream, so an exact range always sits inside some
207
+ * semantic range); the guarantee the builder provides is per-tier ordering.
208
+ */
209
+ export function manifestSorted(m) {
210
+ const key = (r) => `${r.seqStart.toString()},${r.byteStart}`;
211
+ const sorted = (rows) => rows.every((r, i) => i === 0 || key(rows[i - 1].range) <= key(r.range));
212
+ return sorted(m.semantic) && sorted(m.exact);
213
+ }
214
+ /** Build the flag-gated typed reporter (mirrors ledger/topology reporters). */
215
+ function createShardReporter(emit) {
216
+ const fire = (event, fields) => {
217
+ if (!VC4A_ENABLED())
218
+ return;
219
+ if (!emit)
220
+ return;
221
+ try {
222
+ emit(event, fields);
223
+ }
224
+ catch {
225
+ /* non-fatal observability — never break the agent loop */
226
+ }
227
+ };
228
+ return {
229
+ manifestBuilt(fields) {
230
+ fire("vector_cortex_shard_manifest_built", fields);
231
+ },
232
+ protectedSpanRejected(fields) {
233
+ fire("vector_cortex_protected_span_rejected", fields);
234
+ },
235
+ };
236
+ }
237
+ export { createShardReporter };