pi-mega-compact 0.19.3 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/dist/config/vector-cortex.js +24 -0
  2. package/dist/config.js +1 -1
  3. package/dist/extensions/dashboard-server/routes-cache.test/_helpers.js +2 -0
  4. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +2 -0
  5. package/dist/extensions/dashboard-server/routes-vector-cortex-helpers.js +100 -0
  6. package/dist/extensions/dashboard-server/routes-vector-cortex-residual.js +51 -0
  7. package/dist/extensions/dashboard-server/routes-vector-cortex-shards.js +48 -0
  8. package/dist/extensions/dashboard-server/routes-vector-cortex.js +2 -0
  9. package/dist/extensions/dashboard-server/routes.js +1 -1
  10. package/dist/extensions/dashboard-server/server.js +41 -16
  11. package/dist/extensions/dashboard-server/tailscale.js +32 -0
  12. package/dist/src/config/vector-cortex.js +24 -0
  13. package/dist/src/config.js +1 -1
  14. package/dist/src/vector-cortex/residual/codec.js +224 -0
  15. package/dist/src/vector-cortex/residual/dct.js +158 -0
  16. package/dist/src/vector-cortex/residual/fixture-payload.js +77 -0
  17. package/dist/src/vector-cortex/residual/gf256.js +182 -0
  18. package/dist/src/vector-cortex/residual/parity.js +245 -0
  19. package/dist/src/vector-cortex/residual/quantize.js +200 -0
  20. package/dist/src/vector-cortex/residual/stream.js +124 -0
  21. package/dist/src/vector-cortex/residual/types.js +55 -0
  22. package/dist/src/vector-cortex/shards/exact.js +134 -0
  23. package/dist/src/vector-cortex/shards/manifest.js +237 -0
  24. package/dist/src/vector-cortex/shards/semantic.js +137 -0
  25. package/dist/src/vector-cortex/shards/types.js +51 -0
  26. package/dist/vector-cortex/residual/codec.js +224 -0
  27. package/dist/vector-cortex/residual/dct.js +158 -0
  28. package/dist/vector-cortex/residual/fixture-payload.js +77 -0
  29. package/dist/vector-cortex/residual/gf256.js +182 -0
  30. package/dist/vector-cortex/residual/parity.js +245 -0
  31. package/dist/vector-cortex/residual/quantize.js +200 -0
  32. package/dist/vector-cortex/residual/stream.js +124 -0
  33. package/dist/vector-cortex/residual/types.js +55 -0
  34. package/dist/vector-cortex/shards/exact.js +134 -0
  35. package/dist/vector-cortex/shards/manifest.js +237 -0
  36. package/dist/vector-cortex/shards/semantic.js +137 -0
  37. package/dist/vector-cortex/shards/types.js +51 -0
  38. package/extensions/dashboard-client/dist/assets/{AreaChart-DqiFSnDx.js → AreaChart-n4z5w9Ox.js} +2 -2
  39. package/extensions/dashboard-client/dist/assets/{AreaChart-DqiFSnDx.js.map → AreaChart-n4z5w9Ox.js.map} +1 -1
  40. package/extensions/dashboard-client/dist/assets/{BarChart-Cqg-PQMK.js → BarChart-BLGjWWBr.js} +2 -2
  41. package/extensions/dashboard-client/dist/assets/{BarChart-Cqg-PQMK.js.map → BarChart-BLGjWWBr.js.map} +1 -1
  42. package/extensions/dashboard-client/dist/assets/{CacheTab-BPNs0xIF.js → CacheTab-CAnrKKVK.js} +2 -2
  43. package/extensions/dashboard-client/dist/assets/{CacheTab-BPNs0xIF.js.map → CacheTab-CAnrKKVK.js.map} +1 -1
  44. package/extensions/dashboard-client/dist/assets/{EventsTab-BQf7OGkf.js → EventsTab-D-GL7Ngq.js} +2 -2
  45. package/extensions/dashboard-client/dist/assets/{EventsTab-BQf7OGkf.js.map → EventsTab-D-GL7Ngq.js.map} +1 -1
  46. package/extensions/dashboard-client/dist/assets/{HealthTab-B14TaSPN.js → HealthTab-C4x5qQ8s.js} +2 -2
  47. package/extensions/dashboard-client/dist/assets/{HealthTab-B14TaSPN.js.map → HealthTab-C4x5qQ8s.js.map} +1 -1
  48. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-D8lkRoa1.js → MaintenanceTab-BSPs0eq5.js} +2 -2
  49. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-D8lkRoa1.js.map → MaintenanceTab-BSPs0eq5.js.map} +1 -1
  50. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-BOWtGfpx.js → MemoryMapTab-BWEUtTSu.js} +2 -2
  51. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-BOWtGfpx.js.map → MemoryMapTab-BWEUtTSu.js.map} +1 -1
  52. package/extensions/dashboard-client/dist/assets/{MetricsTab-7XI9cWaX.js → MetricsTab-Dhm1c3Cx.js} +2 -2
  53. package/extensions/dashboard-client/dist/assets/{MetricsTab-7XI9cWaX.js.map → MetricsTab-Dhm1c3Cx.js.map} +1 -1
  54. package/extensions/dashboard-client/dist/assets/{OverviewTab-C9KaGYvh.js → OverviewTab-Cm6cySiQ.js} +2 -2
  55. package/extensions/dashboard-client/dist/assets/{OverviewTab-C9KaGYvh.js.map → OverviewTab-Cm6cySiQ.js.map} +1 -1
  56. package/extensions/dashboard-client/dist/assets/{ReposTab-BwESNhox.js → ReposTab-DxLN1N4K.js} +2 -2
  57. package/extensions/dashboard-client/dist/assets/{ReposTab-BwESNhox.js.map → ReposTab-DxLN1N4K.js.map} +1 -1
  58. package/extensions/dashboard-client/dist/assets/{SessionsTab-DKH55Rda.js → SessionsTab-CZuI4ISW.js} +2 -2
  59. package/extensions/dashboard-client/dist/assets/{SessionsTab-DKH55Rda.js.map → SessionsTab-CZuI4ISW.js.map} +1 -1
  60. package/extensions/dashboard-client/dist/assets/{SetupTab-zDxf-nJd.js → SetupTab-C_0VOomm.js} +2 -2
  61. package/extensions/dashboard-client/dist/assets/{SetupTab-zDxf-nJd.js.map → SetupTab-C_0VOomm.js.map} +1 -1
  62. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-C9isog4L.js → TimeSavedCard-BhbaRLmi.js} +2 -2
  63. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-C9isog4L.js.map → TimeSavedCard-BhbaRLmi.js.map} +1 -1
  64. package/extensions/dashboard-client/dist/assets/{TurnsTab-ciI6ydvj.js → TurnsTab-CXJmbjje.js} +2 -2
  65. package/extensions/dashboard-client/dist/assets/{TurnsTab-ciI6ydvj.js.map → TurnsTab-CXJmbjje.js.map} +1 -1
  66. package/extensions/dashboard-client/dist/assets/VectorCortexTab-BGe9hu3r.js +2 -0
  67. package/extensions/dashboard-client/dist/assets/VectorCortexTab-BGe9hu3r.js.map +1 -0
  68. package/extensions/dashboard-client/dist/assets/{WikiTab-DcYHc734.js → WikiTab-C5uvMCzp.js} +2 -2
  69. package/extensions/dashboard-client/dist/assets/{WikiTab-DcYHc734.js.map → WikiTab-C5uvMCzp.js.map} +1 -1
  70. package/extensions/dashboard-client/dist/assets/{button-DF8lx4bt.js → button-Cl_fxJcg.js} +2 -2
  71. package/extensions/dashboard-client/dist/assets/{button-DF8lx4bt.js.map → button-Cl_fxJcg.js.map} +1 -1
  72. package/extensions/dashboard-client/dist/assets/{card-DIoDheSZ.js → card-BH8Le5YM.js} +2 -2
  73. package/extensions/dashboard-client/dist/assets/{card-DIoDheSZ.js.map → card-BH8Le5YM.js.map} +1 -1
  74. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-BFRzeYX3.js → generateCategoricalChart-sZ6ZNtBi.js} +2 -2
  75. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-BFRzeYX3.js.map → generateCategoricalChart-sZ6ZNtBi.js.map} +1 -1
  76. package/extensions/dashboard-client/dist/assets/{index-DSZON0Yi.js → index-CX9zv8cJ.js} +3 -3
  77. package/extensions/dashboard-client/dist/assets/{index-DSZON0Yi.js.map → index-CX9zv8cJ.js.map} +1 -1
  78. package/extensions/dashboard-client/dist/assets/{switch-l4qo22mU.js → switch-CXJhcjhM.js} +2 -2
  79. package/extensions/dashboard-client/dist/assets/{switch-l4qo22mU.js.map → switch-CXJhcjhM.js.map} +1 -1
  80. package/extensions/dashboard-client/dist/assets/{toggle-DPn6td1_.js → toggle-CoLtIc6y.js} +2 -2
  81. package/extensions/dashboard-client/dist/assets/{toggle-DPn6td1_.js.map → toggle-CoLtIc6y.js.map} +1 -1
  82. package/extensions/dashboard-client/dist/assets/{useSSE-BLmolqTJ.js → useSSE-fSXho606.js} +2 -2
  83. package/extensions/dashboard-client/dist/assets/{useSSE-BLmolqTJ.js.map → useSSE-fSXho606.js.map} +1 -1
  84. package/extensions/dashboard-client/dist/index.html +1 -1
  85. package/extensions/dashboard-client/src/api/vector-cortex.ts +9 -0
  86. package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +35 -0
  87. package/extensions/dashboard-client/src/types/vector-cortex.ts +13 -0
  88. package/extensions/dashboard-server/api-contracts/vector-cortex.ts +53 -0
  89. package/extensions/dashboard-server/routes-cache.test/_helpers.ts +2 -0
  90. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +12 -0
  91. package/extensions/dashboard-server/routes-vector-cortex-helpers.ts +110 -0
  92. package/extensions/dashboard-server/routes-vector-cortex-residual.ts +60 -0
  93. package/extensions/dashboard-server/routes-vector-cortex-shards.ts +57 -0
  94. package/extensions/dashboard-server/routes-vector-cortex.ts +2 -0
  95. package/extensions/dashboard-server/routes.ts +2 -0
  96. package/extensions/dashboard-server/server.ts +46 -17
  97. package/extensions/dashboard-server/tailscale.ts +32 -0
  98. package/package.json +1 -1
  99. package/src/config/vector-cortex.ts +26 -0
  100. package/src/config.ts +2 -0
  101. package/src/vector-cortex/residual/codec.ts +292 -0
  102. package/src/vector-cortex/residual/dct.ts +166 -0
  103. package/src/vector-cortex/residual/fixture-payload.ts +90 -0
  104. package/src/vector-cortex/residual/gf256.ts +196 -0
  105. package/src/vector-cortex/residual/parity.ts +283 -0
  106. package/src/vector-cortex/residual/quantize.ts +230 -0
  107. package/src/vector-cortex/residual/stream.ts +135 -0
  108. package/src/vector-cortex/residual/types.ts +236 -0
  109. package/src/vector-cortex/shards/exact.ts +141 -0
  110. package/src/vector-cortex/shards/manifest.ts +276 -0
  111. package/src/vector-cortex/shards/semantic.ts +156 -0
  112. package/src/vector-cortex/shards/types.ts +212 -0
  113. package/extensions/dashboard-client/dist/assets/VectorCortexTab-qTZ7NWng.js +0 -2
  114. package/extensions/dashboard-client/dist/assets/VectorCortexTab-qTZ7NWng.js.map +0 -1
@@ -0,0 +1,134 @@
1
+ /**
2
+ * vector-cortex/shards/exact.ts — exact tier partition (VC4A).
3
+ *
4
+ * Carves the session's PROTECTED spans — every tool call/result pair, every
5
+ * anchor, every invalid UTF-8 event — into `ExactShardV1` payloads of the
6
+ * ORIGINAL bytes, verbatim and never normalized (SHD-UTF8-002: invalid bytes
7
+ * are exact-only and preserved unchanged — the codec's strict-fatal classifier
8
+ * means an invalid event has no lossy text to re-encode; we copy `originalBytes`
9
+ * directly). Each protected span is ATOMIC: a tool call/result pair is one
10
+ * `ProtectedSpan`, so no exact shard EVER splits a pair, even when the pair
11
+ * straddles the target-size boundary (SHD-PAIR-001).
12
+ *
13
+ * The partition walks the protected spans in canonical stream order (ascending
14
+ * first-seq), accrues each span's bytes, and closes an exact shard the moment
15
+ * adding the next atomic span would exceed `targetSize`. A single span over the
16
+ * budget still occupies exactly one shard — the pair case never splits.
17
+ *
18
+ * Triad: A = used with the semantic tier; B = the same exact spans preserved
19
+ * alongside extractive-derived semantic content; C = exact anchors/current
20
+ * transcript ONLY (this partition alone, with zero semantic shards).
21
+ *
22
+ * Pure/deterministic: hashes + array walking only, no storage, no console, no
23
+ * network (PREVENT-PI-004 / PREVENT-011).
24
+ */
25
+ import { createHash } from "node:crypto";
26
+ import { cumulativeOffsets } from "./semantic.js";
27
+ /** Deterministic exact-shard digest over the verbatim original bytes. */
28
+ function sealExactDigest(sessionId, bytes) {
29
+ const h = createHash("sha256");
30
+ h.update(`exact-shard-v1|${sessionId}|`);
31
+ h.update(bytes);
32
+ return `sha256:${h.digest("hex")}`;
33
+ }
34
+ /** Combine per-span cases into one shard case (invalid dominates, then tool-pair). */
35
+ function combineCase(span, acc) {
36
+ if (span.case === "invalid-utf8" || acc === "invalid-utf8")
37
+ return "invalid-utf8";
38
+ if (span.case === "tool-pair" || acc === "tool-pair")
39
+ return "tool-pair";
40
+ return "anchor";
41
+ }
42
+ /** Concatenate an ordered group of events' original bytes, verbatim. */
43
+ function concatBytes(events) {
44
+ let len = 0;
45
+ for (const e of events)
46
+ len += e.originalBytes.length;
47
+ const out = new Uint8Array(len);
48
+ let off = 0;
49
+ for (const e of events) {
50
+ out.set(e.originalBytes, off);
51
+ off += e.originalBytes.length;
52
+ }
53
+ return out;
54
+ }
55
+ /**
56
+ * Partition the exact tier (task 3). Returns `ok:true` with a list of
57
+ * `ExactShardV1` whose (sorted) ranges are contiguous, non-overlapping and
58
+ * jointly cover every protected span exactly once, each preserving the original
59
+ * bytes verbatim. Defensive failures mirror the semantic tier: a bad target size
60
+ * rejects with `SHD_INVALID_TARGET_SIZE`; a protected span referencing an event
61
+ * outside the session's stream, or a mixed-session span, rejects with
62
+ * `SHD_CROSS_SESSION`.
63
+ */
64
+ export function partitionExact(input) {
65
+ const { sessionId, events, protectedSpans, targetSize } = input;
66
+ if (!Number.isFinite(targetSize) || targetSize <= 0) {
67
+ return { ok: false, code: "SHD_INVALID_TARGET_SIZE" };
68
+ }
69
+ const stream = new Set(events.map((e) => e.seq.toString()));
70
+ for (const span of protectedSpans) {
71
+ if (span.events.some((e) => e.sessionId !== sessionId || !stream.has(e.seq.toString()))) {
72
+ return { ok: false, code: "SHD_CROSS_SESSION" };
73
+ }
74
+ }
75
+ const offsets = cumulativeOffsets(events);
76
+ if (protectedSpans.length === 0)
77
+ return { ok: true, shards: [] };
78
+ // Process protected spans in canonical stream order (ascending first-seq;
79
+ // equal first-seq preserves input order for determinism).
80
+ const ordered = [...protectedSpans].sort((a, b) => {
81
+ const as = a.events.length ? a.events[0].seq : 0n;
82
+ const bs = b.events.length ? b.events[0].seq : 0n;
83
+ return as < bs ? -1 : as > bs ? 1 : 0;
84
+ });
85
+ const shards = [];
86
+ let batch = [];
87
+ let batchBytes = 0;
88
+ let batchCase = "anchor";
89
+ const spanBytes = (s) => s.events.reduce((n, e) => n + e.originalBytes.length, 0);
90
+ const flush = () => {
91
+ if (batch.length === 0)
92
+ return;
93
+ const first = batch[0];
94
+ const last = batch[batch.length - 1];
95
+ const firstEv = first.events[0];
96
+ const lastEv = last.events[last.events.length - 1];
97
+ const start = offsets.of(firstEv.seq);
98
+ const end = offsets.of(lastEv.seq);
99
+ const byteStart = start ? start.byteStart : 0;
100
+ const byteEnd = end ? end.byteEnd : byteStart;
101
+ const bytes = concatBytes(batch.flatMap((s) => s.events));
102
+ shards.push({
103
+ schema: "exact-shard-v1",
104
+ sessionId,
105
+ range: {
106
+ sessionId,
107
+ seqStart: firstEv.seq,
108
+ seqEnd: lastEv.seq,
109
+ byteStart,
110
+ byteEnd,
111
+ },
112
+ kind: "exact",
113
+ originalBytes: bytes,
114
+ digest: sealExactDigest(sessionId, bytes),
115
+ byteCount: batchBytes,
116
+ case: batchCase,
117
+ });
118
+ batch = [];
119
+ batchBytes = 0;
120
+ batchCase = "anchor";
121
+ };
122
+ for (const span of ordered) {
123
+ const sz = spanBytes(span);
124
+ // Close at a complete-span boundary when adding this atomic span would
125
+ // exceed the budget and we already hold at least one complete span.
126
+ if (batch.length > 0 && batchBytes + sz > targetSize)
127
+ flush();
128
+ batch.push(span);
129
+ batchBytes += sz;
130
+ batchCase = combineCase(span, batchCase);
131
+ }
132
+ flush();
133
+ return { ok: true, shards };
134
+ }
@@ -0,0 +1,237 @@
1
+ /**
2
+ * vector-cortex/shards/manifest.ts — shard manifest assembly + validation (VC4A).
3
+ *
4
+ * Owns `assembleShardManifest` (the deterministic build) and
5
+ * `validateShardManifest` (task 4). The builder takes the already-partitioned
6
+ * semantic + exact tiers plus the protected spans and emits a `ShardManifestV1`
7
+ * whose ranges are disjoint and sorted by `(seqStart, byteStart)`. The validator
8
+ * enforces the two structural rules the sprint acceptance pins:
9
+ *
10
+ * - SHD_RANGE_OVERLAP — any two manifest shard ranges (semantic or exact)
11
+ * overlap in the session byte stream;
12
+ * - SHD_PROTECTED_GAP — the exact shards do not cover every protected byte
13
+ * exactly once (a protected span is unterminated by an exact shard, or an
14
+ * exact shard covers a byte no protected span claims).
15
+ *
16
+ * Emits `vector_cortex_shard_manifest_built` on a successful assembly and
17
+ * `vector_cortex_protected_span_rejected` when validation fails, via the same
18
+ * flag-gated reporter pattern VC3A/VC3B/VC3C use (`MEGACOMPACT_VC4A=0` emits
19
+ * nothing — byte-identical predecessor).
20
+ *
21
+ * Pure/deterministic: hashes + range math only, no storage, no console, no
22
+ * network (PREVENT-PI-004 / PREVENT-011).
23
+ */
24
+ import { createHash } from "node:crypto";
25
+ import { VC4A_ENABLED } from "../../config/vector-cortex.js";
26
+ /** A manifest shard's range, normalized for sorting / digest purposes. */
27
+ function rangeKey(r) {
28
+ return `${r.seqStart.toString()},${r.byteStart}`;
29
+ }
30
+ /**
31
+ * Merge byte intervals into disjoint, sorted intervals (half-open). Overlapping
32
+ * AND touching intervals merge into one — this produces the "union" used for
33
+ * coverage math, never used to hide an overlap (overlap is checked separately in
34
+ * `hasOverlap` before any merging).
35
+ */
36
+ function mergeIntervals(ranges) {
37
+ const sorted = [...ranges]
38
+ .map((r) => ({ start: r.byteStart, end: r.byteEnd }))
39
+ .sort((a, b) => (a.start !== b.start ? a.start - b.start : a.end - b.end));
40
+ const out = [];
41
+ for (const iv of sorted) {
42
+ const last = out[out.length - 1];
43
+ if (last !== undefined && iv.start <= last.end) {
44
+ out[out.length - 1] = { start: last.start, end: Math.max(last.end, iv.end) };
45
+ continue;
46
+ }
47
+ out.push(iv);
48
+ }
49
+ return out;
50
+ }
51
+ /** Difference: the bytes of `base` intervals NOT covered by `cover` intervals. */
52
+ function subtractCover(base, cover) {
53
+ const ordered = cover
54
+ .slice()
55
+ .sort((a, b) => (a.start !== b.start ? a.start - b.start : a.end - b.end));
56
+ const gaps = [];
57
+ for (const iv of base) {
58
+ let cursor = iv.start;
59
+ for (const c of ordered) {
60
+ if (c.end <= cursor)
61
+ continue;
62
+ if (c.start > cursor) {
63
+ gaps.push({ start: cursor, end: Math.min(c.start, iv.end) });
64
+ if (c.start >= iv.end)
65
+ break;
66
+ }
67
+ cursor = Math.max(cursor, c.end);
68
+ if (cursor >= iv.end)
69
+ break;
70
+ }
71
+ if (cursor < iv.end)
72
+ gaps.push({ start: cursor, end: iv.end });
73
+ }
74
+ return gaps;
75
+ }
76
+ /** True when any two ranges overlap (share at least one byte). */
77
+ function hasOverlap(ranges) {
78
+ const sorted = [...ranges].sort((a, b) => (a.byteStart !== b.byteStart ? a.byteStart - b.byteStart : a.byteEnd - b.byteEnd));
79
+ for (let i = 1; i < sorted.length; i++) {
80
+ const prev = sorted[i - 1];
81
+ const cur = sorted[i];
82
+ if (cur.byteStart < prev.byteEnd)
83
+ return true;
84
+ }
85
+ return false;
86
+ }
87
+ /**
88
+ * Validate a manifest (task 4). Disjointness is a PER-TIER property: semantic
89
+ * shard ranges must be pairwise disjoint among themselves, and exact shard
90
+ * ranges pairwise disjoint among themselves (else `SHD_RANGE_OVERLAP`). The two
91
+ * tiers are NOT disjoint from each other — exact shards are a verbatim SUBSET of
92
+ * the semantic stream's bytes (a protected tool-pair's bytes are part of the
93
+ * semantic content), so a semantic range and an exact range overlapping is
94
+ * expected and correct. The cross-tier property is complete protected-span
95
+ * coverage: the exact shards must tile every protected span byte exactly once,
96
+ * no gap and no extra coverage (else `SHD_PROTECTED_GAP`). A manifest missing
97
+ * protected spans is malformed (`SHD_PROTECTED_GAP`). `generationDigest` is NOT
98
+ * verified here (it is a deterministic projection; the caller verifies it
99
+ * against the rebuilt digest).
100
+ */
101
+ export function validateShardManifest(manifest) {
102
+ // Disjointness is enforced within each tier separately. An exact shard sitting
103
+ // inside a semantic shard is legitimate (exact ⊂ semantic).
104
+ if (hasOverlap(manifest.semantic.map((s) => s.range))) {
105
+ return { ok: false, code: "SHD_RANGE_OVERLAP" };
106
+ }
107
+ if (hasOverlap(manifest.exact.map((s) => s.range))) {
108
+ return { ok: false, code: "SHD_RANGE_OVERLAP" };
109
+ }
110
+ const protectedUnion = mergeIntervals(manifest.protectedSpans);
111
+ const exactUnion = mergeIntervals(manifest.exact.map((s) => s.range));
112
+ // Every protected byte must be covered by at least one exact shard (no gap),
113
+ // and no exact byte may fall outside the protected union (no extra coverage).
114
+ const gaps = subtractCover(protectedUnion, exactUnion);
115
+ const extras = subtractCover(exactUnion, protectedUnion);
116
+ if (gaps.length > 0 || extras.length > 0) {
117
+ return { ok: false, code: "SHD_PROTECTED_GAP" };
118
+ }
119
+ return { ok: true };
120
+ }
121
+ /**
122
+ * Deterministic manifest digest over a shard set — order-independent because it
123
+ * hashes the shards sorted by `(kind, seqStart, byteStart, digest)` (never input
124
+ * order). One SHA-256 over the canonical parts; the caller compares it against
125
+ * the stored `generationDigest`.
126
+ */
127
+ export function shardManifestDigest(sessionId, sourceHighWater, semantic, exact) {
128
+ const rows = [
129
+ ...semantic.map((s) => ["semantic", s.range, s.digest]),
130
+ ...exact.map((s) => ["exact", s.range, s.digest]),
131
+ ].sort((a, b) => {
132
+ if (a[0] !== b[0])
133
+ return a[0] < b[0] ? -1 : 1;
134
+ const rk = rangeKey(a[1]);
135
+ const rk2 = rangeKey(b[1]);
136
+ return rk !== rk2 ? (rk < rk2 ? -1 : 1) : a[2] < b[2] ? -1 : a[2] > b[2] ? 1 : 0;
137
+ });
138
+ const h = createHash("sha256");
139
+ h.update(`shard-manifest-v1|${sessionId}|${sourceHighWater.toString()}|`);
140
+ for (const [kind, r, digest] of rows) {
141
+ h.update(`${kind}|${r.seqStart.toString()}|${r.seqEnd.toString()}|${r.byteStart}|${r.byteEnd}|${digest}~`);
142
+ }
143
+ return `sha256:${h.digest("hex")}`;
144
+ }
145
+ /**
146
+ * Assemble a `ShardManifestV1` from already-partitioned tiers (deterministic).
147
+ * Shards are sorted by `(seqStart, byteStart)`; `generationDigest`, `byteTotal`
148
+ * and `shardCount` are computed from the sorted set. The caller typically runs
149
+ * this through `assembleAndValidate` so the structural rules are enforced too.
150
+ */
151
+ export function buildShardManifest(input) {
152
+ const cmp = (a, b) => {
153
+ if (a.range.seqStart !== b.range.seqStart) {
154
+ return a.range.seqStart < b.range.seqStart ? -1 : 1;
155
+ }
156
+ return a.range.byteStart !== b.range.byteStart
157
+ ? a.range.byteStart < b.range.byteStart ? -1 : 1
158
+ : 0;
159
+ };
160
+ const semantic = [...input.semantic].sort(cmp);
161
+ const exact = [...input.exact].sort(cmp);
162
+ const byteTotal = semantic.reduce((n, s) => n + s.byteCount, 0) +
163
+ exact.reduce((n, s) => n + s.byteCount, 0);
164
+ const generationDigest = shardManifestDigest(input.sessionId, input.sourceHighWater, semantic, exact);
165
+ return {
166
+ schema: "shard-manifest-v1",
167
+ sessionId: input.sessionId,
168
+ sourceHighWater: input.sourceHighWater,
169
+ semantic,
170
+ exact,
171
+ protectedSpans: [...input.protectedSpans],
172
+ byteTotal,
173
+ shardCount: semantic.length + exact.length,
174
+ generationDigest,
175
+ };
176
+ }
177
+ /**
178
+ * Assembly + validation seam (task 5). Builds the manifest, validates it, and
179
+ * emits `vector_cortex_shard_manifest_built` on success or
180
+ * `vector_cortex_protected_span_rejected` on failure (flag-gated). Non-fatal:
181
+ * a validation failure returns `{ok:false, code}` and NEVER throws into the host.
182
+ */
183
+ export function assembleAndValidate(input, emit) {
184
+ const manifest = buildShardManifest(input);
185
+ const validation = validateShardManifest(manifest);
186
+ const reporter = createShardReporter(emit);
187
+ if (!validation.ok) {
188
+ reporter.protectedSpanRejected({
189
+ sessionId: input.sessionId,
190
+ code: validation.code,
191
+ });
192
+ return { ok: false, code: validation.code };
193
+ }
194
+ reporter.manifestBuilt({
195
+ sessionId: input.sessionId,
196
+ sourceHighWater: input.sourceHighWater.toString(),
197
+ shardCount: manifest.shardCount,
198
+ byteTotal: manifest.byteTotal,
199
+ generationDigest: manifest.generationDigest,
200
+ });
201
+ return { ok: true, manifest };
202
+ }
203
+ /**
204
+ * Whether each tier's ranges are internally sorted by (seqStart, byteStart).
205
+ * The two tiers are NOT globally ordered against each other (exact shards are a
206
+ * subset of the semantic stream, so an exact range always sits inside some
207
+ * semantic range); the guarantee the builder provides is per-tier ordering.
208
+ */
209
+ export function manifestSorted(m) {
210
+ const key = (r) => `${r.seqStart.toString()},${r.byteStart}`;
211
+ const sorted = (rows) => rows.every((r, i) => i === 0 || key(rows[i - 1].range) <= key(r.range));
212
+ return sorted(m.semantic) && sorted(m.exact);
213
+ }
214
+ /** Build the flag-gated typed reporter (mirrors ledger/topology reporters). */
215
+ function createShardReporter(emit) {
216
+ const fire = (event, fields) => {
217
+ if (!VC4A_ENABLED())
218
+ return;
219
+ if (!emit)
220
+ return;
221
+ try {
222
+ emit(event, fields);
223
+ }
224
+ catch {
225
+ /* non-fatal observability — never break the agent loop */
226
+ }
227
+ };
228
+ return {
229
+ manifestBuilt(fields) {
230
+ fire("vector_cortex_shard_manifest_built", fields);
231
+ },
232
+ protectedSpanRejected(fields) {
233
+ fire("vector_cortex_protected_span_rejected", fields);
234
+ },
235
+ };
236
+ }
237
+ export { createShardReporter };
@@ -0,0 +1,137 @@
1
+ /**
2
+ * vector-cortex/shards/semantic.ts — semantic tier partition (VC4A, mode A/B).
3
+ *
4
+ * Partitions a session's canonical EventV2 stream into `SemanticShardV1`
5
+ * chunks, cutting ONLY at complete record boundaries. Every shard's `range`
6
+ * names the exact inclusive seq span and the half-open source byte window it
7
+ * covers in canonical stream order; a shard NEVER splits a record (or a tool
8
+ * call/result pair). Semantic shards carry DERIVED aggregate content (a
9
+ * deterministic digest + token estimate + byte/event counts) and never
10
+ * reproduce exact text — exact bytes are the exact tier's job.
11
+ *
12
+ * The partition is deterministic and size-bounded: it walks the sorted events,
13
+ * accrues each record's bytes, and closes a shard the moment adding the next
14
+ * complete record would exceed `targetSize`. A single record that is already
15
+ * over the budget gets its own shard (a shard is always at least one complete
16
+ * record, so over-budget records are never split).
17
+ *
18
+ * Triad: A = this semantic tier plus exact shards; B = the same range shape
19
+ * derived from extractive heads with the semantic encoder disabled (the caller
20
+ * may substitute the derived payload, since the range+digest contract is
21
+ * identical); C = exact anchors/current transcript only (no semantic shards at
22
+ * all, handled by the exact tier alone).
23
+ *
24
+ * Pure/deterministic: hashes + array walking only, no storage, no console, no
25
+ * network (PREVENT-PI-004 / PREVENT-011).
26
+ */
27
+ import { createHash } from "node:crypto";
28
+ /**
29
+ * Compute the canonical per-record byte offsets for a session's sorted events.
30
+ * Events are sorted by seq (ascending) defensively; the byte stream is the
31
+ * concatenation of each event's `originalBytes` in that order. A shared pure
32
+ * helper so semantic + exact tiers agree on source ranges (the two partitioning
33
+ * algorithms remain otherwise independent). Returns `undefined`-guarding lookup
34
+ * plus the total length; empty input yields an empty map with total 0.
35
+ */
36
+ export function cumulativeOffsets(events) {
37
+ const map = new Map();
38
+ let off = 0;
39
+ const sorted = [...events].sort((a, b) => (a.seq < b.seq ? -1 : a.seq > b.seq ? 1 : 0));
40
+ for (const e of sorted) {
41
+ const len = e.originalBytes.length;
42
+ map.set(e.seq.toString(), { byteStart: off, byteEnd: off + len });
43
+ off += len;
44
+ }
45
+ return {
46
+ of: (seq) => map.get(seq.toString()),
47
+ total: off,
48
+ };
49
+ }
50
+ /** Approximate token count for a valid UTF-8 text (whitespace-separated runs). */
51
+ function approxTokens(text) {
52
+ if (text.length === 0)
53
+ return 0;
54
+ const runs = text.match(/\S+/g);
55
+ return runs ? runs.length : 0;
56
+ }
57
+ /** Deterministic SHARD digest over a semantic shard's derived payload. */
58
+ function sealedSemanticDigest(sessionId, events, range, tokenEstimate) {
59
+ const h = createHash("sha256");
60
+ h.update(`semantic-shard-v1|${sessionId}|`);
61
+ h.update(`${range.seqStart.toString()}|${range.seqEnd.toString()}|`);
62
+ h.update(`${range.byteStart}|${range.byteEnd}|`);
63
+ h.update(`${tokenEstimate}|`);
64
+ // One `~`-joined run of the covered records' authoritative byte digests in
65
+ // seq order, so the shard digest depends on exactly which records it covers.
66
+ for (const e of [...events].sort((a, b) => (a.seq < b.seq ? -1 : a.seq > b.seq ? 1 : 0))) {
67
+ h.update(`${e.seq.toString()}:${e.bytesDigest}~`);
68
+ }
69
+ return `sha256:${h.digest("hex")}`;
70
+ }
71
+ /**
72
+ * Partition the semantic tier (task 2). Returns `ok:true` with a list of
73
+ * `SemanticShardV1` whose (sorted) ranges are contiguous, non-overlapping and
74
+ * jointly cover every event exactly once. Failures are defensive: a non-positive
75
+ * or non-finite `targetSize` rejects with `SHD_INVALID_TARGET_SIZE`; mixed-session
76
+ * events reject with `SHD_CROSS_SESSION`.
77
+ */
78
+ export function partitionSemantic(input) {
79
+ const { sessionId, events, targetSize } = input;
80
+ if (!Number.isFinite(targetSize) || targetSize <= 0) {
81
+ return { ok: false, code: "SHD_INVALID_TARGET_SIZE" };
82
+ }
83
+ if (events.some((e) => e.sessionId !== sessionId)) {
84
+ return { ok: false, code: "SHD_CROSS_SESSION" };
85
+ }
86
+ const offsets = cumulativeOffsets(events);
87
+ const sorted = [...events].sort((a, b) => (a.seq < b.seq ? -1 : a.seq > b.seq ? 1 : 0));
88
+ if (sorted.length === 0)
89
+ return { ok: true, shards: [] };
90
+ const shards = [];
91
+ let batch = [];
92
+ let batchBytes = 0;
93
+ let batchTokens = 0;
94
+ const flush = () => {
95
+ if (batch.length === 0)
96
+ return;
97
+ const first = batch[0];
98
+ const last = batch[batch.length - 1];
99
+ const start = offsets.of(first.seq);
100
+ const end = offsets.of(last.seq);
101
+ const byteStart = start ? start.byteStart : 0;
102
+ const byteEnd = end ? end.byteEnd : byteStart;
103
+ const tokenEstimate = batchTokens;
104
+ shards.push({
105
+ schema: "semantic-shard-v1",
106
+ sessionId,
107
+ range: {
108
+ sessionId,
109
+ seqStart: first.seq,
110
+ seqEnd: last.seq,
111
+ byteStart,
112
+ byteEnd,
113
+ },
114
+ kind: "semantic",
115
+ digest: sealedSemanticDigest(sessionId, batch, { seqStart: first.seq, seqEnd: last.seq, byteStart, byteEnd }, tokenEstimate),
116
+ byteCount: batchBytes,
117
+ eventCount: batch.length,
118
+ tokenEstimate,
119
+ });
120
+ batch = [];
121
+ batchBytes = 0;
122
+ batchTokens = 0;
123
+ };
124
+ for (const e of sorted) {
125
+ const len = e.originalBytes.length;
126
+ const tokens = e.utf8.valid ? approxTokens(e.utf8.text) : Math.ceil(len / 4);
127
+ // A batch that already has a complete record and would exceed the budget
128
+ // (and the next record itself fits, or it doesn't) closes at a boundary.
129
+ if (batch.length > 0 && batchBytes + len > targetSize)
130
+ flush();
131
+ batch.push(e);
132
+ batchBytes += len;
133
+ batchTokens += tokens;
134
+ }
135
+ flush();
136
+ return { ok: true, shards };
137
+ }
@@ -0,0 +1,51 @@
1
+ /**
2
+ * vector-cortex/shards/types.ts — dual-tier shard contract (VC4A).
3
+ *
4
+ * Owns `SemanticShardV1` / `ExactShardV1` / `ShardManifestV1` — the shared
5
+ * partition contract of the sprint failure triad:
6
+ *
7
+ * A = semantic + exact shards (normal path);
8
+ * B = extractive + exact shards with the semantic encoder disabled;
9
+ * C = exact anchors/current transcript only.
10
+ *
11
+ * A session's canonical byte stream is partitioned ONLY at complete EventV2
12
+ * record boundaries (VC1A byte authority — an EventV2's `originalBytes` + its
13
+ * SHA-256 `bytesDigest` are authoritative, invalid UTF-8 is never normalized).
14
+ * A `ShardRange` names a contiguous run of complete records by inclusive
15
+ * seq `[seqStart..seqEnd]` and by the half-open source byte range
16
+ * `[byteStart, byteEnd)` they occupy in canonical stream order (record `seq`
17
+ * carries every byte from `offset(seq)` to `offset(seq)+len(seq)`).
18
+ *
19
+ * Semantic shards carry derived aggregate vectors/token estimates and NEVER
20
+ * reproduce exact text (the RESIDUAL_CODEC / exact-payload shards own exact
21
+ * bytes). Exact shards carry the EXACT original bytes, unchanged, for the
22
+ * protected spans: every tool call/result pair, every anchor, and every invalid
23
+ * UTF-8 event (SHD-UTF8-002 — invalid bytes are exact-only and never
24
+ * normalized/re-encoded). A tool call/result pair that straddles a target-size
25
+ * boundary stays in ONE exact shard (SHD-PAIR-001).
26
+ *
27
+ * Consumes only reviewer-accepted predecessor contracts (VC1A EventV2) and the
28
+ * [common contracts](../../../../docs/vector-cortex/CONTRACTS.md). Pure
29
+ * types/schema + small predicates: no storage, no console, no network
30
+ * (PREVENT-PI-004 / PREVENT-011). The acceptance contract and the conformance
31
+ * IDs (SHD-001..020 + named SHD-PAIR-001 / SHD-UTF8-002 / SHD-RANGE-003) live
32
+ * here.
33
+ */
34
+ /**
35
+ * Registered SHD conformance ID range (SHD-001..020). The acceptance test reads
36
+ * these rows from the v2 manifest and asserts each returns its manifest
37
+ * `ok`/`code`. The named assertions (SHD-PAIR-001 / SHD-UTF8-002 /
38
+ * SHD-RANGE-003) live in the acceptance test.
39
+ */
40
+ export const SHD_IDS = [
41
+ "SHD-001", "SHD-002", "SHD-003", "SHD-004", "SHD-005",
42
+ "SHD-006", "SHD-007", "SHD-008", "SHD-009", "SHD-010",
43
+ "SHD-011", "SHD-012", "SHD-013", "SHD-014", "SHD-015",
44
+ "SHD-016", "SHD-017", "SHD-018", "SHD-019", "SHD-020",
45
+ ];
46
+ /** Named SHD conformance assertions. */
47
+ export const SHD_NAMED_IDS = [
48
+ "SHD-PAIR-001",
49
+ "SHD-UTF8-002",
50
+ "SHD-RANGE-003",
51
+ ];