pi-mega-compact 0.18.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/assets/vector-cortex/encoder-v1/manifest.json +1 -0
  2. package/assets/vector-cortex/encoder-v1/model-card.json +1 -0
  3. package/assets/vector-cortex/encoder-v1/model.onnx +0 -0
  4. package/assets/vector-cortex/encoder-v1/tokenizer.json +1 -0
  5. package/dist/config/vector-cortex.js +18 -0
  6. package/dist/config.js +117 -0
  7. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +4 -0
  8. package/dist/extensions/dashboard-server/routes-vector-cortex.js +89 -1
  9. package/dist/log.js +47 -0
  10. package/dist/src/config/vector-cortex.js +18 -0
  11. package/dist/src/config.js +1 -1
  12. package/dist/src/vector-cortex/encoder/calibrate.js +174 -0
  13. package/dist/src/vector-cortex/encoder/emit-vc2b.js +63 -0
  14. package/dist/src/vector-cortex/encoder/emit-vc2c.js +59 -0
  15. package/dist/src/vector-cortex/encoder/fallback.js +62 -0
  16. package/dist/src/vector-cortex/encoder/heads.js +113 -0
  17. package/dist/src/vector-cortex/encoder/lexical.js +104 -0
  18. package/dist/src/vector-cortex/encoder/router.js +115 -0
  19. package/dist/src/vector-cortex/encoder/select.js +172 -0
  20. package/dist/src/vector-cortex/encoder/trigram.js +75 -0
  21. package/dist/src/vector-cortex/encoder/types.js +84 -0
  22. package/dist/vector-cortex/encoder/calibrate.js +174 -0
  23. package/dist/vector-cortex/encoder/emit-vc2b.js +63 -0
  24. package/dist/vector-cortex/encoder/emit-vc2c.js +59 -0
  25. package/dist/vector-cortex/encoder/fallback.js +62 -0
  26. package/dist/vector-cortex/encoder/heads.js +113 -0
  27. package/dist/vector-cortex/encoder/lexical.js +104 -0
  28. package/dist/vector-cortex/encoder/router.js +115 -0
  29. package/dist/vector-cortex/encoder/select.js +172 -0
  30. package/dist/vector-cortex/encoder/trigram.js +75 -0
  31. package/dist/vector-cortex/encoder/types.js +84 -0
  32. package/extensions/dashboard-client/dist/assets/{AreaChart-BQXFF7UI.js → AreaChart-DywU5SSt.js} +2 -2
  33. package/extensions/dashboard-client/dist/assets/{AreaChart-BQXFF7UI.js.map → AreaChart-DywU5SSt.js.map} +1 -1
  34. package/extensions/dashboard-client/dist/assets/{BarChart-BVtoz9Eg.js → BarChart-But7WN6K.js} +2 -2
  35. package/extensions/dashboard-client/dist/assets/{BarChart-BVtoz9Eg.js.map → BarChart-But7WN6K.js.map} +1 -1
  36. package/extensions/dashboard-client/dist/assets/{CacheTab-trI2TIGy.js → CacheTab-7aZWa1DR.js} +2 -2
  37. package/extensions/dashboard-client/dist/assets/{CacheTab-trI2TIGy.js.map → CacheTab-7aZWa1DR.js.map} +1 -1
  38. package/extensions/dashboard-client/dist/assets/{EventsTab-CNUbk9__.js → EventsTab-Bf_boNG2.js} +2 -2
  39. package/extensions/dashboard-client/dist/assets/{EventsTab-CNUbk9__.js.map → EventsTab-Bf_boNG2.js.map} +1 -1
  40. package/extensions/dashboard-client/dist/assets/{HealthTab-K-bd_CM-.js → HealthTab-AbGc1nXP.js} +2 -2
  41. package/extensions/dashboard-client/dist/assets/{HealthTab-K-bd_CM-.js.map → HealthTab-AbGc1nXP.js.map} +1 -1
  42. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-CoEioKzx.js → MaintenanceTab-BfHHhn5F.js} +2 -2
  43. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-CoEioKzx.js.map → MaintenanceTab-BfHHhn5F.js.map} +1 -1
  44. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-CJo24ulv.js → MemoryMapTab-DrcWti0-.js} +2 -2
  45. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-CJo24ulv.js.map → MemoryMapTab-DrcWti0-.js.map} +1 -1
  46. package/extensions/dashboard-client/dist/assets/{MetricsTab-DCNlQWWP.js → MetricsTab-ALiglXQq.js} +2 -2
  47. package/extensions/dashboard-client/dist/assets/{MetricsTab-DCNlQWWP.js.map → MetricsTab-ALiglXQq.js.map} +1 -1
  48. package/extensions/dashboard-client/dist/assets/{OverviewTab-CQQlyIQ6.js → OverviewTab-nnaJdNyR.js} +2 -2
  49. package/extensions/dashboard-client/dist/assets/{OverviewTab-CQQlyIQ6.js.map → OverviewTab-nnaJdNyR.js.map} +1 -1
  50. package/extensions/dashboard-client/dist/assets/{ReposTab-Cqm82_yn.js → ReposTab-BTK0nofM.js} +2 -2
  51. package/extensions/dashboard-client/dist/assets/{ReposTab-Cqm82_yn.js.map → ReposTab-BTK0nofM.js.map} +1 -1
  52. package/extensions/dashboard-client/dist/assets/{SessionsTab-A8xTAdET.js → SessionsTab-RXPNqO6x.js} +2 -2
  53. package/extensions/dashboard-client/dist/assets/{SessionsTab-A8xTAdET.js.map → SessionsTab-RXPNqO6x.js.map} +1 -1
  54. package/extensions/dashboard-client/dist/assets/{SetupTab-CqaSgaZl.js → SetupTab-Dab0AVfs.js} +2 -2
  55. package/extensions/dashboard-client/dist/assets/{SetupTab-CqaSgaZl.js.map → SetupTab-Dab0AVfs.js.map} +1 -1
  56. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-C_f3q8GR.js → TimeSavedCard-DIVG8hdt.js} +2 -2
  57. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-C_f3q8GR.js.map → TimeSavedCard-DIVG8hdt.js.map} +1 -1
  58. package/extensions/dashboard-client/dist/assets/{TurnsTab-BEsafACo.js → TurnsTab-9uz2fBwx.js} +2 -2
  59. package/extensions/dashboard-client/dist/assets/{TurnsTab-BEsafACo.js.map → TurnsTab-9uz2fBwx.js.map} +1 -1
  60. package/extensions/dashboard-client/dist/assets/{VectorCortexTab-DD1CMK80.js → VectorCortexTab-DnKY61EJ.js} +2 -2
  61. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DnKY61EJ.js.map +1 -0
  62. package/extensions/dashboard-client/dist/assets/{WikiTab-Bf01M2_Z.js → WikiTab-CpcgaggS.js} +2 -2
  63. package/extensions/dashboard-client/dist/assets/{WikiTab-Bf01M2_Z.js.map → WikiTab-CpcgaggS.js.map} +1 -1
  64. package/extensions/dashboard-client/dist/assets/{button-BEJPTk_E.js → button-B0Q2fuKY.js} +2 -2
  65. package/extensions/dashboard-client/dist/assets/{button-BEJPTk_E.js.map → button-B0Q2fuKY.js.map} +1 -1
  66. package/extensions/dashboard-client/dist/assets/{card-DK1V382s.js → card-Cnik3TFr.js} +2 -2
  67. package/extensions/dashboard-client/dist/assets/{card-DK1V382s.js.map → card-Cnik3TFr.js.map} +1 -1
  68. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-CRtaPphL.js → generateCategoricalChart-DrSwgn1V.js} +2 -2
  69. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-CRtaPphL.js.map → generateCategoricalChart-DrSwgn1V.js.map} +1 -1
  70. package/extensions/dashboard-client/dist/assets/{index-CB6BslE4.js → index-DraytcOY.js} +3 -3
  71. package/extensions/dashboard-client/dist/assets/{index-CB6BslE4.js.map → index-DraytcOY.js.map} +1 -1
  72. package/extensions/dashboard-client/dist/assets/{switch-C4FV6B9k.js → switch-DVQ1aB9b.js} +2 -2
  73. package/extensions/dashboard-client/dist/assets/{switch-C4FV6B9k.js.map → switch-DVQ1aB9b.js.map} +1 -1
  74. package/extensions/dashboard-client/dist/assets/{toggle-GBmeENMd.js → toggle-Dkppx-49.js} +2 -2
  75. package/extensions/dashboard-client/dist/assets/{toggle-GBmeENMd.js.map → toggle-Dkppx-49.js.map} +1 -1
  76. package/extensions/dashboard-client/dist/assets/{useSSE-BOKZGSn1.js → useSSE-CPkjU4Gx.js} +2 -2
  77. package/extensions/dashboard-client/dist/assets/{useSSE-BOKZGSn1.js.map → useSSE-CPkjU4Gx.js.map} +1 -1
  78. package/extensions/dashboard-client/dist/index.html +1 -1
  79. package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +2 -0
  80. package/extensions/dashboard-client/src/types/vector-cortex.ts +11 -0
  81. package/extensions/dashboard-server/api-contracts/vector-cortex.ts +11 -0
  82. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +14 -0
  83. package/extensions/dashboard-server/routes-vector-cortex.ts +108 -1
  84. package/package.json +2 -1
  85. package/src/config/vector-cortex.ts +20 -0
  86. package/src/config.ts +2 -0
  87. package/src/vector-cortex/encoder/calibrate.ts +225 -0
  88. package/src/vector-cortex/encoder/emit-vc2b.ts +82 -0
  89. package/src/vector-cortex/encoder/emit-vc2c.ts +70 -0
  90. package/src/vector-cortex/encoder/fallback.ts +99 -0
  91. package/src/vector-cortex/encoder/heads.ts +142 -0
  92. package/src/vector-cortex/encoder/lexical.ts +123 -0
  93. package/src/vector-cortex/encoder/router.ts +163 -0
  94. package/src/vector-cortex/encoder/select.ts +237 -0
  95. package/src/vector-cortex/encoder/trigram.ts +85 -0
  96. package/src/vector-cortex/encoder/types.ts +226 -0
  97. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DD1CMK80.js.map +0 -1
@@ -17,11 +17,16 @@
17
17
 
18
18
  import type { IncomingMessage, ServerResponse } from "node:http";
19
19
  import type { RouteContext } from "./routes-core.js";
20
- import { VC0A_ENABLED, VC0C_ENABLED, VC1B_ENABLED } from "../../src/config.js";
20
+ import { createHash } from "node:crypto";
21
+ import { readFileSync, statSync } from "node:fs";
22
+ import { join, dirname } from "node:path";
23
+ import { fileURLToPath } from "node:url";
24
+ import { VC0A_ENABLED, VC0C_ENABLED, VC1B_ENABLED, VC2C_ENABLED } from "../../src/config.js";
21
25
  import { readEvalRows } from "../../src/vector-cortex/eval/persist.js";
22
26
  import { summarizeEvalRows } from "../../src/vector-cortex/eval/reader.js";
23
27
  import { createVectorCortexSafety } from "../mega-runtime/vector-cortex-safety.js";
24
28
  import { createLedgerStore } from "../../src/vector-cortex/ledger/store.js";
29
+ import { readEncoderManifest, verifyEncoderAsset, detectPlatform } from "../../src/vector-cortex/encoder/asset.js";
25
30
  import type {
26
31
  VectorCortexEvaluationSummary,
27
32
  VectorCortexHealthCard,
@@ -65,6 +70,102 @@ function sendJson(res: ServerResponse, status: number, body: unknown): void {
65
70
  res.end(JSON.stringify(body));
66
71
  }
67
72
 
73
+ /** Resolve the committed encoder-v1 asset dir by walking up to the repo root. */
74
+ function encoderAssetDir(): string | null {
75
+ let dir = dirname(fileURLToPath(import.meta.url));
76
+ const rel = join("assets", "vector-cortex", "encoder-v1");
77
+ for (let i = 0; i < 8; i++) {
78
+ const candidate = join(dir, rel);
79
+ try {
80
+ // guardrails-allow PREVENT-PI-004: local asset filesystem read (loopback)
81
+ readFileSync(join(candidate, "manifest.json"));
82
+ return candidate;
83
+ } catch {
84
+ /* keep walking */
85
+ }
86
+ const next = dirname(dir);
87
+ if (next === dir) break;
88
+ dir = next;
89
+ }
90
+ return null;
91
+ }
92
+
93
+ /**
94
+ * VC2C encoder health facts (task 5): the SHA-256 of the committed qualified
95
+ * manifest (asset digest) and the encoder triad mode. Reader-only aggregate
96
+ * (digest prefix, no bytes). Gated on MEGACOMPACT_VC2C: when the flag is OFF
97
+ * (rollback), the card reports mode C with an absent digest — consistent with
98
+ * the rest of the card, whose enabled/mode fields reflect active (flag-gated)
99
+ * state. When ON: "A" when the committed asset verifies on this host, "B" when
100
+ * present but not verified (demotion), "C" when absent.
101
+ *
102
+ * The verification hashes the full ONNX + tokenizer bytes, which is expensive
103
+ * on a periodically-polled endpoint as the shipped asset grows. The result is
104
+ * memoized against a cheap key (manifest bytes + file sizes/mtimes + platform);
105
+ * any on-disk mutation invalidates the key, so the cache never goes stale while
106
+ * avoiding the repeated large-file hashing. A failed digest read returns null
107
+ * (an absent digest), never a misleading zero-sentinel.
108
+ */
109
+ type EncoderHealthFacts = { assetDigest: string | null; mode: "A" | "B" | "C" };
110
+
111
+ interface HealthFactCache {
112
+ key: string;
113
+ facts: EncoderHealthFacts;
114
+ }
115
+
116
+ let healthCache: HealthFactCache | null = null;
117
+
118
+ /** Stable cache key: manifest bytes + (size,mtime) of the asset files + platform. */
119
+ function healthFactCacheKey(dir: string, manifestPath: string): string | null {
120
+ const platform = detectPlatform();
121
+ if (!platform) return null;
122
+ const parts: string[] = [platform];
123
+ try {
124
+ for (const p of [manifestPath, join(dir, "model.onnx"), join(dir, "tokenizer.json")]) {
125
+ const s = statSync(p);
126
+ parts.push(`${s.size}:${s.mtimeMs}`);
127
+ }
128
+ // guardrails-allow PREVENT-PI-004: local manifest filesystem read (loopback)
129
+ parts.push(createHash("sha256").update(readFileSync(manifestPath)).digest("hex"));
130
+ } catch {
131
+ return null;
132
+ }
133
+ return parts.join("|");
134
+ }
135
+
136
+ function computeEncoderHealthFacts(dir: string): EncoderHealthFacts | null {
137
+ const manifest = readEncoderManifest(dir);
138
+ if (manifest === null) return { assetDigest: null, mode: "C" };
139
+ const manifestPath = join(dir, "manifest.json");
140
+ const key = healthFactCacheKey(dir, manifestPath);
141
+ if (key === null) return null;
142
+ if (healthCache !== null && healthCache.key === key) return healthCache.facts;
143
+ let digest: string;
144
+ try {
145
+ // guardrails-allow PREVENT-PI-004: local manifest filesystem read (loopback)
146
+ digest = createHash("sha256").update(readFileSync(manifestPath)).digest("hex");
147
+ } catch {
148
+ return { assetDigest: null, mode: "B" };
149
+ }
150
+ const verify = verifyEncoderAsset(dir, manifest, detectPlatform());
151
+ const facts: EncoderHealthFacts = { assetDigest: digest, mode: verify.ok ? "A" : "B" };
152
+ healthCache = { key, facts };
153
+ return facts;
154
+ }
155
+
156
+ function encoderHealthFacts(): EncoderHealthFacts {
157
+ if (!VC2C_ENABLED()) {
158
+ // Rollback: MEGACOMPACT_VC2C=0 selects C — report the inactive (flag-gated)
159
+ // mode with no asset digest rather than an on-disk-computed A/B.
160
+ return { assetDigest: null, mode: "C" };
161
+ }
162
+ const dir = encoderAssetDir();
163
+ if (dir === null) return { assetDigest: null, mode: "C" };
164
+ const facts = computeEncoderHealthFacts(dir);
165
+ if (facts === null) return { assetDigest: null, mode: "C" };
166
+ return facts;
167
+ }
168
+
68
169
  /** Reader-only aggregate GET /api/vector-cortex/evaluation. */
69
170
  export function handleVectorCortexEvaluation(
70
171
  req: IncomingMessage,
@@ -133,6 +234,8 @@ export function handleVectorCortexHealth(
133
234
  const safety = enabled ? createVectorCortexSafety({ stateDir: ctx.stateDir }) : null;
134
235
  const card = safety ? safety.health() : null;
135
236
 
237
+ const enc = encoderHealthFacts();
238
+
136
239
  const fallback: VectorCortexHealthCard = {
137
240
  enabled: false,
138
241
  mode: "C",
@@ -152,6 +255,8 @@ export function handleVectorCortexHealth(
152
255
  updatedAt: new Date().toISOString(),
153
256
  aggregate: "CLOSED_A",
154
257
  stateSource: "ephemeral",
258
+ encoderAssetDigest: enc.assetDigest,
259
+ encoderMode: enc.mode,
155
260
  };
156
261
  if (!card) {
157
262
  sendJson(res, 200, fallback);
@@ -191,6 +296,8 @@ export function handleVectorCortexHealth(
191
296
  updatedAt: new Date().toISOString(),
192
297
  aggregate,
193
298
  stateSource: card.stateSource,
299
+ encoderAssetDigest: enc.assetDigest,
300
+ encoderMode: enc.mode,
194
301
  };
195
302
  sendJson(res, 200, body);
196
303
  return true;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-mega-compact",
3
- "version": "0.18.0",
3
+ "version": "0.19.0",
4
4
  "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
5
5
  "type": "module",
6
6
  "license": "BSD-3-Clause",
@@ -27,6 +27,7 @@
27
27
  "extensions",
28
28
  "extensions/dashboard-client/dist",
29
29
  "src",
30
+ "assets/vector-cortex/encoder-v1",
30
31
  "README.md",
31
32
  "LICENSE",
32
33
  "!dist/**/*.test.js",
@@ -80,6 +80,26 @@ export const VC1C_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC1C");
80
80
  */
81
81
  export const VC2A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC2A");
82
82
 
83
+ /**
84
+ * VC2B — multi-head encoder (VectorSetV1 / HeadCalibrationDraft).
85
+ * Default ON. `MEGACOMPACT_VC2B=0` disables and is byte-identical to the
86
+ * predecessor (the encoder emits no per-head vectors and no fallback-selected
87
+ * event; the trigram/lexical paths themselves are unchanged and are the
88
+ * predecessor's mode-B/C producers). The real consumers are the encoder-heads
89
+ * emit seam and the multi-head encoder producers (heads/trigram/lexical).
90
+ */
91
+ export const VC2B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC2B");
92
+
93
+ /**
94
+ * VC2C — encoder qualification + calibration (QualifiedEncoderV1 / CalibrationV1).
95
+ * Default ON. `MEGACOMPACT_VC2C=0` disables and is byte-identical to the
96
+ * predecessor (mode C: no qualification manifest is read or selected, the
97
+ * calibrate/select/fallback seams are idle, zero `vector_cortex_encoder_qualification_*`
98
+ * emissions; the trigram/lexical paths are unchanged). The real consumers are
99
+ * the encoder-qualification emit seam and the calibrate/select seams.
100
+ */
101
+ export const VC2C_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC2C");
102
+
83
103
  // ---------------------------------------------------------------------------
84
104
  // Breaker state machine constants (TRIAD_RESILIENCE.md §breaker).
85
105
  // Rolled numbers for one 60s window; VC0C consumes these at its breaker seam.
package/src/config.ts CHANGED
@@ -158,6 +158,8 @@ export {
158
158
  VC1B_ENABLED,
159
159
  VC1C_ENABLED,
160
160
  VC2A_ENABLED,
161
+ VC2B_ENABLED,
162
+ VC2C_ENABLED,
161
163
  BREAKER_WINDOW_MS,
162
164
  BREAKER_MIN_ATTEMPTS,
163
165
  BREAKER_PERF_FAILURES,
@@ -0,0 +1,225 @@
1
+ /**
2
+ * vector-cortex/encoder/calibrate.ts — VC2C calibration fit (task 2).
3
+ *
4
+ * Fits a `CalibrationV1` using ONLY the calibration split. Held-out
5
+ * (test/eval) labels are STRICTLY PROHIBITED from the fit inputs: the fit
6
+ * function rejects (ENC_QUALIFICATION_HELD_OUT_IN_FIT) any example whose
7
+ * `itemId` appears in the caller's held-out set. Ties in example score are
8
+ * broken deterministically by item ID (stable score/id ties), never by arrival
9
+ * order, so the fit is invariant to row order.
10
+ *
11
+ * The calibration split assignment is grouped by repository+session
12
+ * (EVALUATION.md §corpus): every label-bearing item carries a `repository` +
13
+ * `session` group, and the split digest is the canonical SHA-256 over the
14
+ * sorted group list. A single repo/session group NEVER crosses split boundaries
15
+ * (the caller seeds a group wholly into the calibration split or not at all).
16
+ *
17
+ * The fit itself is a deterministic seeded per-head temperature + threshold
18
+ * calibration over the calibration-only examples (real isotonic/Platt weights
19
+ * land with trained weights; the contract, split isolation, held-out
20
+ * prohibition, stable ties, and frozen-temperature/threshold surface are all
21
+ * normative here). Pi-agnostic, zero network (PREVENT-PI-004), no `any`
22
+ * (PREVENT-011).
23
+ */
24
+
25
+ import { createHash } from "node:crypto";
26
+ import {
27
+ ENCODER_HEAD_ORDER,
28
+ ENCODER_SEED,
29
+ ENC_QUALIFICATION_FAIL,
30
+ type CalibrationV1,
31
+ type EncoderHeadName,
32
+ } from "./types.js";
33
+
34
+ /** A scored, label-bearing calibration item. `heldOutIds` forbid review labels. */
35
+ export interface CalibrationExample {
36
+ readonly itemId: string;
37
+ readonly head: EncoderHeadName;
38
+ readonly score: number;
39
+ /** Binary ground-truth label (0/1) against which the raw score is calibrated. */
40
+ readonly label: 0 | 1;
41
+ /** Repository group (EVALUATION.md §corpus) — the split unit. */
42
+ readonly repository: string;
43
+ /** Session group (within the repository). */
44
+ readonly session: string;
45
+ }
46
+
47
+ export interface CalibrationFitOptions {
48
+ /** Seed of the deterministic fit (defaults to ENCODER_SEED). */
49
+ readonly seed?: number;
50
+ }
51
+
52
+ export type CalibrationFitResult =
53
+ | { ok: true; calibration: CalibrationV1 }
54
+ | { ok: false; code: string; reason: string };
55
+
56
+ /** Canonical digests of a sorted stable representation (order-invariant). */
57
+ function digestStrings(values: readonly string[]): string {
58
+ const sorted = [...values].sort();
59
+ return createHash("sha256").update(sorted.join("\n")).digest("hex");
60
+ }
61
+
62
+ /**
63
+ * Compute the calibration split digest for a set of (repository, session) groups
64
+ * present in the fit. The group list is canonicalized (sorted, deduped) so the
65
+ * digest is invariant to row order within the input. A caller that seeds WHOLE
66
+ * groups into the calibration split guarantees no group crosses a boundary.
67
+ */
68
+ /**
69
+ * Render one (repository, session) group as an injective canonical string.
70
+ * Each field is length-prefixed (`<len>:<value>`), so two distinct pairs can
71
+ * never collide to the same rendering — e.g. `{r:"a", s:"b::c"}` renders as
72
+ * `1:a3:b::c` while `{r:"a::b", s:"c"}` renders as `4:a::b1:c`. A plain
73
+ * `repository::session` join would conflate those when identifiers happen to
74
+ * contain "::"; length-prefixing makes the split digest sound for arbitrary
75
+ * repository/session identifiers.
76
+ */
77
+ function renderGroup(g: { repository: string; session: string }): string {
78
+ return `${g.repository.length}:${g.repository}${g.session.length}:${g.session}`;
79
+ }
80
+
81
+ export function calibrationSplitDigest(groups: readonly { repository: string; session: string }[]): string {
82
+ const rendered = new Set<string>();
83
+ for (const g of groups) rendered.add(renderGroup(g));
84
+ return digestStrings([...rendered]);
85
+ }
86
+
87
+ /** A deterministic 32-bit LCG step (matches the heads/runtime projectors). */
88
+ function nextState(state: number): number {
89
+ return (state * 1664525 + 1013904223) >>> 0;
90
+ }
91
+
92
+ /**
93
+ * Deterministic per-head temperature in a stable, healthy range (e.g. 0.8..1.5).
94
+ *
95
+ * The caller-supplied `seed` is mixed into the LCG state through independent
96
+ * steps so it ALWAYS affects the temperature — even when it equals ENCODER_SEED.
97
+ * (A naive `ENCODER_SEED ^ head.length ^ seed` cancels the two seed terms when
98
+ * `seed === ENCODER_SEED`, leaving a pure function of the head — the default
99
+ * path would make the seed option inert. Here the seed seeds the PRNG first,
100
+ * then the head name is folded in, so both vary the fit independently.) The fit
101
+ * stays deterministic for a fixed (seed, head) across processes.
102
+ */
103
+ function fitTemperature(head: EncoderHeadName, seed: number): number {
104
+ let state = (seed >>> 0) ^ 0x9e3779b9;
105
+ state = nextState(state);
106
+ state = (state ^ (head.length >>> 0)) >>> 0;
107
+ state = nextState(state);
108
+ state = (state ^ 0x85ebca6b) >>> 0;
109
+ state = nextState(state);
110
+ const r = (state / 4294967296) % 1;
111
+ return 0.8 + r * 0.7; // 0.8 .. 1.5
112
+ }
113
+
114
+ /**
115
+ * Deterministic per-head decision threshold derived from the calibration
116
+ * distribution. For a head with BOTH classes present the threshold is the
117
+ * midpoint between the highest-scoring negative (label 0) and the lowest-scoring
118
+ * positive (label 1) — a true between-class balance point that NEVER lands on a
119
+ * negative example's own score (code-review Q03): a future inference at the
120
+ * highest calibration negative is still classified negative, and one at the
121
+ * lowest calibration positive is still classified positive. Degenerate heads with
122
+ * a single class fall back conservatively (no positives -> just above the top
123
+ * observed score; no negatives -> just below the lowest observed positive) and an
124
+ * empty head defaults to 0.5. Scoring is order-invariant: ties in score resolve by
125
+ * item ID bytewise (stable score/id ties), never by arrival order, so the fit is
126
+ * invariant to row order. This frozen threshold is a normative placeholder (real
127
+ * trained weights land later).
128
+ */
129
+ function fitThreshold(
130
+ head: EncoderHeadName,
131
+ examples: readonly CalibrationExample[],
132
+ ): number {
133
+ const headEx = examples
134
+ .filter((e) => e.head === head)
135
+ .slice()
136
+ .sort((a, b) => (a.score - b.score) || (a.itemId < b.itemId ? -1 : a.itemId > b.itemId ? 1 : 0));
137
+ if (headEx.length === 0) return 0.5;
138
+ let highestNeg = -Infinity;
139
+ let lowestPos = Infinity;
140
+ for (const e of headEx) {
141
+ if (e.label === 0) highestNeg = Math.max(highestNeg, e.score);
142
+ else lowestPos = Math.min(lowestPos, e.score);
143
+ }
144
+ if (lowestPos === Infinity) {
145
+ // Only negatives observed — no positive class to balance against. Set the
146
+ // threshold just above the top observed score so no observed negative is
147
+ // re-admitted (a genuine future positive must exceed all calibration negatives).
148
+ return Math.max(0.5, highestNeg + 0.05);
149
+ }
150
+ if (highestNeg === -Infinity) {
151
+ // Only positives observed — set the threshold just below the lowest observed
152
+ // positive so every observed positive is admitted.
153
+ return Math.max(0, lowestPos - 0.05);
154
+ }
155
+ // Both classes present: the midpoint strictly between the highest negative and
156
+ // the lowest positive is the between-class balance point (Q03).
157
+ return (highestNeg + lowestPos) / 2;
158
+ }
159
+
160
+ /**
161
+ * Fit `CalibrationV1` over the calibration split only (task 2).
162
+ *
163
+ * - rejects any item whose `itemId` is in `heldOutIds` (held-out labels are
164
+ * prohibited from fit inputs).
165
+ * - treats `groups` as the calibration split units; the emitted split digest
166
+ * covers only the groups ACTUALLY present in the fit examples (plus the
167
+ * declared `groups`, when supplied — see below).
168
+ * - stable score/id ties (never arrival order).
169
+ *
170
+ * Held-out labels are prohibited by construction: the caller passes the full set
171
+ * of held-out item IDs, and the fit fails loudly if any calibration input is
172
+ * actually a held-out item — the fit can never silently learn from review labels.
173
+ */
174
+ export function fitCalibration(
175
+ examples: readonly CalibrationExample[],
176
+ options: CalibrationFitOptions & {
177
+ /** Item IDs that belong to the held-out (test/eval) split. Any calibration
178
+ * input whose id appears here is a fit violation. */
179
+ readonly heldOutIds?: readonly string[];
180
+ /** Declared calibration groups (repository+session). When supplied, the split
181
+ * digest covers these; otherwise it covers the groups actually in `examples`.
182
+ * Whole groups never cross a split boundary when the caller seeds a group
183
+ * into exactly one split. */
184
+ readonly groups?: readonly { repository: string; session: string }[];
185
+ } = {},
186
+ ): CalibrationFitResult {
187
+ const seed = options.seed ?? ENCODER_SEED;
188
+ const heldOut = new Set(options.heldOutIds ?? []);
189
+
190
+ for (const e of examples) {
191
+ if (heldOut.has(e.itemId)) {
192
+ return {
193
+ ok: false,
194
+ code: ENC_QUALIFICATION_FAIL.HELD_OUT_IN_FIT,
195
+ reason: `held-out item ${e.itemId} leaked into calibration fit`,
196
+ };
197
+ }
198
+ }
199
+
200
+ // Stable order by (score, itemId) — arrival order never affects the digest or
201
+ // the fit; ties resolve by item ID bytewise (stable score/id ties).
202
+ const stable = examples
203
+ .slice()
204
+ .sort((a, b) => (a.score - b.score) || (a.itemId < b.itemId ? -1 : a.itemId > b.itemId ? 1 : 0));
205
+
206
+ const usedGroups = options.groups ?? stable.map((e) => ({ repository: e.repository, session: e.session }));
207
+ const splitDigest = calibrationSplitDigest(usedGroups);
208
+ const temperatures: Record<EncoderHeadName, number> = {} as Record<EncoderHeadName, number>;
209
+ const thresholds: Record<EncoderHeadName, number> = {} as Record<EncoderHeadName, number>;
210
+ for (const head of ENCODER_HEAD_ORDER) {
211
+ temperatures[head] = fitTemperature(head, seed);
212
+ thresholds[head] = fitThreshold(head, stable);
213
+ }
214
+
215
+ const calibration: CalibrationV1 = {
216
+ schema: "calibration-v1",
217
+ headOrder: [...ENCODER_HEAD_ORDER],
218
+ calibrationSplitDigest: splitDigest,
219
+ fittedOnCalibrationOnly: true,
220
+ temperatures,
221
+ thresholds,
222
+ seed,
223
+ };
224
+ return { ok: true, calibration };
225
+ }
@@ -0,0 +1,82 @@
1
+ /**
2
+ * vector-cortex/encoder/emit-vc2b.ts — VC2B observability seam.
3
+ *
4
+ * Owns the two VC2B events (task 5), gated on `MEGACOMPACT_VC2B` so the
5
+ * flag-OFF path emits zero events (mode C parity, byte-identical predecessor):
6
+ *
7
+ * vector_cortex_encoder_heads_emitted — a multi-head VectorSetV1 produced
8
+ * vector_cortex_encoder_fallback_selected — a mode B/C fallback selected
9
+ *
10
+ * No dashboard or API change is necessary for this internal sprint (task 5).
11
+ * Every event is a JSON line with `ts` + `event` (ENGINEERING_PRACTICES §8); the
12
+ * emitters are non-fatal (never break the agent loop). Pi-agnostic, zero network
13
+ * (PREVENT-PI-004), no `any` (PREVENT-011).
14
+ */
15
+
16
+ import { VC2B_ENABLED } from "../../config/vector-cortex.js";
17
+ import { Logger } from "../../log.js";
18
+
19
+ export type EncoderEmit = (event: string, fields: Record<string, unknown>) => void;
20
+
21
+ /** The two-event VC2B reporter surface. */
22
+ export interface EncoderHeadsReporter {
23
+ readonly headsEmitted: (fields: Record<string, unknown>) => void;
24
+ readonly fallbackSelected: (fields: Record<string, unknown>) => void;
25
+ }
26
+
27
+ /** A flag-gated no-op reporter (zero emissions, structural no-op). */
28
+ export const NOOP_VC2B_REPORTER: EncoderHeadsReporter = {
29
+ headsEmitted: () => {},
30
+ fallbackSelected: () => {},
31
+ };
32
+
33
+ /**
34
+ * The default emitter: routes both VC2B events into the append-only structured
35
+ * logger (`src/log.ts`) as JSON lines with `ts` + `event`. Supplying `emit:` to
36
+ * `createEncoderHeadsReporter` replaces this with a caller-provided sink (used
37
+ * by tests and downstream consumers). Making the default a REAL producer means a
38
+ * caller that just invokes the producer seam (`encodeOrFallback`, `encodeVectorSet`,
39
+ * `selectTrigramBFallback`, `selectLexicalC`) without injecting an emitter still
40
+ * yields structured telemetry instead of silently dropping every event (task 5,
41
+ * code-review Q01). Best-effort: the logger swallows all I/O errors.
42
+ */
43
+ function defaultEmitFor(logPath: string | undefined): EncoderEmit {
44
+ const logger = new Logger(logPath === undefined ? {} : { path: logPath });
45
+ return (event, fields) => {
46
+ logger.info(event, fields);
47
+ };
48
+ }
49
+
50
+ /** Options for the default logger-backed sink (used when no `emit` is injected). */
51
+ export interface EncoderHeadsEmitOptions {
52
+ /** Where the default structured sink writes (defaults to the global log path). */
53
+ readonly logPath?: string;
54
+ }
55
+
56
+ /**
57
+ * Flag-gated emit, defaulting to a real logger-backed sink. The returned
58
+ * reporter is itself flag-gated (`VC2B_ENABLED`), so wiring it into a producer
59
+ * seam yields zero emissions when `MEGACOMPACT_VC2B=0` (byte-identical to the
60
+ * predecessor). Pass an explicit `emit` to route elsewhere (tests, downstream
61
+ * consumers); omit it to emit real structured log lines (Q01: the default is a
62
+ * live producer, not a silent no-op). `opts.logPath` only redirects the default
63
+ * sink and is ignored when `emit` is supplied.
64
+ */
65
+ export function createEncoderHeadsReporter(
66
+ emit?: EncoderEmit,
67
+ opts: EncoderHeadsEmitOptions = {},
68
+ ): EncoderHeadsReporter {
69
+ const sink = emit ?? defaultEmitFor(opts.logPath);
70
+ const fire = (event: string, fields: Record<string, unknown>): void => {
71
+ if (!VC2B_ENABLED()) return;
72
+ try {
73
+ sink(event, { ...fields, ts: new Date().toISOString() });
74
+ } catch {
75
+ /* non-fatal observability */
76
+ }
77
+ };
78
+ return {
79
+ headsEmitted: (fields) => fire("vector_cortex_encoder_heads_emitted", fields),
80
+ fallbackSelected: (fields) => fire("vector_cortex_encoder_fallback_selected", fields),
81
+ };
82
+ }
@@ -0,0 +1,70 @@
1
+ /**
2
+ * vector-cortex/encoder/emit-vc2c.ts — VC2C observability seam.
3
+ *
4
+ * Owns the two VC2C events (task 5), gated on `MEGACOMPACT_VC2C` so the
5
+ * flag-OFF path emits zero events (mode C parity, byte-identical predecessor):
6
+ *
7
+ * vector_cortex_encoder_qualification_passed — a QualifiedEncoderV1 produced (A)
8
+ * vector_cortex_encoder_qualification_demoted — qualification failed; all of A demoted
9
+ *
10
+ * Every event is a JSON line with `ts` + `event` (ENGINEERING_PRACTICES §8); the
11
+ * `ts` is the numeric epoch-ms timestamp the `Logger` injects (LogEntry.ts is
12
+ * `number`), so these events are consistent with the rest of the log stream
13
+ * (code-review Q04 — no ISO-string ts override). The emitters are non-fatal
14
+ * (never break the agent loop). Pi-agnostic, zero network (PREVENT-PI-004), no
15
+ * `any` (PREVENT-011).
16
+ */
17
+
18
+ import { VC2C_ENABLED } from "../../config/vector-cortex.js";
19
+ import { Logger } from "../../log.js";
20
+
21
+ export type EncoderEmit = (event: string, fields: Record<string, unknown>) => void;
22
+
23
+ /** The two-event VC2C reporter surface (qualification passed / demoted). */
24
+ export interface EncoderQualificationReporter {
25
+ readonly qualificationPassed: (fields: Record<string, unknown>) => void;
26
+ readonly qualificationDemoted: (fields: Record<string, unknown>) => void;
27
+ }
28
+
29
+ /**
30
+ * The default emitter: routes both VC2C events into the append-only structured
31
+ * logger (`src/log.ts`) as JSON lines with `event` (the logger injects the
32
+ * numeric `ts`). Supplying `emit:` to `createEncoderQualificationReporter`
33
+ * replaces this with a caller-provided sink (used by tests and downstream
34
+ * consumers). Making the default a REAL producer means a caller that just
35
+ * invokes the producer seam (`selectQualifiedEncoder`) without injecting an
36
+ * emitter still yields structured telemetry instead of silently dropping every
37
+ * event (task 5). Best-effort: the logger swallows all I/O errors.
38
+ */
39
+ function defaultEmitFor(): EncoderEmit {
40
+ const logger = new Logger();
41
+ return (event, fields) => {
42
+ logger.info(event, fields);
43
+ };
44
+ }
45
+
46
+ /**
47
+ * Flag-gated emit, defaulting to a real logger-backed sink. The returned
48
+ * reporter is itself flag-gated (`VC2C_ENABLED`), so wiring it into a producer
49
+ * seam yields zero emissions when `MEGACOMPACT_VC2C=0` (byte-identical to the
50
+ * predecessor). Pass an explicit `emit` to route elsewhere (tests, downstream
51
+ * consumers); omit it to emit real structured log lines. The `ts` is set by the
52
+ * underlying sink (numeric epoch-ms from `Logger`), not overridden here (Q04).
53
+ */
54
+ export function createEncoderQualificationReporter(
55
+ emit?: EncoderEmit,
56
+ ): EncoderQualificationReporter {
57
+ const sink = emit ?? defaultEmitFor();
58
+ const fire = (event: string, fields: Record<string, unknown>): void => {
59
+ if (!VC2C_ENABLED()) return;
60
+ try {
61
+ sink(event, fields);
62
+ } catch {
63
+ /* non-fatal observability */
64
+ }
65
+ };
66
+ return {
67
+ qualificationPassed: (fields) => fire("vector_cortex_encoder_qualification_passed", fields),
68
+ qualificationDemoted: (fields) => fire("vector_cortex_encoder_qualification_demoted", fields),
69
+ };
70
+ }
@@ -0,0 +1,99 @@
1
+ /**
2
+ * vector-cortex/encoder/fallback.ts — VC2C qualification fallback selection.
3
+ *
4
+ * After a qualification demotion (qualify/select fails to produce mode A), this
5
+ * seam selects the mode-B/C fallback and routes the encode request to the
6
+ * independently initialized VC2B producers (trigram B / lexical C). It is the
7
+ * breaker-recovery handoff for the encoder triad:
8
+ *
9
+ * A = fully qualified learned asset (QualifiedEncoderV1).
10
+ * B = asset-free trigram, forced by ANY one failed qualification threshold
11
+ * (or a qualification-manifest digest mismatch).
12
+ * C = token/phrase lexical, forced when A is absent AND B itself errors
13
+ * (injected B error), or when the caller forces C.
14
+ *
15
+ * Breaker recovery follows TRIAD_RESILIENCE: a demotion may probe/promote, never
16
+ * directly re-qualify A; the fallback only ever SELECTS B or C locally, never a
17
+ * remote fetch (PREVENT-PI-004). C states its loss of old semantic context.
18
+ *
19
+ * Pi-agnostic, zero network (PREVENT-PI-004), no `any` (PREVENT-011).
20
+ */
21
+
22
+ import { createEncoderHeadsReporter, type EncoderHeadsReporter } from "./emit-vc2b.js";
23
+ import { embedTrigram512, selectTrigramBFallback, ENCODER_TRIGRAM_WIDTH } from "./trigram.js";
24
+ import { embedLexical, selectLexicalC, ENCODER_LEXICAL_WIDTH, ENCODER_LEXICAL_LIMITATION } from "./lexical.js";
25
+
26
+ export type QualificationFallbackVerdict =
27
+ | {
28
+ readonly ok: true;
29
+ readonly mode: "B";
30
+ readonly vector: Float32Array;
31
+ readonly width: number;
32
+ /** The qualification failure code that forced B (never null on this path). */
33
+ readonly code: string;
34
+ readonly limitation: null;
35
+ }
36
+ | {
37
+ readonly ok: true;
38
+ readonly mode: "C";
39
+ readonly vector: Float32Array;
40
+ readonly width: number;
41
+ /** The qualification or B failure code that forced C. */
42
+ readonly code: string;
43
+ readonly limitation: string;
44
+ };
45
+
46
+ export interface FallbackSelectionOptions {
47
+ readonly reporter?: EncoderHeadsReporter;
48
+ /** Simulates a B initializer failure (forces C despite B being available). */
49
+ readonly injectBError?: boolean;
50
+ /** Force lexical C directly (skips the B attempt). */
51
+ readonly forceC?: boolean;
52
+ }
53
+
54
+ /** Deterministic text derived from an int token sequence (mirrors router.ts). */
55
+ function textFromTokens(tokens: readonly number[]): string {
56
+ return tokens.join("-");
57
+ }
58
+
59
+ /**
60
+ * Select the B/C fallback for a failed qualification and produce the encode
61
+ * vector. When `injectBError` or `forceC` is set, selection lands on lexical C
62
+ * (absent A + injected B error -> C); otherwise a qualification THRESHOLD or
63
+ * DIGEST mismatch selects trigram B. Never returns without an explicit verdict.
64
+ */
65
+ export function selectQualificationFallback(
66
+ qualificationCode: string,
67
+ tokens: readonly number[],
68
+ options: FallbackSelectionOptions = {},
69
+ ): QualificationFallbackVerdict {
70
+ const reporter = options.reporter ?? createEncoderHeadsReporter();
71
+ const tokensText = textFromTokens(tokens);
72
+
73
+ // A absent + injected/forced B error -> C (lexical) with the documented
74
+ // semantic-context limitation.
75
+ if (options.forceC || options.injectBError) {
76
+ selectLexicalC({ reporter });
77
+ const vector = embedLexical(tokensText);
78
+ return {
79
+ ok: true,
80
+ mode: "C",
81
+ vector,
82
+ width: ENCODER_LEXICAL_WIDTH,
83
+ code: options.injectBError ? "ENC_B_INJECTED_ERROR" : qualificationCode,
84
+ limitation: ENCODER_LEXICAL_LIMITATION,
85
+ };
86
+ }
87
+
88
+ // A qualification demotion selects the independently initialized trigram B.
89
+ selectTrigramBFallback({ reporter });
90
+ const vector = embedTrigram512(tokensText);
91
+ return {
92
+ ok: true,
93
+ mode: "B",
94
+ vector,
95
+ width: ENCODER_TRIGRAM_WIDTH,
96
+ code: qualificationCode,
97
+ limitation: null,
98
+ };
99
+ }