pi-mega-compact 0.20.35 → 0.20.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/config/vector-cortex-ml5a.js +28 -0
  2. package/dist/config/vector-cortex-ml5b.js +26 -0
  3. package/dist/config/vector-cortex.js +5 -5
  4. package/dist/config.js +1 -1
  5. package/dist/extensions/dashboard-server/routes-rag-settings-vector-cortex.js +1 -0
  6. package/dist/monitoring.js +172 -0
  7. package/dist/src/config/vector-cortex-ml5a.js +28 -0
  8. package/dist/src/config/vector-cortex-ml5b.js +26 -0
  9. package/dist/src/config/vector-cortex.js +5 -5
  10. package/dist/src/config.js +1 -1
  11. package/dist/src/monitoring.js +19 -0
  12. package/dist/src/store/backfill.js +1 -0
  13. package/dist/src/vector-cortex/encoder/bench-export.js +13 -0
  14. package/dist/src/vector-cortex/encoder/bench.js +100 -0
  15. package/dist/src/vector-cortex/encoder/calibrate.js +55 -0
  16. package/dist/src/vector-cortex/encoder/heads.js +87 -0
  17. package/dist/src/vector-cortex/encoder/select.js +10 -0
  18. package/dist/vector-cortex/encoder/bench-export.js +13 -0
  19. package/dist/vector-cortex/encoder/bench.js +100 -0
  20. package/dist/vector-cortex/encoder/calibrate.js +55 -0
  21. package/dist/vector-cortex/encoder/heads.js +87 -0
  22. package/dist/vector-cortex/encoder/select.js +10 -0
  23. package/dist/vectorStore/dedup-audit.js +104 -0
  24. package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts +6 -0
  25. package/package.json +1 -1
  26. package/src/config/vector-cortex-ml5a.ts +30 -0
  27. package/src/config/vector-cortex-ml5b.ts +28 -0
  28. package/src/config/vector-cortex.ts +5 -5
  29. package/src/config.ts +2 -0
  30. package/src/monitoring.ts +24 -0
  31. package/src/store/backfill.ts +1 -0
  32. package/src/vector-cortex/encoder/bench-export.ts +65 -0
  33. package/src/vector-cortex/encoder/bench.ts +109 -0
  34. package/src/vector-cortex/encoder/calibrate.ts +49 -0
  35. package/src/vector-cortex/encoder/heads.ts +106 -0
  36. package/src/vector-cortex/encoder/select.ts +17 -0
@@ -19,6 +19,8 @@
19
19
  *
20
20
  * Pi-agnostic, zero network (PREVENT-PI-004), no `any` (PREVENT-011).
21
21
  */
22
+ import { readFileSync } from "node:fs";
23
+ import { ML5A_ENABLED } from "../../config/vector-cortex.js";
22
24
  import { ENCODER_HEAD_DIMS, ENCODER_HEAD_ORDER, ENCODER_HEAD_LOSS_WEIGHTS, ENCODER_HEAD_LOSS_SUM, ENCODER_SEED, } from "./types.js";
23
25
  import { createEncoderHeadsReporter, NOOP_VC2B_REPORTER, } from "./emit-vc2b.js";
24
26
  /** The stable head index of a head name (its position in ENCODER_HEAD_ORDER). */
@@ -110,4 +112,89 @@ export function encodeVectorSet(tokens, options = {}) {
110
112
  export function headLossWeights() {
111
113
  return { ...ENCODER_HEAD_LOSS_WEIGHTS };
112
114
  }
115
+ /** True when every head's output dim + weight length matches the contract. */
116
+ export function headsShapeValid(t) {
117
+ return ENCODER_HEAD_ORDER.every((h) => t.dims[h] === ENCODER_HEAD_DIMS[h] && t.weights[h].length === ENCODER_HEAD_DIMS[h] * t.trunkDim);
118
+ }
119
+ /**
120
+ * Load a `trained-heads-v1` artifact into a `HeadProjectionTable`. Gated on
121
+ * MEGACOMPACT_ML5_A: flag-off, absent file, malformed JSON, wrong schema,
122
+ * wrong seed, or a shape mismatch each return null (non-fatal). Deterministic
123
+ * and local (PREVENT-PI-004).
124
+ */
125
+ export function loadHeadProjections(path) {
126
+ if (!ML5A_ENABLED())
127
+ return null;
128
+ let raw;
129
+ try {
130
+ raw = readFileSync(path, "utf8");
131
+ }
132
+ catch {
133
+ return null;
134
+ }
135
+ let parsed;
136
+ try {
137
+ parsed = JSON.parse(raw);
138
+ }
139
+ catch {
140
+ return null;
141
+ }
142
+ const r = parsed;
143
+ if (!r || r["schema"] !== "trained-heads-v1")
144
+ return null;
145
+ if (r["seed"] !== ENCODER_SEED)
146
+ return null;
147
+ const dims = r["dims"];
148
+ const heads = r["heads"];
149
+ if (!dims || !heads)
150
+ return null;
151
+ const trunkDim = Number(r["trunkDim"] ?? 0);
152
+ if (!Number.isFinite(trunkDim) || trunkDim <= 0)
153
+ return null;
154
+ const weights = {};
155
+ const temperatures = {};
156
+ for (const h of ENCODER_HEAD_ORDER) {
157
+ const hd = heads[h];
158
+ if (!hd || typeof hd !== "object")
159
+ return null;
160
+ const w = hd["weights"];
161
+ if (!Array.isArray(w))
162
+ return null;
163
+ weights[h] = Float32Array.from(w);
164
+ if (Number(hd["dim"] ?? 0) !== ENCODER_HEAD_DIMS[h])
165
+ return null;
166
+ temperatures[h] = Number(hd["temperature"] ?? 1);
167
+ if (!Number.isFinite(dims[h]))
168
+ return null;
169
+ }
170
+ const table = {
171
+ schema: "trained-heads-v1",
172
+ seed: Number(r["seed"]),
173
+ trunkDim,
174
+ dims: { semantic: 384, dependency: 128, contradiction: 128, cacheStability: 64, payloadRouting: 32 },
175
+ weights: weights,
176
+ temperatures: temperatures,
177
+ };
178
+ if (!headsShapeValid(table))
179
+ return null;
180
+ return table;
181
+ }
182
+ /**
183
+ * Project a trunk embedding through a trained head's real weights, applying the
184
+ * row-major matrix then L2-normalizing (all-zero on zero norm). Returns a
185
+ * `HeadVector` of the head's normative dimension.
186
+ */
187
+ export function projectHeadFromTrunk(head, trunk, table) {
188
+ const dim = ENCODER_HEAD_DIMS[head];
189
+ const W = table.weights[head];
190
+ const t = table.trunkDim;
191
+ const out = new Float32Array(dim);
192
+ for (let i = 0; i < dim; i++) {
193
+ let acc = 0;
194
+ for (let j = 0; j < t; j++)
195
+ acc += W[i * t + j] * (trunk[j] ?? 0);
196
+ out[i] = acc;
197
+ }
198
+ return { head, dim, values: l2Normalize(out) };
199
+ }
113
200
  export { ENCODER_HEAD_ORDER, ENCODER_HEAD_DIMS, ENCODER_HEAD_LOSS_SUM, ENCODER_SEED, NOOP_VC2B_REPORTER };
@@ -33,6 +33,8 @@
33
33
  * `any` (PREVENT-011).
34
34
  */
35
35
  import { createHash } from "node:crypto";
36
+ import { ML5A_ENABLED } from "../../config/vector-cortex.js";
37
+ import { loadHeadProjections } from "./heads.js";
36
38
  import { ENC_QUALIFICATION_FAIL, EVALUATION_THRESHOLDS, } from "./types.js";
37
39
  import { createEncoderQualificationReporter, } from "./emit-vc2c.js";
38
40
  /** Canonical digest over a CalibrationV1's stable identity (split digest, heads,
@@ -118,6 +120,14 @@ export function selectQualifiedEncoder(candidate, options = {}) {
118
120
  }
119
121
  // Atomic: collect EVERY failed field across asset + all heads + reconstruction.
120
122
  const failed = [];
123
+ // ML5-A: real trained-head weights must load for mode A. When the gate is on
124
+ // and a trained-heads path is pinned, an unloadable/wrong-seed/malformed
125
+ // artifact is a qualification failure (any failed field demotes ALL of A).
126
+ if (ML5A_ENABLED() && candidate.trainedHeadsPath !== undefined) {
127
+ if (loadHeadProjections(candidate.trainedHeadsPath) === null) {
128
+ failed.push("head.weights.trainedHeadsPath");
129
+ }
130
+ }
121
131
  assetPasses(candidate.asset, failed);
122
132
  const heads = ["semantic", "dependency", "contradiction", "cacheStability", "payloadRouting"];
123
133
  for (const h of heads) {
@@ -0,0 +1,13 @@
1
+ /**
2
+ * vector-cortex/encoder/bench-export.ts — ML5-B bench result contract.
3
+ *
4
+ * BenchResultV1 is the typed surface `bench.ts` parses from the qualification
5
+ * harness (`scripts/ml5/bench-onnx-prod.mjs`) and the dashboard / evidence
6
+ * tooling consume. It carries AGGREGATE measurements + a digest only — never
7
+ * chunk/message content (EVAL-REDACT-002).
8
+ *
9
+ * Contract-first (ENGINEERING_PRACTICES §3): this types file is the reviewed
10
+ * gate; implementations import from it. Pi-agnostic, dependency-free
11
+ * (PREVENT-PI-004 / PREVENT-011).
12
+ */
13
+ export {};
@@ -0,0 +1,100 @@
1
+ /**
2
+ * vector-cortex/encoder/bench.ts — ML5-B bench runner (consumer-facing shell).
3
+ *
4
+ * Calls `scripts/ml5/bench-onnx-prod.mjs` via child_process, parses the
5
+ * BenchResultV1 it emits, and writes the four `vector_cortex_encoder_bench_*`
6
+ * events to the monitoring events.log (the dashboard / ML5-D surface consume
7
+ * them later). This is NOT a runtime path — it is developer/evidence tooling.
8
+ *
9
+ * Events written (all best-effort / non-fatal):
10
+ * - vector_cortex_encoder_bench_p95_ms
11
+ * - vector_cortex_encoder_bench_rss_mib
12
+ * - vector_cortex_encoder_bench_opset_ok
13
+ * - vector_cortex_encoder_bench_deterministic
14
+ *
15
+ * Pi-agnostic, dependency-free (PREVENT-PI-004 — the child bench is pure local
16
+ * computation). No `any` (PREVENT-011).
17
+ */
18
+ import { spawnSync } from "node:child_process";
19
+ import { dirname, join } from "node:path";
20
+ import { fileURLToPath } from "node:url";
21
+ import { getStateDir } from "../../store.js";
22
+ import { defaultEventsPath, logBenchEvent } from "../../monitoring.js";
23
+ const HERE = dirname(fileURLToPath(import.meta.url));
24
+ const REPO_ROOT = join(HERE, "..", "..", "..");
25
+ const BENCH_SCRIPT = join(REPO_ROOT, "scripts", "ml5", "bench-onnx-prod.mjs");
26
+ /** Default events.log beside the state dir (mirrors defaultEventsPath). */
27
+ function benchEventsPath(stateDir) {
28
+ return defaultEventsPath(stateDir);
29
+ }
30
+ /**
31
+ * Run the ONNX bench once and record its four events. Returns the parsed
32
+ * BenchResultV1. On any failure (script missing, non-zero exit, unparsable
33
+ * output) it returns a degraded result with gates.all:false — never throws, so
34
+ * the caller's agent loop is never broken (non-fatal store/write contract).
35
+ */
36
+ export function runBench(stateDir = getStateDir()) {
37
+ const noop = (error) => ({
38
+ timestamp: Date.now(),
39
+ platform: `${process.platform}-${process.arch}`,
40
+ encoderNative: false,
41
+ threads: 4,
42
+ tokens: 512,
43
+ corpusTokens: 0,
44
+ p95Ms: null,
45
+ rssMib: null,
46
+ rssBaselineMib: null,
47
+ rssMarginalMib: null,
48
+ opset: null,
49
+ deterministic: false,
50
+ digest: null,
51
+ gates: { latency: false, rss: false, opset: false, determinism: false, all: false },
52
+ error,
53
+ });
54
+ const fallback = (error) => {
55
+ const r = noop(error);
56
+ emitEvents(stateDir, r);
57
+ return r;
58
+ };
59
+ try {
60
+ const res = spawnSync(process.execPath, ["--expose-gc", BENCH_SCRIPT], {
61
+ cwd: REPO_ROOT,
62
+ encoding: "utf8",
63
+ timeout: 600_000,
64
+ });
65
+ const stdout = (res.stdout ?? "").trim();
66
+ if (res.status === null) {
67
+ return fallback("bench timed out or failed to spawn");
68
+ }
69
+ const parsed = JSON.parse(stdout || "");
70
+ if (!isBenchResultV1(parsed)) {
71
+ return fallback("bench output was not a BenchResultV1");
72
+ }
73
+ emitEvents(stateDir, parsed);
74
+ return parsed;
75
+ }
76
+ catch (e) {
77
+ return fallback(`bench failed: ${e?.message ?? String(e)}`);
78
+ }
79
+ }
80
+ function isBenchResultV1(v) {
81
+ if (typeof v !== "object" || v === null)
82
+ return false;
83
+ const o = v;
84
+ return (typeof o.timestamp === "number" &&
85
+ typeof o.platform === "string" &&
86
+ typeof o.encoderNative === "boolean" &&
87
+ typeof o.threads === "number" &&
88
+ typeof o.tokens === "number" &&
89
+ typeof o.corpusTokens === "number" &&
90
+ typeof o.gates === "object" && o.gates !== null &&
91
+ typeof o.gates.all === "boolean");
92
+ }
93
+ function emitEvents(stateDir, r) {
94
+ const path = benchEventsPath(stateDir);
95
+ const run = { platform: r.platform, encoderNative: r.encoderNative, threads: r.threads, tokens: r.tokens, digest: r.digest, corpusTokens: r.corpusTokens };
96
+ logBenchEvent(path, "vector_cortex_encoder_bench_p95_ms", { ...run, p95Ms: r.p95Ms, pass: r.gates.latency });
97
+ logBenchEvent(path, "vector_cortex_encoder_bench_rss_mib", { ...run, rssMib: r.rssMib, rssBaselineMib: r.rssBaselineMib, rssMarginalMib: r.rssMarginalMib, pass: r.gates.rss });
98
+ logBenchEvent(path, "vector_cortex_encoder_bench_opset_ok", { ...run, opset: r.opset, pass: r.gates.opset });
99
+ logBenchEvent(path, "vector_cortex_encoder_bench_deterministic", { ...run, deterministic: r.deterministic, pass: r.gates.determinism });
100
+ }
@@ -22,6 +22,8 @@
22
22
  * (PREVENT-011).
23
23
  */
24
24
  import { createHash } from "node:crypto";
25
+ import { readFileSync } from "node:fs";
26
+ import { ML5A_ENABLED } from "../../config/vector-cortex.js";
25
27
  import { ENCODER_HEAD_ORDER, ENCODER_SEED, ENC_QUALIFICATION_FAIL, } from "./types.js";
26
28
  /** Canonical digests of a sorted stable representation (order-invariant). */
27
29
  function digestStrings(values) {
@@ -172,3 +174,56 @@ export function fitCalibration(examples, options = {}) {
172
174
  };
173
175
  return { ok: true, calibration };
174
176
  }
177
+ /**
178
+ * Load a persisted `CalibrationV1` artifact (schema "calibration-v1") from disk.
179
+ * ML5-A: gated on MEGACOMPACT_ML5_A; flag-off, absent file, malformed JSON,
180
+ * wrong schema, non-canonical five-head order, or non-finite temp/threshold each
181
+ * return null (non-fatal, never throws). Deterministic, local (PREVENT-PI-004).
182
+ */
183
+ export function loadCalibrationV1(path) {
184
+ if (!ML5A_ENABLED())
185
+ return null;
186
+ let raw;
187
+ try {
188
+ raw = readFileSync(path, "utf8");
189
+ }
190
+ catch {
191
+ return null;
192
+ }
193
+ let parsed;
194
+ try {
195
+ parsed = JSON.parse(raw);
196
+ }
197
+ catch {
198
+ return null;
199
+ }
200
+ const r = parsed;
201
+ if (!r || r["schema"] !== "calibration-v1")
202
+ return null;
203
+ const order = r["headOrder"];
204
+ if (!Array.isArray(order))
205
+ return null;
206
+ if (order.length !== ENCODER_HEAD_ORDER.length || !ENCODER_HEAD_ORDER.every((h, i) => order[i] === h)) {
207
+ return null;
208
+ }
209
+ const temperatures = r["temperatures"];
210
+ const thresholds = r["thresholds"];
211
+ const splitDigest = r["calibrationSplitDigest"];
212
+ if (!temperatures || !thresholds || typeof splitDigest !== "string" || splitDigest.length !== 64)
213
+ return null;
214
+ for (const h of ENCODER_HEAD_ORDER) {
215
+ const t = Number(temperatures[h]);
216
+ const th = Number(thresholds[h]);
217
+ if (!Number.isFinite(t) || !Number.isFinite(th))
218
+ return null;
219
+ }
220
+ return {
221
+ schema: "calibration-v1",
222
+ headOrder: [...ENCODER_HEAD_ORDER],
223
+ calibrationSplitDigest: splitDigest,
224
+ fittedOnCalibrationOnly: true,
225
+ temperatures: { ...temperatures },
226
+ thresholds: { ...thresholds },
227
+ seed: Number(r["seed"] ?? ENCODER_SEED),
228
+ };
229
+ }
@@ -19,6 +19,8 @@
19
19
  *
20
20
  * Pi-agnostic, zero network (PREVENT-PI-004), no `any` (PREVENT-011).
21
21
  */
22
+ import { readFileSync } from "node:fs";
23
+ import { ML5A_ENABLED } from "../../config/vector-cortex.js";
22
24
  import { ENCODER_HEAD_DIMS, ENCODER_HEAD_ORDER, ENCODER_HEAD_LOSS_WEIGHTS, ENCODER_HEAD_LOSS_SUM, ENCODER_SEED, } from "./types.js";
23
25
  import { createEncoderHeadsReporter, NOOP_VC2B_REPORTER, } from "./emit-vc2b.js";
24
26
  /** The stable head index of a head name (its position in ENCODER_HEAD_ORDER). */
@@ -110,4 +112,89 @@ export function encodeVectorSet(tokens, options = {}) {
110
112
  export function headLossWeights() {
111
113
  return { ...ENCODER_HEAD_LOSS_WEIGHTS };
112
114
  }
115
+ /** True when every head's output dim + weight length matches the contract. */
116
+ export function headsShapeValid(t) {
117
+ return ENCODER_HEAD_ORDER.every((h) => t.dims[h] === ENCODER_HEAD_DIMS[h] && t.weights[h].length === ENCODER_HEAD_DIMS[h] * t.trunkDim);
118
+ }
119
+ /**
120
+ * Load a `trained-heads-v1` artifact into a `HeadProjectionTable`. Gated on
121
+ * MEGACOMPACT_ML5_A: flag-off, absent file, malformed JSON, wrong schema,
122
+ * wrong seed, or a shape mismatch each return null (non-fatal). Deterministic
123
+ * and local (PREVENT-PI-004).
124
+ */
125
+ export function loadHeadProjections(path) {
126
+ if (!ML5A_ENABLED())
127
+ return null;
128
+ let raw;
129
+ try {
130
+ raw = readFileSync(path, "utf8");
131
+ }
132
+ catch {
133
+ return null;
134
+ }
135
+ let parsed;
136
+ try {
137
+ parsed = JSON.parse(raw);
138
+ }
139
+ catch {
140
+ return null;
141
+ }
142
+ const r = parsed;
143
+ if (!r || r["schema"] !== "trained-heads-v1")
144
+ return null;
145
+ if (r["seed"] !== ENCODER_SEED)
146
+ return null;
147
+ const dims = r["dims"];
148
+ const heads = r["heads"];
149
+ if (!dims || !heads)
150
+ return null;
151
+ const trunkDim = Number(r["trunkDim"] ?? 0);
152
+ if (!Number.isFinite(trunkDim) || trunkDim <= 0)
153
+ return null;
154
+ const weights = {};
155
+ const temperatures = {};
156
+ for (const h of ENCODER_HEAD_ORDER) {
157
+ const hd = heads[h];
158
+ if (!hd || typeof hd !== "object")
159
+ return null;
160
+ const w = hd["weights"];
161
+ if (!Array.isArray(w))
162
+ return null;
163
+ weights[h] = Float32Array.from(w);
164
+ if (Number(hd["dim"] ?? 0) !== ENCODER_HEAD_DIMS[h])
165
+ return null;
166
+ temperatures[h] = Number(hd["temperature"] ?? 1);
167
+ if (!Number.isFinite(dims[h]))
168
+ return null;
169
+ }
170
+ const table = {
171
+ schema: "trained-heads-v1",
172
+ seed: Number(r["seed"]),
173
+ trunkDim,
174
+ dims: { semantic: 384, dependency: 128, contradiction: 128, cacheStability: 64, payloadRouting: 32 },
175
+ weights: weights,
176
+ temperatures: temperatures,
177
+ };
178
+ if (!headsShapeValid(table))
179
+ return null;
180
+ return table;
181
+ }
182
+ /**
183
+ * Project a trunk embedding through a trained head's real weights, applying the
184
+ * row-major matrix then L2-normalizing (all-zero on zero norm). Returns a
185
+ * `HeadVector` of the head's normative dimension.
186
+ */
187
+ export function projectHeadFromTrunk(head, trunk, table) {
188
+ const dim = ENCODER_HEAD_DIMS[head];
189
+ const W = table.weights[head];
190
+ const t = table.trunkDim;
191
+ const out = new Float32Array(dim);
192
+ for (let i = 0; i < dim; i++) {
193
+ let acc = 0;
194
+ for (let j = 0; j < t; j++)
195
+ acc += W[i * t + j] * (trunk[j] ?? 0);
196
+ out[i] = acc;
197
+ }
198
+ return { head, dim, values: l2Normalize(out) };
199
+ }
113
200
  export { ENCODER_HEAD_ORDER, ENCODER_HEAD_DIMS, ENCODER_HEAD_LOSS_SUM, ENCODER_SEED, NOOP_VC2B_REPORTER };
@@ -33,6 +33,8 @@
33
33
  * `any` (PREVENT-011).
34
34
  */
35
35
  import { createHash } from "node:crypto";
36
+ import { ML5A_ENABLED } from "../../config/vector-cortex.js";
37
+ import { loadHeadProjections } from "./heads.js";
36
38
  import { ENC_QUALIFICATION_FAIL, EVALUATION_THRESHOLDS, } from "./types.js";
37
39
  import { createEncoderQualificationReporter, } from "./emit-vc2c.js";
38
40
  /** Canonical digest over a CalibrationV1's stable identity (split digest, heads,
@@ -118,6 +120,14 @@ export function selectQualifiedEncoder(candidate, options = {}) {
118
120
  }
119
121
  // Atomic: collect EVERY failed field across asset + all heads + reconstruction.
120
122
  const failed = [];
123
+ // ML5-A: real trained-head weights must load for mode A. When the gate is on
124
+ // and a trained-heads path is pinned, an unloadable/wrong-seed/malformed
125
+ // artifact is a qualification failure (any failed field demotes ALL of A).
126
+ if (ML5A_ENABLED() && candidate.trainedHeadsPath !== undefined) {
127
+ if (loadHeadProjections(candidate.trainedHeadsPath) === null) {
128
+ failed.push("head.weights.trainedHeadsPath");
129
+ }
130
+ }
121
131
  assetPasses(candidate.asset, failed);
122
132
  const heads = ["semantic", "dependency", "contradiction", "cacheStability", "payloadRouting"];
123
133
  for (const h of heads) {
@@ -0,0 +1,104 @@
1
+ /**
2
+ * dedup-audit.ts — durable audit trail for dedup tier decisions
3
+ * (external-audit item #2).
4
+ *
5
+ * Before this module a tier decision existed only as the in-process `onTier`
6
+ * callback that paints the live UI; nothing survived the process, so an
7
+ * operator could not answer "which layer collapsed this region, onto what, at
8
+ * what similarity?" — the inputs needed to tune the thresholds in
9
+ * config/dedup.ts. Here each decision is appended to the repo's events.log as
10
+ * one structured JSON line (see `DedupAuditEvent` below).
11
+ *
12
+ * The event type and its append helper live HERE rather than in monitoring.ts:
13
+ * monitoring.ts already owns three concerns (decision events, the dashboard.json
14
+ * metrics snapshot, FP alerting) and sits close to its 300-line soft limit, so
15
+ * co-locating the shape with the only recorder that produces it keeps both files
16
+ * under the headroom gate. monitoring.ts re-exports both for callers (and the
17
+ * dashboard SSE tail) that treat it as the events.log barrel.
18
+ *
19
+ * Design constraints:
20
+ * - PURE INSTRUMENTATION. Nothing in this file may influence a dedup outcome.
21
+ * - Best-effort/non-fatal: `logDedupAudit` swallows IO errors, and the emitter
22
+ * itself is wrapped so a malformed field can never break add().
23
+ * - Honest fields only: a value is emitted only where the caller actually
24
+ * computed it. L0/L1 are hash/verify tiers and pass no `similarity`.
25
+ * - Signal, not chatter: callers emit on DECISIONS (a match, a scored
26
+ * candidate, the final outcome), never on every "scanning" transition.
27
+ * - Flag-gated by cfg.DEDUP_AUDIT (default ON; OFF writes nothing at all).
28
+ *
29
+ * PREVENT-PI-004: local filesystem append only, no network.
30
+ */
31
+ import { appendFileSync, mkdirSync } from "node:fs";
32
+ import { dirname } from "node:path";
33
+ import { defaultEventsPath } from "../monitoring.js";
34
+ /**
35
+ * Append one audit event to events.log (best-effort, never throws).
36
+ *
37
+ * Same append-one-JSON-line contract as monitoring.ts's logDecision — an
38
+ * unwritable path is swallowed so instrumentation can never break add().
39
+ */
40
+ export function logDedupAudit(path, ev) {
41
+ try {
42
+ mkdirSync(dirname(path), { recursive: true });
43
+ appendFileSync(path, `${JSON.stringify(ev)}\n`, "utf-8");
44
+ }
45
+ catch {
46
+ /* best-effort — never break the extension on a log failure */
47
+ }
48
+ }
49
+ /** Build a recorder bound to one add() cascade. */
50
+ export function dedupAuditRecorder(ctx, scope) {
51
+ const base = {
52
+ sessionId: scope.sessionId,
53
+ originalTokenEstimate: scope.originalTokenEstimate,
54
+ tokenEstimate: scope.tokenEstimate,
55
+ };
56
+ return {
57
+ deduped: (tier, matchedEntry, dedupReason, similarity) => emitDedupAudit(ctx, {
58
+ ...base,
59
+ tier,
60
+ status: "deduped",
61
+ matchedEntry,
62
+ dedupReason,
63
+ ...(similarity === undefined ? {} : { similarity }),
64
+ }),
65
+ passed: (tier, matchedEntry, similarity) => emitDedupAudit(ctx, {
66
+ ...base,
67
+ tier,
68
+ status: "passed",
69
+ matchedEntry,
70
+ similarity,
71
+ }),
72
+ stored: (storedEntry, dedupReason, tokenEstimate) => emitDedupAudit(ctx, {
73
+ ...base,
74
+ tier: "new",
75
+ status: "stored",
76
+ storedEntry,
77
+ dedupReason,
78
+ tokenEstimate,
79
+ }),
80
+ };
81
+ }
82
+ /**
83
+ * Append one dedup decision to events.log.
84
+ *
85
+ * Resolves the target path from the explicit `eventsPath` when a caller opted
86
+ * in (Sprint 14 monitoring / tests), otherwise from the store's own per-repo
87
+ * state dir — production never passes `eventsPath`, so defaulting is what makes
88
+ * the audit trail actually exist on a real device.
89
+ */
90
+ export function emitDedupAudit(ctx, input) {
91
+ if (!ctx.auditEnabled)
92
+ return;
93
+ try {
94
+ const path = ctx.eventsPath ?? defaultEventsPath(ctx.stateDir);
95
+ logDedupAudit(path, {
96
+ type: "dedup_audit",
97
+ ts: new Date().toISOString(),
98
+ ...input,
99
+ });
100
+ }
101
+ catch {
102
+ /* instrumentation must never break the add() path */
103
+ }
104
+ }
@@ -222,5 +222,11 @@ export const VECTOR_CORTEX_SETTINGS: SettingGroup = {
222
222
  "Dashboard per-turn prompt-cache visibility: surfaces the per-turn stable-prefix ratio trend (GET /api/prefix-stability) in the CacheTab PrefixStabilityCard. Reads aggregate ratios/counts from the local monitoring events log only — no payload bytes. OFF = byte-identical predecessor (PC-B-era): /api/prefix-stability returns 404 and the CacheTab omits the PrefixStabilityCard.",
223
223
  true,
224
224
  ),
225
+ boolDirect(
226
+ "MEGACOMPACT_ML5_A",
227
+ "ML5-A Five-Head Training Load",
228
+ "ML5-A real trained-head loading: loadHeadProjections (trained-heads-v1) feeds selectQualifiedEncoder (trainedHeadsPath atomic demotion) + loadCalibrationV1. ON (default) = a pinned trained-heads path must load for mode A. OFF = loaders return null and selection ignores trainedHeadsPath — byte-identical to the placeholder-weighted VC2C path.",
229
+ true,
230
+ ),
225
231
  ],
226
232
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-mega-compact",
3
- "version": "0.20.35",
3
+ "version": "0.20.38",
4
4
  "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
5
5
  "type": "module",
6
6
  "license": "BSD-3-Clause",
@@ -0,0 +1,30 @@
1
+ /**
2
+ * config/vector-cortex-ml5a.ts — ML5-A five-head training + calibration flag.
3
+ *
4
+ * Extracted from vector-cortex.ts so that file stays under the 300-line soft
5
+ * limit (soft-as-hard gate), exactly as vector-cortex-vc9a.ts..vector-cortex-vc9d.ts
6
+ * were. This is the ML5 training sprint flag. vector-cortex.ts re-exports the
7
+ * ENUM below and root src/config.ts re-exports it, so no consumer import path
8
+ * changes.
9
+ *
10
+ * The split is purely mechanical: ML5A_ENABLED is byte-identical in name,
11
+ * semantics, and default to the definition it replaces, and vector-cortex.ts
12
+ * re-exports it so every existing `from "./config/vector-cortex.js"` import
13
+ * keeps resolving unchanged.
14
+ *
15
+ * Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
16
+ */
17
+
18
+ import { sprintFlag } from "./vector-cortex-flag.js";
19
+
20
+ /**
21
+ * ML5-A — five-head training + calibrated onnx asset. Default ON.
22
+ * `MEGACOMPACT_ML5_A=0` disables and is byte-identical to the placeholder
23
+ * predecessor (VC2C-era): `calibrate.ts`/`heads.ts` keep serving the LCG fake
24
+ * projections and the placeholder `fitTemperature`/`fitThreshold`, mode B
25
+ * trigram continues serving, and no trained artifact is loaded (a fresh/no
26
+ * corpus also no-ops gracefully — asset_emitted:false, placeholder behavior,
27
+ * byte-identical). This flag MUST also be a dashboard SETTINGS toggle (visible
28
+ * in config UI, never in EXCLUDED_SETTINGS), mirroring VC4A..VC9D.
29
+ */
30
+ export const ML5A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_ML5_A");
@@ -0,0 +1,28 @@
1
+ /**
2
+ * config/vector-cortex-ml5b.ts — ML5-B production bench harness flag.
3
+ *
4
+ * Sibling extract mirroring vector-cortex-ml5a.ts, so vector-cortex.ts stays
5
+ * under its 300-line soft limit (soft-as-hard gate). This is the ONNX Runtime
6
+ * evaluation/benchmark sprint flag. vector-cortex.ts re-exports the ENUM below
7
+ * and root src/config.ts re-exports it, so no consumer import path changes.
8
+ *
9
+ * ML5-B introduces NO runtime code path: the bench harness and corpus export
10
+ * are developer/evidence tooling (scripts/) plus a consumer-facing TypeScript
11
+ * shell (src/vector-cortex/encoder/bench.ts) that only writes monitoring
12
+ * events. The flag records intent and scopes the sprint's evidence assets; it
13
+ * gates nothing at runtime today. There is no HTTP endpoint and no dashboard
14
+ * change, so there is no SETTINGS toggle and no EXCLUDED_SETTINGS interaction.
15
+ *
16
+ * Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
17
+ */
18
+
19
+ import { sprintFlag } from "./vector-cortex-flag.js";
20
+
21
+ /**
22
+ * ML5-B — production bench harness (ONNX Runtime eval). Default ON.
23
+ * `MEGACOMPACT_ML5_B=0` disables and is byte-identical to the ML5-A survivor:
24
+ * no bench endpoint exists and mode B continues to serve all clients exactly as
25
+ * before. The flag does not gate the harness itself — the harness is an on-demand
26
+ * developer tool with no runtime path.
27
+ */
28
+ export const ML5B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_ML5_B");
@@ -269,18 +269,18 @@ export const VC8A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8A");
269
269
  */
270
270
  export const VC8B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8B");
271
271
 
272
- // VC8C (canary selection + external Rust parity) extracted to
273
- // vector-cortex-vc8c.ts to keep this file under the 300-line soft limit.
274
- // Re-exported here so every existing `from "./config/vector-cortex.js"`
275
- // import keeps resolving unchanged.
272
+ // VC8C (canary selection + Rust parity) extracted to vector-cortex-vc8c.ts;
273
+ // re-exported so existing `./config/vector-cortex.js` imports keep resolving.
276
274
  export { VC8C_ENABLED } from "./vector-cortex-vc8c.js";
277
275
 
278
- // VC9A/VC9B/VC9C/VC9D split to vector-cortex-vc9{a,b,c,d}.ts to stay under the 300-line soft limit.
276
+ // VC9A/VC9B/VC9C/VC9D/PCC/ML5A split to sibling files to stay under the 300-line soft limit.
279
277
  export { VC9A_ENABLED } from "./vector-cortex-vc9a.js";
280
278
  export { VC9B_ENABLED } from "./vector-cortex-vc9b.js";
281
279
  export { VC9C_ENABLED } from "./vector-cortex-vc9c.js";
282
280
  export { VC9D_ENABLED } from "./vector-cortex-vc9d.js";
283
281
  export { PCC_ENABLED } from "./vector-cortex-pcc.js";
282
+ export { ML5A_ENABLED } from "./vector-cortex-ml5a.js";
283
+ export { ML5B_ENABLED } from "./vector-cortex-ml5b.js";
284
284
 
285
285
  // Breaker constants (TRIAD_RESILIENCE.md §breaker) extracted to vector-cortex-breakers.ts.
286
286
  export {
package/src/config.ts CHANGED
@@ -183,6 +183,8 @@ export {
183
183
  VC9C_ENABLED,
184
184
  VC9D_ENABLED,
185
185
  PCC_ENABLED,
186
+ ML5A_ENABLED,
187
+ ML5B_ENABLED,
186
188
  BREAKER_WINDOW_MS,
187
189
  BREAKER_MIN_ATTEMPTS,
188
190
  BREAKER_PERF_FAILURES,