pi-mega-compact 0.20.35 → 0.20.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/vector-cortex-ml5a.js +28 -0
- package/dist/config/vector-cortex-ml5b.js +26 -0
- package/dist/config/vector-cortex.js +5 -5
- package/dist/config.js +1 -1
- package/dist/extensions/dashboard-server/routes-rag-settings-vector-cortex.js +1 -0
- package/dist/monitoring.js +172 -0
- package/dist/src/config/vector-cortex-ml5a.js +28 -0
- package/dist/src/config/vector-cortex-ml5b.js +26 -0
- package/dist/src/config/vector-cortex.js +5 -5
- package/dist/src/config.js +1 -1
- package/dist/src/monitoring.js +19 -0
- package/dist/src/store/backfill.js +1 -0
- package/dist/src/vector-cortex/encoder/bench-export.js +13 -0
- package/dist/src/vector-cortex/encoder/bench.js +100 -0
- package/dist/src/vector-cortex/encoder/calibrate.js +55 -0
- package/dist/src/vector-cortex/encoder/heads.js +87 -0
- package/dist/src/vector-cortex/encoder/select.js +10 -0
- package/dist/vector-cortex/encoder/bench-export.js +13 -0
- package/dist/vector-cortex/encoder/bench.js +100 -0
- package/dist/vector-cortex/encoder/calibrate.js +55 -0
- package/dist/vector-cortex/encoder/heads.js +87 -0
- package/dist/vector-cortex/encoder/select.js +10 -0
- package/dist/vectorStore/dedup-audit.js +104 -0
- package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts +6 -0
- package/package.json +1 -1
- package/src/config/vector-cortex-ml5a.ts +30 -0
- package/src/config/vector-cortex-ml5b.ts +28 -0
- package/src/config/vector-cortex.ts +5 -5
- package/src/config.ts +2 -0
- package/src/monitoring.ts +24 -0
- package/src/store/backfill.ts +1 -0
- package/src/vector-cortex/encoder/bench-export.ts +65 -0
- package/src/vector-cortex/encoder/bench.ts +109 -0
- package/src/vector-cortex/encoder/calibrate.ts +49 -0
- package/src/vector-cortex/encoder/heads.ts +106 -0
- package/src/vector-cortex/encoder/select.ts +17 -0
|
@@ -19,6 +19,8 @@
|
|
|
19
19
|
*
|
|
20
20
|
* Pi-agnostic, zero network (PREVENT-PI-004), no `any` (PREVENT-011).
|
|
21
21
|
*/
|
|
22
|
+
import { readFileSync } from "node:fs";
|
|
23
|
+
import { ML5A_ENABLED } from "../../config/vector-cortex.js";
|
|
22
24
|
import { ENCODER_HEAD_DIMS, ENCODER_HEAD_ORDER, ENCODER_HEAD_LOSS_WEIGHTS, ENCODER_HEAD_LOSS_SUM, ENCODER_SEED, } from "./types.js";
|
|
23
25
|
import { createEncoderHeadsReporter, NOOP_VC2B_REPORTER, } from "./emit-vc2b.js";
|
|
24
26
|
/** The stable head index of a head name (its position in ENCODER_HEAD_ORDER). */
|
|
@@ -110,4 +112,89 @@ export function encodeVectorSet(tokens, options = {}) {
|
|
|
110
112
|
export function headLossWeights() {
|
|
111
113
|
return { ...ENCODER_HEAD_LOSS_WEIGHTS };
|
|
112
114
|
}
|
|
115
|
+
/** True when every head's output dim + weight length matches the contract. */
|
|
116
|
+
export function headsShapeValid(t) {
|
|
117
|
+
return ENCODER_HEAD_ORDER.every((h) => t.dims[h] === ENCODER_HEAD_DIMS[h] && t.weights[h].length === ENCODER_HEAD_DIMS[h] * t.trunkDim);
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Load a `trained-heads-v1` artifact into a `HeadProjectionTable`. Gated on
|
|
121
|
+
* MEGACOMPACT_ML5_A: flag-off, absent file, malformed JSON, wrong schema,
|
|
122
|
+
* wrong seed, or a shape mismatch each return null (non-fatal). Deterministic
|
|
123
|
+
* and local (PREVENT-PI-004).
|
|
124
|
+
*/
|
|
125
|
+
export function loadHeadProjections(path) {
|
|
126
|
+
if (!ML5A_ENABLED())
|
|
127
|
+
return null;
|
|
128
|
+
let raw;
|
|
129
|
+
try {
|
|
130
|
+
raw = readFileSync(path, "utf8");
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
let parsed;
|
|
136
|
+
try {
|
|
137
|
+
parsed = JSON.parse(raw);
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
const r = parsed;
|
|
143
|
+
if (!r || r["schema"] !== "trained-heads-v1")
|
|
144
|
+
return null;
|
|
145
|
+
if (r["seed"] !== ENCODER_SEED)
|
|
146
|
+
return null;
|
|
147
|
+
const dims = r["dims"];
|
|
148
|
+
const heads = r["heads"];
|
|
149
|
+
if (!dims || !heads)
|
|
150
|
+
return null;
|
|
151
|
+
const trunkDim = Number(r["trunkDim"] ?? 0);
|
|
152
|
+
if (!Number.isFinite(trunkDim) || trunkDim <= 0)
|
|
153
|
+
return null;
|
|
154
|
+
const weights = {};
|
|
155
|
+
const temperatures = {};
|
|
156
|
+
for (const h of ENCODER_HEAD_ORDER) {
|
|
157
|
+
const hd = heads[h];
|
|
158
|
+
if (!hd || typeof hd !== "object")
|
|
159
|
+
return null;
|
|
160
|
+
const w = hd["weights"];
|
|
161
|
+
if (!Array.isArray(w))
|
|
162
|
+
return null;
|
|
163
|
+
weights[h] = Float32Array.from(w);
|
|
164
|
+
if (Number(hd["dim"] ?? 0) !== ENCODER_HEAD_DIMS[h])
|
|
165
|
+
return null;
|
|
166
|
+
temperatures[h] = Number(hd["temperature"] ?? 1);
|
|
167
|
+
if (!Number.isFinite(dims[h]))
|
|
168
|
+
return null;
|
|
169
|
+
}
|
|
170
|
+
const table = {
|
|
171
|
+
schema: "trained-heads-v1",
|
|
172
|
+
seed: Number(r["seed"]),
|
|
173
|
+
trunkDim,
|
|
174
|
+
dims: { semantic: 384, dependency: 128, contradiction: 128, cacheStability: 64, payloadRouting: 32 },
|
|
175
|
+
weights: weights,
|
|
176
|
+
temperatures: temperatures,
|
|
177
|
+
};
|
|
178
|
+
if (!headsShapeValid(table))
|
|
179
|
+
return null;
|
|
180
|
+
return table;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Project a trunk embedding through a trained head's real weights, applying the
|
|
184
|
+
* row-major matrix then L2-normalizing (all-zero on zero norm). Returns a
|
|
185
|
+
* `HeadVector` of the head's normative dimension.
|
|
186
|
+
*/
|
|
187
|
+
export function projectHeadFromTrunk(head, trunk, table) {
|
|
188
|
+
const dim = ENCODER_HEAD_DIMS[head];
|
|
189
|
+
const W = table.weights[head];
|
|
190
|
+
const t = table.trunkDim;
|
|
191
|
+
const out = new Float32Array(dim);
|
|
192
|
+
for (let i = 0; i < dim; i++) {
|
|
193
|
+
let acc = 0;
|
|
194
|
+
for (let j = 0; j < t; j++)
|
|
195
|
+
acc += W[i * t + j] * (trunk[j] ?? 0);
|
|
196
|
+
out[i] = acc;
|
|
197
|
+
}
|
|
198
|
+
return { head, dim, values: l2Normalize(out) };
|
|
199
|
+
}
|
|
113
200
|
export { ENCODER_HEAD_ORDER, ENCODER_HEAD_DIMS, ENCODER_HEAD_LOSS_SUM, ENCODER_SEED, NOOP_VC2B_REPORTER };
|
|
@@ -33,6 +33,8 @@
|
|
|
33
33
|
* `any` (PREVENT-011).
|
|
34
34
|
*/
|
|
35
35
|
import { createHash } from "node:crypto";
|
|
36
|
+
import { ML5A_ENABLED } from "../../config/vector-cortex.js";
|
|
37
|
+
import { loadHeadProjections } from "./heads.js";
|
|
36
38
|
import { ENC_QUALIFICATION_FAIL, EVALUATION_THRESHOLDS, } from "./types.js";
|
|
37
39
|
import { createEncoderQualificationReporter, } from "./emit-vc2c.js";
|
|
38
40
|
/** Canonical digest over a CalibrationV1's stable identity (split digest, heads,
|
|
@@ -118,6 +120,14 @@ export function selectQualifiedEncoder(candidate, options = {}) {
|
|
|
118
120
|
}
|
|
119
121
|
// Atomic: collect EVERY failed field across asset + all heads + reconstruction.
|
|
120
122
|
const failed = [];
|
|
123
|
+
// ML5-A: real trained-head weights must load for mode A. When the gate is on
|
|
124
|
+
// and a trained-heads path is pinned, an unloadable/wrong-seed/malformed
|
|
125
|
+
// artifact is a qualification failure (any failed field demotes ALL of A).
|
|
126
|
+
if (ML5A_ENABLED() && candidate.trainedHeadsPath !== undefined) {
|
|
127
|
+
if (loadHeadProjections(candidate.trainedHeadsPath) === null) {
|
|
128
|
+
failed.push("head.weights.trainedHeadsPath");
|
|
129
|
+
}
|
|
130
|
+
}
|
|
121
131
|
assetPasses(candidate.asset, failed);
|
|
122
132
|
const heads = ["semantic", "dependency", "contradiction", "cacheStability", "payloadRouting"];
|
|
123
133
|
for (const h of heads) {
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/bench-export.ts — ML5-B bench result contract.
|
|
3
|
+
*
|
|
4
|
+
* BenchResultV1 is the typed surface `bench.ts` parses from the qualification
|
|
5
|
+
* harness (`scripts/ml5/bench-onnx-prod.mjs`) and the dashboard / evidence
|
|
6
|
+
* tooling consume. It carries AGGREGATE measurements + a digest only — never
|
|
7
|
+
* chunk/message content (EVAL-REDACT-002).
|
|
8
|
+
*
|
|
9
|
+
* Contract-first (ENGINEERING_PRACTICES §3): this types file is the reviewed
|
|
10
|
+
* gate; implementations import from it. Pi-agnostic, dependency-free
|
|
11
|
+
* (PREVENT-PI-004 / PREVENT-011).
|
|
12
|
+
*/
|
|
13
|
+
export {};
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/bench.ts — ML5-B bench runner (consumer-facing shell).
|
|
3
|
+
*
|
|
4
|
+
* Calls `scripts/ml5/bench-onnx-prod.mjs` via child_process, parses the
|
|
5
|
+
* BenchResultV1 it emits, and writes the four `vector_cortex_encoder_bench_*`
|
|
6
|
+
* events to the monitoring events.log (the dashboard / ML5-D surface consume
|
|
7
|
+
* them later). This is NOT a runtime path — it is developer/evidence tooling.
|
|
8
|
+
*
|
|
9
|
+
* Events written (all best-effort / non-fatal):
|
|
10
|
+
* - vector_cortex_encoder_bench_p95_ms
|
|
11
|
+
* - vector_cortex_encoder_bench_rss_mib
|
|
12
|
+
* - vector_cortex_encoder_bench_opset_ok
|
|
13
|
+
* - vector_cortex_encoder_bench_deterministic
|
|
14
|
+
*
|
|
15
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 — the child bench is pure local
|
|
16
|
+
* computation). No `any` (PREVENT-011).
|
|
17
|
+
*/
|
|
18
|
+
import { spawnSync } from "node:child_process";
|
|
19
|
+
import { dirname, join } from "node:path";
|
|
20
|
+
import { fileURLToPath } from "node:url";
|
|
21
|
+
import { getStateDir } from "../../store.js";
|
|
22
|
+
import { defaultEventsPath, logBenchEvent } from "../../monitoring.js";
|
|
23
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
24
|
+
const REPO_ROOT = join(HERE, "..", "..", "..");
|
|
25
|
+
const BENCH_SCRIPT = join(REPO_ROOT, "scripts", "ml5", "bench-onnx-prod.mjs");
|
|
26
|
+
/** Default events.log beside the state dir (mirrors defaultEventsPath). */
|
|
27
|
+
function benchEventsPath(stateDir) {
|
|
28
|
+
return defaultEventsPath(stateDir);
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Run the ONNX bench once and record its four events. Returns the parsed
|
|
32
|
+
* BenchResultV1. On any failure (script missing, non-zero exit, unparsable
|
|
33
|
+
* output) it returns a degraded result with gates.all:false — never throws, so
|
|
34
|
+
* the caller's agent loop is never broken (non-fatal store/write contract).
|
|
35
|
+
*/
|
|
36
|
+
export function runBench(stateDir = getStateDir()) {
|
|
37
|
+
const noop = (error) => ({
|
|
38
|
+
timestamp: Date.now(),
|
|
39
|
+
platform: `${process.platform}-${process.arch}`,
|
|
40
|
+
encoderNative: false,
|
|
41
|
+
threads: 4,
|
|
42
|
+
tokens: 512,
|
|
43
|
+
corpusTokens: 0,
|
|
44
|
+
p95Ms: null,
|
|
45
|
+
rssMib: null,
|
|
46
|
+
rssBaselineMib: null,
|
|
47
|
+
rssMarginalMib: null,
|
|
48
|
+
opset: null,
|
|
49
|
+
deterministic: false,
|
|
50
|
+
digest: null,
|
|
51
|
+
gates: { latency: false, rss: false, opset: false, determinism: false, all: false },
|
|
52
|
+
error,
|
|
53
|
+
});
|
|
54
|
+
const fallback = (error) => {
|
|
55
|
+
const r = noop(error);
|
|
56
|
+
emitEvents(stateDir, r);
|
|
57
|
+
return r;
|
|
58
|
+
};
|
|
59
|
+
try {
|
|
60
|
+
const res = spawnSync(process.execPath, ["--expose-gc", BENCH_SCRIPT], {
|
|
61
|
+
cwd: REPO_ROOT,
|
|
62
|
+
encoding: "utf8",
|
|
63
|
+
timeout: 600_000,
|
|
64
|
+
});
|
|
65
|
+
const stdout = (res.stdout ?? "").trim();
|
|
66
|
+
if (res.status === null) {
|
|
67
|
+
return fallback("bench timed out or failed to spawn");
|
|
68
|
+
}
|
|
69
|
+
const parsed = JSON.parse(stdout || "");
|
|
70
|
+
if (!isBenchResultV1(parsed)) {
|
|
71
|
+
return fallback("bench output was not a BenchResultV1");
|
|
72
|
+
}
|
|
73
|
+
emitEvents(stateDir, parsed);
|
|
74
|
+
return parsed;
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
return fallback(`bench failed: ${e?.message ?? String(e)}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
function isBenchResultV1(v) {
|
|
81
|
+
if (typeof v !== "object" || v === null)
|
|
82
|
+
return false;
|
|
83
|
+
const o = v;
|
|
84
|
+
return (typeof o.timestamp === "number" &&
|
|
85
|
+
typeof o.platform === "string" &&
|
|
86
|
+
typeof o.encoderNative === "boolean" &&
|
|
87
|
+
typeof o.threads === "number" &&
|
|
88
|
+
typeof o.tokens === "number" &&
|
|
89
|
+
typeof o.corpusTokens === "number" &&
|
|
90
|
+
typeof o.gates === "object" && o.gates !== null &&
|
|
91
|
+
typeof o.gates.all === "boolean");
|
|
92
|
+
}
|
|
93
|
+
function emitEvents(stateDir, r) {
|
|
94
|
+
const path = benchEventsPath(stateDir);
|
|
95
|
+
const run = { platform: r.platform, encoderNative: r.encoderNative, threads: r.threads, tokens: r.tokens, digest: r.digest, corpusTokens: r.corpusTokens };
|
|
96
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_p95_ms", { ...run, p95Ms: r.p95Ms, pass: r.gates.latency });
|
|
97
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_rss_mib", { ...run, rssMib: r.rssMib, rssBaselineMib: r.rssBaselineMib, rssMarginalMib: r.rssMarginalMib, pass: r.gates.rss });
|
|
98
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_opset_ok", { ...run, opset: r.opset, pass: r.gates.opset });
|
|
99
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_deterministic", { ...run, deterministic: r.deterministic, pass: r.gates.determinism });
|
|
100
|
+
}
|
|
@@ -22,6 +22,8 @@
|
|
|
22
22
|
* (PREVENT-011).
|
|
23
23
|
*/
|
|
24
24
|
import { createHash } from "node:crypto";
|
|
25
|
+
import { readFileSync } from "node:fs";
|
|
26
|
+
import { ML5A_ENABLED } from "../../config/vector-cortex.js";
|
|
25
27
|
import { ENCODER_HEAD_ORDER, ENCODER_SEED, ENC_QUALIFICATION_FAIL, } from "./types.js";
|
|
26
28
|
/** Canonical digests of a sorted stable representation (order-invariant). */
|
|
27
29
|
function digestStrings(values) {
|
|
@@ -172,3 +174,56 @@ export function fitCalibration(examples, options = {}) {
|
|
|
172
174
|
};
|
|
173
175
|
return { ok: true, calibration };
|
|
174
176
|
}
|
|
177
|
+
/**
|
|
178
|
+
* Load a persisted `CalibrationV1` artifact (schema "calibration-v1") from disk.
|
|
179
|
+
* ML5-A: gated on MEGACOMPACT_ML5_A; flag-off, absent file, malformed JSON,
|
|
180
|
+
* wrong schema, non-canonical five-head order, or non-finite temp/threshold each
|
|
181
|
+
* return null (non-fatal, never throws). Deterministic, local (PREVENT-PI-004).
|
|
182
|
+
*/
|
|
183
|
+
export function loadCalibrationV1(path) {
|
|
184
|
+
if (!ML5A_ENABLED())
|
|
185
|
+
return null;
|
|
186
|
+
let raw;
|
|
187
|
+
try {
|
|
188
|
+
raw = readFileSync(path, "utf8");
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
let parsed;
|
|
194
|
+
try {
|
|
195
|
+
parsed = JSON.parse(raw);
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
const r = parsed;
|
|
201
|
+
if (!r || r["schema"] !== "calibration-v1")
|
|
202
|
+
return null;
|
|
203
|
+
const order = r["headOrder"];
|
|
204
|
+
if (!Array.isArray(order))
|
|
205
|
+
return null;
|
|
206
|
+
if (order.length !== ENCODER_HEAD_ORDER.length || !ENCODER_HEAD_ORDER.every((h, i) => order[i] === h)) {
|
|
207
|
+
return null;
|
|
208
|
+
}
|
|
209
|
+
const temperatures = r["temperatures"];
|
|
210
|
+
const thresholds = r["thresholds"];
|
|
211
|
+
const splitDigest = r["calibrationSplitDigest"];
|
|
212
|
+
if (!temperatures || !thresholds || typeof splitDigest !== "string" || splitDigest.length !== 64)
|
|
213
|
+
return null;
|
|
214
|
+
for (const h of ENCODER_HEAD_ORDER) {
|
|
215
|
+
const t = Number(temperatures[h]);
|
|
216
|
+
const th = Number(thresholds[h]);
|
|
217
|
+
if (!Number.isFinite(t) || !Number.isFinite(th))
|
|
218
|
+
return null;
|
|
219
|
+
}
|
|
220
|
+
return {
|
|
221
|
+
schema: "calibration-v1",
|
|
222
|
+
headOrder: [...ENCODER_HEAD_ORDER],
|
|
223
|
+
calibrationSplitDigest: splitDigest,
|
|
224
|
+
fittedOnCalibrationOnly: true,
|
|
225
|
+
temperatures: { ...temperatures },
|
|
226
|
+
thresholds: { ...thresholds },
|
|
227
|
+
seed: Number(r["seed"] ?? ENCODER_SEED),
|
|
228
|
+
};
|
|
229
|
+
}
|
|
@@ -19,6 +19,8 @@
|
|
|
19
19
|
*
|
|
20
20
|
* Pi-agnostic, zero network (PREVENT-PI-004), no `any` (PREVENT-011).
|
|
21
21
|
*/
|
|
22
|
+
import { readFileSync } from "node:fs";
|
|
23
|
+
import { ML5A_ENABLED } from "../../config/vector-cortex.js";
|
|
22
24
|
import { ENCODER_HEAD_DIMS, ENCODER_HEAD_ORDER, ENCODER_HEAD_LOSS_WEIGHTS, ENCODER_HEAD_LOSS_SUM, ENCODER_SEED, } from "./types.js";
|
|
23
25
|
import { createEncoderHeadsReporter, NOOP_VC2B_REPORTER, } from "./emit-vc2b.js";
|
|
24
26
|
/** The stable head index of a head name (its position in ENCODER_HEAD_ORDER). */
|
|
@@ -110,4 +112,89 @@ export function encodeVectorSet(tokens, options = {}) {
|
|
|
110
112
|
export function headLossWeights() {
|
|
111
113
|
return { ...ENCODER_HEAD_LOSS_WEIGHTS };
|
|
112
114
|
}
|
|
115
|
+
/** True when every head's output dim + weight length matches the contract. */
|
|
116
|
+
export function headsShapeValid(t) {
|
|
117
|
+
return ENCODER_HEAD_ORDER.every((h) => t.dims[h] === ENCODER_HEAD_DIMS[h] && t.weights[h].length === ENCODER_HEAD_DIMS[h] * t.trunkDim);
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Load a `trained-heads-v1` artifact into a `HeadProjectionTable`. Gated on
|
|
121
|
+
* MEGACOMPACT_ML5_A: flag-off, absent file, malformed JSON, wrong schema,
|
|
122
|
+
* wrong seed, or a shape mismatch each return null (non-fatal). Deterministic
|
|
123
|
+
* and local (PREVENT-PI-004).
|
|
124
|
+
*/
|
|
125
|
+
export function loadHeadProjections(path) {
|
|
126
|
+
if (!ML5A_ENABLED())
|
|
127
|
+
return null;
|
|
128
|
+
let raw;
|
|
129
|
+
try {
|
|
130
|
+
raw = readFileSync(path, "utf8");
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
let parsed;
|
|
136
|
+
try {
|
|
137
|
+
parsed = JSON.parse(raw);
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
const r = parsed;
|
|
143
|
+
if (!r || r["schema"] !== "trained-heads-v1")
|
|
144
|
+
return null;
|
|
145
|
+
if (r["seed"] !== ENCODER_SEED)
|
|
146
|
+
return null;
|
|
147
|
+
const dims = r["dims"];
|
|
148
|
+
const heads = r["heads"];
|
|
149
|
+
if (!dims || !heads)
|
|
150
|
+
return null;
|
|
151
|
+
const trunkDim = Number(r["trunkDim"] ?? 0);
|
|
152
|
+
if (!Number.isFinite(trunkDim) || trunkDim <= 0)
|
|
153
|
+
return null;
|
|
154
|
+
const weights = {};
|
|
155
|
+
const temperatures = {};
|
|
156
|
+
for (const h of ENCODER_HEAD_ORDER) {
|
|
157
|
+
const hd = heads[h];
|
|
158
|
+
if (!hd || typeof hd !== "object")
|
|
159
|
+
return null;
|
|
160
|
+
const w = hd["weights"];
|
|
161
|
+
if (!Array.isArray(w))
|
|
162
|
+
return null;
|
|
163
|
+
weights[h] = Float32Array.from(w);
|
|
164
|
+
if (Number(hd["dim"] ?? 0) !== ENCODER_HEAD_DIMS[h])
|
|
165
|
+
return null;
|
|
166
|
+
temperatures[h] = Number(hd["temperature"] ?? 1);
|
|
167
|
+
if (!Number.isFinite(dims[h]))
|
|
168
|
+
return null;
|
|
169
|
+
}
|
|
170
|
+
const table = {
|
|
171
|
+
schema: "trained-heads-v1",
|
|
172
|
+
seed: Number(r["seed"]),
|
|
173
|
+
trunkDim,
|
|
174
|
+
dims: { semantic: 384, dependency: 128, contradiction: 128, cacheStability: 64, payloadRouting: 32 },
|
|
175
|
+
weights: weights,
|
|
176
|
+
temperatures: temperatures,
|
|
177
|
+
};
|
|
178
|
+
if (!headsShapeValid(table))
|
|
179
|
+
return null;
|
|
180
|
+
return table;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Project a trunk embedding through a trained head's real weights, applying the
|
|
184
|
+
* row-major matrix then L2-normalizing (all-zero on zero norm). Returns a
|
|
185
|
+
* `HeadVector` of the head's normative dimension.
|
|
186
|
+
*/
|
|
187
|
+
export function projectHeadFromTrunk(head, trunk, table) {
|
|
188
|
+
const dim = ENCODER_HEAD_DIMS[head];
|
|
189
|
+
const W = table.weights[head];
|
|
190
|
+
const t = table.trunkDim;
|
|
191
|
+
const out = new Float32Array(dim);
|
|
192
|
+
for (let i = 0; i < dim; i++) {
|
|
193
|
+
let acc = 0;
|
|
194
|
+
for (let j = 0; j < t; j++)
|
|
195
|
+
acc += W[i * t + j] * (trunk[j] ?? 0);
|
|
196
|
+
out[i] = acc;
|
|
197
|
+
}
|
|
198
|
+
return { head, dim, values: l2Normalize(out) };
|
|
199
|
+
}
|
|
113
200
|
export { ENCODER_HEAD_ORDER, ENCODER_HEAD_DIMS, ENCODER_HEAD_LOSS_SUM, ENCODER_SEED, NOOP_VC2B_REPORTER };
|
|
@@ -33,6 +33,8 @@
|
|
|
33
33
|
* `any` (PREVENT-011).
|
|
34
34
|
*/
|
|
35
35
|
import { createHash } from "node:crypto";
|
|
36
|
+
import { ML5A_ENABLED } from "../../config/vector-cortex.js";
|
|
37
|
+
import { loadHeadProjections } from "./heads.js";
|
|
36
38
|
import { ENC_QUALIFICATION_FAIL, EVALUATION_THRESHOLDS, } from "./types.js";
|
|
37
39
|
import { createEncoderQualificationReporter, } from "./emit-vc2c.js";
|
|
38
40
|
/** Canonical digest over a CalibrationV1's stable identity (split digest, heads,
|
|
@@ -118,6 +120,14 @@ export function selectQualifiedEncoder(candidate, options = {}) {
|
|
|
118
120
|
}
|
|
119
121
|
// Atomic: collect EVERY failed field across asset + all heads + reconstruction.
|
|
120
122
|
const failed = [];
|
|
123
|
+
// ML5-A: real trained-head weights must load for mode A. When the gate is on
|
|
124
|
+
// and a trained-heads path is pinned, an unloadable/wrong-seed/malformed
|
|
125
|
+
// artifact is a qualification failure (any failed field demotes ALL of A).
|
|
126
|
+
if (ML5A_ENABLED() && candidate.trainedHeadsPath !== undefined) {
|
|
127
|
+
if (loadHeadProjections(candidate.trainedHeadsPath) === null) {
|
|
128
|
+
failed.push("head.weights.trainedHeadsPath");
|
|
129
|
+
}
|
|
130
|
+
}
|
|
121
131
|
assetPasses(candidate.asset, failed);
|
|
122
132
|
const heads = ["semantic", "dependency", "contradiction", "cacheStability", "payloadRouting"];
|
|
123
133
|
for (const h of heads) {
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dedup-audit.ts — durable audit trail for dedup tier decisions
|
|
3
|
+
* (external-audit item #2).
|
|
4
|
+
*
|
|
5
|
+
* Before this module a tier decision existed only as the in-process `onTier`
|
|
6
|
+
* callback that paints the live UI; nothing survived the process, so an
|
|
7
|
+
* operator could not answer "which layer collapsed this region, onto what, at
|
|
8
|
+
* what similarity?" — the inputs needed to tune the thresholds in
|
|
9
|
+
* config/dedup.ts. Here each decision is appended to the repo's events.log as
|
|
10
|
+
* one structured JSON line (see `DedupAuditEvent` below).
|
|
11
|
+
*
|
|
12
|
+
* The event type and its append helper live HERE rather than in monitoring.ts:
|
|
13
|
+
* monitoring.ts already owns three concerns (decision events, the dashboard.json
|
|
14
|
+
* metrics snapshot, FP alerting) and sits close to its 300-line soft limit, so
|
|
15
|
+
* co-locating the shape with the only recorder that produces it keeps both files
|
|
16
|
+
* under the headroom gate. monitoring.ts re-exports both for callers (and the
|
|
17
|
+
* dashboard SSE tail) that treat it as the events.log barrel.
|
|
18
|
+
*
|
|
19
|
+
* Design constraints:
|
|
20
|
+
* - PURE INSTRUMENTATION. Nothing in this file may influence a dedup outcome.
|
|
21
|
+
* - Best-effort/non-fatal: `logDedupAudit` swallows IO errors, and the emitter
|
|
22
|
+
* itself is wrapped so a malformed field can never break add().
|
|
23
|
+
* - Honest fields only: a value is emitted only where the caller actually
|
|
24
|
+
* computed it. L0/L1 are hash/verify tiers and pass no `similarity`.
|
|
25
|
+
* - Signal, not chatter: callers emit on DECISIONS (a match, a scored
|
|
26
|
+
* candidate, the final outcome), never on every "scanning" transition.
|
|
27
|
+
* - Flag-gated by cfg.DEDUP_AUDIT (default ON; OFF writes nothing at all).
|
|
28
|
+
*
|
|
29
|
+
* PREVENT-PI-004: local filesystem append only, no network.
|
|
30
|
+
*/
|
|
31
|
+
import { appendFileSync, mkdirSync } from "node:fs";
|
|
32
|
+
import { dirname } from "node:path";
|
|
33
|
+
import { defaultEventsPath } from "../monitoring.js";
|
|
34
|
+
/**
|
|
35
|
+
* Append one audit event to events.log (best-effort, never throws).
|
|
36
|
+
*
|
|
37
|
+
* Same append-one-JSON-line contract as monitoring.ts's logDecision — an
|
|
38
|
+
* unwritable path is swallowed so instrumentation can never break add().
|
|
39
|
+
*/
|
|
40
|
+
export function logDedupAudit(path, ev) {
|
|
41
|
+
try {
|
|
42
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
43
|
+
appendFileSync(path, `${JSON.stringify(ev)}\n`, "utf-8");
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
/* best-effort — never break the extension on a log failure */
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/** Build a recorder bound to one add() cascade. */
|
|
50
|
+
export function dedupAuditRecorder(ctx, scope) {
|
|
51
|
+
const base = {
|
|
52
|
+
sessionId: scope.sessionId,
|
|
53
|
+
originalTokenEstimate: scope.originalTokenEstimate,
|
|
54
|
+
tokenEstimate: scope.tokenEstimate,
|
|
55
|
+
};
|
|
56
|
+
return {
|
|
57
|
+
deduped: (tier, matchedEntry, dedupReason, similarity) => emitDedupAudit(ctx, {
|
|
58
|
+
...base,
|
|
59
|
+
tier,
|
|
60
|
+
status: "deduped",
|
|
61
|
+
matchedEntry,
|
|
62
|
+
dedupReason,
|
|
63
|
+
...(similarity === undefined ? {} : { similarity }),
|
|
64
|
+
}),
|
|
65
|
+
passed: (tier, matchedEntry, similarity) => emitDedupAudit(ctx, {
|
|
66
|
+
...base,
|
|
67
|
+
tier,
|
|
68
|
+
status: "passed",
|
|
69
|
+
matchedEntry,
|
|
70
|
+
similarity,
|
|
71
|
+
}),
|
|
72
|
+
stored: (storedEntry, dedupReason, tokenEstimate) => emitDedupAudit(ctx, {
|
|
73
|
+
...base,
|
|
74
|
+
tier: "new",
|
|
75
|
+
status: "stored",
|
|
76
|
+
storedEntry,
|
|
77
|
+
dedupReason,
|
|
78
|
+
tokenEstimate,
|
|
79
|
+
}),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Append one dedup decision to events.log.
|
|
84
|
+
*
|
|
85
|
+
* Resolves the target path from the explicit `eventsPath` when a caller opted
|
|
86
|
+
* in (Sprint 14 monitoring / tests), otherwise from the store's own per-repo
|
|
87
|
+
* state dir — production never passes `eventsPath`, so defaulting is what makes
|
|
88
|
+
* the audit trail actually exist on a real device.
|
|
89
|
+
*/
|
|
90
|
+
export function emitDedupAudit(ctx, input) {
|
|
91
|
+
if (!ctx.auditEnabled)
|
|
92
|
+
return;
|
|
93
|
+
try {
|
|
94
|
+
const path = ctx.eventsPath ?? defaultEventsPath(ctx.stateDir);
|
|
95
|
+
logDedupAudit(path, {
|
|
96
|
+
type: "dedup_audit",
|
|
97
|
+
ts: new Date().toISOString(),
|
|
98
|
+
...input,
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
/* instrumentation must never break the add() path */
|
|
103
|
+
}
|
|
104
|
+
}
|
|
@@ -222,5 +222,11 @@ export const VECTOR_CORTEX_SETTINGS: SettingGroup = {
|
|
|
222
222
|
"Dashboard per-turn prompt-cache visibility: surfaces the per-turn stable-prefix ratio trend (GET /api/prefix-stability) in the CacheTab PrefixStabilityCard. Reads aggregate ratios/counts from the local monitoring events log only — no payload bytes. OFF = byte-identical predecessor (PC-B-era): /api/prefix-stability returns 404 and the CacheTab omits the PrefixStabilityCard.",
|
|
223
223
|
true,
|
|
224
224
|
),
|
|
225
|
+
boolDirect(
|
|
226
|
+
"MEGACOMPACT_ML5_A",
|
|
227
|
+
"ML5-A Five-Head Training Load",
|
|
228
|
+
"ML5-A real trained-head loading: loadHeadProjections (trained-heads-v1) feeds selectQualifiedEncoder (trainedHeadsPath atomic demotion) + loadCalibrationV1. ON (default) = a pinned trained-heads path must load for mode A. OFF = loaders return null and selection ignores trainedHeadsPath — byte-identical to the placeholder-weighted VC2C path.",
|
|
229
|
+
true,
|
|
230
|
+
),
|
|
225
231
|
],
|
|
226
232
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.20.
|
|
3
|
+
"version": "0.20.38",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-3-Clause",
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* config/vector-cortex-ml5a.ts — ML5-A five-head training + calibration flag.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from vector-cortex.ts so that file stays under the 300-line soft
|
|
5
|
+
* limit (soft-as-hard gate), exactly as vector-cortex-vc9a.ts..vector-cortex-vc9d.ts
|
|
6
|
+
* were. This is the ML5 training sprint flag. vector-cortex.ts re-exports the
|
|
7
|
+
* ENUM below and root src/config.ts re-exports it, so no consumer import path
|
|
8
|
+
* changes.
|
|
9
|
+
*
|
|
10
|
+
* The split is purely mechanical: ML5A_ENABLED is byte-identical in name,
|
|
11
|
+
* semantics, and default to the definition it replaces, and vector-cortex.ts
|
|
12
|
+
* re-exports it so every existing `from "./config/vector-cortex.js"` import
|
|
13
|
+
* keeps resolving unchanged.
|
|
14
|
+
*
|
|
15
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { sprintFlag } from "./vector-cortex-flag.js";
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* ML5-A — five-head training + calibrated onnx asset. Default ON.
|
|
22
|
+
* `MEGACOMPACT_ML5_A=0` disables and is byte-identical to the placeholder
|
|
23
|
+
* predecessor (VC2C-era): `calibrate.ts`/`heads.ts` keep serving the LCG fake
|
|
24
|
+
* projections and the placeholder `fitTemperature`/`fitThreshold`, mode B
|
|
25
|
+
* trigram continues serving, and no trained artifact is loaded (a fresh/no
|
|
26
|
+
* corpus also no-ops gracefully — asset_emitted:false, placeholder behavior,
|
|
27
|
+
* byte-identical). This flag MUST also be a dashboard SETTINGS toggle (visible
|
|
28
|
+
* in config UI, never in EXCLUDED_SETTINGS), mirroring VC4A..VC9D.
|
|
29
|
+
*/
|
|
30
|
+
export const ML5A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_ML5_A");
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* config/vector-cortex-ml5b.ts — ML5-B production bench harness flag.
|
|
3
|
+
*
|
|
4
|
+
* Sibling extract mirroring vector-cortex-ml5a.ts, so vector-cortex.ts stays
|
|
5
|
+
* under its 300-line soft limit (soft-as-hard gate). This is the ONNX Runtime
|
|
6
|
+
* evaluation/benchmark sprint flag. vector-cortex.ts re-exports the ENUM below
|
|
7
|
+
* and root src/config.ts re-exports it, so no consumer import path changes.
|
|
8
|
+
*
|
|
9
|
+
* ML5-B introduces NO runtime code path: the bench harness and corpus export
|
|
10
|
+
* are developer/evidence tooling (scripts/) plus a consumer-facing TypeScript
|
|
11
|
+
* shell (src/vector-cortex/encoder/bench.ts) that only writes monitoring
|
|
12
|
+
* events. The flag records intent and scopes the sprint's evidence assets; it
|
|
13
|
+
* gates nothing at runtime today. There is no HTTP endpoint and no dashboard
|
|
14
|
+
* change, so there is no SETTINGS toggle and no EXCLUDED_SETTINGS interaction.
|
|
15
|
+
*
|
|
16
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { sprintFlag } from "./vector-cortex-flag.js";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* ML5-B — production bench harness (ONNX Runtime eval). Default ON.
|
|
23
|
+
* `MEGACOMPACT_ML5_B=0` disables and is byte-identical to the ML5-A survivor:
|
|
24
|
+
* no bench endpoint exists and mode B continues to serve all clients exactly as
|
|
25
|
+
* before. The flag does not gate the harness itself — the harness is an on-demand
|
|
26
|
+
* developer tool with no runtime path.
|
|
27
|
+
*/
|
|
28
|
+
export const ML5B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_ML5_B");
|
|
@@ -269,18 +269,18 @@ export const VC8A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8A");
|
|
|
269
269
|
*/
|
|
270
270
|
export const VC8B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8B");
|
|
271
271
|
|
|
272
|
-
// VC8C (canary selection +
|
|
273
|
-
// vector-cortex
|
|
274
|
-
// Re-exported here so every existing `from "./config/vector-cortex.js"`
|
|
275
|
-
// import keeps resolving unchanged.
|
|
272
|
+
// VC8C (canary selection + Rust parity) extracted to vector-cortex-vc8c.ts;
|
|
273
|
+
// re-exported so existing `./config/vector-cortex.js` imports keep resolving.
|
|
276
274
|
export { VC8C_ENABLED } from "./vector-cortex-vc8c.js";
|
|
277
275
|
|
|
278
|
-
// VC9A/VC9B/VC9C/VC9D split to
|
|
276
|
+
// VC9A/VC9B/VC9C/VC9D/PCC/ML5A split to sibling files to stay under the 300-line soft limit.
|
|
279
277
|
export { VC9A_ENABLED } from "./vector-cortex-vc9a.js";
|
|
280
278
|
export { VC9B_ENABLED } from "./vector-cortex-vc9b.js";
|
|
281
279
|
export { VC9C_ENABLED } from "./vector-cortex-vc9c.js";
|
|
282
280
|
export { VC9D_ENABLED } from "./vector-cortex-vc9d.js";
|
|
283
281
|
export { PCC_ENABLED } from "./vector-cortex-pcc.js";
|
|
282
|
+
export { ML5A_ENABLED } from "./vector-cortex-ml5a.js";
|
|
283
|
+
export { ML5B_ENABLED } from "./vector-cortex-ml5b.js";
|
|
284
284
|
|
|
285
285
|
// Breaker constants (TRIAD_RESILIENCE.md §breaker) extracted to vector-cortex-breakers.ts.
|
|
286
286
|
export {
|