pi-mega-compact 0.20.36 → 0.20.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/vector-cortex-ml5b.js +26 -0
- package/dist/config/vector-cortex.js +1 -0
- package/dist/config.js +1 -1
- package/dist/monitoring.js +172 -0
- package/dist/src/config/vector-cortex-ml5b.js +26 -0
- package/dist/src/config/vector-cortex.js +1 -0
- package/dist/src/config.js +1 -1
- package/dist/src/monitoring.js +19 -0
- package/dist/src/store/backfill.js +1 -0
- package/dist/src/vector-cortex/encoder/bench-export.js +13 -0
- package/dist/src/vector-cortex/encoder/bench.js +100 -0
- package/dist/vector-cortex/encoder/bench-export.js +13 -0
- package/dist/vector-cortex/encoder/bench.js +100 -0
- package/dist/vectorStore/dedup-audit.js +104 -0
- package/package.json +1 -1
- package/src/config/vector-cortex-ml5b.ts +28 -0
- package/src/config/vector-cortex.ts +1 -0
- package/src/config.ts +1 -0
- package/src/monitoring.ts +24 -0
- package/src/store/backfill.ts +1 -0
- package/src/vector-cortex/encoder/bench-export.ts +65 -0
- package/src/vector-cortex/encoder/bench.ts +109 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* config/vector-cortex-ml5b.ts — ML5-B production bench harness flag.
|
|
3
|
+
*
|
|
4
|
+
* Sibling extract mirroring vector-cortex-ml5a.ts, so vector-cortex.ts stays
|
|
5
|
+
* under its 300-line soft limit (soft-as-hard gate). This is the ONNX Runtime
|
|
6
|
+
* evaluation/benchmark sprint flag. vector-cortex.ts re-exports the ENUM below
|
|
7
|
+
* and root src/config.ts re-exports it, so no consumer import path changes.
|
|
8
|
+
*
|
|
9
|
+
* ML5-B introduces NO runtime code path: the bench harness and corpus export
|
|
10
|
+
* are developer/evidence tooling (scripts/) plus a consumer-facing TypeScript
|
|
11
|
+
* shell (src/vector-cortex/encoder/bench.ts) that only writes monitoring
|
|
12
|
+
* events. The flag records intent and scopes the sprint's evidence assets; it
|
|
13
|
+
* gates nothing at runtime today. There is no HTTP endpoint and no dashboard
|
|
14
|
+
* change, so there is no SETTINGS toggle and no EXCLUDED_SETTINGS interaction.
|
|
15
|
+
*
|
|
16
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
|
|
17
|
+
*/
|
|
18
|
+
import { sprintFlag } from "./vector-cortex-flag.js";
|
|
19
|
+
/**
|
|
20
|
+
* ML5-B — production bench harness (ONNX Runtime eval). Default ON.
|
|
21
|
+
* `MEGACOMPACT_ML5_B=0` disables and is byte-identical to the ML5-A survivor:
|
|
22
|
+
* no bench endpoint exists and mode B continues to serve all clients exactly as
|
|
23
|
+
* before. The flag does not gate the harness itself — the harness is an on-demand
|
|
24
|
+
* developer tool with no runtime path.
|
|
25
|
+
*/
|
|
26
|
+
export const ML5B_ENABLED = () => sprintFlag("MEGACOMPACT_ML5_B");
|
|
@@ -249,5 +249,6 @@ export { VC9C_ENABLED } from "./vector-cortex-vc9c.js";
|
|
|
249
249
|
export { VC9D_ENABLED } from "./vector-cortex-vc9d.js";
|
|
250
250
|
export { PCC_ENABLED } from "./vector-cortex-pcc.js";
|
|
251
251
|
export { ML5A_ENABLED } from "./vector-cortex-ml5a.js";
|
|
252
|
+
export { ML5B_ENABLED } from "./vector-cortex-ml5b.js";
|
|
252
253
|
// Breaker constants (TRIAD_RESILIENCE.md §breaker) extracted to vector-cortex-breakers.ts.
|
|
253
254
|
export { BREAKER_WINDOW_MS, BREAKER_MIN_ATTEMPTS, BREAKER_PERF_FAILURES, BREAKER_PERF_FAILURE_RATE, BREAKER_CORRECTNESS_FAILURES, BREAKER_COOLDOWN_MS, BREAKER_PROBE_COUNT, BREAKER_RETRY_BASE_MS, BREAKER_RETRY_CAP_MS, BREAKER_RETRY_JITTER, BREAKER_HYSTERESIS_FAILURE_RATE, BREAKER_HYSTERESIS_BUDGET_P95_MS, BREAKER_MIN_HEALTHY_RESIDENCE_MS, } from "./vector-cortex-breakers.js";
|
package/dist/config.js
CHANGED
|
@@ -114,4 +114,4 @@ export const NEW_UI = () => ragEnabled("MEGACOMPACT_NEW_UI");
|
|
|
114
114
|
// default ON, `=0`/`_DISABLED` off. Re-exported from src/config/vector-cortex.ts
|
|
115
115
|
// so root consumers share one source of truth.
|
|
116
116
|
// ---------------------------------------------------------------------------
|
|
117
|
-
export { VC0A_ENABLED, VC0B_ENABLED, VC1A_ENABLED, VC0C_ENABLED, VC1B_ENABLED, VC1C_ENABLED, VC2A_ENABLED, VC2B_ENABLED, VC2C_ENABLED, VC3A_ENABLED, VC3B_ENABLED, VC3C_ENABLED, VC4A_ENABLED, VC4B_ENABLED, VC4C_ENABLED, VC5A_ENABLED, VC5B_ENABLED, VC5C_ENABLED, VC6A_ENABLED, VC6B_ENABLED, VC6C_ENABLED, VC7A_ENABLED, VC7B_ENABLED, VC7C_ENABLED, VC8A_ENABLED, VC8B_ENABLED, VC8C_ENABLED, VC9A_ENABLED, VC9B_ENABLED, VC9C_ENABLED, VC9D_ENABLED, PCC_ENABLED, ML5A_ENABLED, BREAKER_WINDOW_MS, BREAKER_MIN_ATTEMPTS, BREAKER_PERF_FAILURES, BREAKER_PERF_FAILURE_RATE, BREAKER_CORRECTNESS_FAILURES, BREAKER_COOLDOWN_MS, BREAKER_PROBE_COUNT, BREAKER_RETRY_BASE_MS, BREAKER_RETRY_CAP_MS, BREAKER_RETRY_JITTER, BREAKER_HYSTERESIS_FAILURE_RATE, BREAKER_HYSTERESIS_BUDGET_P95_MS, BREAKER_MIN_HEALTHY_RESIDENCE_MS, } from "./config/vector-cortex.js";
|
|
117
|
+
export { VC0A_ENABLED, VC0B_ENABLED, VC1A_ENABLED, VC0C_ENABLED, VC1B_ENABLED, VC1C_ENABLED, VC2A_ENABLED, VC2B_ENABLED, VC2C_ENABLED, VC3A_ENABLED, VC3B_ENABLED, VC3C_ENABLED, VC4A_ENABLED, VC4B_ENABLED, VC4C_ENABLED, VC5A_ENABLED, VC5B_ENABLED, VC5C_ENABLED, VC6A_ENABLED, VC6B_ENABLED, VC6C_ENABLED, VC7A_ENABLED, VC7B_ENABLED, VC7C_ENABLED, VC8A_ENABLED, VC8B_ENABLED, VC8C_ENABLED, VC9A_ENABLED, VC9B_ENABLED, VC9C_ENABLED, VC9D_ENABLED, PCC_ENABLED, ML5A_ENABLED, ML5B_ENABLED, BREAKER_WINDOW_MS, BREAKER_MIN_ATTEMPTS, BREAKER_PERF_FAILURES, BREAKER_PERF_FAILURE_RATE, BREAKER_CORRECTNESS_FAILURES, BREAKER_COOLDOWN_MS, BREAKER_PROBE_COUNT, BREAKER_RETRY_BASE_MS, BREAKER_RETRY_CAP_MS, BREAKER_RETRY_JITTER, BREAKER_HYSTERESIS_FAILURE_RATE, BREAKER_HYSTERESIS_BUDGET_P95_MS, BREAKER_MIN_HEALTHY_RESIDENCE_MS, } from "./config/vector-cortex.js";
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* monitoring.ts — local dedup monitoring + alerting (Sprint 14, Phase 7).
|
|
3
|
+
*
|
|
4
|
+
* Per-decision structured events go to `events.log` (append-only JSON).
|
|
5
|
+
* Aggregate metrics (hit rate, FP rate, per-tier p95 latency, storage) go to
|
|
6
|
+
* `dashboard.json` — the SAME local-only file the /dashboard UI reads. There is
|
|
7
|
+
* NO Prometheus port and NO network listener (PREVENT-PI-004). Alerting is local
|
|
8
|
+
* only: an FP-rate breach flips the tier to MARK_ONLY and writes a warning.
|
|
9
|
+
*
|
|
10
|
+
* Best-effort: logging/metrics never throw into the add()/search() path.
|
|
11
|
+
*/
|
|
12
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync, appendFileSync } from "node:fs";
|
|
13
|
+
import { dirname, join } from "node:path";
|
|
14
|
+
import { STATE_DIR_DEFAULT } from "./config.js";
|
|
15
|
+
const TIERS = ["L0", "L1", "L2", "RAPTOR"];
|
|
16
|
+
function emptyMetrics() {
|
|
17
|
+
const dec = {};
|
|
18
|
+
const dp = {};
|
|
19
|
+
const fp = {};
|
|
20
|
+
const lat = {};
|
|
21
|
+
for (const t of TIERS) {
|
|
22
|
+
dec[t] = 0;
|
|
23
|
+
dp[t] = 0;
|
|
24
|
+
fp[t] = 0;
|
|
25
|
+
lat[t] = [];
|
|
26
|
+
}
|
|
27
|
+
return { decisions: dec, deduped: dp, falsePositives: fp, latency: lat, storageBytes: 0 };
|
|
28
|
+
}
|
|
29
|
+
/** Append a structured decision event to events.log (best-effort). */
|
|
30
|
+
export function logDecision(path, ev) {
|
|
31
|
+
try {
|
|
32
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
33
|
+
appendFileSync(path, `${JSON.stringify(ev)}\n`);
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
/* never break the extension on a log failure */
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Load metrics from dashboard.json, or return a fresh empty snapshot.
|
|
41
|
+
* Kept simple + synchronous (no network).
|
|
42
|
+
*/
|
|
43
|
+
export function loadMetrics(path) {
|
|
44
|
+
try {
|
|
45
|
+
if (existsSync(path)) {
|
|
46
|
+
const raw = readFileSync(path, "utf-8");
|
|
47
|
+
const parsed = JSON.parse(raw);
|
|
48
|
+
const base = emptyMetrics();
|
|
49
|
+
return {
|
|
50
|
+
decisions: { ...base.decisions, ...(parsed.decisions ?? {}) },
|
|
51
|
+
deduped: { ...base.deduped, ...(parsed.deduped ?? {}) },
|
|
52
|
+
falsePositives: { ...base.falsePositives, ...(parsed.falsePositives ?? {}) },
|
|
53
|
+
latency: { ...base.latency, ...(parsed.latency ?? {}) },
|
|
54
|
+
storageBytes: parsed.storageBytes ?? 0,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
/* corrupt metrics → fresh */
|
|
60
|
+
}
|
|
61
|
+
return emptyMetrics();
|
|
62
|
+
}
|
|
63
|
+
/** Persist metrics to dashboard.json (best-effort). */
|
|
64
|
+
export function saveMetrics(path, m) {
|
|
65
|
+
try {
|
|
66
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
67
|
+
writeFileSync(path, JSON.stringify(m));
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
/* never break the extension */
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/** Compute the p95 latency (ms) for a tier from its samples. */
|
|
74
|
+
export function p95(samples) {
|
|
75
|
+
if (samples.length === 0)
|
|
76
|
+
return 0;
|
|
77
|
+
const sorted = [...samples].sort((a, b) => a - b);
|
|
78
|
+
const idx = Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95));
|
|
79
|
+
return sorted[idx];
|
|
80
|
+
}
|
|
81
|
+
/** FP rate for a tier over the current window (0..1). */
|
|
82
|
+
export function fpRate(m, tier) {
|
|
83
|
+
const decisions = m.decisions[tier] ?? 0;
|
|
84
|
+
if (decisions === 0)
|
|
85
|
+
return 0;
|
|
86
|
+
return (m.falsePositives[tier] ?? 0) / decisions;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Evaluate FP-rate breaches against the config thresholds. A breached fuzzy tier
|
|
90
|
+
* (L0 vs L1/L2 have different thresholds) is auto-downgraded to MARK_ONLY — the
|
|
91
|
+
* local re-map of "alertmanager" (QA #18/#19): record but don't collapse, no
|
|
92
|
+
* remote alert. Returns the tiers flipped so the caller can mutate its config.
|
|
93
|
+
*/
|
|
94
|
+
export function evaluateAlerts(m, cfg) {
|
|
95
|
+
const breached = [];
|
|
96
|
+
const warnings = [];
|
|
97
|
+
for (const tier of TIERS) {
|
|
98
|
+
const rate = fpRate(m, tier);
|
|
99
|
+
const limit = tier === "L0" ? cfg.FP_RATE_L0 : cfg.FP_RATE_L1L2;
|
|
100
|
+
if (rate > limit) {
|
|
101
|
+
breached.push(tier);
|
|
102
|
+
warnings.push(`DEDUP FP BREACH tier=${tier} rate=${rate.toFixed(4)} > ${limit}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return { breached, warnings };
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Record one decision into the metrics snapshot (mutates `m` in place) and
|
|
109
|
+
* returns the updated snapshot. Caps stored latency samples to keep memory
|
|
110
|
+
* bounded (last 1000 per tier).
|
|
111
|
+
*/
|
|
112
|
+
export function recordDecision(m, tier, result, latencyMs, falsePositive = false) {
|
|
113
|
+
m.decisions[tier] = (m.decisions[tier] ?? 0) + 1;
|
|
114
|
+
if (result === "deduped")
|
|
115
|
+
m.deduped[tier] = (m.deduped[tier] ?? 0) + 1;
|
|
116
|
+
if (falsePositive)
|
|
117
|
+
m.falsePositives[tier] = (m.falsePositives[tier] ?? 0) + 1;
|
|
118
|
+
const arr = m.latency[tier] ?? (m.latency[tier] = []);
|
|
119
|
+
arr.push(latencyMs);
|
|
120
|
+
if (arr.length > 1000)
|
|
121
|
+
arr.shift();
|
|
122
|
+
return m;
|
|
123
|
+
}
|
|
124
|
+
/** Default metrics path alongside the state dir. */
|
|
125
|
+
export function defaultMetricsPath(stateDir = STATE_DIR_DEFAULT) {
|
|
126
|
+
return join(stateDir, "dashboard.json");
|
|
127
|
+
}
|
|
128
|
+
/** Default events-log path alongside the state dir. */
|
|
129
|
+
export function defaultEventsPath(stateDir = STATE_DIR_DEFAULT) {
|
|
130
|
+
return join(stateDir, "events.log");
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Append a RecallQualityEvent to events.log (best-effort, non-fatal).
|
|
134
|
+
* Follows the same append-one-JSON-line pattern as logDecision.
|
|
135
|
+
*/
|
|
136
|
+
export function logRecallQuality(path, ev) {
|
|
137
|
+
try {
|
|
138
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
139
|
+
appendFileSync(path, JSON.stringify(ev) + "\n", "utf-8");
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
/* best-effort — never break the extension */
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
// ---------------------------------------------------------------------------
|
|
146
|
+
// ML5-B encoder bench events (consumer-facing for the dashboard + evidence)
|
|
147
|
+
// ---------------------------------------------------------------------------
|
|
148
|
+
/**
|
|
149
|
+
* Append a structured ML5-B bench event to events.log (best-effort, non-fatal).
|
|
150
|
+
* Mirrors the extension's appendEvent schema ({ ts, event, ...fields }) so the
|
|
151
|
+
* dashboard live-stream tail and evidence tooling parse the four
|
|
152
|
+
* `vector_cortex_encoder_bench_*` events identically. The bench is developer/
|
|
153
|
+
* evidence tooling with no runtime gating; this only records its results.
|
|
154
|
+
*/
|
|
155
|
+
export function logBenchEvent(path, event, fields) {
|
|
156
|
+
try {
|
|
157
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
158
|
+
appendFileSync(path, JSON.stringify({ ts: Date.now(), event, ...fields }) + "\n", "utf8");
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
/* best-effort — never break the caller */
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
// ---------------------------------------------------------------------------
|
|
165
|
+
// Dedup audit trail (external-audit item #2)
|
|
166
|
+
// ---------------------------------------------------------------------------
|
|
167
|
+
// The event shape and its append helper are DEFINED in vectorStore/dedup-audit.ts,
|
|
168
|
+
// co-located with the only recorder that emits them (this file already carries
|
|
169
|
+
// decision events + the metrics snapshot + FP alerting and must stay under its
|
|
170
|
+
// 300-line soft limit). Re-exported here so callers that treat monitoring.ts as
|
|
171
|
+
// the events.log barrel — including the dashboard SSE tail — keep one import.
|
|
172
|
+
export { logDedupAudit } from "./vectorStore/dedup-audit.js";
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* config/vector-cortex-ml5b.ts — ML5-B production bench harness flag.
|
|
3
|
+
*
|
|
4
|
+
* Sibling extract mirroring vector-cortex-ml5a.ts, so vector-cortex.ts stays
|
|
5
|
+
* under its 300-line soft limit (soft-as-hard gate). This is the ONNX Runtime
|
|
6
|
+
* evaluation/benchmark sprint flag. vector-cortex.ts re-exports the ENUM below
|
|
7
|
+
* and root src/config.ts re-exports it, so no consumer import path changes.
|
|
8
|
+
*
|
|
9
|
+
* ML5-B introduces NO runtime code path: the bench harness and corpus export
|
|
10
|
+
* are developer/evidence tooling (scripts/) plus a consumer-facing TypeScript
|
|
11
|
+
* shell (src/vector-cortex/encoder/bench.ts) that only writes monitoring
|
|
12
|
+
* events. The flag records intent and scopes the sprint's evidence assets; it
|
|
13
|
+
* gates nothing at runtime today. There is no HTTP endpoint and no dashboard
|
|
14
|
+
* change, so there is no SETTINGS toggle and no EXCLUDED_SETTINGS interaction.
|
|
15
|
+
*
|
|
16
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
|
|
17
|
+
*/
|
|
18
|
+
import { sprintFlag } from "./vector-cortex-flag.js";
|
|
19
|
+
/**
|
|
20
|
+
* ML5-B — production bench harness (ONNX Runtime eval). Default ON.
|
|
21
|
+
* `MEGACOMPACT_ML5_B=0` disables and is byte-identical to the ML5-A survivor:
|
|
22
|
+
* no bench endpoint exists and mode B continues to serve all clients exactly as
|
|
23
|
+
* before. The flag does not gate the harness itself — the harness is an on-demand
|
|
24
|
+
* developer tool with no runtime path.
|
|
25
|
+
*/
|
|
26
|
+
export const ML5B_ENABLED = () => sprintFlag("MEGACOMPACT_ML5_B");
|
|
@@ -249,5 +249,6 @@ export { VC9C_ENABLED } from "./vector-cortex-vc9c.js";
|
|
|
249
249
|
export { VC9D_ENABLED } from "./vector-cortex-vc9d.js";
|
|
250
250
|
export { PCC_ENABLED } from "./vector-cortex-pcc.js";
|
|
251
251
|
export { ML5A_ENABLED } from "./vector-cortex-ml5a.js";
|
|
252
|
+
export { ML5B_ENABLED } from "./vector-cortex-ml5b.js";
|
|
252
253
|
// Breaker constants (TRIAD_RESILIENCE.md §breaker) extracted to vector-cortex-breakers.ts.
|
|
253
254
|
export { BREAKER_WINDOW_MS, BREAKER_MIN_ATTEMPTS, BREAKER_PERF_FAILURES, BREAKER_PERF_FAILURE_RATE, BREAKER_CORRECTNESS_FAILURES, BREAKER_COOLDOWN_MS, BREAKER_PROBE_COUNT, BREAKER_RETRY_BASE_MS, BREAKER_RETRY_CAP_MS, BREAKER_RETRY_JITTER, BREAKER_HYSTERESIS_FAILURE_RATE, BREAKER_HYSTERESIS_BUDGET_P95_MS, BREAKER_MIN_HEALTHY_RESIDENCE_MS, } from "./vector-cortex-breakers.js";
|
package/dist/src/config.js
CHANGED
|
@@ -114,4 +114,4 @@ export const NEW_UI = () => ragEnabled("MEGACOMPACT_NEW_UI");
|
|
|
114
114
|
// default ON, `=0`/`_DISABLED` off. Re-exported from src/config/vector-cortex.ts
|
|
115
115
|
// so root consumers share one source of truth.
|
|
116
116
|
// ---------------------------------------------------------------------------
|
|
117
|
-
export { VC0A_ENABLED, VC0B_ENABLED, VC1A_ENABLED, VC0C_ENABLED, VC1B_ENABLED, VC1C_ENABLED, VC2A_ENABLED, VC2B_ENABLED, VC2C_ENABLED, VC3A_ENABLED, VC3B_ENABLED, VC3C_ENABLED, VC4A_ENABLED, VC4B_ENABLED, VC4C_ENABLED, VC5A_ENABLED, VC5B_ENABLED, VC5C_ENABLED, VC6A_ENABLED, VC6B_ENABLED, VC6C_ENABLED, VC7A_ENABLED, VC7B_ENABLED, VC7C_ENABLED, VC8A_ENABLED, VC8B_ENABLED, VC8C_ENABLED, VC9A_ENABLED, VC9B_ENABLED, VC9C_ENABLED, VC9D_ENABLED, PCC_ENABLED, ML5A_ENABLED, BREAKER_WINDOW_MS, BREAKER_MIN_ATTEMPTS, BREAKER_PERF_FAILURES, BREAKER_PERF_FAILURE_RATE, BREAKER_CORRECTNESS_FAILURES, BREAKER_COOLDOWN_MS, BREAKER_PROBE_COUNT, BREAKER_RETRY_BASE_MS, BREAKER_RETRY_CAP_MS, BREAKER_RETRY_JITTER, BREAKER_HYSTERESIS_FAILURE_RATE, BREAKER_HYSTERESIS_BUDGET_P95_MS, BREAKER_MIN_HEALTHY_RESIDENCE_MS, } from "./config/vector-cortex.js";
|
|
117
|
+
export { VC0A_ENABLED, VC0B_ENABLED, VC1A_ENABLED, VC0C_ENABLED, VC1B_ENABLED, VC1C_ENABLED, VC2A_ENABLED, VC2B_ENABLED, VC2C_ENABLED, VC3A_ENABLED, VC3B_ENABLED, VC3C_ENABLED, VC4A_ENABLED, VC4B_ENABLED, VC4C_ENABLED, VC5A_ENABLED, VC5B_ENABLED, VC5C_ENABLED, VC6A_ENABLED, VC6B_ENABLED, VC6C_ENABLED, VC7A_ENABLED, VC7B_ENABLED, VC7C_ENABLED, VC8A_ENABLED, VC8B_ENABLED, VC8C_ENABLED, VC9A_ENABLED, VC9B_ENABLED, VC9C_ENABLED, VC9D_ENABLED, PCC_ENABLED, ML5A_ENABLED, ML5B_ENABLED, BREAKER_WINDOW_MS, BREAKER_MIN_ATTEMPTS, BREAKER_PERF_FAILURES, BREAKER_PERF_FAILURE_RATE, BREAKER_CORRECTNESS_FAILURES, BREAKER_COOLDOWN_MS, BREAKER_PROBE_COUNT, BREAKER_RETRY_BASE_MS, BREAKER_RETRY_CAP_MS, BREAKER_RETRY_JITTER, BREAKER_HYSTERESIS_FAILURE_RATE, BREAKER_HYSTERESIS_BUDGET_P95_MS, BREAKER_MIN_HEALTHY_RESIDENCE_MS, } from "./config/vector-cortex.js";
|
package/dist/src/monitoring.js
CHANGED
|
@@ -143,6 +143,25 @@ export function logRecallQuality(path, ev) {
|
|
|
143
143
|
}
|
|
144
144
|
}
|
|
145
145
|
// ---------------------------------------------------------------------------
|
|
146
|
+
// ML5-B encoder bench events (consumer-facing for the dashboard + evidence)
|
|
147
|
+
// ---------------------------------------------------------------------------
|
|
148
|
+
/**
|
|
149
|
+
* Append a structured ML5-B bench event to events.log (best-effort, non-fatal).
|
|
150
|
+
* Mirrors the extension's appendEvent schema ({ ts, event, ...fields }) so the
|
|
151
|
+
* dashboard live-stream tail and evidence tooling parse the four
|
|
152
|
+
* `vector_cortex_encoder_bench_*` events identically. The bench is developer/
|
|
153
|
+
* evidence tooling with no runtime gating; this only records its results.
|
|
154
|
+
*/
|
|
155
|
+
export function logBenchEvent(path, event, fields) {
|
|
156
|
+
try {
|
|
157
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
158
|
+
appendFileSync(path, JSON.stringify({ ts: Date.now(), event, ...fields }) + "\n", "utf8");
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
/* best-effort — never break the caller */
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
// ---------------------------------------------------------------------------
|
|
146
165
|
// Dedup audit trail (external-audit item #2)
|
|
147
166
|
// ---------------------------------------------------------------------------
|
|
148
167
|
// The event shape and its append helper are DEFINED in vectorStore/dedup-audit.ts,
|
|
@@ -94,6 +94,7 @@ export function backfillContentHashes(stateDir = getStateDir()) {
|
|
|
94
94
|
}
|
|
95
95
|
if (THROTTLE_MS > 0) {
|
|
96
96
|
// No-op in this synchronous build; placeholder for future streaming backfill.
|
|
97
|
+
// guardrails-allow PREVENT-STUB-001: ML5-C
|
|
97
98
|
}
|
|
98
99
|
return { processed, updated, duplicatesResolved };
|
|
99
100
|
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/bench-export.ts — ML5-B bench result contract.
|
|
3
|
+
*
|
|
4
|
+
* BenchResultV1 is the typed surface `bench.ts` parses from the qualification
|
|
5
|
+
* harness (`scripts/ml5/bench-onnx-prod.mjs`) and the dashboard / evidence
|
|
6
|
+
* tooling consume. It carries AGGREGATE measurements + a digest only — never
|
|
7
|
+
* chunk/message content (EVAL-REDACT-002).
|
|
8
|
+
*
|
|
9
|
+
* Contract-first (ENGINEERING_PRACTICES §3): this types file is the reviewed
|
|
10
|
+
* gate; implementations import from it. Pi-agnostic, dependency-free
|
|
11
|
+
* (PREVENT-PI-004 / PREVENT-011).
|
|
12
|
+
*/
|
|
13
|
+
export {};
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/bench.ts — ML5-B bench runner (consumer-facing shell).
|
|
3
|
+
*
|
|
4
|
+
* Calls `scripts/ml5/bench-onnx-prod.mjs` via child_process, parses the
|
|
5
|
+
* BenchResultV1 it emits, and writes the four `vector_cortex_encoder_bench_*`
|
|
6
|
+
* events to the monitoring events.log (the dashboard / ML5-D surface consume
|
|
7
|
+
* them later). This is NOT a runtime path — it is developer/evidence tooling.
|
|
8
|
+
*
|
|
9
|
+
* Events written (all best-effort / non-fatal):
|
|
10
|
+
* - vector_cortex_encoder_bench_p95_ms
|
|
11
|
+
* - vector_cortex_encoder_bench_rss_mib
|
|
12
|
+
* - vector_cortex_encoder_bench_opset_ok
|
|
13
|
+
* - vector_cortex_encoder_bench_deterministic
|
|
14
|
+
*
|
|
15
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 — the child bench is pure local
|
|
16
|
+
* computation). No `any` (PREVENT-011).
|
|
17
|
+
*/
|
|
18
|
+
import { spawnSync } from "node:child_process";
|
|
19
|
+
import { dirname, join } from "node:path";
|
|
20
|
+
import { fileURLToPath } from "node:url";
|
|
21
|
+
import { getStateDir } from "../../store.js";
|
|
22
|
+
import { defaultEventsPath, logBenchEvent } from "../../monitoring.js";
|
|
23
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
24
|
+
const REPO_ROOT = join(HERE, "..", "..", "..");
|
|
25
|
+
const BENCH_SCRIPT = join(REPO_ROOT, "scripts", "ml5", "bench-onnx-prod.mjs");
|
|
26
|
+
/** Default events.log beside the state dir (mirrors defaultEventsPath). */
|
|
27
|
+
function benchEventsPath(stateDir) {
|
|
28
|
+
return defaultEventsPath(stateDir);
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Run the ONNX bench once and record its four events. Returns the parsed
|
|
32
|
+
* BenchResultV1. On any failure (script missing, non-zero exit, unparsable
|
|
33
|
+
* output) it returns a degraded result with gates.all:false — never throws, so
|
|
34
|
+
* the caller's agent loop is never broken (non-fatal store/write contract).
|
|
35
|
+
*/
|
|
36
|
+
export function runBench(stateDir = getStateDir()) {
|
|
37
|
+
const noop = (error) => ({
|
|
38
|
+
timestamp: Date.now(),
|
|
39
|
+
platform: `${process.platform}-${process.arch}`,
|
|
40
|
+
encoderNative: false,
|
|
41
|
+
threads: 4,
|
|
42
|
+
tokens: 512,
|
|
43
|
+
corpusTokens: 0,
|
|
44
|
+
p95Ms: null,
|
|
45
|
+
rssMib: null,
|
|
46
|
+
rssBaselineMib: null,
|
|
47
|
+
rssMarginalMib: null,
|
|
48
|
+
opset: null,
|
|
49
|
+
deterministic: false,
|
|
50
|
+
digest: null,
|
|
51
|
+
gates: { latency: false, rss: false, opset: false, determinism: false, all: false },
|
|
52
|
+
error,
|
|
53
|
+
});
|
|
54
|
+
const fallback = (error) => {
|
|
55
|
+
const r = noop(error);
|
|
56
|
+
emitEvents(stateDir, r);
|
|
57
|
+
return r;
|
|
58
|
+
};
|
|
59
|
+
try {
|
|
60
|
+
const res = spawnSync(process.execPath, ["--expose-gc", BENCH_SCRIPT], {
|
|
61
|
+
cwd: REPO_ROOT,
|
|
62
|
+
encoding: "utf8",
|
|
63
|
+
timeout: 600_000,
|
|
64
|
+
});
|
|
65
|
+
const stdout = (res.stdout ?? "").trim();
|
|
66
|
+
if (res.status === null) {
|
|
67
|
+
return fallback("bench timed out or failed to spawn");
|
|
68
|
+
}
|
|
69
|
+
const parsed = JSON.parse(stdout || "");
|
|
70
|
+
if (!isBenchResultV1(parsed)) {
|
|
71
|
+
return fallback("bench output was not a BenchResultV1");
|
|
72
|
+
}
|
|
73
|
+
emitEvents(stateDir, parsed);
|
|
74
|
+
return parsed;
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
return fallback(`bench failed: ${e?.message ?? String(e)}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
function isBenchResultV1(v) {
|
|
81
|
+
if (typeof v !== "object" || v === null)
|
|
82
|
+
return false;
|
|
83
|
+
const o = v;
|
|
84
|
+
return (typeof o.timestamp === "number" &&
|
|
85
|
+
typeof o.platform === "string" &&
|
|
86
|
+
typeof o.encoderNative === "boolean" &&
|
|
87
|
+
typeof o.threads === "number" &&
|
|
88
|
+
typeof o.tokens === "number" &&
|
|
89
|
+
typeof o.corpusTokens === "number" &&
|
|
90
|
+
typeof o.gates === "object" && o.gates !== null &&
|
|
91
|
+
typeof o.gates.all === "boolean");
|
|
92
|
+
}
|
|
93
|
+
function emitEvents(stateDir, r) {
|
|
94
|
+
const path = benchEventsPath(stateDir);
|
|
95
|
+
const run = { platform: r.platform, encoderNative: r.encoderNative, threads: r.threads, tokens: r.tokens, digest: r.digest, corpusTokens: r.corpusTokens };
|
|
96
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_p95_ms", { ...run, p95Ms: r.p95Ms, pass: r.gates.latency });
|
|
97
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_rss_mib", { ...run, rssMib: r.rssMib, rssBaselineMib: r.rssBaselineMib, rssMarginalMib: r.rssMarginalMib, pass: r.gates.rss });
|
|
98
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_opset_ok", { ...run, opset: r.opset, pass: r.gates.opset });
|
|
99
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_deterministic", { ...run, deterministic: r.deterministic, pass: r.gates.determinism });
|
|
100
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/bench-export.ts — ML5-B bench result contract.
|
|
3
|
+
*
|
|
4
|
+
* BenchResultV1 is the typed surface `bench.ts` parses from the qualification
|
|
5
|
+
* harness (`scripts/ml5/bench-onnx-prod.mjs`) and the dashboard / evidence
|
|
6
|
+
* tooling consume. It carries AGGREGATE measurements + a digest only — never
|
|
7
|
+
* chunk/message content (EVAL-REDACT-002).
|
|
8
|
+
*
|
|
9
|
+
* Contract-first (ENGINEERING_PRACTICES §3): this types file is the reviewed
|
|
10
|
+
* gate; implementations import from it. Pi-agnostic, dependency-free
|
|
11
|
+
* (PREVENT-PI-004 / PREVENT-011).
|
|
12
|
+
*/
|
|
13
|
+
export {};
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/bench.ts — ML5-B bench runner (consumer-facing shell).
|
|
3
|
+
*
|
|
4
|
+
* Calls `scripts/ml5/bench-onnx-prod.mjs` via child_process, parses the
|
|
5
|
+
* BenchResultV1 it emits, and writes the four `vector_cortex_encoder_bench_*`
|
|
6
|
+
* events to the monitoring events.log (the dashboard / ML5-D surface consume
|
|
7
|
+
* them later). This is NOT a runtime path — it is developer/evidence tooling.
|
|
8
|
+
*
|
|
9
|
+
* Events written (all best-effort / non-fatal):
|
|
10
|
+
* - vector_cortex_encoder_bench_p95_ms
|
|
11
|
+
* - vector_cortex_encoder_bench_rss_mib
|
|
12
|
+
* - vector_cortex_encoder_bench_opset_ok
|
|
13
|
+
* - vector_cortex_encoder_bench_deterministic
|
|
14
|
+
*
|
|
15
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 — the child bench is pure local
|
|
16
|
+
* computation). No `any` (PREVENT-011).
|
|
17
|
+
*/
|
|
18
|
+
import { spawnSync } from "node:child_process";
|
|
19
|
+
import { dirname, join } from "node:path";
|
|
20
|
+
import { fileURLToPath } from "node:url";
|
|
21
|
+
import { getStateDir } from "../../store.js";
|
|
22
|
+
import { defaultEventsPath, logBenchEvent } from "../../monitoring.js";
|
|
23
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
24
|
+
const REPO_ROOT = join(HERE, "..", "..", "..");
|
|
25
|
+
const BENCH_SCRIPT = join(REPO_ROOT, "scripts", "ml5", "bench-onnx-prod.mjs");
|
|
26
|
+
/** Default events.log beside the state dir (mirrors defaultEventsPath). */
|
|
27
|
+
function benchEventsPath(stateDir) {
|
|
28
|
+
return defaultEventsPath(stateDir);
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Run the ONNX bench once and record its four events. Returns the parsed
|
|
32
|
+
* BenchResultV1. On any failure (script missing, non-zero exit, unparsable
|
|
33
|
+
* output) it returns a degraded result with gates.all:false — never throws, so
|
|
34
|
+
* the caller's agent loop is never broken (non-fatal store/write contract).
|
|
35
|
+
*/
|
|
36
|
+
export function runBench(stateDir = getStateDir()) {
|
|
37
|
+
const noop = (error) => ({
|
|
38
|
+
timestamp: Date.now(),
|
|
39
|
+
platform: `${process.platform}-${process.arch}`,
|
|
40
|
+
encoderNative: false,
|
|
41
|
+
threads: 4,
|
|
42
|
+
tokens: 512,
|
|
43
|
+
corpusTokens: 0,
|
|
44
|
+
p95Ms: null,
|
|
45
|
+
rssMib: null,
|
|
46
|
+
rssBaselineMib: null,
|
|
47
|
+
rssMarginalMib: null,
|
|
48
|
+
opset: null,
|
|
49
|
+
deterministic: false,
|
|
50
|
+
digest: null,
|
|
51
|
+
gates: { latency: false, rss: false, opset: false, determinism: false, all: false },
|
|
52
|
+
error,
|
|
53
|
+
});
|
|
54
|
+
const fallback = (error) => {
|
|
55
|
+
const r = noop(error);
|
|
56
|
+
emitEvents(stateDir, r);
|
|
57
|
+
return r;
|
|
58
|
+
};
|
|
59
|
+
try {
|
|
60
|
+
const res = spawnSync(process.execPath, ["--expose-gc", BENCH_SCRIPT], {
|
|
61
|
+
cwd: REPO_ROOT,
|
|
62
|
+
encoding: "utf8",
|
|
63
|
+
timeout: 600_000,
|
|
64
|
+
});
|
|
65
|
+
const stdout = (res.stdout ?? "").trim();
|
|
66
|
+
if (res.status === null) {
|
|
67
|
+
return fallback("bench timed out or failed to spawn");
|
|
68
|
+
}
|
|
69
|
+
const parsed = JSON.parse(stdout || "");
|
|
70
|
+
if (!isBenchResultV1(parsed)) {
|
|
71
|
+
return fallback("bench output was not a BenchResultV1");
|
|
72
|
+
}
|
|
73
|
+
emitEvents(stateDir, parsed);
|
|
74
|
+
return parsed;
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
return fallback(`bench failed: ${e?.message ?? String(e)}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
function isBenchResultV1(v) {
|
|
81
|
+
if (typeof v !== "object" || v === null)
|
|
82
|
+
return false;
|
|
83
|
+
const o = v;
|
|
84
|
+
return (typeof o.timestamp === "number" &&
|
|
85
|
+
typeof o.platform === "string" &&
|
|
86
|
+
typeof o.encoderNative === "boolean" &&
|
|
87
|
+
typeof o.threads === "number" &&
|
|
88
|
+
typeof o.tokens === "number" &&
|
|
89
|
+
typeof o.corpusTokens === "number" &&
|
|
90
|
+
typeof o.gates === "object" && o.gates !== null &&
|
|
91
|
+
typeof o.gates.all === "boolean");
|
|
92
|
+
}
|
|
93
|
+
function emitEvents(stateDir, r) {
|
|
94
|
+
const path = benchEventsPath(stateDir);
|
|
95
|
+
const run = { platform: r.platform, encoderNative: r.encoderNative, threads: r.threads, tokens: r.tokens, digest: r.digest, corpusTokens: r.corpusTokens };
|
|
96
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_p95_ms", { ...run, p95Ms: r.p95Ms, pass: r.gates.latency });
|
|
97
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_rss_mib", { ...run, rssMib: r.rssMib, rssBaselineMib: r.rssBaselineMib, rssMarginalMib: r.rssMarginalMib, pass: r.gates.rss });
|
|
98
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_opset_ok", { ...run, opset: r.opset, pass: r.gates.opset });
|
|
99
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_deterministic", { ...run, deterministic: r.deterministic, pass: r.gates.determinism });
|
|
100
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dedup-audit.ts — durable audit trail for dedup tier decisions
|
|
3
|
+
* (external-audit item #2).
|
|
4
|
+
*
|
|
5
|
+
* Before this module a tier decision existed only as the in-process `onTier`
|
|
6
|
+
* callback that paints the live UI; nothing survived the process, so an
|
|
7
|
+
* operator could not answer "which layer collapsed this region, onto what, at
|
|
8
|
+
* what similarity?" — the inputs needed to tune the thresholds in
|
|
9
|
+
* config/dedup.ts. Here each decision is appended to the repo's events.log as
|
|
10
|
+
* one structured JSON line (see `DedupAuditEvent` below).
|
|
11
|
+
*
|
|
12
|
+
* The event type and its append helper live HERE rather than in monitoring.ts:
|
|
13
|
+
* monitoring.ts already owns three concerns (decision events, the dashboard.json
|
|
14
|
+
* metrics snapshot, FP alerting) and sits close to its 300-line soft limit, so
|
|
15
|
+
* co-locating the shape with the only recorder that produces it keeps both files
|
|
16
|
+
* under the headroom gate. monitoring.ts re-exports both for callers (and the
|
|
17
|
+
* dashboard SSE tail) that treat it as the events.log barrel.
|
|
18
|
+
*
|
|
19
|
+
* Design constraints:
|
|
20
|
+
* - PURE INSTRUMENTATION. Nothing in this file may influence a dedup outcome.
|
|
21
|
+
* - Best-effort/non-fatal: `logDedupAudit` swallows IO errors, and the emitter
|
|
22
|
+
* itself is wrapped so a malformed field can never break add().
|
|
23
|
+
* - Honest fields only: a value is emitted only where the caller actually
|
|
24
|
+
* computed it. L0/L1 are hash/verify tiers and pass no `similarity`.
|
|
25
|
+
* - Signal, not chatter: callers emit on DECISIONS (a match, a scored
|
|
26
|
+
* candidate, the final outcome), never on every "scanning" transition.
|
|
27
|
+
* - Flag-gated by cfg.DEDUP_AUDIT (default ON; OFF writes nothing at all).
|
|
28
|
+
*
|
|
29
|
+
* PREVENT-PI-004: local filesystem append only, no network.
|
|
30
|
+
*/
|
|
31
|
+
import { appendFileSync, mkdirSync } from "node:fs";
|
|
32
|
+
import { dirname } from "node:path";
|
|
33
|
+
import { defaultEventsPath } from "../monitoring.js";
|
|
34
|
+
/**
|
|
35
|
+
* Append one audit event to events.log (best-effort, never throws).
|
|
36
|
+
*
|
|
37
|
+
* Same append-one-JSON-line contract as monitoring.ts's logDecision — an
|
|
38
|
+
* unwritable path is swallowed so instrumentation can never break add().
|
|
39
|
+
*/
|
|
40
|
+
export function logDedupAudit(path, ev) {
|
|
41
|
+
try {
|
|
42
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
43
|
+
appendFileSync(path, `${JSON.stringify(ev)}\n`, "utf-8");
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
/* best-effort — never break the extension on a log failure */
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/** Build a recorder bound to one add() cascade. */
|
|
50
|
+
export function dedupAuditRecorder(ctx, scope) {
|
|
51
|
+
const base = {
|
|
52
|
+
sessionId: scope.sessionId,
|
|
53
|
+
originalTokenEstimate: scope.originalTokenEstimate,
|
|
54
|
+
tokenEstimate: scope.tokenEstimate,
|
|
55
|
+
};
|
|
56
|
+
return {
|
|
57
|
+
deduped: (tier, matchedEntry, dedupReason, similarity) => emitDedupAudit(ctx, {
|
|
58
|
+
...base,
|
|
59
|
+
tier,
|
|
60
|
+
status: "deduped",
|
|
61
|
+
matchedEntry,
|
|
62
|
+
dedupReason,
|
|
63
|
+
...(similarity === undefined ? {} : { similarity }),
|
|
64
|
+
}),
|
|
65
|
+
passed: (tier, matchedEntry, similarity) => emitDedupAudit(ctx, {
|
|
66
|
+
...base,
|
|
67
|
+
tier,
|
|
68
|
+
status: "passed",
|
|
69
|
+
matchedEntry,
|
|
70
|
+
similarity,
|
|
71
|
+
}),
|
|
72
|
+
stored: (storedEntry, dedupReason, tokenEstimate) => emitDedupAudit(ctx, {
|
|
73
|
+
...base,
|
|
74
|
+
tier: "new",
|
|
75
|
+
status: "stored",
|
|
76
|
+
storedEntry,
|
|
77
|
+
dedupReason,
|
|
78
|
+
tokenEstimate,
|
|
79
|
+
}),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Append one dedup decision to events.log.
|
|
84
|
+
*
|
|
85
|
+
* Resolves the target path from the explicit `eventsPath` when a caller opted
|
|
86
|
+
* in (Sprint 14 monitoring / tests), otherwise from the store's own per-repo
|
|
87
|
+
* state dir — production never passes `eventsPath`, so defaulting is what makes
|
|
88
|
+
* the audit trail actually exist on a real device.
|
|
89
|
+
*/
|
|
90
|
+
export function emitDedupAudit(ctx, input) {
|
|
91
|
+
if (!ctx.auditEnabled)
|
|
92
|
+
return;
|
|
93
|
+
try {
|
|
94
|
+
const path = ctx.eventsPath ?? defaultEventsPath(ctx.stateDir);
|
|
95
|
+
logDedupAudit(path, {
|
|
96
|
+
type: "dedup_audit",
|
|
97
|
+
ts: new Date().toISOString(),
|
|
98
|
+
...input,
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
/* instrumentation must never break the add() path */
|
|
103
|
+
}
|
|
104
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.20.
|
|
3
|
+
"version": "0.20.38",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-3-Clause",
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* config/vector-cortex-ml5b.ts — ML5-B production bench harness flag.
|
|
3
|
+
*
|
|
4
|
+
* Sibling extract mirroring vector-cortex-ml5a.ts, so vector-cortex.ts stays
|
|
5
|
+
* under its 300-line soft limit (soft-as-hard gate). This is the ONNX Runtime
|
|
6
|
+
* evaluation/benchmark sprint flag. vector-cortex.ts re-exports the ENUM below
|
|
7
|
+
* and root src/config.ts re-exports it, so no consumer import path changes.
|
|
8
|
+
*
|
|
9
|
+
* ML5-B introduces NO runtime code path: the bench harness and corpus export
|
|
10
|
+
* are developer/evidence tooling (scripts/) plus a consumer-facing TypeScript
|
|
11
|
+
* shell (src/vector-cortex/encoder/bench.ts) that only writes monitoring
|
|
12
|
+
* events. The flag records intent and scopes the sprint's evidence assets; it
|
|
13
|
+
* gates nothing at runtime today. There is no HTTP endpoint and no dashboard
|
|
14
|
+
* change, so there is no SETTINGS toggle and no EXCLUDED_SETTINGS interaction.
|
|
15
|
+
*
|
|
16
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { sprintFlag } from "./vector-cortex-flag.js";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* ML5-B — production bench harness (ONNX Runtime eval). Default ON.
|
|
23
|
+
* `MEGACOMPACT_ML5_B=0` disables and is byte-identical to the ML5-A survivor:
|
|
24
|
+
* no bench endpoint exists and mode B continues to serve all clients exactly as
|
|
25
|
+
* before. The flag does not gate the harness itself — the harness is an on-demand
|
|
26
|
+
* developer tool with no runtime path.
|
|
27
|
+
*/
|
|
28
|
+
export const ML5B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_ML5_B");
|
|
@@ -280,6 +280,7 @@ export { VC9C_ENABLED } from "./vector-cortex-vc9c.js";
|
|
|
280
280
|
export { VC9D_ENABLED } from "./vector-cortex-vc9d.js";
|
|
281
281
|
export { PCC_ENABLED } from "./vector-cortex-pcc.js";
|
|
282
282
|
export { ML5A_ENABLED } from "./vector-cortex-ml5a.js";
|
|
283
|
+
export { ML5B_ENABLED } from "./vector-cortex-ml5b.js";
|
|
283
284
|
|
|
284
285
|
// Breaker constants (TRIAD_RESILIENCE.md §breaker) extracted to vector-cortex-breakers.ts.
|
|
285
286
|
export {
|
package/src/config.ts
CHANGED
package/src/monitoring.ts
CHANGED
|
@@ -205,6 +205,30 @@ export function logRecallQuality(path: string, ev: RecallQualityEvent): void {
|
|
|
205
205
|
}
|
|
206
206
|
}
|
|
207
207
|
|
|
208
|
+
// ---------------------------------------------------------------------------
|
|
209
|
+
// ML5-B encoder bench events (consumer-facing for the dashboard + evidence)
|
|
210
|
+
// ---------------------------------------------------------------------------
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Append a structured ML5-B bench event to events.log (best-effort, non-fatal).
|
|
214
|
+
* Mirrors the extension's appendEvent schema ({ ts, event, ...fields }) so the
|
|
215
|
+
* dashboard live-stream tail and evidence tooling parse the four
|
|
216
|
+
* `vector_cortex_encoder_bench_*` events identically. The bench is developer/
|
|
217
|
+
* evidence tooling with no runtime gating; this only records its results.
|
|
218
|
+
*/
|
|
219
|
+
export function logBenchEvent(
|
|
220
|
+
path: string,
|
|
221
|
+
event: string,
|
|
222
|
+
fields: Record<string, unknown>,
|
|
223
|
+
): void {
|
|
224
|
+
try {
|
|
225
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
226
|
+
appendFileSync(path, JSON.stringify({ ts: Date.now(), event, ...fields }) + "\n", "utf8");
|
|
227
|
+
} catch {
|
|
228
|
+
/* best-effort — never break the caller */
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
208
232
|
// ---------------------------------------------------------------------------
|
|
209
233
|
// Dedup audit trail (external-audit item #2)
|
|
210
234
|
// ---------------------------------------------------------------------------
|
package/src/store/backfill.ts
CHANGED
|
@@ -134,6 +134,7 @@ export function backfillContentHashes(stateDir: string = getStateDir()): Backfil
|
|
|
134
134
|
|
|
135
135
|
if (THROTTLE_MS > 0) {
|
|
136
136
|
// No-op in this synchronous build; placeholder for future streaming backfill.
|
|
137
|
+
// guardrails-allow PREVENT-STUB-001: ML5-C
|
|
137
138
|
}
|
|
138
139
|
|
|
139
140
|
return { processed, updated, duplicatesResolved };
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/bench-export.ts — ML5-B bench result contract.
|
|
3
|
+
*
|
|
4
|
+
* BenchResultV1 is the typed surface `bench.ts` parses from the qualification
|
|
5
|
+
* harness (`scripts/ml5/bench-onnx-prod.mjs`) and the dashboard / evidence
|
|
6
|
+
* tooling consume. It carries AGGREGATE measurements + a digest only — never
|
|
7
|
+
* chunk/message content (EVAL-REDACT-002).
|
|
8
|
+
*
|
|
9
|
+
* Contract-first (ENGINEERING_PRACTICES §3): this types file is the reviewed
|
|
10
|
+
* gate; implementations import from it. Pi-agnostic, dependency-free
|
|
11
|
+
* (PREVENT-PI-004 / PREVENT-011).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/** The four ML5-B bench gates, each independently measured. */
|
|
15
|
+
export interface BenchGatesV1 {
|
|
16
|
+
/** p95 latency at 512 tokens on `threads` threads <= 40 ms. */
|
|
17
|
+
readonly latency: boolean;
|
|
18
|
+
/** steady-state marginal RSS over the process baseline <= 150 MiB. */
|
|
19
|
+
readonly rss: boolean;
|
|
20
|
+
/** the loaded model's declared opset_import equals 17. */
|
|
21
|
+
readonly opset: boolean;
|
|
22
|
+
/** SHA-256 of the embedding output identical across 3 runs (maxAbsDelta=0). */
|
|
23
|
+
readonly determinism: boolean;
|
|
24
|
+
/** conjunctive: every gate passed. */
|
|
25
|
+
readonly all: boolean;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* BenchResultV1 — one qualification run of the ONNX encoder bench.
|
|
30
|
+
*
|
|
31
|
+
* Shape is fixed by the ML5-B spec (task 4). `p95Ms`/`rssMib`/`rssMarginalMib`/
|
|
32
|
+
* `digest` are null when the runtime package is absent (degraded run) or a
|
|
33
|
+
* gate could not be measured; `gates.all` is false in that case and `error`
|
|
34
|
+
* (optional) records the honest degradation reason.
|
|
35
|
+
*/
|
|
36
|
+
export interface BenchResultV1 {
|
|
37
|
+
readonly timestamp: number;
|
|
38
|
+
/** `${process.platform}-${process.arch}` (e.g. linux-x64, darwin-arm64). */
|
|
39
|
+
readonly platform: string;
|
|
40
|
+
/** true = onnxruntime-node (native); false = onnxruntime-web (WASM). */
|
|
41
|
+
readonly encoderNative: boolean;
|
|
42
|
+
/** intraOpNumThreads used for the latency gate (normative 4). */
|
|
43
|
+
readonly threads: number;
|
|
44
|
+
/** token count per inference (normative 512). */
|
|
45
|
+
readonly tokens: number;
|
|
46
|
+
/** total tokens in the corpus the bench streamed over. */
|
|
47
|
+
readonly corpusTokens: number;
|
|
48
|
+
/** p95 latency in ms (null on degraded/absent runtime). */
|
|
49
|
+
readonly p95Ms: number | null;
|
|
50
|
+
/** steady-state RSS (MiB) over the process baseline, post-GC. */
|
|
51
|
+
readonly rssMib: number | null;
|
|
52
|
+
/** RSS (MiB) sampled at process start before loading the encoder. */
|
|
53
|
+
readonly rssBaselineMib: number | null;
|
|
54
|
+
/** rssMib - rssBaselineMib: the encoder's marginal footprint. */
|
|
55
|
+
readonly rssMarginalMib: number | null;
|
|
56
|
+
/** declared opset_import (17); null when no asset manifest is readable. */
|
|
57
|
+
readonly opset: number | null;
|
|
58
|
+
/** true when the output SHA-256 is identical across 3 runs. */
|
|
59
|
+
readonly deterministic: boolean;
|
|
60
|
+
/** SHA-256 of the embedding output buffer (null on degraded/absent). */
|
|
61
|
+
readonly digest: string | null;
|
|
62
|
+
readonly gates: BenchGatesV1;
|
|
63
|
+
/** Optional: honest degradation / failure reason (no runtime package, etc.). */
|
|
64
|
+
readonly error?: string;
|
|
65
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/bench.ts — ML5-B bench runner (consumer-facing shell).
|
|
3
|
+
*
|
|
4
|
+
* Calls `scripts/ml5/bench-onnx-prod.mjs` via child_process, parses the
|
|
5
|
+
* BenchResultV1 it emits, and writes the four `vector_cortex_encoder_bench_*`
|
|
6
|
+
* events to the monitoring events.log (the dashboard / ML5-D surface consume
|
|
7
|
+
* them later). This is NOT a runtime path — it is developer/evidence tooling.
|
|
8
|
+
*
|
|
9
|
+
* Events written (all best-effort / non-fatal):
|
|
10
|
+
* - vector_cortex_encoder_bench_p95_ms
|
|
11
|
+
* - vector_cortex_encoder_bench_rss_mib
|
|
12
|
+
* - vector_cortex_encoder_bench_opset_ok
|
|
13
|
+
* - vector_cortex_encoder_bench_deterministic
|
|
14
|
+
*
|
|
15
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 — the child bench is pure local
|
|
16
|
+
* computation). No `any` (PREVENT-011).
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { spawnSync } from "node:child_process";
|
|
20
|
+
import { dirname, join } from "node:path";
|
|
21
|
+
import { fileURLToPath } from "node:url";
|
|
22
|
+
import { getStateDir } from "../../store.js";
|
|
23
|
+
import { defaultEventsPath, logBenchEvent } from "../../monitoring.js";
|
|
24
|
+
import type { BenchResultV1 } from "./bench-export.js";
|
|
25
|
+
|
|
26
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
27
|
+
const REPO_ROOT = join(HERE, "..", "..", "..");
|
|
28
|
+
const BENCH_SCRIPT = join(REPO_ROOT, "scripts", "ml5", "bench-onnx-prod.mjs");
|
|
29
|
+
|
|
30
|
+
/** Default events.log beside the state dir (mirrors defaultEventsPath). */
|
|
31
|
+
function benchEventsPath(stateDir: string): string {
|
|
32
|
+
return defaultEventsPath(stateDir);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Run the ONNX bench once and record its four events. Returns the parsed
|
|
37
|
+
* BenchResultV1. On any failure (script missing, non-zero exit, unparsable
|
|
38
|
+
* output) it returns a degraded result with gates.all:false — never throws, so
|
|
39
|
+
* the caller's agent loop is never broken (non-fatal store/write contract).
|
|
40
|
+
*/
|
|
41
|
+
export function runBench(stateDir: string = getStateDir()): BenchResultV1 {
|
|
42
|
+
const noop = (error: string): BenchResultV1 => ({
|
|
43
|
+
timestamp: Date.now(),
|
|
44
|
+
platform: `${process.platform}-${process.arch}`,
|
|
45
|
+
encoderNative: false,
|
|
46
|
+
threads: 4,
|
|
47
|
+
tokens: 512,
|
|
48
|
+
corpusTokens: 0,
|
|
49
|
+
p95Ms: null,
|
|
50
|
+
rssMib: null,
|
|
51
|
+
rssBaselineMib: null,
|
|
52
|
+
rssMarginalMib: null,
|
|
53
|
+
opset: null,
|
|
54
|
+
deterministic: false,
|
|
55
|
+
digest: null,
|
|
56
|
+
gates: { latency: false, rss: false, opset: false, determinism: false, all: false },
|
|
57
|
+
error,
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
const fallback = (error: string): BenchResultV1 => {
|
|
61
|
+
const r = noop(error);
|
|
62
|
+
emitEvents(stateDir, r);
|
|
63
|
+
return r;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
try {
|
|
67
|
+
const res = spawnSync(process.execPath, ["--expose-gc", BENCH_SCRIPT], {
|
|
68
|
+
cwd: REPO_ROOT,
|
|
69
|
+
encoding: "utf8",
|
|
70
|
+
timeout: 600_000,
|
|
71
|
+
});
|
|
72
|
+
const stdout = (res.stdout ?? "").trim();
|
|
73
|
+
if (res.status === null) {
|
|
74
|
+
return fallback("bench timed out or failed to spawn");
|
|
75
|
+
}
|
|
76
|
+
const parsed: unknown = JSON.parse(stdout || "");
|
|
77
|
+
if (!isBenchResultV1(parsed)) {
|
|
78
|
+
return fallback("bench output was not a BenchResultV1");
|
|
79
|
+
}
|
|
80
|
+
emitEvents(stateDir, parsed);
|
|
81
|
+
return parsed;
|
|
82
|
+
} catch (e) {
|
|
83
|
+
return fallback(`bench failed: ${(e as Error)?.message ?? String(e)}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function isBenchResultV1(v: unknown): v is BenchResultV1 {
|
|
88
|
+
if (typeof v !== "object" || v === null) return false;
|
|
89
|
+
const o = v as Record<string, unknown>;
|
|
90
|
+
return (
|
|
91
|
+
typeof o.timestamp === "number" &&
|
|
92
|
+
typeof o.platform === "string" &&
|
|
93
|
+
typeof o.encoderNative === "boolean" &&
|
|
94
|
+
typeof o.threads === "number" &&
|
|
95
|
+
typeof o.tokens === "number" &&
|
|
96
|
+
typeof o.corpusTokens === "number" &&
|
|
97
|
+
typeof o.gates === "object" && o.gates !== null &&
|
|
98
|
+
typeof (o.gates as Record<string, unknown>).all === "boolean"
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function emitEvents(stateDir: string, r: BenchResultV1): void {
|
|
103
|
+
const path = benchEventsPath(stateDir);
|
|
104
|
+
const run = { platform: r.platform, encoderNative: r.encoderNative, threads: r.threads, tokens: r.tokens, digest: r.digest, corpusTokens: r.corpusTokens };
|
|
105
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_p95_ms", { ...run, p95Ms: r.p95Ms, pass: r.gates.latency });
|
|
106
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_rss_mib", { ...run, rssMib: r.rssMib, rssBaselineMib: r.rssBaselineMib, rssMarginalMib: r.rssMarginalMib, pass: r.gates.rss });
|
|
107
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_opset_ok", { ...run, opset: r.opset, pass: r.gates.opset });
|
|
108
|
+
logBenchEvent(path, "vector_cortex_encoder_bench_deterministic", { ...run, deterministic: r.deterministic, pass: r.gates.determinism });
|
|
109
|
+
}
|