pi-mega-compact 0.20.40 → 0.20.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/vector-cortex-dedup-attr.js +14 -0
- package/dist/config/vector-cortex-ml5e.js +30 -0
- package/dist/config/vector-cortex.js +5 -5
- package/dist/config.js +1 -1
- package/dist/extensions/dashboard-server/api-contracts/dedup-attribution.js +13 -0
- package/dist/extensions/dashboard-server/api-contracts/endpoints/registry-ext.js +7 -0
- package/dist/extensions/dashboard-server/route-dispatch.js +3 -1
- package/dist/extensions/dashboard-server/routes-dedup-attribution.js +150 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-vector-cortex.js +2 -0
- package/dist/extensions/dashboard-server/routes.js +1 -0
- package/dist/src/config/vector-cortex-dedup-attr.js +14 -0
- package/dist/src/config/vector-cortex-ml5e.js +30 -0
- package/dist/src/config/vector-cortex.js +5 -5
- package/dist/src/config.js +1 -1
- package/dist/src/vector-cortex/dedup-attr/rollup.js +97 -0
- package/dist/src/vector-cortex/encoder/promotion.js +58 -0
- package/dist/vector-cortex/dedup-attr/rollup.js +97 -0
- package/dist/vector-cortex/encoder/promotion.js +58 -0
- package/extensions/dashboard-client/dist/assets/{AreaChart-BT7YPcp7.js → AreaChart-Cn6X7epL.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{AreaChart-BT7YPcp7.js.map → AreaChart-Cn6X7epL.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{BarChart-DceGcBUh.js → BarChart-APy1Wc7z.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{BarChart-DceGcBUh.js.map → BarChart-APy1Wc7z.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{CacheTab-B9lCouRl.js → CacheTab-CEv_Ss9O.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{CacheTab-B9lCouRl.js.map → CacheTab-CEv_Ss9O.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{EventsTab-oqJ11jY7.js → EventsTab-8jV1Pr0C.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{EventsTab-oqJ11jY7.js.map → EventsTab-8jV1Pr0C.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{HealthTab-Dbosf2Ik.js → HealthTab-B2VgEz1j.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{HealthTab-Dbosf2Ik.js.map → HealthTab-B2VgEz1j.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-6TiApCVE.js → MaintenanceTab-m37AlTbg.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-6TiApCVE.js.map → MaintenanceTab-m37AlTbg.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MemoryMapTab-cBsE4mTf.js → MemoryMapTab-Co2jbqdT.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MemoryMapTab-cBsE4mTf.js.map → MemoryMapTab-Co2jbqdT.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MetricsTab-CX-X9hhy.js → MetricsTab-D0yV9X6D.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MetricsTab-CX-X9hhy.js.map → MetricsTab-D0yV9X6D.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{OverviewTab-CpPYaTJF.js → OverviewTab-J5TjXB14.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{OverviewTab-CpPYaTJF.js.map → OverviewTab-J5TjXB14.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{ReposTab-D_Ot4QbU.js → ReposTab-BadNIkWu.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{ReposTab-D_Ot4QbU.js.map → ReposTab-BadNIkWu.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SessionsTab-BUzO3iGK.js → SessionsTab-CdKShJuR.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SessionsTab-BUzO3iGK.js.map → SessionsTab-CdKShJuR.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SetupTab-Be4w9v4j.js → SetupTab-CPemizDx.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SetupTab-Be4w9v4j.js.map → SetupTab-CPemizDx.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Cx-g6CXH.js → TimeSavedCard-TvULibwW.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Cx-g6CXH.js.map → TimeSavedCard-TvULibwW.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TurnsTab-CC_7T7ad.js → TurnsTab-CglFIcYy.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TurnsTab-CC_7T7ad.js.map → TurnsTab-CglFIcYy.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{VcStatusBadge-ZTItXSiH.js → VcStatusBadge-H2bBNhrR.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{VcStatusBadge-ZTItXSiH.js.map → VcStatusBadge-H2bBNhrR.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{VectorCortexTab-DgAUZ4Pg.js → VectorCortexTab-CXPv3sYQ.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{VectorCortexTab-DgAUZ4Pg.js.map → VectorCortexTab-CXPv3sYQ.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{WikiTab-DnKupXTv.js → WikiTab-BNfN175_.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{WikiTab-DnKupXTv.js.map → WikiTab-BNfN175_.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{button-CFt3e0yc.js → button-DjN7aqD2.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{button-CFt3e0yc.js.map → button-DjN7aqD2.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{card-C6ZjsHrl.js → card-DLHA5XP8.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{card-C6ZjsHrl.js.map → card-DLHA5XP8.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{client-extra-DdEdGTIe.js → client-extra-ZT6mIQZS.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{client-extra-DdEdGTIe.js.map → client-extra-ZT6mIQZS.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-C9_fu2hc.js → generateCategoricalChart-DAl4dRVG.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-C9_fu2hc.js.map → generateCategoricalChart-DAl4dRVG.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{index-DY81TY4f.js → index-H-sfROoW.js} +16 -16
- package/extensions/dashboard-client/dist/assets/index-H-sfROoW.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{switch-BOqa1d6x.js → switch-BCxvJYon.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{switch-BOqa1d6x.js.map → switch-BCxvJYon.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{toggle-DToFaIEd.js → toggle-DviUMYeg.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{toggle-DToFaIEd.js.map → toggle-DviUMYeg.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{useSSE-CZqkhU18.js → useSSE-DP2O8vHY.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{useSSE-CZqkhU18.js.map → useSSE-DP2O8vHY.js.map} +1 -1
- package/extensions/dashboard-client/dist/index.html +1 -1
- package/extensions/dashboard-server/api-contracts/dedup-attribution.ts +69 -0
- package/extensions/dashboard-server/api-contracts/endpoints/registry-ext.ts +15 -0
- package/extensions/dashboard-server/api-contracts/index.ts +8 -0
- package/extensions/dashboard-server/route-dispatch.ts +2 -0
- package/extensions/dashboard-server/routes-dedup-attribution.ts +166 -0
- package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts +12 -0
- package/extensions/dashboard-server/routes.ts +1 -0
- package/package.json +1 -1
- package/src/config/vector-cortex-dedup-attr.ts +16 -0
- package/src/config/vector-cortex-ml5e.ts +32 -0
- package/src/config/vector-cortex.ts +5 -5
- package/src/config.ts +2 -0
- package/src/vector-cortex/dedup-attr/rollup.ts +136 -0
- package/src/vector-cortex/encoder/promotion.ts +129 -0
- package/extensions/dashboard-client/dist/assets/index-DY81TY4f.js.map +0 -1
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dashboard-server/routes-dedup-attribution.ts — DEDUP-ATTR tier attribution route.
|
|
3
|
+
*
|
|
4
|
+
* GET /api/dedup-tier-attribution?windowMs=<n> — reader-only aggregate answering
|
|
5
|
+
* "L0/L1/L2/new percent of dedup decisions in window W". Reads the repo's local
|
|
6
|
+
* events.log (per-repo stateDir, bounded 8 MiB tail), parses `dedup_audit` lines
|
|
7
|
+
* (PREVENT-001 guarded), and sends the pure rollup with a shared-derived status.
|
|
8
|
+
*
|
|
9
|
+
* Flag-off (MEGACOMPACT_DEDUP_ATTR=0) returns 404 and writes NO durable cache
|
|
10
|
+
* file — byte-identical predecessor. Reader-only: emits tier counts + shares,
|
|
11
|
+
* never matched checkpoint paths/text, never raw user query (EVAL-REDACT-002).
|
|
12
|
+
*
|
|
13
|
+
* Memoization mirrors routes-vector-cortex-health.ts: an in-memory facts cache
|
|
14
|
+
* keyed on {events.log mtime+size, windowMs} is served when ≤5s old, so repeated
|
|
15
|
+
* dashboard polls stay cheap while any on-disk log mutation still invalidates.
|
|
16
|
+
*
|
|
17
|
+
* Guardrails: PREVENT-PI-004 (local filesystem read + loopback response only),
|
|
18
|
+
* PREVENT-001 (JSON.parse guarded), PREVENT-011 (no `any`).
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
22
|
+
import { readFileSync, statSync, writeFileSync, mkdirSync } from "node:fs";
|
|
23
|
+
import { join, dirname } from "node:path";
|
|
24
|
+
|
|
25
|
+
import type { RouteContext } from "./routes-core.js";
|
|
26
|
+
import { sendJson } from "./routes-vector-cortex-shared.js";
|
|
27
|
+
import { DEDUP_ATTR_ENABLED } from "../../src/config.js";
|
|
28
|
+
import { computeDedupTierRollup } from "../../src/vector-cortex/dedup-attr/rollup.js";
|
|
29
|
+
import type { DedupTierAttributionResponse } from "./api-contracts/dedup-attribution.js";
|
|
30
|
+
import type { DedupAuditEvent } from "../../src/vectorStore/dedup-audit.js";
|
|
31
|
+
import { deriveVcStatus } from "./vc-status.js";
|
|
32
|
+
|
|
33
|
+
const DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1000; // 24h
|
|
34
|
+
const MAX_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
|
|
35
|
+
const MAX_TAIL_BYTES = 8 * 1024 * 1024; // 8 MiB
|
|
36
|
+
const MEMO_TTL_MS = 5000; // ≤5s per stateDir
|
|
37
|
+
|
|
38
|
+
interface MemoEntry {
|
|
39
|
+
key: string;
|
|
40
|
+
at: number;
|
|
41
|
+
body: DedupTierAttributionResponse;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
let memo: MemoEntry | null = null;
|
|
45
|
+
|
|
46
|
+
/** Parse one events.log line into a DedupAuditEvent, or null (PREVENT-001). */
|
|
47
|
+
function parseAuditLine(line: string): DedupAuditEvent | null {
|
|
48
|
+
if (!line.includes('"dedup_audit"')) return null;
|
|
49
|
+
let parsed: unknown;
|
|
50
|
+
try {
|
|
51
|
+
parsed = JSON.parse(line);
|
|
52
|
+
} catch {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
56
|
+
const obj = parsed as Record<string, unknown>;
|
|
57
|
+
if (obj.type !== "dedup_audit") return null;
|
|
58
|
+
if (typeof obj.ts !== "string") return null;
|
|
59
|
+
const tier = obj.tier;
|
|
60
|
+
const status = obj.status;
|
|
61
|
+
if (tier !== "L0" && tier !== "L1" && tier !== "L2" && tier !== "new") return null;
|
|
62
|
+
if (status !== "deduped" && status !== "passed" && status !== "stored") return null;
|
|
63
|
+
// sessionId is never read by the rollup; a parsed line may omit richer fields.
|
|
64
|
+
return { type: "dedup_audit", ts: obj.ts, tier, status, sessionId: "" };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Read the bounded tail of events.log into parsed dedup_audit events. */
|
|
68
|
+
function readAuditEvents(eventsPath: string): DedupAuditEvent[] {
|
|
69
|
+
let buf: Buffer;
|
|
70
|
+
try {
|
|
71
|
+
buf = readFileSync(eventsPath); // guardrails-allow PREVENT-PI-004: local events.log read (loopback dashboard)
|
|
72
|
+
} catch {
|
|
73
|
+
return [];
|
|
74
|
+
}
|
|
75
|
+
const tail = buf.length > MAX_TAIL_BYTES ? buf.subarray(buf.length - MAX_TAIL_BYTES) : buf;
|
|
76
|
+
const text = tail.toString("utf-8");
|
|
77
|
+
const lines = text.split("\n");
|
|
78
|
+
// Drop a leading fragment when we truncated mid-line (first line may be partial).
|
|
79
|
+
const start = buf.length > MAX_TAIL_BYTES ? 1 : 0;
|
|
80
|
+
const out: DedupAuditEvent[] = [];
|
|
81
|
+
for (let i = start; i < lines.length; i++) {
|
|
82
|
+
const line = lines[i];
|
|
83
|
+
if (!line) continue;
|
|
84
|
+
const ev = parseAuditLine(line);
|
|
85
|
+
if (ev) out.push(ev);
|
|
86
|
+
}
|
|
87
|
+
return out;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Resolve windowMs from the ?windowMs= query param (default 24h, capped 30d). */
|
|
91
|
+
function resolveWindowMs(url: string): number {
|
|
92
|
+
if (!url.includes("?")) return DEFAULT_WINDOW_MS;
|
|
93
|
+
const params = new URLSearchParams(url.slice(url.indexOf("?")));
|
|
94
|
+
const raw = params.get("windowMs");
|
|
95
|
+
if (raw == null) return DEFAULT_WINDOW_MS;
|
|
96
|
+
const n = Number(raw);
|
|
97
|
+
if (!Number.isFinite(n) || n <= 0) return DEFAULT_WINDOW_MS;
|
|
98
|
+
return Math.min(n, MAX_WINDOW_MS);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Best-effort durable snapshot write (never breaks the response). */
|
|
102
|
+
function writeDurableSnapshot(stateDir: string, rollup: DedupTierAttributionResponse): void {
|
|
103
|
+
try {
|
|
104
|
+
mkdirSync(dirname(join(stateDir, "dedup-tier-attribution.json")), { recursive: true });
|
|
105
|
+
writeFileSync(
|
|
106
|
+
join(stateDir, "dedup-tier-attribution.json"),
|
|
107
|
+
`${JSON.stringify(rollup)}\n`,
|
|
108
|
+
"utf-8",
|
|
109
|
+
);
|
|
110
|
+
} catch {
|
|
111
|
+
/* best-effort — instrumentation must never break the read path */
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function handleDedupTierAttribution(
|
|
116
|
+
req: IncomingMessage,
|
|
117
|
+
res: ServerResponse,
|
|
118
|
+
ctx: RouteContext,
|
|
119
|
+
): boolean {
|
|
120
|
+
const url = req.url ?? "";
|
|
121
|
+
const path = url.split("?", 1)[0] ?? url;
|
|
122
|
+
if (path !== "/api/dedup-tier-attribution") return false;
|
|
123
|
+
if (req.method !== "GET") {
|
|
124
|
+
sendJson(res, 405, { error: "method_not_allowed" });
|
|
125
|
+
return true;
|
|
126
|
+
}
|
|
127
|
+
if (!DEDUP_ATTR_ENABLED()) {
|
|
128
|
+
// Flag-off: 404 + no durable cache write — byte-identical predecessor.
|
|
129
|
+
sendJson(res, 404, { error: "not_found" });
|
|
130
|
+
return true;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const eventsPath = ctx.eventsPath;
|
|
134
|
+
const windowMs = resolveWindowMs(url);
|
|
135
|
+
const now = new Date();
|
|
136
|
+
|
|
137
|
+
// Memoized-facts pattern (mirrors routes-vector-cortex-health.ts): serve a
|
|
138
|
+
// still-fresh entry keyed on {events.log mtime+size, windowMs}.
|
|
139
|
+
let keyBase: { mtimeMs: number; size: number } | null = null;
|
|
140
|
+
try {
|
|
141
|
+
const s = statSync(eventsPath); // guardrails-allow PREVENT-PI-004: local events.log stat (loopback)
|
|
142
|
+
keyBase = { mtimeMs: s.mtimeMs, size: s.size };
|
|
143
|
+
} catch {
|
|
144
|
+
keyBase = { mtimeMs: 0, size: 0 };
|
|
145
|
+
}
|
|
146
|
+
const key = `${keyBase.mtimeMs}:${keyBase.size}:${windowMs}`;
|
|
147
|
+
if (memo !== null && memo.key === key && now.getTime() - memo.at <= MEMO_TTL_MS) {
|
|
148
|
+
sendJson(res, 200, memo.body);
|
|
149
|
+
return true;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const auditEvents = readAuditEvents(eventsPath);
|
|
153
|
+
const rollup = computeDedupTierRollup(auditEvents, windowMs, now);
|
|
154
|
+
const status = deriveVcStatus({
|
|
155
|
+
enabled: true,
|
|
156
|
+
hasData: auditEvents.length > 0,
|
|
157
|
+
structuralOnly: false,
|
|
158
|
+
});
|
|
159
|
+
const body: DedupTierAttributionResponse = { ...rollup, status };
|
|
160
|
+
|
|
161
|
+
memo = { key, at: now.getTime(), body };
|
|
162
|
+
writeDurableSnapshot(ctx.stateDir, body);
|
|
163
|
+
|
|
164
|
+
sendJson(res, 200, body);
|
|
165
|
+
return true;
|
|
166
|
+
}
|
|
@@ -240,5 +240,17 @@ export const VECTOR_CORTEX_SETTINGS: SettingGroup = {
|
|
|
240
240
|
"ML5-D dashboard 'Improve Cortex' surface: the ModelImprovementCard + POST /api/cortex/improve + GET /api/cortex/improve/status/:jobId. ON (default) = the card renders and Improve launches a local ML5-A training job. OFF = both improve endpoints return 404 and VectorCortexTab omits the card, byte-identical to ML5-C.",
|
|
241
241
|
true,
|
|
242
242
|
),
|
|
243
|
+
boolDirect(
|
|
244
|
+
"MEGACOMPACT_ML5_E",
|
|
245
|
+
"ML5-E Nightly Retraining Cron",
|
|
246
|
+
"ML5-E nightly retraining + feedback loop: the user's own post-redaction conversation turns become fresh training signal, the five heads are re-fit nightly via system cron, calibration is re-validated, and mode-A promotion is re-checked without human intervention. The cron is system-configured (crontab -e); the extension never installs or writes a crontab. Candidates are written to ~/.pi/mega-compact-encoder/candidates/ and promoted via the ML5-D Improve Cortex flow. ON (default) = the scripts may be invoked by the system cron. OFF = scripts are never invoked; byte-identical to ML5-D.",
|
|
247
|
+
true,
|
|
248
|
+
),
|
|
249
|
+
boolDirect(
|
|
250
|
+
"MEGACOMPACT_DEDUP_ATTR",
|
|
251
|
+
"Dedup Tier Attribution Rollup",
|
|
252
|
+
"Dedup tier-attribution rollup: per-tier dedup catch shares (L0/L1/L2/new percent of dedup decisions) read from the local events.log dedup_audit stream (GET /api/dedup-tier-attribution). OFF = 404 + no cache file, byte-identical predecessor.",
|
|
253
|
+
true,
|
|
254
|
+
),
|
|
243
255
|
],
|
|
244
256
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.20.
|
|
3
|
+
"version": "0.20.42",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-3-Clause",
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* config/vector-cortex-dedup-attr.ts — DEDUP-ATTR tier-attribution flag.
|
|
3
|
+
*
|
|
4
|
+
* The single positive sprint flag for the dedup tier-attribution rollup
|
|
5
|
+
* (external-audit item #6). Default ON; `MEGACOMPACT_DEDUP_ATTR=0` / `=false` /
|
|
6
|
+
* `_DISABLED=true` disables and is byte-identical to the predecessor: the
|
|
7
|
+
* `/api/dedup-tier-attribution` endpoint 404s and no rollup cache file is
|
|
8
|
+
* written.
|
|
9
|
+
*
|
|
10
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { sprintFlag } from "./vector-cortex-flag.js";
|
|
14
|
+
|
|
15
|
+
/** Positive sprint flag: `=0` or `_DISABLED=true` disables (default ON). */
|
|
16
|
+
export const DEDUP_ATTR_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_DEDUP_ATTR");
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* config/vector-cortex-ml5e.ts — ML5-E nightly retraining cron + feedback loop.
|
|
3
|
+
*
|
|
4
|
+
* Sibling extract mirroring vector-cortex-ml5a.ts / ml5b.ts / ml5c.ts / ml5d.ts,
|
|
5
|
+
* so vector-cortex.ts stays under its 300-line soft limit (soft-as-hard gate).
|
|
6
|
+
* vector-cortex.ts re-exports the ENUM below and root src/config.ts re-exports
|
|
7
|
+
* it, so no consumer import path changes.
|
|
8
|
+
*
|
|
9
|
+
* ML5-E turns one-shot training (ML5-D's Improve button) into a living loop:
|
|
10
|
+
* the user's own post-redaction conversation turns become fresh training signal,
|
|
11
|
+
* the five heads are re-fit nightly, calibration is re-validated, and mode-A
|
|
12
|
+
* promotion is re-checked without human intervention. The cron is system-
|
|
13
|
+
* configured (`crontab -e` on the operator device); the extension never installs
|
|
14
|
+
* or writes a crontab. The flag gates **invocation** only — the scripts
|
|
15
|
+
* (retrain-nightly.mjs, promotion-gate.mjs, promotion.ts) exist on disk
|
|
16
|
+
* regardless; `=0` means the runtime never invokes them and the dashboard's
|
|
17
|
+
* Improve Cortex flow is byte-identical to the ML5-D survivor.
|
|
18
|
+
*
|
|
19
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { sprintFlag } from "./vector-cortex-flag.js";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* ML5-E — nightly retraining cron + feedback loop. Default ON.
|
|
26
|
+
* `MEGACOMPACT_ML5_E=0` disables and is byte-identical to the ML5-D survivor:
|
|
27
|
+
* the nightly retraining scripts are never invoked by the runtime, the
|
|
28
|
+
* promotion gate is inert, and the dashboard's Improve Cortex flow is
|
|
29
|
+
* byte-identical to the ML5-D tab. The flag gates invocation only — the scripts
|
|
30
|
+
* exist on disk regardless.
|
|
31
|
+
*/
|
|
32
|
+
export const ML5E_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_ML5_E");
|
|
@@ -11,9 +11,8 @@
|
|
|
11
11
|
|
|
12
12
|
import { sprintFlag } from "./vector-cortex-flag.js";
|
|
13
13
|
|
|
14
|
-
// VC0/VC1/VC2 foundation-phase flags
|
|
15
|
-
//
|
|
16
|
-
// `from "./config/vector-cortex.js"` import keeps resolving unchanged.
|
|
14
|
+
// VC0/VC1/VC2 foundation-phase flags in vector-cortex-early.ts; re-exported
|
|
15
|
+
// so every existing `from "./config/vector-cortex.js"` import resolves unchanged.
|
|
17
16
|
export {
|
|
18
17
|
VC0A_ENABLED,
|
|
19
18
|
VC0B_ENABLED,
|
|
@@ -269,9 +268,8 @@ export const VC8A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8A");
|
|
|
269
268
|
*/
|
|
270
269
|
export const VC8B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8B");
|
|
271
270
|
|
|
272
|
-
// VC8C
|
|
271
|
+
// VC8C extracted to vector-cortex-vc8c.ts; re-exported so existing imports keep resolving.
|
|
273
272
|
export { VC8C_ENABLED } from "./vector-cortex-vc8c.js";
|
|
274
|
-
// Sibling extracts to stay under the 300-line soft limit.
|
|
275
273
|
export { VC9A_ENABLED } from "./vector-cortex-vc9a.js";
|
|
276
274
|
export { VC9B_ENABLED } from "./vector-cortex-vc9b.js";
|
|
277
275
|
export { VC9C_ENABLED } from "./vector-cortex-vc9c.js";
|
|
@@ -281,6 +279,8 @@ export { ML5A_ENABLED } from "./vector-cortex-ml5a.js";
|
|
|
281
279
|
export { ML5B_ENABLED } from "./vector-cortex-ml5b.js";
|
|
282
280
|
export { ML5C_ENABLED } from "./vector-cortex-ml5c.js";
|
|
283
281
|
export { ML5D_ENABLED } from "./vector-cortex-ml5d.js";
|
|
282
|
+
export { ML5E_ENABLED } from "./vector-cortex-ml5e.js";
|
|
283
|
+
export { DEDUP_ATTR_ENABLED } from "./vector-cortex-dedup-attr.js";
|
|
284
284
|
|
|
285
285
|
// Breaker constants (TRIAD_RESILIENCE.md §breaker) extracted to vector-cortex-breakers.ts.
|
|
286
286
|
export {
|
package/src/config.ts
CHANGED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/dedup-attr/rollup.ts — DEDUP-ATTR pure tier-attribution rollup.
|
|
3
|
+
*
|
|
4
|
+
* Pure function: reads an array of parsed dedup-audit events, buckets them by
|
|
5
|
+
* tier over a query window, and returns the per-tier catch shares. No I/O, no
|
|
6
|
+
* clock — the route passes `now: Date` (new Date()) at request time, so this
|
|
7
|
+
* module is deterministically reproducible and trivially testable (DEDUP-ATTR-004).
|
|
8
|
+
*
|
|
9
|
+
* Missing `similarity` is fine here: L0/L1 are hash/verify tiers and emit no
|
|
10
|
+
* score, and the rollup never consults it. Malformed lines (non-object, wrong
|
|
11
|
+
* type) are skipped silently — the route's events.log reader already guards its
|
|
12
|
+
* JSON.parse (PREVENT-001), and this layer defends against a hostile/mixed
|
|
13
|
+
* `dedup_audit` array as well.
|
|
14
|
+
*
|
|
15
|
+
* The returned shape is structurally identical to the authoritative contract in
|
|
16
|
+
* `extensions/dashboard-server/api-contracts/dedup-attribution.ts` (the existing
|
|
17
|
+
* src-mirrors-dashboard-contract pattern); src/ stays pi-agnostic and never
|
|
18
|
+
* imports dashboard types. Status here is computed from the pure signal the
|
|
19
|
+
* rollup owns (a non-empty window) — the route additionally derives the sent
|
|
20
|
+
* `status` via `deriveVcStatus` from the shared vc-status module.
|
|
21
|
+
*
|
|
22
|
+
* Guardrails: PREVENT-PI-004 (zero I/O/network), PREVENT-011 (no `any`).
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import type { DedupAuditEvent } from "../../vectorStore/dedup-audit.js";
|
|
26
|
+
|
|
27
|
+
/** Per-tier dedup/passed counts (same shape as the api-contract DedupTierCounts). */
|
|
28
|
+
export interface DedupTierRollupCounts {
|
|
29
|
+
/** Decisions this tier collapsed onto a matched checkpoint. */
|
|
30
|
+
readonly deduped: number;
|
|
31
|
+
/** Decisions this tier scored a candidate but did not collapse. */
|
|
32
|
+
readonly passed: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The status the rollup can produce from its pure signal. */
|
|
36
|
+
export type RollupStatus = "live" | "awaiting_data";
|
|
37
|
+
|
|
38
|
+
/** Purely-computed tier attribution for one window (mirrors DedupTierRollupV1). */
|
|
39
|
+
export interface DedupTierRollup {
|
|
40
|
+
readonly schema: "dedup-tier-rollup-v1";
|
|
41
|
+
readonly windowStart: string;
|
|
42
|
+
readonly windowEnd: string;
|
|
43
|
+
readonly totalDecisions: number;
|
|
44
|
+
readonly byTier: {
|
|
45
|
+
readonly l0: DedupTierRollupCounts;
|
|
46
|
+
readonly l1: DedupTierRollupCounts;
|
|
47
|
+
readonly l2: DedupTierRollupCounts;
|
|
48
|
+
readonly new: number;
|
|
49
|
+
};
|
|
50
|
+
readonly l0Share: number;
|
|
51
|
+
readonly l1Share: number;
|
|
52
|
+
readonly l2Share: number;
|
|
53
|
+
readonly status: RollupStatus;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
interface Accumulator {
|
|
57
|
+
l0: { deduped: number; passed: number };
|
|
58
|
+
l1: { deduped: number; passed: number };
|
|
59
|
+
l2: { deduped: number; passed: number };
|
|
60
|
+
storedNew: number;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Roll up dedup-audit events over [now - windowMs, now] into per-tier shares.
|
|
65
|
+
*
|
|
66
|
+
* Only events whose ISO-8601 `ts` falls inside the window are counted; events of
|
|
67
|
+
* any other `type` (or a missing/unparseable `ts`) are skipped silently. Shares
|
|
68
|
+
* are fractions of the window's total decisions, each in [0, 1]; an empty window
|
|
69
|
+
* yields all-zero shares with `status:"awaiting_data"` — never a fabricated
|
|
70
|
+
* zero-share table presented as real.
|
|
71
|
+
*/
|
|
72
|
+
export function computeDedupTierRollup(
|
|
73
|
+
events: readonly DedupAuditEvent[],
|
|
74
|
+
windowMs: number,
|
|
75
|
+
now: Date,
|
|
76
|
+
): DedupTierRollup {
|
|
77
|
+
const winMs = windowMs > 0 ? windowMs : 24 * 60 * 60 * 1000;
|
|
78
|
+
const windowStart = now.getTime() - winMs;
|
|
79
|
+
const windowEnd = now.getTime();
|
|
80
|
+
|
|
81
|
+
const acc: Accumulator = {
|
|
82
|
+
l0: { deduped: 0, passed: 0 },
|
|
83
|
+
l1: { deduped: 0, passed: 0 },
|
|
84
|
+
l2: { deduped: 0, passed: 0 },
|
|
85
|
+
storedNew: 0,
|
|
86
|
+
};
|
|
87
|
+
let total = 0;
|
|
88
|
+
|
|
89
|
+
for (const ev of events) {
|
|
90
|
+
if (!ev || typeof ev !== "object") continue;
|
|
91
|
+
if (ev.type !== "dedup_audit") continue;
|
|
92
|
+
const ts = Date.parse(ev.ts);
|
|
93
|
+
if (Number.isNaN(ts) || ts < windowStart || ts > windowEnd) continue;
|
|
94
|
+
total += 1;
|
|
95
|
+
switch (ev.tier) {
|
|
96
|
+
case "L0":
|
|
97
|
+
if (ev.status === "deduped") acc.l0.deduped += 1;
|
|
98
|
+
else if (ev.status === "passed") acc.l0.passed += 1;
|
|
99
|
+
break;
|
|
100
|
+
case "L1":
|
|
101
|
+
if (ev.status === "deduped") acc.l1.deduped += 1;
|
|
102
|
+
else if (ev.status === "passed") acc.l1.passed += 1;
|
|
103
|
+
break;
|
|
104
|
+
case "L2":
|
|
105
|
+
if (ev.status === "deduped") acc.l2.deduped += 1;
|
|
106
|
+
else if (ev.status === "passed") acc.l2.passed += 1;
|
|
107
|
+
break;
|
|
108
|
+
case "new":
|
|
109
|
+
acc.storedNew += 1;
|
|
110
|
+
break;
|
|
111
|
+
default:
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const byTier = {
|
|
117
|
+
l0: { deduped: acc.l0.deduped, passed: acc.l0.passed },
|
|
118
|
+
l1: { deduped: acc.l1.deduped, passed: acc.l1.passed },
|
|
119
|
+
l2: { deduped: acc.l2.deduped, passed: acc.l2.passed },
|
|
120
|
+
new: acc.storedNew,
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
const shareOf = (n: number): number => (total > 0 ? n / total : 0);
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
schema: "dedup-tier-rollup-v1",
|
|
127
|
+
windowStart: new Date(windowStart).toISOString(),
|
|
128
|
+
windowEnd: now.toISOString(),
|
|
129
|
+
totalDecisions: total,
|
|
130
|
+
byTier,
|
|
131
|
+
l0Share: shareOf(acc.l0.deduped + acc.l0.passed),
|
|
132
|
+
l1Share: shareOf(acc.l1.deduped + acc.l1.passed),
|
|
133
|
+
l2Share: shareOf(acc.l2.deduped + acc.l2.passed),
|
|
134
|
+
status: total > 0 ? "live" : "awaiting_data",
|
|
135
|
+
};
|
|
136
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/promotion.ts — ML5-E PromotionV1 ledger row type +
|
|
3
|
+
* append-only manifest helper + atomic digest-swap rollback.
|
|
4
|
+
*
|
|
5
|
+
* Pure-function discipline: zero I/O, no clock, no storage, no network
|
|
6
|
+
* (PREVENT-PI-004 / PREVENT-011). Every function is total over its inputs.
|
|
7
|
+
* The cron orchestrator (scripts/ml5/retrain-nightly.mjs) and the promotion
|
|
8
|
+
* gate (scripts/ml5/promotion-gate.mjs) consume these helpers; the acceptance
|
|
9
|
+
* aggregator (ml5e-acceptance.test.ts) validates them synthetically without a
|
|
10
|
+
* real training run.
|
|
11
|
+
*
|
|
12
|
+
* The promotion manifest is an APPEND-ONLY ledger: every new asset is a new
|
|
13
|
+
* entry, never an overwrite. Rollback restores a prior entry by SHA-256 via
|
|
14
|
+
* an atomic digest swap (flip the committed pointer to the prior entry's
|
|
15
|
+
* digest in one step) — no partial state is possible because the swap is a
|
|
16
|
+
* single value assignment.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
// PromotionV1 — the ledger row recording one nightly training + promotion run.
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
|
|
23
|
+
/** Schema tag for the promotion ledger row. */
|
|
24
|
+
export const PROMOTION_SCHEMA = "promotion-v1" as const;
|
|
25
|
+
|
|
26
|
+
/** Per-head calibration verdict: pass or fail. */
|
|
27
|
+
export interface HeadVerdict {
|
|
28
|
+
readonly head: string;
|
|
29
|
+
readonly pass: boolean;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** PromotionV1 — a single ledger row recording one training + gate run.
|
|
33
|
+
* Digest-only: carries SHA-256 identifiers, never payload content. */
|
|
34
|
+
export interface PromotionV1 {
|
|
35
|
+
readonly schema: typeof PROMOTION_SCHEMA;
|
|
36
|
+
/** ISO-8601 timestamp of the run. */
|
|
37
|
+
readonly ts: string;
|
|
38
|
+
/** SHA-256 of the corpus digest used for this training run. */
|
|
39
|
+
readonly corpusDigest: string;
|
|
40
|
+
/** SHA-256 of the newly trained asset manifest (null when no-op). */
|
|
41
|
+
readonly assetDigest: string | null;
|
|
42
|
+
/** SHA-256 of the previously committed asset (the incumbent). */
|
|
43
|
+
readonly priorAssetDigest: string | null;
|
|
44
|
+
/** Per-head calibration verdicts (all five heads). */
|
|
45
|
+
readonly headVerdicts: readonly HeadVerdict[];
|
|
46
|
+
/** True when all five heads pass their per-head thresholds. */
|
|
47
|
+
readonly fiveHeadsOk: boolean;
|
|
48
|
+
/** True when the new asset beats the incumbent on held-out dev set. */
|
|
49
|
+
readonly heldOutBeat: boolean;
|
|
50
|
+
/** The gate decision: promoted (eligible) or demoted (new asset rejected).
|
|
51
|
+
* `noop` is defined by the spec but unused — the corpus-digest check in
|
|
52
|
+
* retrain-nightly.mjs exits before any ledger append when there is nothing
|
|
53
|
+
* to train on. */
|
|
54
|
+
readonly verdict: "promoted" | "demoted" | "noop";
|
|
55
|
+
/** Present only when verdict is "demoted": the demotion event name. */
|
|
56
|
+
readonly demotedEvent: "demoted_new_asset" | null;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// ---------------------------------------------------------------------------
|
|
60
|
+
// Append-only manifest — every new asset is a NEW entry, never an overwrite.
|
|
61
|
+
// ---------------------------------------------------------------------------
|
|
62
|
+
|
|
63
|
+
/** A single entry in the append-only asset manifest. */
|
|
64
|
+
export interface AssetManifestEntry {
|
|
65
|
+
readonly assetDigest: string;
|
|
66
|
+
readonly ts: string;
|
|
67
|
+
readonly source: string;
|
|
68
|
+
readonly verdict: string;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** The append-only manifest: an ordered list of asset entries plus the
|
|
72
|
+
* committed pointer (the digest of the currently active asset). */
|
|
73
|
+
export interface AssetManifest {
|
|
74
|
+
readonly entries: readonly AssetManifestEntry[];
|
|
75
|
+
/** SHA-256 of the currently committed (active) asset. */
|
|
76
|
+
committed: string | null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Append a new asset entry to the manifest. Never mutates `entries`;
|
|
80
|
+
* returns a new manifest with the entry appended and `committed` updated. */
|
|
81
|
+
export function appendAsset(
|
|
82
|
+
manifest: AssetManifest,
|
|
83
|
+
entry: AssetManifestEntry,
|
|
84
|
+
): AssetManifest {
|
|
85
|
+
return {
|
|
86
|
+
entries: [...manifest.entries, entry],
|
|
87
|
+
committed: entry.assetDigest,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Restore a prior asset by SHA-256: atomically swap the committed pointer
|
|
92
|
+
* to the prior entry. The prior entry MUST exist in the manifest (append-only
|
|
93
|
+
* guarantees it was never overwritten). Returns null when the digest is not
|
|
94
|
+
* found (defensive: append-only means it should always be findable). */
|
|
95
|
+
export function rollbackTo(
|
|
96
|
+
manifest: AssetManifest,
|
|
97
|
+
digest: string,
|
|
98
|
+
): AssetManifest | null {
|
|
99
|
+
const found = manifest.entries.find((e) => e.assetDigest === digest);
|
|
100
|
+
if (!found) return null;
|
|
101
|
+
return {
|
|
102
|
+
entries: manifest.entries,
|
|
103
|
+
committed: found.assetDigest,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// ---------------------------------------------------------------------------
|
|
108
|
+
// Pure decision rules.
|
|
109
|
+
// ---------------------------------------------------------------------------
|
|
110
|
+
|
|
111
|
+
/** Evaluate the promotion gate: promote only when all five heads pass AND the
|
|
112
|
+
* new asset beats the committed asset on the held-out dev set. Otherwise
|
|
113
|
+
* demote. No training data is ever the held-out set. */
|
|
114
|
+
export function promoteDecision(
|
|
115
|
+
fiveHeadsOk: boolean,
|
|
116
|
+
heldOutBeat: boolean,
|
|
117
|
+
): "promoted" | "demoted" {
|
|
118
|
+
return fiveHeadsOk && heldOutBeat ? "promoted" : "demoted";
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Whether a rollback is needed: the current asset at position N regressed
|
|
122
|
+
* relative to the asset at position N-1. True only when regression is
|
|
123
|
+
* confirmed and a prior asset digest exists to restore. */
|
|
124
|
+
export function rollbackNeeded(
|
|
125
|
+
regressionConfirmed: boolean,
|
|
126
|
+
priorAssetDigest: string | null,
|
|
127
|
+
): boolean {
|
|
128
|
+
return regressionConfirmed && priorAssetDigest !== null;
|
|
129
|
+
}
|