pi-mega-compact 0.20.39 → 0.20.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/vector-cortex-ml5d.js +28 -0
- package/dist/config/vector-cortex-ml5e.js +30 -0
- package/dist/config/vector-cortex.js +6 -6
- package/dist/config.js +1 -1
- package/dist/extensions/dashboard-server/api-contracts/cortex-improve.js +14 -0
- package/dist/extensions/dashboard-server/api-contracts/endpoints/registry-ext.js +13 -0
- package/dist/extensions/dashboard-server/route-dispatch.js +5 -1
- package/dist/extensions/dashboard-server/routes-cortex-improve.js +220 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-vector-cortex.js +2 -0
- package/dist/extensions/dashboard-server/routes.js +1 -0
- package/dist/extensions/mega-runtime/dashboard-snapshot.js +2 -2
- package/dist/src/config/vector-cortex-ml5d.js +28 -0
- package/dist/src/config/vector-cortex-ml5e.js +30 -0
- package/dist/src/config/vector-cortex.js +6 -6
- package/dist/src/config.js +1 -1
- package/dist/src/vector-cortex/encoder/promotion.js +58 -0
- package/dist/src/vector-cortex/improve.js +24 -0
- package/dist/vector-cortex/encoder/promotion.js +58 -0
- package/dist/vector-cortex/improve.js +24 -0
- package/extensions/dashboard-client/dist/assets/{AreaChart-DjXp8u2Y.js → AreaChart-BT7YPcp7.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{AreaChart-DjXp8u2Y.js.map → AreaChart-BT7YPcp7.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{BarChart-BZbus-dR.js → BarChart-DceGcBUh.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{BarChart-BZbus-dR.js.map → BarChart-DceGcBUh.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{CacheTab-DQ2CjxVW.js → CacheTab-B9lCouRl.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{CacheTab-DQ2CjxVW.js.map → CacheTab-B9lCouRl.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{EventsTab-DUpnOV1p.js → EventsTab-oqJ11jY7.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{EventsTab-DUpnOV1p.js.map → EventsTab-oqJ11jY7.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{HealthTab-Bul_xjcn.js → HealthTab-Dbosf2Ik.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{HealthTab-Bul_xjcn.js.map → HealthTab-Dbosf2Ik.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DLUjdKd3.js → MaintenanceTab-6TiApCVE.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DLUjdKd3.js.map → MaintenanceTab-6TiApCVE.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-cBsE4mTf.js +2 -0
- package/extensions/dashboard-client/dist/assets/{MemoryMapTab-BFy0hb6F.js.map → MemoryMapTab-cBsE4mTf.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MetricsTab-BpYuUkq3.js → MetricsTab-CX-X9hhy.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MetricsTab-BpYuUkq3.js.map → MetricsTab-CX-X9hhy.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{OverviewTab-z7yuT-uq.js → OverviewTab-CpPYaTJF.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{OverviewTab-z7yuT-uq.js.map → OverviewTab-CpPYaTJF.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{ReposTab-ClORoDy_.js → ReposTab-D_Ot4QbU.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{ReposTab-ClORoDy_.js.map → ReposTab-D_Ot4QbU.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SessionsTab-CUpehIt9.js → SessionsTab-BUzO3iGK.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SessionsTab-CUpehIt9.js.map → SessionsTab-BUzO3iGK.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SetupTab-Cii80Fdz.js → SetupTab-Be4w9v4j.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SetupTab-Cii80Fdz.js.map → SetupTab-Be4w9v4j.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-BV2iCSdV.js → TimeSavedCard-Cx-g6CXH.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-BV2iCSdV.js.map → TimeSavedCard-Cx-g6CXH.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TurnsTab-1T86rHnD.js → TurnsTab-CC_7T7ad.js} +3 -3
- package/extensions/dashboard-client/dist/assets/{TurnsTab-1T86rHnD.js.map → TurnsTab-CC_7T7ad.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{VcStatusBadge-DKPleja8.js → VcStatusBadge-ZTItXSiH.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{VcStatusBadge-DKPleja8.js.map → VcStatusBadge-ZTItXSiH.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/VectorCortexTab-DgAUZ4Pg.js +2 -0
- package/extensions/dashboard-client/dist/assets/VectorCortexTab-DgAUZ4Pg.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/WikiTab-DnKupXTv.js +2 -0
- package/extensions/dashboard-client/dist/assets/{WikiTab-DKTILa1M.js.map → WikiTab-DnKupXTv.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/button-CFt3e0yc.js +2 -0
- package/extensions/dashboard-client/dist/assets/{button-5fSU8BZj.js.map → button-CFt3e0yc.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{card-BFUnpXE_.js → card-C6ZjsHrl.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{card-BFUnpXE_.js.map → card-C6ZjsHrl.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/client-extra-DdEdGTIe.js +2 -0
- package/extensions/dashboard-client/dist/assets/client-extra-DdEdGTIe.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-xCzi6pWn.js → generateCategoricalChart-C9_fu2hc.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-xCzi6pWn.js.map → generateCategoricalChart-C9_fu2hc.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{index-DqjmZ88m.js → index-DY81TY4f.js} +11 -11
- package/extensions/dashboard-client/dist/assets/index-DY81TY4f.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{switch-CeZUPf_e.js → switch-BOqa1d6x.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{switch-CeZUPf_e.js.map → switch-BOqa1d6x.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{toggle-MQQwdkwD.js → toggle-DToFaIEd.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{toggle-MQQwdkwD.js.map → toggle-DToFaIEd.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{useSSE-BetS7QHN.js → useSSE-CZqkhU18.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{useSSE-BetS7QHN.js.map → useSSE-CZqkhU18.js.map} +1 -1
- package/extensions/dashboard-client/dist/index.html +1 -1
- package/extensions/dashboard-client/src/api/client-extra.ts +19 -1
- package/extensions/dashboard-client/src/components/ModelImprovementCard.tsx +141 -0
- package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +6 -0
- package/extensions/dashboard-client/src/types/cortex-improve.ts +42 -0
- package/extensions/dashboard-server/api-contracts/cortex-improve.ts +82 -0
- package/extensions/dashboard-server/api-contracts/endpoints/registry-ext.ts +31 -0
- package/extensions/dashboard-server/api-contracts/index.ts +12 -0
- package/extensions/dashboard-server/route-dispatch.ts +4 -0
- package/extensions/dashboard-server/routes-cortex-improve.ts +256 -0
- package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts +12 -0
- package/extensions/dashboard-server/routes.ts +4 -0
- package/extensions/mega-runtime/dashboard-snapshot.ts +2 -2
- package/package.json +1 -1
- package/src/config/vector-cortex-ml5d.ts +30 -0
- package/src/config/vector-cortex-ml5e.ts +32 -0
- package/src/config/vector-cortex.ts +6 -6
- package/src/config.ts +2 -0
- package/src/vector-cortex/encoder/promotion.ts +129 -0
- package/src/vector-cortex/improve.ts +28 -0
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-BFy0hb6F.js +0 -2
- package/extensions/dashboard-client/dist/assets/VectorCortexTab-BRzvd_pU.js +0 -2
- package/extensions/dashboard-client/dist/assets/VectorCortexTab-BRzvd_pU.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/WikiTab-DKTILa1M.js +0 -2
- package/extensions/dashboard-client/dist/assets/button-5fSU8BZj.js +0 -2
- package/extensions/dashboard-client/dist/assets/client-extra-BZbw21wu.js +0 -2
- package/extensions/dashboard-client/dist/assets/client-extra-BZbw21wu.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/index-DqjmZ88m.js.map +0 -1
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dashboard-client/src/types/cortex-improve.ts — ML5-D Improve Cortex client types.
|
|
3
|
+
*
|
|
4
|
+
* Client-side mirrors of the server contract (extensions/dashboard-server/
|
|
5
|
+
* api-contracts/cortex-improve.ts), kept in lockstep so the ModelImprovementCard
|
|
6
|
+
* is fully typed end-to-end. The client bundle does not import the server tree,
|
|
7
|
+
* so these are redeclared here (PREVENT-011: no `any`; every type explicit).
|
|
8
|
+
* Aggregate values only — mode/verdict/digest/reason/progress, never payload
|
|
9
|
+
* bytes or corpus content (EVAL-REDACT-002).
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/** The five-head qualification verdict projected by an improve job. */
|
|
13
|
+
export interface QualificationV1 {
|
|
14
|
+
readonly mode: "A" | "B" | "C";
|
|
15
|
+
readonly assetDigestPrefix: string | null;
|
|
16
|
+
readonly verdict: "qualified" | "demoted";
|
|
17
|
+
readonly reasonCode?: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Response body for POST /api/cortex/improve. */
|
|
21
|
+
export interface CortexImproveStart {
|
|
22
|
+
readonly status: "improving";
|
|
23
|
+
readonly jobId: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Response body for GET /api/cortex/improve/status/:jobId. */
|
|
27
|
+
export type CortexImproveStatus =
|
|
28
|
+
| {
|
|
29
|
+
readonly status: "improving";
|
|
30
|
+
readonly progress: number;
|
|
31
|
+
}
|
|
32
|
+
| {
|
|
33
|
+
readonly status: "qualified";
|
|
34
|
+
readonly progress: number;
|
|
35
|
+
readonly verdict: QualificationV1;
|
|
36
|
+
readonly assetDigest: string;
|
|
37
|
+
}
|
|
38
|
+
| {
|
|
39
|
+
readonly status: "demoted_to_B";
|
|
40
|
+
readonly progress: number;
|
|
41
|
+
readonly reason: string;
|
|
42
|
+
};
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* api-contracts/cortex-improve.ts — ML5-D "Improve Cortex" job API types.
|
|
3
|
+
*
|
|
4
|
+
* Types for POST /api/cortex/improve (launch a local ML5-A training job that
|
|
5
|
+
* re-qualifies the five heads against the latest local corpus) and GET
|
|
6
|
+
* /api/cortex/improve/status/:jobId (poll the in-process job state to a
|
|
7
|
+
* terminal qualified / demoted_to_B verdict). Local-only, aggregate surfaces:
|
|
8
|
+
* mode / verdict / digest / reason / progress — never message content or corpus
|
|
9
|
+
* rows (EVAL-REDACT-002).
|
|
10
|
+
*
|
|
11
|
+
* PREVENT-PI-004: type definitions only, no network code.
|
|
12
|
+
* PREVENT-011: no `any` type — all types are explicit.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The five-head qualification verdict projected from the improve job.
|
|
17
|
+
* Aggregate-only: current surrogate mode + asset digest prefix + a reason code
|
|
18
|
+
* when the asset failed to qualify (never weights or corpus content).
|
|
19
|
+
*/
|
|
20
|
+
export interface QualificationV1 {
|
|
21
|
+
/** Surrogate encoder mode the improve job would promote to on success. */
|
|
22
|
+
readonly mode: "A" | "B" | "C";
|
|
23
|
+
/** First 12 hex chars of the produced asset manifest digest, or null. */
|
|
24
|
+
readonly assetDigestPrefix: string | null;
|
|
25
|
+
/** Terminal verdict: qualified when the heads fit within gate, else demoted. */
|
|
26
|
+
readonly verdict: "qualified" | "demoted";
|
|
27
|
+
/** A stable reason code when verdict is "demoted" (e.g. ENC_ASSET_UNREADABLE). */
|
|
28
|
+
readonly reasonCode?: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Response body for POST /api/cortex/improve (confirmation-gated). */
|
|
32
|
+
export interface CortexImproveStart {
|
|
33
|
+
/** Always "improving" immediately after the job is spawned. */
|
|
34
|
+
readonly status: "improving";
|
|
35
|
+
/** Opaque job token (sha256 of ts+random, hex-sliced), pollable via status. */
|
|
36
|
+
readonly jobId: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Lifecycle status of a Cortex improve job (in-process, restart-scoped). */
|
|
40
|
+
export type CortexImproveJobStatus =
|
|
41
|
+
| "improving"
|
|
42
|
+
| "qualified"
|
|
43
|
+
| "demoted_to_B";
|
|
44
|
+
|
|
45
|
+
/** Progressing (non-terminal) response for GET /api/cortex/improve/status/:jobId. */
|
|
46
|
+
export interface CortexImproveProgress {
|
|
47
|
+
readonly status: "improving";
|
|
48
|
+
/** 0..1 progress through the local ML5-A training run. */
|
|
49
|
+
readonly progress: number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Terminal success: the five heads re-qualified under mode A. */
|
|
53
|
+
export interface CortexImproveQualified {
|
|
54
|
+
readonly status: "qualified";
|
|
55
|
+
readonly progress: number;
|
|
56
|
+
readonly verdict: QualificationV1;
|
|
57
|
+
readonly assetDigest: string;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Terminal demotion: the re-trained heads failed to qualify — stay mode B. */
|
|
61
|
+
export interface CortexImproveDemoted {
|
|
62
|
+
readonly status: "demoted_to_B";
|
|
63
|
+
readonly progress: number;
|
|
64
|
+
readonly reason: string;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Response body for GET /api/cortex/improve/status/:jobId. */
|
|
68
|
+
export type CortexImproveStatus =
|
|
69
|
+
| CortexImproveProgress
|
|
70
|
+
| CortexImproveQualified
|
|
71
|
+
| CortexImproveDemoted;
|
|
72
|
+
|
|
73
|
+
/** Error body returned when the improve surface is disabled (flag off). */
|
|
74
|
+
export interface CortexImproveDisabled {
|
|
75
|
+
readonly error: "disabled";
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Request body for POST /api/cortex/improve. `confirm:true` is the server-side hard requirement. */
|
|
79
|
+
export interface CortexImproveStartRequest {
|
|
80
|
+
/** Required explicit confirmation. Absent/false → 400 confirmation_required. */
|
|
81
|
+
readonly confirm: boolean;
|
|
82
|
+
}
|
|
@@ -25,6 +25,11 @@ import type {
|
|
|
25
25
|
SetupCortexActionLogResponse,
|
|
26
26
|
} from "../setup-cortex.js";
|
|
27
27
|
import type { PrefixStabilityResponse } from "../prefix-stability.js";
|
|
28
|
+
import type {
|
|
29
|
+
CortexImproveStartRequest,
|
|
30
|
+
CortexImproveStart,
|
|
31
|
+
CortexImproveStatus,
|
|
32
|
+
} from "../cortex-improve.js";
|
|
28
33
|
|
|
29
34
|
/** Additive endpoint entries spread into the ENDPOINTS registry. */
|
|
30
35
|
export const EXTRA_ENDPOINTS = {
|
|
@@ -123,4 +128,30 @@ export const EXTRA_ENDPOINTS = {
|
|
|
123
128
|
undefined,
|
|
124
129
|
PrefixStabilityResponse
|
|
125
130
|
>,
|
|
131
|
+
|
|
132
|
+
// ─── ML5-D Improve Cortex (dashboard Vector Cortex tab) ───────────
|
|
133
|
+
|
|
134
|
+
/** POST /api/cortex/improve — launch a local ML5-A training job. */
|
|
135
|
+
improveCortex: {
|
|
136
|
+
method: "POST",
|
|
137
|
+
path: "/api/cortex/improve",
|
|
138
|
+
description:
|
|
139
|
+
"Launch a local ML5-A training job that re-qualifies the five heads against the latest local corpus and returns an opaque jobId. Requires confirm:true; flag-off returns 404. Local-only — never fetches anything.",
|
|
140
|
+
} as const satisfies EndpointDef<
|
|
141
|
+
"POST",
|
|
142
|
+
CortexImproveStartRequest,
|
|
143
|
+
CortexImproveStart
|
|
144
|
+
>,
|
|
145
|
+
|
|
146
|
+
/** GET /api/cortex/improve/status/:jobId — poll an improve job. */
|
|
147
|
+
improveCortexStatus: {
|
|
148
|
+
method: "GET",
|
|
149
|
+
path: "/api/cortex/improve/status/:jobId",
|
|
150
|
+
description:
|
|
151
|
+
"Poll an in-process Cortex improve job to terminal qualified / demoted_to_B. Unknown jobId or flag-off returns 404. Read-only, in-memory job state.",
|
|
152
|
+
} as const satisfies EndpointDef<
|
|
153
|
+
"GET",
|
|
154
|
+
undefined,
|
|
155
|
+
CortexImproveStatus
|
|
156
|
+
>,
|
|
126
157
|
} as const;
|
|
@@ -323,3 +323,15 @@ export type {
|
|
|
323
323
|
ModelThresholdDeleteResponse,
|
|
324
324
|
ModelThresholdsError,
|
|
325
325
|
} from "./model-thresholds.js";
|
|
326
|
+
|
|
327
|
+
// ML5-D — dashboard "Improve Cortex" job API
|
|
328
|
+
export type {
|
|
329
|
+
QualificationV1,
|
|
330
|
+
CortexImproveStart,
|
|
331
|
+
CortexImproveStatus,
|
|
332
|
+
CortexImproveProgress,
|
|
333
|
+
CortexImproveQualified,
|
|
334
|
+
CortexImproveDemoted,
|
|
335
|
+
CortexImproveDisabled,
|
|
336
|
+
CortexImproveStartRequest,
|
|
337
|
+
} from "./cortex-improve.js";
|
|
@@ -60,6 +60,8 @@ import {
|
|
|
60
60
|
handleSetupCortexStatus,
|
|
61
61
|
handleSetupCortexAction,
|
|
62
62
|
handleSetupCortexActionLog,
|
|
63
|
+
handleImproveCortex,
|
|
64
|
+
handleImproveCortexStatus,
|
|
63
65
|
} from "./routes.js";
|
|
64
66
|
// VC6C repair lives in its own module (routes-vector-cortex-repair.ts) so the
|
|
65
67
|
// heal route file stays well under the 400-line extension soft limit.
|
|
@@ -154,5 +156,7 @@ export function dispatchRoutes(
|
|
|
154
156
|
if (handleSetupCortexStatus(req, res, ctx)) return true;
|
|
155
157
|
if (handleSetupCortexAction(req, res, ctx)) return true;
|
|
156
158
|
if (handleSetupCortexActionLog(req, res, ctx)) return true;
|
|
159
|
+
if (handleImproveCortex(req, res, ctx)) return true;
|
|
160
|
+
if (handleImproveCortexStatus(req, res, ctx)) return true;
|
|
157
161
|
return false;
|
|
158
162
|
}
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dashboard-server/routes-cortex-improve.ts — ML5-D "Improve Cortex" job routes.
|
|
3
|
+
*
|
|
4
|
+
* POST /api/cortex/improve — confirmation-gated local ML5-A training
|
|
5
|
+
* GET /api/cortex/improve/status/:jobId — poll an in-process job to terminal
|
|
6
|
+
*
|
|
7
|
+
* The Improve action launches the committed ML5-A training pipeline
|
|
8
|
+
* (training/vector-cortex/train.py) as a background child_process against the
|
|
9
|
+
* latest local corpus, then re-qualifies the five heads. Job state is kept
|
|
10
|
+
* in-process in a Map<jobId, JobState> on this module; nothing is persisted
|
|
11
|
+
* across restarts (the status endpoint is read-only, in-memory).
|
|
12
|
+
*
|
|
13
|
+
* LOCAL ONLY (PREVENT-PI-004): the job spawns a local python process and reads
|
|
14
|
+
* local files — never a fetch. The `onnxruntime-node` native path is attempted
|
|
15
|
+
* only when MEGACOMPACT_ENCODER_NATIVE=1 (default OFF); otherwise the harness
|
|
16
|
+
* uses the WASM/trigram fallback. The status payload surfaces mode/verdict/
|
|
17
|
+
* digest/reason/progress only — never message or corpus content (EVAL-REDACT-002).
|
|
18
|
+
*
|
|
19
|
+
* Guardrails: PREVENT-011 (no `any`), PREVENT-001 (guarded JSON.parse via
|
|
20
|
+
* readJsonBody), flag-off byte-identical (no card + endpoints 404).
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
24
|
+
import { spawn } from "node:child_process";
|
|
25
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
26
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
27
|
+
import { join, dirname } from "node:path";
|
|
28
|
+
import { fileURLToPath } from "node:url";
|
|
29
|
+
import type { RouteContext } from "./routes-core.js";
|
|
30
|
+
import { sendJson, readJsonBody } from "./routes-vector-cortex-shared.js";
|
|
31
|
+
import { ML5D_ENABLED } from "../../src/config.js";
|
|
32
|
+
import { qualifyDecision } from "../../src/vector-cortex/improve.js";
|
|
33
|
+
import type {
|
|
34
|
+
CortexImproveStart,
|
|
35
|
+
CortexImproveStatus,
|
|
36
|
+
QualificationV1,
|
|
37
|
+
} from "./api-contracts/cortex-improve.js";
|
|
38
|
+
|
|
39
|
+
/** Resolve the committed training entry by walking up to the repo root. */
|
|
40
|
+
function trainingScript(): string | null {
|
|
41
|
+
let dir = dirname(fileURLToPath(import.meta.url));
|
|
42
|
+
const rel = join("training", "vector-cortex", "train.py");
|
|
43
|
+
for (let i = 0; i < 8; i++) {
|
|
44
|
+
// guardrails-allow PREVENT-PI-004: local script read (loopback)
|
|
45
|
+
const candidate = join(dir, rel);
|
|
46
|
+
if (existsSync(candidate)) return candidate;
|
|
47
|
+
const next = dirname(dir);
|
|
48
|
+
if (next === dir) break;
|
|
49
|
+
dir = next;
|
|
50
|
+
}
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** An in-process improve job (restart-scoped, never persisted). */
|
|
55
|
+
interface JobState {
|
|
56
|
+
status: "improving" | "qualified" | "demoted_to_B";
|
|
57
|
+
progress: number;
|
|
58
|
+
verdict?: QualificationV1;
|
|
59
|
+
assetDigest?: string;
|
|
60
|
+
reason?: string;
|
|
61
|
+
updatedAt: string;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** In-process job registry; nothing survives a server restart. */
|
|
65
|
+
const JOBS = new Map<string, JobState>();
|
|
66
|
+
|
|
67
|
+
/** Opaque job token: sha256(ts+random) hex-sliced. */
|
|
68
|
+
function newJobId(): string {
|
|
69
|
+
return createHash("sha256")
|
|
70
|
+
.update(`${Date.now()}:${randomBytes(16).toString("hex")}`)
|
|
71
|
+
.digest("hex")
|
|
72
|
+
.slice(0, 24);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Spawn the ML5-A training pipeline in the background and drive the job state
|
|
77
|
+
* to a terminal qualified / demoted_to_B verdict. Honest: an empty corpus (the
|
|
78
|
+
* host state) makes train.py no-op and the job ends demoted_to_B.
|
|
79
|
+
*/
|
|
80
|
+
function startJob(jobId: string): void {
|
|
81
|
+
const script = trainingScript();
|
|
82
|
+
const boot = Date.now();
|
|
83
|
+
JOBS.set(jobId, { status: "improving", progress: 0, updatedAt: new Date().toISOString() });
|
|
84
|
+
|
|
85
|
+
if (script === null) {
|
|
86
|
+
settle(jobId, { status: "demoted_to_B", reason: "ENC_TRAIN_PIPELINE_ABSENT", progress: 1 });
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const env = { ...process.env };
|
|
91
|
+
const proc = spawn("python3", [script], {
|
|
92
|
+
env,
|
|
93
|
+
cwd: join(dirname(script), "..", ".."),
|
|
94
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
let output = "";
|
|
98
|
+
const step = () => {
|
|
99
|
+
const pct = Math.min(0.9, 0.1 + ((Date.now() - boot) / 60000) * 0.6);
|
|
100
|
+
const cur = JOBS.get(jobId);
|
|
101
|
+
if (cur && cur.status === "improving" && pct > cur.progress) {
|
|
102
|
+
JOBS.set(jobId, { ...cur, progress: pct, updatedAt: new Date().toISOString() });
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
const timer = setInterval(step, 2000);
|
|
106
|
+
const stop = () => clearInterval(timer);
|
|
107
|
+
|
|
108
|
+
proc.stdout?.on("data", (d: Buffer) => {
|
|
109
|
+
output += d.toString();
|
|
110
|
+
});
|
|
111
|
+
proc.stderr?.on("data", (d: Buffer) => {
|
|
112
|
+
output += d.toString();
|
|
113
|
+
});
|
|
114
|
+
proc.on("error", (err) => {
|
|
115
|
+
stop();
|
|
116
|
+
settle(jobId, {
|
|
117
|
+
status: "demoted_to_B",
|
|
118
|
+
reason: `ENC_TRAIN_SPAWN_FAILED:${err.message}`,
|
|
119
|
+
progress: 1,
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
proc.on("close", (code) => {
|
|
123
|
+
stop();
|
|
124
|
+
// Read the produced trained-heads artifact digest. The report line includes
|
|
125
|
+
// `trainedHeadsPath`; verify the file exists after a successful run.
|
|
126
|
+
const match = /trainedHeadsPath":\s*"([^"]+)"/.exec(output);
|
|
127
|
+
const assetPath = match?.[1];
|
|
128
|
+
let digest: string | null = null;
|
|
129
|
+
if (assetPath) {
|
|
130
|
+
// guardrails-allow PREVENT-PI-004: local artifact read (loopback)
|
|
131
|
+
try {
|
|
132
|
+
digest = createHash("sha256")
|
|
133
|
+
.update(readFileSync(assetPath))
|
|
134
|
+
.digest("hex")
|
|
135
|
+
.slice(0, 12);
|
|
136
|
+
} catch {
|
|
137
|
+
digest = null;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
const decision = qualifyDecision(code ?? -1, digest);
|
|
141
|
+
if (decision === "qualified") {
|
|
142
|
+
settle(jobId, {
|
|
143
|
+
status: "qualified",
|
|
144
|
+
progress: 1,
|
|
145
|
+
assetDigest: digest ?? undefined,
|
|
146
|
+
verdict: { mode: "A", assetDigestPrefix: digest ?? null, verdict: "qualified" },
|
|
147
|
+
});
|
|
148
|
+
} else {
|
|
149
|
+
settle(jobId, {
|
|
150
|
+
status: "demoted_to_B",
|
|
151
|
+
reason: assetPath
|
|
152
|
+
? "trained asset did not verify — demoted to mode B"
|
|
153
|
+
: "empty corpus (no groups) — no qualified asset emitted",
|
|
154
|
+
progress: 1,
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Write a terminal job state (idempotent — only advances from improving). */
|
|
161
|
+
function settle(
|
|
162
|
+
jobId: string,
|
|
163
|
+
terminal: { status: "qualified" | "demoted_to_B"; progress: number; verdict?: QualificationV1; assetDigest?: string; reason?: string },
|
|
164
|
+
): void {
|
|
165
|
+
const cur = JOBS.get(jobId);
|
|
166
|
+
if (cur && cur.status !== "improving") return;
|
|
167
|
+
JOBS.set(jobId, { ...terminal, updatedAt: new Date().toISOString() });
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Flag-off response, byte-identical regardless of request (ML5-D absent). */
|
|
171
|
+
function sendDisabled(res: ServerResponse): void {
|
|
172
|
+
sendJson(res, 404, { error: "disabled" });
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* POST /api/cortex/improve (ML5-D). Returns true when it claims the request.
|
|
177
|
+
* Requires `confirm:true` server-side (mirrors the client window.confirm).
|
|
178
|
+
*/
|
|
179
|
+
export function handleImproveCortex(
|
|
180
|
+
req: IncomingMessage,
|
|
181
|
+
res: ServerResponse,
|
|
182
|
+
_ctx: RouteContext,
|
|
183
|
+
): boolean {
|
|
184
|
+
const url = req.url ?? "";
|
|
185
|
+
if (url !== "/api/cortex/improve") return false;
|
|
186
|
+
if (req.method !== "POST") {
|
|
187
|
+
sendJson(res, 405, { error: "method_not_allowed" });
|
|
188
|
+
return true;
|
|
189
|
+
}
|
|
190
|
+
if (!ML5D_ENABLED()) {
|
|
191
|
+
sendDisabled(res);
|
|
192
|
+
return true;
|
|
193
|
+
}
|
|
194
|
+
readJsonBody(req, (parsed) => {
|
|
195
|
+
if (!parsed.ok) {
|
|
196
|
+
sendJson(res, 400, { error: "invalid_body" });
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
if (parsed.value.confirm !== true) {
|
|
200
|
+
sendJson(res, 400, { error: "confirmation_required" });
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
const jobId = newJobId();
|
|
204
|
+
startJob(jobId);
|
|
205
|
+
const body: CortexImproveStart = { status: "improving", jobId };
|
|
206
|
+
sendJson(res, 200, body);
|
|
207
|
+
});
|
|
208
|
+
return true;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* GET /api/cortex/improve/status/:jobId (ML5-D). Returns true when it claims the
|
|
213
|
+
* request. Flag-off or unknown jobId → 404. Read-only in-memory.
|
|
214
|
+
*/
|
|
215
|
+
export function handleImproveCortexStatus(
|
|
216
|
+
req: IncomingMessage,
|
|
217
|
+
res: ServerResponse,
|
|
218
|
+
_ctx: RouteContext,
|
|
219
|
+
): boolean {
|
|
220
|
+
const url = req.url ?? "";
|
|
221
|
+
const match = /^\/api\/cortex\/improve\/status\/([A-Za-z0-9]+)$/.exec(url);
|
|
222
|
+
if (!match) return false;
|
|
223
|
+
if (req.method !== "GET") {
|
|
224
|
+
sendJson(res, 405, { error: "method_not_allowed" });
|
|
225
|
+
return true;
|
|
226
|
+
}
|
|
227
|
+
if (!ML5D_ENABLED()) {
|
|
228
|
+
sendDisabled(res);
|
|
229
|
+
return true;
|
|
230
|
+
}
|
|
231
|
+
const job = JOBS.get(match[1]);
|
|
232
|
+
if (!job) {
|
|
233
|
+
sendJson(res, 404, { error: "job_not_found" });
|
|
234
|
+
return true;
|
|
235
|
+
}
|
|
236
|
+
// Narrow job.status into the discriminated CortexImproveStatus variant. The
|
|
237
|
+
// terminal shapes require their payload fields, so construct per-state.
|
|
238
|
+
const body: CortexImproveStatus =
|
|
239
|
+
job.status === "improving"
|
|
240
|
+
? { status: "improving", progress: job.progress }
|
|
241
|
+
: job.status === "qualified"
|
|
242
|
+
? {
|
|
243
|
+
status: "qualified",
|
|
244
|
+
progress: job.progress,
|
|
245
|
+
verdict:
|
|
246
|
+
job.verdict ?? { mode: "A", assetDigestPrefix: null, verdict: "qualified" },
|
|
247
|
+
assetDigest: job.assetDigest ?? "",
|
|
248
|
+
}
|
|
249
|
+
: {
|
|
250
|
+
status: "demoted_to_B",
|
|
251
|
+
progress: job.progress,
|
|
252
|
+
reason: job.reason ?? "UNKNOWN_TERMINAL",
|
|
253
|
+
};
|
|
254
|
+
sendJson(res, 200, body);
|
|
255
|
+
return true;
|
|
256
|
+
}
|
|
@@ -234,5 +234,17 @@ export const VECTOR_CORTEX_SETTINGS: SettingGroup = {
|
|
|
234
234
|
"ML5-C runtime backend selection (WASM vs native): selects the ONNX runtime backend based on the ML5-B bench record and platform support. ON (default) = the runtime-selection dispatch runs and emits vector_cortex_runtime_selected. OFF = no selection runs — encoder serves mode B trigram, byte-identical to ML5-B.",
|
|
235
235
|
true,
|
|
236
236
|
),
|
|
237
|
+
boolDirect(
|
|
238
|
+
"MEGACOMPACT_ML5_D",
|
|
239
|
+
"ML5-D Dashboard Improve Cortex",
|
|
240
|
+
"ML5-D dashboard 'Improve Cortex' surface: the ModelImprovementCard + POST /api/cortex/improve + GET /api/cortex/improve/status/:jobId. ON (default) = the card renders and Improve launches a local ML5-A training job. OFF = both improve endpoints return 404 and VectorCortexTab omits the card, byte-identical to ML5-C.",
|
|
241
|
+
true,
|
|
242
|
+
),
|
|
243
|
+
boolDirect(
|
|
244
|
+
"MEGACOMPACT_ML5_E",
|
|
245
|
+
"ML5-E Nightly Retraining Cron",
|
|
246
|
+
"ML5-E nightly retraining + feedback loop: the user's own post-redaction conversation turns become fresh training signal, the five heads are re-fit nightly via system cron, calibration is re-validated, and mode-A promotion is re-checked without human intervention. The cron is system-configured (crontab -e); the extension never installs or writes a crontab. Candidates are written to ~/.pi/mega-compact-encoder/candidates/ and promoted via the ML5-D Improve Cortex flow. ON (default) = the scripts may be invoked by the system cron. OFF = scripts are never invoked; byte-identical to ML5-D.",
|
|
247
|
+
true,
|
|
248
|
+
),
|
|
237
249
|
],
|
|
238
250
|
};
|
|
@@ -159,8 +159,8 @@ export function buildDashboardSnapshot(ctx: SnapshotBuildContext): DashboardSnap
|
|
|
159
159
|
// per-session in rt and there's no cumulative counter yet.
|
|
160
160
|
total: ctx.repo.dedupCollapsed,
|
|
161
161
|
sessionTokensSaved: ctx.rt.cacheHitTokens,
|
|
162
|
-
// Cumulative tokens saved across all sessions in this repo
|
|
163
|
-
//
|
|
162
|
+
// Cumulative tokens saved across all sessions in this repo = the
|
|
163
|
+
// real repo counter (audit Table 1 stub 8 closed — no rolled-up math).
|
|
164
164
|
totalTokensSaved: ctx.repo.tokensSaved,
|
|
165
165
|
},
|
|
166
166
|
compacts: {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.20.
|
|
3
|
+
"version": "0.20.41",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-3-Clause",
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* config/vector-cortex-ml5d.ts — ML5-D dashboard "Improve Cortex" flag.
|
|
3
|
+
*
|
|
4
|
+
* Sibling extract mirroring vector-cortex-ml5a.ts / vector-cortex-ml5b.ts /
|
|
5
|
+
* vector-cortex-ml5c.ts, so vector-cortex.ts stays under its 300-line soft
|
|
6
|
+
* limit (soft-as-hard gate). This is the dashboard "Improve Cortex" surface +
|
|
7
|
+
* promote workflow sprint flag.
|
|
8
|
+
* vector-cortex.ts re-exports the ENUM below and root src/config.ts re-exports
|
|
9
|
+
* it, so no consumer import path changes.
|
|
10
|
+
*
|
|
11
|
+
* ML5-D adds the dashboard ModelImprovementCard + the POST /api/cortex/improve
|
|
12
|
+
* and GET /api/cortex/improve/status/:jobId endpoints. The flag gates that
|
|
13
|
+
* surface only; when OFF the endpoints return 404/disabled and VectorCortexTab
|
|
14
|
+
* renders exactly as before (byte-identical to the ML5-C-era tab — no
|
|
15
|
+
* ModelImprovementCard, no improve job is ever spawned).
|
|
16
|
+
*
|
|
17
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { sprintFlag } from "./vector-cortex-flag.js";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* ML5-D — dashboard "Improve Cortex" surface + promote workflow. Default ON.
|
|
24
|
+
* `MEGACOMPACT_ML5_D=0` disables and is byte-identical to the ML5-C survivor:
|
|
25
|
+
* both `/api/cortex/improve*` endpoints return 404 and VectorCortexTab omits the
|
|
26
|
+
* ModelImprovementCard. The flag gates the dashboard surface only; it does not
|
|
27
|
+
* gate the underlying ML5-A training pipeline or the encoder's mode-A/B/C
|
|
28
|
+
* selection (those are governed by ML5-A and the encoder independently).
|
|
29
|
+
*/
|
|
30
|
+
export const ML5D_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_ML5_D");
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* config/vector-cortex-ml5e.ts — ML5-E nightly retraining cron + feedback loop.
|
|
3
|
+
*
|
|
4
|
+
* Sibling extract mirroring vector-cortex-ml5a.ts / ml5b.ts / ml5c.ts / ml5d.ts,
|
|
5
|
+
* so vector-cortex.ts stays under its 300-line soft limit (soft-as-hard gate).
|
|
6
|
+
* vector-cortex.ts re-exports the ENUM below and root src/config.ts re-exports
|
|
7
|
+
* it, so no consumer import path changes.
|
|
8
|
+
*
|
|
9
|
+
* ML5-E turns one-shot training (ML5-D's Improve button) into a living loop:
|
|
10
|
+
* the user's own post-redaction conversation turns become fresh training signal,
|
|
11
|
+
* the five heads are re-fit nightly, calibration is re-validated, and mode-A
|
|
12
|
+
* promotion is re-checked without human intervention. The cron is system-
|
|
13
|
+
* configured (`crontab -e` on the operator device); the extension never installs
|
|
14
|
+
* or writes a crontab. The flag gates **invocation** only — the scripts
|
|
15
|
+
* (retrain-nightly.mjs, promotion-gate.mjs, promotion.ts) exist on disk
|
|
16
|
+
* regardless; `=0` means the runtime never invokes them and the dashboard's
|
|
17
|
+
* Improve Cortex flow is byte-identical to the ML5-D survivor.
|
|
18
|
+
*
|
|
19
|
+
* Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { sprintFlag } from "./vector-cortex-flag.js";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* ML5-E — nightly retraining cron + feedback loop. Default ON.
|
|
26
|
+
* `MEGACOMPACT_ML5_E=0` disables and is byte-identical to the ML5-D survivor:
|
|
27
|
+
* the nightly retraining scripts are never invoked by the runtime, the
|
|
28
|
+
* promotion gate is inert, and the dashboard's Improve Cortex flow is
|
|
29
|
+
* byte-identical to the ML5-D tab. The flag gates invocation only — the scripts
|
|
30
|
+
* exist on disk regardless.
|
|
31
|
+
*/
|
|
32
|
+
export const ML5E_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_ML5_E");
|
|
@@ -11,9 +11,8 @@
|
|
|
11
11
|
|
|
12
12
|
import { sprintFlag } from "./vector-cortex-flag.js";
|
|
13
13
|
|
|
14
|
-
// VC0/VC1/VC2 foundation-phase flags
|
|
15
|
-
//
|
|
16
|
-
// `from "./config/vector-cortex.js"` import keeps resolving unchanged.
|
|
14
|
+
// VC0/VC1/VC2 foundation-phase flags in vector-cortex-early.ts; re-exported
|
|
15
|
+
// so every existing `from "./config/vector-cortex.js"` import resolves unchanged.
|
|
17
16
|
export {
|
|
18
17
|
VC0A_ENABLED,
|
|
19
18
|
VC0B_ENABLED,
|
|
@@ -269,10 +268,9 @@ export const VC8A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8A");
|
|
|
269
268
|
*/
|
|
270
269
|
export const VC8B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8B");
|
|
271
270
|
|
|
272
|
-
// VC8C
|
|
273
|
-
// re-exported so existing `./config/vector-cortex.js` imports keep resolving.
|
|
271
|
+
// VC8C extracted to vector-cortex-vc8c.ts; re-exported so existing imports keep resolving.
|
|
274
272
|
export { VC8C_ENABLED } from "./vector-cortex-vc8c.js";
|
|
275
|
-
// Sibling extracts
|
|
273
|
+
// Sibling extracts: stay under the 300-line soft limit.
|
|
276
274
|
export { VC9A_ENABLED } from "./vector-cortex-vc9a.js";
|
|
277
275
|
export { VC9B_ENABLED } from "./vector-cortex-vc9b.js";
|
|
278
276
|
export { VC9C_ENABLED } from "./vector-cortex-vc9c.js";
|
|
@@ -281,6 +279,8 @@ export { PCC_ENABLED } from "./vector-cortex-pcc.js";
|
|
|
281
279
|
export { ML5A_ENABLED } from "./vector-cortex-ml5a.js";
|
|
282
280
|
export { ML5B_ENABLED } from "./vector-cortex-ml5b.js";
|
|
283
281
|
export { ML5C_ENABLED } from "./vector-cortex-ml5c.js";
|
|
282
|
+
export { ML5D_ENABLED } from "./vector-cortex-ml5d.js";
|
|
283
|
+
export { ML5E_ENABLED } from "./vector-cortex-ml5e.js";
|
|
284
284
|
|
|
285
285
|
// Breaker constants (TRIAD_RESILIENCE.md §breaker) extracted to vector-cortex-breakers.ts.
|
|
286
286
|
export {
|