pi-mega-compact 0.20.38 → 0.20.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/dist/config/vector-cortex-ml5c.js +28 -0
  2. package/dist/config/vector-cortex-ml5d.js +28 -0
  3. package/dist/config/vector-cortex.js +4 -3
  4. package/dist/config.js +1 -1
  5. package/dist/extensions/dashboard-server/api-contracts/cortex-improve.js +14 -0
  6. package/dist/extensions/dashboard-server/api-contracts/endpoints/registry-ext.js +13 -0
  7. package/dist/extensions/dashboard-server/route-dispatch.js +5 -1
  8. package/dist/extensions/dashboard-server/routes-cortex-improve.js +220 -0
  9. package/dist/extensions/dashboard-server/routes-rag-settings-vector-cortex.js +2 -0
  10. package/dist/extensions/dashboard-server/routes.js +1 -0
  11. package/dist/extensions/mega-runtime/dashboard-snapshot.js +2 -2
  12. package/dist/src/config/vector-cortex-ml5c.js +28 -0
  13. package/dist/src/config/vector-cortex-ml5d.js +28 -0
  14. package/dist/src/config/vector-cortex.js +4 -3
  15. package/dist/src/config.js +1 -1
  16. package/dist/src/store/backfill.js +0 -9
  17. package/dist/src/vector-cortex/encoder/runtime-emit.js +42 -0
  18. package/dist/src/vector-cortex/encoder/runtime-native.js +77 -0
  19. package/dist/src/vector-cortex/encoder/runtime-select.js +122 -0
  20. package/dist/src/vector-cortex/encoder/runtime-stub.js +35 -0
  21. package/dist/src/vector-cortex/encoder/runtime-wasm.js +71 -0
  22. package/dist/src/vector-cortex/encoder/runtime.js +49 -61
  23. package/dist/src/vector-cortex/improve.js +24 -0
  24. package/dist/vector-cortex/encoder/runtime-emit.js +42 -0
  25. package/dist/vector-cortex/encoder/runtime-native.js +77 -0
  26. package/dist/vector-cortex/encoder/runtime-select.js +122 -0
  27. package/dist/vector-cortex/encoder/runtime-stub.js +35 -0
  28. package/dist/vector-cortex/encoder/runtime-wasm.js +71 -0
  29. package/dist/vector-cortex/encoder/runtime.js +49 -61
  30. package/dist/vector-cortex/improve.js +24 -0
  31. package/extensions/dashboard-client/dist/assets/{AreaChart-DjXp8u2Y.js → AreaChart-BT7YPcp7.js} +2 -2
  32. package/extensions/dashboard-client/dist/assets/{AreaChart-DjXp8u2Y.js.map → AreaChart-BT7YPcp7.js.map} +1 -1
  33. package/extensions/dashboard-client/dist/assets/{BarChart-BZbus-dR.js → BarChart-DceGcBUh.js} +2 -2
  34. package/extensions/dashboard-client/dist/assets/{BarChart-BZbus-dR.js.map → BarChart-DceGcBUh.js.map} +1 -1
  35. package/extensions/dashboard-client/dist/assets/{CacheTab-DQ2CjxVW.js → CacheTab-B9lCouRl.js} +2 -2
  36. package/extensions/dashboard-client/dist/assets/{CacheTab-DQ2CjxVW.js.map → CacheTab-B9lCouRl.js.map} +1 -1
  37. package/extensions/dashboard-client/dist/assets/{EventsTab-DUpnOV1p.js → EventsTab-oqJ11jY7.js} +2 -2
  38. package/extensions/dashboard-client/dist/assets/{EventsTab-DUpnOV1p.js.map → EventsTab-oqJ11jY7.js.map} +1 -1
  39. package/extensions/dashboard-client/dist/assets/{HealthTab-Bul_xjcn.js → HealthTab-Dbosf2Ik.js} +2 -2
  40. package/extensions/dashboard-client/dist/assets/{HealthTab-Bul_xjcn.js.map → HealthTab-Dbosf2Ik.js.map} +1 -1
  41. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DLUjdKd3.js → MaintenanceTab-6TiApCVE.js} +2 -2
  42. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DLUjdKd3.js.map → MaintenanceTab-6TiApCVE.js.map} +1 -1
  43. package/extensions/dashboard-client/dist/assets/MemoryMapTab-cBsE4mTf.js +2 -0
  44. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-BFy0hb6F.js.map → MemoryMapTab-cBsE4mTf.js.map} +1 -1
  45. package/extensions/dashboard-client/dist/assets/{MetricsTab-BpYuUkq3.js → MetricsTab-CX-X9hhy.js} +2 -2
  46. package/extensions/dashboard-client/dist/assets/{MetricsTab-BpYuUkq3.js.map → MetricsTab-CX-X9hhy.js.map} +1 -1
  47. package/extensions/dashboard-client/dist/assets/{OverviewTab-z7yuT-uq.js → OverviewTab-CpPYaTJF.js} +2 -2
  48. package/extensions/dashboard-client/dist/assets/{OverviewTab-z7yuT-uq.js.map → OverviewTab-CpPYaTJF.js.map} +1 -1
  49. package/extensions/dashboard-client/dist/assets/{ReposTab-ClORoDy_.js → ReposTab-D_Ot4QbU.js} +2 -2
  50. package/extensions/dashboard-client/dist/assets/{ReposTab-ClORoDy_.js.map → ReposTab-D_Ot4QbU.js.map} +1 -1
  51. package/extensions/dashboard-client/dist/assets/{SessionsTab-CUpehIt9.js → SessionsTab-BUzO3iGK.js} +2 -2
  52. package/extensions/dashboard-client/dist/assets/{SessionsTab-CUpehIt9.js.map → SessionsTab-BUzO3iGK.js.map} +1 -1
  53. package/extensions/dashboard-client/dist/assets/{SetupTab-Cii80Fdz.js → SetupTab-Be4w9v4j.js} +2 -2
  54. package/extensions/dashboard-client/dist/assets/{SetupTab-Cii80Fdz.js.map → SetupTab-Be4w9v4j.js.map} +1 -1
  55. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-BV2iCSdV.js → TimeSavedCard-Cx-g6CXH.js} +2 -2
  56. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-BV2iCSdV.js.map → TimeSavedCard-Cx-g6CXH.js.map} +1 -1
  57. package/extensions/dashboard-client/dist/assets/{TurnsTab-1T86rHnD.js → TurnsTab-CC_7T7ad.js} +3 -3
  58. package/extensions/dashboard-client/dist/assets/{TurnsTab-1T86rHnD.js.map → TurnsTab-CC_7T7ad.js.map} +1 -1
  59. package/extensions/dashboard-client/dist/assets/{VcStatusBadge-DKPleja8.js → VcStatusBadge-ZTItXSiH.js} +2 -2
  60. package/extensions/dashboard-client/dist/assets/{VcStatusBadge-DKPleja8.js.map → VcStatusBadge-ZTItXSiH.js.map} +1 -1
  61. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DgAUZ4Pg.js +2 -0
  62. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DgAUZ4Pg.js.map +1 -0
  63. package/extensions/dashboard-client/dist/assets/WikiTab-DnKupXTv.js +2 -0
  64. package/extensions/dashboard-client/dist/assets/{WikiTab-DKTILa1M.js.map → WikiTab-DnKupXTv.js.map} +1 -1
  65. package/extensions/dashboard-client/dist/assets/button-CFt3e0yc.js +2 -0
  66. package/extensions/dashboard-client/dist/assets/{button-5fSU8BZj.js.map → button-CFt3e0yc.js.map} +1 -1
  67. package/extensions/dashboard-client/dist/assets/{card-BFUnpXE_.js → card-C6ZjsHrl.js} +2 -2
  68. package/extensions/dashboard-client/dist/assets/{card-BFUnpXE_.js.map → card-C6ZjsHrl.js.map} +1 -1
  69. package/extensions/dashboard-client/dist/assets/client-extra-DdEdGTIe.js +2 -0
  70. package/extensions/dashboard-client/dist/assets/client-extra-DdEdGTIe.js.map +1 -0
  71. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-xCzi6pWn.js → generateCategoricalChart-C9_fu2hc.js} +2 -2
  72. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-xCzi6pWn.js.map → generateCategoricalChart-C9_fu2hc.js.map} +1 -1
  73. package/extensions/dashboard-client/dist/assets/{index-DqjmZ88m.js → index-DY81TY4f.js} +11 -11
  74. package/extensions/dashboard-client/dist/assets/index-DY81TY4f.js.map +1 -0
  75. package/extensions/dashboard-client/dist/assets/{switch-CeZUPf_e.js → switch-BOqa1d6x.js} +2 -2
  76. package/extensions/dashboard-client/dist/assets/{switch-CeZUPf_e.js.map → switch-BOqa1d6x.js.map} +1 -1
  77. package/extensions/dashboard-client/dist/assets/{toggle-MQQwdkwD.js → toggle-DToFaIEd.js} +2 -2
  78. package/extensions/dashboard-client/dist/assets/{toggle-MQQwdkwD.js.map → toggle-DToFaIEd.js.map} +1 -1
  79. package/extensions/dashboard-client/dist/assets/{useSSE-BetS7QHN.js → useSSE-CZqkhU18.js} +2 -2
  80. package/extensions/dashboard-client/dist/assets/{useSSE-BetS7QHN.js.map → useSSE-CZqkhU18.js.map} +1 -1
  81. package/extensions/dashboard-client/dist/index.html +1 -1
  82. package/extensions/dashboard-client/src/api/client-extra.ts +19 -1
  83. package/extensions/dashboard-client/src/components/ModelImprovementCard.tsx +141 -0
  84. package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +6 -0
  85. package/extensions/dashboard-client/src/types/cortex-improve.ts +42 -0
  86. package/extensions/dashboard-server/api-contracts/cortex-improve.ts +82 -0
  87. package/extensions/dashboard-server/api-contracts/endpoints/registry-ext.ts +31 -0
  88. package/extensions/dashboard-server/api-contracts/index.ts +12 -0
  89. package/extensions/dashboard-server/route-dispatch.ts +4 -0
  90. package/extensions/dashboard-server/routes-cortex-improve.ts +256 -0
  91. package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts +12 -0
  92. package/extensions/dashboard-server/routes.ts +4 -0
  93. package/extensions/mega-runtime/dashboard-snapshot.ts +2 -2
  94. package/package.json +1 -1
  95. package/src/config/vector-cortex-ml5c.ts +30 -0
  96. package/src/config/vector-cortex-ml5d.ts +30 -0
  97. package/src/config/vector-cortex.ts +4 -4
  98. package/src/config.ts +2 -0
  99. package/src/store/backfill.ts +0 -7
  100. package/src/vector-cortex/encoder/runtime-emit.ts +47 -0
  101. package/src/vector-cortex/encoder/runtime-native.ts +117 -0
  102. package/src/vector-cortex/encoder/runtime-select.ts +167 -0
  103. package/src/vector-cortex/encoder/runtime-stub.ts +38 -0
  104. package/src/vector-cortex/encoder/runtime-wasm.ts +110 -0
  105. package/src/vector-cortex/encoder/runtime.ts +59 -66
  106. package/src/vector-cortex/improve.ts +28 -0
  107. package/extensions/dashboard-client/dist/assets/MemoryMapTab-BFy0hb6F.js +0 -2
  108. package/extensions/dashboard-client/dist/assets/VectorCortexTab-BRzvd_pU.js +0 -2
  109. package/extensions/dashboard-client/dist/assets/VectorCortexTab-BRzvd_pU.js.map +0 -1
  110. package/extensions/dashboard-client/dist/assets/WikiTab-DKTILa1M.js +0 -2
  111. package/extensions/dashboard-client/dist/assets/button-5fSU8BZj.js +0 -2
  112. package/extensions/dashboard-client/dist/assets/client-extra-BZbw21wu.js +0 -2
  113. package/extensions/dashboard-client/dist/assets/client-extra-BZbw21wu.js.map +0 -1
  114. package/extensions/dashboard-client/dist/assets/index-DqjmZ88m.js.map +0 -1
@@ -0,0 +1,42 @@
1
+ /**
2
+ * dashboard-client/src/types/cortex-improve.ts — ML5-D Improve Cortex client types.
3
+ *
4
+ * Client-side mirrors of the server contract (extensions/dashboard-server/
5
+ * api-contracts/cortex-improve.ts), kept in lockstep so the ModelImprovementCard
6
+ * is fully typed end-to-end. The client bundle does not import the server tree,
7
+ * so these are redeclared here (PREVENT-011: no `any`; every type explicit).
8
+ * Aggregate values only — mode/verdict/digest/reason/progress, never payload
9
+ * bytes or corpus content (EVAL-REDACT-002).
10
+ */
11
+
12
+ /** The five-head qualification verdict projected by an improve job. */
13
+ export interface QualificationV1 {
14
+ readonly mode: "A" | "B" | "C";
15
+ readonly assetDigestPrefix: string | null;
16
+ readonly verdict: "qualified" | "demoted";
17
+ readonly reasonCode?: string;
18
+ }
19
+
20
+ /** Response body for POST /api/cortex/improve. */
21
+ export interface CortexImproveStart {
22
+ readonly status: "improving";
23
+ readonly jobId: string;
24
+ }
25
+
26
+ /** Response body for GET /api/cortex/improve/status/:jobId. */
27
+ export type CortexImproveStatus =
28
+ | {
29
+ readonly status: "improving";
30
+ readonly progress: number;
31
+ }
32
+ | {
33
+ readonly status: "qualified";
34
+ readonly progress: number;
35
+ readonly verdict: QualificationV1;
36
+ readonly assetDigest: string;
37
+ }
38
+ | {
39
+ readonly status: "demoted_to_B";
40
+ readonly progress: number;
41
+ readonly reason: string;
42
+ };
@@ -0,0 +1,82 @@
1
+ /**
2
+ * api-contracts/cortex-improve.ts — ML5-D "Improve Cortex" job API types.
3
+ *
4
+ * Types for POST /api/cortex/improve (launch a local ML5-A training job that
5
+ * re-qualifies the five heads against the latest local corpus) and GET
6
+ * /api/cortex/improve/status/:jobId (poll the in-process job state to a
7
+ * terminal qualified / demoted_to_B verdict). Local-only, aggregate surfaces:
8
+ * mode / verdict / digest / reason / progress — never message content or corpus
9
+ * rows (EVAL-REDACT-002).
10
+ *
11
+ * PREVENT-PI-004: type definitions only, no network code.
12
+ * PREVENT-011: no `any` type — all types are explicit.
13
+ */
14
+
15
+ /**
16
+ * The five-head qualification verdict projected from the improve job.
17
+ * Aggregate-only: current surrogate mode + asset digest prefix + a reason code
18
+ * when the asset failed to qualify (never weights or corpus content).
19
+ */
20
+ export interface QualificationV1 {
21
+ /** Surrogate encoder mode the improve job would promote to on success. */
22
+ readonly mode: "A" | "B" | "C";
23
+ /** First 12 hex chars of the produced asset manifest digest, or null. */
24
+ readonly assetDigestPrefix: string | null;
25
+ /** Terminal verdict: qualified when the heads fit within gate, else demoted. */
26
+ readonly verdict: "qualified" | "demoted";
27
+ /** A stable reason code when verdict is "demoted" (e.g. ENC_ASSET_UNREADABLE). */
28
+ readonly reasonCode?: string;
29
+ }
30
+
31
+ /** Response body for POST /api/cortex/improve (confirmation-gated). */
32
+ export interface CortexImproveStart {
33
+ /** Always "improving" immediately after the job is spawned. */
34
+ readonly status: "improving";
35
+ /** Opaque job token (sha256 of ts+random, hex-sliced), pollable via status. */
36
+ readonly jobId: string;
37
+ }
38
+
39
+ /** Lifecycle status of a Cortex improve job (in-process, restart-scoped). */
40
+ export type CortexImproveJobStatus =
41
+ | "improving"
42
+ | "qualified"
43
+ | "demoted_to_B";
44
+
45
+ /** Progressing (non-terminal) response for GET /api/cortex/improve/status/:jobId. */
46
+ export interface CortexImproveProgress {
47
+ readonly status: "improving";
48
+ /** 0..1 progress through the local ML5-A training run. */
49
+ readonly progress: number;
50
+ }
51
+
52
+ /** Terminal success: the five heads re-qualified under mode A. */
53
+ export interface CortexImproveQualified {
54
+ readonly status: "qualified";
55
+ readonly progress: number;
56
+ readonly verdict: QualificationV1;
57
+ readonly assetDigest: string;
58
+ }
59
+
60
+ /** Terminal demotion: the re-trained heads failed to qualify — stay mode B. */
61
+ export interface CortexImproveDemoted {
62
+ readonly status: "demoted_to_B";
63
+ readonly progress: number;
64
+ readonly reason: string;
65
+ }
66
+
67
+ /** Response body for GET /api/cortex/improve/status/:jobId. */
68
+ export type CortexImproveStatus =
69
+ | CortexImproveProgress
70
+ | CortexImproveQualified
71
+ | CortexImproveDemoted;
72
+
73
+ /** Error body returned when the improve surface is disabled (flag off). */
74
+ export interface CortexImproveDisabled {
75
+ readonly error: "disabled";
76
+ }
77
+
78
+ /** Request body for POST /api/cortex/improve. `confirm:true` is the server-side hard requirement. */
79
+ export interface CortexImproveStartRequest {
80
+ /** Required explicit confirmation. Absent/false → 400 confirmation_required. */
81
+ readonly confirm: boolean;
82
+ }
@@ -25,6 +25,11 @@ import type {
25
25
  SetupCortexActionLogResponse,
26
26
  } from "../setup-cortex.js";
27
27
  import type { PrefixStabilityResponse } from "../prefix-stability.js";
28
+ import type {
29
+ CortexImproveStartRequest,
30
+ CortexImproveStart,
31
+ CortexImproveStatus,
32
+ } from "../cortex-improve.js";
28
33
 
29
34
  /** Additive endpoint entries spread into the ENDPOINTS registry. */
30
35
  export const EXTRA_ENDPOINTS = {
@@ -123,4 +128,30 @@ export const EXTRA_ENDPOINTS = {
123
128
  undefined,
124
129
  PrefixStabilityResponse
125
130
  >,
131
+
132
+ // ─── ML5-D Improve Cortex (dashboard Vector Cortex tab) ───────────
133
+
134
+ /** POST /api/cortex/improve — launch a local ML5-A training job. */
135
+ improveCortex: {
136
+ method: "POST",
137
+ path: "/api/cortex/improve",
138
+ description:
139
+ "Launch a local ML5-A training job that re-qualifies the five heads against the latest local corpus and returns an opaque jobId. Requires confirm:true; flag-off returns 404. Local-only — never fetches anything.",
140
+ } as const satisfies EndpointDef<
141
+ "POST",
142
+ CortexImproveStartRequest,
143
+ CortexImproveStart
144
+ >,
145
+
146
+ /** GET /api/cortex/improve/status/:jobId — poll an improve job. */
147
+ improveCortexStatus: {
148
+ method: "GET",
149
+ path: "/api/cortex/improve/status/:jobId",
150
+ description:
151
+ "Poll an in-process Cortex improve job to terminal qualified / demoted_to_B. Unknown jobId or flag-off returns 404. Read-only, in-memory job state.",
152
+ } as const satisfies EndpointDef<
153
+ "GET",
154
+ undefined,
155
+ CortexImproveStatus
156
+ >,
126
157
  } as const;
@@ -323,3 +323,15 @@ export type {
323
323
  ModelThresholdDeleteResponse,
324
324
  ModelThresholdsError,
325
325
  } from "./model-thresholds.js";
326
+
327
+ // ML5-D — dashboard "Improve Cortex" job API
328
+ export type {
329
+ QualificationV1,
330
+ CortexImproveStart,
331
+ CortexImproveStatus,
332
+ CortexImproveProgress,
333
+ CortexImproveQualified,
334
+ CortexImproveDemoted,
335
+ CortexImproveDisabled,
336
+ CortexImproveStartRequest,
337
+ } from "./cortex-improve.js";
@@ -60,6 +60,8 @@ import {
60
60
  handleSetupCortexStatus,
61
61
  handleSetupCortexAction,
62
62
  handleSetupCortexActionLog,
63
+ handleImproveCortex,
64
+ handleImproveCortexStatus,
63
65
  } from "./routes.js";
64
66
  // VC6C repair lives in its own module (routes-vector-cortex-repair.ts) so the
65
67
  // heal route file stays well under the 400-line extension soft limit.
@@ -154,5 +156,7 @@ export function dispatchRoutes(
154
156
  if (handleSetupCortexStatus(req, res, ctx)) return true;
155
157
  if (handleSetupCortexAction(req, res, ctx)) return true;
156
158
  if (handleSetupCortexActionLog(req, res, ctx)) return true;
159
+ if (handleImproveCortex(req, res, ctx)) return true;
160
+ if (handleImproveCortexStatus(req, res, ctx)) return true;
157
161
  return false;
158
162
  }
@@ -0,0 +1,256 @@
1
+ /**
2
+ * dashboard-server/routes-cortex-improve.ts — ML5-D "Improve Cortex" job routes.
3
+ *
4
+ * POST /api/cortex/improve — confirmation-gated local ML5-A training
5
+ * GET /api/cortex/improve/status/:jobId — poll an in-process job to terminal
6
+ *
7
+ * The Improve action launches the committed ML5-A training pipeline
8
+ * (training/vector-cortex/train.py) as a background child_process against the
9
+ * latest local corpus, then re-qualifies the five heads. Job state is kept
10
+ * in-process in a Map<jobId, JobState> on this module; nothing is persisted
11
+ * across restarts (the status endpoint is read-only, in-memory).
12
+ *
13
+ * LOCAL ONLY (PREVENT-PI-004): the job spawns a local python process and reads
14
+ * local files — never a fetch. The `onnxruntime-node` native path is attempted
15
+ * only when MEGACOMPACT_ENCODER_NATIVE=1 (default OFF); otherwise the harness
16
+ * uses the WASM/trigram fallback. The status payload surfaces mode/verdict/
17
+ * digest/reason/progress only — never message or corpus content (EVAL-REDACT-002).
18
+ *
19
+ * Guardrails: PREVENT-011 (no `any`), PREVENT-001 (guarded JSON.parse via
20
+ * readJsonBody), flag-off byte-identical (no card + endpoints 404).
21
+ */
22
+
23
+ import type { IncomingMessage, ServerResponse } from "node:http";
24
+ import { spawn } from "node:child_process";
25
+ import { createHash, randomBytes } from "node:crypto";
26
+ import { readFileSync, existsSync } from "node:fs";
27
+ import { join, dirname } from "node:path";
28
+ import { fileURLToPath } from "node:url";
29
+ import type { RouteContext } from "./routes-core.js";
30
+ import { sendJson, readJsonBody } from "./routes-vector-cortex-shared.js";
31
+ import { ML5D_ENABLED } from "../../src/config.js";
32
+ import { qualifyDecision } from "../../src/vector-cortex/improve.js";
33
+ import type {
34
+ CortexImproveStart,
35
+ CortexImproveStatus,
36
+ QualificationV1,
37
+ } from "./api-contracts/cortex-improve.js";
38
+
39
+ /** Resolve the committed training entry by walking up to the repo root. */
40
+ function trainingScript(): string | null {
41
+ let dir = dirname(fileURLToPath(import.meta.url));
42
+ const rel = join("training", "vector-cortex", "train.py");
43
+ for (let i = 0; i < 8; i++) {
44
+ // guardrails-allow PREVENT-PI-004: local script read (loopback)
45
+ const candidate = join(dir, rel);
46
+ if (existsSync(candidate)) return candidate;
47
+ const next = dirname(dir);
48
+ if (next === dir) break;
49
+ dir = next;
50
+ }
51
+ return null;
52
+ }
53
+
54
+ /** An in-process improve job (restart-scoped, never persisted). */
55
+ interface JobState {
56
+ status: "improving" | "qualified" | "demoted_to_B";
57
+ progress: number;
58
+ verdict?: QualificationV1;
59
+ assetDigest?: string;
60
+ reason?: string;
61
+ updatedAt: string;
62
+ }
63
+
64
+ /** In-process job registry; nothing survives a server restart. */
65
+ const JOBS = new Map<string, JobState>();
66
+
67
+ /** Opaque job token: sha256(ts+random) hex-sliced. */
68
+ function newJobId(): string {
69
+ return createHash("sha256")
70
+ .update(`${Date.now()}:${randomBytes(16).toString("hex")}`)
71
+ .digest("hex")
72
+ .slice(0, 24);
73
+ }
74
+
75
+ /**
76
+ * Spawn the ML5-A training pipeline in the background and drive the job state
77
+ * to a terminal qualified / demoted_to_B verdict. Honest: an empty corpus (the
78
+ * host state) makes train.py no-op and the job ends demoted_to_B.
79
+ */
80
+ function startJob(jobId: string): void {
81
+ const script = trainingScript();
82
+ const boot = Date.now();
83
+ JOBS.set(jobId, { status: "improving", progress: 0, updatedAt: new Date().toISOString() });
84
+
85
+ if (script === null) {
86
+ settle(jobId, { status: "demoted_to_B", reason: "ENC_TRAIN_PIPELINE_ABSENT", progress: 1 });
87
+ return;
88
+ }
89
+
90
+ const env = { ...process.env };
91
+ const proc = spawn("python3", [script], {
92
+ env,
93
+ cwd: join(dirname(script), "..", ".."),
94
+ stdio: ["ignore", "pipe", "pipe"],
95
+ });
96
+
97
+ let output = "";
98
+ const step = () => {
99
+ const pct = Math.min(0.9, 0.1 + ((Date.now() - boot) / 60000) * 0.6);
100
+ const cur = JOBS.get(jobId);
101
+ if (cur && cur.status === "improving" && pct > cur.progress) {
102
+ JOBS.set(jobId, { ...cur, progress: pct, updatedAt: new Date().toISOString() });
103
+ }
104
+ };
105
+ const timer = setInterval(step, 2000);
106
+ const stop = () => clearInterval(timer);
107
+
108
+ proc.stdout?.on("data", (d: Buffer) => {
109
+ output += d.toString();
110
+ });
111
+ proc.stderr?.on("data", (d: Buffer) => {
112
+ output += d.toString();
113
+ });
114
+ proc.on("error", (err) => {
115
+ stop();
116
+ settle(jobId, {
117
+ status: "demoted_to_B",
118
+ reason: `ENC_TRAIN_SPAWN_FAILED:${err.message}`,
119
+ progress: 1,
120
+ });
121
+ });
122
+ proc.on("close", (code) => {
123
+ stop();
124
+ // Read the produced trained-heads artifact digest. The report line includes
125
+ // `trainedHeadsPath`; verify the file exists after a successful run.
126
+ const match = /trainedHeadsPath":\s*"([^"]+)"/.exec(output);
127
+ const assetPath = match?.[1];
128
+ let digest: string | null = null;
129
+ if (assetPath) {
130
+ // guardrails-allow PREVENT-PI-004: local artifact read (loopback)
131
+ try {
132
+ digest = createHash("sha256")
133
+ .update(readFileSync(assetPath))
134
+ .digest("hex")
135
+ .slice(0, 12);
136
+ } catch {
137
+ digest = null;
138
+ }
139
+ }
140
+ const decision = qualifyDecision(code ?? -1, digest);
141
+ if (decision === "qualified") {
142
+ settle(jobId, {
143
+ status: "qualified",
144
+ progress: 1,
145
+ assetDigest: digest ?? undefined,
146
+ verdict: { mode: "A", assetDigestPrefix: digest ?? null, verdict: "qualified" },
147
+ });
148
+ } else {
149
+ settle(jobId, {
150
+ status: "demoted_to_B",
151
+ reason: assetPath
152
+ ? "trained asset did not verify — demoted to mode B"
153
+ : "empty corpus (no groups) — no qualified asset emitted",
154
+ progress: 1,
155
+ });
156
+ }
157
+ });
158
+ }
159
+
160
+ /** Write a terminal job state (idempotent — only advances from improving). */
161
+ function settle(
162
+ jobId: string,
163
+ terminal: { status: "qualified" | "demoted_to_B"; progress: number; verdict?: QualificationV1; assetDigest?: string; reason?: string },
164
+ ): void {
165
+ const cur = JOBS.get(jobId);
166
+ if (cur && cur.status !== "improving") return;
167
+ JOBS.set(jobId, { ...terminal, updatedAt: new Date().toISOString() });
168
+ }
169
+
170
+ /** Flag-off response, byte-identical regardless of request (ML5-D absent). */
171
+ function sendDisabled(res: ServerResponse): void {
172
+ sendJson(res, 404, { error: "disabled" });
173
+ }
174
+
175
+ /**
176
+ * POST /api/cortex/improve (ML5-D). Returns true when it claims the request.
177
+ * Requires `confirm:true` server-side (mirrors the client window.confirm).
178
+ */
179
+ export function handleImproveCortex(
180
+ req: IncomingMessage,
181
+ res: ServerResponse,
182
+ _ctx: RouteContext,
183
+ ): boolean {
184
+ const url = req.url ?? "";
185
+ if (url !== "/api/cortex/improve") return false;
186
+ if (req.method !== "POST") {
187
+ sendJson(res, 405, { error: "method_not_allowed" });
188
+ return true;
189
+ }
190
+ if (!ML5D_ENABLED()) {
191
+ sendDisabled(res);
192
+ return true;
193
+ }
194
+ readJsonBody(req, (parsed) => {
195
+ if (!parsed.ok) {
196
+ sendJson(res, 400, { error: "invalid_body" });
197
+ return;
198
+ }
199
+ if (parsed.value.confirm !== true) {
200
+ sendJson(res, 400, { error: "confirmation_required" });
201
+ return;
202
+ }
203
+ const jobId = newJobId();
204
+ startJob(jobId);
205
+ const body: CortexImproveStart = { status: "improving", jobId };
206
+ sendJson(res, 200, body);
207
+ });
208
+ return true;
209
+ }
210
+
211
+ /**
212
+ * GET /api/cortex/improve/status/:jobId (ML5-D). Returns true when it claims the
213
+ * request. Flag-off or unknown jobId → 404. Read-only in-memory.
214
+ */
215
+ export function handleImproveCortexStatus(
216
+ req: IncomingMessage,
217
+ res: ServerResponse,
218
+ _ctx: RouteContext,
219
+ ): boolean {
220
+ const url = req.url ?? "";
221
+ const match = /^\/api\/cortex\/improve\/status\/([A-Za-z0-9]+)$/.exec(url);
222
+ if (!match) return false;
223
+ if (req.method !== "GET") {
224
+ sendJson(res, 405, { error: "method_not_allowed" });
225
+ return true;
226
+ }
227
+ if (!ML5D_ENABLED()) {
228
+ sendDisabled(res);
229
+ return true;
230
+ }
231
+ const job = JOBS.get(match[1]);
232
+ if (!job) {
233
+ sendJson(res, 404, { error: "job_not_found" });
234
+ return true;
235
+ }
236
+ // Narrow job.status into the discriminated CortexImproveStatus variant. The
237
+ // terminal shapes require their payload fields, so construct per-state.
238
+ const body: CortexImproveStatus =
239
+ job.status === "improving"
240
+ ? { status: "improving", progress: job.progress }
241
+ : job.status === "qualified"
242
+ ? {
243
+ status: "qualified",
244
+ progress: job.progress,
245
+ verdict:
246
+ job.verdict ?? { mode: "A", assetDigestPrefix: null, verdict: "qualified" },
247
+ assetDigest: job.assetDigest ?? "",
248
+ }
249
+ : {
250
+ status: "demoted_to_B",
251
+ progress: job.progress,
252
+ reason: job.reason ?? "UNKNOWN_TERMINAL",
253
+ };
254
+ sendJson(res, 200, body);
255
+ return true;
256
+ }
@@ -228,5 +228,17 @@ export const VECTOR_CORTEX_SETTINGS: SettingGroup = {
228
228
  "ML5-A real trained-head loading: loadHeadProjections (trained-heads-v1) feeds selectQualifiedEncoder (trainedHeadsPath atomic demotion) + loadCalibrationV1. ON (default) = a pinned trained-heads path must load for mode A. OFF = loaders return null and selection ignores trainedHeadsPath — byte-identical to the placeholder-weighted VC2C path.",
229
229
  true,
230
230
  ),
231
+ boolDirect(
232
+ "MEGACOMPACT_ML5_C",
233
+ "ML5-C Runtime Decision + Packaging",
234
+ "ML5-C runtime backend selection (WASM vs native): selects the ONNX runtime backend based on the ML5-B bench record and platform support. ON (default) = the runtime-selection dispatch runs and emits vector_cortex_runtime_selected. OFF = no selection runs — encoder serves mode B trigram, byte-identical to ML5-B.",
235
+ true,
236
+ ),
237
+ boolDirect(
238
+ "MEGACOMPACT_ML5_D",
239
+ "ML5-D Dashboard Improve Cortex",
240
+ "ML5-D dashboard 'Improve Cortex' surface: the ModelImprovementCard + POST /api/cortex/improve + GET /api/cortex/improve/status/:jobId. ON (default) = the card renders and Improve launches a local ML5-A training job. OFF = both improve endpoints return 404 and VectorCortexTab omits the card, byte-identical to ML5-C.",
241
+ true,
242
+ ),
231
243
  ],
232
244
  };
@@ -62,3 +62,7 @@ export {
62
62
  handleSetupCortexAction,
63
63
  handleSetupCortexActionLog,
64
64
  } from "./routes-setup-cortex-actions.js";
65
+ export {
66
+ handleImproveCortex,
67
+ handleImproveCortexStatus,
68
+ } from "./routes-cortex-improve.js";
@@ -159,8 +159,8 @@ export function buildDashboardSnapshot(ctx: SnapshotBuildContext): DashboardSnap
159
159
  // per-session in rt and there's no cumulative counter yet.
160
160
  total: ctx.repo.dedupCollapsed,
161
161
  sessionTokensSaved: ctx.rt.cacheHitTokens,
162
- // Cumulative tokens saved across all sessions in this repo
163
- // was a placeholder (dedupCollapsed * 100).
162
+ // Cumulative tokens saved across all sessions in this repo = the
163
+ // real repo counter (audit Table 1 stub 8 closed — no rolled-up math).
164
164
  totalTokensSaved: ctx.repo.tokensSaved,
165
165
  },
166
166
  compacts: {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-mega-compact",
3
- "version": "0.20.38",
3
+ "version": "0.20.40",
4
4
  "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
5
5
  "type": "module",
6
6
  "license": "BSD-3-Clause",
@@ -0,0 +1,30 @@
1
+ /**
2
+ * config/vector-cortex-ml5c.ts — ML5-C runtime decision + packaging flag.
3
+ *
4
+ * Sibling extract mirroring vector-cortex-ml5a.ts / vector-cortex-ml5b.ts, so
5
+ * vector-cortex.ts stays under its 300-line soft limit (soft-as-hard gate).
6
+ * This is the ONNX Runtime backend selection + packaging sprint flag.
7
+ * vector-cortex.ts re-exports the ENUM below and root src/config.ts re-exports
8
+ * it, so no consumer import path changes.
9
+ *
10
+ * ML5-C selects the ONNX runtime backend (WASM vs native) based on the ML5-B
11
+ * bench record and platform support. The flag gates the runtime-selection
12
+ * dispatch only; when OFF the encoder serves mode B trigram exactly as before
13
+ * (byte-identical to the ML5-B survivor — no `vector_cortex_runtime_selected`
14
+ * event is emitted, no session-selection dispatch runs).
15
+ *
16
+ * Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
17
+ */
18
+
19
+ import { sprintFlag } from "./vector-cortex-flag.js";
20
+
21
+ /**
22
+ * ML5-C — runtime decision + packaging (WASM vs native). Default ON.
23
+ * `MEGACOMPACT_ML5_C=0` disables and is byte-identical to the ML5-B survivor:
24
+ * no runtime selection runs — the encoder continues to serve mode B trigram,
25
+ * exactly as before, with no `vector_cortex_runtime_selected` event emitted.
26
+ * The flag gates the runtime-selection dispatch only; it does not gate the
27
+ * underlying WASM/native backends (which are exercised by ML5-B's bench
28
+ * harness and ML5-A's trained asset independently).
29
+ */
30
+ export const ML5C_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_ML5_C");
@@ -0,0 +1,30 @@
1
+ /**
2
+ * config/vector-cortex-ml5d.ts — ML5-D dashboard "Improve Cortex" flag.
3
+ *
4
+ * Sibling extract mirroring vector-cortex-ml5a.ts / vector-cortex-ml5b.ts /
5
+ * vector-cortex-ml5c.ts, so vector-cortex.ts stays under its 300-line soft
6
+ * limit (soft-as-hard gate). This is the dashboard "Improve Cortex" surface +
7
+ * promote workflow sprint flag.
8
+ * vector-cortex.ts re-exports the ENUM below and root src/config.ts re-exports
9
+ * it, so no consumer import path changes.
10
+ *
11
+ * ML5-D adds the dashboard ModelImprovementCard + the POST /api/cortex/improve
12
+ * and GET /api/cortex/improve/status/:jobId endpoints. The flag gates that
13
+ * surface only; when OFF the endpoints return 404/disabled and VectorCortexTab
14
+ * renders exactly as before (byte-identical to the ML5-C-era tab — no
15
+ * ModelImprovementCard, no improve job is ever spawned).
16
+ *
17
+ * Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
18
+ */
19
+
20
+ import { sprintFlag } from "./vector-cortex-flag.js";
21
+
22
+ /**
23
+ * ML5-D — dashboard "Improve Cortex" surface + promote workflow. Default ON.
24
+ * `MEGACOMPACT_ML5_D=0` disables and is byte-identical to the ML5-C survivor:
25
+ * both `/api/cortex/improve*` endpoints return 404 and VectorCortexTab omits the
26
+ * ModelImprovementCard. The flag gates the dashboard surface only; it does not
27
+ * gate the underlying ML5-A training pipeline or the encoder's mode-A/B/C
28
+ * selection (those are governed by ML5-A and the encoder independently).
29
+ */
30
+ export const ML5D_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_ML5_D");
@@ -269,11 +269,9 @@ export const VC8A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8A");
269
269
  */
270
270
  export const VC8B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8B");
271
271
 
272
- // VC8C (canary selection + Rust parity) extracted to vector-cortex-vc8c.ts;
273
- // re-exported so existing `./config/vector-cortex.js` imports keep resolving.
272
+ // VC8C (canary selection + Rust parity) extracted to vector-cortex-vc8c.ts; re-exported so existing imports keep resolving.
274
273
  export { VC8C_ENABLED } from "./vector-cortex-vc8c.js";
275
-
276
- // VC9A/VC9B/VC9C/VC9D/PCC/ML5A split to sibling files to stay under the 300-line soft limit.
274
+ // Sibling extracts to stay under the 300-line soft limit.
277
275
  export { VC9A_ENABLED } from "./vector-cortex-vc9a.js";
278
276
  export { VC9B_ENABLED } from "./vector-cortex-vc9b.js";
279
277
  export { VC9C_ENABLED } from "./vector-cortex-vc9c.js";
@@ -281,6 +279,8 @@ export { VC9D_ENABLED } from "./vector-cortex-vc9d.js";
281
279
  export { PCC_ENABLED } from "./vector-cortex-pcc.js";
282
280
  export { ML5A_ENABLED } from "./vector-cortex-ml5a.js";
283
281
  export { ML5B_ENABLED } from "./vector-cortex-ml5b.js";
282
+ export { ML5C_ENABLED } from "./vector-cortex-ml5c.js";
283
+ export { ML5D_ENABLED } from "./vector-cortex-ml5d.js";
284
284
 
285
285
  // Breaker constants (TRIAD_RESILIENCE.md §breaker) extracted to vector-cortex-breakers.ts.
286
286
  export {
package/src/config.ts CHANGED
@@ -185,6 +185,8 @@ export {
185
185
  PCC_ENABLED,
186
186
  ML5A_ENABLED,
187
187
  ML5B_ENABLED,
188
+ ML5C_ENABLED,
189
+ ML5D_ENABLED,
188
190
  BREAKER_WINDOW_MS,
189
191
  BREAKER_MIN_ATTEMPTS,
190
192
  BREAKER_PERF_FAILURES,
@@ -28,7 +28,6 @@ import { defaultEmbedder } from "../embedder.js";
28
28
  import { getStateDir } from "../store.js";
29
29
 
30
30
  const BATCH = 1000;
31
- const THROTTLE_MS = 0; // synchronous backfill; no cross-process yield needed
32
31
 
33
32
  /** Backfill phases, in order (Sprint 14 full-pipeline wiring). */
34
33
  export type BackfillPhase = "L0" | "L1" | "L2" | "RAPTOR";
@@ -132,11 +131,6 @@ export function backfillContentHashes(stateDir: string = getStateDir()): Backfil
132
131
  ).run(lastSid, lastId, updated, duplicatesResolved);
133
132
  }
134
133
 
135
- if (THROTTLE_MS > 0) {
136
- // No-op in this synchronous build; placeholder for future streaming backfill.
137
- // guardrails-allow PREVENT-STUB-001: ML5-C
138
- }
139
-
140
134
  return { processed, updated, duplicatesResolved };
141
135
  }
142
136
 
@@ -220,7 +214,6 @@ export function backfillPhase(
220
214
  });
221
215
  savePhaseCursor(db, phase, cursor ?? null, processed);
222
216
  batches++;
223
- if (THROTTLE_MS > 0) { const end = Date.now() + THROTTLE_MS; while (Date.now() < end) { /* throttle */ } }
224
217
  if (opts.interruptAfterBatches && batches >= opts.interruptAfterBatches) {
225
218
  interrupted = true;
226
219
  break;