pi-mega-compact 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/config/vector-cortex.js +20 -0
  2. package/dist/dedup/digest.js +30 -0
  3. package/dist/dedup/l1-lsh-v2.js +60 -0
  4. package/dist/dedup/l1-lsh.js +52 -0
  5. package/dist/dedup/l1-minhash-v2.js +193 -0
  6. package/dist/dedup/l1-minhash.js +91 -0
  7. package/dist/dedup/l1-verify.js +54 -0
  8. package/dist/dedup/mmr.js +45 -0
  9. package/dist/dedup/normalize.js +39 -0
  10. package/dist/dedup/raptor/buildHistory.js +164 -0
  11. package/dist/dedup/raptor/guardrails.js +83 -0
  12. package/dist/dedup/raptor/incremental.js +304 -0
  13. package/dist/dedup/raptor/index.js +190 -0
  14. package/dist/dedup/raptor/kmeans.js +152 -0
  15. package/dist/dedup/raptor/multilevel.js +184 -0
  16. package/dist/dedup/raptor/retrieval.js +93 -0
  17. package/dist/dedup/raptor/summarizer.js +86 -0
  18. package/dist/dedup/raptor/tree.js +202 -0
  19. package/dist/dedup/topk.js +60 -0
  20. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +2 -0
  21. package/dist/extensions/mega-events/context-handler/dbMirrorAppend.js +63 -0
  22. package/dist/extensions/mega-events/context-handler/gateCheck.js +59 -0
  23. package/dist/extensions/mega-events/context-handler/liveTrim.js +178 -0
  24. package/dist/extensions/mega-events/context-handler/pipelineRun.js +37 -0
  25. package/dist/extensions/mega-events/context-handler.js +39 -305
  26. package/dist/src/config/vector-cortex.js +20 -0
  27. package/dist/src/config.js +1 -1
  28. package/dist/src/dedup/l1-lsh-v2.js +60 -0
  29. package/dist/src/dedup/l1-minhash-v2.js +193 -0
  30. package/dist/src/dedup/normalize.js +1 -1
  31. package/dist/src/vector-cortex/conformance/emit.js +46 -0
  32. package/dist/src/vector-cortex/conformance/manifest.js +235 -0
  33. package/dist/src/vector-cortex/conformance/runner.js +111 -0
  34. package/dist/src/vector-cortex/conformance/triadB-reader.js +95 -0
  35. package/dist/src/vector-cortex/encoder/asset.js +142 -0
  36. package/dist/src/vector-cortex/encoder/emit.js +42 -0
  37. package/dist/src/vector-cortex/encoder/runtime.js +228 -0
  38. package/dist/src/vector-cortex/encoder/types.js +85 -0
  39. package/dist/src/vector-cortex/migrations/minhash-v2.js +209 -0
  40. package/dist/vector-cortex/conformance/emit.js +46 -0
  41. package/dist/vector-cortex/conformance/manifest.js +235 -0
  42. package/dist/vector-cortex/conformance/runner.js +111 -0
  43. package/dist/vector-cortex/conformance/triadB-reader.js +95 -0
  44. package/dist/vector-cortex/encoder/asset.js +142 -0
  45. package/dist/vector-cortex/encoder/emit.js +42 -0
  46. package/dist/vector-cortex/encoder/runtime.js +228 -0
  47. package/dist/vector-cortex/encoder/types.js +85 -0
  48. package/dist/vector-cortex/migrations/minhash-v2.js +209 -0
  49. package/extensions/dashboard-client/dist/assets/{AreaChart-CAnWomhw.js → AreaChart-BQXFF7UI.js} +2 -2
  50. package/extensions/dashboard-client/dist/assets/{AreaChart-CAnWomhw.js.map → AreaChart-BQXFF7UI.js.map} +1 -1
  51. package/extensions/dashboard-client/dist/assets/{BarChart-lZ2-s71k.js → BarChart-BVtoz9Eg.js} +2 -2
  52. package/extensions/dashboard-client/dist/assets/{BarChart-lZ2-s71k.js.map → BarChart-BVtoz9Eg.js.map} +1 -1
  53. package/extensions/dashboard-client/dist/assets/{CacheTab-BL2Thh66.js → CacheTab-trI2TIGy.js} +2 -2
  54. package/extensions/dashboard-client/dist/assets/{CacheTab-BL2Thh66.js.map → CacheTab-trI2TIGy.js.map} +1 -1
  55. package/extensions/dashboard-client/dist/assets/{EventsTab-DEh2Z9fp.js → EventsTab-CNUbk9__.js} +2 -2
  56. package/extensions/dashboard-client/dist/assets/{EventsTab-DEh2Z9fp.js.map → EventsTab-CNUbk9__.js.map} +1 -1
  57. package/extensions/dashboard-client/dist/assets/{HealthTab-DTwKclxk.js → HealthTab-K-bd_CM-.js} +2 -2
  58. package/extensions/dashboard-client/dist/assets/{HealthTab-DTwKclxk.js.map → HealthTab-K-bd_CM-.js.map} +1 -1
  59. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DDfBEdbX.js → MaintenanceTab-CoEioKzx.js} +2 -2
  60. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DDfBEdbX.js.map → MaintenanceTab-CoEioKzx.js.map} +1 -1
  61. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-Dgfmg-0Q.js → MemoryMapTab-CJo24ulv.js} +2 -2
  62. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-Dgfmg-0Q.js.map → MemoryMapTab-CJo24ulv.js.map} +1 -1
  63. package/extensions/dashboard-client/dist/assets/{MetricsTab-DOcMm0Tx.js → MetricsTab-DCNlQWWP.js} +2 -2
  64. package/extensions/dashboard-client/dist/assets/{MetricsTab-DOcMm0Tx.js.map → MetricsTab-DCNlQWWP.js.map} +1 -1
  65. package/extensions/dashboard-client/dist/assets/{OverviewTab-DHT5aoHg.js → OverviewTab-CQQlyIQ6.js} +2 -2
  66. package/extensions/dashboard-client/dist/assets/{OverviewTab-DHT5aoHg.js.map → OverviewTab-CQQlyIQ6.js.map} +1 -1
  67. package/extensions/dashboard-client/dist/assets/{ReposTab-CGAmtR_c.js → ReposTab-Cqm82_yn.js} +2 -2
  68. package/extensions/dashboard-client/dist/assets/{ReposTab-CGAmtR_c.js.map → ReposTab-Cqm82_yn.js.map} +1 -1
  69. package/extensions/dashboard-client/dist/assets/{SessionsTab-BIgx6iKT.js → SessionsTab-A8xTAdET.js} +2 -2
  70. package/extensions/dashboard-client/dist/assets/{SessionsTab-BIgx6iKT.js.map → SessionsTab-A8xTAdET.js.map} +1 -1
  71. package/extensions/dashboard-client/dist/assets/{SetupTab-ADghKuEI.js → SetupTab-CqaSgaZl.js} +2 -2
  72. package/extensions/dashboard-client/dist/assets/{SetupTab-ADghKuEI.js.map → SetupTab-CqaSgaZl.js.map} +1 -1
  73. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-C33UiiOM.js → TimeSavedCard-C_f3q8GR.js} +2 -2
  74. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-C33UiiOM.js.map → TimeSavedCard-C_f3q8GR.js.map} +1 -1
  75. package/extensions/dashboard-client/dist/assets/{TurnsTab-DdbL9wPN.js → TurnsTab-BEsafACo.js} +2 -2
  76. package/extensions/dashboard-client/dist/assets/{TurnsTab-DdbL9wPN.js.map → TurnsTab-BEsafACo.js.map} +1 -1
  77. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DD1CMK80.js +2 -0
  78. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DD1CMK80.js.map +1 -0
  79. package/extensions/dashboard-client/dist/assets/{WikiTab-Ch00OFKs.js → WikiTab-Bf01M2_Z.js} +2 -2
  80. package/extensions/dashboard-client/dist/assets/{WikiTab-Ch00OFKs.js.map → WikiTab-Bf01M2_Z.js.map} +1 -1
  81. package/extensions/dashboard-client/dist/assets/{button-ByeEdo2g.js → button-BEJPTk_E.js} +2 -2
  82. package/extensions/dashboard-client/dist/assets/{button-ByeEdo2g.js.map → button-BEJPTk_E.js.map} +1 -1
  83. package/extensions/dashboard-client/dist/assets/{card-DesoLXvc.js → card-DK1V382s.js} +2 -2
  84. package/extensions/dashboard-client/dist/assets/{card-DesoLXvc.js.map → card-DK1V382s.js.map} +1 -1
  85. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-CCAB7Iz4.js → generateCategoricalChart-CRtaPphL.js} +2 -2
  86. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-CCAB7Iz4.js.map → generateCategoricalChart-CRtaPphL.js.map} +1 -1
  87. package/extensions/dashboard-client/dist/assets/{index-B6cgyAPr.js → index-CB6BslE4.js} +3 -3
  88. package/extensions/dashboard-client/dist/assets/{index-B6cgyAPr.js.map → index-CB6BslE4.js.map} +1 -1
  89. package/extensions/dashboard-client/dist/assets/{switch-DIZC-oe-.js → switch-C4FV6B9k.js} +2 -2
  90. package/extensions/dashboard-client/dist/assets/{switch-DIZC-oe-.js.map → switch-C4FV6B9k.js.map} +1 -1
  91. package/extensions/dashboard-client/dist/assets/{toggle-CIoWaH2e.js → toggle-GBmeENMd.js} +2 -2
  92. package/extensions/dashboard-client/dist/assets/{toggle-CIoWaH2e.js.map → toggle-GBmeENMd.js.map} +1 -1
  93. package/extensions/dashboard-client/dist/assets/{useSSE-Bv4943l1.js → useSSE-BOKZGSn1.js} +2 -2
  94. package/extensions/dashboard-client/dist/assets/{useSSE-Bv4943l1.js.map → useSSE-BOKZGSn1.js.map} +1 -1
  95. package/extensions/dashboard-client/dist/index.html +1 -1
  96. package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +1 -1
  97. package/extensions/dashboard-client/tsconfig.json +1 -1
  98. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +12 -0
  99. package/extensions/mega-events/context-handler/dbMirrorAppend.ts +93 -0
  100. package/extensions/mega-events/context-handler/gateCheck.ts +101 -0
  101. package/extensions/mega-events/context-handler/liveTrim.ts +241 -0
  102. package/extensions/mega-events/context-handler/pipelineRun.ts +79 -0
  103. package/extensions/mega-events/context-handler.ts +45 -347
  104. package/package.json +1 -1
  105. package/src/config/vector-cortex.ts +22 -0
  106. package/src/config.ts +2 -0
  107. package/src/dedup/l1-lsh-v2.ts +78 -0
  108. package/src/dedup/l1-minhash-v2.ts +221 -0
  109. package/src/dedup/normalize.ts +1 -1
  110. package/src/store/sqlite/game-scores.ts +2 -1
  111. package/src/vector-cortex/conformance/emit.ts +56 -0
  112. package/src/vector-cortex/conformance/manifest.ts +298 -0
  113. package/src/vector-cortex/conformance/runner.ts +165 -0
  114. package/src/vector-cortex/conformance/triadB-reader.ts +110 -0
  115. package/src/vector-cortex/encoder/asset.ts +155 -0
  116. package/src/vector-cortex/encoder/emit.ts +51 -0
  117. package/src/vector-cortex/encoder/runtime.ts +283 -0
  118. package/src/vector-cortex/encoder/types.ts +165 -0
  119. package/src/vector-cortex/migrations/minhash-v2.ts +263 -0
  120. package/extensions/dashboard-client/dist/assets/VectorCortexTab-Cl1pbEdR.js +0 -2
  121. package/extensions/dashboard-client/dist/assets/VectorCortexTab-Cl1pbEdR.js.map +0 -1
@@ -2,45 +2,32 @@
2
2
  * mega-events/context-handler.ts — the context event handler (auto-trigger).
3
3
  *
4
4
  * Delegate-shell (extensions split): handles the live-trim compaction pipeline —
5
- * DB-mirror append, fast-gate threshold check, pipeline invocation, legacy
6
- * durable compact, and the live-trim message reconstruction that feeds pi's
7
- * transformContext. The extracted pieces live in ./context-handler/:
5
+ * DB-mirror append, fast-gate threshold check, replay/debounce, pipeline
6
+ * invocation, legacy durable compact, and the live-trim message reconstruction
7
+ * that feeds pi's transformContext. The extracted pieces live in ./context-handler/:
8
8
  * - messageText.ts (messageContentText — best-effort text extraction)
9
9
  * - tailResult.ts (buildTailResult — recall-tail injection factory)
10
10
  * - afterCompact.ts (persistEpochAndMaintain — epoch/wiki/seed/dedup writes)
11
+ * - dbMirrorAppend.ts (appendMirrorAndLedger — mirror + VC1B ledger append)
12
+ * - gateCheck.ts (evaluateGate — S29 fast-gate threshold evaluation)
13
+ * - pipelineRun.ts (invokePipeline — adaptive-compression runCompact)
14
+ * - liveTrim.ts (buildLiveTrimView — S16 live-trim reconstruction + cap)
11
15
  */
12
16
  import type {
13
17
  ExtensionAPI,
14
18
  ExtensionContext,
15
19
  ContextEvent,
16
20
  } from "@earendil-works/pi-coding-agent";
17
- import type { AgentMessage } from "@earendil-works/pi-agent-core";
18
- import {
19
- openStore,
20
- resolveModelThreshold,
21
- DEFAULT_SAFETY_MARGIN_PCT,
22
- DEFAULT_FIRE_POINT_PCT,
23
- } from "../../src/store/sqlite.js";
24
- import { autoCompactCheck } from "../../src/compact.js";
25
- import {
26
- estimateSessionTokens,
27
- estimateBlockTokens,
28
- estimateMessageTokens,
29
- } from "../../src/tokens.js";
21
+ import { estimateSessionTokens } from "../../src/tokens.js";
30
22
  import type { MegaRuntime } from "../mega-runtime.js";
31
- import { runCompact, piCompactWouldNoop } from "../mega-pipeline.js";
32
- import {
33
- pressureFromPct,
34
- pressureRatio,
35
- type MegaConfig,
36
- } from "../mega-config.js";
37
- import { appendMirrorMessages } from "./mirror-append.js";
38
- import { appendMessagesToLedger } from "../mega-runtime/vector-cortex-ledger.js";
39
- import { epochIdFor } from "../../src/mirror/epoch.js";
40
- import { computeLiveTrimCut, liveTrimSummaryMessage } from "../mega-trim.js";
41
- import { messageContentText } from "./context-handler/messageText.js";
23
+ import { piCompactWouldNoop } from "../mega-pipeline.js";
24
+ import type { MegaConfig } from "../mega-config.js";
42
25
  import { buildTailResult } from "./context-handler/tailResult.js";
43
26
  import { persistEpochAndMaintain } from "./context-handler/afterCompact.js";
27
+ import { appendMirrorAndLedger } from "./context-handler/dbMirrorAppend.js";
28
+ import { evaluateGate } from "./context-handler/gateCheck.js";
29
+ import { invokePipeline } from "./context-handler/pipelineRun.js";
30
+ import { buildLiveTrimView } from "./context-handler/liveTrim.js";
44
31
 
45
32
  /** Register the context event handler (live-trim auto-trigger). */
46
33
  export function registerContextHandler(
@@ -101,122 +88,14 @@ export function registerContextHandler(
101
88
 
102
89
  const view = viewForFallback ?? runtime.engineView(messages);
103
90
 
104
- // S27 DB-mirror: append incoming messages to raw_transcript.
105
- // Runs BEFORE fast-gate so every message is captured, even if we
106
- // don't compact this turn. Append is idempotent (content_hash PK).
107
- // F3: high-water mark (mirror-append.ts) skips already-processed
108
- // messages on subsequent events. On fork/rewind (shorter list or
109
- // boundary hash mismatch) the mark is dropped, falling back to a
110
- // full reprocess.
111
- if (config.dbMirror) {
112
- try {
113
- const db = openStore(runtime.currentStateDir);
114
- appendMirrorMessages(
115
- db,
116
- messages,
117
- runtime.rt.sessionId,
118
- epochIdFor(runtime.rt.sessionId),
119
- runtime.currentTurn,
120
- );
121
- // P2.2: populate conversation_thread + tool_results tables for
122
- // prompt-cache analytics and durable separation. The live-array
123
- // separation (buildSeparatedPrompt / buildCacheOptimizedPrompt in
124
- // tailResult above) is sufficient for the prompt-construction path;
125
- // these DB writes persist the split for post-hoc analysis, dashboard
126
- // queries, and future readers. Non-fatal — failure here never breaks
127
- // the agent loop (PREVENT-PI-004: zero network, local SQLite only).
128
- {
129
- const sid = runtime.rt.sessionId;
130
- const turn = runtime.currentTurn;
131
- const now = Date.now();
132
- const threadStmt = db.prepare(
133
- "INSERT OR IGNORE INTO conversation_thread (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)",
134
- );
135
- const toolStmt = db.prepare(
136
- "INSERT OR IGNORE INTO tool_results (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)",
137
- );
138
- for (const m of messages) {
139
- const role = m.role;
140
- const content = messageContentText(m);
141
- if (role === "user" || role === "assistant") {
142
- threadStmt.run(sid, role, content, turn, now);
143
- } else if (role === "toolResult" || role === "bashExecution") {
144
- toolStmt.run(sid, role, content, turn, now);
145
- }
146
- }
147
- }
148
- } catch (e) {
149
- runtime.logger.warn("db-mirror-append-fail", { error: String(e) });
150
- }
151
- }
91
+ // S27 DB-mirror + VC1B ledger append. Runs BEFORE the fast-gate so every
92
+ // message is captured, even if we don't compact this turn. Non-fatal.
93
+ appendMirrorAndLedger(runtime, config, messages);
152
94
 
153
- // VC1B (S1): canonical messages -> v2 ledger occurrences. Flag-OFF opens
154
- // no DB (byte-identical predecessor); non-fatal. onFailure surfaces
155
- // per-append rejections (e.g. EVT_SEQ_REGRESSION on rewind/fork) as
156
- // structured warnings rather than swallowing them silently.
157
- try {
158
- appendMessagesToLedger(
159
- runtime.currentStateDir,
160
- runtime.rt.sessionId,
161
- messages,
162
- runtime.logger,
163
- );
164
- } catch (e) {
165
- runtime.logger.warn("vc1b-ledger-append-fail", { error: String(e) });
166
- }
167
-
168
- // S52 / v0.16.1: per-model threshold override. The user can tune the
169
- // fire point + safety margin PER MODEL (different providers' models range
170
- // 8K-1M+ context, so one global tier % is wrong). Falls back to env/default
171
- // when no override row exists. Computed once here + reused in the tail cap
172
- // below; the lookup is a single SQLite PK hit (cheap; cached after the
173
- // first read in a session).
174
- const _modelIdForThreshold = runtime.currentModel?.modelId ?? null;
175
- const _perModelThreshold = resolveModelThreshold(_modelIdForThreshold, {
176
- safetyMarginFallback: DEFAULT_SAFETY_MARGIN_PCT,
177
- firePointFallback:
178
- config.tierPct != null
179
- ? Math.round(config.tierPct * 100)
180
- : DEFAULT_FIRE_POINT_PCT,
181
- stateDir: runtime.currentStateDir,
182
- });
183
- // S29 FAST GATE: drive the auto-trigger off the context % (the number the
184
- // menu bar shows), NOT the token count — the model under-reports tokens,
185
- // so a token-only gate misses the overshoot that causes max-output-tokens
186
- // truncation. The fire point is the per-model override when present,
187
- // otherwise the tier's percent threshold (tierPct) unless overridden by
188
- // MEGACOMPACT_AUTO_PCT_TRIGGER. `custom` (absolute
189
- // MEGACOMPACT_THRESHOLD_TOKENS, tierPct null) is an explicit opt-out of
190
- // percent scaling — it keeps the token gate. When pct is unavailable
191
- // (window unknown / a model that doesn't report percent) a tiered config
192
- // falls back to the token gate (S27 boot-fallback guarantee) instead of
193
- // skipping compaction — a percent-only gate would regress that.
194
- let gatePassed = false;
195
- if (config.tierPct != null && pct != null) {
196
- // Per-model override is a % (10-90); tierPct is a fraction (0.1-1.0).
197
- // Prefer the override; fall back to autoPctTrigger + tierPct.
198
- const tierPctFraction = config.autoPctTrigger ?? config.tierPct;
199
- const perModelFraction = _perModelThreshold.firePointPct / 100;
200
- const firePct =
201
- _modelIdForThreshold != null ? perModelFraction : tierPctFraction;
202
- gatePassed = pct / 100 >= firePct;
203
- } else {
204
- // custom tier OR tiered-but-pct-unavailable → token gate (S27 fallback).
205
- if (currentTokens < runtime.effectiveThreshold) {
206
- runtime.diagCtxFastGate++;
207
- return tailResult() ?? undefined;
208
- }
209
- const check = autoCompactCheck(currentTokens, runtime.effectiveThreshold); // SERVER-STYLE CONFIRM (local)
210
- if (!check.shouldCompact) {
211
- runtime.diagCtxNoCompact++;
212
- return tailResult() ?? undefined;
213
- }
214
- gatePassed = true;
215
- }
216
- if (!gatePassed) {
217
- runtime.diagCtxFastGate++;
218
- return tailResult() ?? undefined;
219
- }
95
+ // S29 FAST GATE: drive the auto-trigger off the context percent (see
96
+ // gateCheck.ts). Returns a tailed view when the gate does not pass.
97
+ const gate = evaluateGate(runtime, config, { pct, currentTokens, tailResult });
98
+ if (gate.kind === "return") return gate.view;
220
99
 
221
100
  // D.2: Replay MUST be exempt from debounce — replay is free (no compute,
222
101
  // no re-write) and prevents unnecessary KV-cache invalidation. Check
@@ -266,40 +145,20 @@ export function registerContextHandler(
266
145
  }
267
146
  runtime.debounceUntil = now + 2000;
268
147
 
269
- // Adaptive compression (Fix E): scale compression strength + keepFrom depth
270
- // with how close we are to the model context limit. Null-safe: when the
271
- // token-fallback path ran (pct unavailable) use the token-basis pressure
272
- // (the same basis the runtime `pressure` getter uses for custom/no-window).
273
- const pressure =
274
- pct != null
275
- ? pressureFromPct(pct)
276
- : pressureRatio(currentTokens, runtime.effectiveThreshold);
277
- const ran = runCompact(pi, runtime, config, ctx, messages, {
278
- compressionPressure: pressure,
148
+ // Adaptive-compression pipeline invocation (see pipelineRun.ts). Returns a
149
+ // tailed view ("return") when compaction skipped; otherwise "proceed" with
150
+ // the result + pressure consumed by the live-trim stage below.
151
+ const pipeline = invokePipeline(pi, runtime, config, ctx, {
152
+ messages,
153
+ pct,
154
+ currentTokens,
155
+ tailResult,
279
156
  });
280
- // D.3: skip paths fall back to replay instead of returning empty.
281
- // If runCompact skipped and we have a valid trimCache, replay it
282
- // (free stability win) — otherwise defer to the next event.
283
- if (ran.skipped) {
284
- runtime.diagCtxRunSkipped++;
285
- if (
286
- runtime.trimCache &&
287
- runtime.trimCache.checkpointId === runtime.rt.lastCheckpointId &&
288
- runtime.trimCache.cut <= messages.length
289
- ) {
290
- const recent = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized by computeLiveTrimCut (src/boundary.ts); replayed verbatim, transcript only grows within an epoch.
291
- runtime.diagLiveTrimFires++;
292
- runtime.diagLiveTrimReplays++;
293
- runtime.snapshot(ctx);
294
- const skipView = [{ ...runtime.trimCache.summaryAgentMsg }, ...recent];
295
- return tailResult(skipView) ?? { messages: skipView };
296
- }
297
- return tailResult() ?? undefined;
298
- }
157
+ if (pipeline.kind === "return") return pipeline.view;
299
158
 
300
159
  // S27 DB-mirror: write checkpoint_epoch + stamp turn epochs + auto-wiki +
301
160
  // topic seed + fire-and-forget dedup. Best-effort + non-fatal.
302
- await persistEpochAndMaintain(runtime, config, ran);
161
+ await persistEpochAndMaintain(runtime, config, pipeline.ran);
303
162
 
304
163
  // LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
305
164
  // manual compact path aborts the in-flight turn — only used behind the flag.
@@ -351,180 +210,19 @@ export function registerContextHandler(
351
210
  }
352
211
 
353
212
  // S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
354
- // Non-destructive: pi keeps the real transcript; only this LLM call sees the
355
- // trimmed window. We compute the cut on the engine view (pure, tested) then
356
- // slice the ORIGINAL pi AgentMessage[] from that index (lossless alignment,
357
- // mirroring dropCompactedRange) and prepend a user-role summary message.
358
- // A build failure or unsafe cut returns nothing (no trim this call — the
359
- // next context event retries). The anchor floor is read live from env (the
360
- // config value is the cached default) so it can be tuned per-test / per-run
361
- // without reloading the module.
362
- try {
363
- const anchorEnv = process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
364
- const anchorUserMessages =
365
- anchorEnv != null &&
366
- anchorEnv !== "" &&
367
- Number.isFinite(Number(anchorEnv))
368
- ? Number(anchorEnv)
369
- : config.anchorUserMessages;
370
- const cut = computeLiveTrimCut(view, {
371
- compactedFrom: ran.result.compactedFrom,
372
- summary: ran.result.summary,
373
- anchorUserMessages,
374
- // CRITICAL-OVER ESCAPE HATCH: when context is at/over ~90% of the
375
- // window, relief takes priority over the anchor floor. Without this,
376
- // computeLiveTrimCut bails to null (can't satisfy the floor) and the
377
- // model is fed a raw overflow that errors every turn — the
378
- // "Already compacted" + overflow death-spiral (2026-08-01 incident).
379
- // A thin anchor is recoverable; an overflowed session is not.
380
- //
381
- // CRITICAL: pct is null for OpenAI-compatible providers that don't
382
- // report usage.percent (e.g. neuralwatt). Without the token-pressure
383
- // fallback the hatch never armed → cut=null → raw overflow → 400
384
- // "conversation too long even after compaction" (2026-08-03 incident
385
- // on glm-5.2-short, 200K window). Now also fires on pressure >= 0.9
386
- // (token-basis) so the hatch arms regardless of whether the provider
387
- // reports pct.
388
- criticalOver: (pct ?? 0) >= 90 || pressure >= 0.9,
389
- });
390
- if (cut === null) {
391
- runtime.diagCtxCutNull++;
392
- runtime.logger.info("live-trim-skip", {
393
- sessionId: runtime.rt.sessionId,
394
- compactedFrom: ran.result.compactedFrom,
395
- viewLen: view.length,
396
- anchorUserMessages,
397
- criticalOver: (pct ?? 0) >= 90,
398
- });
399
- return tailResult() ?? undefined; // unsafe / below anchor floor — no trim this call
400
- }
401
- const summaryMsg = liveTrimSummaryMessage({
402
- compactedFrom: ran.result.compactedFrom,
403
- summary: ran.result.summary,
404
- anchorUserMessages: config.anchorUserMessages,
405
- });
406
- // Synthesize a user-role AgentMessage carrying the compacted summary.
407
- const summaryAgentMsg = {
408
- role: "user" as const,
409
- content: summaryMsg.text,
410
- // v0.8.6: stable timestamp across the epoch (NOT Date.now()) so the
411
- // summary message bytes — and thus the KV-cache prefix — don't drift
412
- // on every replay within the same compaction epoch.
413
- timestamp: runtime.rt.lastCompactAt ?? Date.now(),
414
- } as unknown as AgentMessage;
415
- const recentRaw = messages.slice(cut); // guardrails-allow PREVENT-PI-002: `cut` is the pre-sanitized `compactedFrom` produced by src/boundary.ts computeDropRange, so the preserved run begins on a toolPair-safe index.
416
-
417
- // FIX 2 (2026-08-03 incident): TOKEN-BUDGET CAP on the live-trim view.
418
- // Compaction fires at tier% of the window (140K for a 200K window),
419
- // but a SINGLE turn can inject a huge tool output (file read, bash) that
420
- // jumps context from 139K → 199K+ before the next gate fires. When that
421
- // happens [summary + preserved tail] can STILL exceed the model window,
422
- // and the provider rejects with 400 "conversation too long even after
423
- // compaction". The anchor floor (PREVENT-PI-001) keeps ≥N user messages
424
- // but has NO token cap, so a 2-message tail of two 80K bash outputs sails
425
- // right past the window.
426
- //
427
- // Cap: when the model context window is known, reserve room for the
428
- // summary + the model's max output tokens + a 10% safety margin, then
429
- // drop oldest preserved messages from the front of `recentRaw` until the
430
- // tail fits. Never drops below the FINAL message (always keep the latest
431
- // turn so the agent can respond). This is a last-resort HARD cap — it
432
- // only fires when the preserved tail alone is oversized, which is rare.
433
- const ctxWindow = runtime.lastCtxWindow;
434
- // Reuse the per-model threshold resolved at the gate (single lookup).
435
- const modelThreshold = _perModelThreshold;
436
- // Reserve room for output tokens. Use the model's reported max output
437
- // when known; fall back to 10% of the window (scales with any model —
438
- // 20K for a 200K window, 100K for a 1M window) so we never let the
439
- // preserved tail eat the model's output budget when maxTokens is unknown.
440
- const maxOutput =
441
- runtime.currentModel?.maxTokens && runtime.currentModel.maxTokens > 0
442
- ? runtime.currentModel.maxTokens
443
- : Math.ceil(ctxWindow * 0.1);
444
- let recent = recentRaw;
445
- if (ctxWindow > 0 && recentRaw.length > 1) {
446
- const summaryTokens = estimateBlockTokens(summaryMsg.text);
447
- // Reserve: summary + max output + per-model safety margin (0-20%).
448
- const safetyMargin = Math.ceil(
449
- ctxWindow * (modelThreshold.safetyMarginPct / 100),
450
- );
451
- const budget = ctxWindow - maxOutput - safetyMargin - summaryTokens;
452
- if (budget > 0) {
453
- // Walk recent from the front, dropping oldest first until the
454
- // remaining tail fits. Use the AgentMessage→engine-text estimate via
455
- // messageContentText (already imported) + estimateMessageTokens.
456
- let tailTokens = 0;
457
- for (let i = recentRaw.length - 1; i >= 0; i--) {
458
- const m = recentRaw[i];
459
- tailTokens += estimateMessageTokens({
460
- text: messageContentText(m),
461
- });
462
- if (tailTokens > budget) {
463
- // Keep from i+1 onward; but never fewer than the final message.
464
- const startIdx = Math.min(i + 1, recentRaw.length - 1);
465
- if (startIdx > 0) {
466
- recent = recentRaw.slice(startIdx);
467
- runtime.logger.warn("live-trim-tail-cap", {
468
- sessionId: runtime.rt.sessionId,
469
- dropped: startIdx,
470
- tailTokens,
471
- safetyMarginPct: modelThreshold.safetyMarginPct,
472
- budget,
473
- ctxWindow,
474
- });
475
- }
476
- break;
477
- }
478
- }
479
- }
480
- }
481
-
482
- // v0.8.6: cache the trim view so subsequent gated calls in this epoch
483
- // replay it verbatim (stabilizing the KV-cache prefix) instead of
484
- // regenerating a fresh summary + sentinel every fire.
485
- runtime.trimCache = {
486
- // v0.8.7: key the replay cache on the STABLE epoch signal
487
- // (rt.lastCheckpointId) instead of ran.result.checkpointId, which is
488
- // dedup-volatile: on a re-compact that dedups onto a DIFFERENT existing
489
- // checkpoint, result.checkpointId is the matched id (engine.ts:188) while
490
- // lastCheckpointId is only updated on a genuinely new checkpoint
491
- // (compact.ts:100-104). Keying on result.checkpointId would make
492
- // trimCache.checkpointId != rt.lastCheckpointId forever after that
493
- // dedup fire, disabling replay for the rest of the epoch (the
494
- // alternating cache-miss that 0.8.6 meant to fix). Prefer the stable
495
- // signal; fall back to result.checkpointId then the epoch timestamp
496
- // only for the no-checkpoint edge case.
497
- checkpointId:
498
- runtime.rt.lastCheckpointId ??
499
- ran.result.checkpointId ??
500
- `epoch-${runtime.rt.lastCompactAt ?? Date.now()}`,
501
- cut,
502
- summaryAgentMsg,
503
- ctxPct: pct ?? null,
504
- ctxTokens: currentTokens,
505
- };
506
- runtime.snapshot(ctx);
507
- // DIAG (team-run relief): confirm the live trim actually fires + how big
508
- // the window still is. The return is non-durable (per-LLM-call only), so
509
- // this is the signal that the model is being fed a compacted view while
510
- // the on-disk transcript + context meter keep growing.
511
- runtime.diagLiveTrimFires++;
512
- runtime.logger.info("live-trim", {
513
- sessionId: runtime.rt.sessionId,
514
- inputMsgs: messages.length,
515
- outputMsgs: recent.length + 1,
516
- compactedFrom: cut,
517
- ctxPct: pct,
518
- ctxTokens: usage?.tokens ?? null,
519
- });
520
- return (
521
- tailResult([summaryAgentMsg, ...recent]) ?? {
522
- messages: [summaryAgentMsg, ...recent],
523
- }
524
- );
525
- } catch {
526
- runtime.diagCtxThrown++;
527
- return tailResult() ?? undefined; // non-fatal: no trim this call; the next context event retries
528
- }
213
+ // See liveTrim.ts. Non-destructive: pi keeps the real transcript; only this
214
+ // LLM call sees the trimmed window. Returns undefined on unsafe cut/throw
215
+ // the next context event retries.
216
+ return buildLiveTrimView(runtime, config, ctx, {
217
+ messages,
218
+ view,
219
+ pct,
220
+ currentTokens,
221
+ usageTokens: usage?.tokens,
222
+ pressure: pipeline.pressure,
223
+ ran: pipeline.ran,
224
+ perModelThreshold: gate.perModelThreshold,
225
+ tailResult,
226
+ });
529
227
  });
530
228
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-mega-compact",
3
- "version": "0.17.0",
3
+ "version": "0.18.0",
4
4
  "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
5
5
  "type": "module",
6
6
  "license": "BSD-3-Clause",
@@ -58,6 +58,28 @@ export const VC1B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC1B");
58
58
  */
59
59
  export const VC0C_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC0C");
60
60
 
61
+ /**
62
+ * VC1C — cross-language conformance v2 (FixtureManifestV2 / DowngradeReport /
63
+ * MinHashV2 + M4 minhash-v2 migration).
64
+ * Default ON. `MEGACOMPACT_VC1C=0` disables and is byte-identical to the
65
+ * predecessor (mode C: a v2 conformance runner that accepts authority fixtures
66
+ * and the manifest validator idle; the sync dedup scan stays on the v1 path;
67
+ * zero `vector_cortex_*` VC1C emissions). The real consumers are the conformance
68
+ * emit seam, the minhash-v2 backfill seam and the downgrade-export seam.
69
+ */
70
+ export const VC1C_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC1C");
71
+
72
+ /**
73
+ * VC2A — offline model runtime and asset decision (ModelManifestV1 /
74
+ * EncoderRuntime).
75
+ * Default ON. `MEGACOMPACT_VC2A=0` disables and is byte-identical to the
76
+ * predecessor (mode C: no asset manifest is read/verified, the encoder runtime
77
+ * idles in mode C, zero `vector_cortex_encoder_*` emissions; the trigram/lexical
78
+ * paths are unchanged). The real consumers are the encoder emit seam and the
79
+ * encoder runtime's A/B/C selection.
80
+ */
81
+ export const VC2A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC2A");
82
+
61
83
  // ---------------------------------------------------------------------------
62
84
  // Breaker state machine constants (TRIAD_RESILIENCE.md §breaker).
63
85
  // Rolled numbers for one 60s window; VC0C consumes these at its breaker seam.
package/src/config.ts CHANGED
@@ -156,6 +156,8 @@ export {
156
156
  VC1A_ENABLED,
157
157
  VC0C_ENABLED,
158
158
  VC1B_ENABLED,
159
+ VC1C_ENABLED,
160
+ VC2A_ENABLED,
159
161
  BREAKER_WINDOW_MS,
160
162
  BREAKER_MIN_ATTEMPTS,
161
163
  BREAKER_PERF_FAILURES,
@@ -0,0 +1,78 @@
1
+ /**
2
+ * l1-lsh-v2.ts — Locality-Sensitive Hashing banding over MinHashV2 signatures
3
+ * (VC1C, M4 minhash-v2).
4
+ *
5
+ * Splits the frozen 2048-byte v2 signature (256 x u64 LE slots) into 64 bands
6
+ * of FOUR u64 values each. Each band yields a deterministic bucket key derived
7
+ * from its 4 u64 little-endian bytes (32 bytes) hashed with a 64-bit FNV-1a,
8
+ * scoped by session id and the frozen v2 version tag so buckets never collide
9
+ * across sessions or the v1 path. Deterministic: same (sessionId,
10
+ * signatureBytes) -> same 64 bucket keys, every run, every language.
11
+ *
12
+ * Mixed v1/v2 comparison and bucket mixing is REJECTED upstream with
13
+ * `MINHASH_VERSION_MISMATCH` (see minhash-v2 migration); this module only ever
14
+ * bands a v2 signature.
15
+ *
16
+ * Pure compute, no deps, no storage, no network (PREVENT-PI-004 / PREVENT-011).
17
+ */
18
+
19
+ import {
20
+ MINHASH_VERSION,
21
+ NUM_HASHES_V2,
22
+ U64_BYTES,
23
+ } from "./l1-minhash-v2.js";
24
+
25
+ /** Number of bands (frozen). */
26
+ export const BANDS_V2 = 64 as const;
27
+
28
+ /** u64 slots per band (frozen): 64 * 4 = 256 total slots. */
29
+ export const VALUES_PER_BAND_V2 = 4 as const;
30
+
31
+ /** Band byte length: 4 u64 LE = 32 bytes. */
32
+ export const BAND_BYTES_V2 = VALUES_PER_BAND_V2 * U64_BYTES;
33
+
34
+ /** Total signature byte length this scheme expects (256 * 8 = 2048). */
35
+ export const SIGNATURE_BYTES_EXPECTED_V2 = NUM_HASHES_V2 * U64_BYTES;
36
+
37
+ const MASK64 = 0xffffffffffffffffn;
38
+
39
+ /** 64-bit FNV-1a over a byte buffer. */
40
+ function fnv1a64Bytes(buf: Uint8Array): bigint {
41
+ let h = 0xcbf29ce484222325n;
42
+ for (let i = 0; i < buf.length; i++) {
43
+ h ^= BigInt(buf[i] ?? 0);
44
+ h = (h * 0x100000001b3n) & MASK64;
45
+ }
46
+ return h;
47
+ }
48
+
49
+ /**
50
+ * Compute the 64 LSH bucket keys for a v2 signature within a session. `bytes`
51
+ * is the 2048-byte little-endian signature (see encodeSignatureV2). Each bucket
52
+ * key is the hex of an 8-byte little-endian FNV-1a-64 over the band's 32 bytes
53
+ * prefixed by `<sessionId>|v2|band-` — so the "bucket bytes" are deterministic
54
+ * and reproducible across languages.
55
+ */
56
+ export function lshBandsV2(
57
+ bytes: Uint8Array,
58
+ sessionId: string,
59
+ version: number = MINHASH_VERSION,
60
+ ): string[] {
61
+ if (bytes.length !== SIGNATURE_BYTES_EXPECTED_V2) {
62
+ throw new Error(
63
+ `lshBandsV2: expected ${SIGNATURE_BYTES_EXPECTED_V2} signature bytes, got ${bytes.length}`,
64
+ );
65
+ }
66
+ const keys: string[] = [];
67
+ const prefix = Buffer.from(`${sessionId}|v${version}|`, "utf8");
68
+ for (let band = 0; band < BANDS_V2; band++) {
69
+ const start = band * BAND_BYTES_V2;
70
+ const bandBytes = bytes.slice(start, start + BAND_BYTES_V2);
71
+ const combined = Buffer.concat([prefix, Buffer.from(bandBytes)]);
72
+ const h = fnv1a64Bytes(combined);
73
+ const out = Buffer.allocUnsafe(U64_BYTES);
74
+ out.writeBigUInt64LE(h, 0);
75
+ keys.push(`b${band}:${out.toString("hex")}`);
76
+ }
77
+ return keys;
78
+ }