pi-mega-compact 0.6.9 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -8
- package/dist/extensions/dashboard-server.js +17 -6
- package/dist/extensions/mega-commands.js +12 -1
- package/dist/extensions/mega-compact.test.js +286 -51
- package/dist/extensions/mega-config.js +67 -5
- package/dist/extensions/mega-events.js +151 -27
- package/dist/extensions/mega-runtime.js +163 -32
- package/dist/extensions/openclaw-mega-compact.js +291 -0
- package/dist/src/dedup-engine.test.js +63 -38
- package/dist/src/minilm.js +92 -0
- package/dist/src/wordpiece.js +129 -0
- package/extensions/dashboard-server.ts +17 -6
- package/extensions/mega-commands.ts +12 -1
- package/extensions/mega-compact.test.ts +947 -516
- package/extensions/mega-config.ts +84 -6
- package/extensions/mega-dashboard.ts +11 -0
- package/extensions/mega-events.ts +558 -360
- package/extensions/mega-runtime.ts +168 -32
- package/package.json +1 -1
- package/src/dedup-engine.test.ts +103 -42
|
@@ -14,13 +14,13 @@ import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
|
14
14
|
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
15
15
|
import { join, dirname } from "node:path";
|
|
16
16
|
import { fileURLToPath } from "node:url";
|
|
17
|
-
import { readFileSync } from "node:fs";
|
|
17
|
+
import { readFileSync, appendFileSync, mkdirSync } from "node:fs";
|
|
18
18
|
import { VectorStore } from "../src/vectorStore.js";
|
|
19
19
|
import { toEngineMessages } from "../src/adapt.js";
|
|
20
20
|
import { normalizeSessionId } from "../src/store.js";
|
|
21
21
|
import { Logger } from "../src/log.js";
|
|
22
22
|
import { recordModelSnapshot, latestModelSnapshot, upsertRepoRegistry, recordRepoModel, type ModelSnapshot } from "../src/store/sqlite.js";
|
|
23
|
-
import { repoStateDir, resolveRepoRoot, pressureRatio, pressureFromPct, pressureBand, type MegaConfig, type PressureBand } from "./mega-config.js";
|
|
23
|
+
import { repoStateDir, resolveRepoRoot, pressureRatio, pressureFromPct, pressureBand, effectiveThresholdTokens, type MegaConfig, type PressureBand } from "./mega-config.js";
|
|
24
24
|
import { Dashboard, type DashboardSnapshot } from "./mega-dashboard.js";
|
|
25
25
|
|
|
26
26
|
export const STATUS_KEY = "mega-compact";
|
|
@@ -74,6 +74,50 @@ export const C = {
|
|
|
74
74
|
|
|
75
75
|
const PULSE = ["◐", "◓", "◑", "◒"];
|
|
76
76
|
|
|
77
|
+
// ── Full-width widget panel helpers ────────────────────────────────────────
|
|
78
|
+
// pi's above-editor widget renderer (a Container of Text lines) does NOT pass
|
|
79
|
+
// a terminal width to setWidget(), so lines render left-aligned by default. To
|
|
80
|
+
// make the widget read as a full-width status panel we pad each line to the
|
|
81
|
+
// real terminal width with a background fill. NOTE: C.reset is a FULL SGR
|
|
82
|
+
// reset, so we re-apply the panel bg after every reset to keep the background
|
|
83
|
+
// continuous under colored text (and under pi's own trailing reset).
|
|
84
|
+
const PANEL_BG = "\x1b[48;5;236m"; // dark slate panel background
|
|
85
|
+
const PANEL_RST = "\x1b[0m" + PANEL_BG; // reset fg but retain panel bg
|
|
86
|
+
|
|
87
|
+
/** Visible cell width of a string, ignoring ANSI SGR/OSC escapes. */
|
|
88
|
+
function visibleWidth(s: string): number {
|
|
89
|
+
const stripped = s
|
|
90
|
+
.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "")
|
|
91
|
+
.replace(/\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)/g, "");
|
|
92
|
+
let w = 0;
|
|
93
|
+
for (const ch of stripped) {
|
|
94
|
+
const cp = ch.codePointAt(0) ?? 0;
|
|
95
|
+
const wide = cp >= 0x1100 && (
|
|
96
|
+
(cp <= 0x115f) || (cp >= 0x2e80 && cp <= 0x303e) ||
|
|
97
|
+
(cp >= 0x3041 && cp <= 0x33ff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
|
98
|
+
(cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0xa000 && cp <= 0xa4cf) ||
|
|
99
|
+
(cp >= 0xac00 && cp <= 0xd7a3) || (cp >= 0xf900 && cp <= 0xfaff) ||
|
|
100
|
+
(cp >= 0xfe30 && cp <= 0xfe4f) || (cp >= 0xff00 && cp <= 0xff60) ||
|
|
101
|
+
(cp >= 0xffe0 && cp <= 0xffe6) || (cp >= 0x1f300 && cp <= 0x1faff) ||
|
|
102
|
+
(cp >= 0x20000 && cp <= 0x3fffd)
|
|
103
|
+
);
|
|
104
|
+
w += wide ? 2 : 1;
|
|
105
|
+
}
|
|
106
|
+
return w;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Pad a content string (with ANSI colors) to `width` cells using panel bg. */
|
|
110
|
+
function panelLine(content: string, width: number): string {
|
|
111
|
+
const withBg = PANEL_BG + content.replace(/\x1b\[0m/g, PANEL_RST);
|
|
112
|
+
const pad = Math.max(0, width - visibleWidth(withBg));
|
|
113
|
+
return withBg + " ".repeat(pad) + "\x1b[0m";
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** A full-width hairline bar (top/bottom border of the panel). */
|
|
117
|
+
function panelBar(width: number, ch = "─"): string {
|
|
118
|
+
return PANEL_BG + ch.repeat(Math.max(0, width)) + "\x1b[0m";
|
|
119
|
+
}
|
|
120
|
+
|
|
77
121
|
interface TickerEntry { text: string; at: number; }
|
|
78
122
|
|
|
79
123
|
export class MegaRuntime {
|
|
@@ -166,20 +210,58 @@ export class MegaRuntime {
|
|
|
166
210
|
diagCtxThrown = 0; // live-trim try threw (caught)
|
|
167
211
|
|
|
168
212
|
/**
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
*
|
|
173
|
-
|
|
174
|
-
|
|
213
|
+
* S26 capture instrumentation: the "model_snapshots empty → $0.00 cost card"
|
|
214
|
+
* bug was invisible because captureModel swallowed the DB write in a silent
|
|
215
|
+
* `catch {}`. These always-updated counters (zero cost) let a headless test or
|
|
216
|
+
* a live capture tell whether captureModel ran and whether the snapshot landed.
|
|
217
|
+
*/
|
|
218
|
+
diagCaptureModelCalls = 0; // captureModel entered with a populated ctx.model
|
|
219
|
+
diagCaptureModelFails = 0; // recordModelSnapshot threw → model_snapshots stays empty
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Live 0–1 pressure — how full the context window is relative to the
|
|
223
|
+
* compaction threshold.
|
|
224
|
+
*
|
|
225
|
+
* RECONCILE (BACKLOG dual-basis flicker): when the model context window is
|
|
226
|
+
* known we base pressure consistently on the *percentage* basis
|
|
227
|
+
* (`lastCtxPercent / (tierPct*100)`). This keeps the band stable whether the
|
|
228
|
+
* latest context event carried a token count or only a percentage, so the
|
|
229
|
+
* threshold comparison doesn't jump when a token-count event arrives vs a
|
|
230
|
+
* percent-only event. We only fall back to the token-count basis
|
|
231
|
+
* (`config.thresholdTokens`) when the window is unknown (e.g. before the first
|
|
232
|
+
* context event, or a `custom` tier with no tierPct). Always finite + in [0,1].
|
|
175
233
|
*/
|
|
176
234
|
get pressure(): number {
|
|
235
|
+
if (this.lastCtxWindow > 0 && this.config.tierPct != null && this.lastCtxPercent != null) {
|
|
236
|
+
// pressureFromPct(x) = x/100, and x = lastCtxPercent/tierPct, so this is
|
|
237
|
+
// exactly the intended lastCtxPercent/(tierPct*100) 0–1 ratio: at the
|
|
238
|
+
// fire point (lastCtxPercent == tierPct*100) pressure == 1.0, matching the
|
|
239
|
+
// token-based pressureRatio(currentTokens, effectiveThreshold) reading so
|
|
240
|
+
// the band doesn't jump when a token-count vs percent-only event arrives.
|
|
241
|
+
return pressureFromPct(this.lastCtxPercent / this.config.tierPct);
|
|
242
|
+
}
|
|
177
243
|
if (this.lastCtxTokens != null && this.lastCtxTokens > 0 && this.config.thresholdTokens > 0) {
|
|
178
244
|
return pressureRatio(this.lastCtxTokens, this.config.thresholdTokens);
|
|
179
245
|
}
|
|
180
246
|
return pressureFromPct(this.lastCtxPercent);
|
|
181
247
|
}
|
|
182
248
|
|
|
249
|
+
/**
|
|
250
|
+
* The live compaction FIRE POINT in tokens: the effective threshold scaled by
|
|
251
|
+
* the current model context window (`tierPct * window`) when known, else the
|
|
252
|
+
* boot fallback `config.thresholdTokens`. This is what the FAST GATE /
|
|
253
|
+
* `autoCompactCheck` / agent_end durable-trigger compare against, so
|
|
254
|
+
* compaction fires at tier% of the window for ANY model size (200k or 1M),
|
|
255
|
+
* always below pi's native auto-compaction (~80% of window).
|
|
256
|
+
*/
|
|
257
|
+
get effectiveThreshold(): number {
|
|
258
|
+
return effectiveThresholdTokens({
|
|
259
|
+
tierPct: this.config.tierPct,
|
|
260
|
+
fallbackThreshold: this.config.thresholdTokens,
|
|
261
|
+
window: this.lastCtxWindow,
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
|
|
183
265
|
/** Live discrete pressure band (low/medium/high/ultra/mega) over `pressure`. */
|
|
184
266
|
get pressureBand(): PressureBand {
|
|
185
267
|
return pressureBand(this.pressure);
|
|
@@ -251,8 +333,14 @@ export class MegaRuntime {
|
|
|
251
333
|
outputRate: modelSnap.outputRate,
|
|
252
334
|
}
|
|
253
335
|
: undefined;
|
|
254
|
-
|
|
255
|
-
|
|
336
|
+
// effectiveThresholdPct: the live fire point as a % of the window (null for
|
|
337
|
+
// `custom`, which has no tierPct). Used by armed/ready + the dashboard.
|
|
338
|
+
const effectiveThresholdPct = this.config.tierPct != null ? this.config.tierPct * 100 : null;
|
|
339
|
+
// armed lights at/above the REAL fire point: max(effectiveThresholdPct,
|
|
340
|
+
// fastGatePct). fastGatePct already equals tierPct*100 by default, but a
|
|
341
|
+
// MEGACOMPACT_FAST_GATE_PCT override can raise it, so we take the max.
|
|
342
|
+
const armed = this.lastCtxPercent != null && this.lastCtxPercent >= Math.max(effectiveThresholdPct ?? 0, this.config.fastGatePct);
|
|
343
|
+
const ready = armed && (this.lastCtxTokens ?? 0) >= this.effectiveThreshold;
|
|
256
344
|
this.dashboard.snapshot({
|
|
257
345
|
version: 1,
|
|
258
346
|
updatedAt: new Date().toISOString(),
|
|
@@ -263,7 +351,9 @@ export class MegaRuntime {
|
|
|
263
351
|
pressure: this.pressure,
|
|
264
352
|
config: {
|
|
265
353
|
fastGatePct: this.config.fastGatePct,
|
|
266
|
-
thresholdTokens: this.
|
|
354
|
+
thresholdTokens: this.effectiveThreshold,
|
|
355
|
+
tierPct: this.config.tierPct,
|
|
356
|
+
effectiveThresholdPct,
|
|
267
357
|
anchorUserMessages: this.config.anchorUserMessages,
|
|
268
358
|
preserveRecent: this.config.preserveRecent,
|
|
269
359
|
auto: this.config.auto,
|
|
@@ -280,7 +370,7 @@ export class MegaRuntime {
|
|
|
280
370
|
dedupAttempts: this.rt.dedupAttempts,
|
|
281
371
|
},
|
|
282
372
|
context: { tokens: this.lastCtxTokens, percent: this.lastCtxPercent, contextWindow: this.lastCtxWindow },
|
|
283
|
-
trigger: { armed, ready, currentTokens: this.lastCtxTokens, thresholdTokens: this.
|
|
373
|
+
trigger: { armed, ready, currentTokens: this.lastCtxTokens, thresholdTokens: this.effectiveThreshold, fastGatePct: this.config.fastGatePct, tierPct: this.config.tierPct, effectiveThresholdPct },
|
|
284
374
|
crew: { activeAgents: this.activeAgents, currentTurn: this.currentTurn },
|
|
285
375
|
store: { checkpointCount: st.checkpointCount, totalTokenEstimate: st.totalTokenEstimate, originalTokens: st.originalTokens, tokensSaved: this.rt.tokensSaved, injectedCount: st.injectedCount, dedupHitRate: st.dedupHitRate, storageDedupRate: st.storageDedupRate, dedupAttempts: st.dedupAttempts, dedupCollapsed: st.dedupCollapsed },
|
|
286
376
|
// Reconciled token accounting (single canonical formula, session + repo).
|
|
@@ -348,7 +438,14 @@ export class MegaRuntime {
|
|
|
348
438
|
x >= 1_000_000 ? `${(x / 1_000_000).toFixed(1)}mil`
|
|
349
439
|
: x >= 1000 ? `${(x / 1000).toFixed(1)}k`
|
|
350
440
|
: `${Math.round(x)}`;
|
|
351
|
-
|
|
441
|
+
// Agents view: ALWAYS show the agent line so status is visible even when
|
|
442
|
+
// idle (previously hidden at 0). 🤖 N agents when active, dimmed 🤖 idle
|
|
443
|
+
// when none — this is the restored "agents view" (count + status). Real
|
|
444
|
+
// per-agent/sub-agent token usage is scoped in Sprint 27.
|
|
445
|
+
const agentLabel = this.activeAgents > 0
|
|
446
|
+
? `🤖 ${this.activeAgents} agent${this.activeAgents === 1 ? "" : "s"}`
|
|
447
|
+
: `${C.dim}🤖 idle${C.reset}`;
|
|
448
|
+
const agentStr = ` │ ${agentLabel}`;
|
|
352
449
|
const turnStr = this.currentTurn > 0 ? ` │ turn ${this.currentTurn}` : "";
|
|
353
450
|
// Phase 3 — pulsing status glyph while a compaction is in flight.
|
|
354
451
|
const pulse = this.pulsing ? `${C.cyan}${PULSE[Math.floor(Date.now() / 250) % PULSE.length]}${C.reset} ` : "";
|
|
@@ -381,37 +478,40 @@ export class MegaRuntime {
|
|
|
381
478
|
const ctxPct = this.lastCtxPercent != null ? this.lastCtxPercent / 100 : 0;
|
|
382
479
|
const sTxt = (sessPct * 100).toFixed(sessPct * 100 >= 10 ? 0 : 1);
|
|
383
480
|
const rTxt = (repoPct * 100).toFixed(repoPct * 100 >= 10 ? 0 : 1);
|
|
481
|
+
// Full-width panel: read the real terminal width and pad each line with a
|
|
482
|
+
// panel background so the above-editor widget reads as a full-width status
|
|
483
|
+
// bar. pi's widget renderer does not pass width to setWidget(), so we pad
|
|
484
|
+
// ourselves. Falls back to 200 cols when stdout.columns is unavailable.
|
|
485
|
+
const W = process.stdout?.columns ?? 200;
|
|
384
486
|
const lines = [
|
|
487
|
+
// top border — full-width hairline
|
|
488
|
+
panelBar(W, "─"),
|
|
385
489
|
// L1 — header: tier + ctx-fill bar (20-cell, green=room→red=full) +
|
|
386
|
-
// tokens + status glyph + checkpoints + agents/turn.
|
|
387
|
-
//
|
|
388
|
-
` ${C.amber}⚡ ${tierLabel}${C.reset} v${C.bold}${ownVersion()}${C.reset} ${ramp(ctxPct, 20)} ${C.bold}${pctStr}${C.reset} ${tokStr}/${maxStr} │ ${triggerLabel} │ ${st.checkpointCount} chk${agentStr}${turnStr}`,
|
|
490
|
+
// tokens + status glyph + checkpoints + agents/turn. The context bar is
|
|
491
|
+
// the only live-moving bar; the whole block is padded to full width.
|
|
492
|
+
panelLine(` ${C.amber}⚡ ${tierLabel}${C.reset} v${C.bold}${ownVersion()}${C.reset} ${ramp(ctxPct, 20)} ${C.bold}${pctStr}${C.reset} ${tokStr}/${maxStr} │ ${triggerLabel} │ ${st.checkpointCount} chk${agentStr}${turnStr}`, W),
|
|
389
493
|
// L2 — savings EXPLAINED, not bar'd. The freed/(freed+kept) ratio
|
|
390
|
-
// saturates near 100% once cumulative freed dwarfs live kept
|
|
391
|
-
//
|
|
392
|
-
|
|
393
|
-
// down to M, freeing X%". Plus repo-wide chk/session counts.
|
|
394
|
-
` ${C.magenta}dup ${dedupStr}${C.reset} │ ${C.gray}sess${C.reset} ${fmt(sessIn)}→${fmt(sessKept)} kept ${C.green}(${sTxt}% freed)${C.reset} · ${C.gray}all-time${C.reset} ${fmt(repoIn)}→${fmt(repoKept)} kept ${C.blue}(${rTxt}% freed)${C.reset} │ ${repo.checkpointCount} chk/${repo.sessionCount} sess`,
|
|
494
|
+
// saturates near 100% once cumulative freed dwarfs live kept, so a bar
|
|
495
|
+
// is visually useless; show the compaction story instead.
|
|
496
|
+
panelLine(` ${C.magenta}dup ${dedupStr}${C.reset} │ ${C.gray}sess${C.reset} ${fmt(sessIn)}→${fmt(sessKept)} kept ${C.green}(${sTxt}% freed)${C.reset} · ${C.gray}all-time${C.reset} ${fmt(repoIn)}→${fmt(repoKept)} kept ${C.blue}(${rTxt}% freed)${C.reset} │ ${repo.checkpointCount} chk/${repo.sessionCount} sess`, W),
|
|
395
497
|
];
|
|
396
498
|
// Live "now processing" line + why + recent deduped/compacted events,
|
|
397
|
-
// collapsed to ONE rotating line (fresh only)
|
|
398
|
-
// (≤5 most-recent events) is cycled one-per-repaint so the line scrolls
|
|
399
|
-
// through recent files in real time while activity fires. We rotate on a
|
|
400
|
-
// 250ms step (same cadence as the pulse), using an event counter as the
|
|
401
|
-
// deterministic phase so consecutive repaints advance the visible entry.
|
|
499
|
+
// collapsed to ONE rotating line (fresh only); padded to full width.
|
|
402
500
|
const fresh = Date.now() - this.lastActivityAt < 4000;
|
|
403
501
|
if (this.tierTrace && fresh) {
|
|
404
|
-
lines.push(` ${pulse}${this.tierTrace}
|
|
502
|
+
lines.push(panelLine(` ${pulse}${this.tierTrace}`, W));
|
|
405
503
|
} else if (this.ticker.length > 0) {
|
|
406
504
|
const step = Math.floor(Date.now() / 250);
|
|
407
505
|
const idx = this.ticker.length - 1 - (step % this.ticker.length);
|
|
408
506
|
const head = this.ticker[idx].text;
|
|
409
507
|
const why = this.lastWhy ? ` ${C.gray}· ${this.lastWhy}${C.reset}` : "";
|
|
410
508
|
const more = this.ticker.length > 1 ? ` ${C.dim}(+${this.ticker.length - 1} more)${C.reset}` : "";
|
|
411
|
-
lines.push(` ${fresh ? C.teal : C.dim}${head}${why}${more}${C.reset}
|
|
509
|
+
lines.push(panelLine(` ${fresh ? C.teal : C.dim}${head}${why}${more}${C.reset}`, W));
|
|
412
510
|
} else if (this.pulsing) {
|
|
413
|
-
lines.push(` ${pulse}${C.teal}compacting…${C.reset}
|
|
511
|
+
lines.push(panelLine(` ${pulse}${C.teal}compacting…${C.reset}`, W));
|
|
414
512
|
}
|
|
513
|
+
// bottom border — full-width hairline closes the panel
|
|
514
|
+
lines.push(panelBar(W, "─"));
|
|
415
515
|
// (Accounting folded into L2's "in→kept (X% freed)" framing — freed =
|
|
416
516
|
// in − kept is implied, and the saturated-ratio bars are gone.)
|
|
417
517
|
ctx.ui.setWidget(WIDGET_KEY, lines, { placement: "aboveEditor" });
|
|
@@ -454,7 +554,7 @@ export class MegaRuntime {
|
|
|
454
554
|
*/
|
|
455
555
|
captureModel(ctx: ExtensionContext): void {
|
|
456
556
|
const m = ctx.model;
|
|
457
|
-
if (!m) return;
|
|
557
|
+
if (!m) { this.appendEvent("captureModel:no-model", { cwd: ctx.cwd }); return; }
|
|
458
558
|
if (this.currentModel && this.currentModel.modelId === m.id && this.currentModel.provider === m.provider) return;
|
|
459
559
|
let providerName: string | null = null;
|
|
460
560
|
try { providerName = ctx.modelRegistry?.getProviderDisplayName(m.provider) ?? null; } catch { /* optional */ }
|
|
@@ -470,9 +570,27 @@ export class MegaRuntime {
|
|
|
470
570
|
reasoning: !!m.reasoning,
|
|
471
571
|
};
|
|
472
572
|
this.currentModel = { ...snap, capturedAt: Date.now() };
|
|
573
|
+
this.diagCaptureModelCalls++;
|
|
574
|
+
const repo = resolveRepoRoot(ctx.cwd) ?? this.currentStateDir;
|
|
575
|
+
// S26: previously a single silent `catch {}` hid every capture failure, so
|
|
576
|
+
// model_snapshots stayed empty and the cost card read $0.00 with zero signal.
|
|
577
|
+
// Split per-write + append to events.log (always-on, dashboard live-streams
|
|
578
|
+
// it) + bump a DIAG counter so a live capture surfaces the root cause.
|
|
473
579
|
try {
|
|
474
|
-
const repo = resolveRepoRoot(ctx.cwd) ?? this.currentStateDir;
|
|
475
580
|
recordModelSnapshot(repo, snap, this.currentStateDir);
|
|
581
|
+
this.appendEvent("captureModel:recorded", {
|
|
582
|
+
repo, modelId: snap.modelId, provider: snap.provider,
|
|
583
|
+
inputRate: snap.inputRate, outputRate: snap.outputRate,
|
|
584
|
+
});
|
|
585
|
+
} catch (e) {
|
|
586
|
+
this.diagCaptureModelFails++;
|
|
587
|
+
this.appendEvent("captureModel:record-failed", {
|
|
588
|
+
repo, modelId: snap.modelId,
|
|
589
|
+
error: e instanceof Error ? e.message : String(e),
|
|
590
|
+
stack: e instanceof Error ? e.stack : undefined,
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
try {
|
|
476
594
|
// Denormalize the active model into the machine-wide index so the
|
|
477
595
|
// All-repos dashboard table can show provider/model per repo without
|
|
478
596
|
// opening every repo's DB. Best-effort + non-fatal.
|
|
@@ -485,7 +603,25 @@ export class MegaRuntime {
|
|
|
485
603
|
stateDir: this.currentStateDir,
|
|
486
604
|
displayName: repo.split(/[\\/]/).filter(Boolean).pop() ?? repo,
|
|
487
605
|
});
|
|
488
|
-
} catch {
|
|
606
|
+
} catch (e) {
|
|
607
|
+
this.appendEvent("captureModel:index-record-failed", {
|
|
608
|
+
repo, modelId: snap.modelId,
|
|
609
|
+
error: e instanceof Error ? e.message : String(e),
|
|
610
|
+
});
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/**
|
|
615
|
+
* Append a structured line to the repo's events.log — the always-on
|
|
616
|
+
* diagnostics sink the dashboard live-streams. Unlike this.logger (gated by
|
|
617
|
+
* config.debug), this fires in production, so capture failures surface during
|
|
618
|
+
* a real capture even with debugging off. Best-effort + non-fatal.
|
|
619
|
+
*/
|
|
620
|
+
private appendEvent(event: string, fields: Record<string, unknown>): void {
|
|
621
|
+
try {
|
|
622
|
+
mkdirSync(this.currentStateDir, { recursive: true });
|
|
623
|
+
appendFileSync(join(this.currentStateDir, "events.log"), JSON.stringify({ ts: Date.now(), event, ...fields }) + "\n");
|
|
624
|
+
} catch { /* non-fatal */ }
|
|
489
625
|
}
|
|
490
626
|
|
|
491
627
|
/** S21: state dir of the currently bound repo (where memories live). */
|
package/package.json
CHANGED
package/src/dedup-engine.test.ts
CHANGED
|
@@ -24,6 +24,12 @@ import { autoCompactCheck } from "./compact.js";
|
|
|
24
24
|
import { loadDedupConfig, type DedupConfigShape } from "./config/dedup.js";
|
|
25
25
|
import type { EngineMessage } from "./types.js";
|
|
26
26
|
|
|
27
|
+
// Real percentage-based threshold config. Replaces the previous LOCAL replica of
|
|
28
|
+
// COMPACT_TIERS + resolveThresholdFromEnv that asserted the OLD static token
|
|
29
|
+
// amounts — importing the live source of truth keeps tests in sync with the
|
|
30
|
+
// source (thresholds are tierPct × the model's context window, not fixed tokens).
|
|
31
|
+
import { TIER_PCT, effectiveThresholdTokens, loadConfig } from "../extensions/mega-config.js";
|
|
32
|
+
|
|
27
33
|
// recallAndInline may or may not be exported; import safely.
|
|
28
34
|
import * as recallMod from "./recall.js";
|
|
29
35
|
|
|
@@ -540,48 +546,63 @@ describe("Edge Cases", () => {
|
|
|
540
546
|
});
|
|
541
547
|
});
|
|
542
548
|
|
|
543
|
-
// -------------------- 7. Tier Switching --------------------
|
|
549
|
+
// -------------------- 7. Tier Switching (percentage-based) --------------------
|
|
550
|
+
|
|
551
|
+
// Replaces the previous LOCAL replica of COMPACT_TIERS + resolveThresholdFromEnv
|
|
552
|
+
// that asserted the OLD static token amounts. We now import the REAL config
|
|
553
|
+
// helpers from extensions/mega-config.js so the tests track the live source of
|
|
554
|
+
// truth: thresholds are tierPct × the model's context window (not fixed tokens).
|
|
555
|
+
|
|
556
|
+
describe("Tier Switching — percentage-based thresholds", () => {
|
|
557
|
+
// Documented tierPct fractions (single source of truth in mega-config.ts).
|
|
558
|
+
it("each named tier carries the documented tierPct fraction", () => {
|
|
559
|
+
assert.equal(TIER_PCT.low, 0.5);
|
|
560
|
+
assert.equal(TIER_PCT.medium, 0.6);
|
|
561
|
+
assert.equal(TIER_PCT.high, 0.7);
|
|
562
|
+
assert.equal(TIER_PCT.ultra, 0.7);
|
|
563
|
+
assert.equal(TIER_PCT.mega, 0.75);
|
|
564
|
+
});
|
|
544
565
|
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
["
|
|
550
|
-
["
|
|
551
|
-
["
|
|
552
|
-
["
|
|
566
|
+
// Boot fallback threshold (sane gate before the first context event supplies a
|
|
567
|
+
// window): round(tierPct × 200_000). Resolved through the REAL loadConfig().
|
|
568
|
+
it("MEGACOMPACT_TIER env resolves to the boot fallback threshold via real config", () => {
|
|
569
|
+
const tiers: Array<[keyof typeof TIER_PCT, number]> = [
|
|
570
|
+
["low", 100_000], // 0.50 × 200_000
|
|
571
|
+
["medium", 120_000], // 0.60 × 200_000
|
|
572
|
+
["high", 140_000], // 0.70 × 200_000
|
|
573
|
+
["ultra", 140_000], // 0.70 × 200_000
|
|
574
|
+
["mega", 150_000], // 0.75 × 200_000
|
|
553
575
|
];
|
|
554
|
-
|
|
555
|
-
for (const [tier, expectedThreshold] of tiers) {
|
|
576
|
+
for (const [tier, expectedBoot] of tiers) {
|
|
556
577
|
const original = process.env.MEGACOMPACT_TIER;
|
|
578
|
+
delete process.env.MEGACOMPACT_THRESHOLD_TOKENS;
|
|
557
579
|
process.env.MEGACOMPACT_TIER = tier;
|
|
558
580
|
try {
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
581
|
+
const cfg = loadConfig();
|
|
582
|
+
assert.equal(cfg.tier, tier, `tier ${tier} should resolve`);
|
|
583
|
+
assert.equal(cfg.tierPct, TIER_PCT[tier], `tier ${tier} tierPct`);
|
|
562
584
|
assert.equal(
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
`tier ${tier} should
|
|
585
|
+
cfg.thresholdTokens,
|
|
586
|
+
expectedBoot,
|
|
587
|
+
`tier ${tier} boot fallback threshold should be ${expectedBoot}`,
|
|
566
588
|
);
|
|
567
589
|
} finally {
|
|
568
|
-
if (original === undefined)
|
|
569
|
-
|
|
570
|
-
} else {
|
|
571
|
-
process.env.MEGACOMPACT_TIER = original;
|
|
572
|
-
}
|
|
590
|
+
if (original === undefined) delete process.env.MEGACOMPACT_TIER;
|
|
591
|
+
else process.env.MEGACOMPACT_TIER = original;
|
|
573
592
|
}
|
|
574
593
|
}
|
|
575
594
|
});
|
|
576
595
|
|
|
577
|
-
it("explicit MEGACOMPACT_THRESHOLD_TOKENS overrides tier", () => {
|
|
596
|
+
it("explicit MEGACOMPACT_THRESHOLD_TOKENS overrides tier (custom stays absolute)", () => {
|
|
578
597
|
const originalTier = process.env.MEGACOMPACT_TIER;
|
|
579
598
|
const originalThreshold = process.env.MEGACOMPACT_THRESHOLD_TOKENS;
|
|
580
|
-
process.env.MEGACOMPACT_TIER
|
|
599
|
+
delete process.env.MEGACOMPACT_TIER;
|
|
581
600
|
process.env.MEGACOMPACT_THRESHOLD_TOKENS = "123456";
|
|
582
601
|
try {
|
|
583
|
-
const
|
|
584
|
-
assert.equal(
|
|
602
|
+
const cfg = loadConfig();
|
|
603
|
+
assert.equal(cfg.tier, "custom", "explicit token threshold → custom tier");
|
|
604
|
+
assert.equal(cfg.tierPct, null, "custom tier has no tierPct (stays absolute)");
|
|
605
|
+
assert.equal(cfg.thresholdTokens, 123_456, "explicit token threshold should win");
|
|
585
606
|
} finally {
|
|
586
607
|
if (originalTier === undefined) delete process.env.MEGACOMPACT_TIER;
|
|
587
608
|
else process.env.MEGACOMPACT_TIER = originalTier;
|
|
@@ -591,19 +612,59 @@ describe("Tier Switching", () => {
|
|
|
591
612
|
});
|
|
592
613
|
});
|
|
593
614
|
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
615
|
+
describe("effectiveThresholdTokens — tierPct × model window", () => {
|
|
616
|
+
// The real compaction fire point. Tiered → scales with the window so it always
|
|
617
|
+
// fires BELOW pi's native ~80% auto-compact for any model size. Custom (null
|
|
618
|
+
// tierPct) → absolute explicitThreshold, never percent-scaled.
|
|
619
|
+
|
|
620
|
+
it("scales tierPct × window for a 200k model", () => {
|
|
621
|
+
assert.equal(
|
|
622
|
+
effectiveThresholdTokens({ tierPct: TIER_PCT.low, fallbackThreshold: 100_000, window: 200_000 }),
|
|
623
|
+
100_000,
|
|
624
|
+
);
|
|
625
|
+
assert.equal(
|
|
626
|
+
effectiveThresholdTokens({ tierPct: TIER_PCT.mega, fallbackThreshold: 150_000, window: 200_000 }),
|
|
627
|
+
150_000,
|
|
628
|
+
);
|
|
629
|
+
});
|
|
630
|
+
|
|
631
|
+
it("scales tierPct × window for a 1M model", () => {
|
|
632
|
+
assert.equal(
|
|
633
|
+
effectiveThresholdTokens({ tierPct: TIER_PCT.low, fallbackThreshold: 500_000, window: 1_000_000 }),
|
|
634
|
+
500_000,
|
|
635
|
+
);
|
|
636
|
+
assert.equal(
|
|
637
|
+
effectiveThresholdTokens({ tierPct: TIER_PCT.mega, fallbackThreshold: 750_000, window: 1_000_000 }),
|
|
638
|
+
750_000,
|
|
639
|
+
);
|
|
640
|
+
});
|
|
641
|
+
|
|
642
|
+
it("falls back to the boot threshold when the window is 0/unknown", () => {
|
|
643
|
+
assert.equal(
|
|
644
|
+
effectiveThresholdTokens({ tierPct: TIER_PCT.mega, fallbackThreshold: 150_000, window: 0 }),
|
|
645
|
+
150_000,
|
|
646
|
+
);
|
|
647
|
+
assert.equal(
|
|
648
|
+
effectiveThresholdTokens({ tierPct: TIER_PCT.low, fallbackThreshold: 100_000, window: -5 }),
|
|
649
|
+
100_000,
|
|
650
|
+
);
|
|
651
|
+
});
|
|
652
|
+
|
|
653
|
+
it("custom (tierPct null) stays an absolute threshold regardless of window", () => {
|
|
654
|
+
assert.equal(
|
|
655
|
+
effectiveThresholdTokens({ tierPct: null, fallbackThreshold: 100_000, window: 200_000, explicitThreshold: 123456 }),
|
|
656
|
+
123456,
|
|
657
|
+
"explicit absolute wins (200k window)",
|
|
658
|
+
);
|
|
659
|
+
assert.equal(
|
|
660
|
+
effectiveThresholdTokens({ tierPct: null, fallbackThreshold: 100_000, window: 1_000_000, explicitThreshold: 123456 }),
|
|
661
|
+
123456,
|
|
662
|
+
"explicit absolute wins (1M window)",
|
|
663
|
+
);
|
|
664
|
+
assert.equal(
|
|
665
|
+
effectiveThresholdTokens({ tierPct: null, fallbackThreshold: 100_000, window: 200_000 }),
|
|
666
|
+
100_000,
|
|
667
|
+
"no explicit → boot fallback",
|
|
668
|
+
);
|
|
669
|
+
});
|
|
670
|
+
});
|