pi-mega-compact 0.7.2 → 0.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/mega-config.js +1 -0
- package/dist/extensions/mega-events.js +117 -4
- package/dist/extensions/mega-events.test.js +47 -0
- package/dist/extensions/mega-runtime.js +225 -97
- package/dist/src/mirror/dedup.js +44 -0
- package/dist/src/mirror/epoch.js +36 -0
- package/dist/src/mirror/mirror.test.js +185 -0
- package/dist/src/recall.js +37 -0
- package/dist/src/store/sqlite.dbmirror.test.js +175 -0
- package/dist/src/store/sqlite.js +248 -1
- package/extensions/mega-config.ts +8 -0
- package/extensions/mega-events.test.ts +55 -0
- package/extensions/mega-events.ts +126 -4
- package/extensions/mega-runtime.ts +941 -737
- package/package.json +1 -1
- package/src/mirror/dedup.ts +57 -0
- package/src/mirror/epoch.ts +37 -0
- package/src/mirror/mirror.test.ts +240 -0
- package/src/recall.ts +38 -0
- package/src/store/sqlite.dbmirror.test.ts +219 -0
- package/src/store/sqlite.ts +378 -1
|
@@ -16,9 +16,9 @@ import { VectorStore } from "../src/vectorStore.js";
|
|
|
16
16
|
import { toEngineMessages } from "../src/adapt.js";
|
|
17
17
|
import { normalizeSessionId } from "../src/store.js";
|
|
18
18
|
import { Logger } from "../src/log.js";
|
|
19
|
-
import { recordModelSnapshot, latestModelSnapshot, upsertRepoRegistry, recordRepoModel } from "../src/store/sqlite.js";
|
|
19
|
+
import { recordModelSnapshot, latestModelSnapshot, upsertRepoRegistry, recordRepoModel, } from "../src/store/sqlite.js";
|
|
20
20
|
import { detectCrossRepoDrift } from "../src/driftDetection.js";
|
|
21
|
-
import { repoStateDir, resolveRepoRoot, pressureRatio, pressureFromPct, pressureBand, effectiveThresholdTokens } from "./mega-config.js";
|
|
21
|
+
import { repoStateDir, resolveRepoRoot, pressureRatio, pressureFromPct, pressureBand, effectiveThresholdTokens, } from "./mega-config.js";
|
|
22
22
|
import { Dashboard } from "./mega-dashboard.js";
|
|
23
23
|
export const STATUS_KEY = "mega-compact";
|
|
24
24
|
export const WIDGET_KEY = "mega-compact-stats";
|
|
@@ -74,39 +74,51 @@ function visibleWidth(s) {
|
|
|
74
74
|
let w = 0;
|
|
75
75
|
for (const ch of stripped) {
|
|
76
76
|
const cp = ch.codePointAt(0) ?? 0;
|
|
77
|
-
const wide = cp >= 0x1100 &&
|
|
78
|
-
(cp
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
77
|
+
const wide = cp >= 0x1100 &&
|
|
78
|
+
(cp <= 0x115f ||
|
|
79
|
+
(cp >= 0x2e80 && cp <= 0x303e) ||
|
|
80
|
+
(cp >= 0x3041 && cp <= 0x33ff) ||
|
|
81
|
+
(cp >= 0x3400 && cp <= 0x4dbf) ||
|
|
82
|
+
(cp >= 0x4e00 && cp <= 0x9fff) ||
|
|
83
|
+
(cp >= 0xa000 && cp <= 0xa4cf) ||
|
|
84
|
+
(cp >= 0xac00 && cp <= 0xd7a3) ||
|
|
85
|
+
(cp >= 0xf900 && cp <= 0xfaff) ||
|
|
86
|
+
(cp >= 0xfe30 && cp <= 0xfe4f) ||
|
|
87
|
+
(cp >= 0xff00 && cp <= 0xff60) ||
|
|
88
|
+
(cp >= 0xffe0 && cp <= 0xffe6) ||
|
|
89
|
+
(cp >= 0x1f300 && cp <= 0x1faff) ||
|
|
90
|
+
(cp >= 0x20000 && cp <= 0x3fffd));
|
|
84
91
|
w += wide ? 2 : 1;
|
|
85
92
|
}
|
|
86
93
|
return w;
|
|
87
94
|
}
|
|
88
95
|
/** Pad a content string (with ANSI colors) to `width` cells using panel bg. */
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
96
|
+
/** Wrap a string (with ANSI codes) to fit within `maxWidth` visible chars.
|
|
97
|
+
* Splits at │ separators or whitespace when possible. */
|
|
98
|
+
function wrapLine(text, maxWidth) {
|
|
99
|
+
if (maxWidth <= 0)
|
|
100
|
+
return [text];
|
|
101
|
+
const result = [];
|
|
102
|
+
let current = "";
|
|
103
|
+
let currentW = 0;
|
|
104
|
+
// Split at │ boundaries first
|
|
105
|
+
const segments = text.split("│");
|
|
106
|
+
for (let i = 0; i < segments.length; i++) {
|
|
107
|
+
const seg = (i > 0 ? "│" : "") + segments[i];
|
|
108
|
+
const segW = visibleWidth(PANEL_BG + seg.replace(/\x1b\[0m/g, PANEL_RST));
|
|
109
|
+
if (currentW + segW <= maxWidth || currentW === 0) {
|
|
110
|
+
current += seg;
|
|
111
|
+
currentW += segW;
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
result.push(current);
|
|
115
|
+
current = seg;
|
|
116
|
+
currentW = segW;
|
|
107
117
|
}
|
|
108
118
|
}
|
|
109
|
-
|
|
119
|
+
if (current)
|
|
120
|
+
result.push(current);
|
|
121
|
+
return result;
|
|
110
122
|
}
|
|
111
123
|
function panelLine(content, width) {
|
|
112
124
|
const withBg = PANEL_BG + content.replace(/\x1b\[0m/g, PANEL_RST);
|
|
@@ -120,8 +132,10 @@ function panelBar(width, ch = "─") {
|
|
|
120
132
|
/** Token-count formatter: M at/above 1e6, k at/above 1e3, raw below.
|
|
121
133
|
* 5,472,700 → "5.5mil", 24,100 → "24.1k", 142 → "142". */
|
|
122
134
|
function fmtTokens(x) {
|
|
123
|
-
return x >= 1_000_000
|
|
124
|
-
|
|
135
|
+
return x >= 1_000_000
|
|
136
|
+
? `${(x / 1_000_000).toFixed(1)}mil`
|
|
137
|
+
: x >= 1000
|
|
138
|
+
? `${(x / 1000).toFixed(1)}k`
|
|
125
139
|
: `${Math.round(x)}`;
|
|
126
140
|
}
|
|
127
141
|
/** Retro gradient bar — `w` cells shaded by fill position (green→amber→red).
|
|
@@ -136,8 +150,10 @@ function ramp(pct, w = 12) {
|
|
|
136
150
|
for (let i = 0; i < full; i++)
|
|
137
151
|
out += (i / w < 0.6 ? C.green : i / w < 0.85 ? C.amber : C.red) + "█";
|
|
138
152
|
if (fracCell)
|
|
139
|
-
out +=
|
|
140
|
-
|
|
153
|
+
out +=
|
|
154
|
+
(full / w < 0.6 ? C.green : full / w < 0.85 ? C.amber : C.red) + fracCell;
|
|
155
|
+
out +=
|
|
156
|
+
C.dim + "░".repeat(Math.max(0, w - full - (fracCell ? 1 : 0))) + C.reset;
|
|
141
157
|
return out;
|
|
142
158
|
}
|
|
143
159
|
/** Human "time since" string from a millisecond delta (or null → "never"). */
|
|
@@ -175,6 +191,7 @@ export class MegaRuntime {
|
|
|
175
191
|
dedupAttempts: 0,
|
|
176
192
|
tokensSaved: 0,
|
|
177
193
|
lastCompactAt: null,
|
|
194
|
+
lastNativeCompactAt: null,
|
|
178
195
|
};
|
|
179
196
|
debounceUntil = 0;
|
|
180
197
|
// S16: debounce for the agent_end resume nudge (avoid busy-loops).
|
|
@@ -239,6 +256,7 @@ export class MegaRuntime {
|
|
|
239
256
|
diagBeforeCompactSupplied = 0; // session_before_compact supplied our trim
|
|
240
257
|
diagAgentEndIdle = 0; // agent_end with activeAgents===0
|
|
241
258
|
diagAgentEndDurable = 0; // agent_end fired ctx.compact() (mid-run durable trim)
|
|
259
|
+
diagAgentEndDurableSkipRecent = 0; // agent_end skipped ctx.compact() — compaction in last 10s (race guard)
|
|
242
260
|
// Per-skip-path counters for the team-run diagnosis.
|
|
243
261
|
diagCtxFastGate = 0; // returned at token fast-gate (below threshold)
|
|
244
262
|
diagCtxNoCompact = 0; // autoCompactCheck().shouldCompact === false
|
|
@@ -268,7 +286,9 @@ export class MegaRuntime {
|
|
|
268
286
|
* context event, or a `custom` tier with no tierPct). Always finite + in [0,1].
|
|
269
287
|
*/
|
|
270
288
|
get pressure() {
|
|
271
|
-
if (this.lastCtxWindow > 0 &&
|
|
289
|
+
if (this.lastCtxWindow > 0 &&
|
|
290
|
+
this.config.tierPct != null &&
|
|
291
|
+
this.lastCtxPercent != null) {
|
|
272
292
|
// pressureFromPct(x) = x/100, and x = lastCtxPercent/tierPct, so this is
|
|
273
293
|
// exactly the intended lastCtxPercent/(tierPct*100) 0–1 ratio: at the
|
|
274
294
|
// fire point (lastCtxPercent == tierPct*100) pressure == 1.0, matching the
|
|
@@ -276,7 +296,9 @@ export class MegaRuntime {
|
|
|
276
296
|
// the band doesn't jump when a token-count vs percent-only event arrives.
|
|
277
297
|
return pressureFromPct(this.lastCtxPercent / this.config.tierPct);
|
|
278
298
|
}
|
|
279
|
-
if (this.lastCtxTokens != null &&
|
|
299
|
+
if (this.lastCtxTokens != null &&
|
|
300
|
+
this.lastCtxTokens > 0 &&
|
|
301
|
+
this.config.thresholdTokens > 0) {
|
|
280
302
|
return pressureRatio(this.lastCtxTokens, this.config.thresholdTokens);
|
|
281
303
|
}
|
|
282
304
|
return pressureFromPct(this.lastCtxPercent);
|
|
@@ -302,8 +324,14 @@ export class MegaRuntime {
|
|
|
302
324
|
}
|
|
303
325
|
constructor(config) {
|
|
304
326
|
this.config = config;
|
|
305
|
-
this.store = new VectorStore({
|
|
306
|
-
|
|
327
|
+
this.store = new VectorStore({
|
|
328
|
+
dedupSim: config.dedupSim,
|
|
329
|
+
stateDir: config.stateDir,
|
|
330
|
+
});
|
|
331
|
+
this.logger = new Logger({
|
|
332
|
+
enabled: config.debug,
|
|
333
|
+
path: join(config.stateDir, "mega-compact.log"),
|
|
334
|
+
});
|
|
307
335
|
this.dashboard = new Dashboard(config.stateDir);
|
|
308
336
|
this.currentStateDir = config.stateDir;
|
|
309
337
|
}
|
|
@@ -314,14 +342,22 @@ export class MegaRuntime {
|
|
|
314
342
|
* and events are fully isolated. Falls back to the global default outside git.
|
|
315
343
|
*/
|
|
316
344
|
bindRepo(cwd) {
|
|
317
|
-
const dir = cwd
|
|
318
|
-
|
|
345
|
+
const dir = cwd
|
|
346
|
+
? repoStateDir(cwd, this.config.stateDir)
|
|
347
|
+
: this.config.stateDir;
|
|
348
|
+
const key = cwd ? (resolveRepoRoot(cwd) ?? dir) : dir;
|
|
319
349
|
if (key === this.activeRepoRoot)
|
|
320
350
|
return dir;
|
|
321
351
|
this.activeRepoRoot = key;
|
|
322
352
|
this.currentStateDir = dir;
|
|
323
|
-
this.store = new VectorStore({
|
|
324
|
-
|
|
353
|
+
this.store = new VectorStore({
|
|
354
|
+
dedupSim: this.config.dedupSim,
|
|
355
|
+
stateDir: dir,
|
|
356
|
+
});
|
|
357
|
+
this.logger = new Logger({
|
|
358
|
+
enabled: this.config.debug,
|
|
359
|
+
path: join(dir, "mega-compact.log"),
|
|
360
|
+
});
|
|
325
361
|
this.dashboard = new Dashboard(dir);
|
|
326
362
|
// Aggregate this repo into the machine-wide index so the multi-repo
|
|
327
363
|
// dashboard (Summary / All-repos tabs) can show it alongside every other
|
|
@@ -331,7 +367,7 @@ export class MegaRuntime {
|
|
|
331
367
|
try {
|
|
332
368
|
const repo = this.store.repoStats();
|
|
333
369
|
const di = this.store.dataInvariant();
|
|
334
|
-
const root = key !== dir ? key : resolveRepoRoot(cwd ?? dir) ?? dir;
|
|
370
|
+
const root = key !== dir ? key : (resolveRepoRoot(cwd ?? dir) ?? dir);
|
|
335
371
|
upsertRepoRegistry({
|
|
336
372
|
repoRoot: root,
|
|
337
373
|
displayName: root.split(/[\\/]/).filter(Boolean).pop() ?? root,
|
|
@@ -371,7 +407,9 @@ export class MegaRuntime {
|
|
|
371
407
|
// armed lights at/above the REAL fire point: max(effectiveThresholdPct,
|
|
372
408
|
// fastGatePct). fastGatePct already equals tierPct*100 by default, but a
|
|
373
409
|
// MEGACOMPACT_FAST_GATE_PCT override can raise it, so we take the max.
|
|
374
|
-
const armed = this.lastCtxPercent != null &&
|
|
410
|
+
const armed = this.lastCtxPercent != null &&
|
|
411
|
+
this.lastCtxPercent >=
|
|
412
|
+
Math.max(effectiveThresholdPct ?? 0, this.config.fastGatePct);
|
|
375
413
|
const ready = armed && (this.lastCtxTokens ?? 0) >= this.effectiveThreshold;
|
|
376
414
|
this.dashboard.snapshot({
|
|
377
415
|
version: 1,
|
|
@@ -401,10 +439,32 @@ export class MegaRuntime {
|
|
|
401
439
|
dedupSkips: this.rt.dedupSkips,
|
|
402
440
|
dedupAttempts: this.rt.dedupAttempts,
|
|
403
441
|
},
|
|
404
|
-
context: {
|
|
405
|
-
|
|
442
|
+
context: {
|
|
443
|
+
tokens: this.lastCtxTokens,
|
|
444
|
+
percent: this.lastCtxPercent,
|
|
445
|
+
contextWindow: this.lastCtxWindow,
|
|
446
|
+
},
|
|
447
|
+
trigger: {
|
|
448
|
+
armed,
|
|
449
|
+
ready,
|
|
450
|
+
currentTokens: this.lastCtxTokens,
|
|
451
|
+
thresholdTokens: this.effectiveThreshold,
|
|
452
|
+
fastGatePct: this.config.fastGatePct,
|
|
453
|
+
tierPct: this.config.tierPct,
|
|
454
|
+
effectiveThresholdPct,
|
|
455
|
+
},
|
|
406
456
|
crew: { activeAgents: this.activeAgents, currentTurn: this.currentTurn },
|
|
407
|
-
store: {
|
|
457
|
+
store: {
|
|
458
|
+
checkpointCount: st.checkpointCount,
|
|
459
|
+
totalTokenEstimate: st.totalTokenEstimate,
|
|
460
|
+
originalTokens: st.originalTokens,
|
|
461
|
+
tokensSaved: this.rt.tokensSaved,
|
|
462
|
+
injectedCount: st.injectedCount,
|
|
463
|
+
dedupHitRate: st.dedupHitRate,
|
|
464
|
+
storageDedupRate: st.storageDedupRate,
|
|
465
|
+
dedupAttempts: st.dedupAttempts,
|
|
466
|
+
dedupCollapsed: st.dedupCollapsed,
|
|
467
|
+
},
|
|
408
468
|
// Reconciled token accounting (single canonical formula, session + repo).
|
|
409
469
|
// Freed = In − Out; In = Freed + Out. session.Freed = rt.tokensSaved (incl.
|
|
410
470
|
// deduped-away originals); repo.Freed = repo.tokensSaved meta counter.
|
|
@@ -413,14 +473,19 @@ export class MegaRuntime {
|
|
|
413
473
|
tokensIn: this.rt.tokensSaved + st.totalTokenEstimate,
|
|
414
474
|
tokensOut: st.totalTokenEstimate,
|
|
415
475
|
tokensFreed: this.rt.tokensSaved,
|
|
416
|
-
compressionPct:
|
|
476
|
+
compressionPct: this.rt.tokensSaved + st.totalTokenEstimate > 0
|
|
477
|
+
? this.rt.tokensSaved /
|
|
478
|
+
(this.rt.tokensSaved + st.totalTokenEstimate)
|
|
479
|
+
: 0,
|
|
417
480
|
dedupPct: st.storageDedupRate,
|
|
418
481
|
},
|
|
419
482
|
repo: {
|
|
420
483
|
tokensIn: repo.tokensSaved + repo.totalTokenEstimate,
|
|
421
484
|
tokensOut: repo.totalTokenEstimate,
|
|
422
485
|
tokensFreed: repo.tokensSaved,
|
|
423
|
-
compressionPct:
|
|
486
|
+
compressionPct: repo.tokensSaved + repo.totalTokenEstimate > 0
|
|
487
|
+
? repo.tokensSaved / (repo.tokensSaved + repo.totalTokenEstimate)
|
|
488
|
+
: 0,
|
|
424
489
|
dedupPct: repo.storageDedupRate,
|
|
425
490
|
},
|
|
426
491
|
},
|
|
@@ -445,14 +510,24 @@ export class MegaRuntime {
|
|
|
445
510
|
// Live stats widget above the editor
|
|
446
511
|
if (ctx) {
|
|
447
512
|
// ── gather widget data (computed per snapshot, rendered per frame) ────
|
|
448
|
-
const tokStr = this.lastCtxTokens != null
|
|
449
|
-
|
|
450
|
-
|
|
513
|
+
const tokStr = this.lastCtxTokens != null
|
|
514
|
+
? `${Math.round(this.lastCtxTokens / 1000)}k`
|
|
515
|
+
: "?";
|
|
516
|
+
const maxStr = this.lastCtxWindow > 0
|
|
517
|
+
? `${Math.round(this.lastCtxWindow / 1000)}k`
|
|
518
|
+
: "?";
|
|
519
|
+
const pctStr = this.lastCtxPercent != null
|
|
520
|
+
? `${Math.round(this.lastCtxPercent * 10) / 10}%`
|
|
521
|
+
: "?%";
|
|
451
522
|
// S24: the tier label is the LIVE pressure band (low/medium/high/ultra/
|
|
452
523
|
// mega), not the static env preset. It climbs as context fills.
|
|
453
524
|
const liveBand = this.pressureBand;
|
|
454
525
|
const tierLabel = `${C.bold}${liveBand}${C.reset}${C.gray}·${this.config.tier}${C.reset}`;
|
|
455
|
-
const triggerLabel = ready
|
|
526
|
+
const triggerLabel = ready
|
|
527
|
+
? `${C.green}● ready${C.reset}`
|
|
528
|
+
: armed
|
|
529
|
+
? `${C.amber}◐ armed${C.reset}`
|
|
530
|
+
: `${C.gray}○ idle${C.reset}`;
|
|
456
531
|
// Storage dedup rate is cumulative (store-wide, per-repo) and survives
|
|
457
532
|
// session resets. Always show a number (decimal for sub-10%).
|
|
458
533
|
const storageRate = st.storageDedupRate; // 0..1
|
|
@@ -477,27 +552,55 @@ export class MegaRuntime {
|
|
|
477
552
|
const ctxPct = this.lastCtxPercent != null ? this.lastCtxPercent / 100 : 0;
|
|
478
553
|
// Model + provider (S26 capture) for the header.
|
|
479
554
|
const modelName = modelSnap?.modelName ?? modelSnap?.modelId ?? "?";
|
|
480
|
-
const modelStr = modelSnap?.provider
|
|
555
|
+
const modelStr = modelSnap?.provider
|
|
556
|
+
? `${modelName}·${modelSnap.provider}`
|
|
557
|
+
: modelName;
|
|
481
558
|
// Since-last-compact (ms; null until first compaction this session).
|
|
482
|
-
const sinceCompact = this.rt.lastCompactAt != null
|
|
559
|
+
const sinceCompact = this.rt.lastCompactAt != null
|
|
560
|
+
? Date.now() - this.rt.lastCompactAt
|
|
561
|
+
: null;
|
|
483
562
|
// Memory store: embedder + compression ratio (original / stored).
|
|
484
563
|
const embedderName = this.embedderName();
|
|
485
564
|
const compRatio = st.originalTokens > 0 && st.totalTokenEstimate > 0
|
|
486
565
|
? st.originalTokens / st.totalTokenEstimate
|
|
487
|
-
:
|
|
566
|
+
: st.originalTokens > 0
|
|
567
|
+
? 1
|
|
568
|
+
: 0;
|
|
488
569
|
const compStr = compRatio >= 1 ? `${compRatio.toFixed(1)}x` : "—";
|
|
489
570
|
// Cross-repo drift status (cached, read-only).
|
|
490
571
|
const driftStatus = this.driftStatus();
|
|
491
572
|
const agentsActive = this.activeAgents > 0;
|
|
492
573
|
this.widgetData = {
|
|
493
574
|
version: ownVersion(),
|
|
494
|
-
tierLabel,
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
575
|
+
tierLabel,
|
|
576
|
+
triggerLabel,
|
|
577
|
+
pctStr,
|
|
578
|
+
tokStr,
|
|
579
|
+
maxStr,
|
|
580
|
+
ctxPct,
|
|
581
|
+
chk: st.checkpointCount,
|
|
582
|
+
agentStr,
|
|
583
|
+
turnStr,
|
|
584
|
+
dedupStr,
|
|
585
|
+
sessIn,
|
|
586
|
+
sessKept,
|
|
587
|
+
sTxt,
|
|
588
|
+
repoIn,
|
|
589
|
+
repoKept,
|
|
590
|
+
rTxt,
|
|
591
|
+
repoChk: repo.checkpointCount,
|
|
592
|
+
repoSess: repo.sessionCount,
|
|
593
|
+
modelStr,
|
|
594
|
+
sinceCompact,
|
|
595
|
+
embedderName,
|
|
596
|
+
compStr,
|
|
597
|
+
driftStatus,
|
|
598
|
+
agentsActive,
|
|
499
599
|
fresh: Date.now() - this.lastActivityAt < 4000,
|
|
500
|
-
ticker: this.ticker,
|
|
600
|
+
ticker: this.ticker,
|
|
601
|
+
lastWhy: this.lastWhy,
|
|
602
|
+
tierTrace: this.tierTrace,
|
|
603
|
+
pulsing: this.pulsing,
|
|
501
604
|
};
|
|
502
605
|
// Auto-fit: register a factory so pi re-renders the panel at the REAL
|
|
503
606
|
// terminal width every frame (tui.columns), instead of guessing with
|
|
@@ -520,31 +623,35 @@ export class MegaRuntime {
|
|
|
520
623
|
buildWidgetLines(width) {
|
|
521
624
|
const wd = this.widgetData;
|
|
522
625
|
if (!wd) {
|
|
523
|
-
return [
|
|
626
|
+
return [
|
|
627
|
+
panelBar(width, "─"),
|
|
628
|
+
panelLine(" mega-compact: warming up…", width),
|
|
629
|
+
panelBar(width, "─"),
|
|
630
|
+
];
|
|
524
631
|
}
|
|
525
|
-
const pulse = wd.pulsing
|
|
632
|
+
const pulse = wd.pulsing
|
|
633
|
+
? `${C.cyan}${PULSE[Math.floor(Date.now() / 250) % PULSE.length]}${C.reset} `
|
|
634
|
+
: "";
|
|
635
|
+
const sep = ` ${C.dim}│${C.reset} `;
|
|
636
|
+
// Build one long content line — let terminal wrap it naturally
|
|
637
|
+
const content = [
|
|
638
|
+
`${C.amber}⚡ ${wd.tierLabel}${C.reset} v${C.bold}${wd.version}${C.reset} ${ramp(wd.ctxPct, 20)} ${C.bold}${wd.pctStr}${C.reset} ${wd.tokStr}/${wd.maxStr}`,
|
|
639
|
+
wd.triggerLabel,
|
|
640
|
+
`${C.cyan}${wd.modelStr}${C.reset}`,
|
|
641
|
+
`${wd.chk} chk${wd.agentStr}${wd.turnStr}`,
|
|
642
|
+
`${C.magenta}dup ${wd.dedupStr}${C.reset}`,
|
|
643
|
+
`${C.gray}sess${C.reset} ${fmtTokens(wd.sessIn)}→${fmtTokens(wd.sessKept)} kept ${C.green}(${wd.sTxt}% freed)${C.reset}`,
|
|
644
|
+
`${C.gray}all-time${C.reset} ${fmtTokens(wd.repoIn)}→${fmtTokens(wd.repoKept)} kept ${C.blue}(${wd.rTxt}% freed)${C.reset}`,
|
|
645
|
+
`${wd.repoChk} chk/${wd.repoSess} sess`,
|
|
646
|
+
`${C.gray}mem${C.reset} ${wd.embedderName} · ${wd.chk} chunks · ${C.blue}comp ${wd.compStr}${C.reset}`,
|
|
647
|
+
`${C.gray}drift${C.reset} ${wd.driftStatus === "ok" ? C.green : C.amber}${wd.driftStatus}${C.reset}`,
|
|
648
|
+
`${C.gray}compact${C.reset} ${sinceCompactStr(wd.sinceCompact)}`,
|
|
649
|
+
].join(sep);
|
|
650
|
+
// Wrap to terminal width and pad each line
|
|
651
|
+
const wrapped = wrapLine(content, width - 2); // 2-char indent
|
|
526
652
|
const lines = [
|
|
527
|
-
// top border
|
|
528
653
|
panelBar(width, "─"),
|
|
529
|
-
|
|
530
|
-
spreadGroups(width, [
|
|
531
|
-
`${C.amber}⚡ ${wd.tierLabel}${C.reset} v${C.bold}${wd.version}${C.reset} ${ramp(wd.ctxPct, 20)} ${C.bold}${wd.pctStr}${C.reset} ${wd.tokStr}/${wd.maxStr}`,
|
|
532
|
-
wd.triggerLabel,
|
|
533
|
-
`${C.cyan}${wd.modelStr}${C.reset}`,
|
|
534
|
-
`${wd.chk} chk${wd.agentStr}${wd.turnStr}`,
|
|
535
|
-
]),
|
|
536
|
-
// L2 — savings reconciled (session + all-time)
|
|
537
|
-
spreadGroups(width, [
|
|
538
|
-
`${C.magenta}dup ${wd.dedupStr}${C.reset}`,
|
|
539
|
-
`${C.gray}sess${C.reset} ${fmtTokens(wd.sessIn)}→${fmtTokens(wd.sessKept)} kept ${C.green}(${wd.sTxt}% freed)${C.reset} · ${C.gray}all-time${C.reset} ${fmtTokens(wd.repoIn)}→${fmtTokens(wd.repoKept)} kept ${C.blue}(${wd.rTxt}% freed)${C.reset}`,
|
|
540
|
-
`${wd.repoChk} chk/${wd.repoSess} sess`,
|
|
541
|
-
]),
|
|
542
|
-
// L3 — memory store + compression + drift + since-compact (NEW)
|
|
543
|
-
spreadGroups(width, [
|
|
544
|
-
`${C.gray}mem${C.reset} ${wd.embedderName} · ${wd.chk} chunks · ${C.blue}comp ${wd.compStr}${C.reset}`,
|
|
545
|
-
`${C.gray}drift${C.reset} ${wd.driftStatus === "ok" ? C.green : C.amber}${wd.driftStatus}${C.reset}`,
|
|
546
|
-
`${C.gray}compact${C.reset} ${sinceCompactStr(wd.sinceCompact)}`,
|
|
547
|
-
]),
|
|
654
|
+
...wrapped.map((l) => panelLine(l, width)),
|
|
548
655
|
];
|
|
549
656
|
// L4 — agents block (S27, count + status; per-agent tokens gated on P0)
|
|
550
657
|
if (wd.agentsActive) {
|
|
@@ -559,7 +666,9 @@ export class MegaRuntime {
|
|
|
559
666
|
const idx = wd.ticker.length - 1 - (step % wd.ticker.length);
|
|
560
667
|
const head = wd.ticker[idx].text;
|
|
561
668
|
const why = wd.lastWhy ? ` ${C.gray}· ${wd.lastWhy}${C.reset}` : "";
|
|
562
|
-
const more = wd.ticker.length > 1
|
|
669
|
+
const more = wd.ticker.length > 1
|
|
670
|
+
? ` ${C.dim}(+${wd.ticker.length - 1} more)${C.reset}`
|
|
671
|
+
: "";
|
|
563
672
|
lines.push(panelLine(` ${wd.fresh ? C.teal : C.dim}${head}${why}${more}${C.reset}`, width));
|
|
564
673
|
}
|
|
565
674
|
else if (wd.pulsing) {
|
|
@@ -573,7 +682,8 @@ export class MegaRuntime {
|
|
|
573
682
|
embedderName() {
|
|
574
683
|
// MINILM_EMBEDDER flag lives in src/config/dedup.ts; read the same env var
|
|
575
684
|
// the embedder factory uses so the label matches what's actually running.
|
|
576
|
-
return process.env.MEGACOMPACT_MINILM === "true" ||
|
|
685
|
+
return process.env.MEGACOMPACT_MINILM === "true" ||
|
|
686
|
+
process.env.MEGACOMPACT_MINILM === "1"
|
|
577
687
|
? "MiniLM"
|
|
578
688
|
: "Trigram";
|
|
579
689
|
}
|
|
@@ -611,6 +721,7 @@ export class MegaRuntime {
|
|
|
611
721
|
dedupAttempts: 0,
|
|
612
722
|
tokensSaved: 0,
|
|
613
723
|
lastCompactAt: null,
|
|
724
|
+
lastNativeCompactAt: null,
|
|
614
725
|
};
|
|
615
726
|
this.statusKey = undefined;
|
|
616
727
|
this.activeAgents = 0;
|
|
@@ -633,13 +744,18 @@ export class MegaRuntime {
|
|
|
633
744
|
this.appendEvent("captureModel:no-model", { cwd: ctx.cwd });
|
|
634
745
|
return;
|
|
635
746
|
}
|
|
636
|
-
if (this.currentModel &&
|
|
747
|
+
if (this.currentModel &&
|
|
748
|
+
this.currentModel.modelId === m.id &&
|
|
749
|
+
this.currentModel.provider === m.provider)
|
|
637
750
|
return;
|
|
638
751
|
let providerName = null;
|
|
639
752
|
try {
|
|
640
|
-
providerName =
|
|
753
|
+
providerName =
|
|
754
|
+
ctx.modelRegistry?.getProviderDisplayName(m.provider) ?? null;
|
|
755
|
+
}
|
|
756
|
+
catch {
|
|
757
|
+
/* optional */
|
|
641
758
|
}
|
|
642
|
-
catch { /* optional */ }
|
|
643
759
|
const snap = {
|
|
644
760
|
provider: m.provider,
|
|
645
761
|
providerName,
|
|
@@ -661,14 +777,18 @@ export class MegaRuntime {
|
|
|
661
777
|
try {
|
|
662
778
|
recordModelSnapshot(repo, snap, this.currentStateDir);
|
|
663
779
|
this.appendEvent("captureModel:recorded", {
|
|
664
|
-
repo,
|
|
665
|
-
|
|
780
|
+
repo,
|
|
781
|
+
modelId: snap.modelId,
|
|
782
|
+
provider: snap.provider,
|
|
783
|
+
inputRate: snap.inputRate,
|
|
784
|
+
outputRate: snap.outputRate,
|
|
666
785
|
});
|
|
667
786
|
}
|
|
668
787
|
catch (e) {
|
|
669
788
|
this.diagCaptureModelFails++;
|
|
670
789
|
this.appendEvent("captureModel:record-failed", {
|
|
671
|
-
repo,
|
|
790
|
+
repo,
|
|
791
|
+
modelId: snap.modelId,
|
|
672
792
|
error: e instanceof Error ? e.message : String(e),
|
|
673
793
|
stack: e instanceof Error ? e.stack : undefined,
|
|
674
794
|
});
|
|
@@ -689,7 +809,8 @@ export class MegaRuntime {
|
|
|
689
809
|
}
|
|
690
810
|
catch (e) {
|
|
691
811
|
this.appendEvent("captureModel:index-record-failed", {
|
|
692
|
-
repo,
|
|
812
|
+
repo,
|
|
813
|
+
modelId: snap.modelId,
|
|
693
814
|
error: e instanceof Error ? e.message : String(e),
|
|
694
815
|
});
|
|
695
816
|
}
|
|
@@ -705,7 +826,9 @@ export class MegaRuntime {
|
|
|
705
826
|
mkdirSync(this.currentStateDir, { recursive: true });
|
|
706
827
|
appendFileSync(join(this.currentStateDir, "events.log"), JSON.stringify({ ts: Date.now(), event, ...fields }) + "\n");
|
|
707
828
|
}
|
|
708
|
-
catch {
|
|
829
|
+
catch {
|
|
830
|
+
/* non-fatal */
|
|
831
|
+
}
|
|
709
832
|
}
|
|
710
833
|
/** S21: state dir of the currently bound repo (where memories live). */
|
|
711
834
|
getStateDir() {
|
|
@@ -715,10 +838,13 @@ export class MegaRuntime {
|
|
|
715
838
|
makeTierCallback(ctx) {
|
|
716
839
|
const order = ["L0", "L1", "L2", "new"];
|
|
717
840
|
const seen = new Map();
|
|
718
|
-
const glyph = (status) => status === "deduped"
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
841
|
+
const glyph = (status) => status === "deduped"
|
|
842
|
+
? `${C.green}✓${C.reset}`
|
|
843
|
+
: status === "passed"
|
|
844
|
+
? `${C.dim}○${C.reset}`
|
|
845
|
+
: status === "scanning"
|
|
846
|
+
? `${C.amber}…${C.reset}`
|
|
847
|
+
: `${C.cyan}●${C.reset}`;
|
|
722
848
|
return (ev) => {
|
|
723
849
|
const label = ev.tier === "new"
|
|
724
850
|
? `${C.cyan}stored${C.reset}`
|
|
@@ -735,7 +861,9 @@ export class MegaRuntime {
|
|
|
735
861
|
try {
|
|
736
862
|
this.snapshot(ctx);
|
|
737
863
|
}
|
|
738
|
-
catch {
|
|
864
|
+
catch {
|
|
865
|
+
/* non-fatal */
|
|
866
|
+
}
|
|
739
867
|
};
|
|
740
868
|
}
|
|
741
869
|
// Phase 3 — recall/activity ticker ring buffer.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dedup.ts — S27 Task 6: Fork snapshot → compress/dedupe pipeline.
|
|
3
|
+
*
|
|
4
|
+
* After the served window is handed to pi, asynchronously:
|
|
5
|
+
* 1. Read raw_transcript rows [0..cut_index] for the epoch
|
|
6
|
+
* 2. For each row, compute content_hash (reuse digest from dedup/)
|
|
7
|
+
* 3. INSERT OR IGNORE INTO dedup_mirror (stores bytes once per unique hash)
|
|
8
|
+
* 4. Update raw_transcript.content_ref to point to dedup_mirror
|
|
9
|
+
* 5. Increment dedup_mirror.ref_count for existing hashes
|
|
10
|
+
*
|
|
11
|
+
* Pi-agnostic: no pi runtime imports (src/ invariant).
|
|
12
|
+
*/
|
|
13
|
+
import { upsertDedupMirror, updateRawTranscriptRef, listRawTranscriptRange, getDedupRatio, } from "../store/sqlite.js";
|
|
14
|
+
import { computeContentDigest } from "../dedup/digest.js";
|
|
15
|
+
/**
|
|
16
|
+
* Deduplicate raw transcript rows for a session range.
|
|
17
|
+
* Fire-and-forget: errors are logged, not thrown.
|
|
18
|
+
*
|
|
19
|
+
* @returns Number of rows deduplicated, or -1 on error.
|
|
20
|
+
*/
|
|
21
|
+
export function dedupTranscript(db, sessionId, fromSeq, toSeq) {
|
|
22
|
+
try {
|
|
23
|
+
const rows = listRawTranscriptRange(db, sessionId, fromSeq, toSeq);
|
|
24
|
+
let deduped = 0;
|
|
25
|
+
for (const row of rows) {
|
|
26
|
+
const contentHash = computeContentDigest(row.contentBytes).contentHash;
|
|
27
|
+
const isNew = upsertDedupMirror(db, contentHash, row.contentBytes, row.seq);
|
|
28
|
+
updateRawTranscriptRef(db, sessionId, row.seq, contentHash);
|
|
29
|
+
if (!isNew) {
|
|
30
|
+
deduped++;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return deduped;
|
|
34
|
+
}
|
|
35
|
+
catch (err) {
|
|
36
|
+
// Fire-and-forget: log but don't throw
|
|
37
|
+
console.error("[mega-compact] dedupTranscript failed:", err);
|
|
38
|
+
return -1;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Get dedup ratio for a session.
|
|
43
|
+
*/
|
|
44
|
+
export { getDedupRatio };
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* epoch.ts — deterministic epoch-id derivation for the S27 DB-mirror.
|
|
3
|
+
*
|
|
4
|
+
* The epoch id MUST be a pure function of the checkpoint it decorates so that
|
|
5
|
+
* replaying / refreshing the same compaction yields the SAME epoch id (idempotent
|
|
6
|
+
* appends + ON CONFLICT refresh). No Date.now / uuid / crypto — this is the
|
|
7
|
+
* only source of randomness-free epoch naming in the mirror stack.
|
|
8
|
+
*
|
|
9
|
+
* - epochIdFor(cp) → "epoch:" + cp (human-traceable back to its checkpoint)
|
|
10
|
+
* - epochNonceFor(cp) → FNV-1a 32-bit hash (cheap, well-distributed nonce)
|
|
11
|
+
*
|
|
12
|
+
* Pi-agnostic: no pi runtime imports (src/ invariant).
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* FNV-1a 32-bit nonce for a checkpoint id. Deterministic and RNG-free:
|
|
16
|
+
* h = 0x811c9dc5; for each char: h ^= codePoint; h = Math.imul(h, 0x01000193);
|
|
17
|
+
* return h >>> 0 (unsigned).
|
|
18
|
+
*/
|
|
19
|
+
export function epochNonceFor(checkpointId) {
|
|
20
|
+
let h = 0x811c9dc5;
|
|
21
|
+
for (let i = 0; i < checkpointId.length; i++) {
|
|
22
|
+
const cp = checkpointId.codePointAt(i);
|
|
23
|
+
if (cp === undefined)
|
|
24
|
+
continue;
|
|
25
|
+
h ^= cp;
|
|
26
|
+
h = Math.imul(h, 0x01000193);
|
|
27
|
+
}
|
|
28
|
+
return h >>> 0;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Deterministic epoch id: "epoch:" + checkpointId. Trivially traceable back to
|
|
32
|
+
* the source checkpoint, and stable under replay (refresh-safe upserts).
|
|
33
|
+
*/
|
|
34
|
+
export function epochIdFor(checkpointId) {
|
|
35
|
+
return "epoch:" + checkpointId;
|
|
36
|
+
}
|