pi-mega-compact 0.4.14 → 0.4.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server.js +30 -9
- package/dist/extensions/mega-commands.js +223 -0
- package/dist/extensions/mega-compact.js +19 -1073
- package/dist/extensions/mega-compact.test.js +28 -0
- package/dist/extensions/mega-config.js +100 -0
- package/dist/extensions/mega-dashboard-cmds.js +214 -0
- package/dist/extensions/mega-dashboard.js +35 -0
- package/dist/extensions/mega-events.js +167 -0
- package/dist/extensions/mega-pipeline.js +140 -0
- package/dist/extensions/mega-runtime.js +370 -0
- package/dist/src/store/sqlite.js +60 -0
- package/extensions/dashboard-server.ts +30 -9
- package/extensions/mega-commands.ts +250 -0
- package/extensions/mega-compact.test.ts +30 -0
- package/extensions/mega-compact.ts +20 -1214
- package/extensions/mega-config.ts +120 -0
- package/extensions/mega-dashboard-cmds.ts +209 -0
- package/extensions/mega-dashboard.ts +107 -0
- package/extensions/mega-events.ts +180 -0
- package/extensions/mega-pipeline.ts +179 -0
- package/extensions/mega-runtime.ts +386 -0
- package/package.json +1 -1
- package/src/store/sqlite.ts +96 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* mega-pipeline.ts — the compaction + recall pipelines.
|
|
3
|
+
*
|
|
4
|
+
* `runCompact` runs the full Trident pipeline (fast-gate aside) and persists a
|
|
5
|
+
* checkpoint. `doRecall` is the unified Layer-5 recall entry point. Both mutate
|
|
6
|
+
* the shared MegaRuntime (token accounting, ticker, status, events) and are
|
|
7
|
+
* driven by the event + command handlers in mega-events.ts / mega-commands.ts.
|
|
8
|
+
*/
|
|
9
|
+
import { compactSession } from "../src/engine.js";
|
|
10
|
+
import { recallAndInline } from "../src/recall.js";
|
|
11
|
+
import { normalizeSessionId } from "../src/store.js";
|
|
12
|
+
import { touchSession, logDaily } from "../src/store/sqlite.js";
|
|
13
|
+
import { C, MARKER_TYPE, } from "./mega-runtime.js";
|
|
14
|
+
import { resolveRepoRoot } from "./mega-config.js";
|
|
15
|
+
/** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
|
|
16
|
+
export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
|
|
17
|
+
runtime.bindRepo(ctx.cwd);
|
|
18
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
19
|
+
runtime.resetRuntime(sid);
|
|
20
|
+
runtime.rt.sessionId = sid;
|
|
21
|
+
const view = runtime.engineView(messages);
|
|
22
|
+
const keepFrom = opts.keepFrom ?? Math.max(0, view.length - config.preserveRecent);
|
|
23
|
+
if (keepFrom <= 0)
|
|
24
|
+
return { skipped: true };
|
|
25
|
+
runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
|
|
26
|
+
const result = compactSession({
|
|
27
|
+
sessionId: sid,
|
|
28
|
+
messages: view,
|
|
29
|
+
keepFrom,
|
|
30
|
+
summary: opts.summary,
|
|
31
|
+
timestamp: Date.now(),
|
|
32
|
+
onTier: runtime.makeTierCallback(ctx),
|
|
33
|
+
}, runtime.store);
|
|
34
|
+
runtime.pulsing = false;
|
|
35
|
+
if (result.skipped)
|
|
36
|
+
return { skipped: true };
|
|
37
|
+
if (!result.deduped) {
|
|
38
|
+
runtime.rt.persistedThisSession = true;
|
|
39
|
+
runtime.rt.lastCheckpointId = result.checkpointId;
|
|
40
|
+
}
|
|
41
|
+
runtime.rt.lastCompactedFrom = result.compactedFrom;
|
|
42
|
+
runtime.rt.lastCompactedTokens = result.tokenEstimate;
|
|
43
|
+
runtime.rt.dedupAttempts++;
|
|
44
|
+
// Honest "tokens saved" for this session-instance only:
|
|
45
|
+
// new checkpoint → original − stored
|
|
46
|
+
// deduped onto existing → whole original region (nothing new stored)
|
|
47
|
+
// Resets to 0 on session_start (rt is rebuilt) — so a fresh session shows 0
|
|
48
|
+
// while the repo's cumulative saved (SQLite meta) keeps the running total.
|
|
49
|
+
const saved = result.deduped
|
|
50
|
+
? result.originalTokenEstimate
|
|
51
|
+
: Math.max(0, result.originalTokenEstimate - result.tokenEstimate);
|
|
52
|
+
runtime.rt.tokensSaved += saved;
|
|
53
|
+
if (result.deduped)
|
|
54
|
+
runtime.rt.dedupSkips++;
|
|
55
|
+
// Grow the rolling "saved" goal so the progress bar always has a fresh
|
|
56
|
+
// denominator (we don't want it pinned at 100% once we pass an old target).
|
|
57
|
+
if (runtime.rt.tokensSaved > runtime.savedGoal)
|
|
58
|
+
runtime.savedGoal = Math.ceil((runtime.rt.tokensSaved * 1.25) / 10_000) * 10_000;
|
|
59
|
+
// Live toolbar "now processing" line: what file/region just got compacted or
|
|
60
|
+
// deduped. Reset to the last-seen action after a few seconds (see snapshot).
|
|
61
|
+
const files = result.filesModified ?? [];
|
|
62
|
+
const fileLabel = files.length
|
|
63
|
+
? files.map((f) => f.split("/").pop() ?? f).slice(0, 2).join(", ")
|
|
64
|
+
: result.regionHash.slice(0, 8);
|
|
65
|
+
runtime.currentActivity = result.deduped
|
|
66
|
+
? `♻ deduped ${fileLabel}`
|
|
67
|
+
: `🗜 compacted ${result.checkpointId} · ${fileLabel}`;
|
|
68
|
+
runtime.lastActivityAt = Date.now();
|
|
69
|
+
// Explain-why line: surfaced while fresh. Pulls the dedup reason (which for
|
|
70
|
+
// L2 includes the cosine sim) so the user sees WHY a region was kept/dropped.
|
|
71
|
+
runtime.lastWhy = result.deduped
|
|
72
|
+
? `why: deduped@${result.dedupReason ?? "tier"}`
|
|
73
|
+
: `why: compacted → ${result.checkpointId}`;
|
|
74
|
+
// Recall/activity ticker: record this event in the ring buffer.
|
|
75
|
+
const savedK = (saved / 1000).toFixed(1);
|
|
76
|
+
runtime.pushTicker(result.deduped
|
|
77
|
+
? `${C.green}♻${C.reset} deduped ${fileLabel} · ${savedK}k saved`
|
|
78
|
+
: `${C.cyan}🗜${C.reset} ${result.checkpointId} · +${savedK}k · ${fileLabel}`);
|
|
79
|
+
// The per-tier trace has settled into the final outcome — fold it back into
|
|
80
|
+
// the activity line and stop showing the live trace.
|
|
81
|
+
runtime.tierTrace = undefined;
|
|
82
|
+
// Record session activity + a daily-log entry in the per-repo SQLite store
|
|
83
|
+
// (foundation for resume-sessions / daily-log features). Best-effort — never
|
|
84
|
+
// block a compaction on bookkeeping.
|
|
85
|
+
try {
|
|
86
|
+
const root = resolveRepoRoot(ctx.cwd);
|
|
87
|
+
touchSession(sid, root, runtime.currentStateDir);
|
|
88
|
+
logDaily(sid, "compact", result.checkpointId, saved, runtime.currentStateDir);
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
/* non-fatal: stats bookkeeping only */
|
|
92
|
+
}
|
|
93
|
+
// Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
|
|
94
|
+
// skip re-vectorizing an already-compacted region (zero token cost).
|
|
95
|
+
pi.appendEntry(MARKER_TYPE, {
|
|
96
|
+
checkpointId: result.checkpointId,
|
|
97
|
+
regionHash: result.regionHash,
|
|
98
|
+
tokenEstimate: result.tokenEstimate,
|
|
99
|
+
deduped: result.deduped,
|
|
100
|
+
});
|
|
101
|
+
runtime.setStatus(ctx, runtime.rt.persistedThisSession
|
|
102
|
+
? `mega-compact: ${result.checkpointId} · ${saved} tok saved`
|
|
103
|
+
: `mega-compact: ready`);
|
|
104
|
+
runtime.logger.info("compact", {
|
|
105
|
+
sessionId: sid,
|
|
106
|
+
checkpointId: result.checkpointId ?? "(deduped)",
|
|
107
|
+
deduped: result.deduped,
|
|
108
|
+
tokenEstimate: saved,
|
|
109
|
+
compactedFrom: result.compactedFrom,
|
|
110
|
+
});
|
|
111
|
+
runtime.dashboard.event("compact", {
|
|
112
|
+
sessionId: sid,
|
|
113
|
+
checkpointId: result.checkpointId ?? "(deduped)",
|
|
114
|
+
deduped: result.deduped,
|
|
115
|
+
tokenEstimate: saved,
|
|
116
|
+
compactedFrom: result.compactedFrom,
|
|
117
|
+
});
|
|
118
|
+
runtime.snapshot(ctx);
|
|
119
|
+
return { skipped: false, result, keepFrom, saved };
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Unified recall (Layer 5). The ONE path that injects. Returns the recall
|
|
123
|
+
* result; callers decide whether to stage it for before_agent_start (resume)
|
|
124
|
+
* or report it (command).
|
|
125
|
+
*/
|
|
126
|
+
export function doRecall(runtime, config, ctx, query, source) {
|
|
127
|
+
runtime.bindRepo(ctx.cwd);
|
|
128
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
129
|
+
const result = recallAndInline({ sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true }, runtime.store);
|
|
130
|
+
runtime.dashboard.event("recall", { source, query: query.slice(0, 120), injected: result.toInject.length, empty: result.empty });
|
|
131
|
+
if (!result.empty && result.toInject.length > 0) {
|
|
132
|
+
const top = result.toInject[0];
|
|
133
|
+
const scorePct = Math.round((top.score ?? 0) * 100);
|
|
134
|
+
const files = top.checkpoint.filesModified ?? [];
|
|
135
|
+
const label = files.length ? files.map((f) => f.split("/").pop() ?? f).slice(0, 2).join(", ") : top.checkpoint.checkpointId;
|
|
136
|
+
runtime.pushTicker(`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`);
|
|
137
|
+
runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
|
|
138
|
+
}
|
|
139
|
+
return result;
|
|
140
|
+
}
|
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* mega-runtime.ts — the shared live state of the mega-compact extension.
|
|
3
|
+
*
|
|
4
|
+
* The original mega-compact.ts was a single large closure over ~20 mutable
|
|
5
|
+
* variables. This module lifts that state into a `MegaRuntime` class so the
|
|
6
|
+
* event/command/pipeline modules can share it without re-declaring it. All
|
|
7
|
+
* behavior (store/dashboard rebinding, dashboard snapshot shape, the
|
|
8
|
+
* above-editor widget math, model capture) is preserved byte-for-byte from the
|
|
9
|
+
* original closure.
|
|
10
|
+
*/
|
|
11
|
+
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
import { VectorStore } from "../src/vectorStore.js";
|
|
14
|
+
import { toEngineMessages } from "../src/adapt.js";
|
|
15
|
+
import { normalizeSessionId } from "../src/store.js";
|
|
16
|
+
import { Logger } from "../src/log.js";
|
|
17
|
+
import { recordModelSnapshot } from "../src/store/sqlite.js";
|
|
18
|
+
import { repoStateDir, resolveRepoRoot } from "./mega-config.js";
|
|
19
|
+
import { Dashboard } from "./mega-dashboard.js";
|
|
20
|
+
export const STATUS_KEY = "mega-compact";
|
|
21
|
+
export const WIDGET_KEY = "mega-compact-stats";
|
|
22
|
+
export const MARKER_TYPE = "mega-compact-marker";
|
|
23
|
+
/** ANSI palette for the toolbar. The pi TUI's Text component preserves ANSI
|
|
24
|
+
* escape codes (see wrapTextWithAnsi), so raw escapes render as colors. No
|
|
25
|
+
* chalk dependency needed — these are just strings. */
|
|
26
|
+
export const C = {
|
|
27
|
+
reset: "\x1b[0m",
|
|
28
|
+
dim: "\x1b[2m",
|
|
29
|
+
bold: "\x1b[1m",
|
|
30
|
+
amber: "\x1b[38;5;214m", // tier / ready
|
|
31
|
+
green: "\x1b[38;5;120m", // saved
|
|
32
|
+
cyan: "\x1b[38;5;51m", // used / live activity
|
|
33
|
+
teal: "\x1b[38;5;37m", // processing (compress/dedup)
|
|
34
|
+
magenta: "\x1b[38;5;201m", // dedup rate
|
|
35
|
+
blue: "\x1b[38;5;75m", // repo totals
|
|
36
|
+
gray: "\x1b[38;5;245m", // labels
|
|
37
|
+
};
|
|
38
|
+
const PULSE = ["◐", "◓", "◑", "◒"];
|
|
39
|
+
export class MegaRuntime {
|
|
40
|
+
config;
|
|
41
|
+
// Store/dashboard/logger are rebound per-repo by bindRepo() so each git repo
|
|
42
|
+
// gets its own isolated state dir. They start bound to the global default.
|
|
43
|
+
store;
|
|
44
|
+
logger;
|
|
45
|
+
dashboard;
|
|
46
|
+
activeRepoRoot = null;
|
|
47
|
+
currentStateDir;
|
|
48
|
+
// The only mutable per-session state. Reset on session_start / session_tree.
|
|
49
|
+
rt = {
|
|
50
|
+
sessionId: normalizeSessionId(undefined),
|
|
51
|
+
persistedThisSession: false,
|
|
52
|
+
lastCheckpointId: undefined,
|
|
53
|
+
lastCompactedFrom: 0,
|
|
54
|
+
lastCompactedTokens: 0,
|
|
55
|
+
dedupSkips: 0,
|
|
56
|
+
dedupAttempts: 0,
|
|
57
|
+
tokensSaved: 0,
|
|
58
|
+
};
|
|
59
|
+
debounceUntil = 0;
|
|
60
|
+
// Agent tracking for real-time widget updates
|
|
61
|
+
activeAgents = 0;
|
|
62
|
+
currentTurn = 0;
|
|
63
|
+
// Recall block produced by auto-inline (resume/branch) that the next
|
|
64
|
+
// before_agent_start should prepend to the system prompt. Unset after use.
|
|
65
|
+
pendingRecallBlock;
|
|
66
|
+
statusKey; // current status text for dashboard
|
|
67
|
+
// Active model/provider (for real cost estimation). Captured from ctx.model
|
|
68
|
+
// on model_select + session_start; persisted to SQL so cost + the dashboard
|
|
69
|
+
// can read it without a live ctx.
|
|
70
|
+
currentModel;
|
|
71
|
+
// Live "what it's doing right now" line for the toolbar. Set on each
|
|
72
|
+
// compaction; shown in teal while recent, then kept as the last-seen action so
|
|
73
|
+
// the widget is never blank. Cleared on session reset.
|
|
74
|
+
currentActivity;
|
|
75
|
+
lastActivityAt = 0;
|
|
76
|
+
// Live per-tier dedup trace (Phase 1): e.g. "L0 ✓ → L1 ✓ → L2 0.91 → stored".
|
|
77
|
+
// Built from the store's sync onTier callback during a compaction so the user
|
|
78
|
+
// watches each tier evaluate in real time. Cleared once the outcome settles.
|
|
79
|
+
tierTrace;
|
|
80
|
+
// Phase 3 — standout toolbar state.
|
|
81
|
+
// Recall/activity ticker: a small ring buffer (≤5) of recent compact/recall
|
|
82
|
+
// events so the widget shows a live history instead of a single last action.
|
|
83
|
+
ticker = [];
|
|
84
|
+
TICKER_MAX = 5;
|
|
85
|
+
// Pulsing status: set true while a compaction is in flight, cleared on result.
|
|
86
|
+
pulsing = false;
|
|
87
|
+
// Rolling "saved" goal for the progress bar — grows as we save more, so the
|
|
88
|
+
// bar always has a meaningful denominator (never sits at 100% forever).
|
|
89
|
+
savedGoal = 50_000;
|
|
90
|
+
// Last explain-why line (dedup reason / anchor-kept / superseded), surfaced
|
|
91
|
+
// while fresh.
|
|
92
|
+
lastWhy = undefined;
|
|
93
|
+
// Context tracking for the dashboard (updated in the context handler).
|
|
94
|
+
lastCtxTokens = null;
|
|
95
|
+
lastCtxPercent = null;
|
|
96
|
+
lastCtxWindow = 0;
|
|
97
|
+
constructor(config) {
|
|
98
|
+
this.config = config;
|
|
99
|
+
this.store = new VectorStore({ dedupSim: config.dedupSim, stateDir: config.stateDir });
|
|
100
|
+
this.logger = new Logger({ enabled: config.debug, path: join(config.stateDir, "mega-compact.log") });
|
|
101
|
+
this.dashboard = new Dashboard(config.stateDir);
|
|
102
|
+
this.currentStateDir = config.stateDir;
|
|
103
|
+
}
|
|
104
|
+
// ---- per-repo binding -----------------------------------------------------
|
|
105
|
+
/**
|
|
106
|
+
* Point store/dashboard/logger at the current repo's state dir. Rebuilds the
|
|
107
|
+
* instances only when the repo root changes, so cross-repo dedup stats, db,
|
|
108
|
+
* and events are fully isolated. Falls back to the global default outside git.
|
|
109
|
+
*/
|
|
110
|
+
bindRepo(cwd) {
|
|
111
|
+
const dir = cwd ? repoStateDir(cwd, this.config.stateDir) : this.config.stateDir;
|
|
112
|
+
const key = cwd ? resolveRepoRoot(cwd) ?? dir : dir;
|
|
113
|
+
if (key === this.activeRepoRoot)
|
|
114
|
+
return dir;
|
|
115
|
+
this.activeRepoRoot = key;
|
|
116
|
+
this.currentStateDir = dir;
|
|
117
|
+
this.store = new VectorStore({ dedupSim: this.config.dedupSim, stateDir: dir });
|
|
118
|
+
this.logger = new Logger({ enabled: this.config.debug, path: join(dir, "mega-compact.log") });
|
|
119
|
+
this.dashboard = new Dashboard(dir);
|
|
120
|
+
return dir;
|
|
121
|
+
}
|
|
122
|
+
// ---- dashboard snapshot + widget ------------------------------------------
|
|
123
|
+
/** Collect live state and write it to disk (+ paint the above-editor widget). */
|
|
124
|
+
snapshot(ctx) {
|
|
125
|
+
if (ctx)
|
|
126
|
+
this.bindRepo(ctx.cwd);
|
|
127
|
+
const st = this.store.stats(this.rt.sessionId);
|
|
128
|
+
const repo = this.store.repoStats();
|
|
129
|
+
const di = this.store.dataInvariant();
|
|
130
|
+
const armed = this.lastCtxPercent != null && this.lastCtxPercent >= this.config.fastGatePct;
|
|
131
|
+
const ready = armed && (this.lastCtxTokens ?? 0) >= this.config.thresholdTokens;
|
|
132
|
+
this.dashboard.snapshot({
|
|
133
|
+
version: 1,
|
|
134
|
+
updatedAt: new Date().toISOString(),
|
|
135
|
+
tier: this.config.tier,
|
|
136
|
+
config: {
|
|
137
|
+
fastGatePct: this.config.fastGatePct,
|
|
138
|
+
thresholdTokens: this.config.thresholdTokens,
|
|
139
|
+
anchorUserMessages: this.config.anchorUserMessages,
|
|
140
|
+
preserveRecent: this.config.preserveRecent,
|
|
141
|
+
auto: this.config.auto,
|
|
142
|
+
autoInline: this.config.autoInline,
|
|
143
|
+
},
|
|
144
|
+
session: {
|
|
145
|
+
id: this.rt.sessionId,
|
|
146
|
+
state: this.statusKey ?? "idle",
|
|
147
|
+
persistedThisSession: this.rt.persistedThisSession,
|
|
148
|
+
lastCheckpointId: this.rt.lastCheckpointId ?? null,
|
|
149
|
+
lastCompactedFrom: this.rt.lastCompactedFrom,
|
|
150
|
+
lastCompactedTokens: this.rt.lastCompactedTokens,
|
|
151
|
+
dedupSkips: this.rt.dedupSkips,
|
|
152
|
+
dedupAttempts: this.rt.dedupAttempts,
|
|
153
|
+
},
|
|
154
|
+
context: { tokens: this.lastCtxTokens, percent: this.lastCtxPercent, contextWindow: this.lastCtxWindow },
|
|
155
|
+
trigger: { armed, ready, currentTokens: this.lastCtxTokens, thresholdTokens: this.config.thresholdTokens, fastGatePct: this.config.fastGatePct },
|
|
156
|
+
crew: { activeAgents: this.activeAgents, currentTurn: this.currentTurn },
|
|
157
|
+
store: { checkpointCount: st.checkpointCount, totalTokenEstimate: st.totalTokenEstimate, originalTokens: st.originalTokens, tokensSaved: this.rt.tokensSaved, injectedCount: st.injectedCount, dedupHitRate: st.dedupHitRate, storageDedupRate: st.storageDedupRate, dedupAttempts: st.dedupAttempts, dedupCollapsed: st.dedupCollapsed },
|
|
158
|
+
repo: {
|
|
159
|
+
checkpointCount: repo.checkpointCount,
|
|
160
|
+
totalTokenEstimate: repo.totalTokenEstimate,
|
|
161
|
+
originalTokens: repo.originalTokens,
|
|
162
|
+
tokensSaved: repo.tokensSaved,
|
|
163
|
+
sessionCount: repo.sessionCount,
|
|
164
|
+
dedupAttempts: repo.dedupAttempts,
|
|
165
|
+
dedupCollapsed: repo.dedupCollapsed,
|
|
166
|
+
storageDedupRate: repo.storageDedupRate,
|
|
167
|
+
},
|
|
168
|
+
integrity: {
|
|
169
|
+
regionsRetained: di.regionsRetained,
|
|
170
|
+
compressedOriginalBytes: di.compressedOriginalBytes,
|
|
171
|
+
duplicatesCollapsed: di.duplicatesCollapsed,
|
|
172
|
+
bytesPermanentlyDeleted: di.bytesPermanentlyDeleted,
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
// Live stats widget above the editor
|
|
176
|
+
if (ctx) {
|
|
177
|
+
const tokStr = this.lastCtxTokens != null ? `${Math.round(this.lastCtxTokens / 1000)}k` : "?";
|
|
178
|
+
const maxStr = this.lastCtxWindow > 0 ? `${Math.round(this.lastCtxWindow / 1000)}k` : "?";
|
|
179
|
+
const pctStr = this.lastCtxPercent != null ? `${Math.round(this.lastCtxPercent * 10) / 10}%` : "?%";
|
|
180
|
+
const triggerLabel = ready ? `${C.green}● ready${C.reset}` : armed ? `${C.amber}◐ armed${C.reset}` : `${C.gray}○ idle${C.reset}`;
|
|
181
|
+
// Storage dedup rate is cumulative (store-wide, per-repo) and survives
|
|
182
|
+
// session resets. Always show a number: 0% before any compaction, a
|
|
183
|
+
// decimal for sub-10% rates so small-but-real dedup isn't rounded away.
|
|
184
|
+
const storageRate = st.storageDedupRate; // 0..1
|
|
185
|
+
const dedupStr = storageRate * 100 >= 10
|
|
186
|
+
? `${Math.round(storageRate * 100)}%`
|
|
187
|
+
: `${(storageRate * 100).toFixed(1)}%`;
|
|
188
|
+
// saved = tokens removed from context (cumulative original − stored).
|
|
189
|
+
// Show BOTH this-session (rt.tokensSaved) and repo-wide-total
|
|
190
|
+
// (repo.tokensSaved) so the user sees per-session progress vs the running
|
|
191
|
+
// repo total. "used" = stored checkpoint tokens (repo.totalTokenEstimate
|
|
192
|
+
// vs st.totalTokenEstimate). Use "k" only at/above 1000 so small-but-real
|
|
193
|
+
// numbers stay visible (previously Math.round(x/1000) zeroed <1000).
|
|
194
|
+
const fmt = (x) => (x >= 1000 ? `${(x / 1000).toFixed(1)}k` : `${x}`);
|
|
195
|
+
const savedStr = `${C.green}${fmt(this.rt.tokensSaved)} sess${C.reset} / ${C.blue}${fmt(repo.tokensSaved)} repo${C.reset}`;
|
|
196
|
+
const usedStr = `${C.cyan}${fmt(st.totalTokenEstimate)} sess${C.reset} / ${C.blue}${fmt(repo.totalTokenEstimate)} repo${C.reset}`;
|
|
197
|
+
const agentStr = this.activeAgents > 0 ? ` │ 🤖 ${this.activeAgents} agent${this.activeAgents === 1 ? "" : "s"}` : "";
|
|
198
|
+
const turnStr = this.currentTurn > 0 ? ` │ turn ${this.currentTurn}` : "";
|
|
199
|
+
// Phase 3 — pulsing status glyph while a compaction is in flight.
|
|
200
|
+
const pulse = this.pulsing ? `${C.cyan}${PULSE[Math.floor(Date.now() / 250) % PULSE.length]}${C.reset} ` : "";
|
|
201
|
+
const lines = [
|
|
202
|
+
` ${C.amber}⚡ ${this.config.tier}${C.reset} │ ${tokStr}/${maxStr} tokens (${C.bold}${pctStr}${C.reset}) │ ${st.checkpointCount} saved${agentStr}${turnStr}`,
|
|
203
|
+
` ${triggerLabel} │ ${C.magenta}repeat-skipped: ${dedupStr}${C.reset} │ ${C.gray}memory held:${C.reset} ${usedStr} │ ${C.gray}space freed:${C.reset} ${savedStr}`,
|
|
204
|
+
];
|
|
205
|
+
// Phase 3 — compact progress bar: session tokens saved toward the rolling goal.
|
|
206
|
+
if (this.rt.tokensSaved > 0) {
|
|
207
|
+
const goal = Math.max(this.savedGoal, 1);
|
|
208
|
+
const pct = Math.min(100, Math.round((this.rt.tokensSaved / goal) * 100));
|
|
209
|
+
const filled = Math.round((pct / 100) * 10);
|
|
210
|
+
const bar = "▓".repeat(filled) + "░".repeat(10 - filled);
|
|
211
|
+
lines.push(` ${C.green}saved ${fmt(this.rt.tokensSaved)} ${bar}${C.reset} ${pct}% of ${fmt(goal)}`);
|
|
212
|
+
}
|
|
213
|
+
// Live "now processing" line — teal while fresh (≤4s), then the last-seen
|
|
214
|
+
// action keeps the widget lively. Cleared on session reset.
|
|
215
|
+
const fresh = Date.now() - this.lastActivityAt < 4000;
|
|
216
|
+
if (this.tierTrace && fresh) {
|
|
217
|
+
lines.push(` ${pulse}${this.tierTrace}`);
|
|
218
|
+
}
|
|
219
|
+
else if (this.currentActivity) {
|
|
220
|
+
lines.push(` ${fresh ? C.teal : C.dim}${this.currentActivity}${C.reset}`);
|
|
221
|
+
}
|
|
222
|
+
else if (this.pulsing) {
|
|
223
|
+
lines.push(` ${pulse}${C.teal}compacting…${C.reset}`);
|
|
224
|
+
}
|
|
225
|
+
// Phase 3 — explain-why line (fresh only).
|
|
226
|
+
if (this.lastWhy && fresh)
|
|
227
|
+
lines.push(` ${C.gray}${this.lastWhy}${C.reset}`);
|
|
228
|
+
// Phase 3 — recall/activity ticker (most-recent first), fresh only.
|
|
229
|
+
if (fresh) {
|
|
230
|
+
for (let i = this.ticker.length - 1; i >= 0; i--) {
|
|
231
|
+
if (lines.length >= 9)
|
|
232
|
+
break; // leave room for the hint line (MAX 10)
|
|
233
|
+
lines.push(` ${i === this.ticker.length - 1 ? "" : C.dim}${this.ticker[i].text}${C.reset}`);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
// Plain-language hint so first-time users understand the widget. Always
|
|
237
|
+
// last, dimmed. "/mega-help explains these terms."
|
|
238
|
+
if (lines.length < 10) {
|
|
239
|
+
lines.push(` ${C.dim}auto-compresses old context to free space · nothing deleted · /mega-help${C.reset}`);
|
|
240
|
+
}
|
|
241
|
+
ctx.ui.setWidget(WIDGET_KEY, lines, { placement: "aboveEditor" });
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
setStatus(ctx, text) {
|
|
245
|
+
this.statusKey = text;
|
|
246
|
+
ctx.ui.setStatus(STATUS_KEY, text);
|
|
247
|
+
}
|
|
248
|
+
resetRuntime(sessionId) {
|
|
249
|
+
const sid = normalizeSessionId(sessionId);
|
|
250
|
+
if (this.rt.sessionId === sid && this.rt.persistedThisSession)
|
|
251
|
+
return; // same session, keep checkpoint memory
|
|
252
|
+
this.rt = {
|
|
253
|
+
sessionId: sid,
|
|
254
|
+
persistedThisSession: false,
|
|
255
|
+
lastCheckpointId: undefined,
|
|
256
|
+
lastCompactedFrom: 0,
|
|
257
|
+
lastCompactedTokens: 0,
|
|
258
|
+
dedupSkips: 0,
|
|
259
|
+
dedupAttempts: 0,
|
|
260
|
+
tokensSaved: 0,
|
|
261
|
+
};
|
|
262
|
+
this.statusKey = undefined;
|
|
263
|
+
this.activeAgents = 0;
|
|
264
|
+
this.currentTurn = 0;
|
|
265
|
+
this.currentActivity = undefined;
|
|
266
|
+
this.lastActivityAt = 0;
|
|
267
|
+
this.tierTrace = undefined;
|
|
268
|
+
this.ticker.length = 0;
|
|
269
|
+
this.pulsing = false;
|
|
270
|
+
this.savedGoal = 50_000;
|
|
271
|
+
this.lastWhy = undefined;
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Capture the active model/provider from ctx.model and persist it so cost
|
|
275
|
+
* estimation + the dashboard can read real pricing. Cheap + idempotent-ish:
|
|
276
|
+
* only writes a new row when the model id changes (models change rarely).
|
|
277
|
+
*/
|
|
278
|
+
captureModel(ctx) {
|
|
279
|
+
const m = ctx.model;
|
|
280
|
+
if (!m)
|
|
281
|
+
return;
|
|
282
|
+
if (this.currentModel && this.currentModel.modelId === m.id && this.currentModel.provider === m.provider)
|
|
283
|
+
return;
|
|
284
|
+
let providerName = null;
|
|
285
|
+
try {
|
|
286
|
+
providerName = ctx.modelRegistry?.getProviderDisplayName(m.provider) ?? null;
|
|
287
|
+
}
|
|
288
|
+
catch { /* optional */ }
|
|
289
|
+
const snap = {
|
|
290
|
+
provider: m.provider,
|
|
291
|
+
providerName,
|
|
292
|
+
modelId: m.id,
|
|
293
|
+
modelName: m.name ?? null,
|
|
294
|
+
inputRate: m.cost?.input ?? 0,
|
|
295
|
+
outputRate: m.cost?.output ?? 0,
|
|
296
|
+
contextWindow: m.contextWindow ?? 0,
|
|
297
|
+
maxTokens: m.maxTokens ?? 0,
|
|
298
|
+
reasoning: !!m.reasoning,
|
|
299
|
+
};
|
|
300
|
+
this.currentModel = { ...snap, capturedAt: Date.now() };
|
|
301
|
+
try {
|
|
302
|
+
const repo = resolveRepoRoot(ctx.cwd) ?? this.currentStateDir;
|
|
303
|
+
recordModelSnapshot(repo, snap, this.currentStateDir);
|
|
304
|
+
}
|
|
305
|
+
catch { /* non-fatal: cost estimation degrades to model-in-memory only */ }
|
|
306
|
+
}
|
|
307
|
+
/** Build the sync onTier callback that paints the live per-tier trace. */
|
|
308
|
+
makeTierCallback(ctx) {
|
|
309
|
+
const order = ["L0", "L1", "L2", "new"];
|
|
310
|
+
const seen = new Map();
|
|
311
|
+
const glyph = (status) => status === "deduped" ? `${C.green}✓${C.reset}` :
|
|
312
|
+
status === "passed" ? `${C.dim}○${C.reset}` :
|
|
313
|
+
status === "scanning" ? `${C.amber}…${C.reset}` :
|
|
314
|
+
`${C.cyan}●${C.reset}`;
|
|
315
|
+
return (ev) => {
|
|
316
|
+
const label = ev.tier === "new"
|
|
317
|
+
? `${C.cyan}stored${C.reset}`
|
|
318
|
+
: `${ev.tier} ${glyph(ev.status)}` +
|
|
319
|
+
(ev.detail ? ` ${C.gray}(${ev.detail})${C.reset}` : "");
|
|
320
|
+
// Show the most recent outcome per tier (collapses re-fires).
|
|
321
|
+
seen.set(ev.tier, label);
|
|
322
|
+
const show = [];
|
|
323
|
+
for (const t of order)
|
|
324
|
+
if (seen.has(t))
|
|
325
|
+
show.push(seen.get(t));
|
|
326
|
+
this.tierTrace = `${C.teal}⚙${C.reset} ${show.join(` ${C.gray}→${C.reset} `)}`;
|
|
327
|
+
this.lastActivityAt = Date.now();
|
|
328
|
+
try {
|
|
329
|
+
this.snapshot(ctx);
|
|
330
|
+
}
|
|
331
|
+
catch { /* non-fatal */ }
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
// Phase 3 — recall/activity ticker ring buffer.
|
|
335
|
+
pushTicker(text) {
|
|
336
|
+
this.ticker.push({ text, at: Date.now() });
|
|
337
|
+
while (this.ticker.length > this.TICKER_MAX)
|
|
338
|
+
this.ticker.shift();
|
|
339
|
+
this.lastActivityAt = Date.now();
|
|
340
|
+
}
|
|
341
|
+
/** Convert the messages pi hands us in the `context` event into the engine view. */
|
|
342
|
+
engineView(messages) {
|
|
343
|
+
return toEngineMessages(messages);
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Latest user message text — used as the auto-inline recall query.
|
|
348
|
+
* Kept as a free function (not instance state) since it only reads ctx.
|
|
349
|
+
*/
|
|
350
|
+
export function recentUserQuery(ctx) {
|
|
351
|
+
try {
|
|
352
|
+
const entries = ctx.sessionManager.getEntries();
|
|
353
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
354
|
+
const msgs = sessionEntryToContextMessages(entries[i]);
|
|
355
|
+
for (let j = msgs.length - 1; j >= 0; j--) {
|
|
356
|
+
if (msgs[j].role === "user") {
|
|
357
|
+
const c = msgs[j].content;
|
|
358
|
+
if (typeof c === "string")
|
|
359
|
+
return c;
|
|
360
|
+
if (Array.isArray(c))
|
|
361
|
+
return c.map((b) => b.text).join(" ");
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
catch {
|
|
367
|
+
/* best-effort */
|
|
368
|
+
}
|
|
369
|
+
return "";
|
|
370
|
+
}
|
package/dist/src/store/sqlite.js
CHANGED
|
@@ -170,6 +170,24 @@ function initSchema(db) {
|
|
|
170
170
|
);
|
|
171
171
|
CREATE INDEX IF NOT EXISTS idx_daily_log_day ON daily_log(day);
|
|
172
172
|
|
|
173
|
+
-- Active model/provider for cost estimation + the future multi-repo
|
|
174
|
+
-- dashboard (Phase 5b). One row per (repo, model change); latest wins.
|
|
175
|
+
CREATE TABLE IF NOT EXISTS model_snapshots (
|
|
176
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
177
|
+
repo_root TEXT NOT NULL,
|
|
178
|
+
provider TEXT NOT NULL,
|
|
179
|
+
provider_name TEXT,
|
|
180
|
+
model_id TEXT NOT NULL,
|
|
181
|
+
model_name TEXT,
|
|
182
|
+
input_rate REAL, -- USD per input token (Model.cost.input)
|
|
183
|
+
output_rate REAL, -- USD per output token (Model.cost.output)
|
|
184
|
+
context_window INTEGER,
|
|
185
|
+
max_tokens INTEGER,
|
|
186
|
+
reasoning INTEGER DEFAULT 0,
|
|
187
|
+
captured_at INTEGER
|
|
188
|
+
);
|
|
189
|
+
CREATE INDEX IF NOT EXISTS idx_model_repo ON model_snapshots(repo_root);
|
|
190
|
+
|
|
173
191
|
-- Lessons learned (future recall/browse feature seed).
|
|
174
192
|
CREATE TABLE IF NOT EXISTS lessons (
|
|
175
193
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -535,6 +553,48 @@ export function repoStats(stateDir = getStateDir()) {
|
|
|
535
553
|
storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
|
|
536
554
|
};
|
|
537
555
|
}
|
|
556
|
+
/** Persist the active model/provider for a repo (latest row wins per repo). */
|
|
557
|
+
export function recordModelSnapshot(repoRoot, snap, stateDir = getStateDir()) {
|
|
558
|
+
const db = openStore(stateDir);
|
|
559
|
+
db.prepare(`INSERT INTO model_snapshots
|
|
560
|
+
(repo_root, provider, provider_name, model_id, model_name, input_rate,
|
|
561
|
+
output_rate, context_window, max_tokens, reasoning, captured_at)
|
|
562
|
+
VALUES (@repo_root, @provider, @provider_name, @model_id, @model_name,
|
|
563
|
+
@input_rate, @output_rate, @context_window, @max_tokens, @reasoning, @captured_at)`).run({
|
|
564
|
+
repo_root: repoRoot,
|
|
565
|
+
provider: snap.provider,
|
|
566
|
+
provider_name: snap.providerName,
|
|
567
|
+
model_id: snap.modelId,
|
|
568
|
+
model_name: snap.modelName,
|
|
569
|
+
input_rate: snap.inputRate,
|
|
570
|
+
output_rate: snap.outputRate,
|
|
571
|
+
context_window: snap.contextWindow,
|
|
572
|
+
max_tokens: snap.maxTokens,
|
|
573
|
+
reasoning: snap.reasoning ? 1 : 0,
|
|
574
|
+
captured_at: Date.now(),
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
/** Most recent model/provider snapshot for a repo, or undefined. */
|
|
578
|
+
export function latestModelSnapshot(stateDir = getStateDir()) {
|
|
579
|
+
const db = openStore(stateDir);
|
|
580
|
+
const row = db
|
|
581
|
+
.prepare(`SELECT * FROM model_snapshots ORDER BY captured_at DESC LIMIT 1`)
|
|
582
|
+
.get();
|
|
583
|
+
if (!row)
|
|
584
|
+
return undefined;
|
|
585
|
+
return {
|
|
586
|
+
provider: row.provider,
|
|
587
|
+
providerName: row.provider_name,
|
|
588
|
+
modelId: row.model_id,
|
|
589
|
+
modelName: row.model_name,
|
|
590
|
+
inputRate: row.input_rate,
|
|
591
|
+
outputRate: row.output_rate,
|
|
592
|
+
contextWindow: row.context_window,
|
|
593
|
+
maxTokens: row.max_tokens,
|
|
594
|
+
reasoning: row.reasoning === 1,
|
|
595
|
+
capturedAt: row.captured_at,
|
|
596
|
+
};
|
|
597
|
+
}
|
|
538
598
|
/** Close and evict a cached connection (test teardown only). */
|
|
539
599
|
export function closeStore(stateDir) {
|
|
540
600
|
const db = cache.get(stateDir);
|
|
@@ -133,6 +133,15 @@ function dashboardHtml(tierName: string): string {
|
|
|
133
133
|
.card.safe h2 { color: #3fb950; }
|
|
134
134
|
.safe-note { font-size: 12px; color: #8b949e; margin: 12px 0 0; line-height: 1.5; }
|
|
135
135
|
.value.ok { color: #3fb950; }
|
|
136
|
+
.label {
|
|
137
|
+
cursor: help;
|
|
138
|
+
border-bottom: 1px dotted #484f58;
|
|
139
|
+
}
|
|
140
|
+
.card.legend { grid-column: 1 / -1; }
|
|
141
|
+
.legend-list { margin: 0; padding-left: 18px; color: #c9d1d9; }
|
|
142
|
+
.legend-list li { margin-bottom: 8px; line-height: 1.5; }
|
|
143
|
+
.legend-list b { color: #f0f6fc; }
|
|
144
|
+
.legend-note { font-size: 12px; color: #8b949e; margin: 12px 0 0; font-style: italic; }
|
|
136
145
|
.card h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .5px; color: #8b949e; margin-bottom: 12px; font-weight: 600; }
|
|
137
146
|
.meter-track { background: #21262d; border-radius: 4px; height: 20px; overflow: hidden; margin: 8px 0; }
|
|
138
147
|
.meter-fill { height: 100%; border-radius: 4px; transition: width .6s ease; min-width: 2px; }
|
|
@@ -190,15 +199,15 @@ function dashboardHtml(tierName: string): string {
|
|
|
190
199
|
<div class="card">
|
|
191
200
|
<h2>Vector Store</h2>
|
|
192
201
|
<div class="stat-grid">
|
|
193
|
-
<span class="label">Checkpoints</span><span class="value" id="st-count">0</span>
|
|
194
|
-
<span class="label">Tokens Stored</span><span class="value" id="st-tokens">0</span>
|
|
195
|
-
<span class="label">Original Tokens</span><span class="value" id="st-orig">0</span>
|
|
196
|
-
<span class="label">Tokens Saved</span><span class="value" id="st-saved">0</span>
|
|
197
|
-
<span class="label">Injected</span><span class="value" id="st-injected">0</span>
|
|
198
|
-
<span class="label">
|
|
199
|
-
<span class="label">Storage Dedup</span><span class="value" id="st-sdedup">0%</span>
|
|
200
|
-
<span class="label">Collapsed</span><span class="value" id="st-collapsed">0</span>
|
|
201
|
-
<span class="label">Last ID</span><span class="value" id="st-lastid">—</span>
|
|
202
|
+
<span class="label" title="A saved summary of a chunk of your conversation that was compacted to free up space.">Checkpoints</span><span class="value" id="st-count">0</span>
|
|
203
|
+
<span class="label" title="How much conversation we are currently holding as compact summaries (the 'memory' this extension keeps). Smaller is better.">Tokens Stored</span><span class="value" id="st-tokens">0</span>
|
|
204
|
+
<span class="label" title="Total size of the original conversation text before it was compacted.">Original Tokens</span><span class="value" id="st-orig">0</span>
|
|
205
|
+
<span class="label" title="How much conversation space we have freed up for you (original size minus the compact summary we kept).">Tokens Saved</span><span class="value" id="st-saved">0</span>
|
|
206
|
+
<span class="label" title="How many times old context was automatically brought back into the conversation because it was relevant to what you were doing.">Injected</span><span class="value" id="st-injected">0</span>
|
|
207
|
+
<span class="label" title="Of the times we recalled old context, how often it was actually on-topic.">Recall Relevance</span><span class="value" id="st-dedup">0%</span>
|
|
208
|
+
<span class="label" title="How often new content matched something we already had, so we skipped storing a duplicate copy. Higher = less wasted space.">Storage Dedup</span><span class="value" id="st-sdedup">0%</span>
|
|
209
|
+
<span class="label" title="How many duplicate chunks we collapsed into one instead of storing separately.">Collapsed</span><span class="value" id="st-collapsed">0</span>
|
|
210
|
+
<span class="label" title="The ID of the most recent saved checkpoint.">Last ID</span><span class="value" id="st-lastid">—</span>
|
|
202
211
|
</div>
|
|
203
212
|
</div>
|
|
204
213
|
<div class="card">
|
|
@@ -241,6 +250,18 @@ function dashboardHtml(tierName: string): string {
|
|
|
241
250
|
<span class="label">Status</span><span class="value" id="cr-status">idle</span>
|
|
242
251
|
</div>
|
|
243
252
|
</div>
|
|
253
|
+
<div class="card legend">
|
|
254
|
+
<h2>What these numbers mean</h2>
|
|
255
|
+
<ul class="legend-list">
|
|
256
|
+
<li><b>Tokens saved</b> — conversation space this extension has freed up for you (it compacted old text into short summaries).</li>
|
|
257
|
+
<li><b>Tokens stored</b> — how much "memory" (compact summaries) the extension is currently holding for this repo.</li>
|
|
258
|
+
<li><b>Injected</b> — times old context was automatically pasted back in because it was relevant to your current task.</li>
|
|
259
|
+
<li><b>Recall relevance</b> — of those, how often the recalled context was actually on-topic.</li>
|
|
260
|
+
<li><b>Storage dedup</b> — how often new content matched something already saved, so a duplicate copy was skipped (saves space).</li>
|
|
261
|
+
<li><b>Data safety</b> — every compacted region is kept verbatim (compressed). Nothing is permanently deleted; you can restore any of it.</li>
|
|
262
|
+
</ul>
|
|
263
|
+
<p class="legend-note">Hover any label above for a quick explanation.</p>
|
|
264
|
+
</div>
|
|
244
265
|
</div>
|
|
245
266
|
|
|
246
267
|
<div class="events">
|