pi-smart-compact 7.5.1 → 7.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +492 -182
- package/dist/index.js +2979 -0
- package/package.json +5 -3
- package/src/constants.ts +0 -140
- package/src/core.ts +0 -360
- package/src/index.ts +0 -175
- package/src/phases/explore.ts +0 -371
- package/src/phases/synthesize.ts +0 -184
- package/src/phases/verify.ts +0 -191
- package/src/types.ts +0 -176
- package/src/ui/overlays.ts +0 -329
- package/src/utils/cache.ts +0 -145
- package/src/utils/damage.ts +0 -153
- package/src/utils/extraction.ts +0 -259
- package/src/utils/fingerprint.ts +0 -190
- package/src/utils/helpers.ts +0 -161
- package/src/utils/message-blocks.ts +0 -21
- package/src/utils/pruning.ts +0 -147
- package/src/utils/tokens.ts +0 -63
package/package.json
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-smart-compact",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.7.0",
|
|
4
4
|
"description": "EESV smart compaction extension for Pi Coding Agent — deterministic extraction, exploration, synthesis, verification with redundancy pruning, project fingerprinting, and damage detection.",
|
|
5
5
|
"license": "MIT",
|
|
6
|
+
"main": "./dist/index.js",
|
|
6
7
|
"author": "Alper Tarhan <alpertarhan@gmail.com>",
|
|
7
8
|
"repository": {
|
|
8
9
|
"type": "git",
|
|
@@ -25,17 +26,18 @@
|
|
|
25
26
|
"llm-context"
|
|
26
27
|
],
|
|
27
28
|
"files": [
|
|
28
|
-
"
|
|
29
|
+
"dist",
|
|
29
30
|
"README.md",
|
|
30
31
|
"LICENSE",
|
|
31
32
|
"CHANGELOG.md"
|
|
32
33
|
],
|
|
33
34
|
"scripts": {
|
|
35
|
+
"build": "rm -rf dist && mkdir dist && bun build ./src/index.ts --outdir ./dist --target bun --external '@earendil-works/*' --external 'typebox'",
|
|
34
36
|
"test": "bun test",
|
|
35
37
|
"typecheck": "bunx tsc --noEmit"
|
|
36
38
|
},
|
|
37
39
|
"pi": {
|
|
38
|
-
"extensions": ["./
|
|
40
|
+
"extensions": ["./dist/index.js"]
|
|
39
41
|
},
|
|
40
42
|
"peerDependencies": {
|
|
41
43
|
"@earendil-works/pi-ai": "*",
|
package/src/constants.ts
DELETED
|
@@ -1,140 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Constants, prompts, and profile defaults.
|
|
3
|
-
*/
|
|
4
|
-
|
|
5
|
-
import type { CompressionProfile, ProfileConfig } from "./types.ts";
|
|
6
|
-
|
|
7
|
-
export const VERSION = "7.5.0";
|
|
8
|
-
export const CHARS_PER_TOKEN = 3.8;
|
|
9
|
-
|
|
10
|
-
export const COMPACT_SYSTEM_PREFIX =
|
|
11
|
-
"You are an expert conversation summarizer for a coding agent. " +
|
|
12
|
-
"Produce structured markdown summaries. " +
|
|
13
|
-
"Follow output format exactly. " +
|
|
14
|
-
"Use EXACT names — never paraphrase code identifiers. " +
|
|
15
|
-
"Trust deterministic extraction data over intuition.";
|
|
16
|
-
|
|
17
|
-
export const PROFILES: Record<CompressionProfile, ProfileConfig> = {
|
|
18
|
-
light: {
|
|
19
|
-
summaryBudgetTokens: 10000,
|
|
20
|
-
keepRecentTokens: 30000,
|
|
21
|
-
minChunkTokens: 800,
|
|
22
|
-
maxChunkTokens: 12000,
|
|
23
|
-
singlePassMaxTokens: 40000,
|
|
24
|
-
batchMaxTokens: 30000,
|
|
25
|
-
},
|
|
26
|
-
balanced: {
|
|
27
|
-
summaryBudgetTokens: 6000,
|
|
28
|
-
keepRecentTokens: 20000,
|
|
29
|
-
minChunkTokens: 500,
|
|
30
|
-
maxChunkTokens: 8000,
|
|
31
|
-
singlePassMaxTokens: 30000,
|
|
32
|
-
batchMaxTokens: 24000,
|
|
33
|
-
},
|
|
34
|
-
aggressive: {
|
|
35
|
-
summaryBudgetTokens: 3000,
|
|
36
|
-
keepRecentTokens: 10000,
|
|
37
|
-
minChunkTokens: 300,
|
|
38
|
-
maxChunkTokens: 6000,
|
|
39
|
-
singlePassMaxTokens: 20000,
|
|
40
|
-
batchMaxTokens: 18000,
|
|
41
|
-
},
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
export const DEFAULT_CONFIG = {
|
|
45
|
-
profile: "balanced" as CompressionProfile,
|
|
46
|
-
profiles: PROFILES,
|
|
47
|
-
summaryModel: null as string | null,
|
|
48
|
-
segmentationModel: null as string | null,
|
|
49
|
-
autoTrigger: true,
|
|
50
|
-
backupEnabled: true,
|
|
51
|
-
backupDir: "",
|
|
52
|
-
};
|
|
53
|
-
|
|
54
|
-
export const NO_OP_RE = /applied:\s*0|no changes applied|nothing to (?:do|change)|0 edits? applied/i;
|
|
55
|
-
export const SHIFT_RE = /simdi|peki|bide|bi de|gecelim|bakalim|yapalim|baska|sonra|tamam simdi|now let|also|next|let's|moving on|switch to/i;
|
|
56
|
-
export const CHOICE_RE = /use\s+\S+\s+(?:instead|not|rather)|don't\s+use|avoid\s+|switch\s+to\s+|go\s+with\s+|prefer\s+/i;
|
|
57
|
-
|
|
58
|
-
// ── Prompt Templates ──
|
|
59
|
-
|
|
60
|
-
export const SINGLE_PASS_PREFIX =
|
|
61
|
-
"Summarize this coding agent conversation. Produce ONE structured summary.\n" +
|
|
62
|
-
"\nRules for Accuracy:\n" +
|
|
63
|
-
"1. Session Type: read-only tool calls = REVIEW, not implementation\n" +
|
|
64
|
-
"2. Status: Check for user complaints before marking \"Done\"\n" +
|
|
65
|
-
"3. Exact Names: Quote specific variable/function/parameter names, don't paraphrase\n" +
|
|
66
|
-
"4. Files: Use the VERIFIED file lists above (deterministically extracted, zero hallucination risk)\n" +
|
|
67
|
-
"\nOutput EXACTLY this format:\n\n" +
|
|
68
|
-
"## Goal\n[What the user is trying to accomplish]\n" +
|
|
69
|
-
"## Constraints & Preferences\n- [CRITICAL: user requirements, preferences, constraints]\n" +
|
|
70
|
-
"## Progress\n### Done\n- [x] [Completed tasks with file references]\n### In Progress\n- [ ] [Current work state]\n### Blocked\n- [Issues]\n" +
|
|
71
|
-
"## Key Decisions\n- **[Decision]**: [Rationale]\n" +
|
|
72
|
-
"## Files Modified\n- [Verified list from deterministic extraction]\n" +
|
|
73
|
-
"## Files Read\n- [Verified list from deterministic extraction]\n" +
|
|
74
|
-
"## Next Steps\n1. [What should happen next]\n" +
|
|
75
|
-
"## Critical Context\n- [Specific data, patterns, info needed to continue]\n- [Error patterns or gotchas]\n" +
|
|
76
|
-
"## Topics Covered\n[Chronological bullet list with priority in brackets]\n";
|
|
77
|
-
|
|
78
|
-
export const SINGLE_PASS_SUFFIX =
|
|
79
|
-
"\n{PREV_CONTEXT}\n\n{EXTRACTION_CONTEXT}\n\n{EXPLORATION_CONTEXT}\n\n<conversation>\n{CONVERSATION}\n</conversation>";
|
|
80
|
-
|
|
81
|
-
export const BATCH_PROMPT_PREFIX =
|
|
82
|
-
"Summarize these conversation segments.\n\nRules for Accuracy:\n" +
|
|
83
|
-
"1. Use EXACT file paths from extraction data\n" +
|
|
84
|
-
"2. Status: only mark \"done\" if there's clear evidence (successful test run, user confirmation)\n" +
|
|
85
|
-
"3. Quote specific values, don't paraphrase code\n\n" +
|
|
86
|
-
"For EACH segment produce:\n" +
|
|
87
|
-
"### {TOPIC_NAME}\n" +
|
|
88
|
-
"**Priority**: [critical|high|normal|low]\n" +
|
|
89
|
-
"**Summary**: [2-4 sentences: what happened, errors, code changes with paths]\n" +
|
|
90
|
-
"**Decisions**: [comma-separated, or \"None\"]\n" +
|
|
91
|
-
"**Modified**: [comma-separated paths, or \"None\"]\n" +
|
|
92
|
-
"**Read**: [comma-separated paths, or \"None\"]\n";
|
|
93
|
-
|
|
94
|
-
export const BATCH_PROMPT_SUFFIX = "\n{EXTRACTION_CONTEXT}\n\n<segments>\n{TEXT}\n</segments>";
|
|
95
|
-
|
|
96
|
-
export const ASSEMBLY_PROMPT_PREFIX =
|
|
97
|
-
"Merge these topic summaries into ONE coherent summary.\n\n" +
|
|
98
|
-
"## IMMUTABLE CONTEXT (do not modify or contradict these facts)\n" +
|
|
99
|
-
"These are deterministically verified from the original conversation. They take priority over ANY summary content below.\n\n" +
|
|
100
|
-
"Rules:\n" +
|
|
101
|
-
"1. Preserve ALL critical/high info. Condense normal, minimize low.\n" +
|
|
102
|
-
"2. Chronological order.\n" +
|
|
103
|
-
"3. The pre-processed data below is GROUND TRUTH — trust it over individual summaries.\n" +
|
|
104
|
-
"4. Files Modified list is deterministically verified — if a summary says a file was modified but it's NOT in the list above, omit it.\n" +
|
|
105
|
-
"5. Key Decisions below are verified — preserve them exactly, do not paraphrase the decision text.\n" +
|
|
106
|
-
"6. Do NOT fabricate file paths, function names, or error messages not present in the verified data.\n\n" +
|
|
107
|
-
"Format:\n" +
|
|
108
|
-
"## Goal\n[Overall objective]\n" +
|
|
109
|
-
"## Constraints & Preferences\n- [CRITICAL requirements, preferences, constraints]\n" +
|
|
110
|
-
"## Progress\n### Done\n- [x] [Completed tasks with file refs]\n### In Progress\n- [ ] [Current work state]\n### Blocked\n- [Issues]\n" +
|
|
111
|
-
"## Key Decisions\n- **[Decision]**: [Rationale]\n" +
|
|
112
|
-
"## Files Modified\n- [Verified deterministic list]\n" +
|
|
113
|
-
"## Files Read\n- [Verified deterministic list]\n" +
|
|
114
|
-
"## Next Steps\n1. [What should happen next]\n" +
|
|
115
|
-
"## Critical Context\n- [Data, patterns, info needed]\n" +
|
|
116
|
-
"## Topics Covered\n[Chronological bullets with priority]\n";
|
|
117
|
-
|
|
118
|
-
export const ASSEMBLY_PROMPT_SUFFIX =
|
|
119
|
-
"\nIMMUTABLE CONTEXT (verified deterministic data):\n- Key Decisions: {DECISIONS}\n- Files Modified (VERIFIED): {MODIFIED}\n- Files Read (VERIFIED): {READ}\n\n{EXPLORATION_CONTEXT}\n{PREV_CONTEXT}\n\n<summaries>{SUMMARIES}</summaries>";
|
|
120
|
-
|
|
121
|
-
// ── Session-type-specific prompt instructions ──
|
|
122
|
-
|
|
123
|
-
export const SESSION_TYPE_INSTRUCTIONS: Record<string, string> = {
|
|
124
|
-
debugging: "Focus on: error chains, root cause analysis, attempted fixes, resolution status. Prioritize error messages and stack traces. Mark files as Done only if all errors resolved.",
|
|
125
|
-
implementation: "Focus on: files created/modified, architectural decisions, feature completeness, test coverage. Prioritize code changes with exact paths.",
|
|
126
|
-
review: "Focus on: files read, issues found, recommendations, approval status. Prioritize findings over changes. Read-only tool calls = REVIEW, not implementation.",
|
|
127
|
-
discussion: "Focus on: decisions made, trade-offs discussed, consensus reached. Prioritize rationale over implementation details.",
|
|
128
|
-
};
|
|
129
|
-
|
|
130
|
-
export const EXPLORER_SYSTEM_PROMPT =
|
|
131
|
-
"You are a conversation analyst. You have deterministic extraction data and can query the raw conversation using tools.\n\n" +
|
|
132
|
-
"Your job:\n" +
|
|
133
|
-
"1. Verify/enrich the extracted boundaries (merge, split, or add as needed)\n" +
|
|
134
|
-
"2. Identify cross-topic relationships\n" +
|
|
135
|
-
"3. Find implicit constraints (user tone, frustration, urgency)\n" +
|
|
136
|
-
"4. Assess completion status accurately\n" +
|
|
137
|
-
"5. Extract the narrative arc\n\n" +
|
|
138
|
-
"Use tools BEFORE forming conclusions. You may make up to 8 tool calls.\n\n" +
|
|
139
|
-
"After exploration, output ONLY a JSON object (no markdown):\n" +
|
|
140
|
-
'{"boundaries":[{"afterIndex":N,"topic":"...","priority":"critical|high|normal|low","confidence":0.0-1.0}],"mainGoal":"...","sessionType":"implementation|review|debugging|discussion","enrichedConstraints":[...],"crossReferences":[...],"statusAssessment":{"done":[...],"inProgress":[...],"blocked":[...]},"criticalContext":[...],"keyDecisions":[...]}';
|
package/src/core.ts
DELETED
|
@@ -1,360 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Core EESV pipeline runner.
|
|
3
|
-
*/
|
|
4
|
-
|
|
5
|
-
import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
|
|
6
|
-
import { convertToLlm, serializeConversation } from "@earendil-works/pi-coding-agent";
|
|
7
|
-
import type { Model, Api } from "@earendil-works/pi-ai";
|
|
8
|
-
import type {
|
|
9
|
-
CompressionProfile, PendingCompaction, LlmMessage, StructuredExtraction,
|
|
10
|
-
ExplorationReport, SmartCompactDetails, ChunkSummary,
|
|
11
|
-
} from "./types.ts";
|
|
12
|
-
import { PROFILES } from "./constants.ts";
|
|
13
|
-
import { estimateTokens, getProviderCaps } from "./utils/tokens.ts";
|
|
14
|
-
import {
|
|
15
|
-
resetCompactSessionId, resetMetrics, appendMetricsLog, getMetricsSummary,
|
|
16
|
-
saveCachedExtraction, loadCachedExtraction, mergeExtractions, cacheOpts,
|
|
17
|
-
} from "./utils/cache.ts";
|
|
18
|
-
import { extractStructured } from "./utils/extraction.ts";
|
|
19
|
-
import { pruneRedundant } from "./utils/pruning.ts";
|
|
20
|
-
import { deriveProjectId, loadProjectFingerprint, saveProjectFingerprint, buildProjectContext } from "./utils/fingerprint.ts";
|
|
21
|
-
import { detectDamage, logDamageReport } from "./utils/damage.ts";
|
|
22
|
-
import {
|
|
23
|
-
loadConfig, backupConversation, getPreviousCompactionContext,
|
|
24
|
-
smartKeepBoundary, createBatches,
|
|
25
|
-
} from "./utils/helpers.ts";
|
|
26
|
-
import { extractText } from "./utils/extraction.ts";
|
|
27
|
-
import { exploreConversation, shouldExplore } from "./phases/explore.ts";
|
|
28
|
-
import { chunkLlmMessages, singlePassCompact, summarizeBatch, assembleLLM, assembleFallback } from "./phases/synthesize.ts";
|
|
29
|
-
import { verifySummary, patchSummary, patchDeterministic } from "./phases/verify.ts";
|
|
30
|
-
import { showProgressOverlay, showResultScreen } from "./ui/overlays.ts";
|
|
31
|
-
|
|
32
|
-
export async function runSmartCompact(
|
|
33
|
-
ctx: ExtensionCommandContext,
|
|
34
|
-
summaryModel: Model<Api>, segModel: Model<Api>,
|
|
35
|
-
profile: CompressionProfile,
|
|
36
|
-
verbose: boolean, dryRun: boolean,
|
|
37
|
-
pendingRef: { value: PendingCompaction | null; createdAt: number },
|
|
38
|
-
isRunning: { value: boolean },
|
|
39
|
-
autoTriggered: boolean,
|
|
40
|
-
userNote?: string,
|
|
41
|
-
skipCompact?: boolean,
|
|
42
|
-
): Promise<void> {
|
|
43
|
-
if (isRunning.value) return;
|
|
44
|
-
isRunning.value = true;
|
|
45
|
-
const pipelineStart = Date.now();
|
|
46
|
-
resetCompactSessionId();
|
|
47
|
-
resetMetrics();
|
|
48
|
-
|
|
49
|
-
if (!summaryModel || !segModel) { isRunning.value = false; if (!autoTriggered) ctx.ui.notify("Model resolve failed", "error"); return; }
|
|
50
|
-
try {
|
|
51
|
-
const config = loadConfig();
|
|
52
|
-
const pc = { ...PROFILES[profile], ...(config.profiles?.[profile] ?? {}) };
|
|
53
|
-
const auth = await ctx.modelRegistry.getApiKeyAndHeaders(summaryModel);
|
|
54
|
-
const segAuth = segModel !== summaryModel ? await ctx.modelRegistry.getApiKeyAndHeaders(segModel) : auth;
|
|
55
|
-
if ((!auth.ok || !auth.apiKey) || (!segAuth.ok || !segAuth.apiKey)) { isRunning.value = false; if (!autoTriggered) ctx.ui.notify("Auth failed", "error"); return; }
|
|
56
|
-
|
|
57
|
-
const usage = ctx.getContextUsage();
|
|
58
|
-
const totalTokens = usage?.tokens ?? 0;
|
|
59
|
-
if (!totalTokens || totalTokens < 5000) { isRunning.value = false; if (!autoTriggered) ctx.ui.notify("Context OK or unknown", "info"); return; }
|
|
60
|
-
|
|
61
|
-
const notify = (msg: string, type: "info" | "success" | "warning" | "error" = "info") => { ctx.ui.notify(msg, type); };
|
|
62
|
-
const ctrl = new AbortController();
|
|
63
|
-
const signal = ctrl.signal;
|
|
64
|
-
const modelLabel = summaryModel.provider + "/" + summaryModel.id;
|
|
65
|
-
notify("Smart compact: " + modelLabel + ", " + profile + ", tokens=" + totalTokens, "info");
|
|
66
|
-
notify("EESV Compact (" + modelLabel + ", " + profile + ") — " + (totalTokens ?? 0).toLocaleString() + "t", "info");
|
|
67
|
-
|
|
68
|
-
const branch = ctx.sessionManager.getBranch();
|
|
69
|
-
interface SessionMessageEntry { type: "message"; id: string; message: unknown }
|
|
70
|
-
const msgs = branch.filter((e: SessionMessageEntry): e is SessionMessageEntry => e.type === "message" && e.message != null);
|
|
71
|
-
if (msgs.length < 3) { isRunning.value = false; return; }
|
|
72
|
-
|
|
73
|
-
let accTokens = 0, keepFrom = msgs.length;
|
|
74
|
-
for (let i = msgs.length - 1; i >= 0; i--) {
|
|
75
|
-
// Use extractText for content instead of JSON.stringify to avoid metadata overhead
|
|
76
|
-
const msg = msgs[i].message as any;
|
|
77
|
-
const contentText = msg?.content ? (typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)) : "";
|
|
78
|
-
accTokens += estimateTokens(contentText);
|
|
79
|
-
if (accTokens >= pc.keepRecentTokens) { keepFrom = i; break; }
|
|
80
|
-
}
|
|
81
|
-
keepFrom = smartKeepBoundary(msgs, keepFrom);
|
|
82
|
-
|
|
83
|
-
const toCompact = msgs.slice(0, keepFrom);
|
|
84
|
-
if (!toCompact.length) { isRunning.value = false; return; }
|
|
85
|
-
const firstKeptId = msgs[keepFrom]?.id ?? msgs[msgs.length - 1]?.id ?? "";
|
|
86
|
-
|
|
87
|
-
if (!autoTriggered) {
|
|
88
|
-
showProgressOverlay(ctx, { phase: 1, phaseName: "Extract", detail: "Preparing...", model: modelLabel, profile });
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
const llmMessages = convertToLlm(toCompact.map(e => e.message)) as LlmMessage[];
|
|
92
|
-
|
|
93
|
-
// ── Pre-compaction redundancy pruning ──
|
|
94
|
-
const pruning = pruneRedundant(llmMessages);
|
|
95
|
-
if (pruning.prunedCount > 0) {
|
|
96
|
-
notify("Pruning: " + pruning.prunedCount + " msgs removed (" + pruning.reasons.map(r => r.count + "x " + r.reason).join(", ") + ")", "info");
|
|
97
|
-
}
|
|
98
|
-
const prunedMessages = pruning.messages;
|
|
99
|
-
const convText = serializeConversation(prunedMessages);
|
|
100
|
-
const convTokens = estimateTokens(convText);
|
|
101
|
-
|
|
102
|
-
const sessionId = ctx.sessionManager.getSessionId?.() ?? "unknown";
|
|
103
|
-
const backupPath = backupConversation(convText, sessionId);
|
|
104
|
-
const prevContext = getPreviousCompactionContext(branch);
|
|
105
|
-
|
|
106
|
-
// ── Project fingerprint (cross-session context) ──
|
|
107
|
-
const projectId = deriveProjectId(extraction);
|
|
108
|
-
const fingerprint = loadProjectFingerprint(projectId);
|
|
109
|
-
if (fingerprint) {
|
|
110
|
-
notify("Project: " + fingerprint.language + (fingerprint.framework ? "/" + fingerprint.framework : "") + " (" + fingerprint.sessionCount + " sessions)", "info");
|
|
111
|
-
}
|
|
112
|
-
const projectCtx = buildProjectContext(fingerprint);
|
|
113
|
-
|
|
114
|
-
// Phase 1
|
|
115
|
-
const cachedExt = loadCachedExtraction(sessionId);
|
|
116
|
-
let extraction: StructuredExtraction;
|
|
117
|
-
if (cachedExt && cachedExt.lastMessageIndex < llmMessages.length - 1) {
|
|
118
|
-
const newMsgs = llmMessages.slice(cachedExt.lastMessageIndex + 1);
|
|
119
|
-
const delta = extractStructured(newMsgs, pc);
|
|
120
|
-
extraction = mergeExtractions(cachedExt.extraction, delta, cachedExt.messageCount);
|
|
121
|
-
notify("Phase 1 Incremental: " + (cachedExt.lastMessageIndex + 1) + " cached + " + newMsgs.length + " new messages", "info");
|
|
122
|
-
} else {
|
|
123
|
-
extraction = extractStructured(llmMessages, pc);
|
|
124
|
-
notify("Phase 1 Full: " + extraction.modifiedFiles.length + " files, " + extraction.errors.length + " errors", "info");
|
|
125
|
-
}
|
|
126
|
-
saveCachedExtraction(sessionId, extraction, llmMessages.length);
|
|
127
|
-
|
|
128
|
-
let finalSummary: string;
|
|
129
|
-
let method: string;
|
|
130
|
-
let llmCalls = 0;
|
|
131
|
-
let summaries: ChunkSummary[] = [];
|
|
132
|
-
let explorationReport: ExplorationReport | null = null;
|
|
133
|
-
let explorationRounds = 0;
|
|
134
|
-
let chunkCount = 0;
|
|
135
|
-
|
|
136
|
-
if (convTokens < pc.singlePassMaxTokens) {
|
|
137
|
-
if (!autoTriggered) showProgressOverlay(ctx, { phase: 2, phaseName: "Explore", detail: "Single-pass (" + convTokens.toLocaleString() + "t)", model: modelLabel, profile, extraction });
|
|
138
|
-
try {
|
|
139
|
-
const r = await singlePassCompact(convText, extraction, null, prevContext + projectCtx, summaryModel, { apiKey: auth.apiKey, headers: auth.headers }, signal);
|
|
140
|
-
finalSummary = r.summary; method = "single-pass"; llmCalls = r.llmCalls;
|
|
141
|
-
} catch (err) {
|
|
142
|
-
notify("Single-pass failed: " + (err instanceof Error ? err.message : String(err)), "warning");
|
|
143
|
-
finalSummary = assembleFallback([], extraction);
|
|
144
|
-
method = "heuristic"; llmCalls = 0;
|
|
145
|
-
}
|
|
146
|
-
} else {
|
|
147
|
-
// Adaptive exploration gate: skip explore for simple sessions
|
|
148
|
-
const needsExploration = shouldExplore(extraction);
|
|
149
|
-
if (needsExploration) {
|
|
150
|
-
if (!autoTriggered) showProgressOverlay(ctx, { phase: 2, phaseName: "Explore", detail: "Exploring...", model: modelLabel, profile, extraction });
|
|
151
|
-
try {
|
|
152
|
-
const expResult = await exploreConversation(llmMessages, extraction, segModel, { apiKey: segAuth.apiKey, headers: segAuth.headers }, prevContext || undefined, userNote, signal, 8, notify);
|
|
153
|
-
explorationReport = expResult.report;
|
|
154
|
-
explorationRounds = expResult.rounds;
|
|
155
|
-
notify("Phase 2 Explore: " + expResult.rounds + " rounds, " + explorationReport.boundaries.length + " boundaries" + (expResult.toolSupported ? "" : " (no tool support)"), "info");
|
|
156
|
-
} catch (err) {
|
|
157
|
-
notify("Phase 2 Explore: failed - " + (err instanceof Error ? err.message : String(err)), "warning");
|
|
158
|
-
}
|
|
159
|
-
} else {
|
|
160
|
-
notify("Phase 2 Explore: skipped (simple session: " + extraction.topics.length + " topics, " + extraction.errors.filter(e => !e.resolved).length + " unresolved errors)", "info");
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
let boundaries: import("./types.ts").TopicBoundary[];
|
|
164
|
-
if (explorationReport?.boundaries.length) {
|
|
165
|
-
// Merge LLM boundaries with heuristic boundaries — don't discard heuristics
|
|
166
|
-
const llmBounds = explorationReport.boundaries.filter(b => b.confidence >= 0.4);
|
|
167
|
-
const heuristicBounds = extraction.topics.map(t => ({
|
|
168
|
-
afterIndex: t.endIndex,
|
|
169
|
-
topic: t.primaryFile ? "Working on " + t.primaryFile.split("/").pop() : "Segment",
|
|
170
|
-
priority: t.errorDensity > 2 ? "high" as const : "normal" as const,
|
|
171
|
-
confidence: 0.6,
|
|
172
|
-
}));
|
|
173
|
-
if (llmBounds.length > 0) {
|
|
174
|
-
// LLM boundaries are primary; fill gaps with heuristic boundaries
|
|
175
|
-
const merged = [...llmBounds];
|
|
176
|
-
for (const hb of heuristicBounds) {
|
|
177
|
-
const nearby = merged.find(m => Math.abs(m.afterIndex - hb.afterIndex) <= 3);
|
|
178
|
-
if (!nearby) merged.push(hb);
|
|
179
|
-
}
|
|
180
|
-
boundaries = merged.sort((a, b) => a.afterIndex - b.afterIndex);
|
|
181
|
-
} else {
|
|
182
|
-
boundaries = heuristicBounds;
|
|
183
|
-
}
|
|
184
|
-
} else {
|
|
185
|
-
boundaries = extraction.topics.map(t => ({
|
|
186
|
-
afterIndex: t.endIndex,
|
|
187
|
-
topic: t.primaryFile ? "Working on " + t.primaryFile.split("/").pop() : "Segment",
|
|
188
|
-
priority: t.errorDensity > 2 ? "high" as const : "normal" as const,
|
|
189
|
-
confidence: 0.6,
|
|
190
|
-
}));
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
const chunks = chunkLlmMessages(llmMessages, boundaries, pc);
|
|
194
|
-
chunkCount = chunks.length;
|
|
195
|
-
notify("Chunked: " + chunkCount + " chunks", "info");
|
|
196
|
-
|
|
197
|
-
const batches = createBatches(chunks, pc.batchMaxTokens);
|
|
198
|
-
const totalBatches = batches.length;
|
|
199
|
-
if (!autoTriggered) showProgressOverlay(ctx, { phase: 3, phaseName: "Synthesize", detail: "0/" + totalBatches + " batches", model: modelLabel, profile, extraction, totalBatches });
|
|
200
|
-
|
|
201
|
-
const caps = getProviderCaps(summaryModel.provider);
|
|
202
|
-
const concurrency = caps.concurrencyLimit;
|
|
203
|
-
|
|
204
|
-
if (totalBatches <= 1) {
|
|
205
|
-
try {
|
|
206
|
-
summaries.push(...await summarizeBatch(batches[0], extraction, summaryModel, { apiKey: auth.apiKey, headers: auth.headers }, signal));
|
|
207
|
-
} catch (err) {
|
|
208
|
-
summaries.push(...batches[0].map(ch => ({
|
|
209
|
-
topic: ch.topic, startIndex: ch.startIndex, endIndex: ch.endIndex,
|
|
210
|
-
summary: "[Failed] " + ch.messages.map((m: any) => extractText(m.content)).join("\n").slice(0, 300),
|
|
211
|
-
keyDecisions: [] as string[], filesModified: [] as string[], filesRead: [] as string[], priority: ch.priority as ChunkSummary["priority"],
|
|
212
|
-
})));
|
|
213
|
-
}
|
|
214
|
-
} else {
|
|
215
|
-
const results: ChunkSummary[][] = new Array(totalBatches);
|
|
216
|
-
const errors: (Error | null)[] = new Array(totalBatches).fill(null);
|
|
217
|
-
let completed = 0;
|
|
218
|
-
|
|
219
|
-
for (let wave = 0; wave < totalBatches; wave += concurrency) {
|
|
220
|
-
const waveBatches = batches.slice(wave, Math.min(wave + concurrency, totalBatches));
|
|
221
|
-
const wavePromises = waveBatches.map(async (batch, i) => {
|
|
222
|
-
const idx = wave + i;
|
|
223
|
-
try {
|
|
224
|
-
results[idx] = await summarizeBatch(batch, extraction, summaryModel, { apiKey: auth.apiKey, headers: auth.headers }, signal);
|
|
225
|
-
} catch (err) {
|
|
226
|
-
errors[idx] = err instanceof Error ? err : new Error(String(err));
|
|
227
|
-
results[idx] = batch.map(ch => ({
|
|
228
|
-
topic: ch.topic, startIndex: ch.startIndex, endIndex: ch.endIndex,
|
|
229
|
-
summary: "[Failed] " + ch.messages.map((m: any) => extractText(m.content)).join("\n").slice(0, 300),
|
|
230
|
-
keyDecisions: [] as string[], filesModified: [] as string[], filesRead: [] as string[], priority: ch.priority as ChunkSummary["priority"],
|
|
231
|
-
}));
|
|
232
|
-
}
|
|
233
|
-
completed++;
|
|
234
|
-
if (!autoTriggered) showProgressOverlay(ctx, { phase: 3, phaseName: "Synthesize", detail: completed + "/" + totalBatches + " batches", model: modelLabel, profile, extraction, totalBatches, currentBatch: completed });
|
|
235
|
-
});
|
|
236
|
-
await Promise.all(wavePromises);
|
|
237
|
-
}
|
|
238
|
-
for (const r of results) if (r) summaries.push(...r);
|
|
239
|
-
for (let i = 0; i < errors.length; i++) if (errors[i]) notify("Batch " + (i + 1) + " failed: " + errors[i]!.message, "warning");
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
if (!autoTriggered) showProgressOverlay(ctx, { phase: 3, phaseName: "Synthesize", detail: "Assembling...", model: modelLabel, profile, extraction, totalBatches: batches.length });
|
|
243
|
-
let assemblyCalls = 1;
|
|
244
|
-
try {
|
|
245
|
-
const r = await assembleLLM(summaries, extraction, explorationReport, summaryModel, { apiKey: auth.apiKey, headers: auth.headers }, pc.summaryBudgetTokens, prevContext, signal);
|
|
246
|
-
if (r?.startsWith("##")) finalSummary = r; else throw new Error("bad");
|
|
247
|
-
} catch {
|
|
248
|
-
finalSummary = assembleFallback(summaries, extraction); assemblyCalls = 0;
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
method = "eesv";
|
|
252
|
-
llmCalls = explorationRounds + batches.length + assemblyCalls;
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
if (!autoTriggered) showProgressOverlay(ctx, { phase: 4, phaseName: "Verify", detail: "Checking...", model: modelLabel, profile, extraction, explorationRounds });
|
|
256
|
-
const verification = verifySummary(finalSummary, extraction);
|
|
257
|
-
if (!verification.ok) {
|
|
258
|
-
if (verification.score < 85) {
|
|
259
|
-
// Deterministic patch first (zero LLM cost)
|
|
260
|
-
notify("Phase 4 Verify: " + verification.gaps.length + " gap(s), score=" + verification.score + ", applying deterministic patch", "warning");
|
|
261
|
-
finalSummary = patchDeterministic(finalSummary, verification.gaps, extraction);
|
|
262
|
-
// Re-verify after patch — only use LLM patch if still bad
|
|
263
|
-
const recheck = verifySummary(finalSummary, extraction);
|
|
264
|
-
if (!recheck.ok && recheck.score < 75) {
|
|
265
|
-
notify("Phase 4 Verify: deterministic patch insufficient (score=" + recheck.score + "), trying LLM patch", "warning");
|
|
266
|
-
try {
|
|
267
|
-
finalSummary = await patchSummary(finalSummary, recheck.gaps, summaryModel, { apiKey: auth.apiKey, headers: auth.headers }, signal);
|
|
268
|
-
llmCalls++;
|
|
269
|
-
} catch { /* accept deterministic patch as-is */ }
|
|
270
|
-
}
|
|
271
|
-
} else {
|
|
272
|
-
notify("Phase 4 Verify: " + verification.gaps.length + " gap(s), score=" + verification.score + " ≥ 85 — skipping patch", "info");
|
|
273
|
-
}
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
const detModified = extraction.modifiedFiles.map(f => f.path);
|
|
277
|
-
const detRead = extraction.readFiles;
|
|
278
|
-
const estimatedAfter = estimateTokens(finalSummary) + accTokens;
|
|
279
|
-
const tokensSaved = Math.max(0, totalTokens - estimatedAfter);
|
|
280
|
-
|
|
281
|
-
const pipelineInfo = method === "eesv"
|
|
282
|
-
? "EESV: Extract > Explore (" + explorationRounds + "r) > Synthesize (" + (chunkCount || 1) + " chunks) > Verify (" + (verification.ok ? "pass" : verification.gaps.length + " gaps") + ")"
|
|
283
|
-
: method + " (" + (chunkCount || 1) + " chunks, " + llmCalls + " calls)";
|
|
284
|
-
const pipelineMs = Date.now() - pipelineStart;
|
|
285
|
-
const durationStr = pipelineMs < 1000 ? pipelineMs + "ms" : (pipelineMs / 1000).toFixed(1) + "s";
|
|
286
|
-
notify("Done: " + pipelineInfo + " — saved " + (tokensSaved ?? 0).toLocaleString() + "t (" + durationStr + ")", "success");
|
|
287
|
-
|
|
288
|
-
const details: SmartCompactDetails = {
|
|
289
|
-
method: method as SmartCompactDetails["method"],
|
|
290
|
-
chunkCount: chunkCount || 1,
|
|
291
|
-
topics: summaries.length ? summaries.map(s => s.topic) : [method],
|
|
292
|
-
readFiles: detRead, modifiedFiles: detModified,
|
|
293
|
-
totalMessages: toCompact.length, totalTokensSummarized: convTokens,
|
|
294
|
-
llmCalls, profile, backupPath, tokensSaved,
|
|
295
|
-
verified: verification.ok, gaps: verification.gaps,
|
|
296
|
-
explorationRounds, explorationBoundaries: explorationReport?.boundaries.length ?? 0,
|
|
297
|
-
model: modelLabel, qualityScore: verification.score,
|
|
298
|
-
tokensBefore: totalTokens,
|
|
299
|
-
};
|
|
300
|
-
|
|
301
|
-
if (dryRun) {
|
|
302
|
-
notify("DRY RUN (" + method + ", " + profile + ") — " + toCompact.length + " msgs, " + llmCalls + " calls", "info");
|
|
303
|
-
return;
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
pendingRef.value = { summary: finalSummary, firstKeptEntryId: firstKeptId, tokensBefore: totalTokens, details };
|
|
307
|
-
pendingRef.createdAt = Date.now();
|
|
308
|
-
|
|
309
|
-
// ── Save project fingerprint for cross-session context ──
|
|
310
|
-
saveProjectFingerprint(projectId, extraction);
|
|
311
|
-
|
|
312
|
-
appendMetricsLog(sessionId);
|
|
313
|
-
|
|
314
|
-
// ── Damage detection: check if previous compaction caused issues ──
|
|
315
|
-
// This reads post-compaction messages from the current branch to detect regression
|
|
316
|
-
try {
|
|
317
|
-
const postCompactMsgs = msgs.slice(keepFrom).map(e => convertToLlm([e.message])).flat().map((m: any) => m as LlmMessage);
|
|
318
|
-
if (postCompactMsgs.length > 2) {
|
|
319
|
-
// Only detect if there are enough post-compaction messages
|
|
320
|
-
const lastCompaction = branch.filter((e: any) => e.type === "compaction").slice(-1)[0] as any;
|
|
321
|
-
if (lastCompaction?.details) {
|
|
322
|
-
const prevDetails = lastCompaction.details as SmartCompactDetails;
|
|
323
|
-
const prevExtraction = extractStructured(postCompactMsgs.slice(0, Math.min(15, postCompactMsgs.length)), pc);
|
|
324
|
-
const damage = detectDamage(postCompactMsgs.slice(0, Math.min(15, postCompactMsgs.length)), prevExtraction, prevDetails);
|
|
325
|
-
if (damage.damageScore > 0) {
|
|
326
|
-
notify("Previous compaction damage: " + damage.summary, "warning");
|
|
327
|
-
}
|
|
328
|
-
logDamageReport(sessionId, damage, prevDetails);
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
} catch { /* damage detection is best effort */ }
|
|
332
|
-
const ms = getMetricsSummary();
|
|
333
|
-
if (ms.totalCalls > 0) {
|
|
334
|
-
notify("Metrics: " + ms.totalCalls + " calls, " + ms.totalInput + "t in, " + ms.totalOutput + "t out, cache " + Math.round(ms.cacheHitRate * 100) + "%, " + ms.avgLatency + "ms avg", "info");
|
|
335
|
-
}
|
|
336
|
-
if (!autoTriggered) {
|
|
337
|
-
try {
|
|
338
|
-
const timeout = new Promise<void>(resolve => setTimeout(resolve, 5000));
|
|
339
|
-
await Promise.race([showResultScreen(ctx, details, extraction), timeout]);
|
|
340
|
-
} catch {
|
|
341
|
-
notify("Result screen skipped", "info");
|
|
342
|
-
}
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
if (!skipCompact) {
|
|
346
|
-
ctx.compact({
|
|
347
|
-
customInstructions: "Use pre-computed smart summary from /smart-compact",
|
|
348
|
-
onComplete: () => { if (!autoTriggered) ctx.ui.notify("Applied \u2713", "success"); },
|
|
349
|
-
onError: e => { if (!autoTriggered) ctx.ui.notify("Failed: " + e.message, "error"); },
|
|
350
|
-
});
|
|
351
|
-
}
|
|
352
|
-
} finally {
|
|
353
|
-
isRunning.value = false;
|
|
354
|
-
const pipelineMs = Date.now() - pipelineStart;
|
|
355
|
-
if (autoTriggered) {
|
|
356
|
-
ctx.ui.notify("Compaction completed in " + (pipelineMs < 1000 ? pipelineMs + "ms" : (pipelineMs / 1000).toFixed(1) + "s"), "info");
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
}
|
|
360
|
-
|