ultimate-pi 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/PACKAGING.md +3 -2
- package/.pi/extensions/lib/harness-vcc-settings.ts +50 -0
- package/.pi/extensions/ultimate-pi-vcc.ts +17 -0
- package/.pi/harness/docs/adrs/0030-inhouse-vcc-compaction.md +40 -0
- package/.pi/harness/docs/adrs/README.md +1 -0
- package/.pi/prompts/harness-setup.md +2 -2
- package/.pi/scripts/vendor-pi-vcc-settings.stub.ts +8 -0
- package/.pi/scripts/vendor-sync-pi-vcc.sh +40 -0
- package/.pi/settings.example.json +1 -6
- package/CHANGELOG.md +9 -7
- package/THIRD_PARTY_NOTICES.md +8 -22
- package/package.json +7 -6
- package/vendor/pi-vcc/README.md +215 -0
- package/vendor/pi-vcc/UPSTREAM_PIN.md +12 -0
- package/vendor/pi-vcc/demo.gif +0 -0
- package/vendor/pi-vcc/index.ts +12 -0
- package/vendor/pi-vcc/package.json +26 -0
- package/vendor/pi-vcc/scripts/audit-sessions.ts +88 -0
- package/vendor/pi-vcc/scripts/benchmark-real-sessions.ts +25 -0
- package/vendor/pi-vcc/scripts/compare-before-after.ts +36 -0
- package/vendor/pi-vcc/scripts/dump-branch-output.ts +20 -0
- package/vendor/pi-vcc/src/commands/pi-vcc.ts +36 -0
- package/vendor/pi-vcc/src/commands/vcc-recall.ts +65 -0
- package/vendor/pi-vcc/src/core/brief.ts +381 -0
- package/vendor/pi-vcc/src/core/build-sections.ts +79 -0
- package/vendor/pi-vcc/src/core/content.ts +60 -0
- package/vendor/pi-vcc/src/core/filter-noise.ts +42 -0
- package/vendor/pi-vcc/src/core/format-recall.ts +27 -0
- package/vendor/pi-vcc/src/core/format.ts +49 -0
- package/vendor/pi-vcc/src/core/lineage.ts +26 -0
- package/vendor/pi-vcc/src/core/load-messages.ts +41 -0
- package/vendor/pi-vcc/src/core/normalize.ts +66 -0
- package/vendor/pi-vcc/src/core/recall-scope.ts +14 -0
- package/vendor/pi-vcc/src/core/render-entries.ts +55 -0
- package/vendor/pi-vcc/src/core/report.ts +237 -0
- package/vendor/pi-vcc/src/core/sanitize.ts +5 -0
- package/vendor/pi-vcc/src/core/search-entries.ts +221 -0
- package/vendor/pi-vcc/src/core/settings.ts +8 -0
- package/vendor/pi-vcc/src/core/skill-collapse.ts +35 -0
- package/vendor/pi-vcc/src/core/summarize.ts +157 -0
- package/vendor/pi-vcc/src/core/tool-args.ts +14 -0
- package/vendor/pi-vcc/src/details.ts +7 -0
- package/vendor/pi-vcc/src/extract/commits.ts +69 -0
- package/vendor/pi-vcc/src/extract/files.ts +80 -0
- package/vendor/pi-vcc/src/extract/goals.ts +79 -0
- package/vendor/pi-vcc/src/extract/preferences.ts +55 -0
- package/vendor/pi-vcc/src/hooks/before-compact.ts +314 -0
- package/vendor/pi-vcc/src/sections.ts +12 -0
- package/vendor/pi-vcc/src/tools/recall.ts +109 -0
- package/vendor/pi-vcc/src/types.ts +14 -0
- package/vendor/pi-vcc/tests/before-compact-hook.test.ts +204 -0
- package/vendor/pi-vcc/tests/before-compact.test.ts +145 -0
- package/vendor/pi-vcc/tests/brief.test.ts +206 -0
- package/vendor/pi-vcc/tests/build-sections.test.ts +59 -0
- package/vendor/pi-vcc/tests/compile.test.ts +80 -0
- package/vendor/pi-vcc/tests/content.test.ts +31 -0
- package/vendor/pi-vcc/tests/extract-goals.test.ts +86 -0
- package/vendor/pi-vcc/tests/extract-preferences.test.ts +30 -0
- package/vendor/pi-vcc/tests/filter-noise.test.ts +61 -0
- package/vendor/pi-vcc/tests/fixtures.ts +61 -0
- package/vendor/pi-vcc/tests/format-recall.test.ts +30 -0
- package/vendor/pi-vcc/tests/format.test.ts +62 -0
- package/vendor/pi-vcc/tests/lineage.test.ts +33 -0
- package/vendor/pi-vcc/tests/load-messages.test.ts +51 -0
- package/vendor/pi-vcc/tests/normalize.test.ts +97 -0
- package/vendor/pi-vcc/tests/real-sessions.test.ts +38 -0
- package/vendor/pi-vcc/tests/recall-expand.test.ts +15 -0
- package/vendor/pi-vcc/tests/recall-scope.test.ts +32 -0
- package/vendor/pi-vcc/tests/recall-tool-scope.test.ts +67 -0
- package/vendor/pi-vcc/tests/render-entries.test.ts +62 -0
- package/vendor/pi-vcc/tests/report.test.ts +44 -0
- package/vendor/pi-vcc/tests/sanitize.test.ts +24 -0
- package/vendor/pi-vcc/tests/search-entries.test.ts +144 -0
- package/vendor/pi-vcc/tests/support/load-session.ts +23 -0
- package/vendor/pi-vcc/tests/support/real-sessions.ts +51 -0
- package/.pi/pi-vcc-config.json +0 -4
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { basename, dirname } from "node:path";
|
|
2
|
+
import { compile } from "../src/core/summarize";
|
|
3
|
+
import { normalize } from "../src/core/normalize";
|
|
4
|
+
import { filterNoise } from "../src/core/filter-noise";
|
|
5
|
+
import { renderMessage } from "../src/core/render-entries";
|
|
6
|
+
import { prepareSessionSamples } from "../tests/support/real-sessions";
|
|
7
|
+
import { loadSessionMessages } from "../tests/support/load-session";
|
|
8
|
+
|
|
9
|
+
const SEP = "=".repeat(80);
|
|
10
|
+
const samples = await prepareSessionSamples(10);
|
|
11
|
+
|
|
12
|
+
for (const sample of samples) {
|
|
13
|
+
const loaded = loadSessionMessages(sample.copy);
|
|
14
|
+
const { messages } = loaded;
|
|
15
|
+
|
|
16
|
+
const rawBlocks = normalize(messages);
|
|
17
|
+
const filteredBlocks = filterNoise(rawBlocks);
|
|
18
|
+
const afterText = compile({ messages });
|
|
19
|
+
|
|
20
|
+
const rendered = messages.map((m, i) => renderMessage(m, i));
|
|
21
|
+
const beforeChars = rendered.reduce((s, e) => s + e.summary.length, 0);
|
|
22
|
+
|
|
23
|
+
const project = dirname(sample.source).split("--").filter(Boolean).pop() ?? "unknown";
|
|
24
|
+
|
|
25
|
+
const goalSection = afterText.match(/\[Session Goal\]\n([\s\S]*?)(?=\n\n\[|$)/)?.[1] ?? "(empty)";
|
|
26
|
+
const stateSection = afterText.match(/\[Current State\]\n([\s\S]*?)(?=\n\n\[|$)/)?.[1] ?? "(empty)";
|
|
27
|
+
const doneSection = afterText.match(/\[What Was Done\]\n([\s\S]*?)(?=\n\n\[|$)/)?.[1] ?? "(empty)";
|
|
28
|
+
const problemsSection = afterText.match(/\[Open Problems\]\n([\s\S]*?)(?=\n\n\[|$)/)?.[1] ?? "(empty)";
|
|
29
|
+
const nextSection = afterText.match(/\[Next Best Steps\]\n([\s\S]*?)(?=\n\n\[|$)/)?.[1] ?? "(empty)";
|
|
30
|
+
|
|
31
|
+
const doneLines = doneSection.split("\n").filter(l => l.trim());
|
|
32
|
+
const problemLines = problemsSection.split("\n").filter(l => l.trim());
|
|
33
|
+
|
|
34
|
+
// Detect issues
|
|
35
|
+
const issues: string[] = [];
|
|
36
|
+
|
|
37
|
+
// 1. Goal quality
|
|
38
|
+
const goalLines = goalSection.split("\n").map(l => l.replace(/^- /, "").trim()).filter(Boolean);
|
|
39
|
+
if (goalLines[0] && goalLines[0].length < 5) issues.push(`GOAL_TOO_SHORT: "${goalLines[0]}"`);
|
|
40
|
+
if (goalLines.length === 0) issues.push("GOAL_EMPTY");
|
|
41
|
+
|
|
42
|
+
// 2. Sensitive data in What Was Done
|
|
43
|
+
if (/sshpass|password|secret|token=|api[_-]?key/i.test(doneSection)) {
|
|
44
|
+
issues.push("SENSITIVE_DATA_IN_DONE");
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// 3. Raw code/minified JS in summary
|
|
48
|
+
if (/\{[a-zA-Z$_]+:[a-zA-Z$_]+,[a-zA-Z$_]+:/.test(afterText) || /var [a-zA-Z]+=/.test(afterText)) {
|
|
49
|
+
issues.push("RAW_CODE_LEAK");
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// 4. Open problems count
|
|
53
|
+
if (problemLines.length > 10) issues.push(`PROBLEMS_OVERCOUNT: ${problemLines.length}`);
|
|
54
|
+
|
|
55
|
+
// 5. Next steps empty
|
|
56
|
+
if (nextSection === "(empty)") issues.push("NEXT_STEPS_EMPTY");
|
|
57
|
+
|
|
58
|
+
// 6. What Was Done too verbose
|
|
59
|
+
if (doneLines.length > 15) issues.push(`DONE_TOO_VERBOSE: ${doneLines.length} lines`);
|
|
60
|
+
|
|
61
|
+
// 7. Summary too large (>10K chars)
|
|
62
|
+
if (afterText.length > 10000) issues.push(`SUMMARY_TOO_LARGE: ${afterText.length} chars`);
|
|
63
|
+
|
|
64
|
+
console.log(SEP);
|
|
65
|
+
console.log(`PROJECT: ${project}`);
|
|
66
|
+
console.log(`FILE: ${basename(sample.source)}`);
|
|
67
|
+
console.log(`Size: ${(sample.size / 1024).toFixed(0)}KB | Msgs: ${messages.length} | Blocks raw: ${rawBlocks.length} -> filtered: ${filteredBlocks.length}`);
|
|
68
|
+
console.log(`Before: ${beforeChars} chars | After: ${afterText.length} chars | Ratio: ${(beforeChars / afterText.length).toFixed(1)}x`);
|
|
69
|
+
console.log(`Issues: ${issues.length === 0 ? "NONE" : issues.join(", ")}`);
|
|
70
|
+
console.log("");
|
|
71
|
+
console.log("--- GOAL ---");
|
|
72
|
+
console.log(goalSection.slice(0, 300));
|
|
73
|
+
console.log("");
|
|
74
|
+
console.log("--- CURRENT STATE (first 300c) ---");
|
|
75
|
+
console.log(stateSection.slice(0, 300));
|
|
76
|
+
console.log("");
|
|
77
|
+
console.log("--- WHAT WAS DONE (first 5 lines) ---");
|
|
78
|
+
console.log(doneLines.slice(0, 5).join("\n"));
|
|
79
|
+
console.log(`... (${doneLines.length} total lines)`);
|
|
80
|
+
console.log("");
|
|
81
|
+
console.log("--- OPEN PROBLEMS (first 5 lines) ---");
|
|
82
|
+
console.log(problemLines.slice(0, 5).join("\n"));
|
|
83
|
+
console.log(`... (${problemLines.length} total lines)`);
|
|
84
|
+
console.log("");
|
|
85
|
+
console.log("--- NEXT STEPS ---");
|
|
86
|
+
console.log(nextSection.slice(0, 300));
|
|
87
|
+
console.log("");
|
|
88
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { performance } from "node:perf_hooks";
|
|
2
|
+
import { basename } from "node:path";
|
|
3
|
+
import { buildCompactReport } from "../src/core/report";
|
|
4
|
+
import { prepareSessionSamples } from "../tests/support/real-sessions";
|
|
5
|
+
import { loadSessionMessages } from "../tests/support/load-session";
|
|
6
|
+
|
|
7
|
+
const samples = await prepareSessionSamples(2);
|
|
8
|
+
for (const sample of samples) {
|
|
9
|
+
const loaded = loadSessionMessages(sample.copy);
|
|
10
|
+
const start = performance.now();
|
|
11
|
+
const report = buildCompactReport({ messages: loaded.messages });
|
|
12
|
+
const elapsedMs = performance.now() - start;
|
|
13
|
+
console.log(JSON.stringify({
|
|
14
|
+
sourceFile: basename(sample.source),
|
|
15
|
+
sourceSizeBytes: sample.size,
|
|
16
|
+
copiedToTemp: true,
|
|
17
|
+
loadedMessages: loaded.messageCount,
|
|
18
|
+
skippedMessages: loaded.skippedCount,
|
|
19
|
+
compileMs: Number(elapsedMs.toFixed(2)),
|
|
20
|
+
before: report.before,
|
|
21
|
+
after: report.after,
|
|
22
|
+
compression: report.compression,
|
|
23
|
+
recall: report.recall,
|
|
24
|
+
}, null, 2));
|
|
25
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { basename } from "node:path";
|
|
2
|
+
import { compile } from "../src/core/summarize";
|
|
3
|
+
import { renderMessage } from "../src/core/render-entries";
|
|
4
|
+
import { clip } from "../src/core/content";
|
|
5
|
+
import { prepareSessionSamples } from "../tests/support/real-sessions";
|
|
6
|
+
import { loadSessionMessages } from "../tests/support/load-session";
|
|
7
|
+
|
|
8
|
+
const SEP = "=".repeat(80);
|
|
9
|
+
const samples = await prepareSessionSamples(2);
|
|
10
|
+
|
|
11
|
+
for (const sample of samples) {
|
|
12
|
+
const loaded = loadSessionMessages(sample.copy);
|
|
13
|
+
const { messages } = loaded;
|
|
14
|
+
|
|
15
|
+
const rendered = messages.map((m, i) => renderMessage(m, i));
|
|
16
|
+
const beforeLines = rendered.map(
|
|
17
|
+
(e) => `#${e.index} [${e.role}] ${clip(e.summary, 300)}`,
|
|
18
|
+
);
|
|
19
|
+
const beforeText = beforeLines.join("\n");
|
|
20
|
+
const afterText = compile({ messages });
|
|
21
|
+
|
|
22
|
+
console.log(SEP);
|
|
23
|
+
console.log(`FILE: ${basename(sample.source)}`);
|
|
24
|
+
console.log(`Messages: ${messages.length} | Before chars: ${beforeText.length} | After chars: ${afterText.length}`);
|
|
25
|
+
console.log(`Compression: ${(beforeText.length / afterText.length).toFixed(1)}x`);
|
|
26
|
+
console.log(SEP);
|
|
27
|
+
|
|
28
|
+
console.log("\n--- BEFORE (raw context, first 40 + last 20 entries) ---\n");
|
|
29
|
+
for (const line of beforeLines.slice(0, 40)) console.log(line);
|
|
30
|
+
if (beforeLines.length > 60) console.log(`\n... (${beforeLines.length - 60} entries omitted) ...\n`);
|
|
31
|
+
for (const line of beforeLines.slice(-20)) console.log(line);
|
|
32
|
+
|
|
33
|
+
console.log("\n--- AFTER (pi-vcc compiled summary) ---\n");
|
|
34
|
+
console.log(afterText);
|
|
35
|
+
console.log("\n");
|
|
36
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { basename } from "node:path";
|
|
2
|
+
import { compile } from "../src/core/summarize";
|
|
3
|
+
import { prepareSessionSamples } from "../tests/support/real-sessions";
|
|
4
|
+
import { loadSessionMessages } from "../tests/support/load-session";
|
|
5
|
+
import { writeFileSync, mkdirSync } from "node:fs";
|
|
6
|
+
|
|
7
|
+
const outDir = "/tmp/pi-vcc-compare";
|
|
8
|
+
const branch = process.argv[2] || "unknown";
|
|
9
|
+
mkdirSync(`${outDir}/${branch}`, { recursive: true });
|
|
10
|
+
|
|
11
|
+
const samples = await prepareSessionSamples(5);
|
|
12
|
+
for (const sample of samples) {
|
|
13
|
+
const name = basename(sample.source).slice(0, 30);
|
|
14
|
+
const loaded = loadSessionMessages(sample.copy);
|
|
15
|
+
const summary = compile({ messages: loaded.messages });
|
|
16
|
+
const outFile = `${outDir}/${branch}/${name}.txt`;
|
|
17
|
+
writeFileSync(outFile, summary);
|
|
18
|
+
console.log(`${name} (${loaded.messageCount} msgs) => ${summary.length} chars`);
|
|
19
|
+
}
|
|
20
|
+
console.log(`\nSaved to ${outDir}/${branch}/`);
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
2
|
+
import { getLastCompactionStats, PI_VCC_COMPACT_INSTRUCTION } from "../hooks/before-compact";
|
|
3
|
+
|
|
4
|
+
const formatTokens = (n: number): string => {
|
|
5
|
+
if (n >= 1000) return `${(n / 1000).toFixed(1)}k`;
|
|
6
|
+
return String(n);
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
export const registerPiVccCommand = (pi: ExtensionAPI) => {
|
|
10
|
+
pi.registerCommand("pi-vcc", {
|
|
11
|
+
description: "Compact conversation with pi-vcc structured summary",
|
|
12
|
+
handler: async (_args, ctx) => {
|
|
13
|
+
ctx.compact({
|
|
14
|
+
customInstructions: PI_VCC_COMPACT_INSTRUCTION,
|
|
15
|
+
onComplete: () => {
|
|
16
|
+
const stats = getLastCompactionStats();
|
|
17
|
+
if (stats) {
|
|
18
|
+
ctx.ui.notify(
|
|
19
|
+
`pi-vcc: ${stats.summarized} source entries processed; tail kept ${stats.kept} (~${formatTokens(stats.keptTokensEst)} tok).`,
|
|
20
|
+
"info",
|
|
21
|
+
);
|
|
22
|
+
} else {
|
|
23
|
+
ctx.ui.notify("Compacted with pi-vcc", "info");
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
onError: (err) => {
|
|
27
|
+
if (err.message === "Compaction cancelled" || err.message === "Already compacted") {
|
|
28
|
+
ctx.ui.notify("Nothing to compact", "warning");
|
|
29
|
+
} else {
|
|
30
|
+
ctx.ui.notify(`Compaction failed: ${err.message}`, "error");
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
},
|
|
35
|
+
});
|
|
36
|
+
};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
2
|
+
import { loadAllMessages } from "../core/load-messages";
|
|
3
|
+
import { searchEntries } from "../core/search-entries";
|
|
4
|
+
import { formatRecallOutput } from "../core/format-recall";
|
|
5
|
+
import { getActiveLineageEntryIds } from "../core/lineage";
|
|
6
|
+
import { parseRecallScope } from "../core/recall-scope";
|
|
7
|
+
|
|
8
|
+
const PAGE_SIZE = 5;
|
|
9
|
+
const DEFAULT_RECENT = 25;
|
|
10
|
+
|
|
11
|
+
export const registerVccRecallCommand = (pi: ExtensionAPI) => {
|
|
12
|
+
pi.registerCommand("pi-vcc-recall", {
|
|
13
|
+
description: "Search session history. Defaults to active lineage; add scope:all for off-lineage branches.",
|
|
14
|
+
handler: async (args: string, ctx) => {
|
|
15
|
+
const sessionFile = ctx.sessionManager.getSessionFile();
|
|
16
|
+
if (!sessionFile) {
|
|
17
|
+
ctx.ui.notify("No session file available.", "error");
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const raw = args.trim();
|
|
22
|
+
const parsed = parseRecallScope(raw);
|
|
23
|
+
const lineageEntryIds = parsed.scope === "lineage"
|
|
24
|
+
? getActiveLineageEntryIds(ctx.sessionManager)
|
|
25
|
+
: undefined;
|
|
26
|
+
if (!parsed.text) {
|
|
27
|
+
// No query: show recent
|
|
28
|
+
const { rendered } = loadAllMessages(sessionFile, false, lineageEntryIds);
|
|
29
|
+
const recent = rendered.slice(-DEFAULT_RECENT);
|
|
30
|
+
const output = (parsed.scope === "all" ? "Scope: all\n\n" : "") + formatRecallOutput(recent);
|
|
31
|
+
pi.sendMessage({ customType: "vcc-recall", content: output, display: true }, { triggerTurn: true });
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Parse page:N from args
|
|
36
|
+
const pageMatch = parsed.text.match(/\bpage:(\d+)\b/i);
|
|
37
|
+
const page = pageMatch ? Math.max(1, parseInt(pageMatch[1], 10)) : 1;
|
|
38
|
+
const query = parsed.text.replace(/\bpage:\d+\b/i, "").trim();
|
|
39
|
+
|
|
40
|
+
if (!query) {
|
|
41
|
+
const { rendered } = loadAllMessages(sessionFile, false, lineageEntryIds);
|
|
42
|
+
const recent = rendered.slice(-DEFAULT_RECENT);
|
|
43
|
+
const output = (parsed.scope === "all" ? "Scope: all\n\n" : "") + formatRecallOutput(recent);
|
|
44
|
+
pi.sendMessage({ customType: "vcc-recall", content: output, display: true }, { triggerTurn: true });
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const { rendered, rawMessages } = loadAllMessages(sessionFile, false, lineageEntryIds);
|
|
49
|
+
const allResults = searchEntries(rendered, rawMessages, query);
|
|
50
|
+
|
|
51
|
+
const start = (page - 1) * PAGE_SIZE;
|
|
52
|
+
const pageResults = allResults.slice(start, start + PAGE_SIZE);
|
|
53
|
+
const totalPages = Math.ceil(allResults.length / PAGE_SIZE);
|
|
54
|
+
const scopeSuffix = parsed.scope === "all" ? " (scope: all)" : "";
|
|
55
|
+
const header = totalPages > 1
|
|
56
|
+
? `Page ${page}/${totalPages} (${allResults.length} total matches${scopeSuffix})`
|
|
57
|
+
: `${allResults.length} matches${scopeSuffix}`;
|
|
58
|
+
const footer = page < totalPages
|
|
59
|
+
? `\n--- /pi-vcc-recall ${query}${parsed.scope === "all" ? " scope:all" : ""} page:${page + 1} ---`
|
|
60
|
+
: "";
|
|
61
|
+
const output = formatRecallOutput(pageResults, query, header) + footer;
|
|
62
|
+
pi.sendMessage({ customType: "vcc-recall", content: output, display: true }, { triggerTurn: true });
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
};
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
import type { NormalizedBlock } from "../types";
|
|
2
|
+
import { clip, firstLine } from "./content";
|
|
3
|
+
import { extractPath } from "./tool-args";
|
|
4
|
+
import { collapseSkillText } from "./skill-collapse";
|
|
5
|
+
|
|
6
|
+
const TRUNCATE_USER = 256;
|
|
7
|
+
const TRUNCATE_ASSISTANT = 200;
|
|
8
|
+
|
|
9
|
+
// Strip common self-reflective assistant prefixes that carry no semantic info.
|
|
10
|
+
// Conservative list: only removes the leading filler, preserves the actual content.
|
|
11
|
+
const SELF_TALK_PREFIX_RE =
|
|
12
|
+
/^\s*(?:hmm|wait|actually|oh|okay|ok|well|so)[,.!\s-]+/i;
|
|
13
|
+
|
|
14
|
+
// ── noise filtering ──
|
|
15
|
+
|
|
16
|
+
const isNoiseUser = (text: string): boolean => {
|
|
17
|
+
return !text.trim();
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
// ── truncation ──
|
|
21
|
+
|
|
22
|
+
// Unicode-aware word segmentation via Intl.Segmenter (built-in, zero dependency)
|
|
23
|
+
const segmenter = new Intl.Segmenter(undefined, { granularity: "word" });
|
|
24
|
+
|
|
25
|
+
/** Check if segment is a word (Bun's isWordLike is unreliable for alphanumeric tokens) */
|
|
26
|
+
const isWord = (seg: { segment: string; isWordLike: boolean }): boolean =>
|
|
27
|
+
seg.isWordLike || /[\p{L}\p{N}]/u.test(seg.segment);
|
|
28
|
+
|
|
29
|
+
// Common stop words — don't count toward budget
|
|
30
|
+
const STOP_WORDS = new Set([
|
|
31
|
+
"a", "an", "the", "is", "are", "was", "were", "be", "been", "being",
|
|
32
|
+
"have", "has", "had", "do", "does", "did", "will", "would", "could",
|
|
33
|
+
"should", "may", "might", "shall", "can", "need", "must",
|
|
34
|
+
"to", "of", "in", "for", "on", "with", "at", "by", "from", "as",
|
|
35
|
+
"into", "through", "during", "before", "after", "above", "below",
|
|
36
|
+
"between", "under", "over",
|
|
37
|
+
"and", "but", "or", "nor", "not", "so", "yet", "both", "either",
|
|
38
|
+
"neither", "each", "every", "all", "any", "few", "more", "most",
|
|
39
|
+
"other", "some", "such", "no",
|
|
40
|
+
"that", "this", "these", "those", "it", "its",
|
|
41
|
+
"i", "me", "my", "we", "our", "you", "your", "he", "him", "his",
|
|
42
|
+
"she", "her", "they", "them", "their", "who", "which", "what",
|
|
43
|
+
"if", "then", "than", "when", "where", "how", "just", "also",
|
|
44
|
+
]);
|
|
45
|
+
|
|
46
|
+
const truncateTokens = (text: string, limit: number): string => {
|
|
47
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
48
|
+
let count = 0;
|
|
49
|
+
let lastEnd = 0;
|
|
50
|
+
for (const seg of segmenter.segment(flat)) {
|
|
51
|
+
if (isWord(seg)) {
|
|
52
|
+
if (!STOP_WORDS.has(seg.segment.toLowerCase())) {
|
|
53
|
+
count++;
|
|
54
|
+
if (count > limit) {
|
|
55
|
+
return flat.slice(0, lastEnd).trimEnd() + "...(truncated)";
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
lastEnd = seg.index + seg.segment.length;
|
|
60
|
+
}
|
|
61
|
+
return flat;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
// ── bash command compression ──
|
|
65
|
+
|
|
66
|
+
const BASH_CAP = 120;
|
|
67
|
+
const PIPE_TAIL_RE = /\s*\|\s*(?:head|tail|sort|wc|column|tr|cut|awk|uniq|python3|node|bun)(?:\s[^|]*)?$/;
|
|
68
|
+
|
|
69
|
+
/** Semantic compression: strip cd prefix, pipe tail formatting, cap length */
|
|
70
|
+
const compressBash = (raw: string): string => {
|
|
71
|
+
// Flatten multi-line: take first meaningful line
|
|
72
|
+
let cmd = raw.split("\n").map(l => l.trim()).filter(Boolean)[0] ?? raw;
|
|
73
|
+
// Strip cd <path> && prefix
|
|
74
|
+
cmd = cmd.replace(/^cd\s+\S+\s*&&\s*/, "");
|
|
75
|
+
// Strip pipe tail formatting commands (up to 3 times)
|
|
76
|
+
for (let i = 0; i < 3; i++) {
|
|
77
|
+
const stripped = cmd.replace(PIPE_TAIL_RE, "");
|
|
78
|
+
if (stripped === cmd) break;
|
|
79
|
+
cmd = stripped;
|
|
80
|
+
}
|
|
81
|
+
if (cmd.length > BASH_CAP) {
|
|
82
|
+
return cmd.slice(0, BASH_CAP - 3) + "...";
|
|
83
|
+
}
|
|
84
|
+
return cmd;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
// ── tool summary ──
|
|
88
|
+
|
|
89
|
+
const TOOL_SUMMARY_FIELDS: Record<string, string> = {
|
|
90
|
+
Read: "file_path", Edit: "file_path", Write: "file_path",
|
|
91
|
+
read: "file_path", edit: "file_path", write: "file_path",
|
|
92
|
+
Glob: "pattern", Grep: "pattern",
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
const toolOneLiner = (name: string, args: Record<string, unknown>): string => {
|
|
96
|
+
const field = TOOL_SUMMARY_FIELDS[name];
|
|
97
|
+
if (field && typeof args[field] === "string") {
|
|
98
|
+
return `* ${name} "${args[field] as string}"`;
|
|
99
|
+
}
|
|
100
|
+
const path = extractPath(args);
|
|
101
|
+
if (path) return `* ${name} "${path}"`;
|
|
102
|
+
if (name === "bash" || name === "Bash") {
|
|
103
|
+
const raw = (args.command ?? args.description ?? "") as string;
|
|
104
|
+
const cmd = compressBash(raw);
|
|
105
|
+
return `* ${name} "${cmd}"`;
|
|
106
|
+
}
|
|
107
|
+
if (typeof args.query === "string") {
|
|
108
|
+
return `* ${name} "${clip(args.query as string, 60)}"`;
|
|
109
|
+
}
|
|
110
|
+
return `* ${name}`;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
export interface BriefLine {
|
|
114
|
+
/** Section header like "[user]", "[assistant]", "[tool_error] bash" */
|
|
115
|
+
header: string;
|
|
116
|
+
/** Content lines for this section */
|
|
117
|
+
lines: string[];
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Structured transcript entry for JSON output */
|
|
121
|
+
export interface TranscriptEntry {
|
|
122
|
+
role: "user" | "assistant" | "tool_error";
|
|
123
|
+
text?: string;
|
|
124
|
+
tool?: string;
|
|
125
|
+
cmd?: string;
|
|
126
|
+
ref?: string;
|
|
127
|
+
/** Collapse count when identical tool calls are grouped */
|
|
128
|
+
count?: number;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Build BriefLine sections from NormalizedBlocks.
|
|
133
|
+
*/
|
|
134
|
+
export const buildBriefSections = (blocks: NormalizedBlock[]): BriefLine[] => {
|
|
135
|
+
const sections: BriefLine[] = [];
|
|
136
|
+
let lastHeader = "";
|
|
137
|
+
|
|
138
|
+
const push = (header: string, line: string) => {
|
|
139
|
+
if (header === lastHeader && sections.length > 0) {
|
|
140
|
+
sections[sections.length - 1].lines.push(line);
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
sections.push({ header, lines: [line] });
|
|
144
|
+
lastHeader = header;
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
for (const b of blocks) {
|
|
148
|
+
switch (b.kind) {
|
|
149
|
+
case "user": {
|
|
150
|
+
if (isNoiseUser(b.text)) break;
|
|
151
|
+
const text = truncateTokens(collapseSkillText(b.text), TRUNCATE_USER);
|
|
152
|
+
if (text) {
|
|
153
|
+
const ref = b.sourceIndex != null ? ` (#${b.sourceIndex})` : "";
|
|
154
|
+
push("[user]", text + ref);
|
|
155
|
+
}
|
|
156
|
+
lastHeader = "[user]";
|
|
157
|
+
break;
|
|
158
|
+
}
|
|
159
|
+
case "assistant": {
|
|
160
|
+
let raw = b.text;
|
|
161
|
+
// Strip leading self-talk prefix (up to 2x; assistants sometimes chain "Hmm, actually, ...")
|
|
162
|
+
for (let i = 0; i < 2; i++) {
|
|
163
|
+
const stripped = raw.replace(SELF_TALK_PREFIX_RE, "");
|
|
164
|
+
if (stripped === raw) break;
|
|
165
|
+
raw = stripped;
|
|
166
|
+
}
|
|
167
|
+
const text = truncateTokens(raw, TRUNCATE_ASSISTANT);
|
|
168
|
+
if (text) {
|
|
169
|
+
const ref = b.sourceIndex != null ? ` (#${b.sourceIndex})` : "";
|
|
170
|
+
push("[assistant]", text + ref);
|
|
171
|
+
}
|
|
172
|
+
break;
|
|
173
|
+
}
|
|
174
|
+
case "tool_call": {
|
|
175
|
+
// Skip malformed tool calls from streaming providers (empty name / fragmented args).
|
|
176
|
+
if (!b.name || b.name.trim() === "") break;
|
|
177
|
+
const ref = b.sourceIndex != null ? ` (#${b.sourceIndex})` : "";
|
|
178
|
+
const summary = toolOneLiner(b.name, b.args) + ref;
|
|
179
|
+
push("[assistant]", summary);
|
|
180
|
+
break;
|
|
181
|
+
}
|
|
182
|
+
case "tool_result": {
|
|
183
|
+
if (b.isError) {
|
|
184
|
+
const body = firstLine(b.text, 150);
|
|
185
|
+
// Drop empty/placeholder error bodies — keep the line only if it carries info.
|
|
186
|
+
if (!body || body === "(no output)") break;
|
|
187
|
+
const ref = b.sourceIndex != null ? ` (#${b.sourceIndex})` : "";
|
|
188
|
+
const header = `[tool_error] ${b.name}${ref}`;
|
|
189
|
+
push(header, body);
|
|
190
|
+
lastHeader = header;
|
|
191
|
+
}
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
case "thinking":
|
|
195
|
+
break;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// Collapse consecutive identical tool lines (same text, different #ref)
|
|
200
|
+
for (const sec of sections) {
|
|
201
|
+
if (sec.header !== "[assistant]") continue;
|
|
202
|
+
const out: string[] = [];
|
|
203
|
+
for (const line of sec.lines) {
|
|
204
|
+
if (!line.startsWith("* ")) { out.push(line); continue; }
|
|
205
|
+
const ref = line.match(/\(#(\d+)\)$/)?.[1] ?? "";
|
|
206
|
+
const base = ref ? line.slice(0, -(ref.length + 3)).trimEnd() : line;
|
|
207
|
+
const last = out.length > 0 ? out[out.length - 1] : "";
|
|
208
|
+
const m = last.match(/^(.*) \((#[\d, #]+)\) x(\d+)$/);
|
|
209
|
+
if (m && m[1] === base) {
|
|
210
|
+
out[out.length - 1] = `${base} (${m[2]}, #${ref}) x${parseInt(m[3]) + 1}`;
|
|
211
|
+
} else if (last.match(/\(#\d+\)$/) && last.replace(/\s*\(#\d+\)$/, "") === base) {
|
|
212
|
+
const prevRef = last.match(/\(#(\d+)\)$/)?.[1];
|
|
213
|
+
out[out.length - 1] = `${base} (#${prevRef}, #${ref}) x2`;
|
|
214
|
+
} else {
|
|
215
|
+
out.push(line);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
sec.lines = out;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// Cap tool calls per [assistant] turn — keep tail (latest actions tend to
|
|
222
|
+
// be the deciding edits/writes; head is usually exploration noise).
|
|
223
|
+
const TOOL_CALLS_PER_TURN = 8;
|
|
224
|
+
for (const sec of sections) {
|
|
225
|
+
if (sec.header !== "[assistant]") continue;
|
|
226
|
+
const toolIdxs = sec.lines
|
|
227
|
+
.map((l, i) => (l.startsWith("* ") ? i : -1))
|
|
228
|
+
.filter((i) => i >= 0);
|
|
229
|
+
if (toolIdxs.length <= TOOL_CALLS_PER_TURN) continue;
|
|
230
|
+
const dropCount = toolIdxs.length - TOOL_CALLS_PER_TURN;
|
|
231
|
+
const dropSet = new Set(toolIdxs.slice(0, dropCount));
|
|
232
|
+
const firstKeptToolIdx = toolIdxs[dropCount];
|
|
233
|
+
const next: string[] = [];
|
|
234
|
+
let inserted = false;
|
|
235
|
+
for (let i = 0; i < sec.lines.length; i++) {
|
|
236
|
+
if (dropSet.has(i)) continue;
|
|
237
|
+
if (!inserted && i === firstKeptToolIdx) {
|
|
238
|
+
next.push(`* (${dropCount} earlier tool-call entries omitted)`);
|
|
239
|
+
inserted = true;
|
|
240
|
+
}
|
|
241
|
+
next.push(sec.lines[i]);
|
|
242
|
+
}
|
|
243
|
+
sec.lines = next;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Collapse consecutive identical [tool_error] sections (same tool, same body).
|
|
247
|
+
// E.g. 20 back-to-back `[tool_error] bash (#N) ... Command aborted` become one
|
|
248
|
+
// `[tool_error] bash (#refs...) x20` entry.
|
|
249
|
+
const collapsedErrors: BriefLine[] = [];
|
|
250
|
+
for (const sec of sections) {
|
|
251
|
+
const m = sec.header.match(/^\[tool_error\]\s+(\S+?)(?:\s*\(#(\d+)\))?$/);
|
|
252
|
+
if (!m || sec.lines.length !== 1) {
|
|
253
|
+
collapsedErrors.push(sec);
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
const tool = m[1];
|
|
257
|
+
const ref = m[2];
|
|
258
|
+
const body = sec.lines[0];
|
|
259
|
+
const prev = collapsedErrors[collapsedErrors.length - 1];
|
|
260
|
+
const prevMatch = prev?.header.match(
|
|
261
|
+
/^\[tool_error\]\s+(\S+?)\s*\(((?:#\d+(?:,\s*)?)+)\)(?:\s*x(\d+))?$/,
|
|
262
|
+
);
|
|
263
|
+
if (prev && prevMatch && prevMatch[1] === tool && prev.lines.length === 1 && prev.lines[0] === body) {
|
|
264
|
+
const refs = prevMatch[2] + (ref ? `, #${ref}` : "");
|
|
265
|
+
const count = prevMatch[3] ? parseInt(prevMatch[3]) + 1 : 2;
|
|
266
|
+
prev.header = `[tool_error] ${tool} (${refs}) x${count}`;
|
|
267
|
+
} else {
|
|
268
|
+
collapsedErrors.push(sec);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
sections.length = 0;
|
|
272
|
+
sections.push(...collapsedErrors);
|
|
273
|
+
|
|
274
|
+
return sections;
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Stringify BriefLine sections into text format.
|
|
279
|
+
*/
|
|
280
|
+
export const stringifyBrief = (sections: BriefLine[]): string => {
|
|
281
|
+
|
|
282
|
+
// Emit sections -- suppress blank lines between consecutive tool summaries
|
|
283
|
+
const out: string[] = [];
|
|
284
|
+
for (let i = 0; i < sections.length; i++) {
|
|
285
|
+
const sec = sections[i];
|
|
286
|
+
if (i > 0) {
|
|
287
|
+
const prev = sections[i - 1];
|
|
288
|
+
const prevIsTools = prev.header === "[assistant]" &&
|
|
289
|
+
prev.lines.every((l) => l.startsWith("* "));
|
|
290
|
+
const curIsTools = sec.header === "[assistant]" &&
|
|
291
|
+
sec.lines.every((l) => l.startsWith("* "));
|
|
292
|
+
if (!(prevIsTools && curIsTools)) {
|
|
293
|
+
out.push("");
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
out.push(sec.header);
|
|
297
|
+
for (const line of sec.lines) {
|
|
298
|
+
out.push(line);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
return out.join("\n");
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
/** Parse a text line into a structured TranscriptEntry */
|
|
306
|
+
const parseToolLine = (line: string): { tool: string; cmd?: string; ref?: string; count?: number } | null => {
|
|
307
|
+
// * bash "cmd" (#5)
|
|
308
|
+
// * bash "cmd" (#1, #3) x2
|
|
309
|
+
// * tilth "query" (#7)
|
|
310
|
+
const m = line.match(/^\* (\S+)\s*(?:"([^"]*)")?\s*(?:\((#[\d, #]+)\))?\s*(?:x(\d+))?$/);
|
|
311
|
+
if (!m) return null;
|
|
312
|
+
return {
|
|
313
|
+
tool: m[1],
|
|
314
|
+
cmd: m[2] || undefined,
|
|
315
|
+
ref: m[3] || undefined,
|
|
316
|
+
count: m[4] ? parseInt(m[4]) : undefined,
|
|
317
|
+
};
|
|
318
|
+
};
|
|
319
|
+
|
|
320
|
+
const extractRef = (text: string): { clean: string; ref?: string } => {
|
|
321
|
+
const m = text.match(/\s*\(#(\d+)\)$/);
|
|
322
|
+
if (!m) return { clean: text };
|
|
323
|
+
return { clean: text.slice(0, m.index).trimEnd(), ref: `#${m[1]}` };
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* Convert BriefLine sections to structured TranscriptEntry array for JSON output.
|
|
328
|
+
*/
|
|
329
|
+
export const sectionsToTranscript = (sections: BriefLine[]): TranscriptEntry[] => {
|
|
330
|
+
const entries: TranscriptEntry[] = [];
|
|
331
|
+
|
|
332
|
+
for (const sec of sections) {
|
|
333
|
+
if (sec.header === "[user]") {
|
|
334
|
+
for (const line of sec.lines) {
|
|
335
|
+
const { clean, ref } = extractRef(line);
|
|
336
|
+
entries.push({ role: "user", text: clean, ...(ref && { ref }) });
|
|
337
|
+
}
|
|
338
|
+
} else if (sec.header === "[assistant]") {
|
|
339
|
+
for (const line of sec.lines) {
|
|
340
|
+
if (line.startsWith("* ")) {
|
|
341
|
+
const parsed = parseToolLine(line);
|
|
342
|
+
if (parsed) {
|
|
343
|
+
entries.push({
|
|
344
|
+
role: "assistant",
|
|
345
|
+
tool: parsed.tool,
|
|
346
|
+
...(parsed.cmd && { cmd: parsed.cmd }),
|
|
347
|
+
...(parsed.ref && { ref: parsed.ref }),
|
|
348
|
+
...(parsed.count && { count: parsed.count }),
|
|
349
|
+
});
|
|
350
|
+
} else {
|
|
351
|
+
// Fallback: unparseable tool line
|
|
352
|
+
const { clean, ref } = extractRef(line.slice(2));
|
|
353
|
+
entries.push({ role: "assistant", text: clean, ...(ref && { ref }) });
|
|
354
|
+
}
|
|
355
|
+
} else {
|
|
356
|
+
const { clean, ref } = extractRef(line);
|
|
357
|
+
entries.push({ role: "assistant", text: clean, ...(ref && { ref }) });
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
} else if (sec.header.startsWith("[tool_error]")) {
|
|
361
|
+
// [tool_error] bash (#5)
|
|
362
|
+
const headerMatch = sec.header.match(/^\[tool_error\]\s+(\S+)\s*(?:\(#(\d+)\))?/);
|
|
363
|
+
const tool = headerMatch?.[1] ?? "unknown";
|
|
364
|
+
const ref = headerMatch?.[2] ? `#${headerMatch[2]}` : undefined;
|
|
365
|
+
for (const line of sec.lines) {
|
|
366
|
+
entries.push({
|
|
367
|
+
role: "tool_error",
|
|
368
|
+
tool,
|
|
369
|
+
text: line,
|
|
370
|
+
...(ref && { ref }),
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
return entries;
|
|
377
|
+
};
|
|
378
|
+
|
|
379
|
+
/** Convenience: build sections from blocks and stringify to text */
|
|
380
|
+
export const compileBrief = (blocks: NormalizedBlock[]): string =>
|
|
381
|
+
stringifyBrief(buildBriefSections(blocks));
|