pi-condense 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/PRUNING.md +26 -6
- package/README.md +9 -3
- package/index.ts +18 -1
- package/package.json +1 -1
- package/src/budget.test.ts +49 -2
- package/src/budget.ts +22 -8
- package/src/chain-compressor.test.ts +419 -7
- package/src/chain-compressor.ts +144 -3
- package/src/chain-range-prune.test.ts +9 -0
- package/src/chain-range-prune.ts +1 -1
- package/src/commands.test.ts +10 -0
- package/src/commands.ts +6 -2
- package/src/diagnostics.test.ts +11 -2
- package/src/diagnostics.ts +1 -0
- package/src/id-collision.integration.test.ts +20 -2
- package/src/indexer.test.ts +211 -2
- package/src/indexer.ts +45 -4
- package/src/query-tool.test.ts +29 -0
- package/src/range-compression.integration.test.ts +395 -2
- package/src/spill.ts +19 -6
- package/src/types.ts +26 -7
package/src/chain-compressor.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { CUSTOM_TYPE_CHAIN } from "./types.js";
|
|
2
|
-
import type { ChainRange, ChainCompressionEntry } from "./types.js";
|
|
2
|
+
import type { ChainRange, ChainCompressionEntry, ToolCallRecord } from "./types.js";
|
|
3
3
|
import type { ToolCallIndexer } from "./indexer.js";
|
|
4
4
|
import type { BlockRefIssuer } from "./block-refs.js";
|
|
5
|
-
import {
|
|
5
|
+
import type { DiagnosticSink } from "./diagnostics.js";
|
|
6
|
+
import { bareToolCallId, occKey, parseOccKey, resultTimestampOf } from "./occurrence-key.js";
|
|
7
|
+
import { resolveRange } from "./chain-range-prune.js";
|
|
8
|
+
import { extractToolResultText } from "./batch-capture.js";
|
|
6
9
|
|
|
7
10
|
/**
|
|
8
11
|
* Grace ids are keyed the same way `recovery-grace.ts` keys them: occurrence
|
|
@@ -65,6 +68,17 @@ export interface ChainCompressorIndexerDeps {
|
|
|
65
68
|
getPerBatchSummariesForToolCallIds(toolCallIds: string[]): string[];
|
|
66
69
|
getToolRefsForToolCallIds(toolCallIds: string[]): string[];
|
|
67
70
|
registerChain(entry: import("./types.js").ChainCompressionEntry): void;
|
|
71
|
+
getIndex(): Map<string, ToolCallRecord>;
|
|
72
|
+
backfillChainRecords(
|
|
73
|
+
records: ToolCallRecord[],
|
|
74
|
+
opts: {
|
|
75
|
+
spillThreshold: number;
|
|
76
|
+
spillPreviewBytes: number;
|
|
77
|
+
sessionDir: string;
|
|
78
|
+
sessionId: string;
|
|
79
|
+
appendEntry: (customType: string, data?: unknown) => void;
|
|
80
|
+
},
|
|
81
|
+
): Promise<import("./types.js").SummaryToolCallRef[]>;
|
|
68
82
|
}
|
|
69
83
|
|
|
70
84
|
export interface CompressEligibleDeps {
|
|
@@ -81,6 +95,85 @@ export interface CompressEligibleDeps {
|
|
|
81
95
|
* the renderer falls back to the per-batch concatenation.
|
|
82
96
|
*/
|
|
83
97
|
fuseRange?: (perBatchSummaryText: string) => Promise<string | null>;
|
|
98
|
+
/** MUST be the same withClosingMessage(...) array chain detection ran on - raw branch messages spuriously fail span resolution on the message_end path (see doc/specs/2026-08-14-uncovered-chain-deterministic-backfill.md). */
|
|
99
|
+
messages: any[];
|
|
100
|
+
diagnostics: Pick<DiagnosticSink, "report">;
|
|
101
|
+
backfill: { spillThreshold: number; spillPreviewBytes: number; sessionDir: string; sessionId: string };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Pure span walk backing the deterministic zero-LLM branch. Excludes
|
|
106
|
+
* protected middles (relocated verbatim at render, never phase-1 stubbed)
|
|
107
|
+
* and already-indexed occurrence keys (retry idempotence).
|
|
108
|
+
*/
|
|
109
|
+
export function extractChainRecords(
|
|
110
|
+
messages: any[],
|
|
111
|
+
chain: Pick<ChainRange, "startUserTimestamp" | "finalAssistantTimestamp" | "protectedToolCallIds">,
|
|
112
|
+
isIndexed: (occurrenceKey: string) => boolean,
|
|
113
|
+
): ToolCallRecord[] {
|
|
114
|
+
const range = resolveRange(chain, messages);
|
|
115
|
+
if (!range) return [];
|
|
116
|
+
const protectedIds = new Set(chain.protectedToolCallIds ?? []);
|
|
117
|
+
const open = new Map<string, { toolName: string; args: unknown }>();
|
|
118
|
+
const records: ToolCallRecord[] = [];
|
|
119
|
+
for (let i = range.startIndex + 1; i < range.endIndex; i++) {
|
|
120
|
+
const msg = messages[i];
|
|
121
|
+
if (msg.role === "assistant" && Array.isArray(msg.content)) {
|
|
122
|
+
for (const block of msg.content) {
|
|
123
|
+
if (block.type === "toolCall") open.set(block.id, { toolName: block.name, args: block.input ?? block.args ?? block.arguments ?? {} });
|
|
124
|
+
}
|
|
125
|
+
} else if (msg.role === "toolResult") {
|
|
126
|
+
const call = open.get(msg.toolCallId);
|
|
127
|
+
if (!call) continue;
|
|
128
|
+
if (protectedIds.has(msg.toolCallId)) continue;
|
|
129
|
+
const resultTimestamp = resultTimestampOf(msg.timestamp);
|
|
130
|
+
if (resultTimestamp === undefined) continue;
|
|
131
|
+
const key = occKey(msg.toolCallId, resultTimestamp);
|
|
132
|
+
if (isIndexed(key)) continue;
|
|
133
|
+
records.push({
|
|
134
|
+
toolCallId: msg.toolCallId,
|
|
135
|
+
toolName: call.toolName,
|
|
136
|
+
args: call.args as Record<string, unknown>,
|
|
137
|
+
resultText: extractToolResultText(msg),
|
|
138
|
+
isError: msg.isError === true,
|
|
139
|
+
turnIndex: -1, // backfilled records have no batch turn; query tool renders "Turn: -1" (pinned)
|
|
140
|
+
timestamp: resultTimestamp,
|
|
141
|
+
resultTimestamp,
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return records;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const EXCERPT_CAP = 200;
|
|
149
|
+
function excerpt(args: unknown): string {
|
|
150
|
+
const s = JSON.stringify(args) ?? "";
|
|
151
|
+
return s.length <= EXCERPT_CAP ? s : s.slice(0, EXCERPT_CAP) + "...";
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Deterministic zero-LLM body. Grammar pinned by tests - change both together. */
|
|
155
|
+
export function buildDeterministicBody(records: ToolCallRecord[], refs: string[]): string {
|
|
156
|
+
const counts = new Map<string, number>();
|
|
157
|
+
for (const r of records) counts.set(r.toolName, (counts.get(r.toolName) ?? 0) + 1);
|
|
158
|
+
const histogram = [...counts.entries()]
|
|
159
|
+
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
160
|
+
.map(([name, n]) => `${name} x${n}`)
|
|
161
|
+
.join(", ");
|
|
162
|
+
const at = (r: ToolCallRecord) => r.resultTimestamp ?? r.timestamp;
|
|
163
|
+
const sorted = [...records].sort((a, b) => at(a) - at(b));
|
|
164
|
+
const first = sorted[0];
|
|
165
|
+
const last = sorted[sorted.length - 1];
|
|
166
|
+
const seconds = Math.round((at(last) - at(first)) / 1000);
|
|
167
|
+
const refsLine = refs.length > 0 ? refs.join(", ") : records.map((r) => r.toolCallId).join(", ");
|
|
168
|
+
return [
|
|
169
|
+
"Deterministic chain compression (no per-batch summary existed for this span; raw outputs recoverable via context_tree_query).",
|
|
170
|
+
`Calls: ${records.length}`,
|
|
171
|
+
`Tools: ${histogram}`,
|
|
172
|
+
`Span: ${new Date(at(first)).toISOString()} -> ${new Date(at(last)).toISOString()} (${seconds}s)`,
|
|
173
|
+
`First: ${first.toolName} ${excerpt(first.args)}`,
|
|
174
|
+
`Last: ${last.toolName} ${excerpt(last.args)}`,
|
|
175
|
+
`Refs: ${refsLine}`,
|
|
176
|
+
].join("\n");
|
|
84
177
|
}
|
|
85
178
|
|
|
86
179
|
export interface CompressEligibleResult {
|
|
@@ -121,7 +214,55 @@ export async function compressEligible(
|
|
|
121
214
|
const lookupKeys = chain.middleOccurrenceKeys?.length ? chain.middleOccurrenceKeys : chain.middleToolCallIds;
|
|
122
215
|
|
|
123
216
|
if (!deps.indexer.hasPerBatchSummaryCoveringAny(lookupKeys)) {
|
|
124
|
-
|
|
217
|
+
// Deterministic zero-LLM fallback (spec 2026-08-14). Fail-closed: any
|
|
218
|
+
// failure below preserves the historical no-summary skip.
|
|
219
|
+
const index = deps.indexer.getIndex();
|
|
220
|
+
const indexed: ToolCallRecord[] = [];
|
|
221
|
+
for (const key of lookupKeys) {
|
|
222
|
+
const r = index.get(key);
|
|
223
|
+
if (r) indexed.push(r);
|
|
224
|
+
}
|
|
225
|
+
const fresh = extractChainRecords(deps.messages, chain, (k) => index.has(k));
|
|
226
|
+
if (fresh.length === 0 && indexed.length === 0) {
|
|
227
|
+
const protectedIds = new Set(chain.protectedToolCallIds ?? []);
|
|
228
|
+
const fullyProtected =
|
|
229
|
+
chain.middleToolCallIds.length > 0 && chain.middleToolCallIds.every((id) => protectedIds.has(id));
|
|
230
|
+
if (!fullyProtected) {
|
|
231
|
+
// Genuine span mismatch - nothing extractable, nothing durable.
|
|
232
|
+
deps.diagnostics.report(
|
|
233
|
+
"backfill-empty",
|
|
234
|
+
String(chain.startUserTimestamp),
|
|
235
|
+
`middles=${chain.middleToolCallIds.length}`,
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
skipped.push({ startUserTimestamp: chain.startUserTimestamp, reason: "no-summary" });
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
try {
|
|
242
|
+
if (fresh.length > 0) {
|
|
243
|
+
await deps.indexer.backfillChainRecords(fresh, { ...deps.backfill, appendEntry: deps.appendEntry });
|
|
244
|
+
}
|
|
245
|
+
} catch {
|
|
246
|
+
skipped.push({ startUserTimestamp: chain.startUserTimestamp, reason: "no-summary" });
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
const allRecords = [...indexed, ...fresh];
|
|
250
|
+
const toolRefs = deps.indexer.getToolRefsForToolCallIds(lookupKeys);
|
|
251
|
+
const entry: ChainCompressionEntry = {
|
|
252
|
+
blockId: deps.blockRefs.issue(),
|
|
253
|
+
startUserTimestamp: chain.startUserTimestamp,
|
|
254
|
+
droppedToolCallIds: chain.middleToolCallIds,
|
|
255
|
+
finalAssistantTimestamp: chain.finalAssistantTimestamp,
|
|
256
|
+
toolRefs,
|
|
257
|
+
compressedAt: deps.now(),
|
|
258
|
+
rangeSummaryText: buildDeterministicBody(allRecords, toolRefs),
|
|
259
|
+
bodySource: "deterministic",
|
|
260
|
+
...(chain.protectedToolCallIds?.length ? { protectedToolCallIds: chain.protectedToolCallIds } : {}),
|
|
261
|
+
...(chain.middleOccurrenceKeys?.length ? { droppedOccurrenceKeys: chain.middleOccurrenceKeys } : {}),
|
|
262
|
+
};
|
|
263
|
+
deps.appendEntry(CUSTOM_TYPE_CHAIN, entry);
|
|
264
|
+
deps.indexer.registerChain(entry);
|
|
265
|
+
compressedEntries.push(entry);
|
|
125
266
|
continue;
|
|
126
267
|
}
|
|
127
268
|
|
|
@@ -627,6 +627,15 @@ describe("resolveRange", () => {
|
|
|
627
627
|
msgs[2] = { ...msgs[2], timestamp: 1200 };
|
|
628
628
|
expect(resolveRange(entry(), msgs)).toEqual({ startIndex: 0, endIndex: 3 });
|
|
629
629
|
});
|
|
630
|
+
|
|
631
|
+
test("resolveRange accepts a minimal timestamp pair (backfill span walk)", () => {
|
|
632
|
+
const messages = [
|
|
633
|
+
{ role: "user", timestamp: 100 },
|
|
634
|
+
{ role: "assistant", timestamp: 200 },
|
|
635
|
+
];
|
|
636
|
+
const range = resolveRange({ startUserTimestamp: 100, finalAssistantTimestamp: 200 }, messages);
|
|
637
|
+
expect(range).toEqual({ startIndex: 0, endIndex: 1 });
|
|
638
|
+
});
|
|
630
639
|
});
|
|
631
640
|
|
|
632
641
|
describe("applyChainCompressions - positional", () => {
|
package/src/chain-range-prune.ts
CHANGED
|
@@ -75,7 +75,7 @@ export function buildSyntheticChainMessage(
|
|
|
75
75
|
* turns (doc/specs/2026-08-12-toolcall-id-collisions.md).
|
|
76
76
|
*/
|
|
77
77
|
export function resolveRange(
|
|
78
|
-
entry: ChainCompressionEntry,
|
|
78
|
+
entry: Pick<ChainCompressionEntry, "startUserTimestamp" | "finalAssistantTimestamp">,
|
|
79
79
|
messages: any[],
|
|
80
80
|
): { startIndex: number; endIndex: number } | null {
|
|
81
81
|
if (entry.finalAssistantTimestamp === null) return null;
|
package/src/commands.test.ts
CHANGED
|
@@ -201,6 +201,16 @@ describe("diagnostic counters on the status line", () => {
|
|
|
201
201
|
"\u2502 prune: ON \u00b7 diag u1",
|
|
202
202
|
);
|
|
203
203
|
});
|
|
204
|
+
|
|
205
|
+
it("appends b<N> for backfill-empty alongside u/m/o", () => {
|
|
206
|
+
const text = pruneStatusText(cfg(true), undefined, {
|
|
207
|
+
"unresolved-range": 2,
|
|
208
|
+
"range-id-mismatch": 0,
|
|
209
|
+
"orphan-sweep": 1,
|
|
210
|
+
"backfill-empty": 2,
|
|
211
|
+
});
|
|
212
|
+
expect(text).toBe("prune: ON \u00b7 diag u2/o1/b2");
|
|
213
|
+
});
|
|
204
214
|
});
|
|
205
215
|
|
|
206
216
|
describe("context metrics suffix on the status line", () => {
|
package/src/commands.ts
CHANGED
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
} from "./types.js";
|
|
25
25
|
import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
|
|
26
26
|
import { saveConfig } from "./config.js";
|
|
27
|
+
import { MAX_BUDGET_WINDOW } from "./budget.js";
|
|
27
28
|
import { formatTokens, formatCost, formatCharProgress, formatCompactCount } from "./stats.js";
|
|
28
29
|
import { Container, Text, SettingsList, type SettingItem } from "@earendil-works/pi-tui";
|
|
29
30
|
import { DynamicBorder, getSettingsListTheme } from "@earendil-works/pi-coding-agent";
|
|
@@ -71,6 +72,7 @@ export function pruneStatusText(
|
|
|
71
72
|
diagnostics["unresolved-range"] ? `u${diagnostics["unresolved-range"]}` : "",
|
|
72
73
|
diagnostics["range-id-mismatch"] ? `m${diagnostics["range-id-mismatch"]}` : "",
|
|
73
74
|
diagnostics["orphan-sweep"] ? `o${diagnostics["orphan-sweep"]}` : "",
|
|
75
|
+
diagnostics["backfill-empty"] ? `b${diagnostics["backfill-empty"]}` : "",
|
|
74
76
|
].filter(Boolean)
|
|
75
77
|
: [];
|
|
76
78
|
const suffix = diag.length > 0 ? ` \u00b7 diag ${diag.join("/")}` : "";
|
|
@@ -234,10 +236,12 @@ function maxTimeoutDescription(config: ContextPruneConfig): string {
|
|
|
234
236
|
}
|
|
235
237
|
|
|
236
238
|
function autoBudgetThresholdDescription(config: ContextPruneConfig): string {
|
|
239
|
+
const cap = `${MAX_BUDGET_WINDOW / 1000}k`;
|
|
237
240
|
if (config.autoBudgetThreshold == null) {
|
|
238
|
-
return `Token-budget auto-flush: force a prune when context usage reaches this share of the window, regardless of prune-on mode. Currently off. Pick a percentage to enable.`;
|
|
241
|
+
return `Token-budget auto-flush: force a prune when context usage reaches this share of the window (or ${cap} tokens, whichever comes first), regardless of prune-on mode. Currently off. Pick a percentage to enable.`;
|
|
239
242
|
}
|
|
240
|
-
|
|
243
|
+
const pct = Math.round(config.autoBudgetThreshold * 100);
|
|
244
|
+
return `Token-budget auto-flush: force a prune when context usage reaches ${pct}% of the window or ${cap} tokens, whichever comes first, regardless of prune-on mode. The ${cap} ceiling keeps this reachable on huge-window models. Set to Off to disable.`;
|
|
241
245
|
}
|
|
242
246
|
|
|
243
247
|
function protectedToolsDisplay(list: string[]): string {
|
package/src/diagnostics.test.ts
CHANGED
|
@@ -46,7 +46,7 @@ describe("DiagnosticSink", () => {
|
|
|
46
46
|
|
|
47
47
|
test("counts are per kind and start at zero", () => {
|
|
48
48
|
const { sink } = sinkWithLog();
|
|
49
|
-
expect(sink.counts()).toEqual({ "unresolved-range": 0, "range-id-mismatch": 0, "orphan-sweep": 0 });
|
|
49
|
+
expect(sink.counts()).toEqual({ "unresolved-range": 0, "range-id-mismatch": 0, "orphan-sweep": 0, "backfill-empty": 0 });
|
|
50
50
|
sink.report("orphan-sweep", "a,b", "swept 2");
|
|
51
51
|
expect(sink.counts()["orphan-sweep"]).toBe(1);
|
|
52
52
|
});
|
|
@@ -96,7 +96,7 @@ describe("DiagnosticSink", () => {
|
|
|
96
96
|
sink.report("unresolved-range", "b5", "x");
|
|
97
97
|
sink.report("orphan-sweep", "a", "y");
|
|
98
98
|
sink.reset();
|
|
99
|
-
expect(sink.counts()).toEqual({ "unresolved-range": 0, "range-id-mismatch": 0, "orphan-sweep": 0 });
|
|
99
|
+
expect(sink.counts()).toEqual({ "unresolved-range": 0, "range-id-mismatch": 0, "orphan-sweep": 0, "backfill-empty": 0 });
|
|
100
100
|
});
|
|
101
101
|
|
|
102
102
|
test("reset() allows a previously-seen (kind, dedupKey) to report again", () => {
|
|
@@ -112,3 +112,12 @@ describe("DiagnosticSink", () => {
|
|
|
112
112
|
expect(sink.counts()["unresolved-range"]).toBe(1);
|
|
113
113
|
});
|
|
114
114
|
});
|
|
115
|
+
|
|
116
|
+
test("counts backfill-empty like any other kind", () => {
|
|
117
|
+
const appended: Array<{ type: string; data: any }> = [];
|
|
118
|
+
const sink = new DiagnosticSink((type, data) => appended.push({ type, data }));
|
|
119
|
+
sink.report("backfill-empty", "b5", "middleCount=0");
|
|
120
|
+
sink.report("backfill-empty", "b5", "middleCount=0"); // deduped
|
|
121
|
+
expect(sink.counts()["backfill-empty"]).toBe(1);
|
|
122
|
+
expect(appended.length).toBe(1);
|
|
123
|
+
});
|
package/src/diagnostics.ts
CHANGED
|
@@ -89,6 +89,9 @@ const primeIndexer = async (
|
|
|
89
89
|
blockRefs: { issue: () => `b${nextBlock++}` } as any,
|
|
90
90
|
appendEntry: append,
|
|
91
91
|
now: () => 9000,
|
|
92
|
+
messages,
|
|
93
|
+
diagnostics: { report: () => {} },
|
|
94
|
+
backfill: { spillThreshold: 100_000, spillPreviewBytes: 500, sessionDir: "/tmp/unused", sessionId: "s" },
|
|
92
95
|
});
|
|
93
96
|
|
|
94
97
|
return { indexer, appended, refsByToolCallId };
|
|
@@ -151,7 +154,17 @@ describe("id collision, end to end", () => {
|
|
|
151
154
|
expectNoOrphanToolResults(out.messages);
|
|
152
155
|
});
|
|
153
156
|
|
|
154
|
-
|
|
157
|
+
// Regression for the bare-id keying bug (ref #8): registerSummaryBody keyed
|
|
158
|
+
// with `tc.toolCallId` (no resultTimestamp) mismatches hasPerBatchSummaryCoveringAny's
|
|
159
|
+
// occurrence-key lookups, so both chains fall through to the "no per-batch
|
|
160
|
+
// summary covers this span" branch. Pre-2026-08-14 that branch was a permanent
|
|
161
|
+
// no-summary skip (the bug this test used to pin). Since the deterministic
|
|
162
|
+
// backfill fallback (doc/specs/2026-08-14-uncovered-chain-deterministic-backfill.md),
|
|
163
|
+
// that branch instead compresses deterministically from already-indexed
|
|
164
|
+
// records (both chains' tool calls were indexed via addBatch, just not
|
|
165
|
+
// summary-covered under the right key) - so the keying bug can no longer
|
|
166
|
+
// strand a chain through this path. Pin the new correct behavior instead.
|
|
167
|
+
test("live-flush bare-id keying bug: chains compress deterministically instead of stranding", async () => {
|
|
155
168
|
const messages = buildSession();
|
|
156
169
|
// Mirrors the production BUG exactly: `tc.toolCallId` with no resultTimestamp,
|
|
157
170
|
// matching index.ts's pre-fix `batch.toolCalls.map((tc) => tc.toolCallId)`.
|
|
@@ -159,7 +172,12 @@ describe("id collision, end to end", () => {
|
|
|
159
172
|
const out = pruneMessages(messages, indexer, chainConfig);
|
|
160
173
|
|
|
161
174
|
const synthetics = syntheticsOf(out.messages);
|
|
162
|
-
expect(synthetics).toHaveLength(
|
|
175
|
+
expect(synthetics).toHaveLength(2);
|
|
176
|
+
for (const s of synthetics) {
|
|
177
|
+
expect(s.content[0].text).toContain("Deterministic chain compression");
|
|
178
|
+
expect(s.content[0].text).toMatch(/Refs: t\d+/);
|
|
179
|
+
}
|
|
180
|
+
expectNoOrphanToolResults(out.messages);
|
|
163
181
|
});
|
|
164
182
|
|
|
165
183
|
test("re-rendering the same session is deep-equal", async () => {
|
package/src/indexer.test.ts
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { mkdtemp, readFile } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
2
5
|
import { ToolCallIndexer } from "./indexer.js";
|
|
3
|
-
import { CUSTOM_TYPE_INDEX, CUSTOM_TYPE_SUMMARY, CUSTOM_TYPE_DEDUP_ALIAS } from "./types.js";
|
|
4
|
-
import type { CapturedBatch } from "./types.js";
|
|
6
|
+
import { CUSTOM_TYPE_INDEX, CUSTOM_TYPE_SUMMARY, CUSTOM_TYPE_DEDUP_ALIAS, CUSTOM_TYPE_CHAIN } from "./types.js";
|
|
7
|
+
import type { CapturedBatch, ToolCallRecord } from "./types.js";
|
|
8
|
+
import { occKey } from "./occurrence-key.js";
|
|
5
9
|
|
|
6
10
|
const batch = (
|
|
7
11
|
turnIndex: number,
|
|
@@ -334,3 +338,208 @@ describe("session rebuild", () => {
|
|
|
334
338
|
expect(indexer.isSummarized("bash_8")).toBe(true);
|
|
335
339
|
});
|
|
336
340
|
});
|
|
341
|
+
|
|
342
|
+
function backfillOpts(appended: Array<{ type: string; data: any }>, tmpDir: string) {
|
|
343
|
+
return {
|
|
344
|
+
spillThreshold: 100_000,
|
|
345
|
+
spillPreviewBytes: 500,
|
|
346
|
+
sessionDir: tmpDir,
|
|
347
|
+
sessionId: "test-session",
|
|
348
|
+
appendEntry: (type: string, data?: unknown) => appended.push({ type, data }),
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function record(id: string, ts: number, over: Partial<ToolCallRecord> = {}): ToolCallRecord {
|
|
353
|
+
return {
|
|
354
|
+
toolCallId: id,
|
|
355
|
+
toolName: "bash",
|
|
356
|
+
args: { command: "ls" },
|
|
357
|
+
resultText: "out-" + id,
|
|
358
|
+
isError: false,
|
|
359
|
+
turnIndex: -1,
|
|
360
|
+
timestamp: ts,
|
|
361
|
+
resultTimestamp: ts,
|
|
362
|
+
...over,
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
describe("backfillChainRecords", () => {
|
|
367
|
+
test("happy path: indexes records, returns refs, persists one entry", async () => {
|
|
368
|
+
const indexer = new ToolCallIndexer();
|
|
369
|
+
const appended: Array<{ type: string; data: any }> = [];
|
|
370
|
+
const records = [record("a", 1), record("b", 2)];
|
|
371
|
+
|
|
372
|
+
const refs = await indexer.backfillChainRecords(records, backfillOpts(appended, "/tmp/unused"));
|
|
373
|
+
|
|
374
|
+
expect(refs).toHaveLength(2);
|
|
375
|
+
for (const ref of refs) expect(ref.shortId).toMatch(/^t\d+$/);
|
|
376
|
+
|
|
377
|
+
expect(indexer.getRecord("a")?.toolCallId).toBe("a");
|
|
378
|
+
expect(indexer.resolveToolCallId(refs[0].shortId)).toBe(occKey("a", 1));
|
|
379
|
+
|
|
380
|
+
expect(appended).toHaveLength(1);
|
|
381
|
+
expect(appended[0].type).toBe(CUSTOM_TYPE_INDEX);
|
|
382
|
+
expect(appended[0].data.backfilled).toBe(true);
|
|
383
|
+
expect(appended[0].data.refs).toHaveLength(2);
|
|
384
|
+
expect(appended[0].data.toolCalls).toHaveLength(2);
|
|
385
|
+
});
|
|
386
|
+
|
|
387
|
+
test("dedup exclusion: backfilled records never seed contentHashToOriginal", async () => {
|
|
388
|
+
const indexer = new ToolCallIndexer();
|
|
389
|
+
const appended: Array<{ type: string; data: any }> = [];
|
|
390
|
+
await indexer.backfillChainRecords([record("a", 1)], backfillOpts(appended, "/tmp/unused"));
|
|
391
|
+
|
|
392
|
+
expect(indexer.lookupByContent("bash", "out-a")).toBeUndefined();
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
test("append failure aborts atomically: no partial index state", async () => {
|
|
396
|
+
const indexer = new ToolCallIndexer();
|
|
397
|
+
const opts = {
|
|
398
|
+
...backfillOpts([], "/tmp/unused"),
|
|
399
|
+
appendEntry: () => {
|
|
400
|
+
throw new Error("append failed");
|
|
401
|
+
},
|
|
402
|
+
};
|
|
403
|
+
|
|
404
|
+
await expect(indexer.backfillChainRecords([record("a", 1)], opts)).rejects.toThrow();
|
|
405
|
+
|
|
406
|
+
expect(indexer.getIndex().size).toBe(0);
|
|
407
|
+
expect(indexer.getShortRefForToolCallId(occKey("a", 1))).toBeUndefined();
|
|
408
|
+
});
|
|
409
|
+
|
|
410
|
+
test("spill: large result text is written to a sidecar blob", async () => {
|
|
411
|
+
const tmpDir = await mkdtemp(join(tmpdir(), "pi-condense-backfill-"));
|
|
412
|
+
const indexer = new ToolCallIndexer();
|
|
413
|
+
const appended: Array<{ type: string; data: any }> = [];
|
|
414
|
+
const bigText = "x".repeat(200_001);
|
|
415
|
+
const opts = { ...backfillOpts(appended, tmpDir), spillThreshold: 200_000 };
|
|
416
|
+
|
|
417
|
+
await indexer.backfillChainRecords([record("a", 1, { resultText: bigText })], opts);
|
|
418
|
+
|
|
419
|
+
const persistedRecord = appended[0].data.toolCalls[0];
|
|
420
|
+
expect(persistedRecord.spillPath).toBeTruthy();
|
|
421
|
+
expect(persistedRecord.resultText).toBe("");
|
|
422
|
+
expect(persistedRecord.resultPreview).toBeTruthy();
|
|
423
|
+
expect(persistedRecord.contentHash).toBeTruthy();
|
|
424
|
+
|
|
425
|
+
const blobContents = await readFile(persistedRecord.spillPath, "utf-8");
|
|
426
|
+
expect(blobContents).toBe(bigText);
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
test("reconstruction round-trip: backfilled entry replays without seeding dedup", async () => {
|
|
430
|
+
const indexer = new ToolCallIndexer();
|
|
431
|
+
const appended: Array<{ type: string; data: any }> = [];
|
|
432
|
+
const refs = await indexer.backfillChainRecords([record("a", 1)], backfillOpts(appended, "/tmp/unused"));
|
|
433
|
+
|
|
434
|
+
const indexEntry = { type: "custom", customType: CUSTOM_TYPE_INDEX, data: appended[0].data };
|
|
435
|
+
const rebuilt = new ToolCallIndexer();
|
|
436
|
+
const ctx = { sessionManager: { getBranch: () => [indexEntry] } } as any;
|
|
437
|
+
rebuilt.reconstructFromSession(ctx);
|
|
438
|
+
|
|
439
|
+
expect(rebuilt.getRecord("a")?.toolCallId).toBe("a");
|
|
440
|
+
expect(rebuilt.resolveToolCallId(refs[0].shortId)).toBe(occKey("a", 1));
|
|
441
|
+
expect(rebuilt.lookupByContent("bash", "out-a")).toBeUndefined();
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
test("legacy round-trip unchanged: pre-change entries (no backfilled/refs) still seed dedup", () => {
|
|
445
|
+
const indexEntry = {
|
|
446
|
+
type: "custom",
|
|
447
|
+
customType: CUSTOM_TYPE_INDEX,
|
|
448
|
+
data: { toolCalls: [record("a", 1)] },
|
|
449
|
+
};
|
|
450
|
+
const indexer = new ToolCallIndexer();
|
|
451
|
+
const ctx = { sessionManager: { getBranch: () => [indexEntry] } } as any;
|
|
452
|
+
indexer.reconstructFromSession(ctx);
|
|
453
|
+
|
|
454
|
+
expect(indexer.lookupByContent("bash", "out-a")).toBe(occKey("a", 1));
|
|
455
|
+
});
|
|
456
|
+
|
|
457
|
+
test("full legacy session round-trip: pre-change context-prune-chain + context-prune-index entries reconstruct to identical state", () => {
|
|
458
|
+
// Mirrors a real pre-change session: a summarized batch (index entry +
|
|
459
|
+
// per-batch summary entry, both pre-existing shapes) followed by a
|
|
460
|
+
// pre-change chain-compression entry (no bodySource - that field didn't
|
|
461
|
+
// exist yet) recording the range-drop decision over that batch's calls.
|
|
462
|
+
const indexEntry = {
|
|
463
|
+
type: "custom",
|
|
464
|
+
customType: CUSTOM_TYPE_INDEX,
|
|
465
|
+
data: {
|
|
466
|
+
toolCalls: [
|
|
467
|
+
{
|
|
468
|
+
toolCallId: "tc1",
|
|
469
|
+
toolName: "bash",
|
|
470
|
+
args: { cmd: "ls" },
|
|
471
|
+
resultText: "listing",
|
|
472
|
+
isError: false,
|
|
473
|
+
turnIndex: 0,
|
|
474
|
+
timestamp: 100,
|
|
475
|
+
resultTimestamp: 110,
|
|
476
|
+
},
|
|
477
|
+
{
|
|
478
|
+
toolCallId: "tc2",
|
|
479
|
+
toolName: "read",
|
|
480
|
+
args: { path: "f" },
|
|
481
|
+
resultText: "file contents",
|
|
482
|
+
isError: false,
|
|
483
|
+
turnIndex: 0,
|
|
484
|
+
timestamp: 100,
|
|
485
|
+
resultTimestamp: 210,
|
|
486
|
+
},
|
|
487
|
+
],
|
|
488
|
+
},
|
|
489
|
+
};
|
|
490
|
+
const summaryEntry = {
|
|
491
|
+
type: "custom_message",
|
|
492
|
+
customType: CUSTOM_TYPE_SUMMARY,
|
|
493
|
+
content: "summarized tc1 and tc2",
|
|
494
|
+
details: {
|
|
495
|
+
toolCallRefs: [
|
|
496
|
+
{ shortId: "t1", toolCallId: "tc1", resultTimestamp: 110 },
|
|
497
|
+
{ shortId: "t2", toolCallId: "tc2", resultTimestamp: 210 },
|
|
498
|
+
],
|
|
499
|
+
toolNames: ["bash", "read"],
|
|
500
|
+
turnIndex: 0,
|
|
501
|
+
timestamp: 100,
|
|
502
|
+
},
|
|
503
|
+
};
|
|
504
|
+
// Pre-change shape: no `bodySource` field (introduced by the uncovered-
|
|
505
|
+
// chain backfill feature; its absence must preserve existing semantics).
|
|
506
|
+
const chainEntry = {
|
|
507
|
+
type: "custom",
|
|
508
|
+
customType: CUSTOM_TYPE_CHAIN,
|
|
509
|
+
data: {
|
|
510
|
+
blockId: "b1",
|
|
511
|
+
startUserTimestamp: 50,
|
|
512
|
+
droppedToolCallIds: ["tc1", "tc2"],
|
|
513
|
+
droppedOccurrenceKeys: [occKey("tc1", 110), occKey("tc2", 210)],
|
|
514
|
+
finalAssistantTimestamp: 300,
|
|
515
|
+
toolRefs: ["t1", "t2"],
|
|
516
|
+
compressedAt: 12345,
|
|
517
|
+
rangeSummaryText: "fused range summary of tc1+tc2",
|
|
518
|
+
},
|
|
519
|
+
};
|
|
520
|
+
|
|
521
|
+
const indexer = new ToolCallIndexer();
|
|
522
|
+
const ctx = {
|
|
523
|
+
// Deep-clone so the byte-identity assertion below compares against the
|
|
524
|
+
// original fixture, not the same object reference the registry stored.
|
|
525
|
+
sessionManager: { getBranch: () => structuredClone([indexEntry, summaryEntry, chainEntry]) },
|
|
526
|
+
} as any;
|
|
527
|
+
indexer.reconstructFromSession(ctx);
|
|
528
|
+
|
|
529
|
+
// Chain registry populated, entry reconstructs byte-identical to what was persisted.
|
|
530
|
+
const entries = indexer.getChainEntries();
|
|
531
|
+
expect(entries).toHaveLength(1);
|
|
532
|
+
expect(entries[0]).toEqual(chainEntry.data as any);
|
|
533
|
+
expect(entries[0].bodySource).toBeUndefined();
|
|
534
|
+
|
|
535
|
+
// Refs resolve exactly as a pre-change session would have resolved them.
|
|
536
|
+
expect(indexer.getRecord("t1")?.resultText).toBe("listing");
|
|
537
|
+
expect(indexer.getRecord("t2")?.resultText).toBe("file contents");
|
|
538
|
+
expect(indexer.resolveToolCallId("t1")).toBe(occKey("tc1", 110));
|
|
539
|
+
expect(indexer.resolveToolCallId("t2")).toBe(occKey("tc2", 210));
|
|
540
|
+
|
|
541
|
+
// Dedup canonical seeded (non-backfilled legacy index entry -> normal addBatch-equivalent seeding).
|
|
542
|
+
expect(indexer.lookupByContent("bash", "listing")).toBe(occKey("tc1", 110));
|
|
543
|
+
expect(indexer.lookupByContent("read", "file contents")).toBe(occKey("tc2", 210));
|
|
544
|
+
});
|
|
545
|
+
});
|
package/src/indexer.ts
CHANGED
|
@@ -19,6 +19,8 @@ import {
|
|
|
19
19
|
} from "./summary-refs.js";
|
|
20
20
|
import { hashToolResult } from "./content-hash.js";
|
|
21
21
|
import { bareToolCallId, occKey, parseOccKey } from "./occurrence-key.js";
|
|
22
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
23
|
+
import { applySpill, blobDirFor, blobPathFor } from "./spill.js";
|
|
22
24
|
|
|
23
25
|
export class ToolCallIndexer {
|
|
24
26
|
/** occurrence key (`id@resultTimestamp`, or bare id for legacy) -> record */
|
|
@@ -78,17 +80,22 @@ export class ToolCallIndexer {
|
|
|
78
80
|
for (const entry of branch) {
|
|
79
81
|
if (entry.type === "custom" && (entry as any).customType === CUSTOM_TYPE_INDEX) {
|
|
80
82
|
const data = (entry as any).data as IndexEntryData;
|
|
83
|
+
const backfilled = data.backfilled === true;
|
|
81
84
|
if (data && Array.isArray(data.toolCalls)) {
|
|
82
85
|
for (const toolCall of data.toolCalls) {
|
|
83
86
|
const key = this.indexRecord(toolCall);
|
|
84
87
|
// First-seen wins so the contentHashToOriginal map matches what
|
|
85
|
-
// addBatch would have produced at append time.
|
|
86
|
-
|
|
87
|
-
if (!
|
|
88
|
-
|
|
88
|
+
// addBatch would have produced at append time. Backfilled records
|
|
89
|
+
// never seed this map (dedup poison guard - see backfillChainRecords).
|
|
90
|
+
if (!backfilled) {
|
|
91
|
+
const hash = toolCall.contentHash ?? hashToolResult(toolCall.toolName, toolCall.resultText);
|
|
92
|
+
if (!this.contentHashToOriginal.has(hash)) {
|
|
93
|
+
this.contentHashToOriginal.set(hash, key);
|
|
94
|
+
}
|
|
89
95
|
}
|
|
90
96
|
}
|
|
91
97
|
}
|
|
98
|
+
if (backfilled && Array.isArray(data.refs)) this.registerSummaryRefs(data.refs);
|
|
92
99
|
continue;
|
|
93
100
|
}
|
|
94
101
|
|
|
@@ -503,4 +510,38 @@ export class ToolCallIndexer {
|
|
|
503
510
|
|
|
504
511
|
appendEntry(CUSTOM_TYPE_INDEX, { toolCalls: records } as IndexEntryData);
|
|
505
512
|
}
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* Atomic recoverability backfill for an uncovered chain (spec
|
|
516
|
+
* 2026-08-14-uncovered-chain-deterministic-backfill). Append-before-commit:
|
|
517
|
+
* in-memory maps are touched only after the index entry persisted. Records
|
|
518
|
+
* never seed contentHashToOriginal (dedup poison guard). Refs ride the
|
|
519
|
+
* entry so they survive session restart without a summary message.
|
|
520
|
+
*/
|
|
521
|
+
async backfillChainRecords(
|
|
522
|
+
records: ToolCallRecord[],
|
|
523
|
+
opts: {
|
|
524
|
+
spillThreshold: number;
|
|
525
|
+
spillPreviewBytes: number;
|
|
526
|
+
sessionDir: string;
|
|
527
|
+
sessionId: string;
|
|
528
|
+
appendEntry: (customType: string, data?: unknown) => void;
|
|
529
|
+
},
|
|
530
|
+
): Promise<SummaryToolCallRef[]> {
|
|
531
|
+
for (const r of records) {
|
|
532
|
+
if (r.resultText.length < opts.spillThreshold) continue;
|
|
533
|
+
const key = occKey(r.toolCallId, r.resultTimestamp);
|
|
534
|
+
const path = blobPathFor(opts.sessionDir, opts.sessionId, key);
|
|
535
|
+
await mkdir(blobDirFor(opts.sessionDir, opts.sessionId), { recursive: true });
|
|
536
|
+
await writeFile(path, r.resultText, "utf-8"); // throw = abort backfill (fail-closed)
|
|
537
|
+
applySpill(r, path, opts.spillPreviewBytes);
|
|
538
|
+
}
|
|
539
|
+
const calls = records.map((r) => ({ toolCallId: r.toolCallId, resultTimestamp: r.resultTimestamp }));
|
|
540
|
+
const { refs, nextIndex } = buildShortToolCallRefs(calls, this.nextShortAliasNumber);
|
|
541
|
+
this.nextShortAliasNumber = nextIndex; // burned numbers on failure are acceptable (monotonic, opaque)
|
|
542
|
+
opts.appendEntry(CUSTOM_TYPE_INDEX, { toolCalls: records, backfilled: true, refs } satisfies IndexEntryData);
|
|
543
|
+
for (const r of records) this.indexRecord(r);
|
|
544
|
+
this.registerSummaryRefs(refs);
|
|
545
|
+
return refs;
|
|
546
|
+
}
|
|
506
547
|
}
|
package/src/query-tool.test.ts
CHANGED
|
@@ -114,4 +114,33 @@ describe("context_tree_query occurrence handling", () => {
|
|
|
114
114
|
expect(text).not.toContain("undefined");
|
|
115
115
|
expect(text).not.toContain("@legacy");
|
|
116
116
|
});
|
|
117
|
+
|
|
118
|
+
test("backfilled record (turnIndex -1) renders 'Turn: -1' (cosmetic pin, ref #10)", async () => {
|
|
119
|
+
const idx = new ToolCallIndexer();
|
|
120
|
+
const appended: Array<{ type: string; data: any }> = [];
|
|
121
|
+
await idx.backfillChainRecords(
|
|
122
|
+
[
|
|
123
|
+
{
|
|
124
|
+
toolCallId: "bash_bf",
|
|
125
|
+
toolName: "bash",
|
|
126
|
+
args: { command: "ls" },
|
|
127
|
+
resultText: "backfilled-output",
|
|
128
|
+
isError: false,
|
|
129
|
+
turnIndex: -1,
|
|
130
|
+
timestamp: 1,
|
|
131
|
+
resultTimestamp: 1,
|
|
132
|
+
},
|
|
133
|
+
],
|
|
134
|
+
{
|
|
135
|
+
spillThreshold: 100_000,
|
|
136
|
+
spillPreviewBytes: 500,
|
|
137
|
+
sessionDir: "/tmp/unused",
|
|
138
|
+
sessionId: "test-session",
|
|
139
|
+
appendEntry: (type: string, data?: unknown) => appended.push({ type, data }),
|
|
140
|
+
},
|
|
141
|
+
);
|
|
142
|
+
|
|
143
|
+
const text = await runTool(idx, ["bash_bf@1"]);
|
|
144
|
+
expect(text).toContain("Turn: -1");
|
|
145
|
+
});
|
|
117
146
|
});
|