pi-condense 2.7.0 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,11 @@
1
1
  import { CUSTOM_TYPE_CHAIN } from "./types.js";
2
- import type { ChainRange, ChainCompressionEntry } from "./types.js";
2
+ import type { ChainRange, ChainCompressionEntry, ToolCallRecord } from "./types.js";
3
3
  import type { ToolCallIndexer } from "./indexer.js";
4
4
  import type { BlockRefIssuer } from "./block-refs.js";
5
- import { bareToolCallId, parseOccKey } from "./occurrence-key.js";
5
+ import type { DiagnosticSink } from "./diagnostics.js";
6
+ import { bareToolCallId, occKey, parseOccKey, resultTimestampOf } from "./occurrence-key.js";
7
+ import { resolveRange } from "./chain-range-prune.js";
8
+ import { extractToolResultText } from "./batch-capture.js";
6
9
 
7
10
  /**
8
11
  * Grace ids are keyed the same way `recovery-grace.ts` keys them: occurrence
@@ -65,6 +68,17 @@ export interface ChainCompressorIndexerDeps {
65
68
  getPerBatchSummariesForToolCallIds(toolCallIds: string[]): string[];
66
69
  getToolRefsForToolCallIds(toolCallIds: string[]): string[];
67
70
  registerChain(entry: import("./types.js").ChainCompressionEntry): void;
71
+ getIndex(): Map<string, ToolCallRecord>;
72
+ backfillChainRecords(
73
+ records: ToolCallRecord[],
74
+ opts: {
75
+ spillThreshold: number;
76
+ spillPreviewBytes: number;
77
+ sessionDir: string;
78
+ sessionId: string;
79
+ appendEntry: (customType: string, data?: unknown) => void;
80
+ },
81
+ ): Promise<import("./types.js").SummaryToolCallRef[]>;
68
82
  }
69
83
 
70
84
  export interface CompressEligibleDeps {
@@ -81,6 +95,85 @@ export interface CompressEligibleDeps {
81
95
  * the renderer falls back to the per-batch concatenation.
82
96
  */
83
97
  fuseRange?: (perBatchSummaryText: string) => Promise<string | null>;
98
+ /** MUST be the same withClosingMessage(...) array chain detection ran on - raw branch messages spuriously fail span resolution on the message_end path (see doc/specs/2026-08-14-uncovered-chain-deterministic-backfill.md). */
99
+ messages: any[];
100
+ diagnostics: Pick<DiagnosticSink, "report">;
101
+ backfill: { spillThreshold: number; spillPreviewBytes: number; sessionDir: string; sessionId: string };
102
+ }
103
+
104
+ /**
105
+ * Pure span walk backing the deterministic zero-LLM branch. Excludes
106
+ * protected middles (relocated verbatim at render, never phase-1 stubbed)
107
+ * and already-indexed occurrence keys (retry idempotence).
108
+ */
109
+ export function extractChainRecords(
110
+ messages: any[],
111
+ chain: Pick<ChainRange, "startUserTimestamp" | "finalAssistantTimestamp" | "protectedToolCallIds">,
112
+ isIndexed: (occurrenceKey: string) => boolean,
113
+ ): ToolCallRecord[] {
114
+ const range = resolveRange(chain, messages);
115
+ if (!range) return [];
116
+ const protectedIds = new Set(chain.protectedToolCallIds ?? []);
117
+ const open = new Map<string, { toolName: string; args: unknown }>();
118
+ const records: ToolCallRecord[] = [];
119
+ for (let i = range.startIndex + 1; i < range.endIndex; i++) {
120
+ const msg = messages[i];
121
+ if (msg.role === "assistant" && Array.isArray(msg.content)) {
122
+ for (const block of msg.content) {
123
+ if (block.type === "toolCall") open.set(block.id, { toolName: block.name, args: block.input ?? block.args ?? block.arguments ?? {} });
124
+ }
125
+ } else if (msg.role === "toolResult") {
126
+ const call = open.get(msg.toolCallId);
127
+ if (!call) continue;
128
+ if (protectedIds.has(msg.toolCallId)) continue;
129
+ const resultTimestamp = resultTimestampOf(msg.timestamp);
130
+ if (resultTimestamp === undefined) continue;
131
+ const key = occKey(msg.toolCallId, resultTimestamp);
132
+ if (isIndexed(key)) continue;
133
+ records.push({
134
+ toolCallId: msg.toolCallId,
135
+ toolName: call.toolName,
136
+ args: call.args as Record<string, unknown>,
137
+ resultText: extractToolResultText(msg),
138
+ isError: msg.isError === true,
139
+ turnIndex: -1, // backfilled records have no batch turn; query tool renders "Turn: -1" (pinned)
140
+ timestamp: resultTimestamp,
141
+ resultTimestamp,
142
+ });
143
+ }
144
+ }
145
+ return records;
146
+ }
147
+
148
+ const EXCERPT_CAP = 200;
149
+ function excerpt(args: unknown): string {
150
+ const s = JSON.stringify(args) ?? "";
151
+ return s.length <= EXCERPT_CAP ? s : s.slice(0, EXCERPT_CAP) + "...";
152
+ }
153
+
154
+ /** Deterministic zero-LLM body. Grammar pinned by tests - change both together. */
155
+ export function buildDeterministicBody(records: ToolCallRecord[], refs: string[]): string {
156
+ const counts = new Map<string, number>();
157
+ for (const r of records) counts.set(r.toolName, (counts.get(r.toolName) ?? 0) + 1);
158
+ const histogram = [...counts.entries()]
159
+ .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
160
+ .map(([name, n]) => `${name} x${n}`)
161
+ .join(", ");
162
+ const at = (r: ToolCallRecord) => r.resultTimestamp ?? r.timestamp;
163
+ const sorted = [...records].sort((a, b) => at(a) - at(b));
164
+ const first = sorted[0];
165
+ const last = sorted[sorted.length - 1];
166
+ const seconds = Math.round((at(last) - at(first)) / 1000);
167
+ const refsLine = refs.length > 0 ? refs.join(", ") : records.map((r) => r.toolCallId).join(", ");
168
+ return [
169
+ "Deterministic chain compression (no per-batch summary existed for this span; raw outputs recoverable via context_tree_query).",
170
+ `Calls: ${records.length}`,
171
+ `Tools: ${histogram}`,
172
+ `Span: ${new Date(at(first)).toISOString()} -> ${new Date(at(last)).toISOString()} (${seconds}s)`,
173
+ `First: ${first.toolName} ${excerpt(first.args)}`,
174
+ `Last: ${last.toolName} ${excerpt(last.args)}`,
175
+ `Refs: ${refsLine}`,
176
+ ].join("\n");
84
177
  }
85
178
 
86
179
  export interface CompressEligibleResult {
@@ -121,7 +214,55 @@ export async function compressEligible(
121
214
  const lookupKeys = chain.middleOccurrenceKeys?.length ? chain.middleOccurrenceKeys : chain.middleToolCallIds;
122
215
 
123
216
  if (!deps.indexer.hasPerBatchSummaryCoveringAny(lookupKeys)) {
124
- skipped.push({ startUserTimestamp: chain.startUserTimestamp, reason: "no-summary" });
217
+ // Deterministic zero-LLM fallback (spec 2026-08-14). Fail-closed: any
218
+ // failure below preserves the historical no-summary skip.
219
+ const index = deps.indexer.getIndex();
220
+ const indexed: ToolCallRecord[] = [];
221
+ for (const key of lookupKeys) {
222
+ const r = index.get(key);
223
+ if (r) indexed.push(r);
224
+ }
225
+ const fresh = extractChainRecords(deps.messages, chain, (k) => index.has(k));
226
+ if (fresh.length === 0 && indexed.length === 0) {
227
+ const protectedIds = new Set(chain.protectedToolCallIds ?? []);
228
+ const fullyProtected =
229
+ chain.middleToolCallIds.length > 0 && chain.middleToolCallIds.every((id) => protectedIds.has(id));
230
+ if (!fullyProtected) {
231
+ // Genuine span mismatch - nothing extractable, nothing durable.
232
+ deps.diagnostics.report(
233
+ "backfill-empty",
234
+ String(chain.startUserTimestamp),
235
+ `middles=${chain.middleToolCallIds.length}`,
236
+ );
237
+ }
238
+ skipped.push({ startUserTimestamp: chain.startUserTimestamp, reason: "no-summary" });
239
+ continue;
240
+ }
241
+ try {
242
+ if (fresh.length > 0) {
243
+ await deps.indexer.backfillChainRecords(fresh, { ...deps.backfill, appendEntry: deps.appendEntry });
244
+ }
245
+ } catch {
246
+ skipped.push({ startUserTimestamp: chain.startUserTimestamp, reason: "no-summary" });
247
+ continue;
248
+ }
249
+ const allRecords = [...indexed, ...fresh];
250
+ const toolRefs = deps.indexer.getToolRefsForToolCallIds(lookupKeys);
251
+ const entry: ChainCompressionEntry = {
252
+ blockId: deps.blockRefs.issue(),
253
+ startUserTimestamp: chain.startUserTimestamp,
254
+ droppedToolCallIds: chain.middleToolCallIds,
255
+ finalAssistantTimestamp: chain.finalAssistantTimestamp,
256
+ toolRefs,
257
+ compressedAt: deps.now(),
258
+ rangeSummaryText: buildDeterministicBody(allRecords, toolRefs),
259
+ bodySource: "deterministic",
260
+ ...(chain.protectedToolCallIds?.length ? { protectedToolCallIds: chain.protectedToolCallIds } : {}),
261
+ ...(chain.middleOccurrenceKeys?.length ? { droppedOccurrenceKeys: chain.middleOccurrenceKeys } : {}),
262
+ };
263
+ deps.appendEntry(CUSTOM_TYPE_CHAIN, entry);
264
+ deps.indexer.registerChain(entry);
265
+ compressedEntries.push(entry);
125
266
  continue;
126
267
  }
127
268
 
@@ -627,6 +627,15 @@ describe("resolveRange", () => {
627
627
  msgs[2] = { ...msgs[2], timestamp: 1200 };
628
628
  expect(resolveRange(entry(), msgs)).toEqual({ startIndex: 0, endIndex: 3 });
629
629
  });
630
+
631
+ test("resolveRange accepts a minimal timestamp pair (backfill span walk)", () => {
632
+ const messages = [
633
+ { role: "user", timestamp: 100 },
634
+ { role: "assistant", timestamp: 200 },
635
+ ];
636
+ const range = resolveRange({ startUserTimestamp: 100, finalAssistantTimestamp: 200 }, messages);
637
+ expect(range).toEqual({ startIndex: 0, endIndex: 1 });
638
+ });
630
639
  });
631
640
 
632
641
  describe("applyChainCompressions - positional", () => {
@@ -75,7 +75,7 @@ export function buildSyntheticChainMessage(
75
75
  * turns (doc/specs/2026-08-12-toolcall-id-collisions.md).
76
76
  */
77
77
  export function resolveRange(
78
- entry: ChainCompressionEntry,
78
+ entry: Pick<ChainCompressionEntry, "startUserTimestamp" | "finalAssistantTimestamp">,
79
79
  messages: any[],
80
80
  ): { startIndex: number; endIndex: number } | null {
81
81
  if (entry.finalAssistantTimestamp === null) return null;
@@ -201,6 +201,16 @@ describe("diagnostic counters on the status line", () => {
201
201
  "\u2502 prune: ON \u00b7 diag u1",
202
202
  );
203
203
  });
204
+
205
+ it("appends b<N> for backfill-empty alongside u/m/o", () => {
206
+ const text = pruneStatusText(cfg(true), undefined, {
207
+ "unresolved-range": 2,
208
+ "range-id-mismatch": 0,
209
+ "orphan-sweep": 1,
210
+ "backfill-empty": 2,
211
+ });
212
+ expect(text).toBe("prune: ON \u00b7 diag u2/o1/b2");
213
+ });
204
214
  });
205
215
 
206
216
  describe("context metrics suffix on the status line", () => {
package/src/commands.ts CHANGED
@@ -24,6 +24,7 @@ import {
24
24
  } from "./types.js";
25
25
  import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
26
26
  import { saveConfig } from "./config.js";
27
+ import { MAX_BUDGET_WINDOW } from "./budget.js";
27
28
  import { formatTokens, formatCost, formatCharProgress, formatCompactCount } from "./stats.js";
28
29
  import { Container, Text, SettingsList, type SettingItem } from "@earendil-works/pi-tui";
29
30
  import { DynamicBorder, getSettingsListTheme } from "@earendil-works/pi-coding-agent";
@@ -71,6 +72,7 @@ export function pruneStatusText(
71
72
  diagnostics["unresolved-range"] ? `u${diagnostics["unresolved-range"]}` : "",
72
73
  diagnostics["range-id-mismatch"] ? `m${diagnostics["range-id-mismatch"]}` : "",
73
74
  diagnostics["orphan-sweep"] ? `o${diagnostics["orphan-sweep"]}` : "",
75
+ diagnostics["backfill-empty"] ? `b${diagnostics["backfill-empty"]}` : "",
74
76
  ].filter(Boolean)
75
77
  : [];
76
78
  const suffix = diag.length > 0 ? ` \u00b7 diag ${diag.join("/")}` : "";
@@ -234,10 +236,12 @@ function maxTimeoutDescription(config: ContextPruneConfig): string {
234
236
  }
235
237
 
236
238
  function autoBudgetThresholdDescription(config: ContextPruneConfig): string {
239
+ const cap = `${MAX_BUDGET_WINDOW / 1000}k`;
237
240
  if (config.autoBudgetThreshold == null) {
238
- return `Token-budget auto-flush: force a prune when context usage reaches this share of the window, regardless of prune-on mode. Currently off. Pick a percentage to enable.`;
241
+ return `Token-budget auto-flush: force a prune when context usage reaches this share of the window (or ${cap} tokens, whichever comes first), regardless of prune-on mode. Currently off. Pick a percentage to enable.`;
239
242
  }
240
- return `Token-budget auto-flush: force a prune when context usage reaches ${Math.round(config.autoBudgetThreshold * 100)}% of the window, regardless of prune-on mode. Set to Off to disable.`;
243
+ const pct = Math.round(config.autoBudgetThreshold * 100);
244
+ return `Token-budget auto-flush: force a prune when context usage reaches ${pct}% of the window or ${cap} tokens, whichever comes first, regardless of prune-on mode. The ${cap} ceiling keeps this reachable on huge-window models. Set to Off to disable.`;
241
245
  }
242
246
 
243
247
  function protectedToolsDisplay(list: string[]): string {
@@ -46,7 +46,7 @@ describe("DiagnosticSink", () => {
46
46
 
47
47
  test("counts are per kind and start at zero", () => {
48
48
  const { sink } = sinkWithLog();
49
- expect(sink.counts()).toEqual({ "unresolved-range": 0, "range-id-mismatch": 0, "orphan-sweep": 0 });
49
+ expect(sink.counts()).toEqual({ "unresolved-range": 0, "range-id-mismatch": 0, "orphan-sweep": 0, "backfill-empty": 0 });
50
50
  sink.report("orphan-sweep", "a,b", "swept 2");
51
51
  expect(sink.counts()["orphan-sweep"]).toBe(1);
52
52
  });
@@ -96,7 +96,7 @@ describe("DiagnosticSink", () => {
96
96
  sink.report("unresolved-range", "b5", "x");
97
97
  sink.report("orphan-sweep", "a", "y");
98
98
  sink.reset();
99
- expect(sink.counts()).toEqual({ "unresolved-range": 0, "range-id-mismatch": 0, "orphan-sweep": 0 });
99
+ expect(sink.counts()).toEqual({ "unresolved-range": 0, "range-id-mismatch": 0, "orphan-sweep": 0, "backfill-empty": 0 });
100
100
  });
101
101
 
102
102
  test("reset() allows a previously-seen (kind, dedupKey) to report again", () => {
@@ -112,3 +112,12 @@ describe("DiagnosticSink", () => {
112
112
  expect(sink.counts()["unresolved-range"]).toBe(1);
113
113
  });
114
114
  });
115
+
116
+ test("counts backfill-empty like any other kind", () => {
117
+ const appended: Array<{ type: string; data: any }> = [];
118
+ const sink = new DiagnosticSink((type, data) => appended.push({ type, data }));
119
+ sink.report("backfill-empty", "b5", "middleCount=0");
120
+ sink.report("backfill-empty", "b5", "middleCount=0"); // deduped
121
+ expect(sink.counts()["backfill-empty"]).toBe(1);
122
+ expect(appended.length).toBe(1);
123
+ });
@@ -13,6 +13,7 @@ export class DiagnosticSink {
13
13
  "unresolved-range": 0,
14
14
  "range-id-mismatch": 0,
15
15
  "orphan-sweep": 0,
16
+ "backfill-empty": 0,
16
17
  };
17
18
 
18
19
  constructor(private readonly appendEntry: (customType: string, data?: unknown) => void) {}
@@ -89,6 +89,9 @@ const primeIndexer = async (
89
89
  blockRefs: { issue: () => `b${nextBlock++}` } as any,
90
90
  appendEntry: append,
91
91
  now: () => 9000,
92
+ messages,
93
+ diagnostics: { report: () => {} },
94
+ backfill: { spillThreshold: 100_000, spillPreviewBytes: 500, sessionDir: "/tmp/unused", sessionId: "s" },
92
95
  });
93
96
 
94
97
  return { indexer, appended, refsByToolCallId };
@@ -151,7 +154,17 @@ describe("id collision, end to end", () => {
151
154
  expectNoOrphanToolResults(out.messages);
152
155
  });
153
156
 
154
- test("live-flush bare-id keying bug: chains are skipped as no-summary, no synthetics emitted", async () => {
157
+ // Regression for the bare-id keying bug (ref #8): registerSummaryBody keyed
158
+ // with `tc.toolCallId` (no resultTimestamp) mismatches hasPerBatchSummaryCoveringAny's
159
+ // occurrence-key lookups, so both chains fall through to the "no per-batch
160
+ // summary covers this span" branch. Pre-2026-08-14 that branch was a permanent
161
+ // no-summary skip (the bug this test used to pin). Since the deterministic
162
+ // backfill fallback (doc/specs/2026-08-14-uncovered-chain-deterministic-backfill.md),
163
+ // that branch instead compresses deterministically from already-indexed
164
+ // records (both chains' tool calls were indexed via addBatch, just not
165
+ // summary-covered under the right key) - so the keying bug can no longer
166
+ // strand a chain through this path. Pin the new correct behavior instead.
167
+ test("live-flush bare-id keying bug: chains compress deterministically instead of stranding", async () => {
155
168
  const messages = buildSession();
156
169
  // Mirrors the production BUG exactly: `tc.toolCallId` with no resultTimestamp,
157
170
  // matching index.ts's pre-fix `batch.toolCalls.map((tc) => tc.toolCallId)`.
@@ -159,7 +172,12 @@ describe("id collision, end to end", () => {
159
172
  const out = pruneMessages(messages, indexer, chainConfig);
160
173
 
161
174
  const synthetics = syntheticsOf(out.messages);
162
- expect(synthetics).toHaveLength(0);
175
+ expect(synthetics).toHaveLength(2);
176
+ for (const s of synthetics) {
177
+ expect(s.content[0].text).toContain("Deterministic chain compression");
178
+ expect(s.content[0].text).toMatch(/Refs: t\d+/);
179
+ }
180
+ expectNoOrphanToolResults(out.messages);
163
181
  });
164
182
 
165
183
  test("re-rendering the same session is deep-equal", async () => {
@@ -1,7 +1,11 @@
1
1
  import { describe, expect, test } from "bun:test";
2
+ import { mkdtemp, readFile } from "node:fs/promises";
3
+ import { tmpdir } from "node:os";
4
+ import { join } from "node:path";
2
5
  import { ToolCallIndexer } from "./indexer.js";
3
- import { CUSTOM_TYPE_INDEX, CUSTOM_TYPE_SUMMARY, CUSTOM_TYPE_DEDUP_ALIAS } from "./types.js";
4
- import type { CapturedBatch } from "./types.js";
6
+ import { CUSTOM_TYPE_INDEX, CUSTOM_TYPE_SUMMARY, CUSTOM_TYPE_DEDUP_ALIAS, CUSTOM_TYPE_CHAIN } from "./types.js";
7
+ import type { CapturedBatch, ToolCallRecord } from "./types.js";
8
+ import { occKey } from "./occurrence-key.js";
5
9
 
6
10
  const batch = (
7
11
  turnIndex: number,
@@ -334,3 +338,208 @@ describe("session rebuild", () => {
334
338
  expect(indexer.isSummarized("bash_8")).toBe(true);
335
339
  });
336
340
  });
341
+
342
+ function backfillOpts(appended: Array<{ type: string; data: any }>, tmpDir: string) {
343
+ return {
344
+ spillThreshold: 100_000,
345
+ spillPreviewBytes: 500,
346
+ sessionDir: tmpDir,
347
+ sessionId: "test-session",
348
+ appendEntry: (type: string, data?: unknown) => appended.push({ type, data }),
349
+ };
350
+ }
351
+
352
+ function record(id: string, ts: number, over: Partial<ToolCallRecord> = {}): ToolCallRecord {
353
+ return {
354
+ toolCallId: id,
355
+ toolName: "bash",
356
+ args: { command: "ls" },
357
+ resultText: "out-" + id,
358
+ isError: false,
359
+ turnIndex: -1,
360
+ timestamp: ts,
361
+ resultTimestamp: ts,
362
+ ...over,
363
+ };
364
+ }
365
+
366
+ describe("backfillChainRecords", () => {
367
+ test("happy path: indexes records, returns refs, persists one entry", async () => {
368
+ const indexer = new ToolCallIndexer();
369
+ const appended: Array<{ type: string; data: any }> = [];
370
+ const records = [record("a", 1), record("b", 2)];
371
+
372
+ const refs = await indexer.backfillChainRecords(records, backfillOpts(appended, "/tmp/unused"));
373
+
374
+ expect(refs).toHaveLength(2);
375
+ for (const ref of refs) expect(ref.shortId).toMatch(/^t\d+$/);
376
+
377
+ expect(indexer.getRecord("a")?.toolCallId).toBe("a");
378
+ expect(indexer.resolveToolCallId(refs[0].shortId)).toBe(occKey("a", 1));
379
+
380
+ expect(appended).toHaveLength(1);
381
+ expect(appended[0].type).toBe(CUSTOM_TYPE_INDEX);
382
+ expect(appended[0].data.backfilled).toBe(true);
383
+ expect(appended[0].data.refs).toHaveLength(2);
384
+ expect(appended[0].data.toolCalls).toHaveLength(2);
385
+ });
386
+
387
+ test("dedup exclusion: backfilled records never seed contentHashToOriginal", async () => {
388
+ const indexer = new ToolCallIndexer();
389
+ const appended: Array<{ type: string; data: any }> = [];
390
+ await indexer.backfillChainRecords([record("a", 1)], backfillOpts(appended, "/tmp/unused"));
391
+
392
+ expect(indexer.lookupByContent("bash", "out-a")).toBeUndefined();
393
+ });
394
+
395
+ test("append failure aborts atomically: no partial index state", async () => {
396
+ const indexer = new ToolCallIndexer();
397
+ const opts = {
398
+ ...backfillOpts([], "/tmp/unused"),
399
+ appendEntry: () => {
400
+ throw new Error("append failed");
401
+ },
402
+ };
403
+
404
+ await expect(indexer.backfillChainRecords([record("a", 1)], opts)).rejects.toThrow();
405
+
406
+ expect(indexer.getIndex().size).toBe(0);
407
+ expect(indexer.getShortRefForToolCallId(occKey("a", 1))).toBeUndefined();
408
+ });
409
+
410
+ test("spill: large result text is written to a sidecar blob", async () => {
411
+ const tmpDir = await mkdtemp(join(tmpdir(), "pi-condense-backfill-"));
412
+ const indexer = new ToolCallIndexer();
413
+ const appended: Array<{ type: string; data: any }> = [];
414
+ const bigText = "x".repeat(200_001);
415
+ const opts = { ...backfillOpts(appended, tmpDir), spillThreshold: 200_000 };
416
+
417
+ await indexer.backfillChainRecords([record("a", 1, { resultText: bigText })], opts);
418
+
419
+ const persistedRecord = appended[0].data.toolCalls[0];
420
+ expect(persistedRecord.spillPath).toBeTruthy();
421
+ expect(persistedRecord.resultText).toBe("");
422
+ expect(persistedRecord.resultPreview).toBeTruthy();
423
+ expect(persistedRecord.contentHash).toBeTruthy();
424
+
425
+ const blobContents = await readFile(persistedRecord.spillPath, "utf-8");
426
+ expect(blobContents).toBe(bigText);
427
+ });
428
+
429
+ test("reconstruction round-trip: backfilled entry replays without seeding dedup", async () => {
430
+ const indexer = new ToolCallIndexer();
431
+ const appended: Array<{ type: string; data: any }> = [];
432
+ const refs = await indexer.backfillChainRecords([record("a", 1)], backfillOpts(appended, "/tmp/unused"));
433
+
434
+ const indexEntry = { type: "custom", customType: CUSTOM_TYPE_INDEX, data: appended[0].data };
435
+ const rebuilt = new ToolCallIndexer();
436
+ const ctx = { sessionManager: { getBranch: () => [indexEntry] } } as any;
437
+ rebuilt.reconstructFromSession(ctx);
438
+
439
+ expect(rebuilt.getRecord("a")?.toolCallId).toBe("a");
440
+ expect(rebuilt.resolveToolCallId(refs[0].shortId)).toBe(occKey("a", 1));
441
+ expect(rebuilt.lookupByContent("bash", "out-a")).toBeUndefined();
442
+ });
443
+
444
+ test("legacy round-trip unchanged: pre-change entries (no backfilled/refs) still seed dedup", () => {
445
+ const indexEntry = {
446
+ type: "custom",
447
+ customType: CUSTOM_TYPE_INDEX,
448
+ data: { toolCalls: [record("a", 1)] },
449
+ };
450
+ const indexer = new ToolCallIndexer();
451
+ const ctx = { sessionManager: { getBranch: () => [indexEntry] } } as any;
452
+ indexer.reconstructFromSession(ctx);
453
+
454
+ expect(indexer.lookupByContent("bash", "out-a")).toBe(occKey("a", 1));
455
+ });
456
+
457
+ test("full legacy session round-trip: pre-change context-prune-chain + context-prune-index entries reconstruct to identical state", () => {
458
+ // Mirrors a real pre-change session: a summarized batch (index entry +
459
+ // per-batch summary entry, both pre-existing shapes) followed by a
460
+ // pre-change chain-compression entry (no bodySource - that field didn't
461
+ // exist yet) recording the range-drop decision over that batch's calls.
462
+ const indexEntry = {
463
+ type: "custom",
464
+ customType: CUSTOM_TYPE_INDEX,
465
+ data: {
466
+ toolCalls: [
467
+ {
468
+ toolCallId: "tc1",
469
+ toolName: "bash",
470
+ args: { cmd: "ls" },
471
+ resultText: "listing",
472
+ isError: false,
473
+ turnIndex: 0,
474
+ timestamp: 100,
475
+ resultTimestamp: 110,
476
+ },
477
+ {
478
+ toolCallId: "tc2",
479
+ toolName: "read",
480
+ args: { path: "f" },
481
+ resultText: "file contents",
482
+ isError: false,
483
+ turnIndex: 0,
484
+ timestamp: 100,
485
+ resultTimestamp: 210,
486
+ },
487
+ ],
488
+ },
489
+ };
490
+ const summaryEntry = {
491
+ type: "custom_message",
492
+ customType: CUSTOM_TYPE_SUMMARY,
493
+ content: "summarized tc1 and tc2",
494
+ details: {
495
+ toolCallRefs: [
496
+ { shortId: "t1", toolCallId: "tc1", resultTimestamp: 110 },
497
+ { shortId: "t2", toolCallId: "tc2", resultTimestamp: 210 },
498
+ ],
499
+ toolNames: ["bash", "read"],
500
+ turnIndex: 0,
501
+ timestamp: 100,
502
+ },
503
+ };
504
+ // Pre-change shape: no `bodySource` field (introduced by the uncovered-
505
+ // chain backfill feature; its absence must preserve existing semantics).
506
+ const chainEntry = {
507
+ type: "custom",
508
+ customType: CUSTOM_TYPE_CHAIN,
509
+ data: {
510
+ blockId: "b1",
511
+ startUserTimestamp: 50,
512
+ droppedToolCallIds: ["tc1", "tc2"],
513
+ droppedOccurrenceKeys: [occKey("tc1", 110), occKey("tc2", 210)],
514
+ finalAssistantTimestamp: 300,
515
+ toolRefs: ["t1", "t2"],
516
+ compressedAt: 12345,
517
+ rangeSummaryText: "fused range summary of tc1+tc2",
518
+ },
519
+ };
520
+
521
+ const indexer = new ToolCallIndexer();
522
+ const ctx = {
523
+ // Deep-clone so the byte-identity assertion below compares against the
524
+ // original fixture, not the same object reference the registry stored.
525
+ sessionManager: { getBranch: () => structuredClone([indexEntry, summaryEntry, chainEntry]) },
526
+ } as any;
527
+ indexer.reconstructFromSession(ctx);
528
+
529
+ // Chain registry populated, entry reconstructs byte-identical to what was persisted.
530
+ const entries = indexer.getChainEntries();
531
+ expect(entries).toHaveLength(1);
532
+ expect(entries[0]).toEqual(chainEntry.data as any);
533
+ expect(entries[0].bodySource).toBeUndefined();
534
+
535
+ // Refs resolve exactly as a pre-change session would have resolved them.
536
+ expect(indexer.getRecord("t1")?.resultText).toBe("listing");
537
+ expect(indexer.getRecord("t2")?.resultText).toBe("file contents");
538
+ expect(indexer.resolveToolCallId("t1")).toBe(occKey("tc1", 110));
539
+ expect(indexer.resolveToolCallId("t2")).toBe(occKey("tc2", 210));
540
+
541
+ // Dedup canonical seeded (non-backfilled legacy index entry -> normal addBatch-equivalent seeding).
542
+ expect(indexer.lookupByContent("bash", "listing")).toBe(occKey("tc1", 110));
543
+ expect(indexer.lookupByContent("read", "file contents")).toBe(occKey("tc2", 210));
544
+ });
545
+ });
package/src/indexer.ts CHANGED
@@ -19,6 +19,8 @@ import {
19
19
  } from "./summary-refs.js";
20
20
  import { hashToolResult } from "./content-hash.js";
21
21
  import { bareToolCallId, occKey, parseOccKey } from "./occurrence-key.js";
22
+ import { mkdir, writeFile } from "node:fs/promises";
23
+ import { applySpill, blobDirFor, blobPathFor } from "./spill.js";
22
24
 
23
25
  export class ToolCallIndexer {
24
26
  /** occurrence key (`id@resultTimestamp`, or bare id for legacy) -> record */
@@ -78,17 +80,22 @@ export class ToolCallIndexer {
78
80
  for (const entry of branch) {
79
81
  if (entry.type === "custom" && (entry as any).customType === CUSTOM_TYPE_INDEX) {
80
82
  const data = (entry as any).data as IndexEntryData;
83
+ const backfilled = data.backfilled === true;
81
84
  if (data && Array.isArray(data.toolCalls)) {
82
85
  for (const toolCall of data.toolCalls) {
83
86
  const key = this.indexRecord(toolCall);
84
87
  // First-seen wins so the contentHashToOriginal map matches what
85
- // addBatch would have produced at append time.
86
- const hash = toolCall.contentHash ?? hashToolResult(toolCall.toolName, toolCall.resultText);
87
- if (!this.contentHashToOriginal.has(hash)) {
88
- this.contentHashToOriginal.set(hash, key);
88
+ // addBatch would have produced at append time. Backfilled records
89
+ // never seed this map (dedup poison guard - see backfillChainRecords).
90
+ if (!backfilled) {
91
+ const hash = toolCall.contentHash ?? hashToolResult(toolCall.toolName, toolCall.resultText);
92
+ if (!this.contentHashToOriginal.has(hash)) {
93
+ this.contentHashToOriginal.set(hash, key);
94
+ }
89
95
  }
90
96
  }
91
97
  }
98
+ if (backfilled && Array.isArray(data.refs)) this.registerSummaryRefs(data.refs);
92
99
  continue;
93
100
  }
94
101
 
@@ -503,4 +510,38 @@ export class ToolCallIndexer {
503
510
 
504
511
  appendEntry(CUSTOM_TYPE_INDEX, { toolCalls: records } as IndexEntryData);
505
512
  }
513
+
514
+ /**
515
+ * Atomic recoverability backfill for an uncovered chain (spec
516
+ * 2026-08-14-uncovered-chain-deterministic-backfill). Append-before-commit:
517
+ * in-memory maps are touched only after the index entry persisted. Records
518
+ * never seed contentHashToOriginal (dedup poison guard). Refs ride the
519
+ * entry so they survive session restart without a summary message.
520
+ */
521
+ async backfillChainRecords(
522
+ records: ToolCallRecord[],
523
+ opts: {
524
+ spillThreshold: number;
525
+ spillPreviewBytes: number;
526
+ sessionDir: string;
527
+ sessionId: string;
528
+ appendEntry: (customType: string, data?: unknown) => void;
529
+ },
530
+ ): Promise<SummaryToolCallRef[]> {
531
+ for (const r of records) {
532
+ if (r.resultText.length < opts.spillThreshold) continue;
533
+ const key = occKey(r.toolCallId, r.resultTimestamp);
534
+ const path = blobPathFor(opts.sessionDir, opts.sessionId, key);
535
+ await mkdir(blobDirFor(opts.sessionDir, opts.sessionId), { recursive: true });
536
+ await writeFile(path, r.resultText, "utf-8"); // throw = abort backfill (fail-closed)
537
+ applySpill(r, path, opts.spillPreviewBytes);
538
+ }
539
+ const calls = records.map((r) => ({ toolCallId: r.toolCallId, resultTimestamp: r.resultTimestamp }));
540
+ const { refs, nextIndex } = buildShortToolCallRefs(calls, this.nextShortAliasNumber);
541
+ this.nextShortAliasNumber = nextIndex; // burned numbers on failure are acceptable (monotonic, opaque)
542
+ opts.appendEntry(CUSTOM_TYPE_INDEX, { toolCalls: records, backfilled: true, refs } satisfies IndexEntryData);
543
+ for (const r of records) this.indexRecord(r);
544
+ this.registerSummaryRefs(refs);
545
+ return refs;
546
+ }
506
547
  }
@@ -114,4 +114,33 @@ describe("context_tree_query occurrence handling", () => {
114
114
  expect(text).not.toContain("undefined");
115
115
  expect(text).not.toContain("@legacy");
116
116
  });
117
+
118
+ test("backfilled record (turnIndex -1) renders 'Turn: -1' (cosmetic pin, ref #10)", async () => {
119
+ const idx = new ToolCallIndexer();
120
+ const appended: Array<{ type: string; data: any }> = [];
121
+ await idx.backfillChainRecords(
122
+ [
123
+ {
124
+ toolCallId: "bash_bf",
125
+ toolName: "bash",
126
+ args: { command: "ls" },
127
+ resultText: "backfilled-output",
128
+ isError: false,
129
+ turnIndex: -1,
130
+ timestamp: 1,
131
+ resultTimestamp: 1,
132
+ },
133
+ ],
134
+ {
135
+ spillThreshold: 100_000,
136
+ spillPreviewBytes: 500,
137
+ sessionDir: "/tmp/unused",
138
+ sessionId: "test-session",
139
+ appendEntry: (type: string, data?: unknown) => appended.push({ type, data }),
140
+ },
141
+ );
142
+
143
+ const text = await runTool(idx, ["bash_bf@1"]);
144
+ expect(text).toContain("Turn: -1");
145
+ });
117
146
  });