pi-condense 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +73 -0
  2. package/LICENSE +22 -0
  3. package/PRUNING.md +1028 -0
  4. package/README.md +243 -0
  5. package/index.ts +858 -0
  6. package/package.json +56 -0
  7. package/src/batch-capture.ts +226 -0
  8. package/src/block-refs.test.ts +42 -0
  9. package/src/block-refs.ts +16 -0
  10. package/src/budget.test.ts +66 -0
  11. package/src/budget.ts +39 -0
  12. package/src/chain-compressor.test.ts +283 -0
  13. package/src/chain-compressor.ts +132 -0
  14. package/src/chain-detector.test.ts +302 -0
  15. package/src/chain-detector.ts +128 -0
  16. package/src/chain-range-prune.test.ts +522 -0
  17. package/src/chain-range-prune.ts +128 -0
  18. package/src/commands.test.ts +67 -0
  19. package/src/commands.ts +1207 -0
  20. package/src/config.ts +126 -0
  21. package/src/content-hash.ts +35 -0
  22. package/src/error-purge.test.ts +186 -0
  23. package/src/error-purge.ts +71 -0
  24. package/src/frontier.ts +62 -0
  25. package/src/indexer.ts +393 -0
  26. package/src/nested-placeholders.test.ts +82 -0
  27. package/src/nested-placeholders.ts +20 -0
  28. package/src/oversized-spill.integration.test.ts +73 -0
  29. package/src/protected.test.ts +62 -0
  30. package/src/protected.ts +51 -0
  31. package/src/pruner.test.ts +508 -0
  32. package/src/pruner.ts +156 -0
  33. package/src/query-tool.ts +78 -0
  34. package/src/range-compression.integration.test.ts +252 -0
  35. package/src/spill.test.ts +102 -0
  36. package/src/spill.ts +90 -0
  37. package/src/stats.test.ts +114 -0
  38. package/src/stats.ts +190 -0
  39. package/src/summarizer.test.ts +17 -0
  40. package/src/summarizer.ts +262 -0
  41. package/src/summary-refs.ts +61 -0
  42. package/src/thinking-strip.test.ts +175 -0
  43. package/src/thinking-strip.ts +42 -0
  44. package/src/tree-browser.ts +382 -0
  45. package/src/types.ts +764 -0
package/src/indexer.ts ADDED
@@ -0,0 +1,393 @@
1
+ import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
2
+ import type {
3
+ CapturedBatch,
4
+ ChainCompressionEntry,
5
+ DedupAliasEntryData,
6
+ IndexEntryData,
7
+ ToolCallRecord,
8
+ } from "./types.js";
9
+ import {
10
+ CUSTOM_TYPE_CHAIN,
11
+ CUSTOM_TYPE_DEDUP_ALIAS,
12
+ CUSTOM_TYPE_INDEX,
13
+ CUSTOM_TYPE_SUMMARY,
14
+ } from "./types.js";
15
+ import {
16
+ buildShortToolCallRefs,
17
+ normalizeSummaryToolCallRefs,
18
+ type SummaryToolCallRef,
19
+ } from "./summary-refs.js";
20
+ import { hashToolResult } from "./content-hash.js";
21
+
22
+ export class ToolCallIndexer {
23
+ private index = new Map<string, ToolCallRecord>();
24
+ private aliasToToolCallId = new Map<string, string>();
25
+ private toolCallIdToAlias = new Map<string, string>();
26
+ private nextShortAliasNumber = 1;
27
+ /**
28
+ * hash → original toolCallId. Populated as records enter the indexer
29
+ * (`addBatch`) and on `reconstructFromSession`. Drives the pre-flush
30
+ * dedup pass via `lookupByContent`.
31
+ */
32
+ private contentHashToOriginal = new Map<string, string>();
33
+ /**
34
+ * Duplicate toolCallId → original toolCallId. Populated by
35
+ * `registerDuplicate` during the pre-flush dedup pass and rebuilt from
36
+ * CUSTOM_TYPE_DEDUP_ALIAS entries on reconstruction.
37
+ *
38
+ * Both `isSummarized` and `resolveToolCallId` consult this map so
39
+ * `pruneMessages` stub-replaces dup toolResults and `context_tree_query`
40
+ * resolves dup ids to the original record.
41
+ */
42
+ private dedupAliasToOriginal = new Map<string, string>();
43
+ /**
44
+ * Per-batch summary bodies for chain-compression summary text lookup.
45
+ * Each entry maps a set of toolCallIds to the summary's markdown body.
46
+ * Populated from CUSTOM_TYPE_SUMMARY entries at rebuild time and via
47
+ * `registerSummaryBody` after a successful flush.
48
+ */
49
+ private summaryBodies: Array<{ toolCallIds: string[]; text: string }> = [];
50
+ /** Compressed chains, keyed on startUserTimestamp for O(1) dedup checks. */
51
+ private chainRegistry = new Map<number, ChainCompressionEntry>();
52
+
53
+ /**
54
+ * Rebuilds the in-memory index from session history by scanning all
55
+ * custom entries with customType === CUSTOM_TYPE_INDEX.
56
+ */
57
+ reconstructFromSession(ctx: ExtensionContext): void {
58
+ this.index.clear();
59
+ this.aliasToToolCallId.clear();
60
+ this.toolCallIdToAlias.clear();
61
+ this.contentHashToOriginal.clear();
62
+ this.dedupAliasToOriginal.clear();
63
+ this.nextShortAliasNumber = 1;
64
+ this.summaryBodies = [];
65
+ this.chainRegistry.clear();
66
+
67
+ // Two passes so dedup aliases land AFTER the original short refs they
68
+ // need to reuse, regardless of the underlying append order.
69
+ const branch = ctx.sessionManager.getBranch();
70
+ const dedupAliasEntries: DedupAliasEntryData[] = [];
71
+
72
+ for (const entry of branch) {
73
+ if (entry.type === "custom" && (entry as any).customType === CUSTOM_TYPE_INDEX) {
74
+ const data = (entry as any).data as IndexEntryData;
75
+ if (data && Array.isArray(data.toolCalls)) {
76
+ for (const toolCall of data.toolCalls) {
77
+ this.index.set(toolCall.toolCallId, toolCall);
78
+ // First-seen wins so the contentHashToOriginal map matches what
79
+ // addBatch would have produced at append time.
80
+ const hash = toolCall.contentHash ?? hashToolResult(toolCall.toolName, toolCall.resultText);
81
+ if (!this.contentHashToOriginal.has(hash)) {
82
+ this.contentHashToOriginal.set(hash, toolCall.toolCallId);
83
+ }
84
+ }
85
+ }
86
+ continue;
87
+ }
88
+
89
+ if (entry.type === "custom_message" && (entry as any).customType === CUSTOM_TYPE_SUMMARY) {
90
+ const refs = normalizeSummaryToolCallRefs((entry as any).details);
91
+ this.registerSummaryRefs(refs);
92
+ const raw = (entry as any).content;
93
+ const text =
94
+ typeof raw === "string"
95
+ ? raw
96
+ : Array.isArray(raw)
97
+ ? raw
98
+ .filter((c: any) => c.type === "text")
99
+ .map((c: any) => c.text as string)
100
+ .join("\n")
101
+ : "";
102
+ if (text) {
103
+ this.summaryBodies.push({ toolCallIds: refs.map((r) => r.toolCallId), text });
104
+ }
105
+ continue;
106
+ }
107
+
108
+ if (entry.type === "custom" && (entry as any).customType === CUSTOM_TYPE_CHAIN) {
109
+ const data = (entry as any).data as ChainCompressionEntry;
110
+ if (data?.blockId && typeof data.startUserTimestamp === "number") {
111
+ this.chainRegistry.set(data.startUserTimestamp, data);
112
+ }
113
+ continue;
114
+ }
115
+
116
+ if (entry.type === "custom" && (entry as any).customType === CUSTOM_TYPE_DEDUP_ALIAS) {
117
+ const data = (entry as any).data as DedupAliasEntryData;
118
+ if (data?.newToolCallId && data?.originalToolCallId) {
119
+ dedupAliasEntries.push(data);
120
+ }
121
+ }
122
+ }
123
+
124
+ for (const data of dedupAliasEntries) {
125
+ this.dedupAliasToOriginal.set(data.newToolCallId, data.originalToolCallId);
126
+ const originalShortRef = this.toolCallIdToAlias.get(data.originalToolCallId);
127
+ if (originalShortRef) {
128
+ // Keep `getShortRefForToolCallId(dupId)` returning the SAME short ref
129
+ // as the original so pruneMessages emits a consistent `tN` for both.
130
+ this.toolCallIdToAlias.set(data.newToolCallId, originalShortRef);
131
+ }
132
+ }
133
+ }
134
+
135
+ /**
136
+ * Returns true if the given toolCallId has been pruned — either because
137
+ * its full record is in the index, or because it has been registered as
138
+ * an alias of an already-indexed original via the content-hash dedup pass.
139
+ *
140
+ * `pruneMessages` uses this to decide whether to stub-replace a
141
+ * ToolResultMessage; both cases need the same treatment.
142
+ */
143
+ isSummarized(toolCallId: string): boolean {
144
+ return this.index.has(toolCallId) || this.dedupAliasToOriginal.has(toolCallId);
145
+ }
146
+
147
+ /**
148
+ * Returns the full runtime index map.
149
+ */
150
+ getIndex(): Map<string, ToolCallRecord> {
151
+ return this.index;
152
+ }
153
+
154
+ /**
155
+ * Register short aliases for a summary message so future recovery queries can
156
+ * resolve the short ids back to the persisted toolCallIds.
157
+ */
158
+ registerSummaryRefs(refs: SummaryToolCallRef[]): void {
159
+ for (const ref of refs) {
160
+ if (!ref.shortId || !ref.toolCallId) continue;
161
+ if (ref.shortId !== ref.toolCallId) {
162
+ this.aliasToToolCallId.set(ref.shortId, ref.toolCallId);
163
+ this.toolCallIdToAlias.set(ref.toolCallId, ref.shortId);
164
+ }
165
+ const match = /^t(\d+)$/.exec(ref.shortId);
166
+ if (match) {
167
+ this.nextShortAliasNumber = Math.max(this.nextShortAliasNumber, Number(match[1]) + 1);
168
+ }
169
+ }
170
+ }
171
+
172
+ /**
173
+ * Allocates short aliases for a batch's tool calls and registers them in the
174
+ * runtime alias map.
175
+ */
176
+ allocateSummaryRefs(batch: CapturedBatch): SummaryToolCallRef[] {
177
+ const toolCallIds = batch.toolCalls.map((tc) => tc.toolCallId);
178
+ const { refs, nextIndex } = buildShortToolCallRefs(toolCallIds, this.nextShortAliasNumber);
179
+ this.nextShortAliasNumber = nextIndex;
180
+ return refs;
181
+ }
182
+
183
+ /**
184
+ * Resolve a short alias, a duplicate's toolCallId, or a full toolCallId
185
+ * to the canonical toolCallId backing it.
186
+ *
187
+ * Order:
188
+ * 1. Direct hit in `this.index` (canonical id).
189
+ * 2. Dedup alias → underlying original toolCallId.
190
+ * 3. Short-ref (`t3`) → underlying toolCallId.
191
+ *
192
+ * Used by `getRecord`/`lookupToolCalls` so `context_tree_query` returns
193
+ * the original record for both short refs and dedup'd ids.
194
+ */
195
+ resolveToolCallId(toolCallIdOrAlias: string): string | undefined {
196
+ if (this.index.has(toolCallIdOrAlias)) return toolCallIdOrAlias;
197
+ const dedupTarget = this.dedupAliasToOriginal.get(toolCallIdOrAlias);
198
+ if (dedupTarget) return dedupTarget;
199
+ return this.aliasToToolCallId.get(toolCallIdOrAlias);
200
+ }
201
+
202
+ /**
203
+ * Returns the short alias (e.g. "t1") registered for the given
204
+ * toolCallId, or undefined if none was registered. Legacy summaries
205
+ * written before short-refs were introduced map shortId === toolCallId
206
+ * and intentionally return undefined here so callers (e.g. the
207
+ * pruner stub) can fall back to the toolCallId itself.
208
+ */
209
+ getShortRefForToolCallId(toolCallId: string): string | undefined {
210
+ return this.toolCallIdToAlias.get(toolCallId);
211
+ }
212
+
213
+ /**
214
+ * Look up a single record by toolCallId or short alias (used by query tool).
215
+ */
216
+ getRecord(toolCallIdOrAlias: string): ToolCallRecord | undefined {
217
+ const resolved = this.resolveToolCallId(toolCallIdOrAlias);
218
+ if (!resolved) return undefined;
219
+ return this.index.get(resolved);
220
+ }
221
+
222
+ /**
223
+ * Looks up multiple tool call records by ID. Skips any IDs not found.
224
+ */
225
+ lookupToolCalls(toolCallIds: string[]): ToolCallRecord[] {
226
+ const results: ToolCallRecord[] = [];
227
+ for (const id of toolCallIds) {
228
+ const record = this.getRecord(id);
229
+ if (record !== undefined) {
230
+ results.push(record);
231
+ }
232
+ }
233
+ return results;
234
+ }
235
+
236
+ /**
237
+ * Returns the toolCallId of an already-indexed record whose
238
+ * `(toolName, normalize(resultText))` matches the supplied input, or
239
+ * `undefined` if there is no match. Driven by the in-memory
240
+ * `contentHashToOriginal` map; only consults records that entered the
241
+ * indexer via `addBatch` (i.e. previous successful prunes) or were
242
+ * replayed at reconstruction time.
243
+ *
244
+ * Returns `undefined` for hash misses; consumers should treat that as
245
+ * "not a duplicate".
246
+ */
247
+ lookupByContent(toolName: string, resultText: string): string | undefined {
248
+ const hash = hashToolResult(toolName, resultText);
249
+ return this.contentHashToOriginal.get(hash);
250
+ }
251
+
252
+ /**
253
+ * Registers `newToolCallId` as a duplicate of `originalToolCallId`. The new
254
+ * id reuses the original's short alias (so `pruneMessages` emits the same
255
+ * `tN` ref for both) and is persisted via the supplied `appendEntry` so
256
+ * reconstruction can replay it later.
257
+ *
258
+ * No-op when `newToolCallId === originalToolCallId` (defensive).
259
+ */
260
+ registerDuplicate(
261
+ newToolCallId: string,
262
+ originalToolCallId: string,
263
+ appendEntry: (customType: string, data?: unknown) => void,
264
+ ): void {
265
+ if (newToolCallId === originalToolCallId) return;
266
+ this.dedupAliasToOriginal.set(newToolCallId, originalToolCallId);
267
+ const originalShortRef = this.toolCallIdToAlias.get(originalToolCallId);
268
+ if (originalShortRef) {
269
+ this.toolCallIdToAlias.set(newToolCallId, originalShortRef);
270
+ }
271
+ const payload: DedupAliasEntryData = { newToolCallId, originalToolCallId };
272
+ appendEntry(CUSTOM_TYPE_DEDUP_ALIAS, payload);
273
+ }
274
+
275
+ /**
276
+ * Stores summary body text keyed by the toolCallIds it covers.
277
+ * Called after a successful flush so `getPerBatchSummaryTextForToolCallIds`
278
+ * can serve chain summaries without re-scanning session entries.
279
+ */
280
+ registerSummaryBody(toolCallIds: string[], text: string): void {
281
+ if (text && toolCallIds.length > 0) {
282
+ this.summaryBodies.push({ toolCallIds, text });
283
+ }
284
+ }
285
+
286
+ /** Returns true if at least one stored summary covers any of the given toolCallIds. */
287
+ hasPerBatchSummaryCoveringAny(toolCallIds: string[]): boolean {
288
+ if (toolCallIds.length === 0) return false;
289
+ const idSet = new Set(toolCallIds);
290
+ return this.summaryBodies.some((s) => s.toolCallIds.some((id) => idSet.has(id)));
291
+ }
292
+
293
+ /**
294
+ * Returns the distinct per-batch summary texts whose toolCallIds overlap the
295
+ * given set (dedup'd by text). Used to build the synthetic chain body and as
296
+ * the fusion input for the range summarizer; the >= 2 count gates fusion.
297
+ */
298
+ getPerBatchSummariesForToolCallIds(toolCallIds: string[]): string[] {
299
+ if (toolCallIds.length === 0) return [];
300
+ const idSet = new Set(toolCallIds);
301
+ const texts: string[] = [];
302
+ const seen = new Set<string>();
303
+ for (const s of this.summaryBodies) {
304
+ if (s.toolCallIds.some((id) => idSet.has(id)) && !seen.has(s.text)) {
305
+ seen.add(s.text);
306
+ texts.push(s.text);
307
+ }
308
+ }
309
+ return texts;
310
+ }
311
+
312
+ /**
313
+ * Returns the concatenated summary text for all per-batch summaries whose
314
+ * toolCallIds overlap the given set, joined with "\n\n".
315
+ * Used by chain-range-prune to build the synthetic chain message body.
316
+ */
317
+ getPerBatchSummaryTextForToolCallIds(toolCallIds: string[]): string {
318
+ return this.getPerBatchSummariesForToolCallIds(toolCallIds).join("\n\n");
319
+ }
320
+
321
+ /**
322
+ * Returns the short t<N> refs for the given toolCallIds.
323
+ * Skips ids with no registered short ref (tool calls not yet summarized).
324
+ */
325
+ getToolRefsForToolCallIds(toolCallIds: string[]): string[] {
326
+ const refs: string[] = [];
327
+ for (const id of toolCallIds) {
328
+ const ref = this.toolCallIdToAlias.get(id);
329
+ if (ref) refs.push(ref);
330
+ }
331
+ return refs;
332
+ }
333
+
334
+ /** Registers a chain entry in the in-memory registry. Called by chain-compressor after persisting. */
335
+ registerChain(entry: ChainCompressionEntry): void {
336
+ this.chainRegistry.set(entry.startUserTimestamp, entry);
337
+ }
338
+
339
+ /** Returns all compressed chain entries sorted by startUserTimestamp ascending. */
340
+ getChainEntries(): ChainCompressionEntry[] {
341
+ return [...this.chainRegistry.values()].sort((a, b) => a.startUserTimestamp - b.startUserTimestamp);
342
+ }
343
+
344
+ /** O(n) scan over the chain registry by blockId. Registry is small (bounded by session chain count). */
345
+ findChainEntryByBlockId(blockId: string): ChainCompressionEntry | undefined {
346
+ for (const entry of this.chainRegistry.values()) {
347
+ if (entry.blockId === blockId) return entry;
348
+ }
349
+ return undefined;
350
+ }
351
+
352
+ /**
353
+ * Adds all tool calls from a captured batch to the runtime index and
354
+ * persists an IndexEntryData entry to the session via the supplied
355
+ * appendEntry callback. The callback exists so callers can route the
356
+ * append through either `pi.appendEntry` (runtime delivery) or
357
+ * `ctx.sessionManager.appendCustomEntry` (session delivery), without the
358
+ * indexer needing to know which one is active.
359
+ */
360
+ addBatch(
361
+ batch: CapturedBatch,
362
+ appendEntry: (customType: string, data?: unknown) => void,
363
+ ): void {
364
+ const records: ToolCallRecord[] = [];
365
+
366
+ for (const tc of batch.toolCalls) {
367
+ const record: ToolCallRecord = {
368
+ toolCallId: tc.toolCallId,
369
+ toolName: tc.toolName,
370
+ args: tc.args,
371
+ resultText: tc.resultText,
372
+ isError: tc.isError,
373
+ turnIndex: batch.turnIndex,
374
+ timestamp: batch.timestamp,
375
+ ...(tc.spillPath !== undefined ? { spillPath: tc.spillPath } : {}),
376
+ ...(tc.spillBytes !== undefined ? { spillBytes: tc.spillBytes } : {}),
377
+ ...(tc.resultPreview !== undefined ? { resultPreview: tc.resultPreview } : {}),
378
+ ...(tc.contentHash !== undefined ? { contentHash: tc.contentHash } : {}),
379
+ };
380
+ this.index.set(record.toolCallId, record);
381
+ records.push(record);
382
+ // Populate the dedup hash map AFTER the record is indexed so a future
383
+ // flush can dedup against this record. First-seen wins to keep the
384
+ // canonical id stable across multiple identical entries.
385
+ const hash = record.contentHash ?? hashToolResult(record.toolName, record.resultText);
386
+ if (!this.contentHashToOriginal.has(hash)) {
387
+ this.contentHashToOriginal.set(hash, record.toolCallId);
388
+ }
389
+ }
390
+
391
+ appendEntry(CUSTOM_TYPE_INDEX, { toolCalls: records } as IndexEntryData);
392
+ }
393
+ }
@@ -0,0 +1,82 @@
1
+ import { describe, expect, test } from "bun:test";
2
+ import { substituteBlockRefs } from "./nested-placeholders.js";
3
+
4
+ const lookup = (id: string): string | undefined => {
5
+ const map: Record<string, string> = {
6
+ b1: "summary of chain one",
7
+ b2: "summary of chain two",
8
+ b3: "chain three with {b2} inside",
9
+ };
10
+ return map[id];
11
+ };
12
+
13
+ describe("substituteBlockRefs", () => {
14
+ test("empty text returns empty string", () => {
15
+ expect(substituteBlockRefs("", lookup)).toBe("");
16
+ });
17
+
18
+ test("text with no placeholders is unchanged", () => {
19
+ const text = "no placeholders here, just prose";
20
+ expect(substituteBlockRefs(text, lookup)).toBe(text);
21
+ });
22
+
23
+ test("single {b1} is substituted", () => {
24
+ expect(substituteBlockRefs("{b1}", lookup)).toBe("summary of chain one");
25
+ });
26
+
27
+ test("multiple references in one text are all resolved", () => {
28
+ expect(substituteBlockRefs("{b1} and {b2}", lookup)).toBe(
29
+ "summary of chain one and summary of chain two",
30
+ );
31
+ });
32
+
33
+ test("missing blockId leaves the literal {bN} in place", () => {
34
+ expect(substituteBlockRefs("{b99}", lookup)).toBe("{b99}");
35
+ });
36
+
37
+ test("missing blockId among valid ones leaves only the unknown one", () => {
38
+ expect(substituteBlockRefs("{b1} and {b99}", lookup)).toBe("summary of chain one and {b99}");
39
+ });
40
+
41
+ test("self-reference is refused even when lookup has a value", () => {
42
+ // b3 is in the lookup, but selfBlockId=b3 means it must stay literal
43
+ expect(substituteBlockRefs("{b3} foo", lookup, { selfBlockId: "b3" })).toBe("{b3} foo");
44
+ });
45
+
46
+ test("self-reference refused, other refs still substituted", () => {
47
+ expect(substituteBlockRefs("{b3} and {b1}", lookup, { selfBlockId: "b3" })).toBe(
48
+ "{b3} and summary of chain one",
49
+ );
50
+ });
51
+
52
+ test("one-level only: substituted text containing {b2} is NOT re-expanded", () => {
53
+ // b3 → "chain three with {b2} inside"
54
+ // The {b2} inside that expansion must survive as a literal in the output.
55
+ const result = substituteBlockRefs("{b3}", lookup);
56
+ expect(result).toBe("chain three with {b2} inside");
57
+ // Explicitly: {b2} in the output is the literal string, not "summary of chain two"
58
+ expect(result).not.toContain("summary of chain two");
59
+ });
60
+
61
+ test("adjacent placeholders handled cleanly", () => {
62
+ expect(substituteBlockRefs("{b1}{b2}", lookup)).toBe(
63
+ "summary of chain onesummary of chain two",
64
+ );
65
+ });
66
+
67
+ test("placeholders embedded in surrounding prose", () => {
68
+ expect(substituteBlockRefs("Before {b1} middle {b2} after", lookup)).toBe(
69
+ "Before summary of chain one middle summary of chain two after",
70
+ );
71
+ });
72
+
73
+ test("lookup returning undefined leaves placeholder literal (null-safe contract)", () => {
74
+ // `resolved ?? match` only falls through to the literal when lookup returns undefined.
75
+ // pruner.ts collapses empty-string summary bodies to undefined before calling here
76
+ // (`|| undefined`), so this test pins the undefined-→-literal path that the
77
+ // pruner relies on to avoid silently erasing placeholders when summary bodies are missing.
78
+ const missingLookup = (_: string): string | undefined => undefined;
79
+ expect(substituteBlockRefs("{b1}", missingLookup)).toBe("{b1}");
80
+ expect(substituteBlockRefs("{b1} text", missingLookup)).toBe("{b1} text");
81
+ });
82
+ });
@@ -0,0 +1,20 @@
1
+ // One-level substitution only: the replaced text is not re-scanned.
2
+ // This is intentional — if summary B contains "{b1}", that placeholder
3
+ // was already resolved (or left literal) when B was originally generated.
4
+ // Re-scanning would require cycle detection across an arbitrary graph and
5
+ // could silently expand stale content if the session is replayed later.
6
+ const BLOCK_REF_RE = /\{b(\d+)\}/g;
7
+
8
+ export function substituteBlockRefs(
9
+ text: string,
10
+ blockSummaryLookup: (blockId: string) => string | undefined,
11
+ options?: { selfBlockId?: string },
12
+ ): string {
13
+ const selfBlockId = options?.selfBlockId;
14
+ return text.replace(BLOCK_REF_RE, (match, digits) => {
15
+ const blockId = `b${digits}`;
16
+ if (blockId === selfBlockId) return match;
17
+ const resolved = blockSummaryLookup(blockId);
18
+ return resolved ?? match;
19
+ });
20
+ }
@@ -0,0 +1,73 @@
1
+ import { describe, it, expect } from "bun:test";
2
+ import { mkdtemp, readFile, rm } from "node:fs/promises";
3
+ import { tmpdir } from "node:os";
4
+ import { join } from "node:path";
5
+ import { ToolCallIndexer } from "./indexer.js";
6
+ import { spillOversizedBatch, blobPathFor } from "./spill.js";
7
+ import { pruneMessages } from "./pruner.js";
8
+ import type { CapturedBatch } from "./types.js";
9
+ import { CUSTOM_TYPE_INDEX } from "./types.js";
10
+
11
+ const cfg = { spillThreshold: 10, spillPreviewBytes: 16, dedupByContentHash: true };
12
+ const batch = (tc: any): CapturedBatch => ({ turnIndex: 0, timestamp: 1, assistantText: "", toolCalls: [tc] });
13
+
14
+ describe("oversized spill end-to-end", () => {
15
+ it("spills, stubs in context, keeps full body on disk, survives reconstruct", async () => {
16
+ const dir = await mkdtemp(join(tmpdir(), "spill-e2e-"));
17
+ try {
18
+ const indexer = new ToolCallIndexer();
19
+ const entries: any[] = [];
20
+ const appendEntry = (customType: string, data?: unknown) => {
21
+ entries.push({ type: "custom", customType, data });
22
+ };
23
+
24
+ const body = "BIG\n".repeat(1000);
25
+ await spillOversizedBatch({
26
+ batch: batch({ toolCallId: "tc1", toolName: "fetch", args: { url: "u" }, resultText: body, isError: false }),
27
+ indexer,
28
+ config: cfg,
29
+ sessionDir: dir,
30
+ sessionId: "sid",
31
+ appendEntry,
32
+ });
33
+
34
+ // (a) full body on disk
35
+ expect(await readFile(blobPathFor(dir, "sid", "tc1"), "utf-8")).toBe(body);
36
+
37
+ // (b) persisted index entry has spillPath + preview, NOT the full body
38
+ const idxEntry = entries.find((e) => e.customType === CUSTOM_TYPE_INDEX);
39
+ expect(idxEntry).toBeTruthy();
40
+ const persisted = idxEntry.data.toolCalls[0];
41
+ expect(persisted.spillPath).toBe(blobPathFor(dir, "sid", "tc1"));
42
+ expect(persisted.resultText).toBe("");
43
+ expect(persisted.resultPreview.length).toBeGreaterThan(0);
44
+ expect(persisted.contentHash).toBeTruthy();
45
+
46
+ // (c) pruneMessages emits the mechanical spill stub (no summary, no LLM)
47
+ const msgs = [
48
+ {
49
+ role: "toolResult",
50
+ toolCallId: "tc1",
51
+ toolName: "fetch",
52
+ content: [{ type: "text", text: body }],
53
+ isError: false,
54
+ timestamp: 1,
55
+ },
56
+ ];
57
+ const { messages: out, pruned } = pruneMessages(msgs as any, indexer);
58
+ expect(pruned).toBe(true);
59
+ expect((out[0] as any).content[0].text).toContain(blobPathFor(dir, "sid", "tc1"));
60
+ expect((out[0] as any).content[0].text).not.toContain("Summarized in pruner summary");
61
+
62
+ // (d) reconstruct from the persisted entries: record still resolves, hash intact
63
+ const indexer2 = new ToolCallIndexer();
64
+ const fakeCtx = { sessionManager: { getBranch: () => entries } } as any;
65
+ indexer2.reconstructFromSession(fakeCtx);
66
+ const rec = indexer2.getRecord("tc1");
67
+ expect(rec?.spillPath).toBe(blobPathFor(dir, "sid", "tc1"));
68
+ expect(indexer2.lookupByContent("fetch", body)).toBe("tc1");
69
+ } finally {
70
+ await rm(dir, { recursive: true, force: true });
71
+ }
72
+ });
73
+ });
@@ -0,0 +1,62 @@
1
+ import { describe, expect, test } from "bun:test";
2
+ import { globToRegExp, isProtected } from "./protected.js";
3
+
4
+ describe("globToRegExp", () => {
5
+ test("** crosses path segments", () => {
6
+ expect(globToRegExp("**/skills/**/*.md").test("/Users/x/.agents/skills/release/SKILL.md")).toBe(true);
7
+ expect(globToRegExp("**/skills/**/*.md").test("a/skills/b/c/notes.md")).toBe(true);
8
+ });
9
+
10
+ test("**/ matches zero directories", () => {
11
+ expect(globToRegExp("**/SKILL.md").test("SKILL.md")).toBe(true);
12
+ expect(globToRegExp("**/skills/**/*.md").test("skills/foo.md")).toBe(true);
13
+ });
14
+
15
+ test("* and ? are segment-local", () => {
16
+ expect(globToRegExp("skills/*.md").test("skills/a/b.md")).toBe(false);
17
+ expect(globToRegExp("skills/?.md").test("skills/a.md")).toBe(true);
18
+ expect(globToRegExp("skills/?.md").test("skills//x.md")).toBe(false);
19
+ });
20
+
21
+ test("regex metacharacters in pattern are literals", () => {
22
+ expect(globToRegExp("a+b/(c).md").test("a+b/(c).md")).toBe(true);
23
+ expect(globToRegExp("a.md").test("aXmd")).toBe(false);
24
+ });
25
+
26
+ test("full-path anchored match", () => {
27
+ expect(globToRegExp("skills/a.md").test("xskills/a.md")).toBe(false);
28
+ expect(globToRegExp("skills/a.md").test("skills/a.md.bak")).toBe(false);
29
+ });
30
+
31
+ test("case-sensitive", () => {
32
+ expect(globToRegExp("**/SKILL.md").test("a/skill.md")).toBe(false);
33
+ });
34
+ });
35
+
36
+ describe("isProtected", () => {
37
+ const cfg = { protectedTools: ["todowrite"], protectedPaths: ["**/skills/**/*.md"] };
38
+
39
+ test("matches by tool name regardless of args", () => {
40
+ expect(isProtected("todowrite", undefined, cfg)).toBe(true);
41
+ });
42
+
43
+ test("matches by path glob", () => {
44
+ expect(isProtected("read", { path: "/h/skills/x/SKILL.md" }, cfg)).toBe(true);
45
+ expect(isProtected("read", { path: "/h/src/app.ts" }, cfg)).toBe(false);
46
+ });
47
+
48
+ test("backslashes normalized to forward slashes", () => {
49
+ expect(isProtected("read", { path: "h\\skills\\x\\SKILL.md" }, cfg)).toBe(true);
50
+ });
51
+
52
+ test("non-string / missing path is not protected", () => {
53
+ expect(isProtected("read", { path: 42 }, cfg)).toBe(false);
54
+ expect(isProtected("read", {}, cfg)).toBe(false);
55
+ expect(isProtected("read", undefined, cfg)).toBe(false);
56
+ expect(isProtected("read", null, cfg)).toBe(false);
57
+ });
58
+
59
+ test("empty config protects nothing", () => {
60
+ expect(isProtected("read", { path: "skills/a/SKILL.md" }, { protectedTools: [], protectedPaths: [] })).toBe(false);
61
+ });
62
+ });
@@ -0,0 +1,51 @@
1
+ /** Structural pick of ContextPruneConfig — keeps this module dependency-free. */
2
+ export interface ProtectionConfig {
3
+ protectedTools: readonly string[];
4
+ protectedPaths: readonly string[];
5
+ }
6
+
7
+ // Compile-once: pattern -> RegExp is pure, so the cache never needs
8
+ // invalidation. Patterns only come from config arrays, so growth is bounded.
9
+ const patternCache = new Map<string, RegExp>();
10
+
11
+ export function globToRegExp(pattern: string): RegExp {
12
+ const cached = patternCache.get(pattern);
13
+ if (cached) return cached;
14
+ let re = "";
15
+ let i = 0;
16
+ while (i < pattern.length) {
17
+ const ch = pattern[i];
18
+ if (ch === "*") {
19
+ if (pattern[i + 1] === "*") {
20
+ if (pattern[i + 2] === "/") {
21
+ re += "(?:[^/]*/)*"; // `**/` — zero or more whole directories
22
+ i += 3;
23
+ } else {
24
+ re += ".*"; // bare `**`
25
+ i += 2;
26
+ }
27
+ } else {
28
+ re += "[^/]*"; // `*` — segment-local
29
+ i += 1;
30
+ }
31
+ } else if (ch === "?") {
32
+ re += "[^/]";
33
+ i += 1;
34
+ } else {
35
+ re += ch.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
36
+ i += 1;
37
+ }
38
+ }
39
+ const compiled = new RegExp(`^${re}$`);
40
+ patternCache.set(pattern, compiled);
41
+ return compiled;
42
+ }
43
+
44
+ export function isProtected(toolName: string, args: unknown, config: ProtectionConfig): boolean {
45
+ if (config.protectedTools.includes(toolName)) return true;
46
+ if (config.protectedPaths.length === 0) return false;
47
+ const path = (args as Record<string, unknown> | null | undefined)?.path;
48
+ if (typeof path !== "string") return false;
49
+ const normalized = path.replace(/\\/g, "/");
50
+ return config.protectedPaths.some((p) => globToRegExp(p).test(normalized));
51
+ }