pi-condense 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/LICENSE +22 -0
- package/PRUNING.md +1028 -0
- package/README.md +243 -0
- package/index.ts +858 -0
- package/package.json +56 -0
- package/src/batch-capture.ts +226 -0
- package/src/block-refs.test.ts +42 -0
- package/src/block-refs.ts +16 -0
- package/src/budget.test.ts +66 -0
- package/src/budget.ts +39 -0
- package/src/chain-compressor.test.ts +283 -0
- package/src/chain-compressor.ts +132 -0
- package/src/chain-detector.test.ts +302 -0
- package/src/chain-detector.ts +128 -0
- package/src/chain-range-prune.test.ts +522 -0
- package/src/chain-range-prune.ts +128 -0
- package/src/commands.test.ts +67 -0
- package/src/commands.ts +1207 -0
- package/src/config.ts +126 -0
- package/src/content-hash.ts +35 -0
- package/src/error-purge.test.ts +186 -0
- package/src/error-purge.ts +71 -0
- package/src/frontier.ts +62 -0
- package/src/indexer.ts +393 -0
- package/src/nested-placeholders.test.ts +82 -0
- package/src/nested-placeholders.ts +20 -0
- package/src/oversized-spill.integration.test.ts +73 -0
- package/src/protected.test.ts +62 -0
- package/src/protected.ts +51 -0
- package/src/pruner.test.ts +508 -0
- package/src/pruner.ts +156 -0
- package/src/query-tool.ts +78 -0
- package/src/range-compression.integration.test.ts +252 -0
- package/src/spill.test.ts +102 -0
- package/src/spill.ts +90 -0
- package/src/stats.test.ts +114 -0
- package/src/stats.ts +190 -0
- package/src/summarizer.test.ts +17 -0
- package/src/summarizer.ts +262 -0
- package/src/summary-refs.ts +61 -0
- package/src/thinking-strip.test.ts +175 -0
- package/src/thinking-strip.ts +42 -0
- package/src/tree-browser.ts +382 -0
- package/src/types.ts +764 -0
package/src/indexer.ts
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import type {
|
|
3
|
+
CapturedBatch,
|
|
4
|
+
ChainCompressionEntry,
|
|
5
|
+
DedupAliasEntryData,
|
|
6
|
+
IndexEntryData,
|
|
7
|
+
ToolCallRecord,
|
|
8
|
+
} from "./types.js";
|
|
9
|
+
import {
|
|
10
|
+
CUSTOM_TYPE_CHAIN,
|
|
11
|
+
CUSTOM_TYPE_DEDUP_ALIAS,
|
|
12
|
+
CUSTOM_TYPE_INDEX,
|
|
13
|
+
CUSTOM_TYPE_SUMMARY,
|
|
14
|
+
} from "./types.js";
|
|
15
|
+
import {
|
|
16
|
+
buildShortToolCallRefs,
|
|
17
|
+
normalizeSummaryToolCallRefs,
|
|
18
|
+
type SummaryToolCallRef,
|
|
19
|
+
} from "./summary-refs.js";
|
|
20
|
+
import { hashToolResult } from "./content-hash.js";
|
|
21
|
+
|
|
22
|
+
export class ToolCallIndexer {
|
|
23
|
+
private index = new Map<string, ToolCallRecord>();
|
|
24
|
+
private aliasToToolCallId = new Map<string, string>();
|
|
25
|
+
private toolCallIdToAlias = new Map<string, string>();
|
|
26
|
+
private nextShortAliasNumber = 1;
|
|
27
|
+
/**
|
|
28
|
+
* hash → original toolCallId. Populated as records enter the indexer
|
|
29
|
+
* (`addBatch`) and on `reconstructFromSession`. Drives the pre-flush
|
|
30
|
+
* dedup pass via `lookupByContent`.
|
|
31
|
+
*/
|
|
32
|
+
private contentHashToOriginal = new Map<string, string>();
|
|
33
|
+
/**
|
|
34
|
+
* Duplicate toolCallId → original toolCallId. Populated by
|
|
35
|
+
* `registerDuplicate` during the pre-flush dedup pass and rebuilt from
|
|
36
|
+
* CUSTOM_TYPE_DEDUP_ALIAS entries on reconstruction.
|
|
37
|
+
*
|
|
38
|
+
* Both `isSummarized` and `resolveToolCallId` consult this map so
|
|
39
|
+
* `pruneMessages` stub-replaces dup toolResults and `context_tree_query`
|
|
40
|
+
* resolves dup ids to the original record.
|
|
41
|
+
*/
|
|
42
|
+
private dedupAliasToOriginal = new Map<string, string>();
|
|
43
|
+
/**
|
|
44
|
+
* Per-batch summary bodies for chain-compression summary text lookup.
|
|
45
|
+
* Each entry maps a set of toolCallIds to the summary's markdown body.
|
|
46
|
+
* Populated from CUSTOM_TYPE_SUMMARY entries at rebuild time and via
|
|
47
|
+
* `registerSummaryBody` after a successful flush.
|
|
48
|
+
*/
|
|
49
|
+
private summaryBodies: Array<{ toolCallIds: string[]; text: string }> = [];
|
|
50
|
+
/** Compressed chains, keyed on startUserTimestamp for O(1) dedup checks. */
|
|
51
|
+
private chainRegistry = new Map<number, ChainCompressionEntry>();
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Rebuilds the in-memory index from session history by scanning all
|
|
55
|
+
* custom entries with customType === CUSTOM_TYPE_INDEX.
|
|
56
|
+
*/
|
|
57
|
+
reconstructFromSession(ctx: ExtensionContext): void {
|
|
58
|
+
this.index.clear();
|
|
59
|
+
this.aliasToToolCallId.clear();
|
|
60
|
+
this.toolCallIdToAlias.clear();
|
|
61
|
+
this.contentHashToOriginal.clear();
|
|
62
|
+
this.dedupAliasToOriginal.clear();
|
|
63
|
+
this.nextShortAliasNumber = 1;
|
|
64
|
+
this.summaryBodies = [];
|
|
65
|
+
this.chainRegistry.clear();
|
|
66
|
+
|
|
67
|
+
// Two passes so dedup aliases land AFTER the original short refs they
|
|
68
|
+
// need to reuse, regardless of the underlying append order.
|
|
69
|
+
const branch = ctx.sessionManager.getBranch();
|
|
70
|
+
const dedupAliasEntries: DedupAliasEntryData[] = [];
|
|
71
|
+
|
|
72
|
+
for (const entry of branch) {
|
|
73
|
+
if (entry.type === "custom" && (entry as any).customType === CUSTOM_TYPE_INDEX) {
|
|
74
|
+
const data = (entry as any).data as IndexEntryData;
|
|
75
|
+
if (data && Array.isArray(data.toolCalls)) {
|
|
76
|
+
for (const toolCall of data.toolCalls) {
|
|
77
|
+
this.index.set(toolCall.toolCallId, toolCall);
|
|
78
|
+
// First-seen wins so the contentHashToOriginal map matches what
|
|
79
|
+
// addBatch would have produced at append time.
|
|
80
|
+
const hash = toolCall.contentHash ?? hashToolResult(toolCall.toolName, toolCall.resultText);
|
|
81
|
+
if (!this.contentHashToOriginal.has(hash)) {
|
|
82
|
+
this.contentHashToOriginal.set(hash, toolCall.toolCallId);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (entry.type === "custom_message" && (entry as any).customType === CUSTOM_TYPE_SUMMARY) {
|
|
90
|
+
const refs = normalizeSummaryToolCallRefs((entry as any).details);
|
|
91
|
+
this.registerSummaryRefs(refs);
|
|
92
|
+
const raw = (entry as any).content;
|
|
93
|
+
const text =
|
|
94
|
+
typeof raw === "string"
|
|
95
|
+
? raw
|
|
96
|
+
: Array.isArray(raw)
|
|
97
|
+
? raw
|
|
98
|
+
.filter((c: any) => c.type === "text")
|
|
99
|
+
.map((c: any) => c.text as string)
|
|
100
|
+
.join("\n")
|
|
101
|
+
: "";
|
|
102
|
+
if (text) {
|
|
103
|
+
this.summaryBodies.push({ toolCallIds: refs.map((r) => r.toolCallId), text });
|
|
104
|
+
}
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (entry.type === "custom" && (entry as any).customType === CUSTOM_TYPE_CHAIN) {
|
|
109
|
+
const data = (entry as any).data as ChainCompressionEntry;
|
|
110
|
+
if (data?.blockId && typeof data.startUserTimestamp === "number") {
|
|
111
|
+
this.chainRegistry.set(data.startUserTimestamp, data);
|
|
112
|
+
}
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (entry.type === "custom" && (entry as any).customType === CUSTOM_TYPE_DEDUP_ALIAS) {
|
|
117
|
+
const data = (entry as any).data as DedupAliasEntryData;
|
|
118
|
+
if (data?.newToolCallId && data?.originalToolCallId) {
|
|
119
|
+
dedupAliasEntries.push(data);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
for (const data of dedupAliasEntries) {
|
|
125
|
+
this.dedupAliasToOriginal.set(data.newToolCallId, data.originalToolCallId);
|
|
126
|
+
const originalShortRef = this.toolCallIdToAlias.get(data.originalToolCallId);
|
|
127
|
+
if (originalShortRef) {
|
|
128
|
+
// Keep `getShortRefForToolCallId(dupId)` returning the SAME short ref
|
|
129
|
+
// as the original so pruneMessages emits a consistent `tN` for both.
|
|
130
|
+
this.toolCallIdToAlias.set(data.newToolCallId, originalShortRef);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Returns true if the given toolCallId has been pruned — either because
|
|
137
|
+
* its full record is in the index, or because it has been registered as
|
|
138
|
+
* an alias of an already-indexed original via the content-hash dedup pass.
|
|
139
|
+
*
|
|
140
|
+
* `pruneMessages` uses this to decide whether to stub-replace a
|
|
141
|
+
* ToolResultMessage; both cases need the same treatment.
|
|
142
|
+
*/
|
|
143
|
+
isSummarized(toolCallId: string): boolean {
|
|
144
|
+
return this.index.has(toolCallId) || this.dedupAliasToOriginal.has(toolCallId);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Returns the full runtime index map.
|
|
149
|
+
*/
|
|
150
|
+
getIndex(): Map<string, ToolCallRecord> {
|
|
151
|
+
return this.index;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Register short aliases for a summary message so future recovery queries can
|
|
156
|
+
* resolve the short ids back to the persisted toolCallIds.
|
|
157
|
+
*/
|
|
158
|
+
registerSummaryRefs(refs: SummaryToolCallRef[]): void {
|
|
159
|
+
for (const ref of refs) {
|
|
160
|
+
if (!ref.shortId || !ref.toolCallId) continue;
|
|
161
|
+
if (ref.shortId !== ref.toolCallId) {
|
|
162
|
+
this.aliasToToolCallId.set(ref.shortId, ref.toolCallId);
|
|
163
|
+
this.toolCallIdToAlias.set(ref.toolCallId, ref.shortId);
|
|
164
|
+
}
|
|
165
|
+
const match = /^t(\d+)$/.exec(ref.shortId);
|
|
166
|
+
if (match) {
|
|
167
|
+
this.nextShortAliasNumber = Math.max(this.nextShortAliasNumber, Number(match[1]) + 1);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Allocates short aliases for a batch's tool calls and registers them in the
|
|
174
|
+
* runtime alias map.
|
|
175
|
+
*/
|
|
176
|
+
allocateSummaryRefs(batch: CapturedBatch): SummaryToolCallRef[] {
|
|
177
|
+
const toolCallIds = batch.toolCalls.map((tc) => tc.toolCallId);
|
|
178
|
+
const { refs, nextIndex } = buildShortToolCallRefs(toolCallIds, this.nextShortAliasNumber);
|
|
179
|
+
this.nextShortAliasNumber = nextIndex;
|
|
180
|
+
return refs;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Resolve a short alias, a duplicate's toolCallId, or a full toolCallId
|
|
185
|
+
* to the canonical toolCallId backing it.
|
|
186
|
+
*
|
|
187
|
+
* Order:
|
|
188
|
+
* 1. Direct hit in `this.index` (canonical id).
|
|
189
|
+
* 2. Dedup alias → underlying original toolCallId.
|
|
190
|
+
* 3. Short-ref (`t3`) → underlying toolCallId.
|
|
191
|
+
*
|
|
192
|
+
* Used by `getRecord`/`lookupToolCalls` so `context_tree_query` returns
|
|
193
|
+
* the original record for both short refs and dedup'd ids.
|
|
194
|
+
*/
|
|
195
|
+
resolveToolCallId(toolCallIdOrAlias: string): string | undefined {
|
|
196
|
+
if (this.index.has(toolCallIdOrAlias)) return toolCallIdOrAlias;
|
|
197
|
+
const dedupTarget = this.dedupAliasToOriginal.get(toolCallIdOrAlias);
|
|
198
|
+
if (dedupTarget) return dedupTarget;
|
|
199
|
+
return this.aliasToToolCallId.get(toolCallIdOrAlias);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Returns the short alias (e.g. "t1") registered for the given
|
|
204
|
+
* toolCallId, or undefined if none was registered. Legacy summaries
|
|
205
|
+
* written before short-refs were introduced map shortId === toolCallId
|
|
206
|
+
* and intentionally return undefined here so callers (e.g. the
|
|
207
|
+
* pruner stub) can fall back to the toolCallId itself.
|
|
208
|
+
*/
|
|
209
|
+
getShortRefForToolCallId(toolCallId: string): string | undefined {
|
|
210
|
+
return this.toolCallIdToAlias.get(toolCallId);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Look up a single record by toolCallId or short alias (used by query tool).
|
|
215
|
+
*/
|
|
216
|
+
getRecord(toolCallIdOrAlias: string): ToolCallRecord | undefined {
|
|
217
|
+
const resolved = this.resolveToolCallId(toolCallIdOrAlias);
|
|
218
|
+
if (!resolved) return undefined;
|
|
219
|
+
return this.index.get(resolved);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Looks up multiple tool call records by ID. Skips any IDs not found.
|
|
224
|
+
*/
|
|
225
|
+
lookupToolCalls(toolCallIds: string[]): ToolCallRecord[] {
|
|
226
|
+
const results: ToolCallRecord[] = [];
|
|
227
|
+
for (const id of toolCallIds) {
|
|
228
|
+
const record = this.getRecord(id);
|
|
229
|
+
if (record !== undefined) {
|
|
230
|
+
results.push(record);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
return results;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Returns the toolCallId of an already-indexed record whose
|
|
238
|
+
* `(toolName, normalize(resultText))` matches the supplied input, or
|
|
239
|
+
* `undefined` if there is no match. Driven by the in-memory
|
|
240
|
+
* `contentHashToOriginal` map; only consults records that entered the
|
|
241
|
+
* indexer via `addBatch` (i.e. previous successful prunes) or were
|
|
242
|
+
* replayed at reconstruction time.
|
|
243
|
+
*
|
|
244
|
+
* Returns `undefined` for hash misses; consumers should treat that as
|
|
245
|
+
* "not a duplicate".
|
|
246
|
+
*/
|
|
247
|
+
lookupByContent(toolName: string, resultText: string): string | undefined {
|
|
248
|
+
const hash = hashToolResult(toolName, resultText);
|
|
249
|
+
return this.contentHashToOriginal.get(hash);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Registers `newToolCallId` as a duplicate of `originalToolCallId`. The new
|
|
254
|
+
* id reuses the original's short alias (so `pruneMessages` emits the same
|
|
255
|
+
* `tN` ref for both) and is persisted via the supplied `appendEntry` so
|
|
256
|
+
* reconstruction can replay it later.
|
|
257
|
+
*
|
|
258
|
+
* No-op when `newToolCallId === originalToolCallId` (defensive).
|
|
259
|
+
*/
|
|
260
|
+
registerDuplicate(
|
|
261
|
+
newToolCallId: string,
|
|
262
|
+
originalToolCallId: string,
|
|
263
|
+
appendEntry: (customType: string, data?: unknown) => void,
|
|
264
|
+
): void {
|
|
265
|
+
if (newToolCallId === originalToolCallId) return;
|
|
266
|
+
this.dedupAliasToOriginal.set(newToolCallId, originalToolCallId);
|
|
267
|
+
const originalShortRef = this.toolCallIdToAlias.get(originalToolCallId);
|
|
268
|
+
if (originalShortRef) {
|
|
269
|
+
this.toolCallIdToAlias.set(newToolCallId, originalShortRef);
|
|
270
|
+
}
|
|
271
|
+
const payload: DedupAliasEntryData = { newToolCallId, originalToolCallId };
|
|
272
|
+
appendEntry(CUSTOM_TYPE_DEDUP_ALIAS, payload);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Stores summary body text keyed by the toolCallIds it covers.
|
|
277
|
+
* Called after a successful flush so `getPerBatchSummaryTextForToolCallIds`
|
|
278
|
+
* can serve chain summaries without re-scanning session entries.
|
|
279
|
+
*/
|
|
280
|
+
registerSummaryBody(toolCallIds: string[], text: string): void {
|
|
281
|
+
if (text && toolCallIds.length > 0) {
|
|
282
|
+
this.summaryBodies.push({ toolCallIds, text });
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** Returns true if at least one stored summary covers any of the given toolCallIds. */
|
|
287
|
+
hasPerBatchSummaryCoveringAny(toolCallIds: string[]): boolean {
|
|
288
|
+
if (toolCallIds.length === 0) return false;
|
|
289
|
+
const idSet = new Set(toolCallIds);
|
|
290
|
+
return this.summaryBodies.some((s) => s.toolCallIds.some((id) => idSet.has(id)));
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Returns the distinct per-batch summary texts whose toolCallIds overlap the
|
|
295
|
+
* given set (dedup'd by text). Used to build the synthetic chain body and as
|
|
296
|
+
* the fusion input for the range summarizer; the >= 2 count gates fusion.
|
|
297
|
+
*/
|
|
298
|
+
getPerBatchSummariesForToolCallIds(toolCallIds: string[]): string[] {
|
|
299
|
+
if (toolCallIds.length === 0) return [];
|
|
300
|
+
const idSet = new Set(toolCallIds);
|
|
301
|
+
const texts: string[] = [];
|
|
302
|
+
const seen = new Set<string>();
|
|
303
|
+
for (const s of this.summaryBodies) {
|
|
304
|
+
if (s.toolCallIds.some((id) => idSet.has(id)) && !seen.has(s.text)) {
|
|
305
|
+
seen.add(s.text);
|
|
306
|
+
texts.push(s.text);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return texts;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/**
|
|
313
|
+
* Returns the concatenated summary text for all per-batch summaries whose
|
|
314
|
+
* toolCallIds overlap the given set, joined with "\n\n".
|
|
315
|
+
* Used by chain-range-prune to build the synthetic chain message body.
|
|
316
|
+
*/
|
|
317
|
+
getPerBatchSummaryTextForToolCallIds(toolCallIds: string[]): string {
|
|
318
|
+
return this.getPerBatchSummariesForToolCallIds(toolCallIds).join("\n\n");
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* Returns the short t<N> refs for the given toolCallIds.
|
|
323
|
+
* Skips ids with no registered short ref (tool calls not yet summarized).
|
|
324
|
+
*/
|
|
325
|
+
getToolRefsForToolCallIds(toolCallIds: string[]): string[] {
|
|
326
|
+
const refs: string[] = [];
|
|
327
|
+
for (const id of toolCallIds) {
|
|
328
|
+
const ref = this.toolCallIdToAlias.get(id);
|
|
329
|
+
if (ref) refs.push(ref);
|
|
330
|
+
}
|
|
331
|
+
return refs;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/** Registers a chain entry in the in-memory registry. Called by chain-compressor after persisting. */
|
|
335
|
+
registerChain(entry: ChainCompressionEntry): void {
|
|
336
|
+
this.chainRegistry.set(entry.startUserTimestamp, entry);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/** Returns all compressed chain entries sorted by startUserTimestamp ascending. */
|
|
340
|
+
getChainEntries(): ChainCompressionEntry[] {
|
|
341
|
+
return [...this.chainRegistry.values()].sort((a, b) => a.startUserTimestamp - b.startUserTimestamp);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/** O(n) scan over the chain registry by blockId. Registry is small (bounded by session chain count). */
|
|
345
|
+
findChainEntryByBlockId(blockId: string): ChainCompressionEntry | undefined {
|
|
346
|
+
for (const entry of this.chainRegistry.values()) {
|
|
347
|
+
if (entry.blockId === blockId) return entry;
|
|
348
|
+
}
|
|
349
|
+
return undefined;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Adds all tool calls from a captured batch to the runtime index and
|
|
354
|
+
* persists an IndexEntryData entry to the session via the supplied
|
|
355
|
+
* appendEntry callback. The callback exists so callers can route the
|
|
356
|
+
* append through either `pi.appendEntry` (runtime delivery) or
|
|
357
|
+
* `ctx.sessionManager.appendCustomEntry` (session delivery), without the
|
|
358
|
+
* indexer needing to know which one is active.
|
|
359
|
+
*/
|
|
360
|
+
addBatch(
|
|
361
|
+
batch: CapturedBatch,
|
|
362
|
+
appendEntry: (customType: string, data?: unknown) => void,
|
|
363
|
+
): void {
|
|
364
|
+
const records: ToolCallRecord[] = [];
|
|
365
|
+
|
|
366
|
+
for (const tc of batch.toolCalls) {
|
|
367
|
+
const record: ToolCallRecord = {
|
|
368
|
+
toolCallId: tc.toolCallId,
|
|
369
|
+
toolName: tc.toolName,
|
|
370
|
+
args: tc.args,
|
|
371
|
+
resultText: tc.resultText,
|
|
372
|
+
isError: tc.isError,
|
|
373
|
+
turnIndex: batch.turnIndex,
|
|
374
|
+
timestamp: batch.timestamp,
|
|
375
|
+
...(tc.spillPath !== undefined ? { spillPath: tc.spillPath } : {}),
|
|
376
|
+
...(tc.spillBytes !== undefined ? { spillBytes: tc.spillBytes } : {}),
|
|
377
|
+
...(tc.resultPreview !== undefined ? { resultPreview: tc.resultPreview } : {}),
|
|
378
|
+
...(tc.contentHash !== undefined ? { contentHash: tc.contentHash } : {}),
|
|
379
|
+
};
|
|
380
|
+
this.index.set(record.toolCallId, record);
|
|
381
|
+
records.push(record);
|
|
382
|
+
// Populate the dedup hash map AFTER the record is indexed so a future
|
|
383
|
+
// flush can dedup against this record. First-seen wins to keep the
|
|
384
|
+
// canonical id stable across multiple identical entries.
|
|
385
|
+
const hash = record.contentHash ?? hashToolResult(record.toolName, record.resultText);
|
|
386
|
+
if (!this.contentHashToOriginal.has(hash)) {
|
|
387
|
+
this.contentHashToOriginal.set(hash, record.toolCallId);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
appendEntry(CUSTOM_TYPE_INDEX, { toolCalls: records } as IndexEntryData);
|
|
392
|
+
}
|
|
393
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { substituteBlockRefs } from "./nested-placeholders.js";
|
|
3
|
+
|
|
4
|
+
const lookup = (id: string): string | undefined => {
|
|
5
|
+
const map: Record<string, string> = {
|
|
6
|
+
b1: "summary of chain one",
|
|
7
|
+
b2: "summary of chain two",
|
|
8
|
+
b3: "chain three with {b2} inside",
|
|
9
|
+
};
|
|
10
|
+
return map[id];
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
describe("substituteBlockRefs", () => {
|
|
14
|
+
test("empty text returns empty string", () => {
|
|
15
|
+
expect(substituteBlockRefs("", lookup)).toBe("");
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
test("text with no placeholders is unchanged", () => {
|
|
19
|
+
const text = "no placeholders here, just prose";
|
|
20
|
+
expect(substituteBlockRefs(text, lookup)).toBe(text);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
test("single {b1} is substituted", () => {
|
|
24
|
+
expect(substituteBlockRefs("{b1}", lookup)).toBe("summary of chain one");
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("multiple references in one text are all resolved", () => {
|
|
28
|
+
expect(substituteBlockRefs("{b1} and {b2}", lookup)).toBe(
|
|
29
|
+
"summary of chain one and summary of chain two",
|
|
30
|
+
);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test("missing blockId leaves the literal {bN} in place", () => {
|
|
34
|
+
expect(substituteBlockRefs("{b99}", lookup)).toBe("{b99}");
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test("missing blockId among valid ones leaves only the unknown one", () => {
|
|
38
|
+
expect(substituteBlockRefs("{b1} and {b99}", lookup)).toBe("summary of chain one and {b99}");
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("self-reference is refused even when lookup has a value", () => {
|
|
42
|
+
// b3 is in the lookup, but selfBlockId=b3 means it must stay literal
|
|
43
|
+
expect(substituteBlockRefs("{b3} foo", lookup, { selfBlockId: "b3" })).toBe("{b3} foo");
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("self-reference refused, other refs still substituted", () => {
|
|
47
|
+
expect(substituteBlockRefs("{b3} and {b1}", lookup, { selfBlockId: "b3" })).toBe(
|
|
48
|
+
"{b3} and summary of chain one",
|
|
49
|
+
);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test("one-level only: substituted text containing {b2} is NOT re-expanded", () => {
|
|
53
|
+
// b3 → "chain three with {b2} inside"
|
|
54
|
+
// The {b2} inside that expansion must survive as a literal in the output.
|
|
55
|
+
const result = substituteBlockRefs("{b3}", lookup);
|
|
56
|
+
expect(result).toBe("chain three with {b2} inside");
|
|
57
|
+
// Explicitly: {b2} in the output is the literal string, not "summary of chain two"
|
|
58
|
+
expect(result).not.toContain("summary of chain two");
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("adjacent placeholders handled cleanly", () => {
|
|
62
|
+
expect(substituteBlockRefs("{b1}{b2}", lookup)).toBe(
|
|
63
|
+
"summary of chain onesummary of chain two",
|
|
64
|
+
);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test("placeholders embedded in surrounding prose", () => {
|
|
68
|
+
expect(substituteBlockRefs("Before {b1} middle {b2} after", lookup)).toBe(
|
|
69
|
+
"Before summary of chain one middle summary of chain two after",
|
|
70
|
+
);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("lookup returning undefined leaves placeholder literal (null-safe contract)", () => {
|
|
74
|
+
// `resolved ?? match` only falls through to the literal when lookup returns undefined.
|
|
75
|
+
// pruner.ts collapses empty-string summary bodies to undefined before calling here
|
|
76
|
+
// (`|| undefined`), so this test pins the undefined-→-literal path that the
|
|
77
|
+
// pruner relies on to avoid silently erasing placeholders when summary bodies are missing.
|
|
78
|
+
const missingLookup = (_: string): string | undefined => undefined;
|
|
79
|
+
expect(substituteBlockRefs("{b1}", missingLookup)).toBe("{b1}");
|
|
80
|
+
expect(substituteBlockRefs("{b1} text", missingLookup)).toBe("{b1} text");
|
|
81
|
+
});
|
|
82
|
+
});
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// One-level substitution only: the replaced text is not re-scanned.
|
|
2
|
+
// This is intentional — if summary B contains "{b1}", that placeholder
|
|
3
|
+
// was already resolved (or left literal) when B was originally generated.
|
|
4
|
+
// Re-scanning would require cycle detection across an arbitrary graph and
|
|
5
|
+
// could silently expand stale content if the session is replayed later.
|
|
6
|
+
const BLOCK_REF_RE = /\{b(\d+)\}/g;
|
|
7
|
+
|
|
8
|
+
export function substituteBlockRefs(
|
|
9
|
+
text: string,
|
|
10
|
+
blockSummaryLookup: (blockId: string) => string | undefined,
|
|
11
|
+
options?: { selfBlockId?: string },
|
|
12
|
+
): string {
|
|
13
|
+
const selfBlockId = options?.selfBlockId;
|
|
14
|
+
return text.replace(BLOCK_REF_RE, (match, digits) => {
|
|
15
|
+
const blockId = `b${digits}`;
|
|
16
|
+
if (blockId === selfBlockId) return match;
|
|
17
|
+
const resolved = blockSummaryLookup(blockId);
|
|
18
|
+
return resolved ?? match;
|
|
19
|
+
});
|
|
20
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { describe, it, expect } from "bun:test";
|
|
2
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { ToolCallIndexer } from "./indexer.js";
|
|
6
|
+
import { spillOversizedBatch, blobPathFor } from "./spill.js";
|
|
7
|
+
import { pruneMessages } from "./pruner.js";
|
|
8
|
+
import type { CapturedBatch } from "./types.js";
|
|
9
|
+
import { CUSTOM_TYPE_INDEX } from "./types.js";
|
|
10
|
+
|
|
11
|
+
const cfg = { spillThreshold: 10, spillPreviewBytes: 16, dedupByContentHash: true };
|
|
12
|
+
const batch = (tc: any): CapturedBatch => ({ turnIndex: 0, timestamp: 1, assistantText: "", toolCalls: [tc] });
|
|
13
|
+
|
|
14
|
+
describe("oversized spill end-to-end", () => {
|
|
15
|
+
it("spills, stubs in context, keeps full body on disk, survives reconstruct", async () => {
|
|
16
|
+
const dir = await mkdtemp(join(tmpdir(), "spill-e2e-"));
|
|
17
|
+
try {
|
|
18
|
+
const indexer = new ToolCallIndexer();
|
|
19
|
+
const entries: any[] = [];
|
|
20
|
+
const appendEntry = (customType: string, data?: unknown) => {
|
|
21
|
+
entries.push({ type: "custom", customType, data });
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
const body = "BIG\n".repeat(1000);
|
|
25
|
+
await spillOversizedBatch({
|
|
26
|
+
batch: batch({ toolCallId: "tc1", toolName: "fetch", args: { url: "u" }, resultText: body, isError: false }),
|
|
27
|
+
indexer,
|
|
28
|
+
config: cfg,
|
|
29
|
+
sessionDir: dir,
|
|
30
|
+
sessionId: "sid",
|
|
31
|
+
appendEntry,
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
// (a) full body on disk
|
|
35
|
+
expect(await readFile(blobPathFor(dir, "sid", "tc1"), "utf-8")).toBe(body);
|
|
36
|
+
|
|
37
|
+
// (b) persisted index entry has spillPath + preview, NOT the full body
|
|
38
|
+
const idxEntry = entries.find((e) => e.customType === CUSTOM_TYPE_INDEX);
|
|
39
|
+
expect(idxEntry).toBeTruthy();
|
|
40
|
+
const persisted = idxEntry.data.toolCalls[0];
|
|
41
|
+
expect(persisted.spillPath).toBe(blobPathFor(dir, "sid", "tc1"));
|
|
42
|
+
expect(persisted.resultText).toBe("");
|
|
43
|
+
expect(persisted.resultPreview.length).toBeGreaterThan(0);
|
|
44
|
+
expect(persisted.contentHash).toBeTruthy();
|
|
45
|
+
|
|
46
|
+
// (c) pruneMessages emits the mechanical spill stub (no summary, no LLM)
|
|
47
|
+
const msgs = [
|
|
48
|
+
{
|
|
49
|
+
role: "toolResult",
|
|
50
|
+
toolCallId: "tc1",
|
|
51
|
+
toolName: "fetch",
|
|
52
|
+
content: [{ type: "text", text: body }],
|
|
53
|
+
isError: false,
|
|
54
|
+
timestamp: 1,
|
|
55
|
+
},
|
|
56
|
+
];
|
|
57
|
+
const { messages: out, pruned } = pruneMessages(msgs as any, indexer);
|
|
58
|
+
expect(pruned).toBe(true);
|
|
59
|
+
expect((out[0] as any).content[0].text).toContain(blobPathFor(dir, "sid", "tc1"));
|
|
60
|
+
expect((out[0] as any).content[0].text).not.toContain("Summarized in pruner summary");
|
|
61
|
+
|
|
62
|
+
// (d) reconstruct from the persisted entries: record still resolves, hash intact
|
|
63
|
+
const indexer2 = new ToolCallIndexer();
|
|
64
|
+
const fakeCtx = { sessionManager: { getBranch: () => entries } } as any;
|
|
65
|
+
indexer2.reconstructFromSession(fakeCtx);
|
|
66
|
+
const rec = indexer2.getRecord("tc1");
|
|
67
|
+
expect(rec?.spillPath).toBe(blobPathFor(dir, "sid", "tc1"));
|
|
68
|
+
expect(indexer2.lookupByContent("fetch", body)).toBe("tc1");
|
|
69
|
+
} finally {
|
|
70
|
+
await rm(dir, { recursive: true, force: true });
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
});
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { globToRegExp, isProtected } from "./protected.js";
|
|
3
|
+
|
|
4
|
+
describe("globToRegExp", () => {
|
|
5
|
+
test("** crosses path segments", () => {
|
|
6
|
+
expect(globToRegExp("**/skills/**/*.md").test("/Users/x/.agents/skills/release/SKILL.md")).toBe(true);
|
|
7
|
+
expect(globToRegExp("**/skills/**/*.md").test("a/skills/b/c/notes.md")).toBe(true);
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
test("**/ matches zero directories", () => {
|
|
11
|
+
expect(globToRegExp("**/SKILL.md").test("SKILL.md")).toBe(true);
|
|
12
|
+
expect(globToRegExp("**/skills/**/*.md").test("skills/foo.md")).toBe(true);
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test("* and ? are segment-local", () => {
|
|
16
|
+
expect(globToRegExp("skills/*.md").test("skills/a/b.md")).toBe(false);
|
|
17
|
+
expect(globToRegExp("skills/?.md").test("skills/a.md")).toBe(true);
|
|
18
|
+
expect(globToRegExp("skills/?.md").test("skills//x.md")).toBe(false);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test("regex metacharacters in pattern are literals", () => {
|
|
22
|
+
expect(globToRegExp("a+b/(c).md").test("a+b/(c).md")).toBe(true);
|
|
23
|
+
expect(globToRegExp("a.md").test("aXmd")).toBe(false);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
test("full-path anchored match", () => {
|
|
27
|
+
expect(globToRegExp("skills/a.md").test("xskills/a.md")).toBe(false);
|
|
28
|
+
expect(globToRegExp("skills/a.md").test("skills/a.md.bak")).toBe(false);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("case-sensitive", () => {
|
|
32
|
+
expect(globToRegExp("**/SKILL.md").test("a/skill.md")).toBe(false);
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
describe("isProtected", () => {
|
|
37
|
+
const cfg = { protectedTools: ["todowrite"], protectedPaths: ["**/skills/**/*.md"] };
|
|
38
|
+
|
|
39
|
+
test("matches by tool name regardless of args", () => {
|
|
40
|
+
expect(isProtected("todowrite", undefined, cfg)).toBe(true);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("matches by path glob", () => {
|
|
44
|
+
expect(isProtected("read", { path: "/h/skills/x/SKILL.md" }, cfg)).toBe(true);
|
|
45
|
+
expect(isProtected("read", { path: "/h/src/app.ts" }, cfg)).toBe(false);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("backslashes normalized to forward slashes", () => {
|
|
49
|
+
expect(isProtected("read", { path: "h\\skills\\x\\SKILL.md" }, cfg)).toBe(true);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test("non-string / missing path is not protected", () => {
|
|
53
|
+
expect(isProtected("read", { path: 42 }, cfg)).toBe(false);
|
|
54
|
+
expect(isProtected("read", {}, cfg)).toBe(false);
|
|
55
|
+
expect(isProtected("read", undefined, cfg)).toBe(false);
|
|
56
|
+
expect(isProtected("read", null, cfg)).toBe(false);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("empty config protects nothing", () => {
|
|
60
|
+
expect(isProtected("read", { path: "skills/a/SKILL.md" }, { protectedTools: [], protectedPaths: [] })).toBe(false);
|
|
61
|
+
});
|
|
62
|
+
});
|
package/src/protected.ts
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/** Structural pick of ContextPruneConfig — keeps this module dependency-free. */
|
|
2
|
+
export interface ProtectionConfig {
|
|
3
|
+
protectedTools: readonly string[];
|
|
4
|
+
protectedPaths: readonly string[];
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
// Compile-once: pattern -> RegExp is pure, so the cache never needs
|
|
8
|
+
// invalidation. Patterns only come from config arrays, so growth is bounded.
|
|
9
|
+
const patternCache = new Map<string, RegExp>();
|
|
10
|
+
|
|
11
|
+
export function globToRegExp(pattern: string): RegExp {
|
|
12
|
+
const cached = patternCache.get(pattern);
|
|
13
|
+
if (cached) return cached;
|
|
14
|
+
let re = "";
|
|
15
|
+
let i = 0;
|
|
16
|
+
while (i < pattern.length) {
|
|
17
|
+
const ch = pattern[i];
|
|
18
|
+
if (ch === "*") {
|
|
19
|
+
if (pattern[i + 1] === "*") {
|
|
20
|
+
if (pattern[i + 2] === "/") {
|
|
21
|
+
re += "(?:[^/]*/)*"; // `**/` — zero or more whole directories
|
|
22
|
+
i += 3;
|
|
23
|
+
} else {
|
|
24
|
+
re += ".*"; // bare `**`
|
|
25
|
+
i += 2;
|
|
26
|
+
}
|
|
27
|
+
} else {
|
|
28
|
+
re += "[^/]*"; // `*` — segment-local
|
|
29
|
+
i += 1;
|
|
30
|
+
}
|
|
31
|
+
} else if (ch === "?") {
|
|
32
|
+
re += "[^/]";
|
|
33
|
+
i += 1;
|
|
34
|
+
} else {
|
|
35
|
+
re += ch.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
36
|
+
i += 1;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const compiled = new RegExp(`^${re}$`);
|
|
40
|
+
patternCache.set(pattern, compiled);
|
|
41
|
+
return compiled;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function isProtected(toolName: string, args: unknown, config: ProtectionConfig): boolean {
|
|
45
|
+
if (config.protectedTools.includes(toolName)) return true;
|
|
46
|
+
if (config.protectedPaths.length === 0) return false;
|
|
47
|
+
const path = (args as Record<string, unknown> | null | undefined)?.path;
|
|
48
|
+
if (typeof path !== "string") return false;
|
|
49
|
+
const normalized = path.replace(/\\/g, "/");
|
|
50
|
+
return config.protectedPaths.some((p) => globToRegExp(p).test(normalized));
|
|
51
|
+
}
|