pi-condense 2.4.3 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/PRUNING.md +96 -54
- package/README.md +6 -1
- package/index.ts +28 -42
- package/package.json +1 -1
- package/src/batch-capture.test.ts +75 -1
- package/src/batch-capture.ts +22 -13
- package/src/chain-compressor.test.ts +114 -0
- package/src/chain-compressor.ts +29 -4
- package/src/chain-detector.test.ts +49 -0
- package/src/chain-detector.ts +7 -0
- package/src/chain-range-prune.test.ts +342 -7
- package/src/chain-range-prune.ts +161 -48
- package/src/commands.test.ts +31 -2
- package/src/commands.ts +25 -35
- package/src/config.test.ts +27 -1
- package/src/diagnostics.test.ts +114 -0
- package/src/diagnostics.ts +46 -0
- package/src/frontier.test.ts +138 -16
- package/src/frontier.ts +0 -1
- package/src/id-collision.integration.test.ts +251 -0
- package/src/indexer.test.ts +336 -0
- package/src/indexer.ts +168 -55
- package/src/occurrence-key.test.ts +57 -0
- package/src/occurrence-key.ts +36 -0
- package/src/orphan-sweep.test.ts +67 -0
- package/src/orphan-sweep.ts +40 -0
- package/src/oversized-spill.integration.test.ts +7 -2
- package/src/pruner.test.ts +471 -64
- package/src/pruner.ts +84 -54
- package/src/query-tool.test.ts +117 -0
- package/src/query-tool.ts +47 -31
- package/src/range-compression.integration.test.ts +7 -44
- package/src/recovery-grace.test.ts +13 -0
- package/src/recovery-grace.ts +12 -3
- package/src/spill.test.ts +108 -1
- package/src/spill.ts +5 -3
- package/src/summary-refs.test.ts +51 -1
- package/src/summary-refs.ts +15 -4
- package/src/test-support.ts +54 -0
- package/src/tree-browser.ts +2 -1
- package/src/types.ts +56 -49
- package/src/thinking-strip.test.ts +0 -257
- package/src/thinking-strip.ts +0 -83
package/src/indexer.ts
CHANGED
|
@@ -18,22 +18,27 @@ import {
|
|
|
18
18
|
type SummaryToolCallRef,
|
|
19
19
|
} from "./summary-refs.js";
|
|
20
20
|
import { hashToolResult } from "./content-hash.js";
|
|
21
|
+
import { bareToolCallId, occKey, parseOccKey } from "./occurrence-key.js";
|
|
21
22
|
|
|
22
23
|
export class ToolCallIndexer {
|
|
24
|
+
/** occurrence key (`id@resultTimestamp`, or bare id for legacy) -> record */
|
|
23
25
|
private index = new Map<string, ToolCallRecord>();
|
|
26
|
+
/** bare toolCallId -> its occurrence keys, in insertion order */
|
|
27
|
+
private bareIdToKeys = new Map<string, string[]>();
|
|
24
28
|
private aliasToToolCallId = new Map<string, string>();
|
|
25
29
|
private toolCallIdToAlias = new Map<string, string>();
|
|
26
30
|
private nextShortAliasNumber = 1;
|
|
27
31
|
/**
|
|
28
|
-
* hash
|
|
29
|
-
* (`addBatch`) and on `reconstructFromSession`.
|
|
30
|
-
* dedup pass via `lookupByContent`.
|
|
32
|
+
* hash -> original occurrence key (or legacy bare id). Populated as
|
|
33
|
+
* records enter the indexer (`addBatch`) and on `reconstructFromSession`.
|
|
34
|
+
* Drives the pre-flush dedup pass via `lookupByContent`.
|
|
31
35
|
*/
|
|
32
36
|
private contentHashToOriginal = new Map<string, string>();
|
|
33
37
|
/**
|
|
34
|
-
* Duplicate
|
|
35
|
-
*
|
|
36
|
-
* CUSTOM_TYPE_DEDUP_ALIAS entries on
|
|
38
|
+
* Duplicate occurrence key (or legacy bare id) -> original occurrence key
|
|
39
|
+
* (or legacy bare id). Populated by `registerDuplicate` during the
|
|
40
|
+
* pre-flush dedup pass and rebuilt from CUSTOM_TYPE_DEDUP_ALIAS entries on
|
|
41
|
+
* reconstruction.
|
|
37
42
|
*
|
|
38
43
|
* Both `isSummarized` and `resolveToolCallId` consult this map so
|
|
39
44
|
* `pruneMessages` stub-replaces dup toolResults and `context_tree_query`
|
|
@@ -56,6 +61,7 @@ export class ToolCallIndexer {
|
|
|
56
61
|
*/
|
|
57
62
|
reconstructFromSession(ctx: ExtensionContext): void {
|
|
58
63
|
this.index.clear();
|
|
64
|
+
this.bareIdToKeys.clear();
|
|
59
65
|
this.aliasToToolCallId.clear();
|
|
60
66
|
this.toolCallIdToAlias.clear();
|
|
61
67
|
this.contentHashToOriginal.clear();
|
|
@@ -74,12 +80,12 @@ export class ToolCallIndexer {
|
|
|
74
80
|
const data = (entry as any).data as IndexEntryData;
|
|
75
81
|
if (data && Array.isArray(data.toolCalls)) {
|
|
76
82
|
for (const toolCall of data.toolCalls) {
|
|
77
|
-
this.
|
|
83
|
+
const key = this.indexRecord(toolCall);
|
|
78
84
|
// First-seen wins so the contentHashToOriginal map matches what
|
|
79
85
|
// addBatch would have produced at append time.
|
|
80
86
|
const hash = toolCall.contentHash ?? hashToolResult(toolCall.toolName, toolCall.resultText);
|
|
81
87
|
if (!this.contentHashToOriginal.has(hash)) {
|
|
82
|
-
this.contentHashToOriginal.set(hash,
|
|
88
|
+
this.contentHashToOriginal.set(hash, key);
|
|
83
89
|
}
|
|
84
90
|
}
|
|
85
91
|
}
|
|
@@ -100,7 +106,7 @@ export class ToolCallIndexer {
|
|
|
100
106
|
.join("\n")
|
|
101
107
|
: "";
|
|
102
108
|
if (text) {
|
|
103
|
-
this.summaryBodies.push({ toolCallIds: refs.map((r) => r.toolCallId), text });
|
|
109
|
+
this.summaryBodies.push({ toolCallIds: refs.map((r) => occKey(r.toolCallId, r.resultTimestamp)), text });
|
|
104
110
|
}
|
|
105
111
|
continue;
|
|
106
112
|
}
|
|
@@ -122,26 +128,52 @@ export class ToolCallIndexer {
|
|
|
122
128
|
}
|
|
123
129
|
|
|
124
130
|
for (const data of dedupAliasEntries) {
|
|
125
|
-
|
|
126
|
-
const
|
|
131
|
+
const newKey = occKey(data.newToolCallId, data.newResultTimestamp);
|
|
132
|
+
const originalKey = occKey(data.originalToolCallId, data.originalResultTimestamp);
|
|
133
|
+
this.dedupAliasToOriginal.set(newKey, originalKey);
|
|
134
|
+
const originalShortRef = this.toolCallIdToAlias.get(originalKey);
|
|
127
135
|
if (originalShortRef) {
|
|
128
136
|
// Keep `getShortRefForToolCallId(dupId)` returning the SAME short ref
|
|
129
137
|
// as the original so pruneMessages emits a consistent `tN` for both.
|
|
130
|
-
this.toolCallIdToAlias.set(
|
|
138
|
+
this.toolCallIdToAlias.set(newKey, originalShortRef);
|
|
131
139
|
}
|
|
132
140
|
}
|
|
133
141
|
}
|
|
134
142
|
|
|
135
143
|
/**
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
144
|
+
* Indexes a single record under its occurrence key and updates the
|
|
145
|
+
* bare-id reverse index + legacy-shape tracking. Shared by `addBatch` and
|
|
146
|
+
* `reconstructFromSession` so both paths key identically.
|
|
147
|
+
*/
|
|
148
|
+
private indexRecord(record: ToolCallRecord): string {
|
|
149
|
+
const key = occKey(record.toolCallId, record.resultTimestamp);
|
|
150
|
+
this.index.set(key, record);
|
|
151
|
+
const keys = this.bareIdToKeys.get(record.toolCallId) ?? [];
|
|
152
|
+
if (!keys.includes(key)) keys.push(key);
|
|
153
|
+
this.bareIdToKeys.set(record.toolCallId, keys);
|
|
154
|
+
return key;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Returns true if the given occurrence key has been pruned - either
|
|
159
|
+
* because its full record is in the index, or because it has been
|
|
160
|
+
* registered as an alias of an already-indexed original via the
|
|
161
|
+
* content-hash dedup pass.
|
|
162
|
+
*
|
|
163
|
+
* STRICT lookup: no bare-id uniquification happens here, unlike
|
|
164
|
+
* `resolveToolCallId`/`getRecord`/`getRecordsForId`. A bare-id fallback
|
|
165
|
+
* in this method would be a silent correctness bug - it would report an
|
|
166
|
+
* unrelated LIVE tool result as summarized merely because an older
|
|
167
|
+
* occurrence of the same provider-reused id was summarized. Callers that
|
|
168
|
+
* need bare-id resolution use `resolveToolCallId` (unambiguous case) or
|
|
169
|
+
* `getRecordsForId` (all occurrences).
|
|
139
170
|
*
|
|
140
171
|
* `pruneMessages` uses this to decide whether to stub-replace a
|
|
141
|
-
* ToolResultMessage; both
|
|
172
|
+
* ToolResultMessage; both index and dedup-alias hits need the same
|
|
173
|
+
* treatment.
|
|
142
174
|
*/
|
|
143
|
-
isSummarized(
|
|
144
|
-
return this.index.has(
|
|
175
|
+
isSummarized(occurrenceKey: string): boolean {
|
|
176
|
+
return this.index.has(occurrenceKey) || this.dedupAliasToOriginal.has(occurrenceKey);
|
|
145
177
|
}
|
|
146
178
|
|
|
147
179
|
/**
|
|
@@ -158,9 +190,10 @@ export class ToolCallIndexer {
|
|
|
158
190
|
registerSummaryRefs(refs: SummaryToolCallRef[]): void {
|
|
159
191
|
for (const ref of refs) {
|
|
160
192
|
if (!ref.shortId || !ref.toolCallId) continue;
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
this.
|
|
193
|
+
const key = occKey(ref.toolCallId, ref.resultTimestamp);
|
|
194
|
+
if (ref.shortId !== key) {
|
|
195
|
+
this.aliasToToolCallId.set(ref.shortId, key);
|
|
196
|
+
this.toolCallIdToAlias.set(key, ref.shortId);
|
|
164
197
|
}
|
|
165
198
|
const match = /^t(\d+)$/.exec(ref.shortId);
|
|
166
199
|
if (match) {
|
|
@@ -174,44 +207,54 @@ export class ToolCallIndexer {
|
|
|
174
207
|
* runtime alias map.
|
|
175
208
|
*/
|
|
176
209
|
allocateSummaryRefs(batch: CapturedBatch): SummaryToolCallRef[] {
|
|
177
|
-
const
|
|
178
|
-
const { refs, nextIndex } = buildShortToolCallRefs(
|
|
210
|
+
const calls = batch.toolCalls.map((tc) => ({ toolCallId: tc.toolCallId, resultTimestamp: tc.resultTimestamp }));
|
|
211
|
+
const { refs, nextIndex } = buildShortToolCallRefs(calls, this.nextShortAliasNumber);
|
|
179
212
|
this.nextShortAliasNumber = nextIndex;
|
|
180
213
|
return refs;
|
|
181
214
|
}
|
|
182
215
|
|
|
183
216
|
/**
|
|
184
|
-
* Resolve a short alias, a duplicate's
|
|
185
|
-
* to the canonical
|
|
217
|
+
* Resolve a short alias, a duplicate's occurrence key, or a full occurrence
|
|
218
|
+
* key (or legacy bare id) to the canonical occurrence key backing it.
|
|
186
219
|
*
|
|
187
220
|
* Order:
|
|
188
|
-
* 1. Direct hit in `this.index` (canonical
|
|
189
|
-
* 2. Dedup alias → underlying original
|
|
190
|
-
* 3. Short-ref (`t3`) → underlying
|
|
221
|
+
* 1. Direct hit in `this.index` (canonical occurrence key).
|
|
222
|
+
* 2. Dedup alias → underlying original occurrence key.
|
|
223
|
+
* 3. Short-ref (`t3`) → underlying occurrence key.
|
|
224
|
+
* 4. Bare id with exactly ONE occurrence → that occurrence's key.
|
|
225
|
+
*
|
|
226
|
+
* A bare id with several occurrences resolves to undefined here — that
|
|
227
|
+
* ambiguity is fail-closed by design; callers that must handle collisions
|
|
228
|
+
* use `getRecordsForId`.
|
|
191
229
|
*
|
|
192
230
|
* Used by `getRecord`/`lookupToolCalls` so `context_tree_query` returns
|
|
193
231
|
* the original record for both short refs and dedup'd ids.
|
|
194
232
|
*/
|
|
195
|
-
resolveToolCallId(
|
|
196
|
-
if (this.index.has(
|
|
197
|
-
const dedupTarget = this.dedupAliasToOriginal.get(
|
|
233
|
+
resolveToolCallId(input: string): string | undefined {
|
|
234
|
+
if (this.index.has(input)) return input;
|
|
235
|
+
const dedupTarget = this.dedupAliasToOriginal.get(input);
|
|
198
236
|
if (dedupTarget) return dedupTarget;
|
|
199
|
-
|
|
237
|
+
const aliased = this.aliasToToolCallId.get(input);
|
|
238
|
+
if (aliased) return aliased;
|
|
239
|
+
const keys = this.bareIdToKeys.get(input);
|
|
240
|
+
if (keys && keys.length === 1) return keys[0];
|
|
241
|
+
return undefined;
|
|
200
242
|
}
|
|
201
243
|
|
|
202
244
|
/**
|
|
203
|
-
* Returns the short alias (e.g. "t1") registered for the given
|
|
204
|
-
*
|
|
205
|
-
* written before short-refs were introduced map shortId ===
|
|
206
|
-
* and intentionally return undefined here so callers (e.g. the
|
|
207
|
-
*
|
|
245
|
+
* Returns the short alias (e.g. "t1") registered for the given occurrence
|
|
246
|
+
* key (or legacy bare id), or undefined if none was registered. Legacy
|
|
247
|
+
* summaries written before short-refs were introduced map shortId === key
|
|
248
|
+
* and intentionally return undefined here so callers (e.g. the pruner
|
|
249
|
+
* stub) can fall back to the key itself.
|
|
208
250
|
*/
|
|
209
|
-
getShortRefForToolCallId(
|
|
210
|
-
return this.toolCallIdToAlias.get(
|
|
251
|
+
getShortRefForToolCallId(occurrenceKey: string): string | undefined {
|
|
252
|
+
return this.toolCallIdToAlias.get(occurrenceKey);
|
|
211
253
|
}
|
|
212
254
|
|
|
213
255
|
/**
|
|
214
|
-
* Look up a single record by
|
|
256
|
+
* Look up a single record by occurrence key, short alias, or (unambiguous)
|
|
257
|
+
* bare id (used by query tool).
|
|
215
258
|
*/
|
|
216
259
|
getRecord(toolCallIdOrAlias: string): ToolCallRecord | undefined {
|
|
217
260
|
const resolved = this.resolveToolCallId(toolCallIdOrAlias);
|
|
@@ -220,7 +263,8 @@ export class ToolCallIndexer {
|
|
|
220
263
|
}
|
|
221
264
|
|
|
222
265
|
/**
|
|
223
|
-
* Looks up multiple tool call records by
|
|
266
|
+
* Looks up multiple tool call records by occurrence key / short alias.
|
|
267
|
+
* Skips any not found.
|
|
224
268
|
*/
|
|
225
269
|
lookupToolCalls(toolCallIds: string[]): ToolCallRecord[] {
|
|
226
270
|
const results: ToolCallRecord[] = [];
|
|
@@ -233,6 +277,67 @@ export class ToolCallIndexer {
|
|
|
233
277
|
return results;
|
|
234
278
|
}
|
|
235
279
|
|
|
280
|
+
/**
|
|
281
|
+
* Every record a bare toolCallId, occurrence key, or short ref can denote,
|
|
282
|
+
* sorted by `resultTimestamp ?? timestamp` ascending. A bare id with
|
|
283
|
+
* multiple occurrences returns all of them; an occurrence key/short ref
|
|
284
|
+
* returns exactly the one record it resolves to.
|
|
285
|
+
*
|
|
286
|
+
* The sort is display order for a multi-match listing, not a causal
|
|
287
|
+
* clock: it mixes a tool-result timestamp with a batch-capture timestamp,
|
|
288
|
+
* which are both epoch ms from the same session and adequate for a
|
|
289
|
+
* listing but not a strict ordering guarantee.
|
|
290
|
+
*/
|
|
291
|
+
getRecordsForId(input: string): ToolCallRecord[] {
|
|
292
|
+
const keys = this.bareIdToKeys.get(input);
|
|
293
|
+
const records = (keys ?? [])
|
|
294
|
+
.map((k) => this.index.get(k))
|
|
295
|
+
.filter((r): r is ToolCallRecord => r !== undefined);
|
|
296
|
+
|
|
297
|
+
// Dedup-alias occurrences aren't tracked in `bareIdToKeys` (it only
|
|
298
|
+
// covers indexed records), so a bare id whose collision was content-
|
|
299
|
+
// deduplicated would otherwise be silently omitted here. Resolve each
|
|
300
|
+
// matching alias to the record it aliases, but label it with the
|
|
301
|
+
// ALIAS's own occurrence timestamp (not the original's) so a reader can
|
|
302
|
+
// tell the two occurrences apart.
|
|
303
|
+
for (const [aliasKey, originalKey] of this.dedupAliasToOriginal) {
|
|
304
|
+
if (bareToolCallId(aliasKey) !== input) continue;
|
|
305
|
+
const original = this.index.get(originalKey);
|
|
306
|
+
if (!original) continue;
|
|
307
|
+
const { resultTimestamp } = parseOccKey(aliasKey);
|
|
308
|
+
records.push({ ...original, resultTimestamp });
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
if (records.length > 0) {
|
|
312
|
+
return records.sort((a, b) => (a.resultTimestamp ?? a.timestamp) - (b.resultTimestamp ?? b.timestamp));
|
|
313
|
+
}
|
|
314
|
+
const record = this.getRecord(input);
|
|
315
|
+
return record ? [record] : [];
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* True when the bare id is LEGACY-ONLY: a record is indexed under the
|
|
320
|
+
* bare key AND that bare id has no occurrence-keyed siblings. A legacy
|
|
321
|
+
* record has no `resultTimestamp`, so `occKey` keys it under its bare id
|
|
322
|
+
* - meaning it already lives in `index` under that exact string; this is
|
|
323
|
+
* a derivation, not a separate container.
|
|
324
|
+
*
|
|
325
|
+
* A session that spans the upgrade can hold BOTH a legacy bare record
|
|
326
|
+
* and modern occurrence records under the same bare id (e.g. legacy
|
|
327
|
+
* `bash_23` plus `bash_23@2150`). In that mixed case this must return
|
|
328
|
+
* false: a live, unrelated `bash_23@9150` result is not the legacy one,
|
|
329
|
+
* and a permissive true here would stub it with the stale legacy content
|
|
330
|
+
* - the exact collision this bare-id path exists to avoid re-introducing.
|
|
331
|
+
* `bareIdToKeys` already tracks every key minted under a bare id, so a
|
|
332
|
+
* single-key entry is the discriminant. The pruner's only sanctioned
|
|
333
|
+
* bare-id path.
|
|
334
|
+
*/
|
|
335
|
+
hasLegacyBareRecord(toolCallId: string): boolean {
|
|
336
|
+
if (!this.index.has(toolCallId)) return false;
|
|
337
|
+
const keys = this.bareIdToKeys.get(toolCallId);
|
|
338
|
+
return keys !== undefined && keys.length === 1;
|
|
339
|
+
}
|
|
340
|
+
|
|
236
341
|
/**
|
|
237
342
|
* Returns the toolCallId of an already-indexed record whose
|
|
238
343
|
* `(toolName, normalize(resultText))` matches the supplied input, or
|
|
@@ -250,25 +355,32 @@ export class ToolCallIndexer {
|
|
|
250
355
|
}
|
|
251
356
|
|
|
252
357
|
/**
|
|
253
|
-
* Registers `
|
|
254
|
-
* id reuses the original's short alias
|
|
255
|
-
* `tN` ref for both) and is persisted
|
|
256
|
-
* reconstruction can replay it later.
|
|
358
|
+
* Registers `newKey` as a duplicate of `originalKey` (each an occurrence
|
|
359
|
+
* key, or a legacy bare id). The new id reuses the original's short alias
|
|
360
|
+
* (so `pruneMessages` emits the same `tN` ref for both) and is persisted
|
|
361
|
+
* via the supplied `appendEntry` so reconstruction can replay it later.
|
|
257
362
|
*
|
|
258
|
-
* No-op when `
|
|
363
|
+
* No-op when `newKey === originalKey` (defensive).
|
|
259
364
|
*/
|
|
260
365
|
registerDuplicate(
|
|
261
|
-
|
|
262
|
-
|
|
366
|
+
newKey: string,
|
|
367
|
+
originalKey: string,
|
|
263
368
|
appendEntry: (customType: string, data?: unknown) => void,
|
|
264
369
|
): void {
|
|
265
|
-
if (
|
|
266
|
-
this.dedupAliasToOriginal.set(
|
|
267
|
-
const originalShortRef = this.toolCallIdToAlias.get(
|
|
370
|
+
if (newKey === originalKey) return;
|
|
371
|
+
this.dedupAliasToOriginal.set(newKey, originalKey);
|
|
372
|
+
const originalShortRef = this.toolCallIdToAlias.get(originalKey);
|
|
268
373
|
if (originalShortRef) {
|
|
269
|
-
this.toolCallIdToAlias.set(
|
|
374
|
+
this.toolCallIdToAlias.set(newKey, originalShortRef);
|
|
270
375
|
}
|
|
271
|
-
const
|
|
376
|
+
const { toolCallId: newToolCallId, resultTimestamp: newResultTimestamp } = parseOccKey(newKey);
|
|
377
|
+
const { toolCallId: originalToolCallId, resultTimestamp: originalResultTimestamp } = parseOccKey(originalKey);
|
|
378
|
+
const payload: DedupAliasEntryData = {
|
|
379
|
+
newToolCallId,
|
|
380
|
+
originalToolCallId,
|
|
381
|
+
...(newResultTimestamp !== undefined ? { newResultTimestamp } : {}),
|
|
382
|
+
...(originalResultTimestamp !== undefined ? { originalResultTimestamp } : {}),
|
|
383
|
+
};
|
|
272
384
|
appendEntry(CUSTOM_TYPE_DEDUP_ALIAS, payload);
|
|
273
385
|
}
|
|
274
386
|
|
|
@@ -372,19 +484,20 @@ export class ToolCallIndexer {
|
|
|
372
484
|
isError: tc.isError,
|
|
373
485
|
turnIndex: batch.turnIndex,
|
|
374
486
|
timestamp: batch.timestamp,
|
|
487
|
+
...(tc.resultTimestamp !== undefined ? { resultTimestamp: tc.resultTimestamp } : {}),
|
|
375
488
|
...(tc.spillPath !== undefined ? { spillPath: tc.spillPath } : {}),
|
|
376
489
|
...(tc.spillBytes !== undefined ? { spillBytes: tc.spillBytes } : {}),
|
|
377
490
|
...(tc.resultPreview !== undefined ? { resultPreview: tc.resultPreview } : {}),
|
|
378
491
|
...(tc.contentHash !== undefined ? { contentHash: tc.contentHash } : {}),
|
|
379
492
|
};
|
|
380
|
-
this.
|
|
493
|
+
const key = this.indexRecord(record);
|
|
381
494
|
records.push(record);
|
|
382
495
|
// Populate the dedup hash map AFTER the record is indexed so a future
|
|
383
496
|
// flush can dedup against this record. First-seen wins to keep the
|
|
384
497
|
// canonical id stable across multiple identical entries.
|
|
385
498
|
const hash = record.contentHash ?? hashToolResult(record.toolName, record.resultText);
|
|
386
499
|
if (!this.contentHashToOriginal.has(hash)) {
|
|
387
|
-
this.contentHashToOriginal.set(hash,
|
|
500
|
+
this.contentHashToOriginal.set(hash, key);
|
|
388
501
|
}
|
|
389
502
|
}
|
|
390
503
|
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { occKey, parseOccKey, bareToolCallId, resultTimestampOf } from "./occurrence-key.js";
|
|
3
|
+
|
|
4
|
+
describe("occKey", () => {
|
|
5
|
+
test("joins id and timestamp", () => {
|
|
6
|
+
expect(occKey("bash_23", 1700)).toBe("bash_23@1700");
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
test("returns the bare id when the timestamp is absent (legacy shape)", () => {
|
|
10
|
+
expect(occKey("bash_23", undefined)).toBe("bash_23");
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
test("round-trips through parseOccKey", () => {
|
|
14
|
+
expect(parseOccKey("bash_23@1700")).toEqual({ toolCallId: "bash_23", resultTimestamp: 1700 });
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test("treats a key with no separator as a bare id", () => {
|
|
18
|
+
expect(parseOccKey("bash_23")).toEqual({ toolCallId: "bash_23" });
|
|
19
|
+
expect("resultTimestamp" in parseOccKey("bash_23")).toBe(false);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test("treats a trailing non-numeric segment as part of a bare id", () => {
|
|
23
|
+
// github-copilot ids embed base64 payloads after a '|' and may contain '@'
|
|
24
|
+
expect(parseOccKey("call_abc@sha")).toEqual({ toolCallId: "call_abc@sha" });
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("splits on the LAST separator so ids containing '@' survive", () => {
|
|
28
|
+
expect(parseOccKey("call@abc@1700")).toEqual({ toolCallId: "call@abc", resultTimestamp: 1700 });
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("bareToolCallId strips the occurrence suffix", () => {
|
|
32
|
+
expect(bareToolCallId("bash_23@1700")).toBe("bash_23");
|
|
33
|
+
expect(bareToolCallId("bash_23")).toBe("bash_23");
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
describe("resultTimestampOf", () => {
|
|
38
|
+
test("passes a number through", () => {
|
|
39
|
+
expect(resultTimestampOf(1700)).toBe(1700);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("returns undefined for undefined", () => {
|
|
43
|
+
expect(resultTimestampOf(undefined)).toBeUndefined();
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("returns undefined for a string", () => {
|
|
47
|
+
expect(resultTimestampOf("1700")).toBeUndefined();
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("returns undefined for null", () => {
|
|
51
|
+
expect(resultTimestampOf(null)).toBeUndefined();
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("passes NaN through (typeof NaN === 'number'; a NaN timestamp cannot arise from pi's typed ToolResultMessage.timestamp)", () => {
|
|
55
|
+
expect(resultTimestampOf(NaN)).toBeNaN();
|
|
56
|
+
});
|
|
57
|
+
});
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider tool-call ids are unique only within one response (see
|
|
3
|
+
* doc/specs/2026-08-12-toolcall-id-collisions.md), so bare ids cannot key
|
|
4
|
+
* session-durable records. The ToolResultMessage timestamp is the one
|
|
5
|
+
* discriminant readable identically at capture time and at render time.
|
|
6
|
+
*
|
|
7
|
+
* A key with no parsable numeric suffix IS a bare id - that is the legacy
|
|
8
|
+
* shape for records persisted before this field existed.
|
|
9
|
+
*/
|
|
10
|
+
const SEP = "@";
|
|
11
|
+
|
|
12
|
+
export function occKey(toolCallId: string, resultTimestamp?: number): string {
|
|
13
|
+
return resultTimestamp === undefined ? toolCallId : `${toolCallId}${SEP}${resultTimestamp}`;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Narrows an untrusted value to the numeric timestamp discriminant, or
|
|
18
|
+
* `undefined` if it isn't a number. Shared by every ingress point that reads
|
|
19
|
+
* a ToolResultMessage-shaped `.timestamp` off data of uncertain provenance
|
|
20
|
+
* (live turn events, session JSON, summary details JSON).
|
|
21
|
+
*/
|
|
22
|
+
export function resultTimestampOf(value: unknown): number | undefined {
|
|
23
|
+
return typeof value === "number" ? value : undefined;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function parseOccKey(key: string): { toolCallId: string; resultTimestamp?: number } {
|
|
27
|
+
const i = key.lastIndexOf(SEP);
|
|
28
|
+
if (i <= 0) return { toolCallId: key };
|
|
29
|
+
const suffix = key.slice(i + 1);
|
|
30
|
+
if (!/^\d+$/.test(suffix)) return { toolCallId: key };
|
|
31
|
+
return { toolCallId: key.slice(0, i), resultTimestamp: Number(suffix) };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function bareToolCallId(key: string): string {
|
|
35
|
+
return parseOccKey(key).toolCallId;
|
|
36
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { sweepOrphanToolResults } from "./orphan-sweep.js";
|
|
3
|
+
|
|
4
|
+
const asst = (ts: number, ids: string[]) => ({
|
|
5
|
+
role: "assistant",
|
|
6
|
+
content: ids.map((id) => ({ type: "toolCall", id, name: "bash", input: {} })),
|
|
7
|
+
timestamp: ts,
|
|
8
|
+
});
|
|
9
|
+
const res = (ts: number, id: string) => ({
|
|
10
|
+
role: "toolResult",
|
|
11
|
+
toolCallId: id,
|
|
12
|
+
toolName: "bash",
|
|
13
|
+
content: [{ type: "text", text: "ok" }],
|
|
14
|
+
isError: false,
|
|
15
|
+
timestamp: ts,
|
|
16
|
+
});
|
|
17
|
+
const user = (ts: number) => ({ role: "user", content: [{ type: "text", text: "go" }], timestamp: ts });
|
|
18
|
+
|
|
19
|
+
describe("sweepOrphanToolResults", () => {
|
|
20
|
+
test("returns the SAME array reference when there is no orphan", () => {
|
|
21
|
+
const msgs = [user(1), asst(2, ["a"]), res(3, "a")];
|
|
22
|
+
const out = sweepOrphanToolResults(msgs);
|
|
23
|
+
expect(out.messages).toBe(msgs);
|
|
24
|
+
expect(out.sweptIds).toEqual([]);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("removes a toolResult whose call was never opened", () => {
|
|
28
|
+
const msgs = [user(1), asst(2, ["a"]), res(3, "a"), res(4, "ghost")];
|
|
29
|
+
const out = sweepOrphanToolResults(msgs);
|
|
30
|
+
expect(out.sweptIds).toEqual(["ghost"]);
|
|
31
|
+
expect(out.messages).toHaveLength(3);
|
|
32
|
+
expect(out.messages.some((m: any) => m.toolCallId === "ghost")).toBe(false);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("open-call tracking is per turn: a validly used id does not license a later orphan", () => {
|
|
36
|
+
// 'a' is opened and consumed in turn 1; the later 'a' result has no opener
|
|
37
|
+
const msgs = [user(1), asst(2, ["a"]), res(3, "a"), asst(4, ["b"]), res(5, "b"), res(6, "a")];
|
|
38
|
+
const out = sweepOrphanToolResults(msgs);
|
|
39
|
+
expect(out.sweptIds).toEqual(["a"]);
|
|
40
|
+
expect(out.messages).toHaveLength(5);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("legitimate reuse of the same bare id across turns is kept, not swept", () => {
|
|
44
|
+
const msgs = [asst(1, ["a"]), res(2, "a"), asst(3, ["a"]), res(4, "a")];
|
|
45
|
+
const out = sweepOrphanToolResults(msgs);
|
|
46
|
+
expect(out.sweptIds).toEqual([]);
|
|
47
|
+
expect(out.messages).toBe(msgs);
|
|
48
|
+
expect(out.messages).toHaveLength(4);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("a duplicate result for one call is swept (id consumed once)", () => {
|
|
52
|
+
const msgs = [asst(1, ["a"]), res(2, "a"), res(3, "a")];
|
|
53
|
+
const out = sweepOrphanToolResults(msgs);
|
|
54
|
+
expect(out.sweptIds).toEqual(["a"]);
|
|
55
|
+
expect(out.messages).toHaveLength(2);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("keeps results for every id of a multi-call assistant turn", () => {
|
|
59
|
+
const msgs = [asst(1, ["a", "b"]), res(2, "a"), res(3, "b")];
|
|
60
|
+
expect(sweepOrphanToolResults(msgs).messages).toBe(msgs);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("non-assistant messages between a call and its result do not close the open set", () => {
|
|
64
|
+
const msgs = [asst(1, ["a"]), { role: "custom", customType: "x", timestamp: 2 }, res(3, "a")];
|
|
65
|
+
expect(sweepOrphanToolResults(msgs).messages).toBe(msgs);
|
|
66
|
+
});
|
|
67
|
+
});
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* pi-ai repairs orphan tool calls only; providers reject orphan tool results.
|
|
3
|
+
*
|
|
4
|
+
* Open-call tracking is PER TURN: an assistant message replaces the open set
|
|
5
|
+
* with its own toolCall ids. A cumulative seen-set would let an id used
|
|
6
|
+
* validly in an early turn license a later genuine orphan - exactly the
|
|
7
|
+
* id-collision case this exists for.
|
|
8
|
+
*
|
|
9
|
+
* Returns the input array reference when nothing is swept, preserving the
|
|
10
|
+
* no-op / prompt-cache-prefix invariant of
|
|
11
|
+
* doc/specs/2026-08-04-pruner-noop-serialization.md.
|
|
12
|
+
*/
|
|
13
|
+
export function sweepOrphanToolResults(messages: any[]): { messages: any[]; sweptIds: string[] } {
|
|
14
|
+
let open = new Set<string>();
|
|
15
|
+
const orphanIndices = new Set<number>();
|
|
16
|
+
const sweptIds: string[] = [];
|
|
17
|
+
|
|
18
|
+
for (let i = 0; i < messages.length; i++) {
|
|
19
|
+
const msg = messages[i];
|
|
20
|
+
if (msg.role === "assistant") {
|
|
21
|
+
open = new Set(
|
|
22
|
+
(msg.content ?? [])
|
|
23
|
+
.filter((c: any) => c.type === "toolCall")
|
|
24
|
+
.map((c: any) => c.id as string),
|
|
25
|
+
);
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
if (msg.role === "toolResult") {
|
|
29
|
+
if (open.has(msg.toolCallId)) {
|
|
30
|
+
open.delete(msg.toolCallId);
|
|
31
|
+
} else {
|
|
32
|
+
orphanIndices.add(i);
|
|
33
|
+
sweptIds.push(msg.toolCallId);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if (orphanIndices.size === 0) return { messages, sweptIds: [] };
|
|
39
|
+
return { messages: messages.filter((_, i) => !orphanIndices.has(i)), sweptIds };
|
|
40
|
+
}
|
|
@@ -45,6 +45,11 @@ describe("oversized spill end-to-end", () => {
|
|
|
45
45
|
|
|
46
46
|
// (c) pruneMessages emits the mechanical spill stub (no summary, no LLM)
|
|
47
47
|
const msgs = [
|
|
48
|
+
{
|
|
49
|
+
role: "assistant",
|
|
50
|
+
content: [{ type: "toolCall", id: "tc1", name: "fetch", input: {} }],
|
|
51
|
+
timestamp: 0,
|
|
52
|
+
},
|
|
48
53
|
{
|
|
49
54
|
role: "toolResult",
|
|
50
55
|
toolCallId: "tc1",
|
|
@@ -56,8 +61,8 @@ describe("oversized spill end-to-end", () => {
|
|
|
56
61
|
];
|
|
57
62
|
const { messages: out, pruned } = pruneMessages(msgs as any, indexer);
|
|
58
63
|
expect(pruned).toBe(true);
|
|
59
|
-
expect((out[
|
|
60
|
-
expect((out[
|
|
64
|
+
expect((out[1] as any).content[0].text).toContain(blobPathFor(dir, "sid", "tc1"));
|
|
65
|
+
expect((out[1] as any).content[0].text).not.toContain("Summarized in pruner summary");
|
|
61
66
|
|
|
62
67
|
// (d) reconstruct from the persisted entries: record still resolves, hash intact
|
|
63
68
|
const indexer2 = new ToolCallIndexer();
|