pi-condense 2.4.3 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/PRUNING.md +96 -54
- package/README.md +6 -1
- package/index.ts +28 -42
- package/package.json +1 -1
- package/src/batch-capture.test.ts +75 -1
- package/src/batch-capture.ts +22 -13
- package/src/chain-compressor.test.ts +114 -0
- package/src/chain-compressor.ts +29 -4
- package/src/chain-detector.test.ts +49 -0
- package/src/chain-detector.ts +7 -0
- package/src/chain-range-prune.test.ts +342 -7
- package/src/chain-range-prune.ts +161 -48
- package/src/commands.test.ts +31 -2
- package/src/commands.ts +25 -35
- package/src/config.test.ts +27 -1
- package/src/diagnostics.test.ts +114 -0
- package/src/diagnostics.ts +46 -0
- package/src/frontier.test.ts +138 -16
- package/src/frontier.ts +0 -1
- package/src/id-collision.integration.test.ts +251 -0
- package/src/indexer.test.ts +336 -0
- package/src/indexer.ts +168 -55
- package/src/occurrence-key.test.ts +57 -0
- package/src/occurrence-key.ts +36 -0
- package/src/orphan-sweep.test.ts +67 -0
- package/src/orphan-sweep.ts +40 -0
- package/src/oversized-spill.integration.test.ts +7 -2
- package/src/pruner.test.ts +471 -64
- package/src/pruner.ts +84 -54
- package/src/query-tool.test.ts +117 -0
- package/src/query-tool.ts +47 -31
- package/src/range-compression.integration.test.ts +7 -44
- package/src/recovery-grace.test.ts +13 -0
- package/src/recovery-grace.ts +12 -3
- package/src/spill.test.ts +108 -1
- package/src/spill.ts +5 -3
- package/src/summary-refs.test.ts +51 -1
- package/src/summary-refs.ts +15 -4
- package/src/test-support.ts +54 -0
- package/src/tree-browser.ts +2 -1
- package/src/types.ts +56 -49
- package/src/thinking-strip.test.ts +0 -257
- package/src/thinking-strip.ts +0 -83
|
@@ -4,9 +4,12 @@ import {
|
|
|
4
4
|
buildSyntheticChainMessage,
|
|
5
5
|
isPerBatchSummaryMessage,
|
|
6
6
|
perBatchSummaryOverlapsDropped,
|
|
7
|
+
resolveRange,
|
|
7
8
|
withoutThinkingBlocks,
|
|
8
9
|
} from "./chain-range-prune.js";
|
|
10
|
+
import { CUSTOM_TYPE_SUMMARY } from "./types.js";
|
|
9
11
|
import type { ChainCompressionEntry } from "./types.js";
|
|
12
|
+
import { expectNoOrphanToolResults, expectZeroOrphanSweep } from "./test-support.js";
|
|
10
13
|
|
|
11
14
|
|
|
12
15
|
function userMsg(timestamp: number, text = "do the thing"): any {
|
|
@@ -56,6 +59,19 @@ function summaryMsg(timestamp: number, toolCallIds: string[]): any {
|
|
|
56
59
|
};
|
|
57
60
|
}
|
|
58
61
|
|
|
62
|
+
function summaryMsgOcc(timestamp: number, refs: { toolCallId: string; resultTimestamp: number }[]): any {
|
|
63
|
+
return {
|
|
64
|
+
role: "custom",
|
|
65
|
+
customType: "context-prune-summary",
|
|
66
|
+
content: "summary text",
|
|
67
|
+
display: false,
|
|
68
|
+
details: {
|
|
69
|
+
toolCallRefs: refs.map((r, i) => ({ shortId: `t${i + 1}`, toolCallId: r.toolCallId, resultTimestamp: r.resultTimestamp })),
|
|
70
|
+
},
|
|
71
|
+
timestamp,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
59
75
|
function entry(
|
|
60
76
|
blockId: string,
|
|
61
77
|
startUserTimestamp: number,
|
|
@@ -92,19 +108,31 @@ describe("isPerBatchSummaryMessage", () => {
|
|
|
92
108
|
});
|
|
93
109
|
|
|
94
110
|
describe("perBatchSummaryOverlapsDropped", () => {
|
|
95
|
-
test("returns true when at least one toolCallRef is in the dropped set", () => {
|
|
111
|
+
test("returns true when at least one legacy (no resultTimestamp) toolCallRef is in the dropped bare set", () => {
|
|
96
112
|
const msg = summaryMsg(999, ["tc1", "tc2"]);
|
|
97
|
-
expect(perBatchSummaryOverlapsDropped(msg, new Set(["tc1"]))).toBe(true);
|
|
113
|
+
expect(perBatchSummaryOverlapsDropped(msg, new Set(), new Set(["tc1"]))).toBe(true);
|
|
98
114
|
});
|
|
99
115
|
|
|
100
|
-
test("returns false when no toolCallRefs are in the dropped set", () => {
|
|
116
|
+
test("returns false when no legacy toolCallRefs are in the dropped bare set", () => {
|
|
101
117
|
const msg = summaryMsg(999, ["tc3"]);
|
|
102
|
-
expect(perBatchSummaryOverlapsDropped(msg, new Set(["tc1", "tc2"]))).toBe(false);
|
|
118
|
+
expect(perBatchSummaryOverlapsDropped(msg, new Set(), new Set(["tc1", "tc2"]))).toBe(false);
|
|
103
119
|
});
|
|
104
120
|
|
|
105
121
|
test("returns false when details is missing", () => {
|
|
106
122
|
const msg = { role: "custom", customType: "context-prune-summary", content: "x", timestamp: 1 };
|
|
107
|
-
expect(perBatchSummaryOverlapsDropped(msg, new Set(["tc1"]))).toBe(false);
|
|
123
|
+
expect(perBatchSummaryOverlapsDropped(msg, new Set(), new Set(["tc1"]))).toBe(false);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test("an occurrence-keyed ref matches only its exact occurrence, not the bare id of a different one", () => {
|
|
127
|
+
const msg = summaryMsgOcc(999, [{ toolCallId: "tc1", resultTimestamp: 3150 }]);
|
|
128
|
+
// Dropped set contains a DIFFERENT occurrence of the same bare id (tc1@2000), plus the bare
|
|
129
|
+
// id in the legacy fallback set — neither should cause a match for the live tc1@3150 ref.
|
|
130
|
+
expect(perBatchSummaryOverlapsDropped(msg, new Set(["tc1@2000"]), new Set(["tc1"]))).toBe(false);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("an occurrence-keyed ref matches its exact occurrence in droppedOccKeys", () => {
|
|
134
|
+
const msg = summaryMsgOcc(999, [{ toolCallId: "tc1", resultTimestamp: 2000 }]);
|
|
135
|
+
expect(perBatchSummaryOverlapsDropped(msg, new Set(["tc1@2000"]), new Set())).toBe(true);
|
|
108
136
|
});
|
|
109
137
|
});
|
|
110
138
|
|
|
@@ -245,12 +273,39 @@ describe("applyChainCompressions", () => {
|
|
|
245
273
|
});
|
|
246
274
|
|
|
247
275
|
test("does not suppress per-batch summary whose toolCallRefs do not overlap", () => {
|
|
276
|
+
// Summary placed AFTER the resolved range (agent-message batching order) -
|
|
277
|
+
// positionally-inside summaries are dropped unconditionally regardless of
|
|
278
|
+
// coverage; coverage-based suppression only applies outside the range.
|
|
248
279
|
const msgs = [
|
|
249
280
|
userMsg(100),
|
|
250
281
|
assistantWithTools(200, ["tc1"]),
|
|
251
282
|
toolResult(300, "tc1"),
|
|
252
|
-
summaryMsg(350, ["tc2"]), // different toolCallId
|
|
253
283
|
assistantText(400),
|
|
284
|
+
summaryMsg(450, ["tc2"]), // different toolCallId, outside the range
|
|
285
|
+
];
|
|
286
|
+
const e = entry("b1", 100, ["tc1"], 400);
|
|
287
|
+
const result = applyChainCompressions(msgs, [e], noopSummary, false);
|
|
288
|
+
const hasCustomSummary = result.some(
|
|
289
|
+
(m: any) => m.role === "custom" && m.customType === "context-prune-summary",
|
|
290
|
+
);
|
|
291
|
+
expect(hasCustomSummary).toBe(true);
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
test("a live turn's per-batch summary survives even when it reuses a compressed chain's bare id (occurrence collision)", () => {
|
|
295
|
+
// Compressed chain drops tc1@300 (dropped range: user@100..assistant@400).
|
|
296
|
+
// A later, LIVE turn reuses the bare id "tc1" with a different resultTimestamp (3150) and
|
|
297
|
+
// is never dropped (outside the resolved range). Its per-batch summary references that
|
|
298
|
+
// live occurrence and must not be suppressed by the earlier drop of the same bare id.
|
|
299
|
+
const msgs = [
|
|
300
|
+
userMsg(100),
|
|
301
|
+
assistantWithTools(200, ["tc1"]),
|
|
302
|
+
toolResult(300, "tc1"),
|
|
303
|
+
assistantText(400),
|
|
304
|
+
userMsg(3000),
|
|
305
|
+
assistantWithTools(3100, ["tc1"]),
|
|
306
|
+
toolResult(3150, "tc1"),
|
|
307
|
+
summaryMsgOcc(3200, [{ toolCallId: "tc1", resultTimestamp: 3150 }]),
|
|
308
|
+
assistantText(3300),
|
|
254
309
|
];
|
|
255
310
|
const e = entry("b1", 100, ["tc1"], 400);
|
|
256
311
|
const result = applyChainCompressions(msgs, [e], noopSummary, false);
|
|
@@ -496,7 +551,7 @@ describe("applyChainCompressions", () => {
|
|
|
496
551
|
};
|
|
497
552
|
const messages = [
|
|
498
553
|
{ role: "user", timestamp: 1, content: [{ type: "text", text: "go" }] },
|
|
499
|
-
{ role: "assistant", timestamp:
|
|
554
|
+
{ role: "assistant", timestamp: 9, content: [{ type: "text", text: "done" }] },
|
|
500
555
|
];
|
|
501
556
|
const out = applyChainCompressions(messages, [e] as any, () => "SUMMARY", false);
|
|
502
557
|
const synthetic = out.find((m: any) => typeof m.content?.[0]?.text === "string" && m.content[0].text.startsWith("<compressed-chain"));
|
|
@@ -520,3 +575,283 @@ describe("applyChainCompressions", () => {
|
|
|
520
575
|
expect(synthetic?.content[0].text).toContain("{b99}");
|
|
521
576
|
});
|
|
522
577
|
});
|
|
578
|
+
|
|
579
|
+
describe("resolveRange", () => {
|
|
580
|
+
const base = () => [
|
|
581
|
+
{ role: "user", content: [{ type: "text", text: "go" }], timestamp: 1000 },
|
|
582
|
+
{ role: "assistant", content: [{ type: "toolCall", id: "bash_1", name: "bash", input: {} }], timestamp: 1100 },
|
|
583
|
+
{ role: "toolResult", toolCallId: "bash_1", toolName: "bash", content: [{ type: "text", text: "x" }], isError: false, timestamp: 1150 },
|
|
584
|
+
{ role: "assistant", content: [{ type: "text", text: "done" }], timestamp: 1200 },
|
|
585
|
+
];
|
|
586
|
+
const entry = (over: any = {}) => ({
|
|
587
|
+
blockId: "b1",
|
|
588
|
+
startUserTimestamp: 1000,
|
|
589
|
+
droppedToolCallIds: ["bash_1"],
|
|
590
|
+
droppedOccurrenceKeys: ["bash_1@1150"],
|
|
591
|
+
finalAssistantTimestamp: 1200,
|
|
592
|
+
toolRefs: ["t1"],
|
|
593
|
+
compressedAt: 5000,
|
|
594
|
+
...over,
|
|
595
|
+
});
|
|
596
|
+
|
|
597
|
+
test("resolves the unique role-gated boundaries", () => {
|
|
598
|
+
expect(resolveRange(entry(), base())).toEqual({ startIndex: 0, endIndex: 3 });
|
|
599
|
+
});
|
|
600
|
+
|
|
601
|
+
test("returns null when finalAssistantTimestamp is null", () => {
|
|
602
|
+
expect(resolveRange(entry({ finalAssistantTimestamp: null }), base())).toBeNull();
|
|
603
|
+
});
|
|
604
|
+
|
|
605
|
+
test("returns null when the start timestamp matches two user messages", () => {
|
|
606
|
+
const msgs = [...base(), { role: "user", content: [{ type: "text", text: "dup" }], timestamp: 1000 }];
|
|
607
|
+
expect(resolveRange(entry(), msgs)).toBeNull();
|
|
608
|
+
});
|
|
609
|
+
|
|
610
|
+
test("returns null when a boundary message is absent", () => {
|
|
611
|
+
expect(resolveRange(entry({ startUserTimestamp: 999 }), base())).toBeNull();
|
|
612
|
+
});
|
|
613
|
+
|
|
614
|
+
test("returns null when the end precedes the start", () => {
|
|
615
|
+
// Both boundaries resolve uniquely, but the assistant match sits before
|
|
616
|
+
// the user match positionally - startIndex < endIndex must still hold.
|
|
617
|
+
const msgs = [
|
|
618
|
+
{ role: "assistant", content: [{ type: "text", text: "early" }], timestamp: 900 },
|
|
619
|
+
{ role: "user", content: [{ type: "text", text: "go" }], timestamp: 1000 },
|
|
620
|
+
{ role: "assistant", content: [{ type: "text", text: "done" }], timestamp: 1200 },
|
|
621
|
+
];
|
|
622
|
+
expect(resolveRange(entry({ finalAssistantTimestamp: 900 }), msgs)).toBeNull();
|
|
623
|
+
});
|
|
624
|
+
|
|
625
|
+
test("ignores a toolResult sharing the final assistant timestamp (role gating)", () => {
|
|
626
|
+
const msgs = base();
|
|
627
|
+
msgs[2] = { ...msgs[2], timestamp: 1200 };
|
|
628
|
+
expect(resolveRange(entry(), msgs)).toEqual({ startIndex: 0, endIndex: 3 });
|
|
629
|
+
});
|
|
630
|
+
});
|
|
631
|
+
|
|
632
|
+
describe("applyChainCompressions - positional", () => {
|
|
633
|
+
// Incident fixture: chains b5/b7 compressed, then a live turn reusing bash_23.
|
|
634
|
+
const incident = () => [
|
|
635
|
+
{ role: "user", content: [{ type: "text", text: "1" }], timestamp: 1000 },
|
|
636
|
+
{ role: "assistant", content: [{ type: "toolCall", id: "bash_18", name: "bash", input: {} }], timestamp: 1100 },
|
|
637
|
+
{ role: "toolResult", toolCallId: "bash_18", toolName: "bash", content: [{ type: "text", text: "a" }], isError: false, timestamp: 1150 },
|
|
638
|
+
{ role: "assistant", content: [{ type: "text", text: "done 1" }], timestamp: 1200 },
|
|
639
|
+
{ role: "user", content: [{ type: "text", text: "2" }], timestamp: 2000 },
|
|
640
|
+
{ role: "assistant", content: [{ type: "toolCall", id: "bash_23", name: "bash", input: {} }], timestamp: 2100 },
|
|
641
|
+
{ role: "toolResult", toolCallId: "bash_23", toolName: "bash", content: [{ type: "text", text: "b" }], isError: false, timestamp: 2150 },
|
|
642
|
+
{ role: "assistant", content: [{ type: "text", text: "done 2" }], timestamp: 2200 },
|
|
643
|
+
{ role: "user", content: [{ type: "text", text: "3" }], timestamp: 3000 },
|
|
644
|
+
{ role: "assistant", content: [{ type: "toolCall", id: "bash_23", name: "bash", input: {} }, { type: "toolCall", id: "gauntlet_setting_24", name: "gauntlet_setting", input: {} }], timestamp: 3100 },
|
|
645
|
+
{ role: "toolResult", toolCallId: "bash_23", toolName: "bash", content: [{ type: "text", text: "LIVE" }], isError: false, timestamp: 3150 },
|
|
646
|
+
{ role: "toolResult", toolCallId: "gauntlet_setting_24", toolName: "gauntlet_setting", content: [{ type: "text", text: "LIVE2" }], isError: false, timestamp: 3160 },
|
|
647
|
+
];
|
|
648
|
+
const entries = [
|
|
649
|
+
// compressedAt kept below 3000 so the "live turn" ts>=3000 filter in the
|
|
650
|
+
// survives-with-both-results test below doesn't also catch the synthetics.
|
|
651
|
+
{ blockId: "b5", startUserTimestamp: 1000, droppedToolCallIds: ["bash_18"], droppedOccurrenceKeys: ["bash_18@1150"], finalAssistantTimestamp: 1200, toolRefs: ["t1"], compressedAt: 1050 },
|
|
652
|
+
{ blockId: "b7", startUserTimestamp: 2000, droppedToolCallIds: ["bash_23"], droppedOccurrenceKeys: ["bash_23@2150"], finalAssistantTimestamp: 2200, toolRefs: ["t2"], compressedAt: 2050 },
|
|
653
|
+
];
|
|
654
|
+
const summaryFor = (e: any) => `summary ${e.blockId}`;
|
|
655
|
+
|
|
656
|
+
test("the live turn reusing a dropped id survives with both of its results", () => {
|
|
657
|
+
const out = applyChainCompressions(incident(), entries as any, summaryFor, false);
|
|
658
|
+
const live = out.filter((m: any) => (m.timestamp ?? 0) >= 3000);
|
|
659
|
+
expect(live).toHaveLength(4);
|
|
660
|
+
expect(live.filter((m: any) => m.role === "toolResult").map((m: any) => m.toolCallId)).toEqual([
|
|
661
|
+
"bash_23",
|
|
662
|
+
"gauntlet_setting_24",
|
|
663
|
+
]);
|
|
664
|
+
});
|
|
665
|
+
|
|
666
|
+
test("drops exactly the two chain interiors and inserts both synthetics", () => {
|
|
667
|
+
const out = applyChainCompressions(incident(), entries as any, summaryFor, false);
|
|
668
|
+
expect(out).toHaveLength(12 - 4 + 2);
|
|
669
|
+
const synthetics = out.filter((m: any) => m.role === "user" && m.content?.[0]?.text?.startsWith("<compressed-chain"));
|
|
670
|
+
expect(synthetics).toHaveLength(2);
|
|
671
|
+
expect(out.indexOf(synthetics[0])).toBe(1);
|
|
672
|
+
});
|
|
673
|
+
|
|
674
|
+
test("no toolResult survives without its toolCall", () => {
|
|
675
|
+
const out = applyChainCompressions(incident(), entries as any, summaryFor, false);
|
|
676
|
+
expectNoOrphanToolResults(out);
|
|
677
|
+
});
|
|
678
|
+
|
|
679
|
+
test("an unresolved entry drops nothing and inserts no synthetic", () => {
|
|
680
|
+
const reports: any[] = [];
|
|
681
|
+
const sink = { report: (kind: string, dedupKey: string, detail: string) => reports.push({ kind, dedupKey, detail }), counts: () => ({}) as any };
|
|
682
|
+
const bad = [{ ...entries[0], finalAssistantTimestamp: null }];
|
|
683
|
+
const msgs = incident();
|
|
684
|
+
const out = applyChainCompressions(msgs, bad as any, summaryFor, false, undefined, sink as any);
|
|
685
|
+
expect(out).toBe(msgs);
|
|
686
|
+
expect(reports.map((r) => r.kind)).toEqual(["unresolved-range"]);
|
|
687
|
+
});
|
|
688
|
+
|
|
689
|
+
test("re-applying the same entries is a no-op (synthetic preserved)", () => {
|
|
690
|
+
const first = applyChainCompressions(incident(), entries as any, summaryFor, false);
|
|
691
|
+
const second = applyChainCompressions(first, entries as any, summaryFor, false);
|
|
692
|
+
expect(second).toEqual(first);
|
|
693
|
+
});
|
|
694
|
+
|
|
695
|
+
test("a third-party custom message inside a range survives", () => {
|
|
696
|
+
const msgs = incident();
|
|
697
|
+
msgs.splice(2, 0, { role: "custom", customType: "other-extension", content: "keepme", timestamp: 1120 } as any);
|
|
698
|
+
const out = applyChainCompressions(msgs, entries as any, summaryFor, false);
|
|
699
|
+
expect(out.some((m: any) => m.customType === "other-extension")).toBe(true);
|
|
700
|
+
});
|
|
701
|
+
|
|
702
|
+
test("a per-batch summary AFTER the range is still suppressed (agent-message mode)", () => {
|
|
703
|
+
const msgs = incident();
|
|
704
|
+
msgs.splice(4, 0, {
|
|
705
|
+
role: "custom",
|
|
706
|
+
customType: CUSTOM_TYPE_SUMMARY,
|
|
707
|
+
content: "batch summary",
|
|
708
|
+
details: { toolCallRefs: [{ shortId: "t1", toolCallId: "bash_18", resultTimestamp: 1150 }] },
|
|
709
|
+
timestamp: 1300,
|
|
710
|
+
} as any);
|
|
711
|
+
const out = applyChainCompressions(msgs, entries as any, summaryFor, false);
|
|
712
|
+
expect(out.some((m: any) => m.customType === CUSTOM_TYPE_SUMMARY)).toBe(false);
|
|
713
|
+
});
|
|
714
|
+
|
|
715
|
+
test("strips thinking at the resolved endIndex only", () => {
|
|
716
|
+
const msgs = incident();
|
|
717
|
+
msgs[3] = { ...msgs[3], content: [{ type: "thinking", thinking: "hmm" }, { type: "text", text: "done 1" }] } as any;
|
|
718
|
+
const out = applyChainCompressions(msgs, entries as any, summaryFor, true);
|
|
719
|
+
const end = out.find((m: any) => m.timestamp === 1200);
|
|
720
|
+
expect(end.content.some((c: any) => c.type === "thinking")).toBe(false);
|
|
721
|
+
});
|
|
722
|
+
|
|
723
|
+
test("reports range-id-mismatch when the range drops a different id set", () => {
|
|
724
|
+
const reports: any[] = [];
|
|
725
|
+
const sink = { report: (kind: string, dedupKey: string, detail: string) => reports.push({ kind, dedupKey, detail }), counts: () => ({}) as any };
|
|
726
|
+
const skewed = [{ ...entries[0], droppedToolCallIds: ["bash_18", "ghost_1"] }];
|
|
727
|
+
applyChainCompressions(incident(), skewed as any, summaryFor, false, undefined, sink as any);
|
|
728
|
+
expect(reports.map((r) => r.kind)).toContain("range-id-mismatch");
|
|
729
|
+
});
|
|
730
|
+
|
|
731
|
+
test("skips an entry whose start falls strictly inside another entry's range", () => {
|
|
732
|
+
// b9 opens at the user message at index 4, which sits inside a wide b8
|
|
733
|
+
// range (index 0 -> 7). b8 wins; b9 contributes nothing and is reported.
|
|
734
|
+
const wide = { blockId: "b8", startUserTimestamp: 1000, droppedToolCallIds: [], finalAssistantTimestamp: 2200, toolRefs: [], compressedAt: 9002 };
|
|
735
|
+
const nestedInner = { blockId: "b9", startUserTimestamp: 2000, droppedToolCallIds: [], finalAssistantTimestamp: 2200, toolRefs: [], compressedAt: 9003 };
|
|
736
|
+
const reports: any[] = [];
|
|
737
|
+
const sink = { report: (kind: string, dedupKey: string, detail: string) => reports.push({ kind, dedupKey, detail }), counts: () => ({}) as any };
|
|
738
|
+
const out = applyChainCompressions(incident(), [wide, nestedInner] as any, summaryFor, false, undefined, sink as any);
|
|
739
|
+
const synthetics = out.filter((m: any) => m.role === "user" && m.content?.[0]?.text?.startsWith("<compressed-chain"));
|
|
740
|
+
expect(synthetics.map((m: any) => /id="([^"]+)"/.exec(m.content[0].text)![1])).toEqual(["b8"]);
|
|
741
|
+
expect(reports.filter((r) => r.kind === "unresolved-range").map((r) => r.dedupKey)).toEqual(["overlap:b9"]);
|
|
742
|
+
expectNoOrphanToolResults(out);
|
|
743
|
+
});
|
|
744
|
+
|
|
745
|
+
test("a genuinely unresolvable boundary and a benign overlap skip both report unresolved-range but with distinct dedup keys", () => {
|
|
746
|
+
const wide = { blockId: "b8", startUserTimestamp: 1000, droppedToolCallIds: [], finalAssistantTimestamp: 2200, toolRefs: [], compressedAt: 9002 };
|
|
747
|
+
const nestedInner = { blockId: "b9", startUserTimestamp: 2000, droppedToolCallIds: [], finalAssistantTimestamp: 2200, toolRefs: [], compressedAt: 9003 };
|
|
748
|
+
const brokenBoundary = { ...entries[0], blockId: "b11", finalAssistantTimestamp: null };
|
|
749
|
+
const reports: any[] = [];
|
|
750
|
+
const sink = { report: (kind: string, dedupKey: string, detail: string) => reports.push({ kind, dedupKey, detail }), counts: () => ({}) as any };
|
|
751
|
+
applyChainCompressions(incident(), [wide, nestedInner, brokenBoundary] as any, summaryFor, false, undefined, sink as any);
|
|
752
|
+
const kinds = reports.filter((r) => r.kind === "unresolved-range");
|
|
753
|
+
const dedupKeys = kinds.map((r) => r.dedupKey).sort();
|
|
754
|
+
expect(dedupKeys).toEqual(["b11", "overlap:b9"]);
|
|
755
|
+
expect(new Set(dedupKeys).size).toBe(dedupKeys.length);
|
|
756
|
+
const overlapReport = kinds.find((r) => r.dedupKey === "overlap:b9")!;
|
|
757
|
+
const boundaryReport = kinds.find((r) => r.dedupKey === "b11")!;
|
|
758
|
+
expect(overlapReport.detail).toContain("b8");
|
|
759
|
+
expect(boundaryReport.detail).toContain("start=");
|
|
760
|
+
expect(boundaryReport.detail).toContain("final=");
|
|
761
|
+
});
|
|
762
|
+
|
|
763
|
+
test("two entries resolving to the same startIndex insert one synthetic", () => {
|
|
764
|
+
const twin = { ...entries[0], blockId: "b10", compressedAt: 9004 };
|
|
765
|
+
const out = applyChainCompressions(incident(), [entries[0], twin] as any, summaryFor, false);
|
|
766
|
+
const synthetics = out.filter((m: any) => m.role === "user" && m.content?.[0]?.text?.startsWith("<compressed-chain"));
|
|
767
|
+
expect(synthetics).toHaveLength(1);
|
|
768
|
+
});
|
|
769
|
+
|
|
770
|
+
describe("G4/C3: orphan-sweep zero-fire proof", () => {
|
|
771
|
+
// Runs the real orphan sweep (src/orphan-sweep.ts, backing pruner.ts Phase 4)
|
|
772
|
+
// over the output of every clean applyChainCompressions fixture in this
|
|
773
|
+
// describe block. None of these are expected to leave an orphan behind -
|
|
774
|
+
// if one does, that is a real finding (chain-range-prune would be relying
|
|
775
|
+
// on the sweep as a crutch, not producing clean output on its own).
|
|
776
|
+
const fixtures: Array<[string, () => any[]]> = [
|
|
777
|
+
["the live turn reusing a dropped id survives with both of its results", () => applyChainCompressions(incident(), entries as any, summaryFor, false)],
|
|
778
|
+
["drops exactly the two chain interiors and inserts both synthetics", () => applyChainCompressions(incident(), entries as any, summaryFor, false)],
|
|
779
|
+
["re-applying the same entries is a no-op", () => applyChainCompressions(applyChainCompressions(incident(), entries as any, summaryFor, false), entries as any, summaryFor, false)],
|
|
780
|
+
["a third-party custom message inside a range survives", () => {
|
|
781
|
+
const msgs = incident();
|
|
782
|
+
msgs.splice(2, 0, { role: "custom", customType: "other-extension", content: "keepme", timestamp: 1120 } as any);
|
|
783
|
+
return applyChainCompressions(msgs, entries as any, summaryFor, false);
|
|
784
|
+
}],
|
|
785
|
+
["strips thinking at the resolved endIndex only", () => {
|
|
786
|
+
const msgs = incident();
|
|
787
|
+
msgs[3] = { ...msgs[3], content: [{ type: "thinking", thinking: "hmm" }, { type: "text", text: "done 1" }] } as any;
|
|
788
|
+
return applyChainCompressions(msgs, entries as any, summaryFor, true);
|
|
789
|
+
}],
|
|
790
|
+
["skips an entry whose start falls strictly inside another entry's range", () => {
|
|
791
|
+
const wide = { blockId: "b8", startUserTimestamp: 1000, droppedToolCallIds: [], finalAssistantTimestamp: 2200, toolRefs: [], compressedAt: 9002 };
|
|
792
|
+
const nestedInner = { blockId: "b9", startUserTimestamp: 2000, droppedToolCallIds: [], finalAssistantTimestamp: 2200, toolRefs: [], compressedAt: 9003 };
|
|
793
|
+
return applyChainCompressions(incident(), [wide, nestedInner] as any, summaryFor, false);
|
|
794
|
+
}],
|
|
795
|
+
["two entries resolving to the same startIndex insert one synthetic", () => {
|
|
796
|
+
const twin = { ...entries[0], blockId: "b10", compressedAt: 9004 };
|
|
797
|
+
return applyChainCompressions(incident(), [entries[0], twin] as any, summaryFor, false);
|
|
798
|
+
}],
|
|
799
|
+
["single dropped chain, ordering invariant fixture", () => {
|
|
800
|
+
const msgs = [
|
|
801
|
+
userMsg(100),
|
|
802
|
+
assistantWithTools(200, ["tc1"]),
|
|
803
|
+
toolResult(300, "tc1"),
|
|
804
|
+
assistantText(400),
|
|
805
|
+
userMsg(500),
|
|
806
|
+
assistantText(600),
|
|
807
|
+
];
|
|
808
|
+
return applyChainCompressions(msgs, [entry("b1", 100, ["tc1"], 400)], noopSummary, false);
|
|
809
|
+
}],
|
|
810
|
+
["multiple chains in one pass", () => {
|
|
811
|
+
const msgs = [
|
|
812
|
+
userMsg(100),
|
|
813
|
+
assistantWithTools(200, ["tc1"]),
|
|
814
|
+
toolResult(300, "tc1"),
|
|
815
|
+
assistantText(400),
|
|
816
|
+
userMsg(500),
|
|
817
|
+
assistantWithTools(600, ["tc2"]),
|
|
818
|
+
toolResult(700, "tc2"),
|
|
819
|
+
assistantText(800),
|
|
820
|
+
userMsg(900),
|
|
821
|
+
assistantText(1000),
|
|
822
|
+
];
|
|
823
|
+
const e1 = entry("b1", 100, ["tc1"], 400, ["t1"]);
|
|
824
|
+
const e2 = entry("b2", 500, ["tc2"], 800, ["t2"]);
|
|
825
|
+
return applyChainCompressions(msgs, [e1, e2], noopSummary, false);
|
|
826
|
+
}],
|
|
827
|
+
["protected output relocation", () => {
|
|
828
|
+
const e = {
|
|
829
|
+
blockId: "b1",
|
|
830
|
+
startUserTimestamp: 1,
|
|
831
|
+
droppedToolCallIds: ["tc-read", "tc-todo"],
|
|
832
|
+
protectedToolCallIds: ["tc-todo"],
|
|
833
|
+
finalAssistantTimestamp: 9,
|
|
834
|
+
toolRefs: ["t1", "t2"],
|
|
835
|
+
compressedAt: 100,
|
|
836
|
+
};
|
|
837
|
+
const messages = [
|
|
838
|
+
{ role: "user", timestamp: 1, content: [{ type: "text", text: "go" }] },
|
|
839
|
+
{ role: "assistant", timestamp: 2, content: [
|
|
840
|
+
{ type: "toolCall", id: "tc-read", name: "read" },
|
|
841
|
+
{ type: "toolCall", id: "tc-todo", name: "todowrite" },
|
|
842
|
+
] },
|
|
843
|
+
{ role: "toolResult", toolCallId: "tc-read", toolName: "read", content: [{ type: "text", text: "FILE" }] },
|
|
844
|
+
{ role: "toolResult", toolCallId: "tc-todo", toolName: "todowrite", content: [{ type: "text", text: "PLAN-STATE" }] },
|
|
845
|
+
{ role: "assistant", timestamp: 9, content: [{ type: "text", text: "done" }] },
|
|
846
|
+
];
|
|
847
|
+
return applyChainCompressions(messages, [e] as any, () => "SUMMARY", false);
|
|
848
|
+
}],
|
|
849
|
+
];
|
|
850
|
+
|
|
851
|
+
for (const [name, run] of fixtures) {
|
|
852
|
+
test(`zero orphan sweeps: ${name}`, () => {
|
|
853
|
+
expectZeroOrphanSweep(run());
|
|
854
|
+
});
|
|
855
|
+
}
|
|
856
|
+
});
|
|
857
|
+
});
|
package/src/chain-range-prune.ts
CHANGED
|
@@ -3,14 +3,35 @@ import { CUSTOM_TYPE_SUMMARY } from "./types.js";
|
|
|
3
3
|
import type { ChainCompressionEntry } from "./types.js";
|
|
4
4
|
import { substituteBlockRefs } from "./nested-placeholders.js";
|
|
5
5
|
import { extractToolResultText } from "./batch-capture.js";
|
|
6
|
+
import { bareToolCallId, occKey, resultTimestampOf } from "./occurrence-key.js";
|
|
7
|
+
import type { DiagnosticSink } from "./diagnostics.js";
|
|
6
8
|
|
|
7
9
|
export function isPerBatchSummaryMessage(msg: any): boolean {
|
|
8
10
|
return msg.role === "custom" && msg.customType === CUSTOM_TYPE_SUMMARY;
|
|
9
11
|
}
|
|
10
12
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
return
|
|
13
|
+
/** Refs a per-batch summary message carries, as (toolCallId, resultTimestamp) pairs. */
|
|
14
|
+
function summaryRefs(msg: any): { toolCallId: string; resultTimestamp?: number }[] {
|
|
15
|
+
return msg.details?.toolCallRefs ?? [];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* A ref with a `resultTimestamp` is matched by exact occurrence key against
|
|
20
|
+
* `droppedOccKeys` - a live turn reusing a dropped chain's bare id must not
|
|
21
|
+
* suppress that live turn's summary. A ref with no `resultTimestamp` (legacy)
|
|
22
|
+
* falls back to bare-id membership in `droppedBareIds`, since no occurrence
|
|
23
|
+
* discriminant was ever recorded for it.
|
|
24
|
+
*/
|
|
25
|
+
export function perBatchSummaryOverlapsDropped(
|
|
26
|
+
msg: any,
|
|
27
|
+
droppedOccKeys: Set<string>,
|
|
28
|
+
droppedBareIds: Set<string>,
|
|
29
|
+
): boolean {
|
|
30
|
+
return summaryRefs(msg).some((r) =>
|
|
31
|
+
r.resultTimestamp !== undefined
|
|
32
|
+
? droppedOccKeys.has(occKey(r.toolCallId, r.resultTimestamp))
|
|
33
|
+
: droppedBareIds.has(bareToolCallId(r.toolCallId)),
|
|
34
|
+
);
|
|
14
35
|
}
|
|
15
36
|
|
|
16
37
|
export function withoutThinkingBlocks(msg: AssistantMessage): AssistantMessage {
|
|
@@ -44,12 +65,46 @@ export function buildSyntheticChainMessage(
|
|
|
44
65
|
};
|
|
45
66
|
}
|
|
46
67
|
|
|
68
|
+
/**
|
|
69
|
+
* Resolves a persisted chain entry to a positional index range.
|
|
70
|
+
*
|
|
71
|
+
* Role-gated and unique-match-or-nothing: exactly one user message at
|
|
72
|
+
* startUserTimestamp, exactly one assistant at finalAssistantTimestamp, and
|
|
73
|
+
* start < end. Otherwise null - the entry drops nothing. Fail-closed is the
|
|
74
|
+
* whole point: an id-set or timestamp-window fallback is what deleted live
|
|
75
|
+
* turns (doc/specs/2026-08-12-toolcall-id-collisions.md).
|
|
76
|
+
*/
|
|
77
|
+
export function resolveRange(
|
|
78
|
+
entry: ChainCompressionEntry,
|
|
79
|
+
messages: any[],
|
|
80
|
+
): { startIndex: number; endIndex: number } | null {
|
|
81
|
+
if (entry.finalAssistantTimestamp === null) return null;
|
|
82
|
+
let startIndex = -1;
|
|
83
|
+
let startMatches = 0;
|
|
84
|
+
let endIndex = -1;
|
|
85
|
+
let endMatches = 0;
|
|
86
|
+
for (let i = 0; i < messages.length; i++) {
|
|
87
|
+
const msg = messages[i];
|
|
88
|
+
if (msg.role === "user" && msg.timestamp === entry.startUserTimestamp) {
|
|
89
|
+
startMatches++;
|
|
90
|
+
if (startIndex < 0) startIndex = i;
|
|
91
|
+
} else if (msg.role === "assistant" && msg.timestamp === entry.finalAssistantTimestamp) {
|
|
92
|
+
endMatches++;
|
|
93
|
+
if (endIndex < 0) endIndex = i;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (startMatches !== 1 || endMatches !== 1) return null;
|
|
97
|
+
if (!(startIndex < endIndex)) return null;
|
|
98
|
+
return { startIndex, endIndex };
|
|
99
|
+
}
|
|
100
|
+
|
|
47
101
|
export function applyChainCompressions(
|
|
48
102
|
messages: any[],
|
|
49
103
|
chainEntries: ChainCompressionEntry[],
|
|
50
104
|
summaryTextForChain: (entry: ChainCompressionEntry) => string,
|
|
51
105
|
stripFinalThinking: boolean,
|
|
52
106
|
blockSummaryLookup?: (blockId: string) => string | undefined,
|
|
107
|
+
diagnostics?: DiagnosticSink,
|
|
53
108
|
): any[] {
|
|
54
109
|
if (chainEntries.length === 0) return messages;
|
|
55
110
|
|
|
@@ -65,64 +120,122 @@ export function applyChainCompressions(
|
|
|
65
120
|
}
|
|
66
121
|
}
|
|
67
122
|
|
|
68
|
-
|
|
69
|
-
const
|
|
123
|
+
// 1. resolve every entry to a range; unresolved entries contribute nothing
|
|
124
|
+
const resolved: { entry: ChainCompressionEntry; startIndex: number; endIndex: number }[] = [];
|
|
125
|
+
for (const entry of chainEntries) {
|
|
126
|
+
const range = resolveRange(entry, messages);
|
|
127
|
+
if (!range) {
|
|
128
|
+
diagnostics?.report(
|
|
129
|
+
"unresolved-range",
|
|
130
|
+
entry.blockId,
|
|
131
|
+
`blockId=${entry.blockId} start=${entry.startUserTimestamp} final=${entry.finalAssistantTimestamp}`,
|
|
132
|
+
);
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
resolved.push({ entry, ...range });
|
|
136
|
+
}
|
|
70
137
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
138
|
+
// 2. drop entries nested inside another entry's range, and de-duplicate
|
|
139
|
+
// entries that resolved to the same startIndex (one synthetic per slot)
|
|
140
|
+
const accepted: typeof resolved = [];
|
|
141
|
+
const claimedStart = new Set<number>();
|
|
142
|
+
for (const candidate of resolved) {
|
|
143
|
+
const loser = resolved.find(
|
|
144
|
+
(other) => other !== candidate && candidate.startIndex > other.startIndex && candidate.startIndex < other.endIndex,
|
|
145
|
+
);
|
|
146
|
+
if (loser || claimedStart.has(candidate.startIndex)) {
|
|
147
|
+
// Same DiagnosticKind as a genuinely unresolvable boundary, but a
|
|
148
|
+
// distinct dedup-key prefix: this is a benign, expected skip (nested or
|
|
149
|
+
// duplicate range), not a compression failure. Keeping the kind fixed
|
|
150
|
+
// to the spec's set while still making the two cases greppable.
|
|
151
|
+
diagnostics?.report(
|
|
152
|
+
"unresolved-range",
|
|
153
|
+
`overlap:${candidate.entry.blockId}`,
|
|
154
|
+
loser
|
|
155
|
+
? `blockId=${candidate.entry.blockId} skipped: range nests inside blockId=${loser.entry.blockId}`
|
|
156
|
+
: `blockId=${candidate.entry.blockId} skipped: range duplicates startIndex=${candidate.startIndex} already claimed`,
|
|
157
|
+
);
|
|
158
|
+
continue;
|
|
77
159
|
}
|
|
160
|
+
claimedStart.add(candidate.startIndex);
|
|
161
|
+
accepted.push(candidate);
|
|
78
162
|
}
|
|
79
163
|
|
|
164
|
+
if (accepted.length === 0) return messages;
|
|
165
|
+
|
|
166
|
+
// 3. index sets + per-entry facts
|
|
167
|
+
const dropIndices = new Set<number>();
|
|
168
|
+
const stripAtIndex = new Set<number>();
|
|
169
|
+
const insertAfterIndex = new Map<number, { synthetic: any; blockId: string }>();
|
|
170
|
+
const droppedBareIds = new Set<string>();
|
|
171
|
+
const droppedOccKeys = new Set<string>();
|
|
80
172
|
const protectedByBlock = new Map<string, { tool: string; text: string }[]>();
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
173
|
+
|
|
174
|
+
for (const { entry, startIndex, endIndex } of accepted) {
|
|
175
|
+
const protectedIds = new Set(entry.protectedToolCallIds ?? []);
|
|
176
|
+
const inRangeBareIds: string[] = [];
|
|
177
|
+
for (let i = startIndex + 1; i < endIndex; i++) {
|
|
178
|
+
const msg = messages[i];
|
|
179
|
+
dropIndices.add(i);
|
|
180
|
+
if (msg.role === "toolResult") {
|
|
181
|
+
inRangeBareIds.push(msg.toolCallId);
|
|
182
|
+
droppedBareIds.add(msg.toolCallId);
|
|
183
|
+
droppedOccKeys.add(occKey(msg.toolCallId, resultTimestampOf(msg.timestamp)));
|
|
184
|
+
if (protectedIds.has(msg.toolCallId)) {
|
|
185
|
+
const arr = protectedByBlock.get(entry.blockId) ?? [];
|
|
186
|
+
arr.push({ tool: msg.toolName, text: extractToolResultText(msg) });
|
|
187
|
+
protectedByBlock.set(entry.blockId, arr);
|
|
188
|
+
}
|
|
189
|
+
} else if (msg.role === "assistant") {
|
|
190
|
+
for (const block of msg.content ?? []) {
|
|
191
|
+
if (block.type === "toolCall") {
|
|
192
|
+
inRangeBareIds.push(block.id);
|
|
193
|
+
droppedBareIds.add(block.id);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
88
196
|
}
|
|
89
197
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
198
|
+
// Diagnostic-only cross-check: the range always wins.
|
|
199
|
+
const recorded = new Set((entry.droppedToolCallIds ?? []).map(bareToolCallId));
|
|
200
|
+
const actual = new Set(inRangeBareIds);
|
|
201
|
+
if (recorded.size !== actual.size || [...recorded].some((id) => !actual.has(id))) {
|
|
202
|
+
diagnostics?.report(
|
|
203
|
+
"range-id-mismatch",
|
|
204
|
+
entry.blockId,
|
|
205
|
+
`blockId=${entry.blockId} recorded=${recorded.size} actual=${actual.size}`,
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
if (stripFinalThinking) stripAtIndex.add(endIndex);
|
|
209
|
+
insertAfterIndex.set(startIndex, {
|
|
210
|
+
synthetic: buildSyntheticChainMessage(
|
|
211
|
+
entry,
|
|
212
|
+
summaryTextForChain(entry),
|
|
213
|
+
blockSummaryLookup,
|
|
214
|
+
protectedByBlock.get(entry.blockId) ?? [],
|
|
215
|
+
),
|
|
216
|
+
blockId: entry.blockId,
|
|
98
217
|
});
|
|
99
218
|
}
|
|
100
219
|
|
|
220
|
+
// 4. emit. Drops are role-restricted: user-role messages (including the
|
|
221
|
+
// already-inserted synthetic, which is user-role) and third-party custom
|
|
222
|
+
// messages inside a range are preserved. Preserving the synthetic is what
|
|
223
|
+
// makes re-application a true no-op.
|
|
101
224
|
const out: any[] = [];
|
|
102
|
-
for (
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
if (stripFinalAtTimestamp.has(msg.timestamp)) {
|
|
111
|
-
out.push(withoutThinkingBlocks(msg));
|
|
112
|
-
continue;
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
if (isPerBatchSummaryMessage(msg) && perBatchSummaryOverlapsDropped(msg, droppedToolCallIds)) continue;
|
|
225
|
+
for (let i = 0; i < messages.length; i++) {
|
|
226
|
+
const msg = messages[i];
|
|
227
|
+
const isSummary = isPerBatchSummaryMessage(msg);
|
|
228
|
+
const droppable = msg.role === "assistant" || msg.role === "toolResult" || isSummary;
|
|
229
|
+
if (dropIndices.has(i) && droppable) continue;
|
|
230
|
+
// Coverage, not index membership: in agent-message batching the per-batch
|
|
231
|
+
// summary is appended AFTER finalAssistantTimestamp, outside the range.
|
|
232
|
+
if (isSummary && perBatchSummaryOverlapsDropped(msg, droppedOccKeys, droppedBareIds)) continue;
|
|
117
233
|
|
|
118
|
-
out.push(msg);
|
|
234
|
+
if (msg.role === "assistant" && stripAtIndex.has(i)) out.push(withoutThinkingBlocks(msg));
|
|
235
|
+
else out.push(msg);
|
|
119
236
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
if (info && !existingSyntheticBlockIds.has(info.blockId)) {
|
|
123
|
-
out.push(info.synthetic);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
237
|
+
const info = insertAfterIndex.get(i);
|
|
238
|
+
if (info && !existingSyntheticBlockIds.has(info.blockId)) out.push(info.synthetic);
|
|
126
239
|
}
|
|
127
240
|
return out;
|
|
128
241
|
}
|