pi-tuck 1.0.16 → 1.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/CHANGELOG.md +75 -0
  2. package/README.md +7 -4
  3. package/dist/src/budget.d.ts +14 -17
  4. package/dist/src/budget.js +22 -29
  5. package/dist/src/build-id.d.ts +1 -1
  6. package/dist/src/build-id.js +1 -1
  7. package/dist/src/compaction.js +113 -20
  8. package/dist/src/diagnostics.d.ts +1 -1
  9. package/dist/src/history-tools.d.ts +16 -0
  10. package/dist/src/history-tools.js +59 -0
  11. package/dist/src/ledger.d.ts +0 -2
  12. package/dist/src/ledger.js +0 -2
  13. package/dist/src/omissions.d.ts +1 -1
  14. package/dist/src/omissions.js +88 -6
  15. package/dist/src/tuck/core/brief.d.ts +6 -3
  16. package/dist/src/tuck/core/brief.js +27 -9
  17. package/dist/src/tuck/core/filter-noise.d.ts +1 -1
  18. package/dist/src/tuck/core/filter-noise.js +1 -3
  19. package/dist/src/tuck/core/normalize.d.ts +2 -5
  20. package/dist/src/tuck/core/normalize.js +15 -33
  21. package/dist/src/tuck/core/shrink.d.ts +6 -14
  22. package/dist/src/tuck/core/shrink.js +162 -102
  23. package/dist/src/tuck/core/summarize.d.ts +3 -14
  24. package/dist/src/tuck/core/summarize.js +9 -20
  25. package/dist/src/tuck/core/transcript.d.ts +5 -0
  26. package/dist/src/tuck/core/transcript.js +27 -4
  27. package/dist/src/tuck/types.d.ts +1 -1
  28. package/dist/src/tuck.d.ts +9 -17
  29. package/dist/src/tuck.js +384 -212
  30. package/package.json +1 -1
  31. package/src/budget.ts +24 -26
  32. package/src/build-id.ts +1 -1
  33. package/src/compaction.ts +102 -21
  34. package/src/history-tools.ts +57 -0
  35. package/src/ledger.ts +0 -2
  36. package/src/omissions.ts +64 -7
  37. package/src/tuck/core/brief.ts +25 -10
  38. package/src/tuck/core/filter-noise.ts +1 -3
  39. package/src/tuck/core/normalize.ts +11 -27
  40. package/src/tuck/core/shrink.ts +141 -91
  41. package/src/tuck/core/summarize.ts +9 -24
  42. package/src/tuck/core/transcript.ts +26 -4
  43. package/src/tuck/types.ts +3 -3
  44. package/src/tuck.ts +336 -185
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-tuck",
3
- "version": "1.0.16",
3
+ "version": "1.0.18",
4
4
  "description": "Deterministic compaction for Pi with no model call, plus exact recall of earlier history. Search by meaning on Apple silicon.",
5
5
  "keywords": [
6
6
  "pi-package",
package/src/budget.ts CHANGED
@@ -26,12 +26,26 @@ export const validCutAt = (value: unknown): value is number => typeof value ===
26
26
  /** The initial default cut point: 60% of the room before a usable previous cut. Measured against 1 with paired replays (no answer loss) and billed
27
27
  * input under automatic and explicit prompt caching (about 17% less at 128k, 10-14% at 32k). */
28
28
  export const DEFAULT_CUT_AT = 0.6;
29
+ /** Preferred native carry, in the planner's estimate: never more than the
30
+ * keep-recent cut offers or the tail amount the adaptive band assumes. */
31
+ export const carriedTailTokens = (offeredTokens: number): number => Math.min(offeredTokens, MAX_TAIL_TOKENS / 2);
29
32
  export const bandOf = (room: number, cutAt = 1, adaptive?: { system: number; tokensAfter: number; factor: number }): number => {
30
33
  const positive = Math.max(0, room);
31
34
  const base = Math.floor(positive * cutAt);
32
- return Math.min(room - Math.ceil(positive * TRIGGER_MARGIN_FRACTION), adaptive ? Math.max(base, Math.ceil((adaptive.system + adaptive.tokensAfter + MAX_TAIL_TOKENS / 2) * adaptive.factor)) : base);
35
+ return Math.min(room - Math.ceil(positive * TRIGGER_MARGIN_FRACTION), adaptive ? Math.max(base, Math.ceil((adaptive.system + adaptive.tokensAfter + carriedTailTokens(MAX_TAIL_TOKENS)) * adaptive.factor)) : base);
33
36
  };
34
37
 
38
+ /** The cut's one carried-context grant, in the planner's estimate. Combines
39
+ * the existing native/fixed base with the quarter-free historical amount;
40
+ * neither component remains an independent spending limit in the fill.
41
+ * The grant depends only on quantities. */
42
+ export function carriedTarget(window: number, reserve: number, base: number, before: number): number {
43
+ const room = Math.max(0, window - reserve - 1);
44
+ const free = Math.max(0, window - reserve - base - 1);
45
+ const historical = Math.max(0, Math.min(Math.floor(free / 4), before - base - 1));
46
+ return Math.min(room, before - 1, base + historical);
47
+ }
48
+
35
49
  /** The APIs whose request converter (pi-ai openai-responses-shared) sends each signed thinking block back as an input reasoning item. */
36
50
  const REPLAYING_APIS: ReadonlySet<string> = new Set(["openai-responses", "azure-openai-responses", "openai-codex-responses"]);
37
51
  /**
@@ -91,8 +105,8 @@ export const planningReserve = (window: number, targetRoom: number, factor: numb
91
105
  window - Math.floor(targetRoom / factor) + mandatory;
92
106
 
93
107
  export type Target = "band" | "room";
94
- /** A planned cut: the conversation it leaves, and the summary view that produced it (whole dialogue or shrunk). */
95
- export type Plan<D extends object = {}> = ({ draft: true; after: number; mode: string } & D) | { draft: false; capacityRejected: boolean; reason: string };
108
+ /** A planned cut: the conversation it leaves. The optional mode is diagnostic data, not a decision input. */
109
+ export type Plan<D extends object = {}> = ({ draft: true; after: number; mode?: string } & D) | { draft: false; capacityRejected: boolean; reason: string };
96
110
  export type Decision<D extends object = {}> =
97
111
  /** The request fits below the band, or an opportunistic cut cannot help: no cut and no fault; the request goes to the guard. */
98
112
  | { kind: "none" }
@@ -118,39 +132,23 @@ export interface CutInput<D extends object = {}> {
118
132
  }
119
133
  /** What a cut decision resolves to once the planner has been asked. */
120
134
  export type Verdict<D extends object = {}> = Exclude<Decision<D>, { kind: "defer" }>;
121
- /**
122
- * Rule 4. The exact room is planned first. When its plan already lands below the band it is taken as it is (target "room"): it keeps the
123
- * room plan's content exactly, because every allowance of the planner is derived from the reserve and the band's is smaller. Only
124
- * otherwise is the band planned. Inside the band a cut is opportunistic, so a failed plan is never a fault and never falls back to the
125
- * room (a cut that still lands in the band would trigger again every round); the exact room would cut nothing there, so a band cut is
126
- * taken only when the room can plan a cut too and with the same mode (a lossy view the room could not even plan is a quality change with
127
- * no fault to prevent). At or above the room, where a cut is due, a band plan of another mode gives way to the room plan (today's
128
- * view), and a band plan stands when the room has none (a cut instead of a fault).
129
- * A lean band, one the cut point binds (`band < bandOf(room)`), is planned first and the room plan landing below it is not taken for that:
130
- * the room plan keeps a native tail of up to half the room, so it can land just under a band that far below the room, and the next round
131
- * would cut again, freeing little. The room is planned only where the rules above use it: to compare the mode, or when the band has no plan
132
- * and a cut is due. An adaptive band can also reach the margin, in which case the room plan is tried first.
133
- * The margin adds no new decision fault. Exceptions thrown by `plan` are not caught here: they are faults, and they now surface at
134
- * boundaries inside the band too, where no planner ran before the margin.
135
- */
135
+ /** Plan the band first at every cut point. An opportunistic cut must land
136
+ * below it and free the existing minimum. A due cut takes a band draft, or
137
+ * falls back to the room only when the band has no draft. Final reduction and
138
+ * room-fit checks own faults. Planner exceptions propagate. */
136
139
  export function chooseCut<D extends object = {}>(input: CutInput<D> & { opportunistic: boolean }): Verdict<D> {
137
140
  const { before, mandatory, room, band, fits, plan, unreduced } = input;
138
141
  const lands = (p: Plan<D>): p is Extract<Plan<D>, { draft: true }> => p.draft && p.after < before && fits(p.after + mandatory, band);
139
142
  // A cut no one needs yet must be worth its summary: one that frees under a tenth of the room would trigger again within a round or two.
140
- // Without this, a band at the cap takes the room plan on every round where free room is small. A due cut is not asked: it has to happen.
143
+ // A due cut is not asked: it has to happen.
141
144
  const worthwhile = (p: Extract<Plan<D>, { draft: true }>) => !input.usefulOnly || !input.opportunistic || !fits(10 * (before - p.after), room);
142
- let roomPlan: Plan<D> | undefined;
143
- const exact = () => roomPlan ??= plan("room");
144
- if (band >= bandOf(room)) { const e = exact(); if (lands(e) && worthwhile(e)) return { kind: "cut", target: "room", plan: e }; }
145
145
  const preferred = plan("band");
146
146
  if (input.opportunistic) {
147
147
  if (!lands(preferred) || !worthwhile(preferred)) return { kind: "none" };
148
- const e = exact();
149
- return e.draft && e.mode === preferred.mode ? { kind: "cut", target: "band", plan: preferred } : { kind: "none" };
148
+ return { kind: "cut", target: "band", plan: preferred };
150
149
  }
151
- const e = exact();
152
150
  let chosen: Plan<D> = preferred, target: Target = "band";
153
- if (!preferred.draft || (e.draft && e.mode !== preferred.mode)) { chosen = e; target = "room"; }
151
+ if (!preferred.draft) { chosen = plan("room"); target = "room"; }
154
152
  if (!chosen.draft) return { kind: "fault", capacity: chosen.capacityRejected, message: chosen.reason };
155
153
  if (chosen.after >= before || !fits(chosen.after + mandatory, room))
156
154
  return { kind: "fault", capacity: !fits(chosen.after + mandatory, room), message: unreduced };
package/src/build-id.ts CHANGED
@@ -1 +1 @@
1
- export const BUILD_ID = "e695eee2b3e13753eb135cb61b69783a79d72ac4c1ce08f28304b98cca6b5a49";
1
+ export const BUILD_ID = "c53f4e367c1f45ac6b3865cbf25bf99992da4b7e961557e5510369dca81fb087";
package/src/compaction.ts CHANGED
@@ -6,10 +6,10 @@ import { isDeepStrictEqual } from "node:util";
6
6
  import { assertNativeToolRounds } from "./protocol.js";
7
7
  import { projectedMessages } from "./omissions.js";
8
8
  import { applyClips, clipBudgets, clipFor, type Clip } from "./clip.js";
9
- import { DEFAULT_CUT_AT, bandOf, chooseCut, conversationTokens, decideCut, gateCut, incomingEventTokens, incomingTokens, measurableConversationTokens, planningReserve, projectedConversationTokens, requestTokens, systemPart, validCutAt,
9
+ import { DEFAULT_CUT_AT, bandOf, chooseCut, conversationTokens, gateCut, incomingEventTokens, incomingTokens, measurableConversationTokens, planningReserve, projectedConversationTokens, systemPart, validCutAt,
10
10
  type AgentMessage, type Messages, type Plan, type Target, type ToolShape } from "./budget.js";
11
- import { isContextOverflow, type ToolResultMessage } from "@earendil-works/pi-ai";
12
- import { convertToLlm, SessionManager, SettingsManager, type ExtensionAPI, type ExtensionContext, type SessionEntry } from "@earendil-works/pi-coding-agent";
11
+ import { getCurrentSystemMessage, isContextOverflow, type ImageContent, type ToolResultMessage } from "@earendil-works/pi-ai";
12
+ import { calculateContextTokens, convertToLlm, estimateTokens, shouldCompact, SessionManager, SettingsManager, type ExtensionAPI, type ExtensionContext, type SessionEntry } from "@earendil-works/pi-coding-agent";
13
13
  import { Type } from "typebox";
14
14
  import { Check } from "typebox/value";
15
15
  import type { RecallPolicy } from "./recall.js";
@@ -18,6 +18,7 @@ import { MAX_RECALL_CHARACTERS, MAX_RECALL_ENTRY_ID_CHARACTERS, MAX_RECALL_QUERY
18
18
  import { tuck, tuckDraft, tuckObservedDraft, tuckRecall } from "./tuck.js";
19
19
  import { ImageRecallInput, historyImage, historyAround } from "./history-image.js";
20
20
  import { eligibilityKey, eligibilityKeyMoved, memoizedEligibleSources } from "./eligibility-cache";
21
+ import { applyHistoryTools, hasTuckCut, historyToolDeclarations, historyToolNames, stageHistoryTools } from "./history-tools.js";
21
22
 
22
23
  const query = Type.String({ minLength: 1, maxLength: MAX_RECALL_QUERY_CHARACTERS });
23
24
  const entryId = Type.String({ minLength: 1, maxLength: MAX_RECALL_ENTRY_ID_CHARACTERS });
@@ -296,12 +297,22 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
296
297
  const measureFor = (ctx: ExtensionContext) => (messages: Messages) => projectedConversationTokens(applyClips(messages, clips), ctx, options.recallPolicy);
297
298
  // The planner counts with the same function, except that it does not fail on a summary in another format: a cut can replace one
298
299
  // (an operator's /compact repairs a session whose saved summary is foreign). Identical to the strict one for this compactor's summaries.
299
- const plannerMeasureFor = (ctx: ExtensionContext) => (messages: Messages) => measurableConversationTokens(applyClips(messages, clips), ctx, options.recallPolicy);
300
+ const plannerMeasureFor = (ctx: ExtensionContext) => Object.assign(
301
+ (messages: Messages) => measurableConversationTokens(applyClips(messages, clips), ctx, options.recallPolicy),
302
+ { messageTokens: (message: Messages[number]) => conversationTokens(applyClips([message], clips)) });
300
303
  const hostTokens = (ctx: ExtensionContext) => options.hostMessages ? conversationTokens(options.hostMessages(ctx)) : 0;
301
304
  const toolShapes = (names: Iterable<string>): ToolShape[] => {
302
305
  const selected = new Set(names);
303
306
  return pi.getAllTools().filter(tool => selected.has(tool.name)).map(tool => ({ name: tool.name, description: tool.description, parameters: tool.parameters }));
304
307
  };
308
+ // Every decision prices the declaration set a cut must carry, even before the first cut enables our tools on the wire.
309
+ const requestSystemPart = (observedPrompt: string, names: readonly string[], saved: Messages, guardPrompt: string) => {
310
+ const shapes = toolShapes(historyToolNames(pi, names, true));
311
+ // Keep the guard's observed schemas (Pi may enrich a host tool's schema), and add our future declarations to that same view.
312
+ // Taking max(current guard, future raw shapes) alone could lose the enrichment when the future shapes are larger.
313
+ const future = saved.some(message => message.role === "system") ? historyToolDeclarations(saved, shapes, true) : saved;
314
+ return systemPart(observedPrompt, shapes, future, guardPrompt);
315
+ };
305
316
  // Pi reports a manual cut with nothing to do as a failed compaction; the thrown error lacks the event's "Compaction failed: " prefix.
306
317
  const isNoWork = (message: string | undefined) => ["Already compacted", "Nothing to compact (session too small)"].includes(message?.replace(/^Compaction failed: /, "") ?? "");
307
318
  // A missing model is configuration, not capacity.
@@ -352,7 +363,7 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
352
363
  if (!ctx.isIdle() || phase === "starting") return; // a steering or follow-up message joins the running request
353
364
  forgetFailure();
354
365
  phase = "decided";
355
- try { await preflight(ctx, incomingTokens(event.text, event.images), ctx.getSystemPrompt(), pi.getActiveTools()); }
366
+ try { syncHistoryTools(ctx); await preflight(ctx, incomingTokens(event.text, event.images), ctx.getSystemPrompt(), pi.getActiveTools()); syncHistoryTools(ctx); }
356
367
  catch (error) { fail(error, ctx); }
357
368
  });
358
369
  pi.registerTool({
@@ -425,7 +436,7 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
425
436
  // cut: the system part and the host input. A prompt's cut is pi-tuck's own and never comes here.
426
437
  const saved = ctx.sessionManager.buildSessionProjection().messages;
427
438
  const host = hostTokens(ctx);
428
- const system = systemPart(ctx.getSystemPrompt(), toolShapes(pi.getActiveTools()), saved, ctx.getSystemPrompt());
439
+ const system = requestSystemPart(ctx.getSystemPrompt(), pi.getActiveTools(), saved, ctx.getSystemPrompt());
429
440
  const mandatory = system + host, band = decisionBand(ctx, room, system);
430
441
  const before = plannerMeasureFor(ctx)(saved);
431
442
  // Pi's own usage threshold (reason "threshold") counts the last reply's output tokens, reasoning included, plus the messages after
@@ -466,6 +477,13 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
466
477
  });
467
478
  pi.on("session_before_tree", (event) => event.preparation.userWantsSummary ? { cancel: true } : undefined);
468
479
  pi.on("session_start", () => { clips.clear(); promptCut = undefined; recorder.stop("session-transition"); proposalId = undefined; failure = undefined; admittedRun = false; pendingBoundaryCommit = false; identity = ""; factor = 1; pending = undefined; phase = "idle"; });
480
+ const historyToolsEnabled = (ctx: ExtensionContext) => Boolean(promptCut) || hasTuckCut(ctx.sessionManager.getBranch());
481
+ const syncHistoryTools = (ctx: ExtensionContext) => { if (active()) applyHistoryTools(pi, historyToolsEnabled(ctx)); };
482
+ pi.on("session_start", (_event, ctx) => syncHistoryTools(ctx));
483
+ pi.on("session_tree", (_event, ctx) => syncHistoryTools(ctx));
484
+ pi.on("session_compact", (_event, ctx) => { if (!admittedRun) syncHistoryTools(ctx); });
485
+ pi.on("turn_start", (_event, ctx) => syncHistoryTools(ctx));
486
+ pi.on("agent_settled", (_event, ctx) => syncHistoryTools(ctx));
469
487
  // The session as it will be once a cut that is not saved yet is saved, and its conversation as the next request carries it.
470
488
  const cutSession = (ctx: ExtensionContext, cut: CutEntry) => {
471
489
  const header = ctx.sessionManager.getHeader();
@@ -479,7 +497,7 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
479
497
  // custom message starts). The cut is pi-tuck's own, planned as at a tool boundary. It is not Pi's: Pi's cut finder keeps the last
480
498
  // keepRecentTokens, and where those are the whole conversation it has nothing to cut, so every later prompt was refused for good (t06 at
481
499
  // Pi's defaults on a 32k window). Each request of the run carries the cut, and the run's first boundary saves it.
482
- const preflight = async (ctx: ExtensionContext, incomingPart: number, observedPrompt: string, tools: readonly string[]) => {
500
+ const preflight = (ctx: ExtensionContext, incomingPart: number, observedPrompt: string, tools: readonly string[], hostSnapshot?: number) => {
483
501
  promptCut = undefined;
484
502
  promptCutAt = ctx.sessionManager.getBranch().length;
485
503
  if (!active()) return;
@@ -489,9 +507,9 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
489
507
  const { response, room } = limits(ctx);
490
508
  // Public early view only: later handlers, image normalization and tool-loop growth remain the complete-context guard's responsibility.
491
509
  const saved = ctx.sessionManager.buildSessionProjection().messages;
492
- const system = systemPart(observedPrompt, toolShapes(tools), saved, ctx.getSystemPrompt());
510
+ const system = requestSystemPart(observedPrompt, tools, saved, ctx.getSystemPrompt());
493
511
  const band = decisionBand(ctx, room, system);
494
- const host = hostTokens(ctx), measure = measureFor(ctx), before = measure(saved);
512
+ const host = hostSnapshot ?? hostTokens(ctx), measure = measureFor(ctx), before = measure(saved);
495
513
  const incoming = incomingPart + system, mandatory = incoming + host;
496
514
  if (!fits(incoming, room, ctx))
497
515
  throw new CapacityError("Incoming Pi prompt and response reserve cannot fit the estimated request budget");
@@ -525,8 +543,36 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
525
543
  const prompted = phase === "decided";
526
544
  phase = "starting";
527
545
  try {
546
+ syncHistoryTools(ctx);
528
547
  if (prompted) access(); // decided on input; a failure latched there still stops the run
529
- else await preflight(ctx, incomingEventTokens(event), event.systemPrompt, event.systemPromptOptions.selectedTools);
548
+ else await preflight(ctx, incomingEventTokens(event), event.systemPrompt,
549
+ historyToolNames(pi, event.systemPromptOptions.selectedTools, historyToolsEnabled(ctx)));
550
+ // Pi declares tool changes before turn_start, then snapshots executable tools in prepareRequest. Introduce our tools in
551
+ // the request projection: recording an activation message here would shift every later session-global recall index.
552
+ if (historyToolsEnabled(ctx)) stageHistoryTools(pi, ctx.sessionManager.buildSessionProjection().messages);
553
+ else applyHistoryTools(pi, false);
554
+ const staged = pi.getActiveTools();
555
+ event.systemPromptOptions.selectedTools = historyToolNames(pi, event.systemPromptOptions.selectedTools, true).filter(name =>
556
+ name !== "tuck_recall" && name !== "tuck_search" || staged.includes(name));
557
+ } catch (error) { fail(error, ctx); }
558
+ });
559
+ // The incoming user message is about to be appended (including steering inside a run), and later before_agent_start/turn_start
560
+ // handlers have run. Pi's prepareRequest has not yet snapshotted executable tools. A first cut discovered in context_with_system
561
+ // would be too late for that snapshot, even if its declarations were repaired there, so make the late prompt decision here.
562
+ pi.on("message_end", (event, ctx) => {
563
+ if (event.message.role !== "user" || !active() || !admittedRun || promptCut) return;
564
+ try {
565
+ access();
566
+ if (!ctx.model) throw missingModel();
567
+ const message = event.message;
568
+ const incoming = incomingTokens(typeof message.content === "string" ? message.content : message.content.flatMap(b => b.type === "text" ? [b.text] : []).join("\n"),
569
+ typeof message.content === "string" ? [] : message.content.filter((b): b is ImageContent => b.type === "image"));
570
+ const saved = ctx.sessionManager.buildSessionProjection().messages;
571
+ const host = hostTokens(ctx);
572
+ const system = requestSystemPart(ctx.getSystemPrompt(), pi.getActiveTools(), saved, ctx.getSystemPrompt());
573
+ if (fits(conversationTokens(saved) + system + incoming + host, limits(ctx).room, ctx)) return;
574
+ preflight(ctx, incoming, ctx.getSystemPrompt(), pi.getActiveTools(), host);
575
+ syncHistoryTools(ctx);
530
576
  } catch (error) { fail(error, ctx); }
531
577
  });
532
578
  // A cut at a public boundary, drafted against the boundary's own preview: "tool-loop" between the rounds of a run, "run-end" once its
@@ -545,11 +591,27 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
545
591
  // One snapshot per decision. Messages queued for the next request (steering, follow-ups) are in that request but not in the
546
592
  // saved conversation. Pi's peek returns the steering queue, else the follow-up queue; a follow-up is not in the very next
547
593
  // request while tools still run, so it can be counted early: conservative, and nothing else would cut at that later boundary.
548
- const system = systemPart(ctx.getSystemPrompt(), toolShapes(pi.getActiveTools()), saved, ctx.getSystemPrompt());
594
+ const system = requestSystemPart(ctx.getSystemPrompt(), pi.getActiveTools(), saved, ctx.getSystemPrompt());
549
595
  const band = decisionBand(ctx, room, system);
550
596
  const host = hostTokens(ctx);
551
597
  const queued = event.context.pendingMessages.length ? conversationTokens(event.context.pendingMessages as Messages) : 0;
552
598
  const mandatory = system + host + queued;
599
+ // Without usable provider usage, Pi's own threshold uses its unwrapped message estimate as the compiler's density input.
600
+ // Our declarations are request-only, so Pi can miss that threshold. Preserve the same density input at settlement; every
601
+ // budget decision below still uses `system` and the complete conversation measure, including the summary envelope.
602
+ let densityTokens: number | undefined;
603
+ const lastAssistant = saved.findLast(message => message.role === "assistant");
604
+ const lastCut = ctx.sessionManager.getBranch().findLast(entry => entry.type === "compaction");
605
+ if (reason === "run-end" && lastAssistant && (!lastCut || lastAssistant.timestamp > Date.parse(lastCut.timestamp)) &&
606
+ !saved.some(message => message.role === "assistant" &&
607
+ !["error", "aborted"].includes(message.stopReason) && message.usage && calculateContextTokens(message.usage) > 0)) {
608
+ const forecast = historyToolDeclarations(saved, toolShapes(historyToolNames(pi, pi.getActiveTools(), true)), true);
609
+ if (forecast !== saved) {
610
+ const head = getCurrentSystemMessage(convertToLlm(forecast));
611
+ const native = (head ? estimateTokens(head) : 0) + saved.reduce((sum, message) => sum + (message.role === "system" ? 0 : estimateTokens(message)), 0);
612
+ if (shouldCompact(native, ctx.model.contextWindow, settingsFor().getCompactionSettings(ctx.model))) densityTokens = native;
613
+ }
614
+ }
553
615
  const previousSummary = saved.find(message => message.role === "compactionSummary")?.summary;
554
616
  // The cut previews this decision has built, in the order it built them. The clip retry below measures them for what a cut leaves the
555
617
  // request besides the fresh results. Reset per decision, so a retry sizes itself against the cut it is retrying, not an earlier attempt's.
@@ -559,19 +621,22 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
559
621
  previews = [];
560
622
  floorKept = undefined;
561
623
  const before = measure(saved);
562
- return decideCut({
563
- total: before + mandatory, mandatory, before, room, band, fits: (tokens, limit) => fits(tokens, limit, ctx), usefulOnly: options.cutAt === undefined,
624
+ const gate = gateCut({ total: before + mandatory, mandatory, room, band, fits: (tokens, limit) => fits(tokens, limit, ctx) });
625
+ if (gate.kind !== "plan") return gate;
626
+ return chooseCut({
627
+ opportunistic: gate.opportunistic, mandatory, before, room, band, fits: (tokens, limit) => fits(tokens, limit, ctx), usefulOnly: options.cutAt === undefined,
564
628
  unreduced: "Tool-loop preflight compaction did not reduce context within the estimated request budget",
565
629
  plan: (target): Plan<{ compaction: NonNullable<ReturnType<typeof tuckObservedDraft>["compaction"]>; reserve: number }> => {
566
630
  const reserve = planReserve(target, room, band, mandatory, ctx);
567
- const draft = tuckObservedDraft(tuck, { branchEntries: ctx.sessionManager.getBranch(), tokensBefore: before, previousSummary }, ctx, reserve, plannerMeasureFor(ctx), options.recallPolicy, true);
631
+ const draft = tuckObservedDraft(tuck, { branchEntries: ctx.sessionManager.getBranch(), tokensBefore: densityTokens ?? before, previousSummary }, ctx, reserve, plannerMeasureFor(ctx), options.recallPolicy, true);
568
632
  if (!draft.compaction) {
569
633
  // The planner's own floor, so a clip's second budget is measured against what the smallest cut really keeps.
570
634
  if (draft.floorMessages && event.toolResults) {
571
635
  const fresh = new Set(event.toolResults.map(result => result.toolCallId));
572
636
  const kept = measure(draft.floorMessages.filter(message =>
573
637
  !(message.role === "toolResult" && fresh.has((message as ToolResultMessage).toolCallId))));
574
- floorKept = Math.min(floorKept ?? kept, kept);
638
+ // Both targets must leave their protected openings beside the clip.
639
+ floorKept = Math.max(floorKept ?? kept, kept);
575
640
  }
576
641
  return { draft: false, capacityRejected: Boolean(draft.capacityRejected), reason: draft.unavailable };
577
642
  }
@@ -596,7 +661,7 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
596
661
  const branch = ctx.sessionManager.getBranch();
597
662
  const entryIdOf = (toolCallId: string) => branch.findLast(entry => entry.type === "message" && entry.message.role === "toolResult" && entry.message.toolCallId === toolCallId)?.id;
598
663
  const fresh = new Set(event.toolResults.map(result => result.toolCallId));
599
- const kept = previews.length ? Math.min(...previews.map(preview => measure(preview.filter(message =>
664
+ const kept = previews.length ? Math.max(...previews.map(preview => measure(preview.filter(message =>
600
665
  !(message.role === "toolResult" && fresh.has((message as ToolResultMessage).toolCallId)))))) : floorKept;
601
666
  for (const budget of clipBudgets(room, factor, mandatory, kept)) {
602
667
  const round = clipFor(event.toolResults, entryIdOf, budget);
@@ -638,9 +703,19 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
638
703
  settingsFor().getCompactionSettings(ctx.model).enabled;
639
704
  if (!promptCut && !decide) return;
640
705
  const result = atBoundary(event, ctx, "tool-loop", decide);
641
- if (result) pendingBoundaryCommit = true;
706
+ if (result) {
707
+ pendingBoundaryCommit = true;
708
+ }
642
709
  return result;
643
710
  } catch (error) { fail(error, ctx); }
711
+ finally {
712
+ // Keep our part of Pi's next declaration snapshot unchanged; turn_start restores the branch-derived executable set before
713
+ // prepareRequest. This avoids an extra persisted system message at the first cut and on later rounds/runs.
714
+ if (active()) {
715
+ if (historyToolsEnabled(ctx)) stageHistoryTools(pi, ctx.sessionManager.buildSessionProjection().messages);
716
+ else applyHistoryTools(pi, false);
717
+ }
718
+ }
644
719
  });
645
720
  // A request of a run whose prompt's cut is not saved yet: the saved conversation in it, which other handlers may surround, is replaced by
646
721
  // the conversation with the cut applied. Pi sends the saved system message apart from the request's messages, so neither side carries it
@@ -656,9 +731,11 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
656
731
  pi.on("context", (event, ctx) => {
657
732
  try {
658
733
  if (!active()) return;
734
+ // Before a cut, Pi owns the whole request, including its object identities and system-message serialization.
735
+ if (!historyToolsEnabled(ctx)) return clips.size ? { messages: applyClips(event.messages, clips) } : undefined;
659
736
  access();
660
737
  return { messages: applyClips(projectedMessages(promptCut ? cutRequest(event.messages, ctx, promptCut) : event.messages, ctx, options.recallPolicy), clips) };
661
- } catch (error) { try { fail(error, ctx); } finally { return { messages: [] }; } }
738
+ } catch (error) { try { fail(error, ctx); } finally { return historyToolsEnabled(ctx) ? { messages: [] } : undefined; } }
662
739
  });
663
740
  pi.on("session_compact_failed", (event, ctx) => {
664
741
  // This compactor's own refusal of a Pi threshold ask (the gate, or an opportunistic cut that cannot reach the band) reaches here as a
@@ -699,20 +776,24 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
699
776
  if (!ctx.model) throw missingModel();
700
777
  // The complete request, counted after Pi's conversion (summary/custom/bash envelopes), with the effective system once.
701
778
  // Pi applies a forced prompt after this hook; the public prompt already observes it.
702
- const tokens = requestTokens(event.messages, ctx.getSystemPrompt());
779
+ syncHistoryTools(ctx);
780
+ const messages = historyToolsEnabled(ctx) ? historyToolDeclarations(event.messages, toolShapes(pi.getActiveTools()), true) : event.messages;
781
+ const system = requestSystemPart(ctx.getSystemPrompt(), pi.getActiveTools(), messages, ctx.getSystemPrompt());
782
+ const tokens = system + conversationTokens(messages);
703
783
  if (!fits(tokens, limits(ctx).room, ctx))
704
784
  throw new CapacityError("Complete Pi context and response reserve exceed the estimated request budget");
705
785
  if (recorder.enabled && !recorder.waitingForCut) recorder.measureInline(() => {
706
786
  recorder.request();
707
787
  // No system prompt or hidden assistant blocks; no provider payload/headers.
708
- const permitted = event.messages.filter(m => m.role !== "system" &&
788
+ const permitted = messages.filter(m => m.role !== "system" &&
709
789
  !(m.role === "assistant" && m.content.some(b => b.type === "thinking")) &&
710
790
  !(m.role === "custom" && options.recallPolicy?.privateCustomTypes?.includes(m.customType)));
711
791
  recorder.capture("prepared-request", "public-context-with-system-hook-before-later-transforms-not-wire", { messages: permitted, estimatedInputTokens: tokens }, permitted.length !== event.messages.length);
712
792
  });
713
793
  if (tokens > 0 && Number.isSafeInteger(tokens)) pending = {
714
- identity: identityFor(ctx), tokens, messages: event.messages, snapshot: structuredClone(convertToLlm(event.messages)),
794
+ identity: identityFor(ctx), tokens, messages, snapshot: structuredClone(convertToLlm(messages)),
715
795
  };
796
+ if (messages !== event.messages) return { messages };
716
797
 
717
798
  } catch (error) { fail(error, ctx); }
718
799
  });
@@ -0,0 +1,57 @@
1
+ import { getCurrentTools } from "@earendil-works/pi-ai";
2
+ import { convertToLlm, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
3
+ import type { Messages, ToolShape } from "./budget.js";
4
+
5
+ const own = (name: string) => name === "tuck_recall" || name === "tuck_search";
6
+ /** Native Pi compactions do not make pi-tuck history available. No session-wide activation state. */
7
+ export const hasTuckCut = (branch: readonly { type: string; details?: unknown }[]): boolean => branch.some(entry =>
8
+ entry.type === "compaction" && (entry.details as { compactor?: unknown } | null)?.compactor === "pi-tuck");
9
+
10
+ /** Preserve the host's loadout and its order; only our registered tools may change. */
11
+ export function historyToolNames(pi: ExtensionAPI, names: readonly string[], enabled: boolean): string[] {
12
+ if (!enabled) return names.filter(name => !own(name));
13
+ const registered = new Set(pi.getAllTools().map(tool => tool.name));
14
+ return [...names, ...["tuck_recall", "tuck_search"].filter(name => registered.has(name) && !names.includes(name))];
15
+ }
16
+
17
+ export function applyHistoryTools(pi: ExtensionAPI, enabled: boolean): void {
18
+ const current = pi.getActiveTools(), next = historyToolNames(pi, current, enabled);
19
+ if (current.length !== next.length || current.some((name, index) => name !== next[index])) pi.setActiveTools(next);
20
+ }
21
+
22
+ /** Match only our part of Pi's saved declaration snapshot, so its loop records no activation/removal message. Legacy sessions can
23
+ * already declare our tools; new sessions declare them only in the request projection. Executable tools are restored at turn_start. */
24
+ export function stageHistoryTools(pi: ExtensionAPI, messages: Messages): string[] {
25
+ const declared = new Set(getCurrentTools(convertToLlm(messages)).map(tool => tool.name));
26
+ const current = pi.getActiveTools();
27
+ const next = historyToolNames(pi, current, true).filter(name => !own(name) || declared.has(name));
28
+ if (current.length !== next.length || current.some((name, index) => name !== next[index])) pi.setActiveTools(next);
29
+ return next;
30
+ }
31
+
32
+ /** Pi has already declared the request's tools before queued user messages are appended. A prompt cut planned on such a message needs its declarations
33
+ * in that very request. Change only our declarations, leaving all other system content and tools in their original places. */
34
+ export function historyToolDeclarations(messages: Messages, tools: readonly ToolShape[], enabled: boolean): Messages {
35
+ let changed = false;
36
+ const filtered = messages.map(message => {
37
+ if (message.role !== "system") return message;
38
+ const added = message.toolsAdded?.filter(tool => !own(tool.name));
39
+ const removed = message.toolsRemoved?.filter(tool => !own(tool.name));
40
+ if (added?.length === message.toolsAdded?.length && removed?.length === message.toolsRemoved?.length) return message;
41
+ changed = true;
42
+ const { toolsAdded: _, toolsRemoved: __, ...rest } = message;
43
+ return { ...rest, ...(added?.length ? { toolsAdded: added } : {}), ...(removed?.length ? { toolsRemoved: removed } : {}) };
44
+ });
45
+ // Leave an already-correct transcript byte-identical, including the original placement of our declarations.
46
+ const declared = getCurrentTools(convertToLlm(messages));
47
+ const wanted = enabled ? tools.filter(tool => own(tool.name)) : [];
48
+ if (enabled && wanted.every(tool => declared.some(current => own(current.name) && current.name === tool.name)) &&
49
+ declared.filter(tool => own(tool.name)).length === wanted.length) return messages;
50
+ if (!enabled || !wanted.length) return changed ? filtered : messages;
51
+ const at = filtered.findLastIndex(message => message.role === "system");
52
+ if (at < 0) throw new Error("Missing Pi system message for history tool declarations");
53
+ const system = filtered[at]!;
54
+ if (system.role !== "system") throw new Error("Missing Pi system message");
55
+ filtered[at] = { ...system, toolsAdded: [...(system.toolsAdded ?? []), ...wanted as NonNullable<typeof system.toolsAdded>] };
56
+ return filtered;
57
+ }
package/src/ledger.ts CHANGED
@@ -4,8 +4,6 @@ import { isDiscardedResponse } from "./protocol";
4
4
  import { extractPath } from "./tuck/core/tool-args";
5
5
  import type { TranscriptRecord } from "./tuck/core/transcript";
6
6
 
7
- /** The initiating prompt is quoted whole up to this many characters; the rest stays readable through tuck_recall. */
8
- export const MAX_TASK_CONTRACT_CHARACTERS = 12000;
9
7
  /** Owner default; older distinct paths are counted, not listed. */
10
8
  export const MAX_LEDGER_LINES = 60;
11
9
  const LEDGER_TITLE = "Files";
package/src/omissions.ts CHANGED
@@ -1,26 +1,72 @@
1
1
  import type { ExtensionContext, SessionEntry } from '@earendil-works/pi-coding-agent';
2
2
  import { buildGlobalIndexById } from './tuck/core/global-indices';
3
- import { parseRecords, splitSummary, transcriptHeader } from './tuck/core/transcript';
3
+ import { parseRecords, serializeRecord, splitSummary, transcriptHeader } from './tuck/core/transcript';
4
+ import { textOf } from "./tuck/core/content";
5
+ import { isDiscardedResponse } from "./protocol";
4
6
  import { createRecall, type RecallPolicy } from './recall';
5
7
  /** Native branch edits supply exclusion; original global indices supply provenance. */
6
8
  export function omittedSourceIndices(entries: readonly SessionEntry[], branch: readonly SessionEntry[]): ReadonlySet<number> {
7
- const indices = buildGlobalIndexById(entries);
8
9
  const edits = new Set<string>();
9
10
  for (const entry of branch) if (entry.type === 'context_edit') {
10
11
  if (entry.replacement !== null) throw new Error('pi-tuck does not support replacement context edits');
11
12
  edits.add(entry.targetId);
12
13
  }
14
+ if (!edits.size) return new Set();
15
+ const indices = buildGlobalIndexById(entries);
13
16
  return new Set([...edits].flatMap(id => indices.has(id) ? [indices.get(id)!] : []));
14
17
  }
15
18
  /** Preserve surviving record bytes/order, and the summary's format (a v1 summary stays v1). No text matching or semantic inference. */
16
- export function filterOmittedSummary(summary: string, omitted: ReadonlySet<number>): string {
19
+ export function filterOmittedSummary(summary: string, omitted: ReadonlySet<number>, sources?: readonly SessionEntry[], rangeOmitted = omitted): string {
17
20
  if (!omitted.size) return summary;
18
21
  const split = splitSummary(summary);
19
22
  if (!split) throw new Error('Cannot apply native source omission to an unsupported compaction summary');
20
23
  const { body, closing, version } = split;
21
24
  const records = parseRecords(body, version), lines = body ? body.split('\n') : [];
22
- const retained = records.flatMap((record, i) =>
23
- (record.sourceIndex !== undefined && omitted.has(record.sourceIndex)) || record.sourceIndices?.some(index => omitted.has(index)) ? [] : [lines[i]!]);
25
+ // A broad range does not identify the role of an intervening source. Without
26
+ // original sources there is no evidence that an edit removes a counted unit.
27
+ const indices = sources && rangeOmitted.size && records.some(record => record.sourceRange) ? buildGlobalIndexById(sources) : undefined;
28
+ const sourceUnits = new Map<number, { role: string; missing: number }>();
29
+ const shownBySource = new Map<string, string[]>();
30
+ for (const record of records) if (record.kind === 'excerpt' && record.sourceIndex !== undefined && !record.span) {
31
+ const key = `${record.role}:${record.sourceIndex}`, shown = shownBySource.get(key) ?? [];
32
+ shown.push(record.text); shownBySource.set(key, shown);
33
+ }
34
+ if (indices && sources) for (const entry of sources) {
35
+ if (entry.type !== 'message') continue;
36
+ const index = indices.get(entry.id);
37
+ if (index === undefined) continue;
38
+ const message = entry.message;
39
+ const shown = shownBySource.get(`${message.role}:${index}`) ?? [];
40
+ if (message.role === 'user') {
41
+ const text = textOf(message.content);
42
+ sourceUnits.set(index, { role: 'user', missing: Number(Boolean(text.length) && !shown.includes(text)) });
43
+ } else if (message.role === 'assistant' && !isDiscardedResponse(message)) {
44
+ const available = [...shown];
45
+ let missing = 0;
46
+ for (const part of message.content) if (part.type === 'text' && part.text.trim()) {
47
+ const at = available.indexOf(part.text);
48
+ if (at < 0) missing++; else available.splice(at, 1);
49
+ }
50
+ sourceUnits.set(index, { role: 'assistant', missing });
51
+ } else if (message.role === 'toolResult') sourceUnits.set(index, { role: 'toolResult', missing: 1 });
52
+ }
53
+ const retained = records.flatMap((record, at) => {
54
+ if ((record.sourceIndex !== undefined && omitted.has(record.sourceIndex)) || record.sourceIndices?.some(index => omitted.has(index))) return [];
55
+ if (!record.sourceRange || !indices || record.count === undefined) return [lines[at]!];
56
+ const range = record.sourceRange;
57
+ let removed = 0;
58
+ const remaining: number[] = [];
59
+ for (const [index, unit] of sourceUnits) {
60
+ if (index < range.start || index > range.end || unit.role !== record.role || !unit.missing) continue;
61
+ if (rangeOmitted.has(index)) removed += unit.missing; else if (!omitted.has(index)) remaining.push(index);
62
+ }
63
+ if (!removed) return [lines[at]!];
64
+ const count = record.count - removed;
65
+ if (count <= 0) return [];
66
+ const sourceRange = remaining.length ? { start: Math.min(...remaining), end: Math.max(...remaining) } : range;
67
+ const adjusted = { ...record, count, sourceRange };
68
+ return [version === 1 ? JSON.stringify(adjusted) : serializeRecord(adjusted)];
69
+ });
24
70
  return transcriptHeader(version) + '\n' + retained.join('\n') + closing;
25
71
  }
26
72
 
@@ -29,13 +75,14 @@ export function filterCustomQuotes(summary: string, entries: readonly SessionEnt
29
75
  const split = splitSummary(summary);
30
76
  if (!split) throw new Error('Cannot resolve custom sources in an unsupported compaction summary');
31
77
  const { body, closing, version } = split;
78
+ const records = parseRecords(body, version);
79
+ if (!records.some(record => record.role === "custom")) return summary;
32
80
  const omitted = new Set(branch.flatMap(entry => entry.type === 'context_edit' && entry.replacement === null ? [entry.targetId] : []));
33
81
  const eligible = createRecall(policy).isHistorySource, indices = buildGlobalIndexById(entries);
34
82
  const globalCounts = new Map<string, number>(), branchSources = new Map<string, SessionEntry[]>();
35
83
  for (const entry of entries) globalCounts.set(entry.id, (globalCounts.get(entry.id) ?? 0) + 1);
36
84
  for (const entry of branch) { const found = branchSources.get(entry.id) ?? []; found.push(entry); branchSources.set(entry.id, found); }
37
85
  const lines = body ? body.split('\n') : [];
38
- const records = parseRecords(body, version);
39
86
  const retained = records.flatMap((record, i) => {
40
87
  if (record.role !== 'custom') return [lines[i]!];
41
88
  const id = record.sourceEntryId, source = id === undefined ? undefined : branchSources.get(id);
@@ -52,8 +99,18 @@ type ProjectedMessages = ReturnType<ExtensionContext['sessionManager']['buildSes
52
99
  function projectMessages(messages: ProjectedMessages, ctx: ExtensionContext, policy: RecallPolicy | undefined, allowForeign: boolean): ProjectedMessages {
53
100
  const entries = ctx.sessionManager.getEntries(), branch = ctx.sessionManager.getBranch();
54
101
  const omitted = omittedSourceIndices(entries, branch);
102
+ const lastCut = branch.findLastIndex(entry => entry.type === 'compaction');
103
+ const cut = branch[lastCut];
104
+ const afterCut = new Set<number>();
105
+ if (omitted.size && cut?.type === 'compaction') {
106
+ const indices = buildGlobalIndexById(entries);
107
+ for (const entry of branch.slice(lastCut + 1)) if (entry.type === 'context_edit' && entry.replacement === null) {
108
+ const index = indices.get(entry.targetId); if (index !== undefined) afterCut.add(index);
109
+ }
110
+ }
55
111
  return messages.map(message => message.role === 'compactionSummary' && (!allowForeign || splitSummary(message.summary) !== undefined)
56
- ? { ...message, summary: filterCustomQuotes(filterOmittedSummary(message.summary, omitted), entries, branch, policy) } : message);
112
+ ? { ...message, summary: filterCustomQuotes(filterOmittedSummary(message.summary, omitted, entries,
113
+ cut?.type === 'compaction' && cut.summary === message.summary ? afterCut : new Set()), entries, branch, policy) } : message);
57
114
  }
58
115
  export function projectedMessages(messages: ProjectedMessages, ctx: ExtensionContext, policy?: RecallPolicy): ProjectedMessages {
59
116
  return projectMessages(messages, ctx, policy, false);
@@ -123,19 +123,17 @@ export const isDialogue = (block: NormalizedBlock): block is DialogueBlock => bl
123
123
 
124
124
  /** A user or assistant text block the omission notices count when it is unselected: non-empty, and not rendered in the mandatory base. */
125
125
  export const isOmittable = (block: NormalizedBlock, mandatorySources?: ReadonlySet<number>): boolean =>
126
- (block.kind === "user" || block.kind === "assistant") && Boolean(block.text.trim()) &&
126
+ (block.kind === "assistant" || block.kind === "user" && !block.image) && Boolean(block.text.trim()) &&
127
127
  (block.sourceIndex === undefined || !mandatorySources?.has(block.sourceIndex));
128
128
 
129
- /** `sources` are the distinct locators of the omitted blocks in ascending order; at most the first 9 decide the notice. They are written
130
- * as "#N" on the record's line. */
131
- export const omissionRecord = (role: "user" | "assistant", count: number, sources: number[], unlocated: boolean): TranscriptRecord => ({
129
+ /** Count dropped units inside inclusive original-index bounds. The range may
130
+ * contain retained units and other roles; only omitted units of this role count. */
131
+ export const omissionRecord = (role: "user" | "assistant" | "toolResult", count: number, sources: number[], unlocated: boolean): TranscriptRecord => ({
132
132
  kind: "omission", role, count,
133
- ...(sources.length ? { sourceIndices: sources.slice(0, 8) } : {}),
134
- reason: "Completely unselected historical text blocks; count is text blocks, not messages. " +
135
- "The # locators are individual original locators, not a contiguous range. " +
136
- (sources.length > 8 ? "Locator list incomplete. " : "") +
137
- (unlocated ? "Some text blocks have no source locator. " : "") +
138
- "Read originals with tuck_recall; omitted text may qualify retained text. No new authority." });
133
+ ...(sources.length ? { sourceRange: { start: sources.reduce((a, b) => Math.min(a, b)), end: sources.reduce((a, b) => Math.max(a, b)) } } : {}),
134
+ reason: (role === "toolResult" ? "Historical tool actions and observations omitted. " : "Historical text units omitted. ") +
135
+ (unlocated ? "Some units have no source locator. " : "") +
136
+ "Count is omitted units of this role in the index range; read originals with tuck_recall." });
139
137
 
140
138
  const analyses = new WeakMap<DialogueBlock, Analysis>();
141
139
 
@@ -159,10 +157,27 @@ export const blockRecords = (b: DialogueBlock, mandatory: ReadonlySet<number> |
159
157
  return excerptRecords(b.text, analysis, role, b.sourceIndex, clip[0], clip[1], readFrom).map(record => ({ ...record, ...notice }));
160
158
  };
161
159
 
160
+ /** Exact user text prefixes are allowed to split a passage. Unlike assistant
161
+ * passage clips, their explicit continuation points at the unabridged input. */
162
+ export const userPrefixRecords = (block: DialogueBlock, mandatory: ReadonlySet<number> | undefined, take: number): TranscriptRecord[] => {
163
+ if (block.kind !== "user" || block.sourceIndex !== undefined && mandatory?.has(block.sourceIndex)) return [];
164
+ const text = block.originalText ?? block.text;
165
+ if (take <= 0 || !text.length) return [];
166
+ let end = Math.min(text.length, Math.floor(take));
167
+ // Charge the complete code point in the measured candidate, including the
168
+ // low surrogate, so a protected 80-character opening never shrinks to 79.
169
+ if (end < text.length && /[\uD800-\uDBFF]/u.test(text[end - 1]!)) end++;
170
+ const locator = block.sourceIndex === undefined ? undefined : { index: block.sourceIndex };
171
+ return [{ kind: "excerpt", role: "user", sourceIndex: block.sourceIndex, text: text.slice(0, end) },
172
+ ...(end < text.length ? [{ kind: "omission" as const, role: "user" as const, sourceIndex: block.sourceIndex,
173
+ reason: locator ? `tuck_recall ${JSON.stringify(block.startsSource ? { ...locator, offset: end } : locator)}` : "Source has no locator" }] : [])];
174
+ };
175
+
162
176
  /** A source time for each locator a user record names; only briefs of times-carrying blocks have them. */
163
177
  export const decorateRecord = (record: TranscriptRecord, times: NormalizedBlock["sourceTimes"]): TranscriptRecord => {
164
178
  // Times mark the user's turns only: on every record they cost a fifth to a third of the summary's tokens.
165
179
  if (!times || record.role !== "user") return record;
180
+ if (record.sourceRange) return record;
166
181
  const indices = record.sourceIndices ?? (record.sourceIndex === undefined ? [] : [record.sourceIndex]);
167
182
  return indices.length ? { ...record, sourceTimes: indices.map(sourceIndex => ({ sourceIndex, ...(times[sourceIndex] ?? { basis: "unknown" as const }) })) } : record;
168
183
  };