pi-tuck 1.0.16 → 1.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +75 -0
- package/README.md +7 -4
- package/dist/src/budget.d.ts +14 -17
- package/dist/src/budget.js +22 -29
- package/dist/src/build-id.d.ts +1 -1
- package/dist/src/build-id.js +1 -1
- package/dist/src/compaction.js +113 -20
- package/dist/src/diagnostics.d.ts +1 -1
- package/dist/src/history-tools.d.ts +16 -0
- package/dist/src/history-tools.js +59 -0
- package/dist/src/ledger.d.ts +0 -2
- package/dist/src/ledger.js +0 -2
- package/dist/src/omissions.d.ts +1 -1
- package/dist/src/omissions.js +88 -6
- package/dist/src/tuck/core/brief.d.ts +6 -3
- package/dist/src/tuck/core/brief.js +27 -9
- package/dist/src/tuck/core/filter-noise.d.ts +1 -1
- package/dist/src/tuck/core/filter-noise.js +1 -3
- package/dist/src/tuck/core/normalize.d.ts +2 -5
- package/dist/src/tuck/core/normalize.js +15 -33
- package/dist/src/tuck/core/shrink.d.ts +6 -14
- package/dist/src/tuck/core/shrink.js +162 -102
- package/dist/src/tuck/core/summarize.d.ts +3 -14
- package/dist/src/tuck/core/summarize.js +9 -20
- package/dist/src/tuck/core/transcript.d.ts +5 -0
- package/dist/src/tuck/core/transcript.js +27 -4
- package/dist/src/tuck/types.d.ts +1 -1
- package/dist/src/tuck.d.ts +9 -17
- package/dist/src/tuck.js +384 -212
- package/package.json +1 -1
- package/src/budget.ts +24 -26
- package/src/build-id.ts +1 -1
- package/src/compaction.ts +102 -21
- package/src/history-tools.ts +57 -0
- package/src/ledger.ts +0 -2
- package/src/omissions.ts +64 -7
- package/src/tuck/core/brief.ts +25 -10
- package/src/tuck/core/filter-noise.ts +1 -3
- package/src/tuck/core/normalize.ts +11 -27
- package/src/tuck/core/shrink.ts +141 -91
- package/src/tuck/core/summarize.ts +9 -24
- package/src/tuck/core/transcript.ts +26 -4
- package/src/tuck/types.ts +3 -3
- package/src/tuck.ts +336 -185
package/package.json
CHANGED
package/src/budget.ts
CHANGED
|
@@ -26,12 +26,26 @@ export const validCutAt = (value: unknown): value is number => typeof value ===
|
|
|
26
26
|
/** The initial default cut point: 60% of the room before a usable previous cut. Measured against 1 with paired replays (no answer loss) and billed
|
|
27
27
|
* input under automatic and explicit prompt caching (about 17% less at 128k, 10-14% at 32k). */
|
|
28
28
|
export const DEFAULT_CUT_AT = 0.6;
|
|
29
|
+
/** Preferred native carry, in the planner's estimate: never more than the
|
|
30
|
+
* keep-recent cut offers or the tail amount the adaptive band assumes. */
|
|
31
|
+
export const carriedTailTokens = (offeredTokens: number): number => Math.min(offeredTokens, MAX_TAIL_TOKENS / 2);
|
|
29
32
|
export const bandOf = (room: number, cutAt = 1, adaptive?: { system: number; tokensAfter: number; factor: number }): number => {
|
|
30
33
|
const positive = Math.max(0, room);
|
|
31
34
|
const base = Math.floor(positive * cutAt);
|
|
32
|
-
return Math.min(room - Math.ceil(positive * TRIGGER_MARGIN_FRACTION), adaptive ? Math.max(base, Math.ceil((adaptive.system + adaptive.tokensAfter + MAX_TAIL_TOKENS
|
|
35
|
+
return Math.min(room - Math.ceil(positive * TRIGGER_MARGIN_FRACTION), adaptive ? Math.max(base, Math.ceil((adaptive.system + adaptive.tokensAfter + carriedTailTokens(MAX_TAIL_TOKENS)) * adaptive.factor)) : base);
|
|
33
36
|
};
|
|
34
37
|
|
|
38
|
+
/** The cut's one carried-context grant, in the planner's estimate. Combines
|
|
39
|
+
* the existing native/fixed base with the quarter-free historical amount;
|
|
40
|
+
* neither component remains an independent spending limit in the fill.
|
|
41
|
+
* The grant depends only on quantities. */
|
|
42
|
+
export function carriedTarget(window: number, reserve: number, base: number, before: number): number {
|
|
43
|
+
const room = Math.max(0, window - reserve - 1);
|
|
44
|
+
const free = Math.max(0, window - reserve - base - 1);
|
|
45
|
+
const historical = Math.max(0, Math.min(Math.floor(free / 4), before - base - 1));
|
|
46
|
+
return Math.min(room, before - 1, base + historical);
|
|
47
|
+
}
|
|
48
|
+
|
|
35
49
|
/** The APIs whose request converter (pi-ai openai-responses-shared) sends each signed thinking block back as an input reasoning item. */
|
|
36
50
|
const REPLAYING_APIS: ReadonlySet<string> = new Set(["openai-responses", "azure-openai-responses", "openai-codex-responses"]);
|
|
37
51
|
/**
|
|
@@ -91,8 +105,8 @@ export const planningReserve = (window: number, targetRoom: number, factor: numb
|
|
|
91
105
|
window - Math.floor(targetRoom / factor) + mandatory;
|
|
92
106
|
|
|
93
107
|
export type Target = "band" | "room";
|
|
94
|
-
/** A planned cut: the conversation it leaves
|
|
95
|
-
export type Plan<D extends object = {}> = ({ draft: true; after: number; mode
|
|
108
|
+
/** A planned cut: the conversation it leaves. The optional mode is diagnostic data, not a decision input. */
|
|
109
|
+
export type Plan<D extends object = {}> = ({ draft: true; after: number; mode?: string } & D) | { draft: false; capacityRejected: boolean; reason: string };
|
|
96
110
|
export type Decision<D extends object = {}> =
|
|
97
111
|
/** The request fits below the band, or an opportunistic cut cannot help: no cut and no fault; the request goes to the guard. */
|
|
98
112
|
| { kind: "none" }
|
|
@@ -118,39 +132,23 @@ export interface CutInput<D extends object = {}> {
|
|
|
118
132
|
}
|
|
119
133
|
/** What a cut decision resolves to once the planner has been asked. */
|
|
120
134
|
export type Verdict<D extends object = {}> = Exclude<Decision<D>, { kind: "defer" }>;
|
|
121
|
-
/**
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
* room (a cut that still lands in the band would trigger again every round); the exact room would cut nothing there, so a band cut is
|
|
126
|
-
* taken only when the room can plan a cut too and with the same mode (a lossy view the room could not even plan is a quality change with
|
|
127
|
-
* no fault to prevent). At or above the room, where a cut is due, a band plan of another mode gives way to the room plan (today's
|
|
128
|
-
* view), and a band plan stands when the room has none (a cut instead of a fault).
|
|
129
|
-
* A lean band, one the cut point binds (`band < bandOf(room)`), is planned first and the room plan landing below it is not taken for that:
|
|
130
|
-
* the room plan keeps a native tail of up to half the room, so it can land just under a band that far below the room, and the next round
|
|
131
|
-
* would cut again, freeing little. The room is planned only where the rules above use it: to compare the mode, or when the band has no plan
|
|
132
|
-
* and a cut is due. An adaptive band can also reach the margin, in which case the room plan is tried first.
|
|
133
|
-
* The margin adds no new decision fault. Exceptions thrown by `plan` are not caught here: they are faults, and they now surface at
|
|
134
|
-
* boundaries inside the band too, where no planner ran before the margin.
|
|
135
|
-
*/
|
|
135
|
+
/** Plan the band first at every cut point. An opportunistic cut must land
|
|
136
|
+
* below it and free the existing minimum. A due cut takes a band draft, or
|
|
137
|
+
* falls back to the room only when the band has no draft. Final reduction and
|
|
138
|
+
* room-fit checks own faults. Planner exceptions propagate. */
|
|
136
139
|
export function chooseCut<D extends object = {}>(input: CutInput<D> & { opportunistic: boolean }): Verdict<D> {
|
|
137
140
|
const { before, mandatory, room, band, fits, plan, unreduced } = input;
|
|
138
141
|
const lands = (p: Plan<D>): p is Extract<Plan<D>, { draft: true }> => p.draft && p.after < before && fits(p.after + mandatory, band);
|
|
139
142
|
// A cut no one needs yet must be worth its summary: one that frees under a tenth of the room would trigger again within a round or two.
|
|
140
|
-
//
|
|
143
|
+
// A due cut is not asked: it has to happen.
|
|
141
144
|
const worthwhile = (p: Extract<Plan<D>, { draft: true }>) => !input.usefulOnly || !input.opportunistic || !fits(10 * (before - p.after), room);
|
|
142
|
-
let roomPlan: Plan<D> | undefined;
|
|
143
|
-
const exact = () => roomPlan ??= plan("room");
|
|
144
|
-
if (band >= bandOf(room)) { const e = exact(); if (lands(e) && worthwhile(e)) return { kind: "cut", target: "room", plan: e }; }
|
|
145
145
|
const preferred = plan("band");
|
|
146
146
|
if (input.opportunistic) {
|
|
147
147
|
if (!lands(preferred) || !worthwhile(preferred)) return { kind: "none" };
|
|
148
|
-
|
|
149
|
-
return e.draft && e.mode === preferred.mode ? { kind: "cut", target: "band", plan: preferred } : { kind: "none" };
|
|
148
|
+
return { kind: "cut", target: "band", plan: preferred };
|
|
150
149
|
}
|
|
151
|
-
const e = exact();
|
|
152
150
|
let chosen: Plan<D> = preferred, target: Target = "band";
|
|
153
|
-
if (!preferred.draft
|
|
151
|
+
if (!preferred.draft) { chosen = plan("room"); target = "room"; }
|
|
154
152
|
if (!chosen.draft) return { kind: "fault", capacity: chosen.capacityRejected, message: chosen.reason };
|
|
155
153
|
if (chosen.after >= before || !fits(chosen.after + mandatory, room))
|
|
156
154
|
return { kind: "fault", capacity: !fits(chosen.after + mandatory, room), message: unreduced };
|
package/src/build-id.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const BUILD_ID = "
|
|
1
|
+
export const BUILD_ID = "c53f4e367c1f45ac6b3865cbf25bf99992da4b7e961557e5510369dca81fb087";
|
package/src/compaction.ts
CHANGED
|
@@ -6,10 +6,10 @@ import { isDeepStrictEqual } from "node:util";
|
|
|
6
6
|
import { assertNativeToolRounds } from "./protocol.js";
|
|
7
7
|
import { projectedMessages } from "./omissions.js";
|
|
8
8
|
import { applyClips, clipBudgets, clipFor, type Clip } from "./clip.js";
|
|
9
|
-
import { DEFAULT_CUT_AT, bandOf, chooseCut, conversationTokens,
|
|
9
|
+
import { DEFAULT_CUT_AT, bandOf, chooseCut, conversationTokens, gateCut, incomingEventTokens, incomingTokens, measurableConversationTokens, planningReserve, projectedConversationTokens, systemPart, validCutAt,
|
|
10
10
|
type AgentMessage, type Messages, type Plan, type Target, type ToolShape } from "./budget.js";
|
|
11
|
-
import { isContextOverflow, type ToolResultMessage } from "@earendil-works/pi-ai";
|
|
12
|
-
import { convertToLlm, SessionManager, SettingsManager, type ExtensionAPI, type ExtensionContext, type SessionEntry } from "@earendil-works/pi-coding-agent";
|
|
11
|
+
import { getCurrentSystemMessage, isContextOverflow, type ImageContent, type ToolResultMessage } from "@earendil-works/pi-ai";
|
|
12
|
+
import { calculateContextTokens, convertToLlm, estimateTokens, shouldCompact, SessionManager, SettingsManager, type ExtensionAPI, type ExtensionContext, type SessionEntry } from "@earendil-works/pi-coding-agent";
|
|
13
13
|
import { Type } from "typebox";
|
|
14
14
|
import { Check } from "typebox/value";
|
|
15
15
|
import type { RecallPolicy } from "./recall.js";
|
|
@@ -18,6 +18,7 @@ import { MAX_RECALL_CHARACTERS, MAX_RECALL_ENTRY_ID_CHARACTERS, MAX_RECALL_QUERY
|
|
|
18
18
|
import { tuck, tuckDraft, tuckObservedDraft, tuckRecall } from "./tuck.js";
|
|
19
19
|
import { ImageRecallInput, historyImage, historyAround } from "./history-image.js";
|
|
20
20
|
import { eligibilityKey, eligibilityKeyMoved, memoizedEligibleSources } from "./eligibility-cache";
|
|
21
|
+
import { applyHistoryTools, hasTuckCut, historyToolDeclarations, historyToolNames, stageHistoryTools } from "./history-tools.js";
|
|
21
22
|
|
|
22
23
|
const query = Type.String({ minLength: 1, maxLength: MAX_RECALL_QUERY_CHARACTERS });
|
|
23
24
|
const entryId = Type.String({ minLength: 1, maxLength: MAX_RECALL_ENTRY_ID_CHARACTERS });
|
|
@@ -296,12 +297,22 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
296
297
|
const measureFor = (ctx: ExtensionContext) => (messages: Messages) => projectedConversationTokens(applyClips(messages, clips), ctx, options.recallPolicy);
|
|
297
298
|
// The planner counts with the same function, except that it does not fail on a summary in another format: a cut can replace one
|
|
298
299
|
// (an operator's /compact repairs a session whose saved summary is foreign). Identical to the strict one for this compactor's summaries.
|
|
299
|
-
const plannerMeasureFor = (ctx: ExtensionContext) => (
|
|
300
|
+
const plannerMeasureFor = (ctx: ExtensionContext) => Object.assign(
|
|
301
|
+
(messages: Messages) => measurableConversationTokens(applyClips(messages, clips), ctx, options.recallPolicy),
|
|
302
|
+
{ messageTokens: (message: Messages[number]) => conversationTokens(applyClips([message], clips)) });
|
|
300
303
|
const hostTokens = (ctx: ExtensionContext) => options.hostMessages ? conversationTokens(options.hostMessages(ctx)) : 0;
|
|
301
304
|
const toolShapes = (names: Iterable<string>): ToolShape[] => {
|
|
302
305
|
const selected = new Set(names);
|
|
303
306
|
return pi.getAllTools().filter(tool => selected.has(tool.name)).map(tool => ({ name: tool.name, description: tool.description, parameters: tool.parameters }));
|
|
304
307
|
};
|
|
308
|
+
// Every decision prices the declaration set a cut must carry, even before the first cut enables our tools on the wire.
|
|
309
|
+
const requestSystemPart = (observedPrompt: string, names: readonly string[], saved: Messages, guardPrompt: string) => {
|
|
310
|
+
const shapes = toolShapes(historyToolNames(pi, names, true));
|
|
311
|
+
// Keep the guard's observed schemas (Pi may enrich a host tool's schema), and add our future declarations to that same view.
|
|
312
|
+
// Taking max(current guard, future raw shapes) alone could lose the enrichment when the future shapes are larger.
|
|
313
|
+
const future = saved.some(message => message.role === "system") ? historyToolDeclarations(saved, shapes, true) : saved;
|
|
314
|
+
return systemPart(observedPrompt, shapes, future, guardPrompt);
|
|
315
|
+
};
|
|
305
316
|
// Pi reports a manual cut with nothing to do as a failed compaction; the thrown error lacks the event's "Compaction failed: " prefix.
|
|
306
317
|
const isNoWork = (message: string | undefined) => ["Already compacted", "Nothing to compact (session too small)"].includes(message?.replace(/^Compaction failed: /, "") ?? "");
|
|
307
318
|
// A missing model is configuration, not capacity.
|
|
@@ -352,7 +363,7 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
352
363
|
if (!ctx.isIdle() || phase === "starting") return; // a steering or follow-up message joins the running request
|
|
353
364
|
forgetFailure();
|
|
354
365
|
phase = "decided";
|
|
355
|
-
try { await preflight(ctx, incomingTokens(event.text, event.images), ctx.getSystemPrompt(), pi.getActiveTools()); }
|
|
366
|
+
try { syncHistoryTools(ctx); await preflight(ctx, incomingTokens(event.text, event.images), ctx.getSystemPrompt(), pi.getActiveTools()); syncHistoryTools(ctx); }
|
|
356
367
|
catch (error) { fail(error, ctx); }
|
|
357
368
|
});
|
|
358
369
|
pi.registerTool({
|
|
@@ -425,7 +436,7 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
425
436
|
// cut: the system part and the host input. A prompt's cut is pi-tuck's own and never comes here.
|
|
426
437
|
const saved = ctx.sessionManager.buildSessionProjection().messages;
|
|
427
438
|
const host = hostTokens(ctx);
|
|
428
|
-
const system =
|
|
439
|
+
const system = requestSystemPart(ctx.getSystemPrompt(), pi.getActiveTools(), saved, ctx.getSystemPrompt());
|
|
429
440
|
const mandatory = system + host, band = decisionBand(ctx, room, system);
|
|
430
441
|
const before = plannerMeasureFor(ctx)(saved);
|
|
431
442
|
// Pi's own usage threshold (reason "threshold") counts the last reply's output tokens, reasoning included, plus the messages after
|
|
@@ -466,6 +477,13 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
466
477
|
});
|
|
467
478
|
pi.on("session_before_tree", (event) => event.preparation.userWantsSummary ? { cancel: true } : undefined);
|
|
468
479
|
pi.on("session_start", () => { clips.clear(); promptCut = undefined; recorder.stop("session-transition"); proposalId = undefined; failure = undefined; admittedRun = false; pendingBoundaryCommit = false; identity = ""; factor = 1; pending = undefined; phase = "idle"; });
|
|
480
|
+
const historyToolsEnabled = (ctx: ExtensionContext) => Boolean(promptCut) || hasTuckCut(ctx.sessionManager.getBranch());
|
|
481
|
+
const syncHistoryTools = (ctx: ExtensionContext) => { if (active()) applyHistoryTools(pi, historyToolsEnabled(ctx)); };
|
|
482
|
+
pi.on("session_start", (_event, ctx) => syncHistoryTools(ctx));
|
|
483
|
+
pi.on("session_tree", (_event, ctx) => syncHistoryTools(ctx));
|
|
484
|
+
pi.on("session_compact", (_event, ctx) => { if (!admittedRun) syncHistoryTools(ctx); });
|
|
485
|
+
pi.on("turn_start", (_event, ctx) => syncHistoryTools(ctx));
|
|
486
|
+
pi.on("agent_settled", (_event, ctx) => syncHistoryTools(ctx));
|
|
469
487
|
// The session as it will be once a cut that is not saved yet is saved, and its conversation as the next request carries it.
|
|
470
488
|
const cutSession = (ctx: ExtensionContext, cut: CutEntry) => {
|
|
471
489
|
const header = ctx.sessionManager.getHeader();
|
|
@@ -479,7 +497,7 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
479
497
|
// custom message starts). The cut is pi-tuck's own, planned as at a tool boundary. It is not Pi's: Pi's cut finder keeps the last
|
|
480
498
|
// keepRecentTokens, and where those are the whole conversation it has nothing to cut, so every later prompt was refused for good (t06 at
|
|
481
499
|
// Pi's defaults on a 32k window). Each request of the run carries the cut, and the run's first boundary saves it.
|
|
482
|
-
const preflight =
|
|
500
|
+
const preflight = (ctx: ExtensionContext, incomingPart: number, observedPrompt: string, tools: readonly string[], hostSnapshot?: number) => {
|
|
483
501
|
promptCut = undefined;
|
|
484
502
|
promptCutAt = ctx.sessionManager.getBranch().length;
|
|
485
503
|
if (!active()) return;
|
|
@@ -489,9 +507,9 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
489
507
|
const { response, room } = limits(ctx);
|
|
490
508
|
// Public early view only: later handlers, image normalization and tool-loop growth remain the complete-context guard's responsibility.
|
|
491
509
|
const saved = ctx.sessionManager.buildSessionProjection().messages;
|
|
492
|
-
const system =
|
|
510
|
+
const system = requestSystemPart(observedPrompt, tools, saved, ctx.getSystemPrompt());
|
|
493
511
|
const band = decisionBand(ctx, room, system);
|
|
494
|
-
const host = hostTokens(ctx), measure = measureFor(ctx), before = measure(saved);
|
|
512
|
+
const host = hostSnapshot ?? hostTokens(ctx), measure = measureFor(ctx), before = measure(saved);
|
|
495
513
|
const incoming = incomingPart + system, mandatory = incoming + host;
|
|
496
514
|
if (!fits(incoming, room, ctx))
|
|
497
515
|
throw new CapacityError("Incoming Pi prompt and response reserve cannot fit the estimated request budget");
|
|
@@ -525,8 +543,36 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
525
543
|
const prompted = phase === "decided";
|
|
526
544
|
phase = "starting";
|
|
527
545
|
try {
|
|
546
|
+
syncHistoryTools(ctx);
|
|
528
547
|
if (prompted) access(); // decided on input; a failure latched there still stops the run
|
|
529
|
-
else await preflight(ctx, incomingEventTokens(event), event.systemPrompt,
|
|
548
|
+
else await preflight(ctx, incomingEventTokens(event), event.systemPrompt,
|
|
549
|
+
historyToolNames(pi, event.systemPromptOptions.selectedTools, historyToolsEnabled(ctx)));
|
|
550
|
+
// Pi declares tool changes before turn_start, then snapshots executable tools in prepareRequest. Introduce our tools in
|
|
551
|
+
// the request projection: recording an activation message here would shift every later session-global recall index.
|
|
552
|
+
if (historyToolsEnabled(ctx)) stageHistoryTools(pi, ctx.sessionManager.buildSessionProjection().messages);
|
|
553
|
+
else applyHistoryTools(pi, false);
|
|
554
|
+
const staged = pi.getActiveTools();
|
|
555
|
+
event.systemPromptOptions.selectedTools = historyToolNames(pi, event.systemPromptOptions.selectedTools, true).filter(name =>
|
|
556
|
+
name !== "tuck_recall" && name !== "tuck_search" || staged.includes(name));
|
|
557
|
+
} catch (error) { fail(error, ctx); }
|
|
558
|
+
});
|
|
559
|
+
// The incoming user message is about to be appended (including steering inside a run), and later before_agent_start/turn_start
|
|
560
|
+
// handlers have run. Pi's prepareRequest has not yet snapshotted executable tools. A first cut discovered in context_with_system
|
|
561
|
+
// would be too late for that snapshot, even if its declarations were repaired there, so make the late prompt decision here.
|
|
562
|
+
pi.on("message_end", (event, ctx) => {
|
|
563
|
+
if (event.message.role !== "user" || !active() || !admittedRun || promptCut) return;
|
|
564
|
+
try {
|
|
565
|
+
access();
|
|
566
|
+
if (!ctx.model) throw missingModel();
|
|
567
|
+
const message = event.message;
|
|
568
|
+
const incoming = incomingTokens(typeof message.content === "string" ? message.content : message.content.flatMap(b => b.type === "text" ? [b.text] : []).join("\n"),
|
|
569
|
+
typeof message.content === "string" ? [] : message.content.filter((b): b is ImageContent => b.type === "image"));
|
|
570
|
+
const saved = ctx.sessionManager.buildSessionProjection().messages;
|
|
571
|
+
const host = hostTokens(ctx);
|
|
572
|
+
const system = requestSystemPart(ctx.getSystemPrompt(), pi.getActiveTools(), saved, ctx.getSystemPrompt());
|
|
573
|
+
if (fits(conversationTokens(saved) + system + incoming + host, limits(ctx).room, ctx)) return;
|
|
574
|
+
preflight(ctx, incoming, ctx.getSystemPrompt(), pi.getActiveTools(), host);
|
|
575
|
+
syncHistoryTools(ctx);
|
|
530
576
|
} catch (error) { fail(error, ctx); }
|
|
531
577
|
});
|
|
532
578
|
// A cut at a public boundary, drafted against the boundary's own preview: "tool-loop" between the rounds of a run, "run-end" once its
|
|
@@ -545,11 +591,27 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
545
591
|
// One snapshot per decision. Messages queued for the next request (steering, follow-ups) are in that request but not in the
|
|
546
592
|
// saved conversation. Pi's peek returns the steering queue, else the follow-up queue; a follow-up is not in the very next
|
|
547
593
|
// request while tools still run, so it can be counted early: conservative, and nothing else would cut at that later boundary.
|
|
548
|
-
const system =
|
|
594
|
+
const system = requestSystemPart(ctx.getSystemPrompt(), pi.getActiveTools(), saved, ctx.getSystemPrompt());
|
|
549
595
|
const band = decisionBand(ctx, room, system);
|
|
550
596
|
const host = hostTokens(ctx);
|
|
551
597
|
const queued = event.context.pendingMessages.length ? conversationTokens(event.context.pendingMessages as Messages) : 0;
|
|
552
598
|
const mandatory = system + host + queued;
|
|
599
|
+
// Without usable provider usage, Pi's own threshold uses its unwrapped message estimate as the compiler's density input.
|
|
600
|
+
// Our declarations are request-only, so Pi can miss that threshold. Preserve the same density input at settlement; every
|
|
601
|
+
// budget decision below still uses `system` and the complete conversation measure, including the summary envelope.
|
|
602
|
+
let densityTokens: number | undefined;
|
|
603
|
+
const lastAssistant = saved.findLast(message => message.role === "assistant");
|
|
604
|
+
const lastCut = ctx.sessionManager.getBranch().findLast(entry => entry.type === "compaction");
|
|
605
|
+
if (reason === "run-end" && lastAssistant && (!lastCut || lastAssistant.timestamp > Date.parse(lastCut.timestamp)) &&
|
|
606
|
+
!saved.some(message => message.role === "assistant" &&
|
|
607
|
+
!["error", "aborted"].includes(message.stopReason) && message.usage && calculateContextTokens(message.usage) > 0)) {
|
|
608
|
+
const forecast = historyToolDeclarations(saved, toolShapes(historyToolNames(pi, pi.getActiveTools(), true)), true);
|
|
609
|
+
if (forecast !== saved) {
|
|
610
|
+
const head = getCurrentSystemMessage(convertToLlm(forecast));
|
|
611
|
+
const native = (head ? estimateTokens(head) : 0) + saved.reduce((sum, message) => sum + (message.role === "system" ? 0 : estimateTokens(message)), 0);
|
|
612
|
+
if (shouldCompact(native, ctx.model.contextWindow, settingsFor().getCompactionSettings(ctx.model))) densityTokens = native;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
553
615
|
const previousSummary = saved.find(message => message.role === "compactionSummary")?.summary;
|
|
554
616
|
// The cut previews this decision has built, in the order it built them. The clip retry below measures them for what a cut leaves the
|
|
555
617
|
// request besides the fresh results. Reset per decision, so a retry sizes itself against the cut it is retrying, not an earlier attempt's.
|
|
@@ -559,19 +621,22 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
559
621
|
previews = [];
|
|
560
622
|
floorKept = undefined;
|
|
561
623
|
const before = measure(saved);
|
|
562
|
-
|
|
563
|
-
|
|
624
|
+
const gate = gateCut({ total: before + mandatory, mandatory, room, band, fits: (tokens, limit) => fits(tokens, limit, ctx) });
|
|
625
|
+
if (gate.kind !== "plan") return gate;
|
|
626
|
+
return chooseCut({
|
|
627
|
+
opportunistic: gate.opportunistic, mandatory, before, room, band, fits: (tokens, limit) => fits(tokens, limit, ctx), usefulOnly: options.cutAt === undefined,
|
|
564
628
|
unreduced: "Tool-loop preflight compaction did not reduce context within the estimated request budget",
|
|
565
629
|
plan: (target): Plan<{ compaction: NonNullable<ReturnType<typeof tuckObservedDraft>["compaction"]>; reserve: number }> => {
|
|
566
630
|
const reserve = planReserve(target, room, band, mandatory, ctx);
|
|
567
|
-
const draft = tuckObservedDraft(tuck, { branchEntries: ctx.sessionManager.getBranch(), tokensBefore: before, previousSummary }, ctx, reserve, plannerMeasureFor(ctx), options.recallPolicy, true);
|
|
631
|
+
const draft = tuckObservedDraft(tuck, { branchEntries: ctx.sessionManager.getBranch(), tokensBefore: densityTokens ?? before, previousSummary }, ctx, reserve, plannerMeasureFor(ctx), options.recallPolicy, true);
|
|
568
632
|
if (!draft.compaction) {
|
|
569
633
|
// The planner's own floor, so a clip's second budget is measured against what the smallest cut really keeps.
|
|
570
634
|
if (draft.floorMessages && event.toolResults) {
|
|
571
635
|
const fresh = new Set(event.toolResults.map(result => result.toolCallId));
|
|
572
636
|
const kept = measure(draft.floorMessages.filter(message =>
|
|
573
637
|
!(message.role === "toolResult" && fresh.has((message as ToolResultMessage).toolCallId))));
|
|
574
|
-
|
|
638
|
+
// Both targets must leave their protected openings beside the clip.
|
|
639
|
+
floorKept = Math.max(floorKept ?? kept, kept);
|
|
575
640
|
}
|
|
576
641
|
return { draft: false, capacityRejected: Boolean(draft.capacityRejected), reason: draft.unavailable };
|
|
577
642
|
}
|
|
@@ -596,7 +661,7 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
596
661
|
const branch = ctx.sessionManager.getBranch();
|
|
597
662
|
const entryIdOf = (toolCallId: string) => branch.findLast(entry => entry.type === "message" && entry.message.role === "toolResult" && entry.message.toolCallId === toolCallId)?.id;
|
|
598
663
|
const fresh = new Set(event.toolResults.map(result => result.toolCallId));
|
|
599
|
-
const kept = previews.length ? Math.
|
|
664
|
+
const kept = previews.length ? Math.max(...previews.map(preview => measure(preview.filter(message =>
|
|
600
665
|
!(message.role === "toolResult" && fresh.has((message as ToolResultMessage).toolCallId)))))) : floorKept;
|
|
601
666
|
for (const budget of clipBudgets(room, factor, mandatory, kept)) {
|
|
602
667
|
const round = clipFor(event.toolResults, entryIdOf, budget);
|
|
@@ -638,9 +703,19 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
638
703
|
settingsFor().getCompactionSettings(ctx.model).enabled;
|
|
639
704
|
if (!promptCut && !decide) return;
|
|
640
705
|
const result = atBoundary(event, ctx, "tool-loop", decide);
|
|
641
|
-
if (result)
|
|
706
|
+
if (result) {
|
|
707
|
+
pendingBoundaryCommit = true;
|
|
708
|
+
}
|
|
642
709
|
return result;
|
|
643
710
|
} catch (error) { fail(error, ctx); }
|
|
711
|
+
finally {
|
|
712
|
+
// Keep our part of Pi's next declaration snapshot unchanged; turn_start restores the branch-derived executable set before
|
|
713
|
+
// prepareRequest. This avoids an extra persisted system message at the first cut and on later rounds/runs.
|
|
714
|
+
if (active()) {
|
|
715
|
+
if (historyToolsEnabled(ctx)) stageHistoryTools(pi, ctx.sessionManager.buildSessionProjection().messages);
|
|
716
|
+
else applyHistoryTools(pi, false);
|
|
717
|
+
}
|
|
718
|
+
}
|
|
644
719
|
});
|
|
645
720
|
// A request of a run whose prompt's cut is not saved yet: the saved conversation in it, which other handlers may surround, is replaced by
|
|
646
721
|
// the conversation with the cut applied. Pi sends the saved system message apart from the request's messages, so neither side carries it
|
|
@@ -656,9 +731,11 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
656
731
|
pi.on("context", (event, ctx) => {
|
|
657
732
|
try {
|
|
658
733
|
if (!active()) return;
|
|
734
|
+
// Before a cut, Pi owns the whole request, including its object identities and system-message serialization.
|
|
735
|
+
if (!historyToolsEnabled(ctx)) return clips.size ? { messages: applyClips(event.messages, clips) } : undefined;
|
|
659
736
|
access();
|
|
660
737
|
return { messages: applyClips(projectedMessages(promptCut ? cutRequest(event.messages, ctx, promptCut) : event.messages, ctx, options.recallPolicy), clips) };
|
|
661
|
-
} catch (error) { try { fail(error, ctx); } finally { return { messages: [] }; } }
|
|
738
|
+
} catch (error) { try { fail(error, ctx); } finally { return historyToolsEnabled(ctx) ? { messages: [] } : undefined; } }
|
|
662
739
|
});
|
|
663
740
|
pi.on("session_compact_failed", (event, ctx) => {
|
|
664
741
|
// This compactor's own refusal of a Pi threshold ask (the gate, or an opportunistic cut that cannot reach the band) reaches here as a
|
|
@@ -699,20 +776,24 @@ export async function installCompaction(pi: ExtensionAPI, options: CompactionOpt
|
|
|
699
776
|
if (!ctx.model) throw missingModel();
|
|
700
777
|
// The complete request, counted after Pi's conversion (summary/custom/bash envelopes), with the effective system once.
|
|
701
778
|
// Pi applies a forced prompt after this hook; the public prompt already observes it.
|
|
702
|
-
|
|
779
|
+
syncHistoryTools(ctx);
|
|
780
|
+
const messages = historyToolsEnabled(ctx) ? historyToolDeclarations(event.messages, toolShapes(pi.getActiveTools()), true) : event.messages;
|
|
781
|
+
const system = requestSystemPart(ctx.getSystemPrompt(), pi.getActiveTools(), messages, ctx.getSystemPrompt());
|
|
782
|
+
const tokens = system + conversationTokens(messages);
|
|
703
783
|
if (!fits(tokens, limits(ctx).room, ctx))
|
|
704
784
|
throw new CapacityError("Complete Pi context and response reserve exceed the estimated request budget");
|
|
705
785
|
if (recorder.enabled && !recorder.waitingForCut) recorder.measureInline(() => {
|
|
706
786
|
recorder.request();
|
|
707
787
|
// No system prompt or hidden assistant blocks; no provider payload/headers.
|
|
708
|
-
const permitted =
|
|
788
|
+
const permitted = messages.filter(m => m.role !== "system" &&
|
|
709
789
|
!(m.role === "assistant" && m.content.some(b => b.type === "thinking")) &&
|
|
710
790
|
!(m.role === "custom" && options.recallPolicy?.privateCustomTypes?.includes(m.customType)));
|
|
711
791
|
recorder.capture("prepared-request", "public-context-with-system-hook-before-later-transforms-not-wire", { messages: permitted, estimatedInputTokens: tokens }, permitted.length !== event.messages.length);
|
|
712
792
|
});
|
|
713
793
|
if (tokens > 0 && Number.isSafeInteger(tokens)) pending = {
|
|
714
|
-
identity: identityFor(ctx), tokens, messages
|
|
794
|
+
identity: identityFor(ctx), tokens, messages, snapshot: structuredClone(convertToLlm(messages)),
|
|
715
795
|
};
|
|
796
|
+
if (messages !== event.messages) return { messages };
|
|
716
797
|
|
|
717
798
|
} catch (error) { fail(error, ctx); }
|
|
718
799
|
});
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { getCurrentTools } from "@earendil-works/pi-ai";
|
|
2
|
+
import { convertToLlm, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
3
|
+
import type { Messages, ToolShape } from "./budget.js";
|
|
4
|
+
|
|
5
|
+
const own = (name: string) => name === "tuck_recall" || name === "tuck_search";
|
|
6
|
+
/** Native Pi compactions do not make pi-tuck history available. No session-wide activation state. */
|
|
7
|
+
export const hasTuckCut = (branch: readonly { type: string; details?: unknown }[]): boolean => branch.some(entry =>
|
|
8
|
+
entry.type === "compaction" && (entry.details as { compactor?: unknown } | null)?.compactor === "pi-tuck");
|
|
9
|
+
|
|
10
|
+
/** Preserve the host's loadout and its order; only our registered tools may change. */
|
|
11
|
+
export function historyToolNames(pi: ExtensionAPI, names: readonly string[], enabled: boolean): string[] {
|
|
12
|
+
if (!enabled) return names.filter(name => !own(name));
|
|
13
|
+
const registered = new Set(pi.getAllTools().map(tool => tool.name));
|
|
14
|
+
return [...names, ...["tuck_recall", "tuck_search"].filter(name => registered.has(name) && !names.includes(name))];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function applyHistoryTools(pi: ExtensionAPI, enabled: boolean): void {
|
|
18
|
+
const current = pi.getActiveTools(), next = historyToolNames(pi, current, enabled);
|
|
19
|
+
if (current.length !== next.length || current.some((name, index) => name !== next[index])) pi.setActiveTools(next);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Match only our part of Pi's saved declaration snapshot, so its loop records no activation/removal message. Legacy sessions can
|
|
23
|
+
* already declare our tools; new sessions declare them only in the request projection. Executable tools are restored at turn_start. */
|
|
24
|
+
export function stageHistoryTools(pi: ExtensionAPI, messages: Messages): string[] {
|
|
25
|
+
const declared = new Set(getCurrentTools(convertToLlm(messages)).map(tool => tool.name));
|
|
26
|
+
const current = pi.getActiveTools();
|
|
27
|
+
const next = historyToolNames(pi, current, true).filter(name => !own(name) || declared.has(name));
|
|
28
|
+
if (current.length !== next.length || current.some((name, index) => name !== next[index])) pi.setActiveTools(next);
|
|
29
|
+
return next;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Pi has already declared the request's tools before queued user messages are appended. A prompt cut planned on such a message needs its declarations
|
|
33
|
+
* in that very request. Change only our declarations, leaving all other system content and tools in their original places. */
|
|
34
|
+
export function historyToolDeclarations(messages: Messages, tools: readonly ToolShape[], enabled: boolean): Messages {
|
|
35
|
+
let changed = false;
|
|
36
|
+
const filtered = messages.map(message => {
|
|
37
|
+
if (message.role !== "system") return message;
|
|
38
|
+
const added = message.toolsAdded?.filter(tool => !own(tool.name));
|
|
39
|
+
const removed = message.toolsRemoved?.filter(tool => !own(tool.name));
|
|
40
|
+
if (added?.length === message.toolsAdded?.length && removed?.length === message.toolsRemoved?.length) return message;
|
|
41
|
+
changed = true;
|
|
42
|
+
const { toolsAdded: _, toolsRemoved: __, ...rest } = message;
|
|
43
|
+
return { ...rest, ...(added?.length ? { toolsAdded: added } : {}), ...(removed?.length ? { toolsRemoved: removed } : {}) };
|
|
44
|
+
});
|
|
45
|
+
// Leave an already-correct transcript byte-identical, including the original placement of our declarations.
|
|
46
|
+
const declared = getCurrentTools(convertToLlm(messages));
|
|
47
|
+
const wanted = enabled ? tools.filter(tool => own(tool.name)) : [];
|
|
48
|
+
if (enabled && wanted.every(tool => declared.some(current => own(current.name) && current.name === tool.name)) &&
|
|
49
|
+
declared.filter(tool => own(tool.name)).length === wanted.length) return messages;
|
|
50
|
+
if (!enabled || !wanted.length) return changed ? filtered : messages;
|
|
51
|
+
const at = filtered.findLastIndex(message => message.role === "system");
|
|
52
|
+
if (at < 0) throw new Error("Missing Pi system message for history tool declarations");
|
|
53
|
+
const system = filtered[at]!;
|
|
54
|
+
if (system.role !== "system") throw new Error("Missing Pi system message");
|
|
55
|
+
filtered[at] = { ...system, toolsAdded: [...(system.toolsAdded ?? []), ...wanted as NonNullable<typeof system.toolsAdded>] };
|
|
56
|
+
return filtered;
|
|
57
|
+
}
|
package/src/ledger.ts
CHANGED
|
@@ -4,8 +4,6 @@ import { isDiscardedResponse } from "./protocol";
|
|
|
4
4
|
import { extractPath } from "./tuck/core/tool-args";
|
|
5
5
|
import type { TranscriptRecord } from "./tuck/core/transcript";
|
|
6
6
|
|
|
7
|
-
/** The initiating prompt is quoted whole up to this many characters; the rest stays readable through tuck_recall. */
|
|
8
|
-
export const MAX_TASK_CONTRACT_CHARACTERS = 12000;
|
|
9
7
|
/** Owner default; older distinct paths are counted, not listed. */
|
|
10
8
|
export const MAX_LEDGER_LINES = 60;
|
|
11
9
|
const LEDGER_TITLE = "Files";
|
package/src/omissions.ts
CHANGED
|
@@ -1,26 +1,72 @@
|
|
|
1
1
|
import type { ExtensionContext, SessionEntry } from '@earendil-works/pi-coding-agent';
|
|
2
2
|
import { buildGlobalIndexById } from './tuck/core/global-indices';
|
|
3
|
-
import { parseRecords, splitSummary, transcriptHeader } from './tuck/core/transcript';
|
|
3
|
+
import { parseRecords, serializeRecord, splitSummary, transcriptHeader } from './tuck/core/transcript';
|
|
4
|
+
import { textOf } from "./tuck/core/content";
|
|
5
|
+
import { isDiscardedResponse } from "./protocol";
|
|
4
6
|
import { createRecall, type RecallPolicy } from './recall';
|
|
5
7
|
/** Native branch edits supply exclusion; original global indices supply provenance. */
|
|
6
8
|
export function omittedSourceIndices(entries: readonly SessionEntry[], branch: readonly SessionEntry[]): ReadonlySet<number> {
|
|
7
|
-
const indices = buildGlobalIndexById(entries);
|
|
8
9
|
const edits = new Set<string>();
|
|
9
10
|
for (const entry of branch) if (entry.type === 'context_edit') {
|
|
10
11
|
if (entry.replacement !== null) throw new Error('pi-tuck does not support replacement context edits');
|
|
11
12
|
edits.add(entry.targetId);
|
|
12
13
|
}
|
|
14
|
+
if (!edits.size) return new Set();
|
|
15
|
+
const indices = buildGlobalIndexById(entries);
|
|
13
16
|
return new Set([...edits].flatMap(id => indices.has(id) ? [indices.get(id)!] : []));
|
|
14
17
|
}
|
|
15
18
|
/** Preserve surviving record bytes/order, and the summary's format (a v1 summary stays v1). No text matching or semantic inference. */
|
|
16
|
-
export function filterOmittedSummary(summary: string, omitted: ReadonlySet<number
|
|
19
|
+
export function filterOmittedSummary(summary: string, omitted: ReadonlySet<number>, sources?: readonly SessionEntry[], rangeOmitted = omitted): string {
|
|
17
20
|
if (!omitted.size) return summary;
|
|
18
21
|
const split = splitSummary(summary);
|
|
19
22
|
if (!split) throw new Error('Cannot apply native source omission to an unsupported compaction summary');
|
|
20
23
|
const { body, closing, version } = split;
|
|
21
24
|
const records = parseRecords(body, version), lines = body ? body.split('\n') : [];
|
|
22
|
-
|
|
23
|
-
|
|
25
|
+
// A broad range does not identify the role of an intervening source. Without
|
|
26
|
+
// original sources there is no evidence that an edit removes a counted unit.
|
|
27
|
+
const indices = sources && rangeOmitted.size && records.some(record => record.sourceRange) ? buildGlobalIndexById(sources) : undefined;
|
|
28
|
+
const sourceUnits = new Map<number, { role: string; missing: number }>();
|
|
29
|
+
const shownBySource = new Map<string, string[]>();
|
|
30
|
+
for (const record of records) if (record.kind === 'excerpt' && record.sourceIndex !== undefined && !record.span) {
|
|
31
|
+
const key = `${record.role}:${record.sourceIndex}`, shown = shownBySource.get(key) ?? [];
|
|
32
|
+
shown.push(record.text); shownBySource.set(key, shown);
|
|
33
|
+
}
|
|
34
|
+
if (indices && sources) for (const entry of sources) {
|
|
35
|
+
if (entry.type !== 'message') continue;
|
|
36
|
+
const index = indices.get(entry.id);
|
|
37
|
+
if (index === undefined) continue;
|
|
38
|
+
const message = entry.message;
|
|
39
|
+
const shown = shownBySource.get(`${message.role}:${index}`) ?? [];
|
|
40
|
+
if (message.role === 'user') {
|
|
41
|
+
const text = textOf(message.content);
|
|
42
|
+
sourceUnits.set(index, { role: 'user', missing: Number(Boolean(text.length) && !shown.includes(text)) });
|
|
43
|
+
} else if (message.role === 'assistant' && !isDiscardedResponse(message)) {
|
|
44
|
+
const available = [...shown];
|
|
45
|
+
let missing = 0;
|
|
46
|
+
for (const part of message.content) if (part.type === 'text' && part.text.trim()) {
|
|
47
|
+
const at = available.indexOf(part.text);
|
|
48
|
+
if (at < 0) missing++; else available.splice(at, 1);
|
|
49
|
+
}
|
|
50
|
+
sourceUnits.set(index, { role: 'assistant', missing });
|
|
51
|
+
} else if (message.role === 'toolResult') sourceUnits.set(index, { role: 'toolResult', missing: 1 });
|
|
52
|
+
}
|
|
53
|
+
const retained = records.flatMap((record, at) => {
|
|
54
|
+
if ((record.sourceIndex !== undefined && omitted.has(record.sourceIndex)) || record.sourceIndices?.some(index => omitted.has(index))) return [];
|
|
55
|
+
if (!record.sourceRange || !indices || record.count === undefined) return [lines[at]!];
|
|
56
|
+
const range = record.sourceRange;
|
|
57
|
+
let removed = 0;
|
|
58
|
+
const remaining: number[] = [];
|
|
59
|
+
for (const [index, unit] of sourceUnits) {
|
|
60
|
+
if (index < range.start || index > range.end || unit.role !== record.role || !unit.missing) continue;
|
|
61
|
+
if (rangeOmitted.has(index)) removed += unit.missing; else if (!omitted.has(index)) remaining.push(index);
|
|
62
|
+
}
|
|
63
|
+
if (!removed) return [lines[at]!];
|
|
64
|
+
const count = record.count - removed;
|
|
65
|
+
if (count <= 0) return [];
|
|
66
|
+
const sourceRange = remaining.length ? { start: Math.min(...remaining), end: Math.max(...remaining) } : range;
|
|
67
|
+
const adjusted = { ...record, count, sourceRange };
|
|
68
|
+
return [version === 1 ? JSON.stringify(adjusted) : serializeRecord(adjusted)];
|
|
69
|
+
});
|
|
24
70
|
return transcriptHeader(version) + '\n' + retained.join('\n') + closing;
|
|
25
71
|
}
|
|
26
72
|
|
|
@@ -29,13 +75,14 @@ export function filterCustomQuotes(summary: string, entries: readonly SessionEnt
|
|
|
29
75
|
const split = splitSummary(summary);
|
|
30
76
|
if (!split) throw new Error('Cannot resolve custom sources in an unsupported compaction summary');
|
|
31
77
|
const { body, closing, version } = split;
|
|
78
|
+
const records = parseRecords(body, version);
|
|
79
|
+
if (!records.some(record => record.role === "custom")) return summary;
|
|
32
80
|
const omitted = new Set(branch.flatMap(entry => entry.type === 'context_edit' && entry.replacement === null ? [entry.targetId] : []));
|
|
33
81
|
const eligible = createRecall(policy).isHistorySource, indices = buildGlobalIndexById(entries);
|
|
34
82
|
const globalCounts = new Map<string, number>(), branchSources = new Map<string, SessionEntry[]>();
|
|
35
83
|
for (const entry of entries) globalCounts.set(entry.id, (globalCounts.get(entry.id) ?? 0) + 1);
|
|
36
84
|
for (const entry of branch) { const found = branchSources.get(entry.id) ?? []; found.push(entry); branchSources.set(entry.id, found); }
|
|
37
85
|
const lines = body ? body.split('\n') : [];
|
|
38
|
-
const records = parseRecords(body, version);
|
|
39
86
|
const retained = records.flatMap((record, i) => {
|
|
40
87
|
if (record.role !== 'custom') return [lines[i]!];
|
|
41
88
|
const id = record.sourceEntryId, source = id === undefined ? undefined : branchSources.get(id);
|
|
@@ -52,8 +99,18 @@ type ProjectedMessages = ReturnType<ExtensionContext['sessionManager']['buildSes
|
|
|
52
99
|
function projectMessages(messages: ProjectedMessages, ctx: ExtensionContext, policy: RecallPolicy | undefined, allowForeign: boolean): ProjectedMessages {
|
|
53
100
|
const entries = ctx.sessionManager.getEntries(), branch = ctx.sessionManager.getBranch();
|
|
54
101
|
const omitted = omittedSourceIndices(entries, branch);
|
|
102
|
+
const lastCut = branch.findLastIndex(entry => entry.type === 'compaction');
|
|
103
|
+
const cut = branch[lastCut];
|
|
104
|
+
const afterCut = new Set<number>();
|
|
105
|
+
if (omitted.size && cut?.type === 'compaction') {
|
|
106
|
+
const indices = buildGlobalIndexById(entries);
|
|
107
|
+
for (const entry of branch.slice(lastCut + 1)) if (entry.type === 'context_edit' && entry.replacement === null) {
|
|
108
|
+
const index = indices.get(entry.targetId); if (index !== undefined) afterCut.add(index);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
55
111
|
return messages.map(message => message.role === 'compactionSummary' && (!allowForeign || splitSummary(message.summary) !== undefined)
|
|
56
|
-
? { ...message, summary: filterCustomQuotes(filterOmittedSummary(message.summary, omitted
|
|
112
|
+
? { ...message, summary: filterCustomQuotes(filterOmittedSummary(message.summary, omitted, entries,
|
|
113
|
+
cut?.type === 'compaction' && cut.summary === message.summary ? afterCut : new Set()), entries, branch, policy) } : message);
|
|
57
114
|
}
|
|
58
115
|
export function projectedMessages(messages: ProjectedMessages, ctx: ExtensionContext, policy?: RecallPolicy): ProjectedMessages {
|
|
59
116
|
return projectMessages(messages, ctx, policy, false);
|
package/src/tuck/core/brief.ts
CHANGED
|
@@ -123,19 +123,17 @@ export const isDialogue = (block: NormalizedBlock): block is DialogueBlock => bl
|
|
|
123
123
|
|
|
124
124
|
/** A user or assistant text block the omission notices count when it is unselected: non-empty, and not rendered in the mandatory base. */
|
|
125
125
|
export const isOmittable = (block: NormalizedBlock, mandatorySources?: ReadonlySet<number>): boolean =>
|
|
126
|
-
(block.kind === "
|
|
126
|
+
(block.kind === "assistant" || block.kind === "user" && !block.image) && Boolean(block.text.trim()) &&
|
|
127
127
|
(block.sourceIndex === undefined || !mandatorySources?.has(block.sourceIndex));
|
|
128
128
|
|
|
129
|
-
/**
|
|
130
|
-
*
|
|
131
|
-
export const omissionRecord = (role: "user" | "assistant", count: number, sources: number[], unlocated: boolean): TranscriptRecord => ({
|
|
129
|
+
/** Count dropped units inside inclusive original-index bounds. The range may
|
|
130
|
+
* contain retained units and other roles; only omitted units of this role count. */
|
|
131
|
+
export const omissionRecord = (role: "user" | "assistant" | "toolResult", count: number, sources: number[], unlocated: boolean): TranscriptRecord => ({
|
|
132
132
|
kind: "omission", role, count,
|
|
133
|
-
...(sources.length ? {
|
|
134
|
-
reason: "
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
(unlocated ? "Some text blocks have no source locator. " : "") +
|
|
138
|
-
"Read originals with tuck_recall; omitted text may qualify retained text. No new authority." });
|
|
133
|
+
...(sources.length ? { sourceRange: { start: sources.reduce((a, b) => Math.min(a, b)), end: sources.reduce((a, b) => Math.max(a, b)) } } : {}),
|
|
134
|
+
reason: (role === "toolResult" ? "Historical tool actions and observations omitted. " : "Historical text units omitted. ") +
|
|
135
|
+
(unlocated ? "Some units have no source locator. " : "") +
|
|
136
|
+
"Count is omitted units of this role in the index range; read originals with tuck_recall." });
|
|
139
137
|
|
|
140
138
|
const analyses = new WeakMap<DialogueBlock, Analysis>();
|
|
141
139
|
|
|
@@ -159,10 +157,27 @@ export const blockRecords = (b: DialogueBlock, mandatory: ReadonlySet<number> |
|
|
|
159
157
|
return excerptRecords(b.text, analysis, role, b.sourceIndex, clip[0], clip[1], readFrom).map(record => ({ ...record, ...notice }));
|
|
160
158
|
};
|
|
161
159
|
|
|
160
|
+
/** Exact user text prefixes are allowed to split a passage. Unlike assistant
|
|
161
|
+
* passage clips, their explicit continuation points at the unabridged input. */
|
|
162
|
+
export const userPrefixRecords = (block: DialogueBlock, mandatory: ReadonlySet<number> | undefined, take: number): TranscriptRecord[] => {
|
|
163
|
+
if (block.kind !== "user" || block.sourceIndex !== undefined && mandatory?.has(block.sourceIndex)) return [];
|
|
164
|
+
const text = block.originalText ?? block.text;
|
|
165
|
+
if (take <= 0 || !text.length) return [];
|
|
166
|
+
let end = Math.min(text.length, Math.floor(take));
|
|
167
|
+
// Charge the complete code point in the measured candidate, including the
|
|
168
|
+
// low surrogate, so a protected 80-character opening never shrinks to 79.
|
|
169
|
+
if (end < text.length && /[\uD800-\uDBFF]/u.test(text[end - 1]!)) end++;
|
|
170
|
+
const locator = block.sourceIndex === undefined ? undefined : { index: block.sourceIndex };
|
|
171
|
+
return [{ kind: "excerpt", role: "user", sourceIndex: block.sourceIndex, text: text.slice(0, end) },
|
|
172
|
+
...(end < text.length ? [{ kind: "omission" as const, role: "user" as const, sourceIndex: block.sourceIndex,
|
|
173
|
+
reason: locator ? `tuck_recall ${JSON.stringify(block.startsSource ? { ...locator, offset: end } : locator)}` : "Source has no locator" }] : [])];
|
|
174
|
+
};
|
|
175
|
+
|
|
162
176
|
/** A source time for each locator a user record names; only briefs of times-carrying blocks have them. */
|
|
163
177
|
export const decorateRecord = (record: TranscriptRecord, times: NormalizedBlock["sourceTimes"]): TranscriptRecord => {
|
|
164
178
|
// Times mark the user's turns only: on every record they cost a fifth to a third of the summary's tokens.
|
|
165
179
|
if (!times || record.role !== "user") return record;
|
|
180
|
+
if (record.sourceRange) return record;
|
|
166
181
|
const indices = record.sourceIndices ?? (record.sourceIndex === undefined ? [] : [record.sourceIndex]);
|
|
167
182
|
return indices.length ? { ...record, sourceTimes: indices.map(sourceIndex => ({ sourceIndex, ...(times[sourceIndex] ?? { basis: "unknown" as const }) })) } : record;
|
|
168
183
|
};
|