pi-tuck 1.0.16 → 1.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +75 -0
- package/README.md +7 -4
- package/dist/src/budget.d.ts +14 -17
- package/dist/src/budget.js +22 -29
- package/dist/src/build-id.d.ts +1 -1
- package/dist/src/build-id.js +1 -1
- package/dist/src/compaction.js +113 -20
- package/dist/src/diagnostics.d.ts +1 -1
- package/dist/src/history-tools.d.ts +16 -0
- package/dist/src/history-tools.js +59 -0
- package/dist/src/ledger.d.ts +0 -2
- package/dist/src/ledger.js +0 -2
- package/dist/src/omissions.d.ts +1 -1
- package/dist/src/omissions.js +88 -6
- package/dist/src/tuck/core/brief.d.ts +6 -3
- package/dist/src/tuck/core/brief.js +27 -9
- package/dist/src/tuck/core/filter-noise.d.ts +1 -1
- package/dist/src/tuck/core/filter-noise.js +1 -3
- package/dist/src/tuck/core/normalize.d.ts +2 -5
- package/dist/src/tuck/core/normalize.js +15 -33
- package/dist/src/tuck/core/shrink.d.ts +6 -14
- package/dist/src/tuck/core/shrink.js +162 -102
- package/dist/src/tuck/core/summarize.d.ts +3 -14
- package/dist/src/tuck/core/summarize.js +9 -20
- package/dist/src/tuck/core/transcript.d.ts +5 -0
- package/dist/src/tuck/core/transcript.js +27 -4
- package/dist/src/tuck/types.d.ts +1 -1
- package/dist/src/tuck.d.ts +9 -17
- package/dist/src/tuck.js +384 -212
- package/package.json +1 -1
- package/src/budget.ts +24 -26
- package/src/build-id.ts +1 -1
- package/src/compaction.ts +102 -21
- package/src/history-tools.ts +57 -0
- package/src/ledger.ts +0 -2
- package/src/omissions.ts +64 -7
- package/src/tuck/core/brief.ts +25 -10
- package/src/tuck/core/filter-noise.ts +1 -3
- package/src/tuck/core/normalize.ts +11 -27
- package/src/tuck/core/shrink.ts +141 -91
- package/src/tuck/core/summarize.ts +9 -24
- package/src/tuck/core/transcript.ts +26 -4
- package/src/tuck/types.ts +3 -3
- package/src/tuck.ts +336 -185
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { getCurrentTools } from "@earendil-works/pi-ai";
|
|
2
|
+
import { convertToLlm } from "@earendil-works/pi-coding-agent";
|
|
3
|
+
const own = (name) => name === "tuck_recall" || name === "tuck_search";
|
|
4
|
+
/** Native Pi compactions do not make pi-tuck history available. No session-wide activation state. */
|
|
5
|
+
export const hasTuckCut = (branch) => branch.some(entry => entry.type === "compaction" && entry.details?.compactor === "pi-tuck");
|
|
6
|
+
/** Preserve the host's loadout and its order; only our registered tools may change. */
|
|
7
|
+
export function historyToolNames(pi, names, enabled) {
|
|
8
|
+
if (!enabled)
|
|
9
|
+
return names.filter(name => !own(name));
|
|
10
|
+
const registered = new Set(pi.getAllTools().map(tool => tool.name));
|
|
11
|
+
return [...names, ...["tuck_recall", "tuck_search"].filter(name => registered.has(name) && !names.includes(name))];
|
|
12
|
+
}
|
|
13
|
+
export function applyHistoryTools(pi, enabled) {
|
|
14
|
+
const current = pi.getActiveTools(), next = historyToolNames(pi, current, enabled);
|
|
15
|
+
if (current.length !== next.length || current.some((name, index) => name !== next[index]))
|
|
16
|
+
pi.setActiveTools(next);
|
|
17
|
+
}
|
|
18
|
+
/** Match only our part of Pi's saved declaration snapshot, so its loop records no activation/removal message. Legacy sessions can
|
|
19
|
+
* already declare our tools; new sessions declare them only in the request projection. Executable tools are restored at turn_start. */
|
|
20
|
+
export function stageHistoryTools(pi, messages) {
|
|
21
|
+
const declared = new Set(getCurrentTools(convertToLlm(messages)).map(tool => tool.name));
|
|
22
|
+
const current = pi.getActiveTools();
|
|
23
|
+
const next = historyToolNames(pi, current, true).filter(name => !own(name) || declared.has(name));
|
|
24
|
+
if (current.length !== next.length || current.some((name, index) => name !== next[index]))
|
|
25
|
+
pi.setActiveTools(next);
|
|
26
|
+
return next;
|
|
27
|
+
}
|
|
28
|
+
/** Pi has already declared the request's tools before queued user messages are appended. A prompt cut planned on such a message needs its declarations
|
|
29
|
+
* in that very request. Change only our declarations, leaving all other system content and tools in their original places. */
|
|
30
|
+
export function historyToolDeclarations(messages, tools, enabled) {
|
|
31
|
+
let changed = false;
|
|
32
|
+
const filtered = messages.map(message => {
|
|
33
|
+
if (message.role !== "system")
|
|
34
|
+
return message;
|
|
35
|
+
const added = message.toolsAdded?.filter(tool => !own(tool.name));
|
|
36
|
+
const removed = message.toolsRemoved?.filter(tool => !own(tool.name));
|
|
37
|
+
if (added?.length === message.toolsAdded?.length && removed?.length === message.toolsRemoved?.length)
|
|
38
|
+
return message;
|
|
39
|
+
changed = true;
|
|
40
|
+
const { toolsAdded: _, toolsRemoved: __, ...rest } = message;
|
|
41
|
+
return { ...rest, ...(added?.length ? { toolsAdded: added } : {}), ...(removed?.length ? { toolsRemoved: removed } : {}) };
|
|
42
|
+
});
|
|
43
|
+
// Leave an already-correct transcript byte-identical, including the original placement of our declarations.
|
|
44
|
+
const declared = getCurrentTools(convertToLlm(messages));
|
|
45
|
+
const wanted = enabled ? tools.filter(tool => own(tool.name)) : [];
|
|
46
|
+
if (enabled && wanted.every(tool => declared.some(current => own(current.name) && current.name === tool.name)) &&
|
|
47
|
+
declared.filter(tool => own(tool.name)).length === wanted.length)
|
|
48
|
+
return messages;
|
|
49
|
+
if (!enabled || !wanted.length)
|
|
50
|
+
return changed ? filtered : messages;
|
|
51
|
+
const at = filtered.findLastIndex(message => message.role === "system");
|
|
52
|
+
if (at < 0)
|
|
53
|
+
throw new Error("Missing Pi system message for history tool declarations");
|
|
54
|
+
const system = filtered[at];
|
|
55
|
+
if (system.role !== "system")
|
|
56
|
+
throw new Error("Missing Pi system message");
|
|
57
|
+
filtered[at] = { ...system, toolsAdded: [...(system.toolsAdded ?? []), ...wanted] };
|
|
58
|
+
return filtered;
|
|
59
|
+
}
|
package/dist/src/ledger.d.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import type { SessionEntry } from "@earendil-works/pi-coding-agent";
|
|
2
2
|
import type { TranscriptRecord } from "./tuck/core/transcript.js";
|
|
3
|
-
/** The initiating prompt is quoted whole up to this many characters; the rest stays readable through tuck_recall. */
|
|
4
|
-
export declare const MAX_TASK_CONTRACT_CHARACTERS = 12000;
|
|
5
3
|
/** Owner default; older distinct paths are counted, not listed. */
|
|
6
4
|
export declare const MAX_LEDGER_LINES = 60;
|
|
7
5
|
export declare const LEDGER_HEADING = "Files touched by read, write and edit before this summary, newest first, with the last operation:";
|
package/dist/src/ledger.js
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import { isLookupTool } from "./tuck/core/recall-operation.js";
|
|
2
2
|
import { isDiscardedResponse } from "./protocol.js";
|
|
3
3
|
import { extractPath } from "./tuck/core/tool-args.js";
|
|
4
|
-
/** The initiating prompt is quoted whole up to this many characters; the rest stays readable through tuck_recall. */
|
|
5
|
-
export const MAX_TASK_CONTRACT_CHARACTERS = 12000;
|
|
6
4
|
/** Owner default; older distinct paths are counted, not listed. */
|
|
7
5
|
export const MAX_LEDGER_LINES = 60;
|
|
8
6
|
const LEDGER_TITLE = "Files";
|
package/dist/src/omissions.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { type RecallPolicy } from './recall.js';
|
|
|
3
3
|
/** Native branch edits supply exclusion; original global indices supply provenance. */
|
|
4
4
|
export declare function omittedSourceIndices(entries: readonly SessionEntry[], branch: readonly SessionEntry[]): ReadonlySet<number>;
|
|
5
5
|
/** Preserve surviving record bytes/order, and the summary's format (a v1 summary stays v1). No text matching or semantic inference. */
|
|
6
|
-
export declare function filterOmittedSummary(summary: string, omitted: ReadonlySet<number>): string;
|
|
6
|
+
export declare function filterOmittedSummary(summary: string, omitted: ReadonlySet<number>, sources?: readonly SessionEntry[], rangeOmitted?: ReadonlySet<number>): string;
|
|
7
7
|
/** Custom quotes carry native entry identity; no text matching or legacy fallback. */
|
|
8
8
|
export declare function filterCustomQuotes(summary: string, entries: readonly SessionEntry[], branch: readonly SessionEntry[], policy?: RecallPolicy): string;
|
|
9
9
|
type ProjectedMessages = ReturnType<ExtensionContext['sessionManager']['buildSessionProjection']>['messages'];
|
package/dist/src/omissions.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { buildGlobalIndexById } from './tuck/core/global-indices.js';
|
|
2
|
-
import { parseRecords, splitSummary, transcriptHeader } from './tuck/core/transcript.js';
|
|
2
|
+
import { parseRecords, serializeRecord, splitSummary, transcriptHeader } from './tuck/core/transcript.js';
|
|
3
|
+
import { textOf } from "./tuck/core/content.js";
|
|
4
|
+
import { isDiscardedResponse } from "./protocol.js";
|
|
3
5
|
import { createRecall } from './recall.js';
|
|
4
6
|
/** Native branch edits supply exclusion; original global indices supply provenance. */
|
|
5
7
|
export function omittedSourceIndices(entries, branch) {
|
|
6
|
-
const indices = buildGlobalIndexById(entries);
|
|
7
8
|
const edits = new Set();
|
|
8
9
|
for (const entry of branch)
|
|
9
10
|
if (entry.type === 'context_edit') {
|
|
@@ -11,10 +12,13 @@ export function omittedSourceIndices(entries, branch) {
|
|
|
11
12
|
throw new Error('pi-tuck does not support replacement context edits');
|
|
12
13
|
edits.add(entry.targetId);
|
|
13
14
|
}
|
|
15
|
+
if (!edits.size)
|
|
16
|
+
return new Set();
|
|
17
|
+
const indices = buildGlobalIndexById(entries);
|
|
14
18
|
return new Set([...edits].flatMap(id => indices.has(id) ? [indices.get(id)] : []));
|
|
15
19
|
}
|
|
16
20
|
/** Preserve surviving record bytes/order, and the summary's format (a v1 summary stays v1). No text matching or semantic inference. */
|
|
17
|
-
export function filterOmittedSummary(summary, omitted) {
|
|
21
|
+
export function filterOmittedSummary(summary, omitted, sources, rangeOmitted = omitted) {
|
|
18
22
|
if (!omitted.size)
|
|
19
23
|
return summary;
|
|
20
24
|
const split = splitSummary(summary);
|
|
@@ -22,7 +26,71 @@ export function filterOmittedSummary(summary, omitted) {
|
|
|
22
26
|
throw new Error('Cannot apply native source omission to an unsupported compaction summary');
|
|
23
27
|
const { body, closing, version } = split;
|
|
24
28
|
const records = parseRecords(body, version), lines = body ? body.split('\n') : [];
|
|
25
|
-
|
|
29
|
+
// A broad range does not identify the role of an intervening source. Without
|
|
30
|
+
// original sources there is no evidence that an edit removes a counted unit.
|
|
31
|
+
const indices = sources && rangeOmitted.size && records.some(record => record.sourceRange) ? buildGlobalIndexById(sources) : undefined;
|
|
32
|
+
const sourceUnits = new Map();
|
|
33
|
+
const shownBySource = new Map();
|
|
34
|
+
for (const record of records)
|
|
35
|
+
if (record.kind === 'excerpt' && record.sourceIndex !== undefined && !record.span) {
|
|
36
|
+
const key = `${record.role}:${record.sourceIndex}`, shown = shownBySource.get(key) ?? [];
|
|
37
|
+
shown.push(record.text);
|
|
38
|
+
shownBySource.set(key, shown);
|
|
39
|
+
}
|
|
40
|
+
if (indices && sources)
|
|
41
|
+
for (const entry of sources) {
|
|
42
|
+
if (entry.type !== 'message')
|
|
43
|
+
continue;
|
|
44
|
+
const index = indices.get(entry.id);
|
|
45
|
+
if (index === undefined)
|
|
46
|
+
continue;
|
|
47
|
+
const message = entry.message;
|
|
48
|
+
const shown = shownBySource.get(`${message.role}:${index}`) ?? [];
|
|
49
|
+
if (message.role === 'user') {
|
|
50
|
+
const text = textOf(message.content);
|
|
51
|
+
sourceUnits.set(index, { role: 'user', missing: Number(Boolean(text.length) && !shown.includes(text)) });
|
|
52
|
+
}
|
|
53
|
+
else if (message.role === 'assistant' && !isDiscardedResponse(message)) {
|
|
54
|
+
const available = [...shown];
|
|
55
|
+
let missing = 0;
|
|
56
|
+
for (const part of message.content)
|
|
57
|
+
if (part.type === 'text' && part.text.trim()) {
|
|
58
|
+
const at = available.indexOf(part.text);
|
|
59
|
+
if (at < 0)
|
|
60
|
+
missing++;
|
|
61
|
+
else
|
|
62
|
+
available.splice(at, 1);
|
|
63
|
+
}
|
|
64
|
+
sourceUnits.set(index, { role: 'assistant', missing });
|
|
65
|
+
}
|
|
66
|
+
else if (message.role === 'toolResult')
|
|
67
|
+
sourceUnits.set(index, { role: 'toolResult', missing: 1 });
|
|
68
|
+
}
|
|
69
|
+
const retained = records.flatMap((record, at) => {
|
|
70
|
+
if ((record.sourceIndex !== undefined && omitted.has(record.sourceIndex)) || record.sourceIndices?.some(index => omitted.has(index)))
|
|
71
|
+
return [];
|
|
72
|
+
if (!record.sourceRange || !indices || record.count === undefined)
|
|
73
|
+
return [lines[at]];
|
|
74
|
+
const range = record.sourceRange;
|
|
75
|
+
let removed = 0;
|
|
76
|
+
const remaining = [];
|
|
77
|
+
for (const [index, unit] of sourceUnits) {
|
|
78
|
+
if (index < range.start || index > range.end || unit.role !== record.role || !unit.missing)
|
|
79
|
+
continue;
|
|
80
|
+
if (rangeOmitted.has(index))
|
|
81
|
+
removed += unit.missing;
|
|
82
|
+
else if (!omitted.has(index))
|
|
83
|
+
remaining.push(index);
|
|
84
|
+
}
|
|
85
|
+
if (!removed)
|
|
86
|
+
return [lines[at]];
|
|
87
|
+
const count = record.count - removed;
|
|
88
|
+
if (count <= 0)
|
|
89
|
+
return [];
|
|
90
|
+
const sourceRange = remaining.length ? { start: Math.min(...remaining), end: Math.max(...remaining) } : range;
|
|
91
|
+
const adjusted = { ...record, count, sourceRange };
|
|
92
|
+
return [version === 1 ? JSON.stringify(adjusted) : serializeRecord(adjusted)];
|
|
93
|
+
});
|
|
26
94
|
return transcriptHeader(version) + '\n' + retained.join('\n') + closing;
|
|
27
95
|
}
|
|
28
96
|
/** Custom quotes carry native entry identity; no text matching or legacy fallback. */
|
|
@@ -31,6 +99,9 @@ export function filterCustomQuotes(summary, entries, branch, policy = {}) {
|
|
|
31
99
|
if (!split)
|
|
32
100
|
throw new Error('Cannot resolve custom sources in an unsupported compaction summary');
|
|
33
101
|
const { body, closing, version } = split;
|
|
102
|
+
const records = parseRecords(body, version);
|
|
103
|
+
if (!records.some(record => record.role === "custom"))
|
|
104
|
+
return summary;
|
|
34
105
|
const omitted = new Set(branch.flatMap(entry => entry.type === 'context_edit' && entry.replacement === null ? [entry.targetId] : []));
|
|
35
106
|
const eligible = createRecall(policy).isHistorySource, indices = buildGlobalIndexById(entries);
|
|
36
107
|
const globalCounts = new Map(), branchSources = new Map();
|
|
@@ -42,7 +113,6 @@ export function filterCustomQuotes(summary, entries, branch, policy = {}) {
|
|
|
42
113
|
branchSources.set(entry.id, found);
|
|
43
114
|
}
|
|
44
115
|
const lines = body ? body.split('\n') : [];
|
|
45
|
-
const records = parseRecords(body, version);
|
|
46
116
|
const retained = records.flatMap((record, i) => {
|
|
47
117
|
if (record.role !== 'custom')
|
|
48
118
|
return [lines[i]];
|
|
@@ -62,8 +132,20 @@ export function filterCustomQuotes(summary, entries, branch, policy = {}) {
|
|
|
62
132
|
function projectMessages(messages, ctx, policy, allowForeign) {
|
|
63
133
|
const entries = ctx.sessionManager.getEntries(), branch = ctx.sessionManager.getBranch();
|
|
64
134
|
const omitted = omittedSourceIndices(entries, branch);
|
|
135
|
+
const lastCut = branch.findLastIndex(entry => entry.type === 'compaction');
|
|
136
|
+
const cut = branch[lastCut];
|
|
137
|
+
const afterCut = new Set();
|
|
138
|
+
if (omitted.size && cut?.type === 'compaction') {
|
|
139
|
+
const indices = buildGlobalIndexById(entries);
|
|
140
|
+
for (const entry of branch.slice(lastCut + 1))
|
|
141
|
+
if (entry.type === 'context_edit' && entry.replacement === null) {
|
|
142
|
+
const index = indices.get(entry.targetId);
|
|
143
|
+
if (index !== undefined)
|
|
144
|
+
afterCut.add(index);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
65
147
|
return messages.map(message => message.role === 'compactionSummary' && (!allowForeign || splitSummary(message.summary) !== undefined)
|
|
66
|
-
? { ...message, summary: filterCustomQuotes(filterOmittedSummary(message.summary, omitted), entries, branch, policy) } : message);
|
|
148
|
+
? { ...message, summary: filterCustomQuotes(filterOmittedSummary(message.summary, omitted, entries, cut?.type === 'compaction' && cut.summary === message.summary ? afterCut : new Set()), entries, branch, policy) } : message);
|
|
67
149
|
}
|
|
68
150
|
export function projectedMessages(messages, ctx, policy) {
|
|
69
151
|
return projectMessages(messages, ctx, policy, false);
|
|
@@ -7,13 +7,16 @@ export type DialogueBlock = Extract<NormalizedBlock, {
|
|
|
7
7
|
export declare const isDialogue: (block: NormalizedBlock) => block is DialogueBlock;
|
|
8
8
|
/** A user or assistant text block the omission notices count when it is unselected: non-empty, and not rendered in the mandatory base. */
|
|
9
9
|
export declare const isOmittable: (block: NormalizedBlock, mandatorySources?: ReadonlySet<number>) => boolean;
|
|
10
|
-
/**
|
|
11
|
-
*
|
|
12
|
-
export declare const omissionRecord: (role: "user" | "assistant", count: number, sources: number[], unlocated: boolean) => TranscriptRecord;
|
|
10
|
+
/** Count dropped units inside inclusive original-index bounds. The range may
|
|
11
|
+
* contain retained units and other roles; only omitted units of this role count. */
|
|
12
|
+
export declare const omissionRecord: (role: "user" | "assistant" | "toolResult", count: number, sources: number[], unlocated: boolean) => TranscriptRecord;
|
|
13
13
|
/** The word budgets of a clipped excerpt: the passages kept from the start of the text, and from its end. */
|
|
14
14
|
export type Clip = readonly [headWords: number, tailWords: number];
|
|
15
15
|
/** The records of one dialogue block alone, before source times are attached: its whole original text, or, with a `clip`, the excerpts
|
|
16
16
|
* that keep their sanitized-block spans around an omission. A mandatory source renders nothing: the adapter quotes it in its base. */
|
|
17
17
|
export declare const blockRecords: (b: DialogueBlock, mandatory: ReadonlySet<number> | undefined, clip?: Clip) => TranscriptRecord[];
|
|
18
|
+
/** Exact user text prefixes are allowed to split a passage. Unlike assistant
|
|
19
|
+
* passage clips, their explicit continuation points at the unabridged input. */
|
|
20
|
+
export declare const userPrefixRecords: (block: DialogueBlock, mandatory: ReadonlySet<number> | undefined, take: number) => TranscriptRecord[];
|
|
18
21
|
/** A source time for each locator a user record names; only briefs of times-carrying blocks have them. */
|
|
19
22
|
export declare const decorateRecord: (record: TranscriptRecord, times: NormalizedBlock["sourceTimes"]) => TranscriptRecord;
|
|
@@ -134,18 +134,16 @@ const excerptRecords = (text, { passages, costs }, role, sourceIndex, headLimit,
|
|
|
134
134
|
};
|
|
135
135
|
export const isDialogue = (block) => block.kind === "user" || block.kind === "assistant" || block.kind === "notice";
|
|
136
136
|
/** A user or assistant text block the omission notices count when it is unselected: non-empty, and not rendered in the mandatory base. */
|
|
137
|
-
export const isOmittable = (block, mandatorySources) => (block.kind === "
|
|
137
|
+
export const isOmittable = (block, mandatorySources) => (block.kind === "assistant" || block.kind === "user" && !block.image) && Boolean(block.text.trim()) &&
|
|
138
138
|
(block.sourceIndex === undefined || !mandatorySources?.has(block.sourceIndex));
|
|
139
|
-
/**
|
|
140
|
-
*
|
|
139
|
+
/** Count dropped units inside inclusive original-index bounds. The range may
|
|
140
|
+
* contain retained units and other roles; only omitted units of this role count. */
|
|
141
141
|
export const omissionRecord = (role, count, sources, unlocated) => ({
|
|
142
142
|
kind: "omission", role, count,
|
|
143
|
-
...(sources.length ? {
|
|
144
|
-
reason: "
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
(unlocated ? "Some text blocks have no source locator. " : "") +
|
|
148
|
-
"Read originals with tuck_recall; omitted text may qualify retained text. No new authority."
|
|
143
|
+
...(sources.length ? { sourceRange: { start: sources.reduce((a, b) => Math.min(a, b)), end: sources.reduce((a, b) => Math.max(a, b)) } } : {}),
|
|
144
|
+
reason: (role === "toolResult" ? "Historical tool actions and observations omitted. " : "Historical text units omitted. ") +
|
|
145
|
+
(unlocated ? "Some units have no source locator. " : "") +
|
|
146
|
+
"Count is omitted units of this role in the index range; read originals with tuck_recall."
|
|
149
147
|
});
|
|
150
148
|
const analyses = new WeakMap();
|
|
151
149
|
/** The records of one dialogue block alone, before source times are attached: its whole original text, or, with a `clip`, the excerpts
|
|
@@ -167,11 +165,31 @@ export const blockRecords = (b, mandatory, clip) => {
|
|
|
167
165
|
const readFrom = (offset) => locator ? `tuck_recall ${JSON.stringify(exact ? { ...locator, offset } : locator)}` : "source has no locator to read it by";
|
|
168
166
|
return excerptRecords(b.text, analysis, role, b.sourceIndex, clip[0], clip[1], readFrom).map(record => ({ ...record, ...notice }));
|
|
169
167
|
};
|
|
168
|
+
/** Exact user text prefixes are allowed to split a passage. Unlike assistant
|
|
169
|
+
* passage clips, their explicit continuation points at the unabridged input. */
|
|
170
|
+
export const userPrefixRecords = (block, mandatory, take) => {
|
|
171
|
+
if (block.kind !== "user" || block.sourceIndex !== undefined && mandatory?.has(block.sourceIndex))
|
|
172
|
+
return [];
|
|
173
|
+
const text = block.originalText ?? block.text;
|
|
174
|
+
if (take <= 0 || !text.length)
|
|
175
|
+
return [];
|
|
176
|
+
let end = Math.min(text.length, Math.floor(take));
|
|
177
|
+
// Charge the complete code point in the measured candidate, including the
|
|
178
|
+
// low surrogate, so a protected 80-character opening never shrinks to 79.
|
|
179
|
+
if (end < text.length && /[\uD800-\uDBFF]/u.test(text[end - 1]))
|
|
180
|
+
end++;
|
|
181
|
+
const locator = block.sourceIndex === undefined ? undefined : { index: block.sourceIndex };
|
|
182
|
+
return [{ kind: "excerpt", role: "user", sourceIndex: block.sourceIndex, text: text.slice(0, end) },
|
|
183
|
+
...(end < text.length ? [{ kind: "omission", role: "user", sourceIndex: block.sourceIndex,
|
|
184
|
+
reason: locator ? `tuck_recall ${JSON.stringify(block.startsSource ? { ...locator, offset: end } : locator)}` : "Source has no locator" }] : [])];
|
|
185
|
+
};
|
|
170
186
|
/** A source time for each locator a user record names; only briefs of times-carrying blocks have them. */
|
|
171
187
|
export const decorateRecord = (record, times) => {
|
|
172
188
|
// Times mark the user's turns only: on every record they cost a fifth to a third of the summary's tokens.
|
|
173
189
|
if (!times || record.role !== "user")
|
|
174
190
|
return record;
|
|
191
|
+
if (record.sourceRange)
|
|
192
|
+
return record;
|
|
175
193
|
const indices = record.sourceIndices ?? (record.sourceIndex === undefined ? [] : [record.sourceIndex]);
|
|
176
194
|
return indices.length ? { ...record, sourceTimes: indices.map(sourceIndex => ({ sourceIndex, ...(times[sourceIndex] ?? { basis: "unknown" }) })) } : record;
|
|
177
195
|
};
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { NormalizedBlock } from "../types.js";
|
|
2
|
-
/** Suppress lookup echoes
|
|
2
|
+
/** Suppress lookup call echoes. Every result stays available for counted omissions. */
|
|
3
3
|
export declare const filterNoise: (blocks: NormalizedBlock[]) => NormalizedBlock[];
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import { isSourceExport, isLookupTool, isRecallTool } from "./recall-operation.js";
|
|
2
2
|
import { isDialogue } from "./brief.js";
|
|
3
|
-
/** Suppress lookup echoes
|
|
3
|
+
/** Suppress lookup call echoes. Every result stays available for counted omissions. */
|
|
4
4
|
export const filterNoise = (blocks) => blocks.filter(b => !(b.kind === "tool_call" && isLookupTool(b.name) && !(isRecallTool(b.name) && isSourceExport(b.args))) &&
|
|
5
|
-
!(b.kind === "tool_result" && isLookupTool(b.name) &&
|
|
6
|
-
!(isRecallTool(b.name) && b.recallExport)) &&
|
|
7
5
|
!(isDialogue(b) && !b.text.trim()));
|
|
@@ -7,8 +7,5 @@ import type { CompilerMessage, NormalizedBlock } from "../types.js";
|
|
|
7
7
|
* legacy positional behavior is preserved (used by callers/tests that have no
|
|
8
8
|
* index space to map into).
|
|
9
9
|
*/
|
|
10
|
-
/**
|
|
11
|
-
export declare const normalize: (messages: CompilerMessage[], sourceIndices?: Array<number | undefined>,
|
|
12
|
-
messages: CompilerMessage[];
|
|
13
|
-
sourceIndices: Array<number | undefined>;
|
|
14
|
-
}, sourceEntryIds?: readonly string[], sourceTimes?: readonly import("./source-time.js").SourceTime[]) => NormalizedBlock[];
|
|
10
|
+
/** Original-source blocks only; every historical result is counted, without pairing or inferred effects. */
|
|
11
|
+
export declare const normalize: (messages: CompilerMessage[], sourceIndices?: Array<number | undefined>, sourceEntryIds?: readonly string[], sourceTimes?: readonly import("./source-time.js").SourceTime[]) => NormalizedBlock[];
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { isSourceExport } from "./recall-operation.js";
|
|
2
1
|
import { textOf } from "./content.js";
|
|
3
2
|
import { sanitize } from "./sanitize.js";
|
|
4
3
|
const normalizeOne = (msg, msgIndex) => {
|
|
@@ -12,7 +11,7 @@ const normalizeOne = (msg, msgIndex) => {
|
|
|
12
11
|
if (msg.content && typeof msg.content !== "string") {
|
|
13
12
|
for (const part of msg.content) {
|
|
14
13
|
if (part.type === "image") {
|
|
15
|
-
blocks.push({ kind: "user", text: `[image: ${part.mimeType}]`, sourceIndex: msgIndex });
|
|
14
|
+
blocks.push({ kind: "user", image: true, text: `[image: ${part.mimeType}]`, sourceIndex: msgIndex });
|
|
16
15
|
}
|
|
17
16
|
}
|
|
18
17
|
}
|
|
@@ -58,45 +57,28 @@ const normalizeOne = (msg, msgIndex) => {
|
|
|
58
57
|
* legacy positional behavior is preserved (used by callers/tests that have no
|
|
59
58
|
* index space to map into).
|
|
60
59
|
*/
|
|
61
|
-
/**
|
|
62
|
-
export const normalize = (messages, sourceIndices,
|
|
60
|
+
/** Original-source blocks only; every historical result is counted, without pairing or inferred effects. */
|
|
61
|
+
export const normalize = (messages, sourceIndices, sourceEntryIds, sourceTimes) => {
|
|
63
62
|
if (sourceEntryIds && sourceEntryIds.length !== messages.length)
|
|
64
63
|
throw new Error("Original entry IDs must align with compiler messages");
|
|
65
64
|
if (sourceTimes && sourceTimes.length !== messages.length)
|
|
66
65
|
throw new Error("Original times must align with compiler messages");
|
|
67
66
|
const timeByIndex = Object.fromEntries((sourceIndices ?? messages.map((_, i) => i)).flatMap((index, i) => index === undefined || !sourceTimes ? [] : [[index, sourceTimes[i]]]));
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
pairs.set(id, pair);
|
|
76
|
-
}
|
|
77
|
-
return pair;
|
|
78
|
-
};
|
|
79
|
-
(receiptContext?.messages ?? messages).forEach((msg, order) => {
|
|
80
|
-
if (msg.role === "assistant" && Array.isArray(msg.content))
|
|
81
|
-
for (const part of msg.content) {
|
|
82
|
-
if (part.type === "toolCall")
|
|
83
|
-
group(part.id).calls.push({ name: part.name, order, args: part.arguments });
|
|
84
|
-
}
|
|
85
|
-
if (msg.role === "toolResult") {
|
|
86
|
-
if (typeof msg.isError !== "boolean")
|
|
67
|
+
// Identity/status validation is still useful for malformed source history;
|
|
68
|
+
// export pairing is not: every result now contributes to counted omissions.
|
|
69
|
+
for (const message of messages) {
|
|
70
|
+
if (message.role === "toolResult") {
|
|
71
|
+
if (typeof message.toolCallId !== "string" || !message.toolCallId)
|
|
72
|
+
throw new Error("Tool history lacks a recorded call identity");
|
|
73
|
+
if (typeof message.isError !== "boolean")
|
|
87
74
|
throw new Error("Tool receipt lacks a recorded error status");
|
|
88
|
-
group(msg.toolCallId).receipts.push({ name: msg.toolName, order });
|
|
89
75
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
};
|
|
76
|
+
if (message.role === "assistant" && Array.isArray(message.content))
|
|
77
|
+
for (const part of message.content)
|
|
78
|
+
if (part.type === "toolCall" && (typeof part.id !== "string" || !part.id))
|
|
79
|
+
throw new Error("Tool history lacks a recorded call identity");
|
|
80
|
+
}
|
|
96
81
|
return messages.flatMap((msg, i) => normalizeOne(msg, sourceIndices ? sourceIndices[i] : i).map(block => {
|
|
97
|
-
if (block.kind === "tool_result" && msg.role === "toolResult" && paired(msg.toolCallId) &&
|
|
98
|
-
isSourceExport(pairs.get(msg.toolCallId)?.calls[0]?.args))
|
|
99
|
-
block = { ...block, recallExport: true };
|
|
100
82
|
if (sourceTimes)
|
|
101
83
|
block = { ...block, sourceTimes: timeByIndex };
|
|
102
84
|
return block.kind === "notice" && sourceEntryIds?.[i] !== undefined ? { ...block, sourceEntryId: sourceEntryIds[i] } : block;
|
|
@@ -8,26 +8,18 @@ export interface DialogueInput {
|
|
|
8
8
|
sourceIndices: readonly (number | undefined)[];
|
|
9
9
|
sourceEntryIds?: readonly string[];
|
|
10
10
|
}
|
|
11
|
-
/**
|
|
12
|
-
export type Step = "whole" |
|
|
11
|
+
/** Whether the ranked view is whole, selected, or cannot fit its omission metadata. */
|
|
12
|
+
export type Step = "whole" | "ranked" | "none";
|
|
13
13
|
export interface Shrunk {
|
|
14
14
|
/** The summary text, or "" when nothing fits. */
|
|
15
15
|
text: string;
|
|
16
16
|
step: Step;
|
|
17
|
-
/** The
|
|
17
|
+
/** The serialized size of the selected view. */
|
|
18
18
|
size: number;
|
|
19
19
|
}
|
|
20
20
|
/** The whole pre-cut dialogue: user, assistant and notice records; tool actions and observations are omitted. "" when it renders nothing. */
|
|
21
21
|
export declare const wholeDialogue: (input: DialogueInput) => string;
|
|
22
|
-
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* B. the newest assistant units that fit a third of the limit whole, the older ones at tier 3 with the oldest d dropped;
|
|
26
|
-
* C. the newest assistant units that fit a third of the limit at tier 3, the rest dropped; then the oldest u user units clipped to tier 1, 2, 3;
|
|
27
|
-
* D. every user unit at tier 3, the oldest dropped;
|
|
28
|
-
* E. the oldest notices dropped, then the oldest of the remaining assistant units.
|
|
29
|
-
* Every step is a one-parameter family whose size is read from prefix sums of the per-unit lengths, so a step costs O(units). A notice is clipped
|
|
30
|
-
* to its one tier in every configuration but the whole dialogue. The size is arithmetic over unit lengths; the rendered text has the last say:
|
|
31
|
-
* a configuration whose text is over the limit is passed over.
|
|
32
|
-
*/
|
|
22
|
+
/** Lower ranks of the single fill: the existing recent assistant third, then
|
|
23
|
+
* more user turns newest first, then the remaining dialogue newest first.
|
|
24
|
+
* Each lane grows only from its recent end; complete passages remain atomic. */
|
|
33
25
|
export declare const shrinkDialogue: (input: DialogueInput, limit: number) => Shrunk;
|