pi-posthorse 0.9.3 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -26
- package/index.ts +1 -1
- package/package.json +10 -9
- package/src/budget.ts +5 -5
- package/src/context-tools.ts +3 -3
- package/src/contracts.ts +1 -0
- package/src/files.ts +87 -25
- package/src/history-search.ts +31 -15
- package/src/history.ts +1 -1
- package/src/notes.ts +116 -45
- package/src/paging.ts +1 -1
- package/src/recovery-results.ts +5 -2
- package/src/recovery.ts +59 -21
- package/src/retention.ts +203 -0
- package/src/rollover.ts +29 -20
- package/src/session-files.ts +37 -24
- package/src/session-lines.ts +108 -0
package/src/notes.ts
CHANGED
|
@@ -1,13 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
existsSync,
|
|
4
|
-
mkdirSync,
|
|
5
|
-
readdirSync,
|
|
6
|
-
readFileSync,
|
|
7
|
-
realpathSync,
|
|
8
|
-
statSync,
|
|
9
|
-
} from "node:fs";
|
|
1
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { readdir, realpath, stat } from "node:fs/promises";
|
|
10
3
|
import { dirname, isAbsolute, join, normalize, sep } from "node:path";
|
|
4
|
+
import { setImmediate } from "node:timers/promises";
|
|
11
5
|
import {
|
|
12
6
|
formatSize,
|
|
13
7
|
withFileMutationQueue,
|
|
@@ -18,6 +12,7 @@ import { Type } from "typebox";
|
|
|
18
12
|
import { toolCards } from "../ui.ts";
|
|
19
13
|
import {
|
|
20
14
|
MAX_PAGE_CHARS,
|
|
15
|
+
isRecord,
|
|
21
16
|
requireValue,
|
|
22
17
|
textResult,
|
|
23
18
|
type PolicyContext,
|
|
@@ -36,10 +31,25 @@ export type NotesParams = {
|
|
|
36
31
|
};
|
|
37
32
|
type NoteFile = { readonly path: string; readonly size: number; readonly mtimeMs: number };
|
|
38
33
|
type NoteHits = { readonly rows: string[]; readonly more: boolean; readonly skipped: number };
|
|
34
|
+
/** Paged operations take a fresh policy snapshot after their filesystem awaits. */
|
|
35
|
+
type NotesAccess = {
|
|
36
|
+
readonly host: () => PolicyContext;
|
|
37
|
+
readonly pages: PageAccess;
|
|
38
|
+
readonly signal: AbortSignal | undefined;
|
|
39
|
+
};
|
|
39
40
|
/** Larger files are logs or evidence rather than notes; reading them would dominate a search. */
|
|
40
41
|
const SEARCH_FILE_BYTES = 1024 * 1024;
|
|
41
42
|
/** Overlapping reads cut a full search of a 212,000-file notes tree from 39 s to 14 s. */
|
|
42
43
|
const SEARCH_BATCH = 16;
|
|
44
|
+
/**
|
|
45
|
+
* Maximum explicit `stat` calls overlapped per batch. On a 212,751-note tree, 16, 64 and 256 took
|
|
46
|
+
* equal time within host noise. The walk enters one folder at a time, so this bounds our pending
|
|
47
|
+
* `stat` calls at any depth. On filesystems that report unknown entry types, Node 24's
|
|
48
|
+
* `readdir({ withFileTypes: true })` also issues its own `lstat` for every entry at once.
|
|
49
|
+
*/
|
|
50
|
+
const WALK_BATCH = 64;
|
|
51
|
+
/** Formatting all 212,751 rows of one listing at once stalled Pi for 156-163 ms. */
|
|
52
|
+
const FORMAT_BATCH = 10_000;
|
|
43
53
|
/** Search rescans up to the requested offset, so this also caps the matches one call keeps. */
|
|
44
54
|
const SEARCH_OFFSET_LIMIT = 1_000_000;
|
|
45
55
|
function safeJoin(dir: string, path: string): string {
|
|
@@ -49,46 +59,105 @@ function safeJoin(dir: string, path: string): string {
|
|
|
49
59
|
}
|
|
50
60
|
return join(dir, relative);
|
|
51
61
|
}
|
|
52
|
-
|
|
53
|
-
|
|
62
|
+
/** Like `statSync(path, { throwIfNoEntry: false })`: an entry deleted mid-walk is skipped. */
|
|
63
|
+
async function statIfPresent(path: string) {
|
|
64
|
+
try {
|
|
65
|
+
return await stat(path);
|
|
66
|
+
} catch (error) {
|
|
67
|
+
if (isRecord(error) && (error.code === "ENOENT" || error.code === "ENOTDIR")) {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
throw error;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/** A folder's present entries in listing order. */
|
|
74
|
+
async function* folderEntries(directory: string, signal: AbortSignal | undefined) {
|
|
75
|
+
const entries = await readdir(directory, { withFileTypes: true });
|
|
76
|
+
let start = 0;
|
|
77
|
+
while (start < entries.length) {
|
|
78
|
+
signal?.throwIfAborted();
|
|
79
|
+
const batch = entries.slice(start, start + WALK_BATCH);
|
|
80
|
+
const firstFolder = batch.findIndex(
|
|
81
|
+
(entry) => entry.isDirectory() || entry.isSymbolicLink(),
|
|
82
|
+
);
|
|
83
|
+
const paths = batch
|
|
84
|
+
.slice(0, firstFolder === -1 ? batch.length : firstFolder + 1)
|
|
85
|
+
.map((entry) => join(directory, entry.name));
|
|
86
|
+
// Stats overlap only up to the next entry that may be a folder.
|
|
87
|
+
// oxlint-disable-next-line no-await-in-loop
|
|
88
|
+
const infos = await Promise.all(paths.map(statIfPresent));
|
|
89
|
+
for (const [index, path] of paths.entries()) {
|
|
90
|
+
start += 1;
|
|
91
|
+
const info = infos[index];
|
|
92
|
+
if (info !== undefined) {
|
|
93
|
+
yield { path, info };
|
|
94
|
+
}
|
|
95
|
+
if (info?.isDirectory() === true) {
|
|
96
|
+
// Stat later entries only after this folder's subtree is walked, discarding any
|
|
97
|
+
// already taken, so entries removed meanwhile are skipped.
|
|
98
|
+
break;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/** Files depth-first in directory-listing order, yielding between filesystem calls. */
|
|
104
|
+
async function walk(
|
|
105
|
+
directory: string,
|
|
106
|
+
signal: AbortSignal | undefined,
|
|
107
|
+
ancestors: readonly string[] = [],
|
|
108
|
+
): Promise<NoteFile[]> {
|
|
109
|
+
signal?.throwIfAborted();
|
|
110
|
+
const root = await realpath(directory);
|
|
54
111
|
if (ancestors.includes(root)) {
|
|
55
112
|
return [];
|
|
56
113
|
}
|
|
57
114
|
const next = [...ancestors, root];
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
115
|
+
const files: NoteFile[] = [];
|
|
116
|
+
for await (const { path, info } of folderEntries(directory, signal)) {
|
|
117
|
+
if (info.isDirectory()) {
|
|
118
|
+
for (const file of await walk(path, signal, next)) {
|
|
119
|
+
files.push(file);
|
|
120
|
+
}
|
|
121
|
+
} else {
|
|
122
|
+
files.push({ path, size: info.size, mtimeMs: info.mtimeMs });
|
|
63
123
|
}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
: [{ path, size: info.size, mtimeMs: info.mtimeMs }];
|
|
67
|
-
});
|
|
124
|
+
}
|
|
125
|
+
return files;
|
|
68
126
|
}
|
|
69
|
-
function notesUnder(target: string): NoteFile[] {
|
|
70
|
-
const info =
|
|
127
|
+
async function notesUnder(target: string, signal: AbortSignal | undefined): Promise<NoteFile[]> {
|
|
128
|
+
const info = await statIfPresent(target);
|
|
129
|
+
signal?.throwIfAborted();
|
|
71
130
|
if (info === undefined) {
|
|
72
131
|
return [];
|
|
73
132
|
}
|
|
74
133
|
const files = info.isDirectory()
|
|
75
|
-
? walk(target)
|
|
134
|
+
? await walk(target, signal)
|
|
76
135
|
: [{ path: target, size: info.size, mtimeMs: info.mtimeMs }];
|
|
136
|
+
signal?.throwIfAborted();
|
|
77
137
|
return files.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
78
138
|
}
|
|
79
|
-
function listNotes(
|
|
139
|
+
async function listNotes(
|
|
80
140
|
dir: string,
|
|
81
141
|
params: NotesParams,
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
142
|
+
access: NotesAccess,
|
|
143
|
+
): Promise<TextResult> {
|
|
144
|
+
const { host, pages, signal } = access;
|
|
85
145
|
const folder = params.path;
|
|
86
146
|
const target = folder === undefined || folder === "" ? dir : safeJoin(dir, folder);
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
147
|
+
const files = await notesUnder(target, signal);
|
|
148
|
+
const rows: string[] = [];
|
|
149
|
+
for (let start = 0; start < files.length; start += FORMAT_BATCH) {
|
|
150
|
+
// Pi gets an event-loop turn before each batch of rows.
|
|
151
|
+
// oxlint-disable-next-line no-await-in-loop
|
|
152
|
+
await setImmediate();
|
|
153
|
+
signal?.throwIfAborted();
|
|
154
|
+
for (const file of files.slice(start, start + FORMAT_BATCH)) {
|
|
155
|
+
rows.push(
|
|
156
|
+
`${file.path.slice(dir.length + 1)} ${formatSize(file.size)} ${new Date(file.mtimeMs).toISOString().slice(0, 16)}Z`,
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return pages.notes(host(), {
|
|
92
161
|
rows,
|
|
93
162
|
offset: params.offset ?? 0,
|
|
94
163
|
kind: "notes-list",
|
|
@@ -99,9 +168,9 @@ function listNotes(
|
|
|
99
168
|
async function readNote(
|
|
100
169
|
dir: string,
|
|
101
170
|
params: NotesParams,
|
|
102
|
-
|
|
103
|
-
pages: PageAccess,
|
|
171
|
+
access: NotesAccess,
|
|
104
172
|
): Promise<TextResult> {
|
|
173
|
+
const { pages } = access;
|
|
105
174
|
const relative = requireValue(params.path, "path", params.op);
|
|
106
175
|
const path = safeJoin(dir, relative);
|
|
107
176
|
const info = statSync(path, { throwIfNoEntry: false });
|
|
@@ -109,7 +178,7 @@ async function readNote(
|
|
|
109
178
|
throw new Error(`No note at ${relative}. Use op "list" to see available notes.`);
|
|
110
179
|
}
|
|
111
180
|
if (info.isDirectory()) {
|
|
112
|
-
return listNotes(dir, params,
|
|
181
|
+
return listNotes(dir, params, access);
|
|
113
182
|
}
|
|
114
183
|
if (await isBinaryFile(path)) {
|
|
115
184
|
throw new Error(
|
|
@@ -122,7 +191,7 @@ async function readNote(
|
|
|
122
191
|
throw new Error(`Offset ${offset} is past the end of ${relative} (${text.length} chars).`);
|
|
123
192
|
}
|
|
124
193
|
const header = `File: ${path}\n`;
|
|
125
|
-
const host =
|
|
194
|
+
const host = access.host();
|
|
126
195
|
const chars = pages.size(host, { offset }) - header.length;
|
|
127
196
|
if (chars <= 0) {
|
|
128
197
|
pages.error(
|
|
@@ -197,11 +266,13 @@ async function noteHits(
|
|
|
197
266
|
limit: number,
|
|
198
267
|
signal?: AbortSignal,
|
|
199
268
|
): Promise<NoteHits> {
|
|
200
|
-
const notes = notesUnder(dir);
|
|
269
|
+
const notes = await notesUnder(dir, signal);
|
|
201
270
|
const rows: string[] = [];
|
|
202
271
|
let length = -1;
|
|
203
272
|
let skipped = 0;
|
|
204
273
|
for (let start = 0; start < notes.length; start += SEARCH_BATCH) {
|
|
274
|
+
// A batch of notes over 1 MiB reads nothing that would notice cancellation.
|
|
275
|
+
signal?.throwIfAborted();
|
|
205
276
|
// Batches keep note order, so a search stops within one batch of the page limit.
|
|
206
277
|
// oxlint-disable-next-line no-await-in-loop
|
|
207
278
|
const batch = await Promise.all(
|
|
@@ -231,21 +302,20 @@ async function noteHits(
|
|
|
231
302
|
async function searchNotes(
|
|
232
303
|
dir: string,
|
|
233
304
|
params: NotesParams,
|
|
234
|
-
access:
|
|
235
|
-
signal?: AbortSignal,
|
|
305
|
+
access: NotesAccess,
|
|
236
306
|
): Promise<TextResult> {
|
|
237
307
|
const { host, pages } = access;
|
|
238
308
|
const query = requireValue(params.query, "query", params.op);
|
|
239
309
|
const offset = params.offset ?? 0;
|
|
240
310
|
if (offset >= SEARCH_OFFSET_LIMIT) {
|
|
241
311
|
pages.error(
|
|
242
|
-
host,
|
|
312
|
+
host(),
|
|
243
313
|
`Notes search pages end before offset ${SEARCH_OFFSET_LIMIT}; search for something more specific.`,
|
|
244
314
|
);
|
|
245
315
|
}
|
|
246
|
-
const hits = await noteHits(dir, query.toLowerCase(), offset + MAX_PAGE_CHARS, signal);
|
|
316
|
+
const hits = await noteHits(dir, query.toLowerCase(), offset + MAX_PAGE_CHARS, access.signal);
|
|
247
317
|
const skipped = hits.skipped === 1 ? "1 note" : `${hits.skipped} notes`;
|
|
248
|
-
return pages.notes(host, {
|
|
318
|
+
return pages.notes(host(), {
|
|
249
319
|
rows: hits.rows,
|
|
250
320
|
more: hits.more,
|
|
251
321
|
offset,
|
|
@@ -297,17 +367,18 @@ export function registerNotes(
|
|
|
297
367
|
// oxlint-disable-next-line max-params
|
|
298
368
|
async execute(_id, params, signal, _onUpdate, ctx) {
|
|
299
369
|
const dir = notesDir(ctx.cwd);
|
|
370
|
+
const access: NotesAccess = { host: () => policy(ctx), pages, signal };
|
|
300
371
|
switch (params.op) {
|
|
301
372
|
case "list":
|
|
302
|
-
return listNotes(dir, params,
|
|
373
|
+
return listNotes(dir, params, access);
|
|
303
374
|
case "read":
|
|
304
|
-
return readNote(dir, params,
|
|
375
|
+
return readNote(dir, params, access);
|
|
305
376
|
case "write":
|
|
306
377
|
return writeNote(dir, params, signal);
|
|
307
378
|
case "append":
|
|
308
379
|
return appendNote(dir, params, signal);
|
|
309
380
|
case "search":
|
|
310
|
-
return searchNotes(dir, params,
|
|
381
|
+
return searchNotes(dir, params, access);
|
|
311
382
|
}
|
|
312
383
|
},
|
|
313
384
|
});
|
package/src/paging.ts
CHANGED
|
@@ -63,7 +63,7 @@ export class PageReservations {
|
|
|
63
63
|
);
|
|
64
64
|
}
|
|
65
65
|
this.previousUsage = usage;
|
|
66
|
-
const remaining = pageCapacity(ctx, this.toolTokens
|
|
66
|
+
const remaining = pageCapacity(ctx, this.toolTokens) - this.pendingTokens;
|
|
67
67
|
const chars = Math.min(
|
|
68
68
|
MAX_PAGE_CHARS,
|
|
69
69
|
Math.max(0, remaining - PAGE_MARGIN_TOKENS) * 4 -
|
package/src/recovery-results.ts
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
toolIdentity,
|
|
10
10
|
} from "./message-text.ts";
|
|
11
11
|
|
|
12
|
+
export type RecoveryEntry = EntryLike & { readonly retained: boolean };
|
|
12
13
|
type LinkedCall = { readonly entry: EntryLike; readonly block: ToolCallLike };
|
|
13
14
|
export type RecoveryBlock = { readonly header: string; readonly text: string };
|
|
14
15
|
export type RecoveryBatch = { readonly callId?: string; readonly blocks: readonly RecoveryBlock[] };
|
|
@@ -82,12 +83,14 @@ function resultBlock(
|
|
|
82
83
|
};
|
|
83
84
|
}
|
|
84
85
|
/** Async receipts may arrive after a response; completion does not establish receipt delivery. */
|
|
85
|
-
export function recoverableToolResults(
|
|
86
|
+
export function recoverableToolResults(
|
|
87
|
+
entries: readonly RecoveryEntry[],
|
|
88
|
+
): RecoveryBatch | undefined {
|
|
86
89
|
const calls = callsById(entries);
|
|
87
90
|
const start = trailingStart(entries);
|
|
88
91
|
const results = entries.filter((entry, index) => {
|
|
89
92
|
const message = entry.message;
|
|
90
|
-
if (message?.role !== "toolResult" || message.toolCallId === undefined) {
|
|
93
|
+
if (message?.role !== "toolResult" || message.toolCallId === undefined || entry.retained) {
|
|
91
94
|
return false;
|
|
92
95
|
}
|
|
93
96
|
const call = calls.get(message.toolCallId);
|
package/src/recovery.ts
CHANGED
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
recoverableToolResults,
|
|
19
19
|
type RecoveryBatch,
|
|
20
20
|
type RecoveryBlock,
|
|
21
|
+
type RecoveryEntry,
|
|
21
22
|
} from "./recovery-results.ts";
|
|
22
23
|
|
|
23
24
|
const MAX_RECOVERY_RECORD_CHARS = 4_000;
|
|
@@ -121,6 +122,7 @@ function projectedCurrent(
|
|
|
121
122
|
entries: readonly EntryLike[],
|
|
122
123
|
projected: readonly ProjectedEntry[],
|
|
123
124
|
current: readonly EntryLike[] = entries.slice(contextStart(entries).start),
|
|
125
|
+
retained: ReadonlySet<string> = new Set(),
|
|
124
126
|
): EntryLike[] {
|
|
125
127
|
const edits = contextEdits(entries);
|
|
126
128
|
const omittedIds = new Set(
|
|
@@ -130,7 +132,7 @@ function projectedCurrent(
|
|
|
130
132
|
);
|
|
131
133
|
return current.flatMap((entry) => {
|
|
132
134
|
const id = entry.id ?? "";
|
|
133
|
-
if (id
|
|
135
|
+
if (retained.has(id) || omittedIds.has(id)) {
|
|
134
136
|
return [];
|
|
135
137
|
}
|
|
136
138
|
const edit = edits.get(id);
|
|
@@ -211,6 +213,7 @@ function recoveryAncestry(
|
|
|
211
213
|
entries: readonly EntryLike[],
|
|
212
214
|
projected: readonly ProjectedEntry[],
|
|
213
215
|
ownerQuestionRegistered: boolean,
|
|
216
|
+
retained: ReadonlySet<string>,
|
|
214
217
|
): {
|
|
215
218
|
readonly checkpoint?: EntryLike;
|
|
216
219
|
readonly carried: readonly RecoveryRecord[];
|
|
@@ -234,7 +237,7 @@ function recoveryAncestry(
|
|
|
234
237
|
if (owners.length > 0) {
|
|
235
238
|
return {
|
|
236
239
|
checkpoint: automaticBoundary(window.boundary) ? undefined : window.boundary,
|
|
237
|
-
carried: anchors(owners).fixed,
|
|
240
|
+
carried: anchors(owners.filter((record) => !retained.has(record.id))).fixed,
|
|
238
241
|
skipped,
|
|
239
242
|
};
|
|
240
243
|
}
|
|
@@ -262,13 +265,16 @@ function skippedRecords(ids: readonly string[], limit: number): string | undefin
|
|
|
262
265
|
function projectedToolEntries(
|
|
263
266
|
entries: readonly EntryLike[],
|
|
264
267
|
projected: readonly ProjectedEntry[],
|
|
265
|
-
|
|
268
|
+
retained: ReadonlySet<string>,
|
|
269
|
+
): RecoveryEntry[] {
|
|
266
270
|
const edits = contextEdits(entries);
|
|
267
271
|
return projected.flatMap(({ sourceEntry, messages }) =>
|
|
268
272
|
messages
|
|
269
273
|
.filter((message) => message.role === "assistant" || message.role === "toolResult")
|
|
270
274
|
.map((message) => ({
|
|
271
275
|
type: "message",
|
|
276
|
+
// Resolve retention before replacing identity with the history-read reference.
|
|
277
|
+
retained: retained.has(sourceEntry.id ?? ""),
|
|
272
278
|
id: edits.get(sourceEntry.id ?? "")?.id ?? sourceEntry.id,
|
|
273
279
|
message,
|
|
274
280
|
})),
|
|
@@ -378,37 +384,69 @@ function renderIndex(records: readonly RecoveryRecord[], available: number): str
|
|
|
378
384
|
...listed,
|
|
379
385
|
].join("\n");
|
|
380
386
|
}
|
|
381
|
-
|
|
382
|
-
export function buildAutoHandoff(
|
|
387
|
+
function selectedInputs(
|
|
383
388
|
entries: readonly EntryLike[],
|
|
384
389
|
projected: readonly ProjectedEntry[],
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
+
options: {
|
|
391
|
+
readonly ownerQuestionRegistered: boolean;
|
|
392
|
+
readonly retained: ReadonlySet<string>;
|
|
393
|
+
readonly carried: readonly RecoveryRecord[];
|
|
394
|
+
},
|
|
395
|
+
): { readonly fixed: readonly RecoveryRecord[]; readonly others: readonly RecoveryRecord[] } {
|
|
396
|
+
const { ownerQuestionRegistered, retained, carried } = options;
|
|
397
|
+
const current = projectedCurrent(entries, projected);
|
|
398
|
+
const allRecords = ownerRecords(current, projected, ownerQuestionRegistered);
|
|
390
399
|
const records = ownerRecords(
|
|
391
|
-
projectedCurrent(entries, projected),
|
|
400
|
+
projectedCurrent(entries, projected, undefined, retained),
|
|
392
401
|
projected,
|
|
393
402
|
ownerQuestionRegistered,
|
|
394
403
|
);
|
|
395
404
|
const { fixed, others } = anchors(records);
|
|
396
|
-
if (!
|
|
405
|
+
if (!allRecords.some((record) => record.kind === "owner")) {
|
|
397
406
|
fixed.unshift(
|
|
398
|
-
...
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
407
|
+
...carried.map((record) =>
|
|
408
|
+
Object.assign({}, record, {
|
|
409
|
+
label: `${record.label}; carried from an earlier window, possibly already handled; not current progress`,
|
|
410
|
+
}),
|
|
411
|
+
),
|
|
402
412
|
);
|
|
403
413
|
}
|
|
414
|
+
return { fixed, others };
|
|
415
|
+
}
|
|
416
|
+
/** Raw owner anchors and projected tool evidence have distinct provenance and share one bounded payload. */
|
|
417
|
+
export function buildAutoHandoff(
|
|
418
|
+
entries: readonly EntryLike[],
|
|
419
|
+
projected: readonly ProjectedEntry[],
|
|
420
|
+
maxChars: number,
|
|
421
|
+
options: {
|
|
422
|
+
readonly ownerQuestionRegistered: boolean;
|
|
423
|
+
readonly retainedEntryIds?: ReadonlySet<string>;
|
|
424
|
+
},
|
|
425
|
+
): string {
|
|
426
|
+
const { ownerQuestionRegistered, retainedEntryIds: retained = new Set<string>() } = options;
|
|
427
|
+
const preamble =
|
|
428
|
+
retained.size === 0
|
|
429
|
+
? PREAMBLE
|
|
430
|
+
: `${PREAMBLE}\nRecent messages follow separately; their inputs and tool evidence are not repeated in this record.`;
|
|
431
|
+
const history = recoveryAncestry(entries, projected, ownerQuestionRegistered, retained);
|
|
432
|
+
const priorWindow = history.checkpoint;
|
|
433
|
+
const { fixed, others } = selectedInputs(entries, projected, {
|
|
434
|
+
ownerQuestionRegistered,
|
|
435
|
+
retained,
|
|
436
|
+
carried: history.carried,
|
|
437
|
+
});
|
|
438
|
+
const emptyInputHeader =
|
|
439
|
+
retained.size > 0
|
|
440
|
+
? "No discarded owner or visible coordination inputs were selected for this recovery record."
|
|
441
|
+
: "No selected current-window owner or visible coordination inputs were found.";
|
|
404
442
|
const inputHeader =
|
|
405
443
|
fixed.length > 0
|
|
406
444
|
? "Selected owner and coordination inputs (chronological):"
|
|
407
|
-
:
|
|
408
|
-
const batch = recoverableToolResults(projectedToolEntries(entries, projected));
|
|
445
|
+
: emptyInputHeader;
|
|
446
|
+
const batch = recoverableToolResults(projectedToolEntries(entries, projected, retained));
|
|
409
447
|
const header = batchHeader(batch);
|
|
410
448
|
const minimum = [
|
|
411
|
-
|
|
449
|
+
preamble,
|
|
412
450
|
inputHeader,
|
|
413
451
|
...fixed.map((record) => formatRecord(record, 0)),
|
|
414
452
|
header,
|
|
@@ -427,7 +465,7 @@ export function buildAutoHandoff(
|
|
|
427
465
|
(history.skipped.length > 0 ? 1 : 0);
|
|
428
466
|
const fixedBudget = Math.max(
|
|
429
467
|
0,
|
|
430
|
-
maxChars - joined([
|
|
468
|
+
maxChars - joined([preamble, inputHeader, plan.bare]).length - HANDOFF_OVERHEAD_RESERVE,
|
|
431
469
|
);
|
|
432
470
|
const fixedLimit =
|
|
433
471
|
count > 0
|
|
@@ -438,7 +476,7 @@ export function buildAutoHandoff(
|
|
|
438
476
|
skippedRecords(history.skipped, fixedLimit),
|
|
439
477
|
]);
|
|
440
478
|
const fixedParts = [
|
|
441
|
-
|
|
479
|
+
preamble,
|
|
442
480
|
inputHeader,
|
|
443
481
|
...fixed.map((record) => formatRecord(record, fixedLimit)),
|
|
444
482
|
];
|
package/src/retention.ts
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DEFAULT_COMPACTION_SETTINGS,
|
|
3
|
+
estimateTokens,
|
|
4
|
+
findCutPoint,
|
|
5
|
+
type ProjectedSessionEntry,
|
|
6
|
+
type SessionEntry,
|
|
7
|
+
type SessionMessageEntry,
|
|
8
|
+
} from "@earendil-works/pi-coding-agent";
|
|
9
|
+
import { freshPayloadChars } from "./budget.ts";
|
|
10
|
+
import {
|
|
11
|
+
MAX_HANDOFF_CHARS,
|
|
12
|
+
MIN_PAGE_CHARS,
|
|
13
|
+
type EntryLike,
|
|
14
|
+
type MessageLike,
|
|
15
|
+
type PolicyContext,
|
|
16
|
+
} from "./contracts.ts";
|
|
17
|
+
import { buildAutoHandoff } from "./recovery.ts";
|
|
18
|
+
|
|
19
|
+
type RecoveryCapacity = {
|
|
20
|
+
readonly host: PolicyContext;
|
|
21
|
+
readonly toolTokens: number;
|
|
22
|
+
readonly pendingMessages?: readonly MessageLike[];
|
|
23
|
+
readonly firstKeptEntryId?: string;
|
|
24
|
+
};
|
|
25
|
+
type RetainedTail = {
|
|
26
|
+
readonly firstKeptEntryId?: string;
|
|
27
|
+
readonly ids: ReadonlySet<string>;
|
|
28
|
+
};
|
|
29
|
+
type Immutable<T> = T extends string | number | boolean | null | undefined
|
|
30
|
+
? T
|
|
31
|
+
: T extends readonly (infer Item)[]
|
|
32
|
+
? readonly Immutable<Item>[]
|
|
33
|
+
: T extends object
|
|
34
|
+
? { readonly [Key in keyof T]: Immutable<T[Key]> }
|
|
35
|
+
: T;
|
|
36
|
+
type ProjectionEntry = Immutable<ProjectedSessionEntry>;
|
|
37
|
+
const CUT_ROLES = new Set([
|
|
38
|
+
"user",
|
|
39
|
+
"assistant",
|
|
40
|
+
"bashExecution",
|
|
41
|
+
"custom",
|
|
42
|
+
"branchSummary",
|
|
43
|
+
"compactionSummary",
|
|
44
|
+
]);
|
|
45
|
+
function visibleMessages(entry: ProjectionEntry) {
|
|
46
|
+
return entry.messages.filter((message) => message.role !== "system");
|
|
47
|
+
}
|
|
48
|
+
function safeStart(entry: ProjectionEntry): boolean {
|
|
49
|
+
return (
|
|
50
|
+
entry.sourceEntry.type !== "compaction" &&
|
|
51
|
+
visibleMessages(entry).some((message) => CUT_ROLES.has(message.role))
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
function nativeMessage(
|
|
55
|
+
message: ProjectionEntry["messages"][number],
|
|
56
|
+
): SessionMessageEntry["message"] {
|
|
57
|
+
switch (message.role) {
|
|
58
|
+
case "assistant":
|
|
59
|
+
return {
|
|
60
|
+
role: "assistant",
|
|
61
|
+
api: message.api,
|
|
62
|
+
provider: message.provider,
|
|
63
|
+
model: message.model,
|
|
64
|
+
stopReason: message.stopReason,
|
|
65
|
+
usage: message.usage,
|
|
66
|
+
timestamp: message.timestamp,
|
|
67
|
+
content: [...message.content],
|
|
68
|
+
};
|
|
69
|
+
case "toolResult":
|
|
70
|
+
return {
|
|
71
|
+
role: "toolResult",
|
|
72
|
+
toolCallId: message.toolCallId,
|
|
73
|
+
toolName: message.toolName,
|
|
74
|
+
isError: message.isError,
|
|
75
|
+
timestamp: message.timestamp,
|
|
76
|
+
content: [...message.content],
|
|
77
|
+
};
|
|
78
|
+
case "user":
|
|
79
|
+
case "custom":
|
|
80
|
+
return {
|
|
81
|
+
...message,
|
|
82
|
+
content:
|
|
83
|
+
typeof message.content === "string" ? message.content : [...message.content],
|
|
84
|
+
};
|
|
85
|
+
case "system":
|
|
86
|
+
// Compaction checkpoints replace earlier system declarations.
|
|
87
|
+
return { role: "user", content: "", timestamp: message.timestamp };
|
|
88
|
+
case "bashExecution":
|
|
89
|
+
case "branchSummary":
|
|
90
|
+
case "compactionSummary":
|
|
91
|
+
return { ...message };
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/** The exported native cut finder reads raw entries; materialize only projected contributions. */
|
|
95
|
+
function projectedCut(entries: readonly ProjectionEntry[], keep: number): number {
|
|
96
|
+
const materialized = entries.flatMap<SessionEntry>(({ sourceEntry, messages }) =>
|
|
97
|
+
messages.length === 0 || sourceEntry.type === "compaction"
|
|
98
|
+
? [{ ...sourceEntry, type: "custom", customType: "posthorse-projection", data: {} }]
|
|
99
|
+
: messages.map((message) => ({
|
|
100
|
+
id: sourceEntry.id,
|
|
101
|
+
parentId: sourceEntry.parentId,
|
|
102
|
+
timestamp: sourceEntry.timestamp,
|
|
103
|
+
type: "message",
|
|
104
|
+
message: nativeMessage(message),
|
|
105
|
+
})),
|
|
106
|
+
);
|
|
107
|
+
const cut = findCutPoint(materialized, 0, materialized.length, keep);
|
|
108
|
+
const id = materialized[cut.firstKeptEntryIndex]?.id;
|
|
109
|
+
return entries.findIndex((entry) => entry.sourceEntry.id === id);
|
|
110
|
+
}
|
|
111
|
+
function pairedTail(entries: readonly ProjectionEntry[]): boolean {
|
|
112
|
+
const calls = new Set<string>();
|
|
113
|
+
for (const message of entries.flatMap(visibleMessages)) {
|
|
114
|
+
if (message.role === "assistant") {
|
|
115
|
+
for (const part of message.content.filter((block) => block.type === "toolCall")) {
|
|
116
|
+
calls.add(part.id);
|
|
117
|
+
}
|
|
118
|
+
} else if (message.role === "toolResult") {
|
|
119
|
+
if (!calls.delete(message.toolCallId)) {
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return calls.size === 0;
|
|
125
|
+
}
|
|
126
|
+
function boundedTail(
|
|
127
|
+
entries: readonly ProjectionEntry[],
|
|
128
|
+
start: number,
|
|
129
|
+
maxTokens: number,
|
|
130
|
+
): RetainedTail {
|
|
131
|
+
const costs = entries.map((entry) =>
|
|
132
|
+
entry.sourceEntry.type === "compaction"
|
|
133
|
+
? 0
|
|
134
|
+
: visibleMessages(entry).reduce(
|
|
135
|
+
(sum, message) => sum + estimateTokens(nativeMessage(message)),
|
|
136
|
+
0,
|
|
137
|
+
),
|
|
138
|
+
);
|
|
139
|
+
let tokens = costs.slice(start).reduce((sum, cost) => sum + cost, 0);
|
|
140
|
+
for (let index = start; index < entries.length; index++) {
|
|
141
|
+
if (tokens > 0 && tokens <= maxTokens && safeStart(entries[index])) {
|
|
142
|
+
const tail = entries.slice(index);
|
|
143
|
+
if (pairedTail(tail)) {
|
|
144
|
+
return {
|
|
145
|
+
firstKeptEntryId: entries[index].sourceEntry.id,
|
|
146
|
+
ids: new Set(tail.map((entry) => entry.sourceEntry.id)),
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
tokens -= costs[index];
|
|
151
|
+
}
|
|
152
|
+
return { ids: new Set() };
|
|
153
|
+
}
|
|
154
|
+
function retainedTail(
|
|
155
|
+
projected: readonly ProjectionEntry[],
|
|
156
|
+
capacity: RecoveryCapacity,
|
|
157
|
+
maxTokens: number,
|
|
158
|
+
): RetainedTail {
|
|
159
|
+
const keep =
|
|
160
|
+
capacity.host.getCompactionSettings().keepRecentTokens ??
|
|
161
|
+
DEFAULT_COMPACTION_SETTINGS.keepRecentTokens;
|
|
162
|
+
if (!Number.isFinite(keep) || keep <= 0 || maxTokens <= 0) {
|
|
163
|
+
return { ids: new Set() };
|
|
164
|
+
}
|
|
165
|
+
const boundary = projected.findIndex(
|
|
166
|
+
(entry) => entry.sourceEntry.type === "compaction" && entry.messages.length > 0,
|
|
167
|
+
);
|
|
168
|
+
const entries = projected.slice(boundary + 1);
|
|
169
|
+
if (entries.length === 0) {
|
|
170
|
+
return { ids: new Set() };
|
|
171
|
+
}
|
|
172
|
+
const prepared = entries.findIndex(
|
|
173
|
+
(entry) => entry.sourceEntry.id === capacity.firstKeptEntryId,
|
|
174
|
+
);
|
|
175
|
+
const start = prepared >= 0 ? prepared : projectedCut(entries, keep);
|
|
176
|
+
return start < 0 ? { ids: new Set() } : boundedTail(entries, start, maxTokens);
|
|
177
|
+
}
|
|
178
|
+
/** Recent messages and recovery share half the fresh operational budget, leaving work headroom. */
|
|
179
|
+
export function automaticRecovery(
|
|
180
|
+
entries: readonly EntryLike[],
|
|
181
|
+
projected: readonly ProjectionEntry[],
|
|
182
|
+
capacity: RecoveryCapacity,
|
|
183
|
+
ownerQuestionRegistered: boolean,
|
|
184
|
+
): { readonly handoff: string; readonly firstKeptEntryId?: string } | undefined {
|
|
185
|
+
const payload = freshPayloadChars(
|
|
186
|
+
capacity.host,
|
|
187
|
+
capacity.toolTokens,
|
|
188
|
+
capacity.pendingMessages,
|
|
189
|
+
Number.MAX_SAFE_INTEGER,
|
|
190
|
+
);
|
|
191
|
+
const limit = Math.min(MAX_HANDOFF_CHARS, payload);
|
|
192
|
+
if (limit < MIN_PAGE_CHARS) {
|
|
193
|
+
return undefined;
|
|
194
|
+
}
|
|
195
|
+
const retained = retainedTail(projected, capacity, Math.floor((payload - limit) / 4));
|
|
196
|
+
const handoff = buildAutoHandoff(entries, projected, limit, {
|
|
197
|
+
ownerQuestionRegistered,
|
|
198
|
+
retainedEntryIds: retained.ids,
|
|
199
|
+
});
|
|
200
|
+
return handoff.length > limit
|
|
201
|
+
? undefined
|
|
202
|
+
: { handoff, firstKeptEntryId: retained.firstKeptEntryId };
|
|
203
|
+
}
|