pi-context-kit 0.0.0-stage → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/LICENSE +21 -0
- package/README.md +572 -2
- package/extensions/index.ts +509 -0
- package/package.json +49 -4
- package/src/background.ts +104 -0
- package/src/config.ts +228 -0
- package/src/context.ts +108 -0
- package/src/files.ts +54 -0
- package/src/handoff.ts +83 -0
- package/src/maintain.ts +79 -0
- package/src/note.ts +34 -0
- package/src/prune.ts +63 -0
- package/src/recall.ts +212 -0
- package/src/shake.ts +106 -0
- package/src/stale.ts +106 -0
- package/src/tool-calls.ts +18 -0
package/src/recall.ts
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { Worker } from "node:worker_threads";
|
|
2
|
+
import type { ImageContent, TextContent } from "@earendil-works/pi-ai";
|
|
3
|
+
import type { SessionEntry } from "@earendil-works/pi-coding-agent";
|
|
4
|
+
|
|
5
|
+
export interface SearchOptions {
|
|
6
|
+
query: string;
|
|
7
|
+
regex?: boolean;
|
|
8
|
+
offset?: number;
|
|
9
|
+
limit?: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface ReadOptions {
|
|
13
|
+
offset?: number;
|
|
14
|
+
limit?: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const MAX_RESULTS = 50;
|
|
18
|
+
const DEFAULT_RESULTS = 20;
|
|
19
|
+
const MAX_CHARS = 50_000;
|
|
20
|
+
const DEFAULT_CHARS = 20_000;
|
|
21
|
+
const SNIPPET_CHARS = 160;
|
|
22
|
+
const REGEX_DEADLINE_MS = 500;
|
|
23
|
+
|
|
24
|
+
type RegexWorkerResult =
|
|
25
|
+
{ type: "success"; indexes: number[] } | { type: "error"; error: string };
|
|
26
|
+
|
|
27
|
+
/** Searches the raw text of every entry on the branch, including compacted and edited ones. */
|
|
28
|
+
export async function searchEntries(
|
|
29
|
+
branch: readonly SessionEntry[],
|
|
30
|
+
options: SearchOptions,
|
|
31
|
+
signal?: AbortSignal,
|
|
32
|
+
): Promise<string> {
|
|
33
|
+
const indexed = branch.flatMap((entry) => {
|
|
34
|
+
const text = entryText(entry);
|
|
35
|
+
return text ? [{ entry, text }] : [];
|
|
36
|
+
});
|
|
37
|
+
const indexes = options.regex
|
|
38
|
+
? await regexIndexes(
|
|
39
|
+
indexed.map(({ text }) => text),
|
|
40
|
+
options.query,
|
|
41
|
+
signal,
|
|
42
|
+
)
|
|
43
|
+
: indexed.map(({ text }) =>
|
|
44
|
+
text.toLowerCase().indexOf(options.query.toLowerCase()),
|
|
45
|
+
);
|
|
46
|
+
const hits: string[] = [];
|
|
47
|
+
for (let i = indexed.length - 1; i >= 0; i--) {
|
|
48
|
+
const item = indexed[i];
|
|
49
|
+
const index = indexes[i];
|
|
50
|
+
if (item && index !== undefined && index >= 0)
|
|
51
|
+
hits.push(
|
|
52
|
+
`${item.entry.id} ${entryLabel(item.entry)}: ${snippet(item.text, index)}`,
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
if (hits.length === 0) return "No entries match.";
|
|
56
|
+
const offset = clamp(options.offset, 0, hits.length, 0);
|
|
57
|
+
const limit = clamp(options.limit, 1, MAX_RESULTS, DEFAULT_RESULTS);
|
|
58
|
+
const page = hits.slice(offset, offset + limit);
|
|
59
|
+
const end = offset + page.length;
|
|
60
|
+
if (end < hits.length)
|
|
61
|
+
page.push(`[${hits.length} matches. Use offset=${end} for more.]`);
|
|
62
|
+
return page.join("\n");
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Returns one entry's raw text, paged by characters. */
|
|
66
|
+
export function readEntry(
|
|
67
|
+
branch: readonly SessionEntry[],
|
|
68
|
+
entryId: string,
|
|
69
|
+
options: ReadOptions,
|
|
70
|
+
): string {
|
|
71
|
+
const entry = branch.find((candidate) => candidate.id === entryId);
|
|
72
|
+
if (!entry) throw new Error(`No entry ${entryId} on the current branch.`);
|
|
73
|
+
const text = entryText(entry);
|
|
74
|
+
const offset = clamp(options.offset, 0, text.length, 0);
|
|
75
|
+
const limit = clamp(options.limit, 1, MAX_CHARS, DEFAULT_CHARS);
|
|
76
|
+
const end = Math.min(text.length, offset + limit);
|
|
77
|
+
const page = text.slice(offset, end);
|
|
78
|
+
return end < text.length
|
|
79
|
+
? `${page}\n\n[Showing characters ${offset}-${end} of ${text.length}. Use offset=${end} to continue.]`
|
|
80
|
+
: page;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function regexIndexes(
|
|
84
|
+
texts: string[],
|
|
85
|
+
query: string,
|
|
86
|
+
signal: AbortSignal | undefined,
|
|
87
|
+
): Promise<number[]> {
|
|
88
|
+
if (signal?.aborted) return Promise.reject(new Error("Search aborted."));
|
|
89
|
+
const source = `
|
|
90
|
+
const { parentPort, workerData } = require("node:worker_threads");
|
|
91
|
+
try {
|
|
92
|
+
const pattern = new RegExp(workerData.query, "i");
|
|
93
|
+
parentPort.postMessage({ type: "success", indexes: workerData.texts.map((text) => text.search(pattern)) });
|
|
94
|
+
} catch (error) {
|
|
95
|
+
parentPort.postMessage({ type: "error", error: error.message });
|
|
96
|
+
}
|
|
97
|
+
`;
|
|
98
|
+
return new Promise((resolve, reject) => {
|
|
99
|
+
const worker = new Worker(source, {
|
|
100
|
+
eval: true,
|
|
101
|
+
workerData: { query, texts },
|
|
102
|
+
});
|
|
103
|
+
let settled = false;
|
|
104
|
+
const finish = (action: () => void) => {
|
|
105
|
+
if (settled) return;
|
|
106
|
+
settled = true;
|
|
107
|
+
clearTimeout(timer);
|
|
108
|
+
signal?.removeEventListener("abort", onAbort);
|
|
109
|
+
void worker.terminate();
|
|
110
|
+
action();
|
|
111
|
+
};
|
|
112
|
+
const onAbort = () => finish(() => reject(new Error("Search aborted.")));
|
|
113
|
+
const timer = setTimeout(
|
|
114
|
+
() =>
|
|
115
|
+
finish(() =>
|
|
116
|
+
reject(
|
|
117
|
+
new Error(`Regex search exceeded ${REGEX_DEADLINE_MS}ms deadline.`),
|
|
118
|
+
),
|
|
119
|
+
),
|
|
120
|
+
REGEX_DEADLINE_MS,
|
|
121
|
+
);
|
|
122
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
123
|
+
worker.once("message", (result: RegexWorkerResult) => {
|
|
124
|
+
if (result.type === "error")
|
|
125
|
+
finish(() => reject(new Error(`Invalid regex: ${result.error}`)));
|
|
126
|
+
else finish(() => resolve(result.indexes));
|
|
127
|
+
});
|
|
128
|
+
worker.once("error", (error) => finish(() => reject(error)));
|
|
129
|
+
worker.once("exit", (code) => {
|
|
130
|
+
if (code !== 0)
|
|
131
|
+
finish(() =>
|
|
132
|
+
reject(new Error(`Regex worker exited with code ${code}.`)),
|
|
133
|
+
);
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function entryLabel(entry: SessionEntry): string {
|
|
139
|
+
if (entry.type !== "message") return entry.type;
|
|
140
|
+
const { message } = entry;
|
|
141
|
+
return message.role === "toolResult"
|
|
142
|
+
? `toolResult ${message.toolName}`
|
|
143
|
+
: message.role;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function entryText(entry: SessionEntry): string {
|
|
147
|
+
switch (entry.type) {
|
|
148
|
+
case "message":
|
|
149
|
+
return messageText(entry.message);
|
|
150
|
+
case "custom_message":
|
|
151
|
+
return contentText(entry.content);
|
|
152
|
+
case "compaction":
|
|
153
|
+
case "branch_summary":
|
|
154
|
+
return entry.summary;
|
|
155
|
+
default:
|
|
156
|
+
return "";
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function messageText(
|
|
161
|
+
message: Extract<SessionEntry, { type: "message" }>["message"],
|
|
162
|
+
): string {
|
|
163
|
+
switch (message.role) {
|
|
164
|
+
case "user":
|
|
165
|
+
case "custom":
|
|
166
|
+
return contentText(message.content);
|
|
167
|
+
case "assistant":
|
|
168
|
+
return message.content
|
|
169
|
+
.map((block) => {
|
|
170
|
+
if (block.type === "text") return block.text;
|
|
171
|
+
if (block.type === "thinking") return `[thinking] ${block.thinking}`;
|
|
172
|
+
return `[tool call ${block.name} ${JSON.stringify(block.arguments)}]`;
|
|
173
|
+
})
|
|
174
|
+
.join("\n");
|
|
175
|
+
case "toolResult":
|
|
176
|
+
return `[${message.toolName} result${message.isError ? " error" : ""}]\n${contentText(message.content)}`;
|
|
177
|
+
case "system":
|
|
178
|
+
return "";
|
|
179
|
+
default:
|
|
180
|
+
return JSON.stringify(message);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function contentText(
|
|
185
|
+
content: string | readonly (TextContent | ImageContent)[],
|
|
186
|
+
): string {
|
|
187
|
+
if (typeof content === "string") return content;
|
|
188
|
+
return content
|
|
189
|
+
.map((block) =>
|
|
190
|
+
block.type === "text" ? block.text : `[image ${block.mimeType}]`,
|
|
191
|
+
)
|
|
192
|
+
.join("\n");
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function snippet(text: string, index: number): string {
|
|
196
|
+
const start = Math.max(0, index - SNIPPET_CHARS / 2);
|
|
197
|
+
const flat = text
|
|
198
|
+
.slice(start, start + SNIPPET_CHARS)
|
|
199
|
+
.replace(/\s+/g, " ")
|
|
200
|
+
.trim();
|
|
201
|
+
return `${start > 0 ? "…" : ""}${flat}${start + SNIPPET_CHARS < text.length ? "…" : ""}`;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function clamp(
|
|
205
|
+
value: number | undefined,
|
|
206
|
+
min: number,
|
|
207
|
+
max: number,
|
|
208
|
+
fallback: number,
|
|
209
|
+
): number {
|
|
210
|
+
if (value === undefined || !Number.isFinite(value)) return fallback;
|
|
211
|
+
return Math.min(max, Math.max(min, Math.floor(value)));
|
|
212
|
+
}
|
package/src/shake.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { TextContent } from "@earendil-works/pi-ai";
|
|
2
|
+
import type {
|
|
3
|
+
ContextEditableContent,
|
|
4
|
+
ProjectedSessionEntry,
|
|
5
|
+
} from "@earendil-works/pi-coding-agent";
|
|
6
|
+
import {
|
|
7
|
+
type Edit,
|
|
8
|
+
type EditableMessage,
|
|
9
|
+
editableMessage,
|
|
10
|
+
messageTokens,
|
|
11
|
+
withContent,
|
|
12
|
+
} from "./context.ts";
|
|
13
|
+
|
|
14
|
+
export interface ShakeOptions {
|
|
15
|
+
/** Newest context tokens that are never shaken. */
|
|
16
|
+
protectTokens: number;
|
|
17
|
+
/** Shake nothing unless the total savings reach this many tokens. */
|
|
18
|
+
minSavings: number;
|
|
19
|
+
/** Fenced or XML blocks smaller than this stay. */
|
|
20
|
+
blockMinTokens: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** A fenced code block, or an XML-like element with a matching closing tag. */
|
|
24
|
+
const BLOCK =
|
|
25
|
+
/(`{3,}|~{3,})[^\n]*\n[\s\S]*?\n[ \t]*\1|<([A-Za-z][\w.:-]*)(?:\s[^<>]*)?>[\s\S]*?<\/\2>/g;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Elides large blocks and images from older messages. Tool calls and thinking stay intact,
|
|
29
|
+
* so every tool call keeps its result and signed reasoning stays valid.
|
|
30
|
+
*/
|
|
31
|
+
export function shake(
|
|
32
|
+
entries: readonly ProjectedSessionEntry[],
|
|
33
|
+
options: ShakeOptions,
|
|
34
|
+
): Edit[] {
|
|
35
|
+
const edits: Edit[] = [];
|
|
36
|
+
let seen = 0;
|
|
37
|
+
let savings = 0;
|
|
38
|
+
|
|
39
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
40
|
+
const entry = entries[i];
|
|
41
|
+
if (!entry) continue;
|
|
42
|
+
const isProtected = seen < options.protectTokens;
|
|
43
|
+
for (const message of entry.messages) {
|
|
44
|
+
if (message.role !== "system") seen += messageTokens(message);
|
|
45
|
+
}
|
|
46
|
+
const message = editableMessage(entry);
|
|
47
|
+
if (isProtected || !message || message.role === "user") continue;
|
|
48
|
+
|
|
49
|
+
const content = shakeContent(message, entry.sourceEntry.id, options);
|
|
50
|
+
if (content === undefined) continue;
|
|
51
|
+
const saved =
|
|
52
|
+
messageTokens(message) - messageTokens(withContent(message, content));
|
|
53
|
+
if (saved <= 0) continue;
|
|
54
|
+
savings += saved;
|
|
55
|
+
edits.push({ targetId: entry.sourceEntry.id, content });
|
|
56
|
+
}
|
|
57
|
+
return savings >= options.minSavings && edits.length > 0
|
|
58
|
+
? edits.reverse()
|
|
59
|
+
: [];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function shakeContent(
|
|
63
|
+
message: EditableMessage,
|
|
64
|
+
id: string,
|
|
65
|
+
options: ShakeOptions,
|
|
66
|
+
): ContextEditableContent | undefined {
|
|
67
|
+
const text = (value: string) =>
|
|
68
|
+
elideBlocks(value, id, options.blockMinTokens);
|
|
69
|
+
|
|
70
|
+
if (typeof message.content === "string") {
|
|
71
|
+
const next = text(message.content);
|
|
72
|
+
return next === message.content ? undefined : next;
|
|
73
|
+
}
|
|
74
|
+
let changed = false;
|
|
75
|
+
const next = message.content.map((block) => {
|
|
76
|
+
if (block.type === "text") {
|
|
77
|
+
const replaced = text(block.text);
|
|
78
|
+
if (replaced === block.text) return block;
|
|
79
|
+
changed = true;
|
|
80
|
+
return { ...block, text: replaced };
|
|
81
|
+
}
|
|
82
|
+
if (block.type === "image") {
|
|
83
|
+
changed = true;
|
|
84
|
+
return imageNotice(id);
|
|
85
|
+
}
|
|
86
|
+
return block;
|
|
87
|
+
});
|
|
88
|
+
// Blocks keep their kind, and images only occur where text is also allowed.
|
|
89
|
+
return changed ? (next as ContextEditableContent) : undefined;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function elideBlocks(text: string, id: string, minTokens: number): string {
|
|
93
|
+
return text.replace(BLOCK, (block) => {
|
|
94
|
+
const tokens = Math.ceil(block.length / 4);
|
|
95
|
+
return tokens < minTokens
|
|
96
|
+
? block
|
|
97
|
+
: `[Elided block: about ${tokens} tokens. Call recall with entryId "${id}" to read it.]`;
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function imageNotice(id: string): TextContent {
|
|
102
|
+
return {
|
|
103
|
+
type: "text",
|
|
104
|
+
text: `[Image elided. Call recall with entryId "${id}" to see its metadata.]`,
|
|
105
|
+
};
|
|
106
|
+
}
|
package/src/stale.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { ToolResultMessage } from "@earendil-works/pi-ai";
|
|
2
|
+
import type { ProjectedSessionEntry } from "@earendil-works/pi-coding-agent";
|
|
3
|
+
import {
|
|
4
|
+
type Edit,
|
|
5
|
+
editableMessage,
|
|
6
|
+
messageTokens,
|
|
7
|
+
suffixTokens,
|
|
8
|
+
withContent,
|
|
9
|
+
} from "./context.ts";
|
|
10
|
+
import { toolCallsById } from "./tool-calls.ts";
|
|
11
|
+
|
|
12
|
+
export const SUPERSEDED_NOTICE = "[Superseded by a newer read of this file]";
|
|
13
|
+
|
|
14
|
+
interface ReadKey {
|
|
15
|
+
path: string;
|
|
16
|
+
/** Undefined for a read of the whole file. */
|
|
17
|
+
range: string | undefined;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
interface NewerReads {
|
|
21
|
+
anyRead: boolean;
|
|
22
|
+
fullSuccess: boolean;
|
|
23
|
+
successRanges: Set<string>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Replaces read results that a newer read of the same file makes redundant.
|
|
28
|
+
* A newer full read covers every older read; a newer ranged read covers only the same range;
|
|
29
|
+
* a failed read covers only older failed reads. A candidate and its newer messages must fit
|
|
30
|
+
* within `suffixLimit` tokens.
|
|
31
|
+
*/
|
|
32
|
+
export function supersededReads(
|
|
33
|
+
entries: readonly ProjectedSessionEntry[],
|
|
34
|
+
edited: ReadonlySet<string>,
|
|
35
|
+
suffixLimit: number,
|
|
36
|
+
): Edit[] {
|
|
37
|
+
const calls = toolCallsById(entries);
|
|
38
|
+
const suffix = suffixTokens(entries);
|
|
39
|
+
const newer = new Map<string, NewerReads>();
|
|
40
|
+
const edits: Edit[] = [];
|
|
41
|
+
|
|
42
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
43
|
+
const entry = entries[i];
|
|
44
|
+
const message = entry && editableMessage(entry);
|
|
45
|
+
if (!entry || message?.role !== "toolResult" || message.toolName !== "read")
|
|
46
|
+
continue;
|
|
47
|
+
const key = readKey(calls.get(message.toolCallId)?.arguments);
|
|
48
|
+
if (!key) continue;
|
|
49
|
+
|
|
50
|
+
const seen = newer.get(key.path) ?? {
|
|
51
|
+
anyRead: false,
|
|
52
|
+
fullSuccess: false,
|
|
53
|
+
successRanges: new Set<string>(),
|
|
54
|
+
};
|
|
55
|
+
const readCoversOutput =
|
|
56
|
+
!message.isError && !isTruncatedRead(message.content);
|
|
57
|
+
const superseded = message.isError
|
|
58
|
+
? seen.anyRead
|
|
59
|
+
: seen.fullSuccess ||
|
|
60
|
+
(key.range !== undefined && seen.successRanges.has(key.range));
|
|
61
|
+
|
|
62
|
+
if (
|
|
63
|
+
superseded &&
|
|
64
|
+
!edited.has(entry.sourceEntry.id) &&
|
|
65
|
+
(suffix[i] ?? Number.POSITIVE_INFINITY) <= suffixLimit
|
|
66
|
+
) {
|
|
67
|
+
const content = [{ type: "text" as const, text: SUPERSEDED_NOTICE }];
|
|
68
|
+
if (
|
|
69
|
+
messageTokens(withContent(message, content)) < messageTokens(message)
|
|
70
|
+
) {
|
|
71
|
+
edits.push({ targetId: entry.sourceEntry.id, content });
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
seen.anyRead = true;
|
|
76
|
+
if (readCoversOutput) {
|
|
77
|
+
if (key.range === undefined) seen.fullSuccess = true;
|
|
78
|
+
else seen.successRanges.add(key.range);
|
|
79
|
+
}
|
|
80
|
+
newer.set(key.path, seen);
|
|
81
|
+
}
|
|
82
|
+
return edits.reverse();
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function isTruncatedRead(content: ToolResultMessage["content"]): boolean {
|
|
86
|
+
return content.some(
|
|
87
|
+
(block) =>
|
|
88
|
+
block.type === "text" &&
|
|
89
|
+
/\[(?:Showing lines |\d+ more lines in file|Line \d+ is )/.test(
|
|
90
|
+
block.text,
|
|
91
|
+
),
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function readKey(
|
|
96
|
+
args: Record<string, unknown> | undefined,
|
|
97
|
+
): ReadKey | undefined {
|
|
98
|
+
const path = args?.path;
|
|
99
|
+
if (typeof path !== "string" || path.length === 0) return undefined;
|
|
100
|
+
const { offset, limit } = args ?? {};
|
|
101
|
+
const ranged = offset !== undefined || limit !== undefined;
|
|
102
|
+
return {
|
|
103
|
+
path,
|
|
104
|
+
range: ranged ? `${String(offset ?? 1)}:${String(limit ?? "")}` : undefined,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ToolCall } from "@earendil-works/pi-ai";
|
|
2
|
+
import type { ProjectedSessionEntry } from "@earendil-works/pi-coding-agent";
|
|
3
|
+
|
|
4
|
+
/** Tool calls in the projected context, by tool call ID. */
|
|
5
|
+
export function toolCallsById(
|
|
6
|
+
entries: readonly ProjectedSessionEntry[],
|
|
7
|
+
): Map<string, ToolCall> {
|
|
8
|
+
const calls = new Map<string, ToolCall>();
|
|
9
|
+
for (const entry of entries) {
|
|
10
|
+
for (const message of entry.messages) {
|
|
11
|
+
if (message.role !== "assistant") continue;
|
|
12
|
+
for (const block of message.content) {
|
|
13
|
+
if (block.type === "toolCall") calls.set(block.id, block);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
return calls;
|
|
18
|
+
}
|