pi-memoria 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +672 -0
- package/index.ts +260 -0
- package/package.json +71 -0
- package/skills/memoria/SKILL.md +65 -0
- package/skills/memoria-sessions/SKILL.md +66 -0
- package/src/commands.ts +472 -0
- package/src/config.ts +196 -0
- package/src/frontmatter.ts +318 -0
- package/src/index-engine.ts +1044 -0
- package/src/learn.ts +299 -0
- package/src/recall.ts +172 -0
- package/src/rerank.ts +85 -0
- package/src/runtime.ts +1633 -0
- package/src/search.ts +499 -0
- package/src/sessions.ts +592 -0
- package/src/similarity.ts +374 -0
- package/src/store.ts +1643 -0
- package/src/synonyms.ts +105 -0
- package/src/timeexpr.ts +145 -0
- package/src/tokenize.ts +295 -0
- package/src/tools.ts +935 -0
- package/src/transfer.ts +180 -0
- package/src/types.ts +533 -0
- package/src/usage.ts +174 -0
- package/src/util.ts +138 -0
package/src/sessions.ts
ADDED
|
@@ -0,0 +1,592 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Search pi's saved session transcripts.
|
|
3
|
+
*
|
|
4
|
+
* Curated memory (the library) is the fast path, but it only holds what the
|
|
5
|
+
* agent chose to remember. Sometimes the answer to "did we discuss this?" is in
|
|
6
|
+
* an earlier conversation and nowhere else — so this module scans the JSONL
|
|
7
|
+
* transcripts pi writes under `<agent dir>/sessions/` (and `sessions-archive/`
|
|
8
|
+
* if present) and returns bounded, evidence-linked excerpts.
|
|
9
|
+
*
|
|
10
|
+
* Design constraints, in order:
|
|
11
|
+
*
|
|
12
|
+
* 1. **Exhaustive on disk.** No pre-built index that can go stale: every
|
|
13
|
+
* readable transcript is streamed, newest first, so a session saved a minute
|
|
14
|
+
* ago is searchable immediately.
|
|
15
|
+
* 2. **Bounded.** Only user/assistant text is scanned by default; thinking
|
|
16
|
+
* blocks, image payloads and tool plumbing are skipped. Excerpts, results and
|
|
17
|
+
* per-message text are capped, and a wall-clock budget can stop a scan
|
|
18
|
+
* (reporting `partial`).
|
|
19
|
+
* 3. **Cheap on repeat.** Parsed messages are cached by path with mtime+size
|
|
20
|
+
* validation under a byte budget, so the second search in a session is
|
|
21
|
+
* in-memory. Substring prefiltering keeps the scoring pass linear.
|
|
22
|
+
* 4. **Read-only.** Transcripts are never written, and `readWindow` refuses
|
|
23
|
+
* paths outside the configured roots.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { existsSync } from "node:fs";
|
|
27
|
+
import { readdir, readFile, stat } from "node:fs/promises";
|
|
28
|
+
import { basename, join, relative, resolve, sep } from "node:path";
|
|
29
|
+
import { isCjkChar, stem, tokenize } from "./tokenize.ts";
|
|
30
|
+
import { oneLine, truncateChars } from "./util.ts";
|
|
31
|
+
import type { SessionFile, SessionHit, SessionMessage, SessionReadResult, SessionScanStats, SessionSearchOptions, SessionSearchResult, SessionWindowMessage } from "./types.ts";
|
|
32
|
+
|
|
33
|
+
/** Hard cap on a single extracted message, to keep the cache useful. */
|
|
34
|
+
export const MAX_SESSION_MESSAGE_CHARS = 4000;
|
|
35
|
+
/** Directories deeper than this under a root are ignored. */
|
|
36
|
+
const MAX_WALK_DEPTH = 4;
|
|
37
|
+
/** Files larger than this are skipped rather than read into memory. */
|
|
38
|
+
const MAX_FILE_BYTES = 64 * 1024 * 1024;
|
|
39
|
+
/**
|
|
40
|
+
* Upper bound on listed transcript files. Reaching it marks the scan `partial`:
|
|
41
|
+
* a listing this large is a pathological corpus (years of heavy use), and the
|
|
42
|
+
* alternative is an unbounded directory walk inside a prompt.
|
|
43
|
+
*/
|
|
44
|
+
const MAX_SESSION_FILES = 20_000;
|
|
45
|
+
|
|
46
|
+
interface Candidate {
|
|
47
|
+
file: SessionFile;
|
|
48
|
+
message: SessionMessage;
|
|
49
|
+
project: string;
|
|
50
|
+
projectName: string;
|
|
51
|
+
score: number;
|
|
52
|
+
matched: string[];
|
|
53
|
+
exact: boolean;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Loaded from the transcript header or derived from the file name. */
|
|
57
|
+
function deriveProject(path: string, cwd?: string): { project: string; projectName: string } {
|
|
58
|
+
if (cwd) return { project: cwd, projectName: basename(cwd) || cwd };
|
|
59
|
+
const dir = basename(join(path, ".."));
|
|
60
|
+
const decoded = dir.replace(/^--+|-+$/g, "").replace(/-/g, "/");
|
|
61
|
+
return { project: decoded, projectName: basename(decoded) || decoded };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** `<timestamp>_<uuid>.jsonl` → epoch ms, falling back to the file mtime. */
|
|
65
|
+
function startedAtFromName(path: string, fallback: number): number {
|
|
66
|
+
const match = basename(path).match(/^(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z/);
|
|
67
|
+
if (!match) return fallback;
|
|
68
|
+
const parsed = Date.parse(`${match[1]}-${match[2]}-${match[3]}T${match[4]}:${match[5]}:${match[6]}.${match[7]}Z`);
|
|
69
|
+
return Number.isFinite(parsed) ? parsed : fallback;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Normalize text for phrase matching: NFKC, lowercase, punctuation to spaces. */
|
|
73
|
+
export function normalizeForSessionMatch(text: string): string {
|
|
74
|
+
return text
|
|
75
|
+
.normalize("NFKC")
|
|
76
|
+
.toLowerCase()
|
|
77
|
+
.replace(/[^\p{L}\p{N}]+/gu, " ")
|
|
78
|
+
.trim();
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Extract the searchable text of one message, honouring the include flags. */
|
|
82
|
+
export function extractMessageText(role: string, content: unknown, includeTools: boolean): string {
|
|
83
|
+
const parts: string[] = [];
|
|
84
|
+
const push = (value: unknown): void => {
|
|
85
|
+
if (typeof value === "string" && value.trim()) parts.push(value);
|
|
86
|
+
};
|
|
87
|
+
const walkBlocks = (blocks: unknown, blockRole: "user" | "assistant" | "tool"): void => {
|
|
88
|
+
if (typeof blocks === "string") {
|
|
89
|
+
push(blocks);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
if (!Array.isArray(blocks)) return;
|
|
93
|
+
for (const block of blocks) {
|
|
94
|
+
if (!block || typeof block !== "object") continue;
|
|
95
|
+
const typed = block as { type?: string; text?: string; arguments?: unknown; input?: unknown };
|
|
96
|
+
if (typed.type === "text") push(typed.text);
|
|
97
|
+
// Thinking blocks are private reasoning, never evidence.
|
|
98
|
+
if (!includeTools) continue;
|
|
99
|
+
if ((blockRole === "assistant" || blockRole === "tool") && (typed.type === "toolCall" || typed.type === "tool_call")) {
|
|
100
|
+
const args = typed.arguments ?? typed.input;
|
|
101
|
+
const rendered = typeof args === "string" ? args : args === undefined ? "" : JSON.stringify(args);
|
|
102
|
+
if (rendered) push(rendered);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
if (role === "user" || role === "assistant") walkBlocks(content, role);
|
|
107
|
+
else if (role === "toolResult" && includeTools) walkBlocks(content, "tool");
|
|
108
|
+
const joined = parts.join("\n").replace(/\s+/g, " ").trim();
|
|
109
|
+
return truncateChars(joined, MAX_SESSION_MESSAGE_CHARS);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Generic text harvest for `custom` payloads (web results, extension data). */
|
|
113
|
+
function harvestCustomText(value: unknown, depth = 0): string[] {
|
|
114
|
+
if (depth > 4) return [];
|
|
115
|
+
if (typeof value === "string") return value.length > 2 ? [value] : [];
|
|
116
|
+
if (Array.isArray(value)) return value.flatMap((entry) => harvestCustomText(entry, depth + 1));
|
|
117
|
+
if (value && typeof value === "object") {
|
|
118
|
+
const out: string[] = [];
|
|
119
|
+
for (const [key, entry] of Object.entries(value as Record<string, unknown>)) {
|
|
120
|
+
if (["data", "bytes", "base64", "signature", "image", "thinking"].includes(key)) continue;
|
|
121
|
+
out.push(...harvestCustomText(entry, depth + 1));
|
|
122
|
+
}
|
|
123
|
+
return out;
|
|
124
|
+
}
|
|
125
|
+
return [];
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Case-insensitive, punctuation-tolerant regex for a normalized phrase. */
|
|
129
|
+
function phraseRegex(phrase: string): RegExp {
|
|
130
|
+
const parts = phrase.split(" ").filter(Boolean).map((word) => word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
131
|
+
return new RegExp(parts.join("[^\\p{L}\\p{N}]+"), "iu");
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Where a phrase first appears in the original text, or -1. */
|
|
135
|
+
export function findPhrase(text: string, phrases: string[], terms: string[]): number {
|
|
136
|
+
let best = -1;
|
|
137
|
+
for (const phrase of phrases) {
|
|
138
|
+
const match = phraseRegex(phrase).exec(text);
|
|
139
|
+
if (match && (best === -1 || match.index < best)) best = match.index;
|
|
140
|
+
}
|
|
141
|
+
if (best !== -1) return best;
|
|
142
|
+
for (const term of terms) {
|
|
143
|
+
const match = phraseRegex(term).exec(text);
|
|
144
|
+
if (match && (best === -1 || match.index < best)) best = match.index;
|
|
145
|
+
}
|
|
146
|
+
return best;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Build a bounded excerpt around the first match. Falls back to the head of the
|
|
151
|
+
* message when the match cannot be located in the original text.
|
|
152
|
+
*/
|
|
153
|
+
export function buildExcerpt(text: string, phrases: string[], terms: string[], maxChars: number): string {
|
|
154
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
155
|
+
if (flat.length <= maxChars) return flat;
|
|
156
|
+
const at = findPhrase(flat, phrases, terms);
|
|
157
|
+
if (at === -1) return `${truncateChars(flat, Math.max(1, maxChars - 1)).trimEnd()}…`;
|
|
158
|
+
const half = Math.floor(maxChars / 2);
|
|
159
|
+
let start = Math.max(0, at - Math.floor(half / 2));
|
|
160
|
+
if (start > 0) {
|
|
161
|
+
const space = flat.indexOf(" ", start);
|
|
162
|
+
if (space !== -1 && space - start < 30) start = space + 1;
|
|
163
|
+
}
|
|
164
|
+
let end = Math.min(flat.length, start + maxChars);
|
|
165
|
+
if (end < flat.length) {
|
|
166
|
+
const space = flat.lastIndexOf(" ", end);
|
|
167
|
+
if (space > start) end = space;
|
|
168
|
+
}
|
|
169
|
+
let excerpt = flat.slice(start, end).trim();
|
|
170
|
+
if (start > 0) excerpt = `…${excerpt}`;
|
|
171
|
+
if (end < flat.length) excerpt = `${excerpt.replace(/[,;:.\s]+$/, "")}…`;
|
|
172
|
+
return excerpt;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Cache key for a transcript parse. Two variants are kept per file: the cheap
|
|
177
|
+
* user/assistant-only parse (the default) and the full parse including tool
|
|
178
|
+
* plumbing, so an `includeTools` search does not force every later search to
|
|
179
|
+
* pay for tool extraction.
|
|
180
|
+
*/
|
|
181
|
+
function cacheKey(path: string, includeTools: boolean): string {
|
|
182
|
+
return `${path}::${includeTools ? "all" : "core"}`;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
export interface SessionStoreOptions {
|
|
186
|
+
/** Transcript roots to scan, in priority order. */
|
|
187
|
+
roots: string[];
|
|
188
|
+
/** In-memory budget for parsed messages (accounted as twice the file size). */
|
|
189
|
+
cacheBytes?: number;
|
|
190
|
+
/** Default excerpt size. */
|
|
191
|
+
excerptChars?: number;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
interface CachedFile {
|
|
195
|
+
path: string;
|
|
196
|
+
mtimeMs: number;
|
|
197
|
+
size: number;
|
|
198
|
+
/** Raw file length, for reporting. */
|
|
199
|
+
rawBytes: number;
|
|
200
|
+
/** Accounted cache cost. */
|
|
201
|
+
cost: number;
|
|
202
|
+
messages: SessionMessage[];
|
|
203
|
+
header: { cwd?: string; startedAt: number };
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
export class SessionStore {
|
|
207
|
+
readonly roots: string[];
|
|
208
|
+
private cache = new Map<string, CachedFile>();
|
|
209
|
+
private cacheBytes = 0;
|
|
210
|
+
private cacheBudget: number;
|
|
211
|
+
private excerptChars: number;
|
|
212
|
+
|
|
213
|
+
constructor(options: SessionStoreOptions) {
|
|
214
|
+
this.roots = options.roots;
|
|
215
|
+
this.cacheBudget = Math.max(0, options.cacheBytes ?? 32 * 1024 * 1024);
|
|
216
|
+
this.excerptChars = Math.max(80, options.excerptChars ?? 400);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** True when at least one root exists on disk. */
|
|
220
|
+
hasRoots(): boolean {
|
|
221
|
+
return this.roots.some((root) => existsSync(root));
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Transcript files, newest session first.
|
|
226
|
+
*
|
|
227
|
+
* The directory is re-walked on every call: `stat` is cheap, and a cached
|
|
228
|
+
* listing would hand a stale mtime/size to the parse cache, hiding a session
|
|
229
|
+
* that was written seconds ago. Parsing — not walking — is what the cache is
|
|
230
|
+
* for.
|
|
231
|
+
*
|
|
232
|
+
* The walk is itself bounded: `deadline` (a `performance.now()` value) and
|
|
233
|
+
* `maxFiles` stop it, and `truncated` reports that the listing is not the
|
|
234
|
+
* whole corpus. That matters because a ten-thousand-session archive would
|
|
235
|
+
* otherwise stall an automatic recall before the parse budget is even
|
|
236
|
+
* consulted.
|
|
237
|
+
*/
|
|
238
|
+
async listFiles(options: { deadline?: number; maxFiles?: number } = {}): Promise<{ files: SessionFile[]; truncated: boolean }> {
|
|
239
|
+
const state = { files: [] as SessionFile[], truncated: false };
|
|
240
|
+
const deadline = options.deadline ?? Number.POSITIVE_INFINITY;
|
|
241
|
+
const maxFiles = Math.max(1, options.maxFiles ?? MAX_SESSION_FILES);
|
|
242
|
+
for (const root of this.roots) {
|
|
243
|
+
if (!existsSync(root)) continue;
|
|
244
|
+
await this.walk(root, root, 0, state, () => performance.now() > deadline || state.files.length >= maxFiles);
|
|
245
|
+
}
|
|
246
|
+
state.files.sort((a, b) => b.startedAt - a.startedAt);
|
|
247
|
+
return { files: state.files, truncated: state.truncated };
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
private async walk(root: string, dir: string, depth: number, state: { files: SessionFile[]; truncated: boolean }, overBudget: () => boolean): Promise<void> {
|
|
251
|
+
if (depth > MAX_WALK_DEPTH || state.truncated) return;
|
|
252
|
+
let entries;
|
|
253
|
+
try {
|
|
254
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
255
|
+
} catch {
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
// Transcript names start with an ISO timestamp, so name-descending is
|
|
259
|
+
// newest-first: a truncated listing keeps the most recent sessions.
|
|
260
|
+
entries.sort((a, b) => (a.name < b.name ? 1 : a.name > b.name ? -1 : 0));
|
|
261
|
+
for (const entry of entries) {
|
|
262
|
+
if (overBudget()) {
|
|
263
|
+
state.truncated = true;
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
const path = join(dir, entry.name);
|
|
267
|
+
if (entry.isSymbolicLink()) continue;
|
|
268
|
+
if (entry.isDirectory()) {
|
|
269
|
+
if (entry.name.startsWith(".")) continue;
|
|
270
|
+
await this.walk(root, path, depth + 1, state, overBudget);
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
if (!entry.isFile() || !entry.name.endsWith(".jsonl")) continue;
|
|
274
|
+
let stats;
|
|
275
|
+
try {
|
|
276
|
+
stats = await stat(path);
|
|
277
|
+
} catch {
|
|
278
|
+
continue;
|
|
279
|
+
}
|
|
280
|
+
if (stats.size === 0 || stats.size > MAX_FILE_BYTES) continue;
|
|
281
|
+
const cachedCore = this.cache.get(cacheKey(path, false));
|
|
282
|
+
const cachedAll = this.cache.get(cacheKey(path, true));
|
|
283
|
+
const fresh = (entry: CachedFile | undefined): CachedFile | undefined => (entry && entry.mtimeMs === stats.mtimeMs && entry.size === stats.size ? entry : undefined);
|
|
284
|
+
const header = (fresh(cachedCore) ?? fresh(cachedAll))?.header;
|
|
285
|
+
const { project, projectName } = deriveProject(path, header?.cwd);
|
|
286
|
+
state.files.push({
|
|
287
|
+
path,
|
|
288
|
+
root,
|
|
289
|
+
relPath: relative(root, path).split(sep).join("/"),
|
|
290
|
+
project,
|
|
291
|
+
projectName,
|
|
292
|
+
startedAt: header?.startedAt ?? startedAtFromName(path, stats.mtimeMs),
|
|
293
|
+
mtimeMs: stats.mtimeMs,
|
|
294
|
+
size: stats.size,
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** Parse a transcript, reusing the cache when mtime and size are unchanged. */
|
|
300
|
+
async messagesFor(file: SessionFile, includeTools: boolean): Promise<{ messages: SessionMessage[]; cached: boolean; skipped: number; rawBytes: number; cwd?: string }> {
|
|
301
|
+
const key = cacheKey(file.path, includeTools);
|
|
302
|
+
// A full parse can serve the default view, but the default view cannot
|
|
303
|
+
// serve an includeTools search (it never extracted the tool text).
|
|
304
|
+
const cached = includeTools
|
|
305
|
+
? this.cache.get(cacheKey(file.path, true))
|
|
306
|
+
: (this.cache.get(cacheKey(file.path, false)) ?? this.cache.get(cacheKey(file.path, true)));
|
|
307
|
+
if (cached && cached.mtimeMs === file.mtimeMs && cached.size === file.size) {
|
|
308
|
+
// Touch for LRU ordering.
|
|
309
|
+
this.cache.delete(key);
|
|
310
|
+
this.cache.set(key, cached);
|
|
311
|
+
const visible = includeTools ? cached.messages : cached.messages.filter((message) => message.role !== "tool" && message.role !== "summary");
|
|
312
|
+
return { messages: visible, cached: true, skipped: 0, rawBytes: cached.rawBytes, cwd: cached.header.cwd };
|
|
313
|
+
}
|
|
314
|
+
let raw: string;
|
|
315
|
+
try {
|
|
316
|
+
raw = await readFile(file.path, "utf8");
|
|
317
|
+
} catch {
|
|
318
|
+
return { messages: [], cached: false, skipped: 1, rawBytes: 0 };
|
|
319
|
+
}
|
|
320
|
+
const messages: SessionMessage[] = [];
|
|
321
|
+
let skipped = 0;
|
|
322
|
+
let cwd: string | undefined;
|
|
323
|
+
let headerStarted = 0;
|
|
324
|
+
for (const [index, line] of raw.split(/\r?\n/).entries()) {
|
|
325
|
+
if (!line || line.charCodeAt(0) !== 123 /* "{" */) continue;
|
|
326
|
+
let record: Record<string, unknown>;
|
|
327
|
+
try {
|
|
328
|
+
record = JSON.parse(line) as Record<string, unknown>;
|
|
329
|
+
} catch {
|
|
330
|
+
skipped += 1;
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
const type = record.type;
|
|
334
|
+
if (type === "session") {
|
|
335
|
+
if (typeof record.cwd === "string") cwd = record.cwd;
|
|
336
|
+
if (typeof record.timestamp === "string") headerStarted = Date.parse(record.timestamp) || 0;
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
const entryTime = typeof record.timestamp === "number" ? record.timestamp : typeof record.timestamp === "string" ? Date.parse(record.timestamp) || 0 : 0;
|
|
340
|
+
if (type === "message") {
|
|
341
|
+
const message = record.message as { role?: string; content?: unknown; timestamp?: number } | undefined;
|
|
342
|
+
if (!message || typeof message.role !== "string") continue;
|
|
343
|
+
// The system prompt is large and is not evidence.
|
|
344
|
+
if (message.role === "system") continue;
|
|
345
|
+
const role = message.role === "user" || message.role === "assistant" ? message.role : message.role === "toolResult" ? "tool" : undefined;
|
|
346
|
+
if (!role) continue;
|
|
347
|
+
if (role === "tool" && !includeTools) continue;
|
|
348
|
+
const text = extractMessageText(message.role, message.content, includeTools);
|
|
349
|
+
if (!text) continue;
|
|
350
|
+
messages.push({
|
|
351
|
+
line: index + 1,
|
|
352
|
+
role,
|
|
353
|
+
timestamp: typeof message.timestamp === "number" && message.timestamp > 0 ? message.timestamp : entryTime,
|
|
354
|
+
text,
|
|
355
|
+
normalized: "",
|
|
356
|
+
});
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
if (!includeTools) continue;
|
|
360
|
+
if (type === "compaction" && typeof record.summary === "string" && record.summary.trim()) {
|
|
361
|
+
const text = truncateChars(record.summary.replace(/\s+/g, " ").trim(), MAX_SESSION_MESSAGE_CHARS);
|
|
362
|
+
messages.push({ line: index + 1, role: "summary", timestamp: entryTime, text, normalized: "" });
|
|
363
|
+
continue;
|
|
364
|
+
}
|
|
365
|
+
if (type === "custom") {
|
|
366
|
+
const text = truncateChars(oneLine(harvestCustomText(record.data).join(" "), MAX_SESSION_MESSAGE_CHARS), MAX_SESSION_MESSAGE_CHARS);
|
|
367
|
+
if (text) messages.push({ line: index + 1, role: "tool", timestamp: entryTime, text, normalized: "" });
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
// `custom_message` is injected context (e.g. a recall block), not a turn.
|
|
371
|
+
}
|
|
372
|
+
const parsed: CachedFile = {
|
|
373
|
+
path: file.path,
|
|
374
|
+
mtimeMs: file.mtimeMs,
|
|
375
|
+
size: file.size,
|
|
376
|
+
rawBytes: raw.length,
|
|
377
|
+
// Extracted text plus its normalized copy, plus object overhead.
|
|
378
|
+
cost: raw.length * 2,
|
|
379
|
+
messages,
|
|
380
|
+
header: { cwd, startedAt: headerStarted || file.startedAt },
|
|
381
|
+
};
|
|
382
|
+
this.put(parsed, includeTools);
|
|
383
|
+
const visible = includeTools ? messages : messages.filter((message) => message.role !== "tool" && message.role !== "summary");
|
|
384
|
+
return { messages: visible, cached: false, skipped, rawBytes: raw.length, cwd };
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
private put(entry: CachedFile, includeTools: boolean): void {
|
|
388
|
+
const key = cacheKey(entry.path, includeTools);
|
|
389
|
+
const existing = this.cache.get(key);
|
|
390
|
+
if (existing) this.cacheBytes -= existing.cost;
|
|
391
|
+
this.cache.set(key, entry);
|
|
392
|
+
this.cacheBytes += entry.cost;
|
|
393
|
+
while (this.cacheBytes > this.cacheBudget && this.cache.size > 0) {
|
|
394
|
+
const oldest = this.cache.keys().next();
|
|
395
|
+
if (oldest.done) break;
|
|
396
|
+
const evicted = this.cache.get(oldest.value)!;
|
|
397
|
+
this.cache.delete(oldest.value);
|
|
398
|
+
this.cacheBytes -= evicted.cost;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** Drop the parsed-message cache. */
|
|
403
|
+
clear(): void {
|
|
404
|
+
this.cache.clear();
|
|
405
|
+
this.cacheBytes = 0;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
get cachedFileCount(): number {
|
|
409
|
+
return this.cache.size;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/**
|
|
413
|
+
* Search the transcripts.
|
|
414
|
+
*
|
|
415
|
+
* Query terms are matched with the memory tokenizer (identifiers, stems, CJK
|
|
416
|
+
* unigrams) after a cheap substring prefilter, so the full corpus is examined
|
|
417
|
+
* without tokenizing every message. Results are ranked by exact-phrase hits,
|
|
418
|
+
* term coverage and authorship, with recency as the tie-break.
|
|
419
|
+
*/
|
|
420
|
+
async search(query: string, options: SessionSearchOptions = {}): Promise<SessionSearchResult> {
|
|
421
|
+
const started = performance.now();
|
|
422
|
+
const limit = Math.max(1, Math.min(50, options.limit ?? 8));
|
|
423
|
+
const budgetMs = Math.max(0, options.budgetMs ?? 1500);
|
|
424
|
+
const excerptChars = Math.max(80, options.excerptChars ?? this.excerptChars);
|
|
425
|
+
const includeTools = options.includeTools === true;
|
|
426
|
+
const stats: SessionScanStats = { roots: this.roots.filter((root) => existsSync(root)), files: 0, messages: 0, bytes: 0, cachedFiles: 0, skipped: 0, partial: false };
|
|
427
|
+
|
|
428
|
+
const phrases = query
|
|
429
|
+
.split(/\n+/)
|
|
430
|
+
.map((part) => normalizeForSessionMatch(part))
|
|
431
|
+
.filter((part) => part.length >= 3);
|
|
432
|
+
if (phrases.length === 0) return { hits: [], stats, tookMs: Math.round((performance.now() - started) * 100) / 100 };
|
|
433
|
+
|
|
434
|
+
// Distinctive terms: stopwords dropped by the tokenizer, stems kept so
|
|
435
|
+
// "deploying" finds "deploys".
|
|
436
|
+
const termSet = new Set<string>();
|
|
437
|
+
for (const term of tokenize(query)) {
|
|
438
|
+
if (term.length < 2) continue;
|
|
439
|
+
if (term.length === 1 && isCjkChar(term[0])) continue;
|
|
440
|
+
termSet.add(term);
|
|
441
|
+
}
|
|
442
|
+
const terms = [...termSet].sort((a, b) => b.length - a.length).slice(0, 16);
|
|
443
|
+
const scoringTerms = terms.length > 0 ? terms : phrases.flatMap((phrase) => phrase.split(" ").filter((word) => word.length >= 3)).slice(0, 16);
|
|
444
|
+
|
|
445
|
+
const since = options.sinceDays !== undefined ? Date.now() - options.sinceDays * 86_400_000 : undefined;
|
|
446
|
+
const projectFilter = options.project?.trim().toLowerCase();
|
|
447
|
+
// The budget covers the directory walk as well as parsing; a listing that
|
|
448
|
+
// could not be completed is reported as partial.
|
|
449
|
+
const listing = await this.listFiles({ deadline: budgetMs > 0 ? started + budgetMs : undefined });
|
|
450
|
+
if (listing.truncated) stats.partial = true;
|
|
451
|
+
const candidates: Candidate[] = [];
|
|
452
|
+
for (const file of listing.files) {
|
|
453
|
+
if (since !== undefined && file.startedAt < since) continue;
|
|
454
|
+
if (projectFilter && !file.project.toLowerCase().includes(projectFilter) && !file.projectName.toLowerCase().includes(projectFilter)) continue;
|
|
455
|
+
// Checked before every file (a clock read per file is free), so the
|
|
456
|
+
// budget is honoured to within one file rather than one batch.
|
|
457
|
+
if (budgetMs > 0 && performance.now() - started > budgetMs) {
|
|
458
|
+
stats.partial = true;
|
|
459
|
+
break;
|
|
460
|
+
}
|
|
461
|
+
const parsed = await this.messagesFor(file, includeTools);
|
|
462
|
+
stats.files += 1;
|
|
463
|
+
stats.bytes += parsed.rawBytes;
|
|
464
|
+
stats.skipped += parsed.skipped;
|
|
465
|
+
if (parsed.cached) stats.cachedFiles += 1;
|
|
466
|
+
for (const message of parsed.messages) {
|
|
467
|
+
stats.messages += 1;
|
|
468
|
+
if (options.userOnly && message.role !== "user") continue;
|
|
469
|
+
if (!message.normalized) message.normalized = normalizeForSessionMatch(message.text);
|
|
470
|
+
const normalized = message.normalized;
|
|
471
|
+
if (!normalized) continue;
|
|
472
|
+
let exact = 0;
|
|
473
|
+
for (const phrase of phrases) if (normalized.includes(phrase)) exact += 1;
|
|
474
|
+
const matched: string[] = [];
|
|
475
|
+
for (const term of scoringTerms) {
|
|
476
|
+
if (normalized.includes(term)) {
|
|
477
|
+
matched.push(term);
|
|
478
|
+
continue;
|
|
479
|
+
}
|
|
480
|
+
// CJK terms are matched by substring above; other scripts can be
|
|
481
|
+
// matched again after tokenizing (stems, camelCase).
|
|
482
|
+
if (term.length >= 4 && !isCjkChar(term[0])) {
|
|
483
|
+
if (!message.terms) message.terms = new Set(tokenize(message.text));
|
|
484
|
+
if (message.terms.has(term) || message.terms.has(stem(term))) matched.push(term);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
if (exact === 0 && matched.length === 0) continue;
|
|
488
|
+
// Exact phrases and term coverage dominate; the role bonus only
|
|
489
|
+
// breaks ties, in favour of first-hand (user) evidence.
|
|
490
|
+
const roleBonus = message.role === "user" ? 8 : message.role === "assistant" ? 0 : message.role === "summary" ? -2 : -4;
|
|
491
|
+
const coverage = scoringTerms.length > 0 && matched.length >= scoringTerms.length ? 5 : 0;
|
|
492
|
+
// The transcript header records the real cwd; prefer it over the
|
|
493
|
+
// decoded folder name for display and filtering.
|
|
494
|
+
const project = parsed.cwd ?? file.project;
|
|
495
|
+
candidates.push({
|
|
496
|
+
file,
|
|
497
|
+
message,
|
|
498
|
+
project,
|
|
499
|
+
projectName: parsed.cwd ? basename(parsed.cwd) || parsed.cwd : file.projectName,
|
|
500
|
+
score: 100 * exact + 10 * matched.length + roleBonus + coverage,
|
|
501
|
+
matched: matched.slice(0, 8),
|
|
502
|
+
exact: exact > 0,
|
|
503
|
+
});
|
|
504
|
+
if (candidates.length > limit * 8) {
|
|
505
|
+
candidates.sort((a, b) => b.score - a.score);
|
|
506
|
+
candidates.length = limit * 4;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
const hits = candidates
|
|
512
|
+
.sort((a, b) => b.score - a.score || b.message.timestamp - a.message.timestamp)
|
|
513
|
+
.slice(0, limit)
|
|
514
|
+
.map((candidate): SessionHit => ({
|
|
515
|
+
path: candidate.file.path,
|
|
516
|
+
relPath: candidate.file.relPath,
|
|
517
|
+
project: candidate.project,
|
|
518
|
+
projectName: candidate.projectName,
|
|
519
|
+
line: candidate.message.line,
|
|
520
|
+
role: candidate.message.role,
|
|
521
|
+
timestamp: candidate.message.timestamp,
|
|
522
|
+
score: candidate.score,
|
|
523
|
+
matched: candidate.matched,
|
|
524
|
+
exact: candidate.exact,
|
|
525
|
+
excerpt: buildExcerpt(candidate.message.text, phrases, scoringTerms, excerptChars),
|
|
526
|
+
}));
|
|
527
|
+
return { hits, stats, tookMs: Math.round((performance.now() - started) * 100) / 100 };
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/**
|
|
531
|
+
* Read a bounded window of messages around a line, so an agent can verify a
|
|
532
|
+
* hit in context (including later corrections).
|
|
533
|
+
*/
|
|
534
|
+
async readWindow(path: string, line: number, windowSize = 6): Promise<SessionReadResult | undefined> {
|
|
535
|
+
const target = resolve(path);
|
|
536
|
+
const root = this.roots.find((candidate) => target === resolve(candidate) || target.startsWith(`${resolve(candidate)}${sep}`));
|
|
537
|
+
if (!root || !existsSync(target)) return undefined;
|
|
538
|
+
if (!Number.isFinite(line) || line < 1) return undefined;
|
|
539
|
+
const size = Math.max(0, Math.min(40, Math.floor(windowSize)));
|
|
540
|
+
const from = Math.max(1, Math.floor(line) - size);
|
|
541
|
+
const to = Math.floor(line) + size;
|
|
542
|
+
let raw: string;
|
|
543
|
+
try {
|
|
544
|
+
raw = await readFile(target, "utf8");
|
|
545
|
+
} catch {
|
|
546
|
+
return undefined;
|
|
547
|
+
}
|
|
548
|
+
const messages: SessionWindowMessage[] = [];
|
|
549
|
+
let cwd: string | undefined;
|
|
550
|
+
const lines = raw.split(/\r?\n/);
|
|
551
|
+
for (let index = from - 1; index < lines.length && index + 1 <= to; index += 1) {
|
|
552
|
+
const rawLine = lines[index];
|
|
553
|
+
if (!rawLine || rawLine.charCodeAt(0) !== 123) continue;
|
|
554
|
+
let record: Record<string, unknown>;
|
|
555
|
+
try {
|
|
556
|
+
record = JSON.parse(rawLine) as Record<string, unknown>;
|
|
557
|
+
} catch {
|
|
558
|
+
continue;
|
|
559
|
+
}
|
|
560
|
+
if (record.type === "session" && typeof record.cwd === "string") cwd = record.cwd;
|
|
561
|
+
if (record.type !== "message") continue;
|
|
562
|
+
const message = record.message as { role?: string; content?: unknown; timestamp?: number } | undefined;
|
|
563
|
+
if (!message || typeof message.role !== "string" || message.role === "system") continue;
|
|
564
|
+
const role = message.role === "toolResult" ? "tool" : message.role;
|
|
565
|
+
const text = extractMessageText(message.role, message.content, true);
|
|
566
|
+
if (!text) continue;
|
|
567
|
+
const entryTime = typeof record.timestamp === "string" ? Date.parse(record.timestamp) || 0 : 0;
|
|
568
|
+
messages.push({
|
|
569
|
+
line: index + 1,
|
|
570
|
+
role,
|
|
571
|
+
timestamp: typeof message.timestamp === "number" && message.timestamp > 0 ? message.timestamp : entryTime,
|
|
572
|
+
text: truncateChars(text, 2400),
|
|
573
|
+
});
|
|
574
|
+
}
|
|
575
|
+
if (messages.length === 0) return undefined;
|
|
576
|
+
const { projectName } = deriveProject(target, cwd);
|
|
577
|
+
return {
|
|
578
|
+
path: target,
|
|
579
|
+
relPath: relative(root, target).split(sep).join("/"),
|
|
580
|
+
projectName,
|
|
581
|
+
startLine: messages[0].line,
|
|
582
|
+
endLine: messages[messages.length - 1].line,
|
|
583
|
+
messages,
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
/** ISO timestamp for display; empty string when unknown. */
|
|
589
|
+
export function formatSessionTime(timestamp: number): string {
|
|
590
|
+
if (!timestamp) return "unknown";
|
|
591
|
+
return new Date(timestamp).toISOString().replace("T", " ").slice(0, 16);
|
|
592
|
+
}
|