pi-braid 0.43.7 → 0.43.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/TOOLS.md +1 -1
- package/build-info.json +4 -4
- package/dist/extensions/core/prompt-builder/tool-guidelines.js +1 -1
- package/dist/extensions/core/skills-tool/auto-inject.d.ts +21 -0
- package/dist/extensions/core/skills-tool/auto-inject.js +105 -1
- package/dist/extensions/core/skills-tool/index.js +20 -2
- package/dist/extensions/core/skills-tool/loading-guidance.d.ts +4 -1
- package/dist/extensions/core/skills-tool/loading-guidance.js +4 -1
- package/dist/extensions/core/skills-tool/session-records.d.ts +107 -9
- package/dist/extensions/core/skills-tool/session-records.js +141 -5
- package/dist/extensions/enforcement/handlers/analysis/dreamer-handler.js +5 -1
- package/dist/extensions/enforcement/handlers/register-tool-call.js +5 -2
- package/dist/extensions/enforcement/verification-judge.d.ts +7 -0
- package/dist/extensions/enforcement/verification-judge.js +28 -21
- package/dist/extensions/memory/session-recall/ask.d.ts +95 -0
- package/dist/extensions/memory/session-recall/ask.js +376 -0
- package/dist/extensions/memory/session-recall/corpus-index.d.ts +200 -0
- package/dist/extensions/memory/session-recall/corpus-index.js +480 -0
- package/dist/extensions/memory/session-recall/index.js +173 -21
- package/dist/extensions/memory/session-recall/search.d.ts +29 -0
- package/dist/extensions/memory/session-recall/search.js +42 -9
- package/dist/lib/ai/payload-hygiene.d.ts +3 -0
- package/dist/lib/ai/payload-hygiene.js +34 -2
- package/dist/lib/config/braid-paths.d.ts +4 -0
- package/dist/lib/config/braid-paths.js +6 -0
- package/dist/lib/config/braid-settings.d.ts +22 -0
- package/dist/lib/config/braid-settings.js +15 -0
- package/dist/lib/detection/tool-loop-detector.d.ts +95 -46
- package/dist/lib/detection/tool-loop-detector.js +288 -180
- package/dist/lib/reasoning/dreamer.d.ts +2 -1
- package/dist/lib/reasoning/dreamer.js +12 -7
- package/docs/LOCAL_DEV.md +1 -1
- package/docs/SETUP.md +1 -0
- package/package.json +1 -1
- package/src/extensions/core/AGENTS.md +3 -2
- package/src/extensions/core/prompt-builder/tool-guidelines.ts +1 -1
- package/src/extensions/core/skills-tool/auto-inject.ts +118 -1
- package/src/extensions/core/skills-tool/index.ts +21 -1
- package/src/extensions/core/skills-tool/loading-guidance.ts +4 -1
- package/src/extensions/core/skills-tool/session-records.ts +201 -9
- package/src/extensions/enforcement/handlers/analysis/dreamer-handler.ts +5 -1
- package/src/extensions/enforcement/handlers/register-tool-call.ts +5 -2
- package/src/extensions/enforcement/verification-judge.ts +32 -25
- package/src/extensions/memory/AGENTS.md +2 -2
- package/src/extensions/memory/session-recall/AGENTS.md +92 -18
- package/src/extensions/memory/session-recall/ask.ts +435 -0
- package/src/extensions/memory/session-recall/corpus-index.ts +609 -0
- package/src/extensions/memory/session-recall/index.ts +202 -21
- package/src/extensions/memory/session-recall/search.ts +61 -10
- package/src/lib/ai/payload-hygiene.ts +47 -3
- package/src/lib/config/braid-paths.ts +7 -0
- package/src/lib/config/braid-settings.ts +42 -3
- package/src/lib/detection/AGENTS.md +8 -7
- package/src/lib/detection/tool-loop-detector.ts +339 -195
- package/src/lib/reasoning/dreamer.ts +13 -6
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Session corpus index - the user's own turns, extracted once and stored as a
|
|
3
|
+
* derived population.
|
|
4
|
+
*
|
|
5
|
+
* The transcript corpus is measured in gigabytes while every user message in it
|
|
6
|
+
* is measured in megabytes: the human's words are a fraction of one percent of
|
|
7
|
+
* the archive, and the rest is the agent's own reasoning and tool output. A
|
|
8
|
+
* grep over the archive therefore drowns that fraction in noise, and no tool
|
|
9
|
+
* reads it as a population. This module is the population: one row per user
|
|
10
|
+
* turn, carrying the session file, the session id, the project label, the entry
|
|
11
|
+
* timestamp, the line number in the session file, and the text as typed.
|
|
12
|
+
*
|
|
13
|
+
* Derived, never authoritative: every shard is rebuildable from the session
|
|
14
|
+
* files alone. One shard per project under `corpusIndexDir()`, so a query can
|
|
15
|
+
* read one project or all of them without walking a transcript tree.
|
|
16
|
+
*
|
|
17
|
+
* Two properties the readers depend on:
|
|
18
|
+
*
|
|
19
|
+
* - **Fail closed.** A shard is accepted whole or not at all. A manifest that
|
|
20
|
+
* does not parse, a format this build does not know, a row count that
|
|
21
|
+
* disagrees with the manifest, or a row that does not parse makes the shard
|
|
22
|
+
* UNREADABLE - the caller must say it cannot answer rather than answer from
|
|
23
|
+
* half a store. `loadCorpusIndex` reports which shard and which defect.
|
|
24
|
+
* - **Excluded by construction.** Only `message` entries with role `user`
|
|
25
|
+
* contribute text. Assistant turns, tool calls, tool results and
|
|
26
|
+
* extension-injected `custom_message` entries never enter a row, so no key,
|
|
27
|
+
* no tool argument and no environment value can be retrieved from the index.
|
|
28
|
+
* Tool results arrive as user-role messages with no text block, so the text
|
|
29
|
+
* filter drops them without a role list.
|
|
30
|
+
*
|
|
31
|
+
* What is NOT indexed: the compaction archives under `<session dir>/archives/`.
|
|
32
|
+
* An envelope there is a second copy of messages the session body already
|
|
33
|
+
* carries (writeOriginalsArchive stores the pre-compaction originals), so
|
|
34
|
+
* indexing both would count the same words twice.
|
|
35
|
+
*/
|
|
36
|
+
import { createHash } from "node:crypto";
|
|
37
|
+
import { readFile, readdir, rename, stat, writeFile, mkdir } from "node:fs/promises";
|
|
38
|
+
import { basename, isAbsolute, join } from "node:path";
|
|
39
|
+
import { corpusIndexDir } from "../../../lib/config/braid-paths.js";
|
|
40
|
+
import { projectFromPath, walkJsonlFiles } from "./search.js";
|
|
41
|
+
/** Shard layout version. A shard written by another format is unreadable, not
|
|
42
|
+
* migrated: the build regenerates it from the sessions. */
|
|
43
|
+
export const CORPUS_INDEX_FORMAT = 1;
|
|
44
|
+
/** Directory-name segment that holds compaction envelopes, never session bodies. */
|
|
45
|
+
const ARCHIVE_SEGMENT = /(^|[/\\])archives[/\\]/;
|
|
46
|
+
/** Thrown by the shard reader; the message names the shard and the defect. */
|
|
47
|
+
export class CorpusShardError extends Error {
|
|
48
|
+
shardPath;
|
|
49
|
+
defect;
|
|
50
|
+
constructor(shardPath, defect) {
|
|
51
|
+
super(`${shardPath}: ${defect}`);
|
|
52
|
+
this.shardPath = shardPath;
|
|
53
|
+
this.defect = defect;
|
|
54
|
+
this.name = "CorpusShardError";
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** pi's session file name is `<iso-ts>_<session-id>.jsonl`; a shard cites the id. */
|
|
58
|
+
const SESSION_ID_IN_NAME = /_([0-9a-z][0-9a-z-]{7,})\.jsonl$/i;
|
|
59
|
+
/**
|
|
60
|
+
* File-name stem for one project's shard.
|
|
61
|
+
*
|
|
62
|
+
* The project label is decoded from an encoded directory name, which is lossy
|
|
63
|
+
* (`-` is both a separator and a legal character inside a name), so two
|
|
64
|
+
* different projects can decode to one label and two labels can sanitize to one
|
|
65
|
+
* stem. The digest of the label keeps one shard per project; the readable
|
|
66
|
+
* prefix keeps the directory browsable. Every reader goes through here, so a
|
|
67
|
+
* writer and a reader cannot name different files.
|
|
68
|
+
*/
|
|
69
|
+
export function shardSlug(project) {
|
|
70
|
+
const safe = project
|
|
71
|
+
.replace(/[^a-zA-Z0-9._-]+/g, "-")
|
|
72
|
+
.replace(/-{2,}/g, "-")
|
|
73
|
+
.replace(/^[-.]+|[-.]+$/g, "")
|
|
74
|
+
.slice(0, 80) || "project";
|
|
75
|
+
const digest = createHash("sha256").update(project).digest("hex").slice(0, 8);
|
|
76
|
+
return `${safe}-${digest}`;
|
|
77
|
+
}
|
|
78
|
+
/** Absolute path of one project's shard. */
|
|
79
|
+
export function shardPath(indexDir, project) {
|
|
80
|
+
return join(indexDir, `${shardSlug(project)}.jsonl`);
|
|
81
|
+
}
|
|
82
|
+
/** The session id a session file name carries, or the file's stem when it carries none. */
|
|
83
|
+
export function sessionIdFromPath(sessionPath) {
|
|
84
|
+
const name = basename(sessionPath);
|
|
85
|
+
const match = SESSION_ID_IN_NAME.exec(name);
|
|
86
|
+
return match?.[1] ?? name.replace(/\.jsonl$/, "");
|
|
87
|
+
}
|
|
88
|
+
/** The `YYYY-MM-DD` date a session file name starts with, or null. */
|
|
89
|
+
function dateFromName(sessionPath) {
|
|
90
|
+
const match = basename(sessionPath).match(/^(\d{4}-\d{2}-\d{2})T/);
|
|
91
|
+
return match?.[1] ?? null;
|
|
92
|
+
}
|
|
93
|
+
/** Text of a message's content: a string, or the text blocks joined. */
|
|
94
|
+
function textOfContent(content) {
|
|
95
|
+
if (typeof content === "string")
|
|
96
|
+
return content;
|
|
97
|
+
if (!Array.isArray(content))
|
|
98
|
+
return "";
|
|
99
|
+
return content
|
|
100
|
+
.filter((block) => !!block && typeof block === "object")
|
|
101
|
+
.filter((block) => block.type === "text" && typeof block.text === "string")
|
|
102
|
+
.map((block) => block.text)
|
|
103
|
+
.join(" ");
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Every user turn in one session body, in file order.
|
|
107
|
+
*
|
|
108
|
+
* A turn is a `message` entry whose role is `user` and whose content carries
|
|
109
|
+
* text. The role alone is not enough: a tool result is delivered as a
|
|
110
|
+
* user-role message with `tool_result` blocks and no text, and a thought or a
|
|
111
|
+
* blank paste carries no words worth indexing. Blank text is dropped here so
|
|
112
|
+
* the corpus holds only what the human actually typed.
|
|
113
|
+
*/
|
|
114
|
+
export function extractUserTurns(content, opts) {
|
|
115
|
+
const sessionId = sessionIdFromPath(opts.session);
|
|
116
|
+
const fallbackTs = opts.fileDate ? `${opts.fileDate}T00:00:00.000Z` : "";
|
|
117
|
+
const turns = [];
|
|
118
|
+
const lines = content.split("\n");
|
|
119
|
+
for (let i = 0; i < lines.length; i++) {
|
|
120
|
+
const line = lines[i];
|
|
121
|
+
if (!line || !line.includes('"user"'))
|
|
122
|
+
continue;
|
|
123
|
+
let entry;
|
|
124
|
+
try {
|
|
125
|
+
entry = JSON.parse(line);
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
if (entry?.type !== "message")
|
|
131
|
+
continue;
|
|
132
|
+
if (entry.message?.role !== "user")
|
|
133
|
+
continue;
|
|
134
|
+
const text = textOfContent(entry.message.content);
|
|
135
|
+
if (!text.trim())
|
|
136
|
+
continue;
|
|
137
|
+
turns.push({
|
|
138
|
+
session: opts.session,
|
|
139
|
+
session_id: sessionId,
|
|
140
|
+
project: opts.project,
|
|
141
|
+
ts: typeof entry.timestamp === "string" ? entry.timestamp : fallbackTs,
|
|
142
|
+
line: i + 1,
|
|
143
|
+
text,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
return turns;
|
|
147
|
+
}
|
|
148
|
+
function parseManifest(line, shard) {
|
|
149
|
+
let parsed;
|
|
150
|
+
try {
|
|
151
|
+
parsed = JSON.parse(line);
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
throw new CorpusShardError(shard, "first line is not JSON (truncated or foreign file)");
|
|
155
|
+
}
|
|
156
|
+
const manifest = parsed;
|
|
157
|
+
if (!manifest || typeof manifest !== "object") {
|
|
158
|
+
throw new CorpusShardError(shard, "first line is not an object");
|
|
159
|
+
}
|
|
160
|
+
if (manifest.kind !== "manifest") {
|
|
161
|
+
throw new CorpusShardError(shard, "first line is not a manifest");
|
|
162
|
+
}
|
|
163
|
+
if (manifest.format !== CORPUS_INDEX_FORMAT) {
|
|
164
|
+
throw new CorpusShardError(shard, `format ${String(manifest.format)} is not the supported format ${CORPUS_INDEX_FORMAT}`);
|
|
165
|
+
}
|
|
166
|
+
if (typeof manifest.project !== "string" || !manifest.project) {
|
|
167
|
+
throw new CorpusShardError(shard, "manifest carries no project");
|
|
168
|
+
}
|
|
169
|
+
if (typeof manifest.built_at !== "string" || Number.isNaN(Date.parse(manifest.built_at))) {
|
|
170
|
+
throw new CorpusShardError(shard, "manifest carries no readable built_at");
|
|
171
|
+
}
|
|
172
|
+
if (typeof manifest.turns !== "number" || !Number.isFinite(manifest.turns)) {
|
|
173
|
+
throw new CorpusShardError(shard, "manifest carries no turn count");
|
|
174
|
+
}
|
|
175
|
+
if (!Array.isArray(manifest.files)) {
|
|
176
|
+
throw new CorpusShardError(shard, "manifest carries no file list");
|
|
177
|
+
}
|
|
178
|
+
return manifest;
|
|
179
|
+
}
|
|
180
|
+
function parseRow(line, shard, manifest) {
|
|
181
|
+
let parsed;
|
|
182
|
+
try {
|
|
183
|
+
parsed = JSON.parse(line);
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
throw new CorpusShardError(shard, "a row is not JSON (partial write?)");
|
|
187
|
+
}
|
|
188
|
+
const row = parsed;
|
|
189
|
+
if (!row || typeof row !== "object") {
|
|
190
|
+
throw new CorpusShardError(shard, "a row is not an object");
|
|
191
|
+
}
|
|
192
|
+
if (typeof row.session !== "string" || !isAbsolute(row.session)) {
|
|
193
|
+
throw new CorpusShardError(shard, "a row names no absolute session path");
|
|
194
|
+
}
|
|
195
|
+
if (typeof row.project !== "string" || row.project !== manifest.project) {
|
|
196
|
+
throw new CorpusShardError(shard, "a row names a project the manifest does not");
|
|
197
|
+
}
|
|
198
|
+
if (typeof row.session_id !== "string" || !row.session_id) {
|
|
199
|
+
throw new CorpusShardError(shard, "a row carries no session id");
|
|
200
|
+
}
|
|
201
|
+
if (typeof row.ts !== "string" || Number.isNaN(Date.parse(row.ts))) {
|
|
202
|
+
throw new CorpusShardError(shard, "a row carries no readable timestamp");
|
|
203
|
+
}
|
|
204
|
+
if (typeof row.line !== "number" || !Number.isInteger(row.line) || row.line < 1) {
|
|
205
|
+
throw new CorpusShardError(shard, "a row carries no line number");
|
|
206
|
+
}
|
|
207
|
+
if (typeof row.text !== "string" || !row.text.trim()) {
|
|
208
|
+
throw new CorpusShardError(shard, "a row carries no text");
|
|
209
|
+
}
|
|
210
|
+
return row;
|
|
211
|
+
}
|
|
212
|
+
/** Read and validate one shard. Whole or nothing - see CorpusShardError. */
|
|
213
|
+
export async function readShard(shard) {
|
|
214
|
+
let raw;
|
|
215
|
+
try {
|
|
216
|
+
raw = await readFile(shard, "utf-8");
|
|
217
|
+
}
|
|
218
|
+
catch (err) {
|
|
219
|
+
throw new CorpusShardError(shard, `unreadable: ${err.message}`);
|
|
220
|
+
}
|
|
221
|
+
const lines = raw.split("\n").filter((line) => line.trim() !== "");
|
|
222
|
+
if (lines.length === 0) {
|
|
223
|
+
throw new CorpusShardError(shard, "empty file (no manifest)");
|
|
224
|
+
}
|
|
225
|
+
const manifest = parseManifest(lines[0], shard);
|
|
226
|
+
const turns = [];
|
|
227
|
+
for (let i = 1; i < lines.length; i++) {
|
|
228
|
+
turns.push(parseRow(lines[i], shard, manifest));
|
|
229
|
+
}
|
|
230
|
+
if (turns.length !== manifest.turns) {
|
|
231
|
+
throw new CorpusShardError(shard, `${turns.length} rows but the manifest records ${manifest.turns} (partial write?)`);
|
|
232
|
+
}
|
|
233
|
+
return { path: shard, manifest, turns };
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Load every shard under the corpus index directory, or refuse.
|
|
237
|
+
*
|
|
238
|
+
* Refusal is the point: a caller that cannot read its evidence must say so.
|
|
239
|
+
* `reason` names the first unreadable shard and its defect; `shards` lists
|
|
240
|
+
* every shard path the directory held, so a refusal can show its population.
|
|
241
|
+
* A `project` filter narrows which shards are read, and a filter that matches
|
|
242
|
+
* nothing is still a refusal - an empty population cannot answer either.
|
|
243
|
+
*/
|
|
244
|
+
export async function loadCorpusIndex(opts = {}) {
|
|
245
|
+
const dir = opts.indexDir ?? corpusIndexDir();
|
|
246
|
+
let names;
|
|
247
|
+
try {
|
|
248
|
+
names = (await readdir(dir)).filter((name) => name.endsWith(".jsonl")).sort();
|
|
249
|
+
}
|
|
250
|
+
catch {
|
|
251
|
+
return { ok: false, dir, reason: `no index directory at ${dir}`, shards: [] };
|
|
252
|
+
}
|
|
253
|
+
if (names.length === 0) {
|
|
254
|
+
return { ok: false, dir, reason: `no shards in ${dir}`, shards: [] };
|
|
255
|
+
}
|
|
256
|
+
const paths = names.map((name) => join(dir, name));
|
|
257
|
+
const shards = [];
|
|
258
|
+
for (const path of paths) {
|
|
259
|
+
let shard;
|
|
260
|
+
try {
|
|
261
|
+
shard = await readShard(path);
|
|
262
|
+
}
|
|
263
|
+
catch (err) {
|
|
264
|
+
// The reader's contract is a refusal, never a throw: a caller that has to
|
|
265
|
+
// catch to learn "cannot answer" is a caller that will answer anyway.
|
|
266
|
+
const detail = err instanceof CorpusShardError ? err.defect : err.message;
|
|
267
|
+
return { ok: false, dir, reason: `${path}: ${detail}`, shards: paths };
|
|
268
|
+
}
|
|
269
|
+
const wanted = opts.project ?? null;
|
|
270
|
+
if (wanted && shard.manifest.project !== wanted)
|
|
271
|
+
continue;
|
|
272
|
+
shards.push(shard);
|
|
273
|
+
}
|
|
274
|
+
if (shards.length === 0) {
|
|
275
|
+
return {
|
|
276
|
+
ok: false,
|
|
277
|
+
dir,
|
|
278
|
+
reason: `no shard matches project "${opts.project}" (shards: ${names.length})`,
|
|
279
|
+
shards: paths,
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
return { ok: true, dir, shards };
|
|
283
|
+
}
|
|
284
|
+
export async function corpusIndexFreshness(shards, sessionsDir) {
|
|
285
|
+
const indexed = new Map();
|
|
286
|
+
for (const shard of shards) {
|
|
287
|
+
for (const file of shard.manifest.files)
|
|
288
|
+
indexed.set(file.path, file);
|
|
289
|
+
}
|
|
290
|
+
const onDisk = (await walkJsonlFiles(sessionsDir)).filter((p) => !ARCHIVE_SEGMENT.test(p));
|
|
291
|
+
let changed = 0;
|
|
292
|
+
let added = 0;
|
|
293
|
+
let newest = 0;
|
|
294
|
+
const seen = new Set();
|
|
295
|
+
for (const path of onDisk) {
|
|
296
|
+
seen.add(path);
|
|
297
|
+
const known = indexed.get(path);
|
|
298
|
+
if (!known) {
|
|
299
|
+
added++;
|
|
300
|
+
try {
|
|
301
|
+
newest = Math.max(newest, (await stat(path)).mtimeMs);
|
|
302
|
+
}
|
|
303
|
+
catch {
|
|
304
|
+
/* vanished between walk and stat */
|
|
305
|
+
}
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
let info;
|
|
309
|
+
try {
|
|
310
|
+
info = await stat(path);
|
|
311
|
+
}
|
|
312
|
+
catch {
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
if (info.mtimeMs !== known.mtimeMs || info.size !== known.size) {
|
|
316
|
+
changed++;
|
|
317
|
+
newest = Math.max(newest, info.mtimeMs);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
let removed = 0;
|
|
321
|
+
for (const path of indexed.keys())
|
|
322
|
+
if (!seen.has(path))
|
|
323
|
+
removed++;
|
|
324
|
+
return {
|
|
325
|
+
indexed: indexed.size,
|
|
326
|
+
changed,
|
|
327
|
+
added,
|
|
328
|
+
removed,
|
|
329
|
+
newest_change: newest > 0 ? new Date(newest).toISOString() : null,
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
/** Group the session bodies by project label. */
|
|
333
|
+
async function groupByProject(sessionsDir) {
|
|
334
|
+
const grouped = new Map();
|
|
335
|
+
for (const path of await walkJsonlFiles(sessionsDir)) {
|
|
336
|
+
if (ARCHIVE_SEGMENT.test(path))
|
|
337
|
+
continue;
|
|
338
|
+
let info;
|
|
339
|
+
try {
|
|
340
|
+
info = await stat(path);
|
|
341
|
+
}
|
|
342
|
+
catch {
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
345
|
+
const project = projectFromPath(path);
|
|
346
|
+
const list = grouped.get(project);
|
|
347
|
+
const entry = { path, mtimeMs: info.mtimeMs, size: info.size };
|
|
348
|
+
if (list)
|
|
349
|
+
list.push(entry);
|
|
350
|
+
else
|
|
351
|
+
grouped.set(project, [entry]);
|
|
352
|
+
}
|
|
353
|
+
return grouped;
|
|
354
|
+
}
|
|
355
|
+
function manifestLine(manifest) {
|
|
356
|
+
return JSON.stringify(manifest);
|
|
357
|
+
}
|
|
358
|
+
/** Row text bytes, the figure the shard's manifest records. */
|
|
359
|
+
function textBytes(turns) {
|
|
360
|
+
let bytes = 0;
|
|
361
|
+
for (const turn of turns)
|
|
362
|
+
bytes += Buffer.byteLength(turn.text, "utf8");
|
|
363
|
+
return bytes;
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* Walk the session files once and write one shard per project.
|
|
367
|
+
*
|
|
368
|
+
* Incremental by (mtime, size): a session file whose stamp still matches its
|
|
369
|
+
* manifest entry contributes its previous rows without being parsed, so a
|
|
370
|
+
* daily refresh costs what changed rather than the corpus. A shard that cannot
|
|
371
|
+
* be read is not repaired in place - it is rebuilt from its session files,
|
|
372
|
+
* which is the only authority for derived data.
|
|
373
|
+
*/
|
|
374
|
+
export async function buildCorpusIndex(opts) {
|
|
375
|
+
const started = Date.now();
|
|
376
|
+
const dir = opts.indexDir ?? corpusIndexDir();
|
|
377
|
+
const now = opts.now ?? (() => new Date());
|
|
378
|
+
await mkdir(dir, { recursive: true });
|
|
379
|
+
const grouped = await groupByProject(opts.sessionsDir);
|
|
380
|
+
const report = {
|
|
381
|
+
index_dir: dir,
|
|
382
|
+
projects: grouped.size,
|
|
383
|
+
shards: 0,
|
|
384
|
+
sessions: 0,
|
|
385
|
+
turns: 0,
|
|
386
|
+
bytes: 0,
|
|
387
|
+
parsed_files: 0,
|
|
388
|
+
reused_files: 0,
|
|
389
|
+
removed_files: 0,
|
|
390
|
+
skipped: [],
|
|
391
|
+
elapsed_ms: 0,
|
|
392
|
+
};
|
|
393
|
+
for (const [project, files] of grouped) {
|
|
394
|
+
const shard = shardPath(dir, project);
|
|
395
|
+
let previous = null;
|
|
396
|
+
if (!opts.rebuild) {
|
|
397
|
+
try {
|
|
398
|
+
previous = await readShard(shard);
|
|
399
|
+
}
|
|
400
|
+
catch {
|
|
401
|
+
// A corrupt or absent shard costs a rebuild, never a merge: derived
|
|
402
|
+
// data has one authority, the session files.
|
|
403
|
+
previous = null;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
const previousFiles = new Map();
|
|
407
|
+
const previousRows = new Map();
|
|
408
|
+
if (previous && previous.manifest.project === project) {
|
|
409
|
+
for (const file of previous.manifest.files)
|
|
410
|
+
previousFiles.set(file.path, file);
|
|
411
|
+
for (const turn of previous.turns) {
|
|
412
|
+
const rows = previousRows.get(turn.session);
|
|
413
|
+
if (rows)
|
|
414
|
+
rows.push(turn);
|
|
415
|
+
else
|
|
416
|
+
previousRows.set(turn.session, [turn]);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
const turns = [];
|
|
420
|
+
const manifestFiles = [];
|
|
421
|
+
for (const file of files) {
|
|
422
|
+
const known = previousFiles.get(file.path);
|
|
423
|
+
const carried = previousRows.get(file.path);
|
|
424
|
+
if (known && carried && known.mtimeMs === file.mtimeMs && known.size === file.size) {
|
|
425
|
+
for (const turn of carried)
|
|
426
|
+
turns.push(turn);
|
|
427
|
+
manifestFiles.push({ ...file, turns: carried.length });
|
|
428
|
+
report.reused_files++;
|
|
429
|
+
continue;
|
|
430
|
+
}
|
|
431
|
+
let content;
|
|
432
|
+
try {
|
|
433
|
+
content = await readFile(file.path, "utf-8");
|
|
434
|
+
}
|
|
435
|
+
catch (err) {
|
|
436
|
+
report.skipped.push(`${file.path}: ${err.message}`);
|
|
437
|
+
continue;
|
|
438
|
+
}
|
|
439
|
+
const extracted = extractUserTurns(content, {
|
|
440
|
+
session: file.path,
|
|
441
|
+
project,
|
|
442
|
+
fileDate: dateFromName(file.path),
|
|
443
|
+
});
|
|
444
|
+
for (const turn of extracted)
|
|
445
|
+
turns.push(turn);
|
|
446
|
+
manifestFiles.push({ ...file, turns: extracted.length });
|
|
447
|
+
report.parsed_files++;
|
|
448
|
+
}
|
|
449
|
+
// Files the manifest carried that the walk no longer sees. Counted, not
|
|
450
|
+
// guessed at: a shard that silently kept them would cite a deleted file.
|
|
451
|
+
for (const path of previousFiles.keys()) {
|
|
452
|
+
if (!files.some((file) => file.path === path))
|
|
453
|
+
report.removed_files++;
|
|
454
|
+
}
|
|
455
|
+
turns.sort((a, b) => a.session.localeCompare(b.session) || a.line - b.line);
|
|
456
|
+
const manifest = {
|
|
457
|
+
kind: "manifest",
|
|
458
|
+
format: CORPUS_INDEX_FORMAT,
|
|
459
|
+
project,
|
|
460
|
+
built_at: now().toISOString(),
|
|
461
|
+
sessions: manifestFiles.length,
|
|
462
|
+
turns: turns.length,
|
|
463
|
+
bytes: textBytes(turns),
|
|
464
|
+
files: manifestFiles,
|
|
465
|
+
};
|
|
466
|
+
const body = [manifestLine(manifest), ...turns.map((turn) => JSON.stringify(turn))].join("\n");
|
|
467
|
+
const temp = `${shard}.tmp-${process.pid}`;
|
|
468
|
+
await writeFile(temp, `${body}\n`, "utf-8");
|
|
469
|
+
await rename(temp, shard);
|
|
470
|
+
report.shards++;
|
|
471
|
+
report.sessions += manifestFiles.length;
|
|
472
|
+
report.turns += manifest.turns;
|
|
473
|
+
report.bytes += manifest.bytes;
|
|
474
|
+
opts.progress?.(`${project}: ${manifest.turns} turns from ${manifestFiles.length} sessions ` +
|
|
475
|
+
`(${(manifest.bytes / 1_048_576).toFixed(2)} MB)`);
|
|
476
|
+
}
|
|
477
|
+
report.elapsed_ms = Date.now() - started;
|
|
478
|
+
return report;
|
|
479
|
+
}
|
|
480
|
+
//# sourceMappingURL=corpus-index.js.map
|