poe-code 4.0.31 → 4.0.32

package/packages/agent-traces/dist/index-store/store.js

@@ -0,0 +1,263 @@
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { mapConcurrent } from "./concurrency.js";
+import { readHead } from "./head.js";
+const MANIFEST_VERSION = 1;
+const HOT_WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
+const IO_CONCURRENCY = 32;
+function isMissingFile(error) {
+    return (typeof error === "object" &&
+        error !== null &&
+        Object.prototype.hasOwnProperty.call(error, "code") &&
+        error.code === "ENOENT");
+}
+function shardFileName(directory) {
+    return `${createHash("sha256").update(directory).digest("hex").slice(0, 16)}.jsonl`;
+}
+function parseShardLines(contents) {
+    const records = [];
+    for (const line of contents.split("\n")) {
+        if (line.trim().length === 0) {
+            continue;
+        }
+        try {
+            const value = JSON.parse(line);
+            if (typeof value === "object" &&
+                value !== null &&
+                typeof value.path === "string" &&
+                typeof value.id === "string") {
+                records.push(value);
+            }
+        }
+        catch {
+            // Corrupt shard lines are dropped; the next sync re-derives them.
+        }
+    }
+    return records;
+}
+function referenceFromRecord(record) {
+    return {
+        source: record.source,
+        id: record.id,
+        path: record.path,
+        ...(record.cwd !== undefined ? { cwd: record.cwd } : {}),
+        ...(record.updatedAtMs > 0 ? { updatedAt: new Date(record.updatedAtMs) } : {}),
+        ...(record.title !== undefined ? { title: record.title } : {}),
+        ...(record.metadata !== undefined ? { metadata: record.metadata } : {})
+    };
+}
+export async function openTraceIndex(options) {
+    const { dir, fs } = options;
+    const manifestPath = path.join(dir, "manifest.json");
+    const shardsDir = path.join(dir, "shards");
+    const writeAtomic = async (filePath, data) => {
+        const rename = fs.rename;
+        if (typeof rename === "function") {
+            const tempPath = `${filePath}.tmp`;
+            await fs.writeFile(tempPath, data);
+            await rename.call(fs, tempPath, filePath);
+            return;
+        }
+        await fs.writeFile(filePath, data);
+    };
+    const loadManifest = async () => {
+        try {
+            const parsed = JSON.parse(await fs.readFile(manifestPath, "utf8"));
+            if (parsed.version === MANIFEST_VERSION && typeof parsed.shards === "object") {
+                return parsed;
+            }
+        }
+        catch {
+            // Missing or corrupt manifest: start empty and re-derive on sync.
+        }
+        return { version: MANIFEST_VERSION, shards: {} };
+    };
+    const loadShard = async (shard) => {
+        try {
+            return parseShardLines(await fs.readFile(path.join(shardsDir, shard.file), "utf8"));
+        }
+        catch (error) {
+            if (isMissingFile(error)) {
+                return [];
+            }
+            throw error;
+        }
+    };
+    const sync = async (syncOptions) => {
+        const now = syncOptions.now ?? Date.now;
+        const stats = {
+            scannedDirs: 0,
+            statted: 0,
+            headReads: 0,
+            added: 0,
+            updated: 0,
+            removed: 0
+        };
+        await fs.mkdir(shardsDir, { recursive: true });
+        const manifest = await loadManifest();
+        let manifestDirty = false;
+        const scannedSources = new Set();
+        const seenDirectories = new Set();
+        const hotCutoffMs = now() - HOT_WINDOW_MS;
+        for (const reader of syncOptions.readers) {
+            const scan = reader.scan?.bind(reader);
+            const readHeadMetadata = reader.readHeadMetadata?.bind(reader);
+            if (scan === undefined || readHeadMetadata === undefined) {
+                continue;
+            }
+            scannedSources.add(reader.id);
+            for await (const { directory, files } of scan({ homeDir: syncOptions.homeDir, fs })) {
+                stats.scannedDirs += 1;
+                seenDirectories.add(directory);
+                const shardEntry = manifest.shards[directory];
+                const existing = shardEntry === undefined ? [] : await loadShard(shardEntry);
+                const byPath = new Map(existing.map((record) => [record.path, record]));
+                const fileSet = new Set(files);
+                const toStat = files.filter((filePath) => {
+                    const record = byPath.get(filePath);
+                    return record === undefined || record.updatedAtMs >= hotCutoffMs;
+                });
+                stats.statted += toStat.length;
+                const statResults = await mapConcurrent(toStat, IO_CONCURRENCY, async (filePath) => ({
+                    filePath,
+                    stat: await fs.stat(filePath).catch(() => undefined)
+                }));
+                const changed = statResults.filter(({ filePath, stat }) => {
+                    if (stat === undefined || !stat.isFile()) {
+                        return false;
+                    }
+                    const record = byPath.get(filePath);
+                    const mtimeMs = stat.mtime?.getTime() ?? 0;
+                    const size = stat.size ?? -1;
+                    return record === undefined || record.mtimeMs !== mtimeMs || record.size !== size;
+                });
+                const updatedRecords = await mapConcurrent(changed, IO_CONCURRENCY, async ({ filePath, stat }) => {
+                    stats.headReads += 1;
+                    const head = await readHead(fs, filePath).catch(() => undefined);
+                    if (head === undefined) {
+                        return undefined;
+                    }
+                    const metadata = readHeadMetadata(head, filePath);
+                    if (metadata === undefined) {
+                        return undefined;
+                    }
+                    const mtimeMs = stat?.mtime?.getTime() ?? 0;
+                    return {
+                        path: filePath,
+                        source: reader.id,
+                        id: metadata.id,
+                        ...(metadata.cwd !== undefined ? { cwd: metadata.cwd } : {}),
+                        ...(metadata.title !== undefined ? { title: metadata.title } : {}),
+                        ...(metadata.metadata !== undefined ? { metadata: metadata.metadata } : {}),
+                        updatedAtMs: mtimeMs > 0 ? mtimeMs : (metadata.updatedAt?.getTime() ?? 0),
+                        mtimeMs,
+                        size: stat?.size ?? -1
+                    };
+                });
+                const removedCount = existing.filter((record) => !fileSet.has(record.path)).length;
+                const freshRecords = updatedRecords.filter((record) => record !== undefined);
+                if (freshRecords.length === 0 && removedCount === 0) {
+                    syncOptions.onProgress?.({ scannedDirs: stats.scannedDirs, headReads: stats.headReads });
+                    continue;
+                }
+                for (const record of freshRecords) {
+                    if (byPath.has(record.path)) {
+                        stats.updated += 1;
+                    }
+                    else {
+                        stats.added += 1;
+                    }
+                    byPath.set(record.path, record);
+                }
+                for (const record of existing) {
+                    if (!fileSet.has(record.path)) {
+                        byPath.delete(record.path);
+                        stats.removed += 1;
+                    }
+                }
+                const records = [...byPath.values()].sort((a, b) => b.updatedAtMs - a.updatedAtMs);
+                const file = shardEntry?.file ?? shardFileName(directory);
+                if (records.length === 0) {
+                    delete manifest.shards[directory];
+                }
+                else {
+                    await writeAtomic(path.join(shardsDir, file), records.map((record) => JSON.stringify(record)).join("\n") + "\n");
+                    manifest.shards[directory] = {
+                        file,
+                        source: reader.id,
+                        maxUpdatedAtMs: records[0]?.updatedAtMs ?? 0,
+                        count: records.length
+                    };
+                }
+                manifestDirty = true;
+                syncOptions.onProgress?.({ scannedDirs: stats.scannedDirs, headReads: stats.headReads });
+            }
+        }
+        for (const [directory, shard] of Object.entries(manifest.shards)) {
+            if (scannedSources.has(shard.source) && !seenDirectories.has(directory)) {
+                stats.removed += shard.count;
+                delete manifest.shards[directory];
+                manifestDirty = true;
+                const unlink = fs.unlink;
+                if (typeof unlink === "function") {
+                    await unlink.call(fs, path.join(shardsDir, shard.file)).catch(() => undefined);
+                }
+            }
+        }
+        if (manifestDirty) {
+            await writeAtomic(manifestPath, JSON.stringify({ ...manifest, version: MANIFEST_VERSION }));
+        }
+        return stats;
+    };
+    const query = async (queryOptions) => {
+        const manifest = await loadManifest();
+        const sinceMs = queryOptions.since?.getTime();
+        const limit = queryOptions.limit;
+        const entries = Object.values(manifest.shards)
+            .filter((shard) => queryOptions.sources === undefined || queryOptions.sources.includes(shard.source))
+            .sort((a, b) => b.maxUpdatedAtMs - a.maxUpdatedAtMs);
+        let results = [];
+        for (const entry of entries) {
+            if (sinceMs !== undefined && entry.maxUpdatedAtMs < sinceMs) {
+                break;
+            }
+            if (results.length >= limit) {
+                const cutoff = results[results.length - 1]?.updatedAtMs ?? 0;
+                if (entry.maxUpdatedAtMs <= cutoff) {
+                    break;
+                }
+            }
+            for (const record of await loadShard(entry)) {
+                if (sinceMs !== undefined && record.updatedAtMs < sinceMs) {
+                    continue;
+                }
+                if (queryOptions.allWorkspaces !== true &&
+                    queryOptions.cwd !== undefined &&
+                    record.cwd !== queryOptions.cwd) {
+                    continue;
+                }
+                results.push(record);
+            }
+            results.sort((a, b) => b.updatedAtMs - a.updatedAtMs);
+            if (Number.isFinite(limit) && results.length > limit) {
+                results = results.slice(0, limit);
+            }
+        }
+        return results.map(referenceFromRecord);
+    };
+    const rebuild = async (syncOptions) => {
+        const unlink = fs.unlink;
+        if (typeof unlink === "function") {
+            const names = await fs.readdir(shardsDir).catch(() => []);
+            for (const name of names) {
+                await unlink.call(fs, path.join(shardsDir, name)).catch(() => undefined);
+            }
+            await unlink.call(fs, manifestPath).catch(() => undefined);
+        }
+        else {
+            await writeAtomic(manifestPath, JSON.stringify({ version: MANIFEST_VERSION, shards: {} }));
+        }
+        return sync(syncOptions);
+    };
+    return { sync, query, rebuild };
+}

package/packages/agent-traces/dist/index.d.ts

@@ -1,4 +1,6 @@
 export { collectHumanPrompts, collectHumanPromptsFromReaders, collectHumanPromptsWithStats } from "./collect.js";
 export { writeHumanPromptJsonl } from "./jsonl.js";
+export { openTraceIndex } from "./index-store/store.js";
+export type { TraceIndex, TraceIndexQueryOptions, TraceIndexSyncOptions, TraceIndexSyncStats } from "./index-store/store.js";
 export { claudeTraceReader, codexTraceReader, piTraceReader, poeCodeTraceReader, traceReaders } from "./readers/index.js";
-export type { AgentTraceFileSystem, AgentTraceSource, CollectHumanPromptsOptions, CollectHumanPromptsResult, HumanPromptRecord, NormalizedTrace, NormalizedTraceTurn, SqliteTraceDatabase, SqliteTraceDatabaseFactory, TraceDiscoverOptions, TraceReadOptions, TraceReader, TraceReference, TraceUsage } from "./types.js";
+export type { AgentTraceFileSystem, AgentTraceSource, CollectHumanPromptsOptions, CollectHumanPromptsResult, HumanPromptRecord, NormalizedTrace, NormalizedTraceTurn, SqliteTraceDatabase, SqliteTraceDatabaseFactory, TraceDiscoverOptions, TraceHeadMetadata, TraceReadOptions, TraceReader, TraceReference, TraceScanDirectory, TraceScanOptions, TraceUsage } from "./types.js";

package/packages/agent-traces/dist/index.js

@@ -1,3 +1,4 @@
 export { collectHumanPrompts, collectHumanPromptsFromReaders, collectHumanPromptsWithStats } from "./collect.js";
 export { writeHumanPromptJsonl } from "./jsonl.js";
+export { openTraceIndex } from "./index-store/store.js";
 export { claudeTraceReader, codexTraceReader, piTraceReader, poeCodeTraceReader, traceReaders } from "./readers/index.js";

package/packages/agent-traces/dist/readers/claude.js

@@ -1,4 +1,6 @@
 import path from "node:path";
+import { mapConcurrent } from "../index-store/concurrency.js";
+import { readHead } from "../index-store/head.js";
 import { asRecord, newestDate, parseDate, parseJsonLines } from "../line-json.js";
 function isMissingFile(error) {
     return (typeof error === "object" &&
@@ -547,6 +549,7 @@ function parseRecordLine(rawLine) {
         return undefined;
     }
 }
+const DISCOVER_CONCURRENCY = 32;
 export const claudeTraceReader = {
     id: "claude",
     defaultRoots(homeDir) {
@@ -559,24 +562,40 @@ export const claudeTraceReader = {
             : [path.join(projectsRoot, encodeClaudeProjectPath(options.cwd))];
         const references = [];
         for (const directory of directories) {
-            for (const filePath of await listJsonlFiles(options.fs, directory)) {
+            const files = await listJsonlFiles(options.fs, directory);
+            const discovered = await mapConcurrent(files, DISCOVER_CONCURRENCY, async (filePath) => {
                 const updatedAt = await fileUpdatedAt(options.fs, filePath);
                 if (options.since && updatedAt && updatedAt < options.since) {
-                    continue;
+                    return undefined;
                 }
-                const metadata = traceHeadMetadata(await options.fs.readFile(filePath, "utf8"));
-                references.push({
+                const metadata = traceHeadMetadata(await readHead(options.fs, filePath));
+                return {
                     source: "claude",
                     id: metadata.sessionId ?? fileId(filePath),
                     path: filePath,
                     ...(metadata.cwd ? { cwd: metadata.cwd } : {}),
                     ...(updatedAt ? { updatedAt } : {}),
                     ...(metadata.title ? { title: metadata.title } : {})
-                });
-            }
+                };
+            });
+            references.push(...discovered.filter((reference) => reference !== undefined));
         }
         return references;
     },
+    async *scan(options) {
+        const projectsRoot = path.join(options.homeDir, ".claude", "projects");
+        for (const directory of await listProjectDirectories(options.fs, projectsRoot)) {
+            yield { directory, files: await listJsonlFiles(options.fs, directory) };
+        }
+    },
+    readHeadMetadata(head, filePath) {
+        const metadata = traceHeadMetadata(head);
+        return {
+            id: metadata.sessionId ?? fileId(filePath),
+            ...(metadata.cwd ? { cwd: metadata.cwd } : {}),
+            ...(metadata.title ? { title: metadata.title } : {})
+        };
+    },
     async read(reference, options) {
         if (!reference.path) {
             throw new Error(`Claude trace ${reference.id} has no path.`);

package/packages/agent-traces/dist/readers/pi.js

@@ -1,4 +1,6 @@
 import path from "node:path";
+import { mapConcurrent } from "../index-store/concurrency.js";
+import { readHead } from "../index-store/head.js";
 import { asRecord, newestDate, parseDate, parseJsonLines } from "../line-json.js";
 function isMissingFile(error) {
     return (typeof error === "object" &&
@@ -272,6 +274,44 @@ function readTraceContents(filePath, contents) {
         turns
     };
 }
+function headMetadataFromContents(head, filePath) {
+    let id = fileId(filePath);
+    let cwd;
+    let model;
+    let title;
+    const toolCalls = new Map();
+    for (const value of parseJsonLines(head)) {
+        const record = asRecord(value);
+        if (!record) {
+            continue;
+        }
+        if (record.type === "session") {
+            if (typeof record.id === "string") {
+                id = record.id;
+            }
+            if (typeof record.cwd === "string") {
+                cwd = record.cwd;
+            }
+        }
+        if (record.type === "model_change" && typeof record.modelId === "string") {
+            model = record.modelId;
+        }
+        const message = asRecord(record.message);
+        if (message?.role === "assistant" && typeof message.model === "string") {
+            model = message.model;
+        }
+        if (title === undefined) {
+            title = turnsFromMessage(record, toolCalls).find((turn) => turn.role === "human")?.text;
+        }
+    }
+    return {
+        id,
+        ...(cwd ? { cwd } : {}),
+        ...(title ? { title } : {}),
+        ...(model ? { metadata: { model } } : {})
+    };
+}
+const DISCOVER_CONCURRENCY = 32;
 export const piTraceReader = {
     id: "pi",
     defaultRoots(homeDir) {
@@ -284,26 +324,37 @@ export const piTraceReader = {
             : [path.join(sessionsRoot, encodeWorkspacePath(options.cwd))];
         const references = [];
         for (const directory of directories) {
-            for (const filePath of await listJsonlFiles(options.fs, directory)) {
+            const files = await listJsonlFiles(options.fs, directory);
+            const discovered = await mapConcurrent(files, DISCOVER_CONCURRENCY, async (filePath) => {
                 const stat = await options.fs.stat(filePath);
                 const updatedAt = stat.mtime;
                 if (options.since && updatedAt && updatedAt < options.since) {
-                    continue;
+                    return undefined;
                 }
-                const trace = readTraceContents(filePath, await options.fs.readFile(filePath, "utf8"));
-                references.push({
+                const metadata = headMetadataFromContents(await readHead(options.fs, filePath), filePath);
+                return {
                     source: "pi",
-                    id: trace.id,
+                    id: metadata.id,
                     path: filePath,
-                    ...(trace.cwd ? { cwd: trace.cwd } : {}),
+                    ...(metadata.cwd ? { cwd: metadata.cwd } : {}),
                     ...(updatedAt ? { updatedAt } : {}),
-                    ...(trace.title ? { title: trace.title } : {}),
-                    ...(trace.model ? { metadata: { model: trace.model } } : {})
-                });
-            }
+                    ...(metadata.title ? { title: metadata.title } : {}),
+                    ...(metadata.metadata ? { metadata: metadata.metadata } : {})
+                };
+            });
+            references.push(...discovered.filter((reference) => reference !== undefined));
         }
         return references;
     },
+    async *scan(options) {
+        const sessionsRoot = path.join(options.homeDir, ".pi", "agent", "sessions");
+        for (const directory of await listSessionDirectories(options.fs, sessionsRoot)) {
+            yield { directory, files: await listJsonlFiles(options.fs, directory) };
+        }
+    },
+    readHeadMetadata(head, filePath) {
+        return headMetadataFromContents(head, filePath);
+    },
     async read(reference, options) {
         if (!reference.path) {
             throw new Error(`Pi trace ${reference.id} has no path.`);

package/packages/agent-traces/dist/readers/poe-code.js

@@ -1,4 +1,6 @@
 import path from "node:path";
+import { mapConcurrent } from "../index-store/concurrency.js";
+import { readHead } from "../index-store/head.js";
 import { asRecord, newestDate, parseDate, parseJsonLines } from "../line-json.js";
 const HEAD_SCAN_MAX_LINES = 50;
 const REDACTED_LOG_CONTENT = "[redacted]";
@@ -208,6 +210,7 @@ function turnFromRecord(record) {
             return undefined;
     }
 }
+const DISCOVER_CONCURRENCY = 32;
 export const poeCodeTraceReader = {
     id: "poe-code",
     defaultRoots(homeDir) {
@@ -215,26 +218,41 @@ export const poeCodeTraceReader = {
     },
     async discover(options) {
         const root = path.join(options.homeDir, ".poe-code", "spawn-logs");
-        const references = [];
-        for (const filePath of await listJsonlFiles(options.fs, root)) {
+        const files = await listJsonlFiles(options.fs, root);
+        const discovered = await mapConcurrent(files, DISCOVER_CONCURRENCY, async (filePath) => {
             const parsed = parseLogFileName(filePath);
             if (!parsed) {
-                continue;
+                return undefined;
             }
             const updatedAt = await updatedAtForFile(options.fs, filePath, parsed.timestamp);
             if (options.since && updatedAt && updatedAt < options.since) {
-                continue;
+                return undefined;
             }
-            const content = await options.fs.readFile(filePath, "utf8");
-            references.push({
+            const head = await readHead(options.fs, filePath);
+            return {
                 source: "poe-code",
                 id: parsed.sessionId,
                 path: filePath,
                 ...(updatedAt ? { updatedAt } : {}),
-                title: titleFromLogContent(content) ?? parsed.agent
-            });
+                title: titleFromLogContent(head) ?? parsed.agent
+            };
+        });
+        return discovered.filter((reference) => reference !== undefined);
+    },
+    async *scan(options) {
+        const root = path.join(options.homeDir, ".poe-code", "spawn-logs");
+        yield { directory: root, files: await listJsonlFiles(options.fs, root) };
+    },
+    readHeadMetadata(head, filePath) {
+        const parsed = parseLogFileName(filePath);
+        if (!parsed) {
+            return undefined;
         }
-        return references;
+        return {
+            id: parsed.sessionId,
+            title: titleFromLogContent(head) ?? parsed.agent,
+            ...(parsed.timestamp ? { updatedAt: parsed.timestamp } : {})
+        };
     },
     async read(reference, options) {
         if (!reference.path) {

package/packages/agent-traces/dist/types.d.ts

@@ -89,14 +89,32 @@ export interface CollectHumanPromptsOptions {
     allWorkspaces?: boolean;
     fs?: AgentTraceFileSystem;
     sqlite?: SqliteTraceDatabaseFactory;
+    indexDir?: string;
 }
 export interface CollectHumanPromptsResult {
     records: HumanPromptRecord[];
     traceCount: number;
 }
+export interface TraceScanOptions {
+    homeDir: string;
+    fs: AgentTraceFileSystem;
+}
+export interface TraceScanDirectory {
+    directory: string;
+    files: string[];
+}
+export interface TraceHeadMetadata {
+    id: string;
+    cwd?: string;
+    title?: string;
+    updatedAt?: Date;
+    metadata?: Record<string, unknown>;
+}
 export interface TraceReader {
     id: AgentTraceSource;
     defaultRoots(homeDir: string): string[];
     discover(options: TraceDiscoverOptions): Promise<TraceReference[]>;
     read(reference: TraceReference, options: TraceReadOptions): Promise<NormalizedTrace>;
+    scan?(options: TraceScanOptions): AsyncIterable<TraceScanDirectory>;
+    readHeadMetadata?(head: string, filePath: string): TraceHeadMetadata | undefined;
 }