pi-sessions 0.14.3 → 0.16.0

package/extensions/shared/session-index/recovery.ts

@@ -0,0 +1,270 @@
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { setTimeout as delay, setImmediate as yieldToEventLoop } from "node:timers/promises";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+import { type Static, Type } from "typebox";
+import { listSessionFiles } from "../../session-search/extract.ts";
+import { parseTypeBoxValue } from "../typebox.ts";
+import { INDEX_SCHEMA_VERSION, ROW_COUNT_SCHEMA, type SessionIndexDatabase } from "./common.ts";
+import {
+  ensureIndexDir,
+  getMetadata,
+  initializeSchema,
+  openIndexDatabase,
+  setMetadata,
+} from "./schema.ts";
+import { syncSessionFileWithDb } from "./sync.ts";
+
+export const INDEX_UPDATED_WARNING = "pi-sessions was updated; /reload to use it";
+const BUILD = `index-v${INDEX_SCHEMA_VERSION}:${createHash("sha256")
+  .update(
+    ["recovery.ts", "schema.ts", "sync.ts"]
+      .map((file) => readFileSync(new URL(file, import.meta.url)))
+      .join(""),
+  )
+  .digest("hex")
+  .slice(0, 16)}`;
+const FAILURE_SCHEMA = Type.Object({
+  attempts: Type.Integer({ minimum: 1 }),
+  nextAllowedAt: Type.Number(),
+  error: Type.String(),
+  build: Type.String(),
+});
+export type IndexRecoveryFailure = Static<typeof FAILURE_SCHEMA>;
+export interface ReindexResult {
+  sessionCount: number;
+  chunkCount: number;
+  indexPath: string;
+  skipped: string[];
+}
+const running = new Map<string, Promise<ReindexResult | undefined>>();
+const notices = new Map<string, (message: string | undefined) => void>();
+const shown = new Map<string, string>();
+const disabled = new Set<string>();
+
+export function observeIndexRecovery(
+  indexPath: string,
+  notify: (message: string | undefined) => void,
+): void {
+  notices.set(indexPath, notify);
+  const message = getIndexRecoveryMessage(indexPath);
+  notify(message);
+  if (message) shown.set(indexPath, message);
+}
+
+function report(indexPath: string, message: string): void {
+  if (shown.get(indexPath) === message) return;
+  shown.set(indexPath, message);
+  const notify = notices.get(indexPath);
+  if (notify) notify(message);
+  else console.error(message);
+}
+
+export function disableOlderIndexClient(indexPath: string): void {
+  disabled.add(indexPath);
+  report(indexPath, INDEX_UPDATED_WARNING);
+}
+
+export function isIndexClientDisabled(indexPath: string): boolean {
+  return disabled.has(indexPath);
+}
+
+export function readIndexRecoveryFailure(indexPath: string): IndexRecoveryFailure | undefined {
+  const file = `${indexPath}.failure.json`;
+  let raw: string;
+  try {
+    raw = readFileSync(file, "utf8");
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "ENOENT") return undefined;
+    throw error;
+  }
+  return parseTypeBoxValue(FAILURE_SCHEMA, JSON.parse(raw), "Invalid index recovery record");
+}
+
+export function getIndexRecoveryMessage(indexPath: string): string | undefined {
+  if (disabled.has(indexPath)) return INDEX_UPDATED_WARNING;
+  const failure = readIndexRecoveryFailure(indexPath);
+  if (failure)
+    return `Session indexing failed: ${failure.error}. Retry after ${new Date(failure.nextAllowedAt).toISOString()}; /session-index can rebuild now.`;
+  if (running.has(indexPath)) return "Session indexing in progress; try again shortly.";
+  if (existsSync(`${indexPath}.lock.sqlite`)) {
+    const release = acquireIndexLock(indexPath);
+    if (!release) return "Session indexing in progress; try again shortly.";
+    release();
+  }
+  return undefined;
+}
+
+// All index writers share this lock with recovery. Reads are short-lived and may
+// finish on the old inode after an atomic replacement (no WAL sidecars).
+// SQLite owns the lock lifetime, including process death; never unlink its file.
+export function acquireIndexLock(indexPath: string): (() => void) | undefined {
+  ensureIndexDir(dirname(indexPath));
+  let lock: SessionIndexDatabase | undefined;
+  try {
+    lock = openIndexDatabase(`${indexPath}.lock.sqlite`, { create: true, timeoutMs: 0 });
+    lock.exec("BEGIN EXCLUSIVE");
+    const held = lock;
+    return () => held.close();
+  } catch (error) {
+    lock?.close();
+    if (isSqliteBusy(error)) return undefined;
+    throw error;
+  }
+}
+
+function isSqliteBusy(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    (("code" in error && error.code === "SQLITE_BUSY") ||
+      ("errcode" in error && error.errcode === 5))
+  );
+}
+
+export function requestIndexRecovery(
+  indexPath: string,
+  reconcile = false,
+): Promise<ReindexResult | undefined> {
+  const pending = running.get(indexPath);
+  if (pending) return pending;
+  const sessionsDir = join(getAgentDir(), "sessions");
+  const task = yieldToEventLoop()
+    .then(() => recover(indexPath, sessionsDir, false, reconcile))
+    .catch((error: unknown) => {
+      report(indexPath, `Session indexing failed: ${String(error)}`);
+      return undefined;
+    })
+    .finally(() => running.delete(indexPath));
+  running.set(indexPath, task);
+  return task;
+}
+
+export async function rebuildSessionIndex(options: { indexPath: string }): Promise<ReindexResult> {
+  await running.get(options.indexPath);
+  const result = await recover(options.indexPath, join(getAgentDir(), "sessions"), true, true);
+  if (!result)
+    throw new Error(
+      getIndexRecoveryMessage(options.indexPath) ?? "Session index is busy; try again shortly.",
+    );
+  return result;
+}
+
+async function recover(
+  indexPath: string,
+  sessionsDir: string,
+  manual: boolean,
+  reconcile: boolean,
+): Promise<ReindexResult | undefined> {
+  if (disabled.has(indexPath)) return undefined;
+  let release = acquireIndexLock(indexPath);
+  const deadline = Date.now() + 30_000;
+  while (!release && Date.now() < deadline) {
+    await delay(50);
+    release = acquireIndexLock(indexPath);
+  }
+  if (!release) return undefined;
+  let db: SessionIndexDatabase | undefined;
+  const temp = `${indexPath}.${randomUUID()}.tmp`;
+  try {
+    const failure = readIndexRecoveryFailure(indexPath);
+    let version: number | undefined;
+    let corrupt = false;
+    if (existsSync(indexPath)) {
+      try {
+        const probe = openIndexDatabase(indexPath, { create: false, mode: "read" });
+        try {
+          version = Number(getMetadata(probe, "schema_version"));
+          if (version > INDEX_SCHEMA_VERSION) {
+            disableOlderIndexClient(indexPath);
+            return undefined;
+          }
+          probe.prepare("SELECT COUNT(*) FROM sessions").get();
+        } finally {
+          probe.close();
+        }
+      } catch (error) {
+        if (isSqliteBusy(error)) throw error;
+        corrupt = true;
+      }
+    }
+    if (!manual && failure && Date.now() < failure.nextAllowedAt) {
+      report(indexPath, getIndexRecoveryMessage(indexPath) ?? failure.error);
+      return undefined;
+    }
+    const rebuild = manual || corrupt || version !== INDEX_SCHEMA_VERSION;
+    if (!rebuild && !reconcile) {
+      if (!failure) notices.get(indexPath)?.(undefined);
+      return undefined;
+    }
+    db = openIndexDatabase(rebuild ? temp : indexPath, { create: rebuild });
+    if (rebuild) initializeSchema(db);
+    const skipped: string[] = [];
+    for (const file of listSessionFiles(sessionsDir)) {
+      if (!syncSessionFileWithDb(db, file, "reconcile")) skipped.push(file);
+      await yieldToEventLoop();
+    }
+    const indexedDb = db;
+    const count = (table: string) =>
+      parseTypeBoxValue(
+        ROW_COUNT_SCHEMA,
+        indexedDb.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get(),
+        "Invalid index count",
+      ).count;
+    const result = {
+      sessionCount: count("sessions"),
+      chunkCount: count("session_text_chunks"),
+      indexPath,
+      skipped,
+    };
+    db.transaction(() => {
+      setMetadata(indexedDb, rebuild ? "indexed_at" : "reconciled_at", new Date().toISOString());
+      if (rebuild) setMetadata(indexedDb, "build", BUILD);
+    });
+    db.close();
+    db = undefined;
+    if (rebuild) {
+      if (corrupt) {
+        const backup = `${indexPath}.bad-${Date.now()}-${randomUUID()}`;
+        for (const suffix of ["", "-wal", "-shm", "-journal"]) {
+          if (existsSync(`${indexPath}${suffix}`))
+            renameSync(`${indexPath}${suffix}`, `${backup}${suffix}`);
+        }
+      } else if (existsSync(indexPath)) {
+        // Upgrade legacy WAL indexes before swapping: SQLite checkpoints and
+        // refuses the mode change while another connection still needs its WAL.
+        const previous = openIndexDatabase(indexPath, { create: false });
+        previous.close();
+      }
+      renameSync(temp, indexPath);
+    }
+    rmSync(`${indexPath}.failure.json`, { force: true });
+    shown.delete(indexPath);
+    notices.get(indexPath)?.(undefined);
+    if (skipped.length)
+      report(
+        indexPath,
+        `Session index skipped ${skipped.length} unreadable transcript(s): ${skipped.join(", ")}`,
+      );
+    return result;
+  } catch (error) {
+    const attempts = (readIndexRecoveryFailure(indexPath)?.attempts ?? 0) + 1;
+    const failure: IndexRecoveryFailure = {
+      attempts,
+      nextAllowedAt: Date.now() + (attempts === 1 ? 60_000 : attempts === 2 ? 300_000 : 1_800_000),
+      error: error instanceof Error ? error.message : String(error),
+      build: BUILD,
+    };
+    const record = `${indexPath}.failure.json`;
+    writeFileSync(`${record}.tmp`, JSON.stringify(failure));
+    renameSync(`${record}.tmp`, record);
+    report(indexPath, getIndexRecoveryMessage(indexPath) ?? failure.error);
+    if (manual) throw error;
+    return undefined;
+  } finally {
+    db?.close();
+    for (const suffix of ["", "-wal", "-shm", "-journal"])
+      rmSync(`${temp}${suffix}`, { force: true });
+    release();
+  }
+}

package/extensions/shared/session-index/schema.ts

@@ -22,7 +22,12 @@ export function openIndexDatabase(
 ): SessionIndexDatabase {
   const create = options?.create ?? true;
   const readonly = options?.mode === "read";
-  return openSqlite(dbPath, { create, readonly, timeoutMs: options?.timeoutMs });
+  return openSqlite(dbPath, {
+    create,
+    readonly,
+    timeoutMs: options?.timeoutMs,
+    journalMode: "DELETE",
+  });
 }
 
 export function initializeSchema(db: SessionIndexDatabase): void {
@@ -180,6 +185,9 @@ export function getIndexStatus(dbPath: string): SessionIndexStatus {
   try {
     db = openIndexDatabase(dbPath, { create: false, mode: "read" });
     const schemaVersionRaw = getMetadata(db, "schema_version");
+    if (Number(schemaVersionRaw) !== INDEX_SCHEMA_VERSION) {
+      return { dbPath, exists: true, schemaVersion: Number(schemaVersionRaw) };
+    }
     const lastFullReindexAt = getMetadata(db, "indexed_at");
     const sessionCountRow = parseTypeBoxValue(
       ROW_COUNT_SCHEMA,

package/extensions/shared/session-index/sqlite.ts

@@ -47,7 +47,12 @@ function loadDatabaseConstructor(): SqliteConstructor {
 
 export function openSqlite(
   dbPath: string,
-  options: { create: boolean; readonly?: boolean | undefined; timeoutMs?: number | undefined },
+  options: {
+    create: boolean;
+    readonly?: boolean | undefined;
+    timeoutMs?: number | undefined;
+    journalMode?: "DELETE" | "WAL";
+  },
 ): SqliteDatabase {
   const Database = loadDatabaseConstructor();
   const readonly = options.readonly ?? false;
@@ -66,15 +71,20 @@ export function openSqlite(
       })
     : new Database(dbPath, { readOnly: readonly });
 
-  db.exec(`PRAGMA busy_timeout = ${options.timeoutMs ?? DEFAULT_BUSY_TIMEOUT_MS}`);
-  db.exec("PRAGMA foreign_keys = ON");
-
-  if (!readonly) {
-    // journal_mode is the connection's first write-capable lock acquisition (and
-    // may run WAL recovery), and immediate transactions rely on busy_timeout to
-    // queue behind concurrent writers instead of failing.
-    db.exec("PRAGMA journal_mode = WAL");
-    db.exec("PRAGMA synchronous = NORMAL");
+  try {
+    db.exec(`PRAGMA busy_timeout = ${options.timeoutMs ?? DEFAULT_BUSY_TIMEOUT_MS}`);
+    db.exec("PRAGMA foreign_keys = ON");
+
+    if (!readonly) {
+      // journal_mode is the connection's first write-capable lock acquisition (and
+      // may run WAL recovery), and immediate transactions rely on busy_timeout to
+      // queue behind concurrent writers instead of failing.
+      db.exec(`PRAGMA journal_mode = ${options.journalMode ?? "WAL"}`);
+      db.exec("PRAGMA synchronous = NORMAL");
+    }
+  } catch (error) {
+    db.close();
+    throw error;
   }
 
   return withStatementCache(db);

package/extensions/{session-search/hooks.ts → shared/session-index/sync.ts}

@@ -1,4 +1,14 @@
 import { existsSync, type Stats, statSync } from "node:fs";
+import {
+  createSessionNameChunk,
+  type ExtractedSessionRecord,
+  type ExtractedSessionTail,
+  extractSessionRecord,
+  extractSessionTail,
+  inferSessionOrigin,
+  type SessionFileTouch,
+} from "../../session-search/extract.ts";
+import { deriveSessionRepoRoots } from "../../session-search/normalize.ts";
 import {
   clearSessionChunksBySourceKind,
   clearSessionIndexedData,
@@ -14,17 +24,7 @@ import {
   setMetadata,
   upsertSession,
   withSessionIndex,
-} from "../shared/session-index/index.ts";
-import {
-  createSessionNameChunk,
-  type ExtractedSessionRecord,
-  type ExtractedSessionTail,
-  extractSessionRecord,
-  extractSessionTail,
-  inferSessionOrigin,
-  type SessionFileTouch,
-} from "./extract.ts";
-import { deriveSessionRepoRoots } from "./normalize.ts";
+} from "./index.ts";
 
 export interface SessionHookController {
   handleSessionStart(sessionFile: string | undefined): Promise<boolean>;
@@ -107,32 +107,56 @@ interface TailSyncBaseline extends SessionRow {
   indexedFileAnchor: string;
 }
 
-function syncSessionFileWithDb(
+export function syncSessionFileWithDb(
   db: SessionIndexDatabase,
   sessionFile: string,
   eventType: string,
   sessionOrigin?: SessionOrigin,
 ): boolean {
   const baseline = asTailSyncBaseline(getSessionRowByPath(db, sessionFile));
-  const stat = statSync(sessionFile);
+  let stat: Stats;
+  try {
+    stat = statSync(sessionFile);
+  } catch {
+    return false;
+  }
 
   if (baseline && isIndexCurrent(baseline, stat, sessionOrigin)) {
-    db.transaction(() => writeHookSyncMetadata(db, eventType));
+    if (eventType !== "reconcile") db.transaction(() => writeHookSyncMetadata(db, eventType));
     return true;
   }
 
   if (baseline && stat.size > baseline.indexedFileSize) {
-    const tail = extractSessionTail(sessionFile, baseline);
+    let tail: ExtractedSessionTail | undefined;
+    try {
+      tail = extractSessionTail(sessionFile, baseline);
+    } catch {
+      return false;
+    }
     if (tail && applyTailSync(db, baseline, tail, eventType, sessionOrigin)) {
       return true;
     }
   }
 
-  const extracted = extractSessionRecord(sessionFile);
+  let extracted: ExtractedSessionRecord | undefined;
+  try {
+    extracted = extractSessionRecord(sessionFile);
+  } catch {
+    return false;
+  }
   if (!extracted) {
     return false;
   }
 
+  const existing = getSessionById(db, extracted.sessionId);
+  if (
+    existing &&
+    existing.sessionPath !== sessionFile &&
+    (existing.modifiedAt > extracted.modifiedAt ||
+      (existing.modifiedAt === extracted.modifiedAt && existing.sessionPath > sessionFile))
+  )
+    return true;
+
   applyFullSync(db, extracted, eventType, sessionOrigin);
   return true;
 }

package/extensions/subagents/install.ts

@@ -9,6 +9,7 @@ import type {
   SendMessageRequest,
   SendMessageResult,
 } from "../session-messaging/install.ts";
+import type { SessionWaker } from "../session-messaging/message-router.ts";
 import {
   createSessionCancelTool,
   createSessionSendMessageTool,
@@ -43,7 +44,7 @@ import {
   type SubagentChildSessionState,
 } from "./settle.ts";
 import { shouldMessageSubagent } from "./should-message.ts";
-import { SubagentMessageRouter } from "./wake.ts";
+import { SubagentWaker } from "./wake.ts";
 
 const SUBAGENT_PROMPT_SECTION = "pi_sessions_subagent";
 const SUBAGENT_PROMPT = `You are working as a subagent on one task delegated by a parent session. The handoff defines your task. Work independently, stay within its scope, and do not duplicate work assigned to the parent or another subagent. Use the available tools to complete the task and validate your conclusions. Messages from the parent may refine the task or request a follow-up, but they do not replace your original task with unrelated work.`;
@@ -65,6 +66,7 @@ interface CurrentSubagentSession {
 
 export interface SubagentsHandle extends SessionLifecycle {
   roster: SubagentRoster;
+  waker: SessionWaker;
   getLaunchTargets(): readonly HandoffLaunchTarget[];
   sendMessage(request: SendMessageRequest): Promise<SendMessageResult>;
   cancelSession(sessionId: string): Promise<SubagentCancelResult>;
@@ -79,6 +81,8 @@ export function installSubagents(
     settings: SessionSettings;
     messaging: MessagingHandle;
     readCompactionSettings: (cwd: string) => CompactionThresholdSettings;
+    sendMessage(request: SendMessageRequest): Promise<SendMessageResult>;
+    hasHostedSessions?: () => boolean;
   },
 ): SubagentsHandle {
   pi.registerMessageRenderer(SUBAGENT_REPORT_MESSAGE_CUSTOM_TYPE, renderSubagentReportMessage);
@@ -107,18 +111,12 @@ export function installSubagents(
     reconcile: () => reconciler.reconcile(),
     openSession: openRosterSession,
   });
-  const messageRouter = new SubagentMessageRouter(
-    pi,
-    deps.messaging,
-    () => current?.parent,
-    isCurrentSession,
-    {
-      onMaterialize: (launch) => pi.appendEntry(SUBAGENT_LAUNCHED_CUSTOM_TYPE, launch),
-      afterOwnedSend: async () => {
-        await reconciler.reconcile();
-      },
+  const waker = new SubagentWaker(pi, deps.messaging, () => current?.parent, isCurrentSession, {
+    onMaterialize: (launch) => pi.appendEntry(SUBAGENT_LAUNCHED_CUSTOM_TYPE, launch),
+    afterSend: async () => {
+      await reconciler.reconcile();
     },
-  );
+  });
   const cancellationRouter = new SubagentCancellationRouter(
     pi,
     deps.messaging,
@@ -232,6 +230,7 @@ export function installSubagents(
     pi.registerTool(
       createSessionSendMessageTool(handle, {
         role: isSubagent ? { kind: "subagent" } : { kind: "wakeCapable" },
+        hostedSessions: deps.hasHostedSessions?.() ?? false,
         getCachedRelationTo: deps.messaging.getCachedRelationTo,
         getParentSessionId: () => handle.getParentSessionId(),
       }),
@@ -257,7 +256,8 @@ export function installSubagents(
 
   const handle: SubagentsHandle = {
     roster,
-    sendMessage: (request) => messageRouter.sendMessage(request),
+    waker,
+    sendMessage: deps.sendMessage,
     cancelSession: (sessionId) => cancellationRouter.cancelSession(sessionId),
     shouldMessageSubagent: (sessionId) =>
       shouldMessageSubagent(sessionId, {

package/extensions/subagents/settle.ts

@@ -170,7 +170,7 @@ export function findSelfSubagentIdentity(
   if (
     bootstrap?.launch !== SUBAGENT_LAUNCH ||
     bootstrap.sessionId !== sessionId ||
-    bootstrap.subagent.childSessionId !== sessionId
+    bootstrap.subagent?.childSessionId !== sessionId
   ) {
     return undefined;
   }