pi-para 0.7.0 → 0.7.2

package/src/scheduler/index.ts

@@ -158,6 +158,11 @@ export class WikiScheduler {
     // No real embedding provider configured — embed calls would hit the inert
     // shim endpoint (slow TCP timeouts on Windows) and always fail. Skip.
     if (!storeHasApiProviders(store)) return;
+    // A credential/billing failure (401/402/403) cannot heal by retrying every
+    // tick and every session start; it needs a human (top up credits, fix the
+    // key). Pause embedding durably and resume automatically after backoff.
+    const backoffUntil = this.state.getValue(EMBED_BACKOFF_KEY);
+    if (backoffUntil && Date.parse(backoffUntil) > Date.now()) return;
 
     const status = await store.getStatus();
     const pending = status.needsEmbedding ?? 0;
@@ -170,18 +175,39 @@ export class WikiScheduler {
       const diagnostics = [...new Set(attempt.diagnostics)].join(" | ");
       if (!attempt.ok) {
         const reason = attempt.error instanceof Error ? attempt.error.message : String(attempt.error);
-        throw new Error(diagnostics ? `${reason} — ${diagnostics}` : reason);
+        throw this.embedFailure(diagnostics ? `${reason} — ${diagnostics}` : reason);
       }
       const result = attempt.value;
       if (result.errors > 0) {
-        throw new Error(
+        throw this.embedFailure(
           `qmd embed finished with ${result.errors} error(s) (${result.chunksEmbedded} chunk(s) embedded, ${pending} doc(s) were pending)${diagnostics ? ` — ${diagnostics}` : ""}`,
         );
       }
+      this.state.deleteValue(EMBED_BACKOFF_KEY);
     } finally {
       releaseLease(this.state.db, leaseKey, this.holderId);
     }
   }
+
+  /** Record durable backoff for credential/billing embed failures, then return the error. */
+  private embedFailure(message: string): Error {
+    if (isCredentialEmbedError(message)) {
+      const until = new Date(Date.now() + EMBED_CREDENTIAL_BACKOFF_MS).toISOString();
+      try {
+        this.state.setValue(EMBED_BACKOFF_KEY, until);
+      } catch { /* backoff bookkeeping is best-effort */ }
+      return new Error(`${message} — embedding paused until ${until} (credential/billing error; fix the provider and it resumes automatically, BM25 search keeps working)`);
+    }
+    return new Error(message);
+  }
+}
+
+const EMBED_BACKOFF_KEY = "qmd-embed-backoff-until";
+const EMBED_CREDENTIAL_BACKOFF_MS = 6 * 60 * 60_000;
+
+/** Credential/billing embed failures (401/402/403) that retries cannot fix. */
+export function isCredentialEmbedError(message: string): boolean {
+  return /\bAPI 40[123]\b/.test(message);
 }
 
 const schedulers = new Map<string, WikiScheduler>();

package/src/scheduler/session-capture.ts

@@ -1,5 +1,5 @@
-import { appendFile, readFile } from "node:fs/promises";
-import { existsSync, readFileSync } from "node:fs";
+import { appendFile, readFile, rename, writeFile } from "node:fs/promises";
+import { existsSync, readFileSync, statSync } from "node:fs";
 import { basename, dirname, join } from "node:path";
 import type { Model } from "@earendil-works/pi-ai";
 import type { QMDStore } from "qmd-engine";
@@ -62,31 +62,93 @@ export async function appendCompletedSession(
   return { timestamp, sessionPath };
 }
 
+/**
+ * A session file modified this recently is treated as still active: capturing
+ * it now would both re-read a growing file and produce a summary that is
+ * immediately stale, so it stays in the registry for a later pass.
+ */
+export const ACTIVE_SESSION_QUIET_MS = 10 * 60_000;
+
 export async function enqueueCompletedSessionsFromRegistry(
   scheduler: WikiScheduler,
   wikiDir: string,
-  opts: { stateDb?: StateDB } = {},
+  opts: { stateDb?: StateDB; quietMs?: number } = {},
 ): Promise<number[]> {
   const entries = await readCompletedSessionRegistry(wikiDir);
-  return enqueueCompletedSessions(scheduler, entries, opts);
+  const { ids, survivors } = scanCompletedSessions(scheduler, entries, opts);
+  // Compact the registry: keep one line per still-relevant session so the
+  // file (and the startup scan) cannot grow without bound. Processed and
+  // deleted sessions are dropped; unprocessed and still-active ones survive.
+  if (survivors.length < entries.length) {
+    await compactCompletedSessionRegistry(wikiDir, entries, survivors);
+  }
+  return ids;
 }
 
 export function enqueueCompletedSessions(
   scheduler: WikiScheduler,
   entries: CompletedSessionEntry[],
-  opts: { stateDb?: StateDB } = {},
+  opts: { stateDb?: StateDB; quietMs?: number } = {},
 ): number[] {
+  return scanCompletedSessions(scheduler, entries, opts).ids;
+}
+
+function scanCompletedSessions(
+  scheduler: WikiScheduler,
+  entries: CompletedSessionEntry[],
+  opts: { stateDb?: StateDB; quietMs?: number } = {},
+): { ids: number[]; survivors: CompletedSessionEntry[] } {
+  const quietMs = opts.quietMs ?? ACTIVE_SESSION_QUIET_MS;
   const ids: number[] = [];
+  const survivors: CompletedSessionEntry[] = [];
+  const seen = new Set<string>();
   for (const entry of entries) {
+    if (seen.has(entry.sessionPath)) continue;
+    seen.add(entry.sessionPath);
     if (!existsSync(entry.sessionPath)) continue;
+    // Still-active session: skip this pass but keep it registered.
+    try {
+      if (Date.now() - statSync(entry.sessionPath).mtimeMs < quietMs) {
+        survivors.push(entry);
+        continue;
+      }
+    } catch {
+      continue;
+    }
     if (opts.stateDb?.isProcessed(entry.sessionPath)) continue;
+    survivors.push(entry);
     ids.push(scheduler.enqueue(
       "capture-session",
       { sessionPath: entry.sessionPath, scope: detectScopeFromSessionPath(entry.sessionPath) } satisfies CaptureSessionPayload,
       { dedupeKey: `capture-session:${entry.sessionPath}`, priority: 20 },
     ));
   }
-  return ids;
+  return { ids, survivors };
+}
+
+/**
+ * Atomically rewrite the registry with the surviving entries, merging any
+ * lines appended concurrently since `snapshot` was read.
+ */
+async function compactCompletedSessionRegistry(
+  wikiDir: string,
+  snapshot: CompletedSessionEntry[],
+  survivors: CompletedSessionEntry[],
+): Promise<void> {
+  const registryPath = join(wikiDir, ".completed-sessions");
+  try {
+    const snapshotPaths = new Set(snapshot.map((entry) => entry.sessionPath));
+    const current = await readCompletedSessionRegistry(wikiDir);
+    const appendedSinceSnapshot = current.filter((entry) => !snapshotPaths.has(entry.sessionPath));
+    const lines = [...survivors, ...appendedSinceSnapshot]
+      .map((entry) => `${entry.timestamp}|${entry.sessionPath}`)
+      .join("\n");
+    const tmpPath = `${registryPath}.tmp-${process.pid}`;
+    await writeFile(tmpPath, lines ? `${lines}\n` : "", "utf-8");
+    await rename(tmpPath, registryPath);
+  } catch {
+    // Compaction is an optimization; the registry stays valid without it.
+  }
 }
 
 export function createCaptureSessionHandler(deps: CaptureSessionDeps): SchedulerTaskHandler {

package/src/scheduler/state.ts

@@ -89,9 +89,30 @@ export class SchedulerStateDB {
         duration_ms INTEGER,
         error TEXT
       );
+
+      CREATE TABLE IF NOT EXISTS scheduler_kv (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL,
+        updated_at TEXT NOT NULL
+      );
     `);
   }
 
+  getValue(key: string): string | null {
+    const row = this.db.prepare("SELECT value FROM scheduler_kv WHERE key = ?").get(key) as { value: string } | undefined;
+    return row?.value ?? null;
+  }
+
+  setValue(key: string, value: string): void {
+    this.db
+      .prepare("INSERT OR REPLACE INTO scheduler_kv (key, value, updated_at) VALUES (?, ?, ?)")
+      .run(key, value, new Date().toISOString());
+  }
+
+  deleteValue(key: string): void {
+    this.db.prepare("DELETE FROM scheduler_kv WHERE key = ?").run(key);
+  }
+
   enqueue(taskName: string, payload: unknown, opts: EnqueueOptions = {}): number {
     const now = opts.now ?? new Date();
     const availableAt = opts.availableAt ?? now;

package/src/state.ts

@@ -6,7 +6,7 @@
 
 import Database from "better-sqlite3";
 import { createHash } from "node:crypto";
-import { readFileSync } from "node:fs";
+import { readFileSync, statSync } from "node:fs";
 import { join } from "node:path";
 
 export interface PageSummary {
@@ -62,17 +62,48 @@ export class StateDB {
         updated_at TEXT NOT NULL
       );
     `);
+    // Stat fast path for isProcessed(): when size+mtime are unchanged since
+    // the recorded hash, skip re-hashing the whole session file. Critical for
+    // startup catch-up scans over registries with very large session JSONLs.
+    try {
+      this.db.exec("ALTER TABLE processed_sessions ADD COLUMN file_size INTEGER");
+    } catch { /* column exists */ }
+    try {
+      this.db.exec("ALTER TABLE processed_sessions ADD COLUMN file_mtime_ms INTEGER");
+    } catch { /* column exists */ }
   }
 
-  /** Check if a session file has already been processed with the same content. */
+  /**
+   * Check if a session file has already been processed with the same content.
+   * Uses a stat (size+mtime) fast path before falling back to a full content
+   * hash, so registry-wide scans stay cheap even with multi-hundred-MB
+   * session files. A hash match under a changed stat refreshes the cache.
+   */
   isProcessed(sessionPath: string): boolean {
     const row = this.db
-      .prepare("SELECT content_hash FROM processed_sessions WHERE session_path = ?")
-      .get(sessionPath) as { content_hash: string } | undefined;
+      .prepare("SELECT content_hash, file_size, file_mtime_ms FROM processed_sessions WHERE session_path = ?")
+      .get(sessionPath) as { content_hash: string; file_size: number | null; file_mtime_ms: number | null } | undefined;
     if (!row) return false;
 
+    let stat: { size: number; mtimeMs: number };
+    try {
+      stat = statSync(sessionPath);
+    } catch {
+      return false;
+    }
+    // mtimeMs is a float and loses sub-millisecond precision across
+    // utimes/copy round-trips; compare with a small tolerance. Size is exact.
+    if (row.file_size !== null && row.file_mtime_ms !== null &&
+      row.file_size === stat.size && Math.abs(row.file_mtime_ms - stat.mtimeMs) <= 2) {
+      return true;
+    }
+
     const currentHash = hashFile(sessionPath);
-    return row.content_hash === currentHash;
+    if (row.content_hash !== currentHash) return false;
+    this.db
+      .prepare("UPDATE processed_sessions SET file_size = ?, file_mtime_ms = ? WHERE session_path = ?")
+      .run(stat.size, Math.floor(stat.mtimeMs), sessionPath);
+    return true;
   }
 
   /** Get the stored hash for a session (or null if not processed). */
@@ -91,11 +122,12 @@ export class StateDB {
     pagesUpdated: string[],
   ): void {
     const hash = hashFile(sessionPath);
+    const stat = statFile(sessionPath);
     this.db
       .prepare(`
         INSERT OR REPLACE INTO processed_sessions
-        (session_path, content_hash, processed_at, scope, pages_created, pages_updated, error)
-        VALUES (?, ?, ?, ?, ?, ?, NULL)
+        (session_path, content_hash, processed_at, scope, pages_created, pages_updated, error, file_size, file_mtime_ms)
+        VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?)
       `)
       .run(
         sessionPath,
@@ -104,19 +136,23 @@ export class StateDB {
         scope,
         JSON.stringify(pagesCreated),
         JSON.stringify(pagesUpdated),
+        stat?.size ?? null,
+        stat ? Math.floor(stat.mtimeMs) : null,
       );
   }
 
   /** Record a failed processing attempt. */
   recordFailure(sessionPath: string, scope: string, error: string): void {
     const hash = hashFile(sessionPath);
+    const stat = statFile(sessionPath);
     this.db
       .prepare(`
         INSERT OR REPLACE INTO processed_sessions
-        (session_path, content_hash, processed_at, scope, pages_created, pages_updated, error)
-        VALUES (?, ?, ?, ?, '[]', '[]', ?)
+        (session_path, content_hash, processed_at, scope, pages_created, pages_updated, error, file_size, file_mtime_ms)
+        VALUES (?, ?, ?, ?, '[]', '[]', ?, ?, ?)
       `)
-      .run(sessionPath, hash, new Date().toISOString(), scope, error);
+      .run(sessionPath, hash, new Date().toISOString(), scope, error,
+        stat?.size ?? null, stat ? Math.floor(stat.mtimeMs) : null);
   }
 
   /** Get all failed sessions for retry. */
@@ -241,6 +277,14 @@ export class StateDB {
   }
 }
 
+function statFile(filePath: string): { size: number; mtimeMs: number } | null {
+  try {
+    return statSync(filePath);
+  } catch {
+    return null;
+  }
+}
+
 function hashFile(filePath: string): string {
   try {
     const content = readFileSync(filePath);

package/src/wiki.ts

@@ -185,6 +185,7 @@ gepa/output/
 
 # local caches/logs
 *.tmp
+.capture.log*
 `;
 
 // -- Helpers -----------------------------------------------------------------