pi-hermes-memory 0.7.22 → 0.8.0

package/src/store/canonical-storage-path.ts

@@ -0,0 +1,54 @@
+import fs from "node:fs";
+import path from "node:path";
+
+const MAX_SYMLINK_DEPTH = 40;
+
+function pathParts(absolutePath: string): { root: string; parts: string[] } {
+  const root = path.parse(absolutePath).root;
+  return {
+    root,
+    parts: absolutePath.slice(root.length).split(path.sep).filter(Boolean),
+  };
+}
+
+export function canonicalStoragePathSync(filePath: string): string {
+  const input = path.resolve(filePath);
+  let { root, parts } = pathParts(input);
+  let current = root;
+  let depth = 0;
+
+  while (parts.length > 0) {
+    const part = parts.shift()!;
+    const candidate = path.join(current, part);
+    let state: fs.Stats;
+    try {
+      state = fs.lstatSync(candidate);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+      return path.join(fs.realpathSync.native(current), part, ...parts);
+    }
+
+    if (!state.isSymbolicLink()) {
+      current = candidate;
+      continue;
+    }
+
+    if (depth++ >= MAX_SYMLINK_DEPTH) {
+      const error = new Error(`Symbolic link loop detected while resolving ${input}`) as NodeJS.ErrnoException;
+      error.code = "ELOOP";
+      throw error;
+    }
+    const target = fs.readlinkSync(candidate);
+    const targetPath = path.resolve(path.dirname(candidate), target);
+    const targetParts = pathParts(targetPath);
+    root = targetParts.root;
+    current = root;
+    parts = [...targetParts.parts, ...parts];
+  }
+
+  return fs.realpathSync.native(current);
+}
+
+export async function canonicalStoragePath(filePath: string): Promise<string> {
+  return canonicalStoragePathSync(filePath);
+}

package/src/store/db.ts

@@ -2,6 +2,8 @@ import path from 'node:path';
 import fs from 'node:fs';
 import { createRequire } from 'node:module';
 import { SCHEMA_SQL } from './schema.js';
+import { AtomicLockCoordinator } from './atomic-lock-coordinator.js';
+import { canonicalStoragePathSync } from './canonical-storage-path.js';
 
 type StatementLike = {
   run: (...args: any[]) => any;
@@ -34,12 +36,30 @@ type MovedDatabaseFile = {
 };
 
 export interface DatabaseRecoveryResult {
-  strategy: 'rebuilt' | 'recreated-empty';
+  strategy: 'rebuilt' | 'recreated-empty' | 'reused';
   backupPaths: string[];
   recoveredRows?: Record<string, number>;
   error?: string;
 }
 
+export interface DatabaseRecoveryOptions {
+  recoveryLockWaitMs?: number;
+  recoveryLockPollMs?: number;
+  recoveryLockStaleMs?: number;
+  recoveryCircuitLimit?: number;
+  recoveryCircuitWindowMs?: number;
+  recoveryBackupRetention?: number;
+}
+
+interface ResolvedDatabaseRecoveryOptions {
+  recoveryLockWaitMs: number;
+  recoveryLockPollMs: number;
+  recoveryLockStaleMs: number;
+  recoveryCircuitLimit: number;
+  recoveryCircuitWindowMs: number;
+  recoveryBackupRetention: number;
+}
+
 class DatabaseCorruptionError extends Error {
   code = 'SQLITE_CORRUPT';
 
@@ -54,6 +74,14 @@ export const SQLITE_WAL_AUTOCHECKPOINT_PAGES = 1000;
 const DATABASE_FILE_SUFFIXES: readonly DatabaseFileSuffix[] = ['', '-wal', '-shm'];
 const MEMORY_TARGETS = new Set(['memory', 'user', 'failure']);
 const MEMORY_CATEGORIES = new Set(['failure', 'correction', 'insight', 'preference', 'convention', 'tool-quirk']);
+const DEFAULT_RECOVERY_OPTIONS: ResolvedDatabaseRecoveryOptions = {
+  recoveryLockWaitMs: 5000,
+  recoveryLockPollMs: 50,
+  recoveryLockStaleMs: 300000,
+  recoveryCircuitLimit: 3,
+  recoveryCircuitWindowMs: 300000,
+  recoveryBackupRetention: 3,
+};
 
 function quoteIdentifier(identifier: string): string {
   return `"${identifier.replace(/"/g, '""')}"`;
@@ -110,11 +138,27 @@ const Database = loadDatabaseCtor();
 
 export class DatabaseManager {
   private db: DatabaseLike | null = null;
-  private readonly dbPath: string;
+  private readonly displayDbPath: string;
+  private canonicalDbPath: string | null = null;
+  private readonly recoveryOptions: ResolvedDatabaseRecoveryOptions;
   private lastRecovery: DatabaseRecoveryResult | null = null;
+  private openGuard: (() => void) | null = null;
+  private activeRecoveryLease: { coordinator: AtomicLockCoordinator; key: string; token: string } | null = null;
+
+  constructor(memoryDir: string, recoveryOptions: DatabaseRecoveryOptions = {}) {
+    this.displayDbPath = path.join(memoryDir, 'sessions.db');
+    this.recoveryOptions = { ...DEFAULT_RECOVERY_OPTIONS, ...recoveryOptions };
+  }
+
+  private get dbPath(): string {
+    if (!this.canonicalDbPath) {
+      this.canonicalDbPath = canonicalStoragePathSync(this.displayDbPath);
+    }
+    return this.canonicalDbPath;
+  }
 
-  constructor(memoryDir: string) {
-    this.dbPath = path.join(memoryDir, 'sessions.db');
+  setOpenGuard(guard: (() => void) | null): void {
+    this.openGuard = guard;
   }
 
   /**
@@ -147,6 +191,7 @@ export class DatabaseManager {
    */
   getDb(): DatabaseLike {
     if (!this.db) {
+      this.openGuard?.();
       this.db = this.open();
     }
     return this.db;
@@ -181,7 +226,15 @@ export class DatabaseManager {
    */
   recoverFromCorruption(cause?: unknown): DatabaseRecoveryResult {
     this.close();
-    const recovery = this.recoverDatabaseFile(cause);
+    let verifiedDb: DatabaseLike | null = null;
+    let recovery: DatabaseRecoveryResult;
+    try {
+      recovery = this.recoverDatabaseFile(cause, () => {
+        verifiedDb = this.openUnchecked();
+      });
+    } finally {
+      if (verifiedDb) this.safeClose(verifiedDb);
+    }
     this.lastRecovery = recovery;
     return recovery;
   }
@@ -202,9 +255,19 @@ export class DatabaseManager {
         throw err;
       }
 
-      const recovery = this.recoverDatabaseFile(err);
+      let recoveredDb: DatabaseLike | null = null;
+      let recovery: DatabaseRecoveryResult;
+      try {
+        recovery = this.recoverDatabaseFile(err, () => {
+          recoveredDb = this.openUnchecked();
+        });
+      } catch (error) {
+        if (recoveredDb) this.safeClose(recoveredDb);
+        throw error;
+      }
       this.lastRecovery = recovery;
-      return this.openUnchecked();
+      if (!recoveredDb) throw new Error(`SQLite recovery verification did not open ${this.displayDbPath}`);
+      return recoveredDb;
     }
   }
 
@@ -297,7 +360,57 @@ export class DatabaseManager {
     }
   }
 
-  private recoverDatabaseFile(cause?: unknown): DatabaseRecoveryResult {
+  private recoverDatabaseFile(cause: unknown, verify: () => void): DatabaseRecoveryResult {
+    const coordinator = new AtomicLockCoordinator(path.join(path.dirname(this.dbPath), '.pi-hermes-locks.sqlite'));
+    const lockKey = `recovery:${this.dbPath}`;
+    const deadline = Date.now() + Math.max(0, this.recoveryOptions.recoveryLockWaitMs);
+
+    while (true) {
+      const lease = coordinator.tryAcquire(lockKey, { staleMs: this.recoveryOptions.recoveryLockStaleMs });
+      if (!lease) {
+        if (Date.now() >= deadline) {
+          throw new Error(`SQLite recovery already in progress for ${this.displayDbPath}; timed out after ${this.recoveryOptions.recoveryLockWaitMs}ms`);
+        }
+        DatabaseManager.sleepSync(Math.min(
+          this.recoveryOptions.recoveryLockPollMs,
+          Math.max(1, deadline - Date.now()),
+        ));
+        continue;
+      }
+
+      this.activeRecoveryLease = { coordinator, key: lockKey, token: lease.token };
+      try {
+        if (this.currentDatabaseIsHealthy()) {
+          try {
+            verify();
+            this.clearRecoveryFailuresBestEffort();
+            return { strategy: 'reused', backupPaths: [] };
+          } catch (error) {
+            this.recordRecoveryFailure();
+            throw error;
+          }
+        }
+
+        this.assertRecoveryCircuitClosed();
+        try {
+          this.cleanupRecoveryArtifactsBestEffort();
+          const result = this.recoverDatabaseFileUnlocked(cause);
+          verify();
+          this.cleanupRecoveryArtifactsBestEffort();
+          this.clearRecoveryFailuresBestEffort();
+          return result;
+        } catch (error) {
+          this.recordRecoveryFailure();
+          throw error;
+        }
+      } finally {
+        this.activeRecoveryLease = null;
+        lease.release();
+      }
+    }
+  }
+
+  private recoverDatabaseFileUnlocked(cause?: unknown): DatabaseRecoveryResult {
     const backupBase = this.corruptBackupBase();
     let rebuildError: unknown;
 
@@ -317,6 +430,112 @@ export class DatabaseManager {
     };
   }
 
+  private currentDatabaseIsHealthy(): boolean {
+    if (!this.hasExistingMainDatabaseFile()) return false;
+    let db: DatabaseLike | null = null;
+    try {
+      db = new Database(this.dbPath);
+      this.assertIntegrityOk(db, 'quick_check', 'while joining corruption recovery');
+      return true;
+    } catch {
+      return false;
+    } finally {
+      if (db) this.safeClose(db);
+    }
+  }
+
+  private recoveryCircuitPath(): string {
+    return `${this.dbPath}.recovery-state.json`;
+  }
+
+  private recentRecoveryFailures(): number[] {
+    try {
+      const parsed = JSON.parse(fs.readFileSync(this.recoveryCircuitPath(), 'utf-8')) as { failures?: unknown };
+      if (!Array.isArray(parsed.failures)) return [];
+      const cutoff = Date.now() - Math.max(0, this.recoveryOptions.recoveryCircuitWindowMs);
+      return parsed.failures.filter((value): value is number => typeof value === 'number' && value >= cutoff);
+    } catch {
+      return [];
+    }
+  }
+
+  private assertRecoveryCircuitClosed(): void {
+    if (this.recentRecoveryFailures().length >= Math.max(1, this.recoveryOptions.recoveryCircuitLimit)) {
+      throw new Error(
+        `SQLite recovery circuit is open for ${this.displayDbPath}: too many failed recovery attempts within ${this.recoveryOptions.recoveryCircuitWindowMs}ms`,
+      );
+    }
+  }
+
+  private recordRecoveryFailure(): void {
+    const statePath = this.recoveryCircuitPath();
+    const tempPath = `${statePath}.tmp-${process.pid}-${Math.random().toString(16).slice(2, 8)}`;
+    const failures = [...this.recentRecoveryFailures(), Date.now()];
+    try {
+      fs.writeFileSync(tempPath, JSON.stringify({ failures }), { encoding: 'utf-8', mode: 0o600 });
+      fs.renameSync(tempPath, statePath);
+    } finally {
+      fs.rmSync(tempPath, { force: true });
+    }
+  }
+
+  private clearRecoveryFailures(): void {
+    fs.rmSync(this.recoveryCircuitPath(), { force: true });
+  }
+
+  private clearRecoveryFailuresBestEffort(): void {
+    try { this.clearRecoveryFailures(); } catch {}
+  }
+
+  private cleanupRecoveryArtifactsBestEffort(): void {
+    try { this.cleanupRecoveryArtifacts(); } catch {}
+  }
+
+  private cleanupRecoveryArtifacts(): void {
+    const dir = path.dirname(this.dbPath);
+    const databaseName = path.basename(this.dbPath);
+    let names: string[];
+    try {
+      names = fs.readdirSync(dir);
+    } catch {
+      return;
+    }
+
+    for (const name of names) {
+      if (name.startsWith(`${databaseName}.rebuild-`)) {
+        fs.rmSync(path.join(dir, name), { recursive: true, force: true });
+      }
+    }
+
+    const backupGroups = new Map<string, number>();
+    for (const name of names) {
+      if (!name.startsWith(`${databaseName}.corrupt-`)) continue;
+      const group = name.replace(/-(?:wal|shm)$/, '');
+      try {
+        const mtimeMs = fs.statSync(path.join(dir, name)).mtimeMs;
+        backupGroups.set(group, Math.max(backupGroups.get(group) ?? 0, mtimeMs));
+      } catch {
+        // Artifact disappeared while scanning.
+      }
+    }
+
+    const retained = Math.max(0, this.recoveryOptions.recoveryBackupRetention);
+    const expired = [...backupGroups.entries()]
+      .sort((left, right) => right[1] - left[1])
+      .slice(retained);
+    for (const [group] of expired) {
+      for (const suffix of DATABASE_FILE_SUFFIXES) {
+        fs.rmSync(path.join(dir, `${group}${suffix}`), { force: true });
+      }
+    }
+  }
+
+  private static sleepSync(milliseconds: number): void {
+    if (milliseconds <= 0) return;
+    const signal = new Int32Array(new SharedArrayBuffer(4));
+    Atomics.wait(signal, 0, 0, milliseconds);
+  }
+
   private rebuildDatabaseFromReadableRows(backupBase: string): DatabaseRecoveryResult {
     const tempPath = this.rebuildTempPath();
     this.removeDatabaseFileSet(tempPath);
@@ -555,6 +774,7 @@ export class DatabaseManager {
   }
 
   private moveDatabaseFilesToBackup(backupBase: string): MovedDatabaseFile[] {
+    this.assertStillRecoveryOwner();
     const moved: MovedDatabaseFile[] = [];
     for (const suffix of DATABASE_FILE_SUFFIXES) {
       const original = `${this.dbPath}${suffix}`;
@@ -568,6 +788,21 @@ export class DatabaseManager {
     return moved;
   }
 
+  /**
+   * Verifies this instance still holds the recovery lease immediately before
+   * a destructive rename that has no independent compare-and-swap of its
+   * own (unlike the Markdown mutation path, which re-checks a content
+   * fingerprint at publish time). If the lease was reclaimed as stale while
+   * this call was in flight, abort rather than race the new owner.
+   */
+  private assertStillRecoveryOwner(): void {
+    const active = this.activeRecoveryLease;
+    if (!active) return;
+    if (!active.coordinator.isCurrentOwner(active.key, active.token)) {
+      throw new Error(`SQLite recovery lease lost for ${this.displayDbPath}; another process took over`);
+    }
+  }
+
   private restoreMovedDatabaseFiles(moved: MovedDatabaseFile[]): void {
     for (const file of [...moved].reverse()) {
       try {
@@ -765,7 +1000,7 @@ export class DatabaseManager {
    * Get the database file path.
    */
   getPath(): string {
-    return this.dbPath;
+    return this.displayDbPath;
   }
 
   /**

package/src/store/markdown-mutation-lock.ts

@@ -0,0 +1,38 @@
+import * as path from "node:path";
+import { AtomicLockCoordinator, type AtomicLockLease } from "./atomic-lock-coordinator.js";
+import { canonicalStoragePath } from "./canonical-storage-path.js";
+
+const MUTATION_WAIT_MS = 5_000;
+const MUTATION_STALE_MS = 300_000;
+
+export async function canonicalMarkdownIdentity(filePath: string): Promise<string> {
+  return canonicalStoragePath(filePath);
+}
+
+export async function acquireMarkdownMutationLock(filePath: string): Promise<AtomicLockLease> {
+  const identity = await canonicalMarkdownIdentity(filePath);
+  const coordinatorDir = path.dirname(path.dirname(identity));
+  const coordinator = new AtomicLockCoordinator(path.join(coordinatorDir, ".pi-hermes-locks.sqlite"));
+  const lockKey = `mutation:${identity}`;
+  const deadline = Date.now() + MUTATION_WAIT_MS;
+  let lease = coordinator.tryAcquire(lockKey, { staleMs: MUTATION_STALE_MS });
+
+  while (!lease) {
+    if (Date.now() >= deadline) {
+      throw new Error(`Memory mutation already in progress for ${identity}`);
+    }
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    lease = coordinator.tryAcquire(lockKey, { staleMs: MUTATION_STALE_MS });
+  }
+
+  return lease;
+}
+
+export async function withMarkdownMutationLock<T>(filePath: string, operation: () => Promise<T> | T): Promise<T> {
+  const lease = await acquireMarkdownMutationLock(filePath);
+  try {
+    return await operation();
+  } finally {
+    lease.release();
+  }
+}