swarm-mail 0.1.0

package/src/streams/migrations.test.ts

@@ -0,0 +1,359 @@
+/**
+ * Tests for Schema Migration System
+ */
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { PGlite } from "@electric-sql/pglite";
+import {
+  runMigrations,
+  getCurrentVersion,
+  rollbackTo,
+  isMigrationApplied,
+  getPendingMigrations,
+  getAppliedMigrations,
+  migrations,
+} from "./migrations";
+
+describe("Schema Migrations", () => {
+  let db: PGlite;
+
+  beforeEach(async () => {
+    // Use in-memory database for tests
+    db = new PGlite();
+  });
+
+  afterEach(async () => {
+    await db.close();
+  });
+
+  describe("Fresh Install", () => {
+    it("should start with version 0", async () => {
+      const version = await getCurrentVersion(db);
+      expect(version).toBe(0);
+    });
+
+    it("should run all migrations on fresh database", async () => {
+      const result = await runMigrations(db);
+
+      expect(result.applied).toEqual([1, 2, 3, 4]);
+      expect(result.current).toBe(4);
+
+      const version = await getCurrentVersion(db);
+      expect(version).toBe(4);
+    });
+
+    it("should create cursors table with correct schema", async () => {
+      await runMigrations(db);
+
+      // Verify table exists
+      const tableResult = await db.query<{ exists: boolean }>(
+        `SELECT EXISTS (
+          SELECT FROM information_schema.tables 
+          WHERE table_name = 'cursors'
+        ) as exists`,
+      );
+      expect(tableResult.rows[0]?.exists).toBe(true);
+
+      // Verify columns
+      const columnsResult = await db.query<{ column_name: string }>(
+        `SELECT column_name FROM information_schema.columns
+         WHERE table_name = 'cursors'
+         ORDER BY ordinal_position`,
+      );
+      const columns = columnsResult.rows.map((r) => r.column_name);
+      expect(columns).toContain("id");
+      expect(columns).toContain("stream");
+      expect(columns).toContain("checkpoint");
+      expect(columns).toContain("position");
+      expect(columns).toContain("updated_at");
+
+      // Verify unique constraint exists
+      const constraintsResult = await db.query<{ constraint_name: string }>(
+        `SELECT constraint_name FROM information_schema.table_constraints
+         WHERE table_name = 'cursors' AND constraint_type = 'UNIQUE'`,
+      );
+      expect(constraintsResult.rows.length).toBeGreaterThan(0);
+    });
+
+    it("should create deferred table with correct schema", async () => {
+      await runMigrations(db);
+
+      const tableResult = await db.query<{ exists: boolean }>(
+        `SELECT EXISTS (
+          SELECT FROM information_schema.tables 
+          WHERE table_name = 'deferred'
+        ) as exists`,
+      );
+      expect(tableResult.rows[0]?.exists).toBe(true);
+
+      const columnsResult = await db.query<{ column_name: string }>(
+        `SELECT column_name FROM information_schema.columns
+         WHERE table_name = 'deferred'
+         ORDER BY ordinal_position`,
+      );
+      const columns = columnsResult.rows.map((r) => r.column_name);
+      expect(columns).toContain("id");
+      expect(columns).toContain("url");
+      expect(columns).toContain("resolved");
+      expect(columns).toContain("value");
+      expect(columns).toContain("error");
+      expect(columns).toContain("expires_at");
+      expect(columns).toContain("created_at");
+    });
+  });
+
+  describe("Idempotency", () => {
+    it("should be safe to run migrations multiple times", async () => {
+      // First run
+      const result1 = await runMigrations(db);
+      expect(result1.applied).toEqual([1, 2, 3, 4]);
+
+      // Second run - should apply nothing
+      const result2 = await runMigrations(db);
+      expect(result2.applied).toEqual([]);
+      expect(result2.current).toBe(4);
+
+      // Version should still be 2
+      const version = await getCurrentVersion(db);
+      expect(version).toBe(4);
+    });
+  });
+
+  describe("Incremental Upgrade", () => {
+    it("should apply only new migrations", async () => {
+      // Manually apply migration 1
+      await db.exec(migrations[0]!.up);
+      await db.exec(`
+        CREATE TABLE IF NOT EXISTS schema_version (
+          version INTEGER PRIMARY KEY,
+          applied_at BIGINT NOT NULL,
+          description TEXT
+        );
+      `);
+      await db.query(
+        `INSERT INTO schema_version (version, applied_at, description)
+         VALUES ($1, $2, $3)`,
+        [1, Date.now(), migrations[0]!.description],
+      );
+
+      // Now run migrations - should only apply 2
+      const result = await runMigrations(db);
+      expect(result.applied).toEqual([2, 3, 4]);
+      expect(result.current).toBe(4);
+    });
+  });
+
+  describe("Rollback", () => {
+    it("should rollback to target version", async () => {
+      // Apply all migrations
+      await runMigrations(db);
+      expect(await getCurrentVersion(db)).toBe(4);
+
+      // Rollback to version 1
+      const result = await rollbackTo(db, 1);
+      expect(result.rolledBack).toEqual([4, 3, 2]);
+      expect(result.current).toBe(1);
+
+      // Version should be 1
+      const version = await getCurrentVersion(db);
+      expect(version).toBe(1);
+
+      // Cursors table should still exist
+      const cursorsExists = await db.query<{ exists: boolean }>(
+        `SELECT EXISTS (
+          SELECT FROM information_schema.tables 
+          WHERE table_name = 'cursors'
+        ) as exists`,
+      );
+      expect(cursorsExists.rows[0]?.exists).toBe(true);
+
+      // Deferred table should be gone
+      const deferredExists = await db.query<{ exists: boolean }>(
+        `SELECT EXISTS (
+          SELECT FROM information_schema.tables 
+          WHERE table_name = 'deferred'
+        ) as exists`,
+      );
+      expect(deferredExists.rows[0]?.exists).toBe(false);
+    });
+
+    it("should rollback to version 0", async () => {
+      await runMigrations(db);
+
+      const result = await rollbackTo(db, 0);
+      expect(result.rolledBack).toEqual([4, 3, 2, 1]);
+      expect(result.current).toBe(0);
+
+      // All tables should be gone
+      const cursorsExists = await db.query<{ exists: boolean }>(
+        `SELECT EXISTS (
+          SELECT FROM information_schema.tables 
+          WHERE table_name = 'cursors'
+        ) as exists`,
+      );
+      expect(cursorsExists.rows[0]?.exists).toBe(false);
+    });
+
+    it("should do nothing if target version >= current", async () => {
+      await runMigrations(db);
+
+      const result = await rollbackTo(db, 4);
+      expect(result.rolledBack).toEqual([]);
+      expect(result.current).toBe(4);
+    });
+  });
+
+  describe("Migration Status", () => {
+    it("should check if migration is applied", async () => {
+      expect(await isMigrationApplied(db, 1)).toBe(false);
+
+      await runMigrations(db);
+
+      expect(await isMigrationApplied(db, 1)).toBe(true);
+      expect(await isMigrationApplied(db, 2)).toBe(true);
+      expect(await isMigrationApplied(db, 3)).toBe(true);
+      expect(await isMigrationApplied(db, 4)).toBe(true);
+      expect(await isMigrationApplied(db, 3)).toBe(true);
+      expect(await isMigrationApplied(db, 4)).toBe(true);
+    });
+
+    it("should list pending migrations", async () => {
+      const pending1 = await getPendingMigrations(db);
+      expect(pending1).toHaveLength(4);
+      expect(pending1.map((m) => m.version)).toEqual([1, 2, 3, 4]);
+
+      // Apply migration 1
+      const migration = migrations[0];
+      if (!migration) throw new Error("Migration not found");
+
+      await db.exec(migration.up);
+      await db.exec(`
+        CREATE TABLE IF NOT EXISTS schema_version (
+          version INTEGER PRIMARY KEY,
+          applied_at BIGINT NOT NULL,
+          description TEXT
+        );
+      `);
+      await db.query(
+        `INSERT INTO schema_version (version, applied_at, description)
+         VALUES ($1, $2, $3)`,
+        [1, Date.now(), migration.description],
+      );
+
+      const pending2 = await getPendingMigrations(db);
+      expect(pending2).toHaveLength(3);
+      expect(pending2.map((m) => m.version)).toEqual([2, 3, 4]);
+    });
+
+    it("should list applied migrations", async () => {
+      const applied1 = await getAppliedMigrations(db);
+      expect(applied1).toHaveLength(0);
+
+      await runMigrations(db);
+
+      const applied2 = await getAppliedMigrations(db);
+      expect(applied2).toHaveLength(4);
+      expect(applied2.map((m) => m.version)).toEqual([1, 2, 3, 4]);
+      expect(applied2[0]?.description).toBe(
+        "Add cursors table for DurableCursor",
+      );
+    });
+  });
+
+  describe("Data Persistence", () => {
+    it("should preserve data across migrations", async () => {
+      // Apply migration 1 (cursors table)
+      await db.exec(migrations[0]!.up);
+      await db.exec(`
+        CREATE TABLE IF NOT EXISTS schema_version (
+          version INTEGER PRIMARY KEY,
+          applied_at BIGINT NOT NULL,
+          description TEXT
+        );
+      `);
+      await db.query(
+        `INSERT INTO schema_version (version, applied_at, description)
+         VALUES ($1, $2, $3)`,
+        [1, Date.now(), migrations[0]!.description],
+      );
+
+      // Insert test data
+      await db.query(
+        `INSERT INTO cursors (stream, checkpoint, position, updated_at)
+         VALUES ($1, $2, $3, $4)`,
+        ["test-stream", "test-checkpoint", 42, Date.now()],
+      );
+
+      // Apply remaining migrations
+      await runMigrations(db);
+
+      // Data should still be there
+      const result = await db.query<{ position: number }>(
+        `SELECT position FROM cursors WHERE stream = $1`,
+        ["test-stream"],
+      );
+      expect(result.rows[0]?.position).toBe(42);
+    });
+  });
+
+  describe("Error Handling", () => {
+    it("should rollback failed migrations", async () => {
+      // Apply good migration first
+      const migration = migrations[0];
+      if (!migration) throw new Error("Migration not found");
+
+      await db.exec(migration.up);
+      await db.exec(`
+        CREATE TABLE IF NOT EXISTS schema_version (
+          version INTEGER PRIMARY KEY,
+          applied_at BIGINT NOT NULL,
+          description TEXT
+        );
+      `);
+      await db.query(
+        `INSERT INTO schema_version (version, applied_at, description)
+         VALUES ($1, $2, $3)`,
+        [1, Date.now(), migration.description],
+      );
+
+      // Try to run invalid SQL in a transaction
+      try {
+        await db.exec("BEGIN");
+        await db.exec("THIS IS INVALID SQL");
+        await db.exec("COMMIT");
+        throw new Error("Should have thrown");
+      } catch {
+        await db.exec("ROLLBACK");
+        // Expected to fail
+      }
+
+      // Version should still be 1
+      const version = await getCurrentVersion(db);
+      expect(version).toBe(1);
+    });
+  });
+
+  describe("Schema Version Table", () => {
+    it("should record migration metadata", async () => {
+      await runMigrations(db);
+
+      const result = await db.query<{
+        version: number;
+        applied_at: string;
+        description: string;
+      }>(
+        `SELECT version, applied_at, description FROM schema_version ORDER BY version`,
+      );
+
+      expect(result.rows).toHaveLength(4);
+      expect(result.rows[0]?.version).toBe(1);
+      expect(result.rows[0]?.description).toBe(
+        "Add cursors table for DurableCursor",
+      );
+      expect(result.rows[1]?.version).toBe(2);
+
+      // Applied_at should be recent
+      const appliedAt = parseInt(result.rows[0]?.applied_at as string);
+      expect(appliedAt).toBeGreaterThan(Date.now() - 10000);
+    });
+  });
+});

package/src/streams/migrations.ts

@@ -0,0 +1,362 @@
+/**
+ * Schema Migration System
+ *
+ * Handles database schema evolution for the PGLite event store.
+ *
+ * ## How It Works
+ *
+ * 1. Each migration has a unique version number (incrementing integer)
+ * 2. On startup, `runMigrations()` checks current schema version
+ * 3. Migrations are applied in order until schema is current
+ * 4. Version is stored in `schema_version` table
+ *
+ * ## Adding a New Migration
+ *
+ * ```typescript
+ * // In migrations.ts
+ * export const migrations: Migration[] = [
+ *   // ... existing migrations
+ *   {
+ *     version: 3,
+ *     description: "add_new_column",
+ *     up: `ALTER TABLE events ADD COLUMN new_col TEXT`,
+ *     down: `ALTER TABLE events DROP COLUMN new_col`,
+ *   },
+ * ];
+ * ```
+ *
+ * ## Rollback
+ *
+ * Rollback is supported via `rollbackTo(db, targetVersion)`.
+ * Note: Some migrations may not be fully reversible (data loss).
+ *
+ * ## Best Practices
+ *
+ * - Always test migrations on a copy of production data
+ * - Keep migrations small and focused
+ * - Include both `up` and `down` SQL
+ * - Use transactions for multi-statement migrations
+ * - Document any data transformations
+ *
+ * @module migrations
+ */
+import type { PGlite } from "@electric-sql/pglite";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * A database migration definition.
+ */
+export interface Migration {
+  /** Unique version number (must be sequential) */
+  version: number;
+  /** Human-readable migration description */
+  description: string;
+  /** SQL to apply the migration */
+  up: string;
+  /** SQL to rollback the migration (best effort) */
+  down: string;
+}
+
+interface SchemaVersion {
+  version: number;
+  applied_at: number;
+  description: string | null;
+}
+
+// ============================================================================
+// Migration Definitions
+// ============================================================================
+
+export const migrations: Migration[] = [
+  {
+    version: 1,
+    description: "Add cursors table for DurableCursor",
+    up: `
+      CREATE TABLE IF NOT EXISTS cursors (
+        id SERIAL PRIMARY KEY,
+        stream TEXT NOT NULL,
+        checkpoint TEXT NOT NULL,
+        position BIGINT NOT NULL DEFAULT 0,
+        updated_at BIGINT NOT NULL,
+        UNIQUE(stream, checkpoint)
+      );
+      CREATE INDEX IF NOT EXISTS idx_cursors_checkpoint ON cursors(checkpoint);
+      CREATE INDEX IF NOT EXISTS idx_cursors_stream ON cursors(stream);
+    `,
+    down: `DROP TABLE IF EXISTS cursors;`,
+  },
+  {
+    version: 2,
+    description: "Add deferred table for DurableDeferred",
+    up: `
+      CREATE TABLE IF NOT EXISTS deferred (
+        id SERIAL PRIMARY KEY,
+        url TEXT NOT NULL UNIQUE,
+        resolved BOOLEAN NOT NULL DEFAULT FALSE,
+        value JSONB,
+        error TEXT,
+        expires_at BIGINT NOT NULL,
+        created_at BIGINT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_deferred_url ON deferred(url);
+      CREATE INDEX IF NOT EXISTS idx_deferred_expires ON deferred(expires_at);
+      CREATE INDEX IF NOT EXISTS idx_deferred_resolved ON deferred(resolved);
+    `,
+    down: `DROP TABLE IF EXISTS deferred;`,
+  },
+  {
+    version: 3,
+    description: "Add eval_records table for learning system",
+    up: `
+      CREATE TABLE IF NOT EXISTS eval_records (
+        id TEXT PRIMARY KEY,
+        project_key TEXT NOT NULL,
+        task TEXT NOT NULL,
+        context TEXT,
+        strategy TEXT NOT NULL,
+        epic_title TEXT NOT NULL,
+        subtasks JSONB NOT NULL,
+        outcomes JSONB,
+        overall_success BOOLEAN,
+        total_duration_ms INTEGER,
+        total_errors INTEGER,
+        human_accepted BOOLEAN,
+        human_modified BOOLEAN,
+        human_notes TEXT,
+        file_overlap_count INTEGER,
+        scope_accuracy REAL,
+        time_balance_ratio REAL,
+        created_at BIGINT NOT NULL,
+        updated_at BIGINT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_eval_records_project ON eval_records(project_key);
+      CREATE INDEX IF NOT EXISTS idx_eval_records_strategy ON eval_records(strategy);
+    `,
+    down: `DROP TABLE IF EXISTS eval_records;`,
+  },
+  {
+    version: 4,
+    description: "Add swarm_contexts table for context recovery",
+    up: `
+      CREATE TABLE IF NOT EXISTS swarm_contexts (
+        id TEXT PRIMARY KEY,
+        epic_id TEXT NOT NULL,
+        bead_id TEXT NOT NULL,
+        strategy TEXT NOT NULL,
+        files JSONB NOT NULL,
+        dependencies JSONB NOT NULL,
+        directives JSONB NOT NULL,
+        recovery JSONB NOT NULL,
+        created_at BIGINT NOT NULL,
+        updated_at BIGINT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_swarm_contexts_epic ON swarm_contexts(epic_id);
+      CREATE INDEX IF NOT EXISTS idx_swarm_contexts_bead ON swarm_contexts(bead_id);
+    `,
+    down: `DROP TABLE IF EXISTS swarm_contexts;`,
+  },
+];
+
+// ============================================================================
+// Migration Execution
+// ============================================================================
+
+/**
+ * Initialize schema_version table if it doesn't exist
+ */
+async function ensureVersionTable(db: PGlite): Promise<void> {
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS schema_version (
+      version INTEGER PRIMARY KEY,
+      applied_at BIGINT NOT NULL,
+      description TEXT
+    );
+  `);
+}
+
+/**
+ * Get the current schema version
+ *
+ * Returns 0 if no migrations have been applied
+ */
+export async function getCurrentVersion(db: PGlite): Promise<number> {
+  await ensureVersionTable(db);
+
+  const result = await db.query<{ version: number }>(
+    `SELECT MAX(version) as version FROM schema_version`,
+  );
+
+  return result.rows[0]?.version ?? 0;
+}
+
+/**
+ * Get all applied migrations
+ */
+export async function getAppliedMigrations(
+  db: PGlite,
+): Promise<SchemaVersion[]> {
+  await ensureVersionTable(db);
+
+  const result = await db.query<{
+    version: number;
+    applied_at: string;
+    description: string | null;
+  }>(
+    `SELECT version, applied_at, description FROM schema_version ORDER BY version ASC`,
+  );
+
+  return result.rows.map((row) => ({
+    version: row.version,
+    applied_at: parseInt(row.applied_at as string),
+    description: row.description,
+  }));
+}
+
+/**
+ * Run all pending migrations
+ *
+ * Idempotent - safe to run multiple times.
+ * Only runs migrations that haven't been applied yet.
+ */
+export async function runMigrations(db: PGlite): Promise<{
+  applied: number[];
+  current: number;
+}> {
+  await ensureVersionTable(db);
+
+  const currentVersion = await getCurrentVersion(db);
+  const applied: number[] = [];
+
+  // Find migrations that need to be applied
+  const pendingMigrations = migrations.filter(
+    (m) => m.version > currentVersion,
+  );
+
+  if (pendingMigrations.length === 0) {
+    return { applied: [], current: currentVersion };
+  }
+
+  // Sort by version to ensure correct order
+  pendingMigrations.sort((a, b) => a.version - b.version);
+
+  // Apply each migration in a transaction
+  for (const migration of pendingMigrations) {
+    await db.exec("BEGIN");
+    try {
+      // Run the migration SQL
+      await db.exec(migration.up);
+
+      // Record the migration
+      await db.query(
+        `INSERT INTO schema_version (version, applied_at, description)
+         VALUES ($1, $2, $3)`,
+        [migration.version, Date.now(), migration.description],
+      );
+
+      await db.exec("COMMIT");
+      applied.push(migration.version);
+
+      console.log(
+        `[migrations] Applied migration ${migration.version}: ${migration.description}`,
+      );
+    } catch (error) {
+      await db.exec("ROLLBACK");
+      const err = error as Error;
+      console.error(
+        `[migrations] Failed to apply migration ${migration.version}: ${err.message}`,
+      );
+      throw new Error(`Migration ${migration.version} failed: ${err.message}`);
+    }
+  }
+
+  const finalVersion = await getCurrentVersion(db);
+  return { applied, current: finalVersion };
+}
+
+/**
+ * Rollback to a specific version
+ *
+ * WARNING: This will DROP tables and LOSE DATA.
+ * Only use for testing or emergency recovery.
+ */
+export async function rollbackTo(
+  db: PGlite,
+  targetVersion: number,
+): Promise<{
+  rolledBack: number[];
+  current: number;
+}> {
+  const currentVersion = await getCurrentVersion(db);
+  const rolledBack: number[] = [];
+
+  if (targetVersion >= currentVersion) {
+    return { rolledBack: [], current: currentVersion };
+  }
+
+  // Find migrations to rollback (in reverse order)
+  const migrationsToRollback = migrations
+    .filter((m) => m.version > targetVersion && m.version <= currentVersion)
+    .sort((a, b) => b.version - a.version); // Descending order
+
+  for (const migration of migrationsToRollback) {
+    await db.exec("BEGIN");
+    try {
+      // Run the down migration
+      await db.exec(migration.down);
+
+      // Remove from version table
+      await db.query(`DELETE FROM schema_version WHERE version = $1`, [
+        migration.version,
+      ]);
+
+      await db.exec("COMMIT");
+      rolledBack.push(migration.version);
+
+      console.log(
+        `[migrations] Rolled back migration ${migration.version}: ${migration.description}`,
+      );
+    } catch (error) {
+      await db.exec("ROLLBACK");
+      const err = error as Error;
+      console.error(
+        `[migrations] Failed to rollback migration ${migration.version}: ${err.message}`,
+      );
+      throw new Error(
+        `Rollback of migration ${migration.version} failed: ${err.message}`,
+      );
+    }
+  }
+
+  const finalVersion = await getCurrentVersion(db);
+  return { rolledBack, current: finalVersion };
+}
+
+/**
+ * Check if a specific migration has been applied
+ */
+export async function isMigrationApplied(
+  db: PGlite,
+  version: number,
+): Promise<boolean> {
+  await ensureVersionTable(db);
+
+  const result = await db.query<{ count: string }>(
+    `SELECT COUNT(*) as count FROM schema_version WHERE version = $1`,
+    [version],
+  );
+
+  return parseInt(result.rows[0]?.count || "0") > 0;
+}
+
+/**
+ * Get pending migrations (not yet applied)
+ */
+export async function getPendingMigrations(db: PGlite): Promise<Migration[]> {
+  const currentVersion = await getCurrentVersion(db);
+  return migrations
+    .filter((m) => m.version > currentVersion)
+    .sort((a, b) => a.version - b.version);
+}