swarm-mail 0.1.0

package/src/streams/effect/cursor.integration.test.ts

@@ -0,0 +1,418 @@
+/**
+ * DurableCursor Tests
+ *
+ * Tests for Effect-TS cursor service with checkpointing
+ */
+import { describe, expect, it, beforeEach, afterEach } from "vitest";
+import { Effect } from "effect";
+import type { AgentRegisteredEvent } from "../events";
+import { DurableCursor, DurableCursorLayer, type CursorConfig } from "./cursor";
+import {
+  appendEvent,
+  closeDatabase,
+  createEvent,
+  resetDatabase,
+} from "../index";
+
+// ============================================================================
+// Test Utilities
+// ============================================================================
+
+const TEST_PROJECT = "/tmp/cursor-test";
+
+beforeEach(async () => {
+  await resetDatabase(TEST_PROJECT);
+});
+
+afterEach(async () => {
+  await closeDatabase(TEST_PROJECT);
+});
+
+async function cleanup() {
+  await closeDatabase(TEST_PROJECT);
+}
+
+/**
+ * Helper to run Effect programs with DurableCursor service
+ */
+async function runWithCursor<A, E>(
+  effect: Effect.Effect<A, E, DurableCursor>,
+): Promise<A> {
+  return Effect.runPromise(Effect.provide(effect, DurableCursorLayer));
+}
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+describe("DurableCursor", () => {
+  describe("create", () => {
+    it("creates a cursor with initial position 0", async () => {
+      await cleanup();
+
+      const program = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+        const cursor = yield* service.create({
+          stream: "test-stream",
+          checkpoint: "test-checkpoint",
+          projectPath: TEST_PROJECT,
+        });
+
+        const position = yield* cursor.getPosition();
+        return position;
+      });
+
+      const position = await runWithCursor(program);
+      expect(position).toBe(0);
+    });
+
+    it("resumes from last checkpoint position", async () => {
+      await cleanup();
+
+      // First cursor - commit at sequence 5
+      const program1 = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+        const cursor = yield* service.create({
+          stream: "test-stream",
+          checkpoint: "test-checkpoint",
+          projectPath: TEST_PROJECT,
+        });
+
+        yield* cursor.commit(5);
+        return yield* cursor.getPosition();
+      });
+
+      await runWithCursor(program1);
+
+      // Second cursor - should resume at 5
+      const program2 = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+        const cursor = yield* service.create({
+          stream: "test-stream",
+          checkpoint: "test-checkpoint",
+          projectPath: TEST_PROJECT,
+        });
+
+        return yield* cursor.getPosition();
+      });
+
+      const position = await runWithCursor(program2);
+      expect(position).toBe(5);
+    });
+
+    it("supports multiple independent checkpoints", async () => {
+      await cleanup();
+
+      const program = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+
+        const cursor1 = yield* service.create({
+          stream: "test-stream",
+          checkpoint: "checkpoint-a",
+          projectPath: TEST_PROJECT,
+        });
+
+        const cursor2 = yield* service.create({
+          stream: "test-stream",
+          checkpoint: "checkpoint-b",
+          projectPath: TEST_PROJECT,
+        });
+
+        yield* cursor1.commit(10);
+        yield* cursor2.commit(20);
+
+        const pos1 = yield* cursor1.getPosition();
+        const pos2 = yield* cursor2.getPosition();
+
+        return { pos1, pos2 };
+      });
+
+      const result = await runWithCursor(program);
+      expect(result.pos1).toBe(10);
+      expect(result.pos2).toBe(20);
+    });
+  });
+
+  describe("consume", () => {
+    it("consumes events from current position", async () => {
+      await cleanup();
+
+      // Append test events
+      const events = [
+        createEvent("agent_registered", {
+          project_key: "test-project",
+          agent_name: "agent-1",
+          program: "test",
+          model: "test-model",
+        }),
+        createEvent("agent_registered", {
+          project_key: "test-project",
+          agent_name: "agent-2",
+          program: "test",
+          model: "test-model",
+        }),
+        createEvent("agent_registered", {
+          project_key: "test-project",
+          agent_name: "agent-3",
+          program: "test",
+          model: "test-model",
+        }),
+      ];
+
+      for (const event of events) {
+        await appendEvent(event, TEST_PROJECT);
+      }
+
+      // Create cursor and consume outside Effect.gen
+      const program = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+        return yield* service.create({
+          stream: "test-stream",
+          checkpoint: "test-consumer",
+          projectPath: TEST_PROJECT,
+          batchSize: 2,
+        });
+      });
+
+      const cursor = await runWithCursor(program);
+      const consumed: string[] = [];
+
+      for await (const msg of cursor.consume<
+        AgentRegisteredEvent & { id: number; sequence: number }
+      >()) {
+        consumed.push(msg.value.agent_name);
+        await Effect.runPromise(msg.commit());
+      }
+
+      expect(consumed).toHaveLength(3);
+      expect(consumed).toEqual(["agent-1", "agent-2", "agent-3"]);
+    });
+
+    it("resumes consumption from checkpoint", async () => {
+      await cleanup();
+
+      // Append test events
+      const events = [
+        createEvent("agent_registered", {
+          project_key: "test-project",
+          agent_name: "agent-1",
+          program: "test",
+          model: "test-model",
+        }),
+        createEvent("agent_registered", {
+          project_key: "test-project",
+          agent_name: "agent-2",
+          program: "test",
+          model: "test-model",
+        }),
+        createEvent("agent_registered", {
+          project_key: "test-project",
+          agent_name: "agent-3",
+          program: "test",
+          model: "test-model",
+        }),
+      ];
+
+      for (const event of events) {
+        await appendEvent(event, TEST_PROJECT);
+      }
+
+      // First consumer - consume first event only
+      const program1 = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+        return yield* service.create({
+          stream: "test-stream",
+          checkpoint: "resume-test",
+          projectPath: TEST_PROJECT,
+        });
+      });
+
+      const cursor1 = await runWithCursor(program1);
+      const first: string[] = [];
+
+      for await (const msg of cursor1.consume<
+        AgentRegisteredEvent & { id: number; sequence: number }
+      >()) {
+        first.push(msg.value.agent_name);
+        await Effect.runPromise(msg.commit());
+        break; // Consume only first event
+      }
+
+      expect(first).toEqual(["agent-1"]);
+
+      // Second consumer - should resume from checkpoint
+      const program2 = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+        return yield* service.create({
+          stream: "test-stream",
+          checkpoint: "resume-test",
+          projectPath: TEST_PROJECT,
+        });
+      });
+
+      const cursor2 = await runWithCursor(program2);
+      const second: string[] = [];
+
+      for await (const msg of cursor2.consume<
+        AgentRegisteredEvent & { id: number; sequence: number }
+      >()) {
+        second.push(msg.value.agent_name);
+        await Effect.runPromise(msg.commit());
+      }
+
+      expect(second).toEqual(["agent-2", "agent-3"]);
+    });
+
+    it("supports event type filtering", async () => {
+      await cleanup();
+
+      // Append mixed event types
+      await appendEvent(
+        createEvent("agent_registered", {
+          project_key: "test-project",
+          agent_name: "agent-1",
+          program: "test",
+          model: "test-model",
+        }),
+        TEST_PROJECT,
+      );
+
+      await appendEvent(
+        createEvent("message_sent", {
+          project_key: "test-project",
+          from_agent: "agent-1",
+          to_agents: ["agent-2"],
+          subject: "test",
+          body: "test message",
+          importance: "normal",
+          ack_required: false,
+        }),
+        TEST_PROJECT,
+      );
+
+      await appendEvent(
+        createEvent("agent_registered", {
+          project_key: "test-project",
+          agent_name: "agent-2",
+          program: "test",
+          model: "test-model",
+        }),
+        TEST_PROJECT,
+      );
+
+      const program = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+        return yield* service.create({
+          stream: "test-stream",
+          checkpoint: "filter-test",
+          projectPath: TEST_PROJECT,
+          types: ["agent_registered"],
+        });
+      });
+
+      const cursor = await runWithCursor(program);
+      const types: string[] = [];
+
+      for await (const msg of cursor.consume()) {
+        types.push(msg.value.type);
+        await Effect.runPromise(msg.commit());
+      }
+
+      expect(types).toEqual(["agent_registered", "agent_registered"]);
+    });
+
+    it("commits update cursor position", async () => {
+      await cleanup();
+
+      // Append test events
+      await appendEvent(
+        createEvent("agent_registered", {
+          project_key: "test-project",
+          agent_name: "agent-1",
+          program: "test",
+          model: "test-model",
+        }),
+        TEST_PROJECT,
+      );
+
+      const program = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+        return yield* service.create({
+          stream: "test-stream",
+          checkpoint: "commit-test",
+          projectPath: TEST_PROJECT,
+        });
+      });
+
+      const cursor = await runWithCursor(program);
+      const initialPos = await Effect.runPromise(cursor.getPosition());
+
+      let afterCommit = 0;
+      let sequence = 0;
+
+      for await (const msg of cursor.consume()) {
+        await Effect.runPromise(msg.commit());
+        afterCommit = await Effect.runPromise(cursor.getPosition());
+        sequence = msg.sequence;
+        break;
+      }
+
+      expect(initialPos).toBe(0);
+      expect(afterCommit).toBe(sequence);
+      expect(afterCommit).toBeGreaterThan(0);
+    });
+
+    it("handles empty streams gracefully", async () => {
+      await cleanup();
+
+      const program = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+        return yield* service.create({
+          stream: "empty-stream",
+          checkpoint: "empty-test",
+          projectPath: TEST_PROJECT,
+        });
+      });
+
+      const cursor = await runWithCursor(program);
+      const consumed: unknown[] = [];
+
+      for await (const msg of cursor.consume()) {
+        consumed.push(msg);
+      }
+
+      expect(consumed).toHaveLength(0);
+    });
+  });
+
+  describe("commit", () => {
+    it("persists position across cursor instances", async () => {
+      await cleanup();
+
+      const config: CursorConfig = {
+        stream: "test-stream",
+        checkpoint: "persist-test",
+        projectPath: TEST_PROJECT,
+      };
+
+      // First cursor - commit position
+      const program1 = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+        const cursor = yield* service.create(config);
+        yield* cursor.commit(42);
+      });
+
+      await runWithCursor(program1);
+
+      // Second cursor - verify position persisted
+      const program2 = Effect.gen(function* () {
+        const service = yield* DurableCursor;
+        const cursor = yield* service.create(config);
+        return yield* cursor.getPosition();
+      });
+
+      const position = await runWithCursor(program2);
+      expect(position).toBe(42);
+    });
+  });
+});

package/src/streams/effect/cursor.ts

@@ -0,0 +1,288 @@
+/**
+ * DurableCursor - Positioned event stream consumption with checkpointing
+ *
+ * Effect-TS service that wraps event stream reading with cursor state management.
+ * Enables reliable event processing with resumable position tracking.
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const cursor = yield* DurableCursor;
+ *   const consumer = yield* cursor.create({
+ *     stream: "projects/foo/events",
+ *     checkpoint: "agents/bar/position"
+ *   });
+ *
+ *   for await (const msg of consumer.consume()) {
+ *     yield* handleMessage(msg.value);
+ *     yield* msg.commit();
+ *   }
+ * });
+ * ```
+ */
+import { Context, Effect, Ref, Stream } from "effect";
+import { getDatabase } from "../index";
+import { readEvents } from "../store";
+import type { AgentEvent } from "../events";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * Configuration for creating a cursor
+ */
+export interface CursorConfig {
+  /** Stream identifier (e.g. "projects/foo/events") */
+  readonly stream: string;
+  /** Checkpoint identifier (e.g. "agents/bar/position") */
+  readonly checkpoint: string;
+  /** Project path for database location */
+  readonly projectPath?: string;
+  /** Batch size for reading events (default: 100) */
+  readonly batchSize?: number;
+  /** Optional filters for event types */
+  readonly types?: AgentEvent["type"][];
+}
+
+/**
+ * A message from the cursor with commit capability
+ */
+export interface CursorMessage<T = unknown> {
+  /** The event value */
+  readonly value: T;
+  /** Event sequence number */
+  readonly sequence: number;
+  /** Commit this position to the checkpoint */
+  readonly commit: () => Effect.Effect<void>;
+}
+
+/**
+ * A cursor instance for consuming events
+ */
+export interface Cursor {
+  /** Get current position */
+  readonly getPosition: () => Effect.Effect<number>;
+  /** Consume events as an async iterable */
+  readonly consume: <
+    T = AgentEvent & { id: number; sequence: number },
+  >() => AsyncIterable<CursorMessage<T>>;
+  /** Update checkpoint position */
+  readonly commit: (sequence: number) => Effect.Effect<void>;
+}
+
+/**
+ * DurableCursor service interface
+ */
+export interface DurableCursorService {
+  /** Create a new cursor instance */
+  readonly create: (config: CursorConfig) => Effect.Effect<Cursor>;
+}
+
+// ============================================================================
+// Service Definition
+// ============================================================================
+
+/**
+ * DurableCursor Context.Tag
+ */
+export class DurableCursor extends Context.Tag("DurableCursor")<
+  DurableCursor,
+  DurableCursorService
+>() {}
+
+// ============================================================================
+// Implementation
+// ============================================================================
+
+/**
+ * Initialize cursor table schema
+ */
+async function initializeCursorSchema(projectPath?: string): Promise<void> {
+  const db = await getDatabase(projectPath);
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS cursors (
+      id SERIAL PRIMARY KEY,
+      stream TEXT NOT NULL,
+      checkpoint TEXT NOT NULL,
+      position BIGINT NOT NULL DEFAULT 0,
+      updated_at BIGINT NOT NULL,
+      UNIQUE(stream, checkpoint)
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_cursors_stream ON cursors(stream);
+    CREATE INDEX IF NOT EXISTS idx_cursors_checkpoint ON cursors(checkpoint);
+  `);
+}
+
+/**
+ * Load cursor position from database
+ */
+async function loadCursorPosition(
+  stream: string,
+  checkpoint: string,
+  projectPath?: string,
+): Promise<number> {
+  await initializeCursorSchema(projectPath);
+  const db = await getDatabase(projectPath);
+
+  const result = await db.query<{ position: string }>(
+    `SELECT position FROM cursors WHERE stream = $1 AND checkpoint = $2`,
+    [stream, checkpoint],
+  );
+
+  if (result.rows.length === 0) {
+    // Initialize cursor at position 0
+    await db.query(
+      `INSERT INTO cursors (stream, checkpoint, position, updated_at)
+       VALUES ($1, $2, 0, $3)
+       ON CONFLICT (stream, checkpoint) DO NOTHING`,
+      [stream, checkpoint, Date.now()],
+    );
+    return 0;
+  }
+
+  return parseInt(result.rows[0]?.position || "0");
+}
+
+/**
+ * Save cursor position to database
+ */
+async function saveCursorPosition(
+  stream: string,
+  checkpoint: string,
+  position: number,
+  projectPath?: string,
+): Promise<void> {
+  const db = await getDatabase(projectPath);
+
+  await db.query(
+    `INSERT INTO cursors (stream, checkpoint, position, updated_at)
+     VALUES ($1, $2, $3, $4)
+     ON CONFLICT (stream, checkpoint)
+     DO UPDATE SET position = EXCLUDED.position, updated_at = EXCLUDED.updated_at`,
+    [stream, checkpoint, position, Date.now()],
+  );
+}
+
+/**
+ * Create cursor implementation
+ */
+function createCursorImpl(config: CursorConfig): Effect.Effect<Cursor> {
+  return Effect.gen(function* () {
+    // Load initial position from database
+    const initialPosition = yield* Effect.promise(() =>
+      loadCursorPosition(config.stream, config.checkpoint, config.projectPath),
+    );
+
+    // Create mutable reference for current position
+    const positionRef = yield* Ref.make(initialPosition);
+
+    // Commit function - updates database and reference
+    const commitPosition = (sequence: number): Effect.Effect<void> =>
+      Effect.gen(function* () {
+        yield* Effect.promise(() =>
+          saveCursorPosition(
+            config.stream,
+            config.checkpoint,
+            sequence,
+            config.projectPath,
+          ),
+        );
+        yield* Ref.set(positionRef, sequence);
+      });
+
+    // Get current position
+    const getPosition = (): Effect.Effect<number> => Ref.get(positionRef);
+
+    // Consume events as async iterable
+    const consume = <
+      T = AgentEvent & { id: number; sequence: number },
+    >(): AsyncIterable<CursorMessage<T>> => {
+      const batchSize = config.batchSize ?? 100;
+
+      return {
+        [Symbol.asyncIterator]() {
+          let currentBatch: Array<
+            AgentEvent & { id: number; sequence: number }
+          > = [];
+          let batchIndex = 0;
+          let done = false;
+
+          return {
+            async next(): Promise<IteratorResult<CursorMessage<T>>> {
+              // Load next batch if current batch is exhausted
+              if (batchIndex >= currentBatch.length && !done) {
+                const currentPosition = await Effect.runPromise(
+                  Ref.get(positionRef),
+                );
+
+                const events = await readEvents(
+                  {
+                    afterSequence: currentPosition,
+                    limit: batchSize,
+                    types: config.types,
+                  },
+                  config.projectPath,
+                );
+
+                if (events.length === 0) {
+                  done = true;
+                  return { done: true, value: undefined };
+                }
+
+                currentBatch = events;
+                batchIndex = 0;
+              }
+
+              // Return next message from current batch
+              if (batchIndex < currentBatch.length) {
+                const event = currentBatch[batchIndex++];
+                if (!event) {
+                  done = true;
+                  return { done: true, value: undefined };
+                }
+
+                const message: CursorMessage<T> = {
+                  value: event as unknown as T,
+                  sequence: event.sequence,
+                  commit: () => commitPosition(event.sequence),
+                };
+
+                return { done: false, value: message };
+              }
+
+              done = true;
+              return { done: true, value: undefined };
+            },
+          };
+        },
+      };
+    };
+
+    return {
+      getPosition,
+      consume,
+      commit: commitPosition,
+    };
+  });
+}
+
+// ============================================================================
+// Layer
+// ============================================================================
+
+/**
+ * Live implementation of DurableCursor service
+ */
+export const DurableCursorLive = DurableCursor.of({
+  create: createCursorImpl,
+});
+
+/**
+ * Default layer for DurableCursor service
+ */
+export const DurableCursorLayer = Context.make(
+  DurableCursor,
+  DurableCursorLive,
+);