swarm-mail 1.2.2 → 1.4.0

package/README.md

@@ -25,25 +25,35 @@
        🐝        🐝               ██║ ╚═╝ ██║██║  ██║██║███████╗
                                   ╚═╝     ╚═╝╚═╝  ╚═╝╚═╝╚══════╝       🐝
                  🐝
-                             ⚡ Actor-Model Primitives for Agent Coordination ⚡
+                             ⚡ Event Sourcing + Actor Model Primitives ⚡
 ```
 
-Event sourcing primitives for multi-agent coordination. Local-first, no external servers.
+Event sourcing and actor-model primitives for multi-agent coordination. Built on **libSQL** (embedded SQLite) with **Drizzle ORM**. Local-first, no external servers.
 
 **[🌐 swarmtools.ai](https://swarmtools.ai)** | **[📖 Full Documentation](https://swarmtools.ai/docs)**
 
+## What is swarm-mail?
+
+A TypeScript library providing:
+
+1. **Event Store** - Append-only log with automatic projection updates (agents, messages, file reservations)
+2. **Actor Primitives** - DurableMailbox, DurableLock, DurableCursor, DurableDeferred (Effect-TS based)
+3. **Hive** - Git-synced work item tracker (cells, epics, dependencies)
+4. **Semantic Memory** - Vector embeddings for persistent agent learnings (Ollama + pgvector)
+
 ```
 ┌─────────────────────────────────────────────────────────────┐
 │                     SWARM MAIL STACK                        │
 ├─────────────────────────────────────────────────────────────┤
-│  TIER 3: COORDINATION                                       │
+│  COORDINATION                                               │
+│  ├── HiveAdapter - Work item tracking (cells, epics)       │
 │  └── ask<Req, Res>() - Request/Response (RPC-style)        │
 │                                                             │
-│  TIER 2: PATTERNS                                           │
+│  PATTERNS                                                   │
 │  ├── DurableMailbox - Actor inbox with typed envelopes     │
 │  └── DurableLock - CAS-based mutual exclusion              │
 │                                                             │
-│  TIER 1: PRIMITIVES                                         │
+│  PRIMITIVES                                                 │
 │  ├── DurableCursor - Checkpointed stream reader            │
 │  └── DurableDeferred - Distributed promise                 │
 │                                                             │
@@ -51,7 +61,7 @@ Event sourcing primitives for multi-agent coordination. Local-first, no external
 │  └── Semantic Memory - Vector embeddings (pgvector/Ollama) │
 │                                                             │
 │  STORAGE                                                    │
-│  └── PGLite (Embedded Postgres) + Migrations               │
+│  └── libSQL (Embedded SQLite via Drizzle ORM)              │
 └─────────────────────────────────────────────────────────────┘
 ```
 
@@ -61,17 +71,13 @@ Event sourcing primitives for multi-agent coordination. Local-first, no external
 bun add swarm-mail
 ```
 
-## Usage
-
-### Event Store
-
-Append-only event log with automatic projection updates:
+## Quick Start
 
 ```typescript
-import { getSwarmMail } from "swarm-mail";
+import { getSwarmMailLibSQL } from "swarm-mail";
 
-// Create swarm mail instance (automatically creates PGlite adapter)
-const swarmMail = await getSwarmMail("/my/project");
+// Create swarm mail instance (libSQL + Drizzle)
+const swarmMail = await getSwarmMailLibSQL("/my/project");
 
 // Append events
 await swarmMail.appendEvent({
@@ -84,145 +90,141 @@ await swarmMail.appendEvent({
 // Query projections
 const agents = await swarmMail.getAgents();
 const messages = await swarmMail.getInbox("WorkerA", { limit: 5 });
+
+// Clean shutdown
+await swarmMail.close();
 ```
 
-### Durable Primitives (Effect-TS)
+## Core APIs
 
-Built on Effect-TS for type-safe, composable coordination:
+### Event Store
+
+Append-only event log with automatic projection updates:
 
 ```typescript
-import { DurableMailbox, DurableLock, ask } from 'swarm-mail'
-import { Effect } from 'effect'
-
-// Actor mailbox
-const mailbox = DurableMailbox.create<MyMessage>('worker-a')
-await Effect.runPromise(
-  mailbox.send({ type: 'task', payload: 'do something' })
-)
-
-// File locking
-const lock = DurableLock.create('src/auth.ts')
-await Effect.runPromise(
-  lock.acquire({ ttl: 60000 }).pipe(
-    Effect.flatMap(() => /* do work */),
-    Effect.ensuring(lock.release())
-  )
-)
-
-// Request/response
-const response = await Effect.runPromise(
-  ask<Request, Response>('other-agent', { type: 'get-types' })
-)
-```
+import { getSwarmMailLibSQL } from "swarm-mail";
 
-### Database Adapter
+const swarmMail = await getSwarmMailLibSQL("/my/project");
 
-Dependency injection for testing and flexibility:
+// Append events
+await swarmMail.appendEvent({
+  type: "message_sent",
+  from: "WorkerA",
+  to: ["WorkerB"],
+  subject: "Task complete",
+  body: "Auth flow implemented",
+  timestamp: Date.now(),
+});
 
-```typescript
-import { DatabaseAdapter, createSwarmMailAdapter } from 'swarm-mail'
-
-// Implement your own adapter
-const customAdapter: DatabaseAdapter = {
-  query: async (sql, params) => /* ... */,
-  exec: async (sql) => /* ... */,
-  transaction: async (fn) => /* ... */,
-  close: async () => /* ... */
-}
+// Query inbox
+const messages = await swarmMail.getInbox("WorkerB", { 
+  limit: 5,
+  unreadOnly: true 
+});
 
-// Use custom adapter
-const swarmMail = createSwarmMailAdapter(customAdapter, '/my/project')
+// Get thread
+const thread = await swarmMail.getThread("epic-123");
 
-// Or use the convenience layer (built-in PGLite)
-import { getSwarmMail, createInMemorySwarmMail } from 'swarm-mail'
-const swarmMail = await getSwarmMail('/my/project') // persistent
-const swarmMail = await createInMemorySwarmMail() // in-memory
+// Check file reservations
+const conflicts = await swarmMail.checkConflicts([
+  "src/auth.ts"
+], "WorkerA");
 ```
 
-## Deployment
+### Hive (Work Item Tracker)
 
-### Connection Modes
+Git-synced work item tracking with cells and epics:
 
-#### Daemon Mode (Default)
+```typescript
+import { createHiveAdapter } from "swarm-mail";
 
-By default, swarm-mail starts an in-process `PGLiteSocketServer` when you call `getSwarmMail()`. All database operations go through this server, preventing multi-process corruption.
+const hive = await createHiveAdapter({ 
+  projectPath: "/my/project" 
+});
 
-```typescript
-import { getSwarmMail } from 'swarm-mail'
+// Create cell
+const cell = await hive.createCell({
+  title: "Add OAuth",
+  type: "feature",
+  priority: 2,
+});
 
-// Default: starts daemon automatically
-const swarmMail = await getSwarmMail('/my/project')
+// Query cells
+const open = await hive.queryCells({ status: "open" });
+const ready = await hive.queryCells({ ready: true });
 
-// Keep alive - handles cleanup on shutdown
-process.on('SIGTERM', async () => {
-  await swarmMail.close() // Flushes WAL, closes cleanly
-  process.exit(0)
-})
+// Update cell
+await hive.updateCell(cell.id, { 
+  status: "in_progress",
+  description: "Implementing Google OAuth flow"
+});
+
+// Close cell
+await hive.closeCell(cell.id, "Completed: OAuth implemented");
 ```
 
-**Why daemon mode is the default:**
+### Semantic Memory
 
-- **No external dependencies** - Uses `@electric-sql/pglite-socket` in-process
-- **Multi-process safe** - One PGLite instance, multiple clients can connect safely
-- **WAL safety** - Single process prevents WAL accumulation from multiple instances
-- **Proper cleanup** - Graceful shutdown triggers checkpoint, preventing unclean state
-- **Resource efficiency** - One PGLite instance shared across operations
+Vector embeddings for persistent agent learnings:
 
-#### Embedded Mode (Opt-out)
+```typescript
+import { createSemanticMemory } from "swarm-mail";
 
-For single-process use cases where you're certain only one process will access the database, you can opt out of daemon mode:
+const memory = await createSemanticMemory("/my/project");
 
-```bash
-SWARM_MAIL_SOCKET=false
-```
+// Store a learning
+const { id } = await memory.store(
+  "OAuth refresh tokens need 5min buffer before expiry to avoid race conditions",
+  { tags: "auth,tokens,debugging" }
+);
 
-⚠️ **Warning:** Embedded mode is NOT safe for multi-process access. PGLite uses a single connection, so concurrent processes will cause database corruption. Use only when you're absolutely certain only one process will access the database
+// Search by meaning (vector similarity)
+const results = await memory.find("token refresh issues", { limit: 5 });
 
-### WAL Safety Features
+// Get memory by ID
+const mem = await memory.get(id);
 
-PGLite uses PostgreSQL's Write-Ahead Log (WAL) for durability. Swarm Mail includes safeguards against WAL bloat:
+// Validate (resets decay timer)
+await memory.validate(id);
 
-**Automatic checkpointing:**
-```typescript
-// After batch operations, force WAL flush
-await db.checkpoint()
+// Check Ollama health
+const health = await memory.checkHealth();
+// { ollama: true, model: "mxbai-embed-large" }
 ```
 
-**Health monitoring:**
-```typescript
-// Check WAL size (default threshold: 100MB)
-const health = await swarmMail.healthCheck({ walThresholdMb: 100 })
-
-if (!health.walHealth?.healthy) {
-  console.warn(health.walHealth?.message)
-  // "WAL size 120MB exceeds 100MB threshold (15 files)"
-}
+> **Note:** Requires [Ollama](https://ollama.ai/) for vector embeddings. Falls back to full-text search if unavailable.
+>
+> ```bash
+> ollama pull mxbai-embed-large
+> ```
 
-// Get detailed stats
-const stats = await swarmMail.getDatabaseStats()
-// { connected: true, wal: { size: 120_000_000, fileCount: 15 } }
-```
+### Custom Database Setup
 
-**When to checkpoint manually:**
+Bring your own libSQL database:
 
-- After migrations: `await swarmMail.runMigrations(); await db.checkpoint()`
-- After bulk event appends (100+ events)
-- Before long-running operations
+```typescript
+import { createLibSQLAdapter } from "swarm-mail";
+import { drizzle } from "drizzle-orm/libsql";
+import { createClient } from "@libsql/client";
 
-### Ephemeral Instances (Testing)
+// Create libSQL client
+const client = createClient({
+  url: "file:///my/custom/path/swarm.db"
+});
 
-For tests, create isolated in-memory instances:
+const db = drizzle(client);
 
-```typescript
-import { createInMemorySwarmMail } from 'swarm-mail'
+// Create adapter
+const swarmMail = createLibSQLAdapter(db, "/my/project");
 
-const swarmMail = await createInMemorySwarmMail('test-id')
-// ... run tests ...
-await swarmMail.close()
+// Use as normal
+await swarmMail.appendEvent({
+  type: "agent_registered",
+  agent_name: "CustomAgent",
+  timestamp: Date.now(),
+});
 ```
 
-**Don't use ephemeral instances in production** - multiple short-lived processes compound WAL accumulation since each instance creates new PGLite connections without coordinated cleanup.
-
 ## Event Types
 
 ```typescript
@@ -234,6 +236,7 @@ type SwarmMailEvent =
       to: string[];
       subject: string;
       body: string;
+      thread_id?: string;
     }
   | { type: "message_read"; message_id: number; agent_name: string }
   | {
@@ -241,6 +244,7 @@ type SwarmMailEvent =
       agent_name: string;
       paths: string[];
       exclusive: boolean;
+      reason?: string;
     }
   | { type: "file_released"; agent_name: string; paths: string[] }
   | {
@@ -260,7 +264,7 @@ type SwarmMailEvent =
 
 ## Projections
 
-Materialized views derived from events:
+Materialized views automatically updated from events:
 
 | Projection          | Description                        |
 | ------------------- | ---------------------------------- |
@@ -270,46 +274,65 @@ Materialized views derived from events:
 | `swarm_contexts`    | Checkpoint state for recovery      |
 | `eval_records`      | Outcome data for learning          |
 
-### Semantic Memory
+## Testing
 
-Persistent, searchable storage for agent learnings:
+For tests, use in-memory instances:
 
 ```typescript
-import { createMemoryAdapter } from 'swarm-mail/memory'
+import { createInMemorySwarmMail } from "swarm-mail";
+
+describe("my feature", () => {
+  let swarmMail: SwarmMailAdapter;
+
+  beforeAll(async () => {
+    swarmMail = await createInMemorySwarmMail("test");
+  });
+
+  afterAll(async () => {
+    await swarmMail.close();
+  });
+
+  test("appends events", async () => {
+    const result = await swarmMail.appendEvent({
+      type: "agent_registered",
+      agent_name: "TestAgent",
+      timestamp: Date.now(),
+    });
+    
+    expect(result.id).toBeGreaterThan(0);
+  });
+});
+```
 
-const swarmMail = await getSwarmMail('/my/project')
-const db = await swarmMail.getDatabase()
-const memory = await createMemoryAdapter(db)
+## Architecture
 
-// Store a learning
-const { id } = await memory.store(
-  "OAuth refresh tokens need 5min buffer before expiry...",
-  { tags: "auth,tokens,debugging" }
-)
+- **Event Sourcing** - Append-only log, projections are derived
+- **Local-first** - libSQL embedded SQLite, no external servers
+- **Type-safe** - TypeScript with Zod validation and Drizzle ORM
+- **Effect-TS** - Composable, testable actor primitives
+- **Git-synced** - Hive cells stored as JSON + git for team coordination
 
-// Search by meaning (vector similarity)
-const results = await memory.find("token refresh issues")
+## Migration from v0.31
 
-// Check Ollama health
-const health = await memory.checkHealth()
-// { ollama: true, model: "mxbai-embed-large" }
-```
+If you're migrating from PGLite-based swarm-mail:
 
-> **Note:** Requires [Ollama](https://ollama.ai/) for vector embeddings. Falls back to full-text search if unavailable.
->
-> ```bash
-> ollama pull mxbai-embed-large
-> ```
+```typescript
+// OLD (v0.31)
+import { getSwarmMail } from "swarm-mail";
+const swarmMail = await getSwarmMail("/my/project");
 
-> **Deprecation Notice:** The standalone [semantic-memory MCP server](https://github.com/joelhooks/semantic-memory) is deprecated. Use the embedded memory in swarm-mail instead - same API, single PGLite instance, no separate process.
+// NEW (v0.32+)
+import { getSwarmMailLibSQL } from "swarm-mail";
+const swarmMail = await getSwarmMailLibSQL("/my/project");
+```
 
-## Architecture
+**Key changes:**
+- Storage backend: PGLite → libSQL (SQLite-compatible)
+- ORM: Raw SQL → Drizzle ORM
+- Main export: `getSwarmMailLibSQL` (was `getSwarmMail`)
+- Semantic memory: Embedded (was standalone MCP server)
 
-- **Append-only log** - Events are immutable, projections are derived
-- **Local-first** - PGLite embedded Postgres, no external servers
-- **Effect-TS** - Type-safe, composable, testable
-- **Exactly-once** - DurableCursor checkpoints position
-- **Semantic memory** - Vector embeddings with pgvector + Ollama
+See [CHANGELOG.md](./CHANGELOG.md) for full migration guide.
 
 ## API Reference
 
@@ -349,6 +372,13 @@ interface SwarmMailAdapter {
 }
 ```
 
+## Resources
+
+- **Documentation:** [swarmtools.ai/docs](https://swarmtools.ai/docs)
+- **Architecture:** [SWARM-CONTEXT.md](../../SWARM-CONTEXT.md)
+- **Examples:** [examples/](../../examples/)
+- **Changelog:** [CHANGELOG.md](./CHANGELOG.md)
+
 ## License
 
 MIT

package/bin/swarm-db.ts

@@ -0,0 +1,168 @@
+#!/usr/bin/env bun
+/**
+ * swarm-db CLI - Human-facing analytics and SQL queries
+ *
+ * Commands:
+ *   query <sql>             Execute raw SQL (read-only, max 1000 rows)
+ *   analytics <command>     Run pre-built analytics query
+ *   list                    List available analytics commands
+ *
+ * Flags:
+ *   --format <fmt>          Output format: table (default), json, csv, jsonl
+ *   --db <path>             Database path (default: ~/.config/swarm-tools/swarm.db)
+ *   --since <range>         Time range filter (e.g., 7d, 24h, 30m)
+ *   --until <range>         End time filter (e.g., 1d, 12h)
+ *   --project <key>         Filter by project key
+ *   --epic <id>             Filter by epic ID
+ *   --help, -h              Show help
+ *
+ * Examples:
+ *   swarm-db query "SELECT type, COUNT(*) FROM events GROUP BY type"
+ *   swarm-db analytics failed-decompositions --format json
+ *   swarm-db analytics agent-activity --since 7d --format table
+ *   swarm-db list
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+import type { OutputFormat } from "../src/analytics/types.js";
+import {
+	executeAnalyticsCommand,
+	executeQueryCommand,
+	listAnalyticsCommands,
+} from "../src/cli/db.js";
+
+const DEFAULT_DB = join(homedir(), ".config/swarm-tools/swarm.db");
+
+function showHelp() {
+	console.log(`
+swarm-db - Analytics and SQL queries for swarm coordination database
+
+USAGE
+  swarm-db <command> [options]
+
+COMMANDS
+  query <sql>             Execute raw SQL query (read-only, max 1000 rows)
+  analytics <command>     Run pre-built analytics query
+  list                    List all available analytics commands
+
+ANALYTICS COMMANDS
+  Run 'swarm-db list' to see all available analytics commands with descriptions.
+
+FLAGS
+  --format <fmt>          Output format: table (default), json, csv, jsonl
+  --db <path>             Database path (default: ~/.config/swarm-tools/swarm.db)
+  --since <range>         Time range filter (e.g., 7d, 24h, 30m)
+  --until <range>         End time filter (e.g., 1d, 12h)
+  --project <key>         Filter by project key
+  --epic <id>             Filter by epic ID
+  -h, --help              Show this help message
+
+EXAMPLES
+  # Raw SQL query
+  swarm-db query "SELECT type, COUNT(*) FROM events GROUP BY type"
+
+  # Analytics with default table format
+  swarm-db analytics failed-decompositions
+
+  # Analytics with time filter and JSON output
+  swarm-db analytics agent-activity --since 7d --format json
+
+  # Analytics filtered by project
+  swarm-db analytics lock-contention --project /path/to/project --format csv
+
+  # List all available analytics commands
+  swarm-db list
+
+NOTES
+  - SQL queries are read-only for safety (SELECT only)
+  - Maximum 1000 rows returned for raw queries
+  - Analytics queries have built-in limits
+  - Time ranges: d=days, h=hours, m=minutes (e.g., 7d, 24h, 30m)
+`);
+}
+
+async function main() {
+	const { values, positionals } = parseArgs({
+		args: process.argv.slice(2),
+		options: {
+			format: { type: "string", default: "table" },
+			db: { type: "string", default: DEFAULT_DB },
+			since: { type: "string" },
+			until: { type: "string" },
+			project: { type: "string" },
+			epic: { type: "string" },
+			help: { type: "boolean", short: "h", default: false },
+		},
+		allowPositionals: true,
+	});
+
+	if (values.help || positionals.length === 0) {
+		showHelp();
+		process.exit(0);
+	}
+
+	const command = positionals[0];
+	const format = values.format as OutputFormat;
+
+	try {
+		if (command === "list") {
+			// List analytics commands
+			const commands = listAnalyticsCommands();
+			console.log("\nAvailable Analytics Commands:\n");
+			for (const cmd of commands) {
+				console.log(`  ${cmd.name.padEnd(25)} ${cmd.description}`);
+			}
+			console.log(
+				`\nRun 'swarm-db analytics <command>' to execute a command.\n`,
+			);
+		} else if (command === "query") {
+			// Execute raw SQL
+			const sql = positionals[1];
+			if (!sql) {
+				console.error("Error: SQL query required");
+				console.error("Usage: swarm-db query <sql>");
+				process.exit(1);
+			}
+
+			const output = await executeQueryCommand({
+				sql,
+				db: values.db,
+				format,
+			});
+
+			console.log(output);
+		} else if (command === "analytics") {
+			// Execute analytics command
+			const analyticsCmd = positionals[1];
+			if (!analyticsCmd) {
+				console.error("Error: Analytics command required");
+				console.error("Usage: swarm-db analytics <command>");
+				console.error("Run 'swarm-db list' to see available commands");
+				process.exit(1);
+			}
+
+			const output = await executeAnalyticsCommand({
+				command: analyticsCmd,
+				db: values.db,
+				format,
+				since: values.since,
+				until: values.until,
+				project: values.project,
+				epic: values.epic,
+			});
+
+			console.log(output);
+		} else {
+			console.error(`Unknown command: ${command}`);
+			console.error("Run 'swarm-db --help' for usage information");
+			process.exit(1);
+		}
+	} catch (error) {
+		console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
+		process.exit(1);
+	}
+}
+
+main();

package/dist/adapter.d.ts

@@ -12,10 +12,10 @@
  *
  * ## Usage
  * ```typescript
- * import { createPGLiteAdapter } from '@opencode/swarm-mail/adapters/pglite';
+ * import { createLibSQLAdapter } from '@opencode/swarm-mail/adapters/libsql';
  * import { createSwarmMailAdapter } from '@opencode/swarm-mail';
  *
- * const dbAdapter = createPGLiteAdapter({ path: './streams.db' });
+ * const dbAdapter = createLibSQLAdapter({ path: './streams.db' });
  * const swarmMail = createSwarmMailAdapter(dbAdapter, '/path/to/project');
  *
  * // Use the adapter

package/dist/analytics/formatters.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Analytics Query Result Formatters
+ *
+ * Output formatters for query results in various formats.
+ * Each formatter takes a QueryResult and returns a string.
+ */
+import type { QueryResult } from "./types.js";
+/**
+ * Format query result as ASCII table with aligned columns.
+ *
+ * Produces a readable table format with headers, separators, and aligned columns.
+ * Empty results show headers and indicate 0 rows.
+ *
+ * @param result - Query result to format
+ * @returns ASCII table string
+ */
+export declare function formatTable(result: QueryResult): string;
+/**
+ * Format query result as pretty-printed JSON.
+ *
+ * Produces a readable JSON representation of the entire QueryResult object.
+ *
+ * @param result - Query result to format
+ * @returns Pretty-printed JSON string
+ */
+export declare function formatJSON(result: QueryResult): string;
+/**
+ * Format query result as RFC 4180 compliant CSV.
+ *
+ * Produces CSV with:
+ * - Header row with column names
+ * - Data rows with values
+ * - Proper escaping of quotes and commas
+ * - Empty strings for null/undefined values
+ *
+ * @param result - Query result to format
+ * @returns CSV string
+ */
+export declare function formatCSV(result: QueryResult): string;
+/**
+ * Format query result as newline-delimited JSON (JSONL).
+ *
+ * Produces one compact JSON object per line, one line per row.
+ * Empty results produce empty string.
+ *
+ * @param result - Query result to format
+ * @returns JSONL string (newline-delimited JSON objects)
+ */
+export declare function formatJSONL(result: QueryResult): string;
+//# sourceMappingURL=formatters.d.ts.map