pdf-brain 0.1.1 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pdf-brain",
-  "version": "0.1.1",
+  "version": "0.4.0",
   "description": "Local PDF knowledge base with vector search",
   "type": "module",
   "main": "src/index.ts",
@@ -18,7 +18,8 @@
     "@effect/platform": "^0.72.0",
     "@effect/platform-bun": "^0.52.0",
     "@effect/schema": "^0.75.0",
-    "@electric-sql/pglite": "^0.2.17",
+    "@electric-sql/pglite": "^0.3.14",
+    "@electric-sql/pglite-tools": "^0.2.19",
     "effect": "^3.12.0"
   },
   "devDependencies": {

package/scripts/migration/export-pg16.mjs

@@ -0,0 +1,150 @@
+#!/usr/bin/env node
+/**
+ * Export data from PGlite 0.2.x (PostgreSQL 16) database
+ * 
+ * This script exports your existing database using PGlite 0.2.x before upgrading.
+ * 
+ * Prerequisites:
+ *   npm install @electric-sql/pglite@0.2.12
+ * 
+ * Usage:
+ *   node export-pg16.mjs [db-path] [output-dir]
+ * 
+ * Example:
+ *   node export-pg16.mjs ~/.pdf-library/library ~/.pdf-library
+ */
+
+import { PGlite } from '@electric-sql/pglite';
+import { writeFileSync, appendFileSync, existsSync, unlinkSync, mkdirSync } from 'fs';
+import { join, dirname } from 'path';
+
+const args = process.argv.slice(2);
+const dbPath = args[0] || join(process.env.HOME, 'Documents/.pdf-library/library');
+const outputDir = args[1] || dirname(dbPath);
+
+async function main() {
+  console.log('=== PGlite 0.2.x Database Export ===\n');
+  console.log(`Database: ${dbPath}`);
+  console.log(`Output: ${outputDir}\n`);
+
+  // Check if database exists
+  if (!existsSync(dbPath)) {
+    console.error(`Error: Database not found at ${dbPath}`);
+    process.exit(1);
+  }
+
+  // Check PG_VERSION
+  const versionFile = join(dbPath, 'PG_VERSION');
+  if (existsSync(versionFile)) {
+    const version = require('fs').readFileSync(versionFile, 'utf-8').trim();
+    console.log(`PostgreSQL version: ${version}`);
+    if (version !== '16') {
+      console.warn(`Warning: Expected PG version 16, found ${version}`);
+    }
+  }
+
+  console.log('\nOpening database with PGlite 0.2.x...');
+  
+  let db;
+  try {
+    db = await PGlite.create({ dataDir: dbPath });
+  } catch (e) {
+    console.error(`Failed to open database: ${e.message}`);
+    console.error('\nMake sure you have PGlite 0.2.x installed:');
+    console.error('  npm install @electric-sql/pglite@0.2.12');
+    process.exit(1);
+  }
+
+  // Ensure output directory exists
+  if (!existsSync(outputDir)) {
+    mkdirSync(outputDir, { recursive: true });
+  }
+
+  // Export documents
+  console.log('\nExporting documents...');
+  const docs = await db.query('SELECT * FROM documents');
+  console.log(`  Found ${docs.rows.length} documents`);
+  
+  const docsFile = join(outputDir, 'backup-documents.json');
+  writeFileSync(docsFile, JSON.stringify(docs.rows, null, 2));
+  console.log(`  Saved to: ${docsFile}`);
+
+  // Count chunks
+  const chunkCount = await db.query('SELECT COUNT(*) as c FROM chunks');
+  const totalChunks = parseInt(chunkCount.rows[0].c);
+  console.log(`\nExporting ${totalChunks} chunks...`);
+
+  // Export chunks in batches
+  const chunksFile = join(outputDir, 'backup-chunks.jsonl');
+  if (existsSync(chunksFile)) unlinkSync(chunksFile);
+
+  const batchSize = 100;
+  let exported = 0;
+
+  for (let offset = 0; offset < totalChunks; offset += batchSize) {
+    const batch = await db.query(
+      `SELECT * FROM chunks ORDER BY id LIMIT ${batchSize} OFFSET ${offset}`
+    );
+    
+    for (const row of batch.rows) {
+      appendFileSync(chunksFile, JSON.stringify(row) + '\n');
+    }
+    
+    exported += batch.rows.length;
+    if (exported % 10000 === 0 || exported === totalChunks) {
+      console.log(`  Progress: ${exported}/${totalChunks}`);
+    }
+  }
+  console.log(`  Saved to: ${chunksFile}`);
+
+  // Try to export embeddings (may fail if vector extension issues)
+  console.log('\nExporting embeddings...');
+  try {
+    const embCount = await db.query('SELECT COUNT(*) as c FROM embeddings');
+    const totalEmb = parseInt(embCount.rows[0].c);
+    console.log(`  Found ${totalEmb} embeddings`);
+
+    if (totalEmb > 0) {
+      const embFile = join(outputDir, 'backup-embeddings.jsonl');
+      if (existsSync(embFile)) unlinkSync(embFile);
+
+      let embExported = 0;
+      for (let offset = 0; offset < totalEmb; offset += batchSize) {
+        const batch = await db.query(
+          `SELECT chunk_id, embedding::text as embedding FROM embeddings ORDER BY chunk_id LIMIT ${batchSize} OFFSET ${offset}`
+        );
+        
+        for (const row of batch.rows) {
+          appendFileSync(embFile, JSON.stringify(row) + '\n');
+        }
+        
+        embExported += batch.rows.length;
+        if (embExported % 5000 === 0 || embExported === totalEmb) {
+          console.log(`  Progress: ${embExported}/${totalEmb}`);
+        }
+      }
+      console.log(`  Saved to: ${embFile}`);
+    }
+  } catch (e) {
+    console.log(`  Skipped: ${e.message}`);
+    console.log('  (Embeddings can be regenerated after import)');
+  }
+
+  await db.close();
+
+  console.log('\n=== Export Complete ===');
+  console.log('\nBackup files:');
+  console.log(`  - ${join(outputDir, 'backup-documents.json')}`);
+  console.log(`  - ${join(outputDir, 'backup-chunks.jsonl')}`);
+  if (existsSync(join(outputDir, 'backup-embeddings.jsonl'))) {
+    console.log(`  - ${join(outputDir, 'backup-embeddings.jsonl')}`);
+  }
+  console.log('\nNext steps:');
+  console.log('  1. Upgrade to PGlite 0.3.x: bun install');
+  console.log('  2. Import data: bun run scripts/migration/import-pg17.ts');
+}
+
+main().catch(e => {
+  console.error('Export failed:', e);
+  process.exit(1);
+});

package/scripts/migration/import-pg17.ts

@@ -0,0 +1,284 @@
+#!/usr/bin/env bun
+/**
+ * Import backup data into PGlite 0.3.x (PostgreSQL 17) database
+ *
+ * This script imports data exported from PGlite 0.2.x.
+ *
+ * Usage:
+ *   bun run scripts/migration/import-pg17.ts [backup-dir] [db-path]
+ *
+ * Example:
+ *   bun run scripts/migration/import-pg17.ts ~/.pdf-library ~/.pdf-library/library
+ */
+
+import { PGlite } from "@electric-sql/pglite";
+import { vector } from "@electric-sql/pglite/vector";
+import { readFileSync, createReadStream, existsSync } from "fs";
+import { createInterface } from "readline";
+import { join } from "path";
+
+const EMBEDDING_DIM = 1024;
+
+const args = process.argv.slice(2);
+const backupDir = args[0] || join(process.env.HOME!, "Documents/.pdf-library");
+const dbPath = args[1] || join(backupDir, "library");
+
+async function main() {
+  console.log("=== PGlite 0.3.x Database Import ===\n");
+  console.log(`Backup dir: ${backupDir}`);
+  console.log(`Database: ${dbPath}\n`);
+
+  // Check backup files exist
+  const docsFile = join(backupDir, "backup-documents.json");
+  const chunksFile = join(backupDir, "backup-chunks.jsonl");
+
+  if (!existsSync(docsFile)) {
+    console.error(`Error: Documents backup not found: ${docsFile}`);
+    console.error("\nRun the export script first:");
+    console.error("  node scripts/migration/export-pg16.mjs");
+    process.exit(1);
+  }
+
+  if (!existsSync(chunksFile)) {
+    console.error(`Error: Chunks backup not found: ${chunksFile}`);
+    process.exit(1);
+  }
+
+  // Check if database already exists
+  if (existsSync(dbPath)) {
+    const versionFile = join(dbPath, "PG_VERSION");
+    if (existsSync(versionFile)) {
+      const version = readFileSync(versionFile, "utf-8").trim();
+      if (version === "17") {
+        console.error("Error: PG17 database already exists at this location.");
+        console.error("Remove it first if you want to re-import:");
+        console.error(`  rm -r "${dbPath}"`);
+        process.exit(1);
+      }
+    }
+  }
+
+  console.log("Creating new PGlite 0.3.x database...");
+  const db = new PGlite(dbPath, { extensions: { vector } });
+  await db.waitReady;
+
+  console.log("Initializing schema...");
+  await db.exec("CREATE EXTENSION IF NOT EXISTS vector;");
+
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS documents (
+      id TEXT PRIMARY KEY,
+      title TEXT NOT NULL,
+      path TEXT NOT NULL UNIQUE,
+      added_at TIMESTAMPTZ NOT NULL,
+      page_count INTEGER NOT NULL,
+      size_bytes INTEGER NOT NULL,
+      tags JSONB DEFAULT '[]',
+      metadata JSONB DEFAULT '{}'
+    )
+  `);
+
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS chunks (
+      id TEXT PRIMARY KEY,
+      doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+      page INTEGER NOT NULL,
+      chunk_index INTEGER NOT NULL,
+      content TEXT NOT NULL
+    )
+  `);
+
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS embeddings (
+      chunk_id TEXT PRIMARY KEY REFERENCES chunks(id) ON DELETE CASCADE,
+      embedding vector(${EMBEDDING_DIM}) NOT NULL
+    )
+  `);
+
+  // Create indexes
+  await db.exec(`
+    CREATE INDEX IF NOT EXISTS embeddings_hnsw_idx 
+    ON embeddings USING hnsw (embedding vector_cosine_ops)
+  `);
+  await db.exec(`
+    CREATE INDEX IF NOT EXISTS chunks_content_idx 
+    ON chunks USING gin (to_tsvector('english', content))
+  `);
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_doc ON chunks(doc_id)`);
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_docs_path ON documents(path)`);
+
+  console.log("Schema created!\n");
+
+  // Import documents
+  console.log("Importing documents...");
+  const docs = JSON.parse(readFileSync(docsFile, "utf-8"));
+  const docIds = new Set<string>();
+
+  for (const doc of docs) {
+    docIds.add(doc.id);
+    await db.query(
+      `INSERT INTO documents (id, title, path, added_at, page_count, size_bytes, tags, metadata)
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+       ON CONFLICT (id) DO NOTHING`,
+      [
+        doc.id,
+        doc.title,
+        doc.path,
+        doc.added_at,
+        doc.page_count,
+        doc.size_bytes,
+        JSON.stringify(doc.tags || []),
+        JSON.stringify(doc.metadata || {}),
+      ],
+    );
+  }
+  console.log(`  Imported ${docs.length} documents`);
+
+  // Import chunks
+  console.log("\nImporting chunks...");
+  const rl = createInterface({
+    input: createReadStream(chunksFile),
+    crlfDelay: Infinity,
+  });
+
+  let chunkCount = 0;
+  let skipped = 0;
+  let batch: any[] = [];
+  const BATCH_SIZE = 100;
+
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+    const chunk = JSON.parse(line);
+
+    // Skip orphaned chunks
+    if (!docIds.has(chunk.doc_id)) {
+      skipped++;
+      continue;
+    }
+
+    batch.push(chunk);
+
+    if (batch.length >= BATCH_SIZE) {
+      await db.exec("BEGIN");
+      for (const c of batch) {
+        await db.query(
+          `INSERT INTO chunks (id, doc_id, page, chunk_index, content)
+           VALUES ($1, $2, $3, $4, $5)
+           ON CONFLICT (id) DO NOTHING`,
+          [c.id, c.doc_id, c.page, c.chunk_index, c.content],
+        );
+      }
+      await db.exec("COMMIT");
+      chunkCount += batch.length;
+      batch = [];
+
+      if (chunkCount % 10000 === 0) {
+        console.log(`  Progress: ${chunkCount} chunks...`);
+      }
+    }
+  }
+
+  // Final batch
+  if (batch.length > 0) {
+    await db.exec("BEGIN");
+    for (const c of batch) {
+      await db.query(
+        `INSERT INTO chunks (id, doc_id, page, chunk_index, content)
+         VALUES ($1, $2, $3, $4, $5)
+         ON CONFLICT (id) DO NOTHING`,
+        [c.id, c.doc_id, c.page, c.chunk_index, c.content],
+      );
+    }
+    await db.exec("COMMIT");
+    chunkCount += batch.length;
+  }
+
+  console.log(`  Imported ${chunkCount} chunks`);
+  if (skipped > 0) {
+    console.log(`  Skipped ${skipped} orphaned chunks`);
+  }
+
+  // Import embeddings if available
+  const embFile = join(backupDir, "backup-embeddings.jsonl");
+  if (existsSync(embFile)) {
+    console.log("\nImporting embeddings...");
+    const embRl = createInterface({
+      input: createReadStream(embFile),
+      crlfDelay: Infinity,
+    });
+
+    let embCount = 0;
+    let embBatch: any[] = [];
+
+    for await (const line of embRl) {
+      if (!line.trim()) continue;
+      const emb = JSON.parse(line);
+      embBatch.push(emb);
+
+      if (embBatch.length >= 50) {
+        await db.exec("BEGIN");
+        for (const e of embBatch) {
+          await db.query(
+            `INSERT INTO embeddings (chunk_id, embedding)
+             VALUES ($1, $2::vector)
+             ON CONFLICT (chunk_id) DO NOTHING`,
+            [e.chunk_id, e.embedding],
+          );
+        }
+        await db.exec("COMMIT");
+        embCount += embBatch.length;
+        embBatch = [];
+
+        if (embCount % 5000 === 0) {
+          console.log(`  Progress: ${embCount} embeddings...`);
+        }
+      }
+    }
+
+    if (embBatch.length > 0) {
+      await db.exec("BEGIN");
+      for (const e of embBatch) {
+        await db.query(
+          `INSERT INTO embeddings (chunk_id, embedding)
+           VALUES ($1, $2::vector)
+           ON CONFLICT (chunk_id) DO NOTHING`,
+          [e.chunk_id, e.embedding],
+        );
+      }
+      await db.exec("COMMIT");
+      embCount += embBatch.length;
+    }
+
+    console.log(`  Imported ${embCount} embeddings`);
+  }
+
+  // Verify
+  const docResult = await db.query<{ c: number }>(
+    "SELECT COUNT(*) as c FROM documents",
+  );
+  const chunkResult = await db.query<{ c: number }>(
+    "SELECT COUNT(*) as c FROM chunks",
+  );
+  const embResult = await db.query<{ c: number }>(
+    "SELECT COUNT(*) as c FROM embeddings",
+  );
+
+  console.log("\n=== Import Complete ===");
+  console.log(`Documents:  ${docResult.rows[0]?.c}`);
+  console.log(`Chunks:     ${chunkResult.rows[0]?.c}`);
+  console.log(`Embeddings: ${embResult.rows[0]?.c}`);
+
+  if (Number(embResult.rows[0]?.c) === 0) {
+    console.log("\nEmbeddings need to be regenerated.");
+    console.log("Run: bun run scripts/migration/regenerate-embeddings.ts");
+  }
+
+  console.log("\nVerify with: pdf-library stats");
+
+  await db.close();
+}
+
+main().catch((e) => {
+  console.error("Import failed:", e);
+  process.exit(1);
+});

package/scripts/migration/regenerate-embeddings.ts

@@ -0,0 +1,177 @@
+#!/usr/bin/env bun
+/**
+ * Regenerate embeddings for chunks using Ollama
+ *
+ * This script generates embeddings for chunks that don't have them.
+ * Useful after migrating from PGlite 0.2.x where embeddings couldn't be exported.
+ *
+ * Prerequisites:
+ *   - Ollama running: ollama serve
+ *   - Embedding model: ollama pull mxbai-embed-large
+ *
+ * Usage:
+ *   bun run scripts/migration/regenerate-embeddings.ts [db-path]
+ *
+ * Environment:
+ *   OLLAMA_HOST - Ollama server URL (default: http://localhost:11434)
+ *   OLLAMA_MODEL - Embedding model (default: mxbai-embed-large)
+ */
+
+import { PGlite } from "@electric-sql/pglite";
+import { vector } from "@electric-sql/pglite/vector";
+import { join } from "path";
+
+const OLLAMA_HOST = process.env.OLLAMA_HOST || "http://localhost:11434";
+const OLLAMA_MODEL = process.env.OLLAMA_MODEL || "mxbai-embed-large";
+const BATCH_SIZE = 5; // Process 5 chunks at a time (balance speed vs memory)
+
+const args = process.argv.slice(2);
+const dbPath =
+  args[0] || join(process.env.HOME!, "Documents/.pdf-library/library");
+
+interface EmbeddingResponse {
+  embedding: number[];
+}
+
+async function getEmbedding(text: string): Promise<number[]> {
+  const response = await fetch(`${OLLAMA_HOST}/api/embeddings`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ model: OLLAMA_MODEL, prompt: text }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`Ollama error: ${response.status} ${response.statusText}`);
+  }
+
+  const data = (await response.json()) as EmbeddingResponse;
+  return data.embedding;
+}
+
+async function checkOllama(): Promise<boolean> {
+  try {
+    const response = await fetch(`${OLLAMA_HOST}/api/tags`);
+    if (!response.ok) return false;
+
+    // Test embedding generation
+    const testEmb = await getEmbedding("test");
+    console.log(
+      `Ollama ready! Model: ${OLLAMA_MODEL}, Dimension: ${testEmb.length}`,
+    );
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function main() {
+  console.log("=== Embedding Regeneration ===\n");
+  console.log(`Database: ${dbPath}`);
+  console.log(`Ollama: ${OLLAMA_HOST}`);
+  console.log(`Model: ${OLLAMA_MODEL}\n`);
+
+  // Check Ollama
+  console.log("Checking Ollama...");
+  if (!(await checkOllama())) {
+    console.error("\nOllama not available. Make sure it's running:");
+    console.error("  ollama serve");
+    console.error(`  ollama pull ${OLLAMA_MODEL}`);
+    process.exit(1);
+  }
+
+  // Connect to database
+  console.log("\nConnecting to database...");
+  const db = new PGlite(dbPath, { extensions: { vector } });
+  await db.waitReady;
+
+  // Count chunks needing embeddings
+  const countResult = await db.query<{ c: string }>(`
+    SELECT COUNT(*) as c FROM chunks c
+    LEFT JOIN embeddings e ON e.chunk_id = c.id
+    WHERE e.chunk_id IS NULL
+  `);
+  const total = parseInt(countResult.rows[0]?.c || "0");
+
+  console.log(`Chunks needing embeddings: ${total}`);
+
+  if (total === 0) {
+    console.log("\nAll chunks have embeddings!");
+    await db.close();
+    return;
+  }
+
+  // Estimate time
+  const estimatedMinutes = Math.ceil(total / 15 / 60); // ~15 embeddings/sec
+  console.log(`Estimated time: ~${estimatedMinutes} minutes\n`);
+
+  let processed = 0;
+  let errors = 0;
+  const startTime = Date.now();
+
+  while (processed < total) {
+    // Get batch of chunks without embeddings
+    const batch = await db.query<{ id: string; content: string }>(`
+      SELECT c.id, c.content FROM chunks c
+      LEFT JOIN embeddings e ON e.chunk_id = c.id
+      WHERE e.chunk_id IS NULL
+      LIMIT ${BATCH_SIZE}
+    `);
+
+    if (batch.rows.length === 0) break;
+
+    // Generate embeddings for batch
+    for (const row of batch.rows) {
+      try {
+        const embedding = await getEmbedding(row.content);
+        const vectorStr = `[${embedding.join(",")}]`;
+
+        await db.query(
+          `INSERT INTO embeddings (chunk_id, embedding)
+           VALUES ($1, $2::vector)
+           ON CONFLICT (chunk_id) DO NOTHING`,
+          [row.id, vectorStr],
+        );
+
+        processed++;
+      } catch (e) {
+        errors++;
+        console.error(`Error on chunk ${row.id}: ${e}`);
+      }
+    }
+
+    // Progress update every 100 chunks
+    if (processed % 100 === 0 || processed === total) {
+      const elapsed = (Date.now() - startTime) / 1000;
+      const rate = processed / elapsed;
+      const remaining = (total - processed) / rate;
+      const pct = ((processed / total) * 100).toFixed(1);
+
+      console.log(
+        `Progress: ${processed}/${total} (${pct}%) | ` +
+          `${rate.toFixed(1)}/s | ` +
+          `ETA: ${Math.ceil(remaining / 60)}min`,
+      );
+    }
+  }
+
+  // Final stats
+  const embResult = await db.query<{ c: string }>(
+    "SELECT COUNT(*) as c FROM embeddings",
+  );
+  const finalCount = parseInt(embResult.rows[0]?.c || "0");
+
+  console.log("\n=== Complete ===");
+  console.log(`Embeddings generated: ${processed}`);
+  console.log(`Total embeddings: ${finalCount}`);
+  console.log(`Errors: ${errors}`);
+  console.log(
+    `Time: ${((Date.now() - startTime) / 1000 / 60).toFixed(1)} minutes`,
+  );
+
+  await db.close();
+}
+
+main().catch((e) => {
+  console.error("Regeneration failed:", e);
+  process.exit(1);
+});

package/src/cli.ts

@@ -14,6 +14,7 @@ import {
   LibraryConfig,
   URLFetchError,
 } from "./index.js";
+import { Migration, MigrationLive } from "./services/Migration.js";
 
 /**
  * Check if a string is a URL
@@ -84,6 +85,14 @@ Commands:
 
   check                   Check if Ollama is ready
 
+  repair                  Fix database integrity issues
+                          Removes orphaned chunks/embeddings
+
+  migrate                 Database migration utilities
+    --check               Check if migration is needed
+    --import <file>       Import from SQL dump file
+    --generate-script     Generate export script for current database
+
 Options:
   --help, -h              Show this help
 
@@ -91,6 +100,8 @@ Examples:
   pdf-library add ./book.pdf --tags "programming,rust"
   pdf-library add https://example.com/paper.pdf --title "Research Paper"
   pdf-library search "machine learning" --limit 5
+  pdf-library migrate --check
+  pdf-library migrate --import backup.sql
 `;
 
 function parseArgs(args: string[]) {
@@ -314,6 +325,38 @@ const program = Effect.gen(function* () {
       break;
     }
 
+    case "repair": {
+      yield* Console.log("Checking database integrity...\n");
+      const result = yield* library.repair();
+
+      if (
+        result.orphanedChunks === 0 &&
+        result.orphanedEmbeddings === 0 &&
+        result.zeroVectorEmbeddings === 0
+      ) {
+        yield* Console.log("✓ Database is healthy - no repairs needed");
+      } else {
+        yield* Console.log("Repairs completed:");
+        if (result.orphanedChunks > 0) {
+          yield* Console.log(
+            `  • Removed ${result.orphanedChunks} orphaned chunks`,
+          );
+        }
+        if (result.orphanedEmbeddings > 0) {
+          yield* Console.log(
+            `  • Removed ${result.orphanedEmbeddings} orphaned embeddings`,
+          );
+        }
+        if (result.zeroVectorEmbeddings > 0) {
+          yield* Console.log(
+            `  • Removed ${result.zeroVectorEmbeddings} zero-dimension embeddings`,
+          );
+        }
+        yield* Console.log("\n✓ Database repaired");
+      }
+      break;
+    }
+
     default:
       yield* Console.error(`Unknown command: ${command}`);
       yield* Console.log(HELP);
@@ -321,15 +364,89 @@ const program = Effect.gen(function* () {
   }
 });
 
-// Run with error handling
-Effect.runPromise(
-  program.pipe(
-    Effect.provide(PDFLibraryLive),
-    Effect.catchAll((error) =>
-      Effect.gen(function* () {
-        yield* Console.error(`Error: ${error._tag}: ${JSON.stringify(error)}`);
-        process.exit(1);
-      }),
+// Handle migrate command separately (doesn't need full PDFLibrary)
+const args = process.argv.slice(2);
+
+if (args[0] === "migrate") {
+  const migrateProgram = Effect.gen(function* () {
+    const opts = parseArgs(args.slice(1));
+    const migration = yield* Migration;
+    const config = LibraryConfig.fromEnv();
+    const dbPath = config.dbPath.replace(".db", "");
+
+    if (opts.check) {
+      const needed = yield* migration.checkMigrationNeeded(dbPath);
+      if (needed) {
+        yield* Console.log(
+          "Migration needed:\n" + migration.getMigrationMessage(),
+        );
+      } else {
+        yield* Console.log("✓ No migration needed - database is compatible");
+      }
+    } else if (opts.import) {
+      yield* migration.importFromDump(opts.import as string, dbPath);
+      yield* Console.log("✓ Import complete");
+    } else if (opts["generate-script"]) {
+      yield* Console.log(migration.generateExportScript(dbPath));
+    } else {
+      // Default: check and show message
+      const needed = yield* migration.checkMigrationNeeded(dbPath);
+      if (needed) {
+        yield* Console.log(migration.getMigrationMessage());
+      } else {
+        yield* Console.log("✓ No migration needed - database is compatible");
+      }
+    }
+  });
+
+  Effect.runPromise(
+    migrateProgram.pipe(
+      Effect.provide(MigrationLive),
+      Effect.catchAll((error) =>
+        Effect.gen(function* () {
+          if (error._tag === "MigrationError") {
+            yield* Console.error(`Migration Error: ${error.message}`);
+          } else {
+            yield* Console.error(
+              `Error: ${error._tag}: ${JSON.stringify(error)}`,
+            );
+          }
+          process.exit(1);
+        }),
+      ),
     ),
-  ),
-);
+  );
+} else {
+  // Run with error handling
+  Effect.runPromise(
+    program.pipe(
+      Effect.provide(PDFLibraryLive),
+      Effect.catchAll((error) =>
+        Effect.gen(function* () {
+          // Check if it's a database initialization error
+          const errorStr = JSON.stringify(error);
+          if (
+            errorStr.includes("PGlite") ||
+            errorStr.includes("version") ||
+            errorStr.includes("incompatible")
+          ) {
+            yield* Console.error(
+              `Database Error: ${error._tag}: ${JSON.stringify(error)}`,
+            );
+            yield* Console.error(
+              "\nThis may be a database version compatibility issue.",
+            );
+            yield* Console.error(
+              "Run 'pdf-library migrate --check' to diagnose.",
+            );
+          } else {
+            yield* Console.error(
+              `Error: ${error._tag}: ${JSON.stringify(error)}`,
+            );
+          }
+          process.exit(1);
+        }),
+      ),
+    ),
+  );
+}

package/src/index.ts

@@ -289,6 +289,12 @@ export class PDFLibrary extends Effect.Service<PDFLibrary>()("PDFLibrary", {
             libraryPath: config.libraryPath,
           };
         }),
+
+      /**
+       * Repair database integrity issues
+       * Removes orphaned chunks and embeddings
+       */
+      repair: () => db.repair(),
     };
   }),
   dependencies: [OllamaLive, PDFExtractorLive, DatabaseLive],

package/src/services/Database.ts

@@ -76,6 +76,16 @@ export class Database extends Context.Tag("Database")<
       { documents: number; chunks: number; embeddings: number },
       DatabaseError
     >;
+
+    // Maintenance
+    readonly repair: () => Effect.Effect<
+      {
+        orphanedChunks: number;
+        orphanedEmbeddings: number;
+        zeroVectorEmbeddings: number;
+      },
+      DatabaseError
+    >;
   }
 >() {}
 
@@ -97,13 +107,10 @@ export const DatabaseLive = Layer.scoped(
     // PGlite stores data in a directory, not a single file
     const pgDataDir = config.dbPath.replace(".db", "");
 
-    // Initialize PGlite with pgvector extension
-    const db = yield* Effect.tryPromise({
-      try: () =>
-        PGlite.create({
-          dataDir: pgDataDir,
-          extensions: { vector },
-        }),
+    // Initialize PGlite with pgvector extension (0.3.x API)
+    const db = new PGlite(pgDataDir, { extensions: { vector } });
+    yield* Effect.tryPromise({
+      try: () => db.waitReady,
       catch: (e) =>
         new DatabaseError({ reason: `Failed to init PGlite: ${e}` }),
     });
@@ -467,6 +474,70 @@ export const DatabaseLive = Layer.scoped(
           },
           catch: (e) => new DatabaseError({ reason: String(e) }),
         }),
+
+      repair: () =>
+        Effect.tryPromise({
+          try: async () => {
+            // Count orphaned chunks (doc_id not in documents)
+            const orphanedChunksResult = await db.query(`
+              SELECT COUNT(*) as count FROM chunks c
+              WHERE NOT EXISTS (SELECT 1 FROM documents d WHERE d.id = c.doc_id)
+            `);
+            const orphanedChunks = Number(
+              (orphanedChunksResult.rows[0] as { count: number }).count,
+            );
+
+            // Count orphaned embeddings (chunk_id not in chunks)
+            const orphanedEmbeddingsResult = await db.query(`
+              SELECT COUNT(*) as count FROM embeddings e
+              WHERE NOT EXISTS (SELECT 1 FROM chunks c WHERE c.id = e.chunk_id)
+            `);
+            const orphanedEmbeddings = Number(
+              (orphanedEmbeddingsResult.rows[0] as { count: number }).count,
+            );
+
+            // Count zero-dimension embeddings (vector_dims returns 0 or null)
+            // Note: In pgvector, we check for malformed vectors
+            const zeroVectorResult = await db.query(`
+              SELECT COUNT(*) as count FROM embeddings 
+              WHERE embedding IS NULL OR vector_dims(embedding) = 0
+            `);
+            const zeroVectorEmbeddings = Number(
+              (zeroVectorResult.rows[0] as { count: number }).count,
+            );
+
+            // Delete orphaned embeddings first (depends on chunks)
+            if (orphanedEmbeddings > 0) {
+              await db.query(`
+                DELETE FROM embeddings e
+                WHERE NOT EXISTS (SELECT 1 FROM chunks c WHERE c.id = e.chunk_id)
+              `);
+            }
+
+            // Delete orphaned chunks (depends on documents)
+            if (orphanedChunks > 0) {
+              await db.query(`
+                DELETE FROM chunks c
+                WHERE NOT EXISTS (SELECT 1 FROM documents d WHERE d.id = c.doc_id)
+              `);
+            }
+
+            // Delete zero-dimension embeddings
+            if (zeroVectorEmbeddings > 0) {
+              await db.query(`
+                DELETE FROM embeddings 
+                WHERE embedding IS NULL OR vector_dims(embedding) = 0
+              `);
+            }
+
+            return {
+              orphanedChunks,
+              orphanedEmbeddings,
+              zeroVectorEmbeddings,
+            };
+          },
+          catch: (e) => new DatabaseError({ reason: String(e) }),
+        }),
     };
   }),
 );

package/src/services/Migration.ts

@@ -0,0 +1,319 @@
+/**
+ * PGlite Migration Service
+ *
+ * Handles migration from PGlite 0.2.x (PostgreSQL 16) to 0.3.x (PostgreSQL 17).
+ * Since automatic pg_dump migration isn't possible (0.2.x WASM crashes on current runtimes),
+ * provides detection and manual migration paths.
+ */
+
+import { Effect, Context, Layer, Schema } from "effect";
+import { existsSync, readFileSync } from "fs";
+import { join } from "path";
+import { PGlite } from "@electric-sql/pglite";
+import { vector } from "@electric-sql/pglite/vector";
+
+// ============================================================================
+// Errors
+// ============================================================================
+
+export class MigrationError extends Schema.TaggedError<MigrationError>()(
+  "MigrationError",
+  { reason: Schema.String },
+) {}
+
+// ============================================================================
+// Service Definition
+// ============================================================================
+
+export class Migration extends Context.Tag("Migration")<
+  Migration,
+  {
+    /**
+     * Check if database at given path needs migration from PG16 to PG17.
+     * Returns true if migration is needed.
+     */
+    readonly checkMigrationNeeded: (
+      dbPath: string,
+    ) => Effect.Effect<boolean, MigrationError>;
+
+    /**
+     * Get helpful error message with migration options.
+     */
+    readonly getMigrationMessage: () => string;
+
+    /**
+     * Import SQL dump into fresh PG17 database.
+     */
+    readonly importFromDump: (
+      dumpFile: string,
+      dbPath: string,
+    ) => Effect.Effect<void, MigrationError>;
+
+    /**
+     * Generate shell script for manual pg_dump with old PGlite.
+     * User runs this with PGlite 0.2.x installed to export data.
+     */
+    readonly generateExportScript: (dbPath: string) => string;
+  }
+>() {}
+
+// ============================================================================
+// Implementation
+// ============================================================================
+
+export const MigrationLive = Layer.succeed(
+  Migration,
+  Migration.of({
+    checkMigrationNeeded: (dbPath) =>
+      Effect.gen(function* () {
+        const pgDataDir = dbPath.replace(".db", "");
+
+        // If directory doesn't exist, no migration needed (fresh install)
+        if (!existsSync(pgDataDir)) {
+          return false;
+        }
+
+        // Check for PG_VERSION file
+        const versionFile = join(pgDataDir, "PG_VERSION");
+        if (!existsSync(versionFile)) {
+          // Directory exists but no version file - might be corrupted
+          // Attempt to detect by trying to open with PG17
+          return yield* Effect.tryPromise({
+            try: async () => {
+              try {
+                const testDb = await PGlite.create({
+                  dataDir: pgDataDir,
+                  extensions: { vector },
+                });
+                await testDb.close();
+                return false; // Successfully opened - no migration needed
+              } catch (e) {
+                // Failed to open - likely needs migration
+                return true;
+              }
+            },
+            catch: (e) =>
+              new MigrationError({
+                reason: `Failed to check database version: ${e}`,
+              }),
+          });
+        }
+
+        // Read PG_VERSION file
+        const version = yield* Effect.try({
+          try: () => readFileSync(versionFile, "utf8").trim(),
+          catch: (e) =>
+            new MigrationError({
+              reason: `Failed to read PG_VERSION: ${e}`,
+            }),
+        });
+
+        // If version is 16, migration is needed
+        return version === "16";
+      }),
+
+    getMigrationMessage: () => `
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  DATABASE MIGRATION REQUIRED
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+Your existing PDF library database uses PostgreSQL 16 (PGlite 0.2.x).
+This version requires PostgreSQL 17 (PGlite 0.3.x).
+
+Automatic migration is not possible because PGlite 0.2.x WASM crashes on
+current Node.js runtimes. You have two options:
+
+OPTION 1: Fresh Start (Easiest)
+────────────────────────────────
+1. Backup your database directory if desired
+2. Delete the database directory to start fresh
+3. Re-add your PDFs
+
+OPTION 2: Manual Migration (Preserve Data)
+───────────────────────────────────────────
+Migration scripts are provided in the repository.
+
+1. Export data using PGlite 0.2.x:
+   cd /path/to/pdf-library
+   npm install @electric-sql/pglite@0.2.12
+   node scripts/migration/export-pg16.mjs
+
+2. Import into PGlite 0.3.x:
+   bun run scripts/migration/import-pg17.ts
+
+3. Regenerate embeddings (if needed):
+   bun run scripts/migration/regenerate-embeddings.ts
+
+For detailed instructions, see:
+https://github.com/joelhooks/pdf-library#migration
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+`,
+
+    importFromDump: (dumpFile, dbPath) =>
+      Effect.gen(function* () {
+        // Verify dump file exists
+        if (!existsSync(dumpFile)) {
+          return yield* Effect.fail(
+            new MigrationError({
+              reason: `Dump file not found: ${dumpFile}`,
+            }),
+          );
+        }
+
+        // Read SQL dump
+        const sql = yield* Effect.try({
+          try: () => readFileSync(dumpFile, "utf8"),
+          catch: (e) =>
+            new MigrationError({
+              reason: `Failed to read dump file: ${e}`,
+            }),
+        });
+
+        // Create fresh PG17 database
+        const pgDataDir = dbPath.replace(".db", "");
+        const db = yield* Effect.tryPromise({
+          try: () =>
+            PGlite.create({
+              dataDir: pgDataDir,
+              extensions: { vector },
+            }),
+          catch: (e) =>
+            new MigrationError({
+              reason: `Failed to create new database: ${e}`,
+            }),
+        });
+
+        // Execute dump SQL
+        yield* Effect.tryPromise({
+          try: async () => {
+            await db.exec(sql);
+            await db.close();
+          },
+          catch: (e) =>
+            new MigrationError({
+              reason: `Failed to import dump: ${e}`,
+            }),
+        });
+      }),
+
+    generateExportScript: (dbPath) => {
+      const pgDataDir = dbPath.replace(".db", "");
+      const dumpFile = join(process.cwd(), "library-dump.sql");
+
+      // Generate a Node.js script that uses PGlite 0.2.x to export data
+      return `#!/usr/bin/env node
+/**
+ * PDF Library Database Export Script
+ * 
+ * This script exports your database using PGlite 0.2.x.
+ * 
+ * Prerequisites:
+ * 1. Node.js 18+
+ * 2. Install dependencies:
+ *    npm install @electric-sql/pglite@0.2.12
+ * 
+ * Usage:
+ *    node export.js
+ */
+
+const { PGlite } = require('@electric-sql/pglite');
+const { writeFileSync } = require('fs');
+
+(async () => {
+  console.log('Opening database with PGlite 0.2.x...');
+  
+  const db = new PGlite('${pgDataDir}');
+  
+  console.log('Exporting documents table...');
+  const docs = await db.query('SELECT * FROM documents');
+  
+  console.log('Exporting chunks table...');
+  const chunks = await db.query('SELECT * FROM chunks');
+  
+  console.log('Exporting embeddings table...');
+  const embeddings = await db.query('SELECT * FROM embeddings');
+  
+  // Generate SQL dump
+  let sql = '-- PDF Library Database Dump\\n';
+  sql += '-- Generated with PGlite 0.2.x\\n\\n';
+  
+  sql += 'BEGIN;\\n\\n';
+  
+  // Enable extensions
+  sql += 'CREATE EXTENSION IF NOT EXISTS vector;\\n\\n';
+  
+  // Create tables
+  sql += \`CREATE TABLE IF NOT EXISTS documents (
+    id TEXT PRIMARY KEY,
+    title TEXT NOT NULL,
+    path TEXT NOT NULL UNIQUE,
+    added_at TIMESTAMPTZ NOT NULL,
+    page_count INTEGER NOT NULL,
+    size_bytes INTEGER NOT NULL,
+    tags JSONB DEFAULT '[]',
+    metadata JSONB DEFAULT '{}'
+  );\\n\\n\`;
+  
+  sql += \`CREATE TABLE IF NOT EXISTS chunks (
+    id TEXT PRIMARY KEY,
+    doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+    page INTEGER NOT NULL,
+    chunk_index INTEGER NOT NULL,
+    content TEXT NOT NULL
+  );\\n\\n\`;
+  
+  sql += \`CREATE TABLE IF NOT EXISTS embeddings (
+    chunk_id TEXT PRIMARY KEY REFERENCES chunks(id) ON DELETE CASCADE,
+    embedding vector(1024) NOT NULL
+  );\\n\\n\`;
+  
+  // Insert documents
+  for (const row of docs.rows) {
+    const values = [
+      row.id,
+      row.title.replace(/'/g, "''"),
+      row.path.replace(/'/g, "''"),
+      row.added_at,
+      row.page_count,
+      row.size_bytes,
+      JSON.stringify(row.tags),
+      JSON.stringify(row.metadata)
+    ];
+    sql += \`INSERT INTO documents (id, title, path, added_at, page_count, size_bytes, tags, metadata) VALUES ('\${values[0]}', '\${values[1]}', '\${values[2]}', '\${values[3]}', \${values[4]}, \${values[5]}, '\${values[6]}', '\${values[7]}');\\n\`;
+  }
+  
+  sql += '\\n';
+  
+  // Insert chunks
+  for (const row of chunks.rows) {
+    const content = row.content.replace(/'/g, "''");
+    sql += \`INSERT INTO chunks (id, doc_id, page, chunk_index, content) VALUES ('\${row.id}', '\${row.doc_id}', \${row.page}, \${row.chunk_index}, '\${content}');\\n\`;
+  }
+  
+  sql += '\\n';
+  
+  // Insert embeddings
+  for (const row of embeddings.rows) {
+    sql += \`INSERT INTO embeddings (chunk_id, embedding) VALUES ('\${row.chunk_id}', '\${row.embedding}');\\n\`;
+  }
+  
+  sql += '\\nCOMMIT;\\n';
+  
+  // Write dump file
+  writeFileSync('${dumpFile}', sql);
+  
+  await db.close();
+  
+  console.log(\`\\nExport complete! Dump saved to: ${dumpFile}\`);
+  console.log(\`\\nNext steps:\`);
+  console.log(\`  1. Upgrade to PGlite 0.3.x\`);
+  console.log(\`  2. Import dump: pdf-library migration import ${dumpFile}\`);
+})().catch(err => {
+  console.error('Export failed:', err);
+  process.exit(1);
+});
+`;
+    },
+  }),
+);