pdf-brain 0.3.1 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pdf-brain",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "Local PDF knowledge base with vector search",
   "type": "module",
   "main": "src/index.ts",

package/src/cli.ts

@@ -85,6 +85,9 @@ Commands:
 
   check                   Check if Ollama is ready
 
+  repair                  Fix database integrity issues
+                          Removes orphaned chunks/embeddings
+
   migrate                 Database migration utilities
     --check               Check if migration is needed
     --import <file>       Import from SQL dump file
@@ -322,6 +325,38 @@ const program = Effect.gen(function* () {
       break;
     }
 
+    case "repair": {
+      yield* Console.log("Checking database integrity...\n");
+      const result = yield* library.repair();
+
+      if (
+        result.orphanedChunks === 0 &&
+        result.orphanedEmbeddings === 0 &&
+        result.zeroVectorEmbeddings === 0
+      ) {
+        yield* Console.log("✓ Database is healthy - no repairs needed");
+      } else {
+        yield* Console.log("Repairs completed:");
+        if (result.orphanedChunks > 0) {
+          yield* Console.log(
+            `  • Removed ${result.orphanedChunks} orphaned chunks`,
+          );
+        }
+        if (result.orphanedEmbeddings > 0) {
+          yield* Console.log(
+            `  • Removed ${result.orphanedEmbeddings} orphaned embeddings`,
+          );
+        }
+        if (result.zeroVectorEmbeddings > 0) {
+          yield* Console.log(
+            `  • Removed ${result.zeroVectorEmbeddings} zero-dimension embeddings`,
+          );
+        }
+        yield* Console.log("\n✓ Database repaired");
+      }
+      break;
+    }
+
     default:
       yield* Console.error(`Unknown command: ${command}`);
       yield* Console.log(HELP);

package/src/index.ts

@@ -289,6 +289,12 @@ export class PDFLibrary extends Effect.Service<PDFLibrary>()("PDFLibrary", {
             libraryPath: config.libraryPath,
           };
         }),
+
+      /**
+       * Repair database integrity issues
+       * Removes orphaned chunks and embeddings
+       */
+      repair: () => db.repair(),
     };
   }),
   dependencies: [OllamaLive, PDFExtractorLive, DatabaseLive],

package/src/services/Database.ts

@@ -76,6 +76,16 @@ export class Database extends Context.Tag("Database")<
       { documents: number; chunks: number; embeddings: number },
       DatabaseError
     >;
+
+    // Maintenance
+    readonly repair: () => Effect.Effect<
+      {
+        orphanedChunks: number;
+        orphanedEmbeddings: number;
+        zeroVectorEmbeddings: number;
+      },
+      DatabaseError
+    >;
   }
 >() {}
 
@@ -464,6 +474,70 @@ export const DatabaseLive = Layer.scoped(
           },
           catch: (e) => new DatabaseError({ reason: String(e) }),
         }),
+
+      repair: () =>
+        Effect.tryPromise({
+          try: async () => {
+            // Count orphaned chunks (doc_id not in documents)
+            const orphanedChunksResult = await db.query(`
+              SELECT COUNT(*) as count FROM chunks c
+              WHERE NOT EXISTS (SELECT 1 FROM documents d WHERE d.id = c.doc_id)
+            `);
+            const orphanedChunks = Number(
+              (orphanedChunksResult.rows[0] as { count: number }).count,
+            );
+
+            // Count orphaned embeddings (chunk_id not in chunks)
+            const orphanedEmbeddingsResult = await db.query(`
+              SELECT COUNT(*) as count FROM embeddings e
+              WHERE NOT EXISTS (SELECT 1 FROM chunks c WHERE c.id = e.chunk_id)
+            `);
+            const orphanedEmbeddings = Number(
+              (orphanedEmbeddingsResult.rows[0] as { count: number }).count,
+            );
+
+            // Count zero-dimension embeddings (vector_dims returns 0 or null)
+            // Note: In pgvector, we check for malformed vectors
+            const zeroVectorResult = await db.query(`
+              SELECT COUNT(*) as count FROM embeddings 
+              WHERE embedding IS NULL OR vector_dims(embedding) = 0
+            `);
+            const zeroVectorEmbeddings = Number(
+              (zeroVectorResult.rows[0] as { count: number }).count,
+            );
+
+            // Delete orphaned embeddings first (depends on chunks)
+            if (orphanedEmbeddings > 0) {
+              await db.query(`
+                DELETE FROM embeddings e
+                WHERE NOT EXISTS (SELECT 1 FROM chunks c WHERE c.id = e.chunk_id)
+              `);
+            }
+
+            // Delete orphaned chunks (depends on documents)
+            if (orphanedChunks > 0) {
+              await db.query(`
+                DELETE FROM chunks c
+                WHERE NOT EXISTS (SELECT 1 FROM documents d WHERE d.id = c.doc_id)
+              `);
+            }
+
+            // Delete zero-dimension embeddings
+            if (zeroVectorEmbeddings > 0) {
+              await db.query(`
+                DELETE FROM embeddings 
+                WHERE embedding IS NULL OR vector_dims(embedding) = 0
+              `);
+            }
+
+            return {
+              orphanedChunks,
+              orphanedEmbeddings,
+              zeroVectorEmbeddings,
+            };
+          },
+          catch: (e) => new DatabaseError({ reason: String(e) }),
+        }),
     };
   }),
 );