vectra-js 0.9.11 → 0.9.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectra-js",
-  "version": "0.9.11",
+  "version": "0.9.12",
   "description": "A production-ready, provider-agnostic Node.js SDK for End-to-End RAG pipelines.",
   "main": "index.js",
   "scripts": {
@@ -34,7 +34,7 @@
   "author": "Abhishek N",
   "license": "GPL-3.0",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.20.9",
+    "@anthropic-ai/sdk": "^0.71.2",
     "@google/genai": "^1.34.0",
     "dotenv": "^16.6.1",
     "mammoth": "^1.11.0",

package/src/backends/milvus_store.js

@@ -45,6 +45,28 @@ class MilvusVectorStore extends VectorStore {
     return rows.map((r) => ({ id: r.id, content: r.content || '', metadata: r.metadata ? JSON.parse(r.metadata) : {} }));
   }
 
+  async fileExists(sha256, size, lastModified) {
+    if (typeof this.client.query !== 'function') return false;
+    try {
+      const expr = '';
+      const res = await this.client.query({
+        collection_name: this.collection,
+        expr,
+        output_fields: ['content', 'metadata'],
+        limit: 1
+      });
+      const rows = Array.isArray(res) ? res : (res?.data || res?.results || []);
+      return rows.some((r) => {
+        try {
+          const m = r.metadata ? JSON.parse(r.metadata) : {};
+          return m.fileSHA256 === sha256 && m.fileSize === size && m.lastModified === lastModified;
+        } catch (_) { return false; }
+      });
+    } catch (_) {
+      return false;
+    }
+  }
+
   async deleteDocuments({ ids = null, filter = null } = {}) {
     if (typeof this.client.delete !== 'function') throw new Error('deleteDocuments is not supported for this Milvus client');
     if (Array.isArray(ids) && ids.length > 0) {

package/src/backends/postgres_store.js

@@ -24,9 +24,11 @@ class PostgresVectorStore extends VectorStore {
     const tableName = config.tableName || 'document';
     const columnMap = config.columnMap || {};
     this._table = quoteTableName(tableName, 'tableName');
+    this._tableBase = tableName.split('.').pop();
     this._cContent = quoteIdentifier(columnMap.content || 'content', 'columnMap.content');
     this._cMeta = quoteIdentifier(columnMap.metadata || 'metadata', 'columnMap.metadata');
     this._cVec = quoteIdentifier(columnMap.vector || 'vector', 'columnMap.vector');
+    this._cCreatedAt = '"createdAt"';
     
     // We expect config.clientInstance to be a pg.Pool or pg.Client
     if (!this.config.clientInstance) {
@@ -45,7 +47,34 @@ class PostgresVectorStore extends VectorStore {
     // Enable pgvector extension
     await this.client.query('CREATE EXTENSION IF NOT EXISTS vector');
     
-    // Create table if not exists
+    // Detect existing column type to avoid malformed array issues
+    try {
+      const typeCheck = await this.client.query(
+        `SELECT data_type, udt_name 
+         FROM information_schema.columns 
+         WHERE table_name = $1 AND column_name = $2`,
+        [this._tableBase, this._cVec.replace(/"/g, '')]
+      );
+      const row = typeCheck.rows[0];
+      if (row) {
+        const isPgVector = row.udt_name === 'vector';
+        const isArray = row.data_type && row.data_type.toLowerCase().includes('array');
+        if (isArray && !isPgVector) {
+          throw new Error(
+            'Postgres schema mismatch: vector column is double precision[] (array). ' +
+            'Use pgvector type: vector(<dimensions>). ' +
+            'Example: ALTER TABLE ' + this._table + ' ALTER COLUMN ' + this._cVec + ' TYPE vector(1536);'
+          );
+        }
+      }
+    } catch (e) {
+      // Only throw if we explicitly detected array type; otherwise continue
+      if (String(e.message || e).includes('schema mismatch')) {
+        throw e;
+      }
+    }
+    
+    // Create table if not exists (best-effort)
     // Note: We need to know vector dimensions. We'll try to guess or use default 1536
     // If embedding dimensions are provided in config, use them
     // But store config usually doesn't have embedding config directly unless passed down
@@ -65,6 +94,47 @@ class PostgresVectorStore extends VectorStore {
     `;
     await this.client.query(createTableQuery);
     
+    // Ensure required columns exist (non-destructive)
+    try {
+      const res = await this.client.query(
+        `SELECT column_name, data_type, udt_name 
+         FROM information_schema.columns 
+         WHERE table_name = $1`,
+        [this._tableBase]
+      );
+      const cols = new Map(res.rows.map(r => [r.column_name, r]));
+      const contentCol = this._cContent.replace(/"/g, '');
+      const metaCol = this._cMeta.replace(/"/g, '');
+      const vecCol = this._cVec.replace(/"/g, '');
+      const createdAtCol = this._cCreatedAt.replace(/"/g, '');
+      
+      if (!cols.has(contentCol)) {
+        await this.client.query(`ALTER TABLE ${this._table} ADD COLUMN ${this._cContent} TEXT`);
+      }
+      if (!cols.has(metaCol)) {
+        await this.client.query(`ALTER TABLE ${this._table} ADD COLUMN ${this._cMeta} JSONB`);
+      }
+      if (!cols.has(vecCol)) {
+        await this.client.query(`ALTER TABLE ${this._table} ADD COLUMN ${this._cVec} vector(${dim})`);
+      } else {
+        const vinfo = cols.get(vecCol);
+        const isPgVector = vinfo && vinfo.udt_name === 'vector';
+        const isArray = vinfo && vinfo.data_type && vinfo.data_type.toLowerCase().includes('array');
+        if (isArray && !isPgVector) {
+          throw new Error(
+            'Postgres schema mismatch: vector column is double precision[] (array). ' +
+            'Use pgvector type: vector(' + dim + '). ' +
+            'Example: ALTER TABLE ' + this._table + ' ALTER COLUMN ' + this._cVec + ' TYPE vector(' + dim + ');'
+          );
+        }
+      }
+      if (!cols.has(createdAtCol)) {
+        await this.client.query(`ALTER TABLE ${this._table} ADD COLUMN ${this._cCreatedAt} TIMESTAMP WITH TIME ZONE DEFAULT NOW()`);
+      }
+    } catch (_) {
+      // best-effort; ignore
+    }
+    
     // Create HNSW index for faster search
     // checking if index exists is hard in raw sql cross-version, 
     // simpler to CREATE INDEX IF NOT EXISTS which pg supports in recent versions
@@ -72,12 +142,17 @@ class PostgresVectorStore extends VectorStore {
     try {
         await this.client.query(`CREATE INDEX IF NOT EXISTS "${this._table.replace(/"/g, '')}_vec_idx" ON ${this._table} USING hnsw (${this._cVec} vector_cosine_ops)`);
     } catch (e) {
-        console.warn('Could not create vector index (might be fine if not supported):', e.message);
+        // Fallback to ivfflat when hnsw not supported
+        try {
+          await this.client.query(`CREATE INDEX IF NOT EXISTS "${this._table.replace(/"/g, '')}_vec_idx" ON ${this._table} USING ivfflat (${this._cVec} vector_cosine_ops)`);
+        } catch (e2) {
+          console.warn('Could not create vector index (might be fine if not supported):', e.message);
+        }
     }
   }
 
   async addDocuments(docs) {
-    const q = `INSERT INTO ${this._table} ("id", ${this._cContent}, ${this._cMeta}, ${this._cVec}, "createdAt") VALUES ($1, $2, $3, $4, NOW())`;
+    const q = `INSERT INTO ${this._table} ("id", ${this._cContent}, ${this._cMeta}, ${this._cVec}, "createdAt") VALUES ($1, $2, $3, $4, NOW()) ON CONFLICT ("id") DO NOTHING`;
     
     for (const doc of docs) {
       const id = doc.id || uuidv4();
@@ -186,6 +261,22 @@ class PostgresVectorStore extends VectorStore {
     
     return Object.values(combined).sort((a, b) => b.score - a.score).slice(0, limit);
   }
+  
+  async fileExists(sha256, size, lastModified) {
+    try {
+      const q = `
+        SELECT 1 
+        FROM ${this._table} 
+        WHERE ${this._cMeta} @> $1 
+        LIMIT 1
+      `;
+      const metaFilter = JSON.stringify({ fileSHA256: sha256, fileSize: size, lastModified });
+      const res = await this.client.query(q, [metaFilter]);
+      return res.rowCount > 0;
+    } catch (_) {
+      return false;
+    }
+  }
 }
 
 module.exports = { PostgresVectorStore };

package/src/backends/prisma_store.js

@@ -35,7 +35,7 @@ class PrismaVectorStore extends VectorStore {
   }
   async addDocuments(docs) {
     const { clientInstance } = this.config;
-    const q = `INSERT INTO ${this._table} ("id", ${this._cContent}, ${this._cMeta}, ${this._cVec}, "createdAt") VALUES ($1, $2, $3, $4::vector, NOW())`;
+    const q = `INSERT INTO ${this._table} ("id", ${this._cContent}, ${this._cMeta}, ${this._cVec}, "createdAt") VALUES ($1, $2, $3, $4::vector, NOW()) ON CONFLICT ("id") DO NOTHING`;
     for (const doc of docs) {
         const id = doc.id || uuidv4();
         const vec = JSON.stringify(this.normalizeVector(doc.embedding));
@@ -108,6 +108,7 @@ class PrismaVectorStore extends VectorStore {
     const idxFts = `"${base}_content_fts_gin"`;
     try {
       await clientInstance.$executeRawUnsafe('CREATE EXTENSION IF NOT EXISTS vector');
+      await this._ensureColumns();
       await clientInstance.$executeRawUnsafe(`CREATE INDEX IF NOT EXISTS ${idxVec} ON ${this._table} USING ivfflat (${this._cVec} vector_cosine_ops) WITH (lists = 100);`);
       await clientInstance.$executeRawUnsafe(`CREATE INDEX IF NOT EXISTS ${idxFts} ON ${this._table} USING GIN (to_tsvector('english', ${this._cContent}));`);
     } catch (e) {
@@ -115,6 +116,54 @@ class PrismaVectorStore extends VectorStore {
     }
   }
   
+  async _ensureColumns() {
+    const { clientInstance } = this.config;
+    const dim = 1536;
+    const createTableQuery = `
+      CREATE TABLE IF NOT EXISTS ${this._table} (
+        "id" TEXT PRIMARY KEY,
+        ${this._cContent} TEXT,
+        ${this._cMeta} JSONB,
+        ${this._cVec} vector(${dim}),
+        "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+      )
+    `;
+    await clientInstance.$executeRawUnsafe(createTableQuery);
+    try {
+      const res = await clientInstance.$queryRawUnsafe(
+        `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_name = $1`,
+        this._tableBase
+      );
+      const cols = new Map(res.map(r => [r.column_name, r]));
+      const contentCol = this._cContent.replace(/"/g, '');
+      const metaCol = this._cMeta.replace(/"/g, '');
+      const vecCol = this._cVec.replace(/"/g, '');
+      const createdAtCol = 'createdAt';
+      if (!cols.has(contentCol)) {
+        await clientInstance.$executeRawUnsafe(`ALTER TABLE ${this._table} ADD COLUMN ${this._cContent} TEXT`);
+      }
+      if (!cols.has(metaCol)) {
+        await clientInstance.$executeRawUnsafe(`ALTER TABLE ${this._table} ADD COLUMN ${this._cMeta} JSONB`);
+      }
+      if (!cols.has(vecCol)) {
+        await clientInstance.$executeRawUnsafe(`ALTER TABLE ${this._table} ADD COLUMN ${this._cVec} vector(${dim})`);
+      } else {
+        const vinfo = cols.get(vecCol);
+        const isPgVector = vinfo && vinfo.udt_name === 'vector';
+        const isArray = vinfo && vinfo.data_type && vinfo.data_type.toLowerCase().includes('array');
+        if (isArray && !isPgVector) {
+          throw new Error(
+            'Postgres schema mismatch: vector column is double precision[] (array). Use pgvector type: vector(' + dim + '). Example: ALTER TABLE ' + this._table + ' ALTER COLUMN ' + this._cVec + ' TYPE vector(' + dim + ');'
+          );
+        }
+      }
+      if (!cols.has(createdAtCol)) {
+        await clientInstance.$executeRawUnsafe(`ALTER TABLE ${this._table} ADD COLUMN "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT NOW()`);
+      }
+    } catch (_) {
+    }
+  }
+  
   async fileExists(sha256, size, lastModified) {
     const { clientInstance } = this.config;
     const payload = JSON.stringify({ fileSHA256: sha256, fileSize: size, lastModified });

package/src/backends/qdrant_store.js

@@ -52,6 +52,16 @@ class QdrantVectorStore extends VectorStore {
     }
     return out;
   }
+  async fileExists(sha256, size, lastModified) {
+    const filter = this.normalizeFilter({ fileSHA256: sha256, fileSize: size, lastModified });
+    try {
+      const res = await this.client.scroll(this.collection, { limit: 1, filter });
+      const points = res?.points || res?.result?.points || [];
+      return points.length > 0;
+    } catch (_) {
+      return false;
+    }
+  }
   async deleteDocuments({ ids = null, filter = null } = {}) {
     if (typeof this.client.delete !== 'function') throw new Error('deleteDocuments is not supported for this Qdrant client');
     if (Array.isArray(ids) && ids.length > 0) {

package/src/telemetry.js

@@ -103,7 +103,7 @@ class TelemetryManager {
 
     const batch = this.queue.splice(0, this.queue.length);
 
-    if (!global.fetch || !SUPABASE_ANON_KEY) {
+    if (!global.fetch) {
       if (process.env.VECTRA_TELEMETRY_DEBUG) {
         console.log('Telemetry batch (debug):', batch);
       }