vectra-js 0.9.6 → 0.9.8

package/RELEASE_NOTES.md

@@ -0,0 +1,15 @@
+# Vectra-js 0.9.7-beta Release Notes
+
+## New Features
+*   **Native PostgreSQL Support**: Added `PostgresVectorStore` for direct PostgreSQL vector operations without Prisma dependency.
+*   **Enhanced Validation**: Integrated Zod for robust configuration schema validation.
+*   **Observability**: Added SQLite-based logging for better telemetry and debugging.
+
+## Improvements
+*   **Code Quality**: Refactored core logic to reduce magic numbers and improve maintainability (SonarCloud fixes).
+*   **Linting**: Migrated to ESLint flat config and enforced stricter code style (no-var, prefer-const).
+*   **CLI**: Improved stability and error handling in CLI commands.
+
+## Fixes
+*   Fixed potential unhandled promise rejections in observability logger.
+*   Fixed console log noise in production builds.

package/examples/chromadb.js

@@ -0,0 +1,96 @@
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '.env') });
+const { ChromaClient } = require('chromadb');
+const { VectraClient, ProviderType, ChunkingStrategy, RetrievalStrategy } = require('../vectra-js');
+const { LoggingCallbackHandler } = require('../vectra-js/src/callbacks');
+
+async function runSimulation() {
+  console.log('=== Starting Vectra SDK Simulation (Node.js) ===\n');
+
+  const chroma = new ChromaClient({
+    ssl: false,
+    host: "localhost",
+    port: 8000,
+    headers: {},
+  });
+
+  const config = {
+    embedding: {
+      provider: ProviderType.GEMINI,
+      apiKey: process.env.GEMINI_KEY,
+      modelName: 'gemini-embedding-001',
+      dimensions: 1536,
+    },
+    llm: {
+      provider: ProviderType.GEMINI,
+      apiKey: process.env.GEMINI_KEY,
+      modelName: 'gemini-2.5-flash-lite',
+    },
+    chunking: {
+      strategy: ChunkingStrategy.RECURSIVE,
+      chunkSize: 500,
+      chunkOverlap: 200,
+    },
+    database: {
+      type: 'chroma',
+      tableName: 'rag_collection',
+      clientInstance: chroma,
+      columnMap: { content: 'content', vector: 'embedding', metadata: 'metadata' },
+    },
+    retrieval: {
+      strategy: RetrievalStrategy.HYBRID,
+    },
+    reranking: {
+      enabled: true,
+      topN: 5,
+      windowSize: 20,
+      llmConfig: {
+        provider: ProviderType.GEMINI,
+        apiKey: process.env.GEMINI_KEY,
+        modelName: 'gemini-2.5-flash-lite',
+      }
+    },
+    observability: {
+        enabled: true,
+        projectId: "node-test-project",
+        sqlitePath: path.resolve(__dirname, "db/node-observability.db")
+    },
+    callbacks: [
+      new LoggingCallbackHandler(),
+      { onEmbeddingStart: (c) => console.info(`[RAG] Embedding ${c} chunks...`) }
+    ],
+  };
+
+  console.log('Initializing Client...');
+  const client = new VectraClient(config);
+  if (config.database.type === 'prisma' && client.vectorStore.ensureIndexes) {
+    await client.vectorStore.ensureIndexes();
+  }
+  await client.ingestDocuments('data/llm-ebook-part1-1.pdf');
+
+  console.log('\n--- Step 1: Standard Query (Hybrid) ---\n');
+  try {
+    const result = await client.queryRAG('What is LLM?');
+    console.log('Answer:', result.answer);
+  } catch (error) {
+    console.error('Query failed:', error);
+  }
+
+  console.log('\n--- Step 2: Streaming Query ---\n');
+  try {
+    const stream = await client.queryRAG('Tell me more about LLM...', null, true);
+    process.stdout.write('Stream Output: ');
+    for await (const chunk of stream) {
+      if (typeof chunk === 'string') {
+        process.stdout.write(chunk);
+      } else if (chunk && chunk.delta) {
+        process.stdout.write(chunk.delta);
+      }
+    }
+    process.stdout.write('\n');
+  } catch (error) {
+    console.error('Streaming failed:', error);
+  }
+}
+
+runSimulation();

package/examples/pg-prisma.js

@@ -0,0 +1,119 @@
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '.env') });
+const { PrismaClient } = require('@prisma/client');
+const { PrismaPg } = require('@prisma/adapter-pg');
+const { Pool } = require('pg');
+const { VectraClient, ProviderType, ChunkingStrategy, RetrievalStrategy } = require('../vectra-js');
+const { LoggingCallbackHandler } = require('../vectra-js/src/callbacks');
+
+async function runSimulation() {
+  console.log('=== Starting Vectra SDK Simulation (Node.js + Prisma) ===\n');
+
+  const connectionString = process.env.DATABASE_URL;
+  const pool = new Pool({ connectionString });
+  const adapter = new PrismaPg(pool);
+  const prisma = new PrismaClient({ adapter });
+
+  const config = {
+    embedding: {
+      provider: ProviderType.GEMINI,
+      apiKey: process.env.GEMINI_KEY,
+      modelName: 'gemini-embedding-001',
+      dimensions: 1536,
+    },
+    llm: {
+      provider: ProviderType.GEMINI,
+      apiKey: process.env.GEMINI_KEY,
+      modelName: 'gemini-2.5-flash-lite',
+    },
+    chunking: {
+      strategy: ChunkingStrategy.RECURSIVE,
+      chunkSize: 1000,
+      chunkOverlap: 200,
+    },
+    database: {
+      type: 'prisma',
+      tableName: 'Document',
+      clientInstance: prisma,
+      columnMap: { content: 'content', vector: 'embedding', metadata: 'metadata' },
+    },
+    retrieval: {
+      strategy: RetrievalStrategy.HYBRID,
+    },
+    reranking: {
+      enabled: true,
+      topN: 5,
+      windowSize: 20,
+      llmConfig: {
+        provider: ProviderType.GEMINI,
+        apiKey: process.env.GEMINI_KEY,
+        modelName: 'gemini-2.5-flash-lite',
+      }
+    },
+    observability: {
+        enabled: true,
+        projectId: "node-test-project",
+        sqlitePath: path.resolve(__dirname, "db/node-observability.db")
+    },
+    callbacks: [
+      new LoggingCallbackHandler(),
+      { onEmbeddingStart: (c) => console.info(`[RAG] Embedding ${c} chunks...`) }
+    ],
+  };
+
+  console.log('Initializing Client...');
+  const client = new VectraClient(config);
+  
+  if (config.database.type === 'prisma' && client.vectorStore.ensureIndexes) {
+    try {
+        await client.vectorStore.ensureIndexes();
+        console.log('Database indexes ensured.');
+    } catch (e) {
+        console.warn('Index creation warning (may already exist):', e.message);
+    }
+  }
+  
+  // Clean up table for simulation using Prisma
+  try {
+    // Note: Prisma doesn't support deleteMany on tables with unsupported types (like vector) easily in all versions
+    // or sometimes we need to use executeRaw. 
+    // Since Document model has unsupported field, standard deleteMany might work but let's check.
+    // However, it is safer to use raw query if standard model usage is limited.
+    await prisma.$executeRawUnsafe(`DELETE FROM "Document"`);
+    console.log('Cleared existing documents from table.');
+  } catch (e) {
+    console.warn('Could not clear table:', e.message);
+  }
+  
+  await client.ingestDocuments('data/sample.txt');
+
+  console.log('\n--- Step 1: Standard Query (Hybrid) ---\n');
+  try {
+    const result = await client.queryRAG('What is RAG?');
+    console.log('Answer:', result.answer);
+  } catch (error) {
+    console.error('Query failed:', error);
+  }
+
+  console.log('\n--- Step 2: Streaming Query ---\n');
+  try {
+    const stream = await client.queryRAG('Tell me more...', null, true);
+    process.stdout.write('Stream Output: ');
+    for await (const chunk of stream) {
+      if (typeof chunk === 'string') {
+        process.stdout.write(chunk);
+      } else if (chunk && chunk.delta) {
+        process.stdout.write(chunk.delta);
+      }
+    }
+    process.stdout.write('\n');
+  } catch (error) {
+    console.error('Streaming failed:', error);
+  }
+  
+  // Clean up
+  await prisma.$disconnect();
+  await pool.end();
+}
+
+runSimulation();

package/examples/postgress.js

@@ -0,0 +1,115 @@
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '.env') });
+const { PrismaClient } = require('@prisma/client');
+const { PrismaPg } = require('@prisma/adapter-pg');
+const { Pool } = require('pg');
+const { VectraClient, ProviderType, ChunkingStrategy, RetrievalStrategy } = require('../vectra-js');
+const { LoggingCallbackHandler } = require('../vectra-js/src/callbacks');
+
+async function runSimulation() {
+  console.log('=== Starting Vectra SDK Simulation (Node.js + Prisma) ===\n');
+
+  const connectionString = process.env.DATABASE_URL;
+  const pool = new Pool({ connectionString });
+  // const adapter = new PrismaPg({ pool });
+  // const prisma = new PrismaClient({ adapter });
+
+  const config = {
+    embedding: {
+      provider: ProviderType.GEMINI,
+      apiKey: process.env.GEMINI_KEY,
+      modelName: 'gemini-embedding-001',
+      dimensions: 1536,
+    },
+    llm: {
+      provider: ProviderType.GEMINI,
+      apiKey: process.env.GEMINI_KEY,
+      modelName: 'gemini-2.5-flash-lite',
+    },
+    chunking: {
+      strategy: ChunkingStrategy.RECURSIVE,
+      chunkSize: 1000,
+      chunkOverlap: 200,
+    },
+    database: {
+      type: 'postgres',
+      tableName: 'Document',
+      clientInstance: pool,
+      columnMap: { content: 'content', vector: 'embedding', metadata: 'metadata' },
+    },
+    retrieval: {
+      strategy: RetrievalStrategy.HYBRID,
+    },
+    reranking: {
+      enabled: true,
+      topN: 5,
+      windowSize: 20,
+      llmConfig: {
+        provider: ProviderType.GEMINI,
+        apiKey: process.env.GEMINI_KEY,
+        modelName: 'gemini-2.5-flash-lite',
+      }
+    },
+    observability: {
+        enabled: true,
+        projectId: "node-test-project",
+        sqlitePath: path.resolve(__dirname, "db/node-observability.db")
+    },
+    callbacks: [
+      new LoggingCallbackHandler(),
+      { onEmbeddingStart: (c) => console.info(`[RAG] Embedding ${c} chunks...`) }
+    ],
+  };
+
+  console.log('Initializing Client...');
+  const client = new VectraClient(config);
+  
+  if ((config.database.type === 'prisma' || config.database.type === 'postgres') && client.vectorStore.ensureIndexes) {
+    try {
+        await client.vectorStore.ensureIndexes();
+        console.log('Database indexes ensured.');
+    } catch (e) {
+        console.warn('Index creation warning (may already exist):', e.message);
+    }
+  }
+  
+  // Clean up table for simulation
+  if (config.database.type === 'postgres') {
+      try {
+        await pool.query(`DELETE FROM "${config.database.tableName}"`); // Use quoted identifier
+        console.log('Cleared existing documents from table.');
+      } catch (e) {
+        console.warn('Could not clear table:', e.message);
+      }
+  }
+  
+  await client.ingestDocuments('data/sample.txt');
+
+  console.log('\n--- Step 1: Standard Query (Hybrid) ---\n');
+  try {
+    const result = await client.queryRAG('What is RAG?');
+    console.log('Answer:', result.answer);
+  } catch (error) {
+    console.error('Query failed:', error);
+  }
+
+  console.log('\n--- Step 2: Streaming Query ---\n');
+  try {
+    const stream = await client.queryRAG('Tell me more...', null, true);
+    process.stdout.write('Stream Output: ');
+    for await (const chunk of stream) {
+      if (typeof chunk === 'string') {
+        process.stdout.write(chunk);
+      } else if (chunk && chunk.delta) {
+        process.stdout.write(chunk.delta);
+      }
+    }
+    process.stdout.write('\n');
+  } catch (error) {
+    console.error('Streaming failed:', error);
+  }
+
+  await pool.end();
+}
+
+runSimulation();

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "vectra-js",
-  "version": "0.9.6",
+  "version": "0.9.8",
   "description": "A production-ready, provider-agnostic Node.js SDK for End-to-End RAG pipelines.",
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
     "prisma:generate": "prisma generate",
-    "lint": "eslint . --ext .js,.cjs,.mjs",
-    "lint:fix": "eslint . --ext .js,.cjs,.mjs --fix"
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix"
   },
   "bin": {
     "vectra": "bin/vectra.js"
@@ -40,6 +40,7 @@
     "mammoth": "^1.11.0",
     "openai": "^6.15.0",
     "pdf-parse": "^2.4.5",
+    "pg": "^8.16.3",
     "sqlite3": "^5.1.7",
     "uuid": "^9.0.1",
     "xlsx": "^0.18.5",

package/src/backends/gemini.js

@@ -19,14 +19,21 @@ class GeminiBackend {
   }
 
   async embedDocuments(texts) {
-    const res = await this._retry(() => this.client.models.embedContent({
-      model: this.config.modelName,
-      contents: texts,
-      config: { outputDimensionality: this.config.dimensions }
-    }));
-    const out = res?.embeddings || res?.data?.embeddings;
-    if (!out || !Array.isArray(out)) throw new Error('Gemini embedding response missing embeddings');
-    return out.map(e => e.values || e.embedding?.values || e);
+    const BATCH_SIZE = 100;
+    const allEmbeddings = [];
+
+    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+      const batch = texts.slice(i, i + BATCH_SIZE);
+      const res = await this._retry(() => this.client.models.embedContent({
+        model: this.config.modelName,
+        contents: batch,
+        config: { outputDimensionality: this.config.dimensions }
+      }));
+      const out = res?.embeddings || res?.data?.embeddings;
+      if (!out || !Array.isArray(out)) throw new Error('Gemini embedding response missing embeddings');
+      allEmbeddings.push(...out.map(e => e.values || e.embedding?.values || e));
+    }
+    return allEmbeddings;
   }
   async embedQuery(text) {
     const res = await this._retry(() => this.client.models.embedContent({

package/src/backends/openrouter.js

@@ -15,8 +15,8 @@ class OpenRouterBackend {
     });
   }
 
-  async embedDocuments(texts) { throw new Error('OpenRouter does not support embeddings via this SDK.'); }
-  async embedQuery(text) { throw new Error('OpenRouter does not support embeddings via this SDK.'); }
+  async embedDocuments(_) { throw new Error('OpenRouter does not support embeddings via this SDK.'); }
+  async embedQuery(_) { throw new Error('OpenRouter does not support embeddings via this SDK.'); }
 
   async generate(prompt, sys) {
     const msgs = [];

package/src/backends/postgres_store.js

@@ -0,0 +1,191 @@
+const { v4: uuidv4 } = require('uuid');
+const { VectorStore } = require('../interfaces');
+
+const isSafeIdentifier = (value) => typeof value === 'string' && /^[A-Za-z_][A-Za-z0-9_]*$/.test(value);
+const assertSafeIdentifier = (value, label) => {
+  if (!isSafeIdentifier(value)) throw new Error(`Unsafe SQL identifier for ${label}`);
+};
+const quoteIdentifier = (value, label) => {
+  assertSafeIdentifier(value, label);
+  return `"${value}"`;
+};
+const quoteTableName = (value, label) => {
+  if (typeof value !== 'string' || value.trim().length === 0) throw new Error(`Unsafe SQL identifier for ${label}`);
+  const parts = value.split('.').map(p => p.trim()).filter(Boolean);
+  if (parts.length === 0 || parts.length > 2) throw new Error(`Unsafe SQL identifier for ${label}`);
+  parts.forEach((p, i) => assertSafeIdentifier(p, i === 0 && parts.length === 2 ? `${label} schema` : `${label} table`));
+  return parts.map(p => `"${p}"`).join('.');
+};
+
+class PostgresVectorStore extends VectorStore {
+  constructor(config) {
+    super();
+    this.config = config;
+    const tableName = config.tableName || 'document';
+    const columnMap = config.columnMap || {};
+    this._table = quoteTableName(tableName, 'tableName');
+    this._cContent = quoteIdentifier(columnMap.content || 'content', 'columnMap.content');
+    this._cMeta = quoteIdentifier(columnMap.metadata || 'metadata', 'columnMap.metadata');
+    this._cVec = quoteIdentifier(columnMap.vector || 'vector', 'columnMap.vector');
+    
+    // We expect config.clientInstance to be a pg.Pool or pg.Client
+    if (!this.config.clientInstance) {
+        throw new Error('PostgresVectorStore requires a clientInstance (pg.Pool or pg.Client)');
+    }
+    this.client = this.config.clientInstance;
+  }
+
+  normalizeVector(v) {
+    const m = Math.sqrt(v.reduce((s, x) => s + x * x, 0));
+    return m === 0 ? v : v.map(x => x / m);
+  }
+
+  // Helper to ensure table and extension exist
+  async ensureIndexes() {
+    // Enable pgvector extension
+    await this.client.query('CREATE EXTENSION IF NOT EXISTS vector');
+    
+    // Create table if not exists
+    // Note: We need to know vector dimensions. We'll try to guess or use default 1536
+    // If embedding dimensions are provided in config, use them
+    // But store config usually doesn't have embedding config directly unless passed down
+    // For now we will assume the user creates the table or we default to 1536 (OpenAI)
+    // A better approach is to rely on user schema, but for convenience:
+    const dim = 1536; // Default to OpenAI dimension if unknown.
+    // However, if the table exists, we don't change it.
+    
+    const createTableQuery = `
+      CREATE TABLE IF NOT EXISTS ${this._table} (
+        "id" TEXT PRIMARY KEY,
+        ${this._cContent} TEXT,
+        ${this._cMeta} JSONB,
+        ${this._cVec} vector(${dim}),
+        "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+      )
+    `;
+    await this.client.query(createTableQuery);
+    
+    // Create HNSW index for faster search
+    // checking if index exists is hard in raw sql cross-version, 
+    // simpler to CREATE INDEX IF NOT EXISTS which pg supports in recent versions
+    // or catch error
+    try {
+        await this.client.query(`CREATE INDEX IF NOT EXISTS "${this._table.replace(/"/g, '')}_vec_idx" ON ${this._table} USING hnsw (${this._cVec} vector_cosine_ops)`);
+    } catch (e) {
+        console.warn('Could not create vector index (might be fine if not supported):', e.message);
+    }
+  }
+
+  async addDocuments(docs) {
+    const q = `INSERT INTO ${this._table} ("id", ${this._cContent}, ${this._cMeta}, ${this._cVec}, "createdAt") VALUES ($1, $2, $3, $4, NOW())`;
+    
+    for (const doc of docs) {
+      const id = doc.id || uuidv4();
+      const vec = `[${this.normalizeVector(doc.embedding).join(',')}]`; // pgvector format
+      try {
+        await this.client.query(q, [id, doc.content, doc.metadata, vec]);
+      } catch (e) {
+        const msg = e?.message || String(e);
+        if (msg.includes('vector') && msg.includes('dimension')) {
+           throw new Error('DimensionMismatchError: Embedding dimension does not match pgvector column.');
+        }
+        throw e;
+      }
+    }
+  }
+
+  async upsertDocuments(docs) {
+      const q = `
+        INSERT INTO ${this._table} ("id", ${this._cContent}, ${this._cMeta}, ${this._cVec}, "createdAt") 
+        VALUES ($1, $2, $3, $4, NOW()) 
+        ON CONFLICT ("id") 
+        DO UPDATE SET 
+            ${this._cContent} = EXCLUDED.${this._cContent}, 
+            ${this._cMeta} = EXCLUDED.${this._cMeta}, 
+            ${this._cVec} = EXCLUDED.${this._cVec}
+      `;
+      
+      for (const doc of docs) {
+        const id = doc.id || uuidv4();
+        const vec = `[${this.normalizeVector(doc.embedding).join(',')}]`;
+        await this.client.query(q, [id, doc.content, doc.metadata, vec]);
+      }
+  }
+
+  async similaritySearch(vector, limit = 5, filter = null) {
+    const vec = `[${this.normalizeVector(vector).join(',')}]`;
+    let where = ""; 
+    const params = [vec];
+    
+    if (filter) { 
+        where = `WHERE ${this._cMeta} @> $2`; 
+        params.push(filter); 
+    }
+    
+    const limitIdx = params.length + 1;
+    // <=> is cosine distance. 1 - distance = similarity (roughly)
+    const q = `
+        SELECT ${this._cContent} as content, ${this._cMeta} as metadata, 1 - (${this._cVec} <=> $1) as score 
+        FROM ${this._table} 
+        ${where} 
+        ORDER BY ${this._cVec} <=> $1 ASC 
+        LIMIT $${limitIdx}
+    `;
+    params.push(Math.max(1, Number(limit) || 5));
+
+    const res = await this.client.query(q, params);
+    return res.rows.map(r => ({ content: r.content, metadata: r.metadata, score: r.score }));
+  }
+
+  async hybridSearch(text, vector, limit = 5, filter = null) {
+    // 1. Semantic search
+    const semantic = await this.similaritySearch(vector, limit * 2, filter);
+    
+    // 2. Keyword search using to_tsvector
+    // We assume english config 'simple' or 'english'
+    const params = [text];
+    let where = "";
+    if (filter) {
+        where = `AND ${this._cMeta} @> $2`;
+        params.push(filter);
+    }
+    const limitIdx = params.length + 1;
+    
+    const q = `
+        SELECT ${this._cContent} as content, ${this._cMeta} as metadata 
+        FROM ${this._table} 
+        WHERE to_tsvector('english', ${this._cContent}) @@ plainto_tsquery('english', $1) 
+        ${where} 
+        LIMIT $${limitIdx}
+    `;
+    params.push(Math.max(1, Number(limit) || 5) * 2);
+
+    let lexical = [];
+    try {
+        const res = await this.client.query(q, params);
+        lexical = res.rows.map(r => ({ content: r.content, metadata: r.metadata, score: 1.0 }));
+    } catch (e) {
+        console.warn("Keyword search failed (maybe missing indexes):", e.message);
+        lexical = [];
+    }
+
+    // 3. Reciprocal Rank Fusion
+    const combined = {};
+    const add = (list, weight = 1) => {
+      list.forEach((doc, idx) => {
+        const key = doc.content; // Use content as key if id not returned, ideally use id
+        // But doc structure returned by similaritySearch might not have id unless we select it
+        // existing implementations use content as key often in simple RRF
+        const score = 1 / (60 + idx + 1) * weight;
+        if (!combined[key]) combined[key] = { ...doc, score: 0 };
+        combined[key].score += score;
+      });
+    };
+    add(semantic, 1);
+    add(lexical, 1);
+    
+    return Object.values(combined).sort((a, b) => b.score - a.score).slice(0, limit);
+  }
+}
+
+module.exports = { PostgresVectorStore };

package/src/config.js

@@ -113,7 +113,7 @@ const RAGConfigSchema = z.object({
   generation: z.object({ structuredOutput: z.enum(['none','citations']).default('none'), outputFormat: z.enum(['text','json']).default('text') }).optional(),
   prompts: z.object({ query: z.string().optional(), reranking: z.string().optional() }).optional(),
   tracing: z.object({ enable: z.boolean().default(false) }).optional(),
-  callbacks: z.array(z.custom((val) => true)).optional(), 
+  callbacks: z.array(z.custom((_) => true)).optional(), 
   observability: z.object({
     enabled: z.boolean().default(false),
     sqlitePath: z.string().default('vectra-observability.db'),