vectra-js 0.9.6 → 0.9.8

package/src/core.js

@@ -1,6 +1,6 @@
 const fs = require('fs');
 const path = require('path');
-const { RAGConfigSchema, ProviderType, ChunkingStrategy, RetrievalStrategy } = require('./config');
+const { RAGConfigSchema, ProviderType, RetrievalStrategy } = require('./config');
 const crypto = require('crypto');
 const { DocumentProcessor } = require('./processor');
 const { OpenAIBackend } = require('./backends/openai');
@@ -10,6 +10,7 @@ const { OpenRouterBackend } = require('./backends/openrouter');
 const { HuggingFaceBackend } = require('./backends/huggingface');
 const { PrismaVectorStore } = require('./backends/prisma_store');
 const { ChromaVectorStore } = require('./backends/chroma_store');
+const { PostgresVectorStore } = require('./backends/postgres_store');
 const { QdrantVectorStore } = require('./backends/qdrant_store');
 const { MilvusVectorStore } = require('./backends/milvus_store');
 const { LLMReranker } = require('./reranker');
@@ -19,6 +20,18 @@ const { v5: uuidv5 } = require('uuid');
 const { v4: uuidv4 } = require('uuid');
 const SQLiteLogger = require('./observability');
 
+const DEFAULT_TOKEN_BUDGET = 2048;
+const DEFAULT_PREFER_SUMMARY_BELOW = 1024;
+const DEFAULT_SUMMARY_LENGTH = 800;
+const DEFAULT_CHUNK_LENGTH = 1200;
+const DEFAULT_FALLBACK_SUMMARY_LENGTH = 300;
+const DEFAULT_KEYWORD_COUNT = 10;
+const DEFAULT_MEMORY_MESSAGES = 20;
+const DEFAULT_CONCURRENCY_LIMIT = 5;
+const DEFAULT_RETRY_ATTEMPTS = 3;
+const DEFAULT_INITIAL_RETRY_DELAY = 500;
+const DEFAULT_MAX_RETRY_DELAY = 4000;
+
 class VectraClient {
   constructor(config) {
     const parsed = RAGConfigSchema.parse(config);
@@ -49,7 +62,7 @@ class VectraClient {
     this.vectorStore = this.createVectorStore(this.config.database);
     this._embeddingCache = new Map();
     this._metadataEnrichmentEnabled = !!(this.config.metadata && this.config.metadata.enrichment);
-    const mm = this.config.memory?.maxMessages || 20;
+    const mm = this.config.memory?.maxMessages || DEFAULT_MEMORY_MESSAGES;
     if (this.config.memory && this.config.memory.enabled) {
       if (this.config.memory.type === 'in-memory') {
         this.history = new InMemoryHistory(mm);
@@ -98,6 +111,7 @@ class VectraClient {
     if (!dbConfig || !dbConfig.type) throw new Error('Database config missing type');
     const t = dbConfig.type.toLowerCase();
     if (t === 'prisma') return new PrismaVectorStore(dbConfig);
+    if (t === 'postgres') return new PostgresVectorStore(dbConfig);
     if (t === 'chroma') return new ChromaVectorStore(dbConfig);
     if (t === 'qdrant') return new QdrantVectorStore(dbConfig);
     if (t === 'milvus') return new MilvusVectorStore(dbConfig);
@@ -127,14 +141,149 @@ class VectraClient {
         const words = c.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter(w => w.length > 3);
         const freq = {};
         for (const w of words) freq[w] = (freq[w] || 0) + 1;
-        const top = Object.entries(freq).sort((a,b)=>b[1]-a[1]).slice(0,10).map(([w])=>w);
-        const summary = c.slice(0, 300);
+        const top = Object.entries(freq).sort((a,b)=>b[1]-a[1]).slice(0,DEFAULT_KEYWORD_COUNT).map(([w])=>w);
+        const summary = c.slice(0, DEFAULT_FALLBACK_SUMMARY_LENGTH);
         enriched.push({ summary, keywords: top, hypothetical_questions: [] });
       }
     }
     return enriched;
   }
 
+  async _batchEmbedChunks(toEmbed, mapIndex, hashes) {
+    const newEmbeds = [];
+    if (toEmbed.length > 0) {
+      const enabled = !!(this.config.ingestion && this.config.ingestion.rateLimitEnabled);
+      const defaultLimit = (this.config.ingestion && typeof this.config.ingestion.concurrencyLimit === 'number') ? this.config.ingestion.concurrencyLimit : DEFAULT_CONCURRENCY_LIMIT;
+      const limit = enabled ? defaultLimit : toEmbed.length;
+      const batches = [];
+      for (let i = 0; i < toEmbed.length; i += limit) batches.push(toEmbed.slice(i, i + limit));
+      for (const batch of batches) {
+        let attempt = 0; let delay = DEFAULT_INITIAL_RETRY_DELAY;
+        while (true) {
+          try {
+            const out = await this.embedder.embedDocuments(batch);
+            newEmbeds.push(...out);
+            break;
+          } catch (err) {
+            attempt++;
+            if (attempt >= DEFAULT_RETRY_ATTEMPTS) throw err;
+            await new Promise(r => setTimeout(r, delay));
+            delay = Math.min(DEFAULT_MAX_RETRY_DELAY, delay * 2);
+          }
+        }
+      }
+      newEmbeds.forEach((vec, j) => {
+        const h = hashes[mapIndex[j]];
+        this._embeddingCache.set(h, vec);
+      });
+    }
+  }
+
+  async _processDirectory(filePath) {
+    const files = await fs.promises.readdir(filePath);
+    const summary = { processed: 0, succeeded: 0, failed: 0, errors: [] };
+    for (const file of files) {
+      const full = path.join(filePath, file);
+      if (this._isTemporaryFile(full)) continue;
+      summary.processed++;
+      try {
+        await this.ingestDocuments(full);
+        summary.succeeded++;
+      } catch (err) {
+        summary.failed++;
+        summary.errors.push({ file: full, message: err?.message || String(err) });
+        this.trigger('onError', err);
+      }
+    }
+    this.trigger('onIngestSummary', summary);
+  }
+
+  async _validateFile(filePath, stats) {
+    const absPath = path.resolve(filePath);
+    const size = stats.size || 0;
+    const mtime = Math.floor(stats.mtimeMs || Date.now());
+    const md5 = crypto.createHash('md5');
+    const sha = crypto.createHash('sha256');
+    await new Promise((resolve, reject) => {
+      const s = fs.createReadStream(filePath);
+      s.on('data', (chunk) => { md5.update(chunk); sha.update(chunk); });
+      s.on('error', reject);
+      s.on('end', resolve);
+    });
+    const fileMD5 = md5.digest('hex');
+    const fileSHA256 = sha.digest('hex');
+    return { absolutePath: absPath, fileMD5, fileSHA256, fileSize: size, lastModified: mtime, timestamp: Date.now() };
+  }
+
+  async _prepareDocuments(filePath, rawText, chunks, embeddings, hashes, validation) {
+    const metas = this.processor.computeChunkMetadata(filePath, rawText, chunks);
+    const idNamespace = uuidv5('vectra-js', uuidv5.DNS);
+    let documents = chunks.map((content, i) => ({
+      id: uuidv5(`${validation.fileSHA256}:${i}`, idNamespace),
+      content,
+      embedding: embeddings[i],
+      metadata: { 
+        docId: uuidv5(`${validation.fileSHA256}:${i}`, idNamespace),
+        source: filePath,
+        absolutePath: validation.absolutePath,
+        fileMD5: validation.fileMD5,
+        fileSHA256: validation.fileSHA256,
+        fileSize: validation.fileSize,
+        lastModified: validation.lastModified,
+        chunkIndex: i,
+        sha256: hashes[i],
+        fileType: metas[i]?.fileType,
+        docTitle: metas[i]?.docTitle,
+        pageFrom: metas[i]?.pageFrom,
+        pageTo: metas[i]?.pageTo,
+        section: metas[i]?.section
+      }
+    }));
+
+    if (this._metadataEnrichmentEnabled) {
+      const extra = await this._enrichChunkMetadata(chunks);
+      documents = documents.map((d, i) => ({
+        ...d,
+        metadata: {
+          ...d.metadata,
+          summary: extra[i]?.summary,
+          keywords: extra[i]?.keywords,
+          hypothetical_questions: extra[i]?.hypothetical_questions,
+        }
+      }));
+    }
+    return documents;
+  }
+
+  async _storeDocuments(documents, mode, absPath) {
+    if (this.vectorStore && typeof this.vectorStore.ensureIndexes === 'function') {
+      try { await this.vectorStore.ensureIndexes(); } catch (_) {}
+    }
+    
+    if (mode === 'replace' && this.vectorStore && typeof this.vectorStore.deleteDocuments === 'function') {
+      try {
+        await this.vectorStore.deleteDocuments({ filter: { absolutePath: absPath } });
+      } catch (_) {}
+    }
+    
+    let attempt = 0; let delay = DEFAULT_INITIAL_RETRY_DELAY;
+    while (true) {
+      try {
+        if (mode === 'replace' && this.vectorStore && typeof this.vectorStore.upsertDocuments === 'function') {
+          await this.vectorStore.upsertDocuments(documents);
+        } else {
+          await this.vectorStore.addDocuments(documents);
+        }
+        break;
+      } catch (err) {
+        attempt++;
+        if (attempt >= DEFAULT_RETRY_ATTEMPTS) throw err;
+        await new Promise(r => setTimeout(r, delay));
+        delay = Math.min(DEFAULT_MAX_RETRY_DELAY, delay * 2);
+      }
+    }
+  }
+
   async ingestDocuments(filePath) {
     const traceId = uuidv4();
     const rootSpanId = uuidv4();
@@ -146,58 +295,33 @@ class VectraClient {
       const stats = await fs.promises.stat(filePath);
 
       if (stats.isDirectory()) {
-        const files = await fs.promises.readdir(filePath);
-        const summary = { processed: 0, succeeded: 0, failed: 0, errors: [] };
-        for (const file of files) {
-          const full = path.join(filePath, file);
-          if (this._isTemporaryFile(full)) continue;
-          summary.processed++;
-          try {
-            await this.ingestDocuments(full);
-            summary.succeeded++;
-          } catch (err) {
-            summary.failed++;
-            summary.errors.push({ file: full, message: err?.message || String(err) });
-            this.trigger('onError', err);
-          }
-        }
-        this.trigger('onIngestSummary', summary);
+        await this._processDirectory(filePath);
         return;
       }
 
       const t0 = Date.now();
       this.trigger('onIngestStart', filePath);
-      const absPath = path.resolve(filePath);
-      const size = stats.size || 0;
-      const mtime = Math.floor(stats.mtimeMs || Date.now());
-      const md5 = crypto.createHash('md5');
-      const sha = crypto.createHash('sha256');
-      await new Promise((resolve, reject) => {
-        const s = fs.createReadStream(filePath);
-        s.on('data', (chunk) => { md5.update(chunk); sha.update(chunk); });
-        s.on('error', reject);
-        s.on('end', resolve);
-      });
-      const fileMD5 = md5.digest('hex');
-      const fileSHA256 = sha.digest('hex');
-      const validation = { absolutePath: absPath, fileMD5, fileSHA256, fileSize: size, lastModified: mtime, timestamp: Date.now() };
+      
+      const validation = await this._validateFile(filePath, stats);
       this.trigger('onPreIngestionValidation', validation);
+      
       const mode = (this.config.ingestion && this.config.ingestion.mode) ? this.config.ingestion.mode : 'skip';
       let exists = false;
       if (this.vectorStore && typeof this.vectorStore.fileExists === 'function') {
-        try { exists = await this.vectorStore.fileExists(fileSHA256, size, mtime); } catch { exists = false; }
+        try { exists = await this.vectorStore.fileExists(validation.fileSHA256, validation.fileSize, validation.lastModified); } catch { exists = false; }
       }
       if (mode === 'skip' && exists) {
         this.trigger('onIngestSkipped', validation);
         return;
       }
+
       const rawText = await this.processor.loadDocument(filePath);
 
       this.trigger('onChunkingStart', this.config.chunking.strategy);
       const chunks = await this.processor.process(rawText);
 
       this.trigger('onEmbeddingStart', chunks.length);
-      // Compute hashes and use cache for known chunks
+      
       const hashes = chunks.map(c => crypto.createHash('sha256').update(c).digest('hex'));
       const toEmbed = [];
       const mapIndex = [];
@@ -206,104 +330,24 @@ class VectraClient {
         toEmbed.push(chunks[i]);
         mapIndex.push(i);
       });
-      const newEmbeds = [];
-      if (toEmbed.length > 0) {
-        const enabled = !!(this.config.ingestion && this.config.ingestion.rateLimitEnabled);
-        const defaultLimit = (this.config.ingestion && typeof this.config.ingestion.concurrencyLimit === 'number') ? this.config.ingestion.concurrencyLimit : 5;
-        const limit = enabled ? defaultLimit : toEmbed.length;
-        const batches = [];
-        for (let i = 0; i < toEmbed.length; i += limit) batches.push(toEmbed.slice(i, i + limit));
-        for (const batch of batches) {
-          let attempt = 0; let delay = 500;
-          while (true) {
-            try {
-              const out = await this.embedder.embedDocuments(batch);
-              newEmbeds.push(...out);
-              break;
-            } catch (err) {
-              attempt++;
-              if (attempt >= 3) throw err;
-              await new Promise(r => setTimeout(r, delay));
-              delay = Math.min(4000, delay * 2);
-            }
-          }
-        }
-        newEmbeds.forEach((vec, j) => {
-          const h = hashes[mapIndex[j]];
-          this._embeddingCache.set(h, vec);
-        });
-      }
+      
+      await this._batchEmbedChunks(toEmbed, mapIndex, hashes);
+      
       const embeddings = hashes.map((h) => this._embeddingCache.get(h));
 
-      const metas = this.processor.computeChunkMetadata(filePath, rawText, chunks);
-      const idNamespace = uuidv5('vectra-js', uuidv5.DNS);
-      let documents = chunks.map((content, i) => ({
-        id: uuidv5(`${fileSHA256}:${i}`, idNamespace),
-        content,
-        embedding: embeddings[i],
-        metadata: { 
-          docId: uuidv5(`${fileSHA256}:${i}`, idNamespace),
-          source: filePath,
-          absolutePath: absPath,
-          fileMD5,
-          fileSHA256,
-          fileSize: size,
-          lastModified: mtime,
-          chunkIndex: i,
-          sha256: hashes[i],
-          fileType: metas[i]?.fileType,
-          docTitle: metas[i]?.docTitle,
-          pageFrom: metas[i]?.pageFrom,
-          pageTo: metas[i]?.pageTo,
-          section: metas[i]?.section
-        }
-      }));
-
-      if (this._metadataEnrichmentEnabled) {
-        const extra = await this._enrichChunkMetadata(chunks);
-        documents = documents.map((d, i) => ({
-          ...d,
-          metadata: {
-            ...d.metadata,
-            summary: extra[i]?.summary,
-            keywords: extra[i]?.keywords,
-            hypothetical_questions: extra[i]?.hypothetical_questions,
-          }
-        }));
-      }
-
-      if (this.vectorStore && typeof this.vectorStore.ensureIndexes === 'function') {
-        try { await this.vectorStore.ensureIndexes(); } catch (_) {}
-      }
+      const documents = await this._prepareDocuments(filePath, rawText, chunks, embeddings, hashes, validation);
+      
       let existsServer = false;
       if (this.vectorStore && typeof this.vectorStore.fileExists === 'function') {
-        try { existsServer = await this.vectorStore.fileExists(fileSHA256, size, mtime); } catch { existsServer = false; }
+        try { existsServer = await this.vectorStore.fileExists(validation.fileSHA256, validation.fileSize, validation.lastModified); } catch { existsServer = false; }
       }
       if (mode === 'skip' && existsServer) {
         this.trigger('onIngestSkipped', validation);
         return;
       }
-      if (mode === 'replace' && this.vectorStore && typeof this.vectorStore.deleteDocuments === 'function') {
-        try {
-          await this.vectorStore.deleteDocuments({ filter: { absolutePath: absPath } });
-        } catch (_) {}
-      }
-      let attempt = 0; let delay = 500;
-      while (true) {
-        try {
-          if (mode === 'replace' && this.vectorStore && typeof this.vectorStore.upsertDocuments === 'function') {
-            await this.vectorStore.upsertDocuments(documents);
-          } else {
-            await this.vectorStore.addDocuments(documents);
-          }
-          break;
-        } catch (err) {
-          attempt++;
-          if (attempt >= 3) throw err;
-          await new Promise(r => setTimeout(r, delay));
-          delay = Math.min(4000, delay * 2);
-        }
-      }
+      
+      await this._storeDocuments(documents, mode, validation.absolutePath);
+
       const durationMs = Date.now() - t0;
       this.trigger('onIngestEnd', filePath, chunks.length, durationMs);
       
@@ -315,7 +359,7 @@ class VectraClient {
         endTime: Date.now(),
         input: { filePath },
         output: { chunks: chunks.length, durationMs },
-        attributes: { fileSize: size },
+        attributes: { fileSize: validation.fileSize },
         provider,
         modelName
       });
@@ -393,16 +437,16 @@ class VectraClient {
   }
 
   buildContextParts(docs, query) {
-    const budget = (this.config.queryPlanning && this.config.queryPlanning.tokenBudget) ? this.config.queryPlanning.tokenBudget : 2048;
-    const preferSumm = (this.config.queryPlanning && this.config.queryPlanning.preferSummariesBelow) ? this.config.queryPlanning.preferSummariesBelow : 1024;
+    const budget = (this.config.queryPlanning && this.config.queryPlanning.tokenBudget) ? this.config.queryPlanning.tokenBudget : DEFAULT_TOKEN_BUDGET;
+    const preferSumm = (this.config.queryPlanning && this.config.queryPlanning.preferSummariesBelow) ? this.config.queryPlanning.preferSummariesBelow : DEFAULT_PREFER_SUMMARY_BELOW;
     const parts = [];
     let used = 0;
     for (const d of docs) {
       const t = d.metadata?.docTitle || '';
       const sec = d.metadata?.section || '';
       const pages = (d.metadata?.pageFrom && d.metadata?.pageTo) ? `pages ${d.metadata.pageFrom}-${d.metadata.pageTo}` : '';
-      const sum = d.metadata?.summary ? d.metadata.summary : d.content.slice(0, 800);
-      const chosen = (this.tokenEstimate(sum) <= preferSumm) ? sum : d.content.slice(0, 1200);
+      const sum = d.metadata?.summary ? d.metadata.summary : d.content.slice(0, DEFAULT_SUMMARY_LENGTH);
+      const chosen = (this.tokenEstimate(sum) <= preferSumm) ? sum : d.content.slice(0, DEFAULT_CHUNK_LENGTH);
       const part = `${t} ${sec} ${pages}\n${chosen}`;
       const est = this.tokenEstimate(part);
       if (used + est > budget) break;

package/src/observability.js

@@ -17,26 +17,20 @@ class SQLiteLogger {
         const dbPath = path.isAbsolute(rawPath) ? rawPath : path.resolve(process.cwd(), rawPath);
         // Ensure directory exists
         const dbDir = path.dirname(dbPath);
-        console.log(`[SQLiteLogger] dbPath: ${dbPath}, dbDir: ${dbDir}`);
         
         const fs = require('fs');
         if (!fs.existsSync(dbDir)) {
-          console.log(`[SQLiteLogger] Creating directory: ${dbDir}`);
           fs.mkdirSync(dbDir, { recursive: true });
-        } else {
-          console.log(`[SQLiteLogger] Directory exists: ${dbDir}`);
         }
 
         const sqlite3 = require('sqlite3').verbose();
         this.db = new sqlite3.Database(dbPath, (err) => {
             if (err) {
-                console.error('Failed to connect to SQLite database:', err);
                 throw err;
             }
         });
         this.initializeSchema();
     } catch (error) {
-        console.error('Failed to initialize SQLite logger:', error);
         throw error;
     }
   }

package/src/processor.js

@@ -16,8 +16,38 @@ class DocumentProcessor {
     const ext = path.extname(filePath).toLowerCase();
     const buffer = await fs.promises.readFile(filePath);
     if (ext === '.pdf') {
+      let PDFParse = pdf.PDFParse;
+      if (!PDFParse && pdf.default && pdf.default.PDFParse) {
+        PDFParse = pdf.default.PDFParse;
+      }
+
+      if (PDFParse) {
+        // Handle pdf-parse v2
+        const parser = new PDFParse({ data: buffer });
+        const info = await parser.getInfo();
+        const total = info.total;
+        const pages = [];
+        let fullText = '';
+
+        for (let i = 1; i <= total; i++) {
+          const pageRes = await parser.getText({ partial: [i] });
+          const pageText = pageRes.text || '';
+          pages.push(pageText);
+          fullText += pageText + '\n'; 
+        }
+        await parser.destroy();
+        this._lastPages = pages;
+        return fullText;
+      }
+
+      // Fallback for v1 (or if PDFParse class not found)
+      let pdfFunc = pdf;
+      if (typeof pdfFunc !== 'function' && pdfFunc.default) {
+        pdfFunc = pdfFunc.default;
+      }
+      
       const pages = [];
-      const res = await pdf(buffer, {
+      const res = await pdfFunc(buffer, {
         pagerender: pageData => pageData.getTextContent().then(tc => {
           const s = tc.items.map(it => it.str).join(' ');
           pages.push(s);
@@ -95,7 +125,7 @@ class DocumentProcessor {
         } else {
           finalChunks.push(window);
         }
-      } catch (e) { 
+      } catch (_) { 
         // Fallback to window if parsing fails
         finalChunks.push(window); 
       }

package/src/webconfig_server.js

@@ -1,7 +1,7 @@
 const http = require('http');
 const fs = require('fs');
 const path = require('path');
-const { RAGConfigSchema, ProviderType, ChunkingStrategy, RetrievalStrategy } = require('./config');
+const { ProviderType, ChunkingStrategy, RetrievalStrategy } = require('./config');
 const sqlite3 = require('sqlite3').verbose();