vectra-js 0.9.0

package/src/memory.js

@@ -0,0 +1,96 @@
+class InMemoryHistory {
+  constructor(maxMessages = 20) {
+    this.sessions = new Map();
+    this.maxMessages = maxMessages;
+  }
+  addMessage(sessionId, role, content) {
+    if (!sessionId) return;
+    const arr = this.sessions.get(sessionId) || [];
+    arr.push({ role, content, ts: Date.now() });
+    const start = Math.max(0, arr.length - this.maxMessages);
+    this.sessions.set(sessionId, arr.slice(start));
+  }
+  getRecent(sessionId, n = 10) {
+    const arr = this.sessions.get(sessionId) || [];
+    const start = Math.max(0, arr.length - n);
+    return arr.slice(start);
+  }
+}
+class RedisHistory {
+  constructor(client, keyPrefix = 'vectra:chat:', maxMessages = 20) {
+    this.client = client;
+    this.keyPrefix = keyPrefix;
+    this.maxMessages = maxMessages;
+  }
+  async addMessage(sessionId, role, content) {
+    if (!sessionId || !this.client) return;
+    const key = `${this.keyPrefix}${sessionId}`;
+    const payload = JSON.stringify({ role, content, ts: Date.now() });
+    try {
+      if (typeof this.client.rpush === 'function') {
+        await this.client.rpush(key, payload);
+      } else if (typeof this.client.lPush === 'function') {
+        await this.client.lPush(key, payload);
+      }
+      if (typeof this.client.ltrim === 'function') {
+        await this.client.ltrim(key, -this.maxMessages, -1);
+      }
+    } catch (_) {}
+  }
+  async getRecent(sessionId, n = 10) {
+    if (!sessionId || !this.client) return [];
+    const key = `${this.keyPrefix}${sessionId}`;
+    try {
+      let arr = [];
+      if (typeof this.client.lrange === 'function') {
+        arr = await this.client.lrange(key, -n, -1);
+      } else if (typeof this.client.lRange === 'function') {
+        arr = await this.client.lRange(key, -n, -1);
+      }
+      return arr.map(x => {
+        try { return JSON.parse(x); } catch { return { role: 'assistant', content: String(x) }; }
+      });
+    } catch (_) {
+      return [];
+    }
+  }
+}
+class PostgresHistory {
+  constructor(client, tableName = 'ChatMessage', columnMap = { sessionId: 'sessionId', role: 'role', content: 'content', createdAt: 'createdAt' }, maxMessages = 20) {
+    this.client = client;
+    this.tableName = tableName;
+    this.columnMap = columnMap;
+    this.maxMessages = maxMessages;
+  }
+  async addMessage(sessionId, role, content) {
+    if (!sessionId || !this.client) return;
+    const t = this.tableName;
+    const c = this.columnMap;
+    const q = `INSERT INTO "${t}" ("${c.sessionId}","${c.role}","${c.content}","${c.createdAt}") VALUES ($1,$2,$3,NOW())`;
+    try {
+      if (typeof this.client.$executeRawUnsafe === 'function') {
+        await this.client.$executeRawUnsafe(q, sessionId, role, content);
+      } else if (typeof this.client.execute_raw === 'function') {
+        await this.client.execute_raw(q, sessionId, role, content);
+      }
+    } catch (_) {}
+  }
+  async getRecent(sessionId, n = 10) {
+    if (!sessionId || !this.client) return [];
+    const t = this.tableName;
+    const c = this.columnMap;
+    const q = `SELECT "${c.role}" as role, "${c.content}" as content FROM "${t}" WHERE "${c.sessionId}" = $1 ORDER BY "${c.createdAt}" DESC LIMIT ${Math.max(1, n)}`;
+    try {
+      let rows = [];
+      if (typeof this.client.$queryRawUnsafe === 'function') {
+        rows = await this.client.$queryRawUnsafe(q, sessionId);
+      } else if (typeof this.client.query_raw === 'function') {
+        rows = await this.client.query_raw(q, sessionId);
+      }
+      return Array.isArray(rows) ? rows.reverse().map(r => ({ role: r.role, content: r.content })) : [];
+    } catch (_) {
+      return [];
+    }
+  }
+}
+module.exports = { InMemoryHistory, RedisHistory, PostgresHistory };

package/src/processor.js

@@ -0,0 +1,155 @@
+const fs = require('fs');
+const path = require('path');
+const pdf = require('pdf-parse');
+const mammoth = require('mammoth');
+const xlsx = require('xlsx');
+const { ChunkingStrategy } = require('./config');
+
+class DocumentProcessor {
+  constructor(config, agenticLlm) {
+    this.config = config;
+    this.agenticLlm = agenticLlm;
+    this._lastPages = null;
+  }
+
+  async loadDocument(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    const buffer = await fs.promises.readFile(filePath);
+    if (ext === '.pdf') {
+      const pages = [];
+      const res = await pdf(buffer, {
+        pagerender: pageData => pageData.getTextContent().then(tc => {
+          const s = tc.items.map(it => it.str).join(' ');
+          pages.push(s);
+          return s;
+        })
+      });
+      this._lastPages = pages;
+      return res.text;
+    }
+    if (ext === '.docx') return (await mammoth.extractRawText({ buffer })).value;
+    if (['.txt','.md'].includes(ext)) return buffer.toString('utf-8');
+    if (['.xlsx','.xls'].includes(ext)) {
+        const wb = xlsx.read(buffer, { type: 'buffer' });
+        return xlsx.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]]);
+    }
+    throw new Error(`Unsupported file: ${ext}`);
+  }
+
+  async process(text) {
+    return this.config.strategy === ChunkingStrategy.AGENTIC 
+      ? this.agenticSplit(text) 
+      : this.recursiveSplit(text);
+  }
+
+  recursiveSplit(text) {
+    const chunks = [];
+    const sizeChars = Math.max(500, this.config.chunkSize || 1000);
+    const baseOverlap = Math.max(0, this.config.chunkOverlap || 200);
+    const sentences = text.split(/(?<=[.!?])\s+/);
+    let current = '';
+    for (const s of sentences) {
+      const candidate = current.length ? current + ' ' + s : s;
+      if (candidate.length >= sizeChars) {
+        const entropy = this._entropy(candidate);
+        const overlap = Math.min(baseOverlap + Math.floor(entropy * 50), Math.floor(sizeChars / 3));
+        chunks.push(candidate);
+        // create overlap window from end of candidate
+        current = candidate.slice(Math.max(0, candidate.length - overlap));
+      } else {
+        current = candidate;
+      }
+    }
+    if (current) chunks.push(current);
+    return chunks;
+  }
+
+  _entropy(str) {
+    const freq = {};
+    for (const ch of str) freq[ch] = (freq[ch] || 0) + 1;
+    const len = str.length;
+    let H = 0;
+    Object.values(freq).forEach(c => { const p = c / len; H += -p * Math.log2(p); });
+    return H;
+  }
+
+  async agenticSplit(text) {
+    if (!this.agenticLlm) throw new Error("Agentic LLM not configured.");
+    const windows = this.recursiveSplit(text);
+    const finalChunks = [];
+    for (const window of windows) {
+      const prompt = `Split this text into semantically complete propositions. Return a VALID JSON list of strings. Do not include Markdown formatting.\nText: "${window}"`;
+      try {
+        const response = await this.agenticLlm.generate(prompt);
+        // Attempt to clean markdown
+        const cleanJson = response.replace(/```json/g, '').replace(/```/g, '').trim();
+        const parsed = JSON.parse(cleanJson);
+        if (Array.isArray(parsed)) {
+          const dedup = new Set();
+          for (const item of parsed) {
+            if (typeof item === 'string') {
+              const norm = item.trim().replace(/\s+/g, ' ');
+              if (norm.length > 1 && !dedup.has(norm)) { dedup.add(norm); finalChunks.push(norm); }
+            }
+          }
+        } else {
+          finalChunks.push(window);
+        }
+      } catch (e) { 
+        // Fallback to window if parsing fails
+        finalChunks.push(window); 
+      }
+    }
+    return finalChunks;
+  }
+
+  computeChunkMetadata(filePath, rawText, chunks) {
+    const ext = path.extname(filePath).toLowerCase();
+    const title = path.basename(filePath);
+    const positions = [];
+    let cursor = 0;
+    for (const c of chunks) {
+      const idx = rawText.indexOf(c, cursor);
+      const start = idx >= 0 ? idx : 0;
+      const end = start + c.length;
+      positions.push({ start, end });
+      cursor = end;
+    }
+    let pagesMeta = null;
+    if (ext === '.pdf' && Array.isArray(this._lastPages)) {
+      const lens = this._lastPages.map(p => p.length);
+      const cum = [];
+      let acc = 0;
+      for (const l of lens) { acc += l; cum.push(acc); }
+      pagesMeta = positions.map(pos => {
+        const pf = cum.findIndex(x => x >= pos.start) + 1;
+        const pt = cum.findIndex(x => x >= pos.end) + 1;
+        return { pageFrom: pf || 1, pageTo: pt || pf || 1 };
+      });
+    }
+    let sections = null;
+    if (ext === '.md' || ext === '.txt') {
+      const lines = rawText.split(/\n/);
+      let offset = 0;
+      const heads = [];
+      for (const ln of lines) {
+        if (/^#{1,6}\s+/.test(ln)) heads.push({ pos: offset, text: ln.replace(/^#{1,6}\s+/, '') });
+        offset += ln.length + 1;
+      }
+      sections = positions.map(pos => {
+        const candidates = heads.filter(h => h.pos <= pos.start);
+        const h = candidates.length ? candidates[candidates.length - 1] : null;
+        return h ? h.text : null;
+      });
+    }
+    return positions.map((pos, i) => ({
+      fileType: ext,
+      docTitle: title,
+      chunkIndex: i,
+      pageFrom: pagesMeta ? pagesMeta[i].pageFrom : undefined,
+      pageTo: pagesMeta ? pagesMeta[i].pageTo : undefined,
+      section: sections ? sections[i] : undefined
+    }));
+  }
+}
+module.exports = { DocumentProcessor };

package/src/reranker.js

@@ -0,0 +1,26 @@
+class LLMReranker {
+    constructor(llm, config) {
+        this.llm = llm;
+        this.config = config;
+    }
+
+    async rerank(query, documents) {
+        if (!documents || documents.length === 0) return [];
+        const scored = await Promise.all(documents.map(async (doc) => {
+            const score = await this.scoreDocument(query, doc.content);
+            return { ...doc, score };
+        }));
+        scored.sort((a, b) => b.score - a.score);
+        return scored.slice(0, this.config.topN);
+    }
+
+    async scoreDocument(query, content) {
+        const prompt = `Analyze relevance (0-10) of document to query. Return ONLY integer.\nQuery: "${query}"\nDoc: "${content.substring(0, 1000)}..."`;
+        try {
+            const res = await this.llm.generate(prompt);
+            const match = res.match(/\d+/);
+            return match ? parseInt(match[0], 10) : 0;
+        } catch { return 0; }
+    }
+}
+module.exports = { LLMReranker };