vectra-js 0.9.12 → 1.0.1

package/eslint.config.js

@@ -0,0 +1,22 @@
+const globals = require('globals');
+
+module.exports = [
+  {
+    ignores: ['node_modules/**', 'dist/**', 'coverage/**']
+  },
+  {
+    files: ['**/*.js', '**/*.cjs', '**/*.mjs'],
+    languageOptions: {
+      ecmaVersion: 2022,
+      sourceType: 'script',
+      globals: globals.node
+    },
+    rules: {
+      'no-var': 'error',
+      'prefer-const': 'error',
+      'eqeqeq': ['error', 'always'],
+      'curly': ['error', 'multi-line'],
+      'no-console': 'off'
+    }
+  }
+];

package/index.js

@@ -1,11 +1,13 @@
 const config = require('./src/config');
 const callbacks = require('./src/callbacks');
 const core = require('./src/core');
+const interfaces = require('./src/interfaces');
 const reranker = require('./src/reranker');
 
 module.exports = {
   ...config,
   ...callbacks,
   ...core,
+  ...interfaces,
   ...reranker
 };

package/occupy_port.js

@@ -0,0 +1,5 @@
+const http = require('http');
+const server = http.createServer((req, res) => {});
+server.listen(8766, () => {
+    console.log('Occupying port 8766');
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectra-js",
-  "version": "0.9.12",
+  "version": "1.0.1",
   "description": "A production-ready, provider-agnostic Node.js SDK for End-to-End RAG pipelines.",
   "main": "index.js",
   "scripts": {
@@ -30,33 +30,49 @@
   "bugs": {
     "url": "https://github.com/iamabhishek-n/vectra-js/issues"
   },
+  "files": [
+    "bin/vectra.js",
+    "src",
+    "index.js",
+    "LICENSE",
+    "README.md",
+    "package.json",
+    "package-lock.json",
+    "occupy_port.js",
+    "documentation.md",
+    "examples",
+    "docs",
+    "eslint.config.js"
+  ],
   "homepage": "https://github.com/iamabhishek-n/vectra-js#readme",
   "author": "Abhishek N",
   "license": "GPL-3.0",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.71.2",
     "@google/genai": "^1.34.0",
-    "dotenv": "^16.6.1",
     "mammoth": "^1.11.0",
     "openai": "^6.15.0",
     "pdf-parse": "^2.4.5",
     "pg": "^8.16.3",
     "sqlite3": "^5.1.7",
-    "uuid": "^9.0.1",
-    "xlsx": "^0.18.5",
-    "zod": "^3.25.76"
+    "xlsx": "^0.18.5"
   },
   "peerDependencies": {
     "@prisma/client": "^5.0.0"
   },
   "devDependencies": {
+    "@chroma-core/default-embed": "^0.1.9",
+    "chromadb": "^3.4.0",
+    "dotenv": "^17.3.1",
     "eslint": "^9.39.2",
     "globals": "^16.5.0",
-    "prisma": "^7.2.0"
+    "prisma": "^7.2.0",
+    "uuid": "^13.0.0",
+    "zod": "^4.3.6"
   },
   "pnpm": {
     "onlyBuiltDependencies": [
       "sqlite3"
     ]
   }
-}
+}

package/src/backends/chroma_store.js

@@ -10,6 +10,10 @@ class ChromaVectorStore extends VectorStore {
     }
 
     async _init() {
+        if (!this.client) {
+            const { ChromaClient } = require('chromadb');
+            this.client = new ChromaClient();
+        }
         if (!this.collection) {
             this.collection = await this.client.getOrCreateCollection({ name: this.collectionName });
         }
@@ -33,12 +37,19 @@ class ChromaVectorStore extends VectorStore {
         const metadatas = docs.map(d => this._cleanMetadata(d.metadata));
         const documents = docs.map(d => d.content);
 
-        await this.collection.add({
-            ids,
-            embeddings,
-            metadatas,
-            documents
-        });
+        console.log(`Adding ${docs.length} docs to Chroma collection: ${this.collectionName}`);
+        try {
+            await this.collection.add({
+                ids,
+                embeddings,
+                metadatas,
+                documents
+            });
+            console.log("Success adding docs to Chroma.");
+        } catch (e) {
+            console.error("Error in collection.add:", e);
+            throw e;
+        }
     }
 
     async upsertDocuments(docs) {
@@ -89,10 +100,10 @@ class ChromaVectorStore extends VectorStore {
         return out;
     }
 
-    async listDocuments({ filter = null, limit = 100, offset = 0 } = {}) {
+    async listDocuments({ filter = null, limit = 100, cursor = null } = {}) {
         await this._init();
         const lim = Math.max(1, Math.min(1000, Number(limit) || 100));
-        const off = Math.max(0, Number(offset) || 0);
+        const off = cursor ? Number(cursor) : 0;
         const res = await this.collection.get({
             where: filter || undefined,
             limit: lim,
@@ -102,7 +113,9 @@ class ChromaVectorStore extends VectorStore {
         const ids = Array.isArray(res?.ids) ? res.ids : [];
         const documents = Array.isArray(res?.documents) ? res.documents : [];
         const metadatas = Array.isArray(res?.metadatas) ? res.metadatas : [];
-        return ids.map((id, i) => ({ id, content: documents[i], metadata: metadatas[i] }));
+        const docs = ids.map((id, i) => ({ id, content: documents[i], metadata: metadatas[i] }));
+        const nextCursor = docs.length === lim ? String(off + docs.length) : null;
+        return { documents: docs, nextCursor };
     }
 
     async deleteDocuments({ ids = null, filter = null } = {}) {

package/src/backends/huggingface.js

@@ -1,3 +1,5 @@
+const https = require('https');
+
 class HuggingFaceBackend {
   constructor(config) {
     this.config = config;
@@ -7,16 +9,41 @@ class HuggingFaceBackend {
   }
 
   async _post(model, payload) {
-    const res = await fetch(`${this.baseUrl}/${encodeURIComponent(model)}`, {
-      method: 'POST',
-      headers: { 'Authorization': `Bearer ${this.apiKey}`, 'Content-Type': 'application/json' },
-      body: JSON.stringify(payload)
+    return new Promise((resolve, reject) => {
+      const data = JSON.stringify(payload);
+      const url = new URL(`${this.baseUrl}/${encodeURIComponent(model)}`);
+      
+      const options = {
+        hostname: url.hostname,
+        path: url.pathname + url.search,
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${this.apiKey}`,
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(data)
+        }
+      };
+
+      const req = https.request(options, (res) => {
+        let body = '';
+        res.on('data', (chunk) => body += chunk);
+        res.on('end', () => {
+          if (res.statusCode < 200 || res.statusCode >= 300) {
+            reject(new Error(`HF error ${res.statusCode}: ${body}`));
+          } else {
+            try {
+              resolve(JSON.parse(body));
+            } catch(e) {
+              resolve(body);
+            }
+          }
+        });
+      });
+
+      req.on('error', reject);
+      req.write(data);
+      req.end();
     });
-    if (!res.ok) {
-      const t = await res.text();
-      throw new Error(`HF error ${res.status}: ${t}`);
-    }
-    return await res.json();
   }
 
   async embedDocuments(texts) {

package/src/backends/milvus_store.js

@@ -30,19 +30,21 @@ class MilvusVectorStore extends VectorStore {
   }
   async hybridSearch(text, vector, limit = 5, filter = null) { return this.similaritySearch(vector, limit, filter); }
 
-  async listDocuments({ filter = null, limit = 100, offset = 0 } = {}) {
+  async listDocuments({ filter = null, limit = 100, cursor = null } = {}) {
     if (typeof this.client.query !== 'function') throw new Error('listDocuments is not supported for this Milvus client');
-    const lim = Math.max(1, Math.min(1000, Number(limit) || 100));
-    const off = Math.max(0, Number(offset) || 0);
+    const lim = Math.max(1, Number(limit) || 100);
+    const off = cursor ? Number(cursor) : 0;
     const res = await this.client.query({
       collection_name: this.collection,
       expr: filter || '',
-      output_fields: ['content', 'metadata'],
+      output_fields: ['id', 'content', 'metadata'],
       limit: lim,
       offset: off,
     });
     const rows = Array.isArray(res) ? res : (res?.data || res?.results || []);
-    return rows.map((r) => ({ id: r.id, content: r.content || '', metadata: r.metadata ? JSON.parse(r.metadata) : {} }));
+    const docs = rows.map((r) => ({ id: r.id, content: r.content || '', metadata: r.metadata ? JSON.parse(r.metadata) : {} }));
+    const nextCursor = docs.length === lim ? String(off + docs.length) : null;
+    return { documents: docs, nextCursor };
   }
 
   async fileExists(sha256, size, lastModified) {

package/src/backends/postgres_store.js

@@ -37,6 +37,14 @@ class PostgresVectorStore extends VectorStore {
     this.client = this.config.clientInstance;
   }
 
+  async _withConn(fn) {
+    if (typeof this.client.connect === 'function') {
+        const client = await this.client.connect();
+        try { return await fn(client); } finally { client.release(); }
+    }
+    return fn(this.client);
+  }
+
   normalizeVector(v) {
     const m = Math.sqrt(v.reduce((s, x) => s + x * x, 0));
     return m === 0 ? v : v.map(x => x / m);
@@ -44,129 +52,57 @@ class PostgresVectorStore extends VectorStore {
 
   // Helper to ensure table and extension exist
   async ensureIndexes() {
-    // Enable pgvector extension
-    await this.client.query('CREATE EXTENSION IF NOT EXISTS vector');
-    
-    // Detect existing column type to avoid malformed array issues
-    try {
-      const typeCheck = await this.client.query(
-        `SELECT data_type, udt_name 
-         FROM information_schema.columns 
-         WHERE table_name = $1 AND column_name = $2`,
-        [this._tableBase, this._cVec.replace(/"/g, '')]
-      );
-      const row = typeCheck.rows[0];
-      if (row) {
-        const isPgVector = row.udt_name === 'vector';
-        const isArray = row.data_type && row.data_type.toLowerCase().includes('array');
-        if (isArray && !isPgVector) {
-          throw new Error(
-            'Postgres schema mismatch: vector column is double precision[] (array). ' +
-            'Use pgvector type: vector(<dimensions>). ' +
-            'Example: ALTER TABLE ' + this._table + ' ALTER COLUMN ' + this._cVec + ' TYPE vector(1536);'
-          );
-        }
-      }
-    } catch (e) {
-      // Only throw if we explicitly detected array type; otherwise continue
-      if (String(e.message || e).includes('schema mismatch')) {
-        throw e;
-      }
-    }
-    
-    // Create table if not exists (best-effort)
-    // Note: We need to know vector dimensions. We'll try to guess or use default 1536
-    // If embedding dimensions are provided in config, use them
-    // But store config usually doesn't have embedding config directly unless passed down
-    // For now we will assume the user creates the table or we default to 1536 (OpenAI)
-    // A better approach is to rely on user schema, but for convenience:
-    const dim = 1536; // Default to OpenAI dimension if unknown.
-    // However, if the table exists, we don't change it.
-    
-    const createTableQuery = `
-      CREATE TABLE IF NOT EXISTS ${this._table} (
-        "id" TEXT PRIMARY KEY,
-        ${this._cContent} TEXT,
-        ${this._cMeta} JSONB,
-        ${this._cVec} vector(${dim}),
-        "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT NOW()
-      )
-    `;
-    await this.client.query(createTableQuery);
-    
-    // Ensure required columns exist (non-destructive)
-    try {
-      const res = await this.client.query(
-        `SELECT column_name, data_type, udt_name 
-         FROM information_schema.columns 
-         WHERE table_name = $1`,
-        [this._tableBase]
-      );
-      const cols = new Map(res.rows.map(r => [r.column_name, r]));
-      const contentCol = this._cContent.replace(/"/g, '');
-      const metaCol = this._cMeta.replace(/"/g, '');
-      const vecCol = this._cVec.replace(/"/g, '');
-      const createdAtCol = this._cCreatedAt.replace(/"/g, '');
-      
-      if (!cols.has(contentCol)) {
-        await this.client.query(`ALTER TABLE ${this._table} ADD COLUMN ${this._cContent} TEXT`);
-      }
-      if (!cols.has(metaCol)) {
-        await this.client.query(`ALTER TABLE ${this._table} ADD COLUMN ${this._cMeta} JSONB`);
-      }
-      if (!cols.has(vecCol)) {
-        await this.client.query(`ALTER TABLE ${this._table} ADD COLUMN ${this._cVec} vector(${dim})`);
-      } else {
-        const vinfo = cols.get(vecCol);
-        const isPgVector = vinfo && vinfo.udt_name === 'vector';
-        const isArray = vinfo && vinfo.data_type && vinfo.data_type.toLowerCase().includes('array');
-        if (isArray && !isPgVector) {
-          throw new Error(
-            'Postgres schema mismatch: vector column is double precision[] (array). ' +
-            'Use pgvector type: vector(' + dim + '). ' +
-            'Example: ALTER TABLE ' + this._table + ' ALTER COLUMN ' + this._cVec + ' TYPE vector(' + dim + ');'
+    await this._withConn(async (client) => {
+        await client.query('CREATE EXTENSION IF NOT EXISTS vector');
+        
+        try {
+          const typeCheck = await client.query(
+            `SELECT data_type, udt_name FROM information_schema.columns WHERE table_name = $1 AND column_name = $2`,
+            [this._tableBase, this._cVec.replace(/"/g, '')]
           );
+          const row = typeCheck.rows[0];
+          if (row && row.data_type && row.data_type.toLowerCase().includes('array') && row.udt_name !== 'vector') {
+              throw new Error('Postgres schema mismatch: vector column is array. Use vector(<dimensions>).');
+          }
+        } catch (e) {
+          if (String(e.message || e).includes('schema mismatch')) throw e;
         }
-      }
-      if (!cols.has(createdAtCol)) {
-        await this.client.query(`ALTER TABLE ${this._table} ADD COLUMN ${this._cCreatedAt} TIMESTAMP WITH TIME ZONE DEFAULT NOW()`);
-      }
-    } catch (_) {
-      // best-effort; ignore
-    }
-    
-    // Create HNSW index for faster search
-    // checking if index exists is hard in raw sql cross-version, 
-    // simpler to CREATE INDEX IF NOT EXISTS which pg supports in recent versions
-    // or catch error
-    try {
-        await this.client.query(`CREATE INDEX IF NOT EXISTS "${this._table.replace(/"/g, '')}_vec_idx" ON ${this._table} USING hnsw (${this._cVec} vector_cosine_ops)`);
-    } catch (e) {
-        // Fallback to ivfflat when hnsw not supported
+        
+        const dim = 1536;
+        await client.query(`CREATE TABLE IF NOT EXISTS ${this._table} ("id" TEXT PRIMARY KEY, ${this._cContent} TEXT, ${this._cMeta} JSONB, ${this._cVec} vector(${dim}), "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT NOW())`);
+        
+        try {
+          const res = await client.query(`SELECT column_name FROM information_schema.columns WHERE table_name = $1`, [this._tableBase]);
+          const cols = new Set(res.rows.map(r => r.column_name));
+          if (!cols.has(this._cContent.replace(/"/g, ''))) await client.query(`ALTER TABLE ${this._table} ADD COLUMN ${this._cContent} TEXT`);
+          if (!cols.has(this._cMeta.replace(/"/g, ''))) await client.query(`ALTER TABLE ${this._table} ADD COLUMN ${this._cMeta} JSONB`);
+          if (!cols.has(this._cVec.replace(/"/g, ''))) await client.query(`ALTER TABLE ${this._table} ADD COLUMN ${this._cVec} vector(${dim})`);
+          if (!cols.has('createdAt')) await client.query(`ALTER TABLE ${this._table} ADD COLUMN "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT NOW()`);
+        } catch (_) {}
+        
         try {
-          await this.client.query(`CREATE INDEX IF NOT EXISTS "${this._table.replace(/"/g, '')}_vec_idx" ON ${this._table} USING ivfflat (${this._cVec} vector_cosine_ops)`);
-        } catch (e2) {
-          console.warn('Could not create vector index (might be fine if not supported):', e.message);
+            await client.query(`CREATE INDEX IF NOT EXISTS "${this._table.replace(/"/g, '')}_vec_idx" ON ${this._table} USING hnsw (${this._cVec} vector_cosine_ops)`);
+        } catch (e) {
+            try { await client.query(`CREATE INDEX IF NOT EXISTS "${this._table.replace(/"/g, '')}_vec_idx" ON ${this._table} USING ivfflat (${this._cVec} vector_cosine_ops)`); } catch (_) {}
         }
-    }
+    });
   }
 
   async addDocuments(docs) {
     const q = `INSERT INTO ${this._table} ("id", ${this._cContent}, ${this._cMeta}, ${this._cVec}, "createdAt") VALUES ($1, $2, $3, $4, NOW()) ON CONFLICT ("id") DO NOTHING`;
-    
-    for (const doc of docs) {
-      const id = doc.id || uuidv4();
-      const vec = `[${this.normalizeVector(doc.embedding).join(',')}]`; // pgvector format
-      try {
-        await this.client.query(q, [id, doc.content, doc.metadata, vec]);
-      } catch (e) {
-        const msg = e?.message || String(e);
-        if (msg.includes('vector') && msg.includes('dimension')) {
-           throw new Error('DimensionMismatchError: Embedding dimension does not match pgvector column.');
+    await this._withConn(async (client) => {
+      for (const doc of docs) {
+        const id = doc.id || uuidv4();
+        const vec = `[${this.normalizeVector(doc.embedding).join(',')}]`;
+        try {
+          await client.query(q, [id, doc.content, doc.metadata, vec]);
+        } catch (e) {
+          const msg = e?.message || String(e);
+          if (msg.includes('vector') && msg.includes('dimension')) throw new Error('DimensionMismatchError');
+          throw e;
         }
-        throw e;
       }
-    }
+    });
   }
 
   async upsertDocuments(docs) {
@@ -180,11 +116,13 @@ class PostgresVectorStore extends VectorStore {
             ${this._cVec} = EXCLUDED.${this._cVec}
       `;
       
-      for (const doc of docs) {
-        const id = doc.id || uuidv4();
-        const vec = `[${this.normalizeVector(doc.embedding).join(',')}]`;
-        await this.client.query(q, [id, doc.content, doc.metadata, vec]);
-      }
+      await this._withConn(async (client) => {
+        for (const doc of docs) {
+          const id = doc.id || uuidv4();
+          const vec = `[${this.normalizeVector(doc.embedding).join(',')}]`;
+          await client.query(q, [id, doc.content, doc.metadata, vec]);
+        }
+      });
   }
 
   async similaritySearch(vector, limit = 5, filter = null) {
@@ -208,7 +146,7 @@ class PostgresVectorStore extends VectorStore {
     `;
     params.push(Math.max(1, Number(limit) || 5));
 
-    const res = await this.client.query(q, params);
+    const res = await this._withConn(c => c.query(q, params));
     return res.rows.map(r => ({ content: r.content, metadata: r.metadata, score: r.score }));
   }
 
@@ -237,7 +175,7 @@ class PostgresVectorStore extends VectorStore {
 
     let lexical = [];
     try {
-        const res = await this.client.query(q, params);
+        const res = await this._withConn(c => c.query(q, params));
         lexical = res.rows.map(r => ({ content: r.content, metadata: r.metadata, score: 1.0 }));
     } catch (e) {
         console.warn("Keyword search failed (maybe missing indexes):", e.message);
@@ -262,6 +200,29 @@ class PostgresVectorStore extends VectorStore {
     return Object.values(combined).sort((a, b) => b.score - a.score).slice(0, limit);
   }
   
+  async listDocuments({ filter = null, limit = 100, cursor = null } = {}) {
+    return this._withConn(async (client) => {
+      const params = [];
+      const whereParts = [];
+      if (filter) {
+        whereParts.push(`${this._cMeta} @> $${params.length + 1}`);
+        params.push(filter);
+      }
+      if (cursor) {
+        whereParts.push(`"id" > $${params.length + 1}`);
+        params.push(cursor);
+      }
+      const where = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
+      const lim = Math.max(1, Number(limit) || 100);
+      const q = `SELECT "id", ${this._cContent} as content, ${this._cMeta} as metadata FROM ${this._table} ${where} ORDER BY "id" ASC LIMIT $${params.length + 1}`;
+      params.push(lim);
+      const res = await client.query(q, params);
+      const docs = res.rows.map(r => ({ id: r.id, content: r.content, metadata: r.metadata }));
+      const nextCursor = docs.length === lim ? docs[docs.length - 1].id : null;
+      return { documents: docs, nextCursor };
+    });
+  }
+
   async fileExists(sha256, size, lastModified) {
     try {
       const q = `
@@ -271,7 +232,7 @@ class PostgresVectorStore extends VectorStore {
         LIMIT 1
       `;
       const metaFilter = JSON.stringify({ fileSHA256: sha256, fileSize: size, lastModified });
-      const res = await this.client.query(q, [metaFilter]);
+      const res = await this._withConn(c => c.query(q, [metaFilter]));
       return res.rowCount > 0;
     } catch (_) {
       return false;

package/src/backends/prisma_store.js

@@ -176,19 +176,25 @@ class PrismaVectorStore extends VectorStore {
     }
   }
 
-  async listDocuments({ filter = null, limit = 100, offset = 0 } = {}) {
+  async listDocuments({ filter = null, limit = 100, cursor = null } = {}) {
     const { clientInstance } = this.config;
     const params = [];
-    let where = '';
+    const whereParts = [];
     if (filter) {
-      where = `WHERE ${this._cMeta} @> $1::jsonb`;
+      whereParts.push(`${this._cMeta} @> $${params.length + 1}::jsonb`);
       params.push(JSON.stringify(filter));
     }
-    const lim = Math.max(1, Math.min(1000, Number(limit) || 100));
-    const off = Math.max(0, Number(offset) || 0);
-    const q = `SELECT "id" as id, ${this._cContent} as content, ${this._cMeta} as metadata, "createdAt" as createdAt FROM ${this._table} ${where} ORDER BY "createdAt" DESC LIMIT ${lim} OFFSET ${off}`;
+    if (cursor) {
+      whereParts.push(`"id" > $${params.length + 1}`);
+      params.push(cursor);
+    }
+    const where = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
+    const lim = Math.max(1, Number(limit) || 100);
+    const q = `SELECT "id" as id, ${this._cContent} as content, ${this._cMeta} as metadata FROM ${this._table} ${where} ORDER BY "id" ASC LIMIT ${lim}`;
     const res = await clientInstance.$queryRawUnsafe(q, ...params);
-    return res.map(r => ({ id: r.id, content: r.content, metadata: r.metadata, createdAt: r.createdAt }));
+    const docs = res.map(r => ({ id: r.id, content: r.content, metadata: r.metadata }));
+    const nextCursor = docs.length === lim ? docs[docs.length - 1].id : null;
+    return { documents: docs, nextCursor };
   }
 
   async deleteDocuments({ ids = null, filter = null } = {}) {

package/src/backends/qdrant_store.js

@@ -30,27 +30,25 @@ class QdrantVectorStore extends VectorStore {
     return res.map(r => ({ content: r.payload.content, metadata: r.payload.metadata, score: r.score }));
   }
   async hybridSearch(text, vector, limit = 5, filter = null) { return this.similaritySearch(vector, limit, filter); }
-  async listDocuments({ filter = null, limit = 100, offset = 0 } = {}) {
+  async listDocuments({ filter = null, limit = 100, cursor = null } = {}) {
     if (typeof this.client.scroll !== 'function') throw new Error('listDocuments is not supported for this Qdrant client');
     const qFilter = this.normalizeFilter(filter);
-    const lim = Math.max(1, Math.min(1000, Number(limit) || 100));
-    const off = Math.max(0, Number(offset) || 0);
-    let skipped = 0;
-    const out = [];
-    let nextOffset = undefined;
-    while (out.length < lim) {
-      const res = await this.client.scroll(this.collection, { limit: Math.min(256, lim), filter: qFilter, offset: nextOffset });
-      const points = res?.points || res?.result?.points || [];
-      nextOffset = res?.next_page_offset || res?.result?.next_page_offset || res?.next_page_offset;
-      if (!points.length) break;
-      for (const p of points) {
-        if (skipped < off) { skipped++; continue; }
-        out.push({ id: p.id, content: p.payload?.content, metadata: p.payload?.metadata });
-        if (out.length >= lim) break;
-      }
-      if (!nextOffset) break;
-    }
-    return out;
+    const lim = Math.max(1, Number(limit) || 100);
+    const res = await this.client.scroll(this.collection, { 
+      limit: lim, 
+      filter: qFilter, 
+      offset: cursor || undefined,
+      with_payload: true,
+      with_vector: false
+    });
+    const points = res?.points || res?.result?.points || [];
+    const nextCursor = res?.next_page_offset || res?.result?.next_page_offset;
+    const docs = points.map(p => ({
+      id: p.id,
+      content: p.payload?.content,
+      metadata: p.payload?.metadata
+    }));
+    return { documents: docs, nextCursor };
   }
   async fileExists(sha256, size, lastModified) {
     const filter = this.normalizeFilter({ fileSHA256: sha256, fileSize: size, lastModified });