vectra 0.12.2 → 0.14.0

package/src/WebFetcher.ts

@@ -1,227 +1,225 @@
-import axios, { AxiosRequestConfig } from "axios";
 import { TextFetcher } from './types';
 import * as cheerio from 'cheerio';
-import TurndownService  from 'turndown';
+import TurndownService from 'turndown';
 
 const ALLOWED_CONTENT_TYPES = [
-    "text/html",
-    "application/json",
-    "application/xml",
-    "application/javascript",
-    "text/plain",
+  "text/html",
+  "application/json",
+  "application/xml",
+  "application/javascript",
+  "text/plain",
 ];
 
-const DEFAULT_HEADERS = {
-    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-    "Accept-Encoding": "gzip, deflate, sdch, br",
-    "Accept-Language": "en-US,en;q=0.8,ms;q=0.6",
-    "Alt-Used": "LEAVE-THIS-KEY-SET-BY-TOOL",
-    Connection: "keep-alive",
-    Host: "LEAVE-THIS-KEY-SET-BY-TOOL",
-    Referer: "https://www.google.com/",
-    "Sec-Fetch-Dest": "document",
-    "Sec-Fetch-Mode": "navigate",
-    "Sec-Fetch-Site": "cross-site",
-    "Upgrade-Insecure-Requests": "1",
-    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36",
+const DEFAULT_HEADERS: Record<string, string> = {
+  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+  "Accept-Encoding": "gzip, deflate, sdch, br",
+  "Accept-Language": "en-US,en;q=0.8,ms;q=0.6",
+  "Alt-Used": "LEAVE-THIS-KEY-SET-BY-TOOL",
+  Connection: "keep-alive",
+  Host: "LEAVE-THIS-KEY-SET-BY-TOOL",
+  Referer: "https://www.google.com/",
+  "Sec-Fetch-Dest": "document",
+  "Sec-Fetch-Mode": "navigate",
+  "Sec-Fetch-Site": "cross-site",
+  "Upgrade-Insecure-Requests": "1",
+  "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36",
 };
 
 export interface WebFetcherConfig {
-    headers?: Record<string,string>;
-    requestConfig?: AxiosRequestConfig;
-    htmlToMarkdown: boolean;
-    summarizeHtml: boolean;
+  headers?: Record<string, string>;
+  requestConfig?: RequestInit;
+  htmlToMarkdown: boolean;
+  summarizeHtml: boolean;
 }
 
 export class WebFetcher implements TextFetcher {
-    private readonly _config: WebFetcherConfig;
-
-    public constructor(config?: Partial<WebFetcherConfig>) {
-        this._config = Object.assign({
-            htmlToMarkdown: true,
-            summarizeHtml: false,
-        } as WebFetcherConfig, config);
+  private readonly _config: WebFetcherConfig;
+
+  public constructor(config?: Partial<WebFetcherConfig>) {
+    this._config = Object.assign(
+      {
+        htmlToMarkdown: true,
+        summarizeHtml: false,
+      } as WebFetcherConfig,
+      config
+    );
+  }
+
+  public async fetch(
+    uri: string,
+    onDocument: (uri: string, text: string, docType?: string) => Promise<boolean>
+  ): Promise<boolean> {
+    // Clone headers to avoid mutating the original
+    const headers: Record<string, string> = Object.assign({}, DEFAULT_HEADERS, this._config.headers);
+
+    // get hostname from url
+    const host = new URL(uri).hostname;
+    headers['Host'] = host;
+    headers['Alt-Used'] = host;
+
+    // Fetch page and check for errors
+    const response = await fetch(uri, {
+      ...this._config.requestConfig,
+      method: 'GET',
+      headers,
+    });
+    if (response.status >= 400) {
+      throw new Error(`Site returned an HTTP status of ${response.status}`);
     }
 
-    public async fetch(uri: string, onDocument: (uri: string, text: string, docType?: string) => Promise<boolean>): Promise<boolean> {
-        const httpClient = axios.create({
-            validateStatus: () => true,
-        });
-
-        // Clone headers to avoid mutating the original
-        const headers = Object.assign({}, DEFAULT_HEADERS, this._config.headers)
-
-        // get hostname from url
-        const host = new URL(uri).hostname;
-        headers['Host'] = host;
-        headers['Alt-Used'] = host;
-
-        // Fetch page and check for errors
-        const response = await httpClient.get(uri, {
-            headers,
-            ...this._config.requestConfig,
-        });
-        if (response.status >= 400) {
-            throw new Error(`Site returned an HTTP status of ${response.status}`);
-        }
-
-        // Check for valid content type
-        const contentType = response.headers['content-type'];
-        const contentTypeArray = contentType.split(';');
-        if (!contentTypeArray[0] || !ALLOWED_CONTENT_TYPES.includes(contentTypeArray[0])) {
-            throw new Error(`Site returned an invalid content type of ${contentType}`);
-        }
-
-        // Convert content type to doc type
-        const docType = contentTypeArray[0] != 'text/plain' ? contentTypeArray[0].split('/')[1] : undefined;
-        if (docType == 'html' && this._config.htmlToMarkdown) {
-            const text = this.htmlToMarkdown(response.data, uri);
-            return await onDocument(uri, text, 'md');
-        } else {
-            const text = response.data;
-            return await onDocument(uri, text, docType);
-        }
+    // Check for valid content type
+    const contentType = response.headers.get('content-type') ?? '';
+    const contentTypeArray = contentType.split(';');
+    if (!contentTypeArray[0] || !ALLOWED_CONTENT_TYPES.includes(contentTypeArray[0])) {
+      throw new Error(`Site returned an invalid content type of ${contentType}`);
     }
 
-
-    private htmlToMarkdown(html: string, baseUrl: string): string {
-        // Parse HTML and remove scripts
-        const $ = cheerio.load(html, { scriptingEnabled: true });
-
-        // Remove scripts and convert relative links to absolute
-        $('script').remove();
-        $('a').each((i, elem) => {
-            const $el = $(elem);
-            const href = $el.attr("href");
-            if (href && !href.startsWith("http")) {
-                // Try converting to an absolute link
-                try {
-                    $el.attr("href", new URL(href, baseUrl).toString());
-                } catch {
-                    // Leave as is
-                }
-            }
-        });
-
-        // Convert to markdown
-        const body = $('body').html() ?? '';
-        const turndownService = new TurndownService({
-            hr: '\n\n---\n\n',
-        });
-        convertTables(turndownService);
-        const md = turndownService.turndown(body);
-
-        // Remove any overly long header text
-        const contentStart = Math.min(md.indexOf('\n'), md.indexOf(' '));
-        if (contentStart > 64) {
-            return md.slice(contentStart);
-        } else {
-            return md;
-        }
+    // Convert content type to doc type
+    const docType = contentTypeArray[0] != 'text/plain' ? contentTypeArray[0].split('/')[1] : undefined;
+    if (docType == 'html' && this._config.htmlToMarkdown) {
+      const data = await response.text();
+      const text = this.htmlToMarkdown(data, uri);
+      return await onDocument(uri, text, 'md');
+    } else {
+      const text = await response.text();
+      return await onDocument(uri, text, docType);
     }
-}
-
-function convertTables(turndownService: TurndownService): void {
-    turndownService.addRule('tableCell', {
-        filter: ['th', 'td'],
-        replacement: function (content, node) {
-            return cell(content, node)
-        }
-    });
-
-    turndownService.addRule('tableRow', {
-        filter: 'tr',
-        replacement: function (content, node) {
-            var borderCells = ''
-            var alignMap: any = { left: ':--', right: '--:', center: ':-:' }
-
-            if (isHeadingRow(node)) {
-                for (var i = 0; i < node.childNodes.length; i++) {
-                    var border = '---'
-                    var align: string = (
-                        node.childNodes[i].getAttribute('align') || ''
-                    ).toLowerCase()
-
-                    if (align) border = alignMap[align] || border
-
-                    borderCells += cell(border, node.childNodes[i])
-                }
-            }
-            return '\n' + content + (borderCells ? '\n' + borderCells : '')
+  }
+
+  private htmlToMarkdown(html: string, baseUrl: string): string {
+    // Parse HTML
+    const $ = cheerio.load(html, { scriptingEnabled: true });
+
+    // Remove scripts and convert relative links to absolute
+    $("script").remove();
+    $("a").each((_, elem) => {
+      const $el = $(elem);
+      const href = $el.attr("href");
+      if (href && !href.startsWith("http")) {
+        try {
+          $el.attr("href", new URL(href, baseUrl).toString());
+        } catch {
+          // ignore bad URLs
         }
+      }
     });
 
-    turndownService.addRule('table', {
-        filter: ['table'],
-        replacement: function (content, node) {
-            // Ensure there are no blank lines
-            content = content.replace('\n\n', '\n')
-            return '\n\n' + content + '\n\n'
-        }
+    // Convert to markdown
+    const body = $("body").html() ?? "";
+    const turndownService = new TurndownService({
+      hr: "\n\n---\n\n",
     });
+    convertTables(turndownService);
+    const md = turndownService.turndown(body);
+
+    // Remove any overly long header text
+    const firstSpace = md.indexOf(" ");
+    const firstNewline = md.indexOf("\n");
+    const candidates = [firstSpace, firstNewline].filter((i) => i >= 0);
+    const contentStart = candidates.length ? Math.min(...candidates) : -1;
+
+    if (contentStart > 64) {
+      return md.slice(contentStart);
+    } else {
+      return md;
+    }
+  }
+}
 
-    turndownService.addRule('tableSection', {
-        filter: ['thead', 'tbody', 'tfoot'],
-        replacement: function (content) {
-            return content
+function convertTables(turndownService: TurndownService): void {
+  turndownService.addRule("tableCell", {
+    filter: ["th", "td"],
+    replacement: function (content: string, node: any): string {
+      return cell(content, node);
+    },
+  });
+
+  turndownService.addRule("tableRow", {
+    filter: "tr",
+    replacement: function (content: string, node: any): string {
+      let borderCells = "";
+      const alignMap: Record<string, string> = { left: ":--", right: "--:", center: ":-:" };
+
+      if (isHeadingRow(node)) {
+        const children = (node as any).childNodes;
+        for (let i = 0; i < children.length; i++) {
+          let border = "---";
+          const align: string = (children[i].getAttribute?.("align") || "").toLowerCase();
+          if (align) border = (alignMap as any)[align] || border;
+          borderCells += cell(border, children[i]);
         }
-    });
+      }
+      return "\n" + content + (borderCells ? "\n" + borderCells : "");
+    },
+  });
+
+  turndownService.addRule("table", {
+    filter: ["table"],
+    replacement: function (content: string): string {
+      // Ensure there are no blank lines
+      content = content.replace("\n\n", "\n");
+      return "\n\n" + content + "\n\n";
+    },
+  });
+
+  turndownService.addRule("tableSection", {
+    filter: ["thead", "tbody", "tfoot"],
+    replacement: function (content: string): string {
+      return content;
+    },
+  });
 }
 
-const indexOf = Array.prototype.indexOf
-const every = Array.prototype.every
+const indexOf = Array.prototype.indexOf;
+const every = Array.prototype.every;
 
 // A tr is a heading row if:
 // - the parent is a THEAD
 // - or if its the first child of the TABLE or the first TBODY (possibly
 //   following a blank THEAD)
 // - and every cell is a TH
-function isHeadingRow(tr: any) {
-    var parentNode = tr.parentNode
-    return (
-        parentNode.nodeName === 'THEAD' ||
-        (
-            parentNode.firstChild === tr &&
-            (parentNode.nodeName === 'TABLE' || isFirstTbody(parentNode)) &&
-            every.call(tr.childNodes, function (n) { return n.nodeName === 'TH' })
-        )
-    )
+function isHeadingRow(tr: any): boolean {
+  const parentNode = tr.parentNode;
+  return (
+    parentNode.nodeName === "THEAD" ||
+    (parentNode.firstChild === tr &&
+      (parentNode.nodeName === "TABLE" || isFirstTbody(parentNode)) &&
+      every.call(parentNode ? tr.childNodes : [], function (n: any) {
+        return n.nodeName === "TH";
+      }))
+  );
 }
 
-function isFirstTbody(element: any) {
-    var previousSibling = element.previousSibling
-    return (
-        element.nodeName === 'TBODY' && (
-            !previousSibling ||
-            (
-                previousSibling.nodeName === 'THEAD' &&
-                /^\s*$/i.test(previousSibling.textContent)
-            )
-        )
-    )
+function isFirstTbody(element: any): boolean {
+  const previousSibling = element.previousSibling;
+  return (
+    element.nodeName === "TBODY" &&
+    (!previousSibling ||
+      (previousSibling.nodeName === "THEAD" && /^\s*$/i.test(previousSibling.textContent)))
+  );
 }
 
 function cell(content: string, node: any): string {
-    var index = indexOf.call(node.parentNode.childNodes, node)
-    var prefix = ' '
-    if (index === 0) {
-        prefix = '| '
-    }
-    return cleanContent(prefix + content + ' |');
+  const index = indexOf.call(node.parentNode.childNodes, node);
+  let prefix = " ";
+  if (index === 0) {
+    prefix = "| ";
+  }
+  return cleanContent(prefix + content + " |");
 }
 
 function cleanContent(content: string): string {
-    let output = '';
-    const chars = ['\n', '\r', '\t', '\f', '\v', '\u00a0', '\u2028', '\u2029', ' '];
-    for (let i = 0; i < content.length; i++) {
-        if (chars.includes(content[i])) {
-            if (output[output.length - 1] != ' ') {
-                output += ' ';
-            }
-            continue;
-        } else {
-            output += content[i];
-        }
+  let output = "";
+  const chars = ["\n", "\r", "\t", "\f", "\v", "\u00a0", "\u2028", "\u2029", " "];
+  for (let i = 0; i < content.length; i++) {
+    if (chars.includes(content[i])) {
+      if (output[output.length - 1] !== " ") {
+        output += " ";
+      }
+      continue;
+    } else {
+      output += content[i];
     }
-    return output;
+  }
+  return output;
 }

package/src/browser.ts

@@ -0,0 +1,69 @@
+/**
+ * Browser entry point for Vectra.
+ *
+ * This module exports only browser-compatible components.
+ * Node-specific components (LocalFileStorage, FileFetcher, WebFetcher) are excluded.
+ *
+ * For browser usage:
+ * - Use `VirtualFileStorage` for in-memory storage (ephemeral)
+ * - Use `IndexedDBStorage` for persistent browser storage
+ * - Use `BrowserWebFetcher` for fetching web content
+ */
+
+// Core index classes
+export { LocalIndex, CreateIndexConfig } from './LocalIndex';
+export { LocalDocumentIndex, LocalDocumentIndexConfig, DocumentQueryOptions } from './LocalDocumentIndex';
+
+// Document classes
+export { LocalDocument } from './LocalDocument';
+export { LocalDocumentResult } from './LocalDocumentResult';
+
+// Text processing
+export { TextSplitter, TextSplitterConfig } from './TextSplitter';
+export { GPT3Tokenizer } from './GPT3Tokenizer';
+
+// Embeddings
+export {
+    OpenAIEmbeddings,
+    OpenAIEmbeddingsOptions,
+    AzureOpenAIEmbeddingsOptions,
+    OSSEmbeddingsOptions,
+    BaseOpenAIEmbeddingsOptions
+} from './OpenAIEmbeddings';
+export { TransformersEmbeddings, TransformersEmbeddingsOptions } from './TransformersEmbeddings';
+export { TransformersTokenizer } from './TransformersTokenizer';
+
+// Query utilities
+export { ItemSelector } from './ItemSelector';
+
+// Types
+export {
+    EmbeddingsModel,
+    EmbeddingsResponse,
+    EmbeddingsResponseStatus,
+    TextChunk,
+    TextFetcher,
+    IndexStats,
+    IndexItem,
+    MetadataFilter,
+    MetadataTypes,
+    QueryResult,
+    Tokenizer,
+    DocumentChunkMetadata,
+    DocumentCatalogStats,
+    DocumentTextSection,
+    IndexData
+} from './types';
+
+// Storage - browser compatible
+export { FileStorage, FileDetails, ListFilesFilter } from './storage/FileStorage';
+export { FileStorageUtilities } from './storage/FileStorageUtilities';
+export { FileType } from './storage/FileType';
+export { VirtualFileStorage } from './storage/VirtualFileStorage';
+export { IndexedDBStorage } from './storage/IndexedDBStorage';
+
+// Browser-compatible fetcher
+export { BrowserWebFetcher, BrowserWebFetcherConfig } from './BrowserWebFetcher';
+
+// Utilities
+export { pathUtils } from './utils/pathUtils';

package/src/codecs/IndexCodec.ts

@@ -0,0 +1,40 @@
+import { IndexData, MetadataTypes } from '../types';
+
+/**
+ * Catalog structure for document indexes.
+ */
+export interface DocumentCatalog {
+    version: number;
+    count: number;
+    uriToId: { [uri: string]: string };
+    idToUri: { [id: string]: string };
+}
+
+/**
+ * Abstraction for index serialization format.
+ * @remarks
+ * Implementations control how index data, catalogs, and metadata are
+ * serialized to and from Buffers written by the storage backend.
+ */
+export interface IndexCodec {
+    /** File extension used for index files (e.g., '.json', '.pb') */
+    readonly extension: string;
+
+    /** Serialize IndexData to a Buffer for writing to storage */
+    serializeIndex(data: IndexData): Buffer;
+
+    /** Deserialize a Buffer read from storage into IndexData */
+    deserializeIndex(buffer: Buffer): IndexData;
+
+    /** Serialize a DocumentCatalog to a Buffer */
+    serializeCatalog(catalog: DocumentCatalog): Buffer;
+
+    /** Deserialize a Buffer into a DocumentCatalog */
+    deserializeCatalog(buffer: Buffer): DocumentCatalog;
+
+    /** Serialize arbitrary metadata to a Buffer */
+    serializeMetadata(metadata: Record<string, MetadataTypes>): Buffer;
+
+    /** Deserialize a Buffer into metadata */
+    deserializeMetadata(buffer: Buffer): Record<string, MetadataTypes>;
+}

package/src/codecs/JsonCodec.spec.ts

@@ -0,0 +1,70 @@
+import assert from 'node:assert';
+import { JsonCodec } from './JsonCodec';
+import { IndexData } from '../types';
+import { DocumentCatalog } from './IndexCodec';
+
+describe('JsonCodec', () => {
+    const codec = new JsonCodec();
+
+    it('has .json extension', () => {
+        assert.equal(codec.extension, '.json');
+    });
+
+    describe('serializeIndex / deserializeIndex', () => {
+        it('round-trips IndexData', () => {
+            const data: IndexData = {
+                version: 1,
+                metadata_config: { indexed: ['category'] },
+                items: [
+                    { id: 'a', metadata: { category: 'food', score: 0.9 }, vector: [1.5, 2.5, 3.5], norm: 4.5 },
+                    { id: 'b', metadata: {}, vector: [0, 0, 0], norm: 0, metadataFile: 'ext.json' },
+                ],
+            };
+            const buf = codec.serializeIndex(data);
+            assert.ok(Buffer.isBuffer(buf));
+            const result = codec.deserializeIndex(buf);
+            assert.deepStrictEqual(result, data);
+        });
+
+        it('handles empty items', () => {
+            const data: IndexData = { version: 2, metadata_config: {}, items: [] };
+            const result = codec.deserializeIndex(codec.serializeIndex(data));
+            assert.deepStrictEqual(result, data);
+        });
+    });
+
+    describe('serializeCatalog / deserializeCatalog', () => {
+        it('round-trips DocumentCatalog', () => {
+            const catalog: DocumentCatalog = {
+                version: 1,
+                count: 2,
+                uriToId: { 'http://a': 'id-a', 'http://b': 'id-b' },
+                idToUri: { 'id-a': 'http://a', 'id-b': 'http://b' },
+            };
+            const buf = codec.serializeCatalog(catalog);
+            assert.ok(Buffer.isBuffer(buf));
+            const result = codec.deserializeCatalog(buf);
+            assert.deepStrictEqual(result, catalog);
+        });
+
+        it('handles empty catalog', () => {
+            const catalog: DocumentCatalog = { version: 1, count: 0, uriToId: {}, idToUri: {} };
+            const result = codec.deserializeCatalog(codec.serializeCatalog(catalog));
+            assert.deepStrictEqual(result, catalog);
+        });
+    });
+
+    describe('serializeMetadata / deserializeMetadata', () => {
+        it('round-trips metadata with all types', () => {
+            const metadata = { name: 'test', score: 42, active: true };
+            const buf = codec.serializeMetadata(metadata);
+            const result = codec.deserializeMetadata(buf);
+            assert.deepStrictEqual(result, metadata);
+        });
+
+        it('handles empty metadata', () => {
+            const result = codec.deserializeMetadata(codec.serializeMetadata({}));
+            assert.deepStrictEqual(result, {});
+        });
+    });
+});

package/src/codecs/JsonCodec.ts

@@ -0,0 +1,37 @@
+import { IndexData, MetadataTypes } from '../types';
+import { DocumentCatalog, IndexCodec } from './IndexCodec';
+
+/**
+ * JSON codec — default serialization format.
+ * @remarks
+ * Wraps the existing JSON.stringify/JSON.parse behavior. This preserves
+ * full backward compatibility with indexes created before the codec
+ * abstraction was introduced.
+ */
+export class JsonCodec implements IndexCodec {
+    readonly extension = '.json';
+
+    serializeIndex(data: IndexData): Buffer {
+        return Buffer.from(JSON.stringify(data), 'utf-8');
+    }
+
+    deserializeIndex(buffer: Buffer): IndexData {
+        return JSON.parse(buffer.toString('utf-8'));
+    }
+
+    serializeCatalog(catalog: DocumentCatalog): Buffer {
+        return Buffer.from(JSON.stringify(catalog), 'utf-8');
+    }
+
+    deserializeCatalog(buffer: Buffer): DocumentCatalog {
+        return JSON.parse(buffer.toString('utf-8'));
+    }
+
+    serializeMetadata(metadata: Record<string, MetadataTypes>): Buffer {
+        return Buffer.from(JSON.stringify(metadata), 'utf-8');
+    }
+
+    deserializeMetadata(buffer: Buffer): Record<string, MetadataTypes> {
+        return JSON.parse(buffer.toString('utf-8'));
+    }
+}