vectra 0.12.2 → 0.14.0

package/src/BrowserWebFetcher.ts

@@ -0,0 +1,345 @@
+import { TextFetcher } from './types';
+
+/**
+ * Configuration options for BrowserWebFetcher.
+ */
+export interface BrowserWebFetcherConfig {
+    /**
+     * Optional. Whether to convert HTML to a simplified text/markdown format.
+     * @remarks
+     * Defaults to `true`.
+     */
+    htmlToMarkdown?: boolean;
+
+    /**
+     * Optional. Additional headers to include in requests.
+     */
+    headers?: Record<string, string>;
+
+    /**
+     * Optional. Request mode for fetch.
+     * @remarks
+     * Defaults to 'cors'.
+     */
+    mode?: RequestMode;
+
+    /**
+     * Optional. Credentials mode for fetch.
+     * @remarks
+     * Defaults to 'same-origin'.
+     */
+    credentials?: RequestCredentials;
+}
+
+/**
+ * Browser-compatible web fetcher using the native Fetch API.
+ * @remarks
+ * This fetcher works in browsers and Electron renderer processes.
+ * Uses DOMParser instead of cheerio for HTML parsing.
+ */
+export class BrowserWebFetcher implements TextFetcher {
+    private readonly _config: BrowserWebFetcherConfig;
+
+    private static readonly ALLOWED_CONTENT_TYPES = [
+        'text/html',
+        'application/json',
+        'application/xml',
+        'application/javascript',
+        'text/plain',
+        'text/markdown',
+        'text/xml'
+    ];
+
+    /**
+     * Creates a new `BrowserWebFetcher` instance.
+     * @param config Optional configuration options.
+     */
+    constructor(config?: BrowserWebFetcherConfig) {
+        this._config = {
+            htmlToMarkdown: true,
+            mode: 'cors',
+            credentials: 'same-origin',
+            ...config
+        };
+    }
+
+    /**
+     * Fetches content from a URL and passes it to the document handler.
+     * @param uri URL to fetch.
+     * @param onDocument Callback to handle the fetched document.
+     * @returns Promise that resolves to the return value of onDocument.
+     */
+    async fetch(
+        uri: string,
+        onDocument: (uri: string, text: string, docType?: string) => Promise<boolean>
+    ): Promise<boolean> {
+        const response = await fetch(uri, {
+            method: 'GET',
+            headers: this._config.headers,
+            mode: this._config.mode,
+            credentials: this._config.credentials
+        });
+
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+        }
+
+        const contentType = response.headers.get('content-type') || 'text/plain';
+        const mimeType = contentType.split(';')[0].trim().toLowerCase();
+
+        // Validate content type
+        if (!BrowserWebFetcher.ALLOWED_CONTENT_TYPES.some(allowed => mimeType.includes(allowed))) {
+            throw new Error(`Unsupported content type: ${contentType}`);
+        }
+
+        const text = await response.text();
+
+        // Handle HTML content
+        if (mimeType.includes('text/html') && this._config.htmlToMarkdown) {
+            const markdown = this.htmlToMarkdown(text, uri);
+            return onDocument(uri, markdown, 'md');
+        }
+
+        // Determine doc type from content type
+        const docType = this.getDocTypeFromMime(mimeType);
+        return onDocument(uri, text, docType);
+    }
+
+    /**
+     * Converts HTML to a simplified markdown-like format using DOMParser.
+     */
+    private htmlToMarkdown(html: string, baseUrl: string): string {
+        const parser = new DOMParser();
+        const doc = parser.parseFromString(html, 'text/html');
+
+        // Remove unwanted elements
+        const removeSelectors = ['script', 'style', 'noscript', 'iframe', 'svg', 'canvas'];
+        removeSelectors.forEach(selector => {
+            doc.querySelectorAll(selector).forEach(el => el.remove());
+        });
+
+        // Convert relative URLs to absolute
+        doc.querySelectorAll('a[href]').forEach(el => {
+            const href = el.getAttribute('href');
+            if (href && !href.startsWith('http') && !href.startsWith('//') && !href.startsWith('#')) {
+                try {
+                    el.setAttribute('href', new URL(href, baseUrl).toString());
+                } catch {
+                    // Leave as-is if URL parsing fails
+                }
+            }
+        });
+
+        // Process the body
+        const body = doc.body;
+        if (!body) {
+            return html;
+        }
+
+        const lines: string[] = [];
+        this.processNode(body, lines);
+
+        // Clean up the result
+        let result = lines.join('\n');
+
+        // Remove excessive newlines
+        result = result.replace(/\n{3,}/g, '\n\n');
+
+        // Trim leading/trailing whitespace
+        result = result.trim();
+
+        return result;
+    }
+
+    /**
+     * Recursively processes DOM nodes to extract text content.
+     */
+    private processNode(node: Node, lines: string[]): void {
+        if (node.nodeType === Node.TEXT_NODE) {
+            const text = node.textContent?.trim();
+            if (text) {
+                lines.push(text);
+            }
+            return;
+        }
+
+        if (node.nodeType !== Node.ELEMENT_NODE) {
+            return;
+        }
+
+        const el = node as Element;
+        const tagName = el.tagName.toLowerCase();
+
+        // Handle specific elements
+        switch (tagName) {
+            case 'h1':
+                lines.push('');
+                lines.push(`# ${this.getTextContent(el)}`);
+                lines.push('');
+                return;
+            case 'h2':
+                lines.push('');
+                lines.push(`## ${this.getTextContent(el)}`);
+                lines.push('');
+                return;
+            case 'h3':
+                lines.push('');
+                lines.push(`### ${this.getTextContent(el)}`);
+                lines.push('');
+                return;
+            case 'h4':
+                lines.push('');
+                lines.push(`#### ${this.getTextContent(el)}`);
+                lines.push('');
+                return;
+            case 'h5':
+                lines.push('');
+                lines.push(`##### ${this.getTextContent(el)}`);
+                lines.push('');
+                return;
+            case 'h6':
+                lines.push('');
+                lines.push(`###### ${this.getTextContent(el)}`);
+                lines.push('');
+                return;
+            case 'p':
+                lines.push('');
+                this.processChildren(el, lines);
+                lines.push('');
+                return;
+            case 'br':
+                lines.push('');
+                return;
+            case 'hr':
+                lines.push('');
+                lines.push('---');
+                lines.push('');
+                return;
+            case 'a':
+                const href = el.getAttribute('href');
+                const text = this.getTextContent(el);
+                if (href && text) {
+                    lines.push(`[${text}](${href})`);
+                } else if (text) {
+                    lines.push(text);
+                }
+                return;
+            case 'strong':
+            case 'b':
+                lines.push(`**${this.getTextContent(el)}**`);
+                return;
+            case 'em':
+            case 'i':
+                lines.push(`*${this.getTextContent(el)}*`);
+                return;
+            case 'code':
+                lines.push(`\`${this.getTextContent(el)}\``);
+                return;
+            case 'pre':
+                lines.push('');
+                lines.push('```');
+                lines.push(this.getTextContent(el));
+                lines.push('```');
+                lines.push('');
+                return;
+            case 'blockquote':
+                lines.push('');
+                const quoteText = this.getTextContent(el);
+                quoteText.split('\n').forEach(line => {
+                    lines.push(`> ${line}`);
+                });
+                lines.push('');
+                return;
+            case 'ul':
+            case 'ol':
+                lines.push('');
+                el.querySelectorAll(':scope > li').forEach((li, index) => {
+                    const prefix = tagName === 'ol' ? `${index + 1}.` : '-';
+                    lines.push(`${prefix} ${this.getTextContent(li)}`);
+                });
+                lines.push('');
+                return;
+            case 'table':
+                lines.push('');
+                this.processTable(el, lines);
+                lines.push('');
+                return;
+            case 'img':
+                const alt = el.getAttribute('alt') || 'image';
+                const src = el.getAttribute('src');
+                if (src) {
+                    lines.push(`![${alt}](${src})`);
+                }
+                return;
+            default:
+                // For other elements, process children
+                this.processChildren(el, lines);
+        }
+    }
+
+    /**
+     * Processes child nodes of an element.
+     */
+    private processChildren(el: Element, lines: string[]): void {
+        el.childNodes.forEach(child => {
+            this.processNode(child, lines);
+        });
+    }
+
+    /**
+     * Gets clean text content from an element.
+     */
+    private getTextContent(el: Element): string {
+        return (el.textContent || '').replace(/\s+/g, ' ').trim();
+    }
+
+    /**
+     * Processes a table element to markdown format.
+     */
+    private processTable(table: Element, lines: string[]): void {
+        const rows = table.querySelectorAll('tr');
+        let isFirstRow = true;
+
+        rows.forEach(row => {
+            const cells = row.querySelectorAll('th, td');
+            const cellContents: string[] = [];
+
+            cells.forEach(cell => {
+                cellContents.push(this.getTextContent(cell));
+            });
+
+            if (cellContents.length > 0) {
+                lines.push(`| ${cellContents.join(' | ')} |`);
+
+                // Add separator after header row
+                if (isFirstRow) {
+                    lines.push(`| ${cellContents.map(() => '---').join(' | ')} |`);
+                    isFirstRow = false;
+                }
+            }
+        });
+    }
+
+    /**
+     * Maps MIME type to document type.
+     */
+    private getDocTypeFromMime(mimeType: string): string | undefined {
+        const mimeMap: Record<string, string> = {
+            'text/html': 'html',
+            'text/plain': undefined as any,
+            'text/markdown': 'md',
+            'text/xml': 'xml',
+            'application/json': 'json',
+            'application/xml': 'xml',
+            'application/javascript': 'js'
+        };
+
+        for (const [mime, docType] of Object.entries(mimeMap)) {
+            if (mimeType.includes(mime)) {
+                return docType;
+            }
+        }
+
+        return undefined;
+    }
+}

package/src/FileFetcher.spec.ts

@@ -0,0 +1,234 @@
+import { strict as assert } from 'assert';
+import * as sinon from 'sinon';
+import fs from 'node:fs';
+import * as path from 'path';
+import { FileFetcher } from './FileFetcher';
+
+describe('FileFetcher', () => {
+  let fetcher: FileFetcher;
+
+  let statStub: sinon.SinonStub;
+  let readdirStub: sinon.SinonStub;
+  let readFileStub: sinon.SinonStub;
+
+  beforeEach(() => {
+    statStub = sinon.stub(fs.promises, 'stat') as sinon.SinonStub;
+    readdirStub = sinon.stub(fs.promises, 'readdir') as sinon.SinonStub;
+    readFileStub = sinon.stub(fs.promises, 'readFile') as sinon.SinonStub;
+
+    fetcher = new FileFetcher();
+  });
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('resolves true when the path does not exist and does not call onDocument', async () => {
+    statStub.rejects(new Error('not found'));
+    const onDocument = sinon.fake.resolves(true);
+
+    const result = await fetcher.fetch('/nonexistent', onDocument);
+
+    assert.equal(result, true);
+    assert.equal(onDocument.callCount, 0);
+  });
+
+  it('recurses a flat directory and calls fetch for each file', async () => {
+    statStub.callsFake(async (uri: string) => {
+      if (uri === '/dir') return { isDirectory: () => true };
+      if (uri === path.join('/dir', 'file1.txt')) return { isDirectory: () => false };
+      if (uri === path.join('/dir', 'file2.md')) return { isDirectory: () => false };
+      return { isDirectory: () => false };
+    });
+    readdirStub.callsFake(async (uri: string) => {
+      if (uri === '/dir') return ['file1.txt', 'file2.md'];
+      return [];
+    });
+    readFileStub.resolves('content');
+
+    const fetchSpy = sinon.spy(fetcher, 'fetch');
+    const onDocument = sinon.fake.resolves(true);
+
+    const result = await fetcher.fetch('/dir', onDocument);
+
+    assert.equal(result, true);
+    assert.equal(fetchSpy.callCount, 3); // root + 2 files
+    assert(fetchSpy.calledWith(path.join('/dir', 'file1.txt'), sinon.match.func));
+    assert(fetchSpy.calledWith(path.join('/dir', 'file2.md'), sinon.match.func));
+    assert.equal(onDocument.callCount, 2);
+    assert(onDocument.calledWith(path.join('/dir', 'file1.txt'), 'content', 'txt'));
+    assert(onDocument.calledWith(path.join('/dir', 'file2.md'), 'content', 'md'));
+  });
+
+  it('reads a file and passes correct args to onDocument (uri, text, docType)', async () => {
+    statStub.resolves({ isDirectory: () => false });
+    readFileStub.resolves('file content');
+
+    const onDocument = sinon.fake.resolves(true);
+    const result = await fetcher.fetch('/file.txt', onDocument);
+
+    assert.equal(result, true);
+    assert.equal(onDocument.callCount, 1);
+
+    const [uri, text, docType] = onDocument.firstCall.args;
+    assert.equal(uri, '/file.txt');
+    assert.equal(text, 'file content');
+    assert.equal(docType, 'txt');
+  });
+
+  it('handles file with no extension by using last path segment as docType', async () => {
+    statStub.resolves({ isDirectory: () => false });
+    readFileStub.resolves('content');
+
+    const onDocument = sinon.fake.resolves(true);
+    const result = await fetcher.fetch('/file', onDocument);
+
+    assert.equal(result, true);
+    assert.equal(onDocument.callCount, 1);
+
+    const [uri, text, docType] = onDocument.firstCall.args;
+    assert.equal(uri, '/file');
+    assert.equal(text, 'content');
+    assert.equal(docType, 'file');
+  });
+
+  it('awaits onDocument and returns its boolean result (true)', async () => {
+    statStub.resolves({ isDirectory: () => false });
+    readFileStub.resolves('text');
+
+    const onDocumentTrue = sinon.fake.resolves(true);
+    const resultTrue = await fetcher.fetch('/file.txt', onDocumentTrue);
+
+    assert.equal(resultTrue, true);
+  });
+
+  it('awaits onDocument and returns its boolean result (false)', async () => {
+    statStub.resolves({ isDirectory: () => false });
+    readFileStub.resolves('text');
+
+    const onDocumentFalse = sinon.fake.resolves(false);
+    const resultFalse = await fetcher.fetch('/file.txt', onDocumentFalse);
+
+    assert.equal(resultFalse, false);
+  });
+
+  it('propagates a rejection from onDocument as a rejected promise', async () => {
+    statStub.resolves({ isDirectory: () => false });
+    readFileStub.resolves('text');
+
+    const onDocument = sinon.fake.rejects(new Error('fail'));
+    await assert.rejects(() => fetcher.fetch('/file.txt', onDocument), /fail/);
+  });
+
+  it('recurses nested directories and processes files at multiple depths', async () => {
+    statStub.callsFake(async (uri: string) => {
+      if (uri === '/dir' || uri === path.join('/dir', 'subdir')) return { isDirectory: () => true };
+      return { isDirectory: () => false };
+    });
+    readdirStub.callsFake(async (uri: string) => {
+      if (uri === '/dir') return ['file1.txt', 'subdir'];
+      if (uri === path.join('/dir', 'subdir')) return ['file2.md'];
+      return [];
+    });
+    readFileStub.resolves('content');
+
+    const onDocument = sinon.fake.resolves(true);
+    const result = await fetcher.fetch('/dir', onDocument);
+
+    assert.equal(result, true);
+    assert.equal(onDocument.callCount, 2);
+    assert(onDocument.calledWith(path.join('/dir', 'file1.txt'), 'content', 'txt'));
+    assert(onDocument.calledWith(path.join('/dir', 'subdir', 'file2.md'), 'content', 'md'));
+  });
+
+  it('handles an empty directory (no onDocument calls, resolves true)', async () => {
+    statStub.resolves({ isDirectory: () => true });
+    readdirStub.resolves([]);
+
+    const onDocument = sinon.fake.resolves(true);
+    const result = await fetcher.fetch('/emptydir', onDocument);
+
+    assert.equal(result, true);
+    assert.equal(onDocument.callCount, 0);
+  });
+
+  it('recurses directories that contain only subdirectories (no onDocument calls)', async () => {
+    statStub.callsFake(async (uri: string) => {
+      if (
+        uri === '/dir' ||
+        uri === path.join('/dir', 'subdir1') ||
+        uri === path.join('/dir', 'subdir2')
+      ) {
+        return { isDirectory: () => true };
+      }
+      return { isDirectory: () => false };
+    });
+    readdirStub.callsFake(async (uri: string) => {
+      if (uri === '/dir') return ['subdir1', 'subdir2'];
+      if (uri === path.join('/dir', 'subdir1')) return [];
+      if (uri === path.join('/dir', 'subdir2')) return [];
+      return [];
+    });
+
+    const onDocument = sinon.fake.resolves(true);
+    const result = await fetcher.fetch('/dir', onDocument);
+
+    assert.equal(result, true);
+    assert.equal(onDocument.callCount, 0);
+  });
+
+  it('rejects when readdir fails for a directory', async () => {
+    statStub.resolves({ isDirectory: () => true });
+    readdirStub.rejects(new Error('fail'));
+
+    const onDocument = sinon.fake.resolves(true);
+    await assert.rejects(() => fetcher.fetch('/dir', onDocument), /fail/);
+  });
+
+  it('rejects when readFile fails for a file', async () => {
+    statStub.resolves({ isDirectory: () => false });
+    readFileStub.rejects(new Error('fail'));
+
+    const onDocument = sinon.fake.resolves(true);
+    await assert.rejects(() => fetcher.fetch('/file.txt', onDocument), /fail/);
+  });
+
+  it('extracts docType from multi-part extensions (file.tar.gz -> gz)', async () => {
+    statStub.resolves({ isDirectory: () => false });
+    readFileStub.resolves('content');
+
+    const onDocument = sinon.fake.resolves(true);
+    const result = await fetcher.fetch('/file.tar.gz', onDocument);
+
+    assert.equal(result, true);
+    assert.equal(onDocument.callCount, 1);
+
+    const [, , docType] = onDocument.firstCall.args;
+    assert.equal(docType, 'gz');
+  });
+
+  it('returns false when any child in a directory returns false (aggregates to allOk=false)', async () => {
+    statStub.callsFake(async (uri: string) => {
+      if (uri === '/dir') return { isDirectory: () => true };
+      return { isDirectory: () => false };
+    });
+    readdirStub.callsFake(async (uri: string) => {
+      if (uri === '/dir') return ['good.txt', 'bad.txt'];
+      return [];
+    });
+    readFileStub.resolves('content');
+
+    const onDocument = sinon.stub();
+    onDocument.callsFake(async (uri: string) => {
+      if (uri === path.join('/dir', 'bad.txt')) return false;
+      return true;
+    });
+
+    const result = await fetcher.fetch('/dir', onDocument);
+
+    assert.equal(result, false);
+    assert.equal(onDocument.callCount, 2);
+    assert(onDocument.calledWith(path.join('/dir', 'good.txt'), 'content', 'txt'));
+    assert(onDocument.calledWith(path.join('/dir', 'bad.txt'), 'content', 'txt'));
+  });
+});

package/src/FileFetcher.ts

@@ -1,31 +1,43 @@
 import { TextFetcher } from './types';
-import * as fs from 'fs/promises';
-import * as path from 'path';
+import fs from 'node:fs';
+import * as path from 'node:path';
 
 export class FileFetcher implements TextFetcher {
-    public async fetch(uri: string, onDocument: (uri: string, text: string, docType?: string | undefined) => Promise<boolean>): Promise<boolean> {
-        // Does path exist and is it a directory?
-        let isDirectory: boolean;
-        try {
-            const stat = await fs.stat(uri);
-            isDirectory = stat.isDirectory();
-        } catch {
-            return true;
-        }
+  public async fetch(
+    uri: string,
+    onDocument: (uri: string, text: string, docType?: string | undefined) => Promise<boolean>
+  ): Promise<boolean> {
+    // Does path exist and is it a directory?
+    let stat;
+    try {
+      stat = await fs.promises.stat(uri);
+    } catch {
+      // Non-existent path: treat as no-op success
+      return true;
+    }
 
-        // If directory, read all files and recurse
-        if (isDirectory) {
-            const files = await fs.readdir(uri);
-            for (const file of files) {
-                const filePath = path.join(uri, file);
-                await this.fetch(filePath, onDocument);
-            }
-            return true;
-        } else {
-            // Read file and call onDocument
-            const text = await fs.readFile(uri, 'utf8');
-            const parts = uri.split('.');
-            return await onDocument(uri, text, parts.length > 0 ? parts[parts.length - 1].toLowerCase() : undefined);
+    if (stat.isDirectory()) {
+      // Read directory and recurse. If any child returns false, aggregate to false.
+      const entries = await fs.promises.readdir(uri);
+      let allOk = true;
+      for (const file of entries) {
+        const filePath = path.join(uri, file);
+        const ok = await this.fetch(filePath, onDocument);
+        if (!ok) {
+          allOk = false;
         }
+      }
+      return allOk;
+    } else {
+      // Read file and invoke onDocument
+      const text = await fs.promises.readFile(uri, 'utf8');
+      const ext = path.extname(uri);
+      const docType =
+        ext && ext.length > 1
+          ? ext.slice(1).toLowerCase()
+          : path.basename(uri).toLowerCase();
+
+      return await onDocument(uri, text, docType);
     }
-}
+  }
+}