vectra 0.12.2 → 0.14.0

package/src/FolderWatcher.spec.ts

@@ -0,0 +1,288 @@
+import { strict as assert } from 'assert';
+import * as sinon from 'sinon';
+import * as os from 'os';
+import * as path from 'path';
+import fs from 'node:fs';
+import { FolderWatcher } from './FolderWatcher';
+import { LocalDocumentIndex } from './LocalDocumentIndex';
+import { EmbeddingsModel, EmbeddingsResponse } from './types';
+import { LocalFileStorage } from './storage/LocalFileStorage';
+
+// Stub embeddings model that returns deterministic vectors
+class StubEmbeddings implements EmbeddingsModel {
+    public readonly maxTokens = 8000;
+    public async createEmbeddings(inputs: string | string[]): Promise<EmbeddingsResponse> {
+        const texts = Array.isArray(inputs) ? inputs : [inputs];
+        const output = texts.map(() => {
+            const vec = new Array(384).fill(0);
+            vec[0] = 1; // unit vector
+            return vec;
+        });
+        return { status: 'success', output };
+    }
+}
+
+describe('FolderWatcher', () => {
+    let tmpDir: string;
+    let indexDir: string;
+    let watchDir: string;
+    let index: LocalDocumentIndex;
+    let sandbox: sinon.SinonSandbox;
+
+    beforeEach(async () => {
+        sandbox = sinon.createSandbox();
+        tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'vectra-watch-'));
+        indexDir = path.join(tmpDir, 'index');
+        watchDir = path.join(tmpDir, 'watch');
+        await fs.promises.mkdir(indexDir, { recursive: true });
+        await fs.promises.mkdir(watchDir, { recursive: true });
+
+        index = new LocalDocumentIndex({
+            folderPath: indexDir,
+            embeddings: new StubEmbeddings(),
+            storage: new LocalFileStorage(),
+        });
+        await index.createIndex({ version: 1, deleteIfExists: true });
+    });
+
+    afterEach(async () => {
+        sandbox.restore();
+        await fs.promises.rm(tmpDir, { recursive: true, force: true });
+    });
+
+    it('should perform initial sync of existing files', async () => {
+        // Create files before starting watcher
+        await fs.promises.writeFile(path.join(watchDir, 'file1.txt'), 'hello world');
+        await fs.promises.writeFile(path.join(watchDir, 'file2.txt'), 'goodbye world');
+
+        const watcher = new FolderWatcher({ index, paths: [watchDir] });
+        const synced: string[] = [];
+        watcher.on('sync', (uri: string) => synced.push(uri));
+
+        await watcher.start();
+        try {
+            assert.equal(watcher.trackedFileCount, 2);
+            assert.equal(synced.length, 2);
+            // Verify documents exist in index
+            const id1 = await index.getDocumentId(path.join(watchDir, 'file1.txt'));
+            const id2 = await index.getDocumentId(path.join(watchDir, 'file2.txt'));
+            assert.ok(id1, 'file1.txt should be indexed');
+            assert.ok(id2, 'file2.txt should be indexed');
+        } finally {
+            await watcher.stop();
+        }
+    });
+
+    it('should emit ready event after initial sync', async () => {
+        const watcher = new FolderWatcher({ index, paths: [watchDir] });
+        let ready = false;
+        watcher.on('ready', () => { ready = true; });
+
+        await watcher.start();
+        try {
+            assert.equal(ready, true);
+        } finally {
+            await watcher.stop();
+        }
+    });
+
+    it('should filter by extensions', async () => {
+        await fs.promises.writeFile(path.join(watchDir, 'include.txt'), 'included');
+        await fs.promises.writeFile(path.join(watchDir, 'exclude.js'), 'excluded');
+        await fs.promises.writeFile(path.join(watchDir, 'include.md'), 'also included');
+
+        const watcher = new FolderWatcher({
+            index,
+            paths: [watchDir],
+            extensions: ['.txt', '.md']
+        });
+
+        await watcher.start();
+        try {
+            assert.equal(watcher.trackedFileCount, 2);
+            const idTxt = await index.getDocumentId(path.join(watchDir, 'include.txt'));
+            const idMd = await index.getDocumentId(path.join(watchDir, 'include.md'));
+            const idJs = await index.getDocumentId(path.join(watchDir, 'exclude.js'));
+            assert.ok(idTxt, 'include.txt should be indexed');
+            assert.ok(idMd, 'include.md should be indexed');
+            assert.equal(idJs, undefined, 'exclude.js should not be indexed');
+        } finally {
+            await watcher.stop();
+        }
+    });
+
+    it('should handle extensions without leading dot', async () => {
+        await fs.promises.writeFile(path.join(watchDir, 'test.txt'), 'hello');
+
+        const watcher = new FolderWatcher({
+            index,
+            paths: [watchDir],
+            extensions: ['txt']
+        });
+
+        await watcher.start();
+        try {
+            assert.equal(watcher.trackedFileCount, 1);
+        } finally {
+            await watcher.stop();
+        }
+    });
+
+    it('should recurse into subdirectories', async () => {
+        const subDir = path.join(watchDir, 'sub');
+        await fs.promises.mkdir(subDir, { recursive: true });
+        await fs.promises.writeFile(path.join(watchDir, 'root.txt'), 'root');
+        await fs.promises.writeFile(path.join(subDir, 'nested.txt'), 'nested');
+
+        const watcher = new FolderWatcher({ index, paths: [watchDir] });
+
+        await watcher.start();
+        try {
+            assert.equal(watcher.trackedFileCount, 2);
+            const idRoot = await index.getDocumentId(path.join(watchDir, 'root.txt'));
+            const idNested = await index.getDocumentId(path.join(subDir, 'nested.txt'));
+            assert.ok(idRoot, 'root.txt should be indexed');
+            assert.ok(idNested, 'nested.txt should be indexed');
+        } finally {
+            await watcher.stop();
+        }
+    });
+
+    it('should watch individual files', async () => {
+        const singleFile = path.join(tmpDir, 'single.txt');
+        await fs.promises.writeFile(singleFile, 'single file');
+
+        const watcher = new FolderWatcher({ index, paths: [singleFile] });
+
+        await watcher.start();
+        try {
+            assert.equal(watcher.trackedFileCount, 1);
+            const id = await index.getDocumentId(singleFile);
+            assert.ok(id, 'single.txt should be indexed');
+        } finally {
+            await watcher.stop();
+        }
+    });
+
+    it('should watch multiple paths', async () => {
+        const dir2 = path.join(tmpDir, 'watch2');
+        await fs.promises.mkdir(dir2, { recursive: true });
+        await fs.promises.writeFile(path.join(watchDir, 'a.txt'), 'a');
+        await fs.promises.writeFile(path.join(dir2, 'b.txt'), 'b');
+
+        const watcher = new FolderWatcher({ index, paths: [watchDir, dir2] });
+
+        await watcher.start();
+        try {
+            assert.equal(watcher.trackedFileCount, 2);
+        } finally {
+            await watcher.stop();
+        }
+    });
+
+    it('should handle sync() detecting deleted files', async () => {
+        await fs.promises.writeFile(path.join(watchDir, 'ephemeral.txt'), 'temporary');
+
+        const watcher = new FolderWatcher({ index, paths: [watchDir] });
+        await watcher.start();
+        assert.equal(watcher.trackedFileCount, 1);
+
+        // Delete the file
+        await fs.promises.unlink(path.join(watchDir, 'ephemeral.txt'));
+
+        // Manual sync should detect deletion
+        const synced: Array<{ uri: string; action: string }> = [];
+        watcher.on('sync', (uri: string, action: string) => synced.push({ uri, action }));
+        await watcher.sync();
+
+        try {
+            assert.equal(watcher.trackedFileCount, 0);
+            const deleted = synced.find(s => s.action === 'deleted');
+            assert.ok(deleted, 'should have emitted a delete event');
+        } finally {
+            await watcher.stop();
+        }
+    });
+
+    it('should handle sync() detecting updated files', async () => {
+        const filePath = path.join(watchDir, 'mutable.txt');
+        await fs.promises.writeFile(filePath, 'version 1');
+
+        const watcher = new FolderWatcher({ index, paths: [watchDir] });
+        await watcher.start();
+        assert.equal(watcher.trackedFileCount, 1);
+
+        // Update the file (ensure mtime changes)
+        await new Promise(r => setTimeout(r, 50));
+        await fs.promises.writeFile(filePath, 'version 2');
+
+        // Manual sync should detect update
+        const synced: Array<{ uri: string; action: string }> = [];
+        watcher.on('sync', (uri: string, action: string) => synced.push({ uri, action }));
+        await watcher.sync();
+
+        try {
+            assert.equal(watcher.trackedFileCount, 1);
+            const updated = synced.find(s => s.action === 'updated');
+            assert.ok(updated, 'should have emitted an update event');
+        } finally {
+            await watcher.stop();
+        }
+    });
+
+    it('should not throw if path does not exist', async () => {
+        const watcher = new FolderWatcher({
+            index,
+            paths: [path.join(tmpDir, 'nonexistent')]
+        });
+
+        await watcher.start();
+        try {
+            assert.equal(watcher.trackedFileCount, 0);
+        } finally {
+            await watcher.stop();
+        }
+    });
+
+    it('should throw if started twice', async () => {
+        const watcher = new FolderWatcher({ index, paths: [watchDir] });
+        await watcher.start();
+        try {
+            await assert.rejects(() => watcher.start(), /already running/);
+        } finally {
+            await watcher.stop();
+        }
+    });
+
+    it('should report isRunning correctly', async () => {
+        const watcher = new FolderWatcher({ index, paths: [watchDir] });
+        assert.equal(watcher.isRunning, false);
+        await watcher.start();
+        assert.equal(watcher.isRunning, true);
+        await watcher.stop();
+        assert.equal(watcher.isRunning, false);
+    });
+
+    it('should emit error events for sync failures', async () => {
+        await fs.promises.writeFile(path.join(watchDir, 'bad.txt'), 'content');
+
+        // Create watcher with no embeddings to force error
+        const badIndex = new LocalDocumentIndex({
+            folderPath: indexDir,
+            // no embeddings — will throw on upsertDocument
+            storage: new LocalFileStorage(),
+        });
+
+        const watcher = new FolderWatcher({ index: badIndex, paths: [watchDir] });
+        const errors: Array<{ err: Error; uri: string }> = [];
+        watcher.on('error', (err: Error, uri: string) => errors.push({ err, uri }));
+
+        await watcher.start();
+        try {
+            assert.equal(errors.length, 1);
+            assert.ok(errors[0].err.message.includes('Embeddings model not configured'));
+        } finally {
+            await watcher.stop();
+        }
+    });
+});

package/src/FolderWatcher.ts

@@ -0,0 +1,304 @@
+import { EventEmitter } from 'events';
+import fs from 'node:fs';
+import * as path from 'path';
+import { LocalDocumentIndex } from './LocalDocumentIndex';
+
+/**
+ * Configuration for FolderWatcher.
+ */
+export interface FolderWatcherConfig {
+    /**
+     * The LocalDocumentIndex to sync files into.
+     */
+    index: LocalDocumentIndex;
+
+    /**
+     * List of folder or file paths to watch.
+     */
+    paths: string[];
+
+    /**
+     * Optional. File extensions to include (e.g., ['.txt', '.md', '.html']).
+     * @remarks
+     * If not specified, all files are included.
+     */
+    extensions?: string[];
+
+    /**
+     * Optional. Debounce interval in milliseconds for file change events.
+     * @remarks
+     * Default is 500ms. Multiple rapid changes to the same file are collapsed into one sync.
+     */
+    debounceMs?: number;
+}
+
+/**
+ * Events emitted by FolderWatcher.
+ *
+ * - `sync` — emitted after a file is synced. Args: `(uri: string, action: 'added' | 'updated' | 'deleted')`
+ * - `error` — emitted when a sync operation fails. Args: `(error: Error, uri: string)`
+ * - `ready` — emitted after the initial sync completes.
+ */
+export interface FolderWatcherEvents {
+    sync: [uri: string, action: 'added' | 'updated' | 'deleted'];
+    error: [error: Error, uri: string];
+    ready: [];
+}
+
+interface TrackedFile {
+    uri: string;
+    mtimeMs: number;
+}
+
+/**
+ * Watches folders for file changes and automatically syncs them into a LocalDocumentIndex.
+ *
+ * @remarks
+ * Uses Node.js `fs.watch` for efficient filesystem monitoring with debouncing.
+ * Performs an initial full sync on start, then watches for incremental changes.
+ */
+export class FolderWatcher extends EventEmitter {
+    private readonly _index: LocalDocumentIndex;
+    private readonly _paths: string[];
+    private readonly _extensions?: Set<string>;
+    private readonly _debounceMs: number;
+    private readonly _tracked: Map<string, TrackedFile> = new Map();
+    private readonly _pending: Map<string, NodeJS.Timeout> = new Map();
+    private readonly _watchers: fs.FSWatcher[] = [];
+    private _running: boolean = false;
+
+    /**
+     * Creates a new FolderWatcher instance.
+     * @param config Configuration for the watcher.
+     */
+    public constructor(config: FolderWatcherConfig) {
+        super();
+        this._index = config.index;
+        this._paths = config.paths.map(p => path.resolve(p));
+        this._extensions = config.extensions
+            ? new Set(config.extensions.map(e => e.startsWith('.') ? e.toLowerCase() : `.${e.toLowerCase()}`))
+            : undefined;
+        this._debounceMs = config.debounceMs ?? 500;
+    }
+
+    /**
+     * Returns true if the watcher is currently running.
+     */
+    public get isRunning(): boolean {
+        return this._running;
+    }
+
+    /**
+     * Returns the number of tracked files.
+     */
+    public get trackedFileCount(): number {
+        return this._tracked.size;
+    }
+
+    /**
+     * Starts the watcher: performs an initial sync and then watches for changes.
+     */
+    public async start(): Promise<void> {
+        if (this._running) {
+            throw new Error('FolderWatcher is already running');
+        }
+        this._running = true;
+
+        // Initial sync
+        await this._initialSync();
+        this.emit('ready');
+
+        // Set up watchers
+        for (const watchPath of this._paths) {
+            try {
+                const stat = await fs.promises.stat(watchPath);
+                if (stat.isDirectory()) {
+                    this._watchDirectory(watchPath);
+                } else if (stat.isFile()) {
+                    this._watchFile(watchPath);
+                }
+            } catch {
+                // Path doesn't exist — skip
+            }
+        }
+    }
+
+    /**
+     * Stops the watcher and cleans up all resources.
+     */
+    public async stop(): Promise<void> {
+        this._running = false;
+
+        // Close all watchers
+        for (const watcher of this._watchers) {
+            watcher.close();
+        }
+        this._watchers.length = 0;
+
+        // Clear pending debounced operations
+        for (const timeout of this._pending.values()) {
+            clearTimeout(timeout);
+        }
+        this._pending.clear();
+    }
+
+    /**
+     * Performs a full sync: scans all watched paths and upserts/deletes as needed.
+     * @returns Number of files synced (added + updated + deleted).
+     */
+    public async sync(): Promise<number> {
+        let count = 0;
+
+        // Collect current files on disk
+        const currentFiles = new Map<string, number>();
+        for (const watchPath of this._paths) {
+            await this._collectFiles(watchPath, currentFiles);
+        }
+
+        // Upsert new or changed files
+        for (const [filePath, mtimeMs] of currentFiles) {
+            const tracked = this._tracked.get(filePath);
+            if (!tracked || tracked.mtimeMs < mtimeMs) {
+                const ok = await this._syncFile(filePath, mtimeMs);
+                if (ok) count++;
+            }
+        }
+
+        // Delete files that no longer exist
+        for (const [filePath, tracked] of this._tracked) {
+            if (!currentFiles.has(filePath)) {
+                const ok = await this._deleteFile(tracked.uri);
+                if (ok) count++;
+            }
+        }
+
+        return count;
+    }
+
+    // --- Private methods ---
+
+    private async _initialSync(): Promise<void> {
+        await this.sync();
+    }
+
+    private _shouldInclude(filePath: string): boolean {
+        if (!this._extensions) return true;
+        const ext = path.extname(filePath).toLowerCase();
+        return this._extensions.has(ext);
+    }
+
+    private async _collectFiles(dirOrFile: string, out: Map<string, number>): Promise<void> {
+        let stat: fs.Stats;
+        try {
+            stat = await fs.promises.stat(dirOrFile);
+        } catch {
+            return;
+        }
+
+        if (stat.isFile()) {
+            if (this._shouldInclude(dirOrFile)) {
+                out.set(dirOrFile, stat.mtimeMs);
+            }
+        } else if (stat.isDirectory()) {
+            const entries = await fs.promises.readdir(dirOrFile);
+            for (const entry of entries) {
+                await this._collectFiles(path.join(dirOrFile, entry), out);
+            }
+        }
+    }
+
+    private async _syncFile(filePath: string, mtimeMs: number): Promise<boolean> {
+        const wasTracked = this._tracked.has(filePath);
+        const action = wasTracked ? 'updated' : 'added';
+        try {
+            const text = await fs.promises.readFile(filePath, 'utf-8');
+            const ext = path.extname(filePath);
+            const docType = ext ? ext.slice(1).toLowerCase() : undefined;
+            await this._index.upsertDocument(filePath, text, docType);
+            this._tracked.set(filePath, { uri: filePath, mtimeMs });
+            this.emit('sync', filePath, action);
+            return true;
+        } catch (err: unknown) {
+            this.emit('error', err instanceof Error ? err : new Error(String(err)), filePath);
+            return false;
+        }
+    }
+
+    private async _deleteFile(uri: string): Promise<boolean> {
+        try {
+            await this._index.deleteDocument(uri);
+            // Find and remove from tracked by URI
+            for (const [filePath, tracked] of this._tracked) {
+                if (tracked.uri === uri) {
+                    this._tracked.delete(filePath);
+                    break;
+                }
+            }
+            this.emit('sync', uri, 'deleted');
+            return true;
+        } catch (err: unknown) {
+            this.emit('error', err instanceof Error ? err : new Error(String(err)), uri);
+            return false;
+        }
+    }
+
+    private _watchDirectory(dirPath: string): void {
+        try {
+            const watcher = fs.watch(dirPath, { recursive: true }, (eventType, filename) => {
+                if (!this._running || !filename) return;
+                const fullPath = path.join(dirPath, filename);
+                if (this._shouldInclude(fullPath)) {
+                    this._debouncedSync(fullPath);
+                }
+            });
+            watcher.on('error', (err) => {
+                this.emit('error', err, dirPath);
+            });
+            this._watchers.push(watcher);
+        } catch (err: unknown) {
+            this.emit('error', err instanceof Error ? err : new Error(String(err)), dirPath);
+        }
+    }
+
+    private _watchFile(filePath: string): void {
+        try {
+            const watcher = fs.watch(filePath, (eventType) => {
+                if (!this._running) return;
+                this._debouncedSync(filePath);
+            });
+            watcher.on('error', (err) => {
+                this.emit('error', err, filePath);
+            });
+            this._watchers.push(watcher);
+        } catch (err: unknown) {
+            this.emit('error', err instanceof Error ? err : new Error(String(err)), filePath);
+        }
+    }
+
+    private _debouncedSync(filePath: string): void {
+        // Cancel any pending sync for this file
+        const existing = this._pending.get(filePath);
+        if (existing) {
+            clearTimeout(existing);
+        }
+
+        const timeout = setTimeout(async () => {
+            this._pending.delete(filePath);
+            if (!this._running) return;
+
+            try {
+                const stat = await fs.promises.stat(filePath);
+                if (stat.isFile()) {
+                    await this._syncFile(filePath, stat.mtimeMs);
+                }
+            } catch {
+                // File was deleted
+                if (this._tracked.has(filePath)) {
+                    await this._deleteFile(filePath);
+                }
+            }
+        }, this._debounceMs);
+
+        this._pending.set(filePath, timeout);
+    }
+}

package/src/GPT3Tokenizer.spec.ts

@@ -0,0 +1,50 @@
+import { strict as assert } from 'node:assert';
+import { describe, it } from 'mocha';
+import { GPT3Tokenizer } from '../src/GPT3Tokenizer';
+
+describe('GPT3Tokenizer', () => {
+  const tokenizer = new GPT3Tokenizer();
+
+  it('encodes empty string to [] and decodes [] to empty string', () => {
+    const tokens = tokenizer.encode('');
+    assert.deepEqual(tokens, [], 'encode("") should return []');
+
+    const text = tokenizer.decode([]);
+    assert.equal(text, '', 'decode([]) should return empty string');
+  });
+
+  it('round-trips various strings including unicode and punctuation', () => {
+    const samples = [
+      'Hello, world!',
+      'Café 😊 こんにちは 𠜎𠜱𠝹𠱓',
+      'Newlines\nand\ttabs with   multiple   spaces.',
+      '--- *** ===='
+    ];
+
+    for (const s of samples) {
+      const tokens = tokenizer.encode(s);
+      const decoded = tokenizer.decode(tokens);
+      assert.equal(decoded, s, `decode(encode(s)) should equal s for: ${JSON.stringify(s)}`);
+
+      // Validate token array shape: array of non-negative integers
+      assert.ok(Array.isArray(tokens), 'encode should return an array');
+      for (const t of tokens) {
+        assert.equal(typeof t, 'number', 'each token should be a number');
+        assert.ok(Number.isInteger(t), 'each token should be an integer');
+        assert.ok(t >= 0, 'each token should be non-negative');
+      }
+
+      // Encoding should be stable across calls for the same input
+      const tokens2 = tokenizer.encode(s);
+      assert.deepEqual(tokens2, tokens, 'encode should be deterministic for the same input');
+    }
+  });
+
+  it('produces non-empty tokens for typical non-empty input', () => {
+    const s = 'This is a simple test.';
+    const tokens = tokenizer.encode(s);
+    assert.ok(tokens.length > 0, 'expected some tokens for non-empty input');
+    const decoded = tokenizer.decode(tokens);
+    assert.equal(decoded, s, 'decoded text should match original input');
+  });
+});