vectra 0.12.2 → 0.14.0

package/src/codecs/migrateIndex.ts

@@ -0,0 +1,125 @@
+import { pathUtils as path } from '../utils/pathUtils';
+import { FileStorage, LocalFileStorage } from '../storage';
+import { IndexCodec } from './IndexCodec';
+import { JsonCodec } from './JsonCodec';
+import { ProtobufCodec } from './ProtobufCodec';
+
+export type FormatName = 'json' | 'protobuf';
+
+function codecForFormat(format: FormatName): IndexCodec {
+    return format === 'protobuf' ? new ProtobufCodec() : new JsonCodec();
+}
+
+/**
+ * Detects the codec in use for an existing index folder.
+ * @remarks
+ * Checks for `index.json` and `index.pb` files. If both exist, throws an
+ * error directing the user to re-run migration. If neither exists, throws.
+ * @param folderPath Path to the index folder.
+ * @param storage Storage backend to use.
+ * @returns The detected codec.
+ */
+export async function detectCodec(folderPath: string, storage: FileStorage): Promise<IndexCodec> {
+    const hasJson = await storage.pathExists(path.join(folderPath, 'index.json'));
+    const hasPb = await storage.pathExists(path.join(folderPath, 'index.pb'));
+
+    if (hasJson && hasPb) {
+        throw new Error(
+            'Both index.json and index.pb found — the index may be in a partially migrated state. ' +
+            'Run `vectra migrate` or `migrateIndex()` to complete the migration.'
+        );
+    }
+    if (hasPb) return new ProtobufCodec();
+    if (hasJson) return new JsonCodec();
+    throw new Error('No index file found (expected index.json or index.pb).');
+}
+
+export interface MigrateOptions {
+    to: FormatName;
+    storage?: FileStorage;
+}
+
+/**
+ * Migrates an index folder from one serialization format to another.
+ * @param folderPath Path to the index folder.
+ * @param options Migration options.
+ */
+export async function migrateIndex(folderPath: string, options: MigrateOptions): Promise<void> {
+    const storage = options.storage || new LocalFileStorage();
+    const targetCodec = codecForFormat(options.to);
+
+    // Detect current format
+    const sourceCodec = await detectCodec(folderPath, storage);
+
+    if (sourceCodec.extension === targetCodec.extension) {
+        return; // Already in target format
+    }
+
+    // --- Migrate index file ---
+    const sourceIndexPath = path.join(folderPath, `index${sourceCodec.extension}`);
+    const targetIndexPath = path.join(folderPath, `index${targetCodec.extension}`);
+    const indexBuffer = await storage.readFile(sourceIndexPath);
+    const indexData = sourceCodec.deserializeIndex(indexBuffer);
+
+    // Rewrite external metadata file references
+    for (const item of indexData.items) {
+        if (item.metadataFile && item.metadataFile.endsWith(sourceCodec.extension)) {
+            const baseName = item.metadataFile.slice(0, -sourceCodec.extension.length);
+            const oldMetaPath = path.join(folderPath, item.metadataFile);
+            const newMetaFile = `${baseName}${targetCodec.extension}`;
+            const newMetaPath = path.join(folderPath, newMetaFile);
+
+            // Read, re-serialize, write
+            const metaBuf = await storage.readFile(oldMetaPath);
+            const metadata = sourceCodec.deserializeMetadata(metaBuf);
+            await storage.upsertFile(newMetaPath, targetCodec.serializeMetadata(metadata));
+
+            // Update reference
+            item.metadataFile = newMetaFile;
+        }
+    }
+
+    // Write new index
+    await storage.upsertFile(targetIndexPath, targetCodec.serializeIndex(indexData));
+
+    // --- Migrate catalog if present ---
+    const sourceCatalogPath = path.join(folderPath, `catalog${sourceCodec.extension}`);
+    const targetCatalogPath = path.join(folderPath, `catalog${targetCodec.extension}`);
+    if (await storage.pathExists(sourceCatalogPath)) {
+        const catalogBuffer = await storage.readFile(sourceCatalogPath);
+        const catalog = sourceCodec.deserializeCatalog(catalogBuffer);
+
+        // Migrate per-document metadata files
+        for (const docId of Object.values(catalog.idToUri ? catalog.idToUri : {})) {
+            // Document metadata uses documentId as filename base
+        }
+        // Actually the document IDs are the keys of idToUri
+        for (const docId of Object.keys(catalog.idToUri)) {
+            const oldDocMetaPath = path.join(folderPath, `${docId}${sourceCodec.extension}`);
+            if (await storage.pathExists(oldDocMetaPath)) {
+                const docMetaBuf = await storage.readFile(oldDocMetaPath);
+                const docMeta = sourceCodec.deserializeMetadata(docMetaBuf);
+                const newDocMetaPath = path.join(folderPath, `${docId}${targetCodec.extension}`);
+                await storage.upsertFile(newDocMetaPath, targetCodec.serializeMetadata(docMeta));
+                await storage.deleteFile(oldDocMetaPath);
+            }
+        }
+
+        await storage.upsertFile(targetCatalogPath, targetCodec.serializeCatalog(catalog));
+        await storage.deleteFile(sourceCatalogPath);
+    }
+
+    // --- Clean up old files (after new files are safely written) ---
+    // Delete old external metadata files
+    for (const item of sourceCodec.deserializeIndex(indexBuffer).items) {
+        if (item.metadataFile) {
+            const oldMetaPath = path.join(folderPath, item.metadataFile);
+            if (await storage.pathExists(oldMetaPath)) {
+                await storage.deleteFile(oldMetaPath);
+            }
+        }
+    }
+
+    // Delete old index file last
+    await storage.deleteFile(sourceIndexPath);
+}

package/src/codecs/schemas/index.proto

@@ -0,0 +1,34 @@
+syntax = "proto3";
+
+message IndexData {
+  uint32 version = 1;
+  MetadataConfig metadata_config = 2;
+  repeated IndexItem items = 3;
+}
+
+message MetadataConfig {
+  repeated string indexed = 1;
+}
+
+message IndexItem {
+  string id = 1;
+  map<string, MetadataValue> metadata = 2;
+  repeated float vector = 3 [packed = true];
+  double norm = 4;
+  optional string metadata_file = 5;
+}
+
+message MetadataValue {
+  oneof value {
+    string string_value = 1;
+    double number_value = 2;
+    bool bool_value = 3;
+  }
+}
+
+message DocumentCatalog {
+  uint32 version = 1;
+  uint32 count = 2;
+  map<string, string> uri_to_id = 3;
+  map<string, string> id_to_uri = 4;
+}

package/src/index.ts

@@ -1,11 +1,19 @@
+export * from './codecs';
+export * from './storage';
 export * from './FileFetcher';
+export * from './FolderWatcher';
 export * from './GPT3Tokenizer';
 export * from './ItemSelector';
+export * from './LocalEmbeddings';
 export * from './LocalIndex';
 export * from './LocalDocument';
 export * from './LocalDocumentIndex';
 export * from './LocalDocumentResult';
-export * from './OpenAIEmbeddings';
 export * from './TextSplitter';
+export * from './OpenAIEmbeddings';
+export * from './TransformersEmbeddings';
+export * from './TransformersTokenizer';
 export * from './types';
 export * from './WebFetcher';
+export * from './server';
+export * from './utils/pathUtils';

package/src/internals/Colorize.ts

@@ -1,4 +1,5 @@
-const colorizer = require('json-colorizer');
+const colorizerModule = require('json-colorizer');
+const colorizeFn = colorizerModule?.default ?? colorizerModule;
 
 /**
  * @private
@@ -20,20 +21,23 @@ export class Colorize {
         if (typeof output === 'string') {
             return isBm25 ? `\x1b[34m${quote}${output}${quote}\x1b[0m` : `\x1b[32m${quote}${output}${quote}\x1b[0m`;
         } else if (typeof output === 'object' && output !== null) {
-            return colorizer(output, {
-                pretty: true,
-                colors: {
-                    BRACE: 'white',
-                    BRACKET: 'white',
-                    COLON: 'white',
-                    COMMA: 'white',
-                    STRING_KEY: 'white',
-                    STRING_LITERAL: 'green',
-                    NUMBER_LITERAL: 'blue',
-                    BOOLEAN_LITERAL: 'blue',
-                    NULL_LITERAL: 'blue'
-                }
-            });
+            if (typeof colorizeFn === 'function') {
+                return colorizeFn(output, {
+                    pretty: true,
+                    colors: {
+                        BRACE: 'white',
+                        BRACKET: 'white',
+                        COLON: 'white',
+                        COMMA: 'white',
+                        STRING_KEY: 'white',
+                        STRING_LITERAL: 'green',
+                        NUMBER_LITERAL: 'blue',
+                        BOOLEAN_LITERAL: 'blue',
+                        NULL_LITERAL: 'blue'
+                    }
+                });
+            }
+            return JSON.stringify(output, null, 2);
         } else if (typeof output == 'number') {
             return `\x1b[34m${output}${units}\x1b[0m`;
         } else {
@@ -59,6 +63,5 @@ export class Colorize {
 
     public static warning(warning: string): string {
         return `\x1b[33m${warning}\x1b[0m`;
-
     }
 }

package/src/server/IndexManager.ts

@@ -0,0 +1,243 @@
+import * as path from 'path';
+import * as fs from 'fs';
+import { LocalIndex, CreateIndexConfig } from '../LocalIndex';
+import { LocalDocumentIndex, LocalDocumentIndexConfig } from '../LocalDocumentIndex';
+import { LocalFileStorage } from '../storage/LocalFileStorage';
+import { FileStorage } from '../storage';
+import { EmbeddingsModel, MetadataTypes } from '../types';
+import { detectCodec, IndexCodec, JsonCodec, ProtobufCodec } from '../codecs';
+
+export interface ManagedIndex {
+    name: string;
+    index: LocalIndex | LocalDocumentIndex;
+    isDocumentIndex: boolean;
+    format: string;
+}
+
+export interface IndexManagerConfig {
+    /** Single index path (mutually exclusive with rootDir). */
+    indexPath?: string;
+    /** Root directory containing multiple index subdirectories. */
+    rootDir?: string;
+    /** Embeddings model for server-side embedding computation. */
+    embeddings?: EmbeddingsModel;
+    /** Polling interval in ms for auto-detecting new indexes (default: 3000). */
+    scanInterval?: number;
+}
+
+/**
+ * Manages loaded indexes for the gRPC server.
+ * Supports single-index and multi-index modes with auto-detection of new indexes.
+ */
+export class IndexManager {
+    private readonly _config: IndexManagerConfig;
+    private readonly _indexes: Map<string, ManagedIndex> = new Map();
+    private _scanTimer?: ReturnType<typeof setInterval>;
+    private _singleMode: boolean;
+
+    constructor(config: IndexManagerConfig) {
+        this._config = config;
+        this._singleMode = !!config.indexPath;
+    }
+
+    /** Returns all currently loaded indexes. */
+    public get indexes(): Map<string, ManagedIndex> {
+        return this._indexes;
+    }
+
+    /** Returns true if running in single-index mode. */
+    public get isSingleMode(): boolean {
+        return this._singleMode;
+    }
+
+    /**
+     * Initializes the index manager: loads existing indexes and starts auto-detection.
+     */
+    public async initialize(): Promise<void> {
+        if (this._singleMode && this._config.indexPath) {
+            await this.loadSingleIndex(this._config.indexPath);
+        } else if (this._config.rootDir) {
+            await this.scanRootDir();
+            // Start periodic scanning for new indexes
+            const interval = this._config.scanInterval ?? 3000;
+            this._scanTimer = setInterval(() => {
+                this.scanRootDir().catch(() => {/* ignore scan errors */});
+            }, interval);
+        }
+    }
+
+    /**
+     * Shuts down the index manager: stops scanning and flushes all indexes.
+     */
+    public async shutdown(): Promise<void> {
+        if (this._scanTimer) {
+            clearInterval(this._scanTimer);
+            this._scanTimer = undefined;
+        }
+        // No special flush needed — LocalIndex writes on endUpdate()
+        this._indexes.clear();
+    }
+
+    /**
+     * Gets a managed index by name.
+     * In single-index mode, any name (or empty string) returns the single index.
+     */
+    public getIndex(name: string): ManagedIndex | undefined {
+        if (this._singleMode) {
+            // Return the single loaded index regardless of the name provided
+            const entries = Array.from(this._indexes.values());
+            return entries[0];
+        }
+        return this._indexes.get(name);
+    }
+
+    /**
+     * Gets a managed index, throwing NOT_FOUND-appropriate error if missing.
+     */
+    public requireIndex(name: string): ManagedIndex {
+        const managed = this.getIndex(name);
+        if (!managed) {
+            throw new Error(`Index not found: ${name}`);
+        }
+        return managed;
+    }
+
+    /**
+     * Gets a managed document index, throwing if missing or not a document index.
+     */
+    public requireDocumentIndex(name: string): { managed: ManagedIndex; docIndex: LocalDocumentIndex } {
+        const managed = this.requireIndex(name);
+        if (!managed.isDocumentIndex) {
+            throw new Error(`Index "${name}" is not a document index`);
+        }
+        return { managed, docIndex: managed.index as LocalDocumentIndex };
+    }
+
+    /**
+     * Creates a new index on disk and loads it.
+     */
+    public async createIndex(
+        name: string,
+        format: string,
+        isDocumentIndex: boolean,
+        documentConfig?: { version?: number; chunkSize?: number; chunkOverlap?: number }
+    ): Promise<ManagedIndex> {
+        if (this._indexes.has(name)) {
+            throw new Error(`Index already exists: ${name}`);
+        }
+
+        const rootDir = this._config.rootDir;
+        if (!rootDir && !this._singleMode) {
+            throw new Error('Cannot create index: no root directory configured');
+        }
+
+        const folderPath = rootDir ? path.join(rootDir, name) : name;
+        const storage = new LocalFileStorage();
+        const codec = format === 'protobuf' ? new ProtobufCodec() : new JsonCodec();
+
+        let index: LocalIndex | LocalDocumentIndex;
+        if (isDocumentIndex) {
+            const config: LocalDocumentIndexConfig = {
+                folderPath,
+                storage,
+                codec,
+                embeddings: this._config.embeddings,
+                chunkingConfig: {
+                    chunkSize: documentConfig?.chunkSize || 512,
+                    chunkOverlap: documentConfig?.chunkOverlap || 0,
+                },
+            };
+            index = new LocalDocumentIndex(config);
+        } else {
+            index = new LocalIndex(folderPath, undefined, storage, codec);
+        }
+
+        const createConfig: CreateIndexConfig = {
+            version: documentConfig?.version || 1,
+            deleteIfExists: false,
+        };
+        await index.createIndex(createConfig);
+
+        const managed: ManagedIndex = {
+            name,
+            index,
+            isDocumentIndex,
+            format: format || 'json',
+        };
+        this._indexes.set(name, managed);
+        return managed;
+    }
+
+    /**
+     * Deletes an index from disk and unloads it.
+     */
+    public async deleteIndex(name: string): Promise<void> {
+        const managed = this.requireIndex(name);
+        await managed.index.deleteIndex();
+        this._indexes.delete(name);
+    }
+
+    /**
+     * Lists all loaded indexes.
+     */
+    public listIndexes(): ManagedIndex[] {
+        return Array.from(this._indexes.values());
+    }
+
+    private async loadSingleIndex(indexPath: string): Promise<void> {
+        const name = path.basename(indexPath);
+        await this.loadIndex(name, indexPath);
+    }
+
+    private async scanRootDir(): Promise<void> {
+        const rootDir = this._config.rootDir!;
+        if (!fs.existsSync(rootDir)) return;
+
+        const entries = fs.readdirSync(rootDir, { withFileTypes: true });
+        for (const entry of entries) {
+            if (!entry.isDirectory()) continue;
+            if (entry.name.startsWith('.')) continue;
+            if (this._indexes.has(entry.name)) continue;
+
+            const folderPath = path.join(rootDir, entry.name);
+            try {
+                await this.loadIndex(entry.name, folderPath);
+            } catch {
+                // Skip directories that aren't valid indexes
+            }
+        }
+    }
+
+    private async loadIndex(name: string, folderPath: string): Promise<void> {
+        const storage = new LocalFileStorage();
+
+        // Detect codec — throws if no index file found
+        const codec = await detectCodec(folderPath, storage);
+        const format = codec.extension === '.pb' ? 'protobuf' : 'json';
+
+        // Detect if it's a document index (has catalog file)
+        const hasCatalogJson = await storage.pathExists(path.join(folderPath, 'catalog.json'));
+        const hasCatalogPb = await storage.pathExists(path.join(folderPath, 'catalog.pb'));
+        const isDocumentIndex = hasCatalogJson || hasCatalogPb;
+
+        let index: LocalIndex | LocalDocumentIndex;
+        if (isDocumentIndex) {
+            index = new LocalDocumentIndex({
+                folderPath,
+                storage,
+                codec,
+                embeddings: this._config.embeddings,
+            });
+        } else {
+            index = new LocalIndex(folderPath, undefined, storage, codec);
+        }
+
+        const managed: ManagedIndex = {
+            name,
+            index,
+            isDocumentIndex,
+            format,
+        };
+        this._indexes.set(name, managed);
+    }
+}