verso-db 0.1.4 → 0.2.0

package/dist/Collection.js

@@ -0,0 +1,1186 @@
+import { HNSWIndex } from './HNSWIndex.js';
+import { DimensionMismatchError, DuplicateVectorError, VectorDBError } from './errors.js';
+const EMPTY_METADATA = Object.freeze({});
+/** Supported metadata filter operators — hoisted to avoid per-call allocation */
+const SUPPORTED_OPERATORS = new Set(['$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin']);
+const COLLECTION_STATE_VERSION = 1;
+export class Collection {
+    static MAX_INTERNAL_ID = 0xFFFFFFFE;
+    name;
+    dimension;
+    metric;
+    M; // HNSW M parameter
+    efConstruction; // HNSW efConstruction parameter
+    storage;
+    stateKey; // Storage key for atomic state pointer
+    defaultIndexKey;
+    defaultMetaKey;
+    defaultDeletedKey;
+    indexKey; // Storage key for HNSW index data
+    metaKey; // Storage key for metadata
+    deletedKey; // Storage key for deleted IDs
+    hnsw;
+    idMap; // Maps string IDs to internal numeric IDs
+    idReverseMap; // Maps internal numeric IDs to string IDs
+    metadata; // Stores metadata for each vector
+    deletedIds; // Tombstone markers for deleted vectors
+    nextNumericId; // Monotonic counter for generating unique internal IDs
+    activeCount; // O(1) counter for non-deleted vectors
+    autoPersist;
+    dirty;
+    mutationLock;
+    constructor(name, config, storage, options = {}) {
+        this.name = name;
+        this.dimension = config.dimension;
+        this.metric = config.metric ?? 'cosine';
+        this.M = config.M ?? 16;
+        this.efConstruction = config.efConstruction ?? 200;
+        this.storage = storage;
+        this.defaultIndexKey = `${name}/${name}.hnsw`;
+        this.defaultMetaKey = `${name}/${name}.meta`;
+        this.defaultDeletedKey = `${name}/${name}.deleted`;
+        this.stateKey = `${name}/${name}.state`;
+        this.indexKey = this.defaultIndexKey;
+        this.metaKey = this.defaultMetaKey;
+        this.deletedKey = this.defaultDeletedKey;
+        // Initialize HNSW index with the specified parameters
+        this.hnsw = new HNSWIndex(config.dimension, this.metric, this.M, this.efConstruction);
+        this.idMap = new Map();
+        this.idReverseMap = new Map();
+        this.metadata = new Map();
+        this.deletedIds = new Set();
+        this.nextNumericId = 0;
+        this.activeCount = 0;
+        this.autoPersist = options.autoPersist ?? false;
+        this.dirty = false;
+        this.mutationLock = Promise.resolve();
+    }
+    async init() {
+        // Load existing data if files exist
+        await this.loadFromDisk();
+    }
+    async withMutationLock(fn) {
+        const previous = this.mutationLock;
+        let release;
+        this.mutationLock = new Promise((resolve) => {
+            release = resolve;
+        });
+        await previous;
+        try {
+            return await fn();
+        }
+        finally {
+            release();
+        }
+    }
+    async flush() {
+        await this.withMutationLock(async () => {
+            if (!this.dirty)
+                return;
+            await this.saveToDiskUnlocked();
+        });
+    }
+    captureMutationSnapshot() {
+        return {
+            hnswBuffer: this.hnsw.serialize(),
+            idMap: new Map(this.idMap),
+            idReverseMap: new Map(this.idReverseMap),
+            metadata: new Map(this.metadata),
+            deletedIds: new Set(this.deletedIds),
+            nextNumericId: this.nextNumericId,
+            activeCount: this.activeCount,
+            indexKey: this.indexKey,
+            metaKey: this.metaKey,
+            deletedKey: this.deletedKey,
+            dirty: this.dirty,
+        };
+    }
+    restoreMutationSnapshot(snapshot) {
+        const restoredHnsw = HNSWIndex.deserialize(snapshot.hnswBuffer.slice(0));
+        if (this.hnsw && typeof this.hnsw.destroy === 'function') {
+            this.hnsw.destroy();
+        }
+        this.hnsw = restoredHnsw;
+        this.idMap = new Map(snapshot.idMap);
+        this.idReverseMap = new Map(snapshot.idReverseMap);
+        this.metadata = new Map(snapshot.metadata);
+        this.deletedIds = new Set(snapshot.deletedIds);
+        this.nextNumericId = snapshot.nextNumericId;
+        this.activeCount = snapshot.activeCount;
+        this.setActiveStorageKeys(snapshot.indexKey, snapshot.metaKey, snapshot.deletedKey);
+        this.dirty = snapshot.dirty;
+    }
+    /**
+     * Wrap a mutation in snapshot-capture + auto-persist.
+     * The mutation fn should set `this.dirty = true` when it mutates state.
+     * If autoPersist is on and dirty was set, saves after fn completes.
+     * On error, restores from snapshot.
+     */
+    async withAutoSave(fn) {
+        const snapshot = this.autoPersist ? this.captureMutationSnapshot() : null;
+        try {
+            const result = await fn();
+            if (this.dirty && this.autoPersist) {
+                await this.saveToDiskUnlocked();
+            }
+            return result;
+        }
+        catch (err) {
+            if (snapshot) {
+                this.restoreMutationSnapshot(snapshot);
+            }
+            throw err;
+        }
+    }
+    reserveNumericIds(count) {
+        if (!Number.isInteger(count) || count < 0) {
+            throw new VectorDBError(`Invalid ID reservation count: ${count}`, 'VALIDATION_ERROR');
+        }
+        if (count === 0)
+            return this.nextNumericId;
+        const startId = this.nextNumericId;
+        const endId = startId + count - 1;
+        if (!Number.isSafeInteger(endId) || endId > Collection.MAX_INTERNAL_ID) {
+            throw new VectorDBError(`Collection '${this.name}' cannot allocate ${count} internal IDs: exhausted ID space (max ${Collection.MAX_INTERNAL_ID})`, 'VALIDATION_ERROR');
+        }
+        this.nextNumericId = endId + 1;
+        return startId;
+    }
+    setActiveStorageKeys(indexKey, metaKey, deletedKey) {
+        this.indexKey = indexKey;
+        this.metaKey = metaKey;
+        this.deletedKey = deletedKey;
+    }
+    isValidStorageState(value) {
+        if (!value || typeof value !== 'object')
+            return false;
+        const state = value;
+        return (state.version === COLLECTION_STATE_VERSION &&
+            typeof state.indexKey === 'string' &&
+            typeof state.metaKey === 'string' &&
+            typeof state.deletedKey === 'string');
+    }
+    async loadStorageState() {
+        const stateBuffer = await this.storage.read(this.stateKey);
+        if (!stateBuffer)
+            return null;
+        try {
+            const raw = JSON.parse(new TextDecoder().decode(stateBuffer));
+            if (this.isValidStorageState(raw)) {
+                return raw;
+            }
+        }
+        catch {
+            // Ignore malformed state file and fall back to legacy keys.
+        }
+        return null;
+    }
+    getVersionedDataKeys(saveId) {
+        return {
+            indexKey: `${this.name}/${this.name}.${saveId}.hnsw`,
+            metaKey: `${this.name}/${this.name}.${saveId}.meta`,
+            deletedKey: `${this.name}/${this.name}.${saveId}.deleted`,
+        };
+    }
+    isVersionedDataKey(key) {
+        if (key === this.defaultIndexKey ||
+            key === this.defaultMetaKey ||
+            key === this.defaultDeletedKey) {
+            return false;
+        }
+        const prefix = `${this.name}/${this.name}.`;
+        return key.startsWith(prefix) && (key.endsWith('.hnsw') ||
+            key.endsWith('.meta') ||
+            key.endsWith('.deleted'));
+    }
+    validateLoadedIndexConfig(index) {
+        if (index.getDimension() !== this.dimension ||
+            index.getMetric() !== this.metric ||
+            index.getM() !== this.M ||
+            index.getEfConstruction() !== this.efConstruction) {
+            throw new VectorDBError(`Collection '${this.name}' index parameters do not match manifest configuration`, 'CORRUPT_COLLECTION');
+        }
+    }
+    async loadFromDisk() {
+        // Reset in-memory state before loading.
+        this.idMap.clear();
+        this.idReverseMap.clear();
+        this.metadata.clear();
+        this.deletedIds.clear();
+        this.nextNumericId = 0;
+        this.activeCount = 0;
+        const persistedState = await this.loadStorageState();
+        if (persistedState) {
+            this.setActiveStorageKeys(persistedState.indexKey, persistedState.metaKey, persistedState.deletedKey);
+        }
+        else {
+            this.setActiveStorageKeys(this.defaultIndexKey, this.defaultMetaKey, this.defaultDeletedKey);
+        }
+        // Load the binary HNSW index via StorageBackend
+        const indexData = await this.storage.read(this.indexKey);
+        if (indexData) {
+            try {
+                const loaded = HNSWIndex.deserialize(indexData);
+                this.validateLoadedIndexConfig(loaded);
+                this.hnsw = loaded;
+            }
+            catch (e) {
+                throw new VectorDBError(`Collection '${this.name}' index file '${this.indexKey}' is corrupt or unreadable: ${e instanceof Error ? e.message : String(e)}`, 'CORRUPT_COLLECTION');
+            }
+        }
+        // Load metadata via StorageBackend
+        const metaData = await this.storage.read(this.metaKey);
+        if (metaData) {
+            try {
+                const metaContent = new TextDecoder().decode(metaData);
+                const lines = metaContent.split('\n');
+                for (let i = 0; i < lines.length; i++) {
+                    const line = lines[i];
+                    if (line.length === 0 || line.trim().length === 0)
+                        continue;
+                    const parsed = this.parseMetadataLine(line);
+                    if (!parsed) {
+                        throw new VectorDBError(`Collection '${this.name}' metadata file '${this.metaKey}' is corrupt at line ${i + 1}`, 'CORRUPT_COLLECTION');
+                    }
+                    const { id, internalId, metadata } = parsed;
+                    this.idMap.set(id, internalId);
+                    this.idReverseMap.set(internalId, id);
+                    if (metadata !== undefined) {
+                        this.metadata.set(internalId, metadata);
+                    }
+                }
+            }
+            catch (e) {
+                if (e instanceof VectorDBError) {
+                    throw e;
+                }
+                throw new VectorDBError(`Collection '${this.name}' metadata file '${this.metaKey}' is corrupt or unreadable: ${e instanceof Error ? e.message : String(e)}`, 'CORRUPT_COLLECTION');
+            }
+        }
+        // Load deleted IDs via StorageBackend
+        const deletedData = await this.storage.read(this.deletedKey);
+        if (deletedData) {
+            try {
+                const deletedContent = new TextDecoder().decode(deletedData);
+                const deletedArray = JSON.parse(deletedContent);
+                if (!Array.isArray(deletedArray)) {
+                    throw new VectorDBError('Deleted-ID payload is not an array', 'CORRUPT_COLLECTION');
+                }
+                for (let i = 0; i < deletedArray.length; i++) {
+                    const numericId = deletedArray[i];
+                    if (!Number.isInteger(numericId) ||
+                        numericId < 0 ||
+                        numericId > Collection.MAX_INTERNAL_ID) {
+                        throw new VectorDBError(`Invalid deleted internal ID at index ${i}: ${numericId}`, 'CORRUPT_COLLECTION');
+                    }
+                }
+                this.deletedIds = new Set(deletedArray);
+            }
+            catch (e) {
+                throw new VectorDBError(`Collection '${this.name}' deleted-ID file '${this.deletedKey}' is corrupt or unreadable: ${e instanceof Error ? e.message : String(e)}`, 'CORRUPT_COLLECTION');
+            }
+        }
+        this.reconcileLoadedState();
+        this.dirty = false;
+    }
+    async add(config) {
+        return this.addInternal(config, false);
+    }
+    async query(config) {
+        return this.executeQuery(config, 'Query vector');
+    }
+    /**
+     * Batch query for multiple vectors at once.
+     * Shares query semantics with query(), including adaptive candidate expansion
+     * for filters and tombstones.
+     *
+     * @param configs Array of query configurations
+     * @returns Array of query results, one per query
+     */
+    async queryBatch(configs) {
+        if (!Array.isArray(configs)) {
+            throw new VectorDBError('Query batch must be an array of query configs', 'VALIDATION_ERROR');
+        }
+        if (configs.length === 0)
+            return [];
+        for (let i = 0; i < configs.length; i++) {
+            if (!this.isPlainObject(configs[i])) {
+                throw new VectorDBError(`Query ${i} must be a plain object`, 'VALIDATION_ERROR');
+            }
+        }
+        const results = new Array(configs.length);
+        for (let i = 0; i < configs.length; i++) {
+            results[i] = this.executeQuery(configs[i], `Query ${i}`);
+        }
+        return results;
+    }
+    /**
+     * Brute-force KNN search for validation and correctness checking
+     * This checks all vectors and returns the true k nearest neighbors
+     */
+    async queryBruteForce(config) {
+        this.validateQueryInput(config, 'Query vector');
+        const { queryVector, k, filter } = config;
+        // Compute distances to all vectors, applying filter during scan
+        const queryFloat = queryVector instanceof Float32Array ? queryVector : new Float32Array(queryVector);
+        const hasFilter = this.hasFilter(filter);
+        const distances = [];
+        for (const [numericId, vector] of this.hnsw.getAllVectors()) {
+            // Skip deleted vectors
+            if (this.deletedIds.has(numericId))
+                continue;
+            // Apply filter before distance computation when possible
+            if (hasFilter) {
+                const meta = this.metadata.get(numericId) || EMPTY_METADATA;
+                if (!this.matchesFilter(meta, filter))
+                    continue;
+            }
+            const distance = this.hnsw.calculateDistance(queryFloat, vector);
+            distances.push({ id: numericId, distance });
+        }
+        // Sort by distance and take top k
+        distances.sort((a, b) => {
+            const diff = a.distance - b.distance;
+            return diff !== 0 ? diff : a.id - b.id;
+        });
+        const filteredResults = distances.slice(0, k);
+        // Prepare the output
+        const ids = [];
+        const distancesOut = [];
+        const metadata = [];
+        for (const result of filteredResults) {
+            const id = this.idReverseMap.get(result.id);
+            if (id) {
+                ids.push(id);
+                distancesOut.push(result.distance);
+                metadata.push(this.metadata.get(result.id) || EMPTY_METADATA);
+            }
+        }
+        return {
+            ids,
+            distances: distancesOut,
+            metadata
+        };
+    }
+    /**
+     * Upsert vectors (insert or update).
+     *
+     * For existing IDs, the old vectors are tombstone-deleted and new vectors
+     * are inserted. Use compact() periodically to reclaim space from
+     * tombstoned vectors.
+     *
+     * For new IDs, behaves identically to add().
+     */
+    async upsert(config) {
+        return this.addInternal(config, true);
+    }
+    async addInternal(config, isUpsert) {
+        await this.withMutationLock(async () => {
+            this.validateAddInput(config, isUpsert);
+            if (config.ids.length === 0)
+                return;
+            await this.withAutoSave(async () => {
+                const points = new Array(config.ids.length);
+                const startId = this.reserveNumericIds(config.ids.length);
+                for (let i = 0; i < config.ids.length; i++) {
+                    points[i] = { id: startId + i, vector: new Float32Array(config.vectors[i]) };
+                }
+                await this.hnsw.addPointsBulk(points);
+                for (let i = 0; i < config.ids.length; i++) {
+                    const id = config.ids[i];
+                    const metadata = config.metadata ? config.metadata[i] : undefined;
+                    if (isUpsert) {
+                        const replacementId = this.idMap.get(id);
+                        if (replacementId !== undefined) {
+                            if (!this.deletedIds.has(replacementId)) {
+                                this.activeCount--;
+                            }
+                            this.deletedIds.add(replacementId);
+                            this.idReverseMap.delete(replacementId);
+                            this.metadata.delete(replacementId);
+                        }
+                    }
+                    const numericId = startId + i;
+                    this.idMap.set(id, numericId);
+                    this.idReverseMap.set(numericId, id);
+                    if (metadata !== undefined) {
+                        this.metadata.set(numericId, metadata);
+                    }
+                    this.activeCount++;
+                }
+                this.dirty = true;
+            });
+        });
+    }
+    /** Returns the vector dimension for this collection. */
+    getDimension() { return this.dimension; }
+    /** Returns the distance metric for this collection. */
+    getMetric() { return this.metric; }
+    /** Returns the HNSW M parameter for this collection. */
+    getM() { return this.M; }
+    /** Returns the HNSW efConstruction parameter for this collection. */
+    getEfConstruction() { return this.efConstruction; }
+    /**
+     * Returns the number of active (non-deleted) vectors in the collection.
+     * O(1) via maintained counter.
+     */
+    count() {
+        return this.activeCount;
+    }
+    /**
+     * Returns the total number of tracked vectors including deleted (tombstoned) ones.
+     * Use this to determine when compaction might be beneficial.
+     */
+    countWithDeleted() {
+        return this.activeCount + this.deletedIds.size;
+    }
+    /**
+     * Returns the number of deleted (tombstoned) vectors awaiting compaction.
+     */
+    deletedCount() {
+        return this.deletedIds.size;
+    }
+    /**
+     * Mark a vector as deleted (tombstone deletion).
+     * The vector remains in the index but is excluded from search results.
+     * Use compact() to permanently remove deleted vectors and reclaim space.
+     *
+     * @param id The string ID of the vector to delete
+     * @returns true if the vector was deleted, false if it didn't exist or was already deleted
+     */
+    deleteInternal(id) {
+        const numericId = this.idMap.get(id);
+        if (numericId === undefined)
+            return false;
+        if (this.deletedIds.has(numericId))
+            return false;
+        this.deletedIds.add(numericId);
+        this.activeCount--;
+        return true;
+    }
+    async delete(id) {
+        return this.withMutationLock(async () => {
+            if (!this.has(id))
+                return false;
+            return this.withAutoSave(async () => {
+                const deleted = this.deleteInternal(id);
+                if (!deleted)
+                    return false;
+                this.dirty = true;
+                return true;
+            });
+        });
+    }
+    /**
+     * Mark multiple vectors as deleted (tombstone deletion).
+     *
+     * @param ids Array of string IDs to delete
+     * @returns Number of vectors that were successfully deleted
+     */
+    async deleteBatch(ids) {
+        return this.withMutationLock(async () => {
+            return this.withAutoSave(async () => {
+                let deleted = 0;
+                for (const id of ids) {
+                    if (this.deleteInternal(id))
+                        deleted++;
+                }
+                if (deleted === 0)
+                    return 0;
+                this.dirty = true;
+                return deleted;
+            });
+        });
+    }
+    /**
+     * Check if a vector exists and is not deleted.
+     */
+    has(id) {
+        const numericId = this.idMap.get(id);
+        if (numericId === undefined)
+            return false;
+        return !this.deletedIds.has(numericId);
+    }
+    /**
+     * Check if a vector was deleted (tombstoned).
+     */
+    isDeleted(id) {
+        const numericId = this.idMap.get(id);
+        if (numericId === undefined)
+            return false;
+        return this.deletedIds.has(numericId);
+    }
+    /**
+     * Reorder the internal HNSW index for BFS cache locality.
+     * Remaps all internal IDs so that graph neighbors are stored
+     * contiguously in memory, improving search cache hit rates.
+     */
+    async reorderIndex() {
+        await this.withMutationLock(async () => {
+            await this.withAutoSave(async () => {
+                const oldToNew = this.hnsw.reorderForLocality();
+                if (oldToNew.size === 0)
+                    return;
+                // Remap idMap and idReverseMap
+                const newIdMap = new Map();
+                const newIdReverseMap = new Map();
+                for (const [strId, oldNum] of this.idMap) {
+                    const newNum = oldToNew.get(oldNum);
+                    if (newNum !== undefined) {
+                        newIdMap.set(strId, newNum);
+                        newIdReverseMap.set(newNum, strId);
+                    }
+                }
+                this.idMap = newIdMap;
+                this.idReverseMap = newIdReverseMap;
+                // Remap metadata
+                const newMetadata = new Map();
+                for (const [oldNum, meta] of this.metadata) {
+                    const newNum = oldToNew.get(oldNum);
+                    if (newNum !== undefined) {
+                        newMetadata.set(newNum, meta);
+                    }
+                }
+                this.metadata = newMetadata;
+                // Remap deletedIds
+                const newDeletedIds = new Set();
+                for (const oldNum of this.deletedIds) {
+                    const newNum = oldToNew.get(oldNum);
+                    if (newNum !== undefined) {
+                        newDeletedIds.add(newNum);
+                    }
+                }
+                this.deletedIds = newDeletedIds;
+                this.dirty = true;
+            });
+        });
+    }
+    async saveToDisk() {
+        await this.withMutationLock(async () => {
+            await this.saveToDiskUnlocked();
+        });
+    }
+    async saveToDiskUnlocked() {
+        // Ensure the collection directory exists
+        await this.storage.mkdir(this.name);
+        const saveId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
+        const nextKeys = this.getVersionedDataKeys(saveId);
+        const previousKeys = {
+            indexKey: this.indexKey,
+            metaKey: this.metaKey,
+            deletedKey: this.deletedKey,
+        };
+        // Prepare all data before writing
+        const indexBytes = new Uint8Array(this.hnsw.serialize());
+        const metaLines = [];
+        for (const [numericId, id] of this.idReverseMap) {
+            const meta = this.metadata.get(numericId);
+            metaLines.push(JSON.stringify({
+                id,
+                numericId,
+                metadata: meta ?? {},
+            }));
+        }
+        const metaBytes = new TextEncoder().encode(metaLines.join('\n'));
+        const deletedBytes = new TextEncoder().encode(JSON.stringify([...this.deletedIds]));
+        // Data writes in parallel
+        await Promise.all([
+            this.storage.write(nextKeys.indexKey, indexBytes),
+            this.storage.write(nextKeys.metaKey, metaBytes),
+            this.storage.write(nextKeys.deletedKey, deletedBytes),
+        ]);
+        // Commit pointer last. Crash before this write keeps previous committed state.
+        const state = {
+            version: COLLECTION_STATE_VERSION,
+            indexKey: nextKeys.indexKey,
+            metaKey: nextKeys.metaKey,
+            deletedKey: nextKeys.deletedKey,
+        };
+        await this.storage.write(this.stateKey, new TextEncoder().encode(JSON.stringify(state)));
+        this.setActiveStorageKeys(nextKeys.indexKey, nextKeys.metaKey, nextKeys.deletedKey);
+        // Shadow copies in parallel, non-fatal
+        await Promise.all([
+            this.storage.write(this.defaultIndexKey, indexBytes).catch(() => { }),
+            this.storage.write(this.defaultMetaKey, metaBytes).catch(() => { }),
+            this.storage.write(this.defaultDeletedKey, deletedBytes).catch(() => { }),
+        ]);
+        // Best-effort cleanup: only delete old versioned files. Legacy fixed keys are
+        // intentionally left untouched for backward compatibility.
+        const staleKeys = [previousKeys.indexKey, previousKeys.metaKey, previousKeys.deletedKey];
+        for (const key of staleKeys) {
+            if (!this.isVersionedDataKey(key))
+                continue;
+            if (key === this.indexKey || key === this.metaKey || key === this.deletedKey)
+                continue;
+            await this.storage.delete(key).catch(() => { });
+        }
+        this.dirty = false;
+    }
+    executeQuery(config, context) {
+        this.validateQueryInput(config, context);
+        const queryVector = config.queryVector instanceof Float32Array
+            ? config.queryVector
+            : new Float32Array(config.queryVector);
+        const filter = this.hasFilter(config.filter) ? config.filter : undefined;
+        const efSearch = config.efSearch ?? Math.max(config.k * 2, 50);
+        // Determine whether to use quantized search:
+        // - Explicit true/false from config takes precedence
+        // - Otherwise auto-detect: use quantized when quantization is enabled
+        const useQuantized = config.useQuantizedSearch ?? this.hnsw.isQuantizationEnabled();
+        const candidateMultiplier = config.candidateMultiplier ?? 3;
+        const candidates = this.searchCandidates(queryVector, config.k, efSearch, filter, useQuantized, candidateMultiplier);
+        return this.materializeResults(candidates, config.k);
+    }
+    searchCandidates(queryVector, k, efSearch, filter, useQuantized = false, candidateMultiplier = 3) {
+        const totalCandidates = this.countWithDeleted();
+        if (totalCandidates === 0)
+            return [];
+        let requestK = Math.min(Math.max(k * 2, 50), totalCandidates);
+        let requestEf = Math.max(efSearch, requestK);
+        let filtered = [];
+        while (true) {
+            const rawResults = useQuantized
+                ? this.hnsw.searchKNNQuantized(queryVector, requestK, candidateMultiplier, requestEf)
+                : this.hnsw.searchKNN(queryVector, requestK, requestEf);
+            filtered = this.filterAndDeduplicateResults(rawResults, filter);
+            if (filtered.length >= k || requestK >= totalCandidates) {
+                return filtered;
+            }
+            const nextK = Math.min(totalCandidates, requestK * 2);
+            if (nextK === requestK) {
+                return filtered;
+            }
+            requestK = nextK;
+            requestEf = Math.max(requestEf, requestK);
+        }
+    }
+    filterAndDeduplicateResults(results, filter) {
+        const filtered = [];
+        const hasFilter = !!filter;
+        for (const result of results) {
+            if (this.deletedIds.has(result.id)) {
+                continue;
+            }
+            if (hasFilter) {
+                const metadata = this.metadata.get(result.id) || EMPTY_METADATA;
+                if (!this.matchesFilter(metadata, filter)) {
+                    continue;
+                }
+            }
+            filtered.push(result);
+        }
+        filtered.sort((a, b) => {
+            const diff = a.distance - b.distance;
+            return diff !== 0 ? diff : a.id - b.id;
+        });
+        return filtered;
+    }
+    materializeResults(candidates, k) {
+        const resultCount = Math.min(candidates.length, k);
+        const ids = new Array(resultCount);
+        const distances = new Array(resultCount);
+        const metadata = new Array(resultCount);
+        let outIdx = 0;
+        for (let i = 0; i < resultCount; i++) {
+            const candidate = candidates[i];
+            const id = this.idReverseMap.get(candidate.id);
+            if (!id)
+                continue;
+            ids[outIdx] = id;
+            distances[outIdx] = candidate.distance;
+            metadata[outIdx] = this.metadata.get(candidate.id) || EMPTY_METADATA;
+            outIdx++;
+        }
+        if (outIdx < resultCount) {
+            ids.length = outIdx;
+            distances.length = outIdx;
+            metadata.length = outIdx;
+        }
+        return { ids, distances, metadata };
+    }
+    validateQueryInput(config, context) {
+        if (!this.isPlainObject(config)) {
+            throw new VectorDBError(`${context} config must be a plain object`, 'VALIDATION_ERROR');
+        }
+        const queryVector = config.queryVector;
+        if (!(Array.isArray(queryVector) || queryVector instanceof Float32Array)) {
+            throw new VectorDBError(`${context} must include a queryVector (number[] or Float32Array)`, 'VALIDATION_ERROR');
+        }
+        if (!Number.isInteger(config.k) || config.k <= 0) {
+            throw new VectorDBError(`k must be a positive integer, got ${config.k}`, 'VALIDATION_ERROR');
+        }
+        if (config.queryVector.length !== this.dimension) {
+            throw new DimensionMismatchError(this.dimension, config.queryVector.length, context);
+        }
+        for (let i = 0; i < config.queryVector.length; i++) {
+            if (!Number.isFinite(config.queryVector[i])) {
+                throw new VectorDBError(`${context} contains non-finite value at dimension ${i}: ${config.queryVector[i]}`, 'VALIDATION_ERROR');
+            }
+        }
+        if (config.efSearch !== undefined) {
+            if (!Number.isInteger(config.efSearch) || config.efSearch <= 0) {
+                throw new VectorDBError(`efSearch must be a positive integer, got ${config.efSearch}`, 'VALIDATION_ERROR');
+            }
+        }
+        if (config.filter !== undefined) {
+            if (!this.isFilterObject(config.filter)) {
+                throw new VectorDBError('Filter must be a plain object', 'VALIDATION_ERROR');
+            }
+            this.validateFilterOperators(config.filter);
+        }
+    }
+    hasFilter(filter) {
+        return !!filter && Object.keys(filter).length > 0;
+    }
+    reconcileLoadedState() {
+        const indexVectors = this.hnsw.getAllVectors();
+        const existingIds = new Set();
+        let maxNumericId = -1;
+        for (const [id] of indexVectors) {
+            existingIds.add(id);
+            if (id > maxNumericId)
+                maxNumericId = id;
+        }
+        // Collect keys to delete (can't delete during iteration of same Map)
+        const staleIds = [];
+        for (const [id, numericId] of this.idMap) {
+            if (!existingIds.has(numericId)) {
+                staleIds.push(id);
+            }
+        }
+        for (const id of staleIds) {
+            this.idMap.delete(id);
+        }
+        this.idReverseMap.clear();
+        for (const [id, numericId] of this.idMap.entries()) {
+            if (this.idReverseMap.has(numericId)) {
+                throw new VectorDBError(`Collection '${this.name}' metadata is corrupt: duplicate numeric ID ${numericId}`, 'CORRUPT_COLLECTION');
+            }
+            this.idReverseMap.set(numericId, id);
+        }
+        const staleDeletedIds = [];
+        for (const numericId of this.deletedIds) {
+            if (!existingIds.has(numericId)) {
+                staleDeletedIds.push(numericId);
+            }
+        }
+        for (const id of staleDeletedIds) {
+            this.deletedIds.delete(id);
+        }
+        const staleMetaIds = [];
+        for (const numericId of this.metadata.keys()) {
+            if (!this.idReverseMap.has(numericId)) {
+                staleMetaIds.push(numericId);
+            }
+        }
+        for (const id of staleMetaIds) {
+            this.metadata.delete(id);
+        }
+        const missingMappings = [];
+        for (const numericId of existingIds) {
+            if (this.deletedIds.has(numericId))
+                continue;
+            if (!this.idReverseMap.has(numericId)) {
+                missingMappings.push(numericId);
+            }
+        }
+        if (missingMappings.length > 0) {
+            throw new VectorDBError(`Collection '${this.name}' metadata is inconsistent with index: missing ${missingMappings.length} active ID mapping(s)`, 'CORRUPT_COLLECTION');
+        }
+        if (maxNumericId > Collection.MAX_INTERNAL_ID) {
+            throw new VectorDBError(`Collection '${this.name}' contains unsupported internal ID ${maxNumericId} (max ${Collection.MAX_INTERNAL_ID})`, 'CORRUPT_COLLECTION');
+        }
+        this.nextNumericId = maxNumericId + 1;
+        this.activeCount = 0;
+        for (const numericId of this.idMap.values()) {
+            if (!this.deletedIds.has(numericId)) {
+                this.activeCount++;
+            }
+        }
+    }
+    matchesFilter(metadata, filter) {
+        for (const key of Object.keys(filter)) {
+            const value = filter[key];
+            const metaValue = Object.prototype.hasOwnProperty.call(metadata, key)
+                ? metadata[key]
+                : undefined;
+            if (this.isOperatorObject(value)) {
+                // Operator validity already checked by validateFilterOperators() in validateQueryInput()
+                if (metaValue === undefined && value.$ne === undefined && value.$nin === undefined) {
+                    return false;
+                }
+                if (value.$eq !== undefined && !this.deepEqual(metaValue, value.$eq))
+                    return false;
+                if (value.$ne !== undefined && this.deepEqual(metaValue, value.$ne))
+                    return false;
+                // Comparison operators must reject undefined/missing fields explicitly,
+                // because JS comparisons like `undefined > 5` return false (not NaN/error),
+                // causing `!(undefined > 5)` to incorrectly pass.
+                if (value.$gt !== undefined && (metaValue === undefined || !(metaValue > value.$gt)))
+                    return false;
+                if (value.$lt !== undefined && (metaValue === undefined || !(metaValue < value.$lt)))
+                    return false;
+                if (value.$gte !== undefined && (metaValue === undefined || !(metaValue >= value.$gte)))
+                    return false;
+                if (value.$lte !== undefined && (metaValue === undefined || !(metaValue <= value.$lte)))
+                    return false;
+                if (value.$in !== undefined) {
+                    if (!Array.isArray(value.$in) || !this.matchesInOperator(metaValue, value.$in))
+                        return false;
+                }
+                if (value.$nin !== undefined) {
+                    if (!Array.isArray(value.$nin))
+                        return false;
+                    if (this.matchesInOperator(metaValue, value.$nin))
+                        return false;
+                }
+            }
+            else {
+                if (!this.deepEqual(metaValue, value))
+                    return false;
+            }
+        }
+        return true;
+    }
+    validateFilterOperators(filter) {
+        for (const key of Object.keys(filter)) {
+            const value = filter[key];
+            if (!this.isOperatorObject(value))
+                continue;
+            for (const operator of Object.keys(value)) {
+                if (!SUPPORTED_OPERATORS.has(operator)) {
+                    throw new VectorDBError(`Unsupported filter operator '${operator}'`, 'VALIDATION_ERROR');
+                }
+            }
+        }
+    }
+    validateAddInput(config, allowExistingIds) {
+        if (!this.isPlainObject(config)) {
+            throw new VectorDBError('Add config must be a plain object', 'VALIDATION_ERROR');
+        }
+        if (!Array.isArray(config.ids)) {
+            throw new VectorDBError('ids must be an array of strings', 'VALIDATION_ERROR');
+        }
+        if (!Array.isArray(config.vectors)) {
+            throw new VectorDBError('vectors must be an array of number[] or Float32Array', 'VALIDATION_ERROR');
+        }
+        if (config.vectors.length !== config.ids.length) {
+            throw new VectorDBError('Number of vectors must match number of IDs', 'VALIDATION_ERROR');
+        }
+        if (config.metadata !== undefined && !Array.isArray(config.metadata)) {
+            throw new VectorDBError('metadata must be an array when provided', 'VALIDATION_ERROR');
+        }
+        if (config.metadata && config.metadata.length !== config.ids.length) {
+            throw new VectorDBError('Number of metadata entries must match number of IDs', 'VALIDATION_ERROR');
+        }
+        if (config.metadata) {
+            for (let i = 0; i < config.metadata.length; i++) {
+                const meta = config.metadata[i];
+                if (meta === undefined)
+                    continue;
+                if (!this.isPlainObject(meta)) {
+                    throw new VectorDBError(`Metadata at index ${i} must be a plain object`, 'VALIDATION_ERROR');
+                }
+                this.validateMetadataValue(meta, `metadata[${i}]`, new Set());
+            }
+        }
+        const seenIds = new Set();
+        const duplicateIds = new Set();
+        for (let i = 0; i < config.ids.length; i++) {
+            const id = config.ids[i];
+            if (typeof id !== 'string') {
+                throw new VectorDBError(`ID at index ${i} must be a string`, 'VALIDATION_ERROR');
+            }
+            if (seenIds.has(id)) {
+                duplicateIds.add(id);
+            }
+            else {
+                seenIds.add(id);
+            }
+            if (!allowExistingIds && this.idMap.has(id)) {
+                duplicateIds.add(id);
+            }
+        }
+        if (duplicateIds.size > 0) {
+            throw new DuplicateVectorError([...duplicateIds]);
+        }
+        for (let i = 0; i < config.vectors.length; i++) {
+            const vector = config.vectors[i];
+            if (!(Array.isArray(vector) || vector instanceof Float32Array)) {
+                throw new VectorDBError(`Vector at index ${i} must be a number[] or Float32Array`, 'VALIDATION_ERROR');
+            }
+            if (vector.length !== this.dimension) {
+                throw new DimensionMismatchError(this.dimension, vector.length, `Vector at index ${i}`);
+            }
+            for (let d = 0; d < vector.length; d++) {
+                if (!Number.isFinite(vector[d])) {
+                    throw new VectorDBError(`Vector at index ${i} contains non-finite value at dimension ${d}: ${vector[d]}`, 'VALIDATION_ERROR');
+                }
+            }
+        }
+    }
+    validateMetadataValue(value, path, seen) {
+        if (value === null)
+            return;
+        const valueType = typeof value;
+        if (valueType === 'string' || valueType === 'boolean')
+            return;
+        if (valueType === 'number') {
+            if (!Number.isFinite(value)) {
+                throw new VectorDBError(`${path} contains non-finite number ${value}`, 'VALIDATION_ERROR');
+            }
+            return;
+        }
+        if (valueType !== 'object') {
+            throw new VectorDBError(`${path} must contain only JSON-serializable values`, 'VALIDATION_ERROR');
+        }
+        const objectValue = value;
+        if (seen.has(objectValue)) {
+            throw new VectorDBError(`${path} contains a circular reference`, 'VALIDATION_ERROR');
+        }
+        seen.add(objectValue);
+        try {
+            if (Array.isArray(value)) {
+                for (let i = 0; i < value.length; i++) {
+                    if (value[i] === undefined) {
+                        throw new VectorDBError(`${path}[${i}] is undefined and cannot be persisted`, 'VALIDATION_ERROR');
+                    }
+                    this.validateMetadataValue(value[i], `${path}[${i}]`, seen);
+                }
+                return;
+            }
+            if (!this.isPlainObject(value)) {
+                throw new VectorDBError(`${path} must contain only plain JSON objects`, 'VALIDATION_ERROR');
+            }
+            for (const [key, nestedValue] of Object.entries(value)) {
+                if (nestedValue === undefined) {
+                    throw new VectorDBError(`${path}.${key} is undefined and cannot be persisted`, 'VALIDATION_ERROR');
+                }
+                this.validateMetadataValue(nestedValue, `${path}.${key}`, seen);
+            }
+        }
+        finally {
+            seen.delete(objectValue);
+        }
+    }
+    parseMetadataLine(line) {
+        const trimmed = line.trim();
+        if (trimmed.length === 0)
+            return null;
+        // v2+ JSONL format
+        if (trimmed[0] === '{') {
+            const record = JSON.parse(trimmed);
+            if (typeof record.id !== 'string'
+                || !Number.isInteger(record.numericId)
+                || record.numericId < 0) {
+                return null;
+            }
+            if (record.metadata !== undefined && !this.isPlainObject(record.metadata)) {
+                return null;
+            }
+            return {
+                id: record.id,
+                internalId: record.numericId,
+                metadata: record.metadata === undefined ? undefined : record.metadata,
+            };
+        }
+        // Legacy tab-separated format
+        const parts = line.split('\t');
+        if (parts.length < 2)
+            return null;
+        const id = parts[0];
+        const internalId = parseInt(parts[1], 10);
+        if (!Number.isInteger(internalId) || internalId < 0)
+            return null;
+        let metadata;
+        if (parts.length > 2) {
+            const parsedMetadata = JSON.parse(parts.slice(2).join('\t'));
+            if (!this.isPlainObject(parsedMetadata))
+                return null;
+            metadata = parsedMetadata;
+        }
+        return { id, internalId, metadata };
+    }
+    isPlainObject(value) {
+        if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+            return false;
+        }
+        const prototype = Object.getPrototypeOf(value);
+        return prototype === Object.prototype || prototype === null;
+    }
+    isFilterObject(value) {
+        return typeof value === 'object' && value !== null && !Array.isArray(value);
+    }
+    isOperatorObject(value) {
+        if (!this.isPlainObject(value))
+            return false;
+        const keys = Object.keys(value);
+        if (keys.length === 0)
+            return false;
+        for (const key of keys) {
+            if (!key.startsWith('$'))
+                return false;
+        }
+        return true;
+    }
+    deepEqual(a, b) {
+        if (Object.is(a, b))
+            return true;
+        if (Array.isArray(a) && Array.isArray(b)) {
+            if (a.length !== b.length)
+                return false;
+            for (let i = 0; i < a.length; i++) {
+                if (!this.deepEqual(a[i], b[i]))
+                    return false;
+            }
+            return true;
+        }
+        if (this.isPlainObject(a) && this.isPlainObject(b)) {
+            const keysA = Object.keys(a);
+            const keysB = Object.keys(b);
+            if (keysA.length !== keysB.length)
+                return false;
+            for (const key of keysA) {
+                if (!Object.prototype.hasOwnProperty.call(b, key))
+                    return false;
+                if (!this.deepEqual(a[key], b[key]))
+                    return false;
+            }
+            return true;
+        }
+        return false;
+    }
+    matchesInOperator(metaValue, candidates) {
+        if (Array.isArray(metaValue)) {
+            for (const candidate of candidates) {
+                for (const item of metaValue) {
+                    if (this.deepEqual(item, candidate))
+                        return true;
+                }
+            }
+            return false;
+        }
+        for (const candidate of candidates) {
+            if (this.deepEqual(metaValue, candidate))
+                return true;
+        }
+        return false;
+    }
+    /**
+     * Compact the collection by rebuilding the index without deleted vectors.
+     * This permanently removes tombstoned vectors and reclaims space.
+     *
+     * @returns Number of vectors removed during compaction
+     */
+    async compact() {
+        return this.withMutationLock(async () => {
+            if (this.deletedIds.size === 0)
+                return 0;
+            const removedCount = this.deletedIds.size;
+            // Collect all non-deleted vectors
+            const activeVectors = [];
+            for (const [numericId, vector] of this.hnsw.getAllVectors()) {
+                if (!this.deletedIds.has(numericId)) {
+                    const stringId = this.idReverseMap.get(numericId);
+                    if (stringId) {
+                        activeVectors.push({
+                            id: stringId,
+                            numericId,
+                            vector,
+                            meta: this.metadata.get(numericId)
+                        });
+                    }
+                }
+            }
+            // Rebuild into temporary state first so we can roll back on failure.
+            const rebuiltHnsw = new HNSWIndex(this.dimension, this.metric, this.M, this.efConstruction);
+            const rebuiltIdMap = new Map();
+            const rebuiltIdReverseMap = new Map();
+            const rebuiltMetadata = new Map();
+            const rebuiltDeletedIds = new Set();
+            // Prepare bulk insert data and re-populate mappings
+            const points = new Array(activeVectors.length);
+            for (let i = 0; i < activeVectors.length; i++) {
+                const { id, vector, meta } = activeVectors[i];
+                const newNumericId = i;
+                rebuiltIdMap.set(id, newNumericId);
+                rebuiltIdReverseMap.set(newNumericId, id);
+                if (meta) {
+                    rebuiltMetadata.set(newNumericId, meta);
+                }
+                points[i] = { id: newNumericId, vector };
+            }
+            // Use bulk insert for better performance
+            try {
+                await rebuiltHnsw.addPointsBulk(points);
+            }
+            catch (err) {
+                rebuiltHnsw.destroy();
+                throw err;
+            }
+            const previousState = {
+                hnsw: this.hnsw,
+                idMap: this.idMap,
+                idReverseMap: this.idReverseMap,
+                metadata: this.metadata,
+                deletedIds: this.deletedIds,
+                nextNumericId: this.nextNumericId,
+                activeCount: this.activeCount,
+                indexKey: this.indexKey,
+                metaKey: this.metaKey,
+                deletedKey: this.deletedKey,
+                dirty: this.dirty,
+            };
+            this.hnsw = rebuiltHnsw;
+            this.idMap = rebuiltIdMap;
+            this.idReverseMap = rebuiltIdReverseMap;
+            this.metadata = rebuiltMetadata;
+            this.deletedIds = rebuiltDeletedIds;
+            this.nextNumericId = activeVectors.length;
+            this.activeCount = activeVectors.length;
+            this.dirty = true;
+            try {
+                await this.saveToDiskUnlocked();
+            }
+            catch (err) {
+                this.hnsw.destroy();
+                this.hnsw = previousState.hnsw;
+                this.idMap = previousState.idMap;
+                this.idReverseMap = previousState.idReverseMap;
+                this.metadata = previousState.metadata;
+                this.deletedIds = previousState.deletedIds;
+                this.nextNumericId = previousState.nextNumericId;
+                this.activeCount = previousState.activeCount;
+                this.dirty = previousState.dirty;
+                this.setActiveStorageKeys(previousState.indexKey, previousState.metaKey, previousState.deletedKey);
+                throw err;
+            }
+            previousState.hnsw.destroy();
+            return removedCount;
+        });
+    }
+    /**
+     * Destroy the collection, freeing all in-memory resources.
+     * @param save Whether to persist data to storage before destroying (default: true).
+     *             Pass false when the collection is being deleted entirely.
+     */
+    async destroy(save = true) {
+        await this.withMutationLock(async () => {
+            if (save && this.dirty) {
+                await this.saveToDiskUnlocked();
+            }
+            // Destroy the HNSW index to free memory
+            if (this.hnsw && typeof this.hnsw.destroy === 'function') {
+                this.hnsw.destroy();
+            }
+            // Clear all maps to free memory
+            this.idMap.clear();
+            this.idReverseMap.clear();
+            this.metadata.clear();
+            this.deletedIds.clear();
+            this.activeCount = 0;
+            this.dirty = false;
+        });
+    }
+}
+//# sourceMappingURL=Collection.js.map