verso-db 0.1.4 → 0.2.0

package/dist/Collection.d.ts

@@ -1,4 +1,5 @@
 import { DistanceMetric } from './HNSWIndex';
+import type { StorageBackend } from './storage/StorageBackend';
 /**
  * Configuration for adding vectors to a collection.
  */
@@ -6,8 +7,8 @@ export interface AddConfig {
     /** Unique string identifiers for each vector */
     ids: string[];
     /** Array of vectors to add (must match collection dimension) */
-    vectors: number[][];
-    /** Optional metadata for each vector (same length as ids/vectors) */
+    vectors: Array<number[] | Float32Array>;
+    /** Optional metadata for each vector (same length as ids/vectors, plain JSON objects only) */
     metadata?: Array<Record<string, any>>;
 }
 /**
@@ -35,7 +36,7 @@ export interface AddConfig {
  */
 export interface QueryConfig {
     /** The query vector (must match collection dimension) */
-    queryVector: number[];
+    queryVector: number[] | Float32Array;
     /** Number of nearest neighbors to return */
     k: number;
     /**
@@ -52,39 +53,80 @@ export interface QueryConfig {
     filter?: Record<string, any>;
     /** Search effort parameter (higher = better recall, slower). Default: max(k*2, 50) */
     efSearch?: number;
+    /** Use int8 quantized search + float32 rescore. Default: auto (true when quantization enabled) */
+    useQuantizedSearch?: boolean;
+    /** Oversampling multiplier for quantized rescore (default: 3). Higher = better recall, slower. */
+    candidateMultiplier?: number;
 }
 export interface QueryResult {
     ids: string[];
     distances: number[];
     metadata: Array<Record<string, any>>;
 }
+export interface CollectionOptions {
+    /**
+     * Persist mutating operations automatically after they complete.
+     * VectorDB-managed collections enable this by default.
+     */
+    autoPersist?: boolean;
+}
 export declare class Collection {
+    private static readonly MAX_INTERNAL_ID;
     private name;
     private dimension;
     private metric;
     private M;
     private efConstruction;
-    private indexPath;
-    private metaPath;
-    private deletedPath;
+    private storage;
+    private readonly stateKey;
+    private readonly defaultIndexKey;
+    private readonly defaultMetaKey;
+    private readonly defaultDeletedKey;
+    private indexKey;
+    private metaKey;
+    private deletedKey;
     private hnsw;
     private idMap;
     private idReverseMap;
     private metadata;
     private deletedIds;
+    private nextNumericId;
+    private activeCount;
+    private readonly autoPersist;
+    private dirty;
+    private mutationLock;
     constructor(name: string, config: {
         dimension: number;
         metric?: DistanceMetric;
         M?: number;
         efConstruction?: number;
-    }, collectionPath: string);
+    }, storage: StorageBackend, options?: CollectionOptions);
     init(): Promise<void>;
+    private withMutationLock;
+    flush(): Promise<void>;
+    private captureMutationSnapshot;
+    private restoreMutationSnapshot;
+    /**
+     * Wrap a mutation in snapshot-capture + auto-persist.
+     * The mutation fn should set `this.dirty = true` when it mutates state.
+     * If autoPersist is on and dirty was set, saves after fn completes.
+     * On error, restores from snapshot.
+     */
+    private withAutoSave;
+    private reserveNumericIds;
+    private setActiveStorageKeys;
+    private isValidStorageState;
+    private loadStorageState;
+    private getVersionedDataKeys;
+    private isVersionedDataKey;
+    private validateLoadedIndexConfig;
     private loadFromDisk;
     add(config: AddConfig): Promise<void>;
     query(config: QueryConfig): Promise<QueryResult>;
     /**
      * Batch query for multiple vectors at once.
-     * More efficient than calling query() multiple times.
+     * Shares query semantics with query(), including adaptive candidate expansion
+     * for filters and tombstones.
      *
      * @param configs Array of query configurations
      * @returns Array of query results, one per query
@@ -98,24 +140,34 @@ export declare class Collection {
     /**
      * Upsert vectors (insert or update).
      *
-     * NOTE: HNSW indices don't support efficient in-place updates.
-     * Duplicate IDs will throw an error. For updates, rebuild the index
-     * without the old vectors and add the new ones.
+     * For existing IDs, the old vectors are tombstone-deleted and new vectors
+     * are inserted. Use compact() periodically to reclaim space from
+     * tombstoned vectors.
      *
-     * @throws Error if any ID already exists
+     * For new IDs, behaves identically to add().
      */
     upsert(config: AddConfig): Promise<void>;
+    private addInternal;
+    /** Returns the vector dimension for this collection. */
+    getDimension(): number;
+    /** Returns the distance metric for this collection. */
+    getMetric(): DistanceMetric;
+    /** Returns the HNSW M parameter for this collection. */
+    getM(): number;
+    /** Returns the HNSW efConstruction parameter for this collection. */
+    getEfConstruction(): number;
     /**
      * Returns the number of active (non-deleted) vectors in the collection.
+     * O(1) via maintained counter.
      */
     count(): number;
     /**
-     * Returns the total number of vectors including deleted (tombstoned) ones.
+     * Returns the total number of tracked vectors including deleted (tombstoned) ones.
      * Use this to determine when compaction might be beneficial.
      */
     countWithDeleted(): number;
     /**
-     * Returns the number of deleted (tombstoned) vectors.
+     * Returns the number of deleted (tombstoned) vectors awaiting compaction.
      */
     deletedCount(): number;
     /**
@@ -126,14 +178,15 @@ export declare class Collection {
      * @param id The string ID of the vector to delete
      * @returns true if the vector was deleted, false if it didn't exist or was already deleted
      */
-    delete(id: string): boolean;
+    private deleteInternal;
+    delete(id: string): Promise<boolean>;
     /**
      * Mark multiple vectors as deleted (tombstone deletion).
      *
      * @param ids Array of string IDs to delete
      * @returns Number of vectors that were successfully deleted
      */
-    deleteBatch(ids: string[]): number;
+    deleteBatch(ids: string[]): Promise<number>;
     /**
      * Check if a vector exists and is not deleted.
      */
@@ -142,8 +195,31 @@ export declare class Collection {
      * Check if a vector was deleted (tombstoned).
      */
     isDeleted(id: string): boolean;
+    /**
+     * Reorder the internal HNSW index for BFS cache locality.
+     * Remaps all internal IDs so that graph neighbors are stored
+     * contiguously in memory, improving search cache hit rates.
+     */
+    reorderIndex(): Promise<void>;
     saveToDisk(): Promise<void>;
+    private saveToDiskUnlocked;
+    private executeQuery;
+    private searchCandidates;
+    private filterAndDeduplicateResults;
+    private materializeResults;
+    private validateQueryInput;
+    private hasFilter;
+    private reconcileLoadedState;
     private matchesFilter;
+    private validateFilterOperators;
+    private validateAddInput;
+    private validateMetadataValue;
+    private parseMetadataLine;
+    private isPlainObject;
+    private isFilterObject;
+    private isOperatorObject;
+    private deepEqual;
+    private matchesInOperator;
     /**
      * Compact the collection by rebuilding the index without deleted vectors.
      * This permanently removes tombstoned vectors and reclaims space.
@@ -151,6 +227,11 @@ export declare class Collection {
      * @returns Number of vectors removed during compaction
      */
     compact(): Promise<number>;
-    destroy(): Promise<void>;
+    /**
+     * Destroy the collection, freeing all in-memory resources.
+     * @param save Whether to persist data to storage before destroying (default: true).
+     *             Pass false when the collection is being deleted entirely.
+     */
+    destroy(save?: boolean): Promise<void>;
 }
 //# sourceMappingURL=Collection.d.ts.map

package/dist/Collection.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Collection.d.ts","sourceRoot":"","sources":["../src/Collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,cAAc,EAAE,MAAM,aAAa,CAAC;AAIxD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,gDAAgD;IAChD,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,gEAAgE;IAChE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC;IACpB,qEAAqE;IACrE,QAAQ,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;CACvC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,WAAW,WAAW;IAC1B,yDAAyD;IACzD,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,4CAA4C;IAC5C,CAAC,EAAE,MAAM,CAAC;IACV;;;;;;;;;;OAUG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,sFAAsF;IACtF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;CACtC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,CAAC,CAAS;IAClB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,IAAI,CAAY;IACxB,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,UAAU,CAAc;gBAEpB,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,cAAc,CAAC;QAAC,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,cAAc,EAAE,MAAM;IAwBvI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAKb,YAAY;IA8DpB,GAAG,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDrC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAuEtD;;;;;;OAMG;IACG,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IA2FhE;;;OAGG;IACG,eAAe,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IA0DhE;;;;;;;;OAQG;IACG,MAAM,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB9C;;OAEG;IACH,KAAK,IAAI,MAAM;IAIf;;;OAGG;IACH,gBAAgB,IAAI,MAAM;IAI1B;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAS3B;;;;;OAKG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM;IAQlC;;OAEG;IACH,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAMxB;;OAEG;IACH,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAMxB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA6BjC,OAAO,CAAC,aAAa;IAsBrB;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC;IA6D1B,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAe/B"}
+{"version":3,"file":"Collection.d.ts","sourceRoot":"","sources":["../src/Collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,cAAc,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAK/D;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,gDAAgD;IAChD,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,gEAAgE;IAChE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC;IACxC,8FAA8F;IAC9F,QAAQ,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;CACvC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,WAAW,WAAW;IAC1B,yDAAyD;IACzD,WAAW,EAAE,MAAM,EAAE,GAAG,YAAY,CAAC;IACrC,4CAA4C;IAC5C,CAAC,EAAE,MAAM,CAAC;IACV;;;;;;;;;;OAUG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,sFAAsF;IACtF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kGAAkG;IAClG,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kGAAkG;IAClG,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;CACtC;AAaD,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAgBD,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAc;IAErD,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,CAAC,CAAS;IAClB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,IAAI,CAAY;IACxB,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,UAAU,CAAc;IAChC,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,KAAK,CAAU;IACvB,OAAO,CAAC,YAAY,CAAgB;gBAGlC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,cAAc,CAAC;QAAC,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE,EAC3F,OAAO,EAAE,cAAc,EACvB,OAAO,GAAE,iBAAsB;IAmC3B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAKb,gBAAgB;IAexB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAO5B,OAAO,CAAC,uBAAuB;IAgB/B,OAAO,CAAC,uBAAuB;IAiB/B;;;;;OAKG;YACW,YAAY;IAgB1B,OAAO,CAAC,iBAAiB;IAoBzB,OAAO,CAAC,oBAAoB;IAM5B,OAAO,CAAC,mBAAmB;YAWb,gBAAgB;IAgB9B,OAAO,CAAC,oBAAoB;IAQ5B,OAAO,CAAC,kBAAkB;IAgB1B,OAAO,CAAC,yBAAyB;YAcnB,YAAY;IAmGpB,GAAG,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAIrC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAItD;;;;;;;OAOG;IACG,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAmBhE;;;OAGG;IACG,eAAe,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAkDhE;;;;;;;;OAQG;IACG,MAAM,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;YAIhC,WAAW;IA6CzB,wDAAwD;IACxD,YAAY,IAAI,MAAM;IAEtB,uDAAuD;IACvD,SAAS,IAAI,cAAc;IAE3B,wDAAwD;IACxD,IAAI,IAAI,MAAM;IAEd,qEAAqE;IACrE,iBAAiB,IAAI,MAAM;IAE3B;;;OAGG;IACH,KAAK,IAAI,MAAM;IAIf;;;OAGG;IACH,gBAAgB,IAAI,MAAM;IAI1B;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAUhB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAc1C;;;;;OAKG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBjD;;OAEG;IACH,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAMxB;;OAEG;IACH,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAM9B;;;;OAIG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IA4C7B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAMnB,kBAAkB;IAiEhC,OAAO,CAAC,YAAY;IAmBpB,OAAO,CAAC,gBAAgB;IAkCxB,OAAO,CAAC,2BAA2B;IA6BnC,OAAO,CAAC,kBAAkB;IA6B1B,OAAO,CAAC,kBAAkB;IA+C1B,OAAO,CAAC,SAAS;IAIjB,OAAO,CAAC,oBAAoB;IAkF5B,OAAO,CAAC,aAAa;IAuCrB,OAAO,CAAC,uBAAuB;IAY/B,OAAO,CAAC,gBAAgB;IAmFxB,OAAO,CAAC,qBAAqB;IAiD7B,OAAO,CAAC,iBAAiB;IA0CzB,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,SAAS;IAyBjB,OAAO,CAAC,iBAAiB;IAgBzB;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC;IAuGhC;;;;OAIG;IACG,OAAO,CAAC,IAAI,GAAE,OAAc,GAAG,OAAO,CAAC,IAAI,CAAC;CAoBnD"}