vectlite 0.1.12 → 0.9.1

package/README.md

@@ -43,10 +43,12 @@ db.close()
 ### Core
 
 - **Single-file storage** -- one `.vdb` file per database, portable and easy to back up
-- **Dense vectors** -- cosine similarity with automatic HNSW indexing for large collections
+- **Distance metrics** -- cosine (default), euclidean (L2), dot product, manhattan (L1) with SIMD acceleration
+- **Dense vectors** -- automatic HNSW indexing with metric-aware distance functions
 - **Sparse vectors** -- BM25-scored inverted index for keyword retrieval
 - **Hybrid search** -- dense + sparse fusion with linear or RRF strategies
 - **Vector quantization** -- scalar (int8, 4x), binary (32x), and product quantization (PQ) with 2-stage rescoring
+- **Multi-vector / ColBERT** -- late interaction search with per-token MaxSim scoring and 2-bit quantization (~16x compression)
 - **Rich metadata** -- string, number, boolean, null, array, and nested object values
 - **Crash-safe WAL** -- writes land in a write-ahead log first, then checkpoint with `compact()`
 - **Transactions** -- atomic batched writes with `db.transaction()`
@@ -61,6 +63,7 @@ db.close()
 - **MMR diversification** -- `mmrLambda` controls relevance vs. diversity trade-off
 - **Namespaces** -- logical isolation with per-namespace or cross-namespace search
 - **Observability** -- `searchWithStats()` returns timings, BM25 term scores, ANN stats, and per-result explain payloads
+- **Payload indexes** -- keyword and numeric indexes on metadata fields accelerate filtered queries on large collections
 
 ### Data Management
 
@@ -68,14 +71,35 @@ db.close()
 - **Bulk ingestion** -- `bulkIngest()` with deferred index rebuilds for fast imports
 - **Listing & filtered counts** -- `list()` and `count({ namespace, filter })` without a vector query
 - **Delete by filter** -- `deleteByFilter()` for bulk deletion by metadata filter
+- **Partial metadata updates** -- `updateMetadata()` merges a patch without re-writing the vector or rebuilding indexes
 - **Snapshots** -- `db.snapshot(path)` creates a self-contained copy
 - **Backup / Restore** -- `db.backup(dir)` and `vectlite.restore(dir, path)` for full roundtrips
 - **Read-only mode** -- `vectlite.open(path, { readOnly: true })` for safe concurrent readers
 - **Explicit close** -- `db.close()` to release locks deterministically
 - **Lock timeouts** -- `lockTimeout` for bounded lock acquisition waits
+- **TTL / Expiry** -- `setTtl()` / `clearTtl()` or `ttl` option on insert/upsert; expired records auto-filtered from reads and GC'd on compact
+- **Cursor-based pagination** -- `listCursor()` for efficient iteration over large collections
+- **Async API** -- `searchAsync()`, `compactAsync()`, `flushAsync()`, `bulkIngestAsync()` run on the libuv threadpool
 
 ## Usage
 
+### Distance Metrics
+
+```js
+// Default is cosine similarity
+const db = vectlite.open('knowledge.vdb', { dimension: 384 })
+
+// Choose a different metric at creation time
+const db2 = vectlite.open('knowledge.vdb', { dimension: 384, metric: 'euclidean' })
+const db3 = vectlite.open('knowledge.vdb', { dimension: 384, metric: 'dotproduct' })
+const db4 = vectlite.open('knowledge.vdb', { dimension: 384, metric: 'manhattan' })
+
+// Aliases: 'l2', 'dot', 'ip', 'l1'
+console.log(db2.metric) // "euclidean"
+```
+
+The metric is persisted in the database file. Scores are always oriented so that **higher is better**.
+
 ### Hybrid Search
 
 ```js
@@ -114,6 +138,10 @@ products.upsert('p1', embedding, { name: 'Widget', price: 9.99 })
 
 const logs = store.openOrCreateCollection('logs', 128)
 console.log(store.collections()) // ["logs", "products"]
+
+products.close()
+logs.close()
+store.close()
 ```
 
 ### Transactions
@@ -167,6 +195,9 @@ const records = db.list({ namespace: 'docs', filter: { stale: false }, limit: 20
 const count = db.count({ namespace: 'docs', filter: { source: 'blog' } })
 const deleted = db.deleteByFilter({ stale: true }, { namespace: 'docs' })
 
+// Partial metadata update (merge patch -- only touches specified keys)
+db.updateMetadata('doc1', { status: 'reviewed', score: 0.95 })
+
 db.close()
 ```
 
@@ -184,22 +215,42 @@ console.log(outcome.stats.used_ann)     // true
 console.log(outcome.results[0].explain) // Detailed scoring breakdown
 ```
 
+### Payload Indexes
+
+Create keyword or numeric indexes on metadata fields to accelerate filtered queries on large collections. Indexes are automatically used by `search()`, `count()`, and `list()`.
+
+```js
+// Create indexes on frequently-filtered fields
+db.createIndex('source', 'keyword')   // string equality, $in
+db.createIndex('score', 'numeric')    // range queries: $gt, $gte, $lt, $lte
+
+// Filtered queries now use indexes automatically
+const count = db.count({ filter: { source: 'blog' } })
+const results = db.search(query, { k: 10, filter: { score: { $gte: 0.8 } } })
+
+// Inspect and manage indexes
+console.log(db.listIndexes())  // [{ field: 'source', type: 'keyword' }, ...]
+db.dropIndex('score')
+```
+
 ### Vector Quantization
 
-Reduce memory usage and accelerate search with quantized vectors. All methods use a 2-stage pipeline: fast quantized candidate selection followed by exact float32 rescoring.
+Reduce in-memory candidate-index usage and accelerate search with quantized vectors. All methods use a 2-stage pipeline: fast quantized candidate selection followed by exact float32 rescoring.
 
 ```js
-// Scalar quantization (int8) -- 4x memory reduction, minimal recall loss
+// Scalar quantization (int8) -- smaller in-memory candidate index, minimal recall loss
 db.enableQuantization('scalar')
 
-// Binary quantization -- 32x memory reduction, best for normalized embeddings
-db.enableQuantization('binary', JSON.stringify({ rescoreMultiplier: 10 }))
+// Binary quantization -- smallest in-memory candidate index, best for normalized embeddings
+db.enableQuantization('binary', { rescoreMultiplier: 10 })
 
-// Product quantization -- configurable compression for very large datasets
-db.enableQuantization('product', JSON.stringify({ numSubVectors: 16, numCentroids: 256 }))
+// Product quantization -- "pq" and "product" are accepted case-insensitively
+console.log(db.validNumSubVectors()) // valid PQ partitions for this dimension
+db.enableQuantization('pq', { numSubVectors: 16, numCentroids: 256 })
 
 // Search works exactly the same -- quantization accelerates it transparently
 const results = db.search(queryEmbedding, { k: 10 })
+const sameResults = db.search({ query: queryEmbedding, k: 10 })
 
 // Check quantization status
 console.log(db.isQuantized)         // true
@@ -209,7 +260,133 @@ console.log(db.quantizationMethod)  // "scalar", "binary", or "product"
 db.disableQuantization()
 ```
 
-Quantization parameters persist across reopens in a `.vdb.quant` sidecar file. The quantized index auto-rebuilds on inserts and upserts.
+`rescoreMultiplier` controls the number of quantized candidates rescored with exact float32 scoring: `k * rescoreMultiplier`, capped at the collection size. Increase it to trade latency for recall.
+
+For PQ, `numSubVectors` must divide the database dimension. If omitted, Vectlite chooses a compatible default; use `db.validNumSubVectors()` to inspect all valid values.
+
+Quantization does not shrink the `.vdb` file on disk. Vectlite keeps the original float32 vectors for exact rescoring and stores quantization parameters in a `.vdb.quant` sidecar file, so total disk footprint can increase slightly. The quantized index auto-rebuilds on inserts and upserts.
+
+### Multi-Vector / ColBERT Search
+
+Store token-level embeddings (ColBERT, ColPali) and search with MaxSim late interaction scoring.
+
+```js
+// Upsert with per-token ColBERT embeddings
+db.upsertMultiVectors('doc1', denseVector,
+  { colbert: [tokenVec1, tokenVec2] },
+  { metadata: { source: 'paper' } }
+)
+
+// MaxSim search
+const results = db.searchMultiVector('colbert', queryTokenVectors)
+
+// Enable 2-bit quantization (~16x compression)
+db.enableMultiVectorQuantization('colbert')
+
+// Check and disable
+console.log(db.isMultiVectorQuantized('colbert'))  // true
+db.disableMultiVectorQuantization('colbert')
+```
+
+### TTL / Expiry
+
+Records can automatically expire after a time-to-live. Expired records are transparently filtered from all reads and permanently removed on `compact()`.
+
+```js
+// Set TTL on insert/upsert (seconds)
+db.upsert('session1', embedding, { user: 'alice' }, { ttl: 3600 }) // expires in 1 hour
+
+// Set/clear TTL on existing records
+db.setTtl('doc1', 86400)    // expire in 24 hours
+db.clearTtl('doc1')          // remove expiry
+
+// Expired records are invisible to get/list/count/search
+const record = db.get('session1') // null after TTL elapses
+
+// compact() garbage-collects expired records from disk
+db.compact()
+```
+
+### Cursor-Based Pagination
+
+Efficiently iterate over large collections without offset overhead.
+
+```js
+// Paginate 100 records at a time
+let cursor = null
+do {
+  const page = db.listCursor({ limit: 100, cursor })
+  for (const record of page.records) {
+    process(record)
+  }
+  cursor = page.cursor
+} while (cursor !== null)
+
+// Works with namespace and filter
+const page = db.listCursor({ namespace: 'docs', filter: { source: 'blog' }, limit: 50 })
+```
+
+### Async API
+
+Non-blocking versions of heavy operations that run on the libuv threadpool.
+
+```js
+// Async search (returns a Promise)
+const results = await db.searchAsync(queryEmbedding, { k: 10, filter: { source: 'blog' } })
+
+// Async search with stats
+const outcome = await db.searchWithStatsAsync(queryEmbedding, { k: 10 })
+
+// Async maintenance
+await db.flushAsync()
+await db.compactAsync()
+
+// Async bulk ingestion
+const count = await db.bulkIngestAsync(records, { batchSize: 5000 })
+```
+
+### OpenTelemetry Integration
+
+vectlite ships with optional OpenTelemetry tracing. When enabled, every search
+call is wrapped in a span carrying semantic DB attributes and search-specific
+metrics. `@opentelemetry/api` is loaded lazily -- it is **not** a runtime
+dependency.
+
+```js
+const vectlite = require('vectlite')
+
+// Auto-detect: resolves a tracer from @opentelemetry/api if installed
+const tracer = vectlite.configureOpenTelemetry()
+
+// Or supply your own tracer
+vectlite.configureOpenTelemetry({ tracer: myTracer })
+
+// Custom tracer name (default: 'vectlite')
+vectlite.configureOpenTelemetry({ tracerName: 'my-app' })
+
+// Disable
+vectlite.configureOpenTelemetry(false)
+```
+
+When a tracer is active, each `search` / `searchWithStats` / `searchAsync` /
+`searchWithStatsAsync` call creates a `vectlite.search` span with these
+attributes:
+
+| Attribute | Description |
+|---|---|
+| `db.system` | Always `"vectlite"` |
+| `db.operation.name` | Always `"search"` |
+| `vectlite.search.k` | Requested result count |
+| `vectlite.search.namespace` | Target namespace |
+| `vectlite.search.has_dense` | Whether a dense query vector was provided |
+| `vectlite.search.has_sparse` | Whether sparse terms were provided |
+| `vectlite.search.fusion` | Fusion strategy (`"linear"` or `"rrf"`) |
+| `vectlite.search.used_ann` | Whether HNSW was used (set after completion) |
+| `vectlite.search.result_count` | Number of results returned (set after completion) |
+| `vectlite.search.total_us` | Total search time in microseconds (set after completion) |
+
+If a search throws, the span records the exception and sets an error status
+before re-throwing.
 
 ## Database Methods Reference
 
@@ -225,29 +402,46 @@ Quantization parameters persist across reopens in a `.vdb.quant` sidecar file. T
 | `db.delete(id, { namespace })` | Delete a single record |
 | `db.deleteMany(ids, { namespace })` | Delete multiple records by id |
 | `db.deleteByFilter(filter, { namespace })` | Delete all records matching a filter |
+| `db.updateMetadata(id, metadata, { namespace })` | Merge a metadata patch into an existing record (no vector rewrite) |
+| `db.setTtl(id, seconds, { namespace })` | Set time-to-live on a record (seconds from now) |
+| `db.clearTtl(id, { namespace })` | Remove TTL from a record |
 
 ### Read Methods
 
 | Method | Description |
 |---|---|
 | `db.get(id, { namespace })` | Get a single record by id |
-| `db.search(query, options)` | Search and return a list of results |
+| `db.search(query, options)` or `db.search({ query, ...options })` | Search and return a list of results |
 | `db.searchWithStats(query, options)` | Search with detailed performance stats |
 | `db.count({ namespace, filter })` | Count records, optionally scoped by namespace/filter |
 | `db.list({ namespace, filter, limit, offset })` | List records without issuing a vector query |
+| `db.listCursor({ namespace, filter, limit, cursor })` | Cursor-based pagination for large collections |
 | `db.namespaces()` | List all namespaces |
 | `db.dimension` | Vector dimension (property) |
 | `db.path` | Database file path (property) |
+| `db.metric` | Distance metric name: `"cosine"`, `"euclidean"`, `"dotproduct"`, or `"manhattan"` (property) |
 | `db.readOnly` | Whether the database is read-only (property) |
 
+### Index Methods
+
+| Method | Description |
+|---|---|
+| `db.createIndex(field, indexType)` | Create a payload index (`'keyword'` or `'numeric'`) on a metadata field |
+| `db.dropIndex(field)` | Remove an index |
+| `db.listIndexes()` | List all active indexes as `[{ field, type }, ...]` |
+
 ### Quantization Methods
 
 | Method | Description |
 |---|---|
-| `db.enableQuantization(method, optionsJson)` | Enable quantization (`'scalar'`, `'binary'`, or `'product'`) |
+| `db.enableQuantization(method, options)` | Enable quantization (`'scalar'`, `'binary'`, or `'pq'` / `'product'`) |
 | `db.disableQuantization()` | Disable quantization and remove persisted parameters |
 | `db.isQuantized` | Whether quantization is enabled (property) |
 | `db.quantizationMethod` | Active method name or `null` (property) |
+| `db.validNumSubVectors()` | Valid PQ `numSubVectors` values for this database dimension |
+| `db.enableMultiVectorQuantization(space, options)` | Enable 2-bit quantization for a multi-vector space |
+| `db.disableMultiVectorQuantization(space)` | Disable multi-vector quantization for a space |
+| `db.isMultiVectorQuantized(space)` | Whether multi-vector quantization is enabled for a space |
 
 ### Maintenance Methods
 
@@ -260,6 +454,16 @@ Quantization parameters persist across reopens in a `.vdb.quant` sidecar file. T
 | `db.transaction()` | Begin an atomic transaction |
 | `db.close()` | Flush pending state, release the file lock, and invalidate the handle |
 
+### Async Methods
+
+| Method | Description |
+|---|---|
+| `db.searchAsync(query, options)` | Non-blocking search (returns Promise) |
+| `db.searchWithStatsAsync(query, options)` | Non-blocking search with stats (returns Promise) |
+| `db.flushAsync()` | Non-blocking flush/compact (returns Promise) |
+| `db.compactAsync()` | Non-blocking compact (returns Promise) |
+| `db.bulkIngestAsync(records, options)` | Non-blocking bulk import (returns Promise) |
+
 ## Filter Operators
 
 | Operator | Example | Description |

package/index.d.ts

@@ -9,6 +9,7 @@ export type MetadataValue =
 export type Metadata = { [key: string]: MetadataValue }
 export type SparseVector = { [term: string]: number }
 export type NamedVectors = { [name: string]: number[] }
+export type MultiVectors = { [space: string]: number[][] }
 export type Filter = { [key: string]: unknown }
 export type TextEmbedding = ArrayLike<number>
 export type TextEmbeddingResult = TextEmbedding | Promise<TextEmbedding>
@@ -21,6 +22,7 @@ export interface Record {
   vectors: NamedVectors
   sparse: SparseVector
   metadata: Metadata
+  expires_at: number | null
 }
 
 export interface SearchTimings {
@@ -41,6 +43,8 @@ export interface SearchStats {
   ann_loaded_from_disk: boolean
   wal_entries_replayed: number
   fusion: string
+  effective_dimension: number
+  matryoshka_truncated: boolean
   rerank_applied: boolean
   rerank_count: number
   timings: SearchTimings
@@ -80,6 +84,7 @@ export interface WriteOptions {
   namespace?: string | null
   sparse?: SparseVector | null
   vectors?: NamedVectors | null
+  ttl?: number | null
 }
 
 export interface CountOptions {
@@ -92,6 +97,16 @@ export interface ListOptions extends CountOptions {
   offset?: number | null
 }
 
+export interface ListCursorOptions extends CountOptions {
+  limit?: number | null
+  cursor?: string | null
+}
+
+export interface ListCursorResult {
+  records: Record[]
+  cursor: string | null
+}
+
 export interface BulkIngestOptions {
   namespace?: string | null
   batchSize?: number
@@ -110,15 +125,59 @@ export interface SearchOptions {
   vectorName?: string | null
   fusion?: 'linear' | 'rrf'
   rrfK?: number
+  truncateDim?: number | null
   explain?: boolean
   queryVectors?: { [name: string]: number[] } | null
   vectorWeights?: { [name: string]: number } | null
 }
 
+export interface SearchRequest extends SearchOptions {
+  query?: number[] | null
+}
+
+export type QuantizationMethod = 'scalar' | 'int8' | 'binary' | 'product' | 'pq'
+export interface QuantizationOptions {
+  rescoreMultiplier?: number
+  rescore_multiplier?: number
+  numSubVectors?: number
+  num_sub_vectors?: number
+  numCentroids?: number
+  num_centroids?: number
+  trainingIterations?: number
+  training_iterations?: number
+}
+
+export interface MultiVectorWriteOptions {
+  namespace?: string | null
+  metadata?: Metadata | null
+}
+
+export interface MultiVectorSearchOptions {
+  k?: number
+  filter?: Filter | null
+  namespace?: string | null
+}
+
+export interface MultiVectorSearchResult {
+  namespace: string
+  id: string
+  score: number
+  metadata: Metadata
+}
+
+export interface MultiVectorQuantizationOptions {
+  method?: 'two_bit'
+  rescoreMultiplier?: number
+  rescore_multiplier?: number
+}
+
+export type DistanceMetric = 'cosine' | 'euclidean' | 'dotproduct' | 'manhattan' | 'l2' | 'dot' | 'ip' | 'l1'
+
 export interface OpenOptions {
   dimension?: number | null
   readOnly?: boolean
   lockTimeout?: number | null
+  metric?: DistanceMetric | null
 }
 
 export class VectLiteError extends Error {}
@@ -139,12 +198,14 @@ export class Database {
   readonly path: string
   readonly walPath: string
   readonly dimension: number
+  readonly metric: string
   readonly readOnly: boolean
 
   count(options?: CountOptions): number
   namespaces(): string[]
   close(): void
   list(options?: ListOptions): Record[]
+  listCursor(options?: ListCursorOptions): ListCursorResult
   transaction(): Transaction
   insert(id: string, vector: number[], metadata?: Metadata | null, options?: WriteOptions): void
   upsert(id: string, vector: number[], metadata?: Metadata | null, options?: WriteOptions): void
@@ -155,12 +216,37 @@ export class Database {
   delete(id: string, options?: { namespace?: string | null }): boolean
   deleteMany(ids: string[], options?: { namespace?: string | null }): number
   deleteByFilter(filter: Filter, options?: { namespace?: string | null }): number
+  updateMetadata(id: string, metadata: Metadata, options?: { namespace?: string | null }): boolean
+  setTtl(id: string, ttl: number, options?: { namespace?: string | null }): boolean
+  clearTtl(id: string, options?: { namespace?: string | null }): boolean
+  createIndex(field: string, indexType: 'keyword' | 'numeric'): boolean
+  dropIndex(field: string): boolean
+  listIndexes(): Array<{ field: string; type: 'keyword' | 'numeric' }>
+  readonly isQuantized: boolean
+  readonly quantizationMethod: 'scalar' | 'binary' | 'product' | null
+  enableQuantization(method?: QuantizationMethod, options?: QuantizationOptions | string): void
+  disableQuantization(): void
+  validNumSubVectors(): number[]
+  upsertMultiVectors(id: string, vector: number[], multiVectors: MultiVectors, options?: MultiVectorWriteOptions): void
+  searchMultiVector(space: string, queryTokens: number[][], options?: MultiVectorSearchOptions): MultiVectorSearchResult[]
+  enableMultiVectorQuantization(space: string, options?: MultiVectorQuantizationOptions | string): void
+  disableMultiVectorQuantization(space: string): void
+  isMultiVectorQuantized(space: string): boolean
   flush(): void
   compact(): void
   snapshot(dest: string): void
   backup(dest: string): void
+  search(request: SearchRequest): SearchResult[]
   search(query?: number[] | null, options?: SearchOptions): SearchResult[]
+  searchWithStats(request: SearchRequest): SearchResponse
   searchWithStats(query?: number[] | null, options?: SearchOptions): SearchResponse
+  searchAsync(request: SearchRequest): Promise<SearchResult[]>
+  searchAsync(query?: number[] | null, options?: SearchOptions): Promise<SearchResult[]>
+  searchWithStatsAsync(request: SearchRequest): Promise<SearchResponse>
+  searchWithStatsAsync(query?: number[] | null, options?: SearchOptions): Promise<SearchResponse>
+  flushAsync(): Promise<void>
+  compactAsync(): Promise<void>
+  bulkIngestAsync(records: Record[], options?: BulkIngestOptions): Promise<number>
 }
 
 export class Store {
@@ -171,11 +257,36 @@ export class Store {
   openCollectionReadOnly(name: string): Database
   dropCollection(name: string): boolean
   collections(): string[]
+  close(): void
 }
 
 export function open(path: string, options?: OpenOptions): Database
 export function openStore(root: string): Store
 export function restore(source: string, dest: string): Database
+export interface OpenTelemetryOptions {
+  /** Pass `false` or `{ enabled: false }` to disable tracing. */
+  enabled?: boolean
+  /** Supply your own OTel `Tracer` instance. */
+  tracer?: unknown
+  /** Tracer name used when auto-resolving via `@opentelemetry/api`. Defaults to `'vectlite'`. */
+  tracerName?: string
+}
+
+/**
+ * Configure optional OpenTelemetry tracing for search operations.
+ *
+ * When a tracer is active, every `search`, `searchWithStats`, `searchAsync`,
+ * and `searchWithStatsAsync` call is wrapped in a span with semantic
+ * `db.system` / `db.operation.name` attributes and search-specific metrics.
+ *
+ * `@opentelemetry/api` is loaded lazily via `require()` -- it is **not** a
+ * runtime dependency. If the package is not installed the function returns
+ * `null` and search calls remain un-instrumented.
+ *
+ * @returns The resolved tracer, or `null` if tracing could not be configured.
+ */
+export function configureOpenTelemetry(options?: OpenTelemetryOptions | false): unknown | null
+
 export function sparseTerms(text: string): SparseVector
 export function upsertText(
   db: Database,