vectlite 0.1.3 → 0.1.8

package/README.md

@@ -1,75 +1,203 @@
-# Node Binding
+# vectlite
 
-The Node binding now exists in-repo and builds from source.
+[![npm version](https://img.shields.io/npm/v/vectlite.svg)](https://www.npmjs.com/package/vectlite)
+[![Node versions](https://img.shields.io/node/v/vectlite.svg)](https://www.npmjs.com/package/vectlite)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
 
-Current state:
+Embedded vector store for local-first AI applications.
 
-- Rust addon implemented with `napi-rs`
-- JavaScript wrapper and TypeScript declarations included
-- local smoke test available in `bindings/node/tests`
-- npm package can be published as a source-build package
-- installing the npm package requires a working Rust toolchain on the target machine
+**vectlite** is a single-file, zero-dependency vector database written in Rust with Node.js bindings. It gives you dense + sparse hybrid search, HNSW indexing, metadata filtering, transactions, and crash-safe persistence in a single `.vdb` file -- no server, no Docker, no network calls.
 
-## Local Build
-
-From the repository root:
+## Installation
 
 ```bash
-cd bindings/node
-npm run build
+npm install vectlite
 ```
 
-This compiles the Rust addon and writes `bindings/node/vectlite.node`.
+Requires Node.js 18+. Pre-built binaries are available for macOS (x86_64, arm64), Linux (x86_64), and Windows (x86_64). Other platforms fall back to compiling from source (requires Rust/Cargo).
 
-## Local Test
+## Quick Start
 
-```bash
-cd bindings/node
-npm test
+```js
+const vectlite = require('vectlite')
+
+// Create or open a database
+const db = vectlite.open('knowledge.vdb', { dimension: 384 })
+
+// Insert records with vectors, metadata, and sparse terms
+db.upsert('doc1', embedding, { source: 'blog', title: 'Auth Guide' })
+db.upsert('doc2', embedding2, { source: 'notes', title: 'Billing' })
+
+// Search with filters
+const results = db.search(embeddingQuery, { k: 5, filter: { source: 'blog' } })
+
+// Clean up
+db.compact()
 ```
 
-## npm Package Model
+## Features
+
+### Core
 
-The npm package is set up as a source-build package:
+- **Single-file storage** -- one `.vdb` file per database, portable and easy to back up
+- **Dense vectors** -- cosine similarity with automatic HNSW indexing for large collections
+- **Sparse vectors** -- BM25-scored inverted index for keyword retrieval
+- **Hybrid search** -- dense + sparse fusion with linear or RRF strategies
+- **Rich metadata** -- string, number, boolean, null, array, and nested object values
+- **Crash-safe WAL** -- writes land in a write-ahead log first, then checkpoint with `compact()`
+- **Transactions** -- atomic batched writes with `db.transaction()`
+- **File locking** -- advisory locks prevent corruption from concurrent access
 
-- `prepack` stages a self-contained native crate plus the core Rust crate
-- `install` compiles the addon with Cargo on the target machine
-- the published tarball does not ship prebuilt binaries yet
+### Search & Retrieval
 
-That keeps one source of truth for the Rust core, but it means `npm install vectlite` requires:
+- **Metadata filters** -- MongoDB-style operators: `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`, `$in`, `$nin`, `$contains`, `$exists`, `$and`, `$or`, `$not`
+- **Nested filters** -- dot-path traversal (`author.name`), `$elemMatch`, `$size` on arrays and objects
+- **Named vectors** -- multiple vector spaces per record (`vectors: { title: [...], body: [...] }`)
+- **Multi-vector queries** -- weighted search across vector spaces in a single call
+- **MMR diversification** -- `mmrLambda` controls relevance vs. diversity trade-off
+- **Namespaces** -- logical isolation with per-namespace or cross-namespace search
+- **Observability** -- `searchWithStats()` returns timings, BM25 term scores, ANN stats, and per-result explain payloads
 
-- Node 18+
-- Rust/Cargo installed
-- registry/network access to fetch Rust crates during the build
+### Data Management
+
+- **Physical collections** -- `vectlite.openStore()` manages a directory of independent databases
+- **Bulk ingestion** -- `bulkIngest()` with deferred index rebuilds for fast imports
+- **Snapshots** -- `db.snapshot(path)` creates a self-contained copy
+- **Backup / Restore** -- `db.backup(dir)` and `vectlite.restore(dir, path)` for full roundtrips
+- **Read-only mode** -- `vectlite.open(path, { readOnly: true })` for safe concurrent readers
 
 ## Usage
 
+### Hybrid Search
+
 ```js
-const { open, sparseTerms } = require('./index.js')
+const vectlite = require('vectlite')
 
-const db = open('knowledge.vdb', { dimension: 384 })
-db.upsert('doc1', embedding, { source: 'notes', title: 'Auth Guide' })
+const db = vectlite.open('knowledge.vdb', { dimension: 384 })
 
+// Upsert with dense + sparse vectors
+db.upsert(
+  'doc1',
+  denseEmbedding,
+  { source: 'docs', title: 'Auth Setup', text: 'How to configure SSO...' },
+  { sparse: vectlite.sparseTerms('How to configure SSO authentication') },
+)
+
+// Hybrid search
 const results = db.search(queryEmbedding, {
-  k: 5,
-  sparse: sparseTerms('auth guide'),
-  filter: { source: 'notes' },
+  k: 10,
+  sparse: vectlite.sparseTerms('SSO authentication'),
+  fusion: 'rrf',
+  filter: { source: 'docs' },
+  explain: true,
 })
+
+for (const result of results) {
+  console.log(result.id, result.score)
+}
+```
+
+### Collections
+
+```js
+const store = vectlite.openStore('./my_collections')
+const products = store.createCollection('products', 384)
+products.upsert('p1', embedding, { name: 'Widget', price: 9.99 })
+
+const logs = store.openOrCreateCollection('logs', 128)
+console.log(store.collections()) // ["logs", "products"]
+```
+
+### Transactions
+
+```js
+const tx = db.transaction()
+try {
+  tx.upsert('doc1', emb1, { source: 'a' })
+  tx.upsert('doc2', emb2, { source: 'b' })
+  tx.delete('old_doc')
+  tx.commit() // All operations commit atomically
+} catch (err) {
+  tx.rollback() // Roll back on error
+  throw err
+}
+```
+
+### Text Helpers
+
+```js
+async function run() {
+  // embedFn can be sync or async
+  await vectlite.upsertText(db, 'doc1', 'Auth setup guide', embedFn, { source: 'docs' })
+  const results = await vectlite.searchText(db, 'how to authenticate', embedFn, { k: 5 })
+}
 ```
 
-## Scope
+### Snapshots & Backup
 
-The initial Node surface covers the core database and store operations:
+```js
+db.snapshot('/backups/knowledge_2024.vdb') // Self-contained copy
+db.backup('/backups/full/')                // Full backup with ANN sidecars
+
+const restored = vectlite.restore('/backups/full/', 'restored.vdb')
+```
 
-- `open`, `openStore`, `restore`
-- `insert`, `upsert`, `get`, `delete`
-- batch writes and bulk ingest
-- snapshots, backup, compact, flush
-- namespaces and collections
-- dense, sparse, and hybrid search
-- search stats and text helpers
+### Read-Only Mode
 
-Not yet included:
+```js
+const ro = vectlite.open('knowledge.vdb', { readOnly: true })
+const results = ro.search(query, { k: 5 }) // Reads work
+ro.upsert(...)                              // Throws VectLiteError
+```
+
+### Search Diagnostics
+
+```js
+const outcome = db.searchWithStats(query, {
+  k: 5,
+  sparse: terms,
+  explain: true,
+})
+
+console.log(outcome.stats.timings)      // { dense_us: 120, sparse_us: 45, ... }
+console.log(outcome.stats.used_ann)     // true
+console.log(outcome.results[0].explain) // Detailed scoring breakdown
+```
 
-- JS callback rerank hooks
-- prebuilt binaries
+## Filter Operators
+
+| Operator | Example | Description |
+|---|---|---|
+| `$eq` | `{ field: { $eq: 'value' } }` | Equal (also `{ field: 'value' }`) |
+| `$ne` | `{ field: { $ne: 'value' } }` | Not equal |
+| `$gt` / `$gte` | `{ field: { $gt: 5 } }` | Greater than (or equal) |
+| `$lt` / `$lte` | `{ field: { $lt: 20 } }` | Less than (or equal) |
+| `$in` / `$nin` | `{ field: { $in: ['a', 'b'] } }` | In / not in set |
+| `$contains` | `{ field: { $contains: 'auth' } }` | Substring match |
+| `$exists` | `{ field: { $exists: true } }` | Field presence |
+| `$and` / `$or` | `{ $and: [{...}, {...}] }` | Logical combinators |
+| `$not` | `{ $not: {...} }` | Logical negation |
+| `$elemMatch` | `{ tags: { $elemMatch: { $eq: 'rust' } } }` | Match array elements |
+| `$size` | `{ tags: { $size: 3 } }` | Array length |
+| dot-path | `{ 'author.name': 'Alice' }` | Nested field access |
+
+## How It Works
+
+- Records are stored in a compact binary `.vdb` snapshot file
+- Writes go through a crash-safe WAL (`.wal`) before being applied in memory
+- `compact()` folds the WAL into the snapshot and persists HNSW sidecar files
+- Dense search uses HNSW indexes (auto-built for collections above ~128 records)
+- Sparse search uses an inverted index with BM25 scoring
+- Hybrid fusion combines dense + sparse via linear combination or reciprocal rank fusion
+- Advisory file locks (`flock`) prevent concurrent write corruption
+
+## Links
+
+- [Official Documentation](https://vectlite.mcsedition.org/)
+- [GitHub Repository](https://github.com/mcsedition-hub/vectlite)
+- [Issue Tracker](https://github.com/mcsedition-hub/vectlite/issues)
+- [PyPI Package](https://pypi.org/project/vectlite/)
+
+## License
+
+MIT

package/index.d.ts

@@ -10,6 +10,9 @@ export type Metadata = { [key: string]: MetadataValue }
 export type SparseVector = { [term: string]: number }
 export type NamedVectors = { [name: string]: number[] }
 export type Filter = { [key: string]: unknown }
+export type TextEmbedding = ArrayLike<number>
+export type TextEmbeddingResult = TextEmbedding | Promise<TextEmbedding>
+export type TextEmbedder = (text: string) => TextEmbeddingResult
 
 export interface Record {
   namespace: string
@@ -164,19 +167,39 @@ export function upsertText(
   db: Database,
   id: string,
   text: string,
-  embed: (text: string) => ArrayLike<number>,
+  embed: (text: string) => TextEmbedding,
   metadata?: Metadata | null,
   options?: WriteOptions,
 ): void
+export function upsertText(
+  db: Database,
+  id: string,
+  text: string,
+  embed: (text: string) => Promise<TextEmbedding>,
+  metadata?: Metadata | null,
+  options?: WriteOptions,
+): Promise<void>
 export function searchText(
   db: Database,
   text: string,
-  embed: (text: string) => ArrayLike<number>,
+  embed: (text: string) => TextEmbedding,
   options?: SearchOptions,
 ): SearchResult[]
+export function searchText(
+  db: Database,
+  text: string,
+  embed: (text: string) => Promise<TextEmbedding>,
+  options?: SearchOptions,
+): Promise<SearchResult[]>
 export function searchTextWithStats(
   db: Database,
   text: string,
-  embed: (text: string) => ArrayLike<number>,
+  embed: (text: string) => TextEmbedding,
   options?: SearchOptions,
 ): SearchResponse
+export function searchTextWithStats(
+  db: Database,
+  text: string,
+  embed: (text: string) => Promise<TextEmbedding>,
+  options?: SearchOptions,
+): Promise<SearchResponse>

package/index.js

@@ -1,6 +1,66 @@
 'use strict'
 
-const native = require('./vectlite.node')
+const fs = require('node:fs')
+const path = require('node:path')
+
+function linuxLibc() {
+  if (process.platform !== 'linux') {
+    return null
+  }
+
+  const report = process.report?.getReport?.()
+  return report?.header?.glibcVersionRuntime ? 'gnu' : 'musl'
+}
+
+function runtimePrebuildTag() {
+  switch (process.platform) {
+    case 'darwin':
+      if (process.arch === 'x64') return 'darwin-x64'
+      if (process.arch === 'arm64') return 'darwin-arm64'
+      return null
+    case 'linux': {
+      const libc = linuxLibc()
+      if (process.arch === 'x64' && libc === 'gnu') return 'linux-x64-gnu'
+      if (process.arch === 'arm64' && libc === 'gnu') return 'linux-arm64-gnu'
+      return null
+    }
+    case 'win32':
+      if (process.arch === 'x64') return 'win32-x64-msvc'
+      if (process.arch === 'arm64') return 'win32-arm64-msvc'
+      return null
+    default:
+      return null
+  }
+}
+
+function loadNative() {
+  const candidates = []
+  const prebuildTag = runtimePrebuildTag()
+  if (prebuildTag != null) {
+    candidates.push(path.join(__dirname, 'prebuilds', prebuildTag, 'vectlite.node'))
+  }
+  candidates.push(path.join(__dirname, 'vectlite.node'))
+
+  const errors = []
+  for (const candidate of candidates) {
+    if (!fs.existsSync(candidate)) {
+      continue
+    }
+    try {
+      return require(candidate)
+    } catch (error) {
+      errors.push(`${candidate}: ${error?.message ?? String(error)}`)
+    }
+  }
+
+  const detail = errors.length === 0 ? 'No compatible prebuilt binary was found.' : errors.join('\n')
+  throw new Error(
+    `Unable to load the vectlite native addon.\n${detail}\n` +
+      'If this platform is not covered by prebuilt binaries, install Rust/Cargo and reinstall the package.',
+  )
+}
+
+const native = loadNative()
 
 const TOKEN_RE = /[a-z0-9]+/g
 
@@ -25,6 +85,15 @@ function wrapError(fn) {
   }
 }
 
+function wrapAsync(value) {
+  return Promise.resolve(value).catch((error) => {
+    if (error instanceof VectLiteError) {
+      throw error
+    }
+    throw new VectLiteError(error?.message ?? String(error), error)
+  })
+}
+
 function encode(value) {
   return value == null ? null : JSON.stringify(value)
 }
@@ -37,6 +106,18 @@ function asArray(values) {
   return Array.from(values)
 }
 
+function isPromiseLike(value) {
+  return value != null && typeof value.then === 'function'
+}
+
+function embedText(text, embed) {
+  const embedded = wrapError(() => embed(text))
+  if (isPromiseLike(embedded)) {
+    return wrapAsync(embedded).then((vector) => asArray(vector))
+  }
+  return asArray(embedded)
+}
+
 function normalizeWriteOptions(options = {}) {
   return {
     namespace: options.namespace ?? null,
@@ -258,15 +339,30 @@ function upsertText(db, id, text, embed, metadata = null, options = {}) {
   if (payload.text === undefined) {
     payload.text = text
   }
-  return db.upsert(id, asArray(embed(text)), payload, { ...options, sparse: sparseTerms(text) })
+  const vector = embedText(text, embed)
+  const writeOptions = { ...options, sparse: sparseTerms(text) }
+  if (isPromiseLike(vector)) {
+    return wrapAsync(vector.then((resolved) => db.upsert(id, resolved, payload, writeOptions)))
+  }
+  return db.upsert(id, vector, payload, writeOptions)
 }
 
 function searchText(db, text, embed, options = {}) {
-  return db.search(asArray(embed(text)), { ...options, sparse: sparseTerms(text) })
+  const vector = embedText(text, embed)
+  const searchOptions = { ...options, sparse: sparseTerms(text) }
+  if (isPromiseLike(vector)) {
+    return wrapAsync(vector.then((resolved) => db.search(resolved, searchOptions)))
+  }
+  return db.search(vector, searchOptions)
 }
 
 function searchTextWithStats(db, text, embed, options = {}) {
-  return db.searchWithStats(asArray(embed(text)), { ...options, sparse: sparseTerms(text) })
+  const vector = embedText(text, embed)
+  const searchOptions = { ...options, sparse: sparseTerms(text) }
+  if (isPromiseLike(vector)) {
+    return wrapAsync(vector.then((resolved) => db.searchWithStats(resolved, searchOptions)))
+  }
+  return db.searchWithStats(vector, searchOptions)
 }
 
 module.exports = {

package/native/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "vectlite-node"
-version = "0.1.3"
+version = "0.1.8"
 edition = "2024"
 license = "MIT"
 description = "Node.js bindings for vectlite."
-homepage = "https://github.com/mcsedition-hub/vectlite"
+homepage = "https://vectlite.mcsedition.org/"
 repository = "https://github.com/mcsedition-hub/vectlite"
-documentation = "https://github.com/mcsedition-hub/vectlite/tree/main/bindings/node"
+documentation = "https://vectlite.mcsedition.org/"
 build = "build.rs"
 
 [lib]

package/native/vectlite-core/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "vectlite-core"
-version = "0.1.3"
+version = "0.1.8"
 edition = "2024"
 license = "MIT"
 description = "Core storage engine for vectlite."
-homepage = "https://github.com/mcsedition-hub/vectlite"
+homepage = "https://vectlite.mcsedition.org/"
 repository = "https://github.com/mcsedition-hub/vectlite"
-documentation = "https://github.com/mcsedition-hub/vectlite"
+documentation = "https://vectlite.mcsedition.org/"
 
 [lib]
 name = "vectlite"

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "vectlite",
-  "version": "0.1.3",
+  "version": "0.1.8",
   "description": "Embedded vector store for local-first AI applications.",
   "main": "index.js",
   "types": "index.d.ts",
-  "homepage": "https://github.com/mcsedition-hub/vectlite/tree/main/bindings/node",
+  "homepage": "https://vectlite.mcsedition.org/",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/mcsedition-hub/vectlite.git",
@@ -28,12 +28,13 @@
     "index.d.ts",
     "README.md",
     "scripts",
-    "native"
+    "native",
+    "prebuilds"
   ],
   "scripts": {
     "build": "node ./scripts/build-addon.mjs",
-    "test": "npm run build && node --test tests/smoke.test.cjs",
-    "install": "node ./scripts/build-addon.mjs",
+    "test": "npm run build && node --test tests/smoke.test.cjs && node --test tests/prebuild-loader.test.cjs",
+    "install": "node ./scripts/install-addon.mjs",
     "prepack": "node ./scripts/prepare-package.mjs",
     "postpack": "node ./scripts/clean-package.mjs"
   },

package/scripts/collect-prebuilds.mjs

@@ -0,0 +1,27 @@
+import { cpSync, existsSync, mkdirSync, readdirSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import { dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+const packageRoot = resolve(__dirname, '..')
+const sourceRoot = resolve(process.argv[2] ?? join(packageRoot, '..', '..', 'dist', 'node-prebuilds'))
+const destRoot = join(packageRoot, 'prebuilds')
+
+for (const entry of readdirSync(sourceRoot, { withFileTypes: true })) {
+  if (!entry.isDirectory() || !entry.name.startsWith('prebuild-')) {
+    continue
+  }
+
+  const prebuildTag = entry.name.slice('prebuild-'.length)
+  const source = join(sourceRoot, entry.name, 'vectlite.node')
+  if (!existsSync(source)) {
+    console.error(`Missing prebuilt artifact for ${prebuildTag}: ${source}`)
+    process.exit(1)
+  }
+
+  const destDir = join(destRoot, prebuildTag)
+  mkdirSync(destDir, { recursive: true })
+  cpSync(source, join(destDir, 'vectlite.node'))
+  console.log(`Collected ${prebuildTag}`)
+}

package/scripts/install-addon.mjs

@@ -0,0 +1,53 @@
+import { existsSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import { dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { spawnSync } from 'node:child_process'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+const packageRoot = resolve(__dirname, '..')
+
+function linuxLibc() {
+  if (process.platform !== 'linux') {
+    return null
+  }
+
+  const report = process.report?.getReport?.()
+  return report?.header?.glibcVersionRuntime ? 'gnu' : 'musl'
+}
+
+function runtimePrebuildTag() {
+  switch (process.platform) {
+    case 'darwin':
+      if (process.arch === 'x64') return 'darwin-x64'
+      if (process.arch === 'arm64') return 'darwin-arm64'
+      return null
+    case 'linux': {
+      const libc = linuxLibc()
+      if (process.arch === 'x64' && libc === 'gnu') return 'linux-x64-gnu'
+      if (process.arch === 'arm64' && libc === 'gnu') return 'linux-arm64-gnu'
+      return null
+    }
+    case 'win32':
+      if (process.arch === 'x64') return 'win32-x64-msvc'
+      if (process.arch === 'arm64') return 'win32-arm64-msvc'
+      return null
+    default:
+      return null
+  }
+}
+
+const prebuildTag = runtimePrebuildTag()
+const prebuiltPath =
+  prebuildTag == null ? null : join(packageRoot, 'prebuilds', prebuildTag, 'vectlite.node')
+
+if (prebuiltPath != null && existsSync(prebuiltPath)) {
+  console.log(`Using prebuilt addon: ${prebuiltPath}`)
+  process.exit(0)
+}
+
+const result = spawnSync(process.execPath, [join(__dirname, 'build-addon.mjs')], {
+  stdio: 'inherit',
+})
+
+process.exit(result.status ?? 1)

package/scripts/stage-prebuilt.mjs

@@ -0,0 +1,56 @@
+import { cpSync, existsSync, mkdirSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import { dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+const packageRoot = resolve(__dirname, '..')
+
+function linuxLibc() {
+  if (process.platform !== 'linux') {
+    return null
+  }
+
+  const report = process.report?.getReport?.()
+  return report?.header?.glibcVersionRuntime ? 'gnu' : 'musl'
+}
+
+function runtimePrebuildTag() {
+  switch (process.platform) {
+    case 'darwin':
+      if (process.arch === 'x64') return 'darwin-x64'
+      if (process.arch === 'arm64') return 'darwin-arm64'
+      return null
+    case 'linux': {
+      const libc = linuxLibc()
+      if (process.arch === 'x64' && libc === 'gnu') return 'linux-x64-gnu'
+      if (process.arch === 'arm64' && libc === 'gnu') return 'linux-arm64-gnu'
+      return null
+    }
+    case 'win32':
+      if (process.arch === 'x64') return 'win32-x64-msvc'
+      if (process.arch === 'arm64') return 'win32-arm64-msvc'
+      return null
+    default:
+      return null
+  }
+}
+
+const prebuildTag = process.env.VECTLITE_PREBUILD_TAG ?? runtimePrebuildTag()
+if (prebuildTag == null) {
+  console.error('Unable to determine a prebuild tag for this platform.')
+  process.exit(1)
+}
+
+const source = join(packageRoot, 'vectlite.node')
+if (!existsSync(source)) {
+  console.error(`Missing built addon at ${source}. Run the build first.`)
+  process.exit(1)
+}
+
+const destDir = join(packageRoot, 'prebuilds', prebuildTag)
+const dest = join(destDir, 'vectlite.node')
+
+mkdirSync(destDir, { recursive: true })
+cpSync(source, dest)
+console.log(`Staged prebuilt ${prebuildTag}: ${dest}`)