vectlite 0.1.3

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 vectlite contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,75 @@
+# Node Binding
+
+The Node binding now exists in-repo and builds from source.
+
+Current state:
+
+- Rust addon implemented with `napi-rs`
+- JavaScript wrapper and TypeScript declarations included
+- local smoke test available in `bindings/node/tests`
+- npm package can be published as a source-build package
+- installing the npm package requires a working Rust toolchain on the target machine
+
+## Local Build
+
+From the repository root:
+
+```bash
+cd bindings/node
+npm run build
+```
+
+This compiles the Rust addon and writes `bindings/node/vectlite.node`.
+
+## Local Test
+
+```bash
+cd bindings/node
+npm test
+```
+
+## npm Package Model
+
+The npm package is set up as a source-build package:
+
+- `prepack` stages a self-contained native crate plus the core Rust crate
+- `install` compiles the addon with Cargo on the target machine
+- the published tarball does not ship prebuilt binaries yet
+
+That keeps one source of truth for the Rust core, but it means `npm install vectlite` requires:
+
+- Node 18+
+- Rust/Cargo installed
+- registry/network access to fetch Rust crates during the build
+
+## Usage
+
+```js
+const { open, sparseTerms } = require('./index.js')
+
+const db = open('knowledge.vdb', { dimension: 384 })
+db.upsert('doc1', embedding, { source: 'notes', title: 'Auth Guide' })
+
+const results = db.search(queryEmbedding, {
+  k: 5,
+  sparse: sparseTerms('auth guide'),
+  filter: { source: 'notes' },
+})
+```
+
+## Scope
+
+The initial Node surface covers the core database and store operations:
+
+- `open`, `openStore`, `restore`
+- `insert`, `upsert`, `get`, `delete`
+- batch writes and bulk ingest
+- snapshots, backup, compact, flush
+- namespaces and collections
+- dense, sparse, and hybrid search
+- search stats and text helpers
+
+Not yet included:
+
+- JS callback rerank hooks
+- prebuilt binaries

package/index.d.ts

@@ -0,0 +1,182 @@
+export type MetadataValue =
+  | string
+  | number
+  | boolean
+  | null
+  | MetadataValue[]
+  | { [key: string]: MetadataValue }
+
+export type Metadata = { [key: string]: MetadataValue }
+export type SparseVector = { [term: string]: number }
+export type NamedVectors = { [name: string]: number[] }
+export type Filter = { [key: string]: unknown }
+
+export interface Record {
+  namespace: string
+  id: string
+  vector: number[]
+  vectors: NamedVectors
+  sparse: SparseVector
+  metadata: Metadata
+}
+
+export interface SearchTimings {
+  dense_us: number
+  sparse_us: number
+  fusion_us: number
+  total_us: number
+}
+
+export interface SearchStats {
+  used_ann: boolean
+  ann_candidate_count: number
+  exact_fallback: boolean
+  considered_count: number
+  fetch_k: number
+  mmr_applied: boolean
+  sparse_candidate_count: number
+  ann_loaded_from_disk: boolean
+  wal_entries_replayed: number
+  fusion: string
+  rerank_applied: boolean
+  rerank_count: number
+  timings: SearchTimings
+}
+
+export interface ExplainDetails {
+  fusion?: string
+  dense_score?: number
+  sparse_score?: number
+  matched_terms?: string[]
+  vector_name?: string | null
+  dense_rank?: number | null
+  sparse_rank?: number | null
+  bm25_term_scores?: { [term: string]: number }
+}
+
+export interface SearchResult {
+  namespace: string
+  id: string
+  score: number
+  dense_score: number
+  sparse_score: number
+  vector_name: string | null
+  matched_terms: string[]
+  dense_rank: number | null
+  sparse_rank: number | null
+  metadata: Metadata
+  explain?: ExplainDetails
+}
+
+export interface SearchResponse {
+  results: SearchResult[]
+  stats: SearchStats
+}
+
+export interface WriteOptions {
+  namespace?: string | null
+  sparse?: SparseVector | null
+  vectors?: NamedVectors | null
+}
+
+export interface BulkIngestOptions {
+  namespace?: string | null
+  batchSize?: number
+}
+
+export interface SearchOptions {
+  k?: number
+  filter?: Filter | null
+  namespace?: string | null
+  allNamespaces?: boolean
+  sparse?: SparseVector | null
+  denseWeight?: number
+  sparseWeight?: number
+  fetchK?: number
+  mmrLambda?: number | null
+  vectorName?: string | null
+  fusion?: 'linear' | 'rrf'
+  rrfK?: number
+  explain?: boolean
+  queryVectors?: { [name: string]: number[] } | null
+  vectorWeights?: { [name: string]: number } | null
+}
+
+export interface OpenOptions {
+  dimension?: number | null
+  readOnly?: boolean
+}
+
+export class VectLiteError extends Error {}
+
+export class Transaction {
+  count(): number
+  insert(id: string, vector: number[], metadata?: Metadata | null, options?: WriteOptions): void
+  upsert(id: string, vector: number[], metadata?: Metadata | null, options?: WriteOptions): void
+  insertMany(records: Record[], options?: { namespace?: string | null }): number
+  upsertMany(records: Record[], options?: { namespace?: string | null }): number
+  delete(id: string, options?: { namespace?: string | null }): boolean
+  deleteMany(ids: string[], options?: { namespace?: string | null }): number
+  commit(): void
+  rollback(): void
+}
+
+export class Database {
+  readonly path: string
+  readonly walPath: string
+  readonly dimension: number
+  readonly readOnly: boolean
+
+  count(): number
+  namespaces(): string[]
+  transaction(): Transaction
+  insert(id: string, vector: number[], metadata?: Metadata | null, options?: WriteOptions): void
+  upsert(id: string, vector: number[], metadata?: Metadata | null, options?: WriteOptions): void
+  insertMany(records: Record[], options?: { namespace?: string | null }): number
+  upsertMany(records: Record[], options?: { namespace?: string | null }): number
+  bulkIngest(records: Record[], options?: BulkIngestOptions): number
+  get(id: string, options?: { namespace?: string | null }): Record | null
+  delete(id: string, options?: { namespace?: string | null }): boolean
+  deleteMany(ids: string[], options?: { namespace?: string | null }): number
+  flush(): void
+  compact(): void
+  snapshot(dest: string): void
+  backup(dest: string): void
+  search(query?: number[] | null, options?: SearchOptions): SearchResult[]
+  searchWithStats(query?: number[] | null, options?: SearchOptions): SearchResponse
+}
+
+export class Store {
+  readonly root: string
+  createCollection(name: string, dimension: number): Database
+  openCollection(name: string): Database
+  openOrCreateCollection(name: string, dimension: number): Database
+  openCollectionReadOnly(name: string): Database
+  dropCollection(name: string): boolean
+  collections(): string[]
+}
+
+export function open(path: string, options?: OpenOptions): Database
+export function openStore(root: string): Store
+export function restore(source: string, dest: string): Database
+export function sparseTerms(text: string): SparseVector
+export function upsertText(
+  db: Database,
+  id: string,
+  text: string,
+  embed: (text: string) => ArrayLike<number>,
+  metadata?: Metadata | null,
+  options?: WriteOptions,
+): void
+export function searchText(
+  db: Database,
+  text: string,
+  embed: (text: string) => ArrayLike<number>,
+  options?: SearchOptions,
+): SearchResult[]
+export function searchTextWithStats(
+  db: Database,
+  text: string,
+  embed: (text: string) => ArrayLike<number>,
+  options?: SearchOptions,
+): SearchResponse

package/index.js

@@ -0,0 +1,284 @@
+'use strict'
+
+const native = require('./vectlite.node')
+
+const TOKEN_RE = /[a-z0-9]+/g
+
+class VectLiteError extends Error {
+  constructor(message, cause) {
+    super(message)
+    this.name = 'VectLiteError'
+    if (cause !== undefined) {
+      this.cause = cause
+    }
+  }
+}
+
+function wrapError(fn) {
+  try {
+    return fn()
+  } catch (error) {
+    if (error instanceof VectLiteError) {
+      throw error
+    }
+    throw new VectLiteError(error?.message ?? String(error), error)
+  }
+}
+
+function encode(value) {
+  return value == null ? null : JSON.stringify(value)
+}
+
+function decode(value) {
+  return value == null ? null : JSON.parse(value)
+}
+
+function asArray(values) {
+  return Array.from(values)
+}
+
+function normalizeWriteOptions(options = {}) {
+  return {
+    namespace: options.namespace ?? null,
+    sparse: options.sparse ?? null,
+    vectors: options.vectors ?? null,
+  }
+}
+
+function sparseTerms(text) {
+  const counts = {}
+  const tokens = String(text).toLowerCase().match(TOKEN_RE) ?? []
+  if (tokens.length === 0) {
+    return counts
+  }
+
+  const total = tokens.length
+  for (const token of tokens) {
+    counts[token] = (counts[token] ?? 0) + (1 / total)
+  }
+  return counts
+}
+
+class Transaction {
+  constructor(nativeTx) {
+    this._native = nativeTx
+  }
+
+  count() {
+    return wrapError(() => this._native.count())
+  }
+
+  insert(id, vector, metadata = null, options = {}) {
+    const { namespace, sparse, vectors } = normalizeWriteOptions(options)
+    return wrapError(() =>
+      this._native.insert(id, asArray(vector), encode(metadata), namespace, encode(sparse), encode(vectors)),
+    )
+  }
+
+  upsert(id, vector, metadata = null, options = {}) {
+    const { namespace, sparse, vectors } = normalizeWriteOptions(options)
+    return wrapError(() =>
+      this._native.upsert(id, asArray(vector), encode(metadata), namespace, encode(sparse), encode(vectors)),
+    )
+  }
+
+  insertMany(records, options = {}) {
+    return wrapError(() => this._native.insertMany(encode(records), options.namespace ?? null))
+  }
+
+  upsertMany(records, options = {}) {
+    return wrapError(() => this._native.upsertMany(encode(records), options.namespace ?? null))
+  }
+
+  delete(id, options = {}) {
+    return wrapError(() => this._native.delete(id, options.namespace ?? null))
+  }
+
+  deleteMany(ids, options = {}) {
+    return wrapError(() => this._native.deleteMany(ids, options.namespace ?? null))
+  }
+
+  commit() {
+    return wrapError(() => this._native.commit())
+  }
+
+  rollback() {
+    return wrapError(() => this._native.rollback())
+  }
+}
+
+class Database {
+  constructor(nativeDb) {
+    this._native = nativeDb
+  }
+
+  get path() {
+    return wrapError(() => this._native.path)
+  }
+
+  get walPath() {
+    return wrapError(() => this._native.walPath)
+  }
+
+  get dimension() {
+    return wrapError(() => this._native.dimension)
+  }
+
+  get readOnly() {
+    return wrapError(() => this._native.readOnly)
+  }
+
+  count() {
+    return wrapError(() => this._native.count())
+  }
+
+  namespaces() {
+    return wrapError(() => this._native.namespaces())
+  }
+
+  transaction() {
+    return wrapError(() => new Transaction(this._native.transaction()))
+  }
+
+  insert(id, vector, metadata = null, options = {}) {
+    const { namespace, sparse, vectors } = normalizeWriteOptions(options)
+    return wrapError(() =>
+      this._native.insert(id, asArray(vector), encode(metadata), namespace, encode(sparse), encode(vectors)),
+    )
+  }
+
+  upsert(id, vector, metadata = null, options = {}) {
+    const { namespace, sparse, vectors } = normalizeWriteOptions(options)
+    return wrapError(() =>
+      this._native.upsert(id, asArray(vector), encode(metadata), namespace, encode(sparse), encode(vectors)),
+    )
+  }
+
+  insertMany(records, options = {}) {
+    return wrapError(() => this._native.insertMany(encode(records), options.namespace ?? null))
+  }
+
+  upsertMany(records, options = {}) {
+    return wrapError(() => this._native.upsertMany(encode(records), options.namespace ?? null))
+  }
+
+  bulkIngest(records, options = {}) {
+    return wrapError(() =>
+      this._native.bulkIngest(encode(records), options.namespace ?? null, options.batchSize ?? 10_000),
+    )
+  }
+
+  get(id, options = {}) {
+    return wrapError(() => decode(this._native.get(id, options.namespace ?? null)))
+  }
+
+  delete(id, options = {}) {
+    return wrapError(() => this._native.delete(id, options.namespace ?? null))
+  }
+
+  deleteMany(ids, options = {}) {
+    return wrapError(() => this._native.deleteMany(ids, options.namespace ?? null))
+  }
+
+  flush() {
+    return wrapError(() => this._native.flush())
+  }
+
+  compact() {
+    return wrapError(() => this._native.compact())
+  }
+
+  snapshot(dest) {
+    return wrapError(() => this._native.snapshot(dest))
+  }
+
+  backup(dest) {
+    return wrapError(() => this._native.backup(dest))
+  }
+
+  search(query = null, options = {}) {
+    return wrapError(() => decode(this._native.search(query == null ? null : asArray(query), encode(options))))
+  }
+
+  searchWithStats(query = null, options = {}) {
+    return wrapError(() =>
+      decode(this._native.searchWithStats(query == null ? null : asArray(query), encode(options))),
+    )
+  }
+}
+
+class Store {
+  constructor(nativeStore) {
+    this._native = nativeStore
+  }
+
+  get root() {
+    return wrapError(() => this._native.root)
+  }
+
+  createCollection(name, dimension) {
+    return wrapError(() => new Database(this._native.createCollection(name, dimension)))
+  }
+
+  openCollection(name) {
+    return wrapError(() => new Database(this._native.openCollection(name)))
+  }
+
+  openOrCreateCollection(name, dimension) {
+    return wrapError(() => new Database(this._native.openOrCreateCollection(name, dimension)))
+  }
+
+  openCollectionReadOnly(name) {
+    return wrapError(() => new Database(this._native.openCollectionReadOnly(name)))
+  }
+
+  dropCollection(name) {
+    return wrapError(() => this._native.dropCollection(name))
+  }
+
+  collections() {
+    return wrapError(() => this._native.collections())
+  }
+}
+
+function open(path, options = {}) {
+  return wrapError(() => new Database(native.open(path, options.dimension ?? null, options.readOnly ?? false)))
+}
+
+function openStore(root) {
+  return wrapError(() => new Store(native.openStore(root)))
+}
+
+function restore(source, dest) {
+  return wrapError(() => new Database(native.restore(source, dest)))
+}
+
+function upsertText(db, id, text, embed, metadata = null, options = {}) {
+  const payload = { ...(metadata ?? {}) }
+  if (payload.text === undefined) {
+    payload.text = text
+  }
+  return db.upsert(id, asArray(embed(text)), payload, { ...options, sparse: sparseTerms(text) })
+}
+
+function searchText(db, text, embed, options = {}) {
+  return db.search(asArray(embed(text)), { ...options, sparse: sparseTerms(text) })
+}
+
+function searchTextWithStats(db, text, embed, options = {}) {
+  return db.searchWithStats(asArray(embed(text)), { ...options, sparse: sparseTerms(text) })
+}
+
+module.exports = {
+  Database,
+  Store,
+  Transaction,
+  VectLiteError,
+  open,
+  openStore,
+  restore,
+  sparseTerms,
+  upsertText,
+  searchText,
+  searchTextWithStats,
+}

package/native/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "vectlite-node"
+version = "0.1.3"
+edition = "2024"
+license = "MIT"
+description = "Node.js bindings for vectlite."
+homepage = "https://github.com/mcsedition-hub/vectlite"
+repository = "https://github.com/mcsedition-hub/vectlite"
+documentation = "https://github.com/mcsedition-hub/vectlite/tree/main/bindings/node"
+build = "build.rs"
+
+[lib]
+name = "vectlite_node"
+crate-type = ["cdylib"]
+
+[dependencies]
+napi = { version = "3.5.2", default-features = false, features = ["napi8"] }
+napi-derive = "3.3.0"
+serde_json = "1.0"
+vectlite = { package = "vectlite-core", path = "./vectlite-core" }
+
+[build-dependencies]
+napi-build = "2.2.3"

package/native/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    napi_build::setup();
+}