quiver-client 0.22.0

package/dist/client.js

@@ -0,0 +1,431 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+//! A small, dependency-free REST client for Quiver.
+//
+// Mirrors the server's HTTP contract (docs/api/rest-grpc.md): collection CRUD,
+// point upsert/delete/get, and filtered k-NN search, plus the per-collection
+// index choice (the memory-frugal `disk_vamana` path). Embeddings are produced
+// by the caller — Quiver is model-agnostic. Uses the global `fetch`, so it runs
+// on Node >= 20 and modern runtimes with no dependencies.
+const DEFAULT_BASE_URL = "http://127.0.0.1:6333";
+const DEFAULT_TIMEOUT_MS = 30_000;
+/** An error from the Quiver server or the transport. `status` is the HTTP code
+ * when the failure came from the server, or `undefined` for a transport error. */
+export class QuiverError extends Error {
+    status;
+    constructor(message, status) {
+        super(message);
+        this.name = "QuiverError";
+        this.status = status;
+    }
+}
+/** A synchronous-feeling, promise-based Quiver REST client. */
+export class Client {
+    #baseUrl;
+    #headers;
+    #timeoutMs;
+    #fetch;
+    constructor(baseUrl = DEFAULT_BASE_URL, opts = {}) {
+        this.#baseUrl = baseUrl.replace(/\/+$/, "");
+        this.#headers = { "content-type": "application/json" };
+        if (opts.apiKey) {
+            this.#headers["authorization"] = `Bearer ${opts.apiKey}`;
+        }
+        this.#timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+        this.#fetch = opts.fetch ?? globalThis.fetch.bind(globalThis);
+    }
+    // --- collections ---
+    /** Create a collection. Rejects with {@link QuiverError} if the name is taken
+     * or the index/metric combination is unsupported. */
+    async createCollection(name, dim, opts = {}) {
+        const body = { name, dim, metric: opts.metric ?? "l2" };
+        if (opts.index)
+            body["index"] = opts.index;
+        if (opts.pqSubspaces !== undefined)
+            body["pq_subspaces"] = opts.pqSubspaces;
+        if (opts.filterable && opts.filterable.length > 0) {
+            body["filterable"] = opts.filterable.map((f) => ({
+                path: f.path,
+                field_type: f.fieldType ?? "keyword",
+            }));
+        }
+        if (opts.multivector)
+            body["multivector"] = true;
+        if (opts.vectorEncryption && opts.vectorEncryption !== "none") {
+            body["vector_encryption"] = opts.vectorEncryption;
+        }
+        return toCollection(await this.#json("POST", "/v1/collections", body));
+    }
+    /** List all collections. */
+    async listCollections() {
+        const body = (await this.#json("GET", "/v1/collections"));
+        return body.map(toCollection);
+    }
+    /** Fetch one collection's metadata. */
+    async getCollection(name) {
+        return toCollection(await this.#json("GET", `/v1/collections/${encodeURIComponent(name)}`));
+    }
+    /** Delete a collection; resolves to whether it existed. */
+    async deleteCollection(name) {
+        const body = (await this.#json("DELETE", `/v1/collections/${encodeURIComponent(name)}`));
+        return Boolean(body.existed);
+    }
+    // --- points ---
+    /** Insert or replace points; resolves to the number upserted. */
+    async upsert(collection, points) {
+        const body = { points: points.map(pointDict) };
+        const res = (await this.#json("POST", `/v1/collections/${encodeURIComponent(collection)}/points`, body));
+        return Number(res.upserted ?? 0);
+    }
+    /** Delete points by id; resolves to the number deleted. */
+    async deletePoints(collection, ids) {
+        const res = (await this.#json("DELETE", `/v1/collections/${encodeURIComponent(collection)}/points`, { ids }));
+        return Number(res.deleted ?? 0);
+    }
+    /** Fetch a point by id, or `null` if it does not exist. */
+    async getPoint(collection, id) {
+        const path = `/v1/collections/${encodeURIComponent(collection)}/points/${encodeURIComponent(id)}`;
+        const resp = await this.#send("GET", path);
+        if (resp.status === 404)
+            return null;
+        await throwForStatus(resp);
+        const body = (await resp.json());
+        return { id: body.id, score: 0, payload: body.payload, vector: body.vector };
+    }
+    /** Search for the `k` nearest points to `vector`, nearest first. */
+    async search(collection, vector, opts = {}) {
+        const body = {
+            vector,
+            k: opts.k ?? 10,
+            ef_search: opts.efSearch ?? 64,
+            with_payload: opts.withPayload ?? true,
+            with_vector: opts.withVector ?? false,
+        };
+        if (opts.filter !== undefined)
+            body["filter"] = opts.filter;
+        const res = (await this.#json("POST", `/v1/collections/${encodeURIComponent(collection)}/query`, body));
+        return (res.matches ?? []).map((m) => ({
+            id: m.id,
+            score: m.score,
+            payload: m.payload,
+            vector: m.vector,
+        }));
+    }
+    /** Hybrid search fused with Reciprocal Rank Fusion (ADR-0043/0046). Provide a
+     * dense `vector`, a `sparse` query vector, and/or a full-text `queryText` (scored
+     * by BM25) — at least one is required. The same payload `filter` applies to every
+     * side. */
+    async hybridSearch(collection, opts = {}) {
+        if (opts.vector === undefined && opts.sparse === undefined && opts.queryText === undefined) {
+            throw new QuiverError("hybridSearch requires a dense vector, a sparse vector, or a text query");
+        }
+        const body = {
+            k: opts.k ?? 10,
+            ef_search: opts.efSearch ?? 64,
+            rrf_k0: opts.rrfK0 ?? 60,
+            with_payload: opts.withPayload ?? true,
+            with_vector: opts.withVector ?? false,
+        };
+        if (opts.vector !== undefined)
+            body["vector"] = opts.vector;
+        if (opts.queryText !== undefined)
+            body["query_text"] = opts.queryText;
+        if (opts.sparse !== undefined) {
+            body["sparse_indices"] = opts.sparse.indices;
+            body["sparse_values"] = opts.sparse.values;
+        }
+        if (opts.filter !== undefined)
+            body["filter"] = opts.filter;
+        const res = (await this.#json("POST", `/v1/collections/${encodeURIComponent(collection)}/query/hybrid`, body));
+        return (res.matches ?? []).map((m) => ({
+            id: m.id,
+            score: m.score,
+            payload: m.payload,
+            vector: m.vector,
+        }));
+    }
+    /** Embed each point's `text` server-side and upsert it (ADR-0047); the text is
+     * also indexed for BM25. Requires an `[embedding.<collection>]` provider on the
+     * server. Resolves to the number upserted. */
+    async upsertText(collection, points) {
+        const body = {
+            points: points.map((p) => ({
+                id: p.id,
+                text: p.text,
+                ...(p.payload !== undefined ? { payload: p.payload } : {}),
+            })),
+        };
+        const res = (await this.#json("POST", `/v1/collections/${encodeURIComponent(collection)}/points:text`, body));
+        return Number(res.upserted ?? 0);
+    }
+    /** Embed `text` server-side and search dense ⊕ BM25, optionally reranking the
+     * candidate pool in one call (ADR-0047). Requires an `[embedding.<collection>]`
+     * provider (and a `[rerank.<collection>]` provider for `rerank: true`). */
+    async searchText(collection, text, opts = {}) {
+        const body = {
+            text,
+            k: opts.k ?? 10,
+            ef_search: opts.efSearch ?? 64,
+            rrf_k0: opts.rrfK0 ?? 60,
+            with_payload: opts.withPayload ?? true,
+            with_vector: opts.withVector ?? false,
+            rerank: opts.rerank ?? false,
+        };
+        if (opts.filter !== undefined)
+            body["filter"] = opts.filter;
+        const res = (await this.#json("POST", `/v1/collections/${encodeURIComponent(collection)}/query/text`, body));
+        return (res.matches ?? []).map((m) => ({
+            id: m.id,
+            score: m.score,
+            payload: m.payload,
+            vector: m.vector,
+        }));
+    }
+    /** Fetch points without ranking; an optional payload `filter` narrows the set
+     * and `limit` bounds it. The retrieval path for `client_side`-encrypted
+     * collections (ADR-0032): the server returns the entitled set (each payload
+     * carries the sealed vector under `__quiver_vec__`) and you decrypt and rank
+     * locally (see {@link searchClientSide}). Also a general list-points call for
+     * any collection; returned matches carry `score` 0. */
+    async fetch(collection, opts = {}) {
+        const body = {
+            limit: opts.limit ?? 100,
+            with_payload: opts.withPayload ?? true,
+            with_vector: opts.withVector ?? false,
+        };
+        if (opts.filter !== undefined)
+            body["filter"] = opts.filter;
+        const res = (await this.#json("POST", `/v1/collections/${encodeURIComponent(collection)}/fetch`, body));
+        return (res.points ?? []).map((p) => ({
+            id: p.id,
+            score: 0,
+            payload: p.payload,
+            vector: p.vector,
+        }));
+    }
+    /** Upsert a large (sync or async) iterable of points in server-friendly
+     * batches; resolves to the total upserted. `batch` must stay within the
+     * server's `max_batch_size` (ADR-0040, default 1000). `onProgress` is called
+     * with the running total after each batch (may be sync or async). Mirrors the
+     * Python `upsert_iter`. */
+    async upsertIter(collection, points, opts = {}) {
+        const batch = opts.batch ?? 500;
+        let total = 0;
+        let chunk = [];
+        const flush = async () => {
+            if (chunk.length === 0)
+                return;
+            total += await this.upsert(collection, chunk);
+            chunk = [];
+            if (opts.onProgress)
+                await opts.onProgress(total);
+        };
+        // `for await` iterates sync and async iterables alike.
+        for await (const p of points) {
+            chunk.push(p);
+            if (chunk.length >= batch)
+                await flush();
+        }
+        await flush();
+        return total;
+    }
+    /** Yield points page by page, for export / re-embedding. The REST fetch is
+     * limit-bounded without a server cursor, so this returns up to `batch` points
+     * in one page — pass a narrowing `filter` for large collections (a server-side
+     * scroll cursor is a follow-up). Mirrors the Python async `scroll`. */
+    async *scroll(collection, opts = {}) {
+        const page = await this.fetch(collection, {
+            filter: opts.filter,
+            limit: opts.batch ?? 500,
+            withPayload: opts.withPayload ?? true,
+            withVector: opts.withVector ?? false,
+        });
+        for (const m of page)
+            yield m;
+    }
+    /** Delete every point matching `filter`; resolves to the number deleted.
+     * Fetches matching ids (paged by `batch`) and deletes them until none remain —
+     * useful for GDPR erasure and re-indexing. Mirrors the Python `delete_by_filter`. */
+    async deleteByFilter(collection, filter, opts = {}) {
+        const batch = opts.batch ?? 500;
+        let total = 0;
+        for (;;) {
+            const page = await this.fetch(collection, { filter, limit: batch, withPayload: false });
+            const ids = page.map((m) => m.id);
+            if (ids.length === 0)
+                return total;
+            total += await this.deletePoints(collection, ids);
+            if (ids.length < batch)
+                return total;
+        }
+    }
+    /** Take a consistent online snapshot (backup) of the whole database into a
+     * server-local directory, which must not already exist (ADR-0050); admin-only.
+     * Resolves to the captured manifest version and the file/byte counts. */
+    async snapshot(destination) {
+        const res = (await this.#json("POST", "/v1/snapshot", { destination }));
+        return {
+            manifestVersion: Number(res.manifest_version ?? 0),
+            files: Number(res.files ?? 0),
+            bytes: Number(res.bytes ?? 0),
+        };
+    }
+    /** Nearest-neighbour search over a `client_side`-encrypted collection (ADR-0032),
+     * done entirely client-side: {@link fetch} the (optionally filtered) candidate
+     * set, decrypt each vector with `cipher` (a `VectorCipher` from
+     * `quiver-client/vector`), rank by metric, and return the top `k`. The server
+     * never ranks and never sees the key. This mode suits small/medium or
+     * pre-filtered collections; `candidateLimit` bounds how many points are fetched
+     * before ranking. Each result carries the decrypted `vector` and a `score` under
+     * the chosen metric. */
+    async searchClientSide(collection, query, cipher, opts = {}) {
+        const metric = opts.metric ?? "l2";
+        const points = await this.fetch(collection, {
+            filter: opts.filter,
+            limit: opts.candidateLimit ?? 10000,
+            withPayload: true,
+        });
+        const ranked = points.map((m) => {
+            const vector = cipher.open(m.payload);
+            const [ordering, score] = clientSideScore(metric, query, vector);
+            return { ordering, match: { id: m.id, score, payload: m.payload, vector } };
+        });
+        ranked.sort((a, b) => a.ordering - b.ordering);
+        return ranked.slice(0, opts.k ?? 10).map((r) => r.match);
+    }
+    // --- documents (multi-vector / late interaction) ---
+    /** Insert or replace multi-vector documents; resolves to the number upserted. */
+    async upsertDocuments(collection, documents) {
+        const body = { documents: documents.map(documentDict) };
+        const res = (await this.#json("POST", `/v1/collections/${encodeURIComponent(collection)}/documents`, body));
+        return Number(res.upserted ?? 0);
+    }
+    /** Delete multi-vector documents by id; resolves to the number deleted. */
+    async deleteDocuments(collection, ids) {
+        const res = (await this.#json("DELETE", `/v1/collections/${encodeURIComponent(collection)}/documents`, { ids }));
+        return Number(res.deleted ?? 0);
+    }
+    /** Rank documents by MaxSim late interaction against the `query` token set. */
+    async searchMultiVector(collection, query, opts = {}) {
+        const body = {
+            query,
+            k: opts.k ?? 10,
+            ef_search: opts.efSearch ?? 64,
+            with_payload: opts.withPayload ?? true,
+            with_vector: opts.withVector ?? false,
+        };
+        if (opts.filter !== undefined)
+            body["filter"] = opts.filter;
+        const res = (await this.#json("POST", `/v1/collections/${encodeURIComponent(collection)}/documents/query`, body));
+        return (res.matches ?? []).map((m) => ({
+            id: m.id,
+            score: m.score,
+            payload: m.payload,
+            vectors: m.vectors,
+        }));
+    }
+    // --- health ---
+    /** Whether the server's liveness probe succeeds. */
+    async healthz() {
+        try {
+            const resp = await this.#send("GET", "/healthz");
+            return resp.ok;
+        }
+        catch {
+            return false;
+        }
+    }
+    // --- internals ---
+    async #send(method, path, body) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.#timeoutMs);
+        try {
+            return await this.#fetch(`${this.#baseUrl}${path}`, {
+                method,
+                headers: this.#headers,
+                body: body === undefined ? undefined : JSON.stringify(body),
+                signal: controller.signal,
+            });
+        }
+        catch (err) {
+            throw new QuiverError(`request to ${path} failed: ${String(err)}`);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    async #json(method, path, body) {
+        const resp = await this.#send(method, path, body);
+        await throwForStatus(resp);
+        return resp.json();
+    }
+}
+function clientSideScore(metric, query, vector) {
+    if (metric === "l2") {
+        let d = 0;
+        for (let i = 0; i < query.length; i++) {
+            const diff = (query[i] ?? 0) - (vector[i] ?? 0);
+            d += diff * diff;
+        }
+        return [d, d];
+    }
+    let dot = 0;
+    for (let i = 0; i < query.length; i++)
+        dot += (query[i] ?? 0) * (vector[i] ?? 0);
+    if (metric === "dot")
+        return [-dot, dot];
+    let nq = 0;
+    let nv = 0;
+    for (let i = 0; i < query.length; i++)
+        nq += (query[i] ?? 0) ** 2;
+    for (let i = 0; i < vector.length; i++)
+        nv += (vector[i] ?? 0) ** 2;
+    const sim = dot / ((Math.sqrt(nq) || 1) * (Math.sqrt(nv) || 1));
+    return [-sim, sim];
+}
+function toCollection(body) {
+    const b = body;
+    return {
+        name: String(b["name"]),
+        dim: Number(b["dim"]),
+        metric: String(b["metric"]),
+        count: Number(b["count"] ?? 0),
+        index: b["index"] ?? "hnsw",
+        pqSubspaces: b["pq_subspaces"] === undefined ? undefined : Number(b["pq_subspaces"]),
+        filterable: Array.isArray(b["filterable"])
+            ? b["filterable"].map((f) => ({
+                path: String(f["path"]),
+                fieldType: f["field_type"] ?? "keyword",
+            }))
+            : [],
+        multivector: Boolean(b["multivector"]),
+        vectorEncryption: typeof b["vector_encryption"] === "string"
+            ? b["vector_encryption"]
+            : "none",
+    };
+}
+function documentDict(doc) {
+    const out = { id: doc.id, vectors: doc.vectors };
+    if (doc.payload !== undefined)
+        out["payload"] = doc.payload;
+    return out;
+}
+function pointDict(point) {
+    const out = { id: point.id, vector: point.vector };
+    if (point.payload !== undefined && point.payload !== null)
+        out["payload"] = point.payload;
+    return out;
+}
+async function throwForStatus(resp) {
+    if (resp.status < 400)
+        return;
+    let detail;
+    try {
+        const body = (await resp.clone().json());
+        detail = body["detail"] ?? body["title"];
+    }
+    catch {
+        detail = undefined;
+    }
+    const fallback = (await resp.text().catch(() => "")) || `HTTP ${resp.status}`;
+    throw new QuiverError(detail ?? fallback, resp.status);
+}

package/dist/dcpe.d.ts

@@ -0,0 +1,57 @@
+/** DCPE initialisation-vector length in bytes (a 96-bit ChaCha20 nonce). */
+export declare const IV_LEN = 12;
+/** DCPE integrity-tag length in bytes (full HMAC-SHA256 output). */
+export declare const TAG_LEN = 32;
+/** An error from DCPE encryption, decryption, or construction. */
+export declare class DcpeError extends Error {
+    constructor(message: string);
+}
+/** A DCPE-encrypted vector: the ciphertext (upserted and searched like any
+ * vector), the IV seeding its perturbation, and an HMAC-SHA256 integrity tag. */
+export interface EncryptedVector {
+    ciphertext: number[];
+    iv: Uint8Array;
+    tag: Uint8Array;
+}
+/** A fixed, ordering-preserving global affine normalisation (ADR-0035).
+ *
+ * Maps a plaintext `m` to `(m - shift) * scale` before encryption, where `shift`
+ * is a per-dimension translation and `scale` is a **single** positive scalar. Both
+ * steps preserve the L2 distance-comparison ordering (a uniform shift cancels in
+ * any difference; a single positive scalar scales every distance by the same
+ * factor) and are invertible. Supply it once from a one-time measurement of your
+ * corpus and reuse it for the data *and* the queries.
+ *
+ * Per-axis variance whitening (a different scale per dimension) is anisotropic,
+ * re-weights the L2 distance, and so breaks the ordering — it is intentionally not
+ * expressible here. See ADR-0035. */
+export declare class Normalization {
+    readonly shift: number[];
+    readonly scale: number;
+    private constructor();
+    /** Build a normalisation from a per-dimension shift and a single positive scale. */
+    static create(shift: ArrayLike<number>, scale: number): Normalization;
+}
+/** A client-held DCPE key bound to one approximation factor (ADR-0031/0035).
+ *
+ * Construct one cipher per `(key, approximationFactor[, normalization])` and reuse
+ * it; the same factor (and normalisation) must be used for the data and the queries
+ * searched against it. */
+export declare class DcpeCipher {
+    #private;
+    private constructor();
+    /** Build a cipher from a raw 256-bit (32-byte) key. */
+    static fromBytes(key: Uint8Array, approximationFactor: number, normalization?: Normalization | null): DcpeCipher;
+    /** Build a cipher from a 64-character hex-encoded 256-bit key. */
+    static fromHex(hex: string, approximationFactor: number, normalization?: Normalization | null): DcpeCipher;
+    /** The secret, key-derived scaling factor `s ∈ [1, 2)`. Part of the key. */
+    get scale(): number;
+    /** The approximation factor this cipher was built with. */
+    get approximationFactor(): number;
+    /** Encrypt a vector for storage with a fresh random IV. */
+    encrypt(vector: ArrayLike<number>): EncryptedVector;
+    /** Encrypt a query vector for searching against DCPE-encrypted data. */
+    encryptQuery(vector: ArrayLike<number>): number[];
+    /** Verify the integrity tag (constant-time) and recover the plaintext. */
+    decrypt(sealed: EncryptedVector): number[];
+}