presubmit 0.1.0

package/vendor/lancedb/dist/table.js

@@ -0,0 +1,498 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Branches = exports.LocalTable = exports.Table = void 0;
+const arrow_1 = require("./arrow");
+const registry_1 = require("./embedding/registry");
+const merge_1 = require("./merge");
+const query_1 = require("./query");
+const sanitize_1 = require("./sanitize");
+const util_1 = require("./util");
+/**
+ * A Table is a collection of Records in a LanceDB Database.
+ *
+ * A Table object is expected to be long lived and reused for multiple operations.
+ * Table objects will cache a certain amount of index data in memory.  This cache
+ * will be freed when the Table is garbage collected.  To eagerly free the cache you
+ * can call the `close` method.  Once the Table is closed, it cannot be used for any
+ * further operations.
+ *
+ * Tables are created using the methods {@link Connection#createTable}
+ * and {@link Connection#createEmptyTable}. Existing tables are opened
+ * using {@link Connection#openTable}.
+ *
+ * Closing a table is optional.  It not closed, it will be closed when it is garbage
+ * collected.
+ *
+ * @hideconstructor
+ */
+class Table {
+    [Symbol.for("nodejs.util.inspect.custom")]() {
+        return this.display();
+    }
+}
+exports.Table = Table;
+class LocalTable extends Table {
+    inner;
+    constructor(inner) {
+        super();
+        this.inner = inner;
+    }
+    get name() {
+        return this.inner.name;
+    }
+    isOpen() {
+        return this.inner.isOpen();
+    }
+    close() {
+        this.inner.close();
+    }
+    display() {
+        return this.inner.display();
+    }
+    async getEmbeddingFunctions(inner = this.inner) {
+        const schemaBuf = await inner.schema();
+        const schema = (0, arrow_1.tableFromIPC)(schemaBuf).schema;
+        const registry = (0, registry_1.getRegistry)();
+        return registry.parseFunctions(schema.metadata);
+    }
+    /** Get the schema of the table. */
+    async schema() {
+        const schemaBuf = await this.inner.schema();
+        const tbl = (0, arrow_1.tableFromIPC)(schemaBuf);
+        return tbl.schema;
+    }
+    async add(data, options) {
+        const mode = options?.mode ?? "append";
+        const schema = await this.schema();
+        const buffer = await (0, arrow_1.fromDataToBuffer)(data, undefined, schema);
+        // Wrap the user callback so a thrown error doesn't surface as an
+        // unhandled exception (the callback fires from a napi threadsafe
+        // function — exceptions there crash the process).
+        const userProgress = options?.progress;
+        const progress = userProgress
+            ? (p) => {
+                try {
+                    userProgress(p);
+                }
+                catch (e) {
+                    console.warn("Table.add progress callback threw:", e);
+                }
+            }
+            : undefined;
+        return await this.inner.add(buffer, mode, progress);
+    }
+    async update(optsOrUpdates, options) {
+        const isValues = "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
+        const isValuesSql = "valuesSql" in optsOrUpdates &&
+            typeof optsOrUpdates.valuesSql !== "string";
+        const isMap = (obj) => {
+            return obj instanceof Map;
+        };
+        let predicate;
+        let columns;
+        switch (true) {
+            case isMap(optsOrUpdates):
+                columns = Array.from(optsOrUpdates.entries());
+                predicate = options?.where;
+                break;
+            case isValues && isMap(optsOrUpdates.values):
+                columns = Array.from(optsOrUpdates.values.entries()).map(([k, v]) => [
+                    k,
+                    (0, util_1.toSQL)(v),
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            case isValues && !isMap(optsOrUpdates.values):
+                columns = Object.entries(optsOrUpdates.values).map(([k, v]) => [
+                    k,
+                    (0, util_1.toSQL)(v),
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            case isValuesSql && isMap(optsOrUpdates.valuesSql):
+                columns = Array.from(optsOrUpdates.valuesSql.entries());
+                predicate = optsOrUpdates.where;
+                break;
+            case isValuesSql && !isMap(optsOrUpdates.valuesSql):
+                columns = Object.entries(optsOrUpdates.valuesSql).map(([k, v]) => [
+                    k,
+                    v,
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            default:
+                columns = Object.entries(optsOrUpdates);
+                predicate = options?.where;
+        }
+        return await this.inner.update(predicate, columns);
+    }
+    async countRows(filter) {
+        return await this.inner.countRows(filter);
+    }
+    async delete(predicate) {
+        return await this.inner.delete(predicate);
+    }
+    async createIndex(column, options) {
+        // Bit of a hack to get around the fact that TS has no package-scope.
+        // biome-ignore lint/suspicious/noExplicitAny: skip
+        const nativeIndex = options?.config?.inner;
+        await this.inner.createIndex(nativeIndex, column, options?.replace, options?.waitTimeoutSeconds, options?.name, options?.train);
+    }
+    async createIndexAsync(column, options) {
+        // biome-ignore lint/suspicious/noExplicitAny: skip
+        const nativeIndex = options?.config?.inner;
+        return await this.inner.createIndexAsync(nativeIndex, column, options?.replace, options?.waitTimeoutSeconds, options?.name, options?.train);
+    }
+    async dropIndex(name) {
+        await this.inner.dropIndex(name);
+    }
+    async prewarmIndex(name) {
+        await this.inner.prewarmIndex(name);
+    }
+    async prewarmData(columns) {
+        await this.inner.prewarmData(columns);
+    }
+    async waitForIndex(indexNames, timeoutSeconds) {
+        await this.inner.waitForIndex(indexNames, timeoutSeconds);
+    }
+    takeOffsets(offsets) {
+        return new query_1.TakeQuery(this.inner.takeOffsets(offsets));
+    }
+    takeRowIds(rowIds) {
+        const ids = rowIds.map((id) => {
+            if (typeof id === "bigint") {
+                return id;
+            }
+            if (!Number.isInteger(id)) {
+                throw new Error("Row id must be an integer (or bigint)");
+            }
+            if (id < 0) {
+                throw new Error("Row id cannot be negative");
+            }
+            if (!Number.isSafeInteger(id)) {
+                throw new Error("Row id is too large for number; use bigint instead");
+            }
+            return BigInt(id);
+        });
+        return new query_1.TakeQuery(this.inner.takeRowIds(ids));
+    }
+    query() {
+        return new query_1.Query(this.inner);
+    }
+    search(query, queryType = "auto", ftsColumns) {
+        if (typeof query !== "string" && !(0, query_1.instanceOfFullTextQuery)(query)) {
+            if (queryType === "fts") {
+                throw new Error("Cannot perform full text search on a vector query");
+            }
+            return this.vectorSearch(query);
+        }
+        // If the query is a string, we need to determine if it is a vector query or a full text search query
+        if (queryType === "fts") {
+            return this.query().fullTextSearch(query, {
+                columns: ftsColumns,
+            });
+        }
+        if (queryType === "auto") {
+            if ((0, query_1.instanceOfFullTextQuery)(query)) {
+                return this.query().fullTextSearch(query, {
+                    columns: ftsColumns,
+                });
+            }
+            const columns = typeof ftsColumns === "string" ? [ftsColumns] : (ftsColumns ?? null);
+            return (0, query_1.createAutoQuery)(this.inner, query, columns, async (metadata) => {
+                const functions = await (0, registry_1.getRegistry)().parseFunctions(new Map([["embedding_functions", metadata]]));
+                // TODO: Support multiple embedding functions
+                const embeddingFunc = functions
+                    .values()
+                    .next().value;
+                // The route only calls this callback when embedding metadata exists.
+                // parseFunctions either yields a provider or reports malformed metadata.
+                if (!embeddingFunc)
+                    throw new Error("Invalid embedding function metadata");
+                return await embeddingFunc.function.computeQueryEmbeddings(query);
+            });
+        }
+        const queryPromise = this.getEmbeddingFunctions().then(async (functions) => {
+            // TODO: Support multiple embedding functions
+            const embeddingFunc = functions
+                .values()
+                .next().value;
+            if (!embeddingFunc) {
+                return Promise.reject(new Error("No embedding functions are defined in the table"));
+            }
+            return await embeddingFunc.function.computeQueryEmbeddings(query);
+        });
+        return this.query().nearestTo(queryPromise);
+    }
+    vectorSearch(vector) {
+        if ((0, arrow_1.isMultiVector)(vector)) {
+            const query = this.query().nearestTo(vector[0]);
+            for (const v of vector.slice(1)) {
+                query.addQueryVector(v);
+            }
+            return query;
+        }
+        return this.query().nearestTo(vector);
+    }
+    // TODO: Support BatchUDF
+    async addColumns(newColumnTransforms) {
+        // Columns defined by an expression are declared, not materialized here.
+        if (typeof newColumnTransforms === "object" &&
+            !Array.isArray(newColumnTransforms) &&
+            "computed" in newColumnTransforms) {
+            return await this.inner.addComputedColumns(newColumnTransforms.computed);
+        }
+        // Handle single Field -> convert to array of Fields
+        if (newColumnTransforms instanceof arrow_1.Field) {
+            newColumnTransforms = [newColumnTransforms];
+        }
+        // Handle array of Fields -> convert to Schema
+        if (Array.isArray(newColumnTransforms) &&
+            newColumnTransforms.length > 0 &&
+            newColumnTransforms[0] instanceof arrow_1.Field) {
+            const fields = newColumnTransforms;
+            newColumnTransforms = new arrow_1.Schema(fields);
+        }
+        // Handle Schema -> use schema-based approach
+        if (newColumnTransforms instanceof arrow_1.Schema) {
+            const schema = newColumnTransforms;
+            // Convert schema to buffer using Arrow IPC format
+            const emptyTable = (0, arrow_1.makeEmptyTable)(schema);
+            const schemaBuf = await (0, arrow_1.fromTableToBuffer)(emptyTable);
+            return await this.inner.addColumnsWithSchema(schemaBuf);
+        }
+        // Handle SQL expressions (existing functionality)
+        if (Array.isArray(newColumnTransforms)) {
+            return await this.inner.addColumns(newColumnTransforms);
+        }
+        throw new Error("Invalid input type for addColumns");
+    }
+    async refreshColumn(column) {
+        return await this.inner.refreshColumn(column);
+    }
+    async refreshColumnAsync(column) {
+        return await this.inner.refreshColumnAsync(column);
+    }
+    async refreshMaterializedView(full, sourceVersion) {
+        return await this.inner.refreshMaterializedView(full, sourceVersion);
+    }
+    async alterColumns(columnAlterations) {
+        const processedAlterations = columnAlterations.map((alteration) => {
+            if (typeof alteration.dataType === "string") {
+                return {
+                    ...alteration,
+                    dataType: JSON.stringify({ type: alteration.dataType }),
+                };
+            }
+            else if (alteration.dataType === undefined) {
+                return {
+                    ...alteration,
+                    dataType: undefined,
+                };
+            }
+            else {
+                const dataType = (0, sanitize_1.sanitizeType)(alteration.dataType);
+                return {
+                    ...alteration,
+                    dataType: JSON.stringify((0, arrow_1.dataTypeToJson)(dataType)),
+                };
+            }
+        });
+        return await this.inner.alterColumns(processedAlterations);
+    }
+    async updateFieldMetadata(updates) {
+        return await this.inner.updateFieldMetadata(updates);
+    }
+    async dropColumns(columnNames) {
+        return await this.inner.dropColumns(columnNames);
+    }
+    async setUnenforcedPrimaryKey(columns) {
+        const cols = typeof columns === "string" ? [columns] : columns;
+        return await this.inner.setUnenforcedPrimaryKey(cols);
+    }
+    async setLsmWriteSpec(spec) {
+        return await this.inner.setLsmWriteSpec(spec);
+    }
+    async unsetLsmWriteSpec() {
+        return await this.inner.unsetLsmWriteSpec();
+    }
+    async getLsmWriteSpec() {
+        // The native binding types `specType` as a plain `string`; narrow it back
+        // to the public union. The Rust `From` impl only ever emits one of the
+        // three valid values, so the cast is safe.
+        return ((await this.inner.getLsmWriteSpec()) ?? undefined);
+    }
+    async closeLsmWriters() {
+        return await this.inner.closeLsmWriters();
+    }
+    async flushLsm() {
+        return await this.inner.flushLsm();
+    }
+    async compactLsm() {
+        return await this.inner.compactLsm();
+    }
+    async checkpointLsm() {
+        return await this.inner.checkpointLsm();
+    }
+    async getLsmStats(includeGenerationRows = false) {
+        return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined;
+    }
+    async version() {
+        return await this.inner.version();
+    }
+    async checkout(version) {
+        if (typeof version === "string") {
+            return this.inner.checkoutTag(version);
+        }
+        return this.inner.checkout(version);
+    }
+    async checkoutLatest() {
+        await this.inner.checkoutLatest();
+    }
+    async listVersions() {
+        return (await this.inner.listVersions()).map((version) => ({
+            version: version.version,
+            timestamp: new Date(version.timestamp / 1000),
+            metadata: version.metadata,
+        }));
+    }
+    async restore() {
+        await this.inner.restore();
+    }
+    async tags() {
+        return await this.inner.tags();
+    }
+    async branches() {
+        return new Branches(await this.inner.branches());
+    }
+    currentBranch() {
+        return this.inner.currentBranch() ?? null;
+    }
+    async optimize(options) {
+        let cleanupOlderThanMs;
+        if (options?.cleanupOlderThan !== undefined &&
+            options?.cleanupOlderThan !== null) {
+            cleanupOlderThanMs =
+                new Date().getTime() - options.cleanupOlderThan.getTime();
+        }
+        return await this.inner.optimize(cleanupOlderThanMs, options?.deleteUnverified);
+    }
+    async listIndices() {
+        return await this.inner.listIndices();
+    }
+    async tokenize(query, options) {
+        return await this.inner.tokenize(query, options?.column, options?.indexName);
+    }
+    async toArrow() {
+        return await this.query().toArrow();
+    }
+    async indexStats(name) {
+        const stats = await this.inner.indexStats(name);
+        if (stats === null) {
+            return undefined;
+        }
+        return stats;
+    }
+    async stats() {
+        return await this.inner.stats();
+    }
+    async initialStorageOptions() {
+        return await this.inner.initialStorageOptions();
+    }
+    async latestStorageOptions() {
+        return await this.inner.latestStorageOptions();
+    }
+    mergeInsert(on) {
+        on = Array.isArray(on) ? on : [on];
+        return new merge_1.MergeInsertBuilder(this.inner.mergeInsert(on), this.schema());
+    }
+    /**
+     * Check if the table uses the new manifest path scheme.
+     *
+     * This function will return true if the table uses the V2 manifest
+     * path scheme.
+     */
+    async usesV2ManifestPaths() {
+        return await this.inner.usesV2ManifestPaths();
+    }
+    /**
+     * Migrate the table to use the new manifest path scheme.
+     *
+     * This function will rename all V1 manifests to V2 manifest paths.
+     * These paths provide more efficient opening of datasets with many versions
+     * on object stores.
+     *
+     * This function is idempotent, and can be run multiple times without
+     * changing the state of the object store.
+     *
+     * However, it should not be run while other concurrent operations are happening.
+     * And it should also run until completion before resuming other operations.
+     */
+    async migrateManifestPathsV2() {
+        await this.inner.migrateManifestPathsV2();
+    }
+}
+exports.LocalTable = LocalTable;
+/**
+ * Branch manager for a {@link Table}.
+ *
+ * Unlike tags, `create` and `checkout` return a new {@link Table} handle scoped
+ * to the branch; writes on it do not affect `main`.
+ */
+class Branches {
+    #inner;
+    /**
+     * Construct a Branches manager. Internal use only.
+     * @hidden
+     */
+    constructor(inner) {
+        this.#inner = inner;
+    }
+    /** List all branches, mapping name to branch metadata. */
+    async list() {
+        return await this.#inner.list();
+    }
+    /**
+     * Create a branch and return a handle scoped to it.
+     *
+     * @param name Name of the new branch.
+     * @param fromRef Source branch to fork from. Defaults to `main`.
+     * @param fromVersion A specific version on `fromRef`. Defaults to latest.
+     */
+    async create(name, fromRef, fromVersion) {
+        return new LocalTable(await this.#inner.create(name, fromRef, fromVersion));
+    }
+    /**
+     * Check out an existing branch and return a handle scoped to it.
+     *
+     * With `version` set, the returned handle is pinned to that version of the
+     * branch (a read-only, detached view); otherwise it tracks the branch's
+     * latest and stays writable.
+     */
+    async checkout(name, version) {
+        return new LocalTable(await this.#inner.checkout(name, version));
+    }
+    /** Delete a branch. */
+    async delete(name) {
+        return await this.#inner.delete(name);
+    }
+    /** Compare a branch against main without modifying either branch. */
+    async diff(fromBranch) {
+        return (await this.#inner.diff(fromBranch));
+    }
+    /**
+     * Cherry-pick a branch onto main.
+     *
+     * Set `dryRun` to `true` to preview. A failed cherry-pick resolves
+     * with `status: "failed"` instead of throwing.
+     *
+     * @param fromBranch Branch to cherry-pick from.
+     * @param dryRun When true, only preview. Defaults to false.
+     */
+    async cherryPick(fromBranch, dryRun = false) {
+        return (await this.#inner.cherryPick(fromBranch, dryRun));
+    }
+}
+exports.Branches = Branches;

package/vendor/lancedb/dist/util.d.ts

@@ -0,0 +1,14 @@
+export type IntoSql = string | number | boolean | null | Date | ArrayBufferLike | Buffer | IntoSql[];
+export declare function toSQL(value: IntoSql): string;
+export declare function packBits(data: Array<number>): Array<number>;
+export declare class TTLCache {
+    private readonly ttl;
+    private readonly cache;
+    /**
+     * @param ttl Time to live in milliseconds
+     */
+    constructor(ttl: number);
+    get(key: string): any | undefined;
+    set(key: string, value: any): void;
+    delete(key: string): void;
+}

package/vendor/lancedb/dist/util.js

@@ -0,0 +1,77 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TTLCache = void 0;
+exports.toSQL = toSQL;
+exports.packBits = packBits;
+function toSQL(value) {
+    if (typeof value === "string") {
+        return `'${value.replace(/'/g, "''")}'`;
+    }
+    else if (typeof value === "number") {
+        return value.toString();
+    }
+    else if (typeof value === "boolean") {
+        return value ? "TRUE" : "FALSE";
+    }
+    else if (value === null) {
+        return "NULL";
+    }
+    else if (value instanceof Date) {
+        return `'${value.toISOString()}'`;
+    }
+    else if (Array.isArray(value)) {
+        return `[${value.map(toSQL).join(", ")}]`;
+    }
+    else if (Buffer.isBuffer(value)) {
+        return `X'${value.toString("hex")}'`;
+    }
+    else if (value instanceof ArrayBuffer) {
+        return `X'${Buffer.from(value).toString("hex")}'`;
+    }
+    else {
+        throw new Error(`Unsupported value type: ${typeof value} value: (${value})`);
+    }
+}
+function packBits(data) {
+    const packed = Array(data.length >> 3).fill(0);
+    for (let i = 0; i < data.length; i++) {
+        const byte = i >> 3;
+        const bit = i & 7;
+        packed[byte] |= data[i] << bit;
+    }
+    return packed;
+}
+class TTLCache {
+    ttl;
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    cache;
+    /**
+     * @param ttl Time to live in milliseconds
+     */
+    constructor(ttl) {
+        this.ttl = ttl;
+        this.cache = new Map();
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    get(key) {
+        const entry = this.cache.get(key);
+        if (entry === undefined) {
+            return undefined;
+        }
+        if (entry.expires < Date.now()) {
+            this.cache.delete(key);
+            return undefined;
+        }
+        return entry.value;
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    set(key, value) {
+        this.cache.set(key, { value, expires: Date.now() + this.ttl });
+    }
+    delete(key) {
+        this.cache.delete(key);
+    }
+}
+exports.TTLCache = TTLCache;

package/vendor/lancedb/license_header.txt

@@ -0,0 +1,2 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors

package/vendor/lancedb/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@lancedb/lancedb",
+  "description": "LanceDB: A serverless, low-latency vector database for AI applications",
+  "keywords": [
+    "database",
+    "lance",
+    "lancedb",
+    "search",
+    "vector",
+    "vector database",
+    "ann"
+  ],
+  "private": false,
+  "version": "0.38.0",
+  "main": "dist/index.js",
+  "exports": {
+    ".": "./dist/index.js",
+    "./embedding": "./dist/embedding/index.js",
+    "./embedding/openai": "./dist/embedding/openai.js",
+    "./embedding/transformers": "./dist/embedding/transformers.js"
+  },
+  "types": "dist/index.d.ts",
+  "napi": {
+    "binaryName": "lancedb",
+    "targets": [
+      "aarch64-apple-darwin",
+      "x86_64-unknown-linux-gnu",
+      "aarch64-unknown-linux-gnu",
+      "x86_64-unknown-linux-musl",
+      "aarch64-unknown-linux-musl",
+      "x86_64-pc-windows-msvc",
+      "aarch64-pc-windows-msvc"
+    ]
+  },
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/lancedb/lancedb"
+  },
+  "ava": {
+    "timeout": "3m"
+  },
+  "engines": {
+    "node": ">= 22"
+  },
+  "packageManager": "pnpm@11.1.1",
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
+  "os": [
+    "darwin",
+    "linux",
+    "win32"
+  ]
+}