presubmit 0.1.0

package/vendor/lancedb/dist/connection.js

@@ -0,0 +1,298 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalConnection = exports.Connection = void 0;
+exports.cleanseStorageOptions = cleanseStorageOptions;
+const apache_arrow_1 = require("apache-arrow");
+const arrow_1 = require("./arrow");
+const arrow_2 = require("./arrow");
+const registry_1 = require("./embedding/registry");
+const materialized_view_1 = require("./materialized_view");
+const sanitize_1 = require("./sanitize");
+const table_1 = require("./table");
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ *
+ * A Connection is intended to be a long lived object and may hold open
+ * resources such as HTTP connection pools.  This is generally fine and
+ * a single connection should be shared if it is going to be used many
+ * times. However, if you are finished with a connection, you may call
+ * close to eagerly free these resources.  Any call to a Connection
+ * method after it has been closed will result in an error.
+ *
+ * Closing a connection is optional.  Connections will automatically
+ * be closed when they are garbage collected.
+ *
+ * Any created tables are independent and will continue to work even if
+ * the underlying connection has been closed.
+ * @hideconstructor
+ */
+class Connection {
+    [Symbol.for("nodejs.util.inspect.custom")]() {
+        return this.display();
+    }
+}
+exports.Connection = Connection;
+/** @hideconstructor */
+class LocalConnection extends Connection {
+    inner;
+    /** @hidden */
+    constructor(inner) {
+        super();
+        this.inner = inner;
+    }
+    isOpen() {
+        return this.inner.isOpen();
+    }
+    close() {
+        this.inner.close();
+    }
+    display() {
+        return this.inner.display();
+    }
+    async tableNames(namespacePathOrOptions, options) {
+        // Detect if first argument is namespacePath array or options object
+        let namespacePath;
+        let tableNamesOptions;
+        if (Array.isArray(namespacePathOrOptions)) {
+            // First argument is namespacePath array
+            namespacePath = namespacePathOrOptions;
+            tableNamesOptions = options;
+        }
+        else {
+            // First argument is options object (backwards compatibility)
+            namespacePath = undefined;
+            tableNamesOptions = namespacePathOrOptions;
+        }
+        return this.inner.tableNames(namespacePath ?? [], tableNamesOptions?.startAfter, tableNamesOptions?.limit);
+    }
+    async createMaterializedView(name, source, options) {
+        (0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
+        const innerTable = await this.inner.createMaterializedView(name, source, (0, materialized_view_1.normalizeSelect)(options?.select), options?.where, options?.limit);
+        return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
+    }
+    async openMaterializedView(name) {
+        const innerTable = await this.inner.openMaterializedView(name);
+        return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
+    }
+    async listMaterializedViews() {
+        return await this.inner.listMaterializedViews();
+    }
+    async listTables(namespacePathOrOptions, options) {
+        // Detect if first argument is namespacePath array or options object
+        const namespacePath = Array.isArray(namespacePathOrOptions)
+            ? namespacePathOrOptions
+            : undefined;
+        const listTablesOptions = Array.isArray(namespacePathOrOptions)
+            ? options
+            : namespacePathOrOptions;
+        return this.inner.listTables(namespacePath ?? [], listTablesOptions?.pageToken, listTablesOptions?.limit);
+    }
+    async openTable(name, namespacePath, options) {
+        const innerTable = await this.inner.openTable(name, namespacePath ?? [], cleanseStorageOptions(options?.storageOptions), options?.indexCacheSize);
+        let table = new table_1.LocalTable(innerTable);
+        // "main" is the default branch, so treat it as no branch. On a real branch,
+        // scope and pin in one step (yielding "version V of branch B"); otherwise
+        // pin the version, if any, against main.
+        const branch = options?.branch != null && options.branch !== "main"
+            ? options.branch
+            : undefined;
+        if (branch != null) {
+            table = await (await table.branches()).checkout(branch, options?.version);
+        }
+        else if (options?.version != null) {
+            await table.checkout(options.version);
+        }
+        return table;
+    }
+    async cloneTable(targetTableName, sourceUri, options) {
+        const innerTable = await this.inner.cloneTable(targetTableName, sourceUri, options?.targetNamespacePath ?? [], options?.sourceVersion ?? null, options?.sourceTag ?? null, options?.isShallow ?? true);
+        return new table_1.LocalTable(innerTable);
+    }
+    getStorageOptions(options) {
+        if (options?.dataStorageVersion !== undefined) {
+            if (options.storageOptions === undefined) {
+                options.storageOptions = {};
+            }
+            options.storageOptions["newTableDataStorageVersion"] =
+                options.dataStorageVersion;
+        }
+        if (options?.enableV2ManifestPaths !== undefined) {
+            if (options.storageOptions === undefined) {
+                options.storageOptions = {};
+            }
+            options.storageOptions["newTableEnableV2ManifestPaths"] =
+                options.enableV2ManifestPaths ? "true" : "false";
+        }
+        return cleanseStorageOptions(options?.storageOptions);
+    }
+    async createTable(nameOrOptions, dataOrNamespacePath, namespacePathOrOptions, options) {
+        if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
+            // First overload: createTable(options, namespacePath?)
+            const { name, data, ...createOptions } = nameOrOptions;
+            const namespacePath = dataOrNamespacePath;
+            return this._createTableImpl(name, data, namespacePath, createOptions);
+        }
+        // Second overload: createTable(name, data, namespacePath?, options?)
+        const name = nameOrOptions;
+        const data = dataOrNamespacePath;
+        // Detect if third argument is namespacePath array or options object
+        let namespacePath;
+        let createOptions;
+        if (Array.isArray(namespacePathOrOptions)) {
+            // Third argument is namespacePath array
+            namespacePath = namespacePathOrOptions;
+            createOptions = options;
+        }
+        else {
+            // Third argument is options object (backwards compatibility)
+            namespacePath = undefined;
+            createOptions = namespacePathOrOptions;
+        }
+        return this._createTableImpl(name, data, namespacePath, createOptions);
+    }
+    async _createTableImpl(name, data, namespacePath, options) {
+        if (data === undefined) {
+            throw new Error("data is required");
+        }
+        const { buf, mode } = await parseTableData(data, options);
+        const storageOptions = this.getStorageOptions(options);
+        const innerTable = await this.inner.createTable(name, buf, mode, namespacePath ?? [], storageOptions);
+        return new table_1.LocalTable(innerTable);
+    }
+    async createEmptyTable(name, schema, namespacePathOrOptions, options) {
+        // Detect if third argument is namespacePath array or options object
+        let namespacePath;
+        let createOptions;
+        if (Array.isArray(namespacePathOrOptions)) {
+            // Third argument is namespacePath array
+            namespacePath = namespacePathOrOptions;
+            createOptions = options;
+        }
+        else {
+            // Third argument is options object (backwards compatibility)
+            namespacePath = undefined;
+            createOptions = namespacePathOrOptions;
+        }
+        let mode = createOptions?.mode ?? "create";
+        const existOk = createOptions?.existOk ?? false;
+        if (mode === "create" && existOk) {
+            mode = "exist_ok";
+        }
+        let metadata = undefined;
+        if (createOptions?.embeddingFunction !== undefined) {
+            const embeddingFunction = createOptions.embeddingFunction;
+            const registry = (0, registry_1.getRegistry)();
+            metadata = registry.getTableMetadata([embeddingFunction]);
+        }
+        const storageOptions = this.getStorageOptions(createOptions);
+        const table = (0, arrow_2.makeEmptyTable)(schema, metadata);
+        const buf = await (0, arrow_2.fromTableToBuffer)(table);
+        const innerTable = await this.inner.createEmptyTable(name, buf, mode, namespacePath ?? [], storageOptions);
+        return new table_1.LocalTable(innerTable);
+    }
+    async dropTable(name, namespacePath) {
+        return this.inner.dropTable(name, namespacePath ?? []);
+    }
+    async dropTableAsync(name, namespacePath) {
+        return this.inner.dropTableAsync(name, namespacePath ?? []);
+    }
+    async dropAllTables(namespacePath) {
+        return this.inner.dropAllTables(namespacePath ?? []);
+    }
+    describeNamespace(namespacePath) {
+        return this.inner.describeNamespace(namespacePath);
+    }
+    listNamespaces(namespacePath, options) {
+        return this.inner.listNamespaces(namespacePath ?? [], options?.pageToken, options?.limit);
+    }
+    createNamespace(namespacePath, options) {
+        return this.inner.createNamespace(namespacePath, options?.mode, options?.properties);
+    }
+    dropNamespace(namespacePath, options) {
+        return this.inner.dropNamespace(namespacePath, options?.mode, options?.behavior);
+    }
+    async renameTable(currentName, newName, options) {
+        return this.inner.renameTable(currentName, newName, options?.namespacePath ?? [], options?.newNamespacePath);
+    }
+    job(jobId) {
+        return this.inner.job(jobId);
+    }
+    async listJobs() {
+        return this.inner.listJobs();
+    }
+    async getJob(jobId) {
+        return this.inner.getJob(jobId);
+    }
+    async cancelJob(jobId) {
+        return this.inner.cancelJob(jobId);
+    }
+    async jobHistory(jobId) {
+        const buf = await this.inner.jobHistory(jobId);
+        if (buf.length === 0) {
+            return new arrow_2.Table();
+        }
+        return (0, apache_arrow_1.tableFromIPC)(buf);
+    }
+}
+exports.LocalConnection = LocalConnection;
+/**
+ * Takes storage options and makes all the keys snake case.
+ */
+function cleanseStorageOptions(options) {
+    if (options === undefined) {
+        return undefined;
+    }
+    const result = {};
+    for (const [key, value] of Object.entries(options)) {
+        if (value !== undefined) {
+            const newKey = camelToSnakeCase(key);
+            result[newKey] = value;
+        }
+    }
+    return result;
+}
+/**
+ * Convert a string to snake case. It might already be snake case, in which case it is
+ * returned unchanged.
+ */
+function camelToSnakeCase(camel) {
+    if (camel.includes("_")) {
+        // Assume if there is at least one underscore, it is already snake case
+        return camel;
+    }
+    if (camel.toLocaleUpperCase() === camel) {
+        // Assume if the string is all uppercase, it is already snake case
+        return camel;
+    }
+    let result = camel.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+    if (result.startsWith("_")) {
+        result = result.slice(1);
+    }
+    return result;
+}
+async function parseTableData(data, options, streaming = false) {
+    let mode = options?.mode ?? "create";
+    const existOk = options?.existOk ?? false;
+    if (mode === "create" && existOk) {
+        mode = "exist_ok";
+    }
+    let table;
+    if ((0, arrow_1.isArrowTable)(data)) {
+        table = (0, sanitize_1.sanitizeTable)(data);
+    }
+    else {
+        table = (0, arrow_1.makeArrowTable)(data, options);
+    }
+    if (streaming) {
+        const buf = await (0, arrow_1.fromTableToStreamBuffer)(table, options?.embeddingFunction, options?.schema);
+        return { buf, mode };
+    }
+    else {
+        const buf = await (0, arrow_2.fromTableToBuffer)(table, options?.embeddingFunction, options?.schema);
+        return { buf, mode };
+    }
+}

package/vendor/lancedb/dist/embedding/embedding_function.d.ts

@@ -0,0 +1,103 @@
+import "reflect-metadata";
+import { DataType, Float, type IntoVector } from "../arrow";
+/**
+ * Options for a given embedding function
+ */
+export interface FunctionOptions {
+    [key: string]: any;
+}
+export interface EmbeddingFunctionConstructor<T extends EmbeddingFunction = EmbeddingFunction> {
+    new (modelOptions?: T["TOptions"]): T;
+}
+/**
+ * An embedding function that automatically creates vector representation for a given column.
+ *
+ * It's important subclasses pass the **original** options to the super constructor
+ * and then pass those options to `resolveVariables` to resolve any variables before
+ * using them.
+ *
+ * @example
+ * ```ts
+ * class MyEmbeddingFunction extends EmbeddingFunction {
+ *   constructor(options: {model: string, timeout: number}) {
+ *     super(optionsRaw);
+ *     const options = this.resolveVariables(optionsRaw);
+ *     this.model = options.model;
+ *     this.timeout = options.timeout;
+ *   }
+ * }
+ * ```
+ */
+export declare abstract class EmbeddingFunction<T = any, M extends FunctionOptions = FunctionOptions> {
+    #private;
+    /**
+     * @ignore
+     *  This is only used for associating the options type with the class for type checking
+     */
+    readonly TOptions: M;
+    /**
+     * Get the original arguments to the constructor, to serialize them so they
+     * can be used to recreate the embedding function later.
+     */
+    toJSON(): Record<string, any>;
+    constructor();
+    /**
+     * Provide a list of keys in the function options that should be treated as
+     * sensitive. If users pass raw values for these keys, they will be rejected.
+     */
+    protected getSensitiveKeys(): string[];
+    /**
+     * Apply variables to the config.
+     */
+    protected resolveVariables(config: Partial<M>): Partial<M>;
+    /**
+     * Optionally load any resources needed for the embedding function.
+     *
+     * This method is called after the embedding function has been initialized
+     * but before any embeddings are computed. It is useful for loading local models
+     * or other resources that are needed for the embedding function to work.
+     */
+    init?(): Promise<void>;
+    /**
+     * sourceField is used in combination with `LanceSchema` to provide a declarative data model
+     *
+     * @param optionsOrDatatype - The options for the field or the datatype
+     *
+     * @see {@link LanceSchema}
+     */
+    sourceField(optionsOrDatatype: Partial<FieldOptions> | DataType): [DataType, Map<string, EmbeddingFunction>];
+    /**
+     * vectorField is used in combination with `LanceSchema` to provide a declarative data model
+     *
+     * @param optionsOrDatatype - The options for the field
+     *
+     * @see {@link LanceSchema}
+     */
+    vectorField(optionsOrDatatype?: Partial<FieldOptions> | DataType): [DataType, Map<string, EmbeddingFunction>];
+    /** The number of dimensions of the embeddings */
+    ndims(): number | undefined;
+    /** The datatype of the embeddings */
+    abstract embeddingDataType(): Float;
+    /**
+     * Creates a vector representation for the given values.
+     */
+    abstract computeSourceEmbeddings(data: T[]): Promise<number[][] | Float32Array[] | Float64Array[]>;
+    /**
+    Compute the embeddings for a single query
+   */
+    computeQueryEmbeddings(data: T): Promise<Awaited<IntoVector>>;
+}
+/**
+ * an abstract class for implementing embedding functions that take text as input
+ */
+export declare abstract class TextEmbeddingFunction<M extends FunctionOptions = FunctionOptions> extends EmbeddingFunction<string, M> {
+    abstract generateEmbeddings(texts: string[], ...args: any[]): Promise<number[][] | Float32Array[] | Float64Array[]>;
+    computeQueryEmbeddings(data: string): Promise<Awaited<IntoVector>>;
+    embeddingDataType(): Float;
+    sourceField(): [DataType, Map<string, EmbeddingFunction>];
+    computeSourceEmbeddings(data: string[]): Promise<number[][] | Float32Array[] | Float64Array[]>;
+}
+export interface FieldOptions<T extends DataType = DataType> {
+    datatype: T;
+    dims?: number;
+}

package/vendor/lancedb/dist/embedding/embedding_function.js

@@ -0,0 +1,192 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TextEmbeddingFunction = exports.EmbeddingFunction = void 0;
+require("reflect-metadata");
+const arrow_1 = require("../arrow");
+const sanitize_1 = require("../sanitize");
+const registry_1 = require("./registry");
+/**
+ * An embedding function that automatically creates vector representation for a given column.
+ *
+ * It's important subclasses pass the **original** options to the super constructor
+ * and then pass those options to `resolveVariables` to resolve any variables before
+ * using them.
+ *
+ * @example
+ * ```ts
+ * class MyEmbeddingFunction extends EmbeddingFunction {
+ *   constructor(options: {model: string, timeout: number}) {
+ *     super(optionsRaw);
+ *     const options = this.resolveVariables(optionsRaw);
+ *     this.model = options.model;
+ *     this.timeout = options.timeout;
+ *   }
+ * }
+ * ```
+ */
+class EmbeddingFunction {
+    /**
+     * @ignore
+     *  This is only used for associating the options type with the class for type checking
+     */
+    // biome-ignore lint/style/useNamingConvention: we want to keep the name as it is
+    TOptions;
+    #config;
+    /**
+     * Get the original arguments to the constructor, to serialize them so they
+     * can be used to recreate the embedding function later.
+     */
+    // biome-ignore lint/suspicious/noExplicitAny :
+    toJSON() {
+        return JSON.parse(JSON.stringify(this.#config));
+    }
+    constructor() {
+        this.#config = {};
+    }
+    /**
+     * Provide a list of keys in the function options that should be treated as
+     * sensitive. If users pass raw values for these keys, they will be rejected.
+     */
+    getSensitiveKeys() {
+        return [];
+    }
+    /**
+     * Apply variables to the config.
+     */
+    resolveVariables(config) {
+        this.#config = config;
+        const registry = (0, registry_1.getRegistry)();
+        const newConfig = { ...config };
+        for (const [key_, value] of Object.entries(newConfig)) {
+            if (this.getSensitiveKeys().includes(key_) &&
+                !value.startsWith("$var:")) {
+                throw new Error(`The key "${key_}" is sensitive and cannot be set directly. Please use the $var: syntax to set it.`);
+            }
+            // Makes TS happy (https://stackoverflow.com/a/78391854)
+            const key = key_;
+            if (typeof value === "string" && value.startsWith("$var:")) {
+                const [name, defaultValue] = value.slice(5).split(":", 2);
+                const variableValue = registry.getVar(name);
+                if (!variableValue) {
+                    if (defaultValue) {
+                        // biome-ignore lint/suspicious/noExplicitAny:
+                        newConfig[key] = defaultValue;
+                    }
+                    else {
+                        throw new Error(`Variable "${name}" not found`);
+                    }
+                }
+                else {
+                    // biome-ignore lint/suspicious/noExplicitAny:
+                    newConfig[key] = variableValue;
+                }
+            }
+        }
+        return newConfig;
+    }
+    /**
+     * sourceField is used in combination with `LanceSchema` to provide a declarative data model
+     *
+     * @param optionsOrDatatype - The options for the field or the datatype
+     *
+     * @see {@link LanceSchema}
+     */
+    sourceField(optionsOrDatatype) {
+        let datatype = "datatype" in optionsOrDatatype
+            ? optionsOrDatatype.datatype
+            : optionsOrDatatype;
+        if (!datatype) {
+            throw new Error("Datatype is required");
+        }
+        datatype = (0, sanitize_1.sanitizeType)(datatype);
+        const metadata = new Map();
+        metadata.set("source_column_for", this);
+        return [datatype, metadata];
+    }
+    /**
+     * vectorField is used in combination with `LanceSchema` to provide a declarative data model
+     *
+     * @param optionsOrDatatype - The options for the field
+     *
+     * @see {@link LanceSchema}
+     */
+    vectorField(optionsOrDatatype) {
+        let dtype;
+        let vectorType;
+        let dims = this.ndims();
+        // `func.vectorField(new Float32())`
+        if (optionsOrDatatype === undefined) {
+            dtype = new arrow_1.Float32();
+        }
+        else if (!("datatype" in optionsOrDatatype)) {
+            dtype = (0, sanitize_1.sanitizeType)(optionsOrDatatype);
+        }
+        else {
+            // `func.vectorField({
+            //  datatype: new Float32(),
+            //  dims: 10
+            // })`
+            dims = dims ?? optionsOrDatatype?.dims;
+            dtype = (0, sanitize_1.sanitizeType)(optionsOrDatatype?.datatype);
+        }
+        if (dtype !== undefined) {
+            // `func.vectorField(new FixedSizeList(dims, new Field("item", new Float32(), true)))`
+            // or `func.vectorField({datatype: new FixedSizeList(dims, new Field("item", new Float32(), true))})`
+            if ((0, arrow_1.isFixedSizeList)(dtype)) {
+                vectorType = dtype;
+                // `func.vectorField(new Float32())`
+                // or `func.vectorField({datatype: new Float32()})`
+            }
+            else if ((0, arrow_1.isFloat)(dtype)) {
+                // No `ndims` impl and no `{dims: n}` provided;
+                if (dims === undefined) {
+                    throw new Error("ndims is required for vector field");
+                }
+                vectorType = (0, arrow_1.newVectorType)(dims, dtype);
+            }
+            else {
+                throw new Error("Expected FixedSizeList or Float as datatype for vector field");
+            }
+        }
+        else {
+            if (dims === undefined) {
+                throw new Error("ndims is required for vector field");
+            }
+            vectorType = new arrow_1.FixedSizeList(dims, new arrow_1.Field("item", new arrow_1.Float32(), true));
+        }
+        const metadata = new Map();
+        metadata.set("vector_column_for", this);
+        return [vectorType, metadata];
+    }
+    /** The number of dimensions of the embeddings */
+    ndims() {
+        return undefined;
+    }
+    /**
+    Compute the embeddings for a single query
+   */
+    async computeQueryEmbeddings(data) {
+        return this.computeSourceEmbeddings([data]).then((embeddings) => embeddings[0]);
+    }
+}
+exports.EmbeddingFunction = EmbeddingFunction;
+/**
+ * an abstract class for implementing embedding functions that take text as input
+ */
+class TextEmbeddingFunction extends EmbeddingFunction {
+    async computeQueryEmbeddings(data) {
+        return this.generateEmbeddings([data]).then((data) => data[0]);
+    }
+    embeddingDataType() {
+        return new arrow_1.Float32();
+    }
+    sourceField() {
+        return super.sourceField(new arrow_1.Utf8());
+    }
+    computeSourceEmbeddings(data) {
+        return this.generateEmbeddings(data);
+    }
+}
+exports.TextEmbeddingFunction = TextEmbeddingFunction;

package/vendor/lancedb/dist/embedding/index.d.ts

@@ -0,0 +1,37 @@
+import { Schema } from "../arrow";
+import { EmbeddingFunction } from "./embedding_function";
+import { EmbeddingFunctionRegistry } from "./registry";
+export { FieldOptions, EmbeddingFunction, TextEmbeddingFunction, FunctionOptions, EmbeddingFunctionConstructor, } from "./embedding_function";
+export { EmbeddingFunctionRegistry, parseEmbeddingMetadata, register, } from "./registry";
+export type { CreateReturnType, EmbeddingFunctionConfig, EmbeddingFunctionCreate, EmbeddingMetadataEntry, ResolvedEmbeddingFunctionConfig, } from "./registry";
+/**
+ * Get the global embedding function registry.
+ *
+ * LanceDB built-in providers are initialized when this public API is first
+ * used, so importing the root package does not change automatic search
+ * selection for tables without embedding metadata.
+ */
+export declare function getRegistry(): EmbeddingFunctionRegistry;
+/**
+ * Create a schema with embedding functions.
+ *
+ * @param fields
+ * @returns Schema
+ * @example
+ * ```ts
+ * class MyEmbeddingFunction extends EmbeddingFunction {
+ * // ...
+ * }
+ * const func = new MyEmbeddingFunction();
+ * const schema = LanceSchema({
+ *   id: new Int32(),
+ *   text: func.sourceField(new Utf8()),
+ *   vector: func.vectorField(),
+ *   // optional: specify the datatype and/or dimensions
+ *   vector2: func.vectorField({ datatype: new Float32(), dims: 3}),
+ * });
+ *
+ * const table = await db.createTable("my_table", data, { schema });
+ * ```
+ */
+export declare function LanceSchema(fields: Record<string, [object, Map<string, EmbeddingFunction>] | object>): Schema;