presubmit 0.1.0

package/vendor/lancedb/dist/indices.js

@@ -0,0 +1,166 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Index = void 0;
+const native_1 = require("./native");
+class Index {
+    inner;
+    constructor(inner) {
+        this.inner = inner;
+    }
+    /**
+     * Create an IvfPq index
+     *
+     * This index stores a compressed (quantized) copy of every vector.  These vectors
+     * are grouped into partitions of similar vectors.  Each partition keeps track of
+     * a centroid which is the average value of all vectors in the group.
+     *
+     * During a query the centroids are compared with the query vector to find the closest
+     * partitions.  The compressed vectors in these partitions are then searched to find
+     * the closest vectors.
+     *
+     * The compression scheme is called product quantization.  Each vector is divided into
+     * subvectors and then each subvector is quantized into a small number of bits.  the
+     * parameters `num_bits` and `num_subvectors` control this process, providing a tradeoff
+     * between index size (and thus search speed) and index accuracy.
+     *
+     * The partitioning process is called IVF and the `num_partitions` parameter controls how
+     * many groups to create.
+     *
+     * Note that training an IVF PQ index on a large dataset is a slow operation and
+     * currently is also a memory intensive operation.
+     */
+    static ivfPq(options) {
+        return new Index(native_1.Index.ivfPq(options?.distanceType, options?.numPartitions, options?.numSubVectors, options?.numBits, options?.maxIterations, options?.sampleRate));
+    }
+    /**
+     * Create an IvfRq index
+     *
+     * IVF-RQ (RabitQ Quantization) compresses vectors using RabitQ quantization
+     * and organizes them into IVF partitions.
+     *
+     * The compression scheme is called RabitQ quantization. Each dimension is quantized into a small number of bits.
+     * The parameters `num_bits` and `num_partitions` control this process, providing a tradeoff
+     * between index size (and thus search speed) and index accuracy.
+     *
+     * The partitioning process is called IVF and the `num_partitions` parameter controls how
+     * many groups to create.
+     *
+     * Note that training an IVF RQ index on a large dataset is a slow operation and
+     * currently is also a memory intensive operation.
+     */
+    static ivfRq(options) {
+        return new Index(native_1.Index.ivfRq(options?.distanceType, options?.numPartitions, options?.numBits, options?.maxIterations, options?.sampleRate));
+    }
+    /**
+     * Create an IvfFlat index
+     *
+     * This index groups vectors into partitions of similar vectors.  Each partition keeps track of
+     * a centroid which is the average value of all vectors in the group.
+     *
+     * During a query the centroids are compared with the query vector to find the closest
+     * partitions.  The vectors in these partitions are then searched to find
+     * the closest vectors.
+     *
+     * The partitioning process is called IVF and the `num_partitions` parameter controls how
+     * many groups to create.
+     *
+     * Note that training an IVF FLAT index on a large dataset is a slow operation and
+     * currently is also a memory intensive operation.
+     */
+    static ivfFlat(options) {
+        return new Index(native_1.Index.ivfFlat(options?.distanceType, options?.numPartitions, options?.maxIterations, options?.sampleRate));
+    }
+    /**
+     * Create a btree index
+     *
+     * A btree index is an index on a scalar columns.  The index stores a copy of the column
+     * in sorted order.  A header entry is created for each block of rows (currently the
+     * block size is fixed at 4096).  These header entries are stored in a separate
+     * cacheable structure (a btree).  To search for data the header is used to determine
+     * which blocks need to be read from disk.
+     *
+     * For example, a btree index in a table with 1Bi rows requires sizeof(Scalar) * 256Ki
+     * bytes of memory and will generally need to read sizeof(Scalar) * 4096 bytes to find
+     * the correct row ids.
+     *
+     * This index is good for scalar columns with mostly distinct values and does best when
+     * the query is highly selective.
+     *
+     * The btree index does not currently have any parameters though parameters such as the
+     * block size may be added in the future.
+     */
+    static btree() {
+        return new Index(native_1.Index.btree());
+    }
+    /**
+     * Create a bitmap index.
+     *
+     * A `Bitmap` index stores a bitmap for each distinct value in the column for every row.
+     *
+     * This index works best for low-cardinality columns, where the number of unique values
+     * is small (i.e., less than a few hundreds).
+     */
+    static bitmap() {
+        return new Index(native_1.Index.bitmap());
+    }
+    /**
+     * Create a label list index.
+     *
+     * LabelList index is a scalar index that can be used on `List<T>` columns to
+     * support queries with `array_contains_all` and `array_contains_any`
+     * using an underlying bitmap index.
+     */
+    static labelList() {
+        return new Index(native_1.Index.labelList());
+    }
+    /**
+     * Create an FM-Index.
+     *
+     * An FM-Index is a scalar index on string or binary columns that accelerates
+     * substring search, i.e. `contains(col, 'needle')`. Unlike the tokenized
+     * full-text-search index, it matches arbitrary substrings of the raw bytes.
+     */
+    static fm() {
+        return new Index(native_1.Index.fm());
+    }
+    /**
+     * Create a full text search index
+     *
+     * A full text search index is an index on a string column, so that you can conduct full
+     * text searches on the column.
+     *
+     * The results of a full text search are ordered by relevance measured by BM25.
+     *
+     * You can combine filters with full text search.
+     */
+    static fts(options) {
+        return new Index(native_1.Index.fts(options?.withPosition, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.customStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, options?.prefixOnly, options?.blockSize));
+    }
+    /**
+     *
+     * Create a hnswPq index
+     *
+     * HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
+     * It is a variant of the HNSW algorithm that uses product quantization to compress
+     * the vectors.
+     *
+     */
+    static hnswPq(options) {
+        return new Index(native_1.Index.hnswPq(options?.distanceType, options?.numPartitions, options?.numSubVectors, options?.maxIterations, options?.sampleRate, options?.m, options?.efConstruction));
+    }
+    /**
+     *
+     * Create a hnswSq index
+     *
+     * HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
+     * It is a variant of the HNSW algorithm that uses scalar quantization to compress
+     * the vectors.
+     *
+     */
+    static hnswSq(options) {
+        return new Index(native_1.Index.hnswSq(options?.distanceType, options?.numPartitions, options?.maxIterations, options?.sampleRate, options?.m, options?.efConstruction));
+    }
+}
+exports.Index = Index;

package/vendor/lancedb/dist/materialized_view.d.ts

@@ -0,0 +1,69 @@
+import { RefreshMaterializedViewResult } from "./native";
+import { Table } from "./table";
+/** Schema metadata key holding a materialized view's definition. */
+export declare const DEFINITION_META_KEY = "mv.definition";
+/** The query that defines a materialized view. */
+export interface MaterializedViewDefinition {
+    /** Name of the source table, in the same database as the view. */
+    sourceTable: string;
+    /** `[output column, SQL expression]` pairs, in view schema order. */
+    projections: [string, string][];
+    /** SQL predicate selecting the source rows the view holds. */
+    filter?: string;
+    /** Cap on the number of rows the view holds. */
+    limit?: number;
+    /** Source columns the projections and filter read. */
+    inputs: string[];
+}
+/**
+ * The view's columns: column names, `[alias, SQL expression]` pairs, or a
+ * record of the same. A bare name projects itself.
+ */
+export type MaterializedViewSelect = (string | [string, string])[] | Record<string, string>;
+/**
+ * @internal Reject a numeric option N-API would otherwise silently coerce:
+ * `Infinity` reaches Rust as 0, `1.5` as 1.
+ */
+export declare function validateNonNegativeInteger(value: number | undefined, name: string): void;
+/**
+ * @internal Normalize a select argument into `[alias, expression]` pairs.
+ * A bare name projects itself and is quoted, so any valid column name works;
+ * pair and record entries are kept verbatim because their right side is an
+ * expression.
+ */
+export declare function normalizeSelect(select?: MaterializedViewSelect): [string, string][] | undefined;
+/** @internal Parse a definition off a table's stored schema metadata. */
+export declare function definitionFromMetadata(metadata: Map<string, string>, name: string): MaterializedViewDefinition;
+/**
+ * A handle on a materialized view: its table plus its definition.
+ *
+ * Obtained from {@link Connection#createMaterializedView} or
+ * {@link Connection#openMaterializedView}. The view is a normal table --
+ * queries, indexes and search all apply through {@link MaterializedView#table}
+ * -- whose contents are maintained by {@link MaterializedView#refresh}.
+ */
+export declare class MaterializedView {
+    private readonly inner;
+    constructor(table: Table);
+    get name(): string;
+    /** The view, as the table it is. */
+    table(): Table;
+    /** The query that defines the view, read from its stored schema. */
+    definition(): Promise<MaterializedViewDefinition>;
+    /**
+     * Recompute the view from its source.
+     *
+     * The refresh is incremental when the source's changes can be reconciled
+     * into the view -- rows added, changed or removed since the last one --
+     * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
+     * refreshes to that source version instead of the latest.
+     *
+     * Concurrent refreshes of one view do not duplicate its rows. Two that
+     * plan the same source rows conflict on commit, and the loser throws
+     * rather than writing them a second time.
+     */
+    refresh(options?: {
+        full?: boolean;
+        sourceVersion?: number;
+    }): Promise<RefreshMaterializedViewResult>;
+}

package/vendor/lancedb/dist/materialized_view.js

@@ -0,0 +1,112 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MaterializedView = exports.DEFINITION_META_KEY = void 0;
+exports.validateNonNegativeInteger = validateNonNegativeInteger;
+exports.normalizeSelect = normalizeSelect;
+exports.definitionFromMetadata = definitionFromMetadata;
+/** Schema metadata key holding a materialized view's definition. */
+exports.DEFINITION_META_KEY = "mv.definition";
+/**
+ * @internal Reject a numeric option N-API would otherwise silently coerce:
+ * `Infinity` reaches Rust as 0, `1.5` as 1.
+ */
+function validateNonNegativeInteger(value, name) {
+    if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) {
+        throw new Error(`${name} must be a non-negative integer`);
+    }
+}
+/** @internal Quote a column name as a Lance SQL identifier (backticks). */
+function quoteIdentifier(name) {
+    return "`" + name.replace(/`/g, "``") + "`";
+}
+/**
+ * @internal Normalize a select argument into `[alias, expression]` pairs.
+ * A bare name projects itself and is quoted, so any valid column name works;
+ * pair and record entries are kept verbatim because their right side is an
+ * expression.
+ */
+function normalizeSelect(select) {
+    if (select === undefined) {
+        return undefined;
+    }
+    if (Array.isArray(select)) {
+        return select.map((item) => typeof item === "string" ? [item, quoteIdentifier(item)] : item);
+    }
+    return Object.entries(select);
+}
+/** @internal Parse a definition off a table's stored schema metadata. */
+function definitionFromMetadata(metadata, name) {
+    const raw = metadata.get(exports.DEFINITION_META_KEY);
+    if (raw === undefined) {
+        throw new Error(`Table '${name}' is not a materialized view`);
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: raw JSON
+    const value = JSON.parse(raw);
+    if (value.kind !== "select") {
+        throw new Error(`materialized view '${name}' is defined by '${value.kind}', which this ` +
+            "version of lancedb cannot refresh");
+    }
+    const limit = value.limit ?? undefined;
+    // JSON.parse rounds integers past 2^53; every exact u64 parses to a safe
+    // integer and every rounded one does not, so this rejects precisely the
+    // values a number cannot carry.
+    if (limit !== undefined && !Number.isSafeInteger(limit)) {
+        throw new Error(`materialized view '${name}' has a stored limit too large to represent exactly`);
+    }
+    return {
+        sourceTable: value.source_table,
+        // biome-ignore lint/suspicious/noExplicitAny: raw JSON
+        projections: (value.projections ?? []).map((p) => [
+            p.output,
+            p.expression,
+        ]),
+        filter: value.filter ?? undefined,
+        limit,
+        inputs: value.inputs ?? [],
+    };
+}
+/**
+ * A handle on a materialized view: its table plus its definition.
+ *
+ * Obtained from {@link Connection#createMaterializedView} or
+ * {@link Connection#openMaterializedView}. The view is a normal table --
+ * queries, indexes and search all apply through {@link MaterializedView#table}
+ * -- whose contents are maintained by {@link MaterializedView#refresh}.
+ */
+class MaterializedView {
+    inner;
+    constructor(table) {
+        this.inner = table;
+    }
+    get name() {
+        return this.inner.name;
+    }
+    /** The view, as the table it is. */
+    table() {
+        return this.inner;
+    }
+    /** The query that defines the view, read from its stored schema. */
+    async definition() {
+        const schema = await this.inner.schema();
+        return definitionFromMetadata(schema.metadata, this.name);
+    }
+    /**
+     * Recompute the view from its source.
+     *
+     * The refresh is incremental when the source's changes can be reconciled
+     * into the view -- rows added, changed or removed since the last one --
+     * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
+     * refreshes to that source version instead of the latest.
+     *
+     * Concurrent refreshes of one view do not duplicate its rows. Two that
+     * plan the same source rows conflict on commit, and the loser throws
+     * rather than writing them a second time.
+     */
+    async refresh(options) {
+        validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
+        return await this.inner.refreshMaterializedView(options?.full, options?.sourceVersion);
+    }
+}
+exports.MaterializedView = MaterializedView;

package/vendor/lancedb/dist/merge.d.ts

@@ -0,0 +1,104 @@
+import { Data, Schema } from "./arrow";
+import { MergeResult, NativeMergeInsertBuilder } from "./native";
+/** A builder used to create and run a merge insert operation */
+export declare class MergeInsertBuilder {
+    #private;
+    /** Construct a MergeInsertBuilder. __Internal use only.__ */
+    constructor(native: NativeMergeInsertBuilder, schema: Schema | Promise<Schema>);
+    /**
+     * Rows that exist in both the source table (new data) and
+     * the target table (old data) will be updated, replacing
+     * the old row with the corresponding matching row.
+     *
+     * If there are multiple matches then the behavior is undefined.
+     * Currently this causes multiple copies of the row to be created
+     * but that behavior is subject to change.
+     *
+     * An optional condition may be specified.  If it is, then only
+     * matched rows that satisfy the condtion will be updated.  Any
+     * rows that do not satisfy the condition will be left as they
+     * are.  Failing to satisfy the condition does not cause a
+     * "matched row" to become a "not matched" row.
+     *
+     * The condition should be an SQL string.  Use the prefix
+     * target. to refer to rows in the target table (old data)
+     * and the prefix source. to refer to rows in the source
+     * table (new data).
+     *
+     * For example, "target.last_update < source.last_update"
+     */
+    whenMatchedUpdateAll(options?: {
+        where: string;
+    }): MergeInsertBuilder;
+    /**
+     * Rows that exist only in the source table (new data) should
+     * be inserted into the target table.
+     */
+    whenNotMatchedInsertAll(): MergeInsertBuilder;
+    /**
+     * Rows that exist only in the target table (old data) will be
+     * deleted.  An optional condition can be provided to limit what
+     * data is deleted.
+     *
+     * @param options.where - An optional condition to limit what data is deleted
+     */
+    whenNotMatchedBySourceDelete(options?: {
+        where: string;
+    }): MergeInsertBuilder;
+    /**
+     * Controls whether to use indexes for the merge operation.
+     *
+     * When set to `true` (the default), the operation will use an index if available
+     * on the join key for improved performance. When set to `false`, it forces a full
+     * table scan even if an index exists. This can be useful for benchmarking or when
+     * the query optimizer chooses a suboptimal path.
+     *
+     * @param useIndex - Whether to use indices for the merge operation. Defaults to `true`.
+     */
+    useIndex(useIndex: boolean): MergeInsertBuilder;
+    /**
+     * Control MemWAL routing for this merge.
+     *
+     * By default (unset), a `mergeInsert` on a table with an LSM write spec is
+     * routed through Lance's MemWAL shard writer, and a table without one uses the
+     * standard path.
+     *
+     * @param enable - `true` forces MemWAL routing and errors if the table has no
+     * LSM write spec. `false` forces the standard write path even when a spec is set.
+     */
+    useLsm(enable: boolean): MergeInsertBuilder;
+    /**
+     * Controls how an LSM merge checks that its input targets a single shard.
+     *
+     * When a table has an LSM write spec, every row in a `mergeInsert` call must
+     * route to the same shard. When `true` (the default), every row is inspected
+     * to verify this. When `false`, only the first row is inspected and the
+     * shard it routes to is used for the whole input — a faster path for callers
+     * that have already pre-sharded their input. Has no effect on tables without
+     * an LSM write spec.
+     *
+     * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
+     */
+    validateSingleShard(validateSingleShard: boolean): MergeInsertBuilder;
+    /**
+     * Executes the merge insert operation
+     *
+     * @returns {Promise<MergeResult>} the merge result
+     */
+    execute(data: Data, execOptions?: Partial<WriteExecutionOptions>): Promise<MergeResult>;
+}
+export interface WriteExecutionOptions {
+    /**
+     * Maximum time to run the operation before cancelling it.
+     *
+     * By default, there is a 30-second timeout that is only enforced after the
+     * first attempt. This is to prevent spending too long retrying to resolve
+     * conflicts. For example, if a write attempt takes 20 seconds and fails,
+     * the second attempt will be cancelled after 10 seconds, hitting the
+     * 30-second timeout. However, a write that takes one hour and succeeds on the
+     * first attempt will not be cancelled.
+     *
+     * When this is set, the timeout is enforced on all attempts, including the first.
+     */
+    timeoutMs?: number;
+}

package/vendor/lancedb/dist/merge.js

@@ -0,0 +1,120 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MergeInsertBuilder = void 0;
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+const arrow_1 = require("./arrow");
+/** A builder used to create and run a merge insert operation */
+class MergeInsertBuilder {
+    #native;
+    #schema;
+    /** Construct a MergeInsertBuilder. __Internal use only.__ */
+    constructor(native, schema) {
+        this.#native = native;
+        this.#schema = schema;
+    }
+    /**
+     * Rows that exist in both the source table (new data) and
+     * the target table (old data) will be updated, replacing
+     * the old row with the corresponding matching row.
+     *
+     * If there are multiple matches then the behavior is undefined.
+     * Currently this causes multiple copies of the row to be created
+     * but that behavior is subject to change.
+     *
+     * An optional condition may be specified.  If it is, then only
+     * matched rows that satisfy the condtion will be updated.  Any
+     * rows that do not satisfy the condition will be left as they
+     * are.  Failing to satisfy the condition does not cause a
+     * "matched row" to become a "not matched" row.
+     *
+     * The condition should be an SQL string.  Use the prefix
+     * target. to refer to rows in the target table (old data)
+     * and the prefix source. to refer to rows in the source
+     * table (new data).
+     *
+     * For example, "target.last_update < source.last_update"
+     */
+    whenMatchedUpdateAll(options) {
+        return new MergeInsertBuilder(this.#native.whenMatchedUpdateAll(options?.where), this.#schema);
+    }
+    /**
+     * Rows that exist only in the source table (new data) should
+     * be inserted into the target table.
+     */
+    whenNotMatchedInsertAll() {
+        return new MergeInsertBuilder(this.#native.whenNotMatchedInsertAll(), this.#schema);
+    }
+    /**
+     * Rows that exist only in the target table (old data) will be
+     * deleted.  An optional condition can be provided to limit what
+     * data is deleted.
+     *
+     * @param options.where - An optional condition to limit what data is deleted
+     */
+    whenNotMatchedBySourceDelete(options) {
+        return new MergeInsertBuilder(this.#native.whenNotMatchedBySourceDelete(options?.where), this.#schema);
+    }
+    /**
+     * Controls whether to use indexes for the merge operation.
+     *
+     * When set to `true` (the default), the operation will use an index if available
+     * on the join key for improved performance. When set to `false`, it forces a full
+     * table scan even if an index exists. This can be useful for benchmarking or when
+     * the query optimizer chooses a suboptimal path.
+     *
+     * @param useIndex - Whether to use indices for the merge operation. Defaults to `true`.
+     */
+    useIndex(useIndex) {
+        return new MergeInsertBuilder(this.#native.useIndex(useIndex), this.#schema);
+    }
+    /**
+     * Control MemWAL routing for this merge.
+     *
+     * By default (unset), a `mergeInsert` on a table with an LSM write spec is
+     * routed through Lance's MemWAL shard writer, and a table without one uses the
+     * standard path.
+     *
+     * @param enable - `true` forces MemWAL routing and errors if the table has no
+     * LSM write spec. `false` forces the standard write path even when a spec is set.
+     */
+    useLsm(enable) {
+        return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema);
+    }
+    /**
+     * Controls how an LSM merge checks that its input targets a single shard.
+     *
+     * When a table has an LSM write spec, every row in a `mergeInsert` call must
+     * route to the same shard. When `true` (the default), every row is inspected
+     * to verify this. When `false`, only the first row is inspected and the
+     * shard it routes to is used for the whole input — a faster path for callers
+     * that have already pre-sharded their input. Has no effect on tables without
+     * an LSM write spec.
+     *
+     * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
+     */
+    validateSingleShard(validateSingleShard) {
+        return new MergeInsertBuilder(this.#native.validateSingleShard(validateSingleShard), this.#schema);
+    }
+    /**
+     * Executes the merge insert operation
+     *
+     * @returns {Promise<MergeResult>} the merge result
+     */
+    async execute(data, execOptions) {
+        let schema;
+        if (this.#schema instanceof Promise) {
+            schema = await this.#schema;
+            this.#schema = schema; // In case of future calls
+        }
+        else {
+            schema = this.#schema;
+        }
+        if (execOptions?.timeoutMs !== undefined) {
+            this.#native.setTimeout(execOptions.timeoutMs);
+        }
+        const buffer = await (0, arrow_1.fromDataToBuffer)(data, undefined, schema);
+        return await this.#native.execute(buffer);
+    }
+}
+exports.MergeInsertBuilder = MergeInsertBuilder;