presubmit 0.1.0

package/vendor/lancedb/dist/arrow.d.ts

@@ -0,0 +1,296 @@
+import { Table as ArrowTable, Binary, BufferType, DataType, Field, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, RecordBatch, Schema, Struct, Utf8 } from "apache-arrow";
+import { Buffers } from "apache-arrow/data";
+import { type EmbeddingFunction } from "./embedding/embedding_function";
+import { EmbeddingFunctionConfig } from "./embedding/registry";
+export * from "apache-arrow";
+export type SchemaLike = Schema | {
+    fields: FieldLike[];
+    metadata: Map<string, string>;
+    get names(): unknown[];
+};
+export type FieldLike = Field | {
+    type: string;
+    name: string;
+    nullable: boolean;
+    metadata?: Map<string, string>;
+};
+export type DataLike = import("apache-arrow").Data | {
+    type: any;
+    length: number;
+    offset: number;
+    stride: number;
+    nullable: boolean;
+    children: DataLike[];
+    dictionary?: {
+        data: readonly DataLike[];
+    };
+    get nullCount(): number;
+    values: Buffers<any>[BufferType.DATA];
+    typeIds: Buffers<any>[BufferType.TYPE];
+    nullBitmap: Buffers<any>[BufferType.VALIDITY];
+    valueOffsets: Buffers<any>[BufferType.OFFSET];
+};
+export type RecordBatchLike = RecordBatch | {
+    schema: SchemaLike;
+    data: DataLike;
+};
+export type TableLike = ArrowTable | {
+    schema: SchemaLike;
+    batches: RecordBatchLike[];
+};
+export type IntoVector = Float32Array | Float64Array | Uint8Array | number[] | Promise<Float32Array | Float64Array | Uint8Array | number[]>;
+export type MultiVector = IntoVector[];
+export declare function isMultiVector(value: unknown): value is MultiVector;
+export declare function isIntoVector(value: unknown): value is IntoVector;
+/**
+ * Extract the underlying byte buffer and data type from a typed array
+ * for passing to the Rust NAPI layer without precision loss.
+ */
+export declare function extractVectorBuffer(vector: Float32Array | Float64Array | Uint8Array): {
+    data: Uint8Array;
+    dtype: string;
+} | null;
+export declare function isArrowTable(value: object): value is TableLike;
+export declare function isNull(value: unknown): value is Null;
+export declare function isInt(value: unknown): value is Int;
+export declare function isFloat(value: unknown): value is Float;
+export declare function isBinary(value: unknown): value is Binary;
+export declare function isLargeBinary(value: unknown): value is LargeBinary;
+export declare function isUtf8(value: unknown): value is Utf8;
+export declare function isLargeUtf8(value: unknown): value is Utf8;
+export declare function isBool(value: unknown): value is Utf8;
+export declare function isDecimal(value: unknown): value is Utf8;
+export declare function isDate(value: unknown): value is Utf8;
+export declare function isTime(value: unknown): value is Utf8;
+export declare function isTimestamp(value: unknown): value is Utf8;
+export declare function isInterval(value: unknown): value is Utf8;
+export declare function isDuration(value: unknown): value is Utf8;
+export declare function isList(value: unknown): value is List;
+export declare function isStruct(value: unknown): value is Struct;
+export declare function isUnion(value: unknown): value is Struct;
+export declare function isFixedSizeBinary(value: unknown): value is FixedSizeBinary;
+export declare function isFixedSizeList(value: unknown): value is FixedSizeList;
+/** Data type accepted by NodeJS SDK */
+export type Data = Record<string, unknown>[] | TableLike;
+export declare class VectorColumnOptions {
+    /** Vector column type. */
+    type: Float;
+    constructor(values?: Partial<VectorColumnOptions>);
+}
+/** Options to control the makeArrowTable call. */
+export declare class MakeArrowTableOptions {
+    schema?: SchemaLike;
+    vectorColumns: Record<string, VectorColumnOptions>;
+    embeddings?: EmbeddingFunction<unknown>;
+    embeddingFunction?: EmbeddingFunctionConfig;
+    /**
+     * If true then string columns will be encoded with dictionary encoding
+     *
+     * Set this to true if your string columns tend to repeat the same values
+     * often.  For more precise control use the `schema` property to specify the
+     * data type for individual columns.
+     *
+     * If `schema` is provided then this property is ignored.
+     */
+    dictionaryEncodeStrings: boolean;
+    constructor(values?: Partial<MakeArrowTableOptions>);
+}
+/**
+ * An enhanced version of the apache-arrow makeTable function from Apache Arrow
+ * that supports nested fields and embeddings columns.
+ *
+ * (typically you do not need to call this function.  It will be called automatically
+ * when creating a table or adding data to it)
+ *
+ * This function converts an array of Record<String, any> (row-major JS objects)
+ * to an Arrow Table (a columnar structure)
+ *
+ * If a schema is provided then it will be used to determine the resulting array
+ * types.  Fields will also be reordered to fit the order defined by the schema.
+ *
+ * If a schema is not provided then the types will be inferred and the field order
+ * will be controlled by the order of properties in the first record.  If a type
+ * is inferred it will always be nullable.
+ *
+ * If not all fields are found in the data, then a subset of the schema will be
+ * returned.
+ *
+ * If the input is empty then a schema must be provided to create an empty table.
+ *
+ * When a schema is not specified then data types will be inferred.  The inference
+ * rules are as follows:
+ *
+ *  - boolean => Bool
+ *  - number => Float64
+ *  - bigint => Int64
+ *  - String => Utf8
+ *  - Buffer => Binary
+ *  - Record<String, any> => Struct
+ *  - Array<any> => List
+ * @example
+ * ```ts
+ * import { fromTableToBuffer, makeArrowTable } from "../arrow";
+ * import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
+ *
+ * const schema = new Schema([
+ *   new Field("a", new Int32()),
+ *   new Field("b", new Float32()),
+ *   new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
+ *  ]);
+ *  const table = makeArrowTable([
+ *    { a: 1, b: 2, c: [1, 2, 3] },
+ *    { a: 4, b: 5, c: [4, 5, 6] },
+ *    { a: 7, b: 8, c: [7, 8, 9] },
+ *  ], { schema });
+ * ```
+ *
+ * By default it assumes that the column named `vector` is a vector column
+ * and it will be converted into a fixed size list array of type float32.
+ * The `vectorColumns` option can be used to support other vector column
+ * names and data types.
+ *
+ * ```ts
+ * const schema = new Schema([
+ *   new Field("a", new Float64()),
+ *   new Field("b", new Float64()),
+ *   new Field(
+ *     "vector",
+ *     new FixedSizeList(3, new Field("item", new Float32()))
+ *   ),
+ * ]);
+ * const table = makeArrowTable([
+ *   { a: 1, b: 2, vector: [1, 2, 3] },
+ *   { a: 4, b: 5, vector: [4, 5, 6] },
+ *   { a: 7, b: 8, vector: [7, 8, 9] },
+ * ]);
+ * assert.deepEqual(table.schema, schema);
+ * ```
+ *
+ * You can specify the vector column types and names using the options as well
+ *
+ * ```ts
+ * const schema = new Schema([
+ *   new Field('a', new Float64()),
+ *   new Field('b', new Float64()),
+ *   new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
+ *   new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
+ * ]);
+ * const table = makeArrowTable([
+ *   { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+ *   { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+ *   { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
+ * ], {
+ *   vectorColumns: {
+ *     vec1: { type: new Float16() },
+ *     vec2: { type: new Float16() }
+ *   }
+ * }
+ * assert.deepEqual(table.schema, schema)
+ * ```
+ */
+export declare function makeArrowTable(data: Array<Record<string, unknown>>, options?: Partial<MakeArrowTableOptions>, metadata?: Map<string, string>): ArrowTable;
+/**
+ * Create an empty Arrow table with the provided schema
+ */
+export declare function makeEmptyTable(schema: SchemaLike, metadata?: Map<string, string>): ArrowTable;
+/**
+ * Convert an Array of records into an Arrow Table, optionally applying an
+ * embeddings function to it.
+ *
+ * This function calls `makeArrowTable` first to create the Arrow Table.
+ * Any provided `makeTableOptions` (e.g. a schema) will be passed on to
+ * that call.
+ *
+ * The embedding function will be passed a column of values (based on the
+ * `sourceColumn` of the embedding function) and expects to receive back
+ * number[][] which will be converted into a fixed size list column.  By
+ * default this will be a fixed size list of Float32 but that can be
+ * customized by the `embeddingDataType` property of the embedding function.
+ *
+ * If a schema is provided in `makeTableOptions` then it should include the
+ * embedding columns.  If no schema is provded then embedding columns will
+ * be placed at the end of the table, after all of the input columns.
+ */
+export declare function convertToTable(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, makeTableOptions?: Partial<MakeArrowTableOptions>): Promise<ArrowTable>;
+/** Creates the Arrow Type for a Vector column with dimension `dim` */
+export declare function newVectorType<T extends Float>(dim: number, innerType: unknown): FixedSizeList<T>;
+/**
+ * Serialize an Array of records into a buffer using the Arrow IPC File serialization
+ *
+ * This function will call `convertToTable` and pass on `embeddings` and `schema`
+ *
+ * `schema` is required if data is empty
+ */
+export declare function fromRecordsToBuffer(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
+/**
+ * Serialize an Array of records into a buffer using the Arrow IPC Stream serialization
+ *
+ * This function will call `convertToTable` and pass on `embeddings` and `schema`
+ *
+ * `schema` is required if data is empty
+ */
+export declare function fromRecordsToStreamBuffer(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
+/**
+ * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
+ *
+ * This function will apply `embeddings` to the table in a manner similar to
+ * `convertToTable`.
+ *
+ * `schema` is required if the table is empty
+ */
+export declare function fromTableToBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?: SchemaLike): Promise<Buffer>;
+/**
+ * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
+ *
+ * This function will apply `embeddings` to the table in a manner similar to
+ * `convertToTable`.
+ *
+ * `schema` is required if the table is empty
+ */
+export declare function fromDataToBuffer(data: Data, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
+/**
+ * Read a single record batch from a buffer.
+ *
+ * Returns null if the buffer does not contain a record batch
+ */
+export declare function fromBufferToRecordBatch(data: Buffer): Promise<RecordBatch | null>;
+/**
+ * Create a buffer containing a single record batch
+ */
+export declare function fromRecordBatchToBuffer(batch: RecordBatch): Promise<Buffer>;
+/**
+ * Create a buffer containing a single record batch using the Arrow IPC Stream
+ * serialization. Each call produces a self-contained Stream message (schema +
+ * batch + EOS) suitable for incremental decode by `arrow_ipc::reader::StreamReader`.
+ */
+export declare function fromRecordBatchToStreamBuffer(batch: RecordBatch): Promise<Buffer>;
+/**
+ * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
+ *
+ * This function will apply `embeddings` to the table in a manner similar to
+ * `convertToTable`.
+ *
+ * `schema` is required if the table is empty
+ */
+export declare function fromTableToStreamBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?: SchemaLike): Promise<Buffer>;
+/**
+ * Create an empty table with the given schema
+ */
+export declare function createEmptyTable(schema: Schema): ArrowTable;
+/**
+ * Ensures that all nested fields defined in the schema exist in the data,
+ * filling missing fields with null values.
+ */
+export declare function ensureNestedFieldsExist(data: Array<Record<string, unknown>>, schema: Schema): Array<Record<string, unknown>>;
+interface JsonDataType {
+    type: string;
+    fields?: JsonField[];
+    length?: number;
+}
+interface JsonField {
+    name: string;
+    type: JsonDataType;
+    nullable: boolean;
+    metadata: Map<string, string>;
+}
+export declare function dataTypeToJson(dataType: DataType): JsonDataType;