redis-vl 0.1.0-beta.0

package/dist/storage/json-storage.js

@@ -0,0 +1,81 @@
+import { BaseStorage } from './base-storage.js';
+import { SchemaValidationError } from '../errors.js';
+/**
+ * Storage implementation for Redis JSON data type.
+ * Implements JSON-specific logic for validation and read/write operations.
+ */
+export class JsonStorage extends BaseStorage {
+    constructor(schema) {
+        super(schema);
+    }
+    /**
+     * Write documents to Redis as JSON entries using pipelining for performance.
+     */
+    async write(client, data, options = {}) {
+        const { keys, ttl, batchSize = this.defaultBatchSize } = options;
+        // Validate inputs
+        if (data.length === 0) {
+            return [];
+        }
+        if (keys && keys.length !== data.length) {
+            throw new SchemaValidationError('Length of keys does not match the length of data');
+        }
+        // Pass 1: Preprocess and validate all documents
+        const preparedDocs = await this.preprocessAndValidateDocuments(data, options);
+        // Pass 2: Write all valid documents in batches using pipeline
+        const loadedKeys = [];
+        let pipeline = client.multi();
+        let commandCount = 0;
+        for (let i = 0; i < preparedDocs.length; i++) {
+            const { key, doc } = preparedDocs[i];
+            // Add JSON.SET command to pipeline
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            pipeline.json.set(key, '$', doc);
+            commandCount++;
+            // Add EXPIRE command if TTL is provided
+            if (ttl) {
+                pipeline.expire(key, ttl);
+                commandCount++;
+            }
+            loadedKeys.push(key);
+            // Execute pipeline in batches
+            if (commandCount >= batchSize || i === preparedDocs.length - 1) {
+                await pipeline.execAsPipeline();
+                pipeline = client.multi();
+                commandCount = 0;
+            }
+        }
+        return loadedKeys;
+    }
+    /**
+     * Read documents from Redis by keys using pipelining.
+     */
+    async get(client, keys, batchSize = this.defaultBatchSize) {
+        if (!keys || keys.length === 0) {
+            return [];
+        }
+        const results = [];
+        // Process in batches
+        for (let i = 0; i < keys.length; i += batchSize) {
+            const batch = keys.slice(i, i + batchSize);
+            const pipeline = client.multi();
+            // Add JSON.GET commands to pipeline
+            for (const key of batch) {
+                pipeline.json.get(key);
+            }
+            // Execute pipeline and get results
+            const batchResults = (await pipeline.execAsPipeline());
+            // Process batch results
+            for (const data of batchResults) {
+                if (data && typeof data === 'object') {
+                    results.push(data);
+                }
+                else {
+                    results.push(null);
+                }
+            }
+        }
+        return results;
+    }
+}
+//# sourceMappingURL=json-storage.js.map

package/dist/storage/json-storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-storage.js","sourceRoot":"","sources":["../../src/storage/json-storage.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAErD;;;GAGG;AACH,MAAM,OAAO,WAAY,SAAQ,WAAW;IACxC,YAAY,MAAmB;QAC3B,KAAK,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CACP,MAAmB,EACnB,IAA+B,EAC/B,UAAwB,EAAE;QAE1B,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,GAAG,OAAO,CAAC;QAEjE,kBAAkB;QAClB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;QACd,CAAC;QAED,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,qBAAqB,CAAC,kDAAkD,CAAC,CAAC;QACxF,CAAC;QAED,gDAAgD;QAChD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,8BAA8B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAE9E,8DAA8D;QAC9D,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,IAAI,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAErC,mCAAmC;YACnC,8DAA8D;YAC9D,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAU,CAAC,CAAC;YACxC,YAAY,EAAE,CAAC;YAEf,wCAAwC;YACxC,IAAI,GAAG,EAAE,CAAC;gBACN,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBAC1B,YAAY,EAAE,CAAC;YACnB,CAAC;YAED,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAErB,8BAA8B;YAC9B,IAAI,YAAY,IAAI,SAAS,IAAI,CAAC,KAAK,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7D,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC;gBAChC,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;gBAC1B,YAAY,GAAG,CAAC,CAAC;YACrB,CAAC;QACL,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CACL,MAAmB,EACnB,IAAc,EACd,SAAS,GAAG,IAAI,CAAC,gBAAgB;QAEjC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,OAAO,GAAuC,EAAE,CAAC;QAEvD,qBAAqB;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC3C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;YAEhC,oCAAoC;YACpC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;YAED,mCAAmC;YACnC,MAAM,YAAY,GAAG,CAAC,MAAM,QAAQ,CAAC,cAAc,EAAE,CAG5C,CAAC;YAEV,wBAAwB;YACxB,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;gBAC9B,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACnC,OAAO,CAAC,IAAI,CAAC,IAA+B,CAAC,CAAC;gBAClD,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACvB,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ"}

package/dist/utils/distance.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Utility functions for normalizing vector distance metrics.
+ *
+ * Redis supports three distance metrics:
+ * - COSINE: Returns values between 0 and 2
+ * - L2 (Euclidean): Returns unbounded values (0 to infinity)
+ * - IP (Inner Product): Returns values determined by vector magnitude
+ *
+ * These functions normalize distances to similarity scores between 0 and 1:
+ * - 1.0 = perfect match (most similar)
+ * - 0.0 = completely different (least similar)
+ */
+/**
+ * Normalize a COSINE distance to a similarity score between 0 and 1.
+ *
+ * COSINE distance in Redis ranges from 0 (identical vectors) to 2 (opposite vectors).
+ * This function converts it to a similarity score where:
+ * - distance 0 → similarity 1.0 (identical)
+ * - distance 1 → similarity 0.5 (orthogonal)
+ * - distance 2 → similarity 0.0 (opposite)
+ *
+ * Formula: similarity = (2 - distance) / 2
+ *
+ * @param distance - COSINE distance value from Redis (0 to 2)
+ * @returns Normalized similarity score (0 to 1)
+ *
+ * @example
+ * ```typescript
+ * normalizeCosineDistance(0)   // 1.0 (identical)
+ * normalizeCosineDistance(1)   // 0.5 (orthogonal)
+ * normalizeCosineDistance(2)   // 0.0 (opposite)
+ * ```
+ */
+export declare function normalizeCosineDistance(distance: number): number;
+/**
+ * Denormalize a similarity score to a COSINE distance.
+ *
+ * This is the inverse of normalizeCosineDistance.
+ * Converts a similarity score (0 to 1) back to COSINE distance (0 to 2).
+ *
+ * Formula: distance = 2 - 2 * similarity
+ *
+ * @param similarity - Normalized similarity score (0 to 1)
+ * @returns COSINE distance value (0 to 2)
+ *
+ * @example
+ * ```typescript
+ * denormalizeCosineDistance(1.0)  // 0 (identical)
+ * denormalizeCosineDistance(0.5)  // 1 (orthogonal)
+ * denormalizeCosineDistance(0.0)  // 2 (opposite)
+ * ```
+ */
+export declare function denormalizeCosineDistance(similarity: number): number;
+/**
+ * Normalize an L2 (Euclidean) distance to a similarity score between 0 and 1.
+ *
+ * L2 distance in Redis is unbounded (0 to infinity).
+ * This function converts it to a similarity score using the formula:
+ * similarity = 1 / (1 + distance)
+ *
+ * - distance 0 → similarity 1.0 (identical)
+ * - distance 1 → similarity 0.5
+ * - distance ∞ → similarity 0.0 (very different)
+ *
+ * @param distance - L2 distance value from Redis (0 to infinity)
+ * @returns Normalized similarity score (0 to 1)
+ *
+ * @example
+ * ```typescript
+ * normalizeL2Distance(0)    // 1.0 (identical)
+ * normalizeL2Distance(1)    // 0.5
+ * normalizeL2Distance(9)    // 0.1
+ * normalizeL2Distance(99)   // 0.01
+ * ```
+ */
+export declare function normalizeL2Distance(distance: number): number;
+/**
+ * Type definition for distance normalization functions.
+ */
+export type DistanceNormalizer = (distance: number) => number;
+/**
+ * Map of distance metrics to their normalization functions.
+ *
+ * - COSINE: Normalizes from [0, 2] to [0, 1]
+ * - L2: Normalizes from [0, ∞] to [0, 1]
+ * - IP: null (Inner Product is not normalized; use COSINE instead)
+ *
+ * Note: COSINE is normalized Inner Product by definition, so IP
+ * should not be normalized separately.
+ */
+export declare const DISTANCE_NORMALIZERS: Record<string, DistanceNormalizer | null>;
+//# sourceMappingURL=distance.d.ts.map

package/dist/utils/distance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"distance.d.ts","sourceRoot":"","sources":["../../src/utils/distance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,yBAAyB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE5D;AAED;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAAC,QAAQ,EAAE,MAAM,KAAK,MAAM,CAAC;AAE9D;;;;;;;;;GASG;AACH,eAAO,MAAM,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAI1E,CAAC"}

package/dist/utils/distance.js

@@ -0,0 +1,98 @@
+/**
+ * Utility functions for normalizing vector distance metrics.
+ *
+ * Redis supports three distance metrics:
+ * - COSINE: Returns values between 0 and 2
+ * - L2 (Euclidean): Returns unbounded values (0 to infinity)
+ * - IP (Inner Product): Returns values determined by vector magnitude
+ *
+ * These functions normalize distances to similarity scores between 0 and 1:
+ * - 1.0 = perfect match (most similar)
+ * - 0.0 = completely different (least similar)
+ */
+/**
+ * Normalize a COSINE distance to a similarity score between 0 and 1.
+ *
+ * COSINE distance in Redis ranges from 0 (identical vectors) to 2 (opposite vectors).
+ * This function converts it to a similarity score where:
+ * - distance 0 → similarity 1.0 (identical)
+ * - distance 1 → similarity 0.5 (orthogonal)
+ * - distance 2 → similarity 0.0 (opposite)
+ *
+ * Formula: similarity = (2 - distance) / 2
+ *
+ * @param distance - COSINE distance value from Redis (0 to 2)
+ * @returns Normalized similarity score (0 to 1)
+ *
+ * @example
+ * ```typescript
+ * normalizeCosineDistance(0)   // 1.0 (identical)
+ * normalizeCosineDistance(1)   // 0.5 (orthogonal)
+ * normalizeCosineDistance(2)   // 0.0 (opposite)
+ * ```
+ */
+export function normalizeCosineDistance(distance) {
+    return Math.max((2 - distance) / 2, 0);
+}
+/**
+ * Denormalize a similarity score to a COSINE distance.
+ *
+ * This is the inverse of normalizeCosineDistance.
+ * Converts a similarity score (0 to 1) back to COSINE distance (0 to 2).
+ *
+ * Formula: distance = 2 - 2 * similarity
+ *
+ * @param similarity - Normalized similarity score (0 to 1)
+ * @returns COSINE distance value (0 to 2)
+ *
+ * @example
+ * ```typescript
+ * denormalizeCosineDistance(1.0)  // 0 (identical)
+ * denormalizeCosineDistance(0.5)  // 1 (orthogonal)
+ * denormalizeCosineDistance(0.0)  // 2 (opposite)
+ * ```
+ */
+export function denormalizeCosineDistance(similarity) {
+    return Math.max(2 - 2 * similarity, 0);
+}
+/**
+ * Normalize an L2 (Euclidean) distance to a similarity score between 0 and 1.
+ *
+ * L2 distance in Redis is unbounded (0 to infinity).
+ * This function converts it to a similarity score using the formula:
+ * similarity = 1 / (1 + distance)
+ *
+ * - distance 0 → similarity 1.0 (identical)
+ * - distance 1 → similarity 0.5
+ * - distance ∞ → similarity 0.0 (very different)
+ *
+ * @param distance - L2 distance value from Redis (0 to infinity)
+ * @returns Normalized similarity score (0 to 1)
+ *
+ * @example
+ * ```typescript
+ * normalizeL2Distance(0)    // 1.0 (identical)
+ * normalizeL2Distance(1)    // 0.5
+ * normalizeL2Distance(9)    // 0.1
+ * normalizeL2Distance(99)   // 0.01
+ * ```
+ */
+export function normalizeL2Distance(distance) {
+    return 1 / (1 + distance);
+}
+/**
+ * Map of distance metrics to their normalization functions.
+ *
+ * - COSINE: Normalizes from [0, 2] to [0, 1]
+ * - L2: Normalizes from [0, ∞] to [0, 1]
+ * - IP: null (Inner Product is not normalized; use COSINE instead)
+ *
+ * Note: COSINE is normalized Inner Product by definition, so IP
+ * should not be normalized separately.
+ */
+export const DISTANCE_NORMALIZERS = {
+    COSINE: normalizeCosineDistance,
+    L2: normalizeL2Distance,
+    IP: null, // Normalized inner product is COSINE by definition
+};
+//# sourceMappingURL=distance.js.map

package/dist/utils/distance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"distance.js","sourceRoot":"","sources":["../../src/utils/distance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,uBAAuB,CAAC,QAAgB;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,yBAAyB,CAAC,UAAkB;IACxD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAChD,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;AAC9B,CAAC;AAOD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAA8C;IAC3E,MAAM,EAAE,uBAAuB;IAC/B,EAAE,EAAE,mBAAmB;IACvB,EAAE,EAAE,IAAI,EAAE,mDAAmD;CAChE,CAAC"}

package/dist/vectorizers/base-vectorizer.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Base abstract class for all vectorizers.
+ *
+ * Vectorizers are responsible for converting text into numerical embeddings (vectors)
+ * that can be stored in Redis and used for semantic search.
+ *
+ * This class defines the interface that all concrete vectorizer implementations must follow.
+ *
+ * @example
+ * ```typescript
+ * class MyVectorizer extends BaseVectorizer {
+ *   async embed(text: string): Promise<number[]> {
+ *     // Implementation
+ *   }
+ *
+ *   async embedMany(texts: string[], batchSize?: number): Promise<number[][]> {
+ *     // Implementation
+ *   }
+ *
+ *   get dims(): number {
+ *     return 384; // Embedding dimensions
+ *   }
+ *
+ *   get model(): string {
+ *     return 'my-model-name';
+ *   }
+ * }
+ * ```
+ */
+export declare abstract class BaseVectorizer {
+    /**
+     * Generate an embedding for a single text.
+     *
+     * @param text - The text to embed
+     * @returns A promise that resolves to the embedding vector
+     *
+     * @example
+     * ```typescript
+     * const embedding = await vectorizer.embed('Hello world');
+     * console.log(embedding); // [0.1, 0.2, 0.3, ...]
+     * ```
+     */
+    abstract embed(text: string): Promise<number[]>;
+    /**
+     * Generate embeddings for multiple texts.
+     *
+     * This method should be more efficient than calling embed() multiple times
+     * by batching requests to the underlying model.
+     *
+     * @param texts - Array of texts to embed
+     * @param batchSize - Optional batch size for processing (default: 32)
+     * @returns A promise that resolves to an array of embedding vectors
+     *
+     * @example
+     * ```typescript
+     * const embeddings = await vectorizer.embedMany(['Hello', 'World']);
+     * console.log(embeddings); // [[0.1, 0.2, ...], [0.3, 0.4, ...]]
+     * ```
+     */
+    abstract embedMany(texts: string[], batchSize?: number): Promise<number[][]>;
+    /**
+     * Get the dimensionality of the embeddings produced by this vectorizer.
+     *
+     * This should match the length of the arrays returned by embed() and embedMany().
+     *
+     * @returns The number of dimensions in the embedding vectors
+     *
+     * @example
+     * ```typescript
+     * console.log(vectorizer.dims); // 384
+     * const embedding = await vectorizer.embed('test');
+     * console.log(embedding.length === vectorizer.dims); // true
+     * ```
+     */
+    abstract get dims(): number;
+    /**
+     * Get the name/identifier of the model used by this vectorizer.
+     *
+     * @returns The model name or identifier
+     *
+     * @example
+     * ```typescript
+     * console.log(vectorizer.model); // "sentence-transformers/all-MiniLM-L6-v2"
+     * ```
+     */
+    abstract get model(): string;
+}
+//# sourceMappingURL=base-vectorizer.d.ts.map

package/dist/vectorizers/base-vectorizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-vectorizer.d.ts","sourceRoot":"","sources":["../../src/vectorizers/base-vectorizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,8BAAsB,cAAc;IAChC;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAE/C;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAE5E;;;;;;;;;;;;;OAaG;IACH,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC;IAE5B;;;;;;;;;OASG;IACH,QAAQ,KAAK,KAAK,IAAI,MAAM,CAAC;CAChC"}

package/dist/vectorizers/base-vectorizer.js

@@ -0,0 +1,32 @@
+/**
+ * Base abstract class for all vectorizers.
+ *
+ * Vectorizers are responsible for converting text into numerical embeddings (vectors)
+ * that can be stored in Redis and used for semantic search.
+ *
+ * This class defines the interface that all concrete vectorizer implementations must follow.
+ *
+ * @example
+ * ```typescript
+ * class MyVectorizer extends BaseVectorizer {
+ *   async embed(text: string): Promise<number[]> {
+ *     // Implementation
+ *   }
+ *
+ *   async embedMany(texts: string[], batchSize?: number): Promise<number[][]> {
+ *     // Implementation
+ *   }
+ *
+ *   get dims(): number {
+ *     return 384; // Embedding dimensions
+ *   }
+ *
+ *   get model(): string {
+ *     return 'my-model-name';
+ *   }
+ * }
+ * ```
+ */
+export class BaseVectorizer {
+}
+//# sourceMappingURL=base-vectorizer.js.map

package/dist/vectorizers/base-vectorizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-vectorizer.js","sourceRoot":"","sources":["../../src/vectorizers/base-vectorizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAgB,cAAc;CA4DnC"}

package/dist/vectorizers/huggingface-vectorizer.d.ts

@@ -0,0 +1,103 @@
+import { BaseVectorizer } from './base-vectorizer.js';
+/**
+ * Configuration options for HuggingFaceVectorizer
+ */
+export interface HuggingFaceConfig {
+    /**
+     * The Hugging Face model to use for embeddings.
+     *
+     * Examples:
+     * - 'Xenova/all-MiniLM-L6-v2' (384 dimensions, fast)
+     * - 'Xenova/all-mpnet-base-v2' (768 dimensions, better quality)
+     * - 'sentence-transformers/all-MiniLM-L6-v2'
+     */
+    model: string;
+    /**
+     * Device to run inference on.
+     * - 'cpu': Run on CPU (default)
+     * - 'webgpu': Run on GPU (if available)
+     */
+    device?: 'cpu' | 'webgpu';
+    /**
+     * Data type for model weights.
+     * - 'fp32': 32-bit floating point (default, best quality)
+     * - 'fp16': 16-bit floating point (faster, less memory)
+     * - 'q8': 8-bit quantized (fastest, smallest)
+     */
+    dtype?: 'fp32' | 'fp16' | 'q8';
+    /**
+     * Pooling strategy for combining token embeddings.
+     * - 'mean': Average all token embeddings (default)
+     * - 'cls': Use [CLS] token embedding
+     */
+    pooling?: 'mean' | 'cls';
+    /**
+     * Whether to normalize embeddings to unit length (L2 norm = 1).
+     * Default: true
+     */
+    normalize?: boolean;
+}
+/**
+ * HuggingFace vectorizer using Transformers.js for local inference.
+ *
+ * This vectorizer uses the @huggingface/transformers library to generate
+ * embeddings locally without requiring API keys or external services.
+ *
+ * Models are automatically downloaded and cached on first use.
+ *
+ * @example
+ * ```typescript
+ * import { HuggingFaceVectorizer } from 'redis-vl';
+ *
+ * const vectorizer = new HuggingFaceVectorizer({
+ *   model: 'Xenova/all-MiniLM-L6-v2'
+ * });
+ *
+ * // Generate single embedding
+ * const embedding = await vectorizer.embed('Hello world');
+ * console.log(embedding.length); // 384
+ *
+ * // Generate multiple embeddings
+ * const embeddings = await vectorizer.embedMany(['Hello', 'World']);
+ * console.log(embeddings.length); // 2
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Use with preprocessing
+ * await index.load(documents, {
+ *   preprocess: async (doc) => ({
+ *     ...doc,
+ *     embedding: await vectorizer.embed(doc.text)
+ *   })
+ * });
+ * ```
+ */
+export declare class HuggingFaceVectorizer extends BaseVectorizer {
+    private readonly config;
+    private pipelineInstance;
+    private modelDims;
+    constructor(config: HuggingFaceConfig);
+    /**
+     * Lazy load the Transformers.js pipeline.
+     * This ensures the library is only loaded when actually needed.
+     */
+    private loadPipeline;
+    /**
+     * Generate an embedding for a single text.
+     */
+    embed(text: string): Promise<number[]>;
+    /**
+     * Generate embeddings for multiple texts.
+     */
+    embedMany(texts: string[], batchSize?: number): Promise<number[][]>;
+    /**
+     * Get the dimensionality of the embeddings.
+     */
+    get dims(): number;
+    /**
+     * Get the model name.
+     */
+    get model(): string;
+}
+//# sourceMappingURL=huggingface-vectorizer.d.ts.map

package/dist/vectorizers/huggingface-vectorizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"huggingface-vectorizer.d.ts","sourceRoot":"","sources":["../../src/vectorizers/huggingface-vectorizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGtD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAC9B;;;;;;;OAOG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;;OAIG;IACH,MAAM,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC;IAE1B;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAE/B;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAEzB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,qBAAa,qBAAsB,SAAQ,cAAc;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA8B;IACrD,OAAO,CAAC,gBAAgB,CAAa;IACrC,OAAO,CAAC,SAAS,CAAuB;gBAE5B,MAAM,EAAE,iBAAiB;IAiBrC;;;OAGG;YACW,YAAY;IA0B1B;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAmB5C;;OAEG;IACG,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAW,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAsC7E;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAQjB;IAED;;OAEG;IACH,IAAI,KAAK,IAAI,MAAM,CAElB;CACJ"}

package/dist/vectorizers/huggingface-vectorizer.js

@@ -0,0 +1,142 @@
+import { BaseVectorizer } from './base-vectorizer.js';
+import { VectorizerError } from '../errors.js';
+/**
+ * HuggingFace vectorizer using Transformers.js for local inference.
+ *
+ * This vectorizer uses the @huggingface/transformers library to generate
+ * embeddings locally without requiring API keys or external services.
+ *
+ * Models are automatically downloaded and cached on first use.
+ *
+ * @example
+ * ```typescript
+ * import { HuggingFaceVectorizer } from 'redis-vl';
+ *
+ * const vectorizer = new HuggingFaceVectorizer({
+ *   model: 'Xenova/all-MiniLM-L6-v2'
+ * });
+ *
+ * // Generate single embedding
+ * const embedding = await vectorizer.embed('Hello world');
+ * console.log(embedding.length); // 384
+ *
+ * // Generate multiple embeddings
+ * const embeddings = await vectorizer.embedMany(['Hello', 'World']);
+ * console.log(embeddings.length); // 2
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Use with preprocessing
+ * await index.load(documents, {
+ *   preprocess: async (doc) => ({
+ *     ...doc,
+ *     embedding: await vectorizer.embed(doc.text)
+ *   })
+ * });
+ * ```
+ */
+export class HuggingFaceVectorizer extends BaseVectorizer {
+    config;
+    pipelineInstance = null;
+    modelDims = null;
+    constructor(config) {
+        super();
+        if (!config.model || config.model.trim() === '') {
+            throw new VectorizerError('Model name cannot be empty');
+        }
+        // Set defaults
+        this.config = {
+            model: config.model,
+            device: config.device ?? 'cpu',
+            dtype: config.dtype ?? 'fp32',
+            pooling: config.pooling ?? 'mean',
+            normalize: config.normalize ?? true,
+        };
+    }
+    /**
+     * Lazy load the Transformers.js pipeline.
+     * This ensures the library is only loaded when actually needed.
+     */
+    async loadPipeline() {
+        if (this.pipelineInstance) {
+            return this.pipelineInstance;
+        }
+        try {
+            const { pipeline } = await import('@huggingface/transformers');
+            this.pipelineInstance = await pipeline('feature-extraction', this.config.model, {
+                device: this.config.device,
+                dtype: this.config.dtype,
+            });
+            return this.pipelineInstance;
+        }
+        catch (error) {
+            if (error instanceof Error && error.message.includes('Cannot find module')) {
+                throw new VectorizerError('HuggingFaceVectorizer requires @huggingface/transformers. ' +
+                    'Install it with: npm install @huggingface/transformers', { cause: error });
+            }
+            throw error;
+        }
+    }
+    /**
+     * Generate an embedding for a single text.
+     */
+    async embed(text) {
+        const pipeline = await this.loadPipeline();
+        const output = await pipeline(text, {
+            pooling: this.config.pooling,
+            normalize: this.config.normalize,
+        });
+        // Extract the embedding from the output
+        const embedding = Array.from(output.data);
+        // Cache dimensions on first call
+        if (this.modelDims === null) {
+            this.modelDims = embedding.length;
+        }
+        return embedding;
+    }
+    /**
+     * Generate embeddings for multiple texts.
+     */
+    async embedMany(texts, batchSize = 32) {
+        if (texts.length === 0) {
+            return [];
+        }
+        const pipeline = await this.loadPipeline();
+        const results = [];
+        // Process in batches for memory efficiency
+        for (let i = 0; i < texts.length; i += batchSize) {
+            const batch = texts.slice(i, i + batchSize);
+            // Process batch
+            const batchOutputs = await Promise.all(batch.map((text) => pipeline(text, {
+                pooling: this.config.pooling,
+                normalize: this.config.normalize,
+            })));
+            // Extract embeddings from outputs
+            const batchEmbeddings = batchOutputs.map((output) => Array.from(output.data));
+            results.push(...batchEmbeddings);
+        }
+        // Cache dimensions on first call
+        if (this.modelDims === null && results.length > 0) {
+            this.modelDims = results[0].length;
+        }
+        return results;
+    }
+    /**
+     * Get the dimensionality of the embeddings.
+     */
+    get dims() {
+        if (this.modelDims === null) {
+            throw new VectorizerError('Embedding dimensions not yet determined. ' +
+                'Call embed() or embedMany() first to initialize the model.');
+        }
+        return this.modelDims;
+    }
+    /**
+     * Get the model name.
+     */
+    get model() {
+        return this.config.model;
+    }
+}
+//# sourceMappingURL=huggingface-vectorizer.js.map

package/dist/vectorizers/huggingface-vectorizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"huggingface-vectorizer.js","sourceRoot":"","sources":["../../src/vectorizers/huggingface-vectorizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AA6C/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,MAAM,OAAO,qBAAsB,SAAQ,cAAc;IACpC,MAAM,CAA8B;IAC7C,gBAAgB,GAAQ,IAAI,CAAC;IAC7B,SAAS,GAAkB,IAAI,CAAC;IAExC,YAAY,MAAyB;QACjC,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC9C,MAAM,IAAI,eAAe,CAAC,4BAA4B,CAAC,CAAC;QAC5D,CAAC;QAED,eAAe;QACf,IAAI,CAAC,MAAM,GAAG;YACV,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,KAAK;YAC9B,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,MAAM;YAC7B,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI;SACtC,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY;QACtB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,gBAAgB,CAAC;QACjC,CAAC;QAED,IAAI,CAAC;YACD,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;YAE/D,IAAI,CAAC,gBAAgB,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;gBAC5E,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;aAC3B,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC,gBAAgB,CAAC;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBACzE,MAAM,IAAI,eAAe,CACrB,4DAA4D;oBACxD,wDAAwD,EAC5D,EAAE,KAAK,EAAE,KAAK,EAAE,CACnB,CAAC;YACN,CAAC;YACD,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE;YAChC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;SACnC,CAAC,CAAC;QAEH,wCAAwC;QACxC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAa,CAAC;QAEtD,iCAAiC;QACjC,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC;QACtC,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,KAAe,EAAE,YAAoB,EAAE;QACnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,2CAA2C;QAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAE5C,gBAAgB;YAChB,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACf,QAAQ,CAAC,IAAI,EAAE;gBACX,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC5B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;aACnC,CAAC,CACL,CACJ,CAAC;YAEF,kCAAkC;YAClC,MAAM,eAAe,GAAG,YAAY,CAAC,GAAG,CACpC,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAa,CAClD,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;QACrC,CAAC;QAED,iCAAiC;QACjC,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACvC,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,IAAI,IAAI;QACJ,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,eAAe,CACrB,2CAA2C;gBACvC,4DAA4D,CACnE,CAAC;QACN,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAC7B,CAAC;CACJ"}