ruvector 0.2.23 → 0.2.25

package/dist/core/tensor-compress.js

@@ -0,0 +1,432 @@
+"use strict";
+/**
+ * TensorCompress - Adaptive tensor compression for intelligence storage
+ * Provides 10x memory savings with access-frequency based compression
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TensorCompress = void 0;
+const DEFAULT_CONFIG = {
+    hotThreshold: 0.8,
+    warmThreshold: 0.4,
+    coolThreshold: 0.1,
+    coldThreshold: 0.01,
+    autoCompress: true,
+    compressIntervalMs: 60000, // 1 minute
+};
+class TensorCompress {
+    constructor(config = {}) {
+        this.tensors = new Map();
+        this.totalAccesses = 0;
+        this.compressTimer = null;
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        if (this.config.autoCompress) {
+            this.startAutoCompress();
+        }
+    }
+    /**
+     * Store a tensor with automatic compression based on access patterns
+     */
+    store(id, tensor, level) {
+        const data = tensor instanceof Float32Array ? Array.from(tensor) : tensor;
+        const now = Date.now();
+        // Check if updating existing tensor
+        const existing = this.tensors.get(id);
+        const accessCount = existing ? existing.accessCount : 0;
+        const compressed = this.compress(data, level || 'none');
+        this.tensors.set(id, {
+            data: compressed.data,
+            level: compressed.level,
+            originalDim: data.length,
+            accessCount,
+            lastAccess: now,
+            created: existing?.created || now,
+        });
+    }
+    /**
+     * Retrieve and decompress a tensor
+     */
+    get(id) {
+        const tensor = this.tensors.get(id);
+        if (!tensor)
+            return null;
+        // Update access stats
+        tensor.accessCount++;
+        tensor.lastAccess = Date.now();
+        this.totalAccesses++;
+        // Decompress and return
+        return this.decompress(tensor);
+    }
+    /**
+     * Check if tensor exists
+     */
+    has(id) {
+        return this.tensors.has(id);
+    }
+    /**
+     * Delete a tensor
+     */
+    delete(id) {
+        return this.tensors.delete(id);
+    }
+    /**
+     * Get all tensor IDs
+     */
+    keys() {
+        return Array.from(this.tensors.keys());
+    }
+    /**
+     * Compress tensor to specified level
+     */
+    compress(data, level) {
+        switch (level) {
+            case 'none':
+                return { data: new Float32Array(data), level };
+            case 'half':
+                // Float16 simulation using Uint16Array
+                return { data: this.toFloat16(data), level };
+            case 'pq8':
+                // Product quantization to 8-bit
+                return { data: this.toPQ8(data), level };
+            case 'pq4':
+                // Product quantization to 4-bit (packed into Uint8)
+                return { data: this.toPQ4(data), level };
+            case 'binary':
+                // Binary quantization (1-bit per value)
+                return { data: this.toBinary(data), level };
+            default:
+                return { data: new Float32Array(data), level: 'none' };
+        }
+    }
+    /**
+     * Decompress tensor back to Float32Array
+     */
+    decompress(tensor) {
+        const { data, level, originalDim } = tensor;
+        switch (level) {
+            case 'none':
+                return data instanceof Float32Array ? data : new Float32Array(data);
+            case 'half':
+                return this.fromFloat16(data, originalDim);
+            case 'pq8':
+                return this.fromPQ8(data, originalDim);
+            case 'pq4':
+                return this.fromPQ4(data, originalDim);
+            case 'binary':
+                return this.fromBinary(data, originalDim);
+            default:
+                return new Float32Array(data);
+        }
+    }
+    /**
+     * Float16 conversion (approximate)
+     */
+    toFloat16(data) {
+        const result = new Uint16Array(data.length);
+        for (let i = 0; i < data.length; i++) {
+            result[i] = this.floatToHalf(data[i]);
+        }
+        return result;
+    }
+    fromFloat16(data, dim) {
+        const result = new Float32Array(dim);
+        for (let i = 0; i < dim; i++) {
+            result[i] = this.halfToFloat(data[i]);
+        }
+        return result;
+    }
+    floatToHalf(val) {
+        const floatView = new Float32Array(1);
+        const int32View = new Int32Array(floatView.buffer);
+        floatView[0] = val;
+        const x = int32View[0];
+        let bits = (x >> 16) & 0x8000;
+        let m = (x >> 12) & 0x07ff;
+        const e = (x >> 23) & 0xff;
+        if (e < 103)
+            return bits;
+        if (e > 142) {
+            bits |= 0x7c00;
+            bits |= ((e === 255) ? 0 : 1) && (x & 0x007fffff);
+            return bits;
+        }
+        if (e < 113) {
+            m |= 0x0800;
+            bits |= (m >> (114 - e)) + ((m >> (113 - e)) & 1);
+            return bits;
+        }
+        bits |= ((e - 112) << 10) | (m >> 1);
+        bits += (m & 1);
+        return bits;
+    }
+    halfToFloat(val) {
+        const s = (val & 0x8000) >> 15;
+        const e = (val & 0x7C00) >> 10;
+        const f = val & 0x03FF;
+        if (e === 0) {
+            return (s ? -1 : 1) * Math.pow(2, -14) * (f / Math.pow(2, 10));
+        }
+        else if (e === 0x1F) {
+            return f ? NaN : ((s ? -1 : 1) * Infinity);
+        }
+        return (s ? -1 : 1) * Math.pow(2, e - 15) * (1 + f / Math.pow(2, 10));
+    }
+    /**
+     * Product Quantization 8-bit
+     */
+    toPQ8(data) {
+        const result = new Uint8Array(data.length);
+        const min = Math.min(...data);
+        const max = Math.max(...data);
+        const range = max - min || 1;
+        for (let i = 0; i < data.length; i++) {
+            result[i] = Math.round(((data[i] - min) / range) * 255);
+        }
+        // Store min/max in first 8 bytes for reconstruction
+        const output = new Uint8Array(data.length + 8);
+        const view = new DataView(output.buffer);
+        view.setFloat32(0, min, true);
+        view.setFloat32(4, max, true);
+        output.set(result, 8);
+        return output;
+    }
+    fromPQ8(data, dim) {
+        const view = new DataView(data.buffer, data.byteOffset);
+        const min = view.getFloat32(0, true);
+        const max = view.getFloat32(4, true);
+        const range = max - min || 1;
+        const result = new Float32Array(dim);
+        for (let i = 0; i < dim; i++) {
+            result[i] = (data[i + 8] / 255) * range + min;
+        }
+        return result;
+    }
+    /**
+     * Product Quantization 4-bit (packed)
+     */
+    toPQ4(data) {
+        const packedLen = Math.ceil(data.length / 2);
+        const result = new Uint8Array(packedLen + 8);
+        const min = Math.min(...data);
+        const max = Math.max(...data);
+        const range = max - min || 1;
+        const view = new DataView(result.buffer);
+        view.setFloat32(0, min, true);
+        view.setFloat32(4, max, true);
+        for (let i = 0; i < data.length; i += 2) {
+            const v1 = Math.round(((data[i] - min) / range) * 15);
+            const v2 = i + 1 < data.length ? Math.round(((data[i + 1] - min) / range) * 15) : 0;
+            result[8 + i / 2] = (v1 << 4) | v2;
+        }
+        return result;
+    }
+    fromPQ4(data, dim) {
+        const view = new DataView(data.buffer, data.byteOffset);
+        const min = view.getFloat32(0, true);
+        const max = view.getFloat32(4, true);
+        const range = max - min || 1;
+        const result = new Float32Array(dim);
+        for (let i = 0; i < dim; i += 2) {
+            const packed = data[8 + i / 2];
+            result[i] = ((packed >> 4) / 15) * range + min;
+            if (i + 1 < dim) {
+                result[i + 1] = ((packed & 0x0F) / 15) * range + min;
+            }
+        }
+        return result;
+    }
+    /**
+     * Binary quantization (1-bit per value)
+     */
+    toBinary(data) {
+        const packedLen = Math.ceil(data.length / 8);
+        const result = new Uint8Array(packedLen + 4);
+        // Store mean for reconstruction
+        const mean = data.reduce((a, b) => a + b, 0) / data.length;
+        const view = new DataView(result.buffer);
+        view.setFloat32(0, mean, true);
+        for (let i = 0; i < data.length; i++) {
+            if (data[i] >= mean) {
+                const byteIdx = 4 + Math.floor(i / 8);
+                const bitIdx = i % 8;
+                result[byteIdx] |= (1 << bitIdx);
+            }
+        }
+        return result;
+    }
+    fromBinary(data, dim) {
+        const view = new DataView(data.buffer, data.byteOffset);
+        const mean = view.getFloat32(0, true);
+        // Use fixed deviation for reconstruction
+        const deviation = 0.5;
+        const result = new Float32Array(dim);
+        for (let i = 0; i < dim; i++) {
+            const byteIdx = 4 + Math.floor(i / 8);
+            const bitIdx = i % 8;
+            const bit = (data[byteIdx] >> bitIdx) & 1;
+            result[i] = bit ? mean + deviation : mean - deviation;
+        }
+        return result;
+    }
+    /**
+     * Calculate access frequency for a tensor
+     */
+    getAccessFrequency(tensor) {
+        if (this.totalAccesses === 0)
+            return 1;
+        return tensor.accessCount / this.totalAccesses;
+    }
+    /**
+     * Determine optimal compression level based on access frequency
+     */
+    getOptimalLevel(id) {
+        const tensor = this.tensors.get(id);
+        if (!tensor)
+            return 'none';
+        const freq = this.getAccessFrequency(tensor);
+        if (freq > this.config.hotThreshold)
+            return 'none';
+        if (freq > this.config.warmThreshold)
+            return 'half';
+        if (freq > this.config.coolThreshold)
+            return 'pq8';
+        if (freq > this.config.coldThreshold)
+            return 'pq4';
+        return 'binary';
+    }
+    /**
+     * Recompress all tensors based on current access patterns
+     */
+    recompressAll() {
+        const stats = {
+            totalTensors: this.tensors.size,
+            byLevel: { none: 0, half: 0, pq8: 0, pq4: 0, binary: 0 },
+            originalBytes: 0,
+            compressedBytes: 0,
+            savingsPercent: 0,
+        };
+        for (const [id, tensor] of this.tensors) {
+            const optimalLevel = this.getOptimalLevel(id);
+            if (optimalLevel !== tensor.level) {
+                // Decompress and recompress at new level
+                const decompressed = this.decompress(tensor);
+                const recompressed = this.compress(Array.from(decompressed), optimalLevel);
+                tensor.data = recompressed.data;
+                tensor.level = recompressed.level;
+            }
+            stats.byLevel[tensor.level]++;
+            stats.originalBytes += tensor.originalDim * 4; // Float32
+            stats.compressedBytes += this.getCompressedSize(tensor);
+        }
+        stats.savingsPercent = stats.originalBytes > 0
+            ? ((stats.originalBytes - stats.compressedBytes) / stats.originalBytes) * 100
+            : 0;
+        return stats;
+    }
+    /**
+     * Get compressed size in bytes
+     */
+    getCompressedSize(tensor) {
+        const { data, level, originalDim } = tensor;
+        switch (level) {
+            case 'none': return originalDim * 4;
+            case 'half': return originalDim * 2;
+            case 'pq8': return originalDim + 8;
+            case 'pq4': return Math.ceil(originalDim / 2) + 8;
+            case 'binary': return Math.ceil(originalDim / 8) + 4;
+            default: return originalDim * 4;
+        }
+    }
+    /**
+     * Get compression statistics
+     */
+    getStats() {
+        const stats = {
+            totalTensors: this.tensors.size,
+            byLevel: { none: 0, half: 0, pq8: 0, pq4: 0, binary: 0 },
+            originalBytes: 0,
+            compressedBytes: 0,
+            savingsPercent: 0,
+        };
+        for (const tensor of this.tensors.values()) {
+            stats.byLevel[tensor.level]++;
+            stats.originalBytes += tensor.originalDim * 4;
+            stats.compressedBytes += this.getCompressedSize(tensor);
+        }
+        stats.savingsPercent = stats.originalBytes > 0
+            ? ((stats.originalBytes - stats.compressedBytes) / stats.originalBytes) * 100
+            : 0;
+        return stats;
+    }
+    /**
+     * Start auto-compression timer
+     */
+    startAutoCompress() {
+        if (this.compressTimer)
+            return;
+        this.compressTimer = setInterval(() => {
+            this.recompressAll();
+        }, this.config.compressIntervalMs);
+    }
+    /**
+     * Stop auto-compression
+     */
+    stopAutoCompress() {
+        if (this.compressTimer) {
+            clearInterval(this.compressTimer);
+            this.compressTimer = null;
+        }
+    }
+    /**
+     * Export all tensors for persistence
+     */
+    export() {
+        const tensors = {};
+        for (const [id, tensor] of this.tensors) {
+            tensors[id] = {
+                data: Array.from(tensor.data),
+                level: tensor.level,
+                originalDim: tensor.originalDim,
+                accessCount: tensor.accessCount,
+                lastAccess: tensor.lastAccess,
+                created: tensor.created,
+            };
+        }
+        return { tensors, totalAccesses: this.totalAccesses };
+    }
+    /**
+     * Import tensors from persistence
+     */
+    import(data) {
+        this.totalAccesses = data.totalAccesses || 0;
+        for (const [id, tensor] of Object.entries(data.tensors)) {
+            const t = tensor;
+            this.tensors.set(id, {
+                data: this.restoreTypedArray(t.data, t.level),
+                level: t.level,
+                originalDim: t.originalDim,
+                accessCount: t.accessCount || 0,
+                lastAccess: t.lastAccess || Date.now(),
+                created: t.created || Date.now(),
+            });
+        }
+    }
+    restoreTypedArray(data, level) {
+        switch (level) {
+            case 'none': return new Float32Array(data);
+            case 'half': return new Uint16Array(data);
+            case 'pq8':
+            case 'pq4':
+            case 'binary': return new Uint8Array(data);
+            default: return new Float32Array(data);
+        }
+    }
+    /**
+     * Clear all tensors
+     */
+    clear() {
+        this.tensors.clear();
+        this.totalAccesses = 0;
+    }
+}
+exports.TensorCompress = TensorCompress;
+exports.default = TensorCompress;

package/dist/index.d.ts

@@ -0,0 +1,106 @@
+/**
+ * ruvector - High-performance vector database for Node.js
+ *
+ * This package automatically detects and uses the best available implementation:
+ * 1. Native (Rust-based, fastest) - if available for your platform
+ * 2. RVF (persistent store) - if @ruvector/rvf is installed
+ * 3. Stub (testing fallback) - limited functionality
+ *
+ * Also provides safe wrappers for GNN and Attention modules that handle
+ * array type conversions automatically.
+ */
+export * from './types';
+export * from './core';
+export * from './services';
+declare let implementation: any;
+/**
+ * Get the current implementation type
+ */
+export declare function getImplementationType(): 'native' | 'rvf' | 'wasm';
+/**
+ * Check if native implementation is being used
+ */
+export declare function isNative(): boolean;
+/**
+ * Check if RVF implementation is being used
+ */
+export declare function isRvf(): boolean;
+/**
+ * Check if stub/fallback implementation is being used
+ */
+export declare function isWasm(): boolean;
+/**
+ * Get version information
+ */
+export declare function getVersion(): {
+    version: string;
+    implementation: string;
+};
+/**
+ * Wrapper class that automatically handles metadata JSON conversion
+ */
+declare class VectorDBWrapper {
+    private db;
+    constructor(options: {
+        dimensions: number;
+        storagePath?: string;
+        distanceMetric?: string;
+        metric?: string;
+        hnswConfig?: any;
+    });
+    /**
+     * Insert a vector with optional metadata (objects are auto-converted to JSON)
+     */
+    insert(entry: {
+        id?: string;
+        vector: Float32Array | number[];
+        metadata?: Record<string, any>;
+    }): Promise<string>;
+    /**
+     * Insert multiple vectors in batch
+     */
+    insertBatch(entries: Array<{
+        id?: string;
+        vector: Float32Array | number[];
+        metadata?: Record<string, any>;
+    }>): Promise<string[]>;
+    /**
+     * Search for similar vectors (metadata is auto-parsed from JSON)
+     */
+    search(query: {
+        vector: Float32Array | number[];
+        k: number;
+        filter?: Record<string, any>;
+        efSearch?: number;
+    }): Promise<Array<{
+        id: string;
+        score: number;
+        vector?: Float32Array;
+        metadata?: Record<string, any>;
+    }>>;
+    /**
+     * Get a vector by ID (metadata is auto-parsed from JSON)
+     */
+    get(id: string): Promise<{
+        id?: string;
+        vector: Float32Array;
+        metadata?: Record<string, any>;
+    } | null>;
+    /**
+     * Delete a vector by ID
+     */
+    delete(id: string): Promise<boolean>;
+    /**
+     * Get the number of vectors in the database
+     */
+    len(): Promise<number>;
+    /**
+     * Check if the database is empty
+     */
+    isEmpty(): Promise<boolean>;
+}
+export declare const VectorDb: typeof VectorDBWrapper;
+export declare const VectorDB: typeof VectorDBWrapper;
+export declare const NativeVectorDb: any;
+export default implementation;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,cAAc,SAAS,CAAC;AAGxB,cAAc,QAAQ,CAAC;AACvB,cAAc,YAAY,CAAC;AAE3B,QAAA,IAAI,cAAc,EAAE,GAAG,CAAC;AA4DxB;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,QAAQ,GAAG,KAAK,GAAG,MAAM,CAEjE;AAED;;GAEG;AACH,wBAAgB,QAAQ,IAAI,OAAO,CAElC;AAED;;GAEG;AACH,wBAAgB,KAAK,IAAI,OAAO,CAE/B;AAED;;GAEG;AACH,wBAAgB,MAAM,IAAI,OAAO,CAEhC;AAED;;GAEG;AACH,wBAAgB,UAAU,IAAI;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,CAMxE;AA4BD;;GAEG;AACH,cAAM,eAAe;IACnB,OAAO,CAAC,EAAE,CAAM;gBAEJ,OAAO,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,GAAG,CAAA;KAAE;IAe7H;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,YAAY,GAAG,MAAM,EAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IActH;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,YAAY,GAAG,MAAM,EAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAUtI;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE;QAAE,MAAM,EAAE,YAAY,GAAG,MAAM,EAAE,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,YAAY,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAAE,CAAC,CAAC;IAuB1N;;OAEG;IACG,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,YAAY,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAAE,GAAG,IAAI,CAAC;IAW5G;;OAEG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAI1C;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC;IAI5B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;CAGlC;AAGD,eAAO,MAAM,QAAQ,wBAAkB,CAAC;AACxC,eAAO,MAAM,QAAQ,wBAAkB,CAAC;AAGxC,eAAO,MAAM,cAAc,KAA0B,CAAC;AAGtD,eAAe,cAAc,CAAC"}