webinfer 0.0.3 → 0.0.5

package/dist/core/buffer-pool.d.ts

@@ -1,18 +0,0 @@
-/**
- * GPU Buffer Pool - Reduces allocation overhead and memory fragmentation
- */
-export declare class BufferPool {
-    private device;
-    private pools;
-    private sizeClasses;
-    constructor(device: GPUDevice);
-    private getSizeClass;
-    acquire(size: number, usage: GPUBufferUsageFlags): GPUBuffer;
-    release(buffer: GPUBuffer): void;
-    getStats(): {
-        totalBuffers: number;
-        inUse: number;
-        totalBytes: number;
-    };
-    dispose(): void;
-}

package/dist/core/device.d.ts

@@ -1,23 +0,0 @@
-/**
- * WebGPU Device Management
- */
-export interface DeviceInfo {
-    vendor: "apple" | "nvidia" | "intel" | "amd" | "unknown";
-    architecture: string;
-    maxWorkgroupSize: number;
-    maxComputeInvocationsPerWorkgroup: number;
-    maxStorageBufferBindingSize: number;
-}
-export declare class WebInferDevice {
-    private _device;
-    private _info;
-    private constructor();
-    static create(): Promise<WebInferDevice>;
-    private static detectDeviceInfo;
-    get device(): GPUDevice;
-    get info(): DeviceInfo;
-    get limits(): GPUSupportedLimits;
-    createCommandEncoder(): GPUCommandEncoder;
-    submit(commandBuffers: GPUCommandBuffer[]): void;
-    dispose(): void;
-}

package/dist/core/tdr.d.ts

@@ -1,114 +0,0 @@
-/**
- * TDR (Timeout Detection and Recovery) Prevention
- * Handles browser-specific GPU timeout limits and graceful degradation
- */
-/**
- * Browser detection result
- */
-export interface BrowserInfo {
-    name: "chrome" | "safari" | "firefox" | "edge" | "unknown";
-    version: number;
-    isMobile: boolean;
-    hasWebGPU: boolean;
-}
-/**
- * TDR configuration for each browser
- */
-export interface TDRConfig {
-    timeoutMs: number;
-    safetyMargin: number;
-    maxChunkSize: number;
-    supportsTimestampQuery: boolean;
-}
-/**
- * Detect browser information
- */
-export declare function detectBrowser(): BrowserInfo;
-/**
- * Get TDR configuration for current browser
- */
-export declare function getTDRConfig(browser?: BrowserInfo): TDRConfig;
-/**
- * Graceful degradation options
- */
-export interface DegradationOptions {
-    /** Use CPU fallback when GPU fails */
-    enableCpuFallback: boolean;
-    /** Reduce precision to f16 when memory is tight */
-    enablePrecisionReduction: boolean;
-    /** Auto-chunk large sequences */
-    enableAutoChunking: boolean;
-    /** Maximum retries before falling back */
-    maxRetries: number;
-}
-/**
- * Error types for graceful handling
- */
-export type WebGPUErrorType = "device_lost" | "out_of_memory" | "validation" | "timeout" | "unknown";
-/**
- * Classify WebGPU error
- */
-export declare function classifyError(error: unknown): WebGPUErrorType;
-/**
- * TDR-safe execution wrapper
- */
-export declare class TDRGuard {
-    private browser;
-    private config;
-    private options;
-    private lastExecutionTime;
-    constructor(options?: Partial<DegradationOptions>);
-    /**
-     * Get safe execution time limit
-     */
-    getSafeTimeLimit(): number;
-    /**
-     * Check if operation might cause TDR
-     */
-    mightTimeout(estimatedMs: number): boolean;
-    /**
-     * Calculate chunks needed for safe execution
-     */
-    calcChunks(estimatedTotalMs: number): number;
-    /**
-     * Yield to browser main thread
-     */
-    yield(): Promise<void>;
-    /**
-     * Execute with TDR protection
-     */
-    execute<T>(fn: () => Promise<T>, options?: {
-        estimatedMs?: number;
-        onRetry?: (attempt: number, error: unknown) => void;
-        fallback?: () => T | Promise<T>;
-    }): Promise<T>;
-    /**
-     * Get browser info
-     */
-    getBrowserInfo(): BrowserInfo;
-    /**
-     * Get TDR config
-     */
-    getTDRConfig(): TDRConfig;
-    /**
-     * Get last execution time
-     */
-    getLastExecutionTime(): number;
-}
-/**
- * Check WebGPU support and capabilities
- */
-export declare function checkWebGPUSupport(): Promise<{
-    supported: boolean;
-    reason?: string;
-    adapter?: GPUAdapter;
-    limits?: GPUSupportedLimits;
-}>;
-/**
- * Create device with graceful degradation
- */
-export declare function createDeviceWithFallback(adapter: GPUAdapter, options?: {
-    requiredFeatures?: GPUFeatureName[];
-    requiredLimits?: Record<string, number>;
-    onFallback?: (reason: string) => void;
-}): Promise<GPUDevice>;

package/dist/inference/engine.d.ts

@@ -1,69 +0,0 @@
-/**
- * Inference Engine
- * Core engine for running LLM inference
- */
-import type { WebInferDevice } from "../core/device.ts";
-import type { LoadedModel } from "../model/types.ts";
-import type { ForwardResult, InferenceConfig, ModelConfig } from "./types.ts";
-/**
- * Inference Engine
- * Manages model weights and provides forward pass functionality
- */
-export declare class InferenceEngine {
-    private device;
-    private config;
-    private modelConfig;
-    private weights;
-    private loadedModel;
-    private kvCache;
-    private ropeFreqsCos;
-    private ropeFreqsSin;
-    constructor(device: WebInferDevice | null, config?: InferenceConfig);
-    /**
-     * Load model weights from a LoadedModel
-     */
-    loadModel(model: LoadedModel, modelConfig: ModelConfig): Promise<void>;
-    /**
-     * Extract model weights from loaded model
-     */
-    private extractWeights;
-    /**
-     * Initialize KV cache
-     */
-    private initKVCache;
-    /**
-     * Reset KV cache (for new sequence)
-     */
-    resetKVCache(): void;
-    /**
-     * Forward pass (CPU reference implementation)
-     * @param inputIds - Input token IDs [seqLen]
-     * @param startPos - Starting position for KV cache
-     * @returns Logits for the last token
-     */
-    forward(inputIds: Uint32Array, startPos?: number): ForwardResult;
-    /**
-     * Attention forward pass
-     */
-    private attentionForward;
-    /**
-     * Apply RoPE to a single head position
-     */
-    private applyRoPE;
-    /**
-     * FFN forward pass (SwiGLU)
-     */
-    private ffnForward;
-    /**
-     * Get model configuration
-     */
-    getModelConfig(): ModelConfig | null;
-    /**
-     * Check if model is loaded
-     */
-    isLoaded(): boolean;
-    /**
-     * Dispose resources
-     */
-    dispose(): void;
-}

package/dist/inference/generate.d.ts

@@ -1,30 +0,0 @@
-/**
- * Text Generation API
- * High-level API for generating text with LLMs
- */
-import type { InferenceEngine } from "./engine.ts";
-import type { GenerationConfig, GenerationResult, StreamToken } from "./types.ts";
-/**
- * Apply generation config to logits and sample next token
- */
-export declare function sampleNextToken(logits: Float32Array, config: GenerationConfig, generatedTokens?: number[]): number;
-/**
- * Generate tokens from a prompt
- * @param engine - Initialized inference engine with loaded model
- * @param promptTokens - Tokenized prompt
- * @param config - Generation configuration
- * @returns Generation result
- */
-export declare function generate(engine: InferenceEngine, promptTokens: number[] | Uint32Array, config?: Partial<GenerationConfig>): Promise<GenerationResult>;
-/**
- * Generate tokens with streaming (async iterator)
- * @param engine - Initialized inference engine with loaded model
- * @param promptTokens - Tokenized prompt
- * @param config - Generation configuration
- * @yields StreamToken for each generated token
- */
-export declare function generateStream(engine: InferenceEngine, promptTokens: number[] | Uint32Array, config?: Partial<GenerationConfig>): AsyncGenerator<StreamToken, void, unknown>;
-/**
- * Simple greedy decode (no sampling, fastest)
- */
-export declare function greedyDecode(engine: InferenceEngine, promptTokens: number[] | Uint32Array, maxTokens: number, eosTokenId?: number): number[];

package/dist/inference/index.d.ts

@@ -1,7 +0,0 @@
-/**
- * Inference Module
- * High-level API for LLM inference
- */
-export { InferenceEngine } from "./engine.ts";
-export { generate, generateStream, greedyDecode, sampleNextToken, } from "./generate.ts";
-export { DEFAULT_GENERATION_CONFIG, type FinishReason, type ForwardResult, type GenerationConfig, type GenerationResult, type InferenceConfig, type LayerWeights, type ModelConfig, type ModelWeights, normalizeGenerationConfig, type StreamToken, } from "./types.ts";

package/dist/inference/types.d.ts

@@ -1,161 +0,0 @@
-/**
- * Inference Types
- * Configuration and result types for model inference
- */
-import type { DType } from "../core/tensor.ts";
-/**
- * Model architecture configuration
- */
-export interface ModelConfig {
-    /** Model architecture (e.g., "llama", "mistral", "gpt2") */
-    architecture: string;
-    /** Number of transformer layers */
-    numLayers: number;
-    /** Number of attention heads */
-    numHeads: number;
-    /** Number of key-value heads (for GQA, defaults to numHeads) */
-    numKVHeads?: number;
-    /** Hidden/embedding dimension */
-    hiddenSize: number;
-    /** Intermediate size for FFN */
-    intermediateSize: number;
-    /** Vocabulary size */
-    vocabSize: number;
-    /** Maximum sequence length */
-    maxSeqLen: number;
-    /** Head dimension (defaults to hiddenSize / numHeads) */
-    headDim?: number;
-    /** RoPE frequency base */
-    ropeFreqBase?: number;
-    /** RMS norm epsilon */
-    rmsNormEps?: number;
-    /** Data type for computation */
-    dtype?: DType;
-}
-/**
- * Inference engine configuration
- */
-export interface InferenceConfig {
-    /** Maximum batch size */
-    maxBatchSize?: number;
-    /** Maximum sequence length */
-    maxSeqLen?: number;
-    /** Use KV cache for generation */
-    useKVCache?: boolean;
-    /** Memory limit in bytes (optional) */
-    memoryLimit?: number;
-    /** Enable profiling */
-    enableProfiling?: boolean;
-}
-/**
- * Generation / sampling configuration
- */
-export interface GenerationConfig {
-    /** Maximum number of tokens to generate */
-    maxTokens: number;
-    /** Temperature for sampling (0 = greedy) */
-    temperature?: number;
-    /** Top-K sampling (0 = disabled) */
-    topK?: number;
-    /** Top-P / nucleus sampling (1.0 = disabled) */
-    topP?: number;
-    /** Repetition penalty (1.0 = disabled) */
-    repetitionPenalty?: number;
-    /** Stop sequences (generation stops when any is generated) */
-    stopSequences?: number[][];
-    /** EOS token ID */
-    eosTokenId?: number;
-    /** Pad token ID */
-    padTokenId?: number;
-    /** BOS token ID */
-    bosTokenId?: number;
-    /** Stream tokens as they are generated */
-    stream?: boolean;
-    /** Random seed for reproducibility */
-    seed?: number;
-}
-/**
- * Default generation config values
- */
-export declare const DEFAULT_GENERATION_CONFIG: Required<Omit<GenerationConfig, "stopSequences" | "seed">>;
-/**
- * Reason for generation completion
- */
-export type FinishReason = "stop" | "length" | "eos";
-/**
- * Result of text generation
- */
-export interface GenerationResult {
-    /** Generated token IDs */
-    tokens: number[];
-    /** Finish reason */
-    finishReason: FinishReason;
-    /** Number of prompt tokens */
-    promptTokens: number;
-    /** Number of generated tokens */
-    generatedTokens: number;
-    /** Total time in milliseconds */
-    totalTimeMs: number;
-    /** Tokens per second */
-    tokensPerSecond: number;
-}
-/**
- * Streaming generation token
- */
-export interface StreamToken {
-    /** Token ID */
-    tokenId: number;
-    /** Token index in generation */
-    index: number;
-    /** Whether this is the final token */
-    isLast: boolean;
-    /** Finish reason (only set if isLast) */
-    finishReason?: FinishReason;
-}
-/**
- * Forward pass result
- */
-export interface ForwardResult {
-    /** Logits [batch, vocabSize] or [batch, seqLen, vocabSize] */
-    logits: Float32Array;
-    /** Shape of logits */
-    logitsShape: number[];
-}
-/**
- * Validate and normalize generation config
- */
-export declare function normalizeGenerationConfig(config: Partial<GenerationConfig>): GenerationConfig;
-/**
- * Model layer weights
- */
-export interface LayerWeights {
-    /** Attention weights */
-    attention: {
-        qProj: Float32Array;
-        kProj: Float32Array;
-        vProj: Float32Array;
-        oProj: Float32Array;
-    };
-    /** FFN weights */
-    ffn: {
-        gate?: Float32Array;
-        up: Float32Array;
-        down: Float32Array;
-    };
-    /** Normalization */
-    inputNorm: Float32Array;
-    postAttentionNorm: Float32Array;
-}
-/**
- * Full model weights
- */
-export interface ModelWeights {
-    /** Token embeddings */
-    embedTokens: Float32Array;
-    /** Layer weights */
-    layers: LayerWeights[];
-    /** Final norm */
-    finalNorm: Float32Array;
-    /** LM head (output projection) */
-    lmHead: Float32Array;
-}

package/dist/jit/compiler.d.ts

@@ -1,23 +0,0 @@
-/**
- * WGSL Compiler - Generates optimized GPU kernels
- */
-import type { DeviceInfo } from "../core/device.ts";
-import type { KernelCache } from "./kernel-cache.ts";
-export interface MatMulConfig {
-    M: number;
-    N: number;
-    K: number;
-    tileM?: number;
-    tileN?: number;
-    tileK?: number;
-}
-export declare class WGSLCompiler {
-    private device;
-    private cache;
-    private deviceInfo;
-    constructor(device: GPUDevice, cache: KernelCache, deviceInfo: DeviceInfo);
-    private selectTileSize;
-    compileMatMul(config: MatMulConfig): GPUComputePipeline;
-    private generateMatMulWGSL;
-    getCacheStats(): import("./kernel-cache.ts").CacheStats;
-}

package/dist/jit/kernel-cache.d.ts

@@ -1,21 +0,0 @@
-/**
- * Kernel Cache - Caches compiled GPU compute pipelines
- */
-export interface CacheStats {
-    hits: number;
-    misses: number;
-    size: number;
-}
-export declare class KernelCache {
-    private device;
-    private cache;
-    private hits;
-    private misses;
-    constructor(device: GPUDevice);
-    getOrCreate(key: string, createFn: () => GPUComputePipeline): GPUComputePipeline;
-    has(key: string): boolean;
-    get(key: string): GPUComputePipeline | undefined;
-    set(key: string, pipeline: GPUComputePipeline): void;
-    getStats(): CacheStats;
-    clear(): void;
-}

package/dist/model/gguf.d.ts

@@ -1,90 +0,0 @@
-/**
- * GGUF Format Parser
- *
- * GGUF file structure:
- * - 4 bytes: magic "GGUF"
- * - 4 bytes: version (3)
- * - 8 bytes: n_tensors (u64)
- * - 8 bytes: n_kv (u64)
- * - Key-value metadata pairs
- * - Tensor info descriptors
- * - Padding to alignment
- * - Tensor data
- */
-import { GGUFQuantType, type GGUFTensorInfo, type LoadedModel, type LoadOptions, type TensorInfo } from "./types.ts";
-/**
- * Reader helper for GGUF binary format
- */
-declare class GGUFReader {
-    private view;
-    private offset;
-    private textDecoder;
-    constructor(buffer: ArrayBuffer);
-    get position(): number;
-    set position(pos: number);
-    readUint8(): number;
-    readInt8(): number;
-    readUint16(): number;
-    readInt16(): number;
-    readUint32(): number;
-    readInt32(): number;
-    readUint64(): bigint;
-    readInt64(): bigint;
-    readFloat32(): number;
-    readFloat64(): number;
-    readBool(): boolean;
-    readString(): string;
-    alignTo(alignment: number): void;
-}
-/**
- * GGUF header information
- */
-export interface GGUFHeader {
-    magic: number;
-    version: number;
-    nTensors: bigint;
-    nKV: bigint;
-}
-/**
- * Parse the GGUF header
- */
-export declare function parseGGUFHeader(reader: GGUFReader): GGUFHeader;
-/**
- * Parse all metadata key-value pairs
- */
-export declare function parseGGUFMetadata(reader: GGUFReader, nKV: bigint): Map<string, unknown>;
-/**
- * Parse tensor info descriptors
- */
-export declare function parseGGUFTensorInfos(reader: GGUFReader, nTensors: bigint): GGUFTensorInfo[];
-/**
- * Calculate byte size for a GGUF tensor
- */
-export declare function calculateGGUFTensorBytes(type: GGUFQuantType, shape: number[]): number;
-/**
- * Load a GGUF model from an ArrayBuffer
- */
-export declare function loadGGUF(buffer: ArrayBuffer, options?: LoadOptions): LoadedModel;
-/**
- * Load GGUF from a URL
- */
-export declare function loadGGUFFromUrl(url: string, options?: LoadOptions): Promise<LoadedModel>;
-/**
- * Dequantize Q4_0 block to float32
- * Q4_0: 32 values = 2 bytes scale (f16) + 16 bytes data (4-bit packed)
- */
-export declare function dequantizeQ4_0Block(data: Uint8Array, offset: number): Float32Array;
-/**
- * Dequantize Q8_0 block to float32
- * Q8_0: 32 values = 2 bytes scale (f16) + 32 bytes data (int8)
- */
-export declare function dequantizeQ8_0Block(data: Uint8Array, offset: number): Float32Array;
-/**
- * Load and dequantize a GGUF tensor to Float32Array
- */
-export declare function loadGGUFTensor(buffer: ArrayBuffer, info: TensorInfo): Float32Array;
-/**
- * Check if a buffer is a valid GGUF file
- */
-export declare function isGGUF(buffer: ArrayBuffer): boolean;
-export {};

package/dist/model/index.d.ts

@@ -1,16 +0,0 @@
-/**
- * Model Loading Module
- * Supports SafeTensors and GGUF formats
- */
-import type { LoadedModel, LoadOptions, ModelFormat } from "./types.ts";
-export { calculateGGUFTensorBytes, dequantizeQ4_0Block, dequantizeQ8_0Block, type GGUFHeader, isGGUF, loadGGUF, loadGGUFFromUrl, loadGGUFTensor, parseGGUFHeader, parseGGUFMetadata, parseGGUFTensorInfos, } from "./gguf.ts";
-export { getSafetensorsTensorInfos, isSafetensors, loadSafetensors, loadSafetensorsFromUrl, loadSafetensorsTensor, parseSafetensorsHeader, } from "./safetensors.ts";
-export { GGUF_QUANT_BLOCK_SIZE, GGUF_QUANT_BYTES_PER_BLOCK, GGUFMetadataValueType, GGUFQuantType, type GGUFTensorInfo, type LoadedModel, type LoadedTensor, type LoadOptions, type ModelFormat, type ModelMetadata, SAFETENSORS_DTYPE_BYTES, type SafetensorsDType, type SafetensorsHeader, type SafetensorsHeaderEntry, type TensorInfo, } from "./types.ts";
-/**
- * Auto-detect model format from buffer
- */
-export declare function detectModelFormat(buffer: ArrayBuffer): ModelFormat | null;
-/**
- * Load a model file, auto-detecting the format
- */
-export declare function loadModel(source: ArrayBuffer | string, options?: LoadOptions): Promise<LoadedModel>;

package/dist/model/safetensors.d.ts

@@ -1,38 +0,0 @@
-/**
- * SafeTensors Format Parser
- *
- * SafeTensors file structure:
- * - 8 bytes: header size (little-endian u64)
- * - N bytes: JSON header (UTF-8)
- * - Remaining: tensor data (contiguous)
- */
-import { type LoadedModel, type LoadOptions, type SafetensorsHeader, type TensorInfo } from "./types.ts";
-/**
- * Parse the SafeTensors header from a buffer
- * @param buffer - ArrayBuffer containing the SafeTensors file
- * @returns Parsed header and data offset
- */
-export declare function parseSafetensorsHeader(buffer: ArrayBuffer): {
-    header: SafetensorsHeader;
-    dataOffset: number;
-};
-/**
- * Extract tensor information from SafeTensors header
- */
-export declare function getSafetensorsTensorInfos(header: SafetensorsHeader, dataOffset: number): Map<string, TensorInfo>;
-/**
- * Load a single tensor's data from the buffer
- */
-export declare function loadSafetensorsTensor(buffer: ArrayBuffer, info: TensorInfo): Float32Array;
-/**
- * Load a SafeTensors model from an ArrayBuffer
- */
-export declare function loadSafetensors(buffer: ArrayBuffer, options?: LoadOptions): LoadedModel;
-/**
- * Load SafeTensors from a URL
- */
-export declare function loadSafetensorsFromUrl(url: string, options?: LoadOptions): Promise<LoadedModel>;
-/**
- * Check if a buffer is a valid SafeTensors file
- */
-export declare function isSafetensors(buffer: ArrayBuffer): boolean;