webinfer 0.0.2 → 0.0.4

package/dist/attention/paged-kv/page-table.d.ts

@@ -1,99 +0,0 @@
-/**
- * Paged KV Cache - Software page table for efficient memory management
- * Inspired by vLLM's PagedAttention
- */
-import type { WebInferDevice } from "../../core/device.ts";
-/**
- * Configuration for PagedKVCache
- */
-export interface PagedKVCacheConfig {
-    numLayers: number;
-    numHeads: number;
-    headDim: number;
-    pageSize: number;
-    maxPages: number;
-    dtype?: "f32" | "f16";
-}
-/**
- * Entry in the page table for a sequence
- */
-export interface SequenceEntry {
-    seqId: number;
-    pages: number[];
-    length: number;
-}
-/**
- * PagedKVCache - Manages KV cache with paging for efficient memory use
- *
- * Benefits:
- * 1. No memory fragmentation - pages are fixed size
- * 2. Efficient memory sharing - multiple sequences can share cache
- * 3. Dynamic allocation - only allocate pages as needed
- * 4. Easy defragmentation - just remap logical to physical pages
- */
-export declare class PagedKVCache {
-    private device;
-    private config;
-    private keyCache;
-    private valueCache;
-    private pageTable;
-    private freePages;
-    private nextSeqId;
-    constructor(device: WebInferDevice, config: PagedKVCacheConfig);
-    /**
-     * Allocate pages for a new sequence
-     */
-    allocateSequence(initialLength?: number): number;
-    /**
-     * Extend a sequence with new tokens
-     */
-    extendSequence(seqId: number, numNewTokens: number): void;
-    /**
-     * Free a sequence and its pages
-     */
-    freeSequence(seqId: number): void;
-    /**
-     * Get page indices for a sequence
-     */
-    getSequencePages(seqId: number): number[] | null;
-    /**
-     * Get sequence length
-     */
-    getSequenceLength(seqId: number): number;
-    /**
-     * Get the physical page index for a given sequence position
-     */
-    getPageForPosition(seqId: number, position: number): number | null;
-    /**
-     * Get offset within a page for a given position
-     */
-    getOffsetInPage(position: number): number;
-    private allocatePage;
-    private freePage;
-    /**
-     * Get cache statistics
-     */
-    getStats(): {
-        totalPages: number;
-        usedPages: number;
-        freePages: number;
-        numSequences: number;
-        memoryUsedBytes: number;
-        memoryTotalBytes: number;
-    };
-    /**
-     * Get GPU buffers for kernel binding
-     */
-    getBuffers(): {
-        keyCache: GPUBuffer;
-        valueCache: GPUBuffer;
-    };
-    /**
-     * Get configuration
-     */
-    getConfig(): PagedKVCacheConfig;
-    /**
-     * Dispose GPU resources
-     */
-    dispose(): void;
-}

package/dist/attention/scheduler.d.ts

@@ -1,40 +0,0 @@
-/**
- * Attention Scheduler - Prevents TDR (GPU timeout) by splitting long sequences
- */
-import type { WebInferDevice } from "../core/device.ts";
-export interface ChunkPlan {
-    numChunks: number;
-    chunkSize: number;
-    estimatedTimeMs: number;
-}
-/**
- * Attention Scheduler for TDR prevention
- * Splits long sequences into chunks to avoid GPU timeout
- */
-export declare class AttentionScheduler {
-    private device;
-    private tdrLimit;
-    constructor(device: WebInferDevice);
-    private detectTDRLimit;
-    /**
-     * Estimate execution time for attention operation
-     * Based on empirical formula: time ∝ seqLen² × numHeads × headDim
-     */
-    estimateExecutionTime(seqLen: number, numHeads: number, headDim: number): number;
-    /**
-     * Compute chunk plan for given sequence length
-     */
-    computeChunkPlan(seqLen: number, numHeads: number, headDim: number): ChunkPlan;
-    /**
-     * Yield to main thread to prevent TDR
-     */
-    yieldToMain(): Promise<void>;
-    /**
-     * Check if sequence might cause TDR
-     */
-    mightCauseTDR(seqLen: number, numHeads: number, headDim: number): boolean;
-    /**
-     * Get recommended maximum sequence length for single-pass execution
-     */
-    getMaxSinglePassSeqLen(numHeads: number, headDim: number): number;
-}

package/dist/core/buffer-pool.d.ts

@@ -1,18 +0,0 @@
-/**
- * GPU Buffer Pool - Reduces allocation overhead and memory fragmentation
- */
-export declare class BufferPool {
-    private device;
-    private pools;
-    private sizeClasses;
-    constructor(device: GPUDevice);
-    private getSizeClass;
-    acquire(size: number, usage: GPUBufferUsageFlags): GPUBuffer;
-    release(buffer: GPUBuffer): void;
-    getStats(): {
-        totalBuffers: number;
-        inUse: number;
-        totalBytes: number;
-    };
-    dispose(): void;
-}

package/dist/core/device.d.ts

@@ -1,23 +0,0 @@
-/**
- * WebGPU Device Management
- */
-export interface DeviceInfo {
-    vendor: "apple" | "nvidia" | "intel" | "amd" | "unknown";
-    architecture: string;
-    maxWorkgroupSize: number;
-    maxComputeInvocationsPerWorkgroup: number;
-    maxStorageBufferBindingSize: number;
-}
-export declare class WebInferDevice {
-    private _device;
-    private _info;
-    private constructor();
-    static create(): Promise<WebInferDevice>;
-    private static detectDeviceInfo;
-    get device(): GPUDevice;
-    get info(): DeviceInfo;
-    get limits(): GPUSupportedLimits;
-    createCommandEncoder(): GPUCommandEncoder;
-    submit(commandBuffers: GPUCommandBuffer[]): void;
-    dispose(): void;
-}

package/dist/inference/engine.d.ts

@@ -1,69 +0,0 @@
-/**
- * Inference Engine
- * Core engine for running LLM inference
- */
-import type { WebInferDevice } from "../core/device.ts";
-import type { LoadedModel } from "../model/types.ts";
-import type { ModelConfig, InferenceConfig, ForwardResult } from "./types.ts";
-/**
- * Inference Engine
- * Manages model weights and provides forward pass functionality
- */
-export declare class InferenceEngine {
-    private device;
-    private config;
-    private modelConfig;
-    private weights;
-    private loadedModel;
-    private kvCache;
-    private ropeFreqsCos;
-    private ropeFreqsSin;
-    constructor(device: WebInferDevice | null, config?: InferenceConfig);
-    /**
-     * Load model weights from a LoadedModel
-     */
-    loadModel(model: LoadedModel, modelConfig: ModelConfig): Promise<void>;
-    /**
-     * Extract model weights from loaded model
-     */
-    private extractWeights;
-    /**
-     * Initialize KV cache
-     */
-    private initKVCache;
-    /**
-     * Reset KV cache (for new sequence)
-     */
-    resetKVCache(): void;
-    /**
-     * Forward pass (CPU reference implementation)
-     * @param inputIds - Input token IDs [seqLen]
-     * @param startPos - Starting position for KV cache
-     * @returns Logits for the last token
-     */
-    forward(inputIds: Uint32Array, startPos?: number): ForwardResult;
-    /**
-     * Attention forward pass
-     */
-    private attentionForward;
-    /**
-     * Apply RoPE to a single head position
-     */
-    private applyRoPE;
-    /**
-     * FFN forward pass (SwiGLU)
-     */
-    private ffnForward;
-    /**
-     * Get model configuration
-     */
-    getModelConfig(): ModelConfig | null;
-    /**
-     * Check if model is loaded
-     */
-    isLoaded(): boolean;
-    /**
-     * Dispose resources
-     */
-    dispose(): void;
-}

package/dist/inference/generate.d.ts

@@ -1,30 +0,0 @@
-/**
- * Text Generation API
- * High-level API for generating text with LLMs
- */
-import type { InferenceEngine } from "./engine.ts";
-import type { GenerationConfig, GenerationResult, StreamToken } from "./types.ts";
-/**
- * Apply generation config to logits and sample next token
- */
-export declare function sampleNextToken(logits: Float32Array, config: GenerationConfig, generatedTokens?: number[]): number;
-/**
- * Generate tokens from a prompt
- * @param engine - Initialized inference engine with loaded model
- * @param promptTokens - Tokenized prompt
- * @param config - Generation configuration
- * @returns Generation result
- */
-export declare function generate(engine: InferenceEngine, promptTokens: number[] | Uint32Array, config?: Partial<GenerationConfig>): Promise<GenerationResult>;
-/**
- * Generate tokens with streaming (async iterator)
- * @param engine - Initialized inference engine with loaded model
- * @param promptTokens - Tokenized prompt
- * @param config - Generation configuration
- * @yields StreamToken for each generated token
- */
-export declare function generateStream(engine: InferenceEngine, promptTokens: number[] | Uint32Array, config?: Partial<GenerationConfig>): AsyncGenerator<StreamToken, void, unknown>;
-/**
- * Simple greedy decode (no sampling, fastest)
- */
-export declare function greedyDecode(engine: InferenceEngine, promptTokens: number[] | Uint32Array, maxTokens: number, eosTokenId?: number): number[];

package/dist/inference/index.d.ts

@@ -1,7 +0,0 @@
-/**
- * Inference Module
- * High-level API for LLM inference
- */
-export { type ModelConfig, type InferenceConfig, type GenerationConfig, type GenerationResult, type StreamToken, type FinishReason, type ForwardResult, type ModelWeights, type LayerWeights, DEFAULT_GENERATION_CONFIG, normalizeGenerationConfig, } from "./types.ts";
-export { InferenceEngine } from "./engine.ts";
-export { generate, generateStream, greedyDecode, sampleNextToken, } from "./generate.ts";

package/dist/inference/types.d.ts

@@ -1,161 +0,0 @@
-/**
- * Inference Types
- * Configuration and result types for model inference
- */
-import type { DType } from "../core/tensor.ts";
-/**
- * Model architecture configuration
- */
-export interface ModelConfig {
-    /** Model architecture (e.g., "llama", "mistral", "gpt2") */
-    architecture: string;
-    /** Number of transformer layers */
-    numLayers: number;
-    /** Number of attention heads */
-    numHeads: number;
-    /** Number of key-value heads (for GQA, defaults to numHeads) */
-    numKVHeads?: number;
-    /** Hidden/embedding dimension */
-    hiddenSize: number;
-    /** Intermediate size for FFN */
-    intermediateSize: number;
-    /** Vocabulary size */
-    vocabSize: number;
-    /** Maximum sequence length */
-    maxSeqLen: number;
-    /** Head dimension (defaults to hiddenSize / numHeads) */
-    headDim?: number;
-    /** RoPE frequency base */
-    ropeFreqBase?: number;
-    /** RMS norm epsilon */
-    rmsNormEps?: number;
-    /** Data type for computation */
-    dtype?: DType;
-}
-/**
- * Inference engine configuration
- */
-export interface InferenceConfig {
-    /** Maximum batch size */
-    maxBatchSize?: number;
-    /** Maximum sequence length */
-    maxSeqLen?: number;
-    /** Use KV cache for generation */
-    useKVCache?: boolean;
-    /** Memory limit in bytes (optional) */
-    memoryLimit?: number;
-    /** Enable profiling */
-    enableProfiling?: boolean;
-}
-/**
- * Generation / sampling configuration
- */
-export interface GenerationConfig {
-    /** Maximum number of tokens to generate */
-    maxTokens: number;
-    /** Temperature for sampling (0 = greedy) */
-    temperature?: number;
-    /** Top-K sampling (0 = disabled) */
-    topK?: number;
-    /** Top-P / nucleus sampling (1.0 = disabled) */
-    topP?: number;
-    /** Repetition penalty (1.0 = disabled) */
-    repetitionPenalty?: number;
-    /** Stop sequences (generation stops when any is generated) */
-    stopSequences?: number[][];
-    /** EOS token ID */
-    eosTokenId?: number;
-    /** Pad token ID */
-    padTokenId?: number;
-    /** BOS token ID */
-    bosTokenId?: number;
-    /** Stream tokens as they are generated */
-    stream?: boolean;
-    /** Random seed for reproducibility */
-    seed?: number;
-}
-/**
- * Default generation config values
- */
-export declare const DEFAULT_GENERATION_CONFIG: Required<Omit<GenerationConfig, "stopSequences" | "seed">>;
-/**
- * Reason for generation completion
- */
-export type FinishReason = "stop" | "length" | "eos";
-/**
- * Result of text generation
- */
-export interface GenerationResult {
-    /** Generated token IDs */
-    tokens: number[];
-    /** Finish reason */
-    finishReason: FinishReason;
-    /** Number of prompt tokens */
-    promptTokens: number;
-    /** Number of generated tokens */
-    generatedTokens: number;
-    /** Total time in milliseconds */
-    totalTimeMs: number;
-    /** Tokens per second */
-    tokensPerSecond: number;
-}
-/**
- * Streaming generation token
- */
-export interface StreamToken {
-    /** Token ID */
-    tokenId: number;
-    /** Token index in generation */
-    index: number;
-    /** Whether this is the final token */
-    isLast: boolean;
-    /** Finish reason (only set if isLast) */
-    finishReason?: FinishReason;
-}
-/**
- * Forward pass result
- */
-export interface ForwardResult {
-    /** Logits [batch, vocabSize] or [batch, seqLen, vocabSize] */
-    logits: Float32Array;
-    /** Shape of logits */
-    logitsShape: number[];
-}
-/**
- * Validate and normalize generation config
- */
-export declare function normalizeGenerationConfig(config: Partial<GenerationConfig>): GenerationConfig;
-/**
- * Model layer weights
- */
-export interface LayerWeights {
-    /** Attention weights */
-    attention: {
-        qProj: Float32Array;
-        kProj: Float32Array;
-        vProj: Float32Array;
-        oProj: Float32Array;
-    };
-    /** FFN weights */
-    ffn: {
-        gate?: Float32Array;
-        up: Float32Array;
-        down: Float32Array;
-    };
-    /** Normalization */
-    inputNorm: Float32Array;
-    postAttentionNorm: Float32Array;
-}
-/**
- * Full model weights
- */
-export interface ModelWeights {
-    /** Token embeddings */
-    embedTokens: Float32Array;
-    /** Layer weights */
-    layers: LayerWeights[];
-    /** Final norm */
-    finalNorm: Float32Array;
-    /** LM head (output projection) */
-    lmHead: Float32Array;
-}

package/dist/jit/compiler.d.ts

@@ -1,23 +0,0 @@
-/**
- * WGSL Compiler - Generates optimized GPU kernels
- */
-import { KernelCache } from "./kernel-cache.ts";
-import type { DeviceInfo } from "../core/device.ts";
-export interface MatMulConfig {
-    M: number;
-    N: number;
-    K: number;
-    tileM?: number;
-    tileN?: number;
-    tileK?: number;
-}
-export declare class WGSLCompiler {
-    private device;
-    private cache;
-    private deviceInfo;
-    constructor(device: GPUDevice, cache: KernelCache, deviceInfo: DeviceInfo);
-    private selectTileSize;
-    compileMatMul(config: MatMulConfig): GPUComputePipeline;
-    private generateMatMulWGSL;
-    getCacheStats(): import("./kernel-cache.ts").CacheStats;
-}

package/dist/jit/kernel-cache.d.ts

@@ -1,21 +0,0 @@
-/**
- * Kernel Cache - Caches compiled GPU compute pipelines
- */
-export interface CacheStats {
-    hits: number;
-    misses: number;
-    size: number;
-}
-export declare class KernelCache {
-    private device;
-    private cache;
-    private hits;
-    private misses;
-    constructor(device: GPUDevice);
-    getOrCreate(key: string, createFn: () => GPUComputePipeline): GPUComputePipeline;
-    has(key: string): boolean;
-    get(key: string): GPUComputePipeline | undefined;
-    set(key: string, pipeline: GPUComputePipeline): void;
-    getStats(): CacheStats;
-    clear(): void;
-}

package/dist/model/gguf.d.ts

@@ -1,90 +0,0 @@
-/**
- * GGUF Format Parser
- *
- * GGUF file structure:
- * - 4 bytes: magic "GGUF"
- * - 4 bytes: version (3)
- * - 8 bytes: n_tensors (u64)
- * - 8 bytes: n_kv (u64)
- * - Key-value metadata pairs
- * - Tensor info descriptors
- * - Padding to alignment
- * - Tensor data
- */
-import { type GGUFTensorInfo, type TensorInfo, type LoadedModel, type LoadOptions, GGUFQuantType } from "./types.ts";
-/**
- * Reader helper for GGUF binary format
- */
-declare class GGUFReader {
-    private view;
-    private offset;
-    private textDecoder;
-    constructor(buffer: ArrayBuffer);
-    get position(): number;
-    set position(pos: number);
-    readUint8(): number;
-    readInt8(): number;
-    readUint16(): number;
-    readInt16(): number;
-    readUint32(): number;
-    readInt32(): number;
-    readUint64(): bigint;
-    readInt64(): bigint;
-    readFloat32(): number;
-    readFloat64(): number;
-    readBool(): boolean;
-    readString(): string;
-    alignTo(alignment: number): void;
-}
-/**
- * GGUF header information
- */
-export interface GGUFHeader {
-    magic: number;
-    version: number;
-    nTensors: bigint;
-    nKV: bigint;
-}
-/**
- * Parse the GGUF header
- */
-export declare function parseGGUFHeader(reader: GGUFReader): GGUFHeader;
-/**
- * Parse all metadata key-value pairs
- */
-export declare function parseGGUFMetadata(reader: GGUFReader, nKV: bigint): Map<string, unknown>;
-/**
- * Parse tensor info descriptors
- */
-export declare function parseGGUFTensorInfos(reader: GGUFReader, nTensors: bigint): GGUFTensorInfo[];
-/**
- * Calculate byte size for a GGUF tensor
- */
-export declare function calculateGGUFTensorBytes(type: GGUFQuantType, shape: number[]): number;
-/**
- * Load a GGUF model from an ArrayBuffer
- */
-export declare function loadGGUF(buffer: ArrayBuffer, options?: LoadOptions): LoadedModel;
-/**
- * Load GGUF from a URL
- */
-export declare function loadGGUFFromUrl(url: string, options?: LoadOptions): Promise<LoadedModel>;
-/**
- * Dequantize Q4_0 block to float32
- * Q4_0: 32 values = 2 bytes scale (f16) + 16 bytes data (4-bit packed)
- */
-export declare function dequantizeQ4_0Block(data: Uint8Array, offset: number): Float32Array;
-/**
- * Dequantize Q8_0 block to float32
- * Q8_0: 32 values = 2 bytes scale (f16) + 32 bytes data (int8)
- */
-export declare function dequantizeQ8_0Block(data: Uint8Array, offset: number): Float32Array;
-/**
- * Load and dequantize a GGUF tensor to Float32Array
- */
-export declare function loadGGUFTensor(buffer: ArrayBuffer, info: TensorInfo): Float32Array;
-/**
- * Check if a buffer is a valid GGUF file
- */
-export declare function isGGUF(buffer: ArrayBuffer): boolean;
-export {};

package/dist/model/index.d.ts

@@ -1,16 +0,0 @@
-/**
- * Model Loading Module
- * Supports SafeTensors and GGUF formats
- */
-import type { ModelFormat, LoadOptions, LoadedModel } from "./types.ts";
-export { type ModelFormat, type SafetensorsDType, GGUFQuantType, GGUFMetadataValueType, type TensorInfo, type SafetensorsHeader, type SafetensorsHeaderEntry, type ModelMetadata, type GGUFTensorInfo, type LoadedTensor, type LoadedModel, type LoadOptions, SAFETENSORS_DTYPE_BYTES, GGUF_QUANT_BLOCK_SIZE, GGUF_QUANT_BYTES_PER_BLOCK, } from "./types.ts";
-export { parseSafetensorsHeader, getSafetensorsTensorInfos, loadSafetensorsTensor, loadSafetensors, loadSafetensorsFromUrl, isSafetensors, } from "./safetensors.ts";
-export { type GGUFHeader, parseGGUFHeader, parseGGUFMetadata, parseGGUFTensorInfos, calculateGGUFTensorBytes, loadGGUF, loadGGUFFromUrl, loadGGUFTensor, dequantizeQ4_0Block, dequantizeQ8_0Block, isGGUF, } from "./gguf.ts";
-/**
- * Auto-detect model format from buffer
- */
-export declare function detectModelFormat(buffer: ArrayBuffer): ModelFormat | null;
-/**
- * Load a model file, auto-detecting the format
- */
-export declare function loadModel(source: ArrayBuffer | string, options?: LoadOptions): Promise<LoadedModel>;

package/dist/model/safetensors.d.ts

@@ -1,38 +0,0 @@
-/**
- * SafeTensors Format Parser
- *
- * SafeTensors file structure:
- * - 8 bytes: header size (little-endian u64)
- * - N bytes: JSON header (UTF-8)
- * - Remaining: tensor data (contiguous)
- */
-import { type SafetensorsHeader, type TensorInfo, type LoadedModel, type LoadOptions } from "./types.ts";
-/**
- * Parse the SafeTensors header from a buffer
- * @param buffer - ArrayBuffer containing the SafeTensors file
- * @returns Parsed header and data offset
- */
-export declare function parseSafetensorsHeader(buffer: ArrayBuffer): {
-    header: SafetensorsHeader;
-    dataOffset: number;
-};
-/**
- * Extract tensor information from SafeTensors header
- */
-export declare function getSafetensorsTensorInfos(header: SafetensorsHeader, dataOffset: number): Map<string, TensorInfo>;
-/**
- * Load a single tensor's data from the buffer
- */
-export declare function loadSafetensorsTensor(buffer: ArrayBuffer, info: TensorInfo): Float32Array;
-/**
- * Load a SafeTensors model from an ArrayBuffer
- */
-export declare function loadSafetensors(buffer: ArrayBuffer, options?: LoadOptions): LoadedModel;
-/**
- * Load SafeTensors from a URL
- */
-export declare function loadSafetensorsFromUrl(url: string, options?: LoadOptions): Promise<LoadedModel>;
-/**
- * Check if a buffer is a valid SafeTensors file
- */
-export declare function isSafetensors(buffer: ArrayBuffer): boolean;