webinfer 0.0.1

package/dist/inference/engine.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Inference Engine
+ * Core engine for running LLM inference
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import type { LoadedModel } from "../model/types.ts";
+import type { ModelConfig, InferenceConfig, ForwardResult } from "./types.ts";
+/**
+ * Inference Engine
+ * Manages model weights and provides forward pass functionality
+ */
+export declare class InferenceEngine {
+    private device;
+    private config;
+    private modelConfig;
+    private weights;
+    private loadedModel;
+    private kvCache;
+    private ropeFreqsCos;
+    private ropeFreqsSin;
+    constructor(device: WebInferDevice | null, config?: InferenceConfig);
+    /**
+     * Load model weights from a LoadedModel
+     */
+    loadModel(model: LoadedModel, modelConfig: ModelConfig): Promise<void>;
+    /**
+     * Extract model weights from loaded model
+     */
+    private extractWeights;
+    /**
+     * Initialize KV cache
+     */
+    private initKVCache;
+    /**
+     * Reset KV cache (for new sequence)
+     */
+    resetKVCache(): void;
+    /**
+     * Forward pass (CPU reference implementation)
+     * @param inputIds - Input token IDs [seqLen]
+     * @param startPos - Starting position for KV cache
+     * @returns Logits for the last token
+     */
+    forward(inputIds: Uint32Array, startPos?: number): ForwardResult;
+    /**
+     * Attention forward pass
+     */
+    private attentionForward;
+    /**
+     * Apply RoPE to a single head position
+     */
+    private applyRoPE;
+    /**
+     * FFN forward pass (SwiGLU)
+     */
+    private ffnForward;
+    /**
+     * Get model configuration
+     */
+    getModelConfig(): ModelConfig | null;
+    /**
+     * Check if model is loaded
+     */
+    isLoaded(): boolean;
+    /**
+     * Dispose resources
+     */
+    dispose(): void;
+}

package/dist/inference/generate.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Text Generation API
+ * High-level API for generating text with LLMs
+ */
+import type { InferenceEngine } from "./engine.ts";
+import type { GenerationConfig, GenerationResult, StreamToken } from "./types.ts";
+/**
+ * Apply generation config to logits and sample next token
+ */
+export declare function sampleNextToken(logits: Float32Array, config: GenerationConfig, generatedTokens?: number[]): number;
+/**
+ * Generate tokens from a prompt
+ * @param engine - Initialized inference engine with loaded model
+ * @param promptTokens - Tokenized prompt
+ * @param config - Generation configuration
+ * @returns Generation result
+ */
+export declare function generate(engine: InferenceEngine, promptTokens: number[] | Uint32Array, config?: Partial<GenerationConfig>): Promise<GenerationResult>;
+/**
+ * Generate tokens with streaming (async iterator)
+ * @param engine - Initialized inference engine with loaded model
+ * @param promptTokens - Tokenized prompt
+ * @param config - Generation configuration
+ * @yields StreamToken for each generated token
+ */
+export declare function generateStream(engine: InferenceEngine, promptTokens: number[] | Uint32Array, config?: Partial<GenerationConfig>): AsyncGenerator<StreamToken, void, unknown>;
+/**
+ * Simple greedy decode (no sampling, fastest)
+ */
+export declare function greedyDecode(engine: InferenceEngine, promptTokens: number[] | Uint32Array, maxTokens: number, eosTokenId?: number): number[];

package/dist/inference/index.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Inference Module
+ * High-level API for LLM inference
+ */
+export { type ModelConfig, type InferenceConfig, type GenerationConfig, type GenerationResult, type StreamToken, type FinishReason, type ForwardResult, type ModelWeights, type LayerWeights, DEFAULT_GENERATION_CONFIG, normalizeGenerationConfig, } from "./types.ts";
+export { InferenceEngine } from "./engine.ts";
+export { generate, generateStream, greedyDecode, sampleNextToken, } from "./generate.ts";

package/dist/inference/types.d.ts

@@ -0,0 +1,161 @@
+/**
+ * Inference Types
+ * Configuration and result types for model inference
+ */
+import type { DType } from "../core/tensor.ts";
+/**
+ * Model architecture configuration
+ */
+export interface ModelConfig {
+    /** Model architecture (e.g., "llama", "mistral", "gpt2") */
+    architecture: string;
+    /** Number of transformer layers */
+    numLayers: number;
+    /** Number of attention heads */
+    numHeads: number;
+    /** Number of key-value heads (for GQA, defaults to numHeads) */
+    numKVHeads?: number;
+    /** Hidden/embedding dimension */
+    hiddenSize: number;
+    /** Intermediate size for FFN */
+    intermediateSize: number;
+    /** Vocabulary size */
+    vocabSize: number;
+    /** Maximum sequence length */
+    maxSeqLen: number;
+    /** Head dimension (defaults to hiddenSize / numHeads) */
+    headDim?: number;
+    /** RoPE frequency base */
+    ropeFreqBase?: number;
+    /** RMS norm epsilon */
+    rmsNormEps?: number;
+    /** Data type for computation */
+    dtype?: DType;
+}
+/**
+ * Inference engine configuration
+ */
+export interface InferenceConfig {
+    /** Maximum batch size */
+    maxBatchSize?: number;
+    /** Maximum sequence length */
+    maxSeqLen?: number;
+    /** Use KV cache for generation */
+    useKVCache?: boolean;
+    /** Memory limit in bytes (optional) */
+    memoryLimit?: number;
+    /** Enable profiling */
+    enableProfiling?: boolean;
+}
+/**
+ * Generation / sampling configuration
+ */
+export interface GenerationConfig {
+    /** Maximum number of tokens to generate */
+    maxTokens: number;
+    /** Temperature for sampling (0 = greedy) */
+    temperature?: number;
+    /** Top-K sampling (0 = disabled) */
+    topK?: number;
+    /** Top-P / nucleus sampling (1.0 = disabled) */
+    topP?: number;
+    /** Repetition penalty (1.0 = disabled) */
+    repetitionPenalty?: number;
+    /** Stop sequences (generation stops when any is generated) */
+    stopSequences?: number[][];
+    /** EOS token ID */
+    eosTokenId?: number;
+    /** Pad token ID */
+    padTokenId?: number;
+    /** BOS token ID */
+    bosTokenId?: number;
+    /** Stream tokens as they are generated */
+    stream?: boolean;
+    /** Random seed for reproducibility */
+    seed?: number;
+}
+/**
+ * Default generation config values
+ */
+export declare const DEFAULT_GENERATION_CONFIG: Required<Omit<GenerationConfig, "stopSequences" | "seed">>;
+/**
+ * Reason for generation completion
+ */
+export type FinishReason = "stop" | "length" | "eos";
+/**
+ * Result of text generation
+ */
+export interface GenerationResult {
+    /** Generated token IDs */
+    tokens: number[];
+    /** Finish reason */
+    finishReason: FinishReason;
+    /** Number of prompt tokens */
+    promptTokens: number;
+    /** Number of generated tokens */
+    generatedTokens: number;
+    /** Total time in milliseconds */
+    totalTimeMs: number;
+    /** Tokens per second */
+    tokensPerSecond: number;
+}
+/**
+ * Streaming generation token
+ */
+export interface StreamToken {
+    /** Token ID */
+    tokenId: number;
+    /** Token index in generation */
+    index: number;
+    /** Whether this is the final token */
+    isLast: boolean;
+    /** Finish reason (only set if isLast) */
+    finishReason?: FinishReason;
+}
+/**
+ * Forward pass result
+ */
+export interface ForwardResult {
+    /** Logits [batch, vocabSize] or [batch, seqLen, vocabSize] */
+    logits: Float32Array;
+    /** Shape of logits */
+    logitsShape: number[];
+}
+/**
+ * Validate and normalize generation config
+ */
+export declare function normalizeGenerationConfig(config: Partial<GenerationConfig>): GenerationConfig;
+/**
+ * Model layer weights
+ */
+export interface LayerWeights {
+    /** Attention weights */
+    attention: {
+        qProj: Float32Array;
+        kProj: Float32Array;
+        vProj: Float32Array;
+        oProj: Float32Array;
+    };
+    /** FFN weights */
+    ffn: {
+        gate?: Float32Array;
+        up: Float32Array;
+        down: Float32Array;
+    };
+    /** Normalization */
+    inputNorm: Float32Array;
+    postAttentionNorm: Float32Array;
+}
+/**
+ * Full model weights
+ */
+export interface ModelWeights {
+    /** Token embeddings */
+    embedTokens: Float32Array;
+    /** Layer weights */
+    layers: LayerWeights[];
+    /** Final norm */
+    finalNorm: Float32Array;
+    /** LM head (output projection) */
+    lmHead: Float32Array;
+}

package/dist/jit/compiler.d.ts

@@ -0,0 +1,23 @@
+/**
+ * WGSL Compiler - Generates optimized GPU kernels
+ */
+import { KernelCache } from "./kernel-cache.ts";
+import type { DeviceInfo } from "../core/device.ts";
+export interface MatMulConfig {
+    M: number;
+    N: number;
+    K: number;
+    tileM?: number;
+    tileN?: number;
+    tileK?: number;
+}
+export declare class WGSLCompiler {
+    private device;
+    private cache;
+    private deviceInfo;
+    constructor(device: GPUDevice, cache: KernelCache, deviceInfo: DeviceInfo);
+    private selectTileSize;
+    compileMatMul(config: MatMulConfig): GPUComputePipeline;
+    private generateMatMulWGSL;
+    getCacheStats(): import("./kernel-cache.ts").CacheStats;
+}

package/dist/jit/kernel-cache.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Kernel Cache - Caches compiled GPU compute pipelines
+ */
+export interface CacheStats {
+    hits: number;
+    misses: number;
+    size: number;
+}
+export declare class KernelCache {
+    private device;
+    private cache;
+    private hits;
+    private misses;
+    constructor(device: GPUDevice);
+    getOrCreate(key: string, createFn: () => GPUComputePipeline): GPUComputePipeline;
+    has(key: string): boolean;
+    get(key: string): GPUComputePipeline | undefined;
+    set(key: string, pipeline: GPUComputePipeline): void;
+    getStats(): CacheStats;
+    clear(): void;
+}

package/dist/model/gguf.d.ts

@@ -0,0 +1,90 @@
+/**
+ * GGUF Format Parser
+ *
+ * GGUF file structure:
+ * - 4 bytes: magic "GGUF"
+ * - 4 bytes: version (3)
+ * - 8 bytes: n_tensors (u64)
+ * - 8 bytes: n_kv (u64)
+ * - Key-value metadata pairs
+ * - Tensor info descriptors
+ * - Padding to alignment
+ * - Tensor data
+ */
+import { type GGUFTensorInfo, type TensorInfo, type LoadedModel, type LoadOptions, GGUFQuantType } from "./types.ts";
+/**
+ * Reader helper for GGUF binary format
+ */
+declare class GGUFReader {
+    private view;
+    private offset;
+    private textDecoder;
+    constructor(buffer: ArrayBuffer);
+    get position(): number;
+    set position(pos: number);
+    readUint8(): number;
+    readInt8(): number;
+    readUint16(): number;
+    readInt16(): number;
+    readUint32(): number;
+    readInt32(): number;
+    readUint64(): bigint;
+    readInt64(): bigint;
+    readFloat32(): number;
+    readFloat64(): number;
+    readBool(): boolean;
+    readString(): string;
+    alignTo(alignment: number): void;
+}
+/**
+ * GGUF header information
+ */
+export interface GGUFHeader {
+    magic: number;
+    version: number;
+    nTensors: bigint;
+    nKV: bigint;
+}
+/**
+ * Parse the GGUF header
+ */
+export declare function parseGGUFHeader(reader: GGUFReader): GGUFHeader;
+/**
+ * Parse all metadata key-value pairs
+ */
+export declare function parseGGUFMetadata(reader: GGUFReader, nKV: bigint): Map<string, unknown>;
+/**
+ * Parse tensor info descriptors
+ */
+export declare function parseGGUFTensorInfos(reader: GGUFReader, nTensors: bigint): GGUFTensorInfo[];
+/**
+ * Calculate byte size for a GGUF tensor
+ */
+export declare function calculateGGUFTensorBytes(type: GGUFQuantType, shape: number[]): number;
+/**
+ * Load a GGUF model from an ArrayBuffer
+ */
+export declare function loadGGUF(buffer: ArrayBuffer, options?: LoadOptions): LoadedModel;
+/**
+ * Load GGUF from a URL
+ */
+export declare function loadGGUFFromUrl(url: string, options?: LoadOptions): Promise<LoadedModel>;
+/**
+ * Dequantize Q4_0 block to float32
+ * Q4_0: 32 values = 2 bytes scale (f16) + 16 bytes data (4-bit packed)
+ */
+export declare function dequantizeQ4_0Block(data: Uint8Array, offset: number): Float32Array;
+/**
+ * Dequantize Q8_0 block to float32
+ * Q8_0: 32 values = 2 bytes scale (f16) + 32 bytes data (int8)
+ */
+export declare function dequantizeQ8_0Block(data: Uint8Array, offset: number): Float32Array;
+/**
+ * Load and dequantize a GGUF tensor to Float32Array
+ */
+export declare function loadGGUFTensor(buffer: ArrayBuffer, info: TensorInfo): Float32Array;
+/**
+ * Check if a buffer is a valid GGUF file
+ */
+export declare function isGGUF(buffer: ArrayBuffer): boolean;
+export {};

package/dist/model/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Model Loading Module
+ * Supports SafeTensors and GGUF formats
+ */
+import type { ModelFormat, LoadOptions, LoadedModel } from "./types.ts";
+export { type ModelFormat, type SafetensorsDType, GGUFQuantType, GGUFMetadataValueType, type TensorInfo, type SafetensorsHeader, type SafetensorsHeaderEntry, type ModelMetadata, type GGUFTensorInfo, type LoadedTensor, type LoadedModel, type LoadOptions, SAFETENSORS_DTYPE_BYTES, GGUF_QUANT_BLOCK_SIZE, GGUF_QUANT_BYTES_PER_BLOCK, } from "./types.ts";
+export { parseSafetensorsHeader, getSafetensorsTensorInfos, loadSafetensorsTensor, loadSafetensors, loadSafetensorsFromUrl, isSafetensors, } from "./safetensors.ts";
+export { type GGUFHeader, parseGGUFHeader, parseGGUFMetadata, parseGGUFTensorInfos, calculateGGUFTensorBytes, loadGGUF, loadGGUFFromUrl, loadGGUFTensor, dequantizeQ4_0Block, dequantizeQ8_0Block, isGGUF, } from "./gguf.ts";
+/**
+ * Auto-detect model format from buffer
+ */
+export declare function detectModelFormat(buffer: ArrayBuffer): ModelFormat | null;
+/**
+ * Load a model file, auto-detecting the format
+ */
+export declare function loadModel(source: ArrayBuffer | string, options?: LoadOptions): Promise<LoadedModel>;

package/dist/model/safetensors.d.ts

@@ -0,0 +1,38 @@
+/**
+ * SafeTensors Format Parser
+ *
+ * SafeTensors file structure:
+ * - 8 bytes: header size (little-endian u64)
+ * - N bytes: JSON header (UTF-8)
+ * - Remaining: tensor data (contiguous)
+ */
+import { type SafetensorsHeader, type TensorInfo, type LoadedModel, type LoadOptions } from "./types.ts";
+/**
+ * Parse the SafeTensors header from a buffer
+ * @param buffer - ArrayBuffer containing the SafeTensors file
+ * @returns Parsed header and data offset
+ */
+export declare function parseSafetensorsHeader(buffer: ArrayBuffer): {
+    header: SafetensorsHeader;
+    dataOffset: number;
+};
+/**
+ * Extract tensor information from SafeTensors header
+ */
+export declare function getSafetensorsTensorInfos(header: SafetensorsHeader, dataOffset: number): Map<string, TensorInfo>;
+/**
+ * Load a single tensor's data from the buffer
+ */
+export declare function loadSafetensorsTensor(buffer: ArrayBuffer, info: TensorInfo): Float32Array;
+/**
+ * Load a SafeTensors model from an ArrayBuffer
+ */
+export declare function loadSafetensors(buffer: ArrayBuffer, options?: LoadOptions): LoadedModel;
+/**
+ * Load SafeTensors from a URL
+ */
+export declare function loadSafetensorsFromUrl(url: string, options?: LoadOptions): Promise<LoadedModel>;
+/**
+ * Check if a buffer is a valid SafeTensors file
+ */
+export declare function isSafetensors(buffer: ArrayBuffer): boolean;

package/dist/model/types.d.ts

@@ -0,0 +1,182 @@
+/**
+ * Model Loading Types
+ * Common types for SafeTensors and GGUF model loading
+ */
+/**
+ * Supported model formats
+ */
+export type ModelFormat = "safetensors" | "gguf";
+/**
+ * SafeTensors data types
+ */
+export type SafetensorsDType = "F64" | "F32" | "F16" | "BF16" | "I64" | "I32" | "I16" | "I8" | "U8" | "BOOL";
+/**
+ * GGUF quantization types
+ * Based on llama.cpp GGML types
+ */
+export declare enum GGUFQuantType {
+    F32 = 0,
+    F16 = 1,
+    Q4_0 = 2,
+    Q4_1 = 3,
+    Q5_0 = 6,
+    Q5_1 = 7,
+    Q8_0 = 8,
+    Q8_1 = 9,
+    Q2_K = 10,
+    Q3_K = 11,
+    Q4_K = 12,
+    Q5_K = 13,
+    Q6_K = 14,
+    Q8_K = 15,
+    IQ2_XXS = 16,
+    IQ2_XS = 17,
+    IQ3_XXS = 18,
+    IQ1_S = 19,
+    IQ4_NL = 20,
+    IQ3_S = 21,
+    IQ2_S = 22,
+    IQ4_XS = 23,
+    I8 = 24,
+    I16 = 25,
+    I32 = 26,
+    I64 = 27,
+    F64 = 28,
+    BF16 = 29
+}
+/**
+ * GGUF metadata value types
+ */
+export declare enum GGUFMetadataValueType {
+    UINT8 = 0,
+    INT8 = 1,
+    UINT16 = 2,
+    INT16 = 3,
+    UINT32 = 4,
+    INT32 = 5,
+    FLOAT32 = 6,
+    BOOL = 7,
+    STRING = 8,
+    ARRAY = 9,
+    UINT64 = 10,
+    INT64 = 11,
+    FLOAT64 = 12
+}
+/**
+ * Information about a single tensor in a model file
+ */
+export interface TensorInfo {
+    /** Tensor name (e.g., "model.layers.0.attention.wq.weight") */
+    name: string;
+    /** Tensor shape (e.g., [4096, 4096]) */
+    shape: number[];
+    /** Data type or quantization type */
+    dtype: SafetensorsDType | GGUFQuantType;
+    /** Byte offset in the file's data section */
+    offset: number;
+    /** Total bytes for this tensor's data */
+    byteSize: number;
+}
+/**
+ * SafeTensors header entry for a single tensor
+ */
+export interface SafetensorsHeaderEntry {
+    dtype: SafetensorsDType;
+    shape: number[];
+    data_offsets: [number, number];
+}
+/**
+ * Parsed SafeTensors header
+ */
+export interface SafetensorsHeader {
+    tensors: Record<string, SafetensorsHeaderEntry>;
+    __metadata__?: Record<string, string>;
+}
+/**
+ * Model metadata extracted from file headers
+ */
+export interface ModelMetadata {
+    /** Model format */
+    format: ModelFormat;
+    /** Model name (if available) */
+    name?: string;
+    /** Model architecture (e.g., "llama", "mistral", "gpt2") */
+    architecture?: string;
+    /** Context length */
+    contextLength?: number;
+    /** Embedding dimension */
+    embeddingLength?: number;
+    /** Number of layers */
+    numLayers?: number;
+    /** Number of attention heads */
+    numHeads?: number;
+    /** Number of KV heads (for GQA) */
+    numKVHeads?: number;
+    /** Vocabulary size */
+    vocabSize?: number;
+    /** Head dimension */
+    headDim?: number;
+    /** Hidden size for FFN */
+    hiddenSize?: number;
+    /** RoPE frequency base */
+    ropeFreqBase?: number;
+    /** Additional metadata */
+    extra?: Record<string, unknown>;
+}
+/**
+ * GGUF tensor descriptor (from file)
+ */
+export interface GGUFTensorInfo {
+    name: string;
+    nDims: number;
+    dimensions: bigint[];
+    type: GGUFQuantType;
+    offset: bigint;
+}
+/**
+ * Loaded tensor data
+ */
+export interface LoadedTensor {
+    info: TensorInfo;
+    /** Raw data (Float32Array for F32, Uint8Array for quantized) */
+    data: Float32Array | Float16Array | Uint8Array | Int8Array;
+}
+/**
+ * Fully loaded model
+ */
+export interface LoadedModel {
+    /** Model metadata */
+    metadata: ModelMetadata;
+    /** Map of tensor names to tensor info */
+    tensorInfos: Map<string, TensorInfo>;
+    /** Total size of tensor data in bytes */
+    totalBytes: number;
+    /** The raw buffer (for lazy loading) */
+    buffer: ArrayBuffer;
+    /** Offset where tensor data starts */
+    dataOffset: number;
+}
+/**
+ * Model loading options
+ */
+export interface LoadOptions {
+    /** Only load metadata, not tensor data */
+    metadataOnly?: boolean;
+    /** Filter which tensors to load by name pattern */
+    tensorFilter?: (name: string) => boolean;
+    /** Progress callback */
+    onProgress?: (loaded: number, total: number) => void;
+}
+/**
+ * Bytes per element for SafeTensors dtypes
+ */
+export declare const SAFETENSORS_DTYPE_BYTES: Record<SafetensorsDType, number>;
+/**
+ * Block size for GGUF quantization types
+ * Most quantization types process data in blocks
+ */
+export declare const GGUF_QUANT_BLOCK_SIZE: Partial<Record<GGUFQuantType, number>>;
+/**
+ * Bytes per block for GGUF quantization types
+ */
+export declare const GGUF_QUANT_BYTES_PER_BLOCK: Partial<Record<GGUFQuantType, number>>;

package/dist/ops/activations.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Activation Functions
+ * GeLU, SiLU (Swish), ReLU for transformer models
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import { Tensor } from "../core/tensor.ts";
+/**
+ * GeLU (Gaussian Error Linear Unit) - CPU
+ * Used in BERT, GPT-2, etc.
+ * Approximation: x * 0.5 * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3)))
+ */
+export declare function geluCPU(x: Float32Array): Float32Array;
+/**
+ * GeLU exact (using erf) - CPU
+ * More accurate but slower
+ */
+export declare function geluExactCPU(x: Float32Array): Float32Array;
+/**
+ * SiLU (Sigmoid Linear Unit / Swish) - CPU
+ * Used in Llama, Mistral, etc.
+ * Formula: x * sigmoid(x) = x / (1 + exp(-x))
+ */
+export declare function siluCPU(x: Float32Array): Float32Array;
+/**
+ * ReLU (Rectified Linear Unit) - CPU
+ */
+export declare function reluCPU(x: Float32Array): Float32Array;
+/**
+ * Sigmoid - CPU
+ */
+export declare function sigmoidCPU(x: Float32Array): Float32Array;
+/**
+ * GeLU - GPU
+ */
+export declare function gelu(device: WebInferDevice, x: Tensor): Promise<Tensor>;
+/**
+ * SiLU - GPU
+ */
+export declare function silu(device: WebInferDevice, x: Tensor): Promise<Tensor>;
+/**
+ * ReLU - GPU
+ */
+export declare function relu(device: WebInferDevice, x: Tensor): Promise<Tensor>;