webinfer 0.0.3 → 0.0.5

package/dist/attention/block-sparse/format.d.ts

@@ -1,52 +0,0 @@
-/**
- * Block-Sparse CSR Format
- * Enables a single kernel to support all attention variants
- */
-/**
- * Block-Sparse CSR representation of attention mask
- */
-export interface BlockSparseCSR {
-    blockSize: number;
-    rowPtr: Uint32Array;
-    colIdx: Uint32Array;
-    blockMask?: Uint8Array;
-    numRows: number;
-    numCols: number;
-    numBlockRows: number;
-    numBlockCols: number;
-    nnzBlocks: number;
-}
-/**
- * Attention pattern types
- */
-export type AttentionPattern = {
-    type: "dense";
-} | {
-    type: "causal";
-} | {
-    type: "sliding";
-    windowSize: number;
-} | {
-    type: "global-local";
-    globalTokens: number[];
-    localWindow: number;
-} | {
-    type: "custom";
-    mask: boolean[][];
-};
-/**
- * Build BS-CSR from attention pattern
- */
-export declare function buildBlockSparseCSR(seqLen: number, pattern: AttentionPattern, blockSize?: number): BlockSparseCSR;
-/**
- * Calculate sparsity ratio of the mask
- */
-export declare function getSparsityRatio(csr: BlockSparseCSR): number;
-/**
- * Estimate memory savings from sparsity
- */
-export declare function estimateMemorySavings(csr: BlockSparseCSR): {
-    denseBytes: number;
-    sparseBytes: number;
-    savingsRatio: number;
-};

package/dist/attention/block-sparse/patterns/causal.d.ts

@@ -1,16 +0,0 @@
-/**
- * Causal (autoregressive) attention pattern
- * Used in GPT, Llama, and most decoder-only models
- */
-import { type BlockSparseCSR } from "../format.ts";
-/**
- * Build causal attention mask in BS-CSR format
- * Each query position can only attend to positions <= its own position
- */
-export declare function buildCausalMask(seqLen: number, blockSize?: number): BlockSparseCSR;
-/**
- * Get the theoretical sparsity of causal attention
- * For a sequence of length N, causal attention has N*(N+1)/2 non-zero elements
- * out of N*N total, giving ~50% sparsity for large N
- */
-export declare function getCausalSparsity(seqLen: number): number;

package/dist/attention/block-sparse/patterns/sliding.d.ts

@@ -1,22 +0,0 @@
-/**
- * Sliding window attention pattern
- * Used in Mistral and other efficient attention models
- */
-import { type BlockSparseCSR } from "../format.ts";
-/**
- * Build sliding window attention mask in BS-CSR format
- * Each query can only attend to the previous `windowSize` positions
- */
-export declare function buildSlidingWindowMask(seqLen: number, windowSize: number, blockSize?: number): BlockSparseCSR;
-/**
- * Get the theoretical sparsity of sliding window attention
- * For window size W and sequence length N:
- * - First W positions have triangular attention (like causal)
- * - Remaining N-W positions have W+1 attention each
- */
-export declare function getSlidingWindowSparsity(seqLen: number, windowSize: number): number;
-/**
- * Sliding window with causal constraint
- * This is what Mistral uses - combines sliding window with causal masking
- */
-export declare function buildCausalSlidingWindowMask(seqLen: number, windowSize: number, blockSize?: number): BlockSparseCSR;

package/dist/attention/block-sparse/patterns/tree.d.ts

@@ -1,65 +0,0 @@
-/**
- * Tree Attention Pattern
- * Used in speculative decoding (Medusa, EAGLE) and tree-based generation
- */
-import { type BlockSparseCSR } from "../format.ts";
-/**
- * Tree node structure
- */
-export interface TreeNode {
-    /** Token position in sequence */
-    position: number;
-    /** Parent position (-1 for root) */
-    parent: number;
-    /** Depth in tree (0 for root) */
-    depth: number;
-}
-/**
- * Tree attention configuration
- */
-export interface TreeAttentionConfig {
-    /** Total sequence length including prompt */
-    seqLen: number;
-    /** Prompt length (prefix that all tokens attend to) */
-    promptLen: number;
-    /** Tree structure for speculative tokens */
-    tree: TreeNode[];
-    /** Block size for sparse format */
-    blockSize?: number;
-}
-/**
- * Build tree attention mask
- *
- * In tree attention:
- * - All tokens attend to the prompt (positions 0 to promptLen-1)
- * - Tree tokens attend to their ancestors in the tree
- * - Maintains causal property within the tree structure
- */
-export declare function buildTreeMask(config: TreeAttentionConfig): BlockSparseCSR;
-/**
- * Build a simple chain tree (linear speculation)
- * Each token depends on the previous one
- */
-export declare function buildChainTree(numSpecTokens: number): TreeNode[];
-/**
- * Build a wide tree (parallel speculation)
- * All speculative tokens depend only on the prompt
- */
-export declare function buildWideTree(numSpecTokens: number): TreeNode[];
-/**
- * Build a binary tree for speculation
- */
-export declare function buildBinaryTree(depth: number): TreeNode[];
-/**
- * Build Medusa-style tree
- * Multiple heads predict tokens at different positions
- */
-export declare function buildMedusaTree(numHeads: number, tokensPerHead: number): TreeNode[];
-/**
- * Calculate tree sparsity ratio
- */
-export declare function getTreeSparsity(config: TreeAttentionConfig): number;
-/**
- * Validate tree structure
- */
-export declare function validateTree(tree: TreeNode[]): boolean;

package/dist/attention/cascaded-inference.d.ts

@@ -1,29 +0,0 @@
-/**
- * Cascaded Inference - TDR-safe attention for very long sequences
- * Splits attention computation across multiple passes with browser yields
- */
-import type { WebInferDevice } from "../core/device.ts";
-import { Tensor } from "../core/tensor.ts";
-export interface CascadedAttentionConfig {
-    numHeads: number;
-    headDim: number;
-    seqLen: number;
-    scale?: number;
-    causal?: boolean;
-}
-/**
- * Cascaded Attention - Safe for very long sequences
- * Uses Split-K strategy to prevent TDR (GPU timeout)
- *
- * @param device WebInfer device
- * @param q Query tensor [seqLen, numHeads, headDim]
- * @param k Key tensor [seqLen, numHeads, headDim]
- * @param v Value tensor [seqLen, numHeads, headDim]
- * @param config Attention configuration
- * @param onProgress Optional progress callback
- */
-export declare function cascadedAttention(device: WebInferDevice, q: Tensor, k: Tensor, v: Tensor, config: CascadedAttentionConfig, onProgress?: (chunk: number, total: number) => void): Promise<Tensor>;
-/**
- * CPU reference implementation for verification
- */
-export declare function cascadedAttentionCPU(q: Float32Array, k: Float32Array, v: Float32Array, seqLen: number, numHeads: number, headDim: number, causal?: boolean): Float32Array;

package/dist/attention/flash-attention.d.ts

@@ -1,30 +0,0 @@
-/**
- * FlashAttention Implementation for WebGPU
- * Memory-efficient attention using online softmax and tiling
- */
-import type { WebInferDevice } from "../core/device.ts";
-import { Tensor } from "../core/tensor.ts";
-import type { AttentionPattern } from "./block-sparse/format.ts";
-export interface AttentionConfig {
-    numHeads: number;
-    headDim: number;
-    seqLen: number;
-    scale?: number;
-    pattern?: AttentionPattern;
-    blockSize?: number;
-}
-/**
- * FlashAttention forward pass
- * Computes: softmax(Q @ K^T / sqrt(d)) @ V
- *
- * @param device WebInfer device
- * @param q Query tensor [batch, seqLen, numHeads, headDim]
- * @param k Key tensor [batch, seqLen, numHeads, headDim]
- * @param v Value tensor [batch, seqLen, numHeads, headDim]
- * @param config Attention configuration
- */
-export declare function flashAttention(device: WebInferDevice, q: Tensor, k: Tensor, v: Tensor, config: AttentionConfig): Promise<Tensor>;
-/**
- * CPU reference implementation for verification
- */
-export declare function attentionCPU(q: Float32Array, k: Float32Array, v: Float32Array, seqLen: number, numHeads: number, headDim: number, causal?: boolean): Float32Array;

package/dist/attention/index.d.ts

@@ -1,118 +0,0 @@
-/**
- * Attention Module Exports
- */
-import type { WebInferDevice } from "../core/device.ts";
-import { Tensor } from "../core/tensor.ts";
-import type { BlockSparseCSR } from "./block-sparse/format.ts";
-import { PagedKVCache } from "./paged-kv/page-table.ts";
-/**
- * Tensor-like input types
- */
-export type TensorLike = Tensor | Float32Array | {
-    data: Float32Array;
-    shape: number[];
-};
-/**
- * Attention options
- */
-export interface AttentionOptions {
-    q: TensorLike;
-    k: TensorLike;
-    v: TensorLike;
-    causal?: boolean;
-    scale?: number;
-    window?: number;
-    mask?: BlockSparseCSR;
-    returnLse?: boolean;
-}
-/**
- * Attention result with optional LSE
- */
-export interface AttentionResultWithLse {
-    output: Tensor;
-    lse: Tensor;
-}
-/**
- * Simple attention function
- */
-export declare function attention(device: WebInferDevice, options: AttentionOptions): Promise<Tensor>;
-export declare function attention(device: WebInferDevice, options: AttentionOptions & {
-    returnLse: true;
-}): Promise<AttentionResultWithLse>;
-/**
- * BatchAttention configuration
- */
-export interface BatchAttentionConfig {
-    numHeads: number;
-    headDim: number;
-    numKvHeads?: number;
-    maxBatchSize?: number;
-    maxSeqLen?: number;
-}
-/**
- * Prefill input
- */
-export interface PrefillInput {
-    queries: TensorLike[];
-    keys: TensorLike[];
-    values: TensorLike[];
-    causal?: boolean;
-    window?: number;
-}
-/**
- * Decode input
- */
-export interface DecodeInput {
-    query: TensorLike;
-    kvCache: PagedKVCache;
-    seqIds: number[];
-}
-/**
- * BatchAttention - Batched attention for prefill and decode
- */
-export declare class BatchAttention {
-    private device;
-    private config;
-    constructor(device: WebInferDevice, config: BatchAttentionConfig);
-    /**
-     * Prefill: Process variable-length sequences
-     */
-    prefill(input: PrefillInput): Promise<Tensor[]>;
-    /**
-     * Decode: Single token per sequence with KV cache
-     */
-    decode(input: DecodeInput): Promise<Tensor>;
-    getConfig(): Required<BatchAttentionConfig>;
-    dispose(): void;
-}
-/**
- * AttentionKernel configuration
- */
-export interface AttentionKernelConfig {
-    numHeads: number;
-    headDim: number;
-    causal?: boolean;
-    blockSize?: number;
-}
-/**
- * AttentionKernel - Low-level compiled kernel
- */
-export declare class AttentionKernel {
-    private device;
-    private config;
-    private constructor();
-    static compile(device: WebInferDevice, config: AttentionKernelConfig): Promise<AttentionKernel>;
-    execute(input: {
-        q: Tensor;
-        k: Tensor;
-        v: Tensor;
-    }): Promise<Tensor>;
-    dispose(): void;
-}
-export { flashAttention, type AttentionConfig } from "./flash-attention.ts";
-export { buildBlockSparseCSR, getSparsityRatio, estimateMemorySavings, type BlockSparseCSR, type AttentionPattern, } from "./block-sparse/format.ts";
-export { buildCausalMask, getCausalSparsity } from "./block-sparse/patterns/causal.ts";
-export { buildSlidingWindowMask, buildCausalSlidingWindowMask, getSlidingWindowSparsity, } from "./block-sparse/patterns/sliding.ts";
-export { PagedKVCache, type PagedKVCacheConfig, type SequenceEntry, type DefragmentResult, BlockManager, ContinuousBatchScheduler, type BlockManagerConfig, type AllocationPolicy, type AllocationRequest, } from "./paged-kv/index.ts";
-export { pagedAttention, appendToPagedCache, type PagedAttentionConfig, type PagedAttentionInput, } from "./paged-attention.ts";
-export { cascadedAttention, type CascadedAttentionConfig, } from "./cascaded-inference.ts";

package/dist/attention/paged-attention.d.ts

@@ -1,40 +0,0 @@
-/**
- * Paged Attention Implementation for WebGPU
- * Efficient attention computation using paged KV cache (vLLM-style)
- */
-import type { WebInferDevice } from "../core/device.ts";
-import { Tensor } from "../core/tensor.ts";
-import type { PagedKVCache } from "./paged-kv/page-table.ts";
-export interface PagedAttentionConfig {
-    numHeads: number;
-    headDim: number;
-    scale?: number;
-}
-export interface PagedAttentionInput {
-    query: Tensor;
-    kvCache: PagedKVCache;
-    seqIds: number[];
-    positions: number[];
-}
-/**
- * Paged Attention forward pass for decoding
- * Computes attention against paged KV cache for single-token queries
- *
- * @param device WebInfer device
- * @param input Paged attention input (query, kv cache, sequence info)
- * @param config Attention configuration
- */
-export declare function pagedAttention(device: WebInferDevice, input: PagedAttentionInput, config: PagedAttentionConfig): Promise<Tensor>;
-/**
- * Append new KV to paged cache
- */
-export declare function appendToPagedCache(device: WebInferDevice, kvCache: PagedKVCache, seqId: number, key: Tensor, // [numHeads, headDim]
-value: Tensor): Promise<void>;
-/**
- * CPU reference implementation for verification
- */
-export declare function pagedAttentionCPU(q: Float32Array, // [batchSize, numHeads, headDim]
-keyCache: Float32Array, // [maxPages, pageSize, numHeads, headDim]
-valueCache: Float32Array, pageTable: number[][], // [batchSize][pages]
-seqLens: number[], // [batchSize]
-numHeads: number, headDim: number, pageSize: number, maxPages: number): Float32Array;

package/dist/attention/paged-kv/block-manager.d.ts

@@ -1,102 +0,0 @@
-/**
- * Block Manager - High-level memory management for PagedKVCache
- */
-import type { WebInferDevice } from "../../core/device.ts";
-import { PagedKVCache, type PagedKVCacheConfig } from "./page-table.ts";
-/**
- * Block allocation policy
- */
-export type AllocationPolicy = "greedy" | "best-fit" | "first-fit";
-/**
- * Block Manager configuration
- */
-export interface BlockManagerConfig extends PagedKVCacheConfig {
-    policy?: AllocationPolicy;
-    reservedPages?: number;
-}
-/**
- * Request for KV cache allocation
- */
-export interface AllocationRequest {
-    seqId?: number;
-    numTokens: number;
-    priority?: number;
-}
-/**
- * Block Manager - Manages KV cache allocation across multiple sequences
- */
-export declare class BlockManager {
-    private cache;
-    private config;
-    private priorities;
-    constructor(device: WebInferDevice, config: BlockManagerConfig);
-    /**
-     * Check if allocation is possible
-     */
-    canAllocate(request: AllocationRequest): boolean;
-    /**
-     * Allocate or extend a sequence
-     */
-    allocate(request: AllocationRequest): number;
-    /**
-     * Free a sequence
-     */
-    free(seqId: number): void;
-    /**
-     * Evict lowest priority sequences until we have enough free pages
-     */
-    evict(neededPages: number): number[];
-    /**
-     * Get memory utilization
-     */
-    getUtilization(): number;
-    /**
-     * Get the underlying cache
-     */
-    getCache(): PagedKVCache;
-    /**
-     * Get statistics
-     */
-    getStats(): {
-        totalPages: number;
-        usedPages: number;
-        freePages: number;
-        numSequences: number;
-        memoryUsedBytes: number;
-        memoryTotalBytes: number;
-    };
-    /**
-     * Dispose resources
-     */
-    dispose(): void;
-}
-/**
- * Scheduler for continuous batching with PagedAttention
- */
-export declare class ContinuousBatchScheduler {
-    private blockManager;
-    private runningSequences;
-    private waitingQueue;
-    constructor(blockManager: BlockManager);
-    /**
-     * Add a new request to the scheduler
-     */
-    addRequest(request: AllocationRequest): void;
-    /**
-     * Complete a sequence
-     */
-    completeSequence(seqId: number): void;
-    /**
-     * Extend a running sequence
-     */
-    extendSequence(seqId: number, numNewTokens: number): boolean;
-    private scheduleWaiting;
-    /**
-     * Get running sequence count
-     */
-    getRunningCount(): number;
-    /**
-     * Get waiting request count
-     */
-    getWaitingCount(): number;
-}

package/dist/attention/paged-kv/index.d.ts

@@ -1,5 +0,0 @@
-/**
- * PagedKV Module Exports
- */
-export { type AllocationPolicy, type AllocationRequest, BlockManager, type BlockManagerConfig, ContinuousBatchScheduler, } from "./block-manager.ts";
-export { type DefragmentResult, PagedKVCache, type PagedKVCacheConfig, type SequenceEntry, } from "./page-table.ts";

package/dist/attention/paged-kv/page-table.d.ts

@@ -1,165 +0,0 @@
-/**
- * Paged KV Cache - Software page table for efficient memory management
- * Inspired by vLLM's PagedAttention
- */
-import type { WebInferDevice } from "../../core/device.ts";
-/**
- * Configuration for PagedKVCache
- */
-export interface PagedKVCacheConfig {
-    numLayers: number;
-    numHeads: number;
-    headDim: number;
-    pageSize: number;
-    maxPages: number;
-    dtype?: "f32" | "f16";
-}
-/**
- * Entry in the page table for a sequence
- */
-export interface SequenceEntry {
-    seqId: number;
-    pages: number[];
-    length: number;
-}
-/**
- * PagedKVCache - Manages KV cache with paging for efficient memory use
- *
- * Benefits:
- * 1. No memory fragmentation - pages are fixed size
- * 2. Efficient memory sharing - multiple sequences can share cache
- * 3. Dynamic allocation - only allocate pages as needed
- * 4. Easy defragmentation - just remap logical to physical pages
- */
-export declare class PagedKVCache {
-    private device;
-    private config;
-    private keyCache;
-    private valueCache;
-    private pageTable;
-    private freePages;
-    private nextSeqId;
-    constructor(device: WebInferDevice, config: PagedKVCacheConfig);
-    /**
-     * Allocate pages for a new sequence
-     */
-    allocateSequence(initialLength?: number): number;
-    /**
-     * Extend a sequence with new tokens
-     */
-    extendSequence(seqId: number, numNewTokens: number): void;
-    /**
-     * Free a sequence and its pages
-     */
-    freeSequence(seqId: number): void;
-    /**
-     * Get page indices for a sequence
-     */
-    getSequencePages(seqId: number): number[] | null;
-    /**
-     * Get sequence length
-     */
-    getSequenceLength(seqId: number): number;
-    /**
-     * Get the physical page index for a given sequence position
-     */
-    getPageForPosition(seqId: number, position: number): number | null;
-    /**
-     * Get offset within a page for a given position
-     */
-    getOffsetInPage(position: number): number;
-    private allocatePage;
-    private freePage;
-    /**
-     * Get cache statistics
-     */
-    getStats(): {
-        totalPages: number;
-        usedPages: number;
-        freePages: number;
-        numSequences: number;
-        memoryUsedBytes: number;
-        memoryTotalBytes: number;
-    };
-    /**
-     * Get GPU buffers for kernel binding
-     */
-    getBuffers(): {
-        keyCache: GPUBuffer;
-        valueCache: GPUBuffer;
-    };
-    /**
-     * Get configuration
-     */
-    getConfig(): PagedKVCacheConfig;
-    /**
-     * Dispose GPU resources
-     */
-    dispose(): void;
-    /**
-     * Allocate a new sequence (v2 alias)
-     */
-    alloc(initialLength?: number): number;
-    /**
-     * Append KV to sequence (v2 API)
-     */
-    append(seqId: number, kv: {
-        key: Float32Array;
-        value: Float32Array;
-        layer?: number;
-    }): void;
-    /**
-     * Batch append KV for all layers (v2 API)
-     */
-    appendBatch(seqId: number, kv: {
-        keys: Float32Array;
-        values: Float32Array;
-    }): void;
-    /**
-     * Free sequence (v2 alias)
-     */
-    free(seqId: number): void;
-    /**
-     * Get stats (v2 API)
-     */
-    stats(): {
-        usedPages: number;
-        freePages: number;
-        fragmentation: number;
-    };
-    /**
-     * Check if defrag needed (v2 alias)
-     */
-    needsDefrag(threshold?: number): boolean;
-    /**
-     * Check if defragmentation is needed
-     * Returns fragmentation ratio (0 = no fragmentation, 1 = fully fragmented)
-     */
-    getFragmentationRatio(): number;
-    /**
-     * Check if defragmentation would be beneficial
-     */
-    needsDefragmentation(threshold?: number): boolean;
-    /**
-     * Defragment the KV cache by compacting pages
-     * Returns the number of pages moved
-     */
-    defragment(): Promise<DefragmentResult>;
-    /**
-     * Move a single page from one location to another
-     */
-    private movePage;
-    /**
-     * Rebuild the free page list from scratch
-     */
-    private rebuildFreePageList;
-}
-/**
- * Result of defragmentation operation
- */
-export interface DefragmentResult {
-    pagesMoved: number;
-    durationMs: number;
-    fragmentationBefore: number;
-    fragmentationAfter: number;
-}

package/dist/attention/scheduler.d.ts

@@ -1,40 +0,0 @@
-/**
- * Attention Scheduler - Prevents TDR (GPU timeout) by splitting long sequences
- */
-import type { WebInferDevice } from "../core/device.ts";
-export interface ChunkPlan {
-    numChunks: number;
-    chunkSize: number;
-    estimatedTimeMs: number;
-}
-/**
- * Attention Scheduler for TDR prevention
- * Splits long sequences into chunks to avoid GPU timeout
- */
-export declare class AttentionScheduler {
-    private device;
-    private tdrLimit;
-    constructor(device: WebInferDevice);
-    private detectTDRLimit;
-    /**
-     * Estimate execution time for attention operation
-     * Based on empirical formula: time ∝ seqLen² × numHeads × headDim
-     */
-    estimateExecutionTime(seqLen: number, numHeads: number, headDim: number): number;
-    /**
-     * Compute chunk plan for given sequence length
-     */
-    computeChunkPlan(seqLen: number, numHeads: number, headDim: number): ChunkPlan;
-    /**
-     * Yield to main thread to prevent TDR
-     */
-    yieldToMain(): Promise<void>;
-    /**
-     * Check if sequence might cause TDR
-     */
-    mightCauseTDR(seqLen: number, numHeads: number, headDim: number): boolean;
-    /**
-     * Get recommended maximum sequence length for single-pass execution
-     */
-    getMaxSinglePassSeqLen(numHeads: number, headDim: number): number;
-}