webinfer 0.0.1

package/dist/ops/elementwise.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Element-wise Operations
+ * Add, Multiply, Scale, etc. for tensor operations
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import { Tensor } from "../core/tensor.ts";
+/**
+ * Element-wise add (CPU)
+ */
+export declare function addCPU(a: Float32Array, b: Float32Array): Float32Array;
+/**
+ * Element-wise multiply (CPU)
+ */
+export declare function mulCPU(a: Float32Array, b: Float32Array): Float32Array;
+/**
+ * Scale by constant (CPU)
+ */
+export declare function scaleCPU(a: Float32Array, scalar: number): Float32Array;
+/**
+ * Add constant (CPU)
+ */
+export declare function addScalarCPU(a: Float32Array, scalar: number): Float32Array;
+/**
+ * Fused multiply-add: a * b + c (CPU)
+ */
+export declare function fmaCPU(a: Float32Array, b: Float32Array, c: Float32Array): Float32Array;
+/**
+ * Element-wise add (GPU)
+ */
+export declare function add(device: WebInferDevice, a: Tensor, b: Tensor): Promise<Tensor>;
+/**
+ * Element-wise multiply (GPU)
+ */
+export declare function mul(device: WebInferDevice, a: Tensor, b: Tensor): Promise<Tensor>;
+/**
+ * Scale tensor by constant (GPU)
+ */
+export declare function scale(device: WebInferDevice, a: Tensor, scalar: number): Promise<Tensor>;

package/dist/ops/embedding.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Embedding Operations
+ * Token embedding lookup for LLM inference
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import { Tensor } from "../core/tensor.ts";
+/**
+ * Embedding lookup (CPU)
+ * @param embeddings - Embedding table [vocabSize, embeddingDim]
+ * @param tokens - Token indices [seqLen]
+ * @param embeddingDim - Dimension of each embedding
+ * @returns Embedded tokens [seqLen, embeddingDim]
+ */
+export declare function embeddingCPU(embeddings: Float32Array, tokens: number[], embeddingDim: number): Float32Array;
+/**
+ * Embedding lookup (GPU)
+ * @param device - WebInfer device
+ * @param embeddings - Embedding table tensor [vocabSize, embeddingDim]
+ * @param tokens - Token indices tensor [seqLen]
+ * @returns Embedded tokens tensor [seqLen, embeddingDim]
+ */
+export declare function embedding(device: WebInferDevice, embeddings: Tensor, tokens: Tensor): Promise<Tensor>;
+/**
+ * Batched embedding lookup (CPU)
+ * @param embeddings - Embedding table [vocabSize, embeddingDim]
+ * @param tokens - Token indices [batchSize, seqLen]
+ * @param embeddingDim - Dimension of each embedding
+ * @returns Embedded tokens [batchSize, seqLen, embeddingDim]
+ */
+export declare function batchedEmbeddingCPU(embeddings: Float32Array, tokens: number[][], embeddingDim: number): Float32Array;

package/dist/ops/matmul.d.ts

@@ -0,0 +1,21 @@
+/**
+ * MatMul Operator - GPU-accelerated matrix multiplication
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import { Tensor } from "../core/tensor.ts";
+/**
+ * Matrix multiplication: C = A @ B
+ * @param device WebInfer device
+ * @param a Input tensor [M, K]
+ * @param b Input tensor [K, N]
+ * @returns Output tensor [M, N]
+ */
+export declare function matmul(device: WebInferDevice, a: Tensor, b: Tensor): Promise<Tensor>;
+/**
+ * CPU reference implementation for verification
+ */
+export declare function matmulCPU(a: Float32Array, b: Float32Array, M: number, N: number, K: number): Float32Array;
+/**
+ * Get compiler cache statistics
+ */
+export declare function getMatMulCacheStats(device: WebInferDevice): import("../index.ts").CacheStats;

package/dist/ops/normalization.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Normalization Operations
+ * LayerNorm and RMSNorm implementations
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import { Tensor } from "../core/tensor.ts";
+/**
+ * Layer Normalization (CPU)
+ * Normalizes over the last dimension
+ */
+export declare function layerNormCPU(x: Float32Array, weight: Float32Array, bias: Float32Array | null, shape: number[], eps?: number): Float32Array;
+/**
+ * RMS Normalization (CPU)
+ * Used in Llama and other modern LLMs
+ */
+export declare function rmsNormCPU(x: Float32Array, weight: Float32Array, shape: number[], eps?: number): Float32Array;
+/**
+ * Layer Normalization (GPU)
+ */
+export declare function layerNorm(device: WebInferDevice, x: Tensor, weight: Tensor, bias: Tensor | null, eps?: number): Promise<Tensor>;
+/**
+ * RMS Normalization (GPU)
+ */
+export declare function rmsNorm(device: WebInferDevice, x: Tensor, weight: Tensor, eps?: number): Promise<Tensor>;

package/dist/ops/reshape.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Reshape and Transpose Operations
+ * Shape manipulation for tensor operations
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import { Tensor } from "../core/tensor.ts";
+/**
+ * Transpose 2D matrix (CPU)
+ * [M, N] -> [N, M]
+ */
+export declare function transpose2DCPU(x: Float32Array, rows: number, cols: number): Float32Array;
+/**
+ * Transpose last two dimensions (CPU)
+ * [..., M, N] -> [..., N, M]
+ */
+export declare function transposeCPU(x: Float32Array, shape: number[]): {
+    data: Float32Array;
+    shape: number[];
+};
+/**
+ * Reshape tensor (CPU) - just validates and returns new shape
+ * Data layout doesn't change, only interpretation
+ */
+export declare function reshapeCPU(x: Float32Array, oldShape: number[], newShape: number[]): {
+    data: Float32Array;
+    shape: number[];
+};
+/**
+ * Transpose 2D matrix (GPU)
+ */
+export declare function transpose2D(device: WebInferDevice, x: Tensor): Promise<Tensor>;
+/**
+ * Permute dimensions (CPU)
+ * Generalized transpose for any dimension ordering
+ */
+export declare function permuteCPU(x: Float32Array, shape: number[], dims: number[]): {
+    data: Float32Array;
+    shape: number[];
+};

package/dist/ops/rope.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Rotary Position Embedding (RoPE)
+ * Used in Llama, Mistral, and most modern LLMs
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import { Tensor } from "../core/tensor.ts";
+/**
+ * RoPE configuration
+ */
+export interface RoPEConfig {
+    dim: number;
+    maxSeqLen: number;
+    base?: number;
+    scaling?: number;
+}
+/**
+ * Precompute RoPE frequencies
+ * Returns cos and sin values for each position and dimension
+ */
+export declare function computeRoPEFrequencies(config: RoPEConfig): {
+    cos: Float32Array;
+    sin: Float32Array;
+};
+/**
+ * Apply RoPE to query/key tensors (CPU implementation)
+ * Input shape: [seqLen, numHeads, headDim]
+ */
+export declare function ropeCPU(x: Float32Array, positions: number[], cos: Float32Array, sin: Float32Array, seqLen: number, numHeads: number, headDim: number): Float32Array;
+/**
+ * Apply RoPE to query/key tensor (GPU implementation)
+ */
+export declare function rope(device: WebInferDevice, x: Tensor, positions: Tensor, config: RoPEConfig): Promise<Tensor>;

package/dist/ops/softmax.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Softmax Operation
+ * GPU-accelerated softmax for attention and output layers
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import { Tensor } from "../core/tensor.ts";
+/**
+ * Softmax (CPU) - operates along last dimension
+ */
+export declare function softmaxCPU(x: Float32Array, shape: number[]): Float32Array;
+/**
+ * Log-Softmax (CPU) - more numerically stable for cross-entropy loss
+ */
+export declare function logSoftmaxCPU(x: Float32Array, shape: number[]): Float32Array;
+/**
+ * Softmax (GPU) - operates along last dimension
+ */
+export declare function softmaxGPU(device: WebInferDevice, x: Tensor): Promise<Tensor>;

package/dist/quantization/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Quantization Module
+ * INT4 and INT8 quantization for efficient LLM inference
+ */
+export { quantizeToInt8, quantizeToInt4, dequantizeInt8, dequantizeInt4, quantizationError, getMemorySavings, type QuantConfig, type QuantizedTensor, } from "./quantize.ts";
+export { qmatmulInt8CPU, qmatmulInt4CPU, qmatmulInt8BlockCPU, estimateQMatMulFlops, estimateQMatMulBandwidth, } from "./qmatmul.ts";

package/dist/quantization/qmatmul.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Quantized Matrix Multiplication
+ * INT4 and INT8 matmul for efficient LLM inference
+ */
+import type { QuantizedTensor } from "./quantize.ts";
+/**
+ * INT8 MatMul (CPU): float16/32 activations × int8 weights
+ * A: [M, K] float32 activations
+ * B: QuantizedTensor [K, N] int8 weights
+ * Output: [M, N] float32
+ */
+export declare function qmatmulInt8CPU(A: Float32Array, B: QuantizedTensor, M: number, K: number, N: number): Float32Array;
+/**
+ * INT4 MatMul (CPU): float activations × int4 weights
+ * A: [M, K] float32 activations
+ * B: QuantizedTensor [K, N] int4 weights (packed)
+ * Output: [M, N] float32
+ */
+export declare function qmatmulInt4CPU(A: Float32Array, B: QuantizedTensor, M: number, K: number, N: number): Float32Array;
+/**
+ * Optimized INT8 MatMul with block accumulation
+ * Uses SIMD-friendly access patterns
+ */
+export declare function qmatmulInt8BlockCPU(A: Float32Array, B: QuantizedTensor, M: number, K: number, N: number, blockSize?: number): Float32Array;
+/**
+ * Compute expected FLOPS for quantized matmul
+ */
+export declare function estimateQMatMulFlops(M: number, K: number, N: number): number;
+/**
+ * Estimate memory bandwidth for quantized matmul
+ */
+export declare function estimateQMatMulBandwidth(M: number, K: number, N: number, bits: 4 | 8, groupSize: number): {
+    activationBytes: number;
+    weightBytes: number;
+    scaleBytes: number;
+    outputBytes: number;
+    totalBytes: number;
+};

package/dist/quantization/quantize.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Quantization Operations
+ * INT4 and INT8 quantization for memory-efficient LLM inference
+ */
+/**
+ * Quantization configuration
+ */
+export interface QuantConfig {
+    bits: 4 | 8;
+    groupSize: number;
+    symmetric: boolean;
+}
+/**
+ * Quantized tensor representation
+ */
+export interface QuantizedTensor {
+    data: Uint8Array;
+    scales: Float32Array;
+    zeros: Float32Array | null;
+    shape: number[];
+    config: QuantConfig;
+}
+/**
+ * Quantize Float32 tensor to INT8 (CPU)
+ */
+export declare function quantizeToInt8(x: Float32Array, groupSize?: number, symmetric?: boolean): QuantizedTensor;
+/**
+ * Quantize Float32 tensor to INT4 (CPU)
+ * Packs two INT4 values per byte
+ */
+export declare function quantizeToInt4(x: Float32Array, groupSize?: number, symmetric?: boolean): QuantizedTensor;
+/**
+ * Dequantize INT8 tensor back to Float32 (CPU)
+ */
+export declare function dequantizeInt8(qt: QuantizedTensor): Float32Array;
+/**
+ * Dequantize INT4 tensor back to Float32 (CPU)
+ */
+export declare function dequantizeInt4(qt: QuantizedTensor): Float32Array;
+/**
+ * Compute quantization error (MSE)
+ */
+export declare function quantizationError(original: Float32Array, reconstructed: Float32Array): number;
+/**
+ * Get memory savings from quantization
+ */
+export declare function getMemorySavings(originalBytes: number, qt: QuantizedTensor): {
+    originalBytes: number;
+    quantizedBytes: number;
+    savings: number;
+    ratio: number;
+};

package/dist/sampling/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Sampling Module Exports
+ */
+export { topK, topKCPU, topKFilter } from "./top-k.ts";
+export { topPFilter, topPFilterCPU } from "./top-p.ts";
+export { sample, sampleCPU, sampleGreedy, sampleFromProbs, softmax, applyRepetitionPenalty, type SamplingConfig, } from "./sampler.ts";

package/dist/sampling/sampler.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Token Sampler - Combines filtering and sampling strategies
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import { Tensor } from "../core/tensor.ts";
+/**
+ * Sampling configuration
+ */
+export interface SamplingConfig {
+    temperature?: number;
+    topK?: number;
+    topP?: number;
+    repetitionPenalty?: number;
+    seed?: number;
+}
+/**
+ * Apply softmax to logits
+ */
+export declare function softmax(logits: Float32Array): Float32Array;
+/**
+ * Apply repetition penalty to logits
+ */
+export declare function applyRepetitionPenalty(logits: Float32Array, previousTokens: number[], penalty: number): Float32Array;
+/**
+ * Sample a token from probability distribution
+ */
+export declare function sampleFromProbs(probs: Float32Array, random?: () => number): number;
+/**
+ * Greedy sampling (argmax)
+ */
+export declare function sampleGreedy(logits: Float32Array): number;
+/**
+ * Full sampling pipeline
+ */
+export declare function sample(device: WebInferDevice, logits: Tensor, config?: SamplingConfig, previousTokens?: number[]): Promise<number>;
+/**
+ * CPU-only sampling (no GPU tensors)
+ */
+export declare function sampleCPU(logits: Float32Array, config?: SamplingConfig, previousTokens?: number[]): number;

package/dist/sampling/top-k.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Top-K Sampling - GPU-accelerated top-k selection
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import { Tensor } from "../core/tensor.ts";
+/**
+ * GPU-accelerated Top-K selection
+ * Returns the top K values and their indices from logits
+ */
+export declare function topK(device: WebInferDevice, logits: Tensor, k: number): Promise<{
+    values: Tensor;
+    indices: Tensor;
+}>;
+/**
+ * CPU reference implementation
+ */
+export declare function topKCPU(logits: Float32Array, k: number, vocabSize: number): {
+    values: Float32Array;
+    indices: Uint32Array;
+};
+/**
+ * Apply top-k filtering to logits (set non-top-k to -inf)
+ */
+export declare function topKFilter(device: WebInferDevice, logits: Tensor, k: number): Promise<Tensor>;

package/dist/sampling/top-p.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Top-P (Nucleus) Sampling
+ */
+import type { WebInferDevice } from "../core/device.ts";
+import { Tensor } from "../core/tensor.ts";
+/**
+ * Apply top-p (nucleus) filtering to logits
+ * Keeps the smallest set of tokens whose cumulative probability >= p
+ */
+export declare function topPFilter(device: WebInferDevice, logits: Tensor, p: number, temperature?: number): Promise<Tensor>;
+/**
+ * CPU implementation of top-p filtering
+ */
+export declare function topPFilterCPU(logits: Float32Array, p: number, temperature?: number): Float32Array;

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "webinfer",
+  "version": "0.0.1",
+  "description": "High-performance LLM inference kernels for WebGPU",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/guan404ming/webinfer"
+  },
+  "keywords": [
+    "webgpu",
+    "llm",
+    "inference",
+    "gpu",
+    "machine-learning",
+    "transformer",
+    "attention",
+    "matmul"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "dev": "bun --hot benchmarks/server.ts",
+    "bench": "open http://localhost:3000 && bun --hot benchmarks/server.ts",
+    "build": "bun run build:types && bun run build:js",
+    "build:types": "tsc -p tsconfig.build.json",
+    "build:js": "bun build ./src/index.ts --outdir ./dist --target browser --format esm",
+    "test": "bun test",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "bun run build"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "@webgpu/types": "^0.1.49",
+    "onnxruntime-web": "^1.20.1",
+    "typescript": "^5.3.0"
+  },
+  "peerDependencies": {
+    "@webgpu/types": "^0.1.0"
+  }
+}