web-llm-runner 0.1.1

package/lib/types.d.ts

@@ -0,0 +1,202 @@
+import { AppConfig, ChatOptions } from "./config";
+import { ChatCompletionRequest, ChatCompletionRequestBase, ChatCompletionRequestStreaming, ChatCompletionRequestNonStreaming, ChatCompletion, ChatCompletionChunk, CompletionCreateParams, Completion, CompletionCreateParamsBase, CompletionCreateParamsStreaming, CompletionCreateParamsNonStreaming, EmbeddingCreateParams, CreateEmbeddingResponse } from "./openai_api_protocols/index";
+import * as API from "./openai_api_protocols/index";
+/**
+ * Report during intialization.
+ */
+export interface InitProgressReport {
+    progress: number;
+    timeElapsed: number;
+    text: string;
+}
+/**
+ * Callbacks used to report initialization process.
+ */
+export type InitProgressCallback = (report: InitProgressReport) => void;
+/**
+ * A stateful logitProcessor used to post-process logits after forwarding the input and before
+ * sampling the next token. If used with `GenerationConfig.logit_bias`, logit_bias is applied after
+ * `processLogits()` is called.
+ */
+export interface LogitProcessor {
+    /**
+     * Process logits after forward() and before sampling implicitly, happens on the CPU.
+     * @param logits The logits right after forward().
+     * Returns the processed logits.
+     */
+    processLogits: (logits: Float32Array) => Float32Array;
+    /**
+     * Use the sampled token to update the LogitProcessor's internal state. Called implicitly
+     * right after the next token is sampled/committed.
+     * @param token Token sampled from the processed logits.
+     */
+    processSampledToken: (token: number) => void;
+    /**
+     * Called when in `MLCEngine.resetChat()`. Can clear internal states.
+     */
+    resetState: () => void;
+}
+/**
+ * Common interface of MLCEngine that UI can interact with
+ */
+export interface MLCEngineInterface {
+    /**
+     * An object that exposes chat-related APIs.
+     */
+    chat: API.Chat;
+    /**
+     * An object that exposes text completion APIs.
+     */
+    completions: API.Completions;
+    /**
+     * An object that exposes embeddings APIs.
+     */
+    embeddings: API.Embeddings;
+    /**
+     * Set an initialization progress callback function
+     * which reports the progress of model loading.
+     *
+     * This function can be useful to implement an UI that
+     * update as we loading the model.
+     *
+     * @param initProgressCallback The callback function
+     */
+    setInitProgressCallback: (initProgressCallback: InitProgressCallback) => void;
+    /**
+     * @returns The current initialization progress callback function.
+     */
+    getInitProgressCallback: () => InitProgressCallback | undefined;
+    /**
+     * Setter for the engine's appConfig.
+     */
+    setAppConfig: (appConfig: AppConfig) => void;
+    /**
+     * Reload the chat with a new model.
+     *
+     * @param modelId model_id of the model to load, either string or string[]. When multiple models
+     *   are provided, we load all models sequentially. Each modelId needs to either be in
+     *   `webllm.prebuiltAppConfig`, or in `engineConfig.appConfig`.
+     * @param chatOpts Extra options to optionally override the `mlc-chat-config.json` of `modelId`.
+     *   The size of which needs to match that of `modelId`; chatOpts[i] will be used for modelId[i].
+     * @returns A promise when reload finishes.
+     * @throws Throws error when device lost (mostly due to OOM); users should re-call reload(),
+     *   potentially with a smaller model or smaller context window size.
+     * @note This is an async function.
+     */
+    reload: (modelId: string | string[], chatOpts?: ChatOptions | ChatOptions[]) => Promise<void>;
+    /**
+     * OpenAI-style API. Generate a chat completion response for the given conversation and
+     * configuration. Use `engine.chat.completions.create()` to invoke this API.
+     *
+     * @param request A OpenAI-style ChatCompletion request.
+     *
+     * @note The API is completely functional in behavior. That is, a previous request would not
+     * affect the current request's result. Thus, for multi-round chatting, users are responsible for
+     * maintaining the chat history. With that being said, as an implicit internal optimization, if we
+     * detect that the user is performing multi-round chatting, we will preserve the KV cache and only
+     * prefill the new tokens.
+     * @note For requests sent to the same modelId, will block until all previous requests finish.
+     * @note For more, see https://platform.openai.com/docs/api-reference/chat
+     */
+    chatCompletion(request: ChatCompletionRequestNonStreaming): Promise<ChatCompletion>;
+    chatCompletion(request: ChatCompletionRequestStreaming): Promise<AsyncIterable<ChatCompletionChunk>>;
+    chatCompletion(request: ChatCompletionRequestBase): Promise<AsyncIterable<ChatCompletionChunk> | ChatCompletion>;
+    chatCompletion(request: ChatCompletionRequest): Promise<AsyncIterable<ChatCompletionChunk> | ChatCompletion>;
+    /**
+     * OpenAI-style API. Completes a CompletionCreateParams, a text completion with no chat template.
+     * Use `engine.completions.create()` to invoke this API.
+     *
+     * @param request An OpenAI-style Completion request.
+     *
+     * @note For requests sent to the same modelId, will block until all previous requests finish.
+     * @note For more, see https://platform.openai.com/docs/api-reference/completions
+     */
+    completion(request: CompletionCreateParamsNonStreaming): Promise<Completion>;
+    completion(request: CompletionCreateParamsStreaming): Promise<AsyncIterable<Completion>>;
+    completion(request: CompletionCreateParamsBase): Promise<AsyncIterable<Completion> | Completion>;
+    completion(request: CompletionCreateParams): Promise<AsyncIterable<Completion> | Completion>;
+    /**
+     * OpenAI-style API. Creates an embedding vector representing the input text.
+     * Use `engine.embeddings.create()` to invoke this API.
+     *
+     * @param request An OpenAI-style Embeddings request.
+     *
+     * @note For requests sent to the same modelId, will block until all previous requests finish.
+     * @note For more, see https://platform.openai.com/docs/api-reference/embeddings/create
+     */
+    embedding(request: EmbeddingCreateParams): Promise<CreateEmbeddingResponse>;
+    /**
+     * @returns A text summarizing the runtime stats.
+     * @param modelId Only required when multiple models are loaded.
+     * @note This is an async function
+     */
+    runtimeStatsText: (modelId?: string) => Promise<string>;
+    /**
+     * Interrupt the generate process if it is already running.
+     */
+    interruptGenerate: () => void;
+    /**
+     * Explicitly unload the currently loaded model(s) and release the related resources. Waits until
+     * the webgpu device finishes all submitted work and destroys itself.
+     * @note This is an asynchronous function.
+     */
+    unload: () => Promise<void>;
+    /**
+     * Reset the current chat session by clear all memories.
+     * @param keepStats: If True, do not reset the statistics.
+     * @param modelId Only required when multiple models are loaded.
+     */
+    resetChat: (keepStats?: boolean, modelId?: string) => Promise<void>;
+    /**
+     * Get the current generated response.
+     * @param modelId Only required when multiple models are loaded.
+     * @returns The current output message.
+     */
+    getMessage: (modelId?: string) => Promise<string>;
+    /**
+     * Returns the device's maxStorageBufferBindingSize, can be used to guess whether the device
+     * has limited resources like an Android phone.
+     */
+    getMaxStorageBufferBindingSize(): Promise<number>;
+    /**
+     * Returns the device's gpu vendor (e.g. arm, qualcomm, apple) if available. Otherwise return
+     * an empty string.
+     */
+    getGPUVendor(): Promise<string>;
+    /**
+     * Forward the given input tokens to the model, then sample the next token.
+     *
+     * This function has side effects as the model will update its KV cache.
+     *
+     * @param inputIds The input tokens.
+     * @param isPrefill True if prefill, false if decode; only used for statistics.
+     * @param modelId Only required when multiple models are loaded.
+     * @returns Next token sampled.
+     * @note This is an async function.
+     */
+    forwardTokensAndSample(inputIds: Array<number>, isPrefill: boolean, modelId?: string): Promise<number>;
+    /**
+     * Set MLCEngine logging output level
+     *
+     * @param logLevel The new log level
+     */
+    setLogLevel(logLevel: LogLevel): void;
+}
+export declare const LOG_LEVELS: {
+    TRACE: number;
+    DEBUG: number;
+    INFO: number;
+    WARN: number;
+    ERROR: number;
+    SILENT: number;
+};
+export type LogLevel = keyof typeof LOG_LEVELS;
+export type LatencyBreakdown = {
+    logitProcessorTime: number[];
+    logitBiasTime: number[];
+    penaltyTime: number[];
+    sampleTime: number[];
+    totalTime: number[];
+    grammarBitmaskTime: number[];
+};
+//# sourceMappingURL=types.d.ts.map

package/lib/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAClD,OAAO,EACL,qBAAqB,EACrB,yBAAyB,EACzB,8BAA8B,EAC9B,iCAAiC,EACjC,cAAc,EACd,mBAAmB,EACnB,sBAAsB,EACtB,UAAU,EACV,0BAA0B,EAC1B,+BAA+B,EAC/B,kCAAkC,EAClC,qBAAqB,EACrB,uBAAuB,EACxB,MAAM,8BAA8B,CAAC;AACtC,OAAO,KAAK,GAAG,MAAM,8BAA8B,CAAC;AAEpD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,CAAC,MAAM,EAAE,kBAAkB,KAAK,IAAI,CAAC;AAExE;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,aAAa,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,YAAY,CAAC;IAEtD;;;;OAIG;IACH,mBAAmB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAE7C;;OAEG;IACH,UAAU,EAAE,MAAM,IAAI,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE,GAAG,CAAC,WAAW,CAAC;IAE7B;;OAEG;IACH,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC;IAE3B;;;;;;;;OAQG;IACH,uBAAuB,EAAE,CAAC,oBAAoB,EAAE,oBAAoB,KAAK,IAAI,CAAC;IAE9E;;OAEG;IACH,uBAAuB,EAAE,MAAM,oBAAoB,GAAG,SAAS,CAAC;IAEhE;;OAEG;IACH,YAAY,EAAE,CAAC,SAAS,EAAE,SAAS,KAAK,IAAI,CAAC;IAE7C;;;;;;;;;;;;OAYG;IACH,MAAM,EAAE,CACN,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,QAAQ,CAAC,EAAE,WAAW,GAAG,WAAW,EAAE,KACnC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnB;;;;;;;;;;;;;OAaG;IACH,cAAc,CACZ,OAAO,EAAE,iCAAiC,GACzC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC3B,cAAc,CACZ,OAAO,EAAE,8BAA8B,GACtC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC,CAAC;IAC/C,cAAc,CACZ,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,GAAG,cAAc,CAAC,CAAC;IAChE,cAAc,CACZ,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,GAAG,cAAc,CAAC,CAAC;IAEhE;;;;;;;;OAQG;IACH,UAAU,CAAC,OAAO,EAAE,kCAAkC,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAC7E,UAAU,CACR,OAAO,EAAE,+BAA+B,GACvC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC;IACtC,UAAU,CACR,OAAO,EAAE,0BAA0B,GAClC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,CAAC;IACnD,UAAU,CACR,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,CAAC;IAEnD;;;;;;;;OAQG;IACH,SAAS,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAE5E;;;;OAIG;IACH,gBAAgB,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAExD;;OAEG;IACH,iBAAiB,EAAE,MAAM,IAAI,CAAC;IAE9B;;;;OAIG;IACH,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B;;;;OAIG;IACH,SAAS,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpE;;;;OAIG;IACH,UAAU,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAElD;;;OAGG;IACH,8BAA8B,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAElD;;;OAGG;IACH,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhC;;;;;;;;;;OAUG;IACH,sBAAsB,CACpB,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,EACvB,SAAS,EAAE,OAAO,EAClB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnB;;;;OAIG;IACH,WAAW,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;CACvC;AAED,eAAO,MAAM,UAAU;;;;;;;CAOtB,CAAC;AACF,MAAM,MAAM,QAAQ,GAAG,MAAM,OAAO,UAAU,CAAC;AAE/C,MAAM,MAAM,gBAAgB,GAAG;IAC7B,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B,CAAC"}

package/lib/utils.d.ts

@@ -0,0 +1,7 @@
+import { AppConfig, ChatOptions, ModelRecord } from "./config";
+export declare function areArraysEqual(arr1?: Array<any>, arr2?: Array<any>): boolean;
+export declare function areModelRecordsEqual(record1: ModelRecord, record2: ModelRecord): boolean;
+export declare function areAppConfigsEqual(config1?: AppConfig, config2?: AppConfig): boolean;
+export declare function areChatOptionsEqual(options1?: ChatOptions, options2?: ChatOptions): boolean;
+export declare function areChatOptionsListEqual(options1?: ChatOptions[], options2?: ChatOptions[]): boolean;
+//# sourceMappingURL=utils.d.ts.map

package/lib/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAmB,MAAM,UAAU,CAAC;AAGhF,wBAAgB,cAAc,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAQ5E;AAoBD,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,WAAW,GACnB,OAAO,CAkCT;AAED,wBAAgB,kBAAkB,CAChC,OAAO,CAAC,EAAE,SAAS,EACnB,OAAO,CAAC,EAAE,SAAS,GAClB,OAAO,CAwBT;AAED,wBAAgB,mBAAmB,CACjC,QAAQ,CAAC,EAAE,WAAW,EACtB,QAAQ,CAAC,EAAE,WAAW,GACrB,OAAO,CAmBT;AAED,wBAAgB,uBAAuB,CACrC,QAAQ,CAAC,EAAE,WAAW,EAAE,EACxB,QAAQ,CAAC,EAAE,WAAW,EAAE,GACvB,OAAO,CAoBT"}

package/lib/web_worker.d.ts

@@ -0,0 +1,132 @@
+import { AppConfig, ChatOptions, MLCEngineConfig } from "./config";
+import { MLCEngineInterface, InitProgressCallback, LogLevel, LogitProcessor } from "./types";
+import { ChatCompletionRequestBase, ChatCompletionRequestStreaming, ChatCompletionRequestNonStreaming, ChatCompletion, ChatCompletionChunk, Completion, CompletionCreateParamsNonStreaming, CompletionCreateParamsStreaming, CompletionCreateParamsBase, CreateEmbeddingResponse, EmbeddingCreateParams } from "./openai_api_protocols/index";
+import * as API from "./openai_api_protocols/index";
+import { MessageContent, WorkerRequest } from "./message";
+import { MLCEngine } from "./engine";
+/**
+ * Worker handler that can be used in a WebWorker
+ *
+ * @example
+ *
+ * // setup a chat worker handler that routes
+ * // requests to the chat
+ * const engine = new MLCEngine();
+ * cont handler = new WebWorkerMLCEngineHandler(engine);
+ * onmessage = handler.onmessage;
+ */
+export declare class WebWorkerMLCEngineHandler {
+    /**
+     * The modelId and chatOpts that the underlying engine (backend) is currently loaded with.
+     * An engine can be loaded with multiple models, so modelId and chatOpts are lists.
+     *
+     * TODO(webllm-team): This is always in-sync with `this.engine` unless device is lost due to
+     * unexpected reason. Therefore, we should get it from `this.engine` directly and make handler
+     * stateless. Besides, consider if we should add appConfig, or use engine's API to find the
+     * corresponding model record rather than relying on just the modelId.
+     */
+    modelId?: string[];
+    chatOpts?: ChatOptions[];
+    engine: MLCEngine;
+    /** ChatCompletion and Completion share the same chunk generator. Each loaded model has its own. */
+    protected loadedModelIdToAsyncGenerator: Map<string, AsyncGenerator<ChatCompletionChunk | Completion, void, void>>;
+    /**
+     * @param engine A concrete implementation of MLCEngineInterface
+     */
+    constructor();
+    postMessage(msg: any): void;
+    setLogitProcessorRegistry(logitProcessorRegistry?: Map<string, LogitProcessor>): void;
+    handleTask<T extends MessageContent>(uuid: string, task: () => Promise<T>): Promise<void>;
+    onmessage(event: any, onComplete?: (value: any) => void, onError?: () => void): void;
+    /** Check whether frontend expectation matches with backend (modelId and chatOpts). If not (due
+     * to possibly killed service worker), we reload here.
+     * For more, see https://github.com/mlc-ai/web-llm/pull/533
+     */
+    reloadIfUnmatched(expectedModelId: string[], expectedChatOpts?: ChatOptions[]): Promise<void>;
+}
+export interface ChatWorker {
+    onmessage: any;
+    postMessage: (message: any) => void;
+}
+/**
+ * Creates `WebWorkerMLCEngine`, a client that holds the same interface as `MLCEngine`.
+ *
+ * Equivalent to `new webllm.WebWorkerMLCEngine(worker).reload(...)`.
+ *
+ * @param worker The worker that holds the actual MLCEngine, initialized with `new Worker()`.
+ * @param modelId model_id of the model to load, either string or string[]. When multiple models
+ *   are provided, we load all models sequentially. Each modelId needs to either be in
+ *   `webllm.prebuiltAppConfig`, or in `engineCOnfig.appConfig`.
+ * @param engineConfig Optionally configures the engine, see `webllm.MLCEngineConfig` for more.
+ * @param chatOpts Extra options to optionally override the `mlc-chat-config.json` of `modelId`.
+ *   The size of which needs to match that of `modelId`; chatOpts[i] will be used for modelId[i].
+ * @returns An initialized `WebLLM.WebWorkerMLCEngine` with `modelId` loaded.
+ *
+ * @note engineConfig.logitProcessorRegistry is ignored for `CreateWebWorkMLCEngine()`.
+ */
+export declare function CreateWebWorkerMLCEngine(worker: any, modelId: string | string[], engineConfig?: MLCEngineConfig, chatOpts?: ChatOptions | ChatOptions[]): Promise<WebWorkerMLCEngine>;
+/**
+ * A client of MLCEngine that exposes the same interface
+ *
+ * @example
+ *
+ * const chat = new webllm.WebWorkerMLCEngine(new Worker(
+ *   new URL('./worker.ts', import.meta.url),
+ *   {type: 'module'}
+ * ));
+ */
+export declare class WebWorkerMLCEngine implements MLCEngineInterface {
+    worker: ChatWorker;
+    /** For chat.completions.create() */
+    chat: API.Chat;
+    /** For completions.create() */
+    completions: API.Completions;
+    /** For embeddings.create() */
+    embeddings: API.Embeddings;
+    /**
+     * The modelId and chatOpts that the frontend expects the backend engine is currently loaded
+     * with. Needed for service worker. It is the backend and handler's job to match up with the
+     * expectation despite the web/service worker possibly being killed.
+     * Since an engine can load multiple models, both modelId and chatOpts are lists.
+     */
+    modelId?: string[];
+    chatOpts?: ChatOptions[];
+    private initProgressCallback?;
+    private pendingPromise;
+    constructor(worker: ChatWorker, engineConfig?: MLCEngineConfig);
+    setInitProgressCallback(initProgressCallback?: InitProgressCallback): void;
+    getInitProgressCallback(): InitProgressCallback | undefined;
+    setAppConfig(appConfig: AppConfig): void;
+    setLogLevel(logLevel: LogLevel): void;
+    protected getPromise<T extends MessageContent>(msg: WorkerRequest): Promise<T>;
+    reload(modelId: string | string[], chatOpts?: ChatOptions | ChatOptions[]): Promise<void>;
+    getMaxStorageBufferBindingSize(): Promise<number>;
+    getGPUVendor(): Promise<string>;
+    getMessage(modelId?: string): Promise<string>;
+    runtimeStatsText(modelId?: string): Promise<string>;
+    interruptGenerate(): void;
+    unload(): Promise<void>;
+    resetChat(keepStats?: boolean, modelId?: string): Promise<void>;
+    forwardTokensAndSample(inputIds: Array<number>, isPrefill: boolean, modelId?: string): Promise<number>;
+    /**
+     * Every time the generator is called, we post a message to the worker asking it to
+     * decode one step, and we expect to receive a message of `ChatCompletionChunk` from
+     * the worker which we yield. The last message is `void`, meaning the generator has nothing
+     * to yield anymore.
+     *
+     * @param selectedModelId: The model of whose async generator to call next() to get next chunk.
+     *   Needed because an engine can load multiple models.
+     *
+     * @note ChatCompletion and Completion share the same chunk generator.
+     */
+    asyncGenerate(selectedModelId: string): AsyncGenerator<ChatCompletionChunk | Completion, void, void>;
+    chatCompletion(request: ChatCompletionRequestNonStreaming): Promise<ChatCompletion>;
+    chatCompletion(request: ChatCompletionRequestStreaming): Promise<AsyncIterable<ChatCompletionChunk>>;
+    chatCompletion(request: ChatCompletionRequestBase): Promise<AsyncIterable<ChatCompletionChunk> | ChatCompletion>;
+    completion(request: CompletionCreateParamsNonStreaming): Promise<Completion>;
+    completion(request: CompletionCreateParamsStreaming): Promise<AsyncIterable<Completion>>;
+    completion(request: CompletionCreateParamsBase): Promise<AsyncIterable<Completion> | Completion>;
+    embedding(request: EmbeddingCreateParams): Promise<CreateEmbeddingResponse>;
+    onmessage(event: any): void;
+}
+//# sourceMappingURL=web_worker.d.ts.map

package/lib/web_worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"web_worker.d.ts","sourceRoot":"","sources":["../src/web_worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AACnE,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EAEpB,QAAQ,EACR,cAAc,EACf,MAAM,SAAS,CAAC;AACjB,OAAO,EAEL,yBAAyB,EACzB,8BAA8B,EAC9B,iCAAiC,EACjC,cAAc,EACd,mBAAmB,EACnB,UAAU,EACV,kCAAkC,EAClC,+BAA+B,EAC/B,0BAA0B,EAE1B,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,8BAA8B,CAAC;AACtC,OAAO,KAAK,GAAG,MAAM,8BAA8B,CAAC;AACpD,OAAO,EACL,cAAc,EAOd,aAAa,EAOd,MAAM,WAAW,CAAC;AAEnB,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAQrC;;;;;;;;;;GAUG;AACH,qBAAa,yBAAyB;IACpC;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IAElB,MAAM,EAAE,SAAS,CAAC;IACzB,mGAAmG;IACnG,SAAS,CAAC,6BAA6B,EAAE,GAAG,CAC1C,MAAM,EACN,cAAc,CAAC,mBAAmB,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,CAC7D,CAAC;IAEF;;OAEG;;IAiBH,WAAW,CAAC,GAAG,EAAE,GAAG;IAKpB,yBAAyB,CACvB,sBAAsB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC;IAKhD,UAAU,CAAC,CAAC,SAAS,cAAc,EACvC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC;IAqBxB,SAAS,CACP,KAAK,EAAE,GAAG,EACV,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,EACjC,OAAO,CAAC,EAAE,MAAM,IAAI;IA+NtB;;;OAGG;IACG,iBAAiB,CACrB,eAAe,EAAE,MAAM,EAAE,EACzB,gBAAgB,CAAC,EAAE,WAAW,EAAE;CAYnC;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,GAAG,CAAC;IACf,WAAW,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,IAAI,CAAC;CACrC;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,GAAG,EACX,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,YAAY,CAAC,EAAE,eAAe,EAC9B,QAAQ,CAAC,EAAE,WAAW,GAAG,WAAW,EAAE,GACrC,OAAO,CAAC,kBAAkB,CAAC,CAI7B;AAED;;;;;;;;;GASG;AACH,qBAAa,kBAAmB,YAAW,kBAAkB;IACpD,MAAM,EAAE,UAAU,CAAC;IAC1B,oCAAoC;IAC7B,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC;IACtB,+BAA+B;IACxB,WAAW,EAAE,GAAG,CAAC,WAAW,CAAC;IACpC,8BAA8B;IACvB,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC;IAElC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IAEzB,OAAO,CAAC,oBAAoB,CAAC,CAAuB;IACpD,OAAO,CAAC,cAAc,CAAoD;gBAE9D,MAAM,EAAE,UAAU,EAAE,YAAY,CAAC,EAAE,eAAe;IA0B9D,uBAAuB,CAAC,oBAAoB,CAAC,EAAE,oBAAoB;IAInE,uBAAuB,IAAI,oBAAoB,GAAG,SAAS;IAI3D,YAAY,CAAC,SAAS,EAAE,SAAS;IASjC,WAAW,CAAC,QAAQ,EAAE,QAAQ;IAU9B,SAAS,CAAC,UAAU,CAAC,CAAC,SAAS,cAAc,EAC3C,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,CAAC,CAAC;IAwBP,MAAM,CACV,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,QAAQ,CAAC,EAAE,WAAW,GAAG,WAAW,EAAE,GACrC,OAAO,CAAC,IAAI,CAAC;IAsBV,8BAA8B,IAAI,OAAO,CAAC,MAAM,CAAC;IASjD,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAS/B,UAAU,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAW7C,gBAAgB,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWzD,iBAAiB,IAAI,IAAI;IASnB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAWvB,SAAS,CAAC,SAAS,UAAQ,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY7D,sBAAsB,CAC1B,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,EACvB,SAAS,EAAE,OAAO,EAClB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC;IAalB;;;;;;;;;;OAUG;IACI,aAAa,CAClB,eAAe,EAAE,MAAM,GACtB,cAAc,CAAC,mBAAmB,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC;IAmBzD,cAAc,CAClB,OAAO,EAAE,iCAAiC,GACzC,OAAO,CAAC,cAAc,CAAC;IACpB,cAAc,CAClB,OAAO,EAAE,8BAA8B,GACtC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;IACxC,cAAc,CAClB,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,GAAG,cAAc,CAAC;IAmDzD,UAAU,CACd,OAAO,EAAE,kCAAkC,GAC1C,OAAO,CAAC,UAAU,CAAC;IAChB,UAAU,CACd,OAAO,EAAE,+BAA+B,GACvC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IAC/B,UAAU,CACd,OAAO,EAAE,0BAA0B,GAClC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC;IAmD5C,SAAS,CACb,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC,uBAAuB,CAAC;IAgBnC,SAAS,CAAC,KAAK,EAAE,GAAG;CAsCrB"}

package/lib/wrapper/WebLLMWrapper.d.ts

@@ -0,0 +1,20 @@
+export declare class WebLLM {
+    private engine;
+    private worker;
+    private messages;
+    downloadProgress: Record<string, string>;
+    constructor();
+    get models_available(): string[];
+    local_model_available(model_id: string): Promise<boolean>;
+    download_model(model_id: string, workerUrl?: string | URL): Promise<void>;
+    progress(model_id: string): string;
+    delete_model(model_id: string): Promise<void>;
+    chat_stream(prompt: string): AsyncGenerator<string, void, unknown>;
+    chat_no_stream(prompt: string): Promise<string>;
+    reset_chat(): void;
+    generate_stream(prompt: string): AsyncGenerator<string, void, unknown>;
+    generate_no_stream(prompt: string): Promise<string>;
+    private loadContext;
+    private saveContext;
+}
+//# sourceMappingURL=WebLLMWrapper.d.ts.map

package/lib/wrapper/WebLLMWrapper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebLLMWrapper.d.ts","sourceRoot":"","sources":["../../src/wrapper/WebLLMWrapper.ts"],"names":[],"mappings":"AASA,qBAAa,MAAM;IACjB,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAoC;IAC7C,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;;IAOrD,IAAW,gBAAgB,IAAI,MAAM,EAAE,CAEtC;IAEY,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIzD,cAAc,CACzB,QAAQ,EAAE,MAAM,EAEhB,SAAS,GAAE,MAAM,GAAG,GAAiD,GACpE,OAAO,CAAC,IAAI,CAAC;IAYT,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAI5B,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK5C,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC;IAqBnE,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAiBrD,UAAU,IAAI,IAAI;IAMX,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC;IAavE,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWhE,OAAO,CAAC,WAAW;IAYnB,OAAO,CAAC,WAAW;CASpB"}

package/lib/wrapper/llm-worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=llm-worker.d.ts.map

package/lib/wrapper/llm-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-worker.d.ts","sourceRoot":"","sources":["../../src/wrapper/llm-worker.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "web-llm-runner",
+  "version": "0.1.1",
+  "description": "Hardware accelerated language model chats on browsers",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "type": "module",
+  "scripts": {
+    "build": "rollup -c && ./cleanup-index-js.sh",
+    "lint": "npx eslint ./src/ ./tests/ ./examples/ && npx prettier ./src/ ./tests/ ./examples/ --check",
+    "test": "jest --coverage",
+    "format": "prettier --write \"./src/\" \"./examples/\" \"./tests/\"",
+    "prepare": "husky"
+  },
+  "files": [
+    "lib"
+  ],
+  "keywords": [
+    "llm",
+    "large language model",
+    "machine learning"
+  ],
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "@eslint/eslintrc": "^3.3.1",
+    "@eslint/js": "^9.9.0",
+    "@mlc-ai/web-runtime": "^0.24.0-dev2",
+    "@mlc-ai/web-tokenizers": "^0.1.6",
+    "@mlc-ai/web-xgrammar": "0.1.27",
+    "@next/eslint-plugin-next": "^16.0.0",
+    "@rollup/plugin-commonjs": "^29.0.0",
+    "@rollup/plugin-node-resolve": "^16.0.3",
+    "@rollup/plugin-typescript": "^12.3.0",
+    "@types/chrome": "^0.0.266",
+    "@types/jest": "^30.0.0",
+    "@types/serviceworker": "^0.0.86",
+    "@webgpu/types": "^0.1.24",
+    "buffer": "^5.7.1",
+    "eslint": "^9.39.1",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-prettier": "^5.5.4",
+    "husky": "^9.0.11",
+    "jest": "^30.2.0",
+    "prettier": "3.6.2",
+    "process": "^0.11.10",
+    "rollup": "^4.53.3",
+    "rollup-plugin-ignore": "^1.0.10",
+    "ts-jest": "^29.4.6",
+    "tslib": "^2.3.1",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.47.0"
+  },
+  "dependencies": {
+    "loglevel": "^1.9.1"
+  },
+  "overrides": {
+    "test-exclude": "^7.0.1",
+    "glob": "^13.0.0"
+  }
+}