web-llm-runner 0.1.1

package/lib/config.d.ts

@@ -0,0 +1,199 @@
+import { ResponseFormat } from "./openai_api_protocols";
+import { LogitProcessor, InitProgressCallback, LogLevel } from "./types";
+import { ModelIntegrity } from "./integrity";
+/**
+ * Conversation template config
+ */
+export interface ConvTemplateConfig {
+    system_template: string;
+    system_message: string;
+    roles: Record<Role, string>;
+    role_templates?: Partial<Record<Role, string>>;
+    seps: Array<string>;
+    role_content_sep?: string;
+    role_empty_sep?: string;
+    stop_str: Array<string>;
+    system_prefix_token_ids?: Array<number>;
+    stop_token_ids: Array<number>;
+    add_role_after_system_message?: boolean;
+}
+export declare enum Role {
+    user = "user",
+    assistant = "assistant",
+    tool = "tool"
+}
+export declare const DefaultLogLevel: LogLevel;
+/**
+ * Place holders that can be used in role templates.
+ * For example, a role template of
+ * `<<question>> ${MessagePlaceholders.USER} <<function>> ${MessagePlaceholders.FUNCTION}`
+ * will insert the user message to ${MessagePlaceholders.USER}
+ * and insert the function message to ${MessagePlaceholders.FUNCTION}
+ * at run time.
+ */
+export declare enum MessagePlaceholders {
+    system = "{system_message}",
+    user = "{user_message}",
+    assistant = "{assistant_message}",
+    tool = "{tool_message}",
+    function = "{function_string}",
+    hermes_tools = "{hermes_tools}"
+}
+/**
+ * Information about the tokenizer. Currently, only `token_postproc_method` is used to
+ * post process the token table when using grammar.
+ */
+export interface TokenizerInfo {
+    token_postproc_method: string;
+    prepend_space_in_encode: boolean;
+    strip_space_in_decode: boolean;
+}
+/**
+ * Config of one chat model, a data structure representing `mlc-chat-config.json`.
+ * This only corresponds to the chat-related fields and `tokenizer_files` of `mlc-chat-config.json`.
+ * Only these fields affect the conversation in runtime.
+ * i.e. The third part in https://llm.mlc.ai/docs/get_started/mlc_chat_config.html.
+ *
+ * This is initialized in `MLCEngine.reload()` with the model's `mlc-chat-config.json`.
+ */
+export interface ChatConfig {
+    tokenizer_files: Array<string>;
+    tokenizer_info?: TokenizerInfo;
+    token_table_postproc_method?: string;
+    vocab_size: number;
+    conv_config?: Partial<ConvTemplateConfig>;
+    conv_template: ConvTemplateConfig;
+    context_window_size: number;
+    sliding_window_size: number;
+    attention_sink_size: number;
+    repetition_penalty: number;
+    frequency_penalty: number;
+    presence_penalty: number;
+    top_p: number;
+    temperature: number;
+    bos_token_id?: number;
+}
+/**
+ * Custom options that can be used to override known config values.
+ */
+export interface ChatOptions extends Partial<ChatConfig> {
+}
+/**
+ * Optional configurations for `CreateMLCEngine()` and `CreateWebWorkerMLCEngine()`.
+ *
+ * appConfig: Configure the app, including the list of models and whether to use IndexedDB cache.
+ * initProgressCallback: A callback for showing the progress of loading the model.
+ * logitProcessorRegistry: A register for stateful logit processors, see `webllm.LogitProcessor`.
+ *
+ * @note All fields are optional, and `logitProcessorRegistry` is only used for `MLCEngine` and not
+ * other `MLCEngine`s.
+ */
+export interface MLCEngineConfig {
+    appConfig?: AppConfig;
+    initProgressCallback?: InitProgressCallback;
+    logitProcessorRegistry?: Map<string, LogitProcessor>;
+    logLevel?: LogLevel;
+}
+/**
+ * Config for a single generation.
+ * Essentially `ChatConfig` without `tokenizer_files`, `conv_config`, or `conv_template`.
+ * We also support additional fields not present in `mlc-chat-config.json` due to OpenAI-like APIs.
+ *
+ * Note that all values are optional. If unspecified, we use whatever values in `ChatConfig`
+ * initialized during `MLCEngine.reload()`.
+ */
+export interface GenerationConfig {
+    repetition_penalty?: number | null;
+    ignore_eos?: boolean;
+    top_p?: number | null;
+    temperature?: number | null;
+    max_tokens?: number | null;
+    frequency_penalty?: number | null;
+    presence_penalty?: number | null;
+    stop?: string | null | Array<string>;
+    n?: number | null;
+    logit_bias?: Record<string, number> | null;
+    logprobs?: boolean | null;
+    top_logprobs?: number | null;
+    response_format?: ResponseFormat | null;
+    enable_thinking?: boolean | null;
+    enable_latency_breakdown?: boolean | null;
+}
+export declare function postInitAndCheckGenerationConfigValues(config: GenerationConfig): void;
+export declare enum ModelType {
+    "LLM" = 0,
+    "embedding" = 1,
+    "VLM" = 2
+}
+/**
+ * Information for a model.
+ * @param model: the huggingface link to download the model weights, accepting four formats:
+ *    - https://huggingface.co/{USERNAME}/{MODEL}, which we automatically use the main branch
+ *    - https://huggingface.co/{USERNAME}/{MODEL}/, which we automatically use the main branch
+ *    - https://huggingface.co/{USERNAME}/{MODEL}/resolve/{BRANCH}
+ *    - https://huggingface.co/{USERNAME}/{MODEL}/resolve/{BRANCH}/
+ * @param model_id: what we call the model.
+ * @param model_lib: link to the model library (wasm file) the model uses.
+ * @param overrides: partial ChatConfig to override mlc-chat-config.json; can be used to change KVCache settings.
+ * @param vram_required_MB: amount of vram in MB required to run the model (can use
+ *    `utils/vram_requirements` to calculate).
+ * @param low_resource_required: whether the model can run on limited devices (e.g. Android phone).
+ * @param buffer_size_required_bytes: required `maxStorageBufferBindingSize`, different for each device.
+ * @param required_features: feature needed to run this model (e.g. shader-f16).
+ * @param model_type: the intended usecase for the model, if unspecified, default to LLM.
+ * @param integrity: optional SRI hashes to verify downloaded artifacts. See {@link ModelIntegrity}.
+ */
+export interface ModelRecord {
+    model: string;
+    model_id: string;
+    model_lib: string;
+    overrides?: ChatOptions;
+    vram_required_MB?: number;
+    low_resource_required?: boolean;
+    buffer_size_required_bytes?: number;
+    required_features?: Array<string>;
+    model_type?: ModelType;
+    integrity?: ModelIntegrity;
+}
+/**
+ * Extra configuration that can be
+ * passed to the load.
+ *
+ * @param model_list: models to be used.
+ * @param cacheBackend: the backend to use for caching models and other artifacts.
+ * If unspecified, will use the Cache API. For more information, see:
+ * https://developer.mozilla.org/en-US/docs/Web/API/Storage_API/Storage_quotas_and_eviction_criteria#what_technologies_store_data_in_the_browser
+ * Supported values are:
+ * - "cache": browser Cache API.
+ * - "indexeddb": IndexedDB-backed cache.
+ * - "cross-origin": Chrome Cross-Origin Storage extension-backed cache.
+ *
+ * @note Note that the Cache API is more well-tested in WebLLM as of now.
+ */
+export type CacheBackend = "cache" | "indexeddb" | "cross-origin";
+export interface AppConfig {
+    model_list: Array<ModelRecord>;
+    cacheBackend?: CacheBackend;
+}
+export declare function getCacheBackend(appConfig: AppConfig): CacheBackend;
+/**
+ * modelVersion: the prebuilt model libraries that the current npm is compatible with, affects the
+ * `model_lib`s in `prebuiltAppConfig`.
+ *
+ * @note The model version does not have to match the npm version, since not each npm update
+ * requires an update of the model libraries.
+ */
+export declare const modelVersion = "v0_2_80";
+export declare const modelLibURLPrefix = "https://raw.githubusercontent.com/mlc-ai/binary-mlc-llm-libs/main/web-llm-models/";
+/**
+ * Models that support function calling (i.e. usage of `ChatCompletionRequest.tools`). More to come.
+ */
+export declare const functionCallingModelIds: string[];
+/**
+ * Default models and model library mapping to be used if unspecified.
+ *
+ * @note This is the only source of truth of which prebuilt model libraries are compatible with the
+ * current WebLLM npm version.
+ */
+export declare const prebuiltAppConfig: AppConfig;
+//# sourceMappingURL=config.d.ts.map

package/lib/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAQzE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,cAAc,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAC/C,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9B,6BAA6B,CAAC,EAAE,OAAO,CAAC;CACzC;AAED,oBAAY,IAAI;IACd,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,IAAI,SAAS;CACd;AAED,eAAO,MAAM,eAAe,EAAE,QAAiB,CAAC;AAEhD;;;;;;;GAOG;AACH,oBAAY,mBAAmB;IAC7B,MAAM,qBAAqB;IAC3B,IAAI,mBAAmB;IACvB,SAAS,wBAAwB;IACjC,IAAI,mBAAmB;IACvB,QAAQ,sBAAsB;IAC9B,YAAY,mBAAmB;CAChC;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uBAAuB,EAAE,OAAO,CAAC;IACjC,qBAAqB,EAAE,OAAO,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,UAAU;IAEzB,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B,cAAc,CAAC,EAAE,aAAa,CAAC;IAC/B,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC1C,aAAa,EAAE,kBAAkB,CAAC;IAElC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAG5B,kBAAkB,EAAE,MAAM,CAAC;IAE3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AAEH,MAAM,WAAW,WAAY,SAAQ,OAAO,CAAC,UAAU,CAAC;CAAG;AAE3D;;;;;;;;;GASG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,sBAAsB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACrD,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB;IAE/B,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE5B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,eAAe,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IAExC,eAAe,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IACjC,wBAAwB,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;CAC3C;AAED,wBAAgB,sCAAsC,CACpD,MAAM,EAAE,gBAAgB,GACvB,IAAI,CAgFN;AAED,oBAAY,SAAS;IACnB,KAAK,IAAA;IACL,WAAW,IAAA;IACX,KAAK,IAAA;CACN;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,WAAW,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,iBAAiB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAClC,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,SAAS,CAAC,EAAE,cAAc,CAAC;CAC5B;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,WAAW,GAAG,cAAc,CAAC;AAElE,MAAM,WAAW,SAAS;IACxB,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED,wBAAgB,eAAe,CAAC,SAAS,EAAE,SAAS,GAAG,YAAY,CAKlE;AAED;;;;;;GAMG;AACH,eAAO,MAAM,YAAY,YAAY,CAAC;AACtC,eAAO,MAAM,iBAAiB,sFACuD,CAAC;AAEtF;;GAEG;AACH,eAAO,MAAM,uBAAuB,UAMnC,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,iBAAiB,EAAE,SA09D/B,CAAC"}

package/lib/conversation.d.ts

@@ -0,0 +1,107 @@
+import { ChatConfig, ConvTemplateConfig, Role } from "./config";
+import { ChatCompletionContentPart, ChatCompletionContentPartImage, ChatCompletionRequest } from "./openai_api_protocols/index";
+type ImageURL = ChatCompletionContentPartImage.ImageURL;
+/**
+ * Helper to keep track of history conversations.
+ */
+export declare class Conversation {
+    /** Each message is a tuple of (Role, role_name_str, message), where message can be either a
+     *  string or an array of contentPart for possible image input.
+     */
+    messages: Array<[
+        Role,
+        string,
+        string | Array<ChatCompletionContentPart> | undefined
+    ]>;
+    readonly config: ConvTemplateConfig;
+    /** Whether the Conversation object is for text completion with no conversation-style formatting */
+    isTextCompletion: boolean;
+    /** Used when isTextCompletion is true */
+    prompt: string | undefined;
+    function_string: string;
+    use_function_calling: boolean;
+    override_system_message?: string;
+    /**
+     * Tracks whether the last message is an empty thinking block. Should only
+     * be true when we are in the middle of a generation. Will be set to
+     * false when the reply is finished with `finishReply()`.
+     */
+    private isLastMessageEmptyThinkingReplyHeader;
+    constructor(config: ConvTemplateConfig, isTextCompletion?: boolean);
+    private getPromptArrayInternal;
+    /**
+     * Get prompt arrays with the first one as system.
+     *
+     * It is returned as an array of `string | Array<string | ImageURL>`, where each element of
+     * the array represents the formatted message of a role/turn. If the message only contains text,
+     * it will be a string that concatenates the role string, message, and separators. If the
+     * message contains image(s), it will be an array of string and ImageURL in the order of which
+     * they will be prefilled into the model. e.g. it can be something like
+     * [
+     *   "<|system|>\nSome system prompt\n",
+     *   [
+     *     "<|user|>\n",
+     *     imageURL1,
+     *     "\n",
+     *     imageURL2,
+     *     "\n",
+     *     "Some user input<|end|>\n"
+     *   ],
+     * ]
+     *
+     * @returns The prompt array.
+     */
+    getPromptArray(): Array<string | Array<string | ImageURL>>;
+    /**
+     * Get the last round of prompt has not been fed as input.
+     *
+     * @note This function needs to be used with the assumption that
+     *       the caller call appendMessage then appendReplyHeader.
+     *
+     * @returns The prompt array.
+     */
+    getPromptArrayLastRound(): (string | (string | ChatCompletionContentPartImage.ImageURL)[])[];
+    /**
+     * Return prompt in an array for non-conversation text completion.
+     */
+    getPromptArrayTextCompletion(): Array<string>;
+    /**
+     * Resets all states for this.conversation.
+     */
+    reset(): void;
+    getStopStr(): string[];
+    getStopTokens(): number[];
+    appendMessage(role: Role, message: string | Array<ChatCompletionContentPart>, role_name?: string): void;
+    appendReplyHeader(role: Role): void;
+    appendEmptyThinkingReplyHeader(role: Role, emptyThinkingBlockStr: string): void;
+    finishReply(message: string): void;
+}
+export declare function getConversation(conv_template: ConvTemplateConfig, conv_config?: Partial<ConvTemplateConfig>, isTextCompletion?: boolean): Conversation;
+/**
+ * Compare the states of two conversation instances. Equality is defined as their getPromptArray()
+ * should return the exact same things, which is determined by fields: messages, function_string,
+ * use_function_calling, and override_system_message.
+ *
+ * @returns True if `convA` equals to `convB`
+ * @note We assume convA and convB has the same `this.config`.
+ */
+export declare function compareConversationObject(convA: Conversation, convB: Conversation): boolean;
+/**
+ * Get a new Conversation object based on the chat completion request.
+ *
+ * @param request The incoming ChatCompletionRequest
+ * @param includeLastMsg Include last message, by default is false. Set to true for testing only.
+ * @note By default, `request.messages[-1]` is not included as it would be treated as a normal
+ * input to `prefill()`.
+ */
+export declare function getConversationFromChatCompletionRequest(request: ChatCompletionRequest, config: ChatConfig, includeLastMsg?: boolean): Conversation;
+/**
+ * Returns the function string based on the request.tools and request.tool_choice, raises erros if
+ * encounter invalid request.
+ *
+ * @param request The chatCompletionRequest we are about to prefill for.
+ * @returns The string used to set Conversation.function_string
+ */
+export declare function getFunctionCallUsage(request: ChatCompletionRequest): string;
+export {};
+//# sourceMappingURL=conversation.d.ts.map

package/lib/conversation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"conversation.d.ts","sourceRoot":"","sources":["../src/conversation.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,kBAAkB,EAElB,IAAI,EACL,MAAM,UAAU,CAAC;AAClB,OAAO,EACL,yBAAyB,EACzB,8BAA8B,EAE9B,qBAAqB,EACtB,MAAM,8BAA8B,CAAC;AAetC,KAAK,QAAQ,GAAG,8BAA8B,CAAC,QAAQ,CAAC;AAExD;;GAEG;AACH,qBAAa,YAAY;IAEvB;;OAEG;IACI,QAAQ,EAAE,KAAK,CACpB;QAAC,IAAI;QAAE,MAAM;QAAE,MAAM,GAAG,KAAK,CAAC,yBAAyB,CAAC,GAAG,SAAS;KAAC,CACtE,CAAM;IACP,QAAQ,CAAC,MAAM,EAAE,kBAAkB,CAAC;IAEpC,mGAAmG;IAC5F,gBAAgB,EAAE,OAAO,CAAC;IACjC,yCAAyC;IAClC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAE3B,eAAe,SAAM;IACrB,oBAAoB,UAAS;IAC7B,uBAAuB,CAAC,EAAE,MAAM,CAAa;IAEpD;;;;OAIG;IACH,OAAO,CAAC,qCAAqC,CAAS;gBAK1C,MAAM,EAAE,kBAAkB,EAAE,gBAAgB,UAAQ;IAMhE,OAAO,CAAC,sBAAsB;IAmJ9B;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,cAAc,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC;IAO1D;;;;;;;OAOG;IACH,uBAAuB;IAUvB;;OAEG;IACH,4BAA4B,IAAI,KAAK,CAAC,MAAM,CAAC;IAO7C;;OAEG;IACH,KAAK;IAUL,UAAU,IAAI,MAAM,EAAE;IAStB,aAAa;IAIb,aAAa,CACX,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,MAAM,GAAG,KAAK,CAAC,yBAAyB,CAAC,EAClD,SAAS,CAAC,EAAE,MAAM;IAkBpB,iBAAiB,CAAC,IAAI,EAAE,IAAI;IAU5B,8BAA8B,CAAC,IAAI,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM;IAUxE,WAAW,CAAC,OAAO,EAAE,MAAM;CAkB5B;AAED,wBAAgB,eAAe,CAC7B,aAAa,EAAE,kBAAkB,EACjC,WAAW,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACzC,gBAAgB,UAAQ,GACvB,YAAY,CAMd;AAED;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,YAAY,EACnB,KAAK,EAAE,YAAY,GAClB,OAAO,CA0ET;AAED;;;;;;;GAOG;AACH,wBAAgB,wCAAwC,CACtD,OAAO,EAAE,qBAAqB,EAC9B,MAAM,EAAE,UAAU,EAClB,cAAc,UAAQ,GACrB,YAAY,CA8Cd;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,qBAAqB,GAAG,MAAM,CAwC3E"}

package/lib/embedding.d.ts

@@ -0,0 +1,38 @@
+import * as tvmjs from "@mlc-ai/web-runtime";
+import { Tokenizer } from "@mlc-ai/web-tokenizers";
+import { ChatConfig } from "./config";
+export declare class EmbeddingPipeline {
+    private config;
+    private tokenizer;
+    private tvm;
+    private device;
+    private vm;
+    private prefill;
+    private params;
+    private contextWindowSize;
+    private prefillChunkSize;
+    private maxBatchSize;
+    private curRoundEmbedTotalTokens;
+    private curRoundEmbedTotalTime;
+    constructor(tvm: tvmjs.Instance, tokenizer: Tokenizer, config: ChatConfig);
+    embedStep(input: string | Array<string> | Array<number> | Array<Array<number>>): Promise<Array<Array<number>>>;
+    dispose(): void;
+    /**
+     * Synchronize the device.
+     */
+    sync(): Promise<void>;
+    asyncLoadWebGPUPipelines(): Promise<void>;
+    /**
+     * Get the time it took the last `embedStep()` in seconds.
+     */
+    getCurRoundEmbedTotalTime(): number;
+    /**
+     * Get the number of tokens embedded in the last `embedStep()`. This excludes the padded tokens.
+     */
+    getCurRoundEmbedTotalTokens(): number;
+    /**
+     * @returns Prefill tokens per second, starting from the last prefill performed.
+     */
+    getCurRoundEmbedTokensPerSec(): number;
+}
+//# sourceMappingURL=embedding.d.ts.map

package/lib/embedding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../src/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAC;AAE7C,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAStC,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,SAAS,CAAY;IAG7B,OAAO,CAAC,GAAG,CAAiB;IAC5B,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,EAAE,CAAuB;IACjC,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,MAAM,CAAkB;IAGhC,OAAO,CAAC,iBAAiB,CAAM;IAC/B,OAAO,CAAC,gBAAgB,CAAM;IAC9B,OAAO,CAAC,YAAY,CAAM;IAG1B,OAAO,CAAC,wBAAwB,CAAK;IACrC,OAAO,CAAC,sBAAsB,CAAK;gBAEvB,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU;IA8DnE,SAAS,CACb,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GACnE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IA2JhC,OAAO;IAQP;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAKrB,wBAAwB;IAM9B;;OAEG;IACH,yBAAyB,IAAI,MAAM;IAInC;;OAEG;IACH,2BAA2B,IAAI,MAAM;IAIrC;;OAEG;IACH,4BAA4B,IAAI,MAAM;CAGvC"}

package/lib/engine.d.ts

@@ -0,0 +1,140 @@
+import { ChatConfig, ChatOptions, AppConfig, GenerationConfig, MLCEngineConfig } from "./config";
+import { LLMChatPipeline } from "./llm_chat";
+import { ChatCompletionRequest, ChatCompletion, ChatCompletionChunk, ChatCompletionRequestNonStreaming, ChatCompletionRequestStreaming, ChatCompletionRequestBase, CompletionCreateParamsNonStreaming, CompletionCreateParamsStreaming, CompletionCreateParamsBase, CompletionCreateParams, Completion, EmbeddingCreateParams, CreateEmbeddingResponse } from "./openai_api_protocols/index";
+import * as API from "./openai_api_protocols/index";
+import { InitProgressCallback, MLCEngineInterface, LogitProcessor, LogLevel } from "./types";
+/**
+ * Creates `MLCEngine`, and loads `modelId` onto WebGPU.
+ *
+ * Equivalent to `new webllm.MLCEngine().reload(...)`.
+ *
+ * @param modelId model_id of the model to load, either string or string[]. When multiple models
+ *   are provided, we load all models sequentially. Each modelId needs to either be in
+ *   `webllm.prebuiltAppConfig`, or in `engineCOnfig.appConfig`.
+ * @param engineConfig Optionally configures the engine, see `webllm.MLCEngineConfig`.
+ * @param chatOpts Extra options to optionally override the `mlc-chat-config.json` of `modelId`.
+ *   The size of which needs to match that of `modelId`; chatOpts[i] will be used for modelId[i].
+ * @returns An initialized `WebLLM.MLCEngine` with `modelId` loaded.
+ * @throws Throws error when device lost (mostly due to OOM); users should re-call `CreateMLCEngine()`,
+ *   potentially with a smaller model or smaller context window size.
+ */
+export declare function CreateMLCEngine(modelId: string | string[], engineConfig?: MLCEngineConfig, chatOpts?: ChatOptions | ChatOptions[]): Promise<MLCEngine>;
+/**
+ * The main interface of MLCEngine, which loads a model and performs tasks.
+ *
+ * You can either initialize one with `webllm.CreateMLCEngine(modelId)`, or
+ * `webllm.MLCEngine().reload(modelId)`.
+ */
+export declare class MLCEngine implements MLCEngineInterface {
+    /** For chat.completions.create() */
+    chat: API.Chat;
+    /** For completions.create() */
+    completions: API.Completions;
+    /** For embeddings.create() */
+    embeddings: API.Embeddings;
+    /** Maps each loaded model's modelId to its pipeline */
+    private loadedModelIdToPipeline;
+    /** Maps each loaded model's modelId to its chatConfig */
+    private loadedModelIdToChatConfig;
+    /** Maps each loaded model's modelId to its modelType */
+    private loadedModelIdToModelType;
+    /** Maps each loaded model's modelId to a lock. Ensures
+     * each model only processes one request at at time.
+     */
+    private loadedModelIdToLock;
+    private logger;
+    private logitProcessorRegistry?;
+    private initProgressCallback?;
+    private appConfig;
+    private interruptSignal;
+    private deviceLostIsError;
+    private reloadController;
+    constructor(engineConfig?: MLCEngineConfig);
+    setAppConfig(appConfig: AppConfig): void;
+    setInitProgressCallback(initProgressCallback?: InitProgressCallback): void;
+    getInitProgressCallback(): InitProgressCallback | undefined;
+    setLogitProcessorRegistry(logitProcessorRegistry?: Map<string, LogitProcessor>): void;
+    /**
+     * Set MLCEngine logging output level
+     *
+     * @param logLevel The new log level
+     */
+    setLogLevel(logLevel: LogLevel): void;
+    reload(modelId: string | string[], chatOpts?: ChatOptions | ChatOptions[]): Promise<void>;
+    private reloadInternal;
+    unload(): Promise<void>;
+    private _generate;
+    /**
+     * Similar to `_generate()`; but instead of using callback, we use an async iterable.
+     */
+    asyncGenerate(request: ChatCompletionRequestStreaming, model: string, pipeline: LLMChatPipeline, chatConfig: ChatConfig, genConfig: GenerationConfig, timeReceived: number): AsyncGenerator<ChatCompletionChunk, void, void>;
+    asyncGenerate(request: CompletionCreateParamsStreaming, model: string, pipeline: LLMChatPipeline, chatConfig: ChatConfig, genConfig: GenerationConfig, timeReceived: number): AsyncGenerator<Completion, void, void>;
+    interruptGenerate(): Promise<void>;
+    /**
+     * Completes a single ChatCompletionRequest.
+     *
+     * @param request A OpenAI-style ChatCompletion request.
+     *
+     * @note For each choice (i.e. `n`), a request is defined by a single `prefill()` and multiple
+     * `decode()`. This is important as it determines the behavior of various fields including `seed`.
+     */
+    chatCompletion(request: ChatCompletionRequestNonStreaming): Promise<ChatCompletion>;
+    chatCompletion(request: ChatCompletionRequestStreaming): Promise<AsyncIterable<ChatCompletionChunk>>;
+    chatCompletion(request: ChatCompletionRequestBase): Promise<AsyncIterable<ChatCompletionChunk> | ChatCompletion>;
+    /**
+     * Completes a single CompletionCreateParams, a text completion with no chat template.
+     *
+     * @param request A OpenAI-style Completion request.
+     *
+     * @note For each choice (i.e. `n`), a request is defined by a single `prefill()` and multiple
+     * `decode()`. This is important as it determines the behavior of various fields including `seed`.
+     */
+    completion(request: CompletionCreateParamsNonStreaming): Promise<Completion>;
+    completion(request: CompletionCreateParamsStreaming): Promise<AsyncIterable<Completion>>;
+    completion(request: CompletionCreateParamsBase): Promise<AsyncIterable<Completion> | Completion>;
+    embedding(request: EmbeddingCreateParams): Promise<CreateEmbeddingResponse>;
+    getMaxStorageBufferBindingSize(): Promise<number>;
+    getGPUVendor(): Promise<string>;
+    private getLLMStates;
+    private getEmbeddingStates;
+    /**
+     * Return the model, its LLMChatPipeline, and ChatConfig to use. Throws error when unclear which
+     * model to load. Ensure all loadedModelIdToXXX maps contain entry for the selected modelId.
+     * @param requestName The type of request or API to load the model for. Needed for error throwing.
+     * @param modelType The typ of model, determining what type of pipeline to expect.
+     * @param modelId Model the user specified to load via the request. Required when multiple
+     *   models are loaded
+     */
+    private getModelStates;
+    forwardTokensAndSample(inputIds: Array<number>, isPrefill: boolean, modelId?: string): Promise<number>;
+    /**
+     * Get the current generated response.
+     *
+     * @returns The current output message.
+     */
+    getMessage(modelId?: string): Promise<string>;
+    runtimeStatsText(modelId?: string): Promise<string>;
+    resetChat(keepStats?: boolean, modelId?: string): Promise<void>;
+    /**
+     * Run a prefill step with a given input.
+     *
+     * If `input` is a chatCompletionRequest, we treat `input.messages[-1]` as the usual user input.
+     * We then convert `input.messages[:-1]` to a `Conversation` object, representing a conversation
+     * history.
+     *
+     * If the new `Conversation` object matches the current one loaded, it means we are
+     * performing multi-round chatting, so we do not reset, hence reusing KV cache. Otherwise, we
+     * reset every thing, treating the request as something completely new.
+     *
+     * @param input The OpenAI-style prompt to prefill.
+     * @param pipeline The loaded pipeline, hence model, to carry out this prefill.
+     * @param chatConfig The chat config to use for this model.
+     * @param genConfig Generation config.
+     */
+    prefill(input: ChatCompletionRequest | CompletionCreateParams, pipeline: LLMChatPipeline, chatConfig: ChatConfig, genConfig: GenerationConfig): Promise<void>;
+    /**
+     * Run a decode step to decode the next token.
+     */
+    decode(pipeline: LLMChatPipeline, genConfig?: GenerationConfig): Promise<void>;
+}
+//# sourceMappingURL=engine.d.ts.map

package/lib/engine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../src/engine.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,UAAU,EACV,WAAW,EACX,SAAS,EAGT,gBAAgB,EAGhB,eAAe,EAGhB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAEL,qBAAqB,EACrB,cAAc,EACd,mBAAmB,EAEnB,iCAAiC,EACjC,8BAA8B,EAC9B,yBAAyB,EAIzB,kCAAkC,EAClC,+BAA+B,EAC/B,0BAA0B,EAC1B,sBAAsB,EACtB,UAAU,EAEV,qBAAqB,EACrB,uBAAuB,EAExB,MAAM,8BAA8B,CAAC;AACtC,OAAO,KAAK,GAAG,MAAM,8BAA8B,CAAC;AACpD,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAET,MAAM,SAAS,CAAC;AA+BjB;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,YAAY,CAAC,EAAE,eAAe,EAC9B,QAAQ,CAAC,EAAE,WAAW,GAAG,WAAW,EAAE,GACrC,OAAO,CAAC,SAAS,CAAC,CAIpB;AAED;;;;;GAKG;AACH,qBAAa,SAAU,YAAW,kBAAkB;IAElD,oCAAoC;IAC7B,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC;IACtB,+BAA+B;IACxB,WAAW,EAAE,GAAG,CAAC,WAAW,CAAC;IACpC,8BAA8B;IACvB,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC;IAGlC,uDAAuD;IACvD,OAAO,CAAC,uBAAuB,CAG7B;IACF,yDAAyD;IACzD,OAAO,CAAC,yBAAyB,CAA0B;IAC3D,wDAAwD;IACxD,OAAO,CAAC,wBAAwB,CAAyB;IACzD;;OAEG;IACH,OAAO,CAAC,mBAAmB,CAA0B;IAGrD,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,sBAAsB,CAAC,CAA8B;IAC7D,OAAO,CAAC,oBAAoB,CAAC,CAAuB;IACpD,OAAO,CAAC,SAAS,CAAY;IAG7B,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,iBAAiB,CAAQ;IACjC,OAAO,CAAC,gBAAgB,CAA8B;gBAE1C,YAAY,CAAC,EAAE,eAAe;IAsB1C,YAAY,CAAC,SAAS,EAAE,SAAS;IAIjC,uBAAuB,CAAC,oBAAoB,CAAC,EAAE,oBAAoB;IAInE,uBAAuB;IAIvB,yBAAyB,CACvB,sBAAsB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC;IAKtD;;;;OAIG;IACH,WAAW,CAAC,QAAQ,EAAE,QAAQ;IAQxB,MAAM,CACV,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,QAAQ,CAAC,EAAE,WAAW,GAAG,WAAW,EAAE,GACrC,OAAO,CAAC,IAAI,CAAC;YA0CF,cAAc;IAyLtB,MAAM;YAyBE,SAAS;IAwBvB;;OAEG;IACH,aAAa,CACX,OAAO,EAAE,8BAA8B,EACvC,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,gBAAgB,EAC3B,YAAY,EAAE,MAAM,GACnB,cAAc,CAAC,mBAAmB,EAAE,IAAI,EAAE,IAAI,CAAC;IAClD,aAAa,CACX,OAAO,EAAE,+BAA+B,EACxC,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,gBAAgB,EAC3B,YAAY,EAAE,MAAM,GACnB,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC;IAgRnC,iBAAiB;IAQvB;;;;;;;OAOG;IACG,cAAc,CAClB,OAAO,EAAE,iCAAiC,GACzC,OAAO,CAAC,cAAc,CAAC;IACpB,cAAc,CAClB,OAAO,EAAE,8BAA8B,GACtC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;IACxC,cAAc,CAClB,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,GAAG,cAAc,CAAC;IA4K/D;;;;;;;OAOG;IACG,UAAU,CACd,OAAO,EAAE,kCAAkC,GAC1C,OAAO,CAAC,UAAU,CAAC;IAChB,UAAU,CACd,OAAO,EAAE,+BAA+B,GACvC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IAC/B,UAAU,CACd,OAAO,EAAE,0BAA0B,GAClC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC;IAyH5C,SAAS,CACb,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC,uBAAuB,CAAC;IAkD7B,8BAA8B,IAAI,OAAO,CAAC,MAAM,CAAC;IA6BjD,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAcrC,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,kBAAkB;IAW1B;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAgEhB,sBAAsB,CAC1B,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,EACvB,SAAS,EAAE,OAAO,EAClB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC;IAQlB;;;;OAIG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK7C,gBAAgB,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWnD,SAAS,CAAC,SAAS,UAAQ,EAAE,OAAO,CAAC,EAAE,MAAM;IAwBnD;;;;;;;;;;;;;;;OAeG;IACG,OAAO,CACX,KAAK,EAAE,qBAAqB,GAAG,sBAAsB,EACrD,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,gBAAgB;IAwD7B;;OAEG;IACG,MAAM,CAAC,QAAQ,EAAE,eAAe,EAAE,SAAS,CAAC,EAAE,gBAAgB;CAGrE"}