web-llm-runner 0.1.1

package/lib/openai_api_protocols/chat_completion.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chat_completion.d.ts","sourceRoot":"","sources":["../../src/openai_api_protocols/chat_completion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAChE,OAAO,EAGL,SAAS,EACV,MAAM,WAAW,CAAC;AAuBnB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAI9D,qBAAa,IAAI;IACf,OAAO,CAAC,MAAM,CAAqB;IACnC,WAAW,EAAE,WAAW,CAAC;gBAEb,MAAM,EAAE,kBAAkB;CAIvC;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,MAAM,CAAqB;gBAEvB,MAAM,EAAE,kBAAkB;IAItC,MAAM,CAAC,OAAO,EAAE,iCAAiC,GAAG,OAAO,CAAC,cAAc,CAAC;IAC3E,MAAM,CACJ,OAAO,EAAE,8BAA8B,GACtC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;IAC9C,MAAM,CACJ,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,GAAG,cAAc,CAAC;CAMhE;AAID;;;;;;;GAOG;AACH,MAAM,WAAW,yBAAyB;IACxC;;OAEG;IACH,QAAQ,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAE5C;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAExB;;OAEG;IACH,cAAc,CAAC,EAAE,2BAA2B,GAAG,IAAI,CAAC;IAEpD;;OAEG;IACH,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElB;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEjC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEnC;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE3B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAErC;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE5B;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IAE3C;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAE1B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B;;;;;;;OAOG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAErB;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,8BAA8B,CAAC;IAE7C;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAElC;;;;;;;;;;;;;OAaG;IACH,eAAe,CAAC,EAAE,cAAc,CAAC;IAEjC;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,UAAU,CAAC,EAAE;QACX;;;;;WAKG;QACH,eAAe,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;QAEjC;;;WAGG;QACH,wBAAwB,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;KAC3C,CAAC;CACH;AAED,MAAM,WAAW,iCACf,SAAQ,yBAAyB;IACjC;;OAEG;IACH,MAAM,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,8BACf,SAAQ,yBAAyB;IACjC;;OAEG;IACH,MAAM,EAAE,IAAI,CAAC;CACd;AAED,MAAM,MAAM,qBAAqB,GAC7B,iCAAiC,GACjC,8BAA8B,CAAC;AAEnC;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAEtC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,MAAM,EAAE,iBAAiB,CAAC;IAE1B;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,eAAe,CAAC;IAExB;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAE3C;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,MAAM,EAAE,uBAAuB,CAAC;IAEhC;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;OAKG;IACH,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,eAAO,MAAM,sCAAsC,EAAE,KAAK,CAAC,MAAM,CAAM,CAAC;AAExE;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,qBAAqB,EAC9B,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,SAAS,GAC1B,IAAI,CAmLN;AAQD,MAAM,MAAM,yBAAyB,GACjC,6BAA6B,GAC7B,8BAA8B,CAAC;AAEnC,MAAM,WAAW,6BAA6B;IAC5C;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;CACd;AAED,yBAAiB,8BAA8B,CAAC;IAC9C,UAAiB,QAAQ;QACvB;;WAEG;QACH,GAAG,EAAE,MAAM,CAAC;QAEZ;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;KAClC;CACF;AAED,MAAM,WAAW,8BAA8B;IAC7C,SAAS,EAAE,8BAA8B,CAAC,QAAQ,CAAC;IACnD;;OAEG;IACH,IAAI,EAAE,WAAW,CAAC;CACnB;AAID,MAAM,WAAW,6BAA6B;IAC5C;;;OAGG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,QAAQ,EAAE,6BAA6B,CAAC,QAAQ,CAAC;IAEjD;;OAEG;IACH,IAAI,EAAE,UAAU,CAAC;CAClB;AAED,yBAAiB,6BAA6B,CAAC;IAC7C;;OAEG;IACH,UAAiB,QAAQ;QACvB;;;WAGG;QACH,SAAS,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;KACd;CACF;AAID;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B,QAAQ,GACR,MAAM,GACN,WAAW,GACX,MAAM,GACN,UAAU,CAAC;AAEf;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC1C;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,gCAAgC;IAC/C;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;CAChB;AAED,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAEnD;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;;;;OAKG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mCAAmC;IAClD;;OAEG;IACH,IAAI,EAAE,WAAW,CAAC;IAElB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAExB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,UAAU,CAAC,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,0BAA0B,GAClC,gCAAgC,GAChC,8BAA8B,GAC9B,mCAAmC,GACnC,8BAA8B,CAAC;AAInC;;;;;;;;GAQG;AACH,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEzD,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,kBAAkB,CAAC;CACjC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,kBAAkB,CAAC;IAE7B;;OAEG;IACH,IAAI,EAAE,UAAU,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,EAAE,6BAA6B,CAAC,QAAQ,CAAC;IAEjD;;OAEG;IACH,IAAI,EAAE,UAAU,CAAC;CAClB;AAED,yBAAiB,6BAA6B,CAAC;IAC7C,UAAiB,QAAQ;QACvB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;KACd;CACF;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,8BAA8B,GACtC,MAAM,GACN,MAAM,GACN,6BAA6B,CAAC;AAKlC,MAAM,WAAW,UAAU;IACzB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;;;;;;OAQG;IACH,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IAE5B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,0BAA0B;IACzC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;;;;;;OAQG;IACH,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IAE5B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;CACjC;AAGD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAEvB;;OAEG;IACH,IAAI,EAAE,WAAW,CAAC;IAElB;;OAEG;IACH,UAAU,CAAC,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAC;CACnD;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,aAAa,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,KAAK,EAAE;QACL;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,oBAAoB,EAAE,MAAM,CAAC;QAE7B;;WAEG;QACH,mBAAmB,EAAE,MAAM,CAAC;QAE5B;;;WAGG;QACH,qBAAqB,EAAE,MAAM,CAAC;QAE9B;;;WAGG;QACH,uBAAuB,EAAE,MAAM,CAAC;QAEhC;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;;WAGG;QACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAE7B;;;WAGG;QACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;KACrC,CAAC;CACH;AAED;;;;;;GAMG;AACH,MAAM,MAAM,0BAA0B,GAClC,MAAM,GACN,QAAQ,GACR,YAAY,GACZ,OAAO,CAAC;AAEZ,yBAAiB,cAAc,CAAC;IAC9B,UAAiB,MAAM;QACrB;;;;;WAKG;QACH,aAAa,EAAE,0BAA0B,CAAC;QAE1C;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,QAAQ,EAAE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QAEjC;;WAEG;QACH,OAAO,EAAE,qBAAqB,CAAC;KAChC;IAED,UAAiB,MAAM,CAAC;QACtB;;WAEG;QACH,UAAiB,QAAQ;YACvB;;eAEG;YACH,OAAO,EAAE,KAAK,CAAC,0BAA0B,CAAC,GAAG,IAAI,CAAC;SACnD;KACF;CACF;AAED,yBAAiB,mBAAmB,CAAC;IACnC,UAAiB,MAAM;QACrB;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;QAEpB;;;;;WAKG;QACH,aAAa,EAAE,0BAA0B,GAAG,IAAI,CAAC;QAEjD;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;KACnC;IAED,UAAiB,MAAM,CAAC;QACtB;;WAEG;QACH,UAAiB,KAAK;YACpB;;eAEG;YACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;YAExB;;eAEG;YACH,IAAI,CAAC,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,MAAM,CAAC;YAEhD,UAAU,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;SACpC;QAED,UAAiB,KAAK,CAAC;YACrB,UAAiB,QAAQ;gBACvB;;mBAEG;gBACH,KAAK,EAAE,MAAM,CAAC;gBAEd;;mBAEG;gBACH,EAAE,CAAC,EAAE,MAAM,CAAC;gBAEZ,QAAQ,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC;gBAE7B;;mBAEG;gBACH,IAAI,CAAC,EAAE,UAAU,CAAC;aACnB;YAED,UAAiB,QAAQ,CAAC;gBACxB,UAAiB,QAAQ;oBACvB;;;;;uBAKG;oBACH,SAAS,CAAC,EAAE,MAAM,CAAC;oBAEnB;;uBAEG;oBACH,IAAI,CAAC,EAAE,MAAM,CAAC;iBACf;aACF;SACF;QAED;;WAEG;QACH,UAAiB,QAAQ;YACvB;;eAEG;YACH,OAAO,EAAE,KAAK,CAAC,0BAA0B,CAAC,GAAG,IAAI,CAAC;SACnD;KACF;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,GAAG,gBAAgB,CAAC;IAC7D;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,cAAc,CAAC,EAAE,iBAAiB,GAAG,MAAM,CAAC;CAC7C"}

package/lib/openai_api_protocols/completion.d.ts

@@ -0,0 +1,270 @@
+/**
+ * The input to OpenAI API, directly adopted from openai-node with small tweaks:
+ * https://github.com/openai/openai-node/blob/master/src/resources/completions.ts
+ *
+ * Copyright 2024 OpenAI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { MLCEngineInterface } from "../types";
+import { ChatCompletion, ChatCompletionStreamOptions, CompletionUsage, ChatCompletionFinishReason } from "./chat_completion";
+export declare class Completions {
+    private engine;
+    constructor(engine: MLCEngineInterface);
+    create(request: CompletionCreateParamsNonStreaming): Promise<Completion>;
+    create(request: CompletionCreateParamsStreaming): Promise<AsyncIterable<Completion>>;
+    create(request: CompletionCreateParamsBase): Promise<AsyncIterable<Completion> | Completion>;
+}
+/**
+ * OpenAI completion request protocol.
+ *
+ * API reference: https://platform.openai.com/docs/api-reference/completions/create
+ * Followed: https://github.com/openai/openai-node/blob/master/src/resources/completions.ts
+ *
+ * @note `model` is excluded. Instead, call `CreateMLCEngine(model)` or `engine.reload(model)` explicitly before calling this API.
+ */
+export interface CompletionCreateParamsBase {
+    /**
+     * The prompt(s) to generate completions for, encoded as a string.
+     */
+    prompt: string;
+    /**
+     * Echo back the prompt in addition to the completion
+     */
+    echo?: boolean | null;
+    /**
+     * Number between -2.0 and 2.0. Positive values penalize new tokens based on their
+     * existing frequency in the text so far, decreasing the model's likelihood to
+     * repeat the same line verbatim.
+     *
+     * [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation/parameter-details)
+     */
+    frequency_penalty?: number | null;
+    /**
+     * Modify the likelihood of specified tokens appearing in the completion.
+     *
+     * Accepts a JSON object that maps tokens (specified by their token ID, which varies per model)
+     * to an associated bias value from -100 to 100. Typically, you can see `tokenizer.json` of the
+     * model to see which token ID maps to what string. Mathematically, the bias is added to the
+     * logits generated by the model prior to sampling. The exact effect will vary per model, but
+     * values between -1 and 1 should decrease or increase likelihood of selection; values like -100
+     * or 100 should result in a ban or exclusive selection of the relevant token.
+     *
+     * As an example, you can pass `{"16230": -100}` to prevent the `Hello` token from being
+     * generated in Mistral-7B-Instruct-v0.2, according to the mapping in
+     * https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2/raw/main/tokenizer.json.
+     *
+     * @note For stateful and customizable / flexible logit processing, see `webllm.LogitProcessor`.
+     * @note If used in combination with `webllm.LogitProcessor`, `logit_bias` is applied after
+     * `LogitProcessor.processLogits()` is called.
+     */
+    logit_bias?: Record<string, number> | null;
+    /**
+     * Whether to return log probabilities of the output tokens or not.
+     *
+     * If true, returns the log probabilities of each output token returned in the `content` of
+     * `message`.
+     */
+    logprobs?: boolean | null;
+    /**
+     * An integer between 0 and 5 specifying the number of most likely tokens to return
+     * at each token position, each with an associated log probability. `logprobs` must
+     * be set to `true` if this parameter is used.
+     */
+    top_logprobs?: number | null;
+    /**
+     * The maximum number of [tokens](/tokenizer) that can be generated in the
+     * completion.
+     *
+     * The total length of input tokens and generated tokens is limited by the model's
+     * context length.
+     */
+    max_tokens?: number | null;
+    /**
+     * How many completions to generate for each prompt.
+     */
+    n?: number | null;
+    /**
+     * Number between -2.0 and 2.0. Positive values penalize new tokens based on
+     * whether they appear in the text so far, increasing the model's likelihood to
+     * talk about new topics.
+     *
+     * [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation/parameter-details)
+     */
+    presence_penalty?: number | null;
+    /**
+     * Penalizes new tokens based on whether they appear in the prompt and the
+     * generated text so far. Values greater than 1.0 encourage the model to use new
+     * tokens, while values less than 1.0 encourage the model to repeat tokens.
+     */
+    repetition_penalty?: number | null;
+    /**
+     * If specified, our system will make a best effort to sample deterministically,
+     * such that repeated requests with the same `seed` and parameters should return
+     * the same result.
+     *
+     * @note Seeding is done on a request-level rather than choice-level. That is, if `n > 1`, you
+     * would still get different content for each `Choice`. But if two requests with `n = 2` are
+     * processed with the same seed, the two results should be the same (two choices are different).
+     */
+    seed?: number | null;
+    /**
+     * Up to 4 sequences where the API will stop generating further tokens. The
+     * returned text will not contain the stop sequence.
+     */
+    stop?: string | null | Array<string>;
+    /**
+     * If set, partial deltas will be sent. It will be terminated by an empty chunk.
+     */
+    stream?: boolean | null;
+    /**
+     * Options for streaming response. Only set this when you set `stream: true`.
+     */
+    stream_options?: ChatCompletionStreamOptions | null;
+    /**
+     * What sampling temperature to use, between 0 and 2. Higher values like 0.8 will
+     * make the output more random, while lower values like 0.2 will make it more
+     * focused and deterministic.
+     *
+     * We generally recommend altering this or `top_p` but not both.
+     */
+    temperature?: number | null;
+    /**
+     * An alternative to sampling with temperature, called nucleus sampling, where the
+     * model considers the results of the tokens with top_p probability mass. So 0.1
+     * means only the tokens comprising the top 10% probability mass are considered.
+     *
+     * We generally recommend altering this or `temperature` but not both.
+     */
+    top_p?: number | null;
+    /**
+     * If true, will ignore stop string and stop token and generate until max_tokens hit.
+     * If unset, will treat as false.
+     */
+    ignore_eos?: boolean;
+    /**
+     * ID of the model to use. This equals to `ModelRecord.model_id`, which needs to either be in
+     * `webllm.prebuiltAppConfig` or in `engineConfig.appConfig`.
+     *
+     * @note Call `CreateMLCEngine(model)` or `engine.reload(model)` ahead of time.
+     * @note If only one model is loaded in the engine, this field is optional. If multiple models
+     *   are loaded, this is required.
+     */
+    model?: string | null;
+    /**
+     * The suffix that comes after a completion of inserted text.
+     *
+     * @note This field is not supported.
+     */
+    suffix?: string | null;
+    /**
+     * A unique identifier representing your end-user, which can help OpenAI to monitor
+     * and detect abuse.
+     *
+     * @note This field is not supported.
+     */
+    user?: string;
+    /**
+     * Generates `best_of` completions server-side and returns the "best" (the one with
+     * the highest log probability per token). Results cannot be streamed.
+     *
+     * When used with `n`, `best_of` controls the number of candidate completions and
+     * `n` specifies how many to return – `best_of` must be greater than `n`.
+     *
+     * @note This field is not supported.
+     */
+    best_of?: number | null;
+    /**
+     * Fields specific to WebLLM, not present in OpenAI.
+     */
+    extra_body?: {
+        /**
+         * If set to true, the response will include a breakdown of the time spent in various
+         * stages of token sampling.
+         */
+        enable_latency_breakdown?: boolean | null;
+    };
+}
+export type CompletionCreateParams = CompletionCreateParamsNonStreaming | CompletionCreateParamsStreaming;
+export interface CompletionCreateParamsNonStreaming extends CompletionCreateParamsBase {
+    /**
+     * If set, partial deltas will be sent. It will be terminated by an empty chunk.
+     */
+    stream?: false | null;
+}
+export interface CompletionCreateParamsStreaming extends CompletionCreateParamsBase {
+    /**
+     * If set, partial deltas will be sent. It will be terminated by an empty chunk.
+     */
+    stream: true;
+}
+/**
+ * Represents a completion response returned by model, based on the provided input.
+ */
+export interface Completion {
+    /**
+     * A unique identifier for the completion.
+     */
+    id: string;
+    /**
+     * The list of completion choices the model generated for the input prompt.
+     */
+    choices: Array<CompletionChoice>;
+    /**
+     * The Unix timestamp (in seconds) of when the completion was created.
+     */
+    created: number;
+    /**
+     * The model used for completion.
+     */
+    model: string;
+    /**
+     * The object type, which is always "text_completion"
+     */
+    object: "text_completion";
+    /**
+     * This fingerprint represents the backend configuration that the model runs with.
+     *
+     * Can be used in conjunction with the `seed` request parameter to understand when
+     * backend changes have been made that might impact determinism.
+     *
+     * @note Not supported yet.
+     */
+    system_fingerprint?: string;
+    /**
+     * Usage statistics for the completion request.
+     */
+    usage?: CompletionUsage;
+}
+export interface CompletionChoice {
+    /**
+     * The reason the model stopped generating tokens. This will be `stop` if the model
+     * hit a natural stop point or a provided stop sequence, or `length` if the maximum
+     * number of tokens specified in the request was reached.
+     */
+    finish_reason: ChatCompletionFinishReason | null;
+    index: number;
+    /**
+     * A list of message content tokens with log probability information.
+     * @note Different from openai-node, we reuse ChatCompletion's Logprobs.
+     */
+    logprobs?: ChatCompletion.Choice.Logprobs | null;
+    text: string;
+}
+export declare const CompletionCreateParamsUnsupportedFields: Array<string>;
+/**
+ * Post init and verify whether the input of the request is valid. Thus, this function can throw
+ * error or in-place update request.
+ * @param request User's input request.
+ * @param currentModelId The current model loaded that will perform this request.
+ */
+export declare function postInitAndCheckFields(request: CompletionCreateParams, currentModelId: string): void;
+//# sourceMappingURL=completion.d.ts.map

package/lib/openai_api_protocols/completion.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"completion.d.ts","sourceRoot":"","sources":["../../src/openai_api_protocols/completion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAO9C,OAAO,EACL,cAAc,EACd,2BAA2B,EAC3B,eAAe,EACf,0BAA0B,EAC3B,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,WAAW;IACtB,OAAO,CAAC,MAAM,CAAqB;gBAEvB,MAAM,EAAE,kBAAkB;IAItC,MAAM,CAAC,OAAO,EAAE,kCAAkC,GAAG,OAAO,CAAC,UAAU,CAAC;IACxE,MAAM,CACJ,OAAO,EAAE,+BAA+B,GACvC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,CACJ,OAAO,EAAE,0BAA0B,GAClC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC;CAMnD;AAGD;;;;;;;GAOG;AACH,MAAM,WAAW,0BAA0B;IACzC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAEtB;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IAE3C;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAE1B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE3B;;OAEG;IACH,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElB;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEjC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEnC;;;;;;;;OAQG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAErB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAErC;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAExB;;OAEG;IACH,cAAc,CAAC,EAAE,2BAA2B,GAAG,IAAI,CAAC;IAEpD;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE5B;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAItB;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEvB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAExB;;OAEG;IACH,UAAU,CAAC,EAAE;QACX;;;WAGG;QACH,wBAAwB,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;KAC3C,CAAC;CACH;AAED,MAAM,MAAM,sBAAsB,GAC9B,kCAAkC,GAClC,+BAA+B,CAAC;AAEpC,MAAM,WAAW,kCACf,SAAQ,0BAA0B;IAClC;;OAEG;IACH,MAAM,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,+BACf,SAAQ,0BAA0B;IAClC;;OAEG;IACH,MAAM,EAAE,IAAI,CAAC;CACd;AAGD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAEjC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,MAAM,EAAE,iBAAiB,CAAC;IAE1B;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;OAEG;IACH,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B;;;;OAIG;IACH,aAAa,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAEjD,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;IAEjD,IAAI,EAAE,MAAM,CAAC;CACd;AAID,eAAO,MAAM,uCAAuC,EAAE,KAAK,CAAC,MAAM,CAIjE,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,sBAAsB,EAE/B,cAAc,EAAE,MAAM,GACrB,IAAI,CA8BN"}

package/lib/openai_api_protocols/embedding.d.ts

@@ -0,0 +1,125 @@
+/**
+ * The input to OpenAI API, directly adopted from openai-node with small tweaks:
+ * https://github.com/openai/openai-node/blob/master/src/resources/embeddings.ts
+ *
+ * Copyright 2024 OpenAI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { MLCEngineInterface } from "../types";
+export declare class Embeddings {
+    private engine;
+    constructor(engine: MLCEngineInterface);
+    /**
+     * Creates an embedding vector representing the input text.
+     */
+    create(request: EmbeddingCreateParams): Promise<CreateEmbeddingResponse>;
+}
+export interface CreateEmbeddingResponse {
+    /**
+     * The list of embeddings generated by the model.
+     */
+    data: Array<Embedding>;
+    /**
+     * The name of the model used to generate the embedding.
+     */
+    model: string;
+    /**
+     * The object type, which is always "list".
+     */
+    object: "list";
+    /**
+     * The usage information for the request.
+     */
+    usage: CreateEmbeddingResponse.Usage;
+}
+export declare namespace CreateEmbeddingResponse {
+    /**
+     * The usage information for the request.
+     */
+    interface Usage {
+        /**
+         * The number of tokens used by the prompt.
+         */
+        prompt_tokens: number;
+        /**
+         * The total number of tokens used by the request.
+         */
+        total_tokens: number;
+        /**
+         * Fields specific to WebLLM, not present in OpenAI.
+         */
+        extra: {
+            /**
+             * Number of tokens per second for prefilling.
+             */
+            prefill_tokens_per_s: number;
+        };
+    }
+}
+/**
+ * Represents an embedding vector returned by embedding endpoint.
+ */
+export interface Embedding {
+    /**
+     * The embedding vector, which is a list of floats. The length of vector depends on
+     * the model.
+     */
+    embedding: Array<number>;
+    /**
+     * The index of the embedding in the list of embeddings.
+     */
+    index: number;
+    /**
+     * The object type, which is always "embedding".
+     */
+    object: "embedding";
+}
+export interface EmbeddingCreateParams {
+    /**
+     * Input text to embed, encoded as a string or array of tokens. To embed multiple
+     * inputs in a single request, pass an array of strings or array of token arrays.
+     * The input must not exceed the max input tokens for the model, and cannot be an empty string.
+     * If the batch size is too large, multiple forward of the will take place.
+     */
+    input: string | Array<string> | Array<number> | Array<Array<number>>;
+    /**
+     * ID of the model to use. This equals to `ModelRecord.model_id`, which needs to either be in
+     * `webllm.prebuiltAppConfig` or in `engineConfig.appConfig`.
+     *
+     * @note Call `CreateMLCEngine(model)` or `engine.reload(model)` ahead of time.
+     * @note If only one model is loaded in the engine, this field is optional. If multiple models
+     *   are loaded, this is required.
+     */
+    model?: string | null;
+    /**
+     * The format to return the embeddings in.
+     *
+     * @note Currently only support `float`.
+     */
+    encoding_format?: "float" | "base64";
+    /**
+     * The number of dimensions the resulting output embeddings should have.
+     *
+     * @note Not supported.
+     */
+    dimensions?: number;
+    /**
+     * A unique identifier representing your end-user, which can help OpenAI to monitor
+     * and detect abuse.
+     *
+     * @note Not supported.
+     */
+    user?: string;
+}
+export declare const EmbeddingCreateParamsUnsupportedFields: Array<string>;
+export declare function postInitAndCheckFields(request: EmbeddingCreateParams, currentModelId: string): void;
+//# sourceMappingURL=embedding.d.ts.map

package/lib/openai_api_protocols/embedding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/openai_api_protocols/embedding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAOH,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAE9C,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAqB;gBAEvB,MAAM,EAAE,kBAAkB;IAItC;;OAEG;IACH,MAAM,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,uBAAuB,CAAC;CAGzE;AAED,MAAM,WAAW,uBAAuB;IACtC;;OAEG;IACH,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAEvB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,KAAK,EAAE,uBAAuB,CAAC,KAAK,CAAC;CACtC;AAGD,yBAAiB,uBAAuB,CAAC;IACvC;;OAEG;IACH,UAAiB,KAAK;QACpB;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,YAAY,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,KAAK,EAAE;YACL;;eAEG;YACH,oBAAoB,EAAE,MAAM,CAAC;SAC9B,CAAC;KACH;CACF;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEzB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,MAAM,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC;;;;;OAKG;IACH,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAErE;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;;;OAIG;IACH,eAAe,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;IAGrC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,eAAO,MAAM,sCAAsC,EAAE,KAAK,CAAC,MAAM,CAGhE,CAAC;AAEF,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,qBAAqB,EAE9B,cAAc,EAAE,MAAM,GACrB,IAAI,CAmCN"}

package/lib/openai_api_protocols/index.d.ts

@@ -0,0 +1,20 @@
+/**
+ * The input to OpenAI API, directly adopted from openai-node with small tweaks:
+ * https://github.com/openai/openai-node/blob/master/src/resources/chat/completions.ts
+ *
+ * Copyright 2024 OpenAI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { Chat, ChatCompletionRequestBase, ChatCompletionRequestNonStreaming, ChatCompletionRequestStreaming, ChatCompletionRequest, ChatCompletion, ChatCompletionChunk, ChatCompletionRequestUnsupportedFields, postInitAndCheckFields as postInitAndCheckFieldsChatCompletion, ChatCompletionContentPart, ChatCompletionContentPartText, ChatCompletionContentPartImage, ChatCompletionMessageToolCall, ChatCompletionRole, ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam, ChatCompletionAssistantMessageParam, ChatCompletionToolMessageParam, ChatCompletionMessageParam, FunctionParameters, FunctionDefinition, ChatCompletionTool, ChatCompletionNamedToolChoice, ChatCompletionToolChoiceOption, TopLogprob, ChatCompletionTokenLogprob, ChatCompletionMessage, CompletionUsage, ResponseFormat, ChatCompletionFinishReason, } from "./chat_completion";
+export { Completions, CompletionCreateParamsNonStreaming, CompletionCreateParamsStreaming, CompletionCreateParamsBase, CompletionCreateParams, Completion, CompletionChoice, postInitAndCheckFields as postInitAndCheckFieldsCompletion, } from "./completion";
+export { Embeddings, Embedding, EmbeddingCreateParams, CreateEmbeddingResponse, postInitAndCheckFields as postInitAndCheckFieldsEmbedding, } from "./embedding";
+//# sourceMappingURL=index.d.ts.map

package/lib/openai_api_protocols/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/openai_api_protocols/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EACL,IAAI,EACJ,yBAAyB,EACzB,iCAAiC,EACjC,8BAA8B,EAC9B,qBAAqB,EACrB,cAAc,EACd,mBAAmB,EACnB,sCAAsC,EACtC,sBAAsB,IAAI,oCAAoC,EAC9D,yBAAyB,EACzB,6BAA6B,EAC7B,8BAA8B,EAC9B,6BAA6B,EAC7B,kBAAkB,EAClB,gCAAgC,EAChC,8BAA8B,EAC9B,mCAAmC,EACnC,8BAA8B,EAC9B,0BAA0B,EAC1B,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,6BAA6B,EAC7B,8BAA8B,EAC9B,UAAU,EACV,0BAA0B,EAC1B,qBAAqB,EACrB,eAAe,EACf,cAAc,EACd,0BAA0B,GAC3B,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,WAAW,EACX,kCAAkC,EAClC,+BAA+B,EAC/B,0BAA0B,EAC1B,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,sBAAsB,IAAI,gCAAgC,GAC3D,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,UAAU,EACV,SAAS,EACT,qBAAqB,EACrB,uBAAuB,EACvB,sBAAsB,IAAI,+BAA+B,GAC1D,MAAM,aAAa,CAAC"}

package/lib/service_worker.d.ts

@@ -0,0 +1,53 @@
+import { ChatOptions, MLCEngineConfig } from "./config";
+import { WorkerRequest, WorkerResponse } from "./message";
+import { WebWorkerMLCEngineHandler, WebWorkerMLCEngine, ChatWorker } from "./web_worker";
+/**
+ * Worker handler that can be used in a ServiceWorker.
+ *
+ * @example
+ *
+ * const engine = new MLCEngine();
+ * let handler;
+ * chrome.runtime.onConnect.addListener(function (port) {
+ *   if (handler === undefined) {
+ *     handler = new ServiceWorkerMLCEngineHandler(engine, port);
+ *   } else {
+ *     handler.setPort(port);
+ *   }
+ *   port.onMessage.addListener(handler.onmessage.bind(handler));
+ * });
+ */
+export declare class ServiceWorkerMLCEngineHandler extends WebWorkerMLCEngineHandler {
+    private clientRegistry;
+    private initRequestUuid?;
+    constructor();
+    postMessage(message: WorkerResponse): void;
+    onmessage(event: ExtendableMessageEvent, onComplete?: (value: any) => void, onError?: () => void): void;
+}
+export declare class ServiceWorker implements ChatWorker {
+    _onmessage: (event: MessageEvent) => void;
+    get onmessage(): (event: any) => void;
+    set onmessage(handler: (event: any) => void);
+    postMessage(message: WorkerRequest): void;
+}
+/**
+ * Create a ServiceWorkerMLCEngine.
+ *
+ * @param modelId model_id of the model to load, either string or string[]. When multiple models
+ *   are provided, we load all models sequentially. Each modelId needs to either be in
+ *   `webllm.prebuiltAppConfig`, or in `engineCOnfig.appConfig`.
+ * @param engineConfig Optionally configures the engine, see `webllm.MLCEngineConfig` for more.
+ * @param chatOpts Extra options to optionally override the `mlc-chat-config.json` of `modelId`.
+ *   The size of which needs to match that of `modelId`; chatOpts[i] will be used for modelId[i].
+ * @returns An initialized `WebLLM.ServiceWorkerMLCEngine` with `modelId` loaded.
+ */
+export declare function CreateServiceWorkerMLCEngine(modelId: string | string[], engineConfig?: MLCEngineConfig, chatOpts?: ChatOptions | ChatOptions[], keepAliveMs?: number): Promise<ServiceWorkerMLCEngine>;
+/**
+ * A client of MLCEngine that exposes the same interface
+ */
+export declare class ServiceWorkerMLCEngine extends WebWorkerMLCEngine {
+    missedHeartbeat: number;
+    constructor(engineConfig?: MLCEngineConfig, keepAliveMs?: number);
+    onmessage(event: any): void;
+}
+//# sourceMappingURL=service_worker.d.ts.map

package/lib/service_worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"service_worker.d.ts","sourceRoot":"","sources":["../src/service_worker.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AACxD,OAAO,EAAgB,aAAa,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAExE,OAAO,EACL,yBAAyB,EACzB,kBAAkB,EAClB,UAAU,EACX,MAAM,cAAc,CAAC;AAYtB;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,6BAA8B,SAAQ,yBAAyB;IAC1E,OAAO,CAAC,cAAc,CAGlB;IACJ,OAAO,CAAC,eAAe,CAAC,CAAS;;IA+BjC,WAAW,CAAC,OAAO,EAAE,cAAc;IAanC,SAAS,CACP,KAAK,EAAE,sBAAsB,EAC7B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,EACjC,OAAO,CAAC,EAAE,MAAM,IAAI,GACnB,IAAI;CA0DR;AAGD,qBAAa,aAAc,YAAW,UAAU;IAC9C,UAAU,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAY;IAErD,IAAI,SAAS,IAIU,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAF1C;IAED,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,EAO1C;IAED,WAAW,CAAC,OAAO,EAAE,aAAa;CAWnC;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,4BAA4B,CAChD,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,YAAY,CAAC,EAAE,eAAe,EAC9B,QAAQ,CAAC,EAAE,WAAW,GAAG,WAAW,EAAE,EACtC,WAAW,SAAQ,GAClB,OAAO,CAAC,sBAAsB,CAAC,CAgBjC;AAED;;GAEG;AACH,qBAAa,sBAAuB,SAAQ,kBAAkB;IAC5D,eAAe,SAAK;gBAER,YAAY,CAAC,EAAE,eAAe,EAAE,WAAW,SAAQ;IAc/D,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,IAAI;CAkB5B"}

package/lib/support.d.ts

@@ -0,0 +1,117 @@
+/** Util methods. */
+import { Tokenizer } from "@mlc-ai/web-tokenizers";
+import { AppConfig, ModelRecord } from "./config";
+import { ChatCompletionChunk, ChatCompletionContentPartImage, ChatCompletionMessageToolCall } from "./openai_api_protocols/index";
+/**
+ * Based on `p_prob` of size (vocabSize,) which becomes a distribution after calling
+ * `applySoftmaxWithTemperature()`, sample `top_logprobs` top-probable tokens.
+ *
+ * @param num_top_probs: `top_logprobs` from ChatCompletionRequest
+ * @param p_prob: `logitsOnCPUArray`, being a distribution after `applySoftmaxWithTemperature()`.
+ *
+ * Followed implementation of `ComputeTopProbsImpl()` from [https://github.com/mlc-ai/mlc-llm/blob/
+ * 5b8c529e9704abd09b0432da6dcb4b013fdf43b1/cpp/serve/sampler/cpu_sampler.cc].
+ *
+ * @returns Arrays of (tokenID, prob) pairs, ranked from highest prob to least.
+ */
+export declare function getTopProbs(num_top_probs: number, p_prob: Float32Array): Array<[number, number]>;
+/**
+ * Get the token table in the form of a string list of tokens, ordered by their token id.
+ * @param tokenizer A loaded tokenizer.
+ * @note The size of the table (i.e. tokenizer.getVocabSize()) may be smaller than the `vocab_size`
+ * in config.json (length of logits), see https://github.com/QwenLM/Qwen2/issues/147 and
+ * https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/discussions/47.
+ */
+export declare function getTokenTableFromTokenizer(tokenizer: Tokenizer): string[];
+/**
+ * Postprocess the suffix of ModelRecord.model to be "/resolve/main/" if it is not specified otherwise.
+ * e.g. https://huggingface.co/mlc-ai/OpenHermes-2.5-Mistral-7B-q4f16_1-MLC/resolve/main/
+ * @return the href of the final URL.
+ */
+export declare function cleanModelUrl(modelUrl: string): string;
+/**
+ * Json schema used to prompt the model for function calling; directly copied from the official guide.
+ * This represents to a single function call.
+ */
+export declare const officialHermes2FunctionCallSchema = "{\"properties\": {\"arguments\": {\"title\": \"Arguments\", \"type\": \"object\"}, \"name\": {\"title\": \"Name\", \"type\": \"string\"}}, \"required\": [\"arguments\", \"name\"], \"title\": \"FunctionCall\", \"type\": \"object\"}";
+/**
+ * A list of such function calls. Used to specify response format, since the output is expected to
+ * be a list of such function calls.
+ */
+export declare const officialHermes2FunctionCallSchemaArray = "{\"type\":\"array\",\"items\":{\"properties\": {\"arguments\": {\"title\": \"Arguments\", \"type\": \"object\"}, \"name\": {\"title\": \"Name\", \"type\": \"string\"}}, \"required\": [\"arguments\", \"name\"], \"title\": \"FunctionCall\", \"type\": \"object\"}}";
+/**
+ * Full system prompt for Hermes-2-Pro function calling.
+ */
+export declare const hermes2FunctionCallingSystemPrompt = "You are a function calling AI model. You are \nprovided with function signatures within <tools></tools> XML tags. You may call one or more functions \nto assist with the user query. Don't make assumptions about what values to plug into functions. Here \nare the available tools: <tools> {hermes_tools}  </tools>. \nUse the following pydantic model json schema for each tool call you will make: \n{\"properties\": {\"arguments\": {\"title\": \"Arguments\", \"type\": \"object\"}, \"name\": {\"title\": \"Name\", \"type\": \"string\"}}, \"required\": [\"arguments\", \"name\"], \"title\": \"FunctionCall\", \"type\": \"object\"} For each function call return a json object.";
+/**
+ * Given a string outputMessage, parse it as a JSON object and return an array of tool calls.
+ *
+ * Expect outputMessage to be a valid JSON string, and expect it to be an array of Function with
+ * fields `arguments` and `name`.
+ */
+export declare function getToolCallFromOutputMessage(outputMessage: string, isStreaming: false): Array<ChatCompletionMessageToolCall>;
+export declare function getToolCallFromOutputMessage(outputMessage: string, isStreaming: true): Array<ChatCompletionChunk.Choice.Delta.ToolCall>;
+export declare function findModelRecord(modelId: string, appConfig: AppConfig): ModelRecord;
+/**
+ * Return the model to use given the loaded modelIds and requestModel. Throws error when unclear
+ * which model to load.
+ * @param loadedModelIds Models currently loaded in the engine.
+ * @param requestModel Model the user specified to load via the request. Required when multiple
+ *   models are loaded
+ * @param requestName The type of request or API to load the model for. Needed for error throwing.
+ */
+export declare function getModelIdToUse(loadedModelIds: string[], requestModel: string | undefined | null, requestName: string): string;
+/**
+ * TODO: Consider if this is the best strategy (though aligned with mlc-llm). We currently greedily
+ * try to fill up prefillChunkSize. Consider the example with 2048 prefill chunk size:
+ * const inputData = [
+    image1,  // 1921
+    rangeArr(0, 2048),
+    image2,
+  ];
+ * Current approach results in chunks:
+   [image1, rangeArr(0, 127)],
+   [rangeArr(127, 2048)],
+   [image2],
+ * This means 4 embedding kernels and 3 prefill kernels.
+ * While the optimal chunking may be:
+   [image1],
+   [rangeArr(0, 2048)],
+   [image2],
+ * This results in 3 embedding kernels and 3 prefill kernels.
+ * However, greedy strategy is more intuitive and probably more generalizable.
+ */
+/**
+ * Chunk the inputData such that each chunk's total input length is smaller than prefill
+ * chunk size.
+ * @returns [the data chunks, the input length of each chunk]
+ * @note precondition: if inputData has image in it, then prefillChunkSize >= IMAGE_EMBED_SIZE.
+ */
+export declare function getChunkedPrefillInputData(inputData: Array<Array<number> | ImageURL>, prefillChunkSize: number): [Array<Array<number> | ImageURL>[], Array<number>];
+/**
+ * A lock implemented using Promise.
+ *
+ * Referred to:
+ * - https://jackpordi.com/posts/locks-in-js-because-why-not
+ * - https://www.linkedin.com/pulse/asynchronous-locking-using-promises-javascript-abdul-ahad-o7smf/
+ */
+export declare class CustomLock {
+    private acquired;
+    private readonly queue;
+    acquire(): Promise<void>;
+    release(): Promise<void>;
+}
+type ImageURL = ChatCompletionContentPartImage.ImageURL;
+export declare const IMAGE_EMBED_SIZE = 1921;
+/**
+ * Given a url, get the image data. The url can either start with `http` or `data:image`.
+ */
+export declare function getImageDataFromURL(url: string): Promise<ImageData>;
+/**
+ * Given an ImageData, return the RGB array in Uint8ClampedArray. Note the ImageData.data
+ * is RGBA, so we skip every fourth element of the data. The order goes by rows from the
+ * top-left pixel to the bottom-right, in RGB order.
+ */
+export declare function getRGBArrayFromImageData(imageData: ImageData): Uint8ClampedArray;
+export {};
+//# sourceMappingURL=support.d.ts.map

package/lib/support.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"support.d.ts","sourceRoot":"","sources":["../src/support.ts"],"names":[],"mappings":"AAAA,oBAAoB;AACpB,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAuB,WAAW,EAAE,MAAM,UAAU,CAAC;AACvE,OAAO,EACL,mBAAmB,EACnB,8BAA8B,EAC9B,6BAA6B,EAC9B,MAAM,8BAA8B,CAAC;AAWtC;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CACzB,aAAa,EAAE,MAAM,EACrB,MAAM,EAAE,YAAY,GACnB,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAiCzB;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,SAAS,GAAG,MAAM,EAAE,CAOzE;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAMtD;AAKD;;;GAGG;AACH,eAAO,MAAM,iCAAiC,2OAAuM,CAAC;AAEtP;;;GAGG;AACH,eAAO,MAAM,sCAAsC,0QAAkE,CAAC;AAEtH;;GAEG;AACH,eAAO,MAAM,kCAAkC,oqBAKmC,CAAC;AAEnF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,KAAK,GACjB,KAAK,CAAC,6BAA6B,CAAC,CAAC;AACxC,wBAAgB,4BAA4B,CAC1C,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,IAAI,GAChB,KAAK,CAAC,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AAsEpD,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,SAAS,GACnB,WAAW,CAMb;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,cAAc,EAAE,MAAM,EAAE,EACxB,YAAY,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,EACvC,WAAW,EAAE,MAAM,GAClB,MAAM,CAyBR;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;;GAKG;AACH,wBAAgB,0BAA0B,CACxC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,EAC1C,gBAAgB,EAAE,MAAM,GACvB,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAuEpD;AAID;;;;;;GAMG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;IAEvB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAaxB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAkBtC;AAGD,KAAK,QAAQ,GAAG,8BAA8B,CAAC,QAAQ,CAAC;AAGxD,eAAO,MAAM,gBAAgB,OAAO,CAAC;AAErC;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAYzE;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,SAAS,GACnB,iBAAiB,CAQnB"}