use-browser-llm 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 use-local-llm contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,248 @@
1
+ # use-browser-llm
2
+
3
+ A headless React hook for running Large Language Models locally in the
4
+ browser via WebGPU — no backend, no API keys, no data leaving the device.
5
+
6
+ Model inference runs inside a dedicated Web Worker (via
7
+ [`@mlc-ai/web-llm`](https://github.com/mlc-ai/web-llm)), so the main thread
8
+ and your UI never freeze during generation. Model weights are cached in
9
+ IndexedDB across sessions, so repeat visits skip the multi-gigabyte
10
+ re-download.
11
+
12
+ **This package ships no UI components and no styles — only a hook.** It is
13
+ deliberately headless so it stays lightweight and works with whatever
14
+ design system you already use (Tailwind, Shadcn, CSS Modules, plain CSS,
15
+ anything). You bring the chat bubble, the loading spinner, the button —
16
+ `useBrowserLLM()` only gives you the state and the functions to drive them.
17
+
18
+ ## Requirements
19
+
20
+ - React 18 or later
21
+ - A browser with [WebGPU](https://caniuse.com/webgpu) support — check the
22
+ link for current browser coverage, since it's still expanding — see
23
+ [Handling unsupported browsers](#handling-unsupported-browsers) below for
24
+ what happens when it's missing
25
+
26
+ ## Install
27
+
28
+ ```sh
29
+ npm install use-browser-llm
30
+ ```
31
+
32
+ ## Quickstart
33
+
34
+ ```tsx
35
+ import { useState } from "react";
36
+ import { useBrowserLLM } from "use-browser-llm";
37
+
38
+ const MODEL_ID = "Llama-3.2-1B-Instruct-q4f16_1-MLC";
39
+
40
+ export function App() {
41
+ const { status, progress, generate, isGenerating } = useBrowserLLM(MODEL_ID);
42
+ const [reply, setReply] = useState("");
43
+
44
+ if (status === "loading") {
45
+ return <p>Loading model… {Math.round(progress * 100)}%</p>;
46
+ }
47
+
48
+ if (status === "error") {
49
+ return <p>Something went wrong loading the model.</p>;
50
+ }
51
+
52
+ return (
53
+ <div>
54
+ <button
55
+ disabled={status !== "ready" || isGenerating}
56
+ onClick={async () => {
57
+ const text = await generate([
58
+ { role: "user", content: "Say hello in one short sentence." },
59
+ ]);
60
+ setReply(text);
61
+ }}
62
+ >
63
+ {isGenerating ? "Generating…" : "Ask"}
64
+ </button>
65
+ <p>{reply}</p>
66
+ </div>
67
+ );
68
+ }
69
+ ```
70
+
71
+ `useBrowserLLM` takes a model id — any id from
72
+ [`prebuiltAppConfig.model_list`](https://github.com/mlc-ai/web-llm/blob/main/src/config.ts)
73
+ in `@mlc-ai/web-llm` — or `undefined` if you don't want to load a model yet
74
+ (e.g. while the user is still picking one). `status` starts at `"idle"`,
75
+ moves to `"loading"` once a model id is provided, then to `"ready"` or
76
+ `"error"`. `generate()` only resolves once `status === "ready"`; calling it
77
+ earlier rejects immediately (see [Errors](#errors) below).
78
+
79
+ ## Streaming generation
80
+
81
+ For token-by-token output, use `streamGenerate()` instead of `generate()`.
82
+ It returns an `AsyncGenerator`, so you consume it with `for await`:
83
+
84
+ ```tsx
85
+ import { useState } from "react";
86
+ import { useBrowserLLM } from "use-browser-llm";
87
+
88
+ export function StreamingExample() {
89
+ const { status, streamGenerate } = useBrowserLLM(
90
+ "Llama-3.2-1B-Instruct-q4f16_1-MLC",
91
+ );
92
+ const [text, setText] = useState("");
93
+
94
+ async function handleAsk() {
95
+ setText("");
96
+ for await (const token of streamGenerate([
97
+ { role: "user", content: "Write a haiku about the ocean." },
98
+ ])) {
99
+ setText((prev) => prev + token);
100
+ }
101
+ }
102
+
103
+ return (
104
+ <div>
105
+ <button disabled={status !== "ready"} onClick={handleAsk}>
106
+ Ask
107
+ </button>
108
+ <p>{text}</p>
109
+ </div>
110
+ );
111
+ }
112
+ ```
113
+
114
+ Breaking out of the loop early (e.g. a `break` inside the `for await`)
115
+ still stops the underlying worker's inference — you don't need to call
116
+ `abort()` yourself in that case, though you can if you want to stop
117
+ generation from somewhere other than the loop itself (see below).
118
+
119
+ ## Cancellation
120
+
121
+ Both `generate()` and `streamGenerate()` share one `abort()` function that
122
+ stops generation in the worker itself, not just the promise on the main
123
+ thread:
124
+
125
+ ```tsx
126
+ import { useBrowserLLM } from "use-browser-llm";
127
+
128
+ export function CancellableExample() {
129
+ const { status, streamGenerate, abort, isGenerating } = useBrowserLLM(
130
+ "Llama-3.2-1B-Instruct-q4f16_1-MLC",
131
+ );
132
+
133
+ async function handleAsk() {
134
+ for await (const token of streamGenerate([
135
+ { role: "user", content: "Tell me a long story." },
136
+ ])) {
137
+ console.log(token);
138
+ }
139
+ }
140
+
141
+ return (
142
+ <div>
143
+ <button disabled={status !== "ready"} onClick={handleAsk}>
144
+ Ask
145
+ </button>
146
+ <button disabled={!isGenerating} onClick={abort}>
147
+ Stop
148
+ </button>
149
+ </div>
150
+ );
151
+ }
152
+ ```
153
+
154
+ `isGenerating` is `true` for the duration of any `generate()` or
155
+ `streamGenerate()` call, and only one can run at a time — calling either
156
+ while one is already in flight rejects with `HookBusyError` rather than
157
+ silently queuing or corrupting state.
158
+
159
+ ## Handling unsupported browsers
160
+
161
+ When the browser has no usable WebGPU (missing entirely, no adapter, or
162
+ only a software fallback adapter), `status` resolves straight to
163
+ `"unsupported"` — it never passes through `"loading"`, and no worker is
164
+ ever created. Check for it and render your own fallback UI (this package
165
+ ships none):
166
+
167
+ ```tsx
168
+ import { useBrowserLLM } from "use-browser-llm";
169
+
170
+ export function UnsupportedFallbackExample() {
171
+ const { status } = useBrowserLLM("Llama-3.2-1B-Instruct-q4f16_1-MLC");
172
+
173
+ if (status === "unsupported") {
174
+ return (
175
+ <p>
176
+ Your browser doesn't support the local AI features on this page. Try the
177
+ latest Chrome or Edge.
178
+ </p>
179
+ );
180
+ }
181
+
182
+ // ...the rest of your component
183
+ return null;
184
+ }
185
+ ```
186
+
187
+ ## Cache status
188
+
189
+ `cacheStatus` tells you whether the model was already downloaded in a
190
+ previous session, so you can show a different message for "instant load"
191
+ vs. "first-time multi-gigabyte download":
192
+
193
+ ```tsx
194
+ const { cacheStatus } = useBrowserLLM("Llama-3.2-1B-Instruct-q4f16_1-MLC");
195
+ // "idle" | "checking" | "cached" | "downloading"
196
+ ```
197
+
198
+ `cacheStatus` is available before `status` reaches `"ready"`, but for a
199
+ model that loads very quickly it can stay at `"checking"` if the load
200
+ finishes before the cache check itself resolves — check `status` first for
201
+ anything load-gating; `cacheStatus` is informational.
202
+
203
+ ## Errors
204
+
205
+ - `HookNotReadyError` — rejected by `generate()` (immediately) or thrown
206
+ from `streamGenerate()`'s async generator (on its first iteration —
207
+ generators are lazy, so the check runs when you start consuming it, not
208
+ when you call the function) when called before `status === "ready"`.
209
+ - `HookBusyError` — same timing as above, when calling `generate()`/
210
+ `streamGenerate()` while one is already in flight.
211
+ - `UnsupportedError` — the `error` value when `status === "unsupported"`;
212
+ has a `.reason` field (`"no-navigator-gpu"` | `"no-adapter"` |
213
+ `"fallback-adapter"`).
214
+ - `WorkerCrashError` — the `error` value when `status === "error"` and the
215
+ cause was the worker crashing or becoming unresponsive, rather than a
216
+ normal model-load failure.
217
+
218
+ All are exported from `use-browser-llm` for `instanceof` checks.
219
+
220
+ ## API reference
221
+
222
+ ```ts
223
+ function useBrowserLLM(modelId: string | undefined): {
224
+ status: "idle" | "loading" | "ready" | "error" | "unsupported";
225
+ progress: number; // 0-1, meaningful only while status === "loading"
226
+ error: Error | null; // set for "error" and "unsupported"
227
+ cacheStatus: "idle" | "checking" | "cached" | "downloading";
228
+ isGenerating: boolean;
229
+ generationError: Error | null;
230
+ generate(messages: ChatMessage[]): Promise<string>;
231
+ streamGenerate(messages: ChatMessage[]): AsyncGenerator<string, void, void>;
232
+ abort(): void;
233
+ };
234
+
235
+ type ChatMessage = {
236
+ role: "system" | "user" | "assistant";
237
+ content: string;
238
+ };
239
+ ```
240
+
241
+ `ChatMessage` is a plain, self-contained type — e.g.
242
+ `{ role: "user", content: "..." }` — not a re-export of
243
+ `@mlc-ai/web-llm`'s own message type, so this package's public API never
244
+ requires you to know anything about the underlying engine.
245
+
246
+ ## License
247
+
248
+ MIT — see [LICENSE](./LICENSE).
@@ -0,0 +1,78 @@
1
+ /**
2
+ * Public message shape for generate()/streamGenerate() — deliberately a
3
+ * narrower, self-contained re-declaration of the common case from
4
+ * @mlc-ai/web-llm's ChatCompletionMessageParam (system/user/assistant with
5
+ * plain string content), not an import of it. Importing web-llm's type
6
+ * directly would put `@mlc-ai/web-llm` in this package's public .d.ts
7
+ * output, which the package's own architecture rule (only worker.ts ever
8
+ * references web-llm at the type OR value level) exists to prevent.
9
+ *
10
+ * This covers the vast majority of real usage (see README). It's
11
+ * structurally assignable to ChatCompletionMessageParam[] wherever this
12
+ * package passes messages into the real engine, so nothing is lost at the
13
+ * call site — a consumer just doesn't see web-llm's richer (tool-call,
14
+ * image-content-part) variants reflected in their editor, since this
15
+ * package doesn't expose them.
16
+ */
17
+ type ChatRole = "system" | "user" | "assistant";
18
+ interface ChatMessage {
19
+ role: ChatRole;
20
+ content: string;
21
+ }
22
+
23
+ type ModelLoadStatus = "idle" | "loading" | "ready" | "error" | "unsupported";
24
+ type CacheStatus = "idle" | "checking" | "cached" | "downloading";
25
+ interface ModelLoadState {
26
+ status: ModelLoadStatus;
27
+ /** 0-1 while status === "loading"; meaningless otherwise. */
28
+ progress: number;
29
+ /** Set for both "error" and "unsupported" — check `status` first, this
30
+ * carries the detail (an UnsupportedError, WorkerCrashError, or
31
+ * whatever the engine/worker rejected with). */
32
+ error: Error | null;
33
+ /** Whether modelId was already in IndexedDB before this load started.
34
+ * "checking" until the (non-blocking) cache check resolves; settles to
35
+ * "cached" or "downloading" before status reaches "ready" — but is not
36
+ * itself a loading gate, so it's still meaningful to read after ready.
37
+ * Edge case: for a fast/cached model, `status` can reach "ready" before
38
+ * this check resolves at all — cacheStatus then stays "checking"
39
+ * indefinitely for that load (the reducer intentionally drops a late
40
+ * resolution once status is no longer "loading"; see the "cache-status"
41
+ * case below). */
42
+ cacheStatus: CacheStatus;
43
+ }
44
+ interface GenerationState {
45
+ isGenerating: boolean;
46
+ generationError: Error | null;
47
+ }
48
+ interface UseBrowserLLMResult extends ModelLoadState, GenerationState {
49
+ generate(messages: ChatMessage[]): Promise<string>;
50
+ streamGenerate(messages: ChatMessage[]): AsyncGenerator<string, void, void>;
51
+ /** Stops an in-flight generate()/streamGenerate() call, including the
52
+ * worker's inference loop — not just the UI-visible promise. */
53
+ abort(): void;
54
+ }
55
+ /**
56
+ * Model-loading state machine (P1-04) plus generate/streamGenerate/abort
57
+ * (P1-05). Cache-status (P1-06) and the unsupported-browser path (P1-07)
58
+ * extend this hook in later tasks.
59
+ */
60
+ declare function useBrowserLLM(modelId: string | undefined): UseBrowserLLMResult;
61
+
62
+ declare class HookNotReadyError extends Error {
63
+ constructor();
64
+ }
65
+ declare class HookBusyError extends Error {
66
+ constructor();
67
+ }
68
+ declare class UnsupportedError extends Error {
69
+ readonly reason: string;
70
+ constructor(reason: string);
71
+ }
72
+ declare class WorkerCrashError extends Error {
73
+ constructor(message: string);
74
+ }
75
+
76
+ declare const USE_BROWSER_LLM_VERSION = "0.1.0";
77
+
78
+ export { type CacheStatus, type ChatMessage, type ChatRole, type GenerationState, HookBusyError, HookNotReadyError, type ModelLoadState, type ModelLoadStatus, USE_BROWSER_LLM_VERSION, UnsupportedError, type UseBrowserLLMResult, WorkerCrashError, useBrowserLLM };
package/dist/index.js ADDED
@@ -0,0 +1,358 @@
1
+ // src/use-browser-llm.ts
2
+ import { useCallback, useEffect, useReducer, useRef, useState } from "react";
3
+
4
+ // src/engine-client.ts
5
+ import * as Comlink from "comlink";
6
+
7
+ // src/errors.ts
8
+ var HookNotReadyError = class extends Error {
9
+ constructor() {
10
+ super("useBrowserLLM: cannot generate before the model is ready");
11
+ this.name = "HookNotReadyError";
12
+ }
13
+ };
14
+ var HookBusyError = class extends Error {
15
+ constructor() {
16
+ super("useBrowserLLM: a generation is already in progress");
17
+ this.name = "HookBusyError";
18
+ }
19
+ };
20
+ var UnsupportedError = class extends Error {
21
+ reason;
22
+ constructor(reason) {
23
+ super(`useBrowserLLM: WebGPU is unsupported in this browser (${reason})`);
24
+ this.name = "UnsupportedError";
25
+ this.reason = reason;
26
+ }
27
+ };
28
+ var WorkerCrashError = class extends Error {
29
+ constructor(message) {
30
+ super(`useBrowserLLM: the worker crashed or stopped responding (${message})`);
31
+ this.name = "WorkerCrashError";
32
+ }
33
+ };
34
+
35
+ // src/engine-client.ts
36
+ function createEngineClient() {
37
+ const worker = new Worker(new URL("./worker.js", import.meta.url), {
38
+ type: "module"
39
+ });
40
+ const remote = Comlink.wrap(worker);
41
+ return {
42
+ loadModel(modelId, onProgress) {
43
+ return remote.loadModel(modelId, Comlink.proxy(onProgress ?? (() => {
44
+ })));
45
+ },
46
+ generate(messages) {
47
+ return remote.generate(messages);
48
+ },
49
+ streamGenerate(messages, onToken) {
50
+ return remote.streamGenerate(messages, Comlink.proxy(onToken));
51
+ },
52
+ abort() {
53
+ return remote.abort();
54
+ },
55
+ unload() {
56
+ return remote.unload();
57
+ },
58
+ checkCache(modelId) {
59
+ return remote.checkCache(modelId);
60
+ },
61
+ terminate() {
62
+ worker.terminate();
63
+ },
64
+ onCrash(listener) {
65
+ const handleError = (event) => {
66
+ listener(new WorkerCrashError(event.message || "uncaught exception"));
67
+ };
68
+ const handleMessageError = () => {
69
+ listener(new WorkerCrashError("failed to deserialize a worker message"));
70
+ };
71
+ worker.addEventListener("error", handleError);
72
+ worker.addEventListener("messageerror", handleMessageError);
73
+ return () => {
74
+ worker.removeEventListener("error", handleError);
75
+ worker.removeEventListener("messageerror", handleMessageError);
76
+ };
77
+ }
78
+ };
79
+ }
80
+
81
+ // src/detect-webgpu.ts
82
+ async function detectWebGPUSupport() {
83
+ if (typeof navigator === "undefined" || !("gpu" in navigator)) {
84
+ return { supported: false, reason: "no-navigator-gpu" };
85
+ }
86
+ try {
87
+ const adapter = await navigator.gpu.requestAdapter();
88
+ if (!adapter) {
89
+ return { supported: false, reason: "no-adapter" };
90
+ }
91
+ if (adapter.info.isFallbackAdapter) {
92
+ return { supported: false, reason: "fallback-adapter" };
93
+ }
94
+ return { supported: true };
95
+ } catch {
96
+ return { supported: false, reason: "no-adapter" };
97
+ }
98
+ }
99
+
100
+ // src/to-async-generator.ts
101
+ async function* toAsyncGenerator(run) {
102
+ const queue = [];
103
+ let resolveNext = null;
104
+ let done = false;
105
+ let error = null;
106
+ run((token) => {
107
+ queue.push(token);
108
+ resolveNext?.();
109
+ resolveNext = null;
110
+ }).then(
111
+ () => {
112
+ done = true;
113
+ resolveNext?.();
114
+ resolveNext = null;
115
+ },
116
+ (err) => {
117
+ error = err;
118
+ done = true;
119
+ resolveNext?.();
120
+ resolveNext = null;
121
+ }
122
+ );
123
+ while (true) {
124
+ while (queue.length > 0) {
125
+ yield queue.shift();
126
+ }
127
+ if (done) {
128
+ if (error) {
129
+ throw error;
130
+ }
131
+ return;
132
+ }
133
+ await new Promise((resolve) => {
134
+ resolveNext = resolve;
135
+ });
136
+ }
137
+ }
138
+
139
+ // src/use-browser-llm.ts
140
+ var initialState = {
141
+ status: "idle",
142
+ progress: 0,
143
+ error: null,
144
+ cacheStatus: "idle"
145
+ };
146
+ var LOAD_INACTIVITY_TIMEOUT_MS = 3e4;
147
+ function reducer(state, action) {
148
+ switch (action.type) {
149
+ case "reset":
150
+ return initialState;
151
+ case "load-start":
152
+ return {
153
+ status: "loading",
154
+ progress: 0,
155
+ error: null,
156
+ cacheStatus: "checking"
157
+ };
158
+ case "progress":
159
+ return { ...state, progress: action.progress };
160
+ case "cache-status":
161
+ if (state.status !== "loading") {
162
+ return state;
163
+ }
164
+ return { ...state, cacheStatus: action.cacheStatus };
165
+ case "ready":
166
+ return { ...state, status: "ready", progress: 1, error: null };
167
+ case "error":
168
+ return { ...state, status: "error", error: action.error };
169
+ case "unsupported":
170
+ return {
171
+ status: "unsupported",
172
+ progress: 0,
173
+ error: action.error,
174
+ cacheStatus: "idle"
175
+ };
176
+ }
177
+ }
178
+ function useBrowserLLM(modelId) {
179
+ const [state, dispatch] = useReducer(reducer, initialState);
180
+ const clientRef = useRef(null);
181
+ const statusRef = useRef(state.status);
182
+ statusRef.current = state.status;
183
+ const isGeneratingRef = useRef(false);
184
+ const [isGenerating, setIsGenerating] = useState(false);
185
+ const [generationError, setGenerationError] = useState(null);
186
+ useEffect(() => {
187
+ if (!modelId) {
188
+ dispatch({ type: "reset" });
189
+ return;
190
+ }
191
+ let cancelled = false;
192
+ let watchdog = null;
193
+ let unsubscribeCrash = null;
194
+ const clearWatchdog = () => {
195
+ if (watchdog !== null) {
196
+ clearTimeout(watchdog);
197
+ watchdog = null;
198
+ }
199
+ };
200
+ const resetWatchdog = () => {
201
+ clearWatchdog();
202
+ watchdog = setTimeout(() => {
203
+ if (!cancelled) {
204
+ clientRef.current?.terminate();
205
+ clientRef.current = null;
206
+ dispatch({
207
+ type: "error",
208
+ error: new WorkerCrashError(
209
+ `no progress for ${LOAD_INACTIVITY_TIMEOUT_MS / 1e3}s \u2014 the worker may have been killed by the browser (e.g. out of memory)`
210
+ )
211
+ });
212
+ }
213
+ }, LOAD_INACTIVITY_TIMEOUT_MS);
214
+ };
215
+ clientRef.current?.terminate();
216
+ clientRef.current = null;
217
+ void (async () => {
218
+ const support = await detectWebGPUSupport();
219
+ if (cancelled) {
220
+ return;
221
+ }
222
+ if (!support.supported) {
223
+ dispatch({
224
+ type: "unsupported",
225
+ error: new UnsupportedError(support.reason)
226
+ });
227
+ return;
228
+ }
229
+ const client = createEngineClient();
230
+ clientRef.current = client;
231
+ dispatch({ type: "load-start" });
232
+ resetWatchdog();
233
+ unsubscribeCrash = client.onCrash((error) => {
234
+ if (!cancelled) {
235
+ clearWatchdog();
236
+ dispatch({ type: "error", error });
237
+ }
238
+ });
239
+ client.loadModel(modelId, (report) => {
240
+ if (!cancelled) {
241
+ resetWatchdog();
242
+ dispatch({ type: "progress", progress: report.progress });
243
+ }
244
+ }).then(() => {
245
+ if (!cancelled) {
246
+ clearWatchdog();
247
+ dispatch({ type: "ready" });
248
+ }
249
+ }).catch((err) => {
250
+ if (!cancelled) {
251
+ clearWatchdog();
252
+ dispatch({
253
+ type: "error",
254
+ error: err instanceof Error ? err : new Error(String(err))
255
+ });
256
+ }
257
+ });
258
+ client.checkCache(modelId).then((isCached) => {
259
+ if (!cancelled) {
260
+ dispatch({
261
+ type: "cache-status",
262
+ cacheStatus: isCached ? "cached" : "downloading"
263
+ });
264
+ }
265
+ }).catch(() => {
266
+ });
267
+ })();
268
+ return () => {
269
+ cancelled = true;
270
+ clearWatchdog();
271
+ unsubscribeCrash?.();
272
+ clientRef.current?.terminate();
273
+ clientRef.current = null;
274
+ };
275
+ }, [modelId]);
276
+ const generate = useCallback(
277
+ async (messages) => {
278
+ if (statusRef.current !== "ready" || !clientRef.current) {
279
+ throw new HookNotReadyError();
280
+ }
281
+ if (isGeneratingRef.current) {
282
+ throw new HookBusyError();
283
+ }
284
+ isGeneratingRef.current = true;
285
+ setIsGenerating(true);
286
+ setGenerationError(null);
287
+ try {
288
+ return await clientRef.current.generate(messages);
289
+ } catch (err) {
290
+ const error = err instanceof Error ? err : new Error(String(err));
291
+ setGenerationError(error);
292
+ throw error;
293
+ } finally {
294
+ isGeneratingRef.current = false;
295
+ setIsGenerating(false);
296
+ }
297
+ },
298
+ []
299
+ );
300
+ const streamGenerate = useCallback(
301
+ (messages) => streamGenerateImpl(
302
+ statusRef,
303
+ clientRef,
304
+ isGeneratingRef,
305
+ setIsGenerating,
306
+ setGenerationError,
307
+ messages
308
+ ),
309
+ []
310
+ );
311
+ const abort = useCallback(() => {
312
+ clientRef.current?.abort().catch(() => {
313
+ });
314
+ }, []);
315
+ return {
316
+ ...state,
317
+ isGenerating,
318
+ generationError,
319
+ generate,
320
+ streamGenerate,
321
+ abort
322
+ };
323
+ }
324
+ async function* streamGenerateImpl(statusRef, clientRef, isGeneratingRef, setIsGenerating, setGenerationError, messages) {
325
+ if (statusRef.current !== "ready" || !clientRef.current) {
326
+ throw new HookNotReadyError();
327
+ }
328
+ if (isGeneratingRef.current) {
329
+ throw new HookBusyError();
330
+ }
331
+ const client = clientRef.current;
332
+ isGeneratingRef.current = true;
333
+ setIsGenerating(true);
334
+ setGenerationError(null);
335
+ try {
336
+ yield* toAsyncGenerator((onToken) => client.streamGenerate(messages, onToken));
337
+ } catch (err) {
338
+ const error = err instanceof Error ? err : new Error(String(err));
339
+ setGenerationError(error);
340
+ throw error;
341
+ } finally {
342
+ isGeneratingRef.current = false;
343
+ setIsGenerating(false);
344
+ client.abort().catch(() => {
345
+ });
346
+ }
347
+ }
348
+
349
+ // src/index.ts
350
+ var USE_BROWSER_LLM_VERSION = "0.1.0";
351
+ export {
352
+ HookBusyError,
353
+ HookNotReadyError,
354
+ USE_BROWSER_LLM_VERSION,
355
+ UnsupportedError,
356
+ WorkerCrashError,
357
+ useBrowserLLM
358
+ };
@@ -0,0 +1,2 @@
1
+
2
+ export { }
package/dist/worker.js ADDED
@@ -0,0 +1,44 @@
1
+ // src/worker.ts
2
+ import * as Comlink from "comlink";
3
+ import { hasModelInCache, MLCEngine } from "@mlc-ai/web-llm";
4
+
5
+ // src/engine-api-factory.ts
6
+ function createEngineAPI(engine, checkCache) {
7
+ return {
8
+ async loadModel(modelId, onProgress) {
9
+ engine.setInitProgressCallback(onProgress);
10
+ await engine.reload(modelId);
11
+ },
12
+ async generate(messages) {
13
+ const completion = await engine.chatCompletion({
14
+ messages,
15
+ stream: false
16
+ });
17
+ return completion.choices[0]?.message.content ?? "";
18
+ },
19
+ async streamGenerate(messages, onToken) {
20
+ const stream = await engine.chatCompletion({
21
+ messages,
22
+ stream: true
23
+ });
24
+ for await (const chunk of stream) {
25
+ const token = chunk.choices[0]?.delta.content;
26
+ if (token) {
27
+ onToken(token);
28
+ }
29
+ }
30
+ },
31
+ async abort() {
32
+ await engine.interruptGenerate();
33
+ },
34
+ async unload() {
35
+ await engine.unload();
36
+ },
37
+ async checkCache(modelId) {
38
+ return checkCache(modelId);
39
+ }
40
+ };
41
+ }
42
+
43
+ // src/worker.ts
44
+ Comlink.expose(createEngineAPI(new MLCEngine(), hasModelInCache));
package/package.json ADDED
@@ -0,0 +1,65 @@
1
+ {
2
+ "name": "use-browser-llm",
3
+ "version": "0.1.0",
4
+ "description": "Headless React hook for running LLMs locally in the browser via WebGPU, with zero backend and complete privacy.",
5
+ "license": "MIT",
6
+ "repository": {
7
+ "type": "git",
8
+ "url": "git+https://github.com/Muhammad-UmairAli/use-browser-llm.git"
9
+ },
10
+ "homepage": "https://github.com/Muhammad-UmairAli/use-browser-llm#readme",
11
+ "bugs": {
12
+ "url": "https://github.com/Muhammad-UmairAli/use-browser-llm/issues"
13
+ },
14
+ "type": "module",
15
+ "sideEffects": false,
16
+ "engines": {
17
+ "node": ">=18"
18
+ },
19
+ "files": [
20
+ "dist"
21
+ ],
22
+ "exports": {
23
+ ".": {
24
+ "types": "./dist/index.d.ts",
25
+ "import": "./dist/index.js"
26
+ },
27
+ "./package.json": "./package.json"
28
+ },
29
+ "main": "./dist/index.js",
30
+ "types": "./dist/index.d.ts",
31
+ "peerDependencies": {
32
+ "react": ">=18"
33
+ },
34
+ "dependencies": {
35
+ "@mlc-ai/web-llm": "^0.2.79",
36
+ "comlink": "^4.4.2"
37
+ },
38
+ "devDependencies": {
39
+ "@testing-library/dom": "^10.4.1",
40
+ "@testing-library/react": "^16.3.2",
41
+ "@types/node": "^22.10.5",
42
+ "@types/react": "^18.3.18",
43
+ "@types/react-dom": "^18.3.7",
44
+ "@vitest/browser": "2.1.9",
45
+ "@webgpu/types": "^0.1.71",
46
+ "eslint": "^9.18.0",
47
+ "eslint-config-prettier": "^9.1.0",
48
+ "jsdom": "^29.1.1",
49
+ "playwright": "^1.61.1",
50
+ "prettier": "^3.4.2",
51
+ "react": "^18.3.1",
52
+ "react-dom": "^18.3.1",
53
+ "tsup": "^8.3.5",
54
+ "typescript": "^5.7.3",
55
+ "typescript-eslint": "^8.19.1",
56
+ "vitest": "^2.1.8"
57
+ },
58
+ "scripts": {
59
+ "build": "tsup",
60
+ "typecheck": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.worker.json --noEmit",
61
+ "lint": "eslint .",
62
+ "test": "vitest run",
63
+ "test:integration": "vitest run --config vitest.browser.config.ts"
64
+ }
65
+ }