use-browser-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +248 -0
- package/dist/index.d.ts +78 -0
- package/dist/index.js +358 -0
- package/dist/worker.d.ts +2 -0
- package/dist/worker.js +44 -0
- package/package.json +65 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 use-local-llm contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# use-browser-llm
|
|
2
|
+
|
|
3
|
+
A headless React hook for running Large Language Models locally in the
|
|
4
|
+
browser via WebGPU — no backend, no API keys, no data leaving the device.
|
|
5
|
+
|
|
6
|
+
Model inference runs inside a dedicated Web Worker (via
|
|
7
|
+
[`@mlc-ai/web-llm`](https://github.com/mlc-ai/web-llm)), so the main thread
|
|
8
|
+
and your UI never freeze during generation. Model weights are cached in
|
|
9
|
+
IndexedDB across sessions, so repeat visits skip the multi-gigabyte
|
|
10
|
+
re-download.
|
|
11
|
+
|
|
12
|
+
**This package ships no UI components and no styles — only a hook.** It is
|
|
13
|
+
deliberately headless so it stays lightweight and works with whatever
|
|
14
|
+
design system you already use (Tailwind, Shadcn, CSS Modules, plain CSS,
|
|
15
|
+
anything). You bring the chat bubble, the loading spinner, the button —
|
|
16
|
+
`useBrowserLLM()` only gives you the state and the functions to drive them.
|
|
17
|
+
|
|
18
|
+
## Requirements
|
|
19
|
+
|
|
20
|
+
- React 18 or later
|
|
21
|
+
- A browser with [WebGPU](https://caniuse.com/webgpu) support — check the
|
|
22
|
+
link for current browser coverage, since it's still expanding — see
|
|
23
|
+
[Handling unsupported browsers](#handling-unsupported-browsers) below for
|
|
24
|
+
what happens when it's missing
|
|
25
|
+
|
|
26
|
+
## Install
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
npm install use-browser-llm
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Quickstart
|
|
33
|
+
|
|
34
|
+
```tsx
|
|
35
|
+
import { useState } from "react";
|
|
36
|
+
import { useBrowserLLM } from "use-browser-llm";
|
|
37
|
+
|
|
38
|
+
const MODEL_ID = "Llama-3.2-1B-Instruct-q4f16_1-MLC";
|
|
39
|
+
|
|
40
|
+
export function App() {
|
|
41
|
+
const { status, progress, generate, isGenerating } = useBrowserLLM(MODEL_ID);
|
|
42
|
+
const [reply, setReply] = useState("");
|
|
43
|
+
|
|
44
|
+
if (status === "loading") {
|
|
45
|
+
return <p>Loading model… {Math.round(progress * 100)}%</p>;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (status === "error") {
|
|
49
|
+
return <p>Something went wrong loading the model.</p>;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return (
|
|
53
|
+
<div>
|
|
54
|
+
<button
|
|
55
|
+
disabled={status !== "ready" || isGenerating}
|
|
56
|
+
onClick={async () => {
|
|
57
|
+
const text = await generate([
|
|
58
|
+
{ role: "user", content: "Say hello in one short sentence." },
|
|
59
|
+
]);
|
|
60
|
+
setReply(text);
|
|
61
|
+
}}
|
|
62
|
+
>
|
|
63
|
+
{isGenerating ? "Generating…" : "Ask"}
|
|
64
|
+
</button>
|
|
65
|
+
<p>{reply}</p>
|
|
66
|
+
</div>
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
`useBrowserLLM` takes a model id — any id from
|
|
72
|
+
[`prebuiltAppConfig.model_list`](https://github.com/mlc-ai/web-llm/blob/main/src/config.ts)
|
|
73
|
+
in `@mlc-ai/web-llm` — or `undefined` if you don't want to load a model yet
|
|
74
|
+
(e.g. while the user is still picking one). `status` starts at `"idle"`,
|
|
75
|
+
moves to `"loading"` once a model id is provided, then to `"ready"` or
|
|
76
|
+
`"error"`. `generate()` only resolves once `status === "ready"`; calling it
|
|
77
|
+
earlier rejects immediately (see [Errors](#errors) below).
|
|
78
|
+
|
|
79
|
+
## Streaming generation
|
|
80
|
+
|
|
81
|
+
For token-by-token output, use `streamGenerate()` instead of `generate()`.
|
|
82
|
+
It returns an `AsyncGenerator`, so you consume it with `for await`:
|
|
83
|
+
|
|
84
|
+
```tsx
|
|
85
|
+
import { useState } from "react";
|
|
86
|
+
import { useBrowserLLM } from "use-browser-llm";
|
|
87
|
+
|
|
88
|
+
export function StreamingExample() {
|
|
89
|
+
const { status, streamGenerate } = useBrowserLLM(
|
|
90
|
+
"Llama-3.2-1B-Instruct-q4f16_1-MLC",
|
|
91
|
+
);
|
|
92
|
+
const [text, setText] = useState("");
|
|
93
|
+
|
|
94
|
+
async function handleAsk() {
|
|
95
|
+
setText("");
|
|
96
|
+
for await (const token of streamGenerate([
|
|
97
|
+
{ role: "user", content: "Write a haiku about the ocean." },
|
|
98
|
+
])) {
|
|
99
|
+
setText((prev) => prev + token);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
return (
|
|
104
|
+
<div>
|
|
105
|
+
<button disabled={status !== "ready"} onClick={handleAsk}>
|
|
106
|
+
Ask
|
|
107
|
+
</button>
|
|
108
|
+
<p>{text}</p>
|
|
109
|
+
</div>
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Breaking out of the loop early (e.g. a `break` inside the `for await`)
|
|
115
|
+
still stops the underlying worker's inference — you don't need to call
|
|
116
|
+
`abort()` yourself in that case, though you can if you want to stop
|
|
117
|
+
generation from somewhere other than the loop itself (see below).
|
|
118
|
+
|
|
119
|
+
## Cancellation
|
|
120
|
+
|
|
121
|
+
Both `generate()` and `streamGenerate()` share one `abort()` function that
|
|
122
|
+
stops generation in the worker itself, not just the promise on the main
|
|
123
|
+
thread:
|
|
124
|
+
|
|
125
|
+
```tsx
|
|
126
|
+
import { useBrowserLLM } from "use-browser-llm";
|
|
127
|
+
|
|
128
|
+
export function CancellableExample() {
|
|
129
|
+
const { status, streamGenerate, abort, isGenerating } = useBrowserLLM(
|
|
130
|
+
"Llama-3.2-1B-Instruct-q4f16_1-MLC",
|
|
131
|
+
);
|
|
132
|
+
|
|
133
|
+
async function handleAsk() {
|
|
134
|
+
for await (const token of streamGenerate([
|
|
135
|
+
{ role: "user", content: "Tell me a long story." },
|
|
136
|
+
])) {
|
|
137
|
+
console.log(token);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
return (
|
|
142
|
+
<div>
|
|
143
|
+
<button disabled={status !== "ready"} onClick={handleAsk}>
|
|
144
|
+
Ask
|
|
145
|
+
</button>
|
|
146
|
+
<button disabled={!isGenerating} onClick={abort}>
|
|
147
|
+
Stop
|
|
148
|
+
</button>
|
|
149
|
+
</div>
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
`isGenerating` is `true` for the duration of any `generate()` or
|
|
155
|
+
`streamGenerate()` call, and only one can run at a time — calling either
|
|
156
|
+
while one is already in flight rejects with `HookBusyError` rather than
|
|
157
|
+
silently queuing or corrupting state.
|
|
158
|
+
|
|
159
|
+
## Handling unsupported browsers
|
|
160
|
+
|
|
161
|
+
When the browser has no usable WebGPU (missing entirely, no adapter, or
|
|
162
|
+
only a software fallback adapter), `status` resolves straight to
|
|
163
|
+
`"unsupported"` — it never passes through `"loading"`, and no worker is
|
|
164
|
+
ever created. Check for it and render your own fallback UI (this package
|
|
165
|
+
ships none):
|
|
166
|
+
|
|
167
|
+
```tsx
|
|
168
|
+
import { useBrowserLLM } from "use-browser-llm";
|
|
169
|
+
|
|
170
|
+
export function UnsupportedFallbackExample() {
|
|
171
|
+
const { status } = useBrowserLLM("Llama-3.2-1B-Instruct-q4f16_1-MLC");
|
|
172
|
+
|
|
173
|
+
if (status === "unsupported") {
|
|
174
|
+
return (
|
|
175
|
+
<p>
|
|
176
|
+
Your browser doesn't support the local AI features on this page. Try the
|
|
177
|
+
latest Chrome or Edge.
|
|
178
|
+
</p>
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// ...the rest of your component
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
## Cache status
|
|
188
|
+
|
|
189
|
+
`cacheStatus` tells you whether the model was already downloaded in a
|
|
190
|
+
previous session, so you can show a different message for "instant load"
|
|
191
|
+
vs. "first-time multi-gigabyte download":
|
|
192
|
+
|
|
193
|
+
```tsx
|
|
194
|
+
const { cacheStatus } = useBrowserLLM("Llama-3.2-1B-Instruct-q4f16_1-MLC");
|
|
195
|
+
// "idle" | "checking" | "cached" | "downloading"
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
`cacheStatus` is available before `status` reaches `"ready"`, but for a
|
|
199
|
+
model that loads very quickly it can stay at `"checking"` if the load
|
|
200
|
+
finishes before the cache check itself resolves — check `status` first for
|
|
201
|
+
anything load-gating; `cacheStatus` is informational.
|
|
202
|
+
|
|
203
|
+
## Errors
|
|
204
|
+
|
|
205
|
+
- `HookNotReadyError` — rejected by `generate()` (immediately) or thrown
|
|
206
|
+
from `streamGenerate()`'s async generator (on its first iteration —
|
|
207
|
+
generators are lazy, so the check runs when you start consuming it, not
|
|
208
|
+
when you call the function) when called before `status === "ready"`.
|
|
209
|
+
- `HookBusyError` — same timing as above, when calling `generate()`/
|
|
210
|
+
`streamGenerate()` while one is already in flight.
|
|
211
|
+
- `UnsupportedError` — the `error` value when `status === "unsupported"`;
|
|
212
|
+
has a `.reason` field (`"no-navigator-gpu"` | `"no-adapter"` |
|
|
213
|
+
`"fallback-adapter"`).
|
|
214
|
+
- `WorkerCrashError` — the `error` value when `status === "error"` and the
|
|
215
|
+
cause was the worker crashing or becoming unresponsive, rather than a
|
|
216
|
+
normal model-load failure.
|
|
217
|
+
|
|
218
|
+
All are exported from `use-browser-llm` for `instanceof` checks.
|
|
219
|
+
|
|
220
|
+
## API reference
|
|
221
|
+
|
|
222
|
+
```ts
|
|
223
|
+
function useBrowserLLM(modelId: string | undefined): {
|
|
224
|
+
status: "idle" | "loading" | "ready" | "error" | "unsupported";
|
|
225
|
+
progress: number; // 0-1, meaningful only while status === "loading"
|
|
226
|
+
error: Error | null; // set for "error" and "unsupported"
|
|
227
|
+
cacheStatus: "idle" | "checking" | "cached" | "downloading";
|
|
228
|
+
isGenerating: boolean;
|
|
229
|
+
generationError: Error | null;
|
|
230
|
+
generate(messages: ChatMessage[]): Promise<string>;
|
|
231
|
+
streamGenerate(messages: ChatMessage[]): AsyncGenerator<string, void, void>;
|
|
232
|
+
abort(): void;
|
|
233
|
+
};
|
|
234
|
+
|
|
235
|
+
type ChatMessage = {
|
|
236
|
+
role: "system" | "user" | "assistant";
|
|
237
|
+
content: string;
|
|
238
|
+
};
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
`ChatMessage` is a plain, self-contained type — e.g.
|
|
242
|
+
`{ role: "user", content: "..." }` — not a re-export of
|
|
243
|
+
`@mlc-ai/web-llm`'s own message type, so this package's public API never
|
|
244
|
+
requires you to know anything about the underlying engine.
|
|
245
|
+
|
|
246
|
+
## License
|
|
247
|
+
|
|
248
|
+
MIT — see [LICENSE](./LICENSE).
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public message shape for generate()/streamGenerate() — deliberately a
|
|
3
|
+
* narrower, self-contained re-declaration of the common case from
|
|
4
|
+
* @mlc-ai/web-llm's ChatCompletionMessageParam (system/user/assistant with
|
|
5
|
+
* plain string content), not an import of it. Importing web-llm's type
|
|
6
|
+
* directly would put `@mlc-ai/web-llm` in this package's public .d.ts
|
|
7
|
+
* output, which the package's own architecture rule (only worker.ts ever
|
|
8
|
+
* references web-llm at the type OR value level) exists to prevent.
|
|
9
|
+
*
|
|
10
|
+
* This covers the vast majority of real usage (see README). It's
|
|
11
|
+
* structurally assignable to ChatCompletionMessageParam[] wherever this
|
|
12
|
+
* package passes messages into the real engine, so nothing is lost at the
|
|
13
|
+
* call site — a consumer just doesn't see web-llm's richer (tool-call,
|
|
14
|
+
* image-content-part) variants reflected in their editor, since this
|
|
15
|
+
* package doesn't expose them.
|
|
16
|
+
*/
|
|
17
|
+
type ChatRole = "system" | "user" | "assistant";
|
|
18
|
+
interface ChatMessage {
|
|
19
|
+
role: ChatRole;
|
|
20
|
+
content: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
type ModelLoadStatus = "idle" | "loading" | "ready" | "error" | "unsupported";
|
|
24
|
+
type CacheStatus = "idle" | "checking" | "cached" | "downloading";
|
|
25
|
+
interface ModelLoadState {
|
|
26
|
+
status: ModelLoadStatus;
|
|
27
|
+
/** 0-1 while status === "loading"; meaningless otherwise. */
|
|
28
|
+
progress: number;
|
|
29
|
+
/** Set for both "error" and "unsupported" — check `status` first, this
|
|
30
|
+
* carries the detail (an UnsupportedError, WorkerCrashError, or
|
|
31
|
+
* whatever the engine/worker rejected with). */
|
|
32
|
+
error: Error | null;
|
|
33
|
+
/** Whether modelId was already in IndexedDB before this load started.
|
|
34
|
+
* "checking" until the (non-blocking) cache check resolves; settles to
|
|
35
|
+
* "cached" or "downloading" before status reaches "ready" — but is not
|
|
36
|
+
* itself a loading gate, so it's still meaningful to read after ready.
|
|
37
|
+
* Edge case: for a fast/cached model, `status` can reach "ready" before
|
|
38
|
+
* this check resolves at all — cacheStatus then stays "checking"
|
|
39
|
+
* indefinitely for that load (the reducer intentionally drops a late
|
|
40
|
+
* resolution once status is no longer "loading"; see the "cache-status"
|
|
41
|
+
* case below). */
|
|
42
|
+
cacheStatus: CacheStatus;
|
|
43
|
+
}
|
|
44
|
+
interface GenerationState {
|
|
45
|
+
isGenerating: boolean;
|
|
46
|
+
generationError: Error | null;
|
|
47
|
+
}
|
|
48
|
+
interface UseBrowserLLMResult extends ModelLoadState, GenerationState {
|
|
49
|
+
generate(messages: ChatMessage[]): Promise<string>;
|
|
50
|
+
streamGenerate(messages: ChatMessage[]): AsyncGenerator<string, void, void>;
|
|
51
|
+
/** Stops an in-flight generate()/streamGenerate() call, including the
|
|
52
|
+
* worker's inference loop — not just the UI-visible promise. */
|
|
53
|
+
abort(): void;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Model-loading state machine (P1-04) plus generate/streamGenerate/abort
|
|
57
|
+
* (P1-05). Cache-status (P1-06) and the unsupported-browser path (P1-07)
|
|
58
|
+
* extend this hook in later tasks.
|
|
59
|
+
*/
|
|
60
|
+
declare function useBrowserLLM(modelId: string | undefined): UseBrowserLLMResult;
|
|
61
|
+
|
|
62
|
+
declare class HookNotReadyError extends Error {
|
|
63
|
+
constructor();
|
|
64
|
+
}
|
|
65
|
+
declare class HookBusyError extends Error {
|
|
66
|
+
constructor();
|
|
67
|
+
}
|
|
68
|
+
declare class UnsupportedError extends Error {
|
|
69
|
+
readonly reason: string;
|
|
70
|
+
constructor(reason: string);
|
|
71
|
+
}
|
|
72
|
+
declare class WorkerCrashError extends Error {
|
|
73
|
+
constructor(message: string);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
declare const USE_BROWSER_LLM_VERSION = "0.1.0";
|
|
77
|
+
|
|
78
|
+
export { type CacheStatus, type ChatMessage, type ChatRole, type GenerationState, HookBusyError, HookNotReadyError, type ModelLoadState, type ModelLoadStatus, USE_BROWSER_LLM_VERSION, UnsupportedError, type UseBrowserLLMResult, WorkerCrashError, useBrowserLLM };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
// src/use-browser-llm.ts
|
|
2
|
+
import { useCallback, useEffect, useReducer, useRef, useState } from "react";
|
|
3
|
+
|
|
4
|
+
// src/engine-client.ts
|
|
5
|
+
import * as Comlink from "comlink";
|
|
6
|
+
|
|
7
|
+
// src/errors.ts
|
|
8
|
+
var HookNotReadyError = class extends Error {
|
|
9
|
+
constructor() {
|
|
10
|
+
super("useBrowserLLM: cannot generate before the model is ready");
|
|
11
|
+
this.name = "HookNotReadyError";
|
|
12
|
+
}
|
|
13
|
+
};
|
|
14
|
+
var HookBusyError = class extends Error {
|
|
15
|
+
constructor() {
|
|
16
|
+
super("useBrowserLLM: a generation is already in progress");
|
|
17
|
+
this.name = "HookBusyError";
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
var UnsupportedError = class extends Error {
|
|
21
|
+
reason;
|
|
22
|
+
constructor(reason) {
|
|
23
|
+
super(`useBrowserLLM: WebGPU is unsupported in this browser (${reason})`);
|
|
24
|
+
this.name = "UnsupportedError";
|
|
25
|
+
this.reason = reason;
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
var WorkerCrashError = class extends Error {
|
|
29
|
+
constructor(message) {
|
|
30
|
+
super(`useBrowserLLM: the worker crashed or stopped responding (${message})`);
|
|
31
|
+
this.name = "WorkerCrashError";
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
// src/engine-client.ts
|
|
36
|
+
function createEngineClient() {
|
|
37
|
+
const worker = new Worker(new URL("./worker.js", import.meta.url), {
|
|
38
|
+
type: "module"
|
|
39
|
+
});
|
|
40
|
+
const remote = Comlink.wrap(worker);
|
|
41
|
+
return {
|
|
42
|
+
loadModel(modelId, onProgress) {
|
|
43
|
+
return remote.loadModel(modelId, Comlink.proxy(onProgress ?? (() => {
|
|
44
|
+
})));
|
|
45
|
+
},
|
|
46
|
+
generate(messages) {
|
|
47
|
+
return remote.generate(messages);
|
|
48
|
+
},
|
|
49
|
+
streamGenerate(messages, onToken) {
|
|
50
|
+
return remote.streamGenerate(messages, Comlink.proxy(onToken));
|
|
51
|
+
},
|
|
52
|
+
abort() {
|
|
53
|
+
return remote.abort();
|
|
54
|
+
},
|
|
55
|
+
unload() {
|
|
56
|
+
return remote.unload();
|
|
57
|
+
},
|
|
58
|
+
checkCache(modelId) {
|
|
59
|
+
return remote.checkCache(modelId);
|
|
60
|
+
},
|
|
61
|
+
terminate() {
|
|
62
|
+
worker.terminate();
|
|
63
|
+
},
|
|
64
|
+
onCrash(listener) {
|
|
65
|
+
const handleError = (event) => {
|
|
66
|
+
listener(new WorkerCrashError(event.message || "uncaught exception"));
|
|
67
|
+
};
|
|
68
|
+
const handleMessageError = () => {
|
|
69
|
+
listener(new WorkerCrashError("failed to deserialize a worker message"));
|
|
70
|
+
};
|
|
71
|
+
worker.addEventListener("error", handleError);
|
|
72
|
+
worker.addEventListener("messageerror", handleMessageError);
|
|
73
|
+
return () => {
|
|
74
|
+
worker.removeEventListener("error", handleError);
|
|
75
|
+
worker.removeEventListener("messageerror", handleMessageError);
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// src/detect-webgpu.ts
|
|
82
|
+
async function detectWebGPUSupport() {
|
|
83
|
+
if (typeof navigator === "undefined" || !("gpu" in navigator)) {
|
|
84
|
+
return { supported: false, reason: "no-navigator-gpu" };
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
const adapter = await navigator.gpu.requestAdapter();
|
|
88
|
+
if (!adapter) {
|
|
89
|
+
return { supported: false, reason: "no-adapter" };
|
|
90
|
+
}
|
|
91
|
+
if (adapter.info.isFallbackAdapter) {
|
|
92
|
+
return { supported: false, reason: "fallback-adapter" };
|
|
93
|
+
}
|
|
94
|
+
return { supported: true };
|
|
95
|
+
} catch {
|
|
96
|
+
return { supported: false, reason: "no-adapter" };
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// src/to-async-generator.ts
|
|
101
|
+
async function* toAsyncGenerator(run) {
|
|
102
|
+
const queue = [];
|
|
103
|
+
let resolveNext = null;
|
|
104
|
+
let done = false;
|
|
105
|
+
let error = null;
|
|
106
|
+
run((token) => {
|
|
107
|
+
queue.push(token);
|
|
108
|
+
resolveNext?.();
|
|
109
|
+
resolveNext = null;
|
|
110
|
+
}).then(
|
|
111
|
+
() => {
|
|
112
|
+
done = true;
|
|
113
|
+
resolveNext?.();
|
|
114
|
+
resolveNext = null;
|
|
115
|
+
},
|
|
116
|
+
(err) => {
|
|
117
|
+
error = err;
|
|
118
|
+
done = true;
|
|
119
|
+
resolveNext?.();
|
|
120
|
+
resolveNext = null;
|
|
121
|
+
}
|
|
122
|
+
);
|
|
123
|
+
while (true) {
|
|
124
|
+
while (queue.length > 0) {
|
|
125
|
+
yield queue.shift();
|
|
126
|
+
}
|
|
127
|
+
if (done) {
|
|
128
|
+
if (error) {
|
|
129
|
+
throw error;
|
|
130
|
+
}
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
await new Promise((resolve) => {
|
|
134
|
+
resolveNext = resolve;
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// src/use-browser-llm.ts
|
|
140
|
+
var initialState = {
|
|
141
|
+
status: "idle",
|
|
142
|
+
progress: 0,
|
|
143
|
+
error: null,
|
|
144
|
+
cacheStatus: "idle"
|
|
145
|
+
};
|
|
146
|
+
var LOAD_INACTIVITY_TIMEOUT_MS = 3e4;
|
|
147
|
+
function reducer(state, action) {
|
|
148
|
+
switch (action.type) {
|
|
149
|
+
case "reset":
|
|
150
|
+
return initialState;
|
|
151
|
+
case "load-start":
|
|
152
|
+
return {
|
|
153
|
+
status: "loading",
|
|
154
|
+
progress: 0,
|
|
155
|
+
error: null,
|
|
156
|
+
cacheStatus: "checking"
|
|
157
|
+
};
|
|
158
|
+
case "progress":
|
|
159
|
+
return { ...state, progress: action.progress };
|
|
160
|
+
case "cache-status":
|
|
161
|
+
if (state.status !== "loading") {
|
|
162
|
+
return state;
|
|
163
|
+
}
|
|
164
|
+
return { ...state, cacheStatus: action.cacheStatus };
|
|
165
|
+
case "ready":
|
|
166
|
+
return { ...state, status: "ready", progress: 1, error: null };
|
|
167
|
+
case "error":
|
|
168
|
+
return { ...state, status: "error", error: action.error };
|
|
169
|
+
case "unsupported":
|
|
170
|
+
return {
|
|
171
|
+
status: "unsupported",
|
|
172
|
+
progress: 0,
|
|
173
|
+
error: action.error,
|
|
174
|
+
cacheStatus: "idle"
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
function useBrowserLLM(modelId) {
|
|
179
|
+
const [state, dispatch] = useReducer(reducer, initialState);
|
|
180
|
+
const clientRef = useRef(null);
|
|
181
|
+
const statusRef = useRef(state.status);
|
|
182
|
+
statusRef.current = state.status;
|
|
183
|
+
const isGeneratingRef = useRef(false);
|
|
184
|
+
const [isGenerating, setIsGenerating] = useState(false);
|
|
185
|
+
const [generationError, setGenerationError] = useState(null);
|
|
186
|
+
useEffect(() => {
|
|
187
|
+
if (!modelId) {
|
|
188
|
+
dispatch({ type: "reset" });
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
let cancelled = false;
|
|
192
|
+
let watchdog = null;
|
|
193
|
+
let unsubscribeCrash = null;
|
|
194
|
+
const clearWatchdog = () => {
|
|
195
|
+
if (watchdog !== null) {
|
|
196
|
+
clearTimeout(watchdog);
|
|
197
|
+
watchdog = null;
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
const resetWatchdog = () => {
|
|
201
|
+
clearWatchdog();
|
|
202
|
+
watchdog = setTimeout(() => {
|
|
203
|
+
if (!cancelled) {
|
|
204
|
+
clientRef.current?.terminate();
|
|
205
|
+
clientRef.current = null;
|
|
206
|
+
dispatch({
|
|
207
|
+
type: "error",
|
|
208
|
+
error: new WorkerCrashError(
|
|
209
|
+
`no progress for ${LOAD_INACTIVITY_TIMEOUT_MS / 1e3}s \u2014 the worker may have been killed by the browser (e.g. out of memory)`
|
|
210
|
+
)
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
}, LOAD_INACTIVITY_TIMEOUT_MS);
|
|
214
|
+
};
|
|
215
|
+
clientRef.current?.terminate();
|
|
216
|
+
clientRef.current = null;
|
|
217
|
+
void (async () => {
|
|
218
|
+
const support = await detectWebGPUSupport();
|
|
219
|
+
if (cancelled) {
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
if (!support.supported) {
|
|
223
|
+
dispatch({
|
|
224
|
+
type: "unsupported",
|
|
225
|
+
error: new UnsupportedError(support.reason)
|
|
226
|
+
});
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
const client = createEngineClient();
|
|
230
|
+
clientRef.current = client;
|
|
231
|
+
dispatch({ type: "load-start" });
|
|
232
|
+
resetWatchdog();
|
|
233
|
+
unsubscribeCrash = client.onCrash((error) => {
|
|
234
|
+
if (!cancelled) {
|
|
235
|
+
clearWatchdog();
|
|
236
|
+
dispatch({ type: "error", error });
|
|
237
|
+
}
|
|
238
|
+
});
|
|
239
|
+
client.loadModel(modelId, (report) => {
|
|
240
|
+
if (!cancelled) {
|
|
241
|
+
resetWatchdog();
|
|
242
|
+
dispatch({ type: "progress", progress: report.progress });
|
|
243
|
+
}
|
|
244
|
+
}).then(() => {
|
|
245
|
+
if (!cancelled) {
|
|
246
|
+
clearWatchdog();
|
|
247
|
+
dispatch({ type: "ready" });
|
|
248
|
+
}
|
|
249
|
+
}).catch((err) => {
|
|
250
|
+
if (!cancelled) {
|
|
251
|
+
clearWatchdog();
|
|
252
|
+
dispatch({
|
|
253
|
+
type: "error",
|
|
254
|
+
error: err instanceof Error ? err : new Error(String(err))
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
});
|
|
258
|
+
client.checkCache(modelId).then((isCached) => {
|
|
259
|
+
if (!cancelled) {
|
|
260
|
+
dispatch({
|
|
261
|
+
type: "cache-status",
|
|
262
|
+
cacheStatus: isCached ? "cached" : "downloading"
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
}).catch(() => {
|
|
266
|
+
});
|
|
267
|
+
})();
|
|
268
|
+
return () => {
|
|
269
|
+
cancelled = true;
|
|
270
|
+
clearWatchdog();
|
|
271
|
+
unsubscribeCrash?.();
|
|
272
|
+
clientRef.current?.terminate();
|
|
273
|
+
clientRef.current = null;
|
|
274
|
+
};
|
|
275
|
+
}, [modelId]);
|
|
276
|
+
const generate = useCallback(
|
|
277
|
+
async (messages) => {
|
|
278
|
+
if (statusRef.current !== "ready" || !clientRef.current) {
|
|
279
|
+
throw new HookNotReadyError();
|
|
280
|
+
}
|
|
281
|
+
if (isGeneratingRef.current) {
|
|
282
|
+
throw new HookBusyError();
|
|
283
|
+
}
|
|
284
|
+
isGeneratingRef.current = true;
|
|
285
|
+
setIsGenerating(true);
|
|
286
|
+
setGenerationError(null);
|
|
287
|
+
try {
|
|
288
|
+
return await clientRef.current.generate(messages);
|
|
289
|
+
} catch (err) {
|
|
290
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
291
|
+
setGenerationError(error);
|
|
292
|
+
throw error;
|
|
293
|
+
} finally {
|
|
294
|
+
isGeneratingRef.current = false;
|
|
295
|
+
setIsGenerating(false);
|
|
296
|
+
}
|
|
297
|
+
},
|
|
298
|
+
[]
|
|
299
|
+
);
|
|
300
|
+
const streamGenerate = useCallback(
|
|
301
|
+
(messages) => streamGenerateImpl(
|
|
302
|
+
statusRef,
|
|
303
|
+
clientRef,
|
|
304
|
+
isGeneratingRef,
|
|
305
|
+
setIsGenerating,
|
|
306
|
+
setGenerationError,
|
|
307
|
+
messages
|
|
308
|
+
),
|
|
309
|
+
[]
|
|
310
|
+
);
|
|
311
|
+
const abort = useCallback(() => {
|
|
312
|
+
clientRef.current?.abort().catch(() => {
|
|
313
|
+
});
|
|
314
|
+
}, []);
|
|
315
|
+
return {
|
|
316
|
+
...state,
|
|
317
|
+
isGenerating,
|
|
318
|
+
generationError,
|
|
319
|
+
generate,
|
|
320
|
+
streamGenerate,
|
|
321
|
+
abort
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
async function* streamGenerateImpl(statusRef, clientRef, isGeneratingRef, setIsGenerating, setGenerationError, messages) {
|
|
325
|
+
if (statusRef.current !== "ready" || !clientRef.current) {
|
|
326
|
+
throw new HookNotReadyError();
|
|
327
|
+
}
|
|
328
|
+
if (isGeneratingRef.current) {
|
|
329
|
+
throw new HookBusyError();
|
|
330
|
+
}
|
|
331
|
+
const client = clientRef.current;
|
|
332
|
+
isGeneratingRef.current = true;
|
|
333
|
+
setIsGenerating(true);
|
|
334
|
+
setGenerationError(null);
|
|
335
|
+
try {
|
|
336
|
+
yield* toAsyncGenerator((onToken) => client.streamGenerate(messages, onToken));
|
|
337
|
+
} catch (err) {
|
|
338
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
339
|
+
setGenerationError(error);
|
|
340
|
+
throw error;
|
|
341
|
+
} finally {
|
|
342
|
+
isGeneratingRef.current = false;
|
|
343
|
+
setIsGenerating(false);
|
|
344
|
+
client.abort().catch(() => {
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// src/index.ts
|
|
350
|
+
var USE_BROWSER_LLM_VERSION = "0.1.0";
|
|
351
|
+
export {
|
|
352
|
+
HookBusyError,
|
|
353
|
+
HookNotReadyError,
|
|
354
|
+
USE_BROWSER_LLM_VERSION,
|
|
355
|
+
UnsupportedError,
|
|
356
|
+
WorkerCrashError,
|
|
357
|
+
useBrowserLLM
|
|
358
|
+
};
|
package/dist/worker.d.ts
ADDED
package/dist/worker.js
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// src/worker.ts
|
|
2
|
+
import * as Comlink from "comlink";
|
|
3
|
+
import { hasModelInCache, MLCEngine } from "@mlc-ai/web-llm";
|
|
4
|
+
|
|
5
|
+
// src/engine-api-factory.ts
|
|
6
|
+
function createEngineAPI(engine, checkCache) {
|
|
7
|
+
return {
|
|
8
|
+
async loadModel(modelId, onProgress) {
|
|
9
|
+
engine.setInitProgressCallback(onProgress);
|
|
10
|
+
await engine.reload(modelId);
|
|
11
|
+
},
|
|
12
|
+
async generate(messages) {
|
|
13
|
+
const completion = await engine.chatCompletion({
|
|
14
|
+
messages,
|
|
15
|
+
stream: false
|
|
16
|
+
});
|
|
17
|
+
return completion.choices[0]?.message.content ?? "";
|
|
18
|
+
},
|
|
19
|
+
async streamGenerate(messages, onToken) {
|
|
20
|
+
const stream = await engine.chatCompletion({
|
|
21
|
+
messages,
|
|
22
|
+
stream: true
|
|
23
|
+
});
|
|
24
|
+
for await (const chunk of stream) {
|
|
25
|
+
const token = chunk.choices[0]?.delta.content;
|
|
26
|
+
if (token) {
|
|
27
|
+
onToken(token);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
async abort() {
|
|
32
|
+
await engine.interruptGenerate();
|
|
33
|
+
},
|
|
34
|
+
async unload() {
|
|
35
|
+
await engine.unload();
|
|
36
|
+
},
|
|
37
|
+
async checkCache(modelId) {
|
|
38
|
+
return checkCache(modelId);
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// src/worker.ts
|
|
44
|
+
Comlink.expose(createEngineAPI(new MLCEngine(), hasModelInCache));
|
package/package.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "use-browser-llm",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Headless React hook for running LLMs locally in the browser via WebGPU, with zero backend and complete privacy.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "git+https://github.com/Muhammad-UmairAli/use-browser-llm.git"
|
|
9
|
+
},
|
|
10
|
+
"homepage": "https://github.com/Muhammad-UmairAli/use-browser-llm#readme",
|
|
11
|
+
"bugs": {
|
|
12
|
+
"url": "https://github.com/Muhammad-UmairAli/use-browser-llm/issues"
|
|
13
|
+
},
|
|
14
|
+
"type": "module",
|
|
15
|
+
"sideEffects": false,
|
|
16
|
+
"engines": {
|
|
17
|
+
"node": ">=18"
|
|
18
|
+
},
|
|
19
|
+
"files": [
|
|
20
|
+
"dist"
|
|
21
|
+
],
|
|
22
|
+
"exports": {
|
|
23
|
+
".": {
|
|
24
|
+
"types": "./dist/index.d.ts",
|
|
25
|
+
"import": "./dist/index.js"
|
|
26
|
+
},
|
|
27
|
+
"./package.json": "./package.json"
|
|
28
|
+
},
|
|
29
|
+
"main": "./dist/index.js",
|
|
30
|
+
"types": "./dist/index.d.ts",
|
|
31
|
+
"peerDependencies": {
|
|
32
|
+
"react": ">=18"
|
|
33
|
+
},
|
|
34
|
+
"dependencies": {
|
|
35
|
+
"@mlc-ai/web-llm": "^0.2.79",
|
|
36
|
+
"comlink": "^4.4.2"
|
|
37
|
+
},
|
|
38
|
+
"devDependencies": {
|
|
39
|
+
"@testing-library/dom": "^10.4.1",
|
|
40
|
+
"@testing-library/react": "^16.3.2",
|
|
41
|
+
"@types/node": "^22.10.5",
|
|
42
|
+
"@types/react": "^18.3.18",
|
|
43
|
+
"@types/react-dom": "^18.3.7",
|
|
44
|
+
"@vitest/browser": "2.1.9",
|
|
45
|
+
"@webgpu/types": "^0.1.71",
|
|
46
|
+
"eslint": "^9.18.0",
|
|
47
|
+
"eslint-config-prettier": "^9.1.0",
|
|
48
|
+
"jsdom": "^29.1.1",
|
|
49
|
+
"playwright": "^1.61.1",
|
|
50
|
+
"prettier": "^3.4.2",
|
|
51
|
+
"react": "^18.3.1",
|
|
52
|
+
"react-dom": "^18.3.1",
|
|
53
|
+
"tsup": "^8.3.5",
|
|
54
|
+
"typescript": "^5.7.3",
|
|
55
|
+
"typescript-eslint": "^8.19.1",
|
|
56
|
+
"vitest": "^2.1.8"
|
|
57
|
+
},
|
|
58
|
+
"scripts": {
|
|
59
|
+
"build": "tsup",
|
|
60
|
+
"typecheck": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.worker.json --noEmit",
|
|
61
|
+
"lint": "eslint .",
|
|
62
|
+
"test": "vitest run",
|
|
63
|
+
"test:integration": "vitest run --config vitest.browser.config.ts"
|
|
64
|
+
}
|
|
65
|
+
}
|