runntime 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -4
- package/dist/{classifier-ByH6OI6h.mjs → classifier-C6ZR3cyJ.mjs} +38576 -22654
- package/dist/{config-UyxIidZc.d.mts → config-DZyZ2TdH.d.mts} +21 -8
- package/dist/zoo/index.d.mts +206 -63
- package/dist/zoo/index.mjs +2217 -473
- package/dist/zoo/transformers/index.d.mts +10 -10
- package/dist/zoo/transformers/index.mjs +14 -16
- package/package.json +13 -5
|
@@ -11,14 +11,16 @@ declare const ACT_CODE: {
|
|
|
11
11
|
readonly gelu: 2;
|
|
12
12
|
readonly silu: 3;
|
|
13
13
|
readonly relu: 4;
|
|
14
|
+
readonly geluTanh: 5;
|
|
14
15
|
};
|
|
15
16
|
type ActName = keyof typeof ACT_CODE;
|
|
16
17
|
type SlotAct = Exclude<ActName, 'none'>;
|
|
17
18
|
//#endregion
|
|
18
19
|
//#region src/core/graph/ops/matmul.d.ts
|
|
19
|
-
/** Options for matmul. `bias` and `addend` are fused epilogue adds
|
|
20
|
-
*
|
|
21
|
-
*
|
|
20
|
+
/** Options for matmul. `bias` and `addend` are fused epilogue adds; on a
|
|
21
|
+
* quantW weight they need the whole weight, no baseRow or scaleBase view.
|
|
22
|
+
* `baseRow` views into a taller weight without copying, so MoE experts stay
|
|
23
|
+
* views into one fused tensor. */
|
|
22
24
|
interface MatmulOpts {
|
|
23
25
|
bias?: Value;
|
|
24
26
|
addend?: Value;
|
|
@@ -49,7 +51,7 @@ interface ValueMeta {
|
|
|
49
51
|
/** The storage buffer behind a materialized Value. */
|
|
50
52
|
type GpuBufferRef = F32Buffer | F16Buffer | U32Buffer;
|
|
51
53
|
/** Every op a pending node can name. */
|
|
52
|
-
type OpName = 'add' | 'mul' | 'sub' | 'swiglu' | 'swigluChunk' | 'rsqrt' | 'addScalar' | 'sigmoid' | 'mulScalar' | 'clampScalar' | 'tanh' | 'gelu' | 'silu' | 'asinh' | 'matmul' | 'matmulQuantW' | '
|
|
54
|
+
type OpName = 'add' | 'mul' | 'sub' | 'swiglu' | 'swigluChunk' | 'rsqrt' | 'addScalar' | 'sigmoid' | 'mulScalar' | 'clampScalar' | 'tanh' | 'gelu' | 'silu' | 'asinh' | 'matmul' | 'matmulQuantW' | 'matmulGather' | 'moeUp' | 'moeDown' | 'argmaxDot' | 'attn' | 'localAttention' | 'rope' | 'conv1d' | 'gluConv1d' | 'convTranspose2d' | 'conv2dHwc4' | 'maxPool2dHwc4' | 'upsample2dHwc4' | 'avgPool2dHwc4' | 'pad2dHwc4' | 'resizeBilinearHwc4' | 'channelAffineHwc4' | 'concatChannelsHwc4' | 'sliceChannelsHwc4' | 'toHwc4' | 'toChw' | 'ssmScanProject' | 'ssmSelectiveScan' | 'ssmScanMerge' | 'softmax' | 'mean' | 'layerNorm' | 'rmsNorm' | 'topk' | 'astype' | 'transpose' | 'sliceCols' | 'concatCols' | 'sliceRows' | 'concatRows' | 'reshape' | 'sliceChannels' | 'concatChannels' | 'writeRows' | 'gatherRowsFrom' | 'gatherRows';
|
|
53
55
|
/** The recipe for a not-yet-computed Value. Inputs may themselves be pending,
|
|
54
56
|
* so chained ops form the DAG eval() walks. */
|
|
55
57
|
interface PendingOp {
|
|
@@ -154,19 +156,30 @@ interface FusedReadback {
|
|
|
154
156
|
/** toArray with the copy fused into the eval's own submit. */
|
|
155
157
|
toArrayFused(value: Value): Promise<Float32Array>;
|
|
156
158
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
159
|
+
/** Frees pooled buffers a model has stopped using. */
|
|
160
|
+
interface TrimExecutor {
|
|
161
|
+
/** Destroys the free pooled buffers of every size no eval has used since
|
|
162
|
+
* the last trim. A model whose input length varies calls it after each
|
|
163
|
+
* call, or its pool keeps a set of buffers for every length it has seen.
|
|
164
|
+
* Does nothing once captured: the replay owns the pool's buffers. */
|
|
165
|
+
trim(): void;
|
|
166
|
+
}
|
|
167
|
+
type RunntimeExecutor = Executor & Readback & Releaser & CaptureExecutor & FusedReadback & TrimExecutor;
|
|
168
|
+
/** Sets the device every model runs on and returns its executor. Call it
|
|
169
|
+
* once at startup, before the first create<Task>(). */
|
|
160
170
|
declare function initRunntime(root: TgpuRoot): RunntimeExecutor;
|
|
161
171
|
/** Capture and replay: records one frame's dispatches, then re-submits them
|
|
162
172
|
* without walking the graph again. Fixed-shape forwards only. */
|
|
163
173
|
interface CaptureExecutor {
|
|
164
174
|
/** Runs `fn` once, recording every dispatch. The executor is replay-only
|
|
165
|
-
* afterwards. Keep `targets` — their buffers stay valid across replays — and
|
|
175
|
+
* afterwards, until dispose(), which also ends the replays. Keep `targets` — their buffers stay valid across replays — and
|
|
166
176
|
* overwrite inputs in place before each replay. */
|
|
167
177
|
captureFrame<T>(fn: () => T): {
|
|
168
178
|
targets: T;
|
|
169
179
|
replay: () => void;
|
|
180
|
+
/** Records the frame's dispatches into `encoder` without submitting, so
|
|
181
|
+
* other work can run before and after it in one command buffer. */
|
|
182
|
+
encodeReplay: (encoder: GPUCommandEncoder) => void;
|
|
170
183
|
/** Replays once with a timestamp pair per dispatch. Per-dispatch passes
|
|
171
184
|
* remove overlap, so read the sum as relative attribution only. */
|
|
172
185
|
profileReplay: () => Promise<KernelTiming[] | undefined>;
|
package/dist/zoo/index.d.mts
CHANGED
|
@@ -1,17 +1,15 @@
|
|
|
1
|
-
import { n as initRunntime, t as Yolo26Variant } from "../config-
|
|
1
|
+
import { n as initRunntime, t as Yolo26Variant } from "../config-DZyZ2TdH.mjs";
|
|
2
|
+
import { RenderFlag, SampledFlag, TgpuBuffer, TgpuCommandEncoder, TgpuTexture, d } from "typegpu";
|
|
2
3
|
//#region src/core/weights/cache.d.ts
|
|
3
|
-
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* entries are treated as misses (crash artifacts of interrupted writes). */
|
|
4
|
+
/** Keeps downloaded weights between visits. createOpfsCache() is the
|
|
5
|
+
* built-in one; any object with these methods works. An entry with no bytes
|
|
6
|
+
* counts as missing. */
|
|
7
7
|
interface WeightCache {
|
|
8
8
|
get(key: string): Promise<ArrayBuffer | undefined>;
|
|
9
9
|
getOrCompute(key: string, compute: () => Promise<ArrayBufferView>): Promise<ArrayBuffer>;
|
|
10
|
-
/**
|
|
11
|
-
* share a
|
|
12
|
-
*
|
|
13
|
-
* single key. Callers recovering from one corrupt file MUST use this —
|
|
14
|
-
* clear() throws away every other model the app has cached. */
|
|
10
|
+
/** Drops every entry whose key starts with `prefix`. The entries of one
|
|
11
|
+
* file share a prefix, so this removes one bad download and keeps the
|
|
12
|
+
* other models. */
|
|
15
13
|
deletePrefix(prefix: string): Promise<void>;
|
|
16
14
|
/** Drop everything. For a user-facing "clear cache", not error recovery. */
|
|
17
15
|
clear(): Promise<void>;
|
|
@@ -23,8 +21,24 @@ interface WeightCache {
|
|
|
23
21
|
interface RangeSource {
|
|
24
22
|
read(begin: number, end: number): Promise<Uint8Array>;
|
|
25
23
|
}
|
|
24
|
+
/** Wraps bytes already in memory as a RangeSource, for a `modelPath` that is
|
|
25
|
+
* not a URL. */
|
|
26
26
|
export declare function bufferSource(buf: ArrayBuffer | Uint8Array): RangeSource;
|
|
27
27
|
//#endregion
|
|
28
|
+
//#region src/core/error.d.ts
|
|
29
|
+
/** An error with a `code`. Apps check the code, not the message. The
|
|
30
|
+
* message is for people and can change between versions. */
|
|
31
|
+
export declare const RUNNTIME_ERROR_CODES: readonly ["LOAD_FAILED", "LOAD_ABORTED", "CHECKPOINT_MISMATCH", "UNSUPPORTED_DEVICE", "EXECUTION_FAILED", "INVALID_ARGUMENT", "RESOURCE_DISPOSED", "NOT_INITIALIZED"];
|
|
32
|
+
type RunntimeErrorCode = (typeof RUNNTIME_ERROR_CODES)[number];
|
|
33
|
+
export declare class RunntimeError<C extends RunntimeErrorCode = RunntimeErrorCode> extends Error {
|
|
34
|
+
readonly code: C;
|
|
35
|
+
constructor(code: C, message: string, options?: ErrorOptions);
|
|
36
|
+
}
|
|
37
|
+
/** True when `value` is a RunntimeError, with this `code` if one is given.
|
|
38
|
+
* Checks `name` and `code`, not the class, so it works even when a page
|
|
39
|
+
* loads two copies of the library. */
|
|
40
|
+
export declare function isRunntimeError<C extends RunntimeErrorCode>(value: unknown, code?: C): value is RunntimeError<C>;
|
|
41
|
+
//#endregion
|
|
28
42
|
//#region src/core/weights/opfsCache.d.ts
|
|
29
43
|
/** OPFS-backed cache; returns undefined when OPFS is unavailable. */
|
|
30
44
|
export declare function createOpfsCache(namespace: string): Promise<WeightCache | undefined>;
|
|
@@ -46,6 +60,28 @@ interface LoadOptions {
|
|
|
46
60
|
signal?: AbortSignal;
|
|
47
61
|
}
|
|
48
62
|
//#endregion
|
|
63
|
+
//#region src/zoo/tasks/gpu.d.ts
|
|
64
|
+
/** An image already on the GPU. A texel's [0, 1] value stands for a byte
|
|
65
|
+
* 0–255, as in an ImageBuffer. */
|
|
66
|
+
type GpuImage = GPUTexture | GPUExternalTexture | (TgpuTexture & SampledFlag);
|
|
67
|
+
/** A texture the runner draws into. */
|
|
68
|
+
type GpuImageTarget = GPUTexture | (TgpuTexture & RenderFlag);
|
|
69
|
+
/** A buffer laid out as `Schema`. A raw GPUBuffer must hold the same bytes. */
|
|
70
|
+
type GpuTensor<Schema extends d.BaseData> = GPUBuffer | TgpuBuffer<Schema>;
|
|
71
|
+
/** A command encoder to record into. A TgpuCommandEncoder comes from
|
|
72
|
+
* `root['~unstable'].createCommandEncoder()`. */
|
|
73
|
+
type GpuEncoder = GPUCommandEncoder | TgpuCommandEncoder;
|
|
74
|
+
/** The options every encode<Verb> method takes. Every resource must come from
|
|
75
|
+
* the device passed to initRunntime(). */
|
|
76
|
+
interface EncodeOptions<Source, Target> {
|
|
77
|
+
readonly source: Source;
|
|
78
|
+
readonly target: Target;
|
|
79
|
+
/** Records into this encoder, and the caller finishes and submits it,
|
|
80
|
+
* before disposing the runner. It must not have an open pass. Default: a
|
|
81
|
+
* new encoder, submitted before the method returns. */
|
|
82
|
+
readonly encoder?: GpuEncoder;
|
|
83
|
+
}
|
|
84
|
+
//#endregion
|
|
49
85
|
//#region src/zoo/models.d.ts
|
|
50
86
|
/** Ready-made model configs: the Hugging Face URLs of the weights and the
|
|
51
87
|
* tokenizer, plus the options each variant needs. Pass one to a
|
|
@@ -57,8 +93,33 @@ interface LoadOptions {
|
|
|
57
93
|
* `<size>/<precision>/model.safetensors` with `tokenizer.json` at the
|
|
58
94
|
* root where the model has one. */
|
|
59
95
|
export declare const models: {
|
|
60
|
-
readonly
|
|
61
|
-
/**
|
|
96
|
+
readonly embedding: {
|
|
97
|
+
/** Google's EmbeddingGemma 2: text in 100+ languages and code, images
|
|
98
|
+
* and video in one 768-dim space, inputs up to 2048 tokens. 270M
|
|
99
|
+
* parameters in the text model, 170M in the vision tower. */
|
|
100
|
+
readonly EMBEDDINGGEMMA_2: {
|
|
101
|
+
DEFAULT: {
|
|
102
|
+
modelPath: string;
|
|
103
|
+
visionPath: string;
|
|
104
|
+
audioPath: string;
|
|
105
|
+
tokenizerPath: string;
|
|
106
|
+
};
|
|
107
|
+
F16: {
|
|
108
|
+
dtype: "f16";
|
|
109
|
+
modelPath: string;
|
|
110
|
+
visionPath: string;
|
|
111
|
+
audioPath: string;
|
|
112
|
+
tokenizerPath: string;
|
|
113
|
+
};
|
|
114
|
+
F32: {
|
|
115
|
+
dtype: "f32";
|
|
116
|
+
modelPath: string;
|
|
117
|
+
visionPath: string;
|
|
118
|
+
audioPath: string;
|
|
119
|
+
tokenizerPath: string;
|
|
120
|
+
};
|
|
121
|
+
};
|
|
122
|
+
/** all-MiniLM-L6-v2: English text in 384-dim vectors, 22M parameters. */
|
|
62
123
|
readonly ALL_MINILM_L6_V2: {
|
|
63
124
|
DEFAULT: {
|
|
64
125
|
tokenizerPath: string;
|
|
@@ -256,40 +317,122 @@ export declare const models: {
|
|
|
256
317
|
};
|
|
257
318
|
};
|
|
258
319
|
//#endregion
|
|
259
|
-
//#region src/zoo/
|
|
260
|
-
/**
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
320
|
+
//#region src/zoo/embeddinggemma2/config.d.ts
|
|
321
|
+
/** Token budgets an image may take, most tokens for the most detail. */
|
|
322
|
+
declare const IMAGE_TOKEN_BUDGETS: readonly [70, 140, 280, 560, 1120];
|
|
323
|
+
/** The instruction each task puts before the text (the checkpoint's
|
|
324
|
+
* sentence-transformers prompts). `query` and `document` are the two sides
|
|
325
|
+
* of search: queries get one, the texts searched get the other. */
|
|
326
|
+
declare const EMBEDDINGGEMMA2_PROMPTS: {
|
|
327
|
+
readonly query: "task: search result | query: ";
|
|
328
|
+
readonly document: "title: none | text: ";
|
|
329
|
+
readonly questionAnswering: "task: question answering | query: ";
|
|
330
|
+
readonly factChecking: "task: fact checking | query: ";
|
|
331
|
+
readonly classification: "task: classification | query: ";
|
|
332
|
+
readonly clustering: "task: clustering | query: ";
|
|
333
|
+
readonly similarity: "task: sentence similarity | query: ";
|
|
334
|
+
readonly codeRetrieval: "task: code retrieval | query: ";
|
|
335
|
+
};
|
|
336
|
+
//#endregion
|
|
337
|
+
//#region src/zoo/tasks/cv/image.d.ts
|
|
338
|
+
/** Raw CPU image types shared by every vision task. */
|
|
339
|
+
/** Pixel formats an ImageBuffer can carry: which channels, in what order. */
|
|
340
|
+
type ImageFormat = 'rgb' | 'rgba' | 'bgr' | 'bgra' | 'gray';
|
|
341
|
+
/** A raw image in memory: one row after another, all channels of one pixel
|
|
342
|
+
* together (the way a canvas or a camera gives it). */
|
|
343
|
+
interface ImageBuffer {
|
|
344
|
+
readonly data: Uint8Array | Uint8ClampedArray;
|
|
345
|
+
readonly width: number;
|
|
346
|
+
readonly height: number;
|
|
347
|
+
readonly format: ImageFormat;
|
|
348
|
+
readonly layout: 'hwc';
|
|
349
|
+
}
|
|
350
|
+
/** How an image is fitted into the model's input size.
|
|
351
|
+
* - `stretch`: scale width and height separately; nothing is cut or padded.
|
|
352
|
+
* - `letterbox`: keep the aspect ratio, pad the rest with a flat color.
|
|
353
|
+
* - `crop`: keep the aspect ratio, cut the centered square, then scale. */
|
|
354
|
+
type ResizeMode = 'stretch' | 'letterbox' | 'crop';
|
|
355
|
+
/** Wraps raw bytes as an ImageBuffer. Checks the size and the byte count. */
|
|
356
|
+
export declare function imageBuffer(data: Uint8Array | Uint8ClampedArray, width: number, height: number, format?: ImageFormat): ImageBuffer;
|
|
357
|
+
/** Views canvas ImageData as an ImageBuffer. No copy. */
|
|
358
|
+
export declare function imageBufferFromImageData(img: ImageData): ImageBuffer;
|
|
359
|
+
//#endregion
|
|
360
|
+
//#region src/zoo/tasks/multimodal/embedding.d.ts
|
|
361
|
+
/** What a text is for. Each task adds the instruction the model was
|
|
362
|
+
* trained with. */
|
|
363
|
+
type EmbeddingTask = keyof typeof EMBEDDINGGEMMA2_PROMPTS;
|
|
364
|
+
/** Most tokens one image takes: 70, 140, 280, 560 or 1120. */
|
|
365
|
+
type ImageTokenBudget = (typeof IMAGE_TOKEN_BUDGETS)[number];
|
|
366
|
+
/** One piece of an input: text, an image, a video as its frames, or an
|
|
367
|
+
* audio clip as mono samples at 16 kHz. */
|
|
368
|
+
type EmbedPart = string | {
|
|
369
|
+
readonly image: ImageBuffer;
|
|
370
|
+
} | {
|
|
371
|
+
readonly video: readonly ImageBuffer[];
|
|
372
|
+
} | {
|
|
373
|
+
readonly audio: Float32Array;
|
|
374
|
+
};
|
|
375
|
+
/** What embed() takes: one part, or parts in reading order that together
|
|
376
|
+
* make one vector. */
|
|
377
|
+
type EmbedInput = EmbedPart | readonly EmbedPart[];
|
|
378
|
+
/** What to load. Every field is optional; a missing one comes from the
|
|
379
|
+
* model's DEFAULT, EmbeddingGemma 2 without `modelPath`. */
|
|
380
|
+
interface EmbedderModel {
|
|
381
|
+
/** The safetensors weights of EmbeddingGemma 2 or all-MiniLM-L6-v2. The
|
|
382
|
+
* model is read from the file. */
|
|
264
383
|
readonly modelPath?: ModelPath;
|
|
265
|
-
/** The
|
|
384
|
+
/** The vision tower's weights, for images and video. EmbeddingGemma 2
|
|
385
|
+
* only. `null` skips it: a smaller download. */
|
|
386
|
+
readonly visionPath?: ModelPath | null;
|
|
387
|
+
/** The audio tower's weights, for audio. EmbeddingGemma 2 only. `null`
|
|
388
|
+
* skips it: a smaller download. */
|
|
389
|
+
readonly audioPath?: ModelPath | null;
|
|
390
|
+
/** The Hugging Face tokenizer.json. */
|
|
266
391
|
readonly tokenizerPath?: string;
|
|
267
392
|
/** Precision on the GPU. `f16` runs the whole model in half precision,
|
|
268
|
-
* weights and activations, for half the memory traffic
|
|
269
|
-
*
|
|
393
|
+
* weights and activations, for half the memory traffic at a small cost
|
|
394
|
+
* in precision. Needs a device with `shader-f16`. Default: `f16` where
|
|
270
395
|
* the device has it, `f32` elsewhere. Either loads any weight file. */
|
|
271
396
|
readonly dtype?: 'f32' | 'f16';
|
|
272
|
-
/** Longest input in tokens
|
|
273
|
-
*
|
|
397
|
+
/** Longest input in tokens. EmbeddingGemma 2: up to 2048, default 2048.
|
|
398
|
+
* all-MiniLM-L6-v2: up to 512, default 256. Longer text is cut; a longer
|
|
399
|
+
* input with images, video or audio is an error. */
|
|
274
400
|
readonly maxTokens?: number;
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
401
|
+
/** Most tokens one image takes: 70, 140, 280, 560 or 1120. More tokens
|
|
402
|
+
* keep more detail and take longer. Default 280. */
|
|
403
|
+
readonly imageTokens?: ImageTokenBudget;
|
|
404
|
+
}
|
|
405
|
+
/** Per-call options of embed() and embedBatch(). */
|
|
406
|
+
interface EmbedOptions {
|
|
407
|
+
/** What the text is for. A search embeds its queries with `query` and
|
|
408
|
+
* what it searches with `document`. all-MiniLM-L6-v2 ignores it.
|
|
409
|
+
* Default: none. */
|
|
410
|
+
readonly task?: EmbeddingTask;
|
|
411
|
+
/** Length of the returned vectors: 768, 512, 256 or 128 for
|
|
412
|
+
* EmbeddingGemma 2, 384 for all-MiniLM-L6-v2. A shorter vector keeps the
|
|
413
|
+
* first values, made unit length again. Compare only vectors of the same
|
|
414
|
+
* length. Default `dim`. */
|
|
415
|
+
readonly dims?: number;
|
|
416
|
+
}
|
|
417
|
+
/** The runner createEmbedder() returns. */
|
|
418
|
+
interface Embedder {
|
|
419
|
+
/** Length of a vector without `dims`. */
|
|
278
420
|
readonly dim: number;
|
|
279
|
-
/** One
|
|
280
|
-
embed(input:
|
|
281
|
-
/** Many
|
|
282
|
-
|
|
421
|
+
/** One input → unit-length vector. */
|
|
422
|
+
embed(input: EmbedInput, options?: EmbedOptions): Promise<Float32Array>;
|
|
423
|
+
/** Many inputs, one vector each. Much faster than embed() in a loop for
|
|
424
|
+
* text. */
|
|
425
|
+
embedBatch(inputs: readonly EmbedInput[], options?: EmbedOptions): Promise<Float32Array[]>;
|
|
283
426
|
/** Frees the weights and every GPU buffer the runner holds. */
|
|
284
427
|
dispose(): void;
|
|
285
428
|
}
|
|
286
|
-
/**
|
|
287
|
-
*
|
|
429
|
+
/** Cosine similarity of two unit-length vectors: 1 for the same meaning,
|
|
430
|
+
* near 0 for unrelated inputs. */
|
|
288
431
|
export declare function similarity(a: Float32Array, b: Float32Array): number;
|
|
289
|
-
/** Loads the
|
|
290
|
-
* returns the runner. With no config, loads
|
|
291
|
-
* Hugging Face Hub. */
|
|
292
|
-
export declare function
|
|
432
|
+
/** Loads the model in `modelPath` with its tokenizer and towers, warms the
|
|
433
|
+
* GPU, and returns the runner. With no config, loads EmbeddingGemma 2 from
|
|
434
|
+
* the Hugging Face Hub. */
|
|
435
|
+
export declare function createEmbedder(config?: EmbedderModel, opts?: LoadOptions): Promise<Embedder>;
|
|
293
436
|
//#endregion
|
|
294
437
|
//#region src/zoo/tasks/nlp/privacyFilter.d.ts
|
|
295
438
|
/** What to load. With no config, the int8 export from the Hugging Face
|
|
@@ -334,7 +477,7 @@ interface SpeechToTextModel {
|
|
|
334
477
|
readonly arch?: SpeechToTextArch;
|
|
335
478
|
/** The safetensors checkpoint. */
|
|
336
479
|
readonly modelPath?: ModelPath;
|
|
337
|
-
/** The
|
|
480
|
+
/** The Hugging Face tokenizer.json. */
|
|
338
481
|
readonly tokenizerPath?: string;
|
|
339
482
|
/** Precision on the GPU. `f16` runs the whole model in half precision,
|
|
340
483
|
* weights and activations, for half the memory traffic; the text stays
|
|
@@ -385,29 +528,6 @@ export declare function decodeAudio(data: ArrayBuffer): Promise<Float32Array>;
|
|
|
385
528
|
* samples, a lossy re-encode is not needed. */
|
|
386
529
|
export declare function resampleAudio(samples: Float32Array, sourceRate: number): Promise<Float32Array>;
|
|
387
530
|
//#endregion
|
|
388
|
-
//#region src/zoo/tasks/cv/image.d.ts
|
|
389
|
-
/** Raw CPU image types shared by every vision task. */
|
|
390
|
-
/** Pixel formats an ImageBuffer can carry: which channels, in what order. */
|
|
391
|
-
type ImageFormat = 'rgb' | 'rgba' | 'bgr' | 'bgra' | 'gray';
|
|
392
|
-
/** A raw image in memory: one row after another, all channels of one pixel
|
|
393
|
-
* together (the way a canvas or a camera gives it). */
|
|
394
|
-
interface ImageBuffer {
|
|
395
|
-
readonly data: Uint8Array | Uint8ClampedArray;
|
|
396
|
-
readonly width: number;
|
|
397
|
-
readonly height: number;
|
|
398
|
-
readonly format: ImageFormat;
|
|
399
|
-
readonly layout: 'hwc';
|
|
400
|
-
}
|
|
401
|
-
/** How an image is fitted into the model's input size.
|
|
402
|
-
* - `stretch`: scale width and height separately; nothing is cut or padded.
|
|
403
|
-
* - `letterbox`: keep the aspect ratio, pad the rest with a flat color.
|
|
404
|
-
* - `crop`: keep the aspect ratio, cut the centered square, then scale. */
|
|
405
|
-
type ResizeMode = 'stretch' | 'letterbox' | 'crop';
|
|
406
|
-
/** Wraps raw bytes as an ImageBuffer. Checks the size and the byte count. */
|
|
407
|
-
export declare function imageBuffer(data: Uint8Array | Uint8ClampedArray, width: number, height: number, format?: ImageFormat): ImageBuffer;
|
|
408
|
-
/** Views canvas ImageData as an ImageBuffer. No copy. */
|
|
409
|
-
export declare function imageBufferFromImageData(img: ImageData): ImageBuffer;
|
|
410
|
-
//#endregion
|
|
411
531
|
//#region src/zoo/tasks/cv/ops/point.d.ts
|
|
412
532
|
interface Point {
|
|
413
533
|
readonly x: number;
|
|
@@ -614,11 +734,36 @@ interface DepthMap {
|
|
|
614
734
|
readonly height: number;
|
|
615
735
|
readonly data: Float32Array;
|
|
616
736
|
}
|
|
737
|
+
/** The model input as a GPU buffer: normalized RGB, `[channel][row][column]`
|
|
738
|
+
* f32, `inputSize` wide and high. */
|
|
739
|
+
type DepthInput = d.WgslArray<d.WgslArray<d.WgslArray<d.F32>>>;
|
|
740
|
+
/** A depth map as a GPU buffer: `[row][column]` f32, `inputSize` wide and
|
|
741
|
+
* high, the same values as DepthMap.data. */
|
|
742
|
+
type DepthMapData = d.WgslArray<d.WgslArray<d.F32>>;
|
|
743
|
+
interface EncodeEstimateDepthOptions extends EncodeOptions<GpuImage | GpuTensor<DepthInput>, GpuImageTarget | GpuTensor<DepthMapData>> {
|
|
744
|
+
/** Texture targets only. `minmax` maps each map to [0, 1], nearest = 0.
|
|
745
|
+
* Default `none`: the raw values, which an unorm format clamps. */
|
|
746
|
+
readonly normalize?: 'none' | 'minmax';
|
|
747
|
+
}
|
|
617
748
|
interface DepthEstimator {
|
|
618
749
|
/** Width and height of every returned map. */
|
|
619
750
|
readonly inputSize: number;
|
|
751
|
+
/** Buffer layouts for encodeEstimateDepth, at this runner's size. Pass one
|
|
752
|
+
* to `root.createBuffer` to make a buffer of the right type and size. */
|
|
753
|
+
readonly schemas: {
|
|
754
|
+
readonly input: DepthInput;
|
|
755
|
+
readonly output: DepthMapData;
|
|
756
|
+
};
|
|
620
757
|
/** Image in, its depth map out. */
|
|
621
758
|
estimateDepth(image: ImageBuffer): Promise<DepthMap>;
|
|
759
|
+
/** Records a depth estimate from GPU memory into GPU memory, with no CPU
|
|
760
|
+
* copy. A texture source is fitted like an ImageBuffer; a buffer source
|
|
761
|
+
* is the model input as-is. A texture target gets the map stretched to
|
|
762
|
+
* its size in every channel; a buffer target gets the map's values.
|
|
763
|
+
* Work submitted after it sees the result.
|
|
764
|
+
* @throws RunntimeError `INVALID_ARGUMENT` for a resource of the wrong
|
|
765
|
+
* format, usage or size. */
|
|
766
|
+
encodeEstimateDepth(options: EncodeEstimateDepthOptions): void;
|
|
622
767
|
/** Frees the weights and every GPU buffer the runner holds. */
|
|
623
768
|
dispose(): void;
|
|
624
769
|
}
|
|
@@ -626,9 +771,7 @@ interface DepthEstimator {
|
|
|
626
771
|
export declare function createDepthEstimator(config?: DepthEstimatorModel, opts?: LoadOptions): Promise<DepthEstimator>;
|
|
627
772
|
//#endregion
|
|
628
773
|
//#region src/zoo/mobilenetv4/pipeline.d.ts
|
|
629
|
-
/** CPU
|
|
630
|
-
* Preprocess: RGBA canvas pixels → [3, S, S] CHW f32, ImageNet-normalized.
|
|
631
|
-
*/
|
|
774
|
+
/** CPU post-processing around the MobileNetV4 GPU forward. */
|
|
632
775
|
interface Classification {
|
|
633
776
|
classId: number;
|
|
634
777
|
confidence: number;
|
|
@@ -668,4 +811,4 @@ interface ImageClassifier {
|
|
|
668
811
|
* Needs a device with `shader-f16`. */
|
|
669
812
|
export declare function createImageClassifier(config?: ImageClassifierModel, opts?: LoadOptions): Promise<ImageClassifier>;
|
|
670
813
|
//#endregion
|
|
671
|
-
export { type BoundingBox, type BoxFormat, type BoxMap, type Classification, type ClassifyOptions, type CocoLandmark, type DepthEstimator, type DepthEstimatorModel, type DepthMap, type DepthartVariant, type DetectKeypointsOptions, type DetectObjectsOptions, type ImageBuffer, type ImageClassifier, type ImageClassifierModel, type ImageFormat, type InstanceSegmentation, type InstanceSegmenter, type InstanceSegmenterModel, type KeypointDetection, type KeypointDetector, type KeypointDetectorModel, type Landmark, type LoadOptions, type ModelPath, type ObjectDetection, type ObjectDetector, type ObjectDetectorModel, type Point, type PrivacyFilter, type PrivacyFilterModel, type PrivacySpan, type RangeSource, type ResizeMode, type SegmentInstancesOptions, type Size, type SpeechStreamUpdate, type SpeechToText, type SpeechToTextArch, type SpeechToTextModel, type
|
|
814
|
+
export { type BoundingBox, type BoxFormat, type BoxMap, type Classification, type ClassifyOptions, type CocoLandmark, type DepthEstimator, type DepthEstimatorModel, type DepthInput, type DepthMap, type DepthMapData, type DepthartVariant, type DetectKeypointsOptions, type DetectObjectsOptions, type EmbedInput, type EmbedOptions, type EmbedPart, type Embedder, type EmbedderModel, type EmbeddingTask, type EncodeEstimateDepthOptions, type EncodeOptions, type GpuEncoder, type GpuImage, type GpuImageTarget, type GpuTensor, type ImageBuffer, type ImageClassifier, type ImageClassifierModel, type ImageFormat, type ImageTokenBudget, type InstanceSegmentation, type InstanceSegmenter, type InstanceSegmenterModel, type KeypointDetection, type KeypointDetector, type KeypointDetectorModel, type Landmark, type LoadOptions, type ModelPath, type ObjectDetection, type ObjectDetector, type ObjectDetectorModel, type Point, type PrivacyFilter, type PrivacyFilterModel, type PrivacySpan, type RangeSource, type ResizeMode, type RunntimeErrorCode, type SegmentInstancesOptions, type Size, type SpeechStreamUpdate, type SpeechToText, type SpeechToTextArch, type SpeechToTextModel, type WeightCache, type Yolo26Variant, initRunntime };
|