runntime 0.0.1-alpha.0 → 0.0.1-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +169 -2
- package/dist/classifier-ByH6OI6h.mjs +45830 -0
- package/dist/config-UyxIidZc.d.mts +206 -0
- package/dist/zoo/index.d.mts +671 -0
- package/dist/zoo/index.mjs +3012 -0
- package/dist/zoo/transformers/index.d.mts +65 -0
- package/dist/zoo/transformers/index.mjs +674 -0
- package/package.json +68 -5
- package/index.js +0 -1
|
@@ -0,0 +1,3012 @@
|
|
|
1
|
+
import { $ as initRunntime, A as createMinilmEmbedCore, B as Linear, C as decodeTokens, F as createOpfsCache, H as Module, J as derivedTensor, K as warmUp, L as Conv1d, M as EMBEDDING_DIM, N as MINILM_L6, O as MinilmModel, P as models, Q as gpuExecutor, R as Embedding, T as createDetector, U as ModuleList, V as MultiHeadAttention, W as Parameter, Y as fromSafetensors, Z as evalValues, _ as buildRopeTables, a as MOBILENETV4_CROP_FRACTION, at as writeRows, c as createDetector$1, ct as add, d as move, dt as mul, f as moveTiedEmbedding, ft as silu, g as MoonshineDecoder, h as transformDecoder, ht as supportsF16, it as transpose, l as createTranscriber$1, lt as asinh, m as take, mt as defaultRoot, n as IMAGENET_MEAN, nt as gatherRowsFrom, o as DEPTHART_CONFIGS, ot as argmaxDot, p as qualify, pt as tensor, q as bufferSource, r as IMAGENET_STD, rt as slice, s as createEstimator, st as matmul, t as createClassifier, tt as chunk, u as fuseQkv, ut as gelu, v as splitIntoChunks, w as moonshineTokenizerAsset, z as LayerNorm } from "../classifier-ByH6OI6h.mjs";
|
|
2
|
+
//#region src/core/lifetime.ts
|
|
3
|
+
function createResourceScope() {
|
|
4
|
+
const tracked = [];
|
|
5
|
+
let disposed = false;
|
|
6
|
+
return {
|
|
7
|
+
track(resource) {
|
|
8
|
+
tracked.push(resource);
|
|
9
|
+
return resource;
|
|
10
|
+
},
|
|
11
|
+
dispose() {
|
|
12
|
+
disposed = true;
|
|
13
|
+
for (const r of tracked.splice(0).reverse()) r.dispose();
|
|
14
|
+
},
|
|
15
|
+
get disposed() {
|
|
16
|
+
return disposed;
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
//#endregion
|
|
21
|
+
//#region src/zoo/load.ts
|
|
22
|
+
/** Loading helpers shared by every create<Task>() factory: one options
|
|
23
|
+
* shape, one way to open a weights file, one way to fetch a JSON asset. */
|
|
24
|
+
/** Throws when the signal is already aborted. */
|
|
25
|
+
function throwIfAborted(signal) {
|
|
26
|
+
if (signal?.aborted) throw new Error("model load aborted", { cause: signal.reason });
|
|
27
|
+
}
|
|
28
|
+
/** Opens a safetensors file and reads its header. Tensor bytes download
|
|
29
|
+
* later, during loadStateDict. Every failure names the file it came from. */
|
|
30
|
+
async function openWeights(modelPath, opts = {}) {
|
|
31
|
+
throwIfAborted(opts.signal);
|
|
32
|
+
try {
|
|
33
|
+
return await fromSafetensors(modelPath, {
|
|
34
|
+
onBytes: opts.onBytes,
|
|
35
|
+
cache: opts.cache,
|
|
36
|
+
cacheId: opts.cacheId
|
|
37
|
+
});
|
|
38
|
+
} catch (err) {
|
|
39
|
+
throw new Error(`${typeof modelPath === "string" ? modelPath : "weights"}: ${err.message}`, { cause: err });
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Downloads and parses a JSON file (a tokenizer, a config). With a cache,
|
|
43
|
+
* the bytes are saved under `cacheId` (the file name by default) and the
|
|
44
|
+
* next load reads them from there. Every failure names the URL it came
|
|
45
|
+
* from. */
|
|
46
|
+
async function fetchJson(url, opts = {}) {
|
|
47
|
+
throwIfAborted(opts.signal);
|
|
48
|
+
const download = async () => {
|
|
49
|
+
let res;
|
|
50
|
+
try {
|
|
51
|
+
res = await fetch(url, { signal: opts.signal });
|
|
52
|
+
} catch (err) {
|
|
53
|
+
throw new Error(`${url}: ${err.message}`, { cause: err });
|
|
54
|
+
}
|
|
55
|
+
if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`);
|
|
56
|
+
return new Uint8Array(await res.arrayBuffer());
|
|
57
|
+
};
|
|
58
|
+
const bytes = opts.cache ? await opts.cache.getOrCompute(`json/${opts.cacheId ?? url.split("/").pop()}`, download) : await download();
|
|
59
|
+
try {
|
|
60
|
+
return JSON.parse(new TextDecoder().decode(bytes));
|
|
61
|
+
} catch (err) {
|
|
62
|
+
throw new Error(`${url}: not valid JSON`, { cause: err });
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
function minilmTokenizerAsset(json, maxTokens) {
|
|
66
|
+
const bad = (why) => /* @__PURE__ */ new Error(`minilm tokenizer: ${why}`);
|
|
67
|
+
if (typeof json !== "object" || json === null) throw bad("not a JSON object");
|
|
68
|
+
if ("vocab" in json && Array.isArray(json.vocab)) {
|
|
69
|
+
const asset = json;
|
|
70
|
+
if (asset.version !== 1) throw bad(`unsupported asset version ${asset.version}`);
|
|
71
|
+
return maxTokens === void 0 ? asset : {
|
|
72
|
+
...asset,
|
|
73
|
+
maxWordpieces: maxTokens
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
const hf = json;
|
|
77
|
+
if (hf.model?.type !== "WordPiece" || typeof hf.model.vocab !== "object") throw bad("expected a WordPiece tokenizer.json or a version 1 asset");
|
|
78
|
+
const entries = Object.entries(hf.model.vocab);
|
|
79
|
+
const vocab = new Array(entries.length);
|
|
80
|
+
for (const [token, id] of entries) vocab[id] = token;
|
|
81
|
+
const idOf = (token) => {
|
|
82
|
+
const id = hf.model.vocab[token];
|
|
83
|
+
if (id === void 0) throw bad(`vocab has no ${token}`);
|
|
84
|
+
return id;
|
|
85
|
+
};
|
|
86
|
+
return {
|
|
87
|
+
version: 1,
|
|
88
|
+
doLowerCase: hf.normalizer?.lowercase ?? true,
|
|
89
|
+
maxWordpieces: maxTokens ?? 256,
|
|
90
|
+
clsId: idOf("[CLS]"),
|
|
91
|
+
sepId: idOf("[SEP]"),
|
|
92
|
+
padId: idOf("[PAD]"),
|
|
93
|
+
unkId: idOf("[UNK]"),
|
|
94
|
+
vocab
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
const MAX_WORD_CHARS = 100;
|
|
98
|
+
const isWhitespace = (ch) => ch === " " || ch === " " || ch === "\n" || ch === "\r" || /\p{Zs}/u.test(ch);
|
|
99
|
+
const isControl = (ch) => ch !== " " && ch !== "\n" && ch !== "\r" && /\p{C}/u.test(ch);
|
|
100
|
+
const isPunctuation = (ch) => {
|
|
101
|
+
const c = ch.codePointAt(0);
|
|
102
|
+
if (c >= 33 && c <= 47 || c >= 58 && c <= 64 || c >= 91 && c <= 96 || c >= 123 && c <= 126) return true;
|
|
103
|
+
return /\p{P}/u.test(ch);
|
|
104
|
+
};
|
|
105
|
+
const isCjkIdeograph = (c) => c >= 19968 && c <= 40959 || c >= 13312 && c <= 19903 || c >= 131072 && c <= 173791 || c >= 173824 && c <= 177983 || c >= 177984 && c <= 178207 || c >= 178208 && c <= 183983 || c >= 63744 && c <= 64255 || c >= 194560 && c <= 195103;
|
|
106
|
+
function basicTokenize(text, doLowerCase) {
|
|
107
|
+
let cleaned = "";
|
|
108
|
+
for (const ch of text) {
|
|
109
|
+
const c = ch.codePointAt(0);
|
|
110
|
+
if (c === 0 || c === 65533 || isControl(ch)) continue;
|
|
111
|
+
if (isCjkIdeograph(c)) cleaned += ` ${ch} `;
|
|
112
|
+
else cleaned += isWhitespace(ch) ? " " : ch;
|
|
113
|
+
}
|
|
114
|
+
const words = [];
|
|
115
|
+
for (const token of cleaned.split(" ")) {
|
|
116
|
+
if (!token) continue;
|
|
117
|
+
let t = token;
|
|
118
|
+
if (doLowerCase) t = t.toLowerCase().normalize("NFD").replace(/\p{Mn}/gu, "");
|
|
119
|
+
let current = "";
|
|
120
|
+
for (const ch of t) if (isPunctuation(ch)) {
|
|
121
|
+
if (current) words.push(current);
|
|
122
|
+
words.push(ch);
|
|
123
|
+
current = "";
|
|
124
|
+
} else current += ch;
|
|
125
|
+
if (current) words.push(current);
|
|
126
|
+
}
|
|
127
|
+
return words;
|
|
128
|
+
}
|
|
129
|
+
function createMinilmTokenizer(asset) {
|
|
130
|
+
if (asset.version !== 1) throw new Error(`minilm tokenizer: unsupported asset version ${asset.version}`);
|
|
131
|
+
const tokenToId = /* @__PURE__ */ new Map();
|
|
132
|
+
asset.vocab.forEach((token, id) => tokenToId.set(token, id));
|
|
133
|
+
const wordpiece = (word, out) => {
|
|
134
|
+
const chars = [...word];
|
|
135
|
+
if (chars.length > MAX_WORD_CHARS) {
|
|
136
|
+
out.push(asset.unkId);
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
const pieces = [];
|
|
140
|
+
let start = 0;
|
|
141
|
+
while (start < chars.length) {
|
|
142
|
+
let end = chars.length;
|
|
143
|
+
let id;
|
|
144
|
+
while (end > start) {
|
|
145
|
+
const piece = (start > 0 ? "##" : "") + chars.slice(start, end).join("");
|
|
146
|
+
id = tokenToId.get(piece);
|
|
147
|
+
if (id !== void 0) break;
|
|
148
|
+
end--;
|
|
149
|
+
}
|
|
150
|
+
if (id === void 0) {
|
|
151
|
+
out.push(asset.unkId);
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
pieces.push(id);
|
|
155
|
+
start = end;
|
|
156
|
+
}
|
|
157
|
+
out.push(...pieces);
|
|
158
|
+
};
|
|
159
|
+
return {
|
|
160
|
+
clsId: asset.clsId,
|
|
161
|
+
sepId: asset.sepId,
|
|
162
|
+
encode(text) {
|
|
163
|
+
const pieces = [];
|
|
164
|
+
for (const word of basicTokenize(text, asset.doLowerCase)) wordpiece(word, pieces);
|
|
165
|
+
const body = pieces.slice(0, asset.maxWordpieces - 2);
|
|
166
|
+
return [
|
|
167
|
+
asset.clsId,
|
|
168
|
+
...body,
|
|
169
|
+
asset.sepId
|
|
170
|
+
];
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
//#endregion
|
|
175
|
+
//#region src/zoo/tasks/nlp/textEmbedding.ts
|
|
176
|
+
/** Text embedding task: text in, one unit-length vector out. Runs
|
|
177
|
+
* all-MiniLM-L6-v2 on the initRunntime() device. */
|
|
178
|
+
/** Dot product of two unit-length embeddings, which is their cosine
|
|
179
|
+
* similarity: 1 for the same meaning, near 0 for unrelated text. */
|
|
180
|
+
function similarity(a, b) {
|
|
181
|
+
let dot = 0;
|
|
182
|
+
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
183
|
+
return dot;
|
|
184
|
+
}
|
|
185
|
+
const normalize = (v) => {
|
|
186
|
+
let sumSq = 0;
|
|
187
|
+
for (const x of v) sumSq += x * x;
|
|
188
|
+
const inv = 1 / Math.sqrt(sumSq);
|
|
189
|
+
for (let i = 0; i < v.length; i++) v[i] *= inv;
|
|
190
|
+
return v;
|
|
191
|
+
};
|
|
192
|
+
/** Loads the checkpoint and the tokenizer, warms every GPU path once, and
|
|
193
|
+
* returns the runner. With no config, loads all-MiniLM-L6-v2 from the
|
|
194
|
+
* Hugging Face Hub. */
|
|
195
|
+
async function createTextEmbedder(config = {}, opts = {}) {
|
|
196
|
+
const scope = createResourceScope();
|
|
197
|
+
try {
|
|
198
|
+
const device = defaultRoot().device;
|
|
199
|
+
const { modelPath = models.textEmbedding.ALL_MINILM_L6_V2.DEFAULT.modelPath, tokenizerPath = models.textEmbedding.ALL_MINILM_L6_V2.DEFAULT.tokenizerPath, dtype = supportsF16() ? "f16" : "f32", maxTokens } = config;
|
|
200
|
+
const cacheId = (path) => typeof path === "string" ? `minilm-v1/${path.replace(/^https?:\/\//, "")}` : void 0;
|
|
201
|
+
const sd = await openWeights(modelPath, {
|
|
202
|
+
...opts,
|
|
203
|
+
cacheId: opts.cacheId ?? cacheId(modelPath)
|
|
204
|
+
});
|
|
205
|
+
const tokenizer = createMinilmTokenizer(minilmTokenizerAsset(await fetchJson(tokenizerPath, {
|
|
206
|
+
...opts,
|
|
207
|
+
cacheId: cacheId(tokenizerPath)
|
|
208
|
+
}), maxTokens));
|
|
209
|
+
throwIfAborted(opts.signal);
|
|
210
|
+
const model = scope.track(new MinilmModel(MINILM_L6));
|
|
211
|
+
if (dtype === "f16") model.half();
|
|
212
|
+
await model.loadStateDict(sd, { onProgress: opts.onProgress });
|
|
213
|
+
const core = scope.track(createMinilmEmbedCore(model));
|
|
214
|
+
let disposed = false;
|
|
215
|
+
const guard = () => {
|
|
216
|
+
if (disposed) throw new Error("text embedder is disposed");
|
|
217
|
+
};
|
|
218
|
+
const embedTokens = async (ids) => normalize(await core.embedPooled(ids));
|
|
219
|
+
const embedTokensBatch = async (idsList) => {
|
|
220
|
+
if (idsList.length === 0) return [];
|
|
221
|
+
if (idsList.length === 1) return [await embedTokens(idsList[0])];
|
|
222
|
+
const pooled = await core.embedPooledBatch(idsList);
|
|
223
|
+
return idsList.map((_, b) => normalize(pooled.slice(b * EMBEDDING_DIM, (b + 1) * EMBEDDING_DIM)));
|
|
224
|
+
};
|
|
225
|
+
throwIfAborted(opts.signal);
|
|
226
|
+
await warmUp(device, async () => {
|
|
227
|
+
const pair = [tokenizer.clsId, tokenizer.sepId];
|
|
228
|
+
await embedTokensBatch([pair, pair]);
|
|
229
|
+
await core.warmBuckets(pair);
|
|
230
|
+
return embedTokens(pair);
|
|
231
|
+
});
|
|
232
|
+
return {
|
|
233
|
+
dim: EMBEDDING_DIM,
|
|
234
|
+
async embed(input) {
|
|
235
|
+
guard();
|
|
236
|
+
return embedTokens(tokenizer.encode(input));
|
|
237
|
+
},
|
|
238
|
+
async embedBatch(inputs) {
|
|
239
|
+
guard();
|
|
240
|
+
return embedTokensBatch(inputs.map((t) => tokenizer.encode(t)));
|
|
241
|
+
},
|
|
242
|
+
dispose() {
|
|
243
|
+
disposed = true;
|
|
244
|
+
scope.dispose();
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
} catch (err) {
|
|
248
|
+
scope.dispose();
|
|
249
|
+
throw err;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
//#endregion
|
|
253
|
+
//#region src/zoo/tasks/nlp/privacyFilter.ts
|
|
254
|
+
/** Privacy filter task: text in, the personal data it contains out, as
|
|
255
|
+
* labeled spans. Runs OpenAI's privacy-filter on the initRunntime() device. */
|
|
256
|
+
/** Loads the checkpoint, warms the GPU path once, and returns the runner. */
|
|
257
|
+
async function createPrivacyFilter(config = {}, opts = {}) {
|
|
258
|
+
const scope = createResourceScope();
|
|
259
|
+
try {
|
|
260
|
+
const { modelPath = models.privacyFilter.PRIVACY_FILTER.DEFAULT.modelPath } = config;
|
|
261
|
+
const path = typeof modelPath === "string" ? modelPath.replace(/^https?:\/\//, "") : void 0;
|
|
262
|
+
const sd = await openWeights(modelPath, {
|
|
263
|
+
...opts,
|
|
264
|
+
cacheId: opts.cacheId ?? (path && `privacy-filter-v1/${path}`)
|
|
265
|
+
});
|
|
266
|
+
throwIfAborted(opts.signal);
|
|
267
|
+
const detector = scope.track(await createDetector(sd, { onProgress: opts.onProgress }));
|
|
268
|
+
let disposed = false;
|
|
269
|
+
let queue = Promise.resolve();
|
|
270
|
+
return {
|
|
271
|
+
async detect(text) {
|
|
272
|
+
if (disposed) throw new Error("privacy filter is disposed");
|
|
273
|
+
if (typeof text !== "string") throw new Error("detect: text must be a string");
|
|
274
|
+
const run = queue.then(() => detector.detect(text));
|
|
275
|
+
queue = run.catch(() => void 0);
|
|
276
|
+
return (await run).spans;
|
|
277
|
+
},
|
|
278
|
+
dispose() {
|
|
279
|
+
disposed = true;
|
|
280
|
+
scope.dispose();
|
|
281
|
+
}
|
|
282
|
+
};
|
|
283
|
+
} catch (err) {
|
|
284
|
+
scope.dispose();
|
|
285
|
+
throw err;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
//#endregion
|
|
289
|
+
//#region src/zoo/moonshine/segmenter.ts
|
|
290
|
+
function analyzeSegment(audio, opts) {
|
|
291
|
+
const { sampleRate, threshold = .015, hopSeconds = .032, smoothHops = 8, silenceSeconds = .8, maxSegmentSeconds = 15 } = opts;
|
|
292
|
+
if (!(sampleRate > 0)) throw new Error(`analyzeSegment: bad sampleRate ${sampleRate}`);
|
|
293
|
+
if (!(threshold > 0)) throw new Error(`analyzeSegment: threshold must be positive`);
|
|
294
|
+
const hop = Math.max(1, Math.round(hopSeconds * sampleRate));
|
|
295
|
+
const seconds = audio.length / sampleRate;
|
|
296
|
+
const hops = Math.ceil(audio.length / hop);
|
|
297
|
+
const rms = new Float64Array(hops);
|
|
298
|
+
for (let h = 0; h < hops; h++) {
|
|
299
|
+
const start = h * hop;
|
|
300
|
+
const end = Math.min(start + hop, audio.length);
|
|
301
|
+
let sum = 0;
|
|
302
|
+
for (let i = start; i < end; i++) sum += audio[i] * audio[i];
|
|
303
|
+
rms[h] = Math.sqrt(sum / (end - start));
|
|
304
|
+
}
|
|
305
|
+
let lastSpeech = -1;
|
|
306
|
+
let windowSum = 0;
|
|
307
|
+
for (let h = 0; h < hops; h++) {
|
|
308
|
+
windowSum += rms[h];
|
|
309
|
+
if (h >= smoothHops) windowSum -= rms[h - smoothHops];
|
|
310
|
+
if (windowSum / Math.min(h + 1, smoothHops) > threshold) lastSpeech = h;
|
|
311
|
+
}
|
|
312
|
+
const hasSpeech = lastSpeech >= 0;
|
|
313
|
+
const trailingSilenceSeconds = hasSpeech ? (hops - 1 - lastSpeech) * hop / sampleRate : seconds;
|
|
314
|
+
return {
|
|
315
|
+
seconds,
|
|
316
|
+
hasSpeech,
|
|
317
|
+
trailingSilenceSeconds,
|
|
318
|
+
shouldFinalize: hasSpeech && (trailingSilenceSeconds >= silenceSeconds || seconds >= maxSegmentSeconds)
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
//#endregion
|
|
322
|
+
//#region src/zoo/moonshine-streaming/config.ts
|
|
323
|
+
const windows = (layers) => Object.freeze(Array.from({ length: layers }, (_, i) => i < 2 || i >= layers - 2 ? [16, 4] : [16, 0]));
|
|
324
|
+
const MOONSHINE_STREAMING_TINY = Object.freeze({
|
|
325
|
+
enc: Object.freeze({
|
|
326
|
+
hidden: 320,
|
|
327
|
+
ffn: 1280,
|
|
328
|
+
layers: 6,
|
|
329
|
+
heads: 8,
|
|
330
|
+
headDim: 40,
|
|
331
|
+
windows: windows(6)
|
|
332
|
+
}),
|
|
333
|
+
dec: Object.freeze({
|
|
334
|
+
hidden: 320,
|
|
335
|
+
ffn: 1280,
|
|
336
|
+
layers: 6,
|
|
337
|
+
heads: 8,
|
|
338
|
+
headDim: 40,
|
|
339
|
+
rotaryDim: 32,
|
|
340
|
+
ropeTheta: 1e4
|
|
341
|
+
}),
|
|
342
|
+
frameLen: 80,
|
|
343
|
+
vocab: 32768,
|
|
344
|
+
maxPositions: 4096,
|
|
345
|
+
bos: 1,
|
|
346
|
+
eos: 2
|
|
347
|
+
});
|
|
348
|
+
const MOONSHINE_STREAMING_SMALL = Object.freeze({
|
|
349
|
+
enc: Object.freeze({
|
|
350
|
+
hidden: 620,
|
|
351
|
+
ffn: 2480,
|
|
352
|
+
layers: 10,
|
|
353
|
+
heads: 8,
|
|
354
|
+
headDim: 64,
|
|
355
|
+
windows: windows(10)
|
|
356
|
+
}),
|
|
357
|
+
dec: Object.freeze({
|
|
358
|
+
hidden: 512,
|
|
359
|
+
ffn: 2048,
|
|
360
|
+
layers: 10,
|
|
361
|
+
heads: 8,
|
|
362
|
+
headDim: 64,
|
|
363
|
+
rotaryDim: 32,
|
|
364
|
+
ropeTheta: 1e4
|
|
365
|
+
}),
|
|
366
|
+
frameLen: 80,
|
|
367
|
+
vocab: 32768,
|
|
368
|
+
maxPositions: 4096,
|
|
369
|
+
bos: 1,
|
|
370
|
+
eos: 2
|
|
371
|
+
});
|
|
372
|
+
const MOONSHINE_STREAMING_MEDIUM = Object.freeze({
|
|
373
|
+
enc: Object.freeze({
|
|
374
|
+
hidden: 768,
|
|
375
|
+
ffn: 3072,
|
|
376
|
+
layers: 14,
|
|
377
|
+
heads: 10,
|
|
378
|
+
headDim: 64,
|
|
379
|
+
windows: windows(14)
|
|
380
|
+
}),
|
|
381
|
+
dec: Object.freeze({
|
|
382
|
+
hidden: 640,
|
|
383
|
+
ffn: 2560,
|
|
384
|
+
layers: 14,
|
|
385
|
+
heads: 10,
|
|
386
|
+
headDim: 64,
|
|
387
|
+
rotaryDim: 32,
|
|
388
|
+
ropeTheta: 1e4
|
|
389
|
+
}),
|
|
390
|
+
frameLen: 80,
|
|
391
|
+
vocab: 32768,
|
|
392
|
+
maxPositions: 4096,
|
|
393
|
+
bos: 1,
|
|
394
|
+
eos: 2
|
|
395
|
+
});
|
|
396
|
+
function presetFromStateDict(sd) {
|
|
397
|
+
const embedding = sd.tensors.get("model.decoder.embed_tokens.weight");
|
|
398
|
+
const width = embedding?.shape[1];
|
|
399
|
+
const presets = [
|
|
400
|
+
MOONSHINE_STREAMING_TINY,
|
|
401
|
+
MOONSHINE_STREAMING_SMALL,
|
|
402
|
+
MOONSHINE_STREAMING_MEDIUM
|
|
403
|
+
];
|
|
404
|
+
for (const cfg of presets) if (cfg.dec.hidden === width) return cfg;
|
|
405
|
+
throw new Error(embedding === void 0 ? "moonshine-streaming weights: no token embedding, not a Moonshine checkpoint" : `moonshine-streaming weights: embedding width ${width} matches no known size`);
|
|
406
|
+
}
|
|
407
|
+
//#endregion
|
|
408
|
+
//#region src/zoo/moonshine-streaming/frontend.ts
|
|
409
|
+
/** Moonshine v2 frontend: raw 16 kHz mono audio, reshaped on the CPU into
|
|
410
|
+
* non-overlapping 80-sample frames [T, 80] (5 ms, 200 Hz), → encoder frames
|
|
411
|
+
* [T/4, enc.hidden] (~50 Hz):
|
|
412
|
+
* per-frame CMVN → asinh(k·x) → linear [80 → d] (no bias) → silu
|
|
413
|
+
* → causal conv1 k=5 s=2 (d → 2d) → silu → causal conv2 k=5 s=2 (2d → d)
|
|
414
|
+
* CMVN (mean-center, divide by RMS, eps 1e-6) is exactly a bias-free
|
|
415
|
+
* LayerNorm whose weight the loader supplies as ones. k is the checkpoint's
|
|
416
|
+
* learnable asinh compression scale — a [1] Parameter holding exp(log_k),
|
|
417
|
+
* broadcast-multiplied before the asinh. Both convs left-pad kernel−1 zeros
|
|
418
|
+
* (causal), so each keeps ceil(T/2) frames; no activation after conv2. */
|
|
419
|
+
const K = 5;
|
|
420
|
+
const S = 2;
|
|
421
|
+
const PAD = 4;
|
|
422
|
+
function frontendFrames(samples, frameLen) {
|
|
423
|
+
if (!Number.isInteger(samples) || samples < frameLen || samples % frameLen !== 0) throw new Error(`frontendFrames: samples ${samples} must be a positive multiple of frameLen ${frameLen}`);
|
|
424
|
+
const input = samples / frameLen;
|
|
425
|
+
const conv1 = Math.floor((input - 1) / S) + 1;
|
|
426
|
+
return {
|
|
427
|
+
input,
|
|
428
|
+
conv1,
|
|
429
|
+
conv2: Math.floor((conv1 - 1) / S) + 1
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
var StreamingFrontend = class extends Module {
|
|
433
|
+
cmvn;
|
|
434
|
+
k;
|
|
435
|
+
linear;
|
|
436
|
+
conv1;
|
|
437
|
+
conv2;
|
|
438
|
+
constructor(cfg) {
|
|
439
|
+
super();
|
|
440
|
+
const d = cfg.enc.hidden;
|
|
441
|
+
this.cmvn = new LayerNorm(cfg.frameLen, 1e-6);
|
|
442
|
+
this.k = new Parameter({
|
|
443
|
+
elems: 1,
|
|
444
|
+
dtype: "f32",
|
|
445
|
+
dims: [1]
|
|
446
|
+
});
|
|
447
|
+
this.linear = new Linear(cfg.frameLen, d, { bias: false });
|
|
448
|
+
this.conv1 = new Conv1d(d, 2 * d, K, {
|
|
449
|
+
stride: S,
|
|
450
|
+
padLeft: PAD
|
|
451
|
+
});
|
|
452
|
+
this.conv2 = new Conv1d(2 * d, d, K, {
|
|
453
|
+
stride: S,
|
|
454
|
+
padLeft: PAD
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
forward(x) {
|
|
458
|
+
const compressed = asinh(mul(this.cmvn.forward(x), this.k.value));
|
|
459
|
+
const h = silu(this.conv1.forward(silu(this.linear.forward(compressed))));
|
|
460
|
+
return this.conv2.forward(h);
|
|
461
|
+
}
|
|
462
|
+
};
|
|
463
|
+
//#endregion
|
|
464
|
+
//#region src/zoo/moonshine-streaming/encoder.ts
|
|
465
|
+
/** Moonshine v2 encoder: pre-norm transformer layers carrying no positional
|
|
466
|
+
* information. Locality comes entirely from per-layer asymmetric
|
|
467
|
+
* sliding-window attention, with windows converted from HF's strict bounds to
|
|
468
|
+
* the kernel's inclusive form.
|
|
469
|
+
*
|
|
470
|
+
* There is no rope, so the score scale folds into the q columns at load. The
|
|
471
|
+
* loader also adds 1 back to the checkpoint's offset norm gamma, so plain
|
|
472
|
+
* weight-only LayerNorms serve here.
|
|
473
|
+
*
|
|
474
|
+
* The adapter bridges encoder to decoder, and is where positions enter. */
|
|
475
|
+
var EncoderAttn = class extends Module {
|
|
476
|
+
cfg;
|
|
477
|
+
qkv;
|
|
478
|
+
mha;
|
|
479
|
+
out;
|
|
480
|
+
constructor(cfg, window) {
|
|
481
|
+
super();
|
|
482
|
+
this.cfg = cfg;
|
|
483
|
+
const { hidden, heads, headDim } = cfg.enc;
|
|
484
|
+
const qw = heads * headDim;
|
|
485
|
+
this.qkv = new Linear(hidden, 3 * qw, { bias: false });
|
|
486
|
+
this.mha = new MultiHeadAttention({
|
|
487
|
+
qHeads: heads,
|
|
488
|
+
kvHeads: heads,
|
|
489
|
+
headDim,
|
|
490
|
+
windowLeft: window[0] - 1,
|
|
491
|
+
windowRight: Math.max(0, window[1] - 1)
|
|
492
|
+
});
|
|
493
|
+
this.out = new Linear(qw, hidden, { bias: false });
|
|
494
|
+
}
|
|
495
|
+
forward(x, residual) {
|
|
496
|
+
const qkv = this.qkv.forward(x);
|
|
497
|
+
const [q, k, v] = chunk(qkv, 3, 1);
|
|
498
|
+
return this.out.forward(this.mha.forward(q, k, v), { addend: residual });
|
|
499
|
+
}
|
|
500
|
+
};
|
|
501
|
+
var EncoderMlp = class extends Module {
|
|
502
|
+
fc1;
|
|
503
|
+
fc2;
|
|
504
|
+
constructor(cfg) {
|
|
505
|
+
super();
|
|
506
|
+
this.fc1 = new Linear(cfg.enc.hidden, cfg.enc.ffn);
|
|
507
|
+
this.fc2 = new Linear(cfg.enc.ffn, cfg.enc.hidden);
|
|
508
|
+
}
|
|
509
|
+
forward(x, residual) {
|
|
510
|
+
return this.fc2.forward(gelu(this.fc1.forward(x)), { addend: residual });
|
|
511
|
+
}
|
|
512
|
+
};
|
|
513
|
+
var EncoderLayer = class extends Module {
|
|
514
|
+
norm1;
|
|
515
|
+
attn;
|
|
516
|
+
norm2;
|
|
517
|
+
mlp;
|
|
518
|
+
constructor(cfg, window) {
|
|
519
|
+
super();
|
|
520
|
+
this.norm1 = new LayerNorm(cfg.enc.hidden);
|
|
521
|
+
this.attn = new EncoderAttn(cfg, window);
|
|
522
|
+
this.norm2 = new LayerNorm(cfg.enc.hidden);
|
|
523
|
+
this.mlp = new EncoderMlp(cfg);
|
|
524
|
+
}
|
|
525
|
+
forward(x) {
|
|
526
|
+
const h = this.attn.forward(this.norm1.forward(x), x);
|
|
527
|
+
return this.mlp.forward(this.norm2.forward(h), h);
|
|
528
|
+
}
|
|
529
|
+
};
|
|
530
|
+
var StreamingEncoder = class extends Module {
|
|
531
|
+
layers;
|
|
532
|
+
norm;
|
|
533
|
+
constructor(cfg) {
|
|
534
|
+
super();
|
|
535
|
+
this.layers = new ModuleList(cfg.enc.windows.map((w) => new EncoderLayer(cfg, w)));
|
|
536
|
+
this.norm = new LayerNorm(cfg.enc.hidden);
|
|
537
|
+
}
|
|
538
|
+
forward(x) {
|
|
539
|
+
let h = x;
|
|
540
|
+
for (const layer of this.layers.items) h = layer.forward(h);
|
|
541
|
+
return this.norm.forward(h);
|
|
542
|
+
}
|
|
543
|
+
};
|
|
544
|
+
var StreamingAdapter = class extends Module {
|
|
545
|
+
posEmb;
|
|
546
|
+
proj;
|
|
547
|
+
constructor(cfg) {
|
|
548
|
+
super();
|
|
549
|
+
const [eh, dh] = [cfg.enc.hidden, cfg.dec.hidden];
|
|
550
|
+
this.posEmb = new Parameter({
|
|
551
|
+
elems: cfg.maxPositions * eh,
|
|
552
|
+
dtype: "f32",
|
|
553
|
+
dims: [cfg.maxPositions, eh]
|
|
554
|
+
});
|
|
555
|
+
this.proj = eh === dh ? void 0 : new Linear(eh, dh, { bias: false });
|
|
556
|
+
}
|
|
557
|
+
forward(x) {
|
|
558
|
+
const t = x.shape.dims[0];
|
|
559
|
+
const positioned = add(x, slice(this.posEmb.value, 0, 0, t));
|
|
560
|
+
return this.proj ? this.proj.forward(positioned) : positioned;
|
|
561
|
+
}
|
|
562
|
+
};
|
|
563
|
+
//#endregion
|
|
564
|
+
//#region src/zoo/moonshine-streaming/stateDictHooks.ts
|
|
565
|
+
/** Maps the HF Moonshine v2 checkpoint onto the model's parameter tree as lazy
|
|
566
|
+
* derived tensors; nothing is fetched here.
|
|
567
|
+
*
|
|
568
|
+
* The notable transforms: encoder norm gamma gains back its −1 offset, CMVN's
|
|
569
|
+
* missing weight is synthesized as ones, and q/k/v stack into fused weights —
|
|
570
|
+
* scaled on the encoder, which has no rope, and unscaled on the decoder, whose
|
|
571
|
+
* rope tables carry the scale. Everything else renames.
|
|
572
|
+
*
|
|
573
|
+
* Weights stay in torch layout for the core Linear and Conv1d hooks to
|
|
574
|
+
* transpose. Derived byteLengths carry the source file bytes, so load progress
|
|
575
|
+
* still sums to the checkpoint size. */
|
|
576
|
+
function gammaPlusOne(t) {
|
|
577
|
+
return derivedTensor(t.shape, async () => {
|
|
578
|
+
const gamma = await t.f32();
|
|
579
|
+
const out = new Float32Array(gamma.length);
|
|
580
|
+
for (let j = 0; j < gamma.length; j++) out[j] = gamma[j] + 1;
|
|
581
|
+
return out;
|
|
582
|
+
}, t.byteLength);
|
|
583
|
+
}
|
|
584
|
+
function transformStreamingStateDict(sd, prefix, cfg) {
|
|
585
|
+
const q = (name) => qualify(prefix, name);
|
|
586
|
+
const [eh, dh] = [cfg.enc.hidden, cfg.dec.hidden];
|
|
587
|
+
const gammaNorm = (from, to) => sd.tensors.set(to, gammaPlusOne(take(sd, from)));
|
|
588
|
+
moveTiedEmbedding(sd, prefix);
|
|
589
|
+
sd.tensors.set(q("frontend.cmvn.weight"), derivedTensor([cfg.frameLen], async () => new Float32Array(cfg.frameLen).fill(1), 0));
|
|
590
|
+
const logK = take(sd, q("model.encoder.embedder.comp.log_k"));
|
|
591
|
+
sd.tensors.set(q("frontend.k"), derivedTensor([1], async () => Float32Array.from([Math.exp((await logK.f32())[0])]), logK.byteLength));
|
|
592
|
+
move(sd, q("model.encoder.embedder.linear.weight"), q("frontend.linear.weight"));
|
|
593
|
+
move(sd, q("model.encoder.embedder.conv1.weight"), q("frontend.conv1.weight"));
|
|
594
|
+
move(sd, q("model.encoder.embedder.conv1.bias"), q("frontend.conv1.bias"));
|
|
595
|
+
move(sd, q("model.encoder.embedder.conv2.weight"), q("frontend.conv2.weight"));
|
|
596
|
+
move(sd, q("model.encoder.embedder.conv2.bias"), q("frontend.conv2.bias"));
|
|
597
|
+
const encQScale = 1 / Math.sqrt(cfg.enc.headDim);
|
|
598
|
+
for (let i = 0; i < cfg.enc.layers; i++) {
|
|
599
|
+
const hf = q(`model.encoder.layers.${i}`);
|
|
600
|
+
const en = q(`encoder.layers.${i}`);
|
|
601
|
+
gammaNorm(`${hf}.input_layernorm.gamma`, `${en}.norm1.weight`);
|
|
602
|
+
fuseQkv(sd, `${hf}.self_attn`, `${en}.attn.qkv.weight`, eh, encQScale);
|
|
603
|
+
move(sd, `${hf}.self_attn.o_proj.weight`, `${en}.attn.out.weight`);
|
|
604
|
+
gammaNorm(`${hf}.post_attention_layernorm.gamma`, `${en}.norm2.weight`);
|
|
605
|
+
move(sd, `${hf}.mlp.fc1.weight`, `${en}.mlp.fc1.weight`);
|
|
606
|
+
move(sd, `${hf}.mlp.fc1.bias`, `${en}.mlp.fc1.bias`);
|
|
607
|
+
move(sd, `${hf}.mlp.fc2.weight`, `${en}.mlp.fc2.weight`);
|
|
608
|
+
move(sd, `${hf}.mlp.fc2.bias`, `${en}.mlp.fc2.bias`);
|
|
609
|
+
}
|
|
610
|
+
gammaNorm(q("model.encoder.final_norm.gamma"), q("encoder.norm.weight"));
|
|
611
|
+
move(sd, q("model.decoder.pos_emb.weight"), q("adapter.posEmb"));
|
|
612
|
+
if (eh !== dh) move(sd, q("model.decoder.proj.weight"), q("adapter.proj.weight"));
|
|
613
|
+
transformDecoder(sd, prefix, {
|
|
614
|
+
layers: cfg.dec.layers,
|
|
615
|
+
hidden: dh,
|
|
616
|
+
headDim: cfg.dec.headDim
|
|
617
|
+
});
|
|
618
|
+
}
|
|
619
|
+
//#endregion
|
|
620
|
+
//#region src/zoo/moonshine-streaming/model.ts
|
|
621
|
+
/** The full Moonshine v2 module tree. No constructor takes a sequence length,
|
|
622
|
+
* and one loadStateDict binds the whole checkpoint.
|
|
623
|
+
*
|
|
624
|
+
* The decoder is the v1 class: same architecture, different dims, so
|
|
625
|
+
* `decoderView` bridges the two config shapes. Cross K and V read the adapter
|
|
626
|
+
* output rather than the encoder's.
|
|
627
|
+
*
|
|
628
|
+
* The embedding weight serves as both token lookup and the tied LM head. No
|
|
629
|
+
* transposed copy exists, so tying is exact by construction. */
|
|
630
|
+
function decoderView(cfg) {
|
|
631
|
+
return {
|
|
632
|
+
dModel: cfg.dec.hidden,
|
|
633
|
+
ffn: cfg.dec.ffn,
|
|
634
|
+
encLayers: cfg.enc.layers,
|
|
635
|
+
decLayers: cfg.dec.layers,
|
|
636
|
+
heads: cfg.dec.heads,
|
|
637
|
+
headDim: cfg.dec.headDim,
|
|
638
|
+
rotaryDim: cfg.dec.rotaryDim,
|
|
639
|
+
ropeTheta: cfg.dec.ropeTheta,
|
|
640
|
+
vocab: cfg.vocab,
|
|
641
|
+
maxPositions: cfg.maxPositions,
|
|
642
|
+
bos: cfg.bos,
|
|
643
|
+
eos: cfg.eos
|
|
644
|
+
};
|
|
645
|
+
}
|
|
646
|
+
var MoonshineStreamingModel = class extends Module {
|
|
647
|
+
cfg;
|
|
648
|
+
embedding;
|
|
649
|
+
frontend;
|
|
650
|
+
encoder;
|
|
651
|
+
adapter;
|
|
652
|
+
decoder;
|
|
653
|
+
constructor(cfg) {
|
|
654
|
+
super();
|
|
655
|
+
this.cfg = cfg;
|
|
656
|
+
this.embedding = new Embedding(cfg.vocab, cfg.dec.hidden);
|
|
657
|
+
this.frontend = new StreamingFrontend(cfg);
|
|
658
|
+
this.encoder = new StreamingEncoder(cfg);
|
|
659
|
+
this.adapter = new StreamingAdapter(cfg);
|
|
660
|
+
this.decoder = new MoonshineDecoder(decoderView(cfg));
|
|
661
|
+
}
|
|
662
|
+
transformStateDict(sd, prefix) {
|
|
663
|
+
transformStreamingStateDict(sd, prefix, this.cfg);
|
|
664
|
+
}
|
|
665
|
+
embed(ids) {
|
|
666
|
+
return this.embedding.forward(ids);
|
|
667
|
+
}
|
|
668
|
+
embedFrom(ids) {
|
|
669
|
+
return gatherRowsFrom(this.embedding.weight.value, ids);
|
|
670
|
+
}
|
|
671
|
+
logitsLast(h) {
|
|
672
|
+
const w = this.embedding.weight.value;
|
|
673
|
+
if (w.shape.dtype !== h.shape.dtype) throw new Error(`MoonshineStreamingModel.logitsLast: embedding is ${w.shape.dtype} but h is ${h.shape.dtype}`);
|
|
674
|
+
const t = h.shape.dims[0];
|
|
675
|
+
return matmul(w, transpose(slice(h, 0, t - 1, t)));
|
|
676
|
+
}
|
|
677
|
+
nextTokenId(h1) {
|
|
678
|
+
return argmaxDot(this.embedding.weight.value, h1);
|
|
679
|
+
}
|
|
680
|
+
forward() {
|
|
681
|
+
throw new Error("MoonshineStreamingModel: use frontend/encoder/adapter/decoder forwards, embed() and logitsLast()");
|
|
682
|
+
}
|
|
683
|
+
};
|
|
684
|
+
//#endregion
|
|
685
|
+
//#region src/zoo/moonshine-streaming/transcriber.ts
|
|
686
|
+
/** Streaming Moonshine transcriber: the v1 decode loop over the v2 frontend,
|
|
687
|
+
* encoder and adapter. Long clips split on quiet points into per-utterance
|
|
688
|
+
* chunks; each utterance is zero-padded to whole 80-sample frames.
|
|
689
|
+
*
|
|
690
|
+
* Decoding runs in bursts: within a burst each step's argmax id feeds the next
|
|
691
|
+
* step's embedding on device, so the GPU runs the whole burst with no CPU
|
|
692
|
+
* round-trip. A burst may overshoot EOS; those ids are discarded.
|
|
693
|
+
*
|
|
694
|
+
* Rope tables and the per-layer K/V caches are allocated once at
|
|
695
|
+
* DECODE_POSITIONS rows and written in place, so utterances reuse them and
|
|
696
|
+
* transcribe() calls must stay sequential. */
|
|
697
|
+
const SAMPLE_RATE = 16e3;
|
|
698
|
+
const TOKENS_PER_SECOND = 6.5;
|
|
699
|
+
const BURST = 8;
|
|
700
|
+
const DECODE_POSITIONS = 256;
|
|
701
|
+
const MAX_CHUNK_SECONDS = 30;
|
|
702
|
+
const SPLIT_SEARCH_SECONDS = 5;
|
|
703
|
+
async function createTranscriber(sd, tokenizer, opts = {}) {
|
|
704
|
+
const root = defaultRoot();
|
|
705
|
+
if (sd.metadata["model"] && sd.metadata["model"] !== "moonshine-streaming") throw new Error(`moonshine-streaming weights: file is for model '${sd.metadata["model"]}', expected 'moonshine-streaming'`);
|
|
706
|
+
const cfg = opts.cfg ?? presetFromStateDict(sd);
|
|
707
|
+
const dec = decoderView(cfg);
|
|
708
|
+
const perf = opts.perf;
|
|
709
|
+
const burstMax = Math.max(1, Math.min(perf?.burst ?? BURST, DECODE_POSITIONS));
|
|
710
|
+
const ex = gpuExecutor(root, { perf: perf?.gpuPerf });
|
|
711
|
+
const half = opts.weightDtype === "f16";
|
|
712
|
+
const model = new MoonshineStreamingModel(cfg);
|
|
713
|
+
if (half) model.half(root);
|
|
714
|
+
try {
|
|
715
|
+
await model.loadStateDict(sd, {
|
|
716
|
+
root,
|
|
717
|
+
onProgress: opts.onProgress
|
|
718
|
+
});
|
|
719
|
+
} catch (err) {
|
|
720
|
+
model.dispose();
|
|
721
|
+
throw err;
|
|
722
|
+
}
|
|
723
|
+
const actTensor = (data, dims) => tensor(root, data, {
|
|
724
|
+
elems: data.length,
|
|
725
|
+
dtype: half ? "f16" : "f32",
|
|
726
|
+
dims
|
|
727
|
+
});
|
|
728
|
+
const tables = buildRopeTables(DECODE_POSITIONS, dec);
|
|
729
|
+
const decCos = tensor(root, tables.cos, [DECODE_POSITIONS, dec.headDim]);
|
|
730
|
+
const decSin = tensor(root, tables.sin, [DECODE_POSITIONS, dec.headDim]);
|
|
731
|
+
const qw = dec.heads * dec.headDim;
|
|
732
|
+
const kvInit = new Float32Array(DECODE_POSITIONS * qw);
|
|
733
|
+
const selfKV = Array.from({ length: dec.decLayers }, () => ({
|
|
734
|
+
k: actTensor(kvInit, [DECODE_POSITIONS, qw]),
|
|
735
|
+
v: actTensor(kvInit, [DECODE_POSITIONS, qw])
|
|
736
|
+
}));
|
|
737
|
+
const idsBuf = tensor(root, new Float32Array(DECODE_POSITIONS), [DECODE_POSITIONS, 1]);
|
|
738
|
+
const padToFrames = (audio) => {
|
|
739
|
+
if (audio.length === 0) throw new Error("transcribe: empty audio");
|
|
740
|
+
const frames = Math.ceil(audio.length / cfg.frameLen);
|
|
741
|
+
if (frames * cfg.frameLen === audio.length) return audio;
|
|
742
|
+
const padded = new Float32Array(frames * cfg.frameLen);
|
|
743
|
+
padded.set(audio);
|
|
744
|
+
return padded;
|
|
745
|
+
};
|
|
746
|
+
const transcribeUtterance = async (rawAudio) => {
|
|
747
|
+
perf?.onPhase?.("encode");
|
|
748
|
+
const t0 = performance.now();
|
|
749
|
+
const audio = padToFrames(rawAudio);
|
|
750
|
+
const frames = frontendFrames(audio.length, cfg.frameLen);
|
|
751
|
+
const x = actTensor(audio, [frames.input, cfg.frameLen]);
|
|
752
|
+
const adapterOut = model.adapter.forward(model.encoder.forward(model.frontend.forward(x)));
|
|
753
|
+
const cross = model.decoder.precomputeCrossKV(adapterOut);
|
|
754
|
+
evalValues([adapterOut, ...cross.flatMap((c) => [c.k, c.v])], ex);
|
|
755
|
+
if (perf?.syncAfterEncode) await root.device.queue.onSubmittedWorkDone();
|
|
756
|
+
const encodeMs = performance.now() - t0;
|
|
757
|
+
const maxTokens = Math.min(DECODE_POSITIONS, Math.ceil(rawAudio.length / SAMPLE_RATE * TOKENS_PER_SECOND));
|
|
758
|
+
const release = (kvs) => {
|
|
759
|
+
for (const { k, v } of kvs) {
|
|
760
|
+
ex.releaseOutput(k.buffer, k.shape);
|
|
761
|
+
k.markReleased();
|
|
762
|
+
ex.releaseOutput(v.buffer, v.shape);
|
|
763
|
+
v.markReleased();
|
|
764
|
+
}
|
|
765
|
+
};
|
|
766
|
+
ex.releaseOutput(adapterOut.buffer, adapterOut.shape);
|
|
767
|
+
adapterOut.markReleased();
|
|
768
|
+
const ids = [cfg.bos];
|
|
769
|
+
perf?.onPhase?.("decode");
|
|
770
|
+
const t1 = performance.now();
|
|
771
|
+
decode: while (ids.length < maxTokens) {
|
|
772
|
+
const base = ids.length - 1;
|
|
773
|
+
const burst = Math.min(burstMax, maxTokens - ids.length);
|
|
774
|
+
const tBuild = performance.now();
|
|
775
|
+
const idRows = [];
|
|
776
|
+
let x1 = model.embed([ids[base]]);
|
|
777
|
+
for (let j = 0; j < burst; j++) {
|
|
778
|
+
const t = base + j;
|
|
779
|
+
const h = model.decoder.forwardStep(x1, slice(decCos, 0, t, t + 1), slice(decSin, 0, t, t + 1), cross, selfKV, t);
|
|
780
|
+
const idVal = model.nextTokenId(h);
|
|
781
|
+
idRows.push(writeRows(idsBuf, idVal, t));
|
|
782
|
+
if (j + 1 < burst) x1 = model.embedFrom(idVal);
|
|
783
|
+
}
|
|
784
|
+
const tEval = performance.now();
|
|
785
|
+
evalValues(idRows, ex);
|
|
786
|
+
const tRead = performance.now();
|
|
787
|
+
const data = await ex.readback(idsBuf.buffer, {
|
|
788
|
+
elems: base + burst,
|
|
789
|
+
dtype: "f32",
|
|
790
|
+
dims: [base + burst, 1]
|
|
791
|
+
});
|
|
792
|
+
perf?.onBurst?.({
|
|
793
|
+
positions: burst,
|
|
794
|
+
buildMs: tEval - tBuild,
|
|
795
|
+
evalMs: tRead - tEval,
|
|
796
|
+
readbackMs: performance.now() - tRead
|
|
797
|
+
});
|
|
798
|
+
for (let j = 0; j < burst; j++) {
|
|
799
|
+
const best = data[base + j];
|
|
800
|
+
ids.push(best);
|
|
801
|
+
if (best === cfg.eos) break decode;
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
release(cross);
|
|
805
|
+
return {
|
|
806
|
+
ids,
|
|
807
|
+
text: decodeTokens(tokenizer, ids),
|
|
808
|
+
timings: {
|
|
809
|
+
encodeMs,
|
|
810
|
+
decodeMs: performance.now() - t1,
|
|
811
|
+
steps: ids.length - 1
|
|
812
|
+
}
|
|
813
|
+
};
|
|
814
|
+
};
|
|
815
|
+
const chunkOpts = {
|
|
816
|
+
maxSeconds: opts.chunking?.maxSeconds ?? MAX_CHUNK_SECONDS,
|
|
817
|
+
searchSeconds: opts.chunking?.searchSeconds ?? SPLIT_SEARCH_SECONDS,
|
|
818
|
+
sampleRate: SAMPLE_RATE
|
|
819
|
+
};
|
|
820
|
+
const transcribeAll = async (audio) => {
|
|
821
|
+
const chunks = splitIntoChunks(audio, chunkOpts);
|
|
822
|
+
if (chunks.length === 1) return transcribeUtterance(audio);
|
|
823
|
+
const results = [];
|
|
824
|
+
for (const chunk of chunks) results.push(await transcribeUtterance(chunk));
|
|
825
|
+
return {
|
|
826
|
+
ids: results.flatMap((r) => r.ids),
|
|
827
|
+
text: results.map((r) => r.text.trim()).filter((t) => t.length > 0).join(" "),
|
|
828
|
+
timings: results.reduce((sum, r) => ({
|
|
829
|
+
encodeMs: sum.encodeMs + r.timings.encodeMs,
|
|
830
|
+
decodeMs: sum.decodeMs + r.timings.decodeMs,
|
|
831
|
+
steps: sum.steps + r.timings.steps
|
|
832
|
+
}), {
|
|
833
|
+
encodeMs: 0,
|
|
834
|
+
decodeMs: 0,
|
|
835
|
+
steps: 0
|
|
836
|
+
})
|
|
837
|
+
};
|
|
838
|
+
};
|
|
839
|
+
const dispose = () => {
|
|
840
|
+
model.dispose();
|
|
841
|
+
for (const v of [
|
|
842
|
+
decCos,
|
|
843
|
+
decSin,
|
|
844
|
+
idsBuf,
|
|
845
|
+
...selfKV.flatMap((kv) => [kv.k, kv.v])
|
|
846
|
+
]) v.buffer.destroy();
|
|
847
|
+
ex.dispose();
|
|
848
|
+
};
|
|
849
|
+
let inFlight = Promise.resolve();
|
|
850
|
+
let disposed = false;
|
|
851
|
+
const transcribe = (audio) => {
|
|
852
|
+
if (disposed) return Promise.reject(/* @__PURE__ */ new Error("transcriber is disposed"));
|
|
853
|
+
const job = inFlight.then(() => transcribeAll(audio));
|
|
854
|
+
inFlight = job.catch(() => void 0);
|
|
855
|
+
return job;
|
|
856
|
+
};
|
|
857
|
+
return {
|
|
858
|
+
minSamples: cfg.frameLen,
|
|
859
|
+
transcribe,
|
|
860
|
+
dispose: () => {
|
|
861
|
+
disposed = true;
|
|
862
|
+
dispose();
|
|
863
|
+
}
|
|
864
|
+
};
|
|
865
|
+
}
|
|
866
|
+
//#endregion
|
|
867
|
+
//#region src/zoo/tasks/audio/audioInput.ts
|
|
868
|
+
/** Audio input helpers: an encoded clip or raw microphone samples in, the
|
|
869
|
+
* 16 kHz mono Float32Array the speech models take out. Both use the
|
|
870
|
+
* browser's own decoder and resampler through an OfflineAudioContext, so
|
|
871
|
+
* they run in the browser only. Importing the file is safe anywhere, the
|
|
872
|
+
* browser objects appear only inside the functions. */
|
|
873
|
+
/** Sample rate the speech models take. */
|
|
874
|
+
const SPEECH_SAMPLE_RATE = 16e3;
|
|
875
|
+
/** Decodes any clip the browser can play (wav, mp3, ogg, ...) to 16 kHz
|
|
876
|
+
* mono samples. Stereo is mixed down to one channel. */
|
|
877
|
+
async function decodeAudio(data) {
|
|
878
|
+
const decoded = await new OfflineAudioContext(1, 1, SPEECH_SAMPLE_RATE).decodeAudioData(data);
|
|
879
|
+
if (decoded.numberOfChannels === 1) return decoded.getChannelData(0);
|
|
880
|
+
const mix = new OfflineAudioContext(1, decoded.length, SPEECH_SAMPLE_RATE);
|
|
881
|
+
const src = mix.createBufferSource();
|
|
882
|
+
src.buffer = decoded;
|
|
883
|
+
src.connect(mix.destination);
|
|
884
|
+
src.start();
|
|
885
|
+
return (await mix.startRendering()).getChannelData(0);
|
|
886
|
+
}
|
|
887
|
+
/** Resamples raw mono samples recorded at `sourceRate` to 16 kHz. Returns
|
|
888
|
+
* the input as is when it is already at 16 kHz. Use it on microphone
|
|
889
|
+
* samples, a lossy re-encode is not needed. */
|
|
890
|
+
async function resampleAudio(samples, sourceRate) {
|
|
891
|
+
if (sourceRate === 16e3) return samples;
|
|
892
|
+
const frames = Math.ceil(samples.length * SPEECH_SAMPLE_RATE / sourceRate);
|
|
893
|
+
const off = new OfflineAudioContext(1, frames, SPEECH_SAMPLE_RATE);
|
|
894
|
+
const buf = off.createBuffer(1, samples.length, sourceRate);
|
|
895
|
+
buf.copyToChannel(samples, 0);
|
|
896
|
+
const src = off.createBufferSource();
|
|
897
|
+
src.buffer = buf;
|
|
898
|
+
src.connect(off.destination);
|
|
899
|
+
src.start();
|
|
900
|
+
return (await off.startRendering()).getChannelData(0);
|
|
901
|
+
}
|
|
902
|
+
//#endregion
|
|
903
|
+
//#region src/zoo/tasks/audio/speechToText.ts
|
|
904
|
+
/** Speech to text task: 16 kHz mono audio in, as a clip or live from the
|
|
905
|
+
* microphone, text out. Runs Moonshine on the initRunntime() device. */
|
|
906
|
+
/** New audio between two transcriptions of the sentence being spoken. */
|
|
907
|
+
const TICK_SAMPLES = SPEECH_SAMPLE_RATE / 2;
|
|
908
|
+
/** Speech this long commits even without a pause. */
|
|
909
|
+
const MAX_SEGMENT_SECONDS = 15;
|
|
910
|
+
/** Loads the checkpoint and the tokenizer, warms every GPU path once, and
|
|
911
|
+
* returns the runner. With no config, loads Moonshine tiny from the
|
|
912
|
+
* Hugging Face Hub. */
|
|
913
|
+
async function createSpeechToText(config = {}, opts = {}) {
|
|
914
|
+
const scope = createResourceScope();
|
|
915
|
+
try {
|
|
916
|
+
const device = defaultRoot().device;
|
|
917
|
+
const arch = config.arch ?? "moonshine";
|
|
918
|
+
const defaults = arch === "moonshine" ? models.speechToText.MOONSHINE.TINY.DEFAULT : models.speechToText.MOONSHINE_STREAMING.TINY.DEFAULT;
|
|
919
|
+
const { modelPath = defaults.modelPath, tokenizerPath = defaults.tokenizerPath, dtype = supportsF16() ? "f16" : "f32" } = config;
|
|
920
|
+
const cacheId = (path) => typeof path === "string" ? `${arch}-v1/${path.replace(/^https?:\/\//, "")}` : void 0;
|
|
921
|
+
const sd = await openWeights(modelPath, {
|
|
922
|
+
...opts,
|
|
923
|
+
cacheId: opts.cacheId ?? cacheId(modelPath)
|
|
924
|
+
});
|
|
925
|
+
const tokenizer = moonshineTokenizerAsset(await fetchJson(tokenizerPath, {
|
|
926
|
+
...opts,
|
|
927
|
+
cacheId: cacheId(tokenizerPath)
|
|
928
|
+
}));
|
|
929
|
+
throwIfAborted(opts.signal);
|
|
930
|
+
const make = arch === "moonshine" ? createTranscriber$1 : createTranscriber;
|
|
931
|
+
const transcriber = scope.track(await make(sd, tokenizer, {
|
|
932
|
+
weightDtype: dtype,
|
|
933
|
+
onProgress: opts.onProgress
|
|
934
|
+
}));
|
|
935
|
+
throwIfAborted(opts.signal);
|
|
936
|
+
await warmUp(device, async () => {
|
|
937
|
+
const silence = new Float32Array(SPEECH_SAMPLE_RATE);
|
|
938
|
+
return Float32Array.from((await transcriber.transcribe(silence)).ids);
|
|
939
|
+
});
|
|
940
|
+
let disposed = false;
|
|
941
|
+
let queue = Promise.resolve();
|
|
942
|
+
const enqueue = (audio) => {
|
|
943
|
+
const run = queue.then(() => transcriber.transcribe(audio));
|
|
944
|
+
queue = run.catch(() => void 0);
|
|
945
|
+
return run;
|
|
946
|
+
};
|
|
947
|
+
let live;
|
|
948
|
+
async function* liveLoop(session) {
|
|
949
|
+
let last = {
|
|
950
|
+
committed: "",
|
|
951
|
+
nonCommitted: ""
|
|
952
|
+
};
|
|
953
|
+
try {
|
|
954
|
+
while (true) {
|
|
955
|
+
if (!session.stopped && session.fresh < TICK_SAMPLES) {
|
|
956
|
+
await new Promise((resolve) => {
|
|
957
|
+
session.wake = resolve;
|
|
958
|
+
});
|
|
959
|
+
continue;
|
|
960
|
+
}
|
|
961
|
+
const final = session.stopped;
|
|
962
|
+
session.fresh = 0;
|
|
963
|
+
const audio = concat(session.chunks, session.length);
|
|
964
|
+
const segment = analyzeSegment(audio, {
|
|
965
|
+
sampleRate: SPEECH_SAMPLE_RATE,
|
|
966
|
+
maxSegmentSeconds: MAX_SEGMENT_SECONDS
|
|
967
|
+
});
|
|
968
|
+
let text = "";
|
|
969
|
+
if (segment.hasSpeech && audio.length >= transcriber.minSamples) text = (await enqueue(audio)).text.trim();
|
|
970
|
+
else if (segment.seconds >= MAX_SEGMENT_SECONDS) keepTail(session, SPEECH_SAMPLE_RATE);
|
|
971
|
+
let committed = last.committed;
|
|
972
|
+
if (final || segment.shouldFinalize) {
|
|
973
|
+
if (text) committed = committed ? `${committed} ${text}` : text;
|
|
974
|
+
text = "";
|
|
975
|
+
session.chunks = [];
|
|
976
|
+
session.length = 0;
|
|
977
|
+
}
|
|
978
|
+
if (committed !== last.committed || text !== last.nonCommitted) {
|
|
979
|
+
last = {
|
|
980
|
+
committed,
|
|
981
|
+
nonCommitted: text
|
|
982
|
+
};
|
|
983
|
+
yield last;
|
|
984
|
+
}
|
|
985
|
+
if (final) return;
|
|
986
|
+
}
|
|
987
|
+
} finally {
|
|
988
|
+
if (live === session) live = void 0;
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
return {
|
|
992
|
+
async transcribe(audio) {
|
|
993
|
+
if (disposed) throw new Error("speech to text is disposed");
|
|
994
|
+
if (!(audio instanceof Float32Array)) throw new Error("transcribe: audio must be a Float32Array");
|
|
995
|
+
if (audio.length < transcriber.minSamples) return "";
|
|
996
|
+
return (await enqueue(audio)).text;
|
|
997
|
+
},
|
|
998
|
+
stream() {
|
|
999
|
+
if (disposed) throw new Error("speech to text is disposed");
|
|
1000
|
+
if (live) throw new Error("stream: a stream is open, stop it first");
|
|
1001
|
+
live = {
|
|
1002
|
+
chunks: [],
|
|
1003
|
+
length: 0,
|
|
1004
|
+
fresh: 0,
|
|
1005
|
+
stopped: false,
|
|
1006
|
+
wake: () => {}
|
|
1007
|
+
};
|
|
1008
|
+
return liveLoop(live);
|
|
1009
|
+
},
|
|
1010
|
+
streamInsert(samples) {
|
|
1011
|
+
if (!live || live.stopped) throw new Error("streamInsert: no stream is open");
|
|
1012
|
+
if (!(samples instanceof Float32Array)) throw new Error("streamInsert: samples must be a Float32Array");
|
|
1013
|
+
live.chunks.push(samples.slice());
|
|
1014
|
+
live.length += samples.length;
|
|
1015
|
+
live.fresh += samples.length;
|
|
1016
|
+
if (live.fresh >= TICK_SAMPLES) live.wake();
|
|
1017
|
+
},
|
|
1018
|
+
streamStop() {
|
|
1019
|
+
if (!live) return;
|
|
1020
|
+
live.stopped = true;
|
|
1021
|
+
live.wake();
|
|
1022
|
+
},
|
|
1023
|
+
dispose() {
|
|
1024
|
+
disposed = true;
|
|
1025
|
+
if (live) {
|
|
1026
|
+
live.chunks = [];
|
|
1027
|
+
live.length = 0;
|
|
1028
|
+
live.stopped = true;
|
|
1029
|
+
live.wake();
|
|
1030
|
+
}
|
|
1031
|
+
scope.dispose();
|
|
1032
|
+
}
|
|
1033
|
+
};
|
|
1034
|
+
} catch (err) {
|
|
1035
|
+
scope.dispose();
|
|
1036
|
+
throw err;
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
/** Joins the chunks into one array of `length` samples. */
|
|
1040
|
+
function concat(chunks, length) {
|
|
1041
|
+
const out = new Float32Array(length);
|
|
1042
|
+
let at = 0;
|
|
1043
|
+
for (const c of chunks) {
|
|
1044
|
+
out.set(c, at);
|
|
1045
|
+
at += c.length;
|
|
1046
|
+
}
|
|
1047
|
+
return out;
|
|
1048
|
+
}
|
|
1049
|
+
/** Drops chunks from the front until about `samples` remain. */
|
|
1050
|
+
function keepTail(session, samples) {
|
|
1051
|
+
while (session.chunks.length > 1 && session.length - session.chunks[0].length >= samples) session.length -= session.chunks.shift().length;
|
|
1052
|
+
}
|
|
1053
|
+
//#endregion
|
|
1054
|
+
//#region src/zoo/yolo26/pipeline.ts
|
|
1055
|
+
const sigmoid = (x) => 1 / (1 + Math.exp(-x));
|
|
1056
|
+
function scanCells(levels, numClasses, confThreshold) {
|
|
1057
|
+
const logitThreshold = Math.log(confThreshold / (1 - confThreshold));
|
|
1058
|
+
const hits = [];
|
|
1059
|
+
levels.forEach(({ data, h, w, stride }, levelIdx) => {
|
|
1060
|
+
const hw = h * w;
|
|
1061
|
+
for (let i = 0; i < hw; i++) {
|
|
1062
|
+
let bestLogit = -Infinity;
|
|
1063
|
+
let bestClass = 0;
|
|
1064
|
+
for (let c = 0; c < numClasses; c++) {
|
|
1065
|
+
const v = data[(4 + c) * hw + i];
|
|
1066
|
+
if (v > bestLogit) {
|
|
1067
|
+
bestLogit = v;
|
|
1068
|
+
bestClass = c;
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
if (bestLogit < logitThreshold) continue;
|
|
1072
|
+
const score = sigmoid(bestLogit);
|
|
1073
|
+
const ax = i % w + .5;
|
|
1074
|
+
const ay = Math.floor(i / w) + .5;
|
|
1075
|
+
const l = data[i];
|
|
1076
|
+
const t = data[hw + i];
|
|
1077
|
+
const r = data[2 * hw + i];
|
|
1078
|
+
const b = data[3 * hw + i];
|
|
1079
|
+
hits.push({
|
|
1080
|
+
levelIdx,
|
|
1081
|
+
cell: i,
|
|
1082
|
+
det: {
|
|
1083
|
+
x1: (ax - l) * stride,
|
|
1084
|
+
y1: (ay - t) * stride,
|
|
1085
|
+
x2: (ax + r) * stride,
|
|
1086
|
+
y2: (ay + b) * stride,
|
|
1087
|
+
score,
|
|
1088
|
+
classId: bestClass,
|
|
1089
|
+
label: COCO_NAMES[bestClass] ?? `cls${bestClass}`
|
|
1090
|
+
}
|
|
1091
|
+
});
|
|
1092
|
+
}
|
|
1093
|
+
});
|
|
1094
|
+
hits.sort((a, b) => b.det.score - a.det.score);
|
|
1095
|
+
return hits;
|
|
1096
|
+
}
|
|
1097
|
+
function decodeDetections(levels, opts = {}) {
|
|
1098
|
+
const { numClasses = 80, confThreshold = .3, maxDet = 300 } = opts;
|
|
1099
|
+
return scanCells(levels, numClasses, confThreshold).slice(0, maxDet).map((h) => h.det);
|
|
1100
|
+
}
|
|
1101
|
+
function decodeSegmentations(levels, proto, opts = {}) {
|
|
1102
|
+
const { numClasses = 80, numMasks = 32, confThreshold = .3, maxDet = 300, inputSize = 640 } = opts;
|
|
1103
|
+
const protoStride = inputSize / proto.w;
|
|
1104
|
+
const phw = proto.h * proto.w;
|
|
1105
|
+
return scanCells(levels, numClasses, confThreshold).slice(0, maxDet).map(({ levelIdx, cell, det }) => {
|
|
1106
|
+
const { data, h, w } = levels[levelIdx];
|
|
1107
|
+
const hw = h * w;
|
|
1108
|
+
const coeffs = new Float32Array(numMasks);
|
|
1109
|
+
for (let c = 0; c < numMasks; c++) coeffs[c] = data[(4 + numClasses + c) * hw + cell];
|
|
1110
|
+
const px1 = Math.min(proto.w, Math.max(0, Math.floor(det.x1 / protoStride)));
|
|
1111
|
+
const py1 = Math.min(proto.h, Math.max(0, Math.floor(det.y1 / protoStride)));
|
|
1112
|
+
const px2 = Math.min(proto.w, Math.max(px1, Math.ceil(det.x2 / protoStride)));
|
|
1113
|
+
const py2 = Math.min(proto.h, Math.max(py1, Math.ceil(det.y2 / protoStride)));
|
|
1114
|
+
const maskW = px2 - px1;
|
|
1115
|
+
const maskH = py2 - py1;
|
|
1116
|
+
const mask = new Uint8Array(maskW * maskH);
|
|
1117
|
+
for (let y = 0; y < maskH; y++) for (let x = 0; x < maskW; x++) {
|
|
1118
|
+
const pi = (py1 + y) * proto.w + (px1 + x);
|
|
1119
|
+
let v = 0;
|
|
1120
|
+
for (let c = 0; c < numMasks; c++) v += coeffs[c] * proto.data[c * phw + pi];
|
|
1121
|
+
mask[y * maskW + x] = Math.round(255 * sigmoid(v));
|
|
1122
|
+
}
|
|
1123
|
+
return {
|
|
1124
|
+
...det,
|
|
1125
|
+
mask,
|
|
1126
|
+
maskX: px1,
|
|
1127
|
+
maskY: py1,
|
|
1128
|
+
maskW,
|
|
1129
|
+
maskH
|
|
1130
|
+
};
|
|
1131
|
+
});
|
|
1132
|
+
}
|
|
1133
|
+
function decodePoses(levels, opts = {}) {
|
|
1134
|
+
const { numClasses = 1, numKeypoints = 17, confThreshold = .3, maxDet = 300 } = opts;
|
|
1135
|
+
return scanCells(levels, numClasses, confThreshold).slice(0, maxDet).map(({ levelIdx, cell, det }) => {
|
|
1136
|
+
const { data, h, w, stride } = levels[levelIdx];
|
|
1137
|
+
const hw = h * w;
|
|
1138
|
+
const ax = cell % w + .5;
|
|
1139
|
+
const ay = Math.floor(cell / w) + .5;
|
|
1140
|
+
const kpts = [];
|
|
1141
|
+
for (let k = 0; k < numKeypoints; k++) {
|
|
1142
|
+
const o = (4 + numClasses + 3 * k) * hw + cell;
|
|
1143
|
+
kpts.push({
|
|
1144
|
+
x: (data[o] + ax) * stride,
|
|
1145
|
+
y: (data[o + hw] + ay) * stride,
|
|
1146
|
+
v: sigmoid(data[o + 2 * hw])
|
|
1147
|
+
});
|
|
1148
|
+
}
|
|
1149
|
+
return {
|
|
1150
|
+
...det,
|
|
1151
|
+
kpts
|
|
1152
|
+
};
|
|
1153
|
+
});
|
|
1154
|
+
}
|
|
1155
|
+
/** The 17 COCO body landmarks, in model output order. */
|
|
1156
|
+
const COCO_LANDMARKS = [
|
|
1157
|
+
"nose",
|
|
1158
|
+
"leftEye",
|
|
1159
|
+
"rightEye",
|
|
1160
|
+
"leftEar",
|
|
1161
|
+
"rightEar",
|
|
1162
|
+
"leftShoulder",
|
|
1163
|
+
"rightShoulder",
|
|
1164
|
+
"leftElbow",
|
|
1165
|
+
"rightElbow",
|
|
1166
|
+
"leftWrist",
|
|
1167
|
+
"rightWrist",
|
|
1168
|
+
"leftHip",
|
|
1169
|
+
"rightHip",
|
|
1170
|
+
"leftKnee",
|
|
1171
|
+
"rightKnee",
|
|
1172
|
+
"leftAnkle",
|
|
1173
|
+
"rightAnkle"
|
|
1174
|
+
];
|
|
1175
|
+
/** Limbs to draw between COCO keypoints; ear-to-shoulder is left out because it
|
|
1176
|
+
* cuts across the face. */
|
|
1177
|
+
const COCO_SKELETON = [
|
|
1178
|
+
[15, 13],
|
|
1179
|
+
[13, 11],
|
|
1180
|
+
[16, 14],
|
|
1181
|
+
[14, 12],
|
|
1182
|
+
[11, 12],
|
|
1183
|
+
[5, 11],
|
|
1184
|
+
[6, 12],
|
|
1185
|
+
[5, 6],
|
|
1186
|
+
[5, 7],
|
|
1187
|
+
[6, 8],
|
|
1188
|
+
[7, 9],
|
|
1189
|
+
[8, 10],
|
|
1190
|
+
[1, 2],
|
|
1191
|
+
[0, 1],
|
|
1192
|
+
[0, 2],
|
|
1193
|
+
[1, 3],
|
|
1194
|
+
[2, 4]
|
|
1195
|
+
];
|
|
1196
|
+
const COCO_NAMES = [
|
|
1197
|
+
"person",
|
|
1198
|
+
"bicycle",
|
|
1199
|
+
"car",
|
|
1200
|
+
"motorcycle",
|
|
1201
|
+
"airplane",
|
|
1202
|
+
"bus",
|
|
1203
|
+
"train",
|
|
1204
|
+
"truck",
|
|
1205
|
+
"boat",
|
|
1206
|
+
"traffic light",
|
|
1207
|
+
"fire hydrant",
|
|
1208
|
+
"stop sign",
|
|
1209
|
+
"parking meter",
|
|
1210
|
+
"bench",
|
|
1211
|
+
"bird",
|
|
1212
|
+
"cat",
|
|
1213
|
+
"dog",
|
|
1214
|
+
"horse",
|
|
1215
|
+
"sheep",
|
|
1216
|
+
"cow",
|
|
1217
|
+
"elephant",
|
|
1218
|
+
"bear",
|
|
1219
|
+
"zebra",
|
|
1220
|
+
"giraffe",
|
|
1221
|
+
"backpack",
|
|
1222
|
+
"umbrella",
|
|
1223
|
+
"handbag",
|
|
1224
|
+
"tie",
|
|
1225
|
+
"suitcase",
|
|
1226
|
+
"frisbee",
|
|
1227
|
+
"skis",
|
|
1228
|
+
"snowboard",
|
|
1229
|
+
"sports ball",
|
|
1230
|
+
"kite",
|
|
1231
|
+
"baseball bat",
|
|
1232
|
+
"baseball glove",
|
|
1233
|
+
"skateboard",
|
|
1234
|
+
"surfboard",
|
|
1235
|
+
"tennis racket",
|
|
1236
|
+
"bottle",
|
|
1237
|
+
"wine glass",
|
|
1238
|
+
"cup",
|
|
1239
|
+
"fork",
|
|
1240
|
+
"knife",
|
|
1241
|
+
"spoon",
|
|
1242
|
+
"bowl",
|
|
1243
|
+
"banana",
|
|
1244
|
+
"apple",
|
|
1245
|
+
"sandwich",
|
|
1246
|
+
"orange",
|
|
1247
|
+
"broccoli",
|
|
1248
|
+
"carrot",
|
|
1249
|
+
"hot dog",
|
|
1250
|
+
"pizza",
|
|
1251
|
+
"donut",
|
|
1252
|
+
"cake",
|
|
1253
|
+
"chair",
|
|
1254
|
+
"couch",
|
|
1255
|
+
"potted plant",
|
|
1256
|
+
"bed",
|
|
1257
|
+
"dining table",
|
|
1258
|
+
"toilet",
|
|
1259
|
+
"tv",
|
|
1260
|
+
"laptop",
|
|
1261
|
+
"mouse",
|
|
1262
|
+
"remote",
|
|
1263
|
+
"keyboard",
|
|
1264
|
+
"cell phone",
|
|
1265
|
+
"microwave",
|
|
1266
|
+
"oven",
|
|
1267
|
+
"toaster",
|
|
1268
|
+
"sink",
|
|
1269
|
+
"refrigerator",
|
|
1270
|
+
"book",
|
|
1271
|
+
"clock",
|
|
1272
|
+
"vase",
|
|
1273
|
+
"scissors",
|
|
1274
|
+
"teddy bear",
|
|
1275
|
+
"hair drier",
|
|
1276
|
+
"toothbrush"
|
|
1277
|
+
];
|
|
1278
|
+
//#endregion
|
|
1279
|
+
//#region src/zoo/tasks/cv/ops/point.ts
|
|
1280
|
+
/** The resize as numbers: how many input pixels one image pixel takes, and
|
|
1281
|
+
* where the image's top-left corner lands inside the input. */
|
|
1282
|
+
function resizeTransform(options) {
|
|
1283
|
+
const { from, to, resizeMode } = options;
|
|
1284
|
+
switch (resizeMode) {
|
|
1285
|
+
case "stretch": return {
|
|
1286
|
+
scaleX: from.width / to.width,
|
|
1287
|
+
scaleY: from.height / to.height,
|
|
1288
|
+
offsetX: 0,
|
|
1289
|
+
offsetY: 0
|
|
1290
|
+
};
|
|
1291
|
+
case "letterbox": {
|
|
1292
|
+
const scale = Math.min(from.width / to.width, from.height / to.height);
|
|
1293
|
+
return {
|
|
1294
|
+
scaleX: scale,
|
|
1295
|
+
scaleY: scale,
|
|
1296
|
+
offsetX: (from.width - to.width * scale) / 2,
|
|
1297
|
+
offsetY: (from.height - to.height * scale) / 2
|
|
1298
|
+
};
|
|
1299
|
+
}
|
|
1300
|
+
case "crop": {
|
|
1301
|
+
const side = Math.min(to.width, to.height) * (options.cropFraction ?? 1);
|
|
1302
|
+
const scaleX = from.width / side;
|
|
1303
|
+
const scaleY = from.height / side;
|
|
1304
|
+
return {
|
|
1305
|
+
scaleX,
|
|
1306
|
+
scaleY,
|
|
1307
|
+
offsetX: -(to.width - side) / 2 * scaleX,
|
|
1308
|
+
offsetY: -(to.height - side) / 2 * scaleY
|
|
1309
|
+
};
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
/** Maps a point from model-input coordinates to original-image coordinates. */
|
|
1314
|
+
function scalePoint(point, options) {
|
|
1315
|
+
const { scaleX, scaleY, offsetX, offsetY } = resizeTransform(options);
|
|
1316
|
+
return {
|
|
1317
|
+
x: (point.x - offsetX) / scaleX,
|
|
1318
|
+
y: (point.y - offsetY) / scaleY
|
|
1319
|
+
};
|
|
1320
|
+
}
|
|
1321
|
+
//#endregion
|
|
1322
|
+
//#region src/zoo/tasks/cv/ops/box.ts
|
|
1323
|
+
/** Bounding boxes in three formats, plus scaling back onto the original image. */
|
|
1324
|
+
/** Builds a box from four numbers in the given format. */
|
|
1325
|
+
function decodeBox(tuple, format) {
|
|
1326
|
+
const [a, b, c, d] = tuple;
|
|
1327
|
+
switch (format) {
|
|
1328
|
+
case "xyxy": return {
|
|
1329
|
+
format: "xyxy",
|
|
1330
|
+
xmin: a,
|
|
1331
|
+
ymin: b,
|
|
1332
|
+
xmax: c,
|
|
1333
|
+
ymax: d
|
|
1334
|
+
};
|
|
1335
|
+
case "xywh": return {
|
|
1336
|
+
format: "xywh",
|
|
1337
|
+
xmin: a,
|
|
1338
|
+
ymin: b,
|
|
1339
|
+
w: c,
|
|
1340
|
+
h: d
|
|
1341
|
+
};
|
|
1342
|
+
case "cxcywh": return {
|
|
1343
|
+
format: "cxcywh",
|
|
1344
|
+
cx: a,
|
|
1345
|
+
cy: b,
|
|
1346
|
+
w: c,
|
|
1347
|
+
h: d
|
|
1348
|
+
};
|
|
1349
|
+
default: throw new Error(`unknown box format ${String(format)}`);
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
/** Maps a box from model-input coordinates to original-image coordinates. */
|
|
1353
|
+
function scaleBox(box, options) {
|
|
1354
|
+
const { scaleX, scaleY } = resizeTransform(options);
|
|
1355
|
+
switch (box.format) {
|
|
1356
|
+
case "xyxy": {
|
|
1357
|
+
const min = scalePoint({
|
|
1358
|
+
x: box.xmin,
|
|
1359
|
+
y: box.ymin
|
|
1360
|
+
}, options);
|
|
1361
|
+
const max = scalePoint({
|
|
1362
|
+
x: box.xmax,
|
|
1363
|
+
y: box.ymax
|
|
1364
|
+
}, options);
|
|
1365
|
+
return {
|
|
1366
|
+
format: "xyxy",
|
|
1367
|
+
xmin: min.x,
|
|
1368
|
+
ymin: min.y,
|
|
1369
|
+
xmax: max.x,
|
|
1370
|
+
ymax: max.y
|
|
1371
|
+
};
|
|
1372
|
+
}
|
|
1373
|
+
case "xywh": {
|
|
1374
|
+
const min = scalePoint({
|
|
1375
|
+
x: box.xmin,
|
|
1376
|
+
y: box.ymin
|
|
1377
|
+
}, options);
|
|
1378
|
+
return {
|
|
1379
|
+
format: "xywh",
|
|
1380
|
+
xmin: min.x,
|
|
1381
|
+
ymin: min.y,
|
|
1382
|
+
w: box.w / scaleX,
|
|
1383
|
+
h: box.h / scaleY
|
|
1384
|
+
};
|
|
1385
|
+
}
|
|
1386
|
+
case "cxcywh": {
|
|
1387
|
+
const center = scalePoint({
|
|
1388
|
+
x: box.cx,
|
|
1389
|
+
y: box.cy
|
|
1390
|
+
}, options);
|
|
1391
|
+
return {
|
|
1392
|
+
format: "cxcywh",
|
|
1393
|
+
cx: center.x,
|
|
1394
|
+
cy: center.y,
|
|
1395
|
+
w: box.w / scaleX,
|
|
1396
|
+
h: box.h / scaleY
|
|
1397
|
+
};
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
//#endregion
|
|
1402
|
+
//#region src/zoo/tasks/cv/image.ts
|
|
1403
|
+
/** Bytes per pixel for each format. */
|
|
1404
|
+
const FORMAT_CHANNELS = {
|
|
1405
|
+
rgb: 3,
|
|
1406
|
+
rgba: 4,
|
|
1407
|
+
bgr: 3,
|
|
1408
|
+
bgra: 4,
|
|
1409
|
+
gray: 1
|
|
1410
|
+
};
|
|
1411
|
+
/** Throws unless `width` and `height` are positive integers. */
|
|
1412
|
+
function checkImageSize(width, height, what = "image") {
|
|
1413
|
+
if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) throw new Error(`${what}: ${width}x${height} is not a positive size`);
|
|
1414
|
+
}
|
|
1415
|
+
/** Wraps raw bytes as an ImageBuffer. Checks the size and the byte count. */
|
|
1416
|
+
function imageBuffer(data, width, height, format = "rgba") {
|
|
1417
|
+
checkImageSize(width, height);
|
|
1418
|
+
const expected = width * height * FORMAT_CHANNELS[format];
|
|
1419
|
+
if (data.length !== expected) throw new Error(`image: ${data.length} bytes for ${width}x${height} ${format}, expected ${expected}`);
|
|
1420
|
+
return {
|
|
1421
|
+
data,
|
|
1422
|
+
width,
|
|
1423
|
+
height,
|
|
1424
|
+
format,
|
|
1425
|
+
layout: "hwc"
|
|
1426
|
+
};
|
|
1427
|
+
}
|
|
1428
|
+
/** Views canvas ImageData as an ImageBuffer. No copy. */
|
|
1429
|
+
function imageBufferFromImageData(img) {
|
|
1430
|
+
return {
|
|
1431
|
+
data: img.data,
|
|
1432
|
+
width: img.width,
|
|
1433
|
+
height: img.height,
|
|
1434
|
+
format: "rgba",
|
|
1435
|
+
layout: "hwc"
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
//#endregion
|
|
1439
|
+
//#region src/zoo/tasks/cv/utils/imagePreprocessor.ts
|
|
1440
|
+
/** CPU image preprocessor: any ImageBuffer in, the model's RGB CHW f32 input
|
|
1441
|
+
* out. Resizes (stretch / letterbox / crop), converts the pixel format,
|
|
1442
|
+
* moves channels first and normalizes, all in one pass over the output. */
|
|
1443
|
+
/** Index of the R, G and B byte inside one pixel of each format. */
|
|
1444
|
+
const RGB_INDEX = {
|
|
1445
|
+
rgb: [
|
|
1446
|
+
0,
|
|
1447
|
+
1,
|
|
1448
|
+
2
|
|
1449
|
+
],
|
|
1450
|
+
rgba: [
|
|
1451
|
+
0,
|
|
1452
|
+
1,
|
|
1453
|
+
2
|
|
1454
|
+
],
|
|
1455
|
+
bgr: [
|
|
1456
|
+
2,
|
|
1457
|
+
1,
|
|
1458
|
+
0
|
|
1459
|
+
],
|
|
1460
|
+
bgra: [
|
|
1461
|
+
2,
|
|
1462
|
+
1,
|
|
1463
|
+
0
|
|
1464
|
+
],
|
|
1465
|
+
gray: [
|
|
1466
|
+
0,
|
|
1467
|
+
0,
|
|
1468
|
+
0
|
|
1469
|
+
]
|
|
1470
|
+
};
|
|
1471
|
+
function perChannel(v, name) {
|
|
1472
|
+
if (typeof v === "number") return [
|
|
1473
|
+
v,
|
|
1474
|
+
v,
|
|
1475
|
+
v
|
|
1476
|
+
];
|
|
1477
|
+
if (v.length !== 3) throw new Error(`normalize.${name}: expected 3 values, got ${v.length}`);
|
|
1478
|
+
return [
|
|
1479
|
+
v[0],
|
|
1480
|
+
v[1],
|
|
1481
|
+
v[2]
|
|
1482
|
+
];
|
|
1483
|
+
}
|
|
1484
|
+
function buildAxis(outLen, srcLen, scale, offset, interpolation) {
|
|
1485
|
+
const i0 = new Int32Array(outLen);
|
|
1486
|
+
const i1 = new Int32Array(outLen);
|
|
1487
|
+
const w = new Float32Array(outLen);
|
|
1488
|
+
const inside = new Uint8Array(outLen);
|
|
1489
|
+
for (let d = 0; d < outLen; d++) {
|
|
1490
|
+
const s = (d + .5 - offset) / scale;
|
|
1491
|
+
if (s < 0 || s >= srcLen) continue;
|
|
1492
|
+
inside[d] = 1;
|
|
1493
|
+
if (interpolation === "nearest") {
|
|
1494
|
+
i0[d] = i1[d] = Math.min(srcLen - 1, Math.floor(s));
|
|
1495
|
+
continue;
|
|
1496
|
+
}
|
|
1497
|
+
const f = s - .5;
|
|
1498
|
+
const a = Math.floor(f);
|
|
1499
|
+
i0[d] = Math.max(0, a);
|
|
1500
|
+
i1[d] = Math.min(srcLen - 1, a + 1);
|
|
1501
|
+
w[d] = a < 0 ? 0 : f - a;
|
|
1502
|
+
}
|
|
1503
|
+
return {
|
|
1504
|
+
i0,
|
|
1505
|
+
i1,
|
|
1506
|
+
w,
|
|
1507
|
+
inside
|
|
1508
|
+
};
|
|
1509
|
+
}
|
|
1510
|
+
function createImagePreprocessor(options, size) {
|
|
1511
|
+
const { width, height, resizeMode } = {
|
|
1512
|
+
...size,
|
|
1513
|
+
resizeMode: options.resizeMode
|
|
1514
|
+
};
|
|
1515
|
+
checkImageSize(width, height, "input size");
|
|
1516
|
+
const interpolation = options.interpolation ?? "linear";
|
|
1517
|
+
const cropFraction = options.cropFraction ?? 1;
|
|
1518
|
+
if (!(cropFraction > 0 && cropFraction <= 1)) throw new Error(`cropFraction: expected a number in (0, 1], got ${cropFraction}`);
|
|
1519
|
+
const padValue = options.padValue ?? 114;
|
|
1520
|
+
const alpha = perChannel(options.normalizeOpts?.alpha ?? 1 / 255, "alpha");
|
|
1521
|
+
const beta = perChannel(options.normalizeOpts?.beta ?? 0, "beta");
|
|
1522
|
+
const hw = width * height;
|
|
1523
|
+
const scaleOptions = (input) => ({
|
|
1524
|
+
from: {
|
|
1525
|
+
width,
|
|
1526
|
+
height
|
|
1527
|
+
},
|
|
1528
|
+
to: {
|
|
1529
|
+
width: input.width,
|
|
1530
|
+
height: input.height
|
|
1531
|
+
},
|
|
1532
|
+
resizeMode,
|
|
1533
|
+
cropFraction
|
|
1534
|
+
});
|
|
1535
|
+
const process = (input, out) => {
|
|
1536
|
+
out ??= new Float32Array(3 * hw);
|
|
1537
|
+
if (out.length !== 3 * hw) throw new Error(`out: ${out.length} floats, expected ${3 * hw}`);
|
|
1538
|
+
checkImageSize(input.width, input.height);
|
|
1539
|
+
const ch = FORMAT_CHANNELS[input.format];
|
|
1540
|
+
const { data } = input;
|
|
1541
|
+
if (data.length !== input.width * input.height * ch) throw new Error(`image: ${data.length} bytes for ${input.width}x${input.height} ${input.format}`);
|
|
1542
|
+
const [ri, gi, bi] = RGB_INDEX[input.format];
|
|
1543
|
+
const [ar, ag, ab] = alpha;
|
|
1544
|
+
const [br, bg, bb] = beta;
|
|
1545
|
+
const t = resizeTransform(scaleOptions(input));
|
|
1546
|
+
if (t.scaleX === 1 && t.scaleY === 1 && t.offsetX === 0 && t.offsetY === 0) {
|
|
1547
|
+
for (let i = 0; i < hw; i++) {
|
|
1548
|
+
const p = i * ch;
|
|
1549
|
+
out[i] = data[p + ri] * ar + br;
|
|
1550
|
+
out[hw + i] = data[p + gi] * ag + bg;
|
|
1551
|
+
out[2 * hw + i] = data[p + bi] * ab + bb;
|
|
1552
|
+
}
|
|
1553
|
+
return out;
|
|
1554
|
+
}
|
|
1555
|
+
const xs = buildAxis(width, input.width, t.scaleX, t.offsetX, interpolation);
|
|
1556
|
+
const ys = buildAxis(height, input.height, t.scaleY, t.offsetY, interpolation);
|
|
1557
|
+
const padR = padValue * ar + br;
|
|
1558
|
+
const padG = padValue * ag + bg;
|
|
1559
|
+
const padB = padValue * ab + bb;
|
|
1560
|
+
const rowStride = input.width * ch;
|
|
1561
|
+
for (let y = 0; y < height; y++) {
|
|
1562
|
+
const rowIn = ys.inside[y];
|
|
1563
|
+
const r0 = ys.i0[y] * rowStride;
|
|
1564
|
+
const r1 = ys.i1[y] * rowStride;
|
|
1565
|
+
const wy = ys.w[y];
|
|
1566
|
+
for (let x = 0; x < width; x++) {
|
|
1567
|
+
const o = y * width + x;
|
|
1568
|
+
if (!rowIn || !xs.inside[x]) {
|
|
1569
|
+
out[o] = padR;
|
|
1570
|
+
out[hw + o] = padG;
|
|
1571
|
+
out[2 * hw + o] = padB;
|
|
1572
|
+
continue;
|
|
1573
|
+
}
|
|
1574
|
+
const c0 = xs.i0[x] * ch;
|
|
1575
|
+
const c1 = xs.i1[x] * ch;
|
|
1576
|
+
const wx = xs.w[x];
|
|
1577
|
+
const w00 = (1 - wy) * (1 - wx);
|
|
1578
|
+
const w01 = (1 - wy) * wx;
|
|
1579
|
+
const w10 = wy * (1 - wx);
|
|
1580
|
+
const w11 = wy * wx;
|
|
1581
|
+
const r = data[r0 + c0 + ri] * w00 + data[r0 + c1 + ri] * w01 + data[r1 + c0 + ri] * w10 + data[r1 + c1 + ri] * w11;
|
|
1582
|
+
const g = data[r0 + c0 + gi] * w00 + data[r0 + c1 + gi] * w01 + data[r1 + c0 + gi] * w10 + data[r1 + c1 + gi] * w11;
|
|
1583
|
+
const b = data[r0 + c0 + bi] * w00 + data[r0 + c1 + bi] * w01 + data[r1 + c0 + bi] * w10 + data[r1 + c1 + bi] * w11;
|
|
1584
|
+
out[o] = r * ar + br;
|
|
1585
|
+
out[hw + o] = g * ag + bg;
|
|
1586
|
+
out[2 * hw + o] = b * ab + bb;
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
return out;
|
|
1590
|
+
};
|
|
1591
|
+
return {
|
|
1592
|
+
width,
|
|
1593
|
+
height,
|
|
1594
|
+
process,
|
|
1595
|
+
scaleOptions
|
|
1596
|
+
};
|
|
1597
|
+
}
|
|
1598
|
+
//#endregion
|
|
1599
|
+
//#region src/zoo/tasks/cv/objectDetection.ts
|
|
1600
|
+
/** Object detection task: an image in, the objects in it out, as labeled
|
|
1601
|
+
* boxes. Runs YOLO26 on the initRunntime() device. */
|
|
1602
|
+
/** Loads the checkpoint, warms the GPU path once, and returns the runner.
|
|
1603
|
+
* Needs a device with `shader-f16`. */
|
|
1604
|
+
async function createObjectDetector(config = {}, opts = {}) {
|
|
1605
|
+
const scope = createResourceScope();
|
|
1606
|
+
try {
|
|
1607
|
+
const { modelPath = models.objectDetection.YOLO26.DEFAULT.modelPath, variant, inputSize: size, resizeMode = "letterbox" } = config;
|
|
1608
|
+
const path = typeof modelPath === "string" ? modelPath.replace(/^https?:\/\//, "") : void 0;
|
|
1609
|
+
const sd = await openWeights(modelPath, {
|
|
1610
|
+
...opts,
|
|
1611
|
+
cacheId: opts.cacheId ?? (path && `yolo26-v1/${path}`)
|
|
1612
|
+
});
|
|
1613
|
+
throwIfAborted(opts.signal);
|
|
1614
|
+
const detector = scope.track(await createDetector$1(sd, {
|
|
1615
|
+
task: "detect",
|
|
1616
|
+
variant,
|
|
1617
|
+
inputSize: size,
|
|
1618
|
+
onProgress: opts.onProgress
|
|
1619
|
+
}));
|
|
1620
|
+
const { inputSize, numClasses } = detector;
|
|
1621
|
+
const labels = numClasses === COCO_NAMES.length ? COCO_NAMES : Array.from({ length: numClasses }, (_, i) => `class ${i}`);
|
|
1622
|
+
const preprocessor = createImagePreprocessor({ resizeMode }, {
|
|
1623
|
+
width: inputSize,
|
|
1624
|
+
height: inputSize
|
|
1625
|
+
});
|
|
1626
|
+
const pixels = new Float32Array(3 * inputSize * inputSize);
|
|
1627
|
+
let disposed = false;
|
|
1628
|
+
let queue = Promise.resolve();
|
|
1629
|
+
return {
|
|
1630
|
+
labels,
|
|
1631
|
+
async detectObjects(image, options = {}) {
|
|
1632
|
+
if (disposed) throw new Error("object detector is disposed");
|
|
1633
|
+
const { confidenceThreshold = .3, maxDetections = 300 } = options;
|
|
1634
|
+
const run = queue.then(() => detector.run(preprocessor.process(image, pixels)));
|
|
1635
|
+
queue = run.catch(() => void 0);
|
|
1636
|
+
const { levels } = await run;
|
|
1637
|
+
const scale = preprocessor.scaleOptions(image);
|
|
1638
|
+
return decodeDetections(levels, {
|
|
1639
|
+
numClasses,
|
|
1640
|
+
confThreshold: confidenceThreshold,
|
|
1641
|
+
maxDet: maxDetections
|
|
1642
|
+
}).map((d) => {
|
|
1643
|
+
const box = scaleBox(decodeBox([
|
|
1644
|
+
d.x1,
|
|
1645
|
+
d.y1,
|
|
1646
|
+
d.x2,
|
|
1647
|
+
d.y2
|
|
1648
|
+
], "xyxy"), scale);
|
|
1649
|
+
return {
|
|
1650
|
+
label: labels[d.classId],
|
|
1651
|
+
classId: d.classId,
|
|
1652
|
+
confidence: d.score,
|
|
1653
|
+
box: {
|
|
1654
|
+
format: "xyxy",
|
|
1655
|
+
xmin: clamp$2(box.xmin, 0, image.width),
|
|
1656
|
+
ymin: clamp$2(box.ymin, 0, image.height),
|
|
1657
|
+
xmax: clamp$2(box.xmax, 0, image.width),
|
|
1658
|
+
ymax: clamp$2(box.ymax, 0, image.height)
|
|
1659
|
+
}
|
|
1660
|
+
};
|
|
1661
|
+
});
|
|
1662
|
+
},
|
|
1663
|
+
dispose() {
|
|
1664
|
+
disposed = true;
|
|
1665
|
+
scope.dispose();
|
|
1666
|
+
}
|
|
1667
|
+
};
|
|
1668
|
+
} catch (err) {
|
|
1669
|
+
scope.dispose();
|
|
1670
|
+
throw err;
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
const clamp$2 = (v, lo, hi) => Math.min(hi, Math.max(lo, v));
|
|
1674
|
+
//#endregion
|
|
1675
|
+
//#region src/zoo/tasks/cv/instanceSegmentation.ts
|
|
1676
|
+
/** Instance segmentation task: an image in, the objects in it out, each
|
|
1677
|
+
* with a box and a pixel mask. Runs YOLO26 segment on the initRunntime() device. */
|
|
1678
|
+
/** Loads the checkpoint, warms the GPU path once, and returns the runner.
|
|
1679
|
+
* Needs a device with `shader-f16`. */
|
|
1680
|
+
async function createInstanceSegmenter(config = {}, opts = {}) {
|
|
1681
|
+
const scope = createResourceScope();
|
|
1682
|
+
try {
|
|
1683
|
+
const { modelPath = models.instanceSegmentation.YOLO26_SEG.DEFAULT.modelPath, variant, inputSize: size, resizeMode = "letterbox" } = config;
|
|
1684
|
+
const path = typeof modelPath === "string" ? modelPath.replace(/^https?:\/\//, "") : void 0;
|
|
1685
|
+
const sd = await openWeights(modelPath, {
|
|
1686
|
+
...opts,
|
|
1687
|
+
cacheId: opts.cacheId ?? (path && `yolo26-seg-v1/${path}`)
|
|
1688
|
+
});
|
|
1689
|
+
throwIfAborted(opts.signal);
|
|
1690
|
+
const detector = scope.track(await createDetector$1(sd, {
|
|
1691
|
+
task: "segment",
|
|
1692
|
+
variant,
|
|
1693
|
+
inputSize: size,
|
|
1694
|
+
onProgress: opts.onProgress
|
|
1695
|
+
}));
|
|
1696
|
+
const { inputSize, numClasses } = detector;
|
|
1697
|
+
const labels = numClasses === COCO_NAMES.length ? COCO_NAMES : Array.from({ length: numClasses }, (_, i) => `class ${i}`);
|
|
1698
|
+
const preprocessor = createImagePreprocessor({ resizeMode }, {
|
|
1699
|
+
width: inputSize,
|
|
1700
|
+
height: inputSize
|
|
1701
|
+
});
|
|
1702
|
+
const pixels = new Float32Array(3 * inputSize * inputSize);
|
|
1703
|
+
let disposed = false;
|
|
1704
|
+
let queue = Promise.resolve();
|
|
1705
|
+
return {
|
|
1706
|
+
labels,
|
|
1707
|
+
async segmentInstances(image, options = {}) {
|
|
1708
|
+
if (disposed) throw new Error("instance segmenter is disposed");
|
|
1709
|
+
const { confidenceThreshold = .3, maxDetections = 300 } = options;
|
|
1710
|
+
const run = queue.then(() => detector.run(preprocessor.process(image, pixels)));
|
|
1711
|
+
queue = run.catch(() => void 0);
|
|
1712
|
+
const { levels, proto } = await run;
|
|
1713
|
+
if (!proto) throw new Error("yolo26 segment: the forward pass gave no mask prototypes");
|
|
1714
|
+
const scale = preprocessor.scaleOptions(image);
|
|
1715
|
+
const toInput = resizeTransform(scale);
|
|
1716
|
+
const protoStride = inputSize / proto.w;
|
|
1717
|
+
return decodeSegmentations(levels, proto, {
|
|
1718
|
+
numClasses,
|
|
1719
|
+
numMasks: 32,
|
|
1720
|
+
confThreshold: confidenceThreshold,
|
|
1721
|
+
maxDet: maxDetections,
|
|
1722
|
+
inputSize
|
|
1723
|
+
}).map((d) => {
|
|
1724
|
+
const raw = scaleBox(decodeBox([
|
|
1725
|
+
d.x1,
|
|
1726
|
+
d.y1,
|
|
1727
|
+
d.x2,
|
|
1728
|
+
d.y2
|
|
1729
|
+
], "xyxy"), scale);
|
|
1730
|
+
const box = {
|
|
1731
|
+
format: "xyxy",
|
|
1732
|
+
xmin: clamp$1(raw.xmin, 0, image.width),
|
|
1733
|
+
ymin: clamp$1(raw.ymin, 0, image.height),
|
|
1734
|
+
xmax: clamp$1(raw.xmax, 0, image.width),
|
|
1735
|
+
ymax: clamp$1(raw.ymax, 0, image.height)
|
|
1736
|
+
};
|
|
1737
|
+
const cell = (v, axis) => axis === "x" ? (v * toInput.scaleX + toInput.offsetX) / protoStride - d.maskX : (v * toInput.scaleY + toInput.offsetY) / protoStride - d.maskY;
|
|
1738
|
+
const mask = resampleMask(d.mask, d.maskW, d.maskH, cell(box.xmin, "x"), cell(box.ymin, "y"), cell(box.xmax, "x"), cell(box.ymax, "y"));
|
|
1739
|
+
return {
|
|
1740
|
+
label: labels[d.classId],
|
|
1741
|
+
classId: d.classId,
|
|
1742
|
+
confidence: d.score,
|
|
1743
|
+
box,
|
|
1744
|
+
mask
|
|
1745
|
+
};
|
|
1746
|
+
});
|
|
1747
|
+
},
|
|
1748
|
+
dispose() {
|
|
1749
|
+
disposed = true;
|
|
1750
|
+
scope.dispose();
|
|
1751
|
+
}
|
|
1752
|
+
};
|
|
1753
|
+
} catch (err) {
|
|
1754
|
+
scope.dispose();
|
|
1755
|
+
throw err;
|
|
1756
|
+
}
|
|
1757
|
+
}
|
|
1758
|
+
const clamp$1 = (v, lo, hi) => Math.min(hi, Math.max(lo, v));
|
|
1759
|
+
/** Bilinear resample of the cell window [u0,u1)×[v0,v1) of a `w`×`h`
|
|
1760
|
+
* coverage map into a gray image of the same cell count. */
|
|
1761
|
+
function resampleMask(src, w, h, u0, v0, u1, v1) {
|
|
1762
|
+
const width = Math.max(1, Math.round(u1 - u0));
|
|
1763
|
+
const height = Math.max(1, Math.round(v1 - v0));
|
|
1764
|
+
const data = new Uint8Array(width * height);
|
|
1765
|
+
const at = (x, y) => src[clamp$1(y, 0, h - 1) * w + clamp$1(x, 0, w - 1)];
|
|
1766
|
+
const du = (u1 - u0) / width;
|
|
1767
|
+
const dv = (v1 - v0) / height;
|
|
1768
|
+
for (let j = 0; j < height; j++) {
|
|
1769
|
+
const v = v0 + (j + .5) * dv - .5;
|
|
1770
|
+
const y = Math.floor(v);
|
|
1771
|
+
const fy = v - y;
|
|
1772
|
+
for (let i = 0; i < width; i++) {
|
|
1773
|
+
const u = u0 + (i + .5) * du - .5;
|
|
1774
|
+
const x = Math.floor(u);
|
|
1775
|
+
const fx = u - x;
|
|
1776
|
+
const top = at(x, y) * (1 - fx) + at(x + 1, y) * fx;
|
|
1777
|
+
const bottom = at(x, y + 1) * (1 - fx) + at(x + 1, y + 1) * fx;
|
|
1778
|
+
data[j * width + i] = Math.round(top * (1 - fy) + bottom * fy);
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
return {
|
|
1782
|
+
data,
|
|
1783
|
+
width,
|
|
1784
|
+
height,
|
|
1785
|
+
format: "gray",
|
|
1786
|
+
layout: "hwc"
|
|
1787
|
+
};
|
|
1788
|
+
}
|
|
1789
|
+
//#endregion
|
|
1790
|
+
//#region src/zoo/tasks/cv/keypointDetection.ts
|
|
1791
|
+
/** Keypoint detection task: an image in, the people in it out, each with a
|
|
1792
|
+
* box and 17 COCO landmarks. Runs YOLO26 pose on the initRunntime() device. */
|
|
1793
|
+
/** Loads the checkpoint, warms the GPU path once, and returns the runner.
|
|
1794
|
+
* Needs a device with `shader-f16`. */
|
|
1795
|
+
async function createKeypointDetector(config = {}, opts = {}) {
|
|
1796
|
+
const scope = createResourceScope();
|
|
1797
|
+
try {
|
|
1798
|
+
const { modelPath = models.keypointDetection.YOLO26_POSE.DEFAULT.modelPath, variant, inputSize: size, resizeMode = "letterbox" } = config;
|
|
1799
|
+
const path = typeof modelPath === "string" ? modelPath.replace(/^https?:\/\//, "") : void 0;
|
|
1800
|
+
const sd = await openWeights(modelPath, {
|
|
1801
|
+
...opts,
|
|
1802
|
+
cacheId: opts.cacheId ?? (path && `yolo26-pose-v1/${path}`)
|
|
1803
|
+
});
|
|
1804
|
+
throwIfAborted(opts.signal);
|
|
1805
|
+
const detector = scope.track(await createDetector$1(sd, {
|
|
1806
|
+
task: "pose",
|
|
1807
|
+
variant,
|
|
1808
|
+
inputSize: size,
|
|
1809
|
+
onProgress: opts.onProgress
|
|
1810
|
+
}));
|
|
1811
|
+
const { inputSize, numClasses } = detector;
|
|
1812
|
+
const preprocessor = createImagePreprocessor({ resizeMode }, {
|
|
1813
|
+
width: inputSize,
|
|
1814
|
+
height: inputSize
|
|
1815
|
+
});
|
|
1816
|
+
const pixels = new Float32Array(3 * inputSize * inputSize);
|
|
1817
|
+
let disposed = false;
|
|
1818
|
+
let queue = Promise.resolve();
|
|
1819
|
+
return {
|
|
1820
|
+
landmarkNames: COCO_LANDMARKS,
|
|
1821
|
+
skeleton: COCO_SKELETON.map(([a, b]) => [COCO_LANDMARKS[a], COCO_LANDMARKS[b]]),
|
|
1822
|
+
async detectKeypoints(image, options = {}) {
|
|
1823
|
+
if (disposed) throw new Error("keypoint detector is disposed");
|
|
1824
|
+
const { confidenceThreshold = .3, maxDetections = 300 } = options;
|
|
1825
|
+
const run = queue.then(() => detector.run(preprocessor.process(image, pixels)));
|
|
1826
|
+
queue = run.catch(() => void 0);
|
|
1827
|
+
const { levels } = await run;
|
|
1828
|
+
const scale = preprocessor.scaleOptions(image);
|
|
1829
|
+
return decodePoses(levels, {
|
|
1830
|
+
numClasses,
|
|
1831
|
+
numKeypoints: 17,
|
|
1832
|
+
confThreshold: confidenceThreshold,
|
|
1833
|
+
maxDet: maxDetections
|
|
1834
|
+
}).map((p) => {
|
|
1835
|
+
const box = scaleBox(decodeBox([
|
|
1836
|
+
p.x1,
|
|
1837
|
+
p.y1,
|
|
1838
|
+
p.x2,
|
|
1839
|
+
p.y2
|
|
1840
|
+
], "xyxy"), scale);
|
|
1841
|
+
const landmarks = {};
|
|
1842
|
+
p.kpts.forEach((k, i) => {
|
|
1843
|
+
landmarks[COCO_LANDMARKS[i]] = {
|
|
1844
|
+
...scalePoint(k, scale),
|
|
1845
|
+
confidence: k.v
|
|
1846
|
+
};
|
|
1847
|
+
});
|
|
1848
|
+
return {
|
|
1849
|
+
box: {
|
|
1850
|
+
format: "xyxy",
|
|
1851
|
+
xmin: clamp(box.xmin, 0, image.width),
|
|
1852
|
+
ymin: clamp(box.ymin, 0, image.height),
|
|
1853
|
+
xmax: clamp(box.xmax, 0, image.width),
|
|
1854
|
+
ymax: clamp(box.ymax, 0, image.height)
|
|
1855
|
+
},
|
|
1856
|
+
confidence: p.score,
|
|
1857
|
+
landmarks
|
|
1858
|
+
};
|
|
1859
|
+
});
|
|
1860
|
+
},
|
|
1861
|
+
dispose() {
|
|
1862
|
+
disposed = true;
|
|
1863
|
+
scope.dispose();
|
|
1864
|
+
}
|
|
1865
|
+
};
|
|
1866
|
+
} catch (err) {
|
|
1867
|
+
scope.dispose();
|
|
1868
|
+
throw err;
|
|
1869
|
+
}
|
|
1870
|
+
}
|
|
1871
|
+
const clamp = (v, lo, hi) => Math.min(hi, Math.max(lo, v));
|
|
1872
|
+
//#endregion
|
|
1873
|
+
//#region src/zoo/tasks/cv/depthEstimation.ts
|
|
1874
|
+
/** Depth estimation task: an image in, a relative depth map out. Runs
|
|
1875
|
+
* DepthART on the initRunntime() device. */
|
|
1876
|
+
/** Loads the checkpoint, warms the GPU path once, and returns the runner. */
|
|
1877
|
+
async function createDepthEstimator(config = {}, opts = {}) {
|
|
1878
|
+
if (!supportsF16()) throw new Error("createDepthEstimator: this device has no shader-f16, which the conv kernels need. Request it with tgpu.init({ device: { optionalFeatures: ['shader-f16'] } })");
|
|
1879
|
+
const scope = createResourceScope();
|
|
1880
|
+
try {
|
|
1881
|
+
const { modelPath = models.depthEstimation.DEPTHART.DEFAULT.modelPath, variant, resizeMode = "stretch" } = config;
|
|
1882
|
+
const path = typeof modelPath === "string" ? modelPath.replace(/^https?:\/\//, "") : void 0;
|
|
1883
|
+
const sd = await openWeights(modelPath, {
|
|
1884
|
+
...opts,
|
|
1885
|
+
cacheId: opts.cacheId ?? (path && `depthart-v1/${path}`)
|
|
1886
|
+
});
|
|
1887
|
+
throwIfAborted(opts.signal);
|
|
1888
|
+
const estimator = scope.track(await createEstimator(sd, {
|
|
1889
|
+
variant,
|
|
1890
|
+
onProgress: opts.onProgress
|
|
1891
|
+
}));
|
|
1892
|
+
const { inputSize } = estimator;
|
|
1893
|
+
const { mean, std } = DEPTHART_CONFIGS[estimator.variant];
|
|
1894
|
+
const preprocessor = createImagePreprocessor({
|
|
1895
|
+
resizeMode,
|
|
1896
|
+
normalizeOpts: {
|
|
1897
|
+
alpha: std.map((s) => 1 / (255 * s)),
|
|
1898
|
+
beta: mean.map((m, c) => -m / std[c])
|
|
1899
|
+
}
|
|
1900
|
+
}, {
|
|
1901
|
+
width: inputSize,
|
|
1902
|
+
height: inputSize
|
|
1903
|
+
});
|
|
1904
|
+
const pixels = new Float32Array(3 * inputSize * inputSize);
|
|
1905
|
+
let disposed = false;
|
|
1906
|
+
let queue = Promise.resolve();
|
|
1907
|
+
return {
|
|
1908
|
+
inputSize,
|
|
1909
|
+
async estimateDepth(image) {
|
|
1910
|
+
if (disposed) throw new Error("depth estimator is disposed");
|
|
1911
|
+
const run = queue.then(() => estimator.run(preprocessor.process(image, pixels), inputSize, inputSize));
|
|
1912
|
+
queue = run.catch(() => void 0);
|
|
1913
|
+
return {
|
|
1914
|
+
width: inputSize,
|
|
1915
|
+
height: inputSize,
|
|
1916
|
+
data: await run
|
|
1917
|
+
};
|
|
1918
|
+
},
|
|
1919
|
+
dispose() {
|
|
1920
|
+
disposed = true;
|
|
1921
|
+
scope.dispose();
|
|
1922
|
+
}
|
|
1923
|
+
};
|
|
1924
|
+
} catch (err) {
|
|
1925
|
+
scope.dispose();
|
|
1926
|
+
throw err;
|
|
1927
|
+
}
|
|
1928
|
+
}
|
|
1929
|
+
//#endregion
|
|
1930
|
+
//#region src/zoo/mobilenetv4/labels.ts
|
|
1931
|
+
/** The 1000 ImageNet-1k class names, in class index order, as the Hub
|
|
1932
|
+
* config's `id2label` spells them. */
|
|
1933
|
+
const IMAGENET1K_LABELS = [
|
|
1934
|
+
"tench, Tinca tinca",
|
|
1935
|
+
"goldfish, Carassius auratus",
|
|
1936
|
+
"great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias",
|
|
1937
|
+
"tiger shark, Galeocerdo cuvieri",
|
|
1938
|
+
"hammerhead, hammerhead shark",
|
|
1939
|
+
"electric ray, crampfish, numbfish, torpedo",
|
|
1940
|
+
"stingray",
|
|
1941
|
+
"cock",
|
|
1942
|
+
"hen",
|
|
1943
|
+
"ostrich, Struthio camelus",
|
|
1944
|
+
"brambling, Fringilla montifringilla",
|
|
1945
|
+
"goldfinch, Carduelis carduelis",
|
|
1946
|
+
"house finch, linnet, Carpodacus mexicanus",
|
|
1947
|
+
"junco, snowbird",
|
|
1948
|
+
"indigo bunting, indigo finch, indigo bird, Passerina cyanea",
|
|
1949
|
+
"robin, American robin, Turdus migratorius",
|
|
1950
|
+
"bulbul",
|
|
1951
|
+
"jay",
|
|
1952
|
+
"magpie",
|
|
1953
|
+
"chickadee",
|
|
1954
|
+
"water ouzel, dipper",
|
|
1955
|
+
"kite",
|
|
1956
|
+
"bald eagle, American eagle, Haliaeetus leucocephalus",
|
|
1957
|
+
"vulture",
|
|
1958
|
+
"great grey owl, great gray owl, Strix nebulosa",
|
|
1959
|
+
"European fire salamander, Salamandra salamandra",
|
|
1960
|
+
"common newt, Triturus vulgaris",
|
|
1961
|
+
"eft",
|
|
1962
|
+
"spotted salamander, Ambystoma maculatum",
|
|
1963
|
+
"axolotl, mud puppy, Ambystoma mexicanum",
|
|
1964
|
+
"bullfrog, Rana catesbeiana",
|
|
1965
|
+
"tree frog, tree-frog",
|
|
1966
|
+
"tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui",
|
|
1967
|
+
"loggerhead, loggerhead turtle, Caretta caretta",
|
|
1968
|
+
"leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea",
|
|
1969
|
+
"mud turtle",
|
|
1970
|
+
"terrapin",
|
|
1971
|
+
"box turtle, box tortoise",
|
|
1972
|
+
"banded gecko",
|
|
1973
|
+
"common iguana, iguana, Iguana iguana",
|
|
1974
|
+
"American chameleon, anole, Anolis carolinensis",
|
|
1975
|
+
"whiptail, whiptail lizard",
|
|
1976
|
+
"agama",
|
|
1977
|
+
"frilled lizard, Chlamydosaurus kingi",
|
|
1978
|
+
"alligator lizard",
|
|
1979
|
+
"Gila monster, Heloderma suspectum",
|
|
1980
|
+
"green lizard, Lacerta viridis",
|
|
1981
|
+
"African chameleon, Chamaeleo chamaeleon",
|
|
1982
|
+
"Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis",
|
|
1983
|
+
"African crocodile, Nile crocodile, Crocodylus niloticus",
|
|
1984
|
+
"American alligator, Alligator mississipiensis",
|
|
1985
|
+
"triceratops",
|
|
1986
|
+
"thunder snake, worm snake, Carphophis amoenus",
|
|
1987
|
+
"ringneck snake, ring-necked snake, ring snake",
|
|
1988
|
+
"hognose snake, puff adder, sand viper",
|
|
1989
|
+
"green snake, grass snake",
|
|
1990
|
+
"king snake, kingsnake",
|
|
1991
|
+
"garter snake, grass snake",
|
|
1992
|
+
"water snake",
|
|
1993
|
+
"vine snake",
|
|
1994
|
+
"night snake, Hypsiglena torquata",
|
|
1995
|
+
"boa constrictor, Constrictor constrictor",
|
|
1996
|
+
"rock python, rock snake, Python sebae",
|
|
1997
|
+
"Indian cobra, Naja naja",
|
|
1998
|
+
"green mamba",
|
|
1999
|
+
"sea snake",
|
|
2000
|
+
"horned viper, cerastes, sand viper, horned asp, Cerastes cornutus",
|
|
2001
|
+
"diamondback, diamondback rattlesnake, Crotalus adamanteus",
|
|
2002
|
+
"sidewinder, horned rattlesnake, Crotalus cerastes",
|
|
2003
|
+
"trilobite",
|
|
2004
|
+
"harvestman, daddy longlegs, Phalangium opilio",
|
|
2005
|
+
"scorpion",
|
|
2006
|
+
"black and gold garden spider, Argiope aurantia",
|
|
2007
|
+
"barn spider, Araneus cavaticus",
|
|
2008
|
+
"garden spider, Aranea diademata",
|
|
2009
|
+
"black widow, Latrodectus mactans",
|
|
2010
|
+
"tarantula",
|
|
2011
|
+
"wolf spider, hunting spider",
|
|
2012
|
+
"tick",
|
|
2013
|
+
"centipede",
|
|
2014
|
+
"black grouse",
|
|
2015
|
+
"ptarmigan",
|
|
2016
|
+
"ruffed grouse, partridge, Bonasa umbellus",
|
|
2017
|
+
"prairie chicken, prairie grouse, prairie fowl",
|
|
2018
|
+
"peacock",
|
|
2019
|
+
"quail",
|
|
2020
|
+
"partridge",
|
|
2021
|
+
"African grey, African gray, Psittacus erithacus",
|
|
2022
|
+
"macaw",
|
|
2023
|
+
"sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita",
|
|
2024
|
+
"lorikeet",
|
|
2025
|
+
"coucal",
|
|
2026
|
+
"bee eater",
|
|
2027
|
+
"hornbill",
|
|
2028
|
+
"hummingbird",
|
|
2029
|
+
"jacamar",
|
|
2030
|
+
"toucan",
|
|
2031
|
+
"drake",
|
|
2032
|
+
"red-breasted merganser, Mergus serrator",
|
|
2033
|
+
"goose",
|
|
2034
|
+
"black swan, Cygnus atratus",
|
|
2035
|
+
"tusker",
|
|
2036
|
+
"echidna, spiny anteater, anteater",
|
|
2037
|
+
"platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus",
|
|
2038
|
+
"wallaby, brush kangaroo",
|
|
2039
|
+
"koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus",
|
|
2040
|
+
"wombat",
|
|
2041
|
+
"jellyfish",
|
|
2042
|
+
"sea anemone, anemone",
|
|
2043
|
+
"brain coral",
|
|
2044
|
+
"flatworm, platyhelminth",
|
|
2045
|
+
"nematode, nematode worm, roundworm",
|
|
2046
|
+
"conch",
|
|
2047
|
+
"snail",
|
|
2048
|
+
"slug",
|
|
2049
|
+
"sea slug, nudibranch",
|
|
2050
|
+
"chiton, coat-of-mail shell, sea cradle, polyplacophore",
|
|
2051
|
+
"chambered nautilus, pearly nautilus, nautilus",
|
|
2052
|
+
"Dungeness crab, Cancer magister",
|
|
2053
|
+
"rock crab, Cancer irroratus",
|
|
2054
|
+
"fiddler crab",
|
|
2055
|
+
"king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica",
|
|
2056
|
+
"American lobster, Northern lobster, Maine lobster, Homarus americanus",
|
|
2057
|
+
"spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish",
|
|
2058
|
+
"crayfish, crawfish, crawdad, crawdaddy",
|
|
2059
|
+
"hermit crab",
|
|
2060
|
+
"isopod",
|
|
2061
|
+
"white stork, Ciconia ciconia",
|
|
2062
|
+
"black stork, Ciconia nigra",
|
|
2063
|
+
"spoonbill",
|
|
2064
|
+
"flamingo",
|
|
2065
|
+
"little blue heron, Egretta caerulea",
|
|
2066
|
+
"American egret, great white heron, Egretta albus",
|
|
2067
|
+
"bittern",
|
|
2068
|
+
"crane",
|
|
2069
|
+
"limpkin, Aramus pictus",
|
|
2070
|
+
"European gallinule, Porphyrio porphyrio",
|
|
2071
|
+
"American coot, marsh hen, mud hen, water hen, Fulica americana",
|
|
2072
|
+
"bustard",
|
|
2073
|
+
"ruddy turnstone, Arenaria interpres",
|
|
2074
|
+
"red-backed sandpiper, dunlin, Erolia alpina",
|
|
2075
|
+
"redshank, Tringa totanus",
|
|
2076
|
+
"dowitcher",
|
|
2077
|
+
"oystercatcher, oyster catcher",
|
|
2078
|
+
"pelican",
|
|
2079
|
+
"king penguin, Aptenodytes patagonica",
|
|
2080
|
+
"albatross, mollymawk",
|
|
2081
|
+
"grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus",
|
|
2082
|
+
"killer whale, killer, orca, grampus, sea wolf, Orcinus orca",
|
|
2083
|
+
"dugong, Dugong dugon",
|
|
2084
|
+
"sea lion",
|
|
2085
|
+
"Chihuahua",
|
|
2086
|
+
"Japanese spaniel",
|
|
2087
|
+
"Maltese dog, Maltese terrier, Maltese",
|
|
2088
|
+
"Pekinese, Pekingese, Peke",
|
|
2089
|
+
"Shih-Tzu",
|
|
2090
|
+
"Blenheim spaniel",
|
|
2091
|
+
"papillon",
|
|
2092
|
+
"toy terrier",
|
|
2093
|
+
"Rhodesian ridgeback",
|
|
2094
|
+
"Afghan hound, Afghan",
|
|
2095
|
+
"basset, basset hound",
|
|
2096
|
+
"beagle",
|
|
2097
|
+
"bloodhound, sleuthhound",
|
|
2098
|
+
"bluetick",
|
|
2099
|
+
"black-and-tan coonhound",
|
|
2100
|
+
"Walker hound, Walker foxhound",
|
|
2101
|
+
"English foxhound",
|
|
2102
|
+
"redbone",
|
|
2103
|
+
"borzoi, Russian wolfhound",
|
|
2104
|
+
"Irish wolfhound",
|
|
2105
|
+
"Italian greyhound",
|
|
2106
|
+
"whippet",
|
|
2107
|
+
"Ibizan hound, Ibizan Podenco",
|
|
2108
|
+
"Norwegian elkhound, elkhound",
|
|
2109
|
+
"otterhound, otter hound",
|
|
2110
|
+
"Saluki, gazelle hound",
|
|
2111
|
+
"Scottish deerhound, deerhound",
|
|
2112
|
+
"Weimaraner",
|
|
2113
|
+
"Staffordshire bullterrier, Staffordshire bull terrier",
|
|
2114
|
+
"American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier",
|
|
2115
|
+
"Bedlington terrier",
|
|
2116
|
+
"Border terrier",
|
|
2117
|
+
"Kerry blue terrier",
|
|
2118
|
+
"Irish terrier",
|
|
2119
|
+
"Norfolk terrier",
|
|
2120
|
+
"Norwich terrier",
|
|
2121
|
+
"Yorkshire terrier",
|
|
2122
|
+
"wire-haired fox terrier",
|
|
2123
|
+
"Lakeland terrier",
|
|
2124
|
+
"Sealyham terrier, Sealyham",
|
|
2125
|
+
"Airedale, Airedale terrier",
|
|
2126
|
+
"cairn, cairn terrier",
|
|
2127
|
+
"Australian terrier",
|
|
2128
|
+
"Dandie Dinmont, Dandie Dinmont terrier",
|
|
2129
|
+
"Boston bull, Boston terrier",
|
|
2130
|
+
"miniature schnauzer",
|
|
2131
|
+
"giant schnauzer",
|
|
2132
|
+
"standard schnauzer",
|
|
2133
|
+
"Scotch terrier, Scottish terrier, Scottie",
|
|
2134
|
+
"Tibetan terrier, chrysanthemum dog",
|
|
2135
|
+
"silky terrier, Sydney silky",
|
|
2136
|
+
"soft-coated wheaten terrier",
|
|
2137
|
+
"West Highland white terrier",
|
|
2138
|
+
"Lhasa, Lhasa apso",
|
|
2139
|
+
"flat-coated retriever",
|
|
2140
|
+
"curly-coated retriever",
|
|
2141
|
+
"golden retriever",
|
|
2142
|
+
"Labrador retriever",
|
|
2143
|
+
"Chesapeake Bay retriever",
|
|
2144
|
+
"German short-haired pointer",
|
|
2145
|
+
"vizsla, Hungarian pointer",
|
|
2146
|
+
"English setter",
|
|
2147
|
+
"Irish setter, red setter",
|
|
2148
|
+
"Gordon setter",
|
|
2149
|
+
"Brittany spaniel",
|
|
2150
|
+
"clumber, clumber spaniel",
|
|
2151
|
+
"English springer, English springer spaniel",
|
|
2152
|
+
"Welsh springer spaniel",
|
|
2153
|
+
"cocker spaniel, English cocker spaniel, cocker",
|
|
2154
|
+
"Sussex spaniel",
|
|
2155
|
+
"Irish water spaniel",
|
|
2156
|
+
"kuvasz",
|
|
2157
|
+
"schipperke",
|
|
2158
|
+
"groenendael",
|
|
2159
|
+
"malinois",
|
|
2160
|
+
"briard",
|
|
2161
|
+
"kelpie",
|
|
2162
|
+
"komondor",
|
|
2163
|
+
"Old English sheepdog, bobtail",
|
|
2164
|
+
"Shetland sheepdog, Shetland sheep dog, Shetland",
|
|
2165
|
+
"collie",
|
|
2166
|
+
"Border collie",
|
|
2167
|
+
"Bouvier des Flandres, Bouviers des Flandres",
|
|
2168
|
+
"Rottweiler",
|
|
2169
|
+
"German shepherd, German shepherd dog, German police dog, alsatian",
|
|
2170
|
+
"Doberman, Doberman pinscher",
|
|
2171
|
+
"miniature pinscher",
|
|
2172
|
+
"Greater Swiss Mountain dog",
|
|
2173
|
+
"Bernese mountain dog",
|
|
2174
|
+
"Appenzeller",
|
|
2175
|
+
"EntleBucher",
|
|
2176
|
+
"boxer",
|
|
2177
|
+
"bull mastiff",
|
|
2178
|
+
"Tibetan mastiff",
|
|
2179
|
+
"French bulldog",
|
|
2180
|
+
"Great Dane",
|
|
2181
|
+
"Saint Bernard, St Bernard",
|
|
2182
|
+
"Eskimo dog, husky",
|
|
2183
|
+
"malamute, malemute, Alaskan malamute",
|
|
2184
|
+
"Siberian husky",
|
|
2185
|
+
"dalmatian, coach dog, carriage dog",
|
|
2186
|
+
"affenpinscher, monkey pinscher, monkey dog",
|
|
2187
|
+
"basenji",
|
|
2188
|
+
"pug, pug-dog",
|
|
2189
|
+
"Leonberg",
|
|
2190
|
+
"Newfoundland, Newfoundland dog",
|
|
2191
|
+
"Great Pyrenees",
|
|
2192
|
+
"Samoyed, Samoyede",
|
|
2193
|
+
"Pomeranian",
|
|
2194
|
+
"chow, chow chow",
|
|
2195
|
+
"keeshond",
|
|
2196
|
+
"Brabancon griffon",
|
|
2197
|
+
"Pembroke, Pembroke Welsh corgi",
|
|
2198
|
+
"Cardigan, Cardigan Welsh corgi",
|
|
2199
|
+
"toy poodle",
|
|
2200
|
+
"miniature poodle",
|
|
2201
|
+
"standard poodle",
|
|
2202
|
+
"Mexican hairless",
|
|
2203
|
+
"timber wolf, grey wolf, gray wolf, Canis lupus",
|
|
2204
|
+
"white wolf, Arctic wolf, Canis lupus tundrarum",
|
|
2205
|
+
"red wolf, maned wolf, Canis rufus, Canis niger",
|
|
2206
|
+
"coyote, prairie wolf, brush wolf, Canis latrans",
|
|
2207
|
+
"dingo, warrigal, warragal, Canis dingo",
|
|
2208
|
+
"dhole, Cuon alpinus",
|
|
2209
|
+
"African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus",
|
|
2210
|
+
"hyena, hyaena",
|
|
2211
|
+
"red fox, Vulpes vulpes",
|
|
2212
|
+
"kit fox, Vulpes macrotis",
|
|
2213
|
+
"Arctic fox, white fox, Alopex lagopus",
|
|
2214
|
+
"grey fox, gray fox, Urocyon cinereoargenteus",
|
|
2215
|
+
"tabby, tabby cat",
|
|
2216
|
+
"tiger cat",
|
|
2217
|
+
"Persian cat",
|
|
2218
|
+
"Siamese cat, Siamese",
|
|
2219
|
+
"Egyptian cat",
|
|
2220
|
+
"cougar, puma, catamount, mountain lion, painter, panther, Felis concolor",
|
|
2221
|
+
"lynx, catamount",
|
|
2222
|
+
"leopard, Panthera pardus",
|
|
2223
|
+
"snow leopard, ounce, Panthera uncia",
|
|
2224
|
+
"jaguar, panther, Panthera onca, Felis onca",
|
|
2225
|
+
"lion, king of beasts, Panthera leo",
|
|
2226
|
+
"tiger, Panthera tigris",
|
|
2227
|
+
"cheetah, chetah, Acinonyx jubatus",
|
|
2228
|
+
"brown bear, bruin, Ursus arctos",
|
|
2229
|
+
"American black bear, black bear, Ursus americanus, Euarctos americanus",
|
|
2230
|
+
"ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus",
|
|
2231
|
+
"sloth bear, Melursus ursinus, Ursus ursinus",
|
|
2232
|
+
"mongoose",
|
|
2233
|
+
"meerkat, mierkat",
|
|
2234
|
+
"tiger beetle",
|
|
2235
|
+
"ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle",
|
|
2236
|
+
"ground beetle, carabid beetle",
|
|
2237
|
+
"long-horned beetle, longicorn, longicorn beetle",
|
|
2238
|
+
"leaf beetle, chrysomelid",
|
|
2239
|
+
"dung beetle",
|
|
2240
|
+
"rhinoceros beetle",
|
|
2241
|
+
"weevil",
|
|
2242
|
+
"fly",
|
|
2243
|
+
"bee",
|
|
2244
|
+
"ant, emmet, pismire",
|
|
2245
|
+
"grasshopper, hopper",
|
|
2246
|
+
"cricket",
|
|
2247
|
+
"walking stick, walkingstick, stick insect",
|
|
2248
|
+
"cockroach, roach",
|
|
2249
|
+
"mantis, mantid",
|
|
2250
|
+
"cicada, cicala",
|
|
2251
|
+
"leafhopper",
|
|
2252
|
+
"lacewing, lacewing fly",
|
|
2253
|
+
"dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk",
|
|
2254
|
+
"damselfly",
|
|
2255
|
+
"admiral",
|
|
2256
|
+
"ringlet, ringlet butterfly",
|
|
2257
|
+
"monarch, monarch butterfly, milkweed butterfly, Danaus plexippus",
|
|
2258
|
+
"cabbage butterfly",
|
|
2259
|
+
"sulphur butterfly, sulfur butterfly",
|
|
2260
|
+
"lycaenid, lycaenid butterfly",
|
|
2261
|
+
"starfish, sea star",
|
|
2262
|
+
"sea urchin",
|
|
2263
|
+
"sea cucumber, holothurian",
|
|
2264
|
+
"wood rabbit, cottontail, cottontail rabbit",
|
|
2265
|
+
"hare",
|
|
2266
|
+
"Angora, Angora rabbit",
|
|
2267
|
+
"hamster",
|
|
2268
|
+
"porcupine, hedgehog",
|
|
2269
|
+
"fox squirrel, eastern fox squirrel, Sciurus niger",
|
|
2270
|
+
"marmot",
|
|
2271
|
+
"beaver",
|
|
2272
|
+
"guinea pig, Cavia cobaya",
|
|
2273
|
+
"sorrel",
|
|
2274
|
+
"zebra",
|
|
2275
|
+
"hog, pig, grunter, squealer, Sus scrofa",
|
|
2276
|
+
"wild boar, boar, Sus scrofa",
|
|
2277
|
+
"warthog",
|
|
2278
|
+
"hippopotamus, hippo, river horse, Hippopotamus amphibius",
|
|
2279
|
+
"ox",
|
|
2280
|
+
"water buffalo, water ox, Asiatic buffalo, Bubalus bubalis",
|
|
2281
|
+
"bison",
|
|
2282
|
+
"ram, tup",
|
|
2283
|
+
"bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis",
|
|
2284
|
+
"ibex, Capra ibex",
|
|
2285
|
+
"hartebeest",
|
|
2286
|
+
"impala, Aepyceros melampus",
|
|
2287
|
+
"gazelle",
|
|
2288
|
+
"Arabian camel, dromedary, Camelus dromedarius",
|
|
2289
|
+
"llama",
|
|
2290
|
+
"weasel",
|
|
2291
|
+
"mink",
|
|
2292
|
+
"polecat, fitch, foulmart, foumart, Mustela putorius",
|
|
2293
|
+
"black-footed ferret, ferret, Mustela nigripes",
|
|
2294
|
+
"otter",
|
|
2295
|
+
"skunk, polecat, wood pussy",
|
|
2296
|
+
"badger",
|
|
2297
|
+
"armadillo",
|
|
2298
|
+
"three-toed sloth, ai, Bradypus tridactylus",
|
|
2299
|
+
"orangutan, orang, orangutang, Pongo pygmaeus",
|
|
2300
|
+
"gorilla, Gorilla gorilla",
|
|
2301
|
+
"chimpanzee, chimp, Pan troglodytes",
|
|
2302
|
+
"gibbon, Hylobates lar",
|
|
2303
|
+
"siamang, Hylobates syndactylus, Symphalangus syndactylus",
|
|
2304
|
+
"guenon, guenon monkey",
|
|
2305
|
+
"patas, hussar monkey, Erythrocebus patas",
|
|
2306
|
+
"baboon",
|
|
2307
|
+
"macaque",
|
|
2308
|
+
"langur",
|
|
2309
|
+
"colobus, colobus monkey",
|
|
2310
|
+
"proboscis monkey, Nasalis larvatus",
|
|
2311
|
+
"marmoset",
|
|
2312
|
+
"capuchin, ringtail, Cebus capucinus",
|
|
2313
|
+
"howler monkey, howler",
|
|
2314
|
+
"titi, titi monkey",
|
|
2315
|
+
"spider monkey, Ateles geoffroyi",
|
|
2316
|
+
"squirrel monkey, Saimiri sciureus",
|
|
2317
|
+
"Madagascar cat, ring-tailed lemur, Lemur catta",
|
|
2318
|
+
"indri, indris, Indri indri, Indri brevicaudatus",
|
|
2319
|
+
"Indian elephant, Elephas maximus",
|
|
2320
|
+
"African elephant, Loxodonta africana",
|
|
2321
|
+
"lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens",
|
|
2322
|
+
"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca",
|
|
2323
|
+
"barracouta, snoek",
|
|
2324
|
+
"eel",
|
|
2325
|
+
"coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch",
|
|
2326
|
+
"rock beauty, Holocanthus tricolor",
|
|
2327
|
+
"anemone fish",
|
|
2328
|
+
"sturgeon",
|
|
2329
|
+
"gar, garfish, garpike, billfish, Lepisosteus osseus",
|
|
2330
|
+
"lionfish",
|
|
2331
|
+
"puffer, pufferfish, blowfish, globefish",
|
|
2332
|
+
"abacus",
|
|
2333
|
+
"abaya",
|
|
2334
|
+
"academic gown, academic robe, judge's robe",
|
|
2335
|
+
"accordion, piano accordion, squeeze box",
|
|
2336
|
+
"acoustic guitar",
|
|
2337
|
+
"aircraft carrier, carrier, flattop, attack aircraft carrier",
|
|
2338
|
+
"airliner",
|
|
2339
|
+
"airship, dirigible",
|
|
2340
|
+
"altar",
|
|
2341
|
+
"ambulance",
|
|
2342
|
+
"amphibian, amphibious vehicle",
|
|
2343
|
+
"analog clock",
|
|
2344
|
+
"apiary, bee house",
|
|
2345
|
+
"apron",
|
|
2346
|
+
"ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin",
|
|
2347
|
+
"assault rifle, assault gun",
|
|
2348
|
+
"backpack, back pack, knapsack, packsack, rucksack, haversack",
|
|
2349
|
+
"bakery, bakeshop, bakehouse",
|
|
2350
|
+
"balance beam, beam",
|
|
2351
|
+
"balloon",
|
|
2352
|
+
"ballpoint, ballpoint pen, ballpen, Biro",
|
|
2353
|
+
"Band Aid",
|
|
2354
|
+
"banjo",
|
|
2355
|
+
"bannister, banister, balustrade, balusters, handrail",
|
|
2356
|
+
"barbell",
|
|
2357
|
+
"barber chair",
|
|
2358
|
+
"barbershop",
|
|
2359
|
+
"barn",
|
|
2360
|
+
"barometer",
|
|
2361
|
+
"barrel, cask",
|
|
2362
|
+
"barrow, garden cart, lawn cart, wheelbarrow",
|
|
2363
|
+
"baseball",
|
|
2364
|
+
"basketball",
|
|
2365
|
+
"bassinet",
|
|
2366
|
+
"bassoon",
|
|
2367
|
+
"bathing cap, swimming cap",
|
|
2368
|
+
"bath towel",
|
|
2369
|
+
"bathtub, bathing tub, bath, tub",
|
|
2370
|
+
"beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon",
|
|
2371
|
+
"beacon, lighthouse, beacon light, pharos",
|
|
2372
|
+
"beaker",
|
|
2373
|
+
"bearskin, busby, shako",
|
|
2374
|
+
"beer bottle",
|
|
2375
|
+
"beer glass",
|
|
2376
|
+
"bell cote, bell cot",
|
|
2377
|
+
"bib",
|
|
2378
|
+
"bicycle-built-for-two, tandem bicycle, tandem",
|
|
2379
|
+
"bikini, two-piece",
|
|
2380
|
+
"binder, ring-binder",
|
|
2381
|
+
"binoculars, field glasses, opera glasses",
|
|
2382
|
+
"birdhouse",
|
|
2383
|
+
"boathouse",
|
|
2384
|
+
"bobsled, bobsleigh, bob",
|
|
2385
|
+
"bolo tie, bolo, bola tie, bola",
|
|
2386
|
+
"bonnet, poke bonnet",
|
|
2387
|
+
"bookcase",
|
|
2388
|
+
"bookshop, bookstore, bookstall",
|
|
2389
|
+
"bottlecap",
|
|
2390
|
+
"bow",
|
|
2391
|
+
"bow tie, bow-tie, bowtie",
|
|
2392
|
+
"brass, memorial tablet, plaque",
|
|
2393
|
+
"brassiere, bra, bandeau",
|
|
2394
|
+
"breakwater, groin, groyne, mole, bulwark, seawall, jetty",
|
|
2395
|
+
"breastplate, aegis, egis",
|
|
2396
|
+
"broom",
|
|
2397
|
+
"bucket, pail",
|
|
2398
|
+
"buckle",
|
|
2399
|
+
"bulletproof vest",
|
|
2400
|
+
"bullet train, bullet",
|
|
2401
|
+
"butcher shop, meat market",
|
|
2402
|
+
"cab, hack, taxi, taxicab",
|
|
2403
|
+
"caldron, cauldron",
|
|
2404
|
+
"candle, taper, wax light",
|
|
2405
|
+
"cannon",
|
|
2406
|
+
"canoe",
|
|
2407
|
+
"can opener, tin opener",
|
|
2408
|
+
"cardigan",
|
|
2409
|
+
"car mirror",
|
|
2410
|
+
"carousel, carrousel, merry-go-round, roundabout, whirligig",
|
|
2411
|
+
"carpenter's kit, tool kit",
|
|
2412
|
+
"carton",
|
|
2413
|
+
"car wheel",
|
|
2414
|
+
"cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM",
|
|
2415
|
+
"cassette",
|
|
2416
|
+
"cassette player",
|
|
2417
|
+
"castle",
|
|
2418
|
+
"catamaran",
|
|
2419
|
+
"CD player",
|
|
2420
|
+
"cello, violoncello",
|
|
2421
|
+
"cellular telephone, cellular phone, cellphone, cell, mobile phone",
|
|
2422
|
+
"chain",
|
|
2423
|
+
"chainlink fence",
|
|
2424
|
+
"chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour",
|
|
2425
|
+
"chain saw, chainsaw",
|
|
2426
|
+
"chest",
|
|
2427
|
+
"chiffonier, commode",
|
|
2428
|
+
"chime, bell, gong",
|
|
2429
|
+
"china cabinet, china closet",
|
|
2430
|
+
"Christmas stocking",
|
|
2431
|
+
"church, church building",
|
|
2432
|
+
"cinema, movie theater, movie theatre, movie house, picture palace",
|
|
2433
|
+
"cleaver, meat cleaver, chopper",
|
|
2434
|
+
"cliff dwelling",
|
|
2435
|
+
"cloak",
|
|
2436
|
+
"clog, geta, patten, sabot",
|
|
2437
|
+
"cocktail shaker",
|
|
2438
|
+
"coffee mug",
|
|
2439
|
+
"coffeepot",
|
|
2440
|
+
"coil, spiral, volute, whorl, helix",
|
|
2441
|
+
"combination lock",
|
|
2442
|
+
"computer keyboard, keypad",
|
|
2443
|
+
"confectionery, confectionary, candy store",
|
|
2444
|
+
"container ship, containership, container vessel",
|
|
2445
|
+
"convertible",
|
|
2446
|
+
"corkscrew, bottle screw",
|
|
2447
|
+
"cornet, horn, trumpet, trump",
|
|
2448
|
+
"cowboy boot",
|
|
2449
|
+
"cowboy hat, ten-gallon hat",
|
|
2450
|
+
"cradle",
|
|
2451
|
+
"crane",
|
|
2452
|
+
"crash helmet",
|
|
2453
|
+
"crate",
|
|
2454
|
+
"crib, cot",
|
|
2455
|
+
"Crock Pot",
|
|
2456
|
+
"croquet ball",
|
|
2457
|
+
"crutch",
|
|
2458
|
+
"cuirass",
|
|
2459
|
+
"dam, dike, dyke",
|
|
2460
|
+
"desk",
|
|
2461
|
+
"desktop computer",
|
|
2462
|
+
"dial telephone, dial phone",
|
|
2463
|
+
"diaper, nappy, napkin",
|
|
2464
|
+
"digital clock",
|
|
2465
|
+
"digital watch",
|
|
2466
|
+
"dining table, board",
|
|
2467
|
+
"dishrag, dishcloth",
|
|
2468
|
+
"dishwasher, dish washer, dishwashing machine",
|
|
2469
|
+
"disk brake, disc brake",
|
|
2470
|
+
"dock, dockage, docking facility",
|
|
2471
|
+
"dogsled, dog sled, dog sleigh",
|
|
2472
|
+
"dome",
|
|
2473
|
+
"doormat, welcome mat",
|
|
2474
|
+
"drilling platform, offshore rig",
|
|
2475
|
+
"drum, membranophone, tympan",
|
|
2476
|
+
"drumstick",
|
|
2477
|
+
"dumbbell",
|
|
2478
|
+
"Dutch oven",
|
|
2479
|
+
"electric fan, blower",
|
|
2480
|
+
"electric guitar",
|
|
2481
|
+
"electric locomotive",
|
|
2482
|
+
"entertainment center",
|
|
2483
|
+
"envelope",
|
|
2484
|
+
"espresso maker",
|
|
2485
|
+
"face powder",
|
|
2486
|
+
"feather boa, boa",
|
|
2487
|
+
"file, file cabinet, filing cabinet",
|
|
2488
|
+
"fireboat",
|
|
2489
|
+
"fire engine, fire truck",
|
|
2490
|
+
"fire screen, fireguard",
|
|
2491
|
+
"flagpole, flagstaff",
|
|
2492
|
+
"flute, transverse flute",
|
|
2493
|
+
"folding chair",
|
|
2494
|
+
"football helmet",
|
|
2495
|
+
"forklift",
|
|
2496
|
+
"fountain",
|
|
2497
|
+
"fountain pen",
|
|
2498
|
+
"four-poster",
|
|
2499
|
+
"freight car",
|
|
2500
|
+
"French horn, horn",
|
|
2501
|
+
"frying pan, frypan, skillet",
|
|
2502
|
+
"fur coat",
|
|
2503
|
+
"garbage truck, dustcart",
|
|
2504
|
+
"gasmask, respirator, gas helmet",
|
|
2505
|
+
"gas pump, gasoline pump, petrol pump, island dispenser",
|
|
2506
|
+
"goblet",
|
|
2507
|
+
"go-kart",
|
|
2508
|
+
"golf ball",
|
|
2509
|
+
"golfcart, golf cart",
|
|
2510
|
+
"gondola",
|
|
2511
|
+
"gong, tam-tam",
|
|
2512
|
+
"gown",
|
|
2513
|
+
"grand piano, grand",
|
|
2514
|
+
"greenhouse, nursery, glasshouse",
|
|
2515
|
+
"grille, radiator grille",
|
|
2516
|
+
"grocery store, grocery, food market, market",
|
|
2517
|
+
"guillotine",
|
|
2518
|
+
"hair slide",
|
|
2519
|
+
"hair spray",
|
|
2520
|
+
"half track",
|
|
2521
|
+
"hammer",
|
|
2522
|
+
"hamper",
|
|
2523
|
+
"hand blower, blow dryer, blow drier, hair dryer, hair drier",
|
|
2524
|
+
"hand-held computer, hand-held microcomputer",
|
|
2525
|
+
"handkerchief, hankie, hanky, hankey",
|
|
2526
|
+
"hard disc, hard disk, fixed disk",
|
|
2527
|
+
"harmonica, mouth organ, harp, mouth harp",
|
|
2528
|
+
"harp",
|
|
2529
|
+
"harvester, reaper",
|
|
2530
|
+
"hatchet",
|
|
2531
|
+
"holster",
|
|
2532
|
+
"home theater, home theatre",
|
|
2533
|
+
"honeycomb",
|
|
2534
|
+
"hook, claw",
|
|
2535
|
+
"hoopskirt, crinoline",
|
|
2536
|
+
"horizontal bar, high bar",
|
|
2537
|
+
"horse cart, horse-cart",
|
|
2538
|
+
"hourglass",
|
|
2539
|
+
"iPod",
|
|
2540
|
+
"iron, smoothing iron",
|
|
2541
|
+
"jack-o'-lantern",
|
|
2542
|
+
"jean, blue jean, denim",
|
|
2543
|
+
"jeep, landrover",
|
|
2544
|
+
"jersey, T-shirt, tee shirt",
|
|
2545
|
+
"jigsaw puzzle",
|
|
2546
|
+
"jinrikisha, ricksha, rickshaw",
|
|
2547
|
+
"joystick",
|
|
2548
|
+
"kimono",
|
|
2549
|
+
"knee pad",
|
|
2550
|
+
"knot",
|
|
2551
|
+
"lab coat, laboratory coat",
|
|
2552
|
+
"ladle",
|
|
2553
|
+
"lampshade, lamp shade",
|
|
2554
|
+
"laptop, laptop computer",
|
|
2555
|
+
"lawn mower, mower",
|
|
2556
|
+
"lens cap, lens cover",
|
|
2557
|
+
"letter opener, paper knife, paperknife",
|
|
2558
|
+
"library",
|
|
2559
|
+
"lifeboat",
|
|
2560
|
+
"lighter, light, igniter, ignitor",
|
|
2561
|
+
"limousine, limo",
|
|
2562
|
+
"liner, ocean liner",
|
|
2563
|
+
"lipstick, lip rouge",
|
|
2564
|
+
"Loafer",
|
|
2565
|
+
"lotion",
|
|
2566
|
+
"loudspeaker, speaker, speaker unit, loudspeaker system, speaker system",
|
|
2567
|
+
"loupe, jeweler's loupe",
|
|
2568
|
+
"lumbermill, sawmill",
|
|
2569
|
+
"magnetic compass",
|
|
2570
|
+
"mailbag, postbag",
|
|
2571
|
+
"mailbox, letter box",
|
|
2572
|
+
"maillot",
|
|
2573
|
+
"maillot, tank suit",
|
|
2574
|
+
"manhole cover",
|
|
2575
|
+
"maraca",
|
|
2576
|
+
"marimba, xylophone",
|
|
2577
|
+
"mask",
|
|
2578
|
+
"matchstick",
|
|
2579
|
+
"maypole",
|
|
2580
|
+
"maze, labyrinth",
|
|
2581
|
+
"measuring cup",
|
|
2582
|
+
"medicine chest, medicine cabinet",
|
|
2583
|
+
"megalith, megalithic structure",
|
|
2584
|
+
"microphone, mike",
|
|
2585
|
+
"microwave, microwave oven",
|
|
2586
|
+
"military uniform",
|
|
2587
|
+
"milk can",
|
|
2588
|
+
"minibus",
|
|
2589
|
+
"miniskirt, mini",
|
|
2590
|
+
"minivan",
|
|
2591
|
+
"missile",
|
|
2592
|
+
"mitten",
|
|
2593
|
+
"mixing bowl",
|
|
2594
|
+
"mobile home, manufactured home",
|
|
2595
|
+
"Model T",
|
|
2596
|
+
"modem",
|
|
2597
|
+
"monastery",
|
|
2598
|
+
"monitor",
|
|
2599
|
+
"moped",
|
|
2600
|
+
"mortar",
|
|
2601
|
+
"mortarboard",
|
|
2602
|
+
"mosque",
|
|
2603
|
+
"mosquito net",
|
|
2604
|
+
"motor scooter, scooter",
|
|
2605
|
+
"mountain bike, all-terrain bike, off-roader",
|
|
2606
|
+
"mountain tent",
|
|
2607
|
+
"mouse, computer mouse",
|
|
2608
|
+
"mousetrap",
|
|
2609
|
+
"moving van",
|
|
2610
|
+
"muzzle",
|
|
2611
|
+
"nail",
|
|
2612
|
+
"neck brace",
|
|
2613
|
+
"necklace",
|
|
2614
|
+
"nipple",
|
|
2615
|
+
"notebook, notebook computer",
|
|
2616
|
+
"obelisk",
|
|
2617
|
+
"oboe, hautboy, hautbois",
|
|
2618
|
+
"ocarina, sweet potato",
|
|
2619
|
+
"odometer, hodometer, mileometer, milometer",
|
|
2620
|
+
"oil filter",
|
|
2621
|
+
"organ, pipe organ",
|
|
2622
|
+
"oscilloscope, scope, cathode-ray oscilloscope, CRO",
|
|
2623
|
+
"overskirt",
|
|
2624
|
+
"oxcart",
|
|
2625
|
+
"oxygen mask",
|
|
2626
|
+
"packet",
|
|
2627
|
+
"paddle, boat paddle",
|
|
2628
|
+
"paddlewheel, paddle wheel",
|
|
2629
|
+
"padlock",
|
|
2630
|
+
"paintbrush",
|
|
2631
|
+
"pajama, pyjama, pj's, jammies",
|
|
2632
|
+
"palace",
|
|
2633
|
+
"panpipe, pandean pipe, syrinx",
|
|
2634
|
+
"paper towel",
|
|
2635
|
+
"parachute, chute",
|
|
2636
|
+
"parallel bars, bars",
|
|
2637
|
+
"park bench",
|
|
2638
|
+
"parking meter",
|
|
2639
|
+
"passenger car, coach, carriage",
|
|
2640
|
+
"patio, terrace",
|
|
2641
|
+
"pay-phone, pay-station",
|
|
2642
|
+
"pedestal, plinth, footstall",
|
|
2643
|
+
"pencil box, pencil case",
|
|
2644
|
+
"pencil sharpener",
|
|
2645
|
+
"perfume, essence",
|
|
2646
|
+
"Petri dish",
|
|
2647
|
+
"photocopier",
|
|
2648
|
+
"pick, plectrum, plectron",
|
|
2649
|
+
"pickelhaube",
|
|
2650
|
+
"picket fence, paling",
|
|
2651
|
+
"pickup, pickup truck",
|
|
2652
|
+
"pier",
|
|
2653
|
+
"piggy bank, penny bank",
|
|
2654
|
+
"pill bottle",
|
|
2655
|
+
"pillow",
|
|
2656
|
+
"ping-pong ball",
|
|
2657
|
+
"pinwheel",
|
|
2658
|
+
"pirate, pirate ship",
|
|
2659
|
+
"pitcher, ewer",
|
|
2660
|
+
"plane, carpenter's plane, woodworking plane",
|
|
2661
|
+
"planetarium",
|
|
2662
|
+
"plastic bag",
|
|
2663
|
+
"plate rack",
|
|
2664
|
+
"plow, plough",
|
|
2665
|
+
"plunger, plumber's helper",
|
|
2666
|
+
"Polaroid camera, Polaroid Land camera",
|
|
2667
|
+
"pole",
|
|
2668
|
+
"police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria",
|
|
2669
|
+
"poncho",
|
|
2670
|
+
"pool table, billiard table, snooker table",
|
|
2671
|
+
"pop bottle, soda bottle",
|
|
2672
|
+
"pot, flowerpot",
|
|
2673
|
+
"potter's wheel",
|
|
2674
|
+
"power drill",
|
|
2675
|
+
"prayer rug, prayer mat",
|
|
2676
|
+
"printer",
|
|
2677
|
+
"prison, prison house",
|
|
2678
|
+
"projectile, missile",
|
|
2679
|
+
"projector",
|
|
2680
|
+
"puck, hockey puck",
|
|
2681
|
+
"punching bag, punch bag, punching ball, punchball",
|
|
2682
|
+
"purse",
|
|
2683
|
+
"quill, quill pen",
|
|
2684
|
+
"quilt, comforter, comfort, puff",
|
|
2685
|
+
"racer, race car, racing car",
|
|
2686
|
+
"racket, racquet",
|
|
2687
|
+
"radiator",
|
|
2688
|
+
"radio, wireless",
|
|
2689
|
+
"radio telescope, radio reflector",
|
|
2690
|
+
"rain barrel",
|
|
2691
|
+
"recreational vehicle, RV, R.V.",
|
|
2692
|
+
"reel",
|
|
2693
|
+
"reflex camera",
|
|
2694
|
+
"refrigerator, icebox",
|
|
2695
|
+
"remote control, remote",
|
|
2696
|
+
"restaurant, eating house, eating place, eatery",
|
|
2697
|
+
"revolver, six-gun, six-shooter",
|
|
2698
|
+
"rifle",
|
|
2699
|
+
"rocking chair, rocker",
|
|
2700
|
+
"rotisserie",
|
|
2701
|
+
"rubber eraser, rubber, pencil eraser",
|
|
2702
|
+
"rugby ball",
|
|
2703
|
+
"rule, ruler",
|
|
2704
|
+
"running shoe",
|
|
2705
|
+
"safe",
|
|
2706
|
+
"safety pin",
|
|
2707
|
+
"saltshaker, salt shaker",
|
|
2708
|
+
"sandal",
|
|
2709
|
+
"sarong",
|
|
2710
|
+
"sax, saxophone",
|
|
2711
|
+
"scabbard",
|
|
2712
|
+
"scale, weighing machine",
|
|
2713
|
+
"school bus",
|
|
2714
|
+
"schooner",
|
|
2715
|
+
"scoreboard",
|
|
2716
|
+
"screen, CRT screen",
|
|
2717
|
+
"screw",
|
|
2718
|
+
"screwdriver",
|
|
2719
|
+
"seat belt, seatbelt",
|
|
2720
|
+
"sewing machine",
|
|
2721
|
+
"shield, buckler",
|
|
2722
|
+
"shoe shop, shoe-shop, shoe store",
|
|
2723
|
+
"shoji",
|
|
2724
|
+
"shopping basket",
|
|
2725
|
+
"shopping cart",
|
|
2726
|
+
"shovel",
|
|
2727
|
+
"shower cap",
|
|
2728
|
+
"shower curtain",
|
|
2729
|
+
"ski",
|
|
2730
|
+
"ski mask",
|
|
2731
|
+
"sleeping bag",
|
|
2732
|
+
"slide rule, slipstick",
|
|
2733
|
+
"sliding door",
|
|
2734
|
+
"slot, one-armed bandit",
|
|
2735
|
+
"snorkel",
|
|
2736
|
+
"snowmobile",
|
|
2737
|
+
"snowplow, snowplough",
|
|
2738
|
+
"soap dispenser",
|
|
2739
|
+
"soccer ball",
|
|
2740
|
+
"sock",
|
|
2741
|
+
"solar dish, solar collector, solar furnace",
|
|
2742
|
+
"sombrero",
|
|
2743
|
+
"soup bowl",
|
|
2744
|
+
"space bar",
|
|
2745
|
+
"space heater",
|
|
2746
|
+
"space shuttle",
|
|
2747
|
+
"spatula",
|
|
2748
|
+
"speedboat",
|
|
2749
|
+
"spider web, spider's web",
|
|
2750
|
+
"spindle",
|
|
2751
|
+
"sports car, sport car",
|
|
2752
|
+
"spotlight, spot",
|
|
2753
|
+
"stage",
|
|
2754
|
+
"steam locomotive",
|
|
2755
|
+
"steel arch bridge",
|
|
2756
|
+
"steel drum",
|
|
2757
|
+
"stethoscope",
|
|
2758
|
+
"stole",
|
|
2759
|
+
"stone wall",
|
|
2760
|
+
"stopwatch, stop watch",
|
|
2761
|
+
"stove",
|
|
2762
|
+
"strainer",
|
|
2763
|
+
"streetcar, tram, tramcar, trolley, trolley car",
|
|
2764
|
+
"stretcher",
|
|
2765
|
+
"studio couch, day bed",
|
|
2766
|
+
"stupa, tope",
|
|
2767
|
+
"submarine, pigboat, sub, U-boat",
|
|
2768
|
+
"suit, suit of clothes",
|
|
2769
|
+
"sundial",
|
|
2770
|
+
"sunglass",
|
|
2771
|
+
"sunglasses, dark glasses, shades",
|
|
2772
|
+
"sunscreen, sunblock, sun blocker",
|
|
2773
|
+
"suspension bridge",
|
|
2774
|
+
"swab, swob, mop",
|
|
2775
|
+
"sweatshirt",
|
|
2776
|
+
"swimming trunks, bathing trunks",
|
|
2777
|
+
"swing",
|
|
2778
|
+
"switch, electric switch, electrical switch",
|
|
2779
|
+
"syringe",
|
|
2780
|
+
"table lamp",
|
|
2781
|
+
"tank, army tank, armored combat vehicle, armoured combat vehicle",
|
|
2782
|
+
"tape player",
|
|
2783
|
+
"teapot",
|
|
2784
|
+
"teddy, teddy bear",
|
|
2785
|
+
"television, television system",
|
|
2786
|
+
"tennis ball",
|
|
2787
|
+
"thatch, thatched roof",
|
|
2788
|
+
"theater curtain, theatre curtain",
|
|
2789
|
+
"thimble",
|
|
2790
|
+
"thresher, thrasher, threshing machine",
|
|
2791
|
+
"throne",
|
|
2792
|
+
"tile roof",
|
|
2793
|
+
"toaster",
|
|
2794
|
+
"tobacco shop, tobacconist shop, tobacconist",
|
|
2795
|
+
"toilet seat",
|
|
2796
|
+
"torch",
|
|
2797
|
+
"totem pole",
|
|
2798
|
+
"tow truck, tow car, wrecker",
|
|
2799
|
+
"toyshop",
|
|
2800
|
+
"tractor",
|
|
2801
|
+
"trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi",
|
|
2802
|
+
"tray",
|
|
2803
|
+
"trench coat",
|
|
2804
|
+
"tricycle, trike, velocipede",
|
|
2805
|
+
"trimaran",
|
|
2806
|
+
"tripod",
|
|
2807
|
+
"triumphal arch",
|
|
2808
|
+
"trolleybus, trolley coach, trackless trolley",
|
|
2809
|
+
"trombone",
|
|
2810
|
+
"tub, vat",
|
|
2811
|
+
"turnstile",
|
|
2812
|
+
"typewriter keyboard",
|
|
2813
|
+
"umbrella",
|
|
2814
|
+
"unicycle, monocycle",
|
|
2815
|
+
"upright, upright piano",
|
|
2816
|
+
"vacuum, vacuum cleaner",
|
|
2817
|
+
"vase",
|
|
2818
|
+
"vault",
|
|
2819
|
+
"velvet",
|
|
2820
|
+
"vending machine",
|
|
2821
|
+
"vestment",
|
|
2822
|
+
"viaduct",
|
|
2823
|
+
"violin, fiddle",
|
|
2824
|
+
"volleyball",
|
|
2825
|
+
"waffle iron",
|
|
2826
|
+
"wall clock",
|
|
2827
|
+
"wallet, billfold, notecase, pocketbook",
|
|
2828
|
+
"wardrobe, closet, press",
|
|
2829
|
+
"warplane, military plane",
|
|
2830
|
+
"washbasin, handbasin, washbowl, lavabo, wash-hand basin",
|
|
2831
|
+
"washer, automatic washer, washing machine",
|
|
2832
|
+
"water bottle",
|
|
2833
|
+
"water jug",
|
|
2834
|
+
"water tower",
|
|
2835
|
+
"whiskey jug",
|
|
2836
|
+
"whistle",
|
|
2837
|
+
"wig",
|
|
2838
|
+
"window screen",
|
|
2839
|
+
"window shade",
|
|
2840
|
+
"Windsor tie",
|
|
2841
|
+
"wine bottle",
|
|
2842
|
+
"wing",
|
|
2843
|
+
"wok",
|
|
2844
|
+
"wooden spoon",
|
|
2845
|
+
"wool, woolen, woollen",
|
|
2846
|
+
"worm fence, snake fence, snake-rail fence, Virginia fence",
|
|
2847
|
+
"wreck",
|
|
2848
|
+
"yawl",
|
|
2849
|
+
"yurt",
|
|
2850
|
+
"web site, website, internet site, site",
|
|
2851
|
+
"comic book",
|
|
2852
|
+
"crossword puzzle, crossword",
|
|
2853
|
+
"street sign",
|
|
2854
|
+
"traffic light, traffic signal, stoplight",
|
|
2855
|
+
"book jacket, dust cover, dust jacket, dust wrapper",
|
|
2856
|
+
"menu",
|
|
2857
|
+
"plate",
|
|
2858
|
+
"guacamole",
|
|
2859
|
+
"consomme",
|
|
2860
|
+
"hot pot, hotpot",
|
|
2861
|
+
"trifle",
|
|
2862
|
+
"ice cream, icecream",
|
|
2863
|
+
"ice lolly, lolly, lollipop, popsicle",
|
|
2864
|
+
"French loaf",
|
|
2865
|
+
"bagel, beigel",
|
|
2866
|
+
"pretzel",
|
|
2867
|
+
"cheeseburger",
|
|
2868
|
+
"hotdog, hot dog, red hot",
|
|
2869
|
+
"mashed potato",
|
|
2870
|
+
"head cabbage",
|
|
2871
|
+
"broccoli",
|
|
2872
|
+
"cauliflower",
|
|
2873
|
+
"zucchini, courgette",
|
|
2874
|
+
"spaghetti squash",
|
|
2875
|
+
"acorn squash",
|
|
2876
|
+
"butternut squash",
|
|
2877
|
+
"cucumber, cuke",
|
|
2878
|
+
"artichoke, globe artichoke",
|
|
2879
|
+
"bell pepper",
|
|
2880
|
+
"cardoon",
|
|
2881
|
+
"mushroom",
|
|
2882
|
+
"Granny Smith",
|
|
2883
|
+
"strawberry",
|
|
2884
|
+
"orange",
|
|
2885
|
+
"lemon",
|
|
2886
|
+
"fig",
|
|
2887
|
+
"pineapple, ananas",
|
|
2888
|
+
"banana",
|
|
2889
|
+
"jackfruit, jak, jack",
|
|
2890
|
+
"custard apple",
|
|
2891
|
+
"pomegranate",
|
|
2892
|
+
"hay",
|
|
2893
|
+
"carbonara",
|
|
2894
|
+
"chocolate sauce, chocolate syrup",
|
|
2895
|
+
"dough",
|
|
2896
|
+
"meat loaf, meatloaf",
|
|
2897
|
+
"pizza, pizza pie",
|
|
2898
|
+
"potpie",
|
|
2899
|
+
"burrito",
|
|
2900
|
+
"red wine",
|
|
2901
|
+
"espresso",
|
|
2902
|
+
"cup",
|
|
2903
|
+
"eggnog",
|
|
2904
|
+
"alp",
|
|
2905
|
+
"bubble",
|
|
2906
|
+
"cliff, drop, drop-off",
|
|
2907
|
+
"coral reef",
|
|
2908
|
+
"geyser",
|
|
2909
|
+
"lakeside, lakeshore",
|
|
2910
|
+
"promontory, headland, head, foreland",
|
|
2911
|
+
"sandbar, sand bar",
|
|
2912
|
+
"seashore, coast, seacoast, sea-coast",
|
|
2913
|
+
"valley, vale",
|
|
2914
|
+
"volcano",
|
|
2915
|
+
"ballplayer, baseball player",
|
|
2916
|
+
"groom, bridegroom",
|
|
2917
|
+
"scuba diver",
|
|
2918
|
+
"rapeseed",
|
|
2919
|
+
"daisy",
|
|
2920
|
+
"yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum",
|
|
2921
|
+
"corn",
|
|
2922
|
+
"acorn",
|
|
2923
|
+
"hip, rose hip, rosehip",
|
|
2924
|
+
"buckeye, horse chestnut, conker",
|
|
2925
|
+
"coral fungus",
|
|
2926
|
+
"agaric",
|
|
2927
|
+
"gyromitra",
|
|
2928
|
+
"stinkhorn, carrion fungus",
|
|
2929
|
+
"earthstar",
|
|
2930
|
+
"hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa",
|
|
2931
|
+
"bolete",
|
|
2932
|
+
"ear, spike, capitulum",
|
|
2933
|
+
"toilet tissue, toilet paper, bathroom tissue"
|
|
2934
|
+
];
|
|
2935
|
+
//#endregion
|
|
2936
|
+
//#region src/zoo/mobilenetv4/pipeline.ts
|
|
2937
|
+
/** Logits [numClasses] → top-k class probabilities (softmax over ALL classes). */
|
|
2938
|
+
function decodeTopK(logits, k = 5) {
|
|
2939
|
+
let max = -Infinity;
|
|
2940
|
+
for (const v of logits) max = Math.max(max, v);
|
|
2941
|
+
let sum = 0;
|
|
2942
|
+
const exps = new Float64Array(logits.length);
|
|
2943
|
+
for (let i = 0; i < logits.length; i++) {
|
|
2944
|
+
exps[i] = Math.exp(logits[i] - max);
|
|
2945
|
+
sum += exps[i];
|
|
2946
|
+
}
|
|
2947
|
+
return Array.from(exps.keys()).sort((a, b) => exps[b] - exps[a]).slice(0, k).map((classId) => ({
|
|
2948
|
+
classId,
|
|
2949
|
+
confidence: exps[classId] / sum
|
|
2950
|
+
}));
|
|
2951
|
+
}
|
|
2952
|
+
//#endregion
|
|
2953
|
+
//#region src/zoo/tasks/cv/imageClassification.ts
|
|
2954
|
+
/** Image classification task: an image in, the classes it shows out, best
|
|
2955
|
+
* first. Runs MobileNetV4 on the initRunntime() device. */
|
|
2956
|
+
/** Loads the checkpoint, warms the GPU path once, and returns the runner.
|
|
2957
|
+
* Needs a device with `shader-f16`. */
|
|
2958
|
+
async function createImageClassifier(config = {}, opts = {}) {
|
|
2959
|
+
if (!supportsF16()) throw new Error("createImageClassifier: this device has no shader-f16, which the conv kernels need. Request it with tgpu.init({ device: { optionalFeatures: ['shader-f16'] } })");
|
|
2960
|
+
const scope = createResourceScope();
|
|
2961
|
+
try {
|
|
2962
|
+
const { modelPath = models.imageClassification.MOBILENETV4.DEFAULT.modelPath, labels = IMAGENET1K_LABELS, resizeMode = "crop" } = config;
|
|
2963
|
+
const path = typeof modelPath === "string" ? modelPath.replace(/^https?:\/\//, "") : void 0;
|
|
2964
|
+
const sd = await openWeights(modelPath, {
|
|
2965
|
+
...opts,
|
|
2966
|
+
cacheId: opts.cacheId ?? (path && `mobilenetv4-v1/${path}`)
|
|
2967
|
+
});
|
|
2968
|
+
throwIfAborted(opts.signal);
|
|
2969
|
+
const classifier = scope.track(await createClassifier(sd, { onProgress: opts.onProgress }));
|
|
2970
|
+
const { inputSize, numClasses } = classifier;
|
|
2971
|
+
if (labels.length !== numClasses) throw new Error(`labels: ${labels.length} names for a ${numClasses}-class model`);
|
|
2972
|
+
const preprocessor = createImagePreprocessor({
|
|
2973
|
+
resizeMode,
|
|
2974
|
+
cropFraction: MOBILENETV4_CROP_FRACTION,
|
|
2975
|
+
normalizeOpts: {
|
|
2976
|
+
alpha: IMAGENET_STD.map((s) => 1 / (255 * s)),
|
|
2977
|
+
beta: IMAGENET_MEAN.map((m, c) => -m / IMAGENET_STD[c])
|
|
2978
|
+
}
|
|
2979
|
+
}, {
|
|
2980
|
+
width: inputSize,
|
|
2981
|
+
height: inputSize
|
|
2982
|
+
});
|
|
2983
|
+
const pixels = new Float32Array(3 * inputSize * inputSize);
|
|
2984
|
+
let disposed = false;
|
|
2985
|
+
let queue = Promise.resolve();
|
|
2986
|
+
return {
|
|
2987
|
+
labels,
|
|
2988
|
+
inputSize,
|
|
2989
|
+
async classify(image, options = {}) {
|
|
2990
|
+
if (disposed) throw new Error("image classifier is disposed");
|
|
2991
|
+
const { topk = numClasses } = options;
|
|
2992
|
+
if (!Number.isInteger(topk) || topk < 1) throw new Error(`classify: topk must be a positive integer, got ${topk}`);
|
|
2993
|
+
const run = queue.then(() => classifier.run(preprocessor.process(image, pixels)));
|
|
2994
|
+
queue = run.catch(() => void 0);
|
|
2995
|
+
return decodeTopK(await run, Math.min(topk, numClasses)).map(({ classId, confidence }) => ({
|
|
2996
|
+
label: labels[classId],
|
|
2997
|
+
classId,
|
|
2998
|
+
confidence
|
|
2999
|
+
}));
|
|
3000
|
+
},
|
|
3001
|
+
dispose() {
|
|
3002
|
+
disposed = true;
|
|
3003
|
+
scope.dispose();
|
|
3004
|
+
}
|
|
3005
|
+
};
|
|
3006
|
+
} catch (err) {
|
|
3007
|
+
scope.dispose();
|
|
3008
|
+
throw err;
|
|
3009
|
+
}
|
|
3010
|
+
}
|
|
3011
|
+
//#endregion
|
|
3012
|
+
export { COCO_LANDMARKS, SPEECH_SAMPLE_RATE, bufferSource, createDepthEstimator, createImageClassifier, createInstanceSegmenter, createKeypointDetector, createObjectDetector, createOpfsCache, createPrivacyFilter, createSpeechToText, createTextEmbedder, decodeAudio, imageBuffer, imageBufferFromImageData, initRunntime, models, resampleAudio, similarity };
|