tempest-react-sdk 0.69.0 → 0.70.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/vision/core/canvas.cjs +1 -1
- package/dist/vision/core/canvas.cjs.map +1 -1
- package/dist/vision/core/canvas.js +37 -13
- package/dist/vision/core/canvas.js.map +1 -1
- package/dist/vision/core/dtypes.cjs +2 -0
- package/dist/vision/core/dtypes.cjs.map +1 -0
- package/dist/vision/core/dtypes.js +33 -0
- package/dist/vision/core/dtypes.js.map +1 -0
- package/dist/vision/core/graph.cjs +1 -1
- package/dist/vision/core/graph.cjs.map +1 -1
- package/dist/vision/core/graph.js +10 -2
- package/dist/vision/core/graph.js.map +1 -1
- package/dist/vision/core/metadata.cjs +2 -2
- package/dist/vision/core/metadata.cjs.map +1 -1
- package/dist/vision/core/metadata.js +103 -13
- package/dist/vision/core/metadata.js.map +1 -1
- package/dist/vision/core/providers.cjs +1 -1
- package/dist/vision/core/providers.cjs.map +1 -1
- package/dist/vision/core/providers.js +9 -6
- package/dist/vision/core/providers.js.map +1 -1
- package/dist/vision/core/session.cjs +1 -1
- package/dist/vision/core/session.cjs.map +1 -1
- package/dist/vision/core/session.js +115 -27
- package/dist/vision/core/session.js.map +1 -1
- package/dist/vision/fusion.cjs +1 -1
- package/dist/vision/fusion.cjs.map +1 -1
- package/dist/vision/fusion.js +28 -23
- package/dist/vision/fusion.js.map +1 -1
- package/dist/vision/index.cjs +1 -1
- package/dist/vision/index.cjs.map +1 -1
- package/dist/vision/index.js +20 -19
- package/dist/vision/index.js.map +1 -1
- package/dist/vision/io/image.cjs +1 -1
- package/dist/vision/io/image.cjs.map +1 -1
- package/dist/vision/io/image.js +55 -20
- package/dist/vision/io/image.js.map +1 -1
- package/dist/vision/postprocess/detection.cjs +1 -1
- package/dist/vision/postprocess/detection.cjs.map +1 -1
- package/dist/vision/postprocess/detection.js +42 -51
- package/dist/vision/postprocess/detection.js.map +1 -1
- package/dist/vision/postprocess/segmentation.cjs +1 -1
- package/dist/vision/postprocess/segmentation.cjs.map +1 -1
- package/dist/vision/postprocess/segmentation.js +45 -31
- package/dist/vision/postprocess/segmentation.js.map +1 -1
- package/dist/vision/preprocess/pipeline.cjs +1 -1
- package/dist/vision/preprocess/pipeline.cjs.map +1 -1
- package/dist/vision/preprocess/pipeline.js +24 -25
- package/dist/vision/preprocess/pipeline.js.map +1 -1
- package/dist/vision/tasks/base.cjs +1 -1
- package/dist/vision/tasks/base.cjs.map +1 -1
- package/dist/vision/tasks/base.js +22 -7
- package/dist/vision/tasks/base.js.map +1 -1
- package/dist/vision/tasks/classifier.cjs +1 -1
- package/dist/vision/tasks/classifier.cjs.map +1 -1
- package/dist/vision/tasks/classifier.js +64 -55
- package/dist/vision/tasks/classifier.js.map +1 -1
- package/dist/vision/tasks/detectClassify.cjs +1 -1
- package/dist/vision/tasks/detectClassify.cjs.map +1 -1
- package/dist/vision/tasks/detectClassify.js +80 -69
- package/dist/vision/tasks/detectClassify.js.map +1 -1
- package/dist/vision/tasks/detector.cjs +1 -1
- package/dist/vision/tasks/detector.cjs.map +1 -1
- package/dist/vision/tasks/detector.js +74 -68
- package/dist/vision/tasks/detector.js.map +1 -1
- package/dist/vision/tasks/segmenter.cjs +1 -1
- package/dist/vision/tasks/segmenter.cjs.map +1 -1
- package/dist/vision/tasks/segmenter.js +85 -62
- package/dist/vision/tasks/segmenter.js.map +1 -1
- package/dist/vision/types.cjs +1 -1
- package/dist/vision/types.cjs.map +1 -1
- package/dist/vision/types.js +24 -4
- package/dist/vision/types.js.map +1 -1
- package/dist/vision.cjs +1 -1
- package/dist/vision.d.ts +1153 -777
- package/dist/vision.js +22 -21
- package/package.json +1 -1
package/dist/vision.d.ts
CHANGED
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
import type * as ort from 'onnxruntime-web';
|
|
2
2
|
import { RefObject } from 'react';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* Widen a model output to `Float32Array`, without copying when it already is.
|
|
6
|
+
*
|
|
7
|
+
* @param data A tensor's `data`, as ORT returned it.
|
|
8
|
+
* @returns The same values as a `Float32Array`. A float32 output is passed
|
|
9
|
+
* through untouched; anything else is copied once.
|
|
10
|
+
*/
|
|
11
|
+
export declare function asFloat32Array(data: unknown): Float32Array;
|
|
12
|
+
|
|
4
13
|
/**
|
|
5
14
|
* Per-class NMS — boxes are suppressed only by other boxes of the same class.
|
|
6
15
|
*
|
|
@@ -274,6 +283,13 @@ export declare class Classifier extends VisionTask {
|
|
|
274
283
|
call(image: ImageInput, options?: ClassifierPredictOptions): Promise<ClassificationResults[]>;
|
|
275
284
|
/** Run classification on a single image. */
|
|
276
285
|
predict(image: ImageInput, options?: ClassifierPredictOptions): Promise<ClassificationResults[]>;
|
|
286
|
+
/**
|
|
287
|
+
* Resize and pack the image into the tensor the model expects.
|
|
288
|
+
*
|
|
289
|
+
* @param image The decoded input.
|
|
290
|
+
* @param canvas The opaque canvas it was decoded on, when there is one — see
|
|
291
|
+
* {@link ResizePipeline.run}.
|
|
292
|
+
*/
|
|
277
293
|
private _preprocess;
|
|
278
294
|
private _postprocess;
|
|
279
295
|
}
|
|
@@ -475,6 +491,16 @@ export declare function decodeYolo(output: Float32Array, outputDims: readonly nu
|
|
|
475
491
|
* mask coefficients) are ignored — callers can fetch them via the returned
|
|
476
492
|
* {@link DecodedAnchors.anchorIndices}.
|
|
477
493
|
*
|
|
494
|
+
* Candidates are found in two passes, mirroring the Python decoder. The first
|
|
495
|
+
* walks each class row contiguously and only *marks* anchors holding some score
|
|
496
|
+
* at or above the threshold — an anchor's best score clears the threshold
|
|
497
|
+
* exactly when one of its scores does, so no running maximum is needed and the
|
|
498
|
+
* hot loop never writes on a typical frame. The anchor-major scan it replaces
|
|
499
|
+
* read `numClasses` values spaced `numAnchors` apart for every one of the 8400
|
|
500
|
+
* anchors and cost ~1 ms even on a frame with no candidates. The winning class
|
|
501
|
+
* is then picked only for the marked anchors — a few hundred, not 8400 — with
|
|
502
|
+
* the same strict `>` that keeps the lowest class id on a tie.
|
|
503
|
+
*
|
|
478
504
|
* @param data Flat per-anchor output, length `channels * numAnchors`.
|
|
479
505
|
* @param dims Dims as reported by ORT, e.g. `[1, 84, 8400]` (det) or
|
|
480
506
|
* `[1, 116, 8400]` (seg). The leading batch dim must be 1.
|
|
@@ -510,6 +536,11 @@ export declare interface DecodeYoloOptions {
|
|
|
510
536
|
*
|
|
511
537
|
* Compatible with YOLOv8-seg / YOLOv11-seg.
|
|
512
538
|
*
|
|
539
|
+
* The prototypes' batch axis is stripped here; the per-anchor batch axis is
|
|
540
|
+
* validated by {@link decodeYoloAnchors}. Each instance's box is mapped back
|
|
541
|
+
* into input-tensor coordinates and from there into the low-resolution
|
|
542
|
+
* prototype grid, and only the prototype pixels under it are assembled.
|
|
543
|
+
*
|
|
513
544
|
* @param perAnchorData Flat `output0`, length `(4 + numClasses + numMaskCoefs) * numAnchors`.
|
|
514
545
|
* @param perAnchorDims Dims as reported by ORT, e.g. `[1, 116, 8400]`.
|
|
515
546
|
* @param prototypeData Flat `output1`, length `numMaskCoefs * maskH * maskW`.
|
|
@@ -538,6 +569,9 @@ export declare interface DecodeYoloSegOptions {
|
|
|
538
569
|
readonly maskThreshold?: number;
|
|
539
570
|
}
|
|
540
571
|
|
|
572
|
+
/** Cache Storage bucket used when {@link OrtSessionOptions.cache} is `true`. */
|
|
573
|
+
export declare const DEFAULT_MODEL_CACHE = "ort-vision-sdk-models";
|
|
574
|
+
|
|
541
575
|
/**
|
|
542
576
|
* Default execution provider preference order for browser ORT.
|
|
543
577
|
*
|
|
@@ -546,6 +580,9 @@ export declare interface DecodeYoloSegOptions {
|
|
|
546
580
|
*/
|
|
547
581
|
export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
548
582
|
|
|
583
|
+
/** The element type every task preprocesses to, and the fallback when unknown. */
|
|
584
|
+
export declare const DEFAULT_TENSOR_TYPE = "float32";
|
|
585
|
+
|
|
549
586
|
/**
|
|
550
587
|
* Resolve a labels specification into an ordered array of class names.
|
|
551
588
|
*
|
|
@@ -669,6 +706,10 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
669
706
|
* is what lets the graph undo the letterbox transform internally and crop at
|
|
670
707
|
* native resolution instead of from the downscaled copy. That one is **not**
|
|
671
708
|
* letterboxed by definition, so it does not go through the fused path.
|
|
709
|
+
*
|
|
710
|
+
* @param image The decoded input.
|
|
711
|
+
* @param canvas The opaque canvas it was decoded on, when there is one — see
|
|
712
|
+
* {@link LetterboxPipeline.run}.
|
|
672
713
|
*/
|
|
673
714
|
private _preprocess;
|
|
674
715
|
/**
|
|
@@ -803,6 +844,14 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
803
844
|
* belongs to*, and collapsing them would lose one of the two.
|
|
804
845
|
*/
|
|
805
846
|
readonly classification?: ClassificationResult | null;
|
|
847
|
+
/**
|
|
848
|
+
* Binary foreground mask for this object, shaped to `bbox` in original-image
|
|
849
|
+
* pixels — the same contract {@link SegmentationResult.mask} uses.
|
|
850
|
+
*
|
|
851
|
+
* Populated only by a `detect_segment_classify` pipeline, where a
|
|
852
|
+
* segmentation stage ran inside the crop. `null` everywhere else.
|
|
853
|
+
*/
|
|
854
|
+
readonly mask?: Mask | null;
|
|
806
855
|
}
|
|
807
856
|
|
|
808
857
|
/**
|
|
@@ -928,8 +977,26 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
928
977
|
* readback loop, and reuses its output buffer between frames. The buffer is
|
|
929
978
|
* handed straight to ONNX Runtime, so {@link _pipeline.release} must not be
|
|
930
979
|
* called until the run resolves.
|
|
980
|
+
*
|
|
981
|
+
* @param image The decoded input.
|
|
982
|
+
* @param canvas The opaque canvas it was decoded on, when there is one — see
|
|
983
|
+
* {@link LetterboxPipeline.run}.
|
|
931
984
|
*/
|
|
932
985
|
private _preprocess;
|
|
986
|
+
/**
|
|
987
|
+
* Assemble one detection.
|
|
988
|
+
*
|
|
989
|
+
* `croppedImage` is built on first read when the SDK owns the frame's pixels
|
|
990
|
+
* — see {@link memoize} for why. A frame the caller handed over as an
|
|
991
|
+
* `RGBImage` is cropped up front instead: its buffer is theirs, and a video
|
|
992
|
+
* loop refilling it would otherwise hand a late reader the wrong frame.
|
|
993
|
+
*
|
|
994
|
+
* @param original The decoded input.
|
|
995
|
+
* @param bbox Box in original-image coordinates.
|
|
996
|
+
* @param classId Predicted class.
|
|
997
|
+
* @param confidence Detection score.
|
|
998
|
+
* @param lazy Whether `croppedImage` may be deferred.
|
|
999
|
+
*/
|
|
933
1000
|
private _buildResult;
|
|
934
1001
|
private _buildBoxes;
|
|
935
1002
|
}
|
|
@@ -1012,10 +1079,29 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
1012
1079
|
* kept rather than dropped: guessing a provider away would be worse than
|
|
1013
1080
|
* admitting ignorance about it.
|
|
1014
1081
|
*
|
|
1015
|
-
*
|
|
1016
|
-
*
|
|
1082
|
+
* The WebGPU adapter is only requested when `webgpu` is in the list. Asking
|
|
1083
|
+
* unconditionally cost every WASM-only session a `requestAdapter()` round trip
|
|
1084
|
+
* and, on a machine without a GPU, a "No available adapters" console warning
|
|
1085
|
+
* per `create`.
|
|
1086
|
+
*
|
|
1087
|
+
* @param requested Providers in preference order, already resolved, as names
|
|
1088
|
+
* or config objects.
|
|
1089
|
+
* @returns The subset that is not ruled out, in the same order and form.
|
|
1017
1090
|
*/
|
|
1018
|
-
export declare function detectProviders(requested: readonly
|
|
1091
|
+
export declare function detectProviders<T extends ProviderSpec>(requested: readonly T[]): Promise<T[]>;
|
|
1092
|
+
|
|
1093
|
+
/**
|
|
1094
|
+
* The structural surface of `Float16Array` this module needs.
|
|
1095
|
+
*
|
|
1096
|
+
* Declared here rather than pulled from `lib.es2025`: the package targets
|
|
1097
|
+
* ES2022 so it builds for consumers on older TypeScript, and the only thing
|
|
1098
|
+
* needed is "an array-like of numbers ORT recognises as half precision".
|
|
1099
|
+
*/
|
|
1100
|
+
export declare interface Float16ArrayLike {
|
|
1101
|
+
readonly length: number;
|
|
1102
|
+
readonly buffer: ArrayBufferLike;
|
|
1103
|
+
[index: number]: number;
|
|
1104
|
+
}
|
|
1019
1105
|
|
|
1020
1106
|
/**
|
|
1021
1107
|
* Convert an HWC BGR uint8 buffer (OpenCV layout) to the SDK's HWC RGB.
|
|
@@ -1062,6 +1148,17 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
1062
1148
|
/** Value of the `ovs.kind` metadata key for a detector→classifier pipeline. */
|
|
1063
1149
|
export declare const FUSION_KIND_DETECT_CLASSIFY = "detect_classify";
|
|
1064
1150
|
|
|
1151
|
+
/**
|
|
1152
|
+
* Value of `ovs.kind` for a detector→segmenter→classifier pipeline.
|
|
1153
|
+
*
|
|
1154
|
+
* The segmenter runs inside each crop rather than over the whole frame: the
|
|
1155
|
+
* detector finds the boxes, the bridge cuts them out, and the segmenter turns
|
|
1156
|
+
* each crop into a foreground mask that gates the pixels the classifier sees.
|
|
1157
|
+
* Fusing one is a build step on the Python side; the browser only loads the
|
|
1158
|
+
* result.
|
|
1159
|
+
*/
|
|
1160
|
+
export declare const FUSION_KIND_DETECT_SEGMENT_CLASSIFY = "detect_segment_classify";
|
|
1161
|
+
|
|
1065
1162
|
/**
|
|
1066
1163
|
* Raised when a model cannot be driven as a fused detect→classify pipeline.
|
|
1067
1164
|
*
|
|
@@ -1075,7 +1172,7 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
1075
1172
|
|
|
1076
1173
|
/** Everything a fused pipeline declares about how it must be driven. */
|
|
1077
1174
|
export declare interface FusionSpec {
|
|
1078
|
-
/** Pipeline family
|
|
1175
|
+
/** Pipeline family — {@link FUSION_KIND_DETECT_CLASSIFY} or {@link FUSION_KIND_DETECT_SEGMENT_CLASSIFY}. */
|
|
1079
1176
|
readonly kind: string;
|
|
1080
1177
|
/** `[width, height]` the detector stage expects — the resolution to letterbox to. */
|
|
1081
1178
|
readonly inputSize: readonly [number, number];
|
|
@@ -1103,8 +1200,39 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
1103
1200
|
readonly sdkVersion: string;
|
|
1104
1201
|
/** Whether driving this pipeline requires feeding the full-resolution input. */
|
|
1105
1202
|
readonly needsSourceImage: boolean;
|
|
1203
|
+
/** Whether this pipeline emits a {@link OUTPUT_MASKS} tensor. */
|
|
1204
|
+
readonly hasMasks: boolean;
|
|
1205
|
+
/** Class id → name for the segmenter's channels, or `null`. */
|
|
1206
|
+
readonly segmenterNames: readonly string[] | null;
|
|
1207
|
+
/**
|
|
1208
|
+
* Probability above which a segmenter pixel counted as foreground.
|
|
1209
|
+
*
|
|
1210
|
+
* Baked into the graph at fusion time; reported so a caller can say what the
|
|
1211
|
+
* masks mean.
|
|
1212
|
+
*/
|
|
1213
|
+
readonly maskThreshold: number;
|
|
1214
|
+
/** Whether the mask gated the crop before the classifier saw it. */
|
|
1215
|
+
readonly maskApplied: boolean;
|
|
1106
1216
|
}
|
|
1107
1217
|
|
|
1218
|
+
/**
|
|
1219
|
+
* Metadata key naming the level a model was graph-optimized at offline.
|
|
1220
|
+
*
|
|
1221
|
+
* Written by the Python SDK's `optimize_model`; both test suites pin the same
|
|
1222
|
+
* string, so a rename on one side fails. See {@link OrtSession.create} for what
|
|
1223
|
+
* the session does when it finds it.
|
|
1224
|
+
*/
|
|
1225
|
+
export declare const GRAPH_OPTIMIZATION_KEY = "ort_vision_sdk.graph_optimization";
|
|
1226
|
+
|
|
1227
|
+
/**
|
|
1228
|
+
* Whether this environment can build a half-precision tensor at all.
|
|
1229
|
+
*
|
|
1230
|
+
* @returns `true` when the runtime exposes `Float16Array`. ORT rejects any
|
|
1231
|
+
* other typed array for a `float16` tensor, so a `false` here means half
|
|
1232
|
+
* precision is unreachable in this browser no matter what the model declares.
|
|
1233
|
+
*/
|
|
1234
|
+
export declare function hasFloat16Array(): boolean;
|
|
1235
|
+
|
|
1108
1236
|
/** Mean that leaves an image untouched — what an Ultralytics classifier expects. */
|
|
1109
1237
|
export declare const IDENTITY_MEAN: readonly [number, number, number];
|
|
1110
1238
|
|
|
@@ -1112,7 +1240,7 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
1112
1240
|
export declare const IDENTITY_STD: readonly [number, number, number];
|
|
1113
1241
|
|
|
1114
1242
|
/** Anything {@link loadImage} accepts as an image input. */
|
|
1115
|
-
export declare type ImageInput = string | Blob | HTMLImageElement | HTMLCanvasElement | OffscreenCanvas | ImageBitmap | ImageData | RGBImage;
|
|
1243
|
+
export declare type ImageInput = string | Blob | HTMLImageElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | ImageBitmap | VideoFrame | ImageData | RGBImage;
|
|
1116
1244
|
|
|
1117
1245
|
/** Raised when an input image cannot be decoded into the canonical format. */
|
|
1118
1246
|
export declare class ImageLoadError extends OrtVisionError {
|
|
@@ -1235,26 +1363,43 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
1235
1363
|
* into — see {@link ReusableBuffer}.
|
|
1236
1364
|
*
|
|
1237
1365
|
* @param image Source image in the SDK's canonical HWC RGB layout.
|
|
1366
|
+
* @param canvas Optional drawable already holding exactly `image`'s pixels at
|
|
1367
|
+
* `image.width × image.height`, fully opaque. When given it is drawn
|
|
1368
|
+
* straight into the target, skipping the RGB→RGBA rebuild and the
|
|
1369
|
+
* `putImageData` onto the scratch canvas — two full-resolution passes per
|
|
1370
|
+
* frame. The tasks pass the canvas `loadImage` decoded onto; a caller with a
|
|
1371
|
+
* translucent source must not, since the result would composite its alpha.
|
|
1238
1372
|
*/
|
|
1239
|
-
run(image: RGBImage): FusedLetterboxResult;
|
|
1373
|
+
run(image: RGBImage, canvas?: CanvasImageSource | null): FusedLetterboxResult;
|
|
1240
1374
|
/**
|
|
1241
1375
|
* Mark the reusable buffer free again.
|
|
1242
1376
|
*
|
|
1243
1377
|
* Call it once the tensor built from a {@link run} result has been handed to
|
|
1244
1378
|
* ONNX Runtime and the run has resolved — after that the values are inside
|
|
1245
1379
|
* the WASM heap and the buffer can be overwritten.
|
|
1380
|
+
*
|
|
1381
|
+
* Pass the result being released when several runs can be outstanding at
|
|
1382
|
+
* once. A result that got a fresh allocation (`reused: false`) holds nothing
|
|
1383
|
+
* of the pipeline's, so releasing it is a no-op; without that check, the
|
|
1384
|
+
* caller finishing with a fresh buffer would free the shared one while
|
|
1385
|
+
* another caller still had its frame in it. Called without an argument it
|
|
1386
|
+
* frees the shared buffer unconditionally, as before.
|
|
1387
|
+
*
|
|
1388
|
+
* @param result The {@link run} result being released.
|
|
1246
1389
|
*/
|
|
1247
|
-
release(
|
|
1390
|
+
release(result?: {
|
|
1391
|
+
readonly reused: boolean;
|
|
1392
|
+
}): void;
|
|
1248
1393
|
/**
|
|
1249
|
-
*
|
|
1394
|
+
* Paint an image onto the scratch source canvas and return that canvas.
|
|
1250
1395
|
*
|
|
1251
|
-
*
|
|
1252
|
-
* the last one, which for a camera or video source is never after the
|
|
1396
|
+
* The canvas is only reallocated when a frame arrives at a different size
|
|
1397
|
+
* than the last one, which for a camera or video source is never after the
|
|
1398
|
+
* first.
|
|
1253
1399
|
*
|
|
1254
|
-
* @param
|
|
1255
|
-
* @param height Source height in pixels.
|
|
1400
|
+
* @param image Source image in the SDK's canonical HWC RGB layout.
|
|
1256
1401
|
*/
|
|
1257
|
-
private
|
|
1402
|
+
private _paintSource;
|
|
1258
1403
|
}
|
|
1259
1404
|
|
|
1260
1405
|
export declare interface LetterboxResult {
|
|
@@ -1284,6 +1429,10 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
1284
1429
|
/**
|
|
1285
1430
|
* Load an image from any supported source into a HWC uint8 RGB array.
|
|
1286
1431
|
*
|
|
1432
|
+
* A video element must have its current frame available (`readyState >= 2`);
|
|
1433
|
+
* the frame showing at the time of the call is the one decoded. A `VideoFrame`
|
|
1434
|
+
* is read but not closed — it belongs to the caller.
|
|
1435
|
+
*
|
|
1287
1436
|
* @throws {@link ImageLoadError} if the source cannot be decoded or has an unsupported shape.
|
|
1288
1437
|
*/
|
|
1289
1438
|
export declare function loadImage(source: ImageInput): Promise<RGBImage>;
|
|
@@ -1417,6 +1566,13 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
1417
1566
|
* @returns Indices of kept boxes, in descending score order. Boxes tied on
|
|
1418
1567
|
* score are visited lowest-index first, so the survivor of a tie is
|
|
1419
1568
|
* deterministic and matches both `torchvision` and the Python SDK.
|
|
1569
|
+
*
|
|
1570
|
+
* The boxes are copied once into score order, so the inner loop walks a
|
|
1571
|
+
* contiguous `Float32Array` instead of hopping through `order` into the caller's
|
|
1572
|
+
* layout. A pair with no horizontal overlap is skipped before the vertical
|
|
1573
|
+
* extent is computed: its IoU is `0`, which suppresses nothing for any
|
|
1574
|
+
* non-negative threshold — the shortcut is gated on that so a negative
|
|
1575
|
+
* threshold keeps the exact semantics.
|
|
1420
1576
|
*/
|
|
1421
1577
|
export declare function nms(boxes: Float32Array, scores: Float32Array, iouThreshold: number): Int32Array;
|
|
1422
1578
|
|
|
@@ -1512,7 +1668,22 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
1512
1668
|
* `webgpu` on a device without it runs on WASM and is told nothing.
|
|
1513
1669
|
*/
|
|
1514
1670
|
readonly requestedProviders: readonly string[];
|
|
1671
|
+
/**
|
|
1672
|
+
* Tensor type each input declares, keyed by input name.
|
|
1673
|
+
*
|
|
1674
|
+
* Read from the model file, not from the session: `inputMetadata` is
|
|
1675
|
+
* `undefined` on `onnxruntime-web` 1.20.1. Empty when the bytes were never
|
|
1676
|
+
* in hand — a URL loaded with `readMetadata: false` — in which case every
|
|
1677
|
+
* feed is built as float32, the behaviour before this existed.
|
|
1678
|
+
*/
|
|
1679
|
+
private readonly _inputTypes;
|
|
1515
1680
|
private constructor();
|
|
1681
|
+
/**
|
|
1682
|
+
* Tail of the run queue: settles when the latest queued run has finished.
|
|
1683
|
+
*
|
|
1684
|
+
* See {@link run} for why runs are serialized.
|
|
1685
|
+
*/
|
|
1686
|
+
private _runQueue;
|
|
1516
1687
|
/**
|
|
1517
1688
|
* Load an ONNX model into an ORT inference session.
|
|
1518
1689
|
*
|
|
@@ -1526,12 +1697,35 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
1526
1697
|
* it — on a phone that was the difference between a session and
|
|
1527
1698
|
* `Can't create a session. failed to allocate a buffer of size N`.
|
|
1528
1699
|
*
|
|
1700
|
+
* The provider probe and the model download run concurrently: neither
|
|
1701
|
+
* depends on the other, and on a real GPU `requestAdapter()` is not free.
|
|
1702
|
+
* When the model is downloaded here, ORT's runtime is started alongside the
|
|
1703
|
+
* download too — see {@link warmRuntime}.
|
|
1704
|
+
*
|
|
1705
|
+
* A model whose metadata carries {@link GRAPH_OPTIMIZATION_KEY} was already
|
|
1706
|
+
* optimized offline by the Python SDK's `optimize_model`, so it is loaded
|
|
1707
|
+
* with `graphOptimizationLevel: "disabled"` instead of paying the optimizer a
|
|
1708
|
+
* second time — measured on a YOLO11n-seg under WASM, 29 ms of session
|
|
1709
|
+
* creation down to 12 ms with unchanged inference time. An explicit
|
|
1710
|
+
* `graphOptimizationLevel` in `sessionOptions` always wins. The mark is read
|
|
1711
|
+
* with the rest of the metadata, so `readMetadata: false` skips it too.
|
|
1712
|
+
*
|
|
1529
1713
|
* @param model Either a URL string, or a `Uint8Array`/`ArrayBuffer` containing the model bytes.
|
|
1530
1714
|
* @param options Provider list, pass-through `SessionOptions`, and whether to
|
|
1531
1715
|
* read the model's metadata map (see {@link OrtSessionOptions.readMetadata}).
|
|
1532
1716
|
* @throws {@link ModelLoadError} if the model cannot be loaded.
|
|
1533
1717
|
*/
|
|
1534
1718
|
static create(model: ModelSource, options?: OrtSessionOptions): Promise<OrtSession>;
|
|
1719
|
+
/**
|
|
1720
|
+
* Tensor type each input declares, in declaration order.
|
|
1721
|
+
*
|
|
1722
|
+
* `"float32"` for a normal export, `"float16"` for one exported with
|
|
1723
|
+
* `half=True`. ORT matches a feed's type against this exactly, so the tasks
|
|
1724
|
+
* read it and convert at the feed boundary.
|
|
1725
|
+
*/
|
|
1726
|
+
get inputDtypes(): readonly string[];
|
|
1727
|
+
/** Tensor type the first input declares. */
|
|
1728
|
+
get inputDtype(): string;
|
|
1535
1729
|
/** Names of the model's inputs, in declaration order. */
|
|
1536
1730
|
get inputNames(): readonly string[];
|
|
1537
1731
|
/** Name of the first (and usually only) input. */
|
|
@@ -1588,855 +1782,1037 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
|
1588
1782
|
/**
|
|
1589
1783
|
* Run inference and return all outputs.
|
|
1590
1784
|
*
|
|
1785
|
+
* Runs on one session are serialized. ONNX Runtime Web refuses a second
|
|
1786
|
+
* `run` while one is in flight — it rejects with `Session already started` —
|
|
1787
|
+
* so two overlapping `predict()` calls on the same task used to fail. Queued
|
|
1788
|
+
* here, they overlap where they can: while one run executes (in a worker,
|
|
1789
|
+
* with `env.wasm.proxy`, or on the GPU), the next `predict()` decodes and
|
|
1790
|
+
* preprocesses its frame and only waits for its turn at the runtime. A run
|
|
1791
|
+
* that fails does not block the ones behind it.
|
|
1792
|
+
*
|
|
1591
1793
|
* @param feeds Map of input name to `ort.Tensor`. Keys must match {@link inputNames}.
|
|
1592
1794
|
* @throws {@link InferenceError} if ORT raises any error during execution.
|
|
1593
1795
|
*/
|
|
1594
1796
|
run(feeds: Record<string, ort.Tensor>): Promise<Record<string, ort.Tensor>>;
|
|
1595
|
-
}
|
|
1596
|
-
|
|
1597
|
-
export declare interface OrtSessionOptions {
|
|
1598
|
-
/**
|
|
1599
|
-
* Execution providers in preference order. `undefined` uses {@link DEFAULT_PROVIDERS}.
|
|
1600
|
-
*
|
|
1601
|
-
* Naming one explicitly also opts into a `console.warn` when this browser
|
|
1602
|
-
* cannot offer it, instead of falling back in silence.
|
|
1603
|
-
*/
|
|
1604
|
-
readonly providers?: readonly string[];
|
|
1605
|
-
/** Optional ORT session options forwarded to `InferenceSession.create`. */
|
|
1606
|
-
readonly sessionOptions?: ort.InferenceSession.SessionOptions;
|
|
1607
|
-
/**
|
|
1608
|
-
* Whether to read the model's custom metadata map (`names`, `task`, `imgsz`).
|
|
1609
|
-
* Defaults to `true`.
|
|
1610
|
-
*
|
|
1611
|
-
* The runtime does not expose that map, so it is read from the file itself —
|
|
1612
|
-
* which means a URL model is fetched here and handed to ORT as bytes instead
|
|
1613
|
-
* of letting ORT fetch it. That is the same single download either way, and
|
|
1614
|
-
* it is what lets a task resolve its labels off the model. Set to `false` to
|
|
1615
|
-
* keep the URL path untouched and leave {@link OrtSession.metadata} empty.
|
|
1616
|
-
*
|
|
1617
|
-
* `false` is also the escape hatch when a device cannot afford the bytes: the
|
|
1618
|
-
* fetched buffer is dropped before ORT builds the graph (see
|
|
1619
|
-
* {@link OrtSession.create}), but ORT's own load path still keeps the model out
|
|
1620
|
-
* of reach of anything the SDK holds. A session built this way resolves its
|
|
1621
|
-
* input size from the graph as usual — only the class names are lost, so a
|
|
1622
|
-
* caller taking this route has to pass `labels` itself.
|
|
1623
|
-
*/
|
|
1624
|
-
readonly readMetadata?: boolean;
|
|
1625
|
-
}
|
|
1626
|
-
|
|
1627
|
-
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
1628
|
-
/**
|
|
1629
|
-
* Exceptions raised by the SDK.
|
|
1630
|
-
*
|
|
1631
|
-
* All exceptions inherit from {@link OrtVisionError}, so callers can catch
|
|
1632
|
-
* the base class to handle any SDK-originated failure uniformly.
|
|
1633
|
-
*/
|
|
1634
|
-
export declare class OrtVisionError extends Error {
|
|
1635
|
-
constructor(message: string, options?: ErrorOptions);
|
|
1636
|
-
}
|
|
1637
|
-
|
|
1638
|
-
/**
|
|
1639
|
-
* Name of the `[K, 4]` float32 xyxy output, in **letterboxed** input pixels.
|
|
1640
|
-
*
|
|
1641
|
-
* A file fused by `ort-vision-sdk` 0.9.0 or later reports the box that was
|
|
1642
|
-
* actually classified: clamped to the image the crop came from, exactly as
|
|
1643
|
-
* RoiAlign received it. Older files report the raw box, so one that ran off the
|
|
1644
|
-
* frame draws a rectangle wider than the region the classifier saw.
|
|
1645
|
-
*/
|
|
1646
|
-
export declare const OUTPUT_BOXES = "boxes";
|
|
1647
|
-
|
|
1648
|
-
/** Name of the `[K]` int64 detector-class output. */
|
|
1649
|
-
export declare const OUTPUT_CLASSES = "classes";
|
|
1650
|
-
|
|
1651
|
-
/** Name of the `[1]` int64 output holding how many of the `K` rows are real. */
|
|
1652
|
-
export declare const OUTPUT_NUM_DETECTIONS = "num_detections";
|
|
1653
|
-
|
|
1654
|
-
/** Name of the `[K, numClassifierClasses]` float32 classifier output, one row per box. */
|
|
1655
|
-
export declare const OUTPUT_PROBS = "probs";
|
|
1656
|
-
|
|
1657
|
-
/** Name of the `[K]` float32 detection-confidence output. */
|
|
1658
|
-
export declare const OUTPUT_SCORES = "scores";
|
|
1659
|
-
|
|
1660
|
-
/**
|
|
1661
|
-
* Parse a `repr`-encoded `dict[int, str]` class map.
|
|
1662
|
-
*
|
|
1663
|
-
* Split out of {@link modelNames} because the same encoding is reused by a
|
|
1664
|
-
* fused pipeline, which carries one class map per stage and therefore cannot
|
|
1665
|
-
* store both under the single `names` key Ultralytics uses.
|
|
1666
|
-
*
|
|
1667
|
-
* @param encoded The encoded map — e.g. `"{0: 'deworm', 1: 'not_deworm'}"`.
|
|
1668
|
-
* @returns Class names in class-id order, or `null` when the value is missing,
|
|
1669
|
-
* unparseable, not a `dict`, or not keyed by contiguous integers from zero.
|
|
1670
|
-
*/
|
|
1671
|
-
export declare function parseNames(encoded: string | undefined): readonly string[] | null;
|
|
1672
|
-
|
|
1673
|
-
/**
|
|
1674
|
-
* Top-k classification probabilities for a single image.
|
|
1675
|
-
*
|
|
1676
|
-
* Mirrors Ultralytics' `Probs` interface.
|
|
1677
|
-
*/
|
|
1678
|
-
export declare class Probs {
|
|
1679
|
-
readonly data: Float32Array;
|
|
1680
|
-
/** @param data `[numClasses]` per-class probabilities, indexed by class id. */
|
|
1681
|
-
constructor(data: Float32Array);
|
|
1682
|
-
/** Number of classes. */
|
|
1683
|
-
get length(): number;
|
|
1684
|
-
/** `[numClasses]` shape of the underlying vector. */
|
|
1685
|
-
get shape(): readonly [number];
|
|
1686
|
-
/** Index of the most probable class. */
|
|
1687
|
-
get top1(): number;
|
|
1688
|
-
/** Probability of the top-1 class. */
|
|
1689
|
-
get top1conf(): number;
|
|
1690
|
-
/**
|
|
1691
|
-
* Indices of the top-5 most probable classes, descending.
|
|
1692
|
-
*
|
|
1693
|
-
* The array is the memoised selection itself, not a copy — reading it twice
|
|
1694
|
-
* hands back the same object. Treat it as read-only: writing into it edits
|
|
1695
|
-
* what every later read of this `Probs` returns.
|
|
1696
|
-
*/
|
|
1697
|
-
get top5(): Int32Array;
|
|
1698
|
-
/**
|
|
1699
|
-
* Probabilities of the top-5 classes, descending.
|
|
1700
|
-
*
|
|
1701
|
-
* Shares the memoised selection with {@link Probs.top5}, under the same
|
|
1702
|
-
* read-only caveat.
|
|
1703
|
-
*/
|
|
1704
|
-
get top5conf(): Float32Array;
|
|
1705
|
-
private _cache;
|
|
1706
|
-
/**
|
|
1707
|
-
* Memoised {@link Probs._topK}.
|
|
1708
|
-
*
|
|
1709
|
-
* `top5` and `top5conf` are separate getters over the same selection, so a
|
|
1710
|
-
* caller reading both would otherwise pay for it twice — once per frame, in
|
|
1711
|
-
* a camera loop. The probabilities a `Probs` was built from do not change,
|
|
1712
|
-
* so the result is computed once per `k` and kept.
|
|
1713
|
-
*
|
|
1714
|
-
* @param k - How many classes to select.
|
|
1715
|
-
* @returns The cached selection for that `k`.
|
|
1716
|
-
*/
|
|
1717
|
-
private _top;
|
|
1718
|
-
/**
|
|
1719
|
-
* Select the `k` highest probabilities without ordering the rest.
|
|
1720
|
-
*
|
|
1721
|
-
* A full sort to read five entries out of a thousand-class vector costs
|
|
1722
|
-
* O(n log n) plus an index array the size of the vector; keeping `k` slots
|
|
1723
|
-
* ordered by insertion and scanning once costs O(n·k) with no allocation
|
|
1724
|
-
* beyond the result. Measured on 1000 classes over 2000 iterations, 134.5 µs
|
|
1725
|
-
* against 1.5 µs for the same output.
|
|
1726
|
-
*
|
|
1727
|
-
* Ties keep the lower class index first, matching the stable sort this
|
|
1728
|
-
* replaced: a candidate only displaces an entry it is strictly greater
|
|
1729
|
-
* than. That is also what keeps this selection in step with the Python
|
|
1730
|
-
* SDK's `np.argsort(-data, kind="stable")`, where the sort is C and the
|
|
1731
|
-
* cost this avoids does not arise.
|
|
1732
|
-
*
|
|
1733
|
-
* @param k - How many classes to select.
|
|
1734
|
-
* @returns Indices and probabilities, descending by probability.
|
|
1735
|
-
*/
|
|
1736
|
-
private _topK;
|
|
1737
|
-
}
|
|
1738
|
-
|
|
1739
|
-
/** Raised when a requested execution provider is not available. */
|
|
1740
|
-
export declare class ProviderNotAvailableError extends OrtVisionError {
|
|
1741
|
-
}
|
|
1742
|
-
|
|
1743
|
-
/**
|
|
1744
|
-
* Read a pipeline spec out of a model's custom metadata.
|
|
1745
|
-
*
|
|
1746
|
-
* Individual malformed entries fall back to the value a fusion would have used
|
|
1747
|
-
* by default — a single bad float is not a reason to reject an otherwise
|
|
1748
|
-
* loadable pipeline. A malformed resolution is fatal, because there is no safe
|
|
1749
|
-
* default for one.
|
|
1750
|
-
*
|
|
1751
|
-
* @param metadata A model's custom metadata map, as read by
|
|
1752
|
-
* {@link readModelMetadata}.
|
|
1753
|
-
* @returns The decoded spec, or `null` when the model is not a fused pipeline —
|
|
1754
|
-
* it carries no `ovs.kind` entry, or one naming a pipeline kind this version
|
|
1755
|
-
* does not know how to drive.
|
|
1756
|
-
*/
|
|
1757
|
-
export declare function readFusionSpec(metadata: Readonly<Record<string, string>> | undefined): FusionSpec | null;
|
|
1758
|
-
|
|
1759
|
-
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
1760
|
-
/**
|
|
1761
|
-
* Read the metadata an exporter baked into a `.onnx` file.
|
|
1762
|
-
*
|
|
1763
|
-
* `onnxruntime-web` exposes input/output metadata but **not** the model's
|
|
1764
|
-
* custom metadata map, which is where Ultralytics writes `names`, `task` and
|
|
1765
|
-
* `imgsz`. The Python SDK gets it for free from
|
|
1766
|
-
* `InferenceSession.get_modelmeta().custom_metadata_map`; in the browser the
|
|
1767
|
-
* only way to the same information is to read it out of the file, so this
|
|
1768
|
-
* module walks just enough of the ModelProto wire format to collect
|
|
1769
|
-
* `metadata_props`.
|
|
1770
|
-
*
|
|
1771
|
-
* It never throws and never allocates unbounded: a truncated, hostile or
|
|
1772
|
-
* simply unexpected file yields an empty map, and every caller treats that as
|
|
1773
|
-
* "the model says nothing", falling back to what it was given.
|
|
1774
|
-
*/
|
|
1775
|
-
/**
|
|
1776
|
-
* Collect a model's custom metadata map straight out of its bytes.
|
|
1777
|
-
*
|
|
1778
|
-
* @param model The `.onnx` file contents.
|
|
1779
|
-
* @returns Key/value metadata — `names`, `task`, `imgsz`, ... for an
|
|
1780
|
-
* Ultralytics export — or an empty object when the file carries none or
|
|
1781
|
-
* cannot be walked.
|
|
1782
|
-
*/
|
|
1783
|
-
export declare function readModelMetadata(model: Uint8Array | ArrayBufferLike): Readonly<Record<string, string>>;
|
|
1784
|
-
|
|
1785
|
-
/**
|
|
1786
|
-
* Turn an empty result into an error, when the caller asked for that.
|
|
1787
|
-
*
|
|
1788
|
-
* Shared by every task that can come back with nothing — {@link Detector},
|
|
1789
|
-
* {@link Segmenter} and {@link DetectClassify} — so the three agree on when
|
|
1790
|
-
* they throw and on what the message says. The message names the two settings
|
|
1791
|
-
* that decide the outcome, because "no detections" on its own leaves the reader
|
|
1792
|
-
* unable to tell a blank image from a threshold set too high.
|
|
1793
|
-
*
|
|
1794
|
-
* @param count How many detections survived every filter.
|
|
1795
|
-
* @param options The flag for this call, the threshold actually applied (after
|
|
1796
|
-
* any per-call override), the class allowlist if one narrowed the search, and
|
|
1797
|
-
* the source path when the input was one.
|
|
1798
|
-
* @throws {@link NoDetectionsError} when the flag is set and `count` is zero.
|
|
1799
|
-
*/
|
|
1800
|
-
export declare function requireDetections(count: number, options: {
|
|
1801
|
-
readonly raiseOnEmpty: boolean;
|
|
1802
|
-
readonly confThreshold: number;
|
|
1803
|
-
readonly classes: readonly number[] | undefined;
|
|
1804
|
-
readonly path: string | null;
|
|
1805
|
-
}): void;
|
|
1806
|
-
|
|
1807
|
-
/** Resize an image to `(targetWidth, targetHeight)` using high-quality canvas resampling. */
|
|
1808
|
-
export declare function resize(image: RGBImage, targetWidth: number, targetHeight: number): RGBImage;
|
|
1809
|
-
|
|
1810
1797
|
/**
|
|
1811
|
-
*
|
|
1812
|
-
*
|
|
1813
|
-
* The classification counterpart of {@link LetterboxPipeline}. A classifier
|
|
1814
|
-
* stretches to the model's square input instead of letterboxing into it — no
|
|
1815
|
-
* padding, no scale to invert later, because nothing is mapped back onto the
|
|
1816
|
-
* source image afterwards. That difference is why it cannot simply reuse the
|
|
1817
|
-
* letterbox path.
|
|
1798
|
+
* Execute one run immediately; only {@link run}'s queue calls this.
|
|
1818
1799
|
*
|
|
1819
|
-
*
|
|
1820
|
-
*
|
|
1821
|
-
* `Float32Array`s and walks each end to end on every call: about 1.4 MB of
|
|
1822
|
-
* fresh garbage per 224×224 `predict()`, produced at the exact moment a phone
|
|
1823
|
-
* near its memory ceiling can least afford it. Here one `drawImage` resizes,
|
|
1824
|
-
* and one loop reads the resulting RGBA and writes normalized planar float32
|
|
1825
|
-
* into a buffer held across calls.
|
|
1826
|
-
*
|
|
1827
|
-
* Create one per task, not per frame.
|
|
1828
|
-
*/
|
|
1829
|
-
export declare class ResizePipeline {
|
|
1830
|
-
private readonly _targetWidth;
|
|
1831
|
-
private readonly _targetHeight;
|
|
1832
|
-
private readonly _mean;
|
|
1833
|
-
private readonly _std;
|
|
1834
|
-
private readonly _buffer;
|
|
1835
|
-
private _target;
|
|
1836
|
-
private _targetContext;
|
|
1837
|
-
private _source;
|
|
1838
|
-
private _sourceContext;
|
|
1839
|
-
/**
|
|
1840
|
-
* @param targetWidth Model input width in pixels.
|
|
1841
|
-
* @param targetHeight Model input height in pixels.
|
|
1842
|
-
* @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.
|
|
1843
|
-
* @param std Per-channel RGB standard deviation. Defaults to no scaling.
|
|
1800
|
+
* @param feeds Map of input name to `ort.Tensor`.
|
|
1801
|
+
* @throws {@link InferenceError} if ORT raises any error during execution.
|
|
1844
1802
|
*/
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1803
|
+
private _runNow;
|
|
1804
|
+
/**
|
|
1805
|
+
* Rebuild any feed whose type does not match what its input declares.
|
|
1806
|
+
*
|
|
1807
|
+
* ORT matches feed types against the graph exactly: a float32 tensor against
|
|
1808
|
+
* a `half=True` export fails the run with `Unexpected input data type`. Every
|
|
1809
|
+
* preprocessing path in this SDK produces `Float32Array`, on purpose — the
|
|
1810
|
+
* normalization arithmetic belongs in single precision — so the conversion
|
|
1811
|
+
* happens here, once, at the only boundary all of them pass through. A feed
|
|
1812
|
+
* that already carries the declared type is handed on untouched, so a caller
|
|
1813
|
+
* building its own half tensor pays nothing.
|
|
1814
|
+
*
|
|
1815
|
+
* @param feeds The tensors to run with.
|
|
1816
|
+
* @returns The same mapping, with mismatched float32 feeds converted.
|
|
1817
|
+
*/
|
|
1818
|
+
private _asDeclaredTypes;
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
export declare interface OrtSessionOptions {
|
|
1848
1822
|
/**
|
|
1849
|
-
*
|
|
1823
|
+
* Execution providers in preference order. `undefined` uses {@link DEFAULT_PROVIDERS}.
|
|
1850
1824
|
*
|
|
1851
|
-
*
|
|
1852
|
-
*
|
|
1853
|
-
*
|
|
1854
|
-
* input untouched.
|
|
1825
|
+
* Each entry is a name (`"webgpu"`, `"wasm"`) or ORT's config object for that
|
|
1826
|
+
* provider, which passes provider options through untouched —
|
|
1827
|
+
* `{ name: "webgpu", preferredLayout: "NHWC" }`.
|
|
1855
1828
|
*
|
|
1856
|
-
*
|
|
1857
|
-
*
|
|
1858
|
-
*
|
|
1859
|
-
* @param image Source image in the SDK's canonical HWC RGB layout.
|
|
1829
|
+
* Naming one explicitly also opts into a `console.warn` when this browser
|
|
1830
|
+
* cannot offer it, instead of falling back in silence.
|
|
1860
1831
|
*/
|
|
1861
|
-
|
|
1832
|
+
readonly providers?: readonly ProviderSpec[];
|
|
1862
1833
|
/**
|
|
1863
|
-
*
|
|
1834
|
+
* Keep a URL model in the browser's Cache Storage between page loads.
|
|
1835
|
+
*
|
|
1836
|
+
* `true` uses the {@link DEFAULT_MODEL_CACHE} bucket; a string names the
|
|
1837
|
+
* bucket. A later `create` with the same URL reads the bytes from the cache
|
|
1838
|
+
* instead of the network — for a model of several megabytes, the bulk of
|
|
1839
|
+
* what a returning visitor waits for. Defaults to `false`.
|
|
1864
1840
|
*
|
|
1865
|
-
*
|
|
1866
|
-
*
|
|
1867
|
-
* the
|
|
1841
|
+
* The URL is the cache key and nothing expires it: publish a changed model
|
|
1842
|
+
* under a new URL (`yolo.v2.onnx`, `?v=2`) or a new bucket name, or delete
|
|
1843
|
+
* the bucket with `caches.delete(name)`. Ignored for models passed as bytes,
|
|
1844
|
+
* and silently skipped where Cache Storage is unavailable (non-secure
|
|
1845
|
+
* contexts, blocked site data).
|
|
1868
1846
|
*/
|
|
1869
|
-
|
|
1847
|
+
readonly cache?: boolean | string;
|
|
1848
|
+
/** Optional ORT session options forwarded to `InferenceSession.create`. */
|
|
1849
|
+
readonly sessionOptions?: ort.InferenceSession.SessionOptions;
|
|
1870
1850
|
/**
|
|
1871
|
-
*
|
|
1851
|
+
* Whether to read the model's custom metadata map (`names`, `task`, `imgsz`).
|
|
1852
|
+
* Defaults to `true`.
|
|
1872
1853
|
*
|
|
1873
|
-
*
|
|
1874
|
-
*
|
|
1875
|
-
*
|
|
1854
|
+
* The runtime does not expose that map, so it is read from the file itself —
|
|
1855
|
+
* which means a URL model is fetched here and handed to ORT as bytes instead
|
|
1856
|
+
* of letting ORT fetch it. That is the same single download either way, and
|
|
1857
|
+
* it is what lets a task resolve its labels off the model. Set to `false` to
|
|
1858
|
+
* keep the URL path untouched (unless {@link cache} is set, which needs the
|
|
1859
|
+
* bytes) and leave {@link OrtSession.metadata} empty.
|
|
1860
|
+
*
|
|
1861
|
+
* `false` is also the escape hatch when a device cannot afford the bytes: the
|
|
1862
|
+
* fetched buffer is dropped before ORT builds the graph (see
|
|
1863
|
+
* {@link OrtSession.create}), but ORT's own load path still keeps the model out
|
|
1864
|
+
* of reach of anything the SDK holds. A session built this way resolves its
|
|
1865
|
+
* input size from the graph as usual — only the class names are lost, so a
|
|
1866
|
+
* caller taking this route has to pass `labels` itself.
|
|
1876
1867
|
*/
|
|
1877
|
-
|
|
1878
|
-
/** Grow the scratch source canvas to fit an image, reusing it when possible. */
|
|
1879
|
-
private _ensureSource;
|
|
1868
|
+
readonly readMetadata?: boolean;
|
|
1880
1869
|
}
|
|
1881
1870
|
|
|
1871
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
1882
1872
|
/**
|
|
1883
|
-
*
|
|
1884
|
-
*
|
|
1885
|
-
* The allocation-free path is {@link ResizePipeline}; this is the one-shot
|
|
1886
|
-
* form, for a caller who wants the fused behaviour without owning a pipeline.
|
|
1873
|
+
* Exceptions raised by the SDK.
|
|
1887
1874
|
*
|
|
1888
|
-
*
|
|
1889
|
-
*
|
|
1890
|
-
* @param targetHeight Model input height in pixels.
|
|
1891
|
-
* @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.
|
|
1892
|
-
* @param std Per-channel RGB standard deviation. Defaults to no scaling.
|
|
1875
|
+
* All exceptions inherit from {@link OrtVisionError}, so callers can catch
|
|
1876
|
+
* the base class to handle any SDK-originated failure uniformly.
|
|
1893
1877
|
*/
|
|
1894
|
-
export declare
|
|
1895
|
-
|
|
1896
|
-
/** What {@link resolveNormalization} settled on. */
|
|
1897
|
-
export declare interface ResolvedNormalization {
|
|
1898
|
-
/** The preset name, or `"custom"` when the caller supplied the values. */
|
|
1899
|
-
readonly name: string;
|
|
1900
|
-
/** Per-channel mean to subtract. */
|
|
1901
|
-
readonly mean: readonly [number, number, number];
|
|
1902
|
-
/** Per-channel deviation to divide by. */
|
|
1903
|
-
readonly std: readonly [number, number, number];
|
|
1878
|
+
export declare class OrtVisionError extends Error {
|
|
1879
|
+
constructor(message: string, options?: ErrorOptions);
|
|
1904
1880
|
}
|
|
1905
1881
|
|
|
1906
1882
|
/**
|
|
1907
|
-
*
|
|
1883
|
+
* Name of the `[K, 4]` float32 xyxy output, in **letterboxed** input pixels.
|
|
1908
1884
|
*
|
|
1909
|
-
*
|
|
1910
|
-
*
|
|
1911
|
-
*
|
|
1912
|
-
*
|
|
1913
|
-
* reported through `console.warn` instead of being swallowed.
|
|
1914
|
-
*
|
|
1915
|
-
* @param options Graph shape, requested size and per-task fallback.
|
|
1916
|
-
* @returns The `[width, height]` to preprocess to.
|
|
1885
|
+
* A file fused by `ort-vision-sdk` 0.9.0 or later reports the box that was
|
|
1886
|
+
* actually classified: clamped to the image the crop came from, exactly as
|
|
1887
|
+
* RoiAlign received it. Older files report the raw box, so one that ran off the
|
|
1888
|
+
* frame draws a rectangle wider than the region the classifier saw.
|
|
1917
1889
|
*/
|
|
1918
|
-
export declare
|
|
1919
|
-
|
|
1920
|
-
export declare interface ResolveInputSizeOptions {
|
|
1921
|
-
/** Declared shape of the model's image input, from {@link declaredShapesFrom}. */
|
|
1922
|
-
readonly graphShape?: DeclaredShape;
|
|
1923
|
-
/** Size the caller asked for, if any. */
|
|
1924
|
-
readonly requested?: readonly [number, number];
|
|
1925
|
-
/** Size to use when neither the graph nor the caller pins one. */
|
|
1926
|
-
readonly fallback: readonly [number, number];
|
|
1927
|
-
}
|
|
1890
|
+
export declare const OUTPUT_BOXES = "boxes";
|
|
1928
1891
|
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
export declare interface ResolveLabelsOptions {
|
|
1932
|
-
/**
|
|
1933
|
-
* Expected number of classes.
|
|
1934
|
-
*
|
|
1935
|
-
* - When `spec` is `null`/`undefined`, this is required to auto-generate names.
|
|
1936
|
-
* - When `spec` is provided, it validates that the resolved length matches.
|
|
1937
|
-
*/
|
|
1938
|
-
readonly numClasses?: number;
|
|
1939
|
-
}
|
|
1892
|
+
/** Name of the `[K]` int64 detector-class output. */
|
|
1893
|
+
export declare const OUTPUT_CLASSES = "classes";
|
|
1940
1894
|
|
|
1941
1895
|
/**
|
|
1942
|
-
*
|
|
1943
|
-
*
|
|
1944
|
-
* Explicit `mean`/`std` always win — they are the escape hatch for a model whose
|
|
1945
|
-
* preprocessing neither preset describes. Anything they leave open falls back to
|
|
1946
|
-
* the preset, so passing only a `mean` does not silently reset the deviation
|
|
1947
|
-
* to 1.
|
|
1896
|
+
* Name of the `(K, 1, cropH, cropW)` float32 mask output. Only on a
|
|
1897
|
+
* `detect_segment_classify` pipeline.
|
|
1948
1898
|
*
|
|
1949
|
-
*
|
|
1950
|
-
*
|
|
1951
|
-
*
|
|
1952
|
-
* which is exactly why it is worth saying out loud.
|
|
1953
|
-
*
|
|
1954
|
-
* @param metadata The model's custom metadata map, read to detect the family.
|
|
1955
|
-
* @param options The preset asked for, plus any explicit `mean`/`std`.
|
|
1956
|
-
* @throws {RangeError} If `normalization` is not a known preset, or names one
|
|
1957
|
-
* while `mean`/`std` are also supplied — two answers to the same question.
|
|
1899
|
+
* The mask lives in the **crop's** coordinate space, not the original image's,
|
|
1900
|
+
* because that is where the segmenter computed it. Values are 0 or 1 after
|
|
1901
|
+
* thresholding; padded rows are all-zero, like every other output.
|
|
1958
1902
|
*/
|
|
1959
|
-
export declare
|
|
1960
|
-
readonly normalization?: Normalization;
|
|
1961
|
-
readonly mean?: readonly [number, number, number];
|
|
1962
|
-
readonly std?: readonly [number, number, number];
|
|
1963
|
-
}): ResolvedNormalization;
|
|
1903
|
+
export declare const OUTPUT_MASKS = "masks";
|
|
1964
1904
|
|
|
1965
|
-
/**
|
|
1966
|
-
|
|
1967
|
-
*
|
|
1968
|
-
* @param requested Explicit provider list in preference order; `undefined` returns the default.
|
|
1969
|
-
*/
|
|
1970
|
-
export declare function resolveProviders(requested?: readonly string[]): string[];
|
|
1905
|
+
/** Name of the `[1]` int64 output holding how many of the `K` rows are real. */
|
|
1906
|
+
export declare const OUTPUT_NUM_DETECTIONS = "num_detections";
|
|
1971
1907
|
|
|
1972
|
-
|
|
1973
|
-
/**
|
|
1974
|
-
* Public output types returned by the SDK's vision tasks.
|
|
1975
|
-
*
|
|
1976
|
-
* These types form the contract between the SDK and its callers. They mirror
|
|
1977
|
-
* the Python `ort-vision-sdk` output dataclasses 1-to-1.
|
|
1978
|
-
*
|
|
1979
|
-
* Naming is intentionally compatible with the Ultralytics / torchvision idiom
|
|
1980
|
-
* (`cls`, `conf`, `box`, `xyxy`, `xywh`, normalized variants) so code ported
|
|
1981
|
-
* from those projects keeps working with minimal edits. The original verbose
|
|
1982
|
-
* names (`classId`, `className`, `confidence`, `bbox`) are still populated for
|
|
1983
|
-
* backwards compatibility.
|
|
1984
|
-
*/
|
|
1985
|
-
/**
|
|
1986
|
-
* HWC RGB uint8 image — the canonical image format used across the SDK.
|
|
1987
|
-
*
|
|
1988
|
-
* `data.length` must equal `width * height * 3`. The buffer is laid out row
|
|
1989
|
-
* by row, top-to-bottom, with each pixel as `[R, G, B]`.
|
|
1990
|
-
*/
|
|
1991
|
-
export declare class RGBImage {
|
|
1992
|
-
readonly data: Uint8Array;
|
|
1993
|
-
readonly width: number;
|
|
1994
|
-
readonly height: number;
|
|
1995
|
-
constructor(data: Uint8Array, width: number, height: number);
|
|
1996
|
-
}
|
|
1908
|
+
export declare const OUTPUT_PROBS = "probs";
|
|
1997
1909
|
|
|
1998
|
-
/**
|
|
1999
|
-
|
|
2000
|
-
*
|
|
2001
|
-
* Mirrors {@link DetectionResult} and adds the per-instance binary mask
|
|
2002
|
-
* plus a "ready-to-display" background-removed crop.
|
|
2003
|
-
*/
|
|
2004
|
-
export declare interface SegmentationResult {
|
|
2005
|
-
readonly classId: number;
|
|
2006
|
-
readonly className: string;
|
|
2007
|
-
readonly confidence: number;
|
|
2008
|
-
readonly bbox: BoundingBox;
|
|
2009
|
-
/** Alias for `classId` (Ultralytics-style). */
|
|
2010
|
-
readonly cls: number;
|
|
2011
|
-
/** Alias for `className`. */
|
|
2012
|
-
readonly name: string;
|
|
2013
|
-
/** Alias for `confidence` (Ultralytics-style). */
|
|
2014
|
-
readonly conf: number;
|
|
2015
|
-
/** Alias for `bbox` (Ultralytics-style). */
|
|
2016
|
-
readonly box: BoundingBox;
|
|
2017
|
-
/**
|
|
2018
|
-
* Binary mask cropped to `bbox`. Values are `0` (background) or `255`
|
|
2019
|
-
* (foreground). Empty boxes yield a zero-sized `Mask`.
|
|
2020
|
-
*/
|
|
2021
|
-
readonly mask: Mask;
|
|
2022
|
-
/**
|
|
2023
|
-
* The original image cropped to `bbox` with background pixels (where
|
|
2024
|
-
* `mask.data[i] === 0`) zeroed out. Empty boxes yield a zero-sized
|
|
2025
|
-
* `RGBImage`.
|
|
2026
|
-
*/
|
|
2027
|
-
readonly segmentedImage: RGBImage;
|
|
2028
|
-
}
|
|
1910
|
+
/** Name of the `[K]` float32 detection-confidence output. */
|
|
1911
|
+
export declare const OUTPUT_SCORES = "scores";
|
|
2029
1912
|
|
|
2030
1913
|
/**
|
|
2031
|
-
*
|
|
1914
|
+
* Parse a `repr`-encoded `dict[int, str]` class map.
|
|
1915
|
+
*
|
|
1916
|
+
* Split out of {@link modelNames} because the same encoding is reused by a
|
|
1917
|
+
* fused pipeline, which carries one class map per stage and therefore cannot
|
|
1918
|
+
* store both under the single `names` key Ultralytics uses.
|
|
2032
1919
|
*
|
|
2033
|
-
*
|
|
2034
|
-
*
|
|
1920
|
+
* @param encoded The encoded map — e.g. `"{0: 'deworm', 1: 'not_deworm'}"`.
|
|
1921
|
+
* @returns Class names in class-id order, or `null` when the value is missing,
|
|
1922
|
+
* unparseable, not a `dict`, or not keyed by contiguous integers from zero.
|
|
2035
1923
|
*/
|
|
2036
|
-
export declare
|
|
2037
|
-
readonly boxes: Boxes;
|
|
2038
|
-
readonly masks: Masks;
|
|
2039
|
-
readonly detections: readonly SegmentationResult[];
|
|
2040
|
-
readonly names: Readonly<Record<number, string>>;
|
|
2041
|
-
readonly origImg: RGBImage;
|
|
2042
|
-
readonly origShape: readonly [number, number];
|
|
2043
|
-
readonly path: string | null;
|
|
2044
|
-
readonly speed: Readonly<Speed>;
|
|
2045
|
-
constructor(boxes: Boxes, masks: Masks, detections: readonly SegmentationResult[], names: Readonly<Record<number, string>>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly<Speed>);
|
|
2046
|
-
/** Number of surviving instances. */
|
|
2047
|
-
get length(): number;
|
|
2048
|
-
/** Index into the per-instance results. */
|
|
2049
|
-
get(index: number): SegmentationResult | undefined;
|
|
2050
|
-
[Symbol.iterator](): Iterator<SegmentationResult>;
|
|
2051
|
-
}
|
|
1924
|
+
export declare function parseNames(encoded: string | undefined): readonly string[] | null;
|
|
2052
1925
|
|
|
2053
1926
|
/**
|
|
2054
|
-
*
|
|
2055
|
-
*
|
|
2056
|
-
* The model is expected to expose two outputs:
|
|
2057
|
-
*
|
|
2058
|
-
* 1. `output0`: `(1, 4 + numClasses + numMaskCoefs, numAnchors)` — per-anchor
|
|
2059
|
-
* predictions (boxes, class scores, mask coefficients).
|
|
2060
|
-
* 2. `output1`: `(1, numMaskCoefs, maskH, maskW)` — prototype masks.
|
|
1927
|
+
* Top-k classification probabilities for a single image.
|
|
2061
1928
|
*
|
|
2062
|
-
*
|
|
2063
|
-
* single image), mirroring Ultralytics' API. The envelope exposes:
|
|
2064
|
-
*
|
|
2065
|
-
* - `boxes`: bulk numpy view (`xyxy`, `xywh`, `xyxyn`, `xywhn`, `cls`, `conf`).
|
|
2066
|
-
* - `masks`: per-instance binary masks cropped to each box.
|
|
2067
|
-
* - per-instance {@link SegmentationResult} via iteration.
|
|
2068
|
-
*
|
|
2069
|
-
* @example
|
|
2070
|
-
* ```typescript
|
|
2071
|
-
* const seg = await Segmenter.create("/models/yolov8n-seg.onnx");
|
|
2072
|
-
* const r = (await seg.predict("/images/street.jpg"))[0];
|
|
2073
|
-
* for (const inst of r) {
|
|
2074
|
-
* console.log(inst.cls, inst.conf, inst.box.xyxy);
|
|
2075
|
-
* }
|
|
2076
|
-
* ```
|
|
1929
|
+
* Mirrors Ultralytics' `Probs` interface.
|
|
2077
1930
|
*/
|
|
2078
|
-
export declare class
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
1931
|
+
export declare class Probs {
|
|
1932
|
+
readonly data: Float32Array;
|
|
1933
|
+
/** @param data `[numClasses]` per-class probabilities, indexed by class id. */
|
|
1934
|
+
constructor(data: Float32Array);
|
|
1935
|
+
/** Number of classes. */
|
|
1936
|
+
get length(): number;
|
|
1937
|
+
/** `[numClasses]` shape of the underlying vector. */
|
|
1938
|
+
get shape(): readonly [number];
|
|
1939
|
+
/** Index of the most probable class. */
|
|
1940
|
+
get top1(): number;
|
|
1941
|
+
/** Probability of the top-1 class. */
|
|
1942
|
+
get top1conf(): number;
|
|
2090
1943
|
/**
|
|
2091
|
-
*
|
|
2092
|
-
*
|
|
2093
|
-
* The first inference of a session is not representative: WebGPU compiles its
|
|
2094
|
-
* shaders on it and the WASM backend faults in its arenas, which on a phone
|
|
2095
|
-
* can turn the first frame into seconds while every later frame is tens of
|
|
2096
|
-
* milliseconds. Calling this while a loading spinner is still up moves that
|
|
2097
|
-
* cost somewhere the user is already waiting.
|
|
1944
|
+
* Indices of the top-5 most probable classes, descending.
|
|
2098
1945
|
*
|
|
2099
|
-
*
|
|
2100
|
-
*
|
|
1946
|
+
* The array is the memoised selection itself, not a copy — reading it twice
|
|
1947
|
+
* hands back the same object. Treat it as read-only: writing into it edits
|
|
1948
|
+
* what every later read of this `Probs` returns.
|
|
2101
1949
|
*/
|
|
2102
|
-
|
|
1950
|
+
get top5(): Int32Array;
|
|
2103
1951
|
/**
|
|
2104
|
-
*
|
|
1952
|
+
* Probabilities of the top-5 classes, descending.
|
|
2105
1953
|
*
|
|
2106
|
-
*
|
|
2107
|
-
*
|
|
2108
|
-
* fails if it is actually asked to preprocess something.
|
|
1954
|
+
* Shares the memoised selection with {@link Probs.top5}, under the same
|
|
1955
|
+
* read-only caveat.
|
|
2109
1956
|
*/
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
static create(model: ModelSource, options?: SegmenterOptions): Promise<Segmenter>;
|
|
2113
|
-
/** The decoder family used to interpret the model's output. */
|
|
2114
|
-
get head(): SegmenterHead;
|
|
2115
|
-
/** Class labels indexed by class id. */
|
|
2116
|
-
get labels(): readonly string[];
|
|
2117
|
-
/** Class id → class name dict (matches Ultralytics' `model.names`). */
|
|
2118
|
-
get names(): Readonly<Record<number, string>>;
|
|
1957
|
+
get top5conf(): Float32Array;
|
|
1958
|
+
private _cache;
|
|
2119
1959
|
/**
|
|
2120
|
-
*
|
|
1960
|
+
* Memoised {@link Probs._topK}.
|
|
2121
1961
|
*
|
|
2122
|
-
*
|
|
2123
|
-
*
|
|
2124
|
-
*
|
|
2125
|
-
|
|
2126
|
-
get inputSize(): readonly [number, number];
|
|
2127
|
-
/** Number of classes the model predicts. */
|
|
2128
|
-
get numClasses(): number;
|
|
2129
|
-
/** Alias for {@link predict} (parity with PyTorch `nn.Module.__call__`). */
|
|
2130
|
-
call(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
|
|
2131
|
-
/** Run instance segmentation on a single image. */
|
|
2132
|
-
predict(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
|
|
2133
|
-
private _preprocess;
|
|
2134
|
-
private _splitOutputs;
|
|
2135
|
-
private _buildResult;
|
|
2136
|
-
private _buildBoxes;
|
|
2137
|
-
private _buildMasks;
|
|
2138
|
-
}
|
|
2139
|
-
|
|
2140
|
-
/**
|
|
2141
|
-
* Decoder family for the segmentation head.
|
|
2142
|
-
*
|
|
2143
|
-
* - `"yolo-seg"`: YOLO instance-segmentation head with two outputs —
|
|
2144
|
-
* `[1, 4 + nc + nm, N]` per-anchor predictions plus `[1, nm, mh, mw]`
|
|
2145
|
-
* prototype masks. Covers YOLOv8-seg, v11-seg, v26-seg.
|
|
2146
|
-
*
|
|
2147
|
-
* The SDK does **not** auto-detect this — the caller is responsible for
|
|
2148
|
-
* picking a head that matches their export.
|
|
2149
|
-
*/
|
|
2150
|
-
export declare type SegmenterHead = "yolo-seg";
|
|
2151
|
-
|
|
2152
|
-
export declare interface SegmenterOptions extends OrtSessionOptions {
|
|
2153
|
-
/**
|
|
2154
|
-
* Decoder family for the segmentation head. Default `"yolo-seg"` covers
|
|
2155
|
-
* YOLOv8-seg/v11-seg/v26-seg.
|
|
2156
|
-
*/
|
|
2157
|
-
readonly head?: SegmenterHead;
|
|
2158
|
-
/** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */
|
|
2159
|
-
readonly labels?: LabelSpec;
|
|
2160
|
-
/** Number of classes — used to validate the supplied labels. */
|
|
2161
|
-
readonly numClasses?: number;
|
|
2162
|
-
/**
|
|
2163
|
-
* Model input `[width, height]` in pixels for letterboxing.
|
|
1962
|
+
* `top5` and `top5conf` are separate getters over the same selection, so a
|
|
1963
|
+
* caller reading both would otherwise pay for it twice — once per frame, in
|
|
1964
|
+
* a camera loop. The probabilities a `Probs` was built from do not change,
|
|
1965
|
+
* so the result is computed once per `k` and kept.
|
|
2164
1966
|
*
|
|
2165
|
-
*
|
|
2166
|
-
*
|
|
2167
|
-
* Runtime will accept. Defaults to `[640, 640]`.
|
|
2168
|
-
*/
|
|
2169
|
-
readonly inputSize?: readonly [number, number];
|
|
2170
|
-
/** Default minimum class score to keep a candidate. */
|
|
2171
|
-
readonly confThreshold?: number;
|
|
2172
|
-
/** Default IoU threshold for non-maximum suppression. */
|
|
2173
|
-
readonly iouThreshold?: number;
|
|
2174
|
-
/** Maximum number of instances per image. */
|
|
2175
|
-
readonly maxDetections?: number;
|
|
2176
|
-
/**
|
|
2177
|
-
* If `true`, a run that finds nothing throws {@link NoDetectionsError}
|
|
2178
|
-
* instead of returning an empty envelope. Default `false`, because looking
|
|
2179
|
-
* and finding nothing is a successful inference. Turn it on when an empty
|
|
2180
|
-
* result means the surrounding pipeline should stop rather than carry on with
|
|
2181
|
-
* zero rows. Can be overridden per `predict` call.
|
|
2182
|
-
*/
|
|
2183
|
-
readonly raiseOnEmpty?: boolean;
|
|
2184
|
-
/** Probability cutoff applied to soft masks. Defaults to `0.5`. */
|
|
2185
|
-
readonly maskThreshold?: number;
|
|
2186
|
-
}
|
|
2187
|
-
|
|
2188
|
-
export declare interface SegmenterPredictOptions {
|
|
2189
|
-
readonly confThreshold?: number;
|
|
2190
|
-
readonly iouThreshold?: number;
|
|
2191
|
-
/**
|
|
2192
|
-
* If set, keep only instances whose `classId` is in this list.
|
|
2193
|
-
* Mirrors Ultralytics' `model.predict(img, classes=[0, 16])`.
|
|
1967
|
+
* @param k - How many classes to select.
|
|
1968
|
+
* @returns The cached selection for that `k`.
|
|
2194
1969
|
*/
|
|
2195
|
-
|
|
2196
|
-
/** Override the constructor's `raiseOnEmpty` setting for this call. */
|
|
2197
|
-
readonly raiseOnEmpty?: boolean;
|
|
2198
|
-
}
|
|
2199
|
-
|
|
2200
|
-
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
2201
|
-
/**
|
|
2202
|
-
* Classification head postprocessing — softmax + top-k.
|
|
2203
|
-
*/
|
|
2204
|
-
/** Apply numerically-stable softmax to a 1-D vector of logits. */
|
|
2205
|
-
export declare function softmax(logits: Float32Array | readonly number[]): Float32Array;
|
|
2206
|
-
|
|
2207
|
-
/**
|
|
2208
|
-
* Read the spatial input size out of a declared NCHW shape.
|
|
2209
|
-
*
|
|
2210
|
-
* @param shape The declared shape of the model's image input.
|
|
2211
|
-
* @returns `[width, height]` in pixels, or `null` when the shape is not 4D or
|
|
2212
|
-
* leaves either spatial axis dynamic — in which case the model accepts more
|
|
2213
|
-
* than one resolution and there is nothing to correct.
|
|
2214
|
-
*/
|
|
2215
|
-
export declare function spatialInputSize(shape: DeclaredShape): readonly [number, number] | null;
|
|
2216
|
-
|
|
2217
|
-
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
2218
|
-
/**
|
|
2219
|
-
* Per-stage timing for a single `predict()` call.
|
|
2220
|
-
*
|
|
2221
|
-
* Populates the `speed` field every `Results` envelope carries, mirroring
|
|
2222
|
-
* Ultralytics' `results[0].speed`. All values are milliseconds measured with
|
|
2223
|
-
* `performance.now()`.
|
|
2224
|
-
*/
|
|
2225
|
-
/**
|
|
2226
|
-
* Stage durations of one inference, in milliseconds.
|
|
2227
|
-
*
|
|
2228
|
-
* `preprocess`, `inference` and `postprocess` are the three keys Ultralytics
|
|
2229
|
-
* reports, measured over the same boundaries. `load` is specific to this SDK:
|
|
2230
|
-
* `predict()` accepts a URL, `Blob` or DOM element and decodes it internally,
|
|
2231
|
-
* so the fetch/decode cost would otherwise be invisible — and on a cold cache
|
|
2232
|
-
* it dominates everything else.
|
|
2233
|
-
*/
|
|
2234
|
-
export declare interface Speed {
|
|
2235
|
-
/** Fetching and decoding the input into an `RGBImage`. */
|
|
2236
|
-
load: number;
|
|
2237
|
-
/** Letterbox/resize, normalization and tensor packing. */
|
|
2238
|
-
preprocess: number;
|
|
2239
|
-
/** The ONNX Runtime forward pass. */
|
|
2240
|
-
inference: number;
|
|
2241
|
-
/** Decoding raw outputs into results (NMS, mask assembly, top-k). */
|
|
2242
|
-
postprocess: number;
|
|
2243
|
-
}
|
|
2244
|
-
|
|
2245
|
-
/**
|
|
2246
|
-
* Accumulate stage durations while a `predict()` call runs.
|
|
2247
|
-
*
|
|
2248
|
-
* Each `stage()` call closes the previous stage: the elapsed time since the
|
|
2249
|
-
* last boundary is attributed to the name given. This keeps the call sites
|
|
2250
|
-
* free of paired start/stop bookkeeping and guarantees the four stages tile
|
|
2251
|
-
* the whole call without gaps.
|
|
2252
|
-
*/
|
|
2253
|
-
export declare class SpeedTimer {
|
|
2254
|
-
private _last;
|
|
2255
|
-
private readonly _speed;
|
|
2256
|
-
constructor();
|
|
1970
|
+
private _top;
|
|
2257
1971
|
/**
|
|
2258
|
-
*
|
|
1972
|
+
* Select the `k` highest probabilities without ordering the rest.
|
|
2259
1973
|
*
|
|
2260
|
-
*
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
*
|
|
1974
|
+
* A full sort to read five entries out of a thousand-class vector costs
|
|
1975
|
+
* O(n log n) plus an index array the size of the vector; keeping `k` slots
|
|
1976
|
+
* ordered by insertion and scanning once costs O(n·k) with no allocation
|
|
1977
|
+
* beyond the result. Measured on 1000 classes over 2000 iterations, 134.5 µs
|
|
1978
|
+
* against 1.5 µs for the same output.
|
|
2265
1979
|
*
|
|
2266
|
-
*
|
|
1980
|
+
* Ties keep the lower class index first, matching the stable sort this
|
|
1981
|
+
* replaced: a candidate only displaces an entry it is strictly greater
|
|
1982
|
+
* than. That is also what keeps this selection in step with the Python
|
|
1983
|
+
* SDK's `np.argsort(-data, kind="stable")`, where the sort is C and the
|
|
1984
|
+
* cost this avoids does not arise.
|
|
1985
|
+
*
|
|
1986
|
+
* @param k - How many classes to select.
|
|
1987
|
+
* @returns Indices and probabilities, descending by probability.
|
|
2267
1988
|
*/
|
|
2268
|
-
|
|
1989
|
+
private _topK;
|
|
2269
1990
|
}
|
|
2270
1991
|
|
|
2271
|
-
/**
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
* @param hwc Source array of length `width * height * channels`.
|
|
2275
|
-
*/
|
|
2276
|
-
export declare function toCHW(hwc: Float32Array, width: number, height: number, channels?: number): Float32Array;
|
|
1992
|
+
/** Raised when a requested execution provider is not available. */
|
|
1993
|
+
export declare class ProviderNotAvailableError extends OrtVisionError {
|
|
1994
|
+
}
|
|
2277
1995
|
|
|
2278
1996
|
/**
|
|
2279
|
-
*
|
|
1997
|
+
* One execution provider: its name, or ORT's config object for it.
|
|
2280
1998
|
*
|
|
2281
|
-
*
|
|
1999
|
+
* The object form carries provider options straight through to
|
|
2000
|
+
* `InferenceSession.create` — `{ name: "webgpu", preferredLayout: "NHWC" }`,
|
|
2001
|
+
* `{ name: "wasm" }` — which a bare name cannot express.
|
|
2282
2002
|
*/
|
|
2283
|
-
export declare
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
/** Wrap a Float32 buffer into an `ort.Tensor`. */
|
|
2289
|
-
export declare function toFloat32Tensor(data: Float32Array, dims: readonly number[]): ort.Tensor;
|
|
2003
|
+
export declare type ProviderSpec = string | {
|
|
2004
|
+
readonly name: string;
|
|
2005
|
+
readonly [option: string]: unknown;
|
|
2006
|
+
};
|
|
2290
2007
|
|
|
2291
2008
|
/**
|
|
2292
|
-
*
|
|
2009
|
+
* Metadata key the Python SDK's `quantize_model` writes on an INT8 model.
|
|
2293
2010
|
*
|
|
2294
|
-
*
|
|
2011
|
+
* Mirrored by `QUANTIZATION_KEY` in the Python SDK; both test suites pin the
|
|
2012
|
+
* string. See {@link keepQuantizedOffWebGpu} for what a session does with it.
|
|
2295
2013
|
*/
|
|
2296
|
-
export declare
|
|
2297
|
-
|
|
2298
|
-
export declare interface TopKResult {
|
|
2299
|
-
readonly indices: Int32Array;
|
|
2300
|
-
readonly values: Float32Array;
|
|
2301
|
-
}
|
|
2014
|
+
export declare const QUANTIZATION_KEY = "ort_vision_sdk.quantization";
|
|
2302
2015
|
|
|
2303
2016
|
/**
|
|
2304
|
-
*
|
|
2017
|
+
* Read a pipeline spec out of a model's custom metadata.
|
|
2305
2018
|
*
|
|
2306
|
-
*
|
|
2307
|
-
*
|
|
2308
|
-
*
|
|
2019
|
+
* Individual malformed entries fall back to the value a fusion would have used
|
|
2020
|
+
* by default — a single bad float is not a reason to reject an otherwise
|
|
2021
|
+
* loadable pipeline. A malformed resolution is fatal, because there is no safe
|
|
2022
|
+
* default for one.
|
|
2309
2023
|
*
|
|
2310
|
-
* @
|
|
2024
|
+
* @param metadata A model's custom metadata map, as read by
|
|
2025
|
+
* {@link readModelMetadata}.
|
|
2026
|
+
* @returns The decoded spec, or `null` when the model is not a fused pipeline —
|
|
2027
|
+
* it carries no `ovs.kind` entry, or one naming a pipeline kind this version
|
|
2028
|
+
* does not know how to drive.
|
|
2311
2029
|
*/
|
|
2312
|
-
export declare function
|
|
2030
|
+
export declare function readFusionSpec(metadata: Readonly<Record<string, string>> | undefined): FusionSpec | null;
|
|
2313
2031
|
|
|
2314
2032
|
/**
|
|
2315
|
-
*
|
|
2316
|
-
* and expose status/error so the page can render permission and error states.
|
|
2317
|
-
* The stream is automatically released on unmount or retry.
|
|
2318
|
-
*
|
|
2319
|
-
* Defaults to the rear ("environment") camera; desktops fall back to whatever
|
|
2320
|
-
* single camera they expose. Pass `options.constraints` to override, or
|
|
2321
|
-
* `options.enabled: false` to render the surface without prompting for the camera
|
|
2322
|
-
* at all.
|
|
2033
|
+
* Read the element type each graph input declares.
|
|
2323
2034
|
*
|
|
2324
|
-
*
|
|
2325
|
-
*
|
|
2326
|
-
*
|
|
2327
|
-
*
|
|
2328
|
-
*
|
|
2329
|
-
* - `video.play()` rejections are swallowed: autoplay may be blocked, but the
|
|
2330
|
-
* user gesture that opened the camera usually counts and playback resumes on
|
|
2331
|
-
* the next interaction.
|
|
2035
|
+
* `onnxruntime-web` does not expose this. `session.inputMetadata` is
|
|
2036
|
+
* `undefined` on 1.20.1 — the same version where `declaredShapesFrom` comes
|
|
2037
|
+
* back empty — so the declared types have to come from the file, which the SDK
|
|
2038
|
+
* already downloads and walks for {@link readModelMetadata}. Same bytes, one
|
|
2039
|
+
* more pass, no extra request.
|
|
2332
2040
|
*
|
|
2333
|
-
* @param
|
|
2334
|
-
* @returns
|
|
2335
|
-
*
|
|
2336
|
-
*
|
|
2041
|
+
* @param model The `.onnx` file contents.
|
|
2042
|
+
* @returns Input name → `TensorProto.DataType` value. An empty object when the
|
|
2043
|
+
* file carries no readable graph, which every caller treats as "assume
|
|
2044
|
+
* float32" — the behaviour before any of this existed.
|
|
2337
2045
|
*/
|
|
2338
|
-
export declare function
|
|
2339
|
-
|
|
2340
|
-
/** Value returned by {@link useCameraStream}. */
|
|
2341
|
-
export declare interface UseCameraStreamApi {
|
|
2342
|
-
/** Current lifecycle status. */
|
|
2343
|
-
status: CameraStreamStatus;
|
|
2344
|
-
/** The classified error, or `null` while not in the `error` status. */
|
|
2345
|
-
error: CameraStreamError | null;
|
|
2346
|
-
/** Attach to a `<video ref={…} />`. The stream is wired to it once ready. */
|
|
2347
|
-
videoRef: RefObject<HTMLVideoElement | null>;
|
|
2348
|
-
/**
|
|
2349
|
-
* The live stream, or `null`.
|
|
2350
|
-
*
|
|
2351
|
-
* Exposed for the things that need the **track** rather than the picture — the LED
|
|
2352
|
-
* torch (`useTorch`), the real frame size from `getSettings()`, recording it with
|
|
2353
|
-
* `useVideoRecorder`. Do not stop it yourself: the hook owns its lifetime and
|
|
2354
|
-
* releases it on unmount and on `retry()`.
|
|
2355
|
-
*/
|
|
2356
|
-
stream: MediaStream | null;
|
|
2357
|
-
/** Manually re-attempt after an error (e.g. the user changed permissions). */
|
|
2358
|
-
retry: () => void;
|
|
2359
|
-
}
|
|
2360
|
-
|
|
2361
|
-
/** Options for {@link useCameraStream}. */
|
|
2362
|
-
export declare interface UseCameraStreamOptions {
|
|
2363
|
-
/**
|
|
2364
|
-
* Constraints passed to `getUserMedia`. Defaults to the rear
|
|
2365
|
-
* ("environment") camera at Full-HD ideal resolution with audio off.
|
|
2366
|
-
* Read when the stream (re)starts — change it and call `retry()` to apply.
|
|
2367
|
-
*/
|
|
2368
|
-
constraints?: MediaStreamConstraints;
|
|
2369
|
-
/**
|
|
2370
|
-
* Hold off acquiring the camera until this is `true`. Default `true`.
|
|
2371
|
-
*
|
|
2372
|
-
* The point is to be able to *not* prompt. A permission prompt costs the user a
|
|
2373
|
-
* decision and, if they refuse, costs the app the feature permanently — so a
|
|
2374
|
-
* surface that already knows it cannot do its job (a barcode scanner in a browser
|
|
2375
|
-
* with no decoder) must not open the camera to then say so. Flipping this to `false`
|
|
2376
|
-
* also releases a stream that is already live.
|
|
2377
|
-
*/
|
|
2378
|
-
enabled?: boolean;
|
|
2379
|
-
}
|
|
2046
|
+
export declare function readModelInputTypes(model: Uint8Array | ArrayBufferLike): Readonly<Record<string, number>>;
|
|
2380
2047
|
|
|
2048
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
2381
2049
|
/**
|
|
2382
|
-
*
|
|
2383
|
-
* loop and expose the rolling value. Sampling is throttled by `intervalMs` and
|
|
2384
|
-
* paused whenever `enabled` is `false` or the video is not ready yet
|
|
2385
|
-
* (`readyState < 2` or `videoWidth === 0`).
|
|
2050
|
+
* Read the metadata an exporter baked into a `.onnx` file.
|
|
2386
2051
|
*
|
|
2387
|
-
*
|
|
2388
|
-
*
|
|
2052
|
+
* `onnxruntime-web` exposes input/output metadata but **not** the model's
|
|
2053
|
+
* custom metadata map, which is where Ultralytics writes `names`, `task` and
|
|
2054
|
+
* `imgsz`. The Python SDK gets it for free from
|
|
2055
|
+
* `InferenceSession.get_modelmeta().custom_metadata_map`; in the browser the
|
|
2056
|
+
* only way to the same information is to read it out of the file, so this
|
|
2057
|
+
* module walks just enough of the ModelProto wire format to collect
|
|
2058
|
+
* `metadata_props`.
|
|
2389
2059
|
*
|
|
2390
|
-
*
|
|
2391
|
-
*
|
|
2392
|
-
*
|
|
2060
|
+
* It never throws and never allocates unbounded: a truncated, hostile or
|
|
2061
|
+
* simply unexpected file yields an empty map, and every caller treats that as
|
|
2062
|
+
* "the model says nothing", falling back to what it was given.
|
|
2393
2063
|
*/
|
|
2394
|
-
export declare function useLiveLuminance(videoRef: RefObject<HTMLVideoElement | null>, { enabled, intervalMs }?: UseLiveLuminanceOptions): number;
|
|
2395
|
-
|
|
2396
|
-
/** Options for {@link useLiveLuminance}. */
|
|
2397
|
-
export declare interface UseLiveLuminanceOptions {
|
|
2398
|
-
/** When `false` the loop is paused (e.g. while a capture is in flight). Default: `true`. */
|
|
2399
|
-
enabled?: boolean;
|
|
2400
|
-
/** Throttle measurements in milliseconds. Default: `160` (~6 fps), plenty for UX. */
|
|
2401
|
-
intervalMs?: number;
|
|
2402
|
-
}
|
|
2403
|
-
|
|
2404
|
-
export declare const VERSION: string;
|
|
2405
|
-
|
|
2406
|
-
export declare abstract class VisionTask {
|
|
2407
|
-
protected readonly _session: OrtSession;
|
|
2408
|
-
protected constructor(_session: OrtSession);
|
|
2409
|
-
/** The underlying {@link OrtSession} used to run inference. */
|
|
2410
|
-
get session(): OrtSession;
|
|
2411
|
-
}
|
|
2412
|
-
|
|
2413
2064
|
/**
|
|
2414
|
-
*
|
|
2415
|
-
*
|
|
2416
|
-
* Shared by {@link ResizePipeline} and exported for a custom pipeline that
|
|
2417
|
-
* already holds pixels and wants the SDK's exact arithmetic.
|
|
2065
|
+
* Collect a model's custom metadata map straight out of its bytes.
|
|
2418
2066
|
*
|
|
2419
|
-
*
|
|
2420
|
-
*
|
|
2421
|
-
*
|
|
2422
|
-
*
|
|
2423
|
-
*
|
|
2424
|
-
* @param rgba Source pixels, 4 bytes per pixel, `width * height` long.
|
|
2425
|
-
* @param width Image width in pixels.
|
|
2426
|
-
* @param height Image height in pixels.
|
|
2427
|
-
* @param mean Per-channel RGB mean, already in `[0, 1]`.
|
|
2428
|
-
* @param std Per-channel RGB standard deviation.
|
|
2429
|
-
* @param out Destination buffer, `3 * width * height` long.
|
|
2430
|
-
* @param stride Bytes per source pixel: 4 for canvas RGBA, 3 for packed RGB.
|
|
2067
|
+
* @param model The `.onnx` file contents.
|
|
2068
|
+
* @returns Key/value metadata — `names`, `task`, `imgsz`, ... for an
|
|
2069
|
+
* Ultralytics export — or an empty object when the file carries none or
|
|
2070
|
+
* cannot be walked.
|
|
2431
2071
|
*/
|
|
2432
|
-
export declare function
|
|
2072
|
+
export declare function readModelMetadata(model: Uint8Array | ArrayBufferLike): Readonly<Record<string, string>>;
|
|
2433
2073
|
|
|
2434
2074
|
/**
|
|
2435
|
-
*
|
|
2075
|
+
* Turn an empty result into an error, when the caller asked for that.
|
|
2436
2076
|
*
|
|
2437
|
-
*
|
|
2438
|
-
* @
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2077
|
+
* Shared by every task that can come back with nothing — {@link Detector},
|
|
2078
|
+
* {@link Segmenter} and {@link DetectClassify} — so the three agree on when
|
|
2079
|
+
* they throw and on what the message says. The message names the two settings
|
|
2080
|
+
* that decide the outcome, because "no detections" on its own leaves the reader
|
|
2081
|
+
* unable to tell a blank image from a threshold set too high.
|
|
2082
|
+
*
|
|
2083
|
+
* @param count How many detections survived every filter.
|
|
2084
|
+
* @param options The flag for this call, the threshold actually applied (after
|
|
2085
|
+
* any per-call override), the class allowlist if one narrowed the search, and
|
|
2086
|
+
* the source path when the input was one.
|
|
2087
|
+
* @throws {@link NoDetectionsError} when the flag is set and `count` is zero.
|
|
2088
|
+
*/
|
|
2089
|
+
export declare function requireDetections(count: number, options: {
|
|
2090
|
+
readonly raiseOnEmpty: boolean;
|
|
2091
|
+
readonly confThreshold: number;
|
|
2092
|
+
readonly classes: readonly number[] | undefined;
|
|
2093
|
+
readonly path: string | null;
|
|
2094
|
+
}): void;
|
|
2095
|
+
|
|
2096
|
+
/** Resize an image to `(targetWidth, targetHeight)` using high-quality canvas resampling. */
|
|
2097
|
+
export declare function resize(image: RGBImage, targetWidth: number, targetHeight: number): RGBImage;
|
|
2098
|
+
|
|
2099
|
+
/**
|
|
2100
|
+
* Reusable stretch-resize → normalized tensor pipeline for one target size.
|
|
2101
|
+
*
|
|
2102
|
+
* The classification counterpart of {@link LetterboxPipeline}. A classifier
|
|
2103
|
+
* stretches to the model's square input instead of letterboxing into it — no
|
|
2104
|
+
* padding, no scale to invert later, because nothing is mapped back onto the
|
|
2105
|
+
* source image afterwards. That difference is why it cannot simply reuse the
|
|
2106
|
+
* letterbox path.
|
|
2107
|
+
*
|
|
2108
|
+
* What it does share is the technique. The composable route
|
|
2109
|
+
* (`resize` → `normalize` → `toCHW`) allocates an `RGBImage` and two
|
|
2110
|
+
* `Float32Array`s and walks each end to end on every call: about 1.4 MB of
|
|
2111
|
+
* fresh garbage per 224×224 `predict()`, produced at the exact moment a phone
|
|
2112
|
+
* near its memory ceiling can least afford it. Here one `drawImage` resizes,
|
|
2113
|
+
* and one loop reads the resulting RGBA and writes normalized planar float32
|
|
2114
|
+
* into a buffer held across calls.
|
|
2115
|
+
*
|
|
2116
|
+
* Create one per task, not per frame.
|
|
2117
|
+
*/
|
|
2118
|
+
export declare class ResizePipeline {
|
|
2119
|
+
private readonly _targetWidth;
|
|
2120
|
+
private readonly _targetHeight;
|
|
2121
|
+
private readonly _mean;
|
|
2122
|
+
private readonly _std;
|
|
2123
|
+
private readonly _buffer;
|
|
2124
|
+
private _target;
|
|
2125
|
+
private _targetContext;
|
|
2126
|
+
private _source;
|
|
2127
|
+
private _sourceContext;
|
|
2128
|
+
/**
|
|
2129
|
+
* @param targetWidth Model input width in pixels.
|
|
2130
|
+
* @param targetHeight Model input height in pixels.
|
|
2131
|
+
* @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.
|
|
2132
|
+
* @param std Per-channel RGB standard deviation. Defaults to no scaling.
|
|
2133
|
+
*/
|
|
2134
|
+
constructor(targetWidth: number, targetHeight: number, mean?: readonly [number, number, number], std?: readonly [number, number, number]);
|
|
2135
|
+
/** The `[width, height]` this pipeline resizes into. */
|
|
2136
|
+
get targetSize(): readonly [number, number];
|
|
2137
|
+
/**
|
|
2138
|
+
* Resize an image to the target size and write it as normalized planar float32.
|
|
2139
|
+
*
|
|
2140
|
+
* An image that already arrives at the target size skips the canvas entirely
|
|
2141
|
+
* and is read straight out of its packed RGB — which is both faster and what
|
|
2142
|
+
* keeps the result identical to `resize()`, whose own fast path returns the
|
|
2143
|
+
* input untouched.
|
|
2144
|
+
*
|
|
2145
|
+
* Buffer reuse follows {@link ReusableBuffer}: held across calls, replaced when
|
|
2146
|
+
* a consumer detached it by transferring the tensor.
|
|
2147
|
+
*
|
|
2148
|
+
* @param image Source image in the SDK's canonical HWC RGB layout.
|
|
2149
|
+
* @param canvas Optional opaque drawable holding exactly `image`'s pixels —
|
|
2150
|
+
* see {@link LetterboxPipeline.run}. Ignored on the same-size fast path,
|
|
2151
|
+
* which never touches a canvas.
|
|
2152
|
+
*/
|
|
2153
|
+
run(image: RGBImage, canvas?: CanvasImageSource | null): FusedResizeResult;
|
|
2154
|
+
/**
|
|
2155
|
+
* Mark the reusable buffer free again.
|
|
2156
|
+
*
|
|
2157
|
+
* Call it once the tensor built from a {@link run} result has been handed to
|
|
2158
|
+
* ONNX Runtime and the run has resolved — after that the values are inside
|
|
2159
|
+
* the WASM heap and the buffer can be overwritten.
|
|
2160
|
+
*
|
|
2161
|
+
* Pass the result being released when several runs can be outstanding at
|
|
2162
|
+
* once. A result that got a fresh allocation (`reused: false`) holds nothing
|
|
2163
|
+
* of the pipeline's, so releasing it is a no-op; without that check, the
|
|
2164
|
+
* caller finishing with a fresh buffer would free the shared one while
|
|
2165
|
+
* another caller still had its frame in it. Called without an argument it
|
|
2166
|
+
* frees the shared buffer unconditionally, as before.
|
|
2167
|
+
*
|
|
2168
|
+
* @param result The {@link run} result being released.
|
|
2169
|
+
*/
|
|
2170
|
+
release(result?: {
|
|
2171
|
+
readonly reused: boolean;
|
|
2172
|
+
}): void;
|
|
2173
|
+
/**
|
|
2174
|
+
* Build the target canvas on first use.
|
|
2175
|
+
*
|
|
2176
|
+
* Lazily, so a pipeline constructed where no canvas implementation exists
|
|
2177
|
+
* (a Node test, a worker without OffscreenCanvas) only fails if it is asked
|
|
2178
|
+
* to resize something.
|
|
2179
|
+
*/
|
|
2180
|
+
private _ensureTarget;
|
|
2181
|
+
/**
|
|
2182
|
+
* Paint an image onto the scratch source canvas and return that canvas.
|
|
2183
|
+
*
|
|
2184
|
+
* @param image Source image in the SDK's canonical HWC RGB layout.
|
|
2185
|
+
*/
|
|
2186
|
+
private _paintSource;
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
/**
|
|
2190
|
+
* Resize an image into normalized planar float32 without keeping any state.
|
|
2191
|
+
*
|
|
2192
|
+
* The allocation-free path is {@link ResizePipeline}; this is the one-shot
|
|
2193
|
+
* form, for a caller who wants the fused behaviour without owning a pipeline.
|
|
2194
|
+
*
|
|
2195
|
+
* @param image Source image in the SDK's canonical HWC RGB layout.
|
|
2196
|
+
* @param targetWidth Model input width in pixels.
|
|
2197
|
+
* @param targetHeight Model input height in pixels.
|
|
2198
|
+
* @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.
|
|
2199
|
+
* @param std Per-channel RGB standard deviation. Defaults to no scaling.
|
|
2200
|
+
*/
|
|
2201
|
+
export declare function resizeToTensorData(image: RGBImage, targetWidth: number, targetHeight: number, mean?: readonly [number, number, number], std?: readonly [number, number, number]): FusedResizeResult;
|
|
2202
|
+
|
|
2203
|
+
/** What {@link resolveNormalization} settled on. */
|
|
2204
|
+
export declare interface ResolvedNormalization {
|
|
2205
|
+
/** The preset name, or `"custom"` when the caller supplied the values. */
|
|
2206
|
+
readonly name: string;
|
|
2207
|
+
/** Per-channel mean to subtract. */
|
|
2208
|
+
readonly mean: readonly [number, number, number];
|
|
2209
|
+
/** Per-channel deviation to divide by. */
|
|
2210
|
+
readonly std: readonly [number, number, number];
|
|
2211
|
+
}
|
|
2212
|
+
|
|
2213
|
+
/**
|
|
2214
|
+
* Decide the input size a task will preprocess to.
|
|
2215
|
+
*
|
|
2216
|
+
* Precedence is graph → caller → fallback. The graph wins over an explicit
|
|
2217
|
+
* `inputSize` because a static shape is not a preference, it is what ORT will
|
|
2218
|
+
* accept: honoring the caller there would only turn a fixable mismatch into a
|
|
2219
|
+
* failed run. A disagreement is a configuration bug in the caller, so it is
|
|
2220
|
+
* reported through `console.warn` instead of being swallowed.
|
|
2221
|
+
*
|
|
2222
|
+
* @param options Graph shape, requested size and per-task fallback.
|
|
2223
|
+
* @returns The `[width, height]` to preprocess to.
|
|
2224
|
+
*/
|
|
2225
|
+
export declare function resolveInputSize(options: ResolveInputSizeOptions): readonly [number, number];
|
|
2226
|
+
|
|
2227
|
+
export declare interface ResolveInputSizeOptions {
|
|
2228
|
+
/** Declared shape of the model's image input, from {@link declaredShapesFrom}. */
|
|
2229
|
+
readonly graphShape?: DeclaredShape;
|
|
2230
|
+
/** Size the caller asked for, if any. */
|
|
2231
|
+
readonly requested?: readonly [number, number];
|
|
2232
|
+
/** Size to use when neither the graph nor the caller pins one. */
|
|
2233
|
+
readonly fallback: readonly [number, number];
|
|
2234
|
+
}
|
|
2235
|
+
|
|
2236
|
+
export declare function resolveLabels(spec: LabelSpec, options?: ResolveLabelsOptions): readonly string[];
|
|
2237
|
+
|
|
2238
|
+
export declare interface ResolveLabelsOptions {
|
|
2239
|
+
/**
|
|
2240
|
+
* Expected number of classes.
|
|
2241
|
+
*
|
|
2242
|
+
* - When `spec` is `null`/`undefined`, this is required to auto-generate names.
|
|
2243
|
+
* - When `spec` is provided, it validates that the resolved length matches.
|
|
2244
|
+
*/
|
|
2245
|
+
readonly numClasses?: number;
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2248
|
+
/**
|
|
2249
|
+
* Settle which `mean`/`std` to apply, and what to call the choice.
|
|
2250
|
+
*
|
|
2251
|
+
* Explicit `mean`/`std` always win — they are the escape hatch for a model whose
|
|
2252
|
+
* preprocessing neither preset describes. Anything they leave open falls back to
|
|
2253
|
+
* the preset, so passing only a `mean` does not silently reset the deviation
|
|
2254
|
+
* to 1.
|
|
2255
|
+
*
|
|
2256
|
+
* Warns (via `console.warn`) when the model is an Ultralytics export and the
|
|
2257
|
+
* supplied values are not the identity it was trained with. Nothing fails in
|
|
2258
|
+
* that case: the prediction has the right shape and is simply less accurate,
|
|
2259
|
+
* which is exactly why it is worth saying out loud.
|
|
2260
|
+
*
|
|
2261
|
+
* @param metadata The model's custom metadata map, read to detect the family.
|
|
2262
|
+
* @param options The preset asked for, plus any explicit `mean`/`std`.
|
|
2263
|
+
* @throws {RangeError} If `normalization` is not a known preset, or names one
|
|
2264
|
+
* while `mean`/`std` are also supplied — two answers to the same question.
|
|
2265
|
+
*/
|
|
2266
|
+
export declare function resolveNormalization(metadata: Readonly<Record<string, string>>, options?: {
|
|
2267
|
+
readonly normalization?: Normalization;
|
|
2268
|
+
readonly mean?: readonly [number, number, number];
|
|
2269
|
+
readonly std?: readonly [number, number, number];
|
|
2270
|
+
}): ResolvedNormalization;
|
|
2271
|
+
|
|
2272
|
+
/**
|
|
2273
|
+
* Resolve the execution providers to pass to `InferenceSession.create`.
|
|
2274
|
+
*
|
|
2275
|
+
* @param requested Explicit provider list in preference order, as names or
|
|
2276
|
+
* config objects; `undefined` or empty returns the default.
|
|
2277
|
+
*/
|
|
2278
|
+
export declare function resolveProviders<T extends ProviderSpec = string>(requested?: readonly T[]): (T | string)[];
|
|
2279
|
+
|
|
2280
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
2281
|
+
/**
|
|
2282
|
+
* Public output types returned by the SDK's vision tasks.
|
|
2283
|
+
*
|
|
2284
|
+
* These types form the contract between the SDK and its callers. They mirror
|
|
2285
|
+
* the Python `ort-vision-sdk` output dataclasses 1-to-1.
|
|
2286
|
+
*
|
|
2287
|
+
* Naming is intentionally compatible with the Ultralytics / torchvision idiom
|
|
2288
|
+
* (`cls`, `conf`, `box`, `xyxy`, `xywh`, normalized variants) so code ported
|
|
2289
|
+
* from those projects keeps working with minimal edits. The original verbose
|
|
2290
|
+
* names (`classId`, `className`, `confidence`, `bbox`) are still populated for
|
|
2291
|
+
* backwards compatibility.
|
|
2292
|
+
*/
|
|
2293
|
+
/**
|
|
2294
|
+
* HWC RGB uint8 image — the canonical image format used across the SDK.
|
|
2295
|
+
*
|
|
2296
|
+
* `data.length` must equal `width * height * 3`. The buffer is laid out row
|
|
2297
|
+
* by row, top-to-bottom, with each pixel as `[R, G, B]`.
|
|
2298
|
+
*
|
|
2299
|
+
* The pixels can also be produced on first read — see {@link RGBImage.deferred}.
|
|
2300
|
+
* Reading `data` is the same either way.
|
|
2301
|
+
*/
|
|
2302
|
+
export declare class RGBImage {
|
|
2303
|
+
readonly width: number;
|
|
2304
|
+
readonly height: number;
|
|
2305
|
+
private _data;
|
|
2306
|
+
private _materialize;
|
|
2307
|
+
constructor(data: Uint8Array, width: number, height: number);
|
|
2308
|
+
/**
|
|
2309
|
+
* An image whose pixels are computed the first time `data` is read.
|
|
2310
|
+
*
|
|
2311
|
+
* Used by the decoder for frames it can hand to preprocessing without
|
|
2312
|
+
* reading back: a 1080p `getImageData` plus the RGBA→RGB copy is most of
|
|
2313
|
+
* the `load` stage (9.4 ms vs 1.9 ms for the GPU-side copy, measured in
|
|
2314
|
+
* Chromium), and a camera loop that only reads boxes never needs it.
|
|
2315
|
+
*
|
|
2316
|
+
* @param width Width in pixels.
|
|
2317
|
+
* @param height Height in pixels.
|
|
2318
|
+
* @param materialize Produces the `width * height * 3` buffer; called at most
|
|
2319
|
+
* once, and must keep returning the pixels of the moment the image was
|
|
2320
|
+
* made — snapshot whatever it reads from.
|
|
2321
|
+
* @returns The image. Built without the constructor, which would need the
|
|
2322
|
+
* very buffer this defers.
|
|
2323
|
+
*/
|
|
2324
|
+
static deferred(width: number, height: number, materialize: () => Uint8Array): RGBImage;
|
|
2325
|
+
/**
|
|
2326
|
+
* The HWC RGB pixels, materialized on first read for a deferred image.
|
|
2327
|
+
*
|
|
2328
|
+
* @throws {@link ImageLoadError} if a deferred image's buffer has the wrong length.
|
|
2329
|
+
*/
|
|
2330
|
+
get data(): Uint8Array;
|
|
2331
|
+
/**
|
|
2332
|
+
* Validate a buffer's length against the image size.
|
|
2333
|
+
*
|
|
2334
|
+
* @param data The pixels.
|
|
2335
|
+
* @param width Width in pixels.
|
|
2336
|
+
* @param height Height in pixels.
|
|
2337
|
+
* @throws {@link ImageLoadError} if `data.length !== width * height * 3`.
|
|
2338
|
+
*/
|
|
2339
|
+
private static _check;
|
|
2340
|
+
}
|
|
2341
|
+
|
|
2342
|
+
/**
|
|
2343
|
+
* Single segmented instance produced by an instance-segmentation model.
|
|
2344
|
+
*
|
|
2345
|
+
* Mirrors {@link DetectionResult} and adds the per-instance binary mask
|
|
2346
|
+
* plus a "ready-to-display" background-removed crop.
|
|
2347
|
+
*/
|
|
2348
|
+
export declare interface SegmentationResult {
|
|
2349
|
+
readonly classId: number;
|
|
2350
|
+
readonly className: string;
|
|
2351
|
+
readonly confidence: number;
|
|
2352
|
+
readonly bbox: BoundingBox;
|
|
2353
|
+
/** Alias for `classId` (Ultralytics-style). */
|
|
2354
|
+
readonly cls: number;
|
|
2355
|
+
/** Alias for `className`. */
|
|
2356
|
+
readonly name: string;
|
|
2357
|
+
/** Alias for `confidence` (Ultralytics-style). */
|
|
2358
|
+
readonly conf: number;
|
|
2359
|
+
/** Alias for `bbox` (Ultralytics-style). */
|
|
2360
|
+
readonly box: BoundingBox;
|
|
2361
|
+
/**
|
|
2362
|
+
* Binary mask cropped to `bbox`. Values are `0` (background) or `255`
|
|
2363
|
+
* (foreground). Empty boxes yield a zero-sized `Mask`.
|
|
2364
|
+
*/
|
|
2365
|
+
readonly mask: Mask;
|
|
2366
|
+
/**
|
|
2367
|
+
* The original image cropped to `bbox` with background pixels (where
|
|
2368
|
+
* `mask.data[i] === 0`) zeroed out. Empty boxes yield a zero-sized
|
|
2369
|
+
* `RGBImage`.
|
|
2370
|
+
*/
|
|
2371
|
+
readonly segmentedImage: RGBImage;
|
|
2372
|
+
}
|
|
2373
|
+
|
|
2374
|
+
/**
|
|
2375
|
+
* Per-image instance-segmentation envelope (Ultralytics-style `Results`).
|
|
2376
|
+
*
|
|
2377
|
+
* Iterating yields per-instance {@link SegmentationResult} entries. `boxes`
|
|
2378
|
+
* and `masks` mirror Ultralytics' bulk-array views.
|
|
2379
|
+
*/
|
|
2380
|
+
export declare class SegmentationResults implements Iterable<SegmentationResult> {
|
|
2381
|
+
readonly boxes: Boxes;
|
|
2382
|
+
readonly masks: Masks;
|
|
2383
|
+
readonly detections: readonly SegmentationResult[];
|
|
2384
|
+
readonly names: Readonly<Record<number, string>>;
|
|
2385
|
+
readonly origImg: RGBImage;
|
|
2386
|
+
readonly origShape: readonly [number, number];
|
|
2387
|
+
readonly path: string | null;
|
|
2388
|
+
readonly speed: Readonly<Speed>;
|
|
2389
|
+
constructor(boxes: Boxes, masks: Masks, detections: readonly SegmentationResult[], names: Readonly<Record<number, string>>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly<Speed>);
|
|
2390
|
+
/** Number of surviving instances. */
|
|
2391
|
+
get length(): number;
|
|
2392
|
+
/** Index into the per-instance results. */
|
|
2393
|
+
get(index: number): SegmentationResult | undefined;
|
|
2394
|
+
[Symbol.iterator](): Iterator<SegmentationResult>;
|
|
2395
|
+
}
|
|
2396
|
+
|
|
2397
|
+
/**
|
|
2398
|
+
* Instance segmenter for YOLO seg ONNX models (v8-seg / v11-seg / ...).
|
|
2399
|
+
*
|
|
2400
|
+
* The model is expected to expose two outputs:
|
|
2401
|
+
*
|
|
2402
|
+
* 1. `output0`: `(1, 4 + numClasses + numMaskCoefs, numAnchors)` — per-anchor
|
|
2403
|
+
* predictions (boxes, class scores, mask coefficients).
|
|
2404
|
+
* 2. `output1`: `(1, numMaskCoefs, maskH, maskW)` — prototype masks.
|
|
2405
|
+
*
|
|
2406
|
+
* `predict()` returns `Promise<SegmentationResults[]>` (length 1 for a
|
|
2407
|
+
* single image), mirroring Ultralytics' API. The envelope exposes:
|
|
2408
|
+
*
|
|
2409
|
+
* - `boxes`: bulk numpy view (`xyxy`, `xywh`, `xyxyn`, `xywhn`, `cls`, `conf`).
|
|
2410
|
+
* - `masks`: per-instance binary masks cropped to each box.
|
|
2411
|
+
* - per-instance {@link SegmentationResult} via iteration.
|
|
2412
|
+
*
|
|
2413
|
+
* @example
|
|
2414
|
+
* ```typescript
|
|
2415
|
+
* const seg = await Segmenter.create("/models/yolov8n-seg.onnx");
|
|
2416
|
+
* const r = (await seg.predict("/images/street.jpg"))[0];
|
|
2417
|
+
* for (const inst of r) {
|
|
2418
|
+
* console.log(inst.cls, inst.conf, inst.box.xyxy);
|
|
2419
|
+
* }
|
|
2420
|
+
* ```
|
|
2421
|
+
*/
|
|
2422
|
+
export declare class Segmenter extends VisionTask {
|
|
2423
|
+
private readonly _head;
|
|
2424
|
+
private readonly _labels;
|
|
2425
|
+
private readonly _names;
|
|
2426
|
+
private readonly _inputSize;
|
|
2427
|
+
private readonly _confThreshold;
|
|
2428
|
+
private readonly _iouThreshold;
|
|
2429
|
+
private readonly _maxDetections;
|
|
2430
|
+
private readonly _maskThreshold;
|
|
2431
|
+
private readonly _raiseOnEmpty;
|
|
2432
|
+
private constructor();
|
|
2433
|
+
private _pipelineCache;
|
|
2434
|
+
/**
|
|
2435
|
+
* Run the model once on a zero-filled tensor, paying one-time costs up front.
|
|
2436
|
+
*
|
|
2437
|
+
* The first inference of a session is not representative: WebGPU compiles its
|
|
2438
|
+
* shaders on it and the WASM backend faults in its arenas, which on a phone
|
|
2439
|
+
* can turn the first frame into seconds while every later frame is tens of
|
|
2440
|
+
* milliseconds. Calling this while a loading spinner is still up moves that
|
|
2441
|
+
* cost somewhere the user is already waiting.
|
|
2442
|
+
*
|
|
2443
|
+
* @param runs How many warm-up inferences to run. One is enough for WASM;
|
|
2444
|
+
* WebGPU sometimes settles on the second.
|
|
2445
|
+
*/
|
|
2446
|
+
warmup(runs?: number): Promise<void>;
|
|
2447
|
+
/**
|
|
2448
|
+
* The fused preprocessing pipeline, built on first use.
|
|
2449
|
+
*
|
|
2450
|
+
* Lazily, because constructing it allocates canvases: a task built in an
|
|
2451
|
+
* environment without a canvas implementation stays constructible, and only
|
|
2452
|
+
* fails if it is actually asked to preprocess something.
|
|
2453
|
+
*/
|
|
2454
|
+
private get _pipeline();
|
|
2455
|
+
/** Load the model and resolve labels. */
|
|
2456
|
+
static create(model: ModelSource, options?: SegmenterOptions): Promise<Segmenter>;
|
|
2457
|
+
/** The decoder family used to interpret the model's output. */
|
|
2458
|
+
get head(): SegmenterHead;
|
|
2459
|
+
/** Class labels indexed by class id. */
|
|
2460
|
+
get labels(): readonly string[];
|
|
2461
|
+
/** Class id → class name dict (matches Ultralytics' `model.names`). */
|
|
2462
|
+
get names(): Readonly<Record<number, string>>;
|
|
2463
|
+
/**
|
|
2464
|
+
* The `[width, height]` this task preprocesses to.
|
|
2465
|
+
*
|
|
2466
|
+
* Resolved at creation time from the model's graph when it declares a static
|
|
2467
|
+
* input, so reading it back tells you the resolution inference really runs at
|
|
2468
|
+
* — not merely what was requested.
|
|
2469
|
+
*/
|
|
2470
|
+
get inputSize(): readonly [number, number];
|
|
2471
|
+
/** Number of classes the model predicts. */
|
|
2472
|
+
get numClasses(): number;
|
|
2473
|
+
/** Alias for {@link predict} (parity with PyTorch `nn.Module.__call__`). */
|
|
2474
|
+
call(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
|
|
2475
|
+
/** Run instance segmentation on a single image. */
|
|
2476
|
+
predict(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
|
|
2477
|
+
/**
|
|
2478
|
+
* Letterbox and pack the image into the tensor the model expects.
|
|
2479
|
+
*
|
|
2480
|
+
* @param image The decoded input.
|
|
2481
|
+
* @param canvas The opaque canvas it was decoded on, when there is one — see
|
|
2482
|
+
* {@link LetterboxPipeline.run}.
|
|
2483
|
+
*/
|
|
2484
|
+
private _preprocess;
|
|
2485
|
+
private _splitOutputs;
|
|
2486
|
+
/**
|
|
2487
|
+
* Assemble one instance.
|
|
2488
|
+
*
|
|
2489
|
+
* The mask is trimmed to the part of the box inside the frame up front — it
|
|
2490
|
+
* is small and every caller reads it. `segmentedImage`, a masked copy of the
|
|
2491
|
+
* frame's pixels, is built on first read when the SDK owns those pixels and
|
|
2492
|
+
* up front otherwise, for the reason {@link Detector} gives for its crops.
|
|
2493
|
+
*
|
|
2494
|
+
* @param original The decoded input.
|
|
2495
|
+
* @param bbox Box in original-image coordinates.
|
|
2496
|
+
* @param classId Predicted class.
|
|
2497
|
+
* @param confidence Detection score.
|
|
2498
|
+
* @param mask Binary mask shaped to `bbox`.
|
|
2499
|
+
* @param lazy Whether `segmentedImage` may be deferred.
|
|
2500
|
+
*/
|
|
2501
|
+
private _buildResult;
|
|
2502
|
+
private _buildBoxes;
|
|
2503
|
+
private _buildMasks;
|
|
2504
|
+
}
|
|
2505
|
+
|
|
2506
|
+
/**
|
|
2507
|
+
* Decoder family for the segmentation head.
|
|
2508
|
+
*
|
|
2509
|
+
* - `"yolo-seg"`: YOLO instance-segmentation head with two outputs —
|
|
2510
|
+
* `[1, 4 + nc + nm, N]` per-anchor predictions plus `[1, nm, mh, mw]`
|
|
2511
|
+
* prototype masks. Covers YOLOv8-seg, v11-seg, v26-seg.
|
|
2512
|
+
*
|
|
2513
|
+
* The SDK does **not** auto-detect this — the caller is responsible for
|
|
2514
|
+
* picking a head that matches their export.
|
|
2515
|
+
*/
|
|
2516
|
+
export declare type SegmenterHead = "yolo-seg";
|
|
2517
|
+
|
|
2518
|
+
export declare interface SegmenterOptions extends OrtSessionOptions {
|
|
2519
|
+
/**
|
|
2520
|
+
* Decoder family for the segmentation head. Default `"yolo-seg"` covers
|
|
2521
|
+
* YOLOv8-seg/v11-seg/v26-seg.
|
|
2522
|
+
*/
|
|
2523
|
+
readonly head?: SegmenterHead;
|
|
2524
|
+
/** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */
|
|
2525
|
+
readonly labels?: LabelSpec;
|
|
2526
|
+
/** Number of classes — used to validate the supplied labels. */
|
|
2527
|
+
readonly numClasses?: number;
|
|
2528
|
+
/**
|
|
2529
|
+
* Model input `[width, height]` in pixels for letterboxing.
|
|
2530
|
+
*
|
|
2531
|
+
* Only used when the model's graph leaves its spatial axes dynamic: a graph
|
|
2532
|
+
* that declares a static size always wins, since that is the only shape ONNX
|
|
2533
|
+
* Runtime will accept. Defaults to `[640, 640]`.
|
|
2534
|
+
*/
|
|
2535
|
+
readonly inputSize?: readonly [number, number];
|
|
2536
|
+
/** Default minimum class score to keep a candidate. */
|
|
2537
|
+
readonly confThreshold?: number;
|
|
2538
|
+
/** Default IoU threshold for non-maximum suppression. */
|
|
2539
|
+
readonly iouThreshold?: number;
|
|
2540
|
+
/** Maximum number of instances per image. */
|
|
2541
|
+
readonly maxDetections?: number;
|
|
2542
|
+
/**
|
|
2543
|
+
* If `true`, a run that finds nothing throws {@link NoDetectionsError}
|
|
2544
|
+
* instead of returning an empty envelope. Default `false`, because looking
|
|
2545
|
+
* and finding nothing is a successful inference. Turn it on when an empty
|
|
2546
|
+
* result means the surrounding pipeline should stop rather than carry on with
|
|
2547
|
+
* zero rows. Can be overridden per `predict` call.
|
|
2548
|
+
*/
|
|
2549
|
+
readonly raiseOnEmpty?: boolean;
|
|
2550
|
+
/** Probability cutoff applied to soft masks. Defaults to `0.5`. */
|
|
2551
|
+
readonly maskThreshold?: number;
|
|
2552
|
+
}
|
|
2553
|
+
|
|
2554
|
+
export declare interface SegmenterPredictOptions {
|
|
2555
|
+
readonly confThreshold?: number;
|
|
2556
|
+
readonly iouThreshold?: number;
|
|
2557
|
+
/**
|
|
2558
|
+
* If set, keep only instances whose `classId` is in this list.
|
|
2559
|
+
* Mirrors Ultralytics' `model.predict(img, classes=[0, 16])`.
|
|
2560
|
+
*/
|
|
2561
|
+
readonly classes?: readonly number[];
|
|
2562
|
+
/** Override the constructor's `raiseOnEmpty` setting for this call. */
|
|
2563
|
+
readonly raiseOnEmpty?: boolean;
|
|
2564
|
+
}
|
|
2565
|
+
|
|
2566
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
2567
|
+
/**
|
|
2568
|
+
* Classification head postprocessing — softmax + top-k.
|
|
2569
|
+
*/
|
|
2570
|
+
/** Apply numerically-stable softmax to a 1-D vector of logits. */
|
|
2571
|
+
export declare function softmax(logits: Float32Array | readonly number[]): Float32Array;
|
|
2572
|
+
|
|
2573
|
+
/**
|
|
2574
|
+
* Read the spatial input size out of a declared NCHW shape.
|
|
2575
|
+
*
|
|
2576
|
+
* @param shape The declared shape of the model's image input.
|
|
2577
|
+
* @returns `[width, height]` in pixels, or `null` when the shape is not 4D or
|
|
2578
|
+
* leaves either spatial axis dynamic — in which case the model accepts more
|
|
2579
|
+
* than one resolution and there is nothing to correct.
|
|
2580
|
+
*/
|
|
2581
|
+
export declare function spatialInputSize(shape: DeclaredShape): readonly [number, number] | null;
|
|
2582
|
+
|
|
2583
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
2584
|
+
/**
|
|
2585
|
+
* Per-stage timing for a single `predict()` call.
|
|
2586
|
+
*
|
|
2587
|
+
* Populates the `speed` field every `Results` envelope carries, mirroring
|
|
2588
|
+
* Ultralytics' `results[0].speed`. All values are milliseconds measured with
|
|
2589
|
+
* `performance.now()`.
|
|
2590
|
+
*/
|
|
2591
|
+
/**
|
|
2592
|
+
* Stage durations of one inference, in milliseconds.
|
|
2593
|
+
*
|
|
2594
|
+
* `preprocess`, `inference` and `postprocess` are the three keys Ultralytics
|
|
2595
|
+
* reports, measured over the same boundaries. `load` is specific to this SDK:
|
|
2596
|
+
* `predict()` accepts a URL, `Blob` or DOM element and decodes it internally,
|
|
2597
|
+
* so the fetch/decode cost would otherwise be invisible — and on a cold cache
|
|
2598
|
+
* it dominates everything else.
|
|
2599
|
+
*/
|
|
2600
|
+
export declare interface Speed {
|
|
2601
|
+
/** Fetching and decoding the input into an `RGBImage`. */
|
|
2602
|
+
load: number;
|
|
2603
|
+
/** Letterbox/resize, normalization and tensor packing. */
|
|
2604
|
+
preprocess: number;
|
|
2605
|
+
/** The ONNX Runtime forward pass. */
|
|
2606
|
+
inference: number;
|
|
2607
|
+
/** Decoding raw outputs into results (NMS, mask assembly, top-k). */
|
|
2608
|
+
postprocess: number;
|
|
2609
|
+
}
|
|
2610
|
+
|
|
2611
|
+
/**
|
|
2612
|
+
* Accumulate stage durations while a `predict()` call runs.
|
|
2613
|
+
*
|
|
2614
|
+
* Each `stage()` call closes the previous stage: the elapsed time since the
|
|
2615
|
+
* last boundary is attributed to the name given. This keeps the call sites
|
|
2616
|
+
* free of paired start/stop bookkeeping and guarantees the four stages tile
|
|
2617
|
+
* the whole call without gaps.
|
|
2618
|
+
*/
|
|
2619
|
+
export declare class SpeedTimer {
|
|
2620
|
+
private _last;
|
|
2621
|
+
private readonly _speed;
|
|
2622
|
+
constructor();
|
|
2623
|
+
/**
|
|
2624
|
+
* Attribute the time elapsed since the previous boundary to `stage`.
|
|
2625
|
+
*
|
|
2626
|
+
* @param stage Which stage just finished.
|
|
2627
|
+
*/
|
|
2628
|
+
stage(stage: keyof Speed): void;
|
|
2629
|
+
/**
|
|
2630
|
+
* The accumulated durations.
|
|
2631
|
+
*
|
|
2632
|
+
* @returns The `speed` object to hand to the `Results` envelope.
|
|
2633
|
+
*/
|
|
2634
|
+
speed(): Speed;
|
|
2635
|
+
}
|
|
2636
|
+
|
|
2637
|
+
/**
|
|
2638
|
+
* Name the tensor type an ONNX element type maps to.
|
|
2639
|
+
*
|
|
2640
|
+
* @param elemType A `TensorProto.DataType` value read off the graph.
|
|
2641
|
+
* @returns The ORT tensor type name, or `"float32"` for anything unrecognised —
|
|
2642
|
+
* which is what every task produces anyway, so an unknown type surfaces as
|
|
2643
|
+
* ORT's own type error against the real graph rather than as a lookup miss.
|
|
2644
|
+
*/
|
|
2645
|
+
export declare function tensorTypeFor(elemType: number | undefined): string;
|
|
2646
|
+
|
|
2647
|
+
/**
|
|
2648
|
+
* Transpose interleaved HWC data to planar CHW layout.
|
|
2649
|
+
*
|
|
2650
|
+
* @param hwc Source array of length `width * height * channels`.
|
|
2651
|
+
*/
|
|
2652
|
+
export declare function toCHW(hwc: Float32Array, width: number, height: number, channels?: number): Float32Array;
|
|
2653
|
+
|
|
2654
|
+
/**
|
|
2655
|
+
* Convert the SDK's HWC RGB image to an HWC BGR `Uint8Array` (OpenCV layout).
|
|
2656
|
+
*
|
|
2657
|
+
* Useful for round-tripping data to a Python OpenCV consumer.
|
|
2658
|
+
*/
|
|
2659
|
+
export declare function toCv2(image: RGBImage): Uint8Array;
|
|
2660
|
+
|
|
2661
|
+
/** Convert a uint8 image to a `Float32Array` in `[0, 1]` (HWC layout preserved). */
|
|
2662
|
+
export declare function toFloat32(image: RGBImage, scale?: number): Float32Array;
|
|
2663
|
+
|
|
2664
|
+
/** Wrap a Float32 buffer into an `ort.Tensor`. */
|
|
2665
|
+
export declare function toFloat32Tensor(data: Float32Array, dims: readonly number[]): ort.Tensor;
|
|
2666
|
+
|
|
2667
|
+
/**
|
|
2668
|
+
* Return the top-k entries of a 1-D probability vector, sorted descending.
|
|
2669
|
+
*
|
|
2670
|
+
* @param k Number of entries to return; `null` returns all entries.
|
|
2671
|
+
*/
|
|
2672
|
+
export declare function topK(probabilities: Float32Array, k: number | null): TopKResult;
|
|
2673
|
+
|
|
2674
|
+
export declare interface TopKResult {
|
|
2675
|
+
readonly indices: Int32Array;
|
|
2676
|
+
readonly values: Float32Array;
|
|
2677
|
+
}
|
|
2678
|
+
|
|
2679
|
+
/**
|
|
2680
|
+
* Convert an HWC uint8 image to a CHW `Float32Array` scaled to `[0, 1]`.
|
|
2681
|
+
*
|
|
2682
|
+
* Mirrors `torchvision.transforms.ToTensor()` semantics: HWC → CHW,
|
|
2683
|
+
* `uint8 → float32 / 255`. Useful as input to YOLO-style detectors that
|
|
2684
|
+
* don't require ImageNet normalization.
|
|
2685
|
+
*
|
|
2686
|
+
* @returns CHW `Float32Array` of length `width * height * 3`.
|
|
2687
|
+
*/
|
|
2688
|
+
export declare function toTensor(image: RGBImage): Float32Array;
|
|
2689
|
+
|
|
2690
|
+
/**
|
|
2691
|
+
* Acquire a `MediaStream` via `getUserMedia`, attach it to a `<video>` element,
|
|
2692
|
+
* and expose status/error so the page can render permission and error states.
|
|
2693
|
+
* The stream is automatically released on unmount or retry.
|
|
2694
|
+
*
|
|
2695
|
+
* Defaults to the rear ("environment") camera; desktops fall back to whatever
|
|
2696
|
+
* single camera they expose. Pass `options.constraints` to override, or
|
|
2697
|
+
* `options.enabled: false` to render the surface without prompting for the camera
|
|
2698
|
+
* at all.
|
|
2699
|
+
*
|
|
2700
|
+
* Implementation notes:
|
|
2701
|
+
* - Cleanup detaches the stream from a *snapshotted* video node, so it releases
|
|
2702
|
+
* the same element it attached to even if the page remounts the `<video>`.
|
|
2703
|
+
* - When `getUserMedia` is missing, an insecure context is the usual cause, so
|
|
2704
|
+
* the hook prefers that (actionable) error; otherwise it reports `unsupported`.
|
|
2705
|
+
* - `video.play()` rejections are swallowed: autoplay may be blocked, but the
|
|
2706
|
+
* user gesture that opened the camera usually counts and playback resumes on
|
|
2707
|
+
* the next interaction.
|
|
2708
|
+
*
|
|
2709
|
+
* @param options - optional configuration (see {@link UseCameraStreamOptions}).
|
|
2710
|
+
* @returns The stream status, classified error, a `videoRef` to attach, the live
|
|
2711
|
+
* `stream` for whatever needs the track itself, and a `retry()` to re-attempt
|
|
2712
|
+
* acquisition.
|
|
2713
|
+
*/
|
|
2714
|
+
export declare function useCameraStream(options?: UseCameraStreamOptions): UseCameraStreamApi;
|
|
2715
|
+
|
|
2716
|
+
/** Value returned by {@link useCameraStream}. */
|
|
2717
|
+
export declare interface UseCameraStreamApi {
|
|
2718
|
+
/** Current lifecycle status. */
|
|
2719
|
+
status: CameraStreamStatus;
|
|
2720
|
+
/** The classified error, or `null` while not in the `error` status. */
|
|
2721
|
+
error: CameraStreamError | null;
|
|
2722
|
+
/** Attach to a `<video ref={…} />`. The stream is wired to it once ready. */
|
|
2723
|
+
videoRef: RefObject<HTMLVideoElement | null>;
|
|
2724
|
+
/**
|
|
2725
|
+
* The live stream, or `null`.
|
|
2726
|
+
*
|
|
2727
|
+
* Exposed for the things that need the **track** rather than the picture — the LED
|
|
2728
|
+
* torch (`useTorch`), the real frame size from `getSettings()`, recording it with
|
|
2729
|
+
* `useVideoRecorder`. Do not stop it yourself: the hook owns its lifetime and
|
|
2730
|
+
* releases it on unmount and on `retry()`.
|
|
2731
|
+
*/
|
|
2732
|
+
stream: MediaStream | null;
|
|
2733
|
+
/** Manually re-attempt after an error (e.g. the user changed permissions). */
|
|
2734
|
+
retry: () => void;
|
|
2735
|
+
}
|
|
2736
|
+
|
|
2737
|
+
/** Options for {@link useCameraStream}. */
|
|
2738
|
+
export declare interface UseCameraStreamOptions {
|
|
2739
|
+
/**
|
|
2740
|
+
* Constraints passed to `getUserMedia`. Defaults to the rear
|
|
2741
|
+
* ("environment") camera at Full-HD ideal resolution with audio off.
|
|
2742
|
+
* Read when the stream (re)starts — change it and call `retry()` to apply.
|
|
2743
|
+
*/
|
|
2744
|
+
constraints?: MediaStreamConstraints;
|
|
2745
|
+
/**
|
|
2746
|
+
* Hold off acquiring the camera until this is `true`. Default `true`.
|
|
2747
|
+
*
|
|
2748
|
+
* The point is to be able to *not* prompt. A permission prompt costs the user a
|
|
2749
|
+
* decision and, if they refuse, costs the app the feature permanently — so a
|
|
2750
|
+
* surface that already knows it cannot do its job (a barcode scanner in a browser
|
|
2751
|
+
* with no decoder) must not open the camera to then say so. Flipping this to `false`
|
|
2752
|
+
* also releases a stream that is already live.
|
|
2753
|
+
*/
|
|
2754
|
+
enabled?: boolean;
|
|
2755
|
+
}
|
|
2756
|
+
|
|
2757
|
+
/**
|
|
2758
|
+
* Sample mean luminance from a `<video>` source on a `requestAnimationFrame`
|
|
2759
|
+
* loop and expose the rolling value. Sampling is throttled by `intervalMs` and
|
|
2760
|
+
* paused whenever `enabled` is `false` or the video is not ready yet
|
|
2761
|
+
* (`readyState < 2` or `videoWidth === 0`).
|
|
2762
|
+
*
|
|
2763
|
+
* One offscreen canvas is reused across frames to avoid GC pressure. Designed
|
|
2764
|
+
* to feed a live brightness bar / border color on a camera page.
|
|
2765
|
+
*
|
|
2766
|
+
* @param videoRef - ref to the `<video>` element to sample.
|
|
2767
|
+
* @param options - optional configuration (see {@link UseLiveLuminanceOptions}).
|
|
2768
|
+
* @returns The rolling mean luminance in `0..255` (`0` until the first sample).
|
|
2769
|
+
*/
|
|
2770
|
+
export declare function useLiveLuminance(videoRef: RefObject<HTMLVideoElement | null>, { enabled, intervalMs }?: UseLiveLuminanceOptions): number;
|
|
2771
|
+
|
|
2772
|
+
/** Options for {@link useLiveLuminance}. */
|
|
2773
|
+
export declare interface UseLiveLuminanceOptions {
|
|
2774
|
+
/** When `false` the loop is paused (e.g. while a capture is in flight). Default: `true`. */
|
|
2775
|
+
enabled?: boolean;
|
|
2776
|
+
/** Throttle measurements in milliseconds. Default: `160` (~6 fps), plenty for UX. */
|
|
2777
|
+
intervalMs?: number;
|
|
2778
|
+
}
|
|
2779
|
+
|
|
2780
|
+
export declare const VERSION: string;
|
|
2781
|
+
|
|
2782
|
+
export declare abstract class VisionTask {
|
|
2783
|
+
protected readonly _session: OrtSession;
|
|
2784
|
+
protected constructor(_session: OrtSession);
|
|
2785
|
+
/** The underlying {@link OrtSession} used to run inference. */
|
|
2786
|
+
get session(): OrtSession;
|
|
2787
|
+
}
|
|
2788
|
+
|
|
2789
|
+
/**
|
|
2790
|
+
* Write an RGBA buffer as normalized planar float32.
|
|
2791
|
+
*
|
|
2792
|
+
* Shared by {@link ResizePipeline} and exported for a custom pipeline that
|
|
2793
|
+
* already holds pixels and wants the SDK's exact arithmetic.
|
|
2794
|
+
*
|
|
2795
|
+
* The expression is `(value / 255 - mean) / std`, evaluated in that order on
|
|
2796
|
+
* purpose: folding it into a single multiply-add (`value * k + b`) computes the
|
|
2797
|
+
* same quantity but rounds differently, and the tasks' output is asserted to be
|
|
2798
|
+
* bit-identical to the composable `normalize` → `toCHW` path.
|
|
2799
|
+
*
|
|
2800
|
+
* @param rgba Source pixels, 4 bytes per pixel, `width * height` long.
|
|
2801
|
+
* @param width Image width in pixels.
|
|
2802
|
+
* @param height Image height in pixels.
|
|
2803
|
+
* @param mean Per-channel RGB mean, already in `[0, 1]`.
|
|
2804
|
+
* @param std Per-channel RGB standard deviation.
|
|
2805
|
+
* @param out Destination buffer, `3 * width * height` long.
|
|
2806
|
+
* @param stride Bytes per source pixel: 4 for canvas RGBA, 3 for packed RGB.
|
|
2807
|
+
*/
|
|
2808
|
+
export declare function writePlanarFloat32(rgba: Uint8ClampedArray | Uint8Array, width: number, height: number, mean: readonly [number, number, number], std: readonly [number, number, number], out: Float32Array, stride?: number): void;
|
|
2809
|
+
|
|
2810
|
+
/**
|
|
2811
|
+
* Build a zero-filled CHW tensor payload for a warm-up run.
|
|
2812
|
+
*
|
|
2813
|
+
* @param width Model input width in pixels.
|
|
2814
|
+
* @param height Model input height in pixels.
|
|
2815
|
+
*/
|
|
2816
|
+
export declare function zeroTensorData(width: number, height: number): Float32Array;
|
|
2817
|
+
|
|
2818
|
+
export { }
|