tempest-react-sdk 0.69.0 → 0.70.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +1 -1
  2. package/dist/vision/core/canvas.cjs +1 -1
  3. package/dist/vision/core/canvas.cjs.map +1 -1
  4. package/dist/vision/core/canvas.js +37 -13
  5. package/dist/vision/core/canvas.js.map +1 -1
  6. package/dist/vision/core/dtypes.cjs +2 -0
  7. package/dist/vision/core/dtypes.cjs.map +1 -0
  8. package/dist/vision/core/dtypes.js +33 -0
  9. package/dist/vision/core/dtypes.js.map +1 -0
  10. package/dist/vision/core/graph.cjs +1 -1
  11. package/dist/vision/core/graph.cjs.map +1 -1
  12. package/dist/vision/core/graph.js +10 -2
  13. package/dist/vision/core/graph.js.map +1 -1
  14. package/dist/vision/core/metadata.cjs +2 -2
  15. package/dist/vision/core/metadata.cjs.map +1 -1
  16. package/dist/vision/core/metadata.js +103 -13
  17. package/dist/vision/core/metadata.js.map +1 -1
  18. package/dist/vision/core/providers.cjs +1 -1
  19. package/dist/vision/core/providers.cjs.map +1 -1
  20. package/dist/vision/core/providers.js +9 -6
  21. package/dist/vision/core/providers.js.map +1 -1
  22. package/dist/vision/core/session.cjs +1 -1
  23. package/dist/vision/core/session.cjs.map +1 -1
  24. package/dist/vision/core/session.js +115 -27
  25. package/dist/vision/core/session.js.map +1 -1
  26. package/dist/vision/fusion.cjs +1 -1
  27. package/dist/vision/fusion.cjs.map +1 -1
  28. package/dist/vision/fusion.js +28 -23
  29. package/dist/vision/fusion.js.map +1 -1
  30. package/dist/vision/index.cjs +1 -1
  31. package/dist/vision/index.cjs.map +1 -1
  32. package/dist/vision/index.js +20 -19
  33. package/dist/vision/index.js.map +1 -1
  34. package/dist/vision/io/image.cjs +1 -1
  35. package/dist/vision/io/image.cjs.map +1 -1
  36. package/dist/vision/io/image.js +55 -20
  37. package/dist/vision/io/image.js.map +1 -1
  38. package/dist/vision/postprocess/detection.cjs +1 -1
  39. package/dist/vision/postprocess/detection.cjs.map +1 -1
  40. package/dist/vision/postprocess/detection.js +42 -51
  41. package/dist/vision/postprocess/detection.js.map +1 -1
  42. package/dist/vision/postprocess/segmentation.cjs +1 -1
  43. package/dist/vision/postprocess/segmentation.cjs.map +1 -1
  44. package/dist/vision/postprocess/segmentation.js +45 -31
  45. package/dist/vision/postprocess/segmentation.js.map +1 -1
  46. package/dist/vision/preprocess/pipeline.cjs +1 -1
  47. package/dist/vision/preprocess/pipeline.cjs.map +1 -1
  48. package/dist/vision/preprocess/pipeline.js +24 -25
  49. package/dist/vision/preprocess/pipeline.js.map +1 -1
  50. package/dist/vision/tasks/base.cjs +1 -1
  51. package/dist/vision/tasks/base.cjs.map +1 -1
  52. package/dist/vision/tasks/base.js +22 -7
  53. package/dist/vision/tasks/base.js.map +1 -1
  54. package/dist/vision/tasks/classifier.cjs +1 -1
  55. package/dist/vision/tasks/classifier.cjs.map +1 -1
  56. package/dist/vision/tasks/classifier.js +64 -55
  57. package/dist/vision/tasks/classifier.js.map +1 -1
  58. package/dist/vision/tasks/detectClassify.cjs +1 -1
  59. package/dist/vision/tasks/detectClassify.cjs.map +1 -1
  60. package/dist/vision/tasks/detectClassify.js +80 -69
  61. package/dist/vision/tasks/detectClassify.js.map +1 -1
  62. package/dist/vision/tasks/detector.cjs +1 -1
  63. package/dist/vision/tasks/detector.cjs.map +1 -1
  64. package/dist/vision/tasks/detector.js +74 -68
  65. package/dist/vision/tasks/detector.js.map +1 -1
  66. package/dist/vision/tasks/segmenter.cjs +1 -1
  67. package/dist/vision/tasks/segmenter.cjs.map +1 -1
  68. package/dist/vision/tasks/segmenter.js +85 -62
  69. package/dist/vision/tasks/segmenter.js.map +1 -1
  70. package/dist/vision/types.cjs +1 -1
  71. package/dist/vision/types.cjs.map +1 -1
  72. package/dist/vision/types.js +24 -4
  73. package/dist/vision/types.js.map +1 -1
  74. package/dist/vision.cjs +1 -1
  75. package/dist/vision.d.ts +1153 -777
  76. package/dist/vision.js +22 -21
  77. package/package.json +1 -1
package/dist/vision.d.ts CHANGED
@@ -1,6 +1,15 @@
1
1
  import type * as ort from 'onnxruntime-web';
2
2
  import { RefObject } from 'react';
3
3
 
4
+ /**
5
+ * Widen a model output to `Float32Array`, without copying when it already is.
6
+ *
7
+ * @param data A tensor's `data`, as ORT returned it.
8
+ * @returns The same values as a `Float32Array`. A float32 output is passed
9
+ * through untouched; anything else is copied once.
10
+ */
11
+ export declare function asFloat32Array(data: unknown): Float32Array;
12
+
4
13
  /**
5
14
  * Per-class NMS — boxes are suppressed only by other boxes of the same class.
6
15
  *
@@ -274,6 +283,13 @@ export declare class Classifier extends VisionTask {
274
283
  call(image: ImageInput, options?: ClassifierPredictOptions): Promise<ClassificationResults[]>;
275
284
  /** Run classification on a single image. */
276
285
  predict(image: ImageInput, options?: ClassifierPredictOptions): Promise<ClassificationResults[]>;
286
+ /**
287
+ * Resize and pack the image into the tensor the model expects.
288
+ *
289
+ * @param image The decoded input.
290
+ * @param canvas The opaque canvas it was decoded on, when there is one — see
291
+ * {@link ResizePipeline.run}.
292
+ */
277
293
  private _preprocess;
278
294
  private _postprocess;
279
295
  }
@@ -475,6 +491,16 @@ export declare function decodeYolo(output: Float32Array, outputDims: readonly nu
475
491
  * mask coefficients) are ignored — callers can fetch them via the returned
476
492
  * {@link DecodedAnchors.anchorIndices}.
477
493
  *
494
+ * Candidates are found in two passes, mirroring the Python decoder. The first
495
+ * walks each class row contiguously and only *marks* anchors holding some score
496
+ * at or above the threshold — an anchor's best score clears the threshold
497
+ * exactly when one of its scores does, so no running maximum is needed and the
498
+ * hot loop never writes on a typical frame. The anchor-major scan it replaces
499
+ * read `numClasses` values spaced `numAnchors` apart for every one of the 8400
500
+ * anchors and cost ~1 ms even on a frame with no candidates. The winning class
501
+ * is then picked only for the marked anchors — a few hundred, not 8400 — with
502
+ * the same strict `>` that keeps the lowest class id on a tie.
503
+ *
478
504
  * @param data Flat per-anchor output, length `channels * numAnchors`.
479
505
  * @param dims Dims as reported by ORT, e.g. `[1, 84, 8400]` (det) or
480
506
  * `[1, 116, 8400]` (seg). The leading batch dim must be 1.
@@ -510,6 +536,11 @@ export declare interface DecodeYoloOptions {
510
536
  *
511
537
  * Compatible with YOLOv8-seg / YOLOv11-seg.
512
538
  *
539
+ * The prototypes' batch axis is stripped here; the per-anchor batch axis is
540
+ * validated by {@link decodeYoloAnchors}. Each instance's box is mapped back
541
+ * into input-tensor coordinates and from there into the low-resolution
542
+ * prototype grid, and only the prototype pixels under it are assembled.
543
+ *
513
544
  * @param perAnchorData Flat `output0`, length `(4 + numClasses + numMaskCoefs) * numAnchors`.
514
545
  * @param perAnchorDims Dims as reported by ORT, e.g. `[1, 116, 8400]`.
515
546
  * @param prototypeData Flat `output1`, length `numMaskCoefs * maskH * maskW`.
@@ -538,6 +569,9 @@ export declare interface DecodeYoloSegOptions {
538
569
  readonly maskThreshold?: number;
539
570
  }
540
571
 
572
+ /** Cache Storage bucket used when {@link OrtSessionOptions.cache} is `true`. */
573
+ export declare const DEFAULT_MODEL_CACHE = "ort-vision-sdk-models";
574
+
541
575
  /**
542
576
  * Default execution provider preference order for browser ORT.
543
577
  *
@@ -546,6 +580,9 @@ export declare interface DecodeYoloSegOptions {
546
580
  */
547
581
  export declare const DEFAULT_PROVIDERS: readonly string[];
548
582
 
583
+ /** The element type every task preprocesses to, and the fallback when unknown. */
584
+ export declare const DEFAULT_TENSOR_TYPE = "float32";
585
+
549
586
  /**
550
587
  * Resolve a labels specification into an ordered array of class names.
551
588
  *
@@ -669,6 +706,10 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
669
706
  * is what lets the graph undo the letterbox transform internally and crop at
670
707
  * native resolution instead of from the downscaled copy. That one is **not**
671
708
  * letterboxed by definition, so it does not go through the fused path.
709
+ *
710
+ * @param image The decoded input.
711
+ * @param canvas The opaque canvas it was decoded on, when there is one — see
712
+ * {@link LetterboxPipeline.run}.
672
713
  */
673
714
  private _preprocess;
674
715
  /**
@@ -803,6 +844,14 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
803
844
  * belongs to*, and collapsing them would lose one of the two.
804
845
  */
805
846
  readonly classification?: ClassificationResult | null;
847
+ /**
848
+ * Binary foreground mask for this object, shaped to `bbox` in original-image
849
+ * pixels — the same contract {@link SegmentationResult.mask} uses.
850
+ *
851
+ * Populated only by a `detect_segment_classify` pipeline, where a
852
+ * segmentation stage ran inside the crop. `null` everywhere else.
853
+ */
854
+ readonly mask?: Mask | null;
806
855
  }
807
856
 
808
857
  /**
@@ -928,8 +977,26 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
928
977
  * readback loop, and reuses its output buffer between frames. The buffer is
929
978
  * handed straight to ONNX Runtime, so {@link _pipeline.release} must not be
930
979
  * called until the run resolves.
980
+ *
981
+ * @param image The decoded input.
982
+ * @param canvas The opaque canvas it was decoded on, when there is one — see
983
+ * {@link LetterboxPipeline.run}.
931
984
  */
932
985
  private _preprocess;
986
+ /**
987
+ * Assemble one detection.
988
+ *
989
+ * `croppedImage` is built on first read when the SDK owns the frame's pixels
990
+ * — see {@link memoize} for why. A frame the caller handed over as an
991
+ * `RGBImage` is cropped up front instead: its buffer is theirs, and a video
992
+ * loop refilling it would otherwise hand a late reader the wrong frame.
993
+ *
994
+ * @param original The decoded input.
995
+ * @param bbox Box in original-image coordinates.
996
+ * @param classId Predicted class.
997
+ * @param confidence Detection score.
998
+ * @param lazy Whether `croppedImage` may be deferred.
999
+ */
933
1000
  private _buildResult;
934
1001
  private _buildBoxes;
935
1002
  }
@@ -1012,10 +1079,29 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
1012
1079
  * kept rather than dropped: guessing a provider away would be worse than
1013
1080
  * admitting ignorance about it.
1014
1081
  *
1015
- * @param requested Providers in preference order, already resolved.
1016
- * @returns The subset that is not ruled out, in the same order.
1082
+ * The WebGPU adapter is only requested when `webgpu` is in the list. Asking
1083
+ * unconditionally cost every WASM-only session a `requestAdapter()` round trip
1084
+ * and, on a machine without a GPU, a "No available adapters" console warning
1085
+ * per `create`.
1086
+ *
1087
+ * @param requested Providers in preference order, already resolved, as names
1088
+ * or config objects.
1089
+ * @returns The subset that is not ruled out, in the same order and form.
1017
1090
  */
1018
- export declare function detectProviders(requested: readonly string[]): Promise<string[]>;
1091
+ export declare function detectProviders<T extends ProviderSpec>(requested: readonly T[]): Promise<T[]>;
1092
+
1093
+ /**
1094
+ * The structural surface of `Float16Array` this module needs.
1095
+ *
1096
+ * Declared here rather than pulled from `lib.es2025`: the package targets
1097
+ * ES2022 so it builds for consumers on older TypeScript, and the only thing
1098
+ * needed is "an array-like of numbers ORT recognises as half precision".
1099
+ */
1100
+ export declare interface Float16ArrayLike {
1101
+ readonly length: number;
1102
+ readonly buffer: ArrayBufferLike;
1103
+ [index: number]: number;
1104
+ }
1019
1105
 
1020
1106
  /**
1021
1107
  * Convert an HWC BGR uint8 buffer (OpenCV layout) to the SDK's HWC RGB.
@@ -1062,6 +1148,17 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
1062
1148
  /** Value of the `ovs.kind` metadata key for a detector→classifier pipeline. */
1063
1149
  export declare const FUSION_KIND_DETECT_CLASSIFY = "detect_classify";
1064
1150
 
1151
+ /**
1152
+ * Value of `ovs.kind` for a detector→segmenter→classifier pipeline.
1153
+ *
1154
+ * The segmenter runs inside each crop rather than over the whole frame: the
1155
+ * detector finds the boxes, the bridge cuts them out, and the segmenter turns
1156
+ * each crop into a foreground mask that gates the pixels the classifier sees.
1157
+ * Fusing one is a build step on the Python side; the browser only loads the
1158
+ * result.
1159
+ */
1160
+ export declare const FUSION_KIND_DETECT_SEGMENT_CLASSIFY = "detect_segment_classify";
1161
+
1065
1162
  /**
1066
1163
  * Raised when a model cannot be driven as a fused detect→classify pipeline.
1067
1164
  *
@@ -1075,7 +1172,7 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
1075
1172
 
1076
1173
  /** Everything a fused pipeline declares about how it must be driven. */
1077
1174
  export declare interface FusionSpec {
1078
- /** Pipeline family. Only `"detect_classify"` exists today. */
1175
+ /** Pipeline family — {@link FUSION_KIND_DETECT_CLASSIFY} or {@link FUSION_KIND_DETECT_SEGMENT_CLASSIFY}. */
1079
1176
  readonly kind: string;
1080
1177
  /** `[width, height]` the detector stage expects — the resolution to letterbox to. */
1081
1178
  readonly inputSize: readonly [number, number];
@@ -1103,8 +1200,39 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
1103
1200
  readonly sdkVersion: string;
1104
1201
  /** Whether driving this pipeline requires feeding the full-resolution input. */
1105
1202
  readonly needsSourceImage: boolean;
1203
+ /** Whether this pipeline emits a {@link OUTPUT_MASKS} tensor. */
1204
+ readonly hasMasks: boolean;
1205
+ /** Class id → name for the segmenter's channels, or `null`. */
1206
+ readonly segmenterNames: readonly string[] | null;
1207
+ /**
1208
+ * Probability above which a segmenter pixel counted as foreground.
1209
+ *
1210
+ * Baked into the graph at fusion time; reported so a caller can say what the
1211
+ * masks mean.
1212
+ */
1213
+ readonly maskThreshold: number;
1214
+ /** Whether the mask gated the crop before the classifier saw it. */
1215
+ readonly maskApplied: boolean;
1106
1216
  }
1107
1217
 
1218
+ /**
1219
+ * Metadata key naming the level a model was graph-optimized at offline.
1220
+ *
1221
+ * Written by the Python SDK's `optimize_model`; both test suites pin the same
1222
+ * string, so a rename on one side fails. See {@link OrtSession.create} for what
1223
+ * the session does when it finds it.
1224
+ */
1225
+ export declare const GRAPH_OPTIMIZATION_KEY = "ort_vision_sdk.graph_optimization";
1226
+
1227
+ /**
1228
+ * Whether this environment can build a half-precision tensor at all.
1229
+ *
1230
+ * @returns `true` when the runtime exposes `Float16Array`. ORT rejects any
1231
+ * other typed array for a `float16` tensor, so a `false` here means half
1232
+ * precision is unreachable in this browser no matter what the model declares.
1233
+ */
1234
+ export declare function hasFloat16Array(): boolean;
1235
+
1108
1236
  /** Mean that leaves an image untouched — what an Ultralytics classifier expects. */
1109
1237
  export declare const IDENTITY_MEAN: readonly [number, number, number];
1110
1238
 
@@ -1112,7 +1240,7 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
1112
1240
  export declare const IDENTITY_STD: readonly [number, number, number];
1113
1241
 
1114
1242
  /** Anything {@link loadImage} accepts as an image input. */
1115
- export declare type ImageInput = string | Blob | HTMLImageElement | HTMLCanvasElement | OffscreenCanvas | ImageBitmap | ImageData | RGBImage;
1243
+ export declare type ImageInput = string | Blob | HTMLImageElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | ImageBitmap | VideoFrame | ImageData | RGBImage;
1116
1244
 
1117
1245
  /** Raised when an input image cannot be decoded into the canonical format. */
1118
1246
  export declare class ImageLoadError extends OrtVisionError {
@@ -1235,26 +1363,43 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
1235
1363
  * into — see {@link ReusableBuffer}.
1236
1364
  *
1237
1365
  * @param image Source image in the SDK's canonical HWC RGB layout.
1366
+ * @param canvas Optional drawable already holding exactly `image`'s pixels at
1367
+ * `image.width × image.height`, fully opaque. When given it is drawn
1368
+ * straight into the target, skipping the RGB→RGBA rebuild and the
1369
+ * `putImageData` onto the scratch canvas — two full-resolution passes per
1370
+ * frame. The tasks pass the canvas `loadImage` decoded onto; a caller with a
1371
+ * translucent source must not, since the result would composite its alpha.
1238
1372
  */
1239
- run(image: RGBImage): FusedLetterboxResult;
1373
+ run(image: RGBImage, canvas?: CanvasImageSource | null): FusedLetterboxResult;
1240
1374
  /**
1241
1375
  * Mark the reusable buffer free again.
1242
1376
  *
1243
1377
  * Call it once the tensor built from a {@link run} result has been handed to
1244
1378
  * ONNX Runtime and the run has resolved — after that the values are inside
1245
1379
  * the WASM heap and the buffer can be overwritten.
1380
+ *
1381
+ * Pass the result being released when several runs can be outstanding at
1382
+ * once. A result that got a fresh allocation (`reused: false`) holds nothing
1383
+ * of the pipeline's, so releasing it is a no-op; without that check, the
1384
+ * caller finishing with a fresh buffer would free the shared one while
1385
+ * another caller still had its frame in it. Called without an argument it
1386
+ * frees the shared buffer unconditionally, as before.
1387
+ *
1388
+ * @param result The {@link run} result being released.
1246
1389
  */
1247
- release(): void;
1390
+ release(result?: {
1391
+ readonly reused: boolean;
1392
+ }): void;
1248
1393
  /**
1249
- * Grow the scratch source canvas to fit an image, reusing it when possible.
1394
+ * Paint an image onto the scratch source canvas and return that canvas.
1250
1395
  *
1251
- * A canvas is only reallocated when a frame arrives at a different size than
1252
- * the last one, which for a camera or video source is never after the first.
1396
+ * The canvas is only reallocated when a frame arrives at a different size
1397
+ * than the last one, which for a camera or video source is never after the
1398
+ * first.
1253
1399
  *
1254
- * @param width Source width in pixels.
1255
- * @param height Source height in pixels.
1400
+ * @param image Source image in the SDK's canonical HWC RGB layout.
1256
1401
  */
1257
- private _ensureSource;
1402
+ private _paintSource;
1258
1403
  }
1259
1404
 
1260
1405
  export declare interface LetterboxResult {
@@ -1284,6 +1429,10 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
1284
1429
  /**
1285
1430
  * Load an image from any supported source into a HWC uint8 RGB array.
1286
1431
  *
1432
+ * A video element must have its current frame available (`readyState >= 2`);
1433
+ * the frame showing at the time of the call is the one decoded. A `VideoFrame`
1434
+ * is read but not closed — it belongs to the caller.
1435
+ *
1287
1436
  * @throws {@link ImageLoadError} if the source cannot be decoded or has an unsupported shape.
1288
1437
  */
1289
1438
  export declare function loadImage(source: ImageInput): Promise<RGBImage>;
@@ -1417,6 +1566,13 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
1417
1566
  * @returns Indices of kept boxes, in descending score order. Boxes tied on
1418
1567
  * score are visited lowest-index first, so the survivor of a tie is
1419
1568
  * deterministic and matches both `torchvision` and the Python SDK.
1569
+ *
1570
+ * The boxes are copied once into score order, so the inner loop walks a
1571
+ * contiguous `Float32Array` instead of hopping through `order` into the caller's
1572
+ * layout. A pair with no horizontal overlap is skipped before the vertical
1573
+ * extent is computed: its IoU is `0`, which suppresses nothing for any
1574
+ * non-negative threshold — the shortcut is gated on that so a negative
1575
+ * threshold keeps the exact semantics.
1420
1576
  */
1421
1577
  export declare function nms(boxes: Float32Array, scores: Float32Array, iouThreshold: number): Int32Array;
1422
1578
 
@@ -1512,7 +1668,22 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
1512
1668
  * `webgpu` on a device without it runs on WASM and is told nothing.
1513
1669
  */
1514
1670
  readonly requestedProviders: readonly string[];
1671
+ /**
1672
+ * Tensor type each input declares, keyed by input name.
1673
+ *
1674
+ * Read from the model file, not from the session: `inputMetadata` is
1675
+ * `undefined` on `onnxruntime-web` 1.20.1. Empty when the bytes were never
1676
+ * in hand — a URL loaded with `readMetadata: false` — in which case every
1677
+ * feed is built as float32, the behaviour before this existed.
1678
+ */
1679
+ private readonly _inputTypes;
1515
1680
  private constructor();
1681
+ /**
1682
+ * Tail of the run queue: settles when the latest queued run has finished.
1683
+ *
1684
+ * See {@link run} for why runs are serialized.
1685
+ */
1686
+ private _runQueue;
1516
1687
  /**
1517
1688
  * Load an ONNX model into an ORT inference session.
1518
1689
  *
@@ -1526,12 +1697,35 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
1526
1697
  * it — on a phone that was the difference between a session and
1527
1698
  * `Can't create a session. failed to allocate a buffer of size N`.
1528
1699
  *
1700
+ * The provider probe and the model download run concurrently: neither
1701
+ * depends on the other, and on a real GPU `requestAdapter()` is not free.
1702
+ * When the model is downloaded here, ORT's runtime is started alongside the
1703
+ * download too — see {@link warmRuntime}.
1704
+ *
1705
+ * A model whose metadata carries {@link GRAPH_OPTIMIZATION_KEY} was already
1706
+ * optimized offline by the Python SDK's `optimize_model`, so it is loaded
1707
+ * with `graphOptimizationLevel: "disabled"` instead of paying the optimizer a
1708
+ * second time — measured on a YOLO11n-seg under WASM, 29 ms of session
1709
+ * creation down to 12 ms with unchanged inference time. An explicit
1710
+ * `graphOptimizationLevel` in `sessionOptions` always wins. The mark is read
1711
+ * with the rest of the metadata, so `readMetadata: false` skips it too.
1712
+ *
1529
1713
  * @param model Either a URL string, or a `Uint8Array`/`ArrayBuffer` containing the model bytes.
1530
1714
  * @param options Provider list, pass-through `SessionOptions`, and whether to
1531
1715
  * read the model's metadata map (see {@link OrtSessionOptions.readMetadata}).
1532
1716
  * @throws {@link ModelLoadError} if the model cannot be loaded.
1533
1717
  */
1534
1718
  static create(model: ModelSource, options?: OrtSessionOptions): Promise<OrtSession>;
1719
+ /**
1720
+ * Tensor type each input declares, in declaration order.
1721
+ *
1722
+ * `"float32"` for a normal export, `"float16"` for one exported with
1723
+ * `half=True`. ORT matches a feed's type against this exactly, so the tasks
1724
+ * read it and convert at the feed boundary.
1725
+ */
1726
+ get inputDtypes(): readonly string[];
1727
+ /** Tensor type the first input declares. */
1728
+ get inputDtype(): string;
1535
1729
  /** Names of the model's inputs, in declaration order. */
1536
1730
  get inputNames(): readonly string[];
1537
1731
  /** Name of the first (and usually only) input. */
@@ -1588,855 +1782,1037 @@ export declare const DEFAULT_PROVIDERS: readonly string[];
1588
1782
  /**
1589
1783
  * Run inference and return all outputs.
1590
1784
  *
1785
+ * Runs on one session are serialized. ONNX Runtime Web refuses a second
1786
+ * `run` while one is in flight — it rejects with `Session already started` —
1787
+ * so two overlapping `predict()` calls on the same task used to fail. Queued
1788
+ * here, they overlap where they can: while one run executes (in a worker,
1789
+ * with `env.wasm.proxy`, or on the GPU), the next `predict()` decodes and
1790
+ * preprocesses its frame and only waits for its turn at the runtime. A run
1791
+ * that fails does not block the ones behind it.
1792
+ *
1591
1793
  * @param feeds Map of input name to `ort.Tensor`. Keys must match {@link inputNames}.
1592
1794
  * @throws {@link InferenceError} if ORT raises any error during execution.
1593
1795
  */
1594
1796
  run(feeds: Record<string, ort.Tensor>): Promise<Record<string, ort.Tensor>>;
1595
- }
1596
-
1597
- export declare interface OrtSessionOptions {
1598
- /**
1599
- * Execution providers in preference order. `undefined` uses {@link DEFAULT_PROVIDERS}.
1600
- *
1601
- * Naming one explicitly also opts into a `console.warn` when this browser
1602
- * cannot offer it, instead of falling back in silence.
1603
- */
1604
- readonly providers?: readonly string[];
1605
- /** Optional ORT session options forwarded to `InferenceSession.create`. */
1606
- readonly sessionOptions?: ort.InferenceSession.SessionOptions;
1607
- /**
1608
- * Whether to read the model's custom metadata map (`names`, `task`, `imgsz`).
1609
- * Defaults to `true`.
1610
- *
1611
- * The runtime does not expose that map, so it is read from the file itself —
1612
- * which means a URL model is fetched here and handed to ORT as bytes instead
1613
- * of letting ORT fetch it. That is the same single download either way, and
1614
- * it is what lets a task resolve its labels off the model. Set to `false` to
1615
- * keep the URL path untouched and leave {@link OrtSession.metadata} empty.
1616
- *
1617
- * `false` is also the escape hatch when a device cannot afford the bytes: the
1618
- * fetched buffer is dropped before ORT builds the graph (see
1619
- * {@link OrtSession.create}), but ORT's own load path still keeps the model out
1620
- * of reach of anything the SDK holds. A session built this way resolves its
1621
- * input size from the graph as usual — only the class names are lost, so a
1622
- * caller taking this route has to pass `labels` itself.
1623
- */
1624
- readonly readMetadata?: boolean;
1625
- }
1626
-
1627
- /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
1628
- /**
1629
- * Exceptions raised by the SDK.
1630
- *
1631
- * All exceptions inherit from {@link OrtVisionError}, so callers can catch
1632
- * the base class to handle any SDK-originated failure uniformly.
1633
- */
1634
- export declare class OrtVisionError extends Error {
1635
- constructor(message: string, options?: ErrorOptions);
1636
- }
1637
-
1638
- /**
1639
- * Name of the `[K, 4]` float32 xyxy output, in **letterboxed** input pixels.
1640
- *
1641
- * A file fused by `ort-vision-sdk` 0.9.0 or later reports the box that was
1642
- * actually classified: clamped to the image the crop came from, exactly as
1643
- * RoiAlign received it. Older files report the raw box, so one that ran off the
1644
- * frame draws a rectangle wider than the region the classifier saw.
1645
- */
1646
- export declare const OUTPUT_BOXES = "boxes";
1647
-
1648
- /** Name of the `[K]` int64 detector-class output. */
1649
- export declare const OUTPUT_CLASSES = "classes";
1650
-
1651
- /** Name of the `[1]` int64 output holding how many of the `K` rows are real. */
1652
- export declare const OUTPUT_NUM_DETECTIONS = "num_detections";
1653
-
1654
- /** Name of the `[K, numClassifierClasses]` float32 classifier output, one row per box. */
1655
- export declare const OUTPUT_PROBS = "probs";
1656
-
1657
- /** Name of the `[K]` float32 detection-confidence output. */
1658
- export declare const OUTPUT_SCORES = "scores";
1659
-
1660
- /**
1661
- * Parse a `repr`-encoded `dict[int, str]` class map.
1662
- *
1663
- * Split out of {@link modelNames} because the same encoding is reused by a
1664
- * fused pipeline, which carries one class map per stage and therefore cannot
1665
- * store both under the single `names` key Ultralytics uses.
1666
- *
1667
- * @param encoded The encoded map — e.g. `"{0: 'deworm', 1: 'not_deworm'}"`.
1668
- * @returns Class names in class-id order, or `null` when the value is missing,
1669
- * unparseable, not a `dict`, or not keyed by contiguous integers from zero.
1670
- */
1671
- export declare function parseNames(encoded: string | undefined): readonly string[] | null;
1672
-
1673
- /**
1674
- * Top-k classification probabilities for a single image.
1675
- *
1676
- * Mirrors Ultralytics' `Probs` interface.
1677
- */
1678
- export declare class Probs {
1679
- readonly data: Float32Array;
1680
- /** @param data `[numClasses]` per-class probabilities, indexed by class id. */
1681
- constructor(data: Float32Array);
1682
- /** Number of classes. */
1683
- get length(): number;
1684
- /** `[numClasses]` shape of the underlying vector. */
1685
- get shape(): readonly [number];
1686
- /** Index of the most probable class. */
1687
- get top1(): number;
1688
- /** Probability of the top-1 class. */
1689
- get top1conf(): number;
1690
- /**
1691
- * Indices of the top-5 most probable classes, descending.
1692
- *
1693
- * The array is the memoised selection itself, not a copy — reading it twice
1694
- * hands back the same object. Treat it as read-only: writing into it edits
1695
- * what every later read of this `Probs` returns.
1696
- */
1697
- get top5(): Int32Array;
1698
- /**
1699
- * Probabilities of the top-5 classes, descending.
1700
- *
1701
- * Shares the memoised selection with {@link Probs.top5}, under the same
1702
- * read-only caveat.
1703
- */
1704
- get top5conf(): Float32Array;
1705
- private _cache;
1706
- /**
1707
- * Memoised {@link Probs._topK}.
1708
- *
1709
- * `top5` and `top5conf` are separate getters over the same selection, so a
1710
- * caller reading both would otherwise pay for it twice — once per frame, in
1711
- * a camera loop. The probabilities a `Probs` was built from do not change,
1712
- * so the result is computed once per `k` and kept.
1713
- *
1714
- * @param k - How many classes to select.
1715
- * @returns The cached selection for that `k`.
1716
- */
1717
- private _top;
1718
- /**
1719
- * Select the `k` highest probabilities without ordering the rest.
1720
- *
1721
- * A full sort to read five entries out of a thousand-class vector costs
1722
- * O(n log n) plus an index array the size of the vector; keeping `k` slots
1723
- * ordered by insertion and scanning once costs O(n·k) with no allocation
1724
- * beyond the result. Measured on 1000 classes over 2000 iterations, 134.5 µs
1725
- * against 1.5 µs for the same output.
1726
- *
1727
- * Ties keep the lower class index first, matching the stable sort this
1728
- * replaced: a candidate only displaces an entry it is strictly greater
1729
- * than. That is also what keeps this selection in step with the Python
1730
- * SDK's `np.argsort(-data, kind="stable")`, where the sort is C and the
1731
- * cost this avoids does not arise.
1732
- *
1733
- * @param k - How many classes to select.
1734
- * @returns Indices and probabilities, descending by probability.
1735
- */
1736
- private _topK;
1737
- }
1738
-
1739
- /** Raised when a requested execution provider is not available. */
1740
- export declare class ProviderNotAvailableError extends OrtVisionError {
1741
- }
1742
-
1743
- /**
1744
- * Read a pipeline spec out of a model's custom metadata.
1745
- *
1746
- * Individual malformed entries fall back to the value a fusion would have used
1747
- * by default — a single bad float is not a reason to reject an otherwise
1748
- * loadable pipeline. A malformed resolution is fatal, because there is no safe
1749
- * default for one.
1750
- *
1751
- * @param metadata A model's custom metadata map, as read by
1752
- * {@link readModelMetadata}.
1753
- * @returns The decoded spec, or `null` when the model is not a fused pipeline —
1754
- * it carries no `ovs.kind` entry, or one naming a pipeline kind this version
1755
- * does not know how to drive.
1756
- */
1757
- export declare function readFusionSpec(metadata: Readonly<Record<string, string>> | undefined): FusionSpec | null;
1758
-
1759
- /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
1760
- /**
1761
- * Read the metadata an exporter baked into a `.onnx` file.
1762
- *
1763
- * `onnxruntime-web` exposes input/output metadata but **not** the model's
1764
- * custom metadata map, which is where Ultralytics writes `names`, `task` and
1765
- * `imgsz`. The Python SDK gets it for free from
1766
- * `InferenceSession.get_modelmeta().custom_metadata_map`; in the browser the
1767
- * only way to the same information is to read it out of the file, so this
1768
- * module walks just enough of the ModelProto wire format to collect
1769
- * `metadata_props`.
1770
- *
1771
- * It never throws and never allocates unbounded: a truncated, hostile or
1772
- * simply unexpected file yields an empty map, and every caller treats that as
1773
- * "the model says nothing", falling back to what it was given.
1774
- */
1775
- /**
1776
- * Collect a model's custom metadata map straight out of its bytes.
1777
- *
1778
- * @param model The `.onnx` file contents.
1779
- * @returns Key/value metadata — `names`, `task`, `imgsz`, ... for an
1780
- * Ultralytics export — or an empty object when the file carries none or
1781
- * cannot be walked.
1782
- */
1783
- export declare function readModelMetadata(model: Uint8Array | ArrayBufferLike): Readonly<Record<string, string>>;
1784
-
1785
- /**
1786
- * Turn an empty result into an error, when the caller asked for that.
1787
- *
1788
- * Shared by every task that can come back with nothing — {@link Detector},
1789
- * {@link Segmenter} and {@link DetectClassify} — so the three agree on when
1790
- * they throw and on what the message says. The message names the two settings
1791
- * that decide the outcome, because "no detections" on its own leaves the reader
1792
- * unable to tell a blank image from a threshold set too high.
1793
- *
1794
- * @param count How many detections survived every filter.
1795
- * @param options The flag for this call, the threshold actually applied (after
1796
- * any per-call override), the class allowlist if one narrowed the search, and
1797
- * the source path when the input was one.
1798
- * @throws {@link NoDetectionsError} when the flag is set and `count` is zero.
1799
- */
1800
- export declare function requireDetections(count: number, options: {
1801
- readonly raiseOnEmpty: boolean;
1802
- readonly confThreshold: number;
1803
- readonly classes: readonly number[] | undefined;
1804
- readonly path: string | null;
1805
- }): void;
1806
-
1807
- /** Resize an image to `(targetWidth, targetHeight)` using high-quality canvas resampling. */
1808
- export declare function resize(image: RGBImage, targetWidth: number, targetHeight: number): RGBImage;
1809
-
1810
1797
  /**
1811
- * Reusable stretch-resize → normalized tensor pipeline for one target size.
1812
- *
1813
- * The classification counterpart of {@link LetterboxPipeline}. A classifier
1814
- * stretches to the model's square input instead of letterboxing into it — no
1815
- * padding, no scale to invert later, because nothing is mapped back onto the
1816
- * source image afterwards. That difference is why it cannot simply reuse the
1817
- * letterbox path.
1798
+ * Execute one run immediately; only {@link run}'s queue calls this.
1818
1799
  *
1819
- * What it does share is the technique. The composable route
1820
- * (`resize` → `normalize` → `toCHW`) allocates an `RGBImage` and two
1821
- * `Float32Array`s and walks each end to end on every call: about 1.4 MB of
1822
- * fresh garbage per 224×224 `predict()`, produced at the exact moment a phone
1823
- * near its memory ceiling can least afford it. Here one `drawImage` resizes,
1824
- * and one loop reads the resulting RGBA and writes normalized planar float32
1825
- * into a buffer held across calls.
1826
- *
1827
- * Create one per task, not per frame.
1828
- */
1829
- export declare class ResizePipeline {
1830
- private readonly _targetWidth;
1831
- private readonly _targetHeight;
1832
- private readonly _mean;
1833
- private readonly _std;
1834
- private readonly _buffer;
1835
- private _target;
1836
- private _targetContext;
1837
- private _source;
1838
- private _sourceContext;
1839
- /**
1840
- * @param targetWidth Model input width in pixels.
1841
- * @param targetHeight Model input height in pixels.
1842
- * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.
1843
- * @param std Per-channel RGB standard deviation. Defaults to no scaling.
1800
+ * @param feeds Map of input name to `ort.Tensor`.
1801
+ * @throws {@link InferenceError} if ORT raises any error during execution.
1844
1802
  */
1845
- constructor(targetWidth: number, targetHeight: number, mean?: readonly [number, number, number], std?: readonly [number, number, number]);
1846
- /** The `[width, height]` this pipeline resizes into. */
1847
- get targetSize(): readonly [number, number];
1803
+ private _runNow;
1804
+ /**
1805
+ * Rebuild any feed whose type does not match what its input declares.
1806
+ *
1807
+ * ORT matches feed types against the graph exactly: a float32 tensor against
1808
+ * a `half=True` export fails the run with `Unexpected input data type`. Every
1809
+ * preprocessing path in this SDK produces `Float32Array`, on purpose — the
1810
+ * normalization arithmetic belongs in single precision — so the conversion
1811
+ * happens here, once, at the only boundary all of them pass through. A feed
1812
+ * that already carries the declared type is handed on untouched, so a caller
1813
+ * building its own half tensor pays nothing.
1814
+ *
1815
+ * @param feeds The tensors to run with.
1816
+ * @returns The same mapping, with mismatched float32 feeds converted.
1817
+ */
1818
+ private _asDeclaredTypes;
1819
+ }
1820
+
1821
+ export declare interface OrtSessionOptions {
1848
1822
  /**
1849
- * Resize an image to the target size and write it as normalized planar float32.
1823
+ * Execution providers in preference order. `undefined` uses {@link DEFAULT_PROVIDERS}.
1850
1824
  *
1851
- * An image that already arrives at the target size skips the canvas entirely
1852
- * and is read straight out of its packed RGB — which is both faster and what
1853
- * keeps the result identical to `resize()`, whose own fast path returns the
1854
- * input untouched.
1825
+ * Each entry is a name (`"webgpu"`, `"wasm"`) or ORT's config object for that
1826
+ * provider, which passes provider options through untouched —
1827
+ * `{ name: "webgpu", preferredLayout: "NHWC" }`.
1855
1828
  *
1856
- * Buffer reuse follows {@link ReusableBuffer}: held across calls, replaced when
1857
- * a consumer detached it by transferring the tensor.
1858
- *
1859
- * @param image Source image in the SDK's canonical HWC RGB layout.
1829
+ * Naming one explicitly also opts into a `console.warn` when this browser
1830
+ * cannot offer it, instead of falling back in silence.
1860
1831
  */
1861
- run(image: RGBImage): FusedResizeResult;
1832
+ readonly providers?: readonly ProviderSpec[];
1862
1833
  /**
1863
- * Mark the reusable buffer free again.
1834
+ * Keep a URL model in the browser's Cache Storage between page loads.
1835
+ *
1836
+ * `true` uses the {@link DEFAULT_MODEL_CACHE} bucket; a string names the
1837
+ * bucket. A later `create` with the same URL reads the bytes from the cache
1838
+ * instead of the network — for a model of several megabytes, the bulk of
1839
+ * what a returning visitor waits for. Defaults to `false`.
1864
1840
  *
1865
- * Call it once the tensor built from a {@link run} result has been handed to
1866
- * ONNX Runtime and the run has resolved — after that the values are inside
1867
- * the WASM heap and the buffer can be overwritten.
1841
+ * The URL is the cache key and nothing expires it: publish a changed model
1842
+ * under a new URL (`yolo.v2.onnx`, `?v=2`) or a new bucket name, or delete
1843
+ * the bucket with `caches.delete(name)`. Ignored for models passed as bytes,
1844
+ * and silently skipped where Cache Storage is unavailable (non-secure
1845
+ * contexts, blocked site data).
1868
1846
  */
1869
- release(): void;
1847
+ readonly cache?: boolean | string;
1848
+ /** Optional ORT session options forwarded to `InferenceSession.create`. */
1849
+ readonly sessionOptions?: ort.InferenceSession.SessionOptions;
1870
1850
  /**
1871
- * Build the target canvas on first use.
1851
+ * Whether to read the model's custom metadata map (`names`, `task`, `imgsz`).
1852
+ * Defaults to `true`.
1872
1853
  *
1873
- * Lazily, so a pipeline constructed where no canvas implementation exists
1874
- * (a Node test, a worker without OffscreenCanvas) only fails if it is asked
1875
- * to resize something.
1854
+ * The runtime does not expose that map, so it is read from the file itself —
1855
+ * which means a URL model is fetched here and handed to ORT as bytes instead
1856
+ * of letting ORT fetch it. That is the same single download either way, and
1857
+ * it is what lets a task resolve its labels off the model. Set to `false` to
1858
+ * keep the URL path untouched (unless {@link cache} is set, which needs the
1859
+ * bytes) and leave {@link OrtSession.metadata} empty.
1860
+ *
1861
+ * `false` is also the escape hatch when a device cannot afford the bytes: the
1862
+ * fetched buffer is dropped before ORT builds the graph (see
1863
+ * {@link OrtSession.create}), but ORT's own load path still keeps the model out
1864
+ * of reach of anything the SDK holds. A session built this way resolves its
1865
+ * input size from the graph as usual — only the class names are lost, so a
1866
+ * caller taking this route has to pass `labels` itself.
1876
1867
  */
1877
- private _ensureTarget;
1878
- /** Grow the scratch source canvas to fit an image, reusing it when possible. */
1879
- private _ensureSource;
1868
+ readonly readMetadata?: boolean;
1880
1869
  }
1881
1870
 
1871
+ /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
1882
1872
  /**
1883
- * Resize an image into normalized planar float32 without keeping any state.
1884
- *
1885
- * The allocation-free path is {@link ResizePipeline}; this is the one-shot
1886
- * form, for a caller who wants the fused behaviour without owning a pipeline.
1873
+ * Exceptions raised by the SDK.
1887
1874
  *
1888
- * @param image Source image in the SDK's canonical HWC RGB layout.
1889
- * @param targetWidth Model input width in pixels.
1890
- * @param targetHeight Model input height in pixels.
1891
- * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.
1892
- * @param std Per-channel RGB standard deviation. Defaults to no scaling.
1875
+ * All exceptions inherit from {@link OrtVisionError}, so callers can catch
1876
+ * the base class to handle any SDK-originated failure uniformly.
1893
1877
  */
1894
- export declare function resizeToTensorData(image: RGBImage, targetWidth: number, targetHeight: number, mean?: readonly [number, number, number], std?: readonly [number, number, number]): FusedResizeResult;
1895
-
1896
- /** What {@link resolveNormalization} settled on. */
1897
- export declare interface ResolvedNormalization {
1898
- /** The preset name, or `"custom"` when the caller supplied the values. */
1899
- readonly name: string;
1900
- /** Per-channel mean to subtract. */
1901
- readonly mean: readonly [number, number, number];
1902
- /** Per-channel deviation to divide by. */
1903
- readonly std: readonly [number, number, number];
1878
+ export declare class OrtVisionError extends Error {
1879
+ constructor(message: string, options?: ErrorOptions);
1904
1880
  }
1905
1881
 
1906
1882
  /**
1907
- * Decide the input size a task will preprocess to.
1883
+ * Name of the `[K, 4]` float32 xyxy output, in **letterboxed** input pixels.
1908
1884
  *
1909
- * Precedence is graph → caller → fallback. The graph wins over an explicit
1910
- * `inputSize` because a static shape is not a preference, it is what ORT will
1911
- * accept: honoring the caller there would only turn a fixable mismatch into a
1912
- * failed run. A disagreement is a configuration bug in the caller, so it is
1913
- * reported through `console.warn` instead of being swallowed.
1914
- *
1915
- * @param options Graph shape, requested size and per-task fallback.
1916
- * @returns The `[width, height]` to preprocess to.
1885
+ * A file fused by `ort-vision-sdk` 0.9.0 or later reports the box that was
1886
+ * actually classified: clamped to the image the crop came from, exactly as
1887
+ * RoiAlign received it. Older files report the raw box, so one that ran off the
1888
+ * frame draws a rectangle wider than the region the classifier saw.
1917
1889
  */
1918
- export declare function resolveInputSize(options: ResolveInputSizeOptions): readonly [number, number];
1919
-
1920
- export declare interface ResolveInputSizeOptions {
1921
- /** Declared shape of the model's image input, from {@link declaredShapesFrom}. */
1922
- readonly graphShape?: DeclaredShape;
1923
- /** Size the caller asked for, if any. */
1924
- readonly requested?: readonly [number, number];
1925
- /** Size to use when neither the graph nor the caller pins one. */
1926
- readonly fallback: readonly [number, number];
1927
- }
1890
+ export declare const OUTPUT_BOXES = "boxes";
1928
1891
 
1929
- export declare function resolveLabels(spec: LabelSpec, options?: ResolveLabelsOptions): readonly string[];
1930
-
1931
- export declare interface ResolveLabelsOptions {
1932
- /**
1933
- * Expected number of classes.
1934
- *
1935
- * - When `spec` is `null`/`undefined`, this is required to auto-generate names.
1936
- * - When `spec` is provided, it validates that the resolved length matches.
1937
- */
1938
- readonly numClasses?: number;
1939
- }
1892
+ /** Name of the `[K]` int64 detector-class output. */
1893
+ export declare const OUTPUT_CLASSES = "classes";
1940
1894
 
1941
1895
  /**
1942
- * Settle which `mean`/`std` to apply, and what to call the choice.
1943
- *
1944
- * Explicit `mean`/`std` always win — they are the escape hatch for a model whose
1945
- * preprocessing neither preset describes. Anything they leave open falls back to
1946
- * the preset, so passing only a `mean` does not silently reset the deviation
1947
- * to 1.
1896
+ * Name of the `(K, 1, cropH, cropW)` float32 mask output. Only on a
1897
+ * `detect_segment_classify` pipeline.
1948
1898
  *
1949
- * Warns (via `console.warn`) when the model is an Ultralytics export and the
1950
- * supplied values are not the identity it was trained with. Nothing fails in
1951
- * that case: the prediction has the right shape and is simply less accurate,
1952
- * which is exactly why it is worth saying out loud.
1953
- *
1954
- * @param metadata The model's custom metadata map, read to detect the family.
1955
- * @param options The preset asked for, plus any explicit `mean`/`std`.
1956
- * @throws {RangeError} If `normalization` is not a known preset, or names one
1957
- * while `mean`/`std` are also supplied — two answers to the same question.
1899
+ * The mask lives in the **crop's** coordinate space, not the original image's,
1900
+ * because that is where the segmenter computed it. Values are 0 or 1 after
1901
+ * thresholding; padded rows are all-zero, like every other output.
1958
1902
  */
1959
- export declare function resolveNormalization(metadata: Readonly<Record<string, string>>, options?: {
1960
- readonly normalization?: Normalization;
1961
- readonly mean?: readonly [number, number, number];
1962
- readonly std?: readonly [number, number, number];
1963
- }): ResolvedNormalization;
1903
+ export declare const OUTPUT_MASKS = "masks";
1964
1904
 
1965
- /**
1966
- * Resolve the execution providers to pass to `InferenceSession.create`.
1967
- *
1968
- * @param requested Explicit provider list in preference order; `undefined` returns the default.
1969
- */
1970
- export declare function resolveProviders(requested?: readonly string[]): string[];
1905
+ /** Name of the `[1]` int64 output holding how many of the `K` rows are real. */
1906
+ export declare const OUTPUT_NUM_DETECTIONS = "num_detections";
1971
1907
 
1972
- /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
1973
- /**
1974
- * Public output types returned by the SDK's vision tasks.
1975
- *
1976
- * These types form the contract between the SDK and its callers. They mirror
1977
- * the Python `ort-vision-sdk` output dataclasses 1-to-1.
1978
- *
1979
- * Naming is intentionally compatible with the Ultralytics / torchvision idiom
1980
- * (`cls`, `conf`, `box`, `xyxy`, `xywh`, normalized variants) so code ported
1981
- * from those projects keeps working with minimal edits. The original verbose
1982
- * names (`classId`, `className`, `confidence`, `bbox`) are still populated for
1983
- * backwards compatibility.
1984
- */
1985
- /**
1986
- * HWC RGB uint8 image — the canonical image format used across the SDK.
1987
- *
1988
- * `data.length` must equal `width * height * 3`. The buffer is laid out row
1989
- * by row, top-to-bottom, with each pixel as `[R, G, B]`.
1990
- */
1991
- export declare class RGBImage {
1992
- readonly data: Uint8Array;
1993
- readonly width: number;
1994
- readonly height: number;
1995
- constructor(data: Uint8Array, width: number, height: number);
1996
- }
1908
+ export declare const OUTPUT_PROBS = "probs";
1997
1909
 
1998
- /**
1999
- * Single segmented instance produced by an instance-segmentation model.
2000
- *
2001
- * Mirrors {@link DetectionResult} and adds the per-instance binary mask
2002
- * plus a "ready-to-display" background-removed crop.
2003
- */
2004
- export declare interface SegmentationResult {
2005
- readonly classId: number;
2006
- readonly className: string;
2007
- readonly confidence: number;
2008
- readonly bbox: BoundingBox;
2009
- /** Alias for `classId` (Ultralytics-style). */
2010
- readonly cls: number;
2011
- /** Alias for `className`. */
2012
- readonly name: string;
2013
- /** Alias for `confidence` (Ultralytics-style). */
2014
- readonly conf: number;
2015
- /** Alias for `bbox` (Ultralytics-style). */
2016
- readonly box: BoundingBox;
2017
- /**
2018
- * Binary mask cropped to `bbox`. Values are `0` (background) or `255`
2019
- * (foreground). Empty boxes yield a zero-sized `Mask`.
2020
- */
2021
- readonly mask: Mask;
2022
- /**
2023
- * The original image cropped to `bbox` with background pixels (where
2024
- * `mask.data[i] === 0`) zeroed out. Empty boxes yield a zero-sized
2025
- * `RGBImage`.
2026
- */
2027
- readonly segmentedImage: RGBImage;
2028
- }
1910
+ /** Name of the `[K]` float32 detection-confidence output. */
1911
+ export declare const OUTPUT_SCORES = "scores";
2029
1912
 
2030
1913
  /**
2031
- * Per-image instance-segmentation envelope (Ultralytics-style `Results`).
1914
+ * Parse a `repr`-encoded `dict[int, str]` class map.
1915
+ *
1916
+ * Split out of {@link modelNames} because the same encoding is reused by a
1917
+ * fused pipeline, which carries one class map per stage and therefore cannot
1918
+ * store both under the single `names` key Ultralytics uses.
2032
1919
  *
2033
- * Iterating yields per-instance {@link SegmentationResult} entries. `boxes`
2034
- * and `masks` mirror Ultralytics' bulk-array views.
1920
+ * @param encoded The encoded map — e.g. `"{0: 'deworm', 1: 'not_deworm'}"`.
1921
+ * @returns Class names in class-id order, or `null` when the value is missing,
1922
+ * unparseable, not a `dict`, or not keyed by contiguous integers from zero.
2035
1923
  */
2036
- export declare class SegmentationResults implements Iterable<SegmentationResult> {
2037
- readonly boxes: Boxes;
2038
- readonly masks: Masks;
2039
- readonly detections: readonly SegmentationResult[];
2040
- readonly names: Readonly<Record<number, string>>;
2041
- readonly origImg: RGBImage;
2042
- readonly origShape: readonly [number, number];
2043
- readonly path: string | null;
2044
- readonly speed: Readonly<Speed>;
2045
- constructor(boxes: Boxes, masks: Masks, detections: readonly SegmentationResult[], names: Readonly<Record<number, string>>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly<Speed>);
2046
- /** Number of surviving instances. */
2047
- get length(): number;
2048
- /** Index into the per-instance results. */
2049
- get(index: number): SegmentationResult | undefined;
2050
- [Symbol.iterator](): Iterator<SegmentationResult>;
2051
- }
1924
+ export declare function parseNames(encoded: string | undefined): readonly string[] | null;
2052
1925
 
2053
1926
  /**
2054
- * Instance segmenter for YOLO seg ONNX models (v8-seg / v11-seg / ...).
2055
- *
2056
- * The model is expected to expose two outputs:
2057
- *
2058
- * 1. `output0`: `(1, 4 + numClasses + numMaskCoefs, numAnchors)` — per-anchor
2059
- * predictions (boxes, class scores, mask coefficients).
2060
- * 2. `output1`: `(1, numMaskCoefs, maskH, maskW)` — prototype masks.
1927
+ * Top-k classification probabilities for a single image.
2061
1928
  *
2062
- * `predict()` returns `Promise<SegmentationResults[]>` (length 1 for a
2063
- * single image), mirroring Ultralytics' API. The envelope exposes:
2064
- *
2065
- * - `boxes`: bulk numpy view (`xyxy`, `xywh`, `xyxyn`, `xywhn`, `cls`, `conf`).
2066
- * - `masks`: per-instance binary masks cropped to each box.
2067
- * - per-instance {@link SegmentationResult} via iteration.
2068
- *
2069
- * @example
2070
- * ```typescript
2071
- * const seg = await Segmenter.create("/models/yolov8n-seg.onnx");
2072
- * const r = (await seg.predict("/images/street.jpg"))[0];
2073
- * for (const inst of r) {
2074
- * console.log(inst.cls, inst.conf, inst.box.xyxy);
2075
- * }
2076
- * ```
1929
+ * Mirrors Ultralytics' `Probs` interface.
2077
1930
  */
2078
- export declare class Segmenter extends VisionTask {
2079
- private readonly _head;
2080
- private readonly _labels;
2081
- private readonly _names;
2082
- private readonly _inputSize;
2083
- private readonly _confThreshold;
2084
- private readonly _iouThreshold;
2085
- private readonly _maxDetections;
2086
- private readonly _maskThreshold;
2087
- private readonly _raiseOnEmpty;
2088
- private constructor();
2089
- private _pipelineCache;
1931
+ export declare class Probs {
1932
+ readonly data: Float32Array;
1933
+ /** @param data `[numClasses]` per-class probabilities, indexed by class id. */
1934
+ constructor(data: Float32Array);
1935
+ /** Number of classes. */
1936
+ get length(): number;
1937
+ /** `[numClasses]` shape of the underlying vector. */
1938
+ get shape(): readonly [number];
1939
+ /** Index of the most probable class. */
1940
+ get top1(): number;
1941
+ /** Probability of the top-1 class. */
1942
+ get top1conf(): number;
2090
1943
  /**
2091
- * Run the model once on a zero-filled tensor, paying one-time costs up front.
2092
- *
2093
- * The first inference of a session is not representative: WebGPU compiles its
2094
- * shaders on it and the WASM backend faults in its arenas, which on a phone
2095
- * can turn the first frame into seconds while every later frame is tens of
2096
- * milliseconds. Calling this while a loading spinner is still up moves that
2097
- * cost somewhere the user is already waiting.
1944
+ * Indices of the top-5 most probable classes, descending.
2098
1945
  *
2099
- * @param runs How many warm-up inferences to run. One is enough for WASM;
2100
- * WebGPU sometimes settles on the second.
1946
+ * The array is the memoised selection itself, not a copy — reading it twice
1947
+ * hands back the same object. Treat it as read-only: writing into it edits
1948
+ * what every later read of this `Probs` returns.
2101
1949
  */
2102
- warmup(runs?: number): Promise<void>;
1950
+ get top5(): Int32Array;
2103
1951
  /**
2104
- * The fused preprocessing pipeline, built on first use.
1952
+ * Probabilities of the top-5 classes, descending.
2105
1953
  *
2106
- * Lazily, because constructing it allocates canvases: a task built in an
2107
- * environment without a canvas implementation stays constructible, and only
2108
- * fails if it is actually asked to preprocess something.
1954
+ * Shares the memoised selection with {@link Probs.top5}, under the same
1955
+ * read-only caveat.
2109
1956
  */
2110
- private get _pipeline();
2111
- /** Load the model and resolve labels. */
2112
- static create(model: ModelSource, options?: SegmenterOptions): Promise<Segmenter>;
2113
- /** The decoder family used to interpret the model's output. */
2114
- get head(): SegmenterHead;
2115
- /** Class labels indexed by class id. */
2116
- get labels(): readonly string[];
2117
- /** Class id → class name dict (matches Ultralytics' `model.names`). */
2118
- get names(): Readonly<Record<number, string>>;
1957
+ get top5conf(): Float32Array;
1958
+ private _cache;
2119
1959
  /**
2120
- * The `[width, height]` this task preprocesses to.
1960
+ * Memoised {@link Probs._topK}.
2121
1961
  *
2122
- * Resolved at creation time from the model's graph when it declares a static
2123
- * input, so reading it back tells you the resolution inference really runs at
2124
- * — not merely what was requested.
2125
- */
2126
- get inputSize(): readonly [number, number];
2127
- /** Number of classes the model predicts. */
2128
- get numClasses(): number;
2129
- /** Alias for {@link predict} (parity with PyTorch `nn.Module.__call__`). */
2130
- call(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
2131
- /** Run instance segmentation on a single image. */
2132
- predict(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
2133
- private _preprocess;
2134
- private _splitOutputs;
2135
- private _buildResult;
2136
- private _buildBoxes;
2137
- private _buildMasks;
2138
- }
2139
-
2140
- /**
2141
- * Decoder family for the segmentation head.
2142
- *
2143
- * - `"yolo-seg"`: YOLO instance-segmentation head with two outputs —
2144
- * `[1, 4 + nc + nm, N]` per-anchor predictions plus `[1, nm, mh, mw]`
2145
- * prototype masks. Covers YOLOv8-seg, v11-seg, v26-seg.
2146
- *
2147
- * The SDK does **not** auto-detect this — the caller is responsible for
2148
- * picking a head that matches their export.
2149
- */
2150
- export declare type SegmenterHead = "yolo-seg";
2151
-
2152
- export declare interface SegmenterOptions extends OrtSessionOptions {
2153
- /**
2154
- * Decoder family for the segmentation head. Default `"yolo-seg"` covers
2155
- * YOLOv8-seg/v11-seg/v26-seg.
2156
- */
2157
- readonly head?: SegmenterHead;
2158
- /** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */
2159
- readonly labels?: LabelSpec;
2160
- /** Number of classes — used to validate the supplied labels. */
2161
- readonly numClasses?: number;
2162
- /**
2163
- * Model input `[width, height]` in pixels for letterboxing.
1962
+ * `top5` and `top5conf` are separate getters over the same selection, so a
1963
+ * caller reading both would otherwise pay for it twice — once per frame, in
1964
+ * a camera loop. The probabilities a `Probs` was built from do not change,
1965
+ * so the result is computed once per `k` and kept.
2164
1966
  *
2165
- * Only used when the model's graph leaves its spatial axes dynamic: a graph
2166
- * that declares a static size always wins, since that is the only shape ONNX
2167
- * Runtime will accept. Defaults to `[640, 640]`.
2168
- */
2169
- readonly inputSize?: readonly [number, number];
2170
- /** Default minimum class score to keep a candidate. */
2171
- readonly confThreshold?: number;
2172
- /** Default IoU threshold for non-maximum suppression. */
2173
- readonly iouThreshold?: number;
2174
- /** Maximum number of instances per image. */
2175
- readonly maxDetections?: number;
2176
- /**
2177
- * If `true`, a run that finds nothing throws {@link NoDetectionsError}
2178
- * instead of returning an empty envelope. Default `false`, because looking
2179
- * and finding nothing is a successful inference. Turn it on when an empty
2180
- * result means the surrounding pipeline should stop rather than carry on with
2181
- * zero rows. Can be overridden per `predict` call.
2182
- */
2183
- readonly raiseOnEmpty?: boolean;
2184
- /** Probability cutoff applied to soft masks. Defaults to `0.5`. */
2185
- readonly maskThreshold?: number;
2186
- }
2187
-
2188
- export declare interface SegmenterPredictOptions {
2189
- readonly confThreshold?: number;
2190
- readonly iouThreshold?: number;
2191
- /**
2192
- * If set, keep only instances whose `classId` is in this list.
2193
- * Mirrors Ultralytics' `model.predict(img, classes=[0, 16])`.
1967
+ * @param k - How many classes to select.
1968
+ * @returns The cached selection for that `k`.
2194
1969
  */
2195
- readonly classes?: readonly number[];
2196
- /** Override the constructor's `raiseOnEmpty` setting for this call. */
2197
- readonly raiseOnEmpty?: boolean;
2198
- }
2199
-
2200
- /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
2201
- /**
2202
- * Classification head postprocessing — softmax + top-k.
2203
- */
2204
- /** Apply numerically-stable softmax to a 1-D vector of logits. */
2205
- export declare function softmax(logits: Float32Array | readonly number[]): Float32Array;
2206
-
2207
- /**
2208
- * Read the spatial input size out of a declared NCHW shape.
2209
- *
2210
- * @param shape The declared shape of the model's image input.
2211
- * @returns `[width, height]` in pixels, or `null` when the shape is not 4D or
2212
- * leaves either spatial axis dynamic — in which case the model accepts more
2213
- * than one resolution and there is nothing to correct.
2214
- */
2215
- export declare function spatialInputSize(shape: DeclaredShape): readonly [number, number] | null;
2216
-
2217
- /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
2218
- /**
2219
- * Per-stage timing for a single `predict()` call.
2220
- *
2221
- * Populates the `speed` field every `Results` envelope carries, mirroring
2222
- * Ultralytics' `results[0].speed`. All values are milliseconds measured with
2223
- * `performance.now()`.
2224
- */
2225
- /**
2226
- * Stage durations of one inference, in milliseconds.
2227
- *
2228
- * `preprocess`, `inference` and `postprocess` are the three keys Ultralytics
2229
- * reports, measured over the same boundaries. `load` is specific to this SDK:
2230
- * `predict()` accepts a URL, `Blob` or DOM element and decodes it internally,
2231
- * so the fetch/decode cost would otherwise be invisible — and on a cold cache
2232
- * it dominates everything else.
2233
- */
2234
- export declare interface Speed {
2235
- /** Fetching and decoding the input into an `RGBImage`. */
2236
- load: number;
2237
- /** Letterbox/resize, normalization and tensor packing. */
2238
- preprocess: number;
2239
- /** The ONNX Runtime forward pass. */
2240
- inference: number;
2241
- /** Decoding raw outputs into results (NMS, mask assembly, top-k). */
2242
- postprocess: number;
2243
- }
2244
-
2245
- /**
2246
- * Accumulate stage durations while a `predict()` call runs.
2247
- *
2248
- * Each `stage()` call closes the previous stage: the elapsed time since the
2249
- * last boundary is attributed to the name given. This keeps the call sites
2250
- * free of paired start/stop bookkeeping and guarantees the four stages tile
2251
- * the whole call without gaps.
2252
- */
2253
- export declare class SpeedTimer {
2254
- private _last;
2255
- private readonly _speed;
2256
- constructor();
1970
+ private _top;
2257
1971
  /**
2258
- * Attribute the time elapsed since the previous boundary to `stage`.
1972
+ * Select the `k` highest probabilities without ordering the rest.
2259
1973
  *
2260
- * @param stage Which stage just finished.
2261
- */
2262
- stage(stage: keyof Speed): void;
2263
- /**
2264
- * The accumulated durations.
1974
+ * A full sort to read five entries out of a thousand-class vector costs
1975
+ * O(n log n) plus an index array the size of the vector; keeping `k` slots
1976
+ * ordered by insertion and scanning once costs O(n·k) with no allocation
1977
+ * beyond the result. Measured on 1000 classes over 2000 iterations, 134.5 µs
1978
+ * against 1.5 µs for the same output.
2265
1979
  *
2266
- * @returns The `speed` object to hand to the `Results` envelope.
1980
+ * Ties keep the lower class index first, matching the stable sort this
1981
+ * replaced: a candidate only displaces an entry it is strictly greater
1982
+ * than. That is also what keeps this selection in step with the Python
1983
+ * SDK's `np.argsort(-data, kind="stable")`, where the sort is C and the
1984
+ * cost this avoids does not arise.
1985
+ *
1986
+ * @param k - How many classes to select.
1987
+ * @returns Indices and probabilities, descending by probability.
2267
1988
  */
2268
- speed(): Speed;
1989
+ private _topK;
2269
1990
  }
2270
1991
 
2271
- /**
2272
- * Transpose interleaved HWC data to planar CHW layout.
2273
- *
2274
- * @param hwc Source array of length `width * height * channels`.
2275
- */
2276
- export declare function toCHW(hwc: Float32Array, width: number, height: number, channels?: number): Float32Array;
1992
+ /** Raised when a requested execution provider is not available. */
1993
+ export declare class ProviderNotAvailableError extends OrtVisionError {
1994
+ }
2277
1995
 
2278
1996
  /**
2279
- * Convert the SDK's HWC RGB image to an HWC BGR `Uint8Array` (OpenCV layout).
1997
+ * One execution provider: its name, or ORT's config object for it.
2280
1998
  *
2281
- * Useful for round-tripping data to a Python OpenCV consumer.
1999
+ * The object form carries provider options straight through to
2000
+ * `InferenceSession.create` — `{ name: "webgpu", preferredLayout: "NHWC" }`,
2001
+ * `{ name: "wasm" }` — which a bare name cannot express.
2282
2002
  */
2283
- export declare function toCv2(image: RGBImage): Uint8Array;
2284
-
2285
- /** Convert a uint8 image to a `Float32Array` in `[0, 1]` (HWC layout preserved). */
2286
- export declare function toFloat32(image: RGBImage, scale?: number): Float32Array;
2287
-
2288
- /** Wrap a Float32 buffer into an `ort.Tensor`. */
2289
- export declare function toFloat32Tensor(data: Float32Array, dims: readonly number[]): ort.Tensor;
2003
+ export declare type ProviderSpec = string | {
2004
+ readonly name: string;
2005
+ readonly [option: string]: unknown;
2006
+ };
2290
2007
 
2291
2008
  /**
2292
- * Return the top-k entries of a 1-D probability vector, sorted descending.
2009
+ * Metadata key the Python SDK's `quantize_model` writes on an INT8 model.
2293
2010
  *
2294
- * @param k Number of entries to return; `null` returns all entries.
2011
+ * Mirrored by `QUANTIZATION_KEY` in the Python SDK; both test suites pin the
2012
+ * string. See {@link keepQuantizedOffWebGpu} for what a session does with it.
2295
2013
  */
2296
- export declare function topK(probabilities: Float32Array, k: number | null): TopKResult;
2297
-
2298
- export declare interface TopKResult {
2299
- readonly indices: Int32Array;
2300
- readonly values: Float32Array;
2301
- }
2014
+ export declare const QUANTIZATION_KEY = "ort_vision_sdk.quantization";
2302
2015
 
2303
2016
  /**
2304
- * Convert an HWC uint8 image to a CHW `Float32Array` scaled to `[0, 1]`.
2017
+ * Read a pipeline spec out of a model's custom metadata.
2305
2018
  *
2306
- * Mirrors `torchvision.transforms.ToTensor()` semantics: HWC → CHW,
2307
- * `uint8 → float32 / 255`. Useful as input to YOLO-style detectors that
2308
- * don't require ImageNet normalization.
2019
+ * Individual malformed entries fall back to the value a fusion would have used
2020
+ * by default — a single bad float is not a reason to reject an otherwise
2021
+ * loadable pipeline. A malformed resolution is fatal, because there is no safe
2022
+ * default for one.
2309
2023
  *
2310
- * @returns CHW `Float32Array` of length `width * height * 3`.
2024
+ * @param metadata A model's custom metadata map, as read by
2025
+ * {@link readModelMetadata}.
2026
+ * @returns The decoded spec, or `null` when the model is not a fused pipeline —
2027
+ * it carries no `ovs.kind` entry, or one naming a pipeline kind this version
2028
+ * does not know how to drive.
2311
2029
  */
2312
- export declare function toTensor(image: RGBImage): Float32Array;
2030
+ export declare function readFusionSpec(metadata: Readonly<Record<string, string>> | undefined): FusionSpec | null;
2313
2031
 
2314
2032
  /**
2315
- * Acquire a `MediaStream` via `getUserMedia`, attach it to a `<video>` element,
2316
- * and expose status/error so the page can render permission and error states.
2317
- * The stream is automatically released on unmount or retry.
2318
- *
2319
- * Defaults to the rear ("environment") camera; desktops fall back to whatever
2320
- * single camera they expose. Pass `options.constraints` to override, or
2321
- * `options.enabled: false` to render the surface without prompting for the camera
2322
- * at all.
2033
+ * Read the element type each graph input declares.
2323
2034
  *
2324
- * Implementation notes:
2325
- * - Cleanup detaches the stream from a *snapshotted* video node, so it releases
2326
- * the same element it attached to even if the page remounts the `<video>`.
2327
- * - When `getUserMedia` is missing, an insecure context is the usual cause, so
2328
- * the hook prefers that (actionable) error; otherwise it reports `unsupported`.
2329
- * - `video.play()` rejections are swallowed: autoplay may be blocked, but the
2330
- * user gesture that opened the camera usually counts and playback resumes on
2331
- * the next interaction.
2035
+ * `onnxruntime-web` does not expose this. `session.inputMetadata` is
2036
+ * `undefined` on 1.20.1 — the same version where `declaredShapesFrom` comes
2037
+ * back empty — so the declared types have to come from the file, which the SDK
2038
+ * already downloads and walks for {@link readModelMetadata}. Same bytes, one
2039
+ * more pass, no extra request.
2332
2040
  *
2333
- * @param options - optional configuration (see {@link UseCameraStreamOptions}).
2334
- * @returns The stream status, classified error, a `videoRef` to attach, the live
2335
- * `stream` for whatever needs the track itself, and a `retry()` to re-attempt
2336
- * acquisition.
2041
+ * @param model The `.onnx` file contents.
2042
+ * @returns Input name → `TensorProto.DataType` value. An empty object when the
2043
+ * file carries no readable graph, which every caller treats as "assume
2044
+ * float32" — the behaviour before any of this existed.
2337
2045
  */
2338
- export declare function useCameraStream(options?: UseCameraStreamOptions): UseCameraStreamApi;
2339
-
2340
- /** Value returned by {@link useCameraStream}. */
2341
- export declare interface UseCameraStreamApi {
2342
- /** Current lifecycle status. */
2343
- status: CameraStreamStatus;
2344
- /** The classified error, or `null` while not in the `error` status. */
2345
- error: CameraStreamError | null;
2346
- /** Attach to a `<video ref={…} />`. The stream is wired to it once ready. */
2347
- videoRef: RefObject<HTMLVideoElement | null>;
2348
- /**
2349
- * The live stream, or `null`.
2350
- *
2351
- * Exposed for the things that need the **track** rather than the picture — the LED
2352
- * torch (`useTorch`), the real frame size from `getSettings()`, recording it with
2353
- * `useVideoRecorder`. Do not stop it yourself: the hook owns its lifetime and
2354
- * releases it on unmount and on `retry()`.
2355
- */
2356
- stream: MediaStream | null;
2357
- /** Manually re-attempt after an error (e.g. the user changed permissions). */
2358
- retry: () => void;
2359
- }
2360
-
2361
- /** Options for {@link useCameraStream}. */
2362
- export declare interface UseCameraStreamOptions {
2363
- /**
2364
- * Constraints passed to `getUserMedia`. Defaults to the rear
2365
- * ("environment") camera at Full-HD ideal resolution with audio off.
2366
- * Read when the stream (re)starts — change it and call `retry()` to apply.
2367
- */
2368
- constraints?: MediaStreamConstraints;
2369
- /**
2370
- * Hold off acquiring the camera until this is `true`. Default `true`.
2371
- *
2372
- * The point is to be able to *not* prompt. A permission prompt costs the user a
2373
- * decision and, if they refuse, costs the app the feature permanently — so a
2374
- * surface that already knows it cannot do its job (a barcode scanner in a browser
2375
- * with no decoder) must not open the camera to then say so. Flipping this to `false`
2376
- * also releases a stream that is already live.
2377
- */
2378
- enabled?: boolean;
2379
- }
2046
+ export declare function readModelInputTypes(model: Uint8Array | ArrayBufferLike): Readonly<Record<string, number>>;
2380
2047
 
2048
+ /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
2381
2049
  /**
2382
- * Sample mean luminance from a `<video>` source on a `requestAnimationFrame`
2383
- * loop and expose the rolling value. Sampling is throttled by `intervalMs` and
2384
- * paused whenever `enabled` is `false` or the video is not ready yet
2385
- * (`readyState < 2` or `videoWidth === 0`).
2050
+ * Read the metadata an exporter baked into a `.onnx` file.
2386
2051
  *
2387
- * One offscreen canvas is reused across frames to avoid GC pressure. Designed
2388
- * to feed a live brightness bar / border color on a camera page.
2052
+ * `onnxruntime-web` exposes input/output metadata but **not** the model's
2053
+ * custom metadata map, which is where Ultralytics writes `names`, `task` and
2054
+ * `imgsz`. The Python SDK gets it for free from
2055
+ * `InferenceSession.get_modelmeta().custom_metadata_map`; in the browser the
2056
+ * only way to the same information is to read it out of the file, so this
2057
+ * module walks just enough of the ModelProto wire format to collect
2058
+ * `metadata_props`.
2389
2059
  *
2390
- * @param videoRef - ref to the `<video>` element to sample.
2391
- * @param options - optional configuration (see {@link UseLiveLuminanceOptions}).
2392
- * @returns The rolling mean luminance in `0..255` (`0` until the first sample).
2060
+ * It never throws and never allocates unbounded: a truncated, hostile or
2061
+ * simply unexpected file yields an empty map, and every caller treats that as
2062
+ * "the model says nothing", falling back to what it was given.
2393
2063
  */
2394
- export declare function useLiveLuminance(videoRef: RefObject<HTMLVideoElement | null>, { enabled, intervalMs }?: UseLiveLuminanceOptions): number;
2395
-
2396
- /** Options for {@link useLiveLuminance}. */
2397
- export declare interface UseLiveLuminanceOptions {
2398
- /** When `false` the loop is paused (e.g. while a capture is in flight). Default: `true`. */
2399
- enabled?: boolean;
2400
- /** Throttle measurements in milliseconds. Default: `160` (~6 fps), plenty for UX. */
2401
- intervalMs?: number;
2402
- }
2403
-
2404
- export declare const VERSION: string;
2405
-
2406
- export declare abstract class VisionTask {
2407
- protected readonly _session: OrtSession;
2408
- protected constructor(_session: OrtSession);
2409
- /** The underlying {@link OrtSession} used to run inference. */
2410
- get session(): OrtSession;
2411
- }
2412
-
2413
2064
  /**
2414
- * Write an RGBA buffer as normalized planar float32.
2415
- *
2416
- * Shared by {@link ResizePipeline} and exported for a custom pipeline that
2417
- * already holds pixels and wants the SDK's exact arithmetic.
2065
+ * Collect a model's custom metadata map straight out of its bytes.
2418
2066
  *
2419
- * The expression is `(value / 255 - mean) / std`, evaluated in that order on
2420
- * purpose: folding it into a single multiply-add (`value * k + b`) computes the
2421
- * same quantity but rounds differently, and the tasks' output is asserted to be
2422
- * bit-identical to the composable `normalize` → `toCHW` path.
2423
- *
2424
- * @param rgba Source pixels, 4 bytes per pixel, `width * height` long.
2425
- * @param width Image width in pixels.
2426
- * @param height Image height in pixels.
2427
- * @param mean Per-channel RGB mean, already in `[0, 1]`.
2428
- * @param std Per-channel RGB standard deviation.
2429
- * @param out Destination buffer, `3 * width * height` long.
2430
- * @param stride Bytes per source pixel: 4 for canvas RGBA, 3 for packed RGB.
2067
+ * @param model The `.onnx` file contents.
2068
+ * @returns Key/value metadata — `names`, `task`, `imgsz`, ... for an
2069
+ * Ultralytics export — or an empty object when the file carries none or
2070
+ * cannot be walked.
2431
2071
  */
2432
- export declare function writePlanarFloat32(rgba: Uint8ClampedArray | Uint8Array, width: number, height: number, mean: readonly [number, number, number], std: readonly [number, number, number], out: Float32Array, stride?: number): void;
2072
+ export declare function readModelMetadata(model: Uint8Array | ArrayBufferLike): Readonly<Record<string, string>>;
2433
2073
 
2434
2074
  /**
2435
- * Build a zero-filled CHW tensor payload for a warm-up run.
2075
+ * Turn an empty result into an error, when the caller asked for that.
2436
2076
  *
2437
- * @param width Model input width in pixels.
2438
- * @param height Model input height in pixels.
2439
- */
2440
- export declare function zeroTensorData(width: number, height: number): Float32Array;
2441
-
2442
- export { }
2077
+ * Shared by every task that can come back with nothing — {@link Detector},
2078
+ * {@link Segmenter} and {@link DetectClassify} — so the three agree on when
2079
+ * they throw and on what the message says. The message names the two settings
2080
+ * that decide the outcome, because "no detections" on its own leaves the reader
2081
+ * unable to tell a blank image from a threshold set too high.
2082
+ *
2083
+ * @param count How many detections survived every filter.
2084
+ * @param options The flag for this call, the threshold actually applied (after
2085
+ * any per-call override), the class allowlist if one narrowed the search, and
2086
+ * the source path when the input was one.
2087
+ * @throws {@link NoDetectionsError} when the flag is set and `count` is zero.
2088
+ */
2089
+ export declare function requireDetections(count: number, options: {
2090
+ readonly raiseOnEmpty: boolean;
2091
+ readonly confThreshold: number;
2092
+ readonly classes: readonly number[] | undefined;
2093
+ readonly path: string | null;
2094
+ }): void;
2095
+
2096
+ /** Resize an image to `(targetWidth, targetHeight)` using high-quality canvas resampling. */
2097
+ export declare function resize(image: RGBImage, targetWidth: number, targetHeight: number): RGBImage;
2098
+
2099
+ /**
2100
+ * Reusable stretch-resize → normalized tensor pipeline for one target size.
2101
+ *
2102
+ * The classification counterpart of {@link LetterboxPipeline}. A classifier
2103
+ * stretches to the model's square input instead of letterboxing into it — no
2104
+ * padding, no scale to invert later, because nothing is mapped back onto the
2105
+ * source image afterwards. That difference is why it cannot simply reuse the
2106
+ * letterbox path.
2107
+ *
2108
+ * What it does share is the technique. The composable route
2109
+ * (`resize` → `normalize` → `toCHW`) allocates an `RGBImage` and two
2110
+ * `Float32Array`s and walks each end to end on every call: about 1.4 MB of
2111
+ * fresh garbage per 224×224 `predict()`, produced at the exact moment a phone
2112
+ * near its memory ceiling can least afford it. Here one `drawImage` resizes,
2113
+ * and one loop reads the resulting RGBA and writes normalized planar float32
2114
+ * into a buffer held across calls.
2115
+ *
2116
+ * Create one per task, not per frame.
2117
+ */
2118
+ export declare class ResizePipeline {
2119
+ private readonly _targetWidth;
2120
+ private readonly _targetHeight;
2121
+ private readonly _mean;
2122
+ private readonly _std;
2123
+ private readonly _buffer;
2124
+ private _target;
2125
+ private _targetContext;
2126
+ private _source;
2127
+ private _sourceContext;
2128
+ /**
2129
+ * @param targetWidth Model input width in pixels.
2130
+ * @param targetHeight Model input height in pixels.
2131
+ * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.
2132
+ * @param std Per-channel RGB standard deviation. Defaults to no scaling.
2133
+ */
2134
+ constructor(targetWidth: number, targetHeight: number, mean?: readonly [number, number, number], std?: readonly [number, number, number]);
2135
+ /** The `[width, height]` this pipeline resizes into. */
2136
+ get targetSize(): readonly [number, number];
2137
+ /**
2138
+ * Resize an image to the target size and write it as normalized planar float32.
2139
+ *
2140
+ * An image that already arrives at the target size skips the canvas entirely
2141
+ * and is read straight out of its packed RGB — which is both faster and what
2142
+ * keeps the result identical to `resize()`, whose own fast path returns the
2143
+ * input untouched.
2144
+ *
2145
+ * Buffer reuse follows {@link ReusableBuffer}: held across calls, replaced when
2146
+ * a consumer detached it by transferring the tensor.
2147
+ *
2148
+ * @param image Source image in the SDK's canonical HWC RGB layout.
2149
+ * @param canvas Optional opaque drawable holding exactly `image`'s pixels —
2150
+ * see {@link LetterboxPipeline.run}. Ignored on the same-size fast path,
2151
+ * which never touches a canvas.
2152
+ */
2153
+ run(image: RGBImage, canvas?: CanvasImageSource | null): FusedResizeResult;
2154
+ /**
2155
+ * Mark the reusable buffer free again.
2156
+ *
2157
+ * Call it once the tensor built from a {@link run} result has been handed to
2158
+ * ONNX Runtime and the run has resolved — after that the values are inside
2159
+ * the WASM heap and the buffer can be overwritten.
2160
+ *
2161
+ * Pass the result being released when several runs can be outstanding at
2162
+ * once. A result that got a fresh allocation (`reused: false`) holds nothing
2163
+ * of the pipeline's, so releasing it is a no-op; without that check, the
2164
+ * caller finishing with a fresh buffer would free the shared one while
2165
+ * another caller still had its frame in it. Called without an argument it
2166
+ * frees the shared buffer unconditionally, as before.
2167
+ *
2168
+ * @param result The {@link run} result being released.
2169
+ */
2170
+ release(result?: {
2171
+ readonly reused: boolean;
2172
+ }): void;
2173
+ /**
2174
+ * Build the target canvas on first use.
2175
+ *
2176
+ * Lazily, so a pipeline constructed where no canvas implementation exists
2177
+ * (a Node test, a worker without OffscreenCanvas) only fails if it is asked
2178
+ * to resize something.
2179
+ */
2180
+ private _ensureTarget;
2181
+ /**
2182
+ * Paint an image onto the scratch source canvas and return that canvas.
2183
+ *
2184
+ * @param image Source image in the SDK's canonical HWC RGB layout.
2185
+ */
2186
+ private _paintSource;
2187
+ }
2188
+
2189
+ /**
2190
+ * Resize an image into normalized planar float32 without keeping any state.
2191
+ *
2192
+ * The allocation-free path is {@link ResizePipeline}; this is the one-shot
2193
+ * form, for a caller who wants the fused behaviour without owning a pipeline.
2194
+ *
2195
+ * @param image Source image in the SDK's canonical HWC RGB layout.
2196
+ * @param targetWidth Model input width in pixels.
2197
+ * @param targetHeight Model input height in pixels.
2198
+ * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.
2199
+ * @param std Per-channel RGB standard deviation. Defaults to no scaling.
2200
+ */
2201
+ export declare function resizeToTensorData(image: RGBImage, targetWidth: number, targetHeight: number, mean?: readonly [number, number, number], std?: readonly [number, number, number]): FusedResizeResult;
2202
+
2203
+ /** What {@link resolveNormalization} settled on. */
2204
+ export declare interface ResolvedNormalization {
2205
+ /** The preset name, or `"custom"` when the caller supplied the values. */
2206
+ readonly name: string;
2207
+ /** Per-channel mean to subtract. */
2208
+ readonly mean: readonly [number, number, number];
2209
+ /** Per-channel deviation to divide by. */
2210
+ readonly std: readonly [number, number, number];
2211
+ }
2212
+
2213
+ /**
2214
+ * Decide the input size a task will preprocess to.
2215
+ *
2216
+ * Precedence is graph → caller → fallback. The graph wins over an explicit
2217
+ * `inputSize` because a static shape is not a preference, it is what ORT will
2218
+ * accept: honoring the caller there would only turn a fixable mismatch into a
2219
+ * failed run. A disagreement is a configuration bug in the caller, so it is
2220
+ * reported through `console.warn` instead of being swallowed.
2221
+ *
2222
+ * @param options Graph shape, requested size and per-task fallback.
2223
+ * @returns The `[width, height]` to preprocess to.
2224
+ */
2225
+ export declare function resolveInputSize(options: ResolveInputSizeOptions): readonly [number, number];
2226
+
2227
+ export declare interface ResolveInputSizeOptions {
2228
+ /** Declared shape of the model's image input, from {@link declaredShapesFrom}. */
2229
+ readonly graphShape?: DeclaredShape;
2230
+ /** Size the caller asked for, if any. */
2231
+ readonly requested?: readonly [number, number];
2232
+ /** Size to use when neither the graph nor the caller pins one. */
2233
+ readonly fallback: readonly [number, number];
2234
+ }
2235
+
2236
+ export declare function resolveLabels(spec: LabelSpec, options?: ResolveLabelsOptions): readonly string[];
2237
+
2238
+ export declare interface ResolveLabelsOptions {
2239
+ /**
2240
+ * Expected number of classes.
2241
+ *
2242
+ * - When `spec` is `null`/`undefined`, this is required to auto-generate names.
2243
+ * - When `spec` is provided, it validates that the resolved length matches.
2244
+ */
2245
+ readonly numClasses?: number;
2246
+ }
2247
+
2248
+ /**
2249
+ * Settle which `mean`/`std` to apply, and what to call the choice.
2250
+ *
2251
+ * Explicit `mean`/`std` always win — they are the escape hatch for a model whose
2252
+ * preprocessing neither preset describes. Anything they leave open falls back to
2253
+ * the preset, so passing only a `mean` does not silently reset the deviation
2254
+ * to 1.
2255
+ *
2256
+ * Warns (via `console.warn`) when the model is an Ultralytics export and the
2257
+ * supplied values are not the identity it was trained with. Nothing fails in
2258
+ * that case: the prediction has the right shape and is simply less accurate,
2259
+ * which is exactly why it is worth saying out loud.
2260
+ *
2261
+ * @param metadata The model's custom metadata map, read to detect the family.
2262
+ * @param options The preset asked for, plus any explicit `mean`/`std`.
2263
+ * @throws {RangeError} If `normalization` is not a known preset, or names one
2264
+ * while `mean`/`std` are also supplied — two answers to the same question.
2265
+ */
2266
+ export declare function resolveNormalization(metadata: Readonly<Record<string, string>>, options?: {
2267
+ readonly normalization?: Normalization;
2268
+ readonly mean?: readonly [number, number, number];
2269
+ readonly std?: readonly [number, number, number];
2270
+ }): ResolvedNormalization;
2271
+
2272
+ /**
2273
+ * Resolve the execution providers to pass to `InferenceSession.create`.
2274
+ *
2275
+ * @param requested Explicit provider list in preference order, as names or
2276
+ * config objects; `undefined` or empty returns the default.
2277
+ */
2278
+ export declare function resolveProviders<T extends ProviderSpec = string>(requested?: readonly T[]): (T | string)[];
2279
+
2280
+ /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
2281
+ /**
2282
+ * Public output types returned by the SDK's vision tasks.
2283
+ *
2284
+ * These types form the contract between the SDK and its callers. They mirror
2285
+ * the Python `ort-vision-sdk` output dataclasses 1-to-1.
2286
+ *
2287
+ * Naming is intentionally compatible with the Ultralytics / torchvision idiom
2288
+ * (`cls`, `conf`, `box`, `xyxy`, `xywh`, normalized variants) so code ported
2289
+ * from those projects keeps working with minimal edits. The original verbose
2290
+ * names (`classId`, `className`, `confidence`, `bbox`) are still populated for
2291
+ * backwards compatibility.
2292
+ */
2293
+ /**
2294
+ * HWC RGB uint8 image — the canonical image format used across the SDK.
2295
+ *
2296
+ * `data.length` must equal `width * height * 3`. The buffer is laid out row
2297
+ * by row, top-to-bottom, with each pixel as `[R, G, B]`.
2298
+ *
2299
+ * The pixels can also be produced on first read — see {@link RGBImage.deferred}.
2300
+ * Reading `data` is the same either way.
2301
+ */
2302
+ export declare class RGBImage {
2303
+ readonly width: number;
2304
+ readonly height: number;
2305
+ private _data;
2306
+ private _materialize;
2307
+ constructor(data: Uint8Array, width: number, height: number);
2308
+ /**
2309
+ * An image whose pixels are computed the first time `data` is read.
2310
+ *
2311
+ * Used by the decoder for frames it can hand to preprocessing without
2312
+ * reading back: a 1080p `getImageData` plus the RGBA→RGB copy is most of
2313
+ * the `load` stage (9.4 ms vs 1.9 ms for the GPU-side copy, measured in
2314
+ * Chromium), and a camera loop that only reads boxes never needs it.
2315
+ *
2316
+ * @param width Width in pixels.
2317
+ * @param height Height in pixels.
2318
+ * @param materialize Produces the `width * height * 3` buffer; called at most
2319
+ * once, and must keep returning the pixels of the moment the image was
2320
+ * made — snapshot whatever it reads from.
2321
+ * @returns The image. Built without the constructor, which would need the
2322
+ * very buffer this defers.
2323
+ */
2324
+ static deferred(width: number, height: number, materialize: () => Uint8Array): RGBImage;
2325
+ /**
2326
+ * The HWC RGB pixels, materialized on first read for a deferred image.
2327
+ *
2328
+ * @throws {@link ImageLoadError} if a deferred image's buffer has the wrong length.
2329
+ */
2330
+ get data(): Uint8Array;
2331
+ /**
2332
+ * Validate a buffer's length against the image size.
2333
+ *
2334
+ * @param data The pixels.
2335
+ * @param width Width in pixels.
2336
+ * @param height Height in pixels.
2337
+ * @throws {@link ImageLoadError} if `data.length !== width * height * 3`.
2338
+ */
2339
+ private static _check;
2340
+ }
2341
+
2342
+ /**
2343
+ * Single segmented instance produced by an instance-segmentation model.
2344
+ *
2345
+ * Mirrors {@link DetectionResult} and adds the per-instance binary mask
2346
+ * plus a "ready-to-display" background-removed crop.
2347
+ */
2348
+ export declare interface SegmentationResult {
2349
+ readonly classId: number;
2350
+ readonly className: string;
2351
+ readonly confidence: number;
2352
+ readonly bbox: BoundingBox;
2353
+ /** Alias for `classId` (Ultralytics-style). */
2354
+ readonly cls: number;
2355
+ /** Alias for `className`. */
2356
+ readonly name: string;
2357
+ /** Alias for `confidence` (Ultralytics-style). */
2358
+ readonly conf: number;
2359
+ /** Alias for `bbox` (Ultralytics-style). */
2360
+ readonly box: BoundingBox;
2361
+ /**
2362
+ * Binary mask cropped to `bbox`. Values are `0` (background) or `255`
2363
+ * (foreground). Empty boxes yield a zero-sized `Mask`.
2364
+ */
2365
+ readonly mask: Mask;
2366
+ /**
2367
+ * The original image cropped to `bbox` with background pixels (where
2368
+ * `mask.data[i] === 0`) zeroed out. Empty boxes yield a zero-sized
2369
+ * `RGBImage`.
2370
+ */
2371
+ readonly segmentedImage: RGBImage;
2372
+ }
2373
+
2374
+ /**
2375
+ * Per-image instance-segmentation envelope (Ultralytics-style `Results`).
2376
+ *
2377
+ * Iterating yields per-instance {@link SegmentationResult} entries. `boxes`
2378
+ * and `masks` mirror Ultralytics' bulk-array views.
2379
+ */
2380
+ export declare class SegmentationResults implements Iterable<SegmentationResult> {
2381
+ readonly boxes: Boxes;
2382
+ readonly masks: Masks;
2383
+ readonly detections: readonly SegmentationResult[];
2384
+ readonly names: Readonly<Record<number, string>>;
2385
+ readonly origImg: RGBImage;
2386
+ readonly origShape: readonly [number, number];
2387
+ readonly path: string | null;
2388
+ readonly speed: Readonly<Speed>;
2389
+ constructor(boxes: Boxes, masks: Masks, detections: readonly SegmentationResult[], names: Readonly<Record<number, string>>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly<Speed>);
2390
+ /** Number of surviving instances. */
2391
+ get length(): number;
2392
+ /** Index into the per-instance results. */
2393
+ get(index: number): SegmentationResult | undefined;
2394
+ [Symbol.iterator](): Iterator<SegmentationResult>;
2395
+ }
2396
+
2397
+ /**
2398
+ * Instance segmenter for YOLO seg ONNX models (v8-seg / v11-seg / ...).
2399
+ *
2400
+ * The model is expected to expose two outputs:
2401
+ *
2402
+ * 1. `output0`: `(1, 4 + numClasses + numMaskCoefs, numAnchors)` — per-anchor
2403
+ * predictions (boxes, class scores, mask coefficients).
2404
+ * 2. `output1`: `(1, numMaskCoefs, maskH, maskW)` — prototype masks.
2405
+ *
2406
+ * `predict()` returns `Promise<SegmentationResults[]>` (length 1 for a
2407
+ * single image), mirroring Ultralytics' API. The envelope exposes:
2408
+ *
2409
+ * - `boxes`: bulk numpy view (`xyxy`, `xywh`, `xyxyn`, `xywhn`, `cls`, `conf`).
2410
+ * - `masks`: per-instance binary masks cropped to each box.
2411
+ * - per-instance {@link SegmentationResult} via iteration.
2412
+ *
2413
+ * @example
2414
+ * ```typescript
2415
+ * const seg = await Segmenter.create("/models/yolov8n-seg.onnx");
2416
+ * const r = (await seg.predict("/images/street.jpg"))[0];
2417
+ * for (const inst of r) {
2418
+ * console.log(inst.cls, inst.conf, inst.box.xyxy);
2419
+ * }
2420
+ * ```
2421
+ */
2422
+ export declare class Segmenter extends VisionTask {
2423
+ private readonly _head;
2424
+ private readonly _labels;
2425
+ private readonly _names;
2426
+ private readonly _inputSize;
2427
+ private readonly _confThreshold;
2428
+ private readonly _iouThreshold;
2429
+ private readonly _maxDetections;
2430
+ private readonly _maskThreshold;
2431
+ private readonly _raiseOnEmpty;
2432
+ private constructor();
2433
+ private _pipelineCache;
2434
+ /**
2435
+ * Run the model once on a zero-filled tensor, paying one-time costs up front.
2436
+ *
2437
+ * The first inference of a session is not representative: WebGPU compiles its
2438
+ * shaders on it and the WASM backend faults in its arenas, which on a phone
2439
+ * can turn the first frame into seconds while every later frame is tens of
2440
+ * milliseconds. Calling this while a loading spinner is still up moves that
2441
+ * cost somewhere the user is already waiting.
2442
+ *
2443
+ * @param runs How many warm-up inferences to run. One is enough for WASM;
2444
+ * WebGPU sometimes settles on the second.
2445
+ */
2446
+ warmup(runs?: number): Promise<void>;
2447
+ /**
2448
+ * The fused preprocessing pipeline, built on first use.
2449
+ *
2450
+ * Lazily, because constructing it allocates canvases: a task built in an
2451
+ * environment without a canvas implementation stays constructible, and only
2452
+ * fails if it is actually asked to preprocess something.
2453
+ */
2454
+ private get _pipeline();
2455
+ /** Load the model and resolve labels. */
2456
+ static create(model: ModelSource, options?: SegmenterOptions): Promise<Segmenter>;
2457
+ /** The decoder family used to interpret the model's output. */
2458
+ get head(): SegmenterHead;
2459
+ /** Class labels indexed by class id. */
2460
+ get labels(): readonly string[];
2461
+ /** Class id → class name dict (matches Ultralytics' `model.names`). */
2462
+ get names(): Readonly<Record<number, string>>;
2463
+ /**
2464
+ * The `[width, height]` this task preprocesses to.
2465
+ *
2466
+ * Resolved at creation time from the model's graph when it declares a static
2467
+ * input, so reading it back tells you the resolution inference really runs at
2468
+ * — not merely what was requested.
2469
+ */
2470
+ get inputSize(): readonly [number, number];
2471
+ /** Number of classes the model predicts. */
2472
+ get numClasses(): number;
2473
+ /** Alias for {@link predict} (parity with PyTorch `nn.Module.__call__`). */
2474
+ call(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
2475
+ /** Run instance segmentation on a single image. */
2476
+ predict(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
2477
+ /**
2478
+ * Letterbox and pack the image into the tensor the model expects.
2479
+ *
2480
+ * @param image The decoded input.
2481
+ * @param canvas The opaque canvas it was decoded on, when there is one — see
2482
+ * {@link LetterboxPipeline.run}.
2483
+ */
2484
+ private _preprocess;
2485
+ private _splitOutputs;
2486
+ /**
2487
+ * Assemble one instance.
2488
+ *
2489
+ * The mask is trimmed to the part of the box inside the frame up front — it
2490
+ * is small and every caller reads it. `segmentedImage`, a masked copy of the
2491
+ * frame's pixels, is built on first read when the SDK owns those pixels and
2492
+ * up front otherwise, for the reason {@link Detector} gives for its crops.
2493
+ *
2494
+ * @param original The decoded input.
2495
+ * @param bbox Box in original-image coordinates.
2496
+ * @param classId Predicted class.
2497
+ * @param confidence Detection score.
2498
+ * @param mask Binary mask shaped to `bbox`.
2499
+ * @param lazy Whether `segmentedImage` may be deferred.
2500
+ */
2501
+ private _buildResult;
2502
+ private _buildBoxes;
2503
+ private _buildMasks;
2504
+ }
2505
+
2506
+ /**
2507
+ * Decoder family for the segmentation head.
2508
+ *
2509
+ * - `"yolo-seg"`: YOLO instance-segmentation head with two outputs —
2510
+ * `[1, 4 + nc + nm, N]` per-anchor predictions plus `[1, nm, mh, mw]`
2511
+ * prototype masks. Covers YOLOv8-seg, v11-seg, v26-seg.
2512
+ *
2513
+ * The SDK does **not** auto-detect this — the caller is responsible for
2514
+ * picking a head that matches their export.
2515
+ */
2516
+ export declare type SegmenterHead = "yolo-seg";
2517
+
2518
+ export declare interface SegmenterOptions extends OrtSessionOptions {
2519
+ /**
2520
+ * Decoder family for the segmentation head. Default `"yolo-seg"` covers
2521
+ * YOLOv8-seg/v11-seg/v26-seg.
2522
+ */
2523
+ readonly head?: SegmenterHead;
2524
+ /** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */
2525
+ readonly labels?: LabelSpec;
2526
+ /** Number of classes — used to validate the supplied labels. */
2527
+ readonly numClasses?: number;
2528
+ /**
2529
+ * Model input `[width, height]` in pixels for letterboxing.
2530
+ *
2531
+ * Only used when the model's graph leaves its spatial axes dynamic: a graph
2532
+ * that declares a static size always wins, since that is the only shape ONNX
2533
+ * Runtime will accept. Defaults to `[640, 640]`.
2534
+ */
2535
+ readonly inputSize?: readonly [number, number];
2536
+ /** Default minimum class score to keep a candidate. */
2537
+ readonly confThreshold?: number;
2538
+ /** Default IoU threshold for non-maximum suppression. */
2539
+ readonly iouThreshold?: number;
2540
+ /** Maximum number of instances per image. */
2541
+ readonly maxDetections?: number;
2542
+ /**
2543
+ * If `true`, a run that finds nothing throws {@link NoDetectionsError}
2544
+ * instead of returning an empty envelope. Default `false`, because looking
2545
+ * and finding nothing is a successful inference. Turn it on when an empty
2546
+ * result means the surrounding pipeline should stop rather than carry on with
2547
+ * zero rows. Can be overridden per `predict` call.
2548
+ */
2549
+ readonly raiseOnEmpty?: boolean;
2550
+ /** Probability cutoff applied to soft masks. Defaults to `0.5`. */
2551
+ readonly maskThreshold?: number;
2552
+ }
2553
+
2554
+ export declare interface SegmenterPredictOptions {
2555
+ readonly confThreshold?: number;
2556
+ readonly iouThreshold?: number;
2557
+ /**
2558
+ * If set, keep only instances whose `classId` is in this list.
2559
+ * Mirrors Ultralytics' `model.predict(img, classes=[0, 16])`.
2560
+ */
2561
+ readonly classes?: readonly number[];
2562
+ /** Override the constructor's `raiseOnEmpty` setting for this call. */
2563
+ readonly raiseOnEmpty?: boolean;
2564
+ }
2565
+
2566
+ /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
2567
+ /**
2568
+ * Classification head postprocessing — softmax + top-k.
2569
+ */
2570
+ /** Apply numerically-stable softmax to a 1-D vector of logits. */
2571
+ export declare function softmax(logits: Float32Array | readonly number[]): Float32Array;
2572
+
2573
+ /**
2574
+ * Read the spatial input size out of a declared NCHW shape.
2575
+ *
2576
+ * @param shape The declared shape of the model's image input.
2577
+ * @returns `[width, height]` in pixels, or `null` when the shape is not 4D or
2578
+ * leaves either spatial axis dynamic — in which case the model accepts more
2579
+ * than one resolution and there is nothing to correct.
2580
+ */
2581
+ export declare function spatialInputSize(shape: DeclaredShape): readonly [number, number] | null;
2582
+
2583
+ /** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
2584
+ /**
2585
+ * Per-stage timing for a single `predict()` call.
2586
+ *
2587
+ * Populates the `speed` field every `Results` envelope carries, mirroring
2588
+ * Ultralytics' `results[0].speed`. All values are milliseconds measured with
2589
+ * `performance.now()`.
2590
+ */
2591
+ /**
2592
+ * Stage durations of one inference, in milliseconds.
2593
+ *
2594
+ * `preprocess`, `inference` and `postprocess` are the three keys Ultralytics
2595
+ * reports, measured over the same boundaries. `load` is specific to this SDK:
2596
+ * `predict()` accepts a URL, `Blob` or DOM element and decodes it internally,
2597
+ * so the fetch/decode cost would otherwise be invisible — and on a cold cache
2598
+ * it dominates everything else.
2599
+ */
2600
+ export declare interface Speed {
2601
+ /** Fetching and decoding the input into an `RGBImage`. */
2602
+ load: number;
2603
+ /** Letterbox/resize, normalization and tensor packing. */
2604
+ preprocess: number;
2605
+ /** The ONNX Runtime forward pass. */
2606
+ inference: number;
2607
+ /** Decoding raw outputs into results (NMS, mask assembly, top-k). */
2608
+ postprocess: number;
2609
+ }
2610
+
2611
+ /**
2612
+ * Accumulate stage durations while a `predict()` call runs.
2613
+ *
2614
+ * Each `stage()` call closes the previous stage: the elapsed time since the
2615
+ * last boundary is attributed to the name given. This keeps the call sites
2616
+ * free of paired start/stop bookkeeping and guarantees the four stages tile
2617
+ * the whole call without gaps.
2618
+ */
2619
+ export declare class SpeedTimer {
2620
+ private _last;
2621
+ private readonly _speed;
2622
+ constructor();
2623
+ /**
2624
+ * Attribute the time elapsed since the previous boundary to `stage`.
2625
+ *
2626
+ * @param stage Which stage just finished.
2627
+ */
2628
+ stage(stage: keyof Speed): void;
2629
+ /**
2630
+ * The accumulated durations.
2631
+ *
2632
+ * @returns The `speed` object to hand to the `Results` envelope.
2633
+ */
2634
+ speed(): Speed;
2635
+ }
2636
+
2637
+ /**
2638
+ * Name the tensor type an ONNX element type maps to.
2639
+ *
2640
+ * @param elemType A `TensorProto.DataType` value read off the graph.
2641
+ * @returns The ORT tensor type name, or `"float32"` for anything unrecognised —
2642
+ * which is what every task produces anyway, so an unknown type surfaces as
2643
+ * ORT's own type error against the real graph rather than as a lookup miss.
2644
+ */
2645
+ export declare function tensorTypeFor(elemType: number | undefined): string;
2646
+
2647
+ /**
2648
+ * Transpose interleaved HWC data to planar CHW layout.
2649
+ *
2650
+ * @param hwc Source array of length `width * height * channels`.
2651
+ */
2652
+ export declare function toCHW(hwc: Float32Array, width: number, height: number, channels?: number): Float32Array;
2653
+
2654
+ /**
2655
+ * Convert the SDK's HWC RGB image to an HWC BGR `Uint8Array` (OpenCV layout).
2656
+ *
2657
+ * Useful for round-tripping data to a Python OpenCV consumer.
2658
+ */
2659
+ export declare function toCv2(image: RGBImage): Uint8Array;
2660
+
2661
+ /** Convert a uint8 image to a `Float32Array` in `[0, 1]` (HWC layout preserved). */
2662
+ export declare function toFloat32(image: RGBImage, scale?: number): Float32Array;
2663
+
2664
+ /** Wrap a Float32 buffer into an `ort.Tensor`. */
2665
+ export declare function toFloat32Tensor(data: Float32Array, dims: readonly number[]): ort.Tensor;
2666
+
2667
+ /**
2668
+ * Return the top-k entries of a 1-D probability vector, sorted descending.
2669
+ *
2670
+ * @param k Number of entries to return; `null` returns all entries.
2671
+ */
2672
+ export declare function topK(probabilities: Float32Array, k: number | null): TopKResult;
2673
+
2674
+ export declare interface TopKResult {
2675
+ readonly indices: Int32Array;
2676
+ readonly values: Float32Array;
2677
+ }
2678
+
2679
+ /**
2680
+ * Convert an HWC uint8 image to a CHW `Float32Array` scaled to `[0, 1]`.
2681
+ *
2682
+ * Mirrors `torchvision.transforms.ToTensor()` semantics: HWC → CHW,
2683
+ * `uint8 → float32 / 255`. Useful as input to YOLO-style detectors that
2684
+ * don't require ImageNet normalization.
2685
+ *
2686
+ * @returns CHW `Float32Array` of length `width * height * 3`.
2687
+ */
2688
+ export declare function toTensor(image: RGBImage): Float32Array;
2689
+
2690
+ /**
2691
+ * Acquire a `MediaStream` via `getUserMedia`, attach it to a `<video>` element,
2692
+ * and expose status/error so the page can render permission and error states.
2693
+ * The stream is automatically released on unmount or retry.
2694
+ *
2695
+ * Defaults to the rear ("environment") camera; desktops fall back to whatever
2696
+ * single camera they expose. Pass `options.constraints` to override, or
2697
+ * `options.enabled: false` to render the surface without prompting for the camera
2698
+ * at all.
2699
+ *
2700
+ * Implementation notes:
2701
+ * - Cleanup detaches the stream from a *snapshotted* video node, so it releases
2702
+ * the same element it attached to even if the page remounts the `<video>`.
2703
+ * - When `getUserMedia` is missing, an insecure context is the usual cause, so
2704
+ * the hook prefers that (actionable) error; otherwise it reports `unsupported`.
2705
+ * - `video.play()` rejections are swallowed: autoplay may be blocked, but the
2706
+ * user gesture that opened the camera usually counts and playback resumes on
2707
+ * the next interaction.
2708
+ *
2709
+ * @param options - optional configuration (see {@link UseCameraStreamOptions}).
2710
+ * @returns The stream status, classified error, a `videoRef` to attach, the live
2711
+ * `stream` for whatever needs the track itself, and a `retry()` to re-attempt
2712
+ * acquisition.
2713
+ */
2714
+ export declare function useCameraStream(options?: UseCameraStreamOptions): UseCameraStreamApi;
2715
+
2716
+ /** Value returned by {@link useCameraStream}. */
2717
+ export declare interface UseCameraStreamApi {
2718
+ /** Current lifecycle status. */
2719
+ status: CameraStreamStatus;
2720
+ /** The classified error, or `null` while not in the `error` status. */
2721
+ error: CameraStreamError | null;
2722
+ /** Attach to a `<video ref={…} />`. The stream is wired to it once ready. */
2723
+ videoRef: RefObject<HTMLVideoElement | null>;
2724
+ /**
2725
+ * The live stream, or `null`.
2726
+ *
2727
+ * Exposed for the things that need the **track** rather than the picture — the LED
2728
+ * torch (`useTorch`), the real frame size from `getSettings()`, recording it with
2729
+ * `useVideoRecorder`. Do not stop it yourself: the hook owns its lifetime and
2730
+ * releases it on unmount and on `retry()`.
2731
+ */
2732
+ stream: MediaStream | null;
2733
+ /** Manually re-attempt after an error (e.g. the user changed permissions). */
2734
+ retry: () => void;
2735
+ }
2736
+
2737
+ /** Options for {@link useCameraStream}. */
2738
+ export declare interface UseCameraStreamOptions {
2739
+ /**
2740
+ * Constraints passed to `getUserMedia`. Defaults to the rear
2741
+ * ("environment") camera at Full-HD ideal resolution with audio off.
2742
+ * Read when the stream (re)starts — change it and call `retry()` to apply.
2743
+ */
2744
+ constraints?: MediaStreamConstraints;
2745
+ /**
2746
+ * Hold off acquiring the camera until this is `true`. Default `true`.
2747
+ *
2748
+ * The point is to be able to *not* prompt. A permission prompt costs the user a
2749
+ * decision and, if they refuse, costs the app the feature permanently — so a
2750
+ * surface that already knows it cannot do its job (a barcode scanner in a browser
2751
+ * with no decoder) must not open the camera to then say so. Flipping this to `false`
2752
+ * also releases a stream that is already live.
2753
+ */
2754
+ enabled?: boolean;
2755
+ }
2756
+
2757
+ /**
2758
+ * Sample mean luminance from a `<video>` source on a `requestAnimationFrame`
2759
+ * loop and expose the rolling value. Sampling is throttled by `intervalMs` and
2760
+ * paused whenever `enabled` is `false` or the video is not ready yet
2761
+ * (`readyState < 2` or `videoWidth === 0`).
2762
+ *
2763
+ * One offscreen canvas is reused across frames to avoid GC pressure. Designed
2764
+ * to feed a live brightness bar / border color on a camera page.
2765
+ *
2766
+ * @param videoRef - ref to the `<video>` element to sample.
2767
+ * @param options - optional configuration (see {@link UseLiveLuminanceOptions}).
2768
+ * @returns The rolling mean luminance in `0..255` (`0` until the first sample).
2769
+ */
2770
+ export declare function useLiveLuminance(videoRef: RefObject<HTMLVideoElement | null>, { enabled, intervalMs }?: UseLiveLuminanceOptions): number;
2771
+
2772
+ /** Options for {@link useLiveLuminance}. */
2773
+ export declare interface UseLiveLuminanceOptions {
2774
+ /** When `false` the loop is paused (e.g. while a capture is in flight). Default: `true`. */
2775
+ enabled?: boolean;
2776
+ /** Throttle measurements in milliseconds. Default: `160` (~6 fps), plenty for UX. */
2777
+ intervalMs?: number;
2778
+ }
2779
+
2780
+ export declare const VERSION: string;
2781
+
2782
+ export declare abstract class VisionTask {
2783
+ protected readonly _session: OrtSession;
2784
+ protected constructor(_session: OrtSession);
2785
+ /** The underlying {@link OrtSession} used to run inference. */
2786
+ get session(): OrtSession;
2787
+ }
2788
+
2789
+ /**
2790
+ * Write an RGBA buffer as normalized planar float32.
2791
+ *
2792
+ * Shared by {@link ResizePipeline} and exported for a custom pipeline that
2793
+ * already holds pixels and wants the SDK's exact arithmetic.
2794
+ *
2795
+ * The expression is `(value / 255 - mean) / std`, evaluated in that order on
2796
+ * purpose: folding it into a single multiply-add (`value * k + b`) computes the
2797
+ * same quantity but rounds differently, and the tasks' output is asserted to be
2798
+ * bit-identical to the composable `normalize` → `toCHW` path.
2799
+ *
2800
+ * @param rgba Source pixels, 4 bytes per pixel, `width * height` long.
2801
+ * @param width Image width in pixels.
2802
+ * @param height Image height in pixels.
2803
+ * @param mean Per-channel RGB mean, already in `[0, 1]`.
2804
+ * @param std Per-channel RGB standard deviation.
2805
+ * @param out Destination buffer, `3 * width * height` long.
2806
+ * @param stride Bytes per source pixel: 4 for canvas RGBA, 3 for packed RGB.
2807
+ */
2808
+ export declare function writePlanarFloat32(rgba: Uint8ClampedArray | Uint8Array, width: number, height: number, mean: readonly [number, number, number], std: readonly [number, number, number], out: Float32Array, stride?: number): void;
2809
+
2810
+ /**
2811
+ * Build a zero-filled CHW tensor payload for a warm-up run.
2812
+ *
2813
+ * @param width Model input width in pixels.
2814
+ * @param height Model input height in pixels.
2815
+ */
2816
+ export declare function zeroTensorData(width: number, height: number): Float32Array;
2817
+
2818
+ export { }