tempest-react-sdk 0.38.2 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/lib/design/collect.mjs +19 -0
- package/bin/lib/design/functions.mjs +5 -0
- package/bin/lib/design/index.mjs +8 -3
- package/bin/lib/design/index.test.mjs +31 -0
- package/bin/lib/design/scan.mjs +34 -2
- package/bin/lib/design/scan.test.mjs +78 -0
- package/bin/lib/doctor/doctor.e2e.test.mjs +85 -0
- package/bin/tempest.mjs +90 -6
- package/dist/audio/level-meter.cjs.map +1 -1
- package/dist/audio/level-meter.js.map +1 -1
- package/dist/audio/use-audio-recorder.cjs.map +1 -1
- package/dist/audio/use-audio-recorder.js.map +1 -1
- package/dist/auth/passkey.cjs.map +1 -1
- package/dist/auth/passkey.js.map +1 -1
- package/dist/auth/use-passkey.cjs.map +1 -1
- package/dist/auth/use-passkey.js.map +1 -1
- package/dist/br/BrazilMap.cjs.map +1 -1
- package/dist/br/BrazilMap.js.map +1 -1
- package/dist/br/BrazilStateCitySelect.cjs.map +1 -1
- package/dist/br/BrazilStateCitySelect.js.map +1 -1
- package/dist/br/BrazilStateMap.cjs.map +1 -1
- package/dist/br/BrazilStateMap.js.map +1 -1
- package/dist/br/PixQRCode.cjs.map +1 -1
- package/dist/br/PixQRCode.js.map +1 -1
- package/dist/br/boleto.cjs.map +1 -1
- package/dist/br/boleto.js.map +1 -1
- package/dist/br/pix.cjs.map +1 -1
- package/dist/br/pix.js.map +1 -1
- package/dist/br.d.ts +8 -0
- package/dist/capture/media-recorder.cjs.map +1 -1
- package/dist/capture/media-recorder.js.map +1 -1
- package/dist/capture/use-barcode-scanner.cjs.map +1 -1
- package/dist/capture/use-barcode-scanner.js.map +1 -1
- package/dist/capture/use-screen-capture.cjs.map +1 -1
- package/dist/capture/use-screen-capture.js.map +1 -1
- package/dist/capture/use-speech-recognition.cjs.map +1 -1
- package/dist/capture/use-speech-recognition.js.map +1 -1
- package/dist/capture/use-video-recorder.cjs.map +1 -1
- package/dist/capture/use-video-recorder.js.map +1 -1
- package/dist/charts/types.cjs.map +1 -1
- package/dist/charts/types.js.map +1 -1
- package/dist/charts.d.ts +8 -0
- package/dist/components/AIChat/AIChat.cjs.map +1 -1
- package/dist/components/AIChat/AIChat.js.map +1 -1
- package/dist/components/AIChat/AIChatComposer.cjs.map +1 -1
- package/dist/components/AIChat/AIChatComposer.js.map +1 -1
- package/dist/components/AIChat/AIChatTurn.cjs.map +1 -1
- package/dist/components/AIChat/AIChatTurn.js.map +1 -1
- package/dist/components/AppBar/AppBar.cjs.map +1 -1
- package/dist/components/AppBar/AppBar.js.map +1 -1
- package/dist/components/AudioPlayer/AudioPlayer.cjs.map +1 -1
- package/dist/components/AudioPlayer/AudioPlayer.js.map +1 -1
- package/dist/components/AudioRecorder/AudioRecorder.cjs.map +1 -1
- package/dist/components/AudioRecorder/AudioRecorder.js.map +1 -1
- package/dist/components/BarcodeScanner/BarcodeScanner.cjs.map +1 -1
- package/dist/components/BarcodeScanner/BarcodeScanner.js.map +1 -1
- package/dist/components/Button/Button.cjs.map +1 -1
- package/dist/components/Button/Button.js.map +1 -1
- package/dist/components/Calendar/Calendar.cjs.map +1 -1
- package/dist/components/Calendar/Calendar.js.map +1 -1
- package/dist/components/Carousel/Carousel.cjs.map +1 -1
- package/dist/components/Carousel/Carousel.js.map +1 -1
- package/dist/components/Chat/Chat.cjs.map +1 -1
- package/dist/components/Chat/Chat.js.map +1 -1
- package/dist/components/ChipInput/ChipInput.cjs.map +1 -1
- package/dist/components/ChipInput/ChipInput.js.map +1 -1
- package/dist/components/CodeBlock/CodeBlock.cjs.map +1 -1
- package/dist/components/CodeBlock/CodeBlock.js.map +1 -1
- package/dist/components/Combobox/Combobox.cjs.map +1 -1
- package/dist/components/Combobox/Combobox.js.map +1 -1
- package/dist/components/Command/Command.cjs.map +1 -1
- package/dist/components/Command/Command.js.map +1 -1
- package/dist/components/ConfirmDialog/ConfirmDialog.cjs.map +1 -1
- package/dist/components/ConfirmDialog/ConfirmDialog.js.map +1 -1
- package/dist/components/ContextMenu/ContextMenu.cjs.map +1 -1
- package/dist/components/ContextMenu/ContextMenu.js.map +1 -1
- package/dist/components/CopyButton/CopyButton.cjs.map +1 -1
- package/dist/components/CopyButton/CopyButton.js.map +1 -1
- package/dist/components/DataTable/DataTable.cjs.map +1 -1
- package/dist/components/DataTable/DataTable.js.map +1 -1
- package/dist/components/DataTable/DataTable.module.cjs.map +1 -1
- package/dist/components/DataTable/DataTable.module.js.map +1 -1
- package/dist/components/DataTable/EditableCell.cjs.map +1 -1
- package/dist/components/DataTable/EditableCell.js.map +1 -1
- package/dist/components/DatePicker/DatePicker.cjs.map +1 -1
- package/dist/components/DatePicker/DatePicker.js.map +1 -1
- package/dist/components/DateRangePicker/DateRangePicker.cjs.map +1 -1
- package/dist/components/DateRangePicker/DateRangePicker.js.map +1 -1
- package/dist/components/Drawer/Drawer.cjs.map +1 -1
- package/dist/components/Drawer/Drawer.js.map +1 -1
- package/dist/components/DropdownMenu/DropdownMenu.cjs.map +1 -1
- package/dist/components/DropdownMenu/DropdownMenu.js.map +1 -1
- package/dist/components/Dropzone/Dropzone.cjs.map +1 -1
- package/dist/components/Dropzone/Dropzone.js.map +1 -1
- package/dist/components/FileUpload/FileUpload.cjs.map +1 -1
- package/dist/components/FileUpload/FileUpload.js.map +1 -1
- package/dist/components/FilterBar/FilterBar.cjs.map +1 -1
- package/dist/components/FilterBar/FilterBar.js.map +1 -1
- package/dist/components/ImageCropper/ImageCropper.cjs.map +1 -1
- package/dist/components/ImageCropper/ImageCropper.js.map +1 -1
- package/dist/components/InstallBanner/InstallBanner.cjs.map +1 -1
- package/dist/components/InstallBanner/InstallBanner.js.map +1 -1
- package/dist/components/Kanban/Kanban.cjs.map +1 -1
- package/dist/components/Kanban/Kanban.js.map +1 -1
- package/dist/components/Lightbox/Lightbox.cjs.map +1 -1
- package/dist/components/Lightbox/Lightbox.js.map +1 -1
- package/dist/components/ListTile/ListTile.cjs.map +1 -1
- package/dist/components/ListTile/ListTile.js.map +1 -1
- package/dist/components/Markdown/Markdown.cjs.map +1 -1
- package/dist/components/Markdown/Markdown.js.map +1 -1
- package/dist/components/Markdown/markdown-parse.cjs.map +1 -1
- package/dist/components/Markdown/markdown-parse.js.map +1 -1
- package/dist/components/Menubar/Menubar.cjs.map +1 -1
- package/dist/components/Menubar/Menubar.js.map +1 -1
- package/dist/components/Modal/Modal.cjs.map +1 -1
- package/dist/components/Modal/Modal.js.map +1 -1
- package/dist/components/ModalsManager/ModalsManager.cjs.map +1 -1
- package/dist/components/ModalsManager/ModalsManager.js.map +1 -1
- package/dist/components/MultiSelect/MultiSelect.cjs.map +1 -1
- package/dist/components/MultiSelect/MultiSelect.js.map +1 -1
- package/dist/components/NavigationMenu/NavigationMenu.cjs.map +1 -1
- package/dist/components/NavigationMenu/NavigationMenu.js.map +1 -1
- package/dist/components/NotificationCenter/NotificationCenter.cjs.map +1 -1
- package/dist/components/NotificationCenter/NotificationCenter.js.map +1 -1
- package/dist/components/Page/Page.cjs.map +1 -1
- package/dist/components/Page/Page.js.map +1 -1
- package/dist/components/Pagination/Pagination.cjs.map +1 -1
- package/dist/components/Pagination/Pagination.js.map +1 -1
- package/dist/components/PasswordInput/PasswordInput.cjs.map +1 -1
- package/dist/components/PasswordInput/PasswordInput.js.map +1 -1
- package/dist/components/PinInput/PinInput.cjs.map +1 -1
- package/dist/components/PinInput/PinInput.js.map +1 -1
- package/dist/components/Popover/Popover.cjs.map +1 -1
- package/dist/components/Popover/Popover.js.map +1 -1
- package/dist/components/Progress/Progress.cjs.map +1 -1
- package/dist/components/Progress/Progress.js.map +1 -1
- package/dist/components/QRCode/qr-encode.cjs.map +1 -1
- package/dist/components/QRCode/qr-encode.js.map +1 -1
- package/dist/components/Radio/Radio.cjs.map +1 -1
- package/dist/components/Radio/Radio.js.map +1 -1
- package/dist/components/RangeSlider/RangeSlider.cjs.map +1 -1
- package/dist/components/RangeSlider/RangeSlider.js.map +1 -1
- package/dist/components/RatingStars/RatingStars.cjs.map +1 -1
- package/dist/components/RatingStars/RatingStars.js.map +1 -1
- package/dist/components/RefreshIndicator/RefreshIndicator.cjs.map +1 -1
- package/dist/components/RefreshIndicator/RefreshIndicator.js.map +1 -1
- package/dist/components/Resizable/Resizable.cjs.map +1 -1
- package/dist/components/Resizable/Resizable.js.map +1 -1
- package/dist/components/Scheduler/Scheduler.cjs.map +1 -1
- package/dist/components/Scheduler/Scheduler.js.map +1 -1
- package/dist/components/Sidebar/Sidebar.cjs.map +1 -1
- package/dist/components/Sidebar/Sidebar.js.map +1 -1
- package/dist/components/SignaturePad/SignaturePad.cjs.map +1 -1
- package/dist/components/SignaturePad/SignaturePad.js.map +1 -1
- package/dist/components/Slider/Slider.cjs.map +1 -1
- package/dist/components/Slider/Slider.js.map +1 -1
- package/dist/components/Sparkline/Sparkline.cjs.map +1 -1
- package/dist/components/Sparkline/Sparkline.js.map +1 -1
- package/dist/components/StepperInput/StepperInput.cjs.map +1 -1
- package/dist/components/StepperInput/StepperInput.js.map +1 -1
- package/dist/components/Table/Table.cjs.map +1 -1
- package/dist/components/Table/Table.js.map +1 -1
- package/dist/components/TimePicker/TimePicker.cjs.map +1 -1
- package/dist/components/TimePicker/TimePicker.js.map +1 -1
- package/dist/components/Toast/ToastProvider.cjs.map +1 -1
- package/dist/components/Toast/ToastProvider.js.map +1 -1
- package/dist/components/Tour/Tour.cjs.map +1 -1
- package/dist/components/Tour/Tour.js.map +1 -1
- package/dist/components/Transfer/Transfer.cjs.map +1 -1
- package/dist/components/Transfer/Transfer.js.map +1 -1
- package/dist/components/TreeView/TreeView.cjs.map +1 -1
- package/dist/components/TreeView/TreeView.js.map +1 -1
- package/dist/components/VirtualList/VirtualList.cjs.map +1 -1
- package/dist/components/VirtualList/VirtualList.js.map +1 -1
- package/dist/components/VirtualTable/VirtualTable.cjs.map +1 -1
- package/dist/components/VirtualTable/VirtualTable.js.map +1 -1
- package/dist/components/Wizard/Wizard.cjs.map +1 -1
- package/dist/components/Wizard/Wizard.js.map +1 -1
- package/dist/editor/RichTextEditor.cjs.map +1 -1
- package/dist/editor/RichTextEditor.js.map +1 -1
- package/dist/forms/FormField.cjs.map +1 -1
- package/dist/forms/FormField.js.map +1 -1
- package/dist/geo/TrajectoryMap.cjs.map +1 -1
- package/dist/geo/TrajectoryMap.js.map +1 -1
- package/dist/geo/estimate.cjs.map +1 -1
- package/dist/geo/estimate.js.map +1 -1
- package/dist/geo/projection.cjs.map +1 -1
- package/dist/geo/projection.js.map +1 -1
- package/dist/hooks/use-event-listener.cjs.map +1 -1
- package/dist/hooks/use-event-listener.js.map +1 -1
- package/dist/hooks/use-local-storage.cjs.map +1 -1
- package/dist/hooks/use-local-storage.js.map +1 -1
- package/dist/hooks/use-sortable.cjs.map +1 -1
- package/dist/hooks/use-sortable.js.map +1 -1
- package/dist/http/api-client.cjs.map +1 -1
- package/dist/http/api-client.js.map +1 -1
- package/dist/http/errors.cjs.map +1 -1
- package/dist/http/errors.js.map +1 -1
- package/dist/http/resumable-upload.cjs.map +1 -1
- package/dist/http/resumable-upload.js.map +1 -1
- package/dist/http/upload-with-progress.cjs.map +1 -1
- package/dist/http/upload-with-progress.js.map +1 -1
- package/dist/i18n/I18nProvider.cjs.map +1 -1
- package/dist/i18n/I18nProvider.js.map +1 -1
- package/dist/imaging/canvas.cjs.map +1 -1
- package/dist/imaging/canvas.js.map +1 -1
- package/dist/imaging.d.ts +5 -0
- package/dist/logger/logger.cjs.map +1 -1
- package/dist/logger/logger.js.map +1 -1
- package/dist/oauth/GoogleSignIn.cjs.map +1 -1
- package/dist/oauth/GoogleSignIn.js.map +1 -1
- package/dist/offline/create-offline-sync.cjs.map +1 -1
- package/dist/offline/create-offline-sync.js.map +1 -1
- package/dist/perf/cache-size.cjs +1 -1
- package/dist/perf/cache-size.cjs.map +1 -1
- package/dist/perf/cache-size.js +27 -6
- package/dist/perf/cache-size.js.map +1 -1
- package/dist/sse/create-event-stream.cjs.map +1 -1
- package/dist/sse/create-event-stream.js.map +1 -1
- package/dist/styles.css +1 -1
- package/dist/sw/background-sync.cjs.map +1 -1
- package/dist/sw/background-sync.js.map +1 -1
- package/dist/sw/cache.cjs.map +1 -1
- package/dist/sw/cache.js.map +1 -1
- package/dist/sw.d.ts +12 -0
- package/dist/tabular/compact.cjs.map +1 -1
- package/dist/tabular/compact.js.map +1 -1
- package/dist/tabular/predictor.cjs.map +1 -1
- package/dist/tabular/predictor.js.map +1 -1
- package/dist/tabular.d.ts +7 -2
- package/dist/tempest-react-sdk.d.ts +133 -6
- package/dist/theme/ThemeProvider.cjs.map +1 -1
- package/dist/theme/ThemeProvider.js.map +1 -1
- package/dist/theme/color.cjs.map +1 -1
- package/dist/theme/color.js.map +1 -1
- package/dist/theme/create-theme.cjs.map +1 -1
- package/dist/theme/create-theme.js.map +1 -1
- package/dist/theme/data-viz-ramps.cjs +1 -1
- package/dist/theme/data-viz-ramps.cjs.map +1 -1
- package/dist/theme/data-viz-ramps.js +1 -1
- package/dist/theme/data-viz-ramps.js.map +1 -1
- package/dist/utils/storage.cjs.map +1 -1
- package/dist/utils/storage.js.map +1 -1
- package/dist/vision/core/canvas.cjs +1 -1
- package/dist/vision/core/canvas.cjs.map +1 -1
- package/dist/vision/core/canvas.js +4 -4
- package/dist/vision/core/canvas.js.map +1 -1
- package/dist/vision/core/exceptions.cjs +1 -1
- package/dist/vision/core/exceptions.cjs.map +1 -1
- package/dist/vision/core/exceptions.js +2 -2
- package/dist/vision/core/exceptions.js.map +1 -1
- package/dist/vision/core/graph.cjs.map +1 -1
- package/dist/vision/core/graph.js.map +1 -1
- package/dist/vision/core/metadata.cjs +2 -2
- package/dist/vision/core/metadata.cjs.map +1 -1
- package/dist/vision/core/metadata.js +10 -7
- package/dist/vision/core/metadata.js.map +1 -1
- package/dist/vision/core/providers.cjs.map +1 -1
- package/dist/vision/core/providers.js.map +1 -1
- package/dist/vision/core/session.cjs +1 -1
- package/dist/vision/core/session.cjs.map +1 -1
- package/dist/vision/core/session.js +8 -5
- package/dist/vision/core/session.js.map +1 -1
- package/dist/vision/core/timing.cjs.map +1 -1
- package/dist/vision/core/timing.js.map +1 -1
- package/dist/vision/fusion.cjs +2 -0
- package/dist/vision/fusion.cjs.map +1 -0
- package/dist/vision/fusion.js +42 -0
- package/dist/vision/fusion.js.map +1 -0
- package/dist/vision/index.cjs +1 -1
- package/dist/vision/index.cjs.map +1 -1
- package/dist/vision/index.js +23 -20
- package/dist/vision/index.js.map +1 -1
- package/dist/vision/io/image.cjs.map +1 -1
- package/dist/vision/io/image.js.map +1 -1
- package/dist/vision/labels.cjs +1 -1
- package/dist/vision/labels.cjs.map +1 -1
- package/dist/vision/labels.js +7 -4
- package/dist/vision/labels.js.map +1 -1
- package/dist/vision/luminance.cjs.map +1 -1
- package/dist/vision/luminance.js.map +1 -1
- package/dist/vision/postprocess/classification.cjs.map +1 -1
- package/dist/vision/postprocess/classification.js.map +1 -1
- package/dist/vision/postprocess/detection.cjs +1 -1
- package/dist/vision/postprocess/detection.cjs.map +1 -1
- package/dist/vision/postprocess/detection.js +3 -10
- package/dist/vision/postprocess/detection.js.map +1 -1
- package/dist/vision/postprocess/segmentation.cjs +1 -1
- package/dist/vision/postprocess/segmentation.cjs.map +1 -1
- package/dist/vision/postprocess/segmentation.js +11 -15
- package/dist/vision/postprocess/segmentation.js.map +1 -1
- package/dist/vision/preprocess/image.cjs.map +1 -1
- package/dist/vision/preprocess/image.js.map +1 -1
- package/dist/vision/preprocess/pipeline.cjs +2 -0
- package/dist/vision/preprocess/pipeline.cjs.map +1 -0
- package/dist/vision/preprocess/pipeline.js +61 -0
- package/dist/vision/preprocess/pipeline.js.map +1 -0
- package/dist/vision/results.cjs +1 -1
- package/dist/vision/results.cjs.map +1 -1
- package/dist/vision/results.js +23 -2
- package/dist/vision/results.js.map +1 -1
- package/dist/vision/tasks/base.cjs +1 -1
- package/dist/vision/tasks/base.cjs.map +1 -1
- package/dist/vision/tasks/base.js +9 -2
- package/dist/vision/tasks/base.js.map +1 -1
- package/dist/vision/tasks/classifier.cjs.map +1 -1
- package/dist/vision/tasks/classifier.js.map +1 -1
- package/dist/vision/tasks/detectClassify.cjs +2 -0
- package/dist/vision/tasks/detectClassify.cjs.map +1 -0
- package/dist/vision/tasks/detectClassify.js +221 -0
- package/dist/vision/tasks/detectClassify.js.map +1 -0
- package/dist/vision/tasks/detector.cjs +1 -1
- package/dist/vision/tasks/detector.cjs.map +1 -1
- package/dist/vision/tasks/detector.js +54 -29
- package/dist/vision/tasks/detector.js.map +1 -1
- package/dist/vision/tasks/segmenter.cjs +1 -1
- package/dist/vision/tasks/segmenter.cjs.map +1 -1
- package/dist/vision/tasks/segmenter.js +53 -28
- package/dist/vision/tasks/segmenter.js.map +1 -1
- package/dist/vision/types.cjs.map +1 -1
- package/dist/vision/types.js.map +1 -1
- package/dist/vision.cjs +1 -1
- package/dist/vision.d.ts +1517 -956
- package/dist/vision.js +25 -22
- package/dist/vite/tempest-icons.cjs.map +1 -1
- package/dist/vite/tempest-icons.js.map +1 -1
- package/dist/vite/tempest-pwa-icons.cjs.map +1 -1
- package/dist/vite/tempest-pwa-icons.js.map +1 -1
- package/dist/ws/create-web-socket.cjs.map +1 -1
- package/dist/ws/create-web-socket.js.map +1 -1
- package/package.json +1 -1
package/dist/vision.d.ts
CHANGED
|
@@ -11,6 +11,11 @@ import { RefObject } from 'react';
|
|
|
11
11
|
* @param idxs Class index per box, length `N`. Boxes with different `idxs`
|
|
12
12
|
* never suppress each other.
|
|
13
13
|
* @param iouThreshold IoU threshold for suppression within a class.
|
|
14
|
+
* @returns Indices of kept boxes, sorted by descending score across all
|
|
15
|
+
* classes. Survivors from different classes that are tied on score are
|
|
16
|
+
* ordered lowest-index first — an explicit tie-break, because the order the
|
|
17
|
+
* per-class loop emits them in is an implementation detail (here, `Map`
|
|
18
|
+
* insertion order; in Python, sorted class order).
|
|
14
19
|
*/
|
|
15
20
|
export declare function batchedNms(boxes: Float32Array, scores: Float32Array, idxs: Int32Array, iouThreshold: number): Int32Array;
|
|
16
21
|
|
|
@@ -310,6 +315,38 @@ export declare const COCO_CLASSES: readonly string[];
|
|
|
310
315
|
*/
|
|
311
316
|
export declare function computeImageLuminance(source: LuminanceSource, reusableCanvas?: HTMLCanvasElement): number;
|
|
312
317
|
|
|
318
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
319
|
+
/**
|
|
320
|
+
* The contract a fused pipeline carries inside its own `.onnx` file.
|
|
321
|
+
*
|
|
322
|
+
* A pipeline built by the Python SDK's `ort_vision_sdk.compose` is a single
|
|
323
|
+
* graph that already contains the detector, the crop-and-resize bridge and the
|
|
324
|
+
* classifier. Everything the runtime needs to drive it — the letterbox
|
|
325
|
+
* resolution, the crop resolution, where the crops come from, how many
|
|
326
|
+
* detections the graph emits, the class names of both stages — was decided at
|
|
327
|
+
* fusion time and written into the model's metadata.
|
|
328
|
+
*
|
|
329
|
+
* This module reads it back. It is the browser half of a contract whose other
|
|
330
|
+
* half lives in `ort_vision_sdk/fusion.py`: the same keys, the same encodings,
|
|
331
|
+
* the same fallbacks. A pipeline fused once therefore runs identically in
|
|
332
|
+
* Python and in a browser tab, off the same file.
|
|
333
|
+
*
|
|
334
|
+
* Building a pipeline stays a Python-side build step — there is no ONNX
|
|
335
|
+
* protobuf writer here, and there is no reason for one: fusing is something you
|
|
336
|
+
* do once next to your export pipeline, not in a page load.
|
|
337
|
+
*/
|
|
338
|
+
/**
|
|
339
|
+
* Which tensor the bridge crops the detected boxes out of.
|
|
340
|
+
*
|
|
341
|
+
* - `"detector_input"`: the letterboxed tensor already fed to the detector.
|
|
342
|
+
* The fused graph then has a **single** image input, but a small object is
|
|
343
|
+
* cropped out of its downscaled copy.
|
|
344
|
+
* - `"original"`: a second, full-resolution image input. The bridge undoes the
|
|
345
|
+
* letterbox transform in-graph and crops at native resolution. Two tensors to
|
|
346
|
+
* feed, still one session and one model load.
|
|
347
|
+
*/
|
|
348
|
+
export declare type CropSource = "detector_input" | "original";
|
|
349
|
+
|
|
313
350
|
/**
|
|
314
351
|
* One declared dimension: a number when the graph pins it, `null` when the
|
|
315
352
|
* dimension is symbolic (dynamic).
|
|
@@ -436,29 +473,6 @@ export declare interface DecodeYoloSegOptions {
|
|
|
436
473
|
readonly maskThreshold?: number;
|
|
437
474
|
}
|
|
438
475
|
|
|
439
|
-
/**
|
|
440
|
-
* @deprecated since 0.2.0 — use {@link decodeYolo}. Same behavior; the
|
|
441
|
-
* decoder covers v8/v9/v10/v11/v12 detect heads. Will be removed in 0.4.0.
|
|
442
|
-
*/
|
|
443
|
-
export declare function decodeYoloV8(output: Float32Array, outputDims: readonly number[], options: DecodeYoloOptions): DecodedDetection[];
|
|
444
|
-
|
|
445
|
-
/**
|
|
446
|
-
* @deprecated since 0.2.0 — use {@link decodeYoloAnchors}. Will be removed in 0.4.0.
|
|
447
|
-
*/
|
|
448
|
-
export declare function decodeYoloV8Anchors(data: Float32Array, dims: readonly number[], options: DecodeYoloAnchorsOptions): DecodedAnchors;
|
|
449
|
-
|
|
450
|
-
/** @deprecated since 0.2.0 — use {@link DecodeYoloAnchorsOptions}. */
|
|
451
|
-
export declare type DecodeYoloV8AnchorsOptions = DecodeYoloAnchorsOptions;
|
|
452
|
-
|
|
453
|
-
/** @deprecated since 0.2.0 — use {@link DecodeYoloOptions}. */
|
|
454
|
-
export declare type DecodeYoloV8Options = DecodeYoloOptions;
|
|
455
|
-
|
|
456
|
-
/** @deprecated since 0.2.0 — use {@link decodeYoloSeg}. Will be removed in 0.4.0. */
|
|
457
|
-
export declare function decodeYoloV8Seg(perAnchorData: Float32Array, perAnchorDims: readonly number[], prototypeData: Float32Array, prototypeDims: readonly number[], options: DecodeYoloSegOptions): DecodedSegmentation[];
|
|
458
|
-
|
|
459
|
-
/** @deprecated since 0.2.0 — use {@link DecodeYoloSegOptions}. */
|
|
460
|
-
export declare type DecodeYoloV8SegOptions = DecodeYoloSegOptions;
|
|
461
|
-
|
|
462
476
|
/**
|
|
463
477
|
* Default execution provider preference order for browser ORT.
|
|
464
478
|
*
|
|
@@ -468,1037 +482,1584 @@ export declare type DecodeYoloV8SegOptions = DecodeYoloSegOptions;
|
|
|
468
482
|
export declare const DEFAULT_PROVIDERS: readonly string[];
|
|
469
483
|
|
|
470
484
|
/**
|
|
471
|
-
*
|
|
472
|
-
*
|
|
473
|
-
* Such a head declares `(B, 4 + nc, N)` — four box coordinates stacked above one
|
|
474
|
-
* score per class, over `N` candidate anchors. `N` is in the thousands and the
|
|
475
|
-
* batch is 1, so the channel axis is the smallest static axis above 1.
|
|
476
|
-
*
|
|
477
|
-
* @param shape Declared shape of the model's first output.
|
|
478
|
-
* @returns The class count, or `null` when the shape leaves it undeterminable —
|
|
479
|
-
* fully dynamic, or too small to hold boxes plus at least one class.
|
|
480
|
-
*/
|
|
481
|
-
export declare function detectionNumClasses(shape: DeclaredShape): number | null;
|
|
482
|
-
|
|
483
|
-
/**
|
|
484
|
-
* Single detected object produced by an object-detection model.
|
|
485
|
-
*/
|
|
486
|
-
export declare interface DetectionResult {
|
|
487
|
-
readonly classId: number;
|
|
488
|
-
readonly className: string;
|
|
489
|
-
readonly confidence: number;
|
|
490
|
-
readonly bbox: BoundingBox;
|
|
491
|
-
/** Alias for `classId` (Ultralytics-style). */
|
|
492
|
-
readonly cls: number;
|
|
493
|
-
/** Alias for `className`. */
|
|
494
|
-
readonly name: string;
|
|
495
|
-
/** Alias for `confidence` (Ultralytics-style). */
|
|
496
|
-
readonly conf: number;
|
|
497
|
-
/** Alias for `bbox` (Ultralytics-style). */
|
|
498
|
-
readonly box: BoundingBox;
|
|
499
|
-
/**
|
|
500
|
-
* The original image cropped to `bbox`, HWC RGB uint8. Empty boxes
|
|
501
|
-
* (zero area) yield a zero-sized `RGBImage`.
|
|
502
|
-
*/
|
|
503
|
-
readonly croppedImage: RGBImage;
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
/**
|
|
507
|
-
* Per-image detection envelope (Ultralytics-style `Results`).
|
|
508
|
-
*
|
|
509
|
-
* Iterating yields per-instance {@link DetectionResult} entries, so legacy
|
|
510
|
-
* code that did `for (const d of detector.predict(img))` only needs an
|
|
511
|
-
* extra `[0]` to bridge:
|
|
512
|
-
*
|
|
513
|
-
* ```typescript
|
|
514
|
-
* for (const d of (await detector.predict(img))[0]) {
|
|
515
|
-
* console.log(d.cls, d.conf, d.box.xyxy);
|
|
516
|
-
* }
|
|
517
|
-
* ```
|
|
518
|
-
*
|
|
519
|
-
* For numpy-style bulk access, use the `boxes` collection.
|
|
520
|
-
*/
|
|
521
|
-
export declare class DetectionResults implements Iterable<DetectionResult> {
|
|
522
|
-
readonly boxes: Boxes;
|
|
523
|
-
readonly detections: readonly DetectionResult[];
|
|
524
|
-
readonly names: Readonly<Record<number, string>>;
|
|
525
|
-
readonly origImg: RGBImage;
|
|
526
|
-
readonly origShape: readonly [number, number];
|
|
527
|
-
readonly path: string | null;
|
|
528
|
-
readonly speed: Readonly<Speed>;
|
|
529
|
-
constructor(boxes: Boxes, detections: readonly DetectionResult[], names: Readonly<Record<number, string>>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly<Speed>);
|
|
530
|
-
/** Number of surviving detections. */
|
|
531
|
-
get length(): number;
|
|
532
|
-
/** Index into the per-instance detections. */
|
|
533
|
-
get(index: number): DetectionResult | undefined;
|
|
534
|
-
[Symbol.iterator](): Iterator<DetectionResult>;
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
/**
|
|
538
|
-
* Object detector for anchor-free YOLO ONNX models (v8/v9/v10/v11/v12).
|
|
485
|
+
* Resolve a labels specification into an ordered array of class names.
|
|
539
486
|
*
|
|
540
|
-
*
|
|
541
|
-
|
|
542
|
-
* per-instance dataclasses, or use the bulk `boxes` view (`.xyxy`, `.xywh`,
|
|
543
|
-
* `.xyxyn`, `.xywhn`, `.cls`, `.conf`).
|
|
544
|
-
*
|
|
545
|
-
* @example
|
|
546
|
-
* ```typescript
|
|
547
|
-
* const det = await Detector.create("/models/yolov8n.onnx");
|
|
548
|
-
* const results = await det.predict("/images/street.jpg");
|
|
549
|
-
* const r = results[0];
|
|
550
|
-
* console.log(r.boxes.xyxy, r.boxes.cls, r.boxes.conf, r.names);
|
|
551
|
-
* for (const d of r) {
|
|
552
|
-
* console.log(d.cls, d.conf, d.box.xyxy);
|
|
553
|
-
* }
|
|
554
|
-
* ```
|
|
555
|
-
*/
|
|
556
|
-
export declare class Detector extends VisionTask {
|
|
557
|
-
private readonly _head;
|
|
558
|
-
private readonly _labels;
|
|
559
|
-
private readonly _names;
|
|
560
|
-
private readonly _inputSize;
|
|
561
|
-
private readonly _confThreshold;
|
|
562
|
-
private readonly _iouThreshold;
|
|
563
|
-
private readonly _maxDetections;
|
|
564
|
-
private constructor();
|
|
565
|
-
/** Load the model and resolve labels. */
|
|
566
|
-
static create(model: ModelSource, options?: DetectorOptions): Promise<Detector>;
|
|
567
|
-
/** The decoder family used to interpret the model's output. */
|
|
568
|
-
get head(): DetectorHead;
|
|
569
|
-
/** Class labels indexed by class id. */
|
|
570
|
-
get labels(): readonly string[];
|
|
571
|
-
/** Class id → class name dict (matches Ultralytics' `model.names`). */
|
|
572
|
-
get names(): Readonly<Record<number, string>>;
|
|
573
|
-
/**
|
|
574
|
-
* The `[width, height]` this task preprocesses to.
|
|
575
|
-
*
|
|
576
|
-
* Resolved at creation time from the model's graph when it declares a static
|
|
577
|
-
* input, so reading it back tells you the resolution inference really runs at
|
|
578
|
-
* — not merely what was requested.
|
|
579
|
-
*/
|
|
580
|
-
get inputSize(): readonly [number, number];
|
|
581
|
-
/** Number of classes the model predicts. */
|
|
582
|
-
get numClasses(): number;
|
|
583
|
-
/**
|
|
584
|
-
* Alias for {@link predict} — call the detector like a torch `nn.Module`.
|
|
585
|
-
*
|
|
586
|
-
* Use as `det.call(img)` since JavaScript class instances are not callable;
|
|
587
|
-
* for direct invocation, prefer `det.predict(img)`. The full
|
|
588
|
-
* {@link DetectorPredictOptions} (including `classes`) is supported.
|
|
589
|
-
*/
|
|
590
|
-
call(image: ImageInput, options?: DetectorPredictOptions): Promise<DetectionResults[]>;
|
|
591
|
-
/**
|
|
592
|
-
* Run detection on a single image.
|
|
593
|
-
*
|
|
594
|
-
* The returned envelope carries a {@link Speed} breakdown in `speed`,
|
|
595
|
-
* mirroring Ultralytics' `results[0].speed`.
|
|
487
|
+
* @throws {@link LabelMapError} if the spec is invalid, the preset is unknown,
|
|
488
|
+
* or the resolved length disagrees with `numClasses`.
|
|
596
489
|
*/
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
export declare type DetectorHead = "yolo";
|
|
614
|
-
|
|
615
|
-
export declare interface DetectorOptions extends OrtSessionOptions {
|
|
616
|
-
/**
|
|
617
|
-
* Decoder family for the detection head. Default `"yolo"` covers
|
|
618
|
-
* YOLOv8/v9/v10/v11/v12/v26.
|
|
619
|
-
*/
|
|
620
|
-
readonly head?: DetectorHead;
|
|
621
|
-
/** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */
|
|
622
|
-
readonly labels?: LabelSpec;
|
|
623
|
-
/** Number of classes — used to validate the supplied labels. */
|
|
624
|
-
readonly numClasses?: number;
|
|
625
|
-
/**
|
|
626
|
-
* Model input `[width, height]` in pixels for letterboxing.
|
|
627
|
-
*
|
|
628
|
-
* Only used when the model's graph leaves its spatial axes dynamic: a graph
|
|
629
|
-
* that declares a static size always wins, since that is the only shape ONNX
|
|
630
|
-
* Runtime will accept. Defaults to `[640, 640]`.
|
|
631
|
-
*/
|
|
632
|
-
readonly inputSize?: readonly [number, number];
|
|
633
|
-
/** Default minimum class score to keep a candidate. */
|
|
634
|
-
readonly confThreshold?: number;
|
|
635
|
-
/** Default IoU threshold for non-maximum suppression. */
|
|
636
|
-
readonly iouThreshold?: number;
|
|
637
|
-
/** Maximum number of detections per image. */
|
|
638
|
-
readonly maxDetections?: number;
|
|
639
|
-
}
|
|
640
|
-
|
|
641
|
-
export declare interface DetectorPredictOptions {
|
|
642
|
-
/** Override the default confidence threshold. */
|
|
643
|
-
readonly confThreshold?: number;
|
|
644
|
-
/** Override the default IoU threshold. */
|
|
645
|
-
readonly iouThreshold?: number;
|
|
646
|
-
/**
|
|
647
|
-
* If set, keep only detections whose `classId` is in this list.
|
|
648
|
-
* Mirrors Ultralytics' `model.predict(img, classes=[0, 16])`.
|
|
649
|
-
*/
|
|
650
|
-
readonly classes?: readonly number[];
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
/**
|
|
654
|
-
* Convert an HWC BGR uint8 buffer (OpenCV layout) to the SDK's HWC RGB.
|
|
655
|
-
*
|
|
656
|
-
* Use when you receive image bytes from `cv2.imencode` over the wire and
|
|
657
|
-
* want to feed them to the SDK without going through a canvas decode.
|
|
658
|
-
*
|
|
659
|
-
* @param bgr Flat BGR Uint8Array of length `width * height * 3`.
|
|
660
|
-
*/
|
|
661
|
-
export declare function fromCv2(bgr: Uint8Array, width: number, height: number): RGBImage;
|
|
490
|
+
/**
|
|
491
|
+
* Pick the fallback label spec for a model that declares no class names.
|
|
492
|
+
*
|
|
493
|
+
* The COCO preset is the right default for a stock YOLO export and an
|
|
494
|
+
* impossible one for anything else: it names exactly 80 classes, so handing it
|
|
495
|
+
* to a 3-class head makes {@link resolveLabels} throw `Resolved 80 labels but
|
|
496
|
+
* the model has 3 classes` and the task cannot be created at all. A custom
|
|
497
|
+
* model without baked-in `names` is an ordinary thing to have — it should come
|
|
498
|
+
* up as `class_0`, `class_1`, ..., not as a failure.
|
|
499
|
+
*
|
|
500
|
+
* @param numClasses Classes the model predicts, or `undefined` when the output
|
|
501
|
+
* shape does not say.
|
|
502
|
+
* @returns `"coco"` when the preset can describe the model, otherwise `null`,
|
|
503
|
+
* which makes {@link resolveLabels} generate `class_N` names.
|
|
504
|
+
*/
|
|
505
|
+
export declare function defaultLabels(numClasses: number | undefined): LabelSpec;
|
|
662
506
|
|
|
663
|
-
/**
|
|
664
|
-
|
|
507
|
+
/**
|
|
508
|
+
* Detector and classifier running as a single ONNX model.
|
|
509
|
+
*
|
|
510
|
+
* Everything the pipeline needs to know about itself — the resolution to
|
|
511
|
+
* letterbox to, whether it wants the full-resolution image as well, whether its
|
|
512
|
+
* classifier output still needs a softmax, the class names of both stages — was
|
|
513
|
+
* written into the file at fusion time and is read back here. Nothing is
|
|
514
|
+
* restated on the JavaScript side, so nothing can drift out of step with the
|
|
515
|
+
* Python side that built it.
|
|
516
|
+
*
|
|
517
|
+
* @example
|
|
518
|
+
* ```typescript
|
|
519
|
+
* const pipeline = await DetectClassify.create("/models/pipeline.onnx");
|
|
520
|
+
* const result = (await pipeline.predict("/images/flock.jpg"))[0];
|
|
521
|
+
* for (const detection of result) {
|
|
522
|
+
* console.log(detection.name, detection.conf, detection.classification?.name);
|
|
523
|
+
* }
|
|
524
|
+
* ```
|
|
525
|
+
*/
|
|
526
|
+
export declare class DetectClassify extends VisionTask {
|
|
527
|
+
private readonly _spec;
|
|
528
|
+
private readonly _labels;
|
|
529
|
+
private readonly _names;
|
|
530
|
+
private readonly _classifierLabels;
|
|
531
|
+
private readonly _classifierNames;
|
|
532
|
+
private readonly _raiseOnEmpty;
|
|
533
|
+
private constructor();
|
|
534
|
+
private _pipelineCache;
|
|
535
|
+
/**
|
|
536
|
+
* Run the model once on zero-filled inputs, paying one-time costs up front.
|
|
537
|
+
*
|
|
538
|
+
* Worth more here than on a single-stage task: a fused pipeline is two models
|
|
539
|
+
* plus the bridge in one graph, so the first inference compiles shaders for
|
|
540
|
+
* all of it. Calling this while a loading spinner is still up moves that cost
|
|
541
|
+
* somewhere the user is already waiting.
|
|
542
|
+
*
|
|
543
|
+
* @param runs How many warm-up inferences to run. One is enough for WASM;
|
|
544
|
+
* WebGPU sometimes settles on the second.
|
|
545
|
+
*/
|
|
546
|
+
warmup(runs?: number): Promise<void>;
|
|
547
|
+
/**
|
|
548
|
+
* The fused preprocessing pipeline, built on first use.
|
|
549
|
+
*
|
|
550
|
+
* Lazily, because constructing it allocates canvases: a pipeline built in an
|
|
551
|
+
* environment without a canvas implementation stays constructible, and only
|
|
552
|
+
* fails if it is actually asked to preprocess something.
|
|
553
|
+
*/
|
|
554
|
+
private get _pipeline();
|
|
555
|
+
/**
|
|
556
|
+
* Load a fused pipeline and resolve both label spaces.
|
|
557
|
+
*
|
|
558
|
+
* @param model The fused `.onnx` — a URL, an `ArrayBuffer`, or bytes.
|
|
559
|
+
* @param options Label overrides plus the usual session options.
|
|
560
|
+
* @throws {@link FusionError} when the model carries no pipeline metadata,
|
|
561
|
+
* i.e. it is a plain detector or classifier rather than something
|
|
562
|
+
* `ort_vision_sdk.compose` produced.
|
|
563
|
+
*/
|
|
564
|
+
static create(model: ModelSource, options?: DetectClassifyOptions): Promise<DetectClassify>;
|
|
565
|
+
/** The pipeline configuration recorded in the model at fusion time. */
|
|
566
|
+
get spec(): FusionSpec;
|
|
567
|
+
/** The `[width, height]` the detection stage runs at. */
|
|
568
|
+
get inputSize(): readonly [number, number];
|
|
569
|
+
/** Detector class labels indexed by class id. */
|
|
570
|
+
get labels(): readonly string[];
|
|
571
|
+
/** Detector class id → class name (matches Ultralytics' `model.names`). */
|
|
572
|
+
get names(): Readonly<Record<number, string>>;
|
|
573
|
+
/** Classifier class labels indexed by class id. */
|
|
574
|
+
get classifierLabels(): readonly string[];
|
|
575
|
+
/** Classifier class id → class name. */
|
|
576
|
+
get classifierNames(): Readonly<Record<number, string>>;
|
|
577
|
+
/**
|
|
578
|
+
* Alias for {@link predict} — call the pipeline like a torch `nn.Module`.
|
|
579
|
+
*
|
|
580
|
+
* Use as `pipeline.call(img)` since JavaScript class instances are not
|
|
581
|
+
* callable; for direct invocation, prefer `pipeline.predict(img)`.
|
|
582
|
+
*/
|
|
583
|
+
call(image: ImageInput, options?: DetectClassifyPredictOptions): Promise<DetectClassifyResults[]>;
|
|
584
|
+
/**
|
|
585
|
+
* Run the pipeline on a single image.
|
|
586
|
+
*
|
|
587
|
+
* The returned envelope carries a {@link Speed} breakdown in `speed`. Its
|
|
588
|
+
* `inference` figure covers detection *and* classification, since the
|
|
589
|
+
* pipeline runs them as one graph and no boundary between them is observable
|
|
590
|
+
* from outside.
|
|
591
|
+
*/
|
|
592
|
+
predict(image: ImageInput, options?: DetectClassifyPredictOptions): Promise<DetectClassifyResults[]>;
|
|
593
|
+
/**
|
|
594
|
+
* Letterbox the image and build the graph's feeds.
|
|
595
|
+
*
|
|
596
|
+
* The detector input runs through {@link LetterboxPipeline}, which fuses the
|
|
597
|
+
* resize, the padding and the HWC-to-CHW float conversion into one
|
|
598
|
+
* `drawImage` plus one readback, and reuses its output buffer between frames.
|
|
599
|
+
* That buffer goes straight to ONNX Runtime, so `_pipeline.release()` must not
|
|
600
|
+
* be called until the run resolves.
|
|
601
|
+
*
|
|
602
|
+
* A pipeline fused with `cropSource: "original"` also takes the untouched
|
|
603
|
+
* image as a second input, plus the scale and padding of the letterbox — that
|
|
604
|
+
* is what lets the graph undo the letterbox transform internally and crop at
|
|
605
|
+
* native resolution instead of from the downscaled copy. That one is **not**
|
|
606
|
+
* letterboxed by definition, so it does not go through the fused path.
|
|
607
|
+
*/
|
|
608
|
+
private _preprocess;
|
|
609
|
+
/**
|
|
610
|
+
* Map one letterboxed xyxy row back onto the original image.
|
|
611
|
+
*
|
|
612
|
+
* The graph always reports boxes in the detector's letterboxed pixel space,
|
|
613
|
+
* whichever crop source it was fused with, so both sources agree here.
|
|
614
|
+
*/
|
|
615
|
+
private _toOriginal;
|
|
616
|
+
/**
|
|
617
|
+
* Turn one row of the classifier output into a result object.
|
|
618
|
+
*
|
|
619
|
+
* @param row The output row for this detection.
|
|
620
|
+
* @param image The crop the row describes, carried so callers can display
|
|
621
|
+
* what was classified.
|
|
622
|
+
* @param k Optional truncation of the probability list.
|
|
623
|
+
*/
|
|
624
|
+
private _classify;
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
export declare interface DetectClassifyOptions extends OrtSessionOptions {
|
|
628
|
+
/**
|
|
629
|
+
* Class label spec for the **detection** stage — see {@link resolveLabels}.
|
|
630
|
+
* Defaults to the names recorded at fusion time, falling back to the COCO
|
|
631
|
+
* 80-class preset when the fusion recorded none.
|
|
632
|
+
*/
|
|
633
|
+
readonly labels?: LabelSpec;
|
|
634
|
+
/**
|
|
635
|
+
* Class label spec for the **classification** stage. Defaults to the recorded
|
|
636
|
+
* names, falling back to generated `class_<id>` names.
|
|
637
|
+
*/
|
|
638
|
+
readonly classifierLabels?: LabelSpec;
|
|
639
|
+
/**
|
|
640
|
+
* If `true`, a run that finds nothing throws {@link NoDetectionsError}
|
|
641
|
+
* instead of returning an empty envelope. Default `false`, because looking
|
|
642
|
+
* and finding nothing is a successful inference. Turn it on when an empty
|
|
643
|
+
* result means the surrounding pipeline should stop rather than carry on with
|
|
644
|
+
* zero rows. Can be overridden per `predict` call.
|
|
645
|
+
*/
|
|
646
|
+
readonly raiseOnEmpty?: boolean;
|
|
647
|
+
}
|
|
665
648
|
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
649
|
+
export declare interface DetectClassifyPredictOptions {
|
|
650
|
+
/**
|
|
651
|
+
* Drop detections scoring below this. The graph's own NMS threshold was fixed
|
|
652
|
+
* at fusion time and cannot be lowered here — this only filters further.
|
|
653
|
+
*/
|
|
654
|
+
readonly confThreshold?: number;
|
|
655
|
+
/** If set, keep only detections whose detector `classId` is in this list. */
|
|
656
|
+
readonly classes?: readonly number[];
|
|
657
|
+
/** Truncate each detection's `classification.probabilities` to its top-k entries. */
|
|
658
|
+
readonly topK?: number;
|
|
659
|
+
/** Override the constructor's `raiseOnEmpty` setting for this call. */
|
|
660
|
+
readonly raiseOnEmpty?: boolean;
|
|
661
|
+
}
|
|
669
662
|
|
|
670
|
-
/**
|
|
671
|
-
|
|
672
|
-
|
|
663
|
+
/**
|
|
664
|
+
* Per-image envelope for a fused detect→classify pipeline.
|
|
665
|
+
*
|
|
666
|
+
* Structurally a {@link DetectionResults} with a second class map: every
|
|
667
|
+
* detection it yields carries a populated `classification`, and the two stages
|
|
668
|
+
* have their own, unrelated label spaces — a detector that finds `sheep`
|
|
669
|
+
* feeding a classifier that answers `famacha_3` shares no class ids with it.
|
|
670
|
+
* Merging them into one `names` record would make `cls` and
|
|
671
|
+
* `classification.cls` look comparable when they are not.
|
|
672
|
+
*
|
|
673
|
+
* ```typescript
|
|
674
|
+
* const result = (await pipeline.predict("flock.jpg"))[0];
|
|
675
|
+
* for (const detection of result) {
|
|
676
|
+
* console.log(detection.name, detection.conf, detection.classification?.name);
|
|
677
|
+
* }
|
|
678
|
+
* ```
|
|
679
|
+
*/
|
|
680
|
+
export declare class DetectClassifyResults implements Iterable<DetectionResult> {
|
|
681
|
+
readonly boxes: Boxes;
|
|
682
|
+
readonly detections: readonly DetectionResult[];
|
|
683
|
+
readonly names: Readonly<Record<number, string>>;
|
|
684
|
+
readonly classifierNames: Readonly<Record<number, string>>;
|
|
685
|
+
readonly origImg: RGBImage;
|
|
686
|
+
readonly origShape: readonly [number, number];
|
|
687
|
+
readonly path: string | null;
|
|
688
|
+
readonly speed: Readonly<Speed>;
|
|
689
|
+
constructor(boxes: Boxes, detections: readonly DetectionResult[], names: Readonly<Record<number, string>>, classifierNames: Readonly<Record<number, string>>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly<Speed>);
|
|
690
|
+
/** Number of surviving detections. */
|
|
691
|
+
get length(): number;
|
|
692
|
+
/** Index into the per-instance detections. */
|
|
693
|
+
get(index: number): DetectionResult | undefined;
|
|
694
|
+
[Symbol.iterator](): Iterator<DetectionResult>;
|
|
695
|
+
}
|
|
673
696
|
|
|
674
|
-
/**
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
export declare function isLuminanceAcceptable(luminance: number, threshold: number): boolean;
|
|
697
|
+
/**
|
|
698
|
+
* Infer how many classes a YOLO detection/segmentation head emits.
|
|
699
|
+
*
|
|
700
|
+
* Such a head declares `(B, 4 + nc, N)` — four box coordinates stacked above one
|
|
701
|
+
* score per class, over `N` candidate anchors. `N` is in the thousands and the
|
|
702
|
+
* batch is 1, so the channel axis is the smallest static axis above 1.
|
|
703
|
+
*
|
|
704
|
+
* @param shape Declared shape of the model's first output.
|
|
705
|
+
* @returns The class count, or `null` when the shape leaves it undeterminable —
|
|
706
|
+
* fully dynamic, or too small to hold boxes plus at least one class.
|
|
707
|
+
*/
|
|
708
|
+
export declare function detectionNumClasses(shape: DeclaredShape): number | null;
|
|
687
709
|
|
|
688
|
-
/**
|
|
689
|
-
|
|
690
|
-
|
|
710
|
+
/**
|
|
711
|
+
* Single detected object produced by an object-detection model.
|
|
712
|
+
*/
|
|
713
|
+
export declare interface DetectionResult {
|
|
714
|
+
readonly classId: number;
|
|
715
|
+
readonly className: string;
|
|
716
|
+
readonly confidence: number;
|
|
717
|
+
readonly bbox: BoundingBox;
|
|
718
|
+
/** Alias for `classId` (Ultralytics-style). */
|
|
719
|
+
readonly cls: number;
|
|
720
|
+
/** Alias for `className`. */
|
|
721
|
+
readonly name: string;
|
|
722
|
+
/** Alias for `confidence` (Ultralytics-style). */
|
|
723
|
+
readonly conf: number;
|
|
724
|
+
/** Alias for `bbox` (Ultralytics-style). */
|
|
725
|
+
readonly box: BoundingBox;
|
|
726
|
+
/**
|
|
727
|
+
* The original image cropped to `bbox`, HWC RGB uint8. Empty boxes
|
|
728
|
+
* (zero area) yield a zero-sized `RGBImage`.
|
|
729
|
+
*/
|
|
730
|
+
readonly croppedImage: RGBImage;
|
|
731
|
+
/**
|
|
732
|
+
* What a second, classification stage predicted **for this crop** —
|
|
733
|
+
* populated only by {@link DetectClassify}, and `null` for a plain detector.
|
|
734
|
+
*
|
|
735
|
+
* Kept as its own field rather than folded into `classId`/`className`
|
|
736
|
+
* because the two answers are different questions: the detector says *what
|
|
737
|
+
* kind of object this is*, the classifier says *which sub-category the object
|
|
738
|
+
* belongs to*, and collapsing them would lose one of the two.
|
|
739
|
+
*/
|
|
740
|
+
readonly classification?: ClassificationResult | null;
|
|
741
|
+
}
|
|
691
742
|
|
|
692
|
-
/**
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
743
|
+
/**
|
|
744
|
+
* Per-image detection envelope (Ultralytics-style `Results`).
|
|
745
|
+
*
|
|
746
|
+
* Iterating yields per-instance {@link DetectionResult} entries, so legacy
|
|
747
|
+
* code that did `for (const d of detector.predict(img))` only needs an
|
|
748
|
+
* extra `[0]` to bridge:
|
|
749
|
+
*
|
|
750
|
+
* ```typescript
|
|
751
|
+
* for (const d of (await detector.predict(img))[0]) {
|
|
752
|
+
* console.log(d.cls, d.conf, d.box.xyxy);
|
|
753
|
+
* }
|
|
754
|
+
* ```
|
|
755
|
+
*
|
|
756
|
+
* For numpy-style bulk access, use the `boxes` collection.
|
|
757
|
+
*/
|
|
758
|
+
export declare class DetectionResults implements Iterable<DetectionResult> {
|
|
759
|
+
readonly boxes: Boxes;
|
|
760
|
+
readonly detections: readonly DetectionResult[];
|
|
761
|
+
readonly names: Readonly<Record<number, string>>;
|
|
762
|
+
readonly origImg: RGBImage;
|
|
763
|
+
readonly origShape: readonly [number, number];
|
|
764
|
+
readonly path: string | null;
|
|
765
|
+
readonly speed: Readonly<Speed>;
|
|
766
|
+
constructor(boxes: Boxes, detections: readonly DetectionResult[], names: Readonly<Record<number, string>>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly<Speed>);
|
|
767
|
+
/** Number of surviving detections. */
|
|
768
|
+
get length(): number;
|
|
769
|
+
/** Index into the per-instance detections. */
|
|
770
|
+
get(index: number): DetectionResult | undefined;
|
|
771
|
+
[Symbol.iterator](): Iterator<DetectionResult>;
|
|
772
|
+
}
|
|
711
773
|
|
|
712
|
-
/**
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
774
|
+
/**
|
|
775
|
+
* Object detector for anchor-free YOLO ONNX models (v8/v9/v10/v11/v12).
|
|
776
|
+
*
|
|
777
|
+
* `predict()` returns `Promise<DetectionResults[]>` (length 1 for a single
|
|
778
|
+
* image), mirroring Ultralytics' `YOLO("img.jpg")`. Iterate the envelope for
|
|
779
|
+
* per-instance dataclasses, or use the bulk `boxes` view (`.xyxy`, `.xywh`,
|
|
780
|
+
* `.xyxyn`, `.xywhn`, `.cls`, `.conf`).
|
|
781
|
+
*
|
|
782
|
+
* @example
|
|
783
|
+
* ```typescript
|
|
784
|
+
* const det = await Detector.create("/models/yolov8n.onnx");
|
|
785
|
+
* const results = await det.predict("/images/street.jpg");
|
|
786
|
+
* const r = results[0];
|
|
787
|
+
* console.log(r.boxes.xyxy, r.boxes.cls, r.boxes.conf, r.names);
|
|
788
|
+
* for (const d of r) {
|
|
789
|
+
* console.log(d.cls, d.conf, d.box.xyxy);
|
|
790
|
+
* }
|
|
791
|
+
* ```
|
|
792
|
+
*/
|
|
793
|
+
export declare class Detector extends VisionTask {
|
|
794
|
+
private readonly _head;
|
|
795
|
+
private readonly _labels;
|
|
796
|
+
private readonly _names;
|
|
797
|
+
private readonly _inputSize;
|
|
798
|
+
private readonly _confThreshold;
|
|
799
|
+
private readonly _iouThreshold;
|
|
800
|
+
private readonly _maxDetections;
|
|
801
|
+
private readonly _raiseOnEmpty;
|
|
802
|
+
private constructor();
|
|
803
|
+
private _pipelineCache;
|
|
804
|
+
/**
|
|
805
|
+
* Run the model once on a zero-filled tensor, paying one-time costs up front.
|
|
806
|
+
*
|
|
807
|
+
* The first inference of a session is not representative: WebGPU compiles its
|
|
808
|
+
* shaders on it and the WASM backend faults in its arenas, which on a phone
|
|
809
|
+
* can turn the first frame into seconds while every later frame is tens of
|
|
810
|
+
* milliseconds. Calling this while a loading spinner is still up moves that
|
|
811
|
+
* cost somewhere the user is already waiting.
|
|
812
|
+
*
|
|
813
|
+
* @param runs How many warm-up inferences to run. One is enough for WASM;
|
|
814
|
+
* WebGPU sometimes settles on the second.
|
|
815
|
+
*/
|
|
816
|
+
warmup(runs?: number): Promise<void>;
|
|
817
|
+
/**
|
|
818
|
+
* The fused preprocessing pipeline, built on first use.
|
|
819
|
+
*
|
|
820
|
+
* Lazily, because constructing it allocates canvases: a task built in an
|
|
821
|
+
* environment without a canvas implementation stays constructible, and only
|
|
822
|
+
* fails if it is actually asked to preprocess something.
|
|
823
|
+
*/
|
|
824
|
+
private get _pipeline();
|
|
825
|
+
/** Load the model and resolve labels. */
|
|
826
|
+
static create(model: ModelSource, options?: DetectorOptions): Promise<Detector>;
|
|
827
|
+
/** The decoder family used to interpret the model's output. */
|
|
828
|
+
get head(): DetectorHead;
|
|
829
|
+
/** Class labels indexed by class id. */
|
|
830
|
+
get labels(): readonly string[];
|
|
831
|
+
/** Class id → class name dict (matches Ultralytics' `model.names`). */
|
|
832
|
+
get names(): Readonly<Record<number, string>>;
|
|
833
|
+
/**
|
|
834
|
+
* The `[width, height]` this task preprocesses to.
|
|
835
|
+
*
|
|
836
|
+
* Resolved at creation time from the model's graph when it declares a static
|
|
837
|
+
* input, so reading it back tells you the resolution inference really runs at
|
|
838
|
+
* — not merely what was requested.
|
|
839
|
+
*/
|
|
840
|
+
get inputSize(): readonly [number, number];
|
|
841
|
+
/** Number of classes the model predicts. */
|
|
842
|
+
get numClasses(): number;
|
|
843
|
+
/**
|
|
844
|
+
* Alias for {@link predict} — call the detector like a torch `nn.Module`.
|
|
845
|
+
*
|
|
846
|
+
* Use as `det.call(img)` since JavaScript class instances are not callable;
|
|
847
|
+
* for direct invocation, prefer `det.predict(img)`. The full
|
|
848
|
+
* {@link DetectorPredictOptions} (including `classes`) is supported.
|
|
849
|
+
*/
|
|
850
|
+
call(image: ImageInput, options?: DetectorPredictOptions): Promise<DetectionResults[]>;
|
|
851
|
+
/**
|
|
852
|
+
* Run detection on a single image.
|
|
853
|
+
*
|
|
854
|
+
* The returned envelope carries a {@link Speed} breakdown in `speed`,
|
|
855
|
+
* mirroring Ultralytics' `results[0].speed`.
|
|
856
|
+
*/
|
|
857
|
+
predict(image: ImageInput, options?: DetectorPredictOptions): Promise<DetectionResults[]>;
|
|
858
|
+
/**
|
|
859
|
+
* Letterbox and pack the image into the tensor the model expects.
|
|
860
|
+
*
|
|
861
|
+
* Runs through {@link LetterboxPipeline}, which fuses the resize, the
|
|
862
|
+
* padding and the HWC-to-CHW float conversion into one `drawImage` plus one
|
|
863
|
+
* readback loop, and reuses its output buffer between frames. The buffer is
|
|
864
|
+
* handed straight to ONNX Runtime, so {@link _pipeline.release} must not be
|
|
865
|
+
* called until the run resolves.
|
|
866
|
+
*/
|
|
867
|
+
private _preprocess;
|
|
868
|
+
private _buildResult;
|
|
869
|
+
private _buildBoxes;
|
|
870
|
+
}
|
|
720
871
|
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
872
|
+
/**
|
|
873
|
+
* Decoder family for the model's detection head.
|
|
874
|
+
*
|
|
875
|
+
* - `"yolo"`: anchor-free YOLO head with output shape `[1, 4 + nc, N]` —
|
|
876
|
+
* covers YOLOv8, v9, v10, v11, v12, v26 detect exports.
|
|
877
|
+
*
|
|
878
|
+
* The SDK does **not** auto-detect the head from the model — the caller is
|
|
879
|
+
* responsible for picking a head that matches their export. Future families
|
|
880
|
+
* (v5/v6/v7 with `[1, N, 5+nc]`) will be added as new literal members.
|
|
881
|
+
*/
|
|
882
|
+
export declare type DetectorHead = "yolo";
|
|
731
883
|
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
884
|
+
export declare interface DetectorOptions extends OrtSessionOptions {
|
|
885
|
+
/**
|
|
886
|
+
* Decoder family for the detection head. Default `"yolo"` covers
|
|
887
|
+
* YOLOv8/v9/v10/v11/v12/v26.
|
|
888
|
+
*/
|
|
889
|
+
readonly head?: DetectorHead;
|
|
890
|
+
/** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */
|
|
891
|
+
readonly labels?: LabelSpec;
|
|
892
|
+
/** Number of classes — used to validate the supplied labels. */
|
|
893
|
+
readonly numClasses?: number;
|
|
894
|
+
/**
|
|
895
|
+
* Model input `[width, height]` in pixels for letterboxing.
|
|
896
|
+
*
|
|
897
|
+
* Only used when the model's graph leaves its spatial axes dynamic: a graph
|
|
898
|
+
* that declares a static size always wins, since that is the only shape ONNX
|
|
899
|
+
* Runtime will accept. Defaults to `[640, 640]`.
|
|
900
|
+
*/
|
|
901
|
+
readonly inputSize?: readonly [number, number];
|
|
902
|
+
/** Default minimum class score to keep a candidate. */
|
|
903
|
+
readonly confThreshold?: number;
|
|
904
|
+
/** Default IoU threshold for non-maximum suppression. */
|
|
905
|
+
readonly iouThreshold?: number;
|
|
906
|
+
/** Maximum number of detections per image. */
|
|
907
|
+
readonly maxDetections?: number;
|
|
908
|
+
/**
|
|
909
|
+
* If `true`, a run that finds nothing throws {@link NoDetectionsError}
|
|
910
|
+
* instead of returning an empty envelope. Default `false`, because looking
|
|
911
|
+
* and finding nothing is a successful inference. Turn it on when an empty
|
|
912
|
+
* result means the surrounding pipeline should stop rather than carry on with
|
|
913
|
+
* zero rows. Can be overridden per `predict` call.
|
|
914
|
+
*/
|
|
915
|
+
readonly raiseOnEmpty?: boolean;
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
export declare interface DetectorPredictOptions {
|
|
919
|
+
/** Override the default confidence threshold. */
|
|
920
|
+
readonly confThreshold?: number;
|
|
921
|
+
/** Override the default IoU threshold. */
|
|
922
|
+
readonly iouThreshold?: number;
|
|
923
|
+
/**
|
|
924
|
+
* If set, keep only detections whose `classId` is in this list.
|
|
925
|
+
* Mirrors Ultralytics' `model.predict(img, classes=[0, 16])`.
|
|
926
|
+
*/
|
|
927
|
+
readonly classes?: readonly number[];
|
|
928
|
+
/** Override the constructor's `raiseOnEmpty` setting for this call. */
|
|
929
|
+
readonly raiseOnEmpty?: boolean;
|
|
930
|
+
}
|
|
738
931
|
|
|
739
|
-
/**
|
|
740
|
-
* Error raised when a captured frame is too dark to be analysed reliably.
|
|
741
|
-
* Carries the measured luminance and the threshold it failed so callers can
|
|
742
|
-
* surface actionable feedback.
|
|
743
|
-
*/
|
|
744
|
-
export declare class LowLuminanceError extends Error {
|
|
745
|
-
/** Measured mean luminance, `0..255`. */
|
|
746
|
-
readonly luminance: number;
|
|
747
|
-
/** Threshold that was checked against, `0..255`. */
|
|
748
|
-
readonly threshold: number;
|
|
749
932
|
/**
|
|
750
|
-
*
|
|
751
|
-
*
|
|
933
|
+
* Convert an HWC BGR uint8 buffer (OpenCV layout) to the SDK's HWC RGB.
|
|
934
|
+
*
|
|
935
|
+
* Use when you receive image bytes from `cv2.imencode` over the wire and
|
|
936
|
+
* want to feed them to the SDK without going through a canvas decode.
|
|
937
|
+
*
|
|
938
|
+
* @param bgr Flat BGR Uint8Array of length `width * height * 3`.
|
|
752
939
|
*/
|
|
753
|
-
|
|
754
|
-
|
|
940
|
+
export declare function fromCv2(bgr: Uint8Array, width: number, height: number): RGBImage;
|
|
941
|
+
|
|
942
|
+
/** Geometry of a letterbox, plus the planar tensor data it produced. */
|
|
943
|
+
export declare interface FusedLetterboxResult {
|
|
944
|
+
/** CHW float32 in `[0, 1]`, length `3 * targetHeight * targetWidth`. */
|
|
945
|
+
readonly data: Float32Array;
|
|
946
|
+
/** Factor applied to the original image (`< 1` if downscaled). */
|
|
947
|
+
readonly scale: number;
|
|
948
|
+
/** Horizontal padding in pixels. */
|
|
949
|
+
readonly padLeft: number;
|
|
950
|
+
/** Vertical padding in pixels. */
|
|
951
|
+
readonly padTop: number;
|
|
952
|
+
/**
|
|
953
|
+
* Whether {@link data} is the pipeline's reusable buffer.
|
|
954
|
+
*
|
|
955
|
+
* `true` means the next {@link LetterboxPipeline.run} overwrites it, so a
|
|
956
|
+
* caller keeping the values past its own inference has to copy them.
|
|
957
|
+
*/
|
|
958
|
+
readonly reused: boolean;
|
|
959
|
+
}
|
|
755
960
|
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
* Averaging over a small downsample is statistically equivalent for a
|
|
759
|
-
* brightness threshold and orders of magnitude faster than reading every pixel
|
|
760
|
-
* of a full-resolution camera frame.
|
|
761
|
-
*/
|
|
762
|
-
export declare const LUMINANCE_SAMPLE_MAX_EDGE = 256;
|
|
961
|
+
/** Value of the `ovs.kind` metadata key for a detector→classifier pipeline. */
|
|
962
|
+
export declare const FUSION_KIND_DETECT_CLASSIFY = "detect_classify";
|
|
763
963
|
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
964
|
+
/**
|
|
965
|
+
* Raised when a model cannot be driven as a fused detect→classify pipeline.
|
|
966
|
+
*
|
|
967
|
+
* In the browser this means the file carries no `ovs.*` metadata — it is a
|
|
968
|
+
* plain detector or classifier rather than something `ort_vision_sdk.compose`
|
|
969
|
+
* produced — or the graph is missing an output the pipeline contract requires.
|
|
970
|
+
* Building a pipeline is a Python-side build step; the browser only runs one.
|
|
971
|
+
*/
|
|
972
|
+
export declare class FusionError extends OrtVisionError {
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
/** Everything a fused pipeline declares about how it must be driven. */
|
|
976
|
+
export declare interface FusionSpec {
|
|
977
|
+
/** Pipeline family. Only `"detect_classify"` exists today. */
|
|
978
|
+
readonly kind: string;
|
|
979
|
+
/** `[width, height]` the detector stage expects — the resolution to letterbox to. */
|
|
980
|
+
readonly inputSize: readonly [number, number];
|
|
981
|
+
/** `[width, height]` every crop is resampled to inside the graph. */
|
|
982
|
+
readonly cropSize: readonly [number, number];
|
|
983
|
+
/** Which tensor the crops are taken from. */
|
|
984
|
+
readonly cropSource: CropSource;
|
|
985
|
+
/**
|
|
986
|
+
* Fixed number of rows `K` every output carries, surplus zero-padded and
|
|
987
|
+
* counted by {@link OUTPUT_NUM_DETECTIONS}. `null` means the graph emits
|
|
988
|
+
* exactly as many rows as survived NMS.
|
|
989
|
+
*/
|
|
990
|
+
readonly maxDetections: number | null;
|
|
991
|
+
/** Score threshold baked into the graph's NMS node. */
|
|
992
|
+
readonly confThreshold: number;
|
|
993
|
+
/** IoU threshold baked into the graph's NMS node. */
|
|
994
|
+
readonly iouThreshold: number;
|
|
995
|
+
/** Whether the classifier stage emits logits that still need a softmax. */
|
|
996
|
+
readonly applySoftmax: boolean;
|
|
997
|
+
/** Detector class names in class-id order, or `null` when the fusion recorded none. */
|
|
998
|
+
readonly detectorNames: readonly string[] | null;
|
|
999
|
+
/** Classifier class names in class-id order, or `null`. */
|
|
1000
|
+
readonly classifierNames: readonly string[] | null;
|
|
1001
|
+
/** Version of `ort-vision-sdk` that produced the file. */
|
|
1002
|
+
readonly sdkVersion: string;
|
|
1003
|
+
/** Whether driving this pipeline requires feeding the full-resolution input. */
|
|
1004
|
+
readonly needsSourceImage: boolean;
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
/** Anything {@link loadImage} accepts as an image input. */
|
|
1008
|
+
export declare type ImageInput = string | Blob | HTMLImageElement | HTMLCanvasElement | OffscreenCanvas | ImageBitmap | ImageData | RGBImage;
|
|
1009
|
+
|
|
1010
|
+
/** Raised when an input image cannot be decoded into the canonical format. */
|
|
1011
|
+
export declare class ImageLoadError extends OrtVisionError {
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
/** Raised when ONNX Runtime fails while executing a model. */
|
|
1015
|
+
export declare class InferenceError extends OrtVisionError {
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
/** Name of the fused graph's letterboxed detector input, `[1, 3, H, W]` float32 in `[0, 1]`. */
|
|
1019
|
+
export declare const INPUT_IMAGE = "images";
|
|
1020
|
+
|
|
1021
|
+
/** Name of the `[2]` float32 `[padLeft, padTop]`. Only with `cropSource === "original"`. */
|
|
1022
|
+
export declare const INPUT_PAD = "letterbox_pad";
|
|
1023
|
+
|
|
1024
|
+
/** Name of the `[1]` float32 letterbox scale factor. Only with `cropSource === "original"`. */
|
|
1025
|
+
export declare const INPUT_SCALE = "letterbox_scale";
|
|
1026
|
+
|
|
1027
|
+
/** Name of the full-resolution input. Present only when `cropSource === "original"`. */
|
|
1028
|
+
export declare const INPUT_SOURCE = "source_image";
|
|
789
1029
|
|
|
790
|
-
/**
|
|
791
|
-
* Per-instance binary masks for a single image.
|
|
792
|
-
*
|
|
793
|
-
* Each mask is cropped to its instance's bounding box. To paint masks onto
|
|
794
|
-
* a full-image canvas, use `xyxy[i]` as the top-left target.
|
|
795
|
-
*/
|
|
796
|
-
export declare class Masks {
|
|
797
|
-
readonly data: ReadonlyArray<{
|
|
798
|
-
readonly data: Uint8Array;
|
|
799
|
-
readonly width: number;
|
|
800
|
-
readonly height: number;
|
|
801
|
-
}>;
|
|
802
|
-
readonly xyxy: Float32Array;
|
|
803
|
-
readonly origShape: readonly [number, number];
|
|
804
1030
|
/**
|
|
805
|
-
*
|
|
806
|
-
*
|
|
807
|
-
*
|
|
1031
|
+
* Whether a measured luminance clears a brightness threshold.
|
|
1032
|
+
*
|
|
1033
|
+
* `threshold` is intentionally required — a sensible value is
|
|
1034
|
+
* application-specific (it depends on the model, the lighting the model was
|
|
1035
|
+
* trained on, and the acceptable false-reject rate), so the SDK does not bake
|
|
1036
|
+
* in a default.
|
|
1037
|
+
*
|
|
1038
|
+
* @param luminance - measured mean luminance in `0..255`.
|
|
1039
|
+
* @param threshold - minimum acceptable luminance in `0..255`.
|
|
1040
|
+
* @returns `true` when `luminance >= threshold`.
|
|
808
1041
|
*/
|
|
809
|
-
|
|
810
|
-
readonly data: Uint8Array;
|
|
811
|
-
readonly width: number;
|
|
812
|
-
readonly height: number;
|
|
813
|
-
}>, xyxy: Float32Array, origShape: readonly [number, number]);
|
|
814
|
-
/** Number of instance masks. */
|
|
815
|
-
get length(): number;
|
|
816
|
-
/** `[N]` shape of the masks collection. */
|
|
817
|
-
get shape(): readonly [number];
|
|
818
|
-
[Symbol.iterator](): Iterator<{
|
|
819
|
-
readonly data: Uint8Array;
|
|
820
|
-
readonly width: number;
|
|
821
|
-
readonly height: number;
|
|
822
|
-
}>;
|
|
823
|
-
}
|
|
824
|
-
|
|
825
|
-
/** Raised when an ONNX model cannot be loaded into an inference session. */
|
|
826
|
-
export declare class ModelLoadError extends OrtVisionError {
|
|
827
|
-
}
|
|
1042
|
+
export declare function isLuminanceAcceptable(luminance: number, threshold: number): boolean;
|
|
828
1043
|
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
* Ultralytics writes `names` as the Python `repr` of a `dict[int, str]` — e.g.
|
|
833
|
-
* `"{0: 'deworm', 1: 'not_deworm'}"`. The value is parsed structurally (never
|
|
834
|
-
* evaluated), and anything unparseable, non-`dict`, or not keyed by contiguous
|
|
835
|
-
* integers from zero is rejected whole rather than half-applied: a partial name
|
|
836
|
-
* map would silently mislabel predictions.
|
|
837
|
-
*
|
|
838
|
-
* @param metadata A model's custom metadata map.
|
|
839
|
-
* @returns Class names in class-id order, or `null` when the model carries no
|
|
840
|
-
* usable `names` entry.
|
|
841
|
-
*/
|
|
842
|
-
export declare function modelNames(metadata: Readonly<Record<string, string>> | undefined): readonly string[] | null;
|
|
1044
|
+
/** Raised when class labels cannot be resolved from the supplied spec. */
|
|
1045
|
+
export declare class LabelMapError extends OrtVisionError {
|
|
1046
|
+
}
|
|
843
1047
|
|
|
844
|
-
|
|
845
|
-
|
|
1048
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
1049
|
+
/**
|
|
1050
|
+
* Class label resolution: presets, lists, dicts, or auto-generated.
|
|
1051
|
+
*
|
|
1052
|
+
* Tasks call {@link resolveLabels} once at construction time to turn whatever
|
|
1053
|
+
* the caller passed (preset name, array, dict, or `null`) into an ordered
|
|
1054
|
+
* array of class names indexed by class id.
|
|
1055
|
+
*
|
|
1056
|
+
* In the browser there is no filesystem, so this module does not load labels
|
|
1057
|
+
* from a path — fetch the file yourself and pass an array.
|
|
1058
|
+
*/
|
|
1059
|
+
/**
|
|
1060
|
+
* Anything accepted by {@link resolveLabels}.
|
|
1061
|
+
*
|
|
1062
|
+
* - `string[]` / `readonly string[]`: explicit names indexed by class id.
|
|
1063
|
+
* - `Record<number, string>`: sparse mapping (gaps filled with `class_<id>`).
|
|
1064
|
+
* - `string`: a preset name (e.g. `"coco"`).
|
|
1065
|
+
* - `null` / `undefined`: auto-generate `class_0` ... `class_{numClasses-1}`.
|
|
1066
|
+
*/
|
|
1067
|
+
export declare type LabelSpec = readonly string[] | Record<number, string> | string | null | undefined;
|
|
846
1068
|
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
* @param iouThreshold Boxes with IoU above this threshold relative to a kept box are suppressed.
|
|
856
|
-
* @returns Indices of kept boxes, in descending score order.
|
|
857
|
-
*/
|
|
858
|
-
export declare function nms(boxes: Float32Array, scores: Float32Array, iouThreshold: number): Int32Array;
|
|
1069
|
+
/**
|
|
1070
|
+
* Resize preserving aspect ratio, padding to `(targetWidth, targetHeight)`
|
|
1071
|
+
* with a constant fill color.
|
|
1072
|
+
*
|
|
1073
|
+
* Standard YOLO preprocessing — returning `scale` and `padLeft`/`padTop`
|
|
1074
|
+
* lets callers map detections back to the original image coordinates.
|
|
1075
|
+
*/
|
|
1076
|
+
export declare function letterbox(image: RGBImage, targetWidth: number, targetHeight: number, fill?: readonly [number, number, number]): LetterboxResult;
|
|
859
1077
|
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
1078
|
+
/**
|
|
1079
|
+
* Reusable letterbox → tensor pipeline for one target resolution.
|
|
1080
|
+
*
|
|
1081
|
+
* Holds a target canvas and an output buffer across calls, so a steady stream
|
|
1082
|
+
* of frames at the same size allocates nothing. Create one per task, not per
|
|
1083
|
+
* frame.
|
|
1084
|
+
*/
|
|
1085
|
+
export declare class LetterboxPipeline {
|
|
1086
|
+
private readonly _targetWidth;
|
|
1087
|
+
private readonly _targetHeight;
|
|
1088
|
+
private readonly _fill;
|
|
1089
|
+
private readonly _target;
|
|
1090
|
+
private readonly _targetContext;
|
|
1091
|
+
private readonly _buffer;
|
|
1092
|
+
private _source;
|
|
1093
|
+
private _sourceContext;
|
|
1094
|
+
private _bufferInUse;
|
|
1095
|
+
/**
|
|
1096
|
+
* @param targetWidth Model input width in pixels.
|
|
1097
|
+
* @param targetHeight Model input height in pixels.
|
|
1098
|
+
* @param fill RGB padding colour; defaults to YOLO grey.
|
|
1099
|
+
*/
|
|
1100
|
+
constructor(targetWidth: number, targetHeight: number, fill?: readonly [number, number, number]);
|
|
1101
|
+
/** The `[width, height]` this pipeline letterboxes into. */
|
|
1102
|
+
get targetSize(): readonly [number, number];
|
|
1103
|
+
/**
|
|
1104
|
+
* Letterbox an image and write it as planar float32.
|
|
1105
|
+
*
|
|
1106
|
+
* The returned buffer is reused between calls unless a previous result is
|
|
1107
|
+
* still checked out — {@link release} marks it free again. A second `run`
|
|
1108
|
+
* before the first is released allocates a fresh buffer rather than
|
|
1109
|
+
* corrupting it, so concurrent `predict()` calls on one task stay correct at
|
|
1110
|
+
* the cost of the allocation they were trying to avoid.
|
|
1111
|
+
*
|
|
1112
|
+
* @param image Source image in the SDK's canonical HWC RGB layout.
|
|
1113
|
+
*/
|
|
1114
|
+
run(image: RGBImage): FusedLetterboxResult;
|
|
1115
|
+
/**
|
|
1116
|
+
* Mark the reusable buffer free again.
|
|
1117
|
+
*
|
|
1118
|
+
* Call it once the tensor built from a {@link run} result has been handed to
|
|
1119
|
+
* ONNX Runtime and the run has resolved — after that the values are inside
|
|
1120
|
+
* the WASM heap and the buffer can be overwritten.
|
|
1121
|
+
*/
|
|
1122
|
+
release(): void;
|
|
1123
|
+
/**
|
|
1124
|
+
* Grow the scratch source canvas to fit an image, reusing it when possible.
|
|
1125
|
+
*
|
|
1126
|
+
* A canvas is only reallocated when a frame arrives at a different size than
|
|
1127
|
+
* the last one, which for a camera or video source is never after the first.
|
|
1128
|
+
*
|
|
1129
|
+
* @param width Source width in pixels.
|
|
1130
|
+
* @param height Source height in pixels.
|
|
1131
|
+
*/
|
|
1132
|
+
private _ensureSource;
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
export declare interface LetterboxResult {
|
|
1136
|
+
/** The padded image at the target size. */
|
|
1137
|
+
readonly image: RGBImage;
|
|
1138
|
+
/** The factor applied to the original image (`< 1` if downscaled). */
|
|
1139
|
+
readonly scale: number;
|
|
1140
|
+
/** Horizontal padding in pixels (left side; right side has the same or +1). */
|
|
1141
|
+
readonly padLeft: number;
|
|
1142
|
+
/** Vertical padding in pixels (top side). */
|
|
1143
|
+
readonly padTop: number;
|
|
1144
|
+
}
|
|
866
1145
|
|
|
867
|
-
/**
|
|
868
|
-
* Wrap an ONNX Runtime Web `InferenceSession` with convenient metadata access.
|
|
869
|
-
*
|
|
870
|
-
* The wrapper exposes input/output names and the shapes the graph declares,
|
|
871
|
-
* manages execution-provider selection, provides a typed {@link OrtSession.run}
|
|
872
|
-
* method, and releases the native session through {@link OrtSession.release}.
|
|
873
|
-
*/
|
|
874
|
-
export declare class OrtSession {
|
|
875
|
-
private readonly _session;
|
|
876
|
-
readonly providers: readonly string[];
|
|
877
|
-
private readonly _metadata;
|
|
878
|
-
private constructor();
|
|
879
1146
|
/**
|
|
880
|
-
*
|
|
1147
|
+
* Letterbox an image into planar float32 without keeping any state.
|
|
881
1148
|
*
|
|
882
|
-
* The
|
|
883
|
-
*
|
|
884
|
-
* WASM heap and then allocates the graph and the weights on top of that copy;
|
|
885
|
-
* a `readModelMetadata` call placed after `InferenceSession.create` keeps the
|
|
886
|
-
* JavaScript-side buffer reachable across the whole build, so a 5 MB model
|
|
887
|
-
* costs 5 MB of JS heap plus 5 MB of WASM heap plus the weights at the same
|
|
888
|
-
* instant. Reading first makes the buffer collectable as soon as ORT has copied
|
|
889
|
-
* it — on a phone that was the difference between a session and
|
|
890
|
-
* `Can't create a session. failed to allocate a buffer of size N`.
|
|
1149
|
+
* The allocation-free path is {@link LetterboxPipeline}; this is the one-shot
|
|
1150
|
+
* form, for a caller who wants the fused behaviour without owning a pipeline.
|
|
891
1151
|
*
|
|
892
|
-
* @param
|
|
893
|
-
* @param
|
|
894
|
-
*
|
|
895
|
-
* @
|
|
1152
|
+
* @param image Source image in the SDK's canonical HWC RGB layout.
|
|
1153
|
+
* @param targetWidth Model input width in pixels.
|
|
1154
|
+
* @param targetHeight Model input height in pixels.
|
|
1155
|
+
* @param fill RGB padding colour; defaults to YOLO grey.
|
|
1156
|
+
*/
|
|
1157
|
+
export declare function letterboxToTensorData(image: RGBImage, targetWidth: number, targetHeight: number, fill?: readonly [number, number, number]): FusedLetterboxResult;
|
|
1158
|
+
|
|
1159
|
+
/**
|
|
1160
|
+
* Load an image from any supported source into a HWC uint8 RGB array.
|
|
1161
|
+
*
|
|
1162
|
+
* @throws {@link ImageLoadError} if the source cannot be decoded or has an unsupported shape.
|
|
896
1163
|
*/
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
get inputNames(): readonly string[];
|
|
900
|
-
/** Name of the first (and usually only) input. */
|
|
901
|
-
get inputName(): string;
|
|
902
|
-
/** Names of the model's outputs, in declaration order. */
|
|
903
|
-
get outputNames(): readonly string[];
|
|
904
|
-
/**
|
|
905
|
-
* Shapes the graph declares for its inputs, in declaration order.
|
|
906
|
-
*
|
|
907
|
-
* Dynamic (symbolic) axes appear as `null`. Empty shapes mean the runtime
|
|
908
|
-
* reported no metadata — either a non-tensor input, or an `onnxruntime-web`
|
|
909
|
-
* older than 1.21, which predates input metadata.
|
|
910
|
-
*/
|
|
911
|
-
get inputShapes(): readonly DeclaredShape[];
|
|
1164
|
+
export declare function loadImage(source: ImageInput): Promise<RGBImage>;
|
|
1165
|
+
|
|
912
1166
|
/**
|
|
913
|
-
*
|
|
914
|
-
*
|
|
915
|
-
*
|
|
1167
|
+
* Error raised when a captured frame is too dark to be analysed reliably.
|
|
1168
|
+
* Carries the measured luminance and the threshold it failed so callers can
|
|
1169
|
+
* surface actionable feedback.
|
|
916
1170
|
*/
|
|
917
|
-
|
|
1171
|
+
export declare class LowLuminanceError extends Error {
|
|
1172
|
+
/** Measured mean luminance, `0..255`. */
|
|
1173
|
+
readonly luminance: number;
|
|
1174
|
+
/** Threshold that was checked against, `0..255`. */
|
|
1175
|
+
readonly threshold: number;
|
|
1176
|
+
/**
|
|
1177
|
+
* @param luminance - the measured mean luminance in `0..255`.
|
|
1178
|
+
* @param threshold - the threshold the measurement failed to reach.
|
|
1179
|
+
*/
|
|
1180
|
+
constructor(luminance: number, threshold: number);
|
|
1181
|
+
}
|
|
1182
|
+
|
|
918
1183
|
/**
|
|
919
|
-
*
|
|
920
|
-
*
|
|
921
|
-
*
|
|
922
|
-
*
|
|
1184
|
+
* Longest edge (in pixels) the source is downsampled to before sampling.
|
|
1185
|
+
* Averaging over a small downsample is statistically equivalent for a
|
|
1186
|
+
* brightness threshold and orders of magnitude faster than reading every pixel
|
|
1187
|
+
* of a full-resolution camera frame.
|
|
923
1188
|
*/
|
|
924
|
-
|
|
1189
|
+
export declare const LUMINANCE_SAMPLE_MAX_EDGE = 256;
|
|
1190
|
+
|
|
925
1191
|
/**
|
|
926
|
-
*
|
|
1192
|
+
* Drawable source we can sample luminance from.
|
|
927
1193
|
*
|
|
928
|
-
*
|
|
1194
|
+
* The list tracks what `CanvasRenderingContext2D.drawImage` accepts and we can
|
|
1195
|
+
* read a pixel size off, which is what the implementation actually needs.
|
|
1196
|
+
* `ImageBitmap` matters for the decode-downscaled path: `createImageBitmap(blob,
|
|
1197
|
+
* { resizeWidth })` is how a caller avoids materialising a full-resolution
|
|
1198
|
+
* phone photo, and the frame it hands back is the frame whose brightness has to
|
|
1199
|
+
* be checked.
|
|
929
1200
|
*/
|
|
930
|
-
|
|
1201
|
+
export declare type LuminanceSource = HTMLImageElement | HTMLVideoElement | HTMLCanvasElement | ImageBitmap | OffscreenCanvas;
|
|
1202
|
+
|
|
931
1203
|
/**
|
|
932
|
-
*
|
|
933
|
-
* Ultralytics export.
|
|
1204
|
+
* Single-channel binary or grayscale mask, laid out row-major.
|
|
934
1205
|
*
|
|
935
|
-
*
|
|
936
|
-
*
|
|
937
|
-
*
|
|
1206
|
+
* `data.length` must equal `width * height`. For binary masks, values are
|
|
1207
|
+
* `0` (background) or `255` (foreground); soft masks may use the full
|
|
1208
|
+
* `[0, 255]` range.
|
|
938
1209
|
*/
|
|
939
|
-
|
|
1210
|
+
export declare class Mask {
|
|
1211
|
+
readonly data: Uint8Array;
|
|
1212
|
+
readonly width: number;
|
|
1213
|
+
readonly height: number;
|
|
1214
|
+
constructor(data: Uint8Array, width: number, height: number);
|
|
1215
|
+
}
|
|
1216
|
+
|
|
940
1217
|
/**
|
|
941
|
-
*
|
|
1218
|
+
* Per-instance binary masks for a single image.
|
|
942
1219
|
*
|
|
943
|
-
*
|
|
944
|
-
*
|
|
945
|
-
* the runtime is ignored: a session being torn down has nothing left to fail
|
|
946
|
-
* at, and the caller is already moving on.
|
|
1220
|
+
* Each mask is cropped to its instance's bounding box. To paint masks onto
|
|
1221
|
+
* a full-image canvas, use `xyxy[i]` as the top-left target.
|
|
947
1222
|
*/
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
*/
|
|
957
|
-
run(feeds: Record<string, ort.Tensor>): Promise<Record<string, ort.Tensor>>;
|
|
958
|
-
}
|
|
959
|
-
|
|
960
|
-
export declare interface OrtSessionOptions {
|
|
961
|
-
/** Execution providers in preference order. `undefined` uses {@link DEFAULT_PROVIDERS}. */
|
|
962
|
-
readonly providers?: readonly string[];
|
|
963
|
-
/** Optional ORT session options forwarded to `InferenceSession.create`. */
|
|
964
|
-
readonly sessionOptions?: ort.InferenceSession.SessionOptions;
|
|
1223
|
+
export declare class Masks {
|
|
1224
|
+
readonly data: ReadonlyArray<{
|
|
1225
|
+
readonly data: Uint8Array;
|
|
1226
|
+
readonly width: number;
|
|
1227
|
+
readonly height: number;
|
|
1228
|
+
}>;
|
|
1229
|
+
readonly xyxy: Float32Array;
|
|
1230
|
+
readonly origShape: readonly [number, number];
|
|
965
1231
|
/**
|
|
966
|
-
*
|
|
967
|
-
*
|
|
968
|
-
*
|
|
969
|
-
* The runtime does not expose that map, so it is read from the file itself —
|
|
970
|
-
* which means a URL model is fetched here and handed to ORT as bytes instead
|
|
971
|
-
* of letting ORT fetch it. That is the same single download either way, and
|
|
972
|
-
* it is what lets a task resolve its labels off the model. Set to `false` to
|
|
973
|
-
* keep the URL path untouched and leave {@link OrtSession.metadata} empty.
|
|
974
|
-
*
|
|
975
|
-
* `false` is also the escape hatch when a device cannot afford the bytes: the
|
|
976
|
-
* fetched buffer is dropped before ORT builds the graph (see
|
|
977
|
-
* {@link OrtSession.create}), but ORT's own load path still keeps the model out
|
|
978
|
-
* of reach of anything the SDK holds. A session built this way resolves its
|
|
979
|
-
* input size from the graph as usual — only the class names are lost, so a
|
|
980
|
-
* caller taking this route has to pass `labels` itself.
|
|
1232
|
+
* @param data Per-instance binary masks (`Mask` objects from `types.ts`).
|
|
1233
|
+
* @param xyxy Flat `[N, 4]` of bounding-box coordinates in original pixels.
|
|
1234
|
+
* @param origShape `[height, width]` of the original image.
|
|
981
1235
|
*/
|
|
982
|
-
|
|
1236
|
+
constructor(data: ReadonlyArray<{
|
|
1237
|
+
readonly data: Uint8Array;
|
|
1238
|
+
readonly width: number;
|
|
1239
|
+
readonly height: number;
|
|
1240
|
+
}>, xyxy: Float32Array, origShape: readonly [number, number]);
|
|
1241
|
+
/** Number of instance masks. */
|
|
1242
|
+
get length(): number;
|
|
1243
|
+
/** `[N]` shape of the masks collection. */
|
|
1244
|
+
get shape(): readonly [number];
|
|
1245
|
+
[Symbol.iterator](): Iterator<{
|
|
1246
|
+
readonly data: Uint8Array;
|
|
1247
|
+
readonly width: number;
|
|
1248
|
+
readonly height: number;
|
|
1249
|
+
}>;
|
|
983
1250
|
}
|
|
984
1251
|
|
|
985
1252
|
/**
|
|
986
|
-
*
|
|
1253
|
+
* Namespace for every metadata key the fusion writes.
|
|
987
1254
|
*
|
|
988
|
-
*
|
|
989
|
-
* the
|
|
1255
|
+
* Namespaced on purpose: the detector's own Ultralytics metadata (`names`,
|
|
1256
|
+
* `task`, `imgsz`) is carried over into the fused model, and an un-prefixed key
|
|
1257
|
+
* would either collide with it or be mistaken for it.
|
|
990
1258
|
*/
|
|
991
|
-
export declare
|
|
992
|
-
|
|
1259
|
+
export declare const METADATA_PREFIX = "ovs.";
|
|
1260
|
+
|
|
1261
|
+
/** Raised when an ONNX model cannot be loaded into an inference session. */
|
|
1262
|
+
export declare class ModelLoadError extends OrtVisionError {
|
|
993
1263
|
}
|
|
994
1264
|
|
|
995
1265
|
/**
|
|
996
|
-
*
|
|
1266
|
+
* Read the class names an export baked into the model metadata.
|
|
1267
|
+
*
|
|
1268
|
+
* Ultralytics writes `names` as the Python `repr` of a `dict[int, str]` — e.g.
|
|
1269
|
+
* `"{0: 'deworm', 1: 'not_deworm'}"`. The value is parsed structurally (never
|
|
1270
|
+
* evaluated), and anything unparseable, non-`dict`, or not keyed by contiguous
|
|
1271
|
+
* integers from zero is rejected whole rather than half-applied: a partial name
|
|
1272
|
+
* map would silently mislabel predictions.
|
|
997
1273
|
*
|
|
998
|
-
*
|
|
1274
|
+
* @param metadata A model's custom metadata map.
|
|
1275
|
+
* @returns Class names in class-id order, or `null` when the model carries no
|
|
1276
|
+
* usable `names` entry.
|
|
999
1277
|
*/
|
|
1000
|
-
export declare
|
|
1001
|
-
readonly data: Float32Array;
|
|
1002
|
-
/** @param data `[numClasses]` per-class probabilities, indexed by class id. */
|
|
1003
|
-
constructor(data: Float32Array);
|
|
1004
|
-
/** Number of classes. */
|
|
1005
|
-
get length(): number;
|
|
1006
|
-
/** `[numClasses]` shape of the underlying vector. */
|
|
1007
|
-
get shape(): readonly [number];
|
|
1008
|
-
/** Index of the most probable class. */
|
|
1009
|
-
get top1(): number;
|
|
1010
|
-
/** Probability of the top-1 class. */
|
|
1011
|
-
get top1conf(): number;
|
|
1012
|
-
/** Indices of the top-5 most probable classes, descending. */
|
|
1013
|
-
get top5(): Int32Array;
|
|
1014
|
-
/** Probabilities of the top-5 classes, descending. */
|
|
1015
|
-
get top5conf(): Float32Array;
|
|
1016
|
-
private _topK;
|
|
1017
|
-
}
|
|
1278
|
+
export declare function modelNames(metadata: Readonly<Record<string, string>> | undefined): readonly string[] | null;
|
|
1018
1279
|
|
|
1019
|
-
/**
|
|
1020
|
-
export declare
|
|
1021
|
-
}
|
|
1280
|
+
/** Anything `InferenceSession.create` accepts. */
|
|
1281
|
+
export declare type ModelSource = string | ArrayBufferLike | Uint8Array;
|
|
1022
1282
|
|
|
1023
1283
|
/**
|
|
1024
|
-
*
|
|
1284
|
+
* Greedy non-maximum suppression on axis-aligned bounding boxes.
|
|
1025
1285
|
*
|
|
1026
|
-
* `
|
|
1027
|
-
*
|
|
1028
|
-
* `imgsz`. The Python SDK gets it for free from
|
|
1029
|
-
* `InferenceSession.get_modelmeta().custom_metadata_map`; in the browser the
|
|
1030
|
-
* only way to the same information is to read it out of the file, so this
|
|
1031
|
-
* module walks just enough of the ModelProto wire format to collect
|
|
1032
|
-
* `metadata_props`.
|
|
1286
|
+
* Mirrors `torchvision.ops.nms` (keeps boxes with the highest score, drops
|
|
1287
|
+
* any subsequent box whose IoU exceeds the threshold).
|
|
1033
1288
|
*
|
|
1034
|
-
*
|
|
1035
|
-
*
|
|
1036
|
-
*
|
|
1289
|
+
* @param boxes Flat array of length `4 * N` in xyxy order: `[x1,y1,x2,y2, ...]`.
|
|
1290
|
+
* @param scores Detection score per box, length `N`.
|
|
1291
|
+
* @param iouThreshold Boxes with IoU above this threshold relative to a kept box are suppressed.
|
|
1292
|
+
* @returns Indices of kept boxes, in descending score order. Boxes tied on
|
|
1293
|
+
* score are visited lowest-index first, so the survivor of a tie is
|
|
1294
|
+
* deterministic and matches both `torchvision` and the Python SDK.
|
|
1037
1295
|
*/
|
|
1038
|
-
|
|
1039
|
-
* Collect a model's custom metadata map straight out of its bytes.
|
|
1040
|
-
*
|
|
1041
|
-
* @param model The `.onnx` file contents.
|
|
1042
|
-
* @returns Key/value metadata — `names`, `task`, `imgsz`, ... for an
|
|
1043
|
-
* Ultralytics export — or an empty object when the file carries none or
|
|
1044
|
-
* cannot be walked.
|
|
1045
|
-
*/
|
|
1046
|
-
export declare function readModelMetadata(model: Uint8Array | ArrayBufferLike): Readonly<Record<string, string>>;
|
|
1047
|
-
|
|
1048
|
-
/** Resize an image to `(targetWidth, targetHeight)` using high-quality canvas resampling. */
|
|
1049
|
-
export declare function resize(image: RGBImage, targetWidth: number, targetHeight: number): RGBImage;
|
|
1296
|
+
export declare function nms(boxes: Float32Array, scores: Float32Array, iouThreshold: number): Int32Array;
|
|
1050
1297
|
|
|
1051
1298
|
/**
|
|
1052
|
-
*
|
|
1299
|
+
* Raised when a detection task finds nothing and was asked to treat that as an error.
|
|
1053
1300
|
*
|
|
1054
|
-
*
|
|
1055
|
-
*
|
|
1056
|
-
*
|
|
1057
|
-
*
|
|
1058
|
-
*
|
|
1301
|
+
* Only raised when the caller opts in with `raiseOnEmpty: true`. The default
|
|
1302
|
+
* stays an empty result, because "the model looked and found nothing" is a
|
|
1303
|
+
* successful inference, not a failure — a photo of an empty field is a valid
|
|
1304
|
+
* photo. What the flag is for is the opposite situation: a pipeline step whose
|
|
1305
|
+
* *precondition* is that something is there, where an empty result means the
|
|
1306
|
+
* caller should stop rather than quietly carry on with zero rows.
|
|
1059
1307
|
*
|
|
1060
|
-
*
|
|
1061
|
-
*
|
|
1308
|
+
* "Nothing was detected" and "nothing was confident enough" are the same
|
|
1309
|
+
* condition here, since the confidence threshold is what decides what counts as
|
|
1310
|
+
* a detection in the first place.
|
|
1062
1311
|
*/
|
|
1063
|
-
export declare
|
|
1064
|
-
|
|
1065
|
-
export declare interface ResolveInputSizeOptions {
|
|
1066
|
-
/** Declared shape of the model's image input, from {@link declaredShapesFrom}. */
|
|
1067
|
-
readonly graphShape?: DeclaredShape;
|
|
1068
|
-
/** Size the caller asked for, if any. */
|
|
1069
|
-
readonly requested?: readonly [number, number];
|
|
1070
|
-
/** Size to use when neither the graph nor the caller pins one. */
|
|
1071
|
-
readonly fallback: readonly [number, number];
|
|
1312
|
+
export declare class NoDetectionsError extends OrtVisionError {
|
|
1072
1313
|
}
|
|
1073
1314
|
|
|
1074
1315
|
/**
|
|
1075
|
-
*
|
|
1316
|
+
* Convert a uint8 image to a normalized float32 array (HWC layout preserved).
|
|
1076
1317
|
*
|
|
1077
|
-
*
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
export declare function resolveLabels(spec: LabelSpec, options?: ResolveLabelsOptions): readonly string[];
|
|
1318
|
+
* Applies `(pixel * scale - mean) / std` channel-wise.
|
|
1319
|
+
*/
|
|
1320
|
+
export declare function normalize(image: RGBImage, mean: readonly [number, number, number], std: readonly [number, number, number], scale?: number): Float32Array;
|
|
1081
1321
|
|
|
1082
|
-
|
|
1322
|
+
/**
|
|
1323
|
+
* Wrap an ONNX Runtime Web `InferenceSession` with convenient metadata access.
|
|
1324
|
+
*
|
|
1325
|
+
* The wrapper exposes input/output names and the shapes the graph declares,
|
|
1326
|
+
* manages execution-provider selection, provides a typed {@link OrtSession.run}
|
|
1327
|
+
* method, and releases the native session through {@link OrtSession.release}.
|
|
1328
|
+
*/
|
|
1329
|
+
export declare class OrtSession {
|
|
1330
|
+
private readonly _session;
|
|
1331
|
+
readonly providers: readonly string[];
|
|
1332
|
+
private readonly _metadata;
|
|
1333
|
+
private constructor();
|
|
1334
|
+
/**
|
|
1335
|
+
* Load an ONNX model into an ORT inference session.
|
|
1336
|
+
*
|
|
1337
|
+
* The metadata map is read **before** the session is built, and that order is
|
|
1338
|
+
* load-bearing on memory-constrained devices. ORT copies the model into its
|
|
1339
|
+
* WASM heap and then allocates the graph and the weights on top of that copy;
|
|
1340
|
+
* a `readModelMetadata` call placed after `InferenceSession.create` keeps the
|
|
1341
|
+
* JavaScript-side buffer reachable across the whole build, so a 5 MB model
|
|
1342
|
+
* costs 5 MB of JS heap plus 5 MB of WASM heap plus the weights at the same
|
|
1343
|
+
* instant. Reading first makes the buffer collectable as soon as ORT has copied
|
|
1344
|
+
* it — on a phone that was the difference between a session and
|
|
1345
|
+
* `Can't create a session. failed to allocate a buffer of size N`.
|
|
1346
|
+
*
|
|
1347
|
+
* @param model Either a URL string, or a `Uint8Array`/`ArrayBuffer` containing the model bytes.
|
|
1348
|
+
* @param options Provider list, pass-through `SessionOptions`, and whether to
|
|
1349
|
+
* read the model's metadata map (see {@link OrtSessionOptions.readMetadata}).
|
|
1350
|
+
* @throws {@link ModelLoadError} if the model cannot be loaded.
|
|
1351
|
+
*/
|
|
1352
|
+
static create(model: ModelSource, options?: OrtSessionOptions): Promise<OrtSession>;
|
|
1353
|
+
/** Names of the model's inputs, in declaration order. */
|
|
1354
|
+
get inputNames(): readonly string[];
|
|
1355
|
+
/** Name of the first (and usually only) input. */
|
|
1356
|
+
get inputName(): string;
|
|
1357
|
+
/** Names of the model's outputs, in declaration order. */
|
|
1358
|
+
get outputNames(): readonly string[];
|
|
1083
1359
|
/**
|
|
1084
|
-
*
|
|
1360
|
+
* Shapes the graph declares for its inputs, in declaration order.
|
|
1085
1361
|
*
|
|
1086
|
-
*
|
|
1087
|
-
*
|
|
1362
|
+
* Dynamic (symbolic) axes appear as `null`. Empty shapes mean the runtime
|
|
1363
|
+
* reported no metadata — either a non-tensor input, or an `onnxruntime-web`
|
|
1364
|
+
* older than 1.21, which predates input metadata.
|
|
1088
1365
|
*/
|
|
1089
|
-
readonly
|
|
1090
|
-
}
|
|
1091
|
-
|
|
1092
|
-
/**
|
|
1093
|
-
* Resolve the execution providers to pass to `InferenceSession.create`.
|
|
1094
|
-
*
|
|
1095
|
-
* @param requested Explicit provider list in preference order; `undefined` returns the default.
|
|
1096
|
-
*/
|
|
1097
|
-
export declare function resolveProviders(requested?: readonly string[]): string[];
|
|
1098
|
-
|
|
1099
|
-
/**
|
|
1100
|
-
* Public output types returned by the SDK's vision tasks.
|
|
1101
|
-
*
|
|
1102
|
-
* These types form the contract between the SDK and its callers. They mirror
|
|
1103
|
-
* the Python `ort-vision-sdk` output dataclasses 1-to-1.
|
|
1104
|
-
*
|
|
1105
|
-
* Naming is intentionally compatible with the Ultralytics / torchvision idiom
|
|
1106
|
-
* (`cls`, `conf`, `box`, `xyxy`, `xywh`, normalized variants) so code ported
|
|
1107
|
-
* from those projects keeps working with minimal edits. The original verbose
|
|
1108
|
-
* names (`classId`, `className`, `confidence`, `bbox`) are still populated for
|
|
1109
|
-
* backwards compatibility.
|
|
1110
|
-
*/
|
|
1111
|
-
/**
|
|
1112
|
-
* HWC RGB uint8 image — the canonical image format used across the SDK.
|
|
1113
|
-
*
|
|
1114
|
-
* `data.length` must equal `width * height * 3`. The buffer is laid out row
|
|
1115
|
-
* by row, top-to-bottom, with each pixel as `[R, G, B]`.
|
|
1116
|
-
*/
|
|
1117
|
-
export declare class RGBImage {
|
|
1118
|
-
readonly data: Uint8Array;
|
|
1119
|
-
readonly width: number;
|
|
1120
|
-
readonly height: number;
|
|
1121
|
-
constructor(data: Uint8Array, width: number, height: number);
|
|
1122
|
-
}
|
|
1123
|
-
|
|
1124
|
-
/**
|
|
1125
|
-
* Single segmented instance produced by an instance-segmentation model.
|
|
1126
|
-
*
|
|
1127
|
-
* Mirrors {@link DetectionResult} and adds the per-instance binary mask
|
|
1128
|
-
* plus a "ready-to-display" background-removed crop.
|
|
1129
|
-
*/
|
|
1130
|
-
export declare interface SegmentationResult {
|
|
1131
|
-
readonly classId: number;
|
|
1132
|
-
readonly className: string;
|
|
1133
|
-
readonly confidence: number;
|
|
1134
|
-
readonly bbox: BoundingBox;
|
|
1135
|
-
/** Alias for `classId` (Ultralytics-style). */
|
|
1136
|
-
readonly cls: number;
|
|
1137
|
-
/** Alias for `className`. */
|
|
1138
|
-
readonly name: string;
|
|
1139
|
-
/** Alias for `confidence` (Ultralytics-style). */
|
|
1140
|
-
readonly conf: number;
|
|
1141
|
-
/** Alias for `bbox` (Ultralytics-style). */
|
|
1142
|
-
readonly box: BoundingBox;
|
|
1366
|
+
get inputShapes(): readonly DeclaredShape[];
|
|
1143
1367
|
/**
|
|
1144
|
-
*
|
|
1145
|
-
*
|
|
1368
|
+
* Shape the graph declares for its first input, dynamic axes as `null`.
|
|
1369
|
+
*
|
|
1370
|
+
* Empty when the runtime reports no metadata for it.
|
|
1146
1371
|
*/
|
|
1147
|
-
|
|
1372
|
+
get inputShape(): DeclaredShape;
|
|
1148
1373
|
/**
|
|
1149
|
-
*
|
|
1150
|
-
*
|
|
1151
|
-
* `
|
|
1374
|
+
* Shapes the graph declares for its outputs, in declaration order.
|
|
1375
|
+
*
|
|
1376
|
+
* Dynamic (symbolic) axes appear as `null`. Reading them is how a task can
|
|
1377
|
+
* tell how many classes a head emits without being told.
|
|
1152
1378
|
*/
|
|
1153
|
-
|
|
1154
|
-
}
|
|
1155
|
-
|
|
1156
|
-
/**
|
|
1157
|
-
* Per-image instance-segmentation envelope (Ultralytics-style `Results`).
|
|
1158
|
-
*
|
|
1159
|
-
* Iterating yields per-instance {@link SegmentationResult} entries. `boxes`
|
|
1160
|
-
* and `masks` mirror Ultralytics' bulk-array views.
|
|
1161
|
-
*/
|
|
1162
|
-
export declare class SegmentationResults implements Iterable<SegmentationResult> {
|
|
1163
|
-
readonly boxes: Boxes;
|
|
1164
|
-
readonly masks: Masks;
|
|
1165
|
-
readonly detections: readonly SegmentationResult[];
|
|
1166
|
-
readonly names: Readonly<Record<number, string>>;
|
|
1167
|
-
readonly origImg: RGBImage;
|
|
1168
|
-
readonly origShape: readonly [number, number];
|
|
1169
|
-
readonly path: string | null;
|
|
1170
|
-
readonly speed: Readonly<Speed>;
|
|
1171
|
-
constructor(boxes: Boxes, masks: Masks, detections: readonly SegmentationResult[], names: Readonly<Record<number, string>>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly<Speed>);
|
|
1172
|
-
/** Number of surviving instances. */
|
|
1173
|
-
get length(): number;
|
|
1174
|
-
/** Index into the per-instance results. */
|
|
1175
|
-
get(index: number): SegmentationResult | undefined;
|
|
1176
|
-
[Symbol.iterator](): Iterator<SegmentationResult>;
|
|
1177
|
-
}
|
|
1178
|
-
|
|
1179
|
-
/**
|
|
1180
|
-
* Instance segmenter for YOLO seg ONNX models (v8-seg / v11-seg / ...).
|
|
1181
|
-
*
|
|
1182
|
-
* The model is expected to expose two outputs:
|
|
1183
|
-
*
|
|
1184
|
-
* 1. `output0`: `(1, 4 + numClasses + numMaskCoefs, numAnchors)` — per-anchor
|
|
1185
|
-
* predictions (boxes, class scores, mask coefficients).
|
|
1186
|
-
* 2. `output1`: `(1, numMaskCoefs, maskH, maskW)` — prototype masks.
|
|
1187
|
-
*
|
|
1188
|
-
* `predict()` returns `Promise<SegmentationResults[]>` (length 1 for a
|
|
1189
|
-
* single image), mirroring Ultralytics' API. The envelope exposes:
|
|
1190
|
-
*
|
|
1191
|
-
* - `boxes`: bulk numpy view (`xyxy`, `xywh`, `xyxyn`, `xywhn`, `cls`, `conf`).
|
|
1192
|
-
* - `masks`: per-instance binary masks cropped to each box.
|
|
1193
|
-
* - per-instance {@link SegmentationResult} via iteration.
|
|
1194
|
-
*
|
|
1195
|
-
* @example
|
|
1196
|
-
* ```typescript
|
|
1197
|
-
* const seg = await Segmenter.create("/models/yolov8n-seg.onnx");
|
|
1198
|
-
* const r = (await seg.predict("/images/street.jpg"))[0];
|
|
1199
|
-
* for (const inst of r) {
|
|
1200
|
-
* console.log(inst.cls, inst.conf, inst.box.xyxy);
|
|
1201
|
-
* }
|
|
1202
|
-
* ```
|
|
1203
|
-
*/
|
|
1204
|
-
export declare class Segmenter extends VisionTask {
|
|
1205
|
-
private readonly _head;
|
|
1206
|
-
private readonly _labels;
|
|
1207
|
-
private readonly _names;
|
|
1208
|
-
private readonly _inputSize;
|
|
1209
|
-
private readonly _confThreshold;
|
|
1210
|
-
private readonly _iouThreshold;
|
|
1211
|
-
private readonly _maxDetections;
|
|
1212
|
-
private readonly _maskThreshold;
|
|
1213
|
-
private constructor();
|
|
1214
|
-
/** Load the model and resolve labels. */
|
|
1215
|
-
static create(model: ModelSource, options?: SegmenterOptions): Promise<Segmenter>;
|
|
1216
|
-
/** The decoder family used to interpret the model's output. */
|
|
1217
|
-
get head(): SegmenterHead;
|
|
1218
|
-
/** Class labels indexed by class id. */
|
|
1219
|
-
get labels(): readonly string[];
|
|
1220
|
-
/** Class id → class name dict (matches Ultralytics' `model.names`). */
|
|
1221
|
-
get names(): Readonly<Record<number, string>>;
|
|
1379
|
+
get outputShapes(): readonly DeclaredShape[];
|
|
1222
1380
|
/**
|
|
1223
|
-
*
|
|
1381
|
+
* Shape the graph declares for its first output, dynamic axes as `null`.
|
|
1224
1382
|
*
|
|
1225
|
-
*
|
|
1226
|
-
* input, so reading it back tells you the resolution inference really runs at
|
|
1227
|
-
* — not merely what was requested.
|
|
1383
|
+
* Empty when the runtime reports no metadata for it.
|
|
1228
1384
|
*/
|
|
1229
|
-
get
|
|
1230
|
-
/** Number of classes the model predicts. */
|
|
1231
|
-
get numClasses(): number;
|
|
1232
|
-
/** Alias for {@link predict} (parity with PyTorch `nn.Module.__call__`). */
|
|
1233
|
-
call(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
|
|
1234
|
-
/** Run instance segmentation on a single image. */
|
|
1235
|
-
predict(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
|
|
1236
|
-
private _preprocess;
|
|
1237
|
-
private _splitOutputs;
|
|
1238
|
-
private _buildResult;
|
|
1239
|
-
private _buildBoxes;
|
|
1240
|
-
private _buildMasks;
|
|
1241
|
-
}
|
|
1242
|
-
|
|
1243
|
-
/**
|
|
1244
|
-
* Decoder family for the segmentation head.
|
|
1245
|
-
*
|
|
1246
|
-
* - `"yolo-seg"`: YOLO instance-segmentation head with two outputs —
|
|
1247
|
-
* `[1, 4 + nc + nm, N]` per-anchor predictions plus `[1, nm, mh, mw]`
|
|
1248
|
-
* prototype masks. Covers YOLOv8-seg, v11-seg, v26-seg.
|
|
1249
|
-
*
|
|
1250
|
-
* The SDK does **not** auto-detect this — the caller is responsible for
|
|
1251
|
-
* picking a head that matches their export.
|
|
1252
|
-
*/
|
|
1253
|
-
export declare type SegmenterHead = "yolo-seg";
|
|
1254
|
-
|
|
1255
|
-
export declare interface SegmenterOptions extends OrtSessionOptions {
|
|
1385
|
+
get outputShape(): DeclaredShape;
|
|
1256
1386
|
/**
|
|
1257
|
-
*
|
|
1258
|
-
*
|
|
1387
|
+
* The model's custom metadata map — `names`, `task`, `imgsz`, ... for an
|
|
1388
|
+
* Ultralytics export.
|
|
1389
|
+
*
|
|
1390
|
+
* Read from the model's bytes at load time, since the runtime does not expose
|
|
1391
|
+
* it. Empty when the session was created with `readMetadata: false`, from a
|
|
1392
|
+
* URL that could not be fetched here, or from a model carrying no metadata.
|
|
1259
1393
|
*/
|
|
1260
|
-
|
|
1261
|
-
/** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */
|
|
1262
|
-
readonly labels?: LabelSpec;
|
|
1263
|
-
/** Number of classes — used to validate the supplied labels. */
|
|
1264
|
-
readonly numClasses?: number;
|
|
1394
|
+
get metadata(): Readonly<Record<string, string>>;
|
|
1265
1395
|
/**
|
|
1266
|
-
*
|
|
1396
|
+
* Release the native session and free its memory.
|
|
1267
1397
|
*
|
|
1268
|
-
*
|
|
1269
|
-
*
|
|
1270
|
-
*
|
|
1398
|
+
* Call it when a session is discarded while the page lives on — rebuilding a
|
|
1399
|
+
* task at a different input size, swapping in a newer model. A failure from
|
|
1400
|
+
* the runtime is ignored: a session being torn down has nothing left to fail
|
|
1401
|
+
* at, and the caller is already moving on.
|
|
1271
1402
|
*/
|
|
1272
|
-
|
|
1273
|
-
/**
|
|
1274
|
-
|
|
1275
|
-
/**
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1403
|
+
release(): Promise<void>;
|
|
1404
|
+
/** The underlying `onnxruntime-web` session, for advanced use cases. */
|
|
1405
|
+
get raw(): ort.InferenceSession;
|
|
1406
|
+
/**
|
|
1407
|
+
* Run inference and return all outputs.
|
|
1408
|
+
*
|
|
1409
|
+
* @param feeds Map of input name to `ort.Tensor`. Keys must match {@link inputNames}.
|
|
1410
|
+
* @throws {@link InferenceError} if ORT raises any error during execution.
|
|
1411
|
+
*/
|
|
1412
|
+
run(feeds: Record<string, ort.Tensor>): Promise<Record<string, ort.Tensor>>;
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
export declare interface OrtSessionOptions {
|
|
1416
|
+
/** Execution providers in preference order. `undefined` uses {@link DEFAULT_PROVIDERS}. */
|
|
1417
|
+
readonly providers?: readonly string[];
|
|
1418
|
+
/** Optional ORT session options forwarded to `InferenceSession.create`. */
|
|
1419
|
+
readonly sessionOptions?: ort.InferenceSession.SessionOptions;
|
|
1420
|
+
/**
|
|
1421
|
+
* Whether to read the model's custom metadata map (`names`, `task`, `imgsz`).
|
|
1422
|
+
* Defaults to `true`.
|
|
1423
|
+
*
|
|
1424
|
+
* The runtime does not expose that map, so it is read from the file itself —
|
|
1425
|
+
* which means a URL model is fetched here and handed to ORT as bytes instead
|
|
1426
|
+
* of letting ORT fetch it. That is the same single download either way, and
|
|
1427
|
+
* it is what lets a task resolve its labels off the model. Set to `false` to
|
|
1428
|
+
* keep the URL path untouched and leave {@link OrtSession.metadata} empty.
|
|
1429
|
+
*
|
|
1430
|
+
* `false` is also the escape hatch when a device cannot afford the bytes: the
|
|
1431
|
+
* fetched buffer is dropped before ORT builds the graph (see
|
|
1432
|
+
* {@link OrtSession.create}), but ORT's own load path still keeps the model out
|
|
1433
|
+
* of reach of anything the SDK holds. A session built this way resolves its
|
|
1434
|
+
* input size from the graph as usual — only the class names are lost, so a
|
|
1435
|
+
* caller taking this route has to pass `labels` itself.
|
|
1436
|
+
*/
|
|
1437
|
+
readonly readMetadata?: boolean;
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
1286
1441
|
/**
|
|
1287
|
-
*
|
|
1288
|
-
*
|
|
1442
|
+
* Exceptions raised by the SDK.
|
|
1443
|
+
*
|
|
1444
|
+
* All exceptions inherit from {@link OrtVisionError}, so callers can catch
|
|
1445
|
+
* the base class to handle any SDK-originated failure uniformly.
|
|
1289
1446
|
*/
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
/**
|
|
1310
|
-
* Per-stage timing for a single `predict()` call.
|
|
1311
|
-
*
|
|
1312
|
-
* Populates the `speed` field every `Results` envelope carries, mirroring
|
|
1313
|
-
* Ultralytics' `results[0].speed`. All values are milliseconds measured with
|
|
1314
|
-
* `performance.now()`.
|
|
1315
|
-
*/
|
|
1316
|
-
/**
|
|
1317
|
-
* Stage durations of one inference, in milliseconds.
|
|
1318
|
-
*
|
|
1319
|
-
* `preprocess`, `inference` and `postprocess` are the three keys Ultralytics
|
|
1320
|
-
* reports, measured over the same boundaries. `load` is specific to this SDK:
|
|
1321
|
-
* `predict()` accepts a URL, `Blob` or DOM element and decodes it internally,
|
|
1322
|
-
* so the fetch/decode cost would otherwise be invisible — and on a cold cache
|
|
1323
|
-
* it dominates everything else.
|
|
1324
|
-
*/
|
|
1325
|
-
export declare interface Speed {
|
|
1326
|
-
/** Fetching and decoding the input into an `RGBImage`. */
|
|
1327
|
-
load: number;
|
|
1328
|
-
/** Letterbox/resize, normalization and tensor packing. */
|
|
1329
|
-
preprocess: number;
|
|
1330
|
-
/** The ONNX Runtime forward pass. */
|
|
1331
|
-
inference: number;
|
|
1332
|
-
/** Decoding raw outputs into results (NMS, mask assembly, top-k). */
|
|
1333
|
-
postprocess: number;
|
|
1334
|
-
}
|
|
1335
|
-
|
|
1336
|
-
/**
|
|
1337
|
-
* Accumulate stage durations while a `predict()` call runs.
|
|
1338
|
-
*
|
|
1339
|
-
* Each `stage()` call closes the previous stage: the elapsed time since the
|
|
1340
|
-
* last boundary is attributed to the name given. This keeps the call sites
|
|
1341
|
-
* free of paired start/stop bookkeeping and guarantees the four stages tile
|
|
1342
|
-
* the whole call without gaps.
|
|
1343
|
-
*/
|
|
1344
|
-
export declare class SpeedTimer {
|
|
1345
|
-
private _last;
|
|
1346
|
-
private readonly _speed;
|
|
1347
|
-
constructor();
|
|
1447
|
+
export declare class OrtVisionError extends Error {
|
|
1448
|
+
constructor(message: string, options?: ErrorOptions);
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
/** Name of the `[K, 4]` float32 xyxy output, in **letterboxed** input pixels. */
|
|
1452
|
+
export declare const OUTPUT_BOXES = "boxes";
|
|
1453
|
+
|
|
1454
|
+
/** Name of the `[K]` int64 detector-class output. */
|
|
1455
|
+
export declare const OUTPUT_CLASSES = "classes";
|
|
1456
|
+
|
|
1457
|
+
/** Name of the `[1]` int64 output holding how many of the `K` rows are real. */
|
|
1458
|
+
export declare const OUTPUT_NUM_DETECTIONS = "num_detections";
|
|
1459
|
+
|
|
1460
|
+
/** Name of the `[K, numClassifierClasses]` float32 classifier output, one row per box. */
|
|
1461
|
+
export declare const OUTPUT_PROBS = "probs";
|
|
1462
|
+
|
|
1463
|
+
/** Name of the `[K]` float32 detection-confidence output. */
|
|
1464
|
+
export declare const OUTPUT_SCORES = "scores";
|
|
1465
|
+
|
|
1348
1466
|
/**
|
|
1349
|
-
*
|
|
1467
|
+
* Parse a `repr`-encoded `dict[int, str]` class map.
|
|
1350
1468
|
*
|
|
1351
|
-
* @
|
|
1469
|
+
* Split out of {@link modelNames} because the same encoding is reused by a
|
|
1470
|
+
* fused pipeline, which carries one class map per stage and therefore cannot
|
|
1471
|
+
* store both under the single `names` key Ultralytics uses.
|
|
1472
|
+
*
|
|
1473
|
+
* @param encoded The encoded map — e.g. `"{0: 'deworm', 1: 'not_deworm'}"`.
|
|
1474
|
+
* @returns Class names in class-id order, or `null` when the value is missing,
|
|
1475
|
+
* unparseable, not a `dict`, or not keyed by contiguous integers from zero.
|
|
1352
1476
|
*/
|
|
1353
|
-
|
|
1477
|
+
export declare function parseNames(encoded: string | undefined): readonly string[] | null;
|
|
1478
|
+
|
|
1354
1479
|
/**
|
|
1355
|
-
*
|
|
1480
|
+
* Top-k classification probabilities for a single image.
|
|
1356
1481
|
*
|
|
1357
|
-
*
|
|
1482
|
+
* Mirrors Ultralytics' `Probs` interface.
|
|
1358
1483
|
*/
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
/**
|
|
1383
|
-
* Return the top-k entries of a 1-D probability vector, sorted descending.
|
|
1384
|
-
*
|
|
1385
|
-
* @param k Number of entries to return; `null` returns all entries.
|
|
1386
|
-
*/
|
|
1387
|
-
export declare function topK(probabilities: Float32Array, k: number | null): TopKResult;
|
|
1388
|
-
|
|
1389
|
-
export declare interface TopKResult {
|
|
1390
|
-
readonly indices: Int32Array;
|
|
1391
|
-
readonly values: Float32Array;
|
|
1392
|
-
}
|
|
1393
|
-
|
|
1394
|
-
/**
|
|
1395
|
-
* Convert an HWC uint8 image to a CHW `Float32Array` scaled to `[0, 1]`.
|
|
1396
|
-
*
|
|
1397
|
-
* Mirrors `torchvision.transforms.ToTensor()` semantics: HWC → CHW,
|
|
1398
|
-
* `uint8 → float32 / 255`. Useful as input to YOLO-style detectors that
|
|
1399
|
-
* don't require ImageNet normalization.
|
|
1400
|
-
*
|
|
1401
|
-
* @returns CHW `Float32Array` of length `width * height * 3`.
|
|
1402
|
-
*/
|
|
1403
|
-
export declare function toTensor(image: RGBImage): Float32Array;
|
|
1404
|
-
|
|
1405
|
-
/**
|
|
1406
|
-
* Acquire a `MediaStream` via `getUserMedia`, attach it to a `<video>` element,
|
|
1407
|
-
* and expose status/error so the page can render permission and error states.
|
|
1408
|
-
* The stream is automatically released on unmount or retry.
|
|
1409
|
-
*
|
|
1410
|
-
* Defaults to the rear ("environment") camera; desktops fall back to whatever
|
|
1411
|
-
* single camera they expose. Pass `options.constraints` to override, or
|
|
1412
|
-
* `options.enabled: false` to render the surface without prompting for the camera
|
|
1413
|
-
* at all.
|
|
1414
|
-
*
|
|
1415
|
-
* Implementation notes:
|
|
1416
|
-
* - Cleanup detaches the stream from a *snapshotted* video node, so it releases
|
|
1417
|
-
* the same element it attached to even if the page remounts the `<video>`.
|
|
1418
|
-
* - When `getUserMedia` is missing, an insecure context is the usual cause, so
|
|
1419
|
-
* the hook prefers that (actionable) error; otherwise it reports `unsupported`.
|
|
1420
|
-
* - `video.play()` rejections are swallowed: autoplay may be blocked, but the
|
|
1421
|
-
* user gesture that opened the camera usually counts and playback resumes on
|
|
1422
|
-
* the next interaction.
|
|
1423
|
-
*
|
|
1424
|
-
* @param options - optional configuration (see {@link UseCameraStreamOptions}).
|
|
1425
|
-
* @returns The stream status, classified error, a `videoRef` to attach, the live
|
|
1426
|
-
* `stream` for whatever needs the track itself, and a `retry()` to re-attempt
|
|
1427
|
-
* acquisition.
|
|
1428
|
-
*/
|
|
1429
|
-
export declare function useCameraStream(options?: UseCameraStreamOptions): UseCameraStreamApi;
|
|
1430
|
-
|
|
1431
|
-
/** Value returned by {@link useCameraStream}. */
|
|
1432
|
-
export declare interface UseCameraStreamApi {
|
|
1433
|
-
/** Current lifecycle status. */
|
|
1434
|
-
status: CameraStreamStatus;
|
|
1435
|
-
/** The classified error, or `null` while not in the `error` status. */
|
|
1436
|
-
error: CameraStreamError | null;
|
|
1437
|
-
/** Attach to a `<video ref={…} />`. The stream is wired to it once ready. */
|
|
1438
|
-
videoRef: RefObject<HTMLVideoElement | null>;
|
|
1484
|
+
export declare class Probs {
|
|
1485
|
+
readonly data: Float32Array;
|
|
1486
|
+
/** @param data `[numClasses]` per-class probabilities, indexed by class id. */
|
|
1487
|
+
constructor(data: Float32Array);
|
|
1488
|
+
/** Number of classes. */
|
|
1489
|
+
get length(): number;
|
|
1490
|
+
/** `[numClasses]` shape of the underlying vector. */
|
|
1491
|
+
get shape(): readonly [number];
|
|
1492
|
+
/** Index of the most probable class. */
|
|
1493
|
+
get top1(): number;
|
|
1494
|
+
/** Probability of the top-1 class. */
|
|
1495
|
+
get top1conf(): number;
|
|
1496
|
+
/** Indices of the top-5 most probable classes, descending. */
|
|
1497
|
+
get top5(): Int32Array;
|
|
1498
|
+
/** Probabilities of the top-5 classes, descending. */
|
|
1499
|
+
get top5conf(): Float32Array;
|
|
1500
|
+
private _topK;
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
/** Raised when a requested execution provider is not available. */
|
|
1504
|
+
export declare class ProviderNotAvailableError extends OrtVisionError {
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1439
1507
|
/**
|
|
1440
|
-
*
|
|
1508
|
+
* Read a pipeline spec out of a model's custom metadata.
|
|
1441
1509
|
*
|
|
1442
|
-
*
|
|
1443
|
-
*
|
|
1444
|
-
*
|
|
1445
|
-
*
|
|
1510
|
+
* Individual malformed entries fall back to the value a fusion would have used
|
|
1511
|
+
* by default — a single bad float is not a reason to reject an otherwise
|
|
1512
|
+
* loadable pipeline. A malformed resolution is fatal, because there is no safe
|
|
1513
|
+
* default for one.
|
|
1514
|
+
*
|
|
1515
|
+
* @param metadata A model's custom metadata map, as read by
|
|
1516
|
+
* {@link readModelMetadata}.
|
|
1517
|
+
* @returns The decoded spec, or `null` when the model is not a fused pipeline —
|
|
1518
|
+
* it carries no `ovs.kind` entry, or one naming a pipeline kind this version
|
|
1519
|
+
* does not know how to drive.
|
|
1446
1520
|
*/
|
|
1447
|
-
|
|
1448
|
-
/** Manually re-attempt after an error (e.g. the user changed permissions). */
|
|
1449
|
-
retry: () => void;
|
|
1450
|
-
}
|
|
1521
|
+
export declare function readFusionSpec(metadata: Readonly<Record<string, string>> | undefined): FusionSpec | null;
|
|
1451
1522
|
|
|
1452
|
-
|
|
1453
|
-
export declare interface UseCameraStreamOptions {
|
|
1523
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
1454
1524
|
/**
|
|
1455
|
-
*
|
|
1456
|
-
*
|
|
1457
|
-
*
|
|
1525
|
+
* Read the metadata an exporter baked into a `.onnx` file.
|
|
1526
|
+
*
|
|
1527
|
+
* `onnxruntime-web` exposes input/output metadata but **not** the model's
|
|
1528
|
+
* custom metadata map, which is where Ultralytics writes `names`, `task` and
|
|
1529
|
+
* `imgsz`. The Python SDK gets it for free from
|
|
1530
|
+
* `InferenceSession.get_modelmeta().custom_metadata_map`; in the browser the
|
|
1531
|
+
* only way to the same information is to read it out of the file, so this
|
|
1532
|
+
* module walks just enough of the ModelProto wire format to collect
|
|
1533
|
+
* `metadata_props`.
|
|
1534
|
+
*
|
|
1535
|
+
* It never throws and never allocates unbounded: a truncated, hostile or
|
|
1536
|
+
* simply unexpected file yields an empty map, and every caller treats that as
|
|
1537
|
+
* "the model says nothing", falling back to what it was given.
|
|
1458
1538
|
*/
|
|
1459
|
-
constraints?: MediaStreamConstraints;
|
|
1460
1539
|
/**
|
|
1461
|
-
*
|
|
1540
|
+
* Collect a model's custom metadata map straight out of its bytes.
|
|
1462
1541
|
*
|
|
1463
|
-
*
|
|
1464
|
-
*
|
|
1465
|
-
*
|
|
1466
|
-
*
|
|
1467
|
-
* also releases a stream that is already live.
|
|
1542
|
+
* @param model The `.onnx` file contents.
|
|
1543
|
+
* @returns Key/value metadata — `names`, `task`, `imgsz`, ... for an
|
|
1544
|
+
* Ultralytics export — or an empty object when the file carries none or
|
|
1545
|
+
* cannot be walked.
|
|
1468
1546
|
*/
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1547
|
+
export declare function readModelMetadata(model: Uint8Array | ArrayBufferLike): Readonly<Record<string, string>>;
|
|
1548
|
+
|
|
1549
|
+
/**
|
|
1550
|
+
* Turn an empty result into an error, when the caller asked for that.
|
|
1551
|
+
*
|
|
1552
|
+
* Shared by every task that can come back with nothing — {@link Detector},
|
|
1553
|
+
* {@link Segmenter} and {@link DetectClassify} — so the three agree on when
|
|
1554
|
+
* they throw and on what the message says. The message names the two settings
|
|
1555
|
+
* that decide the outcome, because "no detections" on its own leaves the reader
|
|
1556
|
+
* unable to tell a blank image from a threshold set too high.
|
|
1557
|
+
*
|
|
1558
|
+
* @param count How many detections survived every filter.
|
|
1559
|
+
* @param options The flag for this call, the threshold actually applied (after
|
|
1560
|
+
* any per-call override), the class allowlist if one narrowed the search, and
|
|
1561
|
+
* the source path when the input was one.
|
|
1562
|
+
* @throws {@link NoDetectionsError} when the flag is set and `count` is zero.
|
|
1563
|
+
*/
|
|
1564
|
+
export declare function requireDetections(count: number, options: {
|
|
1565
|
+
readonly raiseOnEmpty: boolean;
|
|
1566
|
+
readonly confThreshold: number;
|
|
1567
|
+
readonly classes: readonly number[] | undefined;
|
|
1568
|
+
readonly path: string | null;
|
|
1569
|
+
}): void;
|
|
1570
|
+
|
|
1571
|
+
/** Resize an image to `(targetWidth, targetHeight)` using high-quality canvas resampling. */
|
|
1572
|
+
export declare function resize(image: RGBImage, targetWidth: number, targetHeight: number): RGBImage;
|
|
1573
|
+
|
|
1574
|
+
/**
|
|
1575
|
+
* Decide the input size a task will preprocess to.
|
|
1576
|
+
*
|
|
1577
|
+
* Precedence is graph → caller → fallback. The graph wins over an explicit
|
|
1578
|
+
* `inputSize` because a static shape is not a preference, it is what ORT will
|
|
1579
|
+
* accept: honoring the caller there would only turn a fixable mismatch into a
|
|
1580
|
+
* failed run. A disagreement is a configuration bug in the caller, so it is
|
|
1581
|
+
* reported through `console.warn` instead of being swallowed.
|
|
1582
|
+
*
|
|
1583
|
+
* @param options Graph shape, requested size and per-task fallback.
|
|
1584
|
+
* @returns The `[width, height]` to preprocess to.
|
|
1585
|
+
*/
|
|
1586
|
+
export declare function resolveInputSize(options: ResolveInputSizeOptions): readonly [number, number];
|
|
1587
|
+
|
|
1588
|
+
export declare interface ResolveInputSizeOptions {
|
|
1589
|
+
/** Declared shape of the model's image input, from {@link declaredShapesFrom}. */
|
|
1590
|
+
readonly graphShape?: DeclaredShape;
|
|
1591
|
+
/** Size the caller asked for, if any. */
|
|
1592
|
+
readonly requested?: readonly [number, number];
|
|
1593
|
+
/** Size to use when neither the graph nor the caller pins one. */
|
|
1594
|
+
readonly fallback: readonly [number, number];
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
export declare function resolveLabels(spec: LabelSpec, options?: ResolveLabelsOptions): readonly string[];
|
|
1598
|
+
|
|
1599
|
+
export declare interface ResolveLabelsOptions {
|
|
1600
|
+
/**
|
|
1601
|
+
* Expected number of classes.
|
|
1602
|
+
*
|
|
1603
|
+
* - When `spec` is `null`/`undefined`, this is required to auto-generate names.
|
|
1604
|
+
* - When `spec` is provided, it validates that the resolved length matches.
|
|
1605
|
+
*/
|
|
1606
|
+
readonly numClasses?: number;
|
|
1607
|
+
}
|
|
1608
|
+
|
|
1609
|
+
/**
|
|
1610
|
+
* Resolve the execution providers to pass to `InferenceSession.create`.
|
|
1611
|
+
*
|
|
1612
|
+
* @param requested Explicit provider list in preference order; `undefined` returns the default.
|
|
1613
|
+
*/
|
|
1614
|
+
export declare function resolveProviders(requested?: readonly string[]): string[];
|
|
1615
|
+
|
|
1616
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
1617
|
+
/**
|
|
1618
|
+
* Public output types returned by the SDK's vision tasks.
|
|
1619
|
+
*
|
|
1620
|
+
* These types form the contract between the SDK and its callers. They mirror
|
|
1621
|
+
* the Python `ort-vision-sdk` output dataclasses 1-to-1.
|
|
1622
|
+
*
|
|
1623
|
+
* Naming is intentionally compatible with the Ultralytics / torchvision idiom
|
|
1624
|
+
* (`cls`, `conf`, `box`, `xyxy`, `xywh`, normalized variants) so code ported
|
|
1625
|
+
* from those projects keeps working with minimal edits. The original verbose
|
|
1626
|
+
* names (`classId`, `className`, `confidence`, `bbox`) are still populated for
|
|
1627
|
+
* backwards compatibility.
|
|
1628
|
+
*/
|
|
1629
|
+
/**
|
|
1630
|
+
* HWC RGB uint8 image — the canonical image format used across the SDK.
|
|
1631
|
+
*
|
|
1632
|
+
* `data.length` must equal `width * height * 3`. The buffer is laid out row
|
|
1633
|
+
* by row, top-to-bottom, with each pixel as `[R, G, B]`.
|
|
1634
|
+
*/
|
|
1635
|
+
export declare class RGBImage {
|
|
1636
|
+
readonly data: Uint8Array;
|
|
1637
|
+
readonly width: number;
|
|
1638
|
+
readonly height: number;
|
|
1639
|
+
constructor(data: Uint8Array, width: number, height: number);
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
/**
|
|
1643
|
+
* Single segmented instance produced by an instance-segmentation model.
|
|
1644
|
+
*
|
|
1645
|
+
* Mirrors {@link DetectionResult} and adds the per-instance binary mask
|
|
1646
|
+
* plus a "ready-to-display" background-removed crop.
|
|
1647
|
+
*/
|
|
1648
|
+
export declare interface SegmentationResult {
|
|
1649
|
+
readonly classId: number;
|
|
1650
|
+
readonly className: string;
|
|
1651
|
+
readonly confidence: number;
|
|
1652
|
+
readonly bbox: BoundingBox;
|
|
1653
|
+
/** Alias for `classId` (Ultralytics-style). */
|
|
1654
|
+
readonly cls: number;
|
|
1655
|
+
/** Alias for `className`. */
|
|
1656
|
+
readonly name: string;
|
|
1657
|
+
/** Alias for `confidence` (Ultralytics-style). */
|
|
1658
|
+
readonly conf: number;
|
|
1659
|
+
/** Alias for `bbox` (Ultralytics-style). */
|
|
1660
|
+
readonly box: BoundingBox;
|
|
1661
|
+
/**
|
|
1662
|
+
* Binary mask cropped to `bbox`. Values are `0` (background) or `255`
|
|
1663
|
+
* (foreground). Empty boxes yield a zero-sized `Mask`.
|
|
1664
|
+
*/
|
|
1665
|
+
readonly mask: Mask;
|
|
1666
|
+
/**
|
|
1667
|
+
* The original image cropped to `bbox` with background pixels (where
|
|
1668
|
+
* `mask.data[i] === 0`) zeroed out. Empty boxes yield a zero-sized
|
|
1669
|
+
* `RGBImage`.
|
|
1670
|
+
*/
|
|
1671
|
+
readonly segmentedImage: RGBImage;
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
/**
|
|
1675
|
+
* Per-image instance-segmentation envelope (Ultralytics-style `Results`).
|
|
1676
|
+
*
|
|
1677
|
+
* Iterating yields per-instance {@link SegmentationResult} entries. `boxes`
|
|
1678
|
+
* and `masks` mirror Ultralytics' bulk-array views.
|
|
1679
|
+
*/
|
|
1680
|
+
export declare class SegmentationResults implements Iterable<SegmentationResult> {
|
|
1681
|
+
readonly boxes: Boxes;
|
|
1682
|
+
readonly masks: Masks;
|
|
1683
|
+
readonly detections: readonly SegmentationResult[];
|
|
1684
|
+
readonly names: Readonly<Record<number, string>>;
|
|
1685
|
+
readonly origImg: RGBImage;
|
|
1686
|
+
readonly origShape: readonly [number, number];
|
|
1687
|
+
readonly path: string | null;
|
|
1688
|
+
readonly speed: Readonly<Speed>;
|
|
1689
|
+
constructor(boxes: Boxes, masks: Masks, detections: readonly SegmentationResult[], names: Readonly<Record<number, string>>, origImg: RGBImage, origShape: readonly [number, number], path?: string | null, speed?: Readonly<Speed>);
|
|
1690
|
+
/** Number of surviving instances. */
|
|
1691
|
+
get length(): number;
|
|
1692
|
+
/** Index into the per-instance results. */
|
|
1693
|
+
get(index: number): SegmentationResult | undefined;
|
|
1694
|
+
[Symbol.iterator](): Iterator<SegmentationResult>;
|
|
1695
|
+
}
|
|
1696
|
+
|
|
1697
|
+
/**
|
|
1698
|
+
* Instance segmenter for YOLO seg ONNX models (v8-seg / v11-seg / ...).
|
|
1699
|
+
*
|
|
1700
|
+
* The model is expected to expose two outputs:
|
|
1701
|
+
*
|
|
1702
|
+
* 1. `output0`: `(1, 4 + numClasses + numMaskCoefs, numAnchors)` — per-anchor
|
|
1703
|
+
* predictions (boxes, class scores, mask coefficients).
|
|
1704
|
+
* 2. `output1`: `(1, numMaskCoefs, maskH, maskW)` — prototype masks.
|
|
1705
|
+
*
|
|
1706
|
+
* `predict()` returns `Promise<SegmentationResults[]>` (length 1 for a
|
|
1707
|
+
* single image), mirroring Ultralytics' API. The envelope exposes:
|
|
1708
|
+
*
|
|
1709
|
+
* - `boxes`: bulk numpy view (`xyxy`, `xywh`, `xyxyn`, `xywhn`, `cls`, `conf`).
|
|
1710
|
+
* - `masks`: per-instance binary masks cropped to each box.
|
|
1711
|
+
* - per-instance {@link SegmentationResult} via iteration.
|
|
1712
|
+
*
|
|
1713
|
+
* @example
|
|
1714
|
+
* ```typescript
|
|
1715
|
+
* const seg = await Segmenter.create("/models/yolov8n-seg.onnx");
|
|
1716
|
+
* const r = (await seg.predict("/images/street.jpg"))[0];
|
|
1717
|
+
* for (const inst of r) {
|
|
1718
|
+
* console.log(inst.cls, inst.conf, inst.box.xyxy);
|
|
1719
|
+
* }
|
|
1720
|
+
* ```
|
|
1721
|
+
*/
|
|
1722
|
+
export declare class Segmenter extends VisionTask {
|
|
1723
|
+
private readonly _head;
|
|
1724
|
+
private readonly _labels;
|
|
1725
|
+
private readonly _names;
|
|
1726
|
+
private readonly _inputSize;
|
|
1727
|
+
private readonly _confThreshold;
|
|
1728
|
+
private readonly _iouThreshold;
|
|
1729
|
+
private readonly _maxDetections;
|
|
1730
|
+
private readonly _maskThreshold;
|
|
1731
|
+
private readonly _raiseOnEmpty;
|
|
1732
|
+
private constructor();
|
|
1733
|
+
private _pipelineCache;
|
|
1734
|
+
/**
|
|
1735
|
+
* Run the model once on a zero-filled tensor, paying one-time costs up front.
|
|
1736
|
+
*
|
|
1737
|
+
* The first inference of a session is not representative: WebGPU compiles its
|
|
1738
|
+
* shaders on it and the WASM backend faults in its arenas, which on a phone
|
|
1739
|
+
* can turn the first frame into seconds while every later frame is tens of
|
|
1740
|
+
* milliseconds. Calling this while a loading spinner is still up moves that
|
|
1741
|
+
* cost somewhere the user is already waiting.
|
|
1742
|
+
*
|
|
1743
|
+
* @param runs How many warm-up inferences to run. One is enough for WASM;
|
|
1744
|
+
* WebGPU sometimes settles on the second.
|
|
1745
|
+
*/
|
|
1746
|
+
warmup(runs?: number): Promise<void>;
|
|
1747
|
+
/**
|
|
1748
|
+
* The fused preprocessing pipeline, built on first use.
|
|
1749
|
+
*
|
|
1750
|
+
* Lazily, because constructing it allocates canvases: a task built in an
|
|
1751
|
+
* environment without a canvas implementation stays constructible, and only
|
|
1752
|
+
* fails if it is actually asked to preprocess something.
|
|
1753
|
+
*/
|
|
1754
|
+
private get _pipeline();
|
|
1755
|
+
/** Load the model and resolve labels. */
|
|
1756
|
+
static create(model: ModelSource, options?: SegmenterOptions): Promise<Segmenter>;
|
|
1757
|
+
/** The decoder family used to interpret the model's output. */
|
|
1758
|
+
get head(): SegmenterHead;
|
|
1759
|
+
/** Class labels indexed by class id. */
|
|
1760
|
+
get labels(): readonly string[];
|
|
1761
|
+
/** Class id → class name dict (matches Ultralytics' `model.names`). */
|
|
1762
|
+
get names(): Readonly<Record<number, string>>;
|
|
1763
|
+
/**
|
|
1764
|
+
* The `[width, height]` this task preprocesses to.
|
|
1765
|
+
*
|
|
1766
|
+
* Resolved at creation time from the model's graph when it declares a static
|
|
1767
|
+
* input, so reading it back tells you the resolution inference really runs at
|
|
1768
|
+
* — not merely what was requested.
|
|
1769
|
+
*/
|
|
1770
|
+
get inputSize(): readonly [number, number];
|
|
1771
|
+
/** Number of classes the model predicts. */
|
|
1772
|
+
get numClasses(): number;
|
|
1773
|
+
/** Alias for {@link predict} (parity with PyTorch `nn.Module.__call__`). */
|
|
1774
|
+
call(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
|
|
1775
|
+
/** Run instance segmentation on a single image. */
|
|
1776
|
+
predict(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
|
|
1777
|
+
private _preprocess;
|
|
1778
|
+
private _splitOutputs;
|
|
1779
|
+
private _buildResult;
|
|
1780
|
+
private _buildBoxes;
|
|
1781
|
+
private _buildMasks;
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
/**
|
|
1785
|
+
* Decoder family for the segmentation head.
|
|
1786
|
+
*
|
|
1787
|
+
* - `"yolo-seg"`: YOLO instance-segmentation head with two outputs —
|
|
1788
|
+
* `[1, 4 + nc + nm, N]` per-anchor predictions plus `[1, nm, mh, mw]`
|
|
1789
|
+
* prototype masks. Covers YOLOv8-seg, v11-seg, v26-seg.
|
|
1790
|
+
*
|
|
1791
|
+
* The SDK does **not** auto-detect this — the caller is responsible for
|
|
1792
|
+
* picking a head that matches their export.
|
|
1793
|
+
*/
|
|
1794
|
+
export declare type SegmenterHead = "yolo-seg";
|
|
1795
|
+
|
|
1796
|
+
export declare interface SegmenterOptions extends OrtSessionOptions {
|
|
1797
|
+
/**
|
|
1798
|
+
* Decoder family for the segmentation head. Default `"yolo-seg"` covers
|
|
1799
|
+
* YOLOv8-seg/v11-seg/v26-seg.
|
|
1800
|
+
*/
|
|
1801
|
+
readonly head?: SegmenterHead;
|
|
1802
|
+
/** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */
|
|
1803
|
+
readonly labels?: LabelSpec;
|
|
1804
|
+
/** Number of classes — used to validate the supplied labels. */
|
|
1805
|
+
readonly numClasses?: number;
|
|
1806
|
+
/**
|
|
1807
|
+
* Model input `[width, height]` in pixels for letterboxing.
|
|
1808
|
+
*
|
|
1809
|
+
* Only used when the model's graph leaves its spatial axes dynamic: a graph
|
|
1810
|
+
* that declares a static size always wins, since that is the only shape ONNX
|
|
1811
|
+
* Runtime will accept. Defaults to `[640, 640]`.
|
|
1812
|
+
*/
|
|
1813
|
+
readonly inputSize?: readonly [number, number];
|
|
1814
|
+
/** Default minimum class score to keep a candidate. */
|
|
1815
|
+
readonly confThreshold?: number;
|
|
1816
|
+
/** Default IoU threshold for non-maximum suppression. */
|
|
1817
|
+
readonly iouThreshold?: number;
|
|
1818
|
+
/** Maximum number of instances per image. */
|
|
1819
|
+
readonly maxDetections?: number;
|
|
1820
|
+
/**
|
|
1821
|
+
* If `true`, a run that finds nothing throws {@link NoDetectionsError}
|
|
1822
|
+
* instead of returning an empty envelope. Default `false`, because looking
|
|
1823
|
+
* and finding nothing is a successful inference. Turn it on when an empty
|
|
1824
|
+
* result means the surrounding pipeline should stop rather than carry on with
|
|
1825
|
+
* zero rows. Can be overridden per `predict` call.
|
|
1826
|
+
*/
|
|
1827
|
+
readonly raiseOnEmpty?: boolean;
|
|
1828
|
+
/** Probability cutoff applied to soft masks. Defaults to `0.5`. */
|
|
1829
|
+
readonly maskThreshold?: number;
|
|
1830
|
+
}
|
|
1831
|
+
|
|
1832
|
+
export declare interface SegmenterPredictOptions {
|
|
1833
|
+
readonly confThreshold?: number;
|
|
1834
|
+
readonly iouThreshold?: number;
|
|
1835
|
+
/**
|
|
1836
|
+
* If set, keep only instances whose `classId` is in this list.
|
|
1837
|
+
* Mirrors Ultralytics' `model.predict(img, classes=[0, 16])`.
|
|
1838
|
+
*/
|
|
1839
|
+
readonly classes?: readonly number[];
|
|
1840
|
+
/** Override the constructor's `raiseOnEmpty` setting for this call. */
|
|
1841
|
+
readonly raiseOnEmpty?: boolean;
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1844
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
1845
|
+
/**
|
|
1846
|
+
* Classification head postprocessing — softmax + top-k.
|
|
1847
|
+
*/
|
|
1848
|
+
/** Apply numerically-stable softmax to a 1-D vector of logits. */
|
|
1849
|
+
export declare function softmax(logits: Float32Array | readonly number[]): Float32Array;
|
|
1850
|
+
|
|
1851
|
+
/**
|
|
1852
|
+
* Read the spatial input size out of a declared NCHW shape.
|
|
1853
|
+
*
|
|
1854
|
+
* @param shape The declared shape of the model's image input.
|
|
1855
|
+
* @returns `[width, height]` in pixels, or `null` when the shape is not 4D or
|
|
1856
|
+
* leaves either spatial axis dynamic — in which case the model accepts more
|
|
1857
|
+
* than one resolution and there is nothing to correct.
|
|
1858
|
+
*/
|
|
1859
|
+
export declare function spatialInputSize(shape: DeclaredShape): readonly [number, number] | null;
|
|
1860
|
+
|
|
1861
|
+
/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */
|
|
1862
|
+
/**
|
|
1863
|
+
* Per-stage timing for a single `predict()` call.
|
|
1864
|
+
*
|
|
1865
|
+
* Populates the `speed` field every `Results` envelope carries, mirroring
|
|
1866
|
+
* Ultralytics' `results[0].speed`. All values are milliseconds measured with
|
|
1867
|
+
* `performance.now()`.
|
|
1868
|
+
*/
|
|
1869
|
+
/**
|
|
1870
|
+
* Stage durations of one inference, in milliseconds.
|
|
1871
|
+
*
|
|
1872
|
+
* `preprocess`, `inference` and `postprocess` are the three keys Ultralytics
|
|
1873
|
+
* reports, measured over the same boundaries. `load` is specific to this SDK:
|
|
1874
|
+
* `predict()` accepts a URL, `Blob` or DOM element and decodes it internally,
|
|
1875
|
+
* so the fetch/decode cost would otherwise be invisible — and on a cold cache
|
|
1876
|
+
* it dominates everything else.
|
|
1877
|
+
*/
|
|
1878
|
+
export declare interface Speed {
|
|
1879
|
+
/** Fetching and decoding the input into an `RGBImage`. */
|
|
1880
|
+
load: number;
|
|
1881
|
+
/** Letterbox/resize, normalization and tensor packing. */
|
|
1882
|
+
preprocess: number;
|
|
1883
|
+
/** The ONNX Runtime forward pass. */
|
|
1884
|
+
inference: number;
|
|
1885
|
+
/** Decoding raw outputs into results (NMS, mask assembly, top-k). */
|
|
1886
|
+
postprocess: number;
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
/**
|
|
1890
|
+
* Accumulate stage durations while a `predict()` call runs.
|
|
1891
|
+
*
|
|
1892
|
+
* Each `stage()` call closes the previous stage: the elapsed time since the
|
|
1893
|
+
* last boundary is attributed to the name given. This keeps the call sites
|
|
1894
|
+
* free of paired start/stop bookkeeping and guarantees the four stages tile
|
|
1895
|
+
* the whole call without gaps.
|
|
1896
|
+
*/
|
|
1897
|
+
export declare class SpeedTimer {
|
|
1898
|
+
private _last;
|
|
1899
|
+
private readonly _speed;
|
|
1900
|
+
constructor();
|
|
1901
|
+
/**
|
|
1902
|
+
* Attribute the time elapsed since the previous boundary to `stage`.
|
|
1903
|
+
*
|
|
1904
|
+
* @param stage Which stage just finished.
|
|
1905
|
+
*/
|
|
1906
|
+
stage(stage: keyof Speed): void;
|
|
1907
|
+
/**
|
|
1908
|
+
* The accumulated durations.
|
|
1909
|
+
*
|
|
1910
|
+
* @returns The `speed` object to hand to the `Results` envelope.
|
|
1911
|
+
*/
|
|
1912
|
+
speed(): Speed;
|
|
1913
|
+
}
|
|
1914
|
+
|
|
1915
|
+
/**
|
|
1916
|
+
* Transpose interleaved HWC data to planar CHW layout.
|
|
1917
|
+
*
|
|
1918
|
+
* @param hwc Source array of length `width * height * channels`.
|
|
1919
|
+
*/
|
|
1920
|
+
export declare function toCHW(hwc: Float32Array, width: number, height: number, channels?: number): Float32Array;
|
|
1921
|
+
|
|
1922
|
+
/**
|
|
1923
|
+
* Convert the SDK's HWC RGB image to an HWC BGR `Uint8Array` (OpenCV layout).
|
|
1924
|
+
*
|
|
1925
|
+
* Useful for round-tripping data to a Python OpenCV consumer.
|
|
1926
|
+
*/
|
|
1927
|
+
export declare function toCv2(image: RGBImage): Uint8Array;
|
|
1928
|
+
|
|
1929
|
+
/** Convert a uint8 image to a `Float32Array` in `[0, 1]` (HWC layout preserved). */
|
|
1930
|
+
export declare function toFloat32(image: RGBImage, scale?: number): Float32Array;
|
|
1931
|
+
|
|
1932
|
+
/** Wrap a Float32 buffer into an `ort.Tensor`. */
|
|
1933
|
+
export declare function toFloat32Tensor(data: Float32Array, dims: readonly number[]): ort.Tensor;
|
|
1934
|
+
|
|
1935
|
+
/**
|
|
1936
|
+
* Return the top-k entries of a 1-D probability vector, sorted descending.
|
|
1937
|
+
*
|
|
1938
|
+
* @param k Number of entries to return; `null` returns all entries.
|
|
1939
|
+
*/
|
|
1940
|
+
export declare function topK(probabilities: Float32Array, k: number | null): TopKResult;
|
|
1941
|
+
|
|
1942
|
+
export declare interface TopKResult {
|
|
1943
|
+
readonly indices: Int32Array;
|
|
1944
|
+
readonly values: Float32Array;
|
|
1945
|
+
}
|
|
1946
|
+
|
|
1947
|
+
/**
|
|
1948
|
+
* Convert an HWC uint8 image to a CHW `Float32Array` scaled to `[0, 1]`.
|
|
1949
|
+
*
|
|
1950
|
+
* Mirrors `torchvision.transforms.ToTensor()` semantics: HWC → CHW,
|
|
1951
|
+
* `uint8 → float32 / 255`. Useful as input to YOLO-style detectors that
|
|
1952
|
+
* don't require ImageNet normalization.
|
|
1953
|
+
*
|
|
1954
|
+
* @returns CHW `Float32Array` of length `width * height * 3`.
|
|
1955
|
+
*/
|
|
1956
|
+
export declare function toTensor(image: RGBImage): Float32Array;
|
|
1957
|
+
|
|
1958
|
+
/**
|
|
1959
|
+
* Acquire a `MediaStream` via `getUserMedia`, attach it to a `<video>` element,
|
|
1960
|
+
* and expose status/error so the page can render permission and error states.
|
|
1961
|
+
* The stream is automatically released on unmount or retry.
|
|
1962
|
+
*
|
|
1963
|
+
* Defaults to the rear ("environment") camera; desktops fall back to whatever
|
|
1964
|
+
* single camera they expose. Pass `options.constraints` to override, or
|
|
1965
|
+
* `options.enabled: false` to render the surface without prompting for the camera
|
|
1966
|
+
* at all.
|
|
1967
|
+
*
|
|
1968
|
+
* Implementation notes:
|
|
1969
|
+
* - Cleanup detaches the stream from a *snapshotted* video node, so it releases
|
|
1970
|
+
* the same element it attached to even if the page remounts the `<video>`.
|
|
1971
|
+
* - When `getUserMedia` is missing, an insecure context is the usual cause, so
|
|
1972
|
+
* the hook prefers that (actionable) error; otherwise it reports `unsupported`.
|
|
1973
|
+
* - `video.play()` rejections are swallowed: autoplay may be blocked, but the
|
|
1974
|
+
* user gesture that opened the camera usually counts and playback resumes on
|
|
1975
|
+
* the next interaction.
|
|
1976
|
+
*
|
|
1977
|
+
* @param options - optional configuration (see {@link UseCameraStreamOptions}).
|
|
1978
|
+
* @returns The stream status, classified error, a `videoRef` to attach, the live
|
|
1979
|
+
* `stream` for whatever needs the track itself, and a `retry()` to re-attempt
|
|
1980
|
+
* acquisition.
|
|
1981
|
+
*/
|
|
1982
|
+
export declare function useCameraStream(options?: UseCameraStreamOptions): UseCameraStreamApi;
|
|
1983
|
+
|
|
1984
|
+
/** Value returned by {@link useCameraStream}. */
|
|
1985
|
+
export declare interface UseCameraStreamApi {
|
|
1986
|
+
/** Current lifecycle status. */
|
|
1987
|
+
status: CameraStreamStatus;
|
|
1988
|
+
/** The classified error, or `null` while not in the `error` status. */
|
|
1989
|
+
error: CameraStreamError | null;
|
|
1990
|
+
/** Attach to a `<video ref={…} />`. The stream is wired to it once ready. */
|
|
1991
|
+
videoRef: RefObject<HTMLVideoElement | null>;
|
|
1992
|
+
/**
|
|
1993
|
+
* The live stream, or `null`.
|
|
1994
|
+
*
|
|
1995
|
+
* Exposed for the things that need the **track** rather than the picture — the LED
|
|
1996
|
+
* torch (`useTorch`), the real frame size from `getSettings()`, recording it with
|
|
1997
|
+
* `useVideoRecorder`. Do not stop it yourself: the hook owns its lifetime and
|
|
1998
|
+
* releases it on unmount and on `retry()`.
|
|
1999
|
+
*/
|
|
2000
|
+
stream: MediaStream | null;
|
|
2001
|
+
/** Manually re-attempt after an error (e.g. the user changed permissions). */
|
|
2002
|
+
retry: () => void;
|
|
2003
|
+
}
|
|
2004
|
+
|
|
2005
|
+
/** Options for {@link useCameraStream}. */
|
|
2006
|
+
export declare interface UseCameraStreamOptions {
|
|
2007
|
+
/**
|
|
2008
|
+
* Constraints passed to `getUserMedia`. Defaults to the rear
|
|
2009
|
+
* ("environment") camera at Full-HD ideal resolution with audio off.
|
|
2010
|
+
* Read when the stream (re)starts — change it and call `retry()` to apply.
|
|
2011
|
+
*/
|
|
2012
|
+
constraints?: MediaStreamConstraints;
|
|
2013
|
+
/**
|
|
2014
|
+
* Hold off acquiring the camera until this is `true`. Default `true`.
|
|
2015
|
+
*
|
|
2016
|
+
* The point is to be able to *not* prompt. A permission prompt costs the user a
|
|
2017
|
+
* decision and, if they refuse, costs the app the feature permanently — so a
|
|
2018
|
+
* surface that already knows it cannot do its job (a barcode scanner in a browser
|
|
2019
|
+
* with no decoder) must not open the camera to then say so. Flipping this to `false`
|
|
2020
|
+
* also releases a stream that is already live.
|
|
2021
|
+
*/
|
|
2022
|
+
enabled?: boolean;
|
|
2023
|
+
}
|
|
2024
|
+
|
|
2025
|
+
/**
|
|
2026
|
+
* Sample mean luminance from a `<video>` source on a `requestAnimationFrame`
|
|
2027
|
+
* loop and expose the rolling value. Sampling is throttled by `intervalMs` and
|
|
2028
|
+
* paused whenever `enabled` is `false` or the video is not ready yet
|
|
2029
|
+
* (`readyState < 2` or `videoWidth === 0`).
|
|
2030
|
+
*
|
|
2031
|
+
* One offscreen canvas is reused across frames to avoid GC pressure. Designed
|
|
2032
|
+
* to feed a live brightness bar / border color on a camera page.
|
|
2033
|
+
*
|
|
2034
|
+
* @param videoRef - ref to the `<video>` element to sample.
|
|
2035
|
+
* @param options - optional configuration (see {@link UseLiveLuminanceOptions}).
|
|
2036
|
+
* @returns The rolling mean luminance in `0..255` (`0` until the first sample).
|
|
2037
|
+
*/
|
|
2038
|
+
export declare function useLiveLuminance(videoRef: RefObject<HTMLVideoElement | null>, { enabled, intervalMs }?: UseLiveLuminanceOptions): number;
|
|
2039
|
+
|
|
2040
|
+
/** Options for {@link useLiveLuminance}. */
|
|
2041
|
+
export declare interface UseLiveLuminanceOptions {
|
|
2042
|
+
/** When `false` the loop is paused (e.g. while a capture is in flight). Default: `true`. */
|
|
2043
|
+
enabled?: boolean;
|
|
2044
|
+
/** Throttle measurements in milliseconds. Default: `160` (~6 fps), plenty for UX. */
|
|
2045
|
+
intervalMs?: number;
|
|
2046
|
+
}
|
|
2047
|
+
|
|
2048
|
+
export declare const VERSION: string;
|
|
2049
|
+
|
|
2050
|
+
export declare abstract class VisionTask {
|
|
2051
|
+
protected readonly _session: OrtSession;
|
|
2052
|
+
protected constructor(_session: OrtSession);
|
|
2053
|
+
/** The underlying {@link OrtSession} used to run inference. */
|
|
2054
|
+
get session(): OrtSession;
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2057
|
+
/**
|
|
2058
|
+
* Build a zero-filled CHW tensor payload for a warm-up run.
|
|
2059
|
+
*
|
|
2060
|
+
* @param width Model input width in pixels.
|
|
2061
|
+
* @param height Model input height in pixels.
|
|
2062
|
+
*/
|
|
2063
|
+
export declare function zeroTensorData(width: number, height: number): Float32Array;
|
|
2064
|
+
|
|
2065
|
+
export { }
|