tempest-react-sdk 0.69.0 → 0.70.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/vision/core/canvas.cjs +1 -1
- package/dist/vision/core/canvas.cjs.map +1 -1
- package/dist/vision/core/canvas.js +37 -13
- package/dist/vision/core/canvas.js.map +1 -1
- package/dist/vision/core/dtypes.cjs +2 -0
- package/dist/vision/core/dtypes.cjs.map +1 -0
- package/dist/vision/core/dtypes.js +33 -0
- package/dist/vision/core/dtypes.js.map +1 -0
- package/dist/vision/core/graph.cjs +1 -1
- package/dist/vision/core/graph.cjs.map +1 -1
- package/dist/vision/core/graph.js +10 -2
- package/dist/vision/core/graph.js.map +1 -1
- package/dist/vision/core/metadata.cjs +2 -2
- package/dist/vision/core/metadata.cjs.map +1 -1
- package/dist/vision/core/metadata.js +103 -13
- package/dist/vision/core/metadata.js.map +1 -1
- package/dist/vision/core/providers.cjs +1 -1
- package/dist/vision/core/providers.cjs.map +1 -1
- package/dist/vision/core/providers.js +9 -6
- package/dist/vision/core/providers.js.map +1 -1
- package/dist/vision/core/session.cjs +1 -1
- package/dist/vision/core/session.cjs.map +1 -1
- package/dist/vision/core/session.js +115 -27
- package/dist/vision/core/session.js.map +1 -1
- package/dist/vision/fusion.cjs +1 -1
- package/dist/vision/fusion.cjs.map +1 -1
- package/dist/vision/fusion.js +28 -23
- package/dist/vision/fusion.js.map +1 -1
- package/dist/vision/index.cjs +1 -1
- package/dist/vision/index.cjs.map +1 -1
- package/dist/vision/index.js +20 -19
- package/dist/vision/index.js.map +1 -1
- package/dist/vision/io/image.cjs +1 -1
- package/dist/vision/io/image.cjs.map +1 -1
- package/dist/vision/io/image.js +55 -20
- package/dist/vision/io/image.js.map +1 -1
- package/dist/vision/postprocess/detection.cjs +1 -1
- package/dist/vision/postprocess/detection.cjs.map +1 -1
- package/dist/vision/postprocess/detection.js +42 -51
- package/dist/vision/postprocess/detection.js.map +1 -1
- package/dist/vision/postprocess/segmentation.cjs +1 -1
- package/dist/vision/postprocess/segmentation.cjs.map +1 -1
- package/dist/vision/postprocess/segmentation.js +45 -31
- package/dist/vision/postprocess/segmentation.js.map +1 -1
- package/dist/vision/preprocess/pipeline.cjs +1 -1
- package/dist/vision/preprocess/pipeline.cjs.map +1 -1
- package/dist/vision/preprocess/pipeline.js +24 -25
- package/dist/vision/preprocess/pipeline.js.map +1 -1
- package/dist/vision/tasks/base.cjs +1 -1
- package/dist/vision/tasks/base.cjs.map +1 -1
- package/dist/vision/tasks/base.js +22 -7
- package/dist/vision/tasks/base.js.map +1 -1
- package/dist/vision/tasks/classifier.cjs +1 -1
- package/dist/vision/tasks/classifier.cjs.map +1 -1
- package/dist/vision/tasks/classifier.js +64 -55
- package/dist/vision/tasks/classifier.js.map +1 -1
- package/dist/vision/tasks/detectClassify.cjs +1 -1
- package/dist/vision/tasks/detectClassify.cjs.map +1 -1
- package/dist/vision/tasks/detectClassify.js +80 -69
- package/dist/vision/tasks/detectClassify.js.map +1 -1
- package/dist/vision/tasks/detector.cjs +1 -1
- package/dist/vision/tasks/detector.cjs.map +1 -1
- package/dist/vision/tasks/detector.js +74 -68
- package/dist/vision/tasks/detector.js.map +1 -1
- package/dist/vision/tasks/segmenter.cjs +1 -1
- package/dist/vision/tasks/segmenter.cjs.map +1 -1
- package/dist/vision/tasks/segmenter.js +85 -62
- package/dist/vision/tasks/segmenter.js.map +1 -1
- package/dist/vision/types.cjs +1 -1
- package/dist/vision/types.cjs.map +1 -1
- package/dist/vision/types.js +24 -4
- package/dist/vision/types.js.map +1 -1
- package/dist/vision.cjs +1 -1
- package/dist/vision.d.ts +1153 -777
- package/dist/vision.js +22 -21
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { BoundingBox as e, Mask as t } from "../types.js";
|
|
2
2
|
import { decodeYoloAnchors as n } from "./detection.js";
|
|
3
3
|
//#region src/vision/postprocess/segmentation.ts
|
|
4
|
-
function r(r,
|
|
4
|
+
function r(r, i, s, c, l) {
|
|
5
5
|
let u = c;
|
|
6
6
|
if (u.length === 4) {
|
|
7
7
|
if (u[0] !== 1) throw Error(`decodeYoloSeg: expected batch size 1 in prototypes, got ${u[0]}.`);
|
|
@@ -12,12 +12,12 @@ function r(r, o, s, c, l) {
|
|
|
12
12
|
];
|
|
13
13
|
}
|
|
14
14
|
if (u.length !== 3) throw Error(`decodeYoloSeg: expected 3-D prototypes after batch removal, got dims=${JSON.stringify(c)}.`);
|
|
15
|
-
let d = u[0], f = u[1], p = u[2], m =
|
|
16
|
-
if (m === void 0 || h === void 0) throw Error(`decodeYoloSeg: cannot read channels/numAnchors from dims=${JSON.stringify(
|
|
15
|
+
let d = u[0], f = u[1], p = u[2], m = i.length === 3 ? i[1] : i[0], h = i.length === 3 ? i[2] : i[1];
|
|
16
|
+
if (m === void 0 || h === void 0) throw Error(`decodeYoloSeg: cannot read channels/numAnchors from dims=${JSON.stringify(i)}.`);
|
|
17
17
|
let g = 4 + l.numClasses + d;
|
|
18
18
|
if (m !== g) throw Error(`decodeYoloSeg: channels=${m} does not match 4 + numClasses(${l.numClasses}) + numMaskCoefs(${d}) = ${g}.`);
|
|
19
19
|
if (s.length !== d * f * p) throw Error(`decodeYoloSeg: prototype length ${s.length} does not match dims=${JSON.stringify(c)}.`);
|
|
20
|
-
let _ = n(r,
|
|
20
|
+
let _ = n(r, i, {
|
|
21
21
|
numClasses: l.numClasses,
|
|
22
22
|
originalWidth: l.originalWidth,
|
|
23
23
|
originalHeight: l.originalHeight,
|
|
@@ -31,7 +31,7 @@ function r(r, o, s, c, l) {
|
|
|
31
31
|
if (_.anchorIndices.length === 0) return [];
|
|
32
32
|
let v = l.maskThreshold ?? .5, y = p / l.inputWidth, b = f / l.inputHeight, x = f * p, S = (4 + l.numClasses) * h, C = [];
|
|
33
33
|
for (let n = 0; n < _.anchorIndices.length; n++) {
|
|
34
|
-
let
|
|
34
|
+
let i = _.anchorIndices[n], c = _.boxesXyxy[n * 4], u = _.boxesXyxy[n * 4 + 1], m = _.boxesXyxy[n * 4 + 2], g = _.boxesXyxy[n * 4 + 3], w = new e(c, u, m, g), T = _.classIds[n], E = _.confidences[n], D = Math.max(0, Math.trunc(m) - Math.trunc(c)), O = Math.max(0, Math.trunc(g) - Math.trunc(u));
|
|
35
35
|
if (D === 0 || O === 0) {
|
|
36
36
|
C.push({
|
|
37
37
|
bbox: w,
|
|
@@ -51,25 +51,12 @@ function r(r, o, s, c, l) {
|
|
|
51
51
|
});
|
|
52
52
|
continue;
|
|
53
53
|
}
|
|
54
|
-
let L = F - N, R = I - P, z =
|
|
55
|
-
for (let e = 0; e < R; e++) {
|
|
56
|
-
let t = P + e;
|
|
57
|
-
for (let n = 0; n < L; n++) {
|
|
58
|
-
let a = N + n, c = 0;
|
|
59
|
-
for (let e = 0; e < d; e++) {
|
|
60
|
-
let n = r[S + e * h + o], i = s[e * x + t * p + a];
|
|
61
|
-
c += n * i;
|
|
62
|
-
}
|
|
63
|
-
z[e * L + n] = i(c);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
let B = a(z, L, R, D, O), V = new Uint8Array(D * O);
|
|
67
|
-
for (let e = 0; e < V.length; e++) V[e] = B[e] >= v ? 255 : 0;
|
|
54
|
+
let L = F - N, R = I - P, z = o(a(r, s, S + i, h, d, p, x, N, P, L, R), L, R, D, O, v);
|
|
68
55
|
C.push({
|
|
69
56
|
bbox: w,
|
|
70
57
|
classId: T,
|
|
71
58
|
confidence: E,
|
|
72
|
-
mask: new t(
|
|
59
|
+
mask: new t(z, D, O)
|
|
73
60
|
});
|
|
74
61
|
}
|
|
75
62
|
return C;
|
|
@@ -79,19 +66,46 @@ function i(e) {
|
|
|
79
66
|
let t = Math.exp(e);
|
|
80
67
|
return t / (1 + t);
|
|
81
68
|
}
|
|
82
|
-
function a(e, t, n, r,
|
|
83
|
-
let
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
69
|
+
function a(e, t, n, r, a, o, s, c, l, u, d) {
|
|
70
|
+
let f = new Float64Array(u * d);
|
|
71
|
+
for (let i = 0; i < a; i++) {
|
|
72
|
+
let a = e[n + i * r], p = i * s + c;
|
|
73
|
+
for (let e = 0; e < d; e++) {
|
|
74
|
+
let n = p + (l + e) * o, r = e * u;
|
|
75
|
+
for (let e = 0; e < u; e++) f[r + e] = f[r + e] + a * t[n + e];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
let p = new Float32Array(f.length);
|
|
79
|
+
for (let e = 0; e < f.length; e++) p[e] = i(f[e]);
|
|
80
|
+
return p;
|
|
81
|
+
}
|
|
82
|
+
function o(e, t, n, r, i, a) {
|
|
83
|
+
let o = new Uint8Array(r * i);
|
|
84
|
+
if (r === t && i === n) {
|
|
85
|
+
for (let t = 0; t < o.length; t++) o[t] = e[t] >= a ? 255 : 0;
|
|
86
|
+
return o;
|
|
87
|
+
}
|
|
88
|
+
let s = t / r, c = n / i, l = new Int32Array(r), u = new Int32Array(r), d = new Float64Array(r);
|
|
89
|
+
for (let e = 0; e < r; e++) {
|
|
90
|
+
let n = (e + .5) * s - .5, r = Math.max(0, Math.floor(n));
|
|
91
|
+
l[e] = r, u[e] = Math.min(t - 1, r + 1), d[e] = Math.max(0, Math.min(1, n - r));
|
|
92
|
+
}
|
|
93
|
+
let f = new Float64Array(n * r);
|
|
94
|
+
for (let i = 0; i < n; i++) {
|
|
95
|
+
let n = i * t, a = i * r;
|
|
96
|
+
for (let t = 0; t < r; t++) {
|
|
97
|
+
let r = d[t];
|
|
98
|
+
f[a + t] = e[n + l[t]] * (1 - r) + e[n + u[t]] * r;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
for (let e = 0; e < i; e++) {
|
|
102
|
+
let t = (e + .5) * c - .5, i = Math.max(0, Math.floor(t)), s = Math.min(n - 1, i + 1), l = Math.max(0, Math.min(1, t - i)), u = i * r, d = s * r, p = e * r;
|
|
103
|
+
for (let e = 0; e < r; e++) {
|
|
104
|
+
let t = f[u + e] * (1 - l) + f[d + e] * l;
|
|
105
|
+
o[p + e] = Math.fround(t) >= a ? 255 : 0;
|
|
92
106
|
}
|
|
93
107
|
}
|
|
94
|
-
return
|
|
108
|
+
return o;
|
|
95
109
|
}
|
|
96
110
|
//#endregion
|
|
97
111
|
export { r as decodeYoloSeg };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"segmentation.js","names":[],"sources":["../../../src/vision/postprocess/segmentation.ts"],"sourcesContent":["/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */\n/**\n * Segmentation head postprocessing: YOLO instance-segmentation decoding.\n *\n * Compatible with YOLOv8-seg / YOLOv11-seg exports (and any later seg head\n * sharing the layout). The model produces two output tensors:\n *\n * - `output0` of shape `(1, 4 + numClasses + numMaskCoefs, numAnchors)` — the\n * same per-anchor predictions as plain YOLO detection plus an extra\n * `numMaskCoefs` (typically 32) channels of mask coefficients.\n * - `output1` of shape `(1, numMaskCoefs, maskH, maskW)` — a set of \"prototype\"\n * masks shared across all anchors.\n *\n * The per-anchor decode (xywh→xyxy, undo letterbox, per-class NMS, sort & cap)\n * is delegated to {@link decodeYoloAnchors}; this module only handles the\n * mask-specific work: matmul against prototypes, sigmoid, bilinear resize and\n * thresholding.\n */\n\nimport { decodeYoloAnchors } from \"./detection\";\nimport { BoundingBox, Mask } from \"../types\";\n\nexport interface DecodeYoloSegOptions {\n readonly numClasses: number;\n /** Model input `[width, height]` (post-letterbox). */\n readonly inputWidth: number;\n readonly inputHeight: number;\n /** Original image `[width, height]`. */\n readonly originalWidth: number;\n readonly originalHeight: number;\n /** Letterbox horizontal padding in input-tensor pixels. */\n readonly padLeft: number;\n /** Letterbox vertical padding in input-tensor pixels. */\n readonly padTop: number;\n /** Letterbox scale factor. */\n readonly scale: number;\n readonly confThreshold: number;\n readonly iouThreshold: number;\n readonly maxDetections: number;\n /** Probability cutoff applied to the soft mask. Defaults to `0.5`. */\n readonly maskThreshold?: number;\n}\n\nexport interface DecodedSegmentation {\n readonly bbox: BoundingBox;\n readonly classId: number;\n readonly confidence: number;\n /** Binary mask cropped to `bbox`. Width/height match `bbox.asIntXyxy()` extents. */\n readonly mask: Mask;\n}\n\n/**\n * Decode YOLO segmentation raw outputs into a list of segmented instances.\n *\n * Compatible with YOLOv8-seg / YOLOv11-seg.\n *\n * @param perAnchorData Flat `output0`, length `(4 + numClasses + numMaskCoefs) * numAnchors`.\n * @param perAnchorDims Dims as reported by ORT, e.g. `[1, 116, 8400]`.\n * @param prototypeData Flat `output1`, length `numMaskCoefs * maskH * maskW`.\n * @param prototypeDims Dims as reported by ORT, e.g. `[1, 32, 160, 160]`.\n */\nexport function decodeYoloSeg(\n perAnchorData: Float32Array,\n perAnchorDims: readonly number[],\n prototypeData: Float32Array,\n prototypeDims: readonly number[],\n options: DecodeYoloSegOptions,\n): DecodedSegmentation[] {\n // Strip batch dim from prototypes; perAnchor batch is validated by the helper.\n let pDims = prototypeDims;\n if (pDims.length === 4) {\n if (pDims[0] !== 1) {\n throw new Error(`decodeYoloSeg: expected batch size 1 in prototypes, got ${pDims[0]}.`);\n }\n pDims = [pDims[1] as number, pDims[2] as number, pDims[3] as number];\n }\n if (pDims.length !== 3) {\n throw new Error(\n `decodeYoloSeg: expected 3-D prototypes after batch removal, got dims=${JSON.stringify(prototypeDims)}.`,\n );\n }\n const numMaskCoefs = pDims[0] as number;\n const maskH = pDims[1] as number;\n const maskW = pDims[2] as number;\n\n const channels = perAnchorDims.length === 3 ? perAnchorDims[1] : perAnchorDims[0];\n const numAnchors = perAnchorDims.length === 3 ? perAnchorDims[2] : perAnchorDims[1];\n if (channels === undefined || numAnchors === undefined) {\n throw new Error(\n `decodeYoloSeg: cannot read channels/numAnchors from dims=${JSON.stringify(perAnchorDims)}.`,\n );\n }\n\n const expectedChannels = 4 + options.numClasses + numMaskCoefs;\n if (channels !== expectedChannels) {\n throw new Error(\n `decodeYoloSeg: channels=${channels} does not match 4 + numClasses(${options.numClasses}) + numMaskCoefs(${numMaskCoefs}) = ${expectedChannels}.`,\n );\n }\n if (prototypeData.length !== numMaskCoefs * maskH * maskW) {\n throw new Error(\n `decodeYoloSeg: prototype length ${prototypeData.length} does not match dims=${JSON.stringify(prototypeDims)}.`,\n );\n }\n\n const decoded = decodeYoloAnchors(perAnchorData, perAnchorDims, {\n numClasses: options.numClasses,\n originalWidth: options.originalWidth,\n originalHeight: options.originalHeight,\n padLeft: options.padLeft,\n padTop: options.padTop,\n scale: options.scale,\n confThreshold: options.confThreshold,\n iouThreshold: options.iouThreshold,\n maxDetections: options.maxDetections,\n });\n\n if (decoded.anchorIndices.length === 0) return [];\n\n const maskThreshold = options.maskThreshold ?? 0.5;\n const scaleX = maskW / options.inputWidth;\n const scaleY = maskH / options.inputHeight;\n const protoPlane = maskH * maskW;\n const coefBase = (4 + options.numClasses) * numAnchors;\n\n const results: DecodedSegmentation[] = [];\n\n for (let i = 0; i < decoded.anchorIndices.length; i++) {\n const a = decoded.anchorIndices[i] as number;\n const x1 = decoded.boxesXyxy[i * 4] as number;\n const y1 = decoded.boxesXyxy[i * 4 + 1] as number;\n const x2 = decoded.boxesXyxy[i * 4 + 2] as number;\n const y2 = decoded.boxesXyxy[i * 4 + 3] as number;\n const bbox = new BoundingBox(x1, y1, x2, y2);\n const classId = decoded.classIds[i] as number;\n const confidence = decoded.confidences[i] as number;\n\n const bboxW = Math.max(0, Math.trunc(x2) - Math.trunc(x1));\n const bboxH = Math.max(0, Math.trunc(y2) - Math.trunc(y1));\n\n if (bboxW === 0 || bboxH === 0) {\n results.push({ bbox, classId, confidence, mask: new Mask(new Uint8Array(0), 0, 0) });\n continue;\n }\n\n // Bbox in input-tensor coords, then in low-res mask coords.\n const ibx1 = x1 * options.scale + options.padLeft;\n const iby1 = y1 * options.scale + options.padTop;\n const ibx2 = x2 * options.scale + options.padLeft;\n const iby2 = y2 * options.scale + options.padTop;\n\n const mbx1 = Math.max(0, Math.floor(ibx1 * scaleX));\n const mby1 = Math.max(0, Math.floor(iby1 * scaleY));\n const mbx2 = Math.min(maskW, Math.ceil(ibx2 * scaleX));\n const mby2 = Math.min(maskH, Math.ceil(iby2 * scaleY));\n\n if (mbx2 <= mbx1 || mby2 <= mby1) {\n results.push({\n bbox,\n classId,\n confidence,\n mask: new Mask(new Uint8Array(bboxW * bboxH), bboxW, bboxH),\n });\n continue;\n }\n\n // Compute soft mask only within the prototype region under this bbox.\n const cropW = mbx2 - mbx1;\n const cropH = mby2 - mby1;\n const softCrop = new Float32Array(cropW * cropH);\n for (let y = 0; y < cropH; y++) {\n const py = mby1 + y;\n for (let x = 0; x < cropW; x++) {\n const px = mbx1 + x;\n let sum = 0;\n for (let kk = 0; kk < numMaskCoefs; kk++) {\n const coef = perAnchorData[coefBase + kk * numAnchors + a];\n const proto = prototypeData[kk * protoPlane + py * maskW + px];\n sum += coef * proto;\n }\n softCrop[y * cropW + x] = sigmoid(sum);\n }\n }\n\n const resized = resizeBilinear(softCrop, cropW, cropH, bboxW, bboxH);\n const binary = new Uint8Array(bboxW * bboxH);\n for (let j = 0; j < binary.length; j++) {\n binary[j] = resized[j] >= maskThreshold ? 255 : 0;\n }\n\n results.push({ bbox, classId, confidence, mask: new Mask(binary, bboxW, bboxH) });\n }\n\n return results;\n}\n\nfunction sigmoid(x: number): number {\n if (x >= 0) {\n return 1 / (1 + Math.exp(-x));\n }\n const e = Math.exp(x);\n return e / (1 + e);\n}\n\nfunction resizeBilinear(\n src: Float32Array,\n srcWidth: number,\n srcHeight: number,\n targetWidth: number,\n targetHeight: number,\n): Float32Array {\n const out = new Float32Array(targetWidth * targetHeight);\n if (targetWidth === 0 || targetHeight === 0 || srcWidth === 0 || srcHeight === 0) {\n return out;\n }\n if (targetWidth === srcWidth && targetHeight === srcHeight) {\n out.set(src);\n return out;\n }\n const sx = srcWidth / targetWidth;\n const sy = srcHeight / targetHeight;\n for (let y = 0; y < targetHeight; y++) {\n const yy = (y + 0.5) * sy - 0.5;\n const y0 = Math.max(0, Math.floor(yy));\n const y1 = Math.min(srcHeight - 1, y0 + 1);\n const wy = Math.max(0, Math.min(1, yy - y0));\n for (let x = 0; x < targetWidth; x++) {\n const xx = (x + 0.5) * sx - 0.5;\n const x0 = Math.max(0, Math.floor(xx));\n const x1 = Math.min(srcWidth - 1, x0 + 1);\n const wx = Math.max(0, Math.min(1, xx - x0));\n const v00 = src[y0 * srcWidth + x0];\n const v01 = src[y0 * srcWidth + x1];\n const v10 = src[y1 * srcWidth + x0];\n const v11 = src[y1 * srcWidth + x1];\n const top = v00 * (1 - wx) + v01 * wx;\n const bot = v10 * (1 - wx) + v11 * wx;\n out[y * targetWidth + x] = top * (1 - wy) + bot * wy;\n }\n }\n return out;\n}\n"],"mappings":";;;AA6DA,SAAgB,EACZ,GACA,GACA,GACA,GACA,GACqB;CAErB,IAAI,IAAQ;CACZ,IAAI,EAAM,WAAW,GAAG;EACpB,IAAI,EAAM,OAAO,GACb,MAAU,MAAM,2DAA2D,EAAM,GAAG,EAAE;EAE1F,IAAQ;GAAC,EAAM;GAAc,EAAM;GAAc,EAAM;EAAY;CACvE;CACA,IAAI,EAAM,WAAW,GACjB,MAAU,MACN,wEAAwE,KAAK,UAAU,CAAa,EAAE,EAC1G;CAEJ,IAAM,IAAe,EAAM,IACrB,IAAQ,EAAM,IACd,IAAQ,EAAM,IAEd,IAAW,EAAc,WAAW,IAAI,EAAc,KAAK,EAAc,IACzE,IAAa,EAAc,WAAW,IAAI,EAAc,KAAK,EAAc;CACjF,IAAI,MAAa,KAAA,KAAa,MAAe,KAAA,GACzC,MAAU,MACN,4DAA4D,KAAK,UAAU,CAAa,EAAE,EAC9F;CAGJ,IAAM,IAAmB,IAAI,EAAQ,aAAa;CAClD,IAAI,MAAa,GACb,MAAU,MACN,2BAA2B,EAAS,iCAAiC,EAAQ,WAAW,mBAAmB,EAAa,MAAM,EAAiB,EACnJ;CAEJ,IAAI,EAAc,WAAW,IAAe,IAAQ,GAChD,MAAU,MACN,mCAAmC,EAAc,OAAO,uBAAuB,KAAK,UAAU,CAAa,EAAE,EACjH;CAGJ,IAAM,IAAU,EAAkB,GAAe,GAAe;EAC5D,YAAY,EAAQ;EACpB,eAAe,EAAQ;EACvB,gBAAgB,EAAQ;EACxB,SAAS,EAAQ;EACjB,QAAQ,EAAQ;EAChB,OAAO,EAAQ;EACf,eAAe,EAAQ;EACvB,cAAc,EAAQ;EACtB,eAAe,EAAQ;CAC3B,CAAC;CAED,IAAI,EAAQ,cAAc,WAAW,GAAG,OAAO,CAAC;CAEhD,IAAM,IAAgB,EAAQ,iBAAiB,IACzC,IAAS,IAAQ,EAAQ,YACzB,IAAS,IAAQ,EAAQ,aACzB,IAAa,IAAQ,GACrB,KAAY,IAAI,EAAQ,cAAc,GAEtC,IAAiC,CAAC;CAExC,KAAK,IAAI,IAAI,GAAG,IAAI,EAAQ,cAAc,QAAQ,KAAK;EACnD,IAAM,IAAI,EAAQ,cAAc,IAC1B,IAAK,EAAQ,UAAU,IAAI,IAC3B,IAAK,EAAQ,UAAU,IAAI,IAAI,IAC/B,IAAK,EAAQ,UAAU,IAAI,IAAI,IAC/B,IAAK,EAAQ,UAAU,IAAI,IAAI,IAC/B,IAAO,IAAI,EAAY,GAAI,GAAI,GAAI,CAAE,GACrC,IAAU,EAAQ,SAAS,IAC3B,IAAa,EAAQ,YAAY,IAEjC,IAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,CAAE,IAAI,KAAK,MAAM,CAAE,CAAC,GACnD,IAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,CAAE,IAAI,KAAK,MAAM,CAAE,CAAC;EAEzD,IAAI,MAAU,KAAK,MAAU,GAAG;GAC5B,EAAQ,KAAK;IAAE;IAAM;IAAS;IAAY,MAAM,IAAI,kBAAK,IAAI,WAAY,GAAG,GAAG,CAAC;GAAE,CAAC;GACnF;EACJ;EAGA,IAAM,IAAO,IAAK,EAAQ,QAAQ,EAAQ,SACpC,IAAO,IAAK,EAAQ,QAAQ,EAAQ,QACpC,IAAO,IAAK,EAAQ,QAAQ,EAAQ,SACpC,IAAO,IAAK,EAAQ,QAAQ,EAAQ,QAEpC,IAAO,KAAK,IAAI,GAAG,KAAK,MAAM,IAAO,CAAM,CAAC,GAC5C,IAAO,KAAK,IAAI,GAAG,KAAK,MAAM,IAAO,CAAM,CAAC,GAC5C,IAAO,KAAK,IAAI,GAAO,KAAK,KAAK,IAAO,CAAM,CAAC,GAC/C,IAAO,KAAK,IAAI,GAAO,KAAK,KAAK,IAAO,CAAM,CAAC;EAErD,IAAI,KAAQ,KAAQ,KAAQ,GAAM;GAC9B,EAAQ,KAAK;IACT;IACA;IACA;IACA,MAAM,IAAI,EAAK,IAAI,WAAW,IAAQ,CAAK,GAAG,GAAO,CAAK;GAC9D,CAAC;GACD;EACJ;EAGA,IAAM,IAAQ,IAAO,GACf,IAAQ,IAAO,GACf,IAAW,IAAI,aAAa,IAAQ,CAAK;EAC/C,KAAK,IAAI,IAAI,GAAG,IAAI,GAAO,KAAK;GAC5B,IAAM,IAAK,IAAO;GAClB,KAAK,IAAI,IAAI,GAAG,IAAI,GAAO,KAAK;IAC5B,IAAM,IAAK,IAAO,GACd,IAAM;IACV,KAAK,IAAI,IAAK,GAAG,IAAK,GAAc,KAAM;KACtC,IAAM,IAAO,EAAc,IAAW,IAAK,IAAa,IAClD,IAAQ,EAAc,IAAK,IAAa,IAAK,IAAQ;KAC3D,KAAO,IAAO;IAClB;IACA,EAAS,IAAI,IAAQ,KAAK,EAAQ,CAAG;GACzC;EACJ;EAEA,IAAM,IAAU,EAAe,GAAU,GAAO,GAAO,GAAO,CAAK,GAC7D,IAAS,IAAI,WAAW,IAAQ,CAAK;EAC3C,KAAK,IAAI,IAAI,GAAG,IAAI,EAAO,QAAQ,KAC/B,EAAO,KAAK,EAAQ,MAAM,IAAgB,MAAM;EAGpD,EAAQ,KAAK;GAAE;GAAM;GAAS;GAAY,MAAM,IAAI,EAAK,GAAQ,GAAO,CAAK;EAAE,CAAC;CACpF;CAEA,OAAO;AACX;AAEA,SAAS,EAAQ,GAAmB;CAChC,IAAI,KAAK,GACL,OAAO,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC;CAE/B,IAAM,IAAI,KAAK,IAAI,CAAC;CACpB,OAAO,KAAK,IAAI;AACpB;AAEA,SAAS,EACL,GACA,GACA,GACA,GACA,GACY;CACZ,IAAM,IAAM,IAAI,aAAa,IAAc,CAAY;CACvD,IAAI,MAAgB,KAAK,MAAiB,KAAK,MAAa,KAAK,MAAc,GAC3E,OAAO;CAEX,IAAI,MAAgB,KAAY,MAAiB,GAE7C,OADA,EAAI,IAAI,CAAG,GACJ;CAEX,IAAM,IAAK,IAAW,GAChB,IAAK,IAAY;CACvB,KAAK,IAAI,IAAI,GAAG,IAAI,GAAc,KAAK;EACnC,IAAM,KAAM,IAAI,MAAO,IAAK,IACtB,IAAK,KAAK,IAAI,GAAG,KAAK,MAAM,CAAE,CAAC,GAC/B,IAAK,KAAK,IAAI,IAAY,GAAG,IAAK,CAAC,GACnC,IAAK,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,IAAK,CAAE,CAAC;EAC3C,KAAK,IAAI,IAAI,GAAG,IAAI,GAAa,KAAK;GAClC,IAAM,KAAM,IAAI,MAAO,IAAK,IACtB,IAAK,KAAK,IAAI,GAAG,KAAK,MAAM,CAAE,CAAC,GAC/B,IAAK,KAAK,IAAI,IAAW,GAAG,IAAK,CAAC,GAClC,IAAK,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,IAAK,CAAE,CAAC,GACrC,IAAM,EAAI,IAAK,IAAW,IAC1B,IAAM,EAAI,IAAK,IAAW,IAC1B,IAAM,EAAI,IAAK,IAAW,IAC1B,IAAM,EAAI,IAAK,IAAW,IAC1B,IAAM,KAAO,IAAI,KAAM,IAAM,GAC7B,IAAM,KAAO,IAAI,KAAM,IAAM;GACnC,EAAI,IAAI,IAAc,KAAK,KAAO,IAAI,KAAM,IAAM;EACtD;CACJ;CACA,OAAO;AACX"}
|
|
1
|
+
{"version":3,"file":"segmentation.js","names":[],"sources":["../../../src/vision/postprocess/segmentation.ts"],"sourcesContent":["/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */\n/**\n * Segmentation head postprocessing: YOLO instance-segmentation decoding.\n *\n * Compatible with YOLOv8-seg / YOLOv11-seg exports (and any later seg head\n * sharing the layout). The model produces two output tensors:\n *\n * - `output0` of shape `(1, 4 + numClasses + numMaskCoefs, numAnchors)` — the\n * same per-anchor predictions as plain YOLO detection plus an extra\n * `numMaskCoefs` (typically 32) channels of mask coefficients.\n * - `output1` of shape `(1, numMaskCoefs, maskH, maskW)` — a set of \"prototype\"\n * masks shared across all anchors.\n *\n * The per-anchor decode (xywh→xyxy, undo letterbox, per-class NMS, sort & cap)\n * is delegated to {@link decodeYoloAnchors}; this module only handles the\n * mask-specific work: matmul against prototypes, sigmoid, bilinear resize and\n * thresholding.\n */\n\nimport { decodeYoloAnchors } from \"./detection\";\nimport { BoundingBox, Mask } from \"../types\";\n\nexport interface DecodeYoloSegOptions {\n readonly numClasses: number;\n /** Model input `[width, height]` (post-letterbox). */\n readonly inputWidth: number;\n readonly inputHeight: number;\n /** Original image `[width, height]`. */\n readonly originalWidth: number;\n readonly originalHeight: number;\n /** Letterbox horizontal padding in input-tensor pixels. */\n readonly padLeft: number;\n /** Letterbox vertical padding in input-tensor pixels. */\n readonly padTop: number;\n /** Letterbox scale factor. */\n readonly scale: number;\n readonly confThreshold: number;\n readonly iouThreshold: number;\n readonly maxDetections: number;\n /** Probability cutoff applied to the soft mask. Defaults to `0.5`. */\n readonly maskThreshold?: number;\n}\n\nexport interface DecodedSegmentation {\n readonly bbox: BoundingBox;\n readonly classId: number;\n readonly confidence: number;\n /** Binary mask cropped to `bbox`. Width/height match `bbox.asIntXyxy()` extents. */\n readonly mask: Mask;\n}\n\n/**\n * Decode YOLO segmentation raw outputs into a list of segmented instances.\n *\n * Compatible with YOLOv8-seg / YOLOv11-seg.\n *\n * The prototypes' batch axis is stripped here; the per-anchor batch axis is\n * validated by {@link decodeYoloAnchors}. Each instance's box is mapped back\n * into input-tensor coordinates and from there into the low-resolution\n * prototype grid, and only the prototype pixels under it are assembled.\n *\n * @param perAnchorData Flat `output0`, length `(4 + numClasses + numMaskCoefs) * numAnchors`.\n * @param perAnchorDims Dims as reported by ORT, e.g. `[1, 116, 8400]`.\n * @param prototypeData Flat `output1`, length `numMaskCoefs * maskH * maskW`.\n * @param prototypeDims Dims as reported by ORT, e.g. `[1, 32, 160, 160]`.\n */\nexport function decodeYoloSeg(\n perAnchorData: Float32Array,\n perAnchorDims: readonly number[],\n prototypeData: Float32Array,\n prototypeDims: readonly number[],\n options: DecodeYoloSegOptions,\n): DecodedSegmentation[] {\n let pDims = prototypeDims;\n if (pDims.length === 4) {\n if (pDims[0] !== 1) {\n throw new Error(`decodeYoloSeg: expected batch size 1 in prototypes, got ${pDims[0]}.`);\n }\n pDims = [pDims[1] as number, pDims[2] as number, pDims[3] as number];\n }\n if (pDims.length !== 3) {\n throw new Error(\n `decodeYoloSeg: expected 3-D prototypes after batch removal, got dims=${JSON.stringify(prototypeDims)}.`,\n );\n }\n const numMaskCoefs = pDims[0] as number;\n const maskH = pDims[1] as number;\n const maskW = pDims[2] as number;\n\n const channels = perAnchorDims.length === 3 ? perAnchorDims[1] : perAnchorDims[0];\n const numAnchors = perAnchorDims.length === 3 ? perAnchorDims[2] : perAnchorDims[1];\n if (channels === undefined || numAnchors === undefined) {\n throw new Error(\n `decodeYoloSeg: cannot read channels/numAnchors from dims=${JSON.stringify(perAnchorDims)}.`,\n );\n }\n\n const expectedChannels = 4 + options.numClasses + numMaskCoefs;\n if (channels !== expectedChannels) {\n throw new Error(\n `decodeYoloSeg: channels=${channels} does not match 4 + numClasses(${options.numClasses}) + numMaskCoefs(${numMaskCoefs}) = ${expectedChannels}.`,\n );\n }\n if (prototypeData.length !== numMaskCoefs * maskH * maskW) {\n throw new Error(\n `decodeYoloSeg: prototype length ${prototypeData.length} does not match dims=${JSON.stringify(prototypeDims)}.`,\n );\n }\n\n const decoded = decodeYoloAnchors(perAnchorData, perAnchorDims, {\n numClasses: options.numClasses,\n originalWidth: options.originalWidth,\n originalHeight: options.originalHeight,\n padLeft: options.padLeft,\n padTop: options.padTop,\n scale: options.scale,\n confThreshold: options.confThreshold,\n iouThreshold: options.iouThreshold,\n maxDetections: options.maxDetections,\n });\n\n if (decoded.anchorIndices.length === 0) return [];\n\n const maskThreshold = options.maskThreshold ?? 0.5;\n const scaleX = maskW / options.inputWidth;\n const scaleY = maskH / options.inputHeight;\n const protoPlane = maskH * maskW;\n const coefBase = (4 + options.numClasses) * numAnchors;\n\n const results: DecodedSegmentation[] = [];\n\n for (let i = 0; i < decoded.anchorIndices.length; i++) {\n const a = decoded.anchorIndices[i] as number;\n const x1 = decoded.boxesXyxy[i * 4] as number;\n const y1 = decoded.boxesXyxy[i * 4 + 1] as number;\n const x2 = decoded.boxesXyxy[i * 4 + 2] as number;\n const y2 = decoded.boxesXyxy[i * 4 + 3] as number;\n const bbox = new BoundingBox(x1, y1, x2, y2);\n const classId = decoded.classIds[i] as number;\n const confidence = decoded.confidences[i] as number;\n\n const bboxW = Math.max(0, Math.trunc(x2) - Math.trunc(x1));\n const bboxH = Math.max(0, Math.trunc(y2) - Math.trunc(y1));\n\n if (bboxW === 0 || bboxH === 0) {\n results.push({ bbox, classId, confidence, mask: new Mask(new Uint8Array(0), 0, 0) });\n continue;\n }\n\n const ibx1 = x1 * options.scale + options.padLeft;\n const iby1 = y1 * options.scale + options.padTop;\n const ibx2 = x2 * options.scale + options.padLeft;\n const iby2 = y2 * options.scale + options.padTop;\n\n const mbx1 = Math.max(0, Math.floor(ibx1 * scaleX));\n const mby1 = Math.max(0, Math.floor(iby1 * scaleY));\n const mbx2 = Math.min(maskW, Math.ceil(ibx2 * scaleX));\n const mby2 = Math.min(maskH, Math.ceil(iby2 * scaleY));\n\n if (mbx2 <= mbx1 || mby2 <= mby1) {\n results.push({\n bbox,\n classId,\n confidence,\n mask: new Mask(new Uint8Array(bboxW * bboxH), bboxW, bboxH),\n });\n continue;\n }\n\n const cropW = mbx2 - mbx1;\n const cropH = mby2 - mby1;\n const softCrop = softMaskCrop(\n perAnchorData,\n prototypeData,\n coefBase + a,\n numAnchors,\n numMaskCoefs,\n maskW,\n protoPlane,\n mbx1,\n mby1,\n cropW,\n cropH,\n );\n const binary = resizeBilinearThreshold(softCrop, cropW, cropH, bboxW, bboxH, maskThreshold);\n\n results.push({ bbox, classId, confidence, mask: new Mask(binary, bboxW, bboxH) });\n }\n\n return results;\n}\n\nfunction sigmoid(x: number): number {\n if (x >= 0) {\n return 1 / (1 + Math.exp(-x));\n }\n const e = Math.exp(x);\n return e / (1 + e);\n}\n\n/**\n * Assemble one instance's soft mask over the prototype region under its box.\n *\n * The loop runs coefficient-major: for each of the `numMaskCoefs` prototypes it\n * walks the crop row by row, so every read of `prototypeData` is contiguous.\n * The pixel-major order it replaces jumped a whole prototype plane (25 600\n * floats for a 160×160 head) between consecutive reads, touching a fresh cache\n * line on every multiply. The accumulator is a `Float64Array` so each pixel's\n * sum is still built in double precision and in the same coefficient order as a\n * scalar `let sum = 0` — the result is bit-identical, only the memory walk\n * changed.\n *\n * @param perAnchorData Flat `output0`.\n * @param prototypeData Flat `output1`.\n * @param coefOffset Index of this anchor's first mask coefficient.\n * @param numAnchors Stride between consecutive coefficients of one anchor.\n * @param numMaskCoefs Number of prototypes.\n * @param maskW Prototype width.\n * @param protoPlane Prototype plane size, `maskW * maskH`.\n * @param left First prototype column of the crop.\n * @param top First prototype row of the crop.\n * @param cropW Crop width in prototype pixels.\n * @param cropH Crop height in prototype pixels.\n * @returns The sigmoid of the mask logits, `cropW * cropH`, row-major.\n */\nfunction softMaskCrop(\n perAnchorData: Float32Array,\n prototypeData: Float32Array,\n coefOffset: number,\n numAnchors: number,\n numMaskCoefs: number,\n maskW: number,\n protoPlane: number,\n left: number,\n top: number,\n cropW: number,\n cropH: number,\n): Float32Array {\n const acc = new Float64Array(cropW * cropH);\n for (let kk = 0; kk < numMaskCoefs; kk++) {\n const coef = perAnchorData[coefOffset + kk * numAnchors] as number;\n const planeBase = kk * protoPlane + left;\n for (let y = 0; y < cropH; y++) {\n const src = planeBase + (top + y) * maskW;\n const dst = y * cropW;\n for (let x = 0; x < cropW; x++) {\n acc[dst + x] = (acc[dst + x] as number) + coef * (prototypeData[src + x] as number);\n }\n }\n }\n const soft = new Float32Array(acc.length);\n for (let j = 0; j < acc.length; j++) soft[j] = sigmoid(acc[j] as number);\n return soft;\n}\n\n/**\n * Bilinearly resize a soft mask and threshold it in the same pass.\n *\n * Half-pixel-centre sampling, matching `cv2.resize(..., INTER_LINEAR)` and the\n * Python SDK. The interpolation is separable and runs that way, like the\n * Python `_resize_bilinear`: first along x over the few source rows, into a\n * `Float64Array` so every horizontal blend keeps its exact double value, then\n * along y between two of those rows. Each output pixel costs two reads and one\n * blend instead of four reads and three, with the same values as blending all\n * four directly. The thresholded byte is written directly instead of\n * materialising a `Float32Array` of the resized mask first.\n * Each interpolated value is rounded to float32 with `Math.fround` before the\n * comparison, because that is the value the intermediate array used to hold —\n * comparing the unrounded double would flip pixels sitting exactly on the\n * threshold.\n *\n * @param src Soft mask, `srcWidth * srcHeight`, row-major.\n * @param srcWidth Source width.\n * @param srcHeight Source height.\n * @param targetWidth Output width, at least 1.\n * @param targetHeight Output height, at least 1.\n * @param threshold Probability at or above which a pixel is foreground.\n * @returns `255` for foreground and `0` for background, `targetWidth * targetHeight`.\n */\nfunction resizeBilinearThreshold(\n src: Float32Array,\n srcWidth: number,\n srcHeight: number,\n targetWidth: number,\n targetHeight: number,\n threshold: number,\n): Uint8Array {\n const out = new Uint8Array(targetWidth * targetHeight);\n if (targetWidth === srcWidth && targetHeight === srcHeight) {\n for (let j = 0; j < out.length; j++) out[j] = (src[j] as number) >= threshold ? 255 : 0;\n return out;\n }\n const sx = srcWidth / targetWidth;\n const sy = srcHeight / targetHeight;\n const x0s = new Int32Array(targetWidth);\n const x1s = new Int32Array(targetWidth);\n const wxs = new Float64Array(targetWidth);\n for (let x = 0; x < targetWidth; x++) {\n const xx = (x + 0.5) * sx - 0.5;\n const x0 = Math.max(0, Math.floor(xx));\n x0s[x] = x0;\n x1s[x] = Math.min(srcWidth - 1, x0 + 1);\n wxs[x] = Math.max(0, Math.min(1, xx - x0));\n }\n const across = new Float64Array(srcHeight * targetWidth);\n for (let r = 0; r < srcHeight; r++) {\n const row = r * srcWidth;\n const dst = r * targetWidth;\n for (let x = 0; x < targetWidth; x++) {\n const wx = wxs[x] as number;\n across[dst + x] =\n (src[row + (x0s[x] as number)] as number) * (1 - wx) +\n (src[row + (x1s[x] as number)] as number) * wx;\n }\n }\n for (let y = 0; y < targetHeight; y++) {\n const yy = (y + 0.5) * sy - 0.5;\n const y0 = Math.max(0, Math.floor(yy));\n const y1 = Math.min(srcHeight - 1, y0 + 1);\n const wy = Math.max(0, Math.min(1, yy - y0));\n const top = y0 * targetWidth;\n const bot = y1 * targetWidth;\n const dst = y * targetWidth;\n for (let x = 0; x < targetWidth; x++) {\n const value = (across[top + x] as number) * (1 - wy) + (across[bot + x] as number) * wy;\n out[dst + x] = Math.fround(value) >= threshold ? 255 : 0;\n }\n }\n return out;\n}\n"],"mappings":";;;AAkEA,SAAgB,EACZ,GACA,GACA,GACA,GACA,GACqB;CACrB,IAAI,IAAQ;CACZ,IAAI,EAAM,WAAW,GAAG;EACpB,IAAI,EAAM,OAAO,GACb,MAAU,MAAM,2DAA2D,EAAM,GAAG,EAAE;EAE1F,IAAQ;GAAC,EAAM;GAAc,EAAM;GAAc,EAAM;EAAY;CACvE;CACA,IAAI,EAAM,WAAW,GACjB,MAAU,MACN,wEAAwE,KAAK,UAAU,CAAa,EAAE,EAC1G;CAEJ,IAAM,IAAe,EAAM,IACrB,IAAQ,EAAM,IACd,IAAQ,EAAM,IAEd,IAAW,EAAc,WAAW,IAAI,EAAc,KAAK,EAAc,IACzE,IAAa,EAAc,WAAW,IAAI,EAAc,KAAK,EAAc;CACjF,IAAI,MAAa,KAAA,KAAa,MAAe,KAAA,GACzC,MAAU,MACN,4DAA4D,KAAK,UAAU,CAAa,EAAE,EAC9F;CAGJ,IAAM,IAAmB,IAAI,EAAQ,aAAa;CAClD,IAAI,MAAa,GACb,MAAU,MACN,2BAA2B,EAAS,iCAAiC,EAAQ,WAAW,mBAAmB,EAAa,MAAM,EAAiB,EACnJ;CAEJ,IAAI,EAAc,WAAW,IAAe,IAAQ,GAChD,MAAU,MACN,mCAAmC,EAAc,OAAO,uBAAuB,KAAK,UAAU,CAAa,EAAE,EACjH;CAGJ,IAAM,IAAU,EAAkB,GAAe,GAAe;EAC5D,YAAY,EAAQ;EACpB,eAAe,EAAQ;EACvB,gBAAgB,EAAQ;EACxB,SAAS,EAAQ;EACjB,QAAQ,EAAQ;EAChB,OAAO,EAAQ;EACf,eAAe,EAAQ;EACvB,cAAc,EAAQ;EACtB,eAAe,EAAQ;CAC3B,CAAC;CAED,IAAI,EAAQ,cAAc,WAAW,GAAG,OAAO,CAAC;CAEhD,IAAM,IAAgB,EAAQ,iBAAiB,IACzC,IAAS,IAAQ,EAAQ,YACzB,IAAS,IAAQ,EAAQ,aACzB,IAAa,IAAQ,GACrB,KAAY,IAAI,EAAQ,cAAc,GAEtC,IAAiC,CAAC;CAExC,KAAK,IAAI,IAAI,GAAG,IAAI,EAAQ,cAAc,QAAQ,KAAK;EACnD,IAAM,IAAI,EAAQ,cAAc,IAC1B,IAAK,EAAQ,UAAU,IAAI,IAC3B,IAAK,EAAQ,UAAU,IAAI,IAAI,IAC/B,IAAK,EAAQ,UAAU,IAAI,IAAI,IAC/B,IAAK,EAAQ,UAAU,IAAI,IAAI,IAC/B,IAAO,IAAI,EAAY,GAAI,GAAI,GAAI,CAAE,GACrC,IAAU,EAAQ,SAAS,IAC3B,IAAa,EAAQ,YAAY,IAEjC,IAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,CAAE,IAAI,KAAK,MAAM,CAAE,CAAC,GACnD,IAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,CAAE,IAAI,KAAK,MAAM,CAAE,CAAC;EAEzD,IAAI,MAAU,KAAK,MAAU,GAAG;GAC5B,EAAQ,KAAK;IAAE;IAAM;IAAS;IAAY,MAAM,IAAI,kBAAK,IAAI,WAAY,GAAG,GAAG,CAAC;GAAE,CAAC;GACnF;EACJ;EAEA,IAAM,IAAO,IAAK,EAAQ,QAAQ,EAAQ,SACpC,IAAO,IAAK,EAAQ,QAAQ,EAAQ,QACpC,IAAO,IAAK,EAAQ,QAAQ,EAAQ,SACpC,IAAO,IAAK,EAAQ,QAAQ,EAAQ,QAEpC,IAAO,KAAK,IAAI,GAAG,KAAK,MAAM,IAAO,CAAM,CAAC,GAC5C,IAAO,KAAK,IAAI,GAAG,KAAK,MAAM,IAAO,CAAM,CAAC,GAC5C,IAAO,KAAK,IAAI,GAAO,KAAK,KAAK,IAAO,CAAM,CAAC,GAC/C,IAAO,KAAK,IAAI,GAAO,KAAK,KAAK,IAAO,CAAM,CAAC;EAErD,IAAI,KAAQ,KAAQ,KAAQ,GAAM;GAC9B,EAAQ,KAAK;IACT;IACA;IACA;IACA,MAAM,IAAI,EAAK,IAAI,WAAW,IAAQ,CAAK,GAAG,GAAO,CAAK;GAC9D,CAAC;GACD;EACJ;EAEA,IAAM,IAAQ,IAAO,GACf,IAAQ,IAAO,GAcf,IAAS,EAbE,EACb,GACA,GACA,IAAW,GACX,GACA,GACA,GACA,GACA,GACA,GACA,GACA,CAEmC,GAAU,GAAO,GAAO,GAAO,GAAO,CAAa;EAE1F,EAAQ,KAAK;GAAE;GAAM;GAAS;GAAY,MAAM,IAAI,EAAK,GAAQ,GAAO,CAAK;EAAE,CAAC;CACpF;CAEA,OAAO;AACX;AAEA,SAAS,EAAQ,GAAmB;CAChC,IAAI,KAAK,GACL,OAAO,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC;CAE/B,IAAM,IAAI,KAAK,IAAI,CAAC;CACpB,OAAO,KAAK,IAAI;AACpB;AA2BA,SAAS,EACL,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACA,GACY;CACZ,IAAM,IAAM,IAAI,aAAa,IAAQ,CAAK;CAC1C,KAAK,IAAI,IAAK,GAAG,IAAK,GAAc,KAAM;EACtC,IAAM,IAAO,EAAc,IAAa,IAAK,IACvC,IAAY,IAAK,IAAa;EACpC,KAAK,IAAI,IAAI,GAAG,IAAI,GAAO,KAAK;GAC5B,IAAM,IAAM,KAAa,IAAM,KAAK,GAC9B,IAAM,IAAI;GAChB,KAAK,IAAI,IAAI,GAAG,IAAI,GAAO,KACvB,EAAI,IAAM,KAAM,EAAI,IAAM,KAAgB,IAAQ,EAAc,IAAM;EAE9E;CACJ;CACA,IAAM,IAAO,IAAI,aAAa,EAAI,MAAM;CACxC,KAAK,IAAI,IAAI,GAAG,IAAI,EAAI,QAAQ,KAAK,EAAK,KAAK,EAAQ,EAAI,EAAY;CACvE,OAAO;AACX;AA0BA,SAAS,EACL,GACA,GACA,GACA,GACA,GACA,GACU;CACV,IAAM,IAAM,IAAI,WAAW,IAAc,CAAY;CACrD,IAAI,MAAgB,KAAY,MAAiB,GAAW;EACxD,KAAK,IAAI,IAAI,GAAG,IAAI,EAAI,QAAQ,KAAK,EAAI,KAAM,EAAI,MAAiB,IAAY,MAAM;EACtF,OAAO;CACX;CACA,IAAM,IAAK,IAAW,GAChB,IAAK,IAAY,GACjB,IAAM,IAAI,WAAW,CAAW,GAChC,IAAM,IAAI,WAAW,CAAW,GAChC,IAAM,IAAI,aAAa,CAAW;CACxC,KAAK,IAAI,IAAI,GAAG,IAAI,GAAa,KAAK;EAClC,IAAM,KAAM,IAAI,MAAO,IAAK,IACtB,IAAK,KAAK,IAAI,GAAG,KAAK,MAAM,CAAE,CAAC;EAGrC,AAFA,EAAI,KAAK,GACT,EAAI,KAAK,KAAK,IAAI,IAAW,GAAG,IAAK,CAAC,GACtC,EAAI,KAAK,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,IAAK,CAAE,CAAC;CAC7C;CACA,IAAM,IAAS,IAAI,aAAa,IAAY,CAAW;CACvD,KAAK,IAAI,IAAI,GAAG,IAAI,GAAW,KAAK;EAChC,IAAM,IAAM,IAAI,GACV,IAAM,IAAI;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,GAAa,KAAK;GAClC,IAAM,IAAK,EAAI;GACf,EAAO,IAAM,KACR,EAAI,IAAO,EAAI,OAA6B,IAAI,KAChD,EAAI,IAAO,EAAI,MAA4B;EACpD;CACJ;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,GAAc,KAAK;EACnC,IAAM,KAAM,IAAI,MAAO,IAAK,IACtB,IAAK,KAAK,IAAI,GAAG,KAAK,MAAM,CAAE,CAAC,GAC/B,IAAK,KAAK,IAAI,IAAY,GAAG,IAAK,CAAC,GACnC,IAAK,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,IAAK,CAAE,CAAC,GACrC,IAAM,IAAK,GACX,IAAM,IAAK,GACX,IAAM,IAAI;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,GAAa,KAAK;GAClC,IAAM,IAAS,EAAO,IAAM,MAAiB,IAAI,KAAO,EAAO,IAAM,KAAgB;GACrF,EAAI,IAAM,KAAK,KAAK,OAAO,CAAK,KAAK,IAAY,MAAM;EAC3D;CACJ;CACA,OAAO;AACX"}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const e=require("../core/canvas.cjs");var t=1/255,n=class{_size;_buffer;_inUse=!1;constructor(e){this._size=e,this._buffer=new Float32Array(e)}claim(){return this._inUse?{data:new Float32Array(this._size),reused:!1}:(this._buffer.length!==this._size&&(this._buffer=new Float32Array(this._size)),this._inUse=!0,{data:this._buffer,reused:!0})}release(){this._inUse=!1}},r=class{_targetWidth;_targetHeight;_fill;_target;_targetContext;_buffer;_source=null;_sourceContext=null;constructor(t,r,i=[114,114,114]){if(t<=0||r<=0)throw Error(`Invalid letterbox target ${t}x${r}.`);this._targetWidth=t,this._targetHeight=r,this._fill=i,this._target=e.createCanvas(t,r),this._targetContext=e.get2DContext(this._target,{willReadFrequently:!0}),this._targetContext.imageSmoothingEnabled=!0,this._targetContext.imageSmoothingQuality=`high`,this._buffer=new n(3*r*t)}get targetSize(){return[this._targetWidth,this._targetHeight]}run(n){let r=this._targetWidth,i=this._targetHeight,a=Math.min(r/
|
|
1
|
+
const e=require("../core/canvas.cjs");var t=1/255,n=class{_size;_buffer;_inUse=!1;constructor(e){this._size=e,this._buffer=new Float32Array(e)}claim(){return this._inUse?{data:new Float32Array(this._size),reused:!1}:(this._buffer.length!==this._size&&(this._buffer=new Float32Array(this._size)),this._inUse=!0,{data:this._buffer,reused:!0})}release(){this._inUse=!1}},r=class{_targetWidth;_targetHeight;_fill;_target;_targetContext;_buffer;_source=null;_sourceContext=null;constructor(t,r,i=[114,114,114]){if(t<=0||r<=0)throw Error(`Invalid letterbox target ${t}x${r}.`);this._targetWidth=t,this._targetHeight=r,this._fill=i,this._target=e.createCanvas(t,r),this._targetContext=e.get2DContext(this._target,{willReadFrequently:!0}),this._targetContext.imageSmoothingEnabled=!0,this._targetContext.imageSmoothingQuality=`high`,this._buffer=new n(3*r*t)}get targetSize(){return[this._targetWidth,this._targetHeight]}run(e,n=null){let r=this._targetWidth,i=this._targetHeight,a=Math.min(r/e.width,i/e.height),o=Math.round(e.width*a),s=Math.round(e.height*a),c=Math.floor((r-o)/2),l=Math.floor((i-s)/2),u=n??this._paintSource(e),d=this._targetContext;(c>0||l>0||o!==r||s!==i)&&(d.fillStyle=`rgb(${this._fill[0]},${this._fill[1]},${this._fill[2]})`,d.fillRect(0,0,r,i)),d.drawImage(u,0,0,e.width,e.height,c,l,o,s);let f=d.getImageData(0,0,r,i).data,{data:p,reused:m}=this._buffer.claim(),h=r*i;for(let e=0,n=0;e<h;e++,n+=4)p[e]=f[n]*t,p[h+e]=f[n+1]*t,p[2*h+e]=f[n+2]*t;return{data:p,scale:a,padLeft:c,padTop:l,reused:m}}release(e){(e===void 0||e.reused)&&this._buffer.release()}_paintSource(t){let{width:n,height:r}=t;return(this._source===null||this._source.width!==n||this._source.height!==r)&&(this._source=e.createCanvas(n,r),this._sourceContext=e.get2DContext(this._source)),this._sourceContext.putImageData(e.rgbToImageData(t),0,0),this._source}};function i(e,n,r,i,a,o,s=4){let c=n*r,l=i[0],u=i[1],d=i[2],f=a[0],p=a[1],m=a[2];for(let n=0,r=0;n<c;n++,r+=s)o[n]=(e[r]*t-l)/f,o[c+n]=(e[r+1]*t-u)/p,o[2*c+n]=(e[r+2]*t-d)/m}var a=class{_targetWidth;_targetHeight;_mean;_std;_buffer;_target=null;_targetContext=null;_source=null;_sourceContext=null;constructor(e,t,r=[0,0,0],i=[1,1,1]){if(e<=0||t<=0)throw Error(`Invalid resize target ${e}x${t}.`);this._targetWidth=e,this._targetHeight=t,this._mean=r,this._std=i,this._buffer=new n(3*t*e)}get targetSize(){return[this._targetWidth,this._targetHeight]}run(e,t=null){let n=this._targetWidth,r=this._targetHeight,{data:a,reused:o}=this._buffer.claim();if(e.width===n&&e.height===r)return i(e.data,n,r,this._mean,this._std,a,3),{data:a,reused:o};let s=t??this._paintSource(e),c=this._ensureTarget();c.drawImage(s,0,0,e.width,e.height,0,0,n,r);let l=c.getImageData(0,0,n,r).data;return i(l,n,r,this._mean,this._std,a),{data:a,reused:o}}release(e){(e===void 0||e.reused)&&this._buffer.release()}_ensureTarget(){return this._target===null&&(this._target=e.createCanvas(this._targetWidth,this._targetHeight),this._targetContext=e.get2DContext(this._target,{willReadFrequently:!0}),this._targetContext.imageSmoothingEnabled=!0,this._targetContext.imageSmoothingQuality=`high`),this._targetContext}_paintSource(t){let{width:n,height:r}=t;return(this._source===null||this._source.width!==n||this._source.height!==r)&&(this._source=e.createCanvas(n,r),this._sourceContext=e.get2DContext(this._source)),this._sourceContext.putImageData(e.rgbToImageData(t),0,0),this._source}};function o(e,t,n,r=[0,0,0],i=[1,1,1]){return new a(t,n,r,i).run(e)}function s(e,t){return new Float32Array(3*t*e)}function c(e,t,n,i=[114,114,114]){return new r(t,n,i).run(e)}exports.LetterboxPipeline=r,exports.ResizePipeline=a,exports.letterboxToTensorData=c,exports.resizeToTensorData=o,exports.writePlanarFloat32=i,exports.zeroTensorData=s;
|
|
2
2
|
//# sourceMappingURL=pipeline.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.cjs","names":[],"sources":["../../../src/vision/preprocess/pipeline.ts"],"sourcesContent":["/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */\n/**\n * Fused letterbox → CHW float32 pipeline with reusable buffers.\n *\n * The composable primitives in {@link ./image.js} each allocate and each walk\n * their input end to end, which is the right shape for a library but the wrong\n * shape for a video loop. Chaining them costs eleven full-buffer passes and six\n * large allocations per frame:\n *\n * `getImageData` → RGBA→RGB → RGB→RGBA → `putImageData` → `drawImage` →\n * `getImageData` → RGBA→RGB → fill → row copies → `toFloat32` → `toCHW`.\n *\n * This module collapses the second half of that into two: one `drawImage` that\n * resizes *and* positions the content inside the padded target in a single\n * accelerated operation, and one loop that reads the resulting RGBA and writes\n * planar float32 directly. The intermediate `RGBImage` at target size, the fill\n * loop, the row copies and the two 4.9 MB `Float32Array` allocations all go\n * away.\n *\n * The primitives stay exactly as they are — they are public API and they are\n * what makes a custom pipeline writable. This is the fast path the built-in\n * tasks take.\n */\n\nimport {\n createCanvas,\n get2DContext,\n rgbToImageData,\n type Canvas2D,\n type Context2D,\n} from \"../core/canvas\";\nimport type { RGBImage } from \"../types\";\n\nconst INV_255 = 1 / 255;\n\n/** One claim on a {@link ReusableBuffer}. */\ninterface BufferClaim {\n /** The buffer to write into, always `size` long. */\n readonly data: Float32Array;\n /** Whether {@link data} is the held buffer rather than a fresh allocation. */\n readonly reused: boolean;\n}\n\n/**\n * A `Float32Array` held across calls, handed out one claim at a time.\n *\n * Both pipelines want the same thing — allocate once, write into it every frame,\n * and fall back to a fresh array when a previous result has not been released\n * yet — so the bookkeeping lives here instead of twice.\n *\n * The part that is not obvious is {@link claim} re-allocating a buffer that is\n * *detached*. `ort.env.wasm.proxy` runs ONNX Runtime in a worker and posts the\n * input tensors with their `ArrayBuffer`s in the transfer list, which detaches\n * them on this side. A detached `Float32Array` is silently 0 long: writing to it\n * is a no-op, and the next `InferenceSession.run` rejects with\n * `Tensor's size(N) does not match data length(0)` on every other call — once for\n * the detached buffer, then the throw leaves the claim outstanding so the call\n * after it allocates and succeeds. Treating a buffer that changed length as spent\n * turns that into one extra allocation per transfer, which is what reuse was\n * avoiding, and keeps it correct for any consumer that transfers the tensor\n * rather than copying it.\n */\nclass ReusableBuffer {\n private readonly _size: number;\n private _buffer: Float32Array;\n private _inUse = false;\n\n /** @param size Length in floats of every buffer this hands out. */\n constructor(size: number) {\n this._size = size;\n this._buffer = new Float32Array(size);\n }\n\n /**\n * Take the held buffer, or a fresh one when it is unavailable.\n *\n * Unavailable means either still checked out by an unreleased claim, or\n * detached by whoever it was handed to. The first case allocates for this call\n * only; the second replaces the held buffer, so the allocation is paid once per\n * transfer rather than on every call after it.\n */\n claim(): BufferClaim {\n if (this._inUse) return { data: new Float32Array(this._size), reused: false };\n if (this._buffer.length !== this._size) this._buffer = new Float32Array(this._size);\n this._inUse = true;\n return { data: this._buffer, reused: true };\n }\n\n /** Mark the held buffer free for the next {@link claim}. */\n release(): void {\n this._inUse = false;\n }\n}\n\n/** Geometry of a letterbox, plus the planar tensor data it produced. */\nexport interface FusedLetterboxResult {\n /** CHW float32 in `[0, 1]`, length `3 * targetHeight * targetWidth`. */\n readonly data: Float32Array;\n /** Factor applied to the original image (`< 1` if downscaled). */\n readonly scale: number;\n /** Horizontal padding in pixels. */\n readonly padLeft: number;\n /** Vertical padding in pixels. */\n readonly padTop: number;\n /**\n * Whether {@link data} is the pipeline's reusable buffer.\n *\n * `true` means the next {@link LetterboxPipeline.run} overwrites it, so a\n * caller keeping the values past its own inference has to copy them.\n */\n readonly reused: boolean;\n}\n\n/**\n * Reusable letterbox → tensor pipeline for one target resolution.\n *\n * Holds a target canvas and an output buffer across calls, so a steady stream\n * of frames at the same size allocates nothing. Create one per task, not per\n * frame.\n */\nexport class LetterboxPipeline {\n private readonly _targetWidth: number;\n private readonly _targetHeight: number;\n private readonly _fill: readonly [number, number, number];\n private readonly _target: Canvas2D;\n private readonly _targetContext: Context2D;\n private readonly _buffer: ReusableBuffer;\n private _source: Canvas2D | null = null;\n private _sourceContext: Context2D | null = null;\n\n /**\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param fill RGB padding colour; defaults to YOLO grey.\n */\n constructor(\n targetWidth: number,\n targetHeight: number,\n fill: readonly [number, number, number] = [114, 114, 114],\n ) {\n if (targetWidth <= 0 || targetHeight <= 0) {\n throw new Error(`Invalid letterbox target ${targetWidth}x${targetHeight}.`);\n }\n this._targetWidth = targetWidth;\n this._targetHeight = targetHeight;\n this._fill = fill;\n this._target = createCanvas(targetWidth, targetHeight);\n this._targetContext = get2DContext(this._target, { willReadFrequently: true });\n this._targetContext.imageSmoothingEnabled = true;\n this._targetContext.imageSmoothingQuality = \"high\";\n this._buffer = new ReusableBuffer(3 * targetHeight * targetWidth);\n }\n\n /** The `[width, height]` this pipeline letterboxes into. */\n get targetSize(): readonly [number, number] {\n return [this._targetWidth, this._targetHeight];\n }\n\n /**\n * Letterbox an image and write it as planar float32.\n *\n * The returned buffer is reused between calls unless a previous result is\n * still checked out — {@link release} marks it free again. A second `run`\n * before the first is released allocates a fresh buffer rather than\n * corrupting it, so concurrent `predict()` calls on one task stay correct at\n * the cost of the allocation they were trying to avoid. A buffer that was\n * detached by a consumer that transferred it is replaced rather than written\n * into — see {@link ReusableBuffer}.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n */\n run(image: RGBImage): FusedLetterboxResult {\n const targetWidth = this._targetWidth;\n const targetHeight = this._targetHeight;\n const scale = Math.min(targetWidth / image.width, targetHeight / image.height);\n const scaledWidth = Math.round(image.width * scale);\n const scaledHeight = Math.round(image.height * scale);\n const padLeft = Math.floor((targetWidth - scaledWidth) / 2);\n const padTop = Math.floor((targetHeight - scaledHeight) / 2);\n\n const source = this._ensureSource(image.width, image.height);\n source.putImageData(rgbToImageData(image), 0, 0);\n\n const context = this._targetContext;\n if (\n padLeft > 0 ||\n padTop > 0 ||\n scaledWidth !== targetWidth ||\n scaledHeight !== targetHeight\n ) {\n context.fillStyle = `rgb(${this._fill[0]},${this._fill[1]},${this._fill[2]})`;\n context.fillRect(0, 0, targetWidth, targetHeight);\n }\n context.drawImage(\n this._source as CanvasImageSource,\n 0,\n 0,\n image.width,\n image.height,\n padLeft,\n padTop,\n scaledWidth,\n scaledHeight,\n );\n\n const rgba = context.getImageData(0, 0, targetWidth, targetHeight).data;\n const { data, reused } = this._buffer.claim();\n\n const plane = targetWidth * targetHeight;\n for (let pixel = 0, offset = 0; pixel < plane; pixel++, offset += 4) {\n data[pixel] = (rgba[offset] as number) * INV_255;\n data[plane + pixel] = (rgba[offset + 1] as number) * INV_255;\n data[2 * plane + pixel] = (rgba[offset + 2] as number) * INV_255;\n }\n\n return { data, scale, padLeft, padTop, reused };\n }\n\n /**\n * Mark the reusable buffer free again.\n *\n * Call it once the tensor built from a {@link run} result has been handed to\n * ONNX Runtime and the run has resolved — after that the values are inside\n * the WASM heap and the buffer can be overwritten.\n */\n release(): void {\n this._buffer.release();\n }\n\n /**\n * Grow the scratch source canvas to fit an image, reusing it when possible.\n *\n * A canvas is only reallocated when a frame arrives at a different size than\n * the last one, which for a camera or video source is never after the first.\n *\n * @param width Source width in pixels.\n * @param height Source height in pixels.\n */\n private _ensureSource(width: number, height: number): Context2D {\n if (\n this._source === null ||\n this._source.width !== width ||\n this._source.height !== height\n ) {\n this._source = createCanvas(width, height);\n this._sourceContext = get2DContext(this._source);\n }\n return this._sourceContext as Context2D;\n }\n}\n\n/**\n * Write an RGBA buffer as normalized planar float32.\n *\n * Shared by {@link ResizePipeline} and exported for a custom pipeline that\n * already holds pixels and wants the SDK's exact arithmetic.\n *\n * The expression is `(value / 255 - mean) / std`, evaluated in that order on\n * purpose: folding it into a single multiply-add (`value * k + b`) computes the\n * same quantity but rounds differently, and the tasks' output is asserted to be\n * bit-identical to the composable `normalize` → `toCHW` path.\n *\n * @param rgba Source pixels, 4 bytes per pixel, `width * height` long.\n * @param width Image width in pixels.\n * @param height Image height in pixels.\n * @param mean Per-channel RGB mean, already in `[0, 1]`.\n * @param std Per-channel RGB standard deviation.\n * @param out Destination buffer, `3 * width * height` long.\n * @param stride Bytes per source pixel: 4 for canvas RGBA, 3 for packed RGB.\n */\nexport function writePlanarFloat32(\n rgba: Uint8ClampedArray | Uint8Array,\n width: number,\n height: number,\n mean: readonly [number, number, number],\n std: readonly [number, number, number],\n out: Float32Array,\n stride: number = 4,\n): void {\n const plane = width * height;\n const m0 = mean[0];\n const m1 = mean[1];\n const m2 = mean[2];\n const s0 = std[0];\n const s1 = std[1];\n const s2 = std[2];\n for (let pixel = 0, offset = 0; pixel < plane; pixel++, offset += stride) {\n out[pixel] = ((rgba[offset] as number) * INV_255 - m0) / s0;\n out[plane + pixel] = ((rgba[offset + 1] as number) * INV_255 - m1) / s1;\n out[2 * plane + pixel] = ((rgba[offset + 2] as number) * INV_255 - m2) / s2;\n }\n}\n\n/** Planar tensor data produced by {@link ResizePipeline}. */\nexport interface FusedResizeResult {\n /** CHW float32, normalized, length `3 * targetHeight * targetWidth`. */\n readonly data: Float32Array;\n /**\n * Whether {@link data} is the pipeline's reusable buffer.\n *\n * `true` means the next {@link ResizePipeline.run} overwrites it, so a caller\n * keeping the values past its own inference has to copy them.\n */\n readonly reused: boolean;\n}\n\n/**\n * Reusable stretch-resize → normalized tensor pipeline for one target size.\n *\n * The classification counterpart of {@link LetterboxPipeline}. A classifier\n * stretches to the model's square input instead of letterboxing into it — no\n * padding, no scale to invert later, because nothing is mapped back onto the\n * source image afterwards. That difference is why it cannot simply reuse the\n * letterbox path.\n *\n * What it does share is the technique. The composable route\n * (`resize` → `normalize` → `toCHW`) allocates an `RGBImage` and two\n * `Float32Array`s and walks each end to end on every call: about 1.4 MB of\n * fresh garbage per 224×224 `predict()`, produced at the exact moment a phone\n * near its memory ceiling can least afford it. Here one `drawImage` resizes,\n * and one loop reads the resulting RGBA and writes normalized planar float32\n * into a buffer held across calls.\n *\n * Create one per task, not per frame.\n */\nexport class ResizePipeline {\n private readonly _targetWidth: number;\n private readonly _targetHeight: number;\n private readonly _mean: readonly [number, number, number];\n private readonly _std: readonly [number, number, number];\n private readonly _buffer: ReusableBuffer;\n private _target: Canvas2D | null = null;\n private _targetContext: Context2D | null = null;\n private _source: Canvas2D | null = null;\n private _sourceContext: Context2D | null = null;\n\n /**\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.\n * @param std Per-channel RGB standard deviation. Defaults to no scaling.\n */\n constructor(\n targetWidth: number,\n targetHeight: number,\n mean: readonly [number, number, number] = [0, 0, 0],\n std: readonly [number, number, number] = [1, 1, 1],\n ) {\n if (targetWidth <= 0 || targetHeight <= 0) {\n throw new Error(`Invalid resize target ${targetWidth}x${targetHeight}.`);\n }\n this._targetWidth = targetWidth;\n this._targetHeight = targetHeight;\n this._mean = mean;\n this._std = std;\n this._buffer = new ReusableBuffer(3 * targetHeight * targetWidth);\n }\n\n /** The `[width, height]` this pipeline resizes into. */\n get targetSize(): readonly [number, number] {\n return [this._targetWidth, this._targetHeight];\n }\n\n /**\n * Resize an image to the target size and write it as normalized planar float32.\n *\n * An image that already arrives at the target size skips the canvas entirely\n * and is read straight out of its packed RGB — which is both faster and what\n * keeps the result identical to `resize()`, whose own fast path returns the\n * input untouched.\n *\n * Buffer reuse follows {@link ReusableBuffer}: held across calls, replaced when\n * a consumer detached it by transferring the tensor.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n */\n run(image: RGBImage): FusedResizeResult {\n const targetWidth = this._targetWidth;\n const targetHeight = this._targetHeight;\n const { data, reused } = this._buffer.claim();\n\n if (image.width === targetWidth && image.height === targetHeight) {\n writePlanarFloat32(\n image.data,\n targetWidth,\n targetHeight,\n this._mean,\n this._std,\n data,\n 3,\n );\n return { data, reused };\n }\n\n const source = this._ensureSource(image.width, image.height);\n source.putImageData(rgbToImageData(image), 0, 0);\n\n const context = this._ensureTarget();\n context.drawImage(this._source as CanvasImageSource, 0, 0, targetWidth, targetHeight);\n\n const rgba = context.getImageData(0, 0, targetWidth, targetHeight).data;\n writePlanarFloat32(rgba, targetWidth, targetHeight, this._mean, this._std, data);\n return { data, reused };\n }\n\n /**\n * Mark the reusable buffer free again.\n *\n * Call it once the tensor built from a {@link run} result has been handed to\n * ONNX Runtime and the run has resolved — after that the values are inside\n * the WASM heap and the buffer can be overwritten.\n */\n release(): void {\n this._buffer.release();\n }\n\n /**\n * Build the target canvas on first use.\n *\n * Lazily, so a pipeline constructed where no canvas implementation exists\n * (a Node test, a worker without OffscreenCanvas) only fails if it is asked\n * to resize something.\n */\n private _ensureTarget(): Context2D {\n if (this._target === null) {\n this._target = createCanvas(this._targetWidth, this._targetHeight);\n this._targetContext = get2DContext(this._target, { willReadFrequently: true });\n this._targetContext.imageSmoothingEnabled = true;\n this._targetContext.imageSmoothingQuality = \"high\";\n }\n return this._targetContext as Context2D;\n }\n\n /** Grow the scratch source canvas to fit an image, reusing it when possible. */\n private _ensureSource(width: number, height: number): Context2D {\n if (\n this._source === null ||\n this._source.width !== width ||\n this._source.height !== height\n ) {\n this._source = createCanvas(width, height);\n this._sourceContext = get2DContext(this._source);\n }\n return this._sourceContext as Context2D;\n }\n}\n\n/**\n * Resize an image into normalized planar float32 without keeping any state.\n *\n * The allocation-free path is {@link ResizePipeline}; this is the one-shot\n * form, for a caller who wants the fused behaviour without owning a pipeline.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.\n * @param std Per-channel RGB standard deviation. Defaults to no scaling.\n */\nexport function resizeToTensorData(\n image: RGBImage,\n targetWidth: number,\n targetHeight: number,\n mean: readonly [number, number, number] = [0, 0, 0],\n std: readonly [number, number, number] = [1, 1, 1],\n): FusedResizeResult {\n return new ResizePipeline(targetWidth, targetHeight, mean, std).run(image);\n}\n\n/**\n * Build a zero-filled CHW tensor payload for a warm-up run.\n *\n * @param width Model input width in pixels.\n * @param height Model input height in pixels.\n */\nexport function zeroTensorData(width: number, height: number): Float32Array {\n return new Float32Array(3 * height * width);\n}\n\n/**\n * Letterbox an image into planar float32 without keeping any state.\n *\n * The allocation-free path is {@link LetterboxPipeline}; this is the one-shot\n * form, for a caller who wants the fused behaviour without owning a pipeline.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param fill RGB padding colour; defaults to YOLO grey.\n */\nexport function letterboxToTensorData(\n image: RGBImage,\n targetWidth: number,\n targetHeight: number,\n fill: readonly [number, number, number] = [114, 114, 114],\n): FusedLetterboxResult {\n return new LetterboxPipeline(targetWidth, targetHeight, fill).run(image);\n}\n"],"mappings":"sCAiCA,IAAM,EAAU,EAAI,IA6Bd,EAAN,KAAqB,CACjB,MACA,QACA,OAAiB,GAGjB,YAAY,EAAc,CACtB,KAAK,MAAQ,EACb,KAAK,QAAU,IAAI,aAAa,CAAI,CACxC,CAUA,OAAqB,CAIjB,OAHI,KAAK,OAAe,CAAE,KAAM,IAAI,aAAa,KAAK,KAAK,EAAG,OAAQ,EAAM,GACxE,KAAK,QAAQ,SAAW,KAAK,QAAO,KAAK,QAAU,IAAI,aAAa,KAAK,KAAK,GAClF,KAAK,OAAS,GACP,CAAE,KAAM,KAAK,QAAS,OAAQ,EAAK,EAC9C,CAGA,SAAgB,CACZ,KAAK,OAAS,EAClB,CACJ,EA4Ba,EAAb,KAA+B,CAC3B,aACA,cACA,MACA,QACA,eACA,QACA,QAAmC,KACnC,eAA2C,KAO3C,YACI,EACA,EACA,EAA0C,CAAC,IAAK,IAAK,GAAG,EAC1D,CACE,GAAI,GAAe,GAAK,GAAgB,EACpC,MAAU,MAAM,4BAA4B,EAAY,GAAG,EAAa,EAAE,EAE9E,KAAK,aAAe,EACpB,KAAK,cAAgB,EACrB,KAAK,MAAQ,EACb,KAAK,QAAU,EAAA,aAAa,EAAa,CAAY,EACrD,KAAK,eAAiB,EAAA,aAAa,KAAK,QAAS,CAAE,mBAAoB,EAAK,CAAC,EAC7E,KAAK,eAAe,sBAAwB,GAC5C,KAAK,eAAe,sBAAwB,OAC5C,KAAK,QAAU,IAAI,EAAe,EAAI,EAAe,CAAW,CACpE,CAGA,IAAI,YAAwC,CACxC,MAAO,CAAC,KAAK,aAAc,KAAK,aAAa,CACjD,CAeA,IAAI,EAAuC,CACvC,IAAM,EAAc,KAAK,aACnB,EAAe,KAAK,cACpB,EAAQ,KAAK,IAAI,EAAc,EAAM,MAAO,EAAe,EAAM,MAAM,EACvE,EAAc,KAAK,MAAM,EAAM,MAAQ,CAAK,EAC5C,EAAe,KAAK,MAAM,EAAM,OAAS,CAAK,EAC9C,EAAU,KAAK,OAAO,EAAc,GAAe,CAAC,EACpD,EAAS,KAAK,OAAO,EAAe,GAAgB,CAAC,EAG3D,KADoB,cAAc,EAAM,MAAO,EAAM,MACrD,CAAA,CAAO,aAAa,EAAA,eAAe,CAAK,EAAG,EAAG,CAAC,EAE/C,IAAM,EAAU,KAAK,gBAEjB,EAAU,GACV,EAAS,GACT,IAAgB,GAChB,IAAiB,KAEjB,EAAQ,UAAY,OAAO,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAC3E,EAAQ,SAAS,EAAG,EAAG,EAAa,CAAY,GAEpD,EAAQ,UACJ,KAAK,QACL,EACA,EACA,EAAM,MACN,EAAM,OACN,EACA,EACA,EACA,CACJ,EAEA,IAAM,EAAO,EAAQ,aAAa,EAAG,EAAG,EAAa,CAAY,CAAC,CAAC,KAC7D,CAAE,OAAM,UAAW,KAAK,QAAQ,MAAM,EAEtC,EAAQ,EAAc,EAC5B,IAAK,IAAI,EAAQ,EAAG,EAAS,EAAG,EAAQ,EAAO,IAAS,GAAU,EAC9D,EAAK,GAAU,EAAK,GAAqB,EACzC,EAAK,EAAQ,GAAU,EAAK,EAAS,GAAgB,EACrD,EAAK,EAAI,EAAQ,GAAU,EAAK,EAAS,GAAgB,EAG7D,MAAO,CAAE,OAAM,QAAO,UAAS,SAAQ,QAAO,CAClD,CASA,SAAgB,CACZ,KAAK,QAAQ,QAAQ,CACzB,CAWA,cAAsB,EAAe,EAA2B,CAS5D,OAPI,KAAK,UAAY,MACjB,KAAK,QAAQ,QAAU,GACvB,KAAK,QAAQ,SAAW,KAExB,KAAK,QAAU,EAAA,aAAa,EAAO,CAAM,EACzC,KAAK,eAAiB,EAAA,aAAa,KAAK,OAAO,GAE5C,KAAK,cAChB,CACJ,EAqBA,SAAgB,EACZ,EACA,EACA,EACA,EACA,EACA,EACA,EAAiB,EACb,CACJ,IAAM,EAAQ,EAAQ,EAChB,EAAK,EAAK,GACV,EAAK,EAAK,GACV,EAAK,EAAK,GACV,EAAK,EAAI,GACT,EAAK,EAAI,GACT,EAAK,EAAI,GACf,IAAK,IAAI,EAAQ,EAAG,EAAS,EAAG,EAAQ,EAAO,IAAS,GAAU,EAC9D,EAAI,IAAW,EAAK,GAAqB,EAAU,GAAM,EACzD,EAAI,EAAQ,IAAW,EAAK,EAAS,GAAgB,EAAU,GAAM,EACrE,EAAI,EAAI,EAAQ,IAAW,EAAK,EAAS,GAAgB,EAAU,GAAM,CAEjF,CAkCA,IAAa,EAAb,KAA4B,CACxB,aACA,cACA,MACA,KACA,QACA,QAAmC,KACnC,eAA2C,KAC3C,QAAmC,KACnC,eAA2C,KAQ3C,YACI,EACA,EACA,EAA0C,CAAC,EAAG,EAAG,CAAC,EAClD,EAAyC,CAAC,EAAG,EAAG,CAAC,EACnD,CACE,GAAI,GAAe,GAAK,GAAgB,EACpC,MAAU,MAAM,yBAAyB,EAAY,GAAG,EAAa,EAAE,EAE3E,KAAK,aAAe,EACpB,KAAK,cAAgB,EACrB,KAAK,MAAQ,EACb,KAAK,KAAO,EACZ,KAAK,QAAU,IAAI,EAAe,EAAI,EAAe,CAAW,CACpE,CAGA,IAAI,YAAwC,CACxC,MAAO,CAAC,KAAK,aAAc,KAAK,aAAa,CACjD,CAeA,IAAI,EAAoC,CACpC,IAAM,EAAc,KAAK,aACnB,EAAe,KAAK,cACpB,CAAE,OAAM,UAAW,KAAK,QAAQ,MAAM,EAE5C,GAAI,EAAM,QAAU,GAAe,EAAM,SAAW,EAUhD,OATA,EACI,EAAM,KACN,EACA,EACA,KAAK,MACL,KAAK,KACL,EACA,CACJ,EACO,CAAE,OAAM,QAAO,EAI1B,KADoB,cAAc,EAAM,MAAO,EAAM,MACrD,CAAA,CAAO,aAAa,EAAA,eAAe,CAAK,EAAG,EAAG,CAAC,EAE/C,IAAM,EAAU,KAAK,cAAc,EACnC,EAAQ,UAAU,KAAK,QAA8B,EAAG,EAAG,EAAa,CAAY,EAEpF,IAAM,EAAO,EAAQ,aAAa,EAAG,EAAG,EAAa,CAAY,CAAC,CAAC,KAEnE,OADA,EAAmB,EAAM,EAAa,EAAc,KAAK,MAAO,KAAK,KAAM,CAAI,EACxE,CAAE,OAAM,QAAO,CAC1B,CASA,SAAgB,CACZ,KAAK,QAAQ,QAAQ,CACzB,CASA,eAAmC,CAO/B,OANI,KAAK,UAAY,OACjB,KAAK,QAAU,EAAA,aAAa,KAAK,aAAc,KAAK,aAAa,EACjE,KAAK,eAAiB,EAAA,aAAa,KAAK,QAAS,CAAE,mBAAoB,EAAK,CAAC,EAC7E,KAAK,eAAe,sBAAwB,GAC5C,KAAK,eAAe,sBAAwB,QAEzC,KAAK,cAChB,CAGA,cAAsB,EAAe,EAA2B,CAS5D,OAPI,KAAK,UAAY,MACjB,KAAK,QAAQ,QAAU,GACvB,KAAK,QAAQ,SAAW,KAExB,KAAK,QAAU,EAAA,aAAa,EAAO,CAAM,EACzC,KAAK,eAAiB,EAAA,aAAa,KAAK,OAAO,GAE5C,KAAK,cAChB,CACJ,EAcA,SAAgB,EACZ,EACA,EACA,EACA,EAA0C,CAAC,EAAG,EAAG,CAAC,EAClD,EAAyC,CAAC,EAAG,EAAG,CAAC,EAChC,CACjB,OAAO,IAAI,EAAe,EAAa,EAAc,EAAM,CAAG,CAAC,CAAC,IAAI,CAAK,CAC7E,CAQA,SAAgB,EAAe,EAAe,EAA8B,CACxE,OAAO,IAAI,aAAa,EAAI,EAAS,CAAK,CAC9C,CAaA,SAAgB,EACZ,EACA,EACA,EACA,EAA0C,CAAC,IAAK,IAAK,GAAG,EACpC,CACpB,OAAO,IAAI,EAAkB,EAAa,EAAc,CAAI,CAAC,CAAC,IAAI,CAAK,CAC3E"}
|
|
1
|
+
{"version":3,"file":"pipeline.cjs","names":[],"sources":["../../../src/vision/preprocess/pipeline.ts"],"sourcesContent":["/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */\n/**\n * Fused letterbox → CHW float32 pipeline with reusable buffers.\n *\n * The composable primitives in {@link ./image.js} each allocate and each walk\n * their input end to end, which is the right shape for a library but the wrong\n * shape for a video loop. Chaining them costs eleven full-buffer passes and six\n * large allocations per frame:\n *\n * `getImageData` → RGBA→RGB → RGB→RGBA → `putImageData` → `drawImage` →\n * `getImageData` → RGBA→RGB → fill → row copies → `toFloat32` → `toCHW`.\n *\n * This module collapses the second half of that into two: one `drawImage` that\n * resizes *and* positions the content inside the padded target in a single\n * accelerated operation, and one loop that reads the resulting RGBA and writes\n * planar float32 directly. The intermediate `RGBImage` at target size, the fill\n * loop, the row copies and the two 4.9 MB `Float32Array` allocations all go\n * away.\n *\n * The primitives stay exactly as they are — they are public API and they are\n * what makes a custom pipeline writable. This is the fast path the built-in\n * tasks take.\n */\n\nimport {\n createCanvas,\n get2DContext,\n rgbToImageData,\n type Canvas2D,\n type Context2D,\n} from \"../core/canvas\";\nimport type { RGBImage } from \"../types\";\n\nconst INV_255 = 1 / 255;\n\n/** One claim on a {@link ReusableBuffer}. */\ninterface BufferClaim {\n /** The buffer to write into, always `size` long. */\n readonly data: Float32Array;\n /** Whether {@link data} is the held buffer rather than a fresh allocation. */\n readonly reused: boolean;\n}\n\n/**\n * A `Float32Array` held across calls, handed out one claim at a time.\n *\n * Both pipelines want the same thing — allocate once, write into it every frame,\n * and fall back to a fresh array when a previous result has not been released\n * yet — so the bookkeeping lives here instead of twice.\n *\n * The part that is not obvious is {@link claim} re-allocating a buffer that is\n * *detached*. `ort.env.wasm.proxy` runs ONNX Runtime in a worker and posts the\n * input tensors with their `ArrayBuffer`s in the transfer list, which detaches\n * them on this side. A detached `Float32Array` is silently 0 long: writing to it\n * is a no-op, and the next `InferenceSession.run` rejects with\n * `Tensor's size(N) does not match data length(0)` on every other call — once for\n * the detached buffer, then the throw leaves the claim outstanding so the call\n * after it allocates and succeeds. Treating a buffer that changed length as spent\n * turns that into one extra allocation per transfer, which is what reuse was\n * avoiding, and keeps it correct for any consumer that transfers the tensor\n * rather than copying it.\n */\nclass ReusableBuffer {\n private readonly _size: number;\n private _buffer: Float32Array;\n private _inUse = false;\n\n /** @param size Length in floats of every buffer this hands out. */\n constructor(size: number) {\n this._size = size;\n this._buffer = new Float32Array(size);\n }\n\n /**\n * Take the held buffer, or a fresh one when it is unavailable.\n *\n * Unavailable means either still checked out by an unreleased claim, or\n * detached by whoever it was handed to. The first case allocates for this call\n * only; the second replaces the held buffer, so the allocation is paid once per\n * transfer rather than on every call after it.\n */\n claim(): BufferClaim {\n if (this._inUse) return { data: new Float32Array(this._size), reused: false };\n if (this._buffer.length !== this._size) this._buffer = new Float32Array(this._size);\n this._inUse = true;\n return { data: this._buffer, reused: true };\n }\n\n /** Mark the held buffer free for the next {@link claim}. */\n release(): void {\n this._inUse = false;\n }\n}\n\n/** Geometry of a letterbox, plus the planar tensor data it produced. */\nexport interface FusedLetterboxResult {\n /** CHW float32 in `[0, 1]`, length `3 * targetHeight * targetWidth`. */\n readonly data: Float32Array;\n /** Factor applied to the original image (`< 1` if downscaled). */\n readonly scale: number;\n /** Horizontal padding in pixels. */\n readonly padLeft: number;\n /** Vertical padding in pixels. */\n readonly padTop: number;\n /**\n * Whether {@link data} is the pipeline's reusable buffer.\n *\n * `true` means the next {@link LetterboxPipeline.run} overwrites it, so a\n * caller keeping the values past its own inference has to copy them.\n */\n readonly reused: boolean;\n}\n\n/**\n * Reusable letterbox → tensor pipeline for one target resolution.\n *\n * Holds a target canvas and an output buffer across calls, so a steady stream\n * of frames at the same size allocates nothing. Create one per task, not per\n * frame.\n */\nexport class LetterboxPipeline {\n private readonly _targetWidth: number;\n private readonly _targetHeight: number;\n private readonly _fill: readonly [number, number, number];\n private readonly _target: Canvas2D;\n private readonly _targetContext: Context2D;\n private readonly _buffer: ReusableBuffer;\n private _source: Canvas2D | null = null;\n private _sourceContext: Context2D | null = null;\n\n /**\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param fill RGB padding colour; defaults to YOLO grey.\n */\n constructor(\n targetWidth: number,\n targetHeight: number,\n fill: readonly [number, number, number] = [114, 114, 114],\n ) {\n if (targetWidth <= 0 || targetHeight <= 0) {\n throw new Error(`Invalid letterbox target ${targetWidth}x${targetHeight}.`);\n }\n this._targetWidth = targetWidth;\n this._targetHeight = targetHeight;\n this._fill = fill;\n this._target = createCanvas(targetWidth, targetHeight);\n this._targetContext = get2DContext(this._target, { willReadFrequently: true });\n this._targetContext.imageSmoothingEnabled = true;\n this._targetContext.imageSmoothingQuality = \"high\";\n this._buffer = new ReusableBuffer(3 * targetHeight * targetWidth);\n }\n\n /** The `[width, height]` this pipeline letterboxes into. */\n get targetSize(): readonly [number, number] {\n return [this._targetWidth, this._targetHeight];\n }\n\n /**\n * Letterbox an image and write it as planar float32.\n *\n * The returned buffer is reused between calls unless a previous result is\n * still checked out — {@link release} marks it free again. A second `run`\n * before the first is released allocates a fresh buffer rather than\n * corrupting it, so concurrent `predict()` calls on one task stay correct at\n * the cost of the allocation they were trying to avoid. A buffer that was\n * detached by a consumer that transferred it is replaced rather than written\n * into — see {@link ReusableBuffer}.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param canvas Optional drawable already holding exactly `image`'s pixels at\n * `image.width × image.height`, fully opaque. When given it is drawn\n * straight into the target, skipping the RGB→RGBA rebuild and the\n * `putImageData` onto the scratch canvas — two full-resolution passes per\n * frame. The tasks pass the canvas `loadImage` decoded onto; a caller with a\n * translucent source must not, since the result would composite its alpha.\n */\n run(image: RGBImage, canvas: CanvasImageSource | null = null): FusedLetterboxResult {\n const targetWidth = this._targetWidth;\n const targetHeight = this._targetHeight;\n const scale = Math.min(targetWidth / image.width, targetHeight / image.height);\n const scaledWidth = Math.round(image.width * scale);\n const scaledHeight = Math.round(image.height * scale);\n const padLeft = Math.floor((targetWidth - scaledWidth) / 2);\n const padTop = Math.floor((targetHeight - scaledHeight) / 2);\n\n const drawable = canvas ?? this._paintSource(image);\n\n const context = this._targetContext;\n if (\n padLeft > 0 ||\n padTop > 0 ||\n scaledWidth !== targetWidth ||\n scaledHeight !== targetHeight\n ) {\n context.fillStyle = `rgb(${this._fill[0]},${this._fill[1]},${this._fill[2]})`;\n context.fillRect(0, 0, targetWidth, targetHeight);\n }\n context.drawImage(\n drawable,\n 0,\n 0,\n image.width,\n image.height,\n padLeft,\n padTop,\n scaledWidth,\n scaledHeight,\n );\n\n const rgba = context.getImageData(0, 0, targetWidth, targetHeight).data;\n const { data, reused } = this._buffer.claim();\n\n const plane = targetWidth * targetHeight;\n for (let pixel = 0, offset = 0; pixel < plane; pixel++, offset += 4) {\n data[pixel] = (rgba[offset] as number) * INV_255;\n data[plane + pixel] = (rgba[offset + 1] as number) * INV_255;\n data[2 * plane + pixel] = (rgba[offset + 2] as number) * INV_255;\n }\n\n return { data, scale, padLeft, padTop, reused };\n }\n\n /**\n * Mark the reusable buffer free again.\n *\n * Call it once the tensor built from a {@link run} result has been handed to\n * ONNX Runtime and the run has resolved — after that the values are inside\n * the WASM heap and the buffer can be overwritten.\n *\n * Pass the result being released when several runs can be outstanding at\n * once. A result that got a fresh allocation (`reused: false`) holds nothing\n * of the pipeline's, so releasing it is a no-op; without that check, the\n * caller finishing with a fresh buffer would free the shared one while\n * another caller still had its frame in it. Called without an argument it\n * frees the shared buffer unconditionally, as before.\n *\n * @param result The {@link run} result being released.\n */\n release(result?: { readonly reused: boolean }): void {\n if (result !== undefined && !result.reused) return;\n this._buffer.release();\n }\n\n /**\n * Paint an image onto the scratch source canvas and return that canvas.\n *\n * The canvas is only reallocated when a frame arrives at a different size\n * than the last one, which for a camera or video source is never after the\n * first.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n */\n private _paintSource(image: RGBImage): CanvasImageSource {\n const { width, height } = image;\n if (\n this._source === null ||\n this._source.width !== width ||\n this._source.height !== height\n ) {\n this._source = createCanvas(width, height);\n this._sourceContext = get2DContext(this._source);\n }\n (this._sourceContext as Context2D).putImageData(rgbToImageData(image), 0, 0);\n return this._source as CanvasImageSource;\n }\n}\n\n/**\n * Write an RGBA buffer as normalized planar float32.\n *\n * Shared by {@link ResizePipeline} and exported for a custom pipeline that\n * already holds pixels and wants the SDK's exact arithmetic.\n *\n * The expression is `(value / 255 - mean) / std`, evaluated in that order on\n * purpose: folding it into a single multiply-add (`value * k + b`) computes the\n * same quantity but rounds differently, and the tasks' output is asserted to be\n * bit-identical to the composable `normalize` → `toCHW` path.\n *\n * @param rgba Source pixels, 4 bytes per pixel, `width * height` long.\n * @param width Image width in pixels.\n * @param height Image height in pixels.\n * @param mean Per-channel RGB mean, already in `[0, 1]`.\n * @param std Per-channel RGB standard deviation.\n * @param out Destination buffer, `3 * width * height` long.\n * @param stride Bytes per source pixel: 4 for canvas RGBA, 3 for packed RGB.\n */\nexport function writePlanarFloat32(\n rgba: Uint8ClampedArray | Uint8Array,\n width: number,\n height: number,\n mean: readonly [number, number, number],\n std: readonly [number, number, number],\n out: Float32Array,\n stride: number = 4,\n): void {\n const plane = width * height;\n const m0 = mean[0];\n const m1 = mean[1];\n const m2 = mean[2];\n const s0 = std[0];\n const s1 = std[1];\n const s2 = std[2];\n for (let pixel = 0, offset = 0; pixel < plane; pixel++, offset += stride) {\n out[pixel] = ((rgba[offset] as number) * INV_255 - m0) / s0;\n out[plane + pixel] = ((rgba[offset + 1] as number) * INV_255 - m1) / s1;\n out[2 * plane + pixel] = ((rgba[offset + 2] as number) * INV_255 - m2) / s2;\n }\n}\n\n/** Planar tensor data produced by {@link ResizePipeline}. */\nexport interface FusedResizeResult {\n /** CHW float32, normalized, length `3 * targetHeight * targetWidth`. */\n readonly data: Float32Array;\n /**\n * Whether {@link data} is the pipeline's reusable buffer.\n *\n * `true` means the next {@link ResizePipeline.run} overwrites it, so a caller\n * keeping the values past its own inference has to copy them.\n */\n readonly reused: boolean;\n}\n\n/**\n * Reusable stretch-resize → normalized tensor pipeline for one target size.\n *\n * The classification counterpart of {@link LetterboxPipeline}. A classifier\n * stretches to the model's square input instead of letterboxing into it — no\n * padding, no scale to invert later, because nothing is mapped back onto the\n * source image afterwards. That difference is why it cannot simply reuse the\n * letterbox path.\n *\n * What it does share is the technique. The composable route\n * (`resize` → `normalize` → `toCHW`) allocates an `RGBImage` and two\n * `Float32Array`s and walks each end to end on every call: about 1.4 MB of\n * fresh garbage per 224×224 `predict()`, produced at the exact moment a phone\n * near its memory ceiling can least afford it. Here one `drawImage` resizes,\n * and one loop reads the resulting RGBA and writes normalized planar float32\n * into a buffer held across calls.\n *\n * Create one per task, not per frame.\n */\nexport class ResizePipeline {\n private readonly _targetWidth: number;\n private readonly _targetHeight: number;\n private readonly _mean: readonly [number, number, number];\n private readonly _std: readonly [number, number, number];\n private readonly _buffer: ReusableBuffer;\n private _target: Canvas2D | null = null;\n private _targetContext: Context2D | null = null;\n private _source: Canvas2D | null = null;\n private _sourceContext: Context2D | null = null;\n\n /**\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.\n * @param std Per-channel RGB standard deviation. Defaults to no scaling.\n */\n constructor(\n targetWidth: number,\n targetHeight: number,\n mean: readonly [number, number, number] = [0, 0, 0],\n std: readonly [number, number, number] = [1, 1, 1],\n ) {\n if (targetWidth <= 0 || targetHeight <= 0) {\n throw new Error(`Invalid resize target ${targetWidth}x${targetHeight}.`);\n }\n this._targetWidth = targetWidth;\n this._targetHeight = targetHeight;\n this._mean = mean;\n this._std = std;\n this._buffer = new ReusableBuffer(3 * targetHeight * targetWidth);\n }\n\n /** The `[width, height]` this pipeline resizes into. */\n get targetSize(): readonly [number, number] {\n return [this._targetWidth, this._targetHeight];\n }\n\n /**\n * Resize an image to the target size and write it as normalized planar float32.\n *\n * An image that already arrives at the target size skips the canvas entirely\n * and is read straight out of its packed RGB — which is both faster and what\n * keeps the result identical to `resize()`, whose own fast path returns the\n * input untouched.\n *\n * Buffer reuse follows {@link ReusableBuffer}: held across calls, replaced when\n * a consumer detached it by transferring the tensor.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param canvas Optional opaque drawable holding exactly `image`'s pixels —\n * see {@link LetterboxPipeline.run}. Ignored on the same-size fast path,\n * which never touches a canvas.\n */\n run(image: RGBImage, canvas: CanvasImageSource | null = null): FusedResizeResult {\n const targetWidth = this._targetWidth;\n const targetHeight = this._targetHeight;\n const { data, reused } = this._buffer.claim();\n\n if (image.width === targetWidth && image.height === targetHeight) {\n writePlanarFloat32(\n image.data,\n targetWidth,\n targetHeight,\n this._mean,\n this._std,\n data,\n 3,\n );\n return { data, reused };\n }\n\n const drawable = canvas ?? this._paintSource(image);\n const context = this._ensureTarget();\n context.drawImage(\n drawable,\n 0,\n 0,\n image.width,\n image.height,\n 0,\n 0,\n targetWidth,\n targetHeight,\n );\n\n const rgba = context.getImageData(0, 0, targetWidth, targetHeight).data;\n writePlanarFloat32(rgba, targetWidth, targetHeight, this._mean, this._std, data);\n return { data, reused };\n }\n\n /**\n * Mark the reusable buffer free again.\n *\n * Call it once the tensor built from a {@link run} result has been handed to\n * ONNX Runtime and the run has resolved — after that the values are inside\n * the WASM heap and the buffer can be overwritten.\n *\n * Pass the result being released when several runs can be outstanding at\n * once. A result that got a fresh allocation (`reused: false`) holds nothing\n * of the pipeline's, so releasing it is a no-op; without that check, the\n * caller finishing with a fresh buffer would free the shared one while\n * another caller still had its frame in it. Called without an argument it\n * frees the shared buffer unconditionally, as before.\n *\n * @param result The {@link run} result being released.\n */\n release(result?: { readonly reused: boolean }): void {\n if (result !== undefined && !result.reused) return;\n this._buffer.release();\n }\n\n /**\n * Build the target canvas on first use.\n *\n * Lazily, so a pipeline constructed where no canvas implementation exists\n * (a Node test, a worker without OffscreenCanvas) only fails if it is asked\n * to resize something.\n */\n private _ensureTarget(): Context2D {\n if (this._target === null) {\n this._target = createCanvas(this._targetWidth, this._targetHeight);\n this._targetContext = get2DContext(this._target, { willReadFrequently: true });\n this._targetContext.imageSmoothingEnabled = true;\n this._targetContext.imageSmoothingQuality = \"high\";\n }\n return this._targetContext as Context2D;\n }\n\n /**\n * Paint an image onto the scratch source canvas and return that canvas.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n */\n private _paintSource(image: RGBImage): CanvasImageSource {\n const { width, height } = image;\n if (\n this._source === null ||\n this._source.width !== width ||\n this._source.height !== height\n ) {\n this._source = createCanvas(width, height);\n this._sourceContext = get2DContext(this._source);\n }\n (this._sourceContext as Context2D).putImageData(rgbToImageData(image), 0, 0);\n return this._source as CanvasImageSource;\n }\n}\n\n/**\n * Resize an image into normalized planar float32 without keeping any state.\n *\n * The allocation-free path is {@link ResizePipeline}; this is the one-shot\n * form, for a caller who wants the fused behaviour without owning a pipeline.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.\n * @param std Per-channel RGB standard deviation. Defaults to no scaling.\n */\nexport function resizeToTensorData(\n image: RGBImage,\n targetWidth: number,\n targetHeight: number,\n mean: readonly [number, number, number] = [0, 0, 0],\n std: readonly [number, number, number] = [1, 1, 1],\n): FusedResizeResult {\n return new ResizePipeline(targetWidth, targetHeight, mean, std).run(image);\n}\n\n/**\n * Build a zero-filled CHW tensor payload for a warm-up run.\n *\n * @param width Model input width in pixels.\n * @param height Model input height in pixels.\n */\nexport function zeroTensorData(width: number, height: number): Float32Array {\n return new Float32Array(3 * height * width);\n}\n\n/**\n * Letterbox an image into planar float32 without keeping any state.\n *\n * The allocation-free path is {@link LetterboxPipeline}; this is the one-shot\n * form, for a caller who wants the fused behaviour without owning a pipeline.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param fill RGB padding colour; defaults to YOLO grey.\n */\nexport function letterboxToTensorData(\n image: RGBImage,\n targetWidth: number,\n targetHeight: number,\n fill: readonly [number, number, number] = [114, 114, 114],\n): FusedLetterboxResult {\n return new LetterboxPipeline(targetWidth, targetHeight, fill).run(image);\n}\n"],"mappings":"sCAiCA,IAAM,EAAU,EAAI,IA6Bd,EAAN,KAAqB,CACjB,MACA,QACA,OAAiB,GAGjB,YAAY,EAAc,CACtB,KAAK,MAAQ,EACb,KAAK,QAAU,IAAI,aAAa,CAAI,CACxC,CAUA,OAAqB,CAIjB,OAHI,KAAK,OAAe,CAAE,KAAM,IAAI,aAAa,KAAK,KAAK,EAAG,OAAQ,EAAM,GACxE,KAAK,QAAQ,SAAW,KAAK,QAAO,KAAK,QAAU,IAAI,aAAa,KAAK,KAAK,GAClF,KAAK,OAAS,GACP,CAAE,KAAM,KAAK,QAAS,OAAQ,EAAK,EAC9C,CAGA,SAAgB,CACZ,KAAK,OAAS,EAClB,CACJ,EA4Ba,EAAb,KAA+B,CAC3B,aACA,cACA,MACA,QACA,eACA,QACA,QAAmC,KACnC,eAA2C,KAO3C,YACI,EACA,EACA,EAA0C,CAAC,IAAK,IAAK,GAAG,EAC1D,CACE,GAAI,GAAe,GAAK,GAAgB,EACpC,MAAU,MAAM,4BAA4B,EAAY,GAAG,EAAa,EAAE,EAE9E,KAAK,aAAe,EACpB,KAAK,cAAgB,EACrB,KAAK,MAAQ,EACb,KAAK,QAAU,EAAA,aAAa,EAAa,CAAY,EACrD,KAAK,eAAiB,EAAA,aAAa,KAAK,QAAS,CAAE,mBAAoB,EAAK,CAAC,EAC7E,KAAK,eAAe,sBAAwB,GAC5C,KAAK,eAAe,sBAAwB,OAC5C,KAAK,QAAU,IAAI,EAAe,EAAI,EAAe,CAAW,CACpE,CAGA,IAAI,YAAwC,CACxC,MAAO,CAAC,KAAK,aAAc,KAAK,aAAa,CACjD,CAqBA,IAAI,EAAiB,EAAmC,KAA4B,CAChF,IAAM,EAAc,KAAK,aACnB,EAAe,KAAK,cACpB,EAAQ,KAAK,IAAI,EAAc,EAAM,MAAO,EAAe,EAAM,MAAM,EACvE,EAAc,KAAK,MAAM,EAAM,MAAQ,CAAK,EAC5C,EAAe,KAAK,MAAM,EAAM,OAAS,CAAK,EAC9C,EAAU,KAAK,OAAO,EAAc,GAAe,CAAC,EACpD,EAAS,KAAK,OAAO,EAAe,GAAgB,CAAC,EAErD,EAAW,GAAU,KAAK,aAAa,CAAK,EAE5C,EAAU,KAAK,gBAEjB,EAAU,GACV,EAAS,GACT,IAAgB,GAChB,IAAiB,KAEjB,EAAQ,UAAY,OAAO,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAC3E,EAAQ,SAAS,EAAG,EAAG,EAAa,CAAY,GAEpD,EAAQ,UACJ,EACA,EACA,EACA,EAAM,MACN,EAAM,OACN,EACA,EACA,EACA,CACJ,EAEA,IAAM,EAAO,EAAQ,aAAa,EAAG,EAAG,EAAa,CAAY,CAAC,CAAC,KAC7D,CAAE,OAAM,UAAW,KAAK,QAAQ,MAAM,EAEtC,EAAQ,EAAc,EAC5B,IAAK,IAAI,EAAQ,EAAG,EAAS,EAAG,EAAQ,EAAO,IAAS,GAAU,EAC9D,EAAK,GAAU,EAAK,GAAqB,EACzC,EAAK,EAAQ,GAAU,EAAK,EAAS,GAAgB,EACrD,EAAK,EAAI,EAAQ,GAAU,EAAK,EAAS,GAAgB,EAG7D,MAAO,CAAE,OAAM,QAAO,UAAS,SAAQ,QAAO,CAClD,CAkBA,QAAQ,EAA6C,EAC7C,IAAW,IAAA,IAAc,EAAO,SACpC,KAAK,QAAQ,QAAQ,CACzB,CAWA,aAAqB,EAAoC,CACrD,GAAM,CAAE,QAAO,UAAW,EAU1B,OARI,KAAK,UAAY,MACjB,KAAK,QAAQ,QAAU,GACvB,KAAK,QAAQ,SAAW,KAExB,KAAK,QAAU,EAAA,aAAa,EAAO,CAAM,EACzC,KAAK,eAAiB,EAAA,aAAa,KAAK,OAAO,GAEnD,KAAM,eAA6B,aAAa,EAAA,eAAe,CAAK,EAAG,EAAG,CAAC,EACpE,KAAK,OAChB,CACJ,EAqBA,SAAgB,EACZ,EACA,EACA,EACA,EACA,EACA,EACA,EAAiB,EACb,CACJ,IAAM,EAAQ,EAAQ,EAChB,EAAK,EAAK,GACV,EAAK,EAAK,GACV,EAAK,EAAK,GACV,EAAK,EAAI,GACT,EAAK,EAAI,GACT,EAAK,EAAI,GACf,IAAK,IAAI,EAAQ,EAAG,EAAS,EAAG,EAAQ,EAAO,IAAS,GAAU,EAC9D,EAAI,IAAW,EAAK,GAAqB,EAAU,GAAM,EACzD,EAAI,EAAQ,IAAW,EAAK,EAAS,GAAgB,EAAU,GAAM,EACrE,EAAI,EAAI,EAAQ,IAAW,EAAK,EAAS,GAAgB,EAAU,GAAM,CAEjF,CAkCA,IAAa,EAAb,KAA4B,CACxB,aACA,cACA,MACA,KACA,QACA,QAAmC,KACnC,eAA2C,KAC3C,QAAmC,KACnC,eAA2C,KAQ3C,YACI,EACA,EACA,EAA0C,CAAC,EAAG,EAAG,CAAC,EAClD,EAAyC,CAAC,EAAG,EAAG,CAAC,EACnD,CACE,GAAI,GAAe,GAAK,GAAgB,EACpC,MAAU,MAAM,yBAAyB,EAAY,GAAG,EAAa,EAAE,EAE3E,KAAK,aAAe,EACpB,KAAK,cAAgB,EACrB,KAAK,MAAQ,EACb,KAAK,KAAO,EACZ,KAAK,QAAU,IAAI,EAAe,EAAI,EAAe,CAAW,CACpE,CAGA,IAAI,YAAwC,CACxC,MAAO,CAAC,KAAK,aAAc,KAAK,aAAa,CACjD,CAkBA,IAAI,EAAiB,EAAmC,KAAyB,CAC7E,IAAM,EAAc,KAAK,aACnB,EAAe,KAAK,cACpB,CAAE,OAAM,UAAW,KAAK,QAAQ,MAAM,EAE5C,GAAI,EAAM,QAAU,GAAe,EAAM,SAAW,EAUhD,OATA,EACI,EAAM,KACN,EACA,EACA,KAAK,MACL,KAAK,KACL,EACA,CACJ,EACO,CAAE,OAAM,QAAO,EAG1B,IAAM,EAAW,GAAU,KAAK,aAAa,CAAK,EAC5C,EAAU,KAAK,cAAc,EACnC,EAAQ,UACJ,EACA,EACA,EACA,EAAM,MACN,EAAM,OACN,EACA,EACA,EACA,CACJ,EAEA,IAAM,EAAO,EAAQ,aAAa,EAAG,EAAG,EAAa,CAAY,CAAC,CAAC,KAEnE,OADA,EAAmB,EAAM,EAAa,EAAc,KAAK,MAAO,KAAK,KAAM,CAAI,EACxE,CAAE,OAAM,QAAO,CAC1B,CAkBA,QAAQ,EAA6C,EAC7C,IAAW,IAAA,IAAc,EAAO,SACpC,KAAK,QAAQ,QAAQ,CACzB,CASA,eAAmC,CAO/B,OANI,KAAK,UAAY,OACjB,KAAK,QAAU,EAAA,aAAa,KAAK,aAAc,KAAK,aAAa,EACjE,KAAK,eAAiB,EAAA,aAAa,KAAK,QAAS,CAAE,mBAAoB,EAAK,CAAC,EAC7E,KAAK,eAAe,sBAAwB,GAC5C,KAAK,eAAe,sBAAwB,QAEzC,KAAK,cAChB,CAOA,aAAqB,EAAoC,CACrD,GAAM,CAAE,QAAO,UAAW,EAU1B,OARI,KAAK,UAAY,MACjB,KAAK,QAAQ,QAAU,GACvB,KAAK,QAAQ,SAAW,KAExB,KAAK,QAAU,EAAA,aAAa,EAAO,CAAM,EACzC,KAAK,eAAiB,EAAA,aAAa,KAAK,OAAO,GAEnD,KAAM,eAA6B,aAAa,EAAA,eAAe,CAAK,EAAG,EAAG,CAAC,EACpE,KAAK,OAChB,CACJ,EAcA,SAAgB,EACZ,EACA,EACA,EACA,EAA0C,CAAC,EAAG,EAAG,CAAC,EAClD,EAAyC,CAAC,EAAG,EAAG,CAAC,EAChC,CACjB,OAAO,IAAI,EAAe,EAAa,EAAc,EAAM,CAAG,CAAC,CAAC,IAAI,CAAK,CAC7E,CAQA,SAAgB,EAAe,EAAe,EAA8B,CACxE,OAAO,IAAI,aAAa,EAAI,EAAS,CAAK,CAC9C,CAaA,SAAgB,EACZ,EACA,EACA,EACA,EAA0C,CAAC,IAAK,IAAK,GAAG,EACpC,CACpB,OAAO,IAAI,EAAkB,EAAa,EAAc,CAAI,CAAC,CAAC,IAAI,CAAK,CAC3E"}
|
|
@@ -39,26 +39,25 @@ var r = 1 / 255, i = class {
|
|
|
39
39
|
get targetSize() {
|
|
40
40
|
return [this._targetWidth, this._targetHeight];
|
|
41
41
|
}
|
|
42
|
-
run(e) {
|
|
43
|
-
let
|
|
44
|
-
this.
|
|
45
|
-
let
|
|
46
|
-
(
|
|
47
|
-
let d = u.getImageData(0, 0, t, i).data, { data: f, reused: p } = this._buffer.claim(), m = t * i;
|
|
48
|
-
for (let e = 0, t = 0; e < m; e++, t += 4) f[e] = d[t] * r, f[m + e] = d[t + 1] * r, f[2 * m + e] = d[t + 2] * r;
|
|
42
|
+
run(e, t = null) {
|
|
43
|
+
let n = this._targetWidth, i = this._targetHeight, a = Math.min(n / e.width, i / e.height), o = Math.round(e.width * a), s = Math.round(e.height * a), c = Math.floor((n - o) / 2), l = Math.floor((i - s) / 2), u = t ?? this._paintSource(e), d = this._targetContext;
|
|
44
|
+
(c > 0 || l > 0 || o !== n || s !== i) && (d.fillStyle = `rgb(${this._fill[0]},${this._fill[1]},${this._fill[2]})`, d.fillRect(0, 0, n, i)), d.drawImage(u, 0, 0, e.width, e.height, c, l, o, s);
|
|
45
|
+
let f = d.getImageData(0, 0, n, i).data, { data: p, reused: m } = this._buffer.claim(), h = n * i;
|
|
46
|
+
for (let e = 0, t = 0; e < h; e++, t += 4) p[e] = f[t] * r, p[h + e] = f[t + 1] * r, p[2 * h + e] = f[t + 2] * r;
|
|
49
47
|
return {
|
|
50
|
-
data:
|
|
48
|
+
data: p,
|
|
51
49
|
scale: a,
|
|
52
50
|
padLeft: c,
|
|
53
51
|
padTop: l,
|
|
54
|
-
reused:
|
|
52
|
+
reused: m
|
|
55
53
|
};
|
|
56
54
|
}
|
|
57
|
-
release() {
|
|
58
|
-
this._buffer.release();
|
|
55
|
+
release(e) {
|
|
56
|
+
(e === void 0 || e.reused) && this._buffer.release();
|
|
59
57
|
}
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
_paintSource(r) {
|
|
59
|
+
let { width: i, height: a } = r;
|
|
60
|
+
return (this._source === null || this._source.width !== i || this._source.height !== a) && (this._source = e(i, a), this._sourceContext = t(this._source)), this._sourceContext.putImageData(n(r), 0, 0), this._source;
|
|
62
61
|
}
|
|
63
62
|
};
|
|
64
63
|
function o(e, t, n, i, a, o, s = 4) {
|
|
@@ -90,29 +89,29 @@ var s = class {
|
|
|
90
89
|
get targetSize() {
|
|
91
90
|
return [this._targetWidth, this._targetHeight];
|
|
92
91
|
}
|
|
93
|
-
run(e) {
|
|
94
|
-
let
|
|
95
|
-
if (e.width ===
|
|
92
|
+
run(e, t = null) {
|
|
93
|
+
let n = this._targetWidth, r = this._targetHeight, { data: i, reused: a } = this._buffer.claim();
|
|
94
|
+
if (e.width === n && e.height === r) return o(e.data, n, r, this._mean, this._std, i, 3), {
|
|
96
95
|
data: i,
|
|
97
96
|
reused: a
|
|
98
97
|
};
|
|
99
|
-
this.
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
return o(c, t, r, this._mean, this._std, i), {
|
|
98
|
+
let s = t ?? this._paintSource(e), c = this._ensureTarget();
|
|
99
|
+
c.drawImage(s, 0, 0, e.width, e.height, 0, 0, n, r);
|
|
100
|
+
let l = c.getImageData(0, 0, n, r).data;
|
|
101
|
+
return o(l, n, r, this._mean, this._std, i), {
|
|
104
102
|
data: i,
|
|
105
103
|
reused: a
|
|
106
104
|
};
|
|
107
105
|
}
|
|
108
|
-
release() {
|
|
109
|
-
this._buffer.release();
|
|
106
|
+
release(e) {
|
|
107
|
+
(e === void 0 || e.reused) && this._buffer.release();
|
|
110
108
|
}
|
|
111
109
|
_ensureTarget() {
|
|
112
110
|
return this._target === null && (this._target = e(this._targetWidth, this._targetHeight), this._targetContext = t(this._target, { willReadFrequently: !0 }), this._targetContext.imageSmoothingEnabled = !0, this._targetContext.imageSmoothingQuality = "high"), this._targetContext;
|
|
113
111
|
}
|
|
114
|
-
|
|
115
|
-
|
|
112
|
+
_paintSource(r) {
|
|
113
|
+
let { width: i, height: a } = r;
|
|
114
|
+
return (this._source === null || this._source.width !== i || this._source.height !== a) && (this._source = e(i, a), this._sourceContext = t(this._source)), this._sourceContext.putImageData(n(r), 0, 0), this._source;
|
|
116
115
|
}
|
|
117
116
|
};
|
|
118
117
|
function c(e, t, n, r = [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.js","names":[],"sources":["../../../src/vision/preprocess/pipeline.ts"],"sourcesContent":["/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */\n/**\n * Fused letterbox → CHW float32 pipeline with reusable buffers.\n *\n * The composable primitives in {@link ./image.js} each allocate and each walk\n * their input end to end, which is the right shape for a library but the wrong\n * shape for a video loop. Chaining them costs eleven full-buffer passes and six\n * large allocations per frame:\n *\n * `getImageData` → RGBA→RGB → RGB→RGBA → `putImageData` → `drawImage` →\n * `getImageData` → RGBA→RGB → fill → row copies → `toFloat32` → `toCHW`.\n *\n * This module collapses the second half of that into two: one `drawImage` that\n * resizes *and* positions the content inside the padded target in a single\n * accelerated operation, and one loop that reads the resulting RGBA and writes\n * planar float32 directly. The intermediate `RGBImage` at target size, the fill\n * loop, the row copies and the two 4.9 MB `Float32Array` allocations all go\n * away.\n *\n * The primitives stay exactly as they are — they are public API and they are\n * what makes a custom pipeline writable. This is the fast path the built-in\n * tasks take.\n */\n\nimport {\n createCanvas,\n get2DContext,\n rgbToImageData,\n type Canvas2D,\n type Context2D,\n} from \"../core/canvas\";\nimport type { RGBImage } from \"../types\";\n\nconst INV_255 = 1 / 255;\n\n/** One claim on a {@link ReusableBuffer}. */\ninterface BufferClaim {\n /** The buffer to write into, always `size` long. */\n readonly data: Float32Array;\n /** Whether {@link data} is the held buffer rather than a fresh allocation. */\n readonly reused: boolean;\n}\n\n/**\n * A `Float32Array` held across calls, handed out one claim at a time.\n *\n * Both pipelines want the same thing — allocate once, write into it every frame,\n * and fall back to a fresh array when a previous result has not been released\n * yet — so the bookkeeping lives here instead of twice.\n *\n * The part that is not obvious is {@link claim} re-allocating a buffer that is\n * *detached*. `ort.env.wasm.proxy` runs ONNX Runtime in a worker and posts the\n * input tensors with their `ArrayBuffer`s in the transfer list, which detaches\n * them on this side. A detached `Float32Array` is silently 0 long: writing to it\n * is a no-op, and the next `InferenceSession.run` rejects with\n * `Tensor's size(N) does not match data length(0)` on every other call — once for\n * the detached buffer, then the throw leaves the claim outstanding so the call\n * after it allocates and succeeds. Treating a buffer that changed length as spent\n * turns that into one extra allocation per transfer, which is what reuse was\n * avoiding, and keeps it correct for any consumer that transfers the tensor\n * rather than copying it.\n */\nclass ReusableBuffer {\n private readonly _size: number;\n private _buffer: Float32Array;\n private _inUse = false;\n\n /** @param size Length in floats of every buffer this hands out. */\n constructor(size: number) {\n this._size = size;\n this._buffer = new Float32Array(size);\n }\n\n /**\n * Take the held buffer, or a fresh one when it is unavailable.\n *\n * Unavailable means either still checked out by an unreleased claim, or\n * detached by whoever it was handed to. The first case allocates for this call\n * only; the second replaces the held buffer, so the allocation is paid once per\n * transfer rather than on every call after it.\n */\n claim(): BufferClaim {\n if (this._inUse) return { data: new Float32Array(this._size), reused: false };\n if (this._buffer.length !== this._size) this._buffer = new Float32Array(this._size);\n this._inUse = true;\n return { data: this._buffer, reused: true };\n }\n\n /** Mark the held buffer free for the next {@link claim}. */\n release(): void {\n this._inUse = false;\n }\n}\n\n/** Geometry of a letterbox, plus the planar tensor data it produced. */\nexport interface FusedLetterboxResult {\n /** CHW float32 in `[0, 1]`, length `3 * targetHeight * targetWidth`. */\n readonly data: Float32Array;\n /** Factor applied to the original image (`< 1` if downscaled). */\n readonly scale: number;\n /** Horizontal padding in pixels. */\n readonly padLeft: number;\n /** Vertical padding in pixels. */\n readonly padTop: number;\n /**\n * Whether {@link data} is the pipeline's reusable buffer.\n *\n * `true` means the next {@link LetterboxPipeline.run} overwrites it, so a\n * caller keeping the values past its own inference has to copy them.\n */\n readonly reused: boolean;\n}\n\n/**\n * Reusable letterbox → tensor pipeline for one target resolution.\n *\n * Holds a target canvas and an output buffer across calls, so a steady stream\n * of frames at the same size allocates nothing. Create one per task, not per\n * frame.\n */\nexport class LetterboxPipeline {\n private readonly _targetWidth: number;\n private readonly _targetHeight: number;\n private readonly _fill: readonly [number, number, number];\n private readonly _target: Canvas2D;\n private readonly _targetContext: Context2D;\n private readonly _buffer: ReusableBuffer;\n private _source: Canvas2D | null = null;\n private _sourceContext: Context2D | null = null;\n\n /**\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param fill RGB padding colour; defaults to YOLO grey.\n */\n constructor(\n targetWidth: number,\n targetHeight: number,\n fill: readonly [number, number, number] = [114, 114, 114],\n ) {\n if (targetWidth <= 0 || targetHeight <= 0) {\n throw new Error(`Invalid letterbox target ${targetWidth}x${targetHeight}.`);\n }\n this._targetWidth = targetWidth;\n this._targetHeight = targetHeight;\n this._fill = fill;\n this._target = createCanvas(targetWidth, targetHeight);\n this._targetContext = get2DContext(this._target, { willReadFrequently: true });\n this._targetContext.imageSmoothingEnabled = true;\n this._targetContext.imageSmoothingQuality = \"high\";\n this._buffer = new ReusableBuffer(3 * targetHeight * targetWidth);\n }\n\n /** The `[width, height]` this pipeline letterboxes into. */\n get targetSize(): readonly [number, number] {\n return [this._targetWidth, this._targetHeight];\n }\n\n /**\n * Letterbox an image and write it as planar float32.\n *\n * The returned buffer is reused between calls unless a previous result is\n * still checked out — {@link release} marks it free again. A second `run`\n * before the first is released allocates a fresh buffer rather than\n * corrupting it, so concurrent `predict()` calls on one task stay correct at\n * the cost of the allocation they were trying to avoid. A buffer that was\n * detached by a consumer that transferred it is replaced rather than written\n * into — see {@link ReusableBuffer}.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n */\n run(image: RGBImage): FusedLetterboxResult {\n const targetWidth = this._targetWidth;\n const targetHeight = this._targetHeight;\n const scale = Math.min(targetWidth / image.width, targetHeight / image.height);\n const scaledWidth = Math.round(image.width * scale);\n const scaledHeight = Math.round(image.height * scale);\n const padLeft = Math.floor((targetWidth - scaledWidth) / 2);\n const padTop = Math.floor((targetHeight - scaledHeight) / 2);\n\n const source = this._ensureSource(image.width, image.height);\n source.putImageData(rgbToImageData(image), 0, 0);\n\n const context = this._targetContext;\n if (\n padLeft > 0 ||\n padTop > 0 ||\n scaledWidth !== targetWidth ||\n scaledHeight !== targetHeight\n ) {\n context.fillStyle = `rgb(${this._fill[0]},${this._fill[1]},${this._fill[2]})`;\n context.fillRect(0, 0, targetWidth, targetHeight);\n }\n context.drawImage(\n this._source as CanvasImageSource,\n 0,\n 0,\n image.width,\n image.height,\n padLeft,\n padTop,\n scaledWidth,\n scaledHeight,\n );\n\n const rgba = context.getImageData(0, 0, targetWidth, targetHeight).data;\n const { data, reused } = this._buffer.claim();\n\n const plane = targetWidth * targetHeight;\n for (let pixel = 0, offset = 0; pixel < plane; pixel++, offset += 4) {\n data[pixel] = (rgba[offset] as number) * INV_255;\n data[plane + pixel] = (rgba[offset + 1] as number) * INV_255;\n data[2 * plane + pixel] = (rgba[offset + 2] as number) * INV_255;\n }\n\n return { data, scale, padLeft, padTop, reused };\n }\n\n /**\n * Mark the reusable buffer free again.\n *\n * Call it once the tensor built from a {@link run} result has been handed to\n * ONNX Runtime and the run has resolved — after that the values are inside\n * the WASM heap and the buffer can be overwritten.\n */\n release(): void {\n this._buffer.release();\n }\n\n /**\n * Grow the scratch source canvas to fit an image, reusing it when possible.\n *\n * A canvas is only reallocated when a frame arrives at a different size than\n * the last one, which for a camera or video source is never after the first.\n *\n * @param width Source width in pixels.\n * @param height Source height in pixels.\n */\n private _ensureSource(width: number, height: number): Context2D {\n if (\n this._source === null ||\n this._source.width !== width ||\n this._source.height !== height\n ) {\n this._source = createCanvas(width, height);\n this._sourceContext = get2DContext(this._source);\n }\n return this._sourceContext as Context2D;\n }\n}\n\n/**\n * Write an RGBA buffer as normalized planar float32.\n *\n * Shared by {@link ResizePipeline} and exported for a custom pipeline that\n * already holds pixels and wants the SDK's exact arithmetic.\n *\n * The expression is `(value / 255 - mean) / std`, evaluated in that order on\n * purpose: folding it into a single multiply-add (`value * k + b`) computes the\n * same quantity but rounds differently, and the tasks' output is asserted to be\n * bit-identical to the composable `normalize` → `toCHW` path.\n *\n * @param rgba Source pixels, 4 bytes per pixel, `width * height` long.\n * @param width Image width in pixels.\n * @param height Image height in pixels.\n * @param mean Per-channel RGB mean, already in `[0, 1]`.\n * @param std Per-channel RGB standard deviation.\n * @param out Destination buffer, `3 * width * height` long.\n * @param stride Bytes per source pixel: 4 for canvas RGBA, 3 for packed RGB.\n */\nexport function writePlanarFloat32(\n rgba: Uint8ClampedArray | Uint8Array,\n width: number,\n height: number,\n mean: readonly [number, number, number],\n std: readonly [number, number, number],\n out: Float32Array,\n stride: number = 4,\n): void {\n const plane = width * height;\n const m0 = mean[0];\n const m1 = mean[1];\n const m2 = mean[2];\n const s0 = std[0];\n const s1 = std[1];\n const s2 = std[2];\n for (let pixel = 0, offset = 0; pixel < plane; pixel++, offset += stride) {\n out[pixel] = ((rgba[offset] as number) * INV_255 - m0) / s0;\n out[plane + pixel] = ((rgba[offset + 1] as number) * INV_255 - m1) / s1;\n out[2 * plane + pixel] = ((rgba[offset + 2] as number) * INV_255 - m2) / s2;\n }\n}\n\n/** Planar tensor data produced by {@link ResizePipeline}. */\nexport interface FusedResizeResult {\n /** CHW float32, normalized, length `3 * targetHeight * targetWidth`. */\n readonly data: Float32Array;\n /**\n * Whether {@link data} is the pipeline's reusable buffer.\n *\n * `true` means the next {@link ResizePipeline.run} overwrites it, so a caller\n * keeping the values past its own inference has to copy them.\n */\n readonly reused: boolean;\n}\n\n/**\n * Reusable stretch-resize → normalized tensor pipeline for one target size.\n *\n * The classification counterpart of {@link LetterboxPipeline}. A classifier\n * stretches to the model's square input instead of letterboxing into it — no\n * padding, no scale to invert later, because nothing is mapped back onto the\n * source image afterwards. That difference is why it cannot simply reuse the\n * letterbox path.\n *\n * What it does share is the technique. The composable route\n * (`resize` → `normalize` → `toCHW`) allocates an `RGBImage` and two\n * `Float32Array`s and walks each end to end on every call: about 1.4 MB of\n * fresh garbage per 224×224 `predict()`, produced at the exact moment a phone\n * near its memory ceiling can least afford it. Here one `drawImage` resizes,\n * and one loop reads the resulting RGBA and writes normalized planar float32\n * into a buffer held across calls.\n *\n * Create one per task, not per frame.\n */\nexport class ResizePipeline {\n private readonly _targetWidth: number;\n private readonly _targetHeight: number;\n private readonly _mean: readonly [number, number, number];\n private readonly _std: readonly [number, number, number];\n private readonly _buffer: ReusableBuffer;\n private _target: Canvas2D | null = null;\n private _targetContext: Context2D | null = null;\n private _source: Canvas2D | null = null;\n private _sourceContext: Context2D | null = null;\n\n /**\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.\n * @param std Per-channel RGB standard deviation. Defaults to no scaling.\n */\n constructor(\n targetWidth: number,\n targetHeight: number,\n mean: readonly [number, number, number] = [0, 0, 0],\n std: readonly [number, number, number] = [1, 1, 1],\n ) {\n if (targetWidth <= 0 || targetHeight <= 0) {\n throw new Error(`Invalid resize target ${targetWidth}x${targetHeight}.`);\n }\n this._targetWidth = targetWidth;\n this._targetHeight = targetHeight;\n this._mean = mean;\n this._std = std;\n this._buffer = new ReusableBuffer(3 * targetHeight * targetWidth);\n }\n\n /** The `[width, height]` this pipeline resizes into. */\n get targetSize(): readonly [number, number] {\n return [this._targetWidth, this._targetHeight];\n }\n\n /**\n * Resize an image to the target size and write it as normalized planar float32.\n *\n * An image that already arrives at the target size skips the canvas entirely\n * and is read straight out of its packed RGB — which is both faster and what\n * keeps the result identical to `resize()`, whose own fast path returns the\n * input untouched.\n *\n * Buffer reuse follows {@link ReusableBuffer}: held across calls, replaced when\n * a consumer detached it by transferring the tensor.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n */\n run(image: RGBImage): FusedResizeResult {\n const targetWidth = this._targetWidth;\n const targetHeight = this._targetHeight;\n const { data, reused } = this._buffer.claim();\n\n if (image.width === targetWidth && image.height === targetHeight) {\n writePlanarFloat32(\n image.data,\n targetWidth,\n targetHeight,\n this._mean,\n this._std,\n data,\n 3,\n );\n return { data, reused };\n }\n\n const source = this._ensureSource(image.width, image.height);\n source.putImageData(rgbToImageData(image), 0, 0);\n\n const context = this._ensureTarget();\n context.drawImage(this._source as CanvasImageSource, 0, 0, targetWidth, targetHeight);\n\n const rgba = context.getImageData(0, 0, targetWidth, targetHeight).data;\n writePlanarFloat32(rgba, targetWidth, targetHeight, this._mean, this._std, data);\n return { data, reused };\n }\n\n /**\n * Mark the reusable buffer free again.\n *\n * Call it once the tensor built from a {@link run} result has been handed to\n * ONNX Runtime and the run has resolved — after that the values are inside\n * the WASM heap and the buffer can be overwritten.\n */\n release(): void {\n this._buffer.release();\n }\n\n /**\n * Build the target canvas on first use.\n *\n * Lazily, so a pipeline constructed where no canvas implementation exists\n * (a Node test, a worker without OffscreenCanvas) only fails if it is asked\n * to resize something.\n */\n private _ensureTarget(): Context2D {\n if (this._target === null) {\n this._target = createCanvas(this._targetWidth, this._targetHeight);\n this._targetContext = get2DContext(this._target, { willReadFrequently: true });\n this._targetContext.imageSmoothingEnabled = true;\n this._targetContext.imageSmoothingQuality = \"high\";\n }\n return this._targetContext as Context2D;\n }\n\n /** Grow the scratch source canvas to fit an image, reusing it when possible. */\n private _ensureSource(width: number, height: number): Context2D {\n if (\n this._source === null ||\n this._source.width !== width ||\n this._source.height !== height\n ) {\n this._source = createCanvas(width, height);\n this._sourceContext = get2DContext(this._source);\n }\n return this._sourceContext as Context2D;\n }\n}\n\n/**\n * Resize an image into normalized planar float32 without keeping any state.\n *\n * The allocation-free path is {@link ResizePipeline}; this is the one-shot\n * form, for a caller who wants the fused behaviour without owning a pipeline.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.\n * @param std Per-channel RGB standard deviation. Defaults to no scaling.\n */\nexport function resizeToTensorData(\n image: RGBImage,\n targetWidth: number,\n targetHeight: number,\n mean: readonly [number, number, number] = [0, 0, 0],\n std: readonly [number, number, number] = [1, 1, 1],\n): FusedResizeResult {\n return new ResizePipeline(targetWidth, targetHeight, mean, std).run(image);\n}\n\n/**\n * Build a zero-filled CHW tensor payload for a warm-up run.\n *\n * @param width Model input width in pixels.\n * @param height Model input height in pixels.\n */\nexport function zeroTensorData(width: number, height: number): Float32Array {\n return new Float32Array(3 * height * width);\n}\n\n/**\n * Letterbox an image into planar float32 without keeping any state.\n *\n * The allocation-free path is {@link LetterboxPipeline}; this is the one-shot\n * form, for a caller who wants the fused behaviour without owning a pipeline.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param fill RGB padding colour; defaults to YOLO grey.\n */\nexport function letterboxToTensorData(\n image: RGBImage,\n targetWidth: number,\n targetHeight: number,\n fill: readonly [number, number, number] = [114, 114, 114],\n): FusedLetterboxResult {\n return new LetterboxPipeline(targetWidth, targetHeight, fill).run(image);\n}\n"],"mappings":";;AAiCA,IAAM,IAAU,IAAI,KA6Bd,IAAN,MAAqB;CACjB;CACA;CACA,SAAiB;CAGjB,YAAY,GAAc;EAEtB,AADA,KAAK,QAAQ,GACb,KAAK,UAAU,IAAI,aAAa,CAAI;CACxC;CAUA,QAAqB;EAIjB,OAHI,KAAK,SAAe;GAAE,MAAM,IAAI,aAAa,KAAK,KAAK;GAAG,QAAQ;EAAM,KACxE,KAAK,QAAQ,WAAW,KAAK,UAAO,KAAK,UAAU,IAAI,aAAa,KAAK,KAAK,IAClF,KAAK,SAAS,IACP;GAAE,MAAM,KAAK;GAAS,QAAQ;EAAK;CAC9C;CAGA,UAAgB;EACZ,KAAK,SAAS;CAClB;AACJ,GA4Ba,IAAb,MAA+B;CAC3B;CACA;CACA;CACA;CACA;CACA;CACA,UAAmC;CACnC,iBAA2C;CAO3C,YACI,GACA,GACA,IAA0C;EAAC;EAAK;EAAK;CAAG,GAC1D;EACE,IAAI,KAAe,KAAK,KAAgB,GACpC,MAAU,MAAM,4BAA4B,EAAY,GAAG,EAAa,EAAE;EAS9E,AAPA,KAAK,eAAe,GACpB,KAAK,gBAAgB,GACrB,KAAK,QAAQ,GACb,KAAK,UAAU,EAAa,GAAa,CAAY,GACrD,KAAK,iBAAiB,EAAa,KAAK,SAAS,EAAE,oBAAoB,GAAK,CAAC,GAC7E,KAAK,eAAe,wBAAwB,IAC5C,KAAK,eAAe,wBAAwB,QAC5C,KAAK,UAAU,IAAI,EAAe,IAAI,IAAe,CAAW;CACpE;CAGA,IAAI,aAAwC;EACxC,OAAO,CAAC,KAAK,cAAc,KAAK,aAAa;CACjD;CAeA,IAAI,GAAuC;EACvC,IAAM,IAAc,KAAK,cACnB,IAAe,KAAK,eACpB,IAAQ,KAAK,IAAI,IAAc,EAAM,OAAO,IAAe,EAAM,MAAM,GACvE,IAAc,KAAK,MAAM,EAAM,QAAQ,CAAK,GAC5C,IAAe,KAAK,MAAM,EAAM,SAAS,CAAK,GAC9C,IAAU,KAAK,OAAO,IAAc,KAAe,CAAC,GACpD,IAAS,KAAK,OAAO,IAAe,KAAgB,CAAC;EAG3D,KADoB,cAAc,EAAM,OAAO,EAAM,MACrD,CAAA,CAAO,aAAa,EAAe,CAAK,GAAG,GAAG,CAAC;EAE/C,IAAM,IAAU,KAAK;EAUrB,CARI,IAAU,KACV,IAAS,KACT,MAAgB,KAChB,MAAiB,OAEjB,EAAQ,YAAY,OAAO,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,IAC3E,EAAQ,SAAS,GAAG,GAAG,GAAa,CAAY,IAEpD,EAAQ,UACJ,KAAK,SACL,GACA,GACA,EAAM,OACN,EAAM,QACN,GACA,GACA,GACA,CACJ;EAEA,IAAM,IAAO,EAAQ,aAAa,GAAG,GAAG,GAAa,CAAY,CAAC,CAAC,MAC7D,EAAE,SAAM,cAAW,KAAK,QAAQ,MAAM,GAEtC,IAAQ,IAAc;EAC5B,KAAK,IAAI,IAAQ,GAAG,IAAS,GAAG,IAAQ,GAAO,KAAS,KAAU,GAG9D,AAFA,EAAK,KAAU,EAAK,KAAqB,GACzC,EAAK,IAAQ,KAAU,EAAK,IAAS,KAAgB,GACrD,EAAK,IAAI,IAAQ,KAAU,EAAK,IAAS,KAAgB;EAG7D,OAAO;GAAE;GAAM;GAAO;GAAS;GAAQ;EAAO;CAClD;CASA,UAAgB;EACZ,KAAK,QAAQ,QAAQ;CACzB;CAWA,cAAsB,GAAe,GAA2B;EAS5D,QAPI,KAAK,YAAY,QACjB,KAAK,QAAQ,UAAU,KACvB,KAAK,QAAQ,WAAW,OAExB,KAAK,UAAU,EAAa,GAAO,CAAM,GACzC,KAAK,iBAAiB,EAAa,KAAK,OAAO,IAE5C,KAAK;CAChB;AACJ;AAqBA,SAAgB,EACZ,GACA,GACA,GACA,GACA,GACA,GACA,IAAiB,GACb;CACJ,IAAM,IAAQ,IAAQ,GAChB,IAAK,EAAK,IACV,IAAK,EAAK,IACV,IAAK,EAAK,IACV,IAAK,EAAI,IACT,IAAK,EAAI,IACT,IAAK,EAAI;CACf,KAAK,IAAI,IAAQ,GAAG,IAAS,GAAG,IAAQ,GAAO,KAAS,KAAU,GAG9D,AAFA,EAAI,MAAW,EAAK,KAAqB,IAAU,KAAM,GACzD,EAAI,IAAQ,MAAW,EAAK,IAAS,KAAgB,IAAU,KAAM,GACrE,EAAI,IAAI,IAAQ,MAAW,EAAK,IAAS,KAAgB,IAAU,KAAM;AAEjF;AAkCA,IAAa,IAAb,MAA4B;CACxB;CACA;CACA;CACA;CACA;CACA,UAAmC;CACnC,iBAA2C;CAC3C,UAAmC;CACnC,iBAA2C;CAQ3C,YACI,GACA,GACA,IAA0C;EAAC;EAAG;EAAG;CAAC,GAClD,IAAyC;EAAC;EAAG;EAAG;CAAC,GACnD;EACE,IAAI,KAAe,KAAK,KAAgB,GACpC,MAAU,MAAM,yBAAyB,EAAY,GAAG,EAAa,EAAE;EAM3E,AAJA,KAAK,eAAe,GACpB,KAAK,gBAAgB,GACrB,KAAK,QAAQ,GACb,KAAK,OAAO,GACZ,KAAK,UAAU,IAAI,EAAe,IAAI,IAAe,CAAW;CACpE;CAGA,IAAI,aAAwC;EACxC,OAAO,CAAC,KAAK,cAAc,KAAK,aAAa;CACjD;CAeA,IAAI,GAAoC;EACpC,IAAM,IAAc,KAAK,cACnB,IAAe,KAAK,eACpB,EAAE,SAAM,cAAW,KAAK,QAAQ,MAAM;EAE5C,IAAI,EAAM,UAAU,KAAe,EAAM,WAAW,GAUhD,OATA,EACI,EAAM,MACN,GACA,GACA,KAAK,OACL,KAAK,MACL,GACA,CACJ,GACO;GAAE;GAAM;EAAO;EAI1B,KADoB,cAAc,EAAM,OAAO,EAAM,MACrD,CAAA,CAAO,aAAa,EAAe,CAAK,GAAG,GAAG,CAAC;EAE/C,IAAM,IAAU,KAAK,cAAc;EACnC,EAAQ,UAAU,KAAK,SAA8B,GAAG,GAAG,GAAa,CAAY;EAEpF,IAAM,IAAO,EAAQ,aAAa,GAAG,GAAG,GAAa,CAAY,CAAC,CAAC;EAEnE,OADA,EAAmB,GAAM,GAAa,GAAc,KAAK,OAAO,KAAK,MAAM,CAAI,GACxE;GAAE;GAAM;EAAO;CAC1B;CASA,UAAgB;EACZ,KAAK,QAAQ,QAAQ;CACzB;CASA,gBAAmC;EAO/B,OANI,KAAK,YAAY,SACjB,KAAK,UAAU,EAAa,KAAK,cAAc,KAAK,aAAa,GACjE,KAAK,iBAAiB,EAAa,KAAK,SAAS,EAAE,oBAAoB,GAAK,CAAC,GAC7E,KAAK,eAAe,wBAAwB,IAC5C,KAAK,eAAe,wBAAwB,SAEzC,KAAK;CAChB;CAGA,cAAsB,GAAe,GAA2B;EAS5D,QAPI,KAAK,YAAY,QACjB,KAAK,QAAQ,UAAU,KACvB,KAAK,QAAQ,WAAW,OAExB,KAAK,UAAU,EAAa,GAAO,CAAM,GACzC,KAAK,iBAAiB,EAAa,KAAK,OAAO,IAE5C,KAAK;CAChB;AACJ;AAcA,SAAgB,EACZ,GACA,GACA,GACA,IAA0C;CAAC;CAAG;CAAG;AAAC,GAClD,IAAyC;CAAC;CAAG;CAAG;AAAC,GAChC;CACjB,OAAO,IAAI,EAAe,GAAa,GAAc,GAAM,CAAG,CAAC,CAAC,IAAI,CAAK;AAC7E;AAQA,SAAgB,EAAe,GAAe,GAA8B;CACxE,OAAO,IAAI,aAAa,IAAI,IAAS,CAAK;AAC9C;AAaA,SAAgB,EACZ,GACA,GACA,GACA,IAA0C;CAAC;CAAK;CAAK;AAAG,GACpC;CACpB,OAAO,IAAI,EAAkB,GAAa,GAAc,CAAI,CAAC,CAAC,IAAI,CAAK;AAC3E"}
|
|
1
|
+
{"version":3,"file":"pipeline.js","names":[],"sources":["../../../src/vision/preprocess/pipeline.ts"],"sourcesContent":["/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */\n/**\n * Fused letterbox → CHW float32 pipeline with reusable buffers.\n *\n * The composable primitives in {@link ./image.js} each allocate and each walk\n * their input end to end, which is the right shape for a library but the wrong\n * shape for a video loop. Chaining them costs eleven full-buffer passes and six\n * large allocations per frame:\n *\n * `getImageData` → RGBA→RGB → RGB→RGBA → `putImageData` → `drawImage` →\n * `getImageData` → RGBA→RGB → fill → row copies → `toFloat32` → `toCHW`.\n *\n * This module collapses the second half of that into two: one `drawImage` that\n * resizes *and* positions the content inside the padded target in a single\n * accelerated operation, and one loop that reads the resulting RGBA and writes\n * planar float32 directly. The intermediate `RGBImage` at target size, the fill\n * loop, the row copies and the two 4.9 MB `Float32Array` allocations all go\n * away.\n *\n * The primitives stay exactly as they are — they are public API and they are\n * what makes a custom pipeline writable. This is the fast path the built-in\n * tasks take.\n */\n\nimport {\n createCanvas,\n get2DContext,\n rgbToImageData,\n type Canvas2D,\n type Context2D,\n} from \"../core/canvas\";\nimport type { RGBImage } from \"../types\";\n\nconst INV_255 = 1 / 255;\n\n/** One claim on a {@link ReusableBuffer}. */\ninterface BufferClaim {\n /** The buffer to write into, always `size` long. */\n readonly data: Float32Array;\n /** Whether {@link data} is the held buffer rather than a fresh allocation. */\n readonly reused: boolean;\n}\n\n/**\n * A `Float32Array` held across calls, handed out one claim at a time.\n *\n * Both pipelines want the same thing — allocate once, write into it every frame,\n * and fall back to a fresh array when a previous result has not been released\n * yet — so the bookkeeping lives here instead of twice.\n *\n * The part that is not obvious is {@link claim} re-allocating a buffer that is\n * *detached*. `ort.env.wasm.proxy` runs ONNX Runtime in a worker and posts the\n * input tensors with their `ArrayBuffer`s in the transfer list, which detaches\n * them on this side. A detached `Float32Array` is silently 0 long: writing to it\n * is a no-op, and the next `InferenceSession.run` rejects with\n * `Tensor's size(N) does not match data length(0)` on every other call — once for\n * the detached buffer, then the throw leaves the claim outstanding so the call\n * after it allocates and succeeds. Treating a buffer that changed length as spent\n * turns that into one extra allocation per transfer, which is what reuse was\n * avoiding, and keeps it correct for any consumer that transfers the tensor\n * rather than copying it.\n */\nclass ReusableBuffer {\n private readonly _size: number;\n private _buffer: Float32Array;\n private _inUse = false;\n\n /** @param size Length in floats of every buffer this hands out. */\n constructor(size: number) {\n this._size = size;\n this._buffer = new Float32Array(size);\n }\n\n /**\n * Take the held buffer, or a fresh one when it is unavailable.\n *\n * Unavailable means either still checked out by an unreleased claim, or\n * detached by whoever it was handed to. The first case allocates for this call\n * only; the second replaces the held buffer, so the allocation is paid once per\n * transfer rather than on every call after it.\n */\n claim(): BufferClaim {\n if (this._inUse) return { data: new Float32Array(this._size), reused: false };\n if (this._buffer.length !== this._size) this._buffer = new Float32Array(this._size);\n this._inUse = true;\n return { data: this._buffer, reused: true };\n }\n\n /** Mark the held buffer free for the next {@link claim}. */\n release(): void {\n this._inUse = false;\n }\n}\n\n/** Geometry of a letterbox, plus the planar tensor data it produced. */\nexport interface FusedLetterboxResult {\n /** CHW float32 in `[0, 1]`, length `3 * targetHeight * targetWidth`. */\n readonly data: Float32Array;\n /** Factor applied to the original image (`< 1` if downscaled). */\n readonly scale: number;\n /** Horizontal padding in pixels. */\n readonly padLeft: number;\n /** Vertical padding in pixels. */\n readonly padTop: number;\n /**\n * Whether {@link data} is the pipeline's reusable buffer.\n *\n * `true` means the next {@link LetterboxPipeline.run} overwrites it, so a\n * caller keeping the values past its own inference has to copy them.\n */\n readonly reused: boolean;\n}\n\n/**\n * Reusable letterbox → tensor pipeline for one target resolution.\n *\n * Holds a target canvas and an output buffer across calls, so a steady stream\n * of frames at the same size allocates nothing. Create one per task, not per\n * frame.\n */\nexport class LetterboxPipeline {\n private readonly _targetWidth: number;\n private readonly _targetHeight: number;\n private readonly _fill: readonly [number, number, number];\n private readonly _target: Canvas2D;\n private readonly _targetContext: Context2D;\n private readonly _buffer: ReusableBuffer;\n private _source: Canvas2D | null = null;\n private _sourceContext: Context2D | null = null;\n\n /**\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param fill RGB padding colour; defaults to YOLO grey.\n */\n constructor(\n targetWidth: number,\n targetHeight: number,\n fill: readonly [number, number, number] = [114, 114, 114],\n ) {\n if (targetWidth <= 0 || targetHeight <= 0) {\n throw new Error(`Invalid letterbox target ${targetWidth}x${targetHeight}.`);\n }\n this._targetWidth = targetWidth;\n this._targetHeight = targetHeight;\n this._fill = fill;\n this._target = createCanvas(targetWidth, targetHeight);\n this._targetContext = get2DContext(this._target, { willReadFrequently: true });\n this._targetContext.imageSmoothingEnabled = true;\n this._targetContext.imageSmoothingQuality = \"high\";\n this._buffer = new ReusableBuffer(3 * targetHeight * targetWidth);\n }\n\n /** The `[width, height]` this pipeline letterboxes into. */\n get targetSize(): readonly [number, number] {\n return [this._targetWidth, this._targetHeight];\n }\n\n /**\n * Letterbox an image and write it as planar float32.\n *\n * The returned buffer is reused between calls unless a previous result is\n * still checked out — {@link release} marks it free again. A second `run`\n * before the first is released allocates a fresh buffer rather than\n * corrupting it, so concurrent `predict()` calls on one task stay correct at\n * the cost of the allocation they were trying to avoid. A buffer that was\n * detached by a consumer that transferred it is replaced rather than written\n * into — see {@link ReusableBuffer}.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param canvas Optional drawable already holding exactly `image`'s pixels at\n * `image.width × image.height`, fully opaque. When given it is drawn\n * straight into the target, skipping the RGB→RGBA rebuild and the\n * `putImageData` onto the scratch canvas — two full-resolution passes per\n * frame. The tasks pass the canvas `loadImage` decoded onto; a caller with a\n * translucent source must not, since the result would composite its alpha.\n */\n run(image: RGBImage, canvas: CanvasImageSource | null = null): FusedLetterboxResult {\n const targetWidth = this._targetWidth;\n const targetHeight = this._targetHeight;\n const scale = Math.min(targetWidth / image.width, targetHeight / image.height);\n const scaledWidth = Math.round(image.width * scale);\n const scaledHeight = Math.round(image.height * scale);\n const padLeft = Math.floor((targetWidth - scaledWidth) / 2);\n const padTop = Math.floor((targetHeight - scaledHeight) / 2);\n\n const drawable = canvas ?? this._paintSource(image);\n\n const context = this._targetContext;\n if (\n padLeft > 0 ||\n padTop > 0 ||\n scaledWidth !== targetWidth ||\n scaledHeight !== targetHeight\n ) {\n context.fillStyle = `rgb(${this._fill[0]},${this._fill[1]},${this._fill[2]})`;\n context.fillRect(0, 0, targetWidth, targetHeight);\n }\n context.drawImage(\n drawable,\n 0,\n 0,\n image.width,\n image.height,\n padLeft,\n padTop,\n scaledWidth,\n scaledHeight,\n );\n\n const rgba = context.getImageData(0, 0, targetWidth, targetHeight).data;\n const { data, reused } = this._buffer.claim();\n\n const plane = targetWidth * targetHeight;\n for (let pixel = 0, offset = 0; pixel < plane; pixel++, offset += 4) {\n data[pixel] = (rgba[offset] as number) * INV_255;\n data[plane + pixel] = (rgba[offset + 1] as number) * INV_255;\n data[2 * plane + pixel] = (rgba[offset + 2] as number) * INV_255;\n }\n\n return { data, scale, padLeft, padTop, reused };\n }\n\n /**\n * Mark the reusable buffer free again.\n *\n * Call it once the tensor built from a {@link run} result has been handed to\n * ONNX Runtime and the run has resolved — after that the values are inside\n * the WASM heap and the buffer can be overwritten.\n *\n * Pass the result being released when several runs can be outstanding at\n * once. A result that got a fresh allocation (`reused: false`) holds nothing\n * of the pipeline's, so releasing it is a no-op; without that check, the\n * caller finishing with a fresh buffer would free the shared one while\n * another caller still had its frame in it. Called without an argument it\n * frees the shared buffer unconditionally, as before.\n *\n * @param result The {@link run} result being released.\n */\n release(result?: { readonly reused: boolean }): void {\n if (result !== undefined && !result.reused) return;\n this._buffer.release();\n }\n\n /**\n * Paint an image onto the scratch source canvas and return that canvas.\n *\n * The canvas is only reallocated when a frame arrives at a different size\n * than the last one, which for a camera or video source is never after the\n * first.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n */\n private _paintSource(image: RGBImage): CanvasImageSource {\n const { width, height } = image;\n if (\n this._source === null ||\n this._source.width !== width ||\n this._source.height !== height\n ) {\n this._source = createCanvas(width, height);\n this._sourceContext = get2DContext(this._source);\n }\n (this._sourceContext as Context2D).putImageData(rgbToImageData(image), 0, 0);\n return this._source as CanvasImageSource;\n }\n}\n\n/**\n * Write an RGBA buffer as normalized planar float32.\n *\n * Shared by {@link ResizePipeline} and exported for a custom pipeline that\n * already holds pixels and wants the SDK's exact arithmetic.\n *\n * The expression is `(value / 255 - mean) / std`, evaluated in that order on\n * purpose: folding it into a single multiply-add (`value * k + b`) computes the\n * same quantity but rounds differently, and the tasks' output is asserted to be\n * bit-identical to the composable `normalize` → `toCHW` path.\n *\n * @param rgba Source pixels, 4 bytes per pixel, `width * height` long.\n * @param width Image width in pixels.\n * @param height Image height in pixels.\n * @param mean Per-channel RGB mean, already in `[0, 1]`.\n * @param std Per-channel RGB standard deviation.\n * @param out Destination buffer, `3 * width * height` long.\n * @param stride Bytes per source pixel: 4 for canvas RGBA, 3 for packed RGB.\n */\nexport function writePlanarFloat32(\n rgba: Uint8ClampedArray | Uint8Array,\n width: number,\n height: number,\n mean: readonly [number, number, number],\n std: readonly [number, number, number],\n out: Float32Array,\n stride: number = 4,\n): void {\n const plane = width * height;\n const m0 = mean[0];\n const m1 = mean[1];\n const m2 = mean[2];\n const s0 = std[0];\n const s1 = std[1];\n const s2 = std[2];\n for (let pixel = 0, offset = 0; pixel < plane; pixel++, offset += stride) {\n out[pixel] = ((rgba[offset] as number) * INV_255 - m0) / s0;\n out[plane + pixel] = ((rgba[offset + 1] as number) * INV_255 - m1) / s1;\n out[2 * plane + pixel] = ((rgba[offset + 2] as number) * INV_255 - m2) / s2;\n }\n}\n\n/** Planar tensor data produced by {@link ResizePipeline}. */\nexport interface FusedResizeResult {\n /** CHW float32, normalized, length `3 * targetHeight * targetWidth`. */\n readonly data: Float32Array;\n /**\n * Whether {@link data} is the pipeline's reusable buffer.\n *\n * `true` means the next {@link ResizePipeline.run} overwrites it, so a caller\n * keeping the values past its own inference has to copy them.\n */\n readonly reused: boolean;\n}\n\n/**\n * Reusable stretch-resize → normalized tensor pipeline for one target size.\n *\n * The classification counterpart of {@link LetterboxPipeline}. A classifier\n * stretches to the model's square input instead of letterboxing into it — no\n * padding, no scale to invert later, because nothing is mapped back onto the\n * source image afterwards. That difference is why it cannot simply reuse the\n * letterbox path.\n *\n * What it does share is the technique. The composable route\n * (`resize` → `normalize` → `toCHW`) allocates an `RGBImage` and two\n * `Float32Array`s and walks each end to end on every call: about 1.4 MB of\n * fresh garbage per 224×224 `predict()`, produced at the exact moment a phone\n * near its memory ceiling can least afford it. Here one `drawImage` resizes,\n * and one loop reads the resulting RGBA and writes normalized planar float32\n * into a buffer held across calls.\n *\n * Create one per task, not per frame.\n */\nexport class ResizePipeline {\n private readonly _targetWidth: number;\n private readonly _targetHeight: number;\n private readonly _mean: readonly [number, number, number];\n private readonly _std: readonly [number, number, number];\n private readonly _buffer: ReusableBuffer;\n private _target: Canvas2D | null = null;\n private _targetContext: Context2D | null = null;\n private _source: Canvas2D | null = null;\n private _sourceContext: Context2D | null = null;\n\n /**\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.\n * @param std Per-channel RGB standard deviation. Defaults to no scaling.\n */\n constructor(\n targetWidth: number,\n targetHeight: number,\n mean: readonly [number, number, number] = [0, 0, 0],\n std: readonly [number, number, number] = [1, 1, 1],\n ) {\n if (targetWidth <= 0 || targetHeight <= 0) {\n throw new Error(`Invalid resize target ${targetWidth}x${targetHeight}.`);\n }\n this._targetWidth = targetWidth;\n this._targetHeight = targetHeight;\n this._mean = mean;\n this._std = std;\n this._buffer = new ReusableBuffer(3 * targetHeight * targetWidth);\n }\n\n /** The `[width, height]` this pipeline resizes into. */\n get targetSize(): readonly [number, number] {\n return [this._targetWidth, this._targetHeight];\n }\n\n /**\n * Resize an image to the target size and write it as normalized planar float32.\n *\n * An image that already arrives at the target size skips the canvas entirely\n * and is read straight out of its packed RGB — which is both faster and what\n * keeps the result identical to `resize()`, whose own fast path returns the\n * input untouched.\n *\n * Buffer reuse follows {@link ReusableBuffer}: held across calls, replaced when\n * a consumer detached it by transferring the tensor.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param canvas Optional opaque drawable holding exactly `image`'s pixels —\n * see {@link LetterboxPipeline.run}. Ignored on the same-size fast path,\n * which never touches a canvas.\n */\n run(image: RGBImage, canvas: CanvasImageSource | null = null): FusedResizeResult {\n const targetWidth = this._targetWidth;\n const targetHeight = this._targetHeight;\n const { data, reused } = this._buffer.claim();\n\n if (image.width === targetWidth && image.height === targetHeight) {\n writePlanarFloat32(\n image.data,\n targetWidth,\n targetHeight,\n this._mean,\n this._std,\n data,\n 3,\n );\n return { data, reused };\n }\n\n const drawable = canvas ?? this._paintSource(image);\n const context = this._ensureTarget();\n context.drawImage(\n drawable,\n 0,\n 0,\n image.width,\n image.height,\n 0,\n 0,\n targetWidth,\n targetHeight,\n );\n\n const rgba = context.getImageData(0, 0, targetWidth, targetHeight).data;\n writePlanarFloat32(rgba, targetWidth, targetHeight, this._mean, this._std, data);\n return { data, reused };\n }\n\n /**\n * Mark the reusable buffer free again.\n *\n * Call it once the tensor built from a {@link run} result has been handed to\n * ONNX Runtime and the run has resolved — after that the values are inside\n * the WASM heap and the buffer can be overwritten.\n *\n * Pass the result being released when several runs can be outstanding at\n * once. A result that got a fresh allocation (`reused: false`) holds nothing\n * of the pipeline's, so releasing it is a no-op; without that check, the\n * caller finishing with a fresh buffer would free the shared one while\n * another caller still had its frame in it. Called without an argument it\n * frees the shared buffer unconditionally, as before.\n *\n * @param result The {@link run} result being released.\n */\n release(result?: { readonly reused: boolean }): void {\n if (result !== undefined && !result.reused) return;\n this._buffer.release();\n }\n\n /**\n * Build the target canvas on first use.\n *\n * Lazily, so a pipeline constructed where no canvas implementation exists\n * (a Node test, a worker without OffscreenCanvas) only fails if it is asked\n * to resize something.\n */\n private _ensureTarget(): Context2D {\n if (this._target === null) {\n this._target = createCanvas(this._targetWidth, this._targetHeight);\n this._targetContext = get2DContext(this._target, { willReadFrequently: true });\n this._targetContext.imageSmoothingEnabled = true;\n this._targetContext.imageSmoothingQuality = \"high\";\n }\n return this._targetContext as Context2D;\n }\n\n /**\n * Paint an image onto the scratch source canvas and return that canvas.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n */\n private _paintSource(image: RGBImage): CanvasImageSource {\n const { width, height } = image;\n if (\n this._source === null ||\n this._source.width !== width ||\n this._source.height !== height\n ) {\n this._source = createCanvas(width, height);\n this._sourceContext = get2DContext(this._source);\n }\n (this._sourceContext as Context2D).putImageData(rgbToImageData(image), 0, 0);\n return this._source as CanvasImageSource;\n }\n}\n\n/**\n * Resize an image into normalized planar float32 without keeping any state.\n *\n * The allocation-free path is {@link ResizePipeline}; this is the one-shot\n * form, for a caller who wants the fused behaviour without owning a pipeline.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param mean Per-channel RGB mean in `[0, 1]`. Defaults to no shift.\n * @param std Per-channel RGB standard deviation. Defaults to no scaling.\n */\nexport function resizeToTensorData(\n image: RGBImage,\n targetWidth: number,\n targetHeight: number,\n mean: readonly [number, number, number] = [0, 0, 0],\n std: readonly [number, number, number] = [1, 1, 1],\n): FusedResizeResult {\n return new ResizePipeline(targetWidth, targetHeight, mean, std).run(image);\n}\n\n/**\n * Build a zero-filled CHW tensor payload for a warm-up run.\n *\n * @param width Model input width in pixels.\n * @param height Model input height in pixels.\n */\nexport function zeroTensorData(width: number, height: number): Float32Array {\n return new Float32Array(3 * height * width);\n}\n\n/**\n * Letterbox an image into planar float32 without keeping any state.\n *\n * The allocation-free path is {@link LetterboxPipeline}; this is the one-shot\n * form, for a caller who wants the fused behaviour without owning a pipeline.\n *\n * @param image Source image in the SDK's canonical HWC RGB layout.\n * @param targetWidth Model input width in pixels.\n * @param targetHeight Model input height in pixels.\n * @param fill RGB padding colour; defaults to YOLO grey.\n */\nexport function letterboxToTensorData(\n image: RGBImage,\n targetWidth: number,\n targetHeight: number,\n fill: readonly [number, number, number] = [114, 114, 114],\n): FusedLetterboxResult {\n return new LetterboxPipeline(targetWidth, targetHeight, fill).run(image);\n}\n"],"mappings":";;AAiCA,IAAM,IAAU,IAAI,KA6Bd,IAAN,MAAqB;CACjB;CACA;CACA,SAAiB;CAGjB,YAAY,GAAc;EAEtB,AADA,KAAK,QAAQ,GACb,KAAK,UAAU,IAAI,aAAa,CAAI;CACxC;CAUA,QAAqB;EAIjB,OAHI,KAAK,SAAe;GAAE,MAAM,IAAI,aAAa,KAAK,KAAK;GAAG,QAAQ;EAAM,KACxE,KAAK,QAAQ,WAAW,KAAK,UAAO,KAAK,UAAU,IAAI,aAAa,KAAK,KAAK,IAClF,KAAK,SAAS,IACP;GAAE,MAAM,KAAK;GAAS,QAAQ;EAAK;CAC9C;CAGA,UAAgB;EACZ,KAAK,SAAS;CAClB;AACJ,GA4Ba,IAAb,MAA+B;CAC3B;CACA;CACA;CACA;CACA;CACA;CACA,UAAmC;CACnC,iBAA2C;CAO3C,YACI,GACA,GACA,IAA0C;EAAC;EAAK;EAAK;CAAG,GAC1D;EACE,IAAI,KAAe,KAAK,KAAgB,GACpC,MAAU,MAAM,4BAA4B,EAAY,GAAG,EAAa,EAAE;EAS9E,AAPA,KAAK,eAAe,GACpB,KAAK,gBAAgB,GACrB,KAAK,QAAQ,GACb,KAAK,UAAU,EAAa,GAAa,CAAY,GACrD,KAAK,iBAAiB,EAAa,KAAK,SAAS,EAAE,oBAAoB,GAAK,CAAC,GAC7E,KAAK,eAAe,wBAAwB,IAC5C,KAAK,eAAe,wBAAwB,QAC5C,KAAK,UAAU,IAAI,EAAe,IAAI,IAAe,CAAW;CACpE;CAGA,IAAI,aAAwC;EACxC,OAAO,CAAC,KAAK,cAAc,KAAK,aAAa;CACjD;CAqBA,IAAI,GAAiB,IAAmC,MAA4B;EAChF,IAAM,IAAc,KAAK,cACnB,IAAe,KAAK,eACpB,IAAQ,KAAK,IAAI,IAAc,EAAM,OAAO,IAAe,EAAM,MAAM,GACvE,IAAc,KAAK,MAAM,EAAM,QAAQ,CAAK,GAC5C,IAAe,KAAK,MAAM,EAAM,SAAS,CAAK,GAC9C,IAAU,KAAK,OAAO,IAAc,KAAe,CAAC,GACpD,IAAS,KAAK,OAAO,IAAe,KAAgB,CAAC,GAErD,IAAW,KAAU,KAAK,aAAa,CAAK,GAE5C,IAAU,KAAK;EAUrB,CARI,IAAU,KACV,IAAS,KACT,MAAgB,KAChB,MAAiB,OAEjB,EAAQ,YAAY,OAAO,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,IAC3E,EAAQ,SAAS,GAAG,GAAG,GAAa,CAAY,IAEpD,EAAQ,UACJ,GACA,GACA,GACA,EAAM,OACN,EAAM,QACN,GACA,GACA,GACA,CACJ;EAEA,IAAM,IAAO,EAAQ,aAAa,GAAG,GAAG,GAAa,CAAY,CAAC,CAAC,MAC7D,EAAE,SAAM,cAAW,KAAK,QAAQ,MAAM,GAEtC,IAAQ,IAAc;EAC5B,KAAK,IAAI,IAAQ,GAAG,IAAS,GAAG,IAAQ,GAAO,KAAS,KAAU,GAG9D,AAFA,EAAK,KAAU,EAAK,KAAqB,GACzC,EAAK,IAAQ,KAAU,EAAK,IAAS,KAAgB,GACrD,EAAK,IAAI,IAAQ,KAAU,EAAK,IAAS,KAAgB;EAG7D,OAAO;GAAE;GAAM;GAAO;GAAS;GAAQ;EAAO;CAClD;CAkBA,QAAQ,GAA6C;EAC7C,OAAW,KAAA,KAAc,EAAO,WACpC,KAAK,QAAQ,QAAQ;CACzB;CAWA,aAAqB,GAAoC;EACrD,IAAM,EAAE,UAAO,cAAW;EAU1B,QARI,KAAK,YAAY,QACjB,KAAK,QAAQ,UAAU,KACvB,KAAK,QAAQ,WAAW,OAExB,KAAK,UAAU,EAAa,GAAO,CAAM,GACzC,KAAK,iBAAiB,EAAa,KAAK,OAAO,IAEnD,KAAM,eAA6B,aAAa,EAAe,CAAK,GAAG,GAAG,CAAC,GACpE,KAAK;CAChB;AACJ;AAqBA,SAAgB,EACZ,GACA,GACA,GACA,GACA,GACA,GACA,IAAiB,GACb;CACJ,IAAM,IAAQ,IAAQ,GAChB,IAAK,EAAK,IACV,IAAK,EAAK,IACV,IAAK,EAAK,IACV,IAAK,EAAI,IACT,IAAK,EAAI,IACT,IAAK,EAAI;CACf,KAAK,IAAI,IAAQ,GAAG,IAAS,GAAG,IAAQ,GAAO,KAAS,KAAU,GAG9D,AAFA,EAAI,MAAW,EAAK,KAAqB,IAAU,KAAM,GACzD,EAAI,IAAQ,MAAW,EAAK,IAAS,KAAgB,IAAU,KAAM,GACrE,EAAI,IAAI,IAAQ,MAAW,EAAK,IAAS,KAAgB,IAAU,KAAM;AAEjF;AAkCA,IAAa,IAAb,MAA4B;CACxB;CACA;CACA;CACA;CACA;CACA,UAAmC;CACnC,iBAA2C;CAC3C,UAAmC;CACnC,iBAA2C;CAQ3C,YACI,GACA,GACA,IAA0C;EAAC;EAAG;EAAG;CAAC,GAClD,IAAyC;EAAC;EAAG;EAAG;CAAC,GACnD;EACE,IAAI,KAAe,KAAK,KAAgB,GACpC,MAAU,MAAM,yBAAyB,EAAY,GAAG,EAAa,EAAE;EAM3E,AAJA,KAAK,eAAe,GACpB,KAAK,gBAAgB,GACrB,KAAK,QAAQ,GACb,KAAK,OAAO,GACZ,KAAK,UAAU,IAAI,EAAe,IAAI,IAAe,CAAW;CACpE;CAGA,IAAI,aAAwC;EACxC,OAAO,CAAC,KAAK,cAAc,KAAK,aAAa;CACjD;CAkBA,IAAI,GAAiB,IAAmC,MAAyB;EAC7E,IAAM,IAAc,KAAK,cACnB,IAAe,KAAK,eACpB,EAAE,SAAM,cAAW,KAAK,QAAQ,MAAM;EAE5C,IAAI,EAAM,UAAU,KAAe,EAAM,WAAW,GAUhD,OATA,EACI,EAAM,MACN,GACA,GACA,KAAK,OACL,KAAK,MACL,GACA,CACJ,GACO;GAAE;GAAM;EAAO;EAG1B,IAAM,IAAW,KAAU,KAAK,aAAa,CAAK,GAC5C,IAAU,KAAK,cAAc;EACnC,EAAQ,UACJ,GACA,GACA,GACA,EAAM,OACN,EAAM,QACN,GACA,GACA,GACA,CACJ;EAEA,IAAM,IAAO,EAAQ,aAAa,GAAG,GAAG,GAAa,CAAY,CAAC,CAAC;EAEnE,OADA,EAAmB,GAAM,GAAa,GAAc,KAAK,OAAO,KAAK,MAAM,CAAI,GACxE;GAAE;GAAM;EAAO;CAC1B;CAkBA,QAAQ,GAA6C;EAC7C,OAAW,KAAA,KAAc,EAAO,WACpC,KAAK,QAAQ,QAAQ;CACzB;CASA,gBAAmC;EAO/B,OANI,KAAK,YAAY,SACjB,KAAK,UAAU,EAAa,KAAK,cAAc,KAAK,aAAa,GACjE,KAAK,iBAAiB,EAAa,KAAK,SAAS,EAAE,oBAAoB,GAAK,CAAC,GAC7E,KAAK,eAAe,wBAAwB,IAC5C,KAAK,eAAe,wBAAwB,SAEzC,KAAK;CAChB;CAOA,aAAqB,GAAoC;EACrD,IAAM,EAAE,UAAO,cAAW;EAU1B,QARI,KAAK,YAAY,QACjB,KAAK,QAAQ,UAAU,KACvB,KAAK,QAAQ,WAAW,OAExB,KAAK,UAAU,EAAa,GAAO,CAAM,GACzC,KAAK,iBAAiB,EAAa,KAAK,OAAO,IAEnD,KAAM,eAA6B,aAAa,EAAe,CAAK,GAAG,GAAG,CAAC,GACpE,KAAK;CAChB;AACJ;AAcA,SAAgB,EACZ,GACA,GACA,GACA,IAA0C;CAAC;CAAG;CAAG;AAAC,GAClD,IAAyC;CAAC;CAAG;CAAG;AAAC,GAChC;CACjB,OAAO,IAAI,EAAe,GAAa,GAAc,GAAM,CAAG,CAAC,CAAC,IAAI,CAAK;AAC7E;AAQA,SAAgB,EAAe,GAAe,GAA8B;CACxE,OAAO,IAAI,aAAa,IAAI,IAAS,CAAK;AAC9C;AAaA,SAAgB,EACZ,GACA,GACA,GACA,IAA0C;CAAC;CAAK;CAAK;AAAG,GACpC;CACpB,OAAO,IAAI,EAAkB,GAAa,GAAc,CAAI,CAAC,CAAC,IAAI,CAAK;AAC3E"}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const e=require("../core/exceptions.cjs");var
|
|
1
|
+
const e=require("../core/exceptions.cjs"),t=require("../types.cjs");var n=class{_session;constructor(e){this._session=e}get session(){return this._session}};function r(e){return e.toFixed(6).replace(/0+$/,``).replace(/\.$/,``)}function i(t,n){if(!n.raiseOnEmpty||t>0)return;let i=n.path?` in ${n.path}`:``,a=n.classes===void 0?``:` among classes [${[...n.classes].sort((e,t)=>e-t).join(`, `)}]`;throw new e.NoDetectionsError(`No detections${i}${a}: nothing cleared confThreshold=${r(n.confThreshold)}.`)}function a(e,n){let[r,i,a,o]=n.asIntXyxy(),s=Math.max(0,r),c=Math.max(0,i),l=Math.min(e.width,a),u=Math.min(e.height,o);if(l<=s||u<=c)return new t.RGBImage(new Uint8Array,0,0);let d=l-s,f=u-c,p=new Uint8Array(d*f*3);for(let t=0;t<f;t++){let n=((c+t)*e.width+s)*3;p.set(e.data.subarray(n,n+d*3),t*d*3)}return new t.RGBImage(p,d,f)}function o(e){let t=!1,n;return()=>(t||=(n=e(),!0),n)}exports.VisionTask=n,exports.cropToBox=a,exports.memoize=o,exports.requireDetections=i;
|
|
2
2
|
//# sourceMappingURL=base.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base.cjs","names":[],"sources":["../../../src/vision/tasks/base.ts"],"sourcesContent":["/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */\n/**\n * Common foundation for task-oriented vision SDK objects.\n *\n * The base only owns the {@link OrtSession} — label resolution lives in each\n * subclass because how `numClasses` is read from the model differs per task.\n */\n\nimport { NoDetectionsError } from \"../core/exceptions\";\nimport type { OrtSession } from \"../core/session\";\n\nexport abstract class VisionTask {\n protected constructor(protected readonly _session: OrtSession) {}\n\n /** The underlying {@link OrtSession} used to run inference. */\n get session(): OrtSession {\n return this._session;\n }\n}\n\n/**\n * Render a confidence threshold the way the Python SDK renders it.\n *\n * JavaScript and Python disagree on how a number becomes text. A whole\n * threshold is `1` here and `1.0` there; JavaScript holds off on exponent\n * notation until `1e-7` while Python switches at `1e-5`. A fused pipeline is\n * built once and runs under both runtimes from the same file, so a message that\n * quotes the threshold has to quote it identically — otherwise the two SDKs\n * describe the same run with two different numbers.\n *\n * Six decimals with the trailing zeros trimmed covers every threshold a caller\n * can meaningfully set and agrees byte for byte with `_format_threshold` in the\n * Python SDK's `tasks/base.py`. The pairing is fixed by a shared table in both\n * test suites, so a change on one side that is not mirrored on the other fails.\n *\n * @param value The threshold to render.\n * @returns The threshold as text, without trailing zeros or a trailing dot.\n */\nfunction formatThreshold(value: number): string {\n return value.toFixed(6).replace(/0+$/, \"\").replace(/\\.$/, \"\");\n}\n\n/**\n * Turn an empty result into an error, when the caller asked for that.\n *\n * Shared by every task that can come back with nothing — {@link Detector},\n * {@link Segmenter} and {@link DetectClassify} — so the three agree on when\n * they throw and on what the message says. The message names the two settings\n * that decide the outcome, because \"no detections\" on its own leaves the reader\n * unable to tell a blank image from a threshold set too high.\n *\n * @param count How many detections survived every filter.\n * @param options The flag for this call, the threshold actually applied (after\n * any per-call override), the class allowlist if one narrowed the search, and\n * the source path when the input was one.\n * @throws {@link NoDetectionsError} when the flag is set and `count` is zero.\n */\nexport function requireDetections(\n count: number,\n options: {\n readonly raiseOnEmpty: boolean;\n readonly confThreshold: number;\n readonly classes: readonly number[] | undefined;\n readonly path: string | null;\n },\n): void {\n if (!options.raiseOnEmpty || count > 0) return;\n const where = options.path ? ` in ${options.path}` : \"\";\n const narrowed =\n options.classes === undefined\n ? \"\"\n : ` among classes [${[...options.classes].sort((a, b) => a - b).join(\", \")}]`;\n throw new NoDetectionsError(\n `No detections${where}${narrowed}: nothing cleared confThreshold=${formatThreshold(options.confThreshold)}.`,\n );\n}\n"],"mappings":"
|
|
1
|
+
{"version":3,"file":"base.cjs","names":[],"sources":["../../../src/vision/tasks/base.ts"],"sourcesContent":["/** @generated Vendored from @mauriciobenjamin700/ort-vision-sdk-web. Do not hand-edit — regenerate with `npm run vendor:vision`. */\n/**\n * Common foundation for task-oriented vision SDK objects.\n *\n * The base only owns the {@link OrtSession} — label resolution lives in each\n * subclass because how `numClasses` is read from the model differs per task.\n */\n\nimport { NoDetectionsError } from \"../core/exceptions\";\nimport type { OrtSession } from \"../core/session\";\nimport { type BoundingBox, RGBImage } from \"../types\";\n\nexport abstract class VisionTask {\n protected constructor(protected readonly _session: OrtSession) {}\n\n /** The underlying {@link OrtSession} used to run inference. */\n get session(): OrtSession {\n return this._session;\n }\n}\n\n/**\n * Render a confidence threshold the way the Python SDK renders it.\n *\n * JavaScript and Python disagree on how a number becomes text. A whole\n * threshold is `1` here and `1.0` there; JavaScript holds off on exponent\n * notation until `1e-7` while Python switches at `1e-5`. A fused pipeline is\n * built once and runs under both runtimes from the same file, so a message that\n * quotes the threshold has to quote it identically — otherwise the two SDKs\n * describe the same run with two different numbers.\n *\n * Six decimals with the trailing zeros trimmed covers every threshold a caller\n * can meaningfully set and agrees byte for byte with `_format_threshold` in the\n * Python SDK's `tasks/base.py`. The pairing is fixed by a shared table in both\n * test suites, so a change on one side that is not mirrored on the other fails.\n *\n * @param value The threshold to render.\n * @returns The threshold as text, without trailing zeros or a trailing dot.\n */\nfunction formatThreshold(value: number): string {\n return value.toFixed(6).replace(/0+$/, \"\").replace(/\\.$/, \"\");\n}\n\n/**\n * Turn an empty result into an error, when the caller asked for that.\n *\n * Shared by every task that can come back with nothing — {@link Detector},\n * {@link Segmenter} and {@link DetectClassify} — so the three agree on when\n * they throw and on what the message says. The message names the two settings\n * that decide the outcome, because \"no detections\" on its own leaves the reader\n * unable to tell a blank image from a threshold set too high.\n *\n * @param count How many detections survived every filter.\n * @param options The flag for this call, the threshold actually applied (after\n * any per-call override), the class allowlist if one narrowed the search, and\n * the source path when the input was one.\n * @throws {@link NoDetectionsError} when the flag is set and `count` is zero.\n */\nexport function requireDetections(\n count: number,\n options: {\n readonly raiseOnEmpty: boolean;\n readonly confThreshold: number;\n readonly classes: readonly number[] | undefined;\n readonly path: string | null;\n },\n): void {\n if (!options.raiseOnEmpty || count > 0) return;\n const where = options.path ? ` in ${options.path}` : \"\";\n const narrowed =\n options.classes === undefined\n ? \"\"\n : ` among classes [${[...options.classes].sort((a, b) => a - b).join(\", \")}]`;\n throw new NoDetectionsError(\n `No detections${where}${narrowed}: nothing cleared confThreshold=${formatThreshold(options.confThreshold)}.`,\n );\n}\n\n/**\n * Cut the box region out of an image.\n *\n * The box is truncated to whole pixels and clamped to the image, the same\n * rounding every task applies, so a crop always matches `bbox.asIntXyxy()`\n * within the frame.\n *\n * @param image The source image.\n * @param bbox The box, in the image's pixel coordinates.\n * @returns The cropped region, or a zero-sized image for a box with no area.\n */\nexport function cropToBox(image: RGBImage, bbox: BoundingBox): RGBImage {\n const [rawX1, rawY1, rawX2, rawY2] = bbox.asIntXyxy();\n const x1 = Math.max(0, rawX1);\n const y1 = Math.max(0, rawY1);\n const x2 = Math.min(image.width, rawX2);\n const y2 = Math.min(image.height, rawY2);\n if (x2 <= x1 || y2 <= y1) return new RGBImage(new Uint8Array(0), 0, 0);\n\n const width = x2 - x1;\n const height = y2 - y1;\n const out = new Uint8Array(width * height * 3);\n for (let row = 0; row < height; row++) {\n const offset = ((y1 + row) * image.width + x1) * 3;\n out.set(image.data.subarray(offset, offset + width * 3), row * width * 3);\n }\n return new RGBImage(out, width, height);\n}\n\n/**\n * Wrap a computation so it runs on first call and is remembered after.\n *\n * Backs the per-detection images (`croppedImage`, `segmentedImage`): most\n * callers read a box and a class and never look at the pixels, yet building\n * them eagerly copied every box region out of the frame on every\n * `predict()` — up to `maxDetections` allocations nobody asked for.\n *\n * @param compute Produces the value; called at most once.\n * @returns A getter returning the value, computing it on the first call.\n */\nexport function memoize<T>(compute: () => T): () => T {\n let done = false;\n let value: T;\n return () => {\n if (!done) {\n value = compute();\n done = true;\n }\n return value;\n };\n}\n"],"mappings":"oEAYA,IAAsB,EAAtB,KAAiC,CACY,SAAzC,YAAsB,EAAyC,CAAtB,KAAA,SAAA,CAAuB,CAGhE,IAAI,SAAsB,CACtB,OAAO,KAAK,QAChB,CACJ,EAoBA,SAAS,EAAgB,EAAuB,CAC5C,OAAO,EAAM,QAAQ,CAAC,CAAC,CAAC,QAAQ,MAAO,EAAE,CAAC,CAAC,QAAQ,MAAO,EAAE,CAChE,CAiBA,SAAgB,EACZ,EACA,EAMI,CACJ,GAAI,CAAC,EAAQ,cAAgB,EAAQ,EAAG,OACxC,IAAM,EAAQ,EAAQ,KAAO,OAAO,EAAQ,OAAS,GAC/C,EACF,EAAQ,UAAY,IAAA,GACd,GACA,mBAAmB,CAAC,GAAG,EAAQ,OAAO,CAAC,CAAC,MAAM,EAAG,IAAM,EAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,GACnF,MAAM,IAAI,EAAA,kBACN,gBAAgB,IAAQ,EAAS,kCAAkC,EAAgB,EAAQ,aAAa,EAAE,EAC9G,CACJ,CAaA,SAAgB,EAAU,EAAiB,EAA6B,CACpE,GAAM,CAAC,EAAO,EAAO,EAAO,GAAS,EAAK,UAAU,EAC9C,EAAK,KAAK,IAAI,EAAG,CAAK,EACtB,EAAK,KAAK,IAAI,EAAG,CAAK,EACtB,EAAK,KAAK,IAAI,EAAM,MAAO,CAAK,EAChC,EAAK,KAAK,IAAI,EAAM,OAAQ,CAAK,EACvC,GAAI,GAAM,GAAM,GAAM,EAAI,OAAO,IAAI,EAAA,SAAS,IAAI,WAAe,EAAG,CAAC,EAErE,IAAM,EAAQ,EAAK,EACb,EAAS,EAAK,EACd,EAAM,IAAI,WAAW,EAAQ,EAAS,CAAC,EAC7C,IAAK,IAAI,EAAM,EAAG,EAAM,EAAQ,IAAO,CACnC,IAAM,IAAW,EAAK,GAAO,EAAM,MAAQ,GAAM,EACjD,EAAI,IAAI,EAAM,KAAK,SAAS,EAAQ,EAAS,EAAQ,CAAC,EAAG,EAAM,EAAQ,CAAC,CAC5E,CACA,OAAO,IAAI,EAAA,SAAS,EAAK,EAAO,CAAM,CAC1C,CAaA,SAAgB,EAAW,EAA2B,CAClD,IAAI,EAAO,GACP,EACJ,WACI,AAEI,KADA,EAAQ,EAAQ,EACT,IAEJ,EAEf"}
|