zorya-classifier 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/README.md +24 -8
  2. package/dist/app/classifier-config.d.ts +30 -0
  3. package/dist/app/classifier-config.js +46 -0
  4. package/dist/app/classifier-contracts.d.ts +32 -0
  5. package/dist/app/classifier-contracts.js +1 -0
  6. package/dist/app/classifier-representation.d.ts +5 -0
  7. package/dist/app/classifier-representation.js +33 -0
  8. package/dist/app/classifier-runtime.d.ts +7 -0
  9. package/dist/app/classifier-runtime.js +71 -0
  10. package/dist/app/classifier-spatial.d.ts +31 -0
  11. package/dist/app/classifier-spatial.js +78 -0
  12. package/dist/app/cli-arguments.d.ts +2 -2
  13. package/dist/app/create-classifier.d.ts +4 -29
  14. package/dist/app/create-classifier.js +37 -237
  15. package/dist/cli.js +1 -1
  16. package/dist/entities/classification.d.ts +98 -0
  17. package/dist/entities/classification.js +28 -0
  18. package/dist/entities/contracts.d.ts +6 -471
  19. package/dist/entities/contracts.js +6 -97
  20. package/dist/entities/decision.d.ts +211 -0
  21. package/dist/entities/decision.js +33 -0
  22. package/dist/entities/geometry.d.ts +37 -0
  23. package/dist/entities/geometry.js +26 -0
  24. package/dist/entities/image.d.ts +39 -0
  25. package/dist/entities/image.js +15 -0
  26. package/dist/entities/localization.d.ts +125 -0
  27. package/dist/entities/localization.js +13 -0
  28. package/dist/entities/object-proposals.d.ts +4 -1
  29. package/dist/entities/segmentation-metadata.d.ts +70 -0
  30. package/dist/entities/segmentation-metadata.js +21 -1
  31. package/dist/entities/segmentation-verification.d.ts +1 -0
  32. package/dist/entities/segmentation.d.ts +6 -1
  33. package/dist/entities/validation.d.ts +2 -0
  34. package/dist/entities/validation.js +9 -0
  35. package/dist/entities/whole-object.d.ts +4 -0
  36. package/dist/features/choice-beam-layout.js +2 -7
  37. package/dist/features/proposals/presentation.d.ts +21 -0
  38. package/dist/features/proposals/presentation.js +122 -0
  39. package/dist/features/proposals/validation.d.ts +138 -0
  40. package/dist/features/proposals/validation.js +125 -0
  41. package/dist/features/segmentation/box-check.js +10 -23
  42. package/dist/features/segmentation/mask.js +2 -7
  43. package/dist/features/segmentation/region-result.d.ts +18 -0
  44. package/dist/features/segmentation/region-result.js +171 -0
  45. package/dist/features/segmentation/source-mask.d.ts +6 -0
  46. package/dist/features/segmentation/source-mask.js +32 -0
  47. package/dist/features/segmentation/validation.d.ts +9 -0
  48. package/dist/features/segmentation/validation.js +94 -0
  49. package/dist/infrastructure/decision-transport/errors.d.ts +8 -0
  50. package/dist/infrastructure/decision-transport/errors.js +34 -0
  51. package/dist/infrastructure/decision-transport/images.d.ts +1 -0
  52. package/dist/infrastructure/decision-transport/images.js +52 -0
  53. package/dist/infrastructure/{openrouter-common.d.ts → decision-transport/request.d.ts} +6 -15
  54. package/dist/infrastructure/decision-transport/request.js +162 -0
  55. package/dist/infrastructure/decision-transport/response.d.ts +1 -0
  56. package/dist/infrastructure/decision-transport/response.js +72 -0
  57. package/dist/infrastructure/decision-transport/route-identity.d.ts +11 -0
  58. package/dist/infrastructure/decision-transport/route-identity.js +23 -0
  59. package/dist/infrastructure/openrouter-vision.js +1 -1
  60. package/dist/infrastructure/openrouter.js +7 -19
  61. package/dist/infrastructure/raster.js +3 -8
  62. package/dist/infrastructure/segmentation/grabcut/index.d.ts +5 -1
  63. package/dist/infrastructure/segmentation/grabcut/index.js +6 -5
  64. package/dist/infrastructure/segmentation/grabcut/masks.d.ts +49 -0
  65. package/dist/infrastructure/segmentation/grabcut/masks.js +89 -0
  66. package/dist/infrastructure/segmentation/grabcut/model-asset.d.ts +4 -0
  67. package/dist/infrastructure/segmentation/grabcut/model-asset.js +4 -0
  68. package/dist/infrastructure/segmentation/grabcut/runtime.d.ts +70 -0
  69. package/dist/infrastructure/segmentation/grabcut/runtime.js +96 -0
  70. package/dist/infrastructure/segmentation/grabcut/worker-inference.d.ts +4 -43
  71. package/dist/infrastructure/segmentation/grabcut/worker-inference.js +4 -187
  72. package/dist/infrastructure/segmentation/mobile-sam/index.d.ts +3 -58
  73. package/dist/infrastructure/segmentation/mobile-sam/index.js +2 -472
  74. package/dist/infrastructure/segmentation/mobile-sam/input.d.ts +46 -0
  75. package/dist/infrastructure/segmentation/mobile-sam/input.js +83 -0
  76. package/dist/infrastructure/segmentation/mobile-sam/model-assets.d.ts +8 -0
  77. package/dist/infrastructure/segmentation/mobile-sam/model-assets.js +8 -0
  78. package/dist/infrastructure/segmentation/mobile-sam/object-proposals.d.ts +7 -0
  79. package/dist/infrastructure/segmentation/mobile-sam/object-proposals.js +124 -0
  80. package/dist/infrastructure/segmentation/mobile-sam/options.d.ts +43 -0
  81. package/dist/infrastructure/segmentation/mobile-sam/options.js +19 -0
  82. package/dist/infrastructure/segmentation/mobile-sam/proposal-masks.d.ts +26 -0
  83. package/dist/infrastructure/segmentation/mobile-sam/proposal-masks.js +71 -0
  84. package/dist/infrastructure/segmentation/mobile-sam/segmentation-masks.d.ts +112 -0
  85. package/dist/infrastructure/segmentation/mobile-sam/segmentation-masks.js +230 -0
  86. package/dist/infrastructure/segmentation/mobile-sam/segmenter.d.ts +22 -0
  87. package/dist/infrastructure/segmentation/mobile-sam/segmenter.js +268 -0
  88. package/dist/infrastructure/segmentation/mobile-sam/worker-expand.d.ts +7 -0
  89. package/dist/infrastructure/segmentation/mobile-sam/worker-expand.js +188 -0
  90. package/dist/infrastructure/segmentation/mobile-sam/worker-inference.d.ts +6 -149
  91. package/dist/infrastructure/segmentation/mobile-sam/worker-inference.js +6 -845
  92. package/dist/infrastructure/segmentation/mobile-sam/worker-models.d.ts +14 -0
  93. package/dist/infrastructure/segmentation/mobile-sam/worker-models.js +72 -0
  94. package/dist/infrastructure/segmentation/mobile-sam/worker-proposals.d.ts +7 -0
  95. package/dist/infrastructure/segmentation/mobile-sam/worker-proposals.js +159 -0
  96. package/dist/infrastructure/segmentation/mobile-sam/worker-segment.d.ts +4 -0
  97. package/dist/infrastructure/segmentation/mobile-sam/worker-segment.js +126 -0
  98. package/dist/infrastructure/segmentation/native-protocol.d.ts +59 -34
  99. package/dist/infrastructure/segmentation/native-protocol.js +26 -16
  100. package/dist/infrastructure/segmentation/proposal-presentation-input.d.ts +20 -0
  101. package/dist/infrastructure/segmentation/proposal-presentation-input.js +107 -0
  102. package/dist/infrastructure/segmentation/proposal-presentation-pixels.d.ts +8 -0
  103. package/dist/infrastructure/segmentation/proposal-presentation-pixels.js +93 -0
  104. package/dist/infrastructure/segmentation/proposal-presentation.js +10 -201
  105. package/dist/infrastructure/vision-chat/options.d.ts +62 -0
  106. package/dist/infrastructure/vision-chat/options.js +46 -0
  107. package/dist/infrastructure/vision-chat/request.d.ts +23 -0
  108. package/dist/infrastructure/vision-chat/request.js +109 -0
  109. package/dist/infrastructure/vision-chat/response.d.ts +3 -0
  110. package/dist/infrastructure/vision-chat/response.js +105 -0
  111. package/dist/infrastructure/vision-chat.d.ts +4 -61
  112. package/dist/infrastructure/vision-chat.js +8 -271
  113. package/dist/openai-compatible.js +1 -1
  114. package/dist/processes/extract-from-region.js +11 -216
  115. package/dist/processes/extract.d.ts +2 -8
  116. package/dist/processes/extract.js +12 -128
  117. package/dist/processes/proposal-location.js +5 -251
  118. package/docs/0.2.0-plan.md +3 -3
  119. package/docs/README.ru.md +42 -23
  120. package/docs/acceptance.md +5 -5
  121. package/docs/agents.ru.md +95 -0
  122. package/docs/api.md +4 -2
  123. package/docs/architecture.md +33 -5
  124. package/docs/architecture.ru.md +146 -0
  125. package/docs/benchmarks.md +4 -2
  126. package/docs/development.ru.md +141 -0
  127. package/docs/operations.md +4 -4
  128. package/docs/provenance.json +5 -1
  129. package/docs/publishing.md +8 -0
  130. package/docs/refactoring-plan.ru.md +183 -0
  131. package/docs/v02-usage.md +8 -6
  132. package/docs/vision-results.md +2 -0
  133. package/package.json +5 -3
  134. package/dist/infrastructure/boundary-view.d.ts +0 -2
  135. package/dist/infrastructure/boundary-view.js +0 -263
  136. package/dist/infrastructure/openrouter-common.js +0 -317
  137. package/dist/infrastructure/text-representation.d.ts +0 -23
  138. package/dist/infrastructure/text-representation.js +0 -226
  139. package/docs/evidence/v02/categorical-heldout-v4/README.md +0 -35
  140. package/docs/evidence/v02/proposal-resource-soak-261dc23.md +0 -15
@@ -1,845 +1,6 @@
1
- import { createHash } from 'node:crypto';
2
- import { open } from 'node:fs/promises';
3
- import sharp from 'sharp';
4
- import { packBinaryMask } from '../../../features/segmentation/packed-mask.js';
5
- import { objectProposalLimits } from '../../../entities/object-proposals.js';
6
- const encoderIdentity = {
7
- bytes: 28_157_093,
8
- sha256: '580f5fb648ea1062c0aabc26217aed56921985f03f0cbbd852bba81d760cc749',
9
- };
10
- const decoderIdentity = {
11
- bytes: 16_496_559,
12
- sha256: '8976b90a87ba50a6a72217a5ff994f7d25ce16f2229fcc1ed259e1294c622ffe',
13
- };
14
- async function verifiedBytes(file, identity) {
15
- const handle = await open(file, 'r');
16
- try {
17
- const stat = await handle.stat();
18
- if (!stat.isFile() || stat.size !== identity.bytes)
19
- throw new Error('Pinned model size mismatch');
20
- const bytes = Buffer.alloc(identity.bytes);
21
- let offset = 0;
22
- while (offset < bytes.length) {
23
- const read = await handle.read(bytes, offset, bytes.length - offset, null);
24
- if (read.bytesRead === 0)
25
- throw new Error('Pinned model read was incomplete');
26
- offset += read.bytesRead;
27
- }
28
- if ((await handle.read(Buffer.alloc(1), 0, 1, null)).bytesRead !== 0)
29
- throw new Error('Pinned model changed while reading');
30
- if (createHash('sha256').update(bytes).digest('hex') !== identity.sha256)
31
- throw new Error('Pinned model SHA256 mismatch');
32
- return bytes;
33
- }
34
- finally {
35
- await handle.close();
36
- }
37
- }
38
- export async function probeMobileSam(options) {
39
- const ort = await import('onnxruntime-node');
40
- if (!ort
41
- .listSupportedBackends()
42
- .some((item) => item.name === options.executionProvider && item.bundled))
43
- throw new Error('Explicitly requested ONNX execution provider is unavailable');
44
- const sessionOptions = {
45
- executionProviders: options.executionProvider === 'dml'
46
- ? [{ name: 'dml', deviceId: options.deviceId ?? 0 }]
47
- : ['cpu'],
48
- intraOpNumThreads: options.threads,
49
- interOpNumThreads: 1,
50
- executionMode: 'sequential',
51
- enableMemPattern: options.executionProvider !== 'dml',
52
- graphOptimizationLevel: 'all',
53
- };
54
- const encoderBytes = await verifiedBytes(options.encoderPath, encoderIdentity);
55
- const decoderBytes = await verifiedBytes(options.decoderPath, decoderIdentity);
56
- let encoder;
57
- let decoder;
58
- let releaseFailed = false;
59
- try {
60
- encoder = await ort.InferenceSession.create(encoderBytes, sessionOptions);
61
- decoder = await ort.InferenceSession.create(decoderBytes, sessionOptions);
62
- if (!encoder.inputNames.includes('input_image') ||
63
- !encoder.outputNames.includes('image_embeddings') ||
64
- !decoder.inputNames.includes('image_embeddings') ||
65
- !decoder.outputNames.includes('masks') ||
66
- !decoder.outputNames.includes('iou_predictions'))
67
- throw new Error('Pinned MobileSAM graph tensor names are incompatible');
68
- }
69
- finally {
70
- const closed = await Promise.allSettled([
71
- ...(encoder ? [encoder.release()] : []),
72
- ...(decoder ? [decoder.release()] : []),
73
- ]);
74
- releaseFailed = closed.some((item) => item.status === 'rejected');
75
- }
76
- if (releaseFailed)
77
- throw new Error('MobileSAM probe session release failed');
78
- }
79
- function inside(rect, x, y) {
80
- return x >= rect.left && y >= rect.top && x < rect.left + rect.width && y < rect.top + rect.height;
81
- }
82
- /** Sort by model quality and use decoder token order as a deterministic tie break. */
83
- export function selectBestMaskCandidate(candidates) {
84
- return [...candidates].sort((a, b) => b.score - a.score || a.token - b.token)[0];
85
- }
86
- export function mobileSamSegmentationPrompt(input) {
87
- if (input.prompt === 'whole-region')
88
- return {
89
- coordinates: Float32Array.of(input.wholeRegion.left * input.xScale, input.wholeRegion.top * input.yScale, (input.wholeRegion.left + input.wholeRegion.width) * input.xScale, (input.wholeRegion.top + input.wholeRegion.height) * input.yScale),
90
- labels: Float32Array.of(2, 3),
91
- };
92
- const centerX = input.seedRegion.left + (input.seedRegion.width - 1) / 2;
93
- const centerY = input.seedRegion.top + (input.seedRegion.height - 1) / 2;
94
- const pointX = centerX * input.xScale;
95
- const pointY = centerY * input.yScale;
96
- return {
97
- coordinates: Float32Array.of(pointX, pointY, pointX, pointY),
98
- labels: Float32Array.of(1, -1),
99
- };
100
- }
101
- export function collectMobileSamSegmentationCandidates(input) {
102
- const pixels = input.width * input.height;
103
- if (input.rgba.length !== pixels * 4 ||
104
- input.logits.length !== pixels * 4 ||
105
- input.scores.length !== 4)
106
- throw new Error('Invalid MobileSAM decoder output dimensions');
107
- const candidates = [];
108
- for (let token = 0; token < 4; token++) {
109
- const mask = new Uint8Array(pixels);
110
- let seedHit = false;
111
- for (let index = 0; index < pixels; index++) {
112
- const value = input.logits[token * pixels + index];
113
- if (value === undefined || !Number.isFinite(value))
114
- throw new Error('Non-finite mask logit');
115
- if (value <= 0)
116
- continue;
117
- const x = index % input.width;
118
- const y = Math.floor(index / input.width);
119
- if (!inside(input.roi, x, y))
120
- continue;
121
- if (input.prompt === 'seed-region' && input.rgba[index * 4 + 3] === 0)
122
- continue;
123
- mask[index] = 1;
124
- if (inside(input.seedRegion, x, y))
125
- seedHit = true;
126
- }
127
- const score = input.scores[token];
128
- if (score === undefined || !Number.isFinite(score))
129
- throw new Error('Non-finite MobileSAM quality diagnostic');
130
- if (seedHit)
131
- candidates.push({ token, score, mask });
132
- }
133
- return candidates;
134
- }
135
- export function selectMobileSamSegmentationMask(input) {
136
- const best = selectBestMaskCandidate(input.candidates);
137
- if (best)
138
- return best.mask;
139
- if (input.prompt === 'seed-region')
140
- return new Uint8Array(input.width * input.height);
141
- throw new Error('MobileSAM produced no seed-grounded mask');
142
- }
143
- /** Full-source segmentation masks use the segmentation pixel cap, not the proposal cap. */
144
- export function packFullSourceBinaryMask(mask, width, height) {
145
- const pixels = Number.isSafeInteger(width) && Number.isSafeInteger(height) ? width * height : NaN;
146
- if (!Number.isSafeInteger(width) ||
147
- !Number.isSafeInteger(height) ||
148
- width < 1 ||
149
- height < 1 ||
150
- !Number.isSafeInteger(pixels) ||
151
- pixels > 16_777_216 ||
152
- mask.length !== pixels)
153
- throw new Error('Invalid full-source mask dimensions');
154
- const packed = new Uint8Array(Math.ceil(pixels / 8));
155
- for (let index = 0; index < pixels; index++) {
156
- const value = mask[index];
157
- if (value !== 0 && value !== 1)
158
- throw new Error('Full-source mask is not binary');
159
- if (value === 1)
160
- packed[index >> 3] = (packed[index >> 3] ?? 0) | (1 << (index & 7));
161
- }
162
- return packed;
163
- }
164
- /** Strictly decode the LSB-first full-source mask representation used by expansion IPC. */
165
- export function unpackFullSourceBinaryMask(data, width, height) {
166
- const pixels = Number.isSafeInteger(width) && Number.isSafeInteger(height) ? width * height : NaN;
167
- const expectedBytes = Number.isSafeInteger(pixels) ? Math.ceil(pixels / 8) : NaN;
168
- if (!Number.isSafeInteger(width) ||
169
- !Number.isSafeInteger(height) ||
170
- width < 1 ||
171
- height < 1 ||
172
- !Number.isSafeInteger(pixels) ||
173
- pixels > 16_777_216 ||
174
- data.length !== expectedBytes)
175
- throw new Error('Invalid packed full-source mask dimensions');
176
- const unusedBits = expectedBytes * 8 - pixels;
177
- if (unusedBits > 0) {
178
- const allowed = (1 << (8 - unusedBits)) - 1;
179
- if (((data[expectedBytes - 1] ?? 0) & ~allowed) !== 0)
180
- throw new Error('Packed full-source mask has nonzero padding bits');
181
- }
182
- const output = new Uint8Array(pixels);
183
- for (let index = 0; index < pixels; index++)
184
- output[index] = ((data[index >> 3] ?? 0) >> (index & 7)) & 1;
185
- return output;
186
- }
187
- /** Validate a prior full-source mask and return its foreground area and bounds. */
188
- export function validateMobileSamPreviousMask(input) {
189
- const pixels = Number.isSafeInteger(input.width) && Number.isSafeInteger(input.height)
190
- ? input.width * input.height
191
- : NaN;
192
- if (!Number.isSafeInteger(input.width) ||
193
- !Number.isSafeInteger(input.height) ||
194
- input.width < 1 ||
195
- input.height < 1 ||
196
- !Number.isSafeInteger(pixels) ||
197
- pixels > 16_777_216 ||
198
- input.rgba.length !== pixels * 4 ||
199
- input.previousMask.length !== pixels ||
200
- !Number.isSafeInteger(input.roi.left) ||
201
- !Number.isSafeInteger(input.roi.top) ||
202
- !Number.isSafeInteger(input.roi.width) ||
203
- !Number.isSafeInteger(input.roi.height) ||
204
- input.roi.left < 0 ||
205
- input.roi.top < 0 ||
206
- input.roi.width < 1 ||
207
- input.roi.height < 1 ||
208
- input.roi.left + input.roi.width > input.width ||
209
- input.roi.top + input.roi.height > input.height)
210
- throw new Error('Invalid previous mask dimensions or ROI');
211
- let area = 0;
212
- let left = input.width;
213
- let top = input.height;
214
- let right = -1;
215
- let bottom = -1;
216
- for (let index = 0; index < pixels; index++) {
217
- const value = input.previousMask[index];
218
- if (value !== 0 && value !== 1)
219
- throw new Error('Previous mask is not binary');
220
- if (value === 0)
221
- continue;
222
- const x = index % input.width;
223
- const y = Math.floor(index / input.width);
224
- if (!inside(input.roi, x, y) || input.rgba[index * 4 + 3] === 0)
225
- throw new Error('Previous mask foreground is outside visible ROI pixels');
226
- area++;
227
- left = Math.min(left, x);
228
- top = Math.min(top, y);
229
- right = Math.max(right, x);
230
- bottom = Math.max(bottom, y);
231
- }
232
- if (area === 0)
233
- throw new Error('Previous mask has no foreground');
234
- return { area, bounds: { left, top, right, bottom } };
235
- }
236
- /** Select the smallest actual decoder mask that strictly encloses prior foreground. */
237
- export function selectEnclosingMobileSamCandidate(input) {
238
- const pixels = input.width * input.height;
239
- const previous = validateMobileSamPreviousMask(input);
240
- if (input.logits.length !== pixels * 4 || input.scores.length !== 4)
241
- throw new Error('Invalid MobileSAM decoder output dimensions');
242
- let best;
243
- let currentMask;
244
- try {
245
- for (let token = 0; token < 4; token++) {
246
- const score = input.scores[token];
247
- if (score === undefined || !Number.isFinite(score))
248
- throw new Error('Non-finite MobileSAM quality diagnostic');
249
- const mask = new Uint8Array(pixels);
250
- currentMask = mask;
251
- let area = 0;
252
- let left = input.width;
253
- let top = input.height;
254
- let right = -1;
255
- let bottom = -1;
256
- let containsPrevious = true;
257
- const base = token * pixels;
258
- for (let index = 0; index < pixels; index++) {
259
- const value = input.logits[base + index];
260
- if (value === undefined || !Number.isFinite(value))
261
- throw new Error('Non-finite MobileSAM mask logit');
262
- const x = index % input.width;
263
- const y = Math.floor(index / input.width);
264
- const foreground = value > 0 && input.rgba[index * 4 + 3] !== 0 && inside(input.roi, x, y);
265
- if (input.previousMask[index] === 1 && !foreground)
266
- containsPrevious = false;
267
- if (!foreground)
268
- continue;
269
- mask[index] = 1;
270
- area++;
271
- left = Math.min(left, x);
272
- top = Math.min(top, y);
273
- right = Math.max(right, x);
274
- bottom = Math.max(bottom, y);
275
- }
276
- const strictBoundsGrowth = left <= previous.bounds.left &&
277
- top <= previous.bounds.top &&
278
- right >= previous.bounds.right &&
279
- bottom >= previous.bounds.bottom &&
280
- (left < previous.bounds.left ||
281
- top < previous.bounds.top ||
282
- right > previous.bounds.right ||
283
- bottom > previous.bounds.bottom);
284
- if (!containsPrevious || area <= previous.area || !strictBoundsGrowth) {
285
- mask.fill(0);
286
- continue;
287
- }
288
- const candidate = { token, score, mask, area, bounds: { left, top, right, bottom } };
289
- if (!best ||
290
- candidate.area < best.area ||
291
- (candidate.area === best.area && candidate.score > best.score) ||
292
- (candidate.area === best.area &&
293
- candidate.score === best.score &&
294
- candidate.token < best.token)) {
295
- best?.mask.fill(0);
296
- best = candidate;
297
- }
298
- else
299
- mask.fill(0);
300
- }
301
- return best;
302
- }
303
- catch (error) {
304
- currentMask?.fill(0);
305
- best?.mask.fill(0);
306
- throw error;
307
- }
308
- }
309
- export async function runMobileSamTask(request) {
310
- // Deliberately loaded inside the owned child. Core/entry imports never initialize ONNX Runtime.
311
- const ort = await import('onnxruntime-node');
312
- const providers = ort.listSupportedBackends();
313
- if (!providers.some((item) => item.name === request.options.executionProvider && item.bundled))
314
- throw new Error('Explicitly requested ONNX execution provider is unavailable');
315
- const sessionOptions = {
316
- executionProviders: request.options.executionProvider === 'dml'
317
- ? [{ name: 'dml', deviceId: request.options.deviceId ?? 0 }]
318
- : ['cpu'],
319
- intraOpNumThreads: request.options.threads,
320
- interOpNumThreads: 1,
321
- executionMode: 'sequential',
322
- enableMemPattern: request.options.executionProvider !== 'dml',
323
- graphOptimizationLevel: 'all',
324
- };
325
- const encoder = await ort.InferenceSession.create(await verifiedBytes(request.options.encoderPath, encoderIdentity), sessionOptions);
326
- let decoder;
327
- let result;
328
- let releaseFailed = false;
329
- try {
330
- decoder = await ort.InferenceSession.create(await verifiedBytes(request.options.decoderPath, decoderIdentity), sessionOptions);
331
- const rgba = Buffer.from(request.rgba, 'base64');
332
- if (rgba.length !== request.width * request.height * 4)
333
- throw new Error('Invalid RGBA input size');
334
- const scale = 1024 / Math.max(request.width, request.height);
335
- const resizedWidth = Math.round(request.width * scale);
336
- const resizedHeight = Math.round(request.height * scale);
337
- if (resizedWidth < 1 || resizedHeight < 1)
338
- throw new Error('Unrepresentable image aspect ratio');
339
- const pipeline = sharp(rgba, {
340
- raw: { width: request.width, height: request.height, channels: 4 },
341
- });
342
- let rgb;
343
- try {
344
- rgb = await pipeline
345
- .removeAlpha()
346
- .resize(resizedWidth, resizedHeight, { fit: 'fill', kernel: 'linear' })
347
- .raw()
348
- .toBuffer();
349
- }
350
- finally {
351
- pipeline.destroy();
352
- }
353
- if (rgb.length !== resizedWidth * resizedHeight * 3)
354
- throw new Error('Invalid resized RGB data');
355
- const imageTensor = new ort.Tensor('float32', Float32Array.from(rgb), [
356
- resizedHeight,
357
- resizedWidth,
358
- 3,
359
- ]);
360
- try {
361
- const encoded = await encoder.run({ input_image: imageTensor });
362
- const embedding = encoded.image_embeddings;
363
- if (!(embedding instanceof ort.Tensor) ||
364
- embedding.type !== 'float32' ||
365
- JSON.stringify(embedding.dims) !== JSON.stringify([1, 256, 64, 64]) ||
366
- !(embedding.data instanceof Float32Array))
367
- throw new Error('Unexpected MobileSAM encoder output');
368
- const xScale = resizedWidth / request.width;
369
- const yScale = resizedHeight / request.height;
370
- const prompt = mobileSamSegmentationPrompt({
371
- prompt: request.prompt,
372
- seedRegion: request.seedRegion,
373
- wholeRegion: request.wholeRegion,
374
- xScale,
375
- yScale,
376
- });
377
- const decoded = await decoder.run({
378
- image_embeddings: embedding,
379
- point_coords: new ort.Tensor('float32', prompt.coordinates, [1, 2, 2]),
380
- point_labels: new ort.Tensor('float32', prompt.labels, [1, 2]),
381
- mask_input: new ort.Tensor('float32', new Float32Array(256 * 256), [1, 1, 256, 256]),
382
- has_mask_input: new ort.Tensor('float32', Float32Array.of(0), [1]),
383
- orig_im_size: new ort.Tensor('float32', Float32Array.of(request.height, request.width), [
384
- 2,
385
- ]),
386
- });
387
- const masks = decoded.masks;
388
- const scores = decoded.iou_predictions;
389
- if (!(masks instanceof ort.Tensor) ||
390
- masks.type !== 'float32' ||
391
- JSON.stringify(masks.dims) !== JSON.stringify([1, 4, request.height, request.width]) ||
392
- !(masks.data instanceof Float32Array) ||
393
- !(scores instanceof ort.Tensor) ||
394
- scores.type !== 'float32' ||
395
- JSON.stringify(scores.dims) !== JSON.stringify([1, 4]) ||
396
- !(scores.data instanceof Float32Array) ||
397
- !scores.data.every(Number.isFinite))
398
- throw new Error('Unexpected MobileSAM decoder output');
399
- const candidates = collectMobileSamSegmentationCandidates({
400
- width: request.width,
401
- height: request.height,
402
- rgba,
403
- logits: masks.data,
404
- scores: scores.data,
405
- roi: request.roi,
406
- seedRegion: request.seedRegion,
407
- prompt: request.prompt,
408
- });
409
- const mask = selectMobileSamSegmentationMask({
410
- candidates,
411
- width: request.width,
412
- height: request.height,
413
- prompt: request.prompt,
414
- });
415
- result = {
416
- id: request.id,
417
- status: 'segmented',
418
- width: request.width,
419
- height: request.height,
420
- mask: Buffer.from(mask).toString('base64'),
421
- method: request.prompt === 'seed-region' ? 'mobile-sam-seed-point-v1' : 'mobile-sam-box-prompt',
422
- };
423
- }
424
- finally {
425
- imageTensor.dispose();
426
- }
427
- }
428
- finally {
429
- const releases = await Promise.allSettled([
430
- encoder.release(),
431
- ...(decoder ? [decoder.release()] : []),
432
- ]);
433
- releaseFailed = releases.some((release) => release.status === 'rejected');
434
- }
435
- if (releaseFailed)
436
- throw new Error('MobileSAM native session release failed');
437
- if (!result)
438
- throw new Error('MobileSAM completed without a mask result');
439
- return result;
440
- }
441
- /** Run one same-seed decode and return only the smallest strict enclosing mask. */
442
- export async function runMobileSamExpandTask(request) {
443
- const ort = await import('onnxruntime-node');
444
- if (!ort
445
- .listSupportedBackends()
446
- .some((item) => item.name === request.options.executionProvider && item.bundled))
447
- throw new Error('Explicitly requested ONNX execution provider is unavailable');
448
- const sessionOptions = {
449
- executionProviders: request.options.executionProvider === 'dml'
450
- ? [{ name: 'dml', deviceId: request.options.deviceId ?? 0 }]
451
- : ['cpu'],
452
- intraOpNumThreads: request.options.threads,
453
- interOpNumThreads: 1,
454
- executionMode: 'sequential',
455
- enableMemPattern: request.options.executionProvider !== 'dml',
456
- graphOptimizationLevel: 'all',
457
- };
458
- const encoder = await ort.InferenceSession.create(await verifiedBytes(request.options.encoderPath, encoderIdentity), sessionOptions);
459
- let decoder;
460
- let imageTensor;
461
- let embedding;
462
- let result;
463
- let releaseFailed = false;
464
- let rgba;
465
- let packedPrevious;
466
- let previousMask;
467
- let selectedMask;
468
- try {
469
- decoder = await ort.InferenceSession.create(await verifiedBytes(request.options.decoderPath, decoderIdentity), sessionOptions);
470
- rgba = Buffer.from(request.rgba, 'base64');
471
- packedPrevious = Buffer.from(request.previousMask, 'base64');
472
- if (rgba.length !== request.width * request.height * 4)
473
- throw new Error('Invalid expansion RGBA input size');
474
- previousMask = unpackFullSourceBinaryMask(packedPrevious, request.width, request.height);
475
- validateMobileSamPreviousMask({
476
- width: request.width,
477
- height: request.height,
478
- rgba,
479
- previousMask,
480
- roi: request.roi,
481
- });
482
- const scale = 1024 / Math.max(request.width, request.height);
483
- const resizedWidth = Math.round(request.width * scale);
484
- const resizedHeight = Math.round(request.height * scale);
485
- if (resizedWidth < 1 || resizedHeight < 1)
486
- throw new Error('Unrepresentable image aspect ratio');
487
- const pipeline = sharp(rgba, {
488
- raw: { width: request.width, height: request.height, channels: 4 },
489
- });
490
- let rgb;
491
- try {
492
- rgb = await pipeline
493
- .removeAlpha()
494
- .resize(resizedWidth, resizedHeight, { fit: 'fill', kernel: 'linear' })
495
- .raw()
496
- .toBuffer();
497
- }
498
- finally {
499
- pipeline.destroy();
500
- }
501
- if (rgb.length !== resizedWidth * resizedHeight * 3)
502
- throw new Error('Invalid resized RGB data');
503
- imageTensor = new ort.Tensor('float32', Float32Array.from(rgb), [
504
- resizedHeight,
505
- resizedWidth,
506
- 3,
507
- ]);
508
- rgb.fill(0);
509
- const encoded = await encoder.run({ input_image: imageTensor });
510
- const embeddingTensor = encoded.image_embeddings;
511
- if (!(embeddingTensor instanceof ort.Tensor) ||
512
- embeddingTensor.type !== 'float32' ||
513
- JSON.stringify(embeddingTensor.dims) !== JSON.stringify([1, 256, 64, 64]) ||
514
- !(embeddingTensor.data instanceof Float32Array))
515
- throw new Error('Unexpected MobileSAM encoder output');
516
- embedding = embeddingTensor;
517
- const prompt = mobileSamSegmentationPrompt({
518
- prompt: request.prompt,
519
- seedRegion: request.seedRegion,
520
- wholeRegion: request.wholeRegion,
521
- xScale: resizedWidth / request.width,
522
- yScale: resizedHeight / request.height,
523
- });
524
- const pointCoords = new ort.Tensor('float32', prompt.coordinates, [1, 2, 2]);
525
- const pointLabels = new ort.Tensor('float32', prompt.labels, [1, 2]);
526
- const maskInput = new ort.Tensor('float32', new Float32Array(256 * 256), [1, 1, 256, 256]);
527
- const hasMaskInput = new ort.Tensor('float32', Float32Array.of(0), [1]);
528
- const originalSize = new ort.Tensor('float32', Float32Array.of(request.height, request.width), [
529
- 2,
530
- ]);
531
- let decoded;
532
- try {
533
- decoded = await decoder.run({
534
- image_embeddings: embedding,
535
- point_coords: pointCoords,
536
- point_labels: pointLabels,
537
- mask_input: maskInput,
538
- has_mask_input: hasMaskInput,
539
- orig_im_size: originalSize,
540
- });
541
- const masks = decoded.masks;
542
- const scores = decoded.iou_predictions;
543
- if (!(masks instanceof ort.Tensor) ||
544
- masks.type !== 'float32' ||
545
- JSON.stringify(masks.dims) !== JSON.stringify([1, 4, request.height, request.width]) ||
546
- !(masks.data instanceof Float32Array) ||
547
- !(scores instanceof ort.Tensor) ||
548
- scores.type !== 'float32' ||
549
- JSON.stringify(scores.dims) !== JSON.stringify([1, 4]) ||
550
- !(scores.data instanceof Float32Array))
551
- throw new Error('Unexpected MobileSAM expansion decoder output');
552
- const candidate = selectEnclosingMobileSamCandidate({
553
- width: request.width,
554
- height: request.height,
555
- rgba,
556
- logits: masks.data,
557
- scores: scores.data,
558
- previousMask,
559
- roi: request.roi,
560
- });
561
- if (candidate) {
562
- selectedMask = candidate.mask;
563
- const packed = packFullSourceBinaryMask(selectedMask, request.width, request.height);
564
- const packedBuffer = Buffer.from(packed);
565
- try {
566
- result = {
567
- id: request.id,
568
- status: 'expanded',
569
- width: request.width,
570
- height: request.height,
571
- mask: packedBuffer.toString('base64'),
572
- method: 'mobile-sam-enclosing-mask-v1',
573
- qualityScore: candidate.score,
574
- };
575
- }
576
- finally {
577
- packedBuffer.fill(0);
578
- packed.fill(0);
579
- }
580
- }
581
- else {
582
- result = {
583
- id: request.id,
584
- status: 'unknown',
585
- reason: 'no-enclosing-mask',
586
- width: request.width,
587
- height: request.height,
588
- method: 'mobile-sam-enclosing-mask-v1',
589
- };
590
- }
591
- }
592
- finally {
593
- prompt.coordinates.fill(0);
594
- prompt.labels.fill(0);
595
- for (const tensor of [pointCoords, pointLabels, maskInput, hasMaskInput, originalSize])
596
- if (tensor.data instanceof Float32Array)
597
- tensor.data.fill(0);
598
- pointCoords.dispose();
599
- pointLabels.dispose();
600
- maskInput.dispose();
601
- hasMaskInput.dispose();
602
- originalSize.dispose();
603
- if (decoded)
604
- for (const tensor of Object.values(decoded))
605
- if (tensor instanceof ort.Tensor) {
606
- if (tensor.data instanceof Float32Array || tensor.data instanceof Uint8Array)
607
- tensor.data.fill(0);
608
- tensor.dispose();
609
- }
610
- }
611
- }
612
- finally {
613
- imageTensor?.dispose();
614
- if (embedding && embedding !== imageTensor) {
615
- if (embedding.data instanceof Float32Array)
616
- embedding.data.fill(0);
617
- embedding.dispose();
618
- }
619
- previousMask?.fill(0);
620
- selectedMask?.fill(0);
621
- rgba?.fill(0);
622
- packedPrevious?.fill(0);
623
- const releases = await Promise.allSettled([
624
- encoder.release(),
625
- ...(decoder ? [decoder.release()] : []),
626
- ]);
627
- releaseFailed = releases.some((release) => release.status === 'rejected');
628
- }
629
- if (releaseFailed)
630
- throw new Error('MobileSAM expansion session release failed');
631
- if (!result)
632
- throw new Error('MobileSAM completed without an expansion result');
633
- return result;
634
- }
635
- /** Produce the fixed row-major 3x3 prompt grid from the actual seed rectangle. */
636
- export function mobileSamProposalPoints(rect) {
637
- return Array.from({ length: 9 }, (_, index) => {
638
- const row = Math.floor(index / 3);
639
- const column = index % 3;
640
- return {
641
- x: rect.left + Math.round(((rect.width - 1) * column) / 2),
642
- y: rect.top + Math.round(((rect.height - 1) * row) / 2),
643
- };
644
- });
645
- }
646
- /** Turn one decoder response into four owned packed candidates immediately. */
647
- export function packMobileSamProposalCandidates(input) {
648
- const pixels = input.width * input.height;
649
- if (!Number.isSafeInteger(input.width) ||
650
- !Number.isSafeInteger(input.height) ||
651
- input.width < 1 ||
652
- input.height < 1 ||
653
- pixels > 2_097_152 ||
654
- input.rgba.length !== pixels * 4 ||
655
- input.logits.length !== pixels * 4 ||
656
- input.scores.length !== 4 ||
657
- !Number.isInteger(input.pointIndex) ||
658
- input.pointIndex < 0 ||
659
- input.pointIndex > 8 ||
660
- input.roi.left < 0 ||
661
- input.roi.top < 0 ||
662
- input.roi.width < 1 ||
663
- input.roi.height < 1 ||
664
- input.roi.left + input.roi.width > input.width ||
665
- input.roi.top + input.roi.height > input.height)
666
- throw new Error('Invalid MobileSAM proposal response dimensions');
667
- const proposals = [];
668
- for (let token = 0; token < 4; token++) {
669
- const qualityScore = input.scores[token];
670
- if (qualityScore === undefined || !Number.isFinite(qualityScore))
671
- throw new Error('Invalid MobileSAM proposal quality score');
672
- const mask = new Uint8Array(pixels);
673
- const base = token * pixels;
674
- for (let index = 0; index < pixels; index++) {
675
- const value = input.logits[base + index];
676
- if (value === undefined || !Number.isFinite(value))
677
- throw new Error('Non-finite MobileSAM proposal logit');
678
- if (value <= 0 || input.rgba[index * 4 + 3] === 0)
679
- continue;
680
- const x = index % input.width;
681
- const y = Math.floor(index / input.width);
682
- if (inside(input.roi, x, y))
683
- mask[index] = 1;
684
- }
685
- const id = `C${String(input.pointIndex * 4 + token).padStart(2, '0')}`;
686
- const packed = packBinaryMask(mask, input.width, input.height);
687
- proposals.push({ id, mask: Buffer.from(packed.data).toString('base64'), qualityScore });
688
- }
689
- return proposals;
690
- }
691
- /** One encoder pass followed by nine independent positive-point decoder calls. */
692
- export async function runMobileSamObjectProposalTask(request) {
693
- const ort = await import('onnxruntime-node');
694
- if (!ort.listSupportedBackends().some((item) => item.name === 'cpu' && item.bundled))
695
- throw new Error('Explicitly requested ONNX CPU execution provider is unavailable');
696
- const sessionOptions = {
697
- executionProviders: ['cpu'],
698
- intraOpNumThreads: request.options.threads,
699
- interOpNumThreads: 1,
700
- executionMode: 'sequential',
701
- enableMemPattern: true,
702
- graphOptimizationLevel: 'all',
703
- };
704
- const encoder = await ort.InferenceSession.create(await verifiedBytes(request.options.encoderPath, encoderIdentity), sessionOptions);
705
- let decoder;
706
- let embeddingTensor;
707
- let reply;
708
- let releaseFailed = false;
709
- try {
710
- decoder = await ort.InferenceSession.create(await verifiedBytes(request.options.decoderPath, decoderIdentity), sessionOptions);
711
- const rgba = Buffer.from(request.rgba, 'base64');
712
- const pixels = request.width * request.height;
713
- if (rgba.length !== pixels * 4 || pixels > 2_097_152)
714
- throw new Error('Invalid or oversized RGBA proposal input');
715
- const scale = 1024 / Math.max(request.width, request.height);
716
- const resizedWidth = Math.round(request.width * scale);
717
- const resizedHeight = Math.round(request.height * scale);
718
- const pipeline = sharp(rgba, {
719
- raw: { width: request.width, height: request.height, channels: 4 },
720
- });
721
- let rgb;
722
- try {
723
- rgb = await pipeline
724
- .removeAlpha()
725
- .resize(resizedWidth, resizedHeight, { fit: 'fill', kernel: 'linear' })
726
- .raw()
727
- .toBuffer();
728
- }
729
- finally {
730
- pipeline.destroy();
731
- }
732
- if (rgb.length !== resizedWidth * resizedHeight * 3)
733
- throw new Error('Invalid resized RGB proposal input');
734
- const imageTensor = new ort.Tensor('float32', Float32Array.from(rgb), [
735
- resizedHeight,
736
- resizedWidth,
737
- 3,
738
- ]);
739
- try {
740
- const encoded = await encoder.run({ input_image: imageTensor });
741
- const embedding = encoded.image_embeddings;
742
- if (embedding instanceof ort.Tensor)
743
- embeddingTensor = embedding;
744
- if (!(embedding instanceof ort.Tensor) ||
745
- embedding.type !== 'float32' ||
746
- JSON.stringify(embedding.dims) !== JSON.stringify([1, 256, 64, 64]) ||
747
- !(embedding.data instanceof Float32Array))
748
- throw new Error('Unexpected MobileSAM encoder output');
749
- const xScale = resizedWidth / request.width;
750
- const yScale = resizedHeight / request.height;
751
- const proposals = [];
752
- let packedBytes = 0;
753
- const points = mobileSamProposalPoints(request.seedRegion);
754
- for (let pointIndex = 0; pointIndex < points.length; pointIndex++) {
755
- const point = points[pointIndex];
756
- if (!point)
757
- throw new Error('Missing MobileSAM proposal point');
758
- let coordinates;
759
- let labels;
760
- let maskInput;
761
- let hasMaskInput;
762
- let originalSize;
763
- let decoded;
764
- try {
765
- coordinates = new ort.Tensor('float32', Float32Array.of(point.x * xScale, point.y * yScale, point.x * xScale, point.y * yScale), [1, 2, 2]);
766
- labels = new ort.Tensor('float32', Float32Array.of(1, -1), [1, 2]);
767
- maskInput = new ort.Tensor('float32', new Float32Array(256 * 256), [1, 1, 256, 256]);
768
- hasMaskInput = new ort.Tensor('float32', Float32Array.of(0), [1]);
769
- originalSize = new ort.Tensor('float32', Float32Array.of(request.height, request.width), [
770
- 2,
771
- ]);
772
- decoded = await decoder.run({
773
- image_embeddings: embedding,
774
- point_coords: coordinates,
775
- point_labels: labels,
776
- mask_input: maskInput,
777
- has_mask_input: hasMaskInput,
778
- orig_im_size: originalSize,
779
- });
780
- const masks = decoded.masks;
781
- const scores = decoded.iou_predictions;
782
- if (!(masks instanceof ort.Tensor) ||
783
- masks.type !== 'float32' ||
784
- JSON.stringify(masks.dims) !== JSON.stringify([1, 4, request.height, request.width]) ||
785
- !(masks.data instanceof Float32Array) ||
786
- !(scores instanceof ort.Tensor) ||
787
- scores.type !== 'float32' ||
788
- JSON.stringify(scores.dims) !== JSON.stringify([1, 4]) ||
789
- !(scores.data instanceof Float32Array) ||
790
- !scores.data.every(Number.isFinite))
791
- throw new Error('Unexpected MobileSAM proposal decoder output');
792
- const candidates = packMobileSamProposalCandidates({
793
- width: request.width,
794
- height: request.height,
795
- rgba,
796
- roi: request.roi,
797
- pointIndex,
798
- logits: masks.data,
799
- scores: scores.data,
800
- });
801
- for (const candidate of candidates) {
802
- packedBytes += Buffer.from(candidate.mask, 'base64').byteLength;
803
- if (packedBytes > objectProposalLimits.packedBytes)
804
- throw new Error('Packed MobileSAM proposals exceed the aggregate byte limit');
805
- proposals.push(candidate);
806
- }
807
- }
808
- finally {
809
- coordinates?.dispose();
810
- labels?.dispose();
811
- maskInput?.dispose();
812
- hasMaskInput?.dispose();
813
- originalSize?.dispose();
814
- if (decoded)
815
- for (const tensor of Object.values(decoded))
816
- if (tensor instanceof ort.Tensor)
817
- tensor.dispose();
818
- }
819
- }
820
- reply = {
821
- id: request.id,
822
- status: 'proposals',
823
- width: request.width,
824
- height: request.height,
825
- proposals,
826
- };
827
- }
828
- finally {
829
- embeddingTensor?.dispose();
830
- imageTensor.dispose();
831
- }
832
- }
833
- finally {
834
- const releases = await Promise.allSettled([
835
- encoder.release(),
836
- ...(decoder ? [decoder.release()] : []),
837
- ]);
838
- releaseFailed = releases.some((release) => release.status === 'rejected');
839
- }
840
- if (releaseFailed)
841
- throw new Error('MobileSAM proposal session release failed');
842
- if (!reply)
843
- throw new Error('MobileSAM completed without proposals');
844
- return reply;
845
- }
1
+ export { probeMobileSam } from './worker-models.js';
2
+ export { runMobileSamTask } from './worker-segment.js';
3
+ export { runMobileSamExpandTask } from './worker-expand.js';
4
+ export { runMobileSamObjectProposalTask } from './worker-proposals.js';
5
+ export { selectBestMaskCandidate, mobileSamSegmentationPrompt, collectMobileSamSegmentationCandidates, selectMobileSamSegmentationMask, packFullSourceBinaryMask, unpackFullSourceBinaryMask, validateMobileSamPreviousMask, selectEnclosingMobileSamCandidate, } from './segmentation-masks.js';
6
+ export { mobileSamProposalPoints, packMobileSamProposalCandidates } from './proposal-masks.js';