zorya-classifier 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/app/classifier-config.d.ts +30 -0
- package/dist/app/classifier-config.js +46 -0
- package/dist/app/classifier-contracts.d.ts +32 -0
- package/dist/app/classifier-contracts.js +1 -0
- package/dist/app/classifier-representation.d.ts +5 -0
- package/dist/app/classifier-representation.js +33 -0
- package/dist/app/classifier-runtime.d.ts +7 -0
- package/dist/app/classifier-runtime.js +71 -0
- package/dist/app/classifier-spatial.d.ts +31 -0
- package/dist/app/classifier-spatial.js +78 -0
- package/dist/app/cli-arguments.d.ts +2 -2
- package/dist/app/create-classifier.d.ts +4 -29
- package/dist/app/create-classifier.js +37 -237
- package/dist/cli.js +1 -1
- package/dist/entities/classification.d.ts +98 -0
- package/dist/entities/classification.js +28 -0
- package/dist/entities/contracts.d.ts +6 -471
- package/dist/entities/contracts.js +6 -97
- package/dist/entities/decision.d.ts +211 -0
- package/dist/entities/decision.js +33 -0
- package/dist/entities/geometry.d.ts +37 -0
- package/dist/entities/geometry.js +26 -0
- package/dist/entities/image.d.ts +39 -0
- package/dist/entities/image.js +15 -0
- package/dist/entities/localization.d.ts +125 -0
- package/dist/entities/localization.js +13 -0
- package/dist/entities/object-proposals.d.ts +4 -1
- package/dist/entities/segmentation-metadata.d.ts +70 -0
- package/dist/entities/segmentation-metadata.js +21 -1
- package/dist/entities/segmentation-verification.d.ts +1 -0
- package/dist/entities/segmentation.d.ts +6 -1
- package/dist/entities/validation.d.ts +2 -0
- package/dist/entities/validation.js +9 -0
- package/dist/entities/whole-object.d.ts +4 -0
- package/dist/features/choice-beam-layout.js +2 -7
- package/dist/features/proposals/presentation.d.ts +21 -0
- package/dist/features/proposals/presentation.js +122 -0
- package/dist/features/proposals/validation.d.ts +138 -0
- package/dist/features/proposals/validation.js +125 -0
- package/dist/features/segmentation/box-check.js +10 -23
- package/dist/features/segmentation/mask.js +2 -7
- package/dist/features/segmentation/region-result.d.ts +18 -0
- package/dist/features/segmentation/region-result.js +171 -0
- package/dist/features/segmentation/source-mask.d.ts +6 -0
- package/dist/features/segmentation/source-mask.js +32 -0
- package/dist/features/segmentation/validation.d.ts +9 -0
- package/dist/features/segmentation/validation.js +94 -0
- package/dist/infrastructure/decision-transport/errors.d.ts +8 -0
- package/dist/infrastructure/decision-transport/errors.js +34 -0
- package/dist/infrastructure/decision-transport/images.d.ts +1 -0
- package/dist/infrastructure/decision-transport/images.js +52 -0
- package/dist/infrastructure/{openrouter-common.d.ts → decision-transport/request.d.ts} +6 -15
- package/dist/infrastructure/decision-transport/request.js +162 -0
- package/dist/infrastructure/decision-transport/response.d.ts +1 -0
- package/dist/infrastructure/decision-transport/response.js +72 -0
- package/dist/infrastructure/decision-transport/route-identity.d.ts +11 -0
- package/dist/infrastructure/decision-transport/route-identity.js +23 -0
- package/dist/infrastructure/openrouter-vision.js +1 -1
- package/dist/infrastructure/openrouter.js +7 -19
- package/dist/infrastructure/raster.js +3 -8
- package/dist/infrastructure/segmentation/grabcut/index.d.ts +5 -1
- package/dist/infrastructure/segmentation/grabcut/index.js +6 -5
- package/dist/infrastructure/segmentation/grabcut/masks.d.ts +49 -0
- package/dist/infrastructure/segmentation/grabcut/masks.js +89 -0
- package/dist/infrastructure/segmentation/grabcut/model-asset.d.ts +4 -0
- package/dist/infrastructure/segmentation/grabcut/model-asset.js +4 -0
- package/dist/infrastructure/segmentation/grabcut/runtime.d.ts +70 -0
- package/dist/infrastructure/segmentation/grabcut/runtime.js +96 -0
- package/dist/infrastructure/segmentation/grabcut/worker-inference.d.ts +4 -43
- package/dist/infrastructure/segmentation/grabcut/worker-inference.js +4 -187
- package/dist/infrastructure/segmentation/mobile-sam/index.d.ts +3 -58
- package/dist/infrastructure/segmentation/mobile-sam/index.js +2 -472
- package/dist/infrastructure/segmentation/mobile-sam/input.d.ts +46 -0
- package/dist/infrastructure/segmentation/mobile-sam/input.js +83 -0
- package/dist/infrastructure/segmentation/mobile-sam/model-assets.d.ts +8 -0
- package/dist/infrastructure/segmentation/mobile-sam/model-assets.js +8 -0
- package/dist/infrastructure/segmentation/mobile-sam/object-proposals.d.ts +7 -0
- package/dist/infrastructure/segmentation/mobile-sam/object-proposals.js +124 -0
- package/dist/infrastructure/segmentation/mobile-sam/options.d.ts +43 -0
- package/dist/infrastructure/segmentation/mobile-sam/options.js +19 -0
- package/dist/infrastructure/segmentation/mobile-sam/proposal-masks.d.ts +26 -0
- package/dist/infrastructure/segmentation/mobile-sam/proposal-masks.js +71 -0
- package/dist/infrastructure/segmentation/mobile-sam/segmentation-masks.d.ts +112 -0
- package/dist/infrastructure/segmentation/mobile-sam/segmentation-masks.js +230 -0
- package/dist/infrastructure/segmentation/mobile-sam/segmenter.d.ts +22 -0
- package/dist/infrastructure/segmentation/mobile-sam/segmenter.js +268 -0
- package/dist/infrastructure/segmentation/mobile-sam/worker-expand.d.ts +7 -0
- package/dist/infrastructure/segmentation/mobile-sam/worker-expand.js +188 -0
- package/dist/infrastructure/segmentation/mobile-sam/worker-inference.d.ts +6 -149
- package/dist/infrastructure/segmentation/mobile-sam/worker-inference.js +6 -845
- package/dist/infrastructure/segmentation/mobile-sam/worker-models.d.ts +14 -0
- package/dist/infrastructure/segmentation/mobile-sam/worker-models.js +72 -0
- package/dist/infrastructure/segmentation/mobile-sam/worker-proposals.d.ts +7 -0
- package/dist/infrastructure/segmentation/mobile-sam/worker-proposals.js +159 -0
- package/dist/infrastructure/segmentation/mobile-sam/worker-segment.d.ts +4 -0
- package/dist/infrastructure/segmentation/mobile-sam/worker-segment.js +126 -0
- package/dist/infrastructure/segmentation/native-protocol.d.ts +59 -34
- package/dist/infrastructure/segmentation/native-protocol.js +26 -16
- package/dist/infrastructure/segmentation/proposal-presentation-input.d.ts +20 -0
- package/dist/infrastructure/segmentation/proposal-presentation-input.js +107 -0
- package/dist/infrastructure/segmentation/proposal-presentation-pixels.d.ts +8 -0
- package/dist/infrastructure/segmentation/proposal-presentation-pixels.js +93 -0
- package/dist/infrastructure/segmentation/proposal-presentation.js +10 -201
- package/dist/infrastructure/vision-chat/options.d.ts +62 -0
- package/dist/infrastructure/vision-chat/options.js +46 -0
- package/dist/infrastructure/vision-chat/request.d.ts +23 -0
- package/dist/infrastructure/vision-chat/request.js +109 -0
- package/dist/infrastructure/vision-chat/response.d.ts +3 -0
- package/dist/infrastructure/vision-chat/response.js +105 -0
- package/dist/infrastructure/vision-chat.d.ts +4 -61
- package/dist/infrastructure/vision-chat.js +8 -271
- package/dist/openai-compatible.js +1 -1
- package/dist/processes/extract-from-region.js +11 -216
- package/dist/processes/extract.d.ts +2 -8
- package/dist/processes/extract.js +12 -128
- package/dist/processes/proposal-location.js +5 -251
- package/docs/0.2.0-plan.md +3 -3
- package/docs/README.ru.md +42 -23
- package/docs/acceptance.md +5 -5
- package/docs/agents.ru.md +95 -0
- package/docs/api.md +4 -2
- package/docs/architecture.md +33 -5
- package/docs/architecture.ru.md +146 -0
- package/docs/benchmarks.md +4 -2
- package/docs/development.ru.md +141 -0
- package/docs/operations.md +4 -4
- package/docs/provenance.json +5 -1
- package/docs/publishing.md +8 -0
- package/docs/refactoring-plan.ru.md +183 -0
- package/docs/v02-usage.md +8 -6
- package/docs/vision-results.md +2 -0
- package/package.json +5 -3
- package/dist/infrastructure/boundary-view.d.ts +0 -2
- package/dist/infrastructure/boundary-view.js +0 -263
- package/dist/infrastructure/openrouter-common.js +0 -317
- package/dist/infrastructure/text-representation.d.ts +0 -23
- package/dist/infrastructure/text-representation.js +0 -226
- package/docs/evidence/v02/categorical-heldout-v4/README.md +0 -35
- package/docs/evidence/v02/proposal-resource-soak-261dc23.md +0 -15
|
@@ -1,472 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
import { z } from 'zod';
|
|
4
|
-
import { objectProposalLimits } from '../../../entities/object-proposals.js';
|
|
5
|
-
import { limits, pixelRectSchema } from '../../../entities/contracts.js';
|
|
6
|
-
import { NativeChildPool } from '../native-child.js';
|
|
7
|
-
import { verifyPinnedAsset } from '../verify-asset.js';
|
|
8
|
-
import { probeBackend } from '../probe-backend.js';
|
|
9
|
-
import { nativeErrorReplySchema, nativeMobileSamExpansionReplySchema, nativeObjectProposalReplySchema, nativeSegmentationReplySchema, } from '../native-protocol.js';
|
|
10
|
-
import { packFullSourceBinaryMask, unpackFullSourceBinaryMask, validateMobileSamPreviousMask, } from './worker-inference.js';
|
|
11
|
-
const optionsSchema = z
|
|
12
|
-
.object({
|
|
13
|
-
encoderPath: z.string().min(1).max(4096),
|
|
14
|
-
decoderPath: z.string().min(1).max(4096),
|
|
15
|
-
executionProvider: z.enum(['cpu', 'dml']).default('cpu'),
|
|
16
|
-
deviceId: z.number().int().min(0).max(32).optional(),
|
|
17
|
-
threads: z.number().int().min(1).max(8).default(2),
|
|
18
|
-
})
|
|
19
|
-
.strict();
|
|
20
|
-
const proposalOptionsSchema = z
|
|
21
|
-
.object({
|
|
22
|
-
encoderPath: z.string().min(1).max(4096),
|
|
23
|
-
decoderPath: z.string().min(1).max(4096),
|
|
24
|
-
executionProvider: z.literal('cpu').default('cpu'),
|
|
25
|
-
threads: z.number().int().min(1).max(8).default(2),
|
|
26
|
-
})
|
|
27
|
-
.strict();
|
|
28
|
-
/** Explicit pinned MobileSAM inference in a separately owned Node child process. */
|
|
29
|
-
export async function createMobileSamSegmenter(options) {
|
|
30
|
-
const checked = optionsSchema.safeParse(options);
|
|
31
|
-
if (!checked.success)
|
|
32
|
-
throw new ClassifierError('INVALID_INPUT', 'Invalid MobileSAM options');
|
|
33
|
-
const parsed = checked.data;
|
|
34
|
-
if (parsed.executionProvider === 'dml' && process.platform !== 'win32')
|
|
35
|
-
throw new ClassifierError('UNSUPPORTED', 'DirectML is supported only on Windows');
|
|
36
|
-
try {
|
|
37
|
-
import.meta.resolve('onnxruntime-node');
|
|
38
|
-
}
|
|
39
|
-
catch {
|
|
40
|
-
throw new ClassifierError('UNSUPPORTED', 'onnxruntime-node is not installed');
|
|
41
|
-
}
|
|
42
|
-
await verifyPinnedAsset(parsed.encoderPath, {
|
|
43
|
-
bytes: 28_157_093,
|
|
44
|
-
sha256: '580f5fb648ea1062c0aabc26217aed56921985f03f0cbbd852bba81d760cc749',
|
|
45
|
-
}, 'MobileSAM encoder');
|
|
46
|
-
await verifyPinnedAsset(parsed.decoderPath, {
|
|
47
|
-
bytes: 16_496_559,
|
|
48
|
-
sha256: '8976b90a87ba50a6a72217a5ff994f7d25ce16f2229fcc1ed259e1294c622ffe',
|
|
49
|
-
}, 'MobileSAM decoder');
|
|
50
|
-
const sourceWorker = import.meta.url.endsWith('.ts');
|
|
51
|
-
const workerUrl = new URL(sourceWorker ? '../native-worker.ts' : '../native-worker.js', import.meta.url);
|
|
52
|
-
await probeBackend(workerUrl, 'mobile-sam', { options: parsed });
|
|
53
|
-
const pool = new NativeChildPool({
|
|
54
|
-
workerUrl,
|
|
55
|
-
maxActive: 1,
|
|
56
|
-
maxWaiting: 2,
|
|
57
|
-
jobTimeoutMs: 10_000,
|
|
58
|
-
abortGraceMs: 250,
|
|
59
|
-
encode: (job) => Buffer.from(JSON.stringify({ id: job.id, ...job.payload })),
|
|
60
|
-
decode: (raw, id) => {
|
|
61
|
-
const error = nativeErrorReplySchema.safeParse(raw);
|
|
62
|
-
if (error.success) {
|
|
63
|
-
if (error.data.id !== id)
|
|
64
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native child error ID mismatch');
|
|
65
|
-
throw new ClassifierError(error.data.code, error.data.message);
|
|
66
|
-
}
|
|
67
|
-
const reply = nativeSegmentationReplySchema.safeParse(raw);
|
|
68
|
-
if (reply.success && reply.data.id === id)
|
|
69
|
-
return reply.data;
|
|
70
|
-
const expansion = nativeMobileSamExpansionReplySchema.safeParse(raw);
|
|
71
|
-
if (!expansion.success || expansion.data.id !== id)
|
|
72
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native child returned an invalid or mismatched reply');
|
|
73
|
-
return expansion.data;
|
|
74
|
-
},
|
|
75
|
-
});
|
|
76
|
-
return createMobileSamPortForPool(pool, parsed);
|
|
77
|
-
}
|
|
78
|
-
/** Internal adapter seam: pool ownership stays with the caller. */
|
|
79
|
-
export function createMobileSamPortForPool(pool, parsed) {
|
|
80
|
-
return {
|
|
81
|
-
async segment(input) {
|
|
82
|
-
const started = performance.now();
|
|
83
|
-
assertActive(input.signal);
|
|
84
|
-
let image;
|
|
85
|
-
let maskData;
|
|
86
|
-
try {
|
|
87
|
-
const raw = await pool.runLazy(input.signal, () => {
|
|
88
|
-
image = snapshotInput(input);
|
|
89
|
-
const rgba = Buffer.from(image.data);
|
|
90
|
-
let rgbaBase64;
|
|
91
|
-
try {
|
|
92
|
-
rgbaBase64 = rgba.toString('base64');
|
|
93
|
-
}
|
|
94
|
-
finally {
|
|
95
|
-
rgba.fill(0);
|
|
96
|
-
}
|
|
97
|
-
return {
|
|
98
|
-
kind: 'mobile-sam',
|
|
99
|
-
width: image.width,
|
|
100
|
-
height: image.height,
|
|
101
|
-
rgba: rgbaBase64,
|
|
102
|
-
seedRegion: image.seedRegion,
|
|
103
|
-
wholeRegion: image.wholeRegion,
|
|
104
|
-
roi: image.roi,
|
|
105
|
-
prompt: image.prompt,
|
|
106
|
-
options: parsed,
|
|
107
|
-
};
|
|
108
|
-
});
|
|
109
|
-
if (!image)
|
|
110
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native segmentation omitted its input snapshot');
|
|
111
|
-
const checkedReply = nativeSegmentationReplySchema.safeParse(raw);
|
|
112
|
-
if (!checkedReply.success ||
|
|
113
|
-
checkedReply.data.width !== image.width ||
|
|
114
|
-
checkedReply.data.height !== image.height)
|
|
115
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native child returned an invalid segmentation reply');
|
|
116
|
-
maskData = Buffer.from(checkedReply.data.mask, 'base64');
|
|
117
|
-
if (maskData.length !== image.width * image.height ||
|
|
118
|
-
maskData.toString('base64') !== checkedReply.data.mask)
|
|
119
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native child mask dimensions or encoding are invalid');
|
|
120
|
-
return {
|
|
121
|
-
status: 'segmented',
|
|
122
|
-
mask: {
|
|
123
|
-
data: new Uint8Array(maskData),
|
|
124
|
-
width: image.width,
|
|
125
|
-
height: image.height,
|
|
126
|
-
format: 'binary-u8',
|
|
127
|
-
},
|
|
128
|
-
metadata: {
|
|
129
|
-
method: checkedReply.data.method,
|
|
130
|
-
backend: 'mobile-sam',
|
|
131
|
-
durationMs: Math.max(0, performance.now() - started),
|
|
132
|
-
input: { width: image.width, height: image.height },
|
|
133
|
-
},
|
|
134
|
-
};
|
|
135
|
-
}
|
|
136
|
-
finally {
|
|
137
|
-
maskData?.fill(0);
|
|
138
|
-
image?.data.fill(0);
|
|
139
|
-
}
|
|
140
|
-
},
|
|
141
|
-
async expand(input, previousMask) {
|
|
142
|
-
const started = performance.now();
|
|
143
|
-
assertActive(input.signal);
|
|
144
|
-
let image;
|
|
145
|
-
let previous;
|
|
146
|
-
let resultData;
|
|
147
|
-
let transferred = false;
|
|
148
|
-
try {
|
|
149
|
-
const raw = await pool.runLazy(input.signal, () => {
|
|
150
|
-
image = snapshotInput(input);
|
|
151
|
-
previous = snapshotPreviousMask(input, previousMask, image);
|
|
152
|
-
const packedPrevious = packFullSourceBinaryMask(previous, image.width, image.height);
|
|
153
|
-
let previousMaskBase64;
|
|
154
|
-
try {
|
|
155
|
-
const packedBuffer = Buffer.from(packedPrevious);
|
|
156
|
-
try {
|
|
157
|
-
previousMaskBase64 = packedBuffer.toString('base64');
|
|
158
|
-
}
|
|
159
|
-
finally {
|
|
160
|
-
packedBuffer.fill(0);
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
finally {
|
|
164
|
-
packedPrevious.fill(0);
|
|
165
|
-
}
|
|
166
|
-
const rgbaBuffer = Buffer.from(image.data);
|
|
167
|
-
let rgbaBase64;
|
|
168
|
-
try {
|
|
169
|
-
rgbaBase64 = rgbaBuffer.toString('base64');
|
|
170
|
-
}
|
|
171
|
-
finally {
|
|
172
|
-
rgbaBuffer.fill(0);
|
|
173
|
-
}
|
|
174
|
-
return {
|
|
175
|
-
kind: 'mobile-sam-expand',
|
|
176
|
-
width: image.width,
|
|
177
|
-
height: image.height,
|
|
178
|
-
rgba: rgbaBase64,
|
|
179
|
-
previousMask: previousMaskBase64,
|
|
180
|
-
seedRegion: image.seedRegion,
|
|
181
|
-
wholeRegion: image.wholeRegion,
|
|
182
|
-
roi: image.roi,
|
|
183
|
-
prompt: image.prompt,
|
|
184
|
-
options: parsed,
|
|
185
|
-
};
|
|
186
|
-
});
|
|
187
|
-
if (!image || !previous)
|
|
188
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native expansion omitted its input snapshot');
|
|
189
|
-
const checkedReply = nativeMobileSamExpansionReplySchema.safeParse(raw);
|
|
190
|
-
if (!checkedReply.success ||
|
|
191
|
-
checkedReply.data.width !== image.width ||
|
|
192
|
-
checkedReply.data.height !== image.height)
|
|
193
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native child returned an invalid expansion reply');
|
|
194
|
-
if (checkedReply.data.status === 'unknown')
|
|
195
|
-
return {
|
|
196
|
-
status: 'unknown',
|
|
197
|
-
reason: 'no-enclosing-mask',
|
|
198
|
-
metadata: {
|
|
199
|
-
method: checkedReply.data.method,
|
|
200
|
-
backend: 'mobile-sam',
|
|
201
|
-
durationMs: Math.max(0, performance.now() - started),
|
|
202
|
-
input: { width: image.width, height: image.height },
|
|
203
|
-
},
|
|
204
|
-
};
|
|
205
|
-
const packedBytes = Buffer.from(checkedReply.data.mask, 'base64');
|
|
206
|
-
let data;
|
|
207
|
-
try {
|
|
208
|
-
if (packedBytes.toString('base64') !== checkedReply.data.mask)
|
|
209
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native expansion mask encoding is invalid');
|
|
210
|
-
try {
|
|
211
|
-
data = unpackFullSourceBinaryMask(packedBytes, image.width, image.height);
|
|
212
|
-
resultData = data;
|
|
213
|
-
}
|
|
214
|
-
catch {
|
|
215
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native expansion packed mask is invalid');
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
finally {
|
|
219
|
-
packedBytes.fill(0);
|
|
220
|
-
}
|
|
221
|
-
let oldStats;
|
|
222
|
-
let newStats;
|
|
223
|
-
try {
|
|
224
|
-
oldStats = validateMobileSamPreviousMask({
|
|
225
|
-
width: image.width,
|
|
226
|
-
height: image.height,
|
|
227
|
-
rgba: image.data,
|
|
228
|
-
previousMask: previous,
|
|
229
|
-
roi: image.roi,
|
|
230
|
-
});
|
|
231
|
-
newStats = validateMobileSamPreviousMask({
|
|
232
|
-
width: image.width,
|
|
233
|
-
height: image.height,
|
|
234
|
-
rgba: image.data,
|
|
235
|
-
previousMask: data,
|
|
236
|
-
roi: image.roi,
|
|
237
|
-
});
|
|
238
|
-
}
|
|
239
|
-
catch {
|
|
240
|
-
data.fill(0);
|
|
241
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native expanded mask is invalid');
|
|
242
|
-
}
|
|
243
|
-
let hasNewForeground = false;
|
|
244
|
-
for (let index = 0; index < data.length; index++) {
|
|
245
|
-
if (previous[index] === 1 && data[index] !== 1) {
|
|
246
|
-
data.fill(0);
|
|
247
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native expansion lost prior foreground');
|
|
248
|
-
}
|
|
249
|
-
if (data[index] === 1 && previous[index] === 0)
|
|
250
|
-
hasNewForeground = true;
|
|
251
|
-
}
|
|
252
|
-
if (!hasNewForeground ||
|
|
253
|
-
newStats.area <= oldStats.area ||
|
|
254
|
-
newStats.bounds.left > oldStats.bounds.left ||
|
|
255
|
-
newStats.bounds.top > oldStats.bounds.top ||
|
|
256
|
-
newStats.bounds.right < oldStats.bounds.right ||
|
|
257
|
-
newStats.bounds.bottom < oldStats.bounds.bottom ||
|
|
258
|
-
(newStats.bounds.left === oldStats.bounds.left &&
|
|
259
|
-
newStats.bounds.top === oldStats.bounds.top &&
|
|
260
|
-
newStats.bounds.right === oldStats.bounds.right &&
|
|
261
|
-
newStats.bounds.bottom === oldStats.bounds.bottom)) {
|
|
262
|
-
data.fill(0);
|
|
263
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native expansion is not a strict enclosing mask');
|
|
264
|
-
}
|
|
265
|
-
transferred = true;
|
|
266
|
-
return {
|
|
267
|
-
status: 'segmented',
|
|
268
|
-
mask: { data, width: image.width, height: image.height, format: 'binary-u8' },
|
|
269
|
-
metadata: {
|
|
270
|
-
method: checkedReply.data.method,
|
|
271
|
-
backend: 'mobile-sam',
|
|
272
|
-
durationMs: Math.max(0, performance.now() - started),
|
|
273
|
-
input: { width: image.width, height: image.height },
|
|
274
|
-
qualityScore: checkedReply.data.qualityScore,
|
|
275
|
-
},
|
|
276
|
-
};
|
|
277
|
-
}
|
|
278
|
-
finally {
|
|
279
|
-
image?.data.fill(0);
|
|
280
|
-
previous?.fill(0);
|
|
281
|
-
if (resultData && !transferred)
|
|
282
|
-
resultData.fill(0);
|
|
283
|
-
}
|
|
284
|
-
},
|
|
285
|
-
settle: () => pool.settle(),
|
|
286
|
-
dispose: () => pool.dispose(),
|
|
287
|
-
};
|
|
288
|
-
}
|
|
289
|
-
/** Explicit CPU-only, caller-owned one-shot 3x3-point / 36-mask proposal adapter. */
|
|
290
|
-
export async function createMobileSamObjectProposals(options) {
|
|
291
|
-
const checked = proposalOptionsSchema.safeParse(options);
|
|
292
|
-
if (!checked.success)
|
|
293
|
-
throw new ClassifierError('INVALID_INPUT', 'Invalid MobileSAM proposal options');
|
|
294
|
-
const parsed = checked.data;
|
|
295
|
-
try {
|
|
296
|
-
import.meta.resolve('onnxruntime-node');
|
|
297
|
-
}
|
|
298
|
-
catch {
|
|
299
|
-
throw new ClassifierError('UNSUPPORTED', 'onnxruntime-node is not installed');
|
|
300
|
-
}
|
|
301
|
-
await verifyPinnedAsset(parsed.encoderPath, {
|
|
302
|
-
bytes: 28_157_093,
|
|
303
|
-
sha256: '580f5fb648ea1062c0aabc26217aed56921985f03f0cbbd852bba81d760cc749',
|
|
304
|
-
}, 'MobileSAM encoder');
|
|
305
|
-
await verifyPinnedAsset(parsed.decoderPath, {
|
|
306
|
-
bytes: 16_496_559,
|
|
307
|
-
sha256: '8976b90a87ba50a6a72217a5ff994f7d25ce16f2229fcc1ed259e1294c622ffe',
|
|
308
|
-
}, 'MobileSAM decoder');
|
|
309
|
-
const sourceWorker = import.meta.url.endsWith('.ts');
|
|
310
|
-
const workerUrl = new URL(sourceWorker ? '../native-worker.ts' : '../native-worker.js', import.meta.url);
|
|
311
|
-
// Probe the explicitly requested CPU provider in an owned child, disposing it before return.
|
|
312
|
-
await probeBackend(workerUrl, 'mobile-sam', { options: { ...parsed, executionProvider: 'cpu' } });
|
|
313
|
-
const pool = new NativeChildPool({
|
|
314
|
-
workerUrl,
|
|
315
|
-
maxActive: 1,
|
|
316
|
-
maxWaiting: 2,
|
|
317
|
-
jobTimeoutMs: 10_000,
|
|
318
|
-
abortGraceMs: 250,
|
|
319
|
-
encode: (job) => Buffer.from(JSON.stringify({ id: job.id, ...job.payload })),
|
|
320
|
-
decode: (raw, id) => {
|
|
321
|
-
const error = nativeErrorReplySchema.safeParse(raw);
|
|
322
|
-
if (error.success) {
|
|
323
|
-
if (error.data.id !== id)
|
|
324
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native child error ID mismatch');
|
|
325
|
-
throw new ClassifierError(error.data.code, error.data.message);
|
|
326
|
-
}
|
|
327
|
-
const reply = nativeObjectProposalReplySchema.safeParse(raw);
|
|
328
|
-
if (!reply.success || reply.data.id !== id)
|
|
329
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Native child returned invalid proposals');
|
|
330
|
-
return reply.data;
|
|
331
|
-
},
|
|
332
|
-
});
|
|
333
|
-
return {
|
|
334
|
-
async propose(input) {
|
|
335
|
-
const started = performance.now();
|
|
336
|
-
assertActive(input.signal);
|
|
337
|
-
let image;
|
|
338
|
-
const raw = await pool.runLazy(input.signal, () => {
|
|
339
|
-
const snapshot = snapshotProposalInput(input);
|
|
340
|
-
image = snapshot.image;
|
|
341
|
-
return {
|
|
342
|
-
kind: 'mobile-sam-proposals',
|
|
343
|
-
width: image.width,
|
|
344
|
-
height: image.height,
|
|
345
|
-
rgba: Buffer.from(image.data).toString('base64'),
|
|
346
|
-
seedRegion: snapshot.seedRegion,
|
|
347
|
-
roi: snapshot.roi,
|
|
348
|
-
options: { ...parsed, executionProvider: 'cpu' },
|
|
349
|
-
};
|
|
350
|
-
});
|
|
351
|
-
const reply = nativeObjectProposalReplySchema.parse(raw);
|
|
352
|
-
if (reply.width !== image.width || reply.height !== image.height)
|
|
353
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Proposal dimensions do not match the input');
|
|
354
|
-
const expectedBytes = Math.ceil((image.width * image.height) / 8);
|
|
355
|
-
let aggregateBytes = 0;
|
|
356
|
-
const proposals = reply.proposals.map((proposal, index) => {
|
|
357
|
-
const expectedId = `C${String(index).padStart(2, '0')}`;
|
|
358
|
-
const bytes = Buffer.from(proposal.mask, 'base64');
|
|
359
|
-
aggregateBytes += bytes.byteLength;
|
|
360
|
-
if (proposal.id !== expectedId ||
|
|
361
|
-
bytes.byteLength !== expectedBytes ||
|
|
362
|
-
bytes.toString('base64') !== proposal.mask ||
|
|
363
|
-
aggregateBytes > objectProposalLimits.packedBytes)
|
|
364
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Proposal masks are malformed or oversized');
|
|
365
|
-
const unusedBits = expectedBytes * 8 - image.width * image.height;
|
|
366
|
-
const allowed = unusedBits ? (1 << (8 - unusedBits)) - 1 : 255;
|
|
367
|
-
if ((bytes[expectedBytes - 1] & ~allowed) !== 0)
|
|
368
|
-
throw new ClassifierError('INVALID_RESPONSE', 'Proposal mask has nonzero padding bits');
|
|
369
|
-
return {
|
|
370
|
-
id: proposal.id,
|
|
371
|
-
mask: {
|
|
372
|
-
data: new Uint8Array(bytes),
|
|
373
|
-
width: image.width,
|
|
374
|
-
height: image.height,
|
|
375
|
-
format: 'binary-lsb0',
|
|
376
|
-
},
|
|
377
|
-
qualityScore: proposal.qualityScore,
|
|
378
|
-
};
|
|
379
|
-
});
|
|
380
|
-
return {
|
|
381
|
-
proposals,
|
|
382
|
-
metadata: {
|
|
383
|
-
method: 'mobile-sam-nine-point-proposals-v1',
|
|
384
|
-
backend: 'mobile-sam',
|
|
385
|
-
durationMs: Math.max(0, performance.now() - started),
|
|
386
|
-
input: { width: image.width, height: image.height },
|
|
387
|
-
},
|
|
388
|
-
};
|
|
389
|
-
},
|
|
390
|
-
settle: () => pool.settle(),
|
|
391
|
-
dispose: () => pool.dispose(),
|
|
392
|
-
};
|
|
393
|
-
}
|
|
394
|
-
function snapshotInput(input) {
|
|
395
|
-
const image = input.image;
|
|
396
|
-
const pixels = image.width * image.height;
|
|
397
|
-
const prompt = input.prompt ?? 'whole-region';
|
|
398
|
-
if (image.format !== 'rgba8' ||
|
|
399
|
-
!(image.data instanceof Uint8Array) ||
|
|
400
|
-
!Number.isSafeInteger(image.width) ||
|
|
401
|
-
!Number.isSafeInteger(image.height) ||
|
|
402
|
-
!Number.isSafeInteger(pixels) ||
|
|
403
|
-
image.width < 1 ||
|
|
404
|
-
image.height < 1 ||
|
|
405
|
-
pixels > limits.imagePixels ||
|
|
406
|
-
image.data.length !== pixels * 4)
|
|
407
|
-
throw new ClassifierError('INVALID_INPUT', 'Segmentation requires canonical RGBA8 within the image pixel limit');
|
|
408
|
-
if ((prompt !== 'whole-region' && prompt !== 'seed-region') ||
|
|
409
|
-
!pixelRectSchema.safeParse(input.seedRegion).success ||
|
|
410
|
-
!pixelRectSchema.safeParse(input.wholeRegion).success ||
|
|
411
|
-
!pixelRectSchema.safeParse(input.roi).success)
|
|
412
|
-
throw new ClassifierError('INVALID_INPUT', 'Segmentation regions must be positive integer rectangles');
|
|
413
|
-
for (const region of [input.seedRegion, input.wholeRegion, input.roi])
|
|
414
|
-
if (region.left + region.width > image.width || region.top + region.height > image.height)
|
|
415
|
-
throw new ClassifierError('INVALID_INPUT', 'Segmentation region lies outside the source image');
|
|
416
|
-
return {
|
|
417
|
-
data: new Uint8Array(image.data),
|
|
418
|
-
width: image.width,
|
|
419
|
-
height: image.height,
|
|
420
|
-
seedRegion: { ...input.seedRegion },
|
|
421
|
-
wholeRegion: { ...input.wholeRegion },
|
|
422
|
-
roi: { ...input.roi },
|
|
423
|
-
prompt,
|
|
424
|
-
};
|
|
425
|
-
}
|
|
426
|
-
function snapshotPreviousMask(input, previousMask, image) {
|
|
427
|
-
if (!previousMask ||
|
|
428
|
-
previousMask.format !== 'binary-u8' ||
|
|
429
|
-
!(previousMask.data instanceof Uint8Array) ||
|
|
430
|
-
previousMask.width !== image.width ||
|
|
431
|
-
previousMask.height !== image.height ||
|
|
432
|
-
previousMask.data.length !== image.width * image.height)
|
|
433
|
-
throw new ClassifierError('INVALID_INPUT', 'Expansion requires a full-source binary mask');
|
|
434
|
-
try {
|
|
435
|
-
validateMobileSamPreviousMask({
|
|
436
|
-
width: image.width,
|
|
437
|
-
height: image.height,
|
|
438
|
-
rgba: image.data,
|
|
439
|
-
previousMask: previousMask.data,
|
|
440
|
-
roi: image.roi,
|
|
441
|
-
});
|
|
442
|
-
}
|
|
443
|
-
catch {
|
|
444
|
-
throw new ClassifierError('INVALID_INPUT', 'Expansion mask must be nonempty binary foreground inside visible ROI pixels');
|
|
445
|
-
}
|
|
446
|
-
return new Uint8Array(previousMask.data);
|
|
447
|
-
}
|
|
448
|
-
function snapshotProposalInput(input) {
|
|
449
|
-
const { image } = input;
|
|
450
|
-
const pixels = image.width * image.height;
|
|
451
|
-
if (pixels > objectProposalLimits.imagePixels)
|
|
452
|
-
throw new ClassifierError('RESOURCE_LIMIT', 'Proposal image exceeds the two-megapixel limit');
|
|
453
|
-
if (image.format !== 'rgba8' ||
|
|
454
|
-
!(image.data instanceof Uint8Array) ||
|
|
455
|
-
!Number.isSafeInteger(image.width) ||
|
|
456
|
-
!Number.isSafeInteger(image.height) ||
|
|
457
|
-
!Number.isSafeInteger(pixels) ||
|
|
458
|
-
image.width < 1 ||
|
|
459
|
-
image.height < 1 ||
|
|
460
|
-
image.data.length !== pixels * 4)
|
|
461
|
-
throw new ClassifierError('INVALID_INPUT', 'Proposals require canonical RGBA8 pixels');
|
|
462
|
-
for (const region of [input.seedRegion, input.roi])
|
|
463
|
-
if (!pixelRectSchema.safeParse(region).success ||
|
|
464
|
-
region.left + region.width > image.width ||
|
|
465
|
-
region.top + region.height > image.height)
|
|
466
|
-
throw new ClassifierError('INVALID_INPUT', 'Proposal region lies outside the source image');
|
|
467
|
-
return {
|
|
468
|
-
image: { data: new Uint8Array(image.data), width: image.width, height: image.height },
|
|
469
|
-
seedRegion: { ...input.seedRegion },
|
|
470
|
-
roi: { ...input.roi },
|
|
471
|
-
};
|
|
472
|
-
}
|
|
1
|
+
export { createMobileSamSegmenter, createMobileSamPortForPool } from './segmenter.js';
|
|
2
|
+
export { createMobileSamObjectProposals } from './object-proposals.js';
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { BinaryMask, SegmentationInput } from '../../../entities/segmentation.js';
|
|
2
|
+
import type { ObjectProposalInput } from '../../../entities/object-proposals.js';
|
|
3
|
+
export declare function snapshotInput(input: SegmentationInput): {
|
|
4
|
+
data: Uint8Array<ArrayBuffer>;
|
|
5
|
+
width: number;
|
|
6
|
+
height: number;
|
|
7
|
+
seedRegion: {
|
|
8
|
+
left: number;
|
|
9
|
+
top: number;
|
|
10
|
+
width: number;
|
|
11
|
+
height: number;
|
|
12
|
+
};
|
|
13
|
+
wholeRegion: {
|
|
14
|
+
left: number;
|
|
15
|
+
top: number;
|
|
16
|
+
width: number;
|
|
17
|
+
height: number;
|
|
18
|
+
};
|
|
19
|
+
roi: {
|
|
20
|
+
left: number;
|
|
21
|
+
top: number;
|
|
22
|
+
width: number;
|
|
23
|
+
height: number;
|
|
24
|
+
};
|
|
25
|
+
prompt: "whole-region" | "seed-region";
|
|
26
|
+
};
|
|
27
|
+
export declare function snapshotPreviousMask(input: SegmentationInput, previousMask: BinaryMask, image: ReturnType<typeof snapshotInput>): Uint8Array<ArrayBuffer>;
|
|
28
|
+
export declare function snapshotProposalInput(input: ObjectProposalInput): {
|
|
29
|
+
image: {
|
|
30
|
+
data: Uint8Array<ArrayBuffer>;
|
|
31
|
+
width: number;
|
|
32
|
+
height: number;
|
|
33
|
+
};
|
|
34
|
+
seedRegion: {
|
|
35
|
+
left: number;
|
|
36
|
+
top: number;
|
|
37
|
+
width: number;
|
|
38
|
+
height: number;
|
|
39
|
+
};
|
|
40
|
+
roi: {
|
|
41
|
+
left: number;
|
|
42
|
+
top: number;
|
|
43
|
+
width: number;
|
|
44
|
+
height: number;
|
|
45
|
+
};
|
|
46
|
+
};
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { ClassifierError } from '../../../entities/errors.js';
|
|
2
|
+
import { objectProposalLimits } from '../../../entities/object-proposals.js';
|
|
3
|
+
import { limits, pixelRectSchema } from '../../../entities/contracts.js';
|
|
4
|
+
import { validateMobileSamPreviousMask } from './segmentation-masks.js';
|
|
5
|
+
export function snapshotInput(input) {
|
|
6
|
+
const image = input.image;
|
|
7
|
+
const pixels = image.width * image.height;
|
|
8
|
+
const prompt = input.prompt ?? 'whole-region';
|
|
9
|
+
if (image.format !== 'rgba8' ||
|
|
10
|
+
!(image.data instanceof Uint8Array) ||
|
|
11
|
+
!Number.isSafeInteger(image.width) ||
|
|
12
|
+
!Number.isSafeInteger(image.height) ||
|
|
13
|
+
!Number.isSafeInteger(pixels) ||
|
|
14
|
+
image.width < 1 ||
|
|
15
|
+
image.height < 1 ||
|
|
16
|
+
pixels > limits.imagePixels ||
|
|
17
|
+
image.data.length !== pixels * 4)
|
|
18
|
+
throw new ClassifierError('INVALID_INPUT', 'Segmentation requires canonical RGBA8 within the image pixel limit');
|
|
19
|
+
if ((prompt !== 'whole-region' && prompt !== 'seed-region') ||
|
|
20
|
+
!pixelRectSchema.safeParse(input.seedRegion).success ||
|
|
21
|
+
!pixelRectSchema.safeParse(input.wholeRegion).success ||
|
|
22
|
+
!pixelRectSchema.safeParse(input.roi).success)
|
|
23
|
+
throw new ClassifierError('INVALID_INPUT', 'Segmentation regions must be positive integer rectangles');
|
|
24
|
+
for (const region of [input.seedRegion, input.wholeRegion, input.roi])
|
|
25
|
+
if (region.left + region.width > image.width || region.top + region.height > image.height)
|
|
26
|
+
throw new ClassifierError('INVALID_INPUT', 'Segmentation region lies outside the source image');
|
|
27
|
+
return {
|
|
28
|
+
data: new Uint8Array(image.data),
|
|
29
|
+
width: image.width,
|
|
30
|
+
height: image.height,
|
|
31
|
+
seedRegion: { ...input.seedRegion },
|
|
32
|
+
wholeRegion: { ...input.wholeRegion },
|
|
33
|
+
roi: { ...input.roi },
|
|
34
|
+
prompt,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
export function snapshotPreviousMask(input, previousMask, image) {
|
|
38
|
+
if (!previousMask ||
|
|
39
|
+
previousMask.format !== 'binary-u8' ||
|
|
40
|
+
!(previousMask.data instanceof Uint8Array) ||
|
|
41
|
+
previousMask.width !== image.width ||
|
|
42
|
+
previousMask.height !== image.height ||
|
|
43
|
+
previousMask.data.length !== image.width * image.height)
|
|
44
|
+
throw new ClassifierError('INVALID_INPUT', 'Expansion requires a full-source binary mask');
|
|
45
|
+
try {
|
|
46
|
+
validateMobileSamPreviousMask({
|
|
47
|
+
width: image.width,
|
|
48
|
+
height: image.height,
|
|
49
|
+
rgba: image.data,
|
|
50
|
+
previousMask: previousMask.data,
|
|
51
|
+
roi: image.roi,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
throw new ClassifierError('INVALID_INPUT', 'Expansion mask must be nonempty binary foreground inside visible ROI pixels');
|
|
56
|
+
}
|
|
57
|
+
return new Uint8Array(previousMask.data);
|
|
58
|
+
}
|
|
59
|
+
export function snapshotProposalInput(input) {
|
|
60
|
+
const { image } = input;
|
|
61
|
+
const pixels = image.width * image.height;
|
|
62
|
+
if (pixels > objectProposalLimits.imagePixels)
|
|
63
|
+
throw new ClassifierError('RESOURCE_LIMIT', 'Proposal image exceeds the two-megapixel limit');
|
|
64
|
+
if (image.format !== 'rgba8' ||
|
|
65
|
+
!(image.data instanceof Uint8Array) ||
|
|
66
|
+
!Number.isSafeInteger(image.width) ||
|
|
67
|
+
!Number.isSafeInteger(image.height) ||
|
|
68
|
+
!Number.isSafeInteger(pixels) ||
|
|
69
|
+
image.width < 1 ||
|
|
70
|
+
image.height < 1 ||
|
|
71
|
+
image.data.length !== pixels * 4)
|
|
72
|
+
throw new ClassifierError('INVALID_INPUT', 'Proposals require canonical RGBA8 pixels');
|
|
73
|
+
for (const region of [input.seedRegion, input.roi])
|
|
74
|
+
if (!pixelRectSchema.safeParse(region).success ||
|
|
75
|
+
region.left + region.width > image.width ||
|
|
76
|
+
region.top + region.height > image.height)
|
|
77
|
+
throw new ClassifierError('INVALID_INPUT', 'Proposal region lies outside the source image');
|
|
78
|
+
return {
|
|
79
|
+
image: { data: new Uint8Array(image.data), width: image.width, height: image.height },
|
|
80
|
+
seedRegion: { ...input.seedRegion },
|
|
81
|
+
roi: { ...input.roi },
|
|
82
|
+
};
|
|
83
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare const encoderIdentity: Readonly<{
|
|
2
|
+
readonly bytes: 28157093;
|
|
3
|
+
readonly sha256: "580f5fb648ea1062c0aabc26217aed56921985f03f0cbbd852bba81d760cc749";
|
|
4
|
+
}>;
|
|
5
|
+
export declare const decoderIdentity: Readonly<{
|
|
6
|
+
readonly bytes: 16496559;
|
|
7
|
+
readonly sha256: "8976b90a87ba50a6a72217a5ff994f7d25ce16f2229fcc1ed259e1294c622ffe";
|
|
8
|
+
}>;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export const encoderIdentity = Object.freeze({
|
|
2
|
+
bytes: 28_157_093,
|
|
3
|
+
sha256: '580f5fb648ea1062c0aabc26217aed56921985f03f0cbbd852bba81d760cc749',
|
|
4
|
+
});
|
|
5
|
+
export const decoderIdentity = Object.freeze({
|
|
6
|
+
bytes: 16_496_559,
|
|
7
|
+
sha256: '8976b90a87ba50a6a72217a5ff994f7d25ce16f2229fcc1ed259e1294c622ffe',
|
|
8
|
+
});
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { ObjectProposalPort } from '../../../entities/object-proposals.js';
|
|
2
|
+
import { type MobileSamObjectProposalsOptions } from './options.js';
|
|
3
|
+
/**
|
|
4
|
+
* Explicit CPU-only proposals: historical 3x3/36 default or opt-in seed-center/four.
|
|
5
|
+
* @experimental Native proposal adapter; unqualified for general spatial or mask accuracy.
|
|
6
|
+
*/
|
|
7
|
+
export declare function createMobileSamObjectProposals(options: MobileSamObjectProposalsOptions): Promise<ObjectProposalPort>;
|