zorya-classifier 0.2.1 → 0.2.2

package/dist/infrastructure/segmentation/mobile-sam/segmenter.js

@@ -0,0 +1,268 @@
+import { performance } from 'node:perf_hooks';
+import { ClassifierError, assertActive } from '../../../entities/errors.js';
+import { NativeChildPool } from '../native-child.js';
+import { verifyPinnedAsset } from '../verify-asset.js';
+import { probeBackend } from '../probe-backend.js';
+import { nativeErrorReplySchema, nativeMobileSamExpansionReplySchema, nativeSegmentationReplySchema, } from '../native-protocol.js';
+import { decoderIdentity, encoderIdentity } from './model-assets.js';
+import { optionsSchema } from './options.js';
+import { snapshotInput, snapshotPreviousMask } from './input.js';
+import { packFullSourceBinaryMask, unpackFullSourceBinaryMask, validateMobileSamPreviousMask, } from './segmentation-masks.js';
+/**
+ * Explicit pinned MobileSAM inference in a separately owned Node child process.
+ * @experimental Native segmentation adapter; unqualified for general mask or object accuracy.
+ */
+export async function createMobileSamSegmenter(options) {
+    const checked = optionsSchema.safeParse(options);
+    if (!checked.success)
+        throw new ClassifierError('INVALID_INPUT', 'Invalid MobileSAM options');
+    const parsed = checked.data;
+    if (parsed.executionProvider === 'dml' && process.platform !== 'win32')
+        throw new ClassifierError('UNSUPPORTED', 'DirectML is supported only on Windows');
+    try {
+        import.meta.resolve('onnxruntime-node');
+    }
+    catch {
+        throw new ClassifierError('UNSUPPORTED', 'onnxruntime-node is not installed');
+    }
+    await verifyPinnedAsset(parsed.encoderPath, encoderIdentity, 'MobileSAM encoder');
+    await verifyPinnedAsset(parsed.decoderPath, decoderIdentity, 'MobileSAM decoder');
+    const sourceWorker = import.meta.url.endsWith('.ts');
+    const workerUrl = new URL(sourceWorker ? '../native-worker.ts' : '../native-worker.js', import.meta.url);
+    await probeBackend(workerUrl, 'mobile-sam', { options: parsed });
+    const pool = new NativeChildPool({
+        workerUrl,
+        maxActive: 1,
+        maxWaiting: 2,
+        jobTimeoutMs: 10_000,
+        abortGraceMs: 250,
+        encode: (job) => Buffer.from(JSON.stringify({ id: job.id, ...job.payload })),
+        decode: (raw, id) => {
+            const error = nativeErrorReplySchema.safeParse(raw);
+            if (error.success) {
+                if (error.data.id !== id)
+                    throw new ClassifierError('INVALID_RESPONSE', 'Native child error ID mismatch');
+                throw new ClassifierError(error.data.code, error.data.message);
+            }
+            const reply = nativeSegmentationReplySchema.safeParse(raw);
+            if (reply.success && reply.data.id === id)
+                return reply.data;
+            const expansion = nativeMobileSamExpansionReplySchema.safeParse(raw);
+            if (!expansion.success || expansion.data.id !== id)
+                throw new ClassifierError('INVALID_RESPONSE', 'Native child returned an invalid or mismatched reply');
+            return expansion.data;
+        },
+    });
+    return createMobileSamPortForPool(pool, parsed);
+}
+/** Internal adapter seam: pool ownership stays with the caller. */
+export function createMobileSamPortForPool(pool, parsed) {
+    return {
+        async segment(input) {
+            const started = performance.now();
+            assertActive(input.signal);
+            let image;
+            let maskData;
+            try {
+                const raw = await pool.runLazy(input.signal, () => {
+                    image = snapshotInput(input);
+                    const rgba = Buffer.from(image.data);
+                    let rgbaBase64;
+                    try {
+                        rgbaBase64 = rgba.toString('base64');
+                    }
+                    finally {
+                        rgba.fill(0);
+                    }
+                    return {
+                        kind: 'mobile-sam',
+                        width: image.width,
+                        height: image.height,
+                        rgba: rgbaBase64,
+                        seedRegion: image.seedRegion,
+                        wholeRegion: image.wholeRegion,
+                        roi: image.roi,
+                        prompt: image.prompt,
+                        options: parsed,
+                    };
+                });
+                if (!image)
+                    throw new ClassifierError('INVALID_RESPONSE', 'Native segmentation omitted its input snapshot');
+                const checkedReply = nativeSegmentationReplySchema.safeParse(raw);
+                if (!checkedReply.success ||
+                    checkedReply.data.width !== image.width ||
+                    checkedReply.data.height !== image.height)
+                    throw new ClassifierError('INVALID_RESPONSE', 'Native child returned an invalid segmentation reply');
+                maskData = Buffer.from(checkedReply.data.mask, 'base64');
+                if (maskData.length !== image.width * image.height ||
+                    maskData.toString('base64') !== checkedReply.data.mask)
+                    throw new ClassifierError('INVALID_RESPONSE', 'Native child mask dimensions or encoding are invalid');
+                return {
+                    status: 'segmented',
+                    mask: {
+                        data: new Uint8Array(maskData),
+                        width: image.width,
+                        height: image.height,
+                        format: 'binary-u8',
+                    },
+                    metadata: {
+                        method: checkedReply.data.method,
+                        backend: 'mobile-sam',
+                        durationMs: Math.max(0, performance.now() - started),
+                        input: { width: image.width, height: image.height },
+                    },
+                };
+            }
+            finally {
+                maskData?.fill(0);
+                image?.data.fill(0);
+            }
+        },
+        async expand(input, previousMask) {
+            const started = performance.now();
+            assertActive(input.signal);
+            let image;
+            let previous;
+            let resultData;
+            let transferred = false;
+            try {
+                const raw = await pool.runLazy(input.signal, () => {
+                    image = snapshotInput(input);
+                    previous = snapshotPreviousMask(input, previousMask, image);
+                    const packedPrevious = packFullSourceBinaryMask(previous, image.width, image.height);
+                    let previousMaskBase64;
+                    try {
+                        const packedBuffer = Buffer.from(packedPrevious);
+                        try {
+                            previousMaskBase64 = packedBuffer.toString('base64');
+                        }
+                        finally {
+                            packedBuffer.fill(0);
+                        }
+                    }
+                    finally {
+                        packedPrevious.fill(0);
+                    }
+                    const rgbaBuffer = Buffer.from(image.data);
+                    let rgbaBase64;
+                    try {
+                        rgbaBase64 = rgbaBuffer.toString('base64');
+                    }
+                    finally {
+                        rgbaBuffer.fill(0);
+                    }
+                    return {
+                        kind: 'mobile-sam-expand',
+                        width: image.width,
+                        height: image.height,
+                        rgba: rgbaBase64,
+                        previousMask: previousMaskBase64,
+                        seedRegion: image.seedRegion,
+                        wholeRegion: image.wholeRegion,
+                        roi: image.roi,
+                        prompt: image.prompt,
+                        options: parsed,
+                    };
+                });
+                if (!image || !previous)
+                    throw new ClassifierError('INVALID_RESPONSE', 'Native expansion omitted its input snapshot');
+                const checkedReply = nativeMobileSamExpansionReplySchema.safeParse(raw);
+                if (!checkedReply.success ||
+                    checkedReply.data.width !== image.width ||
+                    checkedReply.data.height !== image.height)
+                    throw new ClassifierError('INVALID_RESPONSE', 'Native child returned an invalid expansion reply');
+                if (checkedReply.data.status === 'unknown')
+                    return {
+                        status: 'unknown',
+                        reason: 'no-enclosing-mask',
+                        metadata: {
+                            method: checkedReply.data.method,
+                            backend: 'mobile-sam',
+                            durationMs: Math.max(0, performance.now() - started),
+                            input: { width: image.width, height: image.height },
+                        },
+                    };
+                const packedBytes = Buffer.from(checkedReply.data.mask, 'base64');
+                let data;
+                try {
+                    if (packedBytes.toString('base64') !== checkedReply.data.mask)
+                        throw new ClassifierError('INVALID_RESPONSE', 'Native expansion mask encoding is invalid');
+                    try {
+                        data = unpackFullSourceBinaryMask(packedBytes, image.width, image.height);
+                        resultData = data;
+                    }
+                    catch {
+                        throw new ClassifierError('INVALID_RESPONSE', 'Native expansion packed mask is invalid');
+                    }
+                }
+                finally {
+                    packedBytes.fill(0);
+                }
+                let oldStats;
+                let newStats;
+                try {
+                    oldStats = validateMobileSamPreviousMask({
+                        width: image.width,
+                        height: image.height,
+                        rgba: image.data,
+                        previousMask: previous,
+                        roi: image.roi,
+                    });
+                    newStats = validateMobileSamPreviousMask({
+                        width: image.width,
+                        height: image.height,
+                        rgba: image.data,
+                        previousMask: data,
+                        roi: image.roi,
+                    });
+                }
+                catch {
+                    data.fill(0);
+                    throw new ClassifierError('INVALID_RESPONSE', 'Native expanded mask is invalid');
+                }
+                let hasNewForeground = false;
+                for (let index = 0; index < data.length; index++) {
+                    if (previous[index] === 1 && data[index] !== 1) {
+                        data.fill(0);
+                        throw new ClassifierError('INVALID_RESPONSE', 'Native expansion lost prior foreground');
+                    }
+                    if (data[index] === 1 && previous[index] === 0)
+                        hasNewForeground = true;
+                }
+                if (!hasNewForeground ||
+                    newStats.area <= oldStats.area ||
+                    newStats.bounds.left > oldStats.bounds.left ||
+                    newStats.bounds.top > oldStats.bounds.top ||
+                    newStats.bounds.right < oldStats.bounds.right ||
+                    newStats.bounds.bottom < oldStats.bounds.bottom ||
+                    (newStats.bounds.left === oldStats.bounds.left &&
+                        newStats.bounds.top === oldStats.bounds.top &&
+                        newStats.bounds.right === oldStats.bounds.right &&
+                        newStats.bounds.bottom === oldStats.bounds.bottom)) {
+                    data.fill(0);
+                    throw new ClassifierError('INVALID_RESPONSE', 'Native expansion is not a strict enclosing mask');
+                }
+                transferred = true;
+                return {
+                    status: 'segmented',
+                    mask: { data, width: image.width, height: image.height, format: 'binary-u8' },
+                    metadata: {
+                        method: checkedReply.data.method,
+                        backend: 'mobile-sam',
+                        durationMs: Math.max(0, performance.now() - started),
+                        input: { width: image.width, height: image.height },
+                        qualityScore: checkedReply.data.qualityScore,
+                    },
+                };
+            }
+            finally {
+                image?.data.fill(0);
+                previous?.fill(0);
+                if (resultData && !transferred)
+                    resultData.fill(0);
+            }
+        },
+        settle: () => pool.settle(),
+        dispose: () => pool.dispose(),
+    };
+}

package/dist/infrastructure/segmentation/mobile-sam/worker-expand.d.ts

@@ -0,0 +1,7 @@
+import type { NativeMobileSamExpansionReply, NativeSegmentationRequest } from '../native-protocol.js';
+type ExpansionRequest = Extract<NativeSegmentationRequest, {
+    kind: 'mobile-sam-expand';
+}>;
+/** Run one same-seed decode and return only the smallest strict enclosing mask. */
+export declare function runMobileSamExpandTask(request: ExpansionRequest): Promise<NativeMobileSamExpansionReply>;
+export {};

package/dist/infrastructure/segmentation/mobile-sam/worker-expand.js

@@ -0,0 +1,188 @@
+import sharp from 'sharp';
+import { buildMobileSamSessionOptions, decoderIdentity, encoderIdentity, verifiedBytes, } from './worker-models.js';
+import { mobileSamSegmentationPrompt, packFullSourceBinaryMask, selectEnclosingMobileSamCandidate, unpackFullSourceBinaryMask, validateMobileSamPreviousMask, } from './segmentation-masks.js';
+/** Run one same-seed decode and return only the smallest strict enclosing mask. */
+export async function runMobileSamExpandTask(request) {
+    const ort = await import('onnxruntime-node');
+    if (!ort
+        .listSupportedBackends()
+        .some((item) => item.name === request.options.executionProvider && item.bundled))
+        throw new Error('Explicitly requested ONNX execution provider is unavailable');
+    const sessionOptions = buildMobileSamSessionOptions(request.options);
+    const encoder = await ort.InferenceSession.create(await verifiedBytes(request.options.encoderPath, encoderIdentity), sessionOptions);
+    let decoder;
+    let imageTensor;
+    let embedding;
+    let result;
+    let releaseFailed = false;
+    let rgba;
+    let packedPrevious;
+    let previousMask;
+    let selectedMask;
+    try {
+        decoder = await ort.InferenceSession.create(await verifiedBytes(request.options.decoderPath, decoderIdentity), sessionOptions);
+        rgba = Buffer.from(request.rgba, 'base64');
+        packedPrevious = Buffer.from(request.previousMask, 'base64');
+        if (rgba.length !== request.width * request.height * 4)
+            throw new Error('Invalid expansion RGBA input size');
+        previousMask = unpackFullSourceBinaryMask(packedPrevious, request.width, request.height);
+        validateMobileSamPreviousMask({
+            width: request.width,
+            height: request.height,
+            rgba,
+            previousMask,
+            roi: request.roi,
+        });
+        const scale = 1024 / Math.max(request.width, request.height);
+        const resizedWidth = Math.round(request.width * scale);
+        const resizedHeight = Math.round(request.height * scale);
+        if (resizedWidth < 1 || resizedHeight < 1)
+            throw new Error('Unrepresentable image aspect ratio');
+        const pipeline = sharp(rgba, {
+            raw: { width: request.width, height: request.height, channels: 4 },
+        });
+        let rgb;
+        try {
+            rgb = await pipeline
+                .removeAlpha()
+                .resize(resizedWidth, resizedHeight, { fit: 'fill', kernel: 'linear' })
+                .raw()
+                .toBuffer();
+        }
+        finally {
+            pipeline.destroy();
+        }
+        if (rgb.length !== resizedWidth * resizedHeight * 3)
+            throw new Error('Invalid resized RGB data');
+        imageTensor = new ort.Tensor('float32', Float32Array.from(rgb), [
+            resizedHeight,
+            resizedWidth,
+            3,
+        ]);
+        rgb.fill(0);
+        const encoded = await encoder.run({ input_image: imageTensor });
+        const embeddingTensor = encoded.image_embeddings;
+        if (!(embeddingTensor instanceof ort.Tensor) ||
+            embeddingTensor.type !== 'float32' ||
+            JSON.stringify(embeddingTensor.dims) !== JSON.stringify([1, 256, 64, 64]) ||
+            !(embeddingTensor.data instanceof Float32Array))
+            throw new Error('Unexpected MobileSAM encoder output');
+        embedding = embeddingTensor;
+        const prompt = mobileSamSegmentationPrompt({
+            prompt: request.prompt,
+            seedRegion: request.seedRegion,
+            wholeRegion: request.wholeRegion,
+            xScale: resizedWidth / request.width,
+            yScale: resizedHeight / request.height,
+        });
+        const pointCoords = new ort.Tensor('float32', prompt.coordinates, [1, 2, 2]);
+        const pointLabels = new ort.Tensor('float32', prompt.labels, [1, 2]);
+        const maskInput = new ort.Tensor('float32', new Float32Array(256 * 256), [1, 1, 256, 256]);
+        const hasMaskInput = new ort.Tensor('float32', Float32Array.of(0), [1]);
+        const originalSize = new ort.Tensor('float32', Float32Array.of(request.height, request.width), [
+            2,
+        ]);
+        let decoded;
+        try {
+            decoded = await decoder.run({
+                image_embeddings: embedding,
+                point_coords: pointCoords,
+                point_labels: pointLabels,
+                mask_input: maskInput,
+                has_mask_input: hasMaskInput,
+                orig_im_size: originalSize,
+            });
+            const masks = decoded.masks;
+            const scores = decoded.iou_predictions;
+            if (!(masks instanceof ort.Tensor) ||
+                masks.type !== 'float32' ||
+                JSON.stringify(masks.dims) !== JSON.stringify([1, 4, request.height, request.width]) ||
+                !(masks.data instanceof Float32Array) ||
+                !(scores instanceof ort.Tensor) ||
+                scores.type !== 'float32' ||
+                JSON.stringify(scores.dims) !== JSON.stringify([1, 4]) ||
+                !(scores.data instanceof Float32Array))
+                throw new Error('Unexpected MobileSAM expansion decoder output');
+            const candidate = selectEnclosingMobileSamCandidate({
+                width: request.width,
+                height: request.height,
+                rgba,
+                logits: masks.data,
+                scores: scores.data,
+                previousMask,
+                roi: request.roi,
+            });
+            if (candidate) {
+                selectedMask = candidate.mask;
+                const packed = packFullSourceBinaryMask(selectedMask, request.width, request.height);
+                const packedBuffer = Buffer.from(packed);
+                try {
+                    result = {
+                        id: request.id,
+                        status: 'expanded',
+                        width: request.width,
+                        height: request.height,
+                        mask: packedBuffer.toString('base64'),
+                        method: 'mobile-sam-enclosing-mask-v1',
+                        qualityScore: candidate.score,
+                    };
+                }
+                finally {
+                    packedBuffer.fill(0);
+                    packed.fill(0);
+                }
+            }
+            else {
+                result = {
+                    id: request.id,
+                    status: 'unknown',
+                    reason: 'no-enclosing-mask',
+                    width: request.width,
+                    height: request.height,
+                    method: 'mobile-sam-enclosing-mask-v1',
+                };
+            }
+        }
+        finally {
+            prompt.coordinates.fill(0);
+            prompt.labels.fill(0);
+            for (const tensor of [pointCoords, pointLabels, maskInput, hasMaskInput, originalSize])
+                if (tensor.data instanceof Float32Array)
+                    tensor.data.fill(0);
+            pointCoords.dispose();
+            pointLabels.dispose();
+            maskInput.dispose();
+            hasMaskInput.dispose();
+            originalSize.dispose();
+            if (decoded)
+                for (const tensor of Object.values(decoded))
+                    if (tensor instanceof ort.Tensor) {
+                        if (tensor.data instanceof Float32Array || tensor.data instanceof Uint8Array)
+                            tensor.data.fill(0);
+                        tensor.dispose();
+                    }
+        }
+    }
+    finally {
+        imageTensor?.dispose();
+        if (embedding && embedding !== imageTensor) {
+            if (embedding.data instanceof Float32Array)
+                embedding.data.fill(0);
+            embedding.dispose();
+        }
+        previousMask?.fill(0);
+        selectedMask?.fill(0);
+        rgba?.fill(0);
+        packedPrevious?.fill(0);
+        const releases = await Promise.allSettled([
+            encoder.release(),
+            ...(decoder ? [decoder.release()] : []),
+        ]);
+        releaseFailed = releases.some((release) => release.status === 'rejected');
+    }
+    if (releaseFailed)
+        throw new Error('MobileSAM expansion session release failed');
+    if (!result)
+        throw new Error('MobileSAM completed without an expansion result');
+    return result;
+}

package/dist/infrastructure/segmentation/mobile-sam/worker-inference.d.ts

@@ -1,149 +1,6 @@
-import type { NativeMobileSamExpansionReply, NativeSegmentationReply, NativeSegmentationRequest } from '../native-protocol.js';
-import type { NativeObjectProposalReply } from '../native-protocol.js';
-export declare function probeMobileSam(options: Extract<NativeSegmentationRequest, {
-    kind: 'probe-mobile-sam';
-}>['options']): Promise<void>;
-interface MaskCandidate {
-    token: number;
-    score: number;
-    mask: Uint8Array;
-}
-/** Sort by model quality and use decoder token order as a deterministic tie break. */
-export declare function selectBestMaskCandidate<T extends MaskCandidate>(candidates: readonly T[]): T | undefined;
-export declare function mobileSamSegmentationPrompt(input: {
-    prompt: 'whole-region' | 'seed-region';
-    seedRegion: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-    wholeRegion: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-    xScale: number;
-    yScale: number;
-}): {
-    coordinates: Float32Array<ArrayBuffer>;
-    labels: Float32Array<ArrayBuffer>;
-};
-export declare function collectMobileSamSegmentationCandidates(input: {
-    width: number;
-    height: number;
-    rgba: Uint8Array;
-    logits: Float32Array;
-    scores: Float32Array;
-    roi: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-    seedRegion: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-    prompt: 'whole-region' | 'seed-region';
-}): MaskCandidate[];
-export declare function selectMobileSamSegmentationMask(input: {
-    candidates: readonly MaskCandidate[];
-    width: number;
-    height: number;
-    prompt: 'whole-region' | 'seed-region';
-}): Uint8Array;
-interface EnclosingMobileSamCandidate {
-    token: number;
-    score: number;
-    mask: Uint8Array;
-    area: number;
-    bounds: {
-        left: number;
-        top: number;
-        right: number;
-        bottom: number;
-    };
-}
-/** Full-source segmentation masks use the segmentation pixel cap, not the proposal cap. */
-export declare function packFullSourceBinaryMask(mask: Uint8Array, width: number, height: number): Uint8Array;
-/** Strictly decode the LSB-first full-source mask representation used by expansion IPC. */
-export declare function unpackFullSourceBinaryMask(data: Uint8Array, width: number, height: number): Uint8Array;
-/** Validate a prior full-source mask and return its foreground area and bounds. */
-export declare function validateMobileSamPreviousMask(input: {
-    width: number;
-    height: number;
-    rgba: Uint8Array;
-    previousMask: Uint8Array;
-    roi: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-}): {
-    area: number;
-    bounds: {
-        left: number;
-        top: number;
-        right: number;
-        bottom: number;
-    };
-};
-/** Select the smallest actual decoder mask that strictly encloses prior foreground. */
-export declare function selectEnclosingMobileSamCandidate(input: {
-    width: number;
-    height: number;
-    rgba: Uint8Array;
-    logits: Float32Array;
-    scores: Float32Array;
-    previousMask: Uint8Array;
-    roi: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-}): EnclosingMobileSamCandidate | undefined;
-export declare function runMobileSamTask(request: Extract<NativeSegmentationRequest, {
-    kind: 'mobile-sam';
-}>): Promise<NativeSegmentationReply>;
-type ExpansionRequest = Extract<NativeSegmentationRequest, {
-    kind: 'mobile-sam-expand';
-}>;
-/** Run one same-seed decode and return only the smallest strict enclosing mask. */
-export declare function runMobileSamExpandTask(request: ExpansionRequest): Promise<NativeMobileSamExpansionReply>;
-type ProposalRequest = Extract<NativeSegmentationRequest, {
-    kind: 'mobile-sam-proposals';
-}>;
-/** Produce the fixed row-major 3x3 prompt grid from the actual seed rectangle. */
-export declare function mobileSamProposalPoints(rect: {
-    left: number;
-    top: number;
-    width: number;
-    height: number;
-}): Array<{
-    x: number;
-    y: number;
-}>;
-/** Turn one decoder response into four owned packed candidates immediately. */
-export declare function packMobileSamProposalCandidates(input: {
-    width: number;
-    height: number;
-    rgba: Uint8Array;
-    roi: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-    pointIndex: number;
-    logits: Float32Array;
-    scores: Float32Array;
-}): NativeObjectProposalReply['proposals'];
-/** One encoder pass followed by nine independent positive-point decoder calls. */
-export declare function runMobileSamObjectProposalTask(request: ProposalRequest): Promise<NativeObjectProposalReply>;
-export {};
+export { probeMobileSam } from './worker-models.js';
+export { runMobileSamTask } from './worker-segment.js';
+export { runMobileSamExpandTask } from './worker-expand.js';
+export { runMobileSamObjectProposalTask } from './worker-proposals.js';
+export { selectBestMaskCandidate, mobileSamSegmentationPrompt, collectMobileSamSegmentationCandidates, selectMobileSamSegmentationMask, packFullSourceBinaryMask, unpackFullSourceBinaryMask, validateMobileSamPreviousMask, selectEnclosingMobileSamCandidate, } from './segmentation-masks.js';
+export { mobileSamProposalPoints, packMobileSamProposalCandidates } from './proposal-masks.js';