zorya-classifier 0.2.0 → 0.2.2

package/dist/infrastructure/segmentation/mobile-sam/worker-models.d.ts

@@ -0,0 +1,14 @@
+import type { InferenceSession } from 'onnxruntime-node';
+import type { NativeSegmentationRequest } from '../native-protocol.js';
+import { decoderIdentity, encoderIdentity } from './model-assets.js';
+export declare function verifiedBytes(file: string, identity: {
+    bytes: number;
+    sha256: string;
+}): Promise<Buffer<ArrayBuffer>>;
+export declare function buildMobileSamSessionOptions(options: Extract<NativeSegmentationRequest, {
+    kind: 'mobile-sam';
+}>['options']): InferenceSession.SessionOptions;
+export declare function probeMobileSam(options: Extract<NativeSegmentationRequest, {
+    kind: 'probe-mobile-sam';
+}>['options']): Promise<void>;
+export { encoderIdentity, decoderIdentity };

package/dist/infrastructure/segmentation/mobile-sam/worker-models.js

@@ -0,0 +1,72 @@
+import { createHash } from 'node:crypto';
+import { open } from 'node:fs/promises';
+import { decoderIdentity, encoderIdentity } from './model-assets.js';
+export async function verifiedBytes(file, identity) {
+    const handle = await open(file, 'r');
+    try {
+        const stat = await handle.stat();
+        if (!stat.isFile() || stat.size !== identity.bytes)
+            throw new Error('Pinned model size mismatch');
+        const bytes = Buffer.alloc(identity.bytes);
+        let offset = 0;
+        while (offset < bytes.length) {
+            const read = await handle.read(bytes, offset, bytes.length - offset, null);
+            if (read.bytesRead === 0)
+                throw new Error('Pinned model read was incomplete');
+            offset += read.bytesRead;
+        }
+        if ((await handle.read(Buffer.alloc(1), 0, 1, null)).bytesRead !== 0)
+            throw new Error('Pinned model changed while reading');
+        if (createHash('sha256').update(bytes).digest('hex') !== identity.sha256)
+            throw new Error('Pinned model SHA256 mismatch');
+        return bytes;
+    }
+    finally {
+        await handle.close();
+    }
+}
+export function buildMobileSamSessionOptions(options) {
+    return {
+        executionProviders: options.executionProvider === 'dml'
+            ? [{ name: 'dml', deviceId: options.deviceId ?? 0 }]
+            : ['cpu'],
+        intraOpNumThreads: options.threads,
+        interOpNumThreads: 1,
+        executionMode: 'sequential',
+        enableMemPattern: options.executionProvider !== 'dml',
+        graphOptimizationLevel: 'all',
+    };
+}
+export async function probeMobileSam(options) {
+    const ort = await import('onnxruntime-node');
+    if (!ort
+        .listSupportedBackends()
+        .some((item) => item.name === options.executionProvider && item.bundled))
+        throw new Error('Explicitly requested ONNX execution provider is unavailable');
+    const sessionOptions = buildMobileSamSessionOptions(options);
+    const encoderBytes = await verifiedBytes(options.encoderPath, encoderIdentity);
+    const decoderBytes = await verifiedBytes(options.decoderPath, decoderIdentity);
+    let encoder;
+    let decoder;
+    let releaseFailed = false;
+    try {
+        encoder = await ort.InferenceSession.create(encoderBytes, sessionOptions);
+        decoder = await ort.InferenceSession.create(decoderBytes, sessionOptions);
+        if (!encoder.inputNames.includes('input_image') ||
+            !encoder.outputNames.includes('image_embeddings') ||
+            !decoder.inputNames.includes('image_embeddings') ||
+            !decoder.outputNames.includes('masks') ||
+            !decoder.outputNames.includes('iou_predictions'))
+            throw new Error('Pinned MobileSAM graph tensor names are incompatible');
+    }
+    finally {
+        const closed = await Promise.allSettled([
+            ...(encoder ? [encoder.release()] : []),
+            ...(decoder ? [decoder.release()] : []),
+        ]);
+        releaseFailed = closed.some((item) => item.status === 'rejected');
+    }
+    if (releaseFailed)
+        throw new Error('MobileSAM probe session release failed');
+}
+export { encoderIdentity, decoderIdentity };

package/dist/infrastructure/segmentation/mobile-sam/worker-proposals.d.ts

@@ -0,0 +1,7 @@
+import type { NativeObjectProposalReply, NativeSegmentationRequest } from '../native-protocol.js';
+type ProposalRequest = Extract<NativeSegmentationRequest, {
+    kind: 'mobile-sam-proposals';
+}>;
+/** One encoder pass followed by a profile-bounded set of positive-point decoder calls. */
+export declare function runMobileSamObjectProposalTask(request: ProposalRequest): Promise<NativeObjectProposalReply>;
+export {};

package/dist/infrastructure/segmentation/mobile-sam/worker-proposals.js

@@ -0,0 +1,159 @@
+import sharp from 'sharp';
+import { objectProposalLimits } from '../../../entities/object-proposals.js';
+import { decoderIdentity, encoderIdentity, verifiedBytes } from './worker-models.js';
+import { mobileSamProposalPoints, packMobileSamProposalCandidates } from './proposal-masks.js';
+/** One encoder pass followed by a profile-bounded set of positive-point decoder calls. */
+export async function runMobileSamObjectProposalTask(request) {
+    const ort = await import('onnxruntime-node');
+    if (!ort.listSupportedBackends().some((item) => item.name === 'cpu' && item.bundled))
+        throw new Error('Explicitly requested ONNX CPU execution provider is unavailable');
+    const sessionOptions = {
+        executionProviders: ['cpu'],
+        intraOpNumThreads: request.options.threads,
+        interOpNumThreads: 1,
+        executionMode: 'sequential',
+        enableMemPattern: true,
+        graphOptimizationLevel: 'all',
+    };
+    const encoder = await ort.InferenceSession.create(await verifiedBytes(request.options.encoderPath, encoderIdentity), sessionOptions);
+    let decoder;
+    let embeddingTensor;
+    let reply;
+    let releaseFailed = false;
+    try {
+        decoder = await ort.InferenceSession.create(await verifiedBytes(request.options.decoderPath, decoderIdentity), sessionOptions);
+        const rgba = Buffer.from(request.rgba, 'base64');
+        const pixels = request.width * request.height;
+        if (rgba.length !== pixels * 4 || pixels > 2_097_152)
+            throw new Error('Invalid or oversized RGBA proposal input');
+        const scale = 1024 / Math.max(request.width, request.height);
+        const resizedWidth = Math.round(request.width * scale);
+        const resizedHeight = Math.round(request.height * scale);
+        const pipeline = sharp(rgba, {
+            raw: { width: request.width, height: request.height, channels: 4 },
+        });
+        let rgb;
+        try {
+            rgb = await pipeline
+                .removeAlpha()
+                .resize(resizedWidth, resizedHeight, { fit: 'fill', kernel: 'linear' })
+                .raw()
+                .toBuffer();
+        }
+        finally {
+            pipeline.destroy();
+        }
+        if (rgb.length !== resizedWidth * resizedHeight * 3)
+            throw new Error('Invalid resized RGB proposal input');
+        const imageTensor = new ort.Tensor('float32', Float32Array.from(rgb), [
+            resizedHeight,
+            resizedWidth,
+            3,
+        ]);
+        try {
+            const encoded = await encoder.run({ input_image: imageTensor });
+            const embedding = encoded.image_embeddings;
+            if (embedding instanceof ort.Tensor)
+                embeddingTensor = embedding;
+            if (!(embedding instanceof ort.Tensor) ||
+                embedding.type !== 'float32' ||
+                JSON.stringify(embedding.dims) !== JSON.stringify([1, 256, 64, 64]) ||
+                !(embedding.data instanceof Float32Array))
+                throw new Error('Unexpected MobileSAM encoder output');
+            const xScale = resizedWidth / request.width;
+            const yScale = resizedHeight / request.height;
+            const proposals = [];
+            let packedBytes = 0;
+            const points = mobileSamProposalPoints(request.seedRegion, request.sampling);
+            for (let pointIndex = 0; pointIndex < points.length; pointIndex++) {
+                const point = points[pointIndex];
+                if (!point)
+                    throw new Error('Missing MobileSAM proposal point');
+                let coordinates;
+                let labels;
+                let maskInput;
+                let hasMaskInput;
+                let originalSize;
+                let decoded;
+                try {
+                    coordinates = new ort.Tensor('float32', Float32Array.of(point.x * xScale, point.y * yScale, point.x * xScale, point.y * yScale), [1, 2, 2]);
+                    labels = new ort.Tensor('float32', Float32Array.of(1, -1), [1, 2]);
+                    maskInput = new ort.Tensor('float32', new Float32Array(256 * 256), [1, 1, 256, 256]);
+                    hasMaskInput = new ort.Tensor('float32', Float32Array.of(0), [1]);
+                    originalSize = new ort.Tensor('float32', Float32Array.of(request.height, request.width), [
+                        2,
+                    ]);
+                    decoded = await decoder.run({
+                        image_embeddings: embedding,
+                        point_coords: coordinates,
+                        point_labels: labels,
+                        mask_input: maskInput,
+                        has_mask_input: hasMaskInput,
+                        orig_im_size: originalSize,
+                    });
+                    const masks = decoded.masks;
+                    const scores = decoded.iou_predictions;
+                    if (!(masks instanceof ort.Tensor) ||
+                        masks.type !== 'float32' ||
+                        JSON.stringify(masks.dims) !== JSON.stringify([1, 4, request.height, request.width]) ||
+                        !(masks.data instanceof Float32Array) ||
+                        !(scores instanceof ort.Tensor) ||
+                        scores.type !== 'float32' ||
+                        JSON.stringify(scores.dims) !== JSON.stringify([1, 4]) ||
+                        !(scores.data instanceof Float32Array) ||
+                        !scores.data.every(Number.isFinite))
+                        throw new Error('Unexpected MobileSAM proposal decoder output');
+                    const candidates = packMobileSamProposalCandidates({
+                        width: request.width,
+                        height: request.height,
+                        rgba,
+                        roi: request.roi,
+                        pointIndex,
+                        logits: masks.data,
+                        scores: scores.data,
+                    });
+                    for (const candidate of candidates) {
+                        packedBytes += Buffer.from(candidate.mask, 'base64').byteLength;
+                        if (packedBytes > objectProposalLimits.packedBytes)
+                            throw new Error('Packed MobileSAM proposals exceed the aggregate byte limit');
+                        proposals.push(candidate);
+                    }
+                }
+                finally {
+                    coordinates?.dispose();
+                    labels?.dispose();
+                    maskInput?.dispose();
+                    hasMaskInput?.dispose();
+                    originalSize?.dispose();
+                    if (decoded)
+                        for (const tensor of Object.values(decoded))
+                            if (tensor instanceof ort.Tensor)
+                                tensor.dispose();
+                }
+            }
+            reply = {
+                id: request.id,
+                status: 'proposals',
+                width: request.width,
+                height: request.height,
+                proposals,
+            };
+        }
+        finally {
+            embeddingTensor?.dispose();
+            imageTensor.dispose();
+        }
+    }
+    finally {
+        const releases = await Promise.allSettled([
+            encoder.release(),
+            ...(decoder ? [decoder.release()] : []),
+        ]);
+        releaseFailed = releases.some((release) => release.status === 'rejected');
+    }
+    if (releaseFailed)
+        throw new Error('MobileSAM proposal session release failed');
+    if (!reply)
+        throw new Error('MobileSAM completed without proposals');
+    return reply;
+}

package/dist/infrastructure/segmentation/mobile-sam/worker-segment.d.ts

@@ -0,0 +1,4 @@
+import type { NativeSegmentationReply, NativeSegmentationRequest } from '../native-protocol.js';
+export declare function runMobileSamTask(request: Extract<NativeSegmentationRequest, {
+    kind: 'mobile-sam';
+}>): Promise<NativeSegmentationReply>;

package/dist/infrastructure/segmentation/mobile-sam/worker-segment.js

@@ -0,0 +1,126 @@
+import sharp from 'sharp';
+import { buildMobileSamSessionOptions, decoderIdentity, encoderIdentity, verifiedBytes, } from './worker-models.js';
+import { collectMobileSamSegmentationCandidates, mobileSamSegmentationPrompt, selectMobileSamSegmentationMask, } from './segmentation-masks.js';
+export async function runMobileSamTask(request) {
+    // Deliberately loaded inside the owned child. Core/entry imports never initialize ONNX Runtime.
+    const ort = await import('onnxruntime-node');
+    const providers = ort.listSupportedBackends();
+    if (!providers.some((item) => item.name === request.options.executionProvider && item.bundled))
+        throw new Error('Explicitly requested ONNX execution provider is unavailable');
+    const sessionOptions = buildMobileSamSessionOptions(request.options);
+    const encoder = await ort.InferenceSession.create(await verifiedBytes(request.options.encoderPath, encoderIdentity), sessionOptions);
+    let decoder;
+    let result;
+    let releaseFailed = false;
+    try {
+        decoder = await ort.InferenceSession.create(await verifiedBytes(request.options.decoderPath, decoderIdentity), sessionOptions);
+        const rgba = Buffer.from(request.rgba, 'base64');
+        if (rgba.length !== request.width * request.height * 4)
+            throw new Error('Invalid RGBA input size');
+        const scale = 1024 / Math.max(request.width, request.height);
+        const resizedWidth = Math.round(request.width * scale);
+        const resizedHeight = Math.round(request.height * scale);
+        if (resizedWidth < 1 || resizedHeight < 1)
+            throw new Error('Unrepresentable image aspect ratio');
+        const pipeline = sharp(rgba, {
+            raw: { width: request.width, height: request.height, channels: 4 },
+        });
+        let rgb;
+        try {
+            rgb = await pipeline
+                .removeAlpha()
+                .resize(resizedWidth, resizedHeight, { fit: 'fill', kernel: 'linear' })
+                .raw()
+                .toBuffer();
+        }
+        finally {
+            pipeline.destroy();
+        }
+        if (rgb.length !== resizedWidth * resizedHeight * 3)
+            throw new Error('Invalid resized RGB data');
+        const imageTensor = new ort.Tensor('float32', Float32Array.from(rgb), [
+            resizedHeight,
+            resizedWidth,
+            3,
+        ]);
+        try {
+            const encoded = await encoder.run({ input_image: imageTensor });
+            const embedding = encoded.image_embeddings;
+            if (!(embedding instanceof ort.Tensor) ||
+                embedding.type !== 'float32' ||
+                JSON.stringify(embedding.dims) !== JSON.stringify([1, 256, 64, 64]) ||
+                !(embedding.data instanceof Float32Array))
+                throw new Error('Unexpected MobileSAM encoder output');
+            const xScale = resizedWidth / request.width;
+            const yScale = resizedHeight / request.height;
+            const prompt = mobileSamSegmentationPrompt({
+                prompt: request.prompt,
+                seedRegion: request.seedRegion,
+                wholeRegion: request.wholeRegion,
+                xScale,
+                yScale,
+            });
+            const decoded = await decoder.run({
+                image_embeddings: embedding,
+                point_coords: new ort.Tensor('float32', prompt.coordinates, [1, 2, 2]),
+                point_labels: new ort.Tensor('float32', prompt.labels, [1, 2]),
+                mask_input: new ort.Tensor('float32', new Float32Array(256 * 256), [1, 1, 256, 256]),
+                has_mask_input: new ort.Tensor('float32', Float32Array.of(0), [1]),
+                orig_im_size: new ort.Tensor('float32', Float32Array.of(request.height, request.width), [
+                    2,
+                ]),
+            });
+            const masks = decoded.masks;
+            const scores = decoded.iou_predictions;
+            if (!(masks instanceof ort.Tensor) ||
+                masks.type !== 'float32' ||
+                JSON.stringify(masks.dims) !== JSON.stringify([1, 4, request.height, request.width]) ||
+                !(masks.data instanceof Float32Array) ||
+                !(scores instanceof ort.Tensor) ||
+                scores.type !== 'float32' ||
+                JSON.stringify(scores.dims) !== JSON.stringify([1, 4]) ||
+                !(scores.data instanceof Float32Array) ||
+                !scores.data.every(Number.isFinite))
+                throw new Error('Unexpected MobileSAM decoder output');
+            const candidates = collectMobileSamSegmentationCandidates({
+                width: request.width,
+                height: request.height,
+                rgba,
+                logits: masks.data,
+                scores: scores.data,
+                roi: request.roi,
+                seedRegion: request.seedRegion,
+                prompt: request.prompt,
+            });
+            const mask = selectMobileSamSegmentationMask({
+                candidates,
+                width: request.width,
+                height: request.height,
+                prompt: request.prompt,
+            });
+            result = {
+                id: request.id,
+                status: 'segmented',
+                width: request.width,
+                height: request.height,
+                mask: Buffer.from(mask).toString('base64'),
+                method: request.prompt === 'seed-region' ? 'mobile-sam-seed-point-v1' : 'mobile-sam-box-prompt',
+            };
+        }
+        finally {
+            imageTensor.dispose();
+        }
+    }
+    finally {
+        const releases = await Promise.allSettled([
+            encoder.release(),
+            ...(decoder ? [decoder.release()] : []),
+        ]);
+        releaseFailed = releases.some((release) => release.status === 'rejected');
+    }
+    if (releaseFailed)
+        throw new Error('MobileSAM native session release failed');
+    if (!result)
+        throw new Error('MobileSAM completed without a mask result');
+    return result;
+}