zorya-classifier 0.1.0 → 0.2.0

package/dist/infrastructure/segmentation/native-child.d.ts

@@ -0,0 +1,23 @@
+interface NativeChildPoolOptions<T> {
+    workerUrl: URL;
+    maxActive?: number;
+    maxWaiting?: number;
+    jobTimeoutMs?: number;
+    abortGraceMs?: number;
+    encode(job: {
+        id: string;
+        payload: T;
+    }): Uint8Array;
+    decode(value: unknown, id: string): unknown;
+}
+/** Owns every spawned child until its exit event confirms physical settlement. */
+export declare class NativeChildPool<T> {
+    #private;
+    constructor(options: NativeChildPoolOptions<T>);
+    run(payload: T, signal: AbortSignal): Promise<unknown>;
+    /** Admission is reserved before buildPayload can snapshot or serialize caller-owned data. */
+    runLazy(signal: AbortSignal, buildPayload: () => T): Promise<unknown>;
+    settle(): Promise<void>;
+    dispose(): Promise<void>;
+}
+export {};

package/dist/infrastructure/segmentation/native-child.js

@@ -0,0 +1,297 @@
+import { fork } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { ClassifierError } from '../../entities/errors.js';
+import { fileURLToPath } from 'node:url';
+import { channel } from 'node:diagnostics_channel';
+const resourceChannel = channel('zorya.segmentation.child-resource');
+/** Owns every spawned child until its exit event confirms physical settlement. */
+export class NativeChildPool {
+    #workerUrl;
+    #maxActive;
+    #maxWaiting;
+    #jobTimeoutMs;
+    #abortGraceMs;
+    #encode;
+    #decode;
+    #queue = [];
+    #physical = new Set();
+    #lifetime = new AbortController();
+    #active = 0;
+    #closing;
+    #poisoned;
+    constructor(options) {
+        this.#workerUrl = options.workerUrl;
+        this.#maxActive = options.maxActive ?? 1;
+        this.#maxWaiting = options.maxWaiting ?? 2;
+        this.#jobTimeoutMs = options.jobTimeoutMs ?? 10_000;
+        this.#abortGraceMs = options.abortGraceMs ?? 250;
+        this.#encode = options.encode;
+        this.#decode = options.decode;
+    }
+    run(payload, signal) {
+        return this.runLazy(signal, () => payload);
+    }
+    /** Admission is reserved before buildPayload can snapshot or serialize caller-owned data. */
+    runLazy(signal, buildPayload) {
+        if (signal.aborted)
+            return Promise.reject(cancelledError(signal.reason));
+        if (this.#closing)
+            return Promise.reject(new ClassifierError('DISPOSED', 'Native child pool is disposed'));
+        if (this.#poisoned)
+            return Promise.reject(nativeFailure(this.#poisoned));
+        if (this.#queue.length >= this.#maxWaiting && this.#active >= this.#maxActive)
+            return Promise.reject(new ClassifierError('CAPACITY', 'Native child queue is full'));
+        let resolve;
+        let reject;
+        const promise = new Promise((yes, no) => {
+            resolve = yes;
+            reject = no;
+        });
+        const job = {
+            id: randomUUID(),
+            payload: undefined,
+            buildPayload,
+            payloadReady: false,
+            signal: AbortSignal.any([signal, this.#lifetime.signal]),
+            resolve,
+            reject,
+            promise,
+            started: false,
+            done: false,
+        };
+        job.abortListener = () => this.#abort(job, cancelledError(job.signal.reason));
+        job.signal.addEventListener('abort', job.abortListener, { once: true });
+        this.#queue.push(job);
+        try {
+            job.payload = job.buildPayload();
+            job.payloadReady = true;
+        }
+        catch (error) {
+            this.#finishQueued(job, error instanceof ClassifierError
+                ? error
+                : new ClassifierError('INVALID_INPUT', 'Unable to snapshot native task input'));
+            this.#pump();
+            return promise;
+        }
+        if (job.signal.aborted)
+            job.abortListener();
+        else
+            this.#pump();
+        return promise;
+    }
+    #pump() {
+        while (!this.#closing && !this.#poisoned && this.#active < this.#maxActive) {
+            const job = this.#queue.shift();
+            if (!job)
+                return;
+            if (job.done || job.signal.aborted) {
+                this.#finishQueued(job, job.signal.reason ?? new Error('Native task cancelled'));
+                continue;
+            }
+            this.#start(job);
+        }
+    }
+    #start(job) {
+        if (!job.payloadReady) {
+            this.#finishQueued(job, new ClassifierError('INVALID_INPUT', 'Native task payload was not prepared'));
+            return;
+        }
+        job.started = true;
+        this.#active++;
+        this.#physical.add(job);
+        const workerPath = fileURLToPath(this.#workerUrl);
+        let child;
+        try {
+            child = fork(workerPath, [], {
+                execArgv: workerPath.endsWith('.ts') ? ['--import', 'tsx'] : [],
+                stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
+            });
+        }
+        catch (error) {
+            job.failure = nativeFailure(error);
+            this.#poison(job.failure);
+            this.#endPhysical(job);
+            return;
+        }
+        job.child = child;
+        if (child.pid !== undefined)
+            resourceChannel.publish({ type: 'start', jobId: job.id, pid: child.pid });
+        let output = '';
+        let outputBytes = 0;
+        child.stdout?.setEncoding('utf8');
+        child.stdout?.on('data', (chunk) => {
+            outputBytes += Buffer.byteLength(chunk);
+            if (outputBytes > 24 * 1024 * 1024) {
+                job.failure = new ClassifierError('RESOURCE_LIMIT', 'Native child output exceeded its limit');
+                child.kill();
+                return;
+            }
+            output += chunk;
+        });
+        child.stderr?.resume();
+        child.stdin?.on('error', (error) => {
+            job.failure ??= nativeFailure(error);
+        });
+        let telemetryConfigured = false;
+        child.on('message', (message) => {
+            if (typeof message !== 'object' || message === null || !('type' in message)) {
+                job.failure ??= new ClassifierError('INVALID_RESPONSE', 'Native child sent an invalid control message');
+                child.kill();
+                return;
+            }
+            if (message.type === 'ready' && !telemetryConfigured) {
+                telemetryConfigured = true;
+                child.send({ type: 'telemetry', enabled: resourceChannel.hasSubscribers });
+                return;
+            }
+            if (message.type === 'memory' && 'memory' in message) {
+                const memory = message.memory;
+                if (typeof memory === 'object' && memory !== null && isMemoryUsage(memory))
+                    resourceChannel.publish({ type: 'memory', jobId: job.id, pid: child.pid, ...memory });
+                return;
+            }
+            job.failure ??= new ClassifierError('INVALID_RESPONSE', 'Native child sent an invalid control message');
+            child.kill();
+        });
+        child.on('error', (error) => {
+            job.failure ??= nativeFailure(error);
+        });
+        child.on('close', (code, signal) => {
+            if (!job.failure && code !== 0)
+                job.failure = new ClassifierError('PROVIDER', `Native child exited with code ${String(code)} (${String(signal)})`);
+            if (!job.failure) {
+                try {
+                    const raw = JSON.parse(output);
+                    job.reply = this.#decode(raw, job.id);
+                }
+                catch (error) {
+                    job.failure =
+                        error instanceof ClassifierError
+                            ? error
+                            : new ClassifierError('INVALID_RESPONSE', 'Native child returned an invalid reply');
+                }
+            }
+            child.stdin?.removeAllListeners();
+            child.stdout?.removeAllListeners();
+            child.stderr?.removeAllListeners();
+            child.stdin?.destroy();
+            child.stdout?.destroy();
+            child.stderr?.destroy();
+            this.#endPhysical(job);
+        });
+        job.timer = setTimeout(() => this.#abort(job, new ClassifierError('TIMEOUT', 'Native segmentation exceeded its job limit')), this.#jobTimeoutMs);
+        try {
+            const bytes = this.#encode({ id: job.id, payload: job.payload });
+            if (bytes.byteLength > 96 * 1024 * 1024)
+                throw new ClassifierError('RESOURCE_LIMIT', 'Native child input exceeded its size limit');
+            child.stdin?.end(bytes);
+        }
+        catch (error) {
+            job.failure = nativeFailure(error);
+            child.kill();
+        }
+        if (job.signal.aborted)
+            this.#abort(job, cancelledError(job.signal.reason));
+    }
+    #abort(job, reason) {
+        if (job.done)
+            return;
+        if (!job.started) {
+            const queued = this.#queue.indexOf(job);
+            if (queued >= 0)
+                this.#queue.splice(queued, 1);
+            this.#finishQueued(job, reason);
+            return;
+        }
+        job.failure ??= reason;
+        if (job.abortTimer)
+            return;
+        job.abortTimer = setTimeout(() => {
+            if (!job.child || !this.#physical.has(job))
+                return;
+            job.child.kill('SIGKILL');
+            job.exitTimer = setTimeout(() => {
+                if (this.#physical.has(job))
+                    this.#poison(reason);
+            }, 2000);
+        }, this.#abortGraceMs);
+    }
+    #poison(reason) {
+        this.#poisoned ??= nativeFailure(reason);
+        while (this.#queue.length) {
+            const job = this.#queue.shift();
+            if (job)
+                this.#finishQueued(job, this.#poisoned);
+        }
+    }
+    #finishQueued(job, error) {
+        if (job.done)
+            return;
+        job.done = true;
+        const queued = this.#queue.indexOf(job);
+        if (queued >= 0)
+            this.#queue.splice(queued, 1);
+        job.signal.removeEventListener('abort', job.abortListener);
+        job.reject(error);
+    }
+    #endPhysical(job) {
+        if (!this.#physical.delete(job))
+            return;
+        if (job.child?.pid !== undefined)
+            resourceChannel.publish({ type: 'close', jobId: job.id, pid: job.child.pid });
+        this.#active--;
+        clearTimeout(job.timer);
+        clearTimeout(job.abortTimer);
+        clearTimeout(job.exitTimer);
+        job.signal.removeEventListener('abort', job.abortListener);
+        if (!job.done) {
+            job.done = true;
+            if (job.failure !== undefined)
+                job.reject(job.failure);
+            else
+                job.resolve(job.reply);
+        }
+        this.#pump();
+    }
+    async settle() {
+        while (this.#queue.length || this.#physical.size) {
+            await Promise.allSettled([...[...this.#physical].map((job) => job.promise)]);
+            if (this.#queue.length && this.#active === 0)
+                this.#pump();
+            if (this.#physical.size)
+                await new Promise((resolve) => setTimeout(resolve, 5));
+        }
+    }
+    dispose() {
+        if (!this.#closing) {
+            this.#closing = this.#dispose();
+            this.#lifetime.abort(new Error('Native child pool disposed'));
+        }
+        return this.#closing;
+    }
+    async #dispose() {
+        this.#lifetime.abort(new Error('Native child pool disposed'));
+        for (const job of [...this.#queue])
+            this.#abort(job, this.#lifetime.signal.reason);
+        for (const job of this.#physical)
+            this.#abort(job, this.#lifetime.signal.reason);
+        await this.settle();
+    }
+}
+function cancelledError(reason) {
+    if (reason instanceof ClassifierError)
+        return reason;
+    const message = reason instanceof Error && reason.message ? reason.message : 'Native task cancelled';
+    return new ClassifierError('ABORTED', message);
+}
+function nativeFailure(error) {
+    if (error instanceof ClassifierError)
+        return error;
+    const code = typeof error === 'object' && error !== null && 'code' in error ? error.code : undefined;
+    if (code === 'ENOENT')
+        return new ClassifierError('UNSUPPORTED', 'Native worker executable is unavailable');
+    return new ClassifierError('PROVIDER', 'Native child process failed');
+}
+function isMemoryUsage(value) {
+    return ['rss', 'heapTotal', 'heapUsed', 'external', 'arrayBuffers'].every((key) => key in value && typeof Reflect.get(value, key) === 'number');
+}