streamby-core 0.38.0 → 0.40.0

package/dist/services/workerJobs.js

@@ -0,0 +1,459 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.publicJob = exports.WORKER_CALLBACK_PATH = void 0;
+exports.enqueueWorkerJob = enqueueWorkerJob;
+exports.enqueueAutomaticJobs = enqueueAutomaticJobs;
+exports.retryWorkerJob = retryWorkerJob;
+exports.dispatchJob = dispatchJob;
+exports.startWorkerSweeper = startWorkerSweeper;
+exports.authorizeCallback = authorizeCallback;
+exports.reportProgress = reportProgress;
+exports.resultUploadUrl = resultUploadUrl;
+exports.completeJob = completeJob;
+exports.failWorkerJob = failWorkerJob;
+const crypto_1 = __importDefault(require("crypto"));
+const mongodb_1 = require("mongodb");
+const manager_1 = require("../models/manager");
+const storage_1 = require("../providers/storage");
+const nosqlDb_1 = require("../utils/nosqlDb");
+const assetCategories_1 = require("../utils/assetCategories");
+const connectionResolver_1 = require("./connectionResolver");
+const assetVersions_1 = require("./assetVersions");
+const jobQueue_1 = require("./jobQueue");
+const jobStore_1 = require("./jobStore");
+const workers_1 = require("./workers");
+// A worker's job, from start to end (the README has the contract a worker follows):
+//
+// 1. Asked for (a person, an agent, or a deliverable marked): it's kept as pending.
+// 2. Sent to its worker: a POST to its endpoint with what to process, a short-lived URL to
+//    download the asset, and where and how to call back (a token only this attempt has).
+// 3. The worker calls back: asks where to upload its result, reports progress, and ends it
+//    (complete, with a report; or fail, with its error).
+// 4. Completed: the result is kept as a new version of the asset, or as a new asset derived
+//    from it (the original is never touched), with the worker's report.
+//
+// A job that can't be sent, or that its worker doesn't finish in time, is sent again (up to
+// workerOptions.maxAttempts); then it fails, with its error, and can be retried by hand.
+exports.WORKER_CALLBACK_PATH = '/worker-jobs';
+const DEFAULT_MAX_ATTEMPTS = 3;
+const RETRY_BASE_MS = 30000;
+const DOWNLOAD_TTL_SECONDS = 3600;
+const UPLOAD_TTL_SECONDS = 3600;
+const filesCollection = () => (0, nosqlDb_1.nosqlDb)()?.collection('storage_files') ?? null;
+const hashOf = (token) => crypto_1.default.createHash('sha256').update(token).digest('hex');
+const extensionOf = (name) => (name.includes('.') ? `.${name.split('.').pop().toLowerCase()}` : '');
+/**
+ * As whom a job acts: whoever asked for it, with the platform role and name they had then, so
+ * a host's canUseBuiltin decides on sending it as it did on asking for it.
+ */
+const authOf = (job) => ({
+    userId: job.userId,
+    username: job.payload?.requester?.username ?? '',
+    role: job.payload?.requester?.role ?? 'viewer',
+});
+const findProject = (projectId) => (0, manager_1.getModel)('projects').findOne({ _id: projectId });
+// The storage an asset is in: its storage connection's, or the default one.
+async function fileAdapter(project, fileDoc, auth, config) {
+    const connId = fileDoc.storageConnectionId;
+    if (connId && (project.storageConnections ?? []).some((c) => c.id === connId)) {
+        const resolved = await (0, connectionResolver_1.resolveStorageAdapter)(project, connId, config, auth);
+        if ('error' in resolved)
+            throw new workers_1.WorkerError(resolved.error, resolved.status);
+        return resolved;
+    }
+    return config.adapter || (0, storage_1.createStorageProvider)(config.storageProviders);
+}
+/** Asks a worker to process an asset, as `auth`: the job, pending, to be sent right away. */
+async function enqueueWorkerJob(project, request, auth, config, callbackBase) {
+    const projectId = String(project._id ?? project.id);
+    if (typeof request.assetId !== 'string' || !request.assetId)
+        throw new workers_1.WorkerError('assetId is required');
+    const ref = request.worker;
+    if (!ref || typeof ref.id !== 'string' || typeof ref.source !== 'string') {
+        throw new workers_1.WorkerError("worker is required: { source: 'builtin' | 'project', id }");
+    }
+    const worker = await (0, workers_1.resolveWorker)(project, { source: ref.source, id: ref.id }, auth, config);
+    if (typeof request.kind !== 'string' || !worker.kinds.includes(request.kind)) {
+        throw new workers_1.WorkerError(`kind must be one of the worker's: ${worker.kinds.join(', ')}`);
+    }
+    const resultMode = (request.resultMode ?? 'derived');
+    if (resultMode !== 'derived' && resultMode !== 'version')
+        throw new workers_1.WorkerError("resultMode must be 'derived' or 'version'");
+    const profile = request.profile ?? {};
+    if (typeof profile !== 'object' || Array.isArray(profile) || profile === null)
+        throw new workers_1.WorkerError('profile must be an object');
+    const files = filesCollection();
+    if (!files)
+        throw new workers_1.WorkerError('Database not available', 500);
+    const fileDoc = await files.findOne({ fileId: request.assetId, projectId });
+    if (!fileDoc)
+        throw new workers_1.WorkerError('Asset not found', 404);
+    if (!worker.categories.includes(fileDoc.category)) {
+        throw new workers_1.WorkerError(`This worker takes ${worker.categories.join(', ')}, not ${fileDoc.category}`);
+    }
+    const versionId = typeof request.versionId === 'string' && request.versionId ? request.versionId : null;
+    if (versionId && !(await (0, assetVersions_1.findVersion)(projectId, request.assetId, versionId)))
+        throw new workers_1.WorkerError('Version not found', 404);
+    if (resultMode === 'version' && !(await fileAdapter(project, fileDoc, auth, config)).copyFile) {
+        throw new workers_1.WorkerError("This asset's storage can't keep versions: ask for a derived result", 400);
+    }
+    const requester = { role: auth.role, username: auth.username };
+    const job = await (0, jobQueue_1.createJob)('process', auth.userId, projectId, { callbackBase, requester }, {
+        assetId: request.assetId,
+        versionId,
+        worker: worker.ref,
+        kind: request.kind,
+        profile: profile,
+        resultMode,
+        attempts: 0,
+        maxAttempts: config.workerOptions?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
+        nextAttemptAt: new Date(),
+    });
+    kick(job.jobId, config);
+    return job;
+}
+/** Runs the workers set to process, by themselves, an asset of their categories marked as a deliverable. */
+async function enqueueAutomaticJobs(project, assetId, versionId, auth, config, callbackBase) {
+    const projectId = String(project._id ?? project.id);
+    const fileDoc = await filesCollection()?.findOne({ fileId: assetId, projectId });
+    if (!fileDoc)
+        return [];
+    const jobs = [];
+    for (const worker of await (0, workers_1.availableWorkers)(project, auth, config)) {
+        if (!worker.autoKind || !worker.categories.includes(fileDoc.category))
+            continue;
+        try {
+            jobs.push(await enqueueWorkerJob(project, { assetId, versionId, worker: { source: worker.source, id: worker.id }, kind: worker.autoKind }, auth, config, callbackBase));
+        }
+        catch (err) {
+            console.error(`❌ StreamBy workers: couldn't start ${worker.name} for asset ${assetId}:`, err instanceof Error ? err.message : err);
+        }
+    }
+    return jobs;
+}
+/** A failed worker job, back in line. */
+async function retryWorkerJob(jobId, config) {
+    const job = await (0, jobStore_1.jobStore)().claim(jobId, ['failed'], {
+        status: 'pending', stage: 'queued', progress: 0, error: null, attempts: 0,
+        nextAttemptAt: new Date(), leaseUntil: null, callbackHash: null, upload: null, updatedAt: new Date(),
+    });
+    if (job) {
+        (0, jobQueue_1.notify)(job);
+        kick(job.jobId, config);
+    }
+    return job;
+}
+// ── Sending a job to its worker ──
+const kick = (jobId, config) => {
+    setImmediate(() => { dispatchJob(jobId, config).catch(err => console.error('❌ StreamBy workers: dispatch failed:', err?.message ?? err)); });
+};
+const retryDelay = (attempts) => RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1);
+/** Ends an attempt that went wrong: sent again later while attempts are left, or failed. */
+async function attemptFailed(job, error) {
+    const attempts = job.attempts ?? 1;
+    const left = attempts < (job.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
+    const updated = await (0, jobStore_1.jobStore)().update(job.jobId, left
+        ? { status: 'pending', stage: 'retrying', error, nextAttemptAt: new Date(Date.now() + retryDelay(attempts)), leaseUntil: null, callbackHash: null, upload: null, updatedAt: new Date() }
+        : { status: 'failed', stage: 'failed', error, leaseUntil: null, callbackHash: null, upload: null, updatedAt: new Date() });
+    if (updated)
+        (0, jobQueue_1.notify)(updated);
+}
+/**
+ * Sends a pending job (or one its worker didn't finish in time) to its worker. Of several
+ * instances, only the one that claims it does.
+ */
+async function dispatchJob(jobId, config) {
+    const now = new Date();
+    const current = await (0, jobStore_1.jobStore)().get(jobId);
+    if (!current?.worker)
+        return;
+    // Its worker had it and the time's up: sent again, or failed.
+    if (current.status === 'processing') {
+        if (!current.leaseUntil || new Date(current.leaseUntil) > now)
+            return;
+        const timedOut = await (0, jobStore_1.jobStore)().claim(jobId, ['processing'], { leaseUntil: null, updatedAt: now }, { leaseEndedBy: now });
+        if (timedOut)
+            await attemptFailed(timedOut, "The worker didn't finish in time");
+        return;
+    }
+    if (current.status !== 'pending' || (current.nextAttemptAt && new Date(current.nextAttemptAt) > now))
+        return;
+    const project = await findProject(current.projectId);
+    let worker;
+    try {
+        if (!project)
+            throw new workers_1.WorkerError('Project not found', 404);
+        worker = await (0, workers_1.resolveWorker)(project, current.worker, authOf(current), config);
+    }
+    catch (err) {
+        const failed = await (0, jobStore_1.jobStore)().claim(jobId, ['pending'], {
+            status: 'failed', stage: 'failed', error: err instanceof Error ? err.message : String(err), updatedAt: now,
+        });
+        if (failed)
+            (0, jobQueue_1.notify)(failed);
+        return;
+    }
+    const token = crypto_1.default.randomBytes(32).toString('base64url');
+    const job = await (0, jobStore_1.jobStore)().claim(jobId, ['pending'], {
+        status: 'processing', stage: 'sent', attempts: (current.attempts ?? 0) + 1, callbackHash: hashOf(token),
+        leaseUntil: new Date(now.getTime() + worker.timeoutSeconds * 1000), nextAttemptAt: null, updatedAt: now,
+    });
+    if (!job)
+        return; // another instance took it
+    (0, jobQueue_1.notify)(job);
+    try {
+        const files = filesCollection();
+        const fileDoc = files && await files.findOne({ fileId: job.assetId, projectId: job.projectId });
+        if (!fileDoc)
+            throw new workers_1.WorkerError('Asset not found', 404);
+        const version = job.versionId ? await (0, assetVersions_1.findVersion)(job.projectId, job.assetId, job.versionId) : null;
+        if (job.versionId && !version)
+            throw new workers_1.WorkerError('Version not found', 404);
+        const adapter = await fileAdapter(project, fileDoc, authOf(job), config);
+        if (!adapter.getPresignedGetUrl)
+            throw new workers_1.WorkerError("This asset's storage can't give download URLs");
+        const fileName = version?.fileName ?? fileDoc.displayName;
+        const download = await adapter.getPresignedGetUrl(version?.storageKey ?? fileDoc.storageKey, DOWNLOAD_TTL_SECONDS, fileName);
+        const callbackBase = config.workerOptions?.callbackBaseUrl ?? job.payload?.callbackBase;
+        // A project's worker: its name may resolve, now, to the server's own network.
+        if (worker.ref.source === 'project')
+            await (0, workers_1.assertPublicEndpoint)(worker.endpoint, config);
+        const response = await fetch(worker.endpoint, {
+            method: 'POST',
+            redirect: 'error',
+            headers: { 'Content-Type': 'application/json', ...(worker.credential ? { Authorization: `Bearer ${worker.credential}` } : {}) },
+            body: JSON.stringify({
+                jobId: job.jobId,
+                kind: job.kind,
+                profile: job.profile ?? {},
+                attempt: job.attempts,
+                asset: {
+                    id: job.assetId, versionId: version?.versionId ?? null, fileName,
+                    contentType: version?.contentType ?? fileDoc.contentType ?? null, category: fileDoc.category,
+                },
+                download: { url: download, expiresAt: new Date(now.getTime() + DOWNLOAD_TTL_SECONDS * 1000).toISOString() },
+                callback: { url: `${String(callbackBase).replace(/\/$/, '')}${exports.WORKER_CALLBACK_PATH}/${job.jobId}`, token },
+                deadline: job.leaseUntil,
+            }),
+            signal: AbortSignal.timeout(30000),
+        });
+        if (!response.ok) {
+            // Its answer stays in the server's log: an endpoint's status isn't the user's to see.
+            console.error(`❌ StreamBy workers: ${worker.ref.name} answered ${response.status} to job ${job.jobId}`);
+            throw new Error('The worker refused the job');
+        }
+        const accepted = await (0, jobStore_1.jobStore)().claim(jobId, ['processing'], { stage: 'accepted', updatedAt: new Date() }, { callbackHash: hashOf(token) });
+        if (accepted)
+            (0, jobQueue_1.notify)(accepted);
+    }
+    catch (err) {
+        const message = err instanceof Error ? (err.name === 'TimeoutError' ? "The worker didn't answer" : err.message) : String(err);
+        // Only this attempt's own: the worker may have called back already.
+        const still = await (0, jobStore_1.jobStore)().get(jobId);
+        if (still?.status !== 'processing' || still.callbackHash !== hashOf(token))
+            return;
+        if (err instanceof workers_1.PrivateEndpointError) {
+            // Not sent, and never will be: no attempt is left for an endpoint at a private address.
+            const failed = await (0, jobStore_1.jobStore)().update(jobId, {
+                status: 'failed', stage: 'failed', error: message, leaseUntil: null, callbackHash: null, upload: null, updatedAt: new Date(),
+            });
+            if (failed)
+                (0, jobQueue_1.notify)(failed);
+            return;
+        }
+        await attemptFailed(still, message);
+    }
+}
+/** Looks, now and then, for jobs to send (again). Every instance does; claiming keeps them from doubling. */
+function startWorkerSweeper(config) {
+    const interval = config.workerOptions?.sweepIntervalMs ?? 30000;
+    if (!interval)
+        return () => { };
+    const timer = setInterval(async () => {
+        try {
+            for (const job of await (0, jobStore_1.jobStore)().due(new Date(), 20))
+                await dispatchJob(job.jobId, config);
+        }
+        catch (err) {
+            console.error('❌ StreamBy workers: sweep failed:', err instanceof Error ? err.message : err);
+        }
+    }, interval);
+    timer.unref?.();
+    return () => clearInterval(timer);
+}
+// ── The worker's callbacks ──
+/** The job a callback is about, if its token is this attempt's and the job is in progress. */
+async function authorizeCallback(jobId, authorization) {
+    const token = authorization?.match(/^Bearer\s+(.+)$/i)?.[1]?.trim();
+    if (!token)
+        throw new workers_1.WorkerError('A callback needs its job token (Authorization: Bearer ...)', 401);
+    const job = await (0, jobQueue_1.getJob)(jobId);
+    const expected = job?.callbackHash ? Buffer.from(job.callbackHash, 'hex') : null;
+    const given = Buffer.from(hashOf(token), 'hex');
+    if (!job || !expected || expected.length !== given.length || !crypto_1.default.timingSafeEqual(expected, given)) {
+        throw new workers_1.WorkerError('Invalid job token', 401);
+    }
+    if (job.status !== 'processing')
+        throw new workers_1.WorkerError(`The job is ${job.status}`, 409);
+    return job;
+}
+async function reportProgress(job, body) {
+    const patch = { updatedAt: new Date() };
+    if (body?.progress !== undefined) {
+        const progress = Number(body.progress);
+        if (!Number.isFinite(progress) || progress < 0 || progress > 100)
+            throw new workers_1.WorkerError('progress must be from 0 to 100');
+        patch.progress = Math.round(progress);
+    }
+    if (body?.stage !== undefined) {
+        if (typeof body.stage !== 'string' || !body.stage.trim() || body.stage.length > 60)
+            throw new workers_1.WorkerError('stage must be a short text');
+        patch.stage = body.stage.trim();
+    }
+    const updated = await (0, jobStore_1.jobStore)().claim(job.jobId, ['processing'], patch);
+    if (updated)
+        (0, jobQueue_1.notify)(updated);
+    return updated;
+}
+/** Where the worker uploads its result: a key of its own until the job completes. */
+async function resultUploadUrl(job, body, config) {
+    const fileName = typeof body?.fileName === 'string' ? body.fileName.trim() : '';
+    const contentType = typeof body?.contentType === 'string' ? body.contentType.trim() : '';
+    if (!fileName || !contentType || fileName.includes('/'))
+        throw new workers_1.WorkerError('fileName and contentType are required');
+    const project = await findProject(job.projectId);
+    const fileDoc = await filesCollection()?.findOne({ fileId: job.assetId, projectId: job.projectId });
+    if (!project || !fileDoc)
+        throw new workers_1.WorkerError('Asset not found', 404);
+    if (!(0, assetCategories_1.validateContentType)(contentType, fileName, fileDoc.category)) {
+        throw new workers_1.WorkerError(`The result must be of the asset's category, ${fileDoc.category}`);
+    }
+    const adapter = await fileAdapter(project, fileDoc, authOf(job), config);
+    if (!adapter.getPresignedUploadUrl)
+        throw new workers_1.WorkerError("This asset's storage can't give upload URLs");
+    const storageKey = job.resultMode === 'version'
+        ? `${String(fileDoc.storageKey).replace(/\.[^./]*$/, '')}.job-${job.jobId}${extensionOf(fileName)}`
+        : `${job.projectId}/${fileDoc.category}/${crypto_1.default.randomUUID()}${extensionOf(fileName)}`;
+    const url = await adapter.getPresignedUploadUrl(storageKey, contentType, UPLOAD_TTL_SECONDS);
+    const updated = await (0, jobStore_1.jobStore)().claim(job.jobId, ['processing'], {
+        upload: { storageKey, fileName, contentType }, stage: 'uploading', updatedAt: new Date(),
+    });
+    if (!updated)
+        throw new workers_1.WorkerError('The job is no longer in progress', 409);
+    (0, jobQueue_1.notify)(updated);
+    return {
+        url, method: 'PUT', headers: { 'Content-Type': contentType },
+        expiresAt: new Date(Date.now() + UPLOAD_TTL_SECONDS * 1000).toISOString(),
+    };
+}
+/** The worker's result, kept: a new version of the asset, or a new asset derived from it. */
+async function completeJob(job, body, config) {
+    if (!job.upload)
+        throw new workers_1.WorkerError('Upload the result first (upload-url)', 409);
+    const report = body?.report ?? null;
+    if (report !== null && (typeof report !== 'object' || Array.isArray(report)))
+        throw new workers_1.WorkerError('report must be an object');
+    if (JSON.stringify(report ?? {}).length > 64000)
+        throw new workers_1.WorkerError('report is too large');
+    // Claimed first, its token spent: a second callback (or a timeout) can't complete it twice.
+    const claimed = await (0, jobStore_1.jobStore)().claim(job.jobId, ['processing'], { stage: 'saving', leaseUntil: null, callbackHash: null, updatedAt: new Date() }, { callbackHash: job.callbackHash ?? '' });
+    if (!claimed)
+        throw new workers_1.WorkerError('The job is no longer in progress', 409);
+    try {
+        const result = await keepResult(job, report, config);
+        const done = await (0, jobStore_1.jobStore)().update(job.jobId, {
+            status: 'completed', stage: 'complete', progress: 100, error: null, report, result, upload: null, updatedAt: new Date(),
+        });
+        if (done)
+            (0, jobQueue_1.notify)(done);
+        return done;
+    }
+    catch (err) {
+        const failed = await (0, jobStore_1.jobStore)().update(job.jobId, {
+            status: 'failed', stage: 'failed', error: `Its result couldn't be kept: ${err instanceof Error ? err.message : err}`, updatedAt: new Date(),
+        });
+        if (failed)
+            (0, jobQueue_1.notify)(failed);
+        throw err;
+    }
+}
+async function keepResult(job, report, config) {
+    const project = await findProject(job.projectId);
+    const files = filesCollection();
+    const fileDoc = files && await files.findOne({ fileId: job.assetId, projectId: job.projectId });
+    if (!project || !files || !fileDoc)
+        throw new workers_1.WorkerError('Asset not found', 404);
+    const adapter = await fileAdapter(project, fileDoc, authOf(job), config);
+    const { storageKey, fileName, contentType } = job.upload;
+    const meta = { report, jobId: job.jobId, kind: job.kind, worker: job.worker };
+    let result;
+    if (job.resultMode === 'version') {
+        if (!adapter.copyFile)
+            throw new workers_1.WorkerError("This asset's storage can't keep versions", 400);
+        // The asset keeps its key (and URL): its current version is archived, the result takes its place.
+        const version = await (0, assetVersions_1.archiveCurrentVersion)(fileDoc, { fileName, contentType, userId: job.userId }, adapter.copyFile.bind(adapter));
+        await adapter.copyFile(storageKey, fileDoc.storageKey);
+        if (adapter.deleteFile)
+            await adapter.deleteFile(storageKey).catch(() => { });
+        await files.updateOne({ fileId: job.assetId, projectId: job.projectId }, { $set: { contentType, updatedAt: new Date() } });
+        await (0, assetVersions_1.versionsCollection)()?.updateOne({ versionId: version.versionId }, { $set: meta });
+        result = { assetId: job.assetId, versionId: version.versionId, mode: 'version' };
+    }
+    else {
+        // A new asset, linked to the one (and the version) it came from.
+        const fileId = crypto_1.default.randomUUID();
+        const derived = {
+            _id: new mongodb_1.ObjectId(), fileId, projectId: job.projectId,
+            ...(fileDoc.storageConnectionId ? { storageConnectionId: fileDoc.storageConnectionId } : {}),
+            category: fileDoc.category, storageKey, displayName: fileName, contentType,
+            uploadedBy: job.userId, createdAt: new Date(), folderId: fileDoc.folderId ?? null,
+            derivedFrom: { assetId: job.assetId, versionId: job.versionId ?? null, jobId: job.jobId, kind: job.kind },
+        };
+        await files.insertOne(derived);
+        await (0, assetVersions_1.recordFirstVersion)(derived);
+        const [first] = (await (0, assetVersions_1.versionsCollection)()?.find({ assetId: fileId, projectId: job.projectId }).limit(1).toArray()) ?? [];
+        if (first)
+            await (0, assetVersions_1.versionsCollection)()?.updateOne({ versionId: first.versionId }, { $set: meta });
+        result = { assetId: fileId, versionId: first?.versionId ?? '', mode: 'derived' };
+    }
+    return result;
+}
+/** The worker couldn't do it: sent again if it says it may work later (and attempts are left), or failed. */
+async function failWorkerJob(job, body, config) {
+    const error = typeof body?.error === 'string' && body.error.trim() ? body.error.trim().slice(0, 2000) : 'The worker failed';
+    const claimed = await (0, jobStore_1.jobStore)().claim(job.jobId, ['processing'], { leaseUntil: null, callbackHash: null, updatedAt: new Date() }, { callbackHash: job.callbackHash ?? '' });
+    if (!claimed)
+        throw new workers_1.WorkerError('The job is no longer in progress', 409);
+    await discardUpload(claimed, config);
+    if (body?.retryable === true)
+        await attemptFailed(claimed, error);
+    else {
+        const failed = await (0, jobStore_1.jobStore)().update(job.jobId, { status: 'failed', stage: 'failed', error, callbackHash: null, upload: null, updatedAt: new Date() });
+        if (failed)
+            (0, jobQueue_1.notify)(failed);
+    }
+    return (0, jobStore_1.jobStore)().get(job.jobId);
+}
+// A failed attempt's upload, if any, isn't kept.
+async function discardUpload(job, config) {
+    if (!job.upload)
+        return;
+    try {
+        const project = await findProject(job.projectId);
+        const fileDoc = await filesCollection()?.findOne({ fileId: job.assetId, projectId: job.projectId });
+        if (!project || !fileDoc)
+            return;
+        const adapter = await fileAdapter(project, fileDoc, authOf(job), config);
+        await adapter.deleteFile?.(job.upload.storageKey);
+    }
+    catch { /* a leftover upload only */ }
+}
+/** What's shown of a job: never its callback token's hash, nor its upload's key. */
+const publicJob = ({ callbackHash: _hash, upload: _upload, payload, ...job }) => ({
+    ...job,
+    payload: job.jobType === 'process' ? {} : payload,
+});
+exports.publicJob = publicJob;

package/dist/services/workers.d.ts

@@ -0,0 +1,78 @@
+import { Auth, StorageCategoryName, StreamByConfig, WorkerRef } from '../types';
+export declare const STORAGE_CATEGORIES: StorageCategoryName[];
+export declare const WORKER_PROVIDERS: readonly ["streamby-worker", "flamenco", "deadline", "rebusfarm", "sheepit", "custom"];
+export declare const DEFAULT_WORKER_TIMEOUT_SECONDS = 3600;
+/** A project's worker, as stored. */
+export interface WorkerConnection {
+    id: string;
+    projectId: string;
+    name: string;
+    provider: string;
+    apiUrl: string;
+    encryptedCredential?: string;
+    kinds?: string[];
+    categories?: StorageCategoryName[];
+    autoKind?: string | null;
+    timeoutSeconds?: number;
+    description?: string;
+    createdBy?: string;
+    createdAt: Date;
+}
+/** What a project's member sees of a worker: never its credential. */
+export interface WorkerSummary {
+    source: WorkerRef['source'];
+    id: string;
+    name: string;
+    provider: string;
+    kinds: string[];
+    categories: StorageCategoryName[];
+    autoKind: string | null;
+    description: string | null;
+    hasCredential: boolean;
+    createdBy: string | null;
+}
+/** A worker ready to be sent a job. */
+export interface ResolvedWorker {
+    ref: WorkerRef;
+    endpoint: string;
+    credential: string | null;
+    kinds: string[];
+    categories: StorageCategoryName[];
+    autoKind: string | null;
+    timeoutSeconds: number;
+}
+export declare class WorkerError extends Error {
+    status: number;
+    constructor(message: string, status?: number);
+}
+/** A worker endpoint at a local or private address: never sent to, whatever the attempt. */
+export declare class PrivateEndpointError extends WorkerError {
+}
+export declare const projectWorkers: (project: any) => WorkerConnection[];
+/** The workers a project's member can send jobs to: the project's own, and the host's they may use. */
+export declare function availableWorkers(project: any, auth: Auth, config: StreamByConfig): Promise<WorkerSummary[]>;
+/** A worker of the project, or of the host, ready to be sent a job as `auth`. */
+export declare function resolveWorker(project: any, ref: {
+    source: string;
+    id: string;
+}, auth: Auth, config: StreamByConfig): Promise<ResolvedWorker>;
+export declare function checkWorkerEndpoint(value: unknown, config: StreamByConfig): string;
+/**
+ * Throws if a project's worker endpoint resolves, now, to a local or private address (a
+ * public-looking name pointing at the server's own network).
+ */
+export declare function assertPublicEndpoint(endpoint: string, config: StreamByConfig): Promise<void>;
+export interface WorkerInput {
+    name?: unknown;
+    provider?: unknown;
+    endpoint?: unknown;
+    credential?: unknown;
+    kinds?: unknown;
+    categories?: unknown;
+    autoKind?: unknown;
+    timeoutSeconds?: unknown;
+    description?: unknown;
+}
+/** A project's new worker, checked and with its credential encrypted. */
+export declare function newWorkerConnection(input: WorkerInput, projectId: string, userId: string, config: StreamByConfig): WorkerConnection;
+export declare const summarizeProjectWorker: (worker: WorkerConnection) => WorkerSummary;