streamby-core 0.38.0 → 0.40.0

package/dist/services/workers.js

@@ -0,0 +1,179 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.summarizeProjectWorker = exports.projectWorkers = exports.PrivateEndpointError = exports.WorkerError = exports.DEFAULT_WORKER_TIMEOUT_SECONDS = exports.WORKER_PROVIDERS = exports.STORAGE_CATEGORIES = void 0;
+exports.availableWorkers = availableWorkers;
+exports.resolveWorker = resolveWorker;
+exports.checkWorkerEndpoint = checkWorkerEndpoint;
+exports.assertPublicEndpoint = assertPublicEndpoint;
+exports.newWorkerConnection = newWorkerConnection;
+const crypto_1 = __importDefault(require("crypto"));
+const auth_1 = require("../utils/auth");
+const builtinAccess_1 = require("../utils/builtinAccess");
+const encryption_1 = require("../utils/encryption");
+const privateAddress_1 = require("../utils/privateAddress");
+// Processing workers: external services StreamBy sends jobs to (see workerJobs). A project
+// can use the host's built-in ones (StreamByConfig.workers, gated by canUseBuiltin) and its
+// own, connected from the project. A project's workers are kept in its
+// `renderFarmConnections` list (render farms are workers too); each one's credential is
+// encrypted, and it's used only while whoever connected it is still in the project.
+exports.STORAGE_CATEGORIES = ['images', 'audios', 'videos', '3d-models'];
+exports.WORKER_PROVIDERS = ['streamby-worker', 'flamenco', 'deadline', 'rebusfarm', 'sheepit', 'custom'];
+exports.DEFAULT_WORKER_TIMEOUT_SECONDS = 3600;
+class WorkerError extends Error {
+    constructor(message, status = 400) {
+        super(message);
+        this.status = status;
+    }
+}
+exports.WorkerError = WorkerError;
+/** A worker endpoint at a local or private address: never sent to, whatever the attempt. */
+class PrivateEndpointError extends WorkerError {
+}
+exports.PrivateEndpointError = PrivateEndpointError;
+const listOf = (value) => (Array.isArray(value) ? value.filter((v) => typeof v === 'string') : []);
+const projectWorkers = (project) => Array.isArray(project?.renderFarmConnections) ? project.renderFarmConnections : [];
+exports.projectWorkers = projectWorkers;
+const builtinSummary = (worker) => ({
+    source: 'builtin', id: worker.id, name: worker.name, provider: 'streamby-worker',
+    kinds: worker.kinds ?? [], categories: worker.categories ?? [], autoKind: worker.autoKind ?? null,
+    description: null, hasCredential: !!worker.credential, createdBy: null,
+});
+const projectSummary = (worker) => ({
+    source: 'project', id: worker.id, name: worker.name, provider: worker.provider,
+    kinds: listOf(worker.kinds), categories: listOf(worker.categories),
+    autoKind: worker.autoKind ?? null, description: worker.description ?? null,
+    hasCredential: !!worker.encryptedCredential, createdBy: worker.createdBy ?? null,
+});
+/** The workers a project's member can send jobs to: the project's own, and the host's they may use. */
+async function availableWorkers(project, auth, config) {
+    const builtins = [];
+    for (const worker of config.workers ?? []) {
+        if (await (0, builtinAccess_1.assertBuiltinAccess)(auth, worker.id, config, 'worker'))
+            builtins.push(builtinSummary(worker));
+    }
+    return [...builtins, ...(0, exports.projectWorkers)(project).map(projectSummary)];
+}
+/** A worker of the project, or of the host, ready to be sent a job as `auth`. */
+async function resolveWorker(project, ref, auth, config) {
+    if (ref.source === 'builtin') {
+        const worker = (config.workers ?? []).find(w => w.id === ref.id);
+        if (!worker)
+            throw new WorkerError('Worker not found', 404);
+        if (!(await (0, builtinAccess_1.assertBuiltinAccess)(auth, worker.id, config, 'worker')))
+            throw new WorkerError('This worker is not available to you', 403);
+        return {
+            ref: { source: 'builtin', id: worker.id, name: worker.name },
+            endpoint: worker.endpoint, credential: worker.credential ?? null,
+            kinds: worker.kinds ?? [], categories: worker.categories ?? [], autoKind: worker.autoKind ?? null,
+            timeoutSeconds: worker.timeoutSeconds ?? exports.DEFAULT_WORKER_TIMEOUT_SECONDS,
+        };
+    }
+    if (ref.source === 'project') {
+        const worker = (0, exports.projectWorkers)(project).find(w => w.id === ref.id);
+        if (!worker)
+            throw new WorkerError('Worker not found', 404);
+        // Tied to whoever connected it: once they leave the project, its credential isn't used.
+        if (worker.createdBy && !(0, auth_1.activeMemberOf)(project, worker.createdBy)) {
+            throw new WorkerError('Whoever connected this worker is no longer in the project: connect it again', 409);
+        }
+        return {
+            ref: { source: 'project', id: worker.id, name: worker.name },
+            endpoint: worker.apiUrl, credential: worker.encryptedCredential ? (0, encryption_1.decrypt)(worker.encryptedCredential) : null,
+            kinds: listOf(worker.kinds), categories: listOf(worker.categories),
+            autoKind: worker.autoKind ?? null, timeoutSeconds: worker.timeoutSeconds ?? exports.DEFAULT_WORKER_TIMEOUT_SECONDS,
+        };
+    }
+    throw new WorkerError("A worker's source is 'builtin' or 'project'");
+}
+// A project's worker is reached from StreamBy's server: only over https, and never at a local
+// or private address (unless the host allows it, for development). The address a name
+// resolves to is checked too, when it's connected and right before each job is sent.
+function checkWorkerEndpoint(value, config) {
+    let url;
+    try {
+        url = new URL(String(value));
+    }
+    catch {
+        throw new WorkerError('endpoint must be a URL');
+    }
+    const local = config.workerOptions?.allowLocalEndpoints === true;
+    if (url.username || url.password)
+        throw new WorkerError("endpoint can't carry credentials: give them as credential");
+    if (!local && url.protocol !== 'https:')
+        throw new WorkerError('endpoint must use https');
+    if (!local && (0, privateAddress_1.isPrivateHostName)(url.hostname))
+        throw new WorkerError("endpoint can't be a local or private address");
+    if (!['https:', 'http:'].includes(url.protocol))
+        throw new WorkerError('endpoint must use https');
+    return url.toString();
+}
+/**
+ * Throws if a project's worker endpoint resolves, now, to a local or private address (a
+ * public-looking name pointing at the server's own network).
+ */
+async function assertPublicEndpoint(endpoint, config) {
+    if (config.workerOptions?.allowLocalEndpoints === true)
+        return;
+    let host;
+    try {
+        host = new URL(endpoint).hostname;
+    }
+    catch {
+        throw new WorkerError('endpoint must be a URL');
+    }
+    let isPrivate;
+    try {
+        isPrivate = await (0, privateAddress_1.resolvesToPrivateAddress)(host);
+    }
+    catch {
+        throw new WorkerError(`The worker's host (${host}) couldn't be resolved`, 400);
+    }
+    if (isPrivate)
+        throw new PrivateEndpointError("The worker's endpoint resolves to a local or private address", 400);
+}
+/** A project's new worker, checked and with its credential encrypted. */
+function newWorkerConnection(input, projectId, userId, config) {
+    const name = typeof input.name === 'string' ? input.name.trim() : '';
+    if (!name)
+        throw new WorkerError('A worker needs a name');
+    const provider = input.provider === undefined ? 'streamby-worker' : input.provider;
+    if (!exports.WORKER_PROVIDERS.includes(provider))
+        throw new WorkerError(`provider must be one of ${exports.WORKER_PROVIDERS.join(', ')}`);
+    const endpoint = checkWorkerEndpoint(input.endpoint, config);
+    const kinds = listOf(input.kinds).map(k => k.trim()).filter(Boolean);
+    if (!kinds.length || kinds.length !== listOf(input.kinds).length)
+        throw new WorkerError('kinds must list what the worker does (e.g. gltf-optimize)');
+    const categories = listOf(input.categories);
+    if (!categories.length || !categories.every(c => exports.STORAGE_CATEGORIES.includes(c))) {
+        throw new WorkerError(`categories must list asset categories it takes: ${exports.STORAGE_CATEGORIES.join(', ')}`);
+    }
+    const autoKind = input.autoKind === undefined || input.autoKind === null || input.autoKind === '' ? null : input.autoKind;
+    if (autoKind !== null && (typeof autoKind !== 'string' || !kinds.includes(autoKind)))
+        throw new WorkerError('autoKind must be one of its kinds, or null');
+    const timeoutSeconds = input.timeoutSeconds === undefined ? undefined : Number(input.timeoutSeconds);
+    if (timeoutSeconds !== undefined && !(Number.isInteger(timeoutSeconds) && timeoutSeconds >= 60 && timeoutSeconds <= 7 * 86400)) {
+        throw new WorkerError('timeoutSeconds must be a whole number of seconds, from 60 to a week');
+    }
+    const credential = typeof input.credential === 'string' && input.credential ? input.credential : null;
+    if (credential && !(0, encryption_1.isEncryptionKeySet)())
+        throw new WorkerError('Credentials cannot be stored: no encryption key is set', 500);
+    return {
+        id: crypto_1.default.randomUUID(),
+        projectId,
+        name,
+        provider: provider,
+        apiUrl: endpoint,
+        ...(credential ? { encryptedCredential: (0, encryption_1.encrypt)(credential) } : {}),
+        kinds: [...new Set(kinds)],
+        categories: [...new Set(categories)],
+        autoKind: autoKind,
+        ...(timeoutSeconds !== undefined ? { timeoutSeconds } : {}),
+        ...(typeof input.description === 'string' && input.description.trim() ? { description: input.description.trim() } : {}),
+        createdBy: userId,
+        createdAt: new Date(),
+    };
+}
+exports.summarizeProjectWorker = projectSummary;

package/dist/types/index.d.ts

@@ -149,7 +149,14 @@ export interface StreamByConfig {
     websocket?: {
         server: WebSocketServer;
     };
-    canUseBuiltin?: (auth: Auth, builtinId: string, kind: 'database' | 'storage') => boolean | Promise<boolean>;
+    canUseBuiltin?: (auth: Auth, builtinId: string, kind: 'database' | 'storage' | 'worker') => boolean | Promise<boolean>;
+    workers?: WorkerConfig[];
+    workerOptions?: {
+        callbackBaseUrl?: string;
+        maxAttempts?: number;
+        sweepIntervalMs?: number;
+        allowLocalEndpoints?: boolean;
+    };
     oauthProviders?: {
         jira?: OAuthClientConfig;
         google?: OAuthClientConfig;
@@ -320,7 +327,7 @@ export interface QcReportRecord {
     overallPassed: boolean;
     generatedAt: Date;
 }
-export type JobType = 'ingest' | 'transcode' | 'caption' | 'thumbnail' | 'render' | 'format-convert' | 'lod' | 'distribute' | 'qc' | 'transcription' | 'upscale' | 'generate-asset';
+export type JobType = 'ingest' | 'transcode' | 'caption' | 'thumbnail' | 'render' | 'format-convert' | 'lod' | 'distribute' | 'qc' | 'transcription' | 'upscale' | 'generate-asset' | 'process';
 export type AiProvider = 'openai' | 'deepgram' | 'assemblyai' | 'stabilityai' | 'meshy' | 'elevenlabs' | 'custom';
 export interface PipelineSuggestion {
     rationale: string;
@@ -439,7 +446,50 @@ export interface JobRecord {
     error?: string;
     createdAt: Date;
     updatedAt: Date;
+    worker?: WorkerRef | null;
+    kind?: string | null;
+    profile?: Record<string, unknown> | null;
+    versionId?: string | null;
+    resultMode?: WorkerResultMode | null;
+    attempts?: number;
+    maxAttempts?: number;
+    nextAttemptAt?: Date | null;
+    leaseUntil?: Date | null;
+    callbackHash?: string | null;
+    upload?: {
+        storageKey: string;
+        fileName: string;
+        contentType: string;
+    } | null;
+    report?: Record<string, unknown> | null;
+    result?: {
+        assetId: string;
+        versionId: string;
+        mode: WorkerResultMode;
+    } | null;
+}
+export type WorkerResultMode = 'version' | 'derived';
+export type WorkerSource = 'builtin' | 'project';
+export interface WorkerRef {
+    source: WorkerSource;
+    id: string;
+    name: string;
 }
+/**
+ * A processing worker a host offers every project (`StreamByConfig.workers`). StreamBy sends
+ * it jobs (see the README's worker contract) and keeps what it makes.
+ */
+export interface WorkerConfig {
+    id: string;
+    name: string;
+    endpoint: string;
+    credential?: string;
+    kinds: string[];
+    categories: StorageCategoryName[];
+    autoKind?: string | null;
+    timeoutSeconds?: number;
+}
+export type StorageCategoryName = 'images' | 'audios' | 'videos' | '3d-models';
 export interface AssetMetadataRecord {
     assetId: string;
     projectId: string;

package/dist/utils/assetCategories.d.ts

@@ -0,0 +1,4 @@
+export type StorageCategory = 'images' | 'audios' | 'videos' | '3d-models';
+export declare const VALID_CATEGORIES: StorageCategory[];
+/** Whether a file (its content type, or for 3D models its extension) belongs in a category. */
+export declare function validateContentType(contentType: string, filename: string, category: StorageCategory): boolean;

package/dist/utils/assetCategories.js

@@ -0,0 +1,21 @@
+"use strict";
+// The categories assets are kept in, and what each one takes.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VALID_CATEGORIES = void 0;
+exports.validateContentType = validateContentType;
+exports.VALID_CATEGORIES = ['images', 'audios', 'videos', '3d-models'];
+const VALID_3D_EXTENSIONS = new Set(['.glb', '.gltf', '.obj', '.fbx', '.stl', '.ply']);
+/** Whether a file (its content type, or for 3D models its extension) belongs in a category. */
+function validateContentType(contentType, filename, category) {
+    if (category === 'images')
+        return contentType.startsWith('image/');
+    if (category === 'audios')
+        return contentType.startsWith('audio/');
+    if (category === 'videos')
+        return contentType.startsWith('video/');
+    if (category === '3d-models') {
+        const ext = filename.split('.').pop()?.toLowerCase();
+        return ext ? VALID_3D_EXTENSIONS.has(`.${ext}`) : false;
+    }
+    return false;
+}

package/dist/utils/builtinAccess.d.ts

@@ -1,5 +1,5 @@
 import { Auth, StreamByConfig } from '../types';
-export declare function assertBuiltinAccess(auth: Auth, builtinId: string, config: StreamByConfig, kind: 'database' | 'storage'): Promise<boolean>;
+export declare function assertBuiltinAccess(auth: Auth, builtinId: string, config: StreamByConfig, kind: 'database' | 'storage' | 'worker'): Promise<boolean>;
 export declare function isBuiltinStorageId(id: string, config: StreamByConfig): boolean;
 export declare function isBuiltinDb(id: string, config: StreamByConfig): boolean;
 export declare const BUILTIN_STORAGE_DISPLAY: Record<string, string>;

package/dist/utils/builtinAccess.js

@@ -4,9 +4,10 @@ exports.BUILTIN_DB_DISPLAY = exports.BUILTIN_STORAGE_DISPLAY = void 0;
 exports.assertBuiltinAccess = assertBuiltinAccess;
 exports.isBuiltinStorageId = isBuiltinStorageId;
 exports.isBuiltinDb = isBuiltinDb;
-// Gates access to a built-in database/storage provider. Delegates to config.canUseBuiltin
-// when the mounting app implements it (e.g. Nhexa-API checking user_subscriptions);
-// defaults to allow when absent, preserving pre-BYOC behavior for deploys without gating.
+// Gates access to a built-in database, storage provider or worker. Delegates to
+// config.canUseBuiltin when the mounting app implements it (e.g. Nhexa-API checking
+// user_subscriptions); defaults to allow when absent, preserving pre-BYOC behavior for deploys
+// without gating.
 async function assertBuiltinAccess(auth, builtinId, config, kind) {
     if (!config.canUseBuiltin)
         return true;

package/dist/utils/privateAddress.d.ts

@@ -0,0 +1,10 @@
+/** Whether an IP address (v4 or v6) is a local, private or otherwise non-public one. */
+export declare function isPrivateIp(ip: string): boolean;
+/** Whether a URL's host, as written, is a local name or a non-public IP address. */
+export declare function isPrivateHostName(hostname: string): boolean;
+/**
+ * Whether a host, resolved now, reaches a non-public address: a name can resolve to one even
+ * if it doesn't look like it, and resolve differently later, so it's checked right before each
+ * request.
+ */
+export declare function resolvesToPrivateAddress(hostname: string): Promise<boolean>;

package/dist/utils/privateAddress.js

@@ -0,0 +1,85 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isPrivateIp = isPrivateIp;
+exports.isPrivateHostName = isPrivateHostName;
+exports.resolvesToPrivateAddress = resolvesToPrivateAddress;
+const dns_1 = require("dns");
+const net_1 = require("net");
+// Whether an address is one StreamBy's server must not be made to reach on someone's behalf: a
+// loopback, private, link-local, shared, benchmarking, multicast or reserved one (anything a
+// request from the server could use to reach its own network).
+const ipv4Parts = (ip) => ip.split('.').map(Number);
+function isPrivateIpv4(ip) {
+    const [a, b] = ipv4Parts(ip);
+    return a === 0 // 0.0.0.0/8 ("this" network)
+        || a === 10 // 10.0.0.0/8
+        || a === 127 // 127.0.0.0/8 loopback
+        || (a === 100 && b >= 64 && b <= 127) // 100.64.0.0/10 shared (CGNAT)
+        || (a === 169 && b === 254) // 169.254.0.0/16 link-local (cloud metadata)
+        || (a === 172 && b >= 16 && b <= 31) // 172.16.0.0/12
+        || (a === 192 && b === 0 && ipv4Parts(ip)[2] === 0) // 192.0.0.0/24 protocol assignments
+        || (a === 192 && b === 168) // 192.168.0.0/16
+        || (a === 198 && (b === 18 || b === 19)) // 198.18.0.0/15 benchmarking
+        || a >= 224; // 224.0.0.0/4 multicast, and 240.0.0.0/4 reserved
+}
+// An IPv6 address's 8 groups, expanded (an embedded IPv4 written as dotted quad included).
+function ipv6Groups(ip) {
+    let text = ip.toLowerCase().replace(/%.*$/, '');
+    const dotted = text.match(/(\d+\.\d+\.\d+\.\d+)$/);
+    if (dotted) {
+        const [a, b, c, d] = ipv4Parts(dotted[1]);
+        text = text.slice(0, -dotted[1].length) + `${((a << 8) | b).toString(16)}:${((c << 8) | d).toString(16)}`;
+    }
+    const [head, tail] = text.split('::');
+    const left = head ? head.split(':') : [];
+    const right = tail !== undefined && tail ? tail.split(':') : [];
+    const missing = 8 - left.length - right.length;
+    if (tail === undefined ? left.length !== 8 : missing < 0)
+        return null;
+    return [...left, ...Array(tail === undefined ? 0 : missing).fill('0'), ...right].map(group => parseInt(group, 16));
+}
+function isPrivateIpv6(ip) {
+    const groups = ipv6Groups(ip);
+    if (!groups || groups.some(Number.isNaN))
+        return true; // unreadable: not reached
+    if (groups.every(g => g === 0))
+        return true; // :: unspecified
+    if (groups.slice(0, 7).every(g => g === 0) && groups[7] === 1)
+        return true; // ::1 loopback
+    // ::ffff:a.b.c.d (IPv4-mapped): judged as the IPv4 address it carries.
+    if (groups.slice(0, 5).every(g => g === 0) && groups[5] === 0xffff) {
+        return isPrivateIpv4(`${groups[6] >> 8}.${groups[6] & 0xff}.${groups[7] >> 8}.${groups[7] & 0xff}`);
+    }
+    return (groups[0] & 0xfe00) === 0xfc00 // fc00::/7 unique local (e.g. fdaa::, Fly's network)
+        || (groups[0] & 0xffc0) === 0xfe80 // fe80::/10 link-local
+        || (groups[0] & 0xff00) === 0xff00; // ff00::/8 multicast
+}
+/** Whether an IP address (v4 or v6) is a local, private or otherwise non-public one. */
+function isPrivateIp(ip) {
+    const bare = ip.replace(/^\[|\]$/g, '');
+    if ((0, net_1.isIP)(bare) === 4)
+        return isPrivateIpv4(bare);
+    if ((0, net_1.isIP)(bare) === 6)
+        return isPrivateIpv6(bare);
+    return true;
+}
+const LOCAL_NAME = /^(localhost|.*\.localhost|.*\.local|.*\.internal)$/i;
+/** Whether a URL's host, as written, is a local name or a non-public IP address. */
+function isPrivateHostName(hostname) {
+    const bare = hostname.replace(/^\[|\]$/g, '');
+    return (0, net_1.isIP)(bare) ? isPrivateIp(bare) : LOCAL_NAME.test(bare);
+}
+/**
+ * Whether a host, resolved now, reaches a non-public address: a name can resolve to one even
+ * if it doesn't look like it, and resolve differently later, so it's checked right before each
+ * request.
+ */
+async function resolvesToPrivateAddress(hostname) {
+    const bare = hostname.replace(/^\[|\]$/g, '');
+    if (isPrivateHostName(bare))
+        return true;
+    if ((0, net_1.isIP)(bare))
+        return false;
+    const addresses = await dns_1.promises.lookup(bare, { all: true });
+    return !addresses.length || addresses.some(({ address }) => isPrivateIp(address));
+}

package/dist/utils/sanitize.d.ts

@@ -14,5 +14,6 @@ export declare function sanitizeIntegrationConnection<T extends object>(conn: T)
 export declare function sanitizeIntegrationConnections<T extends object>(conns: T[] | undefined): (T & {
     hasCredential: true;
 })[];
+export declare function exportSummaries(exports: any[] | undefined): any[];
 export declare function sanitizeProject(project: any): any;
 export declare function sanitizeUserIntegration(integration: any): any;

package/dist/utils/sanitize.js

@@ -4,6 +4,7 @@ exports.sanitizeConnection = sanitizeConnection;
 exports.sanitizeConnections = sanitizeConnections;
 exports.sanitizeIntegrationConnection = sanitizeIntegrationConnection;
 exports.sanitizeIntegrationConnections = sanitizeIntegrationConnections;
+exports.exportSummaries = exportSummaries;
 exports.sanitizeProject = sanitizeProject;
 exports.sanitizeUserIntegration = sanitizeUserIntegration;
 function sanitizeConnection(conn) {
@@ -23,6 +24,19 @@ function sanitizeIntegrationConnection(conn) {
 function sanitizeIntegrationConnections(conns) {
     return (conns ?? []).map(sanitizeIntegrationConnection);
 }
+// A project's exports as they're listed: what each one is, not its schema nor its credentials.
+// The same in the project (GET /projects/:id) and in its own list (GET /projects/:id/exports).
+function exportSummaries(exports) {
+    return (exports || []).map((e) => ({
+        id: e.id,
+        name: e.name,
+        description: e.description,
+        method: e.method,
+        private: e.private,
+        useConnections: e.useConnections,
+        useCredentials: e.useCredentials,
+    }));
+}
 function sanitizeProject(project) {
     if (!project)
         return project;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "streamby-core",
-  "version": "0.38.0",
+  "version": "0.40.0",
   "description": "StreamBy middleware framework for media storage management",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",