streamby-core 0.33.0 → 0.35.0

package/dist/services/pipeline.js

@@ -1,292 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.executePipeline = executePipeline;
-const connectionManager_1 = require("../adapters/database/connectionManager");
-const encryption_1 = require("../utils/encryption");
-const dbConnection_1 = require("./dbConnection");
-const jobQueue_1 = require("./jobQueue");
-const mediaProcessor_1 = require("./mediaProcessor");
-const vfxProcessor_1 = require("./vfxProcessor");
-const aiProcessor_1 = require("./aiProcessor");
-const storage_1 = require("../providers/storage");
-function applyFilterConfig(payload, config) {
-    const isArr = Array.isArray(payload);
-    let r = isArr ? [...payload] : payload;
-    if (config.conditions?.length) {
-        const matches = (item) => config.conditions.every(c => {
-            const v = item?.[c.field];
-            switch (c.op) {
-                case 'eq': return String(v) === c.value;
-                case 'neq': return String(v) !== c.value;
-                case 'gt': return Number(v) > Number(c.value);
-                case 'lt': return Number(v) < Number(c.value);
-                case 'gte': return Number(v) >= Number(c.value);
-                case 'lte': return Number(v) <= Number(c.value);
-                case 'contains': return String(v).includes(c.value);
-                case 'startsWith': return String(v).startsWith(c.value);
-                case 'endsWith': return String(v).endsWith(c.value);
-                default: return true;
-            }
-        });
-        r = isArr ? r.filter(matches) : (matches(r) ? r : null);
-        if (r === null)
-            return null;
-    }
-    if (config.includeFields?.length) {
-        const pick = (item) => {
-            if (!item || typeof item !== 'object')
-                return item;
-            const o = {};
-            for (const f of config.includeFields)
-                if (f in item)
-                    o[f] = item[f];
-            return o;
-        };
-        r = isArr ? r.map(pick) : pick(r);
-    }
-    if (config.renameFields?.length) {
-        const ren = (item) => {
-            if (!item || typeof item !== 'object')
-                return item;
-            const o = { ...item };
-            for (const { from, to } of config.renameFields) {
-                if (from in o) {
-                    o[to] = o[from];
-                    delete o[from];
-                }
-            }
-            return o;
-        };
-        r = isArr ? r.map(ren) : ren(r);
-    }
-    if (config.limit && isArr)
-        r = r.slice(0, config.limit);
-    if (config.wrapKey)
-        r = { [config.wrapKey]: r };
-    return r;
-}
-function getSources(nodes, edges, targetId, targetHandle) {
-    return edges
-        .filter(e => e.target === targetId && e.targetHandle === targetHandle)
-        .map(e => nodes.find(n => n.id === e.source))
-        .filter((n) => n !== undefined);
-}
-function getTarget(nodes, edges, sourceId, sourceHandle) {
-    const edge = edges.find(e => e.source === sourceId && e.sourceHandle === sourceHandle);
-    return edge ? (nodes.find(n => n.id === edge.target) ?? null) : null;
-}
-async function executePipeline(nodeSchema, project, config) {
-    const nodes = nodeSchema.nodes;
-    const edges = nodeSchema.edges;
-    // 1. Data layer — nodes connected to streamby in-bottom
-    const dataSources = getSources(nodes, edges, 'streamby', 'in-bottom');
-    const dataResults = [];
-    for (const node of dataSources) {
-        if (node.type === 'jsonInputNode') {
-            try {
-                dataResults.push(JSON.parse(node.data?.jsonString || 'null'));
-            }
-            catch {
-                dataResults.push(null);
-            }
-        }
-        else if (node.type === 'dataSourceNode') {
-            const tableName = (node.data?.tableName || node.data?.subtitle || node.data?.label);
-            if (!tableName)
-                continue;
-            const connectionId = node.data?.connectionId;
-            const recordId = node.data?.recordId;
-            const projectIdentifier = project._id?.toString() ?? project.id;
-            const fetchData = async (client, type) => recordId
-                ? (0, dbConnection_1.queryRecordByIdInternal)(client, type, tableName, recordId, projectIdentifier)
-                : (0, dbConnection_1.queryRecordsInternal)(client, type, tableName, 500, 0, projectIdentifier);
-            if (connectionId) {
-                const { client, type } = (0, connectionManager_1.getConnection)(connectionId);
-                dataResults.push(await fetchData(client, type));
-            }
-            else {
-                // legacy fallback for nodes saved before connectionId was introduced
-                const targetDb = config.databases?.find((db) => db.type === project.dbType && db.main)
-                    ?? config.databases?.find((db) => db.type === project.dbType);
-                if (!targetDb)
-                    throw new Error(`No database connection for type ${project.dbType}`);
-                const { client, type } = (0, connectionManager_1.getConnection)(targetDb.id);
-                dataResults.push(await fetchData(client, type));
-            }
-        }
-        else if (node.type === 'apiConnectionNode') {
-            const apiConnection = project.apiConnections?.find((c) => c.id === node.data?.connectionId);
-            if (!apiConnection)
-                continue;
-            const headers = { 'Content-Type': 'application/json' };
-            if (apiConnection.credentialId && (0, encryption_1.isEncryptionKeySet)()) {
-                const credential = project.credentials?.find((c) => c.id === apiConnection.credentialId);
-                if (credential) {
-                    const decrypted = (0, encryption_1.decrypt)(credential.encryptedValue);
-                    const prefix = apiConnection.prefix ? `${apiConnection.prefix} ` : '';
-                    headers['Authorization'] = `${prefix}${decrypted}`;
-                }
-            }
-            const response = await fetch(apiConnection.apiUrl, { method: apiConnection.method || 'GET', headers });
-            if (!response.ok)
-                throw new Error(`API connection fetch failed: ${response.statusText}`);
-            dataResults.push(await response.json());
-        }
-    }
-    let payload = dataResults.length === 1 ? dataResults[0] : dataResults;
-    // 2. Process layer — chain from streamby out-top → processNode in-process → out-process → ...
-    const adapter = (0, storage_1.createStorageProvider)(config.storageProviders);
-    const projectId = project._id?.toString() ?? project.id;
-    // userId is not available in pipeline context; use a sentinel for job attribution
-    const systemUserId = 'pipeline';
-    let processNode = getTarget(nodes, edges, 'streamby', 'out-top');
-    while (processNode) {
-        const nodeData = processNode.data ?? {};
-        const fileId = nodeData.fileId;
-        if (processNode.type === 'transcodeNode' && fileId) {
-            const job = (0, jobQueue_1.createJob)('transcode', systemUserId, projectId, nodeData);
-            setImmediate(() => (0, mediaProcessor_1.runTranscodeJob)(job.jobId, fileId, projectId, {
-                codec: nodeData.codec,
-                resolution: nodeData.resolution,
-                outputFormat: nodeData.outputFormat,
-                bitrate: nodeData.bitrate,
-                audioCodec: nodeData.audioCodec,
-            }, adapter));
-            payload = { ...payload, jobId: job.jobId, jobType: 'transcode' };
-        }
-        else if (processNode.type === 'captionNode' && fileId) {
-            const job = (0, jobQueue_1.createJob)('caption', systemUserId, projectId, nodeData);
-            setImmediate(() => (0, mediaProcessor_1.runCaptionJob)(job.jobId, fileId, projectId, {
-                sourceLanguage: nodeData.sourceLanguage,
-                outputFormat: nodeData.outputFormat,
-                provider: nodeData.provider,
-            }));
-            payload = { ...payload, jobId: job.jobId, jobType: 'caption' };
-        }
-        else if (processNode.type === 'thumbnailNode' && fileId) {
-            const job = (0, jobQueue_1.createJob)('thumbnail', systemUserId, projectId, nodeData);
-            setImmediate(() => (0, mediaProcessor_1.runThumbnailJob)(job.jobId, fileId, projectId, {
-                timecode: nodeData.timecode,
-                resolution: nodeData.resolution,
-                strategy: nodeData.strategy,
-            }));
-            payload = { ...payload, jobId: job.jobId, jobType: 'thumbnail' };
-        }
-        else if (processNode.type === 'ingestNode' && fileId) {
-            const job = (0, jobQueue_1.createJob)('ingest', systemUserId, projectId, nodeData);
-            setImmediate(() => (0, mediaProcessor_1.runIngestJob)(job.jobId, fileId, projectId));
-            payload = { ...payload, jobId: job.jobId, jobType: 'ingest' };
-        }
-        else if (processNode.type === 'renderJobNode' && fileId) {
-            const job = (0, jobQueue_1.createJob)('render', systemUserId, projectId, nodeData);
-            setImmediate(() => (0, vfxProcessor_1.runRenderJob)(job.jobId, fileId, projectId, {
-                renderer: nodeData.renderer,
-                renderFarmConnectionId: nodeData.renderFarmConnectionId,
-                frameRange: nodeData.frameRange,
-                resolution: nodeData.resolution,
-                samples: nodeData.samples,
-                outputFormat: nodeData.outputFormat,
-            }));
-            payload = { ...payload, jobId: job.jobId, jobType: 'render' };
-        }
-        else if (processNode.type === 'formatConvertNode' && fileId) {
-            const job = (0, jobQueue_1.createJob)('format-convert', systemUserId, projectId, nodeData);
-            setImmediate(() => (0, vfxProcessor_1.runFormatConvertJob)(job.jobId, fileId, projectId, {
-                inputFormat: nodeData.inputFormat,
-                outputFormat: nodeData.outputFormat,
-                applyTransforms: nodeData.applyTransforms,
-                embedTextures: nodeData.embedTextures,
-            }));
-            payload = { ...payload, jobId: job.jobId, jobType: 'format-convert' };
-        }
-        else if (processNode.type === 'lodNode' && fileId) {
-            const job = (0, jobQueue_1.createJob)('lod', systemUserId, projectId, nodeData);
-            setImmediate(() => (0, vfxProcessor_1.runLodJob)(job.jobId, fileId, projectId, {
-                levels: nodeData.levels,
-                reductionRatios: nodeData.reductionRatios,
-                algorithm: nodeData.algorithm,
-                outputFormat: nodeData.outputFormat,
-            }));
-            payload = { ...payload, jobId: job.jobId, jobType: 'lod' };
-        }
-        else if (processNode.type === 'qcCheckNode' && fileId) {
-            const checks = nodeData.checks ?? [];
-            const checkResults = checks.map((name) => ({
-                name, passed: true, value: 'ok', threshold: 'ok',
-            }));
-            payload = { ...payload, qcReport: { checks: checkResults, overallPassed: true } };
-        }
-        else if (processNode.type === 'transcriptionNode' && fileId) {
-            const job = (0, jobQueue_1.createJob)('transcription', systemUserId, projectId, nodeData);
-            setImmediate(() => (0, aiProcessor_1.runTranscriptionJob)(job.jobId, fileId, projectId, {
-                model: nodeData.model,
-                sourceLanguage: nodeData.sourceLanguage,
-                outputFormats: nodeData.outputFormats,
-                provider: nodeData.provider,
-                credentialId: nodeData.credentialId,
-            }));
-            payload = { ...payload, jobId: job.jobId, jobType: 'transcription' };
-        }
-        else if (processNode.type === 'upscaleNode' && fileId) {
-            const job = (0, jobQueue_1.createJob)('upscale', systemUserId, projectId, nodeData);
-            setImmediate(() => (0, aiProcessor_1.runUpscaleJob)(job.jobId, fileId, projectId, {
-                scale: nodeData.scale,
-                model: nodeData.model,
-                mode: nodeData.mode,
-                credentialId: nodeData.credentialId,
-            }));
-            payload = { ...payload, jobId: job.jobId, jobType: 'upscale' };
-        }
-        else if (processNode.type === 'proceduralAssetNode') {
-            const job = (0, jobQueue_1.createJob)('generate-asset', systemUserId, projectId, nodeData);
-            setImmediate(() => (0, aiProcessor_1.runGenerateAssetJob)(job.jobId, projectId, {
-                assetType: nodeData.assetType,
-                provider: nodeData.provider,
-                prompt: nodeData.prompt ?? '',
-                seed: nodeData.seed,
-                credentialId: nodeData.credentialId,
-            }));
-            payload = { ...payload, jobId: job.jobId, jobType: 'generate-asset' };
-        }
-        else if (processNode.type === 'pipelineSuggestNode') {
-            const suggestion = await (0, aiProcessor_1.buildPipelineSuggestion)({ nodes, edges }, Object.keys(nodeSchema));
-            payload = { ...payload, pipelineSuggestion: suggestion };
-        }
-        processNode = getTarget(nodes, edges, processNode.id, 'out-process');
-    }
-    // 3. Filter + output lane — streamby out-right → filterNode|annotationNode|deliverableNode|distributionNode
-    const outputLaneTypes = new Set(['filterNode', 'annotationNode', 'deliverableNode', 'distributionNode']);
-    let outputNode = getTarget(nodes, edges, 'streamby', 'out-right');
-    while (outputNode) {
-        if (outputNode.type === 'filterNode') {
-            const cfg = outputNode.data?.filterConfig;
-            if (cfg)
-                payload = applyFilterConfig(payload, cfg);
-        }
-        else if (outputNode.type === 'deliverableNode') {
-            payload = {
-                ...payload,
-                deliverable: {
-                    type: outputNode.data?.deliverableType ?? 'asset-bundle',
-                    version: outputNode.data?.deliverableVersion ?? '1.0.0',
-                },
-            };
-        }
-        else if (outputNode.type === 'distributionNode') {
-            // Distribution is async — the actual publish is handled by the host worker
-            payload = {
-                ...payload,
-                distribution: {
-                    target: outputNode.data?.distributionTarget,
-                    connectionId: outputNode.data?.distributionConnectionId,
-                    channel: outputNode.data?.channel,
-                    status: 'queued',
-                },
-            };
-        }
-        // annotationNode — pass-through; annotations are stored separately via the review API
-        const next = getTarget(nodes, edges, outputNode.id, 'out-filter');
-        outputNode = next && outputLaneTypes.has(next.type) ? next : null;
-    }
-    return payload;
-}