streamby-core 0.31.0 → 0.32.1

package/dist/services/connectionResolver.d.ts

@@ -0,0 +1,19 @@
+import { Pool } from 'pg';
+import { MongoClient } from 'mongodb';
+import { StreamByConfig, Auth, DbConnection, StorageConnection, StorageAdapter } from '../types';
+export type ResolvedDbClient = {
+    client: Pool | MongoClient;
+    type: 'sql' | 'nosql';
+    ephemeral: boolean;
+};
+export type ResolveError = {
+    error: string;
+    status: number;
+};
+export declare function findDbConnection(project: any, connId: string): DbConnection | undefined;
+export declare function findStorageConnection(project: any, connId: string): StorageConnection | undefined;
+export declare function resolveDbConnectionClient(project: any, connId: string, config: StreamByConfig, auth: Auth): Promise<ResolvedDbClient | ResolveError>;
+export declare function releaseDbClient(resolved: ResolvedDbClient): Promise<void>;
+export declare function resolveStorageAdapter(project: any, connId: string, config: StreamByConfig & {
+    adapter?: StorageAdapter;
+}, auth: Auth): Promise<StorageAdapter | ResolveError>;

package/dist/services/connectionResolver.js

@@ -0,0 +1,132 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findDbConnection = findDbConnection;
+exports.findStorageConnection = findStorageConnection;
+exports.resolveDbConnectionClient = resolveDbConnectionClient;
+exports.releaseDbClient = releaseDbClient;
+exports.resolveStorageAdapter = resolveStorageAdapter;
+const pg_1 = require("pg");
+const mongodb_1 = require("mongodb");
+const connectionManager_1 = require("../adapters/database/connectionManager");
+const builtinAccess_1 = require("../utils/builtinAccess");
+const userIntegration_1 = require("./userIntegration");
+const encryption_1 = require("../utils/encryption");
+const storage_1 = require("../providers/storage");
+const s3_1 = require("../adapters/storage/s3");
+// Matches by row id first; falls back to matching a builtin row by integrationId (the
+// config id, e.g. 'mongo') for nodes/connections saved before TCORE-69 gave builtin rows
+// their own generated id — same legacy-id class as connFilter in storage.ts.
+function findDbConnection(project, connId) {
+    return project.dbConnections?.find((c) => c.id === connId)
+        ?? project.dbConnections?.find((c) => c.source === 'builtin' && c.integrationId === connId);
+}
+function findStorageConnection(project, connId) {
+    return project.storageConnections?.find((c) => c.id === connId)
+        ?? project.storageConnections?.find((c) => c.source === 'builtin' && c.integrationId === connId);
+}
+// `connId` must be the `id` of a real row in project.dbConnections — never a bare
+// config id (config.databases[].id). Builtin rows reuse the shared pooled client keyed by
+// conn.integrationId (config id); BYOC/manual rows open a short-lived client from their own
+// decrypted credential, which the caller must close via `releaseDbClient` once done.
+async function resolveDbConnectionClient(project, connId, config, auth) {
+    const conn = findDbConnection(project, connId);
+    if (!conn)
+        return { error: 'DB connection not found', status: 404 };
+    if (conn.source === 'builtin') {
+        const builtinId = conn.integrationId;
+        if (!(await (0, builtinAccess_1.assertBuiltinAccess)(auth, builtinId, config, 'database'))) {
+            return { error: 'Access to this built-in database is not permitted', status: 403 };
+        }
+        const db = (config.databases ?? []).find(d => d.id === builtinId);
+        if (!db)
+            return { error: `Database '${builtinId}' not found in config`, status: 404 };
+        try {
+            const { client, type } = (0, connectionManager_1.getConnection)(builtinId);
+            return { client: client, type, ephemeral: false };
+        }
+        catch (e) {
+            return { error: e.message, status: 503 };
+        }
+    }
+    let connectionString;
+    if (conn.source === 'integration') {
+        if (!conn.integrationId)
+            return { error: 'Connection is missing its integrationId', status: 500 };
+        const credential = await (0, userIntegration_1.getDecryptedIntegrationCredentialById)(conn.integrationId);
+        if (!credential)
+            return { error: 'Integration not found', status: 400 };
+        connectionString = credential;
+    }
+    else {
+        if (!(0, encryption_1.isEncryptionKeySet)())
+            return { error: 'Encryption key is not set', status: 500 };
+        if (!conn.encryptedCredential)
+            return { error: 'Connection has no stored credential', status: 400 };
+        connectionString = (0, encryption_1.decrypt)(conn.encryptedCredential);
+    }
+    const type = conn.dbType === 'postgresql' ? 'sql' : 'nosql';
+    try {
+        if (type === 'sql') {
+            const pool = new pg_1.Pool({ connectionString });
+            await pool.query('SELECT 1');
+            return { client: pool, type, ephemeral: true };
+        }
+        const client = new mongodb_1.MongoClient(connectionString);
+        await client.connect();
+        return { client, type, ephemeral: true };
+    }
+    catch (e) {
+        return { error: `Failed to connect: ${e.message}`, status: 503 };
+    }
+}
+// Only closes clients this resolver opened itself (`ephemeral: true`) — the shared pooled
+// builtin clients are owned by connectionManager and must outlive a single request.
+async function releaseDbClient(resolved) {
+    if (!resolved.ephemeral)
+        return;
+    if (resolved.type === 'sql')
+        await resolved.client.end();
+    else
+        await resolved.client.close();
+}
+async function resolveStorageAdapter(project, connId, config, auth) {
+    const conn = findStorageConnection(project, connId);
+    if (!conn)
+        return { error: 'Storage connection not found', status: 404 };
+    if (conn.source === 'builtin') {
+        const builtinId = conn.integrationId;
+        if (!(await (0, builtinAccess_1.assertBuiltinAccess)(auth, builtinId, config, 'storage'))) {
+            return { error: 'Access to this built-in storage is not permitted', status: 403 };
+        }
+        const provider = config.storageProviders.find(p => p.id === builtinId);
+        if (!provider)
+            return { error: 'Storage provider not found', status: 404 };
+        if (provider.type === 's3')
+            return new s3_1.S3Adapter(provider.config);
+        return config.adapter || (0, storage_1.createStorageProvider)(config.storageProviders);
+    }
+    if (conn.source === 'integration') {
+        if (!conn.integrationId)
+            return { error: 'Connection is missing its integrationId', status: 500 };
+        try {
+            const s3Config = await (0, userIntegration_1.getDecryptedIntegrationCredentialById)(conn.integrationId);
+            if (!s3Config)
+                return { error: 'Integration not found', status: 400 };
+            return new s3_1.S3Adapter(s3Config);
+        }
+        catch (e) {
+            return { error: `Failed to initialize storage adapter: ${e.message}`, status: 500 };
+        }
+    }
+    if (!(0, encryption_1.isEncryptionKeySet)())
+        return { error: 'Encryption key not set', status: 500 };
+    if (!conn.encryptedCredential)
+        return { error: 'Connection has no stored credential', status: 400 };
+    try {
+        const s3Config = JSON.parse((0, encryption_1.decrypt)(conn.encryptedCredential));
+        return new s3_1.S3Adapter(s3Config);
+    }
+    catch (e) {
+        return { error: `Failed to initialize storage adapter: ${e.message}`, status: 500 };
+    }
+}

package/dist/services/executor.d.ts

@@ -1,2 +1,7 @@
 import { NodeSchema, StreamByConfig } from '../types';
+export declare class NodeExecutionError extends Error {
+    nodeId: string;
+    nodeType: string;
+    constructor(nodeId: string, nodeType: string, message: string);
+}
 export declare function executeExport(nodeSchema: NodeSchema, project: any, config: StreamByConfig): Promise<any>;

package/dist/services/executor.js

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.NodeExecutionError = void 0;
 exports.executeExport = executeExport;
-const connectionManager_1 = require("../adapters/database/connectionManager");
 const encryption_1 = require("../utils/encryption");
 const dbConnection_1 = require("./dbConnection");
+const connectionResolver_1 = require("./connectionResolver");
 const jobQueue_1 = require("./jobQueue");
 const mediaProcessor_1 = require("./mediaProcessor");
 const vfxProcessor_1 = require("./vfxProcessor");
@@ -75,6 +76,21 @@ function getTarget(nodes, edges, sourceId, sourceHandle) {
     const edge = edges.find(e => e.source === sourceId && e.sourceHandle === sourceHandle);
     return edge ? (nodes.find(n => n.id === edge.target) ?? null) : null;
 }
+class NodeExecutionError extends Error {
+    constructor(nodeId, nodeType, message) {
+        super(message);
+        this.nodeId = nodeId;
+        this.nodeType = nodeType;
+        this.name = 'NodeExecutionError';
+    }
+}
+exports.NodeExecutionError = NodeExecutionError;
+// Export execution is public (gated by allowedOrigin, not user identity — see
+// export.ts:339) — there is no real Auth to pass through. This sentinel satisfies
+// resolveDbConnectionClient's signature without adding a gate that never existed;
+// canUseBuiltin gating is for connection *management* by a user, not for serving an
+// already-published export.
+const SYSTEM_AUTH = { userId: 'system', username: 'system', role: 'admin' };
 async function executeExport(nodeSchema, project, config) {
     const nodes = nodeSchema.nodes;
     const edges = nodeSchema.edges;
@@ -94,24 +110,26 @@ async function executeExport(nodeSchema, project, config) {
             const tableName = (node.data?.tableName || node.data?.subtitle || node.data?.label);
             if (!tableName)
                 continue;
+            // dataSourceNode always declares its own database connection explicitly (persistence:
+            // 'database' in types/nodePersistence.ts) — a project has no notion of a "default"
+            // connection, since a project can have many database/storage/API providers connected.
             const connectionId = node.data?.connectionId;
+            if (!connectionId) {
+                throw new NodeExecutionError(node.id, node.type, `Node '${node.id}' (${node.type}) has no database connection — connect one on the node.`);
+            }
             const recordId = node.data?.recordId;
             const projectIdentifier = project._id?.toString() ?? project.id;
-            const fetchData = async (client, type) => recordId
-                ? (0, dbConnection_1.queryRecordByIdInternal)(client, type, tableName, recordId, projectIdentifier)
-                : (0, dbConnection_1.queryRecordsInternal)(client, type, tableName, 500, 0, projectIdentifier);
-            if (connectionId) {
-                const { client, type } = (0, connectionManager_1.getConnection)(connectionId);
-                dataResults.push(await fetchData(client, type));
+            const resolved = await (0, connectionResolver_1.resolveDbConnectionClient)(project, connectionId, config, SYSTEM_AUTH);
+            if ('error' in resolved) {
+                throw new NodeExecutionError(node.id, node.type, `Node '${node.id}' (${node.type}): ${resolved.error}`);
             }
-            else {
-                // legacy fallback for nodes saved before connectionId was introduced
-                const targetDb = config.databases?.find((db) => db.type === project.dbType && db.main)
-                    ?? config.databases?.find((db) => db.type === project.dbType);
-                if (!targetDb)
-                    throw new Error(`No database connection for type ${project.dbType}`);
-                const { client, type } = (0, connectionManager_1.getConnection)(targetDb.id);
-                dataResults.push(await fetchData(client, type));
+            try {
+                dataResults.push(recordId
+                    ? await (0, dbConnection_1.queryRecordByIdInternal)(resolved.client, resolved.type, tableName, recordId, projectIdentifier)
+                    : await (0, dbConnection_1.queryRecordsInternal)(resolved.client, resolved.type, tableName, 500, 0, projectIdentifier));
+            }
+            finally {
+                await (0, connectionResolver_1.releaseDbClient)(resolved);
             }
         }
         else if (node.type === 'connectionNode') {
@@ -170,6 +188,17 @@ async function executeExport(nodeSchema, project, config) {
             payload = { ...payload, jobId: job.jobId, jobType: 'thumbnail' };
         }
         else if (processNode.type === 'ingestNode' && fileId) {
+            // ingestNode always declares its own storage connection explicitly (persistence:
+            // 'storage' in types/nodePersistence.ts) — a project has no notion of a "default"
+            // connection, since a project can have many database/storage/API providers connected.
+            const storageConnectionId = nodeData.storageConnectionId;
+            if (!storageConnectionId) {
+                throw new NodeExecutionError(processNode.id, processNode.type, `Node '${processNode.id}' (${processNode.type}) has no storage connection — connect one on the node.`);
+            }
+            const storageResolved = await (0, connectionResolver_1.resolveStorageAdapter)(project, storageConnectionId, config, SYSTEM_AUTH);
+            if ('error' in storageResolved) {
+                throw new NodeExecutionError(processNode.id, processNode.type, `Node '${processNode.id}' (${processNode.type}): ${storageResolved.error}`);
+            }
             const job = (0, jobQueue_1.createJob)('ingest', systemUserId, projectId, nodeData);
             setImmediate(() => (0, mediaProcessor_1.runIngestJob)(job.jobId, fileId, projectId));
             payload = { ...payload, jobId: job.jobId, jobType: 'ingest' };

package/dist/types/index.d.ts

@@ -78,6 +78,18 @@ export interface ProjectInfo {
     storageConnections?: StorageConnection[];
     pipelines?: PipelineRef[];
     builtinBackfilledAt?: Date;
+    workflow?: Workflow;
+}
+export type NodePersistence = 'database' | 'storage' | 'both' | 'none';
+export interface Workflow {
+    id: string;
+    name: string;
+    description?: string;
+    status: string;
+    projectId: string;
+    nodeSchema: NodeSchema | null;
+    createdAt: Date;
+    updatedAt: Date;
 }
 export type ConnectionMethod = 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE';
 export interface Connection {

package/dist/types/nodePersistence.d.ts

@@ -0,0 +1,3 @@
+import { NodePersistence } from './index';
+export declare const NODE_PERSISTENCE: Record<string, NodePersistence>;
+export declare function getNodePersistence(nodeType: string): NodePersistence;

package/dist/types/nodePersistence.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NODE_PERSISTENCE = void 0;
+exports.getNodePersistence = getNodePersistence;
+// Only node types that write to a real database/storage connection need an entry here.
+// Process nodes (transcodeNode, captionNode, etc.) operate on a fileId already resolved by
+// the internal job pipeline (mediaProcessor.ts), not directly on project connections, so
+// they're 'none' — same for filter/deliverable/distribution/annotation/connection nodes.
+exports.NODE_PERSISTENCE = {
+    dataSourceNode: 'database',
+    ingestNode: 'storage',
+};
+function getNodePersistence(nodeType) {
+    return exports.NODE_PERSISTENCE[nodeType] ?? 'none';
+}

package/package.json

@@ -1,63 +1,63 @@
-{
-  "name": "streamby-core",
-  "version": "0.31.0",
-  "description": "StreamBy middleware framework for media storage management",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "files": [
-    "dist"
-  ],
-  "scripts": {
-    "build": "tsc",
-    "test": "vitest",
-    "dev": "ts-node-dev --respawn --clear dev.ts",
-    "prepublishOnly": "npm run build"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/GwerhDev/StreamBy-Package.git"
-  },
-  "keywords": [
-    "streamby",
-    "middleware",
-    "upload",
-    "storage",
-    "s3",
-    "express",
-    "typescript"
-  ],
-  "author": "TerminalCore Labs",
-  "license": "MIT",
-  "dependencies": {
-    "@aws-sdk/client-s3": "^3.528.0",
-    "@aws-sdk/s3-request-presigner": "^3.787.0",
-    "@prisma/client": "^6.12.0",
-    "bcrypt": "^5.1.1",
-    "dotenv": "^16.5.0",
-    "express": "^4.0.0",
-    "mongodb": "^6.17.0",
-    "mongoose": "^8.13.2",
-    "node-fetch": "^3.0.0",
-    "pg": "^8.16.3",
-    "prisma": "^6.12.0",
-    "ws": "^8.20.0"
-  },
-  "devDependencies": {
-    "@types/bcrypt": "^5.0.2",
-    "@types/cors": "^2.8.18",
-    "@types/express": "^4.17.17",
-    "@types/mongodb": "^4.0.7",
-    "@types/node": "^22.15.21",
-    "@types/pg": "^8.15.4",
-    "@types/supertest": "^6.0.3",
-    "@types/ws": "^8.18.1",
-    "cors": "^2.8.5",
-    "supertest": "^7.1.4",
-    "ts-node-dev": "^2.0.0",
-    "typescript": "^5.8.3",
-    "vitest": "^3.1.1"
-  },
-  "peerDependencies": {
-    "express": "^4.0.0"
-  }
-}
+{
+  "name": "streamby-core",
+  "version": "0.32.1",
+  "description": "StreamBy middleware framework for media storage management",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest",
+    "dev": "ts-node-dev --respawn --clear dev.ts",
+    "prepublishOnly": "npm run build"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/GwerhDev/StreamBy-Package.git"
+  },
+  "keywords": [
+    "streamby",
+    "middleware",
+    "upload",
+    "storage",
+    "s3",
+    "express",
+    "typescript"
+  ],
+  "author": "TerminalCore Labs",
+  "license": "MIT",
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.528.0",
+    "@aws-sdk/s3-request-presigner": "^3.787.0",
+    "@prisma/client": "^6.12.0",
+    "bcrypt": "^5.1.1",
+    "dotenv": "^16.5.0",
+    "express": "^4.0.0",
+    "mongodb": "^6.17.0",
+    "mongoose": "^8.13.2",
+    "node-fetch": "^3.0.0",
+    "pg": "^8.16.3",
+    "prisma": "^6.12.0",
+    "ws": "^8.20.0"
+  },
+  "devDependencies": {
+    "@types/bcrypt": "^5.0.2",
+    "@types/cors": "^2.8.18",
+    "@types/express": "^4.17.17",
+    "@types/mongodb": "^4.0.7",
+    "@types/node": "^22.15.21",
+    "@types/pg": "^8.15.4",
+    "@types/supertest": "^6.0.3",
+    "@types/ws": "^8.18.1",
+    "cors": "^2.8.5",
+    "supertest": "^7.1.4",
+    "ts-node-dev": "^2.0.0",
+    "typescript": "^5.8.3",
+    "vitest": "^3.1.1"
+  },
+  "peerDependencies": {
+    "express": "^4.0.0"
+  }
+}

package/dist/utils/crypto.d.ts

@@ -1,3 +0,0 @@
-export declare function encrypt(text: string, secretKey: string): string;
-export declare function decrypt(text: string, secretKey: string): string;
-export declare function hash(text: string): string;

package/dist/utils/crypto.js

@@ -1,32 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.encrypt = encrypt;
-exports.decrypt = decrypt;
-exports.hash = hash;
-const crypto_1 = __importDefault(require("crypto"));
-const ALGORITHM = 'aes-256-cbc';
-const IV_LENGTH = 16;
-function encrypt(text, secretKey) {
-    const iv = crypto_1.default.randomBytes(IV_LENGTH);
-    const key = crypto_1.default.createHash('sha256').update(String(secretKey)).digest('base64').substr(0, 32);
-    const cipher = crypto_1.default.createCipheriv(ALGORITHM, Buffer.from(key), iv);
-    let encrypted = cipher.update(text);
-    encrypted = Buffer.concat([encrypted, cipher.final()]);
-    return iv.toString('hex') + ':' + encrypted.toString('hex');
-}
-function decrypt(text, secretKey) {
-    const textParts = text.split(':');
-    const iv = Buffer.from(textParts.shift(), 'hex');
-    const encryptedText = Buffer.from(textParts.join(':'), 'hex');
-    const key = crypto_1.default.createHash('sha256').update(String(secretKey)).digest('base64').substr(0, 32);
-    const decipher = crypto_1.default.createDecipheriv(ALGORITHM, Buffer.from(key), iv);
-    let decrypted = decipher.update(encryptedText);
-    decrypted = Buffer.concat([decrypted, decipher.final()]);
-    return decrypted.toString();
-}
-function hash(text) {
-    return crypto_1.default.createHash('sha256').update(text).digest('hex');
-}