streamby-core 0.39.0 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +84 -0
- package/LICENSE +661 -661
- package/README.md +785 -573
- package/dist/adapters/database/connectionManager.js +182 -124
- package/dist/adapters/database/sql.js +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.js +9 -1
- package/dist/mcp/audit.d.ts +5 -2
- package/dist/mcp/audit.js +8 -0
- package/dist/mcp/server.js +3 -3
- package/dist/mcp/tools.js +41 -0
- package/dist/middleware/createRouter.js +14 -1
- package/dist/middleware/routes/ai.js +3 -3
- package/dist/middleware/routes/deliverable.d.ts +4 -2
- package/dist/middleware/routes/deliverable.js +16 -7
- package/dist/middleware/routes/mcp.js +11 -4
- package/dist/middleware/routes/media.js +7 -6
- package/dist/middleware/routes/{apiConnection.d.ts → oauthResource.d.ts} +1 -1
- package/dist/middleware/routes/oauthResource.js +18 -0
- package/dist/middleware/routes/project.js +5 -1
- package/dist/middleware/routes/renderFarm.js +4 -77
- package/dist/middleware/routes/storage.js +13 -27
- package/dist/middleware/routes/worker.d.ts +7 -0
- package/dist/middleware/routes/worker.js +125 -0
- package/dist/middleware/routes/workerCallback.d.ts +5 -0
- package/dist/middleware/routes/workerCallback.js +31 -0
- package/dist/models/Model.js +9 -9
- package/dist/services/aiProcessor.js +11 -11
- package/dist/services/apiTokenPermissions.js +6 -5
- package/dist/services/auth.d.ts +2 -0
- package/dist/services/auth.js +36 -10
- package/dist/services/executor.js +10 -10
- package/dist/services/jobQueue.d.ts +6 -4
- package/dist/services/jobQueue.js +28 -16
- package/dist/services/jobStore.d.ts +27 -0
- package/dist/services/jobStore.js +177 -0
- package/dist/services/mediaProcessor.js +16 -16
- package/dist/services/oauthTokens.d.ts +60 -0
- package/dist/services/oauthTokens.js +164 -0
- package/dist/services/vfxProcessor.js +12 -12
- package/dist/services/workerJobs.d.ts +73 -0
- package/dist/services/workerJobs.js +459 -0
- package/dist/services/workers.d.ts +78 -0
- package/dist/services/workers.js +179 -0
- package/dist/types/index.d.ts +64 -2
- package/dist/utils/assetCategories.d.ts +4 -0
- package/dist/utils/assetCategories.js +21 -0
- package/dist/utils/builtinAccess.d.ts +1 -1
- package/dist/utils/builtinAccess.js +4 -3
- package/dist/utils/privateAddress.d.ts +10 -0
- package/dist/utils/privateAddress.js +85 -0
- package/package.json +1 -1
- package/dist/middleware/routes/apiConnection.js +0 -100
- package/dist/middleware/routes/credential.d.ts +0 -3
- package/dist/middleware/routes/credential.js +0 -80
- package/dist/services/apiConnection.d.ts +0 -18
- package/dist/services/apiConnection.js +0 -65
- package/dist/services/credential.d.ts +0 -8
- package/dist/services/credential.js +0 -70
- package/dist/services/pipeline.d.ts +0 -2
- package/dist/services/pipeline.js +0 -292
- package/dist/utils/crypto.d.ts +0 -3
- package/dist/utils/crypto.js +0 -32
|
@@ -13,7 +13,7 @@ function getNosqlDb() {
|
|
|
13
13
|
return (0, connectionManager_1.getConnection)(ids[0]).client.db();
|
|
14
14
|
}
|
|
15
15
|
async function runTranscriptionJob(jobId, fileId, projectId, options) {
|
|
16
|
-
(0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
|
|
16
|
+
await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
|
|
17
17
|
try {
|
|
18
18
|
const db = getNosqlDb();
|
|
19
19
|
if (!db)
|
|
@@ -24,7 +24,7 @@ async function runTranscriptionJob(jobId, fileId, projectId, options) {
|
|
|
24
24
|
if (!fileDoc.contentType?.startsWith('video/') && !fileDoc.contentType?.startsWith('audio/')) {
|
|
25
25
|
throw new Error('Transcription only supports video and audio files');
|
|
26
26
|
}
|
|
27
|
-
(0, jobQueue_1.updateJob)(jobId, { stage: 'queued-provider', progress: 20 });
|
|
27
|
+
await (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-provider', progress: 20 });
|
|
28
28
|
await db.collection('media_metadata').updateOne({ assetId: fileId, projectId }, {
|
|
29
29
|
$set: {
|
|
30
30
|
assetId: fileId,
|
|
@@ -41,14 +41,14 @@ async function runTranscriptionJob(jobId, fileId, projectId, options) {
|
|
|
41
41
|
extractedAt: new Date(),
|
|
42
42
|
},
|
|
43
43
|
}, { upsert: true });
|
|
44
|
-
(0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
|
|
44
|
+
await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
|
|
45
45
|
}
|
|
46
46
|
catch (err) {
|
|
47
|
-
(0, jobQueue_1.failJob)(jobId, err.message);
|
|
47
|
+
await (0, jobQueue_1.failJob)(jobId, err.message);
|
|
48
48
|
}
|
|
49
49
|
}
|
|
50
50
|
async function runUpscaleJob(jobId, fileId, projectId, options) {
|
|
51
|
-
(0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
|
|
51
|
+
await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
|
|
52
52
|
try {
|
|
53
53
|
const db = getNosqlDb();
|
|
54
54
|
if (!db)
|
|
@@ -56,7 +56,7 @@ async function runUpscaleJob(jobId, fileId, projectId, options) {
|
|
|
56
56
|
const fileDoc = await db.collection('storage_files').findOne({ fileId, projectId });
|
|
57
57
|
if (!fileDoc)
|
|
58
58
|
throw new Error(`File ${fileId} not found`);
|
|
59
|
-
(0, jobQueue_1.updateJob)(jobId, { stage: 'queued-upscale', progress: 20 });
|
|
59
|
+
await (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-upscale', progress: 20 });
|
|
60
60
|
await db.collection('media_metadata').updateOne({ assetId: fileId, projectId }, {
|
|
61
61
|
$set: {
|
|
62
62
|
assetId: fileId,
|
|
@@ -72,14 +72,14 @@ async function runUpscaleJob(jobId, fileId, projectId, options) {
|
|
|
72
72
|
extractedAt: new Date(),
|
|
73
73
|
},
|
|
74
74
|
}, { upsert: true });
|
|
75
|
-
(0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
|
|
75
|
+
await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
|
|
76
76
|
}
|
|
77
77
|
catch (err) {
|
|
78
|
-
(0, jobQueue_1.failJob)(jobId, err.message);
|
|
78
|
+
await (0, jobQueue_1.failJob)(jobId, err.message);
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
81
|
async function runGenerateAssetJob(jobId, projectId, options) {
|
|
82
|
-
(0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'queued-generation', progress: 10 });
|
|
82
|
+
await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'queued-generation', progress: 10 });
|
|
83
83
|
try {
|
|
84
84
|
const db = getNosqlDb();
|
|
85
85
|
if (!db)
|
|
@@ -95,10 +95,10 @@ async function runGenerateAssetJob(jobId, projectId, options) {
|
|
|
95
95
|
status: 'pending',
|
|
96
96
|
requestedAt: new Date(),
|
|
97
97
|
});
|
|
98
|
-
(0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 20 });
|
|
98
|
+
await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 20 });
|
|
99
99
|
}
|
|
100
100
|
catch (err) {
|
|
101
|
-
(0, jobQueue_1.failJob)(jobId, err.message);
|
|
101
|
+
await (0, jobQueue_1.failJob)(jobId, err.message);
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
104
|
async function buildPipelineSuggestion(nodeSchema, availableNodeTypes) {
|
|
@@ -74,10 +74,10 @@ const RULES = [
|
|
|
74
74
|
// Rights.
|
|
75
75
|
rule('read', `${P}/assets/[^/]+/rights`, 'rights:read'),
|
|
76
76
|
rule(['PUT'], `${P}/assets/[^/]+/rights`, 'rights:write'),
|
|
77
|
-
// Connections (API, database, storage, distribution,
|
|
78
|
-
rule('read', `${P}/(connections|get-connection|distribution-connections|
|
|
79
|
-
rule('write', `${P}/(connections|distribution-connections|
|
|
80
|
-
rule('delete', `${P}/(connections|distribution-connections|
|
|
77
|
+
// Connections (API, database, storage, distribution, workers): the project's settings.
|
|
78
|
+
rule('read', `${P}/(connections|get-connection|distribution-connections|workers)(/.+)?`, 'project:read'),
|
|
79
|
+
rule('write', `${P}/(connections|distribution-connections|workers)(/.+)?`, 'project:update'),
|
|
80
|
+
rule('delete', `${P}/(connections|distribution-connections|workers)/.+`, 'project:update'),
|
|
81
81
|
// Production.
|
|
82
82
|
rule('read', `${P}/production/.+`, 'production:read'),
|
|
83
83
|
rule('write', `${P}/production/.+`, 'production:write'),
|
|
@@ -92,7 +92,8 @@ const RULES = [
|
|
|
92
92
|
rule(['POST'], `${P}/exports`, 'exports:write'),
|
|
93
93
|
rule(['PATCH', 'DELETE'], `${P}/exports/[^/]+`, 'exports:write'),
|
|
94
94
|
// Jobs: transcoding, rendering, AI, publishing a deliverable.
|
|
95
|
-
rule(
|
|
95
|
+
rule('read', `${P}/jobs`, 'jobs:read'),
|
|
96
|
+
rule(['POST'], `${P}/(jobs/[^/]+(/retry)?|pipeline-suggest|deliverables/[^/]+/publish)`, 'jobs:run'),
|
|
96
97
|
];
|
|
97
98
|
/** The rule for a request made with an API token, or undefined: not reachable with one. */
|
|
98
99
|
function routeRuleFor(method, path) {
|
package/dist/services/auth.d.ts
CHANGED
|
@@ -6,5 +6,7 @@ export declare const isPublicRoute: (method: string, path: string) => boolean;
|
|
|
6
6
|
export declare const scopeReachesProject: (scope: Auth["apiToken"], projectId: string) => boolean;
|
|
7
7
|
/** Whether a token has a permission (a request without a token: always). */
|
|
8
8
|
export declare const tokenAllows: (scope: Auth["apiToken"], permission: ApiTokenPermission) => boolean;
|
|
9
|
+
/** Whether an app (an OAuth request) may make this request's method on its path. */
|
|
10
|
+
export declare const oauthMayDelete: (method: string, path: string) => boolean;
|
|
9
11
|
export declare const enforceApiTokenScope: (req: Request, res: Response, next: NextFunction) => void | Response<any, Record<string, any>>;
|
|
10
12
|
export declare const authenticate: (config: StreamByConfig) => (req: Request, res: Response, next: NextFunction) => Promise<void | Response<any, Record<string, any>>>;
|
package/dist/services/auth.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.authenticate = exports.enforceApiTokenScope = exports.tokenAllows = exports.scopeReachesProject = exports.isPublicRoute = void 0;
|
|
3
|
+
exports.authenticate = exports.enforceApiTokenScope = exports.oauthMayDelete = exports.tokenAllows = exports.scopeReachesProject = exports.isPublicRoute = void 0;
|
|
4
4
|
const auth_1 = require("../utils/auth");
|
|
5
5
|
const apiTokenPermissions_1 = require("./apiTokenPermissions");
|
|
6
|
+
const oauthTokens_1 = require("./oauthTokens");
|
|
6
7
|
// The one route open without a session: a project's public export, read
|
|
7
8
|
// (GET /:projectId/export/:exportName, see routes/export). Exactly that path -- not any path
|
|
8
9
|
// that has an "export" segment somewhere.
|
|
@@ -33,13 +34,29 @@ exports.scopeReachesProject = scopeReachesProject;
|
|
|
33
34
|
/** Whether a token has a permission (a request without a token: always). */
|
|
34
35
|
const tokenAllows = (scope, permission) => !scope || scope.permissions.includes(permission);
|
|
35
36
|
exports.tokenAllows = tokenAllows;
|
|
37
|
+
// ── OAuth apps ──
|
|
38
|
+
// An app the user signed in to was approved to make and change things, never to delete them:
|
|
39
|
+
// its permissions come from its scopes, and some of those (project:update, workflow:write,
|
|
40
|
+
// exports:write) also reach routes that delete (a connection, a worker, an integration, a
|
|
41
|
+
// pipeline, an export). So an app's DELETE is refused, whatever its route's permission.
|
|
42
|
+
// The one DELETE it may make is taking a deliverable's mark off: that forgets the mark only
|
|
43
|
+
// (the asset, its versions and its files stay, and it can be marked again), and it's the
|
|
44
|
+
// counterpart of marking one, which assets:write already allows.
|
|
45
|
+
const OAUTH_DELETE_ALLOWED = new RegExp(`^/projects/[^/]+/deliverables/[^/]+/?$`);
|
|
46
|
+
/** Whether an app (an OAuth request) may make this request's method on its path. */
|
|
47
|
+
const oauthMayDelete = (method, path) => method.toUpperCase() !== 'DELETE' || OAUTH_DELETE_ALLOWED.test(path);
|
|
48
|
+
exports.oauthMayDelete = oauthMayDelete;
|
|
36
49
|
const enforceApiTokenScope = (req, res, next) => {
|
|
37
|
-
const
|
|
50
|
+
const auth = req.auth;
|
|
51
|
+
const scope = auth?.apiToken;
|
|
38
52
|
if (!scope)
|
|
39
53
|
return next();
|
|
40
54
|
// The MCP server handles a token itself: its tools call the routes here, each one checked.
|
|
41
55
|
if (req.path === '/mcp')
|
|
42
56
|
return next();
|
|
57
|
+
if (auth?.oauth && !(0, exports.oauthMayDelete)(req.method, req.path)) {
|
|
58
|
+
return res.status(403).json({ error: 'An app connected with OAuth can\'t delete' });
|
|
59
|
+
}
|
|
43
60
|
const rule = (0, apiTokenPermissions_1.routeRuleFor)(req.method, req.path);
|
|
44
61
|
if (!rule || (rule.userTokenOnly && scope.kind !== 'user')) {
|
|
45
62
|
return res.status(403).json({ error: 'Not available with an API token' });
|
|
@@ -53,25 +70,34 @@ const enforceApiTokenScope = (req, res, next) => {
|
|
|
53
70
|
next();
|
|
54
71
|
};
|
|
55
72
|
exports.enforceApiTokenScope = enforceApiTokenScope;
|
|
73
|
+
const hasBearer = (req) => /^Bearer\s+\S/i.test(req.headers.authorization ?? '');
|
|
56
74
|
const authenticate = (config) => async (req, res, next) => {
|
|
57
75
|
if ((0, exports.isPublicRoute)(req.method, req.path)) {
|
|
58
76
|
return next();
|
|
59
77
|
}
|
|
78
|
+
// A client without a valid token is told where to sign in (RFC 9728), when StreamBy is set
|
|
79
|
+
// up as an OAuth resource.
|
|
80
|
+
const unauthorized = () => {
|
|
81
|
+
if (config.oauth)
|
|
82
|
+
res.setHeader('WWW-Authenticate', (0, oauthTokens_1.wwwAuthenticate)(config.oauth, hasBearer(req)));
|
|
83
|
+
return res.status(401).json({ error: 'Unauthorized' });
|
|
84
|
+
};
|
|
60
85
|
try {
|
|
61
86
|
const auth = await config.authProvider(req);
|
|
62
|
-
if (!auth || !auth.userId || !auth.role)
|
|
63
|
-
return
|
|
64
|
-
}
|
|
87
|
+
if (!auth || !auth.userId || !auth.role)
|
|
88
|
+
return unauthorized();
|
|
65
89
|
// A project token's principal is only ever a project's token (see utils/auth).
|
|
66
|
-
if ((0, auth_1.isProjectTokenPrincipal)(auth.userId) && auth.apiToken?.kind !== 'project')
|
|
67
|
-
return
|
|
68
|
-
|
|
90
|
+
if ((0, auth_1.isProjectTokenPrincipal)(auth.userId) && auth.apiToken?.kind !== 'project')
|
|
91
|
+
return unauthorized();
|
|
92
|
+
// An app acting for the user reaches only what its scopes grant: never without them.
|
|
93
|
+
if (auth.oauth && auth.apiToken?.kind !== 'user')
|
|
94
|
+
return unauthorized();
|
|
69
95
|
req.auth = auth;
|
|
70
96
|
next();
|
|
71
97
|
}
|
|
72
98
|
catch (error) {
|
|
73
|
-
console.error('Authentication failed:', error);
|
|
74
|
-
return
|
|
99
|
+
console.error('Authentication failed:', error instanceof Error ? error.message : error);
|
|
100
|
+
return unauthorized();
|
|
75
101
|
}
|
|
76
102
|
};
|
|
77
103
|
exports.authenticate = authenticate;
|
|
@@ -216,7 +216,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
|
|
|
216
216
|
if (isPublic && JOB_NODES.has(processNode.type))
|
|
217
217
|
throw notPublic(processNode);
|
|
218
218
|
if (processNode.type === 'transcodeNode' && fileId) {
|
|
219
|
-
const job = (0, jobQueue_1.createJob)('transcode', systemUserId, projectId, nodeData);
|
|
219
|
+
const job = await (0, jobQueue_1.createJob)('transcode', systemUserId, projectId, nodeData);
|
|
220
220
|
setImmediate(() => (0, mediaProcessor_1.runTranscodeJob)(job.jobId, fileId, projectId, {
|
|
221
221
|
codec: nodeData.codec,
|
|
222
222
|
resolution: nodeData.resolution,
|
|
@@ -227,7 +227,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
|
|
|
227
227
|
payload = { ...payload, jobId: job.jobId, jobType: 'transcode' };
|
|
228
228
|
}
|
|
229
229
|
else if (processNode.type === 'captionNode' && fileId) {
|
|
230
|
-
const job = (0, jobQueue_1.createJob)('caption', systemUserId, projectId, nodeData);
|
|
230
|
+
const job = await (0, jobQueue_1.createJob)('caption', systemUserId, projectId, nodeData);
|
|
231
231
|
setImmediate(() => (0, mediaProcessor_1.runCaptionJob)(job.jobId, fileId, projectId, {
|
|
232
232
|
sourceLanguage: nodeData.sourceLanguage,
|
|
233
233
|
outputFormat: nodeData.outputFormat,
|
|
@@ -236,7 +236,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
|
|
|
236
236
|
payload = { ...payload, jobId: job.jobId, jobType: 'caption' };
|
|
237
237
|
}
|
|
238
238
|
else if (processNode.type === 'thumbnailNode' && fileId) {
|
|
239
|
-
const job = (0, jobQueue_1.createJob)('thumbnail', systemUserId, projectId, nodeData);
|
|
239
|
+
const job = await (0, jobQueue_1.createJob)('thumbnail', systemUserId, projectId, nodeData);
|
|
240
240
|
setImmediate(() => (0, mediaProcessor_1.runThumbnailJob)(job.jobId, fileId, projectId, {
|
|
241
241
|
timecode: nodeData.timecode,
|
|
242
242
|
resolution: nodeData.resolution,
|
|
@@ -256,12 +256,12 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
|
|
|
256
256
|
if ('error' in storageResolved) {
|
|
257
257
|
throw new NodeExecutionError(processNode.id, processNode.type, `Node '${processNode.id}' (${processNode.type}): ${storageResolved.error}`);
|
|
258
258
|
}
|
|
259
|
-
const job = (0, jobQueue_1.createJob)('ingest', systemUserId, projectId, nodeData);
|
|
259
|
+
const job = await (0, jobQueue_1.createJob)('ingest', systemUserId, projectId, nodeData);
|
|
260
260
|
setImmediate(() => (0, mediaProcessor_1.runIngestJob)(job.jobId, fileId, projectId));
|
|
261
261
|
payload = { ...payload, jobId: job.jobId, jobType: 'ingest' };
|
|
262
262
|
}
|
|
263
263
|
else if (processNode.type === 'renderJobNode' && fileId) {
|
|
264
|
-
const job = (0, jobQueue_1.createJob)('render', systemUserId, projectId, nodeData);
|
|
264
|
+
const job = await (0, jobQueue_1.createJob)('render', systemUserId, projectId, nodeData);
|
|
265
265
|
setImmediate(() => (0, vfxProcessor_1.runRenderJob)(job.jobId, fileId, projectId, {
|
|
266
266
|
renderer: nodeData.renderer,
|
|
267
267
|
renderFarmConnectionId: nodeData.renderFarmConnectionId,
|
|
@@ -273,7 +273,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
|
|
|
273
273
|
payload = { ...payload, jobId: job.jobId, jobType: 'render' };
|
|
274
274
|
}
|
|
275
275
|
else if (processNode.type === 'formatConvertNode' && fileId) {
|
|
276
|
-
const job = (0, jobQueue_1.createJob)('format-convert', systemUserId, projectId, nodeData);
|
|
276
|
+
const job = await (0, jobQueue_1.createJob)('format-convert', systemUserId, projectId, nodeData);
|
|
277
277
|
setImmediate(() => (0, vfxProcessor_1.runFormatConvertJob)(job.jobId, fileId, projectId, {
|
|
278
278
|
inputFormat: nodeData.inputFormat,
|
|
279
279
|
outputFormat: nodeData.outputFormat,
|
|
@@ -283,7 +283,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
|
|
|
283
283
|
payload = { ...payload, jobId: job.jobId, jobType: 'format-convert' };
|
|
284
284
|
}
|
|
285
285
|
else if (processNode.type === 'lodNode' && fileId) {
|
|
286
|
-
const job = (0, jobQueue_1.createJob)('lod', systemUserId, projectId, nodeData);
|
|
286
|
+
const job = await (0, jobQueue_1.createJob)('lod', systemUserId, projectId, nodeData);
|
|
287
287
|
setImmediate(() => (0, vfxProcessor_1.runLodJob)(job.jobId, fileId, projectId, {
|
|
288
288
|
levels: nodeData.levels,
|
|
289
289
|
reductionRatios: nodeData.reductionRatios,
|
|
@@ -300,7 +300,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
|
|
|
300
300
|
payload = { ...payload, qcReport: { checks: checkResults, overallPassed: true } };
|
|
301
301
|
}
|
|
302
302
|
else if (processNode.type === 'transcriptionNode' && fileId) {
|
|
303
|
-
const job = (0, jobQueue_1.createJob)('transcription', systemUserId, projectId, nodeData);
|
|
303
|
+
const job = await (0, jobQueue_1.createJob)('transcription', systemUserId, projectId, nodeData);
|
|
304
304
|
setImmediate(() => (0, aiProcessor_1.runTranscriptionJob)(job.jobId, fileId, projectId, {
|
|
305
305
|
model: nodeData.model,
|
|
306
306
|
sourceLanguage: nodeData.sourceLanguage,
|
|
@@ -311,7 +311,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
|
|
|
311
311
|
payload = { ...payload, jobId: job.jobId, jobType: 'transcription' };
|
|
312
312
|
}
|
|
313
313
|
else if (processNode.type === 'upscaleNode' && fileId) {
|
|
314
|
-
const job = (0, jobQueue_1.createJob)('upscale', systemUserId, projectId, nodeData);
|
|
314
|
+
const job = await (0, jobQueue_1.createJob)('upscale', systemUserId, projectId, nodeData);
|
|
315
315
|
setImmediate(() => (0, aiProcessor_1.runUpscaleJob)(job.jobId, fileId, projectId, {
|
|
316
316
|
scale: nodeData.scale,
|
|
317
317
|
model: nodeData.model,
|
|
@@ -321,7 +321,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
|
|
|
321
321
|
payload = { ...payload, jobId: job.jobId, jobType: 'upscale' };
|
|
322
322
|
}
|
|
323
323
|
else if (processNode.type === 'proceduralAssetNode') {
|
|
324
|
-
const job = (0, jobQueue_1.createJob)('generate-asset', systemUserId, projectId, nodeData);
|
|
324
|
+
const job = await (0, jobQueue_1.createJob)('generate-asset', systemUserId, projectId, nodeData);
|
|
325
325
|
setImmediate(() => (0, aiProcessor_1.runGenerateAssetJob)(job.jobId, projectId, {
|
|
326
326
|
assetType: nodeData.assetType,
|
|
327
327
|
provider: nodeData.provider,
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { JobRecord, JobType } from '../types';
|
|
2
|
-
export declare function createJob(jobType: JobType, userId: string, projectId: string, payload: Record<string, any>): JobRecord
|
|
3
|
-
export declare function getJob(jobId: string): JobRecord |
|
|
4
|
-
export declare function
|
|
5
|
-
export declare function
|
|
2
|
+
export declare function createJob(jobType: JobType, userId: string, projectId: string, payload: Record<string, any>, extra?: Partial<JobRecord>): Promise<JobRecord>;
|
|
3
|
+
export declare function getJob(jobId: string): Promise<JobRecord | null>;
|
|
4
|
+
export declare function listJobs(projectId: string, assetId?: string): Promise<JobRecord[]>;
|
|
5
|
+
export declare function notify(job: JobRecord): void;
|
|
6
|
+
export declare function updateJob(jobId: string, updates: Partial<JobRecord>): Promise<JobRecord | null>;
|
|
7
|
+
export declare function failJob(jobId: string, error: string): Promise<JobRecord | null>;
|
|
@@ -5,16 +5,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.createJob = createJob;
|
|
7
7
|
exports.getJob = getJob;
|
|
8
|
+
exports.listJobs = listJobs;
|
|
9
|
+
exports.notify = notify;
|
|
8
10
|
exports.updateJob = updateJob;
|
|
9
11
|
exports.failJob = failJob;
|
|
10
12
|
const crypto_1 = __importDefault(require("crypto"));
|
|
11
13
|
const wsHub_1 = require("./wsHub");
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
const jobStore_1 = require("./jobStore");
|
|
15
|
+
// Jobs, kept in the main database (see jobStore): they survive restarts and every instance
|
|
16
|
+
// sees them. Each change is sent to whoever started the job (a `jobEvent`).
|
|
17
|
+
async function createJob(jobType, userId, projectId, payload, extra = {}) {
|
|
15
18
|
const now = new Date();
|
|
16
19
|
const job = {
|
|
17
|
-
jobId,
|
|
20
|
+
jobId: crypto_1.default.randomUUID(),
|
|
18
21
|
jobType,
|
|
19
22
|
userId,
|
|
20
23
|
projectId,
|
|
@@ -24,31 +27,40 @@ function createJob(jobType, userId, projectId, payload) {
|
|
|
24
27
|
payload,
|
|
25
28
|
createdAt: now,
|
|
26
29
|
updatedAt: now,
|
|
30
|
+
...extra,
|
|
27
31
|
};
|
|
28
|
-
|
|
29
|
-
|
|
32
|
+
const saved = await (0, jobStore_1.jobStore)().insert(job);
|
|
33
|
+
notify(saved);
|
|
34
|
+
return saved;
|
|
35
|
+
}
|
|
36
|
+
async function getJob(jobId) {
|
|
37
|
+
return (0, jobStore_1.jobStore)().get(jobId);
|
|
30
38
|
}
|
|
31
|
-
function
|
|
32
|
-
return
|
|
39
|
+
async function listJobs(projectId, assetId) {
|
|
40
|
+
return (0, jobStore_1.jobStore)().list({ projectId, assetId });
|
|
33
41
|
}
|
|
34
|
-
function
|
|
35
|
-
const job = jobs.get(jobId);
|
|
36
|
-
if (!job)
|
|
37
|
-
return;
|
|
38
|
-
Object.assign(job, updates, { updatedAt: new Date() });
|
|
42
|
+
function notify(job) {
|
|
39
43
|
(0, wsHub_1.emitToUser)(job.userId, {
|
|
40
44
|
type: 'jobEvent',
|
|
41
45
|
data: {
|
|
42
46
|
jobId: job.jobId,
|
|
43
47
|
jobType: job.jobType,
|
|
48
|
+
projectId: job.projectId,
|
|
44
49
|
assetId: job.assetId,
|
|
50
|
+
status: job.status,
|
|
45
51
|
stage: job.stage,
|
|
46
52
|
progress: job.progress,
|
|
47
|
-
message:
|
|
53
|
+
message: job.stage,
|
|
48
54
|
error: job.error,
|
|
49
55
|
},
|
|
50
56
|
});
|
|
51
57
|
}
|
|
52
|
-
function
|
|
53
|
-
|
|
58
|
+
async function updateJob(jobId, updates) {
|
|
59
|
+
const job = await (0, jobStore_1.jobStore)().update(jobId, { ...updates, updatedAt: new Date() });
|
|
60
|
+
if (job)
|
|
61
|
+
notify(job);
|
|
62
|
+
return job;
|
|
63
|
+
}
|
|
64
|
+
async function failJob(jobId, error) {
|
|
65
|
+
return updateJob(jobId, { status: 'failed', stage: 'failed', error });
|
|
54
66
|
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { JobRecord, StreamByConfig } from '../types';
|
|
2
|
+
export interface JobStore {
|
|
3
|
+
insert(job: JobRecord): Promise<JobRecord>;
|
|
4
|
+
get(jobId: string): Promise<JobRecord | null>;
|
|
5
|
+
list(filter: {
|
|
6
|
+
projectId: string;
|
|
7
|
+
assetId?: string;
|
|
8
|
+
limit?: number;
|
|
9
|
+
}): Promise<JobRecord[]>;
|
|
10
|
+
update(jobId: string, patch: Partial<JobRecord>): Promise<JobRecord | null>;
|
|
11
|
+
/**
|
|
12
|
+
* Changes a job only if it's still in one of `statuses` (any, if none), and, if given, its
|
|
13
|
+
* lease ended by `leaseEndedBy` and its callback hash is `callbackHash`.
|
|
14
|
+
*/
|
|
15
|
+
claim(jobId: string, statuses: JobRecord['status'][], patch: Partial<JobRecord>, only?: ClaimConditions): Promise<JobRecord | null>;
|
|
16
|
+
/** Worker jobs to look at now: queued and due, or dispatched with their lease ended. */
|
|
17
|
+
due(now: Date, limit: number): Promise<JobRecord[]>;
|
|
18
|
+
/** A deleted project's jobs. */
|
|
19
|
+
deleteProject(projectId: string): Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
export interface ClaimConditions {
|
|
22
|
+
leaseEndedBy?: Date;
|
|
23
|
+
callbackHash?: string;
|
|
24
|
+
}
|
|
25
|
+
/** The router's main database, where jobs are kept. */
|
|
26
|
+
export declare const setJobStoreDatabase: (config: StreamByConfig) => void;
|
|
27
|
+
export declare function jobStore(): JobStore;
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.setJobStoreDatabase = void 0;
|
|
4
|
+
exports.jobStore = jobStore;
|
|
5
|
+
const connectionManager_1 = require("../adapters/database/connectionManager");
|
|
6
|
+
const JSON_FIELDS = new Set(['payload', 'report', 'result', 'worker', 'profile', 'upload']);
|
|
7
|
+
// ── PostgreSQL ──
|
|
8
|
+
const sqlStore = (pool) => {
|
|
9
|
+
const table = 'streamby.jobs';
|
|
10
|
+
const fromRow = (row) => {
|
|
11
|
+
if (!row)
|
|
12
|
+
return null;
|
|
13
|
+
const { id, ...job } = row;
|
|
14
|
+
return { ...job, jobId: id };
|
|
15
|
+
};
|
|
16
|
+
const toColumns = (patch) => {
|
|
17
|
+
const entries = Object.entries(patch).filter(([key, value]) => key !== 'jobId' && value !== undefined);
|
|
18
|
+
return {
|
|
19
|
+
names: entries.map(([key]) => `"${key}"`),
|
|
20
|
+
values: entries.map(([key, value]) => (JSON_FIELDS.has(key) && value !== null ? JSON.stringify(value) : value)),
|
|
21
|
+
};
|
|
22
|
+
};
|
|
23
|
+
return {
|
|
24
|
+
async insert(job) {
|
|
25
|
+
const { names, values } = toColumns(job);
|
|
26
|
+
const placeholders = values.map((_, i) => `$${i + 2}`);
|
|
27
|
+
const { rows } = await pool.query(`INSERT INTO ${table} (id, ${names.join(', ')}) VALUES ($1, ${placeholders.join(', ')}) RETURNING *`, [job.jobId, ...values]);
|
|
28
|
+
return fromRow(rows[0]);
|
|
29
|
+
},
|
|
30
|
+
async get(jobId) {
|
|
31
|
+
const { rows } = await pool.query(`SELECT * FROM ${table} WHERE id = $1`, [jobId]);
|
|
32
|
+
return fromRow(rows[0]);
|
|
33
|
+
},
|
|
34
|
+
async list({ projectId, assetId, limit = 100 }) {
|
|
35
|
+
const { rows } = await pool.query(`SELECT * FROM ${table} WHERE "projectId" = $1 ${assetId ? 'AND "assetId" = $3' : ''} ORDER BY "createdAt" DESC LIMIT $2`, assetId ? [projectId, limit, assetId] : [projectId, limit]);
|
|
36
|
+
return rows.map(row => fromRow(row));
|
|
37
|
+
},
|
|
38
|
+
async update(jobId, patch) {
|
|
39
|
+
return this.claim(jobId, [], patch);
|
|
40
|
+
},
|
|
41
|
+
async claim(jobId, statuses, patch, only = {}) {
|
|
42
|
+
const { names, values } = toColumns(patch);
|
|
43
|
+
const sets = names.map((name, i) => `${name} = $${i + 2}`).join(', ');
|
|
44
|
+
const params = [jobId, ...values];
|
|
45
|
+
let where = 'id = $1';
|
|
46
|
+
if (statuses.length) {
|
|
47
|
+
params.push(statuses);
|
|
48
|
+
where += ` AND status = ANY($${params.length})`;
|
|
49
|
+
}
|
|
50
|
+
if (only.leaseEndedBy) {
|
|
51
|
+
params.push(only.leaseEndedBy);
|
|
52
|
+
where += ` AND ("leaseUntil" IS NULL OR "leaseUntil" <= $${params.length})`;
|
|
53
|
+
}
|
|
54
|
+
if (only.callbackHash) {
|
|
55
|
+
params.push(only.callbackHash);
|
|
56
|
+
where += ` AND "callbackHash" = $${params.length}`;
|
|
57
|
+
}
|
|
58
|
+
const { rows } = await pool.query(`UPDATE ${table} SET ${sets} WHERE ${where} RETURNING *`, params);
|
|
59
|
+
return fromRow(rows[0]);
|
|
60
|
+
},
|
|
61
|
+
async due(now, limit) {
|
|
62
|
+
const { rows } = await pool.query(`SELECT * FROM ${table}
|
|
63
|
+
WHERE worker IS NOT NULL AND (
|
|
64
|
+
(status = 'pending' AND ("nextAttemptAt" IS NULL OR "nextAttemptAt" <= $1))
|
|
65
|
+
OR (status = 'processing' AND "leaseUntil" <= $1))
|
|
66
|
+
ORDER BY "createdAt" LIMIT $2`, [now, limit]);
|
|
67
|
+
return rows.map(row => fromRow(row));
|
|
68
|
+
},
|
|
69
|
+
async deleteProject(projectId) {
|
|
70
|
+
await pool.query(`DELETE FROM ${table} WHERE "projectId" = $1`, [projectId]);
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
};
|
|
74
|
+
// ── MongoDB ──
|
|
75
|
+
const mongoStore = (client) => {
|
|
76
|
+
const col = () => client.db().collection('jobs');
|
|
77
|
+
const fromDoc = (doc) => {
|
|
78
|
+
if (!doc)
|
|
79
|
+
return null;
|
|
80
|
+
const { _id, ...job } = doc;
|
|
81
|
+
return job;
|
|
82
|
+
};
|
|
83
|
+
const clean = (patch) => Object.fromEntries(Object.entries(patch).filter(([, value]) => value !== undefined));
|
|
84
|
+
return {
|
|
85
|
+
async insert(job) {
|
|
86
|
+
await col().insertOne({ ...clean(job) });
|
|
87
|
+
return job;
|
|
88
|
+
},
|
|
89
|
+
async get(jobId) {
|
|
90
|
+
return fromDoc(await col().findOne({ jobId }));
|
|
91
|
+
},
|
|
92
|
+
async list({ projectId, assetId, limit = 100 }) {
|
|
93
|
+
const docs = await col().find({ projectId, ...(assetId ? { assetId } : {}) }).sort({ createdAt: -1 }).limit(limit).toArray();
|
|
94
|
+
return docs.map(doc => fromDoc(doc));
|
|
95
|
+
},
|
|
96
|
+
async update(jobId, patch) {
|
|
97
|
+
return this.claim(jobId, [], patch);
|
|
98
|
+
},
|
|
99
|
+
async claim(jobId, statuses, patch, only = {}) {
|
|
100
|
+
const filter = { jobId };
|
|
101
|
+
if (statuses.length)
|
|
102
|
+
filter.status = { $in: statuses };
|
|
103
|
+
if (only.leaseEndedBy)
|
|
104
|
+
filter.$or = [{ leaseUntil: null }, { leaseUntil: { $exists: false } }, { leaseUntil: { $lte: only.leaseEndedBy } }];
|
|
105
|
+
if (only.callbackHash)
|
|
106
|
+
filter.callbackHash = only.callbackHash;
|
|
107
|
+
return fromDoc(await col().findOneAndUpdate(filter, { $set: clean(patch) }, { returnDocument: 'after' }));
|
|
108
|
+
},
|
|
109
|
+
async due(now, limit) {
|
|
110
|
+
const docs = await col().find({
|
|
111
|
+
worker: { $ne: null },
|
|
112
|
+
$or: [
|
|
113
|
+
{ status: 'pending', $or: [{ nextAttemptAt: null }, { nextAttemptAt: { $lte: now } }] },
|
|
114
|
+
{ status: 'processing', leaseUntil: { $lte: now } },
|
|
115
|
+
],
|
|
116
|
+
}).sort({ createdAt: 1 }).limit(limit).toArray();
|
|
117
|
+
return docs.map(doc => fromDoc(doc));
|
|
118
|
+
},
|
|
119
|
+
async deleteProject(projectId) {
|
|
120
|
+
await col().deleteMany({ projectId });
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
};
|
|
124
|
+
// ── In memory (no main database) ──
|
|
125
|
+
const memoryStore = () => {
|
|
126
|
+
const jobs = new Map();
|
|
127
|
+
const copy = (job) => (job ? { ...job } : null);
|
|
128
|
+
return {
|
|
129
|
+
async insert(job) { jobs.set(job.jobId, { ...job }); return { ...job }; },
|
|
130
|
+
async get(jobId) { return copy(jobs.get(jobId)); },
|
|
131
|
+
async list({ projectId, assetId, limit = 100 }) {
|
|
132
|
+
return [...jobs.values()]
|
|
133
|
+
.filter(job => job.projectId === projectId && (!assetId || job.assetId === assetId))
|
|
134
|
+
.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime())
|
|
135
|
+
.slice(0, limit)
|
|
136
|
+
.map(job => ({ ...job }));
|
|
137
|
+
},
|
|
138
|
+
async update(jobId, patch) { return this.claim(jobId, [], patch); },
|
|
139
|
+
async claim(jobId, statuses, patch, only = {}) {
|
|
140
|
+
const job = jobs.get(jobId);
|
|
141
|
+
if (!job || (statuses.length && !statuses.includes(job.status)))
|
|
142
|
+
return null;
|
|
143
|
+
if (only.leaseEndedBy && job.leaseUntil && new Date(job.leaseUntil) > only.leaseEndedBy)
|
|
144
|
+
return null;
|
|
145
|
+
if (only.callbackHash && job.callbackHash !== only.callbackHash)
|
|
146
|
+
return null;
|
|
147
|
+
Object.assign(job, Object.fromEntries(Object.entries(patch).filter(([, value]) => value !== undefined)));
|
|
148
|
+
return { ...job };
|
|
149
|
+
},
|
|
150
|
+
async due(now, limit) {
|
|
151
|
+
return [...jobs.values()]
|
|
152
|
+
.filter(job => job.worker && ((job.status === 'pending' && (!job.nextAttemptAt || new Date(job.nextAttemptAt) <= now))
|
|
153
|
+
|| (job.status === 'processing' && job.leaseUntil && new Date(job.leaseUntil) <= now)))
|
|
154
|
+
.slice(0, limit)
|
|
155
|
+
.map(job => ({ ...job }));
|
|
156
|
+
},
|
|
157
|
+
async deleteProject(projectId) {
|
|
158
|
+
for (const [id, job] of jobs)
|
|
159
|
+
if (job.projectId === projectId)
|
|
160
|
+
jobs.delete(id);
|
|
161
|
+
},
|
|
162
|
+
};
|
|
163
|
+
};
|
|
164
|
+
let mainDbId;
|
|
165
|
+
let memory;
|
|
166
|
+
/** The router's main database, where jobs are kept. */
|
|
167
|
+
const setJobStoreDatabase = (config) => {
|
|
168
|
+
mainDbId = config.databases?.find(db => db.main)?.id;
|
|
169
|
+
};
|
|
170
|
+
exports.setJobStoreDatabase = setJobStoreDatabase;
|
|
171
|
+
function jobStore() {
|
|
172
|
+
if (mainDbId && (0, connectionManager_1.getConnectedIds)().includes(mainDbId)) {
|
|
173
|
+
const entry = (0, connectionManager_1.getConnection)(mainDbId);
|
|
174
|
+
return entry.type === 'sql' ? sqlStore(entry.client) : mongoStore(entry.client);
|
|
175
|
+
}
|
|
176
|
+
return (memory ?? (memory = memoryStore()));
|
|
177
|
+
}
|