streamby-core 0.39.0 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CHANGELOG.md +84 -0
  2. package/LICENSE +661 -661
  3. package/README.md +785 -573
  4. package/dist/adapters/database/connectionManager.js +182 -124
  5. package/dist/adapters/database/sql.js +1 -1
  6. package/dist/index.d.ts +3 -1
  7. package/dist/index.js +9 -1
  8. package/dist/mcp/audit.d.ts +5 -2
  9. package/dist/mcp/audit.js +8 -0
  10. package/dist/mcp/server.js +3 -3
  11. package/dist/mcp/tools.js +41 -0
  12. package/dist/middleware/createRouter.js +14 -1
  13. package/dist/middleware/routes/ai.js +3 -3
  14. package/dist/middleware/routes/deliverable.d.ts +4 -2
  15. package/dist/middleware/routes/deliverable.js +16 -7
  16. package/dist/middleware/routes/mcp.js +11 -4
  17. package/dist/middleware/routes/media.js +7 -6
  18. package/dist/middleware/routes/{apiConnection.d.ts → oauthResource.d.ts} +1 -1
  19. package/dist/middleware/routes/oauthResource.js +18 -0
  20. package/dist/middleware/routes/project.js +5 -1
  21. package/dist/middleware/routes/renderFarm.js +4 -77
  22. package/dist/middleware/routes/storage.js +13 -27
  23. package/dist/middleware/routes/worker.d.ts +7 -0
  24. package/dist/middleware/routes/worker.js +125 -0
  25. package/dist/middleware/routes/workerCallback.d.ts +5 -0
  26. package/dist/middleware/routes/workerCallback.js +31 -0
  27. package/dist/models/Model.js +9 -9
  28. package/dist/services/aiProcessor.js +11 -11
  29. package/dist/services/apiTokenPermissions.js +6 -5
  30. package/dist/services/auth.d.ts +2 -0
  31. package/dist/services/auth.js +36 -10
  32. package/dist/services/executor.js +10 -10
  33. package/dist/services/jobQueue.d.ts +6 -4
  34. package/dist/services/jobQueue.js +28 -16
  35. package/dist/services/jobStore.d.ts +27 -0
  36. package/dist/services/jobStore.js +177 -0
  37. package/dist/services/mediaProcessor.js +16 -16
  38. package/dist/services/oauthTokens.d.ts +60 -0
  39. package/dist/services/oauthTokens.js +164 -0
  40. package/dist/services/vfxProcessor.js +12 -12
  41. package/dist/services/workerJobs.d.ts +73 -0
  42. package/dist/services/workerJobs.js +459 -0
  43. package/dist/services/workers.d.ts +78 -0
  44. package/dist/services/workers.js +179 -0
  45. package/dist/types/index.d.ts +64 -2
  46. package/dist/utils/assetCategories.d.ts +4 -0
  47. package/dist/utils/assetCategories.js +21 -0
  48. package/dist/utils/builtinAccess.d.ts +1 -1
  49. package/dist/utils/builtinAccess.js +4 -3
  50. package/dist/utils/privateAddress.d.ts +10 -0
  51. package/dist/utils/privateAddress.js +85 -0
  52. package/package.json +1 -1
  53. package/dist/middleware/routes/apiConnection.js +0 -100
  54. package/dist/middleware/routes/credential.d.ts +0 -3
  55. package/dist/middleware/routes/credential.js +0 -80
  56. package/dist/services/apiConnection.d.ts +0 -18
  57. package/dist/services/apiConnection.js +0 -65
  58. package/dist/services/credential.d.ts +0 -8
  59. package/dist/services/credential.js +0 -70
  60. package/dist/services/pipeline.d.ts +0 -2
  61. package/dist/services/pipeline.js +0 -292
  62. package/dist/utils/crypto.d.ts +0 -3
  63. package/dist/utils/crypto.js +0 -32
@@ -13,7 +13,7 @@ function getNosqlDb() {
13
13
  return (0, connectionManager_1.getConnection)(ids[0]).client.db();
14
14
  }
15
15
  async function runTranscriptionJob(jobId, fileId, projectId, options) {
16
- (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
16
+ await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
17
17
  try {
18
18
  const db = getNosqlDb();
19
19
  if (!db)
@@ -24,7 +24,7 @@ async function runTranscriptionJob(jobId, fileId, projectId, options) {
24
24
  if (!fileDoc.contentType?.startsWith('video/') && !fileDoc.contentType?.startsWith('audio/')) {
25
25
  throw new Error('Transcription only supports video and audio files');
26
26
  }
27
- (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-provider', progress: 20 });
27
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-provider', progress: 20 });
28
28
  await db.collection('media_metadata').updateOne({ assetId: fileId, projectId }, {
29
29
  $set: {
30
30
  assetId: fileId,
@@ -41,14 +41,14 @@ async function runTranscriptionJob(jobId, fileId, projectId, options) {
41
41
  extractedAt: new Date(),
42
42
  },
43
43
  }, { upsert: true });
44
- (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
44
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
45
45
  }
46
46
  catch (err) {
47
- (0, jobQueue_1.failJob)(jobId, err.message);
47
+ await (0, jobQueue_1.failJob)(jobId, err.message);
48
48
  }
49
49
  }
50
50
  async function runUpscaleJob(jobId, fileId, projectId, options) {
51
- (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
51
+ await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
52
52
  try {
53
53
  const db = getNosqlDb();
54
54
  if (!db)
@@ -56,7 +56,7 @@ async function runUpscaleJob(jobId, fileId, projectId, options) {
56
56
  const fileDoc = await db.collection('storage_files').findOne({ fileId, projectId });
57
57
  if (!fileDoc)
58
58
  throw new Error(`File ${fileId} not found`);
59
- (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-upscale', progress: 20 });
59
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-upscale', progress: 20 });
60
60
  await db.collection('media_metadata').updateOne({ assetId: fileId, projectId }, {
61
61
  $set: {
62
62
  assetId: fileId,
@@ -72,14 +72,14 @@ async function runUpscaleJob(jobId, fileId, projectId, options) {
72
72
  extractedAt: new Date(),
73
73
  },
74
74
  }, { upsert: true });
75
- (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
75
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
76
76
  }
77
77
  catch (err) {
78
- (0, jobQueue_1.failJob)(jobId, err.message);
78
+ await (0, jobQueue_1.failJob)(jobId, err.message);
79
79
  }
80
80
  }
81
81
  async function runGenerateAssetJob(jobId, projectId, options) {
82
- (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'queued-generation', progress: 10 });
82
+ await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'queued-generation', progress: 10 });
83
83
  try {
84
84
  const db = getNosqlDb();
85
85
  if (!db)
@@ -95,10 +95,10 @@ async function runGenerateAssetJob(jobId, projectId, options) {
95
95
  status: 'pending',
96
96
  requestedAt: new Date(),
97
97
  });
98
- (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 20 });
98
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 20 });
99
99
  }
100
100
  catch (err) {
101
- (0, jobQueue_1.failJob)(jobId, err.message);
101
+ await (0, jobQueue_1.failJob)(jobId, err.message);
102
102
  }
103
103
  }
104
104
  async function buildPipelineSuggestion(nodeSchema, availableNodeTypes) {
@@ -74,10 +74,10 @@ const RULES = [
74
74
  // Rights.
75
75
  rule('read', `${P}/assets/[^/]+/rights`, 'rights:read'),
76
76
  rule(['PUT'], `${P}/assets/[^/]+/rights`, 'rights:write'),
77
- // Connections (API, database, storage, distribution, render farm): the project's settings.
78
- rule('read', `${P}/(connections|get-connection|distribution-connections|render-farm-connections)(/.+)?`, 'project:read'),
79
- rule('write', `${P}/(connections|distribution-connections|render-farm-connections)(/.+)?`, 'project:update'),
80
- rule('delete', `${P}/(connections|distribution-connections|render-farm-connections)/.+`, 'project:update'),
77
+ // Connections (API, database, storage, distribution, workers): the project's settings.
78
+ rule('read', `${P}/(connections|get-connection|distribution-connections|workers)(/.+)?`, 'project:read'),
79
+ rule('write', `${P}/(connections|distribution-connections|workers)(/.+)?`, 'project:update'),
80
+ rule('delete', `${P}/(connections|distribution-connections|workers)/.+`, 'project:update'),
81
81
  // Production.
82
82
  rule('read', `${P}/production/.+`, 'production:read'),
83
83
  rule('write', `${P}/production/.+`, 'production:write'),
@@ -92,7 +92,8 @@ const RULES = [
92
92
  rule(['POST'], `${P}/exports`, 'exports:write'),
93
93
  rule(['PATCH', 'DELETE'], `${P}/exports/[^/]+`, 'exports:write'),
94
94
  // Jobs: transcoding, rendering, AI, publishing a deliverable.
95
- rule(['POST'], `${P}/(jobs/[^/]+|pipeline-suggest|deliverables/[^/]+/publish)`, 'jobs:run'),
95
+ rule('read', `${P}/jobs`, 'jobs:read'),
96
+ rule(['POST'], `${P}/(jobs/[^/]+(/retry)?|pipeline-suggest|deliverables/[^/]+/publish)`, 'jobs:run'),
96
97
  ];
97
98
  /** The rule for a request made with an API token, or undefined: not reachable with one. */
98
99
  function routeRuleFor(method, path) {
@@ -6,5 +6,7 @@ export declare const isPublicRoute: (method: string, path: string) => boolean;
6
6
  export declare const scopeReachesProject: (scope: Auth["apiToken"], projectId: string) => boolean;
7
7
  /** Whether a token has a permission (a request without a token: always). */
8
8
  export declare const tokenAllows: (scope: Auth["apiToken"], permission: ApiTokenPermission) => boolean;
9
+ /** Whether an app (an OAuth request) may make this request's method on its path. */
10
+ export declare const oauthMayDelete: (method: string, path: string) => boolean;
9
11
  export declare const enforceApiTokenScope: (req: Request, res: Response, next: NextFunction) => void | Response<any, Record<string, any>>;
10
12
  export declare const authenticate: (config: StreamByConfig) => (req: Request, res: Response, next: NextFunction) => Promise<void | Response<any, Record<string, any>>>;
@@ -1,8 +1,9 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.authenticate = exports.enforceApiTokenScope = exports.tokenAllows = exports.scopeReachesProject = exports.isPublicRoute = void 0;
3
+ exports.authenticate = exports.enforceApiTokenScope = exports.oauthMayDelete = exports.tokenAllows = exports.scopeReachesProject = exports.isPublicRoute = void 0;
4
4
  const auth_1 = require("../utils/auth");
5
5
  const apiTokenPermissions_1 = require("./apiTokenPermissions");
6
+ const oauthTokens_1 = require("./oauthTokens");
6
7
  // The one route open without a session: a project's public export, read
7
8
  // (GET /:projectId/export/:exportName, see routes/export). Exactly that path -- not any path
8
9
  // that has an "export" segment somewhere.
@@ -33,13 +34,29 @@ exports.scopeReachesProject = scopeReachesProject;
33
34
  /** Whether a token has a permission (a request without a token: always). */
34
35
  const tokenAllows = (scope, permission) => !scope || scope.permissions.includes(permission);
35
36
  exports.tokenAllows = tokenAllows;
37
+ // ── OAuth apps ──
38
+ // An app the user signed in to was approved to make and change things, never to delete them:
39
+ // its permissions come from its scopes, and some of those (project:update, workflow:write,
40
+ // exports:write) also reach routes that delete (a connection, a worker, an integration, a
41
+ // pipeline, an export). So an app's DELETE is refused, whatever its route's permission.
42
+ // The one DELETE it may make is taking a deliverable's mark off: that forgets the mark only
43
+ // (the asset, its versions and its files stay, and it can be marked again), and it's the
44
+ // counterpart of marking one, which assets:write already allows.
45
+ const OAUTH_DELETE_ALLOWED = new RegExp(`^/projects/[^/]+/deliverables/[^/]+/?$`);
46
+ /** Whether an app (an OAuth request) may make this request's method on its path. */
47
+ const oauthMayDelete = (method, path) => method.toUpperCase() !== 'DELETE' || OAUTH_DELETE_ALLOWED.test(path);
48
+ exports.oauthMayDelete = oauthMayDelete;
36
49
  const enforceApiTokenScope = (req, res, next) => {
37
- const scope = req.auth?.apiToken;
50
+ const auth = req.auth;
51
+ const scope = auth?.apiToken;
38
52
  if (!scope)
39
53
  return next();
40
54
  // The MCP server handles a token itself: its tools call the routes here, each one checked.
41
55
  if (req.path === '/mcp')
42
56
  return next();
57
+ if (auth?.oauth && !(0, exports.oauthMayDelete)(req.method, req.path)) {
58
+ return res.status(403).json({ error: 'An app connected with OAuth can\'t delete' });
59
+ }
43
60
  const rule = (0, apiTokenPermissions_1.routeRuleFor)(req.method, req.path);
44
61
  if (!rule || (rule.userTokenOnly && scope.kind !== 'user')) {
45
62
  return res.status(403).json({ error: 'Not available with an API token' });
@@ -53,25 +70,34 @@ const enforceApiTokenScope = (req, res, next) => {
53
70
  next();
54
71
  };
55
72
  exports.enforceApiTokenScope = enforceApiTokenScope;
73
+ const hasBearer = (req) => /^Bearer\s+\S/i.test(req.headers.authorization ?? '');
56
74
  const authenticate = (config) => async (req, res, next) => {
57
75
  if ((0, exports.isPublicRoute)(req.method, req.path)) {
58
76
  return next();
59
77
  }
78
+ // A client without a valid token is told where to sign in (RFC 9728), when StreamBy is set
79
+ // up as an OAuth resource.
80
+ const unauthorized = () => {
81
+ if (config.oauth)
82
+ res.setHeader('WWW-Authenticate', (0, oauthTokens_1.wwwAuthenticate)(config.oauth, hasBearer(req)));
83
+ return res.status(401).json({ error: 'Unauthorized' });
84
+ };
60
85
  try {
61
86
  const auth = await config.authProvider(req);
62
- if (!auth || !auth.userId || !auth.role) {
63
- return res.status(401).json({ error: 'Unauthorized' });
64
- }
87
+ if (!auth || !auth.userId || !auth.role)
88
+ return unauthorized();
65
89
  // A project token's principal is only ever a project's token (see utils/auth).
66
- if ((0, auth_1.isProjectTokenPrincipal)(auth.userId) && auth.apiToken?.kind !== 'project') {
67
- return res.status(401).json({ error: 'Unauthorized' });
68
- }
90
+ if ((0, auth_1.isProjectTokenPrincipal)(auth.userId) && auth.apiToken?.kind !== 'project')
91
+ return unauthorized();
92
+ // An app acting for the user reaches only what its scopes grant: never without them.
93
+ if (auth.oauth && auth.apiToken?.kind !== 'user')
94
+ return unauthorized();
69
95
  req.auth = auth;
70
96
  next();
71
97
  }
72
98
  catch (error) {
73
- console.error('Authentication failed:', error);
74
- return res.status(401).json({ error: 'Unauthorized' });
99
+ console.error('Authentication failed:', error instanceof Error ? error.message : error);
100
+ return unauthorized();
75
101
  }
76
102
  };
77
103
  exports.authenticate = authenticate;
@@ -216,7 +216,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
216
216
  if (isPublic && JOB_NODES.has(processNode.type))
217
217
  throw notPublic(processNode);
218
218
  if (processNode.type === 'transcodeNode' && fileId) {
219
- const job = (0, jobQueue_1.createJob)('transcode', systemUserId, projectId, nodeData);
219
+ const job = await (0, jobQueue_1.createJob)('transcode', systemUserId, projectId, nodeData);
220
220
  setImmediate(() => (0, mediaProcessor_1.runTranscodeJob)(job.jobId, fileId, projectId, {
221
221
  codec: nodeData.codec,
222
222
  resolution: nodeData.resolution,
@@ -227,7 +227,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
227
227
  payload = { ...payload, jobId: job.jobId, jobType: 'transcode' };
228
228
  }
229
229
  else if (processNode.type === 'captionNode' && fileId) {
230
- const job = (0, jobQueue_1.createJob)('caption', systemUserId, projectId, nodeData);
230
+ const job = await (0, jobQueue_1.createJob)('caption', systemUserId, projectId, nodeData);
231
231
  setImmediate(() => (0, mediaProcessor_1.runCaptionJob)(job.jobId, fileId, projectId, {
232
232
  sourceLanguage: nodeData.sourceLanguage,
233
233
  outputFormat: nodeData.outputFormat,
@@ -236,7 +236,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
236
236
  payload = { ...payload, jobId: job.jobId, jobType: 'caption' };
237
237
  }
238
238
  else if (processNode.type === 'thumbnailNode' && fileId) {
239
- const job = (0, jobQueue_1.createJob)('thumbnail', systemUserId, projectId, nodeData);
239
+ const job = await (0, jobQueue_1.createJob)('thumbnail', systemUserId, projectId, nodeData);
240
240
  setImmediate(() => (0, mediaProcessor_1.runThumbnailJob)(job.jobId, fileId, projectId, {
241
241
  timecode: nodeData.timecode,
242
242
  resolution: nodeData.resolution,
@@ -256,12 +256,12 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
256
256
  if ('error' in storageResolved) {
257
257
  throw new NodeExecutionError(processNode.id, processNode.type, `Node '${processNode.id}' (${processNode.type}): ${storageResolved.error}`);
258
258
  }
259
- const job = (0, jobQueue_1.createJob)('ingest', systemUserId, projectId, nodeData);
259
+ const job = await (0, jobQueue_1.createJob)('ingest', systemUserId, projectId, nodeData);
260
260
  setImmediate(() => (0, mediaProcessor_1.runIngestJob)(job.jobId, fileId, projectId));
261
261
  payload = { ...payload, jobId: job.jobId, jobType: 'ingest' };
262
262
  }
263
263
  else if (processNode.type === 'renderJobNode' && fileId) {
264
- const job = (0, jobQueue_1.createJob)('render', systemUserId, projectId, nodeData);
264
+ const job = await (0, jobQueue_1.createJob)('render', systemUserId, projectId, nodeData);
265
265
  setImmediate(() => (0, vfxProcessor_1.runRenderJob)(job.jobId, fileId, projectId, {
266
266
  renderer: nodeData.renderer,
267
267
  renderFarmConnectionId: nodeData.renderFarmConnectionId,
@@ -273,7 +273,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
273
273
  payload = { ...payload, jobId: job.jobId, jobType: 'render' };
274
274
  }
275
275
  else if (processNode.type === 'formatConvertNode' && fileId) {
276
- const job = (0, jobQueue_1.createJob)('format-convert', systemUserId, projectId, nodeData);
276
+ const job = await (0, jobQueue_1.createJob)('format-convert', systemUserId, projectId, nodeData);
277
277
  setImmediate(() => (0, vfxProcessor_1.runFormatConvertJob)(job.jobId, fileId, projectId, {
278
278
  inputFormat: nodeData.inputFormat,
279
279
  outputFormat: nodeData.outputFormat,
@@ -283,7 +283,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
283
283
  payload = { ...payload, jobId: job.jobId, jobType: 'format-convert' };
284
284
  }
285
285
  else if (processNode.type === 'lodNode' && fileId) {
286
- const job = (0, jobQueue_1.createJob)('lod', systemUserId, projectId, nodeData);
286
+ const job = await (0, jobQueue_1.createJob)('lod', systemUserId, projectId, nodeData);
287
287
  setImmediate(() => (0, vfxProcessor_1.runLodJob)(job.jobId, fileId, projectId, {
288
288
  levels: nodeData.levels,
289
289
  reductionRatios: nodeData.reductionRatios,
@@ -300,7 +300,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
300
300
  payload = { ...payload, qcReport: { checks: checkResults, overallPassed: true } };
301
301
  }
302
302
  else if (processNode.type === 'transcriptionNode' && fileId) {
303
- const job = (0, jobQueue_1.createJob)('transcription', systemUserId, projectId, nodeData);
303
+ const job = await (0, jobQueue_1.createJob)('transcription', systemUserId, projectId, nodeData);
304
304
  setImmediate(() => (0, aiProcessor_1.runTranscriptionJob)(job.jobId, fileId, projectId, {
305
305
  model: nodeData.model,
306
306
  sourceLanguage: nodeData.sourceLanguage,
@@ -311,7 +311,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
311
311
  payload = { ...payload, jobId: job.jobId, jobType: 'transcription' };
312
312
  }
313
313
  else if (processNode.type === 'upscaleNode' && fileId) {
314
- const job = (0, jobQueue_1.createJob)('upscale', systemUserId, projectId, nodeData);
314
+ const job = await (0, jobQueue_1.createJob)('upscale', systemUserId, projectId, nodeData);
315
315
  setImmediate(() => (0, aiProcessor_1.runUpscaleJob)(job.jobId, fileId, projectId, {
316
316
  scale: nodeData.scale,
317
317
  model: nodeData.model,
@@ -321,7 +321,7 @@ async function executeExport(nodeSchema, project, config, { public: isPublic = t
321
321
  payload = { ...payload, jobId: job.jobId, jobType: 'upscale' };
322
322
  }
323
323
  else if (processNode.type === 'proceduralAssetNode') {
324
- const job = (0, jobQueue_1.createJob)('generate-asset', systemUserId, projectId, nodeData);
324
+ const job = await (0, jobQueue_1.createJob)('generate-asset', systemUserId, projectId, nodeData);
325
325
  setImmediate(() => (0, aiProcessor_1.runGenerateAssetJob)(job.jobId, projectId, {
326
326
  assetType: nodeData.assetType,
327
327
  provider: nodeData.provider,
@@ -1,5 +1,7 @@
1
1
  import { JobRecord, JobType } from '../types';
2
- export declare function createJob(jobType: JobType, userId: string, projectId: string, payload: Record<string, any>): JobRecord;
3
- export declare function getJob(jobId: string): JobRecord | undefined;
4
- export declare function updateJob(jobId: string, updates: Partial<Pick<JobRecord, 'status' | 'stage' | 'progress' | 'assetId' | 'error'>>): void;
5
- export declare function failJob(jobId: string, error: string): void;
2
+ export declare function createJob(jobType: JobType, userId: string, projectId: string, payload: Record<string, any>, extra?: Partial<JobRecord>): Promise<JobRecord>;
3
+ export declare function getJob(jobId: string): Promise<JobRecord | null>;
4
+ export declare function listJobs(projectId: string, assetId?: string): Promise<JobRecord[]>;
5
+ export declare function notify(job: JobRecord): void;
6
+ export declare function updateJob(jobId: string, updates: Partial<JobRecord>): Promise<JobRecord | null>;
7
+ export declare function failJob(jobId: string, error: string): Promise<JobRecord | null>;
@@ -5,16 +5,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.createJob = createJob;
7
7
  exports.getJob = getJob;
8
+ exports.listJobs = listJobs;
9
+ exports.notify = notify;
8
10
  exports.updateJob = updateJob;
9
11
  exports.failJob = failJob;
10
12
  const crypto_1 = __importDefault(require("crypto"));
11
13
  const wsHub_1 = require("./wsHub");
12
- const jobs = new Map();
13
- function createJob(jobType, userId, projectId, payload) {
14
- const jobId = crypto_1.default.randomUUID();
14
+ const jobStore_1 = require("./jobStore");
15
+ // Jobs, kept in the main database (see jobStore): they survive restarts and every instance
16
+ // sees them. Each change is sent to whoever started the job (a `jobEvent`).
17
+ async function createJob(jobType, userId, projectId, payload, extra = {}) {
15
18
  const now = new Date();
16
19
  const job = {
17
- jobId,
20
+ jobId: crypto_1.default.randomUUID(),
18
21
  jobType,
19
22
  userId,
20
23
  projectId,
@@ -24,31 +27,40 @@ function createJob(jobType, userId, projectId, payload) {
24
27
  payload,
25
28
  createdAt: now,
26
29
  updatedAt: now,
30
+ ...extra,
27
31
  };
28
- jobs.set(jobId, job);
29
- return job;
32
+ const saved = await (0, jobStore_1.jobStore)().insert(job);
33
+ notify(saved);
34
+ return saved;
35
+ }
36
+ async function getJob(jobId) {
37
+ return (0, jobStore_1.jobStore)().get(jobId);
30
38
  }
31
- function getJob(jobId) {
32
- return jobs.get(jobId);
39
+ async function listJobs(projectId, assetId) {
40
+ return (0, jobStore_1.jobStore)().list({ projectId, assetId });
33
41
  }
34
- function updateJob(jobId, updates) {
35
- const job = jobs.get(jobId);
36
- if (!job)
37
- return;
38
- Object.assign(job, updates, { updatedAt: new Date() });
42
+ function notify(job) {
39
43
  (0, wsHub_1.emitToUser)(job.userId, {
40
44
  type: 'jobEvent',
41
45
  data: {
42
46
  jobId: job.jobId,
43
47
  jobType: job.jobType,
48
+ projectId: job.projectId,
44
49
  assetId: job.assetId,
50
+ status: job.status,
45
51
  stage: job.stage,
46
52
  progress: job.progress,
47
- message: updates.stage,
53
+ message: job.stage,
48
54
  error: job.error,
49
55
  },
50
56
  });
51
57
  }
52
- function failJob(jobId, error) {
53
- updateJob(jobId, { status: 'failed', stage: 'failed', error });
58
+ async function updateJob(jobId, updates) {
59
+ const job = await (0, jobStore_1.jobStore)().update(jobId, { ...updates, updatedAt: new Date() });
60
+ if (job)
61
+ notify(job);
62
+ return job;
63
+ }
64
+ async function failJob(jobId, error) {
65
+ return updateJob(jobId, { status: 'failed', stage: 'failed', error });
54
66
  }
@@ -0,0 +1,27 @@
1
+ import { JobRecord, StreamByConfig } from '../types';
2
+ export interface JobStore {
3
+ insert(job: JobRecord): Promise<JobRecord>;
4
+ get(jobId: string): Promise<JobRecord | null>;
5
+ list(filter: {
6
+ projectId: string;
7
+ assetId?: string;
8
+ limit?: number;
9
+ }): Promise<JobRecord[]>;
10
+ update(jobId: string, patch: Partial<JobRecord>): Promise<JobRecord | null>;
11
+ /**
12
+ * Changes a job only if it's still in one of `statuses` (any, if none), and, if given, its
13
+ * lease ended by `leaseEndedBy` and its callback hash is `callbackHash`.
14
+ */
15
+ claim(jobId: string, statuses: JobRecord['status'][], patch: Partial<JobRecord>, only?: ClaimConditions): Promise<JobRecord | null>;
16
+ /** Worker jobs to look at now: queued and due, or dispatched with their lease ended. */
17
+ due(now: Date, limit: number): Promise<JobRecord[]>;
18
+ /** A deleted project's jobs. */
19
+ deleteProject(projectId: string): Promise<void>;
20
+ }
21
+ export interface ClaimConditions {
22
+ leaseEndedBy?: Date;
23
+ callbackHash?: string;
24
+ }
25
+ /** The router's main database, where jobs are kept. */
26
+ export declare const setJobStoreDatabase: (config: StreamByConfig) => void;
27
+ export declare function jobStore(): JobStore;
@@ -0,0 +1,177 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.setJobStoreDatabase = void 0;
4
+ exports.jobStore = jobStore;
5
+ const connectionManager_1 = require("../adapters/database/connectionManager");
6
+ const JSON_FIELDS = new Set(['payload', 'report', 'result', 'worker', 'profile', 'upload']);
7
+ // ── PostgreSQL ──
8
+ const sqlStore = (pool) => {
9
+ const table = 'streamby.jobs';
10
+ const fromRow = (row) => {
11
+ if (!row)
12
+ return null;
13
+ const { id, ...job } = row;
14
+ return { ...job, jobId: id };
15
+ };
16
+ const toColumns = (patch) => {
17
+ const entries = Object.entries(patch).filter(([key, value]) => key !== 'jobId' && value !== undefined);
18
+ return {
19
+ names: entries.map(([key]) => `"${key}"`),
20
+ values: entries.map(([key, value]) => (JSON_FIELDS.has(key) && value !== null ? JSON.stringify(value) : value)),
21
+ };
22
+ };
23
+ return {
24
+ async insert(job) {
25
+ const { names, values } = toColumns(job);
26
+ const placeholders = values.map((_, i) => `$${i + 2}`);
27
+ const { rows } = await pool.query(`INSERT INTO ${table} (id, ${names.join(', ')}) VALUES ($1, ${placeholders.join(', ')}) RETURNING *`, [job.jobId, ...values]);
28
+ return fromRow(rows[0]);
29
+ },
30
+ async get(jobId) {
31
+ const { rows } = await pool.query(`SELECT * FROM ${table} WHERE id = $1`, [jobId]);
32
+ return fromRow(rows[0]);
33
+ },
34
+ async list({ projectId, assetId, limit = 100 }) {
35
+ const { rows } = await pool.query(`SELECT * FROM ${table} WHERE "projectId" = $1 ${assetId ? 'AND "assetId" = $3' : ''} ORDER BY "createdAt" DESC LIMIT $2`, assetId ? [projectId, limit, assetId] : [projectId, limit]);
36
+ return rows.map(row => fromRow(row));
37
+ },
38
+ async update(jobId, patch) {
39
+ return this.claim(jobId, [], patch);
40
+ },
41
+ async claim(jobId, statuses, patch, only = {}) {
42
+ const { names, values } = toColumns(patch);
43
+ const sets = names.map((name, i) => `${name} = $${i + 2}`).join(', ');
44
+ const params = [jobId, ...values];
45
+ let where = 'id = $1';
46
+ if (statuses.length) {
47
+ params.push(statuses);
48
+ where += ` AND status = ANY($${params.length})`;
49
+ }
50
+ if (only.leaseEndedBy) {
51
+ params.push(only.leaseEndedBy);
52
+ where += ` AND ("leaseUntil" IS NULL OR "leaseUntil" <= $${params.length})`;
53
+ }
54
+ if (only.callbackHash) {
55
+ params.push(only.callbackHash);
56
+ where += ` AND "callbackHash" = $${params.length}`;
57
+ }
58
+ const { rows } = await pool.query(`UPDATE ${table} SET ${sets} WHERE ${where} RETURNING *`, params);
59
+ return fromRow(rows[0]);
60
+ },
61
+ async due(now, limit) {
62
+ const { rows } = await pool.query(`SELECT * FROM ${table}
63
+ WHERE worker IS NOT NULL AND (
64
+ (status = 'pending' AND ("nextAttemptAt" IS NULL OR "nextAttemptAt" <= $1))
65
+ OR (status = 'processing' AND "leaseUntil" <= $1))
66
+ ORDER BY "createdAt" LIMIT $2`, [now, limit]);
67
+ return rows.map(row => fromRow(row));
68
+ },
69
+ async deleteProject(projectId) {
70
+ await pool.query(`DELETE FROM ${table} WHERE "projectId" = $1`, [projectId]);
71
+ },
72
+ };
73
+ };
74
+ // ── MongoDB ──
75
+ const mongoStore = (client) => {
76
+ const col = () => client.db().collection('jobs');
77
+ const fromDoc = (doc) => {
78
+ if (!doc)
79
+ return null;
80
+ const { _id, ...job } = doc;
81
+ return job;
82
+ };
83
+ const clean = (patch) => Object.fromEntries(Object.entries(patch).filter(([, value]) => value !== undefined));
84
+ return {
85
+ async insert(job) {
86
+ await col().insertOne({ ...clean(job) });
87
+ return job;
88
+ },
89
+ async get(jobId) {
90
+ return fromDoc(await col().findOne({ jobId }));
91
+ },
92
+ async list({ projectId, assetId, limit = 100 }) {
93
+ const docs = await col().find({ projectId, ...(assetId ? { assetId } : {}) }).sort({ createdAt: -1 }).limit(limit).toArray();
94
+ return docs.map(doc => fromDoc(doc));
95
+ },
96
+ async update(jobId, patch) {
97
+ return this.claim(jobId, [], patch);
98
+ },
99
+ async claim(jobId, statuses, patch, only = {}) {
100
+ const filter = { jobId };
101
+ if (statuses.length)
102
+ filter.status = { $in: statuses };
103
+ if (only.leaseEndedBy)
104
+ filter.$or = [{ leaseUntil: null }, { leaseUntil: { $exists: false } }, { leaseUntil: { $lte: only.leaseEndedBy } }];
105
+ if (only.callbackHash)
106
+ filter.callbackHash = only.callbackHash;
107
+ return fromDoc(await col().findOneAndUpdate(filter, { $set: clean(patch) }, { returnDocument: 'after' }));
108
+ },
109
+ async due(now, limit) {
110
+ const docs = await col().find({
111
+ worker: { $ne: null },
112
+ $or: [
113
+ { status: 'pending', $or: [{ nextAttemptAt: null }, { nextAttemptAt: { $lte: now } }] },
114
+ { status: 'processing', leaseUntil: { $lte: now } },
115
+ ],
116
+ }).sort({ createdAt: 1 }).limit(limit).toArray();
117
+ return docs.map(doc => fromDoc(doc));
118
+ },
119
+ async deleteProject(projectId) {
120
+ await col().deleteMany({ projectId });
121
+ },
122
+ };
123
+ };
124
+ // ── In memory (no main database) ──
125
+ const memoryStore = () => {
126
+ const jobs = new Map();
127
+ const copy = (job) => (job ? { ...job } : null);
128
+ return {
129
+ async insert(job) { jobs.set(job.jobId, { ...job }); return { ...job }; },
130
+ async get(jobId) { return copy(jobs.get(jobId)); },
131
+ async list({ projectId, assetId, limit = 100 }) {
132
+ return [...jobs.values()]
133
+ .filter(job => job.projectId === projectId && (!assetId || job.assetId === assetId))
134
+ .sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime())
135
+ .slice(0, limit)
136
+ .map(job => ({ ...job }));
137
+ },
138
+ async update(jobId, patch) { return this.claim(jobId, [], patch); },
139
+ async claim(jobId, statuses, patch, only = {}) {
140
+ const job = jobs.get(jobId);
141
+ if (!job || (statuses.length && !statuses.includes(job.status)))
142
+ return null;
143
+ if (only.leaseEndedBy && job.leaseUntil && new Date(job.leaseUntil) > only.leaseEndedBy)
144
+ return null;
145
+ if (only.callbackHash && job.callbackHash !== only.callbackHash)
146
+ return null;
147
+ Object.assign(job, Object.fromEntries(Object.entries(patch).filter(([, value]) => value !== undefined)));
148
+ return { ...job };
149
+ },
150
+ async due(now, limit) {
151
+ return [...jobs.values()]
152
+ .filter(job => job.worker && ((job.status === 'pending' && (!job.nextAttemptAt || new Date(job.nextAttemptAt) <= now))
153
+ || (job.status === 'processing' && job.leaseUntil && new Date(job.leaseUntil) <= now)))
154
+ .slice(0, limit)
155
+ .map(job => ({ ...job }));
156
+ },
157
+ async deleteProject(projectId) {
158
+ for (const [id, job] of jobs)
159
+ if (job.projectId === projectId)
160
+ jobs.delete(id);
161
+ },
162
+ };
163
+ };
164
+ let mainDbId;
165
+ let memory;
166
+ /** The router's main database, where jobs are kept. */
167
+ const setJobStoreDatabase = (config) => {
168
+ mainDbId = config.databases?.find(db => db.main)?.id;
169
+ };
170
+ exports.setJobStoreDatabase = setJobStoreDatabase;
171
+ function jobStore() {
172
+ if (mainDbId && (0, connectionManager_1.getConnectedIds)().includes(mainDbId)) {
173
+ const entry = (0, connectionManager_1.getConnection)(mainDbId);
174
+ return entry.type === 'sql' ? sqlStore(entry.client) : mongoStore(entry.client);
175
+ }
176
+ return (memory ?? (memory = memoryStore()));
177
+ }