streamby-core 0.38.0 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +108 -1
  3. package/dist/adapters/database/connectionManager.js +55 -0
  4. package/dist/mcp/tools.js +41 -0
  5. package/dist/middleware/createRouter.js +11 -1
  6. package/dist/middleware/routes/ai.js +3 -3
  7. package/dist/middleware/routes/deliverable.d.ts +4 -2
  8. package/dist/middleware/routes/deliverable.js +16 -7
  9. package/dist/middleware/routes/export.js +16 -0
  10. package/dist/middleware/routes/media.js +7 -6
  11. package/dist/middleware/routes/project.js +36 -10
  12. package/dist/middleware/routes/renderFarm.js +4 -77
  13. package/dist/middleware/routes/storage.js +13 -27
  14. package/dist/middleware/routes/worker.d.ts +7 -0
  15. package/dist/middleware/routes/worker.js +125 -0
  16. package/dist/middleware/routes/workerCallback.d.ts +5 -0
  17. package/dist/middleware/routes/workerCallback.js +31 -0
  18. package/dist/services/aiProcessor.js +11 -11
  19. package/dist/services/apiTokenPermissions.js +8 -7
  20. package/dist/services/executor.js +10 -10
  21. package/dist/services/jobQueue.d.ts +6 -4
  22. package/dist/services/jobQueue.js +28 -16
  23. package/dist/services/jobStore.d.ts +27 -0
  24. package/dist/services/jobStore.js +177 -0
  25. package/dist/services/mediaProcessor.js +16 -16
  26. package/dist/services/vfxProcessor.js +12 -12
  27. package/dist/services/workerJobs.d.ts +73 -0
  28. package/dist/services/workerJobs.js +459 -0
  29. package/dist/services/workers.d.ts +78 -0
  30. package/dist/services/workers.js +179 -0
  31. package/dist/types/index.d.ts +52 -2
  32. package/dist/utils/assetCategories.d.ts +4 -0
  33. package/dist/utils/assetCategories.js +21 -0
  34. package/dist/utils/builtinAccess.d.ts +1 -1
  35. package/dist/utils/builtinAccess.js +4 -3
  36. package/dist/utils/privateAddress.d.ts +10 -0
  37. package/dist/utils/privateAddress.js +85 -0
  38. package/dist/utils/sanitize.d.ts +1 -0
  39. package/dist/utils/sanitize.js +14 -0
  40. package/package.json +1 -1
@@ -5,16 +5,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.createJob = createJob;
7
7
  exports.getJob = getJob;
8
+ exports.listJobs = listJobs;
9
+ exports.notify = notify;
8
10
  exports.updateJob = updateJob;
9
11
  exports.failJob = failJob;
10
12
  const crypto_1 = __importDefault(require("crypto"));
11
13
  const wsHub_1 = require("./wsHub");
12
- const jobs = new Map();
13
- function createJob(jobType, userId, projectId, payload) {
14
- const jobId = crypto_1.default.randomUUID();
14
+ const jobStore_1 = require("./jobStore");
15
+ // Jobs, kept in the main database (see jobStore): they survive restarts and every instance
16
+ // sees them. Each change is sent to whoever started the job (a `jobEvent`).
17
+ async function createJob(jobType, userId, projectId, payload, extra = {}) {
15
18
  const now = new Date();
16
19
  const job = {
17
- jobId,
20
+ jobId: crypto_1.default.randomUUID(),
18
21
  jobType,
19
22
  userId,
20
23
  projectId,
@@ -24,31 +27,40 @@ function createJob(jobType, userId, projectId, payload) {
24
27
  payload,
25
28
  createdAt: now,
26
29
  updatedAt: now,
30
+ ...extra,
27
31
  };
28
- jobs.set(jobId, job);
29
- return job;
32
+ const saved = await (0, jobStore_1.jobStore)().insert(job);
33
+ notify(saved);
34
+ return saved;
35
+ }
36
+ async function getJob(jobId) {
37
+ return (0, jobStore_1.jobStore)().get(jobId);
30
38
  }
31
- function getJob(jobId) {
32
- return jobs.get(jobId);
39
+ async function listJobs(projectId, assetId) {
40
+ return (0, jobStore_1.jobStore)().list({ projectId, assetId });
33
41
  }
34
- function updateJob(jobId, updates) {
35
- const job = jobs.get(jobId);
36
- if (!job)
37
- return;
38
- Object.assign(job, updates, { updatedAt: new Date() });
42
+ function notify(job) {
39
43
  (0, wsHub_1.emitToUser)(job.userId, {
40
44
  type: 'jobEvent',
41
45
  data: {
42
46
  jobId: job.jobId,
43
47
  jobType: job.jobType,
48
+ projectId: job.projectId,
44
49
  assetId: job.assetId,
50
+ status: job.status,
45
51
  stage: job.stage,
46
52
  progress: job.progress,
47
- message: updates.stage,
53
+ message: job.stage,
48
54
  error: job.error,
49
55
  },
50
56
  });
51
57
  }
52
- function failJob(jobId, error) {
53
- updateJob(jobId, { status: 'failed', stage: 'failed', error });
58
+ async function updateJob(jobId, updates) {
59
+ const job = await (0, jobStore_1.jobStore)().update(jobId, { ...updates, updatedAt: new Date() });
60
+ if (job)
61
+ notify(job);
62
+ return job;
63
+ }
64
+ async function failJob(jobId, error) {
65
+ return updateJob(jobId, { status: 'failed', stage: 'failed', error });
54
66
  }
@@ -0,0 +1,27 @@
1
+ import { JobRecord, StreamByConfig } from '../types';
2
+ export interface JobStore {
3
+ insert(job: JobRecord): Promise<JobRecord>;
4
+ get(jobId: string): Promise<JobRecord | null>;
5
+ list(filter: {
6
+ projectId: string;
7
+ assetId?: string;
8
+ limit?: number;
9
+ }): Promise<JobRecord[]>;
10
+ update(jobId: string, patch: Partial<JobRecord>): Promise<JobRecord | null>;
11
+ /**
12
+ * Changes a job only if it's still in one of `statuses` (any, if none), and, if given, its
13
+ * lease ended by `leaseEndedBy` and its callback hash is `callbackHash`.
14
+ */
15
+ claim(jobId: string, statuses: JobRecord['status'][], patch: Partial<JobRecord>, only?: ClaimConditions): Promise<JobRecord | null>;
16
+ /** Worker jobs to look at now: queued and due, or dispatched with their lease ended. */
17
+ due(now: Date, limit: number): Promise<JobRecord[]>;
18
+ /** A deleted project's jobs. */
19
+ deleteProject(projectId: string): Promise<void>;
20
+ }
21
+ export interface ClaimConditions {
22
+ leaseEndedBy?: Date;
23
+ callbackHash?: string;
24
+ }
25
+ /** The router's main database, where jobs are kept. */
26
+ export declare const setJobStoreDatabase: (config: StreamByConfig) => void;
27
+ export declare function jobStore(): JobStore;
@@ -0,0 +1,177 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.setJobStoreDatabase = void 0;
4
+ exports.jobStore = jobStore;
5
+ const connectionManager_1 = require("../adapters/database/connectionManager");
6
+ const JSON_FIELDS = new Set(['payload', 'report', 'result', 'worker', 'profile', 'upload']);
7
+ // ── PostgreSQL ──
8
+ const sqlStore = (pool) => {
9
+ const table = 'streamby.jobs';
10
+ const fromRow = (row) => {
11
+ if (!row)
12
+ return null;
13
+ const { id, ...job } = row;
14
+ return { ...job, jobId: id };
15
+ };
16
+ const toColumns = (patch) => {
17
+ const entries = Object.entries(patch).filter(([key, value]) => key !== 'jobId' && value !== undefined);
18
+ return {
19
+ names: entries.map(([key]) => `"${key}"`),
20
+ values: entries.map(([key, value]) => (JSON_FIELDS.has(key) && value !== null ? JSON.stringify(value) : value)),
21
+ };
22
+ };
23
+ return {
24
+ async insert(job) {
25
+ const { names, values } = toColumns(job);
26
+ const placeholders = values.map((_, i) => `$${i + 2}`);
27
+ const { rows } = await pool.query(`INSERT INTO ${table} (id, ${names.join(', ')}) VALUES ($1, ${placeholders.join(', ')}) RETURNING *`, [job.jobId, ...values]);
28
+ return fromRow(rows[0]);
29
+ },
30
+ async get(jobId) {
31
+ const { rows } = await pool.query(`SELECT * FROM ${table} WHERE id = $1`, [jobId]);
32
+ return fromRow(rows[0]);
33
+ },
34
+ async list({ projectId, assetId, limit = 100 }) {
35
+ const { rows } = await pool.query(`SELECT * FROM ${table} WHERE "projectId" = $1 ${assetId ? 'AND "assetId" = $3' : ''} ORDER BY "createdAt" DESC LIMIT $2`, assetId ? [projectId, limit, assetId] : [projectId, limit]);
36
+ return rows.map(row => fromRow(row));
37
+ },
38
+ async update(jobId, patch) {
39
+ return this.claim(jobId, [], patch);
40
+ },
41
+ async claim(jobId, statuses, patch, only = {}) {
42
+ const { names, values } = toColumns(patch);
43
+ const sets = names.map((name, i) => `${name} = $${i + 2}`).join(', ');
44
+ const params = [jobId, ...values];
45
+ let where = 'id = $1';
46
+ if (statuses.length) {
47
+ params.push(statuses);
48
+ where += ` AND status = ANY($${params.length})`;
49
+ }
50
+ if (only.leaseEndedBy) {
51
+ params.push(only.leaseEndedBy);
52
+ where += ` AND ("leaseUntil" IS NULL OR "leaseUntil" <= $${params.length})`;
53
+ }
54
+ if (only.callbackHash) {
55
+ params.push(only.callbackHash);
56
+ where += ` AND "callbackHash" = $${params.length}`;
57
+ }
58
+ const { rows } = await pool.query(`UPDATE ${table} SET ${sets} WHERE ${where} RETURNING *`, params);
59
+ return fromRow(rows[0]);
60
+ },
61
+ async due(now, limit) {
62
+ const { rows } = await pool.query(`SELECT * FROM ${table}
63
+ WHERE worker IS NOT NULL AND (
64
+ (status = 'pending' AND ("nextAttemptAt" IS NULL OR "nextAttemptAt" <= $1))
65
+ OR (status = 'processing' AND "leaseUntil" <= $1))
66
+ ORDER BY "createdAt" LIMIT $2`, [now, limit]);
67
+ return rows.map(row => fromRow(row));
68
+ },
69
+ async deleteProject(projectId) {
70
+ await pool.query(`DELETE FROM ${table} WHERE "projectId" = $1`, [projectId]);
71
+ },
72
+ };
73
+ };
74
+ // ── MongoDB ──
75
+ const mongoStore = (client) => {
76
+ const col = () => client.db().collection('jobs');
77
+ const fromDoc = (doc) => {
78
+ if (!doc)
79
+ return null;
80
+ const { _id, ...job } = doc;
81
+ return job;
82
+ };
83
+ const clean = (patch) => Object.fromEntries(Object.entries(patch).filter(([, value]) => value !== undefined));
84
+ return {
85
+ async insert(job) {
86
+ await col().insertOne({ ...clean(job) });
87
+ return job;
88
+ },
89
+ async get(jobId) {
90
+ return fromDoc(await col().findOne({ jobId }));
91
+ },
92
+ async list({ projectId, assetId, limit = 100 }) {
93
+ const docs = await col().find({ projectId, ...(assetId ? { assetId } : {}) }).sort({ createdAt: -1 }).limit(limit).toArray();
94
+ return docs.map(doc => fromDoc(doc));
95
+ },
96
+ async update(jobId, patch) {
97
+ return this.claim(jobId, [], patch);
98
+ },
99
+ async claim(jobId, statuses, patch, only = {}) {
100
+ const filter = { jobId };
101
+ if (statuses.length)
102
+ filter.status = { $in: statuses };
103
+ if (only.leaseEndedBy)
104
+ filter.$or = [{ leaseUntil: null }, { leaseUntil: { $exists: false } }, { leaseUntil: { $lte: only.leaseEndedBy } }];
105
+ if (only.callbackHash)
106
+ filter.callbackHash = only.callbackHash;
107
+ return fromDoc(await col().findOneAndUpdate(filter, { $set: clean(patch) }, { returnDocument: 'after' }));
108
+ },
109
+ async due(now, limit) {
110
+ const docs = await col().find({
111
+ worker: { $ne: null },
112
+ $or: [
113
+ { status: 'pending', $or: [{ nextAttemptAt: null }, { nextAttemptAt: { $lte: now } }] },
114
+ { status: 'processing', leaseUntil: { $lte: now } },
115
+ ],
116
+ }).sort({ createdAt: 1 }).limit(limit).toArray();
117
+ return docs.map(doc => fromDoc(doc));
118
+ },
119
+ async deleteProject(projectId) {
120
+ await col().deleteMany({ projectId });
121
+ },
122
+ };
123
+ };
124
+ // ── In memory (no main database) ──
125
+ const memoryStore = () => {
126
+ const jobs = new Map();
127
+ const copy = (job) => (job ? { ...job } : null);
128
+ return {
129
+ async insert(job) { jobs.set(job.jobId, { ...job }); return { ...job }; },
130
+ async get(jobId) { return copy(jobs.get(jobId)); },
131
+ async list({ projectId, assetId, limit = 100 }) {
132
+ return [...jobs.values()]
133
+ .filter(job => job.projectId === projectId && (!assetId || job.assetId === assetId))
134
+ .sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime())
135
+ .slice(0, limit)
136
+ .map(job => ({ ...job }));
137
+ },
138
+ async update(jobId, patch) { return this.claim(jobId, [], patch); },
139
+ async claim(jobId, statuses, patch, only = {}) {
140
+ const job = jobs.get(jobId);
141
+ if (!job || (statuses.length && !statuses.includes(job.status)))
142
+ return null;
143
+ if (only.leaseEndedBy && job.leaseUntil && new Date(job.leaseUntil) > only.leaseEndedBy)
144
+ return null;
145
+ if (only.callbackHash && job.callbackHash !== only.callbackHash)
146
+ return null;
147
+ Object.assign(job, Object.fromEntries(Object.entries(patch).filter(([, value]) => value !== undefined)));
148
+ return { ...job };
149
+ },
150
+ async due(now, limit) {
151
+ return [...jobs.values()]
152
+ .filter(job => job.worker && ((job.status === 'pending' && (!job.nextAttemptAt || new Date(job.nextAttemptAt) <= now))
153
+ || (job.status === 'processing' && job.leaseUntil && new Date(job.leaseUntil) <= now)))
154
+ .slice(0, limit)
155
+ .map(job => ({ ...job }));
156
+ },
157
+ async deleteProject(projectId) {
158
+ for (const [id, job] of jobs)
159
+ if (job.projectId === projectId)
160
+ jobs.delete(id);
161
+ },
162
+ };
163
+ };
164
+ let mainDbId;
165
+ let memory;
166
+ /** The router's main database, where jobs are kept. */
167
+ const setJobStoreDatabase = (config) => {
168
+ mainDbId = config.databases?.find(db => db.main)?.id;
169
+ };
170
+ exports.setJobStoreDatabase = setJobStoreDatabase;
171
+ function jobStore() {
172
+ if (mainDbId && (0, connectionManager_1.getConnectedIds)().includes(mainDbId)) {
173
+ const entry = (0, connectionManager_1.getConnection)(mainDbId);
174
+ return entry.type === 'sql' ? sqlStore(entry.client) : mongoStore(entry.client);
175
+ }
176
+ return (memory ?? (memory = memoryStore()));
177
+ }
@@ -45,7 +45,7 @@ function inferMetadata(contentType, size) {
45
45
  return meta;
46
46
  }
47
47
  async function runIngestJob(jobId, fileId, projectId) {
48
- (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10 });
48
+ await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10 });
49
49
  try {
50
50
  const filesCol = getFilesCollection();
51
51
  if (!filesCol)
@@ -53,7 +53,7 @@ async function runIngestJob(jobId, fileId, projectId) {
53
53
  const fileDoc = await filesCol.findOne({ fileId, projectId });
54
54
  if (!fileDoc)
55
55
  throw new Error(`File ${fileId} not found in project ${projectId}`);
56
- (0, jobQueue_1.updateJob)(jobId, { stage: 'extracting-metadata', progress: 40, assetId: fileId });
56
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'extracting-metadata', progress: 40, assetId: fileId });
57
57
  const metaCol = getMetadataCollection();
58
58
  if (metaCol) {
59
59
  const meta = inferMetadata(fileDoc.contentType ?? '', fileDoc.size ?? 0);
@@ -67,7 +67,7 @@ async function runIngestJob(jobId, fileId, projectId) {
67
67
  },
68
68
  }, { upsert: true });
69
69
  }
70
- (0, jobQueue_1.updateJob)(jobId, { stage: 'creating-version', progress: 70 });
70
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'creating-version', progress: 70 });
71
71
  const versionsCol = getVersionsCollection();
72
72
  // Its upload already recorded the version it's in; only an asset from before versions
73
73
  // were kept gets one here.
@@ -85,14 +85,14 @@ async function runIngestJob(jobId, fileId, projectId) {
85
85
  createdAt: new Date(),
86
86
  });
87
87
  }
88
- (0, jobQueue_1.updateJob)(jobId, { status: 'completed', stage: 'complete', progress: 100 });
88
+ await (0, jobQueue_1.updateJob)(jobId, { status: 'completed', stage: 'complete', progress: 100 });
89
89
  }
90
90
  catch (err) {
91
- (0, jobQueue_1.failJob)(jobId, err.message);
91
+ await (0, jobQueue_1.failJob)(jobId, err.message);
92
92
  }
93
93
  }
94
94
  async function runTranscodeJob(jobId, fileId, projectId, options, _adapter) {
95
- (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'queued', progress: 5, assetId: fileId });
95
+ await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'queued', progress: 5, assetId: fileId });
96
96
  try {
97
97
  const filesCol = getFilesCollection();
98
98
  if (!filesCol)
@@ -103,7 +103,7 @@ async function runTranscodeJob(jobId, fileId, projectId, options, _adapter) {
103
103
  if (!fileDoc.contentType?.startsWith('video/') && !fileDoc.contentType?.startsWith('audio/')) {
104
104
  throw new Error('Transcode only supports video and audio files');
105
105
  }
106
- (0, jobQueue_1.updateJob)(jobId, { stage: 'transcoding', progress: 20 });
106
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'transcoding', progress: 20 });
107
107
  // Store the transcode request as a pending task — actual FFmpeg execution
108
108
  // happens in the host environment (Nhexa-API worker). The job record is the
109
109
  // handoff contract: the worker reads it, processes the file, and calls back
@@ -127,14 +127,14 @@ async function runTranscodeJob(jobId, fileId, projectId, options, _adapter) {
127
127
  },
128
128
  }, { upsert: true });
129
129
  }
130
- (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
130
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
131
131
  }
132
132
  catch (err) {
133
- (0, jobQueue_1.failJob)(jobId, err.message);
133
+ await (0, jobQueue_1.failJob)(jobId, err.message);
134
134
  }
135
135
  }
136
136
  async function runThumbnailJob(jobId, fileId, projectId, options) {
137
- (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
137
+ await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
138
138
  try {
139
139
  const filesCol = getFilesCollection();
140
140
  if (!filesCol)
@@ -142,7 +142,7 @@ async function runThumbnailJob(jobId, fileId, projectId, options) {
142
142
  const fileDoc = await filesCol.findOne({ fileId, projectId });
143
143
  if (!fileDoc)
144
144
  throw new Error(`File ${fileId} not found`);
145
- (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
145
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
146
146
  const metaCol = getMetadataCollection();
147
147
  if (metaCol) {
148
148
  await metaCol.updateOne({ assetId: fileId, projectId }, {
@@ -160,14 +160,14 @@ async function runThumbnailJob(jobId, fileId, projectId, options) {
160
160
  },
161
161
  }, { upsert: true });
162
162
  }
163
- (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 40 });
163
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 40 });
164
164
  }
165
165
  catch (err) {
166
- (0, jobQueue_1.failJob)(jobId, err.message);
166
+ await (0, jobQueue_1.failJob)(jobId, err.message);
167
167
  }
168
168
  }
169
169
  async function runCaptionJob(jobId, fileId, projectId, options) {
170
- (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
170
+ await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
171
171
  try {
172
172
  const filesCol = getFilesCollection();
173
173
  if (!filesCol)
@@ -195,9 +195,9 @@ async function runCaptionJob(jobId, fileId, projectId, options) {
195
195
  },
196
196
  }, { upsert: true });
197
197
  }
198
- (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
198
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
199
199
  }
200
200
  catch (err) {
201
- (0, jobQueue_1.failJob)(jobId, err.message);
201
+ await (0, jobQueue_1.failJob)(jobId, err.message);
202
202
  }
203
203
  }
@@ -13,7 +13,7 @@ function getNosqlDb() {
13
13
  return (0, connectionManager_1.getConnection)(ids[0]).client.db();
14
14
  }
15
15
  async function runRenderJob(jobId, fileId, projectId, options) {
16
- (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
16
+ await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
17
17
  try {
18
18
  const db = getNosqlDb();
19
19
  if (!db)
@@ -21,7 +21,7 @@ async function runRenderJob(jobId, fileId, projectId, options) {
21
21
  const fileDoc = await db.collection('storage_files').findOne({ fileId, projectId });
22
22
  if (!fileDoc)
23
23
  throw new Error(`File ${fileId} not found`);
24
- (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-farm', progress: 20 });
24
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-farm', progress: 20 });
25
25
  // Store pending render task — picked up by the host render farm worker
26
26
  await db.collection('media_metadata').updateOne({ assetId: fileId, projectId }, {
27
27
  $set: {
@@ -40,14 +40,14 @@ async function runRenderJob(jobId, fileId, projectId, options) {
40
40
  extractedAt: new Date(),
41
41
  },
42
42
  }, { upsert: true });
43
- (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
43
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
44
44
  }
45
45
  catch (err) {
46
- (0, jobQueue_1.failJob)(jobId, err.message);
46
+ await (0, jobQueue_1.failJob)(jobId, err.message);
47
47
  }
48
48
  }
49
49
  async function runFormatConvertJob(jobId, fileId, projectId, options) {
50
- (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
50
+ await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
51
51
  try {
52
52
  const db = getNosqlDb();
53
53
  if (!db)
@@ -55,7 +55,7 @@ async function runFormatConvertJob(jobId, fileId, projectId, options) {
55
55
  const fileDoc = await db.collection('storage_files').findOne({ fileId, projectId });
56
56
  if (!fileDoc)
57
57
  throw new Error(`File ${fileId} not found`);
58
- (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-convert', progress: 25 });
58
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-convert', progress: 25 });
59
59
  await db.collection('media_metadata').updateOne({ assetId: fileId, projectId }, {
60
60
  $set: {
61
61
  assetId: fileId,
@@ -71,14 +71,14 @@ async function runFormatConvertJob(jobId, fileId, projectId, options) {
71
71
  extractedAt: new Date(),
72
72
  },
73
73
  }, { upsert: true });
74
- (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 35 });
74
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 35 });
75
75
  }
76
76
  catch (err) {
77
- (0, jobQueue_1.failJob)(jobId, err.message);
77
+ await (0, jobQueue_1.failJob)(jobId, err.message);
78
78
  }
79
79
  }
80
80
  async function runLodJob(jobId, fileId, projectId, options) {
81
- (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
81
+ await (0, jobQueue_1.updateJob)(jobId, { status: 'processing', stage: 'resolving', progress: 10, assetId: fileId });
82
82
  try {
83
83
  const db = getNosqlDb();
84
84
  if (!db)
@@ -86,7 +86,7 @@ async function runLodJob(jobId, fileId, projectId, options) {
86
86
  const fileDoc = await db.collection('storage_files').findOne({ fileId, projectId });
87
87
  if (!fileDoc)
88
88
  throw new Error(`File ${fileId} not found`);
89
- (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-lod', progress: 25 });
89
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'queued-lod', progress: 25 });
90
90
  await db.collection('media_metadata').updateOne({ assetId: fileId, projectId }, {
91
91
  $set: {
92
92
  assetId: fileId,
@@ -102,10 +102,10 @@ async function runLodJob(jobId, fileId, projectId, options) {
102
102
  extractedAt: new Date(),
103
103
  },
104
104
  }, { upsert: true });
105
- (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
105
+ await (0, jobQueue_1.updateJob)(jobId, { stage: 'pending-worker', progress: 30 });
106
106
  }
107
107
  catch (err) {
108
- (0, jobQueue_1.failJob)(jobId, err.message);
108
+ await (0, jobQueue_1.failJob)(jobId, err.message);
109
109
  }
110
110
  }
111
111
  async function resolveAssetDependencyGraph(projectId, rootAssetId, maxDepth = 5) {
@@ -0,0 +1,73 @@
1
+ import { Auth, JobRecord, StorageAdapter, StreamByConfig, WorkerRef, WorkerResultMode } from '../types';
2
+ export declare const WORKER_CALLBACK_PATH = "/worker-jobs";
3
+ type RouterConfig = StreamByConfig & {
4
+ adapter?: StorageAdapter;
5
+ };
6
+ export interface WorkerJobRequest {
7
+ assetId: unknown;
8
+ versionId?: unknown;
9
+ worker: unknown;
10
+ kind: unknown;
11
+ profile?: unknown;
12
+ resultMode?: unknown;
13
+ }
14
+ /** Asks a worker to process an asset, as `auth`: the job, pending, to be sent right away. */
15
+ export declare function enqueueWorkerJob(project: any, request: WorkerJobRequest, auth: Auth, config: RouterConfig, callbackBase: string): Promise<JobRecord>;
16
+ /** Runs the workers set to process, by themselves, an asset of their categories marked as a deliverable. */
17
+ export declare function enqueueAutomaticJobs(project: any, assetId: string, versionId: string | null, auth: Auth, config: RouterConfig, callbackBase: string): Promise<JobRecord[]>;
18
+ /** A failed worker job, back in line. */
19
+ export declare function retryWorkerJob(jobId: string, config: RouterConfig): Promise<JobRecord | null>;
20
+ /**
21
+ * Sends a pending job (or one its worker didn't finish in time) to its worker. Of several
22
+ * instances, only the one that claims it does.
23
+ */
24
+ export declare function dispatchJob(jobId: string, config: RouterConfig): Promise<void>;
25
+ /** Looks, now and then, for jobs to send (again). Every instance does; claiming keeps them from doubling. */
26
+ export declare function startWorkerSweeper(config: RouterConfig): () => void;
27
+ /** The job a callback is about, if its token is this attempt's and the job is in progress. */
28
+ export declare function authorizeCallback(jobId: string, authorization: string | undefined): Promise<JobRecord>;
29
+ export declare function reportProgress(job: JobRecord, body: any): Promise<JobRecord | null>;
30
+ /** Where the worker uploads its result: a key of its own until the job completes. */
31
+ export declare function resultUploadUrl(job: JobRecord, body: any, config: RouterConfig): Promise<{
32
+ url: string;
33
+ method: string;
34
+ headers: {
35
+ 'Content-Type': any;
36
+ };
37
+ expiresAt: string;
38
+ }>;
39
+ /** The worker's result, kept: a new version of the asset, or a new asset derived from it. */
40
+ export declare function completeJob(job: JobRecord, body: any, config: RouterConfig): Promise<JobRecord>;
41
+ /** The worker couldn't do it: sent again if it says it may work later (and attempts are left), or failed. */
42
+ export declare function failWorkerJob(job: JobRecord, body: any, config: RouterConfig): Promise<JobRecord | null>;
43
+ /** What's shown of a job: never its callback token's hash, nor its upload's key. */
44
+ export declare const publicJob: ({ callbackHash: _hash, upload: _upload, payload, ...job }: JobRecord) => {
45
+ payload: Record<string, any>;
46
+ jobId: string;
47
+ jobType: import("../types").JobType;
48
+ userId: string;
49
+ projectId: string;
50
+ assetId?: string;
51
+ status: import("../types").JobStatus;
52
+ stage: string;
53
+ progress: number;
54
+ error?: string;
55
+ createdAt: Date;
56
+ updatedAt: Date;
57
+ worker?: WorkerRef | null;
58
+ kind?: string | null;
59
+ profile?: Record<string, unknown> | null;
60
+ versionId?: string | null;
61
+ resultMode?: WorkerResultMode | null;
62
+ attempts?: number;
63
+ maxAttempts?: number;
64
+ nextAttemptAt?: Date | null;
65
+ leaseUntil?: Date | null;
66
+ report?: Record<string, unknown> | null;
67
+ result?: {
68
+ assetId: string;
69
+ versionId: string;
70
+ mode: WorkerResultMode;
71
+ } | null;
72
+ };
73
+ export {};