streamby-core 0.39.0 β 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/LICENSE +661 -661
- package/README.md +680 -573
- package/dist/adapters/database/connectionManager.js +179 -124
- package/dist/adapters/database/sql.js +1 -1
- package/dist/mcp/tools.js +41 -0
- package/dist/middleware/createRouter.js +11 -1
- package/dist/middleware/routes/ai.js +3 -3
- package/dist/middleware/routes/deliverable.d.ts +4 -2
- package/dist/middleware/routes/deliverable.js +16 -7
- package/dist/middleware/routes/media.js +7 -6
- package/dist/middleware/routes/project.js +5 -1
- package/dist/middleware/routes/renderFarm.js +4 -77
- package/dist/middleware/routes/storage.js +13 -27
- package/dist/middleware/routes/worker.d.ts +7 -0
- package/dist/middleware/routes/worker.js +125 -0
- package/dist/middleware/routes/workerCallback.d.ts +5 -0
- package/dist/middleware/routes/workerCallback.js +31 -0
- package/dist/models/Model.js +9 -9
- package/dist/services/aiProcessor.js +11 -11
- package/dist/services/apiTokenPermissions.js +6 -5
- package/dist/services/executor.js +10 -10
- package/dist/services/jobQueue.d.ts +6 -4
- package/dist/services/jobQueue.js +28 -16
- package/dist/services/jobStore.d.ts +27 -0
- package/dist/services/jobStore.js +177 -0
- package/dist/services/mediaProcessor.js +16 -16
- package/dist/services/vfxProcessor.js +12 -12
- package/dist/services/workerJobs.d.ts +73 -0
- package/dist/services/workerJobs.js +459 -0
- package/dist/services/workers.d.ts +78 -0
- package/dist/services/workers.js +179 -0
- package/dist/types/index.d.ts +52 -2
- package/dist/utils/assetCategories.d.ts +4 -0
- package/dist/utils/assetCategories.js +21 -0
- package/dist/utils/builtinAccess.d.ts +1 -1
- package/dist/utils/builtinAccess.js +4 -3
- package/dist/utils/privateAddress.d.ts +10 -0
- package/dist/utils/privateAddress.js +85 -0
- package/package.json +1 -1
- package/dist/middleware/routes/apiConnection.d.ts +0 -3
- package/dist/middleware/routes/apiConnection.js +0 -100
- package/dist/middleware/routes/credential.d.ts +0 -3
- package/dist/middleware/routes/credential.js +0 -80
- package/dist/services/apiConnection.d.ts +0 -18
- package/dist/services/apiConnection.js +0 -65
- package/dist/services/credential.d.ts +0 -8
- package/dist/services/credential.js +0 -70
- package/dist/services/pipeline.d.ts +0 -2
- package/dist/services/pipeline.js +0 -292
- package/dist/utils/crypto.d.ts +0 -3
- package/dist/utils/crypto.js +0 -32
package/README.md
CHANGED
|
@@ -1,573 +1,680 @@
|
|
|
1
|
-
# streamby-core
|
|
2
|
-
|
|
3
|
-
[](https://www.gnu.org/licenses/agpl-3.0)
|
|
4
|
-
|
|
5
|
-
Middleware framework for building storage-agnostic media APIs.
|
|
6
|
-
Part of the [StreamBy](https://streamby.nhexa.cl) ecosystem by TerminalCore Labs.
|
|
7
|
-
|
|
8
|
-
---
|
|
9
|
-
|
|
10
|
-
## π What is it?
|
|
11
|
-
|
|
12
|
-
`streamby-core` is a plug-and-play middleware for Express (or compatible frameworks) that enables file uploads, listings and **multi-project access** over services like:
|
|
13
|
-
|
|
14
|
-
- β
AWS S3
|
|
15
|
-
- β
Google Cloud Storage (soon)
|
|
16
|
-
- β
Cloudflare R2 (soon)
|
|
17
|
-
- β
Local/Personal servers (soon)
|
|
18
|
-
|
|
19
|
-
It's designed to be installed inside your existing API as a library β **no need to run a separate backend**.
|
|
20
|
-
|
|
21
|
-
---
|
|
22
|
-
|
|
23
|
-
## π¦ Installation
|
|
24
|
-
|
|
25
|
-
```bash
|
|
26
|
-
npm install streamby-core ws
|
|
27
|
-
npm install -D @types/ws
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
---
|
|
31
|
-
|
|
32
|
-
## π§± Basic usage
|
|
33
|
-
|
|
34
|
-
```ts
|
|
35
|
-
import http from 'http';
|
|
36
|
-
import express from 'express';
|
|
37
|
-
import { WebSocketServer } from 'ws';
|
|
38
|
-
import { createStreamByRouter } from 'streamby-core';
|
|
39
|
-
|
|
40
|
-
const app = express();
|
|
41
|
-
const server = http.createServer(app);
|
|
42
|
-
const wss = new WebSocketServer({ server, path: '/streamby/ws' });
|
|
43
|
-
|
|
44
|
-
app.use('/streamby', express.json(), createStreamByRouter({
|
|
45
|
-
authProvider: async (req) => {
|
|
46
|
-
// validate token/session and return user info
|
|
47
|
-
return { userId: 'user-id', username: 'username', role: 'admin' };
|
|
48
|
-
},
|
|
49
|
-
databases: [
|
|
50
|
-
{ id: 'mongo', type: 'nosql', connectionString: process.env.MONGO_URI! },
|
|
51
|
-
{ id: 'postgres', type: 'sql', connectionString: process.env.POSTGRES_URI!, main: true },
|
|
52
|
-
],
|
|
53
|
-
storageProviders: [
|
|
54
|
-
{
|
|
55
|
-
type: 's3',
|
|
56
|
-
config: {
|
|
57
|
-
bucket: process.env.AWS_BUCKET!,
|
|
58
|
-
region: process.env.AWS_REGION!,
|
|
59
|
-
accessKeyId: process.env.AWS_ACCESS_KEY!,
|
|
60
|
-
secretAccessKey: process.env.AWS_SECRET_KEY!,
|
|
61
|
-
},
|
|
62
|
-
},
|
|
63
|
-
],
|
|
64
|
-
encrypt: process.env.STREAMBY_ENCRYPTION_KEY, // 64 hex characters; see "Encryption and key rotation"
|
|
65
|
-
websocket: { server: wss },
|
|
66
|
-
}));
|
|
67
|
-
|
|
68
|
-
server.listen(3000);
|
|
69
|
-
```
|
|
70
|
-
|
|
71
|
-
> **Important:** use `http.createServer(app)` and pass `server.listen()` instead of `app.listen()`.
|
|
72
|
-
> This is required so the `WebSocketServer` shares the same port as the HTTP server.
|
|
73
|
-
|
|
74
|
-
---
|
|
75
|
-
|
|
76
|
-
## π Encryption and key rotation
|
|
77
|
-
|
|
78
|
-
Credentials stored by StreamBy (connection secrets, export credentials, user integrations,
|
|
79
|
-
distribution and render farm credentials) are encrypted with the key you pass as `encrypt`:
|
|
80
|
-
32 bytes, written as 64 hexadecimal characters. Each stored value records the id of the key
|
|
81
|
-
that encrypted it (a fingerprint, never the key itself), so you can rotate the key:
|
|
82
|
-
|
|
83
|
-
1. Generate a new key, for example with `openssl rand -hex 32`.
|
|
84
|
-
2. Pass the new key as `encrypt` and the previous one in `encryptPreviousKeys`.
|
|
85
|
-
3. Restart. Values encrypted with a previous key still decrypt, and at startup StreamBy
|
|
86
|
-
re-encrypts all of them with the new key.
|
|
87
|
-
4. Once that's done (the startup log says how many records it re-encrypted, and none are left
|
|
88
|
-
on the old key), you can remove the old key from `encryptPreviousKeys`.
|
|
89
|
-
|
|
90
|
-
```ts
|
|
91
|
-
createStreamByRouter({
|
|
92
|
-
// ...
|
|
93
|
-
encrypt: process.env.STREAMBY_ENCRYPTION_KEY, // the current key
|
|
94
|
-
encryptPreviousKeys: [process.env.STREAMBY_PREVIOUS_KEY!], // only while rotating
|
|
95
|
-
});
|
|
96
|
-
```
|
|
97
|
-
|
|
98
|
-
> Upgrading from a version before 0.34.0: deploy with your current key first, and rotate only
|
|
99
|
-
> after that deploy (see the [changelog](CHANGELOG.md)).
|
|
100
|
-
|
|
101
|
-
---
|
|
102
|
-
|
|
103
|
-
## π« API tokens
|
|
104
|
-
|
|
105
|
-
Agents and services work with StreamBy through API tokens, without a browser session. There
|
|
106
|
-
are two kinds:
|
|
107
|
-
|
|
108
|
-
- **A user's token** acts as its user. The user manages their own (`GET`, `POST` and
|
|
109
|
-
`DELETE /streamby/user/api-tokens`). It reaches a list of the user's projects, or `'*'` for
|
|
110
|
-
every project they're a member of, including ones they join later. It never does more than
|
|
111
|
-
its user can: their role in each project still applies.
|
|
112
|
-
- **A project's token** acts on its project alone, as the project's, not as whoever made it:
|
|
113
|
-
it keeps working after they leave the project, for permanent integrations (CI, services, the
|
|
114
|
-
project's agents). Only the project's admins list, make and revoke them (`GET`, `POST` and
|
|
115
|
-
`DELETE /streamby/projects/:id/api-tokens`), and each one records who made it. When they
|
|
116
|
-
leave, the project's admins are notified that it still works. Deleting the project ends
|
|
117
|
-
its tokens.
|
|
118
|
-
|
|
119
|
-
Tokens are managed only with a session: a token can't manage tokens.
|
|
120
|
-
|
|
121
|
-
Each token has a name, **its permissions**, and **an expiry** (7, 30, 90 or 365 days, or
|
|
122
|
-
none). It's `sbt_` followed by 32 random bytes, shown once, when it's made. Only its SHA-256
|
|
123
|
-
hash is stored, with a short prefix to tell it apart.
|
|
124
|
-
|
|
125
|
-
**Permissions**
|
|
126
|
-
|
|
127
|
-
| Resource | Permissions | Covers |
|
|
128
|
-
|---|---|---|
|
|
129
|
-
| Project | `project:read`, `project:update`, `project:delete` | The project, its settings and its connections (API, database, storage, distribution, render farm), integrations and allowed origins. |
|
|
130
|
-
| Members | `members:read`, `members:manage` | Listing members; inviting, changing and removing them, and searching users. |
|
|
131
|
-
| Assets | `assets:read`, `assets:write`, `assets:delete` | Files and folders, upload URLs, metadata, versions, reviews and annotations, QC reports. |
|
|
132
|
-
| Production | `production:read`, `production:write`, `production:delete` | Sequences, shots and tasks. |
|
|
133
|
-
| Workflow | `workflow:read`, `workflow:write` | The workflow and its pipelines. |
|
|
134
|
-
| Exports | `exports:read`, `exports:write` | The project's exports. |
|
|
135
|
-
| Rights | `rights:read`, `rights:write` | Assets' rights. |
|
|
136
|
-
| Jobs | `jobs:read`, `jobs:run` | Following jobs; starting them (transcoding, rendering, AI, publishing a deliverable). |
|
|
137
|
-
| Account | `projects:create` | Creating projects. A user's token only; the token reaches what it creates. |
|
|
138
|
-
|
|
139
|
-
`PROJECT_PERMISSIONS` and `ACCOUNT_PERMISSIONS` are exported. A route no permission covers
|
|
140
|
-
(the user's notifications, integrations and tokens, Explore, accepting an invitation) isn't
|
|
141
|
-
reachable with a token.
|
|
142
|
-
|
|
143
|
-
**Resolving a token**
|
|
144
|
-
|
|
145
|
-
Requests send it as `Authorization: Bearer sbt_...`. Your `authProvider` resolves it with
|
|
146
|
-
`resolveApiToken`, and returns its `userId` with the token's scope in `apiToken`. A project's
|
|
147
|
-
token isn't a user: its `userId` is the token's own (on its project), and it goes by its name.
|
|
148
|
-
|
|
149
|
-
```ts
|
|
150
|
-
import { createStreamByRouter, resolveApiToken, API_TOKEN_PREFIX } from 'streamby-core';
|
|
151
|
-
|
|
152
|
-
createStreamByRouter({
|
|
153
|
-
// ...
|
|
154
|
-
authProvider: async (req) => {
|
|
155
|
-
const bearer = req.headers.authorization?.replace(/^Bearer /, '');
|
|
156
|
-
if (bearer?.startsWith(API_TOKEN_PREFIX)) {
|
|
157
|
-
const resolved = await resolveApiToken(bearer); // null once revoked or expired
|
|
158
|
-
if (!resolved) throw new Error('Invalid API token');
|
|
159
|
-
if (resolved.scope.kind === 'project') {
|
|
160
|
-
return { userId: resolved.userId, username: resolved.name, role: 'viewer', apiToken: resolved.scope };
|
|
161
|
-
}
|
|
162
|
-
const user = await findUser(resolved.userId); // your own user lookup
|
|
163
|
-
return { userId: user.id, username: user.username, role: user.role, apiToken: resolved.scope };
|
|
164
|
-
}
|
|
165
|
-
// ...your session-based authentication
|
|
166
|
-
},
|
|
167
|
-
});
|
|
168
|
-
```
|
|
169
|
-
|
|
170
|
-
A request made with a token goes through only when its route's permission is among the
|
|
171
|
-
token's (otherwise 403), and a project outside the token's projects answers 404; the project
|
|
172
|
-
list shows only the token's projects.
|
|
173
|
-
|
|
174
|
-
---
|
|
175
|
-
|
|
176
|
-
## π€ MCP server
|
|
177
|
-
|
|
178
|
-
StreamBy serves an [MCP](https://modelcontextprotocol.io) server at `POST /streamby/mcp`
|
|
179
|
-
(Streamable HTTP, stateless). Agents such as Claude Code use it to read and work on StreamBy
|
|
180
|
-
projects with an API token (see above). It takes only an API token; a browser session gets 401.
|
|
181
|
-
|
|
182
|
-
Each tool calls StreamBy's own REST API with the caller's token. So it reaches exactly what the
|
|
183
|
-
token allows (and, for a user's token, what its user's role in each project allows), and a
|
|
184
|
-
change is saved just like one made from the UI, which shows it when it loads the project.
|
|
185
|
-
|
|
186
|
-
**Tools**
|
|
187
|
-
|
|
188
|
-
A token sees only the tools whose permissions it has.
|
|
189
|
-
|
|
190
|
-
| Tools | Permissions |
|
|
191
|
-
|---|---|
|
|
192
|
-
| `list_projects` | any token |
|
|
193
|
-
| `get_project` | `project:read` |
|
|
194
|
-
| `create_project` | `projects:create` |
|
|
195
|
-
| `delete_project` | `project:delete` |
|
|
196
|
-
| `list_assets` (by category, optionally in one storage connection, and filtered by name), `get_asset_metadata` | `assets:read` |
|
|
197
|
-
| `update_asset_metadata` | `assets:write` |
|
|
198
|
-
| `delete_asset` | `assets:delete` |
|
|
199
|
-
| `get_asset_rights` / `set_asset_rights` | `rights:read` / `rights:write` |
|
|
200
|
-
| `get_workflow`, `list_pipelines`, `get_pipeline` | `workflow:read` |
|
|
201
|
-
| `create_pipeline`, `update_pipeline` | `workflow:write` |
|
|
202
|
-
| `list_exports` | `project:read` and `exports:read` |
|
|
203
|
-
| `get_export` | `exports:read` |
|
|
204
|
-
| `get_job` | `jobs:read` |
|
|
205
|
-
| `
|
|
206
|
-
| `
|
|
207
|
-
| `
|
|
208
|
-
| `
|
|
209
|
-
| `
|
|
210
|
-
| `
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
bpy
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
the
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
```
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
"
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
"
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
},
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
{
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
}
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
1
|
+
# streamby-core
|
|
2
|
+
|
|
3
|
+
[](https://www.gnu.org/licenses/agpl-3.0)
|
|
4
|
+
|
|
5
|
+
Middleware framework for building storage-agnostic media APIs.
|
|
6
|
+
Part of the [StreamBy](https://streamby.nhexa.cl) ecosystem by TerminalCore Labs.
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
## π What is it?
|
|
11
|
+
|
|
12
|
+
`streamby-core` is a plug-and-play middleware for Express (or compatible frameworks) that enables file uploads, listings and **multi-project access** over services like:
|
|
13
|
+
|
|
14
|
+
- β
AWS S3
|
|
15
|
+
- β
Google Cloud Storage (soon)
|
|
16
|
+
- β
Cloudflare R2 (soon)
|
|
17
|
+
- β
Local/Personal servers (soon)
|
|
18
|
+
|
|
19
|
+
It's designed to be installed inside your existing API as a library β **no need to run a separate backend**.
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
## π¦ Installation
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
npm install streamby-core ws
|
|
27
|
+
npm install -D @types/ws
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
---
|
|
31
|
+
|
|
32
|
+
## π§± Basic usage
|
|
33
|
+
|
|
34
|
+
```ts
|
|
35
|
+
import http from 'http';
|
|
36
|
+
import express from 'express';
|
|
37
|
+
import { WebSocketServer } from 'ws';
|
|
38
|
+
import { createStreamByRouter } from 'streamby-core';
|
|
39
|
+
|
|
40
|
+
const app = express();
|
|
41
|
+
const server = http.createServer(app);
|
|
42
|
+
const wss = new WebSocketServer({ server, path: '/streamby/ws' });
|
|
43
|
+
|
|
44
|
+
app.use('/streamby', express.json(), createStreamByRouter({
|
|
45
|
+
authProvider: async (req) => {
|
|
46
|
+
// validate token/session and return user info
|
|
47
|
+
return { userId: 'user-id', username: 'username', role: 'admin' };
|
|
48
|
+
},
|
|
49
|
+
databases: [
|
|
50
|
+
{ id: 'mongo', type: 'nosql', connectionString: process.env.MONGO_URI! },
|
|
51
|
+
{ id: 'postgres', type: 'sql', connectionString: process.env.POSTGRES_URI!, main: true },
|
|
52
|
+
],
|
|
53
|
+
storageProviders: [
|
|
54
|
+
{
|
|
55
|
+
type: 's3',
|
|
56
|
+
config: {
|
|
57
|
+
bucket: process.env.AWS_BUCKET!,
|
|
58
|
+
region: process.env.AWS_REGION!,
|
|
59
|
+
accessKeyId: process.env.AWS_ACCESS_KEY!,
|
|
60
|
+
secretAccessKey: process.env.AWS_SECRET_KEY!,
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
],
|
|
64
|
+
encrypt: process.env.STREAMBY_ENCRYPTION_KEY, // 64 hex characters; see "Encryption and key rotation"
|
|
65
|
+
websocket: { server: wss },
|
|
66
|
+
}));
|
|
67
|
+
|
|
68
|
+
server.listen(3000);
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
> **Important:** use `http.createServer(app)` and pass `server.listen()` instead of `app.listen()`.
|
|
72
|
+
> This is required so the `WebSocketServer` shares the same port as the HTTP server.
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
76
|
+
## π Encryption and key rotation
|
|
77
|
+
|
|
78
|
+
Credentials stored by StreamBy (connection secrets, export credentials, user integrations,
|
|
79
|
+
distribution and render farm credentials) are encrypted with the key you pass as `encrypt`:
|
|
80
|
+
32 bytes, written as 64 hexadecimal characters. Each stored value records the id of the key
|
|
81
|
+
that encrypted it (a fingerprint, never the key itself), so you can rotate the key:
|
|
82
|
+
|
|
83
|
+
1. Generate a new key, for example with `openssl rand -hex 32`.
|
|
84
|
+
2. Pass the new key as `encrypt` and the previous one in `encryptPreviousKeys`.
|
|
85
|
+
3. Restart. Values encrypted with a previous key still decrypt, and at startup StreamBy
|
|
86
|
+
re-encrypts all of them with the new key.
|
|
87
|
+
4. Once that's done (the startup log says how many records it re-encrypted, and none are left
|
|
88
|
+
on the old key), you can remove the old key from `encryptPreviousKeys`.
|
|
89
|
+
|
|
90
|
+
```ts
|
|
91
|
+
createStreamByRouter({
|
|
92
|
+
// ...
|
|
93
|
+
encrypt: process.env.STREAMBY_ENCRYPTION_KEY, // the current key
|
|
94
|
+
encryptPreviousKeys: [process.env.STREAMBY_PREVIOUS_KEY!], // only while rotating
|
|
95
|
+
});
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
> Upgrading from a version before 0.34.0: deploy with your current key first, and rotate only
|
|
99
|
+
> after that deploy (see the [changelog](CHANGELOG.md)).
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## π« API tokens
|
|
104
|
+
|
|
105
|
+
Agents and services work with StreamBy through API tokens, without a browser session. There
|
|
106
|
+
are two kinds:
|
|
107
|
+
|
|
108
|
+
- **A user's token** acts as its user. The user manages their own (`GET`, `POST` and
|
|
109
|
+
`DELETE /streamby/user/api-tokens`). It reaches a list of the user's projects, or `'*'` for
|
|
110
|
+
every project they're a member of, including ones they join later. It never does more than
|
|
111
|
+
its user can: their role in each project still applies.
|
|
112
|
+
- **A project's token** acts on its project alone, as the project's, not as whoever made it:
|
|
113
|
+
it keeps working after they leave the project, for permanent integrations (CI, services, the
|
|
114
|
+
project's agents). Only the project's admins list, make and revoke them (`GET`, `POST` and
|
|
115
|
+
`DELETE /streamby/projects/:id/api-tokens`), and each one records who made it. When they
|
|
116
|
+
leave, the project's admins are notified that it still works. Deleting the project ends
|
|
117
|
+
its tokens.
|
|
118
|
+
|
|
119
|
+
Tokens are managed only with a session: a token can't manage tokens.
|
|
120
|
+
|
|
121
|
+
Each token has a name, **its permissions**, and **an expiry** (7, 30, 90 or 365 days, or
|
|
122
|
+
none). It's `sbt_` followed by 32 random bytes, shown once, when it's made. Only its SHA-256
|
|
123
|
+
hash is stored, with a short prefix to tell it apart.
|
|
124
|
+
|
|
125
|
+
**Permissions**
|
|
126
|
+
|
|
127
|
+
| Resource | Permissions | Covers |
|
|
128
|
+
|---|---|---|
|
|
129
|
+
| Project | `project:read`, `project:update`, `project:delete` | The project, its settings and its connections (API, database, storage, distribution, render farm), integrations and allowed origins. |
|
|
130
|
+
| Members | `members:read`, `members:manage` | Listing members; inviting, changing and removing them, and searching users. |
|
|
131
|
+
| Assets | `assets:read`, `assets:write`, `assets:delete` | Files and folders, upload URLs, metadata, versions, reviews and annotations, QC reports. |
|
|
132
|
+
| Production | `production:read`, `production:write`, `production:delete` | Sequences, shots and tasks. |
|
|
133
|
+
| Workflow | `workflow:read`, `workflow:write` | The workflow and its pipelines. |
|
|
134
|
+
| Exports | `exports:read`, `exports:write` | The project's exports. |
|
|
135
|
+
| Rights | `rights:read`, `rights:write` | Assets' rights. |
|
|
136
|
+
| Jobs | `jobs:read`, `jobs:run` | Following jobs; starting them (transcoding, rendering, AI, publishing a deliverable). |
|
|
137
|
+
| Account | `projects:create` | Creating projects. A user's token only; the token reaches what it creates. |
|
|
138
|
+
|
|
139
|
+
`PROJECT_PERMISSIONS` and `ACCOUNT_PERMISSIONS` are exported. A route no permission covers
|
|
140
|
+
(the user's notifications, integrations and tokens, Explore, accepting an invitation) isn't
|
|
141
|
+
reachable with a token.
|
|
142
|
+
|
|
143
|
+
**Resolving a token**
|
|
144
|
+
|
|
145
|
+
Requests send it as `Authorization: Bearer sbt_...`. Your `authProvider` resolves it with
|
|
146
|
+
`resolveApiToken`, and returns its `userId` with the token's scope in `apiToken`. A project's
|
|
147
|
+
token isn't a user: its `userId` is the token's own (on its project), and it goes by its name.
|
|
148
|
+
|
|
149
|
+
```ts
|
|
150
|
+
import { createStreamByRouter, resolveApiToken, API_TOKEN_PREFIX } from 'streamby-core';
|
|
151
|
+
|
|
152
|
+
createStreamByRouter({
|
|
153
|
+
// ...
|
|
154
|
+
authProvider: async (req) => {
|
|
155
|
+
const bearer = req.headers.authorization?.replace(/^Bearer /, '');
|
|
156
|
+
if (bearer?.startsWith(API_TOKEN_PREFIX)) {
|
|
157
|
+
const resolved = await resolveApiToken(bearer); // null once revoked or expired
|
|
158
|
+
if (!resolved) throw new Error('Invalid API token');
|
|
159
|
+
if (resolved.scope.kind === 'project') {
|
|
160
|
+
return { userId: resolved.userId, username: resolved.name, role: 'viewer', apiToken: resolved.scope };
|
|
161
|
+
}
|
|
162
|
+
const user = await findUser(resolved.userId); // your own user lookup
|
|
163
|
+
return { userId: user.id, username: user.username, role: user.role, apiToken: resolved.scope };
|
|
164
|
+
}
|
|
165
|
+
// ...your session-based authentication
|
|
166
|
+
},
|
|
167
|
+
});
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
A request made with a token goes through only when its route's permission is among the
|
|
171
|
+
token's (otherwise 403), and a project outside the token's projects answers 404; the project
|
|
172
|
+
list shows only the token's projects.
|
|
173
|
+
|
|
174
|
+
---
|
|
175
|
+
|
|
176
|
+
## π€ MCP server
|
|
177
|
+
|
|
178
|
+
StreamBy serves an [MCP](https://modelcontextprotocol.io) server at `POST /streamby/mcp`
|
|
179
|
+
(Streamable HTTP, stateless). Agents such as Claude Code use it to read and work on StreamBy
|
|
180
|
+
projects with an API token (see above). It takes only an API token; a browser session gets 401.
|
|
181
|
+
|
|
182
|
+
Each tool calls StreamBy's own REST API with the caller's token. So it reaches exactly what the
|
|
183
|
+
token allows (and, for a user's token, what its user's role in each project allows), and a
|
|
184
|
+
change is saved just like one made from the UI, which shows it when it loads the project.
|
|
185
|
+
|
|
186
|
+
**Tools**
|
|
187
|
+
|
|
188
|
+
A token sees only the tools whose permissions it has.
|
|
189
|
+
|
|
190
|
+
| Tools | Permissions |
|
|
191
|
+
|---|---|
|
|
192
|
+
| `list_projects` | any token |
|
|
193
|
+
| `get_project` | `project:read` |
|
|
194
|
+
| `create_project` | `projects:create` |
|
|
195
|
+
| `delete_project` | `project:delete` |
|
|
196
|
+
| `list_assets` (by category, optionally in one storage connection, and filtered by name), `get_asset_metadata` | `assets:read` |
|
|
197
|
+
| `update_asset_metadata` | `assets:write` |
|
|
198
|
+
| `delete_asset` | `assets:delete` |
|
|
199
|
+
| `get_asset_rights` / `set_asset_rights` | `rights:read` / `rights:write` |
|
|
200
|
+
| `get_workflow`, `list_pipelines`, `get_pipeline` | `workflow:read` |
|
|
201
|
+
| `create_pipeline`, `update_pipeline` | `workflow:write` |
|
|
202
|
+
| `list_exports` | `project:read` and `exports:read` |
|
|
203
|
+
| `get_export` | `exports:read` |
|
|
204
|
+
| `get_job`, `list_jobs` | `jobs:read` |
|
|
205
|
+
| `list_workers` | `project:read` |
|
|
206
|
+
| `process_asset`, `retry_job` | `jobs:run` |
|
|
207
|
+
| `list_sequences`, `list_shots`, `list_tasks` | `production:read` |
|
|
208
|
+
| `update_sequence`, `update_shot`, `update_task` | `production:write` |
|
|
209
|
+
| `create_sequence`, `create_shot`, `create_task` | `production:write` and `workflow:write` |
|
|
210
|
+
| `delete_sequence`, `delete_shot`, `delete_task` | `production:delete` |
|
|
211
|
+
| `list_asset_versions`, `get_asset_download_url`, `list_deliverables` | `assets:read` |
|
|
212
|
+
| `request_asset_upload`, `mark_deliverable`, `remove_deliverable` | `assets:write` |
|
|
213
|
+
|
|
214
|
+
The tools take the path a user does in the UI, and each one's description says where its
|
|
215
|
+
result shows there. Production data lives under a production node of the project's workflow:
|
|
216
|
+
if the workflow has none yet, `create_sequence`, `create_shot` and `create_task` first add a
|
|
217
|
+
Shot node to it, as dropping one on the canvas does (so they need `workflow:write`), and make
|
|
218
|
+
nothing if it can't be added.
|
|
219
|
+
|
|
220
|
+
The tools that delete are marked as destructive (`destructiveHint`), so a client can ask before
|
|
221
|
+
running them.
|
|
222
|
+
|
|
223
|
+
**Files**
|
|
224
|
+
|
|
225
|
+
A file never goes through the conversation. `request_asset_upload` makes a new asset (or a new
|
|
226
|
+
version of one) and gives a signed URL to `PUT` its bytes to; `get_asset_download_url` gives a
|
|
227
|
+
signed URL to `GET` an asset's file, its latest version or any earlier one. The tool that has
|
|
228
|
+
the file uploads or downloads it directly, for example Blender's or Godot's own MCP server,
|
|
229
|
+
which can run code. Each URL is for that one file, and given to an API token it works only
|
|
230
|
+
briefly: 15 minutes to upload, 5 to download (`mcp.signedUrlTtl`).
|
|
231
|
+
|
|
232
|
+
**Versions.** Replacing an asset doesn't change its URL: the new version is uploaded to the
|
|
233
|
+
asset's own key, so a site that links to the asset shows the new one with no deploy. Before
|
|
234
|
+
that, the current version is copied to a key of its own (`<key>.v<n>.<ext>`) and recorded, so
|
|
235
|
+
every earlier version can still be downloaded by its id; if it can't be copied, nothing is
|
|
236
|
+
replaced. A storage adapter keeps versions only if it can copy (`copyFile`; the S3 one does);
|
|
237
|
+
otherwise asking for a new version answers 501. Deleting the asset deletes every key of it.
|
|
238
|
+
|
|
239
|
+
Since the URL stays the same, a browser or a CDN may keep serving the earlier version for a
|
|
240
|
+
while. StreamBy doesn't set `Cache-Control` on uploads (a signed URL that did would require
|
|
241
|
+
every client to send the same header): set it where the files are served, for example a short
|
|
242
|
+
`max-age` or `no-cache` on the CDN in front of the bucket, or invalidate the path when an asset
|
|
243
|
+
is replaced.
|
|
244
|
+
|
|
245
|
+
A **deliverable** is an asset, or one of its versions, delivered for a
|
|
246
|
+
production shot or for an export; a shot or an export can have several, and the shot's panel
|
|
247
|
+
in the UI shows them.
|
|
248
|
+
|
|
249
|
+
For example, with Claude connected to StreamBy and to Blender's MCP server, asking *"export the
|
|
250
|
+
scene as glTF and deliver it for shot SH010 of Portfolio"* goes:
|
|
251
|
+
|
|
252
|
+
1. `list_projects`, `list_sequences`, `list_shots`: the project and the shot's id.
|
|
253
|
+
2. `request_asset_upload` with `category: '3d-models'`, `fileName: 'SH010_layout.glb'` and
|
|
254
|
+
`contentType: 'model/gltf-binary'`: the new asset's id, and where to upload it.
|
|
255
|
+
3. In Blender, code that exports the scene and uploads it:
|
|
256
|
+
|
|
257
|
+
```python
|
|
258
|
+
import bpy, urllib.request
|
|
259
|
+
path = bpy.app.tempdir + "SH010_layout.glb"
|
|
260
|
+
bpy.ops.export_scene.gltf(filepath=path, export_format="GLB")
|
|
261
|
+
with open(path, "rb") as f:
|
|
262
|
+
req = urllib.request.Request(UPLOAD_URL, data=f.read(), method="PUT",
|
|
263
|
+
headers={"Content-Type": "model/gltf-binary"})
|
|
264
|
+
urllib.request.urlopen(req)
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
4. `mark_deliverable` with the asset's id and `shotId`: it shows in the shot's panel.
|
|
268
|
+
|
|
269
|
+
Bringing an asset into Blender or Godot is the other way around: `get_asset_download_url`, and
|
|
270
|
+
code there that downloads the URL and imports the file (in Blender,
|
|
271
|
+
`bpy.ops.import_scene.gltf(filepath=...)`). A new version of the same asset is
|
|
272
|
+
`request_asset_upload` with its `assetId`.
|
|
273
|
+
|
|
274
|
+
**Limits and audit**
|
|
275
|
+
|
|
276
|
+
- Each token can make at most 120 requests a minute (a sliding window). Past that, StreamBy
|
|
277
|
+
answers 429 with `Retry-After`. Each instance counts its own requests. Change the limit with
|
|
278
|
+
`mcp.rateLimit`.
|
|
279
|
+
- Every tool call is recorded in the `mcp_audit` table, created at startup in the main database.
|
|
280
|
+
Each row has the user (a project's token: the token itself), the token's id, the project, the
|
|
281
|
+
tool, whether it went through, the status, the error and the duration. A row never includes
|
|
282
|
+
the arguments or the token.
|
|
283
|
+
|
|
284
|
+
```ts
|
|
285
|
+
createStreamByRouter({
|
|
286
|
+
// ...
|
|
287
|
+
mcp: {
|
|
288
|
+
rateLimit: { requests: 120, windowMs: 60_000 }, // the default
|
|
289
|
+
signedUrlTtl: { upload: 900, download: 300 }, // the default, in seconds
|
|
290
|
+
// apiBaseUrl: 'http://127.0.0.1:3000/streamby', // where tools reach the REST API;
|
|
291
|
+
// by default this same process, at the request's local port and mount path
|
|
292
|
+
},
|
|
293
|
+
});
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
### Connecting from Claude Code
|
|
297
|
+
|
|
298
|
+
Make an API token in StreamBy, with only the permissions the agent needs: your own
|
|
299
|
+
(Settings β Security β Personal API tokens), or a project's (in the project, Settings β API
|
|
300
|
+
Tokens). Keep it in an environment variable, never in a file:
|
|
301
|
+
|
|
302
|
+
```bash
|
|
303
|
+
export STREAMBY_TOKEN=sbt_...
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
Add the server to your project's `.mcp.json`. Claude Code expands `${STREAMBY_TOKEN}` from the
|
|
307
|
+
environment:
|
|
308
|
+
|
|
309
|
+
```json
|
|
310
|
+
{
|
|
311
|
+
"mcpServers": {
|
|
312
|
+
"streamby": {
|
|
313
|
+
"type": "http",
|
|
314
|
+
"url": "https://your-host/streamby/mcp",
|
|
315
|
+
"headers": { "Authorization": "Bearer ${STREAMBY_TOKEN}" }
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
### Connecting from Claude Desktop
|
|
322
|
+
|
|
323
|
+
Claude Desktop runs local (stdio) servers from `claude_desktop_config.json`. Reach StreamBy
|
|
324
|
+
through [`mcp-remote`](https://www.npmjs.com/package/mcp-remote), passing the token in its
|
|
325
|
+
`env`. This file holds it in plain text, so use a token with only the permissions you need,
|
|
326
|
+
and an expiry:
|
|
327
|
+
|
|
328
|
+
```json
|
|
329
|
+
{
|
|
330
|
+
"mcpServers": {
|
|
331
|
+
"streamby": {
|
|
332
|
+
"command": "npx",
|
|
333
|
+
"args": ["mcp-remote", "https://your-host/streamby/mcp", "--header", "Authorization:${STREAMBY_AUTH}"],
|
|
334
|
+
"env": { "STREAMBY_AUTH": "Bearer sbt_..." }
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
```
|
|
339
|
+
|
|
340
|
+
The claude.ai connector needs OAuth, which isn't available yet: it's for API tokens only for now.
|
|
341
|
+
|
|
342
|
+
---
|
|
343
|
+
|
|
344
|
+
## βοΈ Processing workers
|
|
345
|
+
|
|
346
|
+
StreamBy doesn't process assets itself (optimizing a glTF, making LODs, transcoding...): it
|
|
347
|
+
sends the work to a **worker**, an external service, and keeps what it makes. Anyone can write a
|
|
348
|
+
compatible worker by following the contract below.
|
|
349
|
+
|
|
350
|
+
**Where workers come from**
|
|
351
|
+
|
|
352
|
+
- **The host's**, offered to every project, in the router config (each one gated, like the other
|
|
353
|
+
built-ins, by `canUseBuiltin` with kind `'worker'`):
|
|
354
|
+
|
|
355
|
+
```ts
|
|
356
|
+
createStreamByRouter({
|
|
357
|
+
// ...
|
|
358
|
+
workers: [{
|
|
359
|
+
id: 'gltf', name: 'glTF optimizer',
|
|
360
|
+
endpoint: 'https://worker.example.com/jobs', credential: process.env.GLTF_WORKER_TOKEN,
|
|
361
|
+
kinds: ['gltf-optimize', 'lod'], // what it does
|
|
362
|
+
categories: ['3d-models'], // the assets it takes
|
|
363
|
+
autoKind: 'gltf-optimize', // optional: run on its own (see below)
|
|
364
|
+
timeoutSeconds: 3600, // optional: how long it may take a job
|
|
365
|
+
}],
|
|
366
|
+
workerOptions: {
|
|
367
|
+
callbackBaseUrl: 'https://api.example.com/streamby', // where workers call back (see below)
|
|
368
|
+
maxAttempts: 3, // how many times a job is sent before it fails
|
|
369
|
+
sweepIntervalMs: 30_000, // how often each instance looks for jobs to send again
|
|
370
|
+
},
|
|
371
|
+
});
|
|
372
|
+
```
|
|
373
|
+
|
|
374
|
+
- **A project's own**, connected from the project (`GET`, `POST` and
|
|
375
|
+
`DELETE /projects/:projectId/workers`; a project's render farms are workers too). Its endpoint
|
|
376
|
+
must be https and never reach a local or private address: not as written, nor as its name
|
|
377
|
+
resolves, which is checked when it's connected and again right before each job is sent (a
|
|
378
|
+
name can point at the server's own network, or start pointing there later). A job for one
|
|
379
|
+
that resolves to such an address fails for good. `workerOptions.allowLocalEndpoints` lifts
|
|
380
|
+
all this, for development. Its credential is stored encrypted, and used only while whoever
|
|
381
|
+
connected it is still in the project.
|
|
382
|
+
|
|
383
|
+
The host's own workers aren't checked this way: the host chose them, and they may well be on
|
|
384
|
+
its private network.
|
|
385
|
+
|
|
386
|
+
`GET /projects/:projectId/workers` lists the workers a member can use.
|
|
387
|
+
|
|
388
|
+
**Asking for a job**
|
|
389
|
+
|
|
390
|
+
`POST /projects/:projectId/jobs/process` with
|
|
391
|
+
`{ assetId, versionId?, worker: { source: 'builtin' | 'project', id }, kind, profile?, resultMode? }`
|
|
392
|
+
(editors and admins; an API token needs `jobs:run`, and the MCP server has `process_asset`).
|
|
393
|
+
`resultMode` decides where the result goes:
|
|
394
|
+
|
|
395
|
+
- `'derived'` (the default): a new asset, linked to the original (`derivedFrom`), which is never
|
|
396
|
+
touched;
|
|
397
|
+
- `'version'`: the asset's new version, at its own key, so its URL serves the result; the
|
|
398
|
+
version it replaces is archived, and can still be downloaded.
|
|
399
|
+
|
|
400
|
+
A worker with an `autoKind` also runs by itself on an asset of its categories when it's marked
|
|
401
|
+
as a deliverable (by a session, or by an API token that has `jobs:run`: one without it marks the
|
|
402
|
+
deliverable and starts nothing).
|
|
403
|
+
|
|
404
|
+
Jobs are kept in the main database (a `jobs` table or collection): they survive restarts, and
|
|
405
|
+
every instance sees them. `GET /projects/:projectId/jobs` lists them (`?assetId=` for one asset's);
|
|
406
|
+
`POST /projects/:projectId/jobs/:jobId/retry` sends a failed one again.
|
|
407
|
+
|
|
408
|
+
**The contract: what a worker receives**
|
|
409
|
+
|
|
410
|
+
StreamBy sends each job with a `POST` to the worker's endpoint
|
|
411
|
+
(`Authorization: Bearer <the worker's credential>`):
|
|
412
|
+
|
|
413
|
+
```json
|
|
414
|
+
{
|
|
415
|
+
"jobId": "β¦", "kind": "gltf-optimize", "profile": { "textureFormat": "ktx2" }, "attempt": 1,
|
|
416
|
+
"asset": { "id": "β¦", "versionId": null, "fileName": "hero.glb", "contentType": "model/gltf-binary", "category": "3d-models" },
|
|
417
|
+
"download": { "url": "https://β¦signedβ¦", "expiresAt": "β¦" },
|
|
418
|
+
"callback": { "url": "https://api.example.com/streamby/worker-jobs/<jobId>", "token": "β¦" },
|
|
419
|
+
"deadline": "β¦"
|
|
420
|
+
}
|
|
421
|
+
```
|
|
422
|
+
|
|
423
|
+
The worker answers `2xx` to take it (any other answer, or none, and the job is sent again later),
|
|
424
|
+
downloads the asset with `download.url`, and does its work before `deadline`.
|
|
425
|
+
|
|
426
|
+
`callback.url` is `workerOptions.callbackBaseUrl` plus the job's path. Without it, it's made
|
|
427
|
+
from the request that asked for the job (`req.protocol` and its host): behind a proxy that ends
|
|
428
|
+
TLS (and without Express's `trust proxy`), that comes out as `http://`, which a worker can't call
|
|
429
|
+
back to. Set `callbackBaseUrl` to the router's public https URL.
|
|
430
|
+
|
|
431
|
+
**The contract: what a worker calls back**
|
|
432
|
+
|
|
433
|
+
Every callback is a `POST` to `callback.url` plus an action, with
|
|
434
|
+
`Authorization: Bearer <callback.token>`. The token is the job's for this attempt alone: a
|
|
435
|
+
callback without it, with another, or after the job ended answers 401.
|
|
436
|
+
|
|
437
|
+
| Action | Body | Answer |
|
|
438
|
+
|---|---|---|
|
|
439
|
+
| `/progress` | `{ "progress": 0-100, "stage": "optimizing" }` (either) | the job |
|
|
440
|
+
| `/upload-url` | `{ "fileName": "hero.glb", "contentType": "model/gltf-binary" }` (of the asset's category) | `{ "url", "method": "PUT", "headers", "expiresAt" }`: upload the result there |
|
|
441
|
+
| `/complete` | `{ "report": { β¦ } }` (optional; any JSON object, kept with the result) | the job, completed, with its `result`: `{ assetId, versionId, mode }` |
|
|
442
|
+
| `/fail` | `{ "error": "β¦", "retryable": false }` | the job, failed with its error (sent again if `retryable` and attempts are left) |
|
|
443
|
+
|
|
444
|
+
A job its worker doesn't complete before its deadline is sent again, up to
|
|
445
|
+
`workerOptions.maxAttempts` times; then it fails, with its error, and can be retried.
|
|
446
|
+
|
|
447
|
+
---
|
|
448
|
+
|
|
449
|
+
## π₯ Project roles
|
|
450
|
+
|
|
451
|
+
What a user can do in a project depends on their role in that project, as an active member
|
|
452
|
+
(an invitation counts once it's accepted). It never depends on the platform role your
|
|
453
|
+
`authProvider` returns.
|
|
454
|
+
|
|
455
|
+
| Role | Can |
|
|
456
|
+
|---|---|
|
|
457
|
+
| `viewer` | See the project and its content, and archive the project in their own list. |
|
|
458
|
+
| `editor` | Everything a viewer can, plus change the project: its details, connections, exports, files and folders, workflow, pipelines, production, reviews, rights, and start jobs. |
|
|
459
|
+
| `admin` | Everything an editor can, plus manage the project's members and delete the project. |
|
|
460
|
+
|
|
461
|
+
A project's public exports (`GET /streamby/:projectId/export/:exportName`) are read without a
|
|
462
|
+
session. They only run nodes that read data: never a node that uses an account's credential,
|
|
463
|
+
nor one that starts a job.
|
|
464
|
+
|
|
465
|
+
---
|
|
466
|
+
|
|
467
|
+
## ποΈ Database Schema Setup
|
|
468
|
+
|
|
469
|
+
`streamby-core` requires a pre-configured database schema for its internal operations. It does not manage database connections directly; instead, it expects the host application to provide initialized connection instances. This ensures full control and flexibility for the host environment.
|
|
470
|
+
|
|
471
|
+
The library will only interact with the specified schema and will not touch other schemas in your database. The `reset` option is opt-in and will only drop and recreate the specified `streamby` schema if explicitly set to `true`.
|
|
472
|
+
|
|
473
|
+
### PostgreSQL Example
|
|
474
|
+
|
|
475
|
+
```ts
|
|
476
|
+
import { Pool } from "pg";
|
|
477
|
+
import { setupStreambyPg } from "streamby-core/pg/setup"; // Adjust path as per your package structure
|
|
478
|
+
|
|
479
|
+
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
|
480
|
+
|
|
481
|
+
// Call setupStreambyPg to ensure the schema and tables are created
|
|
482
|
+
// reset: false (default) ensures idempotency and no data loss
|
|
483
|
+
await setupStreambyPg({ pool, schema: "streamby", reset: false });
|
|
484
|
+
```
|
|
485
|
+
|
|
486
|
+
### MongoDB Example
|
|
487
|
+
|
|
488
|
+
```ts
|
|
489
|
+
import { MongoClient } from "mongodb";
|
|
490
|
+
import { setupStreambyMongo } from "streamby-core/mongo/setup"; // Adjust path as per your package structure
|
|
491
|
+
|
|
492
|
+
const client = new MongoClient(process.env.MONGO_URL!);
|
|
493
|
+
|
|
494
|
+
// Connect the MongoDB client
|
|
495
|
+
await client.connect();
|
|
496
|
+
|
|
497
|
+
// Call setupStreambyMongo to ensure collections and indexes are created
|
|
498
|
+
await setupStreambyMongo({ client, dbName: "streamby" });
|
|
499
|
+
```
|
|
500
|
+
|
|
501
|
+
```ts
|
|
502
|
+
import express from 'express';
|
|
503
|
+
import { createStreamByRouter } from 'streamby-core';
|
|
504
|
+
|
|
505
|
+
const app = express();
|
|
506
|
+
|
|
507
|
+
app.use('/streamby', createStreamByRouter({
|
|
508
|
+
storageProvider: {
|
|
509
|
+
type: 's3',
|
|
510
|
+
config: {
|
|
511
|
+
bucket: 'your-bucket-name',
|
|
512
|
+
region: 'your-region',
|
|
513
|
+
accessKeyId: 'YOUR_AWS_KEY',
|
|
514
|
+
secretAccessKey: 'YOUR_AWS_SECRET'
|
|
515
|
+
}
|
|
516
|
+
},
|
|
517
|
+
authProvider: async (req) => {
|
|
518
|
+
// extract token/cookie and validate
|
|
519
|
+
return {
|
|
520
|
+
userId: 'demo',
|
|
521
|
+
projects: ['demo-project'],
|
|
522
|
+
role: 'admin',
|
|
523
|
+
};
|
|
524
|
+
},
|
|
525
|
+
projectProvider: async (projectId) => {
|
|
526
|
+
// fetch project from your DB
|
|
527
|
+
return {
|
|
528
|
+
id: projectId,
|
|
529
|
+
name: 'Demo Project',
|
|
530
|
+
description: 'Demo for StreamBy integration',
|
|
531
|
+
rootFolders: [],
|
|
532
|
+
settings: { allowUpload: true }
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
}));
|
|
536
|
+
|
|
537
|
+
app.listen(3000);
|
|
538
|
+
```
|
|
539
|
+
|
|
540
|
+
---
|
|
541
|
+
|
|
542
|
+
## π§ͺ Testing
|
|
543
|
+
|
|
544
|
+
```bash
|
|
545
|
+
npm run test
|
|
546
|
+
```
|
|
547
|
+
|
|
548
|
+
Unit tests use [Vitest](https://vitest.dev) and [Supertest](https://www.npmjs.com/package/supertest).
|
|
549
|
+
|
|
550
|
+
---
|
|
551
|
+
|
|
552
|
+
## π Features
|
|
553
|
+
|
|
554
|
+
- π Multi-project access control
|
|
555
|
+
- π Project-aware file uploads & listings
|
|
556
|
+
- π§Ή Modular adapters per storage provider
|
|
557
|
+
- π§° Built-in testability with mock auth & storage
|
|
558
|
+
- β¨ Simple to extend for custom business logic
|
|
559
|
+
|
|
560
|
+
---
|
|
561
|
+
|
|
562
|
+
## πΊοΈ API Endpoints
|
|
563
|
+
|
|
564
|
+
Here's a summary of the key API endpoints provided by `streamby-core`:
|
|
565
|
+
|
|
566
|
+
### `GET /streamby/projects`
|
|
567
|
+
|
|
568
|
+
Lists all projects accessible by the authenticated user.
|
|
569
|
+
|
|
570
|
+
**Query Parameters:**
|
|
571
|
+
- `archived`: (Optional) Filter projects by their archived status.
|
|
572
|
+
- `true`: Returns only archived projects.
|
|
573
|
+
- `false`: Returns only unarchived projects.
|
|
574
|
+
- (Omitted): Returns all projects (both archived and unarchived).
|
|
575
|
+
|
|
576
|
+
**Example Response:**
|
|
577
|
+
```json
|
|
578
|
+
{
|
|
579
|
+
"projects": [
|
|
580
|
+
{
|
|
581
|
+
"id": "project1_id",
|
|
582
|
+
"dbType": "nosql",
|
|
583
|
+
"name": "My NoSQL Project",
|
|
584
|
+
"image": "url_to_image",
|
|
585
|
+
"archived": false
|
|
586
|
+
},
|
|
587
|
+
{
|
|
588
|
+
"id": "project2_id",
|
|
589
|
+
"dbType": "sql",
|
|
590
|
+
"name": "My SQL Project",
|
|
591
|
+
"image": "url_to_image",
|
|
592
|
+
"archived": true
|
|
593
|
+
}
|
|
594
|
+
]
|
|
595
|
+
}
|
|
596
|
+
```
|
|
597
|
+
|
|
598
|
+
### `PATCH /streamby/projects/:id/archive`
|
|
599
|
+
|
|
600
|
+
Archives a specific project. After archiving, it returns the complete list of projects for the user, including the updated project.
|
|
601
|
+
|
|
602
|
+
**Example Response:**
|
|
603
|
+
```json
|
|
604
|
+
{
|
|
605
|
+
"success": true,
|
|
606
|
+
"projects": [
|
|
607
|
+
{
|
|
608
|
+
"id": "project1_id",
|
|
609
|
+
"dbType": "nosql",
|
|
610
|
+
"name": "My NoSQL Project",
|
|
611
|
+
"image": "url_to_image",
|
|
612
|
+
"archived": true
|
|
613
|
+
},
|
|
614
|
+
{
|
|
615
|
+
"id": "project2_id",
|
|
616
|
+
"dbType": "sql",
|
|
617
|
+
"name": "My SQL Project",
|
|
618
|
+
"image": "url_to_image",
|
|
619
|
+
"archived": false
|
|
620
|
+
}
|
|
621
|
+
]
|
|
622
|
+
}
|
|
623
|
+
```
|
|
624
|
+
|
|
625
|
+
### `PATCH /streamby/projects/:id/unarchive`
|
|
626
|
+
|
|
627
|
+
Unarchives a specific project. After unarchiving, it returns the complete list of projects for the user, including the updated project.
|
|
628
|
+
|
|
629
|
+
**Example Response:**
|
|
630
|
+
```json
|
|
631
|
+
{
|
|
632
|
+
"success": true,
|
|
633
|
+
"projects": [
|
|
634
|
+
{
|
|
635
|
+
"id": "project1_id",
|
|
636
|
+
"dbType": "nosql",
|
|
637
|
+
"name": "My NoSQL Project",
|
|
638
|
+
"image": "url_to_image",
|
|
639
|
+
"archived": false
|
|
640
|
+
},
|
|
641
|
+
{
|
|
642
|
+
"id": "project2_id",
|
|
643
|
+
"dbType": "sql",
|
|
644
|
+
"name": "My SQL Project",
|
|
645
|
+
"image": "url_to_image",
|
|
646
|
+
"archived": false
|
|
647
|
+
}
|
|
648
|
+
]
|
|
649
|
+
}
|
|
650
|
+
```
|
|
651
|
+
|
|
652
|
+
---
|
|
653
|
+
|
|
654
|
+
## π£ Roadmap
|
|
655
|
+
|
|
656
|
+
- [ ] Support for Google Cloud Storage
|
|
657
|
+
- [ ] CLI tool for uploading files
|
|
658
|
+
- [ ] Role-based access control middleware
|
|
659
|
+
- [ ] Plugin system for extended routes
|
|
660
|
+
- [ ] Streaming support (HLS, audio, etc.)
|
|
661
|
+
|
|
662
|
+
---
|
|
663
|
+
|
|
664
|
+
## π§βπ» Maintained by
|
|
665
|
+
|
|
666
|
+
**TerminalCore Labs** β part of [Nhexa Entertainment](https://nhexa.cl)
|
|
667
|
+
Developed as open-source infrastructure for creative and media-oriented apps.
|
|
668
|
+
|
|
669
|
+
Visit us [here](https://terminalcore.cl)
|
|
670
|
+
|
|
671
|
+
---
|
|
672
|
+
|
|
673
|
+
## π License
|
|
674
|
+
|
|
675
|
+
Copyright (C) 2026 TerminalCore Labs
|
|
676
|
+
|
|
677
|
+
Licensed under the [GNU Affero General Public License v3.0](./LICENSE) (AGPL-3.0-only).
|
|
678
|
+
|
|
679
|
+
> **Note for consumers:** unlike most installable packages, this one is intentionally AGPL rather than MIT. AGPL's copyleft applies to any project that depends on it -- if you install `streamby-core`, your own project must also be licensed under AGPL-3.0 (or a compatible copyleft license). Confirm this is acceptable for your use case before adding this dependency.
|
|
680
|
+
|