rasler 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +27 -18
- package/src/handlers/operator.js +290 -0
- package/src/handlers/rasl.js +119 -0
- package/src/index.js +6 -6
- package/src/routes/mountPoints.js +5 -20
- package/src/routes/operator.js +38 -250
- package/src/routes/rasl.js +14 -103
- package/src/static.js +13 -39
- package/src/storage/local-blobs.js +19 -0
- package/src/storage/local-db.js +39 -0
- package/src/storage/store.js +83 -96
package/src/routes/operator.js
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
import { Router, json as expressJson } from 'express';
|
|
2
2
|
import multer from 'multer';
|
|
3
|
-
import { CarReader } from '@ipld/car';
|
|
4
|
-
import { base32 } from 'multiformats/bases/base32';
|
|
5
|
-
import * as dagCbor from '@ipld/dag-cbor';
|
|
6
|
-
import { computeDataCid, computeMaslCid, isMaslCid } from '../crypto/cid.js';
|
|
7
|
-
import { createSingleMasl, parseMasl, maslLinkedCids, maslIsBundle } from '../masl/document.js';
|
|
8
3
|
import { realpathSync } from 'fs';
|
|
9
4
|
import { requireApiSecret } from '../middleware/auth.js';
|
|
10
5
|
import { makeOperatorCors } from '../middleware/cors.js';
|
|
11
6
|
import { normalizeMountPath } from '../util/normalizeMountPath.js';
|
|
7
|
+
import {
|
|
8
|
+
handleUpload, handlePin, handleUnpin,
|
|
9
|
+
handleListContent, handleGetContent, handleDeleteContent,
|
|
10
|
+
handleGetStatus,
|
|
11
|
+
} from '../handlers/operator.js';
|
|
12
|
+
import { isMaslCid } from '../crypto/cid.js';
|
|
13
|
+
import { parseMasl, maslIsBundle } from '../masl/document.js';
|
|
12
14
|
|
|
13
15
|
const upload = multer({ storage: multer.memoryStorage() });
|
|
14
16
|
|
|
@@ -49,62 +51,6 @@ const upload = multer({ storage: multer.memoryStorage() });
|
|
|
49
51
|
* type: string
|
|
50
52
|
*/
|
|
51
53
|
|
|
52
|
-
function isCarFile(file) {
|
|
53
|
-
return (
|
|
54
|
-
file.mimetype === 'application/vnd.ipld.car' ||
|
|
55
|
-
file.mimetype === 'application/car' ||
|
|
56
|
-
file.originalname?.toLowerCase().endsWith('.car')
|
|
57
|
-
);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
// Extract, verify, and return all blocks from a CAR file.
|
|
61
|
-
// Returns { maslCid, blocks: Map<cidStr, Uint8Array> } or throws with { status, error, ...extra }.
|
|
62
|
-
async function readCar(fileBuffer) {
|
|
63
|
-
let reader;
|
|
64
|
-
try {
|
|
65
|
-
reader = await CarReader.fromBytes(fileBuffer);
|
|
66
|
-
} catch {
|
|
67
|
-
throw { status: 400, error: 'Invalid CAR file' };
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
const blocks = new Map();
|
|
71
|
-
try {
|
|
72
|
-
for await (const { cid, bytes } of reader.blocks()) {
|
|
73
|
-
const cidStr = cid.toString(base32);
|
|
74
|
-
const isMasl = cid.code === dagCbor.code;
|
|
75
|
-
const actual = isMasl ? await computeMaslCid(bytes) : await computeDataCid(bytes);
|
|
76
|
-
if (actual !== cidStr) throw { status: 400, error: `CID mismatch for block ${cidStr}` };
|
|
77
|
-
blocks.set(cidStr, bytes);
|
|
78
|
-
}
|
|
79
|
-
} catch (err) {
|
|
80
|
-
if (err.status) throw err;
|
|
81
|
-
throw { status: 400, error: `Failed to read CAR blocks: ${err.message}` };
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
const roots = await reader.getRoots();
|
|
85
|
-
const maslRoot = roots.find(cid => cid.code === dagCbor.code);
|
|
86
|
-
if (!maslRoot) throw { status: 400, error: 'CAR root must be a MASL CID (dag-cbor codec)' };
|
|
87
|
-
|
|
88
|
-
const maslCid = maslRoot.toString(base32);
|
|
89
|
-
if (!blocks.has(maslCid)) throw { status: 400, error: 'MASL root block is missing from the CAR' };
|
|
90
|
-
|
|
91
|
-
let doc;
|
|
92
|
-
try { doc = parseMasl(blocks.get(maslCid)); } catch {
|
|
93
|
-
throw { status: 400, error: 'Failed to parse MASL document' };
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
const links = maslLinkedCids(doc);
|
|
97
|
-
const missing = links.filter(l => !blocks.has(l.cid)).map(l => l.cid);
|
|
98
|
-
if (missing.length > 0) throw { status: 400, error: 'CAR is missing linked data CIDs', missing };
|
|
99
|
-
|
|
100
|
-
return { maslCid, blocks, links };
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
// Base operator router: content management and node status (base fields).
|
|
104
|
-
// Auth and CORS are applied here, which also protects any operator extension router
|
|
105
|
-
// routes — Express runs router.use middleware for all requests entering the router
|
|
106
|
-
// even when no route matches, so auth is enforced before the request falls through
|
|
107
|
-
// to the overlay extension.
|
|
108
54
|
export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins = [], staticRoots = [], mountPoints = [] }) {
|
|
109
55
|
const router = Router();
|
|
110
56
|
if (corsOrigins.length > 0) router.use(makeOperatorCors(corsOrigins));
|
|
@@ -163,71 +109,13 @@ export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins =
|
|
|
163
109
|
* $ref: '#/components/schemas/Error'
|
|
164
110
|
*/
|
|
165
111
|
router.post('/upload', upload.array('files'), async (req, res) => {
|
|
166
|
-
const files = req.files
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
const
|
|
172
|
-
|
|
173
|
-
for (const file of files) {
|
|
174
|
-
if (isCarFile(file)) {
|
|
175
|
-
let parsed;
|
|
176
|
-
try {
|
|
177
|
-
parsed = await readCar(file.buffer);
|
|
178
|
-
} catch (err) {
|
|
179
|
-
return res.status(err.status ?? 400).json(
|
|
180
|
-
err.missing ? { error: err.error, missing: err.missing } : { error: err.error }
|
|
181
|
-
);
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
const { maslCid, blocks, links } = parsed;
|
|
185
|
-
const totalBytes = [...blocks.values()].reduce((n, b) => n + b.length, 0);
|
|
186
|
-
if (store.getPoolAvailable() < totalBytes) {
|
|
187
|
-
if (!store.evictIfNeeded(totalBytes)) {
|
|
188
|
-
return res.status(507).json({ error: 'Insufficient storage' });
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
store.putContent(maslCid, blocks.get(maslCid));
|
|
193
|
-
for (const link of links) {
|
|
194
|
-
store.putContent(link.cid, blocks.get(link.cid), { maslCid });
|
|
195
|
-
}
|
|
196
|
-
uploads.push({ filename: file.originalname, maslCid });
|
|
197
|
-
|
|
198
|
-
} else {
|
|
199
|
-
const bytes = file.buffer;
|
|
200
|
-
const name = file.originalname ?? 'upload';
|
|
201
|
-
const type = file.mimetype ?? 'application/octet-stream';
|
|
202
|
-
const size = bytes.length;
|
|
203
|
-
|
|
204
|
-
let dataCid;
|
|
205
|
-
try { dataCid = await computeDataCid(bytes); } catch {
|
|
206
|
-
return res.status(500).json({ error: `CID computation failed for ${name}` });
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
let maslResult;
|
|
210
|
-
try {
|
|
211
|
-
maslResult = await createSingleMasl({ name, type, size, dataCid });
|
|
212
|
-
} catch {
|
|
213
|
-
return res.status(500).json({ error: `MASL creation failed for ${name}` });
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
const { cborBytes, maslCid } = maslResult;
|
|
217
|
-
const totalBytes = bytes.length + cborBytes.length;
|
|
218
|
-
if (store.getPoolAvailable() < totalBytes) {
|
|
219
|
-
if (!store.evictIfNeeded(totalBytes)) {
|
|
220
|
-
return res.status(507).json({ error: 'Insufficient storage' });
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
store.putContent(dataCid, bytes, { maslCid });
|
|
225
|
-
store.putContent(maslCid, cborBytes);
|
|
226
|
-
uploads.push({ filename: name, maslCid });
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
return res.status(200).json({ uploads });
|
|
112
|
+
const files = (req.files ?? []).map(f => ({
|
|
113
|
+
name: f.originalname,
|
|
114
|
+
type: f.mimetype,
|
|
115
|
+
bytes: f.buffer,
|
|
116
|
+
}));
|
|
117
|
+
const result = await handleUpload(store, files);
|
|
118
|
+
return res.status(result.status).json(result.body);
|
|
231
119
|
});
|
|
232
120
|
|
|
233
121
|
/**
|
|
@@ -280,36 +168,10 @@ export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins =
|
|
|
280
168
|
* schema:
|
|
281
169
|
* $ref: '#/components/schemas/Error'
|
|
282
170
|
*/
|
|
283
|
-
router.post('/pin', (req, res) => {
|
|
171
|
+
router.post('/pin', async (req, res) => {
|
|
284
172
|
const { cids } = req.body ?? {};
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
const pinned = new Set();
|
|
290
|
-
|
|
291
|
-
for (const cid of cids) {
|
|
292
|
-
const entry = store.getContent(cid);
|
|
293
|
-
if (!entry) return res.status(404).json({ error: `CID not found: ${cid}` });
|
|
294
|
-
|
|
295
|
-
store.setPinned(cid, true);
|
|
296
|
-
pinned.add(cid);
|
|
297
|
-
|
|
298
|
-
const maslCid = entry.meta.masl_cid;
|
|
299
|
-
if (maslCid && store.hasContent(maslCid)) {
|
|
300
|
-
store.setPinned(maslCid, true);
|
|
301
|
-
pinned.add(maslCid);
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
for (const row of store.listContent()) {
|
|
305
|
-
if (row.masl_cid === cid) {
|
|
306
|
-
store.setPinned(row.cid, true);
|
|
307
|
-
pinned.add(row.cid);
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
return res.status(200).json({ pinned: [...pinned] });
|
|
173
|
+
const result = await handlePin(store, cids);
|
|
174
|
+
return res.status(result.status).json(result.body);
|
|
313
175
|
});
|
|
314
176
|
|
|
315
177
|
/**
|
|
@@ -343,23 +205,9 @@ export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins =
|
|
|
343
205
|
* '401':
|
|
344
206
|
* description: Missing or invalid operator secret
|
|
345
207
|
*/
|
|
346
|
-
router.delete('/pin/:cid', (req, res) => {
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
if (!meta) return res.status(200).json({ status: 'not found' });
|
|
351
|
-
|
|
352
|
-
store.setPinned(cid, false);
|
|
353
|
-
|
|
354
|
-
if (meta.masl_cid && store.hasContent(meta.masl_cid)) {
|
|
355
|
-
store.setPinned(meta.masl_cid, false);
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
for (const row of store.listContent()) {
|
|
359
|
-
if (row.masl_cid === cid) store.setPinned(row.cid, false);
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
return res.status(200).json({ status: 'ok' });
|
|
208
|
+
router.delete('/pin/:cid', async (req, res) => {
|
|
209
|
+
const result = await handleUnpin(store, req.params.cid);
|
|
210
|
+
return res.status(result.status).json(result.body);
|
|
363
211
|
});
|
|
364
212
|
|
|
365
213
|
/**
|
|
@@ -405,25 +253,11 @@ export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins =
|
|
|
405
253
|
* '401':
|
|
406
254
|
* description: Missing or invalid operator secret
|
|
407
255
|
*/
|
|
408
|
-
router.get('/content', (req, res) => {
|
|
256
|
+
router.get('/content', async (req, res) => {
|
|
409
257
|
const limit = Math.min(200, Math.max(1, parseInt(req.query.limit ?? '50', 10) || 50));
|
|
410
258
|
const cursor = req.query.cursor ?? null;
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
const total = store.countContent();
|
|
414
|
-
const nextCursor = items.length === limit ? items[items.length - 1].cid : null;
|
|
415
|
-
|
|
416
|
-
return res.status(200).json({
|
|
417
|
-
total,
|
|
418
|
-
items: items.map(row => ({
|
|
419
|
-
cid: row.cid,
|
|
420
|
-
maslCid: row.masl_cid ?? null,
|
|
421
|
-
size: row.size,
|
|
422
|
-
pinned: row.pinned === 1,
|
|
423
|
-
lastRequested: row.last_requested ?? null,
|
|
424
|
-
})),
|
|
425
|
-
nextCursor,
|
|
426
|
-
});
|
|
259
|
+
const result = await handleListContent(store, limit, cursor);
|
|
260
|
+
return res.status(result.status).json(result.body);
|
|
427
261
|
});
|
|
428
262
|
|
|
429
263
|
/**
|
|
@@ -487,45 +321,14 @@ export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins =
|
|
|
487
321
|
* schema:
|
|
488
322
|
* $ref: '#/components/schemas/Error'
|
|
489
323
|
*/
|
|
490
|
-
router.get('/content/:cid', (req, res) => {
|
|
491
|
-
const
|
|
492
|
-
|
|
493
|
-
if (!meta) return res.status(404).json({ error: 'CID not found' });
|
|
494
|
-
return res.status(200).json({
|
|
495
|
-
cid: meta.cid,
|
|
496
|
-
maslCid: meta.masl_cid ?? null,
|
|
497
|
-
size: meta.size,
|
|
498
|
-
pinned: meta.pinned === 1,
|
|
499
|
-
lastRequested: meta.last_requested ?? null,
|
|
500
|
-
});
|
|
324
|
+
router.get('/content/:cid', async (req, res) => {
|
|
325
|
+
const result = await handleGetContent(store, req.params.cid);
|
|
326
|
+
return res.status(result.status).json(result.body);
|
|
501
327
|
});
|
|
502
328
|
|
|
503
|
-
router.delete('/content/:cid', (req, res) => {
|
|
504
|
-
const
|
|
505
|
-
|
|
506
|
-
if (!entry) return res.status(404).json({ error: 'CID not found' });
|
|
507
|
-
|
|
508
|
-
const deleted = [];
|
|
509
|
-
|
|
510
|
-
if (isMaslCid(cid)) {
|
|
511
|
-
let linkedCids = [];
|
|
512
|
-
try {
|
|
513
|
-
linkedCids = maslLinkedCids(parseMasl(entry.bytes)).map(l => l.cid);
|
|
514
|
-
} catch {
|
|
515
|
-
// Unparseable MASL — fall through and delete just the MASL itself
|
|
516
|
-
}
|
|
517
|
-
for (const linkedCid of linkedCids) {
|
|
518
|
-
if (store.hasContent(linkedCid)) {
|
|
519
|
-
store.deleteContent(linkedCid);
|
|
520
|
-
deleted.push(linkedCid);
|
|
521
|
-
}
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
store.deleteContent(cid);
|
|
526
|
-
deleted.push(cid);
|
|
527
|
-
|
|
528
|
-
return res.status(200).json({ deleted });
|
|
329
|
+
router.delete('/content/:cid', async (req, res) => {
|
|
330
|
+
const result = await handleDeleteContent(store, req.params.cid);
|
|
331
|
+
return res.status(result.status).json(result.body);
|
|
529
332
|
});
|
|
530
333
|
|
|
531
334
|
/**
|
|
@@ -618,14 +421,12 @@ export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins =
|
|
|
618
421
|
const result = [];
|
|
619
422
|
const seen = new Set();
|
|
620
423
|
|
|
621
|
-
// Runtime entries first (they take serving priority).
|
|
622
424
|
for (const mp of store.runtimeMountPoints) {
|
|
623
425
|
const key = `${mp.hostname}|${mp.prefix}`;
|
|
624
426
|
seen.add(key);
|
|
625
427
|
result.push({ hostname: mp.hostname || null, mountPath: mp.prefix || '/', path: null, maslCid: mp.maslCid, source: 'runtime' });
|
|
626
428
|
}
|
|
627
429
|
|
|
628
|
-
// Config-backed entries not overridden by a runtime mapping at the same (hostname, prefix).
|
|
629
430
|
for (const mp of mountPoints) {
|
|
630
431
|
const key = `${mp.hostname}|${mp.prefix}`;
|
|
631
432
|
if (seen.has(key)) continue;
|
|
@@ -831,7 +632,7 @@ export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins =
|
|
|
831
632
|
return normalizeMountPath(req.path.slice('/mount-points/'.length + hostname.length) || '/');
|
|
832
633
|
}
|
|
833
634
|
|
|
834
|
-
function handleMountPointPut(req, res) {
|
|
635
|
+
async function handleMountPointPut(req, res) {
|
|
835
636
|
const { hostname: hostnameParam } = req.params;
|
|
836
637
|
const hostname = hostnameParam === '-' ? '' : hostnameParam;
|
|
837
638
|
const prefix = getMountPrefix(req);
|
|
@@ -842,10 +643,8 @@ export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins =
|
|
|
842
643
|
if (!isMaslCid(maslCid)) {
|
|
843
644
|
return res.status(400).json({ error: 'maslCid must be a dag-cbor CID' });
|
|
844
645
|
}
|
|
845
|
-
const entry = store.getContent(maslCid);
|
|
846
|
-
if (!entry) {
|
|
847
|
-
return res.status(404).json({ error: 'CID not held locally' });
|
|
848
|
-
}
|
|
646
|
+
const entry = await store.getContent(maslCid);
|
|
647
|
+
if (!entry) return res.status(404).json({ error: 'CID not held locally' });
|
|
849
648
|
let doc;
|
|
850
649
|
try { doc = parseMasl(entry.bytes); } catch {
|
|
851
650
|
return res.status(400).json({ error: 'Failed to parse MASL document' });
|
|
@@ -853,12 +652,12 @@ export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins =
|
|
|
853
652
|
if (!maslIsBundle(doc)) {
|
|
854
653
|
return res.status(400).json({ error: 'maslCid must refer to a bundle MASL' });
|
|
855
654
|
}
|
|
856
|
-
store.setPinned(maslCid, true);
|
|
857
|
-
store.setMountPoint(hostname, prefix, maslCid);
|
|
655
|
+
await store.setPinned(maslCid, true);
|
|
656
|
+
await store.setMountPoint(hostname, prefix, maslCid);
|
|
858
657
|
return res.status(200).json({ hostname: hostname || null, mountPath: prefix || '/', maslCid });
|
|
859
658
|
}
|
|
860
659
|
|
|
861
|
-
function handleMountPointDelete(req, res) {
|
|
660
|
+
async function handleMountPointDelete(req, res) {
|
|
862
661
|
const { hostname: hostnameParam } = req.params;
|
|
863
662
|
const hostname = hostnameParam === '-' ? '' : hostnameParam;
|
|
864
663
|
const prefix = getMountPrefix(req);
|
|
@@ -866,7 +665,7 @@ export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins =
|
|
|
866
665
|
if (!exists) {
|
|
867
666
|
return res.status(404).json({ error: 'Runtime virtual host mapping not found' });
|
|
868
667
|
}
|
|
869
|
-
store.deleteMountPoint(hostname, prefix);
|
|
668
|
+
await store.deleteMountPoint(hostname, prefix);
|
|
870
669
|
return res.status(200).json({ status: 'ok' });
|
|
871
670
|
}
|
|
872
671
|
|
|
@@ -918,25 +717,14 @@ export function makeOperatorRouter({ store, selfOrigin, apiSecret, corsOrigins =
|
|
|
918
717
|
* '401':
|
|
919
718
|
* description: Missing or invalid operator secret
|
|
920
719
|
*/
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
res.locals.status = {
|
|
924
|
-
origin: selfOrigin,
|
|
925
|
-
storage: {
|
|
926
|
-
totalCapacity: store.totalCapacity,
|
|
927
|
-
poolUsed: store.getPoolUsed(),
|
|
928
|
-
poolAvailable: store.getPoolAvailable(),
|
|
929
|
-
pinnedUsed: store.getPinnedUsed(),
|
|
930
|
-
pinnedCount: store.countPinned(),
|
|
931
|
-
},
|
|
932
|
-
};
|
|
720
|
+
router.get('/status', async (req, res, next) => {
|
|
721
|
+
res.locals.status = await handleGetStatus(store, selfOrigin);
|
|
933
722
|
next();
|
|
934
723
|
});
|
|
935
724
|
|
|
936
725
|
return router;
|
|
937
726
|
}
|
|
938
727
|
|
|
939
|
-
// Terminator: sends the /status response assembled by base and overlay handlers.
|
|
940
728
|
export function makeOperatorStatusTerminator() {
|
|
941
729
|
const router = Router();
|
|
942
730
|
router.get('/status', (req, res) => res.status(200).json(res.locals.status));
|
package/src/routes/rasl.js
CHANGED
|
@@ -1,122 +1,34 @@
|
|
|
1
1
|
import { Router } from 'express';
|
|
2
|
-
import {
|
|
3
|
-
parseMasl,
|
|
4
|
-
maslContentHeaders,
|
|
5
|
-
maslIsBundle,
|
|
6
|
-
resolveBundleEntry,
|
|
7
|
-
maslLinkedCids,
|
|
8
|
-
findBundleHeadersForCid,
|
|
9
|
-
} from '../masl/document.js';
|
|
10
|
-
import { isMaslCid, cidToUnencodedDigest } from '../crypto/cid.js';
|
|
2
|
+
import { resolveRaslRequest } from '../handlers/rasl.js';
|
|
11
3
|
|
|
12
|
-
// Determine the path suffix after the CID in the request URL.
|
|
13
|
-
// Returns '' for the path-free form (no trailing slash or path segment),
|
|
14
|
-
// '/' for a trailing-slash-only request, '/style.css' for non-root, etc.
|
|
15
|
-
// Using req.path directly avoids the ambiguity in how Express 5 populates
|
|
16
|
-
// req.params.path for trailing-slash-only URLs.
|
|
17
4
|
function getPathAfterCid(req, cid) {
|
|
18
5
|
const cidPrefix = '/.well-known/rasl/' + cid;
|
|
19
6
|
return req.path.startsWith(cidPrefix) ? req.path.slice(cidPrefix.length) : '';
|
|
20
7
|
}
|
|
21
8
|
|
|
22
|
-
// Streams a data CID's bytes to the response. Uses getContentStream so that
|
|
23
|
-
// static (filesystem-backed) entries are served without copying to memory.
|
|
24
|
-
function pipeContent(store, cid, res, _next) {
|
|
25
|
-
const result = store.getContentStream(cid);
|
|
26
|
-
if (!result) return res.status(502).json({ error: 'Content unavailable' });
|
|
27
|
-
res.set('content-length', String(result.meta.size));
|
|
28
|
-
res.status(200);
|
|
29
|
-
result.stream.pipe(res);
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
// Base RASL router: serves locally held content. On miss, calls next() so
|
|
33
|
-
// overlay routing middleware (or a 404 terminator) can take over.
|
|
34
9
|
export function makeRaslRouter({ store }) {
|
|
35
10
|
const router = Router();
|
|
36
11
|
|
|
37
|
-
function handle(req, res, next) {
|
|
12
|
+
async function handle(req, res, next) {
|
|
38
13
|
const { cid } = req.params;
|
|
39
|
-
const
|
|
40
|
-
|
|
14
|
+
const pathSuffix = getPathAfterCid(req, cid);
|
|
15
|
+
const result = await resolveRaslRequest(store, cid, pathSuffix);
|
|
41
16
|
|
|
42
|
-
|
|
17
|
+
if (result === null) return next();
|
|
43
18
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
if (isMaslCid(cid)) {
|
|
50
|
-
// MASL CIDs are always in the blob store (never static).
|
|
51
|
-
const entry = store.getContent(cid);
|
|
52
|
-
if (!entry) return next();
|
|
53
|
-
res.set({ 'content-type': 'application/octet-stream' });
|
|
54
|
-
res.set('unencoded-digest', cidToUnencodedDigest(cid));
|
|
55
|
-
return res.status(200).send(entry.bytes);
|
|
56
|
-
}
|
|
57
|
-
// Data CID: surface the MASL-derived Content-Type when available.
|
|
58
|
-
// This is content negotiation, not path resolution. For bundle MASLs
|
|
59
|
-
// (used by static roots), the content-type lives on the resource entry,
|
|
60
|
-
// not the top-level document.
|
|
61
|
-
let headers = { 'content-type': 'application/octet-stream' };
|
|
62
|
-
if (meta.masl_cid) {
|
|
63
|
-
const maslEntry = store.getContent(meta.masl_cid);
|
|
64
|
-
if (maslEntry) {
|
|
65
|
-
try {
|
|
66
|
-
const doc = parseMasl(maslEntry.bytes);
|
|
67
|
-
headers = maslIsBundle(doc)
|
|
68
|
-
? (findBundleHeadersForCid(doc, cid) ?? headers)
|
|
69
|
-
: maslContentHeaders(doc);
|
|
70
|
-
} catch { /* ignore */ }
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
res.set(headers);
|
|
74
|
-
res.set('unencoded-digest', cidToUnencodedDigest(cid));
|
|
75
|
-
return pipeContent(store, cid, res, next);
|
|
19
|
+
if (result.stream) {
|
|
20
|
+
res.set(result.headers);
|
|
21
|
+
res.set('content-length', String(result.size));
|
|
22
|
+
res.status(result.status);
|
|
23
|
+
return result.stream.pipe(res);
|
|
76
24
|
}
|
|
77
25
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
if (isMaslCid(cid)) {
|
|
83
|
-
const entry = store.getContent(cid);
|
|
84
|
-
if (!entry) return next();
|
|
85
|
-
let doc;
|
|
86
|
-
try { doc = parseMasl(entry.bytes); } catch {
|
|
87
|
-
return res.status(500).json({ error: 'Invalid MASL document' });
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
if (maslIsBundle(doc)) {
|
|
91
|
-
// Bundle Mode: look up path in the resources map.
|
|
92
|
-
const resolved = resolveBundleEntry(doc, path);
|
|
93
|
-
if (!resolved) return res.status(404).json({ error: 'Not found' });
|
|
94
|
-
store.recordRequest(resolved.cid);
|
|
95
|
-
res.set(resolved.headers);
|
|
96
|
-
res.set('unencoded-digest', cidToUnencodedDigest(resolved.cid));
|
|
97
|
-
return pipeContent(store, resolved.cid, res, next);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
// Single Mode: has src but no resources. Only path '/' is valid.
|
|
101
|
-
const links = maslLinkedCids(doc);
|
|
102
|
-
const srcCid = links.length > 0 ? links[0].cid : null;
|
|
103
|
-
if (srcCid) {
|
|
104
|
-
if (path !== '/') return res.status(404).json({ error: 'Not found' });
|
|
105
|
-
store.recordRequest(srcCid);
|
|
106
|
-
res.set(maslContentHeaders(doc));
|
|
107
|
-
res.set('unencoded-digest', cidToUnencodedDigest(srcCid));
|
|
108
|
-
return pipeContent(store, srcCid, res, next);
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
// Other DRISL map (neither resources nor src): fall through to non-MASL handling.
|
|
26
|
+
if (result.bytes) {
|
|
27
|
+
res.set(result.headers);
|
|
28
|
+
return res.status(result.status).send(result.bytes);
|
|
112
29
|
}
|
|
113
30
|
|
|
114
|
-
|
|
115
|
-
// any other path is not found.
|
|
116
|
-
if (path !== '/') return res.status(404).json({ error: 'Not found' });
|
|
117
|
-
res.set({ 'content-type': 'application/octet-stream' });
|
|
118
|
-
res.set('unencoded-digest', cidToUnencodedDigest(cid));
|
|
119
|
-
return pipeContent(store, cid, res, next);
|
|
31
|
+
return res.status(result.status).json(result.body);
|
|
120
32
|
}
|
|
121
33
|
|
|
122
34
|
router.get('/.well-known/rasl/:cid', handle);
|
|
@@ -127,7 +39,6 @@ export function makeRaslRouter({ store }) {
|
|
|
127
39
|
return router;
|
|
128
40
|
}
|
|
129
41
|
|
|
130
|
-
// Terminator: turns a fully-fallthrough RASL request into a 404.
|
|
131
42
|
export function makeRaslNotFoundHandler() {
|
|
132
43
|
const router = Router();
|
|
133
44
|
const notFound = (req, res) => res.status(404).json({ error: 'Not found' });
|
package/src/static.js
CHANGED
|
@@ -4,24 +4,16 @@ import micromatch from 'micromatch';
|
|
|
4
4
|
import { computeDataCid } from './crypto/cid.js';
|
|
5
5
|
import { createBundleMasl, parseMasl } from './masl/document.js';
|
|
6
6
|
import { mimeType } from './util/mime.js';
|
|
7
|
-
|
|
8
|
-
dbPutStaticContent,
|
|
9
|
-
dbGetContentBySourcePath,
|
|
10
|
-
dbListStaticContent,
|
|
11
|
-
dbDeleteContent,
|
|
12
|
-
} from './storage/db.js';
|
|
13
|
-
|
|
14
|
-
// Walks the prev chain from prevMaslCid (depth 2 relative to the new MASL)
|
|
15
|
-
// and unpins any entry whose depth exceeds maxHistory.
|
|
7
|
+
|
|
16
8
|
function pruneHistory(store, prevMaslCid, maxHistory) {
|
|
17
9
|
let cid = prevMaslCid;
|
|
18
|
-
let depth = 2;
|
|
10
|
+
let depth = 2;
|
|
19
11
|
while (cid) {
|
|
20
|
-
const entry = store.getContent(cid);
|
|
12
|
+
const entry = store.db.getContent(cid);
|
|
21
13
|
if (!entry) break;
|
|
22
|
-
if (depth > maxHistory) store.setPinned(cid, false);
|
|
14
|
+
if (depth > maxHistory) store.db.setPinned(cid, false);
|
|
23
15
|
try {
|
|
24
|
-
cid = parseMasl(entry
|
|
16
|
+
cid = parseMasl(store.blobs.get(cid, entry))?.prev?.$link ?? null;
|
|
25
17
|
} catch { break; }
|
|
26
18
|
depth++;
|
|
27
19
|
}
|
|
@@ -39,19 +31,6 @@ async function* walkDir(dir, rootDir, ignore) {
|
|
|
39
31
|
}
|
|
40
32
|
}
|
|
41
33
|
|
|
42
|
-
// Scans rootPath, registers each file's CID in the store's DB (with
|
|
43
|
-
// source_path set), and — when generateMasl is true (the default) —
|
|
44
|
-
// generates a bundle MASL for the root and returns its CID. Files are never
|
|
45
|
-
// copied to the blob store. Symlinks that resolve outside the root are
|
|
46
|
-
// silently skipped.
|
|
47
|
-
//
|
|
48
|
-
// When generateMasl is false, files are stored as plain blobs only (no MASL
|
|
49
|
-
// document is created and null is returned). This is useful for roots that
|
|
50
|
-
// exist purely to make a set of named blobs available by CID.
|
|
51
|
-
//
|
|
52
|
-
// On repeated calls, files whose size and mtime match the stored values are
|
|
53
|
-
// not re-read or re-hashed. DB entries for files that no longer exist are
|
|
54
|
-
// removed.
|
|
55
34
|
export async function indexStaticRoot(rootPath, store, { maxHistory = null, ignore = [], generateMasl = true } = {}) {
|
|
56
35
|
const realRoot = await realpath(rootPath);
|
|
57
36
|
const fileInfos = [];
|
|
@@ -68,8 +47,7 @@ export async function indexStaticRoot(rootPath, store, { maxHistory = null, igno
|
|
|
68
47
|
const { size, mtimeMs } = await stat(realFile);
|
|
69
48
|
const mtime = Math.round(mtimeMs);
|
|
70
49
|
|
|
71
|
-
|
|
72
|
-
const existing = dbGetContentBySourcePath(store.db, realFile);
|
|
50
|
+
const existing = store.db.getContentBySourcePath(realFile);
|
|
73
51
|
let cid;
|
|
74
52
|
if (existing && existing.size === size && existing.source_mtime === mtime) {
|
|
75
53
|
cid = existing.cid;
|
|
@@ -84,41 +62,36 @@ export async function indexStaticRoot(rootPath, store, { maxHistory = null, igno
|
|
|
84
62
|
fileInfos.push({ realPath: realFile, relPath, cid, size, mtime, contentType });
|
|
85
63
|
}
|
|
86
64
|
|
|
87
|
-
|
|
88
|
-
for (const entry of dbListStaticContent(store.db)) {
|
|
65
|
+
for (const entry of store.db.listStaticContent()) {
|
|
89
66
|
if (entry.source_path?.startsWith(realRoot + sep) && !visitedPaths.has(entry.source_path)) {
|
|
90
67
|
changed = true;
|
|
91
|
-
|
|
68
|
+
store.db.deleteContent(entry.cid);
|
|
92
69
|
}
|
|
93
70
|
}
|
|
94
71
|
|
|
95
72
|
if (fileInfos.length === 0) return null;
|
|
96
73
|
|
|
97
74
|
if (!generateMasl) {
|
|
98
|
-
// Nothing changed: files are already registered in the DB — no work needed.
|
|
99
75
|
if (!changed) return null;
|
|
100
76
|
const seen = new Set();
|
|
101
77
|
for (const { cid, size, mtime, realPath } of fileInfos) {
|
|
102
78
|
if (!seen.has(cid)) {
|
|
103
79
|
seen.add(cid);
|
|
104
|
-
|
|
80
|
+
store.db.putStaticContent(cid, { maslCid: null, size, sourcePath: realPath, sourceMtime: mtime });
|
|
105
81
|
}
|
|
106
82
|
}
|
|
107
83
|
return null;
|
|
108
84
|
}
|
|
109
85
|
|
|
110
|
-
|
|
111
|
-
const prevEntry = dbListStaticContent(store.db)
|
|
86
|
+
const prevEntry = store.db.listStaticContent()
|
|
112
87
|
.find(e => e.source_path?.startsWith(realRoot + sep));
|
|
113
88
|
const prevMaslCid = prevEntry?.masl_cid ?? null;
|
|
114
89
|
|
|
115
|
-
// Nothing changed — reuse the existing MASL rather than generating a new one.
|
|
116
90
|
if (!changed && prevMaslCid) {
|
|
117
91
|
store.staticRootMasls.set(realRoot, prevMaslCid);
|
|
118
92
|
return prevMaslCid;
|
|
119
93
|
}
|
|
120
94
|
|
|
121
|
-
// Sort for deterministic MASL CIDs across restarts.
|
|
122
95
|
fileInfos.sort((a, b) => a.relPath.localeCompare(b.relPath));
|
|
123
96
|
|
|
124
97
|
const resources = [];
|
|
@@ -133,7 +106,8 @@ export async function indexStaticRoot(rootPath, store, { maxHistory = null, igno
|
|
|
133
106
|
const name = basename(realRoot);
|
|
134
107
|
const { cborBytes, maslCid } = await createBundleMasl({ name, resources, prevMaslCid });
|
|
135
108
|
|
|
136
|
-
store.putContent(maslCid,
|
|
109
|
+
store.db.putContent(maslCid, { maslCid: null, size: cborBytes.length, pinned: true, lastRequested: null });
|
|
110
|
+
store.blobs.put(maslCid, Buffer.from(cborBytes));
|
|
137
111
|
store.staticRootMasls.set(realRoot, maslCid);
|
|
138
112
|
|
|
139
113
|
if (maxHistory != null && prevMaslCid) pruneHistory(store, prevMaslCid, maxHistory);
|
|
@@ -142,7 +116,7 @@ export async function indexStaticRoot(rootPath, store, { maxHistory = null, igno
|
|
|
142
116
|
for (const { cid, size, mtime, realPath } of fileInfos) {
|
|
143
117
|
if (!seen.has(cid)) {
|
|
144
118
|
seen.add(cid);
|
|
145
|
-
|
|
119
|
+
store.db.putStaticContent(cid, { maslCid, size, sourcePath: realPath, sourceMtime: mtime });
|
|
146
120
|
}
|
|
147
121
|
}
|
|
148
122
|
|