rasler 0.1.0 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rasler",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "RASLer — RASL protocol node implementation with content-addressed storage and MASL support",
   "license": "Apache-2.0",
   "author": "Wes Biggs <github@wbig.gs>",
@@ -20,6 +20,24 @@
   ],
   "type": "module",
   "main": "src/index.js",
+  "exports": {
+    ".": "./src/index.js",
+    "./server": "./src/server.js",
+    "./storage/store": "./src/storage/store.js",
+    "./storage/local-db": "./src/storage/local-db.js",
+    "./storage/local-blobs": "./src/storage/local-blobs.js",
+    "./handlers/rasl": "./src/handlers/rasl.js",
+    "./handlers/operator": "./src/handlers/operator.js",
+    "./routes/rasl": "./src/routes/rasl.js",
+    "./routes/operator": "./src/routes/operator.js",
+    "./routes/mountPoints": "./src/routes/mountPoints.js",
+    "./masl/document": "./src/masl/document.js",
+    "./crypto/cid": "./src/crypto/cid.js",
+    "./middleware/auth": "./src/middleware/auth.js",
+    "./util/normalizeMountPath": "./src/util/normalizeMountPath.js",
+    "./util/parseSize": "./src/util/parseSize.js",
+    "./util/parseJsonConfig": "./src/util/parseJsonConfig.js"
+  },
   "files": [
     "src/",
     "openapi.json",
@@ -28,9 +46,9 @@
   "scripts": {
     "start": "node src/index.js",
     "lint": "eslint src/ scripts/ test/",
-    "test": "npm run lint && NODE_OPTIONS=--experimental-vm-modules jest --runInBand",
-    "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest test/unit --runInBand",
-    "test:integration": "NODE_OPTIONS=--experimental-vm-modules jest test/integration --runInBand",
+    "test": "npm run lint && vitest run",
+    "test:unit": "vitest run test/unit",
+    "test:integration": "vitest run test/integration",
     "generate:openapi": "node scripts/generate-openapi.js",
     "generate:docs": "node scripts/generate-docs.js",
     "get-in-the-car": "node scripts/get-in-the-car.js"
@@ -40,27 +58,18 @@
     "@ipld/dag-cbor": "10.0.1",
     "express": "5.2.1",
     "micromatch": "4.0.8",
-    "multer": "2.1.1",
+    "multer": "2.2.0",
     "multiformats": "14.0.0",
     "swagger-ui-express": "5.0.1"
   },
   "devDependencies": {
     "@eslint/js": "10.0.1",
-    "@jest/globals": "30.4.1",
-    "eslint": "10.4.1",
+    "eslint": "10.5.0",
     "globals": "17.6.0",
-    "jest": "30.4.2",
-    "marked": "18.0.4",
+    "marked": "18.0.5",
     "supertest": "7.2.2",
-    "swagger-jsdoc": "6.3.0"
-  },
-  "jest": {
-    "transform": {},
-    "testEnvironment": "node",
-    "forceExit": true,
-    "testMatch": [
-      "<rootDir>/test/**/*.test.js"
-    ]
+    "swagger-jsdoc": "6.3.0",
+    "vitest": "4.1.9"
   },
   "engines": {
     "node": ">=23.4.0"

package/src/handlers/operator.js

@@ -0,0 +1,290 @@
+import { CarReader } from '@ipld/car';
+import { base32 } from 'multiformats/bases/base32';
+import * as dagCbor from '@ipld/dag-cbor';
+import { computeDataCid, computeMaslCid, isMaslCid } from '../crypto/cid.js';
+import { createSingleMasl, parseMasl, maslLinkedCids, maslIsBundle } from '../masl/document.js';
+import { normalizeMountPath } from '../util/normalizeMountPath.js';
+
+function isCarFilename(name) {
+  return name?.toLowerCase().endsWith('.car');
+}
+
+function isCarMimeType(type) {
+  return type === 'application/vnd.ipld.car' || type === 'application/car';
+}
+
+export function isCarFile({ name, type }) {
+  return isCarMimeType(type) || isCarFilename(name);
+}
+
+// Extract, verify, and return all blocks from a CAR file buffer.
+// Returns { maslCid, blocks: Map<cidStr, Uint8Array>, links } or throws { status, error, ...extra }.
+export async function readCar(fileBuffer) {
+  let reader;
+  try {
+    reader = await CarReader.fromBytes(fileBuffer);
+  } catch {
+    throw { status: 400, error: 'Invalid CAR file' };
+  }
+
+  const blocks = new Map();
+  try {
+    for await (const { cid, bytes } of reader.blocks()) {
+      const cidStr = cid.toString(base32);
+      const isMasl = cid.code === dagCbor.code;
+      const actual = isMasl ? await computeMaslCid(bytes) : await computeDataCid(bytes);
+      if (actual !== cidStr) throw { status: 400, error: `CID mismatch for block ${cidStr}` };
+      blocks.set(cidStr, bytes);
+    }
+  } catch (err) {
+    if (err.status) throw err;
+    throw { status: 400, error: `Failed to read CAR blocks: ${err.message}` };
+  }
+
+  const roots = await reader.getRoots();
+  const maslRoot = roots.find(cid => cid.code === dagCbor.code);
+  if (!maslRoot) throw { status: 400, error: 'CAR root must be a MASL CID (dag-cbor codec)' };
+
+  const maslCid = maslRoot.toString(base32);
+  if (!blocks.has(maslCid)) throw { status: 400, error: 'MASL root block is missing from the CAR' };
+
+  let doc;
+  try { doc = parseMasl(blocks.get(maslCid)); } catch {
+    throw { status: 400, error: 'Failed to parse MASL document' };
+  }
+
+  const links = maslLinkedCids(doc);
+  const missing = links.filter(l => !blocks.has(l.cid)).map(l => l.cid);
+  if (missing.length > 0) throw { status: 400, error: 'CAR is missing linked data CIDs', missing };
+
+  return { maslCid, blocks, links };
+}
+
+// files: Array<{ name: string, type: string, bytes: Uint8Array }>
+// Returns { status, body }
+export async function handleUpload(store, files) {
+  if (!files || files.length === 0) {
+    return { status: 400, body: { error: 'No files provided' } };
+  }
+
+  const uploads = [];
+
+  for (const file of files) {
+    if (isCarFile(file)) {
+      let parsed;
+      try {
+        parsed = await readCar(file.bytes);
+      } catch (err) {
+        return {
+          status: err.status ?? 400,
+          body: err.missing ? { error: err.error, missing: err.missing } : { error: err.error },
+        };
+      }
+
+      const { maslCid, blocks, links } = parsed;
+      const totalBytes = [...blocks.values()].reduce((n, b) => n + b.length, 0);
+      if (await store.getPoolAvailable() < totalBytes) {
+        if (!await store.evictIfNeeded(totalBytes)) {
+          return { status: 507, body: { error: 'Insufficient storage' } };
+        }
+      }
+
+      await store.putContent(maslCid, blocks.get(maslCid));
+      for (const link of links) {
+        await store.putContent(link.cid, blocks.get(link.cid), { maslCid });
+      }
+      uploads.push({ filename: file.name, maslCid });
+
+    } else {
+      const { bytes, name = 'upload', type = 'application/octet-stream' } = file;
+      const size = bytes.length;
+
+      let dataCid;
+      try { dataCid = await computeDataCid(bytes); } catch {
+        return { status: 500, body: { error: `CID computation failed for ${name}` } };
+      }
+
+      let maslResult;
+      try {
+        maslResult = await createSingleMasl({ name, type, size, dataCid });
+      } catch {
+        return { status: 500, body: { error: `MASL creation failed for ${name}` } };
+      }
+
+      const { cborBytes, maslCid } = maslResult;
+      const totalBytes = bytes.length + cborBytes.length;
+      if (await store.getPoolAvailable() < totalBytes) {
+        if (!await store.evictIfNeeded(totalBytes)) {
+          return { status: 507, body: { error: 'Insufficient storage' } };
+        }
+      }
+
+      await store.putContent(dataCid, bytes, { maslCid });
+      await store.putContent(maslCid, cborBytes);
+      uploads.push({ filename: name, maslCid });
+    }
+  }
+
+  return { status: 200, body: { uploads } };
+}
+
+// cids: string[]
+// Returns { status, body }
+export async function handlePin(store, cids) {
+  if (!Array.isArray(cids) || cids.length === 0) {
+    return { status: 400, body: { error: 'cids must be a non-empty array' } };
+  }
+
+  const pinned = new Set();
+
+  for (const cid of cids) {
+    const entry = await store.getContent(cid);
+    if (!entry) return { status: 404, body: { error: `CID not found: ${cid}` } };
+
+    await store.setPinned(cid, true);
+    pinned.add(cid);
+
+    const maslCid = entry.meta.masl_cid;
+    if (maslCid && await store.hasContent(maslCid)) {
+      await store.setPinned(maslCid, true);
+      pinned.add(maslCid);
+    }
+
+    for (const row of await store.listContent()) {
+      if (row.masl_cid === cid) {
+        await store.setPinned(row.cid, true);
+        pinned.add(row.cid);
+      }
+    }
+  }
+
+  return { status: 200, body: { pinned: [...pinned] } };
+}
+
+export async function handleUnpin(store, cid) {
+  const entry = await store.getContent(cid);
+  if (!entry) return { status: 200, body: { status: 'not found' } };
+
+  await store.setPinned(cid, false);
+
+  if (entry.meta.masl_cid && await store.hasContent(entry.meta.masl_cid)) {
+    await store.setPinned(entry.meta.masl_cid, false);
+  }
+
+  for (const row of await store.listContent()) {
+    if (row.masl_cid === cid) await store.setPinned(row.cid, false);
+  }
+
+  return { status: 200, body: { status: 'ok' } };
+}
+
+export async function handleListContent(store, limit, cursor) {
+  const items = await store.listContentPage(limit, cursor);
+  const total = await store.countContent();
+  const nextCursor = items.length === limit ? items[items.length - 1].cid : null;
+  return {
+    status: 200,
+    body: {
+      total,
+      items: items.map(row => ({
+        cid: row.cid,
+        maslCid: row.masl_cid ?? null,
+        size: row.size,
+        pinned: row.pinned === 1,
+        lastRequested: row.last_requested ?? null,
+      })),
+      nextCursor,
+    },
+  };
+}
+
+export async function handleGetContent(store, cid) {
+  const meta = await store.getContentMeta(cid);
+  if (!meta) return { status: 404, body: { error: 'CID not found' } };
+  return {
+    status: 200,
+    body: {
+      cid: meta.cid,
+      maslCid: meta.masl_cid ?? null,
+      size: meta.size,
+      pinned: meta.pinned === 1,
+      lastRequested: meta.last_requested ?? null,
+    },
+  };
+}
+
+export async function handleDeleteContent(store, cid) {
+  const entry = await store.getContent(cid);
+  if (!entry) return { status: 404, body: { error: 'CID not found' } };
+
+  const deleted = [];
+
+  if (isMaslCid(cid)) {
+    let linkedCids = [];
+    try {
+      linkedCids = maslLinkedCids(parseMasl(entry.bytes)).map(l => l.cid);
+    } catch {
+      // Unparseable MASL — fall through and delete just the MASL itself
+    }
+    for (const linkedCid of linkedCids) {
+      if (await store.hasContent(linkedCid)) {
+        await store.deleteContent(linkedCid);
+        deleted.push(linkedCid);
+      }
+    }
+  }
+
+  await store.deleteContent(cid);
+  deleted.push(cid);
+
+  return { status: 200, body: { deleted } };
+}
+
+export async function handleSetMountPoint(store, hostnameParam, prefixParam) {
+  return async function (maslCid) {
+    if (!maslCid || typeof maslCid !== 'string') {
+      return { status: 400, body: { error: 'maslCid is required' } };
+    }
+    if (!isMaslCid(maslCid)) {
+      return { status: 400, body: { error: 'maslCid must be a dag-cbor CID' } };
+    }
+    const entry = await store.getContent(maslCid);
+    if (!entry) return { status: 404, body: { error: 'CID not held locally' } };
+    let doc;
+    try { doc = parseMasl(entry.bytes); } catch {
+      return { status: 400, body: { error: 'Failed to parse MASL document' } };
+    }
+    if (!maslIsBundle(doc)) {
+      return { status: 400, body: { error: 'maslCid must refer to a bundle MASL' } };
+    }
+    const hostname = hostnameParam === '-' ? '' : hostnameParam;
+    const prefix = normalizeMountPath(prefixParam || '/');
+    await store.setPinned(maslCid, true);
+    await store.setMountPoint(hostname, prefix, maslCid);
+    return { status: 200, body: { hostname: hostname || null, mountPath: prefix || '/', maslCid } };
+  };
+}
+
+export async function handleDeleteMountPoint(store, hostnameParam, prefixParam) {
+  const hostname = hostnameParam === '-' ? '' : hostnameParam;
+  const prefix = normalizeMountPath(prefixParam || '/');
+  const exists = store.runtimeMountPoints.some(mp => mp.hostname === hostname && mp.prefix === prefix);
+  if (!exists) {
+    return { status: 404, body: { error: 'Runtime virtual host mapping not found' } };
+  }
+  await store.deleteMountPoint(hostname, prefix);
+  return { status: 200, body: { status: 'ok' } };
+}
+
+export async function handleGetStatus(store, selfOrigin) {
+  return {
+    origin: selfOrigin,
+    storage: {
+      totalCapacity: store.totalCapacity,
+      poolUsed: await store.getPoolUsed(),
+      poolAvailable: await store.getPoolAvailable(),
+      pinnedUsed: await store.getPinnedUsed(),
+      pinnedCount: await store.countPinned(),
+    },
+  };
+}

package/src/handlers/rasl.js

@@ -0,0 +1,119 @@
+import {
+  parseMasl, maslContentHeaders, maslIsBundle,
+  resolveBundleEntry, maslLinkedCids, findBundleHeadersForCid,
+} from '../masl/document.js';
+import { isMaslCid, cidToUnencodedDigest } from '../crypto/cid.js';
+
+// Resolves a RASL request for the given CID and path suffix.
+//
+// pathSuffix is the portion of the URL after the CID:
+//   ''          → path-free form (raw bytes)
+//   '/'         → bundle root
+//   '/foo.css'  → bundle resource
+//
+// Returns one of:
+//   null                                             → local miss; caller should delegate (next())
+//   { status, headers, bytes }                       → complete in-memory response
+//   { status, headers, stream, size }                → streaming response (stream is platform-specific)
+//   { status, body: { error, ...extra } }            → error response
+export async function resolveRaslRequest(store, cid, pathSuffix) {
+  const meta = await store.getContentMeta(cid);
+  if (!meta) return null;
+
+  await store.recordRequest(cid);
+
+  const isPathFree = pathSuffix === '';
+
+  if (isPathFree) {
+    if (isMaslCid(cid)) {
+      const entry = await store.getContent(cid);
+      if (!entry) return null;
+      return {
+        status: 200,
+        headers: {
+          'content-type': 'application/octet-stream',
+          'unencoded-digest': cidToUnencodedDigest(cid),
+        },
+        bytes: entry.bytes,
+      };
+    }
+
+    let headers = { 'content-type': 'application/octet-stream' };
+    if (meta.masl_cid) {
+      const maslEntry = await store.getContent(meta.masl_cid);
+      if (maslEntry) {
+        try {
+          const doc = parseMasl(maslEntry.bytes);
+          headers = maslIsBundle(doc)
+            ? (findBundleHeadersForCid(doc, cid) ?? headers)
+            : maslContentHeaders(doc);
+        } catch { /* ignore */ }
+      }
+    }
+
+    const streamResult = await store.getContentStream(cid);
+    if (!streamResult) return { status: 502, body: { error: 'Content unavailable' } };
+    return {
+      status: 200,
+      headers: { ...headers, 'unencoded-digest': cidToUnencodedDigest(cid) },
+      stream: streamResult.stream,
+      size: streamResult.meta.size,
+    };
+  }
+
+  // Path-bearing: resolve against MASL document structure.
+  const path = pathSuffix;
+
+  if (isMaslCid(cid)) {
+    const entry = await store.getContent(cid);
+    if (!entry) return null;
+    let doc;
+    try { doc = parseMasl(entry.bytes); } catch {
+      return { status: 500, body: { error: 'Invalid MASL document' } };
+    }
+
+    if (maslIsBundle(doc)) {
+      const resolved = resolveBundleEntry(doc, path);
+      if (!resolved) return { status: 404, body: { error: 'Not found' } };
+      await store.recordRequest(resolved.cid);
+      const streamResult = await store.getContentStream(resolved.cid);
+      if (!streamResult) return { status: 502, body: { error: 'Content unavailable' } };
+      return {
+        status: 200,
+        headers: { ...resolved.headers, 'unencoded-digest': cidToUnencodedDigest(resolved.cid) },
+        stream: streamResult.stream,
+        size: streamResult.meta.size,
+      };
+    }
+
+    // Single mode: only path '/' is valid.
+    const links = maslLinkedCids(doc);
+    const srcCid = links.length > 0 ? links[0].cid : null;
+    if (srcCid) {
+      if (path !== '/') return { status: 404, body: { error: 'Not found' } };
+      await store.recordRequest(srcCid);
+      const streamResult = await store.getContentStream(srcCid);
+      if (!streamResult) return { status: 502, body: { error: 'Content unavailable' } };
+      return {
+        status: 200,
+        headers: { ...maslContentHeaders(doc), 'unencoded-digest': cidToUnencodedDigest(srcCid) },
+        stream: streamResult.stream,
+        size: streamResult.meta.size,
+      };
+    }
+  }
+
+  // Non-MASL CID (or unrecognised DRISL map): only path '/' returns raw bytes.
+  if (path !== '/') return { status: 404, body: { error: 'Not found' } };
+  const streamResult = await store.getContentStream(cid);
+  if (!streamResult) return { status: 502, body: { error: 'Content unavailable' } };
+  return {
+    status: 200,
+    headers: {
+      'content-type': 'application/octet-stream',
+      'unencoded-digest': cidToUnencodedDigest(cid),
+    },
+    stream: streamResult.stream,
+    size: streamResult.meta.size,
+  };
+}

package/src/index.js

@@ -1,5 +1,7 @@
 import config from './config.js';
 import { openDb } from './storage/db.js';
+import { makeLocalDb } from './storage/local-db.js';
+import { makeLocalBlobs } from './storage/local-blobs.js';
 import { Store } from './storage/store.js';
 import { createApp, finalizeApp } from './server.js';
 import { makeRaslNotFoundHandler } from './routes/rasl.js';
@@ -8,15 +10,13 @@ import { indexStaticRoots } from './static.js';
 import { startStaticRootWatchers } from './watcher.js';
 
 function main() {
-  const db = openDb(config.dataDir);
-  const store = new Store(db, config.dataDir, config.totalCapacity, {
+  const rawDb = openDb(config.dataDir);
+  const db = makeLocalDb(rawDb);
+  const blobs = makeLocalBlobs(config.dataDir);
+  const store = new Store(db, blobs, config.totalCapacity, {
     staticRoots: config.staticRoots,
   });
 
-  // Index static roots in the background so the server starts immediately.
-  // Watchers start after the initial index so startup writes don't trigger
-  // spurious re-indexes. On a warm restart (no file changes) the mtime cache
-  // makes the window negligible.
   if (config.staticRoots.length > 0) {
     indexStaticRoots(config.staticRoots, store, { maxHistory: config.staticMaxHistory })
       .then(() => startStaticRootWatchers(config.staticRoots, store, { maxHistory: config.staticMaxHistory }));

package/src/routes/mountPoints.js

@@ -4,10 +4,6 @@ import { parseMasl, resolveBundleEntry } from '../masl/document.js';
 import { cidToUnencodedDigest } from '../crypto/cid.js';
 import { OPERATOR_SECRET_HEADER } from '../middleware/auth.js';
 
-// Returns the first mount point whose hostname and prefix match the request,
-// or null. mountPoints must be sorted: longer prefix first, specific hostname
-// before wildcard (hostname='') at equal prefix lengths.
-// hostname='' matches any Host: value (or no Host: header).
 function findMountPoint(mountPoints, hostname, path) {
   for (const mp of mountPoints) {
     if (mp.hostname !== '' && mp.hostname !== hostname) continue;
@@ -18,28 +14,17 @@ function findMountPoint(mountPoints, hostname, path) {
   return null;
 }
 
-// Serves content by resolving the request path against the bundle MASL for
-// the matching mount point. Sits before the RASL router so browser clients
-// can use normal URLs; RASL paths are passed through unchanged.
-//
-// The MASL CID is read from store.staticRootMasls on every request, so it
-// reflects the current indexed version automatically after any re-index.
 export function makeMountPointRouter({ store, mountPoints, selfOrigin }) {
   const router = Router();
 
-  router.use((req, res, next) => {
+  router.use(async (req, res, next) => {
     if (req.method !== 'GET' && req.method !== 'HEAD') return next();
-    // Leave RASL retrieval paths to the RASL router.
     if (req.path.startsWith('/.well-known/rasl/')) return next();
-    // Operator API requests (identified by the secret header) bypass mount-point
-    // routing so authenticated operator paths are always reachable even when a
-    // wildcard mount is configured.
     if (req.headers[OPERATOR_SECRET_HEADER]) return next();
 
     let maslCid;
     let maslPath;
 
-    // Runtime mappings take priority over static-root mappings.
     const runtimeMp = findMountPoint(store.runtimeMountPoints, req.hostname, req.path);
     if (runtimeMp) {
       maslCid = runtimeMp.maslCid;
@@ -57,7 +42,7 @@ export function makeMountPointRouter({ store, mountPoints, selfOrigin }) {
       maslPath = staticMp.prefix ? req.path.slice(staticMp.prefix.length) || '/' : req.path;
     }
 
-    const maslEntry = store.getContent(maslCid);
+    const maslEntry = await store.getContent(maslCid);
     if (!maslEntry) return res.status(503).json({ error: 'MASL unavailable' });
 
     let doc;
@@ -68,10 +53,10 @@ export function makeMountPointRouter({ store, mountPoints, selfOrigin }) {
     const resolved = resolveBundleEntry(doc, maslPath);
     if (!resolved) return res.status(404).send('Not found');
 
-    store.recordRequest(maslCid);
-    store.recordRequest(resolved.cid);
+    await store.recordRequest(maslCid);
+    await store.recordRequest(resolved.cid);
 
-    const result = store.getContentStream(resolved.cid);
+    const result = await store.getContentStream(resolved.cid);
     if (!result) return res.status(502).json({ error: 'Content unavailable' });
 
     res.set(resolved.headers);