rasler 0.1.0

package/src/routes/rasl.js

@@ -0,0 +1,139 @@
+import { Router } from 'express';
+import {
+  parseMasl,
+  maslContentHeaders,
+  maslIsBundle,
+  resolveBundleEntry,
+  maslLinkedCids,
+  findBundleHeadersForCid,
+} from '../masl/document.js';
+import { isMaslCid, cidToUnencodedDigest } from '../crypto/cid.js';
+
+// Determine the path suffix after the CID in the request URL.
+// Returns '' for the path-free form (no trailing slash or path segment),
+// '/' for a trailing-slash-only request, '/style.css' for non-root, etc.
+// Using req.path directly avoids the ambiguity in how Express 5 populates
+// req.params.path for trailing-slash-only URLs.
+function getPathAfterCid(req, cid) {
+  const cidPrefix = '/.well-known/rasl/' + cid;
+  return req.path.startsWith(cidPrefix) ? req.path.slice(cidPrefix.length) : '';
+}
+
+// Streams a data CID's bytes to the response. Uses getContentStream so that
+// static (filesystem-backed) entries are served without copying to memory.
+function pipeContent(store, cid, res, _next) {
+  const result = store.getContentStream(cid);
+  if (!result) return res.status(502).json({ error: 'Content unavailable' });
+  res.set('content-length', String(result.meta.size));
+  res.status(200);
+  result.stream.pipe(res);
+}
+
+// Base RASL router: serves locally held content. On miss, calls next() so
+// overlay routing middleware (or a 404 terminator) can take over.
+export function makeRaslRouter({ store }) {
+  const router = Router();
+
+  function handle(req, res, next) {
+    const { cid } = req.params;
+    const meta = store.getContentMeta(cid);
+    if (!meta) return next();
+
+    store.recordRequest(cid);
+
+    const afterCid = getPathAfterCid(req, cid);
+    const isPathFree = afterCid === '';
+
+    if (isPathFree) {
+      // Path-free form: always raw bytes. No MASL path resolution.
+      if (isMaslCid(cid)) {
+        // MASL CIDs are always in the blob store (never static).
+        const entry = store.getContent(cid);
+        if (!entry) return next();
+        res.set({ 'content-type': 'application/octet-stream' });
+        res.set('unencoded-digest', cidToUnencodedDigest(cid));
+        return res.status(200).send(entry.bytes);
+      }
+      // Data CID: surface the MASL-derived Content-Type when available.
+      // This is content negotiation, not path resolution. For bundle MASLs
+      // (used by static roots), the content-type lives on the resource entry,
+      // not the top-level document.
+      let headers = { 'content-type': 'application/octet-stream' };
+      if (meta.masl_cid) {
+        const maslEntry = store.getContent(meta.masl_cid);
+        if (maslEntry) {
+          try {
+            const doc = parseMasl(maslEntry.bytes);
+            headers = maslIsBundle(doc)
+              ? (findBundleHeadersForCid(doc, cid) ?? headers)
+              : maslContentHeaders(doc);
+          } catch { /* ignore */ }
+        }
+      }
+      res.set(headers);
+      res.set('unencoded-digest', cidToUnencodedDigest(cid));
+      return pipeContent(store, cid, res, next);
+    }
+
+    // Path-bearing form: resolve path against MASL document structure.
+    // afterCid always begins with '/', e.g. '/', '/style.css', '/a/b/c.html'.
+    const path = afterCid;
+
+    if (isMaslCid(cid)) {
+      const entry = store.getContent(cid);
+      if (!entry) return next();
+      let doc;
+      try { doc = parseMasl(entry.bytes); } catch {
+        return res.status(500).json({ error: 'Invalid MASL document' });
+      }
+
+      if (maslIsBundle(doc)) {
+        // Bundle Mode: look up path in the resources map.
+        const resolved = resolveBundleEntry(doc, path);
+        if (!resolved) return res.status(404).json({ error: 'Not found' });
+        store.recordRequest(resolved.cid);
+        res.set(resolved.headers);
+        res.set('unencoded-digest', cidToUnencodedDigest(resolved.cid));
+        return pipeContent(store, resolved.cid, res, next);
+      }
+
+      // Single Mode: has src but no resources. Only path '/' is valid.
+      const links = maslLinkedCids(doc);
+      const srcCid = links.length > 0 ? links[0].cid : null;
+      if (srcCid) {
+        if (path !== '/') return res.status(404).json({ error: 'Not found' });
+        store.recordRequest(srcCid);
+        res.set(maslContentHeaders(doc));
+        res.set('unencoded-digest', cidToUnencodedDigest(srcCid));
+        return pipeContent(store, srcCid, res, next);
+      }
+
+      // Other DRISL map (neither resources nor src): fall through to non-MASL handling.
+    }
+
+    // Non-MASL CID (or unrecognised DRISL map): only path '/' returns raw bytes;
+    // any other path is not found.
+    if (path !== '/') return res.status(404).json({ error: 'Not found' });
+    res.set({ 'content-type': 'application/octet-stream' });
+    res.set('unencoded-digest', cidToUnencodedDigest(cid));
+    return pipeContent(store, cid, res, next);
+  }
+
+  router.get('/.well-known/rasl/:cid', handle);
+  router.get('/.well-known/rasl/:cid/*path', handle);
+  router.head('/.well-known/rasl/:cid', handle);
+  router.head('/.well-known/rasl/:cid/*path', handle);
+
+  return router;
+}
+
+// Terminator: turns a fully-fallthrough RASL request into a 404.
+export function makeRaslNotFoundHandler() {
+  const router = Router();
+  const notFound = (req, res) => res.status(404).json({ error: 'Not found' });
+  router.get('/.well-known/rasl/:cid', notFound);
+  router.get('/.well-known/rasl/:cid/*path', notFound);
+  router.head('/.well-known/rasl/:cid', notFound);
+  router.head('/.well-known/rasl/:cid/*path', notFound);
+  return router;
+}

package/src/server.js

@@ -0,0 +1,83 @@
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, resolve } from 'path';
+import express from 'express';
+import swaggerUi from 'swagger-ui-express';
+import { makeRaslRouter } from './routes/rasl.js';
+import { makeOperatorStatusTerminator } from './routes/operator.js';
+import { makeMountPointRouter } from './routes/mountPoints.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+function mergeSpecs(base, overlay) {
+  return {
+    ...base,
+    ...overlay,
+    info: { ...base.info, ...overlay.info },
+    components: {
+      ...base.components,
+      ...overlay.components,
+      schemas: {
+        ...(base.components?.schemas ?? {}),
+        ...(overlay.components?.schemas ?? {}),
+      },
+      securitySchemes: {
+        ...(base.components?.securitySchemes ?? {}),
+        ...(overlay.components?.securitySchemes ?? {}),
+      },
+    },
+    tags: [
+      ...(base.tags ?? []),
+      ...(overlay.tags ?? []).filter(t => !(base.tags ?? []).some(bt => bt.name === t.name)),
+    ],
+    paths: { ...(base.paths ?? {}), ...(overlay.paths ?? {}) },
+  };
+}
+
+// Mounts RASLer middleware (static files, mount-point router, RASL router,
+// Swagger UI) onto an existing Express app. Does not set trust proxy or mount
+// the operator router — callers handle those before calling finalizeApp().
+// openApiOverlays: array of file paths to OpenAPI overlay specs to merge in.
+export function addRaslerMiddleware(app, { store, config, openApiOverlays = [] }) {
+  app.use(makeMountPointRouter({ store, mountPoints: config.mountPoints ?? [], selfOrigin: config.origin }));
+  app.use(makeRaslRouter({ store }));
+
+  // Swagger UI — mounted early so the browser can load it without credentials.
+  // Use the Authorize button to set x-rasl-operator-secret.
+  if (config.swaggerUi) {
+    const operatorPrefix = config.operatorApiPathPrefix ?? '';
+    let spec = JSON.parse(readFileSync(resolve(__dirname, '..', 'openapi.json'), 'utf8'));
+    for (const overlayPath of openApiOverlays) {
+      const overlay = JSON.parse(readFileSync(overlayPath, 'utf8'));
+      spec = mergeSpecs(spec, overlay);
+    }
+    spec.servers = [{ url: operatorPrefix || '/', description: 'This node' }];
+    const swaggerPath = `${operatorPrefix}/api-docs`;
+    app.use(swaggerPath, swaggerUi.serve, swaggerUi.setup(spec, {
+      swaggerOptions: { persistAuthorization: true },
+    }));
+    console.log(`Swagger UI enabled at ${swaggerPath}`);
+  }
+}
+
+// Convenience factory: creates a new Express app with trust proxy enabled,
+// then calls addRaslerMiddleware. Use addRaslerMiddleware directly when adding
+// RASLer to an existing app.
+export function createApp({ store, config, openApiOverlays = [] }) {
+  const app = express();
+  app.set('trust proxy', 1);
+  addRaslerMiddleware(app, { store, config, openApiOverlays });
+  return app;
+}
+
+// Adds the RASL 404 terminator, operator /status terminator, and error
+// handler. Call after all overlay middleware and the operator router have
+// been mounted.
+export function finalizeApp(app, config) {
+  const prefix = (config.operatorApiPathPrefix ?? '') || '/';
+  app.use(prefix, makeOperatorStatusTerminator());
+  app.use((err, req, res, _next) => {
+    console.error(err);
+    res.status(500).json({ error: 'Internal server error' });
+  });
+}

package/src/static.js

@@ -0,0 +1,168 @@
+import { readFile, readdir, realpath, stat } from 'node:fs/promises';
+import { join, relative, basename, dirname, sep } from 'node:path';
+import micromatch from 'micromatch';
+import { computeDataCid } from './crypto/cid.js';
+import { createBundleMasl, parseMasl } from './masl/document.js';
+import { mimeType } from './util/mime.js';
+import {
+  dbPutStaticContent,
+  dbGetContentBySourcePath,
+  dbListStaticContent,
+  dbDeleteContent,
+} from './storage/db.js';
+
+// Walks the prev chain from prevMaslCid (depth 2 relative to the new MASL)
+// and unpins any entry whose depth exceeds maxHistory.
+function pruneHistory(store, prevMaslCid, maxHistory) {
+  let cid = prevMaslCid;
+  let depth = 2; // new MASL is depth 1, its prev is depth 2
+  while (cid) {
+    const entry = store.getContent(cid);
+    if (!entry) break;
+    if (depth > maxHistory) store.setPinned(cid, false);
+    try {
+      cid = parseMasl(entry.bytes).prev?.$link ?? null;
+    } catch { break; }
+    depth++;
+  }
+}
+
+async function* walkDir(dir, rootDir, ignore) {
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name);
+    if (ignore.length > 0) {
+      const rel = relative(rootDir, full).replace(/\\/g, '/');
+      if (micromatch.isMatch(rel, ignore)) continue;
+    }
+    if (entry.isDirectory()) yield* walkDir(full, rootDir, ignore);
+    else if (entry.isFile()) yield full;
+  }
+}
+
+// Scans rootPath, registers each file's CID in the store's DB (with
+// source_path set), and — when generateMasl is true (the default) —
+// generates a bundle MASL for the root and returns its CID. Files are never
+// copied to the blob store. Symlinks that resolve outside the root are
+// silently skipped.
+//
+// When generateMasl is false, files are stored as plain blobs only (no MASL
+// document is created and null is returned). This is useful for roots that
+// exist purely to make a set of named blobs available by CID.
+//
+// On repeated calls, files whose size and mtime match the stored values are
+// not re-read or re-hashed. DB entries for files that no longer exist are
+// removed.
+export async function indexStaticRoot(rootPath, store, { maxHistory = null, ignore = [], generateMasl = true } = {}) {
+  const realRoot = await realpath(rootPath);
+  const fileInfos = [];
+  const visitedPaths = new Set();
+  let changed = false;
+
+  for await (const filePath of walkDir(realRoot, realRoot, ignore)) {
+    let realFile;
+    try { realFile = await realpath(filePath); } catch { continue; }
+    if (realFile !== realRoot && !realFile.startsWith(realRoot + sep)) continue;
+
+    visitedPaths.add(realFile);
+
+    const { size, mtimeMs } = await stat(realFile);
+    const mtime = Math.round(mtimeMs);
+
+    // Cache hit: size and mtime both match → file unchanged → reuse stored CID.
+    const existing = dbGetContentBySourcePath(store.db, realFile);
+    let cid;
+    if (existing && existing.size === size && existing.source_mtime === mtime) {
+      cid = existing.cid;
+    } else {
+      changed = true;
+      const bytes = await readFile(realFile);
+      cid = await computeDataCid(bytes);
+    }
+
+    const contentType = mimeType(filePath);
+    const relPath = '/' + relative(realRoot, filePath).replace(/\\/g, '/');
+    fileInfos.push({ realPath: realFile, relPath, cid, size, mtime, contentType });
+  }
+
+  // Remove DB entries for files deleted since the last startup.
+  for (const entry of dbListStaticContent(store.db)) {
+    if (entry.source_path?.startsWith(realRoot + sep) && !visitedPaths.has(entry.source_path)) {
+      changed = true;
+      dbDeleteContent(store.db, entry.cid);
+    }
+  }
+
+  if (fileInfos.length === 0) return null;
+
+  if (!generateMasl) {
+    // Nothing changed: files are already registered in the DB — no work needed.
+    if (!changed) return null;
+    const seen = new Set();
+    for (const { cid, size, mtime, realPath } of fileInfos) {
+      if (!seen.has(cid)) {
+        seen.add(cid);
+        dbPutStaticContent(store.db, cid, { maslCid: null, size, sourcePath: realPath, sourceMtime: mtime });
+      }
+    }
+    return null;
+  }
+
+  // Find the existing MASL CID for this root (if any) to link as prev.
+  const prevEntry = dbListStaticContent(store.db)
+    .find(e => e.source_path?.startsWith(realRoot + sep));
+  const prevMaslCid = prevEntry?.masl_cid ?? null;
+
+  // Nothing changed — reuse the existing MASL rather than generating a new one.
+  if (!changed && prevMaslCid) {
+    store.staticRootMasls.set(realRoot, prevMaslCid);
+    return prevMaslCid;
+  }
+
+  // Sort for deterministic MASL CIDs across restarts.
+  fileInfos.sort((a, b) => a.relPath.localeCompare(b.relPath));
+
+  const resources = [];
+  for (const { relPath, cid, size, contentType } of fileInfos) {
+    resources.push({ path: relPath, cid, size, contentType });
+    if (basename(relPath) === 'index.html') {
+      const dir = dirname(relPath);
+      resources.push({ path: dir === '/' ? '/' : dir + '/', cid, size, contentType });
+    }
+  }
+
+  const name = basename(realRoot);
+  const { cborBytes, maslCid } = await createBundleMasl({ name, resources, prevMaslCid });
+
+  store.putContent(maslCid, Buffer.from(cborBytes), { pinned: true });
+  store.staticRootMasls.set(realRoot, maslCid);
+
+  if (maxHistory != null && prevMaslCid) pruneHistory(store, prevMaslCid, maxHistory);
+
+  const seen = new Set();
+  for (const { cid, size, mtime, realPath } of fileInfos) {
+    if (!seen.has(cid)) {
+      seen.add(cid);
+      dbPutStaticContent(store.db, cid, { maslCid, size, sourcePath: realPath, sourceMtime: mtime });
+    }
+  }
+
+  return maslCid;
+}
+
+export async function indexStaticRoots(staticRoots, store, { maxHistory = null } = {}) {
+  for (const root of staticRoots) {
+    const { directory, ignore = [], generateMasl = true } = root;
+    try {
+      const maslCid = await indexStaticRoot(directory, store, { maxHistory, ignore, generateMasl });
+      if (maslCid) {
+        console.log(`Static root indexed: ${directory} → MASL ${maslCid}`);
+      } else if (!generateMasl) {
+        console.log(`Static root indexed: ${directory} (blobs only, no MASL)`);
+      } else {
+        console.warn(`Static root empty, skipped: ${directory}`);
+      }
+    } catch (err) {
+      console.error(`Failed to index static root ${directory}: ${err.message}`);
+    }
+  }
+}

package/src/storage/db.js

@@ -0,0 +1,133 @@
+import { DatabaseSync } from 'node:sqlite';
+import { mkdirSync } from 'fs';
+import { join } from 'path';
+
+const BASE_SCHEMA = `
+CREATE TABLE IF NOT EXISTS content (
+  cid TEXT PRIMARY KEY,
+  masl_cid TEXT,
+  size INTEGER NOT NULL,
+  pinned INTEGER NOT NULL DEFAULT 0,
+  last_requested INTEGER,
+  source_path TEXT,
+  source_mtime INTEGER
+);
+CREATE TABLE IF NOT EXISTS mount_points (
+  hostname TEXT NOT NULL,
+  mount_path TEXT NOT NULL DEFAULT '',
+  masl_cid TEXT NOT NULL,
+  PRIMARY KEY (hostname, mount_path)
+);
+`;
+
+export function openDb(dataDir) {
+  mkdirSync(dataDir, { recursive: true });
+  const db = new DatabaseSync(join(dataDir, 'rasler.db'));
+  db.exec(BASE_SCHEMA);
+  return db;
+}
+
+// ---- Content ----
+
+// pinned = 2: static filesystem-backed content. Never evicted, not counted
+// toward pool or pinned capacity. source_path is the resolved absolute path.
+export function dbPutStaticContent(db, cid, { maslCid = null, size, sourcePath, sourceMtime }) {
+  db.prepare(`
+    INSERT INTO content (cid, masl_cid, size, pinned, source_path, source_mtime, last_requested)
+    VALUES (?, ?, ?, 2, ?, ?, NULL)
+    ON CONFLICT(cid) DO UPDATE SET
+      masl_cid = excluded.masl_cid,
+      size = excluded.size,
+      pinned = 2,
+      source_path = excluded.source_path,
+      source_mtime = excluded.source_mtime
+  `).run(cid, maslCid, size, sourcePath, sourceMtime);
+}
+
+export function dbGetContentBySourcePath(db, sourcePath) {
+  return db.prepare('SELECT * FROM content WHERE source_path = ?').get(sourcePath) ?? null;
+}
+
+export function dbListStaticContent(db) {
+  return db.prepare('SELECT * FROM content WHERE pinned = 2').all();
+}
+
+export function dbPutContent(db, cid, { maslCid = null, size, pinned = 0, lastRequested = null }) {
+  const stmt = db.prepare(`
+    INSERT INTO content (cid, masl_cid, size, pinned, last_requested)
+    VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(cid) DO UPDATE SET
+      masl_cid = excluded.masl_cid,
+      size = excluded.size,
+      pinned = MAX(pinned, excluded.pinned),
+      last_requested = excluded.last_requested
+  `);
+  stmt.run(cid, maslCid, size, pinned ? 1 : 0, lastRequested);
+}
+
+export function dbGetContent(db, cid) {
+  return db.prepare('SELECT * FROM content WHERE cid = ?').get(cid) ?? null;
+}
+
+export function dbHasContent(db, cid) {
+  return !!db.prepare('SELECT 1 FROM content WHERE cid = ?').get(cid);
+}
+
+export function dbListContent(db) {
+  return db.prepare('SELECT * FROM content').all();
+}
+
+export function dbCountContent(db) {
+  return db.prepare('SELECT COUNT(*) AS cnt FROM content').get().cnt;
+}
+
+export function dbListContentPage(db, limit, cursor) {
+  if (cursor) {
+    return db.prepare('SELECT * FROM content WHERE cid > ? ORDER BY cid LIMIT ?').all(cursor, limit);
+  }
+  return db.prepare('SELECT * FROM content ORDER BY cid LIMIT ?').all(limit);
+}
+
+export function dbDeleteContent(db, cid) {
+  db.prepare('DELETE FROM content WHERE cid = ?').run(cid);
+}
+
+export function dbRecordRequest(db, cid) {
+  db.prepare('UPDATE content SET last_requested = ? WHERE cid = ?').run(Date.now(), cid);
+}
+
+export function dbSetPinned(db, cid, pinned) {
+  db.prepare('UPDATE content SET pinned = ? WHERE cid = ?').run(pinned ? 1 : 0, cid);
+}
+
+export function dbGetTotalPoolSize(db) {
+  const row = db.prepare('SELECT SUM(size) AS total FROM content WHERE pinned = 0').get();
+  return row?.total ?? 0;
+}
+
+export function dbGetTotalPinnedSize(db) {
+  const row = db.prepare('SELECT SUM(size) AS total FROM content WHERE pinned = 1').get();
+  return row?.total ?? 0;
+}
+
+export function dbCountPinned(db) {
+  const row = db.prepare('SELECT COUNT(*) AS cnt FROM content WHERE pinned = 1').get();
+  return row?.cnt ?? 0;
+}
+
+// ---- Mount points ----
+
+export function dbSetMountPoint(db, hostname, mountPath, maslCid) {
+  db.prepare(`
+    INSERT INTO mount_points (hostname, mount_path, masl_cid) VALUES (?, ?, ?)
+    ON CONFLICT(hostname, mount_path) DO UPDATE SET masl_cid = excluded.masl_cid
+  `).run(hostname, mountPath, maslCid);
+}
+
+export function dbDeleteMountPoint(db, hostname, mountPath) {
+  db.prepare('DELETE FROM mount_points WHERE hostname = ? AND mount_path = ?').run(hostname, mountPath);
+}
+
+export function dbListMountPoints(db) {
+  return db.prepare('SELECT hostname, mount_path, masl_cid FROM mount_points').all();
+}

package/src/storage/files.js

@@ -0,0 +1,47 @@
+import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, createReadStream } from 'fs';
+import { join } from 'path';
+
+function blobPath(dataDir, cid) {
+  // Use chars [-3, -2] of the CID as the shard directory (e.g. "nq" for ...vnq4).
+  // The first two characters of every CIDv1 base32 string are always "ba", so
+  // sharding by prefix gives no distribution benefit.
+  const shard = cid.slice(-3, -1);
+  return join(dataDir, 'blobs', shard, cid);
+}
+
+export function writeContent(dataDir, cid, bytes) {
+  const path = blobPath(dataDir, cid);
+  mkdirSync(join(path, '..'), { recursive: true });
+  writeFileSync(path, bytes);
+}
+
+export function readContent(dataDir, cid) {
+  const path = blobPath(dataDir, cid);
+  if (!existsSync(path)) return null;
+  return readFileSync(path);
+}
+
+export function deleteContent(dataDir, cid) {
+  const path = blobPath(dataDir, cid);
+  if (existsSync(path)) unlinkSync(path);
+}
+
+export function hasContent(dataDir, cid) {
+  return existsSync(blobPath(dataDir, cid));
+}
+
+export function readContentStream(dataDir, cid) {
+  const path = blobPath(dataDir, cid);
+  if (!existsSync(path)) return null;
+  return createReadStream(path);
+}
+
+export function readContentStreamFromPath(absolutePath) {
+  if (!existsSync(absolutePath)) return null;
+  return createReadStream(absolutePath);
+}
+
+export function readContentFromPath(absolutePath) {
+  if (!existsSync(absolutePath)) return null;
+  return readFileSync(absolutePath);
+}