rasler 0.1.0

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "rasler",
+  "version": "0.1.0",
+  "description": "RASLer — RASL protocol node implementation with content-addressed storage and MASL support",
+  "license": "Apache-2.0",
+  "author": "Wes Biggs <github@wbig.gs>",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/wesbiggs/rasler.git"
+  },
+  "homepage": "https://github.com/wesbiggs/rasler#readme",
+  "keywords": [
+    "rasl",
+    "dasl",
+    "masl",
+    "content-addressed",
+    "ipld",
+    "cid",
+    "decentralized"
+  ],
+  "type": "module",
+  "main": "src/index.js",
+  "files": [
+    "src/",
+    "openapi.json",
+    "LICENSE"
+  ],
+  "scripts": {
+    "start": "node src/index.js",
+    "lint": "eslint src/ scripts/ test/",
+    "test": "npm run lint && NODE_OPTIONS=--experimental-vm-modules jest --runInBand",
+    "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest test/unit --runInBand",
+    "test:integration": "NODE_OPTIONS=--experimental-vm-modules jest test/integration --runInBand",
+    "generate:openapi": "node scripts/generate-openapi.js",
+    "generate:docs": "node scripts/generate-docs.js",
+    "get-in-the-car": "node scripts/get-in-the-car.js"
+  },
+  "dependencies": {
+    "@ipld/car": "5.4.6",
+    "@ipld/dag-cbor": "10.0.1",
+    "express": "5.2.1",
+    "micromatch": "4.0.8",
+    "multer": "2.1.1",
+    "multiformats": "14.0.0",
+    "swagger-ui-express": "5.0.1"
+  },
+  "devDependencies": {
+    "@eslint/js": "10.0.1",
+    "@jest/globals": "30.4.1",
+    "eslint": "10.4.1",
+    "globals": "17.6.0",
+    "jest": "30.4.2",
+    "marked": "18.0.4",
+    "supertest": "7.2.2",
+    "swagger-jsdoc": "6.3.0"
+  },
+  "jest": {
+    "transform": {},
+    "testEnvironment": "node",
+    "forceExit": true,
+    "testMatch": [
+      "<rootDir>/test/**/*.test.js"
+    ]
+  },
+  "engines": {
+    "node": ">=23.4.0"
+  }
+}

package/src/config.js

@@ -0,0 +1,81 @@
+import { resolve } from 'node:path';
+import { existsSync } from 'node:fs';
+import { parseSize } from './util/parseSize.js';
+import { required, optional } from './util/env.js';
+import { normalizeMountPath } from './util/normalizeMountPath.js';
+import { loadRaslerConfig } from './util/loadRaslerConfig.js';
+import { parseJsonStaticRoots, parseJsonMountPoints } from './util/parseJsonConfig.js';
+
+export { parseSize, normalizeMountPath };
+
+const port = parseInt(optional('PORT', '3000'), 10);
+
+// Accept a bare hostname (e.g. "node1.example.com") for backwards compat and
+// default it to https://. A full origin (e.g. "http://localhost:3000") is used
+// as-is so the correct protocol appears in Link: rel="duplicate" headers.
+const originRaw = optional('ORIGIN', `http://localhost:${port}`);
+const originUrl = new URL(/^https?:\/\//.test(originRaw) ? originRaw : `https://${originRaw}`);
+
+const raslerConfig = loadRaslerConfig();
+
+const mountPoints = parseJsonMountPoints(raslerConfig?.mountPoints ?? []);
+
+// Static roots from JSON config, plus any directories referenced by mountPoints.
+const staticRoots = parseJsonStaticRoots(raslerConfig?.staticRoots ?? []);
+const mountPointDirs = new Set(mountPoints.map(mp => mp.directory));
+for (const dir of mountPointDirs) {
+  const existing = staticRoots.find(r => r.directory === dir);
+  if (!existing) {
+    staticRoots.push({ directory: dir, watch: false, ignore: [], generateMasl: true });
+  } else if (!existing.generateMasl) {
+    // Mount-point directories always need a bundle MASL for serving.
+    existing.generateMasl = true;
+  }
+}
+
+// Implicit ./public mount: if the directory exists and no root-level mount is
+// already configured for the origin domain, serve it at the document root.
+const publicDir = resolve(process.cwd(), 'public');
+if (existsSync(publicDir)) {
+  if (!staticRoots.some(r => r.directory === publicDir)) {
+    staticRoots.push({ directory: publicDir, watch: false, ignore: [], generateMasl: true });
+  }
+  if (!mountPoints.some(mp => (mp.hostname === originUrl.hostname || mp.hostname === '') && mp.prefix === '')) {
+    mountPoints.push({ hostname: originUrl.hostname, prefix: '', directory: publicDir });
+    mountPoints.sort((a, b) => b.prefix.length - a.prefix.length);
+  }
+}
+
+const config = Object.freeze({
+  origin: originUrl.origin,
+  domain: originUrl.host,
+  port,
+  dataDir: resolve(optional('DATA_DIR', './data')),
+  totalCapacity: parseSize(optional('TOTAL_CAPACITY', '1G')),
+  apiSecret: (() => {
+    const secret = required('API_SECRET');
+    if (secret === 'change-me-to-a-strong-random-secret') {
+      throw new Error('API_SECRET is still set to the default placeholder — please change it before starting the server');
+    }
+    return secret;
+  })(),
+  swaggerUi: optional('SWAGGER_UI', 'true') === 'true',
+  operatorApiPathPrefix: normalizeMountPath(optional('OPERATOR_API_PATH_PREFIX', '')),
+  operatorCorsOrigins: Array.isArray(raslerConfig?.operatorCorsOrigins)
+    ? raslerConfig.operatorCorsOrigins.filter(s => typeof s === 'string' && s.trim()).map(s => s.trim())
+    : [],
+  // Array of { hostname, prefix, directory } sorted longest-prefix-first.
+  mountPoints,
+  // Array of { directory, watch, ignore } for all static roots (includes mount
+  // point directories and the implicit ./public if it exists).
+  staticRoots,
+  // Maximum number of MASL versions to keep pinned per static root (including
+  // the current one). Older entries are unpinned and become eligible for LRU
+  // eviction. Unset or 0 means no limit.
+  staticMaxHistory: (() => {
+    const n = parseInt(raslerConfig?.staticMaxHistory ?? 0, 10);
+    return Number.isFinite(n) && n >= 1 ? n : null;
+  })(),
+});
+
+export default config;

package/src/crypto/cid.js

@@ -0,0 +1,49 @@
+import { CID } from 'multiformats/cid';
+import { sha256 } from 'multiformats/hashes/sha2';
+import * as raw from 'multiformats/codecs/raw';
+import * as dagCbor from '@ipld/dag-cbor';
+import { base32 } from 'multiformats/bases/base32';
+
+export async function computeDataCid(bytes) {
+  const hash = await sha256.digest(bytes);
+  const cid = CID.create(1, raw.code, hash);
+  return cid.toString(base32);
+}
+
+export async function computeMaslCid(cborBytes) {
+  const hash = await sha256.digest(cborBytes);
+  const cid = CID.create(1, dagCbor.code, hash);
+  return cid.toString(base32);
+}
+
+// Returns true when the CID uses the dag-cbor codec (0x71), meaning it is a MASL document.
+export function isMaslCid(cidString) {
+  try {
+    const cid = CID.parse(cidString, base32);
+    return cid.code === dagCbor.code;
+  } catch {
+    return false;
+  }
+}
+
+// Returns the Unencoded-Digest header value for a CID.
+// The SHA-256 bytes are extracted directly from the CID's multihash — no
+// additional hashing required. Works for both data and MASL CIDs.
+export function cidToUnencodedDigest(cidString) {
+  const cid = CID.parse(cidString, base32);
+  return 'sha-256=:' + Buffer.from(cid.multihash.digest).toString('base64') + ':';
+}
+
+export function getRingPosition(cidString) {
+  const cid = CID.parse(cidString, base32);
+  const digest = cid.multihash.digest;
+  return bufferToBigInt(digest);
+}
+
+function bufferToBigInt(bytes) {
+  let result = 0n;
+  for (const byte of bytes) {
+    result = (result << 8n) | BigInt(byte);
+  }
+  return result;
+}

package/src/index.js

@@ -0,0 +1,45 @@
+import config from './config.js';
+import { openDb } from './storage/db.js';
+import { Store } from './storage/store.js';
+import { createApp, finalizeApp } from './server.js';
+import { makeRaslNotFoundHandler } from './routes/rasl.js';
+import { makeOperatorRouter } from './routes/operator.js';
+import { indexStaticRoots } from './static.js';
+import { startStaticRootWatchers } from './watcher.js';
+
+function main() {
+  const db = openDb(config.dataDir);
+  const store = new Store(db, config.dataDir, config.totalCapacity, {
+    staticRoots: config.staticRoots,
+  });
+
+  // Index static roots in the background so the server starts immediately.
+  // Watchers start after the initial index so startup writes don't trigger
+  // spurious re-indexes. On a warm restart (no file changes) the mtime cache
+  // makes the window negligible.
+  if (config.staticRoots.length > 0) {
+    indexStaticRoots(config.staticRoots, store, { maxHistory: config.staticMaxHistory })
+      .then(() => startStaticRootWatchers(config.staticRoots, store, { maxHistory: config.staticMaxHistory }));
+  }
+
+  const app = createApp({ store, config });
+  const prefix = config.operatorApiPathPrefix || '/';
+
+  app.use(makeRaslNotFoundHandler());
+  app.use(prefix, makeOperatorRouter({
+    store,
+    selfOrigin: config.origin,
+    apiSecret: config.apiSecret,
+    corsOrigins: config.operatorCorsOrigins,
+    staticRoots: config.staticRoots,
+    mountPoints: config.mountPoints,
+  }));
+
+  finalizeApp(app, config);
+
+  app.listen(config.port, () => {
+    console.log(`RASL node ${config.origin} listening on port ${config.port}`);
+  });
+}
+
+main();

package/src/masl/document.js

@@ -0,0 +1,148 @@
+import * as dagCbor from '@ipld/dag-cbor';
+import { computeMaslCid } from '../crypto/cid.js';
+
+// MASL link format: { "$link": cidString }
+function link(cidString) {
+  return { $link: cidString };
+}
+
+function linkCid(linkObj) {
+  return linkObj?.$link ?? null;
+}
+
+// HTTP header keys that MASL documents may carry and that nodes must forward when serving.
+const HTTP_HEADER_KEYS = [
+  'content-type',
+  'content-disposition',
+  'content-encoding',
+  'content-language',
+];
+
+// Collect all present HTTP header fields from a doc or resource entry object.
+function httpHeaders(obj) {
+  const headers = {};
+  for (const key of HTTP_HEADER_KEYS) {
+    if (obj[key] != null) headers[key] = obj[key];
+  }
+  if (!headers['content-type']) headers['content-type'] = 'application/octet-stream';
+  return headers;
+}
+
+export async function createSingleMasl({
+  name, type, size, dataCid,
+  contentEncoding, contentLanguage,
+}) {
+  const doc = {
+    name,
+    type,
+    'content-length': size,
+    src: link(dataCid),
+    'content-type': type,
+    'content-disposition': `inline; filename="${name}"`,
+    ...(contentEncoding != null ? { 'content-encoding': contentEncoding } : {}),
+    ...(contentLanguage != null ? { 'content-language': contentLanguage } : {}),
+  };
+  const cborBytes = dagCbor.encode(doc);
+  const maslCid = await computeMaslCid(cborBytes);
+  return { doc, cborBytes, maslCid };
+}
+
+export async function createBundleMasl({ name, resources, prevMaslCid = null }) {
+  // resources: Array of { path, cid, size, contentType, contentDisposition?,
+  //                        contentEncoding?, contentLanguage? }
+  const resourceMap = {};
+  for (const { path, cid, size, contentType, contentDisposition, contentEncoding, contentLanguage } of resources) {
+    resourceMap[path] = {
+      src: link(cid),
+      'content-length': size,
+      'content-type': contentType ?? 'application/octet-stream',
+      ...(contentDisposition != null ? { 'content-disposition': contentDisposition } : {}),
+      ...(contentEncoding != null   ? { 'content-encoding': contentEncoding }   : {}),
+      ...(contentLanguage != null   ? { 'content-language': contentLanguage }   : {}),
+    };
+  }
+  const doc = {
+    name,
+    ...(prevMaslCid != null ? { prev: link(prevMaslCid) } : {}),
+    resources: resourceMap,
+  };
+  const cborBytes = dagCbor.encode(doc);
+  const maslCid = await computeMaslCid(cborBytes);
+  return { doc, cborBytes, maslCid };
+}
+
+export function parseMasl(cborBytes) {
+  return dagCbor.decode(cborBytes);
+}
+
+// Returns [{ cid: string, size: number|null }] for all directly linked data CIDs.
+export function maslLinkedCids(doc) {
+  const results = [];
+  if (doc.src) {
+    const cid = linkCid(doc.src);
+    if (cid) results.push({ cid, size: doc['content-length'] ?? null });
+  } else if (doc.resources) {
+    for (const entry of Object.values(doc.resources)) {
+      if (entry && typeof entry === 'object') {
+        const cid = linkCid(entry.src);
+        if (cid) results.push({ cid, size: entry['content-length'] ?? null });
+      }
+    }
+  }
+  return results;
+}
+
+// Returns HTTP headers to set when serving a single-mode MASL CID or a data CID
+// whose MASL wrapper is known.
+export function maslContentHeaders(doc) {
+  return httpHeaders(doc);
+}
+
+export function maslIsBundle(doc) {
+  return Boolean(doc.resources);
+}
+
+// Returns true if bytes appear to be a MASL document (dag-cbor map with src.$link
+// or a resources map whose entries have src.$link). Used to reject MASL uploads
+// from POST /pin, which only accepts raw data files.
+export function isMaslDoc(bytes) {
+  try {
+    const doc = dagCbor.decode(bytes);
+    if (!doc || typeof doc !== 'object') return false;
+    if (doc.src?.$link != null) return true;
+    if (doc.resources && typeof doc.resources === 'object') {
+      return Object.values(doc.resources).some(e => e?.src?.$link != null);
+    }
+    return false;
+  } catch {
+    return false;
+  }
+}
+
+// Finds the first resource entry in a bundle whose src CID matches dataCid,
+// and returns its HTTP headers. Used to surface content-type for path-free
+// data CID requests backed by a bundle MASL rather than a single MASL.
+export function findBundleHeadersForCid(doc, dataCid) {
+  if (!doc.resources) return null;
+  for (const entry of Object.values(doc.resources)) {
+    if (entry?.src?.$link === dataCid) return httpHeaders(entry);
+  }
+  return null;
+}
+
+// Returns { cid, headers } for the given bundle path, or null if not found.
+// headers is ready to pass directly to res.set().
+export function resolveBundleEntry(doc, pathSuffix) {
+  if (!doc.resources) return null;
+  const key = pathSuffix || '/';
+  const entry = doc.resources[key];
+  if (!entry) return null;
+  const cid = linkCid(entry.src);
+  if (!cid) return null;
+  return { cid, headers: httpHeaders(entry) };
+}
+
+// Convenience wrapper — returns only the CID string.
+export function resolveBundlePath(doc, pathSuffix) {
+  return resolveBundleEntry(doc, pathSuffix)?.cid ?? null;
+}

package/src/middleware/auth.js

@@ -0,0 +1,11 @@
+export const OPERATOR_SECRET_HEADER = 'x-rasl-operator-secret';
+
+export function requireApiSecret(apiSecret) {
+  return (req, res, next) => {
+    const provided = req.headers[OPERATOR_SECRET_HEADER];
+    if (!provided || provided !== apiSecret) {
+      return res.status(401).json({ error: 'Unauthorized' });
+    }
+    next();
+  };
+}

package/src/middleware/cors.js

@@ -0,0 +1,31 @@
+import { OPERATOR_SECRET_HEADER } from './auth.js';
+
+// Returns Express middleware that adds CORS headers to operator API responses.
+// origins: string[] — pass ['*'] to allow all, or a list of specific origins.
+export function makeOperatorCors(origins) {
+  const allowAll = origins.includes('*');
+
+  return (req, res, next) => {
+    const requestOrigin = req.headers.origin;
+
+    if (!requestOrigin) return next();
+
+    if (allowAll) {
+      res.set('Access-Control-Allow-Origin', '*');
+    } else if (origins.includes(requestOrigin)) {
+      res.set('Access-Control-Allow-Origin', requestOrigin);
+      res.vary('Origin');
+    } else {
+      // Origin not in whitelist — no CORS headers; browser will block the request.
+      if (req.method === 'OPTIONS') return res.sendStatus(204);
+      return next();
+    }
+
+    res.set('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS');
+    res.set('Access-Control-Allow-Headers', `Content-Type, ${OPERATOR_SECRET_HEADER}`);
+
+    if (req.method === 'OPTIONS') return res.sendStatus(204);
+
+    next();
+  };
+}

package/src/routes/mountPoints.js

@@ -0,0 +1,86 @@
+import { Router } from 'express';
+import { realpathSync } from 'fs';
+import { parseMasl, resolveBundleEntry } from '../masl/document.js';
+import { cidToUnencodedDigest } from '../crypto/cid.js';
+import { OPERATOR_SECRET_HEADER } from '../middleware/auth.js';
+
+// Returns the first mount point whose hostname and prefix match the request,
+// or null. mountPoints must be sorted: longer prefix first, specific hostname
+// before wildcard (hostname='') at equal prefix lengths.
+// hostname='' matches any Host: value (or no Host: header).
+function findMountPoint(mountPoints, hostname, path) {
+  for (const mp of mountPoints) {
+    if (mp.hostname !== '' && mp.hostname !== hostname) continue;
+    if (mp.prefix === '' || path === mp.prefix || path.startsWith(mp.prefix + '/')) {
+      return mp;
+    }
+  }
+  return null;
+}
+
+// Serves content by resolving the request path against the bundle MASL for
+// the matching mount point. Sits before the RASL router so browser clients
+// can use normal URLs; RASL paths are passed through unchanged.
+//
+// The MASL CID is read from store.staticRootMasls on every request, so it
+// reflects the current indexed version automatically after any re-index.
+export function makeMountPointRouter({ store, mountPoints, selfOrigin }) {
+  const router = Router();
+
+  router.use((req, res, next) => {
+    if (req.method !== 'GET' && req.method !== 'HEAD') return next();
+    // Leave RASL retrieval paths to the RASL router.
+    if (req.path.startsWith('/.well-known/rasl/')) return next();
+    // Operator API requests (identified by the secret header) bypass mount-point
+    // routing so authenticated operator paths are always reachable even when a
+    // wildcard mount is configured.
+    if (req.headers[OPERATOR_SECRET_HEADER]) return next();
+
+    let maslCid;
+    let maslPath;
+
+    // Runtime mappings take priority over static-root mappings.
+    const runtimeMp = findMountPoint(store.runtimeMountPoints, req.hostname, req.path);
+    if (runtimeMp) {
+      maslCid = runtimeMp.maslCid;
+      maslPath = runtimeMp.prefix ? req.path.slice(runtimeMp.prefix.length) || '/' : req.path;
+    } else {
+      const staticMp = findMountPoint(mountPoints, req.hostname, req.path);
+      if (!staticMp) return next();
+
+      let realRoot;
+      try { realRoot = realpathSync(staticMp.directory); } catch { return next(); }
+
+      maslCid = store.staticRootMasls.get(realRoot) ?? null;
+      if (!maslCid) return res.status(503).json({ error: 'Mount point not yet indexed' });
+
+      maslPath = staticMp.prefix ? req.path.slice(staticMp.prefix.length) || '/' : req.path;
+    }
+
+    const maslEntry = store.getContent(maslCid);
+    if (!maslEntry) return res.status(503).json({ error: 'MASL unavailable' });
+
+    let doc;
+    try { doc = parseMasl(maslEntry.bytes); } catch {
+      return res.status(500).json({ error: 'Invalid MASL document' });
+    }
+
+    const resolved = resolveBundleEntry(doc, maslPath);
+    if (!resolved) return res.status(404).send('Not found');
+
+    store.recordRequest(maslCid);
+    store.recordRequest(resolved.cid);
+
+    const result = store.getContentStream(resolved.cid);
+    if (!result) return res.status(502).json({ error: 'Content unavailable' });
+
+    res.set(resolved.headers);
+    res.set('content-length', String(result.meta.size));
+    res.set('unencoded-digest', cidToUnencodedDigest(resolved.cid));
+    res.set('link', `<${selfOrigin}/.well-known/rasl/${maslCid}${maslPath}>; rel="duplicate"`);
+    res.status(200);
+    result.stream.pipe(res);
+  });
+
+  return router;
+}