rasler 0.1.0

package/src/storage/store.js

@@ -0,0 +1,171 @@
+import {
+  dbPutContent, dbGetContent, dbHasContent, dbListContent, dbDeleteContent,
+  dbRecordRequest, dbSetPinned, dbGetTotalPoolSize, dbGetTotalPinnedSize,
+  dbCountPinned, dbCountContent, dbListContentPage,
+  dbSetMountPoint, dbDeleteMountPoint, dbListMountPoints,
+} from './db.js';
+import { sortMountPoints } from '../util/parseJsonConfig.js';
+import {
+  writeContent, readContent, readContentFromPath,
+  readContentStream, readContentStreamFromPath, deleteContent,
+} from './files.js';
+import { realpathSync } from 'fs';
+import { sep } from 'path';
+
+// Default eviction policy: oldest unpinned by last_requested (LRU).
+// An overlay can supply a network-aware policy that considers
+// replica counts and primary-holder status.
+function defaultFindEvictionCandidate(store) {
+  const row = store.db.prepare(`
+    SELECT cid FROM content
+    WHERE pinned = 0
+    ORDER BY last_requested ASC NULLS FIRST
+    LIMIT 1
+  `).get();
+  return row?.cid ?? null;
+}
+
+export class Store {
+  constructor(db, dataDir, totalCapacity, { findEvictionCandidate, staticRoots = [] } = {}) {
+    this.db = db;
+    this.dataDir = dataDir;
+    this.totalCapacity = totalCapacity;
+    this._findEvictionCandidate = findEvictionCandidate ?? defaultFindEvictionCandidate;
+    // Pre-resolve static roots once so symlink checks at serve time are fast.
+    this._realStaticRoots = staticRoots.map(r => {
+      const dir = typeof r === 'string' ? r : r.directory;
+      try { return realpathSync(dir); } catch { return dir; }
+    });
+    // Populated by indexStaticRoot after each root is indexed. Maps realpath → maslCid.
+    this.staticRootMasls = new Map();
+    // Runtime mount point mappings set via operator API. Persisted in SQLite.
+    // Array of {hostname, prefix, maslCid} — hostname='' means any host.
+    // Sorted: longer prefix first; equal prefix: specific hostname before wildcard.
+    // Takes priority over staticRootMasls in mount-point routing.
+    const runtimeRows = dbListMountPoints(db)
+      .map(row => ({ hostname: row.hostname, prefix: row.mount_path, maslCid: row.masl_cid }));
+    sortMountPoints(runtimeRows);
+    this.runtimeMountPoints = runtimeRows;
+  }
+
+  putContent(cid, bytes, { maslCid = null, pinned = false } = {}) {
+    writeContent(this.dataDir, cid, bytes);
+    dbPutContent(this.db, cid, {
+      maslCid,
+      size: bytes.length,
+      pinned,
+      lastRequested: null,
+    });
+  }
+
+  getContent(cid) {
+    const meta = dbGetContent(this.db, cid);
+    if (!meta) return null;
+    const bytes = meta.source_path
+      ? readContentFromPath(meta.source_path)
+      : readContent(this.dataDir, cid);
+    if (!bytes) return null;
+    return { bytes, meta };
+  }
+
+  // Returns { stream: ReadStream, meta } for efficient large-file serving,
+  // or null if the content is unavailable. For static entries, verifies
+  // that source_path still resolves to a path under a configured static root.
+  getContentStream(cid) {
+    const meta = dbGetContent(this.db, cid);
+    if (!meta) return null;
+    if (meta.source_path) {
+      let realFile;
+      try { realFile = realpathSync(meta.source_path); } catch { return null; }
+      const allowed = this._realStaticRoots.some(
+        root => realFile === root || realFile.startsWith(root + sep)
+      );
+      if (!allowed) return null;
+      const stream = readContentStreamFromPath(meta.source_path);
+      if (!stream) return null;
+      return { stream, meta };
+    }
+    const stream = readContentStream(this.dataDir, cid);
+    if (!stream) return null;
+    return { stream, meta };
+  }
+
+  getContentMeta(cid) {
+    return dbGetContent(this.db, cid);
+  }
+
+  hasContent(cid) {
+    return dbHasContent(this.db, cid);
+  }
+
+  listContent() {
+    return dbListContent(this.db);
+  }
+
+  countContent() {
+    return dbCountContent(this.db);
+  }
+
+  listContentPage(limit, cursor) {
+    return dbListContentPage(this.db, limit, cursor);
+  }
+
+  deleteContent(cid) {
+    const meta = dbGetContent(this.db, cid);
+    // Static content: bytes live on operator's filesystem; only remove the DB record.
+    if (!meta?.source_path) deleteContent(this.dataDir, cid);
+    dbDeleteContent(this.db, cid);
+  }
+
+  recordRequest(cid) {
+    dbRecordRequest(this.db, cid);
+  }
+
+  setPinned(cid, pinned) {
+    dbSetPinned(this.db, cid, pinned);
+  }
+
+  getPoolUsed() {
+    return dbGetTotalPoolSize(this.db);
+  }
+
+  getPinnedUsed() {
+    return dbGetTotalPinnedSize(this.db);
+  }
+
+  getPoolAvailable() {
+    return Math.max(0, this.totalCapacity - this.getPoolUsed() - this.getPinnedUsed());
+  }
+
+  countPinned() {
+    return dbCountPinned(this.db);
+  }
+
+  // Evicts one CID if needed to free requiredBytes. Returns true if eviction
+  // was performed or not needed, false if impossible. The eviction policy is
+  // injected via the constructor; replica-row cleanup happens automatically
+  // via FK cascade on the replicas table.
+  setMountPoint(hostname, prefix, maslCid) {
+    dbSetMountPoint(this.db, hostname, prefix, maslCid);
+    this.runtimeMountPoints = this.runtimeMountPoints.filter(
+      mp => !(mp.hostname === hostname && mp.prefix === prefix)
+    );
+    this.runtimeMountPoints.push({ hostname, prefix, maslCid });
+    sortMountPoints(this.runtimeMountPoints);
+  }
+
+  deleteMountPoint(hostname, prefix) {
+    dbDeleteMountPoint(this.db, hostname, prefix);
+    this.runtimeMountPoints = this.runtimeMountPoints.filter(
+      mp => !(mp.hostname === hostname && mp.prefix === prefix)
+    );
+  }
+
+  evictIfNeeded(requiredBytes) {
+    if (this.getPoolAvailable() >= requiredBytes) return true;
+    const cid = this._findEvictionCandidate(this);
+    if (!cid) return false;
+    this.deleteContent(cid);
+    return true;
+  }
+}

package/src/util/env.js

@@ -0,0 +1,32 @@
+import { readFileSync } from 'fs';
+import { resolve } from 'path';
+
+function loadEnvFile() {
+  try {
+    const raw = readFileSync(resolve(process.cwd(), '.env'), 'utf8');
+    for (const line of raw.split('\n')) {
+      const trimmed = line.trim();
+      if (!trimmed || trimmed.startsWith('#')) continue;
+      const eq = trimmed.indexOf('=');
+      if (eq === -1) continue;
+      const key = trimmed.slice(0, eq).trim();
+      const val = trimmed.slice(eq + 1).trim();
+      if (!(key in process.env)) process.env[key] = val;
+    }
+  } catch {
+    // .env file is optional
+  }
+}
+
+loadEnvFile();
+
+export function required(name) {
+  const val = process.env[name];
+  if (!val) throw new Error(`Missing required environment variable: ${name}`);
+  return val;
+}
+
+export function optional(name, defaultValue) {
+  const val = process.env[name];
+  return val !== undefined && val !== '' ? val : defaultValue;
+}

package/src/util/loadRaslerConfig.js

@@ -0,0 +1,14 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+// Reads rasler.config.json from CWD. Returns the parsed object, or null if
+// the file does not exist. Throws on malformed JSON.
+export function loadRaslerConfig(configPath = 'rasler.config.json') {
+  try {
+    const raw = readFileSync(resolve(process.cwd(), configPath), 'utf8');
+    return JSON.parse(raw);
+  } catch (err) {
+    if (err.code === 'ENOENT') return null;
+    throw new Error(`Failed to parse ${configPath}: ${err.message}`, { cause: err });
+  }
+}

package/src/util/mime.js

@@ -0,0 +1,29 @@
+const MIME_TYPES = {
+  '.html': 'text/html; charset=utf-8',
+  '.css': 'text/css',
+  '.js': 'application/javascript',
+  '.mjs': 'application/javascript',
+  '.json': 'application/json',
+  '.xml': 'application/xml',
+  '.txt': 'text/plain',
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  '.gif': 'image/gif',
+  '.webp': 'image/webp',
+  '.svg': 'image/svg+xml',
+  '.ico': 'image/x-icon',
+  '.woff': 'font/woff',
+  '.woff2': 'font/woff2',
+  '.ttf': 'font/ttf',
+  '.otf': 'font/otf',
+  '.mp4': 'video/mp4',
+  '.webm': 'video/webm',
+  '.pdf': 'application/pdf',
+};
+
+import { extname } from 'path';
+
+export function mimeType(filePath) {
+  return MIME_TYPES[extname(filePath).toLowerCase()] ?? 'application/octet-stream';
+}

package/src/util/normalizeMountPath.js

@@ -0,0 +1,12 @@
+// Normalizes a URL path prefix for use as a mount point:
+//   '/'  → ''   (root — represented as empty string internally)
+//   '/docs/'  → '/docs'
+//   'docs'    → '/docs'  (adds leading slash)
+//   ''        → ''
+export function normalizeMountPath(raw) {
+  let p = (raw ?? '').trim();
+  if (!p) return '';
+  if (!p.startsWith('/')) p = '/' + p;
+  while (p.length > 1 && p.endsWith('/')) p = p.slice(0, -1);
+  return p === '/' ? '' : p;
+}

package/src/util/parseJsonConfig.js

@@ -0,0 +1,57 @@
+import { resolve } from 'node:path';
+import { normalizeMountPath } from './normalizeMountPath.js';
+
+// Parses the staticRoots array from rasler.config.json.
+// Each entry may be a string (path only) or an object { path, watch?, ignore?, generateMasl? }.
+// Returns an array of { directory, watch, ignore, generateMasl }.
+export function parseJsonStaticRoots(entries = []) {
+  return entries.flatMap(entry => {
+    if (typeof entry === 'string') {
+      const p = entry.trim();
+      if (!p) return [];
+      return [{ directory: resolve(p), watch: false, ignore: [], generateMasl: true }];
+    }
+    if (entry !== null && typeof entry === 'object' && typeof entry.path === 'string' && entry.path.trim()) {
+      return [{
+        directory: resolve(entry.path.trim()),
+        watch: entry.watch === true,
+        ignore: Array.isArray(entry.ignore)
+          ? entry.ignore.filter(s => typeof s === 'string')
+          : [],
+        generateMasl: entry.generateMasl !== false,
+      }];
+    }
+    return [];
+  });
+}
+
+// Parses the mountPoints array from rasler.config.json.
+// Each entry: { hostname?, prefix?, directory }.
+// hostname is optional; omitting it (or setting it to '') creates a wildcard entry
+// that matches any Host: header value.
+// Returns an array of { hostname, prefix, directory } sorted longest-prefix-first,
+// with specific-hostname entries before wildcard entries at equal prefix lengths.
+export function parseJsonMountPoints(entries = []) {
+  const points = [];
+  for (const entry of entries) {
+    if (!entry || typeof entry !== 'object') continue;
+    const hostname = typeof entry.hostname === 'string' ? entry.hostname.trim() : '';
+    const directory = typeof entry.directory === 'string' ? entry.directory.trim() : '';
+    if (!directory) continue;
+    points.push({
+      hostname,
+      prefix: normalizeMountPath(entry.prefix || ''),
+      directory: resolve(directory),
+    });
+  }
+  sortMountPoints(points);
+  return points;
+}
+
+// Longer prefix wins; equal prefix: specific hostname before wildcard ('').
+export function sortMountPoints(points) {
+  points.sort((a, b) =>
+    b.prefix.length - a.prefix.length ||
+    (b.hostname ? 1 : 0) - (a.hostname ? 1 : 0)
+  );
+}

package/src/util/parseSize.js

@@ -0,0 +1,16 @@
+const SIZE_UNITS = {
+  '': 1, B: 1,
+  K: 1024,       KB: 1024,
+  M: 1024 ** 2,  MB: 1024 ** 2,
+  G: 1024 ** 3,  GB: 1024 ** 3,
+  T: 1024 ** 4,  TB: 1024 ** 4,
+};
+
+export function parseSize(value) {
+  const str = String(value).trim();
+  const match = str.match(/^(\d+(?:\.\d+)?)\s*([a-zA-Z]*)$/);
+  if (!match) throw new Error(`Invalid size value: "${value}"`);
+  const unit = match[2].toUpperCase();
+  if (!(unit in SIZE_UNITS)) throw new Error(`Unknown size unit "${match[2]}" in: "${value}"`);
+  return Math.round(parseFloat(match[1]) * SIZE_UNITS[unit]);
+}

package/src/watcher.js

@@ -0,0 +1,42 @@
+import { watch } from 'node:fs/promises';
+import { indexStaticRoot } from './static.js';
+
+// Starts a recursive fs watcher for a single static root. Re-indexes the root
+// after a 300 ms debounce whenever any file inside it changes.
+function watchStaticRoot({ directory, ignore, generateMasl = true }, store, { maxHistory }) {
+  let debounceTimer = null;
+
+  (async () => {
+    try {
+      const watcher = watch(directory, { recursive: true });
+      for await (const _ of watcher) {
+        clearTimeout(debounceTimer);
+        debounceTimer = setTimeout(async () => {
+          try {
+            const maslCid = await indexStaticRoot(directory, store, { maxHistory, ignore, generateMasl });
+            if (maslCid) {
+              console.log(`Static root re-indexed: ${directory} → MASL ${maslCid}`);
+            } else if (!generateMasl) {
+              console.log(`Static root re-indexed: ${directory} (blobs only, no MASL)`);
+            } else {
+              console.warn(`Static root re-indexed but empty: ${directory}`);
+            }
+          } catch (err) {
+            console.error(`Failed to re-index static root ${directory}: ${err.message}`);
+          }
+        }, 300);
+      }
+    } catch (err) {
+      if (err.code !== 'ABORT_ERR') {
+        console.error(`Watcher error for ${directory}: ${err.message}`);
+      }
+    }
+  })();
+}
+
+// Starts watchers for every static root that has watch: true.
+export function startStaticRootWatchers(staticRoots, store, { maxHistory = null } = {}) {
+  for (const root of staticRoots) {
+    if (root.watch) watchStaticRoot(root, store, { maxHistory });
+  }
+}