web-agent-bridge 3.2.0 → 3.3.0

package/server/utils/cache.js

@@ -1,125 +1,125 @@
-/**
- * WAB Caching Layer — In-memory cache with TTL for hot data
- * Reduces DB reads for license verification, config, and stats
- */
-class Cache {
-  constructor(defaultTTL = 60000) {
-    this.store = new Map();
-    this.defaultTTL = defaultTTL;
-    this._interval = setInterval(() => this._cleanup(), 120000);
-  }
-
-  get(key) {
-    const entry = this.store.get(key);
-    if (!entry) return undefined;
-    if (Date.now() > entry.expiresAt) {
-      this.store.delete(key);
-      return undefined;
-    }
-    entry.hits++;
-    return entry.value;
-  }
-
-  set(key, value, ttl) {
-    this.store.set(key, {
-      value,
-      expiresAt: Date.now() + (ttl || this.defaultTTL),
-      hits: 0
-    });
-  }
-
-  del(key) {
-    this.store.delete(key);
-  }
-
-  invalidatePattern(pattern) {
-    for (const key of this.store.keys()) {
-      if (key.includes(pattern)) this.store.delete(key);
-    }
-  }
-
-  stats() {
-    return {
-      size: this.store.size,
-      keys: Array.from(this.store.keys())
-    };
-  }
-
-  _cleanup() {
-    const now = Date.now();
-    for (const [key, entry] of this.store) {
-      if (now > entry.expiresAt) this.store.delete(key);
-    }
-  }
-
-  destroy() {
-    clearInterval(this._interval);
-    this.store.clear();
-  }
-}
-
-/**
- * Analytics Queue — Batches analytics inserts for better write performance
- * Flushes every N seconds or when buffer reaches max size.
- * maxBufferTotal caps memory if DB writes fail repeatedly (DoS mitigation).
- */
-class AnalyticsQueue {
-  constructor(db, options = {}) {
-    this.db = db;
-    this.buffer = [];
-    this.maxSize = options.maxSize || 50;
-    this.maxBufferTotal = options.maxBufferTotal || 5000;
-    this.flushInterval = options.flushInterval || 5000;
-    this._timer = setInterval(() => this.flush(), this.flushInterval);
-  }
-
-  push(analytic) {
-    while (this.buffer.length >= this.maxBufferTotal) {
-      this.buffer.shift();
-      console.warn('[WAB] Analytics buffer at cap; dropped oldest event');
-    }
-    this.buffer.push(analytic);
-    if (this.buffer.length >= this.maxSize) {
-      this.flush();
-    }
-  }
-
-  flush() {
-    if (this.buffer.length === 0) return;
-    const batch = this.buffer.splice(0);
-
-    try {
-      const insert = this.db.prepare(
-        `INSERT INTO analytics (site_id, action_name, agent_id, trigger_type, success, metadata) VALUES (?, ?, ?, ?, ?, ?)`
-      );
-      const insertMany = this.db.transaction((items) => {
-        for (const item of items) {
-          insert.run(
-            item.siteId,
-            item.actionName,
-            item.agentId || null,
-            item.triggerType || null,
-            item.success ? 1 : 0,
-            JSON.stringify(item.metadata || {})
-          );
-        }
-      });
-      insertMany(batch);
-    } catch (err) {
-      console.error('[WAB Cache] Analytics batch insert failed:', err.message);
-      while (this.buffer.length + batch.length > this.maxBufferTotal) {
-        batch.shift();
-      }
-      this.buffer.unshift(...batch);
-    }
-  }
-
-  destroy() {
-    clearInterval(this._timer);
-    this.flush();
-  }
-}
-
-const cache = new Cache(60000);
-
-module.exports = { Cache, AnalyticsQueue, cache };
+/**
+ * WAB Caching Layer — In-memory cache with TTL for hot data
+ * Reduces DB reads for license verification, config, and stats
+ */
+class Cache {
+  constructor(defaultTTL = 60000) {
+    this.store = new Map();
+    this.defaultTTL = defaultTTL;
+    this._interval = setInterval(() => this._cleanup(), 120000);
+  }
+
+  get(key) {
+    const entry = this.store.get(key);
+    if (!entry) return undefined;
+    if (Date.now() > entry.expiresAt) {
+      this.store.delete(key);
+      return undefined;
+    }
+    entry.hits++;
+    return entry.value;
+  }
+
+  set(key, value, ttl) {
+    this.store.set(key, {
+      value,
+      expiresAt: Date.now() + (ttl || this.defaultTTL),
+      hits: 0
+    });
+  }
+
+  del(key) {
+    this.store.delete(key);
+  }
+
+  invalidatePattern(pattern) {
+    for (const key of this.store.keys()) {
+      if (key.includes(pattern)) this.store.delete(key);
+    }
+  }
+
+  stats() {
+    return {
+      size: this.store.size,
+      keys: Array.from(this.store.keys())
+    };
+  }
+
+  _cleanup() {
+    const now = Date.now();
+    for (const [key, entry] of this.store) {
+      if (now > entry.expiresAt) this.store.delete(key);
+    }
+  }
+
+  destroy() {
+    clearInterval(this._interval);
+    this.store.clear();
+  }
+}
+
+/**
+ * Analytics Queue — Batches analytics inserts for better write performance
+ * Flushes every N seconds or when buffer reaches max size.
+ * maxBufferTotal caps memory if DB writes fail repeatedly (DoS mitigation).
+ */
+class AnalyticsQueue {
+  constructor(db, options = {}) {
+    this.db = db;
+    this.buffer = [];
+    this.maxSize = options.maxSize || 50;
+    this.maxBufferTotal = options.maxBufferTotal || 5000;
+    this.flushInterval = options.flushInterval || 5000;
+    this._timer = setInterval(() => this.flush(), this.flushInterval);
+  }
+
+  push(analytic) {
+    while (this.buffer.length >= this.maxBufferTotal) {
+      this.buffer.shift();
+      console.warn('[WAB] Analytics buffer at cap; dropped oldest event');
+    }
+    this.buffer.push(analytic);
+    if (this.buffer.length >= this.maxSize) {
+      this.flush();
+    }
+  }
+
+  flush() {
+    if (this.buffer.length === 0) return;
+    const batch = this.buffer.splice(0);
+
+    try {
+      const insert = this.db.prepare(
+        `INSERT INTO analytics (site_id, action_name, agent_id, trigger_type, success, metadata) VALUES (?, ?, ?, ?, ?, ?)`
+      );
+      const insertMany = this.db.transaction((items) => {
+        for (const item of items) {
+          insert.run(
+            item.siteId,
+            item.actionName,
+            item.agentId || null,
+            item.triggerType || null,
+            item.success ? 1 : 0,
+            JSON.stringify(item.metadata || {})
+          );
+        }
+      });
+      insertMany(batch);
+    } catch (err) {
+      console.error('[WAB Cache] Analytics batch insert failed:', err.message);
+      while (this.buffer.length + batch.length > this.maxBufferTotal) {
+        batch.shift();
+      }
+      this.buffer.unshift(...batch);
+    }
+  }
+
+  destroy() {
+    clearInterval(this._timer);
+    this.flush();
+  }
+}
+
+const cache = new Cache(60000);
+
+module.exports = { Cache, AnalyticsQueue, cache };

package/server/utils/migrate.js

@@ -1,81 +1,81 @@
-/**
- * Database Migration Runner
- * Tracks and applies SQL migrations from server/migrations/ in order.
- * Uses a `_migrations` table to record applied migrations.
- */
-const Database = require('better-sqlite3');
-const path = require('path');
-const fs = require('fs');
-
-const DATA_DIR = process.env.NODE_ENV === 'test'
-  ? path.join(__dirname, '..', '..', 'data-test')
-  : path.join(__dirname, '..', '..', 'data');
-if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });
-
-const dbFile = process.env.NODE_ENV === 'test' ? 'wab-test.db' : 'wab.db';
-const db = new Database(path.join(DATA_DIR, dbFile));
-
-// Ensure migrations tracking table exists
-db.exec(`
-  CREATE TABLE IF NOT EXISTS _migrations (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    name TEXT UNIQUE NOT NULL,
-    applied_at TEXT DEFAULT (datetime('now'))
-  );
-`);
-
-const MIGRATIONS_DIR = path.join(__dirname, '..', 'migrations');
-
-function getAppliedMigrations() {
-  return new Set(
-    db.prepare('SELECT name FROM _migrations ORDER BY id').all().map(r => r.name)
-  );
-}
-
-function runMigrations() {
-  if (!fs.existsSync(MIGRATIONS_DIR)) {
-    console.log('No migrations directory found.');
-    return;
-  }
-
-  const files = fs.readdirSync(MIGRATIONS_DIR)
-    .filter(f => f.endsWith('.sql'))
-    .sort();
-
-  const applied = getAppliedMigrations();
-  let count = 0;
-
-  const applyMigration = db.transaction((name, sql) => {
-    db.exec(sql);
-    db.prepare('INSERT INTO _migrations (name) VALUES (?)').run(name);
-  });
-
-  for (const file of files) {
-    if (applied.has(file)) continue;
-
-    const sql = fs.readFileSync(path.join(MIGRATIONS_DIR, file), 'utf8');
-    try {
-      applyMigration(file, sql);
-      console.log(`  ✅ Migration applied: ${file}`);
-      count++;
-    } catch (err) {
-      console.error(`  ❌ Migration failed: ${file} — ${err.message}`);
-      process.exit(1);
-    }
-  }
-
-  if (count === 0) {
-    console.log('  All migrations up to date.');
-  } else {
-    console.log(`  ${count} migration(s) applied.`);
-  }
-}
-
-// Run when called directly: node server/utils/migrate.js
-if (require.main === module) {
-  console.log('Running database migrations...');
-  runMigrations();
-  db.close();
-}
-
-module.exports = { runMigrations };
+/**
+ * Database Migration Runner
+ * Tracks and applies SQL migrations from server/migrations/ in order.
+ * Uses a `_migrations` table to record applied migrations.
+ */
+const Database = require('better-sqlite3');
+const path = require('path');
+const fs = require('fs');
+
+const DATA_DIR = process.env.NODE_ENV === 'test'
+  ? path.join(__dirname, '..', '..', 'data-test')
+  : (process.env.DATA_DIR || path.join(__dirname, '..', '..', 'data'));
+if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });
+
+const dbFile = process.env.NODE_ENV === 'test' ? 'wab-test.db' : 'wab.db';
+const db = new Database(path.join(DATA_DIR, dbFile));
+
+// Ensure migrations tracking table exists
+db.exec(`
+  CREATE TABLE IF NOT EXISTS _migrations (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT UNIQUE NOT NULL,
+    applied_at TEXT DEFAULT (datetime('now'))
+  );
+`);
+
+const MIGRATIONS_DIR = path.join(__dirname, '..', 'migrations');
+
+function getAppliedMigrations() {
+  return new Set(
+    db.prepare('SELECT name FROM _migrations ORDER BY id').all().map(r => r.name)
+  );
+}
+
+function runMigrations() {
+  if (!fs.existsSync(MIGRATIONS_DIR)) {
+    console.log('No migrations directory found.');
+    return;
+  }
+
+  const files = fs.readdirSync(MIGRATIONS_DIR)
+    .filter(f => f.endsWith('.sql'))
+    .sort();
+
+  const applied = getAppliedMigrations();
+  let count = 0;
+
+  const applyMigration = db.transaction((name, sql) => {
+    db.exec(sql);
+    db.prepare('INSERT INTO _migrations (name) VALUES (?)').run(name);
+  });
+
+  for (const file of files) {
+    if (applied.has(file)) continue;
+
+    const sql = fs.readFileSync(path.join(MIGRATIONS_DIR, file), 'utf8');
+    try {
+      applyMigration(file, sql);
+      console.log(`  ✅ Migration applied: ${file}`);
+      count++;
+    } catch (err) {
+      console.error(`  ❌ Migration failed: ${file} — ${err.message}`);
+      process.exit(1);
+    }
+  }
+
+  if (count === 0) {
+    console.log('  All migrations up to date.');
+  } else {
+    console.log(`  ${count} migration(s) applied.`);
+  }
+}
+
+// Run when called directly: node server/utils/migrate.js
+if (require.main === module) {
+  console.log('Running database migrations...');
+  runMigrations();
+  db.close();
+}
+
+module.exports = { runMigrations };

package/server/utils/safe-fetch.js

@@ -0,0 +1,228 @@
+'use strict';
+
+/**
+ * Safe Fetch — SSRF-resistant HTTP client.
+ *
+ * Mitigations applied:
+ *   1. Scheme allow-list (http/https only).
+ *   2. Optional domain allow-list (string globs or "*").
+ *   3. DNS resolution + private/reserved/loopback/link-local CIDR block.
+ *   4. Re-validation on EVERY redirect hop (manual redirect handling).
+ *   5. Hard timeout via AbortController.
+ *   6. Max response body size (default 5 MB) — drains and aborts.
+ *   7. Optional Content-Type allow-list.
+ *
+ * NEVER call native `fetch(url)` directly with user-supplied URLs anywhere
+ * inside this server process. Use this helper.
+ */
+
+const dns = require('node:dns').promises;
+const net = require('node:net');
+
+const PRIVATE_V4_CIDRS = [
+  ['10.0.0.0', 8],
+  ['172.16.0.0', 12],
+  ['192.168.0.0', 16],
+  ['127.0.0.0', 8],
+  ['169.254.0.0', 16],   // link-local (AWS metadata 169.254.169.254 lives here)
+  ['100.64.0.0', 10],    // CGNAT
+  ['0.0.0.0', 8],
+  ['224.0.0.0', 4],      // multicast
+  ['240.0.0.0', 4],      // reserved
+  ['198.18.0.0', 15],    // benchmarking
+  ['192.0.0.0', 24],
+  ['192.0.2.0', 24],     // TEST-NET-1
+  ['198.51.100.0', 24],  // TEST-NET-2
+  ['203.0.113.0', 24],   // TEST-NET-3
+];
+
+const PRIVATE_V6_PREFIXES = ['::1', 'fc', 'fd', 'fe80', 'ff', '::ffff:127.', '::ffff:10.', '::ffff:192.168.', '::ffff:172.', '::', '64:ff9b::'];
+
+function _ipToInt(ip) {
+  return ip.split('.').reduce((acc, oct) => (acc << 8) + parseInt(oct, 10), 0) >>> 0;
+}
+
+function _isPrivateV4(ip) {
+  const ipInt = _ipToInt(ip);
+  return PRIVATE_V4_CIDRS.some(([base, bits]) => {
+    const baseInt = _ipToInt(base);
+    const mask = bits === 0 ? 0 : (~0 << (32 - bits)) >>> 0;
+    return (ipInt & mask) === (baseInt & mask);
+  });
+}
+
+function _isPrivateV6(ip) {
+  const lower = ip.toLowerCase();
+  return PRIVATE_V6_PREFIXES.some((p) => lower === p || lower.startsWith(p));
+}
+
+function isPrivateAddress(ip) {
+  if (!ip) return true;
+  if (net.isIPv4(ip)) return _isPrivateV4(ip);
+  if (net.isIPv6(ip)) return _isPrivateV6(ip);
+  return true; // unknown → treat as private
+}
+
+function _matchesGlob(host, pattern) {
+  if (pattern === '*') return true;
+  if (pattern.startsWith('*.')) {
+    const suffix = pattern.slice(2).toLowerCase();
+    return host === suffix || host.endsWith('.' + suffix);
+  }
+  return host.toLowerCase() === pattern.toLowerCase();
+}
+
+function _allowedHost(host, allowList) {
+  if (!allowList || allowList.length === 0) return true; // no list = allow public
+  return allowList.some((p) => _matchesGlob(host, p));
+}
+
+async function _assertSafeHost(hostname) {
+  // If it's already an IP literal, validate directly.
+  if (net.isIP(hostname)) {
+    if (isPrivateAddress(hostname)) {
+      throw new Error(`SSRF blocked: private/reserved IP ${hostname}`);
+    }
+    return;
+  }
+  let records;
+  try {
+    records = await dns.lookup(hostname, { all: true, verbatim: true });
+  } catch (err) {
+    throw new Error(`DNS resolution failed for ${hostname}: ${err.code || err.message}`);
+  }
+  if (!records || records.length === 0) {
+    throw new Error(`DNS returned no records for ${hostname}`);
+  }
+  for (const r of records) {
+    if (isPrivateAddress(r.address)) {
+      throw new Error(`SSRF blocked: ${hostname} resolves to private/reserved address ${r.address}`);
+    }
+  }
+}
+
+/**
+ * Validate a URL string against the SSRF policy.
+ * @returns {URL} parsed URL
+ */
+async function validateUrl(rawUrl, options = {}) {
+  let parsed;
+  try {
+    parsed = new URL(rawUrl);
+  } catch (e) {
+    throw new Error('Invalid URL');
+  }
+  const allowedSchemes = options.allowedSchemes || ['http:', 'https:'];
+  if (!allowedSchemes.includes(parsed.protocol)) {
+    throw new Error(`Scheme ${parsed.protocol} not allowed`);
+  }
+  if (options.requireHttps && parsed.protocol !== 'https:') {
+    throw new Error('HTTPS required');
+  }
+  // Block credentials in URLs (defeats some auth-smuggling attacks).
+  if (parsed.username || parsed.password) {
+    throw new Error('Credentials in URLs are not allowed');
+  }
+  // Restrict ports to defaults unless explicitly allowed.
+  const allowedPorts = options.allowedPorts || [80, 443];
+  const port = parsed.port ? parseInt(parsed.port, 10) : (parsed.protocol === 'https:' ? 443 : 80);
+  if (!allowedPorts.includes(port)) {
+    throw new Error(`Port ${port} not allowed`);
+  }
+  if (!_allowedHost(parsed.hostname, options.allowList)) {
+    throw new Error(`Host ${parsed.hostname} not in allow-list`);
+  }
+  await _assertSafeHost(parsed.hostname);
+  return parsed;
+}
+
+/**
+ * SSRF-resistant fetch.
+ * Manually follows redirects so each hop is re-validated.
+ */
+async function safeFetch(rawUrl, init = {}, opts = {}) {
+  const maxRedirects = opts.maxRedirects ?? 3;
+  const maxBytes = opts.maxBytes ?? 5 * 1024 * 1024; // 5 MB
+  const timeoutMs = opts.timeoutMs ?? 10000;
+  const allowList = opts.allowList;
+  const allowedContentTypes = opts.allowedContentTypes; // e.g. ['text/html','application/json']
+
+  let currentUrl = rawUrl;
+  for (let hop = 0; hop <= maxRedirects; hop++) {
+    await validateUrl(currentUrl, { allowList, allowedSchemes: opts.allowedSchemes, requireHttps: opts.requireHttps, allowedPorts: opts.allowedPorts });
+
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+    let res;
+    try {
+      res = await fetch(currentUrl, {
+        ...init,
+        redirect: 'manual',
+        signal: controller.signal,
+      });
+    } finally {
+      clearTimeout(timer);
+    }
+
+    if (res.status >= 300 && res.status < 400) {
+      const loc = res.headers.get('location');
+      if (!loc) return _enforceBody(res, maxBytes, allowedContentTypes);
+      if (hop === maxRedirects) throw new Error('Too many redirects');
+      currentUrl = new URL(loc, currentUrl).toString();
+      continue;
+    }
+
+    return _enforceBody(res, maxBytes, allowedContentTypes);
+  }
+  throw new Error('Redirect loop');
+}
+
+async function _enforceBody(res, maxBytes, allowedContentTypes) {
+  const ct = (res.headers.get('content-type') || '').toLowerCase();
+  if (allowedContentTypes && !allowedContentTypes.some((t) => ct.startsWith(t))) {
+    res.body?.cancel?.().catch(() => {});
+    throw new Error(`Content-Type ${ct || 'unknown'} not allowed`);
+  }
+  const declared = parseInt(res.headers.get('content-length') || '0', 10);
+  if (declared && declared > maxBytes) {
+    res.body?.cancel?.().catch(() => {});
+    throw new Error(`Response too large (${declared} bytes, max ${maxBytes})`);
+  }
+  if (!res.body) return res;
+
+  // Buffer the response while enforcing size.
+  const reader = res.body.getReader();
+  const chunks = [];
+  let total = 0;
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    total += value.byteLength;
+    if (total > maxBytes) {
+      reader.cancel().catch(() => {});
+      throw new Error(`Response exceeded ${maxBytes} bytes`);
+    }
+    chunks.push(value);
+  }
+  const buf = Buffer.concat(chunks.map((c) => Buffer.from(c)));
+  // Return a Response-like wrapper matching the parts callers actually use.
+  return {
+    ok: res.ok,
+    status: res.status,
+    statusText: res.statusText,
+    headers: res.headers,
+    url: res.url,
+    redirected: res.redirected,
+    async text() { return buf.toString('utf8'); },
+    async json() { return JSON.parse(buf.toString('utf8')); },
+    async arrayBuffer() { return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); },
+    async buffer() { return buf; },
+  };
+}
+
+module.exports = {
+  safeFetch,
+  validateUrl,
+  isPrivateAddress,
+};