web-agent-bridge 3.4.0 → 3.9.0

package/server/security/url-policy.js

@@ -1,139 +1,139 @@
-'use strict';
-
-/**
- * URL Policy — guards public endpoints (e.g. /api/universal/extract) that
- * accept arbitrary user URLs. Layered on top of the SSRF guard in
- * server/utils/safe-fetch.js, this module enforces:
- *
- *   1. Scheme allow-list (https only by default).
- *   2. TLD/host denylist (configurable via WAB_URL_DENY_HOSTS / DEFAULT_DENY).
- *   3. Path denylist for obvious admin/credential/wp-login style targets that
- *      would suggest abuse.
- *   4. Per-actor (IP / API-key / siteId) rate-limit independent of express
- *      router-level rate limiting.
- *
- * Decisions are recorded in `url_policy_audit` for review.
- */
-
-const crypto = require('crypto');
-const { db } = require('../models/db');
-
-const DEFAULT_DENY_HOSTS = [
-  // Local/private/metadata is already blocked by safe-fetch; these are
-  // additional public hosts that have no legitimate scraping use case.
-  'login.microsoftonline.com',
-  'accounts.google.com',
-  'appleid.apple.com',
-];
-
-const DEFAULT_DENY_PATH_RE = /\/(?:wp-(?:login|admin)|administrator|phpmyadmin|\.git|\.env)(?:\/|\.|$|\?)/i;
-
-const RATE_WINDOW_MS = 60_000;
-const RATE_MAX = parseInt(process.env.WAB_URL_POLICY_RATE_MAX || '30', 10);
-
-function _envHosts() {
-  return String(process.env.WAB_URL_DENY_HOSTS || '')
-    .split(',').map((s) => s.trim().toLowerCase()).filter(Boolean);
-}
-
-db.exec(`
-  CREATE TABLE IF NOT EXISTS url_policy_audit (
-    id TEXT PRIMARY KEY,
-    actor TEXT,
-    url TEXT,
-    decision TEXT NOT NULL CHECK(decision IN ('allowed','blocked','rate_limited')),
-    reason TEXT,
-    created_at TEXT DEFAULT (datetime('now'))
-  );
-  CREATE INDEX IF NOT EXISTS idx_urlpolicy_decision ON url_policy_audit(decision);
-`);
-
-const _rate = new Map(); // actor → [ts]
-
-function _hit(actor) {
-  const now = Date.now();
-  const arr = (_rate.get(actor) || []).filter((t) => now - t < RATE_WINDOW_MS);
-  arr.push(now);
-  _rate.set(actor, arr);
-  return arr.length;
-}
-
-function _audit(actor, url, decision, reason) {
-  try {
-    db.prepare(`INSERT INTO url_policy_audit (id, actor, url, decision, reason)
-                VALUES (?, ?, ?, ?, ?)`).run(
-      crypto.randomUUID(), actor || null, url || null, decision, reason || null);
-  } catch (_) { /* never block on audit failure */ }
-}
-
-/**
- * @param {string} rawUrl
- * @param {object} opts
- * @param {string} [opts.actor] - IP, API key id, or site id
- * @returns {{ ok:boolean, reason?:string, code?:string, parsed?:URL }}
- */
-function check(rawUrl, opts = {}) {
-  const actor = opts.actor || 'anon';
-
-  if (typeof rawUrl !== 'string' || rawUrl.length === 0) {
-    _audit(actor, String(rawUrl).slice(0, 200), 'blocked', 'missing_url');
-    return { ok: false, reason: 'URL is required', code: 'MISSING_URL' };
-  }
-  if (rawUrl.length > 2048) {
-    _audit(actor, rawUrl.slice(0, 200), 'blocked', 'url_too_long');
-    return { ok: false, reason: 'URL exceeds 2048 characters', code: 'URL_TOO_LONG' };
-  }
-
-  let parsed;
-  try { parsed = new URL(rawUrl); }
-  catch {
-    _audit(actor, rawUrl.slice(0, 200), 'blocked', 'invalid_url');
-    return { ok: false, reason: 'Invalid URL', code: 'INVALID_URL' };
-  }
-
-  if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
-    _audit(actor, rawUrl, 'blocked', `scheme:${parsed.protocol}`);
-    return { ok: false, reason: `Scheme ${parsed.protocol} not allowed`, code: 'BAD_SCHEME' };
-  }
-  if (process.env.WAB_URL_POLICY_HTTPS_ONLY === '1' && parsed.protocol !== 'https:') {
-    _audit(actor, rawUrl, 'blocked', 'http_disallowed');
-    return { ok: false, reason: 'HTTPS required', code: 'HTTPS_REQUIRED' };
-  }
-
-  const host = parsed.hostname.toLowerCase();
-  const deny = new Set([...DEFAULT_DENY_HOSTS, ..._envHosts()]);
-  if (deny.has(host)) {
-    _audit(actor, rawUrl, 'blocked', `host_denied:${host}`);
-    return { ok: false, reason: `Host ${host} is denied by policy`, code: 'HOST_DENIED' };
-  }
-
-  if (DEFAULT_DENY_PATH_RE.test(parsed.pathname)) {
-    _audit(actor, rawUrl, 'blocked', `path_denied:${parsed.pathname}`);
-    return { ok: false, reason: 'Path matches abuse pattern', code: 'PATH_DENIED' };
-  }
-
-  const count = _hit(actor);
-  if (count > RATE_MAX) {
-    _audit(actor, rawUrl, 'rate_limited', `count:${count}`);
-    return { ok: false, reason: `Rate limit exceeded (${RATE_MAX} URLs/min per actor)`, code: 'RATE_LIMITED' };
-  }
-
-  _audit(actor, rawUrl, 'allowed', null);
-  return { ok: true, parsed };
-}
-
-function getRecentAudits(limit = 100, decision) {
-  if (decision) {
-    return db.prepare(`SELECT * FROM url_policy_audit WHERE decision = ? ORDER BY rowid DESC LIMIT ?`).all(decision, limit);
-  }
-  return db.prepare(`SELECT * FROM url_policy_audit ORDER BY rowid DESC LIMIT ?`).all(limit);
-}
-
-function actorFromReq(req) {
-  return (req.wabAuth && req.wabAuth.key_id) ||
-         (req.user && req.user.id) ||
-         req.ip ||
-         'anon';
-}
-
-module.exports = { check, getRecentAudits, actorFromReq, RATE_MAX };
+'use strict';
+
+/**
+ * URL Policy — guards public endpoints (e.g. /api/universal/extract) that
+ * accept arbitrary user URLs. Layered on top of the SSRF guard in
+ * server/utils/safe-fetch.js, this module enforces:
+ *
+ *   1. Scheme allow-list (https only by default).
+ *   2. TLD/host denylist (configurable via WAB_URL_DENY_HOSTS / DEFAULT_DENY).
+ *   3. Path denylist for obvious admin/credential/wp-login style targets that
+ *      would suggest abuse.
+ *   4. Per-actor (IP / API-key / siteId) rate-limit independent of express
+ *      router-level rate limiting.
+ *
+ * Decisions are recorded in `url_policy_audit` for review.
+ */
+
+const crypto = require('crypto');
+const { db } = require('../models/db');
+
+const DEFAULT_DENY_HOSTS = [
+  // Local/private/metadata is already blocked by safe-fetch; these are
+  // additional public hosts that have no legitimate scraping use case.
+  'login.microsoftonline.com',
+  'accounts.google.com',
+  'appleid.apple.com',
+];
+
+const DEFAULT_DENY_PATH_RE = /\/(?:wp-(?:login|admin)|administrator|phpmyadmin|\.git|\.env)(?:\/|\.|$|\?)/i;
+
+const RATE_WINDOW_MS = 60_000;
+const RATE_MAX = parseInt(process.env.WAB_URL_POLICY_RATE_MAX || '30', 10);
+
+function _envHosts() {
+  return String(process.env.WAB_URL_DENY_HOSTS || '')
+    .split(',').map((s) => s.trim().toLowerCase()).filter(Boolean);
+}
+
+db.exec(`
+  CREATE TABLE IF NOT EXISTS url_policy_audit (
+    id TEXT PRIMARY KEY,
+    actor TEXT,
+    url TEXT,
+    decision TEXT NOT NULL CHECK(decision IN ('allowed','blocked','rate_limited')),
+    reason TEXT,
+    created_at TEXT DEFAULT (datetime('now'))
+  );
+  CREATE INDEX IF NOT EXISTS idx_urlpolicy_decision ON url_policy_audit(decision);
+`);
+
+const _rate = new Map(); // actor → [ts]
+
+function _hit(actor) {
+  const now = Date.now();
+  const arr = (_rate.get(actor) || []).filter((t) => now - t < RATE_WINDOW_MS);
+  arr.push(now);
+  _rate.set(actor, arr);
+  return arr.length;
+}
+
+function _audit(actor, url, decision, reason) {
+  try {
+    db.prepare(`INSERT INTO url_policy_audit (id, actor, url, decision, reason)
+                VALUES (?, ?, ?, ?, ?)`).run(
+      crypto.randomUUID(), actor || null, url || null, decision, reason || null);
+  } catch (_) { /* never block on audit failure */ }
+}
+
+/**
+ * @param {string} rawUrl
+ * @param {object} opts
+ * @param {string} [opts.actor] - IP, API key id, or site id
+ * @returns {{ ok:boolean, reason?:string, code?:string, parsed?:URL }}
+ */
+function check(rawUrl, opts = {}) {
+  const actor = opts.actor || 'anon';
+
+  if (typeof rawUrl !== 'string' || rawUrl.length === 0) {
+    _audit(actor, String(rawUrl).slice(0, 200), 'blocked', 'missing_url');
+    return { ok: false, reason: 'URL is required', code: 'MISSING_URL' };
+  }
+  if (rawUrl.length > 2048) {
+    _audit(actor, rawUrl.slice(0, 200), 'blocked', 'url_too_long');
+    return { ok: false, reason: 'URL exceeds 2048 characters', code: 'URL_TOO_LONG' };
+  }
+
+  let parsed;
+  try { parsed = new URL(rawUrl); }
+  catch {
+    _audit(actor, rawUrl.slice(0, 200), 'blocked', 'invalid_url');
+    return { ok: false, reason: 'Invalid URL', code: 'INVALID_URL' };
+  }
+
+  if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
+    _audit(actor, rawUrl, 'blocked', `scheme:${parsed.protocol}`);
+    return { ok: false, reason: `Scheme ${parsed.protocol} not allowed`, code: 'BAD_SCHEME' };
+  }
+  if (process.env.WAB_URL_POLICY_HTTPS_ONLY === '1' && parsed.protocol !== 'https:') {
+    _audit(actor, rawUrl, 'blocked', 'http_disallowed');
+    return { ok: false, reason: 'HTTPS required', code: 'HTTPS_REQUIRED' };
+  }
+
+  const host = parsed.hostname.toLowerCase();
+  const deny = new Set([...DEFAULT_DENY_HOSTS, ..._envHosts()]);
+  if (deny.has(host)) {
+    _audit(actor, rawUrl, 'blocked', `host_denied:${host}`);
+    return { ok: false, reason: `Host ${host} is denied by policy`, code: 'HOST_DENIED' };
+  }
+
+  if (DEFAULT_DENY_PATH_RE.test(parsed.pathname)) {
+    _audit(actor, rawUrl, 'blocked', `path_denied:${parsed.pathname}`);
+    return { ok: false, reason: 'Path matches abuse pattern', code: 'PATH_DENIED' };
+  }
+
+  const count = _hit(actor);
+  if (count > RATE_MAX) {
+    _audit(actor, rawUrl, 'rate_limited', `count:${count}`);
+    return { ok: false, reason: `Rate limit exceeded (${RATE_MAX} URLs/min per actor)`, code: 'RATE_LIMITED' };
+  }
+
+  _audit(actor, rawUrl, 'allowed', null);
+  return { ok: true, parsed };
+}
+
+function getRecentAudits(limit = 100, decision) {
+  if (decision) {
+    return db.prepare(`SELECT * FROM url_policy_audit WHERE decision = ? ORDER BY rowid DESC LIMIT ?`).all(decision, limit);
+  }
+  return db.prepare(`SELECT * FROM url_policy_audit ORDER BY rowid DESC LIMIT ?`).all(limit);
+}
+
+function actorFromReq(req) {
+  return (req.wabAuth && req.wabAuth.key_id) ||
+         (req.user && req.user.id) ||
+         req.ip ||
+         'anon';
+}
+
+module.exports = { check, getRecentAudits, actorFromReq, RATE_MAX };

package/server/services/adoption-agent.js

@@ -0,0 +1,182 @@
+/**
+ * Adoption Agent — turns a bare URL into a ready-to-publish wab.json + DNS TXT.
+ *
+ * Heuristic-driven (no LLM required). Steps:
+ *   1) Run sdk/auto-discovery.discover() to extract metadata.
+ *   2) Inspect TLS fingerprint (best effort).
+ *   3) Build a draft wab.json (unsigned) from observed signals.
+ *   4) Build a draft DNS TXT record.
+ *   5) Provide deploy snippets for Cloudflare Worker, Vercel, Netlify, Next.js.
+ *
+ * Pure server-side, used by /api/adopt and bin/wab-init.js (--auto-from).
+ */
+
+'use strict';
+
+const { discover } = require('../../sdk/auto-discovery');
+const tls = require('node:tls');
+
+function _tlsFingerprint(host, timeoutMs = 6000) {
+  return new Promise((resolve) => {
+    const sock = tls.connect({ host, port: 443, servername: host, rejectUnauthorized: false }, () => {
+      const cert = sock.getPeerCertificate(false);
+      sock.end();
+      const fp = (cert && cert.fingerprint256 || '').replace(/:/g, '').toLowerCase();
+      resolve({ fp: fp || null, valid_to: cert && cert.valid_to });
+    });
+    sock.on('error', () => resolve({ fp: null }));
+    sock.setTimeout(timeoutMs, () => { sock.destroy(); resolve({ fp: null }); });
+  });
+}
+
+function _detectStackFromEnv(env) {
+  const out = { type: 'static', signals: [] };
+  if (env.meta && env.meta.og && env.meta.og.site_name) out.signals.push('opengraph');
+  if (env.products && env.products.length) out.signals.push('schema.org/Product');
+  if (env.sitemap && env.sitemap.length) out.signals.push('sitemap.xml');
+
+  // Server header probing is too slow; rely on URL/sitemap heuristics.
+  const allUrls = (env.sitemap || []).join(' ');
+  if (/\?p=\d+|wp-content|wp-json/i.test(allUrls)) out.type = 'wordpress';
+  else if (/\/_next\/|\/__next/.test(allUrls)) out.type = 'nextjs';
+  else if (env.products && env.products.length > 5) out.type = 'ecommerce';
+  return out;
+}
+
+function _suggestActions(env, baseUrl) {
+  const out = [
+    { name: 'home', description: 'Open homepage', url: baseUrl }
+  ];
+  if (env.actions && Array.isArray(env.actions)) {
+    for (const a of env.actions) {
+      if (out.find((x) => x.name === a.name)) continue;
+      out.push(a);
+    }
+  }
+  if (env.products && env.products.length) {
+    out.push({ name: 'browseProducts', description: `${env.products.length} schema.org products discovered`, source: 'schema.org' });
+  }
+  if (env.sitemap && env.sitemap.length) {
+    out.push({ name: 'browseSitemap', description: `${env.sitemap.length} URLs from sitemap.xml`, url: `${baseUrl}/sitemap.xml` });
+  }
+  if (env.meta && env.meta.og && env.meta.og.url) {
+    out.push({ name: 'getOpenGraph', description: 'OpenGraph metadata available', source: 'opengraph' });
+  }
+  return out.slice(0, 12);
+}
+
+function _dnsTxt(host, baseUrl, fingerprint) {
+  let v = `v=wab1; endpoint=${baseUrl}/.well-known/wab.json`;
+  if (fingerprint) v += `; ssl_thumbprint=${fingerprint}`;
+  return { name: `_wab.${host}`, type: 'TXT', value: v };
+}
+
+function _deploySnippets(host, baseUrl, doc) {
+  const docInline = JSON.stringify(doc, null, 2);
+  return {
+    static: {
+      title: 'Static / Apache / nginx',
+      instructions: `Save the wab.json below to:\n  <docroot>/.well-known/wab.json\n\nMake sure it is publicly reachable at:\n  ${baseUrl}/.well-known/wab.json`
+    },
+    cloudflare_worker: {
+      title: '@webagentbridge/cloudflare-worker',
+      install: 'npm i -g wrangler && npm i @webagentbridge/cloudflare-worker',
+      env: {
+        WAB_SITE_NAME: doc.site,
+        WAB_SITE_URL: baseUrl,
+        WAB_ACTIONS_JSON: JSON.stringify(doc.actions || [])
+      },
+      command: 'wrangler deploy'
+    },
+    vercel: {
+      title: '@webagentbridge/edge (Vercel Middleware)',
+      install: 'npm i @webagentbridge/edge',
+      file: 'middleware.ts',
+      content: `import { handleRequest } from '@webagentbridge/edge';
+export const config = { matcher: ['/.well-known/wab.json', '/.well-known/wab-discovery'] };
+export default (req) => handleRequest(req, ${JSON.stringify({ siteName: doc.site, siteUrl: baseUrl, actions: doc.actions || [] }, null, 2)});`
+    },
+    netlify: {
+      title: '@webagentbridge/edge (Netlify Edge Function)',
+      install: 'npm i @webagentbridge/edge',
+      file: 'netlify/edge-functions/wab.js',
+      toml: `[[edge_functions]]\nfunction = "wab"\npath = "/.well-known/wab.json"\n\n[[edge_functions]]\nfunction = "wab"\npath = "/.well-known/wab-discovery"`,
+      content: `import { handleRequest } from '@webagentbridge/edge';
+export default (request, ctx) => handleRequest(request, {
+  siteName: ${JSON.stringify(doc.site)},
+  siteUrl: ${JSON.stringify(baseUrl)},
+  actions: ${JSON.stringify(doc.actions || [])}
+});`
+    },
+    nextjs: {
+      title: '@webagentbridge/next',
+      install: 'npm i @webagentbridge/next',
+      file: 'next.config.js',
+      content: `const { withWAB } = require('@webagentbridge/next');
+module.exports = withWAB({}, {
+  siteName: ${JSON.stringify(doc.site)},
+  siteUrl: ${JSON.stringify(baseUrl)},
+  actions: ${JSON.stringify(doc.actions || [], null, 2)},
+});`
+    },
+    inline_wab_json: docInline
+  };
+}
+
+/**
+ * Suggest a complete adoption package for a URL.
+ *
+ * @param {string} siteUrl
+ * @param {object} [opts]
+ * @param {boolean} [opts.includeTls=true]
+ * @returns {Promise<{ok:boolean, host:string, base_url:string, stack:object, wab_json:object, dns_txt:object, ssl?:object, deploy:object, env:object}>}
+ */
+async function suggest(siteUrl, opts = {}) {
+  if (!siteUrl) return { ok: false, error: 'missing url' };
+  if (!/^https?:\/\//i.test(siteUrl)) siteUrl = `https://${siteUrl}`;
+  let host, baseUrl;
+  try {
+    const u = new URL(siteUrl);
+    host = u.hostname;
+    baseUrl = `${u.protocol}//${u.hostname}`;
+  } catch {
+    return { ok: false, error: 'invalid url' };
+  }
+
+  const env = await discover(baseUrl, { timeoutMs: opts.timeoutMs || 8000 });
+  const stack = _detectStackFromEnv(env);
+  const ssl = opts.includeTls === false ? null : await _tlsFingerprint(host);
+
+  const doc = {
+    version: '1.0',
+    site: (env.site && env.site.name) || host,
+    description: (env.site && env.site.description) || `${host} — generated by Adoption Agent`,
+    url: baseUrl,
+    project_type: stack.type,
+    detected_signals: stack.signals,
+    generated_at: new Date().toISOString(),
+    generator: 'wab-adoption-agent',
+    actions: _suggestActions(env, baseUrl),
+    trust: { signed: false, note: 'Run scripts/sign-wab-domain.js to add an Ed25519 signature.' }
+  };
+
+  return {
+    ok: true,
+    host,
+    base_url: baseUrl,
+    stack,
+    ssl: ssl && ssl.fp ? { fingerprint_sha256: ssl.fp, valid_to: ssl.valid_to } : null,
+    wab_json: doc,
+    dns_txt: _dnsTxt(host, baseUrl, ssl && ssl.fp),
+    deploy: _deploySnippets(host, baseUrl, doc),
+    env_summary: {
+      source: env.source,
+      action_count: doc.actions.length,
+      sitemap_count: env.sitemap ? env.sitemap.length : 0,
+      product_count: env.products ? env.products.length : 0,
+      has_signed_wab: env.source === 'wab.json'
+    }
+  };
+}
+
+module.exports = { suggest };