web-agent-bridge 3.9.1 → 3.9.2

package/package.json

@@ -1,93 +1,93 @@
-{
-  "name": "web-agent-bridge",
-  "version": "3.9.1",
-  "description": "Agent Transaction Bridge — the trust + transaction layer for agentic commerce. Signed intent contracts, idempotent transactions, Ed25519-verifiable receipts, explicit compensation. Plus the original WAB stack: sovereign browser, ShieldQR, SSL health, DNS discovery, agent mesh, and unified gateway for safe AI–website interaction.",
-  "author": "Web Agent Bridge <dev@webagentbridge.com>",
-  "main": "server/index.js",
-  "bin": {
-    "web-agent-bridge": "./bin/cli.js",
-    "wab": "./bin/cli.js",
-    "wab-agent": "./bin/cli.js",
-    "wab-init": "./bin/wab-init.js"
-  },
-  "scripts": {
-    "start": "node server/index.js",
-    "dev": "node server/index.js",
-    "test": "jest --forceExit --detectOpenHandles",
-    "build:script": "node scripts/build.js",
-    "prepublishOnly": "npm test"
-  },
-  "keywords": [
-    "ai",
-    "agent",
-    "bridge",
-    "protocol",
-    "platform",
-    "automation",
-    "web",
-    "ai-agent",
-    "agent-mesh",
-    "sovereign-browser",
-    "phone-shield",
-    "dns-discovery",
-    "api-gateway",
-    "browser-automation",
-    "webdriver-bidi"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/abokenan444/web-agent-bridge.git"
-  },
-  "homepage": "https://github.com/abokenan444/web-agent-bridge#readme",
-  "bugs": {
-    "url": "https://github.com/abokenan444/web-agent-bridge/issues"
-  },
-  "files": [
-    "bin/",
-    "server/",
-    "public/*.html",
-    "public/*.txt",
-    "public/*.xml",
-    "public/*.json",
-    "public/css/",
-    "public/js/",
-    "public/script/",
-    "public/assets/",
-    "public/.well-known/",
-    "script/",
-    "sdk/",
-    "templates/",
-    "examples/",
-    "README.md",
-    "README.ar.md",
-    "LICENSE"
-  ],
-  "engines": {
-    "node": ">=18.0.0"
-  },
-  "license": "MIT",
-  "dependencies": {
-    "bcryptjs": "^3.0.3",
-    "better-sqlite3": "^11.6.0",
-    "cors": "^2.8.5",
-    "dotenv": "^16.4.5",
-    "express": "^4.21.0",
-    "express-rate-limit": "^7.4.1",
-    "helmet": "^8.0.0",
-    "jsonwebtoken": "^9.0.2",
-    "nodemailer": "^8.0.7",
-    "stripe": "^20.4.1",
-    "ws": "^8.20.0"
-  },
-  "devDependencies": {
-    "all-contributors-cli": "^6.26.1",
-    "jest": "^30.3.0",
-    "supertest": "^7.2.2"
-  },
-  "jest": {
-    "testPathIgnorePatterns": [
-      "/node_modules/",
-      "/packages/"
-    ]
-  }
-}
+{
+  "name": "web-agent-bridge",
+  "version": "3.9.2",
+  "description": "Agent Transaction Bridge — the trust + transaction layer for agentic commerce. Signed intent contracts, idempotent transactions, Ed25519-verifiable receipts, explicit compensation. Plus the original WAB stack: sovereign browser, ShieldQR, SSL health, DNS discovery, agent mesh, and unified gateway for safe AI–website interaction.",
+  "author": "Web Agent Bridge <dev@webagentbridge.com>",
+  "main": "server/index.js",
+  "bin": {
+    "web-agent-bridge": "./bin/cli.js",
+    "wab": "./bin/cli.js",
+    "wab-agent": "./bin/cli.js",
+    "wab-init": "./bin/wab-init.js"
+  },
+  "scripts": {
+    "start": "node server/index.js",
+    "dev": "node server/index.js",
+    "test": "jest --forceExit --detectOpenHandles",
+    "build:script": "node scripts/build.js",
+    "prepublishOnly": "npm test"
+  },
+  "keywords": [
+    "ai",
+    "agent",
+    "bridge",
+    "protocol",
+    "platform",
+    "automation",
+    "web",
+    "ai-agent",
+    "agent-mesh",
+    "sovereign-browser",
+    "phone-shield",
+    "dns-discovery",
+    "api-gateway",
+    "browser-automation",
+    "webdriver-bidi"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/abokenan444/web-agent-bridge.git"
+  },
+  "homepage": "https://github.com/abokenan444/web-agent-bridge#readme",
+  "bugs": {
+    "url": "https://github.com/abokenan444/web-agent-bridge/issues"
+  },
+  "files": [
+    "bin/",
+    "server/",
+    "public/*.html",
+    "public/*.txt",
+    "public/*.xml",
+    "public/*.json",
+    "public/css/",
+    "public/js/",
+    "public/script/",
+    "public/assets/",
+    "public/.well-known/",
+    "script/",
+    "sdk/",
+    "templates/",
+    "examples/",
+    "README.md",
+    "README.ar.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "bcryptjs": "^3.0.3",
+    "better-sqlite3": "^11.6.0",
+    "cors": "^2.8.5",
+    "dotenv": "^16.4.5",
+    "express": "^4.21.0",
+    "express-rate-limit": "^7.4.1",
+    "helmet": "^8.0.0",
+    "jsonwebtoken": "^9.0.2",
+    "nodemailer": "^8.0.7",
+    "stripe": "^20.4.1",
+    "ws": "^8.20.0"
+  },
+  "devDependencies": {
+    "all-contributors-cli": "^6.26.1",
+    "jest": "^30.3.0",
+    "supertest": "^7.2.2"
+  },
+  "jest": {
+    "testPathIgnorePatterns": [
+      "/node_modules/",
+      "/packages/"
+    ]
+  }
+}

package/server/index.js

@@ -232,6 +232,15 @@ const licenseLimiter = rateLimit({
   }
 });
 
+// Visitor analytics — record every public page hit (HTML routes only) before
+// they're served by express.static. Skips assets, /api, /admin and other noise.
+try {
+  const visitorTracker = require('./services/visitor-tracker');
+  app.use(visitorTracker.middleware());
+} catch (e) {
+  console.warn('[wab] visitor-tracker disabled:', e.message);
+}
+
 // Whitepaper guard — must run BEFORE express.static so we can apply strict headers
 // and intercept both /whitepaper and /whitepaper.html with the same protections.
 const whitepaperHandler = (req, res) => {

package/server/migrations/021_visitor_analytics.sql

@@ -0,0 +1,31 @@
+-- ─────────────────────────────────────────────────────────────────────────────
+-- Migration 021 — Visitor analytics (page_visits)
+--
+-- Captures every public page request (registered or anonymous) so the admin
+-- panel can show real traffic data. IPs are hashed for privacy.
+-- ─────────────────────────────────────────────────────────────────────────────
+
+CREATE TABLE IF NOT EXISTS page_visits (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  path            TEXT    NOT NULL,
+  query_string    TEXT,
+  referrer        TEXT,
+  host            TEXT,
+  user_agent      TEXT,
+  ip_hash         TEXT,         -- sha256(ip + salt), first 32 chars
+  country         TEXT,         -- best-effort from Cloudflare/CF-IPCountry; nullable
+  device          TEXT,         -- desktop | mobile | tablet | bot
+  is_bot          INTEGER NOT NULL DEFAULT 0,
+  session_id      TEXT,         -- random per-visitor cookie or derived from ip_hash+UA
+  user_id         TEXT,         -- nullable; populated if request carried an auth cookie/token
+  status_code     INTEGER,      -- HTTP status that was returned
+  duration_ms     INTEGER,      -- server-side handler time
+  created_at      TEXT    NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_pv_created       ON page_visits(created_at);
+CREATE INDEX IF NOT EXISTS idx_pv_path          ON page_visits(path);
+CREATE INDEX IF NOT EXISTS idx_pv_ip            ON page_visits(ip_hash);
+CREATE INDEX IF NOT EXISTS idx_pv_session       ON page_visits(session_id);
+CREATE INDEX IF NOT EXISTS idx_pv_is_bot        ON page_visits(is_bot);
+CREATE INDEX IF NOT EXISTS idx_pv_user          ON page_visits(user_id);

package/server/routes/admin.js

@@ -23,6 +23,7 @@ const {
 } = require('../models/db');
 const { sendEmail } = require('../services/email');
 const { createCheckoutSession, createPortalSession, isStripeConfigured, getStripePrices } = require('../services/stripe');
+const visitorTracker = require('../services/visitor-tracker');
 
 // ─── Auth ──────────────────────────────────────────────────────────────
 
@@ -60,6 +61,35 @@ router.get('/me', authenticateAdmin, (req, res) => {
 router.get('/stats', authenticateAdmin, (req, res) => {
   const stats = getAdminStats();
   stats.stripeConfigured = isStripeConfigured();
+
+  // Expose normalized keys that the overview dashboard expects, while keeping
+  // the legacy keys for any older clients still reading them.
+  stats.users  = stats.totalUsers;
+  stats.sites  = stats.totalSites;
+  stats.analytics_total = stats.totalAnalytics;
+  stats.analytics_today = stats.todayAnalytics;
+  stats.revenue30d = (() => {
+    try {
+      const r = db.prepare(
+        `SELECT COALESCE(SUM(amount),0) AS t FROM payments WHERE status='succeeded' AND created_at >= datetime('now','-30 days')`
+      ).get();
+      return r ? r.t : 0;
+    } catch { return 0; }
+  })();
+  stats.active_subscriptions = (() => {
+    try {
+      const r = db.prepare(
+        `SELECT COUNT(*) AS c FROM stripe_subscriptions WHERE status='active'`
+      ).get();
+      return r ? r.c : 0;
+    } catch { return 0; }
+  })();
+
+  // Visitor counts (registered + anonymous).
+  try {
+    Object.assign(stats, visitorTracker.getQuickCounts());
+  } catch { /* table may not exist on first boot before migration */ }
+
   res.json(stats);
 });
 
@@ -69,6 +99,26 @@ router.get('/analytics', authenticateAdmin, (req, res) => {
   res.json(data);
 });
 
+// ─── Visitor analytics (page hits, registered + anonymous) ────────────
+
+router.get('/analytics/visits', authenticateAdmin, (req, res) => {
+  try {
+    const data = visitorTracker.getVisitorAnalytics(req.query.days);
+    res.json({ ok: true, ...data });
+  } catch (e) {
+    res.status(500).json({ ok: false, error: e.message });
+  }
+});
+
+router.get('/analytics/visits/recent', authenticateAdmin, (req, res) => {
+  try {
+    const visits = visitorTracker.getRecentVisits(req.query.limit);
+    res.json({ ok: true, visits });
+  } catch (e) {
+    res.status(500).json({ ok: false, error: e.message });
+  }
+});
+
 // ─── Users Management ─────────────────────────────────────────────────
 
 router.get('/users', authenticateAdmin, (req, res) => {

package/server/services/visitor-tracker.js

@@ -0,0 +1,250 @@
+/**
+ * Visitor tracking — records every public HTML page hit into `page_visits`.
+ * Anonymous-friendly: IPs are hashed (sha256 + IP_HASH_SALT) so we never store raw IPs.
+ *
+ * Exposes:
+ *   - middleware()                — Express middleware to mount before express.static
+ *   - getVisitorAnalytics(days)   — totals + timeline + breakdowns for /api/admin/analytics/visits
+ *   - getRecentVisits(limit)      — latest individual page hits for the admin live feed
+ *   - getQuickCounts()            — visits_24h / visitors_24h / visits_30d for /api/admin/stats
+ */
+
+const crypto = require('crypto');
+const { db } = require('../models/db');
+
+const IP_SALT = process.env.IP_HASH_SALT || process.env.JWT_SECRET || 'wab-visitor-salt-v1';
+
+// ── Bot detection ────────────────────────────────────────────────────
+const BOT_RE = /(bot|crawler|spider|crawl|slurp|bingpreview|facebookexternalhit|preview|monitor|uptime|curl|wget|axios|node-fetch|python-requests|java\/|ahrefs|semrush|petalbot|yandex|baiduspider|duckduckbot|googlebot|applebot|gpt|claude|anthropic|openai|perplexity)/i;
+
+// ── Path filter ──────────────────────────────────────────────────────
+// We track real page requests, not asset/API noise.
+const SKIP_PREFIX = ['/api/', '/css/', '/js/', '/assets/', '/script/', '/v3/', '/v2/', '/v1/', '/latest/', '/.well-known/', '/admin/', '/socket.io', '/favicon', '/sitemap', '/robots.txt', '/feed.xml', '/downloads/'];
+const SKIP_EXT = /\.(?:js|mjs|css|map|png|jpe?g|gif|svg|webp|ico|woff2?|ttf|otf|eot|mp4|webm|mp3|pdf|xml|txt|wasm)$/i;
+
+function shouldTrack(req) {
+  if (req.method !== 'GET' && req.method !== 'HEAD') return false;
+  const p = req.path || '/';
+  if (SKIP_EXT.test(p)) return false;
+  for (const pre of SKIP_PREFIX) if (p.startsWith(pre)) return false;
+  return true;
+}
+
+function hashIp(ip) {
+  if (!ip) return null;
+  return crypto.createHash('sha256').update(IP_SALT + ':' + ip).digest('hex').slice(0, 32);
+}
+
+function detectDevice(ua) {
+  if (!ua) return 'unknown';
+  if (BOT_RE.test(ua)) return 'bot';
+  if (/iPad|Tablet|PlayBook|Silk/i.test(ua)) return 'tablet';
+  if (/Mobi|Android|iPhone|iPod|Opera Mini|IEMobile/i.test(ua)) return 'mobile';
+  return 'desktop';
+}
+
+function extractUserId(req) {
+  // Best-effort: req.user (set by upstream auth middleware) wins; otherwise null.
+  if (req.user && req.user.id) return String(req.user.id);
+  if (req.session && req.session.userId) return String(req.session.userId);
+  return null;
+}
+
+// Lazily prepared so the require() of this module can run before migrations
+// have created the page_visits table (e.g. in tests or during cold boot).
+let _insertVisit = null;
+function getInsertStmt() {
+  if (_insertVisit) return _insertVisit;
+  _insertVisit = db.prepare(`
+    INSERT INTO page_visits
+      (path, query_string, referrer, host, user_agent, ip_hash, country, device, is_bot, session_id, user_id, status_code, duration_ms)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+  return _insertVisit;
+}
+
+function middleware() {
+  return function visitTracker(req, res, next) {
+    if (!shouldTrack(req)) return next();
+    const start = Date.now();
+    res.on('finish', () => {
+      try {
+        const ua      = req.get('user-agent') || null;
+        const ref     = req.get('referer') || req.get('referrer') || null;
+        const ipHash  = hashIp(req.ip);
+        const device  = detectDevice(ua);
+        const isBot   = device === 'bot' ? 1 : 0;
+        const country = req.get('cf-ipcountry') || req.get('x-vercel-ip-country') || null;
+        const session = ipHash && ua
+          ? crypto.createHash('sha256').update(ipHash + ':' + ua).digest('hex').slice(0, 24)
+          : null;
+        const qs = req.url.includes('?') ? req.url.slice(req.url.indexOf('?') + 1, req.url.indexOf('?') + 257) : null;
+        getInsertStmt().run(
+          req.path.slice(0, 512),
+          qs,
+          ref ? ref.slice(0, 512) : null,
+          (req.get('host') || '').slice(0, 255),
+          ua ? ua.slice(0, 512) : null,
+          ipHash,
+          country ? country.slice(0, 4) : null,
+          device,
+          isBot,
+          session,
+          extractUserId(req),
+          res.statusCode,
+          Date.now() - start
+        );
+      } catch (e) {
+        if (process.env.NODE_ENV !== 'production') {
+          console.warn('[visit-tracker] insert failed:', e.message);
+        }
+      }
+    });
+    next();
+  };
+}
+
+// ── Read queries ────────────────────────────────────────────────────
+function getVisitorAnalytics(days) {
+  const n = Math.max(1, Math.min(365, parseInt(days, 10) || 30));
+  const since = new Date(Date.now() - n * 86400000).toISOString();
+
+  const totalsRow = db.prepare(`
+    SELECT
+      COUNT(*) AS pageviews,
+      COUNT(DISTINCT session_id) AS visitors,
+      COALESCE(SUM(CASE WHEN is_bot=1 THEN 1 ELSE 0 END), 0) AS bot_hits,
+      COALESCE(SUM(CASE WHEN user_id IS NOT NULL THEN 1 ELSE 0 END), 0) AS authenticated_hits,
+      COUNT(DISTINCT user_id) AS authenticated_users
+    FROM page_visits WHERE created_at >= ?
+  `).get(since);
+
+  const last24Row = db.prepare(`
+    SELECT COUNT(*) AS pageviews, COUNT(DISTINCT session_id) AS visitors
+    FROM page_visits WHERE created_at >= datetime('now','-1 day')
+  `).get();
+
+  const todayRow = db.prepare(`
+    SELECT COUNT(*) AS pageviews, COUNT(DISTINCT session_id) AS visitors
+    FROM page_visits WHERE date(created_at) = date('now')
+  `).get();
+
+  const timeline = db.prepare(`
+    SELECT date(created_at) AS day,
+           COUNT(*) AS pageviews,
+           COUNT(DISTINCT session_id) AS visitors,
+           SUM(CASE WHEN is_bot=1 THEN 1 ELSE 0 END) AS bots
+    FROM page_visits WHERE created_at >= ?
+    GROUP BY day ORDER BY day
+  `).all(since);
+
+  const topPaths = db.prepare(`
+    SELECT path, COUNT(*) AS pageviews, COUNT(DISTINCT session_id) AS visitors
+    FROM page_visits WHERE created_at >= ? AND is_bot = 0
+    GROUP BY path ORDER BY pageviews DESC LIMIT 25
+  `).all(since);
+
+  const topReferrers = db.prepare(`
+    SELECT
+      COALESCE(NULLIF(substr(referrer, 1, instr(substr(referrer, 9), '/') + 7), ''), 'Direct') AS source,
+      COUNT(*) AS hits
+    FROM page_visits WHERE created_at >= ? AND is_bot = 0
+    GROUP BY source ORDER BY hits DESC LIMIT 15
+  `).all(since);
+
+  const devices = db.prepare(`
+    SELECT device, COUNT(*) AS hits
+    FROM page_visits WHERE created_at >= ?
+    GROUP BY device ORDER BY hits DESC
+  `).all(since);
+
+  const countries = db.prepare(`
+    SELECT COALESCE(country, 'Unknown') AS country, COUNT(*) AS hits
+    FROM page_visits WHERE created_at >= ? AND is_bot = 0
+    GROUP BY country ORDER BY hits DESC LIMIT 20
+  `).all(since);
+
+  const topBots = db.prepare(`
+    SELECT
+      CASE
+        WHEN user_agent LIKE '%Googlebot%' THEN 'Googlebot'
+        WHEN user_agent LIKE '%bingbot%' OR user_agent LIKE '%Bingbot%' THEN 'Bingbot'
+        WHEN user_agent LIKE '%DuckDuckBot%' THEN 'DuckDuckBot'
+        WHEN user_agent LIKE '%AhrefsBot%' THEN 'AhrefsBot'
+        WHEN user_agent LIKE '%SemrushBot%' THEN 'SemrushBot'
+        WHEN user_agent LIKE '%YandexBot%' THEN 'YandexBot'
+        WHEN user_agent LIKE '%Baiduspider%' THEN 'Baiduspider'
+        WHEN user_agent LIKE '%facebookexternalhit%' THEN 'Facebook'
+        WHEN user_agent LIKE '%GPTBot%' OR user_agent LIKE '%ChatGPT%' THEN 'GPTBot'
+        WHEN user_agent LIKE '%anthropic%' OR user_agent LIKE '%Claude%' THEN 'Claude / Anthropic'
+        WHEN user_agent LIKE '%PerplexityBot%' THEN 'Perplexity'
+        WHEN user_agent LIKE '%Applebot%' THEN 'Applebot'
+        ELSE 'Other bot'
+      END AS bot,
+      COUNT(*) AS hits
+    FROM page_visits WHERE created_at >= ? AND is_bot = 1
+    GROUP BY bot ORDER BY hits DESC LIMIT 15
+  `).all(since);
+
+  const signups = db.prepare(`
+    SELECT date(created_at) AS day, COUNT(*) AS count
+    FROM users WHERE created_at >= ? GROUP BY day ORDER BY day
+  `).all(since);
+
+  return {
+    period_days: n,
+    totals: {
+      pageviews: totalsRow.pageviews || 0,
+      visitors:  totalsRow.visitors  || 0,
+      bot_hits:  totalsRow.bot_hits  || 0,
+      human_hits: (totalsRow.pageviews || 0) - (totalsRow.bot_hits || 0),
+      authenticated_hits: totalsRow.authenticated_hits || 0,
+      authenticated_users: totalsRow.authenticated_users || 0,
+      pageviews_24h: last24Row.pageviews || 0,
+      visitors_24h:  last24Row.visitors  || 0,
+      pageviews_today: todayRow.pageviews || 0,
+      visitors_today:  todayRow.visitors  || 0,
+    },
+    timeline,
+    topPaths,
+    topReferrers,
+    devices,
+    countries,
+    topBots,
+    signups,
+  };
+}
+
+function getRecentVisits(limit) {
+  const n = Math.max(1, Math.min(500, parseInt(limit, 10) || 50));
+  return db.prepare(`
+    SELECT id, path, referrer, host, user_agent, country, device, is_bot, session_id, user_id, status_code, duration_ms, created_at
+    FROM page_visits ORDER BY id DESC LIMIT ?
+  `).all(n);
+}
+
+function getQuickCounts() {
+  const row24 = db.prepare(`
+    SELECT COUNT(*) AS pageviews, COUNT(DISTINCT session_id) AS visitors
+    FROM page_visits WHERE created_at >= datetime('now','-1 day')
+  `).get();
+  const row30 = db.prepare(`
+    SELECT COUNT(*) AS pageviews, COUNT(DISTINCT session_id) AS visitors
+    FROM page_visits WHERE created_at >= datetime('now','-30 days')
+  `).get();
+  const total = db.prepare(`SELECT COUNT(*) AS c FROM page_visits`).get();
+  return {
+    pageviews_24h: row24.pageviews || 0,
+    visitors_24h:  row24.visitors  || 0,
+    pageviews_30d: row30.pageviews || 0,
+    visitors_30d:  row30.visitors  || 0,
+    pageviews_total: total.c || 0,
+  };
+}
+
+module.exports = {
+  middleware,
+  getVisitorAnalytics,
+  getRecentVisits,
+  getQuickCounts,
+};