web-agent-bridge 2.9.0 → 3.2.0

package/server/routes/runtime.js

@@ -1940,4 +1940,208 @@ router.get('/cluster/events', (req, res) => {
   res.json({ events });
 });
 
+// ═══════════════════════════════════════════════════════════════════════════
+// CONTAINER ISOLATION
+// ═══════════════════════════════════════════════════════════════════════════
+
+let containerRunner;
+try { containerRunner = require('../runtime/container').containerRunner; } catch {}
+
+/**
+ * Run a task in an isolated container
+ */
+router.post('/containers/run', async (req, res) => {
+  if (!containerRunner) return res.status(501).json({ error: 'Container isolation not available' });
+  try {
+    const result = await containerRunner.runInProcess(
+      req.body.taskId || `ctr_task_${Date.now()}`,
+      req.body.code || '',
+      {
+        params: req.body.params || {},
+        timeout: req.body.timeout || 60000,
+        maxMemory: req.body.maxMemory || 256 * 1024 * 1024,
+        allowNetwork: req.body.allowNetwork !== false,
+      }
+    );
+    res.json(result);
+  } catch (err) {
+    res.status(500).json({ error: err.message });
+  }
+});
+
+/**
+ * List active containers
+ */
+router.get('/containers', (req, res) => {
+  if (!containerRunner) return res.json({ containers: [] });
+  res.json({ containers: containerRunner.listContainers() });
+});
+
+/**
+ * Get container details
+ */
+router.get('/containers/:containerId', (req, res) => {
+  if (!containerRunner) return res.status(404).json({ error: 'Not found' });
+  const c = containerRunner.getContainer(req.params.containerId);
+  if (!c) return res.status(404).json({ error: 'Container not found' });
+  res.json(c);
+});
+
+/**
+ * Kill a container
+ */
+router.post('/containers/:containerId/kill', (req, res) => {
+  if (!containerRunner) return res.status(404).json({ error: 'Not found' });
+  const ok = containerRunner.kill(req.params.containerId);
+  res.json({ success: ok });
+});
+
+/**
+ * Container stats
+ */
+router.get('/containers/stats/summary', (req, res) => {
+  if (!containerRunner) return res.json({ active: 0 });
+  res.json(containerRunner.getStats());
+});
+
+/**
+ * Check Docker availability
+ */
+router.get('/containers/docker/status', (req, res) => {
+  if (!containerRunner) return res.json({ available: false });
+  res.json({ available: containerRunner.isDockerAvailable() });
+});
+
+// ═══════════════════════════════════════════════════════════════════════════
+// EXTERNAL QUEUE MANAGEMENT
+// ═══════════════════════════════════════════════════════════════════════════
+
+let queueModule;
+try { queueModule = require('../runtime/queue'); } catch {}
+
+/**
+ * Queue stats
+ */
+router.get('/queue/stats', (req, res) => {
+  if (!queueModule) return res.json({ backend: 'memory' });
+  const q = queueModule.createQueue('scheduler');
+  res.json(q.getStats());
+});
+
+/**
+ * Purge completed items from queue
+ */
+router.post('/queue/purge', (req, res) => {
+  if (!queueModule) return res.json({ purged: 0 });
+  const q = queueModule.createQueue('scheduler');
+  const purged = q.purgeCompleted(parseInt(req.body.maxAge) || 3600_000);
+  res.json({ purged });
+});
+
+// ═══════════════════════════════════════════════════════════════════════════
+// ENHANCED REPLAY
+// ═══════════════════════════════════════════════════════════════════════════
+
+/**
+ * Export a recording (full data for download)
+ */
+router.get('/replay/recordings/:taskId/export', (req, res) => {
+  const data = replayEngine.exportRecording(req.params.taskId);
+  if (!data) return res.status(404).json({ error: 'Recording not found' });
+  res.json(data);
+});
+
+/**
+ * Import a recording
+ */
+router.post('/replay/recordings/import', (req, res) => {
+  try {
+    const id = replayEngine.importRecording(req.body);
+    res.json({ success: true, recordingId: id });
+  } catch (err) {
+    res.status(400).json({ error: err.message });
+  }
+});
+
+/**
+ * Delete a recording
+ */
+router.delete('/replay/recordings/:taskId', (req, res) => {
+  replayEngine.deleteRecording(req.params.taskId);
+  res.json({ success: true });
+});
+
+/**
+ * Replay from a specific checkpoint
+ */
+router.post('/replay/:taskId/from-checkpoint', async (req, res) => {
+  try {
+    const result = await replayEngine.replay(req.params.taskId, {
+      verify: req.body.verify !== false,
+      continueOnMismatch: !!req.body.continueOnMismatch,
+      fromCheckpoint: req.body.checkpoint,
+    });
+    res.json(result);
+  } catch (err) {
+    res.status(400).json({ error: err.message });
+  }
+});
+
+/**
+ * Purge old recordings
+ */
+router.post('/replay/purge', (req, res) => {
+  const maxAge = parseInt(req.body.maxAge) || 7 * 24 * 3600_000;
+  replayEngine.purgeOld(maxAge);
+  res.json({ success: true });
+});
+
+// ═══════════════════════════════════════════════════════════════════════════
+// WORKER PULL ENDPOINT (for distributed workers)
+// ═══════════════════════════════════════════════════════════════════════════
+
+/**
+ * Workers pull tasks from here
+ */
+router.post('/cluster/nodes/:nodeId/pull', (req, res) => {
+  const limit = parseInt(req.body.limit) || 5;
+  // Fetch pending tasks from the cluster task distributor
+  const tasks = [];
+  try {
+    const pending = distributor.getPendingTasks ? distributor.getPendingTasks(req.params.nodeId, limit) : [];
+    tasks.push(...pending);
+  } catch {}
+  res.json({ tasks });
+});
+
+/**
+ * Worker reports task started
+ */
+router.post('/cluster/tasks/:taskId/started', (req, res) => {
+  bus.emit('cluster.task.started', { taskId: req.params.taskId, nodeId: req.body.nodeId });
+  res.json({ ok: true });
+});
+
+/**
+ * Worker reports task completed
+ */
+router.post('/cluster/tasks/:taskId/completed', (req, res) => {
+  bus.emit('cluster.task.completed', {
+    taskId: req.params.taskId,
+    result: req.body.result,
+  });
+  res.json({ ok: true });
+});
+
+/**
+ * Worker reports task failed
+ */
+router.post('/cluster/tasks/:taskId/failed', (req, res) => {
+  bus.emit('cluster.task.failed', {
+    taskId: req.params.taskId,
+    error: req.body.error,
+  });
+  res.json({ ok: true });
+});
+
 module.exports = router;

package/server/routes/universal.js

@@ -11,7 +11,15 @@ const express = require('express');
 const router = express.Router();
 const scraper = require('../services/universal-scraper');
 const priceIntel = require('../services/price-intelligence');
-const fairness = require('../services/fairness-engine');
+let fairness;
+try { fairness = require('../services/fairness-engine'); } catch {
+  fairness = {
+    calculateFairnessScore: () => ({ score: 0, label: 'unrated' }),
+    rankWithFairness: (_items) => _items,
+    detectDarkPatterns: () => [],
+    getTopFairSites: () => []
+  };
+}
 
 // ─── POST /api/universal/extract ─────────────────────────────────────
 // Extract prices/products from a URL (server-side fetch)

package/server/routes/wab-api.js

@@ -14,12 +14,22 @@ const { findSiteById, findSiteByLicense, recordAnalytic, db } = require('../mode
 const { broadcastAnalytic } = require('../ws');
 const { wabAuthenticateLimiter, wabActionLimiter, searchLimiter } = require('../middleware/rateLimits');
 const { auditLog } = require('../services/security');
-const {
-  calculateNeutralityScore,
-  fairnessWeightedSearch,
-  getDirectoryListings,
-  generateFairnessReport
-} = require('../services/fairness');
+
+// Fairness module is proprietary — provide stubs when not available
+let calculateNeutralityScore, fairnessWeightedSearch, getDirectoryListings, generateFairnessReport;
+try {
+  ({
+    calculateNeutralityScore,
+    fairnessWeightedSearch,
+    getDirectoryListings,
+    generateFairnessReport
+  } = require('../services/fairness'));
+} catch {
+  calculateNeutralityScore = () => ({ score: 0, label: 'unrated' });
+  fairnessWeightedSearch = (_q, candidates) => candidates;
+  getDirectoryListings = () => [];
+  generateFairnessReport = () => ({ status: 'unavailable' });
+}
 
 const WAB_VERSION = '1.2.0';
 const PROTOCOL_VERSION = '1.0';

package/server/runtime/container-worker.js

@@ -0,0 +1,111 @@
+'use strict';
+
+/**
+ * WAB Container Worker — Runs inside a forked child process
+ *
+ * This script is the entry point for process-isolated task execution.
+ * It reads the task definition from a JSON file, executes it,
+ * and sends results back via IPC.
+ *
+ * Security:
+ *   - Runs in a separate process with memory limits (--max-old-space-size)
+ *   - Limited filesystem access (only its tmp directory)
+ *   - Can disable network via environment
+ *   - Timeout enforced by parent
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const taskFile = process.argv[2];
+if (!taskFile) {
+  process.stderr.write('No task file specified\n');
+  process.exit(1);
+}
+
+let taskData;
+try {
+  taskData = JSON.parse(fs.readFileSync(taskFile, 'utf8'));
+} catch (err) {
+  process.stderr.write(`Failed to read task file: ${err.message}\n`);
+  process.exit(1);
+}
+
+// ─── Sandbox Utilities (available to task code) ──────────────────────
+
+const sandbox = {
+  taskId: taskData.taskId,
+  containerId: taskData.containerId,
+  params: taskData.params || {},
+
+  // Send progress updates
+  progress(pct) {
+    if (process.send) process.send({ type: 'progress', progress: pct });
+  },
+
+  // Send log messages
+  log(message) {
+    if (process.send) process.send({ type: 'log', message: String(message).slice(0, 1000) });
+  },
+
+  // Read a param
+  param(key, defaultValue) {
+    return taskData.params[key] !== undefined ? taskData.params[key] : defaultValue;
+  },
+
+  // Filesystem is restricted to tmpDir
+  tmpDir: path.dirname(taskFile),
+
+  readFile(name) {
+    const p = path.join(sandbox.tmpDir, path.basename(name));
+    return fs.readFileSync(p, 'utf8');
+  },
+
+  writeFile(name, content) {
+    const p = path.join(sandbox.tmpDir, path.basename(name));
+    fs.writeFileSync(p, content);
+  },
+};
+
+// ─── Execute Task ────────────────────────────────────────────────────
+
+async function execute() {
+  try {
+    let result;
+
+    if (taskData.code) {
+      // Execute provided code string in a restricted scope
+      const AsyncFunction = Object.getPrototypeOf(async function(){}).constructor;
+      const fn = new AsyncFunction('sandbox', 'params', taskData.code);
+      result = await fn(sandbox, taskData.params);
+    } else if (taskData.module) {
+      // Execute a module (for trusted internal tasks)
+      const mod = require(taskData.module);
+      if (typeof mod.execute === 'function') {
+        result = await mod.execute(taskData.params, sandbox);
+      } else {
+        result = { error: 'Module has no execute() function' };
+      }
+    } else {
+      result = { echo: taskData.params, message: 'No code or module specified' };
+    }
+
+    // Send result back via IPC
+    if (process.send) {
+      process.send({ type: 'result', data: result });
+    }
+
+    // Give IPC time to flush
+    setTimeout(() => process.exit(0), 100);
+  } catch (err) {
+    process.stderr.write(`Task error: ${err.message}\n${err.stack}\n`);
+
+    if (process.send) {
+      process.send({ type: 'result', data: null });
+    }
+
+    setTimeout(() => process.exit(1), 100);
+  }
+}
+
+execute();