web-agent-bridge 2.8.0 → 3.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-agent-bridge",
-  "version": "2.8.0",
+  "version": "3.0.0",
   "description": "Open-source middleware that bridges AI agents and websites — providing a standardized command interface for intelligent automation",
   "main": "server/index.js",
   "bin": {

package/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-agent-bridge-sdk",
-  "version": "2.8.0",
+  "version": "3.0.0",
   "description": "SDK for building AI agents that interact with Web Agent Bridge (WAB)",
   "main": "index.js",
   "license": "MIT",

package/server/index.js

@@ -15,6 +15,7 @@ const { maybeBootstrapAdmin, db } = require('./models/db');
 const { initSearchEngine, search, getSuggestions, getTrendingSearches, getSearchStats, purgeOldCache } = require('./services/search-engine');
 const { processMessage: agentChat } = require('./services/agent-chat');
 const agentTasks = require('./services/agent-tasks');
+const { cluster } = require('./services/cluster');
 
 const authRoutes = require('./routes/auth');
 const apiRoutes = require('./routes/api');
@@ -359,6 +360,9 @@ if (process.env.NODE_ENV !== 'test') {
   // Start Agent OS runtime
   runtime.start();
 
+  // Start Cluster Orchestrator
+  cluster.start();
+
   server.listen(PORT, () => {
     console.log(`\n  ╔══════════════════════════════════════════╗`);
     console.log(`  ║   Web Agent Bridge v${pkg.version}                ║`);

package/server/routes/runtime.js

@@ -36,6 +36,7 @@ const { hostedRuntime } = require('../services/hosted-runtime');
 const { sessionEngine } = require('../runtime/session-engine');
 const vision = require('../services/vision');
 const { lfdEngine } = require('../services/lfd');
+const { cluster, distributor } = require('../services/cluster');
 
 // ═══════════════════════════════════════════════════════════════════════════
 // AUTH MIDDLEWARE
@@ -60,9 +61,7 @@ const PUBLIC_PATHS = [
   '/recipes',
   '/vision/models',
   '/vision/extraction-script',
-  '/recipes',
-  '/vision/models',
-  '/vision/extraction-script',
+  '/cluster/status',
 ];
 
 function authMiddleware(req, res, next) {
@@ -543,6 +542,7 @@ router.get('/observability/health', (req, res) => {
   health.hostedRuntime = hostedRuntime.getStats();
   health.metering = metering.getStats();
   health.lfd = lfdEngine.getStats();
+  health.cluster = cluster.getClusterStatus();
   res.json(health);
 });
 
@@ -1746,4 +1746,402 @@ router.get('/lfd/stats', (req, res) => {
   res.json(lfdEngine.getStats());
 });
 
+// ═══════════════════════════════════════════════════════════════════════════
+// CLUSTER — DISTRIBUTED EXECUTION & WORKER NODES
+// ═══════════════════════════════════════════════════════════════════════════
+
+/**
+ * Get cluster status (public)
+ */
+router.get('/cluster/status', (req, res) => {
+  res.json(cluster.getClusterStatus());
+});
+
+/**
+ * Register a worker node
+ */
+router.post('/cluster/nodes', (req, res) => {
+  try {
+    const result = cluster.registerNode({
+      name: req.body.name,
+      endpoint: req.body.endpoint,
+      region: req.body.region,
+      zone: req.body.zone,
+      role: req.body.role,
+      capacity: req.body.capacity,
+      tags: req.body.tags,
+      hardware: req.body.hardware,
+      version: req.body.version,
+      secret: req.body.secret,
+    });
+    res.json(result);
+  } catch (err) {
+    res.status(400).json({ error: err.message });
+  }
+});
+
+/**
+ * List cluster nodes
+ */
+router.get('/cluster/nodes', (req, res) => {
+  const nodes = cluster.listNodes({
+    region: req.query.region,
+    active: req.query.active === 'true',
+  });
+  res.json({ nodes });
+});
+
+/**
+ * Get a specific node
+ */
+router.get('/cluster/nodes/:nodeId', (req, res) => {
+  const node = cluster.getNode(req.params.nodeId);
+  if (!node) return res.status(404).json({ error: 'Node not found' });
+  res.json(node);
+});
+
+/**
+ * Remove a node
+ */
+router.delete('/cluster/nodes/:nodeId', (req, res) => {
+  const result = cluster.deregisterNode(req.params.nodeId);
+  if (!result) return res.status(404).json({ error: 'Node not found' });
+  res.json(result);
+});
+
+/**
+ * Worker heartbeat
+ */
+router.post('/cluster/nodes/:nodeId/heartbeat', (req, res) => {
+  const result = cluster.heartbeat(req.params.nodeId, {
+    capacityUsed: req.body.capacityUsed,
+    capacityTotal: req.body.capacityTotal,
+    hardware: req.body.hardware,
+    tags: req.body.tags,
+    version: req.body.version,
+  });
+  if (!result) return res.status(404).json({ error: 'Node not found' });
+  res.json(result);
+});
+
+/**
+ * Drain a node (stop new tasks, wait for running)
+ */
+router.post('/cluster/nodes/:nodeId/drain', (req, res) => {
+  const result = cluster.drainNode(req.params.nodeId);
+  if (!result) return res.status(404).json({ error: 'Node not found' });
+  res.json(result);
+});
+
+/**
+ * Cordon a node (prevent scheduling)
+ */
+router.post('/cluster/nodes/:nodeId/cordon', (req, res) => {
+  const result = cluster.cordonNode(req.params.nodeId);
+  if (!result) return res.status(404).json({ error: 'Node not found' });
+  res.json(result);
+});
+
+/**
+ * Uncordon a node (allow scheduling again)
+ */
+router.post('/cluster/nodes/:nodeId/uncordon', (req, res) => {
+  const result = cluster.uncordonNode(req.params.nodeId);
+  if (!result) return res.status(404).json({ error: 'Node not found' });
+  res.json(result);
+});
+
+/**
+ * Submit a task for distributed execution
+ */
+router.post('/cluster/tasks', (req, res) => {
+  try {
+    const result = distributor.submit({
+      type: req.body.type,
+      objective: req.body.objective,
+      params: req.body.params,
+      priority: req.body.priority,
+      affinityTags: req.body.affinityTags,
+      affinityRegion: req.body.affinityRegion,
+      timeout: req.body.timeout,
+      maxAttempts: req.body.maxAttempts,
+      externalId: req.body.externalId,
+    });
+    res.json(result);
+  } catch (err) {
+    res.status(400).json({ error: err.message });
+  }
+});
+
+/**
+ * Get task details
+ */
+router.get('/cluster/tasks/:taskId', (req, res) => {
+  const task = cluster.getTask(req.params.taskId);
+  if (!task) return res.status(404).json({ error: 'Task not found' });
+  res.json(task);
+});
+
+/**
+ * List tasks
+ */
+router.get('/cluster/tasks', (req, res) => {
+  const tasks = cluster.listTasks({
+    status: req.query.status,
+    nodeId: req.query.nodeId,
+    limit: parseInt(req.query.limit) || 50,
+  });
+  res.json({ tasks });
+});
+
+/**
+ * Worker pulls tasks (poll-based)
+ */
+router.post('/cluster/nodes/:nodeId/pull', (req, res) => {
+  const tasks = distributor.pullTasks(req.params.nodeId, parseInt(req.body.limit) || 5);
+  res.json({ tasks });
+});
+
+/**
+ * Worker reports task started
+ */
+router.post('/cluster/tasks/:taskId/started', (req, res) => {
+  const result = cluster.reportTaskStarted(req.params.taskId);
+  if (!result) return res.status(404).json({ error: 'Task not found' });
+  res.json(result);
+});
+
+/**
+ * Worker reports task completed
+ */
+router.post('/cluster/tasks/:taskId/completed', (req, res) => {
+  const result = cluster.reportTaskCompleted(req.params.taskId, req.body.result);
+  if (!result) return res.status(404).json({ error: 'Task not found' });
+  res.json(result);
+});
+
+/**
+ * Worker reports task failed
+ */
+router.post('/cluster/tasks/:taskId/failed', (req, res) => {
+  const result = cluster.reportTaskFailed(req.params.taskId, req.body.error);
+  if (!result) return res.status(404).json({ error: 'Task not found' });
+  res.json(result);
+});
+
+/**
+ * Get cluster events log
+ */
+router.get('/cluster/events', (req, res) => {
+  const events = cluster.getEvents(
+    parseInt(req.query.limit) || 100,
+    req.query.nodeId || null
+  );
+  res.json({ events });
+});
+
+// ═══════════════════════════════════════════════════════════════════════════
+// CONTAINER ISOLATION
+// ═══════════════════════════════════════════════════════════════════════════
+
+let containerRunner;
+try { containerRunner = require('../runtime/container').containerRunner; } catch {}
+
+/**
+ * Run a task in an isolated container
+ */
+router.post('/containers/run', async (req, res) => {
+  if (!containerRunner) return res.status(501).json({ error: 'Container isolation not available' });
+  try {
+    const result = await containerRunner.runInProcess(
+      req.body.taskId || `ctr_task_${Date.now()}`,
+      req.body.code || '',
+      {
+        params: req.body.params || {},
+        timeout: req.body.timeout || 60000,
+        maxMemory: req.body.maxMemory || 256 * 1024 * 1024,
+        allowNetwork: req.body.allowNetwork !== false,
+      }
+    );
+    res.json(result);
+  } catch (err) {
+    res.status(500).json({ error: err.message });
+  }
+});
+
+/**
+ * List active containers
+ */
+router.get('/containers', (req, res) => {
+  if (!containerRunner) return res.json({ containers: [] });
+  res.json({ containers: containerRunner.listContainers() });
+});
+
+/**
+ * Get container details
+ */
+router.get('/containers/:containerId', (req, res) => {
+  if (!containerRunner) return res.status(404).json({ error: 'Not found' });
+  const c = containerRunner.getContainer(req.params.containerId);
+  if (!c) return res.status(404).json({ error: 'Container not found' });
+  res.json(c);
+});
+
+/**
+ * Kill a container
+ */
+router.post('/containers/:containerId/kill', (req, res) => {
+  if (!containerRunner) return res.status(404).json({ error: 'Not found' });
+  const ok = containerRunner.kill(req.params.containerId);
+  res.json({ success: ok });
+});
+
+/**
+ * Container stats
+ */
+router.get('/containers/stats/summary', (req, res) => {
+  if (!containerRunner) return res.json({ active: 0 });
+  res.json(containerRunner.getStats());
+});
+
+/**
+ * Check Docker availability
+ */
+router.get('/containers/docker/status', (req, res) => {
+  if (!containerRunner) return res.json({ available: false });
+  res.json({ available: containerRunner.isDockerAvailable() });
+});
+
+// ═══════════════════════════════════════════════════════════════════════════
+// EXTERNAL QUEUE MANAGEMENT
+// ═══════════════════════════════════════════════════════════════════════════
+
+let queueModule;
+try { queueModule = require('../runtime/queue'); } catch {}
+
+/**
+ * Queue stats
+ */
+router.get('/queue/stats', (req, res) => {
+  if (!queueModule) return res.json({ backend: 'memory' });
+  const q = queueModule.createQueue('scheduler');
+  res.json(q.getStats());
+});
+
+/**
+ * Purge completed items from queue
+ */
+router.post('/queue/purge', (req, res) => {
+  if (!queueModule) return res.json({ purged: 0 });
+  const q = queueModule.createQueue('scheduler');
+  const purged = q.purgeCompleted(parseInt(req.body.maxAge) || 3600_000);
+  res.json({ purged });
+});
+
+// ═══════════════════════════════════════════════════════════════════════════
+// ENHANCED REPLAY
+// ═══════════════════════════════════════════════════════════════════════════
+
+/**
+ * Export a recording (full data for download)
+ */
+router.get('/replay/recordings/:taskId/export', (req, res) => {
+  const data = replayEngine.exportRecording(req.params.taskId);
+  if (!data) return res.status(404).json({ error: 'Recording not found' });
+  res.json(data);
+});
+
+/**
+ * Import a recording
+ */
+router.post('/replay/recordings/import', (req, res) => {
+  try {
+    const id = replayEngine.importRecording(req.body);
+    res.json({ success: true, recordingId: id });
+  } catch (err) {
+    res.status(400).json({ error: err.message });
+  }
+});
+
+/**
+ * Delete a recording
+ */
+router.delete('/replay/recordings/:taskId', (req, res) => {
+  replayEngine.deleteRecording(req.params.taskId);
+  res.json({ success: true });
+});
+
+/**
+ * Replay from a specific checkpoint
+ */
+router.post('/replay/:taskId/from-checkpoint', async (req, res) => {
+  try {
+    const result = await replayEngine.replay(req.params.taskId, {
+      verify: req.body.verify !== false,
+      continueOnMismatch: !!req.body.continueOnMismatch,
+      fromCheckpoint: req.body.checkpoint,
+    });
+    res.json(result);
+  } catch (err) {
+    res.status(400).json({ error: err.message });
+  }
+});
+
+/**
+ * Purge old recordings
+ */
+router.post('/replay/purge', (req, res) => {
+  const maxAge = parseInt(req.body.maxAge) || 7 * 24 * 3600_000;
+  replayEngine.purgeOld(maxAge);
+  res.json({ success: true });
+});
+
+// ═══════════════════════════════════════════════════════════════════════════
+// WORKER PULL ENDPOINT (for distributed workers)
+// ═══════════════════════════════════════════════════════════════════════════
+
+/**
+ * Workers pull tasks from here
+ */
+router.post('/cluster/nodes/:nodeId/pull', (req, res) => {
+  const limit = parseInt(req.body.limit) || 5;
+  // Fetch pending tasks from the cluster task distributor
+  const tasks = [];
+  try {
+    const pending = distributor.getPendingTasks ? distributor.getPendingTasks(req.params.nodeId, limit) : [];
+    tasks.push(...pending);
+  } catch {}
+  res.json({ tasks });
+});
+
+/**
+ * Worker reports task started
+ */
+router.post('/cluster/tasks/:taskId/started', (req, res) => {
+  bus.emit('cluster.task.started', { taskId: req.params.taskId, nodeId: req.body.nodeId });
+  res.json({ ok: true });
+});
+
+/**
+ * Worker reports task completed
+ */
+router.post('/cluster/tasks/:taskId/completed', (req, res) => {
+  bus.emit('cluster.task.completed', {
+    taskId: req.params.taskId,
+    result: req.body.result,
+  });
+  res.json({ ok: true });
+});
+
+/**
+ * Worker reports task failed
+ */
+router.post('/cluster/tasks/:taskId/failed', (req, res) => {
+  bus.emit('cluster.task.failed', {
+    taskId: req.params.taskId,
+    error: req.body.error,
+  });
+  res.json({ ok: true });
+});
+
 module.exports = router;

package/server/runtime/container-worker.js

@@ -0,0 +1,111 @@
+'use strict';
+
+/**
+ * WAB Container Worker — Runs inside a forked child process
+ *
+ * This script is the entry point for process-isolated task execution.
+ * It reads the task definition from a JSON file, executes it,
+ * and sends results back via IPC.
+ *
+ * Security:
+ *   - Runs in a separate process with memory limits (--max-old-space-size)
+ *   - Limited filesystem access (only its tmp directory)
+ *   - Can disable network via environment
+ *   - Timeout enforced by parent
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const taskFile = process.argv[2];
+if (!taskFile) {
+  process.stderr.write('No task file specified\n');
+  process.exit(1);
+}
+
+let taskData;
+try {
+  taskData = JSON.parse(fs.readFileSync(taskFile, 'utf8'));
+} catch (err) {
+  process.stderr.write(`Failed to read task file: ${err.message}\n`);
+  process.exit(1);
+}
+
+// ─── Sandbox Utilities (available to task code) ──────────────────────
+
+const sandbox = {
+  taskId: taskData.taskId,
+  containerId: taskData.containerId,
+  params: taskData.params || {},
+
+  // Send progress updates
+  progress(pct) {
+    if (process.send) process.send({ type: 'progress', progress: pct });
+  },
+
+  // Send log messages
+  log(message) {
+    if (process.send) process.send({ type: 'log', message: String(message).slice(0, 1000) });
+  },
+
+  // Read a param
+  param(key, defaultValue) {
+    return taskData.params[key] !== undefined ? taskData.params[key] : defaultValue;
+  },
+
+  // Filesystem is restricted to tmpDir
+  tmpDir: path.dirname(taskFile),
+
+  readFile(name) {
+    const p = path.join(sandbox.tmpDir, path.basename(name));
+    return fs.readFileSync(p, 'utf8');
+  },
+
+  writeFile(name, content) {
+    const p = path.join(sandbox.tmpDir, path.basename(name));
+    fs.writeFileSync(p, content);
+  },
+};
+
+// ─── Execute Task ────────────────────────────────────────────────────
+
+async function execute() {
+  try {
+    let result;
+
+    if (taskData.code) {
+      // Execute provided code string in a restricted scope
+      const AsyncFunction = Object.getPrototypeOf(async function(){}).constructor;
+      const fn = new AsyncFunction('sandbox', 'params', taskData.code);
+      result = await fn(sandbox, taskData.params);
+    } else if (taskData.module) {
+      // Execute a module (for trusted internal tasks)
+      const mod = require(taskData.module);
+      if (typeof mod.execute === 'function') {
+        result = await mod.execute(taskData.params, sandbox);
+      } else {
+        result = { error: 'Module has no execute() function' };
+      }
+    } else {
+      result = { echo: taskData.params, message: 'No code or module specified' };
+    }
+
+    // Send result back via IPC
+    if (process.send) {
+      process.send({ type: 'result', data: result });
+    }
+
+    // Give IPC time to flush
+    setTimeout(() => process.exit(0), 100);
+  } catch (err) {
+    process.stderr.write(`Task error: ${err.message}\n${err.stack}\n`);
+
+    if (process.send) {
+      process.send({ type: 'result', data: null });
+    }
+
+    setTimeout(() => process.exit(1), 100);
+  }
+}
+
+execute();