web-agent-bridge 2.9.0 → 3.0.0

package/server/runtime/scheduler.js

@@ -4,15 +4,19 @@
  * WAB Runtime - Task Scheduler
  * 
  * Distributes and manages task execution with:
- * - Priority queue
+ * - External queue backend (SQLite/Redis/Memory)
+ * - Priority queue with persistent storage
  * - Retry logic with exponential backoff
  * - Timeout management
  * - Dependency resolution
  * - Concurrency control
+ * - Deterministic replay integration
+ * - Container isolation support
  */
 
 const crypto = require('crypto');
 const { bus } = require('./event-bus');
+const { createQueue } = require('./queue');
 
 // Task states
 const TaskState = {
@@ -28,10 +32,13 @@ const TaskState = {
 
 class Scheduler {
   constructor(options = {}) {
-    this._queue = [];                  // priority queue
+    this._queue = [];                  // fallback in-memory queue
+    this._externalQueue = null;        // external queue backend
     this._running = new Map();         // taskId → task
     this._completed = new Map();       // taskId → result (limited buffer)
     this._handlers = new Map();        // task type → handler function
+    this._replayEngine = null;         // optional replay integration
+    this._containerRunner = null;      // optional container isolation
     this._maxConcurrent = options.maxConcurrent || 20;
     this._maxQueueSize = options.maxQueueSize || 1000;
     this._maxCompleted = options.maxCompleted || 500;
@@ -39,6 +46,31 @@ class Scheduler {
     this._defaultTimeout = options.defaultTimeout || 30000;
     this._processing = false;
     this._stats = { submitted: 0, completed: 0, failed: 0, retried: 0, cancelled: 0, timedOut: 0 };
+
+    // Initialize external queue
+    try {
+      this._externalQueue = createQueue('scheduler', {
+        maxRetries: this._defaultRetries,
+        processTimeout: this._defaultTimeout,
+      });
+    } catch {
+      // Fallback to in-memory
+      this._externalQueue = null;
+    }
+  }
+
+  /**
+   * Attach the replay engine for deterministic recording
+   */
+  setReplayEngine(engine) {
+    this._replayEngine = engine;
+  }
+
+  /**
+   * Attach the container runner for process isolation
+   */
+  setContainerRunner(runner) {
+    this._containerRunner = runner;
   }
 
   /**
@@ -52,10 +84,6 @@ class Scheduler {
    * Submit a task to the scheduler
    */
   submit(task) {
-    if (this._queue.length >= this._maxQueueSize) {
-      throw new Error('Task queue is full');
-    }
-
     const taskEntry = {
       id: task.id || `task_${crypto.randomBytes(16).toString('hex')}`,
       type: task.type || 'general',
@@ -72,6 +100,7 @@ class Scheduler {
       retriesLeft: task.retries !== undefined ? task.retries : this._defaultRetries,
       timeout: task.timeout || this._defaultTimeout,
       deadline: task.deadline || null,
+      isolate: task.isolate || false,  // run in container if true
 
       // Dependencies
       dependsOn: task.dependsOn || [],
@@ -94,16 +123,31 @@ class Scheduler {
       checkpoints: [],
     };
 
-    // Insert by priority (higher priority = earlier in queue)
-    let inserted = false;
-    for (let i = 0; i < this._queue.length; i++) {
-      if (taskEntry.priority > this._queue[i].priority) {
-        this._queue.splice(i, 0, taskEntry);
-        inserted = true;
-        break;
+    // Add to queue (external or in-memory)
+    if (this._externalQueue) {
+      this._externalQueue.enqueue({
+        type: taskEntry.type,
+        data: taskEntry,
+        priority: taskEntry.priority,
+        maxAttempts: taskEntry.retries + 1,
+        timeoutMs: taskEntry.timeout,
+        groupId: taskEntry.agentId || null,
+      });
+    } else {
+      if (this._queue.length >= this._maxQueueSize) {
+        throw new Error('Task queue is full');
       }
+      // Insert by priority (higher priority = earlier in queue)
+      let inserted = false;
+      for (let i = 0; i < this._queue.length; i++) {
+        if (taskEntry.priority > this._queue[i].priority) {
+          this._queue.splice(i, 0, taskEntry);
+          inserted = true;
+          break;
+        }
+      }
+      if (!inserted) this._queue.push(taskEntry);
     }
-    if (!inserted) this._queue.push(taskEntry);
 
     this._stats.submitted++;
     bus.emit('task.queued', { taskId: taskEntry.id, type: taskEntry.type, priority: taskEntry.priority });
@@ -125,28 +169,43 @@ class Scheduler {
     if (this._processing) return;
     this._processing = true;
 
-    while (this._queue.length > 0 && this._running.size < this._maxConcurrent) {
-      const task = this._findNextReady();
-      if (!task) break;
-
-      // Remove from queue
-      const idx = this._queue.indexOf(task);
-      if (idx !== -1) this._queue.splice(idx, 1);
-
-      // Check deadline
-      if (task.deadline && Date.now() > task.deadline) {
-        task.state = TaskState.CANCELLED;
-        task.error = { code: 'DEADLINE_EXCEEDED', message: 'Task deadline has passed' };
-        bus.emit('task.cancelled', { taskId: task.id, reason: 'deadline' });
-        continue;
+    try {
+      while (this._running.size < this._maxConcurrent) {
+        let task = null;
+
+        if (this._externalQueue) {
+          // Dequeue from external queue
+          const item = this._externalQueue.dequeue();
+          if (!item) break;
+          task = item.data;
+          task._queueItemId = item.id;
+        } else {
+          // Dequeue from in-memory queue
+          if (this._queue.length === 0) break;
+          task = this._findNextReady();
+          if (!task) break;
+          const idx = this._queue.indexOf(task);
+          if (idx !== -1) this._queue.splice(idx, 1);
+        }
+
+        // Check deadline
+        if (task.deadline && Date.now() > task.deadline) {
+          task.state = TaskState.CANCELLED;
+          task.error = { code: 'DEADLINE_EXCEEDED', message: 'Task deadline has passed' };
+          if (this._externalQueue && task._queueItemId) {
+            this._externalQueue.fail(task._queueItemId, 'Deadline exceeded');
+          }
+          bus.emit('task.cancelled', { taskId: task.id, reason: 'deadline' });
+          continue;
+        }
+
+        // Execute
+        this._running.set(task.id, task);
+        this._executeTask(task); // fire and forget
       }
-
-      // Execute
-      this._running.set(task.id, task);
-      this._executeTask(task); // fire and forget
+    } finally {
+      this._processing = false;
     }
-
-    this._processing = false;
   }
 
   /**
@@ -174,39 +233,90 @@ class Scheduler {
 
     bus.emit('task.started', { taskId: task.id, attempt: task.attempt, type: task.type });
 
+    // Start deterministic recording
+    if (this._replayEngine) {
+      this._replayEngine.startRecording(task.id, {
+        type: task.type,
+        params: task.params,
+        objective: task.objective,
+        attempt: task.attempt,
+      });
+    }
+
     const handler = this._handlers.get(task.type);
     if (!handler) {
       task.state = TaskState.FAILED;
       task.error = { code: 'NO_HANDLER', message: `No handler for task type: ${task.type}` };
+      if (this._replayEngine) {
+        this._replayEngine.completeRecording(task.id, null, task.error);
+      }
       this._finishTask(task);
       return;
     }
 
     try {
-      const result = await _withTimeout(
-        handler(task.params, {
-          taskId: task.id,
-          agentId: task.agentId,
-          siteId: task.siteId,
-          traceId: task.traceId,
-          steps: task.steps,
-          attempt: task.attempt,
-          reportProgress: (pct, step) => {
-            task.progress = pct;
-            if (step !== undefined) task.currentStep = step;
-            bus.emit('task.progress', { taskId: task.id, progress: pct, step });
-          },
-          checkpoint: (data) => {
-            task.checkpoints.push({ data, timestamp: Date.now(), step: task.currentStep });
-          },
-        }),
-        task.timeout
-      );
+      let result;
+
+      const ctx = {
+        taskId: task.id,
+        agentId: task.agentId,
+        siteId: task.siteId,
+        traceId: task.traceId,
+        steps: task.steps,
+        attempt: task.attempt,
+        reportProgress: (pct, step) => {
+          task.progress = pct;
+          if (step !== undefined) task.currentStep = step;
+          bus.emit('task.progress', { taskId: task.id, progress: pct, step });
+        },
+        checkpoint: (data) => {
+          task.checkpoints.push({ data, timestamp: Date.now(), step: task.currentStep });
+          if (this._replayEngine) {
+            this._replayEngine.saveCheckpoint(task.id, `step-${task.currentStep}`, data);
+          }
+        },
+        recordStep: (step) => {
+          if (this._replayEngine) {
+            this._replayEngine.recordStep(task.id, step);
+          }
+        },
+        recordSideEffect: (effect) => {
+          if (this._replayEngine) {
+            this._replayEngine.recordSideEffect(task.id, effect);
+          }
+        },
+      };
+
+      // Execute in container if isolation requested
+      if (task.isolate && this._containerRunner && typeof task.params === 'object' && task.params._code) {
+        const containerResult = await this._containerRunner.runInProcess(task.id, task.params._code, {
+          params: task.params,
+          timeout: task.timeout,
+          maxMemory: task.params._maxMemory || 256 * 1024 * 1024,
+        });
+        if (!containerResult.success) {
+          throw new Error(containerResult.error || 'Container execution failed');
+        }
+        result = containerResult.result;
+      } else {
+        result = await _withTimeout(handler(task.params, ctx), task.timeout);
+      }
 
       task.state = TaskState.COMPLETED;
       task.result = result;
       task.progress = 100;
       this._stats.completed++;
+
+      // Complete recording
+      if (this._replayEngine) {
+        this._replayEngine.completeRecording(task.id, result);
+      }
+
+      // Mark complete in external queue
+      if (this._externalQueue && task._queueItemId) {
+        this._externalQueue.complete(task._queueItemId, result);
+      }
+
       bus.emit('task.completed', { taskId: task.id, duration: Date.now() - task.startedAt });
     } catch (err) {
       const isTimeout = err.message.includes('timed out');
@@ -222,7 +332,18 @@ class Scheduler {
 
         setTimeout(() => {
           this._running.delete(task.id);
-          this._queue.unshift(task); // Add back to front of queue
+          if (this._externalQueue) {
+            // Re-enqueue in external queue
+            this._externalQueue.enqueue({
+              type: task.type,
+              data: task,
+              priority: task.priority,
+              maxAttempts: task.retriesLeft + 1,
+              timeoutMs: task.timeout,
+            });
+          } else {
+            this._queue.unshift(task);
+          }
           this._processQueue();
         }, backoff);
         return;
@@ -231,6 +352,17 @@ class Scheduler {
       task.state = TaskState.FAILED;
       task.error = { code: isTimeout ? 'TIMEOUT' : 'EXECUTION_ERROR', message: err.message };
       this._stats.failed++;
+
+      // Complete recording with error
+      if (this._replayEngine) {
+        this._replayEngine.completeRecording(task.id, null, err);
+      }
+
+      // Mark failed in external queue
+      if (this._externalQueue && task._queueItemId) {
+        this._externalQueue.fail(task._queueItemId, err.message);
+      }
+
       bus.emit('task.failed', { taskId: task.id, error: err.message, attempts: task.attempt });
     }
 
@@ -347,11 +479,18 @@ class Scheduler {
    * Get scheduler stats
    */
   getStats() {
+    const queueSize = this._externalQueue
+      ? this._externalQueue.size()
+      : this._queue.length;
+
     return {
       ...this._stats,
-      queueSize: this._queue.length,
+      queueSize,
+      queueBackend: this._externalQueue ? this._externalQueue.constructor.name : 'memory',
       runningCount: this._running.size,
       completedBufferSize: this._completed.size,
+      replayEnabled: !!this._replayEngine,
+      containerEnabled: !!this._containerRunner,
     };
   }