scene-capability-engine 3.6.8 → 3.6.10

package/lib/orchestrator/orchestration-engine.js

@@ -28,6 +28,10 @@ const DEFAULT_RATE_LIMIT_SIGNAL_WINDOW_MS = 30000;
 const DEFAULT_RATE_LIMIT_SIGNAL_THRESHOLD = 3;
 const DEFAULT_RATE_LIMIT_SIGNAL_EXTRA_HOLD_MS = 3000;
 const DEFAULT_RATE_LIMIT_DYNAMIC_BUDGET_FLOOR = 1;
+const DEFAULT_RATE_LIMIT_RETRY_SPREAD_MS = 600;
+const DEFAULT_RATE_LIMIT_LAUNCH_HOLD_POLL_MS = 1000;
+const DEFAULT_RATE_LIMIT_DECISION_EVENT_THROTTLE_MS = 1000;
+const MAX_RATE_LIMIT_RETRY_SPREAD_MS = 60000;
 const DEFAULT_AGENT_WAIT_TIMEOUT_SECONDS = 600;
 const AGENT_WAIT_TIMEOUT_GRACE_MS = 30000;
 const RATE_LIMIT_BACKOFF_JITTER_RATIO = 0.5;
@@ -136,12 +140,22 @@ class OrchestrationEngine extends EventEmitter {
     this._rateLimitSignalExtraHoldMs = DEFAULT_RATE_LIMIT_SIGNAL_EXTRA_HOLD_MS;
     /** @type {number} minimum dynamic launch budget floor under sustained pressure */
     this._rateLimitDynamicBudgetFloor = DEFAULT_RATE_LIMIT_DYNAMIC_BUDGET_FLOOR;
+    /** @type {number} deterministic per-spec retry spread to prevent synchronized retry bursts */
+    this._rateLimitRetrySpreadMs = DEFAULT_RATE_LIMIT_RETRY_SPREAD_MS;
+    /** @type {number} polling interval while launch hold is active */
+    this._rateLimitLaunchHoldPollMs = DEFAULT_RATE_LIMIT_LAUNCH_HOLD_POLL_MS;
+    /** @type {number} minimum interval between repeated rate-limit decision events */
+    this._rateLimitDecisionEventThrottleMs = DEFAULT_RATE_LIMIT_DECISION_EVENT_THROTTLE_MS;
     /** @type {number[]} timestamps (ms) of recent spec launches for rolling budget accounting */
     this._rateLimitLaunchTimestamps = [];
     /** @type {number} last launch-budget hold telemetry emission timestamp (ms) */
     this._launchBudgetLastHoldSignalAt = 0;
     /** @type {number} last launch-budget hold duration emitted to telemetry (ms) */
     this._launchBudgetLastHoldMs = 0;
+    /** @type {number} last rate-limit decision event emission timestamp (ms) */
+    this._lastRateLimitDecisionAt = 0;
+    /** @type {string} dedupe key for last rate-limit decision event */
+    this._lastRateLimitDecisionKey = '';
     /** @type {Set<{timer: NodeJS.Timeout|null, resolve: (() => void)|null}>} cancellable sleep waiters */
     this._pendingSleeps = new Set();
     /** @type {number} fallback wait timeout to avoid indefinite hangs when lifecycle events are missing */
@@ -373,10 +387,29 @@ class OrchestrationEngine extends EventEmitter {
         const launchHoldMs = Math.max(rateLimitHoldMs, launchBudgetHoldMs);
         if (launchHoldMs > 0) {
           // Pause new launches when provider asks us to retry later or launch budget is exhausted.
+          const holdReason = launchBudgetHoldMs >= rateLimitHoldMs
+            ? 'launch-budget'
+            : 'rate-limit-retry-hold';
           if (launchBudgetHoldMs > 0) {
             this._onLaunchBudgetHold(launchBudgetHoldMs);
           }
-          await this._sleep(Math.min(launchHoldMs, 1000));
+          const launchHoldPollMs = this._toPositiveInteger(
+            this._rateLimitLaunchHoldPollMs,
+            DEFAULT_RATE_LIMIT_LAUNCH_HOLD_POLL_MS
+          );
+          const holdSleepMs = Math.max(1, Math.min(launchHoldMs, launchHoldPollMs));
+          this._emitRateLimitDecision('launch-hold', {
+            reason: holdReason,
+            holdMs: launchHoldMs,
+            sleepMs: holdSleepMs,
+            pendingSpecs: pending.length,
+            inFlightSpecs: inFlight.size,
+            effectiveMaxParallel: this._toPositiveInteger(
+              this._effectiveMaxParallel,
+              this._toPositiveInteger(maxParallel, 1)
+            ),
+          });
+          await this._sleep(holdSleepMs);
           continue;
         }
 
@@ -606,9 +639,10 @@ class OrchestrationEngine extends EventEmitter {
       this._statusMonitor.incrementRetry(specName);
       this._statusMonitor.updateSpecStatus(specName, 'pending', null, resolvedError);
 
-      const retryDelayMs = isRateLimitError
-        ? this._resolveRateLimitRetryDelayMs(resolvedError, retryCount)
-        : 0;
+      const retryPlan = isRateLimitError
+        ? this._buildRateLimitRetryPlan(specName, retryCount, resolvedError)
+        : null;
+      const retryDelayMs = retryPlan ? retryPlan.totalDelayMs : 0;
       if (retryDelayMs > 0) {
         this._onRateLimitSignal(retryDelayMs);
         const launchHoldMs = this._getRateLimitLaunchHoldRemainingMs();
@@ -616,13 +650,33 @@ class OrchestrationEngine extends EventEmitter {
           specName,
           retryCount,
           retryDelayMs,
+          retryBaseDelayMs: retryPlan ? retryPlan.baseDelayMs : retryDelayMs,
+          retryHintMs: retryPlan ? retryPlan.retryAfterHintMs : 0,
+          retryBackoffMs: retryPlan ? retryPlan.computedBackoffMs : retryDelayMs,
+          retrySpreadMs: retryPlan ? retryPlan.spreadDelayMs : 0,
           launchHoldMs,
           error: resolvedError,
         });
+        this._emitRateLimitDecision('retry', {
+          reason: 'rate-limit-retry',
+          specName,
+          retryCount,
+          retryDelayMs,
+          retryBaseDelayMs: retryPlan ? retryPlan.baseDelayMs : retryDelayMs,
+          retryHintMs: retryPlan ? retryPlan.retryAfterHintMs : 0,
+          retryBackoffMs: retryPlan ? retryPlan.computedBackoffMs : retryDelayMs,
+          retrySpreadMs: retryPlan ? retryPlan.spreadDelayMs : 0,
+          launchHoldMs,
+          pendingRetryCount: this._retryCounts.get(specName) || 0,
+        });
         this.emit('spec:rate-limited', {
           specName,
           retryCount,
           retryDelayMs,
+          retryBaseDelayMs: retryPlan ? retryPlan.baseDelayMs : retryDelayMs,
+          retryHintMs: retryPlan ? retryPlan.retryAfterHintMs : 0,
+          retryBackoffMs: retryPlan ? retryPlan.computedBackoffMs : retryDelayMs,
+          retrySpreadMs: retryPlan ? retryPlan.spreadDelayMs : 0,
           launchHoldMs,
           error: resolvedError,
         });
@@ -638,6 +692,21 @@ class OrchestrationEngine extends EventEmitter {
       // Final failure (Req 5.3)
       this._failedSpecs.add(specName);
       this._statusMonitor.updateSpecStatus(specName, 'failed', agentId, resolvedError);
+      if (isRateLimitError) {
+        this._emitRateLimitDecision('retry-exhausted', {
+          reason: 'rate-limit-retry-budget-exhausted',
+          specName,
+          retryCount,
+          retryLimit,
+          error: resolvedError,
+        });
+        this.emit('spec:rate-limit-exhausted', {
+          specName,
+          retryCount,
+          retryLimit,
+          error: resolvedError,
+        });
+      }
 
       // Sync external status
       await this._syncExternalSafe(specName, 'failed');
@@ -1091,6 +1160,21 @@ class OrchestrationEngine extends EventEmitter {
       config && config.rateLimitDynamicBudgetFloor,
       DEFAULT_RATE_LIMIT_DYNAMIC_BUDGET_FLOOR
     );
+    this._rateLimitRetrySpreadMs = Math.min(
+      MAX_RATE_LIMIT_RETRY_SPREAD_MS,
+      this._toNonNegativeInteger(
+        config && config.rateLimitRetrySpreadMs,
+        DEFAULT_RATE_LIMIT_RETRY_SPREAD_MS
+      )
+    );
+    this._rateLimitLaunchHoldPollMs = this._toPositiveInteger(
+      config && config.rateLimitLaunchHoldPollMs,
+      DEFAULT_RATE_LIMIT_LAUNCH_HOLD_POLL_MS
+    );
+    this._rateLimitDecisionEventThrottleMs = this._toNonNegativeInteger(
+      config && config.rateLimitDecisionEventThrottleMs,
+      DEFAULT_RATE_LIMIT_DECISION_EVENT_THROTTLE_MS
+    );
   }
 
   /**
@@ -1225,6 +1309,12 @@ class OrchestrationEngine extends EventEmitter {
         effectiveMaxParallel: next,
         floor,
       });
+      this._emitRateLimitDecision('parallel-throttled', {
+        reason: 'rate-limit',
+        previousMaxParallel: current,
+        effectiveMaxParallel: next,
+        floor,
+      });
     } else {
       this._effectiveMaxParallel = current;
     }
@@ -1269,6 +1359,12 @@ class OrchestrationEngine extends EventEmitter {
         effectiveMaxParallel: next,
         maxParallel: boundedMax,
       });
+      this._emitRateLimitDecision('parallel-recovered', {
+        reason: 'rate-limit-cooldown',
+        previousMaxParallel: current,
+        effectiveMaxParallel: next,
+        maxParallel: boundedMax,
+      });
     }
   }
 
@@ -1415,6 +1511,13 @@ class OrchestrationEngine extends EventEmitter {
       windowMs,
       used: this._rateLimitLaunchTimestamps.length,
     });
+    this._emitRateLimitDecision('launch-budget-hold', {
+      reason: 'rate-limit-launch-budget',
+      holdMs,
+      budgetPerMinute,
+      windowMs,
+      used: this._rateLimitLaunchTimestamps.length,
+    });
   }
 
   /**
@@ -1606,6 +1709,11 @@ class OrchestrationEngine extends EventEmitter {
     if (extraHoldMs > 0) {
       const currentHoldUntil = this._toNonNegativeInteger(this._rateLimitLaunchHoldUntil, 0);
       this._rateLimitLaunchHoldUntil = Math.max(currentHoldUntil, now + extraHoldMs);
+      this._emitRateLimitDecision('launch-hold-escalated', {
+        reason: 'rate-limit-spike-hold',
+        signalCount,
+        extraHoldMs,
+      });
     }
 
     const configuredBudget = this._toNonNegativeInteger(
@@ -1654,6 +1762,13 @@ class OrchestrationEngine extends EventEmitter {
       windowMs: launchBudgetConfig.windowMs,
       holdMs,
     });
+    this._emitRateLimitDecision('launch-budget-throttled', {
+      reason: 'rate-limit-spike',
+      signalCount,
+      budgetPerMinute: launchBudgetConfig.budgetPerMinute,
+      windowMs: launchBudgetConfig.windowMs,
+      holdMs,
+    });
   }
 
   /**
@@ -1708,6 +1823,12 @@ class OrchestrationEngine extends EventEmitter {
       windowMs: launchBudgetConfig.windowMs,
       holdMs,
     });
+    this._emitRateLimitDecision('launch-budget-recovered', {
+      reason: 'rate-limit-cooldown',
+      budgetPerMinute: launchBudgetConfig.budgetPerMinute,
+      windowMs: launchBudgetConfig.windowMs,
+      holdMs,
+    });
   }
 
   /**
@@ -1730,6 +1851,113 @@ class OrchestrationEngine extends EventEmitter {
     return Math.max(1, Math.min(candidateDelayMs, maxDelayMs));
   }
 
+  /**
+   * Build retry delay details for a rate-limit failure.
+   * Keeps backoff compliant with provider hint while spreading retries across specs.
+   *
+   * @param {string} specName
+   * @param {number} retryCount
+   * @param {string} error
+   * @returns {{computedBackoffMs: number, retryAfterHintMs: number, baseDelayMs: number, spreadDelayMs: number, totalDelayMs: number}}
+   * @private
+   */
+  _buildRateLimitRetryPlan(specName, retryCount, error) {
+    const computedBackoffMs = this._calculateRateLimitBackoffMs(retryCount);
+    const retryAfterHintMs = this._extractRateLimitRetryAfterMs(error);
+    const baseDelayMs = this._resolveRateLimitRetryDelayMs(error, retryCount);
+    const spreadDelayMs = this._calculateRateLimitRetrySpreadMs(specName, retryCount);
+    return {
+      computedBackoffMs,
+      retryAfterHintMs,
+      baseDelayMs,
+      spreadDelayMs,
+      totalDelayMs: baseDelayMs + spreadDelayMs,
+    };
+  }
+
+  /**
+   * Spread same-round retries across specs to avoid synchronized 429 bursts.
+   *
+   * @param {string} specName
+   * @param {number} retryCount
+   * @returns {number}
+   * @private
+   */
+  _calculateRateLimitRetrySpreadMs(specName, retryCount) {
+    const spreadCapMs = Math.min(
+      MAX_RATE_LIMIT_RETRY_SPREAD_MS,
+      this._toNonNegativeInteger(
+        this._rateLimitRetrySpreadMs,
+        DEFAULT_RATE_LIMIT_RETRY_SPREAD_MS
+      )
+    );
+    if (spreadCapMs <= 0) {
+      return 0;
+    }
+
+    const normalizedSpecName = `${specName || ''}`.trim() || 'unknown-spec';
+    const retryOrdinal = this._toNonNegativeInteger(retryCount, 0) + 1;
+    const seed = `${normalizedSpecName}#${retryOrdinal}`;
+    const hash = this._hashString(seed);
+    return hash % (spreadCapMs + 1);
+  }
+
+  /**
+   * Lightweight deterministic hash for retry spread.
+   *
+   * @param {string} value
+   * @returns {number}
+   * @private
+   */
+  _hashString(value) {
+    let hash = 0;
+    const input = `${value || ''}`;
+    for (let idx = 0; idx < input.length; idx++) {
+      hash = ((hash * 31) + input.charCodeAt(idx)) >>> 0;
+    }
+    return hash;
+  }
+
+  /**
+   * Emit machine-readable rate-limit decision telemetry with simple de-dup throttling.
+   *
+   * @param {string} decision
+   * @param {object} payload
+   * @private
+   */
+  _emitRateLimitDecision(decision, payload = {}) {
+    const normalizedDecision = `${decision || ''}`.trim();
+    if (!normalizedDecision) {
+      return;
+    }
+
+    const now = this._getNow();
+    const reason = payload && typeof payload.reason === 'string'
+      ? payload.reason.trim()
+      : '';
+    const dedupeKey = `${normalizedDecision}:${reason}`;
+    const throttleMs = this._toNonNegativeInteger(
+      this._rateLimitDecisionEventThrottleMs,
+      DEFAULT_RATE_LIMIT_DECISION_EVENT_THROTTLE_MS
+    );
+
+    if (
+      throttleMs > 0
+      && dedupeKey === this._lastRateLimitDecisionKey
+      && (now - this._lastRateLimitDecisionAt) < throttleMs
+    ) {
+      return;
+    }
+
+    this._lastRateLimitDecisionAt = now;
+    this._lastRateLimitDecisionKey = dedupeKey;
+    this.emit('rate-limit:decision', {
+      decision: normalizedDecision,
+      at: new Date(now).toISOString(),
+      ...(payload && typeof payload === 'object' ? payload : {}),
+    });
+  }
+
   /**
    * @param {number} ms
    * @returns {Promise<void>}
@@ -1941,6 +2169,8 @@ class OrchestrationEngine extends EventEmitter {
     this._dynamicLaunchBudgetPerMinute = null;
     this._launchBudgetLastHoldSignalAt = 0;
     this._launchBudgetLastHoldMs = 0;
+    this._lastRateLimitDecisionAt = 0;
+    this._lastRateLimitDecisionKey = '';
   }
 }
 

package/lib/orchestrator/orchestrator-config.js

@@ -37,6 +37,9 @@ const KNOWN_KEYS = new Set([
   'rateLimitSignalThreshold',
   'rateLimitSignalExtraHoldMs',
   'rateLimitDynamicBudgetFloor',
+  'rateLimitRetrySpreadMs',
+  'rateLimitLaunchHoldPollMs',
+  'rateLimitDecisionEventThrottleMs',
   'apiKeyEnvVar',
   'bootstrapTemplate',
   'codexArgs',
@@ -57,6 +60,9 @@ const RATE_LIMIT_PROFILE_PRESETS = Object.freeze({
     rateLimitSignalThreshold: 2,
     rateLimitSignalExtraHoldMs: 5000,
     rateLimitDynamicBudgetFloor: 1,
+    rateLimitRetrySpreadMs: 1200,
+    rateLimitLaunchHoldPollMs: 1000,
+    rateLimitDecisionEventThrottleMs: 1000,
   }),
   balanced: Object.freeze({
     rateLimitMaxRetries: 8,
@@ -71,6 +77,9 @@ const RATE_LIMIT_PROFILE_PRESETS = Object.freeze({
     rateLimitSignalThreshold: 3,
     rateLimitSignalExtraHoldMs: 3000,
     rateLimitDynamicBudgetFloor: 1,
+    rateLimitRetrySpreadMs: 600,
+    rateLimitLaunchHoldPollMs: 1000,
+    rateLimitDecisionEventThrottleMs: 1000,
   }),
   aggressive: Object.freeze({
     rateLimitMaxRetries: 6,
@@ -85,6 +94,9 @@ const RATE_LIMIT_PROFILE_PRESETS = Object.freeze({
     rateLimitSignalThreshold: 4,
     rateLimitSignalExtraHoldMs: 2000,
     rateLimitDynamicBudgetFloor: 2,
+    rateLimitRetrySpreadMs: 250,
+    rateLimitLaunchHoldPollMs: 1000,
+    rateLimitDecisionEventThrottleMs: 1000,
   }),
 });
 
@@ -124,6 +136,9 @@ const DEFAULT_CONFIG = Object.freeze({
   rateLimitSignalThreshold: 3,
   rateLimitSignalExtraHoldMs: 3000,
   rateLimitDynamicBudgetFloor: 1,
+  rateLimitRetrySpreadMs: 600,
+  rateLimitLaunchHoldPollMs: 1000,
+  rateLimitDecisionEventThrottleMs: 1000,
   apiKeyEnvVar: 'CODEX_API_KEY',
   bootstrapTemplate: null,
   codexArgs: [],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scene-capability-engine",
-  "version": "3.6.8",
+  "version": "3.6.10",
   "description": "SCE (Scene Capability Engine) - A CLI tool and npm package for spec-driven development with AI coding assistants.",
   "main": "index.js",
   "bin": {