testdriverai 7.8.0-test.4 → 7.8.0-test.40

package/agent/lib/sandbox.js

@@ -1,93 +1,17 @@
-const crypto = require("crypto");
 const Ably = require("ably");
+const axios = require("axios");
 const { events } = require("../events");
 const logger = require("./logger");
 const { version } = require("../../package.json");
-
-function getSentryTraceHeaders(sessionId) {
-  if (!sessionId) return {};
-  const traceId = crypto.createHash("md5").update(sessionId).digest("hex");
-  const spanId = crypto.randomBytes(8).toString("hex");
-  return {
-    "sentry-trace": traceId + "-" + spanId + "-1",
-    baggage:
-      "sentry-trace_id=" +
-      traceId +
-      ",sentry-sample_rate=1.0,sentry-sampled=true",
-  };
-}
-
-function httpPost(apiRoot, path, body, timeout) {
-  const http = require("http");
-  const https = require("https");
-  const url = new URL(apiRoot + path);
-  const transport = url.protocol === "https:" ? https : http;
-  const bodyStr = JSON.stringify(body);
-
-  return new Promise(function (resolve, reject) {
-    var timeoutId = timeout
-      ? setTimeout(function () {
-          req.destroy();
-          reject(
-            new Error("HTTP request timed out after " + timeout + "ms"),
-          );
-        }, timeout)
-      : null;
-
-    var req = transport.request(
-      url,
-      {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          "Content-Length": Buffer.byteLength(bodyStr),
-          "Connection": "close",
-        },
-      },
-      function (res) {
-        var data = "";
-        res.on("data", function (chunk) {
-          data += chunk;
-        });
-        res.on("end", function () {
-          if (timeoutId) clearTimeout(timeoutId);
-          try {
-            var parsed = JSON.parse(data);
-            if (res.statusCode >= 400) {
-              var err = new Error(
-                parsed.errorMessage ||
-                  parsed.message ||
-                  "HTTP " + res.statusCode,
-              );
-              err.responseData = parsed;
-              reject(err);
-            } else {
-              resolve(parsed);
-            }
-          } catch (e) {
-            reject(new Error("Failed to parse API response: " + data));
-          }
-        });
-      },
-    );
-    req.on("error", function (err) {
-      if (timeoutId) clearTimeout(timeoutId);
-      reject(err);
-    });
-    req.write(bodyStr);
-    req.end();
-  });
-}
+const { withRetry, getSentryTraceHeaders } = require("./sdk");
+const sentry = require("../../lib/sentry");
 
 const createSandbox = function (emitter, analytics, sessionInstance) {
   class Sandbox {
     constructor() {
       this._ably = null;
-      this._cmdChannel = null;
-      this._respChannel = null;
-      this._ctrlChannel = null;
-      this._filesChannel = null;
-      this._channelNames = null;
+      this._sessionChannel = null;
+      this._channelName = null;
       this.ps = {};
       this._execBuffers = {}; // accumulate streamed exec.output chunks per requestId
       this.heartbeat = null;
@@ -105,6 +29,12 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       this._lastConnectParams = null;
       this._teamId = null;
       this._sandboxId = null;
+
+      // Rate limiting state for Ably publishes (Ably limits to 50 msg/sec per connection)
+      this._publishLastTime = 0;
+      this._publishMinIntervalMs = 25; // 40 msg/sec max, safely under Ably's 50 limit
+      this._publishCount = 0;
+      this._publishWindowStart = Date.now();
     }
 
     getTraceId() {
@@ -118,7 +48,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       );
     }
 
-    async _initAbly(ablyToken, channelNames) {
+    async _initAbly(ablyToken, channelName) {
       if (this._ably) {
         try {
           this._ably.close();
@@ -126,7 +56,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
           /* ignore */
         }
       }
-      this._channelNames = channelNames;
+      this._channelName = channelName;
       var self = this;
 
       this._ably = new Ably.Realtime({
@@ -134,10 +64,13 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
           callback(null, ablyToken);
         },
         clientId: "sdk-" + this._sandboxId,
+        echoMessages: false,              // don't receive our own published messages
         disconnectedRetryTimeout: 5000,   // retry reconnect every 5s (default 15s)
         suspendedRetryTimeout: 15000,     // retry from suspended every 15s (default 30s)
       });
 
+      logger.log(`[ably] Connecting as sdk-${this._sandboxId}...`);
+
       await new Promise(function (resolve, reject) {
         self._ably.connection.on("connected", resolve);
         self._ably.connection.on("failed", function () {
@@ -148,26 +81,29 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
         }, 30000);
       });
 
-      this._cmdChannel = this._ably.channels.get(channelNames.commands);
-      this._respChannel = this._ably.channels.get(channelNames.responses);
-      this._ctrlChannel = this._ably.channels.get(channelNames.control);
-      this._filesChannel = this._ably.channels.get(channelNames.files);
+      this._sessionChannel = this._ably.channels.get(channelName);
+
+      logger.log(`[ably] Channel initialized: ${channelName}`);
 
-      // Enter presence on control channel so the API can count connected SDK clients
+      // Enter presence on the session channel so the API can count connected SDK clients
       try {
-        await this._ctrlChannel.presence.enter({
+        await this._sessionChannel.presence.enter({
           sandboxId: this._sandboxId,
           connectedAt: Date.now(),
         });
+        logger.log(`[ably] Entered presence on session channel (sandbox=${this._sandboxId})`);
       } catch (e) {
         // Non-fatal — presence is used for concurrency counting, not critical path
-        logger.warn("Failed to enter presence on control channel: " + (e.message || e));
+        logger.warn("Failed to enter presence on session channel: " + (e.message || e));
       }
 
-      this._respChannel.subscribe("response", function (msg) {
+      // Save subscription references for historyBeforeSubscribe() during discontinuity recovery
+      this._responseSubscription = await this._sessionChannel.subscribe("response", function (msg) {
         var message = msg.data;
         if (!message) return;
 
+        logger.log(`[ably] Received response: type=${message.type || 'unknown'} (requestId=${message.requestId || 'none'})`);
+
         if (message.type === "sandbox.progress") {
           emitter.emit(events.sandbox.progress, {
             step: message.step,
@@ -263,9 +199,10 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
         delete self.ps[message.requestId];
       });
 
-      this._filesChannel.subscribe("response", function (msg) {
+      this._fileSubscription = await this._sessionChannel.subscribe("file", function (msg) {
         var message = msg.data;
         if (!message) return;
+        logger.log(`[ably] Received file: type=${message.type || 'unknown'} (requestId=${message.requestId || 'none'})`);
         if (message.requestId && self.ps[message.requestId]) {
           emitter.emit(events.sandbox.received);
           self.ps[message.requestId].resolve(message);
@@ -274,64 +211,168 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
         emitter.emit(events.sandbox.file, message);
       });
 
-      this.heartbeat = setInterval(function () {}, 5000);
+      this.heartbeat = setInterval(function () { }, 5000);
       if (this.heartbeat.unref) this.heartbeat.unref();
 
+      // ─── Periodic stats logging ────────────────────────────────────────
+      this._statsInterval = setInterval(() => {
+        const connState = this._ably ? this._ably.connection.state : 'no-client';
+        const chState = this._sessionChannel ? this._sessionChannel.state : 'null';
+        const pending = Object.keys(this.ps).length;
+        logger.log(`[ably][stats] connection=${connState} | sandbox=${this._sandboxId} | pending=${pending} | channel=${chState}`);
+      }, 10000);
+      if (this._statsInterval.unref) this._statsInterval.unref();
+
       this._ably.connection.on("disconnected", function () {
-        logger.log("Ably disconnected - will auto-reconnect");
+        logger.log("[ably] Connection: disconnected - will auto-reconnect");
       });
 
       this._ably.connection.on("connected", function () {
         // Log reconnection so the user knows the blip was recovered
-        logger.log("Ably reconnected");
+        logger.log("[ably] Connection: reconnected");
       });
 
       this._ably.connection.on("suspended", function () {
-        logger.warn("Ably suspended - connection lost for extended period, will keep retrying");
+        logger.warn("[ably] Connection: suspended - connection lost for extended period, will keep retrying");
       });
 
       this._ably.connection.on("failed", function () {
+        logger.error("[ably] Connection: failed");
         self.apiSocketConnected = false;
         self.instanceSocketConnected = false;
         emitter.emit(events.error.sandbox, "Ably connection failed");
       });
+
+      // ─── Channel discontinuity detection ──────────────────────────────
+      // Set up BEFORE subscribing so we catch any continuity loss during
+      // the initial attachment. Fires at the channel level, covering all
+      // message types (response, file, control).
+      this._sessionChannel.on(function (stateChange) {
+        var current = stateChange.current;
+        var previous = stateChange.previous;
+        var reason = stateChange.reason;
+        var reasonMsg = reason ? (reason.message || reason.code || String(reason)) : '';
+
+        if (current === 'attached' && stateChange.resumed === false && previous) {
+          logger.warn('[ably] Channel DISCONTINUITY detected (resumed=false)' + (reasonMsg ? ' — ' + reasonMsg : ''));
+          emitter.emit(events.sandbox.progress, {
+            step: 'discontinuity',
+            message: 'Recovering missed messages after connection interruption...',
+          });
+          self._recoverFromDiscontinuity();
+        }
+      });
     }
 
     /**
-     * Wrapper around httpPost that retries when the server responds with
-     * CONCURRENCY_LIMIT_EXCEEDED (HTTP 429).  Instead of failing the test
-     * immediately, we wait for a slot to become available — polling every
-     * 10 s until vitest's testTimeout kills the test.
+     * Recover missed messages after a channel discontinuity.
+     * Uses historyBeforeSubscribe() on each subscription, which guarantees
+     * no gap between historical and live messages.
+     */
+    async _recoverFromDiscontinuity() {
+      var subs = [
+        { name: 'response', sub: this._responseSubscription },
+        { name: 'file', sub: this._fileSubscription },
+      ];
+      var totalRecovered = 0;
+      for (var i = 0; i < subs.length; i++) {
+        var entry = subs[i];
+        if (!entry.sub) continue;
+        try {
+          logger.log('[ably] Discontinuity recovery: fetching historyBeforeSubscribe for ' + entry.name + '...');
+          var page = await entry.sub.historyBeforeSubscribe({ limit: 100 });
+          var recovered = 0;
+          while (page) {
+            recovered += page.items.length;
+            page = page.hasNext() ? await page.next() : null;
+          }
+          totalRecovered += recovered;
+          logger.log('[ably] Discontinuity recovery: found ' + recovered + ' ' + entry.name + ' message(s) in gap');
+        } catch (err) {
+          logger.error('[ably] Discontinuity recovery failed for ' + entry.name + ': ' + (err.message || err));
+        }
+      }
+      if (totalRecovered > 0) {
+        logger.warn('[ably] Recovered ' + totalRecovered + ' message(s) that were missed during connection interruption');
+      } else {
+        logger.log('[ably] Discontinuity recovery: no missed messages found');
+      }
+    }
+
+    /**
+     * POST to the API with retry for transient network errors (via withRetry)
+     * and infinite polling for CONCURRENCY_LIMIT_EXCEEDED (until vitest's
+     * testTimeout kills the test).
      */
     async _httpPostWithConcurrencyRetry(path, body, timeout) {
-      var retryInterval = 10000; // 10 seconds between retries
+      var concurrencyRetryInterval = 10000; // 10 seconds between concurrency retries
       var startTime = Date.now();
+      var sessionId = this.sessionInstance ? this.sessionInstance.get() : null;
+
+      var self = this;
+      var makeRequest = function () {
+        return axios({
+          method: "post",
+          url: self.apiRoot + path,
+          data: body,
+          headers: {
+            "Content-Type": "application/json",
+            "User-Agent": "TestDriverSDK/" + version + " (Node.js " + process.version + ")",
+            ...getSentryTraceHeaders(sessionId),
+          },
+          timeout: timeout || 120000,
+        });
+      };
 
       while (true) {
         try {
-          return await httpPost(this.apiRoot, path, body, timeout);
+          var response = await withRetry(makeRequest, {
+            retryConfig: {
+              maxRetries: 3,
+              baseDelayMs: 2000,
+              retryableStatusCodes: [500, 502, 503, 504], // Don't retry 429 — handled below
+            },
+            onRetry: function (attempt, error, delayMs) {
+              var elapsed = Date.now() - startTime;
+              logger.warn(
+                "Transient network error: " + (error.message || error.code) +
+                " — POST " + path +
+                " — retry " + attempt + "/3" +
+                " in " + (delayMs / 1000).toFixed(1) + "s" +
+                " (" + Math.round(elapsed / 1000) + "s elapsed)...",
+              );
+            },
+          });
+          return response.data;
         } catch (err) {
-          var isConcurrencyLimit =
-            err.responseData &&
-            err.responseData.errorCode === "CONCURRENCY_LIMIT_EXCEEDED";
-
-          if (!isConcurrencyLimit) {
-            throw err;
-          }
-
-          var elapsed = Date.now() - startTime;
-
-          logger.log(
-            "Concurrency limit reached — waiting " +
-              retryInterval / 1000 +
+          // Concurrency limit — poll forever until a slot opens
+          var responseData = err.response && err.response.data;
+          if (responseData && responseData.errorCode === "CONCURRENCY_LIMIT_EXCEEDED") {
+            var elapsed = Date.now() - startTime;
+            logger.log(
+              "Concurrency limit reached — waiting " +
+              concurrencyRetryInterval / 1000 +
               "s for a slot to become available (" +
               Math.round(elapsed / 1000) +
               "s elapsed)...",
-          );
-          await new Promise(function (resolve) {
-            var t = setTimeout(resolve, retryInterval);
-            if (t.unref) t.unref();
-          });
+            );
+            await new Promise(function (resolve) {
+              var t = setTimeout(resolve, concurrencyRetryInterval);
+              if (t.unref) t.unref();
+            });
+            continue;
+          }
+
+          // Non-retryable HTTP error — preserve responseData for callers
+          if (responseData) {
+            var httpErr = new Error(
+              responseData.errorMessage || responseData.message || "HTTP " + err.response.status,
+            );
+            httpErr.responseData = responseData;
+            throw httpErr;
+          }
+
+          throw err;
         }
       }
     }
@@ -390,14 +431,14 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       this._teamId = reply.teamId;
 
       if (reply.ably && reply.ably.token) {
-        await this._initAbly(reply.ably.token, reply.ably.channels);
+        await this._initAbly(reply.ably.token, reply.ably.channel);
         this.instanceSocketConnected = true;
 
         // Tell the runner to enable debug log forwarding if debug mode is on
         var debugMode =
           process.env.VERBOSE || process.env.TD_DEBUG;
-        if (debugMode && this._ctrlChannel) {
-          this._ctrlChannel.publish("control", {
+        if (debugMode && this._sessionChannel) {
+          this._sessionChannel.publish("control", {
             type: "debug",
             enabled: true,
           });
@@ -430,7 +471,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
         // agent to signal readiness before sending commands.  Without this
         // gate, commands published before the agent subscribes are lost.
         var self = this;
-        if (!reply.runner && this._ctrlChannel) {
+        if (!reply.runner && this._sessionChannel) {
           logger.log('Waiting for runner agent to signal readiness...');
           var readyTimeout = 120000; // 120s — allows for EC2 boot + agent startup
           await new Promise(function (resolve, reject) {
@@ -439,7 +480,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
               if (resolved) return;
               resolved = true;
               clearTimeout(timer);
-              self._ctrlChannel.unsubscribe('control', onCtrl);
+              self._sessionChannel.unsubscribe('control', onCtrl);
               // Update runner info if provided
               if (data && data.os) reply.runner = reply.runner || {};
               if (data && data.os && reply.runner) reply.runner.os = data.os;
@@ -464,8 +505,13 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
             var timer = setTimeout(function () {
               if (!resolved) {
                 resolved = true;
-                self._ctrlChannel.unsubscribe('control', onCtrl);
-                reject(new Error('Runner agent did not signal readiness within ' + readyTimeout + 'ms'));
+                self._sessionChannel.unsubscribe('control', onCtrl);
+                var err = new Error('Runner agent did not signal readiness within ' + readyTimeout + 'ms');
+                sentry.captureException(err, {
+                  tags: { phase: 'runner_ready', connection_type: 'create' },
+                  extra: { readyTimeout: readyTimeout, sandboxId: reply.sandboxId },
+                });
+                reject(err);
               }
             }, readyTimeout);
             if (timer.unref) timer.unref();
@@ -478,12 +524,12 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
                 finish(data);
               }
             };
-            self._ctrlChannel.subscribe('control', onCtrl);
+            self._sessionChannel.subscribe('control', onCtrl);
 
             // Also check channel history in case runner.ready was published
             // before we subscribed (race condition on fast-booting agents).
             try {
-              self._ctrlChannel.history({ limit: 50 }, function (err, page) {
+              self._sessionChannel.history({ limit: 50 }, function (err, page) {
                 if (err) {
                   logger.warn('History lookup failed (non-fatal): ' + (err.message || err));
                   return;
@@ -538,17 +584,22 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
         // provision the config to the instance via SSM (client-side).
         // This runs from the user's infrastructure where AWS permissions exist,
         // rather than from the API server.
-        if (reply.agentConfig && message.instanceId) {
-          logger.log('Provisioning agent config to instance ' + message.instanceId + ' via SSM...');
-          await this._provisionAgentConfig(message.instanceId, reply.agentConfig);
+        // NOTE: For direct connections, the user MUST provide the AWS instanceId
+        // because the API only knows the sandboxId, not the actual EC2 instance ID.
+        var instanceId = message.instanceId;
+        if (reply.agentConfig && instanceId) {
+          logger.log('Provisioning agent config to instance ' + instanceId + ' via SSM...');
+          await this._provisionAgentConfig(instanceId, reply.agentConfig);
           logger.log('Agent config provisioned successfully.');
+        } else if (reply.agentConfig && !instanceId) {
+          logger.log('Warning: agentConfig returned but no instanceId provided - cannot provision via SSM');
         }
 
         // If the API returned agent credentials (reply.agent present),
         // wait for the runner agent to signal readiness before sending commands.
         // Without this gate, commands published before the agent subscribes are lost.
         var self = this;
-        if (reply.agent && this._ctrlChannel) {
+        if (reply.agent && this._sessionChannel) {
           logger.log('Waiting for runner agent to signal readiness (direct connection)...');
           var readyTimeout = 120000; // 120s — allows for SSM provisioning + agent startup
           await new Promise(function (resolve, reject) {
@@ -557,7 +608,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
               if (resolved) return;
               resolved = true;
               clearTimeout(timer);
-              self._ctrlChannel.unsubscribe('control', onCtrl);
+              self._sessionChannel.unsubscribe('control', onCtrl);
               logger.log('Runner agent ready (direct, os=' + ((data && data.os) || 'unknown') + ', runner v' + ((data && data.runnerVersion) || 'unknown') + ')');
               if (data && data.update) {
                 var u = data.update;
@@ -577,8 +628,13 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
             var timer = setTimeout(function () {
               if (!resolved) {
                 resolved = true;
-                self._ctrlChannel.unsubscribe('control', onCtrl);
-                reject(new Error('Runner agent did not signal readiness within ' + readyTimeout + 'ms (direct connection)'));
+                self._sessionChannel.unsubscribe('control', onCtrl);
+                var err = new Error('Runner agent did not signal readiness within ' + readyTimeout + 'ms (direct connection)');
+                sentry.captureException(err, {
+                  tags: { phase: 'runner_ready', connection_type: 'direct' },
+                  extra: { readyTimeout: readyTimeout, sandboxId: reply.sandboxId, instanceId: message.instanceId },
+                });
+                reject(err);
               }
             }, readyTimeout);
             if (timer.unref) timer.unref();
@@ -591,12 +647,12 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
                 finish(data);
               }
             };
-            self._ctrlChannel.subscribe('control', onCtrl);
+            self._sessionChannel.subscribe('control', onCtrl);
 
             // Also check channel history in case runner.ready was published
             // before we subscribed (race condition on fast-booting agents).
             try {
-              self._ctrlChannel.history({ limit: 50 }, function (err, page) {
+              self._sessionChannel.history({ limit: 50 }, function (err, page) {
                 if (err) {
                   logger.warn('History lookup failed (non-fatal): ' + (err.message || err));
                   return;
@@ -638,7 +694,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
     _sendAbly(message, timeout) {
       if (timeout === undefined) timeout = 300000;
 
-      if (!this._cmdChannel || !this._ably) {
+      if (!this._sessionChannel || !this._ably) {
         return Promise.reject(
           new Error("Sandbox not connected (no Ably client)"),
         );
@@ -731,10 +787,10 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
           rejectPromise(
             new Error(
               "Sandbox message '" +
-                message.type +
-                "' timed out after " +
-                timeout +
-                "ms",
+              message.type +
+              "' timed out after " +
+              timeout +
+              "ms",
             ),
           );
         }
@@ -756,11 +812,10 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       };
 
       if (message.type === "output") {
-        p.catch(function () {});
+        p.catch(function () { });
       }
 
-      this._cmdChannel
-        .publish("command", message)
+      this._throttledPublish(this._sessionChannel, "command", message)
         .then(function () {
           emitter.emit(events.sandbox.sent, message);
         })
@@ -777,6 +832,53 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       return p;
     }
 
+    /**
+     * Throttled publish to stay under Ably's 50 msg/sec per-connection limit.
+     * Also tracks and logs the current publish rate for debugging.
+     * @param {Object} channel - Ably channel to publish on
+     * @param {string} eventName - Event name for the publish
+     * @param {Object} message - Message payload
+     * @returns {Promise} - Resolves when publish completes
+     */
+    async _throttledPublish(channel, eventName, message) {
+      var self = this;
+      var now = Date.now();
+
+      // Rate limiting: wait if too soon since last publish
+      var elapsed = now - this._publishLastTime;
+      if (elapsed < this._publishMinIntervalMs) {
+        var waitMs = this._publishMinIntervalMs - elapsed;
+        await new Promise(function (resolve) {
+          var timer = setTimeout(resolve, waitMs);
+          if (timer.unref) timer.unref();
+        });
+      }
+      this._publishLastTime = Date.now();
+
+      // Metrics: track messages per second
+      this._publishCount++;
+      var windowElapsed = Date.now() - this._publishWindowStart;
+      if (windowElapsed >= 1000) {
+        var rate = (this._publishCount / windowElapsed) * 1000;
+        var rateStr = rate.toFixed(1);
+
+        // Log rate - warning if approaching limit, debug otherwise
+        if (rate > 45) {
+          logger.warn("Ably publish rate: " + rateStr + " msg/sec (approaching 50/sec limit)");
+        } else if (process.env.VERBOSE || process.env.TD_DEBUG) {
+          logger.log("Ably publish rate: " + rateStr + " msg/sec");
+        }
+
+        // Reset window
+        this._publishCount = 0;
+        this._publishWindowStart = Date.now();
+      }
+
+      return channel.publish(eventName, message).then(function () {
+        logger.log(`[ably] Published: channel=${channel.name.split(':').pop()}, event=${eventName}, type=${message.type || 'unknown'} (requestId=${message.requestId || 'none'})`);
+      });
+    }
+
     async auth(apiKey) {
       this.apiKey = apiKey;
       var sessionId = this.sessionInstance
@@ -803,7 +905,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
           logger.log("Trace Report (Share When Reporting Bugs):");
           logger.log(
             "https://testdriver.sentry.io/explore/traces/trace/" +
-              reply.traceId,
+            reply.traceId,
           );
         }
 
@@ -846,7 +948,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       this._sandboxId = reply.sandboxId;
 
       if (reply.ably && reply.ably.token) {
-        await this._initAbly(reply.ably.token, reply.ably.channels);
+        await this._initAbly(reply.ably.token, reply.ably.channel);
       }
 
       this.setConnectionParams({
@@ -895,38 +997,43 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
         clearInterval(this.heartbeat);
         this.heartbeat = null;
       }
+      if (this._statsInterval) {
+        clearInterval(this._statsInterval);
+        this._statsInterval = null;
+      }
 
       // Send end-session control message to runner before disconnecting
-      if (this._ctrlChannel && this._ably?.connection?.state === 'connected') {
+      if (this._sessionChannel && this._ably?.connection?.state === 'connected') {
         try {
-          await this._ctrlChannel.publish('control', { type: 'end-session' });
+          logger.log('[ably] Publishing control: type=end-session');
+          await this._sessionChannel.publish('control', { type: 'end-session' });
         } catch (e) {
           // Ignore - best effort
         }
       }
 
-      // Leave presence on control channel
-      if (this._ctrlChannel) {
+      // Leave presence on session channel
+      if (this._sessionChannel) {
         try {
-          await this._ctrlChannel.presence.leave();
+          logger.log('[ably] Leaving presence on session channel');
+          await this._sessionChannel.presence.leave();
         } catch (e) {
           // ignore - best effort, Ably will auto-leave on disconnect
         }
       }
 
       try {
-        await Promise.allSettled([
-          this._cmdChannel?.detach(),
-          this._respChannel?.detach(),
-          this._ctrlChannel?.detach(),
-          this._filesChannel?.detach(),
-        ].filter(Boolean));
+        logger.log('[ably] Detaching session channel');
+        if (this._sessionChannel) {
+          await this._sessionChannel.detach();
+        }
       } catch (e) {
         /* ignore */
       }
 
       if (this._ably) {
         try {
+          logger.log('[ably] Closing Ably connection');
           this._ably.close();
         } catch (e) {
           /* ignore */
@@ -934,11 +1041,8 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
         this._ably = null;
       }
 
-      this._cmdChannel = null;
-      this._respChannel = null;
-      this._ctrlChannel = null;
-      this._filesChannel = null;
-      this._channelNames = null;
+      this._sessionChannel = null;
+      this._channelName = null;
       this.apiSocketConnected = false;
       this.instanceSocketConnected = false;
       this.authenticated = false;
@@ -961,11 +1065,51 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       const region = process.env.AWS_REGION || 'us-east-2';
 
       // Write SSM parameters to a temp file to avoid shell quoting issues
+      // Log key config details for debugging
+      logger.log('Agent config being provisioned:');
+      logger.log('  sandboxId: ' + agentConfig.sandboxId);
+      logger.log('  apiRoot: ' + agentConfig.apiRoot);
+      logger.log('  channel: ' + (agentConfig.ably?.channel || 'N/A'));
+      logger.log('  token length: ' + (agentConfig.ably?.token ? JSON.stringify(agentConfig.ably.token).length : 0));
+
       const paramsJson = JSON.stringify({
         commands: [
+          // Debug: show existing state
+          "Write-Host '=== Checking existing state ==='",
+          "$task = Get-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction SilentlyContinue",
+          "if ($task) { Write-Host \"Task exists, state: $($task.State)\" } else { Write-Host 'Task does NOT exist!' }",
+          "if (Test-Path 'C:\\Windows\\Temp\\testdriver-agent.json') { Write-Host 'Old config:'; Get-Content 'C:\\Windows\\Temp\\testdriver-agent.json' | Write-Host } else { Write-Host 'Config file does NOT exist yet' }",
+          // Stop any running runner
+          "Write-Host '=== Stopping runner ==='",
+          "Stop-Process -Name node -Force -ErrorAction SilentlyContinue",
+          "Stop-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction SilentlyContinue",
+          // Write config
+          "Write-Host '=== Writing config ==='",
           "$config = '" + configJson.replace(/'/g, "''") + "'",
           "[System.IO.File]::WriteAllText('C:\\Windows\\Temp\\testdriver-agent.json', $config)",
           "Write-Host 'Config written for sandbox " + agentConfig.sandboxId + "'",
+          // Show what was written (redact token)
+          "Write-Host '=== New config (token redacted) ==='",
+          "$cfg = Get-Content 'C:\\Windows\\Temp\\testdriver-agent.json' | ConvertFrom-Json",
+          "Write-Host \"sandboxId: $($cfg.sandboxId)\"",
+          "Write-Host \"apiRoot: $($cfg.apiRoot)\"",
+          "Write-Host \"channel: $($cfg.ably.channel)\"",
+          "Write-Host \"token type: $($cfg.ably.token.GetType().Name)\"",
+          // Start the runner
+          "Write-Host '=== Starting runner ==='",
+          "Start-Sleep -Seconds 1",
+          "Start-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction Stop",
+          "$task = Get-ScheduledTask -TaskName RunTestDriverAgent",
+          "Write-Host \"Task state after start: $($task.State)\"",
+          // Check if node process started
+          "Start-Sleep -Seconds 3",
+          "Write-Host '=== Checking runner process ==='",
+          "$procs = Get-Process -Name node -ErrorAction SilentlyContinue",
+          "if ($procs) { Write-Host \"Node processes: $($procs.Count)\"; $procs | ForEach-Object { Write-Host \"  PID: $($_.Id), StartTime: $($_.StartTime)\" } } else { Write-Host 'No node process found!' }",
+          // Check runner logs
+          "Write-Host '=== Runner log (last 30 lines) ==='",
+          "if (Test-Path 'C:\\testdriver\\logs\\sandbox-agent.log') { Get-Content 'C:\\testdriver\\logs\\sandbox-agent.log' -Tail 30 | Write-Host } else { Write-Host 'No log file found' }",
+          "Write-Host '=== Done ==='",
         ],
       });
       const tmpFile = join(tmpdir(), 'td-provision-' + Date.now() + '.json');
@@ -987,6 +1131,24 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
           '--command-id "' + cmdId + '" --instance-id "' + instanceId + '"',
           { encoding: 'utf-8', timeout: 60000 }
         );
+
+        // Get the command output for debugging
+        try {
+          const invocationOutput = execSync(
+            'aws ssm get-command-invocation --region "' + region + '" ' +
+            '--command-id "' + cmdId + '" --instance-id "' + instanceId + '" --output json',
+            { encoding: 'utf-8', timeout: 30000 }
+          );
+          const invocation = JSON.parse(invocationOutput);
+          if (invocation.StandardOutputContent) {
+            logger.log('SSM output:\n' + invocation.StandardOutputContent);
+          }
+          if (invocation.StandardErrorContent) {
+            logger.warn('SSM errors:\n' + invocation.StandardErrorContent);
+          }
+        } catch (e) {
+          logger.warn('Could not retrieve SSM command output: ' + e.message);
+        }
       } finally {
         try { unlinkSync(tmpFile); } catch (e) { /* ignore */ }
       }