testdriverai 7.4.4 → 7.5.0

package/agent/lib/sandbox.js

@@ -1,32 +1,93 @@
-const WebSocket = require("ws");
 const crypto = require("crypto");
+const Ably = require("ably");
 const { events } = require("../events");
 const logger = require("./logger");
 const { version } = require("../../package.json");
 
-/**
- * Generate Sentry trace headers for distributed tracing
- * Uses the same trace ID derivation as the API (MD5 hash of session ID)
- * @param {string} sessionId - The session ID
- * @returns {Object} Headers object with sentry-trace and baggage
- */
 function getSentryTraceHeaders(sessionId) {
   if (!sessionId) return {};
-
-  // Same logic as API: derive trace ID from session ID
   const traceId = crypto.createHash("md5").update(sessionId).digest("hex");
   const spanId = crypto.randomBytes(8).toString("hex");
-
   return {
-    "sentry-trace": `${traceId}-${spanId}-1`,
-    baggage: `sentry-trace_id=${traceId},sentry-sample_rate=1.0,sentry-sampled=true`,
+    "sentry-trace": traceId + "-" + spanId + "-1",
+    baggage:
+      "sentry-trace_id=" +
+      traceId +
+      ",sentry-sample_rate=1.0,sentry-sampled=true",
   };
 }
 
-const createSandbox = (emitter, analytics, sessionInstance) => {
+function httpPost(apiRoot, path, body, timeout) {
+  const http = require("http");
+  const https = require("https");
+  const url = new URL(apiRoot + path);
+  const transport = url.protocol === "https:" ? https : http;
+  const bodyStr = JSON.stringify(body);
+
+  return new Promise(function (resolve, reject) {
+    var timeoutId = timeout
+      ? setTimeout(function () {
+          req.destroy();
+          reject(
+            new Error("HTTP request timed out after " + timeout + "ms"),
+          );
+        }, timeout)
+      : null;
+
+    var req = transport.request(
+      url,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Content-Length": Buffer.byteLength(bodyStr),
+          "Connection": "close",
+        },
+      },
+      function (res) {
+        var data = "";
+        res.on("data", function (chunk) {
+          data += chunk;
+        });
+        res.on("end", function () {
+          if (timeoutId) clearTimeout(timeoutId);
+          try {
+            var parsed = JSON.parse(data);
+            if (res.statusCode >= 400) {
+              var err = new Error(
+                parsed.errorMessage ||
+                  parsed.message ||
+                  "HTTP " + res.statusCode,
+              );
+              err.responseData = parsed;
+              reject(err);
+            } else {
+              resolve(parsed);
+            }
+          } catch (e) {
+            reject(new Error("Failed to parse API response: " + data));
+          }
+        });
+      },
+    );
+    req.on("error", function (err) {
+      if (timeoutId) clearTimeout(timeoutId);
+      reject(err);
+    });
+    req.write(bodyStr);
+    req.end();
+  });
+}
+
+const createSandbox = function (emitter, analytics, sessionInstance) {
   class Sandbox {
     constructor() {
-      this.socket = null;
+      this._ably = null;
+      this._cmdChannel = null;
+      this._respChannel = null;
+      this._ctrlChannel = null;
+      this._filesChannel = null;
+      this._channelNames = null;
       this.ps = {};
       this.heartbeat = null;
       this.apiSocketConnected = false;
@@ -35,525 +96,739 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
       this.instance = null;
       this.messageId = 0;
       this.uniqueId = Math.random().toString(36).substring(7);
-      this.os = null; // Store OS value to send with every message
-      this.sessionInstance = sessionInstance; // Store session instance to include in messages
-      this.traceId = null; // Sentry trace ID for debugging
-      this.reconnectAttempts = 0;
-      this.maxReconnectAttempts = 10;
-      this.intentionalDisconnect = false;
+      this.os = null;
+      this.sessionInstance = sessionInstance;
+      this.traceId = null;
       this.apiRoot = null;
       this.apiKey = null;
-      this.reconnectTimer = null; // Track reconnect setTimeout
-      this.reconnecting = false; // Prevent duplicate reconnection attempts
-      this.pendingTimeouts = new Map(); // Track per-message timeouts
-      this.pendingRetryQueue = []; // Queue of requests to retry after reconnection
-      this._lastConnectParams = null; // Connection params for reconnection (per-instance, not shared)
+      this._lastConnectParams = null;
+      this._teamId = null;
+      this._sandboxId = null;
     }
 
-    /**
-     * Get the Sentry trace ID for this session
-     * Useful for debugging with customers - they can share this ID to look up their traces
-     * @returns {string|null} The trace ID or null if not authenticated
-     */
     getTraceId() {
       return this.traceId;
     }
 
-    /**
-     * Get the Sentry trace URL for this session
-     * @returns {string|null} The full Sentry trace URL or null if no trace ID
-     */
     getTraceUrl() {
       if (!this.traceId) return null;
-      return `https://testdriver.sentry.io/explore/traces/trace/${this.traceId}`;
+      return (
+        "https://testdriver.sentry.io/explore/traces/trace/" + this.traceId
+      );
     }
 
-    send(message, timeout = 300000) {
-      let resolvePromise;
-      let rejectPromise;
+    async _initAbly(ablyToken, channelNames) {
+      if (this._ably) {
+        try {
+          this._ably.close();
+        } catch (e) {
+          /* ignore */
+        }
+      }
+      this._channelNames = channelNames;
+      var self = this;
+
+      this._ably = new Ably.Realtime({
+        authCallback: function (tokenParams, callback) {
+          callback(null, ablyToken);
+        },
+        clientId: "sdk-" + this._sandboxId,
+        disconnectedRetryTimeout: 5000,   // retry reconnect every 5s (default 15s)
+        suspendedRetryTimeout: 15000,     // retry from suspended every 15s (default 30s)
+      });
 
-      // Check if socket exists and is actually open before sending
-      // This prevents sending to a closed connection (e.g., sandbox killed due to test failure)
-      if (this.socket && this.socket.readyState === WebSocket.OPEN) {
-        this.messageId++;
-        message.requestId = `${this.uniqueId}-${this.messageId}`;
+      await new Promise(function (resolve, reject) {
+        self._ably.connection.on("connected", resolve);
+        self._ably.connection.on("failed", function () {
+          reject(new Error("Ably connection failed"));
+        });
+        setTimeout(function () {
+          reject(new Error("Ably connection timeout"));
+        }, 30000);
+      });
 
-        // If os is set in the message, store it for future messages
-        if (message.os) {
-          this.os = message.os;
+      this._cmdChannel = this._ably.channels.get(channelNames.commands);
+      this._respChannel = this._ably.channels.get(channelNames.responses);
+      this._ctrlChannel = this._ably.channels.get(channelNames.control);
+      this._filesChannel = this._ably.channels.get(channelNames.files);
+
+      this._respChannel.subscribe("response", function (msg) {
+        var message = msg.data;
+        if (!message) return;
+
+        if (message.type === "sandbox.progress") {
+          emitter.emit(events.sandbox.progress, {
+            step: message.step,
+            message: message.message,
+          });
+          return;
         }
 
-        // Add os to every message if it's been set
-        if (this.os && !message.os) {
-          message.os = this.os;
+        if (
+          message.type === "before.file" ||
+          message.type === "after.file" ||
+          message.type === "screenshot.file"
+        ) {
+          emitter.emit(events.sandbox.file, message);
+          return;
         }
 
-        // Add session to every message if available (for interaction tracking)
-        if (this.sessionInstance && !message.session) {
-          const sessionId = this.sessionInstance.get();
-          if (sessionId) {
-            message.session = sessionId;
-          }
+        // Streaming exec output chunks — emit as events, don't resolve the pending promise
+        if (message.type === "exec.output") {
+          emitter.emit(events.exec.output, { chunk: message.chunk, requestId: message.requestId });
+          return;
         }
 
-        // Add sandboxId to every message if we have a connected sandbox
-        // This allows the API to reconnect if the connection was rerouted
-        // Don't inject IP addresses as sandboxId — only valid instance/sandbox IDs
-        if (this._lastConnectParams?.sandboxId && !message.sandboxId) { 
-          const id = this._lastConnectParams.sandboxId;
-          // Only inject if it looks like a valid ID (not an IP address)
-          if (id && !/^\d+\.\d+\.\d+\.\d+$/.test(id)) {
-            message.sandboxId = id;
+        // Runner debug logs — only received when debug mode is enabled
+        if (message.type === "runner.log") {
+          var logLevel = message.level || "info";
+          var logMsg = "[runner] " + (message.message || "");
+          if (logLevel === "error") {
+            logger.error(logMsg);
+          } else {
+            logger.log(logMsg);
           }
+          emitter.emit(events.runner.log, {
+            level: logLevel,
+            message: message.message,
+            timestamp: message.timestamp,
+          });
+          return;
         }
 
-        let p = new Promise((resolve, reject) => {
-          this.socket.send(JSON.stringify(message));
-          emitter.emit(events.sandbox.sent, message);
-          resolvePromise = resolve;
-          rejectPromise = reject;
-        });
-
-        const requestId = message.requestId;
-
-        // Set up timeout to prevent hanging requests
-        const timeoutId = setTimeout(() => {
-          this.pendingTimeouts.delete(requestId);
-          if (this.ps[requestId]) {
-            delete this.ps[requestId];
-            rejectPromise(
-              new Error(
-                `Sandbox message '${message.type}' timed out after ${timeout}ms`,
-              ),
+        if (!message.requestId || !self.ps[message.requestId]) {
+          var debugMode =
+            process.env.VERBOSE || process.env.DEBUG || process.env.TD_DEBUG;
+          if (debugMode) {
+            console.warn(
+              "No pending promise found for requestId:",
+              message.requestId,
             );
           }
-        }, timeout);
-        // Don't let pending timeouts prevent Node process from exiting
-        if (timeoutId.unref) {
-          timeoutId.unref();
+          return;
         }
 
-        // Track timeout so close() can clear it
-        this.pendingTimeouts.set(requestId, timeoutId);
-
-        this.ps[requestId] = {
-          promise: p,
-          resolve: (result) => {
-            clearTimeout(timeoutId);
-            this.pendingTimeouts.delete(requestId);
-            resolvePromise(result);
-          },
-          reject: (error) => {
-            clearTimeout(timeoutId);
-            this.pendingTimeouts.delete(requestId);
-            rejectPromise(error);
-          },
-          message,
-          startTime: Date.now(),
-        };
+        if (message.error) {
+          var pendingMessage =
+            self.ps[message.requestId] &&
+            self.ps[message.requestId].message;
+          if (!pendingMessage || pendingMessage.type !== "output") {
+            emitter.emit(events.error.sandbox, message.errorMessage);
+          }
+          var error = new Error(message.errorMessage || "Sandbox error");
+          error.responseData = message;
+          self.ps[message.requestId].reject(error);
+        } else {
+          emitter.emit(events.sandbox.received);
+          if (self.ps[message.requestId]) {
+            // Unwrap the result from the Ably response envelope
+            // The runner sends { requestId, type, result, success }
+            // But SDK commands expect just the result object
+            var resolvedValue = message.result !== undefined ? message.result : message;
+            self.ps[message.requestId].resolve(resolvedValue);
+          }
+        }
+        delete self.ps[message.requestId];
+      });
 
-        // Fire-and-forget message types: attach .catch() to prevent
-        // unhandled promise rejections if nobody awaits the result
-        const fireAndForgetTypes = ["output"];
-        if (fireAndForgetTypes.includes(message.type)) {
-          p.catch(() => {});
+      this._filesChannel.subscribe("response", function (msg) {
+        var message = msg.data;
+        if (!message) return;
+        if (message.requestId && self.ps[message.requestId]) {
+          emitter.emit(events.sandbox.received);
+          self.ps[message.requestId].resolve(message);
+          delete self.ps[message.requestId];
         }
+        emitter.emit(events.sandbox.file, message);
+      });
 
-        return p;
-      }
+      this.heartbeat = setInterval(function () {}, 5000);
+      if (this.heartbeat.unref) this.heartbeat.unref();
 
-      // Return a rejected promise if socket is not available or not open
-      // This can happen when the sandbox is killed (e.g., due to test failure)
-      const state = this.socket?.readyState;
-      const stateMap = {
-        [WebSocket.CONNECTING]: "connecting",
-        [WebSocket.CLOSING]: "closing",
-        [WebSocket.CLOSED]: "closed",
-      };
-      const stateDesc = stateMap[state] || "unavailable";
-      return Promise.reject(new Error(`Sandbox socket not connected (state: ${stateDesc})`));
-    }
+      this._ably.connection.on("disconnected", function () {
+        logger.log("Ably disconnected - will auto-reconnect");
+      });
 
-    async auth(apiKey) {
-      this.apiKey = apiKey;
-      let reply = await this.send({
-        type: "authenticate",
-        apiKey,
-        version,
+      this._ably.connection.on("connected", function () {
+        // Log reconnection so the user knows the blip was recovered
+        logger.log("Ably reconnected");
       });
 
-      if (reply.success) {
-        this.authenticated = true;
+      this._ably.connection.on("suspended", function () {
+        logger.warn("Ably suspended - connection lost for extended period, will keep retrying");
+      });
 
-        // Log and store the Sentry trace ID for debugging
-        if (reply.traceId) {
-          this.traceId = reply.traceId;
-          logger.log('');
-          logger.log(`🔗 Trace Report (Share When Reporting Bugs):`);
-          logger.log(`https://testdriver.sentry.io/explore/traces/trace/${reply.traceId}`);
-        }
+      this._ably.connection.on("failed", function () {
+        self.apiSocketConnected = false;
+        self.instanceSocketConnected = false;
+        emitter.emit(events.error.sandbox, "Ably connection failed");
+      });
+    }
 
-        emitter.emit(events.sandbox.authenticated, { traceId: reply.traceId });
-        return true;
+    send(message, timeout) {
+      if (timeout === undefined) timeout = 300000;
+      if (message.type === "create" || message.type === "direct") {
+        return this._sendHttp(message, timeout);
       }
+      return this._sendAbly(message, timeout);
     }
 
-    /**
-     * Set connection params for reconnection logic and sandboxId injection.
-     * Use this instead of directly assigning this._lastConnectParams from
-     * external code. Keeps the shape consistent and avoids stale state
-     * leaking across concurrent test runs.
-     * @param {Object|null} params
-     * @param {string} [params.type] - 'direct' for IP-based connections
-     * @param {string} [params.ip] - IP address for direct connections
-     * @param {string} [params.sandboxId] - Sandbox/instance ID
-     * @param {boolean} [params.persist] - Whether to persist the sandbox
-     * @param {number|null} [params.keepAlive] - Keep-alive TTL in ms
-     */
-    setConnectionParams(params) {
-      this._lastConnectParams = params ? { ...params } : null;
-    }
+    async _sendHttp(message, timeout) {
+      var sessionId = this.sessionInstance
+        ? this.sessionInstance.get()
+        : null;
+      var body = {
+        apiKey: this.apiKey,
+        version: version,
+        os: message.os || this.os,
+        session: sessionId,
+      };
 
-    async connect(sandboxId, persist = false, keepAlive = null) {
-      let reply = await this.send({
-        type: "connect",
-        persist,
-        sandboxId,
-        keepAlive,
-      });
+      if (message.type === "create") {
+        body.os = message.os || this.os || "linux";
+        body.resolution = message.resolution;
+        body.ci = message.ci;
+        if (message.ami) body.ami = message.ami;
+        if (message.instanceType) body.instanceType = message.instanceType;
+        if (message.keepAlive !== undefined) body.keepAlive = message.keepAlive;
+      }
 
-      if (reply.success) {
-        // Only store connection params after successful connection
-        // This prevents malformed sandboxId from being attached to subsequent messages
-        this.setConnectionParams({ sandboxId, persist, keepAlive });
-        this.instanceSocketConnected = true;
-        emitter.emit(events.sandbox.connected);
-        // Return the full reply (includes url and sandbox)
-        return reply;
-      } else {
-        // Clear any previous connection params on failure
-        this.setConnectionParams(null);
-        // Throw error to trigger fallback to creating new sandbox
-        throw new Error(reply.errorMessage || "Failed to connect to sandbox");
+      if (message.type === "direct") {
+        body.ip = message.ip;
+        body.resolution = message.resolution;
+        body.ci = message.ci;
+        if (message.instanceId) body.instanceId = message.instanceId;
       }
-    }
 
-    /**
-     * Reconnect to a direct IP-based sandbox after connection loss.
-     * Sends a 'direct' message instead of 'connect' to avoid the API
-     * treating the IP as an AWS instance ID.
-     */
-    async reconnectDirect(ip) {
-      let reply = await this.send({
-        type: "direct",
-        ip,
-      });
+      var reply = await httpPost(
+        this.apiRoot,
+        "/api/v7/sandbox/authenticate",
+        body,
+        timeout,
+      );
 
-      if (reply.success) {
+      if (!reply.success) {
+        var err = new Error(
+          reply.errorMessage || "Failed to allocate sandbox",
+        );
+        err.responseData = reply;
+        throw err;
+      }
+
+      this._sandboxId = reply.sandboxId;
+      this._teamId = reply.teamId;
+
+      if (reply.ably && reply.ably.token) {
+        await this._initAbly(reply.ably.token, reply.ably.channels);
         this.instanceSocketConnected = true;
-        emitter.emit(events.sandbox.connected);
-        return reply;
-      } else {
-        throw new Error(reply.errorMessage || "Failed to reconnect to direct sandbox");
+
+        // Tell the runner to enable debug log forwarding if debug mode is on
+        var debugMode =
+          process.env.VERBOSE || process.env.DEBUG || process.env.TD_DEBUG;
+        if (debugMode && this._ctrlChannel) {
+          this._ctrlChannel.publish("control", {
+            type: "debug",
+            enabled: true,
+          });
+        }
       }
-    }
 
-    async handleConnectionLoss() {
-      if (this.intentionalDisconnect) return;
+      if (message.type === "create") {
+        // E2B (Linux) sandboxes: the API proxies commands and returns a url directly.
+        // No runner agent involved — skip runner.ready wait.
+        if (reply.url) {
+          logger.log(`E2B sandbox ready — url=${reply.url}`);
+          return {
+            success: true,
+            sandbox: {
+              sandboxId: reply.sandboxId,
+              instanceId: reply.sandbox?.sandboxId || reply.sandboxId,
+              os: body.os || 'linux',
+              url: reply.url,
+            },
+          };
+        }
 
-      // Prevent duplicate reconnection attempts (both 'error' and 'close' fire)
-      if (this.reconnecting) return;
-      this.reconnecting = true;
+        const runnerIp = reply.runner && reply.runner.ip;
+        const noVncPort = reply.runner && reply.runner.noVncPort;
+        const runnerVncUrl = reply.runner && reply.runner.vncUrl;
+
+        logger.log(`Runner claimed — ip=${runnerIp || 'none'}, os=${reply.runner?.os || 'unknown'}, noVncPort=${noVncPort || 'not reported'}, vncUrl=${runnerVncUrl || 'not reported'}`);
+
+        // For cloud Windows sandboxes (no runner in reply), wait for the
+        // agent to signal readiness before sending commands.  Without this
+        // gate, commands published before the agent subscribes are lost.
+        var self = this;
+        if (!reply.runner && this._ctrlChannel) {
+          logger.log('Waiting for runner agent to signal readiness...');
+          var readyTimeout = 120000; // 120s — allows for EC2 boot + agent startup
+          await new Promise(function (resolve, reject) {
+            var resolved = false;
+            function finish(data) {
+              if (resolved) return;
+              resolved = true;
+              clearTimeout(timer);
+              self._ctrlChannel.unsubscribe('control', onCtrl);
+              // Update runner info if provided
+              if (data && data.os) reply.runner = reply.runner || {};
+              if (data && data.os && reply.runner) reply.runner.os = data.os;
+              if (data && data.ip && reply.runner) reply.runner.ip = data.ip;
+              logger.log('Runner agent ready (os=' + ((data && data.os) || 'unknown') + ')');
+              resolve();
+            }
 
-      // Remove listeners from the old socket to prevent "No pending promise found" warnings
-      // when late responses arrive on the dying connection
-      if (this.socket) {
-        try {
-          this.socket.removeAllListeners("message");
-        } catch (e) {
-          // Ignore errors removing listeners from closed socket
+            var timer = setTimeout(function () {
+              if (!resolved) {
+                resolved = true;
+                self._ctrlChannel.unsubscribe('control', onCtrl);
+                reject(new Error('Runner agent did not signal readiness within ' + readyTimeout + 'ms'));
+              }
+            }, readyTimeout);
+            if (timer.unref) timer.unref();
+
+            // Listen for live runner.ready messages
+            var onCtrl;
+            onCtrl = function (msg) {
+              var data = msg.data;
+              if (data && data.type === 'runner.ready') {
+                finish(data);
+              }
+            };
+            self._ctrlChannel.subscribe('control', onCtrl);
+
+            // Also check channel history in case runner.ready was published
+            // before we subscribed (race condition on fast-booting agents).
+            try {
+              self._ctrlChannel.history({ limit: 50 }, function (err, page) {
+                if (err) {
+                  logger.warn('History lookup failed (non-fatal): ' + (err.message || err));
+                  return;
+                }
+                if (page && page.items) {
+                  for (var i = 0; i < page.items.length; i++) {
+                    var item = page.items[i];
+                    if (item.name === 'control' && item.data && item.data.type === 'runner.ready') {
+                      logger.log('Found runner.ready in channel history');
+                      finish(item.data);
+                      return;
+                    }
+                  }
+                }
+              });
+            } catch (histErr) {
+              logger.warn('History call threw (non-fatal): ' + (histErr.message || histErr));
+            }
+          });
+        }
+        // Prefer the full vncUrl reported by the runner (infrastructure-agnostic).
+        // Fall back to constructing from ip + noVncPort for older runners.
+        let url;
+        if (runnerVncUrl) {
+          url = runnerVncUrl;
+          logger.log(`Using runner-provided vncUrl: ${url}`);
+        } else if (runnerIp && noVncPort) {
+          url = `http://${runnerIp}:${noVncPort}/vnc_lite.html`;
+          logger.log(`noVNC URL constructed from runner ip+port: ${url}`);
+        } else if (runnerIp) {
+          url = "http://" + runnerIp;
+          logger.warn(`Runner did not report noVNC port — using bare IP: ${url}`);
+        } else {
+          logger.warn('Runner has no IP — preview will not be available');
         }
+        return {
+          success: true,
+          sandbox: {
+            sandboxId: reply.sandboxId,
+            instanceId: reply.sandboxId,
+            os: reply.runner?.os || body.os,
+            ip: runnerIp,
+            url: url,
+            vncPort: noVncPort || undefined,
+            runner: reply.runner,
+          },
+        };
       }
 
-      // Queue pending requests for retry after reconnection
-      // (they were sent on the old socket and will never receive responses)
-      const pendingRequestIds = Object.keys(this.ps);
-      if (pendingRequestIds.length > 0) {
-        console.log(`[Sandbox] Queuing ${pendingRequestIds.length} pending request(s) for retry after reconnection`);
-        for (const requestId of pendingRequestIds) {
-          const pending = this.ps[requestId];
-          if (pending) {
-            // Clear the timeout - we'll set a new one when we retry
-            const timeoutId = this.pendingTimeouts.get(requestId);
-            if (timeoutId) {
-              clearTimeout(timeoutId);
-              this.pendingTimeouts.delete(requestId);
+      if (message.type === "direct") {
+        // If the API provisioned Ably credentials to the instance (reply.agent present),
+        // wait for the runner agent to signal readiness before sending commands.
+        // Without this gate, commands published before the agent subscribes are lost.
+        var self = this;
+        if (reply.agent && this._ctrlChannel) {
+          logger.log('Waiting for runner agent to signal readiness (direct connection)...');
+          var readyTimeout = 120000; // 120s — allows for SSM provisioning + agent startup
+          await new Promise(function (resolve, reject) {
+            var resolved = false;
+            function finish(data) {
+              if (resolved) return;
+              resolved = true;
+              clearTimeout(timer);
+              self._ctrlChannel.unsubscribe('control', onCtrl);
+              logger.log('Runner agent ready (direct, os=' + ((data && data.os) || 'unknown') + ')');
+              resolve();
             }
-            // Queue for retry (store message and promise handlers)
-            this.pendingRetryQueue.push({
-              message: pending.message,
-              resolve: pending.resolve,
-              reject: pending.reject,
-            });
-          }
+
+            var timer = setTimeout(function () {
+              if (!resolved) {
+                resolved = true;
+                self._ctrlChannel.unsubscribe('control', onCtrl);
+                reject(new Error('Runner agent did not signal readiness within ' + readyTimeout + 'ms (direct connection)'));
+              }
+            }, readyTimeout);
+            if (timer.unref) timer.unref();
+
+            // Listen for live runner.ready messages
+            var onCtrl;
+            onCtrl = function (msg) {
+              var data = msg.data;
+              if (data && data.type === 'runner.ready') {
+                finish(data);
+              }
+            };
+            self._ctrlChannel.subscribe('control', onCtrl);
+
+            // Also check channel history in case runner.ready was published
+            // before we subscribed (race condition on fast-booting agents).
+            try {
+              self._ctrlChannel.history({ limit: 50 }, function (err, page) {
+                if (err) {
+                  logger.warn('History lookup failed (non-fatal): ' + (err.message || err));
+                  return;
+                }
+                if (page && page.items) {
+                  for (var i = 0; i < page.items.length; i++) {
+                    var item = page.items[i];
+                    if (item.name === 'control' && item.data && item.data.type === 'runner.ready') {
+                      logger.log('Found runner.ready in channel history (direct)');
+                      finish(item.data);
+                      return;
+                    }
+                  }
+                }
+              });
+            } catch (histErr) {
+              logger.warn('History call threw (non-fatal): ' + (histErr.message || histErr));
+            }
+          });
         }
-        this.ps = {};
+
+        // Construct VNC URL — use port 8080 (nginx noVNC proxy) for Windows instances
+        var directUrl = message.ip ? "http://" + message.ip + ":8080/vnc_lite.html" : undefined;
+
+        return {
+          success: true,
+          instance: {
+            instanceId: reply.sandboxId,
+            sandboxId: reply.sandboxId,
+            ip: message.ip,
+            url: directUrl || "http://" + message.ip,
+          },
+        };
       }
 
-      // Cancel any existing reconnect timer
-      if (this.reconnectTimer) {
-        clearTimeout(this.reconnectTimer);
-        this.reconnectTimer = null;
+      return reply;
+    }
+
+    _sendAbly(message, timeout) {
+      if (timeout === undefined) timeout = 300000;
+
+      if (!this._cmdChannel || !this._ably) {
+        return Promise.reject(
+          new Error("Sandbox not connected (no Ably client)"),
+        );
       }
 
-      if (this.reconnectAttempts >= this.maxReconnectAttempts) {
-        const errorMsg =
-          "Unable to reconnect to TestDriver sandbox after multiple attempts. Please check your internet connection.";
-        emitter.emit(events.error.sandbox, errorMsg);
-        console.error(errorMsg);
-        
-        // Reject all queued requests since reconnection failed
-        if (this.pendingRetryQueue.length > 0) {
-          console.log(`[Sandbox] Rejecting ${this.pendingRetryQueue.length} queued request(s) - reconnection failed`);
-          for (const queued of this.pendingRetryQueue) {
-            queued.reject(new Error("Sandbox reconnection failed after multiple attempts"));
-          }
-          this.pendingRetryQueue = [];
+      // If temporarily disconnected, wait up to 30s for reconnection
+      // instead of failing immediately (dashcam uploads can cause brief blips)
+      var self = this;
+      var connState = this._ably.connection.state;
+      if (connState !== "connected") {
+        if (connState === "disconnected" || connState === "connecting" || connState === "suspended") {
+          logger.log("Ably is " + connState + ", waiting for reconnect before sending...");
+          var waitForConnect = new Promise(function (resolve, reject) {
+            var timer = setTimeout(function () {
+              self._ably.connection.off("connected", onConnected);
+              self._ably.connection.off("failed", onFailed);
+              reject(new Error("Sandbox not connected after waiting 30s (state: " + self._ably.connection.state + ")"));
+            }, 30000);
+            if (timer.unref) timer.unref();
+            function onConnected() {
+              clearTimeout(timer);
+              self._ably.connection.off("failed", onFailed);
+              resolve();
+            }
+            function onFailed() {
+              clearTimeout(timer);
+              self._ably.connection.off("connected", onConnected);
+              reject(new Error("Ably connection failed while waiting to send"));
+            }
+            self._ably.connection.once("connected", onConnected);
+            self._ably.connection.once("failed", onFailed);
+          });
+          return waitForConnect.then(function () {
+            return self._sendAbly(message, timeout);
+          });
         }
-        
-        this.reconnecting = false;
-        return;
+        return Promise.reject(
+          new Error("Sandbox not connected (state: " + connState + ")"),
+        );
       }
 
-      this.reconnectAttempts++;
-      const delay = Math.min(1000 * 2 ** (this.reconnectAttempts - 1), 60000);
+      this.messageId++;
+      message.requestId = this.uniqueId + "-" + this.messageId;
 
-      console.log(
-        `[Sandbox] Connection lost. Reconnecting in ${delay}ms... (Attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})`,
-      );
+      if (message.os) this.os = message.os;
+      if (this.os && !message.os) message.os = this.os;
 
-      this.reconnectTimer = setTimeout(async () => {
-        this.reconnectTimer = null;
-        try {
-          await this.boot(this.apiRoot);
-          if (this.apiKey) {
-            await this.auth(this.apiKey);
-          }
-          // Re-establish sandbox connection on the new API instance
-          // Without this, the new API instance has no connection.desktop
-          // and all Linux operations will fail with "sandbox not initialized"
-          if (this._lastConnectParams) {
-            if (this._lastConnectParams.type === 'direct') {
-              // Direct IP connections must reconnect via 'direct' message, not 'connect'
-              const { ip, persist, keepAlive } = this._lastConnectParams;
-              console.log(`[Sandbox] Re-establishing direct connection (${ip})...`);
-              await this.reconnectDirect(ip);
-            } else {
-              const { sandboxId, persist, keepAlive } = this._lastConnectParams;
-              console.log(`[Sandbox] Re-establishing sandbox connection (${sandboxId})...`);
-              await this.connect(sandboxId, persist, keepAlive);
-            }
-          }
-          console.log("[Sandbox] Reconnected successfully.");
-          
-          // Retry queued requests
-          await this._retryQueuedRequests();
-        } catch (e) {
-          // boot's close handler will trigger handleConnectionLoss again
-        } finally {
-          this.reconnecting = false;
+      if (this.sessionInstance && !message.session) {
+        var sessionId = this.sessionInstance.get();
+        if (sessionId) message.session = sessionId;
+      }
+
+      if (
+        this._lastConnectParams &&
+        this._lastConnectParams.sandboxId &&
+        !message.sandboxId
+      ) {
+        var id = this._lastConnectParams.sandboxId;
+        if (id && !/^\d+\.\d+\.\d+\.\d+$/.test(id)) {
+          message.sandboxId = id;
         }
-      }, delay);
-      // Don't let reconnect timer prevent Node process from exiting
-      if (this.reconnectTimer.unref) {
-        this.reconnectTimer.unref();
       }
-    }
 
-    /**
-     * Retry queued requests after successful reconnection
-     * @private
-     */
-    async _retryQueuedRequests() {
-      if (this.pendingRetryQueue.length === 0) return;
-
-      console.log(`[Sandbox] Retrying ${this.pendingRetryQueue.length} queued request(s)...`);
-      
-      // Take all queued requests and clear the queue
-      const toRetry = this.pendingRetryQueue.splice(0);
-      
-      for (const queued of toRetry) {
-        try {
-          // Re-send the message and resolve/reject the original promise
-          const result = await this.send(queued.message);
-          queued.resolve(result);
-        } catch (err) {
-          queued.reject(err);
+      // Attach Sentry distributed trace headers for runner-side tracing
+      var traceSessionId = this.sessionInstance
+        ? this.sessionInstance.get()
+        : message.session;
+      if (traceSessionId) {
+        var traceHeaders = getSentryTraceHeaders(traceSessionId);
+        if (traceHeaders["sentry-trace"]) {
+          message.sentryTrace = traceHeaders["sentry-trace"];
+          message.baggage = traceHeaders.baggage;
         }
       }
-      
-      console.log(`[Sandbox] Finished retrying queued requests.`);
-    }
 
-    async boot(apiRoot) {
-      if (apiRoot) this.apiRoot = apiRoot;
-      return new Promise((resolve, reject) => {
-        // Get session ID for Sentry trace headers
-        const sessionId = this.sessionInstance?.get();
+      var resolvePromise, rejectPromise;
+      var self = this;
+
+      var p = new Promise(function (resolve, reject) {
+        resolvePromise = resolve;
+        rejectPromise = reject;
+      });
 
-        if (!sessionId) {
-          console.warn(
-            "[Sandbox] No session ID available at boot time - Sentry tracing will not be available",
+      var requestId = message.requestId;
+
+      var timeoutId = setTimeout(function () {
+        if (self.ps[requestId]) {
+          delete self.ps[requestId];
+          rejectPromise(
+            new Error(
+              "Sandbox message '" +
+                message.type +
+                "' timed out after " +
+                timeout +
+                "ms",
+            ),
           );
         }
+      }, timeout);
+      if (timeoutId.unref) timeoutId.unref();
+
+      this.ps[requestId] = {
+        promise: p,
+        resolve: function (result) {
+          clearTimeout(timeoutId);
+          resolvePromise(result);
+        },
+        reject: function (error) {
+          clearTimeout(timeoutId);
+          rejectPromise(error);
+        },
+        message: message,
+        startTime: Date.now(),
+      };
 
-        const sentryHeaders = getSentryTraceHeaders(sessionId);
-
-        // Build WebSocket URL with Sentry trace headers as query params
-        const wsUrl = new URL(apiRoot.replace("https://", "wss://"));
-        if (sentryHeaders["sentry-trace"]) {
-          wsUrl.searchParams.set("sentry-trace", sentryHeaders["sentry-trace"]);
-        }
-        if (sentryHeaders["baggage"]) {
-          wsUrl.searchParams.set("baggage", sentryHeaders["baggage"]);
-        }
+      if (message.type === "output") {
+        p.catch(function () {});
+      }
 
-        this.socket = new WebSocket(wsUrl.toString());
-
-        // handle errors
-        this.socket.on("close", () => {
-          clearInterval(this.heartbeat);
-          this.apiSocketConnected = false;
-          // Also mark instance socket as disconnected to prevent sending messages
-          // to a closed connection (e.g., when sandbox is killed due to test failure)
-          this.instanceSocketConnected = false;
-          // Reset reconnecting flag so handleConnectionLoss can run for this new disconnection
-          this.reconnecting = false;
-          this.handleConnectionLoss();
-          reject();
+      this._cmdChannel
+        .publish("command", message)
+        .then(function () {
+          emitter.emit(events.sandbox.sent, message);
+        })
+        .catch(function (err) {
+          if (self.ps[requestId]) {
+            clearTimeout(timeoutId);
+            delete self.ps[requestId];
+            rejectPromise(
+              new Error("Failed to send message: " + err.message),
+            );
+          }
         });
 
-        this.socket.on("error", (err) => {
-          logger.log("Socket Error");
-          err && logger.log(err);
-          clearInterval(this.heartbeat);
-          emitter.emit(events.error.sandbox, err);
-          this.apiSocketConnected = false;
-          // Don't call handleConnectionLoss here - the 'close' event always fires
-          // after 'error', so let 'close' handle reconnection to avoid duplicate attempts
-          reject(err);
-        });
+      return p;
+    }
 
-        this.socket.on("open", async () => {
-          this.reconnectAttempts = 0;
-          this.reconnecting = false;
-          this.apiSocketConnected = true;
+    async auth(apiKey) {
+      this.apiKey = apiKey;
+      var sessionId = this.sessionInstance
+        ? this.sessionInstance.get()
+        : null;
+
+      var reply = await httpPost(
+        this.apiRoot,
+        "/api/v7/sandbox/authenticate",
+        {
+          apiKey: apiKey,
+          version: version,
+          session: sessionId,
+        },
+      );
 
-          this.heartbeat = setInterval(() => {
-            if (this.socket && this.socket.readyState === WebSocket.OPEN) {
-              this.socket.ping();
-            }
-          }, 5000);
-          // Don't let the heartbeat interval prevent Node process from exiting
-          if (this.heartbeat.unref) {
-            this.heartbeat.unref();
-          }
+      if (reply.success) {
+        this.authenticated = true;
+        this.apiSocketConnected = true;
+        this._teamId = reply.teamId;
 
-          resolve(this);
+        if (reply.traceId) {
+          this.traceId = reply.traceId;
+          logger.log("");
+          logger.log("Trace Report (Share When Reporting Bugs):");
+          logger.log(
+            "https://testdriver.sentry.io/explore/traces/trace/" +
+              reply.traceId,
+          );
+        }
+
+        emitter.emit(events.sandbox.authenticated, {
+          traceId: reply.traceId,
         });
+        return true;
+      }
 
-        this.socket.on("message", async (raw) => {
-          let message = JSON.parse(raw);
+      return false;
+    }
 
-          // Handle progress messages (no requestId needed)
-          if (message.type === "sandbox.progress") {
-            emitter.emit(events.sandbox.progress, {
-              step: message.step,
-              message: message.message,
-            });
-            return;
-          }
+    setConnectionParams(params) {
+      this._lastConnectParams = params ? Object.assign({}, params) : null;
+    }
 
-          if (!this.ps[message.requestId]) {
-            // This can happen during reconnection (ps was cleared) or after timeout
-            // (promise was deleted). Expected during polling loops (e.g. Chrome
-            // debugger readiness checks) where short-timeout exec calls regularly
-            // expire before the sandbox responds.  Only log in debug/verbose mode.
-            if (!this.reconnecting) {
-              const debugMode = process.env.VERBOSE || process.env.DEBUG || process.env.TD_DEBUG;
-              if (debugMode) {
-                console.warn(
-                  "No pending promise found for requestId:",
-                  message.requestId,
-                );
-              }
-            }
-            return;
-          }
+    async connect(sandboxId, persist, keepAlive) {
+      if (persist === undefined) persist = false;
+      if (keepAlive === undefined) keepAlive = null;
+      var sessionId = this.sessionInstance
+        ? this.sessionInstance.get()
+        : null;
+
+      var reply = await httpPost(
+        this.apiRoot,
+        "/api/v7/sandbox/authenticate",
+        {
+          apiKey: this.apiKey,
+          version: version,
+          sandboxId: sandboxId,
+          session: sessionId,
+          keepAlive: keepAlive || undefined,
+        },
+      );
 
-          if (message.error) {
-            // Don't emit error:sandbox for output (log forwarding) messages
-            // to prevent infinite loops: error → log → sendToSandbox → error → ...
-            const pendingMessage = this.ps[message.requestId]?.message;
-            if (pendingMessage?.type !== "output") {
-              emitter.emit(events.error.sandbox, message.errorMessage);
-            }
-            const error = new Error(message.errorMessage || "Sandbox error");
-            error.responseData = message;
-            this.ps[message.requestId].reject(error);
-          } else {
-            emitter.emit(events.sandbox.received);
-            this.ps[message.requestId]?.resolve(message);
-          }
-          delete this.ps[message.requestId];
-        });
+      if (!reply.success) {
+        this.setConnectionParams(null);
+        throw new Error(reply.errorMessage || "Failed to connect to sandbox");
+      }
+
+      this._sandboxId = reply.sandboxId;
+
+      if (reply.ably && reply.ably.token) {
+        await this._initAbly(reply.ably.token, reply.ably.channels);
+      }
+
+      this.setConnectionParams({
+        sandboxId: sandboxId,
+        persist: persist,
+        keepAlive: keepAlive,
       });
+      this.instanceSocketConnected = true;
+      emitter.emit(events.sandbox.connected);
+
+      // Prefer runner-provided vncUrl, fall back to ip+port, then bare IP
+      const reconnectRunner = reply.runner || {};
+      const reconnectVncUrl = reconnectRunner.vncUrl;
+      const reconnectNoVncPort = reconnectRunner.noVncPort;
+      const reconnectIp = reconnectRunner.ip;
+      let reconnectUrl;
+      if (reconnectVncUrl) {
+        reconnectUrl = reconnectVncUrl;
+      } else if (reconnectIp && reconnectNoVncPort) {
+        reconnectUrl = `http://${reconnectIp}:${reconnectNoVncPort}/vnc_lite.html`;
+      } else if (reconnectIp) {
+        reconnectUrl = "http://" + reconnectIp;
+      }
+
+      return {
+        success: true,
+        url: reconnectUrl,
+        sandbox: {
+          sandboxId: reply.sandboxId,
+          instanceId: reply.sandboxId,
+          os: reconnectRunner.os || undefined,
+          ip: reconnectIp || undefined,
+          url: reconnectUrl,
+          vncPort: reconnectNoVncPort || undefined,
+        },
+      };
     }
 
-    /**
-     * Close the WebSocket connection and clean up resources
-     */
-    close() {
-      this.intentionalDisconnect = true;
-      this.reconnecting = false;
-      // Cancel any pending reconnect timer
-      if (this.reconnectTimer) {
-        clearTimeout(this.reconnectTimer);
-        this.reconnectTimer = null;
-      }
+    async boot(apiRoot) {
+      if (apiRoot) this.apiRoot = apiRoot;
+      return this;
+    }
 
+    async close() {
       if (this.heartbeat) {
         clearInterval(this.heartbeat);
         this.heartbeat = null;
       }
 
-      // Clear all pending message timeouts to prevent timers keeping the process alive
-      for (const timeoutId of this.pendingTimeouts.values()) {
-        clearTimeout(timeoutId);
+      // Send end-session control message to runner before disconnecting
+      if (this._ctrlChannel && this._ably?.connection?.state === 'connected') {
+        try {
+          await this._ctrlChannel.publish('control', { type: 'end-session' });
+        } catch (e) {
+          // Ignore - best effort
+        }
+      }
+
+      try {
+        if (this._cmdChannel) this._cmdChannel.detach().catch(() => {});
+        if (this._respChannel) this._respChannel.detach().catch(() => {});
+        if (this._ctrlChannel) this._ctrlChannel.detach().catch(() => {});
+        if (this._filesChannel) this._filesChannel.detach().catch(() => {});
+      } catch (e) {
+        /* ignore */
       }
-      this.pendingTimeouts.clear();
 
-      if (this.socket) {
-        // Remove all listeners before closing to prevent reconnect attempts
-        this.socket.removeAllListeners();
+      if (this._ably) {
         try {
-          this.socket.close();
-        } catch (err) {
-          // Ignore close errors
+          this._ably.close();
+        } catch (e) {
+          /* ignore */
         }
-        this.socket = null;
+        this._ably = null;
       }
 
+      this._cmdChannel = null;
+      this._respChannel = null;
+      this._ctrlChannel = null;
+      this._filesChannel = null;
+      this._channelNames = null;
       this.apiSocketConnected = false;
       this.instanceSocketConnected = false;
       this.authenticated = false;
       this.instance = null;
       this._lastConnectParams = null;
-
-      // Silently clear pending promises and retry queue without rejecting
-      // (rejecting causes unhandled promise rejections during cleanup)
       this.ps = {};
-      this.pendingRetryQueue = [];
     }
   }