qualty 0.1.29 → 0.1.31

package/bin/local-runner.js

@@ -1,8 +1,12 @@
-import { mkdirSync, writeFileSync, mkdtempSync, existsSync, readFileSync, statSync } from "node:fs";
+import { mkdirSync, existsSync, readFileSync, statSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { homedir, tmpdir } from "node:os";
+import { homedir } from "node:os";
 import { createInterface } from "node:readline/promises";
 import process from "node:process";
+import { spawn } from "node:child_process";
+import { resolve4, resolve6 } from "node:dns/promises";
+import { randomBytes } from "node:crypto";
+import { request as httpsRequest } from "node:https";
 
 function safePathSegment(value, fallback) {
   const normalized = String(value || "").trim().replace(/[^a-zA-Z0-9._-]+/g, "_");
@@ -71,6 +75,194 @@ function withDevice(device, body = {}) {
   return { device, ...body };
 }
 
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function parseWsEndpoint(endpoint) {
+  const parsed = new URL(String(endpoint || "").replace(/^ws:/i, "http:").replace(/^wss:/i, "https:"));
+  return {
+    port: Number(parsed.port),
+    path: `${parsed.pathname || "/"}${parsed.search || ""}`,
+  };
+}
+
+function startCloudflaredForPort(token, port) {
+  const baseArgs = [
+    "tunnel",
+    "--no-autoupdate",
+    "--protocol",
+    "http2",
+    "--url",
+    `http://127.0.0.1:${port}`,
+    "run",
+    "--token",
+    token,
+  ];
+  let fallbackStarted = false;
+  const cmd = spawn("cloudflared", baseArgs, { stdio: "inherit" });
+  cmd.on("error", () => {
+    if (fallbackStarted) return;
+    fallbackStarted = true;
+    spawn("npx", ["cloudflared", ...baseArgs], { stdio: "inherit" });
+  });
+  return cmd;
+}
+
+async function resolvePublicHostname(hostname) {
+  const [ipv4, ipv6] = await Promise.allSettled([resolve4(hostname), resolve6(hostname)]);
+  const addresses = [
+    ...(ipv4.status === "fulfilled" ? ipv4.value.map((address) => ({ address, family: 4 })) : []),
+    ...(ipv6.status === "fulfilled" ? ipv6.value.map((address) => ({ address, family: 6 })) : []),
+  ].sort((a, b) => a.family - b.family);
+  if (addresses.length > 0) return addresses;
+
+  try {
+    const dohAddresses = await resolvePublicHostnameViaDoh(hostname);
+    if (dohAddresses.length > 0) return dohAddresses;
+  } catch {
+    /* Surface the original resolver error below; it is usually more familiar. */
+  }
+
+  const reason = ipv4.status === "rejected" ? ipv4.reason : ipv6.reason;
+  throw reason instanceof Error ? reason : new Error(String(reason || `Could not resolve ${hostname}`));
+}
+
+async function resolvePublicHostnameViaDoh(hostname) {
+  const queries = [
+    { type: "A", family: 4 },
+    { type: "AAAA", family: 6 },
+  ];
+  const results = await Promise.allSettled(
+    queries.map(async ({ type, family }) => {
+      const response = await fetch(
+        `https://cloudflare-dns.com/dns-query?name=${encodeURIComponent(hostname)}&type=${type}`,
+        { headers: { Accept: "application/dns-json" } }
+      );
+      if (!response.ok) {
+        throw new Error(`DoH ${type} returned HTTP ${response.status}`);
+      }
+      const body = await response.json();
+      return (Array.isArray(body.Answer) ? body.Answer : [])
+        .filter((answer) => String(answer?.data || "").trim())
+        .map((answer) => ({ address: String(answer.data).trim(), family }));
+    })
+  );
+  return results
+    .flatMap((result) => (result.status === "fulfilled" ? result.value : []))
+    .sort((a, b) => a.family - b.family);
+}
+
+function probeWebSocketUpgrade(connectUrl, addresses, timeoutMs = 10000) {
+  return new Promise((resolve, reject) => {
+    const url = new URL(connectUrl);
+    const req = httpsRequest({
+      protocol: url.protocol === "wss:" ? "https:" : url.protocol,
+      hostname: url.hostname,
+      port: url.port || 443,
+      path: `${url.pathname || "/"}${url.search || ""}`,
+      method: "GET",
+      headers: {
+        Connection: "Upgrade",
+        Upgrade: "websocket",
+        "Sec-WebSocket-Key": randomBytes(16).toString("base64"),
+        "Sec-WebSocket-Version": "13",
+      },
+      timeout: timeoutMs,
+      lookup: (_hostname, options, callback) => {
+        const cb = typeof options === "function" ? options : callback;
+        const lookupOptions = typeof options === "object" && options ? options : {};
+        const picked = addresses[0];
+        if (!picked?.address || !picked?.family) {
+          cb(new Error("No resolved tunnel addresses available"));
+          return;
+        }
+        if (lookupOptions.all) {
+          cb(null, addresses);
+          return;
+        }
+        cb(null, picked.address, picked.family);
+      },
+    });
+
+    req.on("upgrade", (_res, socket) => {
+      socket.destroy();
+      resolve();
+    });
+    req.on("response", (res) => {
+      res.resume();
+      reject(new Error(`WebSocket upgrade returned HTTP ${res.statusCode}`));
+    });
+    req.on("timeout", () => req.destroy(new Error(`WebSocket upgrade timed out after ${timeoutMs}ms`)));
+    req.on("error", reject);
+    req.end();
+  });
+}
+
+async function waitForPublicBrowserTunnel({ connectUrl, hostname, timeoutMs = 60000 }) {
+  const deadline = Date.now() + timeoutMs;
+  let attempts = 0;
+  let lastError = null;
+
+  while (Date.now() < deadline) {
+    attempts += 1;
+    try {
+      const addresses = await resolvePublicHostname(hostname);
+      // eslint-disable-next-line no-console
+      console.log(
+        `[qualty][tunnel] DNS ready for ${hostname}: ${addresses.map((a) => a.address).join(", ")}`
+      );
+
+      await probeWebSocketUpgrade(connectUrl, addresses);
+      // eslint-disable-next-line no-console
+      console.log(`[qualty][tunnel] WebSocket ready after ${attempts} attempt(s): ${hostname}`);
+      return;
+    } catch (err) {
+      lastError = err;
+      const message = String(err?.message || err);
+      // eslint-disable-next-line no-console
+      console.log(`[qualty][tunnel] Waiting for public browser tunnel (${attempts}): ${message}`);
+      await delay(Math.min(1000 + attempts * 500, 5000));
+    }
+  }
+
+  throw new Error(
+    `Public browser tunnel was not reachable within ${Math.round(timeoutMs / 1000)}s for ${hostname}: ${
+      lastError?.message || lastError || "unknown error"
+    }`
+  );
+}
+
+async function waitForCombinationDone({ apiRequest, apiUrl, token, orgId, executionId, device }) {
+  const deadline = Date.now() + 20 * 60 * 1000;
+  while (Date.now() < deadline) {
+    const status = await apiRequest({
+      apiUrl,
+      token,
+      orgId,
+      path: `/api/v1/status/${executionId}`,
+    });
+    const combos = Array.isArray(status?.combinations) ? status.combinations : [];
+    const combo = combos.find((c) => String(c?.device || "").trim() === String(device || "").trim());
+    if (combo && ["completed", "failed", "cancelled"].includes(String(combo.status || ""))) {
+      return {
+        success: combo.status === "completed" && combo.success !== false,
+        error: combo.status === "completed" ? null : combo.explanation || combo.status,
+        combinationsRemaining: combos.filter((c) => !["completed", "failed", "cancelled"].includes(String(c?.status || ""))).length,
+      };
+    }
+    if (["completed", "failed", "cancelled"].includes(String(status?.status || ""))) {
+      return {
+        success: status.status === "completed",
+        error: status.error || status.message || null,
+        combinationsRemaining: 0,
+      };
+    }
+    await delay(2000);
+  }
+  throw new Error(`Timed out waiting for local tunneled run ${executionId} (${device})`);
+}
+
 function localAuthStatePath({ orgId, projectId, profile }) {
   const orgSeg = safePathSegment(orgId || "default-org", "default-org");
   const projectSeg = safePathSegment(projectId || "default-project", "default-project");
@@ -237,46 +429,6 @@ export function logAuthStateDiagnostics(prefix, summary, details = {}) {
   }
 }
 
-export async function logAuthStateAfterInjection(tag, context, { testUrl = "" } = {}) {
-  try {
-    const cookies = await context.cookies();
-    // eslint-disable-next-line no-console
-    console.log(`${tag} injected into browser context: ${cookies.length} cookie(s) loaded`);
-    if (testUrl) {
-      // eslint-disable-next-line no-console
-      console.log(`${tag} localStorage applies after navigation to a matching origin (${testUrl})`);
-    }
-  } catch (err) {
-    // eslint-disable-next-line no-console
-    console.warn(`${tag} could not verify injected auth state: ${err?.message || err}`);
-  }
-}
-
-export async function logAuthStateAfterNavigation(tag, page) {
-  try {
-    const snapshot = await page.evaluate(() => ({
-      origin: window.location.origin,
-      href: window.location.href,
-      localStorageKeys: Object.keys(localStorage),
-      cookieCount: document.cookie ? document.cookie.split(";").filter(Boolean).length : 0,
-    }));
-    const keys = snapshot.localStorageKeys.length ? snapshot.localStorageKeys.join(", ") : "none";
-    // eslint-disable-next-line no-console
-    console.log(
-      `${tag} after navigation (${snapshot.origin}): localStorage keys [${keys}], document.cookie entries ~${snapshot.cookieCount}`
-    );
-    if (snapshot.localStorageKeys.length === 0 && snapshot.cookieCount === 0) {
-      // eslint-disable-next-line no-console
-      console.warn(
-        `${tag} warning: page has no localStorage keys and no cookies after navigation — app will likely appear logged out`
-      );
-    }
-  } catch (err) {
-    // eslint-disable-next-line no-console
-    console.warn(`${tag} could not read auth state after navigation: ${err?.message || err}`);
-  }
-}
-
 const localBindingSetupPromises = new Map();
 
 async function promptYesNo(question, defaultYes = true) {
@@ -530,755 +682,6 @@ async function preflightMissingLocalBindings({
   }
 }
 
-async function downloadProjectAttachments({ apiUrl, token, orgId, projectId, specs }) {
-  const labelToPath = {};
-  if (!projectId || !Array.isArray(specs) || specs.length === 0) return labelToPath;
-  const dir = mkdtempSync(join(tmpdir(), "qualty-attach-"));
-  const orgHeader = String(orgId || "").trim();
-  for (const spec of specs) {
-    const fileId = spec.id;
-    const label = spec.runner_key || spec.label;
-    if (!fileId || !label) continue;
-    const response = await fetch(
-      `${apiUrl}/api/v1/projects/${projectId}/upload-files/${encodeURIComponent(fileId)}`,
-      {
-        method: "GET",
-        headers: {
-          Authorization: `Bearer ${token}`,
-          ...(orgHeader ? { "X-Qualty-Org-Id": orgHeader } : {}),
-        },
-      }
-    );
-    if (!response.ok) continue;
-    const buf = Buffer.from(await response.arrayBuffer());
-    const name = spec.original_filename || `${label}.bin`;
-    const localPath = join(dir, name.replace(/[^a-zA-Z0-9._-]+/g, "_"));
-    writeFileSync(localPath, buf);
-    labelToPath[String(fileId)] = localPath;
-    labelToPath[label] = localPath;
-    if (spec.label && spec.label !== label) labelToPath[spec.label] = localPath;
-  }
-  return labelToPath;
-}
-
-function attachNetworkCollector(page) {
-  const events = [];
-  const onRequest = (req) => {
-    try {
-      events.push({
-        url: req.url(),
-        method: req.method(),
-        resource_type: req.resourceType(),
-        failed: false,
-      });
-    } catch {
-      /* ignore */
-    }
-  };
-  const onRequestFailed = (req) => {
-    try {
-      events.push({
-        url: req.url(),
-        method: req.method(),
-        resource_type: req.resourceType(),
-        failed: true,
-      });
-    } catch {
-      /* ignore */
-    }
-  };
-  page.on("request", onRequest);
-  page.on("requestfailed", onRequestFailed);
-  return {
-    snapshot() {
-      return { events_app_related: events.slice(-500) };
-    },
-  };
-}
-
-function resolveFileUploadArguments(action, labelToPath) {
-  if (!action || !Array.isArray(action.arguments)) return action;
-  const method = String(action.method || "").toLowerCase();
-  if (method !== "file_upload" && method !== "set_input_files") return action;
-  const resolved = action.arguments.map((arg) => {
-    const key = String(arg || "").trim();
-    if (labelToPath[key]) return labelToPath[key];
-    return key;
-  });
-  return { ...action, arguments: resolved };
-}
-
-export async function capturePageContext(page) {
-  try {
-    const text = await page.evaluate(() => {
-      const body = document.body;
-      return body ? body.innerText.slice(0, 12000) : "";
-    });
-    return { page_text: String(text || ""), interactive_summary: "" };
-  } catch {
-    return { page_text: "", interactive_summary: "" };
-  }
-}
-
-function _actLog(stepIndex, attempt, message) {
-  const stepLabel = Number.isFinite(stepIndex) ? stepIndex + 1 : stepIndex;
-  console.log(`[qualty][act][step ${stepLabel}][attempt ${attempt}] ${message}`);
-}
-
-function _summarizeAction(action) {
-  if (!action || typeof action !== "object") return "no-action";
-  const method = String(action.method || "").trim() || "unknown";
-  const selector = String(action.selector || "").trim();
-  return selector ? `${method} selector=${selector}` : method;
-}
-
-async function capturePageContextViaApi({
-  page,
-}) {
-  return capturePageContext(page);
-}
-
-async function _extractInteractiveCandidates(page, limit = 200) {
-  try {
-    const rows = await page.evaluate((maxRows) => {
-      const normalizeText = (v) => (v || "").replace(/\s+/g, " ").trim();
-      const getXPath = (element) => {
-        if (!element || !element.tagName) return "";
-        if (element.id) return `//*[@id="${element.id}"]`;
-        if (element === document.body) return "/html/body";
-        const parent = element.parentElement;
-        if (!parent) return "";
-        const tag = element.tagName.toLowerCase();
-        const sameTag = Array.from(parent.children).filter((c) => c.tagName === element.tagName);
-        const index = sameTag.length > 1 ? `[${sameTag.indexOf(element) + 1}]` : "";
-        const parentXPath = getXPath(parent);
-        return parentXPath ? `${parentXPath}/${tag}${index}` : `//${tag}${index}`;
-      };
-      const isVisible = (el) => {
-        if (!el) return false;
-        const style = window.getComputedStyle(el);
-        if (!style) return false;
-        if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return false;
-        const rect = el.getBoundingClientRect();
-        return rect.width > 0 && rect.height > 0;
-      };
-      const inferRole = (el) => {
-        const explicit = el.getAttribute("role");
-        if (explicit) return explicit;
-        const tag = (el.tagName || "").toLowerCase();
-        if (tag === "a") return "link";
-        if (tag === "button") return "button";
-        if (tag === "input") return "textbox";
-        if (tag === "select") return "listbox";
-        if (tag === "textarea") return "textbox";
-        return null;
-      };
-      const getTestId = (el) =>
-        el.getAttribute("data-testid") ||
-        el.getAttribute("data-test-id") ||
-        el.getAttribute("data-test") ||
-        el.getAttribute("data-qa") ||
-        null;
-
-      const selector =
-        "a,button,input,select,textarea,[role],[tabindex],[onclick],[contenteditable='true']";
-      const nodes = Array.from(document.querySelectorAll(selector));
-      const seen = new Set();
-
-      const out = [];
-      for (const el of nodes) {
-        if (seen.has(el) || !isVisible(el) || out.length >= maxRows) continue;
-        seen.add(el);
-        if (!el || out.length >= maxRows) break;
-        const xpath = getXPath(el);
-        if (!xpath) continue;
-        const rect = el.getBoundingClientRect();
-        const tag = (el.tagName || "").toLowerCase();
-        const role = inferRole(el);
-        const text = normalizeText(el.innerText || el.textContent || "").slice(0, 120) || null;
-        const className =
-          typeof el.className === "string" ? normalizeText(el.className).slice(0, 120) : null;
-        const bbox = {
-          x: Math.round(rect.x),
-          y: Math.round(rect.y),
-          width: Math.round(rect.width),
-          height: Math.round(rect.height),
-          right: Math.round(rect.right),
-          bottom: Math.round(rect.bottom),
-        };
-        const midpoint = {
-          x: Math.round(rect.x + rect.width / 2),
-          y: Math.round(rect.y + rect.height / 2),
-        };
-        out.push({
-          selector: xpath,
-          id: el.id || null,
-          tag: tag || null,
-          role: role || null,
-          text,
-          type: el.getAttribute("type") || null,
-          title: el.getAttribute("title") || null,
-          test_id: getTestId(el),
-          name_attr: el.getAttribute("name") || null,
-          aria_label: el.getAttribute("aria-label") || null,
-          placeholder: el.getAttribute("placeholder") || null,
-          href: el.getAttribute("href") || null,
-          class_name: className,
-          bbox,
-          midpoint,
-          target_coordinate: `${midpoint.x},${midpoint.y}`,
-        });
-      }
-      return out.slice(0, 400);
-    }, Math.max(1, Math.min(500, Number(limit) || 200)));
-    return Array.isArray(rows) ? rows : [];
-  } catch {
-    return [];
-  }
-}
-
-/**
- * Cloud ActHandler parity: attempt 1 (cache / bundle resolve) → 2 (selector LLM) → 3 (heal).
- */
-async function runActStep({
-  apiRequest,
-  apiUrl,
-  token,
-  orgId,
-  executionId,
-  device,
-  stepIndex,
-  page,
-  credentials,
-  labelToPath,
-  step,
-}) {
-  const attemptHistory = [];
-  const excluded = [];
-  let topCandidatesCache = null;
-  let lastErr = null;
-  let action = null;
-  const maxAttempts = 3;
-  let executionSource = null;
-
-  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
-    _actLog(stepIndex, attempt, "observe: requesting plan");
-    let ctx = await capturePageContextViaApi({
-      page,
-    });
-
-    const observeRequest = async (extraBody = null) =>
-      apiRequest({
-        apiUrl,
-        token,
-        orgId,
-        path: `/api/v1/cli/executions/${executionId}/observe`,
-        method: "POST",
-        body: withDevice(device, {
-          step_index: stepIndex,
-          attempt,
-          page_text: ctx.page_text,
-          interactive_summary: ctx.interactive_summary,
-          attempt_history: attemptHistory,
-          excluded_selectors: excluded,
-          top_candidates_cache: topCandidatesCache,
-          ...(extraBody && typeof extraBody === "object" ? extraBody : {}),
-        }),
-      });
-
-    let observe = await observeRequest();
-
-    const status = String(observe.status || "");
-    const source = String(observe.resolution_source || "").trim();
-    _actLog(
-      stepIndex,
-      attempt,
-      `observe: status=${status || "unknown"}${source ? ` source=${source}` : ""}`
-    );
-    if (Array.isArray(observe.top_candidates_cache)) {
-      topCandidatesCache = observe.top_candidates_cache;
-    }
-    if (observe.step_kind === "done" || status === "done") {
-      break;
-    }
-    if (observe.step_kind === "agent" || observe.step_kind === "login" || status === "agent") {
-      throw new Error(observe.error || `${observe.step_kind} steps require agent path`);
-    }
-    if (observe.step_kind === "check" || status === "check") {
-      throw new Error(observe.error || "Check steps are not supported in local CLI yet");
-    }
-    if (status === "heal") {
-      _actLog(stepIndex, attempt, "escalating to agent heal");
-      const healResult = await runAgentStep({
-        apiRequest,
-        apiUrl,
-        token,
-        orgId,
-        executionId,
-        device,
-        stepIndex,
-        instruction: String(observe.remainder_instruction || ""),
-        page,
-        credentials,
-        labelToPath,
-        instructionOverride: true,
-      });
-      return {
-        ok: healResult.ok,
-        error: healResult.error,
-        action: { method: "agent_heal", description: observe.remainder_instruction || "" },
-        skipRemainingSteps: healResult.ok,
-        source: "agent_heal",
-      };
-    }
-    if (status === "retry") {
-      _actLog(stepIndex, attempt, `retry: ${observe.error || "planner requested retry"}`);
-      attemptHistory.push({ attempt, error: observe.error || "retry" });
-      continue;
-    }
-    if (status === "needs_bundle_resolve") {
-      const collectedCandidates = await _extractInteractiveCandidates(page, 200);
-      _actLog(
-        stepIndex,
-        attempt,
-        `needs_bundle_resolve: collected ${collectedCandidates.length} candidates, retrying observe`
-      );
-      observe = await observeRequest({
-        collected_candidates: collectedCandidates,
-        collector_version: "candidate_js_v1",
-      });
-      if (Array.isArray(observe.top_candidates_cache)) {
-        topCandidatesCache = observe.top_candidates_cache;
-      }
-      const retryStatus = String(observe.status || "");
-      const retrySource = String(observe.resolution_source || "").trim();
-      _actLog(
-        stepIndex,
-        attempt,
-        `observe(after candidates): status=${retryStatus || "unknown"}${
-          retrySource ? ` source=${retrySource}` : ""
-        }`
-      );
-      if (retryStatus === "action" && observe.action) {
-        action = resolveFileUploadArguments(observe.action, labelToPath);
-        executionSource = retrySource || source || null;
-        _actLog(stepIndex, attempt, `execute: ${_summarizeAction(action)} source=${executionSource || "unknown"}`);
-        try {
-          await executeAction(page, action, credentials);
-          _actLog(stepIndex, attempt, "execute: success");
-          lastErr = null;
-          break;
-        } catch (err) {
-          lastErr = err.message || String(err);
-          _actLog(stepIndex, attempt, `execute: failed error=${lastErr}`);
-          attemptHistory.push({
-            attempt,
-            method: action.method,
-            selector: action.selector,
-            error: lastErr,
-          });
-          if (action.selector) excluded.push(action.selector);
-          continue;
-        }
-      }
-      if (retryStatus === "retry") {
-        _actLog(stepIndex, attempt, `retry(after candidates): ${observe.error || "planner requested retry"}`);
-        attemptHistory.push({ attempt, error: observe.error || "retry" });
-        continue;
-      }
-      if (retryStatus === "heal") {
-        _actLog(stepIndex, attempt, "escalating to agent heal");
-        const healResult = await runAgentStep({
-          apiRequest,
-          apiUrl,
-          token,
-          orgId,
-          executionId,
-          device,
-          stepIndex,
-          instruction: String(observe.remainder_instruction || ""),
-          page,
-          credentials,
-          labelToPath,
-          instructionOverride: true,
-        });
-        return {
-          ok: healResult.ok,
-          error: healResult.error,
-          action: { method: "agent_heal", description: observe.remainder_instruction || "" },
-          skipRemainingSteps: healResult.ok,
-          source: "agent_heal",
-        };
-      }
-      lastErr = observe.error || "No action returned after bundle resolve";
-      _actLog(stepIndex, attempt, `retry(after candidates): ${lastErr}`);
-      attemptHistory.push({ attempt, error: lastErr });
-      continue;
-    }
-    if (status === "action" && observe.action) {
-      action = resolveFileUploadArguments(observe.action, labelToPath);
-      executionSource = source || null;
-      _actLog(stepIndex, attempt, `execute: ${_summarizeAction(action)} source=${executionSource || "unknown"}`);
-      try {
-        await executeAction(page, action, credentials);
-        _actLog(stepIndex, attempt, "execute: success");
-        lastErr = null;
-        break;
-      } catch (err) {
-        lastErr = err.message || String(err);
-        _actLog(stepIndex, attempt, `execute: failed error=${lastErr}`);
-        attemptHistory.push({
-          attempt,
-          method: action.method,
-          selector: action.selector,
-          error: lastErr,
-        });
-        if (action.selector) excluded.push(action.selector);
-        continue;
-      }
-    }
-    lastErr = observe.error || "No action returned";
-    _actLog(stepIndex, attempt, `retry: ${lastErr}`);
-    attemptHistory.push({ attempt, error: lastErr });
-  }
-
-  return {
-    ok: !lastErr,
-    error: lastErr,
-    action,
-    skipRemainingSteps: false,
-    source: executionSource,
-  };
-}
-
-async function clickAtCoordinate(page, coord, button = "left") {
-  const [x, y] = coord.split(",").map((v) => Number(v.trim()));
-  if (!Number.isFinite(x) || !Number.isFinite(y)) {
-    throw new Error(`Invalid target_coordinate: ${coord}`);
-  }
-  if (button === "right") {
-    await page.mouse.click(x, y, { button: "right" });
-  } else {
-    await page.mouse.click(x, y);
-  }
-}
-
-async function resolveFileInputAtCoordinate(page, x, y) {
-  return page.evaluate(
-    ({ px, py }) => {
-      const el = document.elementFromPoint(px, py);
-      if (!el) return null;
-      const walk = (node) => {
-        if (!node) return null;
-        if (node.tagName && String(node.tagName).toLowerCase() === "input") {
-          const t = (node.getAttribute("type") || "").toLowerCase();
-          if (t === "file") return node;
-        }
-        const label = node.closest && node.closest("label");
-        if (label && label.control && label.control.type === "file") return label.control;
-        return node.parentElement ? walk(node.parentElement) : null;
-      };
-      const found = walk(el);
-      if (!found) return null;
-      if (found.id) return `#${found.id}`;
-      return null;
-    },
-    { px: x, py: y }
-  );
-}
-
-export async function executeAction(page, action, credentials = null) {
-  const method = String(action.method || "click").toLowerCase();
-  const selector = String(action.selector || "").trim();
-  const args = Array.isArray(action.arguments) ? action.arguments.map(String) : [];
-  const coord = String(action.target_coordinate || "").trim();
-
-  if (method === "wait") {
-    const ms = Math.max(0, Number(args[0] || 500));
-    await page.waitForTimeout(ms);
-    return;
-  }
-
-  if (method === "scrollto" && coord) {
-    const [x, y] = coord.split(",").map((v) => Number(v.trim()));
-    await page.evaluate(
-      ({ x, y }) => window.scrollTo(Number(x) || 0, Number(y) || 0),
-      { x, y }
-    );
-    return;
-  }
-
-  if (method === "scroll") {
-    const [x, y] = coord
-      ? coord.split(",").map((v) => Number(v.trim()))
-      : [null, null];
-    const deltaX = Number(args[0] || 0);
-    const deltaY = Number(args[1] || 700);
-    if (Number.isFinite(x) && Number.isFinite(y)) {
-      await page.mouse.move(x, y);
-    }
-    await page.mouse.wheel(deltaX, deltaY);
-    return;
-  }
-
-  if (method === "focused_keypress") {
-    for (const key of args) {
-      if (key) await page.keyboard.press(key);
-    }
-    return;
-  }
-
-  if (method === "fill_sensitive_field" && credentials) {
-    const fieldType = String(action.field_type || "").toLowerCase();
-    const value =
-      fieldType === "password"
-        ? credentials.password || ""
-        : credentials.username || "";
-    if (!value) throw new Error(`No ${fieldType} credential available`);
-    if (coord) await clickAtCoordinate(page, coord);
-    await page.waitForTimeout(200);
-    const isMac = process.platform === "darwin";
-    const selectAll = isMac ? "Meta+A" : "Control+A";
-    await page.keyboard.press(selectAll);
-    await page.keyboard.press("Backspace");
-    await page.keyboard.type(value, { delay: 40 });
-    if (action.press_enter) await page.keyboard.press("Enter");
-    return;
-  }
-
-  const coordinateMethods = new Set([
-    "click",
-    "dblclick",
-    "rightclick",
-    "hover",
-    "dragdrop",
-  ]);
-  if (coord && coordinateMethods.has(method)) {
-    if (method === "click") {
-      await clickAtCoordinate(page, coord, "left");
-      return;
-    }
-    if (method === "rightclick") {
-      await clickAtCoordinate(page, coord, "right");
-      return;
-    }
-    if (method === "dblclick") {
-      const [x, y] = coord.split(",").map((v) => Number(v.trim()));
-      await page.mouse.dblclick(x, y);
-      return;
-    }
-    if (method === "hover") {
-      const [x, y] = coord.split(",").map((v) => Number(v.trim()));
-      await page.mouse.move(x, y);
-      return;
-    }
-    if (method === "dragdrop") {
-      const [sx, sy] = coord.split(",").map((v) => Number(v.trim()));
-      const ex = Number(args[0]);
-      const ey = Number(args[1]);
-      await page.mouse.move(sx, sy);
-      await page.mouse.down();
-      await page.mouse.move(ex, ey);
-      await page.mouse.up();
-      return;
-    }
-  }
-
-  if (method === "file_upload" && coord && !selector) {
-    const [x, y] = coord.split(",").map((v) => Number(v.trim()));
-    const sel = await resolveFileInputAtCoordinate(page, x, y);
-    if (!sel) throw new Error("file_upload: no file input at coordinate");
-    await page.locator(sel).first().setInputFiles(args, { timeout: 30000 });
-    return;
-  }
-
-  if (method === "type" && !selector && coord) {
-    await clickAtCoordinate(page, coord);
-    await page.waitForTimeout(200);
-    await page.keyboard.type(args[0] || "", { delay: 40 });
-    return;
-  }
-
-  if (method === "type" && !selector && !coord) {
-    await page.keyboard.type(args[0] || "", { delay: 40 });
-    return;
-  }
-
-  if (!selector && method !== "focused_keypress") {
-    throw new Error(`Action ${method} requires a selector or target_coordinate`);
-  }
-
-  const locator = page.locator(selector).first();
-
-  switch (method) {
-    case "click":
-      await locator.click({ timeout: 15000 });
-      break;
-    case "dblclick":
-      await locator.dblclick({ timeout: 15000 });
-      break;
-    case "rightclick":
-      await locator.click({ button: "right", timeout: 15000 });
-      break;
-    case "hover":
-      await locator.hover({ timeout: 15000 });
-      break;
-    case "fill":
-    case "type":
-      await locator.fill(args[0] || "", { timeout: 15000 });
-      break;
-    case "press":
-      for (const key of args) {
-        if (key) await locator.press(key, { timeout: 15000 });
-      }
-      break;
-    case "file_upload":
-    case "set_input_files":
-      await locator.setInputFiles(args, { timeout: 30000 });
-      break;
-    case "scroll":
-      await locator.scrollIntoViewIfNeeded({ timeout: 15000 });
-      break;
-    case "scrollto":
-      await locator.scrollIntoViewIfNeeded({ timeout: 15000 });
-      break;
-    case "dragdrop": {
-      const box = await locator.boundingBox();
-      if (!box) throw new Error("dragdrop: element not visible");
-      const sx = box.x + box.width / 2;
-      const sy = box.y + box.height / 2;
-      const ex = Number(args[0]);
-      const ey = Number(args[1]);
-      await page.mouse.move(sx, sy);
-      await page.mouse.down();
-      await page.mouse.move(ex, ey);
-      await page.mouse.up();
-      break;
-    }
-    default:
-      throw new Error(`Unsupported action method: ${method}`);
-  }
-}
-
-async function runAgentStep({
-  apiRequest,
-  apiUrl,
-  token,
-  orgId,
-  executionId,
-  device,
-  stepIndex,
-  instruction,
-  page,
-  credentials,
-  labelToPath,
-  maxTurns = 100,
-  instructionOverride = false,
-}) {
-  let reset = true;
-  let lastErr = null;
-  let turnCount = 0;
-  const pendingFunctionResults = [];
-
-  while (turnCount < maxTurns) {
-    const screenshotB64 = (await page.screenshot({ type: "png" })).toString("base64");
-    const turnBody = withDevice(device, {
-      step_index: stepIndex,
-      screenshot_b64: screenshotB64,
-      page_url: page.url(),
-      reset,
-      function_results: pendingFunctionResults.length ? pendingFunctionResults : undefined,
-    });
-    if (instructionOverride && instruction) {
-      turnBody.instruction_override = instruction;
-    }
-    const turn = await apiRequest({
-      apiUrl,
-      token,
-      orgId,
-      path: `/api/v1/cli/executions/${executionId}/agent-turn`,
-      method: "POST",
-      body: turnBody,
-    });
-    reset = false;
-    pendingFunctionResults.length = 0;
-    turnCount += 1;
-
-    if (turn.status === "error") {
-      lastErr = turn.error || "Agent turn failed";
-      break;
-    }
-    if (turn.status === "done") {
-      lastErr = null;
-      break;
-    }
-    if (turn.status === "continue") {
-      continue;
-    }
-    if (turn.status !== "actions" || !Array.isArray(turn.actions) || !turn.actions.length) {
-      lastErr = turn.error || "Agent returned no actions";
-      break;
-    }
-
-    for (const raw of turn.actions) {
-      const action = resolveFileUploadArguments(raw, labelToPath);
-      let actionOk = true;
-      let actionErr = null;
-      try {
-        if (action.method === "fill_sensitive_field") {
-          await executeAction(page, action, credentials);
-          const fnId = action._function_call_id;
-          if (fnId) {
-            pendingFunctionResults.push({
-              call_id: fnId,
-              success: true,
-              message: "fill_sensitive_field executed locally",
-            });
-          }
-        } else {
-          await executeAction(page, action, credentials);
-        }
-        await page.waitForTimeout(200);
-      } catch (err) {
-        actionOk = false;
-        actionErr = err.message || String(err);
-        lastErr = actionErr;
-      }
-
-      const afterB64 = (await page.screenshot({ type: "png" })).toString("base64");
-      try {
-        await apiRequest({
-          apiUrl,
-          token,
-          orgId,
-          path: `/api/v1/cli/executions/${executionId}/agent-segment`,
-          method: "POST",
-          body: withDevice(device, {
-            step_index: stepIndex,
-            turn: turnCount,
-            turn_thought: turn.turn_thought || "",
-            action,
-            executor_ok: actionOk,
-            error: actionErr || undefined,
-            screenshot_b64: afterB64,
-            page_url: page.url(),
-          }),
-        });
-      } catch (segErr) {
-        console.warn("[CLI] agent-segment upload failed:", segErr.message || segErr);
-      }
-
-      if (!actionOk) break;
-    }
-    if (lastErr) break;
-  }
-
-  if (!lastErr && turnCount >= maxTurns) {
-    lastErr = `Agent exceeded ${maxTurns} turns`;
-  }
-  return { ok: !lastErr, error: lastErr, turns: turnCount };
-}
-
 export async function runLocalExecution({
   apiRequest,
   apiUrl,
@@ -1359,9 +762,6 @@ async function runLocalSingleCombination({
   claim.url = url;
   // eslint-disable-next-line no-console
   console.log(`[qualty] Local run ${executionId} device=${effectiveDevice}`);
-  const viewport = claim.viewport || { width: 1440, height: 900 };
-  const steps = Array.isArray(claim.steps) ? claim.steps : [];
-  const totalSteps = claim.total_steps || steps.length;
   const claimSavedLogin = claim.saved_login_profile && typeof claim.saved_login_profile === "object"
     ? claim.saved_login_profile
     : null;
@@ -1443,237 +843,58 @@ async function runLocalSingleCombination({
     host: "127.0.0.1",
   });
   const cdpEndpoint = browserServer.wsEndpoint();
-  const browser = await chromium.connect(cdpEndpoint);
-  const contextOptions = {
-    viewport: { width: viewport.width || 1440, height: viewport.height || 900 },
-  };
-  if (finalStoragePath) {
-    contextOptions.storageState = finalStoragePath;
-  }
-  const context = await browser.newContext(contextOptions);
-  if (finalStoragePath) {
-    await logAuthStateAfterInjection("[qualty][auth]", context, { testUrl: url });
-  } else if (!noAuthState && (effectiveProfile || claimSavedLogin?.name)) {
-    // eslint-disable-next-line no-console
-    console.log("[qualty][auth] running without saved login state injection");
-  }
-  const page = await context.newPage();
-  const networkCollector = attachNetworkCollector(page);
-  const attachmentSpecs = Array.isArray(claim.file_attachments) ? claim.file_attachments : [];
-  const labelToPath = await downloadProjectAttachments({
-    apiUrl,
-    token,
-    orgId,
-    projectId: claim.project_id,
-    specs: attachmentSpecs,
-  });
-  const missingAttachmentLabels = attachmentSpecs
-    .map((spec) => ({
-      id: String(spec?.id || "").trim(),
-      label: String(spec?.runner_key || spec?.label || "").trim(),
-      rawLabel: String(spec?.label || "").trim(),
-    }))
-    .filter((it) => {
-      if (it.id && labelToPath[it.id]) return false;
-      if (it.label && labelToPath[it.label]) return false;
-      if (it.rawLabel && labelToPath[it.rawLabel]) return false;
-      return true;
-    });
-  if (missingAttachmentLabels.length > 0) {
-    const listed = missingAttachmentLabels
-      .map((it) => it.rawLabel || it.label || it.id || "<unknown>")
-      .join(", ");
-    throw new Error(
-      `Attached file download failed for: ${listed}. Re-attach files in dashboard and retry local run.`
-    );
-  }
-  const credentials = claim.credentials || null;
-
-  const stepsJson = [];
-  let runSuccess = true;
-  let runError = null;
-  const started = Date.now();
-  let lastPrevScreenshotB64 = null;
-  let lastCurrScreenshotB64 = null;
-  let firstPrevScreenshotB64 = null;
 
+  let cloudflared = null;
   try {
-    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60000 });
-    await page.waitForTimeout(500);
-    if (finalStoragePath) {
-      await logAuthStateAfterNavigation("[qualty][auth]", page);
-    }
-
-    for (let stepIndex = 0; stepIndex < totalSteps; stepIndex += 1) {
-      const step = steps[stepIndex] || {};
-      const kind = String(step.kind || "act").toLowerCase();
-      const instruction = step.instruction || step.name || `Step ${stepIndex + 1}`;
-      let action = null;
-      let lastErr = null;
-      const prevBuf = await page.screenshot({ type: "png" });
-
-      if (kind === "agent" || kind === "login") {
-        console.log(`[qualty][agent][step ${stepIndex + 1}] mode=pure_agent kind=${kind}`);
-        const agentResult = await runAgentStep({
-          apiRequest,
-          apiUrl,
-          token,
-          orgId,
-          executionId,
-          device: effectiveDevice,
-          stepIndex,
-          instruction,
-          page,
-          credentials,
-          labelToPath,
-        });
-        lastErr = agentResult.ok ? null : agentResult.error;
-        action = { method: "agent", description: instruction };
-      } else if (kind === "check") {
-        lastErr = "Check steps are not supported in local CLI yet; use cloud run or act steps.";
-      } else {
-        console.log(`[qualty][act][step ${stepIndex + 1}] mode=act`);
-        const actResult = await runActStep({
-          apiRequest,
-          apiUrl,
-          token,
-          orgId,
-          executionId,
-          device: effectiveDevice,
-          stepIndex,
-          page,
-          credentials,
-          labelToPath,
-          step,
-        });
-        lastErr = actResult.ok ? null : actResult.error;
-        action = actResult.action;
-        if (actResult.source) {
-          console.log(`[qualty][act][step ${stepIndex + 1}] resolved_source=${actResult.source}`);
-        }
-        if (actResult.skipRemainingSteps && actResult.ok) {
-          const executorOk = true;
-          const currBuf = await page.screenshot({ type: "png" });
-          const prevB64 = prevBuf.toString("base64");
-          const currB64 = currBuf.toString("base64");
-          lastPrevScreenshotB64 = prevB64;
-          lastCurrScreenshotB64 = currB64;
-          if (firstPrevScreenshotB64 == null) firstPrevScreenshotB64 = prevB64;
-          let row = {
-            name: instruction,
-            description: instruction,
-            status: "passed",
-            action: action
-              ? `${action.method}${action.selector ? ` ${action.selector}` : ""}`
-              : "agent_heal",
-          };
-          const stepResp = await apiRequest({
-            apiUrl,
-            token,
-            orgId,
-            path: `/api/v1/cli/executions/${executionId}/step-result`,
-            method: "POST",
-            body: withDevice(effectiveDevice, {
-              step_index: stepIndex,
-              executor_ok: executorOk,
-              action,
-              result: row,
-              prev_screenshot_b64: prevB64,
-              curr_screenshot_b64: currB64,
-            }),
-          });
-          if (stepResp && stepResp.result && typeof stepResp.result === "object") {
-            row = { ...row, ...stepResp.result };
-          }
-          stepsJson.push(row);
-          break;
-        }
-      }
-
-      const executorOk = !lastErr;
-      const currBuf = await page.screenshot({ type: "png" });
-      const prevB64 = prevBuf.toString("base64");
-      const currB64 = currBuf.toString("base64");
-      lastPrevScreenshotB64 = prevB64;
-      lastCurrScreenshotB64 = currB64;
-      if (firstPrevScreenshotB64 == null) firstPrevScreenshotB64 = prevB64;
-
-      let row = {
-        name: instruction,
-        description: instruction,
-        status: executorOk ? "passed" : "failed",
-        action: action
-          ? `${action.method}${action.selector ? ` ${action.selector}` : ""}`
-          : "",
-        error: lastErr || undefined,
-      };
-
-      const stepResp = await apiRequest({
-        apiUrl,
-        token,
-        orgId,
-        path: `/api/v1/cli/executions/${executionId}/step-result`,
-        method: "POST",
-        body: withDevice(effectiveDevice, {
-          step_index: stepIndex,
-          executor_ok: executorOk,
-          action,
-          result: row,
-          prev_screenshot_b64: prevB64,
-          curr_screenshot_b64: currB64,
-        }),
-      });
-
-      if (stepResp && stepResp.result && typeof stepResp.result === "object") {
-        row = { ...row, ...stepResp.result };
-      }
-      const passed = row.status === "passed";
-      if (!passed) runSuccess = false;
-      stepsJson.push(row);
-
-      if (!passed) {
-        runError = lastErr || row.step_thought || "Step failed";
-        break;
-      }
-      await page.waitForTimeout(300);
+    const wsParts = parseWsEndpoint(cdpEndpoint);
+    if (!Number.isFinite(wsParts.port) || wsParts.port < 1 || wsParts.port > 65535) {
+      throw new Error(`Could not parse local browser WebSocket port from ${cdpEndpoint}`);
     }
-
-    try {
-      lastCurrScreenshotB64 = (await page.screenshot({ type: "png" })).toString("base64");
-    } catch {
-      /* keep last step screenshot */
+    const tunnel = await apiRequest({
+      apiUrl,
+      token,
+      orgId,
+      path: `/api/v1/cli/executions/${executionId}/browser-tunnel`,
+      method: "POST",
+      body: withDevice(effectiveDevice, { local_port: wsParts.port }),
+    });
+    const tunnelToken = tunnel?.tunnel?.token;
+    const hostname = String(tunnel?.tunnel?.hostname || "").trim();
+    if (!tunnelToken || !hostname) {
+      throw new Error("Backend did not return a Cloudflare browser tunnel token and hostname.");
     }
+    cloudflared = startCloudflaredForPort(tunnelToken, wsParts.port);
+    const publicConnectUrl = `wss://${hostname}${wsParts.path}`;
+    // eslint-disable-next-line no-console
+    console.log(`[qualty][tunnel] Probing public browser tunnel ${hostname}`);
+    await waitForPublicBrowserTunnel({ connectUrl: publicConnectUrl, hostname });
+    await apiRequest({
+      apiUrl,
+      token,
+      orgId,
+      path: `/api/v1/cli/executions/${executionId}/register-browser`,
+      method: "POST",
+      body: withDevice(effectiveDevice, {
+        connect_url: publicConnectUrl,
+        connect_kind: "playwright",
+        start_url: url,
+      }),
+    });
+    // eslint-disable-next-line no-console
+    console.log(`[qualty] Registered local browser tunnel for ${effectiveDevice}`);
+    const result = await waitForCombinationDone({
+      apiRequest,
+      apiUrl,
+      token,
+      orgId,
+      executionId,
+      device: effectiveDevice,
+    });
+    return { success: result.success, error: result.error, combinationsRemaining: result.combinationsRemaining, device: effectiveDevice };
   } finally {
-    await context.close().catch(() => {});
-    await browser.close().catch(() => {});
+    if (cloudflared) cloudflared.kill();
     await browserServer.close().catch(() => {});
   }
-
-  const runtimeSec = (Date.now() - started) / 1000;
-  const completeResp = await apiRequest({
-    apiUrl,
-    token,
-    orgId,
-    path: `/api/v1/cli/executions/${executionId}/complete`,
-    method: "POST",
-    body: withDevice(effectiveDevice, {
-      success: runSuccess,
-      steps_json: stepsJson,
-      explanation: runSuccess ? "Local run completed" : runError || "Local run failed",
-      runtime_sec: runtimeSec,
-      error_message: runError,
-      final_screenshot_b64: lastCurrScreenshotB64,
-      prev_screenshot_b64: firstPrevScreenshotB64 || lastPrevScreenshotB64,
-      network_report: networkCollector.snapshot(),
-    }),
-  });
-
-  if (completeResp && completeResp.status) {
-    runSuccess = completeResp.status === "completed" || runSuccess;
-  }
-
-  const combinationsRemaining = Number(completeResp?.combinations_remaining ?? 0);
-  return { success: runSuccess, stepsJson, error: runError, combinationsRemaining, device: effectiveDevice };
 }
 
 async function fetchProjectLocalPort({ apiRequest, apiUrl, token, orgId, projectId }) {

package/bin/qualty.js

@@ -27,7 +27,10 @@ import {
 } from "./auth-ci-bundle.js";
 
 /** CLI backend URL is fixed by Qualty distribution. */
-const DEFAULT_QUALTY_API_URL = "https://qualty-api-production.up.railway.app";
+// const DEFAULT_QUALTY_API_URL = "https://qualty-api-production.up.railway.app";
+// const DEFAULT_QUALTY_API_URL = "http://localhost:8000";
+const DEFAULT_QUALTY_API_URL = "https://qualty-api-development.up.railway.app";
+
 
 const PROJECT_CONFIG_FILENAME = ".qualty.json";
 const DOTENV_FILENAME = ".env";
@@ -2133,8 +2136,20 @@ async function runCi(args) {
 async function resolveSavedJobs({ apiUrl, token, orgId, projectId, suiteId, explicitIds }) {
   const query = new URLSearchParams();
   if (projectId) query.set("project_id", String(projectId));
-  if (suiteId) query.set("suite_id", String(suiteId));
-  if (explicitIds.length > 0) query.set("ids", explicitIds.join(","));
+
+  let effectiveSuiteId = suiteId;
+  let ids = [...explicitIds];
+  // Suite M2M jobs exclude sequence members; expand so ci setup and profile resolution see all tests.
+  if (suiteId && ids.length === 0) {
+    const suiteJobIds = await collectSuiteJobIds({ apiUrl, token, orgId, suiteId });
+    if (suiteJobIds.length > 0) {
+      ids = suiteJobIds;
+      effectiveSuiteId = "";
+    }
+  }
+
+  if (effectiveSuiteId) query.set("suite_id", String(effectiveSuiteId));
+  if (ids.length > 0) query.set("ids", ids.join(","));
   return apiRequest({
     apiUrl,
     token,
@@ -2224,6 +2239,20 @@ async function resolveLocalSuitePlan({ apiUrl, token, orgId, suiteId }) {
   };
 }
 
+async function collectSuiteJobIds({ apiUrl, token, orgId, suiteId }) {
+  const plan = await resolveLocalSuitePlan({ apiUrl, token, orgId, suiteId });
+  const ids = new Set(plan.standaloneJobIds || []);
+  for (const sequence of plan.sequences || []) {
+    for (const stage of sequence.stages || []) {
+      for (const job of stage.jobs || []) {
+        const jobId = String(job.id || "").trim();
+        if (jobId) ids.add(jobId);
+      }
+    }
+  }
+  return [...ids];
+}
+
 async function runLocalSuiteWithSequences({
   apiUrl,
   token,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qualty",
-  "version": "0.1.29",
+  "version": "0.1.31",
   "description": "Qualty CLI for localhost and CI test runs",
   "bin": {
     "qualty": "bin/qualty.js"