surf-cli 2.18.0 → 2.19.0

package/native/doctor.cjs

@@ -405,6 +405,14 @@ function buildRecommendations(report) {
 
 function remoteRecommendations(endpoint, code) {
   const base = [`Confirm Surf is listening on ${endpoint.display}; the remote host listener must allow this Tailnet connection.`];
+  const tlsCodes = new Set([
+    "ERR_TLS_CERT_ALTNAME_INVALID", "UNABLE_TO_VERIFY_LEAF_SIGNATURE", "SELF_SIGNED_CERT_IN_CHAIN",
+    "DEPTH_ZERO_SELF_SIGNED_CERT", "UNABLE_TO_GET_ISSUER_CERT", "UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
+    "CERT_HAS_EXPIRED",
+  ]);
+  if (endpoint.tls?.enabled && (code === "ETIMEDOUT" || tlsCodes.has(code))) {
+    return [...base, "Check the reverse-proxy certificate chain and expected server name (or --remote-tls-server-name); a custom CA replaces system roots."];
+  }
   if (code === "ENOTFOUND") return [...base, "Check the Tailnet DNS name, then run `tailscale status` and `tailscale ping <host>`."].map((item) => item.replace("<host>", endpoint.host));
   if (code === "ETIMEDOUT") return [...base, `Run \`tailscale ping ${endpoint.host}\`; check restrictive Tailnet ACLs/grants and host firewall rules.`];
   if (code === "ECONNREFUSED") return [...base, "Verify the host process is running and bound to the requested TCP port; check restrictive Tailnet ACLs/grants."];
@@ -445,9 +453,10 @@ async function runDoctor(rawOptions = {}, deps = {}) {
       socket = connectEndpoint(target, () => finish({ ok: true, message: "authenticated" }));
       socket.once("error", (error) => finish({ ok: false, code: error.code, message: error.message || String(error) }));
     })))(endpoint, options.connectTimeoutMs);
-    const checks = [{ id: "remote-endpoint", status: "info", message: `Remote endpoint: ${endpoint.display}`, endpoint: endpoint.display }, {
+    const tlsMarker = endpoint.tls?.enabled ? " (TLS)" : "";
+    const checks = [{ id: "remote-endpoint", status: "info", message: `Remote endpoint: ${endpoint.display}${tlsMarker}`, endpoint: endpoint.display }, {
       id: "remote-connect", status: connection.ok ? "pass" : "fail",
-      message: connection.ok ? `Connected to remote endpoint ${endpoint.display}` : `Could not connect to remote endpoint ${endpoint.display}: ${connection.message}`,
+      message: connection.ok ? `Connected to remote endpoint ${endpoint.display}${tlsMarker}` : `Could not connect to remote endpoint ${endpoint.display}${tlsMarker}: ${connection.message}`,
       code: connection.code,
     }, {
       id: "remote-auth", status: connection.ok ? "pass" : connection.code === "EAUTH" ? "fail" : "info",

package/native/endpoint.cjs

@@ -1,7 +1,12 @@
 const net = require("net");
+const fs = require("fs");
+const tls = require("tls");
 const { DEFAULT_SOCKET_PATH } = require("./socket-path.cjs");
 const { authenticateClient } = require("./remote-transport.cjs");
 
+const TLS_HANDSHAKE_TIMEOUT_MS = 5000;
+const HOSTNAME_PATTERN = /^(?=.{1,253}$)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)*[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$/;
+
 function parseRemoteEndpoint(value) {
   if (typeof value !== "string" || !value) throw new Error("--remote requires host:port");
   let host;
@@ -16,7 +21,7 @@ function parseRemoteEndpoint(value) {
     if (!match) throw new Error("remote endpoint must be host:port (IPv6 must be bracketed)");
     [, host, portText] = match;
     if (host.includes("/") || host.includes("@") || host.includes(":") || host === "*" || host.includes("*")) throw new Error("remote endpoint host is invalid");
-    if ((/^\d+(?:\.\d+){3}$/.test(host) && net.isIP(host) !== 4) || (net.isIP(host) !== 4 && !/^(?=.{1,253}$)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)*[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$/.test(host))) {
+    if ((/^\d+(?:\.\d+){3}$/.test(host) && net.isIP(host) !== 4) || (net.isIP(host) !== 4 && !HOSTNAME_PATTERN.test(host))) {
       throw new Error("remote endpoint host is invalid");
     }
     host = host.toLowerCase();
@@ -30,45 +35,107 @@ function parseRemoteEndpoint(value) {
   return { kind: "remote", host, port, display, key: `tcp:${display}`, connectionOptions: { host, port } };
 }
 
+function extractRemoteOptions(args) {
+  const definitions = {
+    "--remote": { name: "remote", missing: "--remote requires host:port" },
+    "--remote-credential": { name: "credential", missing: "--remote-credential requires a file path" },
+    "--remote-tls": { name: "tls", boolean: true },
+    "--remote-tls-ca": { name: "tlsCa", missing: "--remote-tls-ca requires a file path" },
+    "--remote-tls-server-name": { name: "tlsServerName", missing: "--remote-tls-server-name requires a DNS hostname" },
+  };
+  const values = {};
+  const strippedArgs = [];
+  for (let index = 0; index < args.length; index++) {
+    const option = definitions[args[index]];
+    if (!option) {
+      strippedArgs.push(args[index]);
+      continue;
+    }
+    if (Object.hasOwn(values, option.name)) throw new Error(`${args[index]} may only be specified once`);
+    if (option.boolean) {
+      values[option.name] = true;
+      continue;
+    }
+    const value = args[index + 1];
+    if (!value || value.startsWith("--")) throw new Error(option.missing);
+    values[option.name] = value;
+    index++;
+  }
+  return { values, strippedArgs };
+}
+
+function validateServerName(value, source) {
+  if (!value || net.isIP(value) !== 0 || !HOSTNAME_PATTERN.test(value)) {
+    throw new Error(`${source} must be a valid DNS hostname`);
+  }
+  return value.toLowerCase();
+}
+
+function loadTlsCa(caPath, source) {
+  let ca;
+  try {
+    ca = fs.readFileSync(caPath);
+  } catch (error) {
+    throw new Error(`${source} could not read CA file ${caPath}: ${error.message}`);
+  }
+  if (ca.length === 0 || !ca.includes(Buffer.from("-----BEGIN CERTIFICATE-----"))) {
+    throw new Error(`${source} CA file ${caPath} must contain a PEM certificate`);
+  }
+  return ca;
+}
+
 function selectEndpoint(args, env) {
   const selectedEnv = env === undefined ? process.env : env;
-  const remoteIndexes = [];
-  const credentialIndexes = [];
-  for (let i = 0; i < args.length; i++) {
-    if (args[i] === "--remote") remoteIndexes.push(i);
-    if (args[i] === "--remote-credential") credentialIndexes.push(i);
-  }
-  if (remoteIndexes.length > 1) throw new Error("--remote may only be specified once");
-  if (credentialIndexes.length > 1) throw new Error("--remote-credential may only be specified once");
-  let cliRemote;
-  let cliCredential;
-  const strippedArgs = [...args];
-  if (remoteIndexes.length) {
-    const index = remoteIndexes[0];
-    cliRemote = args[index + 1];
-    if (!cliRemote || cliRemote.startsWith("--")) throw new Error("--remote requires host:port");
-    strippedArgs.splice(index, 2);
+  const { values, strippedArgs } = extractRemoteOptions(args);
+  const envTls = selectedEnv.SURF_REMOTE_TLS;
+  if (envTls && envTls !== "1") {
+    throw new Error('SURF_REMOTE_TLS must be "1" to enable TLS; unset it to disable');
   }
-  if (credentialIndexes.length) {
-    const index = credentialIndexes[0];
-    cliCredential = args[index + 1];
-    if (!cliCredential || cliCredential.startsWith("--")) throw new Error("--remote-credential requires a file path");
-    const adjustedIndex = index - (remoteIndexes.length && index > remoteIndexes[0] ? 2 : 0);
-    strippedArgs.splice(adjustedIndex, 2);
+  const remoteValue = values.remote || selectedEnv.SURF_REMOTE;
+  const credentialPath = values.credential || selectedEnv.SURF_REMOTE_CREDENTIAL;
+  const tlsEnabled = values.tls === true || envTls === "1";
+  const caPath = values.tlsCa || selectedEnv.SURF_REMOTE_TLS_CA;
+  const serverNameValue = values.tlsServerName || selectedEnv.SURF_REMOTE_TLS_SERVER_NAME;
+  if (!remoteValue) {
+    if (values.credential) throw new Error("--remote-credential requires a remote endpoint");
+    if (values.tls) throw new Error("--remote-tls requires a remote endpoint");
+    if (selectedEnv.SURF_REMOTE_TLS === "1") throw new Error("SURF_REMOTE_TLS requires a remote endpoint");
+    if (values.tlsCa) throw new Error("--remote-tls-ca requires a remote endpoint");
+    if (selectedEnv.SURF_REMOTE_TLS_CA) throw new Error("SURF_REMOTE_TLS_CA requires a remote endpoint");
+    if (values.tlsServerName) throw new Error("--remote-tls-server-name requires a remote endpoint");
+    if (selectedEnv.SURF_REMOTE_TLS_SERVER_NAME) throw new Error("SURF_REMOTE_TLS_SERVER_NAME requires a remote endpoint");
+    const socketPath = selectedEnv.SURF_SOCKET || DEFAULT_SOCKET_PATH;
+    return { args: strippedArgs, endpoint: { kind: "local", path: socketPath, display: socketPath, key: `unix:${socketPath}`, connectionOptions: socketPath } };
   }
-  const remoteValue = cliRemote || selectedEnv.SURF_REMOTE;
-  if (remoteValue) {
-    const credentialPath = cliCredential || selectedEnv.SURF_REMOTE_CREDENTIAL;
-    if (!credentialPath) throw new Error("remote endpoint requires --remote-credential <path> or SURF_REMOTE_CREDENTIAL");
-    return { args: strippedArgs, endpoint: { ...parseRemoteEndpoint(remoteValue), credentialPath } };
+  if (!credentialPath) throw new Error("remote endpoint requires --remote-credential <path> or SURF_REMOTE_CREDENTIAL");
+  if (caPath && !tlsEnabled) throw new Error(`${values.tlsCa ? "--remote-tls-ca" : "SURF_REMOTE_TLS_CA"} requires TLS to be enabled`);
+  if (serverNameValue && !tlsEnabled) throw new Error(`${values.tlsServerName ? "--remote-tls-server-name" : "SURF_REMOTE_TLS_SERVER_NAME"} requires TLS to be enabled`);
+  const endpoint = { ...parseRemoteEndpoint(remoteValue), credentialPath };
+  if (tlsEnabled) {
+    const tlsOptions = { enabled: true };
+    if (caPath) {
+      tlsOptions.ca = loadTlsCa(caPath, values.tlsCa ? "--remote-tls-ca" : "SURF_REMOTE_TLS_CA");
+      tlsOptions.caPath = caPath;
+    }
+    const serverName = serverNameValue
+      ? validateServerName(serverNameValue, values.tlsServerName ? "--remote-tls-server-name" : "SURF_REMOTE_TLS_SERVER_NAME")
+      : net.isIP(endpoint.host) === 0 ? endpoint.host : undefined;
+    if (serverName) tlsOptions.serverName = serverName;
+    endpoint.tls = tlsOptions;
   }
-  if (cliCredential) throw new Error("--remote-credential requires a remote endpoint");
-  const socketPath = selectedEnv.SURF_SOCKET || DEFAULT_SOCKET_PATH;
-  return { args: strippedArgs, endpoint: { kind: "local", path: socketPath, display: socketPath, key: `unix:${socketPath}`, connectionOptions: socketPath } };
+  return { args: strippedArgs, endpoint };
 }
 
 function createRemoteSocket(endpoint) {
-  const rawSocket = net.createConnection(endpoint.connectionOptions, () => {});
+  const usingTls = endpoint.tls?.enabled === true;
+  const rawSocket = usingTls
+    ? tls.connect({
+      ...endpoint.connectionOptions,
+      rejectUnauthorized: true,
+      ...(endpoint.tls.ca ? { ca: endpoint.tls.ca } : {}),
+      ...(endpoint.tls.serverName ? { servername: endpoint.tls.serverName } : {}),
+    })
+    : net.createConnection(endpoint.connectionOptions, () => {});
   let ready = false;
   let connected = false;
   let destroyed = false;
@@ -106,7 +173,25 @@ function createRemoteSocket(endpoint) {
     get authenticated() { return ready; },
     get connected() { return connected; },
   };
-  rawSocket.once("connect", () => { connected = true; });
+  const transportReadyEvent = usingTls ? "secureConnect" : "connect";
+  rawSocket.once(transportReadyEvent, () => { connected = true; });
+  let handshakeTimer;
+  if (usingTls) {
+    const timeoutMs = endpoint.tls.handshakeTimeoutMs ?? TLS_HANDSHAKE_TIMEOUT_MS;
+    const clearHandshakeTimer = () => {
+      if (handshakeTimer) clearTimeout(handshakeTimer);
+      handshakeTimer = undefined;
+    };
+    handshakeTimer = setTimeout(() => {
+      if (connected || rawSocket.destroyed) return;
+      const error = new Error(`TLS handshake timed out after ${timeoutMs}ms`);
+      error.code = "ETIMEDOUT";
+      rawSocket.destroy(error);
+    }, timeoutMs);
+    rawSocket.once("secureConnect", clearHandshakeTimer);
+    rawSocket.once("error", clearHandshakeTimer);
+    rawSocket.once("close", clearHandshakeTimer);
+  }
   rawSocket.on("error", (error) => {
     if (ready || destroyed) return;
     ready = true;
@@ -133,7 +218,7 @@ function connectEndpoint(endpoint, onConnect) {
     return net.createConnection(endpoint.connectionOptions, onConnect || (() => {}));
   }
   const { rawSocket, proxy, flush } = createRemoteSocket(endpoint);
-  rawSocket.once("connect", () => {
+  rawSocket.once(endpoint.tls?.enabled ? "secureConnect" : "connect", () => {
     authenticateClient(rawSocket, endpoint.credentialPath)
       .then(() => {
         flush();
@@ -168,7 +253,7 @@ function connectEndpoint(endpoint, onConnect) {
 function formatEndpointError(error, endpoint, formatSocketError) {
   if (endpoint.kind === "local") return formatSocketError(error);
   const message = error?.message || String(error);
-  return `Remote endpoint connection failed (${endpoint.display}): ${message}`;
+  return `Remote endpoint connection failed (${endpoint.display}${endpoint.tls?.enabled ? ", TLS" : ""}): ${message}`;
 }
 
-module.exports = { parseRemoteEndpoint, selectEndpoint, connectEndpoint, formatEndpointError };
+module.exports = { TLS_HANDSHAKE_TIMEOUT_MS, parseRemoteEndpoint, selectEndpoint, connectEndpoint, formatEndpointError };

package/native/extract.cjs

@@ -0,0 +1,362 @@
+/** Retries use fresh tabs, so caller scripts must be read-only or idempotent. */
+
+const { applyOptionsPrelude } = require("./script-options.cjs");
+
+const DEFAULT_RETRY_COUNT = 1;
+const DEFAULT_RETRY_DELAY_MS = 500;
+const MAX_RETRY_COUNT = 5;
+const ROW_KEY_CANDIDATES = ["rows", "items", "results", "entries", "records", "data"];
+
+const TRANSIENT_TAB_ERROR_MARKERS = [
+  "navigated or closed",
+  "Detached while handling command",
+  "Cannot find default execution context",
+  "Execution context was destroyed",
+  "Receiving end does not exist",
+  "Content script not loaded",
+  "no longer exists",
+  "Target closed",
+];
+
+const RETRYABLE_ERROR_CODES = new Set(["empty_result", "page_timeout", "tab_gone", "target_gone"]);
+
+const FATAL_READINESS_CODES = new Set(["page_login", "page_challenge", "page_not_found", "page_error"]);
+
+class ExtractError extends Error {
+  constructor(code, message, details = {}) {
+    super(message);
+    this.name = "ExtractError";
+    this.code = code;
+    this.details = details;
+  }
+}
+
+function errorMessageOf(error) {
+  if (error instanceof Error) return error.message;
+  if (typeof error === "string") return error;
+  if (error && typeof error === "object") {
+    const text = error.content?.[0]?.text;
+    if (typeof text === "string") return text;
+    if (typeof error.message === "string") return error.message;
+    return JSON.stringify(error);
+  }
+  return String(error);
+}
+
+function errorCodeOf(error) {
+  if (error && typeof error === "object" && typeof error.code === "string") return error.code;
+  return null;
+}
+
+function isTransientTabError(error) {
+  const message = errorMessageOf(error);
+  return TRANSIENT_TAB_ERROR_MARKERS.some((marker) => message.includes(marker));
+}
+
+/**
+ * Whether a failed attempt is worth a fresh tab. Login bounces, challenges
+ * and not-found pages are not: the next tab lands on the same page.
+ */
+function isRetryableExtractionError(error) {
+  const code = errorCodeOf(error);
+  if (code && FATAL_READINESS_CODES.has(code)) return false;
+  if (code && RETRYABLE_ERROR_CODES.has(code)) return true;
+  return isTransientTabError(error);
+}
+
+function responseText(response) {
+  const text = response?.result?.content?.[0]?.text;
+  return typeof text === "string" ? text : null;
+}
+
+function responseError(response, stage) {
+  if (!response || !response.error) return null;
+  const err = response.error;
+  const code = errorCodeOf(err) || "tool_error";
+  return new ExtractError(code, errorMessageOf(err), { stage, ...(err.details || {}) });
+}
+
+function parseExtractionOutput(text) {
+  if (text === null || text === undefined || text.trim() === "" || text.trim() === "undefined") {
+    throw new ExtractError(
+      "no_output",
+      "The extraction script returned nothing. End it with `return { rows: [...] }` or `return [...]`.",
+    );
+  }
+  try {
+    return JSON.parse(text);
+  } catch (error) {
+    throw new ExtractError("invalid_output", `The extraction script did not return JSON: ${error.message}`, {
+      preview: text.slice(0, 200),
+    });
+  }
+}
+
+/**
+ * Pick the row array out of the script result: the result itself when it
+ * is an array, `--rows <key>` when given, else the first conventional key
+ * holding an array. Returns null when the result has no row concept.
+ */
+function selectRows(data, rowsKey) {
+  if (rowsKey) {
+    const rows = data && typeof data === "object" && !Array.isArray(data) ? data[rowsKey] : undefined;
+    if (!Array.isArray(rows)) {
+      throw new ExtractError("rows_key_missing", `The script result has no array at "${rowsKey}"`, {
+        keys: data && typeof data === "object" ? Object.keys(data) : [],
+      });
+    }
+    return rows;
+  }
+  if (Array.isArray(data)) return data;
+  if (data && typeof data === "object") {
+    for (const key of ROW_KEY_CANDIDATES) {
+      if (Array.isArray(data[key])) return data[key];
+    }
+  }
+  return null;
+}
+
+/**
+ * Zero rows is a failure unless the caller opts in. A logged-out render,
+ * a selector miss or a half-loaded page all look like "no results"; only
+ * the caller knows whether an empty result is plausible.
+ */
+function enforceRowsInvariant(rows, { allowEmpty = false, readiness } = {}) {
+  if (!Array.isArray(rows) || rows.length > 0 || allowEmpty) return;
+  if (readiness?.state === "empty") return;
+  throw new ExtractError(
+    "empty_result",
+    "The extraction returned zero rows (the page may be logged out, blocked, or the selectors missed). Pass --allow-empty to accept an empty result, or --empty-text to recognise the page's own no-results message.",
+    { rows: 0 },
+  );
+}
+
+function cellText(value) {
+  let text;
+  if (value === null || value === undefined) text = "";
+  else if (typeof value === "string") text = value;
+  else text = JSON.stringify(value);
+  return text.replace(/\s+/g, " ").replace(/\|/g, "\\|").trim();
+}
+
+/** Markdown for humans and LLMs: metadata bullets, then a table of rows. */
+function renderExtractionMarkdown(data, rows, { title = "Extraction" } = {}) {
+  const lines = [`# ${title}`, ""];
+  if (data && typeof data === "object" && !Array.isArray(data)) {
+    for (const [key, value] of Object.entries(data)) {
+      if (Array.isArray(value) || (value && typeof value === "object")) continue;
+      lines.push(`- ${key}: ${cellText(value)}`);
+    }
+    if (lines.length > 2) lines.push("");
+  }
+  if (!Array.isArray(rows)) {
+    lines.push("```json", JSON.stringify(data, null, 2), "```");
+    return lines.join("\n");
+  }
+  lines.push(`${rows.length} row${rows.length === 1 ? "" : "s"}`, "");
+  if (rows.length === 0) return lines.join("\n").trimEnd();
+  if (!rows.every((row) => row !== null && typeof row === "object" && !Array.isArray(row))) {
+    for (const row of rows) lines.push(`- ${cellText(row)}`);
+    return lines.join("\n");
+  }
+  const columns = [];
+  for (const row of rows) {
+    for (const key of Object.keys(row)) {
+      if (!columns.includes(key)) columns.push(key);
+    }
+  }
+  lines.push(`| ${columns.join(" | ")} |`);
+  lines.push(`| ${columns.map(() => "---").join(" | ")} |`);
+  for (const row of rows) {
+    lines.push(`| ${columns.map((column) => cellText(row[column])).join(" | ")} |`);
+  }
+  return lines.join("\n");
+}
+
+function normalizeRetry(retry) {
+  const count = Number.isInteger(retry?.count) ? Math.max(0, Math.min(retry.count, MAX_RETRY_COUNT)) : DEFAULT_RETRY_COUNT;
+  const delayMs = Number.isFinite(retry?.delayMs) && retry.delayMs >= 0 ? retry.delayMs : DEFAULT_RETRY_DELAY_MS;
+  return { count, delayMs };
+}
+
+function readinessArgs(ready = {}) {
+  const args = {};
+  if (ready.selector) args.selector = ready.selector;
+  if (ready.text) args.text = ready.text;
+  if (ready.urlPrefix) args.urlPrefix = ready.urlPrefix;
+  if (ready.emptyText) args.emptyText = ready.emptyText;
+  if (ready.timeout !== undefined) args.timeout = ready.timeout;
+  if (ready.interval !== undefined) args.interval = ready.interval;
+  return args;
+}
+
+/** Tab id from the stable structured field on a tab.new host response. */
+function tabIdFromResponse(response) {
+  const failure = responseError(response, "tab.new");
+  if (failure) throw failure;
+  const tabId = response?.result?.tabId;
+  if (Number.isInteger(tabId) && tabId > 0) return tabId;
+  throw new ExtractError("no_tab", "tab.new did not return a structured tab id");
+}
+
+function parseToolJson(response, stage) {
+  const failure = responseError(response, stage);
+  if (failure) throw failure;
+  const text = responseText(response);
+  if (text === null) return null;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return text;
+  }
+}
+
+/** Drop transport-only fields from the readiness metadata returned to callers. */
+function cleanReadiness(readiness) {
+  if (!readiness || typeof readiness !== "object") return readiness;
+  const { id, _resolvedTabId, _resolvedWindowId, _hint, ...publicReadiness } = readiness;
+  return publicReadiness;
+}
+
+async function runAttemptOnTab(executeTool, tabId, settings) {
+  const readiness = cleanReadiness(
+    parseToolJson(await executeTool("wait.ready", readinessArgs(settings.ready), tabId), "wait.ready"),
+  );
+  const code = applyOptionsPrelude(settings.code, settings.options);
+  const jsResponse = await executeTool("js", { code }, tabId);
+  const failure = responseError(jsResponse, "js");
+  if (failure) throw failure;
+  const data = parseExtractionOutput(responseText(jsResponse));
+  const rows = selectRows(data, settings.rowsKey);
+  enforceRowsInvariant(rows, { allowEmpty: settings.allowEmpty, readiness });
+  return { data, rows, readiness };
+}
+
+/**
+ * @param {object} settings
+ * @param {(tool: string, args: object, tabId?: number) => Promise<object>} settings.executeTool
+ *   Sends one tool request. `tabId` overrides the target for owned tabs; when
+ *   it is undefined the caller's default target (session/tab/window) applies.
+ * @param {string} settings.code Page-side script; must `return` JSON.
+ * @param {string} [settings.url] Page to open. Required unless `target` is set.
+ * @param {object} [settings.options] Exposed to the script as SURF_OPTIONS.
+ * @param {object} [settings.ready] wait.ready expectations (selector, text, urlPrefix, emptyText, timeout, interval).
+ * @param {{count?: number, delayMs?: number}} [settings.retry]
+ * @param {boolean} [settings.keepTab] Leave the owned tab open on success.
+ * @param {boolean} [settings.allowEmpty]
+ * @param {string} [settings.rowsKey]
+ * @param {boolean} [settings.target] Use the caller's target instead of an owned tab.
+ * @param {(error: unknown) => boolean} [settings.isRetryable]
+ * @param {(ms: number) => Promise<void>} [settings.sleep]
+ * @param {(event: object) => void} [settings.onEvent]
+ */
+async function runExtraction(settings) {
+  const {
+    executeTool,
+    code,
+    url,
+    target = false,
+    keepTab = false,
+    isRetryable = isRetryableExtractionError,
+    sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+    onEvent = () => {},
+  } = settings;
+  if (typeof executeTool !== "function") throw new Error("runExtraction requires executeTool");
+  if (typeof code !== "string" || code.trim() === "") throw new ExtractError("no_script", "An extraction script is required (--file or --code)");
+  if (!target && !url) throw new ExtractError("no_url", "A URL is required unless --tab-id or --session names the page to read");
+
+  if (target) {
+    // Caller-supplied target: navigate once if asked, never close, never retry.
+    if (url) {
+      const navigation = await executeTool("navigate", { url });
+      const failure = responseError(navigation, "navigate");
+      if (failure) throw failure;
+    }
+    onEvent({ type: "attempt", attempt: 1, of: 1, mode: "target" });
+    const attempt = await runAttemptOnTab(executeTool, undefined, settings);
+    return { ...attempt, rowCount: Array.isArray(attempt.rows) ? attempt.rows.length : null, attempts: 1, mode: "target", url: url ?? null };
+  }
+
+  const retry = normalizeRetry(settings.retry);
+  const attempts = retry.count + 1;
+  let lastError = null;
+  let attemptsMade = 0;
+
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    attemptsMade = attempt;
+    if (attempt > 1) await sleep(retry.delayMs);
+    onEvent({ type: "attempt", attempt, of: attempts, mode: "owned-tab" });
+    let tabId = null;
+    let result;
+    try {
+      tabId = tabIdFromResponse(await executeTool("tab.new", { url }));
+      result = await runAttemptOnTab(executeTool, tabId, settings);
+    } catch (error) {
+      lastError = error;
+      let cleanupError = null;
+      if (tabId) {
+        try {
+          const closed = await executeTool("tab.close", { id: tabId }, tabId);
+          const closeFailure = responseError(closed, "tab.close");
+          if (closeFailure) throw closeFailure;
+        } catch (closeError) {
+          cleanupError = closeError;
+          onEvent({ type: "close-failed", attempt, tabId, error: errorMessageOf(closeError) });
+        }
+      }
+      if (cleanupError) {
+        throw new ExtractError("cleanup_failed", `Extraction failed and the owned tab could not be closed: ${errorMessageOf(cleanupError)}`, {
+          stage: "tab.close",
+          tabId,
+          attempts: attempt,
+          extractionError: { code: errorCodeOf(error), message: errorMessageOf(error) },
+        });
+      }
+      const retryable = attempt < attempts && isRetryable(error);
+      onEvent({ type: "attempt-failed", attempt, of: attempts, error: errorMessageOf(error), code: errorCodeOf(error), retryable });
+      if (!retryable) break;
+      continue;
+    }
+
+    if (!keepTab) {
+      try {
+        const closed = await executeTool("tab.close", { id: tabId }, tabId);
+        const closeFailure = responseError(closed, "tab.close");
+        if (closeFailure) throw closeFailure;
+      } catch (error) {
+        throw new ExtractError("cleanup_failed", `Extraction succeeded but the owned tab could not be closed: ${errorMessageOf(error)}`, {
+          stage: "tab.close",
+          tabId,
+          attempts: attempt,
+          extractionSucceeded: true,
+          rowCount: Array.isArray(result.rows) ? result.rows.length : null,
+        });
+      }
+    }
+    return {
+      ...result,
+      rowCount: Array.isArray(result.rows) ? result.rows.length : null,
+      attempts: attempt,
+      mode: "owned-tab",
+      url,
+      tabId: keepTab ? tabId : null,
+    };
+  }
+  if (lastError instanceof ExtractError) {
+    lastError.details = { ...lastError.details, attempts: attemptsMade };
+    throw lastError;
+  }
+  throw new ExtractError(errorCodeOf(lastError) || "extraction_failed", errorMessageOf(lastError), { attempts: attemptsMade });
+}
+
+module.exports = {
+  ExtractError,
+  enforceRowsInvariant,
+  isRetryableExtractionError,
+  isTransientTabError,
+  parseExtractionOutput,
+  renderExtractionMarkdown,
+  runExtraction,
+  selectRows,
+  tabIdFromResponse,
+};

package/native/host-helpers.cjs

@@ -462,13 +462,37 @@ function formatToolContent(result, log = () => {}, options = {}) {
   }
   
   // Strip internal fields before JSON output
-  const { _resolvedTabId, _hint, ...cleanResult } = result;
+  const { _resolvedTabId, _resolvedWindowId, _hint, ...cleanResult } = result;
   if (_hint) {
     return text(JSON.stringify(cleanResult) + `\n[hint] ${_hint}`);
   }
   return text(JSON.stringify(cleanResult));
 }
 
+/**
+ * Readiness expectations accept both CLI flag spelling (--url-prefix) and
+ * socket API spelling (urlPrefix). Empty values are dropped.
+ */
+function readinessExpectations(a) {
+  const pick = (...keys) => {
+    for (const key of keys) {
+      const value = a[key];
+      if (typeof value === "string" && value.trim() !== "") return value;
+    }
+    return undefined;
+  };
+  const expect = {
+    selector: pick("selector"),
+    text: pick("text"),
+    urlPrefix: pick("urlPrefix", "url-prefix"),
+    emptyText: pick("emptyText", "empty-text"),
+  };
+  for (const key of Object.keys(expect)) {
+    if (expect[key] === undefined) delete expect[key];
+  }
+  return expect;
+}
+
 /**
  * Map computer action to extension message
  */
@@ -871,10 +895,23 @@ function mapToolToMessage(tool, args, tabId) {
       return { type: "WAIT_FOR_URL", pattern: a.pattern || a.url, timeout: a.timeout, ...baseMsg };
     case "wait.dom":
       return { type: "WAIT_FOR_DOM_STABLE", stable: a.stable || 100, timeout: a.timeout || 5000, ...baseMsg };
+    case "wait.ready":
+      return {
+        type: "WAIT_FOR_READY",
+        expect: readinessExpectations(a),
+        timeout: a.timeout,
+        interval: a.interval,
+        accept: a.accept,
+        ...baseMsg,
+      };
+    case "page.readiness":
+      return { type: "PAGE_READINESS", expect: readinessExpectations(a), ...baseMsg };
     case "wait.load":
       return { type: "WAIT_FOR_LOAD", timeout: a.timeout || 30000, ...baseMsg };
     case "frame.list":
       return { type: "GET_FRAMES", ...baseMsg };
+    case "frame.diagnose":
+      return { type: "FRAME_DIAGNOSE", ...baseMsg };
     case "frame.switch":
       return { 
         type: "FRAME_SWITCH", 
@@ -1280,4 +1317,4 @@ function mapToolToMessage(tool, args, tabId) {
   }
 }
 
-module.exports = { mapToolToMessage, mapComputerAction, formatToolContent, formatToolError, buildProviderUploadMessage };
+module.exports = { mapToolToMessage, mapComputerAction, formatToolContent, formatToolError, buildProviderUploadMessage, readinessExpectations };