wb-browser-runtime 0.13.0 → 0.14.0

package/README.md

@@ -122,6 +122,8 @@ Verb arguments support two substitutions at dispatch time:
 
 Both forms are redacted in stdout summaries — only the verb name + selector make it into the log. Expanded values are also scrubbed from `verb.failed` / `slice.failed` error messages before they cross the stdio boundary.
 
+**Escaping.** To emit a literal `{{ … }}` that should *not* be substituted, prefix it with a backslash: `\{{ env.X }}` round-trips to the literal text `{{ env.X }}`. The escape is a single left-to-right pass, so the braces it produces are not re-scanned.
+
 **Missing-value policy.** Set `WB_SUBSTITUTION_ON_MISSING` to choose how a missing `env.X` or `artifacts.X` is handled:
 
 - `warn` (default) — log a stderr warning and substitute an empty string; the verb continues.
@@ -166,10 +168,16 @@ endpoint at session close. Recording is **off by default** — set
 | `WB_RECORDING_SCREENCAST_QUALITY`  | `60`       | JPEG quality (0–100).                             |
 | `WB_RECORDING_RRWEB`               | `1`        | Set `0` to skip rrweb even if recording is on.    |
 | `WB_RECORDING_VIDEO`               | `0` if no `ffmpeg` | Set `0` to skip video even if `ffmpeg` is present. |
+| `WB_RECORDING_MASK_ALL_INPUTS`     | `1`        | rrweb `maskAllInputs`. Set `0` to record input *values* (off by default for safety). |
+| `WB_RECORDING_MASK_TEXT_SELECTOR`  | *(unset)*  | CSS selector whose **text content** rrweb masks (e.g. `.ssn, .acct-balance`). |
+| `WB_RECORDING_BLOCK_SELECTOR`      | *(unset)*  | CSS selector rrweb records as an inert placeholder (contents never captured). |
+| `WB_RECORDING_IGNORE_SELECTOR`     | *(unset)*  | CSS selector whose input events rrweb drops entirely. |
 
 Artifacts are two parallel POSTs per session, `kind ∈ {rrweb, video}`:
 
-- **rrweb** — gzipped JSON (`application/json+gzip`) — `{ run_id, session, event_count, events: [...] }`. DOM mutations + input events captured from every page; defaults mask all inputs for PII.
+- **rrweb** — gzipped JSON (`application/json+gzip`) — `{ run_id, session, event_count, events: [...] }`. DOM mutations + input events captured from every page.
+
+  **PII scope — read this.** `maskAllInputs` (on by default) only redacts the *values* a user types into form fields. It does **not** mask field labels, placeholders, `aria-label`s, `<option>` text, or any other rendered text, and it does not alter the recorded DOM structure. A displayed account number, balance, or name that is page text — not an input value — is captured verbatim. For those, point rrweb at the sensitive nodes with `WB_RECORDING_MASK_TEXT_SELECTOR` (mask the text) or `WB_RECORDING_BLOCK_SELECTOR` (omit the subtree). When in doubt, block the region.
 - **video** — VP9 WebM (`video/webm`) — encoded from JPEG screencast frames via `ffmpeg`. Requires `ffmpeg` on `$PATH` (droplet install: `apt-get install -y ffmpeg`). If `ffmpeg` is missing the video kind silently disables and rrweb continues alone.
 
 Each POST carries headers `Authorization: Bearer <secret>`,
@@ -208,7 +216,7 @@ example, see the `browserbase-hn-upvoted-probe` runbook in the xatabase repo.
 | `assert`     | `assert: <selector>`        | `selector`, `text_contains`, `url_contains`     |
 | `eval`       | `eval: <js>`                | `script`                                        |
 | `save`       | `save: <name>`              | `name`, `value` (captures prior `extract`/`eval` when omitted) |
-| `download`   | `download: <selector>`      | `selector`, `path`, `timeout`, `text_fallback` (clicks + races Playwright `download` event with in-page blob/anchor capture; saves into `$WB_ARTIFACTS_DIR/<path>`) |
+| `download`   | `download: <selector>`      | `selector`, `path`, `timeout`, `text_fallback`, `signed_url` (clicks + races Playwright `download` event, in-page blob/anchor capture, and signed-URL export capture; saves into `$WB_ARTIFACTS_DIR/<path>`) |
 
 `extract`'s `fields` entries are either a CSS selector string (returns
 `textContent`), or `{ selector, attr }` to read an attribute.
@@ -308,9 +316,55 @@ Behaviour:
   doesn't double-save.
 - Emits `slice.artifact_saved` with `source: "download"` and
   `provenance.verb_name: "download"`.
-- On timeout: throws with diagnostics (page URL, selector, both
+- On timeout: throws with diagnostics (page URL, selector, all
   failure reasons) AND emits a `slice.download_failed` frame.
 
+#### Signed-URL export capture
+
+Some SaaS "Download" buttons never trip a Playwright `download` event or an
+in-page Blob. Instead the click calls a same-origin API that returns JSON like
+`{ "download_url": "https://bucket.s3.amazonaws.com/…?<signed>" }` and then
+navigates to that URL — and a page-side `fetch(signedUrl)` fails because the
+object store's CORS policy won't let the app origin read the bytes.
+
+The `download:` verb adds a **third** capture racer for this: it wraps the
+page's `fetch`/`XHR` around the click, inspects small same-origin JSON
+responses for URL-looking fields, and when it finds one pointing at a
+recognized object-store host (S3, GCS, CloudFront, Azure Blob, R2), it
+downloads the bytes **from the sidecar process** (where CORS doesn't apply)
+and saves them like any other artifact.
+
+This is **on by default in `auto` mode** — it only fires when a recognized
+signed host appears in a JSON response around the click, so a normal
+Playwright/blob download is unaffected. Tune or disable it per verb:
+
+```yaml
+- download:
+    selector: 'button:has-text("Download as xlsx")'
+    path: pilot-profit-loss.xlsx
+    timeout: 10s
+    signed_url:
+      enabled: true          # true | false | auto (default auto)
+      hosts:                 # extra non-recognized hosts to accept
+        - pilot-report-downloads.s3.amazonaws.com
+      json_fields:           # restrict to these response field names
+        - download_url
+```
+
+- Set `signed_url: false` to turn the capture off entirely for a verb.
+- In `auto` mode only recognized object-store hosts (or an explicit `hosts:`
+  entry) are fetched. With `enabled: true` an explicit `hosts:`/`json_fields:`
+  match is honored even for an unrecognized host, since you named it.
+- The captured URL's **query string (where signed credentials live) is
+  redacted** everywhere it crosses the stdio boundary — `provenance.signed_url`
+  is `origin+path?<redacted>`; the full URL stays only in sidecar memory for the
+  fetch. Honors `WB_BROWSER_DOWNLOAD_EXTENSIONS`.
+- The saved frame carries `provenance.capture: "signed_url"` plus `api_url`,
+  `field`, `content_type`, and `content_disposition`.
+- A 403 on the signed URL (expired token) emits `slice.download_failed` with
+  `expired: true` and `http_status: 403` so the operator knows to shorten the
+  click→fetch gap.
+
 ## Protocol
 
 Line-framed JSON, one message per line, on stdin/stdout. `stderr` is treated as
@@ -321,9 +375,26 @@ opaque diagnostics by `wb` and printed dimmed to the user's terminal.
 ```
 wb  →  {"type": "hello", "wb_version": "...", "protocol": "wb-sidecar/1"}
 wb  ←  {"type": "ready", "runtime": "wb-browser-runtime", "version": "...",
-        "protocol": "wb-sidecar/1", "supports": ["goto", "click", "fill", ...]}
+        "protocol": "wb-sidecar/1", "min_protocol": "wb-sidecar/1",
+        "supports": ["goto", "click", "fill", ...],
+        "features": ["recording", "pause", "substitution",
+                     "substitution_escape", "download_capture",
+                     "signed_url_download"]}
 ```
 
+The `ready` frame advertises capabilities so a client can feature-detect
+without a hard-coded version→capability map:
+
+- `protocol` — the wire version this runtime speaks.
+- `min_protocol` — the oldest protocol version it can still interoperate with
+  (equal to `protocol` until a breaking frame change ships). A client speaking
+  an older protocol than `min_protocol` should refuse rather than guess.
+- `supports` — the per-verb list (derived from the verb registry).
+- `features` — coarse capability tokens above the verb list.
+
+`version` is read from `package.json` at boot, so it can never drift from the
+published version.
+
 ### Slice
 
 ```

package/bin/wb-browser-runtime.js

@@ -25,9 +25,7 @@
 
 import readline from "node:readline";
 import { chromium } from "playwright-core";
-import { readFileSync } from "node:fs";
 import { send, log } from "../lib/io.js";
-import { resolveInside } from "../lib/util.js";
 import { SessionManager } from "../lib/session-manager.js";
 import {
   RecordingManager,
@@ -40,9 +38,31 @@ import {
   classifyError,
 } from "../lib/failure.js";
 import { installDownloadCapture } from "../lib/download-capture.js";
+import { expand, scrubSecrets } from "../lib/substitution.js";
 import { SUPPORTS, runVerb, verbName } from "../verbs/index.js";
+import pkg from "../package.json" with { type: "json" };
+
+// Read the version from package.json so the `ready` frame can never drift from
+// the published version (it used to be a hand-maintained literal that fell out
+// of sync). Node >=24 supports JSON import attributes natively.
+const VERSION = pkg.version;
+
+// Protocol capability advertisement. `protocol` is the wire version we speak;
+// `min_protocol` is the oldest version a peer may speak and still interoperate
+// (we keep it equal to `protocol` until we ship a breaking frame change).
+// `features` is a coarse capability list above the per-verb `supports` array —
+// a client can feature-detect without hard-coding a version→capability map.
+const PROTOCOL = "wb-sidecar/1";
+const MIN_PROTOCOL = "wb-sidecar/1";
+const FEATURES = [
+  "recording", // rrweb DOM capture + CDP screencast video
+  "pause", // pause_for_human operator handoff
+  "substitution", // {{ env.X }} / {{ artifacts.X }}
+  "substitution_escape", // \{{ literal-brace escape
+  "download_capture", // passive + explicit download artifact capture
+  "signed_url_download", // server-side fetch of in-JSON signed export URLs
+];
 
-const VERSION = "0.8.0";
 const provider = getProvider();
 log(`[provider] ${provider.name}`);
 
@@ -158,108 +178,9 @@ async function ensureSession(name, { profile, restoreSession } = {}) {
     }
   });
 }
-// --- {{ env.X }} / {{ artifacts.X }} substitution --------------------------
-
-const ENV_RE = /\{\{\s*env\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
-// Artifact names are bare identifiers — no dots, no slashes. Anything more
-// exotic would invite path traversal once composed with WB_ARTIFACTS_DIR.
-const ARTIFACT_RE = /\{\{\s*artifacts\.([A-Za-z_][A-Za-z0-9_-]*)\s*\}\}/g;
-
-// Resolved once at module load. `warn` matches historical behavior
-// (log + empty string, runbook continues). `error` throws so a missing OTP
-// or env var fails the slice instead of silently sending an empty value
-// into a Playwright action. `empty` is the silent variant.
-const ON_MISSING = (() => {
-  const raw = (process.env.WB_SUBSTITUTION_ON_MISSING || "warn")
-    .trim()
-    .toLowerCase();
-  if (raw === "error" || raw === "empty" || raw === "warn") return raw;
-  log(
-    `[warn] WB_SUBSTITUTION_ON_MISSING=${raw} is not valid (warn|error|empty); defaulting to warn`,
-  );
-  return "warn";
-})();
-
-function handleMissingSubstitution(kind, name) {
-  const msg = `${kind}.${name} is not set`;
-  if (ON_MISSING === "error") {
-    throw new Error(`substitution: ${msg}`);
-  }
-  if (ON_MISSING === "warn") {
-    log(`[warn] ${msg}; substituting empty string`);
-  }
-  return "";
-}
-
-function readArtifactRaw(name) {
-  const dir = (process.env.WB_ARTIFACTS_DIR || "").trim();
-  if (!dir) {
-    log(`[warn] artifacts.${name} referenced but WB_ARTIFACTS_DIR is not set`);
-    return null;
-  }
-  for (const candidate of [`${name}.txt`, name]) {
-    const full = resolveInside(dir, candidate);
-    if (!full) continue;
-    try {
-      return readFileSync(full, "utf8").trimEnd();
-    } catch {
-      // try next candidate
-    }
-  }
-  return null;
-}
-
-function readArtifact(name, cache) {
-  if (cache && cache.has(name)) {
-    const hit = cache.get(name);
-    if (hit === null) return handleMissingSubstitution("artifacts", name);
-    return hit;
-  }
-  const v = readArtifactRaw(name);
-  if (cache) cache.set(name, v);
-  if (v === null) return handleMissingSubstitution("artifacts", name);
-  return v;
-}
-
-function expand(value, collected, artifactCache) {
-  if (typeof value === "string") {
-    return value
-      .replace(ENV_RE, (_, name) => {
-        const v = process.env[name];
-        if (v === undefined) return handleMissingSubstitution("env", name);
-        if (collected && v.length >= 3) collected.add(v);
-        return v;
-      })
-      .replace(ARTIFACT_RE, (_, name) => {
-        const v = readArtifact(name, artifactCache);
-        if (collected && v && v.length >= 3) collected.add(v);
-        return v;
-      });
-  }
-  if (Array.isArray(value))
-    return value.map((v) => expand(v, collected, artifactCache));
-  if (value && typeof value === "object") {
-    const out = {};
-    for (const [k, v] of Object.entries(value))
-      out[k] = expand(v, collected, artifactCache);
-    return out;
-  }
-  return value;
-}
-
-// Scrub any values that came from {{ env.X }} / {{ artifacts.X }} expansion
-// out of error messages before they cross the stdio boundary — Playwright and
-// fetch errors sometimes echo their inputs (URLs, script bodies, assertion
-// text) and those inputs may contain credentials.
-function scrubSecrets(msg, secrets) {
-  let out = String(msg == null ? "" : msg);
-  if (!secrets) return out;
-  for (const s of secrets) {
-    if (!s) continue;
-    out = out.split(s).join("«***»");
-  }
-  return out;
-}
+// {{ env.X }} / {{ artifacts.X }} substitution + `\{{` escape + secret scrubbing
+// live in lib/substitution.js (extracted so they're unit-testable without
+// booting the sidecar).
 
 // --- Slice handler ----------------------------------------------------------
 
@@ -559,8 +480,10 @@ rl.on("line", (line) => {
         type: "ready",
         runtime: "wb-browser-runtime",
         version: VERSION,
-        protocol: "wb-sidecar/1",
+        protocol: PROTOCOL,
+        min_protocol: MIN_PROTOCOL,
         supports: SUPPORTS,
+        features: FEATURES,
       });
       break;
     case "slice":

package/lib/recording-manager.js

@@ -148,6 +148,29 @@ export function loadRecordingConfig() {
     kinds,
     rrwebSource,
     rrwebMaxEvents,
+    mask: loadMaskConfig(),
+  };
+}
+
+// rrweb's `maskAllInputs` only redacts input *values* — labels, placeholders,
+// aria-labels, option text, and the full DOM structure are still recorded. For
+// genuinely sensitive regions (a displayed SSN, an account balance) the author
+// must point rrweb at the offending nodes with a CSS selector. These knobs
+// expose rrweb's selector options without hard-coding them:
+//   WB_RECORDING_MASK_ALL_INPUTS  default on; set "0" to record input values
+//   WB_RECORDING_MASK_TEXT_SELECTOR  text content of matches → asterisks
+//   WB_RECORDING_BLOCK_SELECTOR      matches recorded as inert placeholders
+//   WB_RECORDING_IGNORE_SELECTOR     matches' input events dropped entirely
+export function loadMaskConfig() {
+  const sel = (name) => {
+    const v = (process.env[name] || "").trim();
+    return v || null;
+  };
+  return {
+    maskAllInputs: process.env.WB_RECORDING_MASK_ALL_INPUTS !== "0",
+    maskTextSelector: sel("WB_RECORDING_MASK_TEXT_SELECTOR"),
+    blockSelector: sel("WB_RECORDING_BLOCK_SELECTOR"),
+    ignoreSelector: sel("WB_RECORDING_IGNORE_SELECTOR"),
   };
 }
 
@@ -221,17 +244,29 @@ export class RecordingManager {
             for (const e of batch) pushRrweb(e);
           }
         });
+        // Build rrweb record options from the resolved mask config. Selector
+        // options are omitted entirely when unset so we don't pass `null` into
+        // rrweb (which would match nothing but still allocate a matcher).
+        const mask = cfg.mask || { maskAllInputs: true };
+        const recordOpts = {
+          maskAllInputs: mask.maskAllInputs !== false,
+        };
+        if (mask.maskTextSelector)
+          recordOpts.maskTextSelector = mask.maskTextSelector;
+        if (mask.blockSelector) recordOpts.blockSelector = mask.blockSelector;
+        if (mask.ignoreSelector)
+          recordOpts.ignoreSelector = mask.ignoreSelector;
+        const recordOptsJson = JSON.stringify(recordOpts);
         const bootstrap = `
 ;(function(){
   if (window.__wbRrwebActive) return;
   window.__wbRrwebActive = true;
   window.__wbRrwebBuffer = [];
   try {
-    rrwebRecord({
+    rrwebRecord(Object.assign({
       emit: function(event){ window.__wbRrwebBuffer.push(event); },
-      sampling: { scroll: 150, media: 800, input: 'last' },
-      maskAllInputs: true
-    });
+      sampling: { scroll: 150, media: 800, input: 'last' }
+    }, ${recordOptsJson}));
   } catch (e) { /* rrweb unavailable on this page (e.g. chrome://) */ }
   var flush = function(){
     var buf = window.__wbRrwebBuffer;

package/lib/signed-url-capture.js

@@ -0,0 +1,275 @@
+// Signed-URL export capture.
+//
+// Some SaaS "Download" buttons never produce a Playwright `download` event or
+// an in-page Blob. Instead the app calls a same-origin API that returns JSON
+// like `{ "download_url": "https://bucket.s3.amazonaws.com/...?<signed>" }`
+// and then navigates to that URL. A page-side `fetch(signedUrl)` usually fails
+// because the object store's CORS policy doesn't allow the app origin to read
+// the bytes — so the only reliable place to fetch it is the sidecar process,
+// where CORS doesn't apply.
+//
+// This module supplies:
+//   - SIGNED_PAGE_HOOK / SIGNED_POLL_SCRIPT — page-side instrumentation that
+//     wraps fetch + XHR, inspects small same-origin JSON responses, and stashes
+//     any http(s) URL fields it finds on `window.__wbSignedCandidates`.
+//   - pure helpers (isSignedHost, redactSignedUrl, extractUrlFields,
+//     pickSignedCandidate, parseSignedConfig) that the `download` verb uses to
+//     decide which captured URL to fetch server-side.
+//
+// The bytes are downloaded by the verb via lib/http.js's retryableFetch. Signed
+// query credentials are redacted everywhere they cross the stdio boundary; the
+// full URL lives only in sidecar memory for the duration of the fetch.
+
+// Object-store / CDN hosts that hand out pre-signed, credential-bearing URLs.
+// A match means "this looks like an export download, not a normal app API call"
+// — the gate that keeps auto-mode from grabbing arbitrary same-origin URLs.
+const SIGNED_HOST_PATTERNS = [
+  // S3: s3.amazonaws.com, s3.us-east-1.amazonaws.com, s3-us-west-2.amazonaws.com,
+  //     bucket.s3.amazonaws.com, bucket.s3.us-east-1.amazonaws.com
+  /(^|\.)s3([.-][a-z0-9-]+)?\.amazonaws\.com$/i,
+  // Google Cloud Storage
+  /(^|\.)storage\.googleapis\.com$/i,
+  /(^|\.)storage\.cloud\.google\.com$/i,
+  // CloudFront
+  /(^|\.)cloudfront\.net$/i,
+  // Azure Blob Storage
+  /\.blob\.core\.windows\.net$/i,
+  // Cloudflare R2
+  /\.r2\.cloudflarestorage\.com$/i,
+];
+
+export function isSignedHost(host) {
+  if (!host) return false;
+  const h = String(host).toLowerCase();
+  return SIGNED_HOST_PATTERNS.some((re) => re.test(h));
+}
+
+// Drop the query string (where signed credentials live) but keep origin + path
+// so diagnostics stay useful. Falls back to a naive split when URL parsing
+// fails so a malformed value still can't leak its query.
+export function redactSignedUrl(url) {
+  const s = String(url || "");
+  try {
+    const u = new URL(s);
+    return u.search ? `${u.origin}${u.pathname}?<redacted>` : `${u.origin}${u.pathname}`;
+  } catch {
+    const i = s.indexOf("?");
+    return i >= 0 ? `${s.slice(0, i)}?<redacted>` : s;
+  }
+}
+
+// Recursively collect every http(s) string value in a parsed JSON object.
+// Bounded in depth and count so a pathological response can't blow the stack
+// or the buffer. Returns `[{ field, url }]` with dotted field paths.
+export function extractUrlFields(data, maxDepth = 6, maxUrls = 30) {
+  const out = [];
+  const visit = (node, fieldPath, depth) => {
+    if (depth > maxDepth || out.length >= maxUrls) return;
+    if (typeof node === "string") {
+      if (/^https?:\/\//i.test(node)) out.push({ field: fieldPath, url: node });
+      return;
+    }
+    if (Array.isArray(node)) {
+      for (let i = 0; i < node.length; i++)
+        visit(node[i], `${fieldPath}[${i}]`, depth + 1);
+      return;
+    }
+    if (node && typeof node === "object") {
+      for (const k of Object.keys(node)) {
+        visit(node[k], fieldPath ? `${fieldPath}.${k}` : k, depth + 1);
+      }
+    }
+  };
+  visit(data, "", 0);
+  return out;
+}
+
+// Leaf field name from a dotted/indexed path: "data.export.download_url" →
+// "download_url", "files[0].url" → "url". Used to match against a caller's
+// `json_fields` allowlist.
+function leafField(fieldPath) {
+  const last = String(fieldPath || "")
+    .split(".")
+    .pop();
+  return last.replace(/\[\d+\]$/, "");
+}
+
+// Normalize `args.signed_url` into a resolved policy.
+//   undefined / "auto"        → { enabled: "auto" }   (recognized hosts only)
+//   false / { enabled: false } → { enabled: false }   (feature off)
+//   true                       → { enabled: true }
+//   { enabled, hosts, json_fields } → that, normalized
+// In "auto" mode only recognized signed hosts (or an explicit `hosts` entry)
+// are captured. In forced mode (`enabled: true`) an explicit `hosts` or
+// `json_fields` match is honored even for an unrecognized host, since the
+// author asked for it by name.
+export function parseSignedConfig(raw) {
+  const norm = (cfg) => ({
+    enabled: cfg.enabled,
+    hosts: Array.isArray(cfg.hosts) ? cfg.hosts.map((h) => String(h).toLowerCase()) : [],
+    jsonFields: cfg.jsonFields && cfg.jsonFields.length ? cfg.jsonFields.map(String) : null,
+  });
+  if (raw === false) return { enabled: false, hosts: [], jsonFields: null };
+  if (raw === true) return norm({ enabled: true });
+  if (raw == null) return norm({ enabled: "auto" });
+  if (typeof raw === "object") {
+    const enabled =
+      raw.enabled === undefined
+        ? "auto"
+        : raw.enabled === true
+          ? true
+          : raw.enabled === false
+            ? false
+            : "auto";
+    if (enabled === false) return { enabled: false, hosts: [], jsonFields: null };
+    const jsonFields = Array.isArray(raw.json_fields)
+      ? raw.json_fields
+      : Array.isArray(raw.jsonFields)
+        ? raw.jsonFields
+        : null;
+    return norm({ enabled, hosts: raw.hosts, jsonFields });
+  }
+  return norm({ enabled: "auto" });
+}
+
+// Choose the best signed-URL candidate from the page-captured list, or null.
+// `candidates` is the shape pushed by SIGNED_PAGE_HOOK:
+//   [{ api_url, urls: [{ field, url }], ts }]
+export function pickSignedCandidate(candidates, opts = {}) {
+  const hosts = opts.hosts || [];
+  const jsonFields = opts.jsonFields || null;
+  const forced = opts.enabled === true;
+  for (const cand of candidates || []) {
+    for (const u of cand.urls || []) {
+      if (jsonFields && !jsonFields.includes(leafField(u.field)) && !jsonFields.includes(u.field))
+        continue;
+      let host = "";
+      try {
+        host = new URL(u.url).host.toLowerCase();
+      } catch {
+        continue;
+      }
+      const hostAllowed =
+        hosts.length > 0 &&
+        hosts.some((h) => host === h || host.endsWith(`.${h}`));
+      const looksSigned = isSignedHost(host);
+      // auto: only recognized signed hosts or an explicit hosts allowlist.
+      // forced: also honor a candidate the author selected via hosts/json_fields.
+      const accept =
+        hostAllowed ||
+        looksSigned ||
+        (forced && (hosts.length > 0 || jsonFields));
+      if (accept) {
+        return { url: u.url, field: u.field, api_url: cand.api_url || null, host };
+      }
+    }
+  }
+  return null;
+}
+
+// Page-side hook installed BEFORE the click. Wraps fetch + XHR to inspect small
+// same-origin JSON responses and stash any http(s) URL fields. Idempotent and
+// fail-open — any error in the wrapper falls through to the original call so the
+// app keeps working. Mirrors the blob hook's "never uninstall" contract.
+export const SIGNED_PAGE_HOOK = `(() => {
+  if (window.__wbSignedInstalled) return;
+  window.__wbSignedInstalled = true;
+  window.__wbSignedCandidates = [];
+  var MAX_BODY = 64 * 1024;
+  var MAX_CAND = 50;
+
+  var sameOrigin = function(u){
+    try { return new URL(u, location.href).origin === location.origin; }
+    catch (e) { return false; }
+  };
+
+  var collect = function(apiUrl, text){
+    try {
+      if (!text || text.length > MAX_BODY) return;
+      var data;
+      try { data = JSON.parse(text); } catch (e) { return; }
+      var urls = [];
+      var visit = function(node, fp, depth){
+        if (depth > 6 || urls.length >= 30) return;
+        if (typeof node === 'string') {
+          if (/^https?:\\/\\//i.test(node)) urls.push({ field: fp, url: node });
+          return;
+        }
+        if (Array.isArray(node)) {
+          for (var i = 0; i < node.length; i++) visit(node[i], fp + '[' + i + ']', depth + 1);
+          return;
+        }
+        if (node && typeof node === 'object') {
+          for (var k in node) {
+            if (Object.prototype.hasOwnProperty.call(node, k)) {
+              visit(node[k], fp ? fp + '.' + k : k, depth + 1);
+            }
+          }
+        }
+      };
+      visit(data, '', 0);
+      if (urls.length && window.__wbSignedCandidates.length < MAX_CAND) {
+        window.__wbSignedCandidates.push({ api_url: apiUrl, urls: urls, ts: Date.now() });
+      }
+    } catch (e) {}
+  };
+
+  var origFetch = window.fetch;
+  if (typeof origFetch === 'function') {
+    window.fetch = function(){
+      var args = arguments;
+      var reqUrl = '';
+      try { reqUrl = (typeof args[0] === 'string') ? args[0] : (args[0] && args[0].url) || ''; } catch (e) {}
+      var p = origFetch.apply(this, args);
+      try {
+        if (sameOrigin(reqUrl) && p && typeof p.then === 'function') {
+          p.then(function(res){
+            try {
+              var ct = (res && res.headers && res.headers.get && res.headers.get('content-type')) || '';
+              if (/json|text/i.test(ct) || ct === '') {
+                res.clone().text().then(function(t){ collect(reqUrl, t); }).catch(function(){});
+              }
+            } catch (e) {}
+            return res;
+          }).catch(function(){});
+        }
+      } catch (e) {}
+      return p;
+    };
+  }
+
+  var XHR = window.XMLHttpRequest;
+  if (XHR && XHR.prototype) {
+    var origOpen = XHR.prototype.open;
+    var origSend = XHR.prototype.send;
+    XHR.prototype.open = function(method, url){
+      try { this.__wbUrl = url; } catch (e) {}
+      return origOpen.apply(this, arguments);
+    };
+    XHR.prototype.send = function(){
+      try {
+        var self = this;
+        this.addEventListener('load', function(){
+          try {
+            if (sameOrigin(self.__wbUrl)) {
+              var rt = '';
+              try {
+                if (self.responseType === '' || self.responseType === 'text') rt = self.responseText;
+              } catch (e) {}
+              if (rt) collect(self.__wbUrl, rt);
+            }
+          } catch (e) {}
+        });
+      } catch (e) {}
+      return origSend.apply(this, arguments);
+    };
+  }
+})()`;
+
+// Read-and-clear of the candidate buffer so successive polls only see new
+// responses (matches the blob hook's read-and-clear contract).
+export const SIGNED_POLL_SCRIPT = `(() => {
+  var c = window.__wbSignedCandidates || [];
+  window.__wbSignedCandidates = [];
+  return c;
+})()`;

package/lib/substitution.js

@@ -0,0 +1,128 @@
+// Verb-argument substitution: {{ env.X }} / {{ artifacts.X }} expansion plus a
+// `\{{` escape for literal template braces. Extracted from the entry point so
+// it's unit-testable without booting the sidecar.
+//
+//   {{ env.NAME }}        → process.env.NAME
+//   {{ artifacts.NAME }}  → contents of $WB_ARTIFACTS_DIR/NAME.txt (or .../NAME)
+//   \{{                   → literal "{{" (escape; braces are NOT re-scanned)
+//
+// `expand(value, collected, artifactCache)` walks strings/arrays/objects.
+// Resolved secret-ish values (≥3 chars) are added to `collected` so the caller
+// can scrub them out of error messages with `scrubSecrets`.
+
+import { readFileSync } from "node:fs";
+import { log } from "./io.js";
+import { resolveInside } from "./util.js";
+
+// One combined pattern, scanned left-to-right in a single pass so the escape
+// branch consumes the braces before either substitution branch can see them.
+// Alternation order matters: the escape must come first.
+//   \{{                   → no capture group (escape)
+//   {{ env.NAME }}        → group 1
+//   {{ artifacts.NAME }}  → group 2
+// Artifact names are bare identifiers — no dots, no slashes — so a name can't
+// compose with WB_ARTIFACTS_DIR into a path-traversal read.
+const SUBST_RE =
+  /\\\{\{|\{\{\s*env\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}|\{\{\s*artifacts\.([A-Za-z_][A-Za-z0-9_-]*)\s*\}\}/g;
+
+let warnedInvalidPolicy = false;
+
+// Resolve the missing-value policy fresh each call (cheap) so the behavior
+// tracks the current env. `warn` matches historical behavior (log + empty
+// string, runbook continues). `error` throws so a missing OTP fails the slice
+// instead of silently sending an empty value into a Playwright action. `empty`
+// is the silent variant.
+function resolveOnMissing() {
+  const raw = (process.env.WB_SUBSTITUTION_ON_MISSING || "warn").trim().toLowerCase();
+  if (raw === "error" || raw === "empty" || raw === "warn") return raw;
+  if (!warnedInvalidPolicy) {
+    warnedInvalidPolicy = true;
+    log(
+      `[warn] WB_SUBSTITUTION_ON_MISSING=${raw} is not valid (warn|error|empty); defaulting to warn`,
+    );
+  }
+  return "warn";
+}
+
+function handleMissingSubstitution(kind, name) {
+  const msg = `${kind}.${name} is not set`;
+  if (resolveOnMissing() === "error") {
+    throw new Error(`substitution: ${msg}`);
+  }
+  if (resolveOnMissing() === "warn") {
+    log(`[warn] ${msg}; substituting empty string`);
+  }
+  return "";
+}
+
+function readArtifactRaw(name) {
+  const dir = (process.env.WB_ARTIFACTS_DIR || "").trim();
+  if (!dir) {
+    log(`[warn] artifacts.${name} referenced but WB_ARTIFACTS_DIR is not set`);
+    return null;
+  }
+  for (const candidate of [`${name}.txt`, name]) {
+    const full = resolveInside(dir, candidate);
+    if (!full) continue;
+    try {
+      return readFileSync(full, "utf8").trimEnd();
+    } catch {
+      // try next candidate
+    }
+  }
+  return null;
+}
+
+function readArtifact(name, cache) {
+  if (cache && cache.has(name)) {
+    const hit = cache.get(name);
+    if (hit === null) return handleMissingSubstitution("artifacts", name);
+    return hit;
+  }
+  const v = readArtifactRaw(name);
+  if (cache) cache.set(name, v);
+  if (v === null) return handleMissingSubstitution("artifacts", name);
+  return v;
+}
+
+export function expand(value, collected, artifactCache) {
+  if (typeof value === "string") {
+    return value.replace(SUBST_RE, (m, envName, artName) => {
+      // Escape branch: `\{{` → literal `{{`. No capture group, so both
+      // envName and artName are undefined here.
+      if (envName === undefined && artName === undefined) return "{{";
+      if (envName !== undefined) {
+        const v = process.env[envName];
+        if (v === undefined) return handleMissingSubstitution("env", envName);
+        if (collected && v.length >= 3) collected.add(v);
+        return v;
+      }
+      const v = readArtifact(artName, artifactCache);
+      if (collected && v && v.length >= 3) collected.add(v);
+      return v;
+    });
+  }
+  if (Array.isArray(value))
+    return value.map((v) => expand(v, collected, artifactCache));
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const [k, v] of Object.entries(value))
+      out[k] = expand(v, collected, artifactCache);
+    return out;
+  }
+  return value;
+}
+
+// Scrub any values that came from {{ env.X }} / {{ artifacts.X }} expansion out
+// of error messages before they cross the stdio boundary — Playwright and fetch
+// errors sometimes echo their inputs (URLs, script bodies, assertion text) and
+// those inputs may contain credentials.
+export function scrubSecrets(msg, secrets) {
+  let out = String(msg == null ? "" : msg);
+  if (!secrets) return out;
+  for (const s of secrets) {
+    if (!s) continue;
+    out = out.split(s).join("«***»");
+  }
+  return out;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wb-browser-runtime",
-  "version": "0.13.0",
+  "version": "0.14.0",
   "description": "Browser sidecar runtime for wb — Playwright over CDP (Browserbase, browser-use) via the wb-sidecar/1 line-framed JSON protocol.",
   "bin": {
     "wb-browser-runtime": "bin/wb-browser-runtime.js"

package/verbs/download.js

@@ -26,6 +26,14 @@ import {
   extensionAllowed,
 } from "../lib/util.js";
 import { HANDLED_MARK } from "../lib/download-capture.js";
+import { retryableFetch } from "../lib/http.js";
+import {
+  SIGNED_PAGE_HOOK,
+  SIGNED_POLL_SCRIPT,
+  parseSignedConfig,
+  pickSignedCandidate,
+  redactSignedUrl,
+} from "../lib/signed-url-capture.js";
 
 const DEFAULT_TIMEOUT_MS = 10_000;
 const POLL_INTERVAL_MS = 50;
@@ -114,15 +122,24 @@ export default {
     const allowlist = parseExtensionAllowlist(
       process.env.WB_BROWSER_DOWNLOAD_EXTENSIONS,
     );
+    const signedCfg = parseSignedConfig(args.signed_url);
+    const signedEnabled = signedCfg.enabled !== false;
 
     // 1) Inject the page-side blob/anchor capture hook BEFORE the click so a
     //    synchronously-dispatched anchor.click() inside the SPA's handler is
     //    observed. Best-effort: a frame mid-navigation can reject evaluate;
     //    the Playwright `download` event still works and is the primary
-    //    signal anyway.
+    //    signal anyway. When signed-URL capture is enabled, install its
+    //    fetch/XHR response hook in the same pre-click window so the API call
+    //    the click triggers is observed from the start.
     try {
       await page.evaluate(PAGE_HOOK);
     } catch {}
+    if (signedEnabled) {
+      try {
+        await page.evaluate(SIGNED_PAGE_HOOK);
+      } catch {}
+    }
 
     // 2) Claim ownership of the next download synchronously — prepended to
     //    BrowserContext listeners so it runs before lib/download-capture.js's
@@ -148,16 +165,25 @@ export default {
       }
     }
 
+    // Shared cancellation token: once the race has a winner, the losing
+    // pollers stop on their next tick instead of spinning page.evaluate against
+    // a (possibly navigating/closing) page for the rest of the timeout window.
+    const stop = { done: false };
+
     try {
-      // 3) Race the two capture sources against the click. The download event
-      //    AND the click run concurrently — Playwright's standard pattern,
-      //    since the click can resolve before or after the download fires.
+      // 3) Race the capture sources against the click. The download event AND
+      //    the click run concurrently — Playwright's standard pattern, since
+      //    the click can resolve before or after the download fires.
       const downloadPromise = page
         .waitForEvent("download", { timeout })
         .then((d) => ({ kind: "playwright", download: d }))
         .catch((e) => ({ kind: "playwright_failed", error: e }));
 
-      const blobPromise = pollForBlob(page, timeout);
+      const blobPromise = pollForBlob(page, timeout, stop);
+
+      const signedPromise = signedEnabled
+        ? pollForSignedUrl(page, timeout, signedCfg, stop)
+        : null;
 
       let clickError = null;
       const clickPromise = (async () => {
@@ -181,7 +207,11 @@ export default {
         }
       })();
 
-      const winner = await raceCaptures(downloadPromise, blobPromise);
+      const winner = await raceCaptures(
+        [downloadPromise, blobPromise, signedPromise].filter(Boolean),
+      );
+      // Winner decided (success or all-failed) — release the losing pollers.
+      stop.done = true;
       // Wait for the click to settle so we surface its error (if any) over
       // a generic "no file captured" — a click that never landed is the
       // more actionable failure.
@@ -208,6 +238,17 @@ export default {
           ctx,
         });
       }
+      if (winner.success && winner.kind === "signed_url") {
+        return await saveSignedUrlDownload({
+          signed: winner.signed,
+          artifactsDir,
+          allowlist,
+          explicitPath,
+          page,
+          ctx,
+          timeout,
+        });
+      }
 
       // No capture won — emit structured failure diagnostics.
       const reasons = winner.failures
@@ -217,6 +258,9 @@ export default {
           }
           if (f.kind === "blob_failed") return `blob hook: ${f.error}`;
           if (f.kind === "blob_timeout") return `blob hook: no capture within ${timeout}ms`;
+          if (f.kind === "signed_failed") return `signed url: ${f.error}`;
+          if (f.kind === "signed_timeout")
+            return `signed url: no signed file URL seen within ${timeout}ms`;
           return f.kind;
         })
         .join("; ");
@@ -233,6 +277,9 @@ export default {
         `download: no file captured within ${timeout}ms after clicking ${args.selector} (page=${safePageUrl(page) || "?"}). ${reasons}`,
       );
     } finally {
+      // Backstop: ensure pollers are released on any exit path (thrown
+      // click/save error, extension rejection, etc.).
+      stop.done = true;
       if (attached && browserContext && typeof browserContext.off === "function") {
         try {
           browserContext.off("download", claim);
@@ -333,13 +380,17 @@ async function saveBlobDownload({
   return `→ ${path.basename(target)}`;
 }
 
-// Race two capture promises. First to report success wins. Both must report
-// before we declare failure, so the diagnostics frame can list every reason
-// the verb didn't see a file. (Promise.race would shortcut on a fast failure
-// and discard the slower success.)
-function raceCaptures(downloadPromise, blobPromise) {
+// Race N capture promises. First to report success wins. Every source must
+// report before we declare failure, so the diagnostics frame can list every
+// reason the verb didn't see a file. (Promise.race would shortcut on a fast
+// failure and discard a slower success.) Each promise resolves to an object
+// whose `kind` names the source: a success kind ("playwright" | "blob" |
+// "signed_url") or a failure kind ("*_failed" | "*_timeout").
+const SUCCESS_KINDS = new Set(["playwright", "blob", "signed_url"]);
+
+function raceCaptures(promises) {
   return new Promise((resolve) => {
-    let outstanding = 2;
+    let outstanding = promises.length;
     const failures = [];
     const finish = (settled) => {
       if (settled.success) {
@@ -349,20 +400,123 @@ function raceCaptures(downloadPromise, blobPromise) {
       failures.push(settled);
       if (--outstanding === 0) resolve({ success: false, failures });
     };
-    downloadPromise.then((r) => {
-      if (r.kind === "playwright") finish({ success: true, ...r });
-      else finish({ success: false, ...r });
+    for (const pr of promises) {
+      pr.then((r) => {
+        if (SUCCESS_KINDS.has(r.kind)) finish({ success: true, ...r });
+        else finish({ success: false, ...r });
+      });
+    }
+  });
+}
+
+async function saveSignedUrlDownload({
+  signed,
+  artifactsDir,
+  allowlist,
+  explicitPath,
+  page,
+  ctx,
+  timeout,
+}) {
+  const redacted = redactSignedUrl(signed.url);
+  // Filename: explicit path: wins, else the signed URL's basename, else a
+  // generic fallback. (S3 keys usually end in the real filename.)
+  let nameFromUrl = "";
+  try {
+    nameFromUrl = path.basename(new URL(signed.url).pathname) || "";
+  } catch {}
+  const suggested = explicitPath || (nameFromUrl.trim() ? nameFromUrl : FALLBACK_NAME);
+  if (!extensionAllowed(suggested, allowlist)) {
+    throw new Error(
+      `download: file "${suggested}" rejected by WB_BROWSER_DOWNLOAD_EXTENSIONS`,
+    );
+  }
+  const target = uniquePathInside(artifactsDir, suggested);
+  if (!target) {
+    throw new Error(
+      `download: refusing to save "${suggested}" — resolves outside $WB_ARTIFACTS_DIR`,
+    );
+  }
+  await fsPromises.mkdir(artifactsDir, { recursive: true });
+
+  // Fetch the signed URL from the sidecar (not the page) so the object store's
+  // CORS policy doesn't block the read. The label is redacted — retry logs must
+  // never echo signed credentials.
+  let res;
+  try {
+    res = await retryableFetch(
+      signed.url,
+      { method: "GET" },
+      `signed-url download (${redacted})`,
+      { timeoutMs: timeout },
+    );
+  } catch (e) {
+    send({
+      type: "slice.download_failed",
+      verb: "download",
+      verb_index: ctx?.index ?? null,
+      capture: "signed_url",
+      api_url: signed.api_url,
+      signed_url: redacted,
+      page_url: safePageUrl(page),
+      reason: `signed url fetch error: ${e?.message || e}`,
     });
-    blobPromise.then((r) => {
-      if (r.kind === "blob") finish({ success: true, ...r });
-      else finish({ success: false, ...r });
+    throw new Error(
+      `download: signed URL fetch failed for ${redacted}: ${e?.message || e}`,
+    );
+  }
+  if (!res.ok) {
+    // A 403 on a pre-signed URL almost always means the token expired before
+    // we fetched it — call that out so the operator knows to shorten the gap.
+    const expired = res.status === 403;
+    send({
+      type: "slice.download_failed",
+      verb: "download",
+      verb_index: ctx?.index ?? null,
+      capture: "signed_url",
+      api_url: signed.api_url,
+      signed_url: redacted,
+      page_url: safePageUrl(page),
+      http_status: res.status,
+      expired,
+      reason: `signed url fetch: HTTP ${res.status}${expired ? " (likely expired)" : ""}`,
     });
+    throw new Error(
+      `download: signed URL fetch returned HTTP ${res.status} for ${redacted}${expired ? " (likely expired)" : ""}`,
+    );
+  }
+  const buf = Buffer.from(await res.arrayBuffer());
+  await fsPromises.writeFile(target, buf);
+  const contentType = safeHeader(res, "content-type");
+  const contentDisposition = safeHeader(res, "content-disposition");
+  send({
+    type: "slice.artifact_saved",
+    filename: path.basename(target),
+    path: target,
+    bytes: buf.length,
+    source: "download",
+    provenance: {
+      url: null,
+      signed_url: redacted,
+      api_url: signed.api_url,
+      field: signed.field,
+      suggested_filename: suggested,
+      page_url: safePageUrl(page),
+      verb_index: ctx?.index ?? null,
+      verb_name: "download",
+      capture: "signed_url",
+      content_type: contentType,
+      content_disposition: contentDisposition,
+      ts: Date.now(),
+    },
   });
+  return `→ ${path.basename(target)}`;
 }
 
-async function pollForBlob(page, timeoutMs) {
+async function pollForBlob(page, timeoutMs, stop) {
   const deadline = Date.now() + timeoutMs;
   while (true) {
+    if (stop?.done) return { kind: "blob_timeout" };
     let result;
     try {
       result = await page.evaluate(POLL_SCRIPT);
@@ -376,6 +530,38 @@ async function pollForBlob(page, timeoutMs) {
   }
 }
 
+// Poll the page-side signed-URL candidate buffer until a candidate matching
+// the configured policy appears or the deadline passes. The bytes are NOT
+// fetched here — the winner is fetched server-side by saveSignedUrlDownload so
+// CORS doesn't apply. Returns the picked candidate; never throws (page
+// evaluate errors degrade to "keep polling").
+async function pollForSignedUrl(page, timeoutMs, signedCfg, stop) {
+  const deadline = Date.now() + timeoutMs;
+  while (true) {
+    if (stop?.done) return { kind: "signed_timeout" };
+    let cands = null;
+    try {
+      cands = await page.evaluate(SIGNED_POLL_SCRIPT);
+    } catch {
+      cands = null;
+    }
+    if (Array.isArray(cands) && cands.length) {
+      const picked = pickSignedCandidate(cands, signedCfg);
+      if (picked) return { kind: "signed_url", signed: picked };
+    }
+    if (Date.now() >= deadline) return { kind: "signed_timeout" };
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+  }
+}
+
+function safeHeader(res, name) {
+  try {
+    return res.headers?.get?.(name) || null;
+  } catch {
+    return null;
+  }
+}
+
 function safePageUrl(page) {
   try {
     return page.url();