wb-browser-runtime 0.4.0 → 0.5.1

package/README.md

@@ -29,6 +29,21 @@ Verb arguments support `{{ env.NAME }}` substitution at dispatch time, so any
 secrets your runbook needs (e.g. `HACKERNEWS_PASSWORD`) get pulled from the
 sidecar process env without ever appearing on stdout.
 
+## Optional: anti-detection
+
+Targets behind Cloudflare / Kasada / DataDome (e.g. Airbase) will reject the
+default Browserbase session fingerprint and serve a non-interactive challenge
+page. Flip either flag on for the affected runs.
+
+| Env var                            | Default | Purpose                                          |
+|------------------------------------|---------|--------------------------------------------------|
+| `BROWSERBASE_ADVANCED_STEALTH`     | *(off)* | Send `browserSettings.advancedStealth: true`. Browserbase Scale-plan-gated — API errors on lower plans. |
+| `BROWSERBASE_PROXIES`              | *(off)* | Send `proxies: true`. Routes through Browserbase residential proxy pool. Incurs extra per-session cost. |
+
+Set `=1` (or `=true`) to enable. `proxies: true` alone clears most Cloudflare
+challenges; add `advancedStealth: true` on top when the target still blocks.
+The sidecar logs the resolved config at session create.
+
 ## Optional: session recording (rrweb + CDP screencast)
 
 Each browser session can be recorded two ways and uploaded to a consumer
@@ -85,10 +100,45 @@ example, see the `browserbase-hn-upvoted-probe` runbook in the xatabase repo.
 | `extract`    | —                           | `selector` (rows), `fields: { name → spec }`    |
 | `assert`     | `assert: <selector>`        | `selector`, `text_contains`, `url_contains`     |
 | `eval`       | `eval: <js>`                | `script`                                        |
+| `save`       | `save: <name>`              | `name`, `value` (captures prior `extract`/`eval` when omitted) |
 
 `extract`'s `fields` entries are either a CSS selector string (returns
 `textContent`), or `{ selector, attr }` to read an attribute.
 
+## Artifacts
+
+`wb` exports `$WB_ARTIFACTS_DIR` to every cell — a per-run directory
+(`~/.wb/runs/<run_id>/artifacts/` by default) where any cell can drop files
+that later cells will read back. The browser `save:` verb is the
+sidecar-side equivalent:
+
+```yaml
+- extract:
+    selector: .order-row
+    fields:
+      id: .order-id
+      total: .total
+- save: orders            # writes $WB_ARTIFACTS_DIR/orders.json
+```
+
+Forms:
+
+- `save: <name>` — captures the previous verb's JSON output (from
+  `extract` or `eval`) into `<name>.json`.
+- `save: { name: orders, value: { ... } }` — writes an inline value.
+- `save: {}` — auto-names the file `cell-<block_index>-<rand>.json`.
+
+Downstream bash/python cells read the file directly:
+
+```bash
+jq '.[0].id' "$WB_ARTIFACTS_DIR/orders.json"
+```
+
+When `WB_ARTIFACTS_UPLOAD_URL` is set (template supports `{run_id}` and
+`{filename}`), `wb` POSTs each new artifact file after the cell that
+produced it completes. Auth reuses `WB_RECORDING_UPLOAD_SECRET`
+(`Authorization: Bearer <…>`); failures are logged and non-fatal.
+
 ## Protocol
 
 Line-framed JSON, one message per line, on stdin/stdout. `stderr` is treated as
@@ -139,7 +189,8 @@ Sidecar exits 0.
 - v0.1 — protocol skeleton (echo only)
 - v0.2 — `slice.session_started` event with stub URL
 - v0.3 — Browserbase + playwright-core, real `goto/fill/click/wait_for/extract/assert`
-- v0.4 — rrweb + CDP screencast recording, uploaded to a consumer endpoint (this)
-- v0.5 — `act:` recovery via Stagehand, `slice.recovered` events
-- v0.6 — `wait_for_mfa` / `wait_for_email_otp` emitting `slice.paused` with
+- v0.4 — rrweb + CDP screencast recording, uploaded to a consumer endpoint
+- v0.5 — `save:` verb + shared `$WB_ARTIFACTS_DIR` for cross-cell data (this)
+- v0.6 — `act:` recovery via Stagehand, `slice.recovered` events
+- v0.7 — `wait_for_mfa` / `wait_for_email_otp` emitting `slice.paused` with
   `resume_url`

package/bin/wb-browser-runtime.js

@@ -41,10 +41,11 @@ const SUPPORTS = [
   "extract",
   "assert",
   "eval",
+  "save",
 ];
 
 const BB_BASE = "https://api.browserbase.com";
-const VERSION = "0.4.0";
+const VERSION = "0.5.0";
 
 // --- Recording config -------------------------------------------------------
 //
@@ -167,15 +168,35 @@ async function bbCreateSession() {
       "BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID must be set",
     );
   }
+
+  // Both flags opt-in per session. advancedStealth is Scale-plan-gated on
+  // Browserbase's side; proxies adds residential-IP cost. Default off so a
+  // misconfigured plan doesn't break unrelated runs (HN, Google Sheets, etc.);
+  // flip per vendor when the target sits behind Cloudflare / similar bot
+  // detection (e.g., Airbase).
+  const envBool = (v) => v === "1" || (typeof v === "string" && v.toLowerCase() === "true");
+  const advancedStealth = envBool(process.env.BROWSERBASE_ADVANCED_STEALTH);
+  const proxies = envBool(process.env.BROWSERBASE_PROXIES);
+
+  // keepAlive:false — slice lifetime is tied to wb process; on shutdown
+  // we explicitly REQUEST_RELEASE so quota isn't burned by orphans.
+  const body = { projectId, keepAlive: false };
+  if (advancedStealth) {
+    body.browserSettings = { advancedStealth: true };
+  }
+  if (proxies) {
+    body.proxies = true;
+  }
+
+  log(`[bb] session create advancedStealth=${advancedStealth} proxies=${proxies}`);
+
   const res = await fetch(`${BB_BASE}/v1/sessions`, {
     method: "POST",
     headers: {
       "X-BB-API-Key": apiKey,
       "Content-Type": "application/json",
     },
-    // keepAlive:false — slice lifetime is tied to wb process; on shutdown
-    // we explicitly REQUEST_RELEASE so quota isn't burned by orphans.
-    body: JSON.stringify({ projectId, keepAlive: false }),
+    body: JSON.stringify(body),
   });
   if (!res.ok) {
     throw new Error(
@@ -597,7 +618,7 @@ function arg(value, primaryKey) {
   return {};
 }
 
-async function runVerb(page, verb, index) {
+async function runVerb(page, verb, index, ctx) {
   const name = verbName(verb);
   const raw = verb[name];
   const a = expand(arg(raw, defaultKey(name)));
@@ -670,6 +691,7 @@ async function runVerb(page, verb, index) {
       // Emit as JSON to stdout so wb captures it in step.complete.stdout.
       // Pretty-printed for readability when a runbook surfaces the output.
       console.log(JSON.stringify(items, null, 2));
+      if (ctx) ctx.lastResult = items;
       return `${rowSelector} → ${items.length} rows`;
     }
     case "assert": {
@@ -695,13 +717,63 @@ async function runVerb(page, verb, index) {
       // Run arbitrary JS in the page; result is JSON-serialized to stdout.
       const result = await page.evaluate(a.script);
       console.log(JSON.stringify(result, null, 2));
+      if (ctx) ctx.lastResult = result;
       return `script ran`;
     }
+    case "save": {
+      // Persist a JSON artifact into $WB_ARTIFACTS_DIR so later cells can read
+      // it and wb can upload it. Captures the previous verb's output unless
+      // the author provides an explicit `value:`.
+      const artifactsDir = (process.env.WB_ARTIFACTS_DIR || "").trim();
+      if (!artifactsDir) {
+        throw new Error(
+          "save: $WB_ARTIFACTS_DIR is not set — run this workbook via `wb run` (wb exports the dir for you)",
+        );
+      }
+      const explicitValue = a.value !== undefined;
+      const payload = explicitValue ? a.value : ctx?.lastResult;
+      if (payload === undefined) {
+        throw new Error(
+          "save: no value provided and no prior extract/eval result to capture",
+        );
+      }
+      const name =
+        typeof a.name === "string" && a.name.trim().length > 0
+          ? sanitizeArtifactName(a.name)
+          : autoArtifactName(ctx?.blockIndex ?? index);
+      const filename = name.endsWith(".json") ? name : `${name}.json`;
+      const full = path.join(artifactsDir, filename);
+      await fsPromises.mkdir(artifactsDir, { recursive: true });
+      await fsPromises.writeFile(
+        full,
+        JSON.stringify(payload, null, 2),
+        "utf8",
+      );
+      send({
+        type: "slice.artifact_saved",
+        filename,
+        path: full,
+        bytes: Buffer.byteLength(JSON.stringify(payload)),
+      });
+      return `→ ${filename}`;
+    }
     default:
       throw new Error(`unsupported verb: ${name}`);
   }
 }
 
+function sanitizeArtifactName(s) {
+  // Keep author-chosen names readable but safe as filenames. Drop anything
+  // that could escape the artifacts dir (slashes, NULs, etc.).
+  return String(s).replace(/[^A-Za-z0-9_.-]+/g, "_").slice(0, 200);
+}
+
+function autoArtifactName(blockIndex) {
+  const rand = randomUUID().replace(/-/g, "").slice(0, 8);
+  const n = Number.isFinite(blockIndex) ? blockIndex : 0;
+  return `cell-${n}-${rand}`;
+}
+
 function defaultKey(name) {
   switch (name) {
     case "goto":
@@ -716,6 +788,8 @@ function defaultKey(name) {
       return "key";
     case "eval":
       return "script";
+    case "save":
+      return "name";
     default:
       return "value";
   }
@@ -733,6 +807,8 @@ async function handleSlice(msg) {
   const verbs = Array.isArray(msg.verbs) ? msg.verbs : [];
   const sessionName = msg.session || "default";
   const restore = msg.restore || null;
+  const blockIndex =
+    typeof msg.block_index === "number" ? msg.block_index : null;
 
   let session;
   try {
@@ -750,11 +826,14 @@ async function handleSlice(msg) {
   // is where we'd jump to verbs[restore.state.verb_index].
   const startAt = restore?.state?.verb_index ?? 0;
 
+  // Per-slice scratch so `save:` can capture the prior verb's JSON output.
+  const sliceCtx = { lastResult: undefined, blockIndex };
+
   for (let i = startAt; i < verbs.length; i++) {
     const v = verbs[i];
     const name = verbName(v);
     try {
-      const summary = await runVerb(session.page, v, i);
+      const summary = await runVerb(session.page, v, i, sliceCtx);
       send({
         type: "verb.complete",
         verb: name,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wb-browser-runtime",
-  "version": "0.4.0",
+  "version": "0.5.1",
   "description": "Browser sidecar runtime for wb — Browserbase + Playwright over the wb-sidecar/1 line-framed JSON protocol.",
   "bin": {
     "wb-browser-runtime": "bin/wb-browser-runtime.js"