typeclaw 0.3.1 → 0.5.0

package/src/agent/tools/curl-impersonate.ts

@@ -0,0 +1,300 @@
+// Shared curl-impersonate spawn primitive.
+//
+// Why this exists: by 2026, every non-trivial public site (DDG, Reuters via
+// Akamai, MarketWatch via Cloudflare, etc.) fingerprints incoming traffic at
+// the TLS handshake (JA3/JA4) and HTTP/2 SETTINGS frame BEFORE any HTTP header
+// is read. Bun's native fetch cannot match Chrome's handshake (upstream issue
+// #11368), so outbound requests get gated by anomaly checks regardless of
+// headers, body shape, or pacing. The fix is to shell out to curl-impersonate
+// (lexiforest fork), which replays Chrome's exact TLS handshake, HTTP/2
+// settings, and header ordering. Pinned by the typeclaw Dockerfile at
+// /usr/local/bin/curl_chrome136 — see src/init/dockerfile.ts for the version
+// and SHA pin.
+//
+// AGENTS.md explicitly warns against adding `-H` overrides because the
+// curl_chrome wrapper already sends the full Chrome header set (correct
+// ordering, sec-ch-ua, sec-fetch-*, accept-encoding, etc.) and any custom
+// header corrupts the impersonation. We therefore expose NO header-override
+// surface from this primitive; add one only when a real caller needs it AND
+// the override is something curl_chrome can't be told to send another way.
+
+import { randomBytes } from 'node:crypto'
+
+import { spawn } from 'bun'
+
+export const CURL_IMPERSONATE_BINARY = 'curl_chrome136'
+export const DEFAULT_TIMEOUT_SECONDS = 30
+
+let curlBinary: string = CURL_IMPERSONATE_BINARY
+
+// Test-only seam: lets *.test.ts point the spawn at a fake `curl_chrome136`
+// script in a tmpdir so we exercise the real Bun.spawn path without depending
+// on a curl-impersonate install on the test host. Production code never calls
+// this — the module-level default above is what production sees.
+export function _setCurlBinaryForTest(binary: string | null): void {
+  curlBinary = binary ?? CURL_IMPERSONATE_BINARY
+}
+
+export type CurlImpersonateRequest = {
+  url: string
+  method?: 'GET' | 'POST'
+  // Form-urlencoded body fields for POST. Each entry is passed as a separate
+  // --data-urlencode argument so curl handles the encoding. Required if
+  // method is 'POST' and you want a body.
+  formFields?: Array<{ name: string; value: string }>
+  // Hard cap on bytes accepted from the response (passed as --max-filesize).
+  // The actual buffer is still bounded by the caller; this just makes curl
+  // bail early instead of streaming gigabytes.
+  maxBytes?: number
+  timeoutSeconds?: number
+  signal?: AbortSignal
+}
+
+export type CurlImpersonateResponse = {
+  body: string
+  finalUrl: string
+  httpStatus: number
+  contentType: string
+  bytesIn: number
+}
+
+// Specific curl exit codes we map to typed errors. The full list is in
+// `man curl` § "EXIT CODES"; these are the only ones we translate at the
+// primitive layer. Everything else surfaces as a generic CurlImpersonateError
+// with stderr attached for caller-side diagnostics.
+export const CURL_EXIT_TIMEOUT = 28
+export const CURL_EXIT_MAX_FILESIZE_PRECHECK = 63
+// Observed empirically (and corroborated by Oracle review): curl returns
+// exit 56 with stderr `Exceeded the maximum allowed file size (...)` when
+// --max-filesize is hit at TRANSFER time (e.g. server omitted Content-Length
+// and curl discovered the overflow mid-stream). The Linux man page lists 56
+// as the more general "Failure in receiving network data," so we additionally
+// gate on a stderr match to avoid mis-classifying real network drops as
+// size-exceeded.
+export const CURL_EXIT_RECV_FAILURE_OR_FILESIZE = 56
+
+export class CurlImpersonateError extends Error {
+  constructor(
+    message: string,
+    public readonly exitCode: number | null,
+    public readonly stderr: string,
+  ) {
+    super(message)
+    this.name = 'CurlImpersonateError'
+  }
+}
+
+export function isCurlExitFilesizeExceeded(error: CurlImpersonateError): boolean {
+  if (error.exitCode === CURL_EXIT_MAX_FILESIZE_PRECHECK) return true
+  if (error.exitCode === CURL_EXIT_RECV_FAILURE_OR_FILESIZE && /maximum.{0,30}file size/i.test(error.stderr)) {
+    return true
+  }
+  return false
+}
+
+export function isCurlExitTimeout(error: CurlImpersonateError): boolean {
+  return error.exitCode === CURL_EXIT_TIMEOUT
+}
+
+export async function curlImpersonate(req: CurlImpersonateRequest): Promise<CurlImpersonateResponse> {
+  const timeoutSeconds = req.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS
+  const method = req.method ?? 'GET'
+
+  // Per-request random sentinel + UTF-8-safe parsing. The static sentinel
+  // approach (previous revision) had a hardening hole: webfetch reads
+  // attacker-controlled pages, and a static sentinel is a public, fixed
+  // string. A page could include the sentinel byte sequence plus fabricated
+  // metadata before the real write-out tail and `indexOf` would split at
+  // the attacker-controlled occurrence. Per-request randomness (96 bits)
+  // removes the attacker's ability to predict the sentinel, and the parser
+  // anchors on the LAST occurrence (curl writes `-w` after the body, so the
+  // real metadata block is always last). Both defenses are needed: random
+  // alone fails if the attacker can read the sentinel from a previous
+  // response and replay it; last-match alone fails if the attacker can
+  // append text after curl's write-out (they can't, but defense in depth).
+  const sentinel = generateSentinel()
+  const writeOutTemplate = `${sentinel}%{http_code}\n%{url_effective}\n%{content_type}\n%{size_download}\n`
+
+  const cmd: string[] = [
+    curlBinary,
+    // `--disable` (alias -q) MUST be the first argument to suppress reading
+    // ~/.curlrc and /etc/curlrc. Without it, a user or attacker-controlled
+    // curlrc could inject --proxy, --header, --resolve, --no-location, etc.,
+    // silently subverting both the Chrome impersonation contract and the
+    // protocol restrictions below. Order is load-bearing: curl ignores
+    // --disable if it appears after any other flag.
+    '--disable',
+    '--silent',
+    '--show-error',
+    // Protocol allowlist. curl-impersonate supports many protocols by default
+    // (ftp, file, dict, etc.). normalizeUrl() already rejects non-http(s) at
+    // the call-site, but redirects are followed by curl after that gate fires
+    // and a 301/302 to ftp://... would otherwise be silently honored. The
+    // `=http,https` syntax means "ONLY these two" rather than "add these to
+    // defaults." --proto-redir governs the redirect chain specifically.
+    '--proto',
+    '=http,https',
+    '--proto-redir',
+    '=http,https',
+    // `--fail-with-body` would make curl exit non-zero on >=400 but still
+    // write the body. We intentionally DO NOT pass it: callers (webfetch,
+    // ddg) want to inspect httpStatus themselves and decide. Curl exits 0
+    // on a 404-with-body in this mode, which matches our contract.
+    '--compressed',
+    '--location',
+    '--max-redirs',
+    '10',
+    '--max-time',
+    String(timeoutSeconds),
+    '-w',
+    writeOutTemplate,
+    '-X',
+    method,
+  ]
+
+  if (req.maxBytes !== undefined) {
+    cmd.push('--max-filesize', String(req.maxBytes))
+  }
+
+  if (req.formFields) {
+    for (const field of req.formFields) {
+      cmd.push('--data-urlencode', `${field.name}=${field.value}`)
+    }
+  }
+
+  // `--` terminates option parsing so a URL beginning with `-` (e.g. an
+  // attacker-supplied "-K /etc/passwd" sneaking through normalizeUrl as
+  // "https://-K /etc/passwd") cannot be reinterpreted as a curl option.
+  cmd.push('--', req.url)
+
+  // Spawn detached so the child becomes the leader of its own process group.
+  // The curl-impersonate wrappers (curl_chrome136 et al.) are bash scripts
+  // that call the real curl-impersonate binary WITHOUT `exec` — meaning the
+  // wrapper is the parent and curl-impersonate is its child. On a plain
+  // SIGKILL to the wrapper PID, the curl child becomes orphaned and keeps
+  // the stdout pipe open until --max-time fires, turning a 50ms abort into
+  // a 30s hang. process.kill(-pid) addresses the negative PID, which signals
+  // the entire process group, killing both atomically. detached: true makes
+  // the child the pgid leader so -pid is well-defined.
+  const proc = spawn({
+    cmd,
+    stdout: 'pipe',
+    stderr: 'pipe',
+    detached: true,
+  })
+
+  const onAbort = () => {
+    try {
+      process.kill(-proc.pid, 'SIGKILL')
+    } catch {
+      proc.kill('SIGKILL')
+    }
+  }
+  req.signal?.addEventListener('abort', onAbort, { once: true })
+
+  try {
+    const [stdoutBuf, stderr, exitCode] = await Promise.all([
+      new Response(proc.stdout).arrayBuffer(),
+      new Response(proc.stderr).text(),
+      proc.exited,
+    ])
+
+    if (req.signal?.aborted) {
+      throw new CurlImpersonateError('aborted', exitCode, stderr)
+    }
+
+    if (exitCode !== 0) {
+      const detail = stderr.trim() || 'no stderr'
+      throw new CurlImpersonateError(`curl-impersonate exited ${exitCode}: ${detail}`, exitCode, stderr)
+    }
+
+    return parseCurlOutput(stdoutBuf, sentinel, stderr)
+  } finally {
+    req.signal?.removeEventListener('abort', onAbort)
+  }
+}
+
+// Generates a per-request sentinel. Format: `\n--TYPECLAW-CURL-META-<hex>--\n`.
+// 24 hex chars = 96 bits of entropy, plenty to defeat any attempt by an
+// attacker-controlled response body to inject a colliding marker. ASCII-only
+// + leading/trailing newlines means it's unambiguous in textual responses
+// and free of NUL bytes (Bun's spawn rejects NULs in argv).
+function generateSentinel(): string {
+  const hex = randomBytes(12).toString('hex')
+  return `\n--TYPECLAW-CURL-META-${hex}--\n`
+}
+
+function parseCurlOutput(buf: ArrayBuffer, sentinel: string, stderr: string): CurlImpersonateResponse {
+  const sentinelBytes = new TextEncoder().encode(sentinel)
+  const bytes = new Uint8Array(buf)
+
+  // Anchor on the LAST occurrence (defense in depth alongside the random
+  // sentinel). curl writes the `-w` output strictly AFTER the body, so the
+  // real metadata block is always the trailing one.
+  const sentinelIndex = lastIndexOfBytes(bytes, sentinelBytes)
+  if (sentinelIndex < 0) {
+    throw new CurlImpersonateError(
+      'curl-impersonate produced no metadata block (sentinel missing). Wrapper or output corruption suspected.',
+      0,
+      stderr,
+    )
+  }
+
+  const bodyBytes = bytes.subarray(0, sentinelIndex)
+  const metaBytes = bytes.subarray(sentinelIndex + sentinelBytes.byteLength)
+  const meta = new TextDecoder('utf-8', { fatal: false }).decode(metaBytes).split('\n')
+
+  const httpStatus = Number(meta[0]?.trim() ?? '0') || 0
+  const finalUrl = (meta[1] ?? '').trim()
+  const contentType = (meta[2] ?? '').trim().toLowerCase()
+  const declaredBytes = Number(meta[3]?.trim() ?? '0') || bodyBytes.byteLength
+
+  const body = new TextDecoder('utf-8', { fatal: false }).decode(bodyBytes)
+
+  return {
+    body,
+    finalUrl,
+    httpStatus,
+    contentType,
+    bytesIn: declaredBytes,
+  }
+}
+
+function lastIndexOfBytes(haystack: Uint8Array, needle: Uint8Array): number {
+  if (needle.byteLength === 0) return haystack.byteLength
+  for (let i = haystack.byteLength - needle.byteLength; i >= 0; i--) {
+    let matched = true
+    for (let j = 0; j < needle.byteLength; j++) {
+      if (haystack[i + j] !== needle[j]) {
+        matched = false
+        break
+      }
+    }
+    if (matched) return i
+  }
+  return -1
+}
+
+// Detect whether curl-impersonate is available on PATH. Used by fetch.ts to
+// decide between the impersonating transport (production: container has the
+// binary pinned in the image) and a Bun.fetch fallback (test/dev: no binary
+// installed). The check is best-effort and cheap — we spawn `--version`
+// and look at exit code. Cached per-process: the binary doesn't appear or
+// disappear at runtime.
+let availabilityCache: boolean | undefined
+
+export async function isCurlImpersonateAvailable(): Promise<boolean> {
+  if (availabilityCache !== undefined) return availabilityCache
+  try {
+    const proc = spawn({ cmd: [curlBinary, '--version'], stdout: 'ignore', stderr: 'ignore' })
+    const code = await proc.exited
+    availabilityCache = code === 0
+  } catch {
+    availabilityCache = false
+  }
+  return availabilityCache
+}
+
+export function _resetAvailabilityCacheForTest(): void {
+  availabilityCache = undefined
+}

package/src/agent/tools/ddg.ts

@@ -7,40 +7,16 @@
 // single bad fingerprint match. `lite` exists for non-browser clients (text
 // browsers, accessibility tools) and historically gates less aggressively —
 // but as of 2026 it ALSO fingerprints at the TLS layer (JA3/JA4) and the
-// HTTP/2 SETTINGS frame, well before any HTTP header is read. Bun's native
-// fetch cannot match Chrome's handshake (upstream issue #11368), so requests
-// from `fetch()` get gated regardless of headers, body shape, or pacing —
-// confirmed empirically over a multi-hour session against a single home IP
-// where real Chromium succeeded continuously while every fetch variant got
-// 202 anomaly-modal or HTTP-200-with-anomaly responses.
-//
-// The fix is to shell out to `curl-impersonate` (lexiforest fork), which
-// replays Chrome's exact TLS handshake + HTTP/2 settings + header ordering.
-// The binary is installed by the typeclaw Dockerfile (see
-// src/init/dockerfile.ts CURL_IMPERSONATE_* constants) at /usr/local/bin/
-// and invoked via the version-pinned wrapper `curl_chrome136`.
-//
-// Why no `-H` overrides: curl_chrome136 already sends the full Chrome 136
-// header set with correct ordering, sec-ch-ua values, etc. Adding our own
-// headers would corrupt the impersonation. The previous code's
-// BROWSER_HEADERS const has been removed for the same reason.
+// HTTP/2 SETTINGS frame, well before any HTTP header is read. The shared
+// curl-impersonate primitive (./curl-impersonate.ts) replays Chrome's exact
+// TLS handshake + HTTP/2 settings + header ordering. See that file's header
+// for the full rationale and AGENTS.md §"Web search" for the original story.
 
-import { spawn } from 'bun'
-
-const DDG_LITE_URL = 'https://lite.duckduckgo.com/lite/'
-const CURL_IMPERSONATE_BINARY = 'curl_chrome136'
-const REQUEST_TIMEOUT_SECONDS = 30
+import { curlImpersonate } from './curl-impersonate'
 
-let curlBinary = CURL_IMPERSONATE_BINARY
+export { _setCurlBinaryForTest } from './curl-impersonate'
 
-// Test-only seam: lets ddg.test.ts and websearch.test.ts point the spawn
-// at a fake `curl_chrome136` script in a tmpdir so we exercise the real
-// Bun.spawn path without depending on a curl-impersonate install on the
-// test host. Production code never calls this — the const-import default
-// above is what production sees.
-export function _setCurlBinaryForTest(binary: string | null): void {
-  curlBinary = binary ?? CURL_IMPERSONATE_BINARY
-}
+const DDG_LITE_URL = 'https://lite.duckduckgo.com/lite/'
 
 export type DdgResult = {
   title: string
@@ -64,64 +40,13 @@ export class DdgCaptchaError extends Error {
 }
 
 export async function fetchDdgHtml(query: string, signal?: AbortSignal): Promise<string> {
-  // Spawn detached so the child becomes the leader of its own process group.
-  // The curl-impersonate wrappers (curl_chrome136 et al.) are bash scripts
-  // that call the real curl-impersonate binary WITHOUT `exec` — meaning the
-  // wrapper is the parent and curl-impersonate is its child. On a plain
-  // SIGKILL to the wrapper PID, the curl child becomes orphaned and keeps
-  // the stdout pipe open until --max-time fires (30s default), turning a
-  // 50ms abort into a 30s hang. process.kill(-pid) addresses the negative
-  // PID, which signals the entire process group, killing both the wrapper
-  // and the inner curl atomically. detached: true is what makes the child
-  // the pgid leader so -pid is well-defined; without it, the child shares
-  // our pgid and we'd nuke our own process.
-  const proc = spawn({
-    cmd: [
-      curlBinary,
-      '--silent',
-      '--show-error',
-      '--fail-with-body',
-      '--compressed',
-      '--max-time',
-      String(REQUEST_TIMEOUT_SECONDS),
-      '-X',
-      'POST',
-      '--data-urlencode',
-      `q=${query}`,
-      DDG_LITE_URL,
-    ],
-    stdout: 'pipe',
-    stderr: 'pipe',
-    detached: true,
+  const response = await curlImpersonate({
+    url: DDG_LITE_URL,
+    method: 'POST',
+    formFields: [{ name: 'q', value: query }],
+    signal,
   })
-
-  const onAbort = () => {
-    try {
-      process.kill(-proc.pid, 'SIGKILL')
-    } catch {
-      proc.kill('SIGKILL')
-    }
-  }
-  signal?.addEventListener('abort', onAbort, { once: true })
-
-  try {
-    const [stdout, stderr, exitCode] = await Promise.all([
-      new Response(proc.stdout).text(),
-      new Response(proc.stderr).text(),
-      proc.exited,
-    ])
-
-    if (signal?.aborted) {
-      throw new Error('aborted')
-    }
-    if (exitCode !== 0) {
-      const detail = stderr.trim() || 'no stderr'
-      throw new Error(`curl-impersonate exited ${exitCode}: ${detail}`)
-    }
-    return stdout
-  } finally {
-    signal?.removeEventListener('abort', onAbort)
-  }
+  return response.body
 }
 
 // The `lite` endpoint's CAPTCHA page is plainer than `html`'s anomaly-modal:

package/src/agent/tools/webfetch/fetch.ts

@@ -1,3 +1,33 @@
+// Webfetch's HTTP transport.
+//
+// Production path (container, curl-impersonate available): we shell out to
+// `curl_chrome136` so outbound requests carry Chrome 136's TLS handshake
+// (JA3/JA4), HTTP/2 SETTINGS frame, and full header set. This is what gets
+// us past the modern bot-detection stacks on Cloudflare/Akamai-protected
+// sites (Reuters, MarketWatch, etc.) when the agent is running from the
+// user's home network — the IP is already residential, so impersonating
+// the browser is the only remaining missing piece. See AGENTS.md §"Web
+// search" and src/agent/tools/curl-impersonate.ts for the full story.
+//
+// Test/dev fallback (curl_chrome136 not on PATH): we transparently fall
+// back to Bun's native `fetch()` with a static User-Agent. This keeps unit
+// tests on developer macOS machines working without forcing every contributor
+// to install curl-impersonate locally. Production runs always have the binary
+// because the typeclaw Dockerfile pins it.
+//
+// Best-effort doctrine: this transport does NOT guarantee the fetch succeeds.
+// Bot-detected sites can still serve 403/CAPTCHA pages. We surface what we
+// got (status, body, final URL) and let the caller decide. The webfetch tool
+// translates non-2xx into a tool-level error message that's useful to the
+// model.
+
+import {
+  CurlImpersonateError,
+  curlImpersonate,
+  isCurlExitFilesizeExceeded,
+  isCurlExitTimeout,
+  isCurlImpersonateAvailable,
+} from '../curl-impersonate'
 import { MAX_RESPONSE_BYTES } from './types'
 
 export type FetchResult = {
@@ -15,7 +45,7 @@ export class WebfetchError extends Error {
   }
 }
 
-const DEFAULT_HEADERS: Record<string, string> = {
+const FALLBACK_HEADERS: Record<string, string> = {
   'User-Agent': 'typeclaw/0 (+https://github.com/code-yeongyu/typeclaw)',
   Accept: 'text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1',
   'Accept-Language': 'en-US,en;q=0.9',
@@ -32,10 +62,83 @@ export function normalizeUrl(input: string): string {
   return `https://${trimmed}`
 }
 
+// Test-only seam: forces fetchWithLimits to use the native-fetch fallback
+// even when curl-impersonate is detected. Used by fetch.test.ts to keep its
+// existing mocked-fetch contract working without the test having to install
+// a fake curl binary. Production code never calls this.
+let forceFallbackForTest = false
+
+export function _setForceFallbackForTest(value: boolean): void {
+  forceFallbackForTest = value
+}
+
 export async function fetchWithLimits(
   url: string,
   timeoutSeconds: number,
   parentSignal?: AbortSignal,
+): Promise<FetchResult> {
+  const useImpersonate = !forceFallbackForTest && (await isCurlImpersonateAvailable())
+  if (useImpersonate) {
+    return fetchWithCurlImpersonate(url, timeoutSeconds, parentSignal)
+  }
+  return fetchWithBunFetch(url, timeoutSeconds, parentSignal)
+}
+
+async function fetchWithCurlImpersonate(
+  url: string,
+  timeoutSeconds: number,
+  parentSignal?: AbortSignal,
+): Promise<FetchResult> {
+  let response
+  try {
+    response = await curlImpersonate({
+      url,
+      method: 'GET',
+      timeoutSeconds,
+      maxBytes: MAX_RESPONSE_BYTES,
+      signal: parentSignal,
+    })
+  } catch (error) {
+    if (parentSignal?.aborted) {
+      throw new WebfetchError('Request aborted')
+    }
+    if (error instanceof CurlImpersonateError) {
+      if (isCurlExitTimeout(error)) {
+        throw new WebfetchError(`Request timed out after ${timeoutSeconds}s`)
+      }
+      if (isCurlExitFilesizeExceeded(error)) {
+        throw new WebfetchError(`Response too large (exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`)
+      }
+      throw new WebfetchError(`Fetch failed: ${error.message}`)
+    }
+    const message = error instanceof Error ? error.message : String(error)
+    throw new WebfetchError(`Fetch failed: ${message}`)
+  }
+
+  if (response.httpStatus < 200 || response.httpStatus >= 300) {
+    throw new WebfetchError(`Fetch failed: HTTP ${response.httpStatus}`)
+  }
+
+  const bodyByteLength = new TextEncoder().encode(response.body).byteLength
+  if (bodyByteLength > MAX_RESPONSE_BYTES) {
+    throw new WebfetchError(
+      `Response too large (${formatBytes(bodyByteLength)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
+    )
+  }
+
+  return {
+    body: response.body,
+    contentType: response.contentType,
+    finalUrl: response.finalUrl || url,
+    httpStatus: response.httpStatus,
+    bytesIn: bodyByteLength,
+  }
+}
+
+async function fetchWithBunFetch(
+  url: string,
+  timeoutSeconds: number,
+  parentSignal?: AbortSignal,
 ): Promise<FetchResult> {
   const controller = new AbortController()
   const timeout = setTimeout(() => controller.abort(new Error('timeout')), timeoutSeconds * 1000)
@@ -43,7 +146,7 @@ export async function fetchWithLimits(
   parentSignal?.addEventListener('abort', onAbort, { once: true })
 
   try {
-    const response = await fetch(url, { headers: DEFAULT_HEADERS, signal: controller.signal, redirect: 'follow' })
+    const response = await fetch(url, { headers: FALLBACK_HEADERS, signal: controller.signal, redirect: 'follow' })
     if (!response.ok) {
       throw new WebfetchError(`Fetch failed: HTTP ${response.status} ${response.statusText}`)
     }

package/src/agent/tools/webfetch/tool.ts

@@ -24,6 +24,10 @@ export const webfetchTool = defineTool({
   description:
     'Fetch a single HTTP(S) URL and return the body, optionally compacted by a strategy. ' +
     'Use this when the user references a specific URL or when websearch surfaced a result you need to read in full. ' +
+    'Outbound requests impersonate Chrome 136 at the TLS, HTTP/2, and header layers ' +
+    '(via curl-impersonate), which helps with TLS/header fingerprint gates on sites behind Cloudflare/Akamai. ' +
+    'It does NOT solve JavaScript challenges, behavioural fingerprinting (mouse/scroll/timing), interactive CAPTCHAs, ' +
+    'or IP-reputation blocks — a 403 from those layers is expected and unrecoverable from this tool. ' +
     'Strategy guide:\n' +
     '- "readability": extract article content as markdown (blogs, docs, news). Default for HTML.\n' +
     '- "jq": query JSON APIs (npm registry, GitHub API). Pass `query` (e.g. ".items[].name").\n' +

package/src/bundled-plugins/agent-browser/shim.ts

@@ -17,6 +17,49 @@ import { AGENT_BROWSER_DASHBOARD_UPSTREAM_PORT } from './dashboard-proxy'
 
 export const REAL_BIN_ENV = 'TYPECLAW_AGENT_BROWSER_REAL_BIN'
 
+// Recent desktop Chrome on Linux x86_64. The shim runs inside the TypeClaw
+// container (always Linux), so a macOS or Windows UA would mismatch the TCP
+// fingerprint, Accept-Language, and JS-side platform — itself a bot signal on
+// stricter sites (Cloudflare, Akamai, PerimeterX). `X11; Linux x86_64` is
+// also correct on linux/arm64 hosts: Chrome on Linux does not expose ARM in
+// the UA string at all (verified against current Chrome 131 releases).
+// The upstream binary defaults to a UA that includes "HeadlessChrome" /
+// a stale Chromium build, which is widely fingerprinted as a bot and
+// silently triggers CAPTCHAs, 403s, blank pages, and A/B-test misrouting.
+// Bump on Chrome major releases — same hygiene as the curl-impersonate pin
+// in src/init/dockerfile.ts.
+export const DEFAULT_USER_AGENT =
+  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
+
+export const USER_AGENT_ENV = 'AGENT_BROWSER_USER_AGENT'
+
+export function hasUserAgentFlag(argv: readonly string[]): boolean {
+  // Matches both `--user-agent <val>` and `--user-agent=<val>`. The upstream
+  // CLI does not document a short alias for --user-agent today (verified via
+  // `agent-browser --help`), so we only check the long form.
+  for (const arg of argv) {
+    if (arg === '--user-agent' || arg.startsWith('--user-agent=')) return true
+  }
+  return false
+}
+
+export function injectUserAgentEnv(
+  argv: readonly string[],
+  env: Record<string, string | undefined>,
+  defaultUa: string = DEFAULT_USER_AGENT,
+): void {
+  // Upstream's precedence is CLI flag > env > default. We only inject the
+  // env when BOTH layers above it are absent so:
+  //   - explicit `--user-agent foo` wins (mobile testing, intentional bot UA)
+  //   - operator-set AGENT_BROWSER_USER_AGENT wins (per-shell override)
+  //   - default UA fills the otherwise-empty slot
+  // `set device "iPhone 14"` is unaffected: it sets UA via CDP at runtime,
+  // not through this env var, so our injection doesn't fight device emulation.
+  if (env[USER_AGENT_ENV] !== undefined && env[USER_AGENT_ENV] !== '') return
+  if (hasUserAgentFlag(argv)) return
+  env[USER_AGENT_ENV] = defaultUa
+}
+
 export type DashboardIntent = 'start' | 'stop' | 'other'
 
 export function classifyDashboardCommand(argv: readonly string[]): DashboardIntent {
@@ -111,6 +154,7 @@ export type ShimOptions = {
   realBin?: string
   upstreamPort?: number
   spawn?: (cmd: string[]) => { exited: Promise<number> }
+  env?: Record<string, string | undefined>
 }
 
 export async function runShim(opts: ShimOptions = {}): Promise<number> {
@@ -118,6 +162,9 @@ export async function runShim(opts: ShimOptions = {}): Promise<number> {
   const realBin = opts.realBin ?? resolveRealAgentBrowserBin()
   const upstreamPort = opts.upstreamPort ?? AGENT_BROWSER_DASHBOARD_UPSTREAM_PORT
   const spawn = opts.spawn ?? defaultSpawn
+  const env = opts.env ?? process.env
+
+  injectUserAgentEnv(argv, env)
 
   const intent = classifyDashboardCommand(argv)
   if (intent !== 'start') {

package/src/bundled-plugins/backup/subagents.ts

@@ -77,6 +77,8 @@ You MAY:
 - Set up a missing upstream branch via \`git push -u origin <branch>\` if it's clear that's the only issue.
 - Retry \`git push\` once after fixing a clear, narrow issue.
 
+**When you run \`git push\` (either to set upstream or to retry), the security plugin's \`gitExfil\` guard will block unless you pass \`acknowledgeGuards: { gitExfil: true }\` on the bash tool call.** You are running on the operator's behalf as the recovery step of a deterministic backup the operator already approved by enabling \`backup.pushToOrigin\`; the ack is justified for this specific retry path. Do NOT use this ack for any other reason — only the one push retry the rules above allow. If your push retry fails again, write the diagnosis and stop; do not ack a second push.
+
 You MUST NOT:
 - Force-push (\`--force\`, \`--force-with-lease\`).
 - Resolve merge conflicts by editing files. If a rebase had conflicts, the runner already aborted it. Leave the repo as-is and explain.