switchback 0.1.0__py3-none-any.whl

switchback/session_trace.py

@@ -0,0 +1,96 @@
+"""Opt-in Playwright trace capture for the browser tiers (req 16).
+
+Off by default. Set ``SCRAPER_TRACE_SESSION=1`` to record a Playwright trace
+(screenshots + DOM snapshots + network) for every browser-tier attempt; each is
+written as a self-contained zip under ``state/traces/`` and is openable with
+``playwright show-trace <zip>``. The HTTP server exposes list / fetch / delete
+endpoints so traces can be pulled and cleaned up on demand.
+
+Capture is wrapped so a tracing failure never breaks a scrape — it just means no
+trace for that attempt. Traces are heavyweight (MBs each); keep this off in
+steady state and flip it on to debug a specific host.
+"""
+from __future__ import annotations
+
+import logging
+import os
+import re
+import time
+
+from .policy.gates import host_of
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_STATE_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "state")
+_STATE_DIR = os.getenv("SCRAPER_STATE_DIR", _DEFAULT_STATE_DIR)
+TRACE_DIR = os.path.join(_STATE_DIR, "traces")
+
+# Trace ids are the zip filename stem; constrain to a safe charset so a request
+# id can never escape TRACE_DIR.
+_ID_RE = re.compile(r"^[A-Za-z0-9._-]+$")
+
+
+def enabled() -> bool:
+    return os.getenv("SCRAPER_TRACE_SESSION") in ("1", "true", "True")
+
+
+def start(context, url: str) -> bool:
+    """Begin tracing on a browser context. No-op (returns False) when disabled or
+    if the context doesn't support tracing."""
+    if not enabled():
+        return False
+    try:
+        context.tracing.start(screenshots=True, snapshots=True, sources=True)
+        return True
+    except Exception as e:
+        logger.warning(f"session_trace: start failed: {e}")
+        return False
+
+
+def stop(context, url: str) -> str | None:
+    """Stop tracing and write the zip; returns its path (or None on failure)."""
+    if not enabled():
+        return None
+    os.makedirs(TRACE_DIR, exist_ok=True)
+    name = f"{host_of(url) or 'unknown'}-{int(time.time() * 1000)}.zip"
+    path = os.path.join(TRACE_DIR, name)
+    try:
+        context.tracing.stop(path=path)
+        logger.info(f"session_trace: wrote {path}")
+        return path
+    except Exception as e:
+        logger.warning(f"session_trace: stop failed: {e}")
+        return None
+
+
+# ── server-side management ───────────────────────────────────────────────────
+
+def list_traces() -> list[dict]:
+    """All captured traces, newest first: id, bytes, modified-at (epoch)."""
+    if not os.path.isdir(TRACE_DIR):
+        return []
+    out = []
+    for fn in os.listdir(TRACE_DIR):
+        if not fn.endswith(".zip"):
+            continue
+        p = os.path.join(TRACE_DIR, fn)
+        st = os.stat(p)
+        out.append({"id": fn[:-4], "bytes": st.st_size, "modified": st.st_mtime})
+    return sorted(out, key=lambda t: -t["modified"])
+
+
+def path_for(trace_id: str) -> str | None:
+    """Resolve a trace id to its zip path, or None if missing/invalid."""
+    if not _ID_RE.match(trace_id or ""):
+        return None
+    p = os.path.join(TRACE_DIR, f"{trace_id}.zip")
+    return p if os.path.exists(p) else None
+
+
+def delete(trace_id: str) -> bool:
+    p = path_for(trace_id)
+    if not p:
+        return False
+    os.remove(p)
+    return True

switchback/tiers/__init__.py

@@ -0,0 +1,24 @@
+"""The cost-ordered cascade. Each tier exposes:
+
+    NAME : str
+    PAID : bool                       # gated/audited if True
+    fetch(url) -> str | None          # markdown on success; None if not
+                                      # applicable; raises on failure.
+
+Order matters — cheapest/cleanest first, paid last.
+"""
+from . import (tier0_apis, tier1_http, tier2_cloudscraper,
+               tier3_browser, tier3b_camoufox, tier_residential, tier4_firecrawl)
+
+TIERS = [
+    tier0_apis,
+    tier1_http,
+    tier2_cloudscraper,
+    tier3_browser,
+    tier3b_camoufox,   # env-gated Firefox stealth (off by default; orthogonal to T3)
+    tier_residential,  # residential-IP CDP browser (off unless BU_CDP_URL set)
+    tier4_firecrawl,
+]
+
+# tier name -> index, for botwall winning-tier routing.
+INDEX = {t.NAME: i for i, t in enumerate(TIERS)}

switchback/tiers/_browser.py

@@ -0,0 +1,50 @@
+"""Shared helpers for the stealth-browser tiers (patchright, camoufox).
+
+Not a tier itself (leading underscore, not in the TIERS registry) — just the
+challenge-resolution mechanic both browsers need.
+
+Akamai Bot Manager / Imperva / Kasada serve a JS-*sensor* interstitial on the
+first load: it sets cookies (e.g. ak_bmsc, _abck) as the sensor script runs, then
+the *real* content is only returned on a re-request. A single settle + reload
+after the sensor runs clears them — provided the egress IP is acceptable. (A hard
+IP block never validates `_abck` and the page stays an interstitial, so the tier
+still falls through; this just stops us snapshotting the interstitial too early on
+IPs that would have passed.)
+"""
+from __future__ import annotations
+
+from ..normalize import html_to_markdown
+from ..policy.gates import _looks_like_botwall
+
+_SETTLE_MS = 5000        # let the sensor JS run and set its cookies
+_POST_RELOAD_MS = 1500   # let the real content paint after the reload
+
+
+def looks_blocked(html: str, url: str) -> bool:
+    """True when the rendered DOM is a bot-wall / sensor interstitial."""
+    return _looks_like_botwall(html_to_markdown(html, base_url=url))
+
+
+def response_bytes(responses) -> int:
+    """Total wire bytes a render pulled across every resource — the residential-cost
+    basis. Reads only the Content-Length header: it's non-blocking. (We deliberately
+    do NOT call resp.body() — on a stalled response body() blocks with no timeout and
+    can freeze the whole render, which is uninterruptible by the cascade deadline.)
+    Responses without a Content-Length are skipped, so this slightly undercounts."""
+    total = 0
+    for resp in responses:
+        try:
+            cl = resp.headers.get("content-length")
+            if cl:
+                total += int(cl)
+        except Exception:
+            pass
+    return total
+
+
+def reload_through_challenge(page, url: str, timeout_ms: int) -> str:
+    """Settle so the bot-manager sensor JS runs, reload once, return fresh html."""
+    page.wait_for_timeout(_SETTLE_MS)
+    page.goto(url, wait_until="networkidle", timeout=timeout_ms)
+    page.wait_for_timeout(_POST_RELOAD_MS)
+    return page.content()

switchback/tiers/tier0_apis.py

@@ -0,0 +1,77 @@
+"""Tier 0 — direct APIs / open mirrors.
+
+Cheapest, cleanest, most reliable. Pattern-routed: returns None when no mirror
+matches (caller falls through to Tier 1). Bypasses the botwall skip-list because
+these are open, stable endpoints — not the scraped page.
+
+Extend here with career-ops-style structured providers (greenhouse/lever/ashby).
+Web *search* (query → URLs) is a different shape and lives in switchback/search.py
+(local SearXNG), not in this fetch cascade.
+"""
+from __future__ import annotations
+
+import re
+from urllib.parse import unquote
+from xml.etree import ElementTree as ET
+
+from ..normalize import html_to_markdown, UA
+from ..policy.gates import check
+
+NAME = "tier0_apis"
+PAID = False
+
+ARXIV_RE = re.compile(r"arxiv\.org/(?:abs|pdf)/(\d{4}\.\d{4,5})(?:v\d+)?(?:\.pdf)?", re.I)
+WIKI_RE = re.compile(r"en\.wikipedia\.org/wiki/([^?#]+)", re.I)
+PMC_RE = re.compile(r"pmc\.ncbi\.nlm\.nih\.gov/articles/(PMC\d+)", re.I)
+
+
+def fetch(url: str) -> str | None:
+    m = ARXIV_RE.search(url)
+    if m:
+        return _arxiv(m.group(1), url)
+    m = WIKI_RE.search(url)
+    if m:
+        return _wikipedia(m.group(1), url)
+    if PMC_RE.search(url):
+        return _europepmc(url)
+    return None  # no mirror — fall through
+
+
+def _arxiv(arxiv_id: str, url: str) -> str:
+    # arxiv wants plain requests + an identifying UA (their published guidance);
+    # impersonating Chrome triggers aggressive 429s from their Akamai front-end.
+    import requests
+    r = requests.get(f"https://export.arxiv.org/api/query?id_list={arxiv_id}",
+                     timeout=15,
+                     headers={"User-Agent": "switchback/1.0 (mailto:akash@theaklabs.com)"})
+    r.raise_for_status()
+    ns = {"atom": "http://www.w3.org/2005/Atom"}
+    entry = ET.fromstring(r.text).find("atom:entry", ns)
+    if entry is None:
+        raise RuntimeError("arxiv: no entry in API response")
+    title = (entry.findtext("atom:title", "", ns) or "").strip()
+    summary = (entry.findtext("atom:summary", "", ns) or "").strip()
+    authors = [a.findtext("atom:name", "", ns) or "" for a in entry.findall("atom:author", ns)]
+    md = (f"# {title}\n\n**Authors:** {', '.join(a for a in authors if a)}\n\n"
+          f"**arXiv:** {arxiv_id}\n\n## Abstract\n\n{summary}")
+    return check(url, md)
+
+
+def _wikipedia(title: str, url: str) -> str:
+    from curl_cffi import requests as cffi
+    r = cffi.get(f"https://en.wikipedia.org/api/rest_v1/page/html/{unquote(title)}",
+                 timeout=15, impersonate="chrome")
+    r.raise_for_status()
+    return check(url, html_to_markdown(r.text, base_url=url))
+
+
+def _europepmc(url: str) -> str:
+    # PMC full text via EuropePMC mirror (avoids reCAPTCHA on ncbi).
+    import requests
+    pmcid = PMC_RE.search(url).group(1)
+    api = f"https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/fullTextXML"
+    r = requests.get(api, timeout=20, headers={"User-Agent": UA})
+    r.raise_for_status()
+    if len(r.text) < 1000:
+        raise RuntimeError(f"europepmc empty: {len(r.text)}")
+    return check(url, html_to_markdown(r.text, base_url=url))

switchback/tiers/tier1_http.py

@@ -0,0 +1,65 @@
+"""Tier 1 — plain HTTP with TLS fingerprint impersonation.
+
+curl_cffi impersonates a real Chrome TLS handshake, which clears many naive bot
+walls without a browser. Handles PDFs inline. Fast and cheap.
+
+The bare "chrome" alias resolves to an old default; we pin recent targets and
+rotate them deterministically per host, so our traffic isn't one shared JA3 yet
+each host stays reproducible (and pairs cleanly with the session cache, which
+records the target that won).
+
+No User-Agent override: the impersonate target already sends a UA that matches
+its TLS fingerprint. Overriding it with a stale string is a detection tell (TLS
+says one Chrome version, the header says another).
+"""
+from __future__ import annotations
+
+import hashlib
+from urllib.parse import urlsplit
+
+from .. import session_cache
+from ..egress import requests_proxies, add_wire_bytes
+from ..normalize import html_to_markdown, pdf_bytes_to_text
+from ..policy.gates import BotWall, check, is_cf_challenge
+
+NAME = "tier1_http"
+PAID = False
+
+# Recent Chrome JA3 targets available in curl_cffi 0.15.x. A small spread of real
+# versions mirrors how live traffic is distributed across Chrome releases.
+_IMPERSONATE_TARGETS = ("chrome131", "chrome136", "chrome142")
+
+
+def _impersonate_for(url: str) -> str:
+    host = urlsplit(url).hostname or ""
+    h = int(hashlib.sha1(host.encode()).hexdigest(), 16)
+    return _IMPERSONATE_TARGETS[h % len(_IMPERSONATE_TARGETS)]
+
+
+def fetch(url: str) -> str:
+    from curl_cffi import requests as cffi
+    # Auth cookies only: the cached cf_clearance is UA-bound to whichever tier
+    # solved it, and CF hosts route straight to Tier 2 on repeat, so replaying it
+    # against Tier 1's distinct impersonate UA would be a mismatch tell.
+    cookie = session_cache.cookie_header(url, include_cache=False)
+    headers = {"Cookie": cookie} if cookie else None
+    r = cffi.get(url, timeout=15, allow_redirects=True,
+                 impersonate=_impersonate_for(url),
+                 proxies=requests_proxies(), headers=headers)
+    add_wire_bytes(len(r.content))  # count even on a block — failed fetches burn bandwidth too
+    if r.status_code >= 400:
+        # A Cloudflare JS challenge often returns 403/503 with the interstitial in
+        # the body. Surface that as a botwall (Tier 2 can solve it) rather than a
+        # hard http_block — which the orchestrator uses to skip Tier 2 entirely.
+        if is_cf_challenge(r.headers, r.text):
+            raise BotWall("cloudflare challenge", vendor="cloudflare")
+        r.raise_for_status()
+    ctype = r.headers.get("Content-Type", "").lower()
+    is_pdf = "application/pdf" in ctype or r.url.lower().split("?")[0].endswith(".pdf")
+    if is_pdf:
+        try:
+            text = pdf_bytes_to_text(r.content)
+        finally:
+            r.close()
+        return check(url, text)
+    return check(url, html_to_markdown(r.text, base_url=r.url))

switchback/tiers/tier2_cloudscraper.py

@@ -0,0 +1,135 @@
+"""Tier 2 — Cloudflare / anti-bot solver (cloudscraper 3.x "Enhanced Edition").
+
+Targets the specific failure the cheaper tiers can't clear: a Cloudflare JS
+challenge / "checking your browser" interstitial. Solves it in-process (no
+browser), then returns the real page. First hit to a CF host sleeps ~5s.
+
+cloudscraper 3.x clears v1/v2/v3 (JS-VM) challenges and Turnstile, with stealth
+on by default (randomized headers, browser quirks, human-like pacing) and
+automatic cf_clearance refresh on 403. Pinned to the GitHub Enhanced Edition;
+PyPI is frozen at 1.2.71 (v1/v2 only, no stealth) — see pyproject.toml.
+
+On hard CAPTCHA variants with no solver configured this raises and the cascade
+falls through to the stealth browser (Tier 3).
+"""
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import threading
+
+from .. import egress, session_cache
+from ..egress import requests_proxies
+from ..normalize import html_to_markdown
+from ..policy.gates import check
+
+logger = logging.getLogger(__name__)
+
+NAME = "tier2_cloudscraper"
+PAID = False
+
+# Wall-clock cap on the whole solve. cloudscraper 3.x *attempts* interactive
+# Turnstile and can loop for minutes on a challenge it can't clear — far past the
+# per-request socket timeout. Capping it here lets the cascade fall through to the
+# stealth browser (which can handle interactive challenges) instead of burning the
+# per-URL deadline. ~25s comfortably covers a real JS/v3 solve (~5-15s).
+_TIMEOUT_S = float(os.getenv("SCRAPER_CLOUDSCRAPER_TIMEOUT_S", "25"))
+
+# Stealth pacing. Kept modest: Tier 2 only fires on CF-suspected hosts, and the
+# real latency win comes from skipping the solve entirely on repeat hits (session
+# cache), not from long inter-request sleeps.
+_STEALTH_OPTIONS = {
+    "min_delay": 0.5,
+    "max_delay": 1.5,
+    "human_like_delays": True,
+    "randomize_headers": True,
+    "browser_quirks": True,
+}
+
+
+_captcha_warned = False
+
+
+def _captcha_opts() -> dict:
+    """Opt-in third-party captcha solver (off by default). When both env vars are
+    set, cloudscraper solves Turnstile / reCAPTCHA / hCaptcha on CF hosts in-process
+    via the provider (2captcha, capsolver, capmonster, anticaptcha, deathbycaptcha,
+    9kw). PAID: the provider bills per solve. cloudscraper resets its solve counter
+    on success, so per-solve counts aren't observable here — track spend in the
+    provider's own dashboard."""
+    provider = os.getenv("SCRAPER_CAPTCHA_PROVIDER")
+    api_key = os.getenv("SCRAPER_CAPTCHA_API_KEY")
+    if not (provider and api_key):
+        return {}
+    global _captcha_warned
+    if not _captcha_warned:
+        logger.warning(f"tier2: captcha solver active (provider={provider}); "
+                       "solves are billed by the provider")
+        _captcha_warned = True
+    return {"captcha": {"provider": provider, "api_key": api_key}}
+
+
+def _interpreter_opts() -> dict:
+    """The v3 JS-VM challenge runs an interpreter. The 3.x default js2py is pure
+    Python — slow and prone to stalling on heavy challenges; Node runs them fast
+    and reliably. Prefer it when present, else fall back to the default."""
+    return {"interpreter": "nodejs"} if shutil.which("node") else {}
+
+
+def _make_scraper():
+    import cloudscraper
+    # enable_stealth / auto_refresh_on_403 are on by default in 3.x; we pass the
+    # stealth tuning explicitly. No UA override: cloudscraper derives a UA (and
+    # matching cipher suite) from the browser dict; a stale override contradicts it.
+    return cloudscraper.create_scraper(
+        browser={"browser": "chrome", "platform": "linux", "mobile": False},
+        enable_stealth=True,
+        stealth_options=_STEALTH_OPTIONS,
+        **_interpreter_opts(),
+        **_captcha_opts(),
+    )
+
+
+def _fetch(url: str) -> str:
+    scraper = _make_scraper()
+    # Replay a cached cf_clearance (skips the ~5s solve) plus any auth cookies.
+    cookies = session_cache.cookies_for(url, include_cache=True)
+    r = scraper.get(url, timeout=20, proxies=requests_proxies(),
+                    cookies=cookies or None)
+    r.raise_for_status()
+    nbytes = len(r.content)
+    md = check(url, html_to_markdown(r.text, base_url=r.url))
+    # Cleared: cache whatever cf cookies the session now holds for next time.
+    session_cache.remember(url, dict(scraper.cookies),
+                           ua=scraper.headers.get("User-Agent", ""))
+    return md, nbytes
+
+
+def fetch(url: str) -> str:
+    # Run the (blocking, occasionally runaway) solve under a hard wall-clock cap.
+    # A daemon worker means an abandoned solve can't block process exit; it dies on
+    # its own socket timeout shortly after. Thread-locals don't inherit, so the
+    # egress scope is re-applied inside the worker.
+    scoped = egress.in_egress_scope()
+    box: dict = {}
+
+    def work():
+        with egress.egress_scope(scoped):
+            try:
+                box["md"], box["bytes"] = _fetch(url)
+            except BaseException as e:  # noqa: BLE001 — propagated to caller below
+                box["err"] = e
+
+    t = threading.Thread(target=work, name="tier2-cloudscraper", daemon=True)
+    t.start()
+    t.join(_TIMEOUT_S)
+    if t.is_alive():
+        raise TimeoutError(
+            f"cloudscraper exceeded {_TIMEOUT_S}s (unsolvable challenge); "
+            "falling through to the stealth browser")
+    # Re-attribute the worker's wire bytes here, in the scope-owning thread.
+    egress.add_wire_bytes(box.get("bytes", 0))
+    if "err" in box:
+        raise box["err"]
+    return box["md"]

switchback/tiers/tier3_browser.py

@@ -0,0 +1,59 @@
+"""Tier 3 — stealth headless browser (patchright).
+
+Renders JS-heavy SPAs. patchright is a hardened Playwright fork that evades the
+common automation fingerprints. Tries domcontentloaded first, then networkidle
+if the DOM is suspiciously small (lazy/JS content).
+
+Future: (a) batch many URLs through one browser (musings does this — big perf
+win); (b) browser-harness mode to drive the user's logged-in Chrome over CDP for
+auth-walled pages (set BU_CDP_URL).
+"""
+from __future__ import annotations
+
+from . import _browser
+from .. import session_cache, session_trace
+from ..concurrency import browser_slot
+from ..egress import playwright_proxy, add_wire_bytes
+from ..normalize import html_to_markdown
+from ..policy.gates import check
+
+NAME = "tier3_browser"
+PAID = False
+
+
+def fetch(url: str, timeout_ms: int = 15000) -> str:
+    from patchright.sync_api import sync_playwright
+    with browser_slot(NAME), sync_playwright() as p:
+        browser = p.chromium.launch(headless=True, proxy=playwright_proxy())
+        ctx = None
+        try:
+            # No user_agent override: patchright ships a real, internally
+            # consistent Chromium fingerprint; overriding the UA desyncs it from
+            # the engine version / client hints and defeats the stealth fork.
+            ctx = browser.new_context()
+            session_trace.start(ctx, url)
+            auth = session_cache.browser_cookies(url)
+            if auth:
+                ctx.add_cookies(auth)
+            page = ctx.new_page()
+            responses: list = []
+            page.on("response", lambda resp: responses.append(resp))
+            page.goto(url, timeout=timeout_ms, wait_until="domcontentloaded")
+            html = page.content()
+            if len(html) < 5000:
+                try:
+                    page.wait_for_load_state("networkidle", timeout=8000)
+                except Exception:
+                    pass
+                html = page.content()
+            # A JS bot-manager (Akamai/Imperva/…) may serve a sensor interstitial
+            # first; settle + reload once to get the real page on an acceptable IP.
+            if _browser.looks_blocked(html, page.url or url):
+                html = _browser.reload_through_challenge(page, url, timeout_ms)
+            add_wire_bytes(_browser.response_bytes(responses))
+            md = html_to_markdown(html, base_url=page.url or url)
+        finally:
+            if ctx is not None:
+                session_trace.stop(ctx, url)
+            browser.close()
+    return check(url, md)

switchback/tiers/tier3b_camoufox.py

@@ -0,0 +1,89 @@
+"""Tier 3b — Camoufox (hardened Firefox), env-gated stealth.
+
+An *orthogonal* fingerprint to the Chromium patchright tier: Camoufox patches
+stealth at the C++ level and is best-in-class for **headless** detection evasion,
+so it can clear hosts where the Chromium browser still gets blocked. It's the
+slowest rung we own (~40s on a hard Cloudflare solve), but it only fires after
+the four cheaper tiers AND patchright all miss, so easy traffic never pays for
+it. ON by default — opt out with SCRAPER_DISABLE_CAMOUFOX=1.
+
+Needs its Firefox build (`camoufox fetch`); if absent the launch raises and the
+cascade falls through to Firecrawl. Tried after the Chromium browser misses,
+before the paid Firecrawl tier.
+"""
+from __future__ import annotations
+
+import logging
+import os
+
+from . import _browser
+from .. import session_cache, session_trace
+from ..concurrency import browser_slot
+from ..egress import playwright_proxy, add_wire_bytes
+from ..normalize import html_to_markdown
+from ..policy.gates import check
+
+logger = logging.getLogger(__name__)
+
+NAME = "tier3b_camoufox"
+PAID = False
+
+_TIMEOUT_MS = int(os.getenv("SCRAPER_CAMOUFOX_TIMEOUT_MS", "45000"))
+
+
+def disabled() -> bool:
+    """On by default; opt out (heavy + slow) with SCRAPER_DISABLE_CAMOUFOX=1."""
+    return bool(os.getenv("SCRAPER_DISABLE_CAMOUFOX"))
+
+
+def _geoip_available() -> bool:
+    """camoufox's geoip matching needs the `geoip2` package (the camoufox[geoip]
+    extra). Without it, requesting geoip raises and kills the whole tier — so we
+    probe and degrade gracefully instead."""
+    try:
+        import geoip2  # noqa: F401
+        return True
+    except Exception:
+        return False
+
+
+def _launch_opts() -> dict:
+    """Camoufox evasion knobs. We launch it bare no longer: humanize + randomized
+    desktop OS make the fingerprint blend in; with a proxy set we also turn on
+    geoip so timezone/locale/geolocation match the proxy's IP (a mismatch there
+    is itself a tell) — but only when the geoip extra is installed."""
+    opts: dict = {"headless": True, "humanize": True,
+                  "os": ["windows", "macos", "linux"]}
+    proxy = playwright_proxy()
+    if proxy:
+        opts["proxy"] = proxy
+        if _geoip_available():
+            opts["geoip"] = True
+        else:
+            logger.warning("camoufox: proxy set but geoip extra missing "
+                           "(pip install camoufox[geoip]); locale/timezone won't "
+                           "match the proxy IP — a possible detection tell")
+    return opts
+
+
+def fetch(url: str) -> str:
+    from camoufox.sync_api import Camoufox
+    with browser_slot(NAME), Camoufox(**_launch_opts()) as browser:
+        page = browser.new_page()
+        responses: list = []
+        page.on("response", lambda resp: responses.append(resp))
+        try:
+            session_trace.start(page.context, url)
+            auth = session_cache.browser_cookies(url)
+            if auth:
+                page.context.add_cookies(auth)
+            page.goto(url, wait_until="networkidle", timeout=_TIMEOUT_MS)
+            html = page.content()
+            # JS bot-manager sensor interstitial → settle + reload once.
+            if _browser.looks_blocked(html, url):
+                html = _browser.reload_through_challenge(page, url, _TIMEOUT_MS)
+            add_wire_bytes(_browser.response_bytes(responses))
+        finally:
+            session_trace.stop(page.context, url)
+            page.close()
+    return check(url, html_to_markdown(html, base_url=url))

switchback/tiers/tier4_firecrawl.py

@@ -0,0 +1,48 @@
+"""Tier 4 — Firecrawl (paid, last resort).
+
+Env-gated: set SCRAPER_DISABLE_FIRECRAWL to skip this tier entirely (URL is then
+dropped). Every invocation is audited and feeds the botwall promotion counter, so
+hosts that keep needing it get auto-skipped. Needs FIRECRAWL_API_KEY.
+"""
+from __future__ import annotations
+
+import os
+import threading
+
+from ..policy.gates import check
+
+NAME = "tier4_firecrawl"
+PAID = True
+
+
+def disabled() -> bool:
+    return bool(os.getenv("SCRAPER_DISABLE_FIRECRAWL"))
+
+
+def _scrape(url: str) -> str:
+    from firecrawl import Firecrawl
+    app = Firecrawl(api_key=os.environ["FIRECRAWL_API_KEY"])
+    doc = app.scrape(url, formats=["markdown"])
+    d = doc.model_dump() if hasattr(doc, "model_dump") else (doc if isinstance(doc, dict) else {})
+    return check(url, (d.get("markdown") or "").strip())
+
+
+def fetch(url: str) -> str:
+    # Run in a dedicated thread: the Firecrawl SDK sets an asyncio event loop on
+    # the calling thread, which then makes a later sync-Playwright browser tier in
+    # the same batch raise "Sync API inside the asyncio loop". A worker thread
+    # confines that loop so the browser tiers stay usable across a multi-URL run.
+    box: dict = {}
+
+    def work():
+        try:
+            box["md"] = _scrape(url)
+        except BaseException as e:  # noqa: BLE001 — re-raised to the caller below
+            box["err"] = e
+
+    t = threading.Thread(target=work, name="tier4-firecrawl", daemon=True)
+    t.start()
+    t.join()
+    if "err" in box:
+        raise box["err"]
+    return box["md"]