switchback 0.2.0__tar.gz → 0.5.0__tar.gz

{switchback-0.2.0 → switchback-0.5.0}/.env.example

@@ -11,7 +11,7 @@
 OTEL_SERVICE_NAME=switchback
 OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 
-# ── Search (Tier-0 SearXNG, query → URLs) ───────────────────────────────────
+# ── Search (SearXNG, query → URLs — separate from the fetch cascade) ─────────
 SEARXNG_URL=http://localhost:8888
 
 # ── Output format ───────────────────────────────────────────────────────────
@@ -27,22 +27,38 @@ SEARXNG_URL=http://localhost:8888
 # fall back to their text for those sources.
 SCRAPER_OUTPUT_FORMAT=markdown
 
-# ── Tier 2.5 · Jina Reader (r.jina.ai) ──────────────────────────────────────
-# Optional: keyless works at 20 RPM. A key gives 500 RPM + a 10M-token grant.
-JINA_API_KEY=
-SCRAPER_JINA_TIMEOUT_S=20
+# ── tier_3 · Cloudflare solver (cloudscraper) ───────────────────────────────
+# Needs the 3.x Enhanced Edition fork (see README); with the frozen PyPI build
+# the tier reports `unavailable` and fails fast. Wall-clock cap on a single solve
+# so an unsolvable challenge can't eat the per-URL deadline before the browser
+# tier runs. Lower (e.g. 12) if tier_3 rarely wins on your hosts.
+SCRAPER_TIER_3_TIMEOUT_S=25
 
-# ── Tier 3b · Camoufox (Firefox stealth) ────────────────────────────────────
+# ── tier_5 · Camoufox (Firefox stealth) ─────────────────────────────────────
 # ON by default. Needs: pip install camoufox && camoufox fetch
 # Set to 1 to turn the tier off entirely.
 SCRAPER_DISABLE_CAMOUFOX=
-SCRAPER_CAMOUFOX_TIMEOUT_MS=45000
+SCRAPER_TIER_5_TIMEOUT_S=45
 
-# ── Tier 4 · Firecrawl (paid, last resort) ──────────────────────────────────
+# ── tier_7 · Firecrawl (paid, last resort) ──────────────────────────────────
 # Required only if this tier runs. Set SCRAPER_DISABLE_FIRECRAWL=1 to skip it.
 FIRECRAWL_API_KEY=
 SCRAPER_DISABLE_FIRECRAWL=
 
+# ── Per-tier timeouts (seconds) ─────────────────────────────────────────────
+# Each tier's wall-clock/socket cap; override any of them. Defaults shown below.
+# tier_3 (=25) and tier_5 (=45) are set live in their sections above; the rest
+# fall back to these defaults. The pre-0.5.0 names are still honored when the new
+# var is unset: SCRAPER_CLOUDSCRAPER_TIMEOUT_S → tier_3,
+# SCRAPER_CAMOUFOX_TIMEOUT_MS → tier_5, SCRAPER_RESIDENTIAL_TIMEOUT_MS → tier_6.
+#SCRAPER_TIER_1_TIMEOUT_S=15    # direct APIs / open mirrors
+#SCRAPER_TIER_2_TIMEOUT_S=15    # plain HTTP + TLS impersonation
+#SCRAPER_TIER_3_TIMEOUT_S=25    # cloudscraper (Cloudflare solver)
+#SCRAPER_TIER_4_TIMEOUT_S=15    # stealth headless browser (patchright)
+#SCRAPER_TIER_5_TIMEOUT_S=45    # camoufox (slowest rung; hard CF solves ~40s)
+#SCRAPER_TIER_6_TIMEOUT_S=30    # residential-IP CDP browser
+#SCRAPER_TIER_7_TIMEOUT_S=15    # Firecrawl (paid) — was unbounded; raise if scrapes get cut off
+
 # ── Orchestrator ────────────────────────────────────────────────────────────
 # Per-URL wall-clock budget (s), checked between tiers. 45s balances latency vs
 # coverage — roughly fits a Camoufox solve (~40s) that starts after the cheaper

switchback-0.5.0/CHANGELOG.md

@@ -0,0 +1,119 @@
+# Changelog
+
+All notable changes to this project are documented here. Format loosely follows
+[Keep a Changelog](https://keepachangelog.com/); this project uses semantic-ish
+versioning while pre-1.0.
+
+## [Unreleased]
+
+## [0.5.0] - 2026-06-30
+
+### Changed
+- **Tiers renamed to plain `tier_1`…`tier_7`** (cost-ordered, contiguous) in place
+  of the old mixed scheme (`tier0_apis`, `tier1_http`, `tier2_cloudscraper`,
+  `tier3_browser`, `tier3b_camoufox`, `tier_residential`, `tier4_firecrawl`). The
+  mapping is positional: `tier_1`=apis, `tier_2`=http, `tier_3`=cloudscraper,
+  `tier_4`=browser, `tier_5`=camoufox, `tier_6`=residential, `tier_7`=firecrawl.
+  **Backwards-compatible:** an existing `state/botwall_db.json` is migrated on load
+  (a host's learned `winning_tier` / `tier_stats` keys are remapped to the new
+  names), so routing survives the upgrade instead of re-probing from scratch.
+
+### Added
+- **Per-tier timeout knobs** — every tier now reads `SCRAPER_TIER_<N>_TIMEOUT_S`
+  (seconds, `N` = 1–7). Defaults: `15` for tiers without a prior budget
+  (apis/http/browser), and the existing budgets are preserved — `tier_3`=25,
+  `tier_5`=45, `tier_6`=30. The previously-unconfigurable/unbounded tiers (apis,
+  http, browser, **firecrawl**) are now bounded and overridable. The pre-rename
+  `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` / `SCRAPER_CAMOUFOX_TIMEOUT_MS` /
+  `SCRAPER_RESIDENTIAL_TIMEOUT_MS` are still honored when the new var is unset.
+  Note: `tier_7` (paid Firecrawl) was previously unbounded; its new `15`s default
+  bounds it — raise `SCRAPER_TIER_7_TIMEOUT_S` if hard hosts get cut off (a scrape
+  killed at the cap may still be billed). `SCRAPER_TIER_RETRIES_<TIER>` overrides
+  follow the new names (e.g. `SCRAPER_TIER_RETRIES_TIER_4`).
+
+## [0.4.0] - 2026-06-29
+
+### Added
+- **Configurable per-tier retries** — a tier can now re-attempt before falling
+  through to the next, more capable one. `SCRAPER_TIER_RETRIES` (global, default
+  `0` = off; `N` → up to `1+N` tries per tier), per-tier overrides
+  `SCRAPER_TIER_RETRIES_<TIER>` (e.g. `SCRAPER_TIER_RETRIES_TIER3_BROWSER=2`), and
+  `SCRAPER_TIER_RETRY_ON` (retryable failure classes; default
+  `timeout,rate_limited,connection` — widen to include `botwall,http_block` behind
+  a rotating residential proxy, where each retry gets a fresh IP). Retries stay
+  bounded by `SCRAPER_DEADLINE_S`, and intermediate retries are traced/logged but
+  **not** persisted to the botwall policy DB, so they never inflate the
+  self-healing skip / `needs_egress` counters. Default `0` keeps behaviour
+  unchanged. Enabling retries on the paid Firecrawl tier bills per attempt.
+
+### Fixed
+- **Quality gate rejects content shells** — the gate no longer passes a page just
+  because it clears the length floor; thin "shell" pages (nav/boilerplate with no
+  real article body) are now treated as a tier miss so the cascade falls through.
+- **Paid last-resort budget reserve** — `SCRAPER_FIRECRAWL_FALLBACK_AFTER_S`
+  (default 25s) stops starting local tiers once enough of the per-URL deadline has
+  elapsed and an enabled paid tier is still ahead, so a hard host can't burn the
+  whole budget before Firecrawl gets a turn.
+
+## [0.3.0] - 2026-06-27
+
+### Added
+- **`unavailable` tier outcome** — when a tier's optional dependency is missing,
+  the wrong version, or not installed yet (frozen PyPI `cloudscraper` instead of
+  the 3.x stealth fork; patchright's Chromium not downloaded during an async
+  cold-start install), the tier now fails fast (~0ms) with a distinct
+  `unavailable` outcome carrying the exact install command, logged once per tier.
+  It ranks above bot-wall in the verdict, so an environment problem is no longer
+  masked as `botwall` — and a missing Tier 2 dependency no longer burns the
+  per-URL solve budget before the browser tier runs.
+- **`switchback --doctor`** — preflight tier-readiness check (doubles as a
+  healthcheck: exit 0 when the capable tiers are ready). Reports whether
+  cloudscraper is the stealth-capable 3.x fork, patchright + Chromium are
+  installed, Camoufox/Node are present, and Firecrawl is configured. Built for
+  cold-start deploys where the browser is installed by a background thread after
+  boot.
+
+### Docs
+- README **Production / cold-start deployment** section and a `.env.example`
+  Tier 2 block: install `patchright install chromium` in the post-boot step, the
+  cloudscraper 3.x fork requirement, Node.js for Tier 2 concurrency, and the
+  `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` budget knob.
+
+## [0.2.0] - 2026-06-25
+
+### Added
+- **Selectable output formats** — `SCRAPER_OUTPUT_FORMAT` (or per-call
+  `scrape(fmt=...)`, CLI `--format`, `/scrape` `{"format": ...}`) selects the
+  content shape: `markdown` (default, unchanged), `markdown_trimmed` (extra
+  ad/nav/boilerplate removed), `html` (raw), or `html_selectors` (cleaned HTML
+  with per-domain `drop`/`selector` applied). Default output is byte-identical;
+  html-family results use a `html` JSON key instead of `markdown`.
+
+## [0.1.0] - 2026-06-23
+
+### Added
+- **Challenge-type learning** — bot-walls are classified by vendor (Cloudflare,
+  DataDome, Akamai, PerimeterX, Incapsula, Google) and counted per host in the
+  botwall DB; the vendor is attached to each event and OTel span (`scrape.challenge`).
+- **Metrics & reporting** — `switchback.reporting` rolls the event log + botwall DB
+  into cost-savings-vs-Firecrawl, coverage, overall/per-tier/per-domain latency
+  (mean/median/min/max/p50/p95), outcomes, error codes by domain, and challenges
+  by domain. Exposed via `GET /metrics` and `GET /metrics/domains` (both accept
+  `?minutes=N`).
+- **Periodic flagging** — `python -m switchback.flags` emits a cron-friendly digest
+  (domains stuck on Firecrawl, escalated to egress, most-challenged) to logs/OTel.
+- **Content cache** — optional URL→result cache (`SCRAPER_CONTENT_TTL_S`, sqlite,
+  off by default) short-circuits re-scrapes before any tier runs.
+- **Login-session refresh** — `SCRAPER_LOGIN_HOOK` (`pkg.module:func`) refreshes a
+  dead logged-in session on demand; cookies overlay every tier and persist.
+- **Exponential backoff** — between-tier backoff with jitter after rate-limit /
+  timeout (`SCRAPER_BACKOFF_BASE_MS` / `SCRAPER_BACKOFF_MAX_MS`, off by default).
+- **Per-domain extraction prefs** — `config/extraction.json` (CSS scope selector +
+  extra drops) applied automatically in the normalize step for every tier.
+- **Session traces** — opt-in Playwright trace capture (`SCRAPER_TRACE_SESSION=1`)
+  for browser tiers, with `GET/DELETE /traces` management endpoints.
+
+### Changed
+- Tier 2's `cloudscraper` moved from a core dependency (which pinned a git-URL
+  fork PyPI can't publish) to the `cloudflare` extra; see the README for installing
+  the 3.x Enhanced Edition fork for full stealth.

{switchback-0.2.0 → switchback-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: switchback
-Version: 0.2.0
+Version: 0.5.0
 Summary: One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool.
 Author-email: Akash Kodavuru <akash@theaklabs.com>
 License: MIT
@@ -121,13 +121,13 @@ That's the whole loop. Add tiers as you need them (see [Install](#install)).
 
 | Tier | Strategy | Cost |
 |---|---|---|
-| 0 | Direct APIs / mirrors (arxiv, wikipedia, EuropePMC; extend: job boards) | free, cleanest |
-| 1 | Plain HTTP + TLS impersonation (`curl_cffi`), incl. PDFs | cheap |
-| 2 | Cloudflare / anti-bot solver (`cloudscraper`, install `.[cloudflare]`) | cheap-ish (~5s/host) |
-| 3 | Stealth headless browser (`patchright`, Chromium) | heavy |
-| 3b | Camoufox (Firefox stealth) — **on by default** (opt out: `SCRAPER_DISABLE_CAMOUFOX`) | heavy + slow (~40s on hard CF) |
-| 3c | Residential-IP browser over CDP (`BU_CDP_URL`) — off unless configured | heavy (remote egress) |
-| 4 | Firecrawl (paid, env-gated, audited) | paid, last resort |
+| tier_1 | Direct APIs / mirrors (arxiv, wikipedia, EuropePMC; extend: job boards) | free, cleanest |
+| tier_2 | Plain HTTP + TLS impersonation (`curl_cffi`), incl. PDFs | cheap |
+| tier_3 | Cloudflare / anti-bot solver (`cloudscraper`, install `.[cloudflare]`) | cheap-ish (~5s/host) |
+| tier_4 | Stealth headless browser (`patchright`, Chromium) | heavy |
+| tier_5 | Camoufox (Firefox stealth) — **on by default** (opt out: `SCRAPER_DISABLE_CAMOUFOX`) | heavy + slow (~40s on hard CF) |
+| tier_6 | Residential-IP browser over CDP (`BU_CDP_URL`) — off unless configured | heavy (remote egress) |
+| tier_7 | Firecrawl (paid, env-gated, audited) | paid, last resort |
 
 Every URL has a wall-clock budget (`SCRAPER_DEADLINE_S`, default 45s) checked between
 tiers so one URL can't run the whole cascade of timeouts. Each tier attempt records
@@ -142,17 +142,17 @@ Search (query → URLs) is separate from the scrape cascade: `switchback.search(
 ## Install
 
 ```bash
-pip install switchback                 # core: normalization + cheap tiers (0/1) + search
-pip install "switchback[cloudflare]"   # + Tier 2 Cloudflare/anti-bot solver (cloudscraper)
+pip install switchback                 # core: normalization + cheap tiers (tier_1/tier_2) + search
+pip install "switchback[cloudflare]"   # + tier_3 Cloudflare/anti-bot solver (cloudscraper)
 pip install "switchback[server]"       # + HTTP service (fastapi, uvicorn) incl. /metrics + /traces
-pip install "switchback[browser]" && patchright install chromium   # + Tier 3 stealth Chromium
-pip install "switchback[camoufox]" && camoufox fetch               # + Tier 3b Firefox stealth
-pip install "switchback[firecrawl]"    # + Tier 4 paid API (needs FIRECRAWL_API_KEY)
+pip install "switchback[browser]" && patchright install chromium   # + tier_4 stealth Chromium
+pip install "switchback[camoufox]" && camoufox fetch               # + tier_5 Firefox stealth
+pip install "switchback[firecrawl]"    # + tier_7 paid API (needs FIRECRAWL_API_KEY)
 pip install "switchback[tracing]"      # + OpenTelemetry -> any OTLP backend
 pip install "switchback[all]"          # everything
 ```
 
-For Tier 2's **full** v3 JS-VM + Turnstile + stealth, install the Enhanced Edition
+For tier_3's **full** v3 JS-VM + Turnstile + stealth, install the Enhanced Edition
 3.x fork (PyPI's `cloudscraper` is the older v1/v2 — PyPI forbids pinning a
 git-URL dep inside a published package, so install it alongside):
 
@@ -163,6 +163,35 @@ pip install "cloudscraper @ git+https://github.com/VeNoMouS/cloudscraper@3.0.0"
 Or run the whole thing as a container:
 `docker build -t switchback . && docker run -p 8799:8799 switchback`.
 
+### Production / cold-start deployment
+
+The two heavy tiers pull dependencies that often can't be baked into a base image
+and land *after* boot (e.g. an async install thread on Azure). Until they're
+ready, those tiers report **`unavailable`** (a distinct outcome carrying the exact
+fix) and the cascade falls through — they are never silently skipped. Checklist:
+
+- **tier_4 is the real workhorse for Cloudflare/JS sites** — make sure its browser
+  is installed: `patchright install chromium` (note: **patchright**, not vanilla
+  `playwright`). On a cold start, run this in your post-boot install step/thread;
+  tier_4 flips to ready once it finishes.
+- **tier_3 needs the cloudscraper 3.x fork** (above) to attempt stealth. With the
+  frozen PyPI `cloudscraper` it reports `unavailable` and fails fast (no wasted
+  solve budget) instead of erroring mid-cascade. tier_3 is a *weak* solver for
+  modern Cloudflare — treat it as a cheap try before the browser, not the primary.
+- **Install Node.js** for tier_3's v3 JS-VM challenges — faster and thread-safe
+  vs. the pure-Python js2py fallback (relevant under concurrent load).
+- **Bound tier_3's solve budget** with `SCRAPER_TIER_3_TIMEOUT_S` (default `25`;
+  the old `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` is still honored) so an unsolvable
+  challenge can't eat the per-URL deadline before the browser tier runs. Lower it
+  (e.g. `12`) if tier_3 rarely wins on your hosts.
+
+**Verify readiness on the box** with the preflight check (doubles as a healthcheck
+— exit 0 when the capable tiers are ready):
+
+```bash
+switchback --doctor          # or: python -m switchback --doctor
+```
+
 ## Use it from your app
 
 Three interchangeable entry points — all return the same shape
@@ -254,16 +283,16 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 <details>
 <summary><b>Env gates</b> — enable/disable tiers and integrations</summary>
 
-- `SCRAPER_DISABLE_FIRECRAWL` — skip Tier 4
-- `FIRECRAWL_API_KEY` — enable Tier 4
-- `SCRAPER_DISABLE_CAMOUFOX` — turn off Tier 3b (on by default; needs `pip install camoufox` + `camoufox fetch`)
-- `BU_CDP_URL` — enable Tier 3c residential browser by pointing at a CDP endpoint
+- `SCRAPER_DISABLE_FIRECRAWL` — skip tier_7
+- `FIRECRAWL_API_KEY` — enable tier_7
+- `SCRAPER_DISABLE_CAMOUFOX` — turn off tier_5 (on by default; needs `pip install camoufox` + `camoufox fetch`)
+- `BU_CDP_URL` — enable tier_6 residential browser by pointing at a CDP endpoint
 - `SCRAPER_PROXY` — route *all* tiers/URLs through a proxy
 - `SCRAPER_EGRESS_PROXY` — route only walled hosts through a proxy (see [Cost-scoped residential egress](#cost-scoped-residential-egress))
 - `SEARXNG_URL` — defaults to `http://localhost:8888`
 - `SCRAPER_STATE_DIR` — where the botwall DB/event log + session cache live
 - `SCRAPER_COOKIES_FILE` — Netscape `cookies.txt` to scrape login-gated hosts (injected into the HTTP and browser tiers)
-- `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into Tier 2 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.
+- `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into tier_3 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.
 </details>
 
 <details>
@@ -271,7 +300,8 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 
 - `SCRAPER_OUTPUT_FORMAT` — output shape: `markdown` (default) · `markdown_trimmed` · `html` · `html_selectors` (see [Output formats](#output-formats))
 - `SCRAPER_DEADLINE_S` — per-URL budget (45s)
-- `SCRAPER_CAMOUFOX_TIMEOUT_MS` — (45000)
+- `SCRAPER_FIRECRAWL_FALLBACK_AFTER_S` — after this many seconds on a URL, stop trying the local tiers and fall back to Firecrawl, so a hard host doesn't burn the whole deadline before the paid last resort gets a turn (25s; 0 = off)
+- `SCRAPER_TIER_<N>_TIMEOUT_S` — per-tier timeout in seconds, `N` = 1–7 (tier_1 apis · tier_2 http · tier_3 cloudscraper · tier_4 browser · tier_5 camoufox · tier_6 residential · tier_7 firecrawl). Defaults: 15/15/**25**/15/**45**/**30**/15 (the three bold ones keep their prior budgets; everything else is 15s). The pre-0.5.0 `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` / `SCRAPER_CAMOUFOX_TIMEOUT_MS` / `SCRAPER_RESIDENTIAL_TIMEOUT_MS` are still honored when the new var is unset. Note: `tier_7` (Firecrawl) was previously unbounded — its 15s default now bounds the paid tier, so raise `SCRAPER_TIER_7_TIMEOUT_S` if slow hosts get cut off
 - `SCRAPER_BROWSER_CONCURRENCY` — max simultaneous headless browsers (default 1)
 - `SCRAPER_BOTWALL_URL_SKIP_COOLDOWN_H` — auto-skip re-test window (24h; 0 = never)
 - `SCRAPER_BOTWALL_EGRESS_AFTER` — local-tier failures before a host escalates to the residential tier (default 2)
@@ -279,6 +309,8 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 - `SCRAPER_DISABLE_SESSION_CACHE` — turn off cf_clearance reuse
 - `SCRAPER_CONTENT_TTL_S` — URL→result cache TTL (**0 = off**; set e.g. 86400 to skip re-scraping a page within a day)
 - `SCRAPER_BACKOFF_BASE_MS` / `SCRAPER_BACKOFF_MAX_MS` — exponential backoff between tiers after a rate-limit/timeout (base 0 = off)
+- `SCRAPER_TIER_RETRIES` — same-tier retries before falling through (default 0 = off; `N` → up to `1+N` tries per tier), with per-tier overrides `SCRAPER_TIER_RETRIES_<TIER>` (e.g. `SCRAPER_TIER_RETRIES_TIER_4=2`)
+- `SCRAPER_TIER_RETRY_ON` — failure classes eligible for a same-tier retry (default `timeout,rate_limited,connection`; widen to include `botwall,http_block` behind a rotating residential proxy so each retry gets a fresh IP). Retries are bounded by `SCRAPER_DEADLINE_S`; enabling them on the paid Firecrawl tier bills per attempt
 - `SCRAPER_LOGIN_HOOK` — `pkg.module:func` returning `{cookie: value}` for a host (see [Logged-in sessions](#logged-in-sessions))
 - `SCRAPER_EXTRACTION_FILE` — per-domain extraction prefs JSON (default `config/extraction.json`)
 - `SCRAPER_TRACE_SESSION` — opt-in: capture a Playwright trace (screenshots + DOM + network) per browser-tier attempt, written to `state/traces/`

{switchback-0.2.0 → switchback-0.5.0}/README.md

@@ -62,13 +62,13 @@ That's the whole loop. Add tiers as you need them (see [Install](#install)).
 
 | Tier | Strategy | Cost |
 |---|---|---|
-| 0 | Direct APIs / mirrors (arxiv, wikipedia, EuropePMC; extend: job boards) | free, cleanest |
-| 1 | Plain HTTP + TLS impersonation (`curl_cffi`), incl. PDFs | cheap |
-| 2 | Cloudflare / anti-bot solver (`cloudscraper`, install `.[cloudflare]`) | cheap-ish (~5s/host) |
-| 3 | Stealth headless browser (`patchright`, Chromium) | heavy |
-| 3b | Camoufox (Firefox stealth) — **on by default** (opt out: `SCRAPER_DISABLE_CAMOUFOX`) | heavy + slow (~40s on hard CF) |
-| 3c | Residential-IP browser over CDP (`BU_CDP_URL`) — off unless configured | heavy (remote egress) |
-| 4 | Firecrawl (paid, env-gated, audited) | paid, last resort |
+| tier_1 | Direct APIs / mirrors (arxiv, wikipedia, EuropePMC; extend: job boards) | free, cleanest |
+| tier_2 | Plain HTTP + TLS impersonation (`curl_cffi`), incl. PDFs | cheap |
+| tier_3 | Cloudflare / anti-bot solver (`cloudscraper`, install `.[cloudflare]`) | cheap-ish (~5s/host) |
+| tier_4 | Stealth headless browser (`patchright`, Chromium) | heavy |
+| tier_5 | Camoufox (Firefox stealth) — **on by default** (opt out: `SCRAPER_DISABLE_CAMOUFOX`) | heavy + slow (~40s on hard CF) |
+| tier_6 | Residential-IP browser over CDP (`BU_CDP_URL`) — off unless configured | heavy (remote egress) |
+| tier_7 | Firecrawl (paid, env-gated, audited) | paid, last resort |
 
 Every URL has a wall-clock budget (`SCRAPER_DEADLINE_S`, default 45s) checked between
 tiers so one URL can't run the whole cascade of timeouts. Each tier attempt records
@@ -83,17 +83,17 @@ Search (query → URLs) is separate from the scrape cascade: `switchback.search(
 ## Install
 
 ```bash
-pip install switchback                 # core: normalization + cheap tiers (0/1) + search
-pip install "switchback[cloudflare]"   # + Tier 2 Cloudflare/anti-bot solver (cloudscraper)
+pip install switchback                 # core: normalization + cheap tiers (tier_1/tier_2) + search
+pip install "switchback[cloudflare]"   # + tier_3 Cloudflare/anti-bot solver (cloudscraper)
 pip install "switchback[server]"       # + HTTP service (fastapi, uvicorn) incl. /metrics + /traces
-pip install "switchback[browser]" && patchright install chromium   # + Tier 3 stealth Chromium
-pip install "switchback[camoufox]" && camoufox fetch               # + Tier 3b Firefox stealth
-pip install "switchback[firecrawl]"    # + Tier 4 paid API (needs FIRECRAWL_API_KEY)
+pip install "switchback[browser]" && patchright install chromium   # + tier_4 stealth Chromium
+pip install "switchback[camoufox]" && camoufox fetch               # + tier_5 Firefox stealth
+pip install "switchback[firecrawl]"    # + tier_7 paid API (needs FIRECRAWL_API_KEY)
 pip install "switchback[tracing]"      # + OpenTelemetry -> any OTLP backend
 pip install "switchback[all]"          # everything
 ```
 
-For Tier 2's **full** v3 JS-VM + Turnstile + stealth, install the Enhanced Edition
+For tier_3's **full** v3 JS-VM + Turnstile + stealth, install the Enhanced Edition
 3.x fork (PyPI's `cloudscraper` is the older v1/v2 — PyPI forbids pinning a
 git-URL dep inside a published package, so install it alongside):
 
@@ -104,6 +104,35 @@ pip install "cloudscraper @ git+https://github.com/VeNoMouS/cloudscraper@3.0.0"
 Or run the whole thing as a container:
 `docker build -t switchback . && docker run -p 8799:8799 switchback`.
 
+### Production / cold-start deployment
+
+The two heavy tiers pull dependencies that often can't be baked into a base image
+and land *after* boot (e.g. an async install thread on Azure). Until they're
+ready, those tiers report **`unavailable`** (a distinct outcome carrying the exact
+fix) and the cascade falls through — they are never silently skipped. Checklist:
+
+- **tier_4 is the real workhorse for Cloudflare/JS sites** — make sure its browser
+  is installed: `patchright install chromium` (note: **patchright**, not vanilla
+  `playwright`). On a cold start, run this in your post-boot install step/thread;
+  tier_4 flips to ready once it finishes.
+- **tier_3 needs the cloudscraper 3.x fork** (above) to attempt stealth. With the
+  frozen PyPI `cloudscraper` it reports `unavailable` and fails fast (no wasted
+  solve budget) instead of erroring mid-cascade. tier_3 is a *weak* solver for
+  modern Cloudflare — treat it as a cheap try before the browser, not the primary.
+- **Install Node.js** for tier_3's v3 JS-VM challenges — faster and thread-safe
+  vs. the pure-Python js2py fallback (relevant under concurrent load).
+- **Bound tier_3's solve budget** with `SCRAPER_TIER_3_TIMEOUT_S` (default `25`;
+  the old `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` is still honored) so an unsolvable
+  challenge can't eat the per-URL deadline before the browser tier runs. Lower it
+  (e.g. `12`) if tier_3 rarely wins on your hosts.
+
+**Verify readiness on the box** with the preflight check (doubles as a healthcheck
+— exit 0 when the capable tiers are ready):
+
+```bash
+switchback --doctor          # or: python -m switchback --doctor
+```
+
 ## Use it from your app
 
 Three interchangeable entry points — all return the same shape
@@ -195,16 +224,16 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 <details>
 <summary><b>Env gates</b> — enable/disable tiers and integrations</summary>
 
-- `SCRAPER_DISABLE_FIRECRAWL` — skip Tier 4
-- `FIRECRAWL_API_KEY` — enable Tier 4
-- `SCRAPER_DISABLE_CAMOUFOX` — turn off Tier 3b (on by default; needs `pip install camoufox` + `camoufox fetch`)
-- `BU_CDP_URL` — enable Tier 3c residential browser by pointing at a CDP endpoint
+- `SCRAPER_DISABLE_FIRECRAWL` — skip tier_7
+- `FIRECRAWL_API_KEY` — enable tier_7
+- `SCRAPER_DISABLE_CAMOUFOX` — turn off tier_5 (on by default; needs `pip install camoufox` + `camoufox fetch`)
+- `BU_CDP_URL` — enable tier_6 residential browser by pointing at a CDP endpoint
 - `SCRAPER_PROXY` — route *all* tiers/URLs through a proxy
 - `SCRAPER_EGRESS_PROXY` — route only walled hosts through a proxy (see [Cost-scoped residential egress](#cost-scoped-residential-egress))
 - `SEARXNG_URL` — defaults to `http://localhost:8888`
 - `SCRAPER_STATE_DIR` — where the botwall DB/event log + session cache live
 - `SCRAPER_COOKIES_FILE` — Netscape `cookies.txt` to scrape login-gated hosts (injected into the HTTP and browser tiers)
-- `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into Tier 2 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.
+- `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into tier_3 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.
 </details>
 
 <details>
@@ -212,7 +241,8 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 
 - `SCRAPER_OUTPUT_FORMAT` — output shape: `markdown` (default) · `markdown_trimmed` · `html` · `html_selectors` (see [Output formats](#output-formats))
 - `SCRAPER_DEADLINE_S` — per-URL budget (45s)
-- `SCRAPER_CAMOUFOX_TIMEOUT_MS` — (45000)
+- `SCRAPER_FIRECRAWL_FALLBACK_AFTER_S` — after this many seconds on a URL, stop trying the local tiers and fall back to Firecrawl, so a hard host doesn't burn the whole deadline before the paid last resort gets a turn (25s; 0 = off)
+- `SCRAPER_TIER_<N>_TIMEOUT_S` — per-tier timeout in seconds, `N` = 1–7 (tier_1 apis · tier_2 http · tier_3 cloudscraper · tier_4 browser · tier_5 camoufox · tier_6 residential · tier_7 firecrawl). Defaults: 15/15/**25**/15/**45**/**30**/15 (the three bold ones keep their prior budgets; everything else is 15s). The pre-0.5.0 `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` / `SCRAPER_CAMOUFOX_TIMEOUT_MS` / `SCRAPER_RESIDENTIAL_TIMEOUT_MS` are still honored when the new var is unset. Note: `tier_7` (Firecrawl) was previously unbounded — its 15s default now bounds the paid tier, so raise `SCRAPER_TIER_7_TIMEOUT_S` if slow hosts get cut off
 - `SCRAPER_BROWSER_CONCURRENCY` — max simultaneous headless browsers (default 1)
 - `SCRAPER_BOTWALL_URL_SKIP_COOLDOWN_H` — auto-skip re-test window (24h; 0 = never)
 - `SCRAPER_BOTWALL_EGRESS_AFTER` — local-tier failures before a host escalates to the residential tier (default 2)
@@ -220,6 +250,8 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 - `SCRAPER_DISABLE_SESSION_CACHE` — turn off cf_clearance reuse
 - `SCRAPER_CONTENT_TTL_S` — URL→result cache TTL (**0 = off**; set e.g. 86400 to skip re-scraping a page within a day)
 - `SCRAPER_BACKOFF_BASE_MS` / `SCRAPER_BACKOFF_MAX_MS` — exponential backoff between tiers after a rate-limit/timeout (base 0 = off)
+- `SCRAPER_TIER_RETRIES` — same-tier retries before falling through (default 0 = off; `N` → up to `1+N` tries per tier), with per-tier overrides `SCRAPER_TIER_RETRIES_<TIER>` (e.g. `SCRAPER_TIER_RETRIES_TIER_4=2`)
+- `SCRAPER_TIER_RETRY_ON` — failure classes eligible for a same-tier retry (default `timeout,rate_limited,connection`; widen to include `botwall,http_block` behind a rotating residential proxy so each retry gets a fresh IP). Retries are bounded by `SCRAPER_DEADLINE_S`; enabling them on the paid Firecrawl tier bills per attempt
 - `SCRAPER_LOGIN_HOOK` — `pkg.module:func` returning `{cookie: value}` for a host (see [Logged-in sessions](#logged-in-sessions))
 - `SCRAPER_EXTRACTION_FILE` — per-domain extraction prefs JSON (default `config/extraction.json`)
 - `SCRAPER_TRACE_SESSION` — opt-in: capture a Playwright trace (screenshots + DOM + network) per browser-tier attempt, written to `state/traces/`

{switchback-0.2.0 → switchback-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "switchback"
-version = "0.2.0"
+version = "0.5.0"
 description = "One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool."
 readme = "README.md"
 requires-python = ">=3.10"

{switchback-0.2.0 → switchback-0.5.0}/switchback/api.py

@@ -55,6 +55,7 @@ def _main() -> int:
                     _os.environ[_k] = _v.strip()
     usage = ("usage: switchback [--format FMT] <url> [<url> ...]\n"
              "       switchback --search <query ...>\n"
+             "       switchback --doctor\n"
              "       (or: python -m switchback <url> ...)\n"
              "  FMT: markdown (default) | markdown_trimmed | html | html_selectors")
     # --help/-h is an explicit request: usage to stdout, exit 0 (don't treat it
@@ -62,6 +63,10 @@ def _main() -> int:
     if any(a in ("--help", "-h") for a in sys.argv[1:]):
         print(usage)
         return 0
+    # --doctor: preflight tier-readiness report (no scrape). Side-effect-free.
+    if "--doctor" in sys.argv[1:]:
+        from .doctor import report
+        return report()
     logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
     setup_logs()  # also ship logs to the OTLP backend when configured
     if len(sys.argv) < 2:

switchback-0.5.0/switchback/doctor.py

@@ -0,0 +1,59 @@
+"""Preflight readiness check — `switchback doctor`.
+
+Reports which tiers can actually run on this box and, when one can't, the exact
+fix. Built for cold-start deploys (e.g. Azure) where the stealth browser is
+installed by a background thread *after* boot: run this to confirm the tiers are
+live before sending traffic, or to see why Tier 2/3 aren't catching anything.
+
+Exit code: 0 if both capable local tiers (cloudscraper + browser) are ready,
+else 1 — so it doubles as a healthcheck.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+
+from .tiers import tier_3, tier_4
+
+
+def _camoufox() -> tuple[bool, str]:
+    if os.getenv("SCRAPER_DISABLE_CAMOUFOX"):
+        return False, "off (SCRAPER_DISABLE_CAMOUFOX set)"
+    try:
+        import camoufox  # noqa: F401
+    except ImportError:
+        return False, 'not installed — pip install "switchback[camoufox]" && camoufox fetch'
+    return True, "camoufox installed"
+
+
+def probe() -> list[tuple[str, bool, str]]:
+    """(label, ok, detail) for each tier/dependency that matters at runtime."""
+    cs_ok, cs_detail = tier_3.available()
+    br_ok, br_detail = tier_4.available()
+    node = shutil.which("node")
+    return [
+        ("tier_3 (cloudscraper)", cs_ok, cs_detail),
+        ("tier_4 (browser)", br_ok, br_detail),
+        ("tier_5 (camoufox)", *_camoufox()),
+        ("node (tier_3 v3 concurrency)", bool(node),
+         node or "not on PATH — tier_3 falls back to slower, thread-fragile js2py"),
+        ("tier_7 (firecrawl)", bool(os.getenv("FIRECRAWL_API_KEY")),
+         "FIRECRAWL_API_KEY set" if os.getenv("FIRECRAWL_API_KEY")
+         else "off (no FIRECRAWL_API_KEY)"),
+    ]
+
+
+def report() -> int:
+    rows = probe()
+    print("switchback doctor — tier readiness\n")
+    for label, ok, detail in rows:
+        mark = "OK  " if ok else "MISS"
+        print(f"  [{mark}] {label:30} {detail}")
+    cs_ok = rows[0][1]
+    br_ok = rows[1][1]
+    if cs_ok and br_ok:
+        print("\nCapable tiers ready.")
+        return 0
+    print("\nOne or more capable tiers are unavailable (see above). On a cold "
+          "start this may resolve once the async install thread finishes.")
+    return 1

{switchback-0.2.0 → switchback-0.5.0}/switchback/flags.py

@@ -10,7 +10,7 @@ Run it from cron / the /loop skill / any scheduler:
     python -m switchback.flags --json          # machine-readable digest
 
 What it flags:
-  • domains still landing on paid Firecrawl (winning_tier == tier4_firecrawl)
+  • domains still landing on paid Firecrawl (winning_tier == tier_7)
   • domains escalated to residential egress (needs_egress)
   • domains throwing the most bot-wall challenges (by vendor)
   • low coverage / negative cost savings in the window
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 
 # A domain is "stuck" if its winning tier is the paid one — these are the hosts
 # that still cost money and are the prime targets for a new tier / cookie / rule.
-_PAID_TIER = "tier4_firecrawl"
+_PAID_TIER = "tier_7"
 
 
 def build_digest(minutes: int | None = None) -> dict: