switchback 0.4.0__tar.gz → 0.5.0__tar.gz

{switchback-0.4.0 → switchback-0.5.0}/.env.example

@@ -11,7 +11,7 @@
 OTEL_SERVICE_NAME=switchback
 OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 
-# ── Search (Tier-0 SearXNG, query → URLs) ───────────────────────────────────
+# ── Search (SearXNG, query → URLs — separate from the fetch cascade) ─────────
 SEARXNG_URL=http://localhost:8888
 
 # ── Output format ───────────────────────────────────────────────────────────
@@ -27,29 +27,38 @@ SEARXNG_URL=http://localhost:8888
 # fall back to their text for those sources.
 SCRAPER_OUTPUT_FORMAT=markdown
 
-# ── Tier 2 · Cloudflare solver (cloudscraper) ───────────────────────────────
+# ── tier_3 · Cloudflare solver (cloudscraper) ───────────────────────────────
 # Needs the 3.x Enhanced Edition fork (see README); with the frozen PyPI build
 # the tier reports `unavailable` and fails fast. Wall-clock cap on a single solve
 # so an unsolvable challenge can't eat the per-URL deadline before the browser
-# tier runs. Lower (e.g. 12) if Tier 2 rarely wins on your hosts.
-SCRAPER_CLOUDSCRAPER_TIMEOUT_S=25
+# tier runs. Lower (e.g. 12) if tier_3 rarely wins on your hosts.
+SCRAPER_TIER_3_TIMEOUT_S=25
 
-# ── Tier 2.5 · Jina Reader (r.jina.ai) ──────────────────────────────────────
-# Optional: keyless works at 20 RPM. A key gives 500 RPM + a 10M-token grant.
-JINA_API_KEY=
-SCRAPER_JINA_TIMEOUT_S=20
-
-# ── Tier 3b · Camoufox (Firefox stealth) ────────────────────────────────────
+# ── tier_5 · Camoufox (Firefox stealth) ─────────────────────────────────────
 # ON by default. Needs: pip install camoufox && camoufox fetch
 # Set to 1 to turn the tier off entirely.
 SCRAPER_DISABLE_CAMOUFOX=
-SCRAPER_CAMOUFOX_TIMEOUT_MS=45000
+SCRAPER_TIER_5_TIMEOUT_S=45
 
-# ── Tier 4 · Firecrawl (paid, last resort) ──────────────────────────────────
+# ── tier_7 · Firecrawl (paid, last resort) ──────────────────────────────────
 # Required only if this tier runs. Set SCRAPER_DISABLE_FIRECRAWL=1 to skip it.
 FIRECRAWL_API_KEY=
 SCRAPER_DISABLE_FIRECRAWL=
 
+# ── Per-tier timeouts (seconds) ─────────────────────────────────────────────
+# Each tier's wall-clock/socket cap; override any of them. Defaults shown below.
+# tier_3 (=25) and tier_5 (=45) are set live in their sections above; the rest
+# fall back to these defaults. The pre-0.5.0 names are still honored when the new
+# var is unset: SCRAPER_CLOUDSCRAPER_TIMEOUT_S → tier_3,
+# SCRAPER_CAMOUFOX_TIMEOUT_MS → tier_5, SCRAPER_RESIDENTIAL_TIMEOUT_MS → tier_6.
+#SCRAPER_TIER_1_TIMEOUT_S=15    # direct APIs / open mirrors
+#SCRAPER_TIER_2_TIMEOUT_S=15    # plain HTTP + TLS impersonation
+#SCRAPER_TIER_3_TIMEOUT_S=25    # cloudscraper (Cloudflare solver)
+#SCRAPER_TIER_4_TIMEOUT_S=15    # stealth headless browser (patchright)
+#SCRAPER_TIER_5_TIMEOUT_S=45    # camoufox (slowest rung; hard CF solves ~40s)
+#SCRAPER_TIER_6_TIMEOUT_S=30    # residential-IP CDP browser
+#SCRAPER_TIER_7_TIMEOUT_S=15    # Firecrawl (paid) — was unbounded; raise if scrapes get cut off
+
 # ── Orchestrator ────────────────────────────────────────────────────────────
 # Per-URL wall-clock budget (s), checked between tiers. 45s balances latency vs
 # coverage — roughly fits a Camoufox solve (~40s) that starts after the cheaper

{switchback-0.4.0 → switchback-0.5.0}/CHANGELOG.md

@@ -6,6 +6,31 @@ versioning while pre-1.0.
 
 ## [Unreleased]
 
+## [0.5.0] - 2026-06-30
+
+### Changed
+- **Tiers renamed to plain `tier_1`…`tier_7`** (cost-ordered, contiguous) in place
+  of the old mixed scheme (`tier0_apis`, `tier1_http`, `tier2_cloudscraper`,
+  `tier3_browser`, `tier3b_camoufox`, `tier_residential`, `tier4_firecrawl`). The
+  mapping is positional: `tier_1`=apis, `tier_2`=http, `tier_3`=cloudscraper,
+  `tier_4`=browser, `tier_5`=camoufox, `tier_6`=residential, `tier_7`=firecrawl.
+  **Backwards-compatible:** an existing `state/botwall_db.json` is migrated on load
+  (a host's learned `winning_tier` / `tier_stats` keys are remapped to the new
+  names), so routing survives the upgrade instead of re-probing from scratch.
+
+### Added
+- **Per-tier timeout knobs** — every tier now reads `SCRAPER_TIER_<N>_TIMEOUT_S`
+  (seconds, `N` = 1–7). Defaults: `15` for tiers without a prior budget
+  (apis/http/browser), and the existing budgets are preserved — `tier_3`=25,
+  `tier_5`=45, `tier_6`=30. The previously-unconfigurable/unbounded tiers (apis,
+  http, browser, **firecrawl**) are now bounded and overridable. The pre-rename
+  `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` / `SCRAPER_CAMOUFOX_TIMEOUT_MS` /
+  `SCRAPER_RESIDENTIAL_TIMEOUT_MS` are still honored when the new var is unset.
+  Note: `tier_7` (paid Firecrawl) was previously unbounded; its new `15`s default
+  bounds it — raise `SCRAPER_TIER_7_TIMEOUT_S` if hard hosts get cut off (a scrape
+  killed at the cap may still be billed). `SCRAPER_TIER_RETRIES_<TIER>` overrides
+  follow the new names (e.g. `SCRAPER_TIER_RETRIES_TIER_4`).
+
 ## [0.4.0] - 2026-06-29
 
 ### Added

{switchback-0.4.0 → switchback-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: switchback
-Version: 0.4.0
+Version: 0.5.0
 Summary: One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool.
 Author-email: Akash Kodavuru <akash@theaklabs.com>
 License: MIT
@@ -121,13 +121,13 @@ That's the whole loop. Add tiers as you need them (see [Install](#install)).
 
 | Tier | Strategy | Cost |
 |---|---|---|
-| 0 | Direct APIs / mirrors (arxiv, wikipedia, EuropePMC; extend: job boards) | free, cleanest |
-| 1 | Plain HTTP + TLS impersonation (`curl_cffi`), incl. PDFs | cheap |
-| 2 | Cloudflare / anti-bot solver (`cloudscraper`, install `.[cloudflare]`) | cheap-ish (~5s/host) |
-| 3 | Stealth headless browser (`patchright`, Chromium) | heavy |
-| 3b | Camoufox (Firefox stealth) — **on by default** (opt out: `SCRAPER_DISABLE_CAMOUFOX`) | heavy + slow (~40s on hard CF) |
-| 3c | Residential-IP browser over CDP (`BU_CDP_URL`) — off unless configured | heavy (remote egress) |
-| 4 | Firecrawl (paid, env-gated, audited) | paid, last resort |
+| tier_1 | Direct APIs / mirrors (arxiv, wikipedia, EuropePMC; extend: job boards) | free, cleanest |
+| tier_2 | Plain HTTP + TLS impersonation (`curl_cffi`), incl. PDFs | cheap |
+| tier_3 | Cloudflare / anti-bot solver (`cloudscraper`, install `.[cloudflare]`) | cheap-ish (~5s/host) |
+| tier_4 | Stealth headless browser (`patchright`, Chromium) | heavy |
+| tier_5 | Camoufox (Firefox stealth) — **on by default** (opt out: `SCRAPER_DISABLE_CAMOUFOX`) | heavy + slow (~40s on hard CF) |
+| tier_6 | Residential-IP browser over CDP (`BU_CDP_URL`) — off unless configured | heavy (remote egress) |
+| tier_7 | Firecrawl (paid, env-gated, audited) | paid, last resort |
 
 Every URL has a wall-clock budget (`SCRAPER_DEADLINE_S`, default 45s) checked between
 tiers so one URL can't run the whole cascade of timeouts. Each tier attempt records
@@ -142,17 +142,17 @@ Search (query → URLs) is separate from the scrape cascade: `switchback.search(
 ## Install
 
 ```bash
-pip install switchback                 # core: normalization + cheap tiers (0/1) + search
-pip install "switchback[cloudflare]"   # + Tier 2 Cloudflare/anti-bot solver (cloudscraper)
+pip install switchback                 # core: normalization + cheap tiers (tier_1/tier_2) + search
+pip install "switchback[cloudflare]"   # + tier_3 Cloudflare/anti-bot solver (cloudscraper)
 pip install "switchback[server]"       # + HTTP service (fastapi, uvicorn) incl. /metrics + /traces
-pip install "switchback[browser]" && patchright install chromium   # + Tier 3 stealth Chromium
-pip install "switchback[camoufox]" && camoufox fetch               # + Tier 3b Firefox stealth
-pip install "switchback[firecrawl]"    # + Tier 4 paid API (needs FIRECRAWL_API_KEY)
+pip install "switchback[browser]" && patchright install chromium   # + tier_4 stealth Chromium
+pip install "switchback[camoufox]" && camoufox fetch               # + tier_5 Firefox stealth
+pip install "switchback[firecrawl]"    # + tier_7 paid API (needs FIRECRAWL_API_KEY)
 pip install "switchback[tracing]"      # + OpenTelemetry -> any OTLP backend
 pip install "switchback[all]"          # everything
 ```
 
-For Tier 2's **full** v3 JS-VM + Turnstile + stealth, install the Enhanced Edition
+For tier_3's **full** v3 JS-VM + Turnstile + stealth, install the Enhanced Edition
 3.x fork (PyPI's `cloudscraper` is the older v1/v2 — PyPI forbids pinning a
 git-URL dep inside a published package, so install it alongside):
 
@@ -170,19 +170,20 @@ and land *after* boot (e.g. an async install thread on Azure). Until they're
 ready, those tiers report **`unavailable`** (a distinct outcome carrying the exact
 fix) and the cascade falls through — they are never silently skipped. Checklist:
 
-- **Tier 3 is the real workhorse for Cloudflare/JS sites** — make sure its browser
+- **tier_4 is the real workhorse for Cloudflare/JS sites** — make sure its browser
   is installed: `patchright install chromium` (note: **patchright**, not vanilla
   `playwright`). On a cold start, run this in your post-boot install step/thread;
-  Tier 3 flips to ready once it finishes.
-- **Tier 2 needs the cloudscraper 3.x fork** (above) to attempt stealth. With the
+  tier_4 flips to ready once it finishes.
+- **tier_3 needs the cloudscraper 3.x fork** (above) to attempt stealth. With the
   frozen PyPI `cloudscraper` it reports `unavailable` and fails fast (no wasted
-  solve budget) instead of erroring mid-cascade. Tier 2 is a *weak* solver for
+  solve budget) instead of erroring mid-cascade. tier_3 is a *weak* solver for
   modern Cloudflare — treat it as a cheap try before the browser, not the primary.
-- **Install Node.js** for Tier 2's v3 JS-VM challenges — faster and thread-safe
+- **Install Node.js** for tier_3's v3 JS-VM challenges — faster and thread-safe
   vs. the pure-Python js2py fallback (relevant under concurrent load).
-- **Bound Tier 2's solve budget** with `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` (default
-  `25`) so an unsolvable challenge can't eat the per-URL deadline before the
-  browser tier runs. Lower it (e.g. `12`) if Tier 2 rarely wins on your hosts.
+- **Bound tier_3's solve budget** with `SCRAPER_TIER_3_TIMEOUT_S` (default `25`;
+  the old `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` is still honored) so an unsolvable
+  challenge can't eat the per-URL deadline before the browser tier runs. Lower it
+  (e.g. `12`) if tier_3 rarely wins on your hosts.
 
 **Verify readiness on the box** with the preflight check (doubles as a healthcheck
 — exit 0 when the capable tiers are ready):
@@ -282,16 +283,16 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 <details>
 <summary><b>Env gates</b> — enable/disable tiers and integrations</summary>
 
-- `SCRAPER_DISABLE_FIRECRAWL` — skip Tier 4
-- `FIRECRAWL_API_KEY` — enable Tier 4
-- `SCRAPER_DISABLE_CAMOUFOX` — turn off Tier 3b (on by default; needs `pip install camoufox` + `camoufox fetch`)
-- `BU_CDP_URL` — enable Tier 3c residential browser by pointing at a CDP endpoint
+- `SCRAPER_DISABLE_FIRECRAWL` — skip tier_7
+- `FIRECRAWL_API_KEY` — enable tier_7
+- `SCRAPER_DISABLE_CAMOUFOX` — turn off tier_5 (on by default; needs `pip install camoufox` + `camoufox fetch`)
+- `BU_CDP_URL` — enable tier_6 residential browser by pointing at a CDP endpoint
 - `SCRAPER_PROXY` — route *all* tiers/URLs through a proxy
 - `SCRAPER_EGRESS_PROXY` — route only walled hosts through a proxy (see [Cost-scoped residential egress](#cost-scoped-residential-egress))
 - `SEARXNG_URL` — defaults to `http://localhost:8888`
 - `SCRAPER_STATE_DIR` — where the botwall DB/event log + session cache live
 - `SCRAPER_COOKIES_FILE` — Netscape `cookies.txt` to scrape login-gated hosts (injected into the HTTP and browser tiers)
-- `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into Tier 2 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.
+- `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into tier_3 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.
 </details>
 
 <details>
@@ -300,7 +301,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 - `SCRAPER_OUTPUT_FORMAT` — output shape: `markdown` (default) · `markdown_trimmed` · `html` · `html_selectors` (see [Output formats](#output-formats))
 - `SCRAPER_DEADLINE_S` — per-URL budget (45s)
 - `SCRAPER_FIRECRAWL_FALLBACK_AFTER_S` — after this many seconds on a URL, stop trying the local tiers and fall back to Firecrawl, so a hard host doesn't burn the whole deadline before the paid last resort gets a turn (25s; 0 = off)
-- `SCRAPER_CAMOUFOX_TIMEOUT_MS` — (45000)
+- `SCRAPER_TIER_<N>_TIMEOUT_S` — per-tier timeout in seconds, `N` = 1–7 (tier_1 apis · tier_2 http · tier_3 cloudscraper · tier_4 browser · tier_5 camoufox · tier_6 residential · tier_7 firecrawl). Defaults: 15/15/**25**/15/**45**/**30**/15 (the three bold ones keep their prior budgets; everything else is 15s). The pre-0.5.0 `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` / `SCRAPER_CAMOUFOX_TIMEOUT_MS` / `SCRAPER_RESIDENTIAL_TIMEOUT_MS` are still honored when the new var is unset. Note: `tier_7` (Firecrawl) was previously unbounded — its 15s default now bounds the paid tier, so raise `SCRAPER_TIER_7_TIMEOUT_S` if slow hosts get cut off
 - `SCRAPER_BROWSER_CONCURRENCY` — max simultaneous headless browsers (default 1)
 - `SCRAPER_BOTWALL_URL_SKIP_COOLDOWN_H` — auto-skip re-test window (24h; 0 = never)
 - `SCRAPER_BOTWALL_EGRESS_AFTER` — local-tier failures before a host escalates to the residential tier (default 2)
@@ -308,7 +309,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 - `SCRAPER_DISABLE_SESSION_CACHE` — turn off cf_clearance reuse
 - `SCRAPER_CONTENT_TTL_S` — URL→result cache TTL (**0 = off**; set e.g. 86400 to skip re-scraping a page within a day)
 - `SCRAPER_BACKOFF_BASE_MS` / `SCRAPER_BACKOFF_MAX_MS` — exponential backoff between tiers after a rate-limit/timeout (base 0 = off)
-- `SCRAPER_TIER_RETRIES` — same-tier retries before falling through (default 0 = off; `N` → up to `1+N` tries per tier), with per-tier overrides `SCRAPER_TIER_RETRIES_<TIER>` (e.g. `SCRAPER_TIER_RETRIES_TIER3_BROWSER=2`)
+- `SCRAPER_TIER_RETRIES` — same-tier retries before falling through (default 0 = off; `N` → up to `1+N` tries per tier), with per-tier overrides `SCRAPER_TIER_RETRIES_<TIER>` (e.g. `SCRAPER_TIER_RETRIES_TIER_4=2`)
 - `SCRAPER_TIER_RETRY_ON` — failure classes eligible for a same-tier retry (default `timeout,rate_limited,connection`; widen to include `botwall,http_block` behind a rotating residential proxy so each retry gets a fresh IP). Retries are bounded by `SCRAPER_DEADLINE_S`; enabling them on the paid Firecrawl tier bills per attempt
 - `SCRAPER_LOGIN_HOOK` — `pkg.module:func` returning `{cookie: value}` for a host (see [Logged-in sessions](#logged-in-sessions))
 - `SCRAPER_EXTRACTION_FILE` — per-domain extraction prefs JSON (default `config/extraction.json`)

{switchback-0.4.0 → switchback-0.5.0}/README.md

@@ -62,13 +62,13 @@ That's the whole loop. Add tiers as you need them (see [Install](#install)).
 
 | Tier | Strategy | Cost |
 |---|---|---|
-| 0 | Direct APIs / mirrors (arxiv, wikipedia, EuropePMC; extend: job boards) | free, cleanest |
-| 1 | Plain HTTP + TLS impersonation (`curl_cffi`), incl. PDFs | cheap |
-| 2 | Cloudflare / anti-bot solver (`cloudscraper`, install `.[cloudflare]`) | cheap-ish (~5s/host) |
-| 3 | Stealth headless browser (`patchright`, Chromium) | heavy |
-| 3b | Camoufox (Firefox stealth) — **on by default** (opt out: `SCRAPER_DISABLE_CAMOUFOX`) | heavy + slow (~40s on hard CF) |
-| 3c | Residential-IP browser over CDP (`BU_CDP_URL`) — off unless configured | heavy (remote egress) |
-| 4 | Firecrawl (paid, env-gated, audited) | paid, last resort |
+| tier_1 | Direct APIs / mirrors (arxiv, wikipedia, EuropePMC; extend: job boards) | free, cleanest |
+| tier_2 | Plain HTTP + TLS impersonation (`curl_cffi`), incl. PDFs | cheap |
+| tier_3 | Cloudflare / anti-bot solver (`cloudscraper`, install `.[cloudflare]`) | cheap-ish (~5s/host) |
+| tier_4 | Stealth headless browser (`patchright`, Chromium) | heavy |
+| tier_5 | Camoufox (Firefox stealth) — **on by default** (opt out: `SCRAPER_DISABLE_CAMOUFOX`) | heavy + slow (~40s on hard CF) |
+| tier_6 | Residential-IP browser over CDP (`BU_CDP_URL`) — off unless configured | heavy (remote egress) |
+| tier_7 | Firecrawl (paid, env-gated, audited) | paid, last resort |
 
 Every URL has a wall-clock budget (`SCRAPER_DEADLINE_S`, default 45s) checked between
 tiers so one URL can't run the whole cascade of timeouts. Each tier attempt records
@@ -83,17 +83,17 @@ Search (query → URLs) is separate from the scrape cascade: `switchback.search(
 ## Install
 
 ```bash
-pip install switchback                 # core: normalization + cheap tiers (0/1) + search
-pip install "switchback[cloudflare]"   # + Tier 2 Cloudflare/anti-bot solver (cloudscraper)
+pip install switchback                 # core: normalization + cheap tiers (tier_1/tier_2) + search
+pip install "switchback[cloudflare]"   # + tier_3 Cloudflare/anti-bot solver (cloudscraper)
 pip install "switchback[server]"       # + HTTP service (fastapi, uvicorn) incl. /metrics + /traces
-pip install "switchback[browser]" && patchright install chromium   # + Tier 3 stealth Chromium
-pip install "switchback[camoufox]" && camoufox fetch               # + Tier 3b Firefox stealth
-pip install "switchback[firecrawl]"    # + Tier 4 paid API (needs FIRECRAWL_API_KEY)
+pip install "switchback[browser]" && patchright install chromium   # + tier_4 stealth Chromium
+pip install "switchback[camoufox]" && camoufox fetch               # + tier_5 Firefox stealth
+pip install "switchback[firecrawl]"    # + tier_7 paid API (needs FIRECRAWL_API_KEY)
 pip install "switchback[tracing]"      # + OpenTelemetry -> any OTLP backend
 pip install "switchback[all]"          # everything
 ```
 
-For Tier 2's **full** v3 JS-VM + Turnstile + stealth, install the Enhanced Edition
+For tier_3's **full** v3 JS-VM + Turnstile + stealth, install the Enhanced Edition
 3.x fork (PyPI's `cloudscraper` is the older v1/v2 — PyPI forbids pinning a
 git-URL dep inside a published package, so install it alongside):
 
@@ -111,19 +111,20 @@ and land *after* boot (e.g. an async install thread on Azure). Until they're
 ready, those tiers report **`unavailable`** (a distinct outcome carrying the exact
 fix) and the cascade falls through — they are never silently skipped. Checklist:
 
-- **Tier 3 is the real workhorse for Cloudflare/JS sites** — make sure its browser
+- **tier_4 is the real workhorse for Cloudflare/JS sites** — make sure its browser
   is installed: `patchright install chromium` (note: **patchright**, not vanilla
   `playwright`). On a cold start, run this in your post-boot install step/thread;
-  Tier 3 flips to ready once it finishes.
-- **Tier 2 needs the cloudscraper 3.x fork** (above) to attempt stealth. With the
+  tier_4 flips to ready once it finishes.
+- **tier_3 needs the cloudscraper 3.x fork** (above) to attempt stealth. With the
   frozen PyPI `cloudscraper` it reports `unavailable` and fails fast (no wasted
-  solve budget) instead of erroring mid-cascade. Tier 2 is a *weak* solver for
+  solve budget) instead of erroring mid-cascade. tier_3 is a *weak* solver for
   modern Cloudflare — treat it as a cheap try before the browser, not the primary.
-- **Install Node.js** for Tier 2's v3 JS-VM challenges — faster and thread-safe
+- **Install Node.js** for tier_3's v3 JS-VM challenges — faster and thread-safe
   vs. the pure-Python js2py fallback (relevant under concurrent load).
-- **Bound Tier 2's solve budget** with `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` (default
-  `25`) so an unsolvable challenge can't eat the per-URL deadline before the
-  browser tier runs. Lower it (e.g. `12`) if Tier 2 rarely wins on your hosts.
+- **Bound tier_3's solve budget** with `SCRAPER_TIER_3_TIMEOUT_S` (default `25`;
+  the old `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` is still honored) so an unsolvable
+  challenge can't eat the per-URL deadline before the browser tier runs. Lower it
+  (e.g. `12`) if tier_3 rarely wins on your hosts.
 
 **Verify readiness on the box** with the preflight check (doubles as a healthcheck
 — exit 0 when the capable tiers are ready):
@@ -223,16 +224,16 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 <details>
 <summary><b>Env gates</b> — enable/disable tiers and integrations</summary>
 
-- `SCRAPER_DISABLE_FIRECRAWL` — skip Tier 4
-- `FIRECRAWL_API_KEY` — enable Tier 4
-- `SCRAPER_DISABLE_CAMOUFOX` — turn off Tier 3b (on by default; needs `pip install camoufox` + `camoufox fetch`)
-- `BU_CDP_URL` — enable Tier 3c residential browser by pointing at a CDP endpoint
+- `SCRAPER_DISABLE_FIRECRAWL` — skip tier_7
+- `FIRECRAWL_API_KEY` — enable tier_7
+- `SCRAPER_DISABLE_CAMOUFOX` — turn off tier_5 (on by default; needs `pip install camoufox` + `camoufox fetch`)
+- `BU_CDP_URL` — enable tier_6 residential browser by pointing at a CDP endpoint
 - `SCRAPER_PROXY` — route *all* tiers/URLs through a proxy
 - `SCRAPER_EGRESS_PROXY` — route only walled hosts through a proxy (see [Cost-scoped residential egress](#cost-scoped-residential-egress))
 - `SEARXNG_URL` — defaults to `http://localhost:8888`
 - `SCRAPER_STATE_DIR` — where the botwall DB/event log + session cache live
 - `SCRAPER_COOKIES_FILE` — Netscape `cookies.txt` to scrape login-gated hosts (injected into the HTTP and browser tiers)
-- `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into Tier 2 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.
+- `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into tier_3 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.
 </details>
 
 <details>
@@ -241,7 +242,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 - `SCRAPER_OUTPUT_FORMAT` — output shape: `markdown` (default) · `markdown_trimmed` · `html` · `html_selectors` (see [Output formats](#output-formats))
 - `SCRAPER_DEADLINE_S` — per-URL budget (45s)
 - `SCRAPER_FIRECRAWL_FALLBACK_AFTER_S` — after this many seconds on a URL, stop trying the local tiers and fall back to Firecrawl, so a hard host doesn't burn the whole deadline before the paid last resort gets a turn (25s; 0 = off)
-- `SCRAPER_CAMOUFOX_TIMEOUT_MS` — (45000)
+- `SCRAPER_TIER_<N>_TIMEOUT_S` — per-tier timeout in seconds, `N` = 1–7 (tier_1 apis · tier_2 http · tier_3 cloudscraper · tier_4 browser · tier_5 camoufox · tier_6 residential · tier_7 firecrawl). Defaults: 15/15/**25**/15/**45**/**30**/15 (the three bold ones keep their prior budgets; everything else is 15s). The pre-0.5.0 `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` / `SCRAPER_CAMOUFOX_TIMEOUT_MS` / `SCRAPER_RESIDENTIAL_TIMEOUT_MS` are still honored when the new var is unset. Note: `tier_7` (Firecrawl) was previously unbounded — its 15s default now bounds the paid tier, so raise `SCRAPER_TIER_7_TIMEOUT_S` if slow hosts get cut off
 - `SCRAPER_BROWSER_CONCURRENCY` — max simultaneous headless browsers (default 1)
 - `SCRAPER_BOTWALL_URL_SKIP_COOLDOWN_H` — auto-skip re-test window (24h; 0 = never)
 - `SCRAPER_BOTWALL_EGRESS_AFTER` — local-tier failures before a host escalates to the residential tier (default 2)
@@ -249,7 +250,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 - `SCRAPER_DISABLE_SESSION_CACHE` — turn off cf_clearance reuse
 - `SCRAPER_CONTENT_TTL_S` — URL→result cache TTL (**0 = off**; set e.g. 86400 to skip re-scraping a page within a day)
 - `SCRAPER_BACKOFF_BASE_MS` / `SCRAPER_BACKOFF_MAX_MS` — exponential backoff between tiers after a rate-limit/timeout (base 0 = off)
-- `SCRAPER_TIER_RETRIES` — same-tier retries before falling through (default 0 = off; `N` → up to `1+N` tries per tier), with per-tier overrides `SCRAPER_TIER_RETRIES_<TIER>` (e.g. `SCRAPER_TIER_RETRIES_TIER3_BROWSER=2`)
+- `SCRAPER_TIER_RETRIES` — same-tier retries before falling through (default 0 = off; `N` → up to `1+N` tries per tier), with per-tier overrides `SCRAPER_TIER_RETRIES_<TIER>` (e.g. `SCRAPER_TIER_RETRIES_TIER_4=2`)
 - `SCRAPER_TIER_RETRY_ON` — failure classes eligible for a same-tier retry (default `timeout,rate_limited,connection`; widen to include `botwall,http_block` behind a rotating residential proxy so each retry gets a fresh IP). Retries are bounded by `SCRAPER_DEADLINE_S`; enabling them on the paid Firecrawl tier bills per attempt
 - `SCRAPER_LOGIN_HOOK` — `pkg.module:func` returning `{cookie: value}` for a host (see [Logged-in sessions](#logged-in-sessions))
 - `SCRAPER_EXTRACTION_FILE` — per-domain extraction prefs JSON (default `config/extraction.json`)

{switchback-0.4.0 → switchback-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "switchback"
-version = "0.4.0"
+version = "0.5.0"
 description = "One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool."
 readme = "README.md"
 requires-python = ">=3.10"

{switchback-0.4.0 → switchback-0.5.0}/switchback/doctor.py

@@ -13,7 +13,7 @@ from __future__ import annotations
 import os
 import shutil
 
-from .tiers import tier2_cloudscraper, tier3_browser
+from .tiers import tier_3, tier_4
 
 
 def _camoufox() -> tuple[bool, str]:
@@ -28,16 +28,16 @@ def _camoufox() -> tuple[bool, str]:
 
 def probe() -> list[tuple[str, bool, str]]:
     """(label, ok, detail) for each tier/dependency that matters at runtime."""
-    cs_ok, cs_detail = tier2_cloudscraper.available()
-    br_ok, br_detail = tier3_browser.available()
+    cs_ok, cs_detail = tier_3.available()
+    br_ok, br_detail = tier_4.available()
     node = shutil.which("node")
     return [
-        ("tier2_cloudscraper", cs_ok, cs_detail),
-        ("tier3_browser", br_ok, br_detail),
-        ("tier3b_camoufox", *_camoufox()),
-        ("node (tier2 v3 concurrency)", bool(node),
-         node or "not on PATH — Tier 2 falls back to slower, thread-fragile js2py"),
-        ("tier4_firecrawl", bool(os.getenv("FIRECRAWL_API_KEY")),
+        ("tier_3 (cloudscraper)", cs_ok, cs_detail),
+        ("tier_4 (browser)", br_ok, br_detail),
+        ("tier_5 (camoufox)", *_camoufox()),
+        ("node (tier_3 v3 concurrency)", bool(node),
+         node or "not on PATH — tier_3 falls back to slower, thread-fragile js2py"),
+        ("tier_7 (firecrawl)", bool(os.getenv("FIRECRAWL_API_KEY")),
          "FIRECRAWL_API_KEY set" if os.getenv("FIRECRAWL_API_KEY")
          else "off (no FIRECRAWL_API_KEY)"),
     ]

{switchback-0.4.0 → switchback-0.5.0}/switchback/flags.py

@@ -10,7 +10,7 @@ Run it from cron / the /loop skill / any scheduler:
     python -m switchback.flags --json          # machine-readable digest
 
 What it flags:
-  • domains still landing on paid Firecrawl (winning_tier == tier4_firecrawl)
+  • domains still landing on paid Firecrawl (winning_tier == tier_7)
   • domains escalated to residential egress (needs_egress)
   • domains throwing the most bot-wall challenges (by vendor)
   • low coverage / negative cost savings in the window
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 
 # A domain is "stuck" if its winning tier is the paid one — these are the hosts
 # that still cost money and are the prime targets for a new tier / cookie / rule.
-_PAID_TIER = "tier4_firecrawl"
+_PAID_TIER = "tier_7"
 
 
 def build_digest(minutes: int | None = None) -> dict:

{switchback-0.4.0 → switchback-0.5.0}/switchback/orchestrator.py

@@ -172,7 +172,7 @@ def _start_index(url: str, db: dict) -> int:
     if botwall.needs_egress(host, db):
         if egress.has_egress_proxy():
             return 0
-        res_i = INDEX.get("tier_residential")
+        res_i = INDEX.get("tier_6")
         if res_i is not None:
             disabled_fn = getattr(TIERS[res_i], "disabled", None)
             if not (disabled_fn and disabled_fn()):
@@ -298,7 +298,7 @@ def _run_cascade(url, host, db, root, t0, deadline) -> ScrapeOutcome:
         # solve attempt and go straight to the browser/egress tiers. A real CF
         # challenge surfaces as `botwall`, not `http_block`, so this never skips a
         # host cloudscraper could actually clear.
-        if tier.NAME == "tier2_cloudscraper" and any(
+        if tier.NAME == "tier_3" and any(
                 a.outcome == "http_block" for a in attempts):
             logger.info(f"{tier.NAME} skipped (prior hard IP block): {url}")
             attempts.append(TierAttempt(tier.NAME, "not_applicable"))

{switchback-0.4.0 → switchback-0.5.0}/switchback/policy/botwall.py

@@ -149,6 +149,40 @@ def _parse_skip_urls_file(path: str) -> dict[str, str]:
     return out
 
 
+# Tiers were renamed to plain tier_1..tier_7 in 0.5.0. A pre-rename DB persists a
+# host's learned winning_tier / tier_stats under the old names; migrate them on
+# load so routing survives the upgrade instead of re-probing the cascade fresh.
+_TIER_RENAME = {
+    "tier0_apis": "tier_1",
+    "tier1_http": "tier_2",
+    "tier2_cloudscraper": "tier_3",
+    "tier3_browser": "tier_4",
+    "tier3b_camoufox": "tier_5",
+    "tier_residential": "tier_6",
+    "tier4_firecrawl": "tier_7",
+}
+
+
+def _migrate_tier_names(hosts: dict) -> bool:
+    """Remap pre-rename tier names in persisted host records. Idempotent."""
+    changed = False
+    for rec in hosts.values():
+        if rec.get("winning_tier") in _TIER_RENAME:
+            rec["winning_tier"] = _TIER_RENAME[rec["winning_tier"]]
+            changed = True
+        stats = rec.get("tier_stats")
+        if isinstance(stats, dict):
+            for old, new in _TIER_RENAME.items():
+                if old in stats:
+                    # Merge into the new key if it somehow already exists, else move.
+                    dst = stats.setdefault(new, {"ok": 0, "miss": 0})
+                    src = stats.pop(old)
+                    dst["ok"] += src.get("ok", 0)
+                    dst["miss"] += src.get("miss", 0)
+                    changed = True
+    return changed
+
+
 def load_db() -> dict:
     db = {"version": 2, "updated_at": "", "hosts": {}, "urls": {}}
     if os.path.exists(DB_PATH):
@@ -159,7 +193,7 @@ def load_db() -> dict:
             logger.error(f"botwall: load failed ({e}); starting fresh")
     hosts = db.setdefault("hosts", {})
     urls = db.setdefault("urls", {})
-    changed = False
+    changed = _migrate_tier_names(hosts)
     for host, reason in SEED_HOSTS.items():
         if host not in hosts:
             hosts[host] = _new_record(reason=reason, status="skip")
@@ -305,7 +339,7 @@ def _track_egress(host: str, tier: str, outcome: str, db: dict) -> None:
     unescalated. We don't count the residential tier's own misses (circular)."""
     if not PROMOTE_EGRESS_AFTER or outcome not in _EGRESS_OUTCOMES:
         return
-    if tier == "tier_residential":
+    if tier == "tier_6":  # residential egress — don't count its own misses
         return
     rec = db.get("hosts", {}).get(host)
     if rec is None or rec.get("needs_egress"):

{switchback-0.4.0 → switchback-0.5.0}/switchback/reporting.py

@@ -39,7 +39,7 @@ FIRECRAWL_USD = float(os.getenv("BENCH_FIRECRAWL_USD", "0.001"))
 HARD_MULT = float(os.getenv("BENCH_FIRECRAWL_HARD_MULT", "5"))
 
 # Tiers whose win means Firecrawl would have billed the hard (stealth) rate.
-_HARD_TIERS = {"tier3_browser", "tier3b_camoufox", "tier_residential", "tier4_firecrawl"}
+_HARD_TIERS = {"tier_4", "tier_5", "tier_6", "tier_7"}
 
 
 def _parse_ts(ts: str) -> datetime | None:

switchback-0.5.0/switchback/tiers/__init__.py

@@ -0,0 +1,24 @@
+"""The cost-ordered cascade. Each tier exposes:
+
+    NAME : str
+    PAID : bool                       # gated/audited if True
+    fetch(url) -> str | None          # markdown on success; None if not
+                                      # applicable; raises on failure.
+
+Order matters — cheapest/cleanest first, paid last.
+"""
+from . import (tier_1, tier_2, tier_3, tier_4, tier_5, tier_6, tier_7)
+
+# Cost-ordered, cheapest first. Plain names; role noted inline.
+TIERS = [
+    tier_1,   # direct APIs / open mirrors
+    tier_2,   # plain HTTP with TLS impersonation
+    tier_3,   # cloudscraper (Cloudflare/anti-bot solver)
+    tier_4,   # stealth headless browser (patchright)
+    tier_5,   # camoufox (Firefox stealth; on by default, orthogonal to tier_4)
+    tier_6,   # residential-IP CDP browser (off unless BU_CDP_URL set)
+    tier_7,   # Firecrawl (paid, last resort)
+]
+
+# tier name -> index, for botwall winning-tier routing.
+INDEX = {t.NAME: i for i, t in enumerate(TIERS)}

switchback-0.4.0/switchback/tiers/tier0_apis.py → switchback-0.5.0/switchback/tiers/tier_1.py

@@ -10,6 +10,7 @@ Web *search* (query → URLs) is a different shape and lives in switchback/searc
 """
 from __future__ import annotations
 
+import os
 import re
 from urllib.parse import unquote
 from xml.etree import ElementTree as ET
@@ -17,9 +18,12 @@ from xml.etree import ElementTree as ET
 from ..normalize import html_to_markdown, UA
 from ..policy.gates import check
 
-NAME = "tier0_apis"
+NAME = "tier_1"
 PAID = False
 
+# Per-tier request timeout (seconds); override with SCRAPER_TIER_1_TIMEOUT_S.
+_TIMEOUT_S = float(os.getenv("SCRAPER_TIER_1_TIMEOUT_S", "15"))
+
 ARXIV_RE = re.compile(r"arxiv\.org/(?:abs|pdf)/(\d{4}\.\d{4,5})(?:v\d+)?(?:\.pdf)?", re.I)
 WIKI_RE = re.compile(r"en\.wikipedia\.org/wiki/([^?#]+)", re.I)
 PMC_RE = re.compile(r"pmc\.ncbi\.nlm\.nih\.gov/articles/(PMC\d+)", re.I)
@@ -42,7 +46,7 @@ def _arxiv(arxiv_id: str, url: str) -> str:
     # impersonating Chrome triggers aggressive 429s from their Akamai front-end.
     import requests
     r = requests.get(f"https://export.arxiv.org/api/query?id_list={arxiv_id}",
-                     timeout=15,
+                     timeout=_TIMEOUT_S,
                      headers={"User-Agent": "switchback/1.0 (mailto:akash@theaklabs.com)"})
     r.raise_for_status()
     ns = {"atom": "http://www.w3.org/2005/Atom"}
@@ -60,7 +64,7 @@ def _arxiv(arxiv_id: str, url: str) -> str:
 def _wikipedia(title: str, url: str) -> str:
     from curl_cffi import requests as cffi
     r = cffi.get(f"https://en.wikipedia.org/api/rest_v1/page/html/{unquote(title)}",
-                 timeout=15, impersonate="chrome")
+                 timeout=_TIMEOUT_S, impersonate="chrome")
     r.raise_for_status()
     return check(url, html_to_markdown(r.text, base_url=url))
 
@@ -70,7 +74,7 @@ def _europepmc(url: str) -> str:
     import requests
     pmcid = PMC_RE.search(url).group(1)
     api = f"https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/fullTextXML"
-    r = requests.get(api, timeout=20, headers={"User-Agent": UA})
+    r = requests.get(api, timeout=_TIMEOUT_S, headers={"User-Agent": UA})
     r.raise_for_status()
     if len(r.text) < 1000:
         raise RuntimeError(f"europepmc empty: {len(r.text)}")

switchback-0.4.0/switchback/tiers/tier1_http.py → switchback-0.5.0/switchback/tiers/tier_2.py

@@ -15,6 +15,7 @@ says one Chrome version, the header says another).
 from __future__ import annotations
 
 import hashlib
+import os
 from urllib.parse import urlsplit
 
 from .. import session_cache
@@ -22,9 +23,12 @@ from ..egress import requests_proxies, add_wire_bytes
 from ..normalize import html_to_markdown, pdf_bytes_to_text
 from ..policy.gates import BotWall, check, is_cf_challenge
 
-NAME = "tier1_http"
+NAME = "tier_2"
 PAID = False
 
+# Per-tier request timeout (seconds); override with SCRAPER_TIER_2_TIMEOUT_S.
+_TIMEOUT_S = float(os.getenv("SCRAPER_TIER_2_TIMEOUT_S", "15"))
+
 # Recent Chrome JA3 targets available in curl_cffi 0.15.x. A small spread of real
 # versions mirrors how live traffic is distributed across Chrome releases.
 _IMPERSONATE_TARGETS = ("chrome131", "chrome136", "chrome142")
@@ -43,7 +47,7 @@ def fetch(url: str) -> str:
     # against Tier 1's distinct impersonate UA would be a mismatch tell.
     cookie = session_cache.cookie_header(url, include_cache=False)
     headers = {"Cookie": cookie} if cookie else None
-    r = cffi.get(url, timeout=15, allow_redirects=True,
+    r = cffi.get(url, timeout=_TIMEOUT_S, allow_redirects=True,
                  impersonate=_impersonate_for(url),
                  proxies=requests_proxies(), headers=headers)
     add_wire_bytes(len(r.content))  # count even on a block — failed fetches burn bandwidth too

switchback-0.4.0/switchback/tiers/tier2_cloudscraper.py → switchback-0.5.0/switchback/tiers/tier_3.py

@@ -26,7 +26,7 @@ from ..policy.gates import Unavailable, check
 
 logger = logging.getLogger(__name__)
 
-NAME = "tier2_cloudscraper"
+NAME = "tier_3"
 PAID = False
 
 # Install hint surfaced when cloudscraper is missing or the frozen PyPI 1.2.71
@@ -61,7 +61,10 @@ def available() -> tuple[bool, str]:
 # per-request socket timeout. Capping it here lets the cascade fall through to the
 # stealth browser (which can handle interactive challenges) instead of burning the
 # per-URL deadline. ~25s comfortably covers a real JS/v3 solve (~5-15s).
-_TIMEOUT_S = float(os.getenv("SCRAPER_CLOUDSCRAPER_TIMEOUT_S", "25"))
+# Back-compat: honor the pre-0.5.0 SCRAPER_CLOUDSCRAPER_TIMEOUT_S if the new var
+# is unset, so a tuned prod value survives the tier rename.
+_TIMEOUT_S = float(os.getenv("SCRAPER_TIER_3_TIMEOUT_S",
+                             os.getenv("SCRAPER_CLOUDSCRAPER_TIMEOUT_S", "25")))
 
 # Stealth pacing. Kept modest: Tier 2 only fires on CF-suspected hosts, and the
 # real latency win comes from skipping the solve entirely on repeat hits (session
@@ -153,7 +156,7 @@ def fetch(url: str) -> str:
             except BaseException as e:  # noqa: BLE001 — propagated to caller below
                 box["err"] = e
 
-    t = threading.Thread(target=work, name="tier2-cloudscraper", daemon=True)
+    t = threading.Thread(target=work, name="tier_3-cloudscraper", daemon=True)
     t.start()
     t.join(_TIMEOUT_S)
     if t.is_alive():

switchback-0.4.0/switchback/tiers/tier3_browser.py → switchback-0.5.0/switchback/tiers/tier_4.py

@@ -19,9 +19,12 @@ from ..egress import playwright_proxy, add_wire_bytes
 from ..normalize import html_to_markdown
 from ..policy.gates import Unavailable, check
 
-NAME = "tier3_browser"
+NAME = "tier_4"
 PAID = False
 
+# Per-tier navigation timeout (seconds); override with SCRAPER_TIER_4_TIMEOUT_S.
+_TIMEOUT_S = float(os.getenv("SCRAPER_TIER_4_TIMEOUT_S", "15"))
+
 # Install hint surfaced when patchright or its Chromium isn't ready — notably
 # during an async cold-start install (the browser binary lands after boot).
 _INSTALL_HINT = 'pip install "switchback[browser]" && patchright install chromium'
@@ -47,7 +50,7 @@ def available() -> tuple[bool, str]:
     return True, "patchright + Chromium ready"
 
 
-def fetch(url: str, timeout_ms: int = 15000) -> str:
+def fetch(url: str, timeout_ms: int = int(_TIMEOUT_S * 1000)) -> str:
     try:
         from patchright.sync_api import sync_playwright
     except ImportError:

switchback-0.4.0/switchback/tiers/tier3b_camoufox.py → switchback-0.5.0/switchback/tiers/tier_5.py

@@ -25,10 +25,16 @@ from ..policy.gates import check
 
 logger = logging.getLogger(__name__)
 
-NAME = "tier3b_camoufox"
+NAME = "tier_5"
 PAID = False
 
-_TIMEOUT_MS = int(os.getenv("SCRAPER_CAMOUFOX_TIMEOUT_MS", "45000"))
+# Per-tier navigation timeout (seconds); override with SCRAPER_TIER_5_TIMEOUT_S.
+# Default kept at 45s — camoufox is the slowest rung (~40s on a hard CF solve).
+# Back-compat: honor the pre-0.5.0 SCRAPER_CAMOUFOX_TIMEOUT_MS (ms) if unset.
+_legacy_ms = os.getenv("SCRAPER_CAMOUFOX_TIMEOUT_MS")
+_TIMEOUT_S = float(os.getenv("SCRAPER_TIER_5_TIMEOUT_S",
+                             str(float(_legacy_ms) / 1000) if _legacy_ms else "45"))
+_TIMEOUT_MS = int(_TIMEOUT_S * 1000)
 
 
 def disabled() -> bool:

switchback-0.4.0/switchback/tiers/tier_residential.py → switchback-0.5.0/switchback/tiers/tier_6.py

@@ -22,10 +22,16 @@ from ..concurrency import browser_slot
 from ..normalize import html_to_markdown
 from ..policy.gates import check
 
-NAME = "tier_residential"
+NAME = "tier_6"
 PAID = False
 
-_TIMEOUT_MS = int(os.getenv("SCRAPER_RESIDENTIAL_TIMEOUT_MS", "30000"))
+# Per-tier navigation timeout (seconds); override with SCRAPER_TIER_6_TIMEOUT_S.
+# Default kept at 30s — remote CDP over a residential proxy is slow to first paint.
+# Back-compat: honor the pre-0.5.0 SCRAPER_RESIDENTIAL_TIMEOUT_MS (ms) if unset.
+_legacy_ms = os.getenv("SCRAPER_RESIDENTIAL_TIMEOUT_MS")
+_TIMEOUT_S = float(os.getenv("SCRAPER_TIER_6_TIMEOUT_S",
+                             str(float(_legacy_ms) / 1000) if _legacy_ms else "30"))
+_TIMEOUT_MS = int(_TIMEOUT_S * 1000)
 
 
 def disabled() -> bool:

switchback-0.4.0/switchback/tiers/tier4_firecrawl.py → switchback-0.5.0/switchback/tiers/tier_7.py

@@ -12,9 +12,15 @@ import threading
 from ..normalize import active_format, render
 from ..policy.gates import check
 
-NAME = "tier4_firecrawl"
+NAME = "tier_7"
 PAID = True
 
+# Per-tier wall-clock cap (seconds); override with SCRAPER_TIER_7_TIMEOUT_S.
+# This paid last resort was previously unbounded; 15s bounds it like the rest, but
+# Firecrawl scrapes can legitimately run longer — raise this if hard hosts get cut
+# off at the finish line (you may still be billed for a scrape killed here).
+_TIMEOUT_S = float(os.getenv("SCRAPER_TIER_7_TIMEOUT_S", "15"))
+
 
 def disabled() -> bool:
     return bool(os.getenv("SCRAPER_DISABLE_FIRECRAWL"))
@@ -49,9 +55,13 @@ def fetch(url: str) -> str:
         except BaseException as e:  # noqa: BLE001 — re-raised to the caller below
             box["err"] = e
 
-    t = threading.Thread(target=work, name="tier4-firecrawl", daemon=True)
+    t = threading.Thread(target=work, name="tier_7-firecrawl", daemon=True)
     t.start()
-    t.join()
+    t.join(_TIMEOUT_S)
+    if t.is_alive():
+        raise TimeoutError(
+            f"firecrawl exceeded {_TIMEOUT_S}s "
+            "(raise SCRAPER_TIER_7_TIMEOUT_S for slow hosts)")
     if "err" in box:
         raise box["err"]
     return box["md"]

{switchback-0.4.0 → switchback-0.5.0}/switchback.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: switchback
-Version: 0.4.0
+Version: 0.5.0
 Summary: One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool.
 Author-email: Akash Kodavuru <akash@theaklabs.com>
 License: MIT
@@ -121,13 +121,13 @@ That's the whole loop. Add tiers as you need them (see [Install](#install)).
 
 | Tier | Strategy | Cost |
 |---|---|---|
-| 0 | Direct APIs / mirrors (arxiv, wikipedia, EuropePMC; extend: job boards) | free, cleanest |
-| 1 | Plain HTTP + TLS impersonation (`curl_cffi`), incl. PDFs | cheap |
-| 2 | Cloudflare / anti-bot solver (`cloudscraper`, install `.[cloudflare]`) | cheap-ish (~5s/host) |
-| 3 | Stealth headless browser (`patchright`, Chromium) | heavy |
-| 3b | Camoufox (Firefox stealth) — **on by default** (opt out: `SCRAPER_DISABLE_CAMOUFOX`) | heavy + slow (~40s on hard CF) |
-| 3c | Residential-IP browser over CDP (`BU_CDP_URL`) — off unless configured | heavy (remote egress) |
-| 4 | Firecrawl (paid, env-gated, audited) | paid, last resort |
+| tier_1 | Direct APIs / mirrors (arxiv, wikipedia, EuropePMC; extend: job boards) | free, cleanest |
+| tier_2 | Plain HTTP + TLS impersonation (`curl_cffi`), incl. PDFs | cheap |
+| tier_3 | Cloudflare / anti-bot solver (`cloudscraper`, install `.[cloudflare]`) | cheap-ish (~5s/host) |
+| tier_4 | Stealth headless browser (`patchright`, Chromium) | heavy |
+| tier_5 | Camoufox (Firefox stealth) — **on by default** (opt out: `SCRAPER_DISABLE_CAMOUFOX`) | heavy + slow (~40s on hard CF) |
+| tier_6 | Residential-IP browser over CDP (`BU_CDP_URL`) — off unless configured | heavy (remote egress) |
+| tier_7 | Firecrawl (paid, env-gated, audited) | paid, last resort |
 
 Every URL has a wall-clock budget (`SCRAPER_DEADLINE_S`, default 45s) checked between
 tiers so one URL can't run the whole cascade of timeouts. Each tier attempt records
@@ -142,17 +142,17 @@ Search (query → URLs) is separate from the scrape cascade: `switchback.search(
 ## Install
 
 ```bash
-pip install switchback                 # core: normalization + cheap tiers (0/1) + search
-pip install "switchback[cloudflare]"   # + Tier 2 Cloudflare/anti-bot solver (cloudscraper)
+pip install switchback                 # core: normalization + cheap tiers (tier_1/tier_2) + search
+pip install "switchback[cloudflare]"   # + tier_3 Cloudflare/anti-bot solver (cloudscraper)
 pip install "switchback[server]"       # + HTTP service (fastapi, uvicorn) incl. /metrics + /traces
-pip install "switchback[browser]" && patchright install chromium   # + Tier 3 stealth Chromium
-pip install "switchback[camoufox]" && camoufox fetch               # + Tier 3b Firefox stealth
-pip install "switchback[firecrawl]"    # + Tier 4 paid API (needs FIRECRAWL_API_KEY)
+pip install "switchback[browser]" && patchright install chromium   # + tier_4 stealth Chromium
+pip install "switchback[camoufox]" && camoufox fetch               # + tier_5 Firefox stealth
+pip install "switchback[firecrawl]"    # + tier_7 paid API (needs FIRECRAWL_API_KEY)
 pip install "switchback[tracing]"      # + OpenTelemetry -> any OTLP backend
 pip install "switchback[all]"          # everything
 ```
 
-For Tier 2's **full** v3 JS-VM + Turnstile + stealth, install the Enhanced Edition
+For tier_3's **full** v3 JS-VM + Turnstile + stealth, install the Enhanced Edition
 3.x fork (PyPI's `cloudscraper` is the older v1/v2 — PyPI forbids pinning a
 git-URL dep inside a published package, so install it alongside):
 
@@ -170,19 +170,20 @@ and land *after* boot (e.g. an async install thread on Azure). Until they're
 ready, those tiers report **`unavailable`** (a distinct outcome carrying the exact
 fix) and the cascade falls through — they are never silently skipped. Checklist:
 
-- **Tier 3 is the real workhorse for Cloudflare/JS sites** — make sure its browser
+- **tier_4 is the real workhorse for Cloudflare/JS sites** — make sure its browser
   is installed: `patchright install chromium` (note: **patchright**, not vanilla
   `playwright`). On a cold start, run this in your post-boot install step/thread;
-  Tier 3 flips to ready once it finishes.
-- **Tier 2 needs the cloudscraper 3.x fork** (above) to attempt stealth. With the
+  tier_4 flips to ready once it finishes.
+- **tier_3 needs the cloudscraper 3.x fork** (above) to attempt stealth. With the
   frozen PyPI `cloudscraper` it reports `unavailable` and fails fast (no wasted
-  solve budget) instead of erroring mid-cascade. Tier 2 is a *weak* solver for
+  solve budget) instead of erroring mid-cascade. tier_3 is a *weak* solver for
   modern Cloudflare — treat it as a cheap try before the browser, not the primary.
-- **Install Node.js** for Tier 2's v3 JS-VM challenges — faster and thread-safe
+- **Install Node.js** for tier_3's v3 JS-VM challenges — faster and thread-safe
   vs. the pure-Python js2py fallback (relevant under concurrent load).
-- **Bound Tier 2's solve budget** with `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` (default
-  `25`) so an unsolvable challenge can't eat the per-URL deadline before the
-  browser tier runs. Lower it (e.g. `12`) if Tier 2 rarely wins on your hosts.
+- **Bound tier_3's solve budget** with `SCRAPER_TIER_3_TIMEOUT_S` (default `25`;
+  the old `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` is still honored) so an unsolvable
+  challenge can't eat the per-URL deadline before the browser tier runs. Lower it
+  (e.g. `12`) if tier_3 rarely wins on your hosts.
 
 **Verify readiness on the box** with the preflight check (doubles as a healthcheck
 — exit 0 when the capable tiers are ready):
@@ -282,16 +283,16 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 <details>
 <summary><b>Env gates</b> — enable/disable tiers and integrations</summary>
 
-- `SCRAPER_DISABLE_FIRECRAWL` — skip Tier 4
-- `FIRECRAWL_API_KEY` — enable Tier 4
-- `SCRAPER_DISABLE_CAMOUFOX` — turn off Tier 3b (on by default; needs `pip install camoufox` + `camoufox fetch`)
-- `BU_CDP_URL` — enable Tier 3c residential browser by pointing at a CDP endpoint
+- `SCRAPER_DISABLE_FIRECRAWL` — skip tier_7
+- `FIRECRAWL_API_KEY` — enable tier_7
+- `SCRAPER_DISABLE_CAMOUFOX` — turn off tier_5 (on by default; needs `pip install camoufox` + `camoufox fetch`)
+- `BU_CDP_URL` — enable tier_6 residential browser by pointing at a CDP endpoint
 - `SCRAPER_PROXY` — route *all* tiers/URLs through a proxy
 - `SCRAPER_EGRESS_PROXY` — route only walled hosts through a proxy (see [Cost-scoped residential egress](#cost-scoped-residential-egress))
 - `SEARXNG_URL` — defaults to `http://localhost:8888`
 - `SCRAPER_STATE_DIR` — where the botwall DB/event log + session cache live
 - `SCRAPER_COOKIES_FILE` — Netscape `cookies.txt` to scrape login-gated hosts (injected into the HTTP and browser tiers)
-- `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into Tier 2 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.
+- `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into tier_3 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.
 </details>
 
 <details>
@@ -300,7 +301,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 - `SCRAPER_OUTPUT_FORMAT` — output shape: `markdown` (default) · `markdown_trimmed` · `html` · `html_selectors` (see [Output formats](#output-formats))
 - `SCRAPER_DEADLINE_S` — per-URL budget (45s)
 - `SCRAPER_FIRECRAWL_FALLBACK_AFTER_S` — after this many seconds on a URL, stop trying the local tiers and fall back to Firecrawl, so a hard host doesn't burn the whole deadline before the paid last resort gets a turn (25s; 0 = off)
-- `SCRAPER_CAMOUFOX_TIMEOUT_MS` — (45000)
+- `SCRAPER_TIER_<N>_TIMEOUT_S` — per-tier timeout in seconds, `N` = 1–7 (tier_1 apis · tier_2 http · tier_3 cloudscraper · tier_4 browser · tier_5 camoufox · tier_6 residential · tier_7 firecrawl). Defaults: 15/15/**25**/15/**45**/**30**/15 (the three bold ones keep their prior budgets; everything else is 15s). The pre-0.5.0 `SCRAPER_CLOUDSCRAPER_TIMEOUT_S` / `SCRAPER_CAMOUFOX_TIMEOUT_MS` / `SCRAPER_RESIDENTIAL_TIMEOUT_MS` are still honored when the new var is unset. Note: `tier_7` (Firecrawl) was previously unbounded — its 15s default now bounds the paid tier, so raise `SCRAPER_TIER_7_TIMEOUT_S` if slow hosts get cut off
 - `SCRAPER_BROWSER_CONCURRENCY` — max simultaneous headless browsers (default 1)
 - `SCRAPER_BOTWALL_URL_SKIP_COOLDOWN_H` — auto-skip re-test window (24h; 0 = never)
 - `SCRAPER_BOTWALL_EGRESS_AFTER` — local-tier failures before a host escalates to the residential tier (default 2)
@@ -308,7 +309,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 - `SCRAPER_DISABLE_SESSION_CACHE` — turn off cf_clearance reuse
 - `SCRAPER_CONTENT_TTL_S` — URL→result cache TTL (**0 = off**; set e.g. 86400 to skip re-scraping a page within a day)
 - `SCRAPER_BACKOFF_BASE_MS` / `SCRAPER_BACKOFF_MAX_MS` — exponential backoff between tiers after a rate-limit/timeout (base 0 = off)
-- `SCRAPER_TIER_RETRIES` — same-tier retries before falling through (default 0 = off; `N` → up to `1+N` tries per tier), with per-tier overrides `SCRAPER_TIER_RETRIES_<TIER>` (e.g. `SCRAPER_TIER_RETRIES_TIER3_BROWSER=2`)
+- `SCRAPER_TIER_RETRIES` — same-tier retries before falling through (default 0 = off; `N` → up to `1+N` tries per tier), with per-tier overrides `SCRAPER_TIER_RETRIES_<TIER>` (e.g. `SCRAPER_TIER_RETRIES_TIER_4=2`)
 - `SCRAPER_TIER_RETRY_ON` — failure classes eligible for a same-tier retry (default `timeout,rate_limited,connection`; widen to include `botwall,http_block` behind a rotating residential proxy so each retry gets a fresh IP). Retries are bounded by `SCRAPER_DEADLINE_S`; enabling them on the paid Firecrawl tier bills per attempt
 - `SCRAPER_LOGIN_HOOK` — `pkg.module:func` returning `{cookie: value}` for a host (see [Logged-in sessions](#logged-in-sessions))
 - `SCRAPER_EXTRACTION_FILE` — per-domain extraction prefs JSON (default `config/extraction.json`)

{switchback-0.4.0 → switchback-0.5.0}/switchback.egg-info/SOURCES.txt

@@ -40,10 +40,10 @@ switchback/policy/botwall.py
 switchback/policy/gates.py
 switchback/tiers/__init__.py
 switchback/tiers/_browser.py
-switchback/tiers/tier0_apis.py
-switchback/tiers/tier1_http.py
-switchback/tiers/tier2_cloudscraper.py
-switchback/tiers/tier3_browser.py
-switchback/tiers/tier3b_camoufox.py
-switchback/tiers/tier4_firecrawl.py
-switchback/tiers/tier_residential.py
+switchback/tiers/tier_1.py
+switchback/tiers/tier_2.py
+switchback/tiers/tier_3.py
+switchback/tiers/tier_4.py
+switchback/tiers/tier_5.py
+switchback/tiers/tier_6.py
+switchback/tiers/tier_7.py

switchback-0.4.0/switchback/tiers/__init__.py

@@ -1,24 +0,0 @@
-"""The cost-ordered cascade. Each tier exposes:
-
-    NAME : str
-    PAID : bool                       # gated/audited if True
-    fetch(url) -> str | None          # markdown on success; None if not
-                                      # applicable; raises on failure.
-
-Order matters — cheapest/cleanest first, paid last.
-"""
-from . import (tier0_apis, tier1_http, tier2_cloudscraper,
-               tier3_browser, tier3b_camoufox, tier_residential, tier4_firecrawl)
-
-TIERS = [
-    tier0_apis,
-    tier1_http,
-    tier2_cloudscraper,
-    tier3_browser,
-    tier3b_camoufox,   # env-gated Firefox stealth (off by default; orthogonal to T3)
-    tier_residential,  # residential-IP CDP browser (off unless BU_CDP_URL set)
-    tier4_firecrawl,
-]
-
-# tier name -> index, for botwall winning-tier routing.
-INDEX = {t.NAME: i for i, t in enumerate(TIERS)}