switchback 0.6.0__tar.gz → 0.6.2__tar.gz

{switchback-0.6.0 → switchback-0.6.2}/CHANGELOG.md

@@ -4,6 +4,37 @@ All notable changes to this project are documented here. Format loosely follows
 [Keep a Changelog](https://keepachangelog.com/); this project uses semantic-ish
 versioning while pre-1.0.
 
+## [0.6.2] - 2026-10-14
+
+### Fixed
+- **Block pages hidden behind inline images were returned as successes.** A
+  bot-wall page that embeds its logo as a base64 `data:` image (e.g. Indeed's
+  Cloudflare block page, ~7k chars of SVG before "You have been blocked") pushed
+  the block phrase past the 600-char head scan *and* cleared the 2000-char length
+  floor on image bytes alone, so a browser tier reported `OK` with the block page
+  as content. The quality gates now ignore inline `data:` payloads when scanning
+  and measuring (returned markdown is unchanged), and Cloudflare's WAF copy "you
+  have been blocked" is a recognised wall (vendor `cloudflare`).
+
+## [0.6.1] - 2026-10-14
+
+### Fixed
+- **camoufox geoip on fresh installs.** camoufox 0.5 swapped `geoip2` for
+  `maxminddb` in its `[geoip]` extra, so tier_5 decided geoip was missing and ran
+  proxied sessions without matching locale/timezone. It now asks camoufox's own
+  flag, which works on 0.4 and 0.5.
+
+### Changed
+- **Newer Chrome TLS fingerprints in tier_2** (`chrome142`, `chrome145`,
+  `chrome146` instead of `chrome131`, `chrome136`, `chrome142`); two-year-old
+  Chrome versions are a fingerprint tell. Needs `curl_cffi>=0.15`, now the floor.
+- **Security floors:** `pypdf>=6.19` (older releases have DoS CVEs on crafted
+  PDFs, which this library parses from the web) and `requests>=2.33`
+  (CVE-2024-35195, CVE-2024-47081, CVE-2026-25645).
+- **arXiv calls no longer send the maintainer's email.** tier_1's arXiv
+  User-Agent named a fixed contact address on every install; it now names the
+  project URL, and `SCRAPER_ARXIV_CONTACT` adds your own address if you want one.
+
 ## [0.6.0] - 2026-10-14
 
 ### Added

{switchback-0.6.0 → switchback-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: switchback
-Version: 0.6.0
+Version: 0.6.2
 Summary: One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool.
 Author-email: Akash Kodavuru <akash@theaklabs.com>
 License: MIT
@@ -27,9 +27,9 @@ License-File: LICENSE
 License-File: NOTICE
 Requires-Dist: markdownify
 Requires-Dist: beautifulsoup4
-Requires-Dist: pypdf
-Requires-Dist: requests
-Requires-Dist: curl_cffi
+Requires-Dist: pypdf>=6.19
+Requires-Dist: requests>=2.33
+Requires-Dist: curl_cffi>=0.15
 Provides-Extra: cloudflare
 Requires-Dist: cloudscraper; extra == "cloudflare"
 Provides-Extra: browser
@@ -398,6 +398,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 - `SCRAPER_PROXY` — route *all* tiers/URLs through a proxy
 - `SCRAPER_EGRESS_PROXY` — route only walled hosts through a proxy (see [Cost-scoped residential egress](#cost-scoped-residential-egress))
 - `SEARXNG_URL` — defaults to `http://localhost:8888`
+- `SCRAPER_ARXIV_CONTACT` — optional email added to the User-Agent of arXiv API calls (arXiv asks clients to identify themselves; the default names only the project URL)
 - `SCRAPER_STATE_DIR` — where the botwall DB/event log + session cache live
 - `SCRAPER_COOKIES_FILE` — Netscape `cookies.txt` to scrape login-gated hosts (injected into the HTTP and browser tiers)
 - `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into tier_3 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.

{switchback-0.6.0 → switchback-0.6.2}/README.md

@@ -333,6 +333,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 - `SCRAPER_PROXY` — route *all* tiers/URLs through a proxy
 - `SCRAPER_EGRESS_PROXY` — route only walled hosts through a proxy (see [Cost-scoped residential egress](#cost-scoped-residential-egress))
 - `SEARXNG_URL` — defaults to `http://localhost:8888`
+- `SCRAPER_ARXIV_CONTACT` — optional email added to the User-Agent of arXiv API calls (arXiv asks clients to identify themselves; the default names only the project URL)
 - `SCRAPER_STATE_DIR` — where the botwall DB/event log + session cache live
 - `SCRAPER_COOKIES_FILE` — Netscape `cookies.txt` to scrape login-gated hosts (injected into the HTTP and browser tiers)
 - `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into tier_3 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.

{switchback-0.6.0 → switchback-0.6.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "switchback"
-version = "0.6.0"
+version = "0.6.2"
 description = "One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -37,9 +37,9 @@ classifiers = [
 dependencies = [
     "markdownify",
     "beautifulsoup4",
-    "pypdf",
-    "requests",
-    "curl_cffi",      # tier1: TLS-impersonating HTTP
+    "pypdf>=6.19",    # older releases hang or blow up memory on crafted PDFs (DoS CVEs)
+    "requests>=2.33", # CVE-2024-35195, CVE-2024-47081, CVE-2026-25645
+    "curl_cffi>=0.15",  # tier_2: TLS-impersonating HTTP (chrome145/146 targets)
 ]
 
 [project.optional-dependencies]

{switchback-0.6.0 → switchback-0.6.2}/switchback/policy/gates.py

@@ -55,11 +55,24 @@ _BOTWALL_MARKERS = (
     ("humans only",                         "datadome"),    # DataDome (e.g. Glassdoor)
     ("access denied",                       "akamai"),      # Akamai / generic 403
     ("unusual traffic from your computer",  "google"),      # Google bot interstitial
+    ("you have been blocked",               "cloudflare"),  # CF WAF block ("Sorry, you have been blocked")
     ("are you a human",                     "generic"),
     ("ddos protection by",                  "generic"),     # generic CDN challenge
 )
 _BOTWALL_HEAD_CHARS = 600
 
+# Inline base64 images: `![](data:image/svg+xml;base64,…)`. Block pages often
+# embed their logo this way, and a single one can run to thousands of chars —
+# enough to push the block phrase out of the head window and to clear the length
+# floor on its own. Base64 never contains ')' or whitespace, so this is exact.
+_INLINE_DATA_URI = re.compile(r"\(data:[^)\s]*\)")
+
+
+def _strip_inline_data(md: str) -> str:
+    """`md` with inline data-URI payloads emptied (`![](data:…)` -> `![]()`), so
+    the gates measure and scan text, not image bytes."""
+    return _INLINE_DATA_URI.sub("()", md)
+
 
 def classify_botwall(md: str | None) -> str | None:
     """Return the vendor of the bot-wall in the head of `md` (cloudflare /
@@ -67,7 +80,7 @@ def classify_botwall(md: str | None) -> str | None:
     the content doesn't look like a wall. First marker match wins."""
     if not md:
         return None
-    head = md[:_BOTWALL_HEAD_CHARS].lower()
+    head = _strip_inline_data(md)[:_BOTWALL_HEAD_CHARS].lower()
     for marker, vendor in _BOTWALL_MARKERS:
         if marker in head:
             return vendor
@@ -189,13 +202,15 @@ def check(url: str, md: str | None) -> str:
     vendor = classify_botwall(md)
     if vendor:
         raise BotWall(f"bot-wall / block page detected ({vendor})", vendor=vendor)
+    # Measure text, not inline image bytes; the caller still gets `md` unchanged.
+    text = _strip_inline_data(md or "")
     gate = min_len_for(url)
-    n = len(md) if md else 0
+    n = len(text)
     if n < gate:
         raise ShortContent(f"body too short: {n} < {gate}")
     # Length cleared, but is it actually content? Reject shells/placeholders so a
     # tier falls through instead of returning a confident false-positive success.
-    shell = _content_shell_reason(md or "")
+    shell = _content_shell_reason(text)
     if shell:
         raise ShortContent(f"no article content: {shell}")
     return md

{switchback-0.6.0 → switchback-0.6.2}/switchback/tiers/tier_1.py

@@ -64,9 +64,11 @@ def _arxiv(arxiv_id: str, url: str) -> str:
     # arxiv wants plain requests + an identifying UA (their published guidance);
     # impersonating Chrome triggers aggressive 429s from their Akamai front-end.
     import requests
+    # The contact is the operator's to give, so it's opt-in; the project URL identifies the client.
+    contact = os.getenv("SCRAPER_ARXIV_CONTACT", "").strip()
+    ua = f"switchback/1.0 (+https://github.com/akash-kr/switchback{'; mailto:' + contact if contact else ''})"
     r = requests.get(f"https://export.arxiv.org/api/query?id_list={arxiv_id}",
-                     timeout=_TIMEOUT_S,
-                     headers={"User-Agent": "switchback/1.0 (mailto:akash@theaklabs.com)"})
+                     timeout=_TIMEOUT_S, headers={"User-Agent": ua})
     r.raise_for_status()
     ns = {"atom": "http://www.w3.org/2005/Atom"}
     entry = ET.fromstring(r.text).find("atom:entry", ns)

{switchback-0.6.0 → switchback-0.6.2}/switchback/tiers/tier_2.py

@@ -37,7 +37,7 @@ _TIMEOUT_S = float(os.getenv("SCRAPER_TIER_2_TIMEOUT_S", "15"))
 
 # Recent Chrome JA3 targets available in curl_cffi 0.15.x. A small spread of real
 # versions mirrors how live traffic is distributed across Chrome releases.
-_IMPERSONATE_TARGETS = ("chrome131", "chrome136", "chrome142")
+_IMPERSONATE_TARGETS = ("chrome142", "chrome145", "chrome146")
 
 
 def _impersonate_for(url: str) -> str:

{switchback-0.6.0 → switchback-0.6.2}/switchback/tiers/tier_5.py

@@ -13,6 +13,7 @@ before the paid Firecrawl tier.
 """
 from __future__ import annotations
 
+import importlib
 import logging
 import os
 
@@ -43,14 +44,15 @@ def disabled() -> bool:
 
 
 def _geoip_available() -> bool:
-    """camoufox's geoip matching needs the `geoip2` package (the camoufox[geoip]
-    extra). Without it, requesting geoip raises and kills the whole tier — so we
-    probe and degrade gracefully instead."""
-    try:
-        import geoip2  # noqa: F401
-        return True
-    except Exception:
-        return False
+    """camoufox's geoip matching needs its camoufox[geoip] extra (`geoip2` before
+    0.5, `maxminddb` from 0.5). Without it, requesting geoip raises and kills the
+    whole tier — so we ask camoufox itself and degrade gracefully instead."""
+    for module in ("camoufox.geolocation", "camoufox.locale"):  # 0.5+, then 0.4
+        try:
+            return bool(importlib.import_module(module).ALLOW_GEOIP)
+        except (ImportError, AttributeError):
+            continue
+    return False
 
 
 def _launch_opts() -> dict:

{switchback-0.6.0 → switchback-0.6.2}/switchback.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: switchback
-Version: 0.6.0
+Version: 0.6.2
 Summary: One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool.
 Author-email: Akash Kodavuru <akash@theaklabs.com>
 License: MIT
@@ -27,9 +27,9 @@ License-File: LICENSE
 License-File: NOTICE
 Requires-Dist: markdownify
 Requires-Dist: beautifulsoup4
-Requires-Dist: pypdf
-Requires-Dist: requests
-Requires-Dist: curl_cffi
+Requires-Dist: pypdf>=6.19
+Requires-Dist: requests>=2.33
+Requires-Dist: curl_cffi>=0.15
 Provides-Extra: cloudflare
 Requires-Dist: cloudscraper; extra == "cloudflare"
 Provides-Extra: browser
@@ -398,6 +398,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 - `SCRAPER_PROXY` — route *all* tiers/URLs through a proxy
 - `SCRAPER_EGRESS_PROXY` — route only walled hosts through a proxy (see [Cost-scoped residential egress](#cost-scoped-residential-egress))
 - `SEARXNG_URL` — defaults to `http://localhost:8888`
+- `SCRAPER_ARXIV_CONTACT` — optional email added to the User-Agent of arXiv API calls (arXiv asks clients to identify themselves; the default names only the project URL)
 - `SCRAPER_STATE_DIR` — where the botwall DB/event log + session cache live
 - `SCRAPER_COOKIES_FILE` — Netscape `cookies.txt` to scrape login-gated hosts (injected into the HTTP and browser tiers)
 - `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into tier_3 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.

{switchback-0.6.0 → switchback-0.6.2}/switchback.egg-info/requires.txt

@@ -1,8 +1,8 @@
 markdownify
 beautifulsoup4
-pypdf
-requests
-curl_cffi
+pypdf>=6.19
+requests>=2.33
+curl_cffi>=0.15
 
 [all]
 cloudscraper