switchback 0.6.1__tar.gz → 0.6.3__tar.gz

{switchback-0.6.1 → switchback-0.6.3}/CHANGELOG.md

@@ -4,6 +4,27 @@ All notable changes to this project are documented here. Format loosely follows
 [Keep a Changelog](https://keepachangelog.com/); this project uses semantic-ish
 versioning while pre-1.0.
 
+## [0.6.3] - 2026-10-14
+
+### Fixed
+- **"Page not found" pages returned as articles by the browser tiers.** tier_4
+  and tier_5 ignored the HTTP status of the page they loaded, so a site answering
+  404 (seen on bhaskarenglish.in, which serves its 404 page to some bot traffic)
+  had its "404 - Page not found" page returned as a success. A 404 or 410 now
+  fails the tier, as it already did in tier_2, and the cascade moves on.
+
+## [0.6.2] - 2026-10-14
+
+### Fixed
+- **Block pages hidden behind inline images were returned as successes.** A
+  bot-wall page that embeds its logo as a base64 `data:` image (e.g. Indeed's
+  Cloudflare block page, ~7k chars of SVG before "You have been blocked") pushed
+  the block phrase past the 600-char head scan *and* cleared the 2000-char length
+  floor on image bytes alone, so a browser tier reported `OK` with the block page
+  as content. The quality gates now ignore inline `data:` payloads when scanning
+  and measuring (returned markdown is unchanged), and Cloudflare's WAF copy "you
+  have been blocked" is a recognised wall (vendor `cloudflare`).
+
 ## [0.6.1] - 2026-10-14
 
 ### Fixed

{switchback-0.6.1 → switchback-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: switchback
-Version: 0.6.1
+Version: 0.6.3
 Summary: One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool.
 Author-email: Akash Kodavuru <akash@theaklabs.com>
 License: MIT

{switchback-0.6.1 → switchback-0.6.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "switchback"
-version = "0.6.1"
+version = "0.6.3"
 description = "One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool."
 readme = "README.md"
 requires-python = ">=3.10"

{switchback-0.6.1 → switchback-0.6.3}/switchback/policy/gates.py

@@ -55,11 +55,24 @@ _BOTWALL_MARKERS = (
     ("humans only",                         "datadome"),    # DataDome (e.g. Glassdoor)
     ("access denied",                       "akamai"),      # Akamai / generic 403
     ("unusual traffic from your computer",  "google"),      # Google bot interstitial
+    ("you have been blocked",               "cloudflare"),  # CF WAF block ("Sorry, you have been blocked")
     ("are you a human",                     "generic"),
     ("ddos protection by",                  "generic"),     # generic CDN challenge
 )
 _BOTWALL_HEAD_CHARS = 600
 
+# Inline base64 images: `![](data:image/svg+xml;base64,…)`. Block pages often
+# embed their logo this way, and a single one can run to thousands of chars —
+# enough to push the block phrase out of the head window and to clear the length
+# floor on its own. Base64 never contains ')' or whitespace, so this is exact.
+_INLINE_DATA_URI = re.compile(r"\(data:[^)\s]*\)")
+
+
+def _strip_inline_data(md: str) -> str:
+    """`md` with inline data-URI payloads emptied (`![](data:…)` -> `![]()`), so
+    the gates measure and scan text, not image bytes."""
+    return _INLINE_DATA_URI.sub("()", md)
+
 
 def classify_botwall(md: str | None) -> str | None:
     """Return the vendor of the bot-wall in the head of `md` (cloudflare /
@@ -67,7 +80,7 @@ def classify_botwall(md: str | None) -> str | None:
     the content doesn't look like a wall. First marker match wins."""
     if not md:
         return None
-    head = md[:_BOTWALL_HEAD_CHARS].lower()
+    head = _strip_inline_data(md)[:_BOTWALL_HEAD_CHARS].lower()
     for marker, vendor in _BOTWALL_MARKERS:
         if marker in head:
             return vendor
@@ -189,13 +202,15 @@ def check(url: str, md: str | None) -> str:
     vendor = classify_botwall(md)
     if vendor:
         raise BotWall(f"bot-wall / block page detected ({vendor})", vendor=vendor)
+    # Measure text, not inline image bytes; the caller still gets `md` unchanged.
+    text = _strip_inline_data(md or "")
     gate = min_len_for(url)
-    n = len(md) if md else 0
+    n = len(text)
     if n < gate:
         raise ShortContent(f"body too short: {n} < {gate}")
     # Length cleared, but is it actually content? Reject shells/placeholders so a
     # tier falls through instead of returning a confident false-positive success.
-    shell = _content_shell_reason(md or "")
+    shell = _content_shell_reason(text)
     if shell:
         raise ShortContent(f"no article content: {shell}")
     return md

{switchback-0.6.1 → switchback-0.6.3}/switchback/tiers/_browser.py

@@ -34,6 +34,14 @@ def looks_blocked(html: str, url: str) -> bool:
     return _looks_like_botwall(html_to_markdown(html, base_url=url))
 
 
+def raise_if_missing(response) -> None:
+    """Fail the tier on a 404/410 document, as tier_2 does; a browser otherwise
+    renders the site's "Page not found" page and returns it as the article."""
+    status = getattr(response, "status", None)
+    if status in (404, 410):
+        raise RuntimeError(f"HTTP Error {status}: page not found")
+
+
 def response_bytes(responses) -> int:
     """Total wire bytes a render pulled across every resource — the residential-cost
     basis. Reads only the Content-Length header: it's non-blocking. (We deliberately

{switchback-0.6.1 → switchback-0.6.3}/switchback/tiers/tier_4.py

@@ -80,7 +80,7 @@ def fetch(url: str, timeout_ms: int = int(_TIMEOUT_S * 1000)) -> str:
             page = ctx.new_page()
             responses: list = []
             page.on("response", lambda resp: responses.append(resp))
-            page.goto(url, timeout=timeout_ms, wait_until="domcontentloaded")
+            _browser.raise_if_missing(page.goto(url, timeout=timeout_ms, wait_until="domcontentloaded"))
             html = page.content()
             if len(html) < 5000:
                 try:

{switchback-0.6.1 → switchback-0.6.3}/switchback/tiers/tier_5.py

@@ -86,7 +86,7 @@ def fetch(url: str) -> str:
             auth = session_cache.browser_cookies(url)
             if auth:
                 page.context.add_cookies(auth)
-            page.goto(url, wait_until="networkidle", timeout=_TIMEOUT_MS)
+            _browser.raise_if_missing(page.goto(url, wait_until="networkidle", timeout=_TIMEOUT_MS))
             html = page.content()
             # JS bot-manager sensor interstitial → settle + reload once.
             if _browser.looks_blocked(html, url):

{switchback-0.6.1 → switchback-0.6.3}/switchback.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: switchback
-Version: 0.6.1
+Version: 0.6.3
 Summary: One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool.
 Author-email: Akash Kodavuru <akash@theaklabs.com>
 License: MIT