switchback 0.1.0__py3-none-any.whl

switchback/tiers/tier_residential.py

@@ -0,0 +1,57 @@
+"""Tier (residential egress) — drive a remote browser over CDP.
+
+The strongest egress lever we own: connect to a browser running on a *residential*
+IP with a real, full Chrome fingerprint, instead of this box's datacenter IP. That
+is the one thing no local fingerprint trick can fake, and it's the actual reason
+hard Cloudflare / DataDome hosts (and datacenter-blocklisted sites) wall us.
+
+Operator-provided seam: export BU_CDP_URL to a CDP endpoint and this tier connects
+to it. Start one with browser-harness:
+
+    start_remote_daemon("scrape", proxyCountryCode="us")   # prints a cdpUrl
+
+then export that URL as BU_CDP_URL. Unset → tier is disabled and the cascade
+skips it. It sits after the local browser tiers (only walls that beat them pay
+this cost) and before paid Firecrawl.
+"""
+from __future__ import annotations
+
+import os
+
+from ..concurrency import browser_slot
+from ..normalize import html_to_markdown
+from ..policy.gates import check
+
+NAME = "tier_residential"
+PAID = False
+
+_TIMEOUT_MS = int(os.getenv("SCRAPER_RESIDENTIAL_TIMEOUT_MS", "30000"))
+
+
+def disabled() -> bool:
+    """Off unless an operator has wired a residential CDP endpoint."""
+    return not os.getenv("BU_CDP_URL")
+
+
+def fetch(url: str) -> str:
+    from patchright.sync_api import sync_playwright
+    cdp = os.environ["BU_CDP_URL"]
+    with browser_slot(NAME), sync_playwright() as p:
+        browser = p.chromium.connect_over_cdp(cdp)
+        try:
+            ctx = browser.contexts[0] if browser.contexts else browser.new_context()
+            page = ctx.new_page()
+            try:
+                page.goto(url, timeout=_TIMEOUT_MS, wait_until="domcontentloaded")
+                if len(page.content()) < 5000:
+                    try:
+                        page.wait_for_load_state("networkidle", timeout=8000)
+                    except Exception:
+                        pass
+                html = page.content()
+            finally:
+                page.close()
+            md = html_to_markdown(html, base_url=page.url or url)
+        finally:
+            browser.close()
+    return check(url, md)

switchback/tracing.py

@@ -0,0 +1,152 @@
+"""OpenTelemetry wiring → any OTLP backend: traces and logs.
+
+Degrades gracefully: if the OTel packages aren't installed the `span()` helper
+becomes a no-op context manager and `setup_logs()` is a no-op, so the engine
+still runs without a backend. Point it at an OTLP backend (Jaeger, Tempo, SigNoz) with:
+
+    export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+    export OTEL_SERVICE_NAME=switchback
+
+Traces: one trace per scraped URL, one span per tier attempt. Span attributes
+use the keys in `Attr` so backend dashboards stay consistent.
+
+Logs: `setup_logs()` routes the stdlib `logging` records to an OTLP backend
+(opt-in — call it from the CLI / app entry point, not from library code, so we
+never hijack a host app's logging). Records emitted inside a span are
+auto-correlated with that trace.
+"""
+from __future__ import annotations
+
+import logging
+import os
+import threading
+from contextlib import contextmanager
+
+
+class Attr:
+    """Canonical span-attribute keys (keep dashboards consistent)."""
+    HOST = "scrape.host"
+    TIER = "scrape.tier"
+    OUTCOME = "scrape.outcome"      # ok | short_content | botwall | http_block |
+                                    # rate_limited | timeout | connection |
+                                    # http_error | error | not_applicable |
+                                    # deadline_exceeded | all_failed | *_skipped
+    ERROR_CLASS = "scrape.error_class"  # normalized failure class (see classify_error)
+    CHALLENGE = "scrape.challenge"      # bot-wall vendor when one was served
+                                        # (cloudflare / datadome / akamai / …)
+    STATUS_CODE = "scrape.status_code"  # upstream HTTP status when known (403/429/…)
+    MD_LEN = "scrape.md_len"
+    SOURCE = "scrape.source_method"
+    ERROR = "scrape.error"
+    COST_USD = "scrape.cost_usd"
+    LATENCY_MS = "scrape.latency_ms"   # per-tier attempt, and total on the root
+    DEADLINE_S = "scrape.deadline_s"   # the per-request budget that was in force
+
+
+_tracer = None
+_init_lock = threading.Lock()
+
+
+def _init():
+    global _tracer
+    if _tracer is not None:
+        return _tracer
+    with _init_lock:
+        if _tracer is not None:
+            return _tracer
+        try:
+            from opentelemetry import trace
+            from opentelemetry.sdk.resources import Resource
+            from opentelemetry.sdk.trace import TracerProvider
+            from opentelemetry.sdk.trace.export import BatchSpanProcessor
+            from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+
+            service = os.getenv("OTEL_SERVICE_NAME", "switchback")
+            provider = TracerProvider(resource=Resource.create({"service.name": service}))
+            # Endpoint comes from OTEL_EXPORTER_OTLP_ENDPOINT; defaults to localhost:4317.
+            provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
+            trace.set_tracer_provider(provider)
+            _tracer = trace.get_tracer("switchback")
+        except Exception:
+            _tracer = False  # tried and unavailable → no-op mode
+    return _tracer
+
+
+_log_provider = None
+
+
+def setup_logs(level: int = logging.INFO) -> bool:
+    """Route stdlib logging → an OTLP backend. Idempotent. Returns False (and
+    does nothing) if OTel isn't installed/configured. Opt-in: call from an app
+    entry point, not from library code."""
+    global _log_provider
+    if _log_provider is not None:
+        return bool(_log_provider)
+    try:
+        from opentelemetry._logs import set_logger_provider
+        from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+        from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+        from opentelemetry.sdk.resources import Resource
+        from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+
+        service = os.getenv("OTEL_SERVICE_NAME", "switchback")
+        provider = LoggerProvider(resource=Resource.create({"service.name": service}))
+        provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
+        set_logger_provider(provider)
+        handler = LoggingHandler(level=level, logger_provider=provider)
+        logging.getLogger().addHandler(handler)
+        _log_provider = provider
+    except Exception:
+        _log_provider = False  # tried and unavailable → no-op
+    return bool(_log_provider)
+
+
+def flush(timeout_ms: int = 5000) -> None:
+    """Force-export buffered spans and logs. Call at the end of a batch/CLI run
+    so telemetry lands before the process exits (the batch processors otherwise
+    flush on their own ~5s timer)."""
+    if _tracer:
+        try:
+            from opentelemetry import trace
+            provider = trace.get_tracer_provider()
+            if hasattr(provider, "force_flush"):
+                provider.force_flush(timeout_ms)
+        except Exception:
+            pass
+    if _log_provider:
+        try:
+            _log_provider.force_flush(timeout_ms)
+        except Exception:
+            pass
+
+
+@contextmanager
+def span(name: str, **attrs):
+    """Start a span. No-op if OTel isn't installed/configured.
+
+    Yields a small object with `.set(key, value)` so tiers can attach the
+    outcome/length/error once they know it.
+    """
+    tracer = _init()
+    if not tracer:
+        yield _NoopSpan()
+        return
+    with tracer.start_as_current_span(name) as sp:
+        for k, v in attrs.items():
+            if v is not None:
+                sp.set_attribute(k, v)
+        yield _RealSpan(sp)
+
+
+class _NoopSpan:
+    def set(self, key, value):
+        pass
+
+
+class _RealSpan:
+    def __init__(self, sp):
+        self._sp = sp
+
+    def set(self, key, value):
+        if value is not None:
+            self._sp.set_attribute(key, value)

switchback-0.1.0.dist-info/METADATA

@@ -0,0 +1,325 @@
+Metadata-Version: 2.4
+Name: switchback
+Version: 0.1.0
+Summary: One cost-ordered scrape cascade (HTTP → stealth browser → paid), shared by every tool.
+Author-email: Akash Kodavuru <akash@theaklabs.com>
+License: MIT
+Project-URL: Homepage, https://github.com/akash-kr/switchback
+Project-URL: Repository, https://github.com/akash-kr/switchback
+Project-URL: Issues, https://github.com/akash-kr/switchback/issues
+Project-URL: Changelog, https://github.com/akash-kr/switchback/blob/main/CHANGELOG.md
+Keywords: scraping,crawler,cloudflare,markdown,cascade,anti-bot,stealth
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: NOTICE
+Requires-Dist: markdownify
+Requires-Dist: beautifulsoup4
+Requires-Dist: pypdf
+Requires-Dist: requests
+Requires-Dist: curl_cffi
+Provides-Extra: cloudflare
+Requires-Dist: cloudscraper; extra == "cloudflare"
+Provides-Extra: browser
+Requires-Dist: patchright; extra == "browser"
+Provides-Extra: camoufox
+Requires-Dist: camoufox[geoip]; extra == "camoufox"
+Provides-Extra: firecrawl
+Requires-Dist: firecrawl-py; extra == "firecrawl"
+Provides-Extra: server
+Requires-Dist: fastapi; extra == "server"
+Requires-Dist: uvicorn; extra == "server"
+Provides-Extra: tracing
+Requires-Dist: opentelemetry-sdk; extra == "tracing"
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc; extra == "tracing"
+Provides-Extra: all
+Requires-Dist: cloudscraper; extra == "all"
+Requires-Dist: patchright; extra == "all"
+Requires-Dist: camoufox[geoip]; extra == "all"
+Requires-Dist: firecrawl-py; extra == "all"
+Requires-Dist: fastapi; extra == "all"
+Requires-Dist: uvicorn; extra == "all"
+Requires-Dist: opentelemetry-sdk; extra == "all"
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+<!-- switchback -->
+
+```
+███████╗██╗    ██╗██╗████████╗ ██████╗██╗  ██╗██████╗  █████╗  ██████╗██╗  ██╗
+██╔════╝██║    ██║██║╚══██╔══╝██╔════╝██║  ██║██╔══██╗██╔══██╗██╔════╝██║ ██╔╝
+███████╗██║ █╗ ██║██║   ██║   ██║     ███████║██████╔╝███████║██║     █████╔╝
+╚════██║██║███╗██║██║   ██║   ██║     ██╔══██║██╔══██╗██╔══██║██║     ██╔═██╗
+███████║╚███╔███╔╝██║   ██║   ╚██████╗██║  ██║██████╔╝██║  ██║╚██████╗██║  ██╗
+╚══════╝ ╚══╝╚══╝ ╚═╝   ╚═╝    ╚═════╝╚═╝  ╚═╝╚═════╝ ╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝
+```
+
+<div align="center">
+
+**One cost-ordered scrape cascade — HTTP → stealth browser → paid — shared by every tool.**
+
+Give it a URL; it tries the cheapest way to get clean Markdown first and only escalates
+to a heavier (slower, costlier) tier when the cheap one is walled. Stops at the first success.
+
+[![PyPI](https://img.shields.io/pypi/v/switchback.svg)](https://pypi.org/project/switchback/)
+[![Python](https://img.shields.io/pypi/pyversions/switchback.svg)](https://pypi.org/project/switchback/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+[![CI](https://github.com/akash-kr/switchback/actions/workflows/ci.yml/badge.svg)](https://github.com/akash-kr/switchback/actions/workflows/ci.yml)
+
+</div>
+
+---
+
+## Why
+
+Most scrapers either give up on hard pages or send *everything* through an expensive
+headless browser / paid API. **switchback** orders the methods by cost and walks them
+cheapest-first, per host, learning which tier wins where so the next run starts there.
+The easy majority stays free; only genuinely-walled hosts pay for the heavy tiers.
+
+- **Cost-ordered cascade** — free APIs → cheap HTTP → anti-bot solver → stealth browser → paid API.
+- **Per-host memory (botwall)** — remembers the winning tier per host, skip-lists hard blockers, auto-skips hosts stuck on the paid tier.
+- **Cost-scoped residential egress** — routes *only* walled hosts through a residential proxy, never the easy majority.
+- **One shape, three entry points** — Python library, CLI (JSON on stdout), or an HTTP service.
+- **Observable** — every attempt is an OpenTelemetry span; logs ship trace-correlated to any OTLP backend (Jaeger, Tempo, SigNoz).
+- **Runs with any subset installed** — each tier imports its deps lazily; a missing one is just a tier miss.
+
+## Quickstart
+
+```bash
+pip install switchback                 # core: cheap tiers (0/1) + search
+```
+
+```python
+from switchback import scrape
+
+for r in scrape(["https://arxiv.org/abs/1706.03762"]):
+    print(r.source_method, len(r.markdown))
+```
+
+```bash
+python -m switchback https://example.com/article    # JSON on stdout — bridge for any language
+```
+
+That's the whole loop. Add tiers as you need them (see [Install](#install)).
+
+## The cascade (stop at first success)
+
+| Tier | Strategy | Cost |
+|---|---|---|
+| 0 | Direct APIs / mirrors (arxiv, wikipedia, EuropePMC; extend: job boards) | free, cleanest |
+| 1 | Plain HTTP + TLS impersonation (`curl_cffi`), incl. PDFs | cheap |
+| 2 | Cloudflare / anti-bot solver (`cloudscraper`, install `.[cloudflare]`) | cheap-ish (~5s/host) |
+| 3 | Stealth headless browser (`patchright`, Chromium) | heavy |
+| 3b | Camoufox (Firefox stealth) — **on by default** (opt out: `SCRAPER_DISABLE_CAMOUFOX`) | heavy + slow (~40s on hard CF) |
+| 3c | Residential-IP browser over CDP (`BU_CDP_URL`) — off unless configured | heavy (remote egress) |
+| 4 | Firecrawl (paid, env-gated, audited) | paid, last resort |
+
+Every URL has a wall-clock budget (`SCRAPER_DEADLINE_S`, default 45s) checked between
+tiers so one URL can't run the whole cascade of timeouts. Each tier attempt records
+latency + outcome (`ok` / `short_content` / `rate_limited` / `miss` / `not_applicable`)
+to its span and the botwall event log; the root span carries total latency and the final
+outcome (incl. `deadline_exceeded`).
+
+Search (query → URLs) is separate from the scrape cascade: `switchback.search()` /
+`python -m switchback.api --search <query>`, backed by a local SearXNG.
+
+
+## Install
+
+```bash
+pip install switchback                 # core: normalization + cheap tiers (0/1) + search
+pip install "switchback[cloudflare]"   # + Tier 2 Cloudflare/anti-bot solver (cloudscraper)
+pip install "switchback[server]"       # + HTTP service (fastapi, uvicorn) incl. /metrics + /traces
+pip install "switchback[browser]" && patchright install chromium   # + Tier 3 stealth Chromium
+pip install "switchback[camoufox]" && camoufox fetch               # + Tier 3b Firefox stealth
+pip install "switchback[firecrawl]"    # + Tier 4 paid API (needs FIRECRAWL_API_KEY)
+pip install "switchback[tracing]"      # + OpenTelemetry -> any OTLP backend
+pip install "switchback[all]"          # everything
+```
+
+For Tier 2's **full** v3 JS-VM + Turnstile + stealth, install the Enhanced Edition
+3.x fork (PyPI's `cloudscraper` is the older v1/v2 — PyPI forbids pinning a
+git-URL dep inside a published package, so install it alongside):
+
+```bash
+pip install "cloudscraper @ git+https://github.com/VeNoMouS/cloudscraper@3.0.0"
+```
+
+Or run the whole thing as a container:
+`docker build -t switchback . && docker run -p 8799:8799 switchback`.
+
+## Use it from your app
+
+Three interchangeable entry points — all return the same shape
+(`[{url, source_method, markdown}]`, successes only):
+
+**Python library**
+```python
+from switchback import scrape
+for r in scrape(["https://arxiv.org/abs/1706.03762"]):
+    print(r.source_method, len(r.markdown))
+
+# Need failures + reasons too? scrape_detailed returns a ScrapeOutcome per URL
+# (ok, final_outcome, error_class, status_code, and the per-tier attempts):
+from switchback import scrape_detailed
+for o in scrape_detailed(["https://www.pcmag.com/news"]):
+    if not o.ok:
+        print(o.url, o.final_outcome, o.error_class, o.status_code)
+```
+
+**CLI** (JSON on stdout — bridge for any language)
+```bash
+python -m switchback https://example.com/article        # or: switchback <url>
+```
+
+**HTTP service** (language-agnostic; one warm process keeps the browser pool hot)
+```bash
+switchback-server                                    # listens on :8799
+curl -s localhost:8799/scrape -d '{"urls":["https://example.com"]}'
+curl 'localhost:8799/search?q=web+scraping'
+```
+
+Non-Python callers: see [clients/node_bridge.md](clients/node_bridge.md). Python
+callers that want HTTP-with-CLI-fallback can drop in
+[clients/python_client.py](clients/python_client.py).
+
+## Cost-scoped residential egress
+
+The dominant reason hard hosts wall you is the **datacenter IP**, not the
+fingerprint. When a host repeatedly walls the local tiers (a 403/429 or a
+bot-wall page, `SCRAPER_BOTWALL_EGRESS_AFTER` times) it's flagged `needs_egress`
+and the cascade reruns through a **residential proxy** — but only for that host:
+
+```bash
+export SCRAPER_EGRESS_PROXY="http://user:pass@p.webshare.io:80"
+```
+
+The easy majority that already succeeds free at the datacenter IP stays direct,
+so you never spend (often metered) residential bandwidth on it. Escalation tries
+the cheap HTTP tiers through the proxy first (~0.2MB/page) before the heavier
+browser tiers. [Webshare](https://www.webshare.io/)'s free plan includes ~1GB/mo
+of residential bandwidth — enough for low-volume hard-host recovery at $0. Use
+`SCRAPER_PROXY` instead to force *every* request through a proxy.
+
+## Metrics & reporting
+
+The engine derives all metrics from its own state files (no external store): the
+botwall event log (one row per tier attempt, incl. the detected challenge vendor)
+and the per-host DB (winning tier, per-vendor `challenge_counts`).
+
+```bash
+curl localhost:8799/metrics            # cost savings vs Firecrawl, coverage,
+                                       # overall + per-tier latency, outcomes
+curl localhost:8799/metrics/domains    # per-domain: error codes, challenges, latency
+python -m switchback.flags             # periodic digest: domains stuck on Firecrawl,
+                                       # escalated to egress, top challenged (cron-friendly)
+```
+
+Both endpoints accept `?minutes=N` to window the event-derived sections. The
+**savings** figure compares engine spend (Firecrawl invocations only) against a
+Firecrawl-everything baseline, charging the hard-page credit multiplier
+(`BENCH_FIRECRAWL_HARD_MULT`) for URLs that needed a browser/residential tier or
+hit a challenge — i.e. exactly the ones Firecrawl bills more for.
+
+## Configuration
+
+All configuration is via environment variables. The engine runs with missing
+pieces: each tier imports its deps lazily and a missing one just counts as a tier
+miss. Tracing no-ops if OTel isn't installed/configured.
+
+<details>
+<summary><b>Tracing (optional)</b></summary>
+
+```bash
+export OTEL_SERVICE_NAME=switchback
+export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+```
+</details>
+
+<details>
+<summary><b>Env gates</b> — enable/disable tiers and integrations</summary>
+
+- `SCRAPER_DISABLE_FIRECRAWL` — skip Tier 4
+- `FIRECRAWL_API_KEY` — enable Tier 4
+- `SCRAPER_DISABLE_CAMOUFOX` — turn off Tier 3b (on by default; needs `pip install camoufox` + `camoufox fetch`)
+- `BU_CDP_URL` — enable Tier 3c residential browser by pointing at a CDP endpoint
+- `SCRAPER_PROXY` — route *all* tiers/URLs through a proxy
+- `SCRAPER_EGRESS_PROXY` — route only walled hosts through a proxy (see [Cost-scoped residential egress](#cost-scoped-residential-egress))
+- `SEARXNG_URL` — defaults to `http://localhost:8888`
+- `SCRAPER_STATE_DIR` — where the botwall DB/event log + session cache live
+- `SCRAPER_COOKIES_FILE` — Netscape `cookies.txt` to scrape login-gated hosts (injected into the HTTP and browser tiers)
+- `SCRAPER_CAPTCHA_PROVIDER` + `SCRAPER_CAPTCHA_API_KEY` — opt-in, off by default: wire a third-party solver (2captcha/capsolver/capmonster/anticaptcha/deathbycaptcha/9kw) into Tier 2 for Turnstile/reCAPTCHA/hCaptcha on CF hosts. **Paid**, billed per solve by the provider.
+</details>
+
+<details>
+<summary><b>Tunables</b> — budgets, timeouts, caches, backoff</summary>
+
+- `SCRAPER_DEADLINE_S` — per-URL budget (45s)
+- `SCRAPER_CAMOUFOX_TIMEOUT_MS` — (45000)
+- `SCRAPER_BROWSER_CONCURRENCY` — max simultaneous headless browsers (default 1)
+- `SCRAPER_BOTWALL_URL_SKIP_COOLDOWN_H` — auto-skip re-test window (24h; 0 = never)
+- `SCRAPER_BOTWALL_EGRESS_AFTER` — local-tier failures before a host escalates to the residential tier (default 2)
+- `SCRAPER_SESSION_TTL_S` — cf_clearance reuse window (1800s)
+- `SCRAPER_DISABLE_SESSION_CACHE` — turn off cf_clearance reuse
+- `SCRAPER_CONTENT_TTL_S` — URL→result cache TTL (**0 = off**; set e.g. 86400 to skip re-scraping a page within a day)
+- `SCRAPER_BACKOFF_BASE_MS` / `SCRAPER_BACKOFF_MAX_MS` — exponential backoff between tiers after a rate-limit/timeout (base 0 = off)
+- `SCRAPER_LOGIN_HOOK` — `pkg.module:func` returning `{cookie: value}` for a host (see [Logged-in sessions](#logged-in-sessions))
+- `SCRAPER_EXTRACTION_FILE` — per-domain extraction prefs JSON (default `config/extraction.json`)
+- `SCRAPER_TRACE_SESSION` — opt-in: capture a Playwright trace (screenshots + DOM + network) per browser-tier attempt, written to `state/traces/`
+- `BENCH_FIRECRAWL_USD` / `BENCH_FIRECRAWL_HARD_MULT` — cost model for the savings report
+</details>
+
+### Logged-in sessions
+Beyond a static `SCRAPER_COOKIES_FILE`, wire `SCRAPER_LOGIN_HOOK` to a callable
+`func(host) -> {cookie: value}`. When an authenticated host trips a login/bot
+wall, the engine calls the hook once, persists the returned cookies per host, and
+overlays them on every tier (and future runs), then re-runs that URL on a fresh
+budget. The hook owns the site-specific login mechanics; the engine stays generic.
+
+### Session traces
+With `SCRAPER_TRACE_SESSION=1`, each browser-tier attempt writes a Playwright
+trace zip to `state/traces/`. Manage them over HTTP — `GET /traces` (list),
+`GET /traces/{id}` (download), `DELETE /traces/{id}` — and open one with
+`playwright show-trace <zip>`. Off by default (traces are MBs each).
+
+### Per-domain extraction
+Markdown of the whole page is the default. To scope a site to its content node or
+strip site-specific noise, declare prefs per host in `config/extraction.json`
+(see [config/extraction.example.json](config/extraction.example.json)); every
+tier's normalize step picks them up automatically.
+
+## Contributing
+
+Issues and PRs welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Start with the
+cascade runner in `switchback/orchestrator.py`.
+
+## Responsible use
+
+This engine is for lawful data collection. You are responsible for respecting
+each target site's Terms of Service, `robots.txt`, and rate limits, and for
+having the right to access the content you fetch. The stealth / anti-bot tiers
+(`cloudscraper`, `patchright`, `camoufox`) exist to handle legitimate access
+friction (e.g. generic bot interstitials on public pages) — not to evade access
+controls, paywalls, or authentication you aren't authorized to bypass. The
+software is provided "as is", without warranty (see [LICENSE](LICENSE)).
+
+## License
+
+MIT — see [LICENSE](LICENSE). Third-party dependencies and their licenses are
+listed in [NOTICE](NOTICE); all are permissive (MIT / BSD-3-Clause / Apache-2.0)
+and compatible with this project's MIT license.

switchback-0.1.0.dist-info/RECORD

@@ -0,0 +1,36 @@
+switchback/__init__.py,sha256=a3WwmcxW3YLBDVMPn5xho8sh-VBIUJC34r6zREpMeKk,506
+switchback/__main__.py,sha256=FNhL9UYwA2BpXukEaEzFSbc5H-FHKDb390n6yUYM6mU,97
+switchback/api.py,sha256=h9kx1o3q1vvxTpr5C-0b3lhORCdk3M_84NreyL3be2A,2973
+switchback/concurrency.py,sha256=FYOjHVa9gMpydYK72Nm5t27lmNBRg_Eh507blHKjLzk,1348
+switchback/content_cache.py,sha256=1Svp4PNFGXnywtrJzT4ptKorB9JlpDw9onmXtWPfKpc,3264
+switchback/egress.py,sha256=_3fqqc8TSw9kuoH8o1YzDST9Tv-tbdEGjHGxTPVFWy0,4211
+switchback/extract.py,sha256=dKWj4vHGewHb9PqBhjw-NDOo-aweLItGC1NWltD4-dY,1825
+switchback/flags.py,sha256=R08P2RzRfijdkhvAN-JxqdsxZZZW9STqSLMRfIbLPZI,4012
+switchback/normalize.py,sha256=hbpTmsVgCGiuJtSoV-SpFcZYiH4ndnF_wb_hwzBTKtM,3266
+switchback/orchestrator.py,sha256=DAzBZBoB7CkSrklUWmzowVWaFxZQ-dot3uJN57cpnPw,15915
+switchback/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+switchback/reporting.py,sha256=9do6pqreH7bHpGyQKKu9TqLrxPxR4bbrMlMb2GYB2V4,9218
+switchback/search.py,sha256=Fg2bz2tPvPM_YOCCQE-u-WScvCyNtFdXsmAbvqvraoE,1352
+switchback/server.py,sha256=8pdFg4pWBFQoExMUzEZUcKl5bOTqvEUwMzIt5-r0QgY,3766
+switchback/session_cache.py,sha256=wBvMMLOUw8hriu4N7VbvXo10k6-cdKJ8CKDuMVoCKjQ,9986
+switchback/session_trace.py,sha256=jJTmEyd7iZVAaDgIFyk4vu2mPAI8iWSH_pLjRUOfxg4,3332
+switchback/tracing.py,sha256=5uTpUfYl8a2gIiaNg94Pk8COK143kBxeA1zKJoei73U,5772
+switchback/policy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+switchback/policy/botwall.py,sha256=uZsoerKM0YsC_y_KtyckIufYeqIZhW2c2TMKhLgWc4g,16187
+switchback/policy/gates.py,sha256=z7eeSWdb1Bqdjcz1qHrUtZ4mIgX8-Hms2Mvk9AEP30I,7103
+switchback/tiers/__init__.py,sha256=seqE3NDnc71XVnf3DuklRx1KfqQDVzudPwxofCflHkY,851
+switchback/tiers/_browser.py,sha256=4kaWxh8wkegyKsnE2pwe9LLv7xJG-eSxq5Gjwz53DFI,2195
+switchback/tiers/tier0_apis.py,sha256=dAwTf_qkdYj1rCNVdH5Qn93JwAoGU_VhMT9YryYFTc8,3086
+switchback/tiers/tier1_http.py,sha256=bpDrOY0lUen2sS9sgF3l_oObkcogasKpuYk-OJSNxto,2902
+switchback/tiers/tier2_cloudscraper.py,sha256=weNXrZj61d5iATf7JVfjL5qG9Q-lsup8SSV9fsQoGe8,5522
+switchback/tiers/tier3_browser.py,sha256=oYG_pDvvG79AwDdyhn1a1EV5NnDZfMp1hYgFO6eDZAY,2531
+switchback/tiers/tier3b_camoufox.py,sha256=UFTIeUQ36r7ixQSjqtswbf9ddMfM9xv2fmKEB2tKs4o,3515
+switchback/tiers/tier4_firecrawl.py,sha256=YZ02zx4Adv8xXALJw2C2XgzZAs4r6wJWMDmWeOlngPQ,1556
+switchback/tiers/tier_residential.py,sha256=IlR0d-OYKYWqMOMp0ZLrliIVRsPXn2wCvWgSls-DIZ4,2097
+switchback-0.1.0.dist-info/licenses/LICENSE,sha256=b8V82Q_eJ8JgOap-Zg7JexHZKRF-g1k6ZPfGn2FJXf8,1071
+switchback-0.1.0.dist-info/licenses/NOTICE,sha256=wloHWGC3lw_iV6GM2AfaIOhBQmoTGWF1j3Gin8YNKrM,1461
+switchback-0.1.0.dist-info/METADATA,sha256=JpEJURvlQYbmbrYFYWIUDaJFXN1ZlQP0JjJD-LfSQbM,16400
+switchback-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+switchback-0.1.0.dist-info/entry_points.txt,sha256=Y51hpCqJxN5MaIhTkkBxE6LBs8N6CoIEF3bX58wqbh0,95
+switchback-0.1.0.dist-info/top_level.txt,sha256=ttbyWWHmZeKuLw0aWB5AGSm4DBRjjJXNruNtJzeuSUk,11
+switchback-0.1.0.dist-info/RECORD,,

switchback-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

switchback-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+switchback = switchback.api:_main
+switchback-server = switchback.server:main

switchback-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Akash Kodavuru
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

switchback-0.1.0.dist-info/licenses/NOTICE

@@ -0,0 +1,34 @@
+switchback
+Copyright (c) 2026 Akash Kodavuru
+Licensed under the MIT License (see LICENSE).
+
+This product depends on third-party components, each under its own license.
+They are installed via pip and are NOT bundled or redistributed in this
+repository — no upstream source or full license texts are vendored here. All
+dependencies use permissive licenses (MIT / BSD-3-Clause / Apache-2.0), which
+are compatible with this project's MIT license.
+
+Third-party components
+----------------------
+
+MIT
+  markdownify        HTML -> Markdown normalization
+  beautifulsoup4     HTML parsing / normalization
+  curl_cffi          tier1: plain HTTP with TLS impersonation
+  cloudscraper       tier2: Cloudflare / anti-bot challenge solver
+  camoufox           tier3b: Firefox stealth (optional)
+  firecrawl-py       tier4: paid last-resort scrape API (optional)
+  fastapi            HTTP service (optional)
+
+BSD-3-Clause
+  pypdf              PDF -> text extraction (tier1 PDFs)
+  uvicorn            ASGI server for the HTTP service (optional)
+
+Apache-2.0
+  requests           HTTP client
+  patchright         tier3: stealth headless Chromium (optional)
+  opentelemetry-sdk                            tracing -> any OTLP backend (optional)
+  opentelemetry-exporter-otlp-proto-grpc       OTLP trace/log export (optional)
+
+Full license texts are available with each package distribution (e.g. in its
+`*.dist-info` directory after install) and from each project's repository.

switchback-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+switchback