switchback 0.1.0__py3-none-any.whl

switchback/reporting.py

@@ -0,0 +1,236 @@
+"""Metrics rollups from the engine's own state — no external store needed.
+
+Reads the two files the policy already writes (see switchback.policy.botwall):
+
+  state/botwall_events.jsonl  — one row per *tier attempt*:
+      {ts, url, host, tier, outcome, md_len, latency_ms, error, status_code, challenge}
+  state/botwall_db.json       — per-host record incl. challenge_counts, winning_tier
+
+and produces the metrics the firecrawl-replacement case is argued on:
+
+  • cost savings vs Firecrawl (free-tier wins are money not spent; hard pages
+    Firecrawl bills more credits for are weighted by HARD_MULT)
+  • latency — overall and per tier and per domain (mean/median/min/max/p50/p95)
+  • coverage (unique-URL success rate)
+  • error codes by domain
+  • challenges / bot-walls by domain
+
+Pure functions returning JSON-serialisable dicts, so the same rollup backs the
+CLI (scrape_stats.py), the HTTP API (/metrics), and the periodic digest
+(switchback.flags). One tier attempt ≈ sequential wall-clock, so a URL's total
+latency is approximated by summing its attempts' latencies.
+"""
+from __future__ import annotations
+
+import json
+import os
+from collections import defaultdict
+from datetime import datetime, timezone
+from statistics import mean, median
+
+from .policy import botwall
+
+# Cost model (estimates — override to your real Firecrawl rates).
+#   FIRECRAWL_USD   $ per basic Firecrawl scrape (matches benchmark.py's default)
+#   HARD_MULT       credit multiplier Firecrawl charges for stealth/JS-rendered
+#                   pages — the hard ones our browser/residential tiers resolve
+#                   for free. Firecrawl's stealth proxy is ~5× basic, hence 5.
+FIRECRAWL_USD = float(os.getenv("BENCH_FIRECRAWL_USD", "0.001"))
+HARD_MULT = float(os.getenv("BENCH_FIRECRAWL_HARD_MULT", "5"))
+
+# Tiers whose win means Firecrawl would have billed the hard (stealth) rate.
+_HARD_TIERS = {"tier3_browser", "tier3b_camoufox", "tier_residential", "tier4_firecrawl"}
+
+
+def _parse_ts(ts: str) -> datetime | None:
+    try:
+        dt = datetime.fromisoformat(ts)
+        return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
+    except (ValueError, TypeError):
+        return None
+
+
+def _percentile(values: list[int], p: float) -> int:
+    if not values:
+        return 0
+    s = sorted(values)
+    idx = max(0, min(len(s) - 1, int(len(s) * p / 100)))
+    return s[idx]
+
+
+def _stats(values: list[int]) -> dict:
+    """min/max/mean/median/p50/p95 over a list of latencies (ms)."""
+    if not values:
+        return {"count": 0, "min": 0, "max": 0, "mean": 0, "median": 0, "p50": 0, "p95": 0}
+    return {
+        "count": len(values),
+        "min": min(values),
+        "max": max(values),
+        "mean": round(mean(values)),
+        "median": round(median(values)),
+        "p50": _percentile(values, 50),
+        "p95": _percentile(values, 95),
+    }
+
+
+def load_events(path: str | None = None, since: datetime | None = None) -> list[dict]:
+    """Read botwall_events.jsonl (optionally only rows at/after `since`)."""
+    path = path or botwall.EVENTS_PATH
+    if not os.path.exists(path):
+        return []
+    out: list[dict] = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                e = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if since is not None:
+                dt = _parse_ts(e.get("ts", ""))
+                if dt is None or dt < since:
+                    continue
+            out.append(e)
+    return out
+
+
+def _by_url(events: list[dict]) -> dict[str, list[dict]]:
+    out: dict[str, list[dict]] = defaultdict(list)
+    for e in events:
+        out[e.get("url") or "?"].append(e)
+    return out
+
+
+def _url_total_ms(attempts: list[dict]) -> int:
+    """Per-URL wall-clock proxy: tiers run sequentially, so sum the attempts."""
+    return sum(a["latency_ms"] for a in attempts if isinstance(a.get("latency_ms"), int))
+
+
+def _coverage(by_url: dict[str, list[dict]]) -> dict:
+    succeeded = {u: a for u, a in by_url.items()
+                 if any(e.get("outcome") == "ok" for e in a)}
+    total = len(by_url)
+    return {
+        "unique_urls": total,
+        "succeeded": len(succeeded),
+        "failed": total - len(succeeded),
+        "success_pct": round(100 * len(succeeded) / total, 1) if total else 0.0,
+    }
+
+
+def _per_tier(events: list[dict]) -> dict:
+    tiers: dict[str, list[dict]] = defaultdict(list)
+    for e in events:
+        tiers[e.get("tier") or "?"].append(e)
+    out = {}
+    for tier, evs in tiers.items():
+        ok = [e for e in evs if e.get("outcome") == "ok"]
+        lats = [e["latency_ms"] for e in ok if isinstance(e.get("latency_ms"), int)]
+        out[tier] = {
+            "attempts": len(evs),
+            "ok": len(ok),
+            "miss": len(evs) - len(ok),
+            "ok_pct": round(100 * len(ok) / len(evs), 1) if evs else 0.0,
+            "latency_ms": _stats(lats),
+        }
+    return out
+
+
+def _outcomes(events: list[dict]) -> dict:
+    counts: dict[str, int] = defaultdict(int)
+    for e in events:
+        counts[e.get("outcome") or "?"] += 1
+    return dict(sorted(counts.items(), key=lambda kv: -kv[1]))
+
+
+def _cost(by_url: dict[str, list[dict]]) -> dict:
+    """Engine spend vs the Firecrawl-everything baseline curiouscats pays today.
+
+    For each unique URL:
+      engine_cost           = FIRECRAWL_USD per actual Firecrawl invocation
+      firecrawl_equivalent  = what scraping that URL via Firecrawl would cost —
+                              HARD_MULT× when it needed a hard tier (Firecrawl
+                              bills stealth/JS pages more), else 1×.
+    savings = baseline − engine. Failed URLs still count toward the baseline
+    (Firecrawl would have been billed for the attempt too)."""
+    engine = 0.0
+    baseline = 0.0
+    firecrawl_calls = 0
+    for attempts in by_url.values():
+        used_fc = sum(1 for a in attempts if a.get("outcome") == "firecrawl_used")
+        firecrawl_calls += used_fc
+        engine += used_fc * FIRECRAWL_USD
+        hard = any((a.get("tier") in _HARD_TIERS) or a.get("challenge") for a in attempts)
+        baseline += FIRECRAWL_USD * (HARD_MULT if hard else 1)
+    return {
+        "firecrawl_usd_per_scrape": FIRECRAWL_USD,
+        "hard_multiplier": HARD_MULT,
+        "firecrawl_invocations": firecrawl_calls,
+        "engine_cost_usd": round(engine, 4),
+        "firecrawl_baseline_usd": round(baseline, 4),
+        "savings_usd": round(baseline - engine, 4),
+        "savings_pct": round(100 * (baseline - engine) / baseline, 1) if baseline else 0.0,
+    }
+
+
+def _domains(events: list[dict], by_url: dict[str, list[dict]], db: dict) -> dict:
+    """Per-host (== per-subdomain) rollup: attempts, error codes, challenges,
+    latency. Challenges come from the durable per-host counts in botwall_db.json
+    so they reflect all history, not just the events window."""
+    host_attempts: dict[str, list[dict]] = defaultdict(list)
+    for e in events:
+        host_attempts[e.get("host") or "?"].append(e)
+
+    # Per-URL total latency grouped by host (host taken from the URL's events).
+    host_url_ms: dict[str, list[int]] = defaultdict(list)
+    for attempts in by_url.values():
+        host = next((a.get("host") for a in attempts if a.get("host")), "?")
+        host_url_ms[host].append(_url_total_ms(attempts))
+
+    hosts_db = db.get("hosts", {})
+    out = {}
+    for host, evs in host_attempts.items():
+        error_codes: dict[str, int] = defaultdict(int)
+        for e in evs:
+            sc = e.get("status_code")
+            if sc:
+                error_codes[str(sc)] += 1
+        rec = hosts_db.get(host, {})
+        out[host] = {
+            "attempts": len(evs),
+            "ok": sum(1 for e in evs if e.get("outcome") == "ok"),
+            "winning_tier": rec.get("winning_tier"),
+            "needs_egress": bool(rec.get("needs_egress")),
+            "error_codes": dict(error_codes),
+            "challenges": dict(rec.get("challenge_counts", {})),
+            "latency_ms": _stats(host_url_ms.get(host, [])),
+        }
+    return dict(sorted(out.items(), key=lambda kv: -kv[1]["attempts"]))
+
+
+def build_report(events: list[dict] | None = None, db: dict | None = None,
+                 since: datetime | None = None) -> dict:
+    """Full metrics rollup. Reads state files when args are omitted."""
+    if events is None:
+        events = load_events(since=since)
+    if db is None:
+        db = botwall.load_db()
+    by_url = _by_url(events)
+    url_totals = [_url_total_ms(a) for a in by_url.values()]
+    return {
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "events": len(events),
+        "coverage": _coverage(by_url),
+        "cost": _cost(by_url),
+        "latency_overall_ms": _stats(url_totals),
+        "latency_per_tier": _per_tier(events),
+        "outcomes": _outcomes(events),
+        "domains": _domains(events, by_url, db),
+    }
+
+
+def domain_report(since: datetime | None = None) -> dict:
+    """Just the per-domain table (error codes + challenges + latency per host)."""
+    return build_report(since=since)["domains"]

switchback/search.py

@@ -0,0 +1,39 @@
+"""Web search via a local SearXNG instance (query → ranked URLs).
+
+Distinct from the scrape cascade: scrape fetches one *known* URL; search
+*discovers* URLs from a query. Returns lightweight results a caller can then
+feed into scrape(). Points at the SearXNG container on localhost:8888 (override
+with SEARXNG_URL); requires its JSON format to be enabled.
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from .tracing import span
+
+SEARXNG_URL = os.getenv("SEARXNG_URL", "http://localhost:8888")
+
+
+@dataclass
+class SearchResult:
+    title: str
+    url: str
+    snippet: str
+    engine: str
+
+
+def search(query: str, n: int = 10) -> list[SearchResult]:
+    """Query the local SearXNG and return up to n results."""
+    import requests
+    with span("search", **{"search.query": query, "search.n": n}) as sp:
+        r = requests.get(f"{SEARXNG_URL}/search",
+                         params={"q": query, "format": "json"}, timeout=15)
+        r.raise_for_status()
+        raw = r.json().get("results", [])[:n]
+        results = [SearchResult(title=x.get("title", ""), url=x.get("url", ""),
+                                snippet=x.get("content", ""),
+                                engine=x.get("engine", ""))
+                   for x in raw]
+        sp.set("search.n_results", len(results))
+        return results

switchback/server.py

@@ -0,0 +1,114 @@
+"""HTTP service — the language-agnostic way to use the engine.
+
+Wraps the same public functions as the library/CLI (`switchback.scrape`,
+`switchback.search`) so any app, in any language, can hit one warm process (which
+keeps the Tier-3 browser pool hot) instead of cold-starting a subprocess.
+
+    pip install -e .[server]        # fastapi + uvicorn
+    switchback-server           # or: python -m switchback.server
+    curl localhost:8799/healthz
+    curl -s localhost:8799/scrape -d '{"urls":["https://example.com"]}'
+    curl 'localhost:8799/search?q=web+scraping'
+
+Local-first by design: no auth, no rate-limiting. Put it behind your own gateway
+if you expose it. Host/port via SCRAPER_HOST / SCRAPER_PORT.
+"""
+from __future__ import annotations
+
+import os
+
+from datetime import datetime, timedelta, timezone
+
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+
+from . import session_trace
+from .api import scrape
+from .reporting import build_report, domain_report
+from .search import search
+from .tracing import setup_logs
+
+app = FastAPI(title="switchback", version="0.1.0")
+
+
+class ScrapeRequest(BaseModel):
+    urls: list[str]
+
+
+@app.get("/healthz")
+def healthz() -> dict:
+    return {"status": "ok"}
+
+
+@app.post("/scrape")
+def scrape_endpoint(req: ScrapeRequest) -> list[dict]:
+    """Run URLs through the cascade. Returns successes only (failed URLs omitted)."""
+    return [{"url": r.url, "source_method": r.source_method, "markdown": r.markdown}
+            for r in scrape(req.urls)]
+
+
+@app.get("/search")
+def search_endpoint(q: str) -> list[dict]:
+    """Query → ranked URLs (SearXNG). Feed results back into /scrape."""
+    return [{"title": h.title, "url": h.url, "snippet": h.snippet, "engine": h.engine}
+            for h in search(q)]
+
+
+def _since(minutes: int | None) -> datetime | None:
+    return datetime.now(timezone.utc) - timedelta(minutes=minutes) if minutes else None
+
+
+@app.get("/metrics")
+def metrics_endpoint(minutes: int | None = None) -> dict:
+    """Metrics rollup from the engine's own state: cost savings vs Firecrawl,
+    coverage, overall + per-tier latency, outcomes, and per-domain detail.
+    Pass ?minutes=N to window the event-derived sections."""
+    return build_report(since=_since(minutes))
+
+
+@app.get("/metrics/domains")
+def metrics_domains_endpoint(minutes: int | None = None) -> dict:
+    """Per-domain table: error codes, challenges/bot-walls, and latency by host."""
+    return domain_report(since=_since(minutes))
+
+
+@app.get("/traces")
+def list_traces_endpoint() -> list[dict]:
+    """Captured Playwright session traces (opt-in via SCRAPER_TRACE_SESSION)."""
+    return session_trace.list_traces()
+
+
+@app.get("/traces/{trace_id}")
+def get_trace_endpoint(trace_id: str):
+    """Download one trace zip (open with `playwright show-trace <zip>`)."""
+    path = session_trace.path_for(trace_id)
+    if not path:
+        raise HTTPException(status_code=404, detail="trace not found")
+    return FileResponse(path, media_type="application/zip",
+                        filename=f"{trace_id}.zip")
+
+
+@app.delete("/traces/{trace_id}")
+def delete_trace_endpoint(trace_id: str) -> dict:
+    """Delete a trace zip."""
+    if not session_trace.delete(trace_id):
+        raise HTTPException(status_code=404, detail="trace not found")
+    return {"deleted": trace_id}
+
+
+def main() -> None:
+    import logging
+    import uvicorn
+
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    setup_logs()  # ship logs to the OTLP backend too when configured
+    uvicorn.run(
+        app,
+        host=os.getenv("SCRAPER_HOST", "0.0.0.0"),
+        port=int(os.getenv("SCRAPER_PORT", "8799")),
+    )
+
+
+if __name__ == "__main__":
+    main()

switchback/session_cache.py

@@ -0,0 +1,274 @@
+"""Cookie provisioning for the HTTP/browser tiers — two sources, merged per hit.
+
+1. Session cache (cf_clearance reuse). After Tier 2 solves a Cloudflare
+   challenge its ``cf_*`` cookies are cached per ``(host, egress_scope)`` and
+   replayed on later hits so the ~5s solve is skipped. cf_clearance is bound to
+   the UA that solved it and the egress IP — hence the scope key and the stored
+   UA. Entries expire after ``SCRAPER_SESSION_TTL_S`` (cf_clearance's typical
+   lifetime); a re-detected wall calls ``forget()`` so a stale cookie self-heals
+   into a fresh solve. Disable with ``SCRAPER_DISABLE_SESSION_CACHE=1``.
+
+2. Auth import (``SCRAPER_COOKIES_FILE``). A Netscape ``cookies.txt`` the user
+   exports from a logged-in browser; domain-matching cookies are sent so the
+   tiers can fetch pages behind a login. Opt-in; unset/absent → no-op.
+
+The cf cache layers on top of auth cookies for the same request. Reads are in
+memory; writes are write-through to ``state/session_cache.json`` (atomic).
+"""
+from __future__ import annotations
+
+import importlib
+import json
+import logging
+import os
+import threading
+import time
+from http.cookiejar import MozillaCookieJar
+from urllib.parse import urlsplit
+
+from . import egress
+from .policy.gates import host_of
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_STATE_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "state")
+_STATE_DIR = os.getenv("SCRAPER_STATE_DIR", _DEFAULT_STATE_DIR)
+CACHE_PATH = os.path.join(_STATE_DIR, "session_cache.json")
+
+_TTL_S = float(os.getenv("SCRAPER_SESSION_TTL_S", "1800"))
+
+# cf_clearance is the durable one; __cf_bm / cf_chl_* are the supporting set.
+def _is_cf_cookie(name: str) -> bool:
+    return name == "cf_clearance" or name.startswith("__cf") or name.startswith("cf_chl")
+
+
+def _enabled() -> bool:
+    return os.getenv("SCRAPER_DISABLE_SESSION_CACHE") not in ("1", "true", "True")
+
+
+# ── cf_clearance session cache ────────────────────────────────────────────────
+
+_LOCK = threading.Lock()
+_DB: dict | None = None  # {"version": 1, "entries": {"<host>\t<scope>": {...}}}
+
+
+def _load() -> dict:
+    global _DB
+    if _DB is not None:
+        return _DB
+    db = {"version": 1, "entries": {}}
+    if os.path.exists(CACHE_PATH):
+        try:
+            with open(CACHE_PATH) as f:
+                db = json.load(f)
+            db.setdefault("entries", {})
+        except Exception as e:
+            logger.error(f"session_cache: load failed ({e}); starting fresh")
+    _DB = db
+    return _DB
+
+
+def _save(db: dict) -> None:
+    os.makedirs(_STATE_DIR, exist_ok=True)
+    tmp = f"{CACHE_PATH}.tmp.{os.getpid()}.{threading.get_ident()}"
+    with open(tmp, "w") as f:
+        json.dump(db, f, indent=2, sort_keys=True)
+    os.replace(tmp, CACHE_PATH)
+
+
+def _key(url: str) -> str:
+    return f"{host_of(url)}\t{egress.scope_label()}"
+
+
+def _cache_cookies(url: str) -> dict:
+    """Fresh cached cf cookies for this host+scope, or {} (also lazily evicts
+    expired entries)."""
+    if not _enabled():
+        return {}
+    with _LOCK:
+        db = _load()
+        key = _key(url)
+        ent = db["entries"].get(key)
+        if not ent:
+            return {}
+        if time.time() - ent.get("ts", 0) > _TTL_S:
+            db["entries"].pop(key, None)
+            _save(db)
+            return {}
+        return dict(ent.get("cookies", {}))
+
+
+def remember(url: str, cookies: dict, ua: str = "") -> None:
+    """Persist the cf cookies from a successful solve for this host+scope."""
+    if not _enabled():
+        return
+    cf = {k: v for k, v in (cookies or {}).items() if _is_cf_cookie(k)}
+    if not cf:
+        return
+    with _LOCK:
+        db = _load()
+        db["entries"][_key(url)] = {"cookies": cf, "ua": ua, "ts": time.time()}
+        _save(db)
+
+
+def forget(url: str) -> None:
+    """Drop the cached entry for this host+scope (a re-detected wall means the
+    cookie is stale or IP-mismatched)."""
+    if not _enabled():
+        return
+    with _LOCK:
+        db = _load()
+        if db["entries"].pop(_key(url), None) is not None:
+            _save(db)
+
+
+# ── auth cookie import (SCRAPER_COOKIES_FILE) ────────────────────────────────
+
+_AUTH_LOCK = threading.Lock()
+_AUTH_JAR: MozillaCookieJar | None = None
+_AUTH_LOADED = False
+
+
+def _auth_jar() -> MozillaCookieJar | None:
+    global _AUTH_JAR, _AUTH_LOADED
+    if _AUTH_LOADED:
+        return _AUTH_JAR
+    with _AUTH_LOCK:
+        if _AUTH_LOADED:
+            return _AUTH_JAR
+        path = os.getenv("SCRAPER_COOKIES_FILE")
+        if path and os.path.exists(path):
+            jar = MozillaCookieJar()
+            try:
+                jar.load(path, ignore_discard=True, ignore_expires=True)
+                _AUTH_JAR = jar
+                logger.info(f"session_cache: loaded auth cookies from {path}")
+            except Exception as e:
+                logger.error(f"session_cache: cookie import failed ({e})")
+        _AUTH_LOADED = True
+    return _AUTH_JAR
+
+
+def _host_matches(cookie_domain: str, host: str) -> bool:
+    d = cookie_domain.lstrip(".")
+    return host == d or host.endswith("." + d)
+
+
+def _auth_cookies(url: str) -> dict:
+    jar = _auth_jar()
+    if not jar:
+        return {}
+    host = host_of(url)
+    return {c.name: c.value for c in jar if _host_matches(c.domain, host)}
+
+
+# ── logged-in session refresh (SCRAPER_LOGIN_HOOK) ───────────────────────────
+#
+# A cookies.txt export goes stale. Configure SCRAPER_LOGIN_HOOK="pkg.module:func"
+# — a callable func(host) -> {cookie_name: value}. When an authed host trips a
+# login / bot wall, the engine calls the hook once, persists the returned cookies
+# per host (in the session cache under "logins"), and overlays them on the
+# cookies.txt jar for every later request and run. The hook owns the site-specific
+# mechanics (drive a browser, hit an auth API, read a secret); the engine stays
+# generic, which is what lets it cover many/varied logged-in sites.
+
+_LOGIN_LOCK = threading.Lock()
+_LOGIN_HOOK = None
+_LOGIN_HOOK_LOADED = False
+
+
+def _login_hook():
+    global _LOGIN_HOOK, _LOGIN_HOOK_LOADED
+    if _LOGIN_HOOK_LOADED:
+        return _LOGIN_HOOK
+    with _LOGIN_LOCK:
+        if not _LOGIN_HOOK_LOADED:
+            spec = os.getenv("SCRAPER_LOGIN_HOOK", "")
+            if spec and ":" in spec:
+                mod, _, fn = spec.partition(":")
+                try:
+                    _LOGIN_HOOK = getattr(importlib.import_module(mod), fn)
+                    logger.info(f"session_cache: login hook loaded ({spec})")
+                except Exception as e:
+                    logger.error(f"session_cache: login hook {spec!r} load failed: {e}")
+            _LOGIN_HOOK_LOADED = True
+    return _LOGIN_HOOK
+
+
+def has_login_hook() -> bool:
+    return _login_hook() is not None
+
+
+def _login_cookies(url: str) -> dict:
+    """Refreshed login cookies stored for this host (host-level, scope-agnostic)."""
+    with _LOCK:
+        db = _load()
+        ent = db.get("logins", {}).get(host_of(url))
+    return dict(ent.get("cookies", {})) if ent else {}
+
+
+def is_authed_host(url: str) -> bool:
+    """True when we hold a logged-in credential for this host — an imported
+    cookies.txt match or previously-refreshed login cookies. Lets the policy tell
+    a dead session (worth re-logging-in for) from a plain bot wall."""
+    return bool(_auth_cookies(url) or _login_cookies(url))
+
+
+def refresh_login(url: str) -> bool:
+    """Invoke the configured login hook for this host and persist the cookies it
+    returns. True if fresh cookies were obtained; no-op without a hook."""
+    hook = _login_hook()
+    if not hook:
+        return False
+    host = host_of(url)
+    try:
+        cookies = hook(host) or {}
+    except Exception as e:
+        logger.error(f"session_cache: login hook failed for {host}: {e}")
+        return False
+    if not cookies:
+        logger.warning(f"session_cache: login hook returned no cookies for {host}")
+        return False
+    with _LOCK:
+        db = _load()
+        db.setdefault("logins", {})[host] = {"cookies": dict(cookies), "ts": time.time()}
+        _save(db)
+    logger.info(f"session_cache: refreshed login for {host} ({len(cookies)} cookies)")
+    return True
+
+
+# ── public API used by the tiers ─────────────────────────────────────────────
+
+def cookies_for(url: str, *, include_cache: bool) -> dict:
+    """Cookies to send for this URL: imported auth, overlaid by refreshed login
+    cookies, plus (when requested) the cached cf_clearance. Freshest wins on a
+    name clash (auth file < refreshed login < cf cache)."""
+    out = _auth_cookies(url)
+    out.update(_login_cookies(url))
+    if include_cache:
+        out.update(_cache_cookies(url))
+    return out
+
+
+def cookie_header(url: str, *, include_cache: bool) -> str | None:
+    """The same cookies as a ``Cookie:`` header value (for curl_cffi, which has
+    no cookies= kwarg), or None when there are none."""
+    c = cookies_for(url, include_cache=include_cache)
+    return "; ".join(f"{k}={v}" for k, v in c.items()) if c else None
+
+
+def browser_cookies(url: str | None = None) -> list[dict]:
+    """Auth + refreshed-login cookies as Playwright ``add_cookies`` records.
+    cf_clearance is not replayed into browsers — they solve natively. Refreshed
+    login cookies are added for the URL's host when a URL is given."""
+    out = []
+    jar = _auth_jar()
+    if jar:
+        out.extend({"name": c.name, "value": c.value,
+                    "domain": c.domain, "path": c.path or "/"} for c in jar)
+    if url:
+        host = host_of(url)
+        out.extend({"name": k, "value": v, "domain": host, "path": "/"}
+                   for k, v in _login_cookies(url).items())
+    return out