dataapp_crawler 0.1.0__py3-none-any.whl

dataapp_crawler/cloudflare.py

@@ -0,0 +1,523 @@
+"""Explicit Cloudflare Challenge parameter capture for an existing browser Page.
+
+This module never navigates, refreshes, contacts a solver, or claims that a
+target website accepted a solution. Install the capture before navigation,
+send the returned task request through the managed resource client, then apply
+the returned token to the exact same page and challenge generation.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import math
+import time
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+from urllib.parse import urlsplit, urlunsplit
+
+from .errors import ConfigurationError, FetchError
+
+_STATE = "__dataappCloudflareChallengeCapture"
+_MAX_CAPTURE_AGE_SECONDS = 180.0
+
+_INSTALL_SCRIPT = f"""
+(() => {{
+  if (window.top !== window || window[{_STATE!r}]?.version === 1) return;
+  const state = {{
+    version: 1,
+    documentId: `${{Date.now()}}-${{Math.random().toString(36).slice(2)}}`,
+    generation: 0,
+    challenges: [],
+    invalidCallbackObserved: false,
+    wrappedObjects: new WeakMap()
+  }};
+  Object.defineProperty(window, {_STATE!r}, {{
+    value: state, configurable: false, enumerable: false, writable: false
+  }});
+
+  const trustedApiUrl = () => {{
+    const values = [];
+    for (const script of Array.from(document.scripts || [])) values.push(script.src || "");
+    try {{
+      for (const entry of performance.getEntriesByType("resource") || []) {{
+        values.push(entry.name || "");
+      }}
+    }} catch (_) {{}}
+    return values.find(value => {{
+      try {{
+        const url = new URL(value, location.href);
+        return url.protocol === "https:" && url.hostname === "challenges.cloudflare.com"
+          && url.pathname.startsWith("/turnstile/")
+          && url.username === "" && url.password === "" && url.hash === "";
+      }} catch (_) {{ return false; }}
+    }}) || null;
+  }};
+
+  const invalidate = (items, widgetId) => {{
+    for (const item of items) {{
+      if (widgetId === undefined || widgetId === null || item.widgetId === widgetId) {{
+        item.active = false;
+      }}
+    }}
+  }};
+
+  const wrap = value => {{
+    if (!value || (typeof value !== "object" && typeof value !== "function")) return value;
+    if (state.wrappedObjects.has(value)) return state.wrappedObjects.get(value);
+    const proxy = new Proxy(value, {{
+      get(target, property, receiver) {{
+        const original = Reflect.get(target, property, receiver);
+        if (property === "render" && typeof original === "function") {{
+          return function(element, params, ...rest) {{
+            const challengeLike = params && typeof params === "object"
+              && typeof params.sitekey === "string" && params.sitekey.length > 0
+              && typeof params.action === "string" && params.action.length > 0
+              && typeof params.cData === "string" && params.cData.length > 0
+              && typeof params.chlPageData === "string" && params.chlPageData.length > 0;
+            if (!challengeLike) return Reflect.apply(original, target, [element, params, ...rest]);
+            if (typeof params.callback !== "function") {{
+              state.invalidCallbackObserved = true;
+              return Reflect.apply(original, target, [element, params, ...rest]);
+            }}
+            const generation = ++state.generation;
+            const captureId = `${{state.documentId}}-${{generation}}`;
+            const item = {{
+              captureId, generation, capturedAtMs: Date.now(), websiteUrl: location.href,
+              websiteKey: params.sitekey, action: params.action, data: params.cData,
+              pageData: params.chlPageData, userAgent: navigator.userAgent,
+              apiJsUrl: trustedApiUrl(), callback: params.callback,
+              callbackInvoked: false, active: true, widgetId: null
+            }};
+            const forwarded = Object.assign({{}}, params, {{
+              callback: function(token, ...callbackRest) {{
+                item.callbackInvoked = true;
+                return Reflect.apply(item.callback, this, [token, ...callbackRest]);
+              }}
+            }});
+            const widgetId = Reflect.apply(original, target, [element, forwarded, ...rest]);
+            item.widgetId = widgetId;
+            state.challenges.push(item);
+            return widgetId;
+          }};
+        }}
+        if ((property === "reset" || property === "remove") && typeof original === "function") {{
+          return function(widgetId, ...rest) {{
+            const existing = Array.from(state.challenges);
+            const result = Reflect.apply(original, target, [widgetId, ...rest]);
+            invalidate(existing, widgetId);
+            return result;
+          }};
+        }}
+        return original;
+      }}
+    }});
+    state.wrappedObjects.set(value, proxy);
+    state.wrappedObjects.set(proxy, proxy);
+    return proxy;
+  }};
+
+  const attach = () => {{
+    let current = window.turnstile;
+    if (!current || typeof current.render !== "function") return;
+    current = wrap(current);
+    try {{
+      Object.defineProperty(window, "turnstile", {{
+        configurable: true,
+        enumerable: true,
+        get: () => current,
+        set: value => {{ current = wrap(value); }}
+      }});
+    }} catch (_) {{
+      try {{ window.turnstile = current; }} catch (_) {{}}
+    }}
+  }};
+  // 先创建 undefined 属性会触发供应商的 "turnstile" in window 重复加载判定。
+  // 外链脚本 load 在其排队的 onload 回调前触发，此时真实 API 已完成导出。
+  document.addEventListener("load", event => {{
+    if (event.target?.tagName === "SCRIPT") attach();
+  }}, true);
+  attach();
+}})();
+"""
+
+_READ_SCRIPT = f"""
+() => {{
+  const state = window[{_STATE!r}];
+  if (!state || state.version !== 1) return {{status: "not_installed"}};
+  if (state.invalidCallbackObserved) return {{status: "invalid_callback"}};
+  const active = state.challenges.filter(item => item.active && !item.callbackInvoked);
+  if (active.length > 1) return {{status: "ambiguous"}};
+  if (active.length === 0) return {{status: "waiting"}};
+  const item = active[0];
+  if (Date.now() - item.capturedAtMs > {int(_MAX_CAPTURE_AGE_SECONDS * 1000)}) {{
+    item.active = false;
+    return {{status: "expired"}};
+  }}
+  return {{
+    status: "ready", document_id: state.documentId, capture_id: item.captureId,
+    generation: item.generation, captured_at_ms: item.capturedAtMs,
+    website_url: item.websiteUrl, website_key: item.websiteKey, action: item.action,
+    data: item.data, page_data: item.pageData, user_agent: item.userAgent,
+    api_js_url: item.apiJsUrl
+  }};
+}}
+"""
+
+_APPLY_SCRIPT = f"""
+input => {{
+  const state = window[{_STATE!r}];
+  if (!state || state.version !== 1 || state.documentId !== input.document_id)
+    return {{applied: false, reason: "document_changed"}};
+  const active = state.challenges.filter(item => item.active && !item.callbackInvoked);
+  if (active.length !== 1) return {{applied: false, reason: active.length ? "ambiguous" : "stale"}};
+  const item = active[0];
+  if (item.captureId !== input.capture_id || item.generation !== input.generation)
+    return {{applied: false, reason: "challenge_changed"}};
+  if (Date.now() - item.capturedAtMs > input.max_age_ms)
+    return {{applied: false, reason: "expired"}};
+  if (navigator.userAgent !== input.user_agent || item.userAgent !== input.user_agent)
+    return {{applied: false, reason: "user_agent_changed"}};
+  if (typeof item.callback !== "function")
+    return {{applied: false, reason: "callback_unavailable"}};
+  item.callbackInvoked = true;
+  Reflect.apply(item.callback, window, [input.token]);
+  return {{applied: true}};
+}}
+"""
+
+
+class _Page(Protocol):
+    async def add_init_script(self, script: str) -> Any: ...
+
+    async def evaluate(self, expression: str, arg: Any = None) -> Any: ...
+
+
+@dataclass(frozen=True, slots=True)
+class CloudflareChallengeParameters:
+    """Current full-page Challenge inputs; representations hide all page values."""
+
+    website_url: str = field(repr=False)
+    website_key: str = field(repr=False)
+    action: str = field(repr=False)
+    data: str = field(repr=False)
+    page_data: str = field(repr=False)
+    user_agent: str = field(repr=False)
+    api_js_url: str | None = field(default=None, repr=False)
+    _document_id: str = field(default="", repr=False)
+    _capture_id: str = field(default="", repr=False)
+    _generation: int = field(default=0, repr=False)
+    _captured_at_ms: int = field(default=0, repr=False)
+
+    def as_task_request(self) -> dict[str, str]:
+        """Return the managed token-mode request without internal page binding fields."""
+
+        request = {
+            "type": "cloudflare_challenge",
+            "website_url": self.website_url,
+            "website_key": self.website_key,
+            "action": self.action,
+            "data": self.data,
+            "page_data": self.page_data,
+            "user_agent": self.user_agent,
+        }
+        if self.api_js_url is not None:
+            request["api_js_url"] = self.api_js_url
+        return request
+
+
+class CloudflareChallengeCapture:
+    """Capture and apply one dynamic full-page Cloudflare Challenge generation."""
+
+    def __init__(self, page: _Page, timeout: float) -> None:
+        self._page = page
+        self._timeout = timeout
+
+    @classmethod
+    async def install(cls, page: _Page, timeout: float = 5.0) -> CloudflareChallengeCapture:
+        """Install before navigation; no navigation or solver call is performed."""
+
+        checked = _timeout(timeout, maximum=30.0)
+        try:
+            await asyncio.wait_for(page.add_init_script(_INSTALL_SCRIPT), timeout=checked)
+            await asyncio.wait_for(page.evaluate(_INSTALL_SCRIPT), timeout=checked)
+        except asyncio.CancelledError:
+            raise
+        except TimeoutError:
+            raise FetchError(
+                "Cloudflare Challenge capture installation timed out",
+                phase="cloudflare_challenge_capture",
+            ) from None
+        except Exception:
+            raise FetchError(
+                "Cloudflare Challenge capture installation failed",
+                phase="cloudflare_challenge_capture",
+            ) from None
+        return cls(page, checked)
+
+    async def wait(self, timeout: float = 30.0) -> CloudflareChallengeParameters:
+        """Wait for one unambiguous render without refreshing or mutating the page."""
+
+        wait_timeout = _timeout(timeout, maximum=180.0)
+        deadline = time.monotonic() + wait_timeout
+        last_read_unavailable = False
+        while True:
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                raise FetchError(
+                    (
+                        "Cloudflare Challenge page remained unavailable until capture timeout"
+                        if last_read_unavailable
+                        else "Cloudflare Challenge parameters were not captured"
+                    ),
+                    phase="cloudflare_challenge_capture",
+                )
+            try:
+                raw = await asyncio.wait_for(
+                    self._page.evaluate(_READ_SCRIPT),
+                    timeout=min(self._timeout, remaining),
+                )
+            except asyncio.CancelledError:
+                raise
+            except TimeoutError:
+                last_read_unavailable = True
+                await asyncio.sleep(min(0.1, max(0.0, deadline - time.monotonic())))
+                continue
+            except Exception as exc:
+                if _is_transient_context_error(exc):
+                    last_read_unavailable = True
+                    await asyncio.sleep(min(0.1, max(0.0, deadline - time.monotonic())))
+                    continue
+                raise FetchError(
+                    "Cloudflare Challenge page read failed",
+                    phase="cloudflare_challenge_capture",
+                ) from None
+            last_read_unavailable = False
+            if not isinstance(raw, dict) or not isinstance(raw.get("status"), str):
+                raise FetchError(
+                    "Cloudflare Challenge capture returned invalid data",
+                    phase="cloudflare_challenge_capture",
+                )
+            status = raw["status"]
+            if status == "ready":
+                return _parameters(raw)
+            if status == "invalid_callback":
+                raise FetchError(
+                    "Cloudflare Challenge callback is not callable",
+                    phase="cloudflare_challenge_capture",
+                )
+            if status == "ambiguous":
+                raise FetchError(
+                    "Multiple Cloudflare Challenges are active",
+                    phase="cloudflare_challenge_capture",
+                )
+            if status == "expired":
+                raise FetchError(
+                    "Cloudflare Challenge parameters expired before solver submission",
+                    phase="cloudflare_challenge_capture",
+                )
+            if status == "not_installed":
+                raise FetchError(
+                    "Cloudflare Challenge capture is not installed in the current document",
+                    phase="cloudflare_challenge_capture",
+                )
+            if status != "waiting":
+                raise FetchError(
+                    "Cloudflare Challenge capture returned an invalid status",
+                    phase="cloudflare_challenge_capture",
+                )
+            remaining = deadline - time.monotonic()
+            await asyncio.sleep(min(0.1, remaining))
+
+    async def apply_token(
+        self,
+        token: str,
+        *,
+        challenge: CloudflareChallengeParameters,
+        user_agent: str | None = None,
+        timeout: float = 5.0,
+    ) -> bool:
+        """Invoke the original page callback; true does not prove site acceptance."""
+
+        checked = _timeout(timeout, maximum=30.0)
+        if not isinstance(challenge, CloudflareChallengeParameters):
+            raise ConfigurationError("challenge must be CloudflareChallengeParameters")
+        _bounded_text("token", token, maximum=8192)
+        if user_agent is None:
+            user_agent = challenge.user_agent
+        _bounded_text("user_agent", user_agent, maximum=512)
+        if user_agent != challenge.user_agent:
+            raise FetchError(
+                "Cloudflare Challenge solution user agent does not match the browser",
+                phase="cloudflare_challenge_apply",
+            )
+        payload = {
+            "token": token,
+            "user_agent": user_agent,
+            "document_id": challenge._document_id,
+            "capture_id": challenge._capture_id,
+            "generation": challenge._generation,
+            "max_age_ms": int(_MAX_CAPTURE_AGE_SECONDS * 1000),
+        }
+        try:
+            raw = await asyncio.wait_for(
+                self._page.evaluate(_APPLY_SCRIPT, payload), timeout=checked
+            )
+        except asyncio.CancelledError:
+            raise
+        except TimeoutError:
+            raise FetchError(
+                "Cloudflare Challenge token application timed out",
+                phase="cloudflare_challenge_apply",
+            ) from None
+        except Exception:
+            raise FetchError(
+                "Cloudflare Challenge token application failed",
+                phase="cloudflare_challenge_apply",
+            ) from None
+        if not isinstance(raw, dict) or not isinstance(raw.get("applied"), bool):
+            raise FetchError(
+                "Cloudflare Challenge token application returned invalid data",
+                phase="cloudflare_challenge_apply",
+            )
+        if raw["applied"]:
+            return True
+        reasons = {
+            "document_changed": "Cloudflare Challenge page changed before token application",
+            "ambiguous": "Multiple Cloudflare Challenges are active",
+            "stale": "Cloudflare Challenge is no longer active",
+            "challenge_changed": "Cloudflare Challenge changed before token application",
+            "expired": "Cloudflare Challenge parameters expired before token application",
+            "user_agent_changed": "Cloudflare Challenge browser user agent changed",
+            "callback_unavailable": "Cloudflare Challenge callback is unavailable",
+        }
+        reason = raw.get("reason")
+        message = (
+            reasons.get(reason, "Cloudflare Challenge token application was rejected")
+            if isinstance(reason, str)
+            else "Cloudflare Challenge token application was rejected"
+        )
+        raise FetchError(
+            message,
+            phase="cloudflare_challenge_apply",
+        )
+
+
+def _timeout(value: float, *, maximum: float) -> float:
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, int | float)
+        or not math.isfinite(value)
+        or value <= 0
+        or value > maximum
+    ):
+        raise ConfigurationError(f"timeout must be between 0 and {int(maximum)} seconds")
+    return float(value)
+
+
+def _is_transient_context_error(error: Exception) -> bool:
+    """Classify only well-known navigation context replacement errors."""
+
+    if error.__class__.__name__ == "TargetClosedError":
+        return False
+    message = str(error).lower()
+    return any(
+        marker in message
+        for marker in (
+            "execution context was destroyed",
+            "cannot find context with specified id",
+            "cannot find execution context",
+        )
+    )
+
+
+def _bounded_text(name: str, value: Any, *, maximum: int) -> str:
+    if (
+        not isinstance(value, str)
+        or not 0 < len(value) <= maximum
+        or any(ord(char) < 32 or ord(char) == 127 for char in value)
+    ):
+        raise ConfigurationError(f"Cloudflare Challenge {name} is invalid")
+    return value
+
+
+def _parameters(raw: dict[str, Any]) -> CloudflareChallengeParameters:
+    try:
+        values = {
+            "website_url": _page_url(raw.get("website_url")),
+            "website_key": _bounded_text("website_key", raw.get("website_key"), maximum=512),
+            "action": _bounded_text("action", raw.get("action"), maximum=64),
+            "data": _bounded_text("data", raw.get("data"), maximum=4096),
+            "page_data": _bounded_text("page_data", raw.get("page_data"), maximum=16384),
+            "user_agent": _bounded_text("user_agent", raw.get("user_agent"), maximum=512),
+        }
+        api_js_url = raw.get("api_js_url")
+        if api_js_url is not None:
+            api_js_url = _api_url(api_js_url)
+        document_id = _bounded_text("document binding", raw.get("document_id"), maximum=256)
+        capture_id = _bounded_text("capture binding", raw.get("capture_id"), maximum=512)
+        generation = raw.get("generation")
+        captured_at_ms = raw.get("captured_at_ms")
+        if (
+            isinstance(generation, bool)
+            or not isinstance(generation, int)
+            or generation <= 0
+            or isinstance(captured_at_ms, bool)
+            or not isinstance(captured_at_ms, int)
+            or captured_at_ms <= 0
+        ):
+            raise ValueError("invalid binding")
+    except (ConfigurationError, FetchError, TypeError, ValueError, OverflowError) as exc:
+        raise FetchError(
+            "Cloudflare Challenge capture returned invalid data",
+            phase="cloudflare_challenge_capture",
+        ) from exc
+    return CloudflareChallengeParameters(
+        **values,
+        api_js_url=api_js_url,
+        _document_id=document_id,
+        _capture_id=capture_id,
+        _generation=generation,
+        _captured_at_ms=captured_at_ms,
+    )
+
+
+def _page_url(value: Any) -> str:
+    value = _bounded_text("website_url", value, maximum=2048)
+    try:
+        parts = urlsplit(value)
+    except ValueError:
+        parts = None
+    if (
+        parts is None
+        or parts.scheme not in {"http", "https"}
+        or not parts.hostname
+        or parts.username is not None
+        or parts.password is not None
+    ):
+        raise FetchError(
+            "Cloudflare Challenge page URL is invalid",
+            phase="cloudflare_challenge_capture",
+        )
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, parts.query, ""))
+
+
+def _api_url(value: Any) -> str:
+    value = _bounded_text("api_js_url", value, maximum=2048)
+    try:
+        parts = urlsplit(value)
+    except ValueError:
+        parts = None
+    if (
+        parts is None
+        or parts.scheme != "https"
+        or (parts.hostname or "").lower() != "challenges.cloudflare.com"
+        or not parts.path.startswith("/turnstile/")
+        or parts.username is not None
+        or parts.password is not None
+        or parts.fragment
+    ):
+        raise FetchError(
+            "Cloudflare Challenge script URL is invalid",
+            phase="cloudflare_challenge_capture",
+        )
+    return value