dataapp_crawler 0.1.0__py3-none-any.whl

dataapp_crawler/captcha.py

@@ -0,0 +1,912 @@
+"""Read-only CAPTCHA detection and solver-component matching.
+
+Detection never navigates, clicks, refreshes, captures images, or invokes a
+solver. It returns fixed evidence codes instead of page values so URLs, site
+keys, tokens, and image content do not leak through result representations.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import math
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from html.parser import HTMLParser
+from typing import Any, Literal
+from urllib.parse import SplitResult, parse_qs, urlsplit
+
+from .errors import ConfigurationError, FetchError, ResponseTooLarge
+
+CaptchaFamily = Literal[
+    "recaptcha_v2",
+    "recaptcha_v2_enterprise",
+    "turnstile",
+    "cloudflare_challenge",
+    "image_to_text",
+    "unknown",
+]
+CaptchaPresence = Literal["access_challenge", "widget_observed", "unknown"]
+CaptchaConfidence = Literal["high", "medium", "low"]
+SolverComponent = Literal["resource_proxy_capmonster", "vision_model"]
+
+_FAMILIES = {
+    "recaptcha_v2",
+    "recaptcha_v2_enterprise",
+    "turnstile",
+    "cloudflare_challenge",
+    "image_to_text",
+    "unknown",
+}
+_PRESENCES = {"access_challenge", "widget_observed", "unknown"}
+_COMPONENTS = {"resource_proxy_capmonster", "vision_model"}
+_SAFE_NAME = re.compile(r"[a-z][a-z0-9_-]{0,63}")
+_BLOCKING_STATUS = {401, 403, 429}
+_TEXT_CHALLENGE_MARKERS = (
+    "verify you are human",
+    "verify that you are human",
+    "checking your browser",
+    "security check",
+    "human verification",
+)
+
+
+@dataclass(frozen=True, slots=True)
+class CaptchaEvidence:
+    """One sanitized reason for a detection, without a raw page value."""
+
+    code: str
+    source: Literal["dom", "resource", "text", "status", "override"]
+
+
+@dataclass(frozen=True, slots=True)
+class SolverMatch:
+    """Structural match between an observation and an integrated component.
+
+    ``can_attempt`` is true only when the caller explicitly declared the
+    component available and all structurally required inputs were observed or
+    overridden. It does not prove a Resource Proxy grant, supplier health,
+    answer correctness, or acceptance by the target website.
+    """
+
+    component: SolverComponent
+    task_type: str
+    integrated: bool
+    configured: bool
+    can_attempt: bool
+    missing_inputs: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class CaptchaObservation:
+    """Immutable CAPTCHA classification with safe evidence and component matches."""
+
+    family: CaptchaFamily
+    vendor: str | None
+    module_hint: str | None
+    confidence: CaptchaConfidence
+    presence: CaptchaPresence
+    evidence: tuple[CaptchaEvidence, ...]
+    solvers: tuple[SolverMatch, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class CaptchaOverride:
+    """Developer-provided classification when site knowledge is stronger.
+
+    Boolean fields only declare that the application already has an input.
+    Actual URLs, keys, image bytes, and tokens stay outside the detector.
+    """
+
+    family: CaptchaFamily
+    presence: CaptchaPresence = "widget_observed"
+    vendor: str | None = None
+    module_hint: str | None = None
+    has_website_url: bool = False
+    has_website_key: bool = False
+    has_image: bool = False
+    has_action: bool = False
+    has_data: bool = False
+    has_page_data: bool = False
+    has_user_agent: bool = False
+
+    def __post_init__(self) -> None:
+        if self.family not in _FAMILIES:
+            raise ConfigurationError("CAPTCHA override family is unsupported")
+        if self.presence not in _PRESENCES:
+            raise ConfigurationError("CAPTCHA override presence is unsupported")
+        for label, value in (("vendor", self.vendor), ("module_hint", self.module_hint)):
+            if value is not None and not _SAFE_NAME.fullmatch(value):
+                raise ConfigurationError(f"CAPTCHA override {label} is invalid")
+        for label, bool_value in (
+            ("has_website_url", self.has_website_url),
+            ("has_website_key", self.has_website_key),
+            ("has_image", self.has_image),
+            ("has_action", self.has_action),
+            ("has_data", self.has_data),
+            ("has_page_data", self.has_page_data),
+            ("has_user_agent", self.has_user_agent),
+        ):
+            if not isinstance(bool_value, bool):
+                raise ConfigurationError(f"CAPTCHA override {label} must be a boolean")
+
+
+@dataclass(slots=True)
+class _Candidate:
+    family: CaptchaFamily
+    vendor: str | None = None
+    module_hint: str | None = None
+    confidence: CaptchaConfidence = "low"
+    presence: CaptchaPresence = "unknown"
+    evidence: list[CaptchaEvidence] = field(default_factory=list)
+    has_website_url: bool = False
+    has_website_key: bool = False
+    has_image: bool = False
+    has_action: bool = False
+    has_data: bool = False
+    has_page_data: bool = False
+    has_user_agent: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class _Node:
+    tag: str
+    attrs: tuple[tuple[str, str], ...]
+
+    def get(self, name: str) -> str:
+        for key, value in self.attrs:
+            if key == name:
+                return value
+        return ""
+
+
+class _DocumentParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.nodes: list[_Node] = []
+        self.text_parts: list[str] = []
+        self.script_parts: list[str] = []
+        self._ignored_text_depth = 0
+        self._script_depth = 0
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        lowered_tag = tag.lower()
+        if lowered_tag in {"script", "style", "noscript"}:
+            self._ignored_text_depth += 1
+        if lowered_tag == "script":
+            self._script_depth += 1
+        self.nodes.append(
+            _Node(
+                lowered_tag,
+                tuple((key.lower(), value or "") for key, value in attrs),
+            )
+        )
+
+    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        self.handle_starttag(tag, attrs)
+        lowered_tag = tag.lower()
+        if lowered_tag == "script":
+            self._script_depth -= 1
+        if lowered_tag in {"script", "style", "noscript"}:
+            self._ignored_text_depth -= 1
+
+    def handle_endtag(self, tag: str) -> None:
+        lowered_tag = tag.lower()
+        if lowered_tag == "script" and self._script_depth:
+            self._script_depth -= 1
+        if lowered_tag in {"script", "style", "noscript"} and self._ignored_text_depth:
+            self._ignored_text_depth -= 1
+
+    def handle_data(self, data: str) -> None:
+        if self._script_depth and data.strip():
+            self.script_parts.append(data)
+        if not self._ignored_text_depth and data.strip():
+            self.text_parts.append(data)
+
+
+def detect_captchas(
+    html: str,
+    *,
+    page_url: str | None = None,
+    status: int | None = None,
+    resource_urls: Sequence[str] = (),
+    overrides: Sequence[CaptchaOverride] = (),
+    available_components: Sequence[SolverComponent] = (),
+    max_html_chars: int = 1_000_000,
+) -> tuple[CaptchaObservation, ...]:
+    """Classify CAPTCHA integrations without performing browser or paid actions.
+
+    Raw URLs and public site keys are inspected only for marker presence and are
+    never returned. ``resource_urls`` is useful for URLs observed by an existing
+    capture; this function never fetches them. A script include alone produces
+    an ``unknown`` clue, not a positive widget or access-challenge verdict.
+    """
+    _validate_inputs(html, page_url, status, resource_urls, overrides, max_html_chars)
+    configured = _configured_components(available_components)
+    parser = _DocumentParser()
+    try:
+        parser.feed(html)
+        parser.close()
+    except (TypeError, ValueError):
+        raise ConfigurationError("CAPTCHA detection received invalid HTML") from None
+
+    text = " ".join(parser.text_parts).lower()
+    blocking = status in _BLOCKING_STATUS or any(
+        marker in text for marker in _TEXT_CHALLENGE_MARKERS
+    )
+    page_url_available = _usable_page_url(page_url)
+    urls = [node.get("src") for node in parser.nodes if node.get("src")]
+    urls.extend(resource_urls)
+    url_text = "\n".join(url.lower() for url in urls)
+    script_text = " ".join(parser.script_parts).lower()
+
+    observations: list[_Candidate] = []
+    recaptcha_nodes = [node for node in parser.nodes if _is_recaptcha_widget(node)]
+    recaptcha_response = any(_is_recaptcha_response(node) for node in parser.nodes)
+    recaptcha_script = any(_is_recaptcha_resource(url) for url in urls)
+    enterprise_script = any(_is_enterprise_recaptcha_resource(url) for url in urls)
+    regular_script = any(_is_regular_recaptcha_resource(url) for url in urls)
+    enterprise_frame = any(_is_enterprise_recaptcha_frame(url) for url in urls)
+    regular_frame = any(_is_regular_recaptcha_frame(url) for url in urls)
+    enterprise_inline = bool(
+        re.search(r"\bgrecaptcha\.enterprise\.(?:execute|ready|render)\s*\(", script_text)
+    )
+    enterprise = enterprise_inline or enterprise_script or enterprise_frame
+    regular = regular_script or regular_frame
+    recaptcha_v3 = bool(
+        re.search(r"\bgrecaptcha(?:\.enterprise)?\.execute\s*\(", script_text)
+    ) or any(_is_recaptcha_v3_resource(url) for url in urls)
+    site_key = bool(recaptcha_nodes) and all(
+        _node_has_website_key(node) for node in recaptcha_nodes
+    )
+    mixed_integration = enterprise and regular
+
+    if recaptcha_nodes and recaptcha_v3:
+        observations.append(
+            _unresolved_recaptcha_candidate(
+                blocking=blocking,
+                page_url_available=page_url_available,
+                evidence_code="recaptcha-v3-widget-ambiguous",
+            )
+        )
+    elif recaptcha_nodes and mixed_integration:
+        observations.append(
+            _clue_candidate("google", "recaptcha-mixed-integration", page_url_available)
+        )
+    elif recaptcha_nodes and (enterprise or regular):
+        family: CaptchaFamily = "recaptcha_v2_enterprise" if enterprise else "recaptcha_v2"
+        candidate = _positive_candidate(
+            family,
+            vendor="google",
+            blocking=blocking,
+            confidence="high",
+            evidence_code=("recaptcha-enterprise-widget" if enterprise else "recaptcha-v2-widget"),
+        )
+        candidate.has_website_url = page_url_available
+        candidate.has_website_key = site_key
+        observations.append(candidate)
+    elif recaptcha_nodes:
+        observations.append(
+            _unresolved_recaptcha_candidate(
+                blocking=blocking,
+                page_url_available=page_url_available,
+                evidence_code="recaptcha-family-unresolved",
+            )
+        )
+    elif (
+        recaptcha_response
+        and not recaptcha_v3
+        and not mixed_integration
+        and (enterprise or regular)
+    ):
+        candidate = _positive_candidate(
+            "recaptcha_v2_enterprise" if enterprise else "recaptcha_v2",
+            vendor="google",
+            blocking=blocking,
+            confidence="medium",
+            evidence_code="recaptcha-response-field",
+        )
+        candidate.has_website_url = page_url_available
+        candidate.has_website_key = any(
+            _resource_has_website_key(url) for url in urls if _is_recaptcha_frame(url)
+        )
+        observations.append(candidate)
+    elif recaptcha_response and not recaptcha_v3 and not mixed_integration:
+        observations.append(
+            _unresolved_recaptcha_candidate(
+                blocking=blocking,
+                page_url_available=page_url_available,
+                evidence_code="recaptcha-response-family-unresolved",
+            )
+        )
+
+    if recaptcha_v3:
+        observations.append(_clue_candidate("google", "recaptcha-v3-marker", page_url_available))
+    elif recaptcha_script and not recaptcha_nodes and not recaptcha_response:
+        observations.append(_clue_candidate("google", "recaptcha-script-only", page_url_available))
+
+    turnstile_nodes = [node for node in parser.nodes if _is_turnstile_widget(node)]
+    turnstile_response = any(_is_turnstile_response(node) for node in parser.nodes)
+    turnstile_script = any(_is_turnstile_resource(url) for url in urls)
+    challenge_options = bool(re.search(r"\b(?:window\.)?_cf_chl_opt\s*=", script_text))
+    challenge_form = any(_is_cloudflare_challenge_form(node) for node in parser.nodes)
+    challenge_resource = any(_is_cloudflare_challenge_resource(url, page_url) for url in urls)
+    challenge_text = any(
+        marker in text
+        for marker in (
+            "just a moment",
+            "performing security verification",
+            "enable javascript and cookies to continue",
+            "checking your browser",
+            "verify you are human",
+            "verify that you are human",
+        )
+    )
+    full_cloudflare = (
+        challenge_options and (challenge_form or challenge_text or challenge_resource)
+    ) or (challenge_form and challenge_resource)
+    if full_cloudflare:
+        candidate = _positive_candidate(
+            "cloudflare_challenge",
+            vendor="cloudflare",
+            blocking=True,
+            confidence="high",
+            evidence_code=(
+                "cloudflare-challenge-options" if challenge_options
+                else "cloudflare-challenge-form"
+            ),
+        )
+        candidate.has_website_url = page_url_available
+        candidate.has_website_key = bool(turnstile_nodes) and all(
+            _node_has_website_key(node) for node in turnstile_nodes
+        )
+        if challenge_form:
+            candidate.evidence.append(CaptchaEvidence("cloudflare-challenge-form", "dom"))
+        if challenge_resource:
+            candidate.evidence.append(CaptchaEvidence("cloudflare-challenge-resource", "resource"))
+        if challenge_text:
+            candidate.evidence.append(CaptchaEvidence("cloudflare-challenge-interstitial", "text"))
+        observations.append(candidate)
+    elif turnstile_nodes or turnstile_response:
+        candidate = _positive_candidate(
+            "turnstile",
+            vendor="cloudflare",
+            blocking=blocking,
+            confidence="high" if turnstile_nodes else "medium",
+            evidence_code="turnstile-widget" if turnstile_nodes else "turnstile-response-field",
+        )
+        candidate.has_website_url = page_url_available
+        candidate.has_website_key = bool(turnstile_nodes) and all(
+            _node_has_website_key(node) for node in turnstile_nodes
+        )
+        observations.append(candidate)
+    elif turnstile_script:
+        observations.append(
+            _clue_candidate("cloudflare", "turnstile-script-only", page_url_available)
+        )
+
+    botdetect_nodes = [node for node in parser.nodes if _is_botdetect_widget(node)]
+    botdetect_resource = (
+        "botdetect" in url_text
+        or "botdetect.init" in script_text
+        or "botdetect.getinstancebyid" in script_text
+    )
+    if botdetect_nodes:
+        candidate = _positive_candidate(
+            "image_to_text",
+            vendor="botdetect",
+            module_hint="botdetect",
+            blocking=blocking,
+            confidence="high",
+            evidence_code="botdetect-widget",
+        )
+        candidate.has_website_url = page_url_available
+        observations.append(candidate)
+    elif botdetect_resource:
+        observations.append(
+            _clue_candidate("botdetect", "botdetect-resource-only", page_url_available)
+        )
+
+    if not botdetect_nodes and _has_generic_image_captcha(parser.nodes):
+        candidate = _positive_candidate(
+            "image_to_text",
+            vendor=None,
+            blocking=blocking,
+            confidence="medium",
+            evidence_code="image-captcha-widget",
+        )
+        candidate.has_website_url = page_url_available
+        observations.append(candidate)
+
+    if status in _BLOCKING_STATUS:
+        _add_evidence(observations, CaptchaEvidence("blocking-http-status", "status"))
+    if any(marker in text for marker in _TEXT_CHALLENGE_MARKERS):
+        _add_evidence(observations, CaptchaEvidence("human-verification-text", "text"))
+
+    for override in overrides:
+        candidate = _candidate_from_override(override, page_url_available)
+        same_family = [item for item in observations if item.family == override.family]
+        if override.vendor is None and len(same_family) == 1:
+            candidate.vendor = same_family[0].vendor
+        unresolved = [
+            item
+            for item in observations
+            if item.family == "unknown"
+            and item.vendor == "google"
+            and any(
+                evidence.code
+                in {"recaptcha-family-unresolved", "recaptcha-response-family-unresolved"}
+                for evidence in item.evidence
+            )
+        ]
+        if (
+            override.family in {"recaptcha_v2", "recaptcha_v2_enterprise"}
+            and override.vendor in {None, "google"}
+            and not same_family
+            and len(unresolved) == 1
+        ):
+            source = unresolved[0]
+            observations.remove(source)
+            candidate.vendor = "google"
+            candidate.presence = _max_presence(candidate.presence, source.presence)
+            candidate.has_website_url |= source.has_website_url
+            candidate.evidence.extend(source.evidence)
+        observations.append(candidate)
+
+    merged = _merge_candidates(observations)
+    return tuple(_finalize(candidate, configured) for candidate in merged)
+
+
+async def detect_captchas_on_page(
+    page: Any,
+    *,
+    timeout: float = 5.0,
+    overrides: Sequence[CaptchaOverride] = (),
+    available_components: Sequence[SolverComponent] = (),
+    max_html_chars: int = 1_000_000,
+) -> tuple[CaptchaObservation, ...]:
+    """Read one existing Page's DOM without navigation or other page mutation.
+
+    ``timeout`` is in seconds. The adapter calls only ``page.content()`` and
+    reads ``page.url``; it does not evaluate scripts, inspect network secrets,
+    capture images, submit forms, or contact a solver.
+    """
+    if (
+        isinstance(timeout, bool)
+        or not isinstance(timeout, (int, float))
+        or not math.isfinite(timeout)
+        or timeout <= 0
+        or timeout > 30
+    ):
+        raise ConfigurationError("CAPTCHA detection timeout must be between 0 and 30 seconds")
+    try:
+        html = await asyncio.wait_for(page.content(), timeout=float(timeout))
+    except TimeoutError:
+        raise TimeoutError("CAPTCHA detection timed out") from None
+    except Exception:
+        raise FetchError("CAPTCHA page inspection failed", phase="captcha_detection") from None
+    if not isinstance(html, str):
+        raise FetchError("Browser returned invalid page content", phase="captcha_detection")
+    page_url = getattr(page, "url", None)
+    if not isinstance(page_url, str):
+        page_url = None
+    return detect_captchas(
+        html,
+        page_url=page_url,
+        overrides=overrides,
+        available_components=available_components,
+        max_html_chars=max_html_chars,
+    )
+
+
+def _validate_inputs(
+    html: str,
+    page_url: str | None,
+    status: int | None,
+    resource_urls: Sequence[str],
+    overrides: Sequence[CaptchaOverride],
+    max_html_chars: int,
+) -> None:
+    if not isinstance(html, str):
+        raise ConfigurationError("CAPTCHA detection HTML must be a string")
+    if (
+        isinstance(max_html_chars, bool)
+        or not isinstance(max_html_chars, int)
+        or not 1 <= max_html_chars <= 5_000_000
+    ):
+        raise ConfigurationError("max_html_chars must be between 1 and 5000000")
+    if len(html) > max_html_chars:
+        raise ResponseTooLarge("CAPTCHA detection HTML exceeds max_html_chars")
+    if page_url is not None and not isinstance(page_url, str):
+        raise ConfigurationError("CAPTCHA detection page_url must be a string")
+    if status is not None and (isinstance(status, bool) or not isinstance(status, int)):
+        raise ConfigurationError("CAPTCHA detection status must be an integer")
+    if isinstance(resource_urls, (str, bytes)) or len(resource_urls) > 128:
+        raise ConfigurationError("resource_urls must contain at most 128 URLs")
+    if any(not isinstance(url, str) or len(url) > 4096 for url in resource_urls):
+        raise ConfigurationError("resource_urls contains an invalid URL")
+    if isinstance(overrides, (str, bytes)) or len(overrides) > 16:
+        raise ConfigurationError("overrides must contain at most 16 items")
+    if any(not isinstance(item, CaptchaOverride) for item in overrides):
+        raise ConfigurationError("overrides must contain CaptchaOverride values")
+
+
+def _configured_components(values: Sequence[SolverComponent]) -> frozenset[str]:
+    if isinstance(values, (str, bytes)):
+        raise ConfigurationError("available_components must be a sequence")
+    configured = set(values)
+    if len(configured) != len(values) or not configured.issubset(_COMPONENTS):
+        raise ConfigurationError("available_components contains invalid or duplicate values")
+    return frozenset(configured)
+
+
+def _usable_page_url(value: str | None) -> bool:
+    if not value or len(value) > 4096:
+        return False
+    try:
+        parts = urlsplit(value)
+    except ValueError:
+        return False
+    return (
+        parts.scheme in {"http", "https"}
+        and bool(parts.hostname)
+        and parts.username is None
+        and parts.password is None
+    )
+
+
+def _is_recaptcha_widget(node: _Node) -> bool:
+    return "g-recaptcha" in node.get("class").lower().split() or (
+        node.tag == "iframe" and _is_recaptcha_frame(node.get("src"))
+    )
+
+
+def _is_recaptcha_response(node: _Node) -> bool:
+    return node.tag in {"input", "textarea"} and _has_exact_form_name(node, "g-recaptcha-response")
+
+
+def _is_enterprise_recaptcha_resource(value: str) -> bool:
+    parts = _trusted_recaptcha_parts(value)
+    return parts is not None and parts.path.lower() == "/recaptcha/enterprise.js"
+
+
+def _is_regular_recaptcha_resource(value: str) -> bool:
+    parts = _trusted_recaptcha_parts(value)
+    return parts is not None and parts.path.lower() == "/recaptcha/api.js"
+
+
+def _is_recaptcha_resource(value: str) -> bool:
+    return _is_regular_recaptcha_resource(value) or _is_enterprise_recaptcha_resource(value)
+
+
+def _is_recaptcha_frame(value: str) -> bool:
+    return _is_regular_recaptcha_frame(value) or _is_enterprise_recaptcha_frame(value)
+
+
+def _is_regular_recaptcha_frame(value: str) -> bool:
+    parts = _trusted_recaptcha_parts(value)
+    if parts is None:
+        return False
+    return parts.path.lower().startswith("/recaptcha/api2/")
+
+
+def _is_enterprise_recaptcha_frame(value: str) -> bool:
+    parts = _trusted_recaptcha_parts(value)
+    if parts is None:
+        return False
+    return parts.path.lower().startswith("/recaptcha/enterprise/")
+
+
+def _trusted_recaptcha_parts(value: str) -> SplitResult | None:
+    try:
+        parts = urlsplit(value)
+    except ValueError:
+        return None
+    if not _is_http_resource(parts) or parts.username or parts.password:
+        return None
+    if (parts.hostname or "").lower() not in {
+        "google.com",
+        "www.google.com",
+        "recaptcha.net",
+        "www.recaptcha.net",
+    }:
+        return None
+    return parts
+
+
+def _is_recaptcha_v3_resource(value: str) -> bool:
+    if not _is_recaptcha_resource(value):
+        return False
+    try:
+        render = parse_qs(urlsplit(value).query).get("render", ())
+    except ValueError:
+        return False
+    return any(item and item.lower() != "explicit" for item in render)
+
+
+def _resource_has_website_key(value: str) -> bool:
+    try:
+        query = parse_qs(urlsplit(value).query)
+    except ValueError:
+        return False
+    return any(bool(query.get(name)) for name in ("k", "sitekey"))
+
+
+def _node_has_website_key(node: _Node) -> bool:
+    if node.get("data-sitekey"):
+        return True
+    if node.tag != "iframe":
+        return False
+    source = node.get("src")
+    return (
+        _is_recaptcha_frame(source) or _is_turnstile_resource(source)
+    ) and _resource_has_website_key(source)
+
+
+def _is_turnstile_widget(node: _Node) -> bool:
+    return "cf-turnstile" in node.get("class").lower().split() or (
+        node.tag == "iframe" and _is_turnstile_resource(node.get("src"))
+    )
+
+
+def _is_cloudflare_challenge_resource(value: str, page_url: str | None) -> bool:
+    try:
+        parts = urlsplit(value)
+        page_host = urlsplit(page_url or "").hostname
+    except ValueError:
+        return False
+    if parts.username or parts.password or parts.scheme not in {"", "http", "https"}:
+        return False
+    if not parts.path.startswith("/cdn-cgi/challenge-platform/"):
+        return False
+    return parts.hostname is None or parts.hostname in {page_host, "challenges.cloudflare.com"}
+
+
+def _is_cloudflare_challenge_form(node: _Node) -> bool:
+    if node.tag != "form" or node.get("id") != "challenge-form":
+        return False
+    try:
+        action = urlsplit(node.get("action"))
+        return action.scheme in {"", "http", "https"} and bool(
+            parse_qs(action.query).get("__cf_chl_f_tk")
+        )
+    except ValueError:
+        return False
+
+
+def _is_turnstile_resource(value: str) -> bool:
+    try:
+        parts = urlsplit(value)
+    except ValueError:
+        return False
+    return (
+        _is_http_resource(parts)
+        and parts.username is None
+        and parts.password is None
+        and (parts.hostname or "").lower() == "challenges.cloudflare.com"
+        and parts.path.lower().startswith("/turnstile/")
+    )
+
+
+def _is_http_resource(parts: SplitResult) -> bool:
+    return parts.scheme in {"http", "https"} or (not parts.scheme and bool(parts.netloc))
+
+
+def _is_turnstile_response(node: _Node) -> bool:
+    return node.tag in {"input", "textarea"} and _has_exact_form_name(node, "cf-turnstile-response")
+
+
+def _is_botdetect_widget(node: _Node) -> bool:
+    tokens = " ".join(node.get(name).lower() for name in ("id", "class", "name"))
+    integration_marker = (
+        "bdc_captcha" in tokens or "bdc-captcha" in tokens or "botdetectcaptcha" in tokens
+    )
+    if node.tag in {"div", "span", "section", "input", "textarea", "canvas"}:
+        return integration_marker
+    if node.tag == "img":
+        return integration_marker or _is_botdetect_image_source(node.get("src"))
+    return False
+
+
+def _is_botdetect_image_source(value: str) -> bool:
+    try:
+        path = urlsplit(value).path.lower()
+    except ValueError:
+        return False
+    return path.rsplit("/", 1)[-1].startswith("botdetectcaptcha.")
+
+
+def _has_exact_form_name(node: _Node, expected: str) -> bool:
+    return node.get("name").lower() == expected or node.get("id").lower() == expected
+
+
+def _has_generic_image_captcha(nodes: Sequence[_Node]) -> bool:
+    image = any(
+        node.tag in {"img", "canvas", "svg"}
+        and _has_generic_captcha_marker(node, include_source=True)
+        for node in nodes
+    )
+    answer = any(
+        node.tag == "input" and _has_generic_captcha_marker(node, include_source=False)
+        for node in nodes
+    )
+    return image and answer
+
+
+def _has_generic_captcha_marker(node: _Node, *, include_source: bool) -> bool:
+    value = " ".join(node.get(name).lower() for name in ("id", "class", "name"))
+    if include_source:
+        try:
+            value = f"{value} {urlsplit(node.get('src')).path.rsplit('/', 1)[-1].lower()}"
+        except ValueError:
+            pass
+    return (
+        "captcha" in value
+        and "recaptcha" not in value
+        and "turnstile" not in value
+        and "botdetect" not in value
+        and "bdc_captcha" not in value
+    )
+
+
+def _positive_candidate(
+    family: CaptchaFamily,
+    *,
+    vendor: str | None,
+    blocking: bool,
+    confidence: CaptchaConfidence,
+    evidence_code: str,
+    module_hint: str | None = None,
+) -> _Candidate:
+    return _Candidate(
+        family=family,
+        vendor=vendor,
+        module_hint=module_hint,
+        confidence=confidence,
+        presence="access_challenge" if blocking else "widget_observed",
+        evidence=[CaptchaEvidence(evidence_code, "dom")],
+    )
+
+
+def _clue_candidate(vendor: str, code: str, has_website_url: bool) -> _Candidate:
+    return _Candidate(
+        family="unknown",
+        vendor=vendor,
+        confidence="low",
+        presence="unknown",
+        evidence=[CaptchaEvidence(code, "resource")],
+        has_website_url=has_website_url,
+    )
+
+
+def _unresolved_recaptcha_candidate(
+    *, blocking: bool, page_url_available: bool, evidence_code: str
+) -> _Candidate:
+    return _Candidate(
+        family="unknown",
+        vendor="google",
+        confidence="medium",
+        presence="access_challenge" if blocking else "widget_observed",
+        evidence=[CaptchaEvidence(evidence_code, "dom")],
+        has_website_url=page_url_available,
+    )
+
+
+def _candidate_from_override(override: CaptchaOverride, page_url_available: bool) -> _Candidate:
+    return _Candidate(
+        family=override.family,
+        vendor=override.vendor,
+        module_hint=override.module_hint,
+        confidence="high",
+        presence=override.presence,
+        evidence=[CaptchaEvidence("developer-override", "override")],
+        has_website_url=override.has_website_url or page_url_available,
+        has_website_key=override.has_website_key,
+        has_image=override.has_image,
+        has_action=override.has_action,
+        has_data=override.has_data,
+        has_page_data=override.has_page_data,
+        has_user_agent=override.has_user_agent,
+    )
+
+
+def _add_evidence(candidates: Sequence[_Candidate], evidence: CaptchaEvidence) -> None:
+    for candidate in candidates:
+        if candidate.presence != "unknown" and evidence not in candidate.evidence:
+            candidate.evidence.append(evidence)
+
+
+def _merge_candidates(candidates: Sequence[_Candidate]) -> list[_Candidate]:
+    merged: dict[tuple[str, str | None], _Candidate] = {}
+    for candidate in candidates:
+        key = (candidate.family, candidate.vendor)
+        existing = merged.get(key)
+        if existing is None:
+            merged[key] = candidate
+            continue
+        existing.module_hint = candidate.module_hint or existing.module_hint
+        existing.confidence = _max_confidence(existing.confidence, candidate.confidence)
+        existing.presence = _max_presence(existing.presence, candidate.presence)
+        existing.has_website_url |= candidate.has_website_url
+        existing.has_website_key |= candidate.has_website_key
+        existing.has_image |= candidate.has_image
+        existing.has_action |= candidate.has_action
+        existing.has_data |= candidate.has_data
+        existing.has_page_data |= candidate.has_page_data
+        existing.has_user_agent |= candidate.has_user_agent
+        for evidence in candidate.evidence:
+            if evidence not in existing.evidence:
+                existing.evidence.append(evidence)
+    return sorted(
+        merged.values(),
+        key=lambda item: (item.family == "unknown", item.family, item.vendor or ""),
+    )
+
+
+def _max_confidence(left: CaptchaConfidence, right: CaptchaConfidence) -> CaptchaConfidence:
+    order = {"low": 0, "medium": 1, "high": 2}
+    return left if order[left] >= order[right] else right
+
+
+def _max_presence(left: CaptchaPresence, right: CaptchaPresence) -> CaptchaPresence:
+    order = {"unknown": 0, "widget_observed": 1, "access_challenge": 2}
+    return left if order[left] >= order[right] else right
+
+
+def _finalize(candidate: _Candidate, configured: frozenset[str]) -> CaptchaObservation:
+    solvers = _solver_matches(candidate, configured)
+    evidence = tuple(sorted(candidate.evidence, key=lambda item: (item.source, item.code)))
+    return CaptchaObservation(
+        family=candidate.family,
+        vendor=candidate.vendor,
+        module_hint=candidate.module_hint,
+        confidence=candidate.confidence,
+        presence=candidate.presence,
+        evidence=evidence,
+        solvers=solvers,
+    )
+
+
+def _solver_matches(candidate: _Candidate, configured: frozenset[str]) -> tuple[SolverMatch, ...]:
+    if candidate.family == "unknown":
+        return ()
+    required: tuple[tuple[str, bool], ...] = (
+        (("image_body", candidate.has_image),)
+        if candidate.family == "image_to_text"
+        else (
+            ("website_url", candidate.has_website_url),
+            ("website_key", candidate.has_website_key),
+        )
+    )
+    if candidate.family == "cloudflare_challenge":
+        required += (
+            ("action", candidate.has_action),
+            ("data", candidate.has_data),
+            ("page_data", candidate.has_page_data),
+            ("user_agent", candidate.has_user_agent),
+        )
+    missing = tuple(name for name, present in required if not present)
+    resource_configured = "resource_proxy_capmonster" in configured
+    matches = [
+        SolverMatch(
+            component="resource_proxy_capmonster",
+            task_type=candidate.family,
+            integrated=True,
+            configured=resource_configured,
+            can_attempt=resource_configured and not missing,
+            missing_inputs=missing,
+        )
+    ]
+    if candidate.family == "image_to_text":
+        vision_configured = "vision_model" in configured
+        matches.append(
+            SolverMatch(
+                component="vision_model",
+                task_type="image_to_text",
+                integrated=True,
+                configured=vision_configured,
+                can_attempt=vision_configured and not missing,
+                missing_inputs=missing,
+            )
+        )
+    return tuple(matches)