fetchgate 0.1.0__py3-none-any.whl

fetchgate/__init__.py

@@ -0,0 +1,21 @@
+"""FetchGate: a deterministic gate at the fetch boundary. A 200 is not a read."""
+
+from .app import default_gate
+from .config import DomainPolicy, FetchPolicy
+from .detect import decide
+from .fetch import build_envelope, fetch
+from .gate import FetchGate, GateOutcome
+from .transports import BrowserTransport, HttpTransport, MockTransport, Transport
+from .types import (
+    GateDisposition, GateResult, ProvenanceEnvelope, RawResponse, Verdict,
+)
+from .verify import envelope_from_dict, recompute_verdict
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "default_gate", "FetchPolicy", "DomainPolicy", "decide", "fetch", "build_envelope",
+    "FetchGate", "GateOutcome", "Transport", "MockTransport", "HttpTransport",
+    "BrowserTransport", "RawResponse", "ProvenanceEnvelope", "GateResult", "Verdict",
+    "GateDisposition", "envelope_from_dict", "recompute_verdict", "__version__",
+]

fetchgate/app.py

@@ -0,0 +1,20 @@
+"""Convenience factory: a gate with the static reader plus the render tier if available."""
+
+from __future__ import annotations
+
+from .config import FetchPolicy
+from .gate import FetchGate
+from .transports import BrowserTransport, HttpTransport
+
+
+def default_gate() -> FetchGate:
+    """A ready gate. Renders JS pages when the [browser] extra is installed."""
+    heavy = None
+    default_mode = "never_render"
+    try:
+        import playwright  # noqa: F401
+        heavy = BrowserTransport()
+        default_mode = "render_allowed"
+    except ImportError:
+        pass
+    return FetchGate(FetchPolicy(default_mode=default_mode), HttpTransport(), heavy)

fetchgate/config.py

@@ -0,0 +1,91 @@
+"""Fetch policy: thresholds and per-domain rules for the classifiers and ladder."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from typing import Optional
+
+# 404 is not a hard fail: soft-404s carry a body and fall through to extraction.
+HARD_FAIL_STATUS = frozenset({401, 403, 407, 408, 429, 451,
+                              500, 501, 502, 503, 504, 505, 506, 507, 508, 510, 511})
+NO_BODY_STATUS = frozenset({204, 205, 304})
+OK_STATUS = frozenset({200, 203, 206, 226})
+
+
+@dataclass(frozen=True)
+class ExtractionProfile:
+    kind: str  # html | structured | xml | plaintext | binary
+    min_extracted_bytes: int = 64          # below this on html -> EMPTY
+    sufficient_extracted_bytes: int = 512  # at/above this -> EXTRACT_OK regardless of ratio
+    min_ratio: float = 0.10                # in the 64..512 band, below this -> THIN
+    min_raw_bytes: int = 64                # EMPTY with raw below this -> empty_transport (FAILED)
+    min_text_chars: int = 1                # plaintext floor
+
+
+@dataclass(frozen=True)
+class DomainPolicy:
+    mode: str = "never_render"  # never_render | render_allowed | always_hard_fail
+
+
+@dataclass(frozen=True)
+class FetchPolicy:
+    policy_id: str = "default"
+    default_mode: str = "never_render"
+    block_cross_origin_redirect: bool = True
+    max_redirects: int = 5
+    unsupported_type_policy: str = "UNKNOWN"
+    domains: dict = field(default_factory=dict)  # registrable-domain -> DomainPolicy
+
+    def profile_for(self, content_type: Optional[str]) -> ExtractionProfile:
+        ct = (content_type or "").split(";")[0].strip().lower()
+        if ct in ("text/html", "application/xhtml+xml"):
+            return ExtractionProfile(kind="html")
+        if ct == "application/json" or ct.endswith("+json"):
+            return ExtractionProfile(kind="structured")
+        if ct in ("application/xml", "text/xml", "application/rss+xml",
+                  "application/atom+xml"):
+            return ExtractionProfile(kind="xml")
+        if ct in ("text/plain", "text/csv", "text/markdown"):
+            return ExtractionProfile(kind="plaintext")
+        return ExtractionProfile(kind="binary")
+
+    def domain_policy(self, url_final: str) -> DomainPolicy:
+        host = _host_of(url_final)
+        reg = _registrable_domain(host)
+        # longest-suffix match against configured domains
+        best: Optional[DomainPolicy] = None
+        best_len = -1
+        for dom, dp in self.domains.items():
+            if host == dom or host.endswith("." + dom) or reg == dom:
+                if len(dom) > best_len:
+                    best, best_len = dp, len(dom)
+        if best is not None:
+            return best
+        return DomainPolicy(mode=self.default_mode)
+
+    def sha256(self) -> str:
+        payload = {
+            "policy_id": self.policy_id,
+            "default_mode": self.default_mode,
+            "block_cross_origin_redirect": self.block_cross_origin_redirect,
+            "max_redirects": self.max_redirects,
+            "unsupported_type_policy": self.unsupported_type_policy,
+            "domains": {k: v.__dict__ for k, v in sorted(self.domains.items())},
+        }
+        raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
+        return hashlib.sha256(raw).hexdigest()
+
+
+def _host_of(url: str) -> str:
+    from urllib.parse import urlparse
+    return (urlparse(url).hostname or "").lower()
+
+
+def _registrable_domain(host: str) -> str:
+    """Naive eTLD+1 (last two labels); production vendors a public-suffix-list."""
+    parts = [p for p in host.split(".") if p]
+    if len(parts) <= 2:
+        return host
+    return ".".join(parts[-2:])

fetchgate/detect.py

@@ -0,0 +1,196 @@
+"""Layers A/C/freshness and decide(): true minimum over layers, then a reason tie-break."""
+
+from __future__ import annotations
+
+from typing import Optional
+from urllib.parse import urlparse
+
+from .config import FetchPolicy, HARD_FAIL_STATUS, NO_BODY_STATUS, OK_STATUS, _registrable_domain
+from .types import (
+    ExtractionClass, Freshness, FreshnessState, GateResult, ProvenanceEnvelope,
+    RawResponse, SEV, TransportClass, TransportSignal, ValiditySeverity,
+    ValiditySignal, Verdict,
+)
+
+# Layer A: transport
+
+def classify_transport(raw: RawResponse, policy: FetchPolicy) -> TransportSignal:
+    req = urlparse(raw.url_requested)
+    fin = urlparse(raw.url_final)
+    downgrade = req.scheme == "https" and fin.scheme == "http"
+    cross = _registrable_domain((req.hostname or "").lower()) != _registrable_domain((fin.hostname or "").lower())
+    max_hit = len(raw.redirect_chain) >= policy.max_redirects
+
+    if downgrade:
+        return TransportSignal(TransportClass.FAIL, "downgrade", cross, max_hit)
+    if raw.transport_error is not None:
+        return TransportSignal(TransportClass.FAIL, raw.transport_error, cross, max_hit)
+    if raw.status is None:
+        return TransportSignal(TransportClass.FAIL, "no_status", cross, max_hit)
+    if raw.status in HARD_FAIL_STATUS:
+        return TransportSignal(TransportClass.FAIL, f"http_{raw.status}", cross, max_hit)
+    if raw.status in NO_BODY_STATUS:
+        return TransportSignal(TransportClass.AMBIGUOUS, "no_content", cross, max_hit)
+    if raw.status in OK_STATUS:
+        return TransportSignal(TransportClass.OK, "ok", cross, max_hit)
+    return TransportSignal(TransportClass.AMBIGUOUS, f"other_{raw.status}", cross, max_hit)
+
+
+# Layer C: content-validity fingerprint (downgrade-only; never FAILED/RETRIEVED)
+
+_CHALLENGE_URL_MARKERS = ("/cdn-cgi/challenge-platform/", "/_incapsula_resource")
+_CHALLENGE_BODY = {
+    "cloudflare": ("just a moment", "challenges.cloudflare.com/turnstile", "cf-chl"),
+    "datadome": ("geo.captcha-delivery.com", "datadome"),
+    "perimeterx": ("access to this page has been denied", "px-captcha"),
+    "akamai": ("reference #", "akamai reference"),
+}
+_SUSPECT_BODY = {
+    "consent": ("this site uses cookies", "__tcfapi", "we value your privacy"),
+    "paywall": ("subscribe to continue", "create a free account to read",
+                "to read the full", "already a subscriber"),
+    "soft_404": ("page not found", "404", "no longer exists", "cannot be found"),
+}
+
+
+def fingerprint_validity(raw: RawResponse) -> ValiditySignal:
+    headers = {k.lower(): (v or "").lower() for k, v in raw.headers.items()}
+    host = (urlparse(raw.url_final).hostname or "").lower()
+    path = (urlparse(raw.url_final).path or "").lower()
+    body = raw.body.decode("utf-8", errors="replace").lower()
+
+    # 1. headers (least rot-prone) -> challenge
+    if "cf-mitigated" in headers and "challenge" in headers.get("cf-mitigated", ""):
+        return ValiditySignal(ValiditySeverity.CHALLENGE, True, "cloudflare_header")
+    if "x-datadome" in headers or "datadome" in headers.get("set-cookie", ""):
+        return ValiditySignal(ValiditySeverity.CHALLENGE, True, "datadome_header")
+    if raw.status == 200 and "www-authenticate" in headers:
+        return ValiditySignal(ValiditySeverity.CHALLENGE, True, "www_authenticate_200")
+
+    # 2. final-url routes
+    if any(m in path for m in _CHALLENGE_URL_MARKERS):
+        return ValiditySignal(ValiditySeverity.CHALLENGE, True, "challenge_url")
+    if host.startswith(("login.", "accounts.", "sso.")):
+        return ValiditySignal(ValiditySeverity.CHALLENGE, True, "auth_redirect")
+    if host.startswith("consent."):
+        return ValiditySignal(ValiditySeverity.SUSPECT, True, "consent_host")
+
+    # 3. body signatures: scan all families, >=2 same-family markers -> challenge.
+    challenge_family = None
+    single_family = None
+    for family, markers in _CHALLENGE_BODY.items():
+        hits = sum(1 for m in markers if m in body)
+        if hits >= 2 and challenge_family is None:
+            challenge_family = family
+        elif hits == 1 and single_family is None:
+            single_family = family
+    if challenge_family is not None:
+        return ValiditySignal(ValiditySeverity.CHALLENGE, True, f"{challenge_family}_body")
+    if single_family is not None:
+        return ValiditySignal(ValiditySeverity.SUSPECT, True, f"{single_family}_single")
+    for family, markers in _SUSPECT_BODY.items():
+        if any(m in body for m in markers):
+            return ValiditySignal(ValiditySeverity.SUSPECT, True, f"{family}")
+
+    return ValiditySignal(ValiditySeverity.CLEAN, False, None)
+
+
+# Freshness
+
+def classify_freshness(
+    fetched_at_epoch: float,
+    required_max_age_seconds: Optional[int],
+    resource_timestamp_epoch: Optional[float],
+    resource_timestamp_iso: Optional[str],
+    resource_timestamp_source: Optional[str],
+) -> Freshness:
+    if required_max_age_seconds is None:
+        return Freshness(None, resource_timestamp_iso, resource_timestamp_source,
+                         None, FreshnessState.NOT_REQUIRED)
+    if resource_timestamp_epoch is None:
+        return Freshness(required_max_age_seconds, None, None, None, FreshnessState.UNKNOWN)
+    age = int(fetched_at_epoch - resource_timestamp_epoch)
+    state = FreshnessState.STALE if age > required_max_age_seconds else FreshnessState.FRESH
+    return Freshness(required_max_age_seconds, resource_timestamp_iso,
+                     resource_timestamp_source, age, state)
+
+
+# decide(): true minimum over all layers, then fixed reason precedence
+
+REASON_PRECEDENCE = [
+    "transport.", "redirect_boundary", "too_many_redirects",
+    "structured_empty", "empty_transport", "stale",
+    "empty_extract", "structured_parse", "unsupported_", "thin_ratio",
+    "freshness_unknown", "challenge_", "validity_suspect",
+]
+
+
+def _precedence(reason: str) -> int:
+    for i, prefix in enumerate(REASON_PRECEDENCE):
+        if reason.startswith(prefix):
+            return i
+    return len(REASON_PRECEDENCE)
+
+
+def _verdict_of(sev: int) -> Verdict:
+    for v, s in SEV.items():
+        if s == sev:
+            return v
+    return Verdict.UNKNOWN
+
+
+def decide(env: ProvenanceEnvelope, policy: FetchPolicy) -> GateResult:
+    contributions: list[tuple[int, str]] = []
+
+    # ---- Layer A: transport ----
+    t = env.signals.transport
+    if t.cls is TransportClass.FAIL:
+        contributions.append((SEV[Verdict.FAILED], "transport." + t.reason))
+    elif t.cls is TransportClass.AMBIGUOUS:
+        contributions.append((SEV[Verdict.UNKNOWN], "transport.no_content"))
+    if t.cross_registrable_domain_redirect and policy.block_cross_origin_redirect:
+        contributions.append((SEV[Verdict.FAILED], "redirect_boundary"))
+    if t.max_redirects_hit:
+        contributions.append((SEV[Verdict.FAILED], "too_many_redirects"))
+
+    # ---- Layer B: extraction ----
+    e = env.signals.extraction.cls
+    profile = policy.profile_for(env.content_type)
+    if e is ExtractionClass.STRUCTURED_EMPTY:
+        contributions.append((SEV[Verdict.FAILED], "structured_empty"))
+    elif e is ExtractionClass.EMPTY:
+        if env.raw_bytes < profile.min_raw_bytes:
+            contributions.append((SEV[Verdict.FAILED], "empty_transport"))
+        else:
+            contributions.append((SEV[Verdict.UNKNOWN], "empty_extract"))
+    elif e is ExtractionClass.THIN:
+        contributions.append((SEV[Verdict.UNKNOWN], "thin_ratio"))
+    elif e is ExtractionClass.STRUCTURED_PARSE_FAIL:
+        contributions.append((SEV[Verdict.UNKNOWN], "structured_parse"))
+    elif e is ExtractionClass.UNSUPPORTED:
+        if policy.unsupported_type_policy != "BYTES_OK":
+            contributions.append((SEV[Verdict.UNKNOWN], "unsupported_" + env.signals.extraction.profile))
+    # EXTRACT_OK contributes nothing
+
+    # ---- Freshness ----
+    f = env.freshness.age_vs_required_freshness
+    if f is FreshnessState.STALE:
+        contributions.append((SEV[Verdict.FAILED], "stale"))
+    elif f is FreshnessState.UNKNOWN:
+        contributions.append((SEV[Verdict.UNKNOWN], "freshness_unknown"))
+
+    # ---- Layer C: content-validity (downgrade-only) ----
+    c = env.signals.content_validity.severity
+    if c is ValiditySeverity.CHALLENGE:
+        fid = env.signals.content_validity.fingerprint_id or "generic"
+        contributions.append((SEV[Verdict.UNKNOWN], "challenge_" + fid))
+    elif c is ValiditySeverity.SUSPECT:
+        contributions.append((SEV[Verdict.UNKNOWN], "validity_suspect"))
+
+    # ---- compose: true minimum, then reason precedence ----
+    if not contributions:
+        return GateResult(Verdict.RETRIEVED, "clean")
+    min_sev = min(sev for sev, _ in contributions)
+    fired = [reason for sev, reason in contributions if sev == min_sev]
+    reason = min(fired, key=_precedence)
+    return GateResult(_verdict_of(min_sev), reason)

fetchgate/envelope.py

@@ -0,0 +1,83 @@
+"""Envelope serialization and hashing. Canonical JSON is bit-stable and holds no verdict."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+
+from .types import ProvenanceEnvelope
+
+
+def content_sha256(raw: bytes) -> str | None:
+    if not raw:
+        return None
+    return hashlib.sha256(raw).hexdigest()
+
+
+def extracted_sha256(text: str) -> str | None:
+    if not text:
+        return None
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def mint_handle(url: str) -> str:
+    """A non-forgeable, gate-issued handle."""
+    return hashlib.sha256(url.encode("utf-8") + os.urandom(16)).hexdigest()[:32]
+
+
+def envelope_to_dict(env: ProvenanceEnvelope) -> dict:
+    s = env.signals
+    return {
+        "envelope_version": env.envelope_version,
+        "runtime_version": env.runtime_version,
+        "request_id": env.request_id,
+        "url_requested": env.url_requested,
+        "url_final": env.url_final,
+        "redirect_chain": [
+            {"status": r.status, "from": r.from_url, "to": r.to_url}
+            for r in env.redirect_chain
+        ],
+        "http_status": env.http_status,
+        "fetched_at": env.fetched_at,
+        "content_type": env.content_type,
+        "raw_bytes": env.raw_bytes,
+        "extracted_bytes": env.extracted_bytes,
+        "renderer_used": env.renderer_used,
+        "truncated": env.truncated,
+        "content_sha256": env.content_sha256,
+        "hash_target": env.hash_target,
+        "extracted_sha256": env.extracted_sha256,
+        "policy_id": env.policy_id,
+        "policy_sha256": env.policy_sha256,
+        "freshness": {
+            "required_max_age_seconds": env.freshness.required_max_age_seconds,
+            "resource_timestamp": env.freshness.resource_timestamp,
+            "resource_timestamp_source": env.freshness.resource_timestamp_source,
+            "age_seconds": env.freshness.age_seconds,
+            "age_vs_required_freshness": env.freshness.age_vs_required_freshness.value,
+        },
+        "signals": {
+            "transport": {
+                "class": s.transport.cls.value,
+                "reason": s.transport.reason,
+                "cross_registrable_domain_redirect": s.transport.cross_registrable_domain_redirect,
+                "max_redirects_hit": s.transport.max_redirects_hit,
+            },
+            "extraction": {
+                "class": s.extraction.cls.value,
+                "profile": s.extraction.profile,
+                "extractor": s.extraction.extractor,
+            },
+            "content_validity": {
+                "severity": s.content_validity.severity.value,
+                "fingerprint_hit": s.content_validity.fingerprint_hit,
+                "fingerprint_id": s.content_validity.fingerprint_id,
+                "fingerprint_pack_version": s.content_validity.fingerprint_pack_version,
+            },
+        },
+    }
+
+
+def canonical_json(env: ProvenanceEnvelope) -> str:
+    return json.dumps(envelope_to_dict(env), sort_keys=True, separators=(",", ":"))

fetchgate/eval/__init__.py

@@ -0,0 +1 @@
+"""Deterministic offline evaluation: replay the spine, write and re-check the manifest."""