fetchgate 0.1.0__py3-none-any.whl

fetchgate/transports.py

@@ -0,0 +1,106 @@
+"""Transports return only a RawResponse, never a verdict."""
+
+from __future__ import annotations
+
+from typing import Protocol
+from urllib.parse import urlparse
+
+from .types import RawResponse
+
+
+class Transport(Protocol):
+    def fetch(self, url: str) -> RawResponse: ...
+
+
+class MockTransport:
+    """Replays captured fixtures; raises on an unknown URL."""
+
+    def __init__(self, fixtures: dict[str, RawResponse], renderer_used: str = "static") -> None:
+        self._fixtures = fixtures
+        self._renderer_used = renderer_used
+
+    def fetch(self, url: str) -> RawResponse:
+        if url not in self._fixtures:
+            raise KeyError(f"MockTransport: no fixture for {url!r}")
+        raw = self._fixtures[url]
+        raw.renderer_used = self._renderer_used
+        return raw
+
+
+class HttpTransport:
+    """Static-tier transport over stdlib urllib; production swaps in httpx."""
+
+    def __init__(self, timeout: float = 10.0, max_bytes: int = 5_000_000) -> None:
+        self._timeout = timeout
+        self._max_bytes = max_bytes
+
+    def fetch(self, url: str) -> RawResponse:
+        import urllib.error
+        import urllib.request
+
+        scheme = urlparse(url).scheme
+        if scheme not in ("http", "https"):
+            raise ValueError(f"unsupported scheme: {scheme!r}")
+
+        req = urllib.request.Request(url, headers={"User-Agent": "FetchGate/0.1 (+honest)"})
+        try:
+            with urllib.request.urlopen(req, timeout=self._timeout) as resp:
+                body = resp.read(self._max_bytes + 1)
+                truncated = len(body) > self._max_bytes
+                if truncated:
+                    body = body[: self._max_bytes]
+                final = resp.geturl()
+                return RawResponse(
+                    url_requested=url,
+                    url_final=final,
+                    status=resp.status,
+                    headers={k: v for k, v in resp.headers.items()},
+                    body=body,
+                    redirect_chain=[],  # urllib does not expose the chain
+                    transport_error=None,
+                    renderer_used="static",
+                    truncated=truncated,
+                )
+        except urllib.error.HTTPError as e:
+            body = e.read() if hasattr(e, "read") else b""
+            return RawResponse(url, e.geturl() if hasattr(e, "geturl") else url,
+                               e.code, {k: v for k, v in (e.headers or {}).items()},
+                               body, renderer_used="static")
+        except urllib.error.URLError as e:
+            reason = str(getattr(e, "reason", "")).lower()
+            err = "timeout" if "timed out" in reason else "dns" if "name" in reason else "reset"
+            return RawResponse(url, url, None, {}, b"", transport_error=err, renderer_used="static")
+
+
+class BrowserTransport:
+    """Headless render tier (Playwright). Honest: default UA, no stealth."""
+
+    def __init__(self, timeout: float = 20.0, settle_ms: int = 2000, max_bytes: int = 5_000_000) -> None:
+        self._timeout = timeout
+        self._settle = settle_ms
+        self._max_bytes = max_bytes
+
+    def fetch(self, url: str) -> RawResponse:
+        if urlparse(url).scheme not in ("http", "https"):
+            raise ValueError(f"unsupported scheme: {urlparse(url).scheme!r}")
+        try:
+            from playwright.sync_api import sync_playwright
+        except ImportError:
+            return RawResponse(url, url, None, {}, b"", transport_error="browser_unavailable", renderer_used="headless")
+        try:
+            with sync_playwright() as p:
+                browser = p.chromium.launch(headless=True)
+                try:
+                    page = browser.new_page()
+                    resp = page.goto(url, wait_until="networkidle", timeout=int(self._timeout * 1000))
+                    page.wait_for_timeout(self._settle)
+                    html = page.content().encode("utf-8")[: self._max_bytes]
+                    headers = dict(resp.headers) if resp else {}
+                    if not any(k.lower() == "content-type" for k in headers):
+                        headers["Content-Type"] = "text/html; charset=utf-8"
+                    return RawResponse(url, page.url, resp.status if resp else None,
+                                       headers, html, renderer_used="headless")
+                finally:
+                    browser.close()
+        except Exception:
+            return RawResponse(url, url, None, {}, b"", transport_error="render_error", renderer_used="headless")

fetchgate/types.py

@@ -0,0 +1,150 @@
+"""Enums and frozen dataclasses for envelopes, signals, and gate results."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Optional
+
+
+class Verdict(str, Enum):
+    RETRIEVED = "RETRIEVED"
+    FAILED = "FAILED"
+    UNKNOWN = "UNKNOWN"
+
+
+class TransportClass(str, Enum):
+    OK = "OK"
+    FAIL = "FAIL"
+    AMBIGUOUS = "AMBIGUOUS"
+
+
+class ExtractionClass(str, Enum):
+    EXTRACT_OK = "EXTRACT_OK"
+    EMPTY = "EMPTY"
+    THIN = "THIN"
+    UNSUPPORTED = "UNSUPPORTED"
+    STRUCTURED_EMPTY = "STRUCTURED_EMPTY"
+    STRUCTURED_PARSE_FAIL = "STRUCTURED_PARSE_FAIL"
+
+
+class ValiditySeverity(str, Enum):
+    CLEAN = "clean"
+    CHALLENGE = "challenge"
+    SUSPECT = "suspect"
+
+
+class FreshnessState(str, Enum):
+    FRESH = "FRESH"
+    STALE = "STALE"
+    UNKNOWN = "UNKNOWN"
+    NOT_REQUIRED = "NOT_REQUIRED"
+
+
+class GateDisposition(str, Enum):
+    ALLOW = "ALLOW"
+    STOP = "STOP"
+
+
+# Severity lattice: FAILED < UNKNOWN < RETRIEVED. decide() returns the minimum.
+SEV = {Verdict.FAILED: 0, Verdict.UNKNOWN: 1, Verdict.RETRIEVED: 2}
+
+
+@dataclass(frozen=True)
+class Redirect:
+    status: int
+    from_url: str
+    to_url: str
+
+
+@dataclass
+class RawResponse:
+    """What a transport returns. Never a verdict; the runtime rebuilds the envelope."""
+
+    url_requested: str
+    url_final: str
+    status: Optional[int]
+    headers: dict
+    body: bytes
+    redirect_chain: list = field(default_factory=list)
+    transport_error: Optional[str] = None  # None | dns | timeout | reset | tls | too_many_redirects | redirect_loop | downgrade
+    elapsed_ms: int = 0
+    renderer_used: str = "static"  # static | headless
+    truncated: bool = False
+
+    @property
+    def ok(self) -> bool:
+        return self.transport_error is None
+
+
+@dataclass(frozen=True)
+class TransportSignal:
+    cls: TransportClass
+    reason: str
+    cross_registrable_domain_redirect: bool = False
+    max_redirects_hit: bool = False
+
+
+@dataclass(frozen=True)
+class ExtractionSignal:
+    cls: ExtractionClass
+    profile: str
+    extractor: str
+
+
+@dataclass(frozen=True)
+class ValiditySignal:
+    severity: ValiditySeverity
+    fingerprint_hit: bool = False
+    fingerprint_id: Optional[str] = None
+    fingerprint_pack_version: str = "2026-07-01"
+
+
+@dataclass(frozen=True)
+class Signals:
+    transport: TransportSignal
+    extraction: ExtractionSignal
+    content_validity: ValiditySignal
+
+
+@dataclass(frozen=True)
+class Freshness:
+    required_max_age_seconds: Optional[int]
+    resource_timestamp: Optional[str]
+    resource_timestamp_source: Optional[str]
+    age_seconds: Optional[int]
+    age_vs_required_freshness: FreshnessState
+
+
+@dataclass(frozen=True)
+class ProvenanceEnvelope:
+    envelope_version: str
+    runtime_version: str
+    request_id: str
+    url_requested: str
+    url_final: str
+    redirect_chain: list
+    http_status: Optional[int]
+    fetched_at: str
+    content_type: Optional[str]
+    raw_bytes: int
+    extracted_bytes: int
+    renderer_used: str
+    truncated: bool
+    content_sha256: Optional[str]
+    hash_target: str
+    extracted_sha256: Optional[str]
+    policy_id: str
+    policy_sha256: str
+    freshness: Freshness
+    signals: Signals
+
+
+@dataclass(frozen=True)
+class GateResult:
+    verdict: Verdict
+    reason: str
+
+    @property
+    def disposition(self) -> GateDisposition:
+        return GateDisposition.ALLOW if self.verdict is Verdict.RETRIEVED else GateDisposition.STOP

fetchgate/verify.py

@@ -0,0 +1,75 @@
+"""Offline verifier: recompute the verdict from stored signals, bound to policy_sha256."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from .config import FetchPolicy
+from .detect import decide
+from .types import (
+    ExtractionClass, ExtractionSignal, Freshness, FreshnessState, GateResult,
+    ProvenanceEnvelope, Redirect, Signals, TransportClass, TransportSignal,
+    ValiditySeverity, ValiditySignal,
+)
+
+
+def envelope_from_dict(d: dict) -> ProvenanceEnvelope:
+    s = d["signals"]
+    fr = d["freshness"]
+    return ProvenanceEnvelope(
+        envelope_version=d["envelope_version"],
+        runtime_version=d["runtime_version"],
+        request_id=d["request_id"],
+        url_requested=d["url_requested"],
+        url_final=d["url_final"],
+        redirect_chain=[Redirect(r["status"], r["from"], r["to"]) for r in d["redirect_chain"]],
+        http_status=d["http_status"],
+        fetched_at=d["fetched_at"],
+        content_type=d["content_type"],
+        raw_bytes=d["raw_bytes"],
+        extracted_bytes=d["extracted_bytes"],
+        renderer_used=d["renderer_used"],
+        truncated=d["truncated"],
+        content_sha256=d["content_sha256"],
+        hash_target=d["hash_target"],
+        extracted_sha256=d["extracted_sha256"],
+        policy_id=d["policy_id"],
+        policy_sha256=d["policy_sha256"],
+        freshness=Freshness(
+            fr["required_max_age_seconds"], fr["resource_timestamp"],
+            fr["resource_timestamp_source"], fr["age_seconds"],
+            FreshnessState(fr["age_vs_required_freshness"]),
+        ),
+        signals=Signals(
+            transport=TransportSignal(
+                TransportClass(s["transport"]["class"]), s["transport"]["reason"],
+                s["transport"]["cross_registrable_domain_redirect"],
+                s["transport"]["max_redirects_hit"],
+            ),
+            extraction=ExtractionSignal(
+                ExtractionClass(s["extraction"]["class"]),
+                s["extraction"]["profile"], s["extraction"]["extractor"],
+            ),
+            content_validity=ValiditySignal(
+                ValiditySeverity(s["content_validity"]["severity"]),
+                s["content_validity"]["fingerprint_hit"],
+                s["content_validity"]["fingerprint_id"],
+                s["content_validity"]["fingerprint_pack_version"],
+            ),
+        ),
+    )
+
+
+class PolicyMismatch(ValueError):
+    """The supplied policy does not match the one recorded in the envelope."""
+
+
+def recompute_verdict(envelope_dict: dict, policy: Optional[FetchPolicy] = None) -> GateResult:
+    """Recompute the verdict; raises PolicyMismatch if policy != the receipt's policy."""
+    policy = policy or FetchPolicy()
+    recorded = envelope_dict.get("policy_sha256")
+    if recorded is not None and policy.sha256() != recorded:
+        raise PolicyMismatch(
+            f"policy_sha256 mismatch: envelope={recorded} supplied={policy.sha256()}"
+        )
+    return decide(envelope_from_dict(envelope_dict), policy)

fetchgate-0.1.0.dist-info/METADATA

@@ -0,0 +1,147 @@
+Metadata-Version: 2.4
+Name: fetchgate
+Version: 0.1.0
+Summary: A deterministic gate at the fetch boundary: a 200 is transport success, not a read.
+Project-URL: Homepage, https://github.com/MoAz06/FetchGate
+Project-URL: Repository, https://github.com/MoAz06/FetchGate
+Project-URL: Issues, https://github.com/MoAz06/FetchGate/issues
+Author: Mohamed Azahrioui
+License: MIT
+License-File: LICENSE
+Keywords: ai-agents,guardrails,llm,mcp,provenance,retrieval
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.11
+Provides-Extra: browser
+Requires-Dist: playwright>=1.44; extra == 'browser'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Provides-Extra: extract
+Requires-Dist: trafilatura>=1.9; extra == 'extract'
+Provides-Extra: mcp
+Requires-Dist: mcp>=1.2; extra == 'mcp'
+Provides-Extra: sign
+Requires-Dist: cryptography>=42; extra == 'sign'
+Description-Content-Type: text/markdown
+
+# FetchGate
+
+**A 200 is not a read.**
+
+When an AI agent fetches a page and it quietly comes back empty (a JavaScript app,
+a 403, a Cloudflare wall), the agent often does not stop. It answers from memory
+and never tells you it did not actually read the page. FetchGate is a small gate
+at the fetch boundary that refuses to let that happen: the agent gets the content
+only when the page was really read, and an honest refusal otherwise.
+
+It is deterministic (no model, no API keys), stdlib-only at its core, and free.
+
+## Quickstart
+
+```bash
+git clone https://github.com/MoAz06/FetchGate
+cd FetchGate
+python demo.py https://example.com https://httpbin.org/status/403
+```
+
+```
+https://example.com
+  verdict     : RETRIEVED  (clean)
+  answerable  : ALLOW
+  bytes       : raw 559 / extracted 127
+  tiers       : static
+  gate-confirmed content (127 chars): Example Domain This domain is for use ...
+
+https://httpbin.org/status/403
+  verdict     : FAILED  (transport.http_403)
+  answerable  : STOP
+  refused: the model gets no content and cannot answer as if it read this page.
+```
+
+Run the tests (no network, no model):
+
+```bash
+pip install -e .
+python -m unittest discover -s tests
+```
+
+## Use it in Claude Desktop or Cursor
+
+```bash
+pip install ".[mcp]"                 # the fetch tool
+pip install ".[browser]"             # optional: render JavaScript pages
+python -m playwright install chromium
+```
+
+Add to your MCP config (details in [docs/mcp.md](docs/mcp.md)):
+
+```json
+{ "mcpServers": { "fetchgate": { "command": "fetchgate-mcp" } } }
+```
+
+Your agent now has a `fetch_url` tool that returns content only for a confirmed
+read, and a refusal otherwise.
+
+## Use it in your own agent
+
+```python
+from fetchgate import default_gate
+
+gate = default_gate()   # renders JS pages if the [browser] extra is installed
+
+def read_url(url: str) -> str:
+    out = gate.guarded_fetch(url)
+    content = gate.content_for(out.handle)
+    if content is None:
+        return f"NOT READ: {url} ({out.result.reason}). Do not answer as if you read it."
+    return content
+```
+
+## How it decides
+
+Every fetch gets one verdict:
+
+- **RETRIEVED**: a real read. The agent may answer and cite it.
+- **FAILED**: a definite non-read (a 4xx or 5xx, a timeout, an https to http
+  downgrade, an empty body). Stop.
+- **UNKNOWN**: fetched, but not enough readable content to confirm a read (often a
+  JavaScript page). Stop, or render and try again.
+
+Three layers feed the verdict, and the result is the strictest of them:
+
+- **transport**: status and redirect chain.
+- **extraction**: readable text kept separate from raw bytes. A 17 byte price JSON
+  is a read, a 400 KB article at 2 percent text is a read, a JS shell with no text
+  is not.
+- **content-validity**: a best-effort fingerprint for Cloudflare, consent,
+  paywall, and soft-404 walls. It can only downgrade to UNKNOWN, never fake a read.
+
+When a page looks thin or empty and the render tier is on, FetchGate escalates to
+a real headless browser and re-checks. It renders, it does not evade: a page still
+walled after an honest render stays a non-read.
+
+## Stated limits
+
+- `content_sha256` is a receipt anchor, not tamper-evidence. A cloaked page hashes
+  cleanly.
+- Provenance is not authenticity. The gate closes fail-silent-on-empty, not
+  fetch-succeeds-on-wrong-content.
+- A read is not an answer. RETRIEVED means enough content arrived, not that it held
+  the fact you need. Claim support is a downstream job.
+- Fail-closed has an availability cost. Anyone who can make your fetch fail can
+  force a refusal. That is the trade for correctness.
+
+## What is built
+
+The deterministic core, the offline evaluation with a signed manifest, the
+build-breaking cheat-test, the MCP server, and the Playwright render tier. Optional
+extras: `[mcp]`, `[browser]`, `[extract]` (trafilatura), `[sign]` (Ed25519).
+
+MIT licensed.

fetchgate-0.1.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+fetchgate/__init__.py,sha256=JVXm2Yreodh_dpAdModgMEoY0ck_Ud_WyEJdkJVwgSc,878
+fetchgate/app.py,sha256=QkTJktb3a2JXnXe-ur0UTgi6_G12KrmlBtHvtiu1jKI,657
+fetchgate/config.py,sha256=yKzfVcwmPhkuszxF705aY2rHWnEFY7NlDOPd9JkOG5U,3694
+fetchgate/detect.py,sha256=wnjEYgxGi2lV1MIC1oHX_K1QWUoNsdAZ20TE86I3-ww,9051
+fetchgate/envelope.py,sha256=zzjQfWSv5EH-DkIJSYnMfi_-zhXmGH2txDSfNhUg5m0,3018
+fetchgate/extract.py,sha256=y_6jHmwHOcmofaa_5WqL7yBRtJsc-ObLKP_zKC4TkL4,3984
+fetchgate/fetch.py,sha256=oZhtzd9e5N5TGjCHfxHJHImt-c4rZ7StOXrSS21-TyQ,3724
+fetchgate/gate.py,sha256=MnbfK0nDTWj9pJkNuFtjK345-U3G_mzGDr9saGqjovw,4075
+fetchgate/mcp_server.py,sha256=t_FtAmbUJUtuTCvebT2SI4adfwerWWVBER2wg1Hc6E4,1875
+fetchgate/transports.py,sha256=YBJ5XFpYE0vPL2AI_rX1zzI4WFqNxXMtXDiL-QZBVs8,4527
+fetchgate/types.py,sha256=ZLUoQV4SK4CWz0wz3dWhOhu3U8mpTmimsGu-NQjc3aU,3450
+fetchgate/verify.py,sha256=fqHuYGTImBVxtpf2Jl_izeaK37vWuIkYxZ8vz8xSnLE,3017
+fetchgate/eval/__init__.py,sha256=09j08Q6i9rs0kMIfPxhT8bPV96PAEzpbY3VpHTjDW54,91
+fetchgate/eval/run_eval.py,sha256=kGMRxZ4rBMZ6eCH-1V3Uz2BiDqBraVoZj6lGRmtmWT4,4392
+fetchgate/eval/spine.py,sha256=gSWLCNMNeKeBYeE-KIJNTk4s5FLjAlmqIdb_HREMK44,6626
+fetchgate/eval/corpus/eval_manifest.json,sha256=hK-Ku3cCQWL3aUwbZqiOsVtk4vOyYvZ0pEivtsJyShU,20712
+fetchgate/eval/corpus/sample_envelope.json,sha256=WqxSZJDE6uwXjznu-Ja7Xv_pp9Ehc8XcXOu04gVTiqo,1482
+fetchgate-0.1.0.dist-info/METADATA,sha256=vxi39xcs2ukxa-ea4A4lv9G4qdDx8Yp9JC5GS8aM3h4,5132
+fetchgate-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+fetchgate-0.1.0.dist-info/entry_points.txt,sha256=SlR-YZR_sOGbvespC0Ihk-vHC81XHmbSwe3QkvInGlY,60
+fetchgate-0.1.0.dist-info/licenses/LICENSE,sha256=8pw6L2DaNOig8vyqHsnRamv51zKosOBeA9SZA9wzrZQ,1074
+fetchgate-0.1.0.dist-info/RECORD,,

fetchgate-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

fetchgate-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+fetchgate-mcp = fetchgate.mcp_server:main

fetchgate-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mohamed Azahrioui
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.