fetchgate 0.1.0__py3-none-any.whl

fetchgate/eval/run_eval.py

@@ -0,0 +1,122 @@
+"""The deterministic headline: run_eval writes the manifest, verify_headline re-checks it."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+
+from ..envelope import envelope_to_dict
+from ..gate import FetchGate
+from ..types import Verdict
+from ..verify import PolicyMismatch, recompute_verdict
+from .spine import FIXED_EPOCH, build_spine
+
+_CORPUS = os.path.join(os.path.dirname(__file__), "corpus")
+_MANIFEST = os.path.join(_CORPUS, "eval_manifest.json")
+
+
+def _canonical(d: dict) -> str:
+    return json.dumps(d, sort_keys=True, separators=(",", ":"))
+
+
+def _sha256(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def run_eval() -> dict:
+    policy, static, heavy, scenarios = build_spine()
+    gate = FetchGate(policy, static, heavy, clock=lambda: FIXED_EPOCH)
+
+    results = []
+    reads_admitted = non_reads = non_reads_answerable = 0
+    for sid, url, expected, bucket in scenarios:
+        outcome = gate.guarded_fetch(url, request_id="req-" + sid)
+        guard = gate.guard_answer(outcome.handle)
+        answerable = guard.verdict is Verdict.RETRIEVED
+        env_dict = envelope_to_dict(outcome.envelope)
+        verdict = outcome.result.verdict
+
+        if verdict is Verdict.RETRIEVED:
+            reads_admitted += 1
+        else:
+            non_reads += 1
+            if answerable:
+                non_reads_answerable += 1
+
+        results.append({
+            "id": sid, "url": url, "bucket": bucket,
+            "expected_verdict": expected,
+            "verdict": verdict.value,
+            "reason": outcome.result.reason,
+            "disposition": outcome.disposition.value,
+            "tiers": outcome.tiers,
+            "answerable": answerable,
+            "content_sha256": outcome.envelope.content_sha256,
+            "envelope": env_dict,
+            "envelope_sha256": _sha256(_canonical(env_dict)),
+        })
+
+    manifest = {
+        "manifest_version": "1",
+        "fetchgate_version": "0.1.0",
+        "clock_epoch": FIXED_EPOCH,
+        "scenarios": results,
+        "result": {
+            "reads_admitted": reads_admitted,
+            "non_reads": non_reads,
+            "non_reads_reaching_answerable_state": non_reads_answerable,
+        },
+        "headline": (
+            f"across {reads_admitted + non_reads} scenarios the gate admitted the "
+            f"{reads_admitted} true reads and refused the {non_reads} non-reads. "
+            f"{non_reads_answerable} non-reads reached answerable state (enforced by "
+            f"construction: the gate mints no handle unless the verdict is RETRIEVED)."
+        ),
+    }
+    return manifest
+
+
+def write_manifest(manifest: dict | None = None) -> str:
+    manifest = manifest or run_eval()
+    os.makedirs(_CORPUS, exist_ok=True)
+    with open(_MANIFEST, "w", encoding="utf-8") as f:
+        json.dump(manifest, f, indent=2, sort_keys=True)
+    return _MANIFEST
+
+
+def verify_headline(manifest: dict | None = None) -> list:
+    """Return a list of problems; empty means the manifest verifies."""
+    if manifest is None:
+        with open(_MANIFEST, encoding="utf-8") as f:
+            manifest = json.load(f)
+    policy = build_spine()[0]  # the policy that produced the receipt
+    problems = []
+    for sc in manifest["scenarios"]:
+        try:
+            recomputed = recompute_verdict(sc["envelope"], policy)
+        except PolicyMismatch as e:
+            problems.append(f"{sc['id']}: {e}")
+            continue
+        if recomputed.verdict.value != sc["verdict"]:
+            problems.append(f"{sc['id']}: verdict recompute {recomputed.verdict.value} != {sc['verdict']}")
+        if _sha256(_canonical(sc["envelope"])) != sc["envelope_sha256"]:
+            problems.append(f"{sc['id']}: envelope sha256 mismatch")
+        if sc["expected_verdict"] != sc["verdict"]:
+            problems.append(f"{sc['id']}: expected {sc['expected_verdict']} got {sc['verdict']}")
+    r = manifest["result"]
+    if r["non_reads_reaching_answerable_state"] != 0:
+        problems.append("a non-read reached answerable state")
+    return problems
+
+
+if __name__ == "__main__":
+    m = run_eval()
+    problems = verify_headline(m)
+    print(m["headline"])
+    if problems:
+        print("VERIFY FAILED:")
+        for p in problems:
+            print("  -", p)
+        raise SystemExit(1)
+    print("verify: OK (recomputed offline from stored signals)")

fetchgate/eval/spine.py

@@ -0,0 +1,124 @@
+"""The 9-scenario spine: byte-frozen fixtures and the domain policy."""
+
+from __future__ import annotations
+
+from ..config import DomainPolicy, FetchPolicy
+from ..transports import MockTransport
+from ..types import RawResponse
+
+FIXED_EPOCH = 1_783_000_000.0  # pinned clock so envelopes are bit-stable
+
+_ARTICLE_BODY = (
+    b"<html><head><title>Reachability</title></head><body>"
+    b"<nav>home about</nav>"
+    b"<article><h1>Most findings are noise</h1>"
+    b"<p>Open any vulnerability scanner on a real codebase and you get a wall of "
+    b"critical findings. Most of them are noise, because the vulnerable function "
+    b"sits in a dependency your code never actually calls.</p>"
+    b"<p>The finding is real. The risk, for your application, is not. A tool "
+    b"answers one question with confidence and leaves a second, more important "
+    b"question unanswered.</p>"
+    b"<p>Reachability is that second question: starting from the entry points, is "
+    b"there an actual path in the code that reaches the vulnerable function.</p></article>"
+    b"<footer>copyright</footer></body></html>"
+)
+
+_JS_SHELL = (
+    b"<html><head><title>Dashboard</title>"
+    b"<script src=\"/app.bundle.js\"></script></head>"
+    b"<body><div id=\"root\"></div>"
+    b"<script>window.__BOOT__=1;/* app renders here */</script></body></html>"
+)
+
+_JS_SHELL_RENDERED = (
+    b"<html><body><main><h1>Live dashboard</h1>"
+    b"<p>The dashboard finished rendering on the client. This is the real content "
+    b"that only exists after the JavaScript executed, and it is what a static "
+    b"reader would have missed entirely.</p>"
+    b"<p>An honest headless render executes the page's own code as a real browser "
+    b"tab would, then the gate re-classifies the rendered DOM.</p></main></body></html>"
+)
+
+_CF_CHALLENGE = (
+    b"<html><head><title>Just a moment...</title>"
+    b"<script src=\"https://challenges.cloudflare.com/turnstile/v0/api.js\"></script>"
+    b"</head><body><div class=\"cf-chl\"><h1>Just a moment...</h1>"
+    b"<span>Enable JavaScript and cookies to continue</span></div></body></html>"
+)
+
+_CONSENT = (
+    b"<html><body><div class=\"consent\"><h1>We value your privacy</h1>"
+    b"<p>This site uses cookies and similar technologies to store and access "
+    b"information on your device. We and our partners process personal data such "
+    b"as browsing behaviour and unique identifiers. You can accept, reject, or "
+    b"manage your preferences at any time. Your choices will be signalled to our "
+    b"partners and will not affect browsing data. Managing preferences may reduce "
+    b"the personalisation you see but will not stop essential cookies that keep "
+    b"the site working correctly for every visitor on every page today.</p>"
+    b"</div></body></html>"
+)
+
+_PAYWALL = (
+    b"<html><body><section class=\"paywall\"><h1>Subscribe to continue reading</h1>"
+    b"<p>You have reached your free article limit for this month. Subscribe to "
+    b"continue reading this article and get unlimited access to award-winning "
+    b"journalism, investigations, and analysis. Already a subscriber? Sign in to "
+    b"pick up where you left off. Our newsroom depends on readers like you to "
+    b"fund the reporting that holds power to account across the country and the "
+    b"wider world, every single day of the year without exception at all.</p>"
+    b"</section></body></html>"
+)
+
+_SOFT_404 = (
+    b"<html><head><title>Page not found</title></head>"
+    b"<body><h1>Page not found</h1><p>404</p></body></html>"
+)
+
+_CF_403 = (
+    b"<html><body><h1>Sorry, you have been blocked</h1>"
+    b"<p>Cloudflare Ray ID: reference #12345</p></body></html>"
+)
+
+
+def _r(url, status, ctype, body, error=None, headers=None):
+    h = {"Content-Type": ctype}
+    if headers:
+        h.update(headers)
+    return RawResponse(url_requested=url, url_final=url, status=status,
+                       headers=h, body=body, transport_error=error)
+
+
+def build_spine():
+    """Return (policy, static_transport, heavy_transport, scenarios)."""
+    static = {
+        "https://example.com/article": _r("https://example.com/article", 200, "text/html; charset=utf-8", _ARTICLE_BODY),
+        "https://api.example.com/price": _r("https://api.example.com/price", 200, "application/json", b'{"price":4.99,"currency":"USD"}'),
+        "https://walled.example.com/page": _r("https://walled.example.com/page", 200, "text/html", _CF_CHALLENGE, headers={"cf-mitigated": "challenge", "Server": "cloudflare"}),
+        "https://news.example.com/story": _r("https://news.example.com/story", 200, "text/html", _CONSENT),
+        "https://paper.example.com/premium": _r("https://paper.example.com/premium", 200, "text/html", _PAYWALL),
+        "https://shop.example.com/gone": _r("https://shop.example.com/gone", 200, "text/html", _SOFT_404),
+        "https://jsapp.example.com/dashboard": _r("https://jsapp.example.com/dashboard", 200, "text/html", _JS_SHELL),
+        "https://blocked.example.com/data": _r("https://blocked.example.com/data", 403, "text/html", _CF_403, headers={"Server": "cloudflare"}),
+        "https://slow.example.com/x": _r("https://slow.example.com/x", None, "", b"", error="timeout"),
+    }
+    heavy = {
+        # render tier: the challenge is still walled; the JS shell hydrates.
+        "https://walled.example.com/page": _r("https://walled.example.com/page", 200, "text/html", _CF_CHALLENGE, headers={"cf-mitigated": "challenge"}),
+        "https://jsapp.example.com/dashboard": _r("https://jsapp.example.com/dashboard", 200, "text/html", _JS_SHELL_RENDERED),
+    }
+    policy = FetchPolicy(domains={
+        "walled.example.com": DomainPolicy(mode="render_allowed"),
+        "jsapp.example.com": DomainPolicy(mode="render_allowed"),
+    })
+    scenarios = [
+        ("real_article", "https://example.com/article", "RETRIEVED", "control"),
+        ("price_json", "https://api.example.com/price", "RETRIEVED", "control"),
+        ("cloudflare_challenge", "https://walled.example.com/page", "UNKNOWN", "blocked"),
+        ("consent_wall", "https://news.example.com/story", "UNKNOWN", "blocked"),
+        ("paywall", "https://paper.example.com/premium", "UNKNOWN", "blocked"),
+        ("soft_404", "https://shop.example.com/gone", "UNKNOWN", "blocked"),
+        ("js_shell_render", "https://jsapp.example.com/dashboard", "RETRIEVED", "blocked"),
+        ("cloudflare_403", "https://blocked.example.com/data", "FAILED", "blocked"),
+        ("timeout", "https://slow.example.com/x", "FAILED", "blocked"),
+    ]
+    return policy, MockTransport(static), MockTransport(heavy, renderer_used="headless"), scenarios

fetchgate/extract.py

@@ -0,0 +1,116 @@
+"""Layer B: extraction. Turns a raw body into an ExtractionClass. Seam: extract_main_text."""
+
+from __future__ import annotations
+
+import json
+from html.parser import HTMLParser
+from typing import Tuple
+
+from .config import ExtractionProfile
+from .types import ExtractionClass
+
+EXTRACTOR_NAME = "stdlib-htmlparser@0.1"
+
+_BOILERPLATE = {"script", "style", "noscript", "template", "head", "title",
+                "nav", "footer", "header", "aside", "form", "svg"}
+
+
+class _MainText(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self._skip_depth = 0
+        self._chunks: list[str] = []
+
+    def handle_starttag(self, tag, attrs):
+        if tag in _BOILERPLATE:
+            self._skip_depth += 1
+
+    def handle_endtag(self, tag):
+        if tag in _BOILERPLATE and self._skip_depth > 0:
+            self._skip_depth -= 1
+
+    def handle_data(self, data):
+        if self._skip_depth == 0:
+            text = data.strip()
+            if text:
+                self._chunks.append(text)
+
+    def text(self) -> str:
+        return " ".join(self._chunks)
+
+
+def extract_main_text(body_text: str) -> str:
+    parser = _MainText()
+    try:
+        parser.feed(body_text)
+    except Exception:
+        return ""
+    return parser.text()
+
+
+def _decode(raw: bytes, content_type: str) -> str:
+    charset = "utf-8"
+    if content_type and "charset=" in content_type:
+        charset = content_type.split("charset=")[-1].split(";")[0].strip() or "utf-8"
+    try:
+        return raw.decode(charset, errors="replace")
+    except (LookupError, ValueError):
+        return raw.decode("utf-8", errors="replace")
+
+
+def classify_extraction(
+    raw: bytes, content_type: str, profile: ExtractionProfile,
+    unsupported_type_policy: str = "UNKNOWN",
+) -> Tuple[ExtractionClass, str]:
+    """Return (ExtractionClass, extracted_text)."""
+    raw_len = len(raw)
+
+    if profile.kind == "html":
+        text = extract_main_text(_decode(raw, content_type))
+        ext_len = len(text.encode("utf-8"))
+        if ext_len >= profile.sufficient_extracted_bytes:
+            return ExtractionClass.EXTRACT_OK, text          # low-ratio long article shield
+        if ext_len < profile.min_extracted_bytes:
+            return ExtractionClass.EMPTY, text               # JS shell / blank
+        ratio = ext_len / raw_len if raw_len else 0.0
+        if ratio >= profile.min_ratio:
+            return ExtractionClass.EXTRACT_OK, text
+        return ExtractionClass.THIN, text
+
+    if profile.kind == "structured":
+        stripped = raw.strip()
+        if not stripped:
+            return ExtractionClass.STRUCTURED_EMPTY, ""
+        try:
+            value = json.loads(stripped.decode("utf-8", errors="replace"))
+        except (json.JSONDecodeError, ValueError):
+            return ExtractionClass.STRUCTURED_PARSE_FAIL, ""
+        if value is None or value == "":
+            return ExtractionClass.STRUCTURED_EMPTY, ""
+        if value == {} or value == []:
+            return ExtractionClass.THIN, ""
+        text = json.dumps(value, separators=(",", ":"), sort_keys=True)
+        return ExtractionClass.EXTRACT_OK, text
+
+    if profile.kind == "xml":
+        try:
+            import xml.etree.ElementTree as ET
+            root = ET.fromstring(raw)
+        except Exception:
+            if not raw.strip():
+                return ExtractionClass.STRUCTURED_EMPTY, ""
+            return ExtractionClass.STRUCTURED_PARSE_FAIL, ""
+        texts = [(e.text or "").strip() for e in root.iter()]
+        joined = " ".join(t for t in texts if t)
+        if joined:
+            return ExtractionClass.EXTRACT_OK, joined
+        return ExtractionClass.THIN, ""
+
+    if profile.kind == "plaintext":
+        text = _decode(raw, content_type)
+        if len(text.strip()) > profile.min_text_chars:
+            return ExtractionClass.EXTRACT_OK, text.strip()
+        return ExtractionClass.EMPTY, ""
+
+    # binary / unmapped
+    return ExtractionClass.UNSUPPORTED, ""

fetchgate/fetch.py

@@ -0,0 +1,112 @@
+"""The fetch runtime: builds a ProvenanceEnvelope out-of-band from a RawResponse."""
+
+from __future__ import annotations
+
+import time
+import uuid
+from datetime import datetime, timezone
+from typing import Callable, Optional, Tuple
+
+from .config import FetchPolicy
+from .detect import classify_freshness, classify_transport, fingerprint_validity
+from .envelope import content_sha256, extracted_sha256
+from .extract import EXTRACTOR_NAME, classify_extraction
+from .transports import Transport
+from .types import (
+    ExtractionSignal, Freshness, ProvenanceEnvelope, RawResponse, Signals,
+)
+
+RUNTIME_VERSION = "fetchgate@0.1.0"
+ENVELOPE_VERSION = "1"
+
+
+def _iso(epoch: float) -> str:
+    dt = datetime.fromtimestamp(epoch, tz=timezone.utc)
+    return dt.strftime("%Y-%m-%dT%H:%M:%S.") + f"{int(dt.microsecond / 1000):03d}Z"
+
+
+def _content_type(headers: dict) -> Optional[str]:
+    for k, v in headers.items():
+        if k.lower() == "content-type":
+            return v
+    return None
+
+
+def build_envelope(
+    raw: RawResponse,
+    policy: FetchPolicy,
+    *,
+    now_epoch: float,
+    request_id: str,
+    required_max_age_seconds: Optional[int] = None,
+    resource_timestamp_epoch: Optional[float] = None,
+    resource_timestamp_iso: Optional[str] = None,
+    resource_timestamp_source: Optional[str] = None,
+) -> Tuple[ProvenanceEnvelope, str]:
+    content_type = _content_type(raw.headers)
+    raw_bytes = len(raw.body)
+    profile = policy.profile_for(content_type)
+
+    ecls, text = classify_extraction(raw.body, content_type or "", profile,
+                                     policy.unsupported_type_policy)
+    extracted_bytes = len(text.encode("utf-8"))
+    transport = classify_transport(raw, policy)
+    validity = fingerprint_validity(raw)
+
+    # Only a caller-supplied timestamp is trusted; the origin's Last-Modified is not.
+    if resource_timestamp_source is None and resource_timestamp_epoch is not None:
+        resource_timestamp_source = "caller_supplied"
+    freshness: Freshness = classify_freshness(
+        now_epoch, required_max_age_seconds, resource_timestamp_epoch,
+        resource_timestamp_iso, resource_timestamp_source,
+    )
+
+    signals = Signals(
+        transport=transport,
+        extraction=ExtractionSignal(ecls, profile.kind, EXTRACTOR_NAME),
+        content_validity=validity,
+    )
+
+    env = ProvenanceEnvelope(
+        envelope_version=ENVELOPE_VERSION,
+        runtime_version=RUNTIME_VERSION,
+        request_id=request_id,
+        url_requested=raw.url_requested,
+        url_final=raw.url_final,
+        redirect_chain=list(raw.redirect_chain),
+        http_status=raw.status,
+        fetched_at=_iso(now_epoch),
+        content_type=content_type,
+        raw_bytes=raw_bytes,
+        extracted_bytes=extracted_bytes,
+        renderer_used=raw.renderer_used,
+        truncated=raw.truncated,
+        content_sha256=content_sha256(raw.body),
+        hash_target="raw_body",
+        extracted_sha256=extracted_sha256(text),
+        policy_id=policy.policy_id,
+        policy_sha256=policy.sha256(),
+        freshness=freshness,
+        signals=signals,
+    )
+    return env, text
+
+
+def fetch(
+    url: str,
+    transport: Transport,
+    policy: FetchPolicy,
+    *,
+    clock: Callable[[], float] = time.time,
+    request_id: Optional[str] = None,
+    required_max_age_seconds: Optional[int] = None,
+    resource_timestamp_epoch: Optional[float] = None,
+) -> Tuple[ProvenanceEnvelope, str]:
+    raw = transport.fetch(url)
+    return build_envelope(
+        raw, policy,
+        now_epoch=clock(),
+        request_id=request_id or str(uuid.uuid4()),
+        required_max_age_seconds=required_max_age_seconds,
+        resource_timestamp_epoch=resource_timestamp_epoch,
+    )

fetchgate/gate.py

@@ -0,0 +1,100 @@
+"""The gate: fetch+register (chokepoint A), content_for, guard_answer (chokepoint B)."""
+
+from __future__ import annotations
+
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Callable, Optional
+
+from .config import FetchPolicy
+from .detect import decide
+from .envelope import mint_handle
+from .fetch import build_envelope
+from .transports import Transport
+from .types import GateDisposition, GateResult, ProvenanceEnvelope, Verdict
+
+
+@dataclass
+class GateOutcome:
+    result: GateResult
+    envelope: ProvenanceEnvelope
+    handle: Optional[str]
+    content: str            # gate-confirmed content, empty unless RETRIEVED
+    tiers: list = field(default_factory=list)
+
+    @property
+    def disposition(self) -> GateDisposition:
+        return self.result.disposition
+
+
+class FetchGate:
+    def __init__(
+        self,
+        policy: FetchPolicy,
+        transport: Transport,
+        heavy_fetch: Optional[Transport] = None,
+        *,
+        clock: Callable[[], float] = time.time,
+    ) -> None:
+        self._policy = policy
+        self._transport = transport
+        self._heavy = heavy_fetch
+        self._clock = clock
+        self._registry: dict[str, tuple[Verdict, str]] = {}
+
+    # ---- Chokepoint A ----
+    def guarded_fetch(self, url: str, *, request_id: Optional[str] = None,
+                      required_max_age_seconds: Optional[int] = None) -> GateOutcome:
+        dp = self._policy.domain_policy(url)
+        rid = request_id or str(uuid.uuid4())
+
+        if dp.mode == "always_hard_fail":
+            env, _ = self._build(self._transport, url, rid, required_max_age_seconds)
+            return self._register(GateResult(Verdict.FAILED, "policy_denylist"), env, "", ["denylist"])
+
+        # Tier 0: static reader
+        env, text = self._build(self._transport, url, rid, required_max_age_seconds)
+        result = decide(env, self._policy)
+        tiers = ["static"]
+        if result.verdict is Verdict.RETRIEVED:
+            return self._register(result, env, text, tiers)
+
+        # Tier 1: headless render when a static read looks thin or empty (maybe JS)
+        if (result.verdict is Verdict.UNKNOWN and result.reason in ("empty_extract", "thin_ratio")
+                and dp.mode == "render_allowed" and self._heavy is not None):
+            env2, text2 = self._build(self._heavy, url, rid, required_max_age_seconds)
+            result2 = decide(env2, self._policy)
+            tiers.append("headless")
+            if result2.verdict is Verdict.RETRIEVED:
+                return self._register(result2, env2, text2, tiers)
+            return self._register(result2, env2, "", tiers)
+
+        return self._register(result, env, "", tiers)
+
+    def _build(self, transport: Transport, url: str, rid: str,
+               max_age: Optional[int]) -> tuple[ProvenanceEnvelope, str]:
+        raw = transport.fetch(url)
+        return build_envelope(raw, self._policy, now_epoch=self._clock(),
+                              request_id=rid, required_max_age_seconds=max_age)
+
+    def _register(self, result: GateResult, env: ProvenanceEnvelope,
+                  content: str, tiers: list) -> GateOutcome:
+        handle = mint_handle(env.url_requested) if result.verdict is Verdict.RETRIEVED else None
+        if handle is not None:
+            self._registry[handle] = (result.verdict, content)
+        return GateOutcome(result, env, handle, content if result.verdict is Verdict.RETRIEVED else "", tiers)
+
+    # ---- content the model may cite ----
+    def content_for(self, handle: Optional[str]) -> Optional[str]:
+        entry = self._registry.get(handle or "")
+        if entry is None or entry[0] is not Verdict.RETRIEVED:
+            return None
+        return entry[1]
+
+    # ---- Chokepoint B (egress) ----
+    def guard_answer(self, handle: Optional[str]) -> GateResult:
+        entry = self._registry.get(handle or "")
+        if entry is not None and entry[0] is Verdict.RETRIEVED:
+            return GateResult(Verdict.RETRIEVED, "clean")
+        return GateResult(Verdict.FAILED, "no_confirmed_read")

fetchgate/mcp_server.py

@@ -0,0 +1,55 @@
+"""FetchGate as an MCP server: a fetch tool that returns content only for a confirmed read."""
+
+from __future__ import annotations
+
+from .app import default_gate
+from .gate import FetchGate
+from .types import Verdict
+
+MAX_CHARS = 20000
+
+
+def gated_fetch_text(gate: FetchGate, url: str, max_chars: int = MAX_CHARS):
+    """Run one URL through the gate. Returns (tool_text, is_read)."""
+    out = gate.guarded_fetch(url)
+    content = gate.content_for(out.handle)
+    if content is None:
+        if out.result.verdict is Verdict.FAILED:
+            what = "could not fetch this page"
+        else:
+            what = "fetched this page but could not confirm readable content (it may be a JavaScript-rendered page)"
+        text = (
+            f"NOT READ: {url} ({out.result.verdict.value}, {out.result.reason}). "
+            f"FetchGate {what}. "
+            f"Do not answer as if you read it; tell the user you could not access it."
+        )
+        return text, False
+    body = content[:max_chars]
+    tail = " [truncated]" if len(content) > max_chars else ""
+    text = f"RETRIEVED {url} ({out.envelope.extracted_bytes} bytes){tail}\n\n{body}"
+    return text, True
+
+
+def build_server(gate: FetchGate | None = None):
+    """Build the FastMCP server. Requires the optional [mcp] extra."""
+    from mcp.server.fastmcp import FastMCP
+
+    gate = gate or default_gate()
+    server = FastMCP("fetchgate")
+
+    @server.tool()
+    def fetch_url(url: str) -> str:
+        """Fetch a URL and return its content ONLY if FetchGate confirms the page was actually read.
+        On a blocked, empty, or failed fetch it returns a refusal instead of content, so you never
+        answer as if you read a page you did not retrieve."""
+        return gated_fetch_text(gate, url)[0]
+
+    return server
+
+
+def main() -> None:
+    build_server().run()
+
+
+if __name__ == "__main__":
+    main()