fetchgate 0.1.0__tar.gz → 0.2.0__tar.gz

fetchgate-0.2.0/.claude-plugin/marketplace.json

@@ -0,0 +1,17 @@
+{
+  "name": "fetchgate",
+  "description": "FetchGate: a deterministic gate at the fetch boundary for AI agents.",
+  "owner": { "name": "Mohamed Azahrioui" },
+  "plugins": [
+    {
+      "name": "fetchgate",
+      "source": "./",
+      "description": "The agent only answers from a page it actually read.",
+      "version": "0.1.0",
+      "homepage": "https://github.com/MoAz06/FetchGate",
+      "repository": "https://github.com/MoAz06/FetchGate",
+      "license": "MIT",
+      "keywords": ["mcp", "ai-agents", "guardrails"]
+    }
+  ]
+}

fetchgate-0.2.0/.claude-plugin/plugin.json

@@ -0,0 +1,16 @@
+{
+  "name": "fetchgate",
+  "displayName": "FetchGate",
+  "description": "A gate at the fetch boundary: the agent only answers from a page it actually read.",
+  "version": "0.1.0",
+  "author": { "name": "Mohamed Azahrioui" },
+  "homepage": "https://github.com/MoAz06/FetchGate",
+  "repository": "https://github.com/MoAz06/FetchGate",
+  "license": "MIT",
+  "keywords": ["mcp", "ai-agents", "retrieval", "guardrails"],
+  "mcpServers": {
+    "fetchgate": {
+      "command": "fetchgate-mcp"
+    }
+  }
+}

{fetchgate-0.1.0 → fetchgate-0.2.0}/.gitignore

@@ -7,3 +7,4 @@ __pycache__/
 dist/
 build/
 .venv/
+*.mcpb

{fetchgate-0.1.0 → fetchgate-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fetchgate
-Version: 0.1.0
+Version: 0.2.0
 Summary: A deterministic gate at the fetch boundary: a 200 is transport success, not a read.
 Project-URL: Homepage, https://github.com/MoAz06/FetchGate
 Project-URL: Repository, https://github.com/MoAz06/FetchGate
@@ -25,6 +25,8 @@ Requires-Dist: pytest>=8; extra == 'dev'
 Requires-Dist: ruff>=0.5; extra == 'dev'
 Provides-Extra: extract
 Requires-Dist: trafilatura>=1.9; extra == 'extract'
+Provides-Extra: http
+Requires-Dist: httpx>=0.27; extra == 'http'
 Provides-Extra: mcp
 Requires-Dist: mcp>=1.2; extra == 'mcp'
 Provides-Extra: sign
@@ -72,10 +74,32 @@ pip install -e .
 python -m unittest discover -s tests
 ```
 
-## Use it in Claude Desktop or Cursor
+## Easiest: one double-click (Claude Desktop)
+
+Download `fetchgate.mcpb` from the [Releases](https://github.com/MoAz06/FetchGate/releases)
+page and double-click it. Claude Desktop shows an Install button, no JSON editing.
+The bundle source is in [bundle/](bundle); rebuild it with
+`npx @anthropic-ai/mcpb pack bundle fetchgate.mcpb`.
+
+## Vibecoders: add it from inside your editor
+
+**Cursor, one click** (needs `uv`, which most setups have):
+
+[Add FetchGate to Cursor](cursor://anysphere.cursor-deeplink/mcp/install?name=fetchgate&config=eyJjb21tYW5kIjogInV2eCIsICJhcmdzIjogWyItLWZyb20iLCAiZmV0Y2hnYXRlW21jcF0iLCAiZmV0Y2hnYXRlLW1jcCJdfQ==)
+
+**Or just ask your AI.** Paste this into Cursor, Claude Code, or Windsurf:
+
+> Add the fetchgate MCP server to this setup. Run `pip install "fetchgate[mcp]"`
+> and register it as an MCP server named `fetchgate` with the command
+> `fetchgate-mcp`. It stops an agent from answering as if it read a page it could
+> not actually fetch.
+
+Your assistant runs the install and wires up the config for you.
+
+## Use it in Claude Desktop or Cursor (manual)
 
 ```bash
-pip install ".[mcp]"                 # the fetch tool
+pip install ".[mcp,http,extract]"    # the fetch tool, real redirect chain, better extraction
 pip install ".[browser]"             # optional: render JavaScript pages
 python -m playwright install chromium
 ```
@@ -89,6 +113,15 @@ Add to your MCP config (details in [docs/mcp.md](docs/mcp.md)):
 Your agent now has a `fetch_url` tool that returns content only for a confirmed
 read, and a refusal otherwise.
 
+This repo is also a Claude Code plugin. After `pip install "fetchgate[mcp]"`:
+
+```bash
+/plugin marketplace add MoAz06/FetchGate
+/plugin install fetchgate@fetchgate
+```
+
+See [docs/claude-code-plugin.md](docs/claude-code-plugin.md).
+
 ## Use it in your own agent
 
 ```python
@@ -142,6 +175,8 @@ walled after an honest render stays a non-read.
 
 The deterministic core, the offline evaluation with a signed manifest, the
 build-breaking cheat-test, the MCP server, and the Playwright render tier. Optional
-extras: `[mcp]`, `[browser]`, `[extract]` (trafilatura), `[sign]` (Ed25519).
+extras: `[mcp]`, `[http]` (httpx redirect chain), `[extract]` (trafilatura),
+`[browser]` (Playwright), `[sign]` (Ed25519). When installed they are used
+automatically; the core stays stdlib-only.
 
 MIT licensed.

{fetchgate-0.1.0 → fetchgate-0.2.0}/README.md

@@ -39,10 +39,32 @@ pip install -e .
 python -m unittest discover -s tests
 ```
 
-## Use it in Claude Desktop or Cursor
+## Easiest: one double-click (Claude Desktop)
+
+Download `fetchgate.mcpb` from the [Releases](https://github.com/MoAz06/FetchGate/releases)
+page and double-click it. Claude Desktop shows an Install button, no JSON editing.
+The bundle source is in [bundle/](bundle); rebuild it with
+`npx @anthropic-ai/mcpb pack bundle fetchgate.mcpb`.
+
+## Vibecoders: add it from inside your editor
+
+**Cursor, one click** (needs `uv`, which most setups have):
+
+[Add FetchGate to Cursor](cursor://anysphere.cursor-deeplink/mcp/install?name=fetchgate&config=eyJjb21tYW5kIjogInV2eCIsICJhcmdzIjogWyItLWZyb20iLCAiZmV0Y2hnYXRlW21jcF0iLCAiZmV0Y2hnYXRlLW1jcCJdfQ==)
+
+**Or just ask your AI.** Paste this into Cursor, Claude Code, or Windsurf:
+
+> Add the fetchgate MCP server to this setup. Run `pip install "fetchgate[mcp]"`
+> and register it as an MCP server named `fetchgate` with the command
+> `fetchgate-mcp`. It stops an agent from answering as if it read a page it could
+> not actually fetch.
+
+Your assistant runs the install and wires up the config for you.
+
+## Use it in Claude Desktop or Cursor (manual)
 
 ```bash
-pip install ".[mcp]"                 # the fetch tool
+pip install ".[mcp,http,extract]"    # the fetch tool, real redirect chain, better extraction
 pip install ".[browser]"             # optional: render JavaScript pages
 python -m playwright install chromium
 ```
@@ -56,6 +78,15 @@ Add to your MCP config (details in [docs/mcp.md](docs/mcp.md)):
 Your agent now has a `fetch_url` tool that returns content only for a confirmed
 read, and a refusal otherwise.
 
+This repo is also a Claude Code plugin. After `pip install "fetchgate[mcp]"`:
+
+```bash
+/plugin marketplace add MoAz06/FetchGate
+/plugin install fetchgate@fetchgate
+```
+
+See [docs/claude-code-plugin.md](docs/claude-code-plugin.md).
+
 ## Use it in your own agent
 
 ```python
@@ -109,6 +140,8 @@ walled after an honest render stays a non-read.
 
 The deterministic core, the offline evaluation with a signed manifest, the
 build-breaking cheat-test, the MCP server, and the Playwright render tier. Optional
-extras: `[mcp]`, `[browser]`, `[extract]` (trafilatura), `[sign]` (Ed25519).
+extras: `[mcp]`, `[http]` (httpx redirect chain), `[extract]` (trafilatura),
+`[browser]` (Playwright), `[sign]` (Ed25519). When installed they are used
+automatically; the core stays stdlib-only.
 
 MIT licensed.

fetchgate-0.2.0/bundle/manifest.json

@@ -0,0 +1,17 @@
+{
+  "manifest_version": "0.2",
+  "name": "fetchgate",
+  "display_name": "FetchGate",
+  "version": "0.1.0",
+  "description": "A gate at the fetch boundary: the agent only answers from a page it actually read.",
+  "author": { "name": "Mohamed Azahrioui" },
+  "homepage": "https://github.com/MoAz06/FetchGate",
+  "server": {
+    "type": "python",
+    "entry_point": "server/main.py",
+    "mcp_config": {
+      "command": "python",
+      "args": ["${__dirname}/server/main.py"]
+    }
+  }
+}

fetchgate-0.2.0/bundle/requirements.txt

@@ -0,0 +1 @@
+fetchgate[mcp]

fetchgate-0.2.0/bundle/server/main.py

@@ -0,0 +1,4 @@
+from fetchgate.mcp_server import main
+
+if __name__ == "__main__":
+    main()

fetchgate-0.2.0/docs/claude-code-plugin.md

@@ -0,0 +1,43 @@
+# FetchGate as a Claude Code plugin
+
+This repo is also a Claude Code plugin. Installing it registers the `fetchgate`
+MCP server so Claude Code gets the `fetch_url` tool.
+
+## Prerequisite
+
+The plugin registers the server config, it does not install the Python package.
+So the `fetchgate-mcp` command has to be on your PATH first:
+
+```bash
+pip install "fetchgate[mcp]"
+pip install "fetchgate[browser]"        # optional: render JavaScript pages
+python -m playwright install chromium   # optional, for the render tier
+```
+
+## Install
+
+```bash
+/plugin marketplace add MoAz06/FetchGate
+/plugin install fetchgate@fetchgate
+/mcp                                     # confirm the fetchgate server is listed
+```
+
+Update later with:
+
+```bash
+/plugin marketplace update fetchgate
+/plugin update fetchgate@fetchgate
+```
+
+## One-click once it is on PyPI
+
+After publishing to PyPI you can drop the manual pip step by having the plugin run
+the server through `uvx` (needs `uv` installed). Change the command in
+`.claude-plugin/plugin.json` to:
+
+```json
+{ "command": "uvx", "args": ["--from", "fetchgate[mcp]", "fetchgate-mcp"] }
+```
+
+Then installing the plugin fetches and runs the server on first use, no separate
+pip install.

fetchgate-0.2.0/docs/config.md

@@ -0,0 +1,54 @@
+# Configuring FetchGate
+
+FetchGate reads a `fetchgate.toml` if it finds one, so you can set policy without
+touching code. This also configures the MCP server, since it uses `default_gate()`.
+
+## Where it looks
+
+In order:
+1. a path you pass to `load_policy(path)`
+2. `$FETCHGATE_CONFIG`
+3. `./fetchgate.toml`
+4. `~/.config/fetchgate/config.toml`
+
+If none exists, sensible defaults apply (render when the `[browser]` extra is
+installed, otherwise static only).
+
+## Example
+
+```toml
+default_mode = "render_allowed"        # never_render | render_allowed | always_hard_fail
+html_extractor = "auto"                # auto | stdlib | trafilatura
+unverified_disposition = "stop"        # stop | warn
+max_redirects = 5
+block_cross_origin_redirect = true
+unsupported_type_policy = "UNKNOWN"    # UNKNOWN | BYTES_OK
+
+[domains]
+"docs.mycompany.com" = "render_allowed"
+"api.stripe.com" = "never_render"
+"pinterest.com" = "always_hard_fail"
+```
+
+## Stop or warn on an unconfirmed read
+
+The verdict is always deterministic. `unverified_disposition` only decides what the
+MCP tool does with an `UNKNOWN` read (a fetch that arrived but could not be confirmed
+as readable, for example a thin or JavaScript-rendered page):
+
+- `stop` (default, fail-closed): return a refusal with no page content.
+- `warn`: return a warning banner plus whatever text was extracted, clearly labelled
+  as unverified, so the agent can proceed with caution instead of being blocked.
+
+A hard `FAILED` (transport error, 403, downgrade) always refuses regardless of this
+setting, because there is no content to hand over.
+
+See [fetchgate.example.toml](../fetchgate.example.toml).
+
+## In code
+
+```python
+from fetchgate import load_policy, FetchGate, HttpTransport
+
+gate = FetchGate(load_policy(), HttpTransport())
+```

fetchgate-0.2.0/docs/evaluation.md

@@ -0,0 +1,37 @@
+# FetchGate evaluation
+
+Two numbers, two tiers.
+
+## Headline (deterministic, offline, no model)
+
+```bash
+python -m fetchgate.eval.run_eval
+```
+
+Replays the 9-scenario spine through the gate and verifies it offline by
+recomputing every verdict from the stored signals (no re-extraction). Result:
+
+> the gate admitted the 3 true reads and refused the 6 non-reads. 0 non-reads
+> reached answerable state.
+
+This needs no model and is re-checkable by anyone.
+
+## Model exhibit (the author runs this once)
+
+```bash
+# deterministic proof of the harness (a mock model that always answers):
+python -m fetchgate.eval.tier_b
+
+# the real number (needs the anthropic SDK and ANTHROPIC_API_KEY):
+python -m fetchgate.eval.tier_b claude-<pinned-model-id>
+```
+
+Over the pages FetchGate calls non-reads, it measures how often an ungated agent
+answers anyway, versus a gated one:
+
+> ungated answered N of M non-reads; gated answered 0; and still answered every
+> real read.
+
+The gated 0 is by construction: on a non-read the gate refuses before the model
+is ever asked. Pin the exact dated model id and temperature 0.0 so the number is
+reproducible.

fetchgate-0.2.0/fetchgate.example.toml

@@ -0,0 +1,22 @@
+# FetchGate configuration. Copy to fetchgate.toml, or point $FETCHGATE_CONFIG at it,
+# or drop it at ~/.config/fetchgate/config.toml.
+
+# never_render | render_allowed | always_hard_fail
+default_mode = "render_allowed"
+
+# auto (trafilatura if installed) | stdlib | trafilatura
+html_extractor = "auto"
+
+# What the MCP tool does with an UNKNOWN read (thin or JS page):
+# stop = refuse with no content (fail-closed), warn = hand over the text with a warning.
+# A hard FAILED always refuses either way.
+unverified_disposition = "stop"        # stop | warn
+
+max_redirects = 5
+block_cross_origin_redirect = true
+unsupported_type_policy = "UNKNOWN"   # UNKNOWN | BYTES_OK
+
+[domains]
+"docs.mycompany.com" = "render_allowed"
+"api.stripe.com" = "never_render"      # structured JSON, rendering is pointless
+"pinterest.com" = "always_hard_fail"   # never even try

{fetchgate-0.1.0 → fetchgate-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "fetchgate"
-version = "0.1.0"
+version = "0.2.0"
 description = "A deterministic gate at the fetch boundary: a 200 is transport success, not a read."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -29,6 +29,7 @@ Issues = "https://github.com/MoAz06/FetchGate/issues"
 
 [project.optional-dependencies]
 mcp = ["mcp>=1.2"]                  # run FetchGate as a local MCP server
+http = ["httpx>=0.27"]             # real redirect chain for the static transport
 extract = ["trafilatura>=1.9"]     # production HTML main-text extractor (seam)
 browser = ["playwright>=1.44"]     # optional headless render tier
 sign = ["cryptography>=42"]        # optional Ed25519 manifest signing

fetchgate-0.2.0/smithery.yaml

@@ -0,0 +1,11 @@
+# Smithery config for FetchGate.
+# Runs the server locally over stdio via uvx, which fetches fetchgate from PyPI
+# on first use, so there is no manual pip step for the user.
+startCommand:
+  type: stdio
+  configSchema:
+    type: object
+    properties: {}
+    additionalProperties: false
+  commandFunction: |
+    (config) => ({ command: 'uvx', args: ['--from', 'fetchgate[mcp]', 'fetchgate-mcp'] })

{fetchgate-0.1.0 → fetchgate-0.2.0}/src/fetchgate/__init__.py

@@ -1,7 +1,7 @@
 """FetchGate: a deterministic gate at the fetch boundary. A 200 is not a read."""
 
 from .app import default_gate
-from .config import DomainPolicy, FetchPolicy
+from .config import DomainPolicy, FetchPolicy, load_policy
 from .detect import decide
 from .fetch import build_envelope, fetch
 from .gate import FetchGate, GateOutcome
@@ -11,10 +11,10 @@ from .types import (
 )
 from .verify import envelope_from_dict, recompute_verdict
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 
 __all__ = [
-    "default_gate", "FetchPolicy", "DomainPolicy", "decide", "fetch", "build_envelope",
+    "default_gate", "load_policy", "FetchPolicy", "DomainPolicy", "decide", "fetch", "build_envelope",
     "FetchGate", "GateOutcome", "Transport", "MockTransport", "HttpTransport",
     "BrowserTransport", "RawResponse", "ProvenanceEnvelope", "GateResult", "Verdict",
     "GateDisposition", "envelope_from_dict", "recompute_verdict", "__version__",

fetchgate-0.2.0/src/fetchgate/app.py

@@ -0,0 +1,21 @@
+"""Convenience factory: a gate from config, with the render tier if available."""
+
+from __future__ import annotations
+
+from .config import load_policy
+from .gate import FetchGate
+from .transports import BrowserTransport, HttpTransport
+
+
+def default_gate() -> FetchGate:
+    """A ready gate. Reads fetchgate.toml if present, renders JS pages when the
+    [browser] extra is installed (unless the config overrides default_mode)."""
+    heavy = None
+    fallback = "never_render"
+    try:
+        import playwright  # noqa: F401
+        heavy = BrowserTransport()
+        fallback = "render_allowed"
+    except ImportError:
+        pass
+    return FetchGate(load_policy(default_mode_fallback=fallback), HttpTransport(), heavy)

{fetchgate-0.1.0 → fetchgate-0.2.0}/src/fetchgate/config.py

@@ -36,6 +36,8 @@ class FetchPolicy:
     block_cross_origin_redirect: bool = True
     max_redirects: int = 5
     unsupported_type_policy: str = "UNKNOWN"
+    html_extractor: str = "auto"  # auto | stdlib | trafilatura
+    unverified_disposition: str = "stop"  # stop | warn: what an unconfirmed read does at the boundary
     domains: dict = field(default_factory=dict)  # registrable-domain -> DomainPolicy
 
     def profile_for(self, content_type: Optional[str]) -> ExtractionProfile:
@@ -72,12 +74,54 @@ class FetchPolicy:
             "block_cross_origin_redirect": self.block_cross_origin_redirect,
             "max_redirects": self.max_redirects,
             "unsupported_type_policy": self.unsupported_type_policy,
+            "html_extractor": self.html_extractor,
+            "unverified_disposition": self.unverified_disposition,
             "domains": {k: v.__dict__ for k, v in sorted(self.domains.items())},
         }
         raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
         return hashlib.sha256(raw).hexdigest()
 
 
+def _read_config(path: Optional[str]) -> Optional[dict]:
+    import os
+    import tomllib
+    candidates = [path, os.environ.get("FETCHGATE_CONFIG"), "fetchgate.toml",
+                  os.path.join(os.path.expanduser("~"), ".config", "fetchgate", "config.toml")]
+    for c in candidates:
+        if c and os.path.isfile(c):
+            with open(c, "rb") as f:
+                return tomllib.load(f)
+    return None
+
+
+def load_policy(path: Optional[str] = None, default_mode_fallback: str = "never_render") -> FetchPolicy:
+    """Build a FetchPolicy from a fetchgate.toml, or defaults if none is found.
+    Looked up in: path arg, $FETCHGATE_CONFIG, ./fetchgate.toml, ~/.config/fetchgate/config.toml."""
+    data = _read_config(path)
+    if data is None:
+        return FetchPolicy(default_mode=default_mode_fallback)
+    raw_domains = data.get("domains") or {}
+    domains = {
+        k: DomainPolicy(mode=(v if isinstance(v, str) else v.get("mode", "never_render")))
+        for k, v in raw_domains.items()
+    }
+    return FetchPolicy(
+        policy_id=data.get("policy_id", "config"),
+        default_mode=data.get("default_mode", default_mode_fallback),
+        block_cross_origin_redirect=bool(data.get("block_cross_origin_redirect", True)),
+        max_redirects=int(data.get("max_redirects", 5)),
+        unsupported_type_policy=data.get("unsupported_type_policy", "UNKNOWN"),
+        html_extractor=data.get("html_extractor", "auto"),
+        unverified_disposition=_disposition(data.get("unverified_disposition", "stop")),
+        domains=domains,
+    )
+
+
+def _disposition(value: object) -> str:
+    v = str(value).strip().lower()
+    return v if v in ("stop", "warn") else "stop"
+
+
 def _host_of(url: str) -> str:
     from urllib.parse import urlparse
     return (urlparse(url).hostname or "").lower()