ghost-api-cli 0.2.0__py3-none-any.whl

ghost_api/net.py

@@ -0,0 +1,235 @@
+"""Polite networking for requests the tool sends itself: rate limiting and robots.txt."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from urllib.parse import urlsplit, urlunsplit
+from urllib.robotparser import RobotFileParser
+
+import httpx
+
+from ghost_api.config import TOOL_NAME, tool_user_agent
+from ghost_api.netguard import (
+    NetworkPolicy,
+    PolicyDenied,
+    ResponseTooLarge,
+    guarded_client,
+    read_limited,
+)
+
+MAX_ROBOTS_BYTES = 512 * 1024
+
+
+class RateLimiter:
+    """Blocks so that calls to `wait()` happen at most `rate` times per second (NFR-18)."""
+
+    def __init__(self, rate: float) -> None:
+        if rate <= 0:
+            raise ValueError("rate must be greater than 0")
+        self.interval = 1.0 / rate
+        self._last = float("-inf")
+
+    def slow_down_to(self, min_interval: float) -> None:
+        self.interval = max(self.interval, min_interval)
+
+    def wait(self) -> None:
+        now = time.monotonic()
+        delay = self._last + self.interval - now
+        if delay > 0:
+            time.sleep(delay)
+        self._last = time.monotonic()
+
+
+@dataclass(frozen=True)
+class RobotsVerdict:
+    allowed: bool
+    reason: str
+    crawl_delay: float | None = None
+
+
+def robots_url(url: str) -> str:
+    parts = urlsplit(url)
+    return urlunsplit((parts.scheme, parts.netloc, "/robots.txt", "", ""))
+
+
+class RobotsPolicy:
+    """robots.txt rules for one host, reusable for every URL the agent visits (NFR-17)."""
+
+    def __init__(
+        self,
+        host: str,
+        parser: RobotFileParser | None,
+        default_allowed: bool,
+        reason: str,
+        agent: str = TOOL_NAME,
+    ) -> None:
+        self.host = host
+        self._parser = parser
+        self._default = default_allowed
+        self.reason = reason
+        self.agent = agent
+
+    @classmethod
+    def allow_all(cls, host: str, reason: str = "robots.txt check skipped") -> RobotsPolicy:
+        return cls(host, None, True, reason)
+
+    @classmethod
+    def from_text(cls, host: str, robots_txt: str, agent: str = TOOL_NAME) -> RobotsPolicy:
+        parser = RobotFileParser()
+        parser.parse(robots_txt.splitlines())
+        return cls(host, parser, True, "robots.txt rules", agent)
+
+    @property
+    def crawl_delay(self) -> float | None:
+        if self._parser is None:
+            return None
+        delay = self._parser.crawl_delay(self.agent)
+        return float(delay) if delay is not None else None
+
+    def allows(self, url: str) -> bool:
+        """Rules apply to the host they came from; other hosts are not checked here."""
+        if (urlsplit(url).hostname or "").lower() != self.host:
+            return True
+        if self._parser is None:
+            return self._default
+        return self._parser.can_fetch(self.agent, url)
+
+    def verdict(self, url: str) -> RobotsVerdict:
+        allowed = self.allows(url)
+        if self._parser is None:
+            reason = self.reason
+        else:
+            reason = "allowed by robots.txt" if allowed else "disallowed by robots.txt"
+        return RobotsVerdict(allowed, reason, self.crawl_delay)
+
+
+class MultiHostRobotsPolicy(RobotsPolicy):
+    """Lazily fetches and caches robots.txt per host, so subdomains are checked too (Issue 3).
+
+    The agent may navigate from ``www.example.com`` to ``api.example.com``.  A plain
+    ``RobotsPolicy`` only knows the rules for one host and returns ``True`` for any other.
+    This wrapper fetches and caches each host's robots.txt on first encounter.
+
+    It exposes the same ``allows(url)`` interface as ``RobotsPolicy`` so it's a drop-in
+    replacement wherever the orchestrator or pipeline calls ``robots.allows(url)``.
+    """
+
+    def __init__(
+        self,
+        seed: RobotsPolicy,
+        limiter: RateLimiter | None = None,
+        fetcher: Callable[[str, RateLimiter | None], RobotsPolicy] | None = None,
+    ) -> None:
+        super().__init__(seed.host, None, True, seed.reason, seed.agent)
+        self._cache: dict[str, RobotsPolicy] = {seed.host: seed}
+        self._limiter = limiter
+        self._fetch = fetcher or fetch_robots
+
+    def _for(self, url: str) -> RobotsPolicy:
+        host = (urlsplit(url).hostname or "").lower()
+        if host not in self._cache:
+            self._cache[host] = self._fetch(url, self._limiter)
+        return self._cache[host]
+
+    @property
+    def crawl_delay(self) -> float | None:
+        return self._cache[self.host].crawl_delay
+
+    def allows(self, url: str) -> bool:
+        """Check the correct host's robots.txt, fetching it lazily if needed."""
+        return self._for(url).allows(url)
+
+    def verdict(self, url: str) -> RobotsVerdict:
+        return self._for(url).verdict(url)
+
+
+class DenyPaths(RobotsPolicy):
+    """robots.txt plus the project's own rules (ghost-api.toml deny_paths) for one host:
+    a denied path is treated exactly like a robots.txt Disallow, everywhere."""
+
+    def __init__(self, inner: RobotsPolicy, deny: tuple[str, ...]) -> None:
+        super().__init__(inner.host, None, True, inner.reason, inner.agent)
+        self.inner = inner
+        self.deny = deny
+
+    @property
+    def crawl_delay(self) -> float | None:
+        return self.inner.crawl_delay
+
+    def _denied(self, url: str) -> bool:
+        parts = urlsplit(url)
+        if (parts.hostname or "").lower() != self.host:
+            return False
+        path = parts.path or "/"
+        return any(path == p or path.startswith(p.rstrip("/") + "/") for p in self.deny)
+
+    def allows(self, url: str) -> bool:
+        return not self._denied(url) and self.inner.allows(url)
+
+    def verdict(self, url: str) -> RobotsVerdict:
+        if self._denied(url):
+            return RobotsVerdict(False, "denied by the project rules (ghost-api.toml)")
+        return self.inner.verdict(url)
+
+
+def evaluate_robots(robots_txt: str, url: str, agent: str = TOOL_NAME) -> RobotsVerdict:
+    """Apply robots.txt rules to a URL. Pure function, so it's easy to test."""
+    host = (urlsplit(url).hostname or "").lower()
+    return RobotsPolicy.from_text(host, robots_txt, agent).verdict(url)
+
+
+def fetch_robots(
+    url: str,
+    limiter: RateLimiter | None = None,
+    policy: NetworkPolicy | None = None,
+    transport: httpx.BaseTransport | None = None,
+) -> RobotsPolicy:
+    """Fetch robots.txt, following RFC 9309 for missing or unreachable files."""
+    host = (urlsplit(url).hostname or "").lower()
+    if limiter:
+        limiter.wait()
+    # Redirects are followed (RFC 9309), but each hop is checked: a public site's
+    # robots.txt must not lead Ghost API to a private address.
+    client = guarded_client(
+        policy or NetworkPolicy.for_target(url),
+        follow_redirects=True,
+        timeout=10,
+        transport=transport,
+    )
+    try:
+        with (
+            client,
+            client.stream(
+                "GET", robots_url(url), headers={"User-Agent": tool_user_agent()}
+            ) as resp,
+        ):
+            text = read_limited(resp, MAX_ROBOTS_BYTES, MAX_ROBOTS_BYTES).text
+    except PolicyDenied as exc:
+        return RobotsPolicy(host, None, False, f"robots.txt {exc}")
+    except ResponseTooLarge:
+        # RFC 9309: at least 500 KiB must be parsed; a bigger file is treated as unreadable.
+        return RobotsPolicy(host, None, False, "robots.txt is too large to read")
+    except httpx.HTTPError as exc:
+        # Unreachable robots.txt: assume everything is disallowed.
+        return RobotsPolicy(host, None, False, f"robots.txt unreachable ({type(exc).__name__})")
+    if resp.status_code in (401, 403):
+        # The site refused to show its rules (Wikipedia does this to clients that don't
+        # identify themselves). Not knowing the rules is not permission.
+        return RobotsPolicy(
+            host,
+            None,
+            False,
+            f"robots.txt is forbidden (HTTP {resp.status_code}), so the site's rules can't be "
+            "read. Set GHOST_API_CONTACT in .env so the site can identify you",
+        )
+    if 400 <= resp.status_code < 500:
+        return RobotsPolicy(host, None, True, f"no robots.txt (HTTP {resp.status_code})")
+    if resp.status_code >= 500:
+        return RobotsPolicy(host, None, False, f"robots.txt server error (HTTP {resp.status_code})")
+    return RobotsPolicy.from_text(host, text)
+
+
+def check_robots(url: str, limiter: RateLimiter | None = None) -> RobotsVerdict:
+    return fetch_robots(url, limiter).verdict(url)

ghost_api/netguard.py

@@ -0,0 +1,203 @@
+"""Where Ghost API may connect: never to this computer or the local network by accident.
+
+A public website can redirect, or make its page call, an address such as 127.0.0.1,
+192.168.1.1 or the cloud metadata address 169.254.169.254. Ghost API runs on the user's
+machine, so following that would let any website reach the user's private network (SSRF).
+
+`NetworkPolicy` decides per destination. Private addresses are blocked unless the user's
+own target is local (the demo shop, tests, a site on the LAN): then the run is in *local
+target mode* and private addresses are allowed, because the user asked for them.
+
+The check resolves the host name and looks at every address it resolves to. The browser's
+traffic goes through `browser/egress.py`, which connects to the exact address that was
+checked, so a host name can't be switched to a private address between check and use.
+Direct requests (httpx) are checked on every redirect hop; httpx resolves the name again
+itself, a small window that is documented in SECURITY.md.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import socket
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urlsplit
+
+import httpcore
+import httpx
+
+IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address
+Resolver = Callable[[str, int], list[str]]
+
+MAX_RESPONSE_BYTES = 5_000_000  # a reply bigger than this (after decompression) is not read
+MAX_COMPRESSED_BYTES = 2_000_000  # bytes on the wire for one reply
+
+
+class PolicyDenied(Exception):
+    """A destination Ghost API refuses to connect to."""
+
+
+class ResponseTooLarge(Exception):
+    """A reply was bigger than the limit; reading stopped (response-too-large)."""
+
+
+def system_resolver(host: str, port: int) -> list[str]:
+    infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
+    return list(dict.fromkeys(str(info[4][0]) for info in infos))
+
+
+def address_problem(ip: IPAddress) -> str | None:
+    """Why an address is not on the public internet, or None when it is."""
+    if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
+        ip = ip.ipv4_mapped
+    if ip.is_loopback:
+        return "this computer (loopback)"
+    if ip.is_link_local:
+        return "a link-local address (e.g. cloud metadata)"
+    if ip.is_unspecified:
+        return "an unspecified address"
+    if ip.is_multicast:
+        return "a multicast address"
+    if ip.is_private or not ip.is_global:
+        return "a private-network address"
+    return None
+
+
+def _literal_ip(host: str) -> IPAddress | None:
+    try:
+        return ipaddress.ip_address(host.strip("[]"))
+    except ValueError:
+        return None
+
+
+@dataclass
+class NetworkPolicy:
+    """`allow_private`: local target mode. `blocked`: tests replace the address rule."""
+
+    allow_private: bool = False
+    resolver: Resolver = system_resolver
+    blocked: Callable[[IPAddress, int], str | None] = field(
+        default=lambda ip, _port: address_problem(ip)
+    )
+    _cache: dict[tuple[str, int], list[str]] = field(default_factory=dict, repr=False)
+
+    @classmethod
+    def for_target(cls, url: str, resolver: Resolver = system_resolver) -> NetworkPolicy:
+        """Local target mode when the user's own target is on this machine or the LAN."""
+        host = (urlsplit(url).hostname or "").lower()
+        local = host == "localhost" or host.endswith(".localhost")
+        if not local:
+            try:
+                addresses = [_literal_ip(host)] if _literal_ip(host) else [
+                    ipaddress.ip_address(a) for a in resolver(host, 443)
+                ]  # fmt: skip
+            except (OSError, ValueError):
+                addresses = []
+            local = any(a is not None and address_problem(a) for a in addresses)
+        return cls(allow_private=local, resolver=resolver)
+
+    def resolve(self, host: str, port: int) -> list[str]:
+        """The addresses `host` may be reached at; raises PolicyDenied if any is blocked.
+
+        All addresses must pass: a name that resolves to one public and one private
+        address could be steered to the private one."""
+        host = host.strip("[]").lower()
+        literal = _literal_ip(host)
+        if literal is not None:
+            addresses = [str(literal)]
+        else:
+            key = (host, port)
+            if key not in self._cache:
+                try:
+                    self._cache[key] = self.resolver(host, port)
+                except OSError as exc:
+                    raise PolicyDenied(f"{host} could not be resolved ({exc})") from exc
+            addresses = self._cache[key]
+        if not addresses:
+            raise PolicyDenied(f"{host} has no address")
+        if not self.allow_private:
+            for address in addresses:
+                problem = self.blocked(ipaddress.ip_address(address), port)
+                if problem:
+                    raise PolicyDenied(
+                        f"blocked {host}: it points to {problem}. Ghost API does not let a "
+                        "website reach your computer or local network"
+                    )
+        return addresses
+
+    def check_url(self, url: str) -> None:
+        parts = urlsplit(url)
+        if parts.scheme not in ("http", "https") or not parts.hostname:
+            raise PolicyDenied(f"blocked {url[:80]}: only http(s) addresses are allowed")
+        port = parts.port or (443 if parts.scheme == "https" else 80)
+        self.resolve(parts.hostname, port)
+
+
+class PinnedBackend(httpcore.SyncBackend):
+    """Connects to the address the policy checked, not to a second DNS answer, so a host
+    name can't be switched to a private address between check and connection (DNS
+    rebinding). TLS still uses the host name, so certificates are checked as usual."""
+
+    def __init__(self, policy: NetworkPolicy) -> None:
+        self.policy = policy
+
+    def connect_tcp(
+        self,
+        host: str,
+        port: int,
+        timeout: float | None = None,
+        local_address: str | None = None,
+        socket_options: Any = None,
+    ) -> httpcore.NetworkStream:
+        address = self.policy.resolve(host, port)[0]  # raises PolicyDenied when blocked
+        return super().connect_tcp(address, port, timeout, local_address, socket_options)
+
+
+def guarded_client(policy: NetworkPolicy, **kwargs: Any) -> httpx.Client:
+    """An httpx client that checks every request it sends, redirect hops included, and
+    connects only to the addresses it checked."""
+
+    def check(request: httpx.Request) -> None:
+        policy.check_url(str(request.url))
+
+    if "transport" not in kwargs or kwargs["transport"] is None:
+        kwargs.pop("transport", None)
+        transport = httpx.HTTPTransport()
+        pool = getattr(transport, "_pool", None)
+        if pool is not None and hasattr(pool, "_network_backend"):
+            pool._network_backend = PinnedBackend(policy)
+        kwargs["transport"] = transport
+    return httpx.Client(event_hooks={"request": [check]}, **kwargs)
+
+
+@dataclass
+class LimitedBody:
+    content: bytes
+    encoding: str | None
+
+    @property
+    def text(self) -> str:
+        return self.content.decode(self.encoding or "utf-8", errors="replace")
+
+
+def read_limited(
+    response: httpx.Response,
+    max_bytes: int = MAX_RESPONSE_BYTES,
+    max_compressed: int = MAX_COMPRESSED_BYTES,
+) -> LimitedBody:
+    """Read a streamed reply, stopping as soon as it is too big (on the wire or unpacked),
+    so a huge or highly compressed reply never fills memory."""
+    declared = response.headers.get("content-length", "")
+    if declared.isdigit() and int(declared) > max_compressed:
+        raise ResponseTooLarge(f"response-too-large: {int(declared):,} bytes declared")
+    chunks: list[bytes] = []
+    size = 0
+    for chunk in response.iter_bytes():
+        size += len(chunk)
+        if size > max_bytes:
+            raise ResponseTooLarge(f"response-too-large: over {max_bytes:,} bytes unpacked")
+        if response.num_bytes_downloaded > max_compressed:
+            raise ResponseTooLarge(f"response-too-large: over {max_compressed:,} bytes")
+        chunks.append(chunk)
+    return LimitedBody(b"".join(chunks), response.encoding)