md4a-fastapi 0.1.0__py3-none-any.whl

md4a/client.py

@@ -0,0 +1,324 @@
+from __future__ import annotations
+
+import ipaddress
+import socket
+import urllib.request
+from collections.abc import Callable
+from dataclasses import dataclass
+from types import TracebackType
+from urllib.parse import urlparse
+
+from . import __version__
+from ._extras import require_server_extra
+
+try:
+    import anyio
+    import httpx
+except ImportError as exc:
+    raise require_server_extra("md4a.client", exc) from exc
+
+MARKDOWN_ACCEPT = "text/markdown"
+
+_NAT64 = ipaddress.ip_network("64:ff9b::/96")
+_IPV6_GLOBAL_UNICAST = ipaddress.ip_network("2000::/3")
+_ADDRESSES = "md4a.addresses"
+_PROXY = "md4a.proxy"
+
+
+def _is_public(value: str) -> bool:
+    """Return true for a globally routable unicast address."""
+    address = ipaddress.ip_address(value)
+    if isinstance(address, ipaddress.IPv6Address):
+        # Judge addresses that carry an IPv4 destination by that destination.
+        embedded = address.ipv4_mapped or address.sixtofour
+        if embedded is None and address.teredo is not None:
+            embedded = address.teredo[1]
+        if embedded is None and address in _NAT64:
+            embedded = ipaddress.IPv4Address(int(address) & 0xFFFFFFFF)
+        if embedded is not None:
+            address = embedded
+        elif address not in _IPV6_GLOBAL_UNICAST:
+            # ipaddress calls some ranges outside it global, such as site-local fec0::/10.
+            return False
+    return address.is_global and not address.is_multicast
+
+
+def _literal_address(host: str) -> str | None:
+    """Return the address a host spells, or None when it is a name to resolve."""
+    try:
+        return str(ipaddress.ip_address(host))
+    except ValueError:
+        pass
+    try:
+        # Forms such as 127.1, 2130706433, and 0x7f000001 are addresses to libc.
+        return socket.inet_ntoa(socket.inet_aton(host))
+    except (OSError, UnicodeError):
+        return None
+
+
+def _exclusion_matches(pattern: httpx.URL, url: httpx.URL) -> bool:
+    if pattern.scheme != "all" and pattern.scheme != url.scheme:
+        return False
+    if pattern.port is not None and pattern.port != url.port:
+        return False
+    host = pattern.host
+    if host in ("", "*"):
+        return True
+    if host.startswith("*."):
+        return url.host.endswith(host[1:]) and len(url.host) > len(host) - 1
+    if host.startswith("*"):
+        return url.host == host[1:] or url.host.endswith(f".{host[1:]}")
+    return url.host == host
+
+
+def _environment_proxy(url: httpx.URL) -> str | None:
+    """Return the environment proxy httpx would send this URL through, if any.
+
+    md4a routes requests itself, so it has to make this decision; it follows
+    httpx's reading of the proxy variables (httpx._utils.get_environment_proxies).
+    """
+    settings = urllib.request.getproxies()
+    rules: dict[str, str | None] = {}
+    for scheme in ("http", "https", "all"):
+        proxy = settings.get(scheme)
+        if proxy:
+            rules[f"{scheme}://"] = proxy if "://" in proxy else f"http://{proxy}"
+    for entry in (item.strip() for item in settings.get("no", "").split(",")):
+        if entry == "*":
+            return None
+        if not entry:
+            continue
+        address = entry.split("/")[0]
+        if "://" in entry:
+            rules[entry] = None
+        elif _parses(ipaddress.IPv4Address, address) or entry.lower() == "localhost":
+            rules[f"all://{entry}"] = None
+        elif _parses(ipaddress.IPv6Address, address):
+            rules[f"all://[{entry}]"] = None
+        else:
+            rules[f"all://*{entry}"] = None
+
+    chosen: tuple[tuple[int, int, int], str | None] | None = None
+    for key, proxy in rules.items():
+        try:
+            pattern = httpx.URL(key)
+        except httpx.InvalidURL:
+            continue
+        if not _exclusion_matches(pattern, url):
+            continue
+        # The most specific rule wins: a port, then the longer host, then a scheme.
+        priority = (
+            pattern.port is None,
+            -len("" if pattern.host == "*" else pattern.host),
+            -len("" if pattern.scheme == "all" else pattern.scheme),
+        )
+        if chosen is None or priority < chosen[0]:
+            chosen = (priority, proxy)
+    return chosen[1] if chosen else None
+
+
+def _parses(kind: type, value: str) -> bool:
+    try:
+        kind(value)
+    except ValueError:
+        return False
+    return True
+
+
+class _RoutingTransport(httpx.AsyncBaseTransport):
+    """Send each request where md4a decided it goes: a proxy, or a checked address.
+
+    Direct requests connect to the addresses that were validated instead of
+    resolving the name again. Only the connection target changes: the hostname is
+    kept for Host, SNI, and certificate verification, and the client still sees the
+    original URL, so redirects and cookies stay scoped to the hostname.
+
+    Each hostname gets its own direct transport. A connection pool only sees the
+    pinned address, so sharing one would let a hostname reuse a connection whose
+    certificate was verified for a different hostname at the same address.
+    """
+
+    def __init__(self, direct_transport: Callable[[], httpx.AsyncBaseTransport]) -> None:
+        self._direct_transport = direct_transport
+        self._direct: dict[str, httpx.AsyncBaseTransport] = {}
+        self._proxies: dict[str, httpx.AsyncBaseTransport] = {}
+        self._entered: dict[int, httpx.AsyncBaseTransport] = {}
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        proxy = request.extensions.get(_PROXY)
+        if proxy is not None:
+            if proxy not in self._proxies:
+                self._proxies[proxy] = await self._enter(httpx.AsyncHTTPTransport(proxy=proxy))
+            return await self._proxies[proxy].handle_async_request(request)
+        addresses = request.extensions.get(_ADDRESSES)
+        if not addresses:
+            raise ValueError("refusing to send a request whose destination was not checked")
+        hostname = request.url.raw_host.decode("ascii")
+        if hostname not in self._direct:
+            self._direct[hostname] = await self._enter(self._direct_transport())
+        direct = self._direct[hostname]
+        extensions = {**request.extensions, "sni_hostname": hostname}
+        for index, address in enumerate(addresses):
+            pinned = httpx.Request(
+                request.method,
+                request.url.copy_with(host=address),
+                headers=request.headers,
+                stream=request.stream,
+                extensions=extensions,
+            )
+            try:
+                return await direct.handle_async_request(pinned)
+            except (httpx.ConnectError, httpx.ConnectTimeout):
+                # Nothing was sent, so the next validated address can be tried.
+                if index == len(addresses) - 1:
+                    raise
+        raise AssertionError("unreachable")
+
+    async def _enter(self, transport: httpx.AsyncBaseTransport) -> httpx.AsyncBaseTransport:
+        # A supplied transport is handed out for every hostname; enter it once.
+        if id(transport) not in self._entered:
+            self._entered[id(transport)] = transport
+            await transport.__aenter__()
+        return transport
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None = None,
+        exc_value: BaseException | None = None,
+        traceback: TracebackType | None = None,
+    ) -> None:
+        entered, self._entered = self._entered, {}
+        for transport in entered.values():
+            await transport.__aexit__(exc_type, exc_value, traceback)
+
+    async def aclose(self) -> None:
+        await self.__aexit__()
+
+
+@dataclass(frozen=True, slots=True)
+class FetchResult:
+    requested_url: str
+    final_url: str
+    status_code: int
+    content_type: str
+    markdown: str | None
+
+    @property
+    def available(self) -> bool:
+        return self.markdown is not None
+
+
+class MarkdownForAgentsClient:
+    """Probe URLs using the Markdown-for-Agents content negotiation header."""
+
+    def __init__(
+        self,
+        *,
+        timeout: float = 15.0,
+        max_bytes: int = 5_000_000,
+        allowed_hosts: set[str] | None = None,
+        allow_private_addresses: bool = False,
+        transport: httpx.AsyncBaseTransport | None = None,
+    ) -> None:
+        self.timeout = timeout
+        self.max_bytes = max_bytes
+        self.allowed_hosts = allowed_hosts
+        self.allow_private_addresses = allow_private_addresses
+        self.transport = transport
+
+    def _validate_url(self, url: str) -> None:
+        parsed = urlparse(url)
+        if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+            raise ValueError("url must be an absolute http(s) URL")
+        if self.allowed_hosts is not None and parsed.hostname not in self.allowed_hosts:
+            raise ValueError(f"host is not allowed: {parsed.hostname}")
+
+    def _direct_transport(self) -> httpx.AsyncBaseTransport:
+        """Return the transport for one hostname's direct connections."""
+        return self.transport or httpx.AsyncHTTPTransport()
+
+    def _proxy_for(self, url: httpx.URL) -> str | None:
+        """Return the environment proxy this URL goes through, if any.
+
+        An explicitly supplied transport is never proxied.
+        """
+        return None if self.transport is not None else _environment_proxy(url)
+
+    async def _route(self, url: str) -> dict[str, object]:
+        """Refuse a non-public destination; return how the request must be sent."""
+        target = httpx.URL(url)
+        host = target.raw_host.decode("ascii")
+        literal = _literal_address(host)
+        if literal is not None:
+            addresses = [literal]
+        else:
+            proxy = self._proxy_for(target)
+            if proxy is not None:
+                # The proxy resolves the name, so there is no address to check or pin.
+                return {_PROXY: proxy}
+            try:
+                with anyio.fail_after(self.timeout):
+                    found = await anyio.getaddrinfo(host, None, type=socket.SOCK_STREAM)
+            except (OSError, TimeoutError) as exc:
+                raise httpx.ConnectError(f"could not resolve host: {host}") from exc
+            addresses = list(dict.fromkeys(str(item[4][0]) for item in found))
+        if not addresses or not all(_is_public(address) for address in addresses):
+            raise ValueError(f"host is not a public address: {host}")
+        proxy = self._proxy_for(target)
+        return {_PROXY: proxy} if proxy is not None else {_ADDRESSES: addresses}
+
+    async def fetch(self, url: str) -> FetchResult:
+        self._validate_url(url)
+        async with httpx.AsyncClient(
+            follow_redirects=False,
+            timeout=self.timeout,
+            # With the check off, httpx routes and resolves as it normally would.
+            transport=(
+                self.transport
+                if self.allow_private_addresses
+                else _RoutingTransport(self._direct_transport)
+            ),
+            headers={"Accept": MARKDOWN_ACCEPT, "User-Agent": f"md4a/{__version__}"},
+        ) as client:
+            current_url = url
+            for _ in range(11):
+                self._validate_url(current_url)
+                route = {} if self.allow_private_addresses else await self._route(current_url)
+                request = client.build_request("GET", current_url, extensions=route)
+                response = await client.send(request, stream=True)
+                if response.is_redirect:
+                    redirect = response.next_request
+                    await response.aclose()
+                    if redirect is None:
+                        raise ValueError("redirect response did not include a usable location")
+                    current_url = str(redirect.url)
+                    continue
+
+                try:
+                    response.raise_for_status()
+                    chunks: list[bytes] = []
+                    size = 0
+                    async for chunk in response.aiter_bytes():
+                        size += len(chunk)
+                        if size > self.max_bytes:
+                            raise ValueError(f"response exceeds {self.max_bytes} bytes")
+                        chunks.append(chunk)
+
+                    content_type = response.headers.get("content-type", "")
+                    media_type = content_type.partition(";")[0].strip().lower()
+                    body = b"".join(chunks)
+                    markdown = (
+                        body.decode(response.encoding or "utf-8")
+                        if media_type == MARKDOWN_ACCEPT
+                        else None
+                    )
+                    return FetchResult(
+                        requested_url=url,
+                        final_url=str(response.url),
+                        status_code=response.status_code,
+                        content_type=content_type,
+                        markdown=markdown,
+                    )
+                finally:
+                    await response.aclose()
+            raise ValueError("too many redirects")

md4a/middleware.py

@@ -0,0 +1,245 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+
+from fastapi import FastAPI
+from starlette.datastructures import Headers, MutableHeaders
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+from ._negotiation import (
+    HTML_REPRESENTATION_HEADERS,
+    MARKDOWN_CONTENT_TYPE,
+    accepts_markdown,
+    declared_charset,
+    html_to_markdown,
+    vary_with_accept,
+)
+from .store import MemoryStore, Store
+
+MarkdownProvider = Callable[[str], str | None]
+
+# ASGI header names are bytes.
+_HTML_REPRESENTATION_HEADERS = {name.encode() for name in HTML_REPRESENTATION_HEADERS}
+
+
+def _declared_length(headers: Headers) -> int:
+    try:
+        return int(headers.get("content-length", "0"))
+    except ValueError:
+        return 0
+
+
+def _declared_charset(headers: Headers) -> str:
+    return declared_charset(headers.get("content-type", ""))
+
+
+class MarkdownForAgentsMiddleware:
+    """Serve Markdown when a GET request explicitly accepts text/markdown.
+
+    Cached/provider content wins. Otherwise the normal FastAPI route executes and
+    an HTML response is converted, cached by request path, and returned as Markdown.
+    Non-GET and non-Markdown requests pass through unchanged.
+
+    Only successful HTML and Markdown responses are buffered, up to max_html_bytes.
+    Other responses, and bodies over that limit, are streamed through unconverted
+    and uncached.
+
+    Cached content is served until the store drops it; give the store a ttl to
+    refresh it, or pass cache_responses=False to stop writing responses to the store.
+
+    Conversion preserves response headers except metadata tied to the HTML bytes.
+    Stores contain only content: security/auth middleware must wrap this middleware
+    to run on cache hits and provider responses as well as converted responses.
+    """
+
+    def __init__(
+        self,
+        app: ASGIApp,
+        *,
+        store: Store | None = None,
+        provider: MarkdownProvider | None = None,
+        convert_html: bool = True,
+        max_html_bytes: int = 5_000_000,
+        cache_responses: bool = True,
+        content_selector: str | None = None,
+        strip_selectors: Sequence[str] = (),
+        escape_misc: bool = False,
+        literal_selectors: Sequence[str] = (),
+    ) -> None:
+        self.app = app
+        self.store = store or MemoryStore()
+        self.provider = provider
+        self.convert_html = convert_html
+        self.max_html_bytes = max_html_bytes
+        self.cache_responses = cache_responses
+        self.content_selector = content_selector
+        self.strip_selectors = tuple(strip_selectors)
+        self.escape_misc = escape_misc
+        self.literal_selectors = tuple(literal_selectors)
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http" or scope["method"] != "GET":
+            await self.app(scope, receive, send)
+            return
+        request_headers = Headers(scope=scope)
+        if not accepts_markdown(request_headers.get("accept", "")):
+            await self.app(scope, receive, send)
+            return
+
+        key = scope.get("raw_path", scope["path"].encode()).decode()
+        if scope.get("query_string"):
+            key += "?" + scope["query_string"].decode()
+        content = self.store.get(key)
+        if content is None and self.provider is not None:
+            content = self.provider(key)
+        if content is not None:
+            await self._send_markdown(send, content)
+            return
+
+        start: Message | None = None
+        media_type = ""
+        body_parts: list[bytes] = []
+        buffered = 0
+        passthrough = False
+
+        async def release() -> None:
+            """Give up on conversion: forward what is held and stop buffering."""
+            nonlocal passthrough
+            assert start is not None
+            passthrough = True
+            await send(start)
+            if body_parts:
+                body = b"".join(body_parts)
+                body_parts.clear()
+                await send({"type": "http.response.body", "body": body, "more_body": True})
+
+        async def capture(message: Message) -> None:
+            nonlocal start, media_type, buffered, passthrough
+            if message["type"] == "http.response.start":
+                start = message
+                response_headers = Headers(raw=start["headers"])
+                media_type = response_headers.get("content-type", "").partition(";")[0].lower()
+                convertible = 200 <= start["status"] < 300 and (
+                    media_type == "text/markdown"
+                    or (self.convert_html and media_type == "text/html")
+                )
+                # Forward all messages, including trailers, when the representation
+                # cannot be converted. Only buffer bodies that may be converted or
+                # cached, and never compressed, partial, or oversized content.
+                passthrough = (
+                    not convertible
+                    or bool(start.get("trailers", False))
+                    or any(
+                        coding.strip().lower() != "identity"
+                        for value in response_headers.getlist("content-encoding")
+                        for coding in value.split(",")
+                    )
+                    or start["status"] == 206
+                    or "content-range" in response_headers
+                    or _declared_length(response_headers) > self.max_html_bytes
+                )
+                if passthrough:
+                    await send(message)
+            elif passthrough or start is None:
+                # Extension messages may precede the start (for example debug).
+                await send(message)
+            elif message["type"] == "http.response.body":
+                body = message.get("body", b"")
+                if buffered + len(body) > self.max_html_bytes:
+                    await release()
+                    await send(message)
+                    return
+                buffered += len(body)
+                body_parts.append(body)
+                if not message.get("more_body", False):
+                    await finish()
+            else:
+                # Not a body this middleware can hold (for example pathsend).
+                await release()
+                await send(message)
+
+        async def finish() -> None:
+            assert start is not None
+            body = b"".join(body_parts)
+            try:
+                text = body.decode(_declared_charset(Headers(raw=start["headers"])))
+                # Some codecs decode to lone surrogates, which cannot be served or stored.
+                text.encode("utf-8")
+            except (LookupError, UnicodeError):
+                text = None
+            if text is None or media_type == "text/markdown":
+                # Undecodable content is returned as the application sent it, uncached.
+                if text is not None and self.cache_responses:
+                    self.store.put(key, text)
+                await send(start)
+                await send({"type": "http.response.body", "body": body})
+                return
+            converted = html_to_markdown(
+                text,
+                content_selector=self.content_selector,
+                strip_selectors=self.strip_selectors,
+                escape_misc=self.escape_misc,
+                literal_selectors=self.literal_selectors,
+            )
+            if self.cache_responses:
+                self.store.put(key, converted)
+            await self._send_markdown(
+                send, converted, status=start["status"], raw_headers=start["headers"]
+            )
+
+        await self.app(scope, receive, capture)
+
+    @staticmethod
+    async def _send_markdown(
+        send: Send,
+        content: str,
+        status: int = 200,
+        *,
+        raw_headers: list[tuple[bytes, bytes]] | None = None,
+    ) -> None:
+        body = content.encode("utf-8")
+        # Copy raw pairs so repeated fields (especially Set-Cookie) survive and
+        # neither the original response nor its mutable headers are modified.
+        message: Message = {
+            "type": "http.response.start",
+            "status": status,
+            "headers": [
+                (name, value)
+                for name, value in (raw_headers or [])
+                if name.lower() not in _HTML_REPRESENTATION_HEADERS
+            ],
+        }
+        headers = MutableHeaders(scope=message)
+        headers["content-type"] = MARKDOWN_CONTENT_TYPE
+        headers["content-length"] = str(len(body))
+        headers["vary"] = vary_with_accept(headers.getlist("vary"))
+        await send(message)
+        await send({"type": "http.response.body", "body": body})
+
+
+def add_md4a(
+    app: FastAPI,
+    *,
+    store: Store | None = None,
+    provider: MarkdownProvider | None = None,
+    convert_html: bool = True,
+    max_html_bytes: int = 5_000_000,
+    cache_responses: bool = True,
+    content_selector: str | None = None,
+    strip_selectors: Sequence[str] = (),
+    escape_misc: bool = False,
+    literal_selectors: Sequence[str] = (),
+) -> None:
+    """Add Markdown-for-Agents negotiation to an existing FastAPI app."""
+    app.add_middleware(
+        MarkdownForAgentsMiddleware,
+        store=store,
+        provider=provider,
+        convert_html=convert_html,
+        max_html_bytes=max_html_bytes,
+        cache_responses=cache_responses,
+        content_selector=content_selector,
+        strip_selectors=strip_selectors,
+        escape_misc=escape_misc,
+        literal_selectors=literal_selectors,
+    )

md4a/py.typed

@@ -0,0 +1 @@
+

md4a/store.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import string
+import time
+from pathlib import Path
+from typing import Protocol
+
+
+class Store(Protocol):
+    def get(self, key: str) -> str | None: ...
+    def put(self, key: str, content: str) -> None: ...
+
+
+def _validated_ttl(ttl: float | None) -> float | None:
+    if ttl is not None and ttl <= 0:
+        raise ValueError("ttl must be greater than 0, or None to never expire")
+    return ttl
+
+
+class MemoryStore:
+    """Process-local buffer store.
+
+    Entries older than ``ttl`` seconds are treated as missing. With ``max_entries``
+    set, storing beyond the limit drops expired entries, then the oldest stored.
+    """
+
+    def __init__(self, *, ttl: float | None = None, max_entries: int | None = None) -> None:
+        if max_entries is not None and max_entries < 1:
+            raise ValueError("max_entries must be at least 1, or None for no limit")
+        self.ttl = _validated_ttl(ttl)
+        self.max_entries = max_entries
+        self.data: dict[str, str] = {}
+        self._stored_at: dict[str, float] = {}
+
+    def _expired(self, key: str) -> bool:
+        return self.ttl is not None and time.monotonic() - self._stored_at[key] >= self.ttl
+
+    def get(self, key: str) -> str | None:
+        if key in self.data and self._expired(key):
+            self.delete(key)
+        return self.data.get(key)
+
+    def put(self, key: str, content: str) -> None:
+        # Remove first so a re-stored key becomes the newest entry.
+        self.delete(key)
+        if self.max_entries is not None and len(self.data) >= self.max_entries:
+            for stale in [item for item in self.data if self._expired(item)]:
+                self.delete(stale)
+            while len(self.data) >= self.max_entries:
+                self.delete(next(iter(self.data)))
+        self.data[key] = content
+        self._stored_at[key] = time.monotonic()
+
+    def delete(self, key: str) -> None:
+        self.data.pop(key, None)
+        self._stored_at.pop(key, None)
+
+    def clear(self) -> None:
+        self.data.clear()
+        self._stored_at.clear()
+
+
+class FileStore:
+    """Filesystem store with hashed filenames and a human-readable index.
+
+    Entries whose file is older than ``ttl`` seconds are removed when read.
+    """
+
+    def __init__(self, directory: str | Path, *, ttl: float | None = None) -> None:
+        self.ttl = _validated_ttl(ttl)
+        self.directory = Path(directory)
+        self.directory.mkdir(parents=True, exist_ok=True)
+
+    def _path(self, key: str) -> Path:
+        digest = hashlib.sha256(key.encode()).hexdigest()
+        return self.directory / f"{digest}.md"
+
+    def _read_index(self) -> dict[str, str]:
+        try:
+            index = json.loads((self.directory / "index.json").read_text(encoding="utf-8"))
+        except (FileNotFoundError, json.JSONDecodeError):
+            return {}
+        return index if isinstance(index, dict) else {}
+
+    def _write_index(self, index: dict[str, str]) -> None:
+        (self.directory / "index.json").write_text(
+            json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+        )
+
+    def get(self, key: str) -> str | None:
+        path = self._path(key)
+        try:
+            if self.ttl is not None and time.time() - path.stat().st_mtime >= self.ttl:
+                self.delete(key)
+                return None
+            return path.read_text(encoding="utf-8")
+        except FileNotFoundError:
+            return None
+
+    def put(self, key: str, content: str) -> None:
+        path = self._path(key)
+        temporary = path.with_suffix(".tmp")
+        temporary.write_text(content, encoding="utf-8")
+        temporary.replace(path)
+        index = self._read_index()
+        index[key] = path.name
+        self._write_index(index)
+
+    def delete(self, key: str) -> None:
+        self._path(key).unlink(missing_ok=True)
+        index = self._read_index()
+        if index.pop(key, None) is not None:
+            self._write_index(index)
+
+    def clear(self) -> None:
+        # Only remove files this store names: a SHA-256 hex digest plus ".md".
+        for path in self.directory.glob("*.md"):
+            if len(path.stem) == 64 and set(path.stem) <= set(string.hexdigits):
+                path.unlink(missing_ok=True)
+        self._write_index({})