md4a-fastapi 0.1.0__py3-none-any.whl

md4a/__init__.py

@@ -0,0 +1,57 @@
+"""Markdown-for-Agents helpers for FastAPI (ASGI) and WSGI applications."""
+
+from __future__ import annotations
+
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+__version__ = "0.1.0"
+
+if TYPE_CHECKING:
+    from .client import FetchResult, MarkdownForAgentsClient
+    from .middleware import MarkdownForAgentsMiddleware, add_md4a
+    from .store import FileStore, MemoryStore
+    from .wsgi import MarkdownForAgentsWSGIMiddleware
+
+# Loaded on first use, so importing md4a.wsgi for a Flask application does not
+# import FastAPI, Starlette or httpx.
+_EXPORTS = {
+    "FetchResult": ".client",
+    "MarkdownForAgentsClient": ".client",
+    "MarkdownForAgentsMiddleware": ".middleware",
+    "add_md4a": ".middleware",
+    "FileStore": ".store",
+    "MemoryStore": ".store",
+    "MarkdownForAgentsWSGIMiddleware": ".wsgi",
+}
+
+__all__ = [
+    "FetchResult",
+    "FileStore",
+    "MarkdownForAgentsClient",
+    "MarkdownForAgentsMiddleware",
+    "MarkdownForAgentsWSGIMiddleware",
+    "MemoryStore",
+    "__version__",
+    "add_md4a",
+]
+
+
+# Importing these eagerly used to make them attributes of the package, so
+# ``md4a.store.MemoryStore`` keeps working after ``import md4a`` alone.
+_SUBMODULES = frozenset({"client", "middleware", "store", "wsgi"})
+
+
+def __getattr__(name: str) -> Any:
+    if name in _EXPORTS:
+        value = getattr(import_module(_EXPORTS[name], __name__), name)
+        globals()[name] = value
+        return value
+    if name in _SUBMODULES:
+        return import_module(f".{name}", __name__)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__() -> list[str]:
+    """Include lazy public exports in module discovery before first access."""
+    return sorted(set(globals()) | set(__all__))

md4a/_extras.py

@@ -0,0 +1,16 @@
+"""Clear errors for features that need the optional ``server`` dependencies."""
+
+from __future__ import annotations
+
+SERVER_EXTRA = frozenset({"anyio", "httpx", "uvicorn"})
+
+
+def require_server_extra(feature: str, exc: ImportError) -> ModuleNotFoundError:
+    """Explain how to install a missing server dependency; re-raise anything else."""
+    if exc.name not in SERVER_EXTRA:
+        raise exc
+    return ModuleNotFoundError(
+        f"{feature} needs {exc.name}, which is part of the optional server dependencies. "
+        "Install them with: pip install 'md4a-fastapi[server]'",
+        name=exc.name,
+    )

md4a/_negotiation.py

@@ -0,0 +1,169 @@
+"""Negotiation rules shared by the ASGI and WSGI middleware.
+
+Nothing here imports a web framework, so either middleware can use it alone.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from email.message import Message as EmailMessage
+from typing import Any
+
+from bs4 import BeautifulSoup
+from markdownify import MarkdownConverter
+
+# A tag name no page uses; markdownify calls convert_<name> for it.
+_LITERAL_TAG = "md4aliteral"
+
+# These describe the original HTML bytes and cannot be reused after conversion.
+HTML_REPRESENTATION_HEADERS = frozenset(
+    {
+        "content-encoding",
+        "content-location",
+        "content-range",
+        "accept-ranges",
+        "etag",
+        "last-modified",
+        "content-md5",
+        "digest",
+        "content-digest",
+        "repr-digest",
+        "transfer-encoding",
+        "trailer",
+    }
+)
+
+MARKDOWN_CONTENT_TYPE = "text/markdown; charset=utf-8"
+
+
+def accepts_markdown(value: str) -> bool:
+    """Return true when text/markdown is an accepted media range (q=0 excluded)."""
+    for item in value.lower().split(","):
+        media, *parameters = (part.strip() for part in item.split(";"))
+        if media != "text/markdown":
+            continue
+        quality = next((p.partition("=")[2] for p in parameters if p.startswith("q=")), "1")
+        try:
+            return float(quality) > 0
+        except ValueError:
+            return False
+    return False
+
+
+def declared_charset(content_type: str) -> str:
+    """Charset named in a Content-Type value, or utf-8 when it names none."""
+    # Quote-aware: a ";" inside a quoted parameter value does not start a parameter.
+    message = EmailMessage()
+    message["content-type"] = content_type
+    return message.get_content_charset() or "utf-8"
+
+
+def vary_with_accept(values: list[str]) -> str:
+    """Merge the response's Vary field values and make sure Accept is among them."""
+    vary = [item.strip() for value in values for item in value.split(",")]
+    if "*" in vary:
+        return "*"
+    if "accept" not in {item.lower() for item in vary}:
+        vary.append("Accept")
+    return ", ".join(item for item in vary if item)
+
+
+# Matched elements whose Markdown only adds block structure around their content
+# keep their tag; any other match (a link, an image, inline formatting, code) is
+# replaced, so its own Markdown syntax cannot wrap the literal text.
+_LITERAL_CONTAINERS = frozenset(
+    {
+        "address",
+        "article",
+        "aside",
+        "blockquote",
+        "caption",
+        "dd",
+        "div",
+        "dt",
+        "figcaption",
+        "footer",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "header",
+        "li",
+        "main",
+        "nav",
+        "p",
+        "section",
+        "td",
+        "th",
+    }
+)
+
+
+def _code_span(text: str, *, in_table: bool = False) -> str:
+    """Text as an inline code span that no Markdown in it can escape from."""
+    text = " ".join(text.split())
+    if not text:
+        return ""
+    if in_table:
+        # Table rows are split into cells before code spans are parsed, so a
+        # pipe must be escaped even inside one; GFM shows "\|" there as "|".
+        text = text.replace("|", "\\|")
+    # A fence longer than any backtick run in the text, padded with a space on
+    # each side so text that starts or ends with a backtick cannot join it.
+    fence = "`" * (max((len(run) for run in re.findall(r"`+", text)), default=0) + 1)
+    # The spaces outside the fences keep them from joining a backtick next to
+    # the span, including the fence of an adjacent literal span.
+    return f" {fence} {text} {fence} "
+
+
+class _Converter(MarkdownConverter):
+    def convert_md4aliteral(self, el: Any, text: str, *args: Any, **kwargs: Any) -> str:
+        return _code_span(el.get_text(), in_table=el.find_parent(["td", "th"]) is not None)
+
+
+def html_to_markdown(
+    html: str,
+    *,
+    content_selector: str | None = None,
+    strip_selectors: Sequence[str] = (),
+    escape_misc: bool = False,
+    literal_selectors: Sequence[str] = (),
+) -> str:
+    """Convert an HTML document to Markdown, optionally only part of it.
+
+    ``strip_selectors`` are CSS selectors whose elements are removed first.
+    ``content_selector`` is a CSS selector; its first match is what gets
+    converted. A page it does not match is converted whole, so a template
+    without that element still yields Markdown rather than nothing.
+
+    ``escape_misc`` escapes Markdown punctuation in page text (``[``, ``]``,
+    ``#``, ``>`` and the like) so it reads as text, not as links or headings.
+    ``literal_selectors`` are CSS selectors whose elements' text is emitted as
+    an inline code span: shown as written, with nothing in it rendered as a
+    link, image, or autolink. Use it for untrusted text such as user comments,
+    together with ``escape_misc`` so a stray backtick in the page's own text
+    cannot open a code span that runs into the literal one.
+    """
+    converter = _Converter(heading_style="ATX", escape_misc=escape_misc)
+    if content_selector is None and not strip_selectors and not literal_selectors:
+        return converter.convert(html)
+    soup = BeautifulSoup(html, "html.parser")
+    for selector in strip_selectors:
+        for element in soup.select(selector):
+            element.decompose()
+    for selector in literal_selectors:
+        for element in soup.select(selector):
+            if element.parent is None:
+                continue  # inside an element already made literal
+            literal = soup.new_tag(_LITERAL_TAG)
+            literal.string = element.get_text()
+            if element.name in _LITERAL_CONTAINERS:
+                element.clear()
+                element.append(literal)
+            else:
+                element.replace_with(literal)
+    content = soup.select_one(content_selector) if content_selector else None
+    return converter.convert(str(content if content is not None else soup))

md4a/app.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from fastapi import FastAPI, HTTPException, Query, Response
+
+from . import __version__
+from ._extras import require_server_extra
+from .store import FileStore
+
+try:
+    import httpx
+
+    from .client import MarkdownForAgentsClient
+except ImportError as exc:
+    raise require_server_extra("The standalone md4a service", exc) from exc
+
+
+def create_app() -> FastAPI:
+    cache_dir = Path(os.getenv("MD4A_CACHE_DIR", ".md4a-cache"))
+    allowed = os.getenv("MD4A_ALLOWED_HOSTS")
+    allowed_hosts = (
+        {host.strip() for host in allowed.split(",") if host.strip()} if allowed else None
+    )
+    store = FileStore(cache_dir)
+    allow_private = os.getenv("MD4A_ALLOW_PRIVATE_ADDRESSES", "").strip().lower() in {
+        "1",
+        "true",
+        "yes",
+    }
+    client = MarkdownForAgentsClient(
+        allowed_hosts=allowed_hosts, allow_private_addresses=allow_private
+    )
+    app = FastAPI(title="md4a", version=__version__)
+
+    @app.get("/health")
+    async def health() -> dict[str, str]:
+        return {"status": "ok"}
+
+    @app.get("/fetch")
+    async def fetch(url: str = Query(..., description="Absolute page URL")) -> Response:
+        try:
+            result = await client.fetch(url)
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+        except httpx.HTTPStatusError as exc:
+            raise HTTPException(
+                status_code=502,
+                detail=f"origin returned HTTP {exc.response.status_code}",
+            ) from exc
+        except httpx.RequestError as exc:
+            raise HTTPException(status_code=502, detail=f"origin request failed: {exc}") from exc
+        if not result.available:
+            raise HTTPException(
+                status_code=406,
+                detail=(
+                    f"origin returned {result.content_type or 'no content type'}, not text/markdown"
+                ),
+            )
+        assert result.markdown is not None
+        store.put(result.final_url, result.markdown)
+        return Response(
+            result.markdown,
+            media_type="text/markdown",
+            headers={"X-MD4A-Source": result.final_url},
+        )
+
+    return app
+
+
+app = create_app()

md4a/cli.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import os
+import sys
+
+from ._extras import require_server_extra
+
+try:
+    import httpx
+    import uvicorn
+
+    from .client import MarkdownForAgentsClient
+except ImportError as exc:
+    # Reported by main() so the md4a command prints one line, not a traceback.
+    _missing_extra: ModuleNotFoundError | None = require_server_extra("The md4a command", exc)
+else:
+    _missing_extra = None
+
+
+def parser() -> argparse.ArgumentParser:
+    result = argparse.ArgumentParser(prog="md4a", description="Markdown-for-Agents utility")
+    sub = result.add_subparsers(dest="command", required=True)
+    fetch = sub.add_parser("fetch", help="fetch and save a Markdown-for-Agents page")
+    fetch.add_argument("url")
+    fetch.add_argument("-o", "--output", help="output file (default: stdout)")
+    fetch.add_argument(
+        "--allowed-host",
+        action="append",
+        metavar="HOST",
+        help="only contact this host; repeatable (default: MD4A_ALLOWED_HOSTS, else unrestricted)",
+    )
+    fetch.add_argument(
+        "--allow-private-addresses",
+        action="store_true",
+        help="permit loopback, private, and other non-public destinations",
+    )
+    fetch.add_argument("--timeout", type=float, default=15.0, help="seconds (default: 15)")
+    fetch.add_argument(
+        "--max-bytes", type=int, default=5_000_000, help="response size limit (default: 5000000)"
+    )
+    serve = sub.add_parser("serve", help="run the standalone FastAPI service")
+    serve.add_argument("--host", default="127.0.0.1")
+    serve.add_argument("--port", type=int, default=8000)
+    return result
+
+
+def _allowed_hosts(options: list[str] | None) -> set[str] | None:
+    if options:
+        return set(options)
+    allowed = os.getenv("MD4A_ALLOWED_HOSTS")
+    return {host.strip() for host in allowed.split(",") if host.strip()} if allowed else None
+
+
+async def _fetch(args: argparse.Namespace) -> int:
+    client = MarkdownForAgentsClient(
+        timeout=args.timeout,
+        max_bytes=args.max_bytes,
+        allowed_hosts=_allowed_hosts(args.allowed_host),
+        allow_private_addresses=args.allow_private_addresses,
+    )
+    try:
+        result = await client.fetch(args.url)
+    except httpx.HTTPStatusError as exc:
+        print(f"md4a: origin returned HTTP {exc.response.status_code}", file=sys.stderr)
+        return 1
+    except (ValueError, httpx.RequestError) as exc:
+        print(f"md4a: {exc}", file=sys.stderr)
+        return 1
+    if not result.available:
+        print(f"No Markdown-for-Agents response ({result.content_type or 'unknown content type'})")
+        return 2
+    assert result.markdown is not None
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as stream:
+            stream.write(result.markdown)
+    else:
+        print(result.markdown, end="")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = parser().parse_args(argv)
+    if _missing_extra is not None:
+        raise SystemExit(f"md4a: {_missing_extra}")
+    if args.command == "serve":
+        uvicorn.run("md4a.app:app", host=args.host, port=args.port)
+    else:
+        raise SystemExit(asyncio.run(_fetch(args)))