amethyst-cli 0.1.0__py3-none-any.whl

amethyst/parse/frontmatter.py

@@ -0,0 +1,62 @@
+"""YAML frontmatter: split it off the token stream, read it as metadata."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import yaml
+from markdown_it.token import Token
+
+from amethyst.errors import InputError
+
+
+def split_frontmatter(tokens: list[Token]) -> tuple[dict[str, Any], list[Token]]:
+    """Pull a leading ``front_matter`` token out and parse it as YAML.
+
+    The parser is given the whole file, delimiters included, so every token's
+    line map stays true to the source — a warning that names line 40 means line
+    40 of the file the author actually wrote. The token is dropped here because
+    nothing downstream should have to know it was ever in the stream.
+    """
+    if not tokens or tokens[0].type != "front_matter":
+        return {}, tokens
+    return parse_frontmatter(tokens[0].content), list(tokens[1:])
+
+
+def parse_frontmatter(text: str) -> dict[str, Any]:
+    """Parse the body of a frontmatter block into a metadata mapping.
+
+    Keys are lowercased so that ``Title:`` and ``title:`` mean the same thing.
+    Values are left exactly as YAML produced them — dates stay dates, lists
+    stay lists — and are flattened to text only where they are displayed.
+    """
+    try:
+        loaded = yaml.safe_load(text)
+    except yaml.YAMLError as exc:
+        raise InputError(
+            "The YAML frontmatter could not be parsed.", hint=_yaml_hint(exc)
+        ) from exc
+
+    if loaded is None:
+        return {}
+    if not isinstance(loaded, dict):
+        raise InputError(
+            f"The frontmatter is a {type(loaded).__name__}, "
+            "not a set of key: value pairs.",
+            hint="Frontmatter looks like `title: My Document`, one field per line.",
+        )
+    return {str(key).strip().lower(): value for key, value in loaded.items()}
+
+
+def _yaml_hint(exc: yaml.YAMLError) -> str | None:
+    """Turn PyYAML's mark into a line number the user can act on.
+
+    The mark counts from the start of the frontmatter body, so add one for the
+    opening ``---`` to get back to a line number in the file itself.
+    """
+    problem = getattr(exc, "problem", None)
+    mark = getattr(exc, "problem_mark", None)
+    if mark is None:
+        return str(problem) if problem else None
+    where = f"line {mark.line + 2} of the file"
+    return f"{problem} at {where}." if problem else f"Check {where}."

amethyst/parse/markdown.py

@@ -0,0 +1,49 @@
+"""Construction of the parser both pipelines share.
+
+One configuration serves both: the PDF path renders these tokens to HTML, the
+DOCX path walks them directly. Keeping the plugin set in a single function is
+what stops the two outputs from disagreeing about what the Markdown meant.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from markdown_it import MarkdownIt
+from mdit_py_plugins.anchors import anchors_plugin
+from mdit_py_plugins.deflist import deflist_plugin
+from mdit_py_plugins.footnote import footnote_plugin
+from mdit_py_plugins.front_matter import front_matter_plugin
+from mdit_py_plugins.tasklists import tasklists_plugin
+
+#: Deepest heading level that gets an ``id``. Anchors cost nothing and an
+#: internal link can point at any heading, so every level gets one; --toc-depth
+#: decides which of them reach the table of contents, separately and later.
+ANCHOR_MAX_LEVEL = 6
+
+#: What markdown-it hands a highlighter: the code, the language written after
+#: the fence, and whatever else was on that line. Returning ``None`` — or
+#: anything empty — leaves markdown-it to escape the code itself, which is
+#: exactly what an unhighlightable block wants.
+Highlight = Callable[[str, str, str], "str | None"]
+
+
+def build_parser(highlight: Highlight | None = None) -> MarkdownIt:
+    """Return the parser Amethyst uses for every document.
+
+    The ``gfm-like`` preset brings tables, strikethrough and linkify. Linkify
+    is not self-contained: it hard-requires ``linkify-it-py`` and raises at
+    *render* time rather than import time when it is absent, which is why that
+    package is a declared dependency rather than an optional extra.
+
+    ``highlight`` is only read when the tokens are rendered to HTML, so parsing
+    a document needs none: it is an option of the HTML renderer that markdown-it
+    happens to keep on the parser.
+    """
+    md = MarkdownIt("gfm-like", {"highlight": highlight} if highlight else None)
+    md.use(front_matter_plugin)
+    md.use(footnote_plugin)
+    md.use(deflist_plugin)
+    md.use(tasklists_plugin)
+    md.use(anchors_plugin, max_level=ANCHOR_MAX_LEVEL)
+    return md

amethyst/remote.py

@@ -0,0 +1,236 @@
+"""Downloading the images a document points at over the network.
+
+This is the only place in Amethyst that opens a socket, and it runs as a step
+of the conversion rather than inside either renderer. That is the whole design:
+by the time a renderer sees the token stream, every image it can have is a file
+on disk, and neither pipeline needs an opinion about HTTP, a cache, a timeout
+or a size limit. The PDF renderer still refuses a remote URL outright — that
+refusal is now a backstop for the ones this step could not get, not the policy.
+
+Downloads are cached, keyed by the URL, so converting the same document twice
+— or a dozen documents that share a logo — makes one request. The cache is
+never invalidated: an image at a URL is treated as the thing that URL names.
+Delete the directory to be rid of it.
+
+The limits are deliberately unfriendly. A document is a document, not a
+browser: one connection at a time, ten seconds each, thirty-two megabytes at
+the outside, and http or https and nothing else — checked again after
+redirects, because a redirect is somebody else's choice of scheme.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import mimetypes
+import os
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+from urllib.parse import unquote, urlsplit
+from urllib.request import Request, urlopen
+
+from amethyst import __version__
+from amethyst.document import Document
+from amethyst.parse.assets import REMOTE_SCHEMES, Asset, AssetKind
+from amethyst.render.base import Warn, discard
+
+#: How long to wait for a response, in seconds. A conversion is interactive:
+#: a wait long enough to look like a hang is worse than a missing picture.
+TIMEOUT = 10.0
+
+#: The most a single image may be. Past this it is not an illustration, and
+#: whatever it is will not lay out on a page anyway.
+MAX_BYTES = 32 * 1024 * 1024
+
+#: Read in chunks so that the size limit can be enforced while reading rather
+#: than after a refusal has already been held in memory.
+CHUNK = 64 * 1024
+
+#: Sent so that a server logging its traffic can see what asked.
+USER_AGENT = f"amethyst/{__version__}"
+
+#: Where downloads are kept, under the user's cache directory.
+CACHE_PARTS = ("amethyst", "images")
+
+#: The environment variable naming the base cache directory, and the directory
+#: to use when it says nothing. Both are the XDG convention, which uv, pip and
+#: most of this tool's neighbours already follow on macOS as well as Linux.
+CACHE_HOME = "XDG_CACHE_HOME"
+DEFAULT_CACHE_HOME = Path.home() / ".cache"
+
+#: Extensions worth trusting from a URL before anything has been fetched. A
+#: path ending in something else gets its extension from the response instead.
+IMAGE_SUFFIXES = frozenset(
+    {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".bmp", ".tif", ".tiff"}
+)
+
+
+def cache_directory() -> Path:
+    """Where downloaded images are kept between runs."""
+    base = os.environ.get(CACHE_HOME)
+    root = Path(base) if base else DEFAULT_CACHE_HOME
+    return root.joinpath(*CACHE_PARTS)
+
+
+def fetch_remote_images(
+    document: Document,
+    *,
+    enabled: bool = True,
+    warn: Warn = discard,
+    cache: Path | None = None,
+) -> None:
+    """Download the document's remote images and point it at the local copies.
+
+    Rewrites the image tokens in place, exactly as asset resolution rewrites a
+    local one, so that a renderer never learns an image was ever remote. An
+    image that cannot be fetched is left as written: the renderer then reports
+    it missing, naming the URL the author typed.
+    """
+    references = _remote_images(document)
+    if not references or not enabled:
+        return
+
+    directory = cache if cache is not None else cache_directory()
+    downloaded: dict[str, Path] = {}
+    for reference in references:
+        path = _download(reference, directory, warn)
+        if path is not None:
+            downloaded[reference] = path
+    if downloaded:
+        _rewrite(document, downloaded)
+
+
+def _remote_images(document: Document) -> list[str]:
+    """Every distinct remote image URL, in the order they are written."""
+    seen: dict[str, None] = {}
+    for asset in document.assets:
+        if asset.is_remote and asset.kind is AssetKind.image:
+            seen.setdefault(asset.reference, None)
+    return list(seen)
+
+
+def _rewrite(document: Document, downloaded: dict[str, Path]) -> None:
+    """Point the tokens, and the recorded assets, at the downloaded files."""
+    for token in document.tokens:
+        if token.type != "inline" or not token.children:
+            continue
+        for child in token.children:
+            if child.type != "image":
+                continue
+            source = child.attrGet("src")
+            path = downloaded.get(source) if isinstance(source, str) else None
+            if path is not None:
+                child.attrSet("src", str(path))
+    document.assets = [
+        Asset(
+            kind=asset.kind,
+            reference=asset.reference,
+            line=asset.line,
+            path=downloaded[asset.reference],
+            is_remote=True,
+        )
+        if asset.is_remote and asset.reference in downloaded
+        else asset
+        for asset in document.assets
+    ]
+
+
+def _download(url: str, directory: Path, warn: Warn) -> Path | None:
+    """Return the local copy of one remote image, fetching it if need be."""
+    key = _key(url)
+    cached = _cached(directory, key)
+    if cached is not None:
+        return cached
+    try:
+        payload, suffix = _read(url)
+    except (HTTPError, URLError, TimeoutError, OSError, ValueError) as exc:
+        warn(f"could not download {url}: {_reason(exc)}.")
+        return None
+    try:
+        return _store(directory, key + suffix, payload)
+    except OSError as exc:
+        detail = exc.strerror or str(exc)
+        warn(f"could not cache {url} in {directory}: {detail.lower()}.")
+        return None
+
+
+def _read(url: str) -> tuple[bytes, str]:
+    """Fetch one URL, refusing anything too big or not over http."""
+    request = Request(url, headers={"User-Agent": USER_AGENT})
+    with urlopen(request, timeout=TIMEOUT) as response:
+        # A redirect is the server's choice of destination, not the author's,
+        # so the scheme is checked again on the URL actually opened.
+        if urlsplit(response.geturl()).scheme not in REMOTE_SCHEMES:
+            raise ValueError("it redirected somewhere that is not http")
+        payload = bytearray()
+        while chunk := response.read(CHUNK):
+            payload += chunk
+            if len(payload) > MAX_BYTES:
+                raise ValueError(f"it is larger than {MAX_BYTES // (1024 * 1024)}MB")
+        suffix = _suffix(url, response.headers.get("Content-Type"))
+    return bytes(payload), suffix
+
+
+def _store(directory: Path, name: str, payload: bytes) -> Path:
+    """Write a download into the cache, whole or not at all.
+
+    Written beside the final name and moved onto it, so that a run stopped
+    halfway leaves no half a picture for the next one to find and trust.
+    """
+    directory.mkdir(parents=True, exist_ok=True)
+    destination = directory / name
+    partial = destination.with_name(f"{name}.part{os.getpid()}")
+    partial.write_bytes(payload)
+    partial.replace(destination)
+    return destination
+
+
+def _cached(directory: Path, key: str) -> Path | None:
+    """The cached download for a key, whatever extension it ended up with."""
+    if not directory.is_dir():
+        return None
+    for entry in sorted(directory.glob(f"{key}*")):
+        if entry.is_file() and ".part" not in entry.name:
+            return entry
+    return None
+
+
+def _key(url: str) -> str:
+    """A filename for a URL: its digest, which is stable and has no path in it."""
+    return hashlib.sha256(url.encode("utf-8")).hexdigest()
+
+
+def _suffix(url: str, content_type: str | None) -> str:
+    """The extension to save a download under.
+
+    Both renderers work out an image's real type from its first bytes, so this
+    is for the benefit of anyone who looks in the cache directory — and for
+    WeasyPrint, which guesses a MIME type from the name of a ``file:`` URL
+    before it falls back to sniffing.
+    """
+    from_url = Path(unquote(urlsplit(url).path)).suffix.lower()
+    if from_url in IMAGE_SUFFIXES:
+        return from_url
+    if content_type:
+        guessed = mimetypes.guess_extension(content_type.split(";")[0].strip())
+        if guessed:
+            return guessed
+    return ""
+
+
+def _reason(exc: BaseException) -> str:
+    """Why a download failed, in the words a person would use."""
+    if isinstance(exc, HTTPError):
+        return f"the server answered {exc.code}"
+    if isinstance(exc, URLError):
+        reason = exc.reason
+        if isinstance(reason, TimeoutError):
+            return "it timed out"
+        return str(reason).strip(" <>") or "the request failed"
+    if isinstance(exc, TimeoutError):
+        return "it timed out"
+    if isinstance(exc, OSError):
+        return (exc.strerror or str(exc)).lower()
+    return str(exc)
+
+
+__all__ = ["MAX_BYTES", "TIMEOUT", "cache_directory", "fetch_remote_images"]

amethyst/render/__init__.py

@@ -0,0 +1,42 @@
+"""Renderers: one parsed document in, one format's bytes out.
+
+Each format gets its own pipeline rather than a shared abstraction over both.
+PDF goes through HTML and CSS, because CSS paged media already solves page
+geometry, running furniture and bookmarks; Word has no equivalent and needs the
+token stream walked directly. What keeps the two outputs recognisably the same
+document is the theme they share, not a common renderer.
+"""
+
+from __future__ import annotations
+
+from amethyst.render.base import (
+    DEFAULT_HIGHLIGHT_STYLE,
+    Renderer,
+    RenderOptions,
+    RenderResult,
+    Warn,
+)
+from amethyst.render.docx import render_docx
+from amethyst.render.highlight import (
+    NO_HIGHLIGHTING,
+    Highlighter,
+    highlight_styles,
+    resolve_highlight_style,
+)
+from amethyst.render.html import render_html
+from amethyst.render.pdf import render_pdf
+
+__all__ = [
+    "DEFAULT_HIGHLIGHT_STYLE",
+    "NO_HIGHLIGHTING",
+    "Highlighter",
+    "RenderOptions",
+    "RenderResult",
+    "Renderer",
+    "Warn",
+    "highlight_styles",
+    "render_docx",
+    "render_html",
+    "render_pdf",
+    "resolve_highlight_style",
+]

amethyst/render/base.py

@@ -0,0 +1,85 @@
+"""What a renderer is: one document and some options in, one file's bytes out.
+
+The protocol is deliberately narrow. HTML and EPUB are out of scope for v1 but
+are the obvious next outputs, and keeping the contract to "bytes, plus whatever
+is worth saying about them" is what makes adding one additive rather than a
+change to everything that calls a renderer.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Protocol
+
+from amethyst.document import Document
+from amethyst.theme import Theme, default_theme
+
+#: How a renderer reports something the user should know but that is not fatal
+#: — an image it could not load, a construct it had to approximate. Renderers
+#: are given this rather than printing, so nothing in this package has an
+#: opinion about where output goes or whether --quiet was passed.
+Warn = Callable[[str], None]
+
+#: The Pygments style a document is highlighted with when the user names none.
+#: It lives here rather than with the highlighter because it is the default of
+#: an option declared below, and :mod:`amethyst.render.highlight` reads it back
+#: from here so that there is one spelling of it.
+DEFAULT_HIGHLIGHT_STYLE = "default"
+
+
+def discard(message: str) -> None:
+    """The default warning sink, for callers that do not want to hear it."""
+
+
+@dataclass(frozen=True)
+class RenderOptions:
+    """Everything a renderer needs that is not part of the document itself.
+
+    These are already resolved: the CLI has merged flags over defaults, so a
+    renderer never has to reason about what was and was not passed. Page
+    geometry is part of the theme rather than a field here — a flag that
+    overrides it does so by handing over an overridden theme, which leaves one
+    place for a renderer to read the sheet size from.
+    """
+
+    theme: Theme = field(default_factory=default_theme)
+    #: Extra stylesheet, appended after everything else so it wins. PDF only.
+    extra_css: Path | None = None
+    page_numbers: bool = True
+    #: Whether to open the document with a table of contents, and how deep to
+    #: take it. The depth is carried even when the contents is off, so that
+    #: nothing has to reason about which of the two flags was passed.
+    toc: bool = False
+    toc_depth: int = 3
+    #: Whether to open the document with a title page built from frontmatter.
+    title_page: bool = False
+    #: The Pygments style code is coloured with, or ``"none"`` for no colour.
+    #: Held as a name rather than as a built highlighter because a renderer
+    #: needs one of its own anyway: a highlighter remembers what it has already
+    #: warned about, and that is per conversion.
+    highlight_style: str = DEFAULT_HIGHLIGHT_STYLE
+    warn: Warn = discard
+
+
+@dataclass(frozen=True)
+class RenderResult:
+    """The finished document, and what is worth telling the user about it."""
+
+    data: bytes
+    #: Page count, where the format has one. DOCX does not — Word decides
+    #: pagination when it opens the file, so there is nothing honest to report.
+    pages: int | None = None
+
+
+class Renderer(Protocol):
+    """One output format.
+
+    A callable protocol rather than a class, because a renderer has no state
+    worth keeping between documents; ``render_pdf`` satisfies it as written.
+    """
+
+    def __call__(self, document: Document, options: RenderOptions) -> RenderResult:
+        """Turn ``document`` into the bytes of one file."""
+        ...