sphinxnotes.render 1.0a42__py3-none-any.whl

sphinxnotes/render/extractx.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING, override
+from abc import ABC, abstractmethod
+
+from sphinx.util.docutils import SphinxDirective
+from docutils.parsers.rst.directives import _directives
+from docutils.parsers.rst.roles import _roles
+
+from .render import HostWrapper
+from .ctxnodes import pending_node
+from .utils import find_current_section, Report, Reporter
+from .utils.ctxproxy import proxy
+
+if TYPE_CHECKING:
+    from typing import Any, Callable, ClassVar
+    from sphinx.application import Sphinx
+    from .render import ParseHost, TransformHost
+
+
+class GlobalExtraContxt(ABC):
+    @abstractmethod
+    def generate(self) -> Any: ...
+
+
+class ParsePhaseExtraContext(ABC):
+    @abstractmethod
+    def generate(self, host: ParseHost) -> Any: ...
+
+
+class TransformPhaseExtraContext(ABC):
+    @abstractmethod
+    def generate(self, host: TransformHost) -> Any: ...
+
+
+# =======================
+# Extra context registion
+# =======================
+
+
+class ExtraContextRegistry:
+    names: set[str]
+    parsing: dict[str, ParsePhaseExtraContext]
+    parsed: dict[str, TransformPhaseExtraContext]
+    post_transform: dict[str, TransformPhaseExtraContext]
+    global_: dict[str, GlobalExtraContxt]
+
+    def __init__(self) -> None:
+        self.names = set()
+        self.parsing = {}
+        self.parsed = {}
+        self.post_transform = {}
+        self.global_ = {}
+
+        self.add_global_context('sphinx', _SphinxExtraContext())
+        self.add_global_context('docutils', _DocutilsExtraContext())
+        self.add_parsing_phase_context('markup', _MarkupExtraContext())
+        self.add_parsing_phase_context('section', _SectionExtraContext())
+        self.add_parsing_phase_context('doc', _DocExtraContext())
+
+    def _name_dedup(self, name: str) -> None:
+        # TODO: allow dup
+        if name in self.names:
+            raise ValueError(f'Context generator {name} already exists')
+        self.names.add(name)
+
+    def add_parsing_phase_context(
+        self, name: str, ctxgen: ParsePhaseExtraContext
+    ) -> None:
+        self._name_dedup(name)
+        self.parsing['_' + name] = ctxgen
+
+    def add_parsed_phase_context(
+        self, name: str, ctxgen: TransformPhaseExtraContext
+    ) -> None:
+        self._name_dedup(name)
+        self.parsed['_' + name] = ctxgen
+
+    def add_post_transform_phase_context(
+        self, name: str, ctxgen: TransformPhaseExtraContext
+    ) -> None:
+        self._name_dedup(name)
+        self.post_transform['_' + name] = ctxgen
+
+    def add_global_context(self, name: str, ctxgen: GlobalExtraContxt):
+        self._name_dedup(name)
+        self.global_['_' + name] = ctxgen
+
+
+# ===================================
+# Bulitin extra context implementions
+# ===================================
+
+
+class _MarkupExtraContext(ParsePhaseExtraContext):
+    @override
+    def generate(self, host: ParseHost) -> Any:
+        isdir = isinstance(host, SphinxDirective)
+        return {
+            'type': 'directive' if isdir else 'role',
+            'name': host.name,
+            'lineno': host.lineno,
+            'rawtext': host.block_text if isdir else host.rawtext,
+        }
+
+
+class _DocExtraContext(ParsePhaseExtraContext):
+    @override
+    def generate(self, host: ParseHost) -> Any:
+        return proxy(HostWrapper(host).doctree)
+
+
+class _SectionExtraContext(ParsePhaseExtraContext):
+    @override
+    def generate(self, host: ParseHost) -> Any:
+        parent = HostWrapper(host).parent
+        return proxy(find_current_section(parent))
+
+
+class _SphinxExtraContext(GlobalExtraContxt):
+    app: ClassVar[Sphinx]
+
+    @override
+    def generate(self) -> Any:
+        return proxy(self.app)
+
+
+class _DocutilsExtraContext(GlobalExtraContxt):
+    @override
+    def generate(self) -> Any:
+        # FIXME: use unexported api
+        return {
+            'directives': _directives,
+            'roles': _roles,
+        }
+
+
+# ========================
+# Extra Context Management
+# ========================
+
+
+class ExtraContextGenerator:
+    node: pending_node
+    report: Report
+
+    registry: ClassVar[ExtraContextRegistry] = ExtraContextRegistry()
+
+    def __init__(self, node: pending_node) -> None:
+        self.node = node
+        self.report = Report(
+            'Extra Context Generation Report',
+            'ERROR',
+            source=node.source,
+            line=node.line,
+        )
+        Reporter(node).append(self.report)
+
+    def on_anytime(self) -> None:
+        for name, ctxgen in self.registry.global_.items():
+            self._safegen(name, lambda: ctxgen.generate())
+
+    def on_parsing(self, host: ParseHost) -> None:
+        for name, ctxgen in self.registry.parsing.items():
+            self._safegen(name, lambda: ctxgen.generate(host))
+
+    def on_parsed(self, host: TransformHost) -> None:
+        for name, ctxgen in self.registry.parsed.items():
+            self._safegen(name, lambda: ctxgen.generate(host))
+
+    def on_post_transform(self, host: TransformHost) -> None:
+        for name, ctxgen in self.registry.post_transform.items():
+            self._safegen(name, lambda: ctxgen.generate(host))
+
+    def _safegen(self, name: str, gen: Callable[[], Any]):
+        try:
+            # ctxgen.generate can be user-defined code, exception of any kind are possible.
+            self.node.extra[name] = gen()
+        except Exception:
+            self.report.text(f'Failed to generate extra context "{name}":')
+            self.report.traceback()
+
+
+def setup(app: Sphinx):
+    _SphinxExtraContext.app = app

sphinxnotes/render/markup.py

@@ -0,0 +1,122 @@
+"""
+sphinxnotes.markup
+~~~~~~~~~~~~~~~~~~
+
+Rendering markup text to doctree nodes.
+
+:copyright: Copyright 2025 by the Shengyu Zhang.
+:license: BSD, see LICENSE for details.
+"""
+
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from docutils import nodes
+from docutils.parsers.rst.states import Struct
+from docutils.utils import new_document
+from sphinx import version_info
+from sphinx.util.docutils import SphinxDirective, SphinxRole
+from sphinx.transforms import SphinxTransform
+from sphinx.environment.collectors.asset import ImageCollector
+
+from .render import Host
+
+if TYPE_CHECKING:
+    from docutils.nodes import Node, system_message
+
+
+@dataclass
+class MarkupRenderer:
+    host: Host
+
+    def render(
+        self, text: str, inline: bool = False
+    ) -> tuple[list[Node], list[system_message]]:
+        if inline:
+            return self._render_inline(text)
+        else:
+            return self._render(text), []
+
+    def _render(self, text: str) -> list[Node]:
+        if isinstance(self.host, SphinxDirective):
+            return self.host.parse_text_to_nodes(text)
+        elif isinstance(self.host, SphinxTransform):
+            # TODO: dont create parser for every time
+            if version_info[0] >= 9:
+                parser = self.host.app.registry.create_source_parser(
+                    'rst', env=self.host.env, config=self.host.config
+                )
+            else:
+                parser = self.host.app.registry.create_source_parser(
+                    self.host.app, 'rst'
+                )
+            settings = self.host.document.settings
+            doc = new_document('<generated text>', settings=settings)
+            parser.parse(text, doc)
+
+            # NOTE: Nodes produced by standalone source parser should be fixed
+            # before returning, cause they missed the processing by certain
+            # Sphinx transforms.
+            self._fix_document(doc)
+
+            return doc.children
+        else:
+            assert False
+
+    def _render_inline(self, text: str) -> tuple[list[Node], list[system_message]]:
+        if isinstance(self.host, SphinxDirective):
+            return self.host.parse_inline(text)
+        if isinstance(self.host, SphinxRole):
+            inliner = self.host.inliner
+            memo = Struct(
+                document=inliner.document,
+                reporter=inliner.reporter,
+                language=inliner.language,
+            )
+
+            return inliner.parse(text, self.host.lineno, memo, inliner.parent)
+        elif isinstance(self.host, SphinxTransform):
+            # Fallback to normal non-inline render then extract inline
+            # elements by self.
+            # FIXME: error seems be ignored?
+            ns = self._render(text)
+            if ns and isinstance(ns[0], nodes.paragraph):
+                ns = ns[0].children
+            return ns, []
+        else:
+            assert False
+
+    def _fix_document(self, document: nodes.document) -> None:
+        assert isinstance(self.host, SphinxTransform)
+
+        """For documents generated by a separate source parser, some preprocessing
+        may be missing. For example:
+
+        - the lack of an ImageCollector result in an incorrect "node['uri']" and
+          a missing "node['candidates']" :class:`nodes.images
+
+        .. note::
+
+           Since this depends on Sphinx's internal implementation, there may be
+           many cases that have not been considered. We can only do our best
+           and fix each error we encounter as possible.
+        """
+
+        ImageCollector().process_doc(self.host.app, document)
+
+    def _fix_image_candidates(self, node: nodes.image) -> None:
+        """This is another way for fixing images node instead of calling
+        ImageCollector().process_doc. Keep it for now.
+        """
+        assert isinstance(self.host, SphinxTransform)
+
+        # NOTE:
+        # :meth:`sphinx.environment.collectors.ImageCollector.process_doc` add
+        # a 'candidates' key to nodes.image, and subclasses of
+        # :class:`sphinx.transforms.post_transforms.BaseImageConverter` require
+        # the key.
+        node['candidates'] = {}
+
+        # Update `node['uri']` to a relative path from srcdir.
+        node['uri'], _ = self.host.env.relfn2path(node['uri'])

sphinxnotes/render/meta.py

@@ -0,0 +1,35 @@
+# This file is generated from sphinx-notes/cookiecutter.
+# DO NOT EDIT!!!
+
+################################################################################
+# Project meta infos.
+################################################################################
+
+from __future__ import annotations
+from importlib import metadata
+
+__project__ = 'sphinxnotes.render'
+__author__ = 'Shengyu Zhang'
+__desc__ = 'A framework to define, constrain, and render data in Sphinx documentation'
+
+try:
+    __version__ = metadata.version('sphinxnotes.render')
+except metadata.PackageNotFoundError:
+    __version__ = 'unknown'
+
+
+################################################################################
+# Sphinx extension utils.
+################################################################################
+
+
+def pre_setup(app):
+    app.require_sphinx('7.0')
+
+
+def post_setup(app):
+    return {
+        'version': __version__,
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

sphinxnotes/render/pipeline.py

@@ -0,0 +1,300 @@
+"""
+sphinxnotes.render.pipeline
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:copyright: Copyright 2026 by the Shengyu Zhang.
+:license: BSD, see LICENSE for details.
+
+This module defines pipeline for rendering data to nodes.
+
+The Pipline
+===========
+
+1. Define context: BaseDataSource generates a :class:`pending_node`, which contains:
+
+   - Context
+   - Template for rendering data to markup text
+   - Possible extra contexts
+
+   See also :class:`BaseDataSource`.
+
+2. Render data: the ``pending_node`` nodes will be rendered
+   (by calling :meth:`pending_node.render`) at some point, depending on
+   :attr:`pending_node.template.phase`.
+
+   The one who calls ``pending_node.render`` is called ``Host``.
+   The ``Host`` host is responsible for rendering the markup text into docutils
+   nodes (See :class:`MarkupRenderer`).
+
+   Phases:
+
+   :``Phase.Parsing``:
+      Called by BaseDataSource ('s subclasses)
+
+   :``Phase.Parsed``:
+      Called by :class:`ParsedHookTransform`.
+
+   :``Phase.Resolving``:
+      Called by :class:`ResolvingHookTransform`.
+
+How context be rendered ``list[nodes.Node]``
+============================================
+
+.. seealso:: :meth:`.ctxnodes.pending_node.render`.
+
+"""
+
+from __future__ import annotations
+from typing import TYPE_CHECKING, override, final, cast
+from abc import abstractmethod, ABC
+
+from docutils import nodes
+from sphinx.util import logging
+from sphinx.util.docutils import SphinxDirective, SphinxRole
+from sphinx.transforms import SphinxTransform
+from sphinx.transforms.post_transforms import SphinxPostTransform, ReferencesResolver
+
+from .render import HostWrapper, Phase, Template, Host, ParseHost, TransformHost
+from .ctx import PendingContext, ResolvedContext
+from .ctxnodes import pending_node
+from .extractx import ExtraContextGenerator
+
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+logger = logging.getLogger(__name__)
+
+
+class Pipeline(ABC):
+    """
+    The core class defines the pipleing of rendering :class:`pending_node`s.
+
+    Subclass is responsible to:
+
+    - call ``queue_xxx`` to add pendin nodes into queue.
+    - override :meth:`process_pending_node` to control when a pending node gets
+      rendered. In this method subclass can also call ``queue_xxx`` to add more
+      pending nodes.
+    - call :meth:`render_queue` to process all queued nodes and
+      returns any that couldn't be rendered in the current phase.
+
+    See Also:
+
+    - :class:`BaseDataSource`: Context source implementation and hook for Phase.Parsing
+    - :class:`ParsedHookTransform`: Built-in hook for Phase.Parsed
+    - :class:`ResolvingHookTransform`: Built-in hook for Phase.Resolving
+    """
+
+    #: Queue of pending node to be rendered.
+    _q: list[pending_node] | None = None
+
+    """Methods to be overrided."""
+
+    def process_pending_node(self, n: pending_node) -> bool:
+        """
+        You can add hooks to pending node here.
+
+        Return ``true`` if you want to render the pending node *now*,
+        otherwise it will be inserted to doctree directly andwaiting to later
+        rendering
+        """
+        ...
+
+    """Helper method for subclasses."""
+
+    @final
+    def queue_pending_node(self, n: pending_node) -> None:
+        if not self._q:
+            self._q = []
+        self._q.append(n)
+
+    @final
+    def queue_context(
+        self, ctx: PendingContext | ResolvedContext, tmpl: Template
+    ) -> pending_node:
+        pending = pending_node(ctx, tmpl)
+        self.queue_pending_node(pending)
+        return pending
+
+    @final
+    def render_queue(self) -> list[pending_node]:
+        """
+        Try rendering all pending nodes in queue.
+
+        If the timing(Phase) is ok, :class:`pending_node` will be rendered
+        (pending.rendered = True); otherwise, the pending node is unchanged.
+
+        If the pending node is already inserted to document, it will not be return.
+        And the corrsponding rendered node will replace it too.
+        """
+
+        logger.debug(
+            f'{type(self)} is running its render queue, '
+            f'{len(self._q or [])} node(s) to render'
+        )
+        ns = []
+        while self._q:
+            pending = self._q.pop()
+
+            ok = self.process_pending_node(pending)
+            logger.debug(
+                f'{type(self)} is trying to render '
+                f'{pending.source}:{pending.line}, ok? {ok}'
+            )
+
+            if not ok:
+                ns.append(pending)
+                continue
+
+            # Generate global extra context for later use.
+            ExtraContextGenerator(pending).on_anytime()
+
+            host = cast(Host, self)
+            pending.render(host)
+
+            if pending.parent is None:
+                ns.append(pending)
+                continue
+
+            if pending.inline:
+                host_ = HostWrapper(host)
+                pending.unwrap_and_replace_self_inline((host_.doctree, pending.parent))
+            else:
+                pending.unwrap_and_replace_self()
+
+        logger.debug(
+            f'{type(self)} runs out of its render queue, '
+            f'{len(self._q or [])} node(s) hanging'
+        )
+
+        return ns
+
+
+class BaseContextSource(Pipeline):
+    """
+    Abstract base class for generateing context, as the source of the rendering
+    pipeline.
+
+    This class also responsible to render context in Phase.Parsing. So the final
+    implementations MUST be subclass of :class:`SphinxDirective` or
+    :class:`SphinxRole`, which provide the execution context and interface for
+    processing reStructuredText markup.
+    """
+
+    """Methods to be implemented."""
+
+    @abstractmethod
+    def current_context(self) -> PendingContext | ResolvedContext:
+        """Return the context to be rendered."""
+        ...
+
+    @abstractmethod
+    def current_template(self) -> Template:
+        """
+        Return the template for rendering the context.
+
+        This method should be implemented to provide the Jinja2 template
+        that will render the context into markup text. The template determines
+        the phase at which rendering occurs.
+
+        Returns:
+            The template to use for rendering.
+        """
+        ...
+
+    """Methods override from parent."""
+
+    @override
+    def process_pending_node(self, n: pending_node) -> bool:
+        host = cast(ParseHost, self)
+
+        # Set source and line.
+        host.set_source_info(n)
+        # Generate and save parsing phase extra context for later use.
+        ExtraContextGenerator(n).on_parsing(host)
+
+        return n.template.phase == Phase.Parsing
+
+
+class BaseContextDirective(BaseContextSource, SphinxDirective):
+    @override
+    def run(self) -> list[nodes.Node]:
+        self.queue_context(self.current_context(), self.current_template())
+
+        ns = []
+        for x in self.render_queue():
+            if not x.rendered:
+                ns.append(x)
+                continue
+            ns += x.unwrap()
+
+        return ns
+
+
+class BaseContextRole(BaseContextSource, SphinxRole):
+    @override
+    def process_pending_node(self, n: pending_node) -> bool:
+        n.inline = True
+        return super().process_pending_node(n)
+
+    @override
+    def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]:
+        pending = self.queue_context(self.current_context(), self.current_template())
+        pending.inline = True
+
+        ns, msgs = [], []
+        for n in self.render_queue():
+            if not n.rendered:
+                ns.append(n)
+                continue
+            ns_, msgs_ = n.unwrap_inline(self.inliner)
+            ns += ns_
+            msgs += msgs_
+
+        return ns, msgs
+
+
+class ParsedHookTransform(SphinxTransform, Pipeline):
+    # Before almost all others.
+    default_priority = 100
+
+    @override
+    def process_pending_node(self, n: pending_node) -> bool:
+        ExtraContextGenerator(n).on_parsed(cast(TransformHost, self))
+        return n.template.phase == Phase.Parsed
+
+    @override
+    def apply(self, **kwargs):
+        for pending in self.document.findall(pending_node):
+            self.queue_pending_node(pending)
+
+        for n in self.render_queue():
+            ...
+
+
+class ResolvingHookTransform(SphinxPostTransform, Pipeline):
+    # After resolving pending_xref
+    default_priority = (ReferencesResolver.default_priority or 10) + 5
+
+    @override
+    def process_pending_node(self, n: pending_node) -> bool:
+        ExtraContextGenerator(n).on_post_transform(cast(TransformHost, self))
+        return n.template.phase == Phase.Resolving
+
+    @override
+    def apply(self, **kwargs):
+        for pending in self.document.findall(pending_node):
+            self.queue_pending_node(pending)
+        ns = self.render_queue()
+
+        # NOTE: Should no node left.
+        assert len(ns) == 0
+
+
+def setup(app: Sphinx) -> None:
+    # Hook for Phase.Parsed.
+    app.add_transform(ParsedHookTransform)
+
+    # Hook for Phase.Resolving.
+    app.add_post_transform(ResolvingHookTransform)

sphinxnotes/render/render.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from enum import Enum
+
+from docutils import nodes
+from sphinx.transforms import SphinxTransform
+from sphinx.util.docutils import SphinxDirective, SphinxRole
+
+
+class Phase(Enum):
+    Parsing = 'parsing'
+    Parsed = 'parsed'
+    Resolving = 'resolving'
+
+    @classmethod
+    def default(cls) -> Phase:
+        return cls.Parsing
+
+
+@dataclass
+class Template:
+    #: Jinja template for rendering the context.
+    text: str
+    #: The render phase.
+    phase: Phase = Phase.default()
+    #: Enable debug output (shown as :class:`nodes.system_message` in document.)
+    debug: bool = False
+
+
+# Possible render host of :meth:`pending_node.render`.
+type Host = ParseHost | TransformHost
+# Host of source parse phase (Phase.Parsing, Phase.Parsed).
+type ParseHost = SphinxDirective | SphinxRole
+# Host of source parse phase (Phase.Parsing, Phase.Parsed).
+type TransformHost = SphinxTransform
+
+
+@dataclass
+class HostWrapper:
+    v: Host
+
+    @property
+    def doctree(self) -> nodes.document:
+        if isinstance(self.v, SphinxDirective):
+            return self.v.state.document
+        elif isinstance(self.v, SphinxRole):
+            return self.v.inliner.document
+        elif isinstance(self.v, SphinxTransform):
+            return self.v.document
+        else:
+            raise NotImplementedError
+
+    @property
+    def parent(self) -> nodes.Element | None:
+        if isinstance(self.v, SphinxDirective):
+            return self.v.state.parent
+        elif isinstance(self.v, SphinxRole):
+            return self.v.inliner.parent
+        else:
+            return None