structured-tutorials 0.1.0__py3-none-any.whl

structured_tutorials/output.py

@@ -0,0 +1,139 @@
+# Copyright (c) 2025 Mathias Ertl
+# Licensed under the MIT License. See LICENSE file for details.
+
+"""Collect functions related to output."""
+
+import logging
+import logging.config
+import sys
+from typing import Any, ClassVar, Literal
+
+from colorama import Fore, Style, just_fix_windows_console
+from termcolor import colored
+
+just_fix_windows_console()  # needed on Windows
+
+LOG_LEVELS = Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+
+
+def error(text: str) -> None:
+    """Output a red/bold line on stderr."""
+    print(colored(text, "red", attrs=["bold"]), file=sys.stderr)
+
+
+class ColorFormatter(logging.Formatter):
+    """Base class for color-based formatters."""
+
+    def __init__(self, *args: Any, no_colors: bool = False, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        # Decide once at formatter creation time
+        self.use_colors = sys.stderr.isatty() and no_colors is False
+
+
+class LevelColorFormatter(ColorFormatter):
+    """Formatter that colors the log level."""
+
+    COLORS: ClassVar[dict[int, str]] = {
+        logging.DEBUG: Fore.CYAN,
+        logging.INFO: Fore.GREEN,
+        logging.WARNING: Fore.YELLOW,
+        logging.ERROR: Fore.RED,
+        logging.CRITICAL: Fore.MAGENTA,
+    }
+
+    def format(self, record: logging.LogRecord) -> str:  # pragma: no cover
+        if not self.use_colors:
+            return super().format(record)
+
+        level_name = record.levelname
+        color = self.COLORS.get(record.levelno, "")
+        record.levelname = f"{color}{level_name.ljust(8)}{Style.RESET_ALL}"
+        try:
+            return super().format(record)
+        finally:
+            record.levelname = level_name  # restore for other handlers
+
+
+class BoldFormatter(ColorFormatter):
+    """Formatter that outputs all messages in bold, if colors are used."""
+
+    def format(self, record: logging.LogRecord) -> str:  # pragma: no cover
+        if not self.use_colors:
+            return super().format(record)
+
+        original = record.msg
+        record.msg = f"{Style.BRIGHT}{original}{Style.RESET_ALL}"
+        try:
+            return super().format(record)
+        finally:
+            record.msg = original  # restore for other handlers
+
+
+class CommandFormatter(logging.Formatter):
+    """Formatter that prepends any log message with a '+ ' (same as `set -x` on a shell)."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        original = record.msg
+        record.msg = f"+ {original}"
+        try:
+            return super().format(record)
+        finally:
+            record.msg = original  # restore for other handlers
+
+
+def setup_logging(level: LOG_LEVELS, no_colors: bool, show_commands: bool) -> None:
+    """Setup logging for the process."""
+    command_log_level: LOG_LEVELS = "INFO" if show_commands else "WARNING"
+
+    config = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "colored": {
+                "()": "structured_tutorials.output.LevelColorFormatter",
+                "format": "%(levelname)-8s | %(message)s",
+                "no_colors": no_colors,
+            },
+            "bold": {
+                "()": "structured_tutorials.output.BoldFormatter",
+                "format": "%(message)s",
+                "no_colors": no_colors,
+            },
+            "command": {
+                "()": "structured_tutorials.output.CommandFormatter",
+                "format": "%(message)s",
+            },
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+                "formatter": "colored",
+            },
+            "part": {
+                "class": "logging.StreamHandler",
+                "formatter": "bold",
+            },
+            "command": {
+                "class": "logging.StreamHandler",
+                "formatter": "command",
+            },
+        },
+        "loggers": {
+            "part": {
+                "handlers": ["part"],
+                "propagate": False,
+                "level": level,
+            },
+            "command": {
+                "handlers": ["command"],
+                "propagate": False,
+                "level": command_log_level,
+            },
+        },
+        "root": {
+            "level": level,
+            "handlers": ["console"],
+        },
+    }
+
+    logging.config.dictConfig(config)

structured_tutorials/runners/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2025 Mathias Ertl
+# Licensed under the MIT License. See LICENSE file for details.

structured_tutorials/runners/base.py

@@ -0,0 +1,115 @@
+# Copyright (c) 2025 Mathias Ertl
+# Licensed under the MIT License. See LICENSE file for details.
+
+"""Base classes for runners."""
+
+import abc
+import io
+import logging
+import shlex
+import subprocess
+import sys
+from copy import deepcopy
+from subprocess import CompletedProcess
+from typing import Any
+
+from jinja2 import Environment
+
+from structured_tutorials.errors import InvalidAlternativesSelectedError
+from structured_tutorials.models import AlternativeModel, TutorialModel
+from structured_tutorials.models.base import CommandType
+from structured_tutorials.models.parts import CleanupCommandModel
+
+command_logger = logging.getLogger("command")
+
+
+class RunnerBase(abc.ABC):
+    """Base class for runners to provide shared functionality."""
+
+    def __init__(
+        self,
+        tutorial: TutorialModel,
+        alternatives: tuple[str, ...] = (),
+        show_command_output: bool = True,
+        interactive: bool = True,
+    ):
+        self.tutorial = tutorial
+        self.context = deepcopy(tutorial.configuration.context)
+        self.context.update(deepcopy(tutorial.configuration.run.context))
+        self.env = Environment(keep_trailing_newline=True)
+        self.cleanup: list[CleanupCommandModel] = []
+        self.alternatives = alternatives
+        self.show_command_output = show_command_output
+        self.interactive = interactive
+
+    def render(self, value: str, **context: Any) -> str:
+        return self.env.from_string(value).render({**self.context, **context})
+
+    def render_command(self, command: CommandType, **context: Any) -> CommandType:
+        if isinstance(command, str):
+            return self.render(command)
+
+        return tuple(self.render(token) for token in command)
+
+    def validate_alternatives(self) -> None:
+        """Validate that for each alternative part, an alternative was selected."""
+        chosen = set(self.alternatives)
+
+        for part_no, part in enumerate(self.tutorial.parts, start=1):
+            if isinstance(part, AlternativeModel):
+                selected = chosen & set(part.alternatives)
+
+                if part.required and len(selected) == 0:
+                    raise InvalidAlternativesSelectedError(f"Part {part_no}: No alternative selected.")
+                elif len(selected) != 1:
+                    raise InvalidAlternativesSelectedError(
+                        f"Part {part_no}: More then one alternative selected: {selected}"
+                    )
+
+    def run_shell_command(
+        self,
+        command: CommandType,
+        show_output: bool,
+        capture_output: bool = False,
+        stdin: int | io.BufferedReader | None = None,
+        input: bytes | None = None,
+    ) -> CompletedProcess[bytes]:
+        # Only show output if runner itself is not configured to hide all output
+        if show_output:
+            show_output = self.show_command_output
+
+        if capture_output:
+            stdout: int | None = subprocess.PIPE
+            stderr: int | None = subprocess.PIPE
+        elif show_output:
+            stdout = stderr = None
+        else:
+            stdout = stderr = subprocess.DEVNULL
+
+        # Render the command (args) as template
+        command = self.render_command(command)
+
+        shell = True
+        logged_command = command  # The command string we pass to the logger
+        if isinstance(command, tuple):
+            shell = False
+            logged_command = shlex.join(logged_command)
+
+        command_logger.info(logged_command)
+        proc = subprocess.run(command, shell=shell, stdin=stdin, input=input, stdout=stdout, stderr=stderr)
+
+        # If we want to show the output and capture it at the same time, we need can only show the streams
+        # separately at the end.
+        if capture_output and show_output:
+            print("--- stdout ---")
+            sys.stdout.buffer.write(proc.stdout + b"\n")
+            sys.stdout.buffer.flush()
+            print("--- stderr ---")
+            sys.stdout.buffer.write(proc.stderr + b"\n")
+            sys.stdout.buffer.flush()
+
+        return proc
+
+    @abc.abstractmethod
+    def run(self) -> None:
+        """Run the tutorial."""

structured_tutorials/runners/local.py

@@ -0,0 +1,254 @@
+# Copyright (c) 2025 Mathias Ertl
+# Licensed under the MIT License. See LICENSE file for details.
+
+"""Runner that runs a tutorial on the local machine."""
+
+import logging
+import os
+import shlex
+import shutil
+import socket
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+from structured_tutorials.errors import CommandOutputTestError, CommandTestError, PromptNotConfirmedError
+from structured_tutorials.models.parts import AlternativeModel, CommandsPartModel, FilePartModel, PromptModel
+from structured_tutorials.models.tests import TestCommandModel, TestOutputModel, TestPortModel
+from structured_tutorials.runners.base import RunnerBase
+from structured_tutorials.utils import chdir, cleanup, git_export
+
+log = logging.getLogger(__name__)
+part_log = logging.getLogger("part")
+
+
+class LocalTutorialRunner(RunnerBase):
+    """Runner implementation that runs a tutorial on the local machine."""
+
+    def run_test(
+        self,
+        test: TestCommandModel | TestPortModel | TestOutputModel,
+        proc: subprocess.CompletedProcess[bytes],
+    ) -> None:
+        # If the test is for an output stream, we can run it right away (the process has already finished).
+        if isinstance(test, TestOutputModel):
+            if test.stream == "stderr":
+                value = proc.stderr
+            else:
+                value = proc.stdout
+
+            if (match := test.regex.search(value)) is not None:
+                self.context.update({k: v.decode("utf-8") for k, v in match.groupdict().items()})
+                return
+            else:
+                decoded = value.decode("utf-8")
+                raise CommandOutputTestError(f"Process did not have the expected output: '{decoded}'")
+
+        # If an initial delay is configured, wait that long
+        if test.delay > 0:
+            time.sleep(test.delay)
+
+        tries = 0
+        while tries <= test.retry:
+            tries += 1
+
+            if isinstance(test, TestCommandModel):
+                test_proc = self.run_shell_command(test.command, show_output=test.show_output)
+
+                if test.status_code == test_proc.returncode:
+                    return
+                else:
+                    log.error("%s: Test command failed.", shlex.join(test_proc.args))
+            else:
+                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                try:
+                    s.connect((test.host, test.port))
+                except Exception:
+                    log.error("%s:%s: failed to connect.", test.host, test.port)
+                else:
+                    return
+
+            wait = test.backoff_factor * (2 ** (tries - 1))
+            if wait > 0 and tries <= test.retry:
+                time.sleep(wait)
+
+        raise CommandTestError("Test did not pass")
+
+    def run_commands(self, part: CommandsPartModel) -> None:
+        for command_config in part.commands:
+            if command_config.run.skip:
+                continue
+
+            # Capture output if any test is for the output.
+            capture_output = any(isinstance(test, TestOutputModel) for test in command_config.run.test)
+
+            proc_input = None
+            if stdin_config := command_config.run.stdin:
+                if stdin_config.contents:
+                    proc_input = self.render(stdin_config.contents).encode("utf-8")
+                elif stdin_config.template:  # source path, but template=True
+                    assert stdin_config.source is not None
+                    with open(self.tutorial.tutorial_root / stdin_config.source) as stream:
+                        stdin_template = stream.read()
+                    proc_input = self.render(stdin_template).encode("utf-8")
+
+            # Run the command and check status code
+            if (
+                command_config.run.stdin
+                and command_config.run.stdin.source
+                and not command_config.run.stdin.template
+            ):
+                with open(self.tutorial.tutorial_root / command_config.run.stdin.source, "rb") as stdin:
+                    proc = self.run_shell_command(
+                        command_config.command,
+                        show_output=command_config.run.show_output,
+                        capture_output=capture_output,
+                        stdin=stdin,
+                    )
+            else:
+                proc = self.run_shell_command(
+                    command_config.command,
+                    show_output=command_config.run.show_output,
+                    capture_output=capture_output,
+                    input=proc_input,
+                )
+
+            # Update list of cleanup commands
+            self.cleanup = list(command_config.run.cleanup) + self.cleanup
+
+            # Handle errors in commands
+            if proc.returncode != command_config.run.status_code:
+                raise RuntimeError(
+                    f"{command_config.command} failed with return code {proc.returncode} "
+                    f"(expected: {command_config.run.status_code})."
+                )
+
+            # Update the context from update_context
+            self.context.update(command_config.run.update_context)
+
+            if command_config.run.chdir is not None:
+                log.info("Changing working directory to %s.", command_config.run.chdir)
+                os.chdir(command_config.run.chdir)
+
+            # Run test commands
+            for test_command_config in command_config.run.test:
+                self.run_test(test_command_config, proc)
+
+    def write_file(self, part: FilePartModel) -> None:
+        """Write a file."""
+        raw_destination = self.render(part.destination)
+        destination = Path(raw_destination)
+
+        if raw_destination.endswith(os.path.sep):
+            # Model validation already validates that the destination does not look like a directory, if no
+            # source is set, but this could be tricked if the destination is a template.
+            if not part.source:
+                raise RuntimeError(
+                    f"{raw_destination}: Destination is directory, but no source given to derive filename."
+                )
+
+            destination.mkdir(parents=True, exist_ok=True)
+            destination = destination / part.source.name
+        elif destination.exists():
+            raise RuntimeError(f"{destination}: Destination already exists.")
+
+        # Create any parent directories
+        destination.parent.mkdir(parents=True, exist_ok=True)
+
+        # If template=False and source is set, we just copy the file as is, without ever reading it
+        if not part.template and part.source:
+            shutil.copy(part.source, destination)
+            return
+
+        if part.source:
+            with open(self.tutorial.tutorial_root / part.source) as source_stream:
+                template = source_stream.read()
+        else:
+            assert isinstance(part.contents, str)  # assured by model validation
+            template = part.contents
+
+        if part.template:
+            contents = self.render(template)
+        else:
+            contents = template
+
+        with open(destination, "w") as destination_stream:
+            destination_stream.write(contents)
+
+    def run_prompt(self, part: PromptModel) -> None:
+        prompt = self.render(part.prompt).strip() + " "
+
+        if part.response == "enter":
+            input(prompt)
+        else:  # type == confirm
+            valid_inputs = ("n", "no", "yes", "y", "")
+            while (response := input(prompt).strip().lower()) not in valid_inputs:
+                print(f"Please enter a valid value ({'/'.join(valid_inputs)}).")
+
+            if response in ("n", "no") or (response == "" and not part.default):
+                error = self.render(part.error, response=response)
+                raise PromptNotConfirmedError(error)
+
+    def run_alternative(self, part: AlternativeModel) -> None:
+        selected = set(self.alternatives) & set(part.alternatives)
+
+        # Note: The CLI agent already verifies this - just assert this to be sure.
+        assert len(selected) <= 1, "More then one part selected."
+
+        if selected:
+            selected_part = part.alternatives[next(iter(selected))]
+            if isinstance(selected_part, CommandsPartModel):
+                self.run_commands(selected_part)
+            elif isinstance(selected_part, FilePartModel):
+                self.write_file(selected_part)
+            else:  # pragma: no cover
+                raise RuntimeError(f"{selected_part} is not supported as alternative.")
+
+    def run_parts(self) -> None:
+        for part in self.tutorial.parts:
+            if part.name:  # pragma: no cover
+                part_log.info(part.name)
+            else:
+                part_log.info(f"Running part {part.id}...")
+
+            if isinstance(part, PromptModel):
+                if self.interactive:
+                    self.run_prompt(part)
+                continue
+
+            if part.run.skip:
+                continue
+
+            if isinstance(part, CommandsPartModel):
+                self.run_commands(part)
+            elif isinstance(part, FilePartModel):
+                self.write_file(part)
+            elif isinstance(part, AlternativeModel):
+                self.run_alternative(part)
+            else:  # pragma: no cover
+                raise RuntimeError(f"{part} is not a tutorial part")
+
+            self.context.update(part.run.update_context)
+
+    def run(self) -> None:
+        if self.tutorial.configuration.run.temporary_directory:
+            with tempfile.TemporaryDirectory() as tmpdir_name:
+                log.info("Switching to temporary directory: %s", tmpdir_name)
+                self.context["cwd"] = self.context["temp_dir"] = Path(tmpdir_name)
+                self.context["orig_cwd"] = Path.cwd()
+
+                with chdir(tmpdir_name), cleanup(self):
+                    self.run_parts()
+        elif self.tutorial.configuration.run.git_export:
+            with tempfile.TemporaryDirectory() as tmpdir_name:
+                work_dir = git_export(tmpdir_name)
+                log.info("Creating git export at: %s", work_dir)
+                self.context["cwd"] = self.context["export_dir"] = work_dir
+                self.context["orig_cwd"] = Path.cwd()
+
+                with chdir(work_dir), cleanup(self):
+                    self.run_parts()
+        else:
+            with cleanup(self):
+                self.run_parts()

structured_tutorials/sphinx/__init__.py

@@ -0,0 +1,35 @@
+"""Sphinx extension for rendering tutorials."""
+
+from pathlib import Path
+
+from sphinx.application import Sphinx
+from sphinx.errors import ExtensionError
+from sphinx.util.typing import ExtensionMetadata
+
+from structured_tutorials import __version__
+from structured_tutorials.sphinx.directives import PartDirective, TutorialDirective
+from structured_tutorials.sphinx.utils import validate_configuration
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+    """Sphinx setup function."""
+    # Add dependency on other extension:
+    # app.setup_extension("sphinx.ext.autodoc")
+    app.connect("config-inited", validate_configuration)
+    app.add_config_value("tutorial_root", Path(app.srcdir), "env", types=[Path])
+    app.add_config_value("structured_tutorial_command_text_width", 75, "env", types=[int])
+
+    app.add_directive("structured-tutorial", TutorialDirective)
+    app.add_directive("structured-tutorial-part", PartDirective)
+
+    try:
+        app.setup_extension("sphinx_inline_tabs")
+    except Exception as exc:  # pragma: no cover
+        raise ExtensionError("structured_tutorials requires sphinx_inline_tabs") from exc
+
+    # return metadata
+    return {
+        "version": __version__,
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

structured_tutorials/sphinx/directives.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2025 Mathias Ertl
+# Licensed under the MIT License. See LICENSE file for details.
+
+"""Directives for Sphinx."""
+
+from typing import TYPE_CHECKING
+
+from docutils.nodes import Node, paragraph
+from docutils.parsers.rst.states import RSTState
+from docutils.statemachine import StringList
+from sphinx.environment import BuildEnvironment
+from sphinx.util.docutils import SphinxDirective
+
+from structured_tutorials.sphinx.utils import TutorialWrapper, get_tutorial_path
+
+if TYPE_CHECKING:
+    from sphinx.environment import _CurrentDocument
+
+
+class CurrentDocumentMixin:
+    """Mixin adding the current document property and context."""
+
+    if TYPE_CHECKING:
+        env: BuildEnvironment
+
+    # NOTE: sphinx 8.2.0 introduced "current_document", temp_data is deprecated and kept only for
+    #   backwards compatability: https://github.com/sphinx-doc/sphinx/pull/13151
+    @property
+    def current_document(self) -> "_CurrentDocument":  # pragma: no cover
+        if hasattr(self.env, "current_document"):
+            return self.env.current_document
+        else:
+            return self.env.temp_data
+
+
+class TutorialDirective(CurrentDocumentMixin, SphinxDirective):
+    """Directive to specify the currently rendered tutorial."""
+
+    has_content = False
+    required_arguments = 1
+    optional_arguments = 0
+
+    def run(self) -> list[Node]:
+        tutorial_arg = self.arguments[0].strip()
+
+        command_text_width: int = self.config.structured_tutorial_command_text_width
+        tutorial_path = get_tutorial_path(self.config.tutorial_root, tutorial_arg)
+        self.current_document["tutorial"] = TutorialWrapper.from_file(
+            tutorial_path, command_text_width=command_text_width
+        )
+
+        # NOTE: `highlighting` directive returns a custom Element for unknown reasons
+        return []
+
+
+class PartDirective(CurrentDocumentMixin, SphinxDirective):
+    """Directive to show a tutorial part."""
+
+    required_arguments = 0
+    optional_arguments = 1
+
+    def run(self) -> list[paragraph]:
+        # Get the named ID if set.
+        part_id = None
+        if self.arguments:
+            part_id = self.arguments[0].strip()
+
+        # Render text
+        tutorial_wrapper: TutorialWrapper = self.current_document["tutorial"]
+        text = tutorial_wrapper.render_part(part_id=part_id)
+
+        source, _lineno = self.get_source_info()
+
+        # Create sphinx node
+        node = paragraph()
+        paragraph.source = source
+        state: RSTState = self.state
+        state.nested_parse(StringList(text.splitlines(), source=source), 0, node)
+        return [node]