paulrun 0.1.0__py3-none-any.whl

paulrun/backends/__init__.py

@@ -0,0 +1,178 @@
+import importlib.metadata
+import os
+import signal
+import subprocess
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from importlib.metadata import EntryPoint
+from pathlib import Path
+from typing import Any, Protocol
+
+ENTRY_POINT_GROUP = "paulrun.backends"
+
+
+@dataclass(frozen=True)
+class Problem:
+    """Something a backend found wrong with a block."""
+
+    message: str
+    line: int | None = None  # line in the block, counting its first line as 1; None if it's about the whole block
+    warning: bool = False  # warnings are reported but don't stop the runbook running
+
+
+class Backend(Protocol):
+    """Runs `run` blocks for one or more languages.
+
+    Backends are classes registered under the `paulrun.backends` entry point group. Each is created
+    once, with no arguments, when backends are loaded.
+    """
+
+    # Read-only, so a backend can set them as plain class attributes.
+    @property
+    def name(self) -> str: ...
+
+    @property
+    def languages(self) -> tuple[str, ...]:
+        """The fence languages this backend runs, e.g. ("sh", "bash")."""
+
+    def validate(self, code: str) -> list[Problem]:
+        """Return the problems found in a block, without running it. An empty list means it's fine.
+
+        code is the block as written, so it can contain <NAME> placeholders. Replacing them with
+        something this language accepts is up to the backend. Any error stops the runbook running;
+        warnings are only reported.
+        """
+
+    def run(
+        self,
+        code: str,
+        *,
+        frontmatter: Mapping[str, Any],
+        env: Mapping[str, str],
+        cwd: Path,
+        output: Callable[[str], None],
+    ) -> int:
+        """Run a block and return its exit code, where 0 means it succeeded.
+
+        Call output with each line the block prints, without its newline, as soon as it's printed.
+        Don't print anything directly; paulrun owns the terminal. frontmatter is the runbook's
+        frontmatter, read-only. env is the block's complete environment and cwd is where it runs.
+        """
+
+
+class Interrupted(KeyboardInterrupt):
+    """Ctrl-C was pressed while a block ran. The block has exited, with exit_code."""
+
+    def __init__(self, exit_code: int):
+        super().__init__(exit_code)
+        self.exit_code = exit_code
+
+
+def stream(command: Sequence[str], *, env: Mapping[str, str], cwd: Path, output: Callable[[str], None]) -> int:
+    """Run command, passing each line it prints to output as it's printed, and return its exit code.
+
+    stdout and stderr are merged, so lines keep the order they were printed in. The command shares
+    paulrun's terminal, so it can still ask for a password on it, as git, ssh and gpg do. This is how
+    the built-in backends run blocks, and other backends can use it too.
+
+    Ctrl-C reaches the command as well as paulrun. On the first press, paulrun waits for the command to
+    stop, still passing on what it prints. A second press kills it and everything it started, and stops
+    reading, so nothing can keep paulrun waiting. Either way, once the command has exited, Interrupted
+    is raised with its exit code.
+    """
+    interrupted = False
+    with subprocess.Popen(command, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as process:
+        assert process.stdout is not None  # set because stdout is a pipe
+        while True:
+            try:
+                # Read bytes so only newlines split lines; text mode would also split on carriage returns.
+                for raw in process.stdout:
+                    output(raw.decode("utf-8", errors="replace").removesuffix("\n"))
+                break
+            except KeyboardInterrupt:
+                if not interrupted:
+                    # The terminal sent Ctrl-C to the command too, so let it stop in its own way.
+                    interrupted = True
+                    continue
+                _kill_tree(process.pid)
+                # Stop reading, so nothing still holding the output open can keep paulrun waiting.
+                process.stdout.close()
+                break
+        process.wait()
+    if interrupted:
+        raise Interrupted(process.returncode)
+    return process.returncode
+
+
+def _kill_tree(pid: int) -> None:
+    """Kill a process and every process it started, however deep.
+
+    Each one is stopped first, so none can start another between being found and being killed.
+    """
+    found = [pid]
+    for parent in found:
+        _signal(parent, signal.SIGSTOP)
+        found.extend(child for child in _children(parent) if child not in found)
+    for process in found:
+        _signal(process, signal.SIGKILL)
+
+
+def _children(pid: int) -> list[int]:
+    """The processes pid started that are still running."""
+    listing = subprocess.run(["ps", "-A", "-o", "pid=", "-o", "ppid="], capture_output=True, text=True).stdout
+    return [int(child) for child, parent in (row.split() for row in listing.splitlines()) if int(parent) == pid]
+
+
+def _signal(pid: int, signal_number: int) -> None:
+    """Signal a process, unless it has already gone."""
+    try:
+        os.kill(pid, signal_number)
+    except ProcessLookupError:
+        pass
+
+
+class BackendError(Exception):
+    """Backends can't be loaded, e.g. two of them claim the same language."""
+
+
+@dataclass(frozen=True)
+class LoadedBackend:
+    backend: Backend
+    entry_point: EntryPoint  # where it was registered, e.g. paulrun's own pyproject.toml or another package's
+
+
+def installed_entry_points() -> Iterable[EntryPoint]:
+    """Every backend registered under the paulrun.backends group by an installed package, paulrun included."""
+    return importlib.metadata.entry_points(group=ENTRY_POINT_GROUP)
+
+
+def load_backend_sources(entry_points: Iterable[EntryPoint] | None = None) -> list[LoadedBackend]:
+    """Load each backend once, with where it came from, from the installed entry points unless others are given.
+
+    Raises BackendError if two backends claim the same language.
+    """
+    if entry_points is None:
+        entry_points = installed_entry_points()
+    loaded: list[LoadedBackend] = []
+    claimed: dict[str, LoadedBackend] = {}
+    for entry_point in entry_points:
+        source = LoadedBackend(entry_point.load()(), entry_point)
+        for language in source.backend.languages:
+            if language in claimed:
+                other = claimed[language]
+                raise BackendError(
+                    f"language {language!r} is claimed by both {other.backend.name!r} ({other.entry_point.value}) "
+                    f"and {source.backend.name!r} ({entry_point.value})"
+                )
+            claimed[language] = source
+        loaded.append(source)
+    return loaded
+
+
+def load_backends(entry_points: Iterable[EntryPoint] | None = None) -> dict[str, Backend]:
+    """Load backends keyed by language, from the installed entry points unless others are given."""
+    return {
+        language: source.backend
+        for source in load_backend_sources(entry_points)
+        for language in source.backend.languages
+    }

paulrun/backends/python.py

@@ -0,0 +1,51 @@
+import shlex
+import sys
+import tempfile
+import warnings
+from collections.abc import Callable, Mapping
+from pathlib import Path
+from typing import Any
+
+from paulrun.backends import Problem, stream
+from paulrun.inputs import PLACEHOLDER
+
+
+class PythonBackend:
+    """Runs each block as a Python script, with the runbook's python command or paulrun's own interpreter."""
+
+    name = "python"
+    languages = ("python", "py")
+
+    def validate(self, code: str) -> list[Problem]:
+        # Swap each placeholder for its bare name, which is a plain identifier wherever it appears, so the code
+        # still compiles and keeps its line numbers. compile() uses paulrun's own interpreter, so syntax only the
+        # runbook's python understands isn't caught.
+        source = PLACEHOLDER.sub(lambda match: match.group(1), code)
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always", SyntaxWarning)
+            try:
+                compile(source, "<block>", "exec")
+            except SyntaxError as e:
+                return [Problem(e.msg, line=e.lineno)]
+        return [
+            Problem(str(warning.message), line=warning.lineno, warning=True)
+            for warning in caught
+            if issubclass(warning.category, SyntaxWarning)
+        ]
+
+    def run(
+        self,
+        code: str,
+        *,
+        frontmatter: Mapping[str, Any],
+        env: Mapping[str, str],
+        cwd: Path,
+        output: Callable[[str], None],
+    ) -> int:
+        python = frontmatter.get("python")
+        command = shlex.split(python) if isinstance(python, str) else [sys.executable]
+        # A file rather than -c, so any interpreter command works, e.g. `uv run python`.
+        with tempfile.TemporaryDirectory(prefix="paulrun-") as directory:
+            script = Path(directory) / "block.py"
+            script.write_text(code, encoding="utf-8")
+            return stream([*command, str(script)], env=env, cwd=cwd, output=output)

paulrun/backends/shell.py

@@ -0,0 +1,80 @@
+import json
+import re
+import shutil
+import subprocess
+from collections.abc import Callable, Mapping
+from pathlib import Path
+from typing import Any
+
+from paulrun.backends import Problem, stream
+from paulrun.inputs import PLACEHOLDER
+
+# How bash -n reports each problem, e.g. "bash: line 3: syntax error near unexpected token `fi'".
+_BASH_ERROR = re.compile(r"^\S*bash: line (\d+): (.*)$")
+
+
+class ShellBackend:
+    """Runs each block as one bash script that stops at the first failing command."""
+
+    name = "shell"
+    languages = ("sh", "bash", "shell")
+
+    def validate(self, code: str) -> list[Problem]:
+        # Bash reads <NAME> as a redirect, so swap each placeholder for its bare name, which is a plain word
+        # wherever it appears.
+        script = PLACEHOLDER.sub(lambda match: match.group(1), code)
+        return _bash_errors(script) or _shellcheck_warnings(script)
+
+    def run(
+        self,
+        code: str,
+        *,
+        frontmatter: Mapping[str, Any],
+        env: Mapping[str, str],
+        cwd: Path,
+        output: Callable[[str], None],
+    ) -> int:
+        # The flags match `set -eo pipefail` without adding a line, so the script bash runs is exactly the
+        # block. The last argument is $0, which bash puts at the start of its error messages.
+        return stream(["bash", "-e", "-o", "pipefail", "-c", code, "paulrun"], env=env, cwd=cwd, output=output)
+
+
+def _bash_errors(script: str) -> list[Problem]:
+    # The script goes in on stdin because bash -c numbers lines from 0 on macOS.
+    result = subprocess.run(["bash", "-n"], input=script, capture_output=True, text=True, errors="replace")
+    if result.returncode == 0:
+        return []
+    problems = []
+    for line in result.stderr.splitlines():
+        if match := _BASH_ERROR.match(line):
+            number, message = int(match.group(1)), match.group(2)
+            # After a syntax error bash prints the offending source line in quotes, which adds nothing.
+            if problems and problems[-1].line == number and message.startswith("`") and message.endswith("'"):
+                continue
+            problems.append(Problem(message, line=number))
+        elif line:
+            problems.append(Problem(line))
+    return problems or [Problem(f"bash -n failed with exit code {result.returncode}")]
+
+
+def _shellcheck() -> str | None:
+    """Where shellcheck is installed, or None if it isn't."""
+    return shutil.which("shellcheck")
+
+
+def _shellcheck_warnings(script: str) -> list[Problem]:
+    """shellcheck's findings, all as warnings, or nothing if shellcheck isn't installed."""
+    shellcheck = _shellcheck()
+    if shellcheck is None:
+        return []
+    result = subprocess.run(
+        [shellcheck, "--shell=bash", "--format=json", "-"], input=script, capture_output=True, text=True
+    )
+    try:
+        findings = json.loads(result.stdout)
+    except json.JSONDecodeError:
+        return [Problem(f"shellcheck failed: {result.stderr.strip() or result.stdout.strip()}", warning=True)]
+    return [
+        Problem(f"SC{finding['code']}: {finding['message']}", line=finding["line"], warning=True)
+        for finding in findings
+    ]

paulrun/cli.py

@@ -0,0 +1,233 @@
+import sys
+from contextlib import nullcontext
+from dataclasses import dataclass
+from importlib.metadata import EntryPoint
+from pathlib import Path
+from typing import Any
+
+import click
+from rich.console import Console
+
+from paulrun import runner, settings
+from paulrun.backends import Backend, BackendError, load_backend_sources, load_backends
+from paulrun.inputs import Input, InputError
+from paulrun.runbook import Runbook, RunbookError, parse, parse_inputs, read_frontmatter
+from paulrun.transcript import Transcript, TranscriptError
+from paulrun.validate import Report, validate
+
+
+@click.group("paulrun")
+@click.version_option(settings.VERSION)
+def cli() -> None:
+    """Run hand-written markdown runbooks."""
+
+
+@cli.command()
+@click.argument("runbook", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option("--dry", is_flag=True, help="Show every block with inputs filled in, without running anything.")
+@click.option("--step", is_flag=True, help="Ask before each run block whether to run it, skip it or quit.")
+def go(runbook: Path, dry: bool, step: bool) -> None:
+    """Run a runbook's steps in order, stopping at the first that fails."""
+    console = Console(highlight=False)
+    parsed, backends, report = _validated(runbook)
+    _print_report(console, runbook, report, summary_if_ok=False)
+    if not report.ok:
+        sys.exit(1)
+    # Nothing runs under --dry, so there's nothing to record.
+    transcript = Transcript.for_runbook(parsed) if parsed.output_path is not None and not dry else None
+
+    def sink(event: runner.Event) -> None:
+        _print_event(console, event, dry=dry)
+        if transcript is not None:
+            transcript(event)
+
+    try:
+        with transcript or nullcontext():
+            result = runner.run(parsed, backends, sink, ClickPrompter(console), dry=dry, step_by_step=step)
+    except (runner.RunnerError, InputError, TranscriptError) as e:
+        raise click.ClickException(str(e)) from e
+    if transcript is not None and transcript.written:
+        console.out(f"transcript: {transcript.path}")
+    if result.status not in ("ok", "dry"):
+        sys.exit(1)
+
+
+@cli.command()
+@click.argument("runbook", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+def check(runbook: Path) -> None:
+    """Check a runbook for problems without running anything. Exits 1 if there are any errors."""
+    console = Console(highlight=False)
+    _, _, report = _validated(runbook)
+    _print_report(console, runbook, report, summary_if_ok=True)
+    if not report.ok:
+        sys.exit(1)
+
+
+@cli.command()
+@click.argument("runbook", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+def inputs(runbook: Path) -> None:
+    """List the inputs a runbook needs, including ones its input blocks ask for part way through.
+
+    Frontmatter inputs are listed even if the rest of the runbook is broken.
+    """
+    console = Console(highlight=False)
+    try:
+        declared: list[tuple[Input, str | None]] = [
+            (input, None) for input in parse_inputs(read_frontmatter(runbook).get("inputs"))
+        ]
+    except RunbookError as e:
+        raise click.ClickException(str(e)) from e
+    declared += _input_blocks(runbook)
+    if not declared:
+        console.out("no inputs")
+        return
+    width = max(len(input.name) for input, _ in declared)
+    for input, step in declared:
+        console.out(f"{input.name:<{width}}  {input.description}")
+        details = [f"pattern: {input.pattern}"] if input.pattern is not None else []
+        details += ["secret"] if input.secret else []
+        details += [f'asked in "{step}"'] if step is not None else []
+        for detail in details:
+            console.out(f"{'':<{width}}  {detail}")
+
+
+def _input_blocks(path: Path) -> list[tuple[Input, str | None]]:
+    """The inputs a runbook's input blocks declare, with the step each is in. Nothing if the body can't be read."""
+    try:
+        parsed = parse(path)
+    except RunbookError:
+        return []
+    return [(block.input, step.name) for step in parsed.steps for block in step.blocks if block.input is not None]
+
+
+@cli.command("backends")
+def list_backends() -> None:
+    """List the installed backends, the languages each runs, and the package that registered it."""
+    console = Console(highlight=False)
+    try:
+        sources = sorted(load_backend_sources(), key=lambda source: source.backend.name)
+    except BackendError as e:
+        raise click.ClickException(str(e)) from e
+    rows = [
+        (source.backend.name, ", ".join(source.backend.languages), _registered_by(source.entry_point))
+        for source in sources
+    ]
+    name_width = max((len(name) for name, _, _ in rows), default=0)
+    languages_width = max((len(languages) for _, languages, _ in rows), default=0)
+    for name, languages, registered_by in rows:
+        console.out(f"{name:<{name_width}}  {languages:<{languages_width}}  {registered_by}")
+
+
+def _registered_by(entry_point: EntryPoint) -> str:
+    """The entry point's target, with the package and version that registered it when that's known."""
+    if entry_point.dist is None:
+        return entry_point.value
+    return f"{entry_point.value} ({entry_point.dist.name} {entry_point.dist.version})"
+
+
+def _validated(path: Path) -> tuple[Runbook, dict[str, Backend], Report]:
+    """Parse and validate a runbook, turning anything that stops it being read into a clean error."""
+    try:
+        parsed = parse(path)
+        backends = load_backends()
+    except (RunbookError, BackendError) as e:
+        raise click.ClickException(str(e)) from e
+    return parsed, backends, validate(parsed, backends)
+
+
+def _print_report(console: Console, path: Path, report: Report, *, summary_if_ok: bool) -> None:
+    """Print each finding as path:line: kind: message, then a count. A clean report prints only if asked."""
+    if not report.findings:
+        if summary_if_ok:
+            console.out(f"{path}: ok", style="green")
+        return
+    for finding in report.findings:
+        kind, style = ("warning", "yellow") if finding.warning else ("error", "red")
+        console.out(f"{path}:{finding.line}: {kind}: {finding.message}", style=style)
+    counts = [_count(len(report.errors), "error"), _count(len(report.warnings), "warning")]
+    console.out()
+    console.out(", ".join(count for count in counts if count), style="bold")
+
+
+def _count(number: int, noun: str) -> str:
+    return f"{number} {noun}{'' if number == 1 else 's'}" if number else ""
+
+
+@dataclass(frozen=True)
+class ClickPrompter:
+    """Prompts in the terminal, hiding what's typed for secrets."""
+
+    console: Console
+
+    def ask(self, input: Input, problem: str | None) -> str:
+        if problem is not None:
+            # console.out, because a pattern such as [a-z] would be read as markup.
+            self.console.out(problem, style="red")
+        return _prompt(f"{input.name} ({input.description})", hide_input=input.secret)
+
+    def confirm(self, question: str) -> str:
+        # An empty default lets enter through as "", which the runner reads as no.
+        return _prompt(question, default="", show_default=False, prompt_suffix=" ")
+
+
+def _prompt(text: str, **options: Any) -> str:
+    """click.prompt, but Ctrl-C raises KeyboardInterrupt so the runner can finish the run as interrupted.
+
+    Click turns both Ctrl-C and running out of input into Abort; only Ctrl-C is turned back.
+    """
+    try:
+        return click.prompt(text, **options)
+    except click.Abort as e:
+        if isinstance(e.__context__, KeyboardInterrupt):
+            raise KeyboardInterrupt from None
+        raise
+
+
+def _print_event(console: Console, event: runner.Event, *, dry: bool = False) -> None:
+    # console.out never reads markup or emoji codes and never wraps, so code and output appear as written.
+    match event:
+        case runner.Overview(title=title, steps=steps, run_blocks=run_blocks, inputs=inputs):
+            console.out(title, style="bold")
+            if inputs:
+                console.out("inputs: " + " ".join(f"{name}={value}" for name, value in inputs))
+            for number, step in enumerate(steps, start=1):
+                console.out(f"  {number}. {step.name}")
+            console.out(f"{run_blocks} run block{'' if run_blocks == 1 else 's'}")
+            if dry:
+                console.out("dry run: nothing will be run", style="yellow")
+        case runner.StepStarted(step=step):
+            console.out()
+            console.out(f"=== {step.name} ===", style="bold")
+        case runner.Docstring(text=text):
+            for line in text.splitlines():
+                console.out(line)
+        case runner.InputBlock(input=input) if dry:
+            console.out(f"input: {input.name} ({input.description}), asked here", style="yellow")
+        case runner.Confirm(text=text):
+            for line in text.splitlines():
+                console.out(line, style="yellow")
+        case runner.BlockStarted(code=code):
+            for line in runner.code_lines(code):
+                console.out(line, style="cyan")
+        case runner.BlockSkipped():
+            console.out("skipped (--step)", style="yellow")
+        case runner.OutputLine(text=text):
+            console.out(text)
+        case runner.BlockFinished(exit_code=exit_code, duration=duration):
+            console.out(
+                f"exit {exit_code} ({runner.format_duration(duration)})", style="green" if exit_code == 0 else "red"
+            )
+        case runner.RunFinished(status="ok", duration=duration):
+            console.out()
+            console.out(f"finished: ok ({runner.format_duration(duration)})", style="green")
+        case runner.RunFinished(status="dry"):
+            console.out()
+            console.out("finished: dry run, nothing was run", style="yellow")
+        case runner.RunFinished(status="aborted" | "interrupted" as status, step=step):
+            console.out()
+            where = "before starting" if step is None else f'at "{step.name}"'
+            console.out(f"finished: {status} {where}", style="red")
+        case runner.RunFinished(step=step, exit_code=exit_code):
+            assert step is not None  # always set when a run fails
+            console.out()
+            console.out(f'finished: failed at "{step.name}" (exit {exit_code})', style="red")

paulrun/inputs.py

@@ -0,0 +1,74 @@
+import re
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from typing import Protocol
+
+PLACEHOLDER = re.compile(r"<([A-Z][A-Z0-9_]*)>")
+
+MASK = "****"
+
+
+@dataclass(frozen=True)
+class Input:
+    name: str
+    description: str
+    pattern: str | None = None
+    secret: bool = False
+
+
+class Prompter(Protocol):
+    """Asks the person running the runbook for values and answers."""
+
+    def ask(self, input: Input, problem: str | None) -> str:
+        """Ask for an input's value, hiding what's typed if it's secret.
+
+        problem is None the first time, then says why the previous answer was refused.
+        """
+
+    def confirm(self, question: str) -> str:
+        """Ask a question, e.g. "Start? [y/N]", and return the answer as typed. Pressing enter gives ""."""
+
+
+class InputError(Exception):
+    """An input can't be collected, e.g. its value from the environment doesn't match its pattern."""
+
+
+def collect(
+    inputs: Iterable[Input], env: Mapping[str, str], prompter: Prompter, *, skip_secrets: bool = False
+) -> dict[str, str]:
+    """Get a value for each input, from env if it's set there, otherwise by asking until the answer matches.
+
+    With skip_secrets, secrets are left out entirely: not read from env, not asked for, and not in the result.
+    """
+    values = {}
+    for input in inputs:
+        if skip_secrets and input.secret:
+            continue
+        # An empty variable counts as unset, so a stray `VERSION=` can't release an empty version.
+        if value := env.get(input.name):
+            if not _matches(input, value):
+                raise InputError(f"{input.name} from the environment must match {input.pattern}")
+        else:
+            value = prompter.ask(input, None)
+            while not _matches(input, value):
+                value = prompter.ask(input, f"{input.name} must match {input.pattern}")
+        values[input.name] = value
+    return values
+
+
+def _matches(input: Input, value: str) -> bool:
+    return input.pattern is None or re.fullmatch(input.pattern, value) is not None
+
+
+def substitute(text: str, values: Mapping[str, str]) -> str:
+    """Replace <NAME> placeholders with their values. Unknown names are left alone."""
+    return PLACEHOLDER.sub(lambda m: values.get(m.group(1), m.group(0)), text)
+
+
+def mask(text: str, secrets: Iterable[str]) -> str:
+    """Replace every occurrence of every secret with ****."""
+    # Longest first, so a secret that contains another is masked whole rather than leaving its tail showing.
+    ordered = sorted({secret for secret in secrets if secret}, key=len, reverse=True)
+    if not ordered:
+        return text
+    return re.sub("|".join(re.escape(secret) for secret in ordered), MASK, text)