dirigent-cli 0.9.0__py3-none-any.whl

dirigent_cli/reaper.py

@@ -0,0 +1,115 @@
+"""Wiring the docker orphan reaper to an instance: what a run id means, and when a pass runs.
+
+The reaping itself is ``dirigent_blocks.reap``, which knows docker and nothing else. This
+module supplies the half it cannot have: the run lookup, which is a database read, and the
+worker chore that puts a pass on a cadence.
+"""
+
+import os
+import shutil
+import tempfile
+from collections.abc import Mapping
+from datetime import timedelta
+from pathlib import Path
+from uuid import UUID
+
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from dirigent_blocks import reap, subprocess
+from dirigent_blocks.docker import DAEMON_ENV
+from dirigent_client.schemas import TERMINAL_RUN_STATUSES
+from dirigent_core.config import Settings
+from dirigent_core.database import session_scope
+from dirigent_core.logging import get_logger
+from dirigent_core.models import Run
+from dirigent_core.worker import Chore
+
+#: The docker config directory is inherited too: a pass runs the host's own CLI, and on a host
+#: whose compose plugin is configured rather than installed system-wide, this is where it says so.
+REAP_ENV = ("DOCKER_CONFIG", *DAEMON_ENV)
+
+_logger = get_logger("worker")
+
+
+def lookup(sessions: async_sessionmaker[AsyncSession]) -> reap.RunLookup:
+    """Build the reaper's run lookup against this instance's database."""
+
+    async def look_up(run_id: UUID) -> reap.RunFact | None:
+        async with session_scope(sessions) as session:
+            run = await session.get(Run, run_id)
+            if run is None:
+                return None
+            return reap.RunFact(status=run.status.value, active=run.status not in TERMINAL_RUN_STATUSES)
+
+    return look_up
+
+
+def environment(root: Path) -> dict[str, str]:
+    """The environment a reaping docker invocation runs under: the worker's daemon, and its home.
+
+    A pass is worker maintenance rather than a step, so it keeps the worker's own ``HOME``
+    instead of a scratch one: that is where a docker install that is not system-wide puts the
+    compose plugin the teardown runs.
+    """
+    built = subprocess.environment(list(REAP_ENV), {}, root)
+    home = os.environ.get("HOME")
+    if home:
+        built["HOME"] = home
+    return built
+
+
+def reachable() -> bool:
+    """Whether this host has a docker CLI at all, which is the cheapest gate there is."""
+    return shutil.which("docker") is not None
+
+
+async def pass_once(
+    settings: Settings, sessions: async_sessionmaker[AsyncSession], *, dry_run: bool = False
+) -> list[reap.Reaped]:
+    """Run one reaping pass against the daemon this host's environment names."""
+    with tempfile.TemporaryDirectory(prefix="dirigent-reap-") as home:
+        root = Path(home)
+        return await reap.reap(
+            environment(root),
+            root,
+            lookup(sessions),
+            grace=settings.docker_reap_grace,
+            dry_run=dry_run,
+        )
+
+
+def record(reaped: reap.Reaped) -> Mapping[str, object]:
+    """The fields one reaped project contributes, whether a record or a log line carries them."""
+    fields: dict[str, object] = {
+        "project": reaped.project,
+        "run_id": str(reaped.run_id),
+        "run_status": reaped.run_status,
+        "torn_down": reaped.torn_down,
+    }
+    if reaped.detail:
+        fields["detail"] = reaped.detail
+    return fields
+
+
+def outcome(reaped: reap.Reaped, *, dry_run: bool = False) -> str:
+    """What happened to one project, in the words a record and a log line both carry."""
+    if dry_run:
+        return "would reap"
+    return "reaped" if reaped.torn_down else "could not reap"
+
+
+def chore(settings: Settings, sessions: async_sessionmaker[AsyncSession]) -> Chore | None:
+    """Build the worker's reaping chore, or none where this worker cannot reach a daemon.
+
+    A worker with no docker CLI on its PATH has no stacks to reap and no way to reap them, so
+    it runs no loop at all rather than one that fails every five minutes.
+    """
+    interval = settings.docker_reap_interval
+    if interval <= timedelta(0) or not reachable():
+        return None
+
+    async def run() -> None:
+        for reaped in await pass_once(settings, sessions):
+            _logger.info(outcome(reaped), kind="docker_reaped", **record(reaped))
+
+    return Chore(name="docker-reap", interval=interval, run=run)

dirigent_cli/scaffold.py

@@ -0,0 +1,63 @@
+"""Scaffolding for a new block pack: a package, one operator, a passing test."""
+
+import re
+from importlib.resources import files
+from pathlib import Path
+from typing import Final, Literal
+
+from pydantic import BaseModel
+
+#: What a pack may be called: a lowercase identifier fragment, because it becomes half a
+#: module name and an entry-point key.
+NAME_PATTERN: Final = re.compile(r"^[a-z][a-z0-9_]*$")
+
+
+class ScaffoldRecord(BaseModel):
+    """One file the scaffold wrote."""
+
+    kind: Literal["scaffold"] = "scaffold"
+    path: str
+
+
+class ScaffoldedRecord(BaseModel):
+    """The closing record: where the pack is and what to run next."""
+
+    kind: Literal["scaffolded"] = "scaffolded"
+    directory: str
+    next: list[str]
+
+
+class ScaffoldError(Exception):
+    """A scaffold that cannot proceed, with the reason as the message."""
+
+
+def scaffold_pack(parent: Path, name: str) -> list[Path]:
+    """Write a new pack under ``parent`` and answer with every file written.
+
+    The directory is refused when it already exists, so running the command twice cannot
+    half-overwrite somebody's edits.
+    """
+    if not NAME_PATTERN.match(name):
+        raise ScaffoldError(f"{name!r} is not a pack name: lowercase letters, digits and _, starting with a letter")
+    root = parent / f"dirigent-{name}"
+    if root.exists():
+        raise ScaffoldError(f"{root} already exists")
+    title = name.replace("_", " ").title().replace(" ", "")
+    module = root / "src" / f"dirigent_{name}"
+    written = {
+        root / "pyproject.toml": _template("pyproject.toml.tmpl", name=name, title=title),
+        root / "README.md": _template("README.md.tmpl", name=name, title=title),
+        module / "__init__.py": _template("__init__.py.tmpl", name=name, title=title),
+        module / "py.typed": "",
+        module / f"{name}.py": _template("operator.py.tmpl", name=name, title=title),
+        root / "tests" / "test_plugin.py": _template("test_plugin.py.tmpl", name=name, title=title),
+    }
+    for path, text in written.items():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text)
+    return list(written)
+
+
+def _template(filename: str, **values: str) -> str:
+    """Read one packaged template and fill its placeholders."""
+    return (files("dirigent_cli") / "templates" / "pack" / filename).read_text().format(**values)

dirigent_cli/schemas.py

@@ -0,0 +1,85 @@
+"""What the closing ``run`` record of a run's stream carries.
+
+A run's stream is written record by record through :mod:`dirigent_core.protocol`; these are
+the shapes of the summaries the last record holds. They are the contract a script filters on,
+and they are also what the end-of-run table and the failure diagnosis are rendered from, so a
+field the rendering needs belongs here rather than in a second query.
+"""
+
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from dirigent_common import JsonMap
+
+
+class StepSummary(BaseModel):
+    """What one settled attempt amounted to, as the closing event carries it."""
+
+    model_config = ConfigDict(frozen=True)
+
+    step: str
+    item: str | None = None
+    block: str
+    status: str
+    depends_on: list[str] = Field(default_factory=list[str])
+    """The steps this one waited for, which is why it ran when it did."""
+
+    warnings: int = 0
+    """How many warnings or errors this attempt logged, whatever it settled as."""
+
+    attempts: int = 1
+    """How many attempts this step took to settle."""
+
+    duration_ms: int | None = None
+    output: JsonMap | None = None
+    error: str | None = None
+    artifact_uri: str | None = None
+    artifact_bytes: int | None = None
+
+
+class FailureSummary(BaseModel):
+    """One failed attempt, with the diagnosis a deleted local database cannot be asked for."""
+
+    model_config = ConfigDict(frozen=True)
+
+    step: str
+    block: str
+    attempt: int
+    error_class: str | None = None
+    error: str | None = None
+    logs: list[str] = Field(default_factory=list[str])
+    input: JsonMap | None = None
+    """What the attempt was given, which is half of why it failed."""
+
+
+class BackfillWindow(BaseModel):
+    """One window a backfill enumerated, as the closing backfill record carries it."""
+
+    model_config = ConfigDict(frozen=True)
+
+    window_start: str
+    window_end: str
+    run_id: str | None = None
+    detail: str | None = None
+
+
+class RunFinished(BaseModel):
+    """The run reached a terminal status; the last line of a run's stream."""
+
+    model_config = ConfigDict(frozen=True)
+
+    run_id: UUID | None = None
+    pipeline: str
+    pipeline_version: int | None = None
+    status: str
+    triggered_by: str | None = None
+    duration_ms: int | None = None
+    items_total: int = 0
+    items_failed: int = 0
+    error: str | None = None
+    exit_code: int = 0
+    steps: list[StepSummary] = Field(default_factory=list[StepSummary])
+    failures: list[FailureSummary] = Field(default_factory=list[FailureSummary])
+    kept_at: str | None = None
+    """Where a ``--keep`` local run left its throwaway instance."""

dirigent_cli/sources.py

@@ -0,0 +1,76 @@
+"""Where a document comes from: a file, a URL, or standard input."""
+
+import sys
+from pathlib import Path
+from typing import Final
+
+import httpx2
+from pydantic import BaseModel, ConfigDict
+
+from dirigent_client.enums import ProvenanceSource
+
+STDIN: Final = "-"
+FETCH_TIMEOUT: Final = 30.0
+
+
+class SourceError(Exception):
+    """A document could not be read from where it was said to be."""
+
+
+class Document(BaseModel):
+    """One document's text, and where it came from."""
+
+    model_config = ConfigDict(frozen=True)
+
+    text: str
+    source: ProvenanceSource
+    ref: str
+
+    @property
+    def label(self) -> str:
+        """Render the origin for a plan line or an error message."""
+        return self.ref
+
+
+def is_url(reference: str) -> bool:
+    """Report whether a reference names a URL rather than a path."""
+    return reference.startswith(("http://", "https://"))
+
+
+def read_document(reference: str) -> Document:
+    """Read a document from a file, a URL, or standard input."""
+    if reference == STDIN:
+        text = sys.stdin.read()
+        if not text.strip():
+            raise SourceError("nothing arrived on standard input")
+        return Document(text=text, source=ProvenanceSource.API, ref="(stdin)")
+    if is_url(reference):
+        return Document(text=fetch(reference), source=ProvenanceSource.URL, ref=reference)
+    path = Path(reference)
+    if not path.is_file():
+        raise SourceError(f"{reference} is neither a file, a URL, nor '-' for standard input")
+    return Document(text=path.read_text(), source=ProvenanceSource.FILE, ref=str(path))
+
+
+def fetch(url: str) -> str:
+    """Fetch a document over HTTP."""
+    try:
+        response = httpx2.get(url, timeout=FETCH_TIMEOUT, follow_redirects=True)
+        response.raise_for_status()
+    except httpx2.HTTPError as error:
+        raise SourceError(f"{url} could not be fetched: {type(error).__name__}: {error}") from error
+    return response.text
+
+
+def read_path(path: Path) -> Document:
+    """Read a document from a path already known to be one, keeping its provenance."""
+    return Document(text=path.read_text(), source=ProvenanceSource.FILE, ref=str(path))
+
+
+def looks_like_a_document(reference: str) -> bool:
+    """Report whether an argument names a document rather than a pipeline.
+
+    Unambiguous only because a pipeline name is a DNS label: it can never contain ``/``,
+    ``.``, or a scheme.
+    """
+    return reference == STDIN or is_url(reference) or Path(reference).is_file()

dirigent_cli/stream.py

@@ -0,0 +1,180 @@
+"""The run's story, as protocol records encoded or rendered for this invocation.
+
+One renderer serves all of it: the events a run emits here, and the same events read back
+off a stored stream by ``dg format``. Verbosity decides which events are emitted; it
+never changes the shape of a line.
+"""
+
+import sys
+from collections.abc import Iterable, Mapping
+from datetime import UTC, datetime
+from typing import Any, Final, cast
+
+from rich.markup import escape
+from rich.style import Style
+
+from dirigent_core.protocol import Format, Record, make, render
+
+#: Supporting text is grey rather than rich's dim attribute, which many terminals render at
+#: too little contrast to read on a dark background.
+MUTED: Final = "grey58"
+
+#: Colours a step is tracked by, assigned in document order and reused for every line it
+#: writes. Colour is the fast path for the eye; the fixed column is what makes the stream
+#: readable where there is no colour at all.
+STEP_COLOURS: Final[tuple[str, ...]] = (
+    "cyan",
+    "magenta",
+    "green",
+    "yellow",
+    "blue",
+    "bright_cyan",
+    "bright_magenta",
+)
+
+#: How each role of a console line is coloured. A level that is routine is left unmarked:
+#: marking every line is the noise that marking exists to cut through.
+ROLES: Final[Mapping[str, str]] = {
+    "time": MUTED,
+    "kind": MUTED,
+    "key": "cyan",
+    "step.": "blue",
+    "level.info": "green",
+    "level.warning": "bold yellow",
+    "level.error": "bold red",
+    "level.critical": "bold red",
+    "level.debug": MUTED,
+}
+
+_step_colours: dict[str, str] = {}
+
+_scratch: str | None = None
+
+
+def track_steps(names: Iterable[str]) -> None:
+    """Give each step a colour, in the order the document wrote them.
+
+    Concurrent steps interleave, so a reader following one step down the page has only its
+    name to go on; a stable colour makes that one glance instead of one search.
+    """
+    _step_colours.clear()
+    for index, name in enumerate(names):
+        _step_colours[name] = STEP_COLOURS[index % len(STEP_COLOURS)]
+
+
+def use_scratch_prefix(prefix: str | None) -> None:
+    """Name the prefix this run's URIs share, so the console rendering can leave it out.
+
+    The prefix is stated once, on the run's own opening event, which is what makes the
+    shorter form a spelling rather than a truncation: it round-trips.
+    """
+    global _scratch  # noqa: PLW0603 - one process-wide run context, set once per invocation
+    _scratch = prefix.rstrip("/") if prefix else None
+
+
+def paint(role: str, text: str) -> str:
+    """Colour one part of a console line, escaping it so rich reads none of it as markup."""
+    style = ROLES.get(role)
+    if style is None and role.startswith("step."):
+        # A stored stream has no tracked colours: dg format never saw the document.
+        style = _step_colours.get(role.removeprefix("step.")) or ROLES.get("step.")
+    safe = escape(text)
+    return f"[{style}]{safe}[/]" if style else safe
+
+
+def ansi(role: str, text: str) -> str:
+    """Colour one part of a line for a stream that is written to rather than printed.
+
+    The logging handler writes to its stream directly, so rich markup would arrive as the
+    literal brackets that spell it. The roles and their colours are the ones the rendered
+    stream uses, resolved to escape sequences here.
+    """
+    style = _style_for(role)
+    if style is None or not _colouring():
+        return text
+    return Style.parse(style).render(text)
+
+
+def _style_for(role: str) -> str | None:
+    """Resolve one role to its style, tracked colours included."""
+    style = ROLES.get(role)
+    if style is None and role.startswith("step."):
+        style = _step_colours.get(role.removeprefix("step.")) or ROLES.get("step.")
+    return style
+
+
+def _colouring() -> bool:
+    """Report whether the diagnostic stream may be sent colour at all."""
+    from dirigent_cli.output import error_console
+
+    return not error_console.no_color and error_console.is_terminal
+
+
+def relative(value: str) -> str:
+    """Spell one URI relative to the prefix this run's URIs share, when it is under it."""
+    return str(_relative(value))
+
+
+def shorten(record: Record) -> Record:
+    """Spell this run's own URIs relative to the prefix they all share.
+
+    Rendering only, and only where the prefix was already stated: two URIs that differ must
+    still read differently, so nothing else about a value is touched.
+    """
+    if _scratch is None:
+        return record
+    return {name: _relative(value) for name, value in record.items()}
+
+
+def _relative(value: Any) -> Any:
+    """Drop the shared scratch prefix from a URI, leaving every other value alone."""
+    if isinstance(value, Mapping):
+        nested = cast("Mapping[str, Any]", value)
+        return {name: _relative(item) for name, item in nested.items()}
+    if isinstance(value, str) and _scratch is not None and value.startswith(f"{_scratch}/"):
+        return value[len(_scratch) + 1 :]
+    return value
+
+
+class Sink:
+    """Where a run's story goes: stdout, in one output, for the whole invocation."""
+
+    def __init__(self, output: Format = "console") -> None:
+        """Fix the output this invocation writes."""
+        self.output: Format = output
+
+    @property
+    def rendered(self) -> bool:
+        """Report whether this invocation is being read by a person."""
+        return self.output == "console"
+
+    def write(self, record: Record) -> None:
+        """Write one record and flush it, so a reader sees it as it happens.
+
+        A rendered invocation goes through the same formatter ``dg format`` uses, so a run
+        watched live and the same run read back off a file look the same.
+        """
+        if self.rendered:
+            _emit(_formatter().render(record))
+        else:
+            sys.stdout.write(render(record, self.output) + "\n")
+        sys.stdout.flush()
+
+    def event(self, kind: str, /, **kwargs: Any) -> None:
+        """Build one record and write it, stamped now when the event carries no moment."""
+        kwargs.setdefault("at", datetime.now(UTC))
+        self.write(make(kind, **kwargs))
+
+
+def _emit(item: Any) -> None:
+    """Print one rendered record, reaching the shared console late to avoid a cycle."""
+    from dirigent_cli.output import emit_rendered
+
+    emit_rendered(item)
+
+
+def _formatter() -> Any:
+    """Reach the console formatter late: it imports this module for the painting."""
+    from dirigent_cli.formatters import Console
+
+    return Console()