cuprum 0.1.0__py3-none-any.whl

cuprum/__init__.py

@@ -0,0 +1,99 @@
+"""cuprum package.
+
+Provides a typed programme catalogue system for managing curated, allowlisted
+executables. Re-exports core types and the default catalogue for convenience.
+
+Example:
+>>> from cuprum import DEFAULT_CATALOGUE, ECHO
+>>> entry = DEFAULT_CATALOGUE.lookup(ECHO)
+>>> entry.project_name
+'core-ops'
+
+"""
+
+from __future__ import annotations
+
+from cuprum.catalogue import (
+    CORE_OPS_PROJECT,
+    DEFAULT_CATALOGUE,
+    DEFAULT_PROJECTS,
+    DOC_TOOL,
+    DOCUMENTATION_PROJECT,
+    ECHO,
+    LS,
+    ProgramCatalogue,
+    ProgramEntry,
+    ProjectSettings,
+    UnknownProgramError,
+)
+from cuprum.context import (
+    AfterHook,
+    AllowRegistration,
+    BeforeHook,
+    CuprumContext,
+    ExecHook,
+    ForbiddenProgramError,
+    HookRegistration,
+    after,
+    allow,
+    before,
+    current_context,
+    get_context,
+    observe,
+    scoped,
+)
+from cuprum.events import ExecEvent
+from cuprum.logging_hooks import LoggingHookRegistration, logging_hook
+from cuprum.program import Program
+from cuprum.sh import (
+    CommandResult,
+    ExecutionContext,
+    Pipeline,
+    PipelineResult,
+    SafeCmd,
+    SafeCmdBuilder,
+)
+
+from . import sh
+
+PACKAGE_NAME = "cuprum"
+
+__all__ = [
+    "CORE_OPS_PROJECT",
+    "DEFAULT_CATALOGUE",
+    "DEFAULT_PROJECTS",
+    "DOCUMENTATION_PROJECT",
+    "DOC_TOOL",
+    "ECHO",
+    "LS",
+    "PACKAGE_NAME",
+    "AfterHook",
+    "AllowRegistration",
+    "BeforeHook",
+    "CommandResult",
+    "CuprumContext",
+    "ExecEvent",
+    "ExecHook",
+    "ExecutionContext",
+    "ForbiddenProgramError",
+    "HookRegistration",
+    "LoggingHookRegistration",
+    "Pipeline",
+    "PipelineResult",
+    "Program",
+    "ProgramCatalogue",
+    "ProgramEntry",
+    "ProjectSettings",
+    "SafeCmd",
+    "SafeCmdBuilder",
+    "UnknownProgramError",
+    "after",
+    "allow",
+    "before",
+    "current_context",
+    "get_context",
+    "logging_hook",
+    "observe",
+    "scoped",
+    "sh",
+]

cuprum/_observability.py

@@ -0,0 +1,77 @@
+"""Internal helpers for structured execution event emission."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import types
+import typing as typ
+
+if typ.TYPE_CHECKING:
+    import collections.abc as cabc
+
+    from cuprum.events import ExecEvent, ExecHook
+
+
+def _freeze_str_mapping(
+    mapping: cabc.Mapping[str, str] | None,
+) -> cabc.Mapping[str, str] | None:
+    if mapping is None:
+        return None
+    return types.MappingProxyType(dict(mapping))
+
+
+def _merge_tags(*tags: cabc.Mapping[str, object] | None) -> cabc.Mapping[str, object]:
+    merged: dict[str, object] = {}
+    for mapping in tags:
+        if not mapping:
+            continue
+        merged.update(mapping)
+    return types.MappingProxyType(merged)
+
+
+def _emit_exec_event(
+    hooks: tuple[ExecHook, ...],
+    event: ExecEvent,
+    *,
+    pending_tasks: list[asyncio.Task[None]],
+) -> None:
+    """Invoke observe hooks and schedule async hooks as background tasks."""
+    for hook in hooks:
+        result = hook(event)
+        if inspect.isawaitable(result):
+            pending_tasks.append(asyncio.create_task(_await_awaitable(result)))
+
+
+async def _await_awaitable(awaitable: cabc.Awaitable[None]) -> None:
+    await awaitable
+
+
+async def _wait_for_exec_hook_tasks(pending_tasks: list[asyncio.Task[None]]) -> None:
+    """Await background observe-hook tasks and surface the first failure.
+
+    Observe hooks may return awaitables; those awaitables are scheduled as tasks
+    by ``_emit_exec_event`` and added to ``pending_tasks``. This helper awaits
+    all pending tasks and re-raises the first ``BaseException`` encountered.
+
+    Notes
+    -----
+    When multiple hooks fail, only the first exception is raised; subsequent
+    exceptions are not surfaced and may be masked by the first failure.
+
+    """
+    if not pending_tasks:
+        return
+    results = await asyncio.gather(*pending_tasks, return_exceptions=True)
+    pending_tasks.clear()
+    for result in results:
+        if isinstance(result, BaseException):
+            raise result
+
+
+__all__ = [
+    "_emit_exec_event",
+    "_freeze_str_mapping",
+    "_merge_tags",
+    "_wait_for_exec_hook_tasks",
+]

cuprum/_pipeline_internals.py

@@ -0,0 +1,278 @@
+"""Internal pipeline execution coordination and fail-fast semantics."""
+
+from __future__ import annotations
+
+import dataclasses as dc
+import sys
+import time
+import typing as typ
+from pathlib import Path
+
+from cuprum._observability import (
+    _emit_exec_event,
+    _freeze_str_mapping,
+    _merge_tags,
+    _wait_for_exec_hook_tasks,
+)
+from cuprum._pipeline_spawn import _spawn_pipeline_processes
+from cuprum._pipeline_streams import (
+    _cancel_stream_tasks,
+    _create_pipe_tasks,
+    _gather_optional_text_tasks,
+    _PipelineRunConfig,
+    _prepare_pipeline_config,
+)
+from cuprum._pipeline_wait import _PipelineWaitResult, _wait_for_pipeline
+from cuprum.context import current_context
+from cuprum.events import ExecEvent
+
+if typ.TYPE_CHECKING:
+    import asyncio
+    import types
+
+    from cuprum.context import AfterHook, BeforeHook
+    from cuprum.events import ExecHook
+    from cuprum.sh import CommandResult, ExecutionContext, PipelineResult, SafeCmd
+
+_MIN_PIPELINE_STAGES = 2
+
+
+def _sh_module() -> types.ModuleType:
+    module = sys.modules.get("cuprum.sh")
+    if module is None:
+        msg = "cuprum.sh must be imported before running pipelines"
+        raise RuntimeError(msg)
+    return module
+
+
+@dc.dataclass(frozen=True, slots=True)
+class _ExecutionHooks:
+    before_hooks: tuple[BeforeHook, ...]
+    after_hooks: tuple[AfterHook, ...]
+    observe_hooks: tuple[ExecHook, ...]
+
+
+def _run_before_hooks(cmd: SafeCmd) -> _ExecutionHooks:
+    """Collect hooks for a command after enforcing the current allowlist."""
+    ctx = current_context()
+    ctx.check_allowed(cmd.program)
+    return _ExecutionHooks(
+        before_hooks=ctx.before_hooks,
+        after_hooks=ctx.after_hooks,
+        observe_hooks=ctx.observe_hooks,
+    )
+
+
+@dc.dataclass(frozen=True, slots=True)
+class _StageObservation:
+    cmd: SafeCmd
+    hooks: _ExecutionHooks
+    tags: typ.Mapping[str, object]
+    cwd: Path | None
+    env_overlay: typ.Mapping[str, str] | None
+    pending_tasks: list[asyncio.Task[None]]
+
+    def emit(
+        self,
+        phase: typ.Literal["plan", "start", "stdout", "stderr", "exit"],
+        details: _EventDetails,
+    ) -> None:
+        if not self.hooks.observe_hooks:
+            return
+        _emit_exec_event(
+            self.hooks.observe_hooks,
+            ExecEvent(
+                phase=phase,
+                program=self.cmd.program,
+                argv=self.cmd.argv_with_program,
+                cwd=self.cwd,
+                env=self.env_overlay,
+                pid=details.pid,
+                timestamp=time.time(),
+                line=details.line,
+                exit_code=details.exit_code,
+                duration_s=details.duration_s,
+                tags=self.tags,
+            ),
+            pending_tasks=self.pending_tasks,
+        )
+
+
+@dc.dataclass(frozen=True, slots=True)
+class _EventDetails:
+    pid: int | None
+    line: str | None = None
+    exit_code: int | None = None
+    duration_s: float | None = None
+
+
+@dc.dataclass(frozen=True, slots=True)
+class _PipelineStageResultInputs:
+    wait_result: _PipelineWaitResult
+    stderr_by_stage: tuple[str | None, ...]
+    final_stdout: str | None
+
+
+def _build_pipeline_observations(
+    parts: tuple[SafeCmd, ...],
+    config: _PipelineRunConfig,
+    *,
+    pending_tasks: list[asyncio.Task[None]],
+) -> tuple[_StageObservation, ...]:
+    hooks_by_stage = tuple(_run_before_hooks(cmd) for cmd in parts)
+    cwd = None if config.ctx.cwd is None else Path(config.ctx.cwd)
+    env_overlay = _freeze_str_mapping(config.ctx.env)
+    return tuple(
+        _StageObservation(
+            cmd=cmd,
+            hooks=hooks,
+            tags=_merge_tags(
+                {
+                    "project": cmd.project.name,
+                    "capture": config.capture,
+                    "echo": config.echo,
+                    "pipeline_stage_index": idx,
+                    "pipeline_stages": len(parts),
+                },
+                config.ctx.tags,
+            ),
+            cwd=cwd,
+            env_overlay=env_overlay,
+            pending_tasks=pending_tasks,
+        )
+        for idx, (cmd, hooks) in enumerate(zip(parts, hooks_by_stage, strict=True))
+    )
+
+
+def _emit_plan_events_and_run_before_hooks(
+    observations: tuple[_StageObservation, ...],
+) -> None:
+    for obs in observations:
+        obs.emit("plan", _EventDetails(pid=None))
+        for hook in obs.hooks.before_hooks:
+            hook(obs.cmd)
+
+
+def _build_pipeline_stage_results(
+    parts: tuple[SafeCmd, ...],
+    observations: tuple[_StageObservation, ...],
+    *,
+    processes: list[asyncio.subprocess.Process],
+    inputs: _PipelineStageResultInputs,
+) -> list[CommandResult]:
+    sh = _sh_module()
+    stage_results: list[CommandResult] = []
+    for idx, obs in enumerate(observations):
+        process = processes[idx]
+        ended_at = inputs.wait_result.ended_at[idx]
+        duration_s = (
+            None
+            if ended_at is None
+            else max(0.0, ended_at - inputs.wait_result.started_at[idx])
+        )
+        obs.emit(
+            "exit",
+            _EventDetails(
+                pid=process.pid,
+                exit_code=inputs.wait_result.exit_codes[idx],
+                duration_s=duration_s,
+            ),
+        )
+        stage_results.append(
+            sh.CommandResult(
+                program=obs.cmd.program,
+                argv=obs.cmd.argv,
+                exit_code=inputs.wait_result.exit_codes[idx],
+                pid=process.pid if process.pid is not None else -1,
+                stdout=inputs.final_stdout if idx == len(parts) - 1 else None,
+                stderr=inputs.stderr_by_stage[idx],
+            ),
+        )
+    return stage_results
+
+
+async def _finalize_pipeline_execution(
+    parts: tuple[SafeCmd, ...],
+    observations: tuple[_StageObservation, ...],
+    stage_results: list[CommandResult],
+    pending_tasks: list[asyncio.Task[None]],
+) -> None:
+    hooks_by_stage = tuple(obs.hooks for obs in observations)
+    _run_pipeline_after_hooks(parts, hooks_by_stage, stage_results)
+    await _wait_for_exec_hook_tasks(pending_tasks)
+
+
+async def _run_pipeline(
+    parts: tuple[SafeCmd, ...],
+    *,
+    capture: bool,
+    echo: bool,
+    context: ExecutionContext | None,
+) -> PipelineResult:
+    """Execute a pipeline and return a structured result."""
+    config = _prepare_pipeline_config(capture=capture, echo=echo, context=context)
+    pending_tasks: list[asyncio.Task[None]] = []
+    observations = _build_pipeline_observations(
+        parts,
+        config,
+        pending_tasks=pending_tasks,
+    )
+    try:
+        _emit_plan_events_and_run_before_hooks(observations)
+        (
+            processes,
+            stderr_tasks,
+            stdout_task,
+            started_at,
+        ) = await _spawn_pipeline_processes(
+            parts,
+            config,
+            observations=observations,
+        )
+    except BaseException:
+        await _wait_for_exec_hook_tasks(pending_tasks)
+        raise
+    try:
+        wait_result = await _wait_for_pipeline(
+            processes,
+            pipe_tasks=_create_pipe_tasks(processes),
+            cancel_grace=config.ctx.cancel_grace,
+            started_at=started_at,
+        )
+        inputs = _PipelineStageResultInputs(
+            wait_result=wait_result,
+            stderr_by_stage=await _gather_optional_text_tasks(stderr_tasks),
+            final_stdout=None if stdout_task is None else await stdout_task,
+        )
+    except BaseException:
+        await _cancel_stream_tasks(stderr_tasks, stdout_task)
+        await _wait_for_exec_hook_tasks(pending_tasks)
+        raise
+    stage_results = _build_pipeline_stage_results(
+        parts,
+        observations,
+        processes=processes,
+        inputs=inputs,
+    )
+    await _finalize_pipeline_execution(
+        parts,
+        observations,
+        stage_results,
+        pending_tasks,
+    )
+
+    return _sh_module().PipelineResult(
+        stages=tuple(stage_results),
+        failure_index=inputs.wait_result.failure_index,
+    )
+
+
+def _run_pipeline_after_hooks(
+    parts: tuple[SafeCmd, ...],
+    hooks_by_stage: tuple[_ExecutionHooks, ...],
+    results: list[CommandResult],
+) -> None:
+    """Run registered after hooks for each pipeline stage."""
+    for cmd, hooks, result in zip(parts, hooks_by_stage, results, strict=True):
+        for hook in hooks.after_hooks:
+            hook(cmd, result)

cuprum/_pipeline_spawn.py

@@ -0,0 +1,20 @@
+"""Pipeline subprocess spawning and stream wiring.
+
+This module is a thin wrapper around the underlying spawn and cleanup helpers.
+The implementations live in ``cuprum._process_lifecycle`` to keep pipeline
+orchestration cohesive and avoid import cycles.
+"""
+
+from __future__ import annotations
+
+from cuprum._process_lifecycle import (
+    _build_spawn_observations,
+    _cleanup_spawned_processes,
+    _spawn_pipeline_processes,
+)
+
+__all__ = [
+    "_build_spawn_observations",
+    "_cleanup_spawned_processes",
+    "_spawn_pipeline_processes",
+]

cuprum/_pipeline_streams.py

@@ -0,0 +1,229 @@
+"""Stream coordination for pipeline execution."""
+
+from __future__ import annotations
+
+import asyncio
+import dataclasses as dc
+import sys
+import typing as typ
+
+from cuprum._streams import _consume_stream, _pump_stream, _StreamConfig
+
+if typ.TYPE_CHECKING:
+    from cuprum._pipeline_internals import _StageObservation
+    from cuprum.sh import ExecutionContext
+
+
+@dc.dataclass(frozen=True, slots=True)
+class _PipelineRunConfig:
+    ctx: ExecutionContext
+    capture: bool
+    echo: bool
+    stdout_sink: typ.IO[str]
+    stderr_sink: typ.IO[str]
+
+    @property
+    def capture_or_echo(self) -> bool:
+        return self.capture or self.echo
+
+    @property
+    def stream_config(self) -> _StreamConfig:
+        return _StreamConfig(
+            capture_output=self.capture,
+            echo_output=self.echo,
+            sink=self.stdout_sink,
+            encoding=self.ctx.encoding,
+            errors=self.ctx.errors,
+        )
+
+
+def _prepare_pipeline_config(
+    *,
+    capture: bool,
+    echo: bool,
+    context: ExecutionContext | None,
+) -> _PipelineRunConfig:
+    """Normalise runtime options for pipeline execution."""
+    from cuprum._pipeline_internals import _sh_module
+
+    sh = _sh_module()
+    ctx = context or sh.ExecutionContext()
+    stdout_sink = ctx.stdout_sink if ctx.stdout_sink is not None else sys.stdout
+    stderr_sink = ctx.stderr_sink if ctx.stderr_sink is not None else sys.stderr
+    return _PipelineRunConfig(
+        ctx=ctx,
+        capture=capture,
+        echo=echo,
+        stdout_sink=stdout_sink,
+        stderr_sink=stderr_sink,
+    )
+
+
+@dc.dataclass(frozen=True, slots=True)
+class _StageStreamConfig:
+    """Stream file descriptor configuration for a pipeline stage."""
+
+    stdin: int
+    stdout: int
+    stderr: int
+
+
+def _get_stage_stream_fds(
+    idx: int,
+    last_idx: int,
+    *,
+    capture_or_echo: bool,
+) -> _StageStreamConfig:
+    """Determine stream file descriptors for a pipeline stage.
+
+    First stage reads from DEVNULL; intermediate stages use pipes for stdin.
+    stdout is piped for intermediate stages or when capturing. stderr is piped
+    only when capturing or echoing.
+    """
+    stdin = asyncio.subprocess.DEVNULL if idx == 0 else asyncio.subprocess.PIPE
+    stdout = (
+        asyncio.subprocess.PIPE
+        if idx != last_idx or capture_or_echo
+        else asyncio.subprocess.DEVNULL
+    )
+    stderr = asyncio.subprocess.PIPE if capture_or_echo else asyncio.subprocess.DEVNULL
+    return _StageStreamConfig(stdin=stdin, stdout=stdout, stderr=stderr)
+
+
+def _create_stage_capture_tasks(
+    process: asyncio.subprocess.Process,
+    config: _PipelineRunConfig,
+    *,
+    is_last_stage: bool,
+    observation: _StageObservation,
+) -> tuple[asyncio.Task[str | None] | None, asyncio.Task[str | None] | None]:
+    """Create stderr and stdout capture tasks for a pipeline stage.
+
+    Returns (stderr_task, stdout_task). stderr is captured for all stages when
+    capture_or_echo is enabled. stdout is only captured for the final stage.
+    """
+    stderr_task: asyncio.Task[str | None] | None = None
+    stdout_task: asyncio.Task[str | None] | None = None
+
+    if not config.capture_or_echo:
+        return stderr_task, stdout_task
+
+    stderr_on_line: typ.Callable[[str], None] | None = None
+    if observation.hooks.observe_hooks:
+
+        def stderr_on_line(line: str) -> None:
+            from cuprum._pipeline_internals import _EventDetails
+
+            observation.emit(
+                "stderr",
+                _EventDetails(pid=process.pid, line=line),
+            )
+
+    stderr_task = asyncio.create_task(
+        _consume_stream(
+            process.stderr,
+            dc.replace(config.stream_config, sink=config.stderr_sink),
+            on_line=stderr_on_line,
+        ),
+    )
+
+    if not is_last_stage:
+        return stderr_task, stdout_task
+
+    stdout_on_line: typ.Callable[[str], None] | None = None
+    if observation.hooks.observe_hooks:
+
+        def stdout_on_line(line: str) -> None:
+            from cuprum._pipeline_internals import _EventDetails
+
+            observation.emit(
+                "stdout",
+                _EventDetails(pid=process.pid, line=line),
+            )
+
+    stdout_task = asyncio.create_task(
+        _consume_stream(
+            process.stdout,
+            config.stream_config,
+            on_line=stdout_on_line,
+        ),
+    )
+
+    return stderr_task, stdout_task
+
+
+def _create_pipe_tasks(
+    processes: list[asyncio.subprocess.Process],
+) -> list[asyncio.Task[None]]:
+    """Create streaming tasks between adjacent pipeline stages."""
+    return [
+        asyncio.create_task(
+            _pump_stream(
+                processes[idx].stdout,
+                processes[idx + 1].stdin,
+            ),
+        )
+        for idx in range(len(processes) - 1)
+    ]
+
+
+def _flatten_stream_tasks(
+    stderr_tasks: list[asyncio.Task[str | None] | None],
+    stdout_task: asyncio.Task[str | None] | None,
+) -> list[asyncio.Task[str | None]]:
+    """Collect all running stream consumer tasks for cancellation cleanup."""
+    tasks = [task for task in stderr_tasks if task is not None]
+    if stdout_task is not None:
+        tasks.append(stdout_task)
+    return tasks
+
+
+async def _cancel_stream_tasks(
+    stderr_tasks: list[asyncio.Task[str | None] | None],
+    stdout_task: asyncio.Task[str | None] | None,
+) -> None:
+    """Cancel stream consumer tasks and await their completion."""
+    tasks = _flatten_stream_tasks(stderr_tasks, stdout_task)
+    for task in tasks:
+        task.cancel()
+    await asyncio.gather(*tasks, return_exceptions=True)
+
+
+async def _gather_optional_text_tasks(
+    tasks: list[asyncio.Task[str | None] | None],
+) -> tuple[str | None, ...]:
+    """Await optional capture tasks, returning a tuple aligned with inputs."""
+    return tuple(
+        await asyncio.gather(
+            *(
+                task if task is not None else asyncio.sleep(0, result=None)
+                for task in tasks
+            ),
+        ),
+    )
+
+
+async def _collect_pipe_results(
+    pipe_tasks: list[asyncio.Task[None]],
+) -> list[object]:
+    """Collect pipe task results, capturing exceptions rather than raising them.
+
+    Uses return_exceptions=True to gather all results including any exceptions
+    that occurred during pipe streaming between pipeline stages.
+    """
+    return list(await asyncio.gather(*pipe_tasks, return_exceptions=True))
+
+
+def _surface_unexpected_pipe_failures(pipe_results: list[object]) -> None:
+    """Raise non-BrokenPipe exceptions from pipe results.
+
+    BrokenPipeError and ConnectionResetError are expected when downstream
+    processes terminate early (e.g., head) and should not fail the pipeline.
+    Other exceptions indicate genuine failures and must be surfaced.
+    """
+    for result in pipe_results:
+        if isinstance(result, Exception) and not isinstance(
+            result,
+            (BrokenPipeError, ConnectionResetError),
+        ):
+            raise result