clankloop 0.0.0__py3-none-any.whl

clankloop/logger.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""Structured logging — JSON, text, scoped buffers, and optional OTel export.
+
+This module configures the standard Python logging system to route logs to
+stderr (as structured text or JSON), to scoped memory buffers for capture
+within a concurrency scope, and optionally to an OpenTelemetry exporter.
+Console logging defaults to stderr so pipeline stdout is never polluted.
+"""
+
+
+import contextvars
+import json
+import logging
+import sys
+from contextlib import contextmanager
+from typing import Any, Iterator
+
+logger = logging.getLogger("clankloop")
+
+# OTel is an optional dependency. The exporter lives in a separate package
+# (opentelemetry-exporter-otlp-proto-http) from the SDK, so both must import.
+try:
+    from opentelemetry._logs import set_logger_provider
+    from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+    from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+    from opentelemetry.exporter.otlp.proto.http.log_exporter import OTLPLogExporter
+
+    HAS_OTEL = True
+except ImportError:
+    HAS_OTEL = False
+
+
+# Derive the standard LogRecord attributes once at class-load time
+# by instantiating a dummy LogRecord and extracting its dict keys.
+_STANDARD_LOGRECORD_KEYS: set[str] = set(
+    logging.LogRecord("dummy", 0, "", 0, "", (), None).__dict__.keys()
+) | {"asctime", "message"}
+
+
+# ContextVar to hold an active list of LogRecords for the current concurrency scope
+_active_buffer: contextvars.ContextVar[list[logging.LogRecord] | None] = (
+    contextvars.ContextVar("_active_buffer", default=None)
+)
+
+
+class _ClankloopHandler(logging.Handler):
+    """Marker base for handlers :func:`setup_logging` owns and manages.
+
+    Idempotent re-configuration checks ``isinstance`` against this base rather
+    than monkey-patching attributes onto stdlib handler instances.
+    """
+
+
+class ScopedBufferHandler(_ClankloopHandler):
+    """Intercepts records and appends them to a context-local memory buffer
+    if one is currently active.
+    """
+
+    def emit(self, record: logging.LogRecord) -> None:
+        buf = _active_buffer.get()
+        if buf is not None:
+            buf.append(record)
+
+
+class _ConsoleHandler(_ClankloopHandler, logging.StreamHandler):
+    """Console stream handler owned by setup_logging."""
+
+
+class JsonFormatter(logging.Formatter):
+    """Formats log records as structured single-line JSON."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        log_data: dict[str, Any] = {
+            "timestamp": self.formatTime(record, self.datefmt),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+        if record.exc_info:
+            log_data["exception"] = self.formatException(record.exc_info)
+        if record.stack_info:
+            log_data["stack"] = self.formatStack(record.stack_info)
+
+        # Extract extra attributes for structured logging
+        for key, value in record.__dict__.items():
+            if key not in _STANDARD_LOGRECORD_KEYS:
+                log_data[key] = value
+
+        return json.dumps(log_data)
+
+
+class StructuredFormatter(logging.Formatter):
+    """Formats log records as structured multi-line text with key=value pairs."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        base = f"[{self.formatTime(record, self.datefmt)}] {record.levelname} in {record.name}: {record.getMessage()}"
+        extra_keys = record.__dict__.keys() - _STANDARD_LOGRECORD_KEYS
+
+        for key in sorted(extra_keys):
+            value = record.__dict__[key]
+            base += f" {key}={value!r}"
+        return base
+
+
+@contextmanager
+def scoped_buffer() -> Iterator[list[logging.LogRecord]]:
+    """Context manager to intercept and capture all logging records emitted
+    within the current thread/task scope in a mutable list.
+    """
+    buf: list[logging.LogRecord] = []
+    token = _active_buffer.set(buf)
+    try:
+        yield buf
+    finally:
+        _active_buffer.reset(token)
+
+
+def setup_logging(
+    level: int = logging.INFO,
+    console: bool = True,
+    json_format: bool = False,
+    otel: bool = False,
+    stream=None,
+) -> None:
+    """Configure the standard Python logging system.
+
+    Sets up a :class:`ScopedBufferHandler` (always), a console handler
+    (optional, text or JSON), and an OTel logging handler (optional).
+
+    Args:
+        level: Root logger level (defaults to ``INFO``).
+        console: Whether to add a console/stream handler.
+        json_format: If True, console output uses :class:`JsonFormatter`;
+            otherwise :class:`StructuredFormatter`.
+        otel: If True, attempt to export logs via OpenTelemetry.
+        stream: Optional stream for the console handler (defaults to stderr).
+    """
+    root = logging.getLogger()
+    root.setLevel(level)
+
+    logging.getLogger("clankloop").setLevel(level)
+
+    for h in list(root.handlers):
+        if isinstance(h, _ClankloopHandler):
+            root.removeHandler(h)
+
+    buffer_handler = ScopedBufferHandler()
+    root.addHandler(buffer_handler)
+
+    if console:
+        console_handler = _ConsoleHandler(
+            stream if stream is not None else sys.stderr
+        )
+        if json_format:
+            console_handler.setFormatter(JsonFormatter())
+        else:
+            console_handler.setFormatter(
+                StructuredFormatter(
+                    fmt="[{asctime}] {levelname} in {name}: {message}",
+                    datefmt="%Y-%m-%d %H:%M:%S",
+                    style="{",
+                )
+            )
+        root.addHandler(console_handler)
+
+    if otel:
+        if HAS_OTEL:
+            provider = LoggerProvider()
+            processor = BatchLogRecordProcessor(OTLPLogExporter())
+            provider.add_log_record_processor(processor)
+            set_logger_provider(provider)
+
+            otel_handler = LoggingHandler(logger_provider=provider)
+            root.addHandler(otel_handler)
+        else:
+            logger.warning(
+                "OTel logging requested but OpenTelemetry SDK or exporter is "
+                "not installed. Falling back without OTel agent export."
+            )

clankloop/loopfile/__init__.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Loopfile parsing and compilation.
+
+The top-level :func:`compile_pipeline` function dispatches to the correct
+version-specific compiler based on the ``version`` key in the raw YAML data.
+Supported versions are listed in :data:`~clankloop.loopfile.versions.SUPPORTED_VERSIONS`.
+"""
+
+from clankloop.loopfile.versions import SUPPORTED_VERSIONS, CURRENT_VERSION, normalize_version
+from clankloop.loopfile.v1.compiler import compile_pipeline as compile_v1
+from clankloop.loopfile.v2.compiler import compile_pipeline as compile_v2
+
+from clankloop.runner import Pipeline
+from typing import Any
+
+
+def compile_pipeline(data: dict[str, Any], pipeline_name: str | None = None) -> Pipeline:
+    """Compile raw loopfile data into a :class:`~clankloop.runner.Pipeline`.
+
+    Args:
+        data: Parsed YAML dictionary (as returned by ``yaml.safe_load``).
+        pipeline_name: Name of the pipeline to compile. Required for v1
+            loopfiles (which can contain multiple pipelines); ignored by v2
+            (one loopfile = one pipeline).
+
+    Returns:
+        A pipeline object ready for registration with a
+        :class:`~clankloop.runner.Runner`.
+
+    Raises:
+        ValueError: If the loopfile version is not supported.
+        RuntimeError: If no compiler matched the version (should not happen).
+    """
+    version = data.get("version")
+
+    if version is None:
+        version = CURRENT_VERSION
+    else:
+        version = normalize_version(version)
+
+    if version not in SUPPORTED_VERSIONS:
+        raise ValueError(
+            f"Unsupported Loopfile version {version!r}. "
+            f"Supported versions: {SUPPORTED_VERSIONS}"
+        )
+
+    if version == "1":
+        return compile_v1(data, pipeline_name)
+
+    if version == "2":
+        return compile_v2(data, pipeline_name)
+
+    raise RuntimeError(f"Unexpectedly did not find loopfile compiler {version!r}")

clankloop/loopfile/v1/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python3
+"""
+Deprecated, no active development. Use :mod:`clankloop.loopfile.v2` instead.
+Loopfile schema v1 — multi-pipeline definitions with flat task lists."""

clankloop/loopfile/v1/compiler.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+"""Loopfile v1 compiler — transforms a v1 loopfile spec into a Pipeline.
+
+Compiles pipelines, tasks, and actions into an :class:`~clankloop.core.graph.ExecutionGraph`
+with a shared environment of parameters, constants, and locals.
+"""
+
+
+from typing import Sequence, Any
+from itertools import takewhile
+
+import clankloop.core.graph as graph
+import clankloop.loopfile.v1.loopfile as loopfile
+from clankloop.runner import ParameterBinding, Pipeline
+from clankloop.core.env import Object, Value, EnvPath
+from clankloop.loopfile.v1 import paths as v1_paths
+
+
+def compile_pipeline(data: dict[str, Any], pipeline_name: str | None) -> Pipeline:
+    """Compile a v1 loopfile into a :class:`~clankloop.runner.Pipeline`.
+
+    Parses the raw YAML data, finds the named pipeline, validates it, and
+    builds an execution graph with the pipeline's parameters, constants,
+    locals, and task-local constants wired into the environment.
+
+    Args:
+        data: Parsed YAML dictionary.
+        pipeline_name: Name of the pipeline to compile. Required — a v1
+            loopfile can contain multiple pipelines.
+
+    Returns:
+        A compiled pipeline ready for execution.
+
+    Raises:
+        ValueError: If *pipeline_name* is not provided.
+        RuntimeError: If the pipeline is not found in the loopfile.
+        ValueError: If the pipeline has no tasks, duplicate task names,
+            or invalid action references.
+    """
+    if pipeline_name is None:
+        raise ValueError(
+            "pipeline_name is required for v1 loopfiles (which can contain "
+            "multiple pipelines)"
+        )
+    loopfile_data = loopfile.parse(data)
+
+    pipeline = next(
+        (p for p in loopfile_data.pipelines if p.name == pipeline_name), None
+    )
+    if pipeline is None:
+        available = ", ".join(repr(p.name) for p in loopfile_data.pipelines)
+        raise RuntimeError(
+            f"Pipeline {pipeline_name!r} not found in Loopfile. "
+            f"Available pipelines: {available}"
+        )
+
+    if not pipeline.tasks:
+        raise ValueError(f"Pipeline {pipeline.name!r} has no tasks")
+
+    _validate_pipeline(pipeline)
+
+    execs: dict[str, graph.Execution] = {}
+
+    pipeline_exports: dict[str, EnvPath] = {}
+
+    pipeline_parameters = {
+        param.name: Value("", immutable=True, assigned=False)
+        for param in loopfile_data.parameters
+    }
+
+    pipeline_locals = {}
+    pipeline_constants = {}
+
+    for pipeline_local_name in pipeline.locals:
+        pipeline_exports[pipeline_local_name] = v1_paths.glob(pipeline_local_name)
+        pipeline_locals[pipeline_local_name] = Value(
+            "", immutable=False, assigned=False
+        )
+
+    for pc_name, pc_value in pipeline.constants.items():
+        pipeline_exports[pc_name] = v1_paths.constant(pc_name)
+        pipeline_constants[pc_name] = Value(pc_value, immutable=True, assigned=True)
+
+    for param in loopfile_data.parameters:
+        pipeline_exports[param.name] = v1_paths.parameter(param.name)
+
+    execution_constants: dict[str, Value] = {}
+
+    for task in pipeline.tasks:
+        # Task local constants need to exported with a local name
+        task_exports: dict[str, EnvPath] = {}
+
+        # v1 has no first-class task-local namespace, so task constants live
+        # in the flat env under a synthetic `task-<task>-<name>` key (see the
+        # `pipeline.current_action` follow-on). The builder owns that shape.
+        for constant_name, contant_value in task.constants.items():
+            contant_key = f"task-{task.name}-{constant_name}"
+            task_exports[contant_key] = v1_paths.task_constant(task.name, constant_name)
+            execution_constants[contant_key] = Value(
+                value=contant_value, immutable=True, assigned=True
+            )
+
+        execs[task.name] = graph.BashExecution(
+            name=task.name, cmds=task.cmds, exports={**pipeline_exports, **task_exports}
+        )
+
+    exec_actions: dict[
+        str, Sequence[tuple[graph.Condition, Sequence[graph.Action]]]
+    ] = {
+        task.name: _compile_task_actions(
+            task, _next_task_name(pipeline, i), pipeline.retries
+        )
+        for i, task in enumerate(pipeline.tasks)
+    }
+
+    parameter_bindings = tuple(
+        ParameterBinding(
+            name=p.name,
+            path=v1_paths.parameter(p.name),
+            required=p.required,
+        )
+        for p in loopfile_data.parameters
+    )
+
+    execution_graph = graph.ExecutionGraph(
+        name=pipeline.name,
+        execs=execs,
+        exec_actions=exec_actions,
+        env=Object(
+            {
+                **pipeline_parameters,
+                **pipeline_constants,
+                **pipeline_locals,
+                **execution_constants,
+            }
+        ),
+    )
+    execution_graph.validate(
+        pipeline.tasks[0].name,
+        parameter_paths=frozenset(str(b.path) for b in parameter_bindings),
+    )
+
+    return Pipeline(
+        name=pipeline.name,
+        graph=execution_graph,
+        entry_task=pipeline.tasks[0].name,
+        parameters=parameter_bindings,
+    )
+
+
+def _validate_pipeline(pipeline: loopfile.PipelineSpec) -> None:
+    task_names = [task.name for task in pipeline.tasks]
+    unique_task_names = set(task_names)
+
+    if len(task_names) != len(unique_task_names):
+        raise ValueError(f"Pipeline {pipeline.name!r} contains duplicate task names")
+
+    local_names = set(pipeline.locals)
+    for task in pipeline.tasks:
+        for action in (task.on_success or []) + (task.on_failure or []):
+            if isinstance(action, loopfile.RetryFromAction):
+                if action.retry_from not in unique_task_names:
+                    raise ValueError(
+                        f"Task {task.name!r} retries from unknown task "
+                        f"{action.retry_from!r}"
+                    )
+            elif isinstance(action, loopfile.SetAction):
+                if action.set not in local_names:
+                    raise ValueError(
+                        f"Task {task.name!r} sets unknown local {action.set!r}"
+                    )
+
+
+def _next_task_name(pipeline: loopfile.PipelineSpec, index: int) -> str | None:
+    if index + 1 >= len(pipeline.tasks):
+        return None
+    return pipeline.tasks[index + 1].name
+
+
+def _compile_task_actions(
+    task: loopfile.Task,
+    next_task: str | None,
+    pipeline_retries: int = 0,
+) -> list[tuple[graph.Condition, list[graph.Action]]]:
+    branches: list[tuple[graph.Condition, list[graph.Action]]] = []
+
+    # 1. Compile OnSuccess branch
+    success_actions = _compile_actions_list(task.on_success or [], is_success=True)
+
+    # If no flow action is present and next task exists, append default NextAction
+    has_flow_action = any(isinstance(act, graph.NextAction) for act in success_actions)
+    if not has_flow_action and next_task is not None:
+        # Check if setup or any set action terminated early via an Abort check
+        aborted = any(
+            isinstance(act, loopfile.AbortAction) for act in (task.on_success or [])
+        )
+        if not aborted:
+            success_actions.append(graph.NextAction(next_task=next_task))
+
+    if success_actions:
+        branches.append((graph.OnSuccessCondition(), success_actions))
+
+    # 2. Compile OnFailure/Retry branches
+    failure_actions = _compile_actions_list(task.on_failure or [], is_success=False)
+
+    # Check for RetryFromAction setup
+    retry_targets = [
+        act.retry_from
+        for act in (task.on_failure or [])
+        if isinstance(act, loopfile.RetryFromAction)
+    ]
+
+    if retry_targets:
+        # If retry condition is hit and we have retries remaining, loop back
+        branches.append(
+            (graph.OnRetryCondition(retries=pipeline_retries), [graph.NextAction(next_task=retry_targets[0])])
+        )
+
+    if failure_actions:
+        branches.append((graph.OnFailureCondition(), failure_actions))
+
+    return branches
+
+
+def _compile_actions_list(
+    actions: list[loopfile.Action],
+    is_success: bool,
+) -> list[graph.Action]:
+    return [
+        _compile_action(action)
+        for action in takewhile(lambda a: not isinstance(a, loopfile.AbortAction), actions)
+        if not isinstance(action, loopfile.RetryFromAction)
+    ]
+
+
+def _compile_action(action: loopfile.Action) -> graph.Action:
+    if isinstance(action, loopfile.SetAction):
+        stdout = action.value == "stdout"
+        stderr = action.value == "stderr"
+        return graph.SetAction(
+            var_name=v1_paths.glob(action.set),
+            value=None if stdout or stderr else action.value,
+            stdout=stdout,
+            stderr=stderr,
+        )
+    assert isinstance(action, loopfile.NextTaskAction)
+    return graph.NextAction(next_task=action.goto)

clankloop/loopfile/v1/loopfile.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Loopfile v1 schema — dataclasses and YAML parsing for v1 loopfiles.
+
+A v1 loopfile defines parameters, constants, and multiple pipelines, each with
+flat task lists. Tasks have bash commands, task-local constants, and
+``on_success`` / ``on_failure`` action lists.
+"""
+
+
+from typing import Any
+
+import attrs
+from attrs import frozen
+import cattrs
+from cattrs import Converter
+
+
+@frozen
+class ParameterSpec:
+    name: str
+    required: bool = False
+
+
+@frozen
+class SetAction:
+    set: str
+    value: str
+
+
+@frozen
+class RetryFromAction:
+    retry_from: str
+
+
+@frozen
+class NextTaskAction:
+    goto: str
+
+
+@frozen
+class AbortAction:
+    pass
+
+
+Action = SetAction | RetryFromAction | NextTaskAction | AbortAction
+
+
+@frozen
+class Task:
+    name: str
+    cmds: str
+    constants: dict[str, Any] = attrs.Factory(dict)
+    on_success: list[Action] | None = None
+    on_failure: list[Action] | None = None
+
+
+@frozen
+class PipelineSpec:
+    name: str
+    retries: int = 0
+    constants: dict[str, Any] = attrs.Factory(dict)
+    locals: list[str] = attrs.Factory(list)
+    tasks: list[Task] = attrs.Factory(list)
+
+
+@frozen
+class Loopfile:
+    parameters: list[ParameterSpec] = attrs.Factory(list)
+    constants: dict[str, Any] = attrs.Factory(dict)
+    pipelines: list[PipelineSpec] = attrs.Factory(list)
+
+
+def _structure_action(data: Any, _type: Any, converter: Converter) -> Action:
+    if not isinstance(data, dict):
+        raise ValueError(f"Expected dict for Action, got {type(data).__name__}")
+    if "abort" in data:
+        return AbortAction()
+    if "retry_from" in data:
+        return converter.structure(data, RetryFromAction)
+    if "set" in data:
+        return converter.structure(data, SetAction)
+    if "goto" in data:
+        return converter.structure(data, NextTaskAction)
+
+    raise ValueError(f"Unrecognised action keys: {set(data.keys())}")
+
+
+def make_converter() -> Converter:
+    converter = cattrs.Converter()
+    converter.register_structure_hook_func(
+        lambda t: t is Action,
+        lambda data, t: _structure_action(data, t, converter),
+    )
+    return converter
+
+
+def parse(data: dict[str, Any]) -> Loopfile:
+    """Parse raw YAML data into a v1 :class:`Loopfile` spec.
+
+    Args:
+        data: Parsed YAML dictionary.
+
+    Returns:
+        A structured :class:`Loopfile` instance.
+    """
+    converter = make_converter()
+    return converter.structure(data, Loopfile)

clankloop/loopfile/v1/paths.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Env-path builders for the v1 loopfile layout.
+
+v1 uses a *flat* environment: parameters, constants, and locals (globals)
+all live as single-segment paths at the root of the env. The builders here
+are the single source of truth for that layout — callers supply the leaf
+name, the builder supplies the (trivial, flat) resolution.
+
+These builders are the place to review the v1 path schema. Future layout
+features (a =pipeline.current_action= alias, array/stack segments) extend
+this module.
+"""
+
+
+from clankloop.core.env import EnvPath
+
+
+def pipeline_name() -> EnvPath:
+    """The pipeline identifier path (flat in v1: =pipeline=)."""
+    return EnvPath.of("pipeline")
+
+
+def constant(name: str) -> EnvPath:
+    """A declared constant's path (flat in v1: =<name>=)."""
+    return EnvPath.of(name)
+
+
+def parameter(name: str) -> EnvPath:
+    """A declared parameter's path (flat in v1: =<name>=)."""
+    return EnvPath.of(name)
+
+
+def glob(name: str) -> EnvPath:
+    """A pipeline global (local) path (flat in v1: =<name>=)."""
+    return EnvPath.of(name)
+
+
+def task_constant(task_name: str, name: str) -> EnvPath:
+    """A task-local constant's path.
+
+    v1 has no first-class task-local namespace, so task constants live in the
+    flat env under a =task-<task>-<name>= synthetic key. This is a known
+    placeholder pending the =pipeline.current_action= alias follow-on; it is
+    kept here rather than inlined at the compiler so the layout is reviewable
+    in one place.
+    """
+    return EnvPath.of(f"task-{task_name}-{name}")

clankloop/loopfile/v2/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+"""Loopfile schema v2 — module system with transforms.
+
+v2 introduces a single-pipeline loopfile with a *module* system. Modules
+(:mod:`~clankloop.loopfile.v2.module`) transform the loopfile before compilation,
+injecting setup/teardown tasks (e.g. working directories, git clones).
+"""