clankloop 0.0.0__py3-none-any.whl

clankloop/loopfile/v2/clankshed_module.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Clankshed module — expands ``clankshed``-type module tasks into bash tasks.
+
+Each clankshed task is expanded into four sub-tasks: work, repair, static-check,
+and review. The flow is: work → static-check → review → next; on failure,
+static-check or review retries from repair.
+"""
+from clankloop.loopfile.v2.module import Module, ModuleDependency
+
+from clankloop.loopfile.v2.loopfile import (
+    Loopfile,
+    BashTask,
+    ModuleTask,
+    Task,
+    NextTaskAction,
+    Action,
+    RetryFromAction,
+    make_converter,
+)
+
+from attrs import frozen
+
+
+@frozen
+class ClankshedSubTask:
+    clankshed_cmd: str
+    on_success: list[Action] | None = None
+    on_failure: list[Action] | None = None
+
+
+@frozen
+class CheckSubTask:
+    cmd: str
+    on_success: list[Action] | None = None
+    on_failure: list[Action] | None = None
+
+
+@frozen
+class ClankshedTaskExtra:
+    work_task: ClankshedSubTask
+    repair_task: ClankshedSubTask
+    review_task: ClankshedSubTask
+    static_check: CheckSubTask
+
+
+_converter = make_converter()
+
+
+class ClankshedModule(Module):
+    VERSION = "1.0.0"
+    DEPENDENCIES: tuple[ModuleDependency, ...] = ()
+    APPLY_BEFORE: tuple[str, ...] = ()
+    APPLY_AFTER: tuple[str, ...] = ()
+
+    def __init__(self, **module_args):
+        self.session = module_args.get("session", "default")
+        self.clankshed_file = module_args.get("clankshed_file", "clankshed.yaml")
+
+    def transform_loopfile(self, src: Loopfile) -> Loopfile:
+        new_tasks: list[Task] = []
+
+        for task in src.tasks:
+            if isinstance(task, BashTask):
+                new_tasks.append(task)
+            elif isinstance(task, ModuleTask) and task.type == "clankshed":
+                new_tasks.extend(self.transform_clankshed_task(task))
+
+        return Loopfile(
+            name=src.name,
+            constants=src.constants,
+            locals=src.locals,
+            tasks=new_tasks,
+            parameters=src.parameters,
+            modules=src.modules,
+        )
+
+    def transform_clankshed_task(self, task: ModuleTask) -> list[BashTask]:
+        """
+        Generate:
+        1. Initial task for starting task
+        2. Repair task
+        3. Static check task
+        4. Review task
+
+        on success: 1 -> 3 -> 4 -> next task in pipeline
+                    2 -> 3 -> 4 -> next task in pipeline
+
+        on retry: 3 -> 2, 4 -> 2
+        """
+        extra = _converter.structure(task.extra_fields, ClankshedTaskExtra)
+
+        work_name = f"{task.name}-work"
+        repair_name = f"{task.name}-repair"
+        static_check_name = f"{task.name}-static-check"
+        review_name = f"{task.name}-review"
+
+        return [
+            BashTask(
+                name=work_name,
+                cmds=f"clankshed run -f {self.clankshed_file} -- {extra.work_task.clankshed_cmd}",
+                constants=task.constants,
+                on_success=[NextTaskAction(goto=static_check_name)],
+            ),
+            BashTask(
+                name=repair_name,
+                cmds=f"clankshed run -f {self.clankshed_file} -- {extra.repair_task.clankshed_cmd}",
+                constants=task.constants,
+            ),
+            BashTask(
+                name=static_check_name,
+                cmds=extra.static_check.cmd,
+                constants=task.constants,
+                on_failure=[RetryFromAction(retry_from=repair_name, retries=2)],
+            ),
+            BashTask(
+                name=review_name,
+                cmds=f"clankshed run -f {self.clankshed_file} -- {extra.review_task.clankshed_cmd}",
+                constants=task.constants,
+                on_failure=[RetryFromAction(retry_from=repair_name, retries=2)],
+            ),
+        ]

clankloop/loopfile/v2/compiler.py

@@ -0,0 +1,331 @@
+#!/usr/bin/env python3
+"""Loopfile v2 compiler — transforms a v2 loopfile spec into a Pipeline.
+
+Applies registered modules to transform the loopfile, then compiles the
+resulting bash tasks into an :class:`~clankloop.core.graph.ExecutionGraph`
+with a structured environment (``pipeline.name``, ``pipeline.constants``,
+``pipeline.parameters``, ``pipeline.globals``).
+"""
+import clankloop.core.graph as graph
+import clankloop.loopfile.v2.loopfile as loopfile
+from clankloop.core.env import Object, Value, ClankTemplate
+from clankloop.loopfile.v2 import paths as v2_paths
+from clankloop.loopfile.v2.module_registry import ModuleRegistry, MissingModuleDependencyError
+from clankloop.loopfile.v2.toposort import CycleError, OrderedEdge, topological_sort
+from clankloop.runner import ParameterBinding, Pipeline
+from typing import Any, Sequence
+from itertools import takewhile
+import logging
+
+logger = logging.getLogger("clankloop")
+
+
+class CircularModuleOrderError(ValueError):
+    """Raised when merged module ordering constraints form a cycle."""
+
+
+def compile_pipeline(
+    data: dict[str, Any],
+    pipeline_name: str | None = None,
+    module_registry: ModuleRegistry | None = None,
+) -> Pipeline:
+    """Compile a v2 loopfile into a :class:`~clankloop.runner.Pipeline`.
+
+    Parses the raw YAML data, applies registered modules to transform the
+    loopfile, then compiles bash tasks into an execution graph.
+
+    Args:
+        data: Parsed YAML dictionary.
+        pipeline_name: Ignored — a v2 loopfile defines exactly one pipeline,
+            named by the loopfile's ``name`` field.
+        module_registry: Optional custom module registry. If ``None``, a
+            fresh registry with builtins is used.
+
+    Returns:
+        A compiled pipeline ready for execution.
+    """
+    loopfile_raw = loopfile.parse(data)
+
+    loopfile_spec = apply_modules(loopfile_raw, module_registry)
+
+    execs: dict[str, graph.Execution] = {
+        t.name: compile_bash_task(t) for t in loopfile_spec.tasks
+    }
+
+    exec_actions: dict[
+        str, Sequence[tuple[graph.Condition, Sequence[graph.Action]]]
+    ] = {
+        task.name: compile_task_actions(task, _next_task_name(loopfile_spec, i))
+        for i, task in enumerate(loopfile_spec.tasks)
+    }
+
+    root_env = Object(
+        {
+            "pipeline": Object(
+                {
+                    "name": Value(
+                        loopfile_spec.name, immutable=True, assigned=True
+                    ),
+                    "constants": Object(
+                        {
+                            k: Value(ClankTemplate(v), immutable=True, assigned=True)
+                            for k, v in loopfile_spec.constants.items()
+                        }
+                    ),
+                    "parameters": Object(
+                        {
+                            ps.name: Value(
+                                "", immutable=False, assigned=False
+                            )
+                            for ps in loopfile_spec.parameters
+                        }
+                    ),
+                    "globals": Object(
+                        {
+                            name: Value(
+                                "", immutable=False, assigned=False
+                            )
+                            for name in loopfile_spec.locals
+                        }
+                    ),
+                }
+            )
+        }
+    )
+
+    parameter_bindings = tuple(
+        ParameterBinding(
+            name=ps.name,
+            path=v2_paths.parameter(ps.name),
+            required=ps.required,
+            default=ps.default,
+        )
+        for ps in loopfile_spec.parameters
+    )
+
+    execution_graph = graph.ExecutionGraph(
+        loopfile_spec.name, execs, exec_actions, root_env
+    )
+    execution_graph.validate(
+        loopfile_spec.tasks[0].name,
+        parameter_paths=frozenset(str(b.path) for b in parameter_bindings),
+        check_data_dependencies=True,
+    )
+
+    return Pipeline(
+        loopfile_spec.name,
+        execution_graph,
+        loopfile_spec.tasks[0].name,  # start_task
+        parameter_bindings,
+    )
+
+
+def apply_modules(
+    loopfile_spec: loopfile.Loopfile, module_registry: ModuleRegistry | None
+) -> loopfile.Loopfile:
+    """Apply registered modules to a loopfile spec in resolved order.
+
+    Resolves the module application order (merging module-declared and
+    user-declared ordering constraints), validates the dependency closure,
+    then applies each module's transform in sequence.
+
+    Args:
+        loopfile_spec: The parsed loopfile spec.
+        module_registry: The module registry to use (falls back to builtins
+            if ``None``).
+
+    Returns:
+        The transformed loopfile spec (all module tasks resolved to bash
+        tasks).
+    """
+    if module_registry is None:
+        module_registry = ModuleRegistry.with_builtins()
+
+    module_order = resolve_modules(loopfile_spec.modules, module_registry)
+
+    spec = loopfile_spec
+    for module_spec in module_order:
+        module = module_registry.get(module_spec.name)(**module_spec.extra_fields)
+        spec = module.transform_loopfile(spec)
+    return spec
+
+
+def resolve_modules(
+    modules_spec: list[loopfile.ModuleSpec],
+    registry: ModuleRegistry,
+) -> list[loopfile.ModuleSpec]:
+    """Validate dependencies and topologically sort modules by merged ordering.
+
+    Three sources of ordering edges are merged into one graph:
+
+    1. Module-declared ``APPLY_BEFORE`` / ``APPLY_AFTER`` (wrapping semantics).
+    2. User-declared ``after`` on ``ModuleSpec`` (supplemental ordering).
+    3. Dependencies are *not* ordering edges — they're presence constraints
+       validated separately.
+
+    If the merged graph has a cycle, the error attributes each edge to its
+    source (module-declared or user-declared) so the conflict is traceable.
+
+    Args:
+        modules_spec: The user's module list from the loopfile.
+        registry: The module registry (source of module-declared facts).
+
+    Returns:
+        Modules in application order.
+
+    Raises:
+        ValueError: If modules are repeated.
+        MissingModuleDependencyError: If a module's dependency is not in the
+            user's module list.
+        CircularModuleOrderError: If the merged ordering constraints form a
+            cycle.
+    """
+    names = [m.name for m in modules_spec]
+    if len(set(names)) != len(names):
+        raise ValueError("Modules are repeated")
+
+    user_modules = {m.name: m for m in modules_spec}
+
+    _validate_dependencies(user_modules, registry)
+
+    edges = _collect_ordering_edges(user_modules, registry)
+    try:
+        ordered = topological_sort(names, edges)
+    except CycleError as exc:
+        raise CircularModuleOrderError(str(exc)) from exc
+    return [user_modules[n] for n in ordered]
+
+
+def _validate_dependencies(
+    user_modules: dict[str, loopfile.ModuleSpec], registry: ModuleRegistry
+) -> None:
+    """Check every module's declared dependencies are in the user's list."""
+    missing = list(
+        (name, dep)
+        for name in user_modules
+        for dep in registry.get(name).DEPENDENCIES
+        if dep.name not in user_modules
+    )
+    if missing:
+        descriptions = ", ".join(
+            f"{name!r} requires {dep.name!r} (version {dep.version})"
+            for name, dep in missing
+        )
+        raise MissingModuleDependencyError(
+            f"Missing module dependencies: {descriptions}"
+        )
+
+
+def _collect_ordering_edges(
+    user_modules: dict[str, loopfile.ModuleSpec], registry: ModuleRegistry
+) -> list[OrderedEdge]:
+    """Merge module-declared and user-declared ordering into one edge list.
+
+    An edge ``(before, after)`` means *before* must be applied before *after*.
+    """
+    apply_before = [
+        OrderedEdge(before=name, after=target, source=f"{name} APPLY_BEFORE {target}")
+        for name in user_modules
+        for target in registry.get(name).APPLY_BEFORE
+    ]
+    apply_after = [
+        OrderedEdge(before=target, after=name, source=f"{name} APPLY_AFTER {target}")
+        for name in user_modules
+        for target in registry.get(name).APPLY_AFTER
+    ]
+    user_declared = [
+        OrderedEdge(before=target, after=spec.name, source=f"user: {spec.name} after {target}")
+        for spec in user_modules.values()
+        for target in spec.after
+    ]
+    return apply_before + apply_after + user_declared
+
+
+def compile_bash_task(task: loopfile.Task) -> graph.BashExecution:
+    if not isinstance(task, loopfile.BashTask):
+        raise ValueError(f"Task {task.name!r} is not a BashTask")
+    if isinstance(task.cmds, str):
+        cmds_tpl = ClankTemplate(task.cmds)
+    else:
+        cmds_tpl = task.cmds
+    exports = dict(task.exports)
+    return graph.BashExecution(task.name, cmds_tpl, exports)
+
+
+def compile_task_actions(
+    task: loopfile.Task,
+    next_task: str | None,
+) -> list[tuple[graph.Condition, list[graph.Action]]]:
+    if not isinstance(task, loopfile.BashTask):
+        raise ValueError(f"Task {task.name!r} is not a BashTask")
+    branches: list[tuple[graph.Condition, list[graph.Action]]] = []
+
+    # 1. Compile OnSuccess branch
+    success_actions = compile_actions_list(task.on_success or [], is_success=True)
+
+    # If no flow action is present and next task exists, append default NextAction
+    has_flow_action = any(isinstance(act, graph.NextAction) for act in success_actions)
+    if not has_flow_action and next_task is not None:
+        # Check if setup or any set action terminated early via an Abort check
+        aborted = any(
+            isinstance(act, loopfile.AbortAction) for act in (task.on_success or [])
+        )
+        if not aborted:
+            success_actions.append(graph.NextAction(next_task=next_task))
+
+    if success_actions:
+        branches.append((graph.OnSuccessCondition(), success_actions))
+
+    # 2. Compile OnFailure/Retry branches
+    failure_actions = compile_actions_list(task.on_failure or [], is_success=False)
+
+    # Check for RetryFromAction setup
+    retry_actions = [
+        act
+        for act in (task.on_failure or [])
+        if isinstance(act, loopfile.RetryFromAction)
+    ]
+
+    if retry_actions:
+        # If retry condition is hit and we have retries remaining, loop back
+        retry = retry_actions[0]
+        branches.append(
+            (graph.OnRetryCondition(retries=retry.retries), [graph.NextAction(next_task=retry.retry_from)])
+        )
+
+    if failure_actions:
+        branches.append((graph.OnFailureCondition(), failure_actions))
+
+    return branches
+
+
+def compile_actions_list(
+    actions: list[loopfile.Action],
+    is_success: bool,
+) -> list[graph.Action]:
+    return [
+        _compile_action(action)
+        for action in takewhile(lambda a: not isinstance(a, loopfile.AbortAction), actions)
+        if not isinstance(action, loopfile.RetryFromAction)
+    ]
+
+
+def _compile_action(action: loopfile.Action) -> graph.Action:
+    if isinstance(action, loopfile.SetAction):
+        stdout = action.value == "stdout"
+        stderr = action.value == "stderr"
+        logger.debug("Creating SetAction %s", action.set)
+        return graph.SetAction(
+            var_name=v2_paths.glob(action.set),
+            value=None if stdout or stderr else action.value,
+            stdout=stdout,
+            stderr=stderr,
+        )
+    assert isinstance(action, loopfile.NextTaskAction)
+    return graph.NextAction(next_task=action.goto)
+
+
+def _next_task_name(loopfile_spec: loopfile.Loopfile, index: int) -> str | None:
+    if index + 1 >= len(loopfile_spec.tasks):
+        return None
+    return loopfile_spec.tasks[index + 1].name

clankloop/loopfile/v2/git_module.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Git module — injects git clone/commit/push tasks around the pipeline.
+
+Adds a ``git-checkout`` task (clones the repository, checks out branches) at
+the start and a ``git-push`` task (commits and pushes) at the end. Configured
+via ``work_repository``, ``src_branch``, ``dst_branch``, and ``commit_template``.
+"""
+from clankloop.loopfile.v2.module import Module, ModuleDependency
+
+from clankloop.loopfile.v2.loopfile import Loopfile, BashTask, NextTaskAction
+
+
+class GitModule(Module):
+    VERSION = "1.0.0"
+    DEPENDENCIES = (
+        ModuleDependency(name="workdir", version=">=1.0"),
+    )
+    APPLY_BEFORE = ("workdir",)
+    APPLY_AFTER: tuple[str, ...] = ()
+
+    def __init__(self, **module_args):
+        self.repo_url = module_args.get("work_repository")
+        self.src_branch = module_args.get("src_branch", "master")
+        self.dst_branch = module_args.get("dst_branch", "master")
+        self.commit_template = module_args.get("commit_template", "Clankloop commit")
+
+    def transform_loopfile(self, src: Loopfile) -> Loopfile:
+        new_constants = {
+            **src.constants,
+            "git_work_repository": self.repo_url,
+            "git_repo_name": self._git_repo_name(),
+            "git_repo_dir": "${pipeline.globals.workdir}/${pipeline.constants.git_repo_name}",
+            "git_src_branch": self.src_branch,
+            "git_dst_branch": self.dst_branch,
+            "git_commit_template": self.commit_template,
+        }
+
+        new_tasks = [
+            BashTask(
+                name="git-checkout",
+                cmds="""
+cd ${pipeline.globals.workdir}
+git clone ${pipeline.constants.git_work_repository}
+cd ${pipeline.constants.git_repo_dir}
+git checkout ${pipeline.constants.git_src_branch}
+git checkout -b ${pipeline.constants.git_dst_branch} || true
+                """,
+                on_failure=[NextTaskAction(goto="workdir-remove")],
+            ),
+            *src.tasks,
+            BashTask(
+                name="git-push",
+                cmds="""
+git commit -m "${pipeline.constants.git_commit_template}"
+git push --set-upstream origin ${pipeline.constants.git_dst_branch}
+                """,
+            ),
+        ]
+        return Loopfile(
+            name=src.name,
+            constants=new_constants,
+            locals=src.locals,
+            tasks=new_tasks,
+            parameters=src.parameters,
+            modules=src.modules,
+        )
+
+    def _git_repo_name(self) -> str:
+        if self.repo_url.endswith(".git"):
+            return self.repo_url.split("/")[-1][:-4]
+        else:
+            return self.repo_url.split("/")[-1]

clankloop/loopfile/v2/loopfile.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Loopfile v2 schema — dataclasses and YAML parsing for v2 loopfiles.
+
+A v2 loopfile defines a single pipeline with bash tasks and module tasks.
+Module tasks are placeholders that are transformed into bash tasks by
+registered modules (e.g. workdir, git, clankshed) before compilation.
+"""
+
+
+from typing import Any
+
+import attrs
+from attrs import frozen, fields
+import cattrs
+from cattrs import Converter
+
+from clankloop.core.env import EnvPath
+
+
+@frozen
+class ParameterSpec:
+    name: str
+    required: bool = False
+    default: str | None = None
+
+
+@frozen
+class SetAction:
+    set: str
+    value: str
+
+
+@frozen
+class RetryFromAction:
+    retry_from: str
+    retries: int
+
+
+@frozen
+class NextTaskAction:
+    goto: str
+
+
+@frozen
+class AbortAction:
+    pass
+
+
+Action = SetAction | RetryFromAction | NextTaskAction | AbortAction
+
+
+@frozen
+class ModuleTask:
+    name: str
+    type: str
+    constants: dict[str, Any] = attrs.Factory(dict)
+    extra_fields: dict[str, Any] = attrs.Factory(dict)
+
+
+@frozen
+class BashTask:
+    name: str
+    cmds: str
+    constants: dict[str, Any] = attrs.Factory(dict)
+    exports: dict[str, EnvPath] = attrs.Factory(dict)
+    on_success: list[Action] | None = None
+    on_failure: list[Action] | None = None
+
+
+Task = BashTask | ModuleTask
+
+
+@frozen
+class ModuleSpec:
+    name: str
+    after: list[str] = attrs.Factory(list)
+    extra_fields: dict[str, Any] = attrs.Factory(dict)
+
+
+@frozen
+class Loopfile:
+    name: str
+    constants: dict[str, Any] = attrs.Factory(dict)
+    locals: list[str] = attrs.Factory(list)
+    tasks: list[Task] = attrs.Factory(list)
+    parameters: list[ParameterSpec] = attrs.Factory(list)
+    modules: list[ModuleSpec] = attrs.Factory(list)
+
+
+def _structure_action(data: Any, _type: Any, converter: Converter) -> Action:
+    if not isinstance(data, dict):
+        raise ValueError(f"Expected dict for Action, got {type(data).__name__}")
+    if "abort" in data:
+        return AbortAction()
+    if "retry_from" in data:
+        return converter.structure(data, RetryFromAction)
+    if "set" in data:
+        return converter.structure(data, SetAction)
+    if "goto" in data:
+        return converter.structure(data, NextTaskAction)
+
+    raise ValueError(f"Unrecognised action keys: {set(data.keys())}")
+
+
+def make_converter() -> Converter:
+    converter = cattrs.Converter()
+
+    def structure_frozen_with_extra(data: dict, cls):
+        known_keys = {f.name for f in fields(cls)}
+        known_data = {}
+        extra_data = {}
+        for key, value in data.items():
+            if key in known_keys:
+                known_data[key] = value
+            else:
+                extra_data[key] = value
+        if "extra_fields" in known_keys:
+            known_data["extra_fields"] = extra_data
+        return converter.structure_attrs_fromdict(known_data, cls)
+
+    converter.register_structure_hook_func(
+        lambda t: t is EnvPath,
+        lambda data, t: EnvPath.parse(data) if isinstance(data, str) else data,
+    )
+    converter.register_structure_hook_func(
+        lambda t: t is Action,
+        lambda data, t: _structure_action(data, t, converter),
+    )
+    converter.register_structure_hook_func(
+        lambda t: t is Task,
+        lambda data, t: (
+            converter.structure(data, ModuleTask)
+            if "type" in data
+            else converter.structure(data, BashTask)
+        ),
+    )
+    converter.register_structure_hook(ModuleSpec, structure_frozen_with_extra)
+    converter.register_structure_hook(ModuleTask, structure_frozen_with_extra)
+    return converter
+
+
+def parse(data: dict[str, Any]) -> Loopfile:
+    """Parse raw YAML data into a v2 :class:`Loopfile` spec.
+
+    Args:
+        data: Parsed YAML dictionary.
+
+    Returns:
+        A structured :class:`Loopfile` instance.
+    """
+    converter = make_converter()
+    return converter.structure(data, Loopfile)