pypeline-runner 1.6.0__tar.gz → 1.7.0__tar.gz

{pypeline_runner-1.6.0 → pypeline_runner-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pypeline-runner
-Version: 1.6.0
+Version: 1.7.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 License: MIT
 Author: cuinixam

{pypeline_runner-1.6.0 → pypeline_runner-1.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pypeline-runner"
-version = "1.6.0"
+version = "1.7.0"
 description = "Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines)."
 authors = ["cuinixam <me@cuinixam.com>"]
 license = "MIT"

pypeline_runner-1.7.0/src/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.7.0"

{pypeline_runner-1.6.0 → pypeline_runner-1.7.0}/src/pypeline/domain/artifacts.py

@@ -10,13 +10,10 @@ CONFIG_FILENAME = "pypeline.yaml"
 class ProjectArtifactsLocator:
     """Provides paths to project artifacts."""
 
-    def __init__(
-        self,
-        project_root_dir: Path,
-    ) -> None:
+    def __init__(self, project_root_dir: Path, config_file: Optional[str] = None) -> None:
         self.project_root_dir = project_root_dir
         self.build_dir = project_root_dir / "build"
-        self.config_file = project_root_dir / CONFIG_FILENAME
+        self.config_file = project_root_dir.joinpath(config_file if config_file else CONFIG_FILENAME)
         self.external_dependencies_dir = self.build_dir / "external"
         scripts_dir = "Scripts" if sys.platform.startswith("win32") else "bin"
         self.venv_scripts_dir = self.project_root_dir.joinpath(".venv").joinpath(scripts_dir)

pypeline_runner-1.7.0/src/pypeline/domain/pipeline.py

@@ -0,0 +1,181 @@
+import importlib
+from abc import abstractmethod
+from dataclasses import dataclass
+from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
+from typing import (
+    Any,
+    Dict,
+    Generic,
+    Iterator,
+    List,
+    Optional,
+    OrderedDict,
+    Protocol,
+    Tuple,
+    Type,
+    TypeAlias,
+    TypeVar,
+    Union,
+)
+
+from mashumaro import DataClassDictMixin
+from py_app_dev.core.exceptions import UserNotificationException
+from py_app_dev.core.runnable import Runnable
+
+from .execution_context import ExecutionContext
+
+
+@dataclass
+class PipelineStepConfig(DataClassDictMixin):
+    #: Step name or class name if file is not specified
+    step: str
+    #: Path to file with step class
+    file: Optional[str] = None
+    #: Python module with step class
+    module: Optional[str] = None
+    #: Step class name
+    class_name: Optional[str] = None
+    #: Command to run. For simple steps that don't need a class. Example: run: [echo, 'Hello World!']
+    run: Optional[Union[str, List[str]]] = None
+    #: Step description
+    description: Optional[str] = None
+    #: Step timeout in seconds
+    timeout_sec: Optional[int] = None
+    #: Custom step configuration
+    config: Optional[Dict[str, Any]] = None
+
+
+PipelineConfig: TypeAlias = Union[List[PipelineStepConfig], OrderedDict[str, List[PipelineStepConfig]]]
+
+TPipelineStep = TypeVar("TPipelineStep", covariant=True)
+
+
+@dataclass
+class PipelineStepReference(Generic[TPipelineStep]):
+    """Once a Step is found, keep the Step class reference to be able to instantiate it later."""
+
+    group_name: Optional[str]
+    _class: Type[TPipelineStep]
+    config: Optional[Dict[str, Any]] = None
+
+    @property
+    def name(self) -> str:
+        return self._class.__name__
+
+
+class PipelineConfigIterator:
+    """
+    Iterates over the pipeline configuration, yielding group name and steps configuration.
+
+    This class abstracts the iteration logic for PipelineConfig, which can be:
+    - A list of steps (group name is None)
+    - An OrderedDict with group names as keys and lists of steps as values.
+
+    The iterator yields tuples of (group_name, steps).
+    """
+
+    def __init__(self, pipeline_config: PipelineConfig) -> None:
+        self._items = pipeline_config.items() if isinstance(pipeline_config, OrderedDict) else [(None, pipeline_config)]
+
+    def __iter__(self) -> Iterator[Tuple[Optional[str], List[PipelineStepConfig]]]:
+        """Return an iterator."""
+        yield from self._items
+
+
+class StepClassFactory(Generic[TPipelineStep], Protocol):
+    def create_step_class(self, step_config: PipelineStepConfig, project_root_dir: Path) -> Type[TPipelineStep]: ...
+
+
+class PipelineLoader(Generic[TPipelineStep]):
+    def __init__(self, pipeline_config: PipelineConfig, project_root_dir: Path, step_class_factory: Optional[StepClassFactory[TPipelineStep]] = None) -> None:
+        self.pipeline_config = pipeline_config
+        self.project_root_dir = project_root_dir
+        self.step_class_factory = step_class_factory
+
+    def load_steps_references(self) -> List[PipelineStepReference[TPipelineStep]]:
+        result = []
+        for group_name, steps_config in PipelineConfigIterator(self.pipeline_config):
+            result.extend(self._load_steps(group_name, steps_config, self.project_root_dir, self.step_class_factory))
+        return result
+
+    @staticmethod
+    def _load_steps(
+        group_name: Optional[str], steps_config: List[PipelineStepConfig], project_root_dir: Path, step_class_factory: Optional[StepClassFactory[TPipelineStep]] = None
+    ) -> List[PipelineStepReference[TPipelineStep]]:
+        result = []
+        for step_config in steps_config:
+            step_class_name = step_config.class_name or step_config.step
+            if step_config.module:
+                step_class = PipelineLoader[TPipelineStep]._load_module_step(step_config.module, step_class_name)
+            elif step_config.file:
+                step_class = PipelineLoader[TPipelineStep]._load_user_step(project_root_dir.joinpath(step_config.file), step_class_name)
+            else:
+                if step_class_factory:
+                    step_class = step_class_factory.create_step_class(step_config, project_root_dir)
+                else:
+                    raise UserNotificationException(
+                        f"Step '{step_class_name}' has no 'module' nor 'file' defined nor a custom step class factory was provided. Please check your pipeline configuration."
+                    )
+            result.append(PipelineStepReference(group_name, step_class, step_config.config))
+        return result
+
+    @staticmethod
+    def _load_user_step(python_file: Path, step_class_name: str) -> Type[TPipelineStep]:
+        # Create a module specification from the file path
+        spec = spec_from_file_location(f"user__{python_file.stem}", python_file)
+        if spec and spec.loader:
+            step_module = module_from_spec(spec)
+            # Import the module
+            spec.loader.exec_module(step_module)
+            try:
+                step_class = getattr(step_module, step_class_name)
+            except AttributeError:
+                raise UserNotificationException(f"Could not load class '{step_class_name}' from file '{python_file}'. Please check your pipeline configuration.") from None
+            return step_class
+        raise UserNotificationException(f"Could not load file '{python_file}'. Please check the file for any errors.")
+
+    @staticmethod
+    def _load_module_step(module_name: str, step_class_name: str) -> Type[TPipelineStep]:
+        try:
+            module = importlib.import_module(module_name)
+            step_class = getattr(module, step_class_name)
+        except ImportError:
+            raise UserNotificationException(f"Could not load module '{module_name}'. Please check your pipeline configuration.") from None
+        except AttributeError:
+            raise UserNotificationException(f"Could not load class '{step_class_name}' from module '{module_name}'. Please check your pipeline configuration.") from None
+        return step_class
+
+
+TExecutionContext = TypeVar("TExecutionContext", bound=ExecutionContext)
+
+
+class PipelineStep(Generic[TExecutionContext], Runnable):
+    """One can create subclasses of PipelineStep that specify the type of ExecutionContext they require."""
+
+    def __init__(self, execution_context: TExecutionContext, group_name: Optional[str], config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(self.get_needs_dependency_management())
+        self.execution_context = execution_context
+        self.group_name = group_name
+        self.config = config
+        self.project_root_dir = self.execution_context.project_root_dir
+
+    @property
+    def output_dir(self) -> Path:
+        output_dir = self.execution_context.create_artifacts_locator().build_dir
+        if self.group_name:
+            output_dir = output_dir / self.group_name
+        return output_dir
+
+    @abstractmethod
+    def update_execution_context(self) -> None:
+        """
+        Even if the step does not need to run ( because it is not outdated ), it can still update the execution context.
+
+        A typical use case is for steps installing software that need to provide the install directories in the execution context even if all tools are already installed.
+        """
+        pass
+
+    def get_needs_dependency_management(self) -> bool:
+        """If false, the step executor will not check for outdated dependencies. This is useful for steps consisting of command lines which shall always run."""
+        return True

{pypeline_runner-1.6.0 → pypeline_runner-1.7.0}/src/pypeline/domain/project_slurper.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Optional
 
 from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.logging import logger
@@ -8,9 +9,9 @@ from .config import PipelineConfig, ProjectConfig
 
 
 class ProjectSlurper:
-    def __init__(self, project_dir: Path) -> None:
+    def __init__(self, project_dir: Path, config_file: Optional[str] = None) -> None:
         self.logger = logger.bind()
-        self.artifacts_locator = ProjectArtifactsLocator(project_dir)
+        self.artifacts_locator = ProjectArtifactsLocator(project_dir, config_file)
         try:
             self.user_config: ProjectConfig = ProjectConfig.from_file(self.artifacts_locator.config_file)
         except FileNotFoundError:

{pypeline_runner-1.6.0 → pypeline_runner-1.7.0}/src/pypeline/main.py

@@ -42,6 +42,7 @@ def init(
 @time_it("run")
 def run(
     project_dir: Path = typer.Option(Path.cwd().absolute(), help="The project directory"),  # noqa: B008,
+    config_file: Optional[str] = typer.Option(None, help="The name of the YAML configuration file containing the pypeline definition."),
     step: Optional[str] = typer.Option(
         None,
         help="Name of the step to run (as written in the pipeline config).",
@@ -67,7 +68,7 @@ def run(
         is_flag=True,
     ),
 ) -> None:
-    project_slurper = ProjectSlurper(project_dir)
+    project_slurper = ProjectSlurper(project_dir, config_file)
     if print:
         logger.info("Pipeline steps:")
         for group, step_configs in PipelineConfigIterator(project_slurper.pipeline):

{pypeline_runner-1.6.0 → pypeline_runner-1.7.0}/src/pypeline/pypeline.py

@@ -1,5 +1,3 @@
-import importlib
-from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
 from typing import (
     Any,
@@ -8,7 +6,6 @@ from typing import (
     List,
     Optional,
     Type,
-    cast,
 )
 
 from py_app_dev.core.exceptions import UserNotificationException
@@ -17,75 +14,18 @@ from py_app_dev.core.runnable import Executor
 
 from .domain.artifacts import ProjectArtifactsLocator
 from .domain.execution_context import ExecutionContext
-from .domain.pipeline import PipelineConfig, PipelineConfigIterator, PipelineStep, PipelineStepConfig, PipelineStepReference, TExecutionContext
+from .domain.pipeline import PipelineConfig, PipelineLoader, PipelineStep, PipelineStepConfig, PipelineStepReference, StepClassFactory, TExecutionContext
 
 
-class PipelineLoader(Generic[TExecutionContext]):
-    """
-    Loads pipeline steps from a pipeline configuration.
-
-    The steps are not instantiated, only the references are returned (lazy load).
-    The pipeline loader needs to know the project root directory to be able to find the
-    user custom local steps.
-    """
-
-    def __init__(self, pipeline_config: PipelineConfig, project_root_dir: Path) -> None:
-        self.pipeline_config = pipeline_config
-        self.project_root_dir = project_root_dir
-
-    def load_steps_references(self) -> List[PipelineStepReference[TExecutionContext]]:
-        result = []
-        for group_name, steps_config in PipelineConfigIterator(self.pipeline_config):
-            result.extend(self._load_steps(group_name, steps_config, self.project_root_dir))
-        return result
-
-    @staticmethod
-    def _load_steps(
-        group_name: Optional[str],
-        steps_config: List[PipelineStepConfig],
-        project_root_dir: Path,
-    ) -> List[PipelineStepReference[TExecutionContext]]:
-        result = []
-        for step_config in steps_config:
-            step_class_name = step_config.class_name or step_config.step
-            if step_config.module:
-                step_class = PipelineLoader._load_module_step(step_config.module, step_class_name)
-            elif step_config.file:
-                step_class = PipelineLoader._load_user_step(project_root_dir.joinpath(step_config.file), step_class_name)
-            elif step_config.run:
-                # We want the run field to always return a list of strings (the command and its arguments).
-                run_command = step_config.run.split(" ") if isinstance(step_config.run, str) else step_config.run
-                step_class = PipelineLoader._create_run_command_step_class(run_command, step_class_name)
-            else:
-                raise UserNotificationException(f"Step '{step_class_name}' has no 'module' nor 'file' nor `run` defined. Please check your pipeline configuration.")
-            result.append(PipelineStepReference[TExecutionContext](group_name, cast(Type[PipelineStep[TExecutionContext]], step_class), step_config.config))
-        return result
-
-    @staticmethod
-    def _load_user_step(python_file: Path, step_class_name: str) -> Type[PipelineStep[ExecutionContext]]:
-        # Create a module specification from the file path
-        spec = spec_from_file_location(f"user__{python_file.stem}", python_file)
-        if spec and spec.loader:
-            step_module = module_from_spec(spec)
-            # Import the module
-            spec.loader.exec_module(step_module)
-            try:
-                step_class = getattr(step_module, step_class_name)
-            except AttributeError:
-                raise UserNotificationException(f"Could not load class '{step_class_name}' from file '{python_file}'. Please check your pipeline configuration.") from None
-            return step_class
-        raise UserNotificationException(f"Could not load file '{python_file}'. Please check the file for any errors.")
-
-    @staticmethod
-    def _load_module_step(module_name: str, step_class_name: str) -> Type[PipelineStep[ExecutionContext]]:
-        try:
-            module = importlib.import_module(module_name)
-            step_class = getattr(module, step_class_name)
-        except ImportError:
-            raise UserNotificationException(f"Could not load module '{module_name}'. Please check your pipeline configuration.") from None
-        except AttributeError:
-            raise UserNotificationException(f"Could not load class '{step_class_name}' from module '{module_name}'. Please check your pipeline configuration.") from None
-        return step_class
+class RunCommandClassFactory(StepClassFactory[PipelineStep[TExecutionContext]]):
+    def create_step_class(self, step_config: PipelineStepConfig, project_root_dir: Path) -> Type[PipelineStep[ExecutionContext]]:
+        _ = project_root_dir  # Unused because we do not need to locate files relative to the project root directory
+        step_name = step_config.class_name or step_config.step
+        if step_config.run:
+            # We want the run field to always return a list of strings (the command and its arguments).
+            run_command = step_config.run.split(" ") if isinstance(step_config.run, str) else step_config.run
+            return self._create_run_command_step_class(run_command, step_name)
+        raise UserNotificationException(f"Step '{step_name}' has `run` command defined. Please check your pipeline configuration.")
 
     @staticmethod
     def _create_run_command_step_class(command: List[str], name: str) -> Type[PipelineStep[ExecutionContext]]:
@@ -133,7 +73,7 @@ class PipelineStepsExecutor(Generic[TExecutionContext]):
     def __init__(
         self,
         execution_context: TExecutionContext,
-        steps_references: List[PipelineStepReference[TExecutionContext]],
+        steps_references: List[PipelineStepReference[PipelineStep[TExecutionContext]]],
         force_run: bool = False,
         dry_run: bool = False,
     ) -> None:
@@ -175,16 +115,15 @@ class PipelineScheduler(Generic[TExecutionContext]):
         self.project_root_dir = project_root_dir
         self.logger = logger.bind()
 
-    def get_steps_to_run(self, step_name: Optional[str] = None, single: bool = False) -> List[PipelineStepReference[TExecutionContext]]:
-        pipeline_loader = PipelineLoader[TExecutionContext](self.pipeline, self.project_root_dir)
-        return self.filter_steps_references(pipeline_loader.load_steps_references(), step_name, single)
+    def get_steps_to_run(self, step_name: Optional[str] = None, single: bool = False) -> List[PipelineStepReference[PipelineStep[TExecutionContext]]]:
+        return self.filter_steps_references(self.create_pipeline_loader(self.pipeline, self.project_root_dir).load_steps_references(), step_name, single)
 
     @staticmethod
     def filter_steps_references(
-        steps_references: List[PipelineStepReference[TExecutionContext]],
+        steps_references: List[PipelineStepReference[PipelineStep[TExecutionContext]]],
         step_name: Optional[str],
         single: Optional[bool],
-    ) -> List[PipelineStepReference[TExecutionContext]]:
+    ) -> List[PipelineStepReference[PipelineStep[TExecutionContext]]]:
         if step_name:
             step_reference = next((step for step in steps_references if step.name == step_name), None)
             if not step_reference:
@@ -193,3 +132,7 @@ class PipelineScheduler(Generic[TExecutionContext]):
                 return [step_reference]
             return [step for step in steps_references if steps_references.index(step) <= steps_references.index(step_reference)]
         return steps_references
+
+    @staticmethod
+    def create_pipeline_loader(pipeline: PipelineConfig, project_root_dir: Path) -> PipelineLoader[PipelineStep[TExecutionContext]]:
+        return PipelineLoader[PipelineStep[TExecutionContext]](pipeline, project_root_dir, RunCommandClassFactory())

pypeline_runner-1.6.0/src/pypeline/__init__.py

@@ -1 +0,0 @@
-__version__ = "1.6.0"

pypeline_runner-1.6.0/src/pypeline/domain/pipeline.py

@@ -1,109 +0,0 @@
-from abc import abstractmethod
-from dataclasses import dataclass
-from pathlib import Path
-from typing import (
-    Any,
-    Dict,
-    Generic,
-    Iterator,
-    List,
-    Optional,
-    OrderedDict,
-    Tuple,
-    Type,
-    TypeAlias,
-    TypeVar,
-    Union,
-)
-
-from mashumaro import DataClassDictMixin
-from py_app_dev.core.runnable import Runnable
-
-from .execution_context import ExecutionContext
-
-
-@dataclass
-class PipelineStepConfig(DataClassDictMixin):
-    #: Step name or class name if file is not specified
-    step: str
-    #: Path to file with step class
-    file: Optional[str] = None
-    #: Python module with step class
-    module: Optional[str] = None
-    #: Step class name
-    class_name: Optional[str] = None
-    #: Command to run. For simple steps that don't need a class. Example: run: [echo, 'Hello World!']
-    run: Optional[Union[str, List[str]]] = None
-    #: Step description
-    description: Optional[str] = None
-    #: Step timeout in seconds
-    timeout_sec: Optional[int] = None
-    #: Custom step configuration
-    config: Optional[Dict[str, Any]] = None
-
-
-PipelineConfig: TypeAlias = Union[List[PipelineStepConfig], OrderedDict[str, List[PipelineStepConfig]]]
-
-
-class PipelineConfigIterator:
-    """
-    Iterates over the pipeline configuration, yielding group name and steps configuration.
-
-    This class abstracts the iteration logic for PipelineConfig, which can be:
-    - A list of steps (group name is None)
-    - An OrderedDict with group names as keys and lists of steps as values.
-
-    The iterator yields tuples of (group_name, steps).
-    """
-
-    def __init__(self, pipeline_config: PipelineConfig) -> None:
-        self._items = pipeline_config.items() if isinstance(pipeline_config, OrderedDict) else [(None, pipeline_config)]
-
-    def __iter__(self) -> Iterator[Tuple[Optional[str], List[PipelineStepConfig]]]:
-        """Return an iterator."""
-        yield from self._items
-
-
-TExecutionContext = TypeVar("TExecutionContext", bound=ExecutionContext)
-
-
-class PipelineStep(Generic[TExecutionContext], Runnable):
-    """One can create subclasses of PipelineStep that specify the type of ExecutionContext they require."""
-
-    def __init__(self, execution_context: TExecutionContext, group_name: Optional[str], config: Optional[Dict[str, Any]] = None) -> None:
-        super().__init__(self.get_needs_dependency_management())
-        self.execution_context = execution_context
-        self.group_name = group_name
-        self.config = config
-        self.project_root_dir = self.execution_context.project_root_dir
-
-    @property
-    def output_dir(self) -> Path:
-        output_dir = self.execution_context.create_artifacts_locator().build_dir
-        if self.group_name:
-            output_dir = output_dir / self.group_name
-        return output_dir
-
-    @abstractmethod
-    def update_execution_context(self) -> None:
-        """
-        Even if the step does not need to run ( because it is not outdated ), it can still update the execution context.
-
-        A typical use case is for steps installing software that need to provide the install directories in the execution context even if all tools are already installed.
-        """
-        pass
-
-    def get_needs_dependency_management(self) -> bool:
-        """If false, the step executor will not check for outdated dependencies. This is useful for steps consisting of command lines which shall always run."""
-        return True
-
-
-class PipelineStepReference(Generic[TExecutionContext]):
-    def __init__(self, group_name: Optional[str], _class: Type[PipelineStep[TExecutionContext]], config: Optional[Dict[str, Any]] = None) -> None:
-        self.group_name = group_name
-        self._class = _class
-        self.config = config
-
-    @property
-    def name(self) -> str:
-        return self._class.__name__