pypeline-runner 0.2.1__tar.gz → 0.3.0__tar.gz

{pypeline_runner-0.2.1 → pypeline_runner-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pypeline-runner
-Version: 0.2.1
+Version: 0.3.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 Home-page: https://github.com/cuinixam/pypeline
 License: MIT
@@ -17,9 +17,9 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries
-Requires-Dist: py-app-dev (>=2.1.1,<3.0.0)
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: typer[all] (>=0.12.0,<0.13.0)
+Requires-Dist: py-app-dev (>=2.2,<3.0)
+Requires-Dist: pyyaml (>=6.0,<7.0)
+Requires-Dist: typer[all] (>=0.12,<0.13)
 Project-URL: Bug Tracker, https://github.com/cuinixam/pypeline/issues
 Project-URL: Changelog, https://github.com/cuinixam/pypeline/blob/main/CHANGELOG.md
 Project-URL: Documentation, https://pypeline-runner.readthedocs.io

{pypeline_runner-0.2.1 → pypeline_runner-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pypeline-runner"
-version = "0.2.1"
+version = "0.3.0"
 description = "Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines)."
 authors = ["cuinixam <me@cuinixam.com>"]
 license = "MIT"
@@ -27,25 +27,25 @@ pypeline = "pypeline.main:main"
 
 [tool.poetry.dependencies]
 python = "^3.10"
-py-app-dev = "^2.1.1"
-typer = {extras = ["all"], version = "^0.12.0"}
-pyyaml = "^6.0.1"
+py-app-dev = "^2.2"
+typer = {extras = ["all"], version = "^0.12"}
+pyyaml = "^6.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.0"
 pytest-cov = "^4.0"
-pre-commit = "^3.1.1"
-ruff = "^0.3.0"
+pre-commit = "^3.1"
+ruff = "^0.3"
 
 [tool.poetry.group.docs.dependencies]
 myst-parser = ">=0.16"
 sphinx = ">=4.0"
-sphinxcontrib-mermaid = "^0.8.1"
-mlx-traceability = "^10.0.0"
-sphinx-copybutton = "^0.5.2"
-sphinx-new-tab-link = "^0.2.2"
-sphinx-book-theme = "^1.1.2"
-sphinx-design = "^0.5.0"
+sphinxcontrib-mermaid = "^0.8"
+mlx-traceability = "^10.0"
+sphinx-copybutton = "^0.5"
+sphinx-new-tab-link = "^0.2"
+sphinx-book-theme = "^1.1"
+sphinx-design = "^0.5"
 
 [tool.semantic_release]
 version_toml = ["pyproject.toml:tool.poetry.version"]

pypeline_runner-0.3.0/src/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{pypeline_runner-0.2.1 → pypeline_runner-0.3.0}/src/pypeline/domain/config.py

@@ -5,10 +5,11 @@ from typing import Any, Dict, Optional
 import yaml
 from mashumaro import DataClassDictMixin
 from py_app_dev.core.exceptions import UserNotificationException
-from py_app_dev.core.pipeline import PipelineConfig
 from yaml.parser import ParserError
 from yaml.scanner import ScannerError
 
+from .pipeline import PipelineConfig
+
 
 @dataclass
 class ProjectConfig(DataClassDictMixin):

pypeline_runner-0.3.0/src/pypeline/domain/pipeline.py

@@ -0,0 +1,67 @@
+from abc import abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    OrderedDict,
+    Type,
+    TypeAlias,
+)
+
+from mashumaro import DataClassDictMixin
+from py_app_dev.core.runnable import Runnable
+
+from .execution_context import ExecutionContext
+
+
+@dataclass
+class PipelineStepConfig(DataClassDictMixin):
+    #: Step name or class name if file is not specified
+    step: str
+    #: Path to file with step class
+    file: Optional[str] = None
+    #: Python module with step class
+    module: Optional[str] = None
+    #: Step class name
+    class_name: Optional[str] = None
+    #: Command to run. For simple steps that don't need a class
+    run: Optional[str] = None
+    #: Step description
+    description: Optional[str] = None
+    #: Step timeout in seconds
+    timeout_sec: Optional[int] = None
+    #: Custom step configuration
+    config: Optional[Dict[str, Any]] = None
+
+
+PipelineConfig: TypeAlias = OrderedDict[str, List[PipelineStepConfig]]
+
+
+class PipelineStep(Runnable):
+    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(self.get_needs_dependency_management())
+        self.execution_context = execution_context
+        self.output_dir = output_dir
+        self.config = config
+        self.project_root_dir = self.execution_context.project_root_dir
+
+    @abstractmethod
+    def update_execution_context(self) -> None:
+        pass
+
+    def get_needs_dependency_management(self) -> bool:
+        return True
+
+
+class PipelineStepReference:
+    def __init__(self, group_name: str, _class: Type[PipelineStep], config: Optional[Dict[str, Any]] = None) -> None:
+        self.group_name = group_name
+        self._class = _class
+        self.config = config
+
+    @property
+    def name(self) -> str:
+        return self._class.__name__

{pypeline_runner-0.2.1 → pypeline_runner-0.3.0}/src/pypeline/main.py

@@ -86,10 +86,14 @@ def run(
     PipelineStepsExecutor(project_slurper.artifacts_locator, steps_references, force_run, dry_run).run()
 
 
-if __name__ == "__main__":
+def main() -> None:
     try:
         setup_logger()
         app()
     except UserNotificationException as e:
         logger.error(f"{e}")
         sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

pypeline_runner-0.3.0/src/pypeline/pypeline.py

@@ -0,0 +1,187 @@
+import importlib
+from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Type,
+)
+
+from py_app_dev.core.exceptions import UserNotificationException
+from py_app_dev.core.logging import logger
+from py_app_dev.core.runnable import Executor
+
+from .domain.artifacts import ProjectArtifactsLocator
+from .domain.execution_context import ExecutionContext
+from .domain.pipeline import PipelineConfig, PipelineStep, PipelineStepConfig, PipelineStepReference
+
+
+class PipelineLoader:
+    """
+    Loads pipeline steps from a pipeline configuration.
+
+    The steps are not instantiated, only the references are returned (lazy load).
+    The pipeline loader needs to know the project root directory to be able to find the
+    user custom local steps.
+    """
+
+    def __init__(self, pipeline_config: PipelineConfig, project_root_dir: Path) -> None:
+        self.pipeline_config = pipeline_config
+        self.project_root_dir = project_root_dir
+
+    def load_steps_references(self) -> List[PipelineStepReference]:
+        result = []
+        for group_name, steps_config in self.pipeline_config.items():
+            result.extend(self._load_steps(group_name, steps_config, self.project_root_dir))
+        return result
+
+    @staticmethod
+    def _load_steps(
+        group_name: str,
+        steps_config: List[PipelineStepConfig],
+        project_root_dir: Path,
+    ) -> List[PipelineStepReference]:
+        result = []
+        for step_config in steps_config:
+            step_class_name = step_config.class_name or step_config.step
+            if step_config.module:
+                step_class = PipelineLoader._load_module_step(step_config.module, step_class_name)
+            elif step_config.file:
+                step_class = PipelineLoader._load_user_step(project_root_dir.joinpath(step_config.file), step_class_name)
+            elif step_config.run:
+                step_class = PipelineLoader._create_run_command_step_class(step_config.run, step_class_name)
+            else:
+                raise UserNotificationException(f"Step '{step_class_name}' has no 'module' nor 'file' nor `run` defined." " Please check your pipeline configuration.")
+            result.append(PipelineStepReference(group_name, step_class, step_config.config))
+        return result
+
+    @staticmethod
+    def _load_user_step(python_file: Path, step_class_name: str) -> Type[PipelineStep]:
+        # Create a module specification from the file path
+        spec = spec_from_file_location(f"user__{step_class_name}", python_file)
+        if spec and spec.loader:
+            step_module = module_from_spec(spec)
+            # Import the module
+            spec.loader.exec_module(step_module)
+            try:
+                step_class = getattr(step_module, step_class_name)
+            except AttributeError:
+                raise UserNotificationException(f"Could not load class '{step_class_name}' from file '{python_file}'." " Please check your pipeline configuration.") from None
+            return step_class
+        raise UserNotificationException(f"Could not load file '{python_file}'." " Please check the file for any errors.")
+
+    @staticmethod
+    def _load_module_step(module_name: str, step_class_name: str) -> Type[PipelineStep]:
+        try:
+            module = importlib.import_module(module_name)
+            step_class = getattr(module, step_class_name)
+        except ImportError:
+            raise UserNotificationException(f"Could not load module '{module_name}'. Please check your pipeline configuration.") from None
+        except AttributeError:
+            raise UserNotificationException(f"Could not load class '{step_class_name}' from module '{module_name}'." " Please check your pipeline configuration.") from None
+        return step_class
+
+    @staticmethod
+    def _create_run_command_step_class(command: str, name: str) -> Type[PipelineStep]:
+        """Dynamically creates a step class for a given command."""
+
+        class DynamicRunCommandStep(PipelineStep):
+            """A simple step that runs a command."""
+
+            def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+                super().__init__(execution_context, output_dir, config)
+                self.command = command
+                self.name = name
+
+            def get_needs_dependency_management(self) -> bool:
+                """A command step does not need dependency management."""
+                return False
+
+            def run(self) -> int:
+                self.execution_context.create_process_executor(
+                    [self.command],
+                    cwd=self.project_root_dir,
+                ).execute()
+                return 0
+
+            def get_name(self) -> str:
+                return self.name
+
+            def get_inputs(self) -> List[Path]:
+                return []
+
+            def get_outputs(self) -> List[Path]:
+                return []
+
+            def update_execution_context(self) -> None:
+                pass
+
+        return DynamicRunCommandStep
+
+
+class PipelineStepsExecutor:
+    """Executes a list of pipeline steps sequentially."""
+
+    def __init__(
+        self,
+        artifacts_locator: ProjectArtifactsLocator,
+        steps_references: List[PipelineStepReference],
+        force_run: bool = False,
+        dry_run: bool = False,
+    ) -> None:
+        self.logger = logger.bind()
+        self.artifacts_locator = artifacts_locator
+        self.steps_references = steps_references
+        self.force_run = force_run
+        self.dry_run = dry_run
+
+    def run(self) -> None:
+        execution_context = ExecutionContext(project_root_dir=self.artifacts_locator.project_root_dir, install_dirs=[])
+        for step_reference in self.steps_references:
+            step_output_dir = self.artifacts_locator.build_dir / step_reference.group_name
+            # Create the step output directory, to make sure that files can be created.
+            step_output_dir.mkdir(parents=True, exist_ok=True)
+            step = step_reference._class(execution_context, step_output_dir, step_reference.config)
+            # Execute the step is necessary. If the step is not dirty, it will not be executed
+            Executor(step.output_dir, self.force_run, self.dry_run).execute(step)
+            # Independent if the step was executed or not, every step shall update the context
+            step.update_execution_context()
+
+        return
+
+
+class PipelineScheduler:
+    """
+    Schedules which steps must be executed based on the provided configuration.
+
+    * If a step name is provided and the single flag is set, only that step will be executed.
+    * If a step name is provided and the single flag is not set, all steps up to the provided step will be executed.
+    * In case a command is provided, only the steps up to that command will be executed.
+    * If no step name is provided, all steps will be executed.
+    """
+
+    def __init__(self, pipeline: PipelineConfig, project_root_dir: Path) -> None:
+        self.pipeline = pipeline
+        self.project_root_dir = project_root_dir
+        self.logger = logger.bind()
+
+    def get_steps_to_run(self, step_name: Optional[str] = None, single: bool = False) -> List[PipelineStepReference]:
+        pipeline_loader = PipelineLoader(self.pipeline, self.project_root_dir)
+        return self.filter_steps_references(pipeline_loader.load_steps_references(), step_name, single)
+
+    @staticmethod
+    def filter_steps_references(
+        steps_references: List[PipelineStepReference],
+        step_name: Optional[str],
+        single: Optional[bool],
+    ) -> List[PipelineStepReference]:
+        if step_name:
+            step_reference = next((step for step in steps_references if step.name == step_name), None)
+            if not step_reference:
+                return []
+            if single:
+                return [step_reference]
+            return [step for step in steps_references if steps_references.index(step) <= steps_references.index(step_reference)]
+        return steps_references

pypeline_runner-0.2.1/src/pypeline/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.1"

pypeline_runner-0.2.1/src/pypeline/domain/pipeline.py

@@ -1,30 +0,0 @@
-from abc import abstractmethod
-from pathlib import Path
-from typing import Any, Dict, Optional, Type
-
-from py_app_dev.core.runnable import Runnable
-
-from .execution_context import ExecutionContext
-
-
-class PipelineStep(Runnable):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
-        self.execution_context = execution_context
-        self.output_dir = output_dir
-        self.config = config
-        self.project_root_dir = self.execution_context.project_root_dir
-
-    @abstractmethod
-    def update_execution_context(self) -> None:
-        pass
-
-
-class PipelineStepReference:
-    def __init__(self, group_name: str, _class: Type[PipelineStep], config: Optional[Dict[str, Any]] = None) -> None:
-        self.group_name = group_name
-        self._class = _class
-        self.config = config
-
-    @property
-    def name(self) -> str:
-        return self._class.__name__

pypeline_runner-0.2.1/src/pypeline/pypeline.py

@@ -1,95 +0,0 @@
-from pathlib import Path
-from typing import List, Optional
-
-from py_app_dev.core.logging import logger
-from py_app_dev.core.pipeline import PipelineConfig
-from py_app_dev.core.pipeline import PipelineLoader as GenericPipelineLoader
-from py_app_dev.core.runnable import Executor
-
-from .domain.artifacts import ProjectArtifactsLocator
-from .domain.execution_context import ExecutionContext
-from .domain.pipeline import PipelineStep, PipelineStepReference
-
-
-class PipelineLoader:
-    """
-    Loads pipeline steps from a pipeline configuration.
-
-    The steps are not instantiated, only the references are returned (lazy load).
-    The pipeline loader needs to know the project root directory to be able to find the
-    user custom local steps.
-    """
-
-    def __init__(self, pipeline_config: PipelineConfig, project_root_dir: Path) -> None:
-        self.pipeline_config = pipeline_config
-        self.project_root_dir = project_root_dir
-        self._loader = GenericPipelineLoader[PipelineStep](self.pipeline_config, self.project_root_dir)
-
-    def load_steps_references(self) -> List[PipelineStepReference]:
-        return [PipelineStepReference(step_reference.group_name, step_reference._class, step_reference.config) for step_reference in self._loader.load_steps()]
-
-
-class PipelineStepsExecutor:
-    """Executes a list of pipeline steps sequentially."""
-
-    def __init__(
-        self,
-        artifacts_locator: ProjectArtifactsLocator,
-        steps_references: List[PipelineStepReference],
-        force_run: bool = False,
-        dry_run: bool = False,
-    ) -> None:
-        self.logger = logger.bind()
-        self.artifacts_locator = artifacts_locator
-        self.steps_references = steps_references
-        self.force_run = force_run
-        self.dry_run = dry_run
-
-    def run(self) -> None:
-        execution_context = ExecutionContext(project_root_dir=self.artifacts_locator.project_root_dir, install_dirs=[])
-        for step_reference in self.steps_references:
-            step_output_dir = self.artifacts_locator.build_dir / step_reference.group_name
-            # Create the step output directory, to make sure that files can be created.
-            step_output_dir.mkdir(parents=True, exist_ok=True)
-            step = step_reference._class(execution_context, step_output_dir, step_reference.config)
-            # Execute the step is necessary. If the step is not dirty, it will not be executed
-            Executor(step.output_dir, self.force_run, self.dry_run).execute(step)
-            # Independent if the step was executed or not, every step shall update the context
-            step.update_execution_context()
-
-        return
-
-
-class PipelineScheduler:
-    """
-    Schedules which steps must be executed based on the provided configuration.
-
-    * If a step name is provided and the single flag is set, only that step will be executed.
-    * If a step name is provided and the single flag is not set, all steps up to the provided step will be executed.
-    * In case a command is provided, only the steps up to that command will be executed.
-    * If no step name is provided, all steps will be executed.
-    """
-
-    def __init__(self, pipeline: PipelineConfig, project_root_dir: Path) -> None:
-        self.pipeline = pipeline
-        self.project_root_dir = project_root_dir
-        self.logger = logger.bind()
-
-    def get_steps_to_run(self, step_name: Optional[str] = None, single: bool = False) -> List[PipelineStepReference]:
-        pipeline_loader = PipelineLoader(self.pipeline, self.project_root_dir)
-        return self.filter_steps_references(pipeline_loader.load_steps_references(), step_name, single)
-
-    @staticmethod
-    def filter_steps_references(
-        steps_references: List[PipelineStepReference],
-        step_name: Optional[str],
-        single: Optional[bool],
-    ) -> List[PipelineStepReference]:
-        if step_name:
-            step_reference = next((step for step in steps_references if step.name == step_name), None)
-            if not step_reference:
-                return []
-            if single:
-                return [step_reference]
-            return [step for step in steps_references if steps_references.index(step) <= steps_references.index(step_reference)]
-        return steps_references