pypeline-runner 1.5.0__py3-none-any.whl → 1.7.0__py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "1.5.0"
+__version__ = "1.7.0"

pypeline/domain/artifacts.py

@@ -10,13 +10,10 @@ CONFIG_FILENAME = "pypeline.yaml"
 class ProjectArtifactsLocator:
     """Provides paths to project artifacts."""
 
-    def __init__(
-        self,
-        project_root_dir: Path,
-    ) -> None:
+    def __init__(self, project_root_dir: Path, config_file: Optional[str] = None) -> None:
         self.project_root_dir = project_root_dir
         self.build_dir = project_root_dir / "build"
-        self.config_file = project_root_dir / CONFIG_FILENAME
+        self.config_file = project_root_dir.joinpath(config_file if config_file else CONFIG_FILENAME)
         self.external_dependencies_dir = self.build_dir / "external"
         scripts_dir = "Scripts" if sys.platform.startswith("win32") else "bin"
         self.venv_scripts_dir = self.project_root_dir.joinpath(".venv").joinpath(scripts_dir)

pypeline/domain/pipeline.py

@@ -1,5 +1,7 @@
+import importlib
 from abc import abstractmethod
 from dataclasses import dataclass
+from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
 from typing import (
     Any,
@@ -9,6 +11,7 @@ from typing import (
     List,
     Optional,
     OrderedDict,
+    Protocol,
     Tuple,
     Type,
     TypeAlias,
@@ -17,6 +20,7 @@ from typing import (
 )
 
 from mashumaro import DataClassDictMixin
+from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.runnable import Runnable
 
 from .execution_context import ExecutionContext
@@ -44,6 +48,21 @@ class PipelineStepConfig(DataClassDictMixin):
 
 PipelineConfig: TypeAlias = Union[List[PipelineStepConfig], OrderedDict[str, List[PipelineStepConfig]]]
 
+TPipelineStep = TypeVar("TPipelineStep", covariant=True)
+
+
+@dataclass
+class PipelineStepReference(Generic[TPipelineStep]):
+    """Once a Step is found, keep the Step class reference to be able to instantiate it later."""
+
+    group_name: Optional[str]
+    _class: Type[TPipelineStep]
+    config: Optional[Dict[str, Any]] = None
+
+    @property
+    def name(self) -> str:
+        return self._class.__name__
+
 
 class PipelineConfigIterator:
     """
@@ -64,6 +83,70 @@ class PipelineConfigIterator:
         yield from self._items
 
 
+class StepClassFactory(Generic[TPipelineStep], Protocol):
+    def create_step_class(self, step_config: PipelineStepConfig, project_root_dir: Path) -> Type[TPipelineStep]: ...
+
+
+class PipelineLoader(Generic[TPipelineStep]):
+    def __init__(self, pipeline_config: PipelineConfig, project_root_dir: Path, step_class_factory: Optional[StepClassFactory[TPipelineStep]] = None) -> None:
+        self.pipeline_config = pipeline_config
+        self.project_root_dir = project_root_dir
+        self.step_class_factory = step_class_factory
+
+    def load_steps_references(self) -> List[PipelineStepReference[TPipelineStep]]:
+        result = []
+        for group_name, steps_config in PipelineConfigIterator(self.pipeline_config):
+            result.extend(self._load_steps(group_name, steps_config, self.project_root_dir, self.step_class_factory))
+        return result
+
+    @staticmethod
+    def _load_steps(
+        group_name: Optional[str], steps_config: List[PipelineStepConfig], project_root_dir: Path, step_class_factory: Optional[StepClassFactory[TPipelineStep]] = None
+    ) -> List[PipelineStepReference[TPipelineStep]]:
+        result = []
+        for step_config in steps_config:
+            step_class_name = step_config.class_name or step_config.step
+            if step_config.module:
+                step_class = PipelineLoader[TPipelineStep]._load_module_step(step_config.module, step_class_name)
+            elif step_config.file:
+                step_class = PipelineLoader[TPipelineStep]._load_user_step(project_root_dir.joinpath(step_config.file), step_class_name)
+            else:
+                if step_class_factory:
+                    step_class = step_class_factory.create_step_class(step_config, project_root_dir)
+                else:
+                    raise UserNotificationException(
+                        f"Step '{step_class_name}' has no 'module' nor 'file' defined nor a custom step class factory was provided. Please check your pipeline configuration."
+                    )
+            result.append(PipelineStepReference(group_name, step_class, step_config.config))
+        return result
+
+    @staticmethod
+    def _load_user_step(python_file: Path, step_class_name: str) -> Type[TPipelineStep]:
+        # Create a module specification from the file path
+        spec = spec_from_file_location(f"user__{python_file.stem}", python_file)
+        if spec and spec.loader:
+            step_module = module_from_spec(spec)
+            # Import the module
+            spec.loader.exec_module(step_module)
+            try:
+                step_class = getattr(step_module, step_class_name)
+            except AttributeError:
+                raise UserNotificationException(f"Could not load class '{step_class_name}' from file '{python_file}'. Please check your pipeline configuration.") from None
+            return step_class
+        raise UserNotificationException(f"Could not load file '{python_file}'. Please check the file for any errors.")
+
+    @staticmethod
+    def _load_module_step(module_name: str, step_class_name: str) -> Type[TPipelineStep]:
+        try:
+            module = importlib.import_module(module_name)
+            step_class = getattr(module, step_class_name)
+        except ImportError:
+            raise UserNotificationException(f"Could not load module '{module_name}'. Please check your pipeline configuration.") from None
+        except AttributeError:
+            raise UserNotificationException(f"Could not load class '{step_class_name}' from module '{module_name}'. Please check your pipeline configuration.") from None
+        return step_class
+
+
 TExecutionContext = TypeVar("TExecutionContext", bound=ExecutionContext)
 
 
@@ -96,14 +179,3 @@ class PipelineStep(Generic[TExecutionContext], Runnable):
     def get_needs_dependency_management(self) -> bool:
         """If false, the step executor will not check for outdated dependencies. This is useful for steps consisting of command lines which shall always run."""
         return True
-
-
-class PipelineStepReference(Generic[TExecutionContext]):
-    def __init__(self, group_name: Optional[str], _class: Type[PipelineStep[TExecutionContext]], config: Optional[Dict[str, Any]] = None) -> None:
-        self.group_name = group_name
-        self._class = _class
-        self.config = config
-
-    @property
-    def name(self) -> str:
-        return self._class.__name__

pypeline/domain/project_slurper.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Optional
 
 from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.logging import logger
@@ -8,9 +9,9 @@ from .config import PipelineConfig, ProjectConfig
 
 
 class ProjectSlurper:
-    def __init__(self, project_dir: Path) -> None:
+    def __init__(self, project_dir: Path, config_file: Optional[str] = None) -> None:
         self.logger = logger.bind()
-        self.artifacts_locator = ProjectArtifactsLocator(project_dir)
+        self.artifacts_locator = ProjectArtifactsLocator(project_dir, config_file)
         try:
             self.user_config: ProjectConfig = ProjectConfig.from_file(self.artifacts_locator.config_file)
         except FileNotFoundError:

pypeline/main.py

@@ -42,6 +42,7 @@ def init(
 @time_it("run")
 def run(
     project_dir: Path = typer.Option(Path.cwd().absolute(), help="The project directory"),  # noqa: B008,
+    config_file: Optional[str] = typer.Option(None, help="The name of the YAML configuration file containing the pypeline definition."),
     step: Optional[str] = typer.Option(
         None,
         help="Name of the step to run (as written in the pipeline config).",
@@ -67,7 +68,7 @@ def run(
         is_flag=True,
     ),
 ) -> None:
-    project_slurper = ProjectSlurper(project_dir)
+    project_slurper = ProjectSlurper(project_dir, config_file)
     if print:
         logger.info("Pipeline steps:")
         for group, step_configs in PipelineConfigIterator(project_slurper.pipeline):

pypeline/pypeline.py

@@ -1,5 +1,3 @@
-import importlib
-from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
 from typing import (
     Any,
@@ -8,7 +6,6 @@ from typing import (
     List,
     Optional,
     Type,
-    cast,
 )
 
 from py_app_dev.core.exceptions import UserNotificationException
@@ -17,75 +14,18 @@ from py_app_dev.core.runnable import Executor
 
 from .domain.artifacts import ProjectArtifactsLocator
 from .domain.execution_context import ExecutionContext
-from .domain.pipeline import PipelineConfig, PipelineConfigIterator, PipelineStep, PipelineStepConfig, PipelineStepReference, TExecutionContext
+from .domain.pipeline import PipelineConfig, PipelineLoader, PipelineStep, PipelineStepConfig, PipelineStepReference, StepClassFactory, TExecutionContext
 
 
-class PipelineLoader(Generic[TExecutionContext]):
-    """
-    Loads pipeline steps from a pipeline configuration.
-
-    The steps are not instantiated, only the references are returned (lazy load).
-    The pipeline loader needs to know the project root directory to be able to find the
-    user custom local steps.
-    """
-
-    def __init__(self, pipeline_config: PipelineConfig, project_root_dir: Path) -> None:
-        self.pipeline_config = pipeline_config
-        self.project_root_dir = project_root_dir
-
-    def load_steps_references(self) -> List[PipelineStepReference[TExecutionContext]]:
-        result = []
-        for group_name, steps_config in PipelineConfigIterator(self.pipeline_config):
-            result.extend(self._load_steps(group_name, steps_config, self.project_root_dir))
-        return result
-
-    @staticmethod
-    def _load_steps(
-        group_name: Optional[str],
-        steps_config: List[PipelineStepConfig],
-        project_root_dir: Path,
-    ) -> List[PipelineStepReference[TExecutionContext]]:
-        result = []
-        for step_config in steps_config:
-            step_class_name = step_config.class_name or step_config.step
-            if step_config.module:
-                step_class = PipelineLoader._load_module_step(step_config.module, step_class_name)
-            elif step_config.file:
-                step_class = PipelineLoader._load_user_step(project_root_dir.joinpath(step_config.file), step_class_name)
-            elif step_config.run:
-                # We want the run field to always return a list of strings (the command and its arguments).
-                run_command = step_config.run.split(" ") if isinstance(step_config.run, str) else step_config.run
-                step_class = PipelineLoader._create_run_command_step_class(run_command, step_class_name)
-            else:
-                raise UserNotificationException(f"Step '{step_class_name}' has no 'module' nor 'file' nor `run` defined." " Please check your pipeline configuration.")
-            result.append(PipelineStepReference[TExecutionContext](group_name, cast(Type[PipelineStep[TExecutionContext]], step_class), step_config.config))
-        return result
-
-    @staticmethod
-    def _load_user_step(python_file: Path, step_class_name: str) -> Type[PipelineStep[ExecutionContext]]:
-        # Create a module specification from the file path
-        spec = spec_from_file_location(f"user__{step_class_name}", python_file)
-        if spec and spec.loader:
-            step_module = module_from_spec(spec)
-            # Import the module
-            spec.loader.exec_module(step_module)
-            try:
-                step_class = getattr(step_module, step_class_name)
-            except AttributeError:
-                raise UserNotificationException(f"Could not load class '{step_class_name}' from file '{python_file}'." " Please check your pipeline configuration.") from None
-            return step_class
-        raise UserNotificationException(f"Could not load file '{python_file}'." " Please check the file for any errors.")
-
-    @staticmethod
-    def _load_module_step(module_name: str, step_class_name: str) -> Type[PipelineStep[ExecutionContext]]:
-        try:
-            module = importlib.import_module(module_name)
-            step_class = getattr(module, step_class_name)
-        except ImportError:
-            raise UserNotificationException(f"Could not load module '{module_name}'. Please check your pipeline configuration.") from None
-        except AttributeError:
-            raise UserNotificationException(f"Could not load class '{step_class_name}' from module '{module_name}'." " Please check your pipeline configuration.") from None
-        return step_class
+class RunCommandClassFactory(StepClassFactory[PipelineStep[TExecutionContext]]):
+    def create_step_class(self, step_config: PipelineStepConfig, project_root_dir: Path) -> Type[PipelineStep[ExecutionContext]]:
+        _ = project_root_dir  # Unused because we do not need to locate files relative to the project root directory
+        step_name = step_config.class_name or step_config.step
+        if step_config.run:
+            # We want the run field to always return a list of strings (the command and its arguments).
+            run_command = step_config.run.split(" ") if isinstance(step_config.run, str) else step_config.run
+            return self._create_run_command_step_class(run_command, step_name)
+        raise UserNotificationException(f"Step '{step_name}' has `run` command defined. Please check your pipeline configuration.")
 
     @staticmethod
     def _create_run_command_step_class(command: List[str], name: str) -> Type[PipelineStep[ExecutionContext]]:
@@ -133,7 +73,7 @@ class PipelineStepsExecutor(Generic[TExecutionContext]):
     def __init__(
         self,
         execution_context: TExecutionContext,
-        steps_references: List[PipelineStepReference[TExecutionContext]],
+        steps_references: List[PipelineStepReference[PipelineStep[TExecutionContext]]],
         force_run: bool = False,
         dry_run: bool = False,
     ) -> None:
@@ -175,16 +115,15 @@ class PipelineScheduler(Generic[TExecutionContext]):
         self.project_root_dir = project_root_dir
         self.logger = logger.bind()
 
-    def get_steps_to_run(self, step_name: Optional[str] = None, single: bool = False) -> List[PipelineStepReference[TExecutionContext]]:
-        pipeline_loader = PipelineLoader[TExecutionContext](self.pipeline, self.project_root_dir)
-        return self.filter_steps_references(pipeline_loader.load_steps_references(), step_name, single)
+    def get_steps_to_run(self, step_name: Optional[str] = None, single: bool = False) -> List[PipelineStepReference[PipelineStep[TExecutionContext]]]:
+        return self.filter_steps_references(self.create_pipeline_loader(self.pipeline, self.project_root_dir).load_steps_references(), step_name, single)
 
     @staticmethod
     def filter_steps_references(
-        steps_references: List[PipelineStepReference[TExecutionContext]],
+        steps_references: List[PipelineStepReference[PipelineStep[TExecutionContext]]],
         step_name: Optional[str],
         single: Optional[bool],
-    ) -> List[PipelineStepReference[TExecutionContext]]:
+    ) -> List[PipelineStepReference[PipelineStep[TExecutionContext]]]:
         if step_name:
             step_reference = next((step for step in steps_references if step.name == step_name), None)
             if not step_reference:
@@ -193,3 +132,7 @@ class PipelineScheduler(Generic[TExecutionContext]):
                 return [step_reference]
             return [step for step in steps_references if steps_references.index(step) <= steps_references.index(step_reference)]
         return steps_references
+
+    @staticmethod
+    def create_pipeline_loader(pipeline: PipelineConfig, project_root_dir: Path) -> PipelineLoader[PipelineStep[TExecutionContext]]:
+        return PipelineLoader[PipelineStep[TExecutionContext]](pipeline, project_root_dir, RunCommandClassFactory())

{pypeline_runner-1.5.0.dist-info → pypeline_runner-1.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pypeline-runner
-Version: 1.5.0
+Version: 1.7.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 License: MIT
 Author: cuinixam

{pypeline_runner-1.5.0.dist-info → pypeline_runner-1.7.0.dist-info}/RECORD

@@ -1,11 +1,11 @@
-pypeline/__init__.py,sha256=X9pTeGHIYZtDOmfb9pnn2DEqWZMAyaPlqdFwEhsuSvk,22
+pypeline/__init__.py,sha256=lEEx5lwinUby-YNl7w-VFGUcjcibsJ9EuzgJ2W7ocrU,22
 pypeline/__run.py,sha256=TCdaX05Qm3g8T4QYryKB25Xxf0L5Km7hFOHe1mK9vI0,350
 pypeline/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/domain/artifacts.py,sha256=qXshnk9umi0AVGV4m5iEiy_MQ5Ad2LDZwI8OULU-qMk,1355
+pypeline/domain/artifacts.py,sha256=5k7cVfHhLmvWXNuHKxXb9ca4Lxu0JytGQqazENCeKEU,1404
 pypeline/domain/config.py,sha256=AlavAaz5hSxa6yaKYnj-x71ClhOtA41yv5Qf2JIE47k,1650
 pypeline/domain/execution_context.py,sha256=ho-WvCVRMUfYo1532eQYabXCHtXDgvSNUkX8S3Cr7Xo,1278
-pypeline/domain/pipeline.py,sha256=mO5tc18nxiAtnLs_MR56u7PAoTJDMZ_KVkCqMzV-YG8,3916
-pypeline/domain/project_slurper.py,sha256=YCho7V1BHjFmC_foxHFaWX8c_VbMJ16XEB4CQBlMrhc,894
+pypeline/domain/pipeline.py,sha256=2BsN2lw2znUxLH--Novyqe6SubVKs6XeHQSQf9yxirw,7788
+pypeline/domain/project_slurper.py,sha256=e3BLV88GvfW3efh0agUWKqMk3oWnL602P5u9jER_o9U,971
 pypeline/kickstart/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/kickstart/create.py,sha256=dWqSdDsqNh7_AnNEsJzsmheR2-xyO6rpUwF-7AwzCYY,2188
 pypeline/kickstart/templates/project/.gitignore,sha256=y8GJoVvRPez1LBokf1NaDOt2X1XtGwKFMF5yjA8AVS0,24
@@ -16,15 +16,15 @@ pypeline/kickstart/templates/project/pypeline.yaml,sha256=EV5Tnu3H33gMT3Ov0t14-j
 pypeline/kickstart/templates/project/pyproject.toml,sha256=yc6RCo-bUo1PXF91XfM-dButgfxU16Uud34NidgJ0zQ,225
 pypeline/kickstart/templates/project/scoopfile.json,sha256=DcfZ8jYf9hmPHM-AWwnPKQJCzRG3fCuYtMeoY01nkag,219
 pypeline/kickstart/templates/project/steps/my_step.py,sha256=_zx01qAVuwn6IMPBUBwKY-IBjS9Gs2m-d51L9sayGug,733
-pypeline/main.py,sha256=GtuOgB9OeNFgbWLHZux80fppYQVwMYNFZoZhDIlJW9c,3457
+pypeline/main.py,sha256=54_-aINmJY6IILSp6swL2kYqNoBQqJw4W2duxY7gcQ4,3607
 pypeline/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/pypeline.py,sha256=fHGocevkWywVVTOkHJ0YSPBKjDwyJ8Zjs37wP4eMAO0,9209
+pypeline/pypeline.py,sha256=FHGS2iNtiuiM4dZDHCbeL-UW1apdCtZeFzl1xZW9qBw,6482
 pypeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/steps/create_venv.py,sha256=vTPSA0gGGzq_QhI1jAgsGchS2s8_ZpEtnWcb0uL8BHE,1934
 pypeline/steps/scoop_install.py,sha256=_YdoCMXLON0eIwck8PJOcNhayx_ka1krBAidw_oRuFE,3373
 pypeline/steps/west_install.py,sha256=hPyr28ksdKsQ0tv0gMNytzupgk1IgjN9CpmaBdX5zps,1947
-pypeline_runner-1.5.0.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
-pypeline_runner-1.5.0.dist-info/METADATA,sha256=BBR8eGiEfCmhMBS-Dgn6lmx7RRYuWUbxitVehIHRMOo,7154
-pypeline_runner-1.5.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-pypeline_runner-1.5.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
-pypeline_runner-1.5.0.dist-info/RECORD,,
+pypeline_runner-1.7.0.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
+pypeline_runner-1.7.0.dist-info/METADATA,sha256=O-Y7JgczzWmfE0m2W7CRH23JJmNDNTrCdxc0P0ZLpkI,7154
+pypeline_runner-1.7.0.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
+pypeline_runner-1.7.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
+pypeline_runner-1.7.0.dist-info/RECORD,,

{pypeline_runner-1.5.0.dist-info → pypeline_runner-1.7.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.0.1
+Generator: poetry-core 2.1.0
 Root-Is-Purelib: true
 Tag: py3-none-any