pypeline-runner 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "0.3.0"
+__version__ = "1.0.0"

pypeline/domain/pipeline.py

@@ -4,11 +4,13 @@ from pathlib import Path
 from typing import (
     Any,
     Dict,
+    Generic,
     List,
     Optional,
     OrderedDict,
     Type,
     TypeAlias,
+    TypeVar,
 )
 
 from mashumaro import DataClassDictMixin
@@ -39,25 +41,39 @@ class PipelineStepConfig(DataClassDictMixin):
 
 PipelineConfig: TypeAlias = OrderedDict[str, List[PipelineStepConfig]]
 
+TExecutionContext = TypeVar("TExecutionContext", bound=ExecutionContext)
 
-class PipelineStep(Runnable):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+
+class PipelineStep(Generic[TExecutionContext], Runnable):
+    """One can create subclasses of PipelineStep that specify the type of ExecutionContext they require."""
+
+    def __init__(self, execution_context: TExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
         super().__init__(self.get_needs_dependency_management())
         self.execution_context = execution_context
-        self.output_dir = output_dir
+        self.group_name = group_name
         self.config = config
         self.project_root_dir = self.execution_context.project_root_dir
 
+    @property
+    def output_dir(self) -> Path:
+        return self.execution_context.create_artifacts_locator().build_dir.joinpath(self.group_name)
+
     @abstractmethod
     def update_execution_context(self) -> None:
+        """
+        Even if the step does not need to run ( because it is not outdated ), it can still update the execution context.
+
+        A typical use case is for steps installing software that need to provide the install directories in the execution context even if all tools are already installed.
+        """
         pass
 
     def get_needs_dependency_management(self) -> bool:
+        """If false, the step executor will not check for outdated dependencies. This is useful for steps consisting of command lines which shall always run."""
         return True
 
 
-class PipelineStepReference:
-    def __init__(self, group_name: str, _class: Type[PipelineStep], config: Optional[Dict[str, Any]] = None) -> None:
+class PipelineStepReference(Generic[TExecutionContext]):
+    def __init__(self, group_name: str, _class: Type[PipelineStep[TExecutionContext]], config: Optional[Dict[str, Any]] = None) -> None:
         self.group_name = group_name
         self._class = _class
         self.config = config

pypeline/kickstart/templates/project/steps/my_step.py

@@ -3,10 +3,11 @@ from typing import List
 
 from py_app_dev.core.logging import logger
 
+from pypeline.domain.execution_context import ExecutionContext
 from pypeline.domain.pipeline import PipelineStep
 
 
-class MyStep(PipelineStep):
+class MyStep(PipelineStep[ExecutionContext]):
     def run(self) -> None:
         logger.info(f"Run {self.get_name()} found install dirs:")
         for install_dir in self.execution_context.install_dirs:

pypeline/main.py

@@ -7,6 +7,7 @@ from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.logging import logger, setup_logger, time_it
 
 from pypeline import __version__
+from pypeline.domain.execution_context import ExecutionContext
 from pypeline.domain.project_slurper import ProjectSlurper
 from pypeline.kickstart.create import KickstartProject
 from pypeline.pypeline import PipelineScheduler, PipelineStepsExecutor
@@ -76,14 +77,14 @@ def run(
     if not project_slurper.pipeline:
         raise UserNotificationException("No pipeline found in the configuration.")
     # Schedule the steps to run
-    steps_references = PipelineScheduler(project_slurper.pipeline, project_dir).get_steps_to_run(step, single)
+    steps_references = PipelineScheduler[ExecutionContext](project_slurper.pipeline, project_dir).get_steps_to_run(step, single)
     if not steps_references:
         if step:
             raise UserNotificationException(f"Step '{step}' not found in the pipeline.")
         logger.info("No steps to run.")
         return
 
-    PipelineStepsExecutor(project_slurper.artifacts_locator, steps_references, force_run, dry_run).run()
+    PipelineStepsExecutor[ExecutionContext](ExecutionContext(project_dir), steps_references, force_run, dry_run).run()
 
 
 def main() -> None:

pypeline/pypeline.py

@@ -4,9 +4,11 @@ from pathlib import Path
 from typing import (
     Any,
     Dict,
+    Generic,
     List,
     Optional,
     Type,
+    cast,
 )
 
 from py_app_dev.core.exceptions import UserNotificationException
@@ -15,10 +17,10 @@ from py_app_dev.core.runnable import Executor
 
 from .domain.artifacts import ProjectArtifactsLocator
 from .domain.execution_context import ExecutionContext
-from .domain.pipeline import PipelineConfig, PipelineStep, PipelineStepConfig, PipelineStepReference
+from .domain.pipeline import PipelineConfig, PipelineStep, PipelineStepConfig, PipelineStepReference, TExecutionContext
 
 
-class PipelineLoader:
+class PipelineLoader(Generic[TExecutionContext]):
     """
     Loads pipeline steps from a pipeline configuration.
 
@@ -31,7 +33,7 @@ class PipelineLoader:
         self.pipeline_config = pipeline_config
         self.project_root_dir = project_root_dir
 
-    def load_steps_references(self) -> List[PipelineStepReference]:
+    def load_steps_references(self) -> List[PipelineStepReference[TExecutionContext]]:
         result = []
         for group_name, steps_config in self.pipeline_config.items():
             result.extend(self._load_steps(group_name, steps_config, self.project_root_dir))
@@ -42,7 +44,7 @@ class PipelineLoader:
         group_name: str,
         steps_config: List[PipelineStepConfig],
         project_root_dir: Path,
-    ) -> List[PipelineStepReference]:
+    ) -> List[PipelineStepReference[TExecutionContext]]:
         result = []
         for step_config in steps_config:
             step_class_name = step_config.class_name or step_config.step
@@ -54,11 +56,11 @@ class PipelineLoader:
                 step_class = PipelineLoader._create_run_command_step_class(step_config.run, step_class_name)
             else:
                 raise UserNotificationException(f"Step '{step_class_name}' has no 'module' nor 'file' nor `run` defined." " Please check your pipeline configuration.")
-            result.append(PipelineStepReference(group_name, step_class, step_config.config))
+            result.append(PipelineStepReference[TExecutionContext](group_name, cast(Type[PipelineStep[TExecutionContext]], step_class), step_config.config))
         return result
 
     @staticmethod
-    def _load_user_step(python_file: Path, step_class_name: str) -> Type[PipelineStep]:
+    def _load_user_step(python_file: Path, step_class_name: str) -> Type[PipelineStep[ExecutionContext]]:
         # Create a module specification from the file path
         spec = spec_from_file_location(f"user__{step_class_name}", python_file)
         if spec and spec.loader:
@@ -73,7 +75,7 @@ class PipelineLoader:
         raise UserNotificationException(f"Could not load file '{python_file}'." " Please check the file for any errors.")
 
     @staticmethod
-    def _load_module_step(module_name: str, step_class_name: str) -> Type[PipelineStep]:
+    def _load_module_step(module_name: str, step_class_name: str) -> Type[PipelineStep[ExecutionContext]]:
         try:
             module = importlib.import_module(module_name)
             step_class = getattr(module, step_class_name)
@@ -84,14 +86,14 @@ class PipelineLoader:
         return step_class
 
     @staticmethod
-    def _create_run_command_step_class(command: str, name: str) -> Type[PipelineStep]:
+    def _create_run_command_step_class(command: str, name: str) -> Type[PipelineStep[ExecutionContext]]:
         """Dynamically creates a step class for a given command."""
 
-        class DynamicRunCommandStep(PipelineStep):
+        class TmpDynamicRunCommandStep(PipelineStep[ExecutionContext]):
             """A simple step that runs a command."""
 
-            def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
-                super().__init__(execution_context, output_dir, config)
+            def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
+                super().__init__(execution_context, group_name, config)
                 self.command = command
                 self.name = name
 
@@ -118,32 +120,35 @@ class PipelineLoader:
             def update_execution_context(self) -> None:
                 pass
 
-        return DynamicRunCommandStep
+        # Dynamically create the class with the given name
+        return type(name, (TmpDynamicRunCommandStep,), {})
 
 
-class PipelineStepsExecutor:
+class PipelineStepsExecutor(Generic[TExecutionContext]):
     """Executes a list of pipeline steps sequentially."""
 
     def __init__(
         self,
-        artifacts_locator: ProjectArtifactsLocator,
-        steps_references: List[PipelineStepReference],
+        execution_context: TExecutionContext,
+        steps_references: List[PipelineStepReference[TExecutionContext]],
         force_run: bool = False,
         dry_run: bool = False,
     ) -> None:
         self.logger = logger.bind()
-        self.artifacts_locator = artifacts_locator
+        self.execution_context = execution_context
         self.steps_references = steps_references
         self.force_run = force_run
         self.dry_run = dry_run
 
+    @property
+    def artifacts_locator(self) -> ProjectArtifactsLocator:
+        return self.execution_context.create_artifacts_locator()
+
     def run(self) -> None:
-        execution_context = ExecutionContext(project_root_dir=self.artifacts_locator.project_root_dir, install_dirs=[])
         for step_reference in self.steps_references:
-            step_output_dir = self.artifacts_locator.build_dir / step_reference.group_name
+            step = step_reference._class(self.execution_context, step_reference.group_name, step_reference.config)
             # Create the step output directory, to make sure that files can be created.
-            step_output_dir.mkdir(parents=True, exist_ok=True)
-            step = step_reference._class(execution_context, step_output_dir, step_reference.config)
+            step.output_dir.mkdir(parents=True, exist_ok=True)
             # Execute the step is necessary. If the step is not dirty, it will not be executed
             Executor(step.output_dir, self.force_run, self.dry_run).execute(step)
             # Independent if the step was executed or not, every step shall update the context
@@ -152,7 +157,7 @@ class PipelineStepsExecutor:
         return
 
 
-class PipelineScheduler:
+class PipelineScheduler(Generic[TExecutionContext]):
     """
     Schedules which steps must be executed based on the provided configuration.
 
@@ -167,16 +172,16 @@ class PipelineScheduler:
         self.project_root_dir = project_root_dir
         self.logger = logger.bind()
 
-    def get_steps_to_run(self, step_name: Optional[str] = None, single: bool = False) -> List[PipelineStepReference]:
-        pipeline_loader = PipelineLoader(self.pipeline, self.project_root_dir)
+    def get_steps_to_run(self, step_name: Optional[str] = None, single: bool = False) -> List[PipelineStepReference[TExecutionContext]]:
+        pipeline_loader = PipelineLoader[TExecutionContext](self.pipeline, self.project_root_dir)
         return self.filter_steps_references(pipeline_loader.load_steps_references(), step_name, single)
 
     @staticmethod
     def filter_steps_references(
-        steps_references: List[PipelineStepReference],
+        steps_references: List[PipelineStepReference[TExecutionContext]],
         step_name: Optional[str],
         single: Optional[bool],
-    ) -> List[PipelineStepReference]:
+    ) -> List[PipelineStepReference[TExecutionContext]]:
         if step_name:
             step_reference = next((step for step in steps_references if step.name == step_name), None)
             if not step_reference:

pypeline/steps/create_venv.py

@@ -15,9 +15,9 @@ class CreateVEnvConfig(DataClassDictMixin):
     bootstrap_script: str = "bootstrap.py"
 
 
-class CreateVEnv(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
-        super().__init__(execution_context, output_dir, config)
+class CreateVEnv(PipelineStep[ExecutionContext]):
+    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, group_name, config)
         self.logger = logger.bind()
         self.logger = logger.bind()
 

pypeline/steps/scoop_install.py

@@ -45,9 +45,9 @@ def create_scoop_wrapper() -> ScoopWrapper:
     return ScoopWrapper()
 
 
-class ScoopInstall(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
-        super().__init__(execution_context, output_dir, config)
+class ScoopInstall(PipelineStep[ExecutionContext]):
+    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, group_name, config)
         self.logger = logger.bind()
         self.execution_info = ScoopInstallExecutionInfo([])
         # One needs to keep track of the installed apps to get the required paths

pypeline/steps/west_install.py

@@ -8,9 +8,9 @@ from ..domain.execution_context import ExecutionContext
 from ..domain.pipeline import PipelineStep
 
 
-class WestInstall(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
-        super().__init__(execution_context, output_dir, config)
+class WestInstall(PipelineStep[ExecutionContext]):
+    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, group_name, config)
         self.logger = logger.bind()
         self.artifacts_locator = execution_context.create_artifacts_locator()
 

{pypeline_runner-0.3.0.dist-info → pypeline_runner-1.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pypeline-runner
-Version: 0.3.0
+Version: 1.0.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 Home-page: https://github.com/cuinixam/pypeline
 License: MIT

{pypeline_runner-0.3.0.dist-info → pypeline_runner-1.0.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
-pypeline/__init__.py,sha256=VrXpHDu3erkzwl_WXrqINBm9xWkcyUy53IQOj042dOs,22
+pypeline/__init__.py,sha256=J-j-u0itpEFT6irdmWmixQqYMadNl1X91TxUmoiLHMI,22
 pypeline/__run.py,sha256=TCdaX05Qm3g8T4QYryKB25Xxf0L5Km7hFOHe1mK9vI0,350
 pypeline/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/domain/artifacts.py,sha256=qXshnk9umi0AVGV4m5iEiy_MQ5Ad2LDZwI8OULU-qMk,1355
 pypeline/domain/config.py,sha256=AlavAaz5hSxa6yaKYnj-x71ClhOtA41yv5Qf2JIE47k,1650
 pypeline/domain/execution_context.py,sha256=0zc3OgXeIMDpgWWYMaDGub7fY5urLLR79yuCaXVxoTQ,1101
-pypeline/domain/pipeline.py,sha256=TbnM1JI1AFEdDfMSfdDVi2DZjFAAWBRMzIycOsui4Vs,1997
+pypeline/domain/pipeline.py,sha256=PlNrRXBaLw3Eu62ytyH3sgbTmt5pKXC4l24GXXhCNII,2925
 pypeline/domain/project_slurper.py,sha256=YCho7V1BHjFmC_foxHFaWX8c_VbMJ16XEB4CQBlMrhc,894
 pypeline/kickstart/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/kickstart/create.py,sha256=Gz2rVfhoQCKgYWlJoefh5Azcgamc9JJtiNsP1DTKdQo,2235
@@ -16,16 +16,16 @@ pypeline/kickstart/templates/project/pypeline.ps1,sha256=tkiXrsMsNM43WJdmWJRhis3
 pypeline/kickstart/templates/project/pypeline.yaml,sha256=i9YSqpS2o-_Bzl69fvdOFVyYRMsDCtDy8usBpk-hqnE,219
 pypeline/kickstart/templates/project/pyproject.toml,sha256=yc6RCo-bUo1PXF91XfM-dButgfxU16Uud34NidgJ0zQ,225
 pypeline/kickstart/templates/project/scoopfile.json,sha256=DcfZ8jYf9hmPHM-AWwnPKQJCzRG3fCuYtMeoY01nkag,219
-pypeline/kickstart/templates/project/steps/my_step.py,sha256=YNRdb7ofqo3PxMQe8Vhtf7K9eZclv-6J0IwMrd3E70w,651
-pypeline/main.py,sha256=gwzbTUu-9_LXHpMDUDJU4gUInA4wGRmngO0VaTAqptk,3260
+pypeline/kickstart/templates/project/steps/my_step.py,sha256=_zx01qAVuwn6IMPBUBwKY-IBjS9Gs2m-d51L9sayGug,733
+pypeline/main.py,sha256=20gfpGlYWN3wlAifTkES4VY3lXDQjwKBYd8KkSr33EE,3355
 pypeline/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/pypeline.py,sha256=YiKZ8IH1t1FsTqJ45dVBkr2I2XRN2Su0ChlJFbUIqBc,8335
+pypeline/pypeline.py,sha256=GUEIPtK6TU9yt-izuIqbRUjbxM3JASoj8nNJ63mN388,8756
 pypeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/steps/create_venv.py,sha256=od1zLVPhNdN-sWqEqYMkX19c5IEt4lIIOcASBeoqUUw,1954
-pypeline/steps/scoop_install.py,sha256=MmoC4Yy1lOhHrfKBrgJKHMcRUNPbRztzUbSgIlazer8,3356
-pypeline/steps/west_install.py,sha256=J1p-62owTDOY6bVTGCr5d9YFOL8DeRUPTnq0TbiaWlA,1930
-pypeline_runner-0.3.0.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
-pypeline_runner-0.3.0.dist-info/METADATA,sha256=f0qMzL-SHEe-n5BJw1DHJ-9WovmEmIaBh1X4bUMaYiY,7156
-pypeline_runner-0.3.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pypeline_runner-0.3.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
-pypeline_runner-0.3.0.dist-info/RECORD,,
+pypeline/steps/create_venv.py,sha256=hXRklcYR38uUPWRDyhUjWu8WEPIaQPSQggM_ocwG7k8,1971
+pypeline/steps/scoop_install.py,sha256=_YdoCMXLON0eIwck8PJOcNhayx_ka1krBAidw_oRuFE,3373
+pypeline/steps/west_install.py,sha256=hPyr28ksdKsQ0tv0gMNytzupgk1IgjN9CpmaBdX5zps,1947
+pypeline_runner-1.0.0.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
+pypeline_runner-1.0.0.dist-info/METADATA,sha256=fDbeBXYd0Lt_7_euh4eG3kUhNC40PVWanF0jY8mysJg,7156
+pypeline_runner-1.0.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pypeline_runner-1.0.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
+pypeline_runner-1.0.0.dist-info/RECORD,,