pypeline-runner 0.3.1__tar.gz → 1.0.0__tar.gz

{pypeline_runner-0.3.1 → pypeline_runner-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pypeline-runner
-Version: 0.3.1
+Version: 1.0.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 Home-page: https://github.com/cuinixam/pypeline
 License: MIT

{pypeline_runner-0.3.1 → pypeline_runner-1.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pypeline-runner"
-version = "0.3.1"
+version = "1.0.0"
 description = "Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines)."
 authors = ["cuinixam <me@cuinixam.com>"]
 license = "MIT"

pypeline_runner-1.0.0/src/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.0.0"

{pypeline_runner-0.3.1 → pypeline_runner-1.0.0}/src/pypeline/domain/pipeline.py

@@ -4,11 +4,13 @@ from pathlib import Path
 from typing import (
     Any,
     Dict,
+    Generic,
     List,
     Optional,
     OrderedDict,
     Type,
     TypeAlias,
+    TypeVar,
 )
 
 from mashumaro import DataClassDictMixin
@@ -39,25 +41,39 @@ class PipelineStepConfig(DataClassDictMixin):
 
 PipelineConfig: TypeAlias = OrderedDict[str, List[PipelineStepConfig]]
 
+TExecutionContext = TypeVar("TExecutionContext", bound=ExecutionContext)
 
-class PipelineStep(Runnable):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+
+class PipelineStep(Generic[TExecutionContext], Runnable):
+    """One can create subclasses of PipelineStep that specify the type of ExecutionContext they require."""
+
+    def __init__(self, execution_context: TExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
         super().__init__(self.get_needs_dependency_management())
         self.execution_context = execution_context
-        self.output_dir = output_dir
+        self.group_name = group_name
         self.config = config
         self.project_root_dir = self.execution_context.project_root_dir
 
+    @property
+    def output_dir(self) -> Path:
+        return self.execution_context.create_artifacts_locator().build_dir.joinpath(self.group_name)
+
     @abstractmethod
     def update_execution_context(self) -> None:
+        """
+        Even if the step does not need to run ( because it is not outdated ), it can still update the execution context.
+
+        A typical use case is for steps installing software that need to provide the install directories in the execution context even if all tools are already installed.
+        """
         pass
 
     def get_needs_dependency_management(self) -> bool:
+        """If false, the step executor will not check for outdated dependencies. This is useful for steps consisting of command lines which shall always run."""
         return True
 
 
-class PipelineStepReference:
-    def __init__(self, group_name: str, _class: Type[PipelineStep], config: Optional[Dict[str, Any]] = None) -> None:
+class PipelineStepReference(Generic[TExecutionContext]):
+    def __init__(self, group_name: str, _class: Type[PipelineStep[TExecutionContext]], config: Optional[Dict[str, Any]] = None) -> None:
         self.group_name = group_name
         self._class = _class
         self.config = config

{pypeline_runner-0.3.1 → pypeline_runner-1.0.0}/src/pypeline/kickstart/templates/project/steps/my_step.py

@@ -3,10 +3,11 @@ from typing import List
 
 from py_app_dev.core.logging import logger
 
+from pypeline.domain.execution_context import ExecutionContext
 from pypeline.domain.pipeline import PipelineStep
 
 
-class MyStep(PipelineStep):
+class MyStep(PipelineStep[ExecutionContext]):
     def run(self) -> None:
         logger.info(f"Run {self.get_name()} found install dirs:")
         for install_dir in self.execution_context.install_dirs:

{pypeline_runner-0.3.1 → pypeline_runner-1.0.0}/src/pypeline/main.py

@@ -7,6 +7,7 @@ from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.logging import logger, setup_logger, time_it
 
 from pypeline import __version__
+from pypeline.domain.execution_context import ExecutionContext
 from pypeline.domain.project_slurper import ProjectSlurper
 from pypeline.kickstart.create import KickstartProject
 from pypeline.pypeline import PipelineScheduler, PipelineStepsExecutor
@@ -76,14 +77,14 @@ def run(
     if not project_slurper.pipeline:
         raise UserNotificationException("No pipeline found in the configuration.")
     # Schedule the steps to run
-    steps_references = PipelineScheduler(project_slurper.pipeline, project_dir).get_steps_to_run(step, single)
+    steps_references = PipelineScheduler[ExecutionContext](project_slurper.pipeline, project_dir).get_steps_to_run(step, single)
     if not steps_references:
         if step:
             raise UserNotificationException(f"Step '{step}' not found in the pipeline.")
         logger.info("No steps to run.")
         return
 
-    PipelineStepsExecutor(project_slurper.artifacts_locator, steps_references, force_run, dry_run).run()
+    PipelineStepsExecutor[ExecutionContext](ExecutionContext(project_dir), steps_references, force_run, dry_run).run()
 
 
 def main() -> None:

{pypeline_runner-0.3.1 → pypeline_runner-1.0.0}/src/pypeline/pypeline.py

@@ -4,9 +4,11 @@ from pathlib import Path
 from typing import (
     Any,
     Dict,
+    Generic,
     List,
     Optional,
     Type,
+    cast,
 )
 
 from py_app_dev.core.exceptions import UserNotificationException
@@ -15,10 +17,10 @@ from py_app_dev.core.runnable import Executor
 
 from .domain.artifacts import ProjectArtifactsLocator
 from .domain.execution_context import ExecutionContext
-from .domain.pipeline import PipelineConfig, PipelineStep, PipelineStepConfig, PipelineStepReference
+from .domain.pipeline import PipelineConfig, PipelineStep, PipelineStepConfig, PipelineStepReference, TExecutionContext
 
 
-class PipelineLoader:
+class PipelineLoader(Generic[TExecutionContext]):
     """
     Loads pipeline steps from a pipeline configuration.
 
@@ -31,7 +33,7 @@ class PipelineLoader:
         self.pipeline_config = pipeline_config
         self.project_root_dir = project_root_dir
 
-    def load_steps_references(self) -> List[PipelineStepReference]:
+    def load_steps_references(self) -> List[PipelineStepReference[TExecutionContext]]:
         result = []
         for group_name, steps_config in self.pipeline_config.items():
             result.extend(self._load_steps(group_name, steps_config, self.project_root_dir))
@@ -42,7 +44,7 @@ class PipelineLoader:
         group_name: str,
         steps_config: List[PipelineStepConfig],
         project_root_dir: Path,
-    ) -> List[PipelineStepReference]:
+    ) -> List[PipelineStepReference[TExecutionContext]]:
         result = []
         for step_config in steps_config:
             step_class_name = step_config.class_name or step_config.step
@@ -54,11 +56,11 @@ class PipelineLoader:
                 step_class = PipelineLoader._create_run_command_step_class(step_config.run, step_class_name)
             else:
                 raise UserNotificationException(f"Step '{step_class_name}' has no 'module' nor 'file' nor `run` defined." " Please check your pipeline configuration.")
-            result.append(PipelineStepReference(group_name, step_class, step_config.config))
+            result.append(PipelineStepReference[TExecutionContext](group_name, cast(Type[PipelineStep[TExecutionContext]], step_class), step_config.config))
         return result
 
     @staticmethod
-    def _load_user_step(python_file: Path, step_class_name: str) -> Type[PipelineStep]:
+    def _load_user_step(python_file: Path, step_class_name: str) -> Type[PipelineStep[ExecutionContext]]:
         # Create a module specification from the file path
         spec = spec_from_file_location(f"user__{step_class_name}", python_file)
         if spec and spec.loader:
@@ -73,7 +75,7 @@ class PipelineLoader:
         raise UserNotificationException(f"Could not load file '{python_file}'." " Please check the file for any errors.")
 
     @staticmethod
-    def _load_module_step(module_name: str, step_class_name: str) -> Type[PipelineStep]:
+    def _load_module_step(module_name: str, step_class_name: str) -> Type[PipelineStep[ExecutionContext]]:
         try:
             module = importlib.import_module(module_name)
             step_class = getattr(module, step_class_name)
@@ -84,14 +86,14 @@ class PipelineLoader:
         return step_class
 
     @staticmethod
-    def _create_run_command_step_class(command: str, name: str) -> Type[PipelineStep]:
+    def _create_run_command_step_class(command: str, name: str) -> Type[PipelineStep[ExecutionContext]]:
         """Dynamically creates a step class for a given command."""
 
-        class TmpDynamicRunCommandStep(PipelineStep):
+        class TmpDynamicRunCommandStep(PipelineStep[ExecutionContext]):
             """A simple step that runs a command."""
 
-            def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
-                super().__init__(execution_context, output_dir, config)
+            def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
+                super().__init__(execution_context, group_name, config)
                 self.command = command
                 self.name = name
 
@@ -122,29 +124,31 @@ class PipelineLoader:
         return type(name, (TmpDynamicRunCommandStep,), {})
 
 
-class PipelineStepsExecutor:
+class PipelineStepsExecutor(Generic[TExecutionContext]):
     """Executes a list of pipeline steps sequentially."""
 
     def __init__(
         self,
-        artifacts_locator: ProjectArtifactsLocator,
-        steps_references: List[PipelineStepReference],
+        execution_context: TExecutionContext,
+        steps_references: List[PipelineStepReference[TExecutionContext]],
         force_run: bool = False,
         dry_run: bool = False,
     ) -> None:
         self.logger = logger.bind()
-        self.artifacts_locator = artifacts_locator
+        self.execution_context = execution_context
         self.steps_references = steps_references
         self.force_run = force_run
         self.dry_run = dry_run
 
+    @property
+    def artifacts_locator(self) -> ProjectArtifactsLocator:
+        return self.execution_context.create_artifacts_locator()
+
     def run(self) -> None:
-        execution_context = ExecutionContext(project_root_dir=self.artifacts_locator.project_root_dir, install_dirs=[])
         for step_reference in self.steps_references:
-            step_output_dir = self.artifacts_locator.build_dir / step_reference.group_name
+            step = step_reference._class(self.execution_context, step_reference.group_name, step_reference.config)
             # Create the step output directory, to make sure that files can be created.
-            step_output_dir.mkdir(parents=True, exist_ok=True)
-            step = step_reference._class(execution_context, step_output_dir, step_reference.config)
+            step.output_dir.mkdir(parents=True, exist_ok=True)
             # Execute the step is necessary. If the step is not dirty, it will not be executed
             Executor(step.output_dir, self.force_run, self.dry_run).execute(step)
             # Independent if the step was executed or not, every step shall update the context
@@ -153,7 +157,7 @@ class PipelineStepsExecutor:
         return
 
 
-class PipelineScheduler:
+class PipelineScheduler(Generic[TExecutionContext]):
     """
     Schedules which steps must be executed based on the provided configuration.
 
@@ -168,16 +172,16 @@ class PipelineScheduler:
         self.project_root_dir = project_root_dir
         self.logger = logger.bind()
 
-    def get_steps_to_run(self, step_name: Optional[str] = None, single: bool = False) -> List[PipelineStepReference]:
-        pipeline_loader = PipelineLoader(self.pipeline, self.project_root_dir)
+    def get_steps_to_run(self, step_name: Optional[str] = None, single: bool = False) -> List[PipelineStepReference[TExecutionContext]]:
+        pipeline_loader = PipelineLoader[TExecutionContext](self.pipeline, self.project_root_dir)
         return self.filter_steps_references(pipeline_loader.load_steps_references(), step_name, single)
 
     @staticmethod
     def filter_steps_references(
-        steps_references: List[PipelineStepReference],
+        steps_references: List[PipelineStepReference[TExecutionContext]],
         step_name: Optional[str],
         single: Optional[bool],
-    ) -> List[PipelineStepReference]:
+    ) -> List[PipelineStepReference[TExecutionContext]]:
         if step_name:
             step_reference = next((step for step in steps_references if step.name == step_name), None)
             if not step_reference:

{pypeline_runner-0.3.1 → pypeline_runner-1.0.0}/src/pypeline/steps/create_venv.py

@@ -15,9 +15,9 @@ class CreateVEnvConfig(DataClassDictMixin):
     bootstrap_script: str = "bootstrap.py"
 
 
-class CreateVEnv(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
-        super().__init__(execution_context, output_dir, config)
+class CreateVEnv(PipelineStep[ExecutionContext]):
+    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, group_name, config)
         self.logger = logger.bind()
         self.logger = logger.bind()
 

{pypeline_runner-0.3.1 → pypeline_runner-1.0.0}/src/pypeline/steps/scoop_install.py

@@ -45,9 +45,9 @@ def create_scoop_wrapper() -> ScoopWrapper:
     return ScoopWrapper()
 
 
-class ScoopInstall(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
-        super().__init__(execution_context, output_dir, config)
+class ScoopInstall(PipelineStep[ExecutionContext]):
+    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, group_name, config)
         self.logger = logger.bind()
         self.execution_info = ScoopInstallExecutionInfo([])
         # One needs to keep track of the installed apps to get the required paths

{pypeline_runner-0.3.1 → pypeline_runner-1.0.0}/src/pypeline/steps/west_install.py

@@ -8,9 +8,9 @@ from ..domain.execution_context import ExecutionContext
 from ..domain.pipeline import PipelineStep
 
 
-class WestInstall(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
-        super().__init__(execution_context, output_dir, config)
+class WestInstall(PipelineStep[ExecutionContext]):
+    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, group_name, config)
         self.logger = logger.bind()
         self.artifacts_locator = execution_context.create_artifacts_locator()
 

pypeline_runner-0.3.1/src/pypeline/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.1"