pypeline-runner 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "0.2.1"
+__version__ = "0.3.0"

pypeline/domain/config.py

@@ -5,10 +5,11 @@ from typing import Any, Dict, Optional
 import yaml
 from mashumaro import DataClassDictMixin
 from py_app_dev.core.exceptions import UserNotificationException
-from py_app_dev.core.pipeline import PipelineConfig
 from yaml.parser import ParserError
 from yaml.scanner import ScannerError
 
+from .pipeline import PipelineConfig
+
 
 @dataclass
 class ProjectConfig(DataClassDictMixin):

pypeline/domain/pipeline.py

@@ -1,14 +1,48 @@
 from abc import abstractmethod
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, Optional, Type
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    OrderedDict,
+    Type,
+    TypeAlias,
+)
 
+from mashumaro import DataClassDictMixin
 from py_app_dev.core.runnable import Runnable
 
 from .execution_context import ExecutionContext
 
 
+@dataclass
+class PipelineStepConfig(DataClassDictMixin):
+    #: Step name or class name if file is not specified
+    step: str
+    #: Path to file with step class
+    file: Optional[str] = None
+    #: Python module with step class
+    module: Optional[str] = None
+    #: Step class name
+    class_name: Optional[str] = None
+    #: Command to run. For simple steps that don't need a class
+    run: Optional[str] = None
+    #: Step description
+    description: Optional[str] = None
+    #: Step timeout in seconds
+    timeout_sec: Optional[int] = None
+    #: Custom step configuration
+    config: Optional[Dict[str, Any]] = None
+
+
+PipelineConfig: TypeAlias = OrderedDict[str, List[PipelineStepConfig]]
+
+
 class PipelineStep(Runnable):
     def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(self.get_needs_dependency_management())
         self.execution_context = execution_context
         self.output_dir = output_dir
         self.config = config
@@ -18,6 +52,9 @@ class PipelineStep(Runnable):
     def update_execution_context(self) -> None:
         pass
 
+    def get_needs_dependency_management(self) -> bool:
+        return True
+
 
 class PipelineStepReference:
     def __init__(self, group_name: str, _class: Type[PipelineStep], config: Optional[Dict[str, Any]] = None) -> None:

pypeline/main.py

@@ -86,10 +86,14 @@ def run(
     PipelineStepsExecutor(project_slurper.artifacts_locator, steps_references, force_run, dry_run).run()
 
 
-if __name__ == "__main__":
+def main() -> None:
     try:
         setup_logger()
         app()
     except UserNotificationException as e:
         logger.error(f"{e}")
         sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

pypeline/pypeline.py

@@ -1,14 +1,21 @@
+import importlib
+from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
-from typing import List, Optional
-
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Type,
+)
+
+from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.logging import logger
-from py_app_dev.core.pipeline import PipelineConfig
-from py_app_dev.core.pipeline import PipelineLoader as GenericPipelineLoader
 from py_app_dev.core.runnable import Executor
 
 from .domain.artifacts import ProjectArtifactsLocator
 from .domain.execution_context import ExecutionContext
-from .domain.pipeline import PipelineStep, PipelineStepReference
+from .domain.pipeline import PipelineConfig, PipelineStep, PipelineStepConfig, PipelineStepReference
 
 
 class PipelineLoader:
@@ -23,10 +30,95 @@ class PipelineLoader:
     def __init__(self, pipeline_config: PipelineConfig, project_root_dir: Path) -> None:
         self.pipeline_config = pipeline_config
         self.project_root_dir = project_root_dir
-        self._loader = GenericPipelineLoader[PipelineStep](self.pipeline_config, self.project_root_dir)
 
     def load_steps_references(self) -> List[PipelineStepReference]:
-        return [PipelineStepReference(step_reference.group_name, step_reference._class, step_reference.config) for step_reference in self._loader.load_steps()]
+        result = []
+        for group_name, steps_config in self.pipeline_config.items():
+            result.extend(self._load_steps(group_name, steps_config, self.project_root_dir))
+        return result
+
+    @staticmethod
+    def _load_steps(
+        group_name: str,
+        steps_config: List[PipelineStepConfig],
+        project_root_dir: Path,
+    ) -> List[PipelineStepReference]:
+        result = []
+        for step_config in steps_config:
+            step_class_name = step_config.class_name or step_config.step
+            if step_config.module:
+                step_class = PipelineLoader._load_module_step(step_config.module, step_class_name)
+            elif step_config.file:
+                step_class = PipelineLoader._load_user_step(project_root_dir.joinpath(step_config.file), step_class_name)
+            elif step_config.run:
+                step_class = PipelineLoader._create_run_command_step_class(step_config.run, step_class_name)
+            else:
+                raise UserNotificationException(f"Step '{step_class_name}' has no 'module' nor 'file' nor `run` defined." " Please check your pipeline configuration.")
+            result.append(PipelineStepReference(group_name, step_class, step_config.config))
+        return result
+
+    @staticmethod
+    def _load_user_step(python_file: Path, step_class_name: str) -> Type[PipelineStep]:
+        # Create a module specification from the file path
+        spec = spec_from_file_location(f"user__{step_class_name}", python_file)
+        if spec and spec.loader:
+            step_module = module_from_spec(spec)
+            # Import the module
+            spec.loader.exec_module(step_module)
+            try:
+                step_class = getattr(step_module, step_class_name)
+            except AttributeError:
+                raise UserNotificationException(f"Could not load class '{step_class_name}' from file '{python_file}'." " Please check your pipeline configuration.") from None
+            return step_class
+        raise UserNotificationException(f"Could not load file '{python_file}'." " Please check the file for any errors.")
+
+    @staticmethod
+    def _load_module_step(module_name: str, step_class_name: str) -> Type[PipelineStep]:
+        try:
+            module = importlib.import_module(module_name)
+            step_class = getattr(module, step_class_name)
+        except ImportError:
+            raise UserNotificationException(f"Could not load module '{module_name}'. Please check your pipeline configuration.") from None
+        except AttributeError:
+            raise UserNotificationException(f"Could not load class '{step_class_name}' from module '{module_name}'." " Please check your pipeline configuration.") from None
+        return step_class
+
+    @staticmethod
+    def _create_run_command_step_class(command: str, name: str) -> Type[PipelineStep]:
+        """Dynamically creates a step class for a given command."""
+
+        class DynamicRunCommandStep(PipelineStep):
+            """A simple step that runs a command."""
+
+            def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+                super().__init__(execution_context, output_dir, config)
+                self.command = command
+                self.name = name
+
+            def get_needs_dependency_management(self) -> bool:
+                """A command step does not need dependency management."""
+                return False
+
+            def run(self) -> int:
+                self.execution_context.create_process_executor(
+                    [self.command],
+                    cwd=self.project_root_dir,
+                ).execute()
+                return 0
+
+            def get_name(self) -> str:
+                return self.name
+
+            def get_inputs(self) -> List[Path]:
+                return []
+
+            def get_outputs(self) -> List[Path]:
+                return []
+
+            def update_execution_context(self) -> None:
+                pass
+
+        return DynamicRunCommandStep
 
 
 class PipelineStepsExecutor:

{pypeline_runner-0.2.1.dist-info → pypeline_runner-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pypeline-runner
-Version: 0.2.1
+Version: 0.3.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 Home-page: https://github.com/cuinixam/pypeline
 License: MIT
@@ -17,9 +17,9 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries
-Requires-Dist: py-app-dev (>=2.1.1,<3.0.0)
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: typer[all] (>=0.12.0,<0.13.0)
+Requires-Dist: py-app-dev (>=2.2,<3.0)
+Requires-Dist: pyyaml (>=6.0,<7.0)
+Requires-Dist: typer[all] (>=0.12,<0.13)
 Project-URL: Bug Tracker, https://github.com/cuinixam/pypeline/issues
 Project-URL: Changelog, https://github.com/cuinixam/pypeline/blob/main/CHANGELOG.md
 Project-URL: Documentation, https://pypeline-runner.readthedocs.io

{pypeline_runner-0.2.1.dist-info → pypeline_runner-0.3.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
-pypeline/__init__.py,sha256=HfjVOrpTnmZ-xVFCYSVmX50EXaBQeJteUHG-PD6iQs8,22
+pypeline/__init__.py,sha256=VrXpHDu3erkzwl_WXrqINBm9xWkcyUy53IQOj042dOs,22
 pypeline/__run.py,sha256=TCdaX05Qm3g8T4QYryKB25Xxf0L5Km7hFOHe1mK9vI0,350
 pypeline/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/domain/artifacts.py,sha256=qXshnk9umi0AVGV4m5iEiy_MQ5Ad2LDZwI8OULU-qMk,1355
-pypeline/domain/config.py,sha256=e2DdbC6Gv4qJzjYaNnX90f2V-uYSHC5F6bViBTe80RE,1663
+pypeline/domain/config.py,sha256=AlavAaz5hSxa6yaKYnj-x71ClhOtA41yv5Qf2JIE47k,1650
 pypeline/domain/execution_context.py,sha256=0zc3OgXeIMDpgWWYMaDGub7fY5urLLR79yuCaXVxoTQ,1101
-pypeline/domain/pipeline.py,sha256=gN2PUdBEYl1D534_XZ-bLwdhBlDr48SR_A8mdJ3ZzJg,980
+pypeline/domain/pipeline.py,sha256=TbnM1JI1AFEdDfMSfdDVi2DZjFAAWBRMzIycOsui4Vs,1997
 pypeline/domain/project_slurper.py,sha256=YCho7V1BHjFmC_foxHFaWX8c_VbMJ16XEB4CQBlMrhc,894
 pypeline/kickstart/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/kickstart/create.py,sha256=Gz2rVfhoQCKgYWlJoefh5Azcgamc9JJtiNsP1DTKdQo,2235
@@ -17,15 +17,15 @@ pypeline/kickstart/templates/project/pypeline.yaml,sha256=i9YSqpS2o-_Bzl69fvdOFV
 pypeline/kickstart/templates/project/pyproject.toml,sha256=yc6RCo-bUo1PXF91XfM-dButgfxU16Uud34NidgJ0zQ,225
 pypeline/kickstart/templates/project/scoopfile.json,sha256=DcfZ8jYf9hmPHM-AWwnPKQJCzRG3fCuYtMeoY01nkag,219
 pypeline/kickstart/templates/project/steps/my_step.py,sha256=YNRdb7ofqo3PxMQe8Vhtf7K9eZclv-6J0IwMrd3E70w,651
-pypeline/main.py,sha256=JLY75JHoQ1Q8F-Yyw6_s41n1CFjKh0971tDiI3drUUY,3227
+pypeline/main.py,sha256=gwzbTUu-9_LXHpMDUDJU4gUInA4wGRmngO0VaTAqptk,3260
 pypeline/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/pypeline.py,sha256=3coRh9KGwR--yDK88ltXisE2nAR8BNUMQ21YHDjoKPY,4359
+pypeline/pypeline.py,sha256=YiKZ8IH1t1FsTqJ45dVBkr2I2XRN2Su0ChlJFbUIqBc,8335
 pypeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/steps/create_venv.py,sha256=od1zLVPhNdN-sWqEqYMkX19c5IEt4lIIOcASBeoqUUw,1954
 pypeline/steps/scoop_install.py,sha256=MmoC4Yy1lOhHrfKBrgJKHMcRUNPbRztzUbSgIlazer8,3356
 pypeline/steps/west_install.py,sha256=J1p-62owTDOY6bVTGCr5d9YFOL8DeRUPTnq0TbiaWlA,1930
-pypeline_runner-0.2.1.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
-pypeline_runner-0.2.1.dist-info/METADATA,sha256=-x52h84lUMHd33QQxo7RzR5A-fy7OihSdmqpFzrwGds,7168
-pypeline_runner-0.2.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pypeline_runner-0.2.1.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
-pypeline_runner-0.2.1.dist-info/RECORD,,
+pypeline_runner-0.3.0.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
+pypeline_runner-0.3.0.dist-info/METADATA,sha256=f0qMzL-SHEe-n5BJw1DHJ-9WovmEmIaBh1X4bUMaYiY,7156
+pypeline_runner-0.3.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pypeline_runner-0.3.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
+pypeline_runner-0.3.0.dist-info/RECORD,,