pypeline-runner 0.1.1__tar.gz → 0.2.1__tar.gz

{pypeline_runner-0.1.1 → pypeline_runner-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pypeline-runner
-Version: 0.1.1
+Version: 0.2.1
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 Home-page: https://github.com/cuinixam/pypeline
 License: MIT
@@ -17,7 +17,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries
-Requires-Dist: py-app-dev (>=2.1.0,<3.0.0)
+Requires-Dist: py-app-dev (>=2.1.1,<3.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: typer[all] (>=0.12.0,<0.13.0)
 Project-URL: Bug Tracker, https://github.com/cuinixam/pypeline/issues
@@ -26,7 +26,7 @@ Project-URL: Documentation, https://pypeline-runner.readthedocs.io
 Project-URL: Repository, https://github.com/cuinixam/pypeline
 Description-Content-Type: text/markdown
 
-# Python Pipeline
+# Pypeline
 
 <p align="center">
   <a href="https://github.com/cuinixam/pypeline/actions/workflows/ci.yml?query=branch%3Amain">
@@ -51,14 +51,27 @@ Description-Content-Type: text/markdown
   </a>
 </p>
 <p align="center">
-  <a href="https://pypi.org/project/pypeline/">
-    <img src="https://img.shields.io/pypi/v/pypeline.svg?logo=python&logoColor=fff&style=flat-square" alt="PyPI Version">
+  <a href="https://pypi.org/project/pypeline-runner/">
+    <img src="https://img.shields.io/pypi/v/pypeline-runner.svg?logo=python&logoColor=fff&style=flat-square" alt="PyPI Version">
   </a>
-  <img src="https://img.shields.io/pypi/pyversions/pypeline.svg?style=flat-square&logo=python&amp;logoColor=fff" alt="Supported Python versions">
-  <img src="https://img.shields.io/pypi/l/pypeline.svg?style=flat-square" alt="License">
+  <img src="https://img.shields.io/pypi/pyversions/pypeline-runner.svg?style=flat-square&logo=python&amp;logoColor=fff" alt="Supported Python versions">
+  <img src="https://img.shields.io/pypi/l/pypeline-runner.svg?style=flat-square" alt="License">
 </p>
 
-Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
+Pypeline is a Python application designed to streamline and automate the software development lifecycle, particularly the pipeline execution processes across various environments such as GitHub and Jenkins.
+The primary motivation for developing Pypeline stemmed from the need to unify and simplify the creation of build, test, and deployment pipelines that are traditionally defined separately across these platforms using GitHub workflows (YAML) and Jenkins pipelines (Jenkinsfile).
+
+**Key Features**
+
+- **Unified Pipeline Definition**: Users can define their entire pipeline in a single YAML file, eliminating the need to switch between different syntaxes and configurations for different CI/CD tools.
+
+- **Extensibility**: Pypeline supports execution steps defined not only through local scripts but also from installed Python packages.
+
+- **Execution Context**: Each step in the pipeline receives an execution context that can be updated during step execution. This allows for the sharing of information and state between steps.
+
+- **Dependency Handling**: Dependency management ensures that only the necessary steps are executed, reducing runtime and resource usage by avoiding unnecessary operations.
+
+- **Ease of Use**: With Pypeline, setting up and running pipelines becomes more straightforward, enabling developers to focus more on coding and less on configuring pipeline specifics.
 
 ## Installation
 
@@ -114,8 +127,5 @@ This project follows the [all-contributors](https://github.com/all-contributors/
 
 ## Credits
 
-This package was created with
-[Copier](https://copier.readthedocs.io/) and the
-[cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template)
-project template.
+This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.
 

{pypeline_runner-0.1.1 → pypeline_runner-0.2.1}/README.md

@@ -1,4 +1,4 @@
-# Python Pipeline
+# Pypeline
 
 <p align="center">
   <a href="https://github.com/cuinixam/pypeline/actions/workflows/ci.yml?query=branch%3Amain">
@@ -23,14 +23,27 @@
   </a>
 </p>
 <p align="center">
-  <a href="https://pypi.org/project/pypeline/">
-    <img src="https://img.shields.io/pypi/v/pypeline.svg?logo=python&logoColor=fff&style=flat-square" alt="PyPI Version">
+  <a href="https://pypi.org/project/pypeline-runner/">
+    <img src="https://img.shields.io/pypi/v/pypeline-runner.svg?logo=python&logoColor=fff&style=flat-square" alt="PyPI Version">
   </a>
-  <img src="https://img.shields.io/pypi/pyversions/pypeline.svg?style=flat-square&logo=python&amp;logoColor=fff" alt="Supported Python versions">
-  <img src="https://img.shields.io/pypi/l/pypeline.svg?style=flat-square" alt="License">
+  <img src="https://img.shields.io/pypi/pyversions/pypeline-runner.svg?style=flat-square&logo=python&amp;logoColor=fff" alt="Supported Python versions">
+  <img src="https://img.shields.io/pypi/l/pypeline-runner.svg?style=flat-square" alt="License">
 </p>
 
-Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
+Pypeline is a Python application designed to streamline and automate the software development lifecycle, particularly the pipeline execution processes across various environments such as GitHub and Jenkins.
+The primary motivation for developing Pypeline stemmed from the need to unify and simplify the creation of build, test, and deployment pipelines that are traditionally defined separately across these platforms using GitHub workflows (YAML) and Jenkins pipelines (Jenkinsfile).
+
+**Key Features**
+
+- **Unified Pipeline Definition**: Users can define their entire pipeline in a single YAML file, eliminating the need to switch between different syntaxes and configurations for different CI/CD tools.
+
+- **Extensibility**: Pypeline supports execution steps defined not only through local scripts but also from installed Python packages.
+
+- **Execution Context**: Each step in the pipeline receives an execution context that can be updated during step execution. This allows for the sharing of information and state between steps.
+
+- **Dependency Handling**: Dependency management ensures that only the necessary steps are executed, reducing runtime and resource usage by avoiding unnecessary operations.
+
+- **Ease of Use**: With Pypeline, setting up and running pipelines becomes more straightforward, enabling developers to focus more on coding and less on configuring pipeline specifics.
 
 ## Installation
 
@@ -86,7 +99,4 @@ This project follows the [all-contributors](https://github.com/all-contributors/
 
 ## Credits
 
-This package was created with
-[Copier](https://copier.readthedocs.io/) and the
-[cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template)
-project template.
+This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.

{pypeline_runner-0.1.1 → pypeline_runner-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pypeline-runner"
-version = "0.1.1"
+version = "0.2.1"
 description = "Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines)."
 authors = ["cuinixam <me@cuinixam.com>"]
 license = "MIT"
@@ -27,7 +27,7 @@ pypeline = "pypeline.main:main"
 
 [tool.poetry.dependencies]
 python = "^3.10"
-py-app-dev = "^2.1.0"
+py-app-dev = "^2.1.1"
 typer = {extras = ["all"], version = "^0.12.0"}
 pyyaml = "^6.0.1"
 

pypeline_runner-0.2.1/src/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.1"

{pypeline_runner-0.1.1 → pypeline_runner-0.2.1}/src/pypeline/domain/config.py

@@ -20,7 +20,10 @@ class ProjectConfig(DataClassDictMixin):
     @classmethod
     def from_file(cls, config_file: Path) -> "ProjectConfig":
         config_dict = cls.parse_to_dict(config_file)
-        return cls.from_dict(config_dict)
+        try:
+            return cls.from_dict(config_dict)
+        except Exception as e:
+            raise UserNotificationException(f"Invalid configuration file '{config_file}'. \nError: {e}") from None
 
     @staticmethod
     def parse_to_dict(config_file: Path) -> Dict[str, Any]:
@@ -31,6 +34,6 @@ class ProjectConfig(DataClassDictMixin):
                 config_dict["file"] = config_file
             return config_dict
         except ScannerError as e:
-            raise UserNotificationException(f"Failed scanning configuration file '{config_file}'. \nError: {e}") from e
+            raise UserNotificationException(f"Failed scanning configuration file '{config_file}'. \nError: {e}") from None
         except ParserError as e:
-            raise UserNotificationException(f"Failed parsing configuration file '{config_file}'. \nError: {e}") from e
+            raise UserNotificationException(f"Failed parsing configuration file '{config_file}'. \nError: {e}") from None

{pypeline_runner-0.1.1 → pypeline_runner-0.2.1}/src/pypeline/domain/pipeline.py

@@ -1,6 +1,6 @@
 from abc import abstractmethod
 from pathlib import Path
-from typing import Type
+from typing import Any, Dict, Optional, Type
 
 from py_app_dev.core.runnable import Runnable
 
@@ -8,9 +8,10 @@ from .execution_context import ExecutionContext
 
 
 class PipelineStep(Runnable):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path) -> None:
+    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
         self.execution_context = execution_context
         self.output_dir = output_dir
+        self.config = config
         self.project_root_dir = self.execution_context.project_root_dir
 
     @abstractmethod
@@ -19,9 +20,10 @@ class PipelineStep(Runnable):
 
 
 class PipelineStepReference:
-    def __init__(self, group_name: str, _class: Type[PipelineStep]) -> None:
+    def __init__(self, group_name: str, _class: Type[PipelineStep], config: Optional[Dict[str, Any]] = None) -> None:
         self.group_name = group_name
         self._class = _class
+        self.config = config
 
     @property
     def name(self) -> str:

{pypeline_runner-0.1.1 → pypeline_runner-0.2.1}/src/pypeline/kickstart/create.py

@@ -31,7 +31,7 @@ class ProjectBuilder:
     @staticmethod
     def _check_target_directory(project_dir: Path) -> None:
         if project_dir.is_dir() and any(project_dir.iterdir()):
-            raise UserNotificationException(f"Project directory '{project_dir}' is not empty." " The target directory shall either be empty or not exist.")
+            raise UserNotificationException(f"Project directory '{project_dir}' is not empty. Use --force to override.")
 
     def build(self) -> None:
         if self.check_target_directory_flag:
@@ -41,18 +41,16 @@ class ProjectBuilder:
 
 
 class KickstartProject:
-    def __init__(self, project_dir: Path, bootstrap_only: bool = False, force: bool = False) -> None:
+    def __init__(self, project_dir: Path, force: bool = False) -> None:
         self.logger = logger.bind()
         self.project_dir = project_dir
-        self.bootstrap_only = bootstrap_only
         self.force = force
 
     def run(self) -> None:
         self.logger.info(f"Kickstart new project in '{self.project_dir.absolute().as_posix()}'")
         project_builder = ProjectBuilder(self.project_dir)
-        project_builder.with_dir("bootstrap")
-        if self.bootstrap_only or self.force:
+        if self.force:
             project_builder.with_disable_target_directory_check()
-        if not self.bootstrap_only:
-            project_builder.with_dir("project")
+        project_builder.with_dir("bootstrap")
+        project_builder.with_dir("project")
         project_builder.build()

{pypeline_runner-0.1.1 → pypeline_runner-0.2.1}/src/pypeline/main.py

@@ -1,6 +1,6 @@
 import sys
 from pathlib import Path
-from typing import List, Optional
+from typing import Optional
 
 import typer
 from py_app_dev.core.exceptions import UserNotificationException
@@ -31,10 +31,9 @@ def version(
 @time_it("init")
 def init(
     project_dir: Path = typer.Option(Path.cwd().absolute(), help="The project directory"),  # noqa: B008
-    bootstrap_only: bool = typer.Option(False, help="Initialize only the bootstrap files."),
     force: bool = typer.Option(False, help="Force the initialization of the project even if the directory is not empty."),
 ) -> None:
-    KickstartProject(project_dir, bootstrap_only, force).run()
+    KickstartProject(project_dir, force).run()
 
 
 @app.command()
@@ -60,10 +59,14 @@ def run(
         help="Force the execution of a step even if it is not dirty.",
         is_flag=True,
     ),
+    dry_run: bool = typer.Option(
+        False,
+        help="Do not run any step, just print the steps that would be executed.",
+        is_flag=True,
+    ),
 ) -> None:
     project_slurper = ProjectSlurper(project_dir)
     if print:
-        logger.warning("TODO: print pipeline steps")
         logger.info("Pipeline steps:")
         for group, step_configs in project_slurper.pipeline.items():
             logger.info(f"    Group: {group}")
@@ -80,24 +83,13 @@ def run(
         logger.info("No steps to run.")
         return
 
-    PipelineStepsExecutor(
-        project_slurper.artifacts_locator,
-        steps_references,
-        force_run,
-    ).run()
+    PipelineStepsExecutor(project_slurper.artifacts_locator, steps_references, force_run, dry_run).run()
 
 
-def main(args: Optional[List[str]] = None) -> int:
+if __name__ == "__main__":
     try:
         setup_logger()
-        if args is None:
-            args = sys.argv[1:]
-        app(args)
-        return 0
+        app()
     except UserNotificationException as e:
         logger.error(f"{e}")
-        return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+        sys.exit(1)

{pypeline_runner-0.1.1 → pypeline_runner-0.2.1}/src/pypeline/pypeline.py

@@ -26,7 +26,7 @@ class PipelineLoader:
         self._loader = GenericPipelineLoader[PipelineStep](self.pipeline_config, self.project_root_dir)
 
     def load_steps_references(self) -> List[PipelineStepReference]:
-        return [PipelineStepReference(step_reference.group_name, step_reference._class) for step_reference in self._loader.load_steps()]
+        return [PipelineStepReference(step_reference.group_name, step_reference._class, step_reference.config) for step_reference in self._loader.load_steps()]
 
 
 class PipelineStepsExecutor:
@@ -37,11 +37,13 @@ class PipelineStepsExecutor:
         artifacts_locator: ProjectArtifactsLocator,
         steps_references: List[PipelineStepReference],
         force_run: bool = False,
+        dry_run: bool = False,
     ) -> None:
         self.logger = logger.bind()
         self.artifacts_locator = artifacts_locator
         self.steps_references = steps_references
         self.force_run = force_run
+        self.dry_run = dry_run
 
     def run(self) -> None:
         execution_context = ExecutionContext(project_root_dir=self.artifacts_locator.project_root_dir, install_dirs=[])
@@ -49,9 +51,9 @@ class PipelineStepsExecutor:
             step_output_dir = self.artifacts_locator.build_dir / step_reference.group_name
             # Create the step output directory, to make sure that files can be created.
             step_output_dir.mkdir(parents=True, exist_ok=True)
-            step = step_reference._class(execution_context, step_output_dir)
+            step = step_reference._class(execution_context, step_output_dir, step_reference.config)
             # Execute the step is necessary. If the step is not dirty, it will not be executed
-            Executor(step.output_dir, self.force_run).execute(step)
+            Executor(step.output_dir, self.force_run, self.dry_run).execute(step)
             # Independent if the step was executed or not, every step shall update the context
             step.update_execution_context()
 

{pypeline_runner-0.1.1 → pypeline_runner-0.2.1}/src/pypeline/steps/create_venv.py

@@ -1,6 +1,8 @@
+from dataclasses import dataclass
 from pathlib import Path
-from typing import List
+from typing import Any, Dict, List, Optional
 
+from mashumaro import DataClassDictMixin
 from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.logging import logger
 
@@ -8,9 +10,14 @@ from ..domain.execution_context import ExecutionContext
 from ..domain.pipeline import PipelineStep
 
 
+@dataclass
+class CreateVEnvConfig(DataClassDictMixin):
+    bootstrap_script: str = "bootstrap.py"
+
+
 class CreateVEnv(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path) -> None:
-        super().__init__(execution_context, output_dir)
+    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, output_dir, config)
         self.logger = logger.bind()
         self.logger = logger.bind()
 
@@ -23,11 +30,12 @@ class CreateVEnv(PipelineStep):
 
     def run(self) -> int:
         self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
-        build_script_path = self.project_root_dir / "bootstrap.py"
-        if not build_script_path.exists():
+        config = CreateVEnvConfig.from_dict(self.config) if self.config else CreateVEnvConfig()
+        bootstrap_script = self.project_root_dir / config.bootstrap_script
+        if not bootstrap_script.exists():
             raise UserNotificationException("Failed to find bootstrap script. Make sure that the project is initialized correctly.")
         self.execution_context.create_process_executor(
-            ["python", build_script_path.as_posix()],
+            ["python", bootstrap_script.as_posix()],
             cwd=self.project_root_dir,
         ).execute()
         self.execution_context.add_install_dirs(self.install_dirs)

{pypeline_runner-0.1.1 → pypeline_runner-0.2.1}/src/pypeline/steps/scoop_install.py

@@ -3,7 +3,7 @@ import json
 import traceback
 from dataclasses import dataclass
 from pathlib import Path
-from typing import ClassVar, List
+from typing import Any, ClassVar, Dict, List, Optional
 
 from mashumaro.config import TO_DICT_ADD_OMIT_NONE_FLAG, BaseConfig
 from mashumaro.mixins.json import DataClassJSONMixin
@@ -46,8 +46,8 @@ def create_scoop_wrapper() -> ScoopWrapper:
 
 
 class ScoopInstall(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path) -> None:
-        super().__init__(execution_context, output_dir)
+    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, output_dir, config)
         self.logger = logger.bind()
         self.execution_info = ScoopInstallExecutionInfo([])
         # One needs to keep track of the installed apps to get the required paths
@@ -77,7 +77,7 @@ class ScoopInstall(PipelineStep):
         return [self.scoop_file]
 
     def get_outputs(self) -> List[Path]:
-        return self.install_dirs
+        return [self.execution_info_file, *self.install_dirs]
 
     def update_execution_context(self) -> None:
         install_dirs = ScoopInstallExecutionInfo.from_json_file(self.execution_info_file).install_dirs

{pypeline_runner-0.1.1 → pypeline_runner-0.2.1}/src/pypeline/steps/west_install.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import List
+from typing import Any, Dict, List, Optional
 
 from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.logging import logger
@@ -9,8 +9,8 @@ from ..domain.pipeline import PipelineStep
 
 
 class WestInstall(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path) -> None:
-        super().__init__(execution_context, output_dir)
+    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, output_dir, config)
         self.logger = logger.bind()
         self.artifacts_locator = execution_context.create_artifacts_locator()
 

pypeline_runner-0.1.1/src/pypeline/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.1.1"