pypeline-runner 0.1.0__tar.gz → 0.2.0__tar.gz

{pypeline_runner-0.1.0 → pypeline_runner-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pypeline-runner
-Version: 0.1.0
+Version: 0.2.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 Home-page: https://github.com/cuinixam/pypeline
 License: MIT
@@ -17,7 +17,8 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries
-Requires-Dist: py-app-dev (>=2.1.0,<3.0.0)
+Requires-Dist: py-app-dev (>=2.1.1,<3.0.0)
+Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: typer[all] (>=0.12.0,<0.13.0)
 Project-URL: Bug Tracker, https://github.com/cuinixam/pypeline/issues
 Project-URL: Changelog, https://github.com/cuinixam/pypeline/blob/main/CHANGELOG.md
@@ -25,14 +26,14 @@ Project-URL: Documentation, https://pypeline-runner.readthedocs.io
 Project-URL: Repository, https://github.com/cuinixam/pypeline
 Description-Content-Type: text/markdown
 
-# Python Pipeline
+# Pypeline
 
 <p align="center">
   <a href="https://github.com/cuinixam/pypeline/actions/workflows/ci.yml?query=branch%3Amain">
     <img src="https://img.shields.io/github/actions/workflow/status/cuinixam/pypeline/ci.yml?branch=main&label=CI&logo=github&style=flat-square" alt="CI Status" >
   </a>
-  <a href="https://pypeline.readthedocs.io">
-    <img src="https://img.shields.io/readthedocs/pypeline.svg?logo=read-the-docs&logoColor=fff&style=flat-square" alt="Documentation Status">
+  <a href="https://pypeline-runner.readthedocs.io">
+    <img src="https://img.shields.io/readthedocs/pypeline-runner.svg?logo=read-the-docs&logoColor=fff&style=flat-square" alt="Documentation Status">
   </a>
   <a href="https://codecov.io/gh/cuinixam/pypeline">
     <img src="https://img.shields.io/codecov/c/github/cuinixam/pypeline.svg?logo=codecov&logoColor=fff&style=flat-square" alt="Test coverage percentage">
@@ -50,20 +51,33 @@ Description-Content-Type: text/markdown
   </a>
 </p>
 <p align="center">
-  <a href="https://pypi.org/project/pypeline/">
-    <img src="https://img.shields.io/pypi/v/pypeline.svg?logo=python&logoColor=fff&style=flat-square" alt="PyPI Version">
+  <a href="https://pypi.org/project/pypeline-runner/">
+    <img src="https://img.shields.io/pypi/v/pypeline-runner.svg?logo=python&logoColor=fff&style=flat-square" alt="PyPI Version">
   </a>
-  <img src="https://img.shields.io/pypi/pyversions/pypeline.svg?style=flat-square&logo=python&amp;logoColor=fff" alt="Supported Python versions">
-  <img src="https://img.shields.io/pypi/l/pypeline.svg?style=flat-square" alt="License">
+  <img src="https://img.shields.io/pypi/pyversions/pypeline-runner.svg?style=flat-square&logo=python&amp;logoColor=fff" alt="Supported Python versions">
+  <img src="https://img.shields.io/pypi/l/pypeline-runner.svg?style=flat-square" alt="License">
 </p>
 
-Configure and execute steps for developing a python package.
+Pypeline is a Python application designed to streamline and automate the software development lifecycle, particularly the pipeline execution processes across various environments such as GitHub and Jenkins.
+The primary motivation for developing Pypeline stemmed from the need to unify and simplify the creation of build, test, and deployment pipelines that are traditionally defined separately across these platforms using GitHub workflows (YAML) and Jenkins pipelines (Jenkinsfile).
+
+**Key Features**
+
+- **Unified Pipeline Definition**: Users can define their entire pipeline in a single YAML file, eliminating the need to switch between different syntaxes and configurations for different CI/CD tools.
+
+- **Extensibility**: Pypeline supports execution steps defined not only through local scripts but also from installed Python packages.
+
+- **Execution Context**: Each step in the pipeline receives an execution context that can be updated during step execution. This allows for the sharing of information and state between steps.
+
+- **Dependency Handling**: Dependency management ensures that only the necessary steps are executed, reducing runtime and resource usage by avoiding unnecessary operations.
+
+- **Ease of Use**: With Pypeline, setting up and running pipelines becomes more straightforward, enabling developers to focus more on coding and less on configuring pipeline specifics.
 
 ## Installation
 
 Install this via pip (or your favourite package manager):
 
-`pip install pypeline`
+`pip install pypeline-runner`
 
 ## Start developing
 
@@ -113,8 +127,5 @@ This project follows the [all-contributors](https://github.com/all-contributors/
 
 ## Credits
 
-This package was created with
-[Copier](https://copier.readthedocs.io/) and the
-[cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template)
-project template.
+This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.
 

{pypeline_runner-0.1.0 → pypeline_runner-0.2.0}/README.md

@@ -1,11 +1,11 @@
-# Python Pipeline
+# Pypeline
 
 <p align="center">
   <a href="https://github.com/cuinixam/pypeline/actions/workflows/ci.yml?query=branch%3Amain">
     <img src="https://img.shields.io/github/actions/workflow/status/cuinixam/pypeline/ci.yml?branch=main&label=CI&logo=github&style=flat-square" alt="CI Status" >
   </a>
-  <a href="https://pypeline.readthedocs.io">
-    <img src="https://img.shields.io/readthedocs/pypeline.svg?logo=read-the-docs&logoColor=fff&style=flat-square" alt="Documentation Status">
+  <a href="https://pypeline-runner.readthedocs.io">
+    <img src="https://img.shields.io/readthedocs/pypeline-runner.svg?logo=read-the-docs&logoColor=fff&style=flat-square" alt="Documentation Status">
   </a>
   <a href="https://codecov.io/gh/cuinixam/pypeline">
     <img src="https://img.shields.io/codecov/c/github/cuinixam/pypeline.svg?logo=codecov&logoColor=fff&style=flat-square" alt="Test coverage percentage">
@@ -23,20 +23,33 @@
   </a>
 </p>
 <p align="center">
-  <a href="https://pypi.org/project/pypeline/">
-    <img src="https://img.shields.io/pypi/v/pypeline.svg?logo=python&logoColor=fff&style=flat-square" alt="PyPI Version">
+  <a href="https://pypi.org/project/pypeline-runner/">
+    <img src="https://img.shields.io/pypi/v/pypeline-runner.svg?logo=python&logoColor=fff&style=flat-square" alt="PyPI Version">
   </a>
-  <img src="https://img.shields.io/pypi/pyversions/pypeline.svg?style=flat-square&logo=python&amp;logoColor=fff" alt="Supported Python versions">
-  <img src="https://img.shields.io/pypi/l/pypeline.svg?style=flat-square" alt="License">
+  <img src="https://img.shields.io/pypi/pyversions/pypeline-runner.svg?style=flat-square&logo=python&amp;logoColor=fff" alt="Supported Python versions">
+  <img src="https://img.shields.io/pypi/l/pypeline-runner.svg?style=flat-square" alt="License">
 </p>
 
-Configure and execute steps for developing a python package.
+Pypeline is a Python application designed to streamline and automate the software development lifecycle, particularly the pipeline execution processes across various environments such as GitHub and Jenkins.
+The primary motivation for developing Pypeline stemmed from the need to unify and simplify the creation of build, test, and deployment pipelines that are traditionally defined separately across these platforms using GitHub workflows (YAML) and Jenkins pipelines (Jenkinsfile).
+
+**Key Features**
+
+- **Unified Pipeline Definition**: Users can define their entire pipeline in a single YAML file, eliminating the need to switch between different syntaxes and configurations for different CI/CD tools.
+
+- **Extensibility**: Pypeline supports execution steps defined not only through local scripts but also from installed Python packages.
+
+- **Execution Context**: Each step in the pipeline receives an execution context that can be updated during step execution. This allows for the sharing of information and state between steps.
+
+- **Dependency Handling**: Dependency management ensures that only the necessary steps are executed, reducing runtime and resource usage by avoiding unnecessary operations.
+
+- **Ease of Use**: With Pypeline, setting up and running pipelines becomes more straightforward, enabling developers to focus more on coding and less on configuring pipeline specifics.
 
 ## Installation
 
 Install this via pip (or your favourite package manager):
 
-`pip install pypeline`
+`pip install pypeline-runner`
 
 ## Start developing
 
@@ -86,7 +99,4 @@ This project follows the [all-contributors](https://github.com/all-contributors/
 
 ## Credits
 
-This package was created with
-[Copier](https://copier.readthedocs.io/) and the
-[cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template)
-project template.
+This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.

{pypeline_runner-0.1.0 → pypeline_runner-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pypeline-runner"
-version = "0.1.0"
+version = "0.2.0"
 description = "Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines)."
 authors = ["cuinixam <me@cuinixam.com>"]
 license = "MIT"
@@ -27,8 +27,9 @@ pypeline = "pypeline.main:main"
 
 [tool.poetry.dependencies]
 python = "^3.10"
-py-app-dev = "^2.1.0"
+py-app-dev = "^2.1.1"
 typer = {extras = ["all"], version = "^0.12.0"}
+pyyaml = "^6.0.1"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.0"

pypeline_runner-0.2.0/src/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

{pypeline_runner-0.1.0 → pypeline_runner-0.2.0}/src/pypeline/domain/pipeline.py

@@ -1,6 +1,6 @@
 from abc import abstractmethod
 from pathlib import Path
-from typing import Type
+from typing import Any, Dict, Optional, Type
 
 from py_app_dev.core.runnable import Runnable
 
@@ -8,9 +8,10 @@ from .execution_context import ExecutionContext
 
 
 class PipelineStep(Runnable):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path) -> None:
+    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
         self.execution_context = execution_context
         self.output_dir = output_dir
+        self.config = config
         self.project_root_dir = self.execution_context.project_root_dir
 
     @abstractmethod
@@ -19,9 +20,10 @@ class PipelineStep(Runnable):
 
 
 class PipelineStepReference:
-    def __init__(self, group_name: str, _class: Type[PipelineStep]) -> None:
+    def __init__(self, group_name: str, _class: Type[PipelineStep], config: Optional[Dict[str, Any]] = None) -> None:
         self.group_name = group_name
         self._class = _class
+        self.config = config
 
     @property
     def name(self) -> str:

{pypeline_runner-0.1.0 → pypeline_runner-0.2.0}/src/pypeline/kickstart/create.py

@@ -31,7 +31,7 @@ class ProjectBuilder:
     @staticmethod
     def _check_target_directory(project_dir: Path) -> None:
         if project_dir.is_dir() and any(project_dir.iterdir()):
-            raise UserNotificationException(f"Project directory '{project_dir}' is not empty." " The target directory shall either be empty or not exist.")
+            raise UserNotificationException(f"Project directory '{project_dir}' is not empty. Use --force to override.")
 
     def build(self) -> None:
         if self.check_target_directory_flag:
@@ -41,18 +41,16 @@ class ProjectBuilder:
 
 
 class KickstartProject:
-    def __init__(self, project_dir: Path, bootstrap_only: bool = False, force: bool = False) -> None:
+    def __init__(self, project_dir: Path, force: bool = False) -> None:
         self.logger = logger.bind()
         self.project_dir = project_dir
-        self.bootstrap_only = bootstrap_only
         self.force = force
 
     def run(self) -> None:
         self.logger.info(f"Kickstart new project in '{self.project_dir.absolute().as_posix()}'")
         project_builder = ProjectBuilder(self.project_dir)
-        project_builder.with_dir("bootstrap")
-        if self.bootstrap_only or self.force:
+        if self.force:
             project_builder.with_disable_target_directory_check()
-        if not self.bootstrap_only:
-            project_builder.with_dir("project")
+        project_builder.with_dir("bootstrap")
+        project_builder.with_dir("project")
         project_builder.build()

pypeline_runner-0.2.0/src/pypeline/kickstart/templates/project/pypeline.ps1

@@ -0,0 +1 @@
+.\.venv\Scripts\pypeline.exe $args

{pypeline_runner-0.1.0 → pypeline_runner-0.2.0}/src/pypeline/kickstart/templates/project/pyproject.toml

@@ -6,3 +6,4 @@ authors = ["Your Name"]
 
 [tool.poetry.dependencies]
 python = ">=3.10,<3.13"
+pypeline-runner = "*"

{pypeline_runner-0.1.0 → pypeline_runner-0.2.0}/src/pypeline/main.py

@@ -13,7 +13,9 @@ from pypeline.pypeline import PipelineScheduler, PipelineStepsExecutor
 
 package_name = "pypeline"
 
-app = typer.Typer(name=package_name, help="Configure and execute steps for developing a python package.", no_args_is_help=True, add_completion=False)
+app = typer.Typer(
+    name=package_name, help="Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).", no_args_is_help=True, add_completion=False
+)
 
 
 @app.callback(invoke_without_command=True)
@@ -29,10 +31,9 @@ def version(
 @time_it("init")
 def init(
     project_dir: Path = typer.Option(Path.cwd().absolute(), help="The project directory"),  # noqa: B008
-    bootstrap_only: bool = typer.Option(False, help="Initialize only the bootstrap files."),
     force: bool = typer.Option(False, help="Force the initialization of the project even if the directory is not empty."),
 ) -> None:
-    KickstartProject(project_dir, bootstrap_only, force).run()
+    KickstartProject(project_dir, force).run()
 
 
 @app.command()
@@ -58,10 +59,14 @@ def run(
         help="Force the execution of a step even if it is not dirty.",
         is_flag=True,
     ),
+    dry_run: bool = typer.Option(
+        False,
+        help="Do not run any step, just print the steps that would be executed.",
+        is_flag=True,
+    ),
 ) -> None:
     project_slurper = ProjectSlurper(project_dir)
     if print:
-        logger.warning("TODO: print pipeline steps")
         logger.info("Pipeline steps:")
         for group, step_configs in project_slurper.pipeline.items():
             logger.info(f"    Group: {group}")
@@ -78,11 +83,7 @@ def run(
         logger.info("No steps to run.")
         return
 
-    PipelineStepsExecutor(
-        project_slurper.artifacts_locator,
-        steps_references,
-        force_run,
-    ).run()
+    PipelineStepsExecutor(project_slurper.artifacts_locator, steps_references, force_run, dry_run).run()
 
 
 def main(args: Optional[List[str]] = None) -> int:

{pypeline_runner-0.1.0 → pypeline_runner-0.2.0}/src/pypeline/pypeline.py

@@ -26,7 +26,7 @@ class PipelineLoader:
         self._loader = GenericPipelineLoader[PipelineStep](self.pipeline_config, self.project_root_dir)
 
     def load_steps_references(self) -> List[PipelineStepReference]:
-        return [PipelineStepReference(step_reference.group_name, step_reference._class) for step_reference in self._loader.load_steps()]
+        return [PipelineStepReference(step_reference.group_name, step_reference._class, step_reference.config) for step_reference in self._loader.load_steps()]
 
 
 class PipelineStepsExecutor:
@@ -37,11 +37,13 @@ class PipelineStepsExecutor:
         artifacts_locator: ProjectArtifactsLocator,
         steps_references: List[PipelineStepReference],
         force_run: bool = False,
+        dry_run: bool = False,
     ) -> None:
         self.logger = logger.bind()
         self.artifacts_locator = artifacts_locator
         self.steps_references = steps_references
         self.force_run = force_run
+        self.dry_run = dry_run
 
     def run(self) -> None:
         execution_context = ExecutionContext(project_root_dir=self.artifacts_locator.project_root_dir, install_dirs=[])
@@ -49,9 +51,9 @@ class PipelineStepsExecutor:
             step_output_dir = self.artifacts_locator.build_dir / step_reference.group_name
             # Create the step output directory, to make sure that files can be created.
             step_output_dir.mkdir(parents=True, exist_ok=True)
-            step = step_reference._class(execution_context, step_output_dir)
+            step = step_reference._class(execution_context, step_output_dir, step_reference.config)
             # Execute the step is necessary. If the step is not dirty, it will not be executed
-            Executor(step.output_dir, self.force_run).execute(step)
+            Executor(step.output_dir, self.force_run, self.dry_run).execute(step)
             # Independent if the step was executed or not, every step shall update the context
             step.update_execution_context()
 

{pypeline_runner-0.1.0 → pypeline_runner-0.2.0}/src/pypeline/steps/create_venv.py

@@ -1,6 +1,8 @@
+from dataclasses import dataclass
 from pathlib import Path
-from typing import List
+from typing import Any, Dict, List, Optional
 
+from mashumaro import DataClassDictMixin
 from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.logging import logger
 
@@ -8,9 +10,14 @@ from ..domain.execution_context import ExecutionContext
 from ..domain.pipeline import PipelineStep
 
 
+@dataclass
+class CreateVEnvConfig(DataClassDictMixin):
+    bootstrap_script: str = "bootstrap.py"
+
+
 class CreateVEnv(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path) -> None:
-        super().__init__(execution_context, output_dir)
+    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, output_dir, config)
         self.logger = logger.bind()
         self.logger = logger.bind()
 
@@ -23,11 +30,12 @@ class CreateVEnv(PipelineStep):
 
     def run(self) -> int:
         self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
-        build_script_path = self.project_root_dir / "bootstrap.py"
-        if not build_script_path.exists():
+        config = CreateVEnvConfig.from_dict(self.config) if self.config else CreateVEnvConfig()
+        bootstrap_script = self.project_root_dir / config.bootstrap_script
+        if not bootstrap_script.exists():
             raise UserNotificationException("Failed to find bootstrap script. Make sure that the project is initialized correctly.")
         self.execution_context.create_process_executor(
-            ["python", build_script_path.as_posix()],
+            ["python", bootstrap_script.as_posix()],
             cwd=self.project_root_dir,
         ).execute()
         self.execution_context.add_install_dirs(self.install_dirs)

{pypeline_runner-0.1.0 → pypeline_runner-0.2.0}/src/pypeline/steps/scoop_install.py

@@ -3,7 +3,7 @@ import json
 import traceback
 from dataclasses import dataclass
 from pathlib import Path
-from typing import ClassVar, List
+from typing import Any, ClassVar, Dict, List, Optional
 
 from mashumaro.config import TO_DICT_ADD_OMIT_NONE_FLAG, BaseConfig
 from mashumaro.mixins.json import DataClassJSONMixin
@@ -46,8 +46,8 @@ def create_scoop_wrapper() -> ScoopWrapper:
 
 
 class ScoopInstall(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path) -> None:
-        super().__init__(execution_context, output_dir)
+    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, output_dir, config)
         self.logger = logger.bind()
         self.execution_info = ScoopInstallExecutionInfo([])
         # One needs to keep track of the installed apps to get the required paths
@@ -77,7 +77,7 @@ class ScoopInstall(PipelineStep):
         return [self.scoop_file]
 
     def get_outputs(self) -> List[Path]:
-        return self.install_dirs
+        return [self.execution_info_file, *self.install_dirs]
 
     def update_execution_context(self) -> None:
         install_dirs = ScoopInstallExecutionInfo.from_json_file(self.execution_info_file).install_dirs

{pypeline_runner-0.1.0 → pypeline_runner-0.2.0}/src/pypeline/steps/west_install.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import List
+from typing import Any, Dict, List, Optional
 
 from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.logging import logger
@@ -9,8 +9,8 @@ from ..domain.pipeline import PipelineStep
 
 
 class WestInstall(PipelineStep):
-    def __init__(self, execution_context: ExecutionContext, output_dir: Path) -> None:
-        super().__init__(execution_context, output_dir)
+    def __init__(self, execution_context: ExecutionContext, output_dir: Path, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, output_dir, config)
         self.logger = logger.bind()
         self.artifacts_locator = execution_context.create_artifacts_locator()
 

pypeline_runner-0.1.0/src/pypeline/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.1.0"