pypeline-runner 1.6.0__tar.gz → 1.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/PKG-INFO +61 -33
  2. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/README.md +60 -32
  3. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/pyproject.toml +2 -1
  4. pypeline_runner-1.8.0/src/pypeline/__init__.py +1 -0
  5. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/domain/artifacts.py +2 -5
  6. pypeline_runner-1.8.0/src/pypeline/domain/pipeline.py +181 -0
  7. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/domain/project_slurper.py +3 -2
  8. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/create.py +55 -55
  9. pypeline_runner-1.8.0/src/pypeline/kickstart/templates/project/bootstrap.py +461 -0
  10. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/poetry.toml +2 -2
  11. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/pypeline.ps1 +7 -7
  12. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/pypeline.yaml +3 -2
  13. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/pyproject.toml +11 -9
  14. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/steps/my_step.py +26 -26
  15. pypeline_runner-1.8.0/src/pypeline/kickstart/templates/project/west.yaml +10 -0
  16. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/main.py +2 -1
  17. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/pypeline.py +19 -76
  18. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/steps/create_venv.py +10 -3
  19. pypeline_runner-1.6.0/src/pypeline/__init__.py +0 -1
  20. pypeline_runner-1.6.0/src/pypeline/domain/pipeline.py +0 -109
  21. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/LICENSE +0 -0
  22. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/__run.py +0 -0
  23. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/domain/__init__.py +0 -0
  24. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/domain/config.py +0 -0
  25. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/domain/execution_context.py +0 -0
  26. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/__init__.py +0 -0
  27. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/.gitignore +0 -0
  28. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/bootstrap.ps1 +0 -0
  29. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/scoopfile.json +0 -0
  30. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/py.typed +0 -0
  31. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/steps/__init__.py +0 -0
  32. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/steps/scoop_install.py +0 -0
  33. {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/steps/west_install.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: pypeline-runner
3
- Version: 1.6.0
3
+ Version: 1.8.0
4
4
  Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
5
5
  License: MIT
6
6
  Author: cuinixam
@@ -26,7 +26,13 @@ Project-URL: Documentation, https://pypeline-runner.readthedocs.io
26
26
  Project-URL: Repository, https://github.com/cuinixam/pypeline
27
27
  Description-Content-Type: text/markdown
28
28
 
29
- # Pypeline
29
+ <p align="center">
30
+ <a href="https://pypeline-runner.readthedocs.io">
31
+ <img align="center" src="https://github.com/cuinixam/pypeline/raw/main/logo.png" width="400"/>
32
+ </a>
33
+ </p>
34
+
35
+ # pypeline - Define Your CI/CD Pipeline Once, Run It Anywhere
30
36
 
31
37
  <p align="center">
32
38
  <a href="https://github.com/cuinixam/pypeline/actions/workflows/ci.yml?query=branch%3Amain">
@@ -58,28 +64,71 @@ Description-Content-Type: text/markdown
58
64
  <img src="https://img.shields.io/pypi/l/pypeline-runner.svg?style=flat-square" alt="License">
59
65
  </p>
60
66
 
61
- Pypeline is a Python application designed to streamline and automate the software development lifecycle, particularly the pipeline execution processes across various environments such as GitHub and Jenkins.
62
- The primary motivation for developing Pypeline stemmed from the need to unify and simplify the creation of build, test, and deployment pipelines that are traditionally defined separately across these platforms using GitHub workflows (YAML) and Jenkins pipelines (Jenkinsfile).
67
+ Pypeline lets you define your build, test, and deployment pipeline in a single YAML file and run it _consistently_ across your local development environment and _any_ CI/CD platform (GitHub Actions, Jenkins, etc.). No more platform-specific configurations – write once, run anywhere.
63
68
 
64
69
  **Key Features**
65
70
 
66
71
  - **Unified Pipeline Definition**: Users can define their entire pipeline in a single YAML file, eliminating the need to switch between different syntaxes and configurations for different CI/CD tools.
67
72
 
68
- - **Extensibility**: Pypeline supports execution steps defined not only through local scripts but also from installed Python packages.
69
-
70
- - **Execution Context**: Each step in the pipeline receives an execution context that can be updated during step execution. This allows for the sharing of information and state between steps.
73
+ - **Extensibility**: Pypeline supports execution steps defined not only through installed Python packages but also from local scripts.
71
74
 
72
- - **Dependency Handling**: Dependency management ensures that only the necessary steps are executed, reducing runtime and resource usage by avoiding unnecessary operations.
75
+ - **Execution Context**: Allow sharing information and state between steps. Each step in the pipeline receives an execution context that can be updated during step execution.
73
76
 
74
- - **Ease of Use**: With Pypeline, setting up and running pipelines becomes more straightforward, enabling developers to focus more on coding and less on configuring pipeline specifics.
77
+ - **Dependency Handling**: Every step can register its dependencies and will only be scheduled if anything has changed.
75
78
 
76
79
  ## Installation
77
80
 
78
- Install this via pip (or your favourite package manager):
81
+ Use pipx (or your favorite package manager) to install and run it in an isolated environment:
82
+
83
+ ```shell
84
+ pipx install pypeline-runner
85
+ ```
86
+
87
+ This will install the `pypeline` command globally, which you can use to run your pipelines.
79
88
 
80
- `pip install pypeline-runner`
89
+ > [!NOTE]
90
+ > The Python package is called `pypeline-runner` because the name `pypeline` was already taken on PyPI.
91
+ > The command-line interface is `pypeline`.
81
92
 
82
- ## Start developing
93
+ Documentation: [pypeline-runner.readthedocs.io](https://pypeline-runner.readthedocs.io)
94
+
95
+ ## Walkthrough: Getting Started with Pypeline
96
+
97
+ To get started run the `init` command to create a sample project:
98
+
99
+ ```shell
100
+ pypeline init --project-dir my-pipeline
101
+ ```
102
+
103
+ The example project pipeline is defined in the `pipeline.yaml` file.
104
+
105
+ ```yaml
106
+ pipeline:
107
+ - step: CreateVEnv
108
+ module: pypeline.steps.create_venv
109
+ config:
110
+ bootstrap_script: .bootstrap/bootstrap.py
111
+ - step: WestInstall
112
+ module: pypeline.steps.west_install
113
+ description: Download external modules
114
+ - step: MyStep
115
+ file: steps/my_step.py
116
+ description: Run a custom script
117
+ ```
118
+
119
+ This pipeline consists of three steps:
120
+
121
+ - `CreateVEnv`: This is a built-in step that creates a Python virtual environment.
122
+ - `WestInstall`: This is a built-in step that downloads external modules using the `west` tool.
123
+ - `MyStep`: This is a custom step that runs a script defined in the `steps/my_step.py` file.
124
+
125
+ You can run the pipeline using the `run` command:
126
+
127
+ ```shell
128
+ pypeline run --project-dir my-pipeline
129
+ ```
130
+
131
+ ## Contributing
83
132
 
84
133
  The project uses Poetry for dependencies management and packaging.
85
134
  Run the `bootstrap.ps1` script to install Python and create the virtual environment.
@@ -88,43 +137,22 @@ Run the `bootstrap.ps1` script to install Python and create the virtual environm
88
137
  .\bootstrap.ps1
89
138
  ```
90
139
 
91
- This will also generate a `poetry.lock` file, you should track this file in version control.
92
-
93
140
  To execute the test suite, call pytest inside Poetry's virtual environment via `poetry run`:
94
141
 
95
142
  ```shell
96
143
  .venv/Scripts/poetry run pytest
97
144
  ```
98
145
 
99
- Check out the Poetry documentation for more information on the available commands.
100
-
101
146
  For those using [VS Code](https://code.visualstudio.com/) there are tasks defined for the most common commands:
102
147
 
103
- - bootstrap
104
- - install dependencies
105
148
  - run tests
106
149
  - run all checks configured for pre-commit
107
150
  - generate documentation
108
151
 
109
152
  See the `.vscode/tasks.json` for more details.
110
153
 
111
- ## Committing changes
112
-
113
154
  This repository uses [commitlint](https://github.com/conventional-changelog/commitlint) for checking if the commit message meets the [conventional commit format](https://www.conventionalcommits.org/en).
114
155
 
115
- ## Contributors ✨
116
-
117
- Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
118
-
119
- <!-- prettier-ignore-start -->
120
- <!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
121
- <!-- markdownlint-disable -->
122
- <!-- markdownlint-enable -->
123
- <!-- ALL-CONTRIBUTORS-LIST:END -->
124
- <!-- prettier-ignore-end -->
125
-
126
- This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
127
-
128
156
  ## Credits
129
157
 
130
158
  This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.
@@ -1,4 +1,10 @@
1
- # Pypeline
1
+ <p align="center">
2
+ <a href="https://pypeline-runner.readthedocs.io">
3
+ <img align="center" src="https://github.com/cuinixam/pypeline/raw/main/logo.png" width="400"/>
4
+ </a>
5
+ </p>
6
+
7
+ # pypeline - Define Your CI/CD Pipeline Once, Run It Anywhere
2
8
 
3
9
  <p align="center">
4
10
  <a href="https://github.com/cuinixam/pypeline/actions/workflows/ci.yml?query=branch%3Amain">
@@ -30,28 +36,71 @@
30
36
  <img src="https://img.shields.io/pypi/l/pypeline-runner.svg?style=flat-square" alt="License">
31
37
  </p>
32
38
 
33
- Pypeline is a Python application designed to streamline and automate the software development lifecycle, particularly the pipeline execution processes across various environments such as GitHub and Jenkins.
34
- The primary motivation for developing Pypeline stemmed from the need to unify and simplify the creation of build, test, and deployment pipelines that are traditionally defined separately across these platforms using GitHub workflows (YAML) and Jenkins pipelines (Jenkinsfile).
39
+ Pypeline lets you define your build, test, and deployment pipeline in a single YAML file and run it _consistently_ across your local development environment and _any_ CI/CD platform (GitHub Actions, Jenkins, etc.). No more platform-specific configurations – write once, run anywhere.
35
40
 
36
41
  **Key Features**
37
42
 
38
43
  - **Unified Pipeline Definition**: Users can define their entire pipeline in a single YAML file, eliminating the need to switch between different syntaxes and configurations for different CI/CD tools.
39
44
 
40
- - **Extensibility**: Pypeline supports execution steps defined not only through local scripts but also from installed Python packages.
41
-
42
- - **Execution Context**: Each step in the pipeline receives an execution context that can be updated during step execution. This allows for the sharing of information and state between steps.
45
+ - **Extensibility**: Pypeline supports execution steps defined not only through installed Python packages but also from local scripts.
43
46
 
44
- - **Dependency Handling**: Dependency management ensures that only the necessary steps are executed, reducing runtime and resource usage by avoiding unnecessary operations.
47
+ - **Execution Context**: Allow sharing information and state between steps. Each step in the pipeline receives an execution context that can be updated during step execution.
45
48
 
46
- - **Ease of Use**: With Pypeline, setting up and running pipelines becomes more straightforward, enabling developers to focus more on coding and less on configuring pipeline specifics.
49
+ - **Dependency Handling**: Every step can register its dependencies and will only be scheduled if anything has changed.
47
50
 
48
51
  ## Installation
49
52
 
50
- Install this via pip (or your favourite package manager):
53
+ Use pipx (or your favorite package manager) to install and run it in an isolated environment:
54
+
55
+ ```shell
56
+ pipx install pypeline-runner
57
+ ```
58
+
59
+ This will install the `pypeline` command globally, which you can use to run your pipelines.
51
60
 
52
- `pip install pypeline-runner`
61
+ > [!NOTE]
62
+ > The Python package is called `pypeline-runner` because the name `pypeline` was already taken on PyPI.
63
+ > The command-line interface is `pypeline`.
53
64
 
54
- ## Start developing
65
+ Documentation: [pypeline-runner.readthedocs.io](https://pypeline-runner.readthedocs.io)
66
+
67
+ ## Walkthrough: Getting Started with Pypeline
68
+
69
+ To get started run the `init` command to create a sample project:
70
+
71
+ ```shell
72
+ pypeline init --project-dir my-pipeline
73
+ ```
74
+
75
+ The example project pipeline is defined in the `pipeline.yaml` file.
76
+
77
+ ```yaml
78
+ pipeline:
79
+ - step: CreateVEnv
80
+ module: pypeline.steps.create_venv
81
+ config:
82
+ bootstrap_script: .bootstrap/bootstrap.py
83
+ - step: WestInstall
84
+ module: pypeline.steps.west_install
85
+ description: Download external modules
86
+ - step: MyStep
87
+ file: steps/my_step.py
88
+ description: Run a custom script
89
+ ```
90
+
91
+ This pipeline consists of three steps:
92
+
93
+ - `CreateVEnv`: This is a built-in step that creates a Python virtual environment.
94
+ - `WestInstall`: This is a built-in step that downloads external modules using the `west` tool.
95
+ - `MyStep`: This is a custom step that runs a script defined in the `steps/my_step.py` file.
96
+
97
+ You can run the pipeline using the `run` command:
98
+
99
+ ```shell
100
+ pypeline run --project-dir my-pipeline
101
+ ```
102
+
103
+ ## Contributing
55
104
 
56
105
  The project uses Poetry for dependencies management and packaging.
57
106
  Run the `bootstrap.ps1` script to install Python and create the virtual environment.
@@ -60,43 +109,22 @@ Run the `bootstrap.ps1` script to install Python and create the virtual environm
60
109
  .\bootstrap.ps1
61
110
  ```
62
111
 
63
- This will also generate a `poetry.lock` file, you should track this file in version control.
64
-
65
112
  To execute the test suite, call pytest inside Poetry's virtual environment via `poetry run`:
66
113
 
67
114
  ```shell
68
115
  .venv/Scripts/poetry run pytest
69
116
  ```
70
117
 
71
- Check out the Poetry documentation for more information on the available commands.
72
-
73
118
  For those using [VS Code](https://code.visualstudio.com/) there are tasks defined for the most common commands:
74
119
 
75
- - bootstrap
76
- - install dependencies
77
120
  - run tests
78
121
  - run all checks configured for pre-commit
79
122
  - generate documentation
80
123
 
81
124
  See the `.vscode/tasks.json` for more details.
82
125
 
83
- ## Committing changes
84
-
85
126
  This repository uses [commitlint](https://github.com/conventional-changelog/commitlint) for checking if the commit message meets the [conventional commit format](https://www.conventionalcommits.org/en).
86
127
 
87
- ## Contributors ✨
88
-
89
- Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
90
-
91
- <!-- prettier-ignore-start -->
92
- <!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
93
- <!-- markdownlint-disable -->
94
- <!-- markdownlint-enable -->
95
- <!-- ALL-CONTRIBUTORS-LIST:END -->
96
- <!-- prettier-ignore-end -->
97
-
98
- This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
99
-
100
128
  ## Credits
101
129
 
102
130
  This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "pypeline-runner"
3
- version = "1.6.0"
3
+ version = "1.8.0"
4
4
  description = "Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines)."
5
5
  authors = ["cuinixam <me@cuinixam.com>"]
6
6
  license = "MIT"
@@ -36,6 +36,7 @@ pytest = "^8.3"
36
36
  pytest-cov = "^6.0"
37
37
  pre-commit = "^4.0"
38
38
  ruff = "^0.9"
39
+ west = "^1.3"
39
40
 
40
41
  [tool.poetry.group.docs.dependencies]
41
42
  myst-parser = ">=0.16"
@@ -0,0 +1 @@
1
+ __version__ = "1.8.0"
@@ -10,13 +10,10 @@ CONFIG_FILENAME = "pypeline.yaml"
10
10
  class ProjectArtifactsLocator:
11
11
  """Provides paths to project artifacts."""
12
12
 
13
- def __init__(
14
- self,
15
- project_root_dir: Path,
16
- ) -> None:
13
+ def __init__(self, project_root_dir: Path, config_file: Optional[str] = None) -> None:
17
14
  self.project_root_dir = project_root_dir
18
15
  self.build_dir = project_root_dir / "build"
19
- self.config_file = project_root_dir / CONFIG_FILENAME
16
+ self.config_file = project_root_dir.joinpath(config_file if config_file else CONFIG_FILENAME)
20
17
  self.external_dependencies_dir = self.build_dir / "external"
21
18
  scripts_dir = "Scripts" if sys.platform.startswith("win32") else "bin"
22
19
  self.venv_scripts_dir = self.project_root_dir.joinpath(".venv").joinpath(scripts_dir)
@@ -0,0 +1,181 @@
1
+ import importlib
2
+ from abc import abstractmethod
3
+ from dataclasses import dataclass
4
+ from importlib.util import module_from_spec, spec_from_file_location
5
+ from pathlib import Path
6
+ from typing import (
7
+ Any,
8
+ Dict,
9
+ Generic,
10
+ Iterator,
11
+ List,
12
+ Optional,
13
+ OrderedDict,
14
+ Protocol,
15
+ Tuple,
16
+ Type,
17
+ TypeAlias,
18
+ TypeVar,
19
+ Union,
20
+ )
21
+
22
+ from mashumaro import DataClassDictMixin
23
+ from py_app_dev.core.exceptions import UserNotificationException
24
+ from py_app_dev.core.runnable import Runnable
25
+
26
+ from .execution_context import ExecutionContext
27
+
28
+
29
+ @dataclass
30
+ class PipelineStepConfig(DataClassDictMixin):
31
+ #: Step name or class name if file is not specified
32
+ step: str
33
+ #: Path to file with step class
34
+ file: Optional[str] = None
35
+ #: Python module with step class
36
+ module: Optional[str] = None
37
+ #: Step class name
38
+ class_name: Optional[str] = None
39
+ #: Command to run. For simple steps that don't need a class. Example: run: [echo, 'Hello World!']
40
+ run: Optional[Union[str, List[str]]] = None
41
+ #: Step description
42
+ description: Optional[str] = None
43
+ #: Step timeout in seconds
44
+ timeout_sec: Optional[int] = None
45
+ #: Custom step configuration
46
+ config: Optional[Dict[str, Any]] = None
47
+
48
+
49
+ PipelineConfig: TypeAlias = Union[List[PipelineStepConfig], OrderedDict[str, List[PipelineStepConfig]]]
50
+
51
+ TPipelineStep = TypeVar("TPipelineStep", covariant=True)
52
+
53
+
54
+ @dataclass
55
+ class PipelineStepReference(Generic[TPipelineStep]):
56
+ """Once a Step is found, keep the Step class reference to be able to instantiate it later."""
57
+
58
+ group_name: Optional[str]
59
+ _class: Type[TPipelineStep]
60
+ config: Optional[Dict[str, Any]] = None
61
+
62
+ @property
63
+ def name(self) -> str:
64
+ return self._class.__name__
65
+
66
+
67
+ class PipelineConfigIterator:
68
+ """
69
+ Iterates over the pipeline configuration, yielding group name and steps configuration.
70
+
71
+ This class abstracts the iteration logic for PipelineConfig, which can be:
72
+ - A list of steps (group name is None)
73
+ - An OrderedDict with group names as keys and lists of steps as values.
74
+
75
+ The iterator yields tuples of (group_name, steps).
76
+ """
77
+
78
+ def __init__(self, pipeline_config: PipelineConfig) -> None:
79
+ self._items = pipeline_config.items() if isinstance(pipeline_config, OrderedDict) else [(None, pipeline_config)]
80
+
81
+ def __iter__(self) -> Iterator[Tuple[Optional[str], List[PipelineStepConfig]]]:
82
+ """Return an iterator."""
83
+ yield from self._items
84
+
85
+
86
+ class StepClassFactory(Generic[TPipelineStep], Protocol):
87
+ def create_step_class(self, step_config: PipelineStepConfig, project_root_dir: Path) -> Type[TPipelineStep]: ...
88
+
89
+
90
+ class PipelineLoader(Generic[TPipelineStep]):
91
+ def __init__(self, pipeline_config: PipelineConfig, project_root_dir: Path, step_class_factory: Optional[StepClassFactory[TPipelineStep]] = None) -> None:
92
+ self.pipeline_config = pipeline_config
93
+ self.project_root_dir = project_root_dir
94
+ self.step_class_factory = step_class_factory
95
+
96
+ def load_steps_references(self) -> List[PipelineStepReference[TPipelineStep]]:
97
+ result = []
98
+ for group_name, steps_config in PipelineConfigIterator(self.pipeline_config):
99
+ result.extend(self._load_steps(group_name, steps_config, self.project_root_dir, self.step_class_factory))
100
+ return result
101
+
102
+ @staticmethod
103
+ def _load_steps(
104
+ group_name: Optional[str], steps_config: List[PipelineStepConfig], project_root_dir: Path, step_class_factory: Optional[StepClassFactory[TPipelineStep]] = None
105
+ ) -> List[PipelineStepReference[TPipelineStep]]:
106
+ result = []
107
+ for step_config in steps_config:
108
+ step_class_name = step_config.class_name or step_config.step
109
+ if step_config.module:
110
+ step_class = PipelineLoader[TPipelineStep]._load_module_step(step_config.module, step_class_name)
111
+ elif step_config.file:
112
+ step_class = PipelineLoader[TPipelineStep]._load_user_step(project_root_dir.joinpath(step_config.file), step_class_name)
113
+ else:
114
+ if step_class_factory:
115
+ step_class = step_class_factory.create_step_class(step_config, project_root_dir)
116
+ else:
117
+ raise UserNotificationException(
118
+ f"Step '{step_class_name}' has no 'module' nor 'file' defined nor a custom step class factory was provided. Please check your pipeline configuration."
119
+ )
120
+ result.append(PipelineStepReference(group_name, step_class, step_config.config))
121
+ return result
122
+
123
+ @staticmethod
124
+ def _load_user_step(python_file: Path, step_class_name: str) -> Type[TPipelineStep]:
125
+ # Create a module specification from the file path
126
+ spec = spec_from_file_location(f"user__{python_file.stem}", python_file)
127
+ if spec and spec.loader:
128
+ step_module = module_from_spec(spec)
129
+ # Import the module
130
+ spec.loader.exec_module(step_module)
131
+ try:
132
+ step_class = getattr(step_module, step_class_name)
133
+ except AttributeError:
134
+ raise UserNotificationException(f"Could not load class '{step_class_name}' from file '{python_file}'. Please check your pipeline configuration.") from None
135
+ return step_class
136
+ raise UserNotificationException(f"Could not load file '{python_file}'. Please check the file for any errors.")
137
+
138
+ @staticmethod
139
+ def _load_module_step(module_name: str, step_class_name: str) -> Type[TPipelineStep]:
140
+ try:
141
+ module = importlib.import_module(module_name)
142
+ step_class = getattr(module, step_class_name)
143
+ except ImportError:
144
+ raise UserNotificationException(f"Could not load module '{module_name}'. Please check your pipeline configuration.") from None
145
+ except AttributeError:
146
+ raise UserNotificationException(f"Could not load class '{step_class_name}' from module '{module_name}'. Please check your pipeline configuration.") from None
147
+ return step_class
148
+
149
+
150
+ TExecutionContext = TypeVar("TExecutionContext", bound=ExecutionContext)
151
+
152
+
153
+ class PipelineStep(Generic[TExecutionContext], Runnable):
154
+ """One can create subclasses of PipelineStep that specify the type of ExecutionContext they require."""
155
+
156
+ def __init__(self, execution_context: TExecutionContext, group_name: Optional[str], config: Optional[Dict[str, Any]] = None) -> None:
157
+ super().__init__(self.get_needs_dependency_management())
158
+ self.execution_context = execution_context
159
+ self.group_name = group_name
160
+ self.config = config
161
+ self.project_root_dir = self.execution_context.project_root_dir
162
+
163
+ @property
164
+ def output_dir(self) -> Path:
165
+ output_dir = self.execution_context.create_artifacts_locator().build_dir
166
+ if self.group_name:
167
+ output_dir = output_dir / self.group_name
168
+ return output_dir
169
+
170
+ @abstractmethod
171
+ def update_execution_context(self) -> None:
172
+ """
173
+ Even if the step does not need to run ( because it is not outdated ), it can still update the execution context.
174
+
175
+ A typical use case is for steps installing software that need to provide the install directories in the execution context even if all tools are already installed.
176
+ """
177
+ pass
178
+
179
+ def get_needs_dependency_management(self) -> bool:
180
+ """If false, the step executor will not check for outdated dependencies. This is useful for steps consisting of command lines which shall always run."""
181
+ return True
@@ -1,4 +1,5 @@
1
1
  from pathlib import Path
2
+ from typing import Optional
2
3
 
3
4
  from py_app_dev.core.exceptions import UserNotificationException
4
5
  from py_app_dev.core.logging import logger
@@ -8,9 +9,9 @@ from .config import PipelineConfig, ProjectConfig
8
9
 
9
10
 
10
11
  class ProjectSlurper:
11
- def __init__(self, project_dir: Path) -> None:
12
+ def __init__(self, project_dir: Path, config_file: Optional[str] = None) -> None:
12
13
  self.logger = logger.bind()
13
- self.artifacts_locator = ProjectArtifactsLocator(project_dir)
14
+ self.artifacts_locator = ProjectArtifactsLocator(project_dir, config_file)
14
15
  try:
15
16
  self.user_config: ProjectConfig = ProjectConfig.from_file(self.artifacts_locator.config_file)
16
17
  except FileNotFoundError:
@@ -1,55 +1,55 @@
1
- import shutil
2
- from pathlib import Path
3
- from typing import List, Optional, Union
4
-
5
- from py_app_dev.core.exceptions import UserNotificationException
6
- from py_app_dev.core.logging import logger
7
-
8
-
9
- class ProjectBuilder:
10
- def __init__(self, project_dir: Path, input_dir: Optional[Path] = None) -> None:
11
- self.project_dir = project_dir
12
- self.input_dir = input_dir if input_dir else Path(__file__).parent.joinpath("templates")
13
-
14
- self.dirs: List[Path] = []
15
- self.check_target_directory_flag = True
16
-
17
- def with_disable_target_directory_check(self) -> "ProjectBuilder":
18
- self.check_target_directory_flag = False
19
- return self
20
-
21
- def with_dir(self, dir: Union[Path, str]) -> "ProjectBuilder":
22
- self.dirs.append(self.resolve_file_path(dir))
23
- return self
24
-
25
- def resolve_file_paths(self, files: List[Path | str]) -> List[Path]:
26
- return [self.resolve_file_path(file) for file in files]
27
-
28
- def resolve_file_path(self, file: Union[Path, str]) -> Path:
29
- return self.input_dir.joinpath(file) if isinstance(file, str) else file
30
-
31
- @staticmethod
32
- def _check_target_directory(project_dir: Path) -> None:
33
- if project_dir.is_dir() and any(project_dir.iterdir()):
34
- raise UserNotificationException(f"Project directory '{project_dir}' is not empty. Use --force to override.")
35
-
36
- def build(self) -> None:
37
- if self.check_target_directory_flag:
38
- self._check_target_directory(self.project_dir)
39
- for dir in self.dirs:
40
- shutil.copytree(dir, self.project_dir, dirs_exist_ok=True)
41
-
42
-
43
- class KickstartProject:
44
- def __init__(self, project_dir: Path, force: bool = False) -> None:
45
- self.logger = logger.bind()
46
- self.project_dir = project_dir
47
- self.force = force
48
-
49
- def run(self) -> None:
50
- self.logger.info(f"Kickstart new project in '{self.project_dir.absolute().as_posix()}'")
51
- project_builder = ProjectBuilder(self.project_dir)
52
- if self.force:
53
- project_builder.with_disable_target_directory_check()
54
- project_builder.with_dir("project")
55
- project_builder.build()
1
+ import shutil
2
+ from pathlib import Path
3
+ from typing import List, Optional, Union
4
+
5
+ from py_app_dev.core.exceptions import UserNotificationException
6
+ from py_app_dev.core.logging import logger
7
+
8
+
9
+ class ProjectBuilder:
10
+ def __init__(self, project_dir: Path, input_dir: Optional[Path] = None) -> None:
11
+ self.project_dir = project_dir
12
+ self.input_dir = input_dir if input_dir else Path(__file__).parent.joinpath("templates")
13
+
14
+ self.dirs: List[Path] = []
15
+ self.check_target_directory_flag = True
16
+
17
+ def with_disable_target_directory_check(self) -> "ProjectBuilder":
18
+ self.check_target_directory_flag = False
19
+ return self
20
+
21
+ def with_dir(self, dir: Union[Path, str]) -> "ProjectBuilder":
22
+ self.dirs.append(self.resolve_file_path(dir))
23
+ return self
24
+
25
+ def resolve_file_paths(self, files: List[Path | str]) -> List[Path]:
26
+ return [self.resolve_file_path(file) for file in files]
27
+
28
+ def resolve_file_path(self, file: Union[Path, str]) -> Path:
29
+ return self.input_dir.joinpath(file) if isinstance(file, str) else file
30
+
31
+ @staticmethod
32
+ def _check_target_directory(project_dir: Path) -> None:
33
+ if project_dir.is_dir() and any(project_dir.iterdir()):
34
+ raise UserNotificationException(f"Project directory '{project_dir}' is not empty. Use --force to override.")
35
+
36
+ def build(self) -> None:
37
+ if self.check_target_directory_flag:
38
+ self._check_target_directory(self.project_dir)
39
+ for dir in self.dirs:
40
+ shutil.copytree(dir, self.project_dir, dirs_exist_ok=True)
41
+
42
+
43
+ class KickstartProject:
44
+ def __init__(self, project_dir: Path, force: bool = False) -> None:
45
+ self.logger = logger.bind()
46
+ self.project_dir = project_dir
47
+ self.force = force
48
+
49
+ def run(self) -> None:
50
+ self.logger.info(f"Kickstart new project in '{self.project_dir.absolute().as_posix()}'")
51
+ project_builder = ProjectBuilder(self.project_dir)
52
+ if self.force:
53
+ project_builder.with_disable_target_directory_check()
54
+ project_builder.with_dir("project")
55
+ project_builder.build()