pypeline-runner 1.6.0__tar.gz → 1.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/PKG-INFO +61 -33
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/README.md +60 -32
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/pyproject.toml +2 -1
- pypeline_runner-1.8.0/src/pypeline/__init__.py +1 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/domain/artifacts.py +2 -5
- pypeline_runner-1.8.0/src/pypeline/domain/pipeline.py +181 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/domain/project_slurper.py +3 -2
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/create.py +55 -55
- pypeline_runner-1.8.0/src/pypeline/kickstart/templates/project/bootstrap.py +461 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/poetry.toml +2 -2
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/pypeline.ps1 +7 -7
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/pypeline.yaml +3 -2
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/pyproject.toml +11 -9
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/steps/my_step.py +26 -26
- pypeline_runner-1.8.0/src/pypeline/kickstart/templates/project/west.yaml +10 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/main.py +2 -1
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/pypeline.py +19 -76
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/steps/create_venv.py +10 -3
- pypeline_runner-1.6.0/src/pypeline/__init__.py +0 -1
- pypeline_runner-1.6.0/src/pypeline/domain/pipeline.py +0 -109
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/LICENSE +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/__run.py +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/domain/__init__.py +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/domain/config.py +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/domain/execution_context.py +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/__init__.py +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/.gitignore +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/bootstrap.ps1 +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/kickstart/templates/project/scoopfile.json +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/py.typed +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/steps/__init__.py +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/steps/scoop_install.py +0 -0
- {pypeline_runner-1.6.0 → pypeline_runner-1.8.0}/src/pypeline/steps/west_install.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: pypeline-runner
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.8.0
|
|
4
4
|
Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: cuinixam
|
|
@@ -26,7 +26,13 @@ Project-URL: Documentation, https://pypeline-runner.readthedocs.io
|
|
|
26
26
|
Project-URL: Repository, https://github.com/cuinixam/pypeline
|
|
27
27
|
Description-Content-Type: text/markdown
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
<p align="center">
|
|
30
|
+
<a href="https://pypeline-runner.readthedocs.io">
|
|
31
|
+
<img align="center" src="https://github.com/cuinixam/pypeline/raw/main/logo.png" width="400"/>
|
|
32
|
+
</a>
|
|
33
|
+
</p>
|
|
34
|
+
|
|
35
|
+
# pypeline - Define Your CI/CD Pipeline Once, Run It Anywhere
|
|
30
36
|
|
|
31
37
|
<p align="center">
|
|
32
38
|
<a href="https://github.com/cuinixam/pypeline/actions/workflows/ci.yml?query=branch%3Amain">
|
|
@@ -58,28 +64,71 @@ Description-Content-Type: text/markdown
|
|
|
58
64
|
<img src="https://img.shields.io/pypi/l/pypeline-runner.svg?style=flat-square" alt="License">
|
|
59
65
|
</p>
|
|
60
66
|
|
|
61
|
-
Pypeline
|
|
62
|
-
The primary motivation for developing Pypeline stemmed from the need to unify and simplify the creation of build, test, and deployment pipelines that are traditionally defined separately across these platforms using GitHub workflows (YAML) and Jenkins pipelines (Jenkinsfile).
|
|
67
|
+
Pypeline lets you define your build, test, and deployment pipeline in a single YAML file and run it _consistently_ across your local development environment and _any_ CI/CD platform (GitHub Actions, Jenkins, etc.). No more platform-specific configurations – write once, run anywhere.
|
|
63
68
|
|
|
64
69
|
**Key Features**
|
|
65
70
|
|
|
66
71
|
- **Unified Pipeline Definition**: Users can define their entire pipeline in a single YAML file, eliminating the need to switch between different syntaxes and configurations for different CI/CD tools.
|
|
67
72
|
|
|
68
|
-
- **Extensibility**: Pypeline supports execution steps defined not only through
|
|
69
|
-
|
|
70
|
-
- **Execution Context**: Each step in the pipeline receives an execution context that can be updated during step execution. This allows for the sharing of information and state between steps.
|
|
73
|
+
- **Extensibility**: Pypeline supports execution steps defined not only through installed Python packages but also from local scripts.
|
|
71
74
|
|
|
72
|
-
- **
|
|
75
|
+
- **Execution Context**: Allow sharing information and state between steps. Each step in the pipeline receives an execution context that can be updated during step execution.
|
|
73
76
|
|
|
74
|
-
- **
|
|
77
|
+
- **Dependency Handling**: Every step can register its dependencies and will only be scheduled if anything has changed.
|
|
75
78
|
|
|
76
79
|
## Installation
|
|
77
80
|
|
|
78
|
-
|
|
81
|
+
Use pipx (or your favorite package manager) to install and run it in an isolated environment:
|
|
82
|
+
|
|
83
|
+
```shell
|
|
84
|
+
pipx install pypeline-runner
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
This will install the `pypeline` command globally, which you can use to run your pipelines.
|
|
79
88
|
|
|
80
|
-
|
|
89
|
+
> [!NOTE]
|
|
90
|
+
> The Python package is called `pypeline-runner` because the name `pypeline` was already taken on PyPI.
|
|
91
|
+
> The command-line interface is `pypeline`.
|
|
81
92
|
|
|
82
|
-
|
|
93
|
+
Documentation: [pypeline-runner.readthedocs.io](https://pypeline-runner.readthedocs.io)
|
|
94
|
+
|
|
95
|
+
## Walkthrough: Getting Started with Pypeline
|
|
96
|
+
|
|
97
|
+
To get started run the `init` command to create a sample project:
|
|
98
|
+
|
|
99
|
+
```shell
|
|
100
|
+
pypeline init --project-dir my-pipeline
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The example project pipeline is defined in the `pipeline.yaml` file.
|
|
104
|
+
|
|
105
|
+
```yaml
|
|
106
|
+
pipeline:
|
|
107
|
+
- step: CreateVEnv
|
|
108
|
+
module: pypeline.steps.create_venv
|
|
109
|
+
config:
|
|
110
|
+
bootstrap_script: .bootstrap/bootstrap.py
|
|
111
|
+
- step: WestInstall
|
|
112
|
+
module: pypeline.steps.west_install
|
|
113
|
+
description: Download external modules
|
|
114
|
+
- step: MyStep
|
|
115
|
+
file: steps/my_step.py
|
|
116
|
+
description: Run a custom script
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
This pipeline consists of three steps:
|
|
120
|
+
|
|
121
|
+
- `CreateVEnv`: This is a built-in step that creates a Python virtual environment.
|
|
122
|
+
- `WestInstall`: This is a built-in step that downloads external modules using the `west` tool.
|
|
123
|
+
- `MyStep`: This is a custom step that runs a script defined in the `steps/my_step.py` file.
|
|
124
|
+
|
|
125
|
+
You can run the pipeline using the `run` command:
|
|
126
|
+
|
|
127
|
+
```shell
|
|
128
|
+
pypeline run --project-dir my-pipeline
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Contributing
|
|
83
132
|
|
|
84
133
|
The project uses Poetry for dependencies management and packaging.
|
|
85
134
|
Run the `bootstrap.ps1` script to install Python and create the virtual environment.
|
|
@@ -88,43 +137,22 @@ Run the `bootstrap.ps1` script to install Python and create the virtual environm
|
|
|
88
137
|
.\bootstrap.ps1
|
|
89
138
|
```
|
|
90
139
|
|
|
91
|
-
This will also generate a `poetry.lock` file, you should track this file in version control.
|
|
92
|
-
|
|
93
140
|
To execute the test suite, call pytest inside Poetry's virtual environment via `poetry run`:
|
|
94
141
|
|
|
95
142
|
```shell
|
|
96
143
|
.venv/Scripts/poetry run pytest
|
|
97
144
|
```
|
|
98
145
|
|
|
99
|
-
Check out the Poetry documentation for more information on the available commands.
|
|
100
|
-
|
|
101
146
|
For those using [VS Code](https://code.visualstudio.com/) there are tasks defined for the most common commands:
|
|
102
147
|
|
|
103
|
-
- bootstrap
|
|
104
|
-
- install dependencies
|
|
105
148
|
- run tests
|
|
106
149
|
- run all checks configured for pre-commit
|
|
107
150
|
- generate documentation
|
|
108
151
|
|
|
109
152
|
See the `.vscode/tasks.json` for more details.
|
|
110
153
|
|
|
111
|
-
## Committing changes
|
|
112
|
-
|
|
113
154
|
This repository uses [commitlint](https://github.com/conventional-changelog/commitlint) for checking if the commit message meets the [conventional commit format](https://www.conventionalcommits.org/en).
|
|
114
155
|
|
|
115
|
-
## Contributors ✨
|
|
116
|
-
|
|
117
|
-
Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
|
|
118
|
-
|
|
119
|
-
<!-- prettier-ignore-start -->
|
|
120
|
-
<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
|
|
121
|
-
<!-- markdownlint-disable -->
|
|
122
|
-
<!-- markdownlint-enable -->
|
|
123
|
-
<!-- ALL-CONTRIBUTORS-LIST:END -->
|
|
124
|
-
<!-- prettier-ignore-end -->
|
|
125
|
-
|
|
126
|
-
This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
|
|
127
|
-
|
|
128
156
|
## Credits
|
|
129
157
|
|
|
130
158
|
This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
|
|
1
|
+
<p align="center">
|
|
2
|
+
<a href="https://pypeline-runner.readthedocs.io">
|
|
3
|
+
<img align="center" src="https://github.com/cuinixam/pypeline/raw/main/logo.png" width="400"/>
|
|
4
|
+
</a>
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
# pypeline - Define Your CI/CD Pipeline Once, Run It Anywhere
|
|
2
8
|
|
|
3
9
|
<p align="center">
|
|
4
10
|
<a href="https://github.com/cuinixam/pypeline/actions/workflows/ci.yml?query=branch%3Amain">
|
|
@@ -30,28 +36,71 @@
|
|
|
30
36
|
<img src="https://img.shields.io/pypi/l/pypeline-runner.svg?style=flat-square" alt="License">
|
|
31
37
|
</p>
|
|
32
38
|
|
|
33
|
-
Pypeline
|
|
34
|
-
The primary motivation for developing Pypeline stemmed from the need to unify and simplify the creation of build, test, and deployment pipelines that are traditionally defined separately across these platforms using GitHub workflows (YAML) and Jenkins pipelines (Jenkinsfile).
|
|
39
|
+
Pypeline lets you define your build, test, and deployment pipeline in a single YAML file and run it _consistently_ across your local development environment and _any_ CI/CD platform (GitHub Actions, Jenkins, etc.). No more platform-specific configurations – write once, run anywhere.
|
|
35
40
|
|
|
36
41
|
**Key Features**
|
|
37
42
|
|
|
38
43
|
- **Unified Pipeline Definition**: Users can define their entire pipeline in a single YAML file, eliminating the need to switch between different syntaxes and configurations for different CI/CD tools.
|
|
39
44
|
|
|
40
|
-
- **Extensibility**: Pypeline supports execution steps defined not only through
|
|
41
|
-
|
|
42
|
-
- **Execution Context**: Each step in the pipeline receives an execution context that can be updated during step execution. This allows for the sharing of information and state between steps.
|
|
45
|
+
- **Extensibility**: Pypeline supports execution steps defined not only through installed Python packages but also from local scripts.
|
|
43
46
|
|
|
44
|
-
- **
|
|
47
|
+
- **Execution Context**: Allow sharing information and state between steps. Each step in the pipeline receives an execution context that can be updated during step execution.
|
|
45
48
|
|
|
46
|
-
- **
|
|
49
|
+
- **Dependency Handling**: Every step can register its dependencies and will only be scheduled if anything has changed.
|
|
47
50
|
|
|
48
51
|
## Installation
|
|
49
52
|
|
|
50
|
-
|
|
53
|
+
Use pipx (or your favorite package manager) to install and run it in an isolated environment:
|
|
54
|
+
|
|
55
|
+
```shell
|
|
56
|
+
pipx install pypeline-runner
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
This will install the `pypeline` command globally, which you can use to run your pipelines.
|
|
51
60
|
|
|
52
|
-
|
|
61
|
+
> [!NOTE]
|
|
62
|
+
> The Python package is called `pypeline-runner` because the name `pypeline` was already taken on PyPI.
|
|
63
|
+
> The command-line interface is `pypeline`.
|
|
53
64
|
|
|
54
|
-
|
|
65
|
+
Documentation: [pypeline-runner.readthedocs.io](https://pypeline-runner.readthedocs.io)
|
|
66
|
+
|
|
67
|
+
## Walkthrough: Getting Started with Pypeline
|
|
68
|
+
|
|
69
|
+
To get started run the `init` command to create a sample project:
|
|
70
|
+
|
|
71
|
+
```shell
|
|
72
|
+
pypeline init --project-dir my-pipeline
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
The example project pipeline is defined in the `pipeline.yaml` file.
|
|
76
|
+
|
|
77
|
+
```yaml
|
|
78
|
+
pipeline:
|
|
79
|
+
- step: CreateVEnv
|
|
80
|
+
module: pypeline.steps.create_venv
|
|
81
|
+
config:
|
|
82
|
+
bootstrap_script: .bootstrap/bootstrap.py
|
|
83
|
+
- step: WestInstall
|
|
84
|
+
module: pypeline.steps.west_install
|
|
85
|
+
description: Download external modules
|
|
86
|
+
- step: MyStep
|
|
87
|
+
file: steps/my_step.py
|
|
88
|
+
description: Run a custom script
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
This pipeline consists of three steps:
|
|
92
|
+
|
|
93
|
+
- `CreateVEnv`: This is a built-in step that creates a Python virtual environment.
|
|
94
|
+
- `WestInstall`: This is a built-in step that downloads external modules using the `west` tool.
|
|
95
|
+
- `MyStep`: This is a custom step that runs a script defined in the `steps/my_step.py` file.
|
|
96
|
+
|
|
97
|
+
You can run the pipeline using the `run` command:
|
|
98
|
+
|
|
99
|
+
```shell
|
|
100
|
+
pypeline run --project-dir my-pipeline
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Contributing
|
|
55
104
|
|
|
56
105
|
The project uses Poetry for dependencies management and packaging.
|
|
57
106
|
Run the `bootstrap.ps1` script to install Python and create the virtual environment.
|
|
@@ -60,43 +109,22 @@ Run the `bootstrap.ps1` script to install Python and create the virtual environm
|
|
|
60
109
|
.\bootstrap.ps1
|
|
61
110
|
```
|
|
62
111
|
|
|
63
|
-
This will also generate a `poetry.lock` file, you should track this file in version control.
|
|
64
|
-
|
|
65
112
|
To execute the test suite, call pytest inside Poetry's virtual environment via `poetry run`:
|
|
66
113
|
|
|
67
114
|
```shell
|
|
68
115
|
.venv/Scripts/poetry run pytest
|
|
69
116
|
```
|
|
70
117
|
|
|
71
|
-
Check out the Poetry documentation for more information on the available commands.
|
|
72
|
-
|
|
73
118
|
For those using [VS Code](https://code.visualstudio.com/) there are tasks defined for the most common commands:
|
|
74
119
|
|
|
75
|
-
- bootstrap
|
|
76
|
-
- install dependencies
|
|
77
120
|
- run tests
|
|
78
121
|
- run all checks configured for pre-commit
|
|
79
122
|
- generate documentation
|
|
80
123
|
|
|
81
124
|
See the `.vscode/tasks.json` for more details.
|
|
82
125
|
|
|
83
|
-
## Committing changes
|
|
84
|
-
|
|
85
126
|
This repository uses [commitlint](https://github.com/conventional-changelog/commitlint) for checking if the commit message meets the [conventional commit format](https://www.conventionalcommits.org/en).
|
|
86
127
|
|
|
87
|
-
## Contributors ✨
|
|
88
|
-
|
|
89
|
-
Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
|
|
90
|
-
|
|
91
|
-
<!-- prettier-ignore-start -->
|
|
92
|
-
<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
|
|
93
|
-
<!-- markdownlint-disable -->
|
|
94
|
-
<!-- markdownlint-enable -->
|
|
95
|
-
<!-- ALL-CONTRIBUTORS-LIST:END -->
|
|
96
|
-
<!-- prettier-ignore-end -->
|
|
97
|
-
|
|
98
|
-
This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
|
|
99
|
-
|
|
100
128
|
## Credits
|
|
101
129
|
|
|
102
130
|
This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "pypeline-runner"
|
|
3
|
-
version = "1.
|
|
3
|
+
version = "1.8.0"
|
|
4
4
|
description = "Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines)."
|
|
5
5
|
authors = ["cuinixam <me@cuinixam.com>"]
|
|
6
6
|
license = "MIT"
|
|
@@ -36,6 +36,7 @@ pytest = "^8.3"
|
|
|
36
36
|
pytest-cov = "^6.0"
|
|
37
37
|
pre-commit = "^4.0"
|
|
38
38
|
ruff = "^0.9"
|
|
39
|
+
west = "^1.3"
|
|
39
40
|
|
|
40
41
|
[tool.poetry.group.docs.dependencies]
|
|
41
42
|
myst-parser = ">=0.16"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.8.0"
|
|
@@ -10,13 +10,10 @@ CONFIG_FILENAME = "pypeline.yaml"
|
|
|
10
10
|
class ProjectArtifactsLocator:
|
|
11
11
|
"""Provides paths to project artifacts."""
|
|
12
12
|
|
|
13
|
-
def __init__(
|
|
14
|
-
self,
|
|
15
|
-
project_root_dir: Path,
|
|
16
|
-
) -> None:
|
|
13
|
+
def __init__(self, project_root_dir: Path, config_file: Optional[str] = None) -> None:
|
|
17
14
|
self.project_root_dir = project_root_dir
|
|
18
15
|
self.build_dir = project_root_dir / "build"
|
|
19
|
-
self.config_file = project_root_dir
|
|
16
|
+
self.config_file = project_root_dir.joinpath(config_file if config_file else CONFIG_FILENAME)
|
|
20
17
|
self.external_dependencies_dir = self.build_dir / "external"
|
|
21
18
|
scripts_dir = "Scripts" if sys.platform.startswith("win32") else "bin"
|
|
22
19
|
self.venv_scripts_dir = self.project_root_dir.joinpath(".venv").joinpath(scripts_dir)
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
from abc import abstractmethod
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from importlib.util import module_from_spec, spec_from_file_location
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import (
|
|
7
|
+
Any,
|
|
8
|
+
Dict,
|
|
9
|
+
Generic,
|
|
10
|
+
Iterator,
|
|
11
|
+
List,
|
|
12
|
+
Optional,
|
|
13
|
+
OrderedDict,
|
|
14
|
+
Protocol,
|
|
15
|
+
Tuple,
|
|
16
|
+
Type,
|
|
17
|
+
TypeAlias,
|
|
18
|
+
TypeVar,
|
|
19
|
+
Union,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
from mashumaro import DataClassDictMixin
|
|
23
|
+
from py_app_dev.core.exceptions import UserNotificationException
|
|
24
|
+
from py_app_dev.core.runnable import Runnable
|
|
25
|
+
|
|
26
|
+
from .execution_context import ExecutionContext
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class PipelineStepConfig(DataClassDictMixin):
|
|
31
|
+
#: Step name or class name if file is not specified
|
|
32
|
+
step: str
|
|
33
|
+
#: Path to file with step class
|
|
34
|
+
file: Optional[str] = None
|
|
35
|
+
#: Python module with step class
|
|
36
|
+
module: Optional[str] = None
|
|
37
|
+
#: Step class name
|
|
38
|
+
class_name: Optional[str] = None
|
|
39
|
+
#: Command to run. For simple steps that don't need a class. Example: run: [echo, 'Hello World!']
|
|
40
|
+
run: Optional[Union[str, List[str]]] = None
|
|
41
|
+
#: Step description
|
|
42
|
+
description: Optional[str] = None
|
|
43
|
+
#: Step timeout in seconds
|
|
44
|
+
timeout_sec: Optional[int] = None
|
|
45
|
+
#: Custom step configuration
|
|
46
|
+
config: Optional[Dict[str, Any]] = None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
PipelineConfig: TypeAlias = Union[List[PipelineStepConfig], OrderedDict[str, List[PipelineStepConfig]]]
|
|
50
|
+
|
|
51
|
+
TPipelineStep = TypeVar("TPipelineStep", covariant=True)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class PipelineStepReference(Generic[TPipelineStep]):
|
|
56
|
+
"""Once a Step is found, keep the Step class reference to be able to instantiate it later."""
|
|
57
|
+
|
|
58
|
+
group_name: Optional[str]
|
|
59
|
+
_class: Type[TPipelineStep]
|
|
60
|
+
config: Optional[Dict[str, Any]] = None
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def name(self) -> str:
|
|
64
|
+
return self._class.__name__
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class PipelineConfigIterator:
|
|
68
|
+
"""
|
|
69
|
+
Iterates over the pipeline configuration, yielding group name and steps configuration.
|
|
70
|
+
|
|
71
|
+
This class abstracts the iteration logic for PipelineConfig, which can be:
|
|
72
|
+
- A list of steps (group name is None)
|
|
73
|
+
- An OrderedDict with group names as keys and lists of steps as values.
|
|
74
|
+
|
|
75
|
+
The iterator yields tuples of (group_name, steps).
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def __init__(self, pipeline_config: PipelineConfig) -> None:
|
|
79
|
+
self._items = pipeline_config.items() if isinstance(pipeline_config, OrderedDict) else [(None, pipeline_config)]
|
|
80
|
+
|
|
81
|
+
def __iter__(self) -> Iterator[Tuple[Optional[str], List[PipelineStepConfig]]]:
|
|
82
|
+
"""Return an iterator."""
|
|
83
|
+
yield from self._items
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class StepClassFactory(Generic[TPipelineStep], Protocol):
|
|
87
|
+
def create_step_class(self, step_config: PipelineStepConfig, project_root_dir: Path) -> Type[TPipelineStep]: ...
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class PipelineLoader(Generic[TPipelineStep]):
|
|
91
|
+
def __init__(self, pipeline_config: PipelineConfig, project_root_dir: Path, step_class_factory: Optional[StepClassFactory[TPipelineStep]] = None) -> None:
|
|
92
|
+
self.pipeline_config = pipeline_config
|
|
93
|
+
self.project_root_dir = project_root_dir
|
|
94
|
+
self.step_class_factory = step_class_factory
|
|
95
|
+
|
|
96
|
+
def load_steps_references(self) -> List[PipelineStepReference[TPipelineStep]]:
|
|
97
|
+
result = []
|
|
98
|
+
for group_name, steps_config in PipelineConfigIterator(self.pipeline_config):
|
|
99
|
+
result.extend(self._load_steps(group_name, steps_config, self.project_root_dir, self.step_class_factory))
|
|
100
|
+
return result
|
|
101
|
+
|
|
102
|
+
@staticmethod
|
|
103
|
+
def _load_steps(
|
|
104
|
+
group_name: Optional[str], steps_config: List[PipelineStepConfig], project_root_dir: Path, step_class_factory: Optional[StepClassFactory[TPipelineStep]] = None
|
|
105
|
+
) -> List[PipelineStepReference[TPipelineStep]]:
|
|
106
|
+
result = []
|
|
107
|
+
for step_config in steps_config:
|
|
108
|
+
step_class_name = step_config.class_name or step_config.step
|
|
109
|
+
if step_config.module:
|
|
110
|
+
step_class = PipelineLoader[TPipelineStep]._load_module_step(step_config.module, step_class_name)
|
|
111
|
+
elif step_config.file:
|
|
112
|
+
step_class = PipelineLoader[TPipelineStep]._load_user_step(project_root_dir.joinpath(step_config.file), step_class_name)
|
|
113
|
+
else:
|
|
114
|
+
if step_class_factory:
|
|
115
|
+
step_class = step_class_factory.create_step_class(step_config, project_root_dir)
|
|
116
|
+
else:
|
|
117
|
+
raise UserNotificationException(
|
|
118
|
+
f"Step '{step_class_name}' has no 'module' nor 'file' defined nor a custom step class factory was provided. Please check your pipeline configuration."
|
|
119
|
+
)
|
|
120
|
+
result.append(PipelineStepReference(group_name, step_class, step_config.config))
|
|
121
|
+
return result
|
|
122
|
+
|
|
123
|
+
@staticmethod
|
|
124
|
+
def _load_user_step(python_file: Path, step_class_name: str) -> Type[TPipelineStep]:
|
|
125
|
+
# Create a module specification from the file path
|
|
126
|
+
spec = spec_from_file_location(f"user__{python_file.stem}", python_file)
|
|
127
|
+
if spec and spec.loader:
|
|
128
|
+
step_module = module_from_spec(spec)
|
|
129
|
+
# Import the module
|
|
130
|
+
spec.loader.exec_module(step_module)
|
|
131
|
+
try:
|
|
132
|
+
step_class = getattr(step_module, step_class_name)
|
|
133
|
+
except AttributeError:
|
|
134
|
+
raise UserNotificationException(f"Could not load class '{step_class_name}' from file '{python_file}'. Please check your pipeline configuration.") from None
|
|
135
|
+
return step_class
|
|
136
|
+
raise UserNotificationException(f"Could not load file '{python_file}'. Please check the file for any errors.")
|
|
137
|
+
|
|
138
|
+
@staticmethod
|
|
139
|
+
def _load_module_step(module_name: str, step_class_name: str) -> Type[TPipelineStep]:
|
|
140
|
+
try:
|
|
141
|
+
module = importlib.import_module(module_name)
|
|
142
|
+
step_class = getattr(module, step_class_name)
|
|
143
|
+
except ImportError:
|
|
144
|
+
raise UserNotificationException(f"Could not load module '{module_name}'. Please check your pipeline configuration.") from None
|
|
145
|
+
except AttributeError:
|
|
146
|
+
raise UserNotificationException(f"Could not load class '{step_class_name}' from module '{module_name}'. Please check your pipeline configuration.") from None
|
|
147
|
+
return step_class
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
TExecutionContext = TypeVar("TExecutionContext", bound=ExecutionContext)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class PipelineStep(Generic[TExecutionContext], Runnable):
|
|
154
|
+
"""One can create subclasses of PipelineStep that specify the type of ExecutionContext they require."""
|
|
155
|
+
|
|
156
|
+
def __init__(self, execution_context: TExecutionContext, group_name: Optional[str], config: Optional[Dict[str, Any]] = None) -> None:
|
|
157
|
+
super().__init__(self.get_needs_dependency_management())
|
|
158
|
+
self.execution_context = execution_context
|
|
159
|
+
self.group_name = group_name
|
|
160
|
+
self.config = config
|
|
161
|
+
self.project_root_dir = self.execution_context.project_root_dir
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def output_dir(self) -> Path:
|
|
165
|
+
output_dir = self.execution_context.create_artifacts_locator().build_dir
|
|
166
|
+
if self.group_name:
|
|
167
|
+
output_dir = output_dir / self.group_name
|
|
168
|
+
return output_dir
|
|
169
|
+
|
|
170
|
+
@abstractmethod
|
|
171
|
+
def update_execution_context(self) -> None:
|
|
172
|
+
"""
|
|
173
|
+
Even if the step does not need to run ( because it is not outdated ), it can still update the execution context.
|
|
174
|
+
|
|
175
|
+
A typical use case is for steps installing software that need to provide the install directories in the execution context even if all tools are already installed.
|
|
176
|
+
"""
|
|
177
|
+
pass
|
|
178
|
+
|
|
179
|
+
def get_needs_dependency_management(self) -> bool:
|
|
180
|
+
"""If false, the step executor will not check for outdated dependencies. This is useful for steps consisting of command lines which shall always run."""
|
|
181
|
+
return True
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
|
+
from typing import Optional
|
|
2
3
|
|
|
3
4
|
from py_app_dev.core.exceptions import UserNotificationException
|
|
4
5
|
from py_app_dev.core.logging import logger
|
|
@@ -8,9 +9,9 @@ from .config import PipelineConfig, ProjectConfig
|
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
class ProjectSlurper:
|
|
11
|
-
def __init__(self, project_dir: Path) -> None:
|
|
12
|
+
def __init__(self, project_dir: Path, config_file: Optional[str] = None) -> None:
|
|
12
13
|
self.logger = logger.bind()
|
|
13
|
-
self.artifacts_locator = ProjectArtifactsLocator(project_dir)
|
|
14
|
+
self.artifacts_locator = ProjectArtifactsLocator(project_dir, config_file)
|
|
14
15
|
try:
|
|
15
16
|
self.user_config: ProjectConfig = ProjectConfig.from_file(self.artifacts_locator.config_file)
|
|
16
17
|
except FileNotFoundError:
|
|
@@ -1,55 +1,55 @@
|
|
|
1
|
-
import shutil
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
from typing import List, Optional, Union
|
|
4
|
-
|
|
5
|
-
from py_app_dev.core.exceptions import UserNotificationException
|
|
6
|
-
from py_app_dev.core.logging import logger
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class ProjectBuilder:
|
|
10
|
-
def __init__(self, project_dir: Path, input_dir: Optional[Path] = None) -> None:
|
|
11
|
-
self.project_dir = project_dir
|
|
12
|
-
self.input_dir = input_dir if input_dir else Path(__file__).parent.joinpath("templates")
|
|
13
|
-
|
|
14
|
-
self.dirs: List[Path] = []
|
|
15
|
-
self.check_target_directory_flag = True
|
|
16
|
-
|
|
17
|
-
def with_disable_target_directory_check(self) -> "ProjectBuilder":
|
|
18
|
-
self.check_target_directory_flag = False
|
|
19
|
-
return self
|
|
20
|
-
|
|
21
|
-
def with_dir(self, dir: Union[Path, str]) -> "ProjectBuilder":
|
|
22
|
-
self.dirs.append(self.resolve_file_path(dir))
|
|
23
|
-
return self
|
|
24
|
-
|
|
25
|
-
def resolve_file_paths(self, files: List[Path | str]) -> List[Path]:
|
|
26
|
-
return [self.resolve_file_path(file) for file in files]
|
|
27
|
-
|
|
28
|
-
def resolve_file_path(self, file: Union[Path, str]) -> Path:
|
|
29
|
-
return self.input_dir.joinpath(file) if isinstance(file, str) else file
|
|
30
|
-
|
|
31
|
-
@staticmethod
|
|
32
|
-
def _check_target_directory(project_dir: Path) -> None:
|
|
33
|
-
if project_dir.is_dir() and any(project_dir.iterdir()):
|
|
34
|
-
raise UserNotificationException(f"Project directory '{project_dir}' is not empty. Use --force to override.")
|
|
35
|
-
|
|
36
|
-
def build(self) -> None:
|
|
37
|
-
if self.check_target_directory_flag:
|
|
38
|
-
self._check_target_directory(self.project_dir)
|
|
39
|
-
for dir in self.dirs:
|
|
40
|
-
shutil.copytree(dir, self.project_dir, dirs_exist_ok=True)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
class KickstartProject:
|
|
44
|
-
def __init__(self, project_dir: Path, force: bool = False) -> None:
|
|
45
|
-
self.logger = logger.bind()
|
|
46
|
-
self.project_dir = project_dir
|
|
47
|
-
self.force = force
|
|
48
|
-
|
|
49
|
-
def run(self) -> None:
|
|
50
|
-
self.logger.info(f"Kickstart new project in '{self.project_dir.absolute().as_posix()}'")
|
|
51
|
-
project_builder = ProjectBuilder(self.project_dir)
|
|
52
|
-
if self.force:
|
|
53
|
-
project_builder.with_disable_target_directory_check()
|
|
54
|
-
project_builder.with_dir("project")
|
|
55
|
-
project_builder.build()
|
|
1
|
+
import shutil
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import List, Optional, Union
|
|
4
|
+
|
|
5
|
+
from py_app_dev.core.exceptions import UserNotificationException
|
|
6
|
+
from py_app_dev.core.logging import logger
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ProjectBuilder:
|
|
10
|
+
def __init__(self, project_dir: Path, input_dir: Optional[Path] = None) -> None:
|
|
11
|
+
self.project_dir = project_dir
|
|
12
|
+
self.input_dir = input_dir if input_dir else Path(__file__).parent.joinpath("templates")
|
|
13
|
+
|
|
14
|
+
self.dirs: List[Path] = []
|
|
15
|
+
self.check_target_directory_flag = True
|
|
16
|
+
|
|
17
|
+
def with_disable_target_directory_check(self) -> "ProjectBuilder":
|
|
18
|
+
self.check_target_directory_flag = False
|
|
19
|
+
return self
|
|
20
|
+
|
|
21
|
+
def with_dir(self, dir: Union[Path, str]) -> "ProjectBuilder":
|
|
22
|
+
self.dirs.append(self.resolve_file_path(dir))
|
|
23
|
+
return self
|
|
24
|
+
|
|
25
|
+
def resolve_file_paths(self, files: List[Path | str]) -> List[Path]:
|
|
26
|
+
return [self.resolve_file_path(file) for file in files]
|
|
27
|
+
|
|
28
|
+
def resolve_file_path(self, file: Union[Path, str]) -> Path:
|
|
29
|
+
return self.input_dir.joinpath(file) if isinstance(file, str) else file
|
|
30
|
+
|
|
31
|
+
@staticmethod
|
|
32
|
+
def _check_target_directory(project_dir: Path) -> None:
|
|
33
|
+
if project_dir.is_dir() and any(project_dir.iterdir()):
|
|
34
|
+
raise UserNotificationException(f"Project directory '{project_dir}' is not empty. Use --force to override.")
|
|
35
|
+
|
|
36
|
+
def build(self) -> None:
|
|
37
|
+
if self.check_target_directory_flag:
|
|
38
|
+
self._check_target_directory(self.project_dir)
|
|
39
|
+
for dir in self.dirs:
|
|
40
|
+
shutil.copytree(dir, self.project_dir, dirs_exist_ok=True)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class KickstartProject:
|
|
44
|
+
def __init__(self, project_dir: Path, force: bool = False) -> None:
|
|
45
|
+
self.logger = logger.bind()
|
|
46
|
+
self.project_dir = project_dir
|
|
47
|
+
self.force = force
|
|
48
|
+
|
|
49
|
+
def run(self) -> None:
|
|
50
|
+
self.logger.info(f"Kickstart new project in '{self.project_dir.absolute().as_posix()}'")
|
|
51
|
+
project_builder = ProjectBuilder(self.project_dir)
|
|
52
|
+
if self.force:
|
|
53
|
+
project_builder.with_disable_target_directory_check()
|
|
54
|
+
project_builder.with_dir("project")
|
|
55
|
+
project_builder.build()
|