nextpipe 0.1.0.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nextpipe-0.1.0.dev0/.github/workflows/publish.yml +141 -0
- nextpipe-0.1.0.dev0/.github/workflows/python-lint.yml +24 -0
- nextpipe-0.1.0.dev0/.github/workflows/python-test.yml +24 -0
- nextpipe-0.1.0.dev0/.gitignore +165 -0
- nextpipe-0.1.0.dev0/.prettierrc.yml +1 -0
- nextpipe-0.1.0.dev0/LICENSE.md +87 -0
- nextpipe-0.1.0.dev0/PKG-INFO +116 -0
- nextpipe-0.1.0.dev0/README.md +3 -0
- nextpipe-0.1.0.dev0/nextpipe/__about__.py +1 -0
- nextpipe-0.1.0.dev0/nextpipe/__init__.py +13 -0
- nextpipe-0.1.0.dev0/nextpipe/decorators.py +166 -0
- nextpipe-0.1.0.dev0/nextpipe/flow.py +281 -0
- nextpipe-0.1.0.dev0/nextpipe/utils.py +68 -0
- nextpipe-0.1.0.dev0/nextpipe.code-workspace +12 -0
- nextpipe-0.1.0.dev0/pyproject.toml +72 -0
- nextpipe-0.1.0.dev0/tests/__init__.py +0 -0
- nextpipe-0.1.0.dev0/tests/test_version.py +10 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
run-name: Publish ${{ inputs.VERSION }} (pre-release - ${{ inputs.IS_PRE_RELEASE }}) by @${{ github.actor }} from ${{ github.ref_name }}
|
|
3
|
+
|
|
4
|
+
on:
|
|
5
|
+
workflow_dispatch:
|
|
6
|
+
inputs:
|
|
7
|
+
VERSION:
|
|
8
|
+
description: "The version to release"
|
|
9
|
+
required: true
|
|
10
|
+
IS_PRE_RELEASE:
|
|
11
|
+
description: "It IS a pre-release"
|
|
12
|
+
required: true
|
|
13
|
+
default: false
|
|
14
|
+
type: boolean
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
bump: # This job is used to bump the version and create a release
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
env:
|
|
20
|
+
VERSION: ${{ inputs.VERSION }}
|
|
21
|
+
GH_TOKEN: ${{ github.token }}
|
|
22
|
+
SSH_AUTH_SOCK: /tmp/ssh_agent.sock
|
|
23
|
+
permissions:
|
|
24
|
+
contents: write
|
|
25
|
+
steps:
|
|
26
|
+
- name: set up Python
|
|
27
|
+
uses: actions/setup-python@v5
|
|
28
|
+
with:
|
|
29
|
+
python-version: "3.12"
|
|
30
|
+
|
|
31
|
+
- name: install dependencies
|
|
32
|
+
run: |
|
|
33
|
+
pip install --upgrade pip
|
|
34
|
+
pip install build hatch
|
|
35
|
+
|
|
36
|
+
- name: configure git with the bot credentials
|
|
37
|
+
run: |
|
|
38
|
+
mkdir -p ~/.ssh
|
|
39
|
+
ssh-keyscan github.com >> ~/.ssh/known_hosts
|
|
40
|
+
ssh-agent -a $SSH_AUTH_SOCK > /dev/null
|
|
41
|
+
ssh-add - <<< "${{ secrets.NEXTMVBOT_SSH_KEY }}"
|
|
42
|
+
|
|
43
|
+
echo "${{ secrets.NEXTMVBOT_SIGNING_KEY }}" > ~/.ssh/signing.key
|
|
44
|
+
chmod 600 ~/.ssh/signing.key
|
|
45
|
+
|
|
46
|
+
git config --global user.name "nextmv-bot"
|
|
47
|
+
git config --global user.email "tech+gh-nextmv-bot@nextmv.io"
|
|
48
|
+
git config --global gpg.format ssh
|
|
49
|
+
git config --global user.signingkey ~/.ssh/signing.key
|
|
50
|
+
|
|
51
|
+
git clone git@github.com:nextmv-io/nextpipe.git
|
|
52
|
+
cd nextpipe
|
|
53
|
+
git switch ${{ github.ref_name }}
|
|
54
|
+
|
|
55
|
+
- name: upgrade version with hatch
|
|
56
|
+
run: hatch version ${{ env.VERSION }}
|
|
57
|
+
working-directory: ./nextpipe
|
|
58
|
+
|
|
59
|
+
- name: commit new version
|
|
60
|
+
run: |
|
|
61
|
+
git add nextpipe/__about__.py
|
|
62
|
+
git commit -S -m "Bump version to $VERSION"
|
|
63
|
+
git push
|
|
64
|
+
git tag $VERSION
|
|
65
|
+
git push origin $VERSION
|
|
66
|
+
working-directory: ./nextpipe
|
|
67
|
+
|
|
68
|
+
- name: create release
|
|
69
|
+
run: |
|
|
70
|
+
PRERELEASE_FLAG=""
|
|
71
|
+
if [ ${{ inputs.IS_PRE_RELEASE }} = true ]; then
|
|
72
|
+
PRERELEASE_FLAG="--prerelease"
|
|
73
|
+
fi
|
|
74
|
+
|
|
75
|
+
gh release create $VERSION \
|
|
76
|
+
--verify-tag \
|
|
77
|
+
--generate-notes \
|
|
78
|
+
--title $VERSION $PRERELEASE_FLAG
|
|
79
|
+
working-directory: ./nextpipe
|
|
80
|
+
|
|
81
|
+
- name: ensure passing build
|
|
82
|
+
run: python -m build
|
|
83
|
+
working-directory: ./nextpipe
|
|
84
|
+
|
|
85
|
+
publish: # This job is used to publish the release to PyPI/TestPyPI
|
|
86
|
+
runs-on: ubuntu-latest
|
|
87
|
+
needs: bump
|
|
88
|
+
strategy:
|
|
89
|
+
matrix:
|
|
90
|
+
include:
|
|
91
|
+
- target-env: pypi
|
|
92
|
+
target-url: https://pypi.org/p/nextpipe
|
|
93
|
+
- target-env: testpypi
|
|
94
|
+
target-url: https://test.pypi.org/p/nextpipe
|
|
95
|
+
environment:
|
|
96
|
+
name: ${{ matrix.target-env }}
|
|
97
|
+
url: ${{ matrix.target-url }}
|
|
98
|
+
permissions:
|
|
99
|
+
contents: read
|
|
100
|
+
id-token: write # This is required for trusted publishing to PyPI
|
|
101
|
+
steps:
|
|
102
|
+
- name: git clone ${{ github.ref_name }}
|
|
103
|
+
uses: actions/checkout@v4
|
|
104
|
+
with:
|
|
105
|
+
ref: ${{ github.ref_name }}
|
|
106
|
+
|
|
107
|
+
- name: set up Python
|
|
108
|
+
uses: actions/setup-python@v5
|
|
109
|
+
with:
|
|
110
|
+
python-version: "3.12"
|
|
111
|
+
|
|
112
|
+
- name: install dependencies
|
|
113
|
+
run: |
|
|
114
|
+
pip install --upgrade pip
|
|
115
|
+
pip install build hatch
|
|
116
|
+
|
|
117
|
+
- name: build binary wheel and source tarball
|
|
118
|
+
run: python -m build
|
|
119
|
+
|
|
120
|
+
- name: Publish package distributions to PyPI
|
|
121
|
+
if: ${{ matrix.target-env == 'pypi' }}
|
|
122
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
123
|
+
with:
|
|
124
|
+
packages-dir: ./dist
|
|
125
|
+
|
|
126
|
+
- name: Publish package distributions to TestPyPI
|
|
127
|
+
if: ${{ matrix.target-env == 'testpypi' }}
|
|
128
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
129
|
+
with:
|
|
130
|
+
repository-url: https://test.pypi.org/legacy/
|
|
131
|
+
packages-dir: ./dist
|
|
132
|
+
|
|
133
|
+
notify:
|
|
134
|
+
runs-on: ubuntu-latest
|
|
135
|
+
needs: publish
|
|
136
|
+
if: ${{ needs.publish.result == 'success' && inputs.IS_PRE_RELEASE == false }}
|
|
137
|
+
steps:
|
|
138
|
+
- name: notify slack
|
|
139
|
+
run: |
|
|
140
|
+
export DATA="{\"text\":\"Release notification - nextpipe ${{ inputs.VERSION }} (see <https://github.com/nextmv-io/nextpipe/releases/${{ inputs.VERSION }}|release notes> / <https://pypi.org/project/nextpipe|PyPI>)\"}"
|
|
141
|
+
curl -X POST -H 'Content-type: application/json' --data "$DATA" ${{ secrets.SLACK_URL_MISSION_CONTROL }}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: python lint
|
|
2
|
+
on: [push]
|
|
3
|
+
jobs:
|
|
4
|
+
python-lint:
|
|
5
|
+
runs-on: ubuntu-latest
|
|
6
|
+
strategy:
|
|
7
|
+
matrix:
|
|
8
|
+
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
|
|
9
|
+
steps:
|
|
10
|
+
- name: git clone
|
|
11
|
+
uses: actions/checkout@v4
|
|
12
|
+
|
|
13
|
+
- name: set up Python ${{ matrix.python-version }}
|
|
14
|
+
uses: actions/setup-python@v5
|
|
15
|
+
with:
|
|
16
|
+
python-version: ${{ matrix.python-version }}
|
|
17
|
+
|
|
18
|
+
- name: install dependencies
|
|
19
|
+
run: |
|
|
20
|
+
python -m pip install --upgrade pip
|
|
21
|
+
pip install .[dev]
|
|
22
|
+
|
|
23
|
+
- name: lint with ruff
|
|
24
|
+
run: ruff check --output-format=github .
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: python test
|
|
2
|
+
on: [push]
|
|
3
|
+
jobs:
|
|
4
|
+
python-test:
|
|
5
|
+
runs-on: ubuntu-latest
|
|
6
|
+
strategy:
|
|
7
|
+
matrix:
|
|
8
|
+
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
|
|
9
|
+
steps:
|
|
10
|
+
- name: git clone
|
|
11
|
+
uses: actions/checkout@v4
|
|
12
|
+
|
|
13
|
+
- name: set up Python ${{ matrix.python-version }}
|
|
14
|
+
uses: actions/setup-python@v5
|
|
15
|
+
with:
|
|
16
|
+
python-version: ${{ matrix.python-version }}
|
|
17
|
+
|
|
18
|
+
- name: install dependencies
|
|
19
|
+
run: |
|
|
20
|
+
python -m pip install --upgrade pip
|
|
21
|
+
pip install .[dev]
|
|
22
|
+
|
|
23
|
+
- name: unit tests
|
|
24
|
+
run: python -m unittest
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py,cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
#Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# poetry
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
102
|
+
#poetry.lock
|
|
103
|
+
|
|
104
|
+
# pdm
|
|
105
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
106
|
+
#pdm.lock
|
|
107
|
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
|
108
|
+
# in version control.
|
|
109
|
+
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
|
110
|
+
.pdm.toml
|
|
111
|
+
.pdm-python
|
|
112
|
+
.pdm-build/
|
|
113
|
+
|
|
114
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
115
|
+
__pypackages__/
|
|
116
|
+
|
|
117
|
+
# Celery stuff
|
|
118
|
+
celerybeat-schedule
|
|
119
|
+
celerybeat.pid
|
|
120
|
+
|
|
121
|
+
# SageMath parsed files
|
|
122
|
+
*.sage.py
|
|
123
|
+
|
|
124
|
+
# Environments
|
|
125
|
+
.env
|
|
126
|
+
.venv
|
|
127
|
+
env/
|
|
128
|
+
venv/
|
|
129
|
+
ENV/
|
|
130
|
+
env.bak/
|
|
131
|
+
venv.bak/
|
|
132
|
+
|
|
133
|
+
# Spyder project settings
|
|
134
|
+
.spyderproject
|
|
135
|
+
.spyproject
|
|
136
|
+
|
|
137
|
+
# Rope project settings
|
|
138
|
+
.ropeproject
|
|
139
|
+
|
|
140
|
+
# mkdocs documentation
|
|
141
|
+
/site
|
|
142
|
+
|
|
143
|
+
# mypy
|
|
144
|
+
.mypy_cache/
|
|
145
|
+
.dmypy.json
|
|
146
|
+
dmypy.json
|
|
147
|
+
|
|
148
|
+
# Pyre type checker
|
|
149
|
+
.pyre/
|
|
150
|
+
|
|
151
|
+
# pytype static type analyzer
|
|
152
|
+
.pytype/
|
|
153
|
+
|
|
154
|
+
# Cython debug symbols
|
|
155
|
+
cython_debug/
|
|
156
|
+
|
|
157
|
+
# PyCharm / Jetbrains
|
|
158
|
+
.idea/
|
|
159
|
+
|
|
160
|
+
# vscode
|
|
161
|
+
.vscode/
|
|
162
|
+
|
|
163
|
+
# Don't include key files
|
|
164
|
+
key*.txt
|
|
165
|
+
key*.json
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
tabWidth: 2
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# LICENSE
|
|
2
|
+
|
|
3
|
+
Business Source License 1.1
|
|
4
|
+
|
|
5
|
+
Parameters
|
|
6
|
+
|
|
7
|
+
Licensor: nextmv.io inc
|
|
8
|
+
Licensed Work: nextpipe
|
|
9
|
+
|
|
10
|
+
Change Date: Four years from the date the Licensed Work is published.
|
|
11
|
+
Change License: GPLv3
|
|
12
|
+
|
|
13
|
+
For information about alternative licensing arrangements for the Software,
|
|
14
|
+
please email info@nextmv.io.
|
|
15
|
+
|
|
16
|
+
Notice
|
|
17
|
+
|
|
18
|
+
The Business Source License (this document, or the “License”) is not an Open
|
|
19
|
+
Source license. However, the Licensed Work will eventually be made available
|
|
20
|
+
under an Open Source License, as stated in this License.
|
|
21
|
+
|
|
22
|
+
License text copyright © 2023 MariaDB plc, All Rights Reserved. “Business Source
|
|
23
|
+
License” is a trademark of MariaDB plc.
|
|
24
|
+
|
|
25
|
+
-----------------------------------------------------------------------------
|
|
26
|
+
|
|
27
|
+
## Terms
|
|
28
|
+
|
|
29
|
+
The Licensor hereby grants you the right to copy, modify, create derivative
|
|
30
|
+
works, redistribute, and make non-production use of the Licensed Work. The
|
|
31
|
+
Licensor may make an Additional Use Grant, above, permitting limited production
|
|
32
|
+
use.
|
|
33
|
+
|
|
34
|
+
Effective on the Change Date, or the fourth anniversary of the first publicly
|
|
35
|
+
available distribution of a specific version of the Licensed Work under this
|
|
36
|
+
License, whichever comes first, the Licensor hereby grants you rights under the
|
|
37
|
+
terms of the Change License, and the rights granted in the paragraph above
|
|
38
|
+
terminate.
|
|
39
|
+
|
|
40
|
+
If your use of the Licensed Work does not comply with the requirements currently
|
|
41
|
+
in effect as described in this License, you must purchase a commercial license
|
|
42
|
+
from the Licensor, its affiliated entities, or authorized resellers, or you must
|
|
43
|
+
refrain from using the Licensed Work.
|
|
44
|
+
|
|
45
|
+
All copies of the original and modified Licensed Work, and derivative works of
|
|
46
|
+
the Licensed Work, are subject to this License. This License applies separately
|
|
47
|
+
for each version of the Licensed Work and the Change Date may vary for each
|
|
48
|
+
version of the Licensed Work released by Licensor.
|
|
49
|
+
|
|
50
|
+
You must conspicuously display this License on each original or modified copy of
|
|
51
|
+
the Licensed Work. If you receive the Licensed Work in original or modified form
|
|
52
|
+
from a third party, the terms and conditions set forth in this License apply to
|
|
53
|
+
your use of that work.
|
|
54
|
+
|
|
55
|
+
Any use of the Licensed Work in violation of this License will automatically
|
|
56
|
+
terminate your rights under this License for the current and all other versions
|
|
57
|
+
of the Licensed Work.
|
|
58
|
+
|
|
59
|
+
This License does not grant you any right in any trademark or logo of Licensor
|
|
60
|
+
or its affiliates (provided that you may use a trademark or logo of Licensor as
|
|
61
|
+
expressly required by this License).TO THE EXTENT PERMITTED BY APPLICABLE LAW,
|
|
62
|
+
THE LICENSED WORK IS PROVIDED ON AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL
|
|
63
|
+
WARRANTIES AND CONDITIONS, EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION)
|
|
64
|
+
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
|
|
65
|
+
NON-INFRINGEMENT, AND TITLE. MariaDB hereby grants you permission to use this
|
|
66
|
+
License’s text to license your works, and to refer to it using the trademark
|
|
67
|
+
“Business Source License”, as long as you comply with the Covenants of Licensor
|
|
68
|
+
below.
|
|
69
|
+
|
|
70
|
+
## Covenants of Licensor
|
|
71
|
+
|
|
72
|
+
In consideration of the right to use this License’s text and the “Business
|
|
73
|
+
Source License” name and trademark, Licensor covenants to MariaDB, and to all
|
|
74
|
+
other recipients of the licensed work to be provided by Licensor:
|
|
75
|
+
|
|
76
|
+
To specify as the Change License the GPL Version 2.0 or any later version, or a
|
|
77
|
+
license that is compatible with GPL Version 2.0 or a later version, where
|
|
78
|
+
“compatible” means that software provided under the Change License can be
|
|
79
|
+
included in a program with software provided under GPL Version 2.0 or a later
|
|
80
|
+
version. Licensor may specify additional Change Licenses without limitation. To
|
|
81
|
+
either: (a) specify an additional grant of rights to use that does not impose
|
|
82
|
+
any additional restriction on the right granted in this License, as the
|
|
83
|
+
Additional Use Grant; or (b) insert the text “None” to specify a Change Date.
|
|
84
|
+
Not to modify this License in any other way.
|
|
85
|
+
|
|
86
|
+
License text copyright © 2023 MariaDB plc, All Rights Reserved. “Business Source
|
|
87
|
+
License” is a trademark of MariaDB plc.
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: nextpipe
|
|
3
|
+
Version: 0.1.0.dev0
|
|
4
|
+
Summary: Framework for Decision Pipeline modeling and execution
|
|
5
|
+
Project-URL: Homepage, https://www.nextmv.io
|
|
6
|
+
Project-URL: Documentation, https://www.nextmv.io/docs
|
|
7
|
+
Project-URL: Repository, https://github.com/nextmv-io/nextpipe
|
|
8
|
+
Author-email: Nextmv <tech@nextmv.io>
|
|
9
|
+
Maintainer-email: Nextmv <tech@nextmv.io>
|
|
10
|
+
License: # LICENSE
|
|
11
|
+
|
|
12
|
+
Business Source License 1.1
|
|
13
|
+
|
|
14
|
+
Parameters
|
|
15
|
+
|
|
16
|
+
Licensor: nextmv.io inc
|
|
17
|
+
Licensed Work: nextpipe
|
|
18
|
+
|
|
19
|
+
Change Date: Four years from the date the Licensed Work is published.
|
|
20
|
+
Change License: GPLv3
|
|
21
|
+
|
|
22
|
+
For information about alternative licensing arrangements for the Software,
|
|
23
|
+
please email info@nextmv.io.
|
|
24
|
+
|
|
25
|
+
Notice
|
|
26
|
+
|
|
27
|
+
The Business Source License (this document, or the “License”) is not an Open
|
|
28
|
+
Source license. However, the Licensed Work will eventually be made available
|
|
29
|
+
under an Open Source License, as stated in this License.
|
|
30
|
+
|
|
31
|
+
License text copyright © 2023 MariaDB plc, All Rights Reserved. “Business Source
|
|
32
|
+
License” is a trademark of MariaDB plc.
|
|
33
|
+
|
|
34
|
+
-----------------------------------------------------------------------------
|
|
35
|
+
|
|
36
|
+
## Terms
|
|
37
|
+
|
|
38
|
+
The Licensor hereby grants you the right to copy, modify, create derivative
|
|
39
|
+
works, redistribute, and make non-production use of the Licensed Work. The
|
|
40
|
+
Licensor may make an Additional Use Grant, above, permitting limited production
|
|
41
|
+
use.
|
|
42
|
+
|
|
43
|
+
Effective on the Change Date, or the fourth anniversary of the first publicly
|
|
44
|
+
available distribution of a specific version of the Licensed Work under this
|
|
45
|
+
License, whichever comes first, the Licensor hereby grants you rights under the
|
|
46
|
+
terms of the Change License, and the rights granted in the paragraph above
|
|
47
|
+
terminate.
|
|
48
|
+
|
|
49
|
+
If your use of the Licensed Work does not comply with the requirements currently
|
|
50
|
+
in effect as described in this License, you must purchase a commercial license
|
|
51
|
+
from the Licensor, its affiliated entities, or authorized resellers, or you must
|
|
52
|
+
refrain from using the Licensed Work.
|
|
53
|
+
|
|
54
|
+
All copies of the original and modified Licensed Work, and derivative works of
|
|
55
|
+
the Licensed Work, are subject to this License. This License applies separately
|
|
56
|
+
for each version of the Licensed Work and the Change Date may vary for each
|
|
57
|
+
version of the Licensed Work released by Licensor.
|
|
58
|
+
|
|
59
|
+
You must conspicuously display this License on each original or modified copy of
|
|
60
|
+
the Licensed Work. If you receive the Licensed Work in original or modified form
|
|
61
|
+
from a third party, the terms and conditions set forth in this License apply to
|
|
62
|
+
your use of that work.
|
|
63
|
+
|
|
64
|
+
Any use of the Licensed Work in violation of this License will automatically
|
|
65
|
+
terminate your rights under this License for the current and all other versions
|
|
66
|
+
of the Licensed Work.
|
|
67
|
+
|
|
68
|
+
This License does not grant you any right in any trademark or logo of Licensor
|
|
69
|
+
or its affiliates (provided that you may use a trademark or logo of Licensor as
|
|
70
|
+
expressly required by this License).TO THE EXTENT PERMITTED BY APPLICABLE LAW,
|
|
71
|
+
THE LICENSED WORK IS PROVIDED ON AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL
|
|
72
|
+
WARRANTIES AND CONDITIONS, EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION)
|
|
73
|
+
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
|
|
74
|
+
NON-INFRINGEMENT, AND TITLE. MariaDB hereby grants you permission to use this
|
|
75
|
+
License’s text to license your works, and to refer to it using the trademark
|
|
76
|
+
“Business Source License”, as long as you comply with the Covenants of Licensor
|
|
77
|
+
below.
|
|
78
|
+
|
|
79
|
+
## Covenants of Licensor
|
|
80
|
+
|
|
81
|
+
In consideration of the right to use this License’s text and the “Business
|
|
82
|
+
Source License” name and trademark, Licensor covenants to MariaDB, and to all
|
|
83
|
+
other recipients of the licensed work to be provided by Licensor:
|
|
84
|
+
|
|
85
|
+
To specify as the Change License the GPL Version 2.0 or any later version, or a
|
|
86
|
+
license that is compatible with GPL Version 2.0 or a later version, where
|
|
87
|
+
“compatible” means that software provided under the Change License can be
|
|
88
|
+
included in a program with software provided under GPL Version 2.0 or a later
|
|
89
|
+
version. Licensor may specify additional Change Licenses without limitation. To
|
|
90
|
+
either: (a) specify an additional grant of rights to use that does not impose
|
|
91
|
+
any additional restriction on the right granted in this License, as the
|
|
92
|
+
Additional Use Grant; or (b) insert the text “None” to specify a Change Date.
|
|
93
|
+
Not to modify this License in any other way.
|
|
94
|
+
|
|
95
|
+
License text copyright © 2023 MariaDB plc, All Rights Reserved. “Business Source
|
|
96
|
+
License” is a trademark of MariaDB plc.
|
|
97
|
+
License-File: LICENSE.md
|
|
98
|
+
Keywords: decision automation,decision engineering,decision pipelines,decision science,decision workflows,decisions,nextmv,operations research,optimization,pipelines,workflows
|
|
99
|
+
Classifier: Operating System :: OS Independent
|
|
100
|
+
Classifier: Programming Language :: Python :: 3
|
|
101
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
102
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
103
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
104
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
105
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
106
|
+
Requires-Python: >=3.8
|
|
107
|
+
Requires-Dist: nextmv>=0.12.0
|
|
108
|
+
Requires-Dist: pathos>=0.3.2
|
|
109
|
+
Requires-Dist: requests>=2.31.0
|
|
110
|
+
Provides-Extra: dev
|
|
111
|
+
Requires-Dist: ruff>=0.6.4; extra == 'dev'
|
|
112
|
+
Description-Content-Type: text/markdown
|
|
113
|
+
|
|
114
|
+
# nextpipe
|
|
115
|
+
|
|
116
|
+
Framework for Decision Pipeline modeling and execution
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "v0.1.0.dev0"
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Framework for Decision Pipeline modeling and execution."""
|
|
2
|
+
|
|
3
|
+
from .__about__ import __version__
|
|
4
|
+
from .decorators import app as app
|
|
5
|
+
from .decorators import needs as needs
|
|
6
|
+
from .decorators import optional as optional
|
|
7
|
+
from .decorators import repeat as repeat
|
|
8
|
+
from .decorators import step as step
|
|
9
|
+
from .flow import FlowGraph as FlowGraph
|
|
10
|
+
from .flow import FlowSpec as FlowSpec
|
|
11
|
+
|
|
12
|
+
VERSION = __version__
|
|
13
|
+
"""The version of the nextpipe package."""
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
from functools import wraps
|
|
3
|
+
from typing import Callable, Dict, List
|
|
4
|
+
|
|
5
|
+
from pathos.multiprocessing import ProcessingPool as Pool
|
|
6
|
+
|
|
7
|
+
from . import utils
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class InputType(Enum):
|
|
11
|
+
JSON = 1
|
|
12
|
+
FILES = 2
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class StepType(Enum):
|
|
16
|
+
DEFAULT = 1
|
|
17
|
+
APP = 2
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Step:
|
|
21
|
+
def __init__(self, function: callable):
|
|
22
|
+
self.function = function
|
|
23
|
+
self.type = StepType.DEFAULT
|
|
24
|
+
self._inputs = {}
|
|
25
|
+
self._output = None
|
|
26
|
+
|
|
27
|
+
def __repr__(self):
|
|
28
|
+
b = f"Step({self.function.__name__}"
|
|
29
|
+
if hasattr(self, "needs"):
|
|
30
|
+
b += f", {self.needs}"
|
|
31
|
+
if hasattr(self, "repeat"):
|
|
32
|
+
b += f", {self.repeat}"
|
|
33
|
+
if hasattr(self, "app"):
|
|
34
|
+
b += f", {self.app}"
|
|
35
|
+
return b + ")"
|
|
36
|
+
|
|
37
|
+
def get_name(self):
|
|
38
|
+
return self.function.__name__
|
|
39
|
+
|
|
40
|
+
def is_needs(self):
|
|
41
|
+
return hasattr(self, "needs")
|
|
42
|
+
|
|
43
|
+
def skip(self):
|
|
44
|
+
return hasattr(self, "optional") and not self.optional.condition(self)
|
|
45
|
+
|
|
46
|
+
def is_repeat(self):
|
|
47
|
+
return hasattr(self, "repeat")
|
|
48
|
+
|
|
49
|
+
def is_app(self):
|
|
50
|
+
return self.type == StepType.APP
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class Needs:
|
|
54
|
+
def __init__(self, predecessors: List[callable]):
|
|
55
|
+
self.predecessors = predecessors
|
|
56
|
+
|
|
57
|
+
def __repr__(self):
|
|
58
|
+
return f"StepNeeds({','.join([p.step.get_name() for p in self.predecessors])})"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class Optional:
|
|
62
|
+
def __init__(self, condition: callable):
|
|
63
|
+
self.condition = condition
|
|
64
|
+
|
|
65
|
+
def __repr__(self):
|
|
66
|
+
return f"StepOnlyIf({self.condition})"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class Repeat:
|
|
70
|
+
def __init__(self, repetitions: int):
|
|
71
|
+
self.repetitions = repetitions
|
|
72
|
+
|
|
73
|
+
def __repr__(self):
|
|
74
|
+
return f"StepRepeat({self.repetitions})"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class App:
|
|
78
|
+
def __init__(
|
|
79
|
+
self,
|
|
80
|
+
app_id: str,
|
|
81
|
+
instance_id: str = "devint",
|
|
82
|
+
input_type: InputType = InputType.JSON,
|
|
83
|
+
parameters: Dict[str, any] = None,
|
|
84
|
+
):
|
|
85
|
+
self.app_id = app_id
|
|
86
|
+
self.instance_id = instance_id
|
|
87
|
+
self.parameters = parameters if parameters else {}
|
|
88
|
+
self.input_type = input_type
|
|
89
|
+
|
|
90
|
+
def __repr__(self):
|
|
91
|
+
return f"StepRun({self.app_id}, {self.instance_id}, {self.parameters}, {self.input_type})"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def needs(predecessors: List[callable]):
|
|
95
|
+
def decorator(function):
|
|
96
|
+
function.step.needs = Needs(predecessors)
|
|
97
|
+
return function
|
|
98
|
+
|
|
99
|
+
return decorator
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def optional(condition: Callable[[Step], bool]):
|
|
103
|
+
def decorator(function):
|
|
104
|
+
function.step.optional = Optional(condition)
|
|
105
|
+
return function
|
|
106
|
+
|
|
107
|
+
return decorator
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def repeat(repetitions: int):
|
|
111
|
+
def decorator(function):
|
|
112
|
+
@wraps(function)
|
|
113
|
+
def wrapper(*args, **kwargs):
|
|
114
|
+
inputs = [(args, kwargs) for _ in range(repetitions)]
|
|
115
|
+
outputs = []
|
|
116
|
+
with Pool(repetitions) as pool:
|
|
117
|
+
outputs = pool.map(utils.wrap_func(function), inputs)
|
|
118
|
+
return outputs
|
|
119
|
+
|
|
120
|
+
wrapper.step.repeat = Repeat(repetitions)
|
|
121
|
+
|
|
122
|
+
return wrapper
|
|
123
|
+
|
|
124
|
+
return decorator
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def app(
|
|
128
|
+
app_id: str,
|
|
129
|
+
instance_id: str = "default",
|
|
130
|
+
parameters: Dict[str, any] = None,
|
|
131
|
+
input_type: InputType = InputType.JSON,
|
|
132
|
+
):
|
|
133
|
+
def decorator(function):
|
|
134
|
+
@wraps(function)
|
|
135
|
+
def wrapper(*args, **kwargs):
|
|
136
|
+
utils.log(f"Running {app_id} version {instance_id}")
|
|
137
|
+
return function(*args, **kwargs)
|
|
138
|
+
|
|
139
|
+
# We need to make sure that all values of the parameters are converted to strings,
|
|
140
|
+
# as no other types are allowed in the JSON.
|
|
141
|
+
converted_parameters = utils.convert_to_string_values(parameters if parameters else {})
|
|
142
|
+
|
|
143
|
+
wrapper.step.app = App(
|
|
144
|
+
app_id=app_id,
|
|
145
|
+
instance_id=instance_id,
|
|
146
|
+
parameters=converted_parameters,
|
|
147
|
+
input_type=input_type,
|
|
148
|
+
)
|
|
149
|
+
wrapper.step.type = StepType.APP
|
|
150
|
+
|
|
151
|
+
return wrapper
|
|
152
|
+
|
|
153
|
+
return decorator
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def step(function):
|
|
157
|
+
@wraps(function)
|
|
158
|
+
def wrapper(*args, **kwargs):
|
|
159
|
+
utils.log(f"Entering {function.__name__}")
|
|
160
|
+
ret_val = function(*args, **kwargs)
|
|
161
|
+
utils.log(f"Finished {function.__name__}")
|
|
162
|
+
return ret_val
|
|
163
|
+
|
|
164
|
+
wrapper.step = Step(function)
|
|
165
|
+
wrapper.is_step = True
|
|
166
|
+
return wrapper
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import base64
|
|
3
|
+
import collections
|
|
4
|
+
import inspect
|
|
5
|
+
import io
|
|
6
|
+
import time
|
|
7
|
+
from typing import List, Optional, Union
|
|
8
|
+
|
|
9
|
+
from nextmv.cloud import Application, Client, StatusV2
|
|
10
|
+
from pathos.multiprocessing import ProcessingPool as Pool
|
|
11
|
+
|
|
12
|
+
from . import utils
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class DAGNode:
|
|
16
|
+
def __init__(self, step_function, step_definition, docstring):
|
|
17
|
+
self.step_function = step_function
|
|
18
|
+
self.step = step_definition
|
|
19
|
+
self.docstring = docstring
|
|
20
|
+
self.successors: List[DAGNode] = []
|
|
21
|
+
|
|
22
|
+
def __repr__(self):
|
|
23
|
+
return f"DAGNode({self.step_function.name})"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def check_cycle(nodes: List[DAGNode]):
|
|
27
|
+
"""
|
|
28
|
+
Checks the given DAG for cycles and returns nodes that are part of a cycle.
|
|
29
|
+
"""
|
|
30
|
+
# Step 1: Calculate in-degree (number of incoming edges) for each node
|
|
31
|
+
in_degree = {node: 0 for node in nodes}
|
|
32
|
+
|
|
33
|
+
for node in nodes:
|
|
34
|
+
for successor in node.successors:
|
|
35
|
+
in_degree[successor] += 1
|
|
36
|
+
|
|
37
|
+
# Step 2: Initialize a queue with all nodes that have in-degree 0
|
|
38
|
+
queue = collections.deque([node for node in nodes if in_degree[node] == 0])
|
|
39
|
+
|
|
40
|
+
# Number of processed nodes
|
|
41
|
+
processed_count = 0
|
|
42
|
+
|
|
43
|
+
# Step 3: Process nodes with in-degree 0
|
|
44
|
+
while queue:
|
|
45
|
+
current_node = queue.popleft()
|
|
46
|
+
processed_count += 1
|
|
47
|
+
|
|
48
|
+
# Decrease the in-degree of each successor by 1
|
|
49
|
+
for successor in current_node.successors:
|
|
50
|
+
in_degree[successor] -= 1
|
|
51
|
+
# If in-degree becomes 0, add it to the queue
|
|
52
|
+
if in_degree[successor] == 0:
|
|
53
|
+
queue.append(successor)
|
|
54
|
+
|
|
55
|
+
# Step 4: Identify the faulty nodes (those still with in-degree > 0)
|
|
56
|
+
faulty_nodes = [node for node in nodes if in_degree[node] > 0]
|
|
57
|
+
|
|
58
|
+
# If there are faulty nodes, there's a cycle
|
|
59
|
+
if faulty_nodes:
|
|
60
|
+
return True, faulty_nodes
|
|
61
|
+
else:
|
|
62
|
+
return False, None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class FlowSpec:
|
|
66
|
+
def __init__(self, name: str, input: dict, client: Optional[Client] = None):
|
|
67
|
+
self.name = name
|
|
68
|
+
self.graph = FlowGraph(self.__class__)
|
|
69
|
+
self.client = client if client is not None else Client()
|
|
70
|
+
self.input = input
|
|
71
|
+
self.results = {}
|
|
72
|
+
|
|
73
|
+
def __repr__(self):
|
|
74
|
+
return f"Flow({self.name})"
|
|
75
|
+
|
|
76
|
+
def run(self):
|
|
77
|
+
open_nodes = set(self.graph.start_nodes)
|
|
78
|
+
closed_nodes = set()
|
|
79
|
+
|
|
80
|
+
# Run the nodes in parallel
|
|
81
|
+
tasks = {}
|
|
82
|
+
with Pool(8) as pool:
|
|
83
|
+
while open_nodes:
|
|
84
|
+
while True:
|
|
85
|
+
# Get the first node from the open nodes which has all its predecessors done
|
|
86
|
+
node = next(
|
|
87
|
+
iter(
|
|
88
|
+
filter(
|
|
89
|
+
lambda n: all(p in closed_nodes for p in n.predecessors),
|
|
90
|
+
open_nodes,
|
|
91
|
+
)
|
|
92
|
+
),
|
|
93
|
+
None,
|
|
94
|
+
)
|
|
95
|
+
if node is None:
|
|
96
|
+
# No more nodes to run at this point. Wait for the remaining tasks to finish.
|
|
97
|
+
break
|
|
98
|
+
open_nodes.remove(node)
|
|
99
|
+
# Run the node asynchronously
|
|
100
|
+
tasks[node] = pool.apipe(
|
|
101
|
+
self.__run_node,
|
|
102
|
+
node,
|
|
103
|
+
self._get_inputs(node),
|
|
104
|
+
self.client,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# Wait until at least one task is done
|
|
108
|
+
task_done = False
|
|
109
|
+
while not task_done:
|
|
110
|
+
time.sleep(0.1)
|
|
111
|
+
# Check if any tasks are done, if not, keep waiting
|
|
112
|
+
for node, task in list(tasks.items()):
|
|
113
|
+
if task.ready():
|
|
114
|
+
# Remove task and mark successors as ready by adding them to the open list.
|
|
115
|
+
result = task.get()
|
|
116
|
+
self.set_result(node, result)
|
|
117
|
+
del tasks[node]
|
|
118
|
+
task_done = True
|
|
119
|
+
closed_nodes.add(node)
|
|
120
|
+
open_nodes.update(node.successors)
|
|
121
|
+
|
|
122
|
+
def set_result(self, step: callable, result: object):
|
|
123
|
+
self.results[step.step] = result
|
|
124
|
+
|
|
125
|
+
def get_result(self, step: callable) -> Union[object, None]:
|
|
126
|
+
return self.results.get(step.step)
|
|
127
|
+
|
|
128
|
+
def _get_inputs(self, node: DAGNode) -> List[object]:
|
|
129
|
+
return (
|
|
130
|
+
[self.get_result(predecessor) for predecessor in node.step.needs.predecessors]
|
|
131
|
+
if node.step.is_needs()
|
|
132
|
+
else [self.input]
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
@staticmethod
|
|
136
|
+
def __run_node(node: DAGNode, inputs: List[object], client: Client) -> Union[List[object], object, None]:
|
|
137
|
+
utils.log(f"Running node {node.step.get_name()}")
|
|
138
|
+
|
|
139
|
+
# Skip the node if it is optional and the condition is not met
|
|
140
|
+
if node.step.skip():
|
|
141
|
+
utils.log(f"Skipping node {node.step.get_name()}")
|
|
142
|
+
return
|
|
143
|
+
|
|
144
|
+
# Run the step
|
|
145
|
+
if node.step.is_app():
|
|
146
|
+
app_step = node.step.app
|
|
147
|
+
repetitions = node.step.repeat.repetitions if node.step.is_repeat() else 1
|
|
148
|
+
# Prepare the input for the app
|
|
149
|
+
# TODO: We only support one predecessor for app steps for now. This may
|
|
150
|
+
# change in the future. We may want to support multiple predecessors for
|
|
151
|
+
# app steps. However, we need to think about how to handle the input and
|
|
152
|
+
# how to expose control over the input to the user.
|
|
153
|
+
if len(inputs) > 1:
|
|
154
|
+
raise Exception(
|
|
155
|
+
f"App steps cannot have more than one predecessor, but {node.step.get_name()} has {len(inputs)}"
|
|
156
|
+
)
|
|
157
|
+
inputs = [
|
|
158
|
+
(
|
|
159
|
+
[], # No nameless arguments
|
|
160
|
+
{ # We use the named arguments to pass the user arguments to the run function
|
|
161
|
+
"input": inputs[0],
|
|
162
|
+
"options": app_step.parameters,
|
|
163
|
+
},
|
|
164
|
+
)
|
|
165
|
+
] * repetitions
|
|
166
|
+
app = Application(client=client, id=app_step.app_id, default_instance_id=app_step.instance_id)
|
|
167
|
+
# Run the app (or multiple runs if it is a repeat step)
|
|
168
|
+
run_ids = [app.new_run(*i[0], **i[1]) for i in inputs]
|
|
169
|
+
outputs = utils.wait_for_runs(app=app, run_ids=run_ids)
|
|
170
|
+
# Check if all runs were successful
|
|
171
|
+
for output in outputs:
|
|
172
|
+
if output.metadata.status_v2 != StatusV2.succeeded:
|
|
173
|
+
raise Exception(
|
|
174
|
+
f"Step {node.step.get_name()} failed with status {output.metadata.status_v2}: "
|
|
175
|
+
+ f"{output.error_log}"
|
|
176
|
+
)
|
|
177
|
+
# Unwrap the result and store it
|
|
178
|
+
# TODO: We may want to store the full RunResult object in certain cases.
|
|
179
|
+
# Maybe this can become a parameter of the step decorator.
|
|
180
|
+
outputs = [output.output for output in outputs]
|
|
181
|
+
return outputs if node.step.is_repeat() else outputs[0]
|
|
182
|
+
else:
|
|
183
|
+
spec = inspect.getfullargspec(node.step.function)
|
|
184
|
+
if len(spec.args) == 0:
|
|
185
|
+
output = node.step.function()
|
|
186
|
+
else:
|
|
187
|
+
output = node.step.function(*inputs)
|
|
188
|
+
return output
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class FlowGraph:
|
|
192
|
+
def __init__(self, flow_spec):
|
|
193
|
+
self.flow_spec = flow_spec
|
|
194
|
+
self.__create_graph(flow_spec)
|
|
195
|
+
self.__debug_print_graph()
|
|
196
|
+
# Create a Mermaid diagram of the graph and log it
|
|
197
|
+
mermaid = self.__to_mermaid()
|
|
198
|
+
utils.log(mermaid)
|
|
199
|
+
mermaid_url = f'https://mermaid.ink/svg/{base64.b64encode(mermaid.encode("utf8")).decode("ascii")}?theme=dark'
|
|
200
|
+
utils.log(f"Mermaid URL: {mermaid_url}")
|
|
201
|
+
|
|
202
|
+
def __create_graph(self, flow_spec):
|
|
203
|
+
module = __import__(flow_spec.__module__)
|
|
204
|
+
class_name = flow_spec.__name__
|
|
205
|
+
tree = ast.parse(inspect.getsource(module)).body
|
|
206
|
+
root = [n for n in tree if isinstance(n, ast.ClassDef) and n.name == class_name][0]
|
|
207
|
+
|
|
208
|
+
# Build the graph
|
|
209
|
+
self.nodes = []
|
|
210
|
+
visitor = StepVisitor(self.nodes, flow_spec)
|
|
211
|
+
visitor.visit(root)
|
|
212
|
+
|
|
213
|
+
# Init nodes for all steps
|
|
214
|
+
nodes_by_step = {node.step: node for node in self.nodes}
|
|
215
|
+
for node in self.nodes:
|
|
216
|
+
node.predecessors = []
|
|
217
|
+
node.successors = []
|
|
218
|
+
|
|
219
|
+
for node in self.nodes:
|
|
220
|
+
if not node.step.is_needs():
|
|
221
|
+
continue
|
|
222
|
+
for predecessor in node.step.needs.predecessors:
|
|
223
|
+
predecessor_node = nodes_by_step[predecessor.step]
|
|
224
|
+
node.predecessors.append(predecessor_node)
|
|
225
|
+
predecessor_node.successors.append(node)
|
|
226
|
+
|
|
227
|
+
self.start_nodes = [node for node in self.nodes if not node.predecessors]
|
|
228
|
+
|
|
229
|
+
# Make sure that all app steps have at most one predecessor.
|
|
230
|
+
# TODO: This may change in the future. See other comment about it in this file.
|
|
231
|
+
for node in self.nodes:
|
|
232
|
+
if node.step.is_app() and len(node.predecessors) > 1:
|
|
233
|
+
raise Exception(
|
|
234
|
+
"App steps cannot have more than one predecessor, "
|
|
235
|
+
+ f"but {node.step.get_name()} has {len(node.predecessors)}"
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
# Check for cycles
|
|
239
|
+
cycle, cycle_nodes = check_cycle(self.nodes)
|
|
240
|
+
if cycle:
|
|
241
|
+
raise Exception(f"Cycle detected in the flow graph, cycle nodes: {cycle_nodes}")
|
|
242
|
+
|
|
243
|
+
def __to_mermaid(self):
|
|
244
|
+
"""Convert the graph to a Mermaid diagram."""
|
|
245
|
+
out = io.StringIO()
|
|
246
|
+
out.write("graph TD\n")
|
|
247
|
+
for node in self.nodes:
|
|
248
|
+
node_name = node.step.get_name()
|
|
249
|
+
if node.step.is_repeat():
|
|
250
|
+
out.write(f" {node_name}{{ }}\n")
|
|
251
|
+
out.write(f" {node_name}_join{{ }}\n")
|
|
252
|
+
repetitions = node.step.repeat.repetitions
|
|
253
|
+
for i in range(repetitions):
|
|
254
|
+
out.write(f" {node_name}_{i}({node_name}_{i})\n")
|
|
255
|
+
out.write(f" {node_name} --> {node_name}_{i}\n")
|
|
256
|
+
out.write(f" {node_name}_{i} --> {node_name}_join\n")
|
|
257
|
+
for successor in node.successors:
|
|
258
|
+
out.write(f" {node_name}_join --> {successor.step.get_name()}\n")
|
|
259
|
+
else:
|
|
260
|
+
out.write(f" {node_name}({node_name})\n")
|
|
261
|
+
for successor in node.successors:
|
|
262
|
+
out.write(f" {node_name} --> {successor.step.get_name()}\n")
|
|
263
|
+
return out.getvalue()
|
|
264
|
+
|
|
265
|
+
def __debug_print_graph(self):
|
|
266
|
+
for node in self.nodes:
|
|
267
|
+
utils.log("Node:")
|
|
268
|
+
utils.log(f" Definition: {node.step}")
|
|
269
|
+
utils.log(f" Docstring: {node.docstring}")
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
class StepVisitor(ast.NodeVisitor):
|
|
273
|
+
def __init__(self, nodes: List[DAGNode], flow: FlowSpec):
|
|
274
|
+
self.nodes = nodes
|
|
275
|
+
self.flow = flow
|
|
276
|
+
super().__init__()
|
|
277
|
+
|
|
278
|
+
def visit_FunctionDef(self, step_function):
|
|
279
|
+
func = getattr(self.flow, step_function.name)
|
|
280
|
+
if hasattr(func, "is_step"):
|
|
281
|
+
self.nodes.append(DAGNode(step_function, func.step, func.__doc__))
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import time
|
|
3
|
+
from functools import wraps
|
|
4
|
+
from typing import Dict, List
|
|
5
|
+
|
|
6
|
+
from nextmv.cloud import Application, RunResult, StatusV2
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def log(message: str) -> None:
|
|
10
|
+
"""Logs a message using stderr."""
|
|
11
|
+
|
|
12
|
+
print(message, file=sys.stderr)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def wrap_func(function):
|
|
16
|
+
"""
|
|
17
|
+
Wraps the given function in a new function that unpacks the arguments given as a tuple.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
@wraps(function)
|
|
21
|
+
def func_wrapper(args):
|
|
22
|
+
return function(*args[0], **args[1])
|
|
23
|
+
|
|
24
|
+
return func_wrapper
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def convert_to_string_values(input_dict: Dict[str, any]) -> Dict[str, str]:
|
|
28
|
+
"""
|
|
29
|
+
Converts all values of the given dictionary to strings.
|
|
30
|
+
"""
|
|
31
|
+
return {key: str(value) for key, value in input_dict.items()}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
_INFINITE_TIMEOUT = sys.maxsize
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def wait_for_runs(
|
|
38
|
+
app: Application,
|
|
39
|
+
run_ids: List[str],
|
|
40
|
+
timeout: int = _INFINITE_TIMEOUT,
|
|
41
|
+
max_backoff: int = 30,
|
|
42
|
+
) -> List[RunResult]:
|
|
43
|
+
"""
|
|
44
|
+
Wait until all runs with the given IDs are finished.
|
|
45
|
+
"""
|
|
46
|
+
# Wait until all runs are finished or the timeout is reached
|
|
47
|
+
missing = set(run_ids)
|
|
48
|
+
backoff = 1
|
|
49
|
+
start_time = time.time()
|
|
50
|
+
while missing and time.time() - start_time < timeout:
|
|
51
|
+
for run_id in missing.copy():
|
|
52
|
+
run_info = app.run_metadata(run_id=run_id)
|
|
53
|
+
if run_info.metadata.status_v2 == StatusV2.succeeded:
|
|
54
|
+
missing.remove(run_id)
|
|
55
|
+
continue
|
|
56
|
+
if run_info.metadata.status_v2 in [
|
|
57
|
+
StatusV2.failed,
|
|
58
|
+
StatusV2.canceled,
|
|
59
|
+
]:
|
|
60
|
+
raise RuntimeError(f"Run {run_id} {run_info.metadata.status_v2}")
|
|
61
|
+
|
|
62
|
+
time.sleep(backoff)
|
|
63
|
+
backoff = min(backoff * 2, max_backoff)
|
|
64
|
+
|
|
65
|
+
if missing:
|
|
66
|
+
raise TimeoutError(f"Timeout of {timeout} seconds reached while waiting.")
|
|
67
|
+
|
|
68
|
+
return [app.run_result(run_id=run_id) for run_id in run_ids]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
build-backend = "hatchling.build"
|
|
3
|
+
requires = ["hatchling >= 1.13.0"]
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
authors = [
|
|
7
|
+
{ email = "tech@nextmv.io", name = "Nextmv" }
|
|
8
|
+
]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Operating System :: OS Independent",
|
|
11
|
+
"Programming Language :: Python :: 3",
|
|
12
|
+
"Programming Language :: Python :: 3.8",
|
|
13
|
+
"Programming Language :: Python :: 3.9",
|
|
14
|
+
"Programming Language :: Python :: 3.10",
|
|
15
|
+
"Programming Language :: Python :: 3.11",
|
|
16
|
+
"Programming Language :: Python :: 3.12"
|
|
17
|
+
]
|
|
18
|
+
dependencies = [
|
|
19
|
+
"requests>=2.31.0",
|
|
20
|
+
"pathos>=0.3.2",
|
|
21
|
+
"nextmv>=0.12.0",
|
|
22
|
+
]
|
|
23
|
+
description = "Framework for Decision Pipeline modeling and execution"
|
|
24
|
+
dynamic = [
|
|
25
|
+
"version",
|
|
26
|
+
]
|
|
27
|
+
keywords = [
|
|
28
|
+
"decision engineering",
|
|
29
|
+
"decision science",
|
|
30
|
+
"decisions",
|
|
31
|
+
"nextmv",
|
|
32
|
+
"optimization",
|
|
33
|
+
"operations research",
|
|
34
|
+
"pipelines",
|
|
35
|
+
"workflows",
|
|
36
|
+
"decision pipelines",
|
|
37
|
+
"decision workflows",
|
|
38
|
+
"decision automation",
|
|
39
|
+
]
|
|
40
|
+
license = { file = "LICENSE.md" }
|
|
41
|
+
maintainers = [
|
|
42
|
+
{ email = "tech@nextmv.io", name = "Nextmv" }
|
|
43
|
+
]
|
|
44
|
+
name = "nextpipe"
|
|
45
|
+
readme = "README.md"
|
|
46
|
+
requires-python = ">=3.8"
|
|
47
|
+
|
|
48
|
+
[project.urls]
|
|
49
|
+
Homepage = "https://www.nextmv.io"
|
|
50
|
+
Documentation = "https://www.nextmv.io/docs"
|
|
51
|
+
Repository = "https://github.com/nextmv-io/nextpipe"
|
|
52
|
+
|
|
53
|
+
[tool.ruff]
|
|
54
|
+
target-version = "py38"
|
|
55
|
+
lint.select = [
|
|
56
|
+
"E", # pycodestyle errors
|
|
57
|
+
"W", # pycodestyle warnings
|
|
58
|
+
"F", # pyflakes
|
|
59
|
+
"I", # isort
|
|
60
|
+
"C", # flake8-comprehensions
|
|
61
|
+
"B", # flake8-bugbear
|
|
62
|
+
"UP", # pyupgrade
|
|
63
|
+
]
|
|
64
|
+
line-length = 120
|
|
65
|
+
|
|
66
|
+
[tool.hatch.version]
|
|
67
|
+
path = "nextpipe/__about__.py"
|
|
68
|
+
|
|
69
|
+
[project.optional-dependencies]
|
|
70
|
+
dev = [
|
|
71
|
+
"ruff>=0.6.4",
|
|
72
|
+
]
|
|
File without changes
|