runnable 0.12.2__py3-none-any.whl → 0.13.0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- runnable/extensions/executor/{local/implementation.py → local.py} +0 -2
- runnable/extensions/nodes.py +16 -1
- runnable/sdk.py +150 -101
- runnable/tasks.py +148 -3
- runnable-0.13.0.dist-info/METADATA +270 -0
- {runnable-0.12.2.dist-info → runnable-0.13.0.dist-info}/RECORD +9 -10
- {runnable-0.12.2.dist-info → runnable-0.13.0.dist-info}/WHEEL +1 -1
- {runnable-0.12.2.dist-info → runnable-0.13.0.dist-info}/entry_points.txt +1 -1
- runnable/extensions/executor/local/__init__.py +0 -0
- runnable-0.12.2.dist-info/METADATA +0 -453
- {runnable-0.12.2.dist-info → runnable-0.13.0.dist-info}/LICENSE +0 -0
@@ -0,0 +1,270 @@
|
|
1
|
+
Metadata-Version: 2.1
|
2
|
+
Name: runnable
|
3
|
+
Version: 0.13.0
|
4
|
+
Summary: A Compute agnostic pipelining software
|
5
|
+
Home-page: https://github.com/vijayvammi/runnable
|
6
|
+
License: Apache-2.0
|
7
|
+
Author: Vijay Vammi
|
8
|
+
Author-email: mesanthu@gmail.com
|
9
|
+
Requires-Python: >=3.9,<3.13
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
16
|
+
Provides-Extra: database
|
17
|
+
Provides-Extra: docker
|
18
|
+
Provides-Extra: notebook
|
19
|
+
Requires-Dist: click
|
20
|
+
Requires-Dist: click-plugins (>=1.1.1,<2.0.0)
|
21
|
+
Requires-Dist: dill (>=0.3.8,<0.4.0)
|
22
|
+
Requires-Dist: docker ; extra == "docker"
|
23
|
+
Requires-Dist: mlflow-skinny
|
24
|
+
Requires-Dist: ploomber-engine (>=0.0.31,<0.0.32) ; extra == "notebook"
|
25
|
+
Requires-Dist: pydantic (>=2.5,<3.0)
|
26
|
+
Requires-Dist: rich (>=13.5.2,<14.0.0)
|
27
|
+
Requires-Dist: ruamel.yaml
|
28
|
+
Requires-Dist: ruamel.yaml.clib
|
29
|
+
Requires-Dist: sqlalchemy ; extra == "database"
|
30
|
+
Requires-Dist: stevedore (>=3.5.0,<4.0.0)
|
31
|
+
Requires-Dist: typing-extensions ; python_version < "3.8"
|
32
|
+
Project-URL: Documentation, https://github.com/vijayvammi/runnable
|
33
|
+
Project-URL: Repository, https://github.com/vijayvammi/runnable
|
34
|
+
Description-Content-Type: text/markdown
|
35
|
+
|
36
|
+
|
37
|
+
|
38
|
+
|
39
|
+
|
40
|
+
|
41
|
+
</p>
|
42
|
+
<hr style="border:2px dotted orange">
|
43
|
+
|
44
|
+
<p align="center">
|
45
|
+
<a href="https://pypi.org/project/runnable/"><img alt="python:" src="https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10-blue.svg"></a>
|
46
|
+
<a href="https://pypi.org/project/runnable/"><img alt="Pypi" src="https://badge.fury.io/py/runnable.svg"></a>
|
47
|
+
<a href="https://github.com/vijayvammi/runnable/blob/main/LICENSE"><img alt"License" src="https://img.shields.io/badge/license-Apache%202.0-blue.svg"></a>
|
48
|
+
<a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
|
49
|
+
<a href="https://github.com/python/mypy"><img alt="MyPy Checked" src="https://www.mypy-lang.org/static/mypy_badge.svg"></a>
|
50
|
+
<a href="https://github.com/vijayvammi/runnable/actions/workflows/release.yaml"><img alt="Tests:" src="https://github.com/vijayvammi/runnable/actions/workflows/release.yaml/badge.svg">
|
51
|
+
</p>
|
52
|
+
<hr style="border:2px dotted orange">
|
53
|
+
|
54
|
+
|
55
|
+
[Please check here for complete documentation](https://astrazeneca.github.io/runnable/)
|
56
|
+
|
57
|
+
## Example
|
58
|
+
|
59
|
+
The below data science flavored code is a well-known
|
60
|
+
[iris example from scikit-learn](https://scikit-learn.org/stable/auto_examples/linear_model/plot_iris_logistic.html).
|
61
|
+
|
62
|
+
|
63
|
+
```python
|
64
|
+
"""
|
65
|
+
Example of Logistic regression using scikit-learn
|
66
|
+
https://scikit-learn.org/stable/auto_examples/linear_model/plot_iris_logistic.html
|
67
|
+
"""
|
68
|
+
|
69
|
+
import matplotlib.pyplot as plt
|
70
|
+
import numpy as np
|
71
|
+
from sklearn import datasets
|
72
|
+
from sklearn.inspection import DecisionBoundaryDisplay
|
73
|
+
from sklearn.linear_model import LogisticRegression
|
74
|
+
|
75
|
+
|
76
|
+
def load_data():
|
77
|
+
# import some data to play with
|
78
|
+
iris = datasets.load_iris()
|
79
|
+
X = iris.data[:, :2] # we only take the first two features.
|
80
|
+
Y = iris.target
|
81
|
+
|
82
|
+
return X, Y
|
83
|
+
|
84
|
+
|
85
|
+
def model_fit(X: np.ndarray, Y: np.ndarray, C: float = 1e5):
|
86
|
+
logreg = LogisticRegression(C=C)
|
87
|
+
logreg.fit(X, Y)
|
88
|
+
|
89
|
+
return logreg
|
90
|
+
|
91
|
+
|
92
|
+
def generate_plots(X: np.ndarray, Y: np.ndarray, logreg: LogisticRegression):
|
93
|
+
_, ax = plt.subplots(figsize=(4, 3))
|
94
|
+
DecisionBoundaryDisplay.from_estimator(
|
95
|
+
logreg,
|
96
|
+
X,
|
97
|
+
cmap=plt.cm.Paired,
|
98
|
+
ax=ax,
|
99
|
+
response_method="predict",
|
100
|
+
plot_method="pcolormesh",
|
101
|
+
shading="auto",
|
102
|
+
xlabel="Sepal length",
|
103
|
+
ylabel="Sepal width",
|
104
|
+
eps=0.5,
|
105
|
+
)
|
106
|
+
|
107
|
+
# Plot also the training points
|
108
|
+
plt.scatter(X[:, 0], X[:, 1], c=Y, edgecolors="k", cmap=plt.cm.Paired)
|
109
|
+
|
110
|
+
plt.xticks(())
|
111
|
+
plt.yticks(())
|
112
|
+
|
113
|
+
plt.savefig("iris_logistic.png")
|
114
|
+
|
115
|
+
# TODO: What is the right value?
|
116
|
+
return 0.6
|
117
|
+
|
118
|
+
|
119
|
+
## Without any orchestration
|
120
|
+
def main():
|
121
|
+
X, Y = load_data()
|
122
|
+
logreg = model_fit(X, Y, C=1.0)
|
123
|
+
generate_plots(X, Y, logreg)
|
124
|
+
|
125
|
+
|
126
|
+
## With runnable orchestration
|
127
|
+
def runnable_pipeline():
|
128
|
+
# The below code can be anywhere
|
129
|
+
from runnable import Catalog, Pipeline, PythonTask, metric, pickled
|
130
|
+
|
131
|
+
# X, Y = load_data()
|
132
|
+
load_data_task = PythonTask(
|
133
|
+
function=load_data,
|
134
|
+
name="load_data",
|
135
|
+
returns=[pickled("X"), pickled("Y")], # (1)
|
136
|
+
)
|
137
|
+
|
138
|
+
# logreg = model_fit(X, Y, C=1.0)
|
139
|
+
model_fit_task = PythonTask(
|
140
|
+
function=model_fit,
|
141
|
+
name="model_fit",
|
142
|
+
returns=[pickled("logreg")],
|
143
|
+
)
|
144
|
+
|
145
|
+
# generate_plots(X, Y, logreg)
|
146
|
+
generate_plots_task = PythonTask(
|
147
|
+
function=generate_plots,
|
148
|
+
name="generate_plots",
|
149
|
+
terminate_with_success=True,
|
150
|
+
catalog=Catalog(put=["iris_logistic.png"]), # (2)
|
151
|
+
returns=[metric("score")],
|
152
|
+
)
|
153
|
+
|
154
|
+
pipeline = Pipeline(
|
155
|
+
steps=[load_data_task, model_fit_task, generate_plots_task],
|
156
|
+
) # (4)
|
157
|
+
|
158
|
+
pipeline.execute()
|
159
|
+
|
160
|
+
return pipeline
|
161
|
+
|
162
|
+
|
163
|
+
if __name__ == "__main__":
|
164
|
+
# main()
|
165
|
+
runnable_pipeline()
|
166
|
+
|
167
|
+
```
|
168
|
+
|
169
|
+
|
170
|
+
1. Return two serialized objects X and Y.
|
171
|
+
2. Store the file `iris_logistic.png` for future reference.
|
172
|
+
3. Define the sequence of tasks.
|
173
|
+
4. Define a pipeline with the tasks
|
174
|
+
|
175
|
+
The difference between native driver and runnable orchestration:
|
176
|
+
|
177
|
+
!!! tip inline end "Notebooks and Shell scripts"
|
178
|
+
|
179
|
+
You can execute notebooks and shell scripts too!!
|
180
|
+
|
181
|
+
They can be written just as you would want them, *plain old notebooks and scripts*.
|
182
|
+
|
183
|
+
|
184
|
+
|
185
|
+
|
186
|
+
<div class="annotate" markdown>
|
187
|
+
|
188
|
+
```diff
|
189
|
+
|
190
|
+
- X, Y = load_data()
|
191
|
+
+load_data_task = PythonTask(
|
192
|
+
+ function=load_data,
|
193
|
+
+ name="load_data",
|
194
|
+
+ returns=[pickled("X"), pickled("Y")], (1)
|
195
|
+
+ )
|
196
|
+
|
197
|
+
-logreg = model_fit(X, Y, C=1.0)
|
198
|
+
+model_fit_task = PythonTask(
|
199
|
+
+ function=model_fit,
|
200
|
+
+ name="model_fit",
|
201
|
+
+ returns=[pickled("logreg")],
|
202
|
+
+ )
|
203
|
+
|
204
|
+
-generate_plots(X, Y, logreg)
|
205
|
+
+generate_plots_task = PythonTask(
|
206
|
+
+ function=generate_plots,
|
207
|
+
+ name="generate_plots",
|
208
|
+
+ terminate_with_success=True,
|
209
|
+
+ catalog=Catalog(put=["iris_logistic.png"]), (2)
|
210
|
+
+ )
|
211
|
+
|
212
|
+
|
213
|
+
+pipeline = Pipeline(
|
214
|
+
+ steps=[load_data_task, model_fit_task, generate_plots_task], (3)
|
215
|
+
|
216
|
+
```
|
217
|
+
</div>
|
218
|
+
|
219
|
+
|
220
|
+
---
|
221
|
+
|
222
|
+
- [x] ```Domain``` code remains completely independent of ```driver``` code.
|
223
|
+
- [x] The ```driver``` function has an equivalent and intuitive runnable expression
|
224
|
+
- [x] Reproducible by default, runnable stores metadata about code/data/config for every execution.
|
225
|
+
- [x] The pipeline is `runnable` in any environment.
|
226
|
+
|
227
|
+
|
228
|
+
## Documentation
|
229
|
+
|
230
|
+
[More details about the project and how to use it available here](https://astrazeneca.github.io/runnable/).
|
231
|
+
|
232
|
+
<hr style="border:2px dotted orange">
|
233
|
+
|
234
|
+
## Installation
|
235
|
+
|
236
|
+
The minimum python version that runnable supports is 3.8
|
237
|
+
|
238
|
+
```shell
|
239
|
+
pip install runnable
|
240
|
+
```
|
241
|
+
|
242
|
+
Please look at the [installation guide](https://astrazeneca.github.io/runnable-core/usage)
|
243
|
+
for more information.
|
244
|
+
|
245
|
+
|
246
|
+
## Pipelines can be:
|
247
|
+
|
248
|
+
### Linear
|
249
|
+
|
250
|
+
A simple linear pipeline with tasks either
|
251
|
+
[python functions](https://astrazeneca.github.io/runnable-core/concepts/task/#python_functions),
|
252
|
+
[notebooks](https://astrazeneca.github.io/runnable-core/concepts/task/#notebooks), or [shell scripts](https://astrazeneca.github.io/runnable-core/concepts/task/#shell)
|
253
|
+
|
254
|
+
[](https://mermaid.live/edit#pako:eNpl0bFuwyAQBuBXQVdZTqTESpxMDJ0ytkszhgwnOCcoNo4OaFVZfvcSx20tGSQ4fn0wHB3o1hBIyLJOWGeDFJ3Iq7r90lfkkA9HHfmTUpnX1hFyLvrHzDLl_qB4-1BOOZGGD3TfSikvTDSNFqdj2sT2vBTr9euQlXNWjqycsN2c7UZWFMUE7udwP0L3y6JenNKiyfvz8t8_b-gavT9QJYY0PcDtjeTLptrAChriBq1JzeoeWkG4UkMKZCoN8k2Bcn1yGEN7_HYaZOBIK4h3g4EOFi-MDcgKa59SMja0_P7s_vAJ_Q_YOH6o)
|
255
|
+
|
256
|
+
### [Parallel branches](https://astrazeneca.github.io/runnable-core/concepts/parallel)
|
257
|
+
|
258
|
+
Execute branches in parallel
|
259
|
+
|
260
|
+
[](https://mermaid.live/edit#pako:eNp9k01rwzAMhv-K8S4ZtJCzDzuMLmWwwkh2KMQ7eImShiZ2sB1KKf3vs52PpsWNT7LySHqlyBeciRwwwUUtTtmBSY2-YsopR8MpQUfAdCdBBekWNBpvv6-EkFICzGAtWcUTDW3wYy20M7lr5QGBK2j-anBAkH4M1z6grnjpy17xAiTwDII07jj6HK8-VnVZBspITnpjztyoVkLLJOy3Qfrdm6gQEu2370Io7WLORo84PbRoA_oOl9BBg4UHbHR58UkMWq_fxjrOnhLRx1nH0SgkjlBjh7ekxNKGc0NelDLknhePI8qf7MVNr_31nm1wwNTeM2Ao6pmf-3y3Mp7WlqA7twOnXfKs17zt-6azmim1gQL1A0NKS3EE8hKZE4Yezm3chIVFiFe4AdmwKjdv7mIjKNYHaIBiYsycySPFlF8NxzotkjPPMNGygxXu2pxp2FSslKzBpGC1Ml7IKy3krn_E7i1f_wEayTcn)
|
261
|
+
|
262
|
+
### [loops or map](https://astrazeneca.github.io/runnable-core/concepts/map)
|
263
|
+
|
264
|
+
Execute a pipeline over an iterable parameter.
|
265
|
+
|
266
|
+
[](https://mermaid.live/edit#pako:eNqVlF1rwjAUhv9KyG4qKNR-3AS2m8nuBgN3Z0Sy5tQG20SSdE7E_76kVVEr2CY3Ied9Tx6Sk3PAmeKACc5LtcsKpi36nlGZFbXciHwfLN79CuWiBLMcEULWGkBSaeosA2OCxbxdXMd89Get2bZASsLiSyuvQE2mJZXIjW27t2rOmQZ3Gp9rD6UjatWnwy7q6zPPukd50WTydmemEiS_QbQ79RwxGoQY9UaMuojRA8TCXexzyHgQZNwbMu5Cxl3IXNX6OWMyiDHpzZh0GZMHjOK3xz2mgxjT3oxplzG9MPp5_nVOhwJjteDwOg3HyFj3L1dCcvh7DUc-iftX18n6Waet1xX8cG908vpKHO6OW7cvkeHm5GR2b3drdvaSGTODHLW37mxabYC8fLgRhlfxpjNdwmEets-Dx7gCXTHBXQc8-D2KbQEVUEzckjO9oZjKo9Ox2qr5XmaYWF3DGNdbzizMBHOVVWGSs9K4XeDCKv3ZttSmsx7_AYa341E)
|
267
|
+
|
268
|
+
### [Arbitrary nesting](https://astrazeneca.github.io/runnable-core/concepts/nesting/)
|
269
|
+
Any nesting of parallel within map and so on.
|
270
|
+
|
@@ -21,15 +21,14 @@ runnable/extensions/executor/argo/specification.yaml,sha256=wXQcm2gOQYqy-IOQIhuc
|
|
21
21
|
runnable/extensions/executor/k8s_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
22
22
|
runnable/extensions/executor/k8s_job/implementation_FF.py,sha256=1IfVG1GRcJcVFzQ-WhkJsmzdJuj51QMxXylY9UrWM0U,10259
|
23
23
|
runnable/extensions/executor/k8s_job/integration_FF.py,sha256=pG6HKhPMgCRIgu1PAnBvsfJQE1FxcjuSiC2I-Hn5sWo,2165
|
24
|
-
runnable/extensions/executor/local
|
25
|
-
runnable/extensions/executor/local/implementation.py,sha256=e8Tzv-FgQmJeUXVut96jeNERTR83JVG_zkQZMEjCVAs,2469
|
24
|
+
runnable/extensions/executor/local.py,sha256=pV1tP9lRn6hwZtt-moKM1SXG_QmohbEF2QGdBbzCpkU,2396
|
26
25
|
runnable/extensions/executor/local_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
27
26
|
runnable/extensions/executor/local_container/implementation.py,sha256=6iwt9tNCQawVEfIErzoqys2hrErWK0DHcAOkO49Ov9w,17322
|
28
27
|
runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
29
28
|
runnable/extensions/executor/mocked/implementation.py,sha256=ChvlcLGpBxO6QwJcoqhBgKBR6NfWVnMdOWKQhMgcEjY,5762
|
30
29
|
runnable/extensions/executor/retry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
31
30
|
runnable/extensions/executor/retry/implementation.py,sha256=-g6PBOhSG7IL4D_IlQOcf9H_En9IXiUzCt-6vKeCB6Q,6892
|
32
|
-
runnable/extensions/nodes.py,sha256=
|
31
|
+
runnable/extensions/nodes.py,sha256=Jz8uJBgvcpdZ8R63EWtzULFE_DjSwsrWgY0P1Z1PYTs,33506
|
33
32
|
runnable/extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
34
33
|
runnable/extensions/run_log_store/chunked_file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
35
34
|
runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=EW2P8lr3eH-pIOsMTJPr5eb-iWc48GQ97W15JzkpC_4,3326
|
@@ -53,12 +52,12 @@ runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
|
|
53
52
|
runnable/nodes.py,sha256=UqR-bJx0Hi7uLSUw_saB7VsNdFh3POKtdgsEPsasHfE,16576
|
54
53
|
runnable/parameters.py,sha256=yZkMDnwnkdYXIwQ8LflBzn50Y0xRGxEvLlxwno6ovvs,5163
|
55
54
|
runnable/pickler.py,sha256=5SDNf0miMUJ3ZauhQdzwk8_t-9jeOqaTjP5bvRnu9sU,2685
|
56
|
-
runnable/sdk.py,sha256=
|
55
|
+
runnable/sdk.py,sha256=sZWiyGVIqaAmmJuc4p1OhuqLBSqS7svN1VuSOqeZ_xA,29254
|
57
56
|
runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
|
58
|
-
runnable/tasks.py,sha256=
|
57
|
+
runnable/tasks.py,sha256=61BYi5DNyTPi8GRlIy8vh2T0DZ1jkTj7gxBx88sK_XY,28058
|
59
58
|
runnable/utils.py,sha256=fXOLoFZKYqh3wQgzA2V-VZOu-dSgLPGqCZIbMmsNzOw,20016
|
60
|
-
runnable-0.
|
61
|
-
runnable-0.
|
62
|
-
runnable-0.
|
63
|
-
runnable-0.
|
64
|
-
runnable-0.
|
59
|
+
runnable-0.13.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
60
|
+
runnable-0.13.0.dist-info/METADATA,sha256=p4fwokQJKU8EJfniOP_DW22yDgSSZmltZDP2reYlBck,10436
|
61
|
+
runnable-0.13.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
62
|
+
runnable-0.13.0.dist-info/entry_points.txt,sha256=cl_K9XPV-qnAVaXfmNuGpZZKaoUu8YwjPsqZW1iSKlo,1479
|
63
|
+
runnable-0.13.0.dist-info/RECORD,,
|
@@ -7,7 +7,7 @@ runnable=runnable.cli:cli
|
|
7
7
|
|
8
8
|
[executor]
|
9
9
|
argo=runnable.extensions.executor.argo.implementation:ArgoExecutor
|
10
|
-
local=runnable.extensions.executor.local
|
10
|
+
local=runnable.extensions.executor.local:LocalExecutor
|
11
11
|
local-container=runnable.extensions.executor.local_container.implementation:LocalContainerExecutor
|
12
12
|
mocked=runnable.extensions.executor.mocked.implementation:MockedExecutor
|
13
13
|
retry=runnable.extensions.executor.retry.implementation:RetryExecutor
|
File without changes
|