runnable 0.12.2__py3-none-any.whl → 0.13.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runnable/extensions/executor/{local/implementation.py → local.py} +0 -2
- runnable/extensions/nodes.py +16 -1
- runnable/sdk.py +150 -101
- runnable/tasks.py +148 -3
- runnable-0.13.0.dist-info/METADATA +270 -0
- {runnable-0.12.2.dist-info → runnable-0.13.0.dist-info}/RECORD +9 -10
- {runnable-0.12.2.dist-info → runnable-0.13.0.dist-info}/WHEEL +1 -1
- {runnable-0.12.2.dist-info → runnable-0.13.0.dist-info}/entry_points.txt +1 -1
- runnable/extensions/executor/local/__init__.py +0 -0
- runnable-0.12.2.dist-info/METADATA +0 -453
- {runnable-0.12.2.dist-info → runnable-0.13.0.dist-info}/LICENSE +0 -0
@@ -0,0 +1,270 @@
|
|
1
|
+
Metadata-Version: 2.1
|
2
|
+
Name: runnable
|
3
|
+
Version: 0.13.0
|
4
|
+
Summary: A Compute agnostic pipelining software
|
5
|
+
Home-page: https://github.com/vijayvammi/runnable
|
6
|
+
License: Apache-2.0
|
7
|
+
Author: Vijay Vammi
|
8
|
+
Author-email: mesanthu@gmail.com
|
9
|
+
Requires-Python: >=3.9,<3.13
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
16
|
+
Provides-Extra: database
|
17
|
+
Provides-Extra: docker
|
18
|
+
Provides-Extra: notebook
|
19
|
+
Requires-Dist: click
|
20
|
+
Requires-Dist: click-plugins (>=1.1.1,<2.0.0)
|
21
|
+
Requires-Dist: dill (>=0.3.8,<0.4.0)
|
22
|
+
Requires-Dist: docker ; extra == "docker"
|
23
|
+
Requires-Dist: mlflow-skinny
|
24
|
+
Requires-Dist: ploomber-engine (>=0.0.31,<0.0.32) ; extra == "notebook"
|
25
|
+
Requires-Dist: pydantic (>=2.5,<3.0)
|
26
|
+
Requires-Dist: rich (>=13.5.2,<14.0.0)
|
27
|
+
Requires-Dist: ruamel.yaml
|
28
|
+
Requires-Dist: ruamel.yaml.clib
|
29
|
+
Requires-Dist: sqlalchemy ; extra == "database"
|
30
|
+
Requires-Dist: stevedore (>=3.5.0,<4.0.0)
|
31
|
+
Requires-Dist: typing-extensions ; python_version < "3.8"
|
32
|
+
Project-URL: Documentation, https://github.com/vijayvammi/runnable
|
33
|
+
Project-URL: Repository, https://github.com/vijayvammi/runnable
|
34
|
+
Description-Content-Type: text/markdown
|
35
|
+
|
36
|
+
|
37
|
+
|
38
|
+
|
39
|
+
|
40
|
+
|
41
|
+
</p>
|
42
|
+
<hr style="border:2px dotted orange">
|
43
|
+
|
44
|
+
<p align="center">
|
45
|
+
<a href="https://pypi.org/project/runnable/"><img alt="python:" src="https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10-blue.svg"></a>
|
46
|
+
<a href="https://pypi.org/project/runnable/"><img alt="Pypi" src="https://badge.fury.io/py/runnable.svg"></a>
|
47
|
+
<a href="https://github.com/vijayvammi/runnable/blob/main/LICENSE"><img alt"License" src="https://img.shields.io/badge/license-Apache%202.0-blue.svg"></a>
|
48
|
+
<a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
|
49
|
+
<a href="https://github.com/python/mypy"><img alt="MyPy Checked" src="https://www.mypy-lang.org/static/mypy_badge.svg"></a>
|
50
|
+
<a href="https://github.com/vijayvammi/runnable/actions/workflows/release.yaml"><img alt="Tests:" src="https://github.com/vijayvammi/runnable/actions/workflows/release.yaml/badge.svg">
|
51
|
+
</p>
|
52
|
+
<hr style="border:2px dotted orange">
|
53
|
+
|
54
|
+
|
55
|
+
[Please check here for complete documentation](https://astrazeneca.github.io/runnable/)
|
56
|
+
|
57
|
+
## Example
|
58
|
+
|
59
|
+
The below data science flavored code is a well-known
|
60
|
+
[iris example from scikit-learn](https://scikit-learn.org/stable/auto_examples/linear_model/plot_iris_logistic.html).
|
61
|
+
|
62
|
+
|
63
|
+
```python
|
64
|
+
"""
|
65
|
+
Example of Logistic regression using scikit-learn
|
66
|
+
https://scikit-learn.org/stable/auto_examples/linear_model/plot_iris_logistic.html
|
67
|
+
"""
|
68
|
+
|
69
|
+
import matplotlib.pyplot as plt
|
70
|
+
import numpy as np
|
71
|
+
from sklearn import datasets
|
72
|
+
from sklearn.inspection import DecisionBoundaryDisplay
|
73
|
+
from sklearn.linear_model import LogisticRegression
|
74
|
+
|
75
|
+
|
76
|
+
def load_data():
|
77
|
+
# import some data to play with
|
78
|
+
iris = datasets.load_iris()
|
79
|
+
X = iris.data[:, :2] # we only take the first two features.
|
80
|
+
Y = iris.target
|
81
|
+
|
82
|
+
return X, Y
|
83
|
+
|
84
|
+
|
85
|
+
def model_fit(X: np.ndarray, Y: np.ndarray, C: float = 1e5):
|
86
|
+
logreg = LogisticRegression(C=C)
|
87
|
+
logreg.fit(X, Y)
|
88
|
+
|
89
|
+
return logreg
|
90
|
+
|
91
|
+
|
92
|
+
def generate_plots(X: np.ndarray, Y: np.ndarray, logreg: LogisticRegression):
|
93
|
+
_, ax = plt.subplots(figsize=(4, 3))
|
94
|
+
DecisionBoundaryDisplay.from_estimator(
|
95
|
+
logreg,
|
96
|
+
X,
|
97
|
+
cmap=plt.cm.Paired,
|
98
|
+
ax=ax,
|
99
|
+
response_method="predict",
|
100
|
+
plot_method="pcolormesh",
|
101
|
+
shading="auto",
|
102
|
+
xlabel="Sepal length",
|
103
|
+
ylabel="Sepal width",
|
104
|
+
eps=0.5,
|
105
|
+
)
|
106
|
+
|
107
|
+
# Plot also the training points
|
108
|
+
plt.scatter(X[:, 0], X[:, 1], c=Y, edgecolors="k", cmap=plt.cm.Paired)
|
109
|
+
|
110
|
+
plt.xticks(())
|
111
|
+
plt.yticks(())
|
112
|
+
|
113
|
+
plt.savefig("iris_logistic.png")
|
114
|
+
|
115
|
+
# TODO: What is the right value?
|
116
|
+
return 0.6
|
117
|
+
|
118
|
+
|
119
|
+
## Without any orchestration
|
120
|
+
def main():
|
121
|
+
X, Y = load_data()
|
122
|
+
logreg = model_fit(X, Y, C=1.0)
|
123
|
+
generate_plots(X, Y, logreg)
|
124
|
+
|
125
|
+
|
126
|
+
## With runnable orchestration
|
127
|
+
def runnable_pipeline():
|
128
|
+
# The below code can be anywhere
|
129
|
+
from runnable import Catalog, Pipeline, PythonTask, metric, pickled
|
130
|
+
|
131
|
+
# X, Y = load_data()
|
132
|
+
load_data_task = PythonTask(
|
133
|
+
function=load_data,
|
134
|
+
name="load_data",
|
135
|
+
returns=[pickled("X"), pickled("Y")], # (1)
|
136
|
+
)
|
137
|
+
|
138
|
+
# logreg = model_fit(X, Y, C=1.0)
|
139
|
+
model_fit_task = PythonTask(
|
140
|
+
function=model_fit,
|
141
|
+
name="model_fit",
|
142
|
+
returns=[pickled("logreg")],
|
143
|
+
)
|
144
|
+
|
145
|
+
# generate_plots(X, Y, logreg)
|
146
|
+
generate_plots_task = PythonTask(
|
147
|
+
function=generate_plots,
|
148
|
+
name="generate_plots",
|
149
|
+
terminate_with_success=True,
|
150
|
+
catalog=Catalog(put=["iris_logistic.png"]), # (2)
|
151
|
+
returns=[metric("score")],
|
152
|
+
)
|
153
|
+
|
154
|
+
pipeline = Pipeline(
|
155
|
+
steps=[load_data_task, model_fit_task, generate_plots_task],
|
156
|
+
) # (4)
|
157
|
+
|
158
|
+
pipeline.execute()
|
159
|
+
|
160
|
+
return pipeline
|
161
|
+
|
162
|
+
|
163
|
+
if __name__ == "__main__":
|
164
|
+
# main()
|
165
|
+
runnable_pipeline()
|
166
|
+
|
167
|
+
```
|
168
|
+
|
169
|
+
|
170
|
+
1. Return two serialized objects X and Y.
|
171
|
+
2. Store the file `iris_logistic.png` for future reference.
|
172
|
+
3. Define the sequence of tasks.
|
173
|
+
4. Define a pipeline with the tasks
|
174
|
+
|
175
|
+
The difference between native driver and runnable orchestration:
|
176
|
+
|
177
|
+
!!! tip inline end "Notebooks and Shell scripts"
|
178
|
+
|
179
|
+
You can execute notebooks and shell scripts too!!
|
180
|
+
|
181
|
+
They can be written just as you would want them, *plain old notebooks and scripts*.
|
182
|
+
|
183
|
+
|
184
|
+
|
185
|
+
|
186
|
+
<div class="annotate" markdown>
|
187
|
+
|
188
|
+
```diff
|
189
|
+
|
190
|
+
- X, Y = load_data()
|
191
|
+
+load_data_task = PythonTask(
|
192
|
+
+ function=load_data,
|
193
|
+
+ name="load_data",
|
194
|
+
+ returns=[pickled("X"), pickled("Y")], (1)
|
195
|
+
+ )
|
196
|
+
|
197
|
+
-logreg = model_fit(X, Y, C=1.0)
|
198
|
+
+model_fit_task = PythonTask(
|
199
|
+
+ function=model_fit,
|
200
|
+
+ name="model_fit",
|
201
|
+
+ returns=[pickled("logreg")],
|
202
|
+
+ )
|
203
|
+
|
204
|
+
-generate_plots(X, Y, logreg)
|
205
|
+
+generate_plots_task = PythonTask(
|
206
|
+
+ function=generate_plots,
|
207
|
+
+ name="generate_plots",
|
208
|
+
+ terminate_with_success=True,
|
209
|
+
+ catalog=Catalog(put=["iris_logistic.png"]), (2)
|
210
|
+
+ )
|
211
|
+
|
212
|
+
|
213
|
+
+pipeline = Pipeline(
|
214
|
+
+ steps=[load_data_task, model_fit_task, generate_plots_task], (3)
|
215
|
+
|
216
|
+
```
|
217
|
+
</div>
|
218
|
+
|
219
|
+
|
220
|
+
---
|
221
|
+
|
222
|
+
- [x] ```Domain``` code remains completely independent of ```driver``` code.
|
223
|
+
- [x] The ```driver``` function has an equivalent and intuitive runnable expression
|
224
|
+
- [x] Reproducible by default, runnable stores metadata about code/data/config for every execution.
|
225
|
+
- [x] The pipeline is `runnable` in any environment.
|
226
|
+
|
227
|
+
|
228
|
+
## Documentation
|
229
|
+
|
230
|
+
[More details about the project and how to use it available here](https://astrazeneca.github.io/runnable/).
|
231
|
+
|
232
|
+
<hr style="border:2px dotted orange">
|
233
|
+
|
234
|
+
## Installation
|
235
|
+
|
236
|
+
The minimum python version that runnable supports is 3.8
|
237
|
+
|
238
|
+
```shell
|
239
|
+
pip install runnable
|
240
|
+
```
|
241
|
+
|
242
|
+
Please look at the [installation guide](https://astrazeneca.github.io/runnable-core/usage)
|
243
|
+
for more information.
|
244
|
+
|
245
|
+
|
246
|
+
## Pipelines can be:
|
247
|
+
|
248
|
+
### Linear
|
249
|
+
|
250
|
+
A simple linear pipeline with tasks either
|
251
|
+
[python functions](https://astrazeneca.github.io/runnable-core/concepts/task/#python_functions),
|
252
|
+
[notebooks](https://astrazeneca.github.io/runnable-core/concepts/task/#notebooks), or [shell scripts](https://astrazeneca.github.io/runnable-core/concepts/task/#shell)
|
253
|
+
|
254
|
+
[](https://mermaid.live/edit#pako:eNpl0bFuwyAQBuBXQVdZTqTESpxMDJ0ytkszhgwnOCcoNo4OaFVZfvcSx20tGSQ4fn0wHB3o1hBIyLJOWGeDFJ3Iq7r90lfkkA9HHfmTUpnX1hFyLvrHzDLl_qB4-1BOOZGGD3TfSikvTDSNFqdj2sT2vBTr9euQlXNWjqycsN2c7UZWFMUE7udwP0L3y6JenNKiyfvz8t8_b-gavT9QJYY0PcDtjeTLptrAChriBq1JzeoeWkG4UkMKZCoN8k2Bcn1yGEN7_HYaZOBIK4h3g4EOFi-MDcgKa59SMja0_P7s_vAJ_Q_YOH6o)
|
255
|
+
|
256
|
+
### [Parallel branches](https://astrazeneca.github.io/runnable-core/concepts/parallel)
|
257
|
+
|
258
|
+
Execute branches in parallel
|
259
|
+
|
260
|
+
[](https://mermaid.live/edit#pako:eNp9k01rwzAMhv-K8S4ZtJCzDzuMLmWwwkh2KMQ7eImShiZ2sB1KKf3vs52PpsWNT7LySHqlyBeciRwwwUUtTtmBSY2-YsopR8MpQUfAdCdBBekWNBpvv6-EkFICzGAtWcUTDW3wYy20M7lr5QGBK2j-anBAkH4M1z6grnjpy17xAiTwDII07jj6HK8-VnVZBspITnpjztyoVkLLJOy3Qfrdm6gQEu2370Io7WLORo84PbRoA_oOl9BBg4UHbHR58UkMWq_fxjrOnhLRx1nH0SgkjlBjh7ekxNKGc0NelDLknhePI8qf7MVNr_31nm1wwNTeM2Ao6pmf-3y3Mp7WlqA7twOnXfKs17zt-6azmim1gQL1A0NKS3EE8hKZE4Yezm3chIVFiFe4AdmwKjdv7mIjKNYHaIBiYsycySPFlF8NxzotkjPPMNGygxXu2pxp2FSslKzBpGC1Ml7IKy3krn_E7i1f_wEayTcn)
|
261
|
+
|
262
|
+
### [loops or map](https://astrazeneca.github.io/runnable-core/concepts/map)
|
263
|
+
|
264
|
+
Execute a pipeline over an iterable parameter.
|
265
|
+
|
266
|
+
[](https://mermaid.live/edit#pako:eNqVlF1rwjAUhv9KyG4qKNR-3AS2m8nuBgN3Z0Sy5tQG20SSdE7E_76kVVEr2CY3Ied9Tx6Sk3PAmeKACc5LtcsKpi36nlGZFbXciHwfLN79CuWiBLMcEULWGkBSaeosA2OCxbxdXMd89Get2bZASsLiSyuvQE2mJZXIjW27t2rOmQZ3Gp9rD6UjatWnwy7q6zPPukd50WTydmemEiS_QbQ79RwxGoQY9UaMuojRA8TCXexzyHgQZNwbMu5Cxl3IXNX6OWMyiDHpzZh0GZMHjOK3xz2mgxjT3oxplzG9MPp5_nVOhwJjteDwOg3HyFj3L1dCcvh7DUc-iftX18n6Waet1xX8cG908vpKHO6OW7cvkeHm5GR2b3drdvaSGTODHLW37mxabYC8fLgRhlfxpjNdwmEets-Dx7gCXTHBXQc8-D2KbQEVUEzckjO9oZjKo9Ox2qr5XmaYWF3DGNdbzizMBHOVVWGSs9K4XeDCKv3ZttSmsx7_AYa341E)
|
267
|
+
|
268
|
+
### [Arbitrary nesting](https://astrazeneca.github.io/runnable-core/concepts/nesting/)
|
269
|
+
Any nesting of parallel within map and so on.
|
270
|
+
|
@@ -21,15 +21,14 @@ runnable/extensions/executor/argo/specification.yaml,sha256=wXQcm2gOQYqy-IOQIhuc
|
|
21
21
|
runnable/extensions/executor/k8s_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
22
22
|
runnable/extensions/executor/k8s_job/implementation_FF.py,sha256=1IfVG1GRcJcVFzQ-WhkJsmzdJuj51QMxXylY9UrWM0U,10259
|
23
23
|
runnable/extensions/executor/k8s_job/integration_FF.py,sha256=pG6HKhPMgCRIgu1PAnBvsfJQE1FxcjuSiC2I-Hn5sWo,2165
|
24
|
-
runnable/extensions/executor/local
|
25
|
-
runnable/extensions/executor/local/implementation.py,sha256=e8Tzv-FgQmJeUXVut96jeNERTR83JVG_zkQZMEjCVAs,2469
|
24
|
+
runnable/extensions/executor/local.py,sha256=pV1tP9lRn6hwZtt-moKM1SXG_QmohbEF2QGdBbzCpkU,2396
|
26
25
|
runnable/extensions/executor/local_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
27
26
|
runnable/extensions/executor/local_container/implementation.py,sha256=6iwt9tNCQawVEfIErzoqys2hrErWK0DHcAOkO49Ov9w,17322
|
28
27
|
runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
29
28
|
runnable/extensions/executor/mocked/implementation.py,sha256=ChvlcLGpBxO6QwJcoqhBgKBR6NfWVnMdOWKQhMgcEjY,5762
|
30
29
|
runnable/extensions/executor/retry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
31
30
|
runnable/extensions/executor/retry/implementation.py,sha256=-g6PBOhSG7IL4D_IlQOcf9H_En9IXiUzCt-6vKeCB6Q,6892
|
32
|
-
runnable/extensions/nodes.py,sha256=
|
31
|
+
runnable/extensions/nodes.py,sha256=Jz8uJBgvcpdZ8R63EWtzULFE_DjSwsrWgY0P1Z1PYTs,33506
|
33
32
|
runnable/extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
34
33
|
runnable/extensions/run_log_store/chunked_file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
35
34
|
runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=EW2P8lr3eH-pIOsMTJPr5eb-iWc48GQ97W15JzkpC_4,3326
|
@@ -53,12 +52,12 @@ runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
|
|
53
52
|
runnable/nodes.py,sha256=UqR-bJx0Hi7uLSUw_saB7VsNdFh3POKtdgsEPsasHfE,16576
|
54
53
|
runnable/parameters.py,sha256=yZkMDnwnkdYXIwQ8LflBzn50Y0xRGxEvLlxwno6ovvs,5163
|
55
54
|
runnable/pickler.py,sha256=5SDNf0miMUJ3ZauhQdzwk8_t-9jeOqaTjP5bvRnu9sU,2685
|
56
|
-
runnable/sdk.py,sha256=
|
55
|
+
runnable/sdk.py,sha256=sZWiyGVIqaAmmJuc4p1OhuqLBSqS7svN1VuSOqeZ_xA,29254
|
57
56
|
runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
|
58
|
-
runnable/tasks.py,sha256=
|
57
|
+
runnable/tasks.py,sha256=61BYi5DNyTPi8GRlIy8vh2T0DZ1jkTj7gxBx88sK_XY,28058
|
59
58
|
runnable/utils.py,sha256=fXOLoFZKYqh3wQgzA2V-VZOu-dSgLPGqCZIbMmsNzOw,20016
|
60
|
-
runnable-0.
|
61
|
-
runnable-0.
|
62
|
-
runnable-0.
|
63
|
-
runnable-0.
|
64
|
-
runnable-0.
|
59
|
+
runnable-0.13.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
60
|
+
runnable-0.13.0.dist-info/METADATA,sha256=p4fwokQJKU8EJfniOP_DW22yDgSSZmltZDP2reYlBck,10436
|
61
|
+
runnable-0.13.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
62
|
+
runnable-0.13.0.dist-info/entry_points.txt,sha256=cl_K9XPV-qnAVaXfmNuGpZZKaoUu8YwjPsqZW1iSKlo,1479
|
63
|
+
runnable-0.13.0.dist-info/RECORD,,
|
@@ -7,7 +7,7 @@ runnable=runnable.cli:cli
|
|
7
7
|
|
8
8
|
[executor]
|
9
9
|
argo=runnable.extensions.executor.argo.implementation:ArgoExecutor
|
10
|
-
local=runnable.extensions.executor.local
|
10
|
+
local=runnable.extensions.executor.local:LocalExecutor
|
11
11
|
local-container=runnable.extensions.executor.local_container.implementation:LocalContainerExecutor
|
12
12
|
mocked=runnable.extensions.executor.mocked.implementation:MockedExecutor
|
13
13
|
retry=runnable.extensions.executor.retry.implementation:RetryExecutor
|
File without changes
|