runnable 0.12.2__py3-none-any.whl → 0.13.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,270 @@
1
+ Metadata-Version: 2.1
2
+ Name: runnable
3
+ Version: 0.13.0
4
+ Summary: A Compute agnostic pipelining software
5
+ Home-page: https://github.com/vijayvammi/runnable
6
+ License: Apache-2.0
7
+ Author: Vijay Vammi
8
+ Author-email: mesanthu@gmail.com
9
+ Requires-Python: >=3.9,<3.13
10
+ Classifier: License :: OSI Approved :: Apache Software License
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.9
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Provides-Extra: database
17
+ Provides-Extra: docker
18
+ Provides-Extra: notebook
19
+ Requires-Dist: click
20
+ Requires-Dist: click-plugins (>=1.1.1,<2.0.0)
21
+ Requires-Dist: dill (>=0.3.8,<0.4.0)
22
+ Requires-Dist: docker ; extra == "docker"
23
+ Requires-Dist: mlflow-skinny
24
+ Requires-Dist: ploomber-engine (>=0.0.31,<0.0.32) ; extra == "notebook"
25
+ Requires-Dist: pydantic (>=2.5,<3.0)
26
+ Requires-Dist: rich (>=13.5.2,<14.0.0)
27
+ Requires-Dist: ruamel.yaml
28
+ Requires-Dist: ruamel.yaml.clib
29
+ Requires-Dist: sqlalchemy ; extra == "database"
30
+ Requires-Dist: stevedore (>=3.5.0,<4.0.0)
31
+ Requires-Dist: typing-extensions ; python_version < "3.8"
32
+ Project-URL: Documentation, https://github.com/vijayvammi/runnable
33
+ Project-URL: Repository, https://github.com/vijayvammi/runnable
34
+ Description-Content-Type: text/markdown
35
+
36
+
37
+
38
+
39
+
40
+
41
+ </p>
42
+ <hr style="border:2px dotted orange">
43
+
44
+ <p align="center">
45
+ <a href="https://pypi.org/project/runnable/"><img alt="python:" src="https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10-blue.svg"></a>
46
+ <a href="https://pypi.org/project/runnable/"><img alt="Pypi" src="https://badge.fury.io/py/runnable.svg"></a>
47
+ <a href="https://github.com/vijayvammi/runnable/blob/main/LICENSE"><img alt"License" src="https://img.shields.io/badge/license-Apache%202.0-blue.svg"></a>
48
+ <a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
49
+ <a href="https://github.com/python/mypy"><img alt="MyPy Checked" src="https://www.mypy-lang.org/static/mypy_badge.svg"></a>
50
+ <a href="https://github.com/vijayvammi/runnable/actions/workflows/release.yaml"><img alt="Tests:" src="https://github.com/vijayvammi/runnable/actions/workflows/release.yaml/badge.svg">
51
+ </p>
52
+ <hr style="border:2px dotted orange">
53
+
54
+
55
+ [Please check here for complete documentation](https://astrazeneca.github.io/runnable/)
56
+
57
+ ## Example
58
+
59
+ The below data science flavored code is a well-known
60
+ [iris example from scikit-learn](https://scikit-learn.org/stable/auto_examples/linear_model/plot_iris_logistic.html).
61
+
62
+
63
+ ```python
64
+ """
65
+ Example of Logistic regression using scikit-learn
66
+ https://scikit-learn.org/stable/auto_examples/linear_model/plot_iris_logistic.html
67
+ """
68
+
69
+ import matplotlib.pyplot as plt
70
+ import numpy as np
71
+ from sklearn import datasets
72
+ from sklearn.inspection import DecisionBoundaryDisplay
73
+ from sklearn.linear_model import LogisticRegression
74
+
75
+
76
+ def load_data():
77
+ # import some data to play with
78
+ iris = datasets.load_iris()
79
+ X = iris.data[:, :2] # we only take the first two features.
80
+ Y = iris.target
81
+
82
+ return X, Y
83
+
84
+
85
+ def model_fit(X: np.ndarray, Y: np.ndarray, C: float = 1e5):
86
+ logreg = LogisticRegression(C=C)
87
+ logreg.fit(X, Y)
88
+
89
+ return logreg
90
+
91
+
92
+ def generate_plots(X: np.ndarray, Y: np.ndarray, logreg: LogisticRegression):
93
+ _, ax = plt.subplots(figsize=(4, 3))
94
+ DecisionBoundaryDisplay.from_estimator(
95
+ logreg,
96
+ X,
97
+ cmap=plt.cm.Paired,
98
+ ax=ax,
99
+ response_method="predict",
100
+ plot_method="pcolormesh",
101
+ shading="auto",
102
+ xlabel="Sepal length",
103
+ ylabel="Sepal width",
104
+ eps=0.5,
105
+ )
106
+
107
+ # Plot also the training points
108
+ plt.scatter(X[:, 0], X[:, 1], c=Y, edgecolors="k", cmap=plt.cm.Paired)
109
+
110
+ plt.xticks(())
111
+ plt.yticks(())
112
+
113
+ plt.savefig("iris_logistic.png")
114
+
115
+ # TODO: What is the right value?
116
+ return 0.6
117
+
118
+
119
+ ## Without any orchestration
120
+ def main():
121
+ X, Y = load_data()
122
+ logreg = model_fit(X, Y, C=1.0)
123
+ generate_plots(X, Y, logreg)
124
+
125
+
126
+ ## With runnable orchestration
127
+ def runnable_pipeline():
128
+ # The below code can be anywhere
129
+ from runnable import Catalog, Pipeline, PythonTask, metric, pickled
130
+
131
+ # X, Y = load_data()
132
+ load_data_task = PythonTask(
133
+ function=load_data,
134
+ name="load_data",
135
+ returns=[pickled("X"), pickled("Y")], # (1)
136
+ )
137
+
138
+ # logreg = model_fit(X, Y, C=1.0)
139
+ model_fit_task = PythonTask(
140
+ function=model_fit,
141
+ name="model_fit",
142
+ returns=[pickled("logreg")],
143
+ )
144
+
145
+ # generate_plots(X, Y, logreg)
146
+ generate_plots_task = PythonTask(
147
+ function=generate_plots,
148
+ name="generate_plots",
149
+ terminate_with_success=True,
150
+ catalog=Catalog(put=["iris_logistic.png"]), # (2)
151
+ returns=[metric("score")],
152
+ )
153
+
154
+ pipeline = Pipeline(
155
+ steps=[load_data_task, model_fit_task, generate_plots_task],
156
+ ) # (4)
157
+
158
+ pipeline.execute()
159
+
160
+ return pipeline
161
+
162
+
163
+ if __name__ == "__main__":
164
+ # main()
165
+ runnable_pipeline()
166
+
167
+ ```
168
+
169
+
170
+ 1. Return two serialized objects X and Y.
171
+ 2. Store the file `iris_logistic.png` for future reference.
172
+ 3. Define the sequence of tasks.
173
+ 4. Define a pipeline with the tasks
174
+
175
+ The difference between native driver and runnable orchestration:
176
+
177
+ !!! tip inline end "Notebooks and Shell scripts"
178
+
179
+ You can execute notebooks and shell scripts too!!
180
+
181
+ They can be written just as you would want them, *plain old notebooks and scripts*.
182
+
183
+
184
+
185
+
186
+ <div class="annotate" markdown>
187
+
188
+ ```diff
189
+
190
+ - X, Y = load_data()
191
+ +load_data_task = PythonTask(
192
+ + function=load_data,
193
+ + name="load_data",
194
+ + returns=[pickled("X"), pickled("Y")], (1)
195
+ + )
196
+
197
+ -logreg = model_fit(X, Y, C=1.0)
198
+ +model_fit_task = PythonTask(
199
+ + function=model_fit,
200
+ + name="model_fit",
201
+ + returns=[pickled("logreg")],
202
+ + )
203
+
204
+ -generate_plots(X, Y, logreg)
205
+ +generate_plots_task = PythonTask(
206
+ + function=generate_plots,
207
+ + name="generate_plots",
208
+ + terminate_with_success=True,
209
+ + catalog=Catalog(put=["iris_logistic.png"]), (2)
210
+ + )
211
+
212
+
213
+ +pipeline = Pipeline(
214
+ + steps=[load_data_task, model_fit_task, generate_plots_task], (3)
215
+
216
+ ```
217
+ </div>
218
+
219
+
220
+ ---
221
+
222
+ - [x] ```Domain``` code remains completely independent of ```driver``` code.
223
+ - [x] The ```driver``` function has an equivalent and intuitive runnable expression
224
+ - [x] Reproducible by default, runnable stores metadata about code/data/config for every execution.
225
+ - [x] The pipeline is `runnable` in any environment.
226
+
227
+
228
+ ## Documentation
229
+
230
+ [More details about the project and how to use it available here](https://astrazeneca.github.io/runnable/).
231
+
232
+ <hr style="border:2px dotted orange">
233
+
234
+ ## Installation
235
+
236
+ The minimum python version that runnable supports is 3.8
237
+
238
+ ```shell
239
+ pip install runnable
240
+ ```
241
+
242
+ Please look at the [installation guide](https://astrazeneca.github.io/runnable-core/usage)
243
+ for more information.
244
+
245
+
246
+ ## Pipelines can be:
247
+
248
+ ### Linear
249
+
250
+ A simple linear pipeline with tasks either
251
+ [python functions](https://astrazeneca.github.io/runnable-core/concepts/task/#python_functions),
252
+ [notebooks](https://astrazeneca.github.io/runnable-core/concepts/task/#notebooks), or [shell scripts](https://astrazeneca.github.io/runnable-core/concepts/task/#shell)
253
+
254
+ [![](https://mermaid.ink/img/pako:eNpl0bFuwyAQBuBXQVdZTqTESpxMDJ0ytkszhgwnOCcoNo4OaFVZfvcSx20tGSQ4fn0wHB3o1hBIyLJOWGeDFJ3Iq7r90lfkkA9HHfmTUpnX1hFyLvrHzDLl_qB4-1BOOZGGD3TfSikvTDSNFqdj2sT2vBTr9euQlXNWjqycsN2c7UZWFMUE7udwP0L3y6JenNKiyfvz8t8_b-gavT9QJYY0PcDtjeTLptrAChriBq1JzeoeWkG4UkMKZCoN8k2Bcn1yGEN7_HYaZOBIK4h3g4EOFi-MDcgKa59SMja0_P7s_vAJ_Q_YOH6o?type=png)](https://mermaid.live/edit#pako:eNpl0bFuwyAQBuBXQVdZTqTESpxMDJ0ytkszhgwnOCcoNo4OaFVZfvcSx20tGSQ4fn0wHB3o1hBIyLJOWGeDFJ3Iq7r90lfkkA9HHfmTUpnX1hFyLvrHzDLl_qB4-1BOOZGGD3TfSikvTDSNFqdj2sT2vBTr9euQlXNWjqycsN2c7UZWFMUE7udwP0L3y6JenNKiyfvz8t8_b-gavT9QJYY0PcDtjeTLptrAChriBq1JzeoeWkG4UkMKZCoN8k2Bcn1yGEN7_HYaZOBIK4h3g4EOFi-MDcgKa59SMja0_P7s_vAJ_Q_YOH6o)
255
+
256
+ ### [Parallel branches](https://astrazeneca.github.io/runnable-core/concepts/parallel)
257
+
258
+ Execute branches in parallel
259
+
260
+ [![](https://mermaid.ink/img/pako:eNp9k01rwzAMhv-K8S4ZtJCzDzuMLmWwwkh2KMQ7eImShiZ2sB1KKf3vs52PpsWNT7LySHqlyBeciRwwwUUtTtmBSY2-YsopR8MpQUfAdCdBBekWNBpvv6-EkFICzGAtWcUTDW3wYy20M7lr5QGBK2j-anBAkH4M1z6grnjpy17xAiTwDII07jj6HK8-VnVZBspITnpjztyoVkLLJOy3Qfrdm6gQEu2370Io7WLORo84PbRoA_oOl9BBg4UHbHR58UkMWq_fxjrOnhLRx1nH0SgkjlBjh7ekxNKGc0NelDLknhePI8qf7MVNr_31nm1wwNTeM2Ao6pmf-3y3Mp7WlqA7twOnXfKs17zt-6azmim1gQL1A0NKS3EE8hKZE4Yezm3chIVFiFe4AdmwKjdv7mIjKNYHaIBiYsycySPFlF8NxzotkjPPMNGygxXu2pxp2FSslKzBpGC1Ml7IKy3krn_E7i1f_wEayTcn?type=png)](https://mermaid.live/edit#pako:eNp9k01rwzAMhv-K8S4ZtJCzDzuMLmWwwkh2KMQ7eImShiZ2sB1KKf3vs52PpsWNT7LySHqlyBeciRwwwUUtTtmBSY2-YsopR8MpQUfAdCdBBekWNBpvv6-EkFICzGAtWcUTDW3wYy20M7lr5QGBK2j-anBAkH4M1z6grnjpy17xAiTwDII07jj6HK8-VnVZBspITnpjztyoVkLLJOy3Qfrdm6gQEu2370Io7WLORo84PbRoA_oOl9BBg4UHbHR58UkMWq_fxjrOnhLRx1nH0SgkjlBjh7ekxNKGc0NelDLknhePI8qf7MVNr_31nm1wwNTeM2Ao6pmf-3y3Mp7WlqA7twOnXfKs17zt-6azmim1gQL1A0NKS3EE8hKZE4Yezm3chIVFiFe4AdmwKjdv7mIjKNYHaIBiYsycySPFlF8NxzotkjPPMNGygxXu2pxp2FSslKzBpGC1Ml7IKy3krn_E7i1f_wEayTcn)
261
+
262
+ ### [loops or map](https://astrazeneca.github.io/runnable-core/concepts/map)
263
+
264
+ Execute a pipeline over an iterable parameter.
265
+
266
+ [![](https://mermaid.ink/img/pako:eNqVlF1rwjAUhv9KyG4qKNR-3AS2m8nuBgN3Z0Sy5tQG20SSdE7E_76kVVEr2CY3Ied9Tx6Sk3PAmeKACc5LtcsKpi36nlGZFbXciHwfLN79CuWiBLMcEULWGkBSaeosA2OCxbxdXMd89Get2bZASsLiSyuvQE2mJZXIjW27t2rOmQZ3Gp9rD6UjatWnwy7q6zPPukd50WTydmemEiS_QbQ79RwxGoQY9UaMuojRA8TCXexzyHgQZNwbMu5Cxl3IXNX6OWMyiDHpzZh0GZMHjOK3xz2mgxjT3oxplzG9MPp5_nVOhwJjteDwOg3HyFj3L1dCcvh7DUc-iftX18n6Waet1xX8cG908vpKHO6OW7cvkeHm5GR2b3drdvaSGTODHLW37mxabYC8fLgRhlfxpjNdwmEets-Dx7gCXTHBXQc8-D2KbQEVUEzckjO9oZjKo9Ox2qr5XmaYWF3DGNdbzizMBHOVVWGSs9K4XeDCKv3ZttSmsx7_AYa341E?type=png)](https://mermaid.live/edit#pako:eNqVlF1rwjAUhv9KyG4qKNR-3AS2m8nuBgN3Z0Sy5tQG20SSdE7E_76kVVEr2CY3Ied9Tx6Sk3PAmeKACc5LtcsKpi36nlGZFbXciHwfLN79CuWiBLMcEULWGkBSaeosA2OCxbxdXMd89Get2bZASsLiSyuvQE2mJZXIjW27t2rOmQZ3Gp9rD6UjatWnwy7q6zPPukd50WTydmemEiS_QbQ79RwxGoQY9UaMuojRA8TCXexzyHgQZNwbMu5Cxl3IXNX6OWMyiDHpzZh0GZMHjOK3xz2mgxjT3oxplzG9MPp5_nVOhwJjteDwOg3HyFj3L1dCcvh7DUc-iftX18n6Waet1xX8cG908vpKHO6OW7cvkeHm5GR2b3drdvaSGTODHLW37mxabYC8fLgRhlfxpjNdwmEets-Dx7gCXTHBXQc8-D2KbQEVUEzckjO9oZjKo9Ox2qr5XmaYWF3DGNdbzizMBHOVVWGSs9K4XeDCKv3ZttSmsx7_AYa341E)
267
+
268
+ ### [Arbitrary nesting](https://astrazeneca.github.io/runnable-core/concepts/nesting/)
269
+ Any nesting of parallel within map and so on.
270
+
@@ -21,15 +21,14 @@ runnable/extensions/executor/argo/specification.yaml,sha256=wXQcm2gOQYqy-IOQIhuc
21
21
  runnable/extensions/executor/k8s_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
22
22
  runnable/extensions/executor/k8s_job/implementation_FF.py,sha256=1IfVG1GRcJcVFzQ-WhkJsmzdJuj51QMxXylY9UrWM0U,10259
23
23
  runnable/extensions/executor/k8s_job/integration_FF.py,sha256=pG6HKhPMgCRIgu1PAnBvsfJQE1FxcjuSiC2I-Hn5sWo,2165
24
- runnable/extensions/executor/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
25
- runnable/extensions/executor/local/implementation.py,sha256=e8Tzv-FgQmJeUXVut96jeNERTR83JVG_zkQZMEjCVAs,2469
24
+ runnable/extensions/executor/local.py,sha256=pV1tP9lRn6hwZtt-moKM1SXG_QmohbEF2QGdBbzCpkU,2396
26
25
  runnable/extensions/executor/local_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
27
26
  runnable/extensions/executor/local_container/implementation.py,sha256=6iwt9tNCQawVEfIErzoqys2hrErWK0DHcAOkO49Ov9w,17322
28
27
  runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
29
28
  runnable/extensions/executor/mocked/implementation.py,sha256=ChvlcLGpBxO6QwJcoqhBgKBR6NfWVnMdOWKQhMgcEjY,5762
30
29
  runnable/extensions/executor/retry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
31
30
  runnable/extensions/executor/retry/implementation.py,sha256=-g6PBOhSG7IL4D_IlQOcf9H_En9IXiUzCt-6vKeCB6Q,6892
32
- runnable/extensions/nodes.py,sha256=SvuaXcy9uqJSG4nj-sFZc-aHswZ0v4HmAoaZNQXcu-Y,32831
31
+ runnable/extensions/nodes.py,sha256=Jz8uJBgvcpdZ8R63EWtzULFE_DjSwsrWgY0P1Z1PYTs,33506
33
32
  runnable/extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
34
33
  runnable/extensions/run_log_store/chunked_file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
35
34
  runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=EW2P8lr3eH-pIOsMTJPr5eb-iWc48GQ97W15JzkpC_4,3326
@@ -53,12 +52,12 @@ runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
53
52
  runnable/nodes.py,sha256=UqR-bJx0Hi7uLSUw_saB7VsNdFh3POKtdgsEPsasHfE,16576
54
53
  runnable/parameters.py,sha256=yZkMDnwnkdYXIwQ8LflBzn50Y0xRGxEvLlxwno6ovvs,5163
55
54
  runnable/pickler.py,sha256=5SDNf0miMUJ3ZauhQdzwk8_t-9jeOqaTjP5bvRnu9sU,2685
56
- runnable/sdk.py,sha256=pmxGO5HSrcEGYR_VmVk6ObfddFHo5slxiaDI6LuIdKM,27852
55
+ runnable/sdk.py,sha256=sZWiyGVIqaAmmJuc4p1OhuqLBSqS7svN1VuSOqeZ_xA,29254
57
56
  runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
58
- runnable/tasks.py,sha256=7sAtFMu6ELD3PJoisNbm47rY2wPKVzz5_h4s_QMep0k,22043
57
+ runnable/tasks.py,sha256=61BYi5DNyTPi8GRlIy8vh2T0DZ1jkTj7gxBx88sK_XY,28058
59
58
  runnable/utils.py,sha256=fXOLoFZKYqh3wQgzA2V-VZOu-dSgLPGqCZIbMmsNzOw,20016
60
- runnable-0.12.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
61
- runnable-0.12.2.dist-info/METADATA,sha256=8u2DRuU-gQvmW66oAqKABw5BYsCB0sJde6E_u-coEiE,17020
62
- runnable-0.12.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
63
- runnable-0.12.2.dist-info/entry_points.txt,sha256=-csEf-FCAqtOboXaBSzpgaTffz1HwlylYPxnppndpFE,1494
64
- runnable-0.12.2.dist-info/RECORD,,
59
+ runnable-0.13.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
60
+ runnable-0.13.0.dist-info/METADATA,sha256=p4fwokQJKU8EJfniOP_DW22yDgSSZmltZDP2reYlBck,10436
61
+ runnable-0.13.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
62
+ runnable-0.13.0.dist-info/entry_points.txt,sha256=cl_K9XPV-qnAVaXfmNuGpZZKaoUu8YwjPsqZW1iSKlo,1479
63
+ runnable-0.13.0.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: poetry-core 1.9.0
2
+ Generator: poetry-core 1.9.1
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
@@ -7,7 +7,7 @@ runnable=runnable.cli:cli
7
7
 
8
8
  [executor]
9
9
  argo=runnable.extensions.executor.argo.implementation:ArgoExecutor
10
- local=runnable.extensions.executor.local.implementation:LocalExecutor
10
+ local=runnable.extensions.executor.local:LocalExecutor
11
11
  local-container=runnable.extensions.executor.local_container.implementation:LocalContainerExecutor
12
12
  mocked=runnable.extensions.executor.mocked.implementation:MockedExecutor
13
13
  retry=runnable.extensions.executor.retry.implementation:RetryExecutor
File without changes