runnable 0.1.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. runnable/__init__.py +34 -0
  2. runnable/catalog.py +141 -0
  3. runnable/cli.py +272 -0
  4. runnable/context.py +34 -0
  5. runnable/datastore.py +687 -0
  6. runnable/defaults.py +182 -0
  7. runnable/entrypoints.py +448 -0
  8. runnable/exceptions.py +94 -0
  9. runnable/executor.py +421 -0
  10. runnable/experiment_tracker.py +139 -0
  11. runnable/extensions/catalog/__init__.py +21 -0
  12. runnable/extensions/catalog/file_system/__init__.py +0 -0
  13. runnable/extensions/catalog/file_system/implementation.py +227 -0
  14. runnable/extensions/catalog/k8s_pvc/__init__.py +0 -0
  15. runnable/extensions/catalog/k8s_pvc/implementation.py +16 -0
  16. runnable/extensions/catalog/k8s_pvc/integration.py +59 -0
  17. runnable/extensions/executor/__init__.py +725 -0
  18. runnable/extensions/executor/argo/__init__.py +0 -0
  19. runnable/extensions/executor/argo/implementation.py +1183 -0
  20. runnable/extensions/executor/argo/specification.yaml +51 -0
  21. runnable/extensions/executor/k8s_job/__init__.py +0 -0
  22. runnable/extensions/executor/k8s_job/implementation_FF.py +259 -0
  23. runnable/extensions/executor/k8s_job/integration_FF.py +69 -0
  24. runnable/extensions/executor/local/__init__.py +0 -0
  25. runnable/extensions/executor/local/implementation.py +70 -0
  26. runnable/extensions/executor/local_container/__init__.py +0 -0
  27. runnable/extensions/executor/local_container/implementation.py +361 -0
  28. runnable/extensions/executor/mocked/__init__.py +0 -0
  29. runnable/extensions/executor/mocked/implementation.py +189 -0
  30. runnable/extensions/experiment_tracker/__init__.py +0 -0
  31. runnable/extensions/experiment_tracker/mlflow/__init__.py +0 -0
  32. runnable/extensions/experiment_tracker/mlflow/implementation.py +94 -0
  33. runnable/extensions/nodes.py +655 -0
  34. runnable/extensions/run_log_store/__init__.py +0 -0
  35. runnable/extensions/run_log_store/chunked_file_system/__init__.py +0 -0
  36. runnable/extensions/run_log_store/chunked_file_system/implementation.py +106 -0
  37. runnable/extensions/run_log_store/chunked_k8s_pvc/__init__.py +0 -0
  38. runnable/extensions/run_log_store/chunked_k8s_pvc/implementation.py +21 -0
  39. runnable/extensions/run_log_store/chunked_k8s_pvc/integration.py +61 -0
  40. runnable/extensions/run_log_store/db/implementation_FF.py +157 -0
  41. runnable/extensions/run_log_store/db/integration_FF.py +0 -0
  42. runnable/extensions/run_log_store/file_system/__init__.py +0 -0
  43. runnable/extensions/run_log_store/file_system/implementation.py +136 -0
  44. runnable/extensions/run_log_store/generic_chunked.py +541 -0
  45. runnable/extensions/run_log_store/k8s_pvc/__init__.py +0 -0
  46. runnable/extensions/run_log_store/k8s_pvc/implementation.py +21 -0
  47. runnable/extensions/run_log_store/k8s_pvc/integration.py +56 -0
  48. runnable/extensions/secrets/__init__.py +0 -0
  49. runnable/extensions/secrets/dotenv/__init__.py +0 -0
  50. runnable/extensions/secrets/dotenv/implementation.py +100 -0
  51. runnable/extensions/secrets/env_secrets/__init__.py +0 -0
  52. runnable/extensions/secrets/env_secrets/implementation.py +42 -0
  53. runnable/graph.py +464 -0
  54. runnable/integration.py +205 -0
  55. runnable/interaction.py +404 -0
  56. runnable/names.py +546 -0
  57. runnable/nodes.py +501 -0
  58. runnable/parameters.py +183 -0
  59. runnable/pickler.py +102 -0
  60. runnable/sdk.py +472 -0
  61. runnable/secrets.py +95 -0
  62. runnable/tasks.py +395 -0
  63. runnable/utils.py +630 -0
  64. runnable-0.3.0.dist-info/METADATA +437 -0
  65. runnable-0.3.0.dist-info/RECORD +69 -0
  66. {runnable-0.1.0.dist-info → runnable-0.3.0.dist-info}/WHEEL +1 -1
  67. runnable-0.3.0.dist-info/entry_points.txt +44 -0
  68. runnable-0.1.0.dist-info/METADATA +0 -16
  69. runnable-0.1.0.dist-info/RECORD +0 -6
  70. /runnable/{.gitkeep → extensions/__init__.py} +0 -0
  71. {runnable-0.1.0.dist-info → runnable-0.3.0.dist-info}/LICENSE +0 -0
@@ -0,0 +1,437 @@
1
+ Metadata-Version: 2.1
2
+ Name: runnable
3
+ Version: 0.3.0
4
+ Summary: A Compute agnostic pipelining software
5
+ Home-page: https://github.com/vijayvammi/runnable
6
+ License: Apache-2.0
7
+ Author: Vijay Vammi
8
+ Author-email: mesanthu@gmail.com
9
+ Requires-Python: >=3.8,<3.13
10
+ Classifier: License :: OSI Approved :: Apache Software License
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.8
13
+ Classifier: Programming Language :: Python :: 3.9
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Provides-Extra: database
18
+ Provides-Extra: docker
19
+ Provides-Extra: mlflow
20
+ Provides-Extra: notebook
21
+ Requires-Dist: click
22
+ Requires-Dist: click-plugins (>=1.1.1,<2.0.0)
23
+ Requires-Dist: docker ; extra == "docker"
24
+ Requires-Dist: mlflow-skinny ; extra == "mlflow"
25
+ Requires-Dist: ploomber-engine (>=0.0.31,<0.0.32) ; extra == "notebook"
26
+ Requires-Dist: pydantic (>=2.5,<3.0)
27
+ Requires-Dist: rich (>=13.5.2,<14.0.0)
28
+ Requires-Dist: ruamel.yaml
29
+ Requires-Dist: ruamel.yaml.clib
30
+ Requires-Dist: sqlalchemy ; extra == "database"
31
+ Requires-Dist: stevedore (>=3.5.0,<4.0.0)
32
+ Requires-Dist: typing-extensions ; python_version < "3.8"
33
+ Project-URL: Documentation, https://github.com/vijayvammi/runnable
34
+ Project-URL: Repository, https://github.com/vijayvammi/runnable
35
+ Description-Content-Type: text/markdown
36
+
37
+
38
+
39
+ # Hello from runnable
40
+
41
+
42
+ <p align="center">
43
+ <img src="https://github.com/AstraZeneca/runnable-core/blob/main/assets/logo-readme.png?raw=true" alt="Logo"/>
44
+ </p>
45
+ <hr style="border:2px dotted orange">
46
+
47
+ <p align="center">
48
+ <a href="https://pypi.org/project/runnable/"><img alt="python:" src="https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10-blue.svg"></a>
49
+ <a href="https://pypi.org/project/runnable/"><img alt="Pypi" src="https://badge.fury.io/py/runnable.svg"></a>
50
+ <a href="https://github.com/AstraZeneca/runnable-core/blob/main/LICENSE"><img alt"License" src="https://img.shields.io/badge/license-Apache%202.0-blue.svg"></a>
51
+ <a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
52
+ <a href="https://github.com/python/mypy"><img alt="MyPy Checked" src="https://www.mypy-lang.org/static/mypy_badge.svg"></a>
53
+ <a href="https://github.com/AstraZeneca/runnable-core/actions/workflows/release.yaml"><img alt="Tests:" src="https://github.com/AstraZeneca/runnable-core/actions/workflows/release.yaml/badge.svg">
54
+ <a href="https://github.com/AstraZeneca/runnable-core/actions/workflows/docs.yaml"><img alt="Docs:" src="https://github.com/AstraZeneca/runnable-core/actions/workflows/docs.yaml/badge.svg">
55
+ </p>
56
+ <hr style="border:2px dotted orange">
57
+
58
+ runnable is a simplified workflow definition language that helps in:
59
+
60
+ - **Streamlined Design Process:** runnable enables users to efficiently plan their pipelines with
61
+ [stubbed nodes](https://astrazeneca.github.io/runnable-core/concepts/stub), along with offering support for various structures such as
62
+ [tasks](https://astrazeneca.github.io/runnable-core/concepts/task), [parallel branches](https://astrazeneca.github.io/runnable-core/concepts/parallel), and [loops or map branches](https://astrazeneca.github.io/runnable-core/concepts/map)
63
+ in both [yaml](https://astrazeneca.github.io/runnable-core/concepts/pipeline) or a [python SDK](https://astrazeneca.github.io/runnable-core/sdk) for maximum flexibility.
64
+
65
+ - **Incremental Development:** Build your pipeline piece by piece with runnable, which allows for the
66
+ implementation of tasks as [python functions](https://astrazeneca.github.io/runnable-core/concepts/task/#python_functions),
67
+ [notebooks](https://astrazeneca.github.io/runnable-core/concepts/task/#notebooks), or [shell scripts](https://astrazeneca.github.io/runnable-core/concepts/task/#shell),
68
+ adapting to the developer's preferred tools and methods.
69
+
70
+ - **Robust Testing:** Ensure your pipeline performs as expected with the ability to test using sampled data. runnable
71
+ also provides the capability to [mock and patch tasks](https://astrazeneca.github.io/runnable-core/configurations/executors/mocked)
72
+ for thorough evaluation before full-scale deployment.
73
+
74
+ - **Seamless Deployment:** Transition from the development stage to production with ease.
75
+ runnable simplifies the process by requiring [only configuration changes](https://astrazeneca.github.io/runnable-core/configurations/overview)
76
+ to adapt to different environments, including support for [argo workflows](https://astrazeneca.github.io/runnable-core/configurations/executors/argo).
77
+
78
+ - **Efficient Debugging:** Quickly identify and resolve issues in pipeline execution with runnable's local
79
+ debugging features. Retrieve data from failed tasks and [retry failures](https://astrazeneca.github.io/runnable-core/concepts/run-log/#retrying_failures)
80
+ using your chosen debugging tools to maintain a smooth development experience.
81
+
82
+ Along with the developer friendly features, runnable also acts as an interface to production grade concepts
83
+ such as [data catalog](https://astrazeneca.github.io/runnable-core/concepts/catalog), [reproducibility](https://astrazeneca.github.io/runnable-core/concepts/run-log),
84
+ [experiment tracking](https://astrazeneca.github.io/runnable-core/concepts/experiment-tracking)
85
+ and secure [access to secrets](https://astrazeneca.github.io/runnable-core/concepts/secrets).
86
+
87
+ <hr style="border:2px dotted orange">
88
+
89
+ ## What does it do?
90
+
91
+
92
+ ![works](assets/work.png)
93
+
94
+ <hr style="border:2px dotted orange">
95
+
96
+ ## Documentation
97
+
98
+ [More details about the project and how to use it available here](https://astrazeneca.github.io/runnable-core/).
99
+
100
+ <hr style="border:2px dotted orange">
101
+
102
+ ## Installation
103
+
104
+ The minimum python version that runnable supports is 3.8
105
+
106
+ ```shell
107
+ pip install runnable
108
+ ```
109
+
110
+ Please look at the [installation guide](https://astrazeneca.github.io/runnable-core/usage)
111
+ for more information.
112
+
113
+ <hr style="border:2px dotted orange">
114
+
115
+ ## Example
116
+
117
+ Your application code. Use pydantic models as DTO.
118
+
119
+ Assumed to be present at ```functions.py```
120
+ ```python
121
+ from pydantic import BaseModel
122
+
123
+ class InnerModel(BaseModel):
124
+ """
125
+ A pydantic model representing a group of related parameters.
126
+ """
127
+
128
+ foo: int
129
+ bar: str
130
+
131
+
132
+ class Parameter(BaseModel):
133
+ """
134
+ A pydantic model representing the parameters of the whole pipeline.
135
+ """
136
+
137
+ x: int
138
+ y: InnerModel
139
+
140
+
141
+ def return_parameter() -> Parameter:
142
+ """
143
+ The annotation of the return type of the function is not mandatory
144
+ but it is a good practice.
145
+
146
+ Returns:
147
+ Parameter: The parameters that should be used in downstream steps.
148
+ """
149
+ # Return type of a function should be a pydantic model
150
+ return Parameter(x=1, y=InnerModel(foo=10, bar="hello world"))
151
+
152
+
153
+ def display_parameter(x: int, y: InnerModel):
154
+ """
155
+ Annotating the arguments of the function is important for
156
+ runnable to understand the type of parameters you want.
157
+
158
+ Input args can be a pydantic model or the individual attributes.
159
+ """
160
+ print(x)
161
+ # >>> prints 1
162
+ print(y)
163
+ # >>> prints InnerModel(foo=10, bar="hello world")
164
+ ```
165
+
166
+ ### Application code using driver functions.
167
+
168
+ The code is runnable without any orchestration framework.
169
+
170
+ ```python
171
+ from functions import return_parameter, display_parameter
172
+
173
+ my_param = return_parameter()
174
+ display_parameter(my_param.x, my_param.y)
175
+ ```
176
+
177
+ ### Orchestration using runnable
178
+
179
+ <table>
180
+ <tr>
181
+ <th>python SDK</th>
182
+ <th>yaml</th>
183
+ </tr>
184
+ <tr>
185
+ <td valign="top"><p>
186
+
187
+ Example present at: ```examples/python-tasks.py```
188
+
189
+ Run it as: ```python examples/python-tasks.py```
190
+
191
+ ```python
192
+ from runnable import Pipeline, Task
193
+
194
+ def main():
195
+ step1 = Task(
196
+ name="step1",
197
+ command="examples.functions.return_parameter",
198
+ )
199
+ step2 = Task(
200
+ name="step2",
201
+ command="examples.functions.display_parameter",
202
+ terminate_with_success=True,
203
+ )
204
+
205
+ step1 >> step2
206
+
207
+ pipeline = Pipeline(
208
+ start_at=step1,
209
+ steps=[step1, step2],
210
+ add_terminal_nodes=True,
211
+ )
212
+
213
+ pipeline.execute()
214
+
215
+
216
+ if __name__ == "__main__":
217
+ main()
218
+ ```
219
+
220
+ </p></td>
221
+
222
+ <td valign="top"><p>
223
+
224
+ Example present at: ```examples/python-tasks.yaml```
225
+
226
+
227
+ Execute via the cli: ```runnable execute -f examples/python-tasks.yaml```
228
+
229
+ ```yaml
230
+ dag:
231
+ description: |
232
+ This is a simple pipeline that does 3 steps in sequence.
233
+ In this example:
234
+ 1. First step: returns a "parameter" x as a Pydantic model
235
+ 2. Second step: Consumes that parameter and prints it
236
+
237
+ This pipeline demonstrates one way to pass small data from one step to another.
238
+
239
+ start_at: step 1
240
+ steps:
241
+ step 1:
242
+ type: task
243
+ command_type: python # (2)
244
+ command: examples.functions.return_parameter # (1)
245
+ next: step 2
246
+ step 2:
247
+ type: task
248
+ command_type: python
249
+ command: examples.functions.display_parameter
250
+ next: success
251
+ success:
252
+ type: success
253
+ fail:
254
+ type: fail
255
+ ```
256
+
257
+ </p></td>
258
+
259
+ </tr>
260
+ </table>
261
+
262
+ ### Transpile to argo workflows
263
+
264
+ No code change, just change the configuration.
265
+
266
+ ```yaml
267
+ executor:
268
+ type: "argo"
269
+ config:
270
+ image: runnable:demo
271
+ persistent_volumes:
272
+ - name: runnable-volume
273
+ mount_path: /mnt
274
+
275
+ run_log_store:
276
+ type: file-system
277
+ config:
278
+ log_folder: /mnt/run_log_store
279
+ ```
280
+
281
+ More details can be found in [argo configuration](https://astrazeneca.github.io/runnable-core/configurations/executors/argo).
282
+
283
+ Execute the code as ```runnable execute -f examples/python-tasks.yaml -c examples/configs/argo-config.yam```
284
+
285
+ <details>
286
+ <summary>Expand</summary>
287
+
288
+ ```yaml
289
+ apiVersion: argoproj.io/v1alpha1
290
+ kind: Workflow
291
+ metadata:
292
+ generateName: runnable-dag-
293
+ annotations: {}
294
+ labels: {}
295
+ spec:
296
+ activeDeadlineSeconds: 172800
297
+ entrypoint: runnable-dag
298
+ podGC:
299
+ strategy: OnPodCompletion
300
+ retryStrategy:
301
+ limit: '0'
302
+ retryPolicy: Always
303
+ backoff:
304
+ duration: '120'
305
+ factor: 2
306
+ maxDuration: '3600'
307
+ serviceAccountName: default-editor
308
+ templates:
309
+ - name: runnable-dag
310
+ failFast: true
311
+ dag:
312
+ tasks:
313
+ - name: step-1-task-uvdp7h
314
+ template: step-1-task-uvdp7h
315
+ depends: ''
316
+ - name: step-2-task-772vg3
317
+ template: step-2-task-772vg3
318
+ depends: step-1-task-uvdp7h.Succeeded
319
+ - name: success-success-igzq2e
320
+ template: success-success-igzq2e
321
+ depends: step-2-task-772vg3.Succeeded
322
+ - name: step-1-task-uvdp7h
323
+ container:
324
+ image: runnable:demo
325
+ command:
326
+ - runnable
327
+ - execute_single_node
328
+ - '{{workflow.parameters.run_id}}'
329
+ - step%1
330
+ - --log-level
331
+ - WARNING
332
+ - --file
333
+ - examples/python-tasks.yaml
334
+ - --config-file
335
+ - examples/configs/argo-config.yaml
336
+ volumeMounts:
337
+ - name: executor-0
338
+ mountPath: /mnt
339
+ imagePullPolicy: ''
340
+ resources:
341
+ limits:
342
+ memory: 1Gi
343
+ cpu: 250m
344
+ requests:
345
+ memory: 1Gi
346
+ cpu: 250m
347
+ - name: step-2-task-772vg3
348
+ container:
349
+ image: runnable:demo
350
+ command:
351
+ - runnable
352
+ - execute_single_node
353
+ - '{{workflow.parameters.run_id}}'
354
+ - step%2
355
+ - --log-level
356
+ - WARNING
357
+ - --file
358
+ - examples/python-tasks.yaml
359
+ - --config-file
360
+ - examples/configs/argo-config.yaml
361
+ volumeMounts:
362
+ - name: executor-0
363
+ mountPath: /mnt
364
+ imagePullPolicy: ''
365
+ resources:
366
+ limits:
367
+ memory: 1Gi
368
+ cpu: 250m
369
+ requests:
370
+ memory: 1Gi
371
+ cpu: 250m
372
+ - name: success-success-igzq2e
373
+ container:
374
+ image: runnable:demo
375
+ command:
376
+ - runnable
377
+ - execute_single_node
378
+ - '{{workflow.parameters.run_id}}'
379
+ - success
380
+ - --log-level
381
+ - WARNING
382
+ - --file
383
+ - examples/python-tasks.yaml
384
+ - --config-file
385
+ - examples/configs/argo-config.yaml
386
+ volumeMounts:
387
+ - name: executor-0
388
+ mountPath: /mnt
389
+ imagePullPolicy: ''
390
+ resources:
391
+ limits:
392
+ memory: 1Gi
393
+ cpu: 250m
394
+ requests:
395
+ memory: 1Gi
396
+ cpu: 250m
397
+ templateDefaults:
398
+ activeDeadlineSeconds: 7200
399
+ timeout: 10800s
400
+ arguments:
401
+ parameters:
402
+ - name: run_id
403
+ value: '{{workflow.uid}}'
404
+ volumes:
405
+ - name: executor-0
406
+ persistentVolumeClaim:
407
+ claimName: runnable-volume
408
+
409
+ ```
410
+
411
+ </details>
412
+
413
+ ## Pipelines can be:
414
+
415
+ ### Linear
416
+
417
+ A simple linear pipeline with tasks either
418
+ [python functions](https://astrazeneca.github.io/runnable-core/concepts/task/#python_functions),
419
+ [notebooks](https://astrazeneca.github.io/runnable-core/concepts/task/#notebooks), or [shell scripts](https://astrazeneca.github.io/runnable-core/concepts/task/#shell)
420
+
421
+ [![](https://mermaid.ink/img/pako:eNpl0bFuwyAQBuBXQVdZTqTESpxMDJ0ytkszhgwnOCcoNo4OaFVZfvcSx20tGSQ4fn0wHB3o1hBIyLJOWGeDFJ3Iq7r90lfkkA9HHfmTUpnX1hFyLvrHzDLl_qB4-1BOOZGGD3TfSikvTDSNFqdj2sT2vBTr9euQlXNWjqycsN2c7UZWFMUE7udwP0L3y6JenNKiyfvz8t8_b-gavT9QJYY0PcDtjeTLptrAChriBq1JzeoeWkG4UkMKZCoN8k2Bcn1yGEN7_HYaZOBIK4h3g4EOFi-MDcgKa59SMja0_P7s_vAJ_Q_YOH6o?type=png)](https://mermaid.live/edit#pako:eNpl0bFuwyAQBuBXQVdZTqTESpxMDJ0ytkszhgwnOCcoNo4OaFVZfvcSx20tGSQ4fn0wHB3o1hBIyLJOWGeDFJ3Iq7r90lfkkA9HHfmTUpnX1hFyLvrHzDLl_qB4-1BOOZGGD3TfSikvTDSNFqdj2sT2vBTr9euQlXNWjqycsN2c7UZWFMUE7udwP0L3y6JenNKiyfvz8t8_b-gavT9QJYY0PcDtjeTLptrAChriBq1JzeoeWkG4UkMKZCoN8k2Bcn1yGEN7_HYaZOBIK4h3g4EOFi-MDcgKa59SMja0_P7s_vAJ_Q_YOH6o)
422
+
423
+ ### [Parallel branches](https://astrazeneca.github.io/runnable-core/concepts/parallel)
424
+
425
+ Execute branches in parallel
426
+
427
+ [![](https://mermaid.ink/img/pako:eNp9k01rwzAMhv-K8S4ZtJCzDzuMLmWwwkh2KMQ7eImShiZ2sB1KKf3vs52PpsWNT7LySHqlyBeciRwwwUUtTtmBSY2-YsopR8MpQUfAdCdBBekWNBpvv6-EkFICzGAtWcUTDW3wYy20M7lr5QGBK2j-anBAkH4M1z6grnjpy17xAiTwDII07jj6HK8-VnVZBspITnpjztyoVkLLJOy3Qfrdm6gQEu2370Io7WLORo84PbRoA_oOl9BBg4UHbHR58UkMWq_fxjrOnhLRx1nH0SgkjlBjh7ekxNKGc0NelDLknhePI8qf7MVNr_31nm1wwNTeM2Ao6pmf-3y3Mp7WlqA7twOnXfKs17zt-6azmim1gQL1A0NKS3EE8hKZE4Yezm3chIVFiFe4AdmwKjdv7mIjKNYHaIBiYsycySPFlF8NxzotkjPPMNGygxXu2pxp2FSslKzBpGC1Ml7IKy3krn_E7i1f_wEayTcn?type=png)](https://mermaid.live/edit#pako:eNp9k01rwzAMhv-K8S4ZtJCzDzuMLmWwwkh2KMQ7eImShiZ2sB1KKf3vs52PpsWNT7LySHqlyBeciRwwwUUtTtmBSY2-YsopR8MpQUfAdCdBBekWNBpvv6-EkFICzGAtWcUTDW3wYy20M7lr5QGBK2j-anBAkH4M1z6grnjpy17xAiTwDII07jj6HK8-VnVZBspITnpjztyoVkLLJOy3Qfrdm6gQEu2370Io7WLORo84PbRoA_oOl9BBg4UHbHR58UkMWq_fxjrOnhLRx1nH0SgkjlBjh7ekxNKGc0NelDLknhePI8qf7MVNr_31nm1wwNTeM2Ao6pmf-3y3Mp7WlqA7twOnXfKs17zt-6azmim1gQL1A0NKS3EE8hKZE4Yezm3chIVFiFe4AdmwKjdv7mIjKNYHaIBiYsycySPFlF8NxzotkjPPMNGygxXu2pxp2FSslKzBpGC1Ml7IKy3krn_E7i1f_wEayTcn)
428
+
429
+ ### [loops or map](https://astrazeneca.github.io/runnable-core/concepts/map)
430
+
431
+ Execute a pipeline over an iterable parameter.
432
+
433
+ [![](https://mermaid.ink/img/pako:eNqVlF1rwjAUhv9KyG4qKNR-3AS2m8nuBgN3Z0Sy5tQG20SSdE7E_76kVVEr2CY3Ied9Tx6Sk3PAmeKACc5LtcsKpi36nlGZFbXciHwfLN79CuWiBLMcEULWGkBSaeosA2OCxbxdXMd89Get2bZASsLiSyuvQE2mJZXIjW27t2rOmQZ3Gp9rD6UjatWnwy7q6zPPukd50WTydmemEiS_QbQ79RwxGoQY9UaMuojRA8TCXexzyHgQZNwbMu5Cxl3IXNX6OWMyiDHpzZh0GZMHjOK3xz2mgxjT3oxplzG9MPp5_nVOhwJjteDwOg3HyFj3L1dCcvh7DUc-iftX18n6Waet1xX8cG908vpKHO6OW7cvkeHm5GR2b3drdvaSGTODHLW37mxabYC8fLgRhlfxpjNdwmEets-Dx7gCXTHBXQc8-D2KbQEVUEzckjO9oZjKo9Ox2qr5XmaYWF3DGNdbzizMBHOVVWGSs9K4XeDCKv3ZttSmsx7_AYa341E?type=png)](https://mermaid.live/edit#pako:eNqVlF1rwjAUhv9KyG4qKNR-3AS2m8nuBgN3Z0Sy5tQG20SSdE7E_76kVVEr2CY3Ied9Tx6Sk3PAmeKACc5LtcsKpi36nlGZFbXciHwfLN79CuWiBLMcEULWGkBSaeosA2OCxbxdXMd89Get2bZASsLiSyuvQE2mJZXIjW27t2rOmQZ3Gp9rD6UjatWnwy7q6zPPukd50WTydmemEiS_QbQ79RwxGoQY9UaMuojRA8TCXexzyHgQZNwbMu5Cxl3IXNX6OWMyiDHpzZh0GZMHjOK3xz2mgxjT3oxplzG9MPp5_nVOhwJjteDwOg3HyFj3L1dCcvh7DUc-iftX18n6Waet1xX8cG908vpKHO6OW7cvkeHm5GR2b3drdvaSGTODHLW37mxabYC8fLgRhlfxpjNdwmEets-Dx7gCXTHBXQc8-D2KbQEVUEzckjO9oZjKo9Ox2qr5XmaYWF3DGNdbzizMBHOVVWGSs9K4XeDCKv3ZttSmsx7_AYa341E)
434
+
435
+ ### [Arbitrary nesting](https://astrazeneca.github.io/runnable-core/concepts/nesting/)
436
+ Any nesting of parallel within map and so on.
437
+
@@ -0,0 +1,69 @@
1
+ runnable/__init__.py,sha256=6xTnI7ff5AT6QgEbzccgEag_4qVQsBRbrKqIIdtZn9k,847
2
+ runnable/catalog.py,sha256=OUaQ73DWfTsMmq2sKlBn0aDz031mupladNGVuF3pWm0,3985
3
+ runnable/cli.py,sha256=2NkFHfo1n1Aeq0gPO-SW_T_GsEY1qS1-IdoAJR_0Qhc,9800
4
+ runnable/context.py,sha256=rP22KqTsLrnpVTe05W80W2jMoAFm8NfkVkYY49Mu2ZA,1025
5
+ runnable/datastore.py,sha256=baSocsV8X25C7ZEACIoAb2344M4SV3s6v7guMDZt_3k,23569
6
+ runnable/defaults.py,sha256=0Hct5MtHHALyrXsRH-CAxyFngP8-EHG3wDZmeE2uWII,4672
7
+ runnable/entrypoints.py,sha256=QbABjy2PBU4tuhFtZ5nyt2hPecNsl4YNmXpbpAZeOFo,15534
8
+ runnable/exceptions.py,sha256=R__RzUWs7Ow7m7yqawi2w09XXI4OqmA67yeXkECM0xw,2419
9
+ runnable/executor.py,sha256=rdupW7LJk2CtwA52vIRIa-AS8hMvMqfvOIakIyr7rxk,16141
10
+ runnable/experiment_tracker.py,sha256=bX2Vr73f3bsdnWqxjMSSiKA-WwqkUHfUzJQqZoQBpvY,3668
11
+ runnable/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
12
+ runnable/extensions/catalog/__init__.py,sha256=uXZ6D-Myr_J4HnBA4F5Hd7LZ0IAjQiFQYxRhMzejhQc,761
13
+ runnable/extensions/catalog/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
14
+ runnable/extensions/catalog/file_system/implementation.py,sha256=UNrJFV_tyMpknFKoCMXbBt-CWL4UpDxcwnMh43zVfpc,8958
15
+ runnable/extensions/catalog/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
16
+ runnable/extensions/catalog/k8s_pvc/implementation.py,sha256=oJDDI0APT7lrtjWmzYJRDHLGn3Vhbn2MdFSRYvFBUpY,436
17
+ runnable/extensions/catalog/k8s_pvc/integration.py,sha256=OfrHbNFN8sR-wsVa4os3ajmWJFSd5H4KOHGVAmjRZTQ,1850
18
+ runnable/extensions/executor/__init__.py,sha256=3A4uEDTiSDax4cLchaVh1KBaeDBk4qxKG6iUMVdp1Hk,30739
19
+ runnable/extensions/executor/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
20
+ runnable/extensions/executor/argo/implementation.py,sha256=wlDSD5RfZmrdQ65abXTZSdh4KUGv-IzQtbHVtDXNUgQ,43795
21
+ runnable/extensions/executor/argo/specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
22
+ runnable/extensions/executor/k8s_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
23
+ runnable/extensions/executor/k8s_job/implementation_FF.py,sha256=1IfVG1GRcJcVFzQ-WhkJsmzdJuj51QMxXylY9UrWM0U,10259
24
+ runnable/extensions/executor/k8s_job/integration_FF.py,sha256=pG6HKhPMgCRIgu1PAnBvsfJQE1FxcjuSiC2I-Hn5sWo,2165
25
+ runnable/extensions/executor/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
26
+ runnable/extensions/executor/local/implementation.py,sha256=r9dSf2lSBGHihbGNhq_GPe3fHKLjf4KI3l-B4w2b5Ls,2468
27
+ runnable/extensions/executor/local_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
28
+ runnable/extensions/executor/local_container/implementation.py,sha256=6kYMgdgE5JxZkVAidxsBSpqkHvyKMfEctgZWSZQEpXA,13979
29
+ runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
30
+ runnable/extensions/executor/mocked/implementation.py,sha256=LmW7IgO5a9dRd7Okf8-nivg3kyIUYYHbOmop8wsv010,7366
31
+ runnable/extensions/experiment_tracker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
32
+ runnable/extensions/experiment_tracker/mlflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
33
+ runnable/extensions/experiment_tracker/mlflow/implementation.py,sha256=sc1Wm1LCf7wBX0BYVx3YVdwsR72AE0qIrzl7cEfIl58,3045
34
+ runnable/extensions/nodes.py,sha256=-l_R2qJ1P0NdSpRu3U_MM5QKtssU5DqIyx0WLHhAsnY,26787
35
+ runnable/extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
36
+ runnable/extensions/run_log_store/chunked_file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
37
+ runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=koHNG-Cv7mpt-rTNC3tiLBR8HcjnQ9L-EvQ4dtKkGRA,3170
38
+ runnable/extensions/run_log_store/chunked_k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
39
+ runnable/extensions/run_log_store/chunked_k8s_pvc/implementation.py,sha256=iGzy-s1eT_kAJP7XgzDLmEMOGrBLvACIiGE_wM62jGE,579
40
+ runnable/extensions/run_log_store/chunked_k8s_pvc/integration.py,sha256=atzdTy5HJ-bZsd6AzDP8kYRI1TshKxviBKeqY359TUs,1979
41
+ runnable/extensions/run_log_store/db/implementation_FF.py,sha256=oEiG5ASWYYbwlBbnryKarQENB-L_yOsnZahbj2U0GdQ,5155
42
+ runnable/extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
43
+ runnable/extensions/run_log_store/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
44
+ runnable/extensions/run_log_store/file_system/implementation.py,sha256=HQ3hr0Z_1z_GPWOE0IJ1xtQSgIZwLnl5k2UVsklL_dA,4194
45
+ runnable/extensions/run_log_store/generic_chunked.py,sha256=It5CI4wGzh8reMgTZxaMOBAQ3afezUJ-x7RlwWBM6YM,19470
46
+ runnable/extensions/run_log_store/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
47
+ runnable/extensions/run_log_store/k8s_pvc/implementation.py,sha256=tLgXy9HUB_vlFVQ0Itk6PpNU3GlCOILN4vA3fm80jXI,542
48
+ runnable/extensions/run_log_store/k8s_pvc/integration.py,sha256=lxQg327mwC0ykhNp5Kg34a9g8o1DzJAhfkiqMGmsABs,1873
49
+ runnable/extensions/secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
50
+ runnable/extensions/secrets/dotenv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
+ runnable/extensions/secrets/dotenv/implementation.py,sha256=3J5pofWahdZbnwnETwpspE5-PKyvmZF_vkfwA1X_bkA,3365
52
+ runnable/extensions/secrets/env_secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
53
+ runnable/extensions/secrets/env_secrets/implementation.py,sha256=5XiHdJvIr0-jkl4fGfEf26UsgE5Q2Z4oCc0RwjlJdJA,1236
54
+ runnable/graph.py,sha256=w7qEnTuh6lvfeVSAbqNlwMufPqWL9espJI5s_G2prHM,15871
55
+ runnable/integration.py,sha256=_jEm5PXl8pbvCmUGKjtssfZ2YtYnMzuRnjr2NySMqzs,7176
56
+ runnable/interaction.py,sha256=DVDQJuMm-n9tT-PDPx6AuUhYvWZX6-jmsHUQX0_TWHw,13003
57
+ runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
58
+ runnable/nodes.py,sha256=7ztYtTf4GthbutwR56lDqu4ANDLrN5zHqJNvLD_PTOo,16458
59
+ runnable/parameters.py,sha256=IT-7OUbYmRQuVtzAsG6L7Q4TkGaovmsQ4ErStcSXwmQ,6434
60
+ runnable/pickler.py,sha256=rrFRc6SMrV6Pxd9r7aMtUou8z-HLL1un4QfH_non4XE,2679
61
+ runnable/sdk.py,sha256=wtoSoSFsd-8NDK9qgOQD9rJBu5wqT-BRikJAyVaWPMo,17631
62
+ runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
63
+ runnable/tasks.py,sha256=eG-L8mB5Kp4m-HChwvcZkXkueS9IA7VQ-tQKmr87lrQ,13604
64
+ runnable/utils.py,sha256=jfgx2_lYCCKUASM7vEGZXdizRFg6EvV9pyZSlhhMKMk,19801
65
+ runnable-0.3.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
66
+ runnable-0.3.0.dist-info/METADATA,sha256=IgHtMhxwFvT0r2H7XPSP1lGzB1Pl7mzMsDDpxnsGBaU,16185
67
+ runnable-0.3.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
68
+ runnable-0.3.0.dist-info/entry_points.txt,sha256=wmEeo0n87KXEXxYLN6vzrd-690-1UhE0niJHMhz7f-o,1640
69
+ runnable-0.3.0.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: poetry-core 1.8.1
2
+ Generator: poetry-core 1.9.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
@@ -0,0 +1,44 @@
1
+ [catalog]
2
+ do-nothing=runnable.catalog:DoNothingCatalog
3
+ file-system=runnable.extensions.catalog.file_system.implementation:FileSystemCatalog
4
+
5
+ [console_scripts]
6
+ runnable=runnable.cli:cli
7
+
8
+ [executor]
9
+ argo=runnable.extensions.executor.argo.implementation:ArgoExecutor
10
+ local=runnable.extensions.executor.local.implementation:LocalExecutor
11
+ local-container=runnable.extensions.executor.local_container.implementation:LocalContainerExecutor
12
+ mocked=runnable.extensions.executor.mocked.implementation:MockedExecutor
13
+
14
+ [experiment_tracker]
15
+ do-nothing=runnable.experiment_tracker:DoNothingTracker
16
+ mlflow=runnable.extensions.experiment_tracker.mlflow.implementation:MLFlowExperimentTracker
17
+
18
+ [nodes]
19
+ dag=runnable.extensions.nodes:DagNode
20
+ fail=runnable.extensions.nodes:FailNode
21
+ map=runnable.extensions.nodes:MapNode
22
+ parallel=runnable.extensions.nodes:ParallelNode
23
+ stub=runnable.extensions.nodes:StubNode
24
+ success=runnable.extensions.nodes:SuccessNode
25
+ task=runnable.extensions.nodes:TaskNode
26
+
27
+ [pickler]
28
+ pickle=runnable.pickler:NativePickler
29
+
30
+ [run_log_store]
31
+ buffered=runnable.datastore:BufferRunLogstore
32
+ chunked-fs=runnable.extensions.run_log_store.chunked_file_system.implementation:ChunkedFileSystemRunLogStore
33
+ file-system=runnable.extensions.run_log_store.file_system.implementation:FileSystemRunLogstore
34
+
35
+ [secrets]
36
+ do-nothing=runnable.secrets:DoNothingSecretManager
37
+ dotenv=runnable.extensions.secrets.dotenv.implementation:DotEnvSecrets
38
+ env-secrets-manager=runnable.extensions.secrets.env_secrets.implementation:EnvSecretsManager
39
+
40
+ [tasks]
41
+ notebook=runnable.tasks:NotebookTaskType
42
+ python=runnable.tasks:PythonTaskType
43
+ shell=runnable.tasks:ShellTaskType
44
+
@@ -1,16 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: runnable
3
- Version: 0.1.0
4
- Summary:
5
- Author: Vijay Vammi
6
- Author-email: vijay.vammi@astrazeneca.com
7
- Requires-Python: >=3.8,<4.0
8
- Classifier: Programming Language :: Python :: 3
9
- Classifier: Programming Language :: Python :: 3.8
10
- Classifier: Programming Language :: Python :: 3.9
11
- Classifier: Programming Language :: Python :: 3.10
12
- Classifier: Programming Language :: Python :: 3.11
13
- Classifier: Programming Language :: Python :: 3.12
14
- Description-Content-Type: text/markdown
15
-
16
-
@@ -1,6 +0,0 @@
1
- runnable/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- runnable/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
- runnable-0.1.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
4
- runnable-0.1.0.dist-info/METADATA,sha256=_TWVl9iRo48RKOEzVjuon4KV2YOOkfHlyOXV2W4dmQ4,495
5
- runnable-0.1.0.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
6
- runnable-0.1.0.dist-info/RECORD,,
File without changes