runnable 0.12.3__py3-none-any.whl → 0.14.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. runnable/__init__.py +0 -11
  2. runnable/catalog.py +27 -5
  3. runnable/cli.py +122 -26
  4. runnable/datastore.py +71 -35
  5. runnable/defaults.py +0 -1
  6. runnable/entrypoints.py +107 -32
  7. runnable/exceptions.py +6 -2
  8. runnable/executor.py +28 -9
  9. runnable/graph.py +37 -12
  10. runnable/integration.py +7 -2
  11. runnable/nodes.py +15 -17
  12. runnable/parameters.py +27 -8
  13. runnable/pickler.py +1 -1
  14. runnable/sdk.py +101 -33
  15. runnable/secrets.py +3 -1
  16. runnable/tasks.py +246 -34
  17. runnable/utils.py +41 -13
  18. {runnable-0.12.3.dist-info → runnable-0.14.0.dist-info}/METADATA +25 -31
  19. runnable-0.14.0.dist-info/RECORD +24 -0
  20. {runnable-0.12.3.dist-info → runnable-0.14.0.dist-info}/WHEEL +1 -1
  21. runnable-0.14.0.dist-info/entry_points.txt +40 -0
  22. runnable/extensions/__init__.py +0 -0
  23. runnable/extensions/catalog/__init__.py +0 -21
  24. runnable/extensions/catalog/file_system/__init__.py +0 -0
  25. runnable/extensions/catalog/file_system/implementation.py +0 -234
  26. runnable/extensions/catalog/k8s_pvc/__init__.py +0 -0
  27. runnable/extensions/catalog/k8s_pvc/implementation.py +0 -16
  28. runnable/extensions/catalog/k8s_pvc/integration.py +0 -59
  29. runnable/extensions/executor/__init__.py +0 -649
  30. runnable/extensions/executor/argo/__init__.py +0 -0
  31. runnable/extensions/executor/argo/implementation.py +0 -1194
  32. runnable/extensions/executor/argo/specification.yaml +0 -51
  33. runnable/extensions/executor/k8s_job/__init__.py +0 -0
  34. runnable/extensions/executor/k8s_job/implementation_FF.py +0 -259
  35. runnable/extensions/executor/k8s_job/integration_FF.py +0 -69
  36. runnable/extensions/executor/local/__init__.py +0 -0
  37. runnable/extensions/executor/local/implementation.py +0 -71
  38. runnable/extensions/executor/local_container/__init__.py +0 -0
  39. runnable/extensions/executor/local_container/implementation.py +0 -446
  40. runnable/extensions/executor/mocked/__init__.py +0 -0
  41. runnable/extensions/executor/mocked/implementation.py +0 -154
  42. runnable/extensions/executor/retry/__init__.py +0 -0
  43. runnable/extensions/executor/retry/implementation.py +0 -168
  44. runnable/extensions/nodes.py +0 -855
  45. runnable/extensions/run_log_store/__init__.py +0 -0
  46. runnable/extensions/run_log_store/chunked_file_system/__init__.py +0 -0
  47. runnable/extensions/run_log_store/chunked_file_system/implementation.py +0 -111
  48. runnable/extensions/run_log_store/chunked_k8s_pvc/__init__.py +0 -0
  49. runnable/extensions/run_log_store/chunked_k8s_pvc/implementation.py +0 -21
  50. runnable/extensions/run_log_store/chunked_k8s_pvc/integration.py +0 -61
  51. runnable/extensions/run_log_store/db/implementation_FF.py +0 -157
  52. runnable/extensions/run_log_store/db/integration_FF.py +0 -0
  53. runnable/extensions/run_log_store/file_system/__init__.py +0 -0
  54. runnable/extensions/run_log_store/file_system/implementation.py +0 -140
  55. runnable/extensions/run_log_store/generic_chunked.py +0 -557
  56. runnable/extensions/run_log_store/k8s_pvc/__init__.py +0 -0
  57. runnable/extensions/run_log_store/k8s_pvc/implementation.py +0 -21
  58. runnable/extensions/run_log_store/k8s_pvc/integration.py +0 -56
  59. runnable/extensions/secrets/__init__.py +0 -0
  60. runnable/extensions/secrets/dotenv/__init__.py +0 -0
  61. runnable/extensions/secrets/dotenv/implementation.py +0 -100
  62. runnable-0.12.3.dist-info/RECORD +0 -64
  63. runnable-0.12.3.dist-info/entry_points.txt +0 -41
  64. {runnable-0.12.3.dist-info → runnable-0.14.0.dist-info/licenses}/LICENSE +0 -0
@@ -1,100 +0,0 @@
1
- import logging
2
- import os
3
-
4
- from runnable import defaults, exceptions, utils
5
- from runnable.secrets import BaseSecrets
6
-
7
- logger = logging.getLogger(defaults.LOGGER_NAME)
8
-
9
-
10
- class DotEnvSecrets(BaseSecrets):
11
- """
12
- A secret manager which uses .env files for secrets.
13
-
14
- We recommend this secrets manager only for local development and should not be used for anything close to
15
- production.
16
- """
17
-
18
- service_name: str = "dotenv"
19
- location: str = defaults.DOTENV_FILE_LOCATION
20
- secrets: dict = {}
21
-
22
- @property
23
- def secrets_location(self):
24
- """
25
- Return the location of the .env file.
26
- If the user has not over-ridden it, it defaults to .env file in the project root.
27
-
28
- Returns:
29
- str: The location of the secrets file
30
- """
31
- return self.location
32
-
33
- def _load_secrets(self):
34
- """
35
- We assume that a dotenv file is of format,
36
- key=value -> secrets[key]='value'
37
- key=value# comment -> secrets[key1]='value1'
38
- key=value2 # comment. -> secrets[key2]='value2'
39
-
40
- Any of the above formats with export or set in front of them.
41
-
42
- We strip the secret value of any empty spaces at the start and end.
43
-
44
- Raises:
45
- Exception: If the file at secrets_location is not found.
46
- Exception: If the secrets are not formatted correctly.
47
- """
48
- # It was loaded in the previous call and need not to be reloaded
49
- if self.secrets:
50
- return
51
-
52
- secrets_location = self.secrets_location
53
- if not utils.does_file_exist(secrets_location):
54
- raise Exception(f"Did not find the secrets file in {secrets_location}")
55
-
56
- with open(secrets_location, "r") as fr:
57
- for secret_line in fr:
58
- # The order of removing fluff around the expression
59
- # the new line
60
- # the comment
61
- # the white space
62
- # Any export or set in front of the key any spaces after that.
63
-
64
- secret_line = secret_line.strip(os.linesep).split("#")[0].strip()
65
-
66
- if secret_line == "":
67
- continue
68
-
69
- secret_line = utils.remove_prefix(secret_line, prefix="export").strip()
70
- secret_line = utils.remove_prefix(secret_line, prefix="EXPORT").strip()
71
- secret_line = utils.remove_prefix(secret_line, prefix="set").strip()
72
- secret_line = utils.remove_prefix(secret_line, prefix="SET").strip()
73
-
74
- data = secret_line.split("=")
75
- if len(data) != 2:
76
- raise Exception("A secret should be of format, secret_name=secret_value[# any comment]")
77
-
78
- key, value = data
79
- self.secrets[key] = value.strip().strip('"').strip(os.linesep)
80
-
81
- def get(self, name: str = "", **kwargs) -> str:
82
- """
83
- Get a secret of name from the secrets file.
84
-
85
-
86
- Args:
87
- name (str): The name of the secret to retrieve
88
-
89
- Raises:
90
- Exception: If the secret by the name is not found.
91
-
92
- Returns:
93
- str: The value of the secret
94
- """
95
- self._load_secrets()
96
-
97
- if name in self.secrets:
98
- return self.secrets[name]
99
-
100
- raise exceptions.SecretNotFoundError(secret_name=name, secret_setting=self.secrets_location)
@@ -1,64 +0,0 @@
1
- runnable/__init__.py,sha256=G5EUe1eaTOEu1UihDBP9F43PKs3yxCYFfp8DOOpPvms,1038
2
- runnable/catalog.py,sha256=22OECi5TrpHErxYIhfx-lJ2vgBUi4-5V9CaYEVm98hE,4138
3
- runnable/cli.py,sha256=RILUrEfzernuKD3dNdXPBkqN_1OgE5GosYRuInj0FVs,9618
4
- runnable/context.py,sha256=QhiXJHRcEBfSKB1ijvL5yB9w44x0HCe7VEiwK1cUJ9U,1124
5
- runnable/datastore.py,sha256=8aQZ15KAMdre7a7G61bNRmcTeJFzOdnx_9O9UP4JQc8,27910
6
- runnable/defaults.py,sha256=r9l3jCPlmMFNdYXOj6X1uIhruO2FyLZ8kuzus9Fd6OQ,4704
7
- runnable/entrypoints.py,sha256=ZdXnjEjtkOqKYyjc0AwqIdfWgxJsKFB8ilHYcUj7oZo,17630
8
- runnable/exceptions.py,sha256=6NIYoTAzdKyGQ9PvW1Hu7b80OS746395KiGDhM7ThH8,2526
9
- runnable/executor.py,sha256=xfBighQ5t_vejohip000XfxLwsgechUE1ZMIJWrZbUA,14484
10
- runnable/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
11
- runnable/extensions/catalog/__init__.py,sha256=uXZ6D-Myr_J4HnBA4F5Hd7LZ0IAjQiFQYxRhMzejhQc,761
12
- runnable/extensions/catalog/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
13
- runnable/extensions/catalog/file_system/implementation.py,sha256=mFPsAwPMNGWbHczpQ84o3mfkPkOEz5zjsT7a3rqNzoE,9092
14
- runnable/extensions/catalog/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
15
- runnable/extensions/catalog/k8s_pvc/implementation.py,sha256=oJDDI0APT7lrtjWmzYJRDHLGn3Vhbn2MdFSRYvFBUpY,436
16
- runnable/extensions/catalog/k8s_pvc/integration.py,sha256=OfrHbNFN8sR-wsVa4os3ajmWJFSd5H4KOHGVAmjRZTQ,1850
17
- runnable/extensions/executor/__init__.py,sha256=-Xw8ZYVOfM-bYAieluEOhd98ncyrAyde8WSPRg1yFN0,26615
18
- runnable/extensions/executor/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
- runnable/extensions/executor/argo/implementation.py,sha256=FE5jDtzv5nlsnOjN9k0VjtEFQQWAAFaSXVJlroi7q6I,43911
20
- runnable/extensions/executor/argo/specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
21
- runnable/extensions/executor/k8s_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
22
- runnable/extensions/executor/k8s_job/implementation_FF.py,sha256=1IfVG1GRcJcVFzQ-WhkJsmzdJuj51QMxXylY9UrWM0U,10259
23
- runnable/extensions/executor/k8s_job/integration_FF.py,sha256=pG6HKhPMgCRIgu1PAnBvsfJQE1FxcjuSiC2I-Hn5sWo,2165
24
- runnable/extensions/executor/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
25
- runnable/extensions/executor/local/implementation.py,sha256=e8Tzv-FgQmJeUXVut96jeNERTR83JVG_zkQZMEjCVAs,2469
26
- runnable/extensions/executor/local_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
27
- runnable/extensions/executor/local_container/implementation.py,sha256=6iwt9tNCQawVEfIErzoqys2hrErWK0DHcAOkO49Ov9w,17322
28
- runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
29
- runnable/extensions/executor/mocked/implementation.py,sha256=ChvlcLGpBxO6QwJcoqhBgKBR6NfWVnMdOWKQhMgcEjY,5762
30
- runnable/extensions/executor/retry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
31
- runnable/extensions/executor/retry/implementation.py,sha256=-g6PBOhSG7IL4D_IlQOcf9H_En9IXiUzCt-6vKeCB6Q,6892
32
- runnable/extensions/nodes.py,sha256=SvuaXcy9uqJSG4nj-sFZc-aHswZ0v4HmAoaZNQXcu-Y,32831
33
- runnable/extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
34
- runnable/extensions/run_log_store/chunked_file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
35
- runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=EW2P8lr3eH-pIOsMTJPr5eb-iWc48GQ97W15JzkpC_4,3326
36
- runnable/extensions/run_log_store/chunked_k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
37
- runnable/extensions/run_log_store/chunked_k8s_pvc/implementation.py,sha256=iGzy-s1eT_kAJP7XgzDLmEMOGrBLvACIiGE_wM62jGE,579
38
- runnable/extensions/run_log_store/chunked_k8s_pvc/integration.py,sha256=atzdTy5HJ-bZsd6AzDP8kYRI1TshKxviBKeqY359TUs,1979
39
- runnable/extensions/run_log_store/db/implementation_FF.py,sha256=oEiG5ASWYYbwlBbnryKarQENB-L_yOsnZahbj2U0GdQ,5155
40
- runnable/extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
41
- runnable/extensions/run_log_store/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
42
- runnable/extensions/run_log_store/file_system/implementation.py,sha256=WxxfGCaDAB5zHMM3zv9aeDwXZ4DhtyzjXOjfjvyDoZ4,4288
43
- runnable/extensions/run_log_store/generic_chunked.py,sha256=PtYK1dheKYdxODwu_ygpGRIHIepgLVaIORSqvsrg0No,19876
44
- runnable/extensions/run_log_store/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
45
- runnable/extensions/run_log_store/k8s_pvc/implementation.py,sha256=tLgXy9HUB_vlFVQ0Itk6PpNU3GlCOILN4vA3fm80jXI,542
46
- runnable/extensions/run_log_store/k8s_pvc/integration.py,sha256=lxQg327mwC0ykhNp5Kg34a9g8o1DzJAhfkiqMGmsABs,1873
47
- runnable/extensions/secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
48
- runnable/extensions/secrets/dotenv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
49
- runnable/extensions/secrets/dotenv/implementation.py,sha256=3J5pofWahdZbnwnETwpspE5-PKyvmZF_vkfwA1X_bkA,3365
50
- runnable/graph.py,sha256=xJ5Q6Rv5T87wOBzQ1eIUvV0Iynnp7sRrZaIeytcDRlA,16313
51
- runnable/integration.py,sha256=eb9qJVZR7Ehg0N1UnGPuyjJvoA-xQ1-xP7AlZHUXHqM,6705
52
- runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
53
- runnable/nodes.py,sha256=UqR-bJx0Hi7uLSUw_saB7VsNdFh3POKtdgsEPsasHfE,16576
54
- runnable/parameters.py,sha256=yZkMDnwnkdYXIwQ8LflBzn50Y0xRGxEvLlxwno6ovvs,5163
55
- runnable/pickler.py,sha256=5SDNf0miMUJ3ZauhQdzwk8_t-9jeOqaTjP5bvRnu9sU,2685
56
- runnable/sdk.py,sha256=sZWiyGVIqaAmmJuc4p1OhuqLBSqS7svN1VuSOqeZ_xA,29254
57
- runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
58
- runnable/tasks.py,sha256=7sAtFMu6ELD3PJoisNbm47rY2wPKVzz5_h4s_QMep0k,22043
59
- runnable/utils.py,sha256=fXOLoFZKYqh3wQgzA2V-VZOu-dSgLPGqCZIbMmsNzOw,20016
60
- runnable-0.12.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
61
- runnable-0.12.3.dist-info/METADATA,sha256=spjKwm0NQ2n_MEfz1c0kqe2NkxpFq6nFQwyJ91xfAso,10436
62
- runnable-0.12.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
63
- runnable-0.12.3.dist-info/entry_points.txt,sha256=-csEf-FCAqtOboXaBSzpgaTffz1HwlylYPxnppndpFE,1494
64
- runnable-0.12.3.dist-info/RECORD,,
@@ -1,41 +0,0 @@
1
- [catalog]
2
- do-nothing=runnable.catalog:DoNothingCatalog
3
- file-system=runnable.extensions.catalog.file_system.implementation:FileSystemCatalog
4
-
5
- [console_scripts]
6
- runnable=runnable.cli:cli
7
-
8
- [executor]
9
- argo=runnable.extensions.executor.argo.implementation:ArgoExecutor
10
- local=runnable.extensions.executor.local.implementation:LocalExecutor
11
- local-container=runnable.extensions.executor.local_container.implementation:LocalContainerExecutor
12
- mocked=runnable.extensions.executor.mocked.implementation:MockedExecutor
13
- retry=runnable.extensions.executor.retry.implementation:RetryExecutor
14
-
15
- [nodes]
16
- dag=runnable.extensions.nodes:DagNode
17
- fail=runnable.extensions.nodes:FailNode
18
- map=runnable.extensions.nodes:MapNode
19
- parallel=runnable.extensions.nodes:ParallelNode
20
- stub=runnable.extensions.nodes:StubNode
21
- success=runnable.extensions.nodes:SuccessNode
22
- task=runnable.extensions.nodes:TaskNode
23
-
24
- [pickler]
25
- pickle=runnable.pickler:NativePickler
26
-
27
- [run_log_store]
28
- buffered=runnable.datastore:BufferRunLogstore
29
- chunked-fs=runnable.extensions.run_log_store.chunked_file_system.implementation:ChunkedFileSystemRunLogStore
30
- file-system=runnable.extensions.run_log_store.file_system.implementation:FileSystemRunLogstore
31
-
32
- [secrets]
33
- do-nothing=runnable.secrets:DoNothingSecretManager
34
- dotenv=runnable.extensions.secrets.dotenv.implementation:DotEnvSecrets
35
- env-secrets=runnable.secrets:EnvSecretsManager
36
-
37
- [tasks]
38
- notebook=runnable.tasks:NotebookTaskType
39
- python=runnable.tasks:PythonTaskType
40
- shell=runnable.tasks:ShellTaskType
41
-