runnable 0.17.1__py3-none-any.whl → 0.19.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. extensions/README.md +0 -0
  2. extensions/__init__.py +0 -0
  3. extensions/catalog/README.md +0 -0
  4. extensions/catalog/file_system.py +253 -0
  5. extensions/catalog/pyproject.toml +14 -0
  6. extensions/job_executor/README.md +0 -0
  7. extensions/job_executor/__init__.py +160 -0
  8. extensions/job_executor/k8s.py +484 -0
  9. extensions/job_executor/k8s_job_spec.yaml +37 -0
  10. extensions/job_executor/local.py +61 -0
  11. extensions/job_executor/local_container.py +192 -0
  12. extensions/job_executor/pyproject.toml +16 -0
  13. extensions/nodes/README.md +0 -0
  14. extensions/nodes/nodes.py +954 -0
  15. extensions/nodes/pyproject.toml +15 -0
  16. extensions/pipeline_executor/README.md +0 -0
  17. extensions/pipeline_executor/__init__.py +644 -0
  18. extensions/pipeline_executor/argo.py +1307 -0
  19. extensions/pipeline_executor/argo_specification.yaml +51 -0
  20. extensions/pipeline_executor/local.py +62 -0
  21. extensions/pipeline_executor/local_container.py +362 -0
  22. extensions/pipeline_executor/mocked.py +161 -0
  23. extensions/pipeline_executor/pyproject.toml +16 -0
  24. extensions/pipeline_executor/retry.py +180 -0
  25. extensions/run_log_store/README.md +0 -0
  26. extensions/run_log_store/__init__.py +0 -0
  27. extensions/run_log_store/chunked_fs.py +113 -0
  28. extensions/run_log_store/db/implementation_FF.py +163 -0
  29. extensions/run_log_store/db/integration_FF.py +0 -0
  30. extensions/run_log_store/file_system.py +145 -0
  31. extensions/run_log_store/generic_chunked.py +599 -0
  32. extensions/run_log_store/pyproject.toml +15 -0
  33. extensions/secrets/README.md +0 -0
  34. extensions/secrets/dotenv.py +62 -0
  35. extensions/secrets/pyproject.toml +15 -0
  36. runnable/__init__.py +1 -0
  37. runnable/catalog.py +1 -2
  38. runnable/entrypoints.py +1 -5
  39. runnable/executor.py +1 -1
  40. runnable/parameters.py +0 -9
  41. runnable/utils.py +5 -25
  42. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/METADATA +1 -7
  43. runnable-0.19.0.dist-info/RECORD +58 -0
  44. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/entry_points.txt +1 -0
  45. runnable-0.17.1.dist-info/RECORD +0 -23
  46. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/WHEEL +0 -0
  47. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/licenses/LICENSE +0 -0
runnable/catalog.py CHANGED
@@ -10,8 +10,6 @@ from runnable.datastore import DataCatalog
10
10
 
11
11
  logger = logging.getLogger(defaults.LOGGER_NAME)
12
12
 
13
- # TODO: Should ** be allowed as glob pattern as it can potentially copy everything to catalog
14
-
15
13
 
16
14
  def is_catalog_out_of_sync(
17
15
  catalog, synced_catalogs=Optional[List[DataCatalog]]
@@ -170,3 +168,4 @@ class DoNothingCatalog(BaseCatalog):
170
168
  Does nothing
171
169
  """
172
170
  logger.info("Using a do-nothing catalog, doing nothing while sync between runs")
171
+ logger.info("Using a do-nothing catalog, doing nothing while sync between runs")
runnable/entrypoints.py CHANGED
@@ -16,9 +16,6 @@ from runnable.executor import BaseJobExecutor, BasePipelineExecutor
16
16
  logger = logging.getLogger(defaults.LOGGER_NAME)
17
17
 
18
18
 
19
- print("") # removes the buffer print
20
-
21
-
22
19
  def get_default_configs() -> RunnableConfig:
23
20
  """
24
21
  User can provide extensions as part of their code base, runnable-config.yaml provides the place to put them.
@@ -128,11 +125,10 @@ def prepare_configurations(
128
125
  "job-executor", None
129
126
  ) # type: ignore
130
127
  if not job_executor_config:
131
- executor_config = cast(
128
+ job_executor_config = cast(
132
129
  ServiceConfig,
133
130
  runnable_defaults.get("job-executor", defaults.DEFAULT_JOB_EXECUTOR),
134
131
  )
135
-
136
132
  assert job_executor_config, "Job executor is not provided"
137
133
  configured_executor = utils.get_provider_by_name_and_type(
138
134
  "job_executor", job_executor_config
runnable/executor.py CHANGED
@@ -11,9 +11,9 @@ import runnable.context as context
11
11
  from runnable import defaults
12
12
  from runnable.datastore import DataCatalog, JobLog, StepLog
13
13
  from runnable.defaults import TypeMapVariable
14
- from runnable.graph import Graph
15
14
 
16
15
  if TYPE_CHECKING: # pragma: no cover
16
+ from runnable.graph import Graph
17
17
  from runnable.nodes import BaseNode
18
18
  from runnable.tasks import BaseTaskType
19
19
 
runnable/parameters.py CHANGED
@@ -15,8 +15,6 @@ from runnable.utils import remove_prefix
15
15
 
16
16
  logger = logging.getLogger(defaults.LOGGER_NAME)
17
17
 
18
- # TODO: Revisit this, it might be a bit too complicated than required
19
-
20
18
 
21
19
  def get_user_set_parameters(remove: bool = False) -> Dict[str, JsonParameter]:
22
20
  """
@@ -50,13 +48,6 @@ def get_user_set_parameters(remove: bool = False) -> Dict[str, JsonParameter]:
50
48
  return parameters
51
49
 
52
50
 
53
- def serialize_parameter_as_str(value: Any) -> str:
54
- if isinstance(value, BaseModel):
55
- return json.dumps(value.model_dump())
56
-
57
- return json.dumps(value)
58
-
59
-
60
51
  def filter_arguments_for_func(
61
52
  func: Callable[..., Any],
62
53
  params: Dict[str, Any],
runnable/utils.py CHANGED
@@ -17,7 +17,7 @@ from ruamel.yaml import YAML
17
17
  from stevedore import driver
18
18
 
19
19
  import runnable.context as context
20
- from runnable import defaults, names
20
+ from runnable import console, defaults, names
21
21
  from runnable.defaults import TypeMapVariable
22
22
 
23
23
  if TYPE_CHECKING: # pragma: no cover
@@ -176,7 +176,7 @@ def is_a_git_repo() -> bool:
176
176
  logger.info("Found the code to be git versioned")
177
177
  return True
178
178
  except BaseException: # pylint: disable=W0702
179
- logger.error("No git repo found, unsafe hash")
179
+ console.print("Not a git repo", style="bold red")
180
180
 
181
181
  return False
182
182
 
@@ -195,27 +195,7 @@ def get_current_code_commit() -> Union[str, None]:
195
195
  logger.info("Found the git commit to be: %s", label)
196
196
  return label
197
197
  except BaseException: # pylint: disable=W0702
198
- logger.exception("Error getting git hash")
199
- raise
200
-
201
-
202
- def archive_git_tracked(name: str):
203
- """Generate a git archive of the tracked files.
204
-
205
- Args:
206
- name (str): The name to give the archive
207
-
208
- Raises:
209
- Exception: If its not a git repo
210
- """
211
- command = f"git archive -v -o {name}.tar.gz --format=tar.gz HEAD"
212
-
213
- if not is_a_git_repo():
214
- raise Exception("Not a git repo")
215
- try:
216
- subprocess.check_output(command.split()).strip().decode("utf-8")
217
- except BaseException: # pylint: disable=W0702
218
- logger.exception("Error archiving repo")
198
+ console.print("Not a git repo, error getting hash", style="bold red")
219
199
  raise
220
200
 
221
201
 
@@ -234,7 +214,7 @@ def is_git_clean() -> Tuple[bool, Union[None, str]]:
234
214
  return True, None
235
215
  return False, label
236
216
  except BaseException: # pylint: disable=W0702
237
- logger.exception("Error checking if the code is git clean")
217
+ console.print("Not a git repo, not clean", style="bold red")
238
218
 
239
219
  return False, None
240
220
 
@@ -253,7 +233,7 @@ def get_git_remote() -> Union[str, None]:
253
233
  logger.info("Found the git remote to be: %s", label)
254
234
  return label
255
235
  except BaseException: # pylint: disable=W0702
256
- logger.exception("Error getting git remote")
236
+ console.print("Not a git repo, no remote", style="bold red")
257
237
  raise
258
238
 
259
239
 
@@ -1,23 +1,17 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: runnable
3
- Version: 0.17.1
3
+ Version: 0.19.0
4
4
  Summary: Add your description here
5
5
  Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
6
6
  License-File: LICENSE
7
7
  Requires-Python: >=3.10
8
- Requires-Dist: catalog
9
8
  Requires-Dist: click-plugins>=1.1.1
10
9
  Requires-Dist: click<=8.1.3
11
10
  Requires-Dist: dill>=0.3.9
12
- Requires-Dist: job-executor
13
- Requires-Dist: nodes
14
- Requires-Dist: pipeline-executor
15
11
  Requires-Dist: pydantic>=2.10.3
16
12
  Requires-Dist: python-dotenv>=1.0.1
17
13
  Requires-Dist: rich>=13.9.4
18
14
  Requires-Dist: ruamel-yaml>=0.18.6
19
- Requires-Dist: run-log-store
20
- Requires-Dist: secrets
21
15
  Requires-Dist: setuptools>=75.6.0
22
16
  Requires-Dist: stevedore>=5.4.0
23
17
  Requires-Dist: typer>=0.15.1
@@ -0,0 +1,58 @@
1
+ extensions/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
+ extensions/catalog/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
+ extensions/catalog/file_system.py,sha256=VZEUx4X-GDSM8rJ_2kiCOyw1eek3roN0CiSB8wdUcOA,9307
5
+ extensions/catalog/pyproject.toml,sha256=lLNxY6v04c8I5QK_zKw_E6sJTArSJRA_V-79ktaA3Hk,279
6
+ extensions/job_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
7
+ extensions/job_executor/__init__.py,sha256=HINaPjBWz04Ni7GqhuDLi0lS0-gYzq52HcOioYueYJE,5513
8
+ extensions/job_executor/k8s.py,sha256=huffLcDIQgTU-Qsz05CekrIwqFZLtFRGFlQAplwPRoE,15258
9
+ extensions/job_executor/k8s_job_spec.yaml,sha256=7aFpxHdO_p6Hkc3YxusUOuAQTD1Myu0yTPX9DrhxbOg,1158
10
+ extensions/job_executor/local.py,sha256=8ebu4TKo6FnFiUflil6fmE7Pk8eSoe1fHNwX8YwI1BQ,1865
11
+ extensions/job_executor/local_container.py,sha256=7G2ARgoPwXbpNPgd5UDxWZqU2ABPIAk7bkNQkC4cNBQ,6585
12
+ extensions/job_executor/pyproject.toml,sha256=UIEgiCYHTXcRWSByNMFuKJFKgxTBpQqTqyUecIsb_Vc,286
13
+ extensions/nodes/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
14
+ extensions/nodes/nodes.py,sha256=ib68QE737ihGLIVp3V2wea13u7lmMZdRvK80bgUkRtA,34645
15
+ extensions/nodes/pyproject.toml,sha256=YTu-ETN3JNFSkMzzWeOwn4m-O2nbRH-PmiPBALDCUw4,278
16
+ extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
17
+ extensions/pipeline_executor/__init__.py,sha256=YnKILiy-SxfnG3rYUoinjh1lfkuAF5QXpPePtn6VxBY,25174
18
+ extensions/pipeline_executor/argo.py,sha256=ClfuU_Of_2f5mvqVgY1QQwwJwXHB0LbzwNArG1x2Axc,44666
19
+ extensions/pipeline_executor/argo_specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
20
+ extensions/pipeline_executor/local.py,sha256=H8s6AdML_9_f-vdGG_6k0y9FbLqAqvA1S_7xMNyARzY,1946
21
+ extensions/pipeline_executor/local_container.py,sha256=UCap8wCbHrtTN5acECBBkvcXkA3SXtrAOGW88JT7ofw,13853
22
+ extensions/pipeline_executor/mocked.py,sha256=SuObJ6Myt7p8duW8sylIp1cYIAnFutsJW1avWaOUY3c,5798
23
+ extensions/pipeline_executor/pyproject.toml,sha256=ykTX7srR10PBYb8LsIwEj8vIPPIEZQ5V_R7VYbZ-ido,291
24
+ extensions/pipeline_executor/retry.py,sha256=KGenhWrLLmOQgzMvqloXHDRJyoNs91t05rRW8aLW6FA,6969
25
+ extensions/run_log_store/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
26
+ extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
27
+ extensions/run_log_store/chunked_fs.py,sha256=ElftNIwBmA2U2QAVGxruhcqepV312M2C9-GWVtiFaMM,3331
28
+ extensions/run_log_store/file_system.py,sha256=SANQ3aFjQeUaq8euvdpwju-8uci9UxdiEDupXtLYppQ,4303
29
+ extensions/run_log_store/generic_chunked.py,sha256=BX0j6S1Fwma3wuitHelUYm69FqXGToh10Zk2kamw6ZY,20253
30
+ extensions/run_log_store/pyproject.toml,sha256=YnmXsFvFG9uv_c0spLYBsNI_1sbktqxtHsOuClyvZ3g,288
31
+ extensions/run_log_store/db/implementation_FF.py,sha256=euTnh0xzNF0e_DyfHQ4W-kG1AwTr8u7OuO3_cZkR5bM,5237
32
+ extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
33
+ extensions/secrets/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
34
+ extensions/secrets/dotenv.py,sha256=FbYYd_pVuJuVuIDIvXbzKuSSQ9GPq7xJXTDbJMTQbhM,1583
35
+ extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
36
+ runnable/__init__.py,sha256=fYkOrbsb-E1rGkrof7kOJ3KboTFH-HriGa-8npn4-50,625
37
+ runnable/catalog.py,sha256=b9N40kTv1IBidzlWjkHcBGyYhq6qIDHZfBuFenzjsMI,4924
38
+ runnable/cli.py,sha256=01zmzOdynEmLI4vWDtSHQ6y1od_Jlc8G1RF69fi2L8g,8446
39
+ runnable/context.py,sha256=by5uepmuCP0dmM9BmsliXihSes5QEFejwAsmekcqylE,1388
40
+ runnable/datastore.py,sha256=9y5enzn6AXLHLdwvgkdjGPrBkVlrcjfbaAHsst-lJzg,32466
41
+ runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
42
+ runnable/entrypoints.py,sha256=P958nFz5WAsgTwd9sW04Q30vtjweYpr3rPsHVY4gh2U,18876
43
+ runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
44
+ runnable/executor.py,sha256=ZPpfKwjDJnta03M2cWIINXcwke2ZDVc_QrIw7kwpHDQ,15547
45
+ runnable/graph.py,sha256=jVjikRLR-so3b2ufmNKpEQ_Ny68qN4bcGDAdXBRKiCY,16574
46
+ runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
47
+ runnable/nodes.py,sha256=YU9u7r1ESzui1uVtJ1dgwdv1ozyJnF2k-MCFieT8CLI,17519
48
+ runnable/parameters.py,sha256=LyQb1d0SaFeI4PJ_yDYt9wArm9ThSPASWb36TwIdDUs,5213
49
+ runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
50
+ runnable/sdk.py,sha256=xN5F4XX8r5wCN131kgN2xG7MkNm0bSGJ3Ukw8prHYJ8,31444
51
+ runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
52
+ runnable/tasks.py,sha256=JnIIYQf3YUidHXIN6hiUIfDnegc7_rJMNXuHW4WS9ig,29378
53
+ runnable/utils.py,sha256=Kwf54tHMVXYK7MCmvAi_FG08U_bHDKIQO-HDpM9X0QI,19500
54
+ runnable-0.19.0.dist-info/METADATA,sha256=tgXvJ1oDrhC59zVORaSTyFKqqLDp785wKWu55NUwSOE,9945
55
+ runnable-0.19.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
56
+ runnable-0.19.0.dist-info/entry_points.txt,sha256=seek5WVGvwYALm8lZ0TfPXwG5NaCeUKjU8urF8k3gvY,1621
57
+ runnable-0.19.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
58
+ runnable-0.19.0.dist-info/RECORD,,
@@ -9,6 +9,7 @@ file-system = extensions.catalog.file_system:FileSystemCatalog
9
9
  k8s-job = extensions.job_executor.k8s:K8sJobExecutor
10
10
  local = extensions.job_executor.local:LocalJobExecutor
11
11
  local-container = extensions.job_executor.local_container:LocalContainerJobExecutor
12
+ mini-k8s-job = extensions.job_executor.k8s:MiniK8sJobExecutor
12
13
 
13
14
  [nodes]
14
15
  dag = extensions.nodes.nodes:DagNode
@@ -1,23 +0,0 @@
1
- runnable/__init__.py,sha256=KqpLDTD1CfdEj2aDyEkSn2KW-_83qyrRrrWLc5lZVM4,624
2
- runnable/catalog.py,sha256=MiEmb-18liAKmgeMdDF41VVn0ZEAVLP8hR33oacQ1zs,4930
3
- runnable/cli.py,sha256=01zmzOdynEmLI4vWDtSHQ6y1od_Jlc8G1RF69fi2L8g,8446
4
- runnable/context.py,sha256=by5uepmuCP0dmM9BmsliXihSes5QEFejwAsmekcqylE,1388
5
- runnable/datastore.py,sha256=9y5enzn6AXLHLdwvgkdjGPrBkVlrcjfbaAHsst-lJzg,32466
6
- runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
7
- runnable/entrypoints.py,sha256=67gPBiIIS4Kd9g6LdoGCraRJPda8K1i7Lp7XcD2iY5k,18913
8
- runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
9
- runnable/executor.py,sha256=Rafu9EECrNq1LBkJmS6KYCekchP5ufrR04mHWG-JzqQ,15543
10
- runnable/graph.py,sha256=jVjikRLR-so3b2ufmNKpEQ_Ny68qN4bcGDAdXBRKiCY,16574
11
- runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
12
- runnable/nodes.py,sha256=YU9u7r1ESzui1uVtJ1dgwdv1ozyJnF2k-MCFieT8CLI,17519
13
- runnable/parameters.py,sha256=g_bJurLjuppFDiDpfFqy6BRF36o_EY0OC5APl7HJFok,5450
14
- runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
15
- runnable/sdk.py,sha256=xN5F4XX8r5wCN131kgN2xG7MkNm0bSGJ3Ukw8prHYJ8,31444
16
- runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
17
- runnable/tasks.py,sha256=JnIIYQf3YUidHXIN6hiUIfDnegc7_rJMNXuHW4WS9ig,29378
18
- runnable/utils.py,sha256=wqyN7lMW56cBqyE59iDE6_i2HXPkvEUCQ-66UQnIwTA,19993
19
- runnable-0.17.1.dist-info/METADATA,sha256=ST_BmhGguYwYrDH0DlUEhR9QCJz9QM09BraYWO9TJbU,10102
20
- runnable-0.17.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
21
- runnable-0.17.1.dist-info/entry_points.txt,sha256=I92DYldRrCb9HCsoum8GjC2UsQrWpuw2kawXTZpkIz4,1559
22
- runnable-0.17.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
23
- runnable-0.17.1.dist-info/RECORD,,