runnable 0.20.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -244,6 +244,7 @@ class TemplateDefaults(BaseModelWIthConfig):
244
244
  image: str
245
245
  image_pull_policy: Optional[ImagePullPolicy] = Field(default=ImagePullPolicy.Always)
246
246
  resources: Resources = Field(default_factory=Resources)
247
+ env: list[EnvVar | SecretEnvVar] = Field(default_factory=list, exclude=True)
247
248
 
248
249
 
249
250
  # User provides this as part of the argoSpec
@@ -365,6 +366,7 @@ class ArgoExecutor(GenericPipelineExecutor):
365
366
  custom_volumes: Optional[list[CustomVolume]] = Field(
366
367
  default_factory=list[CustomVolume]
367
368
  )
369
+ env: list[EnvVar] = Field(default_factory=list[EnvVar])
368
370
 
369
371
  expose_parameters_as_inputs: bool = True
370
372
  secret_from_k8s: Optional[str] = Field(default=None)
@@ -508,6 +510,10 @@ class ArgoExecutor(GenericPipelineExecutor):
508
510
  )
509
511
 
510
512
  self._set_up_initial_container(container_template=core_container_template)
513
+ self._expose_secrets_to_task(
514
+ working_on=node, container_template=core_container_template
515
+ )
516
+ self._set_env_vars_to_task(node, core_container_template)
511
517
 
512
518
  container_template = ContainerTemplate(
513
519
  container=core_container_template,
@@ -523,12 +529,32 @@ class ArgoExecutor(GenericPipelineExecutor):
523
529
 
524
530
  return container_template
525
531
 
532
+ def _set_env_vars_to_task(
533
+ self, working_on: BaseNode, container_template: CoreContainerTemplate
534
+ ):
535
+ if not isinstance(working_on, TaskNode):
536
+ return
537
+ global_envs: dict[str, str] = {}
538
+
539
+ for env_var in self.env:
540
+ global_envs[env_var.name] = env_var.value
541
+
542
+ node_overide = {}
543
+ if hasattr(working_on, "overides"):
544
+ node_overide = working_on.overides
545
+
546
+ global_envs.update(node_overide.get("env", {}))
547
+ for key, value in global_envs.items():
548
+ env_var_to_add = EnvVar(name=key, value=value)
549
+ container_template.env.append(env_var_to_add)
550
+
526
551
  def _expose_secrets_to_task(
527
552
  self,
528
553
  working_on: BaseNode,
529
554
  container_template: CoreContainerTemplate,
530
555
  ):
531
- assert isinstance(working_on, TaskNode)
556
+ if not isinstance(working_on, TaskNode):
557
+ return
532
558
  secrets = working_on.executable.secrets
533
559
  for secret in secrets:
534
560
  assert self.secret_from_k8s is not None
runnable/__init__.py CHANGED
@@ -20,12 +20,14 @@ task_console = Console(record=True)
20
20
  from runnable.sdk import ( # noqa
21
21
  Catalog,
22
22
  Fail,
23
- Job,
24
23
  Map,
24
+ NotebookJob,
25
25
  NotebookTask,
26
26
  Parallel,
27
27
  Pipeline,
28
+ PythonJob,
28
29
  PythonTask,
30
+ ShellJob,
29
31
  ShellTask,
30
32
  Stub,
31
33
  Success,
runnable/entrypoints.py CHANGED
@@ -438,7 +438,7 @@ def set_job_spec_from_python(run_context: context.Context, python_module: str):
438
438
  imported_module = importlib.import_module(module)
439
439
 
440
440
  run_context.from_sdk = True
441
- task = getattr(imported_module, func)().return_task()
441
+ task = getattr(imported_module, func)().get_task()
442
442
  catalog_settings = getattr(imported_module, func)().return_catalog_settings()
443
443
 
444
444
  run_context.job_definition_file = python_module
runnable/sdk.py CHANGED
@@ -822,17 +822,21 @@ class Pipeline(BaseModel):
822
822
  )
823
823
 
824
824
 
825
- class Job(BaseModel):
826
- name: str
827
- task: BaseTask
825
+ class BaseJob(BaseModel):
826
+ catalog: Optional[Catalog] = Field(default=None, alias="catalog")
827
+ overrides: Dict[str, Any] = Field(default_factory=dict, alias="overrides")
828
+ returns: List[Union[str, TaskReturns]] = Field(
829
+ default_factory=list, alias="returns"
830
+ )
831
+ secrets: List[str] = Field(default_factory=list)
828
832
 
829
- def return_task(self) -> RunnableTask:
830
- return self.task.create_job()
833
+ def get_task(self) -> RunnableTask:
834
+ raise NotImplementedError
831
835
 
832
836
  def return_catalog_settings(self) -> Optional[List[str]]:
833
- if self.task.catalog is None:
837
+ if self.catalog is None:
834
838
  return []
835
- return self.task.catalog.put
839
+ return self.catalog.put
836
840
 
837
841
  def _is_called_for_definition(self) -> bool:
838
842
  """
@@ -892,7 +896,7 @@ class Job(BaseModel):
892
896
 
893
897
  run_context.job_definition_file = f"{module_to_call}.py"
894
898
 
895
- job = self.task.create_job()
899
+ job = self.get_task()
896
900
  catalog_settings = self.return_catalog_settings()
897
901
 
898
902
  run_context.executor.submit_job(job, catalog_settings=catalog_settings)
@@ -905,3 +909,60 @@ class Job(BaseModel):
905
909
  return run_context.run_log_store.get_run_log_by_id(
906
910
  run_id=run_context.run_id
907
911
  )
912
+
913
+
914
+ class PythonJob(BaseJob):
915
+ function: Callable = Field(exclude=True)
916
+
917
+ @property
918
+ @computed_field
919
+ def command(self) -> str:
920
+ module = self.function.__module__
921
+ name = self.function.__name__
922
+
923
+ return f"{module}.{name}"
924
+
925
+ def get_task(self) -> RunnableTask:
926
+ # Piggy bank on existing tasks as a hack
927
+ task = PythonTask(
928
+ name="dummy",
929
+ terminate_with_success=True,
930
+ returns=self.returns,
931
+ secrets=self.secrets,
932
+ function=self.function,
933
+ )
934
+ return task.create_node().executable
935
+
936
+
937
+ class NotebookJob(BaseJob):
938
+ notebook: str = Field(serialization_alias="command")
939
+ optional_ploomber_args: Optional[Dict[str, Any]] = Field(
940
+ default=None, alias="optional_ploomber_args"
941
+ )
942
+
943
+ def get_task(self) -> RunnableTask:
944
+ # Piggy bank on existing tasks as a hack
945
+ task = NotebookTask(
946
+ name="dummy",
947
+ terminate_with_success=True,
948
+ returns=self.returns,
949
+ secrets=self.secrets,
950
+ notebook=self.notebook,
951
+ optional_ploomber_args=self.optional_ploomber_args,
952
+ )
953
+ return task.create_node().executable
954
+
955
+
956
+ class ShellJob(BaseJob):
957
+ command: str = Field(alias="command")
958
+
959
+ def get_task(self) -> RunnableTask:
960
+ # Piggy bank on existing tasks as a hack
961
+ task = ShellTask(
962
+ name="dummy",
963
+ terminate_with_success=True,
964
+ returns=self.returns,
965
+ secrets=self.secrets,
966
+ command=self.command,
967
+ )
968
+ return task.create_node().executable
runnable/tasks.py CHANGED
@@ -409,10 +409,10 @@ class NotebookTaskType(BaseTaskType):
409
409
  for key, value in map_variable.items():
410
410
  tag += f"{key}_{value}_"
411
411
 
412
- if self._context.executor._context_node:
412
+ if hasattr(self._context.executor, "_context_node"):
413
413
  tag += self._context.executor._context_node.name
414
414
 
415
- tag = "".join(x for x in tag if x.isalnum())
415
+ tag = "".join(x for x in tag if x.isalnum()).strip("-")
416
416
 
417
417
  output_path = Path(".", self.command)
418
418
  file_name = output_path.parent / (output_path.stem + f"-{tag}_out.ipynb")
runnable/utils.py CHANGED
@@ -109,8 +109,16 @@ def apply_variables(
109
109
  raise Exception("Argument Variables should be dict")
110
110
 
111
111
  json_d = json.dumps(apply_to)
112
- transformed = str_template(json_d).substitute(**variables)
113
- return json.loads(transformed)
112
+ string_template = str_template(json_d)
113
+
114
+ template = string_template.safe_substitute(variables)
115
+
116
+ if "$" in template:
117
+ logger.warning(
118
+ "Not all variables found in the config are found in the variables"
119
+ )
120
+
121
+ return json.loads(template)
114
122
 
115
123
 
116
124
  def get_module_and_attr_names(command: str) -> Tuple[str, str]:
@@ -490,7 +498,7 @@ def get_job_execution_command(over_write_run_id: str = "") -> str:
490
498
  log_level = logging.getLevelName(logger.getEffectiveLevel())
491
499
 
492
500
  action = (
493
- f"runnable execute-job /app/{context.run_context.job_definition_file} {run_id} "
501
+ f"runnable execute-job {context.run_context.job_definition_file} {run_id} "
494
502
  f" --log-level {log_level}"
495
503
  )
496
504
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: runnable
3
- Version: 0.20.0
3
+ Version: 0.22.0
4
4
  Summary: Add your description here
5
5
  Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
6
6
  License-File: LICENSE
@@ -15,7 +15,7 @@ extensions/nodes/nodes.py,sha256=ib68QE737ihGLIVp3V2wea13u7lmMZdRvK80bgUkRtA,346
15
15
  extensions/nodes/pyproject.toml,sha256=YTu-ETN3JNFSkMzzWeOwn4m-O2nbRH-PmiPBALDCUw4,278
16
16
  extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
17
17
  extensions/pipeline_executor/__init__.py,sha256=YnKILiy-SxfnG3rYUoinjh1lfkuAF5QXpPePtn6VxBY,25174
18
- extensions/pipeline_executor/argo.py,sha256=zA-9zeo93Rvyn-7--bLNwRybREUrtmFRC0QL8oXwNio,32154
18
+ extensions/pipeline_executor/argo.py,sha256=81Lv8WO2riYo4KdtgpRabl5YtSgrNGGq8PfMo6YLKUY,33154
19
19
  extensions/pipeline_executor/local.py,sha256=H8s6AdML_9_f-vdGG_6k0y9FbLqAqvA1S_7xMNyARzY,1946
20
20
  extensions/pipeline_executor/local_container.py,sha256=HOT9I-cPDCvgy6_bzNEtl4jPhTyeYSn1GK7lplH3vDA,12515
21
21
  extensions/pipeline_executor/mocked.py,sha256=SuObJ6Myt7p8duW8sylIp1cYIAnFutsJW1avWaOUY3c,5798
@@ -32,13 +32,13 @@ extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
32
32
  extensions/secrets/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
33
33
  extensions/secrets/dotenv.py,sha256=FbYYd_pVuJuVuIDIvXbzKuSSQ9GPq7xJXTDbJMTQbhM,1583
34
34
  extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
35
- runnable/__init__.py,sha256=fYkOrbsb-E1rGkrof7kOJ3KboTFH-HriGa-8npn4-50,625
35
+ runnable/__init__.py,sha256=n14AnTUUEYxXlTJ6-YLT0tMmeFb7Co_3kNldV6pgKSs,662
36
36
  runnable/catalog.py,sha256=b9N40kTv1IBidzlWjkHcBGyYhq6qIDHZfBuFenzjsMI,4924
37
37
  runnable/cli.py,sha256=xHb2VSC16z9IT45SnhUYHuXUrzPqF_pK-jyWiTnz5sM,8670
38
38
  runnable/context.py,sha256=by5uepmuCP0dmM9BmsliXihSes5QEFejwAsmekcqylE,1388
39
39
  runnable/datastore.py,sha256=9y5enzn6AXLHLdwvgkdjGPrBkVlrcjfbaAHsst-lJzg,32466
40
40
  runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
41
- runnable/entrypoints.py,sha256=DxboaCJ4ADZjIHl_oSPsC74ChUCKYic3i8dQqd6U43w,19009
41
+ runnable/entrypoints.py,sha256=5boTvQExAVwujLW-Lmic6pjGY122qwZB8ZnaKlV8Fac,19006
42
42
  runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
43
43
  runnable/executor.py,sha256=ZPpfKwjDJnta03M2cWIINXcwke2ZDVc_QrIw7kwpHDQ,15547
44
44
  runnable/graph.py,sha256=jVjikRLR-so3b2ufmNKpEQ_Ny68qN4bcGDAdXBRKiCY,16574
@@ -46,12 +46,12 @@ runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
46
46
  runnable/nodes.py,sha256=YU9u7r1ESzui1uVtJ1dgwdv1ozyJnF2k-MCFieT8CLI,17519
47
47
  runnable/parameters.py,sha256=LyQb1d0SaFeI4PJ_yDYt9wArm9ThSPASWb36TwIdDUs,5213
48
48
  runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
49
- runnable/sdk.py,sha256=thffRpZgkz1XW4DUY6EIJPyF8NLaTE1puh43dZKW9MU,31595
49
+ runnable/sdk.py,sha256=-FvRoIMlbNFOS_8c2kefYA-mlyE_15rNG3GoN6gKumc,33470
50
50
  runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
51
- runnable/tasks.py,sha256=WdXFUL9mT3zmUc5YoOrTmwfjss02MwNKanf11pn8py4,28497
52
- runnable/utils.py,sha256=h-E6ZmUDcKwrJLjKlu0XXAK4uzfnD0K0EEPpZmeSIfM,19542
53
- runnable-0.20.0.dist-info/METADATA,sha256=njNTjovB_HmjC52tNvGJkj5Ae0aZV4nLeNdjwynXnKE,9945
54
- runnable-0.20.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
55
- runnable-0.20.0.dist-info/entry_points.txt,sha256=seek5WVGvwYALm8lZ0TfPXwG5NaCeUKjU8urF8k3gvY,1621
56
- runnable-0.20.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
57
- runnable-0.20.0.dist-info/RECORD,,
51
+ runnable/tasks.py,sha256=YVKpKxSpsRZWcU3MOqoBoqxeo1XSqv5crkOu6kyu63o,28520
52
+ runnable/utils.py,sha256=HyYo6Xe4O02eHuT_m9oLqzh7dxdhbmtTqKzgFR2bpWE,19712
53
+ runnable-0.22.0.dist-info/METADATA,sha256=KsHHJP5DJOu9S1yv7SEid62OEnSg_eBZfBGbgYUWUiw,9945
54
+ runnable-0.22.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
55
+ runnable-0.22.0.dist-info/entry_points.txt,sha256=seek5WVGvwYALm8lZ0TfPXwG5NaCeUKjU8urF8k3gvY,1621
56
+ runnable-0.22.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
57
+ runnable-0.22.0.dist-info/RECORD,,