runnable 0.20.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

extensions/pipeline_executor/argo.py

@@ -244,6 +244,7 @@ class TemplateDefaults(BaseModelWIthConfig):
     image: str
     image_pull_policy: Optional[ImagePullPolicy] = Field(default=ImagePullPolicy.Always)
     resources: Resources = Field(default_factory=Resources)
+    env: list[EnvVar | SecretEnvVar] = Field(default_factory=list, exclude=True)
 
 
 # User provides this as part of the argoSpec
@@ -365,6 +366,7 @@ class ArgoExecutor(GenericPipelineExecutor):
     custom_volumes: Optional[list[CustomVolume]] = Field(
         default_factory=list[CustomVolume]
     )
+    env: list[EnvVar] = Field(default_factory=list[EnvVar])
 
     expose_parameters_as_inputs: bool = True
     secret_from_k8s: Optional[str] = Field(default=None)
@@ -508,6 +510,10 @@ class ArgoExecutor(GenericPipelineExecutor):
         )
 
         self._set_up_initial_container(container_template=core_container_template)
+        self._expose_secrets_to_task(
+            working_on=node, container_template=core_container_template
+        )
+        self._set_env_vars_to_task(node, core_container_template)
 
         container_template = ContainerTemplate(
             container=core_container_template,
@@ -523,12 +529,32 @@ class ArgoExecutor(GenericPipelineExecutor):
 
         return container_template
 
+    def _set_env_vars_to_task(
+        self, working_on: BaseNode, container_template: CoreContainerTemplate
+    ):
+        if not isinstance(working_on, TaskNode):
+            return
+        global_envs: dict[str, str] = {}
+
+        for env_var in self.env:
+            global_envs[env_var.name] = env_var.value
+
+        node_overide = {}
+        if hasattr(working_on, "overides"):
+            node_overide = working_on.overides
+
+        global_envs.update(node_overide.get("env", {}))
+        for key, value in global_envs.items():
+            env_var_to_add = EnvVar(name=key, value=value)
+            container_template.env.append(env_var_to_add)
+
     def _expose_secrets_to_task(
         self,
         working_on: BaseNode,
         container_template: CoreContainerTemplate,
     ):
-        assert isinstance(working_on, TaskNode)
+        if not isinstance(working_on, TaskNode):
+            return
         secrets = working_on.executable.secrets
         for secret in secrets:
             assert self.secret_from_k8s is not None

runnable/__init__.py

@@ -20,12 +20,14 @@ task_console = Console(record=True)
 from runnable.sdk import (  # noqa
     Catalog,
     Fail,
-    Job,
     Map,
+    NotebookJob,
     NotebookTask,
     Parallel,
     Pipeline,
+    PythonJob,
     PythonTask,
+    ShellJob,
     ShellTask,
     Stub,
     Success,

runnable/entrypoints.py

@@ -438,7 +438,7 @@ def set_job_spec_from_python(run_context: context.Context, python_module: str):
     imported_module = importlib.import_module(module)
 
     run_context.from_sdk = True
-    task = getattr(imported_module, func)().return_task()
+    task = getattr(imported_module, func)().get_task()
     catalog_settings = getattr(imported_module, func)().return_catalog_settings()
 
     run_context.job_definition_file = python_module

runnable/sdk.py

@@ -822,17 +822,21 @@ class Pipeline(BaseModel):
             )
 
 
-class Job(BaseModel):
-    name: str
-    task: BaseTask
+class BaseJob(BaseModel):
+    catalog: Optional[Catalog] = Field(default=None, alias="catalog")
+    overrides: Dict[str, Any] = Field(default_factory=dict, alias="overrides")
+    returns: List[Union[str, TaskReturns]] = Field(
+        default_factory=list, alias="returns"
+    )
+    secrets: List[str] = Field(default_factory=list)
 
-    def return_task(self) -> RunnableTask:
-        return self.task.create_job()
+    def get_task(self) -> RunnableTask:
+        raise NotImplementedError
 
     def return_catalog_settings(self) -> Optional[List[str]]:
-        if self.task.catalog is None:
+        if self.catalog is None:
             return []
-        return self.task.catalog.put
+        return self.catalog.put
 
     def _is_called_for_definition(self) -> bool:
         """
@@ -892,7 +896,7 @@ class Job(BaseModel):
 
             run_context.job_definition_file = f"{module_to_call}.py"
 
-        job = self.task.create_job()
+        job = self.get_task()
         catalog_settings = self.return_catalog_settings()
 
         run_context.executor.submit_job(job, catalog_settings=catalog_settings)
@@ -905,3 +909,60 @@ class Job(BaseModel):
             return run_context.run_log_store.get_run_log_by_id(
                 run_id=run_context.run_id
             )
+
+
+class PythonJob(BaseJob):
+    function: Callable = Field(exclude=True)
+
+    @property
+    @computed_field
+    def command(self) -> str:
+        module = self.function.__module__
+        name = self.function.__name__
+
+        return f"{module}.{name}"
+
+    def get_task(self) -> RunnableTask:
+        # Piggy bank on existing tasks as a hack
+        task = PythonTask(
+            name="dummy",
+            terminate_with_success=True,
+            returns=self.returns,
+            secrets=self.secrets,
+            function=self.function,
+        )
+        return task.create_node().executable
+
+
+class NotebookJob(BaseJob):
+    notebook: str = Field(serialization_alias="command")
+    optional_ploomber_args: Optional[Dict[str, Any]] = Field(
+        default=None, alias="optional_ploomber_args"
+    )
+
+    def get_task(self) -> RunnableTask:
+        # Piggy bank on existing tasks as a hack
+        task = NotebookTask(
+            name="dummy",
+            terminate_with_success=True,
+            returns=self.returns,
+            secrets=self.secrets,
+            notebook=self.notebook,
+            optional_ploomber_args=self.optional_ploomber_args,
+        )
+        return task.create_node().executable
+
+
+class ShellJob(BaseJob):
+    command: str = Field(alias="command")
+
+    def get_task(self) -> RunnableTask:
+        # Piggy bank on existing tasks as a hack
+        task = ShellTask(
+            name="dummy",
+            terminate_with_success=True,
+            returns=self.returns,
+            secrets=self.secrets,
+            command=self.command,
+        )
+        return task.create_node().executable

runnable/tasks.py

@@ -409,10 +409,10 @@ class NotebookTaskType(BaseTaskType):
         for key, value in map_variable.items():
             tag += f"{key}_{value}_"
 
-        if self._context.executor._context_node:
+        if hasattr(self._context.executor, "_context_node"):
             tag += self._context.executor._context_node.name
 
-        tag = "".join(x for x in tag if x.isalnum())
+        tag = "".join(x for x in tag if x.isalnum()).strip("-")
 
         output_path = Path(".", self.command)
         file_name = output_path.parent / (output_path.stem + f"-{tag}_out.ipynb")

runnable/utils.py

@@ -109,8 +109,16 @@ def apply_variables(
         raise Exception("Argument Variables should be dict")
 
     json_d = json.dumps(apply_to)
-    transformed = str_template(json_d).substitute(**variables)
-    return json.loads(transformed)
+    string_template = str_template(json_d)
+
+    template = string_template.safe_substitute(variables)
+
+    if "$" in template:
+        logger.warning(
+            "Not all variables found in the config are found in the variables"
+        )
+
+    return json.loads(template)
 
 
 def get_module_and_attr_names(command: str) -> Tuple[str, str]:
@@ -490,7 +498,7 @@ def get_job_execution_command(over_write_run_id: str = "") -> str:
     log_level = logging.getLevelName(logger.getEffectiveLevel())
 
     action = (
-        f"runnable execute-job /app/{context.run_context.job_definition_file} {run_id} "
+        f"runnable execute-job {context.run_context.job_definition_file} {run_id} "
         f" --log-level {log_level}"
     )
 

{runnable-0.20.0.dist-info → runnable-0.22.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.20.0
+Version: 0.22.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.20.0.dist-info → runnable-0.22.0.dist-info}/RECORD

@@ -15,7 +15,7 @@ extensions/nodes/nodes.py,sha256=ib68QE737ihGLIVp3V2wea13u7lmMZdRvK80bgUkRtA,346
 extensions/nodes/pyproject.toml,sha256=YTu-ETN3JNFSkMzzWeOwn4m-O2nbRH-PmiPBALDCUw4,278
 extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/pipeline_executor/__init__.py,sha256=YnKILiy-SxfnG3rYUoinjh1lfkuAF5QXpPePtn6VxBY,25174
-extensions/pipeline_executor/argo.py,sha256=zA-9zeo93Rvyn-7--bLNwRybREUrtmFRC0QL8oXwNio,32154
+extensions/pipeline_executor/argo.py,sha256=81Lv8WO2riYo4KdtgpRabl5YtSgrNGGq8PfMo6YLKUY,33154
 extensions/pipeline_executor/local.py,sha256=H8s6AdML_9_f-vdGG_6k0y9FbLqAqvA1S_7xMNyARzY,1946
 extensions/pipeline_executor/local_container.py,sha256=HOT9I-cPDCvgy6_bzNEtl4jPhTyeYSn1GK7lplH3vDA,12515
 extensions/pipeline_executor/mocked.py,sha256=SuObJ6Myt7p8duW8sylIp1cYIAnFutsJW1avWaOUY3c,5798
@@ -32,13 +32,13 @@ extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 extensions/secrets/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/secrets/dotenv.py,sha256=FbYYd_pVuJuVuIDIvXbzKuSSQ9GPq7xJXTDbJMTQbhM,1583
 extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
-runnable/__init__.py,sha256=fYkOrbsb-E1rGkrof7kOJ3KboTFH-HriGa-8npn4-50,625
+runnable/__init__.py,sha256=n14AnTUUEYxXlTJ6-YLT0tMmeFb7Co_3kNldV6pgKSs,662
 runnable/catalog.py,sha256=b9N40kTv1IBidzlWjkHcBGyYhq6qIDHZfBuFenzjsMI,4924
 runnable/cli.py,sha256=xHb2VSC16z9IT45SnhUYHuXUrzPqF_pK-jyWiTnz5sM,8670
 runnable/context.py,sha256=by5uepmuCP0dmM9BmsliXihSes5QEFejwAsmekcqylE,1388
 runnable/datastore.py,sha256=9y5enzn6AXLHLdwvgkdjGPrBkVlrcjfbaAHsst-lJzg,32466
 runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
-runnable/entrypoints.py,sha256=DxboaCJ4ADZjIHl_oSPsC74ChUCKYic3i8dQqd6U43w,19009
+runnable/entrypoints.py,sha256=5boTvQExAVwujLW-Lmic6pjGY122qwZB8ZnaKlV8Fac,19006
 runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
 runnable/executor.py,sha256=ZPpfKwjDJnta03M2cWIINXcwke2ZDVc_QrIw7kwpHDQ,15547
 runnable/graph.py,sha256=jVjikRLR-so3b2ufmNKpEQ_Ny68qN4bcGDAdXBRKiCY,16574
@@ -46,12 +46,12 @@ runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
 runnable/nodes.py,sha256=YU9u7r1ESzui1uVtJ1dgwdv1ozyJnF2k-MCFieT8CLI,17519
 runnable/parameters.py,sha256=LyQb1d0SaFeI4PJ_yDYt9wArm9ThSPASWb36TwIdDUs,5213
 runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
-runnable/sdk.py,sha256=thffRpZgkz1XW4DUY6EIJPyF8NLaTE1puh43dZKW9MU,31595
+runnable/sdk.py,sha256=-FvRoIMlbNFOS_8c2kefYA-mlyE_15rNG3GoN6gKumc,33470
 runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
-runnable/tasks.py,sha256=WdXFUL9mT3zmUc5YoOrTmwfjss02MwNKanf11pn8py4,28497
-runnable/utils.py,sha256=h-E6ZmUDcKwrJLjKlu0XXAK4uzfnD0K0EEPpZmeSIfM,19542
-runnable-0.20.0.dist-info/METADATA,sha256=njNTjovB_HmjC52tNvGJkj5Ae0aZV4nLeNdjwynXnKE,9945
-runnable-0.20.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-runnable-0.20.0.dist-info/entry_points.txt,sha256=seek5WVGvwYALm8lZ0TfPXwG5NaCeUKjU8urF8k3gvY,1621
-runnable-0.20.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.20.0.dist-info/RECORD,,
+runnable/tasks.py,sha256=YVKpKxSpsRZWcU3MOqoBoqxeo1XSqv5crkOu6kyu63o,28520
+runnable/utils.py,sha256=HyYo6Xe4O02eHuT_m9oLqzh7dxdhbmtTqKzgFR2bpWE,19712
+runnable-0.22.0.dist-info/METADATA,sha256=KsHHJP5DJOu9S1yv7SEid62OEnSg_eBZfBGbgYUWUiw,9945
+runnable-0.22.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.22.0.dist-info/entry_points.txt,sha256=seek5WVGvwYALm8lZ0TfPXwG5NaCeUKjU8urF8k3gvY,1621
+runnable-0.22.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.22.0.dist-info/RECORD,,