runnable 0.19.1__py3-none-any.whl → 0.20.0__py3-none-any.whl

extensions/pipeline_executor/local_container.py

@@ -155,39 +155,6 @@ class LocalContainerExecutor(GenericPipelineExecutor):
         )
         self.trigger_node_execution(node=node, map_variable=map_variable, **kwargs)
 
-    # def execute_job(self, node: TaskNode):
-    #     """
-    #     Set up the step log and call the execute node
-
-    #     Args:
-    #         node (BaseNode): _description_
-    #     """
-
-    #     step_log = self._context.run_log_store.create_step_log(
-    #         node.name, node._get_step_log_name(map_variable=None)
-    #     )
-
-    #     self.add_code_identities(node=node, step_log=step_log)
-
-    #     step_log.step_type = node.node_type
-    #     step_log.status = defaults.PROCESSING
-    #     self._context.run_log_store.add_step_log(step_log, self._context.run_id)
-
-    #     command = utils.get_job_execution_command(node)
-    #     self._spin_container(node=node, command=command)
-
-    #     # Check the step log status and warn if necessary. Docker errors are generally suppressed.
-    #     step_log = self._context.run_log_store.get_step_log(
-    #         node._get_step_log_name(map_variable=None), self._context.run_id
-    #     )
-    #     if step_log.status != defaults.SUCCESS:
-    #         msg = (
-    #             "Node execution inside the container failed. Please check the logs.\n"
-    #             "Note: If you do not see any docker issue from your side and the code works properly on local execution"
-    #             "please raise a bug report."
-    #         )
-    #         logger.warning(msg)
-
     def trigger_node_execution(
         self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs
     ):

runnable/cli.py

@@ -164,7 +164,7 @@ def fan(
     python_or_yaml_file: Annotated[
         str, typer.Argument(help="The pipeline definition file")
     ],
-    mode: Annotated[FanMode, typer.Option(help="fan in or fan out")],
+    in_or_out: Annotated[str, typer.Argument(help="The fan mode")],
     map_variable: Annotated[
         str,
         typer.Option(
@@ -172,7 +172,7 @@ def fan(
             help="The map variable dictionary in str",
             show_default=True,
         ),
-    ],
+    ] = "",
     config_file: Annotated[
         str,
         typer.Option(
@@ -197,6 +197,14 @@ def fan(
         ),
     ] = LogLevel.INFO,
     tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
+    mode: Annotated[
+        ExecutionMode,
+        typer.Option(
+            "--mode",
+            "-m",
+            help="spec in yaml or python sdk",
+        ),
+    ] = ExecutionMode.YAML,
 ):
     logger.setLevel(log_level.value)
 
@@ -206,6 +214,7 @@ def fan(
         pipeline_file=python_or_yaml_file,
         step_name=step_name,
         mode=mode,
+        in_or_out=in_or_out,
         map_variable=map_variable,
         run_id=run_id,
         tag=tag,

runnable/entrypoints.py

@@ -332,6 +332,7 @@ def execute_single_node(
         run_id=run_id, configuration_file=configuration_file, tag=tag
     )
 
+    # TODO:  Is it useful to make it get from an environment variable
     map_variable_dict = utils.json_to_ordered_dict(map_variable)
 
     step_internal_name = nodes.BaseNode._get_internal_name_from_command_name(step_name)
@@ -354,6 +355,8 @@ def execute_single_node(
         run_context.catalog_handler.put(name=log_file_name, run_id=run_context.run_id)
         os.remove(log_file_name)
 
+    executor.send_return_code()
+
 
 def execute_job_yaml_spec(
     job_definition_file: str,
@@ -492,6 +495,7 @@ def fan(
     pipeline_file: str,
     step_name: str,
     mode: str,
+    in_or_out: str,
     map_variable: str,
     run_id: str,
     tag: str = "",
@@ -551,10 +555,10 @@ def fan(
 
     map_variable_dict = utils.json_to_ordered_dict(map_variable)
 
-    if mode == "in":
+    if in_or_out == "in":
         logger.info("Fanning in for : %s", node_to_execute)
         executor.fan_in(node=node_to_execute, map_variable=map_variable_dict)
-    elif mode == "out":
+    elif in_or_out == "out":
         logger.info("Fanning out for : %s", node_to_execute)
         executor.fan_out(node=node_to_execute, map_variable=map_variable_dict)
     else:

runnable/sdk.py

@@ -419,6 +419,11 @@ class ShellTask(BaseTask):
     def command_type(self) -> str:
         return "shell"
 
+    def create_job(self) -> RunnableTask:
+        self.terminate_with_success = True
+        node = self.create_node()
+        return node.executable
+
 
 class Stub(BaseTraversal):
     """

runnable/tasks.py

@@ -11,7 +11,7 @@ from datetime import datetime
 from pathlib import Path
 from pickle import PicklingError
 from string import Template
-from typing import Any, Dict, List, Literal, Tuple
+from typing import Any, Dict, List, Literal
 
 from pydantic import BaseModel, ConfigDict, Field, field_validator
 from stevedore import driver
@@ -59,20 +59,6 @@ class BaseTaskType(BaseModel):
     def _context(self):
         return context.run_context
 
-    def get_cli_options(self) -> Tuple[str, dict]:
-        """
-        Key is the name of the cli option and value is the value of the cli option.
-        This should always be in sync with the cli options defined in execute_*.
-
-        Returns:
-            str: The name of the cli option.
-            dict: The dict of cli options for the task.
-
-        Raises:
-            NotImplementedError: Base class, not implemented
-        """
-        raise NotImplementedError()
-
     def set_secrets_as_env_variables(self):
         # Preparing the environment for the task execution
         for key in self.secrets:
@@ -262,14 +248,6 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
     task_type: str = Field(default="python", serialization_alias="command_type")
     command: str
 
-    def get_cli_options(self) -> Tuple[str, dict]:
-        """Return the cli options for the task.
-
-        Returns:
-            dict: The cli options for the task
-        """
-        return "function", {"command": self.command}
-
     def execute_command(
         self,
         map_variable: TypeMapVariable = None,
@@ -425,26 +403,22 @@ class NotebookTaskType(BaseTaskType):
 
         return command
 
-    @property
-    def notebook_output_path(self) -> str:
-        # This is to accommodate jobs which does not have a context_node
+    def get_notebook_output_path(self, map_variable: TypeMapVariable = None) -> str:
+        tag = ""
+        map_variable = map_variable or {}
+        for key, value in map_variable.items():
+            tag += f"{key}_{value}_"
+
         if self._context.executor._context_node:
-            node_name = self._context.executor._context_node.internal_name
-            sane_name = "".join(x for x in node_name if x.isalnum())
-        else:
-            sane_name = ""
+            tag += self._context.executor._context_node.name
+
+        tag = "".join(x for x in tag if x.isalnum())
 
         output_path = Path(".", self.command)
-        file_name = output_path.parent / (output_path.stem + f"{sane_name}_out.ipynb")
+        file_name = output_path.parent / (output_path.stem + f"-{tag}_out.ipynb")
 
         return str(file_name)
 
-    def get_cli_options(self) -> Tuple[str, dict]:
-        return "notebook", {
-            "command": self.command,
-            "notebook-output-path": self.notebook_output_path,
-        }
-
     def execute_command(
         self,
         map_variable: TypeMapVariable = None,
@@ -464,7 +438,9 @@ class NotebookTaskType(BaseTaskType):
             import ploomber_engine as pm
             from ploomber_engine.ipython import PloomberClient
 
-            notebook_output_path = self.notebook_output_path
+            notebook_output_path = self.get_notebook_output_path(
+                map_variable=map_variable
+            )
 
             with (
                 self.execution_context(
@@ -476,7 +452,6 @@ class NotebookTaskType(BaseTaskType):
 
                 if map_variable:
                     for key, value in map_variable.items():
-                        notebook_output_path += "_" + str(value)
                         copy_params[key] = JsonParameter(kind="json", value=value)
 
                 # Remove any {v}_unreduced parameters from the parameters

runnable/utils.py

@@ -455,9 +455,9 @@ def get_fan_command(
     log_level = log_level or logging.getLevelName(logger.getEffectiveLevel())
     action = (
         f"runnable fan {run_id} "
-        f"{node._command_friendly_name()} "
-        f"{context.run_context.pipeline_file} "
-        f"--mode {mode} "
+        f"{node._command_friendly_name()} "  # step name
+        f"{context.run_context.pipeline_file} "  # yaml or python
+        f"{mode} "  # in or out
         f"--log-level {log_level} "
     )
     if context.run_context.configuration_file:
@@ -469,8 +469,8 @@ def get_fan_command(
     if map_variable:
         action = action + f" --map-variable '{json.dumps(map_variable)}'"
 
-    if context.run_context.tag:
-        action = action + f" --tag {context.run_context.tag}"
+    if context.run_context.from_sdk:  # execution mode
+        action = action + " --mode python "
 
     return action
 

{runnable-0.19.1.dist-info → runnable-0.20.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.19.1
+Version: 0.20.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.19.1.dist-info → runnable-0.20.0.dist-info}/RECORD

@@ -15,10 +15,9 @@ extensions/nodes/nodes.py,sha256=ib68QE737ihGLIVp3V2wea13u7lmMZdRvK80bgUkRtA,346
 extensions/nodes/pyproject.toml,sha256=YTu-ETN3JNFSkMzzWeOwn4m-O2nbRH-PmiPBALDCUw4,278
 extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/pipeline_executor/__init__.py,sha256=YnKILiy-SxfnG3rYUoinjh1lfkuAF5QXpPePtn6VxBY,25174
-extensions/pipeline_executor/argo.py,sha256=ClfuU_Of_2f5mvqVgY1QQwwJwXHB0LbzwNArG1x2Axc,44666
-extensions/pipeline_executor/argo_specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
+extensions/pipeline_executor/argo.py,sha256=zA-9zeo93Rvyn-7--bLNwRybREUrtmFRC0QL8oXwNio,32154
 extensions/pipeline_executor/local.py,sha256=H8s6AdML_9_f-vdGG_6k0y9FbLqAqvA1S_7xMNyARzY,1946
-extensions/pipeline_executor/local_container.py,sha256=UCap8wCbHrtTN5acECBBkvcXkA3SXtrAOGW88JT7ofw,13853
+extensions/pipeline_executor/local_container.py,sha256=HOT9I-cPDCvgy6_bzNEtl4jPhTyeYSn1GK7lplH3vDA,12515
 extensions/pipeline_executor/mocked.py,sha256=SuObJ6Myt7p8duW8sylIp1cYIAnFutsJW1avWaOUY3c,5798
 extensions/pipeline_executor/pyproject.toml,sha256=ykTX7srR10PBYb8LsIwEj8vIPPIEZQ5V_R7VYbZ-ido,291
 extensions/pipeline_executor/retry.py,sha256=KGenhWrLLmOQgzMvqloXHDRJyoNs91t05rRW8aLW6FA,6969
@@ -35,11 +34,11 @@ extensions/secrets/dotenv.py,sha256=FbYYd_pVuJuVuIDIvXbzKuSSQ9GPq7xJXTDbJMTQbhM,
 extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
 runnable/__init__.py,sha256=fYkOrbsb-E1rGkrof7kOJ3KboTFH-HriGa-8npn4-50,625
 runnable/catalog.py,sha256=b9N40kTv1IBidzlWjkHcBGyYhq6qIDHZfBuFenzjsMI,4924
-runnable/cli.py,sha256=01zmzOdynEmLI4vWDtSHQ6y1od_Jlc8G1RF69fi2L8g,8446
+runnable/cli.py,sha256=xHb2VSC16z9IT45SnhUYHuXUrzPqF_pK-jyWiTnz5sM,8670
 runnable/context.py,sha256=by5uepmuCP0dmM9BmsliXihSes5QEFejwAsmekcqylE,1388
 runnable/datastore.py,sha256=9y5enzn6AXLHLdwvgkdjGPrBkVlrcjfbaAHsst-lJzg,32466
 runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
-runnable/entrypoints.py,sha256=P958nFz5WAsgTwd9sW04Q30vtjweYpr3rPsHVY4gh2U,18876
+runnable/entrypoints.py,sha256=DxboaCJ4ADZjIHl_oSPsC74ChUCKYic3i8dQqd6U43w,19009
 runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
 runnable/executor.py,sha256=ZPpfKwjDJnta03M2cWIINXcwke2ZDVc_QrIw7kwpHDQ,15547
 runnable/graph.py,sha256=jVjikRLR-so3b2ufmNKpEQ_Ny68qN4bcGDAdXBRKiCY,16574
@@ -47,12 +46,12 @@ runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
 runnable/nodes.py,sha256=YU9u7r1ESzui1uVtJ1dgwdv1ozyJnF2k-MCFieT8CLI,17519
 runnable/parameters.py,sha256=LyQb1d0SaFeI4PJ_yDYt9wArm9ThSPASWb36TwIdDUs,5213
 runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
-runnable/sdk.py,sha256=xN5F4XX8r5wCN131kgN2xG7MkNm0bSGJ3Ukw8prHYJ8,31444
+runnable/sdk.py,sha256=thffRpZgkz1XW4DUY6EIJPyF8NLaTE1puh43dZKW9MU,31595
 runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
-runnable/tasks.py,sha256=JnIIYQf3YUidHXIN6hiUIfDnegc7_rJMNXuHW4WS9ig,29378
-runnable/utils.py,sha256=Kwf54tHMVXYK7MCmvAi_FG08U_bHDKIQO-HDpM9X0QI,19500
-runnable-0.19.1.dist-info/METADATA,sha256=INBkKmT9vbToqxJs3UCH4G2Db1G8Gk7mR-Jsk-r99EE,9945
-runnable-0.19.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-runnable-0.19.1.dist-info/entry_points.txt,sha256=seek5WVGvwYALm8lZ0TfPXwG5NaCeUKjU8urF8k3gvY,1621
-runnable-0.19.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.19.1.dist-info/RECORD,,
+runnable/tasks.py,sha256=WdXFUL9mT3zmUc5YoOrTmwfjss02MwNKanf11pn8py4,28497
+runnable/utils.py,sha256=h-E6ZmUDcKwrJLjKlu0XXAK4uzfnD0K0EEPpZmeSIfM,19542
+runnable-0.20.0.dist-info/METADATA,sha256=njNTjovB_HmjC52tNvGJkj5Ae0aZV4nLeNdjwynXnKE,9945
+runnable-0.20.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.20.0.dist-info/entry_points.txt,sha256=seek5WVGvwYALm8lZ0TfPXwG5NaCeUKjU8urF8k3gvY,1621
+runnable-0.20.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.20.0.dist-info/RECORD,,

extensions/pipeline_executor/argo_specification.yaml

@@ -1,51 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Workflow
-metadata:
-  generateName: runnable-dag
-spec:
-  activeDeadlineSeconds: int # max run time of the workflow
-  entrypoint: str
-  nodeSelector: Dict[str, str] # global node selector
-  parallelism: # global level
-  podGC: OnPodCompletion
-  resources: # Should be converted to podSpecPath
-    limits:
-    requests:
-  podSpecPatch: json str representation of resources for defaults
-  retryStrategy: # global level for all templates
-    limit: int
-    retryPolicy: # global level for all templates
-    backoff:
-      duration: str
-      factor: int
-      maxDuration: str
-  serviceAccountName: str # Optionally required
-  templateDefaults:
-    activeDeadlineSeconds: int, for a template
-    timeout: str # max time including the wait time
-    failFast: true
-    volumes:
-  templates:
-    activeDeadlineSeconds: # override
-    nodeSelector: # override
-    retryStrategy: # override
-    tolerations: # override
-    container:
-      command:
-      env:
-      image:
-      imagePullPolicy:
-      volumeMounts:
-      resources:
-        limits:
-        requests:
-    dag:
-      tasks:
-        depends:
-        continueOn:
-  tolerations: # global level for all templates
-    effect: str
-    key: str
-    operator: str
-    value: str
-  volumes: