runnable 0.29.0__py3-none-any.whl → 0.30.1__py3-none-any.whl

extensions/job_executor/local.py

@@ -12,6 +12,12 @@ logger = logging.getLogger(defaults.LOGGER_NAME)
 class LocalJobExecutor(GenericJobExecutor):
     """
     The LocalJobExecutor is a job executor that runs the job locally.
+
+    Configuration:
+
+    pipeline-executor:
+        type: local
+
     """
 
     service_name: str = "local"

extensions/nodes/torch.py

@@ -115,7 +115,9 @@ class TorchNode(DistributedNode, TorchConfig):
         map_variable: TypeMapVariable = None,
         attempt_number: int = 1,
     ) -> StepLog:
-        assert map_variable is None, "TorchNode does not support map_variable"
+        assert (
+            map_variable is None or not map_variable
+        ), "TorchNode does not support map_variable"
 
         step_log = self._context.run_log_store.get_step_log(
             self._get_step_log_name(map_variable), self._context.run_id
@@ -130,6 +132,8 @@ class TorchNode(DistributedNode, TorchConfig):
             self._context.parameters_file or ""
         )
         os.environ["RUNNABLE_TORCH_RUN_ID"] = self._context.run_id
+        # retrieve the master address and port from the parameters
+        # default to localhost and 29500
         launcher = elastic_launch(
             launch_config,
             training_subprocess,
@@ -161,9 +165,17 @@ class TorchNode(DistributedNode, TorchConfig):
 
         return step_log
 
-    # TODO: Not sure we need these methods
     def fan_in(self, map_variable: dict[str, str | int | float] | None = None):
-        assert map_variable is None, "TorchNode does not support map_variable"
+        # Destroy the service
+        # Destroy the statefulset
+        assert (
+            map_variable is None or not map_variable
+        ), "TorchNode does not support map_variable"
 
     def fan_out(self, map_variable: dict[str, str | int | float] | None = None):
-        assert map_variable is None, "TorchNode does not support map_variable"
+        # Create a service
+        # Create a statefulset
+        # Gather the IPs and set them as parameters downstream
+        assert (
+            map_variable is None or not map_variable
+        ), "TorchNode does not support map_variable"

extensions/pipeline_executor/argo.py

@@ -27,6 +27,7 @@ from extensions.nodes.nodes import (
     SuccessNode,
     TaskNode,
 )
+from extensions.nodes.torch import TorchNode
 from extensions.pipeline_executor import GenericPipelineExecutor
 from runnable import defaults, utils
 from runnable.defaults import TypeMapVariable
@@ -370,6 +371,89 @@ class CustomVolume(BaseModelWIthConfig):
 
 
 class ArgoExecutor(GenericPipelineExecutor):
+    """
+    Executes the pipeline using Argo Workflows.
+
+    The defaults configuration is kept similar to the
+    [Argo Workflow spec](https://argo-workflows.readthedocs.io/en/latest/fields/#workflow).
+
+    Configuration:
+
+    ```yaml
+    pipeline-executor:
+      type: argo
+      config:
+        pvc_for_runnable: "my-pvc"
+        custom_volumes:
+          - mount_path: "/tmp"
+            persistent_volume_claim:
+              claim_name: "my-pvc"
+              read_only: false/true
+        expose_parameters_as_inputs: true/false
+        secrets_from_k8s:
+          - key1
+          - key2
+          - ...
+        output_file: "argo-pipeline.yaml"
+        log_level: "DEBUG"/"INFO"/"WARNING"/"ERROR"/"CRITICAL"
+        defaults:
+          image: "my-image"
+          activeDeadlineSeconds: 86400
+          failFast: true
+          nodeSelector:
+            label: value
+          parallelism: 1
+          retryStrategy:
+            backoff:
+            duration: "2m"
+            factor: 2
+            maxDuration: "1h"
+            limit: 0
+            retryPolicy: "Always"
+          timeout: "1h"
+          tolerations:
+          imagePullPolicy: "Always"/"IfNotPresent"/"Never"
+          resources:
+            limits:
+              memory: "1Gi"
+              cpu: "250m"
+              gpu: 0
+            requests:
+              memory: "1Gi"
+              cpu: "250m"
+          env:
+            - name: "MY_ENV"
+            value: "my-value"
+            - name: secret_env
+            secretName: "my-secret"
+            secretKey: "my-key"
+        overrides:
+          key1:
+            ... similar structure to defaults
+
+        argoWorkflow:
+          metadata:
+            annotations:
+              key1: value1
+              key2: value2
+            generateName: "my-workflow"
+            labels:
+              key1: value1
+
+    ```
+
+    As of now, ```runnable``` needs a pvc to store the logs and the catalog; provided by ```pvc_for_runnable```.
+    - ```custom_volumes``` can be used to mount additional volumes to the container.
+
+    - ```expose_parameters_as_inputs``` can be used to expose the initial parameters as inputs to the workflow.
+    - ```secrets_from_k8s``` can be used to expose the secrets from the k8s secret store.
+    - ```output_file``` is the file where the argo pipeline will be dumped.
+    - ```log_level``` is the log level for the containers.
+    - ```defaults``` is the default configuration for all the containers.
+
+
+    """
+
     service_name: str = "argo"
     _is_local: bool = False
     mock: bool = False
@@ -510,6 +594,7 @@ class ArgoExecutor(GenericPipelineExecutor):
             isinstance(node, TaskNode)
             or isinstance(node, StubNode)
             or isinstance(node, SuccessNode)
+            or isinstance(node, TorchNode)
         )
 
         node_override = None
@@ -522,7 +607,7 @@ class ArgoExecutor(GenericPipelineExecutor):
 
         effective_settings = self.defaults.model_dump()
         if node_override:
-            effective_settings.update(node_override.model_dump())
+            effective_settings.update(node_override.model_dump(exclude_none=True))
 
         inputs = inputs or Inputs(parameters=[])
 
@@ -573,7 +658,7 @@ class ArgoExecutor(GenericPipelineExecutor):
     def _set_env_vars_to_task(
         self, working_on: BaseNode, container_template: CoreContainerTemplate
     ):
-        if not isinstance(working_on, TaskNode):
+        if not isinstance(working_on, TaskNode) or isinstance(working_on, TorchNode):
             return
 
         global_envs: dict[str, str] = {}
@@ -792,6 +877,26 @@ class ArgoExecutor(GenericPipelineExecutor):
 
                     self._templates.append(composite_template)
 
+                case "torch":
+                    assert isinstance(working_on, TorchNode)
+                    # TODO: Need to add multi-node functionality
+                    # Check notes on the torch node
+
+                    template_of_container = self._create_container_template(
+                        working_on,
+                        task_name=task_name,
+                        inputs=Inputs(parameters=parameters),
+                    )
+                    assert template_of_container.container is not None
+
+                    if working_on.node_type == "task":
+                        self._expose_secrets_to_task(
+                            working_on,
+                            container_template=template_of_container.container,
+                        )
+
+                    self._templates.append(template_of_container)
+
             self._handle_failures(
                 working_on,
                 dag,

extensions/pipeline_executor/local.py

@@ -18,8 +18,11 @@ class LocalExecutor(GenericPipelineExecutor):
     Also ensure that the local compute is good enough for the compute to happen of all the steps.
 
     Example config:
-    execution:
+
+    ```yaml
+    pipeline-executor:
       type: local
+    ```
 
     """
 

extensions/pipeline_executor/local_container.py

@@ -3,7 +3,6 @@ from pathlib import Path
 from typing import Dict
 
 from pydantic import Field
-from rich import print
 
 from extensions.pipeline_executor import GenericPipelineExecutor
 from runnable import console, defaults, task_console, utils
@@ -20,31 +19,50 @@ class LocalContainerExecutor(GenericPipelineExecutor):
 
     Ensure that the local compute has enough resources to finish all your jobs.
 
-    The image of the run, could either be provided as default in the configuration of the execution engine
-    i.e.:
-    execution:
-      type: 'local-container'
-      config:
-        docker_image: the image you want the code to run in.
-
-    or default image could be over-ridden for a single node by providing a docker_image in the step config.
-    i.e:
-    dag:
-      steps:
-        step:
-          executor_config:
-            local-container:
-                docker_image: The image that you want that single step to run in.
-    This image would only be used for that step only.
-
-    This mode does not build the docker image with the latest code for you, it is still left for the user to build
-    and ensure that the docker image provided is the correct one.
+    Configuration options:
 
-    Example config:
-    execution:
+    ```yaml
+    pipeline-executor:
       type: local-container
       config:
-        docker_image: The default docker image to use if the node does not provide one.
+        docker_image: <required>
+        auto_remove_container: true/false
+        environment:
+          key: value
+        overrides:
+          alternate_config:
+            docker_image: <required>
+            auto_remove_container: true/false
+            environment:
+              key: value
+    ```
+
+    - ```docker_image```: The default docker image to use for all the steps.
+    - ```auto_remove_container```: Remove container after execution
+    - ```environment```: Environment variables to pass to the container
+
+    Overrides give you the ability to override the default docker image for a single step.
+    A step can then then refer to the alternate_config in the task definition.
+
+    Example:
+
+    ```python
+    from runnable import PythonTask
+
+    task = PythonTask(
+        name="alt_task",
+        overrides={
+            "local-container": "alternate_config"
+            }
+        )
+    ```
+
+    In the above example, ```alt_task``` will run in the docker image/configuration
+    as defined in the alternate_config.
+
+    ```runnable``` does not build the docker image for you, it is still left for the user to build
+    and ensure that the docker image provided is the correct one.
+
     """
 
     service_name: str = "local-container"
@@ -221,7 +239,6 @@ class LocalContainerExecutor(GenericPipelineExecutor):
 
         try:
             logger.info(f"Running the command {command}")
-            print(command)
             #  Overrides global config with local
             executor_config = self._resolve_executor_config(node)
 

runnable/executor.py

@@ -156,7 +156,7 @@ class BaseJobExecutor(BaseExecutor):
 # TODO: Consolidate execute_node, trigger_node_execution, _execute_node
 class BasePipelineExecutor(BaseExecutor):
     service_type: str = "pipeline_executor"
-    overrides: dict = {}
+    overrides: dict[str, Any] = {}
 
     _context_node: Optional[BaseNode] = PrivateAttr(default=None)
 

runnable/sdk.py

@@ -325,7 +325,7 @@ class NotebookTask(BaseTask):
         catalog Optional[Catalog]: The files sync data from/to, refer to Catalog.
 
         secrets List[str]: List of secrets to pass to the task. They are exposed as environment variables
-        and removed after execution.
+            and removed after execution.
 
         overrides (Dict[str, Any]): Any overrides to the command.
             Individual tasks can override the global configuration config by referring to the
@@ -391,7 +391,7 @@ class ShellTask(BaseTask):
         catalog Optional[Catalog]: The files sync data from/to, refer to Catalog.
 
         secrets List[str]: List of secrets to pass to the task. They are exposed as environment variables
-        and removed after execution.
+            and removed after execution.
 
         overrides (Dict[str, Any]): Any overrides to the command.
             Individual tasks can override the global configuration config by referring to the
@@ -460,8 +460,6 @@ class Stub(BaseTraversal):
 
 
 class Torch(BaseTraversal, TorchConfig):
-    # Its a wrapper of a python task
-    # TODO: Is there a way to not sync these with the torch node in extensions?
     function: Callable = Field(exclude=True)
     catalog: Optional[Catalog] = Field(default=None, alias="catalog")
     overrides: Dict[str, Any] = Field(default_factory=dict, alias="overrides")

{runnable-0.29.0.dist-info → runnable-0.30.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.29.0
+Version: 0.30.1
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.29.0.dist-info → runnable-0.30.1.dist-info}/RECORD

@@ -10,19 +10,19 @@ extensions/job_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 extensions/job_executor/__init__.py,sha256=E2R6GV5cZTlZdqA5SVJ6ajZFh4oruM0k8AKHkpOZ3W8,5772
 extensions/job_executor/k8s.py,sha256=erzw4UOsOf2JSOiQio5stgW_rMryAsIQSBd8wiL6nBY,16214
 extensions/job_executor/k8s_job_spec.yaml,sha256=7aFpxHdO_p6Hkc3YxusUOuAQTD1Myu0yTPX9DrhxbOg,1158
-extensions/job_executor/local.py,sha256=raobGxwoqZN8c-yCsAa0CDuPLWKuyEttB37U5wsqGF4,1968
+extensions/job_executor/local.py,sha256=3v6F8SOaPbCfPVVmU07RFr1wgs8iC8WoSn6Evfi8o3M,2033
 extensions/job_executor/local_container.py,sha256=8-dLhzY34pOVjJ_x0VmeTwVvYkESXBnp4j-XLsSsgBk,6688
 extensions/job_executor/pyproject.toml,sha256=UIEgiCYHTXcRWSByNMFuKJFKgxTBpQqTqyUecIsb_Vc,286
 extensions/nodes/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/nodes/nodes.py,sha256=s9ub1dqy4qHjRQG6YElCdL7rCOTYNs9RUIrStZ6tEB4,28256
 extensions/nodes/pyproject.toml,sha256=YTu-ETN3JNFSkMzzWeOwn4m-O2nbRH-PmiPBALDCUw4,278
-extensions/nodes/torch.py,sha256=kB4a72YMcrxDDzbR5LffODtrdA7vUo9dRJlaVr8KEEM,5570
+extensions/nodes/torch.py,sha256=oYh4ep9J6CS3r04HURJba5m4v8lzNupWUh4PAXvGgi0,5952
 extensions/nodes/torch_config.py,sha256=yDvDADpnLhQsNtfH8qIztLHQ2LhYiOJEWljxpH9GZzs,1222
 extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/pipeline_executor/__init__.py,sha256=9ZMHcieSYdTiYyjSkc8eT8yhOlKEUFnrbrdbqdOgvP0,24195
-extensions/pipeline_executor/argo.py,sha256=LlXtzcbJyOossTNd-gdC4xrkPe9qYxRc0czdNzNQzlY,34497
-extensions/pipeline_executor/local.py,sha256=orBIhG8QJesA3YqWhsuczjhhlKD_1s62MRNerxv9_Tg,1858
-extensions/pipeline_executor/local_container.py,sha256=PvMXy-zFTnT9hj7jjz1VkaPVJ9Dkrhnd7Hs6eqLMXZ8,12398
+extensions/pipeline_executor/argo.py,sha256=eyIVZbpecU1cPAwdvt56UFRZW2AqxALcBM_Yfvbvhqw,37958
+extensions/pipeline_executor/local.py,sha256=6oWUJ6b6NvIkpeQJBoCT1hbfX4_6WCB4HzMgHZ4ik1A,1887
+extensions/pipeline_executor/local_container.py,sha256=3kZ2QCsrq_YjH9dcAz8v05knKShQ_JtbIU-IA_-G538,12724
 extensions/pipeline_executor/mocked.py,sha256=0sMmypuvstBIv9uQg-WAcPrF3oOFpeEXNi6N8Nzdnl0,5680
 extensions/pipeline_executor/pyproject.toml,sha256=ykTX7srR10PBYb8LsIwEj8vIPPIEZQ5V_R7VYbZ-ido,291
 extensions/pipeline_executor/retry.py,sha256=6ClFXJYtr0M6nWIZiI-mbUGshobOtVH_KADN8JCfvH0,6881
@@ -40,7 +40,6 @@ extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 extensions/secrets/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/secrets/dotenv.py,sha256=nADHXI6KJ_LUYOIe5EbtYH-21OBebSNVr0Pjb1GlZ7w,1573
 extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
-extensions/tasks/torch.py,sha256=uNO4qYMawNH5hPecANCiSUQZnUC8yqw3-rxtM526CeA,1955
 runnable/__init__.py,sha256=swvqdCjeddn40o4zjsluyahdVcU0r1arSRrxmRsvFEQ,673
 runnable/catalog.py,sha256=W_erYbLZ-ffuA9RQuWVqz1DUJOuWayf32ne32IDbAbc,4358
 runnable/cli.py,sha256=3BiKSj95h2Drn__YlchMPZ5rBMafuRb2OGIsVpbsO5Y,8788
@@ -49,18 +48,18 @@ runnable/datastore.py,sha256=ZobM1aVkgeUJ2fZYt63IFDsoNzObwc93hdByegS5YKQ,32396
 runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
 runnable/entrypoints.py,sha256=cDbhtmLUWdBh9K6hNusfQpSd5NadcX8V1K2JEDf_YAg,18984
 runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
-runnable/executor.py,sha256=F0gQjJ10VQSlilqaxYm1gRdXpjmx5DqBP0KCPmz85zg,15021
+runnable/executor.py,sha256=J8-Ri9nBZCb-ao6okePb9FUVlhAaPc0ojQ2l48-FUqc,15031
 runnable/graph.py,sha256=poQz5zcvq89ju_u5sYlunQLPbHnXTaUmjcvstPwvT4U,16536
 runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
 runnable/nodes.py,sha256=d1eLttMAcV7CTwTEqOuNwZqItANoLUkXJ73Xp-srlyI,17811
 runnable/parameters.py,sha256=sT3DNGczivP9z7r4Cp_brbudg1z4J-zjmvrq3ppIrVs,5089
 runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
-runnable/sdk.py,sha256=6OO_vsRuGSjVhME2AJEljs0cjobjIQKa2E3mGoILkZA,35237
+runnable/sdk.py,sha256=NZVQGaL4Zm2hwloRmqEgp8UPbBg9hY1abQGYnOgniPI,35128
 runnable/secrets.py,sha256=4L_dBFxTgr8r_hHUD6RlZEtqaOHDRsFG5PXO5wlvMI0,2324
 runnable/tasks.py,sha256=Qb1IhVxHv68E7vf3M3YCf7MGRHyjmsEEYBpEpiZ4mRI,29062
 runnable/utils.py,sha256=hBr7oGwGL2VgfITlQCTz-a1iwvvf7Mfl-HY8UdENZac,19929
-runnable-0.29.0.dist-info/METADATA,sha256=1jW9CmQUxqDClGKzNPZGMlq1B7M4y8p9FwQf0Cz1bZg,10115
-runnable-0.29.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-runnable-0.29.0.dist-info/entry_points.txt,sha256=PrjKrlfXPZaV_7hz8orGu4FDnatLqnhPOXljyllszdw,1880
-runnable-0.29.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.29.0.dist-info/RECORD,,
+runnable-0.30.1.dist-info/METADATA,sha256=4Y4D0jyK46LpYoZE53b761BJe95eBvxo5QU3R-_-t0Y,10115
+runnable-0.30.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.30.1.dist-info/entry_points.txt,sha256=PrjKrlfXPZaV_7hz8orGu4FDnatLqnhPOXljyllszdw,1880
+runnable-0.30.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.30.1.dist-info/RECORD,,

extensions/tasks/torch.py

@@ -1,52 +0,0 @@
-from typing import List, Optional
-
-from pydantic import Field, field_validator
-
-from runnable import defaults
-from runnable.datastore import StepAttempt
-from runnable.defaults import TypeMapVariable
-from runnable.tasks import BaseTaskType
-
-
-def run_torch_task(
-    rank: int = 1,
-    world_size: int = 1,
-    entrypoint: str = "some function",
-    catalog: Optional[dict[str, List[str]]] = None,
-    task_returns: Optional[List[str]] = None,
-    secrets: Optional[list[str]] = None,
-):
-    # Entry point that creates a python job using simpler python types
-    # and and executes them. The run_id for the job is set to be run_id_rank
-    # Since the configuration file is passes as environmental variable,
-    # The job will use the configuration file to get the required information.
-
-    # In pseudocode, the following is done:
-    # Create the catalog object
-    # Create the secrets and other objects required for the PythonJob
-    # Init the process group using:
-    # https://github.com/pytorch/examples/blob/main/imagenet/main.py#L140
-    # Execute the job, the job is expected to use the environmental variables
-    # to identify the rank or can have them as variable in the signature.
-    # Once the job is executed, we destroy the process group
-    pass
-
-
-class TorchTaskType(BaseTaskType):
-    task_type: str = Field(default="torch", serialization_alias="command_type")
-    command: str
-    num_gpus: int = Field(default=1, description="Number of GPUs to use")
-
-    @field_validator("num_gpus")
-    @classmethod
-    def check_if_cuda_is_available(cls, num_gpus: int) -> int:
-        # Import torch and check if cuda is available
-        # validate if the number of gpus is less than or equal to available gpus
-        return num_gpus
-
-    def execute_command(
-        self,
-        map_variable: TypeMapVariable = None,
-    ) -> StepAttempt:
-        # We have to spawn here
-        return StepAttempt(attempt_number=1, status=defaults.SUCCESS)