runnable 0.30.2__tar.gz → 0.30.4__tar.gz

{runnable-0.30.2 → runnable-0.30.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.30.2
+Version: 0.30.4
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.30.2 → runnable-0.30.4}/extensions/catalog/any_path.py

@@ -94,7 +94,9 @@ class AnyPathCatalog(BaseCatalog):
 
         return data_catalogs
 
-    def put(self, name: str) -> List[DataCatalog]:
+    def put(
+        self, name: str, allow_file_not_found_exc: bool = False
+    ) -> List[DataCatalog]:
         """
         Put the files matching the glob pattern into the catalog.
 
@@ -154,7 +156,7 @@ class AnyPathCatalog(BaseCatalog):
             # TODO: Think about syncing only if the file is changed
             self.upload_to_catalog(file)
 
-        if not data_catalogs:
+        if not data_catalogs and not allow_file_not_found_exc:
             raise Exception(f"Did not find any files matching {name} in {copy_from}")
 
         return data_catalogs

{runnable-0.30.2 → runnable-0.30.4}/extensions/job_executor/__init__.py

@@ -139,6 +139,7 @@ class GenericJobExecutor(BaseJobExecutor):
     def _sync_catalog(
         self,
         catalog_settings=Optional[List[str]],
+        allow_file_not_found_exc: bool = False,
     ) -> List[DataCatalog] | None:
         if not catalog_settings:
             logger.info("No catalog settings found")
@@ -147,7 +148,7 @@ class GenericJobExecutor(BaseJobExecutor):
         data_catalogs = []
         for name_pattern in catalog_settings:
             data_catalog = self._context.catalog_handler.put(
-                name=name_pattern,
+                name=name_pattern, allow_file_not_found_exc=allow_file_not_found_exc
             )
 
             logger.debug(f"Added data catalog: {data_catalog} to job log")

{runnable-0.30.2 → runnable-0.30.4}/extensions/job_executor/k8s.py

@@ -7,7 +7,6 @@ from kubernetes import client
 from kubernetes import config as k8s_config
 from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PrivateAttr
 from pydantic.alias_generators import to_camel
-from rich import print
 
 from extensions.job_executor import GenericJobExecutor
 from runnable import console, defaults, utils
@@ -223,8 +222,13 @@ class GenericK8sJobExecutor(GenericJobExecutor):
         job_log.status = attempt_log.status
         job_log.attempts.append(attempt_log)
 
+        allow_file_not_found_exc = True
+        if job_log.status == defaults.SUCCESS:
+            allow_file_not_found_exc = False
+
         data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
-            catalog_settings=catalog_settings
+            catalog_settings=catalog_settings,
+            allow_file_not_found_exc=allow_file_not_found_exc,
         )
         logger.debug(f"data_catalogs_put: {data_catalogs_put}")
 

{runnable-0.30.2 → runnable-0.30.4}/extensions/job_executor/local.py

@@ -55,11 +55,16 @@ class LocalJobExecutor(GenericJobExecutor):
         job_log.status = attempt_log.status
         job_log.attempts.append(attempt_log)
 
+        allow_file_not_found_exc = True
+        if job_log.status == defaults.SUCCESS:
+            allow_file_not_found_exc = False
+
         data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
-            catalog_settings=catalog_settings
+            catalog_settings=catalog_settings,
+            allow_file_not_found_exc=allow_file_not_found_exc,
         )
-        logger.debug(f"data_catalogs_put: {data_catalogs_put}")
 
+        logger.debug(f"data_catalogs_put: {data_catalogs_put}")
         job_log.add_data_catalogs(data_catalogs_put or [])
 
         console.print("Summary of job")

{runnable-0.30.2 → runnable-0.30.4}/extensions/job_executor/local_container.py

@@ -64,9 +64,15 @@ class LocalContainerJobExecutor(GenericJobExecutor):
         job_log.status = attempt_log.status
         job_log.attempts.append(attempt_log)
 
+        allow_file_not_found_exc = True
+        if job_log.status == defaults.SUCCESS:
+            allow_file_not_found_exc = False
+
         data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
-            catalog_settings=catalog_settings
+            catalog_settings=catalog_settings,
+            allow_file_not_found_exc=allow_file_not_found_exc,
         )
+
         logger.debug(f"data_catalogs_put: {data_catalogs_put}")
 
         job_log.add_data_catalogs(data_catalogs_put or [])

{runnable-0.30.2 → runnable-0.30.4}/extensions/nodes/torch.py

@@ -1,6 +1,8 @@
 import importlib
 import logging
 import os
+import random
+import string
 from datetime import datetime
 from typing import Any, Callable
 
@@ -28,7 +30,13 @@ def training_subprocess():
     command = os.environ.get("RUNNABLE_TORCH_COMMAND")
     run_id = os.environ.get("RUNNABLE_TORCH_RUN_ID", "")
     parameters_files = os.environ.get("RUNNABLE_TORCH_PARAMETERS_FILES", "")
-    process_run_id = run_id + "-" + os.environ.get("RANK", "")
+    process_run_id = (
+        run_id
+        + "-"
+        + os.environ.get("RANK", "")
+        + "-"
+        + "".join(random.choices(string.ascii_lowercase, k=3))
+    )
 
     delete_env_vars_with_prefix("RUNNABLE_")
 
@@ -40,6 +48,13 @@ def training_subprocess():
         job_id=process_run_id,
     )
 
+    from runnable.context import run_context
+
+    job_log = run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id)
+
+    if job_log.status == defaults.FAIL:
+        raise Exception(f"Job {process_run_id} failed")
+
 
 def get_callable_from_dotted_path(dotted_path) -> Callable:
     try:

{runnable-0.30.2 → runnable-0.30.4}/extensions/pipeline_executor/__init__.py

@@ -111,7 +111,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
         )
 
     def _sync_catalog(
-        self, stage: str, synced_catalogs=None
+        self, stage: str, synced_catalogs=None, allow_file_no_found_exc: bool = False
     ) -> Optional[List[DataCatalog]]:
         """
         1). Identify the catalog settings by over-riding node settings with the global settings.
@@ -160,7 +160,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
 
             elif stage == "put":
                 data_catalog = self._context.catalog_handler.put(
-                    name=name_pattern,
+                    name=name_pattern, allow_file_not_found_exc=allow_file_no_found_exc
                 )
             else:
                 raise Exception(f"Stage {stage} not supported")
@@ -233,12 +233,16 @@ class GenericPipelineExecutor(BasePipelineExecutor):
             mock=mock,
         )
 
+        allow_file_not_found_exc = True
         if step_log.status == defaults.SUCCESS:
-            data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
-                stage="put"
-            )
-            logger.debug(f"data_catalogs_put: {data_catalogs_put}")
-            step_log.add_data_catalogs(data_catalogs_put or [])
+            # raise exception if we succeeded but the file was not found
+            allow_file_not_found_exc = False
+
+        data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
+            stage="put", allow_file_no_found_exc=allow_file_not_found_exc
+        )
+        logger.debug(f"data_catalogs_put: {data_catalogs_put}")
+        step_log.add_data_catalogs(data_catalogs_put or [])
 
         # get catalog should always be added to the step log
         step_log.add_data_catalogs(data_catalogs_get or [])

{runnable-0.30.2 → runnable-0.30.4}/extensions/pipeline_executor/argo.py

@@ -658,7 +658,7 @@ class ArgoExecutor(GenericPipelineExecutor):
     def _set_env_vars_to_task(
         self, working_on: BaseNode, container_template: CoreContainerTemplate
     ):
-        if not isinstance(working_on, TaskNode) or isinstance(working_on, TorchNode):
+        if not (isinstance(working_on, TaskNode) or isinstance(working_on, TorchNode)):
             return
 
         global_envs: dict[str, str] = {}

{runnable-0.30.2 → runnable-0.30.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runnable"
-version = "0.30.2"
+version = "0.30.4"
 description = "Add your description here"
 readme = "README.md"
 authors = [

{runnable-0.30.2 → runnable-0.30.4}/runnable/catalog.py

@@ -76,7 +76,9 @@ class BaseCatalog(ABC, BaseModel):
         raise NotImplementedError
 
     @abstractmethod
-    def put(self, name: str) -> List[DataCatalog]:
+    def put(
+        self, name: str, allow_file_not_found_exc: bool = False
+    ) -> List[DataCatalog]:
         """
         Put the file by 'name' from the 'compute_data_folder' in the catalog for the run_id.
 
@@ -137,7 +139,9 @@ class DoNothingCatalog(BaseCatalog):
         logger.info("Using a do-nothing catalog, doing nothing in get")
         return []
 
-    def put(self, name: str) -> List[DataCatalog]:
+    def put(
+        self, name: str, allow_file_not_found_exc: bool = False
+    ) -> List[DataCatalog]:
         """
         Does nothing
         """

{runnable-0.30.2 → runnable-0.30.4}/runnable/executor.py

@@ -123,6 +123,7 @@ class BaseJobExecutor(BaseExecutor):
     def _sync_catalog(
         self,
         catalog_settings: Optional[List[str]],
+        allow_file_not_found_exc: bool = False,
     ) -> Optional[List[DataCatalog]]:
         """
         1). Identify the catalog settings by over-riding node settings with the global settings.
@@ -175,7 +176,10 @@ class BasePipelineExecutor(BaseExecutor):
 
     @abstractmethod
     def _sync_catalog(
-        self, stage: str, synced_catalogs=None
+        self,
+        stage: str,
+        synced_catalogs=None,
+        allow_file_no_found_exc: bool = False,
     ) -> Optional[List[DataCatalog]]:
         """
         1). Identify the catalog settings by over-riding node settings with the global settings.