runnable 0.30.2__py3-none-any.whl → 0.30.4__py3-none-any.whl

extensions/catalog/any_path.py

@@ -94,7 +94,9 @@ class AnyPathCatalog(BaseCatalog):
 
         return data_catalogs
 
-    def put(self, name: str) -> List[DataCatalog]:
+    def put(
+        self, name: str, allow_file_not_found_exc: bool = False
+    ) -> List[DataCatalog]:
         """
         Put the files matching the glob pattern into the catalog.
 
@@ -154,7 +156,7 @@ class AnyPathCatalog(BaseCatalog):
             # TODO: Think about syncing only if the file is changed
             self.upload_to_catalog(file)
 
-        if not data_catalogs:
+        if not data_catalogs and not allow_file_not_found_exc:
             raise Exception(f"Did not find any files matching {name} in {copy_from}")
 
         return data_catalogs

extensions/job_executor/__init__.py

@@ -139,6 +139,7 @@ class GenericJobExecutor(BaseJobExecutor):
     def _sync_catalog(
         self,
         catalog_settings=Optional[List[str]],
+        allow_file_not_found_exc: bool = False,
     ) -> List[DataCatalog] | None:
         if not catalog_settings:
             logger.info("No catalog settings found")
@@ -147,7 +148,7 @@ class GenericJobExecutor(BaseJobExecutor):
         data_catalogs = []
         for name_pattern in catalog_settings:
             data_catalog = self._context.catalog_handler.put(
-                name=name_pattern,
+                name=name_pattern, allow_file_not_found_exc=allow_file_not_found_exc
             )
 
             logger.debug(f"Added data catalog: {data_catalog} to job log")

extensions/job_executor/k8s.py

@@ -7,7 +7,6 @@ from kubernetes import client
 from kubernetes import config as k8s_config
 from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PrivateAttr
 from pydantic.alias_generators import to_camel
-from rich import print
 
 from extensions.job_executor import GenericJobExecutor
 from runnable import console, defaults, utils
@@ -223,8 +222,13 @@ class GenericK8sJobExecutor(GenericJobExecutor):
         job_log.status = attempt_log.status
         job_log.attempts.append(attempt_log)
 
+        allow_file_not_found_exc = True
+        if job_log.status == defaults.SUCCESS:
+            allow_file_not_found_exc = False
+
         data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
-            catalog_settings=catalog_settings
+            catalog_settings=catalog_settings,
+            allow_file_not_found_exc=allow_file_not_found_exc,
         )
         logger.debug(f"data_catalogs_put: {data_catalogs_put}")
 

extensions/job_executor/local.py

@@ -55,11 +55,16 @@ class LocalJobExecutor(GenericJobExecutor):
         job_log.status = attempt_log.status
         job_log.attempts.append(attempt_log)
 
+        allow_file_not_found_exc = True
+        if job_log.status == defaults.SUCCESS:
+            allow_file_not_found_exc = False
+
         data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
-            catalog_settings=catalog_settings
+            catalog_settings=catalog_settings,
+            allow_file_not_found_exc=allow_file_not_found_exc,
         )
-        logger.debug(f"data_catalogs_put: {data_catalogs_put}")
 
+        logger.debug(f"data_catalogs_put: {data_catalogs_put}")
         job_log.add_data_catalogs(data_catalogs_put or [])
 
         console.print("Summary of job")

extensions/job_executor/local_container.py

@@ -64,9 +64,15 @@ class LocalContainerJobExecutor(GenericJobExecutor):
         job_log.status = attempt_log.status
         job_log.attempts.append(attempt_log)
 
+        allow_file_not_found_exc = True
+        if job_log.status == defaults.SUCCESS:
+            allow_file_not_found_exc = False
+
         data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
-            catalog_settings=catalog_settings
+            catalog_settings=catalog_settings,
+            allow_file_not_found_exc=allow_file_not_found_exc,
         )
+
         logger.debug(f"data_catalogs_put: {data_catalogs_put}")
 
         job_log.add_data_catalogs(data_catalogs_put or [])

extensions/nodes/torch.py

@@ -1,6 +1,8 @@
 import importlib
 import logging
 import os
+import random
+import string
 from datetime import datetime
 from typing import Any, Callable
 
@@ -28,7 +30,13 @@ def training_subprocess():
     command = os.environ.get("RUNNABLE_TORCH_COMMAND")
     run_id = os.environ.get("RUNNABLE_TORCH_RUN_ID", "")
     parameters_files = os.environ.get("RUNNABLE_TORCH_PARAMETERS_FILES", "")
-    process_run_id = run_id + "-" + os.environ.get("RANK", "")
+    process_run_id = (
+        run_id
+        + "-"
+        + os.environ.get("RANK", "")
+        + "-"
+        + "".join(random.choices(string.ascii_lowercase, k=3))
+    )
 
     delete_env_vars_with_prefix("RUNNABLE_")
 
@@ -40,6 +48,13 @@ def training_subprocess():
         job_id=process_run_id,
     )
 
+    from runnable.context import run_context
+
+    job_log = run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id)
+
+    if job_log.status == defaults.FAIL:
+        raise Exception(f"Job {process_run_id} failed")
+
 
 def get_callable_from_dotted_path(dotted_path) -> Callable:
     try:

extensions/pipeline_executor/__init__.py

@@ -111,7 +111,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
         )
 
     def _sync_catalog(
-        self, stage: str, synced_catalogs=None
+        self, stage: str, synced_catalogs=None, allow_file_no_found_exc: bool = False
     ) -> Optional[List[DataCatalog]]:
         """
         1). Identify the catalog settings by over-riding node settings with the global settings.
@@ -160,7 +160,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
 
             elif stage == "put":
                 data_catalog = self._context.catalog_handler.put(
-                    name=name_pattern,
+                    name=name_pattern, allow_file_not_found_exc=allow_file_no_found_exc
                 )
             else:
                 raise Exception(f"Stage {stage} not supported")
@@ -233,12 +233,16 @@ class GenericPipelineExecutor(BasePipelineExecutor):
             mock=mock,
         )
 
+        allow_file_not_found_exc = True
         if step_log.status == defaults.SUCCESS:
-            data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
-                stage="put"
-            )
-            logger.debug(f"data_catalogs_put: {data_catalogs_put}")
-            step_log.add_data_catalogs(data_catalogs_put or [])
+            # raise exception if we succeeded but the file was not found
+            allow_file_not_found_exc = False
+
+        data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
+            stage="put", allow_file_no_found_exc=allow_file_not_found_exc
+        )
+        logger.debug(f"data_catalogs_put: {data_catalogs_put}")
+        step_log.add_data_catalogs(data_catalogs_put or [])
 
         # get catalog should always be added to the step log
         step_log.add_data_catalogs(data_catalogs_get or [])

extensions/pipeline_executor/argo.py

@@ -658,7 +658,7 @@ class ArgoExecutor(GenericPipelineExecutor):
     def _set_env_vars_to_task(
         self, working_on: BaseNode, container_template: CoreContainerTemplate
     ):
-        if not isinstance(working_on, TaskNode) or isinstance(working_on, TorchNode):
+        if not (isinstance(working_on, TaskNode) or isinstance(working_on, TorchNode)):
             return
 
         global_envs: dict[str, str] = {}

runnable/catalog.py

@@ -76,7 +76,9 @@ class BaseCatalog(ABC, BaseModel):
         raise NotImplementedError
 
     @abstractmethod
-    def put(self, name: str) -> List[DataCatalog]:
+    def put(
+        self, name: str, allow_file_not_found_exc: bool = False
+    ) -> List[DataCatalog]:
         """
         Put the file by 'name' from the 'compute_data_folder' in the catalog for the run_id.
 
@@ -137,7 +139,9 @@ class DoNothingCatalog(BaseCatalog):
         logger.info("Using a do-nothing catalog, doing nothing in get")
         return []
 
-    def put(self, name: str) -> List[DataCatalog]:
+    def put(
+        self, name: str, allow_file_not_found_exc: bool = False
+    ) -> List[DataCatalog]:
         """
         Does nothing
         """

runnable/executor.py

@@ -123,6 +123,7 @@ class BaseJobExecutor(BaseExecutor):
     def _sync_catalog(
         self,
         catalog_settings: Optional[List[str]],
+        allow_file_not_found_exc: bool = False,
     ) -> Optional[List[DataCatalog]]:
         """
         1). Identify the catalog settings by over-riding node settings with the global settings.
@@ -175,7 +176,10 @@ class BasePipelineExecutor(BaseExecutor):
 
     @abstractmethod
     def _sync_catalog(
-        self, stage: str, synced_catalogs=None
+        self,
+        stage: str,
+        synced_catalogs=None,
+        allow_file_no_found_exc: bool = False,
     ) -> Optional[List[DataCatalog]]:
         """
         1). Identify the catalog settings by over-riding node settings with the global settings.

{runnable-0.30.2.dist-info → runnable-0.30.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.30.2
+Version: 0.30.4
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.30.2.dist-info → runnable-0.30.4.dist-info}/RECORD

@@ -1,26 +1,26 @@
 extensions/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/catalog/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-extensions/catalog/any_path.py,sha256=aNjphoPIyllUfY2uNDFWD1ErM3Px6izSGr0-oGowN8k,7263
+extensions/catalog/any_path.py,sha256=atB5gWPRX6ptW6zwYeCVb_fh0qhs7WAFO9HIsnMZl98,7350
 extensions/catalog/file_system.py,sha256=T_qFPFfrmykoAMc1rjNi_DBb437me8WPRcFglwAK744,1767
 extensions/catalog/minio.py,sha256=R3GvfCxN1GTcs4bQIAWh79_GHDTVd14gnpKlzwFeKUI,2363
 extensions/catalog/pyproject.toml,sha256=lLNxY6v04c8I5QK_zKw_E6sJTArSJRA_V-79ktaA3Hk,279
 extensions/catalog/s3.py,sha256=Sw5t8_kVRprn3uGGJCiHn7M9zw1CLaCOFj6YErtfG0o,287
 extensions/job_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-extensions/job_executor/__init__.py,sha256=E2R6GV5cZTlZdqA5SVJ6ajZFh4oruM0k8AKHkpOZ3W8,5772
-extensions/job_executor/k8s.py,sha256=erzw4UOsOf2JSOiQio5stgW_rMryAsIQSBd8wiL6nBY,16214
+extensions/job_executor/__init__.py,sha256=VeLuYCcShCIYT0TNtAXfUF9tOk4ZHoLzdTEvbsz0spM,5870
+extensions/job_executor/k8s.py,sha256=0V7BL7ERmonVMgCsO-J57cxH__v8KomwukMwepH3qgs,16388
 extensions/job_executor/k8s_job_spec.yaml,sha256=7aFpxHdO_p6Hkc3YxusUOuAQTD1Myu0yTPX9DrhxbOg,1158
-extensions/job_executor/local.py,sha256=3v6F8SOaPbCfPVVmU07RFr1wgs8iC8WoSn6Evfi8o3M,2033
-extensions/job_executor/local_container.py,sha256=8-dLhzY34pOVjJ_x0VmeTwVvYkESXBnp4j-XLsSsgBk,6688
+extensions/job_executor/local.py,sha256=3ZbCFXBvbLlMp10JTmQJJrjBKG2keHI6SH8hEvmHDkA,2230
+extensions/job_executor/local_container.py,sha256=1JcLJ0zrNSNHdubrSO9miN54iwvPLHqKMZ08aOC8WWo,6886
 extensions/job_executor/pyproject.toml,sha256=UIEgiCYHTXcRWSByNMFuKJFKgxTBpQqTqyUecIsb_Vc,286
 extensions/nodes/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/nodes/nodes.py,sha256=s9ub1dqy4qHjRQG6YElCdL7rCOTYNs9RUIrStZ6tEB4,28256
 extensions/nodes/pyproject.toml,sha256=YTu-ETN3JNFSkMzzWeOwn4m-O2nbRH-PmiPBALDCUw4,278
-extensions/nodes/torch.py,sha256=oYh4ep9J6CS3r04HURJba5m4v8lzNupWUh4PAXvGgi0,5952
+extensions/nodes/torch.py,sha256=id0_HVkRcqL9_DPOI-b53vaDwRgVfGB-zZS3yrRej9g,6318
 extensions/nodes/torch_config.py,sha256=yDvDADpnLhQsNtfH8qIztLHQ2LhYiOJEWljxpH9GZzs,1222
 extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-extensions/pipeline_executor/__init__.py,sha256=LApJGwJctEBTDQX9gGHkGV97T6rF_q-_59OzvAJ2X1g,24346
-extensions/pipeline_executor/argo.py,sha256=-mWH5H2lZEldO8er7-tHberm7swqo4CwrtQKUd2VkGA,37944
+extensions/pipeline_executor/__init__.py,sha256=wfigTL2T9OHrmE8b2Ydmb8h6hr-oF--Yc2FectC7WaY,24623
+extensions/pipeline_executor/argo.py,sha256=AEGSWVZulBL6EsvbVCaeBeTl2m_t5ymc6RFpMKhivis,37946
 extensions/pipeline_executor/local.py,sha256=6oWUJ6b6NvIkpeQJBoCT1hbfX4_6WCB4HzMgHZ4ik1A,1887
 extensions/pipeline_executor/local_container.py,sha256=3kZ2QCsrq_YjH9dcAz8v05knKShQ_JtbIU-IA_-G538,12724
 extensions/pipeline_executor/mocked.py,sha256=0sMmypuvstBIv9uQg-WAcPrF3oOFpeEXNi6N8Nzdnl0,5680
@@ -41,14 +41,14 @@ extensions/secrets/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 extensions/secrets/dotenv.py,sha256=nADHXI6KJ_LUYOIe5EbtYH-21OBebSNVr0Pjb1GlZ7w,1573
 extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
 runnable/__init__.py,sha256=swvqdCjeddn40o4zjsluyahdVcU0r1arSRrxmRsvFEQ,673
-runnable/catalog.py,sha256=W_erYbLZ-ffuA9RQuWVqz1DUJOuWayf32ne32IDbAbc,4358
+runnable/catalog.py,sha256=4msQxLhLKlsDDrHFnGauPYe-Or-q9g8_RYCn_4dpxaU,4466
 runnable/cli.py,sha256=3BiKSj95h2Drn__YlchMPZ5rBMafuRb2OGIsVpbsO5Y,8788
 runnable/context.py,sha256=by5uepmuCP0dmM9BmsliXihSes5QEFejwAsmekcqylE,1388
 runnable/datastore.py,sha256=ZobM1aVkgeUJ2fZYt63IFDsoNzObwc93hdByegS5YKQ,32396
 runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
 runnable/entrypoints.py,sha256=cDbhtmLUWdBh9K6hNusfQpSd5NadcX8V1K2JEDf_YAg,18984
 runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
-runnable/executor.py,sha256=J8-Ri9nBZCb-ao6okePb9FUVlhAaPc0ojQ2l48-FUqc,15031
+runnable/executor.py,sha256=UOsYJ3NkTGw4FTR0iePX7AOJzY7vODhZ62aqrwVMO1c,15143
 runnable/graph.py,sha256=poQz5zcvq89ju_u5sYlunQLPbHnXTaUmjcvstPwvT4U,16536
 runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
 runnable/nodes.py,sha256=d1eLttMAcV7CTwTEqOuNwZqItANoLUkXJ73Xp-srlyI,17811
@@ -58,8 +58,8 @@ runnable/sdk.py,sha256=NZVQGaL4Zm2hwloRmqEgp8UPbBg9hY1abQGYnOgniPI,35128
 runnable/secrets.py,sha256=4L_dBFxTgr8r_hHUD6RlZEtqaOHDRsFG5PXO5wlvMI0,2324
 runnable/tasks.py,sha256=Qb1IhVxHv68E7vf3M3YCf7MGRHyjmsEEYBpEpiZ4mRI,29062
 runnable/utils.py,sha256=hBr7oGwGL2VgfITlQCTz-a1iwvvf7Mfl-HY8UdENZac,19929
-runnable-0.30.2.dist-info/METADATA,sha256=M-3XWv_ijqZLfEwzQ6W5IsZaHGOpNxToI8pYNI-SlwQ,10115
-runnable-0.30.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-runnable-0.30.2.dist-info/entry_points.txt,sha256=PrjKrlfXPZaV_7hz8orGu4FDnatLqnhPOXljyllszdw,1880
-runnable-0.30.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.30.2.dist-info/RECORD,,
+runnable-0.30.4.dist-info/METADATA,sha256=S-5zecrqE4tU5MW4Fe1-2F-Q_hLU7fAXZ2oo9xVRRUw,10115
+runnable-0.30.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.30.4.dist-info/entry_points.txt,sha256=PrjKrlfXPZaV_7hz8orGu4FDnatLqnhPOXljyllszdw,1880
+runnable-0.30.4.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.30.4.dist-info/RECORD,,