runnable 0.30.2__tar.gz → 0.30.3__tar.gz

{runnable-0.30.2 → runnable-0.30.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.30.2
+Version: 0.30.3
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.30.2 → runnable-0.30.3}/extensions/nodes/torch.py

@@ -1,6 +1,8 @@
 import importlib
 import logging
 import os
+import random
+import string
 from datetime import datetime
 from typing import Any, Callable
 
@@ -28,7 +30,13 @@ def training_subprocess():
     command = os.environ.get("RUNNABLE_TORCH_COMMAND")
     run_id = os.environ.get("RUNNABLE_TORCH_RUN_ID", "")
     parameters_files = os.environ.get("RUNNABLE_TORCH_PARAMETERS_FILES", "")
-    process_run_id = run_id + "-" + os.environ.get("RANK", "")
+    process_run_id = (
+        run_id
+        + "-"
+        + os.environ.get("RANK", "")
+        + "-"
+        + "".join(random.choices(string.ascii_lowercase, k=3))
+    )
 
     delete_env_vars_with_prefix("RUNNABLE_")
 
@@ -40,6 +48,13 @@ def training_subprocess():
         job_id=process_run_id,
     )
 
+    from runnable.context import run_context
+
+    job_log = run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id)
+
+    if job_log.status == defaults.FAIL:
+        raise Exception(f"Job {process_run_id} failed")
+
 
 def get_callable_from_dotted_path(dotted_path) -> Callable:
     try:

{runnable-0.30.2 → runnable-0.30.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runnable"
-version = "0.30.2"
+version = "0.30.3"
 description = "Add your description here"
 readme = "README.md"
 authors = [