experimaestro 1.10.1__tar.gz → 1.12.0__tar.gz

{experimaestro-1.10.1 → experimaestro-1.12.0}/PKG-INFO

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: experimaestro
-Version: 1.10.1
+Version: 1.12.0
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
+License-File: LICENSE
 Keywords: experiment manager
 Author: Benjamin Piwowarski
 Author-email: benjamin@piwowarski.fr
@@ -20,8 +21,8 @@ Requires-Dist: click (>=8)
 Requires-Dist: decorator (>=5,<6)
 Requires-Dist: docstring-parser (>=0.15,<1)
 Requires-Dist: fasteners (>=0.19,<1)
-Requires-Dist: flask (>=2.3,<3)
-Requires-Dist: flask-socketio (>=5.3,<6)
+Requires-Dist: flask (>=2.3)
+Requires-Dist: flask-socketio (>=5.3)
 Requires-Dist: gevent (>=25)
 Requires-Dist: gevent-websocket (>=0.10)
 Requires-Dist: huggingface-hub (>0.17)

{experimaestro-1.10.1 → experimaestro-1.12.0}/pyproject.toml

@@ -27,8 +27,8 @@ dependencies = [
     "decorator >=5,<6",
     "docstring-parser >=0.15,<1",
     "fasteners >=0.19,<1",
-    "flask >=2.3,<3",
-    "flask-socketio >=5.3,<6",
+    "flask >=2.3",
+    "flask-socketio >=5.3",
     "gevent >=25",
     "gevent-websocket >=0.10",
     "humanfriendly >=10",
@@ -48,7 +48,7 @@ dependencies = [
     "typing-extensions >=4.2; python_version < \"3.12\"",
     "watchdog >=2"
 ]
-version = "1.10.1"
+version = "1.12.0"
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -140,7 +140,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.11.0"
+version = "1.12.0"
 changelog_start_rev = "v1.0.0"
 tag_format = "v$major.$minor.$patch$prerelease"
 # update_changelog_on_bump = true

{experimaestro-1.10.1 → experimaestro-1.12.0}/src/experimaestro/launcherfinder/specs.py

@@ -29,7 +29,7 @@ class CudaSpecification:
     def __repr__(self):
         return (
             f"CUDA({self.model} "
-            f"max={format_size(self.memory)}/min={format_size(self.min_memory)})"
+            f"max={format_size(self.memory, binary=True)}/min={format_size(self.min_memory, binary=True)})"
         )
 
 
@@ -47,6 +47,13 @@ class CPUSpecification:
     cpu_per_gpu: int = 0
     """Number of CPU per GPU (0 if not defined)"""
 
+    def __repr__(self):
+        return (
+            f"CPU("
+            f"mem={format_size(self.memory, binary=True)}, cores={self.cores}"
+            ")"
+        )
+
     def match(self, other: "CPUSpecification"):
         return (self.memory >= other.memory) and (self.cores >= other.cores)
 

{experimaestro-1.10.1 → experimaestro-1.12.0}/src/experimaestro/run.py

@@ -8,6 +8,7 @@ import json
 from typing import List
 import fasteners
 from experimaestro.notifications import progress, report_eoj
+from experimaestro.utils.multiprocessing import delayed_shutdown
 from .core.types import ObjectType
 from experimaestro.utils import logger
 from experimaestro.core.objects import ConfigInformation
@@ -96,6 +97,7 @@ class TaskRunner:
         self.failedpath.write_text(str(code))
         self.cleanup()
         logger.info("Exiting")
+        delayed_shutdown(60, exit_code=code)
         sys.exit(1)
 
     def run(self):

experimaestro-1.12.0/src/experimaestro/utils/multiprocessing.py

@@ -0,0 +1,44 @@
+import logging
+import multiprocessing as mp
+import time
+import os
+import signal
+import threading
+
+
+def delayed_shutdown(delay=60, *, exit_code=1, grace_period=5):
+    """After *delay*'s try a graceful stop, then SIGKILL anything left.
+
+    :param delay: Delay in seconds before killing
+    :param grace_period: Delay in seconds before force-killing a child process
+    :param exit_code: The exit code to use
+    """
+
+    def _killer():
+        time.sleep(delay)
+
+        logging.info("Stall dectected – killing all subprocesses")
+
+        # 1️⃣ Try graceful termination
+        for p in mp.active_children():
+            # sends SIGTERM / TerminateProcess
+            p.terminate()
+
+        alive = mp.active_children()
+        deadline = time.time() + grace_period
+        while alive and time.time() < deadline:
+            alive = [p for p in alive if p.is_alive()]
+            time.sleep(0.1)
+
+        # 2️⃣ Anything still alive? Nuke it.
+        for p in alive:
+            try:
+                os.kill(p.pid, signal.SIGKILL)
+            except OSError:
+                pass
+
+        # 3️⃣ Finally kill the parent
+        os.kill(os.getpid(), signal.SIGKILL)
+
+    # Start the thread (non blocking)
+    threading.Thread(target=_killer, daemon=True).start()