experimaestro 1.10.1__py3-none-any.whl → 1.12.0__py3-none-any.whl

experimaestro/launcherfinder/specs.py

@@ -29,7 +29,7 @@ class CudaSpecification:
     def __repr__(self):
         return (
             f"CUDA({self.model} "
-            f"max={format_size(self.memory)}/min={format_size(self.min_memory)})"
+            f"max={format_size(self.memory, binary=True)}/min={format_size(self.min_memory, binary=True)})"
         )
 
 
@@ -47,6 +47,13 @@ class CPUSpecification:
     cpu_per_gpu: int = 0
     """Number of CPU per GPU (0 if not defined)"""
 
+    def __repr__(self):
+        return (
+            f"CPU("
+            f"mem={format_size(self.memory, binary=True)}, cores={self.cores}"
+            ")"
+        )
+
     def match(self, other: "CPUSpecification"):
         return (self.memory >= other.memory) and (self.cores >= other.cores)
 

experimaestro/run.py

@@ -8,6 +8,7 @@ import json
 from typing import List
 import fasteners
 from experimaestro.notifications import progress, report_eoj
+from experimaestro.utils.multiprocessing import delayed_shutdown
 from .core.types import ObjectType
 from experimaestro.utils import logger
 from experimaestro.core.objects import ConfigInformation
@@ -96,6 +97,7 @@ class TaskRunner:
         self.failedpath.write_text(str(code))
         self.cleanup()
         logger.info("Exiting")
+        delayed_shutdown(60, exit_code=code)
         sys.exit(1)
 
     def run(self):

experimaestro/utils/multiprocessing.py

@@ -0,0 +1,44 @@
+import logging
+import multiprocessing as mp
+import time
+import os
+import signal
+import threading
+
+
+def delayed_shutdown(delay=60, *, exit_code=1, grace_period=5):
+    """After *delay*'s try a graceful stop, then SIGKILL anything left.
+
+    :param delay: Delay in seconds before killing
+    :param grace_period: Delay in seconds before force-killing a child process
+    :param exit_code: The exit code to use
+    """
+
+    def _killer():
+        time.sleep(delay)
+
+        logging.info("Stall dectected – killing all subprocesses")
+
+        # 1️⃣ Try graceful termination
+        for p in mp.active_children():
+            # sends SIGTERM / TerminateProcess
+            p.terminate()
+
+        alive = mp.active_children()
+        deadline = time.time() + grace_period
+        while alive and time.time() < deadline:
+            alive = [p for p in alive if p.is_alive()]
+            time.sleep(0.1)
+
+        # 2️⃣ Anything still alive? Nuke it.
+        for p in alive:
+            try:
+                os.kill(p.pid, signal.SIGKILL)
+            except OSError:
+                pass
+
+        # 3️⃣ Finally kill the parent
+        os.kill(os.getpid(), signal.SIGKILL)
+
+    # Start the thread (non blocking)
+    threading.Thread(target=_killer, daemon=True).start()

{experimaestro-1.10.1.dist-info → experimaestro-1.12.0.dist-info}/METADATA

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: experimaestro
-Version: 1.10.1
+Version: 1.12.0
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
+License-File: LICENSE
 Keywords: experiment manager
 Author: Benjamin Piwowarski
 Author-email: benjamin@piwowarski.fr
@@ -20,8 +21,8 @@ Requires-Dist: click (>=8)
 Requires-Dist: decorator (>=5,<6)
 Requires-Dist: docstring-parser (>=0.15,<1)
 Requires-Dist: fasteners (>=0.19,<1)
-Requires-Dist: flask (>=2.3,<3)
-Requires-Dist: flask-socketio (>=5.3,<6)
+Requires-Dist: flask (>=2.3)
+Requires-Dist: flask-socketio (>=5.3)
 Requires-Dist: gevent (>=25)
 Requires-Dist: gevent-websocket (>=0.10)
 Requires-Dist: huggingface-hub (>0.17)

{experimaestro-1.10.1.dist-info → experimaestro-1.12.0.dist-info}/RECORD

@@ -36,7 +36,7 @@ experimaestro/launcherfinder/__init__.py,sha256=qRUDyv3B9UsAM8Q31mRrZrTZox0Aptwd
 experimaestro/launcherfinder/base.py,sha256=q47SsF_cXdo5O6ZhFKn5385WVFcx8Wd-BcEpd6tRpbs,515
 experimaestro/launcherfinder/parser.py,sha256=MIHhjs2sTVxLHLcc1CgFid9XxhInXker8QdA-GBA-Bk,2364
 experimaestro/launcherfinder/registry.py,sha256=F16A7SE_CgzaTBC-LBlurbthTDvYUa5iyjJftkDNN4M,6420
-experimaestro/launcherfinder/specs.py,sha256=484S4-N22Y10BsyTpNwW0l369ALAJUrXQK9LsYSYsaQ,7705
+experimaestro/launcherfinder/specs.py,sha256=eQC2pwAnvq7kF2xmAfHpg_Wx6_lH6YMf3ZCDwqatjKk,7898
 experimaestro/launchers/__init__.py,sha256=lXn544sgJExr6uirILWzAXu_IfmfyqFZOt4OzRnjHXg,2525
 experimaestro/launchers/direct.py,sha256=JZh6WOPnO6ED_xlOs8pL4MRFmnRhmXzpVxTl-ByaD2A,258
 experimaestro/launchers/oar.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -52,7 +52,7 @@ experimaestro/mypy.py,sha256=M39VFuDrab-ymlCDIF5jys9oKpTwnuBPzb1T8Un5J3s,285
 experimaestro/notifications.py,sha256=TLG9sw6bxukj0Wj4ZXASn-A9sQuET_exzUHtgk8AgRQ,9335
 experimaestro/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/rpyc.py,sha256=ZRKol-3tVoeoUITLNFenLF4dhWBLW_FvSV_GvsypmeI,3605
-experimaestro/run.py,sha256=58ZlIZ2dQ7a0un2iGiyHJhK14zc18BnpEFDis7OyTPA,5222
+experimaestro/run.py,sha256=_szMzKn-67ulT6Wy21dzhB1g6_fiL379JnBC0zJqpIY,5332
 experimaestro/scheduler/__init__.py,sha256=ERmmOxz_9mUkIuccNbzUa5Y6gVLLVDdyc4cCxbCCUbY,20
 experimaestro/scheduler/base.py,sha256=294a_29UtsSfoW4NpL4_6mvDyDslG0qDXGmVkcGRUDA,35829
 experimaestro/scheduler/dependencies.py,sha256=n9XegwrmjayOIxt3xhuTEBVEBGSq4oeVdzz-FviDGXo,1994
@@ -147,11 +147,12 @@ experimaestro/utils/__init__.py,sha256=BdYguxAbR1jOQPV36OgGA31itaMvBJ6WVPV6b4Jn4
 experimaestro/utils/asyncio.py,sha256=9r_vFQs6T6tqmymC_DbHVFhY9YVRO6X48uEuyL_ugP8,726
 experimaestro/utils/jobs.py,sha256=42FAdKcn_v_-M6hcQZPUBr9kbDt1eVsk3a4E8Gc4eu8,2394
 experimaestro/utils/jupyter.py,sha256=JcEo2yQK7x3Cr1tNl5FqGMZOICxCv9DwMvL5xsWdQPk,2183
+experimaestro/utils/multiprocessing.py,sha256=am3DkHP_kmWbpynbck2c9QystCUtPBoSAC0ViBVzndU,1275
 experimaestro/utils/resources.py,sha256=j-nvsTFwmgENMoVGOD2Ap-UD3WU85WkI0IgeSszMCX4,1328
 experimaestro/utils/settings.py,sha256=jpFMqF0DLL4_P1xGal0zVR5cOrdD8O0Y2IOYvnRgN3k,793
 experimaestro/xpmutils.py,sha256=S21eMbDYsHfvmZ1HmKpq5Pz5O-1HnCLYxKbyTBbASyQ,638
-experimaestro-1.10.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-experimaestro-1.10.1.dist-info/METADATA,sha256=5RTdMyryQUQXXnO3VQbI1i0pKFgaVG89LZsyQzaomgw,5689
-experimaestro-1.10.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-experimaestro-1.10.1.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
-experimaestro-1.10.1.dist-info/RECORD,,
+experimaestro-1.12.0.dist-info/METADATA,sha256=BbLWHf4Fg1w4rkSNnAjjdRWf9S4aNx92uKlePpSacQc,5705
+experimaestro-1.12.0.dist-info/WHEEL,sha256=M5asmiAlL6HEcOq52Yi5mmk9KmTVjY2RDPtO4p9DMrc,88
+experimaestro-1.12.0.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
+experimaestro-1.12.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+experimaestro-1.12.0.dist-info/RECORD,,

{experimaestro-1.10.1.dist-info → experimaestro-1.12.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.0
 Root-Is-Purelib: true
 Tag: py3-none-any