experimaestro 1.9.1__py3-none-any.whl → 1.10.1__py3-none-any.whl

experimaestro/cli/filter.py

@@ -1,4 +1,5 @@
 import asyncio
+import logging
 from typing import Any, Callable, Dict, List, Optional
 import pyparsing as pp
 from pathlib import Path
@@ -16,7 +17,11 @@ class JobInformation:
 
     @cached_property
     def params(self):
-        return json.loads((self.path / "params.json").read_text())
+        try:
+            return json.loads((self.path / "params.json").read_text())
+        except Exception:
+            logging.warning("Could not load params.json in %s", self.path)
+            return {"tags": {}}
 
     @cached_property
     def tags(self) -> List[str]:

experimaestro/launcherfinder/parser.py

@@ -51,7 +51,12 @@ def cpu():
 
 
 def duration():
-    return "duration", "=", RegExMatch(r"\d+"), RegExMatch(r"h(ours)?|d(ays)?")
+    return (
+        "duration",
+        "=",
+        RegExMatch(r"\d+"),
+        RegExMatch(r"h(ours?)?|d(ays?)?|m(ins?)?"),
+    )
 
 
 def one_spec():
@@ -67,7 +72,7 @@ def grammar():
 
 class Visitor(PTNodeVisitor):
     def visit_grammar(self, node, children):
-        return [child for child in children]
+        return specs.RequirementUnion(*[child for child in children])
 
     def visit_one_spec(self, node, children):
         return reduce(lambda x, el: x & el, children)

experimaestro/launcherfinder/registry.py

@@ -9,7 +9,7 @@ from omegaconf import DictConfig, OmegaConf, SCMode
 import pkg_resources
 from experimaestro.utils import logger
 from .base import ConnectorConfiguration, TokenConfiguration
-from .specs import HostRequirement
+from .specs import HostRequirement, RequirementUnion
 
 if typing.TYPE_CHECKING:
     from experimaestro.launchers import Launcher
@@ -150,18 +150,18 @@ class LauncherRegistry:
         # Parse specs
         from .parser import parse
 
-        specs = []
+        specs = RequirementUnion()
         for spec in input_specs:
             if isinstance(spec, str):
-                specs.extend(parse(spec))
+                specs.add(parse(spec))
             else:
-                specs.append(spec)
+                specs.add(spec)
 
         # Use launcher function
         from experimaestro.launchers import Launcher
 
         if self.find_launcher_fn is not None:
-            for spec in specs:
+            for spec in specs.requirements:
                 if launcher := self.find_launcher_fn(spec, tags):
                     assert isinstance(
                         launcher, Launcher

experimaestro/launcherfinder/specs.py

@@ -1,4 +1,6 @@
+from abc import ABC, abstractmethod
 import logging
+import math
 from attr import Factory
 from attrs import define
 from copy import copy, deepcopy
@@ -88,7 +90,7 @@ class MatchRequirement:
     requirement: "HostSimpleRequirement"
 
 
-class HostRequirement:
+class HostRequirement(ABC):
     """A requirement must be a disjunction of host requirements"""
 
     requirements: List["HostSimpleRequirement"]
@@ -103,6 +105,12 @@ class HostRequirement:
     def match(self, host: HostSpecification) -> Optional[MatchRequirement]:
         raise NotImplementedError()
 
+    @abstractmethod
+    def multiply_duration(self, coefficient: float) -> "HostRequirement":
+        """Returns a new HostRequirement with a duration multiplied by the
+        provided coefficient"""
+        ...
+
 
 class RequirementUnion(HostRequirement):
     """Ordered list of simple host requirements -- the first one is the priority"""
@@ -112,6 +120,16 @@ class RequirementUnion(HostRequirement):
     def __init__(self, *requirements: "HostSimpleRequirement"):
         self.requirements = list(requirements)
 
+    def add(self, requirement: "HostRequirement"):
+        match requirement:
+            case HostSimpleRequirement():
+                self.requirements.extend(*requirement.requirements)
+            case RequirementUnion():
+                self.requirements.append(requirement)
+            case _:
+                raise RuntimeError("Cannot handle type %s", type(requirement))
+        return self
+
     def match(self, host: HostSpecification) -> Optional[MatchRequirement]:
         """Returns the matched requirement (if any)"""
 
@@ -126,6 +144,14 @@ class RequirementUnion(HostRequirement):
 
         return argmax
 
+    def multiply_duration(self, coefficient: float) -> "RequirementUnion":
+        return RequirementUnion(
+            *[r.multiply_duration(coefficient) for r in self.requirements]
+        )
+
+    def __repr__(self):
+        return " | ".join(repr(r) for r in self.requirements)
+
 
 class HostSimpleRequirement(HostRequirement):
     """Simple host requirement"""
@@ -142,6 +168,11 @@ class HostSimpleRequirement(HostRequirement):
     def __repr__(self):
         return f"Req(cpu={self.cpu}, cuda={self.cuda_gpus}, duration={self.duration})"
 
+    def multiply_duration(self, coefficient: float) -> "HostSimpleRequirement":
+        r = HostSimpleRequirement(self)
+        r.duration = math.ceil(self.duration * coefficient)
+        return r
+
     def __init__(self, *reqs: "HostSimpleRequirement"):
         self.cuda_gpus = []
         self.cpu = CPUSpecification(0, 0)

experimaestro/tests/test_findlauncher.py

@@ -3,9 +3,9 @@ from experimaestro.launcherfinder.specs import (
     CPUSpecification,
     CudaSpecification,
     HostSpecification,
+    RequirementUnion,
     cpu,
     cuda_gpu,
-    HostSimpleRequirement,
 )
 from experimaestro.launcherfinder import parse
 from humanfriendly import parse_size, parse_timespan
@@ -39,6 +39,11 @@ def test_findlauncher_specs():
     assert m is not None
     assert m.requirement is req2
 
+    # Multiply
+    req2 = req.multiply_duration(2)
+    for i in range(2):
+        assert req2.requirements[i].duration == req.requirements[i].duration * 2
+
 
 def test_findlauncher_specs_gpu_mem():
     host = HostSpecification(
@@ -60,8 +65,10 @@ def test_findlauncher_specs_gpu_mem():
 
 
 def test_findlauncher_parse():
-    (r,) = parse("""duration=4 d & cuda(mem=4G) * 2 & cpu(mem=400M, cores=4)""")
-    assert isinstance(r, HostSimpleRequirement)
+    r = parse("""duration=4 d & cuda(mem=4G) * 2 & cpu(mem=400M, cores=4)""")
+    assert isinstance(r, RequirementUnion)
+
+    r = r.requirements[0]
 
     assert len(r.cuda_gpus) == 2
     assert r.cuda_gpus[0].memory == parse_size("4G")
@@ -79,7 +86,7 @@ def slurm_constraint_split(constraint: str):
 
 
 def test_findlauncher_slurm():
-    path = ResourcePathWrapper.create(f"{__package__ }.launchers", "config_slurm")
+    path = ResourcePathWrapper.create(f"{__package__}.launchers", "config_slurm")
 
     assert (path / "launchers.py").is_file()
 

{experimaestro-1.9.1.dist-info → experimaestro-1.10.1.dist-info}/METADATA

@@ -1,52 +1,45 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.9.1
+Version: 1.10.1
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager
 Author: Benjamin Piwowarski
 Author-email: benjamin@piwowarski.fr
-Requires-Python: >=3.9,<4.0
+Requires-Python: >=3.10
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: arpeggio (>=2,<3)
-Requires-Dist: attrs (>=23.1.0,<24.0.0)
+Requires-Dist: attrs (>=23.1.0,<24)
 Requires-Dist: click (>=8)
 Requires-Dist: decorator (>=5,<6)
-Requires-Dist: docstring-parser (>=0.15,<0.16)
-Requires-Dist: fasteners (>=0.19,<0.20)
-Requires-Dist: flask (>=2.3,<3.0)
-Requires-Dist: flask-socketio (>=5.3,<6.0)
-Requires-Dist: gevent (>=24.11.1,<25.0.0)
-Requires-Dist: gevent-websocket (>=0.10,<0.11)
+Requires-Dist: docstring-parser (>=0.15,<1)
+Requires-Dist: fasteners (>=0.19,<1)
+Requires-Dist: flask (>=2.3,<3)
+Requires-Dist: flask-socketio (>=5.3,<6)
+Requires-Dist: gevent (>=25)
+Requires-Dist: gevent-websocket (>=0.10)
 Requires-Dist: huggingface-hub (>0.17)
-Requires-Dist: humanfriendly (>=10,<11)
-Requires-Dist: marshmallow (>=3.20,<4.0)
-Requires-Dist: omegaconf (>=2.3,<3.0)
-Requires-Dist: psutil (>=7)
-Requires-Dist: pyparsing (>=3.1,<4.0)
-Requires-Dist: pytools (>=2023.1.1,<2024.0.0)
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: requests (>=2.31,<3.0)
+Requires-Dist: humanfriendly (>=10)
+Requires-Dist: marshmallow (>=3.20,<4)
+Requires-Dist: mkdocs (>=1.5,<2)
+Requires-Dist: omegaconf (>=2.3,<3)
+Requires-Dist: psutil (>=7,<8)
+Requires-Dist: pyparsing (>=3.1,<4)
+Requires-Dist: pytools (>=2023.1.1,<2024)
+Requires-Dist: pyyaml (>=6.0.1,<7)
+Requires-Dist: requests (>=2.31,<3)
 Requires-Dist: rpyc (>=5,<7)
-Requires-Dist: sortedcontainers (>=2.4,<3.0)
-Requires-Dist: termcolor (>=2.3)
-Requires-Dist: tqdm (>=4.66.1,<5.0.0)
+Requires-Dist: sortedcontainers (>=2.4,<3)
+Requires-Dist: termcolor (>=2.3,<3)
+Requires-Dist: tqdm (>=4.66.1,<5)
 Requires-Dist: typing-extensions (>=4.2) ; python_version < "3.12"
-Requires-Dist: watchdog (>=2,<3)
-Project-URL: Documentation, https://experimaestro-python.readthedocs.io/
-Project-URL: Repository, https://github.com/experimaestro/experimaestro-python
+Requires-Dist: watchdog (>=2)
 Description-Content-Type: text/markdown
 
 [![PyPI version](https://badge.fury.io/py/experimaestro.svg)](https://badge.fury.io/py/experimaestro)

{experimaestro-1.9.1.dist-info → experimaestro-1.10.1.dist-info}/RECORD

@@ -3,7 +3,7 @@ experimaestro/__main__.py,sha256=Dv9lFl03yt1dswd0Xb9NIJRgHpA5_IwH4RfQPEHyFz0,158
 experimaestro/annotations.py,sha256=wyVmPlkXuoT6IxJ-ti8bFo6HxhGY1BYBxM5-pib6shU,8773
 experimaestro/checkers.py,sha256=ZCMbnE_GFC5compWjt-fuHhPImi9fCPjImF8Ow9NqK8,696
 experimaestro/cli/__init__.py,sha256=mzc-qqTFtZnFwCQl7IiwlYXEx08kLGwdntWayCerZ6E,9610
-experimaestro/cli/filter.py,sha256=WsoaxnwgkcbMRlWJdV55333F3cLg7W-9Zbx9e66ZmvY,6148
+experimaestro/cli/filter.py,sha256=bINAF-O7CwJ2u3T6xG_Q_niqDbPeoEASyYLuCeNlbFw,6313
 experimaestro/cli/jobs.py,sha256=BnejUnhKAgMBVgwANfQYj5mLzknXVohveg7NpovNZ8o,7925
 experimaestro/click.py,sha256=6BkeQHEgcxaxzq3xEvEEzwzuBj5-dkfrpOGikuA8L00,1377
 experimaestro/commandline.py,sha256=MJIJcfppGCjNA8ozxXUzbUSeDOlTExuzhxryGx3_lIo,9314
@@ -34,9 +34,9 @@ experimaestro/huggingface.py,sha256=gnVlr6SZnbutYz4PLH0Q77n1TRF-uk-dR-3UFzFqAY0,
 experimaestro/ipc.py,sha256=Xn3tYME83jLEB0nFak3DwEIhpL5IRZpCl3jirBF_jl4,1570
 experimaestro/launcherfinder/__init__.py,sha256=qRUDyv3B9UsAM8Q31mRrZrTZox0AptwdmOY4f2K-TUo,279
 experimaestro/launcherfinder/base.py,sha256=q47SsF_cXdo5O6ZhFKn5385WVFcx8Wd-BcEpd6tRpbs,515
-experimaestro/launcherfinder/parser.py,sha256=w8JyrBtiD15SYSPmzv06H64WDZAOdrz0xyV703mQ6TQ,2287
-experimaestro/launcherfinder/registry.py,sha256=mhFAVgk8t2qWemUlhgM4nftn2x1THm7LTKSyDVsQQ9M,6379
-experimaestro/launcherfinder/specs.py,sha256=j3rObYRUi8WbU9n8220usOIWJ9p68Uti0BjkQRqGiZc,6560
+experimaestro/launcherfinder/parser.py,sha256=MIHhjs2sTVxLHLcc1CgFid9XxhInXker8QdA-GBA-Bk,2364
+experimaestro/launcherfinder/registry.py,sha256=F16A7SE_CgzaTBC-LBlurbthTDvYUa5iyjJftkDNN4M,6420
+experimaestro/launcherfinder/specs.py,sha256=484S4-N22Y10BsyTpNwW0l369ALAJUrXQK9LsYSYsaQ,7705
 experimaestro/launchers/__init__.py,sha256=lXn544sgJExr6uirILWzAXu_IfmfyqFZOt4OzRnjHXg,2525
 experimaestro/launchers/direct.py,sha256=JZh6WOPnO6ED_xlOs8pL4MRFmnRhmXzpVxTl-ByaD2A,258
 experimaestro/launchers/oar.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -119,7 +119,7 @@ experimaestro/tests/tasks/foreign.py,sha256=uhXDOcWozkcm26ybbeEU9RElJpbhjC-zdzml
 experimaestro/tests/test_checkers.py,sha256=Kg5frDNRE3pvWVmmYzyk0tJFNO885KOrK48lSu-NlYA,403
 experimaestro/tests/test_dependencies.py,sha256=xfWrSkvjT45G4FSCL535m1huLT2ghmyW7kvP_XvvCJQ,2005
 experimaestro/tests/test_experiment.py,sha256=QWF9aHewL9hepagrKKFyfikKn3iiZ_lRRXl1LLttta0,1687
-experimaestro/tests/test_findlauncher.py,sha256=8tjpR8bLMi6Gjs7KpY2t64izZso6bmY7vIivMflm-Bc,2965
+experimaestro/tests/test_findlauncher.py,sha256=KPy8ow--NXS1KFCIpxrmEJFRvjo-v-PwlVHVyoVKLPg,3134
 experimaestro/tests/test_forward.py,sha256=9y1zYm7hT_Lx5citxnK7n20cMZ2WJbsaEeY5irCZ9U4,735
 experimaestro/tests/test_identifier.py,sha256=zgfpz5UumQftQTHhdw3nmr044Xd7Ntrh7e2hIAoS_PA,13862
 experimaestro/tests/test_instance.py,sha256=h-8UeoOlNsD-STWq5jbhmxw35CyiwJxtKAaUGmLPgB8,1521
@@ -150,8 +150,8 @@ experimaestro/utils/jupyter.py,sha256=JcEo2yQK7x3Cr1tNl5FqGMZOICxCv9DwMvL5xsWdQP
 experimaestro/utils/resources.py,sha256=j-nvsTFwmgENMoVGOD2Ap-UD3WU85WkI0IgeSszMCX4,1328
 experimaestro/utils/settings.py,sha256=jpFMqF0DLL4_P1xGal0zVR5cOrdD8O0Y2IOYvnRgN3k,793
 experimaestro/xpmutils.py,sha256=S21eMbDYsHfvmZ1HmKpq5Pz5O-1HnCLYxKbyTBbASyQ,638
-experimaestro-1.9.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-experimaestro-1.9.1.dist-info/METADATA,sha256=iwhee9G1vBhkBCM-zMWL_FgRwyZSBV7bZJLsnmVgCwk,6170
-experimaestro-1.9.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-experimaestro-1.9.1.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
-experimaestro-1.9.1.dist-info/RECORD,,
+experimaestro-1.10.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+experimaestro-1.10.1.dist-info/METADATA,sha256=5RTdMyryQUQXXnO3VQbI1i0pKFgaVG89LZsyQzaomgw,5689
+experimaestro-1.10.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+experimaestro-1.10.1.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
+experimaestro-1.10.1.dist-info/RECORD,,