experimaestro 1.9.1__tar.gz → 1.10.0__tar.gz

{experimaestro-1.9.1 → experimaestro-1.10.0}/PKG-INFO

@@ -1,52 +1,45 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.9.1
+Version: 1.10.0
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager
 Author: Benjamin Piwowarski
 Author-email: benjamin@piwowarski.fr
-Requires-Python: >=3.9,<4.0
+Requires-Python: >=3.10
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: arpeggio (>=2,<3)
-Requires-Dist: attrs (>=23.1.0,<24.0.0)
+Requires-Dist: attrs (>=23.1.0,<24)
 Requires-Dist: click (>=8)
 Requires-Dist: decorator (>=5,<6)
-Requires-Dist: docstring-parser (>=0.15,<0.16)
-Requires-Dist: fasteners (>=0.19,<0.20)
-Requires-Dist: flask (>=2.3,<3.0)
-Requires-Dist: flask-socketio (>=5.3,<6.0)
-Requires-Dist: gevent (>=24.11.1,<25.0.0)
-Requires-Dist: gevent-websocket (>=0.10,<0.11)
+Requires-Dist: docstring-parser (>=0.15,<1)
+Requires-Dist: fasteners (>=0.19,<1)
+Requires-Dist: flask (>=2.3,<3)
+Requires-Dist: flask-socketio (>=5.3,<6)
+Requires-Dist: gevent (>=25)
+Requires-Dist: gevent-websocket (>=0.10)
 Requires-Dist: huggingface-hub (>0.17)
-Requires-Dist: humanfriendly (>=10,<11)
-Requires-Dist: marshmallow (>=3.20,<4.0)
-Requires-Dist: omegaconf (>=2.3,<3.0)
-Requires-Dist: psutil (>=7)
-Requires-Dist: pyparsing (>=3.1,<4.0)
-Requires-Dist: pytools (>=2023.1.1,<2024.0.0)
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: requests (>=2.31,<3.0)
+Requires-Dist: humanfriendly (>=10)
+Requires-Dist: marshmallow (>=3.20,<4)
+Requires-Dist: mkdocs (>=1.5,<2)
+Requires-Dist: omegaconf (>=2.3,<3)
+Requires-Dist: psutil (>=7,<8)
+Requires-Dist: pyparsing (>=3.1,<4)
+Requires-Dist: pytools (>=2023.1.1,<2024)
+Requires-Dist: pyyaml (>=6.0.1,<7)
+Requires-Dist: requests (>=2.31,<3)
 Requires-Dist: rpyc (>=5,<7)
-Requires-Dist: sortedcontainers (>=2.4,<3.0)
-Requires-Dist: termcolor (>=2.3)
-Requires-Dist: tqdm (>=4.66.1,<5.0.0)
+Requires-Dist: sortedcontainers (>=2.4,<3)
+Requires-Dist: termcolor (>=2.3,<3)
+Requires-Dist: tqdm (>=4.66.1,<5)
 Requires-Dist: typing-extensions (>=4.2) ; python_version < "3.12"
-Requires-Dist: watchdog (>=2,<3)
-Project-URL: Documentation, https://experimaestro-python.readthedocs.io/
-Project-URL: Repository, https://github.com/experimaestro/experimaestro-python
+Requires-Dist: watchdog (>=2)
 Description-Content-Type: text/markdown
 
 [![PyPI version](https://badge.fury.io/py/experimaestro.svg)](https://badge.fury.io/py/experimaestro)

{experimaestro-1.9.1 → experimaestro-1.10.0}/pyproject.toml

@@ -1,6 +1,8 @@
-[tool.poetry]
+[project]
 name = "experimaestro"
-authors = ["Benjamin Piwowarski <benjamin@piwowarski.fr>"]
+authors = [
+    {name = "Benjamin Piwowarski", email = "benjamin@piwowarski.fr"}
+]
 description = '"Experimaestro is a computer science experiment manager"'
 readme = "README.md"
 license = "GPL-3"
@@ -20,9 +22,39 @@ include = [
     "src/experimaestro/mkdocs/style.css",
     { path="src/experimaestro/server/data/*", format=['sdist', 'wheel']}
 ]
-version = "1.9.1"
 repository = "https://github.com/experimaestro/experimaestro-python"
 documentation = "https://experimaestro-python.readthedocs.io/"
+dynamic = []
+requires-python = ">=3.10"
+dependencies = [
+    "arpeggio >=2,<3",
+    "attrs >=23.1.0,<24",
+    "click >=8",
+    "decorator >=5,<6",
+    "docstring-parser >=0.15,<1",
+    "fasteners >=0.19,<1",
+    "flask >=2.3,<3",
+    "flask-socketio >=5.3,<6",
+    "gevent >=25",
+    "gevent-websocket >=0.10",
+    "humanfriendly >=10",
+    "huggingface-hub >0.17",
+    "marshmallow >=3.20,<4",
+    "mkdocs >=1.5,<2",
+    "omegaconf >=2.3,<3",
+    "psutil >=7,<8",
+    "pyparsing >=3.1,<4",
+    "pytools >=2023.1.1,<2024",
+    "pyyaml >=6.0.1,<7",
+    "requests >=2.31,<3",
+    "rpyc >=5,<7",
+    "sortedcontainers >=2.4,<3",
+    "termcolor >=2.3,<3",
+    "tqdm >=4.66.1,<5",
+    "typing-extensions >=4.2; python_version < \"3.12\"",
+    "watchdog >=2"
+]
+version = "1.10.0"
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -33,38 +65,24 @@ format-jinja = """{%- set pre = [] -%}{%- set metadata = [] -%}
 {%- if revision is not none -%}{{ pre.append("rc" + revision|string) or "" }}{%- endif -%}
 {%- if distance > 0 -%}{{ metadata.append(distance|string) or "" }}{%- endif -%}
 {{ serialize_semver(base, pre, metadata)}}"""
+
+[tool.poetry]
+
+[tool.poetry-dynamic-versioning.files."src/experimaestro/version.py"]
+persistent-substitution = true
+initial-content = """
+  # These version placeholders will be replaced later during substitution.
+  __version__ = "0.0.0"
+  __version_tuple__ = (0, 0, 0)
+"""
+
+[tool.poetry.requires-plugins]
+poetry-dynamic-versioning = { version = ">=1.0.0,<2.0.0", extras = ["plugin"] }
+
 [build-system]
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
 build-backend = "poetry_dynamic_versioning.backend"
 
-[tool.poetry.dependencies]
-python = "^3.9"
-click = ">=8"
-omegaconf = "^2.3"
-typing-extensions = {version = ">=4.2", markers = "python_version < \"3.12\""}
-attrs = "^23.1.0"
-fasteners = "^0.19"
-pyyaml = "^6.0.1"
-psutil = ">=7"
-pytools = "^2023.1.1"
-tqdm = "^4.66.1"
-docstring-parser = "^0.15"
-termcolor = ">=2.3"
-requests = "^2.31"
-sortedcontainers = "^2.4"
-pyparsing = "^3.1"
-humanfriendly = "^10"
-huggingface-hub = ">0.17"
-gevent = "^24.11.1"
-gevent-websocket = "^0.10"
-flask = "^2.3"
-flask-socketio = "^5.3"
-arpeggio = "^2"
-watchdog = "^2"
-marshmallow = "^3.20"
-decorator = "^5"
-rpyc = ">=5,<7"
-
 [tool.poetry.group.ssh]
 optional = true
 
@@ -79,21 +97,21 @@ optional = true
 docutils = "^0.18"
 Pygments = "^2.15"
 
-[tool.poetry.scripts]
+[project.scripts]
 experimaestro = "experimaestro.__main__:main"
 
-[tool.poetry.plugins."mkdocs.plugins"]
+[project.entry-points."mkdocs.plugins"]
 experimaestro = "experimaestro.mkdocs:Documentation"
 
-[tool.poetry.plugins."experimaestro.process"]
+[project.entry-points."experimaestro.process"]
 local = "experimaestro.connectors.local:LocalProcess"
 slurm = "experimaestro.launchers.slurm:BatchSlurmProcess"
 
-[tool.poetry.plugins."experimaestro.connectors"]
+[project.entry-points."experimaestro.connectors"]
 local = "experimaestro.connectors.local:LocalConnector"
 ssh = "experimaestro.connectors.ssh:SshConnector"
 
-[tool.poetry.plugins."experimaestro.tokens"]
+[project.entry-points."experimaestro.tokens"]
 unix = "experimaestro.tokens:CounterToken"
 
 
@@ -122,7 +140,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.9.1"
+version = "1.10.0"
 changelog_start_rev = "v1.0.0"
 tag_format = "v$major.$minor.$patch$prerelease"
 # update_changelog_on_bump = true

{experimaestro-1.9.1 → experimaestro-1.10.0}/src/experimaestro/cli/filter.py

@@ -1,4 +1,5 @@
 import asyncio
+import logging
 from typing import Any, Callable, Dict, List, Optional
 import pyparsing as pp
 from pathlib import Path
@@ -16,7 +17,11 @@ class JobInformation:
 
     @cached_property
     def params(self):
-        return json.loads((self.path / "params.json").read_text())
+        try:
+            return json.loads((self.path / "params.json").read_text())
+        except Exception:
+            logging.warning("Could not load params.json in %s", self.path)
+            return {"tags": {}}
 
     @cached_property
     def tags(self) -> List[str]:

{experimaestro-1.9.1 → experimaestro-1.10.0}/src/experimaestro/launcherfinder/parser.py

@@ -51,7 +51,12 @@ def cpu():
 
 
 def duration():
-    return "duration", "=", RegExMatch(r"\d+"), RegExMatch(r"h(ours)?|d(ays)?")
+    return (
+        "duration",
+        "=",
+        RegExMatch(r"\d+"),
+        RegExMatch(r"h(ours?)?|d(ays?)?|m(ins?)?"),
+    )
 
 
 def one_spec():
@@ -67,7 +72,7 @@ def grammar():
 
 class Visitor(PTNodeVisitor):
     def visit_grammar(self, node, children):
-        return [child for child in children]
+        return specs.RequirementUnion(*[child for child in children])
 
     def visit_one_spec(self, node, children):
         return reduce(lambda x, el: x & el, children)

{experimaestro-1.9.1 → experimaestro-1.10.0}/src/experimaestro/launcherfinder/registry.py

@@ -9,7 +9,7 @@ from omegaconf import DictConfig, OmegaConf, SCMode
 import pkg_resources
 from experimaestro.utils import logger
 from .base import ConnectorConfiguration, TokenConfiguration
-from .specs import HostRequirement
+from .specs import HostRequirement, RequirementUnion
 
 if typing.TYPE_CHECKING:
     from experimaestro.launchers import Launcher
@@ -150,18 +150,18 @@ class LauncherRegistry:
         # Parse specs
         from .parser import parse
 
-        specs = []
+        specs = RequirementUnion()
         for spec in input_specs:
             if isinstance(spec, str):
-                specs.extend(parse(spec))
+                specs.add(parse(spec))
             else:
-                specs.append(spec)
+                specs.add(spec)
 
         # Use launcher function
         from experimaestro.launchers import Launcher
 
         if self.find_launcher_fn is not None:
-            for spec in specs:
+            for spec in specs.requirements:
                 if launcher := self.find_launcher_fn(spec, tags):
                     assert isinstance(
                         launcher, Launcher

{experimaestro-1.9.1 → experimaestro-1.10.0}/src/experimaestro/launcherfinder/specs.py

@@ -1,4 +1,6 @@
+from abc import ABC, abstractmethod
 import logging
+import math
 from attr import Factory
 from attrs import define
 from copy import copy, deepcopy
@@ -88,7 +90,7 @@ class MatchRequirement:
     requirement: "HostSimpleRequirement"
 
 
-class HostRequirement:
+class HostRequirement(ABC):
     """A requirement must be a disjunction of host requirements"""
 
     requirements: List["HostSimpleRequirement"]
@@ -103,6 +105,12 @@ class HostRequirement:
     def match(self, host: HostSpecification) -> Optional[MatchRequirement]:
         raise NotImplementedError()
 
+    @abstractmethod
+    def multiply_duration(self, coefficient: float) -> "HostRequirement":
+        """Returns a new HostRequirement with a duration multiplied by the
+        provided coefficient"""
+        ...
+
 
 class RequirementUnion(HostRequirement):
     """Ordered list of simple host requirements -- the first one is the priority"""
@@ -112,6 +120,16 @@ class RequirementUnion(HostRequirement):
     def __init__(self, *requirements: "HostSimpleRequirement"):
         self.requirements = list(requirements)
 
+    def add(self, requirement: "HostRequirement"):
+        match requirement:
+            case HostSimpleRequirement():
+                self.requirements.extend(*requirement.requirements)
+            case RequirementUnion():
+                self.requirements.append(requirement)
+            case _:
+                raise RuntimeError("Cannot handle type %s", type(requirement))
+        return self
+
     def match(self, host: HostSpecification) -> Optional[MatchRequirement]:
         """Returns the matched requirement (if any)"""
 
@@ -126,6 +144,14 @@ class RequirementUnion(HostRequirement):
 
         return argmax
 
+    def multiply_duration(self, coefficient: float) -> "RequirementUnion":
+        return RequirementUnion(
+            *[r.multiply_duration(coefficient) for r in self.requirements]
+        )
+
+    def __repr__(self):
+        return " | ".join(repr(r) for r in self.requirements)
+
 
 class HostSimpleRequirement(HostRequirement):
     """Simple host requirement"""
@@ -142,6 +168,11 @@ class HostSimpleRequirement(HostRequirement):
     def __repr__(self):
         return f"Req(cpu={self.cpu}, cuda={self.cuda_gpus}, duration={self.duration})"
 
+    def multiply_duration(self, coefficient: float) -> "HostSimpleRequirement":
+        r = HostSimpleRequirement(self)
+        r.duration = math.ceil(self.duration * coefficient)
+        return r
+
     def __init__(self, *reqs: "HostSimpleRequirement"):
         self.cuda_gpus = []
         self.cpu = CPUSpecification(0, 0)

{experimaestro-1.9.1 → experimaestro-1.10.0}/src/experimaestro/tests/test_findlauncher.py

@@ -3,9 +3,9 @@ from experimaestro.launcherfinder.specs import (
     CPUSpecification,
     CudaSpecification,
     HostSpecification,
+    RequirementUnion,
     cpu,
     cuda_gpu,
-    HostSimpleRequirement,
 )
 from experimaestro.launcherfinder import parse
 from humanfriendly import parse_size, parse_timespan
@@ -39,6 +39,11 @@ def test_findlauncher_specs():
     assert m is not None
     assert m.requirement is req2
 
+    # Multiply
+    req2 = req.multiply_duration(2)
+    for i in range(2):
+        assert req2.requirements[i].duration == req.requirements[i].duration * 2
+
 
 def test_findlauncher_specs_gpu_mem():
     host = HostSpecification(
@@ -60,8 +65,10 @@ def test_findlauncher_specs_gpu_mem():
 
 
 def test_findlauncher_parse():
-    (r,) = parse("""duration=4 d & cuda(mem=4G) * 2 & cpu(mem=400M, cores=4)""")
-    assert isinstance(r, HostSimpleRequirement)
+    r = parse("""duration=4 d & cuda(mem=4G) * 2 & cpu(mem=400M, cores=4)""")
+    assert isinstance(r, RequirementUnion)
+
+    r = r.requirements[0]
 
     assert len(r.cuda_gpus) == 2
     assert r.cuda_gpus[0].memory == parse_size("4G")
@@ -79,7 +86,7 @@ def slurm_constraint_split(constraint: str):
 
 
 def test_findlauncher_slurm():
-    path = ResourcePathWrapper.create(f"{__package__ }.launchers", "config_slurm")
+    path = ResourcePathWrapper.create(f"{__package__}.launchers", "config_slurm")
 
     assert (path / "launchers.py").is_file()