experimaestro 1.5.0__py3-none-any.whl → 1.5.1__py3-none-any.whl

experimaestro/experiments/cli.py

@@ -1,4 +1,5 @@
 import inspect
+import itertools
 import json
 import logging
 import sys
@@ -11,7 +12,7 @@ import yaml
 from experimaestro import LauncherRegistry, RunMode, experiment
 from experimaestro.experiments.configuration import ConfigurationBase
 from experimaestro.exceptions import HandledException
-from experimaestro.settings import get_workspace
+from experimaestro.settings import get_settings, get_workspace
 from omegaconf import OmegaConf, SCMode
 from termcolor import cprint
 
@@ -219,21 +220,30 @@ def experiments_cli(  # noqa: C901
         sys.exit(0)
 
     # Move to an object container
-    configuration: schema = OmegaConf.to_container(
+    configuration = OmegaConf.to_container(
         configuration, structured_config_mode=SCMode.INSTANTIATE
     )
 
     # Get the working directory
-    if workdir is None or not Path(workdir).is_dir():
-        workdir = get_workspace(workdir).path.expanduser().resolve()
-        logging.info("Using working directory %s", workdir)
+    settings = get_settings()
+    ws_env = {}
+    workdir = Path(workdir) if workdir else None
+    if (workdir is None) or (not workdir.is_dir()):
+        logging.info("Searching for workspace %s", workdir)
+        ws_settings = get_workspace(str(workdir))
+        workdir = ws_settings.path.expanduser()
+        ws_env = ws_settings.env
+
+    logging.info("Using working directory %s", str(workdir.resolve()))
 
     # --- Runs the experiment
     with experiment(
         workdir, configuration.id, host=host, port=port, run_mode=run_mode
     ) as xp:
         # Set up the environment
-        for key, value in env:
+        # (1) global settings (2) workspace settings and (3) command line settings
+        for key, value in itertools.chain(settings.env.items(), ws_env.items(), env):
+            logging.info("Setting environment: %s=%s", key, value)
             xp.setenv(key, value)
 
         try:

experimaestro/launcherfinder/parser.py

@@ -54,8 +54,12 @@ def duration():
     return "duration", "=", RegExMatch(r"\d+"), RegExMatch(r"h(ours)?|d(ays)?")
 
 
+def one_spec():
+    return OneOrMore(OrderedChoice([duration, cuda, cpu]), sep="&")
+
+
 def grammar():
-    return OneOrMore(OrderedChoice([duration, cuda, cpu]), sep="&"), EndOfFile()
+    return OneOrMore(one_spec, sep="|"), EndOfFile()
 
 
 # ---- Visitor
@@ -63,6 +67,9 @@ def grammar():
 
 class Visitor(PTNodeVisitor):
     def visit_grammar(self, node, children):
+        return [child for child in children]
+
+    def visit_one_spec(self, node, children):
         return reduce(lambda x, el: x & el, children)
 
     def visit_duration(self, node, children):

experimaestro/launcherfinder/registry.py

@@ -219,7 +219,7 @@ class LauncherRegistry:
         raise AssertionError(f"No connector with identifier {identifier}")
 
     def find(
-        self, *specs: Union[HostRequirement, str], tags: Set[str] = set()
+        self, *input_specs: Union[HostRequirement, str], tags: Set[str] = set()
     ) -> Optional["Launcher"]:
         """ "
         Arguments:
@@ -237,7 +237,12 @@ class LauncherRegistry:
         # Parse specs
         from .parser import parse
 
-        specs = [parse(spec) if isinstance(spec, str) else spec for spec in specs]
+        specs = []
+        for spec in input_specs:
+            if isinstance(spec, str):
+                specs.extend(parse(spec))
+            else:
+                specs.append(spec)
 
         # Use launcher function
         if self.find_launcher_fn is not None:
@@ -247,8 +252,6 @@ class LauncherRegistry:
 
         # We have registered launchers
         for spec in specs:
-            if isinstance(spec, str):
-                spec = parse(spec)
             for handler in self.launchers:
                 if (not tags) or any((tag in tags) for tag in handler.tags):
                     if launcher := handler.get(self, spec):

experimaestro/launchers/slurm/base.py

@@ -92,6 +92,8 @@ class SlurmProcessWatcher(threading.Thread):
         self.jobs: Dict[str, SlurmJobState] = {}
 
         self.cv = ThreadingCondition()
+        self.fetched_event = threading.Event()
+        self.updating_jobs = threading.Lock()
         self.start()
 
     @staticmethod
@@ -109,10 +111,18 @@ class SlurmProcessWatcher(threading.Thread):
         with watcher.cv:
             watcher.cv.notify()
 
-    def getjob(self, jobid):
+    def getjob(self, jobid, timeout=None):
         """Allows to share the calls to sacct"""
+
+        # Ensures that we have fetched at least once
+        self.fetched_event.wait()
+
+        # Waits that jobs are refreshed (with a timeout)
         with self.cv:
-            self.cv.wait()
+            self.cv.wait(timeout=timeout)
+
+        # Ensures jobs are not updated right now
+        with self.updating_jobs:
             return self.jobs.get(jobid)
 
     def run(self):
@@ -129,9 +139,9 @@ class SlurmProcessWatcher(threading.Thread):
             builder.stdout = Redirect.pipe(handler)
             builder.environ = self.launcher.launcherenv
             logger.debug("Checking SLURM state with sacct")
-            builder.start()
+            process = builder.start()
 
-            with self.cv:
+            with self.updating_jobs:
                 self.jobs = {}
                 output = handler.output.decode("utf-8")
                 for line in output.split("\n"):
@@ -143,7 +153,11 @@ class SlurmProcessWatcher(threading.Thread):
                             logger.debug("Parsed line: %s", line)
                         except ValueError:
                             logger.error("Could not parse line %s", line)
+            process.kill()
+
+            with self.cv:
                 logger.debug("Jobs %s", self.jobs)
+                self.fetched_event.set()
                 self.cv.notify_all()
 
                 self.cv.wait_for(
@@ -193,7 +207,18 @@ class BatchSlurmProcess(Process):
     def fromspec(cls, connector: Connector, spec: Dict[str, Any]):
         options = {k: v for k, v in spec.get("options", ())}
         launcher = SlurmLauncher(connector=connector, **options)
-        return BatchSlurmProcess(launcher, spec["pid"])
+        process = BatchSlurmProcess(launcher, spec["pid"])
+
+        # Checks that the process is running
+        with SlurmProcessWatcher.get(launcher) as watcher:
+            logger.info("Checking SLURM job %s", process.jobid)
+            jobinfo = watcher.getjob(process.jobid, timeout=0.1)
+            if jobinfo and jobinfo.state.running:
+                logger.debug(
+                    "SLURM job is running (%s), returning process", process.jobid
+                )
+                return process
+        return None
 
 
 def addstream(command: List[str], option: str, redirect: Redirect):

experimaestro/scriptbuilder.py

@@ -91,24 +91,28 @@ class PythonScriptBuilder:
         logger.debug("Writing script %s", scriptpath)
         with scriptpath.open("wt") as out:
             out.write("#!{}\n".format(self.pythonpath))
-            out.write("# Experimaestro generated task\n")
+            out.write("# Experimaestro generated task\n\n")
+            out.write("""import logging\nlogging.basicConfig(level=logging.INFO)\n\n""")
+
+            out.write("\nif __name__ == '__main__':\n\n" "")
 
             # --- Checks locks right away
 
-            out.write("""import logging\nlogging.basicConfig(level=logging.INFO)\n\n""")
-            out.write("""from experimaestro.run import TaskRunner\nimport os\n\n""")
+            out.write(
+                """    from experimaestro.run import TaskRunner\n    import os\n\n"""
+            )
 
-            out.write("lockfiles = [\n")
+            out.write("    lockfiles = [\n")
             for path in self.lockfiles:
-                out.write(f"   '''{relpath(path)}''',\n")
-            out.write("]\n")
+                out.write(f"       '''{relpath(path)}''',\n")
+            out.write("    ]\n")
 
             for name, value in job.environ.items():
-                out.write(f"""os.environ["{name}"] = "{shquote(value)}"\n""")
+                out.write(f"""    os.environ["{name}"] = "{shquote(value)}"\n""")
             out.write("\n")
 
             out.write(
-                f"""TaskRunner("{shquote(connector.resolve(scriptpath))}","""
+                f"""    TaskRunner("{shquote(connector.resolve(scriptpath))}","""
                 """ lockfiles).run()\n"""
             )
 

experimaestro/settings.py

@@ -1,9 +1,9 @@
 import os
 from omegaconf import OmegaConf
-from dataclasses import dataclass, field
+from dataclasses import field, dataclass
 from functools import lru_cache
 from pathlib import Path
-from typing import Optional, List
+from typing import Dict, Optional, List
 
 
 @dataclass
@@ -29,12 +29,18 @@ class WorkspaceSettings:
     path: Path
     """The workspace path"""
 
+    env: Dict[str, str] = field(default_factory=dict)
+    """Workspace specific environment variables"""
+
 
 @dataclass
 class Settings:
     server: ServerSettings = field(default_factory=ServerSettings)
     workspaces: List[WorkspaceSettings] = field(default_factory=list)
 
+    env: Dict[str, str] = field(default_factory=dict)
+    """Default environment variables"""
+
 
 @lru_cache()
 def get_settings(path: Optional[Path] = None) -> Settings:

experimaestro/tests/test_findlauncher.py

@@ -60,7 +60,7 @@ def test_findlauncher_specs_gpu_mem():
 
 
 def test_findlauncher_parse():
-    r = parse("""duration=4 d & cuda(mem=4G) * 2 & cpu(mem=400M, cores=4)""")
+    (r,) = parse("""duration=4 d & cuda(mem=4G) * 2 & cpu(mem=400M, cores=4)""")
     assert isinstance(r, HostSimpleRequirement)
 
     assert len(r.cuda_gpus) == 2

{experimaestro-1.5.0.dist-info → experimaestro-1.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: experimaestro
-Version: 1.5.0
+Version: 1.5.1
 Summary: "Experimaestro is a computer science experiment manager"
 Home-page: https://github.com/experimaestro/experimaestro-python
 License: GPL-3

{experimaestro-1.5.0.dist-info → experimaestro-1.5.1.dist-info}/RECORD

@@ -19,7 +19,7 @@ experimaestro/core/types.py,sha256=oTXD4UjMVoYn_Usxn2C4h6IGhYDTtekKB3O3hfeOynQ,2
 experimaestro/core/utils.py,sha256=JfC3qGUS9b6FUHc2VxIYUI9ysNpXSQ1LjOBkjfZ8n7o,495
 experimaestro/exceptions.py,sha256=cUy83WHM3GeynxmMk6QRr5xsnpqUAdAoc-m3KQVrE2o,44
 experimaestro/experiments/__init__.py,sha256=GcpDUIbCvhnv6rxFdAp4wTffCVNTv-InY6fbQAlTy-o,159
-experimaestro/experiments/cli.py,sha256=vne6skjbNmMIIgmlEJY0b_xoJIxgIjis07xL4byr-0o,7080
+experimaestro/experiments/cli.py,sha256=MXJNRCvSuespStK8BSR82q-3NSPPXm6jq0Rsui4V2BU,7535
 experimaestro/experiments/configuration.py,sha256=8GRqyLG1leF_NbvbFzqpm0yM24O0WjSNmQzvnuLnxxw,1150
 experimaestro/filter.py,sha256=DN1PrmS9yXoOa5Xnv001zbxzpdzvcVZFI9xZFKZ1-6g,5794
 experimaestro/generators.py,sha256=9NQ_TfDfASkArLnO4PF7s5Yoo9KWjlna2DCPzk5gJOI,1230
@@ -27,14 +27,14 @@ experimaestro/huggingface.py,sha256=gnVlr6SZnbutYz4PLH0Q77n1TRF-uk-dR-3UFzFqAY0,
 experimaestro/ipc.py,sha256=ltYqybPm_XfcQC3yiskMfhfI_1dREs-XRu0F83YsNws,1490
 experimaestro/launcherfinder/__init__.py,sha256=jIeT9uRKsIjUv-oyKt0MhFzXJJrSdpJKwM2vL9Sk5YY,294
 experimaestro/launcherfinder/base.py,sha256=NptPJ0e8CktdhOPejocSfI_B4mloeH_EmJrbXruUCSA,1020
-experimaestro/launcherfinder/parser.py,sha256=0qDXgdPce_PsWDy-hKTfxxjXjTAu4FA8moKtyllB2-Q,2129
-experimaestro/launcherfinder/registry.py,sha256=KxlKZDl0PbBIh-JUnn0U8JYSHVt14L3Qox0DKJefwoc,8892
+experimaestro/launcherfinder/parser.py,sha256=pYbfEJw7osnqZWm7fkVhQawhpNU8dLU_6vEjtXdc8E8,2279
+experimaestro/launcherfinder/registry.py,sha256=FKoacw7sIFxfYTRvaJpQs6o67qtBIDJobX0nmEUBU1Y,8927
 experimaestro/launcherfinder/specs.py,sha256=G8za6mEmkVxuZY_ab3OhWJIpONpcBMO_iXeB30sUbhI,6448
 experimaestro/launchers/__init__.py,sha256=lXn544sgJExr6uirILWzAXu_IfmfyqFZOt4OzRnjHXg,2525
 experimaestro/launchers/direct.py,sha256=VJzQNrUGnh-1Ovt6uw4yYIjXNu45QpR-_6V45lcZAfQ,1967
 experimaestro/launchers/oar.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/launchers/slurm/__init__.py,sha256=R1Zwd4phZaXV8FwCYhzfB44n0V4cf-hBQzOc6NkFQ0s,41
-experimaestro/launchers/slurm/base.py,sha256=NoOO4Dj_KP9A0wjVthmDT6PoOquc21JbVsh_hKsmGiU,13200
+experimaestro/launchers/slurm/base.py,sha256=nMoSBMbA901OcneSHVXj8PXGqfv4mVN5G-NPDZ0HXO0,14135
 experimaestro/launchers/slurm/cli.py,sha256=c-S0TImvhZ-ZxFs5-5T2GRDm5manRYRiYudpQLHwsrQ,818
 experimaestro/launchers/slurm/configuration.py,sha256=mtozeuvIZmEfHlvEylwCgBrlVRFHT_jWNAKVxR4Tz1E,19357
 experimaestro/locking.py,sha256=hPT-LuDGZTijpbme8O0kEoB9a3WjdVzI2h31OT44UxE,1477
@@ -54,7 +54,7 @@ experimaestro/scheduler/dependencies.py,sha256=n9XegwrmjayOIxt3xhuTEBVEBGSq4oeVd
 experimaestro/scheduler/environment.py,sha256=ZaSHSgAcZBmIj7b_eS1OvNQOuVLFvuw-qvqtYrc3Vms,2393
 experimaestro/scheduler/services.py,sha256=aCKkNZMULlceabqf-kOs_-C7KPINnjU3Q-I00o5x6iY,2189
 experimaestro/scheduler/workspace.py,sha256=xATJi6-GJcpdwB4alliJnmAuvwt-URUiUKUfq5scUac,1731
-experimaestro/scriptbuilder.py,sha256=-wq4jgH8eKpjKWyU_7SxU7MNJLfPqZ0VdERv6rrkJ88,4235
+experimaestro/scriptbuilder.py,sha256=AYh2LdO__mrCWuFa1JWvdfKm_wnk5hHDr_OjHo1voIE,4357
 experimaestro/server/__init__.py,sha256=F2bzLf2q29Haj2OIbPA26r5WVbaipBNylIozg-As758,10854
 experimaestro/server/data/016b4a6cdced82ab3aa1.ttf,sha256=AD8RVBhWpkmmyCNcYmbIk2IkxdYJ5RRC2iTcVVbRT78,189684
 experimaestro/server/data/0c35d18bf06992036b69.woff2,sha256=gmX2R4Y5fWuDLRygqv3xSa2E5ydZ__qfcnLpGg-wFdE,128352
@@ -80,7 +80,7 @@ experimaestro/server/data/index.js,sha256=f0GvRsfsQ4ayP4en7Q-raZ6buwRXLCswCbzVax
 experimaestro/server/data/index.js.map,sha256=za3MUIjzyyGRI6F5KuBFMTgrFU55xgt0LBrw-4YPHag,3904832
 experimaestro/server/data/login.html,sha256=4dvhSOn6DHp_tbmzqIKrqq2uAo0sAUbgLVD0lTnPp4s,511
 experimaestro/server/data/manifest.json,sha256=EpzHQZzrGh9c1Kf63nrqvI33H1cm0nLYfdh5lDm8ijI,318
-experimaestro/settings.py,sha256=YTUHZROgrGuSAhZIke74bhBrF2CBH7eSj98ZNjmnMsU,1600
+experimaestro/settings.py,sha256=nZO-667XgM-aC6A-RHw1C4gglDlDeIIIJAxs83IgRyI,1807
 experimaestro/sphinx/__init__.py,sha256=heovvtwbYToZM-b6HNi4pJdBoo_97usdEawhMGSK3bk,9560
 experimaestro/sphinx/static/experimaestro.css,sha256=0rEgt1LoDdD-a_R5rVfWZ19zD1gR-1L7q3f4UibIB58,294
 experimaestro/taskglobals.py,sha256=aBjPpo4HQp6E6M3GQ8L6PR4rK2Lu0kD5dS1WjnaGgDc,499
@@ -109,7 +109,7 @@ experimaestro/tests/tasks/all.py,sha256=hrI2CDyeaYrp2IPzXWif-Uu1Uirkndmuih3Jj09C
 experimaestro/tests/tasks/foreign.py,sha256=7IAF525mmMORxSPKQmU1z1B84XPmwsO8PGdxBvYknwU,153
 experimaestro/tests/test_checkers.py,sha256=Kg5frDNRE3pvWVmmYzyk0tJFNO885KOrK48lSu-NlYA,403
 experimaestro/tests/test_dependencies.py,sha256=xfWrSkvjT45G4FSCL535m1huLT2ghmyW7kvP_XvvCJQ,2005
-experimaestro/tests/test_findlauncher.py,sha256=twliDFFI75KPuUJE6EgJwDMf8u3ic1LAOyNOwufFtTw,2964
+experimaestro/tests/test_findlauncher.py,sha256=r34dci01FK9YpjtTHR9fIQ7AMHaMXQOrWtarytcf_Us,2967
 experimaestro/tests/test_forward.py,sha256=XkZ2iOPETVj-kbTyniOQU9gyHXdfvn89GTwpMq9J6qc,780
 experimaestro/tests/test_identifier.py,sha256=fnl7jCUAg86npRrS3yeXtb9JysSKhs5czgFzW9yJ9Q8,13397
 experimaestro/tests/test_instance.py,sha256=awIIMnhiec_qDO6jZBqWDR13ReTzh3arK_60QDY6TLQ,1540
@@ -141,8 +141,8 @@ experimaestro/utils/resources.py,sha256=gDjkrRjo7GULWyXmNXm_u1uqzEIAoAvJydICk56n
 experimaestro/utils/settings.py,sha256=jpFMqF0DLL4_P1xGal0zVR5cOrdD8O0Y2IOYvnRgN3k,793
 experimaestro/utils/yaml.py,sha256=jEjqXqUtJ333wNUdIc0o3LGvdsTQ9AKW9a9CCd-bmGU,6766
 experimaestro/xpmutils.py,sha256=S21eMbDYsHfvmZ1HmKpq5Pz5O-1HnCLYxKbyTBbASyQ,638
-experimaestro-1.5.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-experimaestro-1.5.0.dist-info/METADATA,sha256=Mf0BErVRpCxCp2hnBnbH42ZE1cuK56TUSCO-WG4xMBQ,6265
-experimaestro-1.5.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-experimaestro-1.5.0.dist-info/entry_points.txt,sha256=PhaEili_fDgn5q7rBJGip_uhGkRBq5l3Yuhg91zkcbk,574
-experimaestro-1.5.0.dist-info/RECORD,,
+experimaestro-1.5.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+experimaestro-1.5.1.dist-info/METADATA,sha256=A79XLTCvr5By9ybZMel3_0FZduD20ww4kOamF-4xeQA,6265
+experimaestro-1.5.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+experimaestro-1.5.1.dist-info/entry_points.txt,sha256=PhaEili_fDgn5q7rBJGip_uhGkRBq5l3Yuhg91zkcbk,574
+experimaestro-1.5.1.dist-info/RECORD,,