experimaestro 1.5.6__py3-none-any.whl → 1.5.7__py3-none-any.whl

experimaestro/notifications.py

@@ -44,6 +44,12 @@ class LevelInformation:
         return f"[{self.level}] {self.desc} {int(self.progress*1000)/10}%"
 
 
+class ListenerInformation:
+    def __init__(self, url: str):
+        self.url = url
+        self.error_count = 0
+
+
 class Reporter(threading.Thread):
     NOTIFICATION_FOLDER = ".notifications"
 
@@ -59,7 +65,7 @@ class Reporter(threading.Thread):
         super().__init__(daemon=True)
         self.path = path / Reporter.NOTIFICATION_FOLDER
         self.path.mkdir(exist_ok=True)
-        self.urls: Dict[str, str] = {}
+        self.urls: Dict[str, ListenerInformation] = {}
 
         # Last check of notification URLs
         self.lastcheck = 0
@@ -80,7 +86,7 @@ class Reporter(threading.Thread):
             self.cv.notifyAll()
 
     @staticmethod
-    def isfatal_httperror(e: Exception) -> bool:
+    def isfatal_httperror(e: Exception, info: ListenerInformation) -> bool:
         """Returns True if this HTTP error indicates that the server won't recover"""
         if isinstance(e, HTTPError):
             if e.code >= 400 and e.code < 500:
@@ -90,6 +96,13 @@ class Reporter(threading.Thread):
                 return True
             if isinstance(e.reason, socket.gaierror) and e.reason.errno == -2:
                 return True
+            if isinstance(e.reason, TimeoutError):
+                info.error_count += 1
+
+            # Too many errors
+            if info.error_count > 3:
+                logger.info("Too many errors with %s", info.error_count)
+                return True
 
         return False
 
@@ -100,8 +113,8 @@ class Reporter(threading.Thread):
         mtime = os.path.getmtime(self.path)
         if mtime > self.lastcheck:
             for f in self.path.iterdir():
-                self.urls[f.name] = f.read_text().strip()
-                logger.info("Added new notification URL: %s", self.urls[f.name])
+                self.urls[f.name] = ListenerInformation(f.read_text().strip())
+                logger.info("Added new notification URL: %s", self.urls[f.name].url)
                 f.unlink()
 
             self.lastcheck = os.path.getmtime(self.path)
@@ -128,7 +141,9 @@ class Reporter(threading.Thread):
                         params = level.report()
 
                         # Go over all URLs
-                        for key, baseurl in self.urls.items():
+                        for key, info in self.urls.items():
+                            baseurl = info.url
+
                             url = "{}/progress?{}".format(
                                 baseurl, urllib.parse.urlencode(params)
                             )
@@ -147,7 +162,7 @@ class Reporter(threading.Thread):
                                     url,
                                     e,
                                 )
-                                if Reporter.isfatal_httperror(e):
+                                if Reporter.isfatal_httperror(e, info):
                                     toremove.append(key)
 
                 # Removes unvalid URLs
@@ -165,7 +180,8 @@ class Reporter(threading.Thread):
             self.check_urls()
             if self.urls:
                 # Go over all URLs
-                for key, baseurl in self.urls.items():
+                for key, info in self.urls.items():
+                    baseurl = info.url
                     url = "{}?status=eoj".format(baseurl)
                     try:
                         with urlopen(url) as _:
@@ -243,7 +259,7 @@ class xpm_tqdm(std_tqdm):
 
     def update(self, n=1):
         result = super().update(n)
-        if self.total is not None:
+        if self.total is not None and self.total > 0:
             progress(self.n / self.total, level=self.pos, console=False)
         return result
 

experimaestro/run.py

@@ -155,7 +155,6 @@ class TaskRunner:
                 self.donepath.touch()
 
                 # ... and finish the exit process
-                logger.info("This is the end (TODO: remove this line)")
                 raise
             else:
                 self.handle_error(e.code, None)

experimaestro/scheduler/base.py

@@ -803,11 +803,6 @@ class experiment:
             else None
         )
 
-        # Copy environment variable from main (but do not
-        # override)
-        for key, value in settings.env.items():
-            self.setenv(key, value, override=False)
-
         if os.environ.get("XPM_ENABLEFAULTHANDLER", "0") == "1":
             import faulthandler
 

experimaestro/settings.py

@@ -4,6 +4,7 @@ from dataclasses import field, dataclass
 from functools import lru_cache
 from pathlib import Path
 from typing import Dict, Optional, List
+import logging
 
 
 @dataclass
@@ -59,7 +60,9 @@ def get_settings(path: Optional[Path] = None) -> Settings:
 
         path = path or Path("~/.config/experimaestro/settings.yaml").expanduser()
         if not path.is_file():
-            return schema.to_object()
+            return OmegaConf.to_container(
+                schema, structured_config_mode=SCMode.INSTANTIATE
+            )
 
         conf = OmegaConf.load(path)
         return OmegaConf.to_container(
@@ -78,3 +81,28 @@ def get_workspace(id: Optional[str] = None) -> Optional[WorkspaceSettings]:
                 return workspace
 
     return None
+
+
+def find_workspace(*, workspace: Optional[str] = None, workdir: Optional[Path] = None):
+    """Find workspace"""
+    workdir = Path(workdir) if workdir else None
+
+    if workspace:
+        ws_env = get_workspace(workspace)
+        if ws_env is None:
+            raise RuntimeError("No workspace named %s", workspace)
+
+        logging.info("Using workspace %s", ws_env.id)
+        if workdir:
+            # Overrides working directory
+            logging.info(" override working directory: %s", workdir)
+            ws_env.path = workdir
+    elif workdir:
+        logging.info("Using workdir %s", workdir)
+        ws_env = WorkspaceSettings("", workdir)
+    else:
+        ws_env = get_workspace()
+        assert ws_env is not None, "No workdir or workspace defined, and no default"
+        logging.info("Using default workspace %s", ws_env.id)
+
+    return ws_env

experimaestro/tests/test_tags.py

@@ -2,6 +2,7 @@ from typing import Dict
 from pathlib import Path
 from experimaestro import (
     tag,
+    LightweightTask,
     config,
     argument,
     Config,
@@ -69,6 +70,40 @@ def test_inneroutput():
     assert evaluate.__xpm__.tags() == {"hello": "world"}
 
 
+def test_tags_init_tasks():
+    """Test tags within init tasks"""
+
+    class MyTask(Task):
+        pass
+
+    class InitTask(LightweightTask):
+        pass
+
+    class MyConfig(Config):
+        pass
+
+    class TaskWithOutput(Task):
+        x: Param[MyConfig]
+
+        def task_outputs(self, dep) -> MyConfig:
+            return dep(MyConfig())
+
+    init_task = InitTask().tag("hello", "world")
+    task = MyTask()
+    result = task.submit(run_mode=RunMode.DRY_RUN, init_tasks=[init_task])
+    assert result.tags() == {"hello": "world"}
+
+    other_task = TaskWithOutput(x=MyConfig().tag("hello", "world"))
+    assert other_task.tags() == {"hello": "world"}
+
+    result = other_task.submit(run_mode=RunMode.DRY_RUN)
+    assert isinstance(result, MyConfig)
+    assert result.tags() == {"hello": "world"}
+
+    result = MyTask().submit(run_mode=RunMode.DRY_RUN, init_tasks=[result])
+    assert result.tags() == {"hello": "world"}
+
+
 class TaskDirectoryContext(DirectoryContext):
     def __init__(self, task, path):
         super().__init__(path)

{experimaestro-1.5.6.dist-info → experimaestro-1.5.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: experimaestro
-Version: 1.5.6
+Version: 1.5.7
 Summary: "Experimaestro is a computer science experiment manager"
 Home-page: https://github.com/experimaestro/experimaestro-python
 License: GPL-3

{experimaestro-1.5.6.dist-info → experimaestro-1.5.7.dist-info}/RECORD

@@ -1,7 +1,10 @@
 experimaestro/__init__.py,sha256=btjhjJa9TofgAYpMEw8SLpVH6G-5kSelxsInQdrIu30,1501
-experimaestro/__main__.py,sha256=iRi6yRJVcthXRTVnHQB0TNOKTRmKjdnX6yIQaf-z79o,13528
+experimaestro/__main__.py,sha256=Dv9lFl03yt1dswd0Xb9NIJRgHpA5_IwH4RfQPEHyFz0,158
 experimaestro/annotations.py,sha256=dcpFmo01T12S_y5nIBIQjiXsGsq5S80ZB-58o8tW9wA,8450
 experimaestro/checkers.py,sha256=ZCMbnE_GFC5compWjt-fuHhPImi9fCPjImF8Ow9NqK8,696
+experimaestro/cli/__init__.py,sha256=mzc-qqTFtZnFwCQl7IiwlYXEx08kLGwdntWayCerZ6E,9610
+experimaestro/cli/filter.py,sha256=0jJrD_2cWydovjLO32vTFTK-TxXSs9P8Zxp5WaBF5AE,5790
+experimaestro/cli/jobs.py,sha256=AvSrmoO8UdTDZkPxXT2iHdpcjde7CXQGs7oow0cFqY0,7319
 experimaestro/click.py,sha256=6BkeQHEgcxaxzq3xEvEEzwzuBj5-dkfrpOGikuA8L00,1377
 experimaestro/commandline.py,sha256=NS1ubme8DTJtDS2uWwdHLQiZsl6TSK1LkNxu39c3-cw,9463
 experimaestro/compat.py,sha256=dQqE2ZNHLM2wtdfp7fBRYMfC33qNehVf9J6FGRBUQhs,171
@@ -11,7 +14,7 @@ experimaestro/connectors/ssh.py,sha256=hmvU6bCq6ZsB1Zjz273mzb9pdZdTinUhUqZFJTZl8
 experimaestro/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/core/arguments.py,sha256=dW32opqNEsULYr6nR7Zk8PqHsSCbLPclfXofw27GTpI,5620
 experimaestro/core/context.py,sha256=Q8_ngiHRBZ0laavXRJNiDvdCprrnROVTWaHfrwMdlG4,2638
-experimaestro/core/objects.py,sha256=s9dt7U8LDT9XNVY7mowdt1DvuBH19EO4obips1ZBSHg,63989
+experimaestro/core/objects.py,sha256=vTp_zjBJlYEhe0u55erJ5X-Lc8UuTb45NIFwfVZCPes,64142
 experimaestro/core/objects.pyi,sha256=Adi2OKCW0-B9GROlEUgxxBDK6SHUqccTBHTofRRJE9M,7131
 experimaestro/core/serialization.py,sha256=9tg5ebLF3YeZ_zG9DiTHPLthppvo7io710ohD_dcLTo,3836
 experimaestro/core/serializers.py,sha256=R_CAMyjjfU1oi-eHU6VlEUixJpFayGqEPaYu7VsD9xA,1197
@@ -19,9 +22,8 @@ experimaestro/core/types.py,sha256=oTXD4UjMVoYn_Usxn2C4h6IGhYDTtekKB3O3hfeOynQ,2
 experimaestro/core/utils.py,sha256=JfC3qGUS9b6FUHc2VxIYUI9ysNpXSQ1LjOBkjfZ8n7o,495
 experimaestro/exceptions.py,sha256=cUy83WHM3GeynxmMk6QRr5xsnpqUAdAoc-m3KQVrE2o,44
 experimaestro/experiments/__init__.py,sha256=GcpDUIbCvhnv6rxFdAp4wTffCVNTv-InY6fbQAlTy-o,159
-experimaestro/experiments/cli.py,sha256=5G78YkzZtNU6ygcp_pRJQXYvsMWlguB9_M35-8W_Ors,7916
+experimaestro/experiments/cli.py,sha256=dr-LoNNb7Nh4HCBBh4SzH5QRHXjh5tVppCjchZdZ5Hk,7254
 experimaestro/experiments/configuration.py,sha256=8GRqyLG1leF_NbvbFzqpm0yM24O0WjSNmQzvnuLnxxw,1150
-experimaestro/filter.py,sha256=DN1PrmS9yXoOa5Xnv001zbxzpdzvcVZFI9xZFKZ1-6g,5794
 experimaestro/generators.py,sha256=9NQ_TfDfASkArLnO4PF7s5Yoo9KWjlna2DCPzk5gJOI,1230
 experimaestro/huggingface.py,sha256=gnVlr6SZnbutYz4PLH0Q77n1TRF-uk-dR-3UFzFqAY0,2956
 experimaestro/ipc.py,sha256=ltYqybPm_XfcQC3yiskMfhfI_1dREs-XRu0F83YsNws,1490
@@ -34,7 +36,7 @@ experimaestro/launchers/__init__.py,sha256=lXn544sgJExr6uirILWzAXu_IfmfyqFZOt4Oz
 experimaestro/launchers/direct.py,sha256=VJzQNrUGnh-1Ovt6uw4yYIjXNu45QpR-_6V45lcZAfQ,1967
 experimaestro/launchers/oar.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/launchers/slurm/__init__.py,sha256=R1Zwd4phZaXV8FwCYhzfB44n0V4cf-hBQzOc6NkFQ0s,41
-experimaestro/launchers/slurm/base.py,sha256=nMoSBMbA901OcneSHVXj8PXGqfv4mVN5G-NPDZ0HXO0,14135
+experimaestro/launchers/slurm/base.py,sha256=syIC8Frfz-kWhpYkmPTfMjrskx7_bzvmRMQJ40lkF50,14185
 experimaestro/launchers/slurm/cli.py,sha256=c-S0TImvhZ-ZxFs5-5T2GRDm5manRYRiYudpQLHwsrQ,818
 experimaestro/launchers/slurm/configuration.py,sha256=mtozeuvIZmEfHlvEylwCgBrlVRFHT_jWNAKVxR4Tz1E,19357
 experimaestro/locking.py,sha256=hPT-LuDGZTijpbme8O0kEoB9a3WjdVzI2h31OT44UxE,1477
@@ -44,12 +46,12 @@ experimaestro/mkdocs/base.py,sha256=SwLh9s7BZfrTAZdBaealSqVeLAroDSwLLMOHmLCxMPQ,
 experimaestro/mkdocs/metaloader.py,sha256=qCqnTWhlgxql-oe46E8AbvYdoM311-lQh-msmPnbllQ,1481
 experimaestro/mkdocs/style.css,sha256=42kJ6Ozq_n4Iw5UfJ4-nO1u-HN3ELvV7Vhvj1Xkn7rQ,66
 experimaestro/mypy.py,sha256=M39VFuDrab-ymlCDIF5jys9oKpTwnuBPzb1T8Un5J3s,285
-experimaestro/notifications.py,sha256=936tdH_KYJYBvPVu-ii4Sn_nso_TTs5GqAfO1FDYRgc,8710
+experimaestro/notifications.py,sha256=AiRFoew16Z8E0Pz9dswodOfkY5mFc5zglVPH6LqgQE4,9257
 experimaestro/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/rpyc.py,sha256=ZRKol-3tVoeoUITLNFenLF4dhWBLW_FvSV_GvsypmeI,3605
-experimaestro/run.py,sha256=pRTj7R7N0Tzm1lAjeWyn-ewM0kNFmpU0so8Yhv0BIYs,5294
+experimaestro/run.py,sha256=NTFORDb_RlEK6tWKa7K-_2_bGCdHzzjBJVH5C1ReYtw,5222
 experimaestro/scheduler/__init__.py,sha256=ERmmOxz_9mUkIuccNbzUa5Y6gVLLVDdyc4cCxbCCUbY,20
-experimaestro/scheduler/base.py,sha256=8fwQEBkstnBFtioJ33xD5O0iJQTNltzA2RniSTi0f-4,32120
+experimaestro/scheduler/base.py,sha256=jXjep3VS5cRCjCj6vCxp4t1fGVyqQW2JQNnLp6HKmLQ,31941
 experimaestro/scheduler/dependencies.py,sha256=n9XegwrmjayOIxt3xhuTEBVEBGSq4oeVdzz-FviDGXo,1994
 experimaestro/scheduler/services.py,sha256=aCKkNZMULlceabqf-kOs_-C7KPINnjU3Q-I00o5x6iY,2189
 experimaestro/scheduler/workspace.py,sha256=vyVbYZML28zvmgxfWc2PsKKHGlrAejY43UYlttbm9AU,2280
@@ -79,7 +81,7 @@ experimaestro/server/data/index.js,sha256=f0GvRsfsQ4ayP4en7Q-raZ6buwRXLCswCbzVax
 experimaestro/server/data/index.js.map,sha256=za3MUIjzyyGRI6F5KuBFMTgrFU55xgt0LBrw-4YPHag,3904832
 experimaestro/server/data/login.html,sha256=4dvhSOn6DHp_tbmzqIKrqq2uAo0sAUbgLVD0lTnPp4s,511
 experimaestro/server/data/manifest.json,sha256=EpzHQZzrGh9c1Kf63nrqvI33H1cm0nLYfdh5lDm8ijI,318
-experimaestro/settings.py,sha256=Sh-OefrMEtuJJA9tRe8TiS9-BIy0b41HmHXkf1cLLzM,2181
+experimaestro/settings.py,sha256=UsfKIA4Jx-32hm3xGjBs_b9uvw8M7sOYtzFyUsYSJ8s,3145
 experimaestro/sphinx/__init__.py,sha256=heovvtwbYToZM-b6HNi4pJdBoo_97usdEawhMGSK3bk,9560
 experimaestro/sphinx/static/experimaestro.css,sha256=0rEgt1LoDdD-a_R5rVfWZ19zD1gR-1L7q3f4UibIB58,294
 experimaestro/taskglobals.py,sha256=aBjPpo4HQp6E6M3GQ8L6PR4rK2Lu0kD5dS1WjnaGgDc,499
@@ -119,7 +121,7 @@ experimaestro/tests/test_progress.py,sha256=wtIGQzlV3ldd_wMng11LinVESchW-1J954mC
 experimaestro/tests/test_serializers.py,sha256=xSCezAM9yH_Ix1wr7j0au9SyBv9DtZ7b0zs2-Ynt-VM,2338
 experimaestro/tests/test_snippets.py,sha256=rojnyDjtmAMnSuDUj6Bv9XEgdP8oQf2nVc132JF8vsM,3081
 experimaestro/tests/test_ssh.py,sha256=JhwsS4lJWQeMhtnDfJhQqJ5dwEThqvcNBBgUq1EWvk0,979
-experimaestro/tests/test_tags.py,sha256=kanKx0TOpS7vNs5jOs-J8IuZjyUJ9zMJl2V8dfOh4Lo,1975
+experimaestro/tests/test_tags.py,sha256=vfW99iFfw3m-pcJPy_-mtZsWbAt_Xw5k-u3dWoJbRWw,2921
 experimaestro/tests/test_tasks.py,sha256=bUSB_UT1MTN2P_RPHd4AT5NK-DFsgCVeFKSiXu3bEz8,9429
 experimaestro/tests/test_tokens.py,sha256=cW9qQU4PhbQY4_SgK8ICmKcYq8JVvLRTOYZzdtoS5N8,7826
 experimaestro/tests/test_types.py,sha256=2BbdB4o_2HvFkyhiQ2X3oHM49xcck-phT873NF-p8lA,1257
@@ -140,8 +142,8 @@ experimaestro/utils/resources.py,sha256=gDjkrRjo7GULWyXmNXm_u1uqzEIAoAvJydICk56n
 experimaestro/utils/settings.py,sha256=jpFMqF0DLL4_P1xGal0zVR5cOrdD8O0Y2IOYvnRgN3k,793
 experimaestro/utils/yaml.py,sha256=jEjqXqUtJ333wNUdIc0o3LGvdsTQ9AKW9a9CCd-bmGU,6766
 experimaestro/xpmutils.py,sha256=S21eMbDYsHfvmZ1HmKpq5Pz5O-1HnCLYxKbyTBbASyQ,638
-experimaestro-1.5.6.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-experimaestro-1.5.6.dist-info/METADATA,sha256=X50UbV1oT6bEWSCm2wrLxPcIi-wO90GUC2zd8eg8JLY,6265
-experimaestro-1.5.6.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-experimaestro-1.5.6.dist-info/entry_points.txt,sha256=PhaEili_fDgn5q7rBJGip_uhGkRBq5l3Yuhg91zkcbk,574
-experimaestro-1.5.6.dist-info/RECORD,,
+experimaestro-1.5.7.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+experimaestro-1.5.7.dist-info/METADATA,sha256=setEZ-Qqj09KStW2IrXbenMJzVOE6eR6gPhmldyEW14,6265
+experimaestro-1.5.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+experimaestro-1.5.7.dist-info/entry_points.txt,sha256=PhaEili_fDgn5q7rBJGip_uhGkRBq5l3Yuhg91zkcbk,574
+experimaestro-1.5.7.dist-info/RECORD,,