experimaestro 1.8.0rc1__py3-none-any.whl → 1.8.0rc3__py3-none-any.whl

experimaestro/core/callbacks.py

@@ -0,0 +1,52 @@
+from collections import defaultdict
+import threading
+from typing import Callable, ClassVar, Optional
+from experimaestro.core.objects import ConfigInformation
+from experimaestro.scheduler import Listener, Job, JobState, experiment
+
+
+class TaskEventListener(Listener):
+    INSTANCE: ClassVar[Optional["TaskEventListener"]] = None
+    """The general instance"""
+
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.experiments: set[int] = set()
+
+        self._on_completed: defaultdict[int, Callable] = defaultdict(list)
+
+    @staticmethod
+    def connect(xp: experiment):
+        _self = TaskEventListener.instance()
+        with _self.lock:
+            if id(xp) not in _self.experiments:
+                _self.experiments.add(id(xp))
+                xp.scheduler.addlistener(_self)
+
+    @staticmethod
+    def instance():
+        if TaskEventListener.INSTANCE is None:
+            TaskEventListener.INSTANCE = TaskEventListener()
+
+        return TaskEventListener.INSTANCE
+
+    def job_state(self, job: Job):
+        if job.state == JobState.DONE:
+            with self.lock:
+                for callback in self._on_completed.get(id(job.config.__xpm__), []):
+                    callback()
+
+    @staticmethod
+    def on_completed(
+        config_information: ConfigInformation, callback: Callable[[], None]
+    ):
+        instance = TaskEventListener.instance()
+
+        with instance.lock:
+            instance._on_completed[id(config_information)].append(callback)
+
+            if (
+                config_information.job is not None
+                and config_information.job == JobState.DONE
+            ):
+                callback()

experimaestro/core/objects.py

@@ -47,6 +47,7 @@ from experimaestro.core.types import DeprecatedAttribute, ObjectType
 from .context import SerializationContext, SerializedPath, SerializedPathLoader
 
 if TYPE_CHECKING:
+    from .callbacks import TaskEventListener
     from experimaestro.scheduler.base import Job
     from experimaestro.scheduler.workspace import RunMode
     from experimaestro.launchers import Launcher
@@ -607,6 +608,7 @@ class ConfigInformation:
 
         # State information
         self.job = None
+        self._job_listener: "TaskEventListener" | None = None
 
         #: True when this configuration was loaded from disk
         self.loaded = False
@@ -947,6 +949,15 @@ class ConfigInformation:
         if self.job:
             self.job.watch_output(watched)
 
+    def on_completed(self, callback: Callable[[], None]):
+        """Call a method when the task is completed successfully
+
+        :param callback: _description_
+        """
+        from .callbacks import TaskEventListener
+
+        TaskEventListener.on_completed(self, callback)
+
     def submit(
         self,
         workspace: "Workspace",
@@ -957,6 +968,7 @@ class ConfigInformation:
     ):
         from experimaestro.scheduler import experiment, JobContext
         from experimaestro.scheduler.workspace import RunMode
+        from .callbacks import TaskEventListener
 
         # --- Prepare the object
 
@@ -1009,6 +1021,7 @@ class ConfigInformation:
             workspace.run_mode if run_mode is None else run_mode
         ) or RunMode.NORMAL
         if run_mode == RunMode.NORMAL:
+            TaskEventListener.connect(experiment.CURRENT)
             other = experiment.CURRENT.submit(self.job)
             if other:
                 # Just returns the other task
@@ -2015,6 +2028,9 @@ class Task(LightweightTask):
         """
         self.__xpm__.watch_output(method, callback)
 
+    def on_completed(self, callback: Callable[[], None]):
+        self.__xpm__.on_completed(callback)
+
 
 # --- Utility functions
 

experimaestro/core/serialization.py

@@ -36,7 +36,7 @@ def state_dict(context: SerializationContext, obj: Any):
     :param context: The serialization context
     :param obj: the object to serialize
     """
-    objects = []
+    objects: list[Any] = []
     data = json_object(context, obj, objects)
     return {"objects": objects, "data": data}
 
@@ -50,11 +50,17 @@ def save_definition(obj: Any, context: SerializationContext, path: Path):
 def save(obj: Any, save_directory: Optional[Path]):
     """Saves an object into a disk file
 
+    The serialization process also stores in the given folder the different
+    files or folders that are registered as Path parameters (or
+    meta-parameters).
+
     :param save_directory: The directory in which the object and its data will
         be saved (by default, the object is saved in "definition.json")
     """
     context = SerializationContext(save_directory=save_directory)
-    save_definition(obj, context, save_directory / "definition.json")
+    save_definition(
+        obj, context, save_directory / "definition.json" if save_directory else None
+    )
 
 
 def get_data_loader(path: Union[str, Path, SerializedPathLoader]):
@@ -129,3 +135,47 @@ def from_task_dir(
     content["data"] = {"type": "python", "value": content["objects"][-1]["id"]}
 
     return from_state_dict(content, as_instance=as_instance)
+
+
+def serialize(
+    obj: Any, save_directory: Path, *, init_tasks: list["LightweightTask"] = []
+):
+    """Saves an object into a disk file, including initialization tasks
+
+    The serialization process also stores in the given folder the different
+    files or folders that are registered as Path parameters (or
+    meta-parameters).
+
+    :param save_directory: The directory in which the object and its data will
+        be saved (by default, the object is saved in "definition.json")
+    :param init_tasks: The optional
+    """
+    context = SerializationContext(save_directory=save_directory)
+    save_definition((obj, init_tasks), context, save_directory / "definition.json")
+
+
+def deserialize(
+    path: Union[str, Path, SerializedPathLoader],
+    as_instance: bool = False,
+) -> tuple[Any, List["LightweightTask"]] | Any:
+    """Load data from disk, and initialize the object
+
+    :param path: A directory or a function that transforms relative file path
+        into absolute ones
+    :param as_instance: returns instances instead of configuration objects
+    :returns: either the object (as_instance is true), or a tuple
+    """
+    data_loader = get_data_loader(path)
+
+    with data_loader("definition.json").open("rt") as fh:
+        content = json.load(fh)
+
+    object, init_tasks = from_state_dict(content, as_instance=as_instance)
+
+    if as_instance:
+        for init_task in init_tasks:
+            init_task.execute()
+
+        return object
+
+    return object, init_tasks

experimaestro/experiments/cli.py

@@ -1,3 +1,4 @@
+import datetime
 import importlib
 import inspect
 import json
@@ -292,7 +293,7 @@ def experiments_cli(  # noqa: C901
         sys.exit(0)
 
     # Move to an object container
-    configuration = OmegaConf.to_container(
+    xp_configuration: ConfigurationBase = OmegaConf.to_container(
         configuration, structured_config_mode=SCMode.INSTANTIATE
     )
 
@@ -301,11 +302,22 @@ def experiments_cli(  # noqa: C901
 
     workdir = ws_env.path
 
-    logging.info("Using working directory %s", str(workdir.resolve()))
+    # --- Sets up the experiment ID
 
     # --- Runs the experiment
+    if xp_configuration.add_timestamp:
+        timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M")
+        experiment_id = f"""{xp_configuration.id}-{timestamp}"""
+    else:
+        experiment_id = xp_configuration.id
+
+    logging.info(
+        "Running experiment %s working directory %s",
+        experiment_id,
+        str(workdir.resolve()),
+    )
     with experiment(
-        ws_env, configuration.id, host=host, port=port, run_mode=run_mode
+        ws_env, experiment_id, host=host, port=port, run_mode=run_mode
     ) as xp:
         # Set up the environment
         # (1) global settings (2) workspace settings and (3) command line settings
@@ -318,7 +330,7 @@ def experiments_cli(  # noqa: C901
         try:
             # Run the experiment
             helper.xp = xp
-            helper.run(list(args), configuration)
+            helper.run(list(args), xp_configuration)
 
             # ... and wait
             xp.wait()

experimaestro/experiments/configuration.py

@@ -51,3 +51,6 @@ class ConfigurationBase:
 
     description: str = ""
     """Description of the experiment"""
+
+    add_timestamp: bool = False
+    """Adds a timestamp YYYY_MM_DD-HH_MM to the experiment ID"""

experimaestro/scheduler/base.py

@@ -819,6 +819,7 @@ class experiment:
         """
 
         from experimaestro.server import Server
+        from experimaestro.scheduler import Listener
 
         settings = get_settings()
         if not isinstance(env, WorkspaceSettings):
@@ -835,6 +836,7 @@ class experiment:
         self.xplock = None
         self.old_experiment = None
         self.services: Dict[str, Service] = {}
+        self._job_listener: Optional[Listener] = None
 
         # Get configuration settings
 

experimaestro/tests/test_experiment.py

@@ -41,10 +41,33 @@ def test_experiment_history():
             task_a = TaskA().submit()
             TaskB(task_a=task_a, x=tag(1)).submit()
 
-    # Look at the experiment
-    xp = get_experiment("experiment", workdir=workdir)
+        # Look at the experiment
+        xp = get_experiment("experiment", workdir=workdir)
 
-    (task_a_info,) = xp.get_jobs(TaskA)
-    (task_b_info,) = xp.get_jobs(TaskB)
-    assert task_b_info.tags == {"x": 1}
-    assert task_b_info.depends_on == [task_a_info]
+        (task_a_info,) = xp.get_jobs(TaskA)
+        (task_b_info,) = xp.get_jobs(TaskB)
+        assert task_b_info.tags == {"x": 1}
+        assert task_b_info.depends_on == [task_a_info]
+
+
+class FlagHandler:
+    def __init__(self):
+        self.flag = False
+
+    def set(self):
+        self.flag = True
+
+    def is_set(self):
+        return self.flag
+
+
+def test_experiment_events():
+    """Test handlers"""
+
+    flag = FlagHandler()
+    with TemporaryExperiment("experiment"):
+        task_a = TaskA()
+        task_a.submit()
+        task_a.on_completed(flag.set)
+
+    assert flag.is_set()

{experimaestro-1.8.0rc1.dist-info → experimaestro-1.8.0rc3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.8.0rc1
+Version: 1.8.0rc3
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager

{experimaestro-1.8.0rc1.dist-info → experimaestro-1.8.0rc3.dist-info}/RECORD

@@ -13,17 +13,18 @@ experimaestro/connectors/local.py,sha256=348akOw69t7LgiTBMPG5McCg821I8qfR5GvYNU1
 experimaestro/connectors/ssh.py,sha256=5giqvv1y0QQKF-GI0IFUzI_Z5H8Bj9EuL_Szpvk899Q,8600
 experimaestro/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/core/arguments.py,sha256=7hpkU1f8LJ7JL8kQaD514h9CFSfMotYLsVfMsMmdpWk,6487
+experimaestro/core/callbacks.py,sha256=59JfeUgWcCCdIQ3pvh-xNnoRp9BX8f4iOAkgm16wBzE,1660
 experimaestro/core/context.py,sha256=41jvgudz4sgMDWrqOhPbgFRJHa3klWKvS3l_L661er0,2600
-experimaestro/core/objects.py,sha256=G0_PjcUPmbrQi8lxaxwdTq0dIw2cTCJUHduQ4XEbfFw,67362
+experimaestro/core/objects.py,sha256=yZjLsStxXU1mz21JEYAMfe0UQ5pHHbgweH3E8JS5uWI,67960
 experimaestro/core/objects.pyi,sha256=NElf7J-1rL2l9Td6fQofRj-UQTtt7d0tlO7NUyewqYI,7283
-experimaestro/core/serialization.py,sha256=vYFbeNSgc7Icl9wgmnjZvESCehpfvlweyI_QyldA_CQ,3969
+experimaestro/core/serialization.py,sha256=DB8MvWgE1d9iafgxzDWUG3nUqHjHabkR2o-0xiGSlOU,5674
 experimaestro/core/serializers.py,sha256=R_CAMyjjfU1oi-eHU6VlEUixJpFayGqEPaYu7VsD9xA,1197
 experimaestro/core/types.py,sha256=gSLv9F1HszVxI8jla6e-aVVS7q3KBwSzG1MImUHdGMg,21158
 experimaestro/core/utils.py,sha256=JfC3qGUS9b6FUHc2VxIYUI9ysNpXSQ1LjOBkjfZ8n7o,495
 experimaestro/exceptions.py,sha256=cUy83WHM3GeynxmMk6QRr5xsnpqUAdAoc-m3KQVrE2o,44
 experimaestro/experiments/__init__.py,sha256=GcpDUIbCvhnv6rxFdAp4wTffCVNTv-InY6fbQAlTy-o,159
-experimaestro/experiments/cli.py,sha256=iQrUjY6mmJpT-tj-Ek1QPbzNs27x4jsKjciCMgaTKZw,9722
-experimaestro/experiments/configuration.py,sha256=cFDiUHnUGblJsctAUxAqx0jlM7_Ja_527lzk-4G-44k,1368
+experimaestro/experiments/cli.py,sha256=4jTDzQWtrdm_ZgwdkVmz5WQX9RFVCqxZV5w1e7Yymhs,10085
+experimaestro/experiments/configuration.py,sha256=vVm40BJW5sZNgSDZ0oBzX15jTO9rmOewVYrm0k9iXXY,1466
 experimaestro/generators.py,sha256=DQsEgdMwRUud9suWr-QGxI3vCO5sywP6MVGZWRNQXkk,1372
 experimaestro/huggingface.py,sha256=gnVlr6SZnbutYz4PLH0Q77n1TRF-uk-dR-3UFzFqAY0,2956
 experimaestro/ipc.py,sha256=Xn3tYME83jLEB0nFak3DwEIhpL5IRZpCl3jirBF_jl4,1570
@@ -49,7 +50,7 @@ experimaestro/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/rpyc.py,sha256=ZRKol-3tVoeoUITLNFenLF4dhWBLW_FvSV_GvsypmeI,3605
 experimaestro/run.py,sha256=58ZlIZ2dQ7a0un2iGiyHJhK14zc18BnpEFDis7OyTPA,5222
 experimaestro/scheduler/__init__.py,sha256=ERmmOxz_9mUkIuccNbzUa5Y6gVLLVDdyc4cCxbCCUbY,20
-experimaestro/scheduler/base.py,sha256=9Vh6S3SVEr6VF_Zp3fCf8Jl_ygnEJPVElu4ghOuSapc,35702
+experimaestro/scheduler/base.py,sha256=jcAtlx-Za9TZBPJuOV9X-gQseDLMYn6VZff_Jcgy594,35809
 experimaestro/scheduler/dependencies.py,sha256=n9XegwrmjayOIxt3xhuTEBVEBGSq4oeVdzz-FviDGXo,1994
 experimaestro/scheduler/dynamic_outputs.py,sha256=yYPL98I0nSgDjlE3Sk9dtvovh2PZ6rUDnKjDNnAg1dc,5732
 experimaestro/scheduler/services.py,sha256=aCKkNZMULlceabqf-kOs_-C7KPINnjU3Q-I00o5x6iY,2189
@@ -117,7 +118,7 @@ experimaestro/tests/tasks/all.py,sha256=OMkHsWZkErCmTajiNO7hNhvnk9eKzJC-VatWgabW
 experimaestro/tests/tasks/foreign.py,sha256=uhXDOcWozkcm26ybbeEU9RElJpbhjC-zdzmlSKfPcdY,122
 experimaestro/tests/test_checkers.py,sha256=Kg5frDNRE3pvWVmmYzyk0tJFNO885KOrK48lSu-NlYA,403
 experimaestro/tests/test_dependencies.py,sha256=xfWrSkvjT45G4FSCL535m1huLT2ghmyW7kvP_XvvCJQ,2005
-experimaestro/tests/test_experiment.py,sha256=MXpVsWZpxINBianuTQ8vJ5iwmpIUq-GcS7YwSjLJUxM,1260
+experimaestro/tests/test_experiment.py,sha256=QWF9aHewL9hepagrKKFyfikKn3iiZ_lRRXl1LLttta0,1687
 experimaestro/tests/test_findlauncher.py,sha256=8tjpR8bLMi6Gjs7KpY2t64izZso6bmY7vIivMflm-Bc,2965
 experimaestro/tests/test_forward.py,sha256=9y1zYm7hT_Lx5citxnK7n20cMZ2WJbsaEeY5irCZ9U4,735
 experimaestro/tests/test_identifier.py,sha256=L-r0S0dI9ErOZSmqGOPbbUZBvWJ-uzc3sijoyTPyMYU,13680
@@ -149,8 +150,8 @@ experimaestro/utils/jupyter.py,sha256=JcEo2yQK7x3Cr1tNl5FqGMZOICxCv9DwMvL5xsWdQP
 experimaestro/utils/resources.py,sha256=j-nvsTFwmgENMoVGOD2Ap-UD3WU85WkI0IgeSszMCX4,1328
 experimaestro/utils/settings.py,sha256=jpFMqF0DLL4_P1xGal0zVR5cOrdD8O0Y2IOYvnRgN3k,793
 experimaestro/xpmutils.py,sha256=S21eMbDYsHfvmZ1HmKpq5Pz5O-1HnCLYxKbyTBbASyQ,638
-experimaestro-1.8.0rc1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-experimaestro-1.8.0rc1.dist-info/METADATA,sha256=GKxZQlL20q47h2TzK9Y2ej5ymYPVZ9qjeuXDGM_t6_w,6162
-experimaestro-1.8.0rc1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-experimaestro-1.8.0rc1.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
-experimaestro-1.8.0rc1.dist-info/RECORD,,
+experimaestro-1.8.0rc3.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+experimaestro-1.8.0rc3.dist-info/METADATA,sha256=cnI8ync_pALpUSppAouAA0Clj_ksBwdlMj1m4ZXo3qE,6162
+experimaestro-1.8.0rc3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+experimaestro-1.8.0rc3.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
+experimaestro-1.8.0rc3.dist-info/RECORD,,