experimaestro 0.23.0__py2.py3-none-any.whl → 0.24.0__py2.py3-none-any.whl

experimaestro/__init__.py

@@ -44,11 +44,11 @@ from .core.objects import (
     Config,
     copyconfig,
     setmeta,
-    TaskOutput,
+    ConfigWrapper,
+    ConfigWrapper as TaskOutput,  # maintains compatibility
     Task,
-    SerializedConfig,
-    Serialized,
 )
+from .core.serializers import SerializedConfig, PathBasedSerializedConfig
 from .core.types import Any, SubmitHook, submit_hook_decorator
 from .launchers import Launcher
 from .scheduler.environment import Environment

experimaestro/__main__.py

@@ -12,7 +12,7 @@ import subprocess
 from termcolor import colored, cprint
 
 import experimaestro
-from experimaestro.core.objects import TaskOutput
+from experimaestro.core.objects import ConfigWrapper
 
 # --- Command line main options
 logging.basicConfig(level=logging.INFO)
@@ -142,7 +142,7 @@ def diff(path: Path):
             if new_id != old_id:
                 print(f"{path} differ: {new_id} vs {old_id}")
 
-                if isinstance(value, TaskOutput):
+                if isinstance(value, ConfigWrapper):
                     check(f"{path}.<task>", value.__xpm__.task, new_value.__xpm__.task)
                 else:
                     for arg in value.__xpmtype__.arguments.values():

experimaestro/core/objects.py

@@ -2,6 +2,12 @@
 
 from functools import cached_property
 import json
+
+try:
+    from types import NoneType
+except Exception:
+    # compatibility: python-3.8
+    NoneType = type(None)
 from termcolor import cprint
 import os
 from pathlib import Path
@@ -14,7 +20,6 @@ import inspect
 import importlib
 from typing import (
     Any,
-    Callable,
     ClassVar,
     Dict,
     List,
@@ -169,7 +174,7 @@ class HashComputer:
             for key, value in items:
                 self.update(key, subparam=subparam)
                 self.update(value, subparam=subparam)
-        elif isinstance(value, TaskOutput):
+        elif isinstance(value, ConfigWrapper):
             # Add the task ID...
             self.update(value.__xpm__.task, subparam=subparam)
 
@@ -278,8 +283,14 @@ def updatedependencies(
     elif isinstance(value, (list, set)):
         for el in value:
             updatedependencies(dependencies, el, path, taskids)
-    elif isinstance(value, TaskOutput):
-        dependencies.add(value.__xpm__.task.__xpm__.dependency())
+    elif isinstance(value, ConfigWrapper):
+        # Add the base value (if any)
+        if value.__xpm__.base is not None:
+            value.__xpm__.base.__xpm__.updatedependencies(dependencies, path, taskids)
+
+        # Add the task (if any)
+        if value.__xpm__.task is not None:
+            dependencies.add(value.__xpm__.task.__xpm__.dependency())
     elif isinstance(value, (dict,)):
         for key, val in value.items():
             updatedependencies(dependencies, key, path, taskids)
@@ -419,9 +430,13 @@ class ConfigWalk:
                     result[key] = self(value)
             return result
 
-        if isinstance(x, TaskOutput):
+        if isinstance(x, ConfigWrapper):
             # Process task if different
-            if self.recurse_task and x.__xpm__.task is not x.__unwrap__():
+            if (
+                x.__xpm__.task is not None
+                and self.recurse_task
+                and x.__xpm__.task is not x.__unwrap__()
+            ):
                 self(x.__xpm__.task)
 
             # Processed the wrapped config
@@ -430,6 +445,9 @@ class ConfigWalk:
         if isinstance(x, (float, int, str, Path, Enum)):
             return x
 
+        if isinstance(x, Proxy):
+            return self(x.__unwrap__())
+
         raise NotImplementedError(f"Cannot handle a value of type {type(x)}")
 
 
@@ -448,7 +466,7 @@ class ConfigInformation:
     """Forces this configuration to be a meta-parameter"""
 
     # Set to true when loading from JSON
-    LOADING = False
+    LOADING: ClassVar[bool] = False
 
     def __init__(self, pyobject: "TypeConfig"):
         # The underlying pyobject and XPM type
@@ -477,6 +495,7 @@ class ConfigInformation:
         self._meta = None
 
     def set_meta(self, value: Optional[bool]):
+        """Sets the meta flag"""
         assert not self._sealed, "Configuration is sealed"
         self._meta = value
 
@@ -672,12 +691,17 @@ class ConfigInformation:
             hook(job)
 
     def submit(
-        self, workspace: "Workspace", launcher: "Launcher", run_mode=None
-    ) -> "TaskOutput":
+        self, workspace: "Workspace", launcher: "Launcher", *, run_mode=None, pre=None
+    ) -> "ConfigWrapper":
         from experimaestro.scheduler import experiment, JobContext
         from experimaestro.scheduler.workspace import RunMode
 
+        # --- Handle default values
+
+        pre = pre or []
+
         # --- Prepare the object
+
         if self.job:
             raise Exception("task %s was already submitted" % self)
         if not self.xpmtype.task:
@@ -685,6 +709,7 @@ class ConfigInformation:
 
         # --- Submit the job
 
+        # Creates a new job
         self.job = self.xpmtype.task(
             self.pyobject, launcher=launcher, workspace=workspace, run_mode=run_mode
         )
@@ -759,16 +784,20 @@ class ConfigInformation:
                 hints = get_type_hints(self.pyobject.config)
                 config = hints["return"](**config)
             config.__xpm__.validate()
-            self._taskoutput = TaskOutput(config, self.pyobject)
+            self._taskoutput = ConfigWrapper.__create_taskoutput__(
+                config, self.pyobject
+            )
 
         # New way to handle outputs
         elif hasattr(self.pyobject, "taskoutputs"):
             value = self.pyobject.taskoutputs()
-            self._taskoutput = TaskOutput(value, self.pyobject)
+            self._taskoutput = ConfigWrapper.__create_taskoutput__(value, self.pyobject)
 
         # Otherwise, the output is just the config
         else:
-            self._taskoutput = TaskOutput(self.pyobject, self.pyobject)
+            self._taskoutput = ConfigWrapper.__create_taskoutput__(
+                self.pyobject, self.pyobject
+            )
 
         return self._taskoutput
 
@@ -810,20 +839,13 @@ class ConfigInformation:
                 "value": value.name,
             }
 
-        elif isinstance(value, SerializedTaskOutput):
-            # Reference to a serialized object
-            return {
-                "type": "serialized",
-                "value": id(value.__xpm__.serialized.loader),
-                "path": [c.toJSON() for c in value.__xpm__.path],
-            }
-
-        elif isinstance(value, TaskOutput):
+        elif isinstance(value, ConfigWrapper):
             return {
                 "type": "python",
                 "value": id(value.__unwrap__()),
                 # We add the task for identifier computation
                 "task": id(value.__xpm__.task),
+                "path": [c.toJSON() for c in value.__xpm__.path or []],
             }
 
         elif isinstance(value, Config):
@@ -892,25 +914,18 @@ class ConfigInformation:
     def __collect_objects__(value, objects: List[Dict], context: SerializationContext):
         """Serialize all needed configuration objects, looking at sub
         configurations if necessary"""
-        # objects
-        if isinstance(value, SerializedTaskOutput):
-            loader = value.__xpm__.serialized.loader
-            if id(loader) not in context.serialized:
-                objects.append(
-                    {
-                        "id": id(loader),
-                        "serialized": True,
-                        "module": loader.__class__.__module__,
-                        "type": loader.__class__.__qualname__,
-                        "value": loader.toJSON(),
-                    }
-                )
-            return
-
         # Unwrap if needed
-        if isinstance(value, TaskOutput):
+        if isinstance(value, ConfigWrapper):
             # We will need to output the task configuration objects
-            ConfigInformation.__collect_objects__(value.__xpm__.task, objects, context)
+            if value.__xpm__.task is not None:
+                ConfigInformation.__collect_objects__(
+                    value.__xpm__.task, objects, context
+                )
+
+            if value.__xpm__.base is not None:
+                ConfigInformation.__collect_objects__(
+                    value.__xpm__.base, objects, context
+                )
 
             # Unwrap the value to output it
             value = value.__unwrap__()
@@ -1040,7 +1055,7 @@ class ConfigInformation:
                 if not as_instance:
                     if task_id := value.get("task", None):
                         task = objects[task_id]
-                        return TaskOutput(obj, task)
+                        return ConfigWrapper.__create_taskoutput__(obj, task)
                 return obj
 
             if value["type"] == "serialized":
@@ -1102,6 +1117,7 @@ class ConfigInformation:
         o = None
         objects = {}
         import experimaestro.taskglobals as taskglobals
+        from .serializers import SerializedConfig
 
         for definition in definitions:
             module_name = definition["module"]
@@ -1184,7 +1200,10 @@ class ConfigInformation:
                             assert isinstance(v, Path), "Excepted Path, got {type(v)}"
 
                     if as_instance:
+                        # Unwrap the value if needed
+                        v = unwrap(v)
                         setattr(o, name, v)
+
                         assert (
                             getattr(o, name) is v
                         ), f"Problem with deserialization {name} of {o.__class__}"
@@ -1194,6 +1213,8 @@ class ConfigInformation:
                 if as_instance:
                     # Calls post-init
                     o.__post_init__()
+                    if isinstance(o, SerializedConfig):
+                        o.initialize()
                 else:
                     # Seal and set the identifier
                     if not discard_id:
@@ -1247,6 +1268,13 @@ class ConfigInformation:
             # Call __post_init__
             o.__post_init__()
 
+            # Process a serialized configuration
+            from .serializers import SerializedConfig
+
+            if isinstance(o, SerializedConfig):
+                o.initialize()
+                o = o.__unwrap__()
+
             return o
 
     def fromConfig(self, context: ConfigWalkContext):
@@ -1406,15 +1434,18 @@ class TypeConfig:
             ), f"{context.__class__} is not an instance of ConfigWalkContext"
         return self.__xpm__.fromConfig(context)  # type: ignore
 
-    def submit(self, *, workspace=None, launcher=None, run_mode: "RunMode" = None):
+    def submit(
+        self, *, workspace=None, launcher=None, run_mode: "RunMode" = None, pre=None
+    ):
         """Submit this task
 
         :param workspace: the workspace, defaults to None
         :param launcher: The launcher, defaults to None
         :param run_mode: Run mode (if None, uses the workspace default)
-        :return: a :py:class:TaskOutput object
+        :param pre: Pre-tasks to execute before
+        :return: a :py:class:ConfigWrapper object
         """
-        return self.__xpm__.submit(workspace, launcher, run_mode=run_mode)
+        return self.__xpm__.submit(workspace, launcher, run_mode=run_mode, pre=pre)
 
     def stdout(self):
         return self.__xpm__.job.stdout
@@ -1515,9 +1546,6 @@ class Task(Config):
         raise NotImplementedError()
 
 
-# --- Output proxy
-
-
 class Proxy:
     """A proxy for a value"""
 
@@ -1525,7 +1553,29 @@ class Proxy:
         raise NotImplementedError()
 
 
+def unwrap(v: Any):
+    """Unwrap all proxies"""
+    while isinstance(v, Proxy):
+        v = v.__unwrap__()
+    return v
+
+
+class AttrAccessor:
+    """Access an attribute"""
+
+    def __init__(self, key: str):
+        self.key = key
+
+    def get(self, value):
+        return getattr(value, self.key)
+
+    def toJSON(self):
+        return {"type": "attr", "name": self.key}
+
+
 class ItemAccessor:
+    """Access an array item"""
+
     def __init__(self, key: Any):
         self.key = key
 
@@ -1536,49 +1586,65 @@ class ItemAccessor:
         return value.__getitem__(self.key)
 
 
-class AttrAccessor:
-    def __init__(self, key: Any, default: Any):
-        self.key = key
-        self.default = default
-
-    def get(self, value):
-        return getattr(value, self.key, self.default)
-
-    def toJSON(self):
-        return {"type": "attr", "name": self.key}
+class ConfigWrapperInfo:
+    """Global information about the Configuration wrapper"""
 
+    def __init__(
+        self,
+        value: Any,
+        *,
+        task: Optional[Task] = None,
+        parent: Optional["ConfigWrapperInfo"] = None,
+        base: Optional[Config] = None,
+        path: Optional[List[Path]] = None,
+    ):
+        # Current value
+        self.value = value
+        self.parent = parent
 
-class Serialized:
-    """Simple serialization object"""
+        # The task
+        if isinstance(value, Task):
+            self.task = value
+        else:
+            self.task = task
 
-    def __init__(self, value):
-        self.value = value
+        # Holds serialized config information
+        from .serializers import SerializedConfig
 
-    def toJSON(self):
-        return self.value
+        if isinstance(value, SerializedConfig):
+            self.base = value
+            self.value = value.config
+            self.path = [AttrAccessor("config")]
+        else:
+            self.base = base
+            self.path = path
 
+    def __state_dict__(self):
+        return {"task": self.task, "base": self.base, "path": self.path}
 
-class SerializedConfig:
-    """A serializable configuration
+    def __getitem__(self, key: Any):
+        kwargs = self.__state_dict__()
+        value = self.value.__getitem__(key)
 
-    This can be used to define a loading mechanism when instanciating the
-    configuration
-    """
+        if self.path:
+            kwargs["path"].append(ItemAccessor(key))
 
-    pyobject: Config
-    """The configuration that will be serialized"""
+        return ConfigWrapperInfo(value, **kwargs)
 
-    def __init__(self, pyobject: Config, loader: Callable[[Path], Config]):
-        self.pyobject = pyobject
-        self.loader = loader
+    def __getattr__(self, key: str, *default) -> Any:
+        kwargs = self.__state_dict__()
+        value = getattr(self.value, key, *default)
+        if self.path:
+            kwargs["path"].append(AttrAccessor(key, *default))
+        return ConfigWrapperInfo(value, parent=self, **kwargs)
 
+    def wrap(self):
+        """Wrap a value if needed"""
+        if isinstance(self.value, (str, int, float, Path, bool, NoneType)):
+            return self.value
 
-class TaskOutputInfo:
-    def __init__(self, task: Task):
-        self.task = task
-        self.value = None
-        self.path = None
-        self.serialized = None
+        # Returns a new config wrapper
+        return ConfigWrapper(self)
 
     @property
     def identifier(self):
@@ -1589,92 +1655,64 @@ class TaskOutputInfo:
         return self.task.__xpm__.job
 
     def tags(self):
+        """Returns the tags of the task"""
         tags = self.task.__xpm__.tags()
         return tags
 
     def stdout(self):
+        """Returns the standard output of the associated task"""
         return self.task.__xpm__.job.stdout
 
     def stderr(self):
+        """Returns the standard error of the associated task"""
         return self.task.__xpm__.job.stderr
 
     def wait(self):
+        """Wait for the task to end
+
+        :return: True if the task completed without error
+        """
         from experimaestro.scheduler import JobState
 
         return self.task.__xpm__.job.wait() == JobState.DONE
 
 
-class TaskOutput(Proxy):
+# Cleanup into just one
+class ConfigWrapper(Proxy):
     """Task proxy
 
     This is used when accessing properties *after* having submitted a task,
-    to keep track of the dependencies
+    to keep track of the dependencies, and/or as an accessor when dealing with
+    a serialized config
     """
 
-    def __init__(self, value: Any, task: Union[Task, TaskOutputInfo]):
-        self.__xpm__ = (
-            task if isinstance(task, TaskOutputInfo) else TaskOutputInfo(task)
-        )
-        self.__xpm__.value = value
-
-    def _wrap(self, value):
-        if isinstance(value, SerializedConfig):
-            return SerializedTaskOutput(value.pyobject, value, self.__xpm__.task, [])
-
-        if isinstance(value, (str, int, float, Path, bool)):
-            # No need to wrap if direct
-            return value
+    def __init__(self, info: ConfigWrapperInfo):
+        self.__xpm__ = info
 
-        return TaskOutput(value, self.__xpm__.task)
+    @staticmethod
+    def __create_taskoutput__(value: Any, task: Task):
+        return ConfigWrapperInfo(value, task=task).wrap()
 
     def __getitem__(self, key: Any):
-        return self._wrap(self.__xpm__.value.__getitem__(key))
+        return self.__xpm__[key].wrap()
 
-    def __getattr__(self, key: str, default=None) -> Any:
-        return self._wrap(getattr(self.__xpm__.value, key, default))
+    def __getattr__(self, key: str, *default) -> Any:
+        return self.__xpm__.__getattr__(key, *default).wrap()
 
     def __unwrap__(self):
         return self.__xpm__.value
 
     def __call__(self, *args, **kwargs):
         assert callable(self.__xpm__.value), "Attribute is not a function"
-        __self__ = TaskOutput(self.__xpm__.value.__self__, self.__xpm__.task)
-        return self.__xpm__.value.__func__(__self__, *args, **kwargs)
-
 
-class SerializedTaskOutput(TaskOutput):
-    """Used when serializing a configuration
-
-    Here, we need to keep track of the path to the value we need
-    """
-
-    def __init__(
-        self, value, serialized: SerializedConfig, task: Task, path: List[Any]
-    ):
-        super().__init__(value, task)
-        self.__xpm__.serialized = serialized
-        self.__xpm__.path = path
-
-    def __getitem__(self, key: Any):
-        value = self.__xpm__.value.__getitem__(key)
-        return SerializedTaskOutput(
-            value, self.serialized, self.__xpm__.task, self.path + [ItemAccessor(key)]
-        )
-
-    def __getattr__(self, key: str, default=None) -> Any:
-        value = getattr(self.__xpm__.value, key, default)
-        return SerializedTaskOutput(
-            value,
-            self.__xpm__.serialized,
-            self.__xpm__.task,
-            self.__xpm__.path + [AttrAccessor(key, default)],
-        )
+        __self__ = self.__xpm__.parent.wrap()
+        return self.__xpm__.value.__func__(__self__, *args, **kwargs)
 
 
 # --- Utility functions
 
 
-def copyconfig(config_or_output: Union[Config, TaskOutput], **kwargs):
+def copyconfig(config_or_output: Union[Config, ConfigWrapper], **kwargs):
     """Copy a configuration or task output
 
     Useful to modify a configuration that can be potentially
@@ -1682,7 +1720,7 @@ def copyconfig(config_or_output: Union[Config, TaskOutput], **kwargs):
     a task output).
     """
 
-    if isinstance(config_or_output, TaskOutput):
+    if isinstance(config_or_output, ConfigWrapper):
         output = config_or_output
         config = config_or_output.__unwrap__()
         assert isinstance(config, Config)
@@ -1702,7 +1740,7 @@ def copyconfig(config_or_output: Union[Config, TaskOutput], **kwargs):
         return copy
 
     # wrap in Task output
-    return TaskOutput(copy, output.__xpm__)
+    return ConfigWrapper(copy, output.__xpm__)
 
 
 def setmeta(config: Config, flag: bool):

experimaestro/core/objects.pyi

@@ -181,7 +181,8 @@ class TypeConfig:
         *,
         workspace: Incomplete | None = ...,
         launcher: Incomplete | None = ...,
-        run_mode: RunMode = ...
+        run_mode: RunMode = ...,
+        pre: List[Task] = []
     ): ...
     def stdout(self): ...
     def stderr(self): ...

experimaestro/core/serializers.py

@@ -0,0 +1,52 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import List, TypeVar
+
+from experimaestro import Param
+
+from .objects import Config, Proxy
+from .arguments import DataPath
+
+T = TypeVar("T")
+
+
+class SerializedConfig(Config, Proxy, ABC):
+    """A serializable configuration
+
+    This can be used to define a loading mechanism when instanciating the
+    configuration
+    """
+
+    config: Param[Config]
+    """The configuration that will be serialized"""
+
+    registered: List[Config]
+    """(execution only) List of configurations that use this serialized config"""
+
+    def __post_init__(self):
+        super().__post_init__()
+        self.registered = []
+
+    def register(self, config: Config):
+        self.registered.append(config)
+
+    def __unwrap__(self):
+        return self.config
+
+    @abstractmethod
+    def initialize(self):
+        """Initialize the object
+
+        This might imply loading saved data (e.g. learned models)
+        """
+        ...
+
+
+class PathBasedSerializedConfig(SerializedConfig):
+    """A path based serialized configuration
+
+    The most common case it to have
+    """
+
+    path: DataPath[Path]
+    """Path containing the data"""

experimaestro/huggingface.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 from typing import Optional, Union
-from experimaestro import Config, TaskOutput
+from experimaestro import Config, ConfigWrapper
 from experimaestro.core.context import SerializedPath
 from experimaestro.core.objects import ConfigInformation
 from huggingface_hub import ModelHubMixin, hf_hub_download, snapshot_download
@@ -11,7 +11,7 @@ class ExperimaestroHFHub(ModelHubMixin):
     """Defines models that can be uploaded/downloaded from the Hub"""
 
     def __init__(
-        self, config: Union[Config, TaskOutput], variant: Optional[str] = None
+        self, config: Union[Config, ConfigWrapper], variant: Optional[str] = None
     ):
         self.config = config if isinstance(config, Config) else config.__unwrap__()
         self.variant = variant

experimaestro/tests/test_identifier.py

@@ -17,7 +17,8 @@ from experimaestro import (
     Annotated,
     Task,
 )
-from experimaestro.core.objects import ConfigInformation, TaskOutput, setmeta
+from experimaestro.core.objects import ConfigInformation, ConfigWrapper, setmeta
+from experimaestro.core.serializers import SerializedConfig
 from experimaestro.scheduler.workspace import RunMode
 
 
@@ -56,7 +57,7 @@ class Values:
 
 
 def getidentifier(x):
-    if isinstance(x, TaskOutput):
+    if isinstance(x, ConfigWrapper):
         return x.__xpm__.identifier.all
     return x.__xpm__.identifier.all
 
@@ -387,6 +388,32 @@ def test_identifier_meta_default_array():
     )
 
 
+# --- Check ConfigWrapper
+
+
+class Model(Config):
+    def __post_init__(self):
+        self.initialized = False
+
+
+class Trainer(Config):
+    model: Param[Model]
+
+
+class SerializedModel(SerializedConfig):
+    def initialize(self):
+        self.config.initialized = True
+
+
+def test_identifier_serialized_config():
+    trainer1 = Trainer(model=Model())
+    trainer2 = Trainer(model=SerializedModel(config=Model()))
+    assert_notequal(trainer1, trainer2)
+
+
+# --- Check configuration reloads
+
+
 def check_reload(config):
     old_identifier = config.__xpm__.identifier.all
 
@@ -412,14 +439,14 @@ def test_identifier_reload_config():
     check_reload(IdentifierReloadConfig(id="123"))
 
 
-class IdentifierReloadTaskOutput(Task):
+class IdentifierReloadConfigWrapper(Task):
     id: Param[str]
 
     def taskoutputs(self):
         return IdentifierReloadConfig(id=self.id)
 
 
-class IdentifierReloadTaskOutputDerived(Config):
+class IdentifierReloadConfigWrapperDerived(Config):
     task: Param[IdentifierReloadConfig]
 
 
@@ -427,8 +454,8 @@ def test_identifier_reload_taskoutput():
     """When using a task output, the identifier should not be different"""
 
     # Creates the configuration
-    task = IdentifierReloadTaskOutput(id="123").submit(run_mode=RunMode.DRY_RUN)
-    config = IdentifierReloadTaskOutputDerived(task=task)
+    task = IdentifierReloadConfigWrapper(id="123").submit(run_mode=RunMode.DRY_RUN)
+    config = IdentifierReloadConfigWrapperDerived(task=task)
     check_reload(config)
 
 

experimaestro/tests/test_instance.py

@@ -1,6 +1,7 @@
 from typing import Optional
 from experimaestro import config, Param, Config
 from experimaestro.core.objects import TypeConfig
+from experimaestro.core.serializers import SerializedConfig
 
 
 @config()
@@ -31,27 +32,29 @@ def test_simple_instance():
     assert isinstance(b.a, A.__xpmtype__.basetype)
 
 
-@config()
-class SerializedConfig:
-    x: Param[int] = 1
-    pass
+class Model(Config):
+    def __post_init__(self):
+        self.initialized = False
+
 
+class Trainer(Config):
+    model: Param[Model]
 
-class TestSerialization:
-    """Test that a config can be serialized during execution"""
 
-    def test_instance(self):
-        import pickle
+class SerializedModel(SerializedConfig):
+    def initialize(self):
+        self.config.initialized = True
 
-        a = SerializedConfig(x=2).instance()
-        assert not isinstance(a, TypeConfig)
-        assert isinstance(a, SerializedConfig)
 
-        s_a = pickle.dumps(a)
+def test_instance_serialized():
+    model = SerializedModel(config=Model())
+    trainer = Trainer(model=model)
+    instance = trainer.instance()
 
-        deserialized = pickle.loads(s_a)
-        assert not isinstance(deserialized, TypeConfig)
-        assert deserialized.x == 2
+    assert isinstance(
+        instance.model, Model
+    ), f"The model is not a Model but a {type(instance.model).__qualname__}"
+    assert instance.model.initialized, "The model was not initialized"
 
 
 class ConfigWithOptional(Config):

experimaestro/tests/test_outputs.py

@@ -1,9 +1,7 @@
 """Test for task outputs"""
 
-from experimaestro import Config, Task, Param
-from experimaestro.core.objects import SerializedConfig, Serialized, TaskOutput
+from experimaestro import Config, Task, Param, ConfigWrapper
 from experimaestro.scheduler.workspace import RunMode
-from experimaestro.tests.utils import TemporaryExperiment
 
 
 class B(Config):
@@ -14,19 +12,12 @@ class A(Config):
     b: Param[B]
 
 
-class LoaderA(Serialized):
-    @staticmethod
-    def fromJSON(x) -> A:
-        return A(b=B(x=x)).instance()
-
-
 class Main(Task):
     a: Param[A]
 
     def taskoutputs(self):
         return self.a, {
             "a": self.a,
-            "serialized": SerializedConfig(self.a, LoaderA(self.a.b.x)),
         }
 
     def execute(self):
@@ -44,8 +35,7 @@ def test_output_taskoutput():
     a = A(b=B())
     output, ioutput = Main(a=a).submit(run_mode=RunMode.DRY_RUN)
 
-    assert isinstance(ioutput["serialized"], TaskOutput)
-    assert isinstance(output, TaskOutput), "outputs should be task proxies"
+    assert isinstance(output, ConfigWrapper), "outputs should be task proxies"
 
     # Direct
     Main(a=output)
@@ -58,31 +48,3 @@ def test_output_taskoutput():
 
     # Now, submits
     Main(a=output).submit(run_mode=RunMode.DRY_RUN)
-
-
-def test_output_serialization():
-    """Test output serialization"""
-
-    with TemporaryExperiment("output_serialization", maxwait=5) as xp:
-        a = A(b=B(x=2))
-
-        main0 = Main(a=a)
-        output, ioutput = main0.submit()
-
-        # Direct
-        serialized_a = ioutput["serialized"]
-        main1 = Main(a=serialized_a)
-        main1.submit()
-
-        # Indirect (via attribute)
-        serialized_a = ioutput["serialized"]
-        main2 = Main(a=A(b=serialized_a.b))
-        main2.submit()
-
-        xp.wait()
-
-        for main in (main1, main2):
-            assert main.__xpm__.job.stdout.read_text().strip() == "2"
-            assert len(main.__xpm__.job.dependencies) == 1
-            dep = next(iter(main.__xpm__.job.dependencies))
-            assert dep.origin.config is main0

experimaestro/tests/test_progress.py

@@ -1,13 +1,11 @@
 # Test that progress notification work
 from copy import copy
-import logging
 from pathlib import Path
 import time
 import fasteners
 from typing import List, Tuple, Union
-from experimaestro.commandline import CommandLineJob
 from experimaestro import Task, Annotated, pathgenerator, progress, tqdm
-from experimaestro.core.objects import TaskOutput, logger
+from experimaestro.core.objects import ConfigWrapper, logger
 from experimaestro.notifications import LevelInformation
 from experimaestro.scheduler import Job, Listener
 from queue import Queue
@@ -74,7 +72,7 @@ def test_progress_basic():
         listener = ProgressListener()
         xp.scheduler.addlistener(listener)
 
-        out = ProgressingTask().submit()  # type: TaskOutput
+        out: ConfigWrapper = ProgressingTask().submit()
         path = out.path  # type: Path
         job = out.__xpm__.job
 
@@ -87,9 +85,9 @@ def test_progress_basic():
         for v in progresses:
             writeprogress(path, v)
             if v < 1:
-                l = listener.progresses.get()[0]
-                logger.info("Got %s", l)
-                assert l.progress == v
+                info = listener.progresses.get()[0]
+                logger.info("Got %s", info)
+                assert info.progress == v
 
 
 def test_progress_multiple():
@@ -105,7 +103,7 @@ def test_progress_multiple():
         listener1 = ProgressListener()
         xp1.scheduler.addlistener(listener1)
 
-        out = ProgressingTask().submit()  # type: TaskOutput
+        out = ProgressingTask().submit()  # type: ConfigWrapper
         path = out.path  # type: Path
         job = out.__xpm__.job
 
@@ -219,7 +217,7 @@ def test_progress_nested():
         listener = ProgressListener()
         xp.scheduler.addlistener(listener)
 
-        out = NestedProgressingTask().submit()  # type: TaskOutput
+        out = NestedProgressingTask().submit()  # type: ConfigWrapper
         job = out.__xpm__.job
         path = out.path  # type: Path
 

experimaestro/tests/test_serializers.py

@@ -0,0 +1,54 @@
+from experimaestro import Config, Task, Param, PathBasedSerializedConfig, copyconfig
+from experimaestro.tests.utils import TemporaryExperiment
+
+
+class SubModel(Config):
+    pass
+
+
+class Model(Config):
+    submodel: Param[SubModel]
+
+    def __post_init__(self):
+        self.initialized = False
+        self.submodel.initialized = False
+
+
+class SerializedModel(PathBasedSerializedConfig):
+    def initialize(self):
+        self.config.initialized = True
+        self.config.submodel.initialized = True
+
+
+class Trainer(Task):
+    model: Param[Config]
+
+    def taskoutputs(self):
+        return SerializedModel(config=copyconfig(self.model))
+
+    def execute(self):
+        assert not self.model.initialized, "Model not initialized"
+
+
+class Evaluate(Task):
+    model: Param[Config]
+    is_submodel: Param[bool] = False
+
+    def execute(self):
+        assert self.model.initialized, "Model not initialized"
+        if self.is_submodel:
+            assert isinstance(self.model, SubModel)
+        else:
+            assert isinstance(self.model, Model)
+
+
+def test_serializers_xp():
+    with TemporaryExperiment("serializers", maxwait=10, port=0):
+        model = Model(submodel=SubModel())
+        trained_model: Model = Trainer(model=model).submit()
+
+        # Use the model itself
+        Evaluate(model=trained_model).submit()
+
+        # Use a submodel
+        Evaluate(model=trained_model.submodel, is_submodel=True).submit()

experimaestro/utils/jobs.py

@@ -1,12 +1,12 @@
 import time
 from experimaestro.scheduler import JobState
-from experimaestro.core.objects import TaskOutput
+from experimaestro.core.objects import ConfigWrapper
 from experimaestro.scheduler import Listener
 from threading import Condition
 from tqdm.autonotebook import tqdm
 
 
-def jobmonitor(*outputs: TaskOutput):
+def jobmonitor(*outputs: ConfigWrapper):
     """Follow the progress of a list of jobs (in order)"""
 
     cv = Condition()

experimaestro/version.py

@@ -1,4 +1,4 @@
 # file generated by setuptools_scm
 # don't change, don't track in version control
-__version__ = version = '0.23.0'
-__version_tuple__ = version_tuple = (0, 23, 0)
+__version__ = version = '0.24.0'
+__version_tuple__ = version_tuple = (0, 24, 0)

{experimaestro-0.23.0.dist-info → experimaestro-0.24.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: experimaestro
-Version: 0.23.0
+Version: 0.24.0
 Summary: "Experimaestro is a computer science experiment manager"
 Home-page: https://github.com/experimaestro/experimaestro-python
 Author: Benjamin Piwowarski
@@ -33,12 +33,14 @@ Requires-Dist: pyparsing
 Requires-Dist: humanfriendly
 Requires-Dist: huggingface-hub (~=0.11.1)
 Requires-Dist: gevent
+Requires-Dist: gevent-websocket
 Requires-Dist: flask
 Requires-Dist: flask-socketio
 Requires-Dist: Arpeggio (>=2.0)
 Requires-Dist: watchdog (>2.0.0)
 Requires-Dist: marshmallow
 Requires-Dist: fabric
+Requires-Dist: decorator
 Requires-Dist: rpyc
 Requires-Dist: typing-extensions (>=4.2) ; python_version < "3.11"
 Requires-Dist: cached-property ; python_version < "3.9"
@@ -155,6 +157,21 @@ if __name__ == "__main__":
 
 which can be launched with `python test.py /tmp/helloworld-workdir`
 
+## 0.24.0 (2023-05-23)
+
+### Feat
+
+- serialized configurations
+
+### Fix
+
+- requirement for fabric
+- add gevent-websocket for supporting websockets
+
+### Refactor
+
+- Changed TaskOutput to ConfigWrapper
+
 ## 0.23.0 (2023-04-07)
 
 ### Feat

{experimaestro-0.23.0.dist-info → experimaestro-0.24.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
-experimaestro/__init__.py,sha256=FvCfQt53icBffG2vqLb9zTdBkB04cQhLbiabPmY1yVs,1403
-experimaestro/__main__.py,sha256=DcrhIXrMACuPp40CO-PAKTytHhPaozQwqnGIQ8NfoHs,11192
+experimaestro/__init__.py,sha256=QJipxUiZN_zgzcl1gf3lK67MCWj_-uHJbc6t_LqAIbA,1502
+experimaestro/__main__.py,sha256=wLQ3GUriDiNL2tWWQ1lLbxHrl79OK95we9UH307rSB8,11198
 experimaestro/annotations.py,sha256=laaAy2C57SJTPeRuD0B5jRUN0TgZYckO3fjBIR9laCs,8603
 experimaestro/checkers.py,sha256=ZCMbnE_GFC5compWjt-fuHhPImi9fCPjImF8Ow9NqK8,696
 experimaestro/click.py,sha256=HTfjm4poqfG69MIYztrPWEei5OVnd154UCr2pJm6yp8,2390
@@ -7,7 +7,7 @@ experimaestro/commandline.py,sha256=NS1ubme8DTJtDS2uWwdHLQiZsl6TSK1LkNxu39c3-cw,
 experimaestro/compat.py,sha256=dQqE2ZNHLM2wtdfp7fBRYMfC33qNehVf9J6FGRBUQhs,171
 experimaestro/filter.py,sha256=Qp8ObI3fBGYJjNo2TMu7N3d1rVTdYXP9PVBLFg5JUHk,5775
 experimaestro/generators.py,sha256=9NQ_TfDfASkArLnO4PF7s5Yoo9KWjlna2DCPzk5gJOI,1230
-experimaestro/huggingface.py,sha256=2f0MqNxfth90R7kO6VVJ3K5tXdx95w2d1ljggrL6l_E,3056
+experimaestro/huggingface.py,sha256=HBgQFa544TyudhWo1bxCO_AuR2wHlmUT7Xhvu7q1Y5I,3062
 experimaestro/ipc.py,sha256=0sVzK8mZ4VCRQ5L3piCqTm1o-AVfTfJEXW26P6SOFrY,1694
 experimaestro/locking.py,sha256=hPT-LuDGZTijpbme8O0kEoB9a3WjdVzI2h31OT44UxE,1477
 experimaestro/mypy.py,sha256=M39VFuDrab-ymlCDIF5jys9oKpTwnuBPzb1T8Un5J3s,285
@@ -19,7 +19,7 @@ experimaestro/settings.py,sha256=E6nFyrUB6uBYQlQARczr_YKUn06zRoZ_RbPgDj6jHaU,914
 experimaestro/taskglobals.py,sha256=aBjPpo4HQp6E6M3GQ8L6PR4rK2Lu0kD5dS1WjnaGgDc,499
 experimaestro/tokens.py,sha256=aHT6lN4YLF0ujXl0nNu1sSSx-2ebrYiZFfrFvvmsQ1s,14681
 experimaestro/typingutils.py,sha256=gkEhJ9Fir05xh5v45pYKmrGYiLCykGtlU8sL-ydFXS8,1785
-experimaestro/version.py,sha256=Ktf2waZZK1xthVXoT6iRjDhQfoED_kinKyheuED-ztk,162
+experimaestro/version.py,sha256=ZVPm3jk9go-IY_2KTuk6z9VsibssyAAJYCsrpb1eX2w,162
 experimaestro/xpmutils.py,sha256=S21eMbDYsHfvmZ1HmKpq5Pz5O-1HnCLYxKbyTBbASyQ,638
 experimaestro/connectors/__init__.py,sha256=hxcBSeVLk_7oyiIlS3l-9dGg1NGtShwvRD1tS7f8D2M,5461
 experimaestro/connectors/local.py,sha256=w4IN_pxiPh9jb5u4yIH569ZbVHb6i2LfGhdsGFQ625o,5760
@@ -27,8 +27,9 @@ experimaestro/connectors/ssh.py,sha256=rQv_j3yQ5g-HyX4tvO_E0qD6xkrZyTq5oRYQEC1Wh
 experimaestro/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/core/arguments.py,sha256=KzvPfO92deWsgjhrPULjFuNpr8aCDMbLhCoz5xafrtg,5824
 experimaestro/core/context.py,sha256=70ZhDBobGDl0wEyvZyqyrnWagLs7EiRf0cBSXp1N3Lc,2479
-experimaestro/core/objects.py,sha256=IijmRTPTVMsUKHgVugOJam-Ml_ms83vFfy4C1igN6mc,56141
-experimaestro/core/objects.pyi,sha256=vMy3EWCSwZNky8zr5i80C9lAkx4ON8Y46dcBNQpdE58,7855
+experimaestro/core/objects.py,sha256=SoFKiqOTxQE2zMrlLAsHUBQQ_4nhTKfIPQ9TUD083Rc,57211
+experimaestro/core/objects.pyi,sha256=yo_-tJpHbv-7m56Jr706Zpgtt6rGpEEEfZwer2AgLiw,7885
+experimaestro/core/serializers.py,sha256=xmgdkCJVDTlhsHIHMZ8gxf0cyqAWu_wCZTZFOyht_Dc,1187
 experimaestro/core/types.py,sha256=f8y-N0hKsqxgX_l__Vh5xIh4tudw2Ws2RkECHB3Ki44,19099
 experimaestro/launcherfinder/__init__.py,sha256=sM9U0dDvUkyRIEv5QnU9XZCn5uJ6Q6x4OAbWPmN9qcs,148
 experimaestro/launcherfinder/base.py,sha256=NptPJ0e8CktdhOPejocSfI_B4mloeH_EmJrbXruUCSA,1020
@@ -62,13 +63,13 @@ experimaestro/tests/task_tokens.py,sha256=vgqUa-S_YC2Id9pGOSv40qFTwq1WGZkFhr556Z
 experimaestro/tests/test_checkers.py,sha256=SJ6E60Df6mpXdRvrjIAOVsJpfUgk6O9toOh6gIVc-8k,409
 experimaestro/tests/test_findlauncher.py,sha256=yvp1YoYIfLnt8D2mLTYvM6mzd4iDrETpBudMRtsz_ls,2938
 experimaestro/tests/test_forward.py,sha256=oxhEnH_ovOjxXNH7-Gvqw2IFaJqoFBBCGF_OLjPh488,810
-experimaestro/tests/test_identifier.py,sha256=xg0AgPqim1hq9nAvODROP9Vv7atLL1DFAKXbcEsjp4k,10488
-experimaestro/tests/test_instance.py,sha256=m_4AdxKfa5Zzgn3w6_AOhT9wGrHEhZdDt1BmCEOtY6M,1355
+experimaestro/tests/test_identifier.py,sha256=fsW1YrXrkgHqkDh1wfu0MuY-MbLuUpRT3ro76GH_Lr8,11055
+experimaestro/tests/test_instance.py,sha256=gW8itFn-YRN5r2VU9Dq7FlOMcEMAXAR4lJ8XHMEg5Vk,1494
 experimaestro/tests/test_objects.py,sha256=kotxkhZXI05q6uCX_IvNh0QVXcNXdAR6GIVzx4A47xk,1191
-experimaestro/tests/test_outputs.py,sha256=UrO_QB50lDkVH2gNBco31kCRdga1bpX79sLqE_22hco,2028
+experimaestro/tests/test_outputs.py,sha256=FtAQ0f5Nyg0MVwPrI6_2mcP478NscsGZV6tN4xpxQyk,870
 experimaestro/tests/test_param.py,sha256=ONEydx4bFZKOijf86s30S2WuCsfOG9-rBY4gAWiJ7W8,6781
-experimaestro/tests/test_progress.py,sha256=ZrDHtlEhXcLUYOK3Zn8rNkhWwUXvYs42SVeMTHW1xco,7711
-experimaestro/tests/test_serialization.py,sha256=U1YlaWjxCFO86VOkfKplOtIr0ZHOUJOy9jfoLGLm2qM,1132
+experimaestro/tests/test_progress.py,sha256=wELD8Qj8b7J2d5KYU8k2wzSmHMMn2o67rXDc_iDx3-A,7656
+experimaestro/tests/test_serializers.py,sha256=5bWSRL99keY8qCYwr_orLEf2T24qGRRjDI1TJFjBKlo,1424
 experimaestro/tests/test_snippets.py,sha256=Q_SrQ5GJ3v5SN-gtLSlgqxykONSS0eW9vk4R4TsnpIg,3073
 experimaestro/tests/test_ssh.py,sha256=JhwsS4lJWQeMhtnDfJhQqJ5dwEThqvcNBBgUq1EWvk0,979
 experimaestro/tests/test_tags.py,sha256=cs69ENFEnlkBX_LErql4OCY5JFPCLILkia8Gntqlh18,2669
@@ -101,14 +102,14 @@ experimaestro/tools/diff.py,sha256=VZwoq_kzO8znuckxACrWRyWiRQnOJnNvFXt9_nycAxw,3
 experimaestro/tools/jobs.py,sha256=63bXhJ7RGdczLU_nxu2skGn-9dwgr4r5pD23qH4WeBA,3516
 experimaestro/utils/__init__.py,sha256=BdYguxAbR1jOQPV36OgGA31itaMvBJ6WVPV6b4Jn4xw,2434
 experimaestro/utils/asyncio.py,sha256=zEQQqZW6uHGnFknp_udt9WjjtqLNNMWun9TPL6FOF64,601
-experimaestro/utils/jobs.py,sha256=srnme_mrUl2jdokqRBZM7wWFQH_aq61ceEpiJMRxix8,2450
+experimaestro/utils/jobs.py,sha256=664eh22X_BWXoJWfIFT9rtRjM9QbOJhKkN5CyWs6H-4,2456
 experimaestro/utils/jupyter.py,sha256=JcEo2yQK7x3Cr1tNl5FqGMZOICxCv9DwMvL5xsWdQPk,2183
 experimaestro/utils/resources.py,sha256=gDjkrRjo7GULWyXmNXm_u1uqzEIAoAvJydICk56nOQw,1006
 experimaestro/utils/settings.py,sha256=jpFMqF0DLL4_P1xGal0zVR5cOrdD8O0Y2IOYvnRgN3k,793
 experimaestro/utils/yaml.py,sha256=jEjqXqUtJ333wNUdIc0o3LGvdsTQ9AKW9a9CCd-bmGU,6766
-experimaestro-0.23.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-experimaestro-0.23.0.dist-info/METADATA,sha256=SK-V-y7c9HJeXvhxyicyC2k9u8C6POr_0Jlu4ycZ1Ds,6475
-experimaestro-0.23.0.dist-info/WHEEL,sha256=a-zpFRIJzOq5QfuhBzbhiA1eHTzNCJn8OdRvhdNX0Rk,110
-experimaestro-0.23.0.dist-info/entry_points.txt,sha256=VsHkdvdBt9feNcnofvFyNbEDNjKU3BTSVPJcAGWNgko,591
-experimaestro-0.23.0.dist-info/top_level.txt,sha256=siYS2iOls_-f2iUoulKaaXuQQHroIDN36S8YAI1CFZk,14
-experimaestro-0.23.0.dist-info/RECORD,,
+experimaestro-0.24.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+experimaestro-0.24.0.dist-info/METADATA,sha256=iJuR6W-vAanMpQMFk2I9sNuk1zkbQdsGIJH-iOBCEeQ,6732
+experimaestro-0.24.0.dist-info/WHEEL,sha256=a-zpFRIJzOq5QfuhBzbhiA1eHTzNCJn8OdRvhdNX0Rk,110
+experimaestro-0.24.0.dist-info/entry_points.txt,sha256=VsHkdvdBt9feNcnofvFyNbEDNjKU3BTSVPJcAGWNgko,591
+experimaestro-0.24.0.dist-info/top_level.txt,sha256=siYS2iOls_-f2iUoulKaaXuQQHroIDN36S8YAI1CFZk,14
+experimaestro-0.24.0.dist-info/RECORD,,

experimaestro/tests/test_serialization.py

@@ -1,45 +0,0 @@
-# Test post-experimental serialization
-
-from pathlib import Path
-from experimaestro import Config, DataPath, Task, Param
-from experimaestro.core.objects import ConfigInformation
-from experimaestro.scheduler.workspace import RunMode
-
-
-class A(Config):
-    path: DataPath[Path]
-
-
-class TaskA(Task):
-    id: Param[str]
-
-    def taskoutputs(self):
-        return A()
-
-
-def test_serialization_simple(tmp_path_factory):
-    dir = tmp_path_factory.mktemp("ser")
-
-    a = A(path=Path(__file__))
-    a.__xpm__.serialize(dir)
-
-    des_a = ConfigInformation.deserialize(dir)
-    assert des_a.path != Path(__file__)
-    assert des_a.path.read_text() == Path(__file__).read_text()
-
-
-def test_serialization_identifier(tmp_path_factory):
-    dir = tmp_path_factory.mktemp("ser")
-
-    a = TaskA(id="id").submit(run_mode=RunMode.DRY_RUN)
-    a = a.__unwrap__()
-    a.__xpm__.serialize(dir)
-
-    des_a = ConfigInformation.deserialize(dir)
-
-    des_a_id = des_a.__identifier__()
-
-    assert des_a_id.all == a.__identifier__().all, (
-        "Identifier don't match: "
-        f"expected {a.__identifier__().all.hex()}, got {des_a_id.all.hex()}"
-    )