experimaestro 0.22.0__zip → 0.24.0__zip

experimaestro-0.24.0/src/experimaestro/tests/test_serializers.py

@@ -0,0 +1,54 @@
+from experimaestro import Config, Task, Param, PathBasedSerializedConfig, copyconfig
+from experimaestro.tests.utils import TemporaryExperiment
+
+
+class SubModel(Config):
+    pass
+
+
+class Model(Config):
+    submodel: Param[SubModel]
+
+    def __post_init__(self):
+        self.initialized = False
+        self.submodel.initialized = False
+
+
+class SerializedModel(PathBasedSerializedConfig):
+    def initialize(self):
+        self.config.initialized = True
+        self.config.submodel.initialized = True
+
+
+class Trainer(Task):
+    model: Param[Config]
+
+    def taskoutputs(self):
+        return SerializedModel(config=copyconfig(self.model))
+
+    def execute(self):
+        assert not self.model.initialized, "Model not initialized"
+
+
+class Evaluate(Task):
+    model: Param[Config]
+    is_submodel: Param[bool] = False
+
+    def execute(self):
+        assert self.model.initialized, "Model not initialized"
+        if self.is_submodel:
+            assert isinstance(self.model, SubModel)
+        else:
+            assert isinstance(self.model, Model)
+
+
+def test_serializers_xp():
+    with TemporaryExperiment("serializers", maxwait=10, port=0):
+        model = Model(submodel=SubModel())
+        trained_model: Model = Trainer(model=model).submit()
+
+        # Use the model itself
+        Evaluate(model=trained_model).submit()
+
+        # Use a submodel
+        Evaluate(model=trained_model.submodel, is_submodel=True).submit()

{experimaestro-0.22.0 → experimaestro-0.24.0}/src/experimaestro/utils/jobs.py

@@ -1,12 +1,12 @@
 import time
 from experimaestro.scheduler import JobState
-from experimaestro.core.objects import TaskOutput
+from experimaestro.core.objects import ConfigWrapper
 from experimaestro.scheduler import Listener
 from threading import Condition
 from tqdm.autonotebook import tqdm
 
 
-def jobmonitor(*outputs: TaskOutput):
+def jobmonitor(*outputs: ConfigWrapper):
     """Follow the progress of a list of jobs (in order)"""
 
     cv = Condition()

{experimaestro-0.22.0 → experimaestro-0.24.0}/src/experimaestro/version.py

@@ -1,4 +1,4 @@
 # file generated by setuptools_scm
 # don't change, don't track in version control
-__version__ = version = '0.22.0'
-__version_tuple__ = version_tuple = (0, 22, 0)
+__version__ = version = '0.24.0'
+__version_tuple__ = version_tuple = (0, 24, 0)

{experimaestro-0.22.0 → experimaestro-0.24.0}/src/experimaestro/xpmutils.py

@@ -1,10 +1,10 @@
 """Utilities exposed to users of the experimaestro API"""
 
 from pathlib import Path
-from experimaestro.core.objects import GenerationContext
+from experimaestro.core.objects import ConfigWalkContext
 
 
-class DirectoryContext(GenerationContext):
+class DirectoryContext(ConfigWalkContext):
     """Special generation context used for debugging and testing"""
 
     def __init__(self, path: Path):
@@ -16,7 +16,7 @@ class DirectoryContext(GenerationContext):
         return self._path
 
 
-class EmptyContext(GenerationContext):
+class EmptyContext(ConfigWalkContext):
     """Special generation context used for debugging and testing"""
 
     @property

{experimaestro-0.22.0 → experimaestro-0.24.0}/src/experimaestro.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: experimaestro
-Version: 0.22.0
+Version: 0.24.0
 Summary: "Experimaestro is a computer science experiment manager"
 Home-page: https://github.com/experimaestro/experimaestro-python
 Author: Benjamin Piwowarski
@@ -127,6 +127,27 @@ if __name__ == "__main__":
 
 which can be launched with `python test.py /tmp/helloworld-workdir`
 
+## 0.24.0 (2023-05-23)
+
+### Feat
+
+- serialized configurations
+
+### Fix
+
+- requirement for fabric
+- add gevent-websocket for supporting websockets
+
+### Refactor
+
+- Changed TaskOutput to ConfigWrapper
+
+## 0.23.0 (2023-04-07)
+
+### Feat
+
+- submit hooks to allow e.g. changing the environment variables
+
 ## 0.22.0 (2023-04-05)
 
 ### Feat

{experimaestro-0.22.0 → experimaestro-0.24.0}/src/experimaestro.egg-info/SOURCES.txt

@@ -106,6 +106,7 @@ src/experimaestro/core/arguments.py
 src/experimaestro/core/context.py
 src/experimaestro/core/objects.py
 src/experimaestro/core/objects.pyi
+src/experimaestro/core/serializers.py
 src/experimaestro/core/types.py
 src/experimaestro/launcherfinder/__init__.py
 src/experimaestro/launcherfinder/base.py
@@ -145,7 +146,7 @@ src/experimaestro/tests/test_objects.py
 src/experimaestro/tests/test_outputs.py
 src/experimaestro/tests/test_param.py
 src/experimaestro/tests/test_progress.py
-src/experimaestro/tests/test_serialization.py
+src/experimaestro/tests/test_serializers.py
 src/experimaestro/tests/test_snippets.py
 src/experimaestro/tests/test_ssh.py
 src/experimaestro/tests/test_tags.py

{experimaestro-0.22.0 → experimaestro-0.24.0}/src/experimaestro.egg-info/requires.txt

@@ -12,17 +12,21 @@ sortedcontainers
 pyparsing
 humanfriendly
 huggingface_hub~=0.11.1
-eventlet
+gevent
+gevent-websocket
 flask
 flask-socketio
 Arpeggio>=2.0
 watchdog>2.0.0
 marshmallow
 fabric
+decorator
 rpyc
 
+[:python_version < "3.11"]
+typing_extensions>=4.2
+
 [:python_version < "3.9"]
-typing_extensions>=3.7.4.3
 cached_property
 
 [dev]

experimaestro-0.22.0/src/experimaestro/tests/test_outputs.py

@@ -1,88 +0,0 @@
-"""Test for task outputs"""
-
-from experimaestro import Config, Task, Param
-from experimaestro.core.objects import SerializedConfig, Serialized, TaskOutput
-from experimaestro.scheduler.workspace import RunMode
-from experimaestro.tests.utils import TemporaryExperiment
-
-
-class B(Config):
-    x: Param[int] = 1
-
-
-class A(Config):
-    b: Param[B]
-
-
-class LoaderA(Serialized):
-    @staticmethod
-    def fromJSON(x) -> A:
-        return A(b=B(x=x)).instance()
-
-
-class Main(Task):
-    a: Param[A]
-
-    def taskoutputs(self):
-        return self.a, {
-            "a": self.a,
-            "serialized": SerializedConfig(self.a, LoaderA(self.a.b.x)),
-        }
-
-    def execute(self):
-        print(self.a.b.x)  # noqa: T201
-
-
-class MainB(Task):
-    b: Param[B]
-
-    def execute(self):
-        pass
-
-
-def test_output_taskoutput():
-    a = A(b=B())
-    output, ioutput = Main(a=a).submit(run_mode=RunMode.DRY_RUN)
-
-    assert isinstance(ioutput["serialized"], TaskOutput)
-    assert isinstance(output, TaskOutput), "outputs should be task proxies"
-
-    # Direct
-    Main(a=output)
-
-    # Via getattr
-    Main(a=A(b=output.b))
-
-    # Via getitem
-    Main(a=ioutput["a"])
-
-    # Now, submits
-    Main(a=output).submit(run_mode=RunMode.DRY_RUN)
-
-
-def test_output_serialization():
-    """Test output serialization"""
-
-    with TemporaryExperiment("output_serialization", maxwait=5) as xp:
-        a = A(b=B(x=2))
-
-        main0 = Main(a=a)
-        output, ioutput = main0.submit()
-
-        # Direct
-        serialized_a = ioutput["serialized"]
-        main1 = Main(a=serialized_a)
-        main1.submit()
-
-        # Indirect (via attribute)
-        serialized_a = ioutput["serialized"]
-        main2 = Main(a=A(b=serialized_a.b))
-        main2.submit()
-
-        xp.wait()
-
-        for main in (main1, main2):
-            assert main.__xpm__.job.stdout.read_text().strip() == "2"
-            assert len(main.__xpm__.job.dependencies) == 1
-            dep = next(iter(main.__xpm__.job.dependencies))
-            assert dep.origin.config is main0

experimaestro-0.22.0/src/experimaestro/tests/test_serialization.py

@@ -1,45 +0,0 @@
-# Test post-experimental serialization
-
-from pathlib import Path
-from experimaestro import Config, DataPath, Task, Param
-from experimaestro.core.objects import ConfigInformation
-from experimaestro.scheduler.workspace import RunMode
-
-
-class A(Config):
-    path: DataPath[Path]
-
-
-class TaskA(Task):
-    id: Param[str]
-
-    def taskoutputs(self):
-        return A()
-
-
-def test_serialization_simple(tmp_path_factory):
-    dir = tmp_path_factory.mktemp("ser")
-
-    a = A(path=Path(__file__))
-    a.__xpm__.serialize(dir)
-
-    des_a = ConfigInformation.deserialize(dir)
-    assert des_a.path != Path(__file__)
-    assert des_a.path.read_text() == Path(__file__).read_text()
-
-
-def test_serialization_identifier(tmp_path_factory):
-    dir = tmp_path_factory.mktemp("ser")
-
-    a = TaskA(id="id").submit(run_mode=RunMode.DRY_RUN)
-    a = a.__unwrap__()
-    a.__xpm__.serialize(dir)
-
-    des_a = ConfigInformation.deserialize(dir)
-
-    des_a_id = des_a.__identifier__()
-
-    assert des_a_id.all == a.__identifier__().all, (
-        "Identifier don't match: "
-        f"expected {a.__identifier__().all.hex()}, got {des_a_id.all.hex()}"
-    )