experimaestro 0.22.0__zip → 0.24.0__zip

{experimaestro-0.22.0 → experimaestro-0.24.0}/CHANGELOG.md

@@ -1,3 +1,24 @@
+## 0.24.0 (2023-05-23)
+
+### Feat
+
+- serialized configurations
+
+### Fix
+
+- requirement for fabric
+- add gevent-websocket for supporting websockets
+
+### Refactor
+
+- Changed TaskOutput to ConfigWrapper
+
+## 0.23.0 (2023-04-07)
+
+### Feat
+
+- submit hooks to allow e.g. changing the environment variables
+
 ## 0.22.0 (2023-04-05)
 
 ### Feat

{experimaestro-0.22.0 → experimaestro-0.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: experimaestro
-Version: 0.22.0
+Version: 0.24.0
 Summary: "Experimaestro is a computer science experiment manager"
 Home-page: https://github.com/experimaestro/experimaestro-python
 Author: Benjamin Piwowarski
@@ -127,6 +127,27 @@ if __name__ == "__main__":
 
 which can be launched with `python test.py /tmp/helloworld-workdir`
 
+## 0.24.0 (2023-05-23)
+
+### Feat
+
+- serialized configurations
+
+### Fix
+
+- requirement for fabric
+- add gevent-websocket for supporting websockets
+
+### Refactor
+
+- Changed TaskOutput to ConfigWrapper
+
+## 0.23.0 (2023-04-07)
+
+### Feat
+
+- submit hooks to allow e.g. changing the environment variables
+
 ## 0.22.0 (2023-04-05)
 
 ### Feat

{experimaestro-0.22.0 → experimaestro-0.24.0}/docs/experiments/task.md

@@ -30,6 +30,44 @@ some special variables are defined:
 - task directory can is `self.__taskdir__`
 - when using [sub-parameters](../config#sub-parameters), `self.__maintaskdir__` is the directory of the main task
 
+
+## Restoring object state
+
+It is possible to restore an object state from disk; in that case, `SerializedConfig` (which can
+be subclassed) can be used:
+
+```py3
+from experimaestro import Config, SerializedConfig
+
+class Model(Config):
+    ...
+
+class SerializedModel(SerializedConfig):
+    def initialize(self):
+        # Access the configuration through self.config
+        self.config.initialized = True
+```
+
+A serialized configuration can be used instead of the configuration. This is useful
+to separate the loading mechanism from the configuration logic.
+
+
+The most often use case is when the state can be recovered from disk. In that case,
+`PathBasedSerializedConfig` can be used
+
+```py3
+from experimaestro import Config, PathBasedSerializedConfig
+
+class Model(Config):
+    ...
+
+class SerializedModel(PathBasedSerializedConfig):
+    def initialize(self):
+        # Loads the model from disk
+        data = torch.load(self.path)
+        self.config.load_state_dict(data)
+```
+
 ## Tasks outputs
 
 It is possible to generate a configuration when submitting a task.
@@ -53,7 +91,7 @@ It is possible to generate a configuration when submitting a task.
         parameters: Annotated[Path, pathgenerator("parameters.pth")]
 
         def taskoutputs(self) -> Model:
-            return SerializedConfig(self.model, ModelLoader(str(self.parameters)))
+            return SerializedModel(config=self.model, path=ModelLoader(str(self.parameters)))
 
         def execute(self):
             """Called when this task is run"""
@@ -67,18 +105,18 @@ It is possible to generate a configuration when submitting a task.
 
     ```
 
-## Task output object
+## Wrapper object
 
-Calling submit returns a `TaskOutput` object, which is necessary to keep track
+Calling submit returns a `ConfigWrapper` object, which is necessary to keep track
 of dependencies. This task output is a proxy for the returned object, i.e.
-accessing an attribute wraps it into another `TaskOutput` object.
+accessing an attribute wraps it into another `ConfigWrapper` object.
 
-`TaskOutput` objects have two specific methods and variables:
+`ConfigWrapper` objects have two specific methods and variables:
 
-- `__xpm__`, a `TaskOutputInformation` instance, containing the `task` that was submitted (dependency tracking)
+- `__xpm__`, a `ConfigWrapperInformation` instance, containing the `task` that was submitted (dependency tracking)
 - `__unwrap__` that returns the wrapped value (warning: unwrapping might prevent dependency tracking from working, but might be useful in some corner cases)
 
-For `TaskOutputInformation`, we have:
+For `ConfigWrapperInformation`, we have:
 
 - `stdout()` and `stderr()` that return a `Path` to the file that contains the standard output/error
 - `tags()` that returns the tags of the wrapped variable

{experimaestro-0.22.0 → experimaestro-0.24.0}/pyproject.toml

@@ -4,7 +4,7 @@ fallback_version = "0.0.0-dev"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.22.0"
+version = "0.24.0"
 changelog_start_rev = "0.15.0"
 tag_format = "$version"
 update_changelog_on_bump = true

{experimaestro-0.22.0 → experimaestro-0.24.0}/requirements.txt

@@ -2,8 +2,8 @@
 click
 omegaconf
 
-# Supports Annotated[T, ...]
-typing_extensions >= 3.7.4.3; python_version < "3.9"
+# Supports recent typing
+typing_extensions >= 4.2; python_version < "3.11"
 cached_property; python_version < "3.9"
 
 # Other
@@ -22,7 +22,8 @@ humanfriendly
 huggingface_hub ~= 0.11.1
 
 # server
-eventlet
+gevent
+gevent-websocket
 flask
 flask-socketio
 
@@ -38,6 +39,8 @@ marshmallow
 # SSH support (fabric is built on Paramiko,
 # and allows using OpenSSH config out of the box)
 fabric
+# Bug fix: required by fabric but not in reqs.
+decorator
 
 # Remote code execution
 rpyc

{experimaestro-0.22.0 → experimaestro-0.24.0}/setup.cfg

@@ -70,7 +70,7 @@ ignore = D200, D203, D213, D406, D407   # Google conventions
 
 [flake8]
 doctests = True
-exclude = .git, .eggs, __pycache__, tests/, docs/, build/, dist/
+exclude = .git, .eggs, __pycache__, tests/, docs/, build/, dist/, app/
 max-line-length = 88
 extend-ignore = E203
 

{experimaestro-0.22.0 → experimaestro-0.24.0}/src/experimaestro/__init__.py

@@ -44,22 +44,24 @@ from .core.objects import (
     Config,
     copyconfig,
     setmeta,
-    TaskOutput,
+    ConfigWrapper,
+    ConfigWrapper as TaskOutput,  # maintains compatibility
     Task,
-    SerializedConfig,
-    Serialized,
 )
+from .core.serializers import SerializedConfig, PathBasedSerializedConfig
+from .core.types import Any, SubmitHook, submit_hook_decorator
+from .launchers import Launcher
 from .scheduler.environment import Environment
 from .scheduler.workspace import Workspace, RunMode
 from .scheduler import Scheduler, experiment, FailedExperiment
 from .notifications import progress, tqdm
-from .core.types import Any
 from .checkers import Choices
 from .xpmutils import DirectoryContext
 from .mkdocs.annotations import documentation
+from .scheduler.base import Job
 
 
-def set_launcher(launcher):
+def set_launcher(launcher: Launcher):
     Workspace.CURRENT.launcher = launcher
 
 

{experimaestro-0.22.0 → experimaestro-0.24.0}/src/experimaestro/__main__.py

@@ -12,7 +12,7 @@ import subprocess
 from termcolor import colored, cprint
 
 import experimaestro
-from experimaestro.core.objects import TaskOutput
+from experimaestro.core.objects import ConfigWrapper
 
 # --- Command line main options
 logging.basicConfig(level=logging.INFO)
@@ -125,7 +125,7 @@ def diff(path: Path):
     """Show the reason of the identifier change for a job"""
     from experimaestro.tools.jobs import load_job
     from experimaestro import Config
-    from experimaestro.core.objects import GenerationContext
+    from experimaestro.core.objects import ConfigWalkContext
 
     _, job = load_job(path / "params.json", discard_id=False)
     _, new_job = load_job(path / "params.json")
@@ -142,7 +142,7 @@ def diff(path: Path):
             if new_id != old_id:
                 print(f"{path} differ: {new_id} vs {old_id}")
 
-                if isinstance(value, TaskOutput):
+                if isinstance(value, ConfigWrapper):
                     check(f"{path}.<task>", value.__xpm__.task, new_value.__xpm__.task)
                 else:
                     for arg in value.__xpmtype__.arguments.values():

{experimaestro-0.22.0 → experimaestro-0.24.0}/src/experimaestro/commandline.py

@@ -1,6 +1,5 @@
 """Command line jobs"""
 
-from collections import ChainMap
 import json
 import io
 from pathlib import Path
@@ -260,13 +259,6 @@ class CommandLineJob(Job):
 
         return None
 
-    @property
-    def environ(self):
-        return ChainMap(
-            self.workspace.environment.environ,
-            self.launcher.environ if self.launcher else {},
-        )
-
     @property
     def notificationURL(self):
         if self.launcher and self.launcher.notificationURL: