experimaestro 0.21.0__zip → 0.23.0__zip

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro/core/types.py

@@ -1,6 +1,7 @@
+from abc import ABC, abstractmethod
 import inspect
 import sys
-from typing import Union, Dict, Iterator, List
+from typing import Set, Union, Dict, Iterator, List
 from collections import ChainMap
 from pathlib import Path
 import typing
@@ -17,6 +18,11 @@ if sys.version_info.major == 3 and sys.version_info.minor < 9:
 else:
     from typing import _AnnotatedAlias, get_type_hints
 
+if typing.TYPE_CHECKING:
+    from experimaestro.scheduler.base import Job
+    from experimaestro.launchers import Launcher
+    from experimaestro.core.objects import Config
+
 
 class Identifier:
     def __init__(self, name: str):
@@ -143,7 +149,42 @@ class DeprecatedAttribute:
         self.fn(instance, value)
 
 
+class SubmitHook(ABC):
+    """Hook called before the job is submitted to the scheduler
+
+    This allows modifying e.g. the run environnement
+    """
+
+    @abstractmethod
+    def __call__(self, job: "Job", launcher: "Launcher"):
+        ...
+
+    @abstractmethod
+    def __spec__(self):
+        """Returns an identifier tuple for hashing/equality"""
+        ...
+
+    def __eq__(self, other):
+        if other.__class__ is not self.__class__:
+            return False
+        return self.__spec__ == other.__spec__
+
+    def __hash__(self):
+        return hash((self.__class__, self.__spec__))
+
+
+def submit_hook_decorator(hook: SubmitHook):
+    def decorator(cls: typing.Type["Config"]):
+        cls.__getxpmtype__().submit_hooks.add(hook)
+        return cls
+
+    return decorator
+
+
 class ObjectType(Type):
+    submit_hooks: Set[SubmitHook]
+    """Hooks associated with this configuration"""
+
     """ObjectType contains class-level information about
     experimaestro configurations and tasks
 
@@ -167,6 +208,7 @@ class ObjectType(Type):
         self.taskcommandfactory = None
         self.task = None
         self._title = None
+        self.submit_hooks = set()
 
         # Get the identifier
         if identifier is None and hasattr(tp, "__xpmid__"):
@@ -219,7 +261,7 @@ class ObjectType(Type):
         self.configtype.__module__ = tp.__module__
 
         # Create the type-specific object class
-        # (now, the same as basetype - TODO: remove references)
+        # (now, the same as basetype - but in the future, remove references)
         self.objecttype = self.basetype  # type: type
         self.basetype._ = self.configtype
 

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro/generators.py

@@ -1,15 +1,16 @@
 import inspect
 from pathlib import Path
-from typing import Callable, List, Tuple, Union
+from typing import Callable, Union
 from experimaestro.core.arguments import ArgumentOptions, TypeAnnotation
-from experimaestro.core.objects import GenerationContext, Config
+from experimaestro.core.objects import ConfigWalkContext, Config
 
 
 class Generator:
     """Base class for all generators"""
 
     def isoutput(self):
-        """Returns True if this generator is a task output (e.g. generates a path within the job folder)"""
+        """Returns True if this generator is a task output (e.g. generates a
+        path within the job folder)"""
         return False
 
 
@@ -17,11 +18,11 @@ class PathGenerator(Generator):
     """Generates a path"""
 
     def __init__(
-        self, path: Union[str, Path, Callable[[GenerationContext, Config], Path]]
+        self, path: Union[str, Path, Callable[[ConfigWalkContext, Config], Path]]
     ):
         self.path = path
 
-    def __call__(self, context: GenerationContext, config: Config):
+    def __call__(self, context: ConfigWalkContext, config: Config):
         if inspect.isfunction(self.path):
             path = context.currentpath() / self.path(context, config)  # type: Path
         else:
@@ -34,7 +35,7 @@ class PathGenerator(Generator):
 
 
 class pathgenerator(TypeAnnotation):
-    def __init__(self, value: Union[str, Callable[[GenerationContext, Config], str]]):
+    def __init__(self, value: Union[str, Callable[[ConfigWalkContext, Config], str]]):
         self.value = value
 
     def annotate(self, options: ArgumentOptions):

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro/launchers/__init__.py

@@ -1,25 +1,36 @@
 from pathlib import Path, PosixPath
-from typing import Callable, Dict, List, Optional, Protocol
+from typing import Callable, Dict, List, Optional
 from experimaestro.commandline import AbstractCommand, Job, CommandLineJob
 from experimaestro.connectors import Connector
 from experimaestro.connectors.local import ProcessBuilder, LocalConnector
 from experimaestro.connectors.ssh import SshPath, SshConnector
+from abc import ABC, abstractmethod
 
 
-class ScriptBuilder:
-    lockfiles: List[Path]
-    command: "AbstractCommand"
+class ScriptBuilder(ABC):
     """A script builder is responsible for generating the script
     used to launch a command line job"""
 
+    lockfiles: List[Path]
+    """The files that must be locked before starting the job"""
+
+    command: "AbstractCommand"
+    """Command to be run"""
+
+    @abstractmethod
     def write(self, job: CommandLineJob) -> Path:
-        raise NotImplementedError()
+        """Write the commmand line job
+
+        :params job: The job to be written
+        """
+        ...
 
 
 SubmitListener = Callable[[Job], None]
+"""Listen to job submissions"""
 
 
-class Launcher:
+class Launcher(ABC):
     """A launcher"""
 
     submit_listeners: List[SubmitListener]
@@ -36,9 +47,10 @@ class Launcher:
     def setNotificationURL(self, url: Optional[str]):
         self.notificationURL = url
 
+    @abstractmethod
     def scriptbuilder(self) -> ScriptBuilder:
         """Returns a script builder"""
-        raise NotImplementedError(f"For class {self.__class__}")
+        ...
 
     def addListener(self, listener: SubmitListener):
         self.submit_listeners.append(listener)

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro/scheduler/base.py

@@ -1,4 +1,4 @@
-from collections import defaultdict
+from collections import ChainMap, defaultdict
 import os
 from pathlib import Path
 from shutil import rmtree
@@ -14,7 +14,7 @@ from experimaestro.scheduler.services import Service
 from experimaestro.settings import get_settings
 
 
-from experimaestro.core.objects import Config, GenerationContext
+from experimaestro.core.objects import Config, ConfigWalkContext
 from experimaestro.utils import logger
 from experimaestro.locking import Locks, LockError, Lock
 from experimaestro.tokens import ProcessCounterToken
@@ -151,6 +151,23 @@ class Job(Resource):
         assert self._future, "Cannot wait a not submitted job"
         return self._future.result()
 
+    @property
+    def environ(self):
+        """Returns the job environment
+
+        It is made of (by order of priority):
+
+        1. The job environment
+        1. The launcher environment
+        1. The workspace environment
+
+        """
+        return ChainMap(
+            {},
+            self.launcher.environ if self.launcher else {},
+            self.workspace.environment.environ,
+        )
+
     @property
     def progress(self):
         return self._progress
@@ -292,7 +309,7 @@ class Job(Resource):
         return self._future
 
 
-class JobContext(GenerationContext):
+class JobContext(ConfigWalkContext):
     def __init__(self, job: Job):
         super().__init__()
         self.job = job
@@ -839,6 +856,7 @@ class experiment:
     def __enter__(self):
         logger.info("Locking experiment %s", self.xplockpath)
         self.xplock = self.workspace.connector.lock(self.xplockpath, 0).__enter__()
+        logger.info("Experiment locked")
 
         # Move old jobs into "jobs.bak"
         if self.workspace.run_mode == RunMode.NORMAL:

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro/server/__init__.py

@@ -90,7 +90,7 @@ class Listener(BaseListener, ServiceListener):
         )
 
     def service_state_changed(self, service: Service):
-        self.socketio.emit("service.update", {"state": service.state})
+        self.socketio.emit("service.update", {"state": service.state.name})
 
 
 MIMETYPES = {
@@ -143,10 +143,15 @@ def proxy_response(base_url: str, request: Request, path: str):
 
 
 def start_app(server: "Server"):
+    logging.debug("Starting Flask server...")
     app = Flask("experimaestro")
-    socketio = SocketIO(app, path="/api", async_mode="eventlet")
+
+    logging.debug("Starting Flask server (SocketIO)...")
+    socketio = SocketIO(app, path="/api", async_mode="gevent")
     listener = Listener(server.scheduler, socketio)
 
+    logging.debug("Starting Flask server (setting up socketio)...")
+
     @socketio.on("connect")
     def handle_connect():
         if server.token != request.cookies.get("experimaestro_token", None):
@@ -183,6 +188,8 @@ def start_app(server: "Server"):
         if process is not None:
             process.kill()
 
+    logging.debug("Starting Flask server (setting up routes)...")
+
     @app.route("/services/<path:path>", methods=["GET", "POST"])
     def route_service(path):
         service, *path = path.split("/", 1)
@@ -256,13 +263,14 @@ def start_app(server: "Server"):
 
     # Start the app
     if server.port is None or server.port == 0:
+        logging.info("Searching for an available port")
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        sock.bind(("localhost", 0))
+        sock.bind(("", 0))
         server.port = sock.getsockname()[1]
         sock.close()
 
     logging.info(
-        "Web server started on http://%s:%d?token=%s",
+        "Web server started on http://%s:%d/auth?xpm-token=%s",
         server.host,
         server.port,
         server.token,

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro/utils/__init__.py

@@ -27,6 +27,18 @@ def cleanupdir(path: Path):
     return path
 
 
+def is_notebook():
+    """Returns true if running in a notebook"""
+    try:
+        shell = get_ipython().__class__.__module__  # noqa: F841
+        if shell is not None and shell in ["ipykernel.zmqshell", "google.colab._shell"]:
+            return True
+    except NameError:
+        pass
+
+    return False
+
+
 class ThreadingCondition(threading.Condition):
     # Useful to debug lock problems
     TIMEOUT = float(os.environ.get("XPM_LOCK_TIMEOUT", "-1"))

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro/utils/jobs.py

@@ -21,6 +21,7 @@ def jobmonitor(*outputs: TaskOutput):
 
     for output in outputs:
         try:
+            job = None
             job = output.__xpm__.job
             lastprogress = 0
             while job.scheduler is None:
@@ -34,7 +35,7 @@ def jobmonitor(*outputs: TaskOutput):
 
             # Job already completed
             if job.state.value == JobState.DONE.value:
-                print("Job already completed")
+                print("Job already completed")  # noqa: T201
             else:
                 with tqdm(total=100, unit_scale=True, unit="%") as reporter:
                     progress = 0
@@ -60,8 +61,10 @@ def jobmonitor(*outputs: TaskOutput):
                             reporter.update(100 - progress)
                     else:
                         raise RuntimeError(
-                            f"Job did not complete successfully ({job.state.name}). Check the error log {job.stderr}"
+                            f"Job did not complete successfully ({job.state.name})."
+                            f"Check the error log {job.stderr}"
                         )
 
         finally:
-            job.scheduler.removelistener(listener)
+            if job is not None:
+                job.scheduler.removelistener(listener)

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro/utils/jupyter.py

@@ -1,4 +1,5 @@
 from typing import Callable, Dict, Optional
+import uuid
 import ipywidgets as widgets
 from experimaestro import experiment
 from IPython.display import display
@@ -6,6 +7,8 @@ from .jobs import jobmonitor  # noqa: F401
 
 
 class serverwidget:
+    TOKEN = uuid.uuid4().hex
+
     def __init__(
         self,
         name,
@@ -32,10 +35,13 @@ class serverwidget:
         with self.output:
             if experiment.CURRENT:
                 self.button.description = "Stop experimaestro server"
-                print(f"Server started : http://localhost:{self.port}")
+                print(  # noqa: T201
+                    "Server started : "
+                    f"http://localhost:{self.port}/auth?xpm-token={serverwidget.TOKEN}"
+                )
             else:
                 self.button.description = "Start experimaestro server"
-                print("Server stopped")
+                print("Server stopped")  # noqa: T201
 
     def on_button_clicked(self, b):
         with self.output:
@@ -43,11 +49,15 @@ class serverwidget:
                 try:
                     experiment.CURRENT.__exit__(None, None, None)
                 except Exception:
-                    print("Error while stopping experimaestro")
+                    print("Error while stopping experimaestro")  # noqa: T201
                 self.current = experiment.CURRENT
             else:
                 self.current = experiment(
-                    self.name, self.name, host="localhost", port=self.port
+                    self.name,
+                    self.name,
+                    host="localhost",
+                    port=self.port,
+                    token=serverwidget.TOKEN,
                 ).__enter__()
                 for key, value in self.environment.items():
                     self.current.setenv(key, value)

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro/version.py

@@ -1,4 +1,4 @@
 # file generated by setuptools_scm
 # don't change, don't track in version control
-__version__ = version = '0.21.0'
-__version_tuple__ = version_tuple = (0, 21, 0)
+__version__ = version = '0.23.0'
+__version_tuple__ = version_tuple = (0, 23, 0)

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro/xpmutils.py

@@ -1,10 +1,10 @@
 """Utilities exposed to users of the experimaestro API"""
 
 from pathlib import Path
-from experimaestro.core.objects import GenerationContext
+from experimaestro.core.objects import ConfigWalkContext
 
 
-class DirectoryContext(GenerationContext):
+class DirectoryContext(ConfigWalkContext):
     """Special generation context used for debugging and testing"""
 
     def __init__(self, path: Path):
@@ -16,7 +16,7 @@ class DirectoryContext(GenerationContext):
         return self._path
 
 
-class EmptyContext(GenerationContext):
+class EmptyContext(ConfigWalkContext):
     """Special generation context used for debugging and testing"""
 
     @property

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: experimaestro
-Version: 0.21.0
+Version: 0.23.0
 Summary: "Experimaestro is a computer science experiment manager"
 Home-page: https://github.com/experimaestro/experimaestro-python
 Author: Benjamin Piwowarski
@@ -127,6 +127,23 @@ if __name__ == "__main__":
 
 which can be launched with `python test.py /tmp/helloworld-workdir`
 
+## 0.23.0 (2023-04-07)
+
+### Feat
+
+- submit hooks to allow e.g. changing the environment variables
+
+## 0.22.0 (2023-04-05)
+
+### Feat
+
+- tags as immutable and hashable dicts
+
+### Fix
+
+- corrected service status update for servers
+- improved server
+
 ## 0.21.0 (2023-03-28)
 
 ### Feat

{experimaestro-0.21.0 → experimaestro-0.23.0}/src/experimaestro.egg-info/requires.txt

@@ -8,10 +8,11 @@ tqdm
 docstring_parser
 termcolor>=2.0.0
 requests
+sortedcontainers
 pyparsing
 humanfriendly
 huggingface_hub~=0.11.1
-eventlet
+gevent
 flask
 flask-socketio
 Arpeggio>=2.0
@@ -20,8 +21,10 @@ marshmallow
 fabric
 rpyc
 
+[:python_version < "3.11"]
+typing_extensions>=4.2
+
 [:python_version < "3.9"]
-typing_extensions>=3.7.4.3
 cached_property
 
 [dev]