experimaestro 1.11.1__py3-none-any.whl → 2.0.0b4__py3-none-any.whl

experimaestro/scheduler/workspace.py

@@ -2,10 +2,14 @@ from collections import ChainMap
 from enum import Enum
 from functools import cached_property
 from pathlib import Path
-from typing import Iterator, Optional
+from typing import Optional
 from experimaestro.settings import WorkspaceSettings, Settings
 
 
+# Current workspace version
+WORKSPACE_VERSION = 0
+
+
 class RunMode(str, Enum):
     NORMAL = "normal"
     """Normal run"""
@@ -18,17 +22,16 @@ class RunMode(str, Enum):
 
 
 class Workspace:
-    """An experimental workspace
+    """Workspace environment for experiments
 
-    This workspace is created by an experiment object and is used by launchers
-    to set up jobs
+    This is a simple container for workspace settings, environment, and configuration.
+    Multiple Workspace instances can exist for the same path - the singleton pattern
+    is handled by WorkspaceStateProvider which manages the database per workspace path.
     """
 
     CURRENT = None
     settings: "Settings"
-    worspace: "WorkspaceSettings"
-
-    """Creates a workspace for experiments"""
+    workspace_settings: "WorkspaceSettings"
 
     def __init__(
         self,
@@ -37,6 +40,14 @@ class Workspace:
         launcher=None,
         run_mode: RunMode = None,
     ):
+        """Initialize workspace environment
+
+        Args:
+            settings: Global settings
+            workspace_settings: Workspace-specific settings
+            launcher: Default launcher for this workspace
+            run_mode: Run mode for experiments in this workspace
+        """
         self.settings = settings
         self.workspace_settings = workspace_settings
 
@@ -44,21 +55,47 @@ class Workspace:
         self.notificationURL: Optional[str] = None
         if isinstance(path, Path):
             path = path.absolute()
+        else:
+            path = Path(path).absolute()
         self.path = path
         self.run_mode = run_mode
         self.python_path = []
+
         from ..launchers import Launcher
 
         self.launcher = launcher or Launcher.get(path)
 
         self.env = ChainMap({}, workspace_settings.env, settings.env)
 
+        # Reference counting for nested context managers
+        self._ref_count = 0
+
     def __enter__(self):
-        self.old_workspace = Workspace.CURRENT
-        Workspace.CURRENT = self
+        # Increment reference count
+        self._ref_count += 1
+
+        # Only initialize on first entry
+        if self._ref_count == 1:
+            # Check if a different workspace is already active
+            if Workspace.CURRENT is not None and Workspace.CURRENT.path != self.path:
+                raise RuntimeError(
+                    f"Cannot activate workspace at {self.path} - "
+                    f"workspace at {Workspace.CURRENT.path} is already active. "
+                    "Multiple workspaces are not yet supported."
+                )
+
+            self.old_workspace = Workspace.CURRENT
+            Workspace.CURRENT = self
+
+        return self
 
     def __exit__(self, *args):
-        Workspace.CURRENT = self.old_workspace
+        # Decrement reference count
+        self._ref_count -= 1
+
+        # Only cleanup on last exit
+        if self._ref_count == 0:
+            Workspace.CURRENT = self.old_workspace
 
     @cached_property
     def alt_workspaces(self):
@@ -79,6 +116,11 @@ class Workspace:
         """Folder for jobs"""
         return self.path / "jobs"
 
+    @property
+    def partialspath(self):
+        """Folder for partial job directories (shared checkpoints, etc.)"""
+        return self.path / "partials"
+
     @property
     def experimentspath(self):
         """Folder for experiments"""

experimaestro/scriptbuilder.py

@@ -126,6 +126,13 @@ class PythonScriptBuilder:
             for path in job.python_path:
                 out.write(f"""    sys.path.insert(0, "{shquote(str(path))}")\n""")
 
+            # Write launcher info code (for remaining_time support)
+            launcher_info_code = job.launcher.launcher_info_code()
+            if launcher_info_code:
+                out.write("\n")
+                out.write(launcher_info_code)
+                out.write("\n")
+
             out.write(
                 f"""    TaskRunner("{shquote(connector.resolve(scriptpath))}","""
                 """ lockfiles).run()\n"""

experimaestro/server/__init__.py

@@ -5,10 +5,12 @@ import platform
 import socket
 import uuid
 from experimaestro.scheduler.base import Job
-import pkg_resources
+import sys
 import http
 import threading
-from typing import Optional, Tuple
+from typing import Optional, Tuple, ClassVar
+
+from importlib.resources import files
 from experimaestro.scheduler import Scheduler, Listener as BaseListener
 from experimaestro.scheduler.services import Service, ServiceListener
 from experimaestro.settings import ServerSettings
@@ -46,6 +48,9 @@ def job_details(job):
 
 
 def job_create(job: Job):
+    # Get experiment IDs from job.experiments list
+    experiment_ids = [xp.workdir.name for xp in job.experiments]
+
     return {
         "jobId": job.identifier,
         "taskId": job.name,
@@ -53,28 +58,50 @@ def job_create(job: Job):
         "status": job.state.name.lower(),
         "tags": list(job.tags.items()),
         "progress": progress_state(job),
+        "experimentIds": experiment_ids,  # Add experiment IDs
     }
 
 
 class Listener(BaseListener, ServiceListener):
-    def __init__(self, scheduler: Scheduler, socketio):
-        self.scheduler = scheduler
+    def __init__(self, socketio, state_provider):
         self.socketio = socketio
-        self.scheduler.addlistener(self)
-        self.services = {}
-        for service in self.scheduler.xp.services.values():
-            self.service_add(service)
+        self.state_provider = state_provider
+
+        # Try to get the scheduler (if one is running for active experiments)
+        # Otherwise we're in monitoring mode and don't need scheduler events
+        try:
+            from experimaestro.scheduler import Scheduler
+
+            # Check if a scheduler instance exists (would be created if experiments are running)
+            if Scheduler._instance is not None:
+                self.scheduler = Scheduler._instance
+                self.scheduler.addlistener(self)
+                self.services = {}
+                # Initialize services from all registered experiments
+                for xp in self.scheduler.experiments.values():
+                    for service in xp.services.values():
+                        self.service_add(service)
+            else:
+                # No scheduler running - monitoring mode
+                self.scheduler = None
+                self.services = {}
+        except Exception:
+            # Scheduler not available - monitoring mode
+            self.scheduler = None
+            self.services = {}
 
     def job_submitted(self, job):
         self.socketio.emit("job.add", job_create(job))
 
     def job_state(self, job):
+        experiment_ids = [xp.workdir.name for xp in job.experiments]
         self.socketio.emit(
             "job.update",
             {
                 "jobId": job.identifier,
                 "status": job.state.name.lower(),
                 "progress": progress_state(job),
+                "experimentIds": experiment_ids,
             },
         )
 
@@ -143,13 +170,14 @@ def proxy_response(base_url: str, request: Request, path: str):
     return flask_response
 
 
+# flake8: noqa: C901
 def start_app(server: "Server"):
     logging.debug("Starting Flask server...")
     app = Flask("experimaestro")
 
     logging.debug("Starting Flask server (SocketIO)...")
     socketio = SocketIO(app, path="/api", async_mode="gevent")
-    listener = Listener(server.scheduler, socketio)
+    listener = Listener(socketio, server.state_provider)
 
     logging.debug("Starting Flask server (setting up socketio)...")
 
@@ -159,13 +187,48 @@ def start_app(server: "Server"):
             raise ConnectionRefusedError("invalid token")
 
     @socketio.on("refresh")
-    def handle_refresh():
-        for job in listener.scheduler.jobs.values():
-            emit("job.add", job_create(job))
+    def handle_refresh(experiment_id=None):
+        """Refresh jobs for an experiment (or all experiments if None)"""
+        if experiment_id:
+            # Refresh specific experiment
+            jobs = listener.state_provider.get_jobs(experiment_id)
+            for job_data in jobs:
+                emit("job.add", job_data)
+        else:
+            # Refresh all experiments
+            if listener.scheduler:
+                # Active experiments: get jobs from scheduler
+                for job in listener.scheduler.jobs.values():
+                    emit("job.add", job_create(job))
+            else:
+                # Monitoring mode: get jobs from WorkspaceStateProvider
+                for exp in listener.state_provider.get_experiments():
+                    exp_id = exp["experiment_id"]
+                    jobs = listener.state_provider.get_jobs(exp_id)
+                    for job_data in jobs:
+                        emit("job.add", job_data)
+
+    @socketio.on("experiments")
+    def handle_experiments():
+        """List all experiments"""
+        experiments = listener.state_provider.get_experiments()
+        for exp in experiments:
+            emit("experiment.add", exp)
 
     @socketio.on("job.details")
-    def handle_details(jobid):
-        emit("job.update", job_details(listener.scheduler.jobs[jobid]))
+    def handle_details(data):
+        """Get job details - expects {experimentId, jobId} or just jobId (backward compat)"""
+        # Backward compatibility: if data is a string, treat it as jobId
+        if isinstance(data, str):
+            jobid = data
+            if listener.scheduler:
+                emit("job.update", job_details(listener.scheduler.jobs[jobid]))
+        else:
+            experiment_id = data.get("experimentId")
+            job_id = data.get("jobId")
+            job_data = listener.state_provider.get_job(experiment_id, job_id)
+            if job_data:
+                emit("job.update", job_data)
 
     @socketio.on("services")
     def handle_services_list():
@@ -180,14 +243,26 @@ def start_app(server: "Server"):
             )
 
     @socketio.on("job.kill")
-    def handle_job_kill(jobid: str):
-        job = server.scheduler.jobs[jobid]
-        future = asyncio.run_coroutine_threadsafe(
-            job.aio_process(), server.scheduler.loop
-        )
-        process = future.result()
-        if process is not None:
-            process.kill()
+    def handle_job_kill(data):
+        """Kill a job - expects {experimentId, jobId} or just jobId (backward compat)"""
+        # Backward compatibility: if data is a string, treat it as jobId
+        if isinstance(data, str):
+            jobid = data
+            if listener.scheduler:
+                job = listener.scheduler.jobs[jobid]
+                future = asyncio.run_coroutine_threadsafe(
+                    job.aio_process(), listener.scheduler.loop
+                )
+                process = future.result()
+                if process is not None:
+                    process.kill()
+        else:
+            experiment_id = data.get("experimentId")
+            job_id = data.get("jobId")
+            try:
+                listener.state_provider.kill_job(experiment_id, job_id)
+            except NotImplementedError:
+                logging.warning("kill_job not supported for this state provider")
 
     logging.debug("Starting Flask server (setting up routes)...")
 
@@ -197,7 +272,15 @@ def start_app(server: "Server"):
         if not path:
             return redirect(f"/services/{service}/", http.HTTPStatus.PERMANENT_REDIRECT)
 
-        service = server.scheduler.xp.services.get(service, None)
+        # Get service from all registered experiments
+        scheduler = Scheduler.instance()
+        service_obj = None
+        for xp in scheduler.experiments.values():
+            service_obj = xp.services.get(service, None)
+            if service_obj:
+                break
+
+        service = service_obj
         if service is None:
             return Response(f"Service {service} not found", http.HTTPStatus.NOT_FOUND)
 
@@ -210,7 +293,8 @@ def start_app(server: "Server"):
         progress = float(request.args.get("progress", 0.0))
 
         try:
-            server.scheduler.jobs[jobid].set_progress(
+            scheduler = Scheduler.instance()
+            scheduler.jobs[jobid].set_progress(
                 level,
                 progress,
                 request.args.get("desc", None),
@@ -256,10 +340,17 @@ def start_app(server: "Server"):
 
         datapath = "data/%s" % path
         logging.debug("Looking for %s", datapath)
-        if pkg_resources.resource_exists("experimaestro.server", datapath):
-            mimetype = MIMETYPES[datapath.rsplit(".", 1)[1]]
-            content = pkg_resources.resource_string("experimaestro.server", datapath)
-            return Response(content, mimetype=mimetype)
+
+        try:
+            package_files = files("experimaestro.server")
+            resource_file = package_files / datapath
+            if resource_file.is_file():
+                mimetype = MIMETYPES[datapath.rsplit(".", 1)[1]]
+                content = resource_file.read_bytes()
+                return Response(content, mimetype=mimetype)
+        except (FileNotFoundError, KeyError):
+            pass
+
         return Response("Page not found", status=404)
 
     # Start the app
@@ -292,7 +383,36 @@ def start_app(server: "Server"):
 
 
 class Server:
-    def __init__(self, scheduler: Scheduler, settings: ServerSettings):
+    _instance: ClassVar[Optional["Server"]] = None
+    _lock: ClassVar[threading.Lock] = threading.Lock()
+
+    @staticmethod
+    def instance(settings: ServerSettings = None, state_provider=None) -> "Server":
+        """Get or create the global server instance
+
+        Args:
+            settings: Server settings (optional)
+            state_provider: WorkspaceStateProvider instance (required)
+        """
+        if Server._instance is None:
+            with Server._lock:
+                if Server._instance is None:
+                    if settings is None:
+                        from experimaestro.settings import get_settings
+
+                        settings = get_settings().server
+
+                    # State provider is required - it should be passed explicitly
+                    if state_provider is None:
+                        raise ValueError(
+                            "state_provider parameter is required. "
+                            "Get it via WorkspaceStateProvider.get_instance(workspace.path)"
+                        )
+
+                    Server._instance = Server(settings, state_provider)
+        return Server._instance
+
+    def __init__(self, settings: ServerSettings, state_provider):
         if settings.autohost == "fqdn":
             settings.host = socket.getfqdn()
             logging.info("Auto host name (fqdn): %s", settings.host)
@@ -307,8 +427,8 @@ class Server:
 
         self.host = settings.host or "127.0.0.1"
         self.port = settings.port
-        self.scheduler = scheduler
         self.token = settings.token or uuid.uuid4().hex
+        self.state_provider = state_provider
         self.instance = None
         self.running = False
         self.cv_running = threading.Condition()
@@ -331,13 +451,14 @@ class Server:
                 pass
 
     def start(self):
-        """Start the websocket server in a new process process"""
+        """Start the websocket server in a daemon thread"""
         logging.info("Starting the web server")
 
         # Avoids clutering
         logging.getLogger("geventwebsocket.handler").setLevel(logging.WARNING)
 
-        self.thread = threading.Thread(target=start_app, args=(self,)).start()
+        self.thread = threading.Thread(target=start_app, args=(self,), daemon=True)
+        self.thread.start()
 
         # Wait until we really started
         while True:

experimaestro/server/data/index.css

@@ -22753,131 +22753,6 @@ readers do not read off random characters that represent icons */
   font-weight: bold;
 }
 
-@font-face {
-  font-family: "Material Icons";
-  font-style: normal;
-  font-weight: 400;
-  font-display: block;
-  src: url(/0c35d18bf06992036b69.woff2) format("woff2"), url(/4d73cb90e394b34b7670.woff) format("woff");
-}
-.material-icons {
-  font-family: "Material Icons";
-  font-weight: normal;
-  font-style: normal;
-  font-size: 24px;
-  line-height: 1;
-  letter-spacing: normal;
-  text-transform: none;
-  display: inline-block;
-  white-space: nowrap;
-  word-wrap: normal;
-  direction: ltr;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-  text-rendering: optimizeLegibility;
-  font-feature-settings: "liga";
-}
-
-@font-face {
-  font-family: "Material Icons Outlined";
-  font-style: normal;
-  font-weight: 400;
-  font-display: block;
-  src: url(/6f420cf17cc0d7676fad.woff2) format("woff2"), url(/f882956fd323fd322f31.woff) format("woff");
-}
-.material-icons-outlined {
-  font-family: "Material Icons Outlined";
-  font-weight: normal;
-  font-style: normal;
-  font-size: 24px;
-  line-height: 1;
-  letter-spacing: normal;
-  text-transform: none;
-  display: inline-block;
-  white-space: nowrap;
-  word-wrap: normal;
-  direction: ltr;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-  text-rendering: optimizeLegibility;
-  font-feature-settings: "liga";
-}
-
-@font-face {
-  font-family: "Material Icons Round";
-  font-style: normal;
-  font-weight: 400;
-  font-display: block;
-  src: url(/c380809fd3677d7d6903.woff2) format("woff2"), url(/5d681e2edae8c60630db.woff) format("woff");
-}
-.material-icons-round {
-  font-family: "Material Icons Round";
-  font-weight: normal;
-  font-style: normal;
-  font-size: 24px;
-  line-height: 1;
-  letter-spacing: normal;
-  text-transform: none;
-  display: inline-block;
-  white-space: nowrap;
-  word-wrap: normal;
-  direction: ltr;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-  text-rendering: optimizeLegibility;
-  font-feature-settings: "liga";
-}
-
-@font-face {
-  font-family: "Material Icons Sharp";
-  font-style: normal;
-  font-weight: 400;
-  font-display: block;
-  src: url(/219aa9140e099e6c72ed.woff2) format("woff2"), url(/3a4004a46a653d4b2166.woff) format("woff");
-}
-.material-icons-sharp {
-  font-family: "Material Icons Sharp";
-  font-weight: normal;
-  font-style: normal;
-  font-size: 24px;
-  line-height: 1;
-  letter-spacing: normal;
-  text-transform: none;
-  display: inline-block;
-  white-space: nowrap;
-  word-wrap: normal;
-  direction: ltr;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-  text-rendering: optimizeLegibility;
-  font-feature-settings: "liga";
-}
-
-@font-face {
-  font-family: "Material Icons Two Tone";
-  font-style: normal;
-  font-weight: 400;
-  font-display: block;
-  src: url(/4ef4218c522f1eb6b5b1.woff2) format("woff2"), url(/3baa5b8f3469222b822d.woff) format("woff");
-}
-.material-icons-two-tone {
-  font-family: "Material Icons Two Tone";
-  font-weight: normal;
-  font-style: normal;
-  font-size: 24px;
-  line-height: 1;
-  letter-spacing: normal;
-  text-transform: none;
-  display: inline-block;
-  white-space: nowrap;
-  word-wrap: normal;
-  direction: ltr;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-  text-rendering: optimizeLegibility;
-  font-feature-settings: "liga";
-}
-
 body {
   font-size: 1rem;
 }