PyPI - experimaestro - Versions diffs - 1.6.2__py3-none-any.whl → 1.7.0__py3-none-any.whl - Mend

experimaestro 1.6.2py3-none-any.whl → 1.7.0py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of experimaestro might be problematic. Click here for more details.

Files changed (79) hide show

experimaestro/__init__.py +3 -1
experimaestro/annotations.py +13 -3
experimaestro/cli/filter.py +3 -3
experimaestro/cli/jobs.py +1 -1
experimaestro/commandline.py +3 -7
experimaestro/connectors/__init__.py +22 -10
experimaestro/connectors/local.py +17 -8
experimaestro/connectors/ssh.py +1 -1
experimaestro/core/arguments.py +26 -3
experimaestro/core/objects.py +90 -6
experimaestro/core/objects.pyi +7 -1
experimaestro/core/types.py +33 -2
experimaestro/experiments/cli.py +7 -3
experimaestro/generators.py +6 -1
experimaestro/ipc.py +4 -1
experimaestro/launcherfinder/registry.py +23 -5
experimaestro/launchers/slurm/base.py +47 -9
experimaestro/notifications.py +1 -1
experimaestro/run.py +1 -1
experimaestro/scheduler/base.py +98 -10
experimaestro/scheduler/dynamic_outputs.py +184 -0
experimaestro/scriptbuilder.py +3 -1
experimaestro/server/data/016b4a6cdced82ab3aa1.ttf +0 -0
experimaestro/server/data/0c35d18bf06992036b69.woff2 +0 -0
experimaestro/server/data/1815e00441357e01619e.ttf +0 -0
experimaestro/server/data/219aa9140e099e6c72ed.woff2 +0 -0
experimaestro/server/data/2463b90d9a316e4e5294.woff2 +0 -0
experimaestro/server/data/2582b0e4bcf85eceead0.ttf +0 -0
experimaestro/server/data/3a4004a46a653d4b2166.woff +0 -0
experimaestro/server/data/3baa5b8f3469222b822d.woff +0 -0
experimaestro/server/data/4d73cb90e394b34b7670.woff +0 -0
experimaestro/server/data/4ef4218c522f1eb6b5b1.woff2 +0 -0
experimaestro/server/data/50701fbb8177c2dde530.ttf +0 -0
experimaestro/server/data/5d681e2edae8c60630db.woff +0 -0
experimaestro/server/data/6f420cf17cc0d7676fad.woff2 +0 -0
experimaestro/server/data/878f31251d960bd6266f.woff2 +0 -0
experimaestro/server/data/89999bdf5d835c012025.woff2 +0 -0
experimaestro/server/data/914997e1bdfc990d0897.ttf +0 -0
experimaestro/server/data/b041b1fa4fe241b23445.woff2 +0 -0
experimaestro/server/data/b6879d41b0852f01ed5b.woff2 +0 -0
experimaestro/server/data/c210719e60948b211a12.woff2 +0 -0
experimaestro/server/data/c380809fd3677d7d6903.woff2 +0 -0
experimaestro/server/data/d75e3fd1eb12e9bd6655.ttf +0 -0
experimaestro/server/data/f882956fd323fd322f31.woff +0 -0
experimaestro/server/data/favicon.ico +0 -0
experimaestro/server/data/index.css +22963 -0
experimaestro/server/data/index.css.map +1 -0
experimaestro/server/data/index.html +27 -0
experimaestro/server/data/index.js +101770 -0
experimaestro/server/data/index.js.map +1 -0
experimaestro/server/data/login.html +22 -0
experimaestro/server/data/manifest.json +15 -0
experimaestro/sphinx/__init__.py +7 -17
experimaestro/taskglobals.py +7 -2
experimaestro/tests/definitions_types.py +5 -3
experimaestro/tests/launchers/bin/sbatch +34 -7
experimaestro/tests/launchers/bin/srun +5 -0
experimaestro/tests/launchers/common.py +16 -4
experimaestro/tests/restart.py +6 -3
experimaestro/tests/tasks/all.py +16 -10
experimaestro/tests/tasks/foreign.py +2 -4
experimaestro/tests/test_forward.py +5 -5
experimaestro/tests/test_identifier.py +61 -66
experimaestro/tests/test_instance.py +3 -6
experimaestro/tests/test_param.py +40 -22
experimaestro/tests/test_tags.py +5 -11
experimaestro/tests/test_tokens.py +3 -2
experimaestro/tests/test_types.py +17 -14
experimaestro/tests/test_validation.py +48 -91
experimaestro/tokens.py +16 -5
experimaestro/typingutils.py +7 -0
experimaestro/utils/asyncio.py +6 -2
experimaestro/utils/resources.py +7 -3
{experimaestro-1.6.2.dist-info → experimaestro-1.7.0.dist-info}/METADATA +3 -4
experimaestro-1.7.0.dist-info/RECORD +154 -0
{experimaestro-1.6.2.dist-info → experimaestro-1.7.0.dist-info}/WHEEL +1 -1
experimaestro-1.6.2.dist-info/RECORD +0 -122
{experimaestro-1.6.2.dist-info → experimaestro-1.7.0.dist-info}/LICENSE +0 -0
{experimaestro-1.6.2.dist-info → experimaestro-1.7.0.dist-info}/entry_points.txt +0 -0

experimaestro/server/data/login.html ADDED Viewed

@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta
+      name="viewport"
+      content="width=device-width, initial-scale=1, shrink-to-fit=no"
+    />
+    <meta name="theme-color" content="#000000" />
+    <title>Experimaestro</title>
+  </head>
+  <body>
+    <h1>Experimaestro</h1>
+    <form action="/auth" method="GET">
+      <div>
+        Token
+        <input type="text" name="xpm-token" />
+        <input type="submit" value="submit" />
+      </div>
+    </form>
+  </body>
+</html>

experimaestro/server/data/manifest.json ADDED Viewed

@@ -0,0 +1,15 @@
+{
+  "short_name": "Experimaestro",
+  "name": "Experimaestro Client",
+  "icons": [
+    {
+      "src": "favicon.ico",
+      "sizes": "64x64 32x32 24x24 16x16",
+      "type": "image/x-icon"
+    }
+  ],
+  "start_url": "./index.html",
+  "display": "standalone",
+  "theme_color": "#000000",
+  "background_color": "#ffffff"
+}

experimaestro/sphinx/__init__.py CHANGED Viewed

@@ -10,19 +10,13 @@ from docutils import nodes
 from sphinx.application import Sphinx
 from sphinx import addnodes
 from sphinx.ext.autodoc import ClassDocumenter, Documenter, restify
-from sphinx.locale import _, __
+from sphinx.locale import _
 from sphinx.util import inspect, logging
-from sphinx.domains.python import (
-    PyClasslike,
-    PyAttribute,
-    PyObject,
-    directives,
-    desc_signature,
-    _parse_annotation,
-)
+from sphinx.domains.python import PyClasslike, PyAttribute, directives
+from sphinx.domains.python import PyObject  # noqa: F401
+from sphinx.addnodes import desc_signature
 from sphinx.util.typing import OptionSpec
 from docutils.statemachine import StringList
-import logging
 import re
 from experimaestro import Config, Task
@@ -97,9 +91,6 @@ class ConfigDocumenter(ClassDocumenter):
         can_document = inspect.isclass(member) and issubclass(member, Config)
         return can_document
-    def add_directive_header(self, sig: str) -> None:
-        super().add_directive_header(sig)
     def get_object_members(self, want_all: bool):  # -> Tuple[bool, ObjectMembers]:
         r = super().get_object_members(want_all)
         return r
@@ -157,7 +148,7 @@ class ConfigDocumenter(ClassDocumenter):
         # Our specific code
         if issubclass(self.object, Task):
-            self.add_line(f"   :task:", sourcename)
+            self.add_line("   :task:", sourcename)
         # add inheritance info, if wanted
         if not self.doc_as_attr and self.options.show_inheritance:
@@ -196,7 +187,6 @@ class ConfigDocumenter(ClassDocumenter):
     def add_content(
         self, more_content: Optional[StringList], no_docstring: bool = False
     ) -> None:
         xpminfo = getxpminfo(self.object)
         source_name = self.get_sourcename()
@@ -214,9 +204,9 @@ class ConfigDocumenter(ClassDocumenter):
                     source_name,
                 )
             if argument.generator:
-                self.add_line(f"   :generated:", source_name)
+                self.add_line("   :generated:", source_name)
             elif argument.constant:
-                self.add_line(f"   :constant:", source_name)
+                self.add_line("   :constant:", source_name)
             # self.add_line("", source_name)
             if argument.help:

experimaestro/taskglobals.py CHANGED Viewed

@@ -1,7 +1,6 @@
+from functools import cached_property
 from pathlib import Path
 from typing import Optional
-import os
-import logging
 class Env:
@@ -18,6 +17,12 @@ class Env:
     # - no progress report
     slave: bool = False
+    @cached_property
+    def xpm_path(self):
+        path = self.taskpath / ".experimaestro"
+        path.mkdir(exist_ok=True)
+        return path
     @staticmethod
     def instance():
         if Env._instance is None:

experimaestro/tests/definitions_types.py CHANGED Viewed

@@ -1,15 +1,17 @@
-from experimaestro import argument, Task
+from experimaestro import Param, Task
-@argument("value", type=int)
 class IntegerTask(Task):
+    value: Param[int]
     def execute(self):
         if not isinstance(self.value, int):
             raise AssertionError("Not an integer")
-@argument("value", type=float)
 class FloatTask(Task):
+    value: Param[float]
     def execute(self):
         if not isinstance(self.value, float):
             raise AssertionError("Not a float but %s" % type(self.value))

experimaestro/tests/launchers/bin/sbatch CHANGED Viewed

@@ -2,15 +2,28 @@
 # slurm test suite
+pwd 1>&2
+CURDIR="$(realpath "$(dirname "$0")"/..)"
 # Where we store the jobs
-XPM_SLURM_DIR="$(realpath "$(dirname "$0")"/..)/slurm"
+XPM_SLURM_DIR="${CURDIR}/slurm"
 if ! test -d "$XPM_SLURM_DIR"; then
     echo "Directory $XPM_SLURM_DIR does not exist" 1>&2
     exit 1
 fi
 mkdir -p "$XPM_SLURM_DIR/jobs"
-echo "Slurm directory: $XPM_SLURM_DIR" >&2
+echo "Slurm directory: $XPM_SLURM_DIR" >&2
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+if ! which flock > /dev/null; then
+    echo -e "[${RED}ERROR${NC}] flock is not installed on this system" 1>&2
+    exit 1
+fi
 lockpath() {
     fid="$1"
@@ -18,7 +31,7 @@ lockpath() {
     echo "Locking $path..." 1>&2
     eval exec "$fid<>" $path
-    if ! flock --timeout 2 $fid; then
+    if ! flock --timeout 2 $fid; then
         echo Could not lock "$path" - stopping 1>&2
         exit 017
     fi
@@ -37,12 +50,26 @@ while true; do
         -o) shift; stdout="$1"; shift;;
         -e) shift; stderr="$1"; shift;;
         --*) args+=("$1"); shift;;
-        *) break 2;;
+        *) break 2;;
     esac
 done
-echo "Starting $@ ${args[@]} > $stdout" >&2
-(eval "$@" "${args[@]}"; echo $? > "$XPM_SLURM_DIR/jobs/$$.status") > $stdout 2> $stderr &
+chdir=$(pwd)
+while IFS= read -r line; do
+    case "$line" in
+        "#SBATCH --output="*)stdout=${line#*#SBATCH --output=};;
+        "#SBATCH --error="*) stderr=${line#*#SBATCH --error=};;
+        "#SBATCH --chdir="*) chdir=${line#*#SBATCH --chdir=};;
+    esac
+done < "$1"
+cd "$chdir"
+echo "Starting $@ ${args[@]} > $stdout 2> $stderr" >&2
+(
+    export PATH="${CURDIR}/bin:$PATH"
+    eval "$@" "${args[@]}"
+    echo $? > "$XPM_SLURM_DIR/jobs/$$.status"
+) > $stdout 2> $stderr &
 JOBID="$$"
 date > "$XPM_SLURM_DIR/jobs/$JOBID.start"
 disown
@@ -51,4 +78,4 @@ if test "$parsable" == 0; then
     echo "Submitted batch job ${JOBID}"
 else
     echo "${JOBID};cluster"
-fi
+fi

experimaestro/tests/launchers/bin/srun ADDED Viewed

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Just run the command
+eval "$@"

experimaestro/tests/launchers/common.py CHANGED Viewed

@@ -68,24 +68,36 @@ class WaitUntilTouched(Task):
 def takeback(launcher, datapath, txp1, txp2):
+    """Launch two times the same task (with two experiments)
+    :param launcher: The launcher
+    :param datapath: The path containing the two files that control the task, namely (1) touching which is created by the task when starting, (2) waiting which is controlled here
+    :param txp1: The first experiment
+    :param txp2: The second experiment
+    """
     datapath.mkdir()
     touching = datapath / "touching"
     waiting = datapath / "waiting"
-    with txp1 as xp1:  # flake8: noqa: F841
-        WaitUntilTouched(touching=touching, waiting=waiting).submit(launcher=launcher)
+    with txp1:
+        task: WaitUntilTouched = WaitUntilTouched(
+            touching=touching, waiting=waiting
+        ).submit(launcher=launcher)
         logger.debug("Waiting for task to create 'touching' file")
         while not touching.is_file():
+            if task.__xpm__.job.state.finished():
+                raise Exception("Job has finished... too early")
             time.sleep(0.01)
-        with txp2 as xp2:  # flake8: noqa: F841
+        with txp2:
             result = WaitUntilTouched(touching=touching, waiting=waiting).submit(
                 launcher=launcher
             )
-            logger.debug("Waiting for job to be in running mode")
+            logger.debug("Waiting for job to be running (scheduler)")
             while result.__xpm__.job.state != JobState.RUNNING:
                 time.sleep(0.1)
+            logger.debug("OK, no we can notify the task")
             waiting.touch()

experimaestro/tests/restart.py CHANGED Viewed

@@ -1,12 +1,14 @@
 import time
+from pathlib import Path
 import sys
 from typing import Callable
-from experimaestro import Task, pathoption
+from experimaestro import Task, Meta, field, PathGenerator
 import psutil
 import logging
 import subprocess
 import json
 import signal
 from experimaestro.scheduler.workspace import RunMode
 from experimaestro.tests.utils import TemporaryExperiment, is_posix
 from experimaestro.scheduler import JobState
@@ -26,9 +28,10 @@ if is_posix():
     TERMINATES_FUNC.append(sigint)
-@pathoption("touch", "touch")
-@pathoption("wait", "wait")
 class Restart(Task):
+    touch: Meta[Path] = field(default_factory=PathGenerator("touch"))
+    wait: Meta[Path] = field(default_factory=PathGenerator("wait"))
     def execute(self):
         # Write the file "touch" to notify that we started
         with open(self.touch, "w") as out:

experimaestro/tests/tasks/all.py CHANGED Viewed

@@ -1,11 +1,13 @@
+from pathlib import Path
 import time
 from typing import List
 from experimaestro import (
-    param,
+    Meta,
     Param,
+    field,
     Task,
+    PathGenerator,
     Config,
-    pathoption,
     STDOUT,
     cache,
 )
@@ -18,8 +20,8 @@ class SimpleTask(Task):
         print(self.x)  # noqa: T201
-@pathoption("out", STDOUT)
 class Say(Task):
+    out: Meta[Path] = field(default_factory=PathGenerator(STDOUT))
     word: Param[str]
     def execute(self):
@@ -38,19 +40,20 @@ class Concat(Task):
         print(" ".join(says))  # noqa: T201
-@param("x", type=int)
 class ForeignClassB1(Config):
-    pass
+    x: Param[int]
-@param("b", type=ForeignClassB1)
 class ForeignTaskA(Task):
+    b: Param[ForeignClassB1]
     def execute(self):
         print(self.b.x)  # noqa: T201
-@pathoption("wait", "wait")
 class Fail(Task):
+    wait: Meta[Path] = field(default_factory=PathGenerator("wait"))
     def execute(self):
         while not self.wait.is_file():
             time.sleep(0.01)
@@ -64,14 +67,16 @@ class Fail(Task):
             out.write("hello")
-@param("fail", Fail)
 class FailConsumer(Task):
+    fail: Param[Fail]
     def execute(self):
         return True
-@param("a", int)
 class Method(Task):
+    a: Param[int]
     def execute(self):
         assert self.a == 1
@@ -92,7 +97,8 @@ class CacheConfig(Config):
         return path.read_text()
-@param("data", type=CacheConfig)
 class CacheConfigTask(Task):
+    data: Param[CacheConfig]
     def execute(self):
         assert self.data.get() == "hello"

experimaestro/tests/tasks/foreign.py CHANGED Viewed

@@ -1,8 +1,6 @@
-from experimaestro import param, config
+from experimaestro import Param
 from .all import ForeignClassB1
-@param("y", type=int)
-@config()
 class ForeignClassB2(ForeignClassB1):
-    pass
+    y: Param[int]

experimaestro/tests/test_forward.py CHANGED Viewed

@@ -1,12 +1,12 @@
-from experimaestro import argument, Config
+from experimaestro import Param, Config
 from experimaestro.click import forwardoption
 import click
 def test_main():
-    @argument("epochs", type=int, default=100, help="Number of learning epochs")
     class MyModel(Config):
-        pass
+        epochs: Param[int] = 100
+        """Number of learning epochs"""
     @forwardoption.epochs(MyModel)
     @click.command()
@@ -18,9 +18,9 @@ def test_main():
 def test_rename():
-    @argument("epochs", type=int, default=100, help="Number of learning epochs")
     class MyModel(Config):
-        pass
+        epochs: Param[int] = 100
+        """Number of learning epochs"""
     @forwardoption.epochs(MyModel, "my-epochs")
     @click.command()

experimaestro 1.6.2__py3-none-any.whl → 1.7.0__py3-none-any.whl

Potentially problematic release.

experimaestro 1.6.2py3-none-any.whl → 1.7.0py3-none-any.whl