experimaestro 1.6.1__py3-none-any.whl → 1.15.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. experimaestro/__init__.py +14 -3
  2. experimaestro/annotations.py +13 -3
  3. experimaestro/cli/filter.py +19 -5
  4. experimaestro/cli/jobs.py +12 -5
  5. experimaestro/commandline.py +3 -7
  6. experimaestro/connectors/__init__.py +27 -12
  7. experimaestro/connectors/local.py +19 -10
  8. experimaestro/connectors/ssh.py +1 -1
  9. experimaestro/core/arguments.py +35 -3
  10. experimaestro/core/callbacks.py +52 -0
  11. experimaestro/core/context.py +8 -9
  12. experimaestro/core/identifier.py +301 -0
  13. experimaestro/core/objects/__init__.py +44 -0
  14. experimaestro/core/{objects.py → objects/config.py} +364 -716
  15. experimaestro/core/objects/config_utils.py +58 -0
  16. experimaestro/core/objects/config_walk.py +151 -0
  17. experimaestro/core/objects.pyi +15 -45
  18. experimaestro/core/serialization.py +63 -9
  19. experimaestro/core/serializers.py +1 -8
  20. experimaestro/core/types.py +61 -6
  21. experimaestro/experiments/cli.py +79 -29
  22. experimaestro/experiments/configuration.py +3 -0
  23. experimaestro/generators.py +6 -1
  24. experimaestro/ipc.py +4 -1
  25. experimaestro/launcherfinder/parser.py +8 -3
  26. experimaestro/launcherfinder/registry.py +29 -10
  27. experimaestro/launcherfinder/specs.py +49 -10
  28. experimaestro/launchers/slurm/base.py +51 -13
  29. experimaestro/mkdocs/__init__.py +1 -1
  30. experimaestro/notifications.py +2 -1
  31. experimaestro/run.py +3 -1
  32. experimaestro/scheduler/base.py +114 -6
  33. experimaestro/scheduler/dynamic_outputs.py +184 -0
  34. experimaestro/scheduler/state.py +75 -0
  35. experimaestro/scheduler/workspace.py +2 -1
  36. experimaestro/scriptbuilder.py +13 -2
  37. experimaestro/server/data/0c35d18bf06992036b69.woff2 +0 -0
  38. experimaestro/server/data/1815e00441357e01619e.ttf +0 -0
  39. experimaestro/server/data/219aa9140e099e6c72ed.woff2 +0 -0
  40. experimaestro/server/data/2463b90d9a316e4e5294.woff2 +0 -0
  41. experimaestro/server/data/2582b0e4bcf85eceead0.ttf +0 -0
  42. experimaestro/server/data/3a4004a46a653d4b2166.woff +0 -0
  43. experimaestro/server/data/3baa5b8f3469222b822d.woff +0 -0
  44. experimaestro/server/data/4d73cb90e394b34b7670.woff +0 -0
  45. experimaestro/server/data/4ef4218c522f1eb6b5b1.woff2 +0 -0
  46. experimaestro/server/data/5d681e2edae8c60630db.woff +0 -0
  47. experimaestro/server/data/6f420cf17cc0d7676fad.woff2 +0 -0
  48. experimaestro/server/data/89999bdf5d835c012025.woff2 +0 -0
  49. experimaestro/server/data/914997e1bdfc990d0897.ttf +0 -0
  50. experimaestro/server/data/c210719e60948b211a12.woff2 +0 -0
  51. experimaestro/server/data/c380809fd3677d7d6903.woff2 +0 -0
  52. experimaestro/server/data/f882956fd323fd322f31.woff +0 -0
  53. experimaestro/server/data/favicon.ico +0 -0
  54. experimaestro/server/data/index.css +22963 -0
  55. experimaestro/server/data/index.css.map +1 -0
  56. experimaestro/server/data/index.html +27 -0
  57. experimaestro/server/data/index.js +101770 -0
  58. experimaestro/server/data/index.js.map +1 -0
  59. experimaestro/server/data/login.html +22 -0
  60. experimaestro/server/data/manifest.json +15 -0
  61. experimaestro/settings.py +2 -2
  62. experimaestro/sphinx/__init__.py +7 -17
  63. experimaestro/taskglobals.py +7 -2
  64. experimaestro/tests/core/__init__.py +0 -0
  65. experimaestro/tests/core/test_generics.py +206 -0
  66. experimaestro/tests/definitions_types.py +5 -3
  67. experimaestro/tests/launchers/bin/sbatch +34 -7
  68. experimaestro/tests/launchers/bin/srun +5 -0
  69. experimaestro/tests/launchers/common.py +16 -4
  70. experimaestro/tests/restart.py +9 -4
  71. experimaestro/tests/tasks/all.py +23 -10
  72. experimaestro/tests/tasks/foreign.py +2 -4
  73. experimaestro/tests/test_dependencies.py +0 -6
  74. experimaestro/tests/test_experiment.py +73 -0
  75. experimaestro/tests/test_findlauncher.py +11 -4
  76. experimaestro/tests/test_forward.py +5 -5
  77. experimaestro/tests/test_generators.py +93 -0
  78. experimaestro/tests/test_identifier.py +114 -99
  79. experimaestro/tests/test_instance.py +6 -21
  80. experimaestro/tests/test_objects.py +20 -4
  81. experimaestro/tests/test_param.py +60 -22
  82. experimaestro/tests/test_serializers.py +24 -64
  83. experimaestro/tests/test_tags.py +5 -11
  84. experimaestro/tests/test_tasks.py +10 -23
  85. experimaestro/tests/test_tokens.py +3 -2
  86. experimaestro/tests/test_types.py +20 -17
  87. experimaestro/tests/test_validation.py +48 -91
  88. experimaestro/tokens.py +16 -5
  89. experimaestro/typingutils.py +8 -8
  90. experimaestro/utils/asyncio.py +6 -2
  91. experimaestro/utils/multiprocessing.py +44 -0
  92. experimaestro/utils/resources.py +7 -3
  93. {experimaestro-1.6.1.dist-info → experimaestro-1.15.2.dist-info}/METADATA +27 -34
  94. experimaestro-1.15.2.dist-info/RECORD +159 -0
  95. {experimaestro-1.6.1.dist-info → experimaestro-1.15.2.dist-info}/WHEEL +1 -1
  96. experimaestro-1.6.1.dist-info/RECORD +0 -122
  97. {experimaestro-1.6.1.dist-info → experimaestro-1.15.2.dist-info}/entry_points.txt +0 -0
  98. {experimaestro-1.6.1.dist-info → experimaestro-1.15.2.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,22 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8" />
5
+ <meta
6
+ name="viewport"
7
+ content="width=device-width, initial-scale=1, shrink-to-fit=no"
8
+ />
9
+ <meta name="theme-color" content="#000000" />
10
+ <title>Experimaestro</title>
11
+ </head>
12
+ <body>
13
+ <h1>Experimaestro</h1>
14
+ <form action="/auth" method="GET">
15
+ <div>
16
+ Token
17
+ <input type="text" name="xpm-token" />
18
+ <input type="submit" value="submit" />
19
+ </div>
20
+ </form>
21
+ </body>
22
+ </html>
@@ -0,0 +1,15 @@
1
+ {
2
+ "short_name": "Experimaestro",
3
+ "name": "Experimaestro Client",
4
+ "icons": [
5
+ {
6
+ "src": "favicon.ico",
7
+ "sizes": "64x64 32x32 24x24 16x16",
8
+ "type": "image/x-icon"
9
+ }
10
+ ],
11
+ "start_url": "./index.html",
12
+ "display": "standalone",
13
+ "theme_color": "#000000",
14
+ "background_color": "#ffffff"
15
+ }
experimaestro/settings.py CHANGED
@@ -37,7 +37,7 @@ class WorkspaceSettings:
37
37
 
38
38
  alt_workspaces: List[str] = field(default_factory=list)
39
39
  """Alternative workspaces to find jobs or experiments"""
40
-
40
+
41
41
  def __post_init__(self):
42
42
  self.path = self.path.expanduser().resolve()
43
43
 
@@ -83,7 +83,7 @@ def get_workspace(id: Optional[str] = None) -> Optional[WorkspaceSettings]:
83
83
  return None
84
84
 
85
85
 
86
- def find_workspace(*, workspace: Optional[str] = None, workdir: Optional[Path] = None):
86
+ def find_workspace(*, workspace: Optional[str] = None, workdir: Optional[Path] = None) -> WorkspaceSettings:
87
87
  """Find workspace"""
88
88
  workdir = Path(workdir) if workdir else None
89
89
 
@@ -10,19 +10,13 @@ from docutils import nodes
10
10
  from sphinx.application import Sphinx
11
11
  from sphinx import addnodes
12
12
  from sphinx.ext.autodoc import ClassDocumenter, Documenter, restify
13
- from sphinx.locale import _, __
13
+ from sphinx.locale import _
14
14
  from sphinx.util import inspect, logging
15
- from sphinx.domains.python import (
16
- PyClasslike,
17
- PyAttribute,
18
- PyObject,
19
- directives,
20
- desc_signature,
21
- _parse_annotation,
22
- )
15
+ from sphinx.domains.python import PyClasslike, PyAttribute, directives
16
+ from sphinx.domains.python import PyObject # noqa: F401
17
+ from sphinx.addnodes import desc_signature
23
18
  from sphinx.util.typing import OptionSpec
24
19
  from docutils.statemachine import StringList
25
- import logging
26
20
  import re
27
21
 
28
22
  from experimaestro import Config, Task
@@ -97,9 +91,6 @@ class ConfigDocumenter(ClassDocumenter):
97
91
  can_document = inspect.isclass(member) and issubclass(member, Config)
98
92
  return can_document
99
93
 
100
- def add_directive_header(self, sig: str) -> None:
101
- super().add_directive_header(sig)
102
-
103
94
  def get_object_members(self, want_all: bool): # -> Tuple[bool, ObjectMembers]:
104
95
  r = super().get_object_members(want_all)
105
96
  return r
@@ -157,7 +148,7 @@ class ConfigDocumenter(ClassDocumenter):
157
148
 
158
149
  # Our specific code
159
150
  if issubclass(self.object, Task):
160
- self.add_line(f" :task:", sourcename)
151
+ self.add_line(" :task:", sourcename)
161
152
 
162
153
  # add inheritance info, if wanted
163
154
  if not self.doc_as_attr and self.options.show_inheritance:
@@ -196,7 +187,6 @@ class ConfigDocumenter(ClassDocumenter):
196
187
  def add_content(
197
188
  self, more_content: Optional[StringList], no_docstring: bool = False
198
189
  ) -> None:
199
-
200
190
  xpminfo = getxpminfo(self.object)
201
191
  source_name = self.get_sourcename()
202
192
 
@@ -214,9 +204,9 @@ class ConfigDocumenter(ClassDocumenter):
214
204
  source_name,
215
205
  )
216
206
  if argument.generator:
217
- self.add_line(f" :generated:", source_name)
207
+ self.add_line(" :generated:", source_name)
218
208
  elif argument.constant:
219
- self.add_line(f" :constant:", source_name)
209
+ self.add_line(" :constant:", source_name)
220
210
 
221
211
  # self.add_line("", source_name)
222
212
  if argument.help:
@@ -1,7 +1,6 @@
1
+ from functools import cached_property
1
2
  from pathlib import Path
2
3
  from typing import Optional
3
- import os
4
- import logging
5
4
 
6
5
 
7
6
  class Env:
@@ -18,6 +17,12 @@ class Env:
18
17
  # - no progress report
19
18
  slave: bool = False
20
19
 
20
+ @cached_property
21
+ def xpm_path(self):
22
+ path = self.taskpath / ".experimaestro"
23
+ path.mkdir(exist_ok=True)
24
+ return path
25
+
21
26
  @staticmethod
22
27
  def instance():
23
28
  if Env._instance is None:
File without changes
@@ -0,0 +1,206 @@
1
+ """Tests for the use of generics in configurations"""
2
+
3
+ from typing import Generic, Optional, TypeVar
4
+
5
+ import pytest
6
+ from experimaestro import Config, Param
7
+ from experimaestro.core.arguments import Argument
8
+ from experimaestro.core.types import TypeVarType
9
+
10
+ T = TypeVar("T")
11
+
12
+
13
+ class SimpleConfig(Config):
14
+ pass
15
+
16
+
17
+ class SimpleConfigChild(SimpleConfig):
18
+ pass
19
+
20
+
21
+ class SimpleGenericConfig(Config, Generic[T]):
22
+ x: Param[T]
23
+
24
+
25
+ class SimpleGenericConfigChild(SimpleGenericConfig, Generic[T]):
26
+ """A child class of SimpleGenericConfig that also uses generics"""
27
+
28
+ pass
29
+
30
+
31
+ def test_core_generics_typevar():
32
+ a = SimpleGenericConfig.C(x=1)
33
+
34
+ x_arg = a.__xpmtype__.arguments["x"]
35
+
36
+ # Check correct interpretation of typevar
37
+ assert type(x_arg) is Argument
38
+ assert isinstance(x_arg.type, TypeVarType)
39
+ assert x_arg.type.typevar == T
40
+
41
+ assert isinstance(a.x, int)
42
+
43
+
44
+ def test_core_generics_simple():
45
+ a = SimpleGenericConfig.C(x=2)
46
+
47
+ # OK
48
+ a.x = 3
49
+
50
+ # Fails: changing generics is not allowed
51
+ with pytest.raises(TypeError):
52
+ a.x = "a string"
53
+
54
+ # typevar bindings are local to the instance,
55
+ # so we can create a new instance with a different type
56
+ SimpleGenericConfig.C(x="a string")
57
+
58
+
59
+ class DoubleGenericConfig(Config, Generic[T]):
60
+ x: Param[T]
61
+ y: Param[T]
62
+
63
+
64
+ def test_core_generics_double():
65
+ # OK
66
+ DoubleGenericConfig.C(x=1, y=1)
67
+
68
+ # Fails
69
+ with pytest.raises(TypeError):
70
+ DoubleGenericConfig.C(x=1, y="a")
71
+
72
+ a = DoubleGenericConfig.C(x=1, y=1)
73
+ a.y = 2
74
+ with pytest.raises(TypeError):
75
+ a.x = "b"
76
+
77
+
78
+ def test_core_generics_double_rebind():
79
+ a = DoubleGenericConfig.C(x=1, y=1)
80
+ # Rebinding to a different type should not work
81
+ with pytest.raises(TypeError):
82
+ a.x, a.y = "some", "string"
83
+
84
+
85
+ def test_core_generics_double_plus():
86
+ # Testing with inheritance
87
+ # We allow subclasses of the typevar binding
88
+ # We also allow generalizing up the typevar binding
89
+ # This means that we can use a super class of the typevar binding
90
+
91
+ # Works
92
+ a = DoubleGenericConfig.C(x=SimpleConfigChild.C())
93
+ a.y = SimpleConfig.C()
94
+
95
+ # Works also
96
+ b = DoubleGenericConfig.C(x=SimpleConfig.C())
97
+ b.y = SimpleConfigChild.C()
98
+
99
+ a.x = SimpleConfigChild.C()
100
+
101
+ with pytest.raises(TypeError):
102
+ a.x = "a string"
103
+
104
+
105
+ def test_core_generics_double_type_escalation():
106
+ a = DoubleGenericConfig.C(x=SimpleConfigChild.C())
107
+ a.y = SimpleConfigChild.C()
108
+ # T is now bound to SimpleConfigChild
109
+
110
+ a.y = SimpleConfig.C()
111
+ # T is now bound to SimpleConfig
112
+
113
+ a.y = object()
114
+ # T is now bound to object, which is a super class of SimpleConfigChild
115
+
116
+ # This is allowed, since we are not changing the typevar binding
117
+ a.x = "a string"
118
+
119
+ a.y = dict()
120
+ # This is allowed, since we are not changing the typevar binding
121
+
122
+
123
+ def test_core_generics_double_deep_bind():
124
+ # Since we are deep binding the typevar T to a specific type,
125
+ # we should not be able to have coherent *local-only* type bindings
126
+ # The type bindings are transient
127
+
128
+ with pytest.raises(TypeError):
129
+ DoubleGenericConfig.C(
130
+ x=DoubleGenericConfig.C(x=1, y=2), y=DoubleGenericConfig.C(x=3, y=4)
131
+ )
132
+
133
+
134
+ class NestedConfig(Config, Generic[T]):
135
+ x: Param[DoubleGenericConfig[T]]
136
+ y: Param[SimpleGenericConfig[T]]
137
+
138
+
139
+ def test_core_generics_nested():
140
+ # OK
141
+ NestedConfig.C(x=DoubleGenericConfig.C(x=1, y=1), y=SimpleGenericConfig.C(x=2))
142
+
143
+ # Not OK
144
+ with pytest.raises(TypeError):
145
+ NestedConfig.C(
146
+ x=DoubleGenericConfig.C(x=1, y=1), y=SimpleGenericConfig.C(x="b")
147
+ )
148
+
149
+ with pytest.raises(TypeError):
150
+ a = NestedConfig.C(
151
+ x=DoubleGenericConfig.C(x=1, y=1), y=SimpleGenericConfig.C(x=1)
152
+ )
153
+ a.x.x = "a string"
154
+
155
+
156
+ class TreeGenericConfig(Config, Generic[T]):
157
+ x: Param[T]
158
+ left: Optional["TreeGenericConfig[T]"] = None
159
+ right: Optional["TreeGenericConfig[T]"] = None
160
+
161
+
162
+ class TagTreeGenericConfig(TreeGenericConfig[T], Generic[T]):
163
+ """A tagged version of TreeGenericConfig to test recursive generics"""
164
+
165
+ tag: Param[str] = "default"
166
+
167
+
168
+ def test_core_generics_recursive():
169
+ a = TreeGenericConfig.C(x=1)
170
+ a.left = TreeGenericConfig.C(x=2)
171
+ a.right = TreeGenericConfig.C(x=3)
172
+
173
+ with pytest.raises(TypeError):
174
+ a.left.x = "a string"
175
+
176
+ # OK to use a child class
177
+ a.left = TagTreeGenericConfig.C(x=4, tag="left")
178
+
179
+ with pytest.raises(TypeError):
180
+ a.left.x = "a string"
181
+
182
+
183
+ def test_core_generics_recursive_child():
184
+ # Testing with a child class on the generic value
185
+ a = TreeGenericConfig.C(x=SimpleConfig.C())
186
+ a.left = TreeGenericConfig.C(x=SimpleConfig.C())
187
+ a.right = TreeGenericConfig.C(x=SimpleConfig.C())
188
+
189
+ a.left.x = SimpleConfigChild.C()
190
+
191
+ with pytest.raises(TypeError):
192
+ a.left.x = "a string"
193
+
194
+
195
+ U = TypeVar("U", bound=SimpleConfigChild)
196
+
197
+
198
+ class BoundGenericConfig(Config, Generic[U]):
199
+ x: Param[U]
200
+
201
+
202
+ def test_core_generics_bound_typevar():
203
+ a = BoundGenericConfig.C(x=SimpleConfigChild.C())
204
+ assert isinstance(a.x, SimpleConfigChild)
205
+ with pytest.raises(TypeError):
206
+ a.x = SimpleConfig.C()
@@ -1,15 +1,17 @@
1
- from experimaestro import argument, Task
1
+ from experimaestro import Param, Task
2
2
 
3
3
 
4
- @argument("value", type=int)
5
4
  class IntegerTask(Task):
5
+ value: Param[int]
6
+
6
7
  def execute(self):
7
8
  if not isinstance(self.value, int):
8
9
  raise AssertionError("Not an integer")
9
10
 
10
11
 
11
- @argument("value", type=float)
12
12
  class FloatTask(Task):
13
+ value: Param[float]
14
+
13
15
  def execute(self):
14
16
  if not isinstance(self.value, float):
15
17
  raise AssertionError("Not a float but %s" % type(self.value))
@@ -2,15 +2,28 @@
2
2
 
3
3
  # slurm test suite
4
4
 
5
+ pwd 1>&2
6
+ CURDIR="$(realpath "$(dirname "$0")"/..)"
7
+
5
8
  # Where we store the jobs
6
- XPM_SLURM_DIR="$(realpath "$(dirname "$0")"/..)/slurm"
9
+ XPM_SLURM_DIR="${CURDIR}/slurm"
7
10
  if ! test -d "$XPM_SLURM_DIR"; then
8
11
  echo "Directory $XPM_SLURM_DIR does not exist" 1>&2
9
12
  exit 1
10
13
  fi
11
14
 
12
15
  mkdir -p "$XPM_SLURM_DIR/jobs"
13
- echo "Slurm directory: $XPM_SLURM_DIR" >&2
16
+ echo "Slurm directory: $XPM_SLURM_DIR" >&2
17
+
18
+ RED='\033[0;31m'
19
+ NC='\033[0m' # No Color
20
+
21
+
22
+ if ! which flock > /dev/null; then
23
+ echo -e "[${RED}ERROR${NC}] flock is not installed on this system" 1>&2
24
+ exit 1
25
+ fi
26
+
14
27
 
15
28
  lockpath() {
16
29
  fid="$1"
@@ -18,7 +31,7 @@ lockpath() {
18
31
 
19
32
  echo "Locking $path..." 1>&2
20
33
  eval exec "$fid<>" $path
21
- if ! flock --timeout 2 $fid; then
34
+ if ! flock --timeout 2 $fid; then
22
35
  echo Could not lock "$path" - stopping 1>&2
23
36
  exit 017
24
37
  fi
@@ -37,12 +50,26 @@ while true; do
37
50
  -o) shift; stdout="$1"; shift;;
38
51
  -e) shift; stderr="$1"; shift;;
39
52
  --*) args+=("$1"); shift;;
40
- *) break 2;;
53
+ *) break 2;;
41
54
  esac
42
55
  done
43
56
 
44
- echo "Starting $@ ${args[@]} > $stdout" >&2
45
- (eval "$@" "${args[@]}"; echo $? > "$XPM_SLURM_DIR/jobs/$$.status") > $stdout 2> $stderr &
57
+ chdir=$(pwd)
58
+ while IFS= read -r line; do
59
+ case "$line" in
60
+ "#SBATCH --output="*)stdout=${line#*#SBATCH --output=};;
61
+ "#SBATCH --error="*) stderr=${line#*#SBATCH --error=};;
62
+ "#SBATCH --chdir="*) chdir=${line#*#SBATCH --chdir=};;
63
+ esac
64
+ done < "$1"
65
+
66
+ cd "$chdir"
67
+ echo "Starting $@ ${args[@]} > $stdout 2> $stderr" >&2
68
+ (
69
+ export PATH="${CURDIR}/bin:$PATH"
70
+ eval "$@" "${args[@]}"
71
+ echo $? > "$XPM_SLURM_DIR/jobs/$$.status"
72
+ ) > $stdout 2> $stderr &
46
73
  JOBID="$$"
47
74
  date > "$XPM_SLURM_DIR/jobs/$JOBID.start"
48
75
  disown
@@ -51,4 +78,4 @@ if test "$parsable" == 0; then
51
78
  echo "Submitted batch job ${JOBID}"
52
79
  else
53
80
  echo "${JOBID};cluster"
54
- fi
81
+ fi
@@ -0,0 +1,5 @@
1
+ #!/bin/bash
2
+
3
+ # Just run the command
4
+
5
+ eval "$@"
@@ -68,24 +68,36 @@ class WaitUntilTouched(Task):
68
68
 
69
69
 
70
70
  def takeback(launcher, datapath, txp1, txp2):
71
+ """Launch two times the same task (with two experiments)
72
+
73
+ :param launcher: The launcher
74
+ :param datapath: The path containing the two files that control the task, namely (1) touching which is created by the task when starting, (2) waiting which is controlled here
75
+ :param txp1: The first experiment
76
+ :param txp2: The second experiment
77
+ """
71
78
  datapath.mkdir()
72
79
  touching = datapath / "touching"
73
80
  waiting = datapath / "waiting"
74
81
 
75
- with txp1 as xp1: # flake8: noqa: F841
76
- WaitUntilTouched(touching=touching, waiting=waiting).submit(launcher=launcher)
82
+ with txp1:
83
+ task: WaitUntilTouched = WaitUntilTouched(
84
+ touching=touching, waiting=waiting
85
+ ).submit(launcher=launcher)
77
86
 
78
87
  logger.debug("Waiting for task to create 'touching' file")
79
88
  while not touching.is_file():
89
+ if task.__xpm__.job.state.finished():
90
+ raise Exception("Job has finished... too early")
80
91
  time.sleep(0.01)
81
92
 
82
- with txp2 as xp2: # flake8: noqa: F841
93
+ with txp2:
83
94
  result = WaitUntilTouched(touching=touching, waiting=waiting).submit(
84
95
  launcher=launcher
85
96
  )
86
97
 
87
- logger.debug("Waiting for job to be in running mode")
98
+ logger.debug("Waiting for job to be running (scheduler)")
88
99
  while result.__xpm__.job.state != JobState.RUNNING:
89
100
  time.sleep(0.1)
90
101
 
102
+ logger.debug("OK, no we can notify the task")
91
103
  waiting.touch()
@@ -1,12 +1,14 @@
1
1
  import time
2
+ from pathlib import Path
2
3
  import sys
3
4
  from typing import Callable
4
- from experimaestro import Task, pathoption
5
+ from experimaestro import Task, Meta, field, PathGenerator
5
6
  import psutil
6
7
  import logging
7
8
  import subprocess
8
9
  import json
9
10
  import signal
11
+
10
12
  from experimaestro.scheduler.workspace import RunMode
11
13
  from experimaestro.tests.utils import TemporaryExperiment, is_posix
12
14
  from experimaestro.scheduler import JobState
@@ -25,10 +27,13 @@ TERMINATES_FUNC = [terminate]
25
27
  if is_posix():
26
28
  TERMINATES_FUNC.append(sigint)
27
29
 
30
+ MAX_RESTART_WAIT = 50 # 5 seconds
31
+
28
32
 
29
- @pathoption("touch", "touch")
30
- @pathoption("wait", "wait")
31
33
  class Restart(Task):
34
+ touch: Meta[Path] = field(default_factory=PathGenerator("touch"))
35
+ wait: Meta[Path] = field(default_factory=PathGenerator("wait"))
36
+
32
37
  def execute(self):
33
38
  # Write the file "touch" to notify that we started
34
39
  with open(self.touch, "w") as out:
@@ -78,7 +83,7 @@ def restart(terminate: Callable, experiment):
78
83
  while not task.touch.is_file():
79
84
  time.sleep(0.1)
80
85
  counter += 1
81
- if counter >= 20:
86
+ if counter >= MAX_RESTART_WAIT:
82
87
  terminate(xpmprocess)
83
88
  assert False, "Timeout waiting for task to be executed"
84
89
 
@@ -1,11 +1,13 @@
1
+ from pathlib import Path
1
2
  import time
2
3
  from typing import List
3
4
  from experimaestro import (
4
- param,
5
+ Meta,
5
6
  Param,
7
+ field,
6
8
  Task,
9
+ PathGenerator,
7
10
  Config,
8
- pathoption,
9
11
  STDOUT,
10
12
  cache,
11
13
  )
@@ -17,9 +19,16 @@ class SimpleTask(Task):
17
19
  def execute(self):
18
20
  print(self.x) # noqa: T201
19
21
 
22
+ def task_outputs(self, dep):
23
+ return dep(SimpleTaskOutput.C(task=self))
24
+
25
+
26
+ class SimpleTaskOutput(Config):
27
+ task: Param[SimpleTask]
28
+
20
29
 
21
- @pathoption("out", STDOUT)
22
30
  class Say(Task):
31
+ out: Meta[Path] = field(default_factory=PathGenerator(STDOUT))
23
32
  word: Param[str]
24
33
 
25
34
  def execute(self):
@@ -38,19 +47,20 @@ class Concat(Task):
38
47
  print(" ".join(says)) # noqa: T201
39
48
 
40
49
 
41
- @param("x", type=int)
42
50
  class ForeignClassB1(Config):
43
- pass
51
+ x: Param[int]
44
52
 
45
53
 
46
- @param("b", type=ForeignClassB1)
47
54
  class ForeignTaskA(Task):
55
+ b: Param[ForeignClassB1]
56
+
48
57
  def execute(self):
49
58
  print(self.b.x) # noqa: T201
50
59
 
51
60
 
52
- @pathoption("wait", "wait")
53
61
  class Fail(Task):
62
+ wait: Meta[Path] = field(default_factory=PathGenerator("wait"))
63
+
54
64
  def execute(self):
55
65
  while not self.wait.is_file():
56
66
  time.sleep(0.01)
@@ -64,14 +74,16 @@ class Fail(Task):
64
74
  out.write("hello")
65
75
 
66
76
 
67
- @param("fail", Fail)
68
77
  class FailConsumer(Task):
78
+ fail: Param[Fail]
79
+
69
80
  def execute(self):
70
81
  return True
71
82
 
72
83
 
73
- @param("a", int)
74
84
  class Method(Task):
85
+ a: Param[int]
86
+
75
87
  def execute(self):
76
88
  assert self.a == 1
77
89
 
@@ -92,7 +104,8 @@ class CacheConfig(Config):
92
104
  return path.read_text()
93
105
 
94
106
 
95
- @param("data", type=CacheConfig)
96
107
  class CacheConfigTask(Task):
108
+ data: Param[CacheConfig]
109
+
97
110
  def execute(self):
98
111
  assert self.data.get() == "hello"
@@ -1,8 +1,6 @@
1
- from experimaestro import param, config
1
+ from experimaestro import Param
2
2
  from .all import ForeignClassB1
3
3
 
4
4
 
5
- @param("y", type=int)
6
- @config()
7
5
  class ForeignClassB2(ForeignClassB1):
8
- pass
6
+ y: Param[int]
@@ -85,9 +85,3 @@ def test_dependencies_inner_task_output(xp):
85
85
  a = task_a.submit()
86
86
  b = Inner_TaskB(param_a=a).submit()
87
87
  check_dependencies(b, task_a)
88
-
89
-
90
- def test_dependencies_pre_task(xp):
91
- a = TaskA().submit()
92
- a2 = TaskA().add_pretasks(a).submit()
93
- check_dependencies(a2, a)