experimaestro 1.5.1__py3-none-any.whl → 2.0.0a8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of experimaestro might be problematic. Click here for more details.
- experimaestro/__init__.py +14 -4
- experimaestro/__main__.py +3 -423
- experimaestro/annotations.py +14 -4
- experimaestro/cli/__init__.py +311 -0
- experimaestro/{filter.py → cli/filter.py} +23 -9
- experimaestro/cli/jobs.py +268 -0
- experimaestro/cli/progress.py +269 -0
- experimaestro/click.py +0 -35
- experimaestro/commandline.py +3 -7
- experimaestro/connectors/__init__.py +29 -14
- experimaestro/connectors/local.py +19 -10
- experimaestro/connectors/ssh.py +27 -8
- experimaestro/core/arguments.py +45 -3
- experimaestro/core/callbacks.py +52 -0
- experimaestro/core/context.py +8 -9
- experimaestro/core/identifier.py +310 -0
- experimaestro/core/objects/__init__.py +44 -0
- experimaestro/core/{objects.py → objects/config.py} +399 -772
- experimaestro/core/objects/config_utils.py +58 -0
- experimaestro/core/objects/config_walk.py +151 -0
- experimaestro/core/objects.pyi +15 -45
- experimaestro/core/serialization.py +63 -9
- experimaestro/core/serializers.py +1 -8
- experimaestro/core/types.py +104 -66
- experimaestro/experiments/cli.py +154 -72
- experimaestro/experiments/configuration.py +10 -1
- experimaestro/generators.py +6 -1
- experimaestro/ipc.py +4 -1
- experimaestro/launcherfinder/__init__.py +1 -1
- experimaestro/launcherfinder/base.py +2 -18
- experimaestro/launcherfinder/parser.py +8 -3
- experimaestro/launcherfinder/registry.py +52 -140
- experimaestro/launcherfinder/specs.py +49 -10
- experimaestro/launchers/direct.py +0 -47
- experimaestro/launchers/slurm/base.py +54 -14
- experimaestro/mkdocs/__init__.py +1 -1
- experimaestro/mkdocs/base.py +6 -8
- experimaestro/notifications.py +38 -12
- experimaestro/progress.py +406 -0
- experimaestro/run.py +24 -3
- experimaestro/scheduler/__init__.py +18 -1
- experimaestro/scheduler/base.py +108 -808
- experimaestro/scheduler/dynamic_outputs.py +184 -0
- experimaestro/scheduler/experiment.py +387 -0
- experimaestro/scheduler/jobs.py +475 -0
- experimaestro/scheduler/signal_handler.py +32 -0
- experimaestro/scheduler/state.py +75 -0
- experimaestro/scheduler/workspace.py +27 -8
- experimaestro/scriptbuilder.py +18 -3
- experimaestro/server/__init__.py +36 -5
- experimaestro/server/data/1815e00441357e01619e.ttf +0 -0
- experimaestro/server/data/2463b90d9a316e4e5294.woff2 +0 -0
- experimaestro/server/data/2582b0e4bcf85eceead0.ttf +0 -0
- experimaestro/server/data/89999bdf5d835c012025.woff2 +0 -0
- experimaestro/server/data/914997e1bdfc990d0897.ttf +0 -0
- experimaestro/server/data/c210719e60948b211a12.woff2 +0 -0
- experimaestro/server/data/index.css +5187 -5068
- experimaestro/server/data/index.css.map +1 -1
- experimaestro/server/data/index.js +68887 -68064
- experimaestro/server/data/index.js.map +1 -1
- experimaestro/settings.py +45 -5
- experimaestro/sphinx/__init__.py +7 -17
- experimaestro/taskglobals.py +7 -2
- experimaestro/tests/core/__init__.py +0 -0
- experimaestro/tests/core/test_generics.py +206 -0
- experimaestro/tests/definitions_types.py +5 -3
- experimaestro/tests/launchers/bin/sbatch +34 -7
- experimaestro/tests/launchers/bin/srun +5 -0
- experimaestro/tests/launchers/common.py +17 -5
- experimaestro/tests/launchers/config_slurm/launchers.py +25 -0
- experimaestro/tests/restart.py +10 -5
- experimaestro/tests/tasks/all.py +23 -10
- experimaestro/tests/tasks/foreign.py +2 -4
- experimaestro/tests/test_checkers.py +2 -2
- experimaestro/tests/test_dependencies.py +11 -17
- experimaestro/tests/test_experiment.py +73 -0
- experimaestro/tests/test_file_progress.py +425 -0
- experimaestro/tests/test_file_progress_integration.py +477 -0
- experimaestro/tests/test_findlauncher.py +12 -5
- experimaestro/tests/test_forward.py +5 -5
- experimaestro/tests/test_generators.py +93 -0
- experimaestro/tests/test_identifier.py +182 -158
- experimaestro/tests/test_instance.py +19 -27
- experimaestro/tests/test_objects.py +13 -20
- experimaestro/tests/test_outputs.py +6 -6
- experimaestro/tests/test_param.py +68 -30
- experimaestro/tests/test_progress.py +4 -4
- experimaestro/tests/test_serializers.py +24 -64
- experimaestro/tests/test_ssh.py +7 -0
- experimaestro/tests/test_tags.py +50 -21
- experimaestro/tests/test_tasks.py +42 -51
- experimaestro/tests/test_tokens.py +11 -8
- experimaestro/tests/test_types.py +24 -21
- experimaestro/tests/test_validation.py +67 -110
- experimaestro/tests/token_reschedule.py +1 -1
- experimaestro/tokens.py +24 -13
- experimaestro/tools/diff.py +8 -1
- experimaestro/typingutils.py +20 -11
- experimaestro/utils/asyncio.py +6 -2
- experimaestro/utils/multiprocessing.py +44 -0
- experimaestro/utils/resources.py +11 -3
- {experimaestro-1.5.1.dist-info → experimaestro-2.0.0a8.dist-info}/METADATA +28 -36
- experimaestro-2.0.0a8.dist-info/RECORD +166 -0
- {experimaestro-1.5.1.dist-info → experimaestro-2.0.0a8.dist-info}/WHEEL +1 -1
- {experimaestro-1.5.1.dist-info → experimaestro-2.0.0a8.dist-info}/entry_points.txt +0 -4
- experimaestro/launchers/slurm/cli.py +0 -29
- experimaestro/launchers/slurm/configuration.py +0 -597
- experimaestro/scheduler/environment.py +0 -94
- experimaestro/server/data/016b4a6cdced82ab3aa1.ttf +0 -0
- experimaestro/server/data/50701fbb8177c2dde530.ttf +0 -0
- experimaestro/server/data/878f31251d960bd6266f.woff2 +0 -0
- experimaestro/server/data/b041b1fa4fe241b23445.woff2 +0 -0
- experimaestro/server/data/b6879d41b0852f01ed5b.woff2 +0 -0
- experimaestro/server/data/d75e3fd1eb12e9bd6655.ttf +0 -0
- experimaestro/tests/launchers/config_slurm/launchers.yaml +0 -134
- experimaestro/utils/yaml.py +0 -202
- experimaestro-1.5.1.dist-info/RECORD +0 -148
- {experimaestro-1.5.1.dist-info → experimaestro-2.0.0a8.dist-info/licenses}/LICENSE +0 -0
|
@@ -4,16 +4,14 @@ import json
|
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Dict, List, Optional
|
|
6
6
|
from experimaestro import (
|
|
7
|
-
config,
|
|
8
7
|
Param,
|
|
9
|
-
param,
|
|
10
8
|
deprecate,
|
|
11
9
|
Config,
|
|
12
10
|
Constant,
|
|
13
11
|
Meta,
|
|
14
12
|
Option,
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
PathGenerator,
|
|
14
|
+
field,
|
|
17
15
|
Task,
|
|
18
16
|
LightweightTask,
|
|
19
17
|
)
|
|
@@ -40,6 +38,11 @@ class C(Config):
|
|
|
40
38
|
b: Param[int]
|
|
41
39
|
|
|
42
40
|
|
|
41
|
+
class CField(Config):
|
|
42
|
+
a: Param[int] = field(default_factory=lambda: 1)
|
|
43
|
+
b: Param[int]
|
|
44
|
+
|
|
45
|
+
|
|
43
46
|
class D(Config):
|
|
44
47
|
a: Param[A]
|
|
45
48
|
|
|
@@ -48,8 +51,7 @@ class Float(Config):
|
|
|
48
51
|
value: Param[float]
|
|
49
52
|
|
|
50
53
|
|
|
51
|
-
|
|
52
|
-
class Values:
|
|
54
|
+
class Values(Config):
|
|
53
55
|
value1: Param[float]
|
|
54
56
|
value2: Param[float]
|
|
55
57
|
|
|
@@ -66,68 +68,72 @@ def assert_notequal(a, b, message=""):
|
|
|
66
68
|
assert getidentifier(a) != getidentifier(b), message
|
|
67
69
|
|
|
68
70
|
|
|
69
|
-
def
|
|
70
|
-
assert_equal(A(a=1), A(a=1))
|
|
71
|
+
def test_identifier_int():
|
|
72
|
+
assert_equal(A.C(a=1), A.C(a=1))
|
|
71
73
|
|
|
72
74
|
|
|
73
|
-
def
|
|
74
|
-
assert_notequal(A(a=1), B(a=1))
|
|
75
|
+
def test_identifier_different_type():
|
|
76
|
+
assert_notequal(A.C(a=1), B.C(a=1))
|
|
75
77
|
|
|
76
78
|
|
|
77
|
-
def
|
|
78
|
-
assert_equal(Values(value1=1, value2=2), Values(value2=2, value1=1))
|
|
79
|
+
def test_identifier_order():
|
|
80
|
+
assert_equal(Values.C(value1=1, value2=2), Values.C(value2=2, value1=1))
|
|
79
81
|
|
|
80
82
|
|
|
81
|
-
def
|
|
82
|
-
assert_equal(C(a=1, b=2), C(b=2))
|
|
83
|
+
def test_identifier_default():
|
|
84
|
+
assert_equal(C.C(a=1, b=2), C.C(b=2))
|
|
83
85
|
|
|
84
86
|
|
|
85
|
-
def
|
|
86
|
-
assert_equal(
|
|
87
|
+
def test_identifier_default_field():
|
|
88
|
+
assert_equal(CField.C(a=1, b=2), CField.C(b=2))
|
|
87
89
|
|
|
88
90
|
|
|
89
|
-
def
|
|
90
|
-
assert_equal(
|
|
91
|
+
def test_identifier_inner_eq():
|
|
92
|
+
assert_equal(D.C(a=A.C(a=1)), D.C(a=A.C(a=1)))
|
|
91
93
|
|
|
92
94
|
|
|
93
|
-
def
|
|
94
|
-
assert_equal(Float(value=1
|
|
95
|
+
def test_identifier_float():
|
|
96
|
+
assert_equal(Float.C(value=1), Float.C(value=1))
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_identifier_float2():
|
|
100
|
+
assert_equal(Float.C(value=1.0), Float.C(value=1))
|
|
95
101
|
|
|
96
102
|
|
|
97
103
|
# --- Argument name
|
|
98
104
|
|
|
99
105
|
|
|
100
|
-
def
|
|
106
|
+
def test_identifier_name():
|
|
101
107
|
"""The identifier fully determines the hash code"""
|
|
102
108
|
|
|
103
|
-
|
|
104
|
-
|
|
109
|
+
class Config0(Config):
|
|
110
|
+
__xpmid__ = "test.identifier.argumentname"
|
|
105
111
|
a: Param[int]
|
|
106
112
|
|
|
107
|
-
|
|
108
|
-
|
|
113
|
+
class Config1(Config):
|
|
114
|
+
__xpmid__ = "test.identifier.argumentname"
|
|
109
115
|
b: Param[int]
|
|
110
116
|
|
|
111
|
-
|
|
112
|
-
|
|
117
|
+
class Config3(Config):
|
|
118
|
+
__xpmid__ = "test.identifier.argumentname"
|
|
113
119
|
a: Param[int]
|
|
114
120
|
|
|
115
|
-
assert_notequal(Config0(a=2), Config1(b=2))
|
|
116
|
-
assert_equal(Config0(a=2), Config3(a=2))
|
|
121
|
+
assert_notequal(Config0.C(a=2), Config1.C(b=2))
|
|
122
|
+
assert_equal(Config0.C(a=2), Config3.C(a=2))
|
|
117
123
|
|
|
118
124
|
|
|
119
125
|
# --- Test option
|
|
120
126
|
|
|
121
127
|
|
|
122
|
-
def
|
|
123
|
-
|
|
124
|
-
|
|
128
|
+
def test_identifier_option():
|
|
129
|
+
class OptionConfig(Config):
|
|
130
|
+
__xpmid__ = "test.identifier.option"
|
|
125
131
|
a: Param[int]
|
|
126
132
|
b: Option[int] = 1
|
|
127
133
|
|
|
128
|
-
assert_notequal(OptionConfig(a=2), OptionConfig(a=1))
|
|
129
|
-
assert_equal(OptionConfig(a=1, b=2), OptionConfig(a=1))
|
|
130
|
-
assert_equal(OptionConfig(a=1, b=2), OptionConfig(a=1, b=2))
|
|
134
|
+
assert_notequal(OptionConfig.C(a=2), OptionConfig.C(a=1))
|
|
135
|
+
assert_equal(OptionConfig.C(a=1, b=2), OptionConfig.C(a=1))
|
|
136
|
+
assert_equal(OptionConfig.C(a=1, b=2), OptionConfig.C(a=1, b=2))
|
|
131
137
|
|
|
132
138
|
|
|
133
139
|
# --- Dictionnary
|
|
@@ -142,48 +148,50 @@ def test_identifier_dict():
|
|
|
142
148
|
class A(Config):
|
|
143
149
|
bs: Param[Dict[str, B]]
|
|
144
150
|
|
|
145
|
-
assert_equal(A(bs={"b1": B(x=1)}), A(bs={"b1": B(x=1)}))
|
|
146
|
-
assert_equal(
|
|
151
|
+
assert_equal(A.C(bs={"b1": B.C(x=1)}), A.C(bs={"b1": B.C(x=1)}))
|
|
152
|
+
assert_equal(
|
|
153
|
+
A.C(bs={"b1": B.C(x=1), "b2": B.C(x=2)}),
|
|
154
|
+
A.C(bs={"b2": B.C(x=2), "b1": B.C(x=1)}),
|
|
155
|
+
)
|
|
147
156
|
|
|
148
|
-
assert_notequal(A(bs={"b1": B(x=1)}), A(bs={"b1": B(x=2)}))
|
|
149
|
-
assert_notequal(A(bs={"b1": B(x=1)}), A(bs={"b2": B(x=1)}))
|
|
157
|
+
assert_notequal(A.C(bs={"b1": B.C(x=1)}), A.C(bs={"b1": B.C(x=2)}))
|
|
158
|
+
assert_notequal(A.C(bs={"b1": B.C(x=1)}), A.C(bs={"b2": B.C(x=1)}))
|
|
150
159
|
|
|
151
160
|
|
|
152
161
|
# --- Ignore paths
|
|
153
162
|
|
|
154
163
|
|
|
155
|
-
|
|
156
|
-
class TypeWithPath:
|
|
164
|
+
class TypeWithPath(Config):
|
|
157
165
|
a: Param[int]
|
|
158
166
|
path: Param[Path]
|
|
159
167
|
|
|
160
168
|
|
|
161
|
-
def
|
|
169
|
+
def test_identifier_path():
|
|
162
170
|
"""Path should be ignored"""
|
|
163
|
-
assert_equal(TypeWithPath(a=1, path="/a/b"), TypeWithPath(a=1, path="/c/d"))
|
|
164
|
-
assert_notequal(TypeWithPath(a=2, path="/a/b"), TypeWithPath(a=1, path="/c/d"))
|
|
171
|
+
assert_equal(TypeWithPath.C(a=1, path="/a/b"), TypeWithPath.C(a=1, path="/c/d"))
|
|
172
|
+
assert_notequal(TypeWithPath.C(a=2, path="/a/b"), TypeWithPath.C(a=1, path="/c/d"))
|
|
165
173
|
|
|
166
174
|
|
|
167
175
|
# --- Test with added arguments
|
|
168
176
|
|
|
169
177
|
|
|
170
|
-
def
|
|
178
|
+
def test_identifier_pathoption():
|
|
171
179
|
"""Path arguments should be ignored"""
|
|
172
180
|
|
|
173
|
-
|
|
174
|
-
|
|
181
|
+
class A_with_path(Config):
|
|
182
|
+
__xpmid__ = "pathoption_test"
|
|
175
183
|
a: Param[int]
|
|
176
|
-
path:
|
|
184
|
+
path: Meta[Path] = field(default_factory=PathGenerator("path"))
|
|
177
185
|
|
|
178
|
-
|
|
179
|
-
|
|
186
|
+
class A_without_path(Config):
|
|
187
|
+
__xpmid__ = "pathoption_test"
|
|
180
188
|
a: Param[int]
|
|
181
189
|
|
|
182
|
-
assert_equal(A_with_path(a=1), A_without_path(a=1))
|
|
190
|
+
assert_equal(A_with_path.C(a=1), A_without_path.C(a=1))
|
|
183
191
|
|
|
184
192
|
|
|
185
193
|
def test_identifier_enum():
|
|
186
|
-
"""
|
|
194
|
+
"""test enum parameters"""
|
|
187
195
|
from enum import Enum
|
|
188
196
|
|
|
189
197
|
class EnumParam(Enum):
|
|
@@ -193,8 +201,8 @@ def test_identifier_enum():
|
|
|
193
201
|
class EnumConfig(Config):
|
|
194
202
|
a: Param[EnumParam]
|
|
195
203
|
|
|
196
|
-
assert_notequal(EnumConfig(a=EnumParam.FIRST), EnumConfig(a=EnumParam.SECOND))
|
|
197
|
-
assert_equal(EnumConfig(a=EnumParam.FIRST), EnumConfig(a=EnumParam.FIRST))
|
|
204
|
+
assert_notequal(EnumConfig.C(a=EnumParam.FIRST), EnumConfig.C(a=EnumParam.SECOND))
|
|
205
|
+
assert_equal(EnumConfig.C(a=EnumParam.FIRST), EnumConfig.C(a=EnumParam.FIRST))
|
|
198
206
|
|
|
199
207
|
|
|
200
208
|
def test_identifier_addnone():
|
|
@@ -210,29 +218,28 @@ def test_identifier_addnone():
|
|
|
210
218
|
class A(Config):
|
|
211
219
|
__xpmid__ = "defaultnone"
|
|
212
220
|
|
|
213
|
-
assert_equal(A_with_b(), A())
|
|
214
|
-
assert_notequal(A_with_b(b=B(x=1)), A())
|
|
221
|
+
assert_equal(A_with_b.C(), A.C())
|
|
222
|
+
assert_notequal(A_with_b.C(b=B.C(x=1)), A.C())
|
|
215
223
|
|
|
216
224
|
|
|
217
|
-
def
|
|
225
|
+
def test_identifier_defaultnew():
|
|
218
226
|
"""Path arguments should be ignored"""
|
|
219
227
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
@config("defaultnew")
|
|
223
|
-
class A_with_b:
|
|
224
|
-
pass
|
|
228
|
+
class A_with_b(Config):
|
|
229
|
+
__xpmid__ = "defaultnew"
|
|
225
230
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
class A:
|
|
229
|
-
pass
|
|
231
|
+
a: Param[int]
|
|
232
|
+
b: Param[int] = 1
|
|
230
233
|
|
|
231
|
-
|
|
232
|
-
|
|
234
|
+
class A(Config):
|
|
235
|
+
__xpmid__ = "defaultnew"
|
|
236
|
+
a: Param[int]
|
|
237
|
+
|
|
238
|
+
assert_equal(A_with_b.C(a=1, b=1), A.C(a=1))
|
|
239
|
+
assert_equal(A_with_b.C(a=1), A.C(a=1))
|
|
233
240
|
|
|
234
241
|
|
|
235
|
-
def
|
|
242
|
+
def test_identifier_taskconfigidentifier():
|
|
236
243
|
"""Test whether the embedded task arguments make the configuration different"""
|
|
237
244
|
|
|
238
245
|
class MyConfig(Config):
|
|
@@ -242,36 +249,36 @@ def test_taskconfigidentifier():
|
|
|
242
249
|
x: Param[int]
|
|
243
250
|
|
|
244
251
|
def task_outputs(self, dep):
|
|
245
|
-
return dep(MyConfig(a=1))
|
|
252
|
+
return dep(MyConfig.C(a=1))
|
|
246
253
|
|
|
247
254
|
assert_equal(
|
|
248
|
-
MyTask(x=1).submit(run_mode=RunMode.DRY_RUN),
|
|
249
|
-
MyTask(x=1).submit(run_mode=RunMode.DRY_RUN),
|
|
255
|
+
MyTask.C(x=1).submit(run_mode=RunMode.DRY_RUN),
|
|
256
|
+
MyTask.C(x=1).submit(run_mode=RunMode.DRY_RUN),
|
|
250
257
|
)
|
|
251
258
|
assert_notequal(
|
|
252
|
-
MyTask(x=2).submit(run_mode=RunMode.DRY_RUN),
|
|
253
|
-
MyTask(x=1).submit(run_mode=RunMode.DRY_RUN),
|
|
259
|
+
MyTask.C(x=2).submit(run_mode=RunMode.DRY_RUN),
|
|
260
|
+
MyTask.C(x=1).submit(run_mode=RunMode.DRY_RUN),
|
|
254
261
|
)
|
|
255
262
|
|
|
256
263
|
|
|
257
|
-
def
|
|
264
|
+
def test_identifier_constant():
|
|
258
265
|
"""Test if constants are taken into account for signature computation"""
|
|
259
266
|
|
|
260
|
-
|
|
261
|
-
|
|
267
|
+
class A1(Config):
|
|
268
|
+
__xpmid__ = "test.constant"
|
|
262
269
|
version: Constant[int] = 1
|
|
263
270
|
|
|
264
|
-
|
|
265
|
-
|
|
271
|
+
class A1bis(Config):
|
|
272
|
+
__xpmid__ = "test.constant"
|
|
266
273
|
version: Constant[int] = 1
|
|
267
274
|
|
|
268
|
-
assert_equal(A1(), A1bis())
|
|
275
|
+
assert_equal(A1.C(), A1bis.C())
|
|
269
276
|
|
|
270
|
-
|
|
271
|
-
|
|
277
|
+
class A2(Config):
|
|
278
|
+
__xpmid__ = "test.constant"
|
|
272
279
|
version: Constant[int] = 2
|
|
273
280
|
|
|
274
|
-
assert_notequal(A1(), A2())
|
|
281
|
+
assert_notequal(A1.C(), A2.C())
|
|
275
282
|
|
|
276
283
|
|
|
277
284
|
def test_identifier_deprecated_class():
|
|
@@ -289,10 +296,12 @@ def test_identifier_deprecated_class():
|
|
|
289
296
|
__xpmid__ = "derived"
|
|
290
297
|
|
|
291
298
|
assert_notequal(
|
|
292
|
-
NewConfig(), DerivedConfig(), "A derived configuration has another ID"
|
|
299
|
+
NewConfig.C(), DerivedConfig.C(), "A derived configuration has another ID"
|
|
293
300
|
)
|
|
294
301
|
assert_equal(
|
|
295
|
-
NewConfig(),
|
|
302
|
+
NewConfig.C(),
|
|
303
|
+
OldConfig.C(),
|
|
304
|
+
"Deprecated and new configuration have the same ID",
|
|
296
305
|
)
|
|
297
306
|
|
|
298
307
|
|
|
@@ -304,7 +313,7 @@ def test_identifier_deprecated_attribute():
|
|
|
304
313
|
def value(self, x):
|
|
305
314
|
self.values = [x]
|
|
306
315
|
|
|
307
|
-
assert_equal(Values(values=[1]), Values(value=1))
|
|
316
|
+
assert_equal(Values.C(values=[1]), Values.C(value=1))
|
|
308
317
|
|
|
309
318
|
|
|
310
319
|
class MetaA(Config):
|
|
@@ -327,26 +336,30 @@ def test_identifier_meta():
|
|
|
327
336
|
params: Param[Dict[str, MetaA]]
|
|
328
337
|
|
|
329
338
|
# As meta
|
|
330
|
-
assert_notequal(B(a=MetaA(x=1)), B(a=MetaA(x=2)))
|
|
331
|
-
assert_equal(B(a=setmeta(MetaA(x=1), True)), B(a=setmeta(MetaA(x=2), True)))
|
|
339
|
+
assert_notequal(B.C(a=MetaA.C(x=1)), B.C(a=MetaA.C(x=2)))
|
|
340
|
+
assert_equal(B.C(a=setmeta(MetaA.C(x=1), True)), B.C(a=setmeta(MetaA.C(x=2), True)))
|
|
332
341
|
|
|
333
342
|
# As parameter
|
|
334
|
-
assert_equal(C(a=MetaA(x=1)), C(a=MetaA(x=2)))
|
|
335
|
-
assert_notequal(
|
|
343
|
+
assert_equal(C.C(a=MetaA.C(x=1)), C.C(a=MetaA.C(x=2)))
|
|
344
|
+
assert_notequal(
|
|
345
|
+
C.C(a=setmeta(MetaA.C(x=1), False)), C.C(a=setmeta(MetaA.C(x=2), False))
|
|
346
|
+
)
|
|
336
347
|
|
|
337
348
|
# Array with mixed
|
|
338
349
|
assert_equal(
|
|
339
|
-
ArrayConfig(array=[MetaA(x=1)]),
|
|
340
|
-
ArrayConfig(array=[MetaA(x=1), setmeta(MetaA(x=2), True)]),
|
|
350
|
+
ArrayConfig.C(array=[MetaA.C(x=1)]),
|
|
351
|
+
ArrayConfig.C(array=[MetaA.C(x=1), setmeta(MetaA.C(x=2), True)]),
|
|
341
352
|
)
|
|
342
353
|
|
|
343
354
|
# Array with empty list
|
|
344
|
-
assert_equal(
|
|
355
|
+
assert_equal(
|
|
356
|
+
ArrayConfig.C(array=[]), ArrayConfig.C(array=[setmeta(MetaA.C(x=2), True)])
|
|
357
|
+
)
|
|
345
358
|
|
|
346
359
|
# Dict with mixed
|
|
347
360
|
assert_equal(
|
|
348
|
-
DictConfig(params={"a": MetaA(x=1)}),
|
|
349
|
-
DictConfig(params={"a": MetaA(x=1), "b": setmeta(MetaA(x=2), True)}),
|
|
361
|
+
DictConfig.C(params={"a": MetaA.C(x=1)}),
|
|
362
|
+
DictConfig.C(params={"a": MetaA.C(x=1), "b": setmeta(MetaA.C(x=2), True)}),
|
|
350
363
|
)
|
|
351
364
|
|
|
352
365
|
|
|
@@ -355,14 +368,14 @@ def test_identifier_meta_default_dict():
|
|
|
355
368
|
params: Param[Dict[str, MetaA]] = {}
|
|
356
369
|
|
|
357
370
|
assert_equal(
|
|
358
|
-
DictConfig(params={}),
|
|
359
|
-
DictConfig(params={"b": setmeta(MetaA(x=2), True)}),
|
|
371
|
+
DictConfig.C(params={}),
|
|
372
|
+
DictConfig.C(params={"b": setmeta(MetaA.C(x=2), True)}),
|
|
360
373
|
)
|
|
361
374
|
|
|
362
375
|
# Dict with mixed
|
|
363
376
|
assert_equal(
|
|
364
|
-
DictConfig(params={"a": MetaA(x=1)}),
|
|
365
|
-
DictConfig(params={"a": MetaA(x=1), "b": setmeta(MetaA(x=2), True)}),
|
|
377
|
+
DictConfig.C(params={"a": MetaA.C(x=1)}),
|
|
378
|
+
DictConfig.C(params={"a": MetaA.C(x=1), "b": setmeta(MetaA.C(x=2), True)}),
|
|
366
379
|
)
|
|
367
380
|
|
|
368
381
|
|
|
@@ -372,44 +385,14 @@ def test_identifier_meta_default_array():
|
|
|
372
385
|
|
|
373
386
|
# Array (with default) with mixed
|
|
374
387
|
assert_equal(
|
|
375
|
-
ArrayConfigWithDefault(array=[MetaA(x=1)]),
|
|
376
|
-
ArrayConfigWithDefault(array=[MetaA(x=1), setmeta(MetaA(x=2), True)]),
|
|
388
|
+
ArrayConfigWithDefault.C(array=[MetaA.C(x=1)]),
|
|
389
|
+
ArrayConfigWithDefault.C(array=[MetaA.C(x=1), setmeta(MetaA.C(x=2), True)]),
|
|
377
390
|
)
|
|
378
391
|
# Array (with default) with empty list
|
|
379
392
|
assert_equal(
|
|
380
|
-
ArrayConfigWithDefault(array=[]),
|
|
381
|
-
ArrayConfigWithDefault(array=[setmeta(MetaA(x=2), True)]),
|
|
382
|
-
)
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
def test_identifier_pre_task():
|
|
386
|
-
class MyConfig(Config):
|
|
387
|
-
pass
|
|
388
|
-
|
|
389
|
-
class IdentifierPreLightTask(LightweightTask):
|
|
390
|
-
pass
|
|
391
|
-
|
|
392
|
-
class IdentifierPreTask(Task):
|
|
393
|
-
x: Param[MyConfig]
|
|
394
|
-
|
|
395
|
-
task = IdentifierPreTask(x=MyConfig()).submit(run_mode=RunMode.DRY_RUN)
|
|
396
|
-
task_with_pre = (
|
|
397
|
-
IdentifierPreTask(x=MyConfig())
|
|
398
|
-
.add_pretasks(IdentifierPreLightTask())
|
|
399
|
-
.submit(run_mode=RunMode.DRY_RUN)
|
|
393
|
+
ArrayConfigWithDefault.C(array=[]),
|
|
394
|
+
ArrayConfigWithDefault.C(array=[setmeta(MetaA.C(x=2), True)]),
|
|
400
395
|
)
|
|
401
|
-
task_with_pre_2 = (
|
|
402
|
-
IdentifierPreTask(x=MyConfig())
|
|
403
|
-
.add_pretasks(IdentifierPreLightTask())
|
|
404
|
-
.submit(run_mode=RunMode.DRY_RUN)
|
|
405
|
-
)
|
|
406
|
-
task_with_pre_3 = IdentifierPreTask(
|
|
407
|
-
x=MyConfig().add_pretasks(IdentifierPreLightTask())
|
|
408
|
-
).submit(run_mode=RunMode.DRY_RUN)
|
|
409
|
-
|
|
410
|
-
assert_notequal(task, task_with_pre, "No pre-task")
|
|
411
|
-
assert_equal(task_with_pre, task_with_pre_2, "Same parameters")
|
|
412
|
-
assert_equal(task_with_pre, task_with_pre_3, "Pre-tasks are order-less")
|
|
413
396
|
|
|
414
397
|
|
|
415
398
|
def test_identifier_init_task():
|
|
@@ -422,26 +405,67 @@ def test_identifier_init_task():
|
|
|
422
405
|
class IdentifierInitTask2(Task):
|
|
423
406
|
pass
|
|
424
407
|
|
|
425
|
-
class
|
|
408
|
+
class IdentifierTask(Task):
|
|
426
409
|
x: Param[MyConfig]
|
|
427
410
|
|
|
428
|
-
task =
|
|
429
|
-
task_with_pre =
|
|
411
|
+
task = IdentifierTask.C(x=MyConfig.C()).submit(run_mode=RunMode.DRY_RUN)
|
|
412
|
+
task_with_pre = IdentifierTask.C(x=MyConfig.C()).submit(
|
|
430
413
|
run_mode=RunMode.DRY_RUN,
|
|
431
|
-
init_tasks=[IdentifierInitTask(), IdentifierInitTask2()],
|
|
414
|
+
init_tasks=[IdentifierInitTask.C(), IdentifierInitTask2.C()],
|
|
432
415
|
)
|
|
433
|
-
task_with_pre_2 =
|
|
416
|
+
task_with_pre_2 = IdentifierTask.C(x=MyConfig.C()).submit(
|
|
434
417
|
run_mode=RunMode.DRY_RUN,
|
|
435
|
-
init_tasks=[IdentifierInitTask(), IdentifierInitTask2()],
|
|
418
|
+
init_tasks=[IdentifierInitTask.C(), IdentifierInitTask2.C()],
|
|
436
419
|
)
|
|
437
|
-
task_with_pre_3 =
|
|
420
|
+
task_with_pre_3 = IdentifierTask.C(x=MyConfig.C()).submit(
|
|
438
421
|
run_mode=RunMode.DRY_RUN,
|
|
439
|
-
init_tasks=[IdentifierInitTask2(), IdentifierInitTask()],
|
|
422
|
+
init_tasks=[IdentifierInitTask2.C(), IdentifierInitTask.C()],
|
|
440
423
|
)
|
|
441
424
|
|
|
442
|
-
assert_notequal(task, task_with_pre, "
|
|
425
|
+
assert_notequal(task, task_with_pre, "Should be different with init-task")
|
|
443
426
|
assert_equal(task_with_pre, task_with_pre_2, "Same parameters")
|
|
444
|
-
assert_notequal(task_with_pre, task_with_pre_3, "
|
|
427
|
+
assert_notequal(task_with_pre, task_with_pre_3, "Other parameters")
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def test_identifier_init_task_dep():
|
|
431
|
+
class Loader(LightweightTask):
|
|
432
|
+
param1: Param[float]
|
|
433
|
+
|
|
434
|
+
def execute(self):
|
|
435
|
+
pass
|
|
436
|
+
|
|
437
|
+
class FirstTask(Task):
|
|
438
|
+
def task_outputs(self, dep):
|
|
439
|
+
return dep(Loader.C(param1=1))
|
|
440
|
+
|
|
441
|
+
def execute(self):
|
|
442
|
+
pass
|
|
443
|
+
|
|
444
|
+
class SecondTask(Task):
|
|
445
|
+
param3: Param[int]
|
|
446
|
+
|
|
447
|
+
def execute(self):
|
|
448
|
+
pass
|
|
449
|
+
|
|
450
|
+
# Two identical tasks
|
|
451
|
+
task_a_1 = FirstTask.C()
|
|
452
|
+
task_a_2 = FirstTask.C()
|
|
453
|
+
assert_equal(task_a_1, task_a_2)
|
|
454
|
+
|
|
455
|
+
# We process them with two different init tasks
|
|
456
|
+
loader_1 = task_a_1.submit(
|
|
457
|
+
init_tasks=[Loader.C(param1=0.5)], run_mode=RunMode.DRY_RUN
|
|
458
|
+
)
|
|
459
|
+
loader_2 = task_a_2.submit(
|
|
460
|
+
init_tasks=[Loader.C(param1=5)], run_mode=RunMode.DRY_RUN
|
|
461
|
+
)
|
|
462
|
+
assert_notequal(loader_1, loader_2)
|
|
463
|
+
|
|
464
|
+
# Now, we process
|
|
465
|
+
c_1 = SecondTask.C(param3=2).submit(init_tasks=[loader_1], run_mode=RunMode.DRY_RUN)
|
|
466
|
+
|
|
467
|
+
c_2 = SecondTask.C(param3=2).submit(init_tasks=[loader_2], run_mode=RunMode.DRY_RUN)
|
|
468
|
+
assert_notequal(c_1, c_2)
|
|
445
469
|
|
|
446
470
|
|
|
447
471
|
# --- Check configuration reloads
|
|
@@ -459,7 +483,7 @@ def check_reload(config):
|
|
|
459
483
|
new_config = ConfigInformation.fromParameters(
|
|
460
484
|
data, as_instance=False, discard_id=True
|
|
461
485
|
)
|
|
462
|
-
assert new_config.__xpm__.
|
|
486
|
+
assert new_config.__xpm__._identifier is None
|
|
463
487
|
new_identifier = new_config.__xpm__.identifier.all
|
|
464
488
|
|
|
465
489
|
assert new_identifier == old_identifier
|
|
@@ -471,14 +495,14 @@ class IdentifierReloadConfig(Config):
|
|
|
471
495
|
|
|
472
496
|
def test_identifier_reload_config():
|
|
473
497
|
# Creates the configuration
|
|
474
|
-
check_reload(IdentifierReloadConfig(id="123"))
|
|
498
|
+
check_reload(IdentifierReloadConfig.C(id="123"))
|
|
475
499
|
|
|
476
500
|
|
|
477
501
|
class IdentifierReload(Task):
|
|
478
502
|
id: Param[str]
|
|
479
503
|
|
|
480
|
-
def task_outputs(self, dep):
|
|
481
|
-
return IdentifierReloadConfig(id=self.id)
|
|
504
|
+
def task_outputs(self, dep) -> IdentifierReloadConfig.C:
|
|
505
|
+
return IdentifierReloadConfig.C(id=self.id)
|
|
482
506
|
|
|
483
507
|
|
|
484
508
|
class IdentifierReloadDerived(Config):
|
|
@@ -489,8 +513,8 @@ def test_identifier_reload_taskoutput():
|
|
|
489
513
|
"""When using a task output, the identifier should not be different"""
|
|
490
514
|
|
|
491
515
|
# Creates the configuration
|
|
492
|
-
task = IdentifierReload(id="123").submit(run_mode=RunMode.DRY_RUN)
|
|
493
|
-
config = IdentifierReloadDerived(task=task)
|
|
516
|
+
task = IdentifierReload.C(id="123").submit(run_mode=RunMode.DRY_RUN)
|
|
517
|
+
config = IdentifierReloadDerived.C(task=task)
|
|
494
518
|
check_reload(config)
|
|
495
519
|
|
|
496
520
|
|
|
@@ -511,9 +535,9 @@ def test_identifier_reload_task_direct():
|
|
|
511
535
|
"""When using a direct task output, the identifier should not be different"""
|
|
512
536
|
|
|
513
537
|
# Creates the configuration
|
|
514
|
-
task = IdentifierReloadTask(id="123").submit(run_mode=RunMode.DRY_RUN)
|
|
515
|
-
config = IdentifierReloadTaskDerived(
|
|
516
|
-
task=task, other=IdentifierReloadTaskConfig(x=2)
|
|
538
|
+
task = IdentifierReloadTask.C(id="123").submit(run_mode=RunMode.DRY_RUN)
|
|
539
|
+
config = IdentifierReloadTaskDerived.C(
|
|
540
|
+
task=task, other=IdentifierReloadTaskConfig.C(x=2)
|
|
517
541
|
)
|
|
518
542
|
check_reload(config)
|
|
519
543
|
|
|
@@ -521,9 +545,9 @@ def test_identifier_reload_task_direct():
|
|
|
521
545
|
def test_identifier_reload_meta():
|
|
522
546
|
"""Test identifier don't change when using meta"""
|
|
523
547
|
# Creates the configuration
|
|
524
|
-
task = IdentifierReloadTask(id="123").submit(run_mode=RunMode.DRY_RUN)
|
|
525
|
-
config = IdentifierReloadTaskDerived(
|
|
526
|
-
task=task, other=setmeta(IdentifierReloadTaskConfig(x=2), True)
|
|
548
|
+
task = IdentifierReloadTask.C(id="123").submit(run_mode=RunMode.DRY_RUN)
|
|
549
|
+
config = IdentifierReloadTaskDerived.C(
|
|
550
|
+
task=task, other=setmeta(IdentifierReloadTaskConfig.C(x=2), True)
|
|
527
551
|
)
|
|
528
552
|
check_reload(config)
|
|
529
553
|
|
|
@@ -542,9 +566,9 @@ class LoopC(Config):
|
|
|
542
566
|
|
|
543
567
|
|
|
544
568
|
def test_identifier_loop():
|
|
545
|
-
c = LoopC()
|
|
546
|
-
b = LoopB(param_c=c)
|
|
547
|
-
a = LoopA(param_b=b)
|
|
569
|
+
c = LoopC.C()
|
|
570
|
+
b = LoopB.C(param_c=c)
|
|
571
|
+
a = LoopA.C(param_b=b)
|
|
548
572
|
c.param_a = a
|
|
549
573
|
c.param_b = b
|
|
550
574
|
|
|
@@ -1,35 +1,32 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
|
-
from experimaestro import
|
|
3
|
-
from experimaestro.core.objects import
|
|
2
|
+
from experimaestro import Param, Config
|
|
3
|
+
from experimaestro.core.objects import ConfigMixin
|
|
4
4
|
from experimaestro.core.serializers import SerializationLWTask
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
class A:
|
|
7
|
+
class A(Config):
|
|
9
8
|
x: Param[int] = 1
|
|
10
9
|
|
|
11
10
|
|
|
12
|
-
@config()
|
|
13
11
|
class A1(A):
|
|
14
12
|
pass
|
|
15
13
|
|
|
16
14
|
|
|
17
|
-
|
|
18
|
-
class B:
|
|
15
|
+
class B(Config):
|
|
19
16
|
a: Param[A]
|
|
20
17
|
|
|
21
18
|
|
|
22
19
|
def test_simple_instance():
|
|
23
|
-
a = A1(x=1)
|
|
24
|
-
b = B(a=a)
|
|
20
|
+
a = A1.C(x=1)
|
|
21
|
+
b = B.C(a=a)
|
|
25
22
|
b = b.instance()
|
|
26
23
|
|
|
27
|
-
assert not isinstance(b,
|
|
28
|
-
assert isinstance(b, B.__xpmtype__.
|
|
24
|
+
assert not isinstance(b, ConfigMixin)
|
|
25
|
+
assert isinstance(b, B.__xpmtype__.value_type)
|
|
29
26
|
|
|
30
|
-
assert not isinstance(b.a,
|
|
31
|
-
assert isinstance(b.a, A1.__xpmtype__.
|
|
32
|
-
assert isinstance(b.a, A.__xpmtype__.
|
|
27
|
+
assert not isinstance(b.a, ConfigMixin)
|
|
28
|
+
assert isinstance(b.a, A1.__xpmtype__.value_type)
|
|
29
|
+
assert isinstance(b.a, A.__xpmtype__.value_type)
|
|
33
30
|
|
|
34
31
|
|
|
35
32
|
# --- Test pre tasks
|
|
@@ -49,18 +46,6 @@ class LoadModel(SerializationLWTask):
|
|
|
49
46
|
self.value.initialized = True
|
|
50
47
|
|
|
51
48
|
|
|
52
|
-
def test_instance_serialized():
|
|
53
|
-
model = Model()
|
|
54
|
-
model.add_pretasks(LoadModel(value=model))
|
|
55
|
-
trainer = Evaluator(model=model)
|
|
56
|
-
instance = trainer.instance()
|
|
57
|
-
|
|
58
|
-
assert isinstance(
|
|
59
|
-
instance.model, Model
|
|
60
|
-
), f"The model is not a Model but a {type(instance.model).__qualname__}"
|
|
61
|
-
assert instance.model.initialized, "The model was not initialized"
|
|
62
|
-
|
|
63
|
-
|
|
64
49
|
class ConfigWithOptional(Config):
|
|
65
50
|
x: Param[int] = 1
|
|
66
51
|
y: Param[Optional[int]]
|
|
@@ -68,6 +53,13 @@ class ConfigWithOptional(Config):
|
|
|
68
53
|
|
|
69
54
|
def test_instance_optional():
|
|
70
55
|
"""Test that optional parameters are set to None when calling instance"""
|
|
71
|
-
c = ConfigWithOptional().instance()
|
|
56
|
+
c = ConfigWithOptional.C().instance()
|
|
72
57
|
assert c.x == 1
|
|
73
58
|
assert c.y is None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_instance_keep_config():
|
|
62
|
+
evaluator = Evaluator.C(model=Model.C())
|
|
63
|
+
instance = evaluator.instance(keep=True)
|
|
64
|
+
|
|
65
|
+
assert instance.__config__ is evaluator
|