experimaestro 1.11.1__py3-none-any.whl → 2.0.0a8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of experimaestro might be problematic. Click here for more details.
- experimaestro/annotations.py +1 -1
- experimaestro/cli/__init__.py +10 -11
- experimaestro/cli/progress.py +269 -0
- experimaestro/connectors/__init__.py +2 -2
- experimaestro/core/arguments.py +20 -1
- experimaestro/core/identifier.py +21 -7
- experimaestro/core/objects/config.py +174 -274
- experimaestro/core/objects/config_walk.py +4 -6
- experimaestro/core/objects.pyi +2 -6
- experimaestro/core/serializers.py +1 -8
- experimaestro/core/types.py +35 -57
- experimaestro/launcherfinder/registry.py +3 -3
- experimaestro/mkdocs/base.py +6 -8
- experimaestro/notifications.py +12 -3
- experimaestro/progress.py +406 -0
- experimaestro/scheduler/__init__.py +18 -1
- experimaestro/scheduler/base.py +87 -906
- experimaestro/scheduler/experiment.py +387 -0
- experimaestro/scheduler/jobs.py +475 -0
- experimaestro/scheduler/signal_handler.py +32 -0
- experimaestro/scheduler/state.py +1 -1
- experimaestro/server/__init__.py +36 -5
- experimaestro/settings.py +4 -2
- experimaestro/tests/launchers/common.py +2 -2
- experimaestro/tests/restart.py +1 -1
- experimaestro/tests/tasks/all.py +7 -0
- experimaestro/tests/test_checkers.py +2 -2
- experimaestro/tests/test_dependencies.py +11 -17
- experimaestro/tests/test_experiment.py +3 -3
- experimaestro/tests/test_file_progress.py +425 -0
- experimaestro/tests/test_file_progress_integration.py +477 -0
- experimaestro/tests/test_generators.py +93 -0
- experimaestro/tests/test_identifier.py +155 -135
- experimaestro/tests/test_instance.py +13 -18
- experimaestro/tests/test_objects.py +9 -32
- experimaestro/tests/test_outputs.py +6 -6
- experimaestro/tests/test_param.py +14 -14
- experimaestro/tests/test_progress.py +4 -4
- experimaestro/tests/test_serializers.py +0 -59
- experimaestro/tests/test_tags.py +15 -15
- experimaestro/tests/test_tasks.py +42 -51
- experimaestro/tests/test_tokens.py +8 -6
- experimaestro/tests/test_types.py +10 -10
- experimaestro/tests/test_validation.py +19 -19
- experimaestro/tests/token_reschedule.py +1 -1
- experimaestro/tools/diff.py +8 -1
- experimaestro/typingutils.py +11 -2
- {experimaestro-1.11.1.dist-info → experimaestro-2.0.0a8.dist-info}/METADATA +3 -2
- {experimaestro-1.11.1.dist-info → experimaestro-2.0.0a8.dist-info}/RECORD +52 -44
- {experimaestro-1.11.1.dist-info → experimaestro-2.0.0a8.dist-info}/WHEEL +1 -1
- {experimaestro-1.11.1.dist-info → experimaestro-2.0.0a8.dist-info}/entry_points.txt +0 -0
- {experimaestro-1.11.1.dist-info → experimaestro-2.0.0a8.dist-info/licenses}/LICENSE +0 -0
|
@@ -9,7 +9,6 @@ from experimaestro import taskglobals
|
|
|
9
9
|
|
|
10
10
|
from termcolor import cprint
|
|
11
11
|
from pathlib import Path
|
|
12
|
-
import hashlib
|
|
13
12
|
import logging
|
|
14
13
|
import io
|
|
15
14
|
from enum import Enum
|
|
@@ -20,12 +19,10 @@ from typing import (
|
|
|
20
19
|
Callable,
|
|
21
20
|
ClassVar,
|
|
22
21
|
Dict,
|
|
23
|
-
Iterator,
|
|
24
22
|
List,
|
|
25
23
|
Optional,
|
|
26
24
|
Set,
|
|
27
25
|
Tuple,
|
|
28
|
-
Type,
|
|
29
26
|
TypeVar,
|
|
30
27
|
Union,
|
|
31
28
|
overload,
|
|
@@ -49,7 +46,6 @@ from .config_walk import ConfigWalk, ConfigWalkContext
|
|
|
49
46
|
from .config_utils import (
|
|
50
47
|
getqualattr,
|
|
51
48
|
add_to_path,
|
|
52
|
-
SealedError,
|
|
53
49
|
TaggedValue,
|
|
54
50
|
ObjectStore,
|
|
55
51
|
classproperty,
|
|
@@ -110,6 +106,42 @@ class WatchedOutput:
|
|
|
110
106
|
callback: Callable
|
|
111
107
|
|
|
112
108
|
|
|
109
|
+
def get_generated_paths(
|
|
110
|
+
v: Union["ConfigMixin", list, dict],
|
|
111
|
+
path: list[str] | None = None,
|
|
112
|
+
paths: list[str] | None = None,
|
|
113
|
+
) -> list[str]:
|
|
114
|
+
"""Get the list of generated paths, useful to track down those
|
|
115
|
+
|
|
116
|
+
:param path: The current path
|
|
117
|
+
:param paths: The list of generated paths so far, defaults to None
|
|
118
|
+
:return: The full list of generated paths
|
|
119
|
+
"""
|
|
120
|
+
paths = [] if paths is None else paths
|
|
121
|
+
path = [] if path is None else path
|
|
122
|
+
|
|
123
|
+
if isinstance(v, list):
|
|
124
|
+
for ix, element in enumerate(v):
|
|
125
|
+
get_generated_paths(element, path + [f"[{ix}]"], paths)
|
|
126
|
+
|
|
127
|
+
elif isinstance(v, dict):
|
|
128
|
+
for key, element in v.items():
|
|
129
|
+
get_generated_paths(element, path + [f"[{key}]"], paths)
|
|
130
|
+
|
|
131
|
+
elif isinstance(v, ConfigMixin):
|
|
132
|
+
for key in v.__xpm__._generated_values:
|
|
133
|
+
value = v.__xpm__.values[key]
|
|
134
|
+
if isinstance(value, ConfigMixin) and value.__xpm__._generated_values:
|
|
135
|
+
path.append(key)
|
|
136
|
+
get_generated_paths(value, path, paths)
|
|
137
|
+
path.pop()
|
|
138
|
+
else:
|
|
139
|
+
paths.append(".".join(path + [key]))
|
|
140
|
+
else:
|
|
141
|
+
raise ValueError(f"Cannot handle type {type(v)}")
|
|
142
|
+
return paths
|
|
143
|
+
|
|
144
|
+
|
|
113
145
|
class ConfigInformation:
|
|
114
146
|
"""Holds experimaestro information for a config (or task) instance"""
|
|
115
147
|
|
|
@@ -122,11 +154,11 @@ class ConfigInformation:
|
|
|
122
154
|
def __init__(self, pyobject: "ConfigMixin"):
|
|
123
155
|
# The underlying pyobject and XPM type
|
|
124
156
|
self.pyobject = pyobject
|
|
125
|
-
self.xpmtype = pyobject.__xpmtype__
|
|
157
|
+
self.xpmtype: "ObjectType" = pyobject.__xpmtype__
|
|
126
158
|
self.values = {}
|
|
127
159
|
|
|
128
160
|
# Meta-informations
|
|
129
|
-
self._tags = {}
|
|
161
|
+
self._tags: dict[str, Any] = {}
|
|
130
162
|
self._initinfo = ""
|
|
131
163
|
|
|
132
164
|
self._taskoutput = None
|
|
@@ -142,16 +174,13 @@ class ConfigInformation:
|
|
|
142
174
|
#: True when this configuration was loaded from disk
|
|
143
175
|
self.loaded = False
|
|
144
176
|
|
|
145
|
-
#
|
|
177
|
+
# Explicitly added dependencies
|
|
146
178
|
self.dependencies = []
|
|
147
179
|
|
|
148
180
|
# Concrete type variables resolutions
|
|
149
181
|
# This is used to check typevars coherence
|
|
150
182
|
self.concrete_typevars: Dict[TypeVar, type] = {}
|
|
151
183
|
|
|
152
|
-
# Lightweight tasks
|
|
153
|
-
self.pre_tasks: List["LightweightTask"] = []
|
|
154
|
-
|
|
155
184
|
# Initialization tasks
|
|
156
185
|
self.init_tasks: List["LightweightTask"] = []
|
|
157
186
|
|
|
@@ -160,16 +189,18 @@ class ConfigInformation:
|
|
|
160
189
|
|
|
161
190
|
# Cached information
|
|
162
191
|
|
|
163
|
-
self.
|
|
164
|
-
"""The
|
|
165
|
-
|
|
166
|
-
self._raw_identifier = None
|
|
167
|
-
"""The identifier without taking into account pre-tasks"""
|
|
192
|
+
self._identifier = None
|
|
193
|
+
"""The configuration identifier (cached when sealed)"""
|
|
168
194
|
|
|
169
195
|
self._validated = False
|
|
170
196
|
self._sealed = False
|
|
171
197
|
self._meta = None
|
|
172
198
|
|
|
199
|
+
# This contains the list of generated values (using context) in this
|
|
200
|
+
# configuration or any sub-configuration, is generated. This prevents
|
|
201
|
+
# problem when a configuration with generated values is re-used.
|
|
202
|
+
self._generated_values = []
|
|
203
|
+
|
|
173
204
|
def set_meta(self, value: Optional[bool]):
|
|
174
205
|
"""Sets the meta flag"""
|
|
175
206
|
assert not self._sealed, "Configuration is sealed"
|
|
@@ -187,6 +218,31 @@ class ConfigInformation:
|
|
|
187
218
|
# Not an argument, bypass
|
|
188
219
|
return object.__getattribute__(self.pyobject, name)
|
|
189
220
|
|
|
221
|
+
@staticmethod
|
|
222
|
+
def is_generated_value(argument, value):
|
|
223
|
+
if argument.ignore_generated:
|
|
224
|
+
return False
|
|
225
|
+
|
|
226
|
+
if value is None:
|
|
227
|
+
return False
|
|
228
|
+
|
|
229
|
+
if isinstance(value, (int, str, float, bool, Enum, Path)):
|
|
230
|
+
return False
|
|
231
|
+
|
|
232
|
+
if isinstance(value, ConfigMixin):
|
|
233
|
+
return value.__xpm__._generated_values and value.__xpm__.task is None
|
|
234
|
+
|
|
235
|
+
if isinstance(value, list):
|
|
236
|
+
return any(ConfigInformation.is_generated_value(argument, x) for x in value)
|
|
237
|
+
|
|
238
|
+
if isinstance(value, dict):
|
|
239
|
+
return any(
|
|
240
|
+
ConfigInformation.is_generated_value(argument, x)
|
|
241
|
+
for x in value.values()
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
return False
|
|
245
|
+
|
|
190
246
|
def set(self, k, v, bypass=False):
|
|
191
247
|
from experimaestro.generators import Generator
|
|
192
248
|
|
|
@@ -198,9 +254,21 @@ class ConfigInformation:
|
|
|
198
254
|
if self._sealed and not bypass:
|
|
199
255
|
raise AttributeError(f"Object is read-only (trying to set {k})")
|
|
200
256
|
|
|
257
|
+
if not isinstance(v, ConfigMixin) and isinstance(v, Config):
|
|
258
|
+
raise AttributeError(
|
|
259
|
+
"Configuration (and not objects) should be used. Consider using .C(...)"
|
|
260
|
+
)
|
|
261
|
+
|
|
201
262
|
try:
|
|
202
263
|
argument = self.xpmtype.arguments.get(k, None)
|
|
203
264
|
if argument:
|
|
265
|
+
if ConfigInformation.is_generated_value(argument, v):
|
|
266
|
+
raise AttributeError(
|
|
267
|
+
f"Cannot set {k} to a configuration with generated values. "
|
|
268
|
+
"Here is the list of paths to help you: "
|
|
269
|
+
f"""{', '.join(get_generated_paths(v, [k]))}"""
|
|
270
|
+
)
|
|
271
|
+
|
|
204
272
|
if not bypass and (
|
|
205
273
|
(isinstance(argument.generator, Generator)) or argument.constant
|
|
206
274
|
):
|
|
@@ -302,10 +370,6 @@ class ConfigInformation:
|
|
|
302
370
|
% (k, self.xpmtype, self._initinfo)
|
|
303
371
|
)
|
|
304
372
|
|
|
305
|
-
# Validate pre-tasks
|
|
306
|
-
for pre_task in self.pre_tasks:
|
|
307
|
-
pre_task.__xpm__.validate()
|
|
308
|
-
|
|
309
373
|
# Validate init tasks
|
|
310
374
|
for init_task in self.init_tasks:
|
|
311
375
|
init_task.__xpm__.validate()
|
|
@@ -326,12 +390,21 @@ class ConfigInformation:
|
|
|
326
390
|
Arguments:
|
|
327
391
|
- context: the generation context
|
|
328
392
|
"""
|
|
393
|
+
if generated_keys := [
|
|
394
|
+
k
|
|
395
|
+
for k, v in self.values.items()
|
|
396
|
+
if ConfigInformation.is_generated_value(self.xpmtype.arguments[k], v)
|
|
397
|
+
]:
|
|
398
|
+
raise AttributeError(
|
|
399
|
+
"Cannot seal a configuration with generated values:"
|
|
400
|
+
f"""{",".join(generated_keys)} in {context.currentpath}"""
|
|
401
|
+
)
|
|
329
402
|
|
|
330
403
|
class Sealer(ConfigWalk):
|
|
331
|
-
def preprocess(self, config:
|
|
404
|
+
def preprocess(self, config: ConfigMixin):
|
|
332
405
|
return not config.__xpm__._sealed, config
|
|
333
406
|
|
|
334
|
-
def postprocess(self, stub, config:
|
|
407
|
+
def postprocess(self, stub, config: ConfigMixin, values):
|
|
335
408
|
# Generate values
|
|
336
409
|
from experimaestro.generators import Generator
|
|
337
410
|
|
|
@@ -344,22 +417,42 @@ class ConfigInformation:
|
|
|
344
417
|
continue
|
|
345
418
|
value = argument.generator()
|
|
346
419
|
else:
|
|
420
|
+
# Generate a value
|
|
347
421
|
sig = inspect.signature(argument.generator)
|
|
348
422
|
if len(sig.parameters) == 0:
|
|
349
423
|
value = argument.generator()
|
|
350
424
|
elif len(sig.parameters) == 2:
|
|
425
|
+
# Only in that case do we need to flag this configuration
|
|
426
|
+
# as containing generated values
|
|
427
|
+
if not argument.ignore_generated:
|
|
428
|
+
config.__xpm__._generated_values.append(k)
|
|
429
|
+
else:
|
|
430
|
+
logging.warning("Ignoring %s", k)
|
|
351
431
|
value = argument.generator(self.context, config)
|
|
352
432
|
else:
|
|
353
433
|
assert (
|
|
354
434
|
False
|
|
355
435
|
), "generator has either two parameters (context and config), or none"
|
|
356
436
|
config.__xpm__.set(k, value, bypass=True)
|
|
437
|
+
else:
|
|
438
|
+
value = config.__xpm__.values.get(k)
|
|
357
439
|
except Exception:
|
|
358
440
|
logger.error(
|
|
359
441
|
"While setting %s of %s", argument.name, config.__xpmtype__
|
|
360
442
|
)
|
|
361
443
|
raise
|
|
362
444
|
|
|
445
|
+
# Propagate the generated value flag
|
|
446
|
+
if (
|
|
447
|
+
value is not None
|
|
448
|
+
and isinstance(value, ConfigMixin)
|
|
449
|
+
and value.__xpm__._generated_values
|
|
450
|
+
):
|
|
451
|
+
if not argument.ignore_generated:
|
|
452
|
+
config.__xpm__._generated_values.append(k)
|
|
453
|
+
else:
|
|
454
|
+
logging.warning("Ignoring %s", k)
|
|
455
|
+
|
|
363
456
|
config.__xpm__._sealed = True
|
|
364
457
|
|
|
365
458
|
Sealer(context, recurse_task=True)(self.pyobject)
|
|
@@ -372,90 +465,29 @@ class ConfigInformation:
|
|
|
372
465
|
context = ConfigWalkContext()
|
|
373
466
|
|
|
374
467
|
class Unsealer(ConfigWalk):
|
|
375
|
-
def preprocess(self, config:
|
|
468
|
+
def preprocess(self, config: ConfigMixin):
|
|
376
469
|
return config.__xpm__._sealed, config
|
|
377
470
|
|
|
378
|
-
def postprocess(self, stub, config:
|
|
471
|
+
def postprocess(self, stub, config: ConfigMixin, values):
|
|
379
472
|
config.__xpm__._sealed = False
|
|
380
473
|
config.__xpm__._identifier = None
|
|
381
474
|
|
|
382
475
|
Unsealer(context, recurse_task=True)(self.pyobject)
|
|
383
476
|
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
pre_tasks: Dict[int, "Config"] = {}
|
|
387
|
-
|
|
388
|
-
class PreTaskCollect(ConfigWalk):
|
|
389
|
-
def preprocess(self, config: Config):
|
|
390
|
-
# Do not cross tasks
|
|
391
|
-
return not isinstance(config.__xpm__, Task), config
|
|
392
|
-
|
|
393
|
-
def postprocess(self, stub, config: Config, values):
|
|
394
|
-
pre_tasks.update(
|
|
395
|
-
{id(pre_task): pre_task for pre_task in config.__xpm__.pre_tasks}
|
|
396
|
-
)
|
|
397
|
-
|
|
398
|
-
PreTaskCollect(context, recurse_task=True)(self.pyobject)
|
|
399
|
-
return pre_tasks.values()
|
|
400
|
-
|
|
401
|
-
def identifiers(self, only_raw: bool):
|
|
477
|
+
@property
|
|
478
|
+
def identifier(self):
|
|
402
479
|
"""Computes the unique identifier"""
|
|
403
|
-
from ..identifier import IdentifierComputer
|
|
404
|
-
|
|
405
|
-
raw_identifier = self._raw_identifier
|
|
406
|
-
full_identifier = self._full_identifier
|
|
480
|
+
from ..identifier import IdentifierComputer
|
|
407
481
|
|
|
408
482
|
# Computes raw identifier if needed
|
|
409
|
-
if
|
|
410
|
-
|
|
411
|
-
raw_identifier = IdentifierComputer.compute(self.pyobject)
|
|
412
|
-
if self._sealed:
|
|
413
|
-
self._raw_identifier = raw_identifier
|
|
414
|
-
|
|
415
|
-
if only_raw:
|
|
416
|
-
return raw_identifier, full_identifier
|
|
417
|
-
|
|
418
|
-
# OK, let's compute the full identifier
|
|
419
|
-
if full_identifier is None or not self._sealed:
|
|
420
|
-
# Compute the full identifier by including the pre-tasks
|
|
421
|
-
hasher = hashlib.sha256()
|
|
422
|
-
hasher.update(raw_identifier.all)
|
|
423
|
-
pre_tasks_ids = [
|
|
424
|
-
pre_task.__xpm__.raw_identifier.all
|
|
425
|
-
for pre_task in self.collect_pre_tasks()
|
|
426
|
-
]
|
|
427
|
-
for task_id in sorted(pre_tasks_ids):
|
|
428
|
-
hasher.update(task_id)
|
|
429
|
-
|
|
430
|
-
# Adds init tasks
|
|
431
|
-
if self.init_tasks:
|
|
432
|
-
hasher.update(IdentifierComputer.INIT_TASKS)
|
|
433
|
-
for init_task in self.init_tasks:
|
|
434
|
-
hasher.update(init_task.__xpm__.raw_identifier.all)
|
|
435
|
-
|
|
436
|
-
full_identifier = Identifier(hasher.digest())
|
|
437
|
-
full_identifier.has_loops = raw_identifier.has_loops
|
|
438
|
-
|
|
439
|
-
# Only cache the identifier if sealed
|
|
440
|
-
if self._sealed:
|
|
441
|
-
self._full_identifier = full_identifier
|
|
483
|
+
if self._identifier is not None:
|
|
484
|
+
return self._identifier
|
|
442
485
|
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
raw_identifier, _ = self.identifiers(True)
|
|
449
|
-
return raw_identifier
|
|
450
|
-
|
|
451
|
-
@property
|
|
452
|
-
def full_identifier(self) -> "Identifier":
|
|
453
|
-
"""Computes the unique identifier (with task modifiers)"""
|
|
454
|
-
_, full_identifier = self.identifiers(False)
|
|
455
|
-
return full_identifier
|
|
456
|
-
|
|
457
|
-
identifier = full_identifier
|
|
458
|
-
"""Deprecated: use full_identifier"""
|
|
486
|
+
# Get the main identifier
|
|
487
|
+
identifier = IdentifierComputer.compute(self.pyobject)
|
|
488
|
+
if self._sealed:
|
|
489
|
+
self._identifier = identifier
|
|
490
|
+
return identifier
|
|
459
491
|
|
|
460
492
|
def dependency(self):
|
|
461
493
|
"""Returns a dependency"""
|
|
@@ -470,12 +502,6 @@ class ConfigInformation:
|
|
|
470
502
|
path: List[str],
|
|
471
503
|
taskids: Set[int],
|
|
472
504
|
):
|
|
473
|
-
# Add pre-tasks
|
|
474
|
-
for pre_task in self.pre_tasks:
|
|
475
|
-
pre_task.__xpm__.updatedependencies(
|
|
476
|
-
dependencies, path + ["__pre_tasks__"], taskids
|
|
477
|
-
)
|
|
478
|
-
|
|
479
505
|
# Add initialization tasks
|
|
480
506
|
for init_task in self.init_tasks:
|
|
481
507
|
init_task.__xpm__.updatedependencies(
|
|
@@ -618,10 +644,11 @@ class ConfigInformation:
|
|
|
618
644
|
) or RunMode.NORMAL
|
|
619
645
|
if run_mode == RunMode.NORMAL:
|
|
620
646
|
TaskEventListener.connect(experiment.CURRENT)
|
|
647
|
+
experiment.CURRENT.submit(self.job)
|
|
621
648
|
other = experiment.CURRENT.submit(self.job)
|
|
622
649
|
if other:
|
|
623
|
-
#
|
|
624
|
-
|
|
650
|
+
# Our job = previously submitted job
|
|
651
|
+
self.job = other
|
|
625
652
|
else:
|
|
626
653
|
# Show a warning
|
|
627
654
|
if run_mode == RunMode.GENERATE_ONLY:
|
|
@@ -657,13 +684,6 @@ class ConfigInformation:
|
|
|
657
684
|
|
|
658
685
|
print(file=sys.stderr) # noqa: T201
|
|
659
686
|
|
|
660
|
-
# Handle an output configuration # FIXME: remove
|
|
661
|
-
def mark_output(config: "Config"):
|
|
662
|
-
"""Sets a dependency on the job"""
|
|
663
|
-
assert not isinstance(config, Task), "Cannot set a dependency on a task"
|
|
664
|
-
config.__xpm__.task = self.pyobject
|
|
665
|
-
return config
|
|
666
|
-
|
|
667
687
|
# Mark this configuration also
|
|
668
688
|
self.task = self.pyobject
|
|
669
689
|
|
|
@@ -677,6 +697,9 @@ class ConfigInformation:
|
|
|
677
697
|
def mark_output(self, config: "Config"):
|
|
678
698
|
"""Sets a dependency on the job"""
|
|
679
699
|
assert not isinstance(config, Task), "Cannot set a dependency on a task"
|
|
700
|
+
assert isinstance(
|
|
701
|
+
config, ConfigMixin
|
|
702
|
+
), "Only configurations can be marked as dependent on a task"
|
|
680
703
|
config.__xpm__.task = self.pyobject
|
|
681
704
|
return config
|
|
682
705
|
|
|
@@ -752,9 +775,6 @@ class ConfigInformation:
|
|
|
752
775
|
if self.task is not None and self.task is not self:
|
|
753
776
|
ConfigInformation.__collect_objects__(self.task, objects, context)
|
|
754
777
|
|
|
755
|
-
# Serialize pre-tasks
|
|
756
|
-
ConfigInformation.__collect_objects__(self.pre_tasks, objects, context)
|
|
757
|
-
|
|
758
778
|
# Serialize initialization tasks
|
|
759
779
|
ConfigInformation.__collect_objects__(self.init_tasks, objects, context)
|
|
760
780
|
|
|
@@ -762,14 +782,12 @@ class ConfigInformation:
|
|
|
762
782
|
state_dict = {
|
|
763
783
|
"id": id(self.pyobject),
|
|
764
784
|
"module": self.xpmtype._module,
|
|
765
|
-
"type": self.xpmtype.
|
|
785
|
+
"type": self.xpmtype.value_type.__qualname__,
|
|
766
786
|
"typename": self.xpmtype.name(),
|
|
767
787
|
"identifier": self.identifier.state_dict(),
|
|
768
788
|
}
|
|
769
789
|
|
|
770
790
|
# Add pre/init tasks
|
|
771
|
-
if self.pre_tasks:
|
|
772
|
-
state_dict["pre-tasks"] = [id(pre_task) for pre_task in self.pre_tasks]
|
|
773
791
|
if self.init_tasks:
|
|
774
792
|
state_dict["init-tasks"] = [id(init_task) for init_task in self.init_tasks]
|
|
775
793
|
|
|
@@ -855,6 +873,7 @@ class ConfigInformation:
|
|
|
855
873
|
"workspace": str(context.workspace.path.absolute()),
|
|
856
874
|
"tags": {key: value for key, value in self.tags().items()},
|
|
857
875
|
"version": 2,
|
|
876
|
+
"experimaestro": experimaestro.__version__,
|
|
858
877
|
"objects": self.__get_objects__([], context),
|
|
859
878
|
},
|
|
860
879
|
out,
|
|
@@ -949,34 +968,31 @@ class ConfigInformation:
|
|
|
949
968
|
|
|
950
969
|
@overload
|
|
951
970
|
@staticmethod
|
|
952
|
-
def fromParameters(
|
|
971
|
+
def fromParameters( # noqa: E704
|
|
953
972
|
definitions: List[Dict],
|
|
954
973
|
as_instance=True,
|
|
955
974
|
save_directory: Optional[Path] = None,
|
|
956
975
|
discard_id: bool = False,
|
|
957
|
-
) -> "ConfigMixin":
|
|
958
|
-
...
|
|
976
|
+
) -> "ConfigMixin": ...
|
|
959
977
|
|
|
960
978
|
@overload
|
|
961
979
|
@staticmethod
|
|
962
|
-
def fromParameters(
|
|
980
|
+
def fromParameters( # noqa: E704
|
|
963
981
|
definitions: List[Dict],
|
|
964
982
|
as_instance=False,
|
|
965
983
|
return_tasks=True,
|
|
966
984
|
save_directory: Optional[Path] = None,
|
|
967
985
|
discard_id: bool = False,
|
|
968
|
-
) -> Tuple["Config", List["LightweightTask"]]:
|
|
969
|
-
...
|
|
986
|
+
) -> Tuple["Config", List["LightweightTask"]]: ...
|
|
970
987
|
|
|
971
988
|
@overload
|
|
972
989
|
@staticmethod
|
|
973
|
-
def fromParameters(
|
|
990
|
+
def fromParameters( # noqa: E704
|
|
974
991
|
definitions: List[Dict],
|
|
975
992
|
as_instance=False,
|
|
976
993
|
save_directory: Optional[Path] = None,
|
|
977
994
|
discard_id: bool = False,
|
|
978
|
-
) -> "Config":
|
|
979
|
-
...
|
|
995
|
+
) -> "Config": ...
|
|
980
996
|
|
|
981
997
|
@staticmethod
|
|
982
998
|
def load_objects( # noqa: C901
|
|
@@ -1022,7 +1038,7 @@ class ConfigInformation:
|
|
|
1022
1038
|
|
|
1023
1039
|
# Creates an object (or a config)
|
|
1024
1040
|
if as_instance:
|
|
1025
|
-
o = cls.
|
|
1041
|
+
o = cls.__new__(cls)
|
|
1026
1042
|
else:
|
|
1027
1043
|
o = cls.XPMConfig.__new__(cls.XPMConfig)
|
|
1028
1044
|
assert definition["id"] not in objects, "Duplicate id %s" % definition["id"]
|
|
@@ -1101,12 +1117,6 @@ class ConfigInformation:
|
|
|
1101
1117
|
o.__post_init__()
|
|
1102
1118
|
|
|
1103
1119
|
else:
|
|
1104
|
-
# Sets pre-tasks
|
|
1105
|
-
o.__xpm__.pre_tasks = [
|
|
1106
|
-
objects[pre_task_id]
|
|
1107
|
-
for pre_task_id in definition.get("pre-tasks", [])
|
|
1108
|
-
]
|
|
1109
|
-
|
|
1110
1120
|
if task_id := definition.get("task", None):
|
|
1111
1121
|
o.__xpm__.task = objects[task_id]
|
|
1112
1122
|
|
|
@@ -1140,15 +1150,6 @@ class ConfigInformation:
|
|
|
1140
1150
|
|
|
1141
1151
|
# Run pre-task (or returns them)
|
|
1142
1152
|
if as_instance or return_tasks:
|
|
1143
|
-
# Collect pre-tasks (just once)
|
|
1144
|
-
completed_pretasks = set()
|
|
1145
|
-
pre_tasks = []
|
|
1146
|
-
for definition in definitions:
|
|
1147
|
-
for pre_task_id in definition.get("pre-tasks", []):
|
|
1148
|
-
if pre_task_id not in completed_pretasks:
|
|
1149
|
-
completed_pretasks.add(pre_task_id)
|
|
1150
|
-
pre_tasks.append(objects[pre_task_id])
|
|
1151
|
-
|
|
1152
1153
|
# Collect init tasks
|
|
1153
1154
|
init_tasks = []
|
|
1154
1155
|
for init_task_id in definitions[-1].get("init-tasks", []):
|
|
@@ -1156,14 +1157,11 @@ class ConfigInformation:
|
|
|
1156
1157
|
init_tasks.append(init_task)
|
|
1157
1158
|
|
|
1158
1159
|
if as_instance:
|
|
1159
|
-
for pre_task in pre_tasks:
|
|
1160
|
-
logger.info("Executing pre-task %s", type(pre_task))
|
|
1161
|
-
pre_task.execute()
|
|
1162
1160
|
for init_task in init_tasks:
|
|
1163
1161
|
logger.info("Executing init task %s", type(init_task))
|
|
1164
1162
|
init_task.execute()
|
|
1165
1163
|
else:
|
|
1166
|
-
return o,
|
|
1164
|
+
return o, init_tasks
|
|
1167
1165
|
|
|
1168
1166
|
return o
|
|
1169
1167
|
|
|
@@ -1171,7 +1169,6 @@ class ConfigInformation:
|
|
|
1171
1169
|
def __init__(self, context: ConfigWalkContext, *, objects: ObjectStore = None):
|
|
1172
1170
|
super().__init__(context)
|
|
1173
1171
|
self.objects = ObjectStore() if objects is None else objects
|
|
1174
|
-
self.pre_tasks = {}
|
|
1175
1172
|
|
|
1176
1173
|
def preprocess(self, config: "Config"):
|
|
1177
1174
|
if self.objects.is_constructed(id(config)):
|
|
@@ -1183,7 +1180,7 @@ class ConfigInformation:
|
|
|
1183
1180
|
|
|
1184
1181
|
if o is None:
|
|
1185
1182
|
# Creates an object (and not a config)
|
|
1186
|
-
o = config.
|
|
1183
|
+
o = config.__xpmtype__.value_type()
|
|
1187
1184
|
|
|
1188
1185
|
# Store in cache
|
|
1189
1186
|
self.objects.add_stub(id(config), o)
|
|
@@ -1198,10 +1195,6 @@ class ConfigInformation:
|
|
|
1198
1195
|
# Call __post_init__
|
|
1199
1196
|
stub.__post_init__()
|
|
1200
1197
|
|
|
1201
|
-
# Gather pre-tasks
|
|
1202
|
-
for pre_task in config.__xpm__.pre_tasks:
|
|
1203
|
-
self.pre_tasks[id(pre_task)] = self.stub(pre_task)
|
|
1204
|
-
|
|
1205
1198
|
self.objects.set_constructed(id(config))
|
|
1206
1199
|
return stub
|
|
1207
1200
|
|
|
@@ -1215,10 +1208,6 @@ class ConfigInformation:
|
|
|
1215
1208
|
processor = ConfigInformation.FromPython(context, objects=objects)
|
|
1216
1209
|
last_object = processor(self.pyobject)
|
|
1217
1210
|
|
|
1218
|
-
# Execute pre-tasks
|
|
1219
|
-
for pre_task in processor.pre_tasks.values():
|
|
1220
|
-
pre_task.execute()
|
|
1221
|
-
|
|
1222
1211
|
return last_object
|
|
1223
1212
|
|
|
1224
1213
|
def add_dependencies(self, *dependencies):
|
|
@@ -1242,6 +1231,9 @@ def clone(v):
|
|
|
1242
1231
|
if isinstance(v, Enum):
|
|
1243
1232
|
return v
|
|
1244
1233
|
|
|
1234
|
+
if isinstance(v, tuple):
|
|
1235
|
+
return tuple(clone(x) for x in v)
|
|
1236
|
+
|
|
1245
1237
|
if isinstance(v, Config):
|
|
1246
1238
|
# Create a new instance
|
|
1247
1239
|
kwargs = {
|
|
@@ -1260,6 +1252,11 @@ class ConfigMixin:
|
|
|
1260
1252
|
"""Class for configuration objects"""
|
|
1261
1253
|
|
|
1262
1254
|
__xpmtype__: ObjectType
|
|
1255
|
+
"""The associated XPM type"""
|
|
1256
|
+
|
|
1257
|
+
__xpm__: ConfigInformation
|
|
1258
|
+
"""The __xpm__ object contains all instance specific information about a
|
|
1259
|
+
configuration/task"""
|
|
1263
1260
|
|
|
1264
1261
|
def __init__(self, **kwargs):
|
|
1265
1262
|
"""Initialize the configuration with the given parameters"""
|
|
@@ -1310,8 +1307,8 @@ class ConfigMixin:
|
|
|
1310
1307
|
[f"{key}={value}" for key, value in self.__xpm__.values.items()]
|
|
1311
1308
|
)
|
|
1312
1309
|
return (
|
|
1313
|
-
f"{self.__xpmtype__.
|
|
1314
|
-
f"{self.__xpmtype__.
|
|
1310
|
+
f"{self.__xpmtype__.value_type.__module__}."
|
|
1311
|
+
f"{self.__xpmtype__.value_type.__qualname__}({params})"
|
|
1315
1312
|
)
|
|
1316
1313
|
|
|
1317
1314
|
def tag(self, name, value):
|
|
@@ -1340,9 +1337,20 @@ class ConfigMixin:
|
|
|
1340
1337
|
return self
|
|
1341
1338
|
|
|
1342
1339
|
def instance(
|
|
1343
|
-
self,
|
|
1340
|
+
self,
|
|
1341
|
+
context: ConfigWalkContext = None,
|
|
1342
|
+
*,
|
|
1343
|
+
objects: ObjectStore = None,
|
|
1344
|
+
keep: bool = True,
|
|
1344
1345
|
) -> T:
|
|
1345
|
-
"""Return an instance with the current values
|
|
1346
|
+
"""Return an instance with the current values
|
|
1347
|
+
|
|
1348
|
+
:param context: The context when computing the instance
|
|
1349
|
+
:param objects: The previously built objects (so that we avoid
|
|
1350
|
+
re-creating instances of past configurations)
|
|
1351
|
+
:param keep: register a configuration in the __config__ field of the
|
|
1352
|
+
instance
|
|
1353
|
+
"""
|
|
1346
1354
|
if context is None:
|
|
1347
1355
|
from experimaestro.xpmutils import EmptyContext
|
|
1348
1356
|
|
|
@@ -1351,7 +1359,11 @@ class ConfigMixin:
|
|
|
1351
1359
|
assert isinstance(
|
|
1352
1360
|
context, ConfigWalkContext
|
|
1353
1361
|
), f"{context.__class__} is not an instance of ConfigWalkContext"
|
|
1354
|
-
|
|
1362
|
+
|
|
1363
|
+
instance = self.__xpm__.fromConfig(context, objects=objects) # type: ignore
|
|
1364
|
+
if keep:
|
|
1365
|
+
object.__setattr__(instance, "__config__", self)
|
|
1366
|
+
return instance
|
|
1355
1367
|
|
|
1356
1368
|
def submit(
|
|
1357
1369
|
self,
|
|
@@ -1396,29 +1408,7 @@ class ConfigMixin:
|
|
|
1396
1408
|
attributes)"""
|
|
1397
1409
|
return clone(self)
|
|
1398
1410
|
|
|
1399
|
-
def
|
|
1400
|
-
assert all(
|
|
1401
|
-
[isinstance(task, LightweightTask) for task in tasks]
|
|
1402
|
-
), "One of the pre-tasks are not lightweight tasks"
|
|
1403
|
-
if self.__xpm__._sealed:
|
|
1404
|
-
raise SealedError("Cannot add pre-tasks to a sealed configuration")
|
|
1405
|
-
self.__xpm__.pre_tasks.extend(tasks)
|
|
1406
|
-
return self
|
|
1407
|
-
|
|
1408
|
-
def add_pretasks_from(self, *configs: "Config"):
|
|
1409
|
-
assert all(
|
|
1410
|
-
[isinstance(config, ConfigMixin) for config in configs]
|
|
1411
|
-
), "One of the parameters is not a configuration object"
|
|
1412
|
-
for config in configs:
|
|
1413
|
-
self.add_pretasks(*config.__xpm__.pre_tasks)
|
|
1414
|
-
return self
|
|
1415
|
-
|
|
1416
|
-
@property
|
|
1417
|
-
def pre_tasks(self) -> List["LightweightTask"]:
|
|
1418
|
-
"""Access pre-tasks"""
|
|
1419
|
-
return self.__xpm__.pre_tasks
|
|
1420
|
-
|
|
1421
|
-
def copy_dependencies(self, other: "Config"):
|
|
1411
|
+
def copy_dependencies(self, other: "ConfigMixin"):
|
|
1422
1412
|
"""Add all the dependencies from other configuration"""
|
|
1423
1413
|
|
|
1424
1414
|
# Add task dependency
|
|
@@ -1441,52 +1431,17 @@ class Config:
|
|
|
1441
1431
|
"""The object type holds all the information about a specific subclass
|
|
1442
1432
|
experimaestro metadata"""
|
|
1443
1433
|
|
|
1444
|
-
__xpm__: ConfigInformation
|
|
1445
|
-
"""The __xpm__ object contains all instance specific information about a
|
|
1446
|
-
configuration/task"""
|
|
1447
|
-
|
|
1448
1434
|
@classproperty
|
|
1449
1435
|
def XPMConfig(cls):
|
|
1450
1436
|
if issubclass(cls, ConfigMixin):
|
|
1451
1437
|
return cls
|
|
1452
|
-
return cls.__getxpmtype__().
|
|
1453
|
-
|
|
1454
|
-
@classproperty
|
|
1455
|
-
def XPMValue(cls):
|
|
1456
|
-
"""Returns the value object for this configuration"""
|
|
1457
|
-
if issubclass(cls, ConfigMixin):
|
|
1458
|
-
return cls.__xpmtype__.objecttype
|
|
1459
|
-
|
|
1460
|
-
if value_cls := cls.__dict__.get("__XPMValue__", None):
|
|
1461
|
-
pass
|
|
1462
|
-
else:
|
|
1463
|
-
from ..types import XPMValue
|
|
1464
|
-
|
|
1465
|
-
__objectbases__ = tuple(
|
|
1466
|
-
s.XPMValue
|
|
1467
|
-
for s in cls.__bases__
|
|
1468
|
-
if issubclass(s, Config) and (s is not Config)
|
|
1469
|
-
) or (XPMValue,)
|
|
1470
|
-
|
|
1471
|
-
*tp_qual, tp_name = cls.__qualname__.split(".")
|
|
1472
|
-
value_cls = type(f"{tp_name}.XPMValue", (cls,) + __objectbases__, {})
|
|
1473
|
-
value_cls.__qualname__ = ".".join(tp_qual + [value_cls.__name__])
|
|
1474
|
-
value_cls.__module__ = cls.__module__
|
|
1475
|
-
|
|
1476
|
-
setattr(cls, "__XPMValue__", value_cls)
|
|
1477
|
-
|
|
1478
|
-
return value_cls
|
|
1438
|
+
return cls.__getxpmtype__().config_type
|
|
1479
1439
|
|
|
1480
1440
|
@classproperty
|
|
1481
1441
|
def C(cls):
|
|
1482
1442
|
"""Alias for XPMConfig"""
|
|
1483
1443
|
return cls.XPMConfig
|
|
1484
1444
|
|
|
1485
|
-
@classproperty
|
|
1486
|
-
def V(cls):
|
|
1487
|
-
"""Alias for XPMValue"""
|
|
1488
|
-
return cls.XPMValue
|
|
1489
|
-
|
|
1490
1445
|
@classmethod
|
|
1491
1446
|
def __getxpmtype__(cls) -> "ObjectType":
|
|
1492
1447
|
"""Get (and create if necessary) the Object type associated
|
|
@@ -1503,46 +1458,6 @@ class Config:
|
|
|
1503
1458
|
raise
|
|
1504
1459
|
return xpmtype
|
|
1505
1460
|
|
|
1506
|
-
def __new__(cls: Type[T], *args, **kwargs) -> T:
|
|
1507
|
-
"""Returns an instance of a ConfigMixin (for compatibility, use XPMConfig
|
|
1508
|
-
or C if possible)
|
|
1509
|
-
|
|
1510
|
-
:deprecated: Use Config.C or Config.XPMConfig to construct a new
|
|
1511
|
-
configuration, and Config.V (or Config.XPMValue) for a new value
|
|
1512
|
-
"""
|
|
1513
|
-
# If this is an XPMValue, just return a new instance
|
|
1514
|
-
from experimaestro.core.types import XPMValue
|
|
1515
|
-
|
|
1516
|
-
if issubclass(cls, XPMValue):
|
|
1517
|
-
return object.__new__(cls)
|
|
1518
|
-
|
|
1519
|
-
# If this is the XPMConfig, just return a new instance
|
|
1520
|
-
# __init__ will be called
|
|
1521
|
-
if issubclass(cls, ConfigMixin):
|
|
1522
|
-
return object.__new__(cls)
|
|
1523
|
-
|
|
1524
|
-
# Log a deprecation warning for this way of creating a configuration
|
|
1525
|
-
caller = inspect.getframeinfo(inspect.stack()[1][0])
|
|
1526
|
-
logger.warning(
|
|
1527
|
-
"Creating a configuration using Config.__new__ is deprecated, and will be removed in a future version. "
|
|
1528
|
-
"Use Config.C or Config.XPMConfig to create a new configuration. "
|
|
1529
|
-
"Issue created at %s:%s",
|
|
1530
|
-
str(Path(caller.filename).absolute()),
|
|
1531
|
-
caller.lineno,
|
|
1532
|
-
)
|
|
1533
|
-
|
|
1534
|
-
# otherwise, we use the configuration type
|
|
1535
|
-
o: ConfigMixin = object.__new__(cls.__getxpmtype__().configtype)
|
|
1536
|
-
try:
|
|
1537
|
-
o.__init__(*args, **kwargs)
|
|
1538
|
-
except Exception:
|
|
1539
|
-
logger.error(
|
|
1540
|
-
"Init error in %s:%s"
|
|
1541
|
-
% (str(Path(caller.filename).absolute()), caller.lineno)
|
|
1542
|
-
)
|
|
1543
|
-
raise
|
|
1544
|
-
return o
|
|
1545
|
-
|
|
1546
1461
|
def __validate__(self):
|
|
1547
1462
|
"""Validate the values"""
|
|
1548
1463
|
pass
|
|
@@ -1557,17 +1472,7 @@ class Config:
|
|
|
1557
1472
|
return self.__xpm__.__json__()
|
|
1558
1473
|
|
|
1559
1474
|
def __identifier__(self) -> "Identifier":
|
|
1560
|
-
return self.__xpm__.
|
|
1561
|
-
|
|
1562
|
-
def add_pretasks(self, *tasks: "LightweightTask"):
|
|
1563
|
-
"""Add pre-tasks"""
|
|
1564
|
-
raise AssertionError("This method can only be used during configuration")
|
|
1565
|
-
|
|
1566
|
-
def add_pretasks_from(self, *configs: "Config"):
|
|
1567
|
-
"""Add pre-tasks from the listed configurations"""
|
|
1568
|
-
raise AssertionError(
|
|
1569
|
-
"The 'add_pretasks_from' can only be used during configuration"
|
|
1570
|
-
)
|
|
1475
|
+
return self.__xpm__.identifier
|
|
1571
1476
|
|
|
1572
1477
|
def copy_dependencies(self, other: "Config"):
|
|
1573
1478
|
"""Add pre-tasks from the listed configurations"""
|
|
@@ -1575,11 +1480,6 @@ class Config:
|
|
|
1575
1480
|
"The 'copy_dependencies' method can only be used during configuration"
|
|
1576
1481
|
)
|
|
1577
1482
|
|
|
1578
|
-
@property
|
|
1579
|
-
def pre_tasks(self) -> List["LightweightTask"]:
|
|
1580
|
-
"""Access pre-tasks"""
|
|
1581
|
-
raise AssertionError("Pre-tasks can be accessed only during configuration")
|
|
1582
|
-
|
|
1583
1483
|
def register_task_output(self, method, *args, **kwargs):
|
|
1584
1484
|
# Determine the path for this...
|
|
1585
1485
|
path = taskglobals.Env.instance().xpm_path / "task-outputs.jsonl"
|