experimaestro 1.11.1__py3-none-any.whl → 2.0.0a8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of experimaestro might be problematic. Click here for more details.

Files changed (52) hide show
  1. experimaestro/annotations.py +1 -1
  2. experimaestro/cli/__init__.py +10 -11
  3. experimaestro/cli/progress.py +269 -0
  4. experimaestro/connectors/__init__.py +2 -2
  5. experimaestro/core/arguments.py +20 -1
  6. experimaestro/core/identifier.py +21 -7
  7. experimaestro/core/objects/config.py +174 -274
  8. experimaestro/core/objects/config_walk.py +4 -6
  9. experimaestro/core/objects.pyi +2 -6
  10. experimaestro/core/serializers.py +1 -8
  11. experimaestro/core/types.py +35 -57
  12. experimaestro/launcherfinder/registry.py +3 -3
  13. experimaestro/mkdocs/base.py +6 -8
  14. experimaestro/notifications.py +12 -3
  15. experimaestro/progress.py +406 -0
  16. experimaestro/scheduler/__init__.py +18 -1
  17. experimaestro/scheduler/base.py +87 -906
  18. experimaestro/scheduler/experiment.py +387 -0
  19. experimaestro/scheduler/jobs.py +475 -0
  20. experimaestro/scheduler/signal_handler.py +32 -0
  21. experimaestro/scheduler/state.py +1 -1
  22. experimaestro/server/__init__.py +36 -5
  23. experimaestro/settings.py +4 -2
  24. experimaestro/tests/launchers/common.py +2 -2
  25. experimaestro/tests/restart.py +1 -1
  26. experimaestro/tests/tasks/all.py +7 -0
  27. experimaestro/tests/test_checkers.py +2 -2
  28. experimaestro/tests/test_dependencies.py +11 -17
  29. experimaestro/tests/test_experiment.py +3 -3
  30. experimaestro/tests/test_file_progress.py +425 -0
  31. experimaestro/tests/test_file_progress_integration.py +477 -0
  32. experimaestro/tests/test_generators.py +93 -0
  33. experimaestro/tests/test_identifier.py +155 -135
  34. experimaestro/tests/test_instance.py +13 -18
  35. experimaestro/tests/test_objects.py +9 -32
  36. experimaestro/tests/test_outputs.py +6 -6
  37. experimaestro/tests/test_param.py +14 -14
  38. experimaestro/tests/test_progress.py +4 -4
  39. experimaestro/tests/test_serializers.py +0 -59
  40. experimaestro/tests/test_tags.py +15 -15
  41. experimaestro/tests/test_tasks.py +42 -51
  42. experimaestro/tests/test_tokens.py +8 -6
  43. experimaestro/tests/test_types.py +10 -10
  44. experimaestro/tests/test_validation.py +19 -19
  45. experimaestro/tests/token_reschedule.py +1 -1
  46. experimaestro/tools/diff.py +8 -1
  47. experimaestro/typingutils.py +11 -2
  48. {experimaestro-1.11.1.dist-info → experimaestro-2.0.0a8.dist-info}/METADATA +3 -2
  49. {experimaestro-1.11.1.dist-info → experimaestro-2.0.0a8.dist-info}/RECORD +52 -44
  50. {experimaestro-1.11.1.dist-info → experimaestro-2.0.0a8.dist-info}/WHEEL +1 -1
  51. {experimaestro-1.11.1.dist-info → experimaestro-2.0.0a8.dist-info}/entry_points.txt +0 -0
  52. {experimaestro-1.11.1.dist-info → experimaestro-2.0.0a8.dist-info/licenses}/LICENSE +0 -0
@@ -9,7 +9,6 @@ from experimaestro import taskglobals
9
9
 
10
10
  from termcolor import cprint
11
11
  from pathlib import Path
12
- import hashlib
13
12
  import logging
14
13
  import io
15
14
  from enum import Enum
@@ -20,12 +19,10 @@ from typing import (
20
19
  Callable,
21
20
  ClassVar,
22
21
  Dict,
23
- Iterator,
24
22
  List,
25
23
  Optional,
26
24
  Set,
27
25
  Tuple,
28
- Type,
29
26
  TypeVar,
30
27
  Union,
31
28
  overload,
@@ -49,7 +46,6 @@ from .config_walk import ConfigWalk, ConfigWalkContext
49
46
  from .config_utils import (
50
47
  getqualattr,
51
48
  add_to_path,
52
- SealedError,
53
49
  TaggedValue,
54
50
  ObjectStore,
55
51
  classproperty,
@@ -110,6 +106,42 @@ class WatchedOutput:
110
106
  callback: Callable
111
107
 
112
108
 
109
+ def get_generated_paths(
110
+ v: Union["ConfigMixin", list, dict],
111
+ path: list[str] | None = None,
112
+ paths: list[str] | None = None,
113
+ ) -> list[str]:
114
+ """Get the list of generated paths, useful to track down those
115
+
116
+ :param path: The current path
117
+ :param paths: The list of generated paths so far, defaults to None
118
+ :return: The full list of generated paths
119
+ """
120
+ paths = [] if paths is None else paths
121
+ path = [] if path is None else path
122
+
123
+ if isinstance(v, list):
124
+ for ix, element in enumerate(v):
125
+ get_generated_paths(element, path + [f"[{ix}]"], paths)
126
+
127
+ elif isinstance(v, dict):
128
+ for key, element in v.items():
129
+ get_generated_paths(element, path + [f"[{key}]"], paths)
130
+
131
+ elif isinstance(v, ConfigMixin):
132
+ for key in v.__xpm__._generated_values:
133
+ value = v.__xpm__.values[key]
134
+ if isinstance(value, ConfigMixin) and value.__xpm__._generated_values:
135
+ path.append(key)
136
+ get_generated_paths(value, path, paths)
137
+ path.pop()
138
+ else:
139
+ paths.append(".".join(path + [key]))
140
+ else:
141
+ raise ValueError(f"Cannot handle type {type(v)}")
142
+ return paths
143
+
144
+
113
145
  class ConfigInformation:
114
146
  """Holds experimaestro information for a config (or task) instance"""
115
147
 
@@ -122,11 +154,11 @@ class ConfigInformation:
122
154
  def __init__(self, pyobject: "ConfigMixin"):
123
155
  # The underlying pyobject and XPM type
124
156
  self.pyobject = pyobject
125
- self.xpmtype = pyobject.__xpmtype__ # type: ObjectType
157
+ self.xpmtype: "ObjectType" = pyobject.__xpmtype__
126
158
  self.values = {}
127
159
 
128
160
  # Meta-informations
129
- self._tags = {}
161
+ self._tags: dict[str, Any] = {}
130
162
  self._initinfo = ""
131
163
 
132
164
  self._taskoutput = None
@@ -142,16 +174,13 @@ class ConfigInformation:
142
174
  #: True when this configuration was loaded from disk
143
175
  self.loaded = False
144
176
 
145
- # Explicitely added dependencies
177
+ # Explicitly added dependencies
146
178
  self.dependencies = []
147
179
 
148
180
  # Concrete type variables resolutions
149
181
  # This is used to check typevars coherence
150
182
  self.concrete_typevars: Dict[TypeVar, type] = {}
151
183
 
152
- # Lightweight tasks
153
- self.pre_tasks: List["LightweightTask"] = []
154
-
155
184
  # Initialization tasks
156
185
  self.init_tasks: List["LightweightTask"] = []
157
186
 
@@ -160,16 +189,18 @@ class ConfigInformation:
160
189
 
161
190
  # Cached information
162
191
 
163
- self._full_identifier = None
164
- """The full identifier (with pre-tasks)"""
165
-
166
- self._raw_identifier = None
167
- """The identifier without taking into account pre-tasks"""
192
+ self._identifier = None
193
+ """The configuration identifier (cached when sealed)"""
168
194
 
169
195
  self._validated = False
170
196
  self._sealed = False
171
197
  self._meta = None
172
198
 
199
+ # This contains the list of generated values (using context) in this
200
+ # configuration or any sub-configuration, is generated. This prevents
201
+ # problem when a configuration with generated values is re-used.
202
+ self._generated_values = []
203
+
173
204
  def set_meta(self, value: Optional[bool]):
174
205
  """Sets the meta flag"""
175
206
  assert not self._sealed, "Configuration is sealed"
@@ -187,6 +218,31 @@ class ConfigInformation:
187
218
  # Not an argument, bypass
188
219
  return object.__getattribute__(self.pyobject, name)
189
220
 
221
+ @staticmethod
222
+ def is_generated_value(argument, value):
223
+ if argument.ignore_generated:
224
+ return False
225
+
226
+ if value is None:
227
+ return False
228
+
229
+ if isinstance(value, (int, str, float, bool, Enum, Path)):
230
+ return False
231
+
232
+ if isinstance(value, ConfigMixin):
233
+ return value.__xpm__._generated_values and value.__xpm__.task is None
234
+
235
+ if isinstance(value, list):
236
+ return any(ConfigInformation.is_generated_value(argument, x) for x in value)
237
+
238
+ if isinstance(value, dict):
239
+ return any(
240
+ ConfigInformation.is_generated_value(argument, x)
241
+ for x in value.values()
242
+ )
243
+
244
+ return False
245
+
190
246
  def set(self, k, v, bypass=False):
191
247
  from experimaestro.generators import Generator
192
248
 
@@ -198,9 +254,21 @@ class ConfigInformation:
198
254
  if self._sealed and not bypass:
199
255
  raise AttributeError(f"Object is read-only (trying to set {k})")
200
256
 
257
+ if not isinstance(v, ConfigMixin) and isinstance(v, Config):
258
+ raise AttributeError(
259
+ "Configuration (and not objects) should be used. Consider using .C(...)"
260
+ )
261
+
201
262
  try:
202
263
  argument = self.xpmtype.arguments.get(k, None)
203
264
  if argument:
265
+ if ConfigInformation.is_generated_value(argument, v):
266
+ raise AttributeError(
267
+ f"Cannot set {k} to a configuration with generated values. "
268
+ "Here is the list of paths to help you: "
269
+ f"""{', '.join(get_generated_paths(v, [k]))}"""
270
+ )
271
+
204
272
  if not bypass and (
205
273
  (isinstance(argument.generator, Generator)) or argument.constant
206
274
  ):
@@ -302,10 +370,6 @@ class ConfigInformation:
302
370
  % (k, self.xpmtype, self._initinfo)
303
371
  )
304
372
 
305
- # Validate pre-tasks
306
- for pre_task in self.pre_tasks:
307
- pre_task.__xpm__.validate()
308
-
309
373
  # Validate init tasks
310
374
  for init_task in self.init_tasks:
311
375
  init_task.__xpm__.validate()
@@ -326,12 +390,21 @@ class ConfigInformation:
326
390
  Arguments:
327
391
  - context: the generation context
328
392
  """
393
+ if generated_keys := [
394
+ k
395
+ for k, v in self.values.items()
396
+ if ConfigInformation.is_generated_value(self.xpmtype.arguments[k], v)
397
+ ]:
398
+ raise AttributeError(
399
+ "Cannot seal a configuration with generated values:"
400
+ f"""{",".join(generated_keys)} in {context.currentpath}"""
401
+ )
329
402
 
330
403
  class Sealer(ConfigWalk):
331
- def preprocess(self, config: Config):
404
+ def preprocess(self, config: ConfigMixin):
332
405
  return not config.__xpm__._sealed, config
333
406
 
334
- def postprocess(self, stub, config: Config, values):
407
+ def postprocess(self, stub, config: ConfigMixin, values):
335
408
  # Generate values
336
409
  from experimaestro.generators import Generator
337
410
 
@@ -344,22 +417,42 @@ class ConfigInformation:
344
417
  continue
345
418
  value = argument.generator()
346
419
  else:
420
+ # Generate a value
347
421
  sig = inspect.signature(argument.generator)
348
422
  if len(sig.parameters) == 0:
349
423
  value = argument.generator()
350
424
  elif len(sig.parameters) == 2:
425
+ # Only in that case do we need to flag this configuration
426
+ # as containing generated values
427
+ if not argument.ignore_generated:
428
+ config.__xpm__._generated_values.append(k)
429
+ else:
430
+ logging.warning("Ignoring %s", k)
351
431
  value = argument.generator(self.context, config)
352
432
  else:
353
433
  assert (
354
434
  False
355
435
  ), "generator has either two parameters (context and config), or none"
356
436
  config.__xpm__.set(k, value, bypass=True)
437
+ else:
438
+ value = config.__xpm__.values.get(k)
357
439
  except Exception:
358
440
  logger.error(
359
441
  "While setting %s of %s", argument.name, config.__xpmtype__
360
442
  )
361
443
  raise
362
444
 
445
+ # Propagate the generated value flag
446
+ if (
447
+ value is not None
448
+ and isinstance(value, ConfigMixin)
449
+ and value.__xpm__._generated_values
450
+ ):
451
+ if not argument.ignore_generated:
452
+ config.__xpm__._generated_values.append(k)
453
+ else:
454
+ logging.warning("Ignoring %s", k)
455
+
363
456
  config.__xpm__._sealed = True
364
457
 
365
458
  Sealer(context, recurse_task=True)(self.pyobject)
@@ -372,90 +465,29 @@ class ConfigInformation:
372
465
  context = ConfigWalkContext()
373
466
 
374
467
  class Unsealer(ConfigWalk):
375
- def preprocess(self, config: Config):
468
+ def preprocess(self, config: ConfigMixin):
376
469
  return config.__xpm__._sealed, config
377
470
 
378
- def postprocess(self, stub, config: Config, values):
471
+ def postprocess(self, stub, config: ConfigMixin, values):
379
472
  config.__xpm__._sealed = False
380
473
  config.__xpm__._identifier = None
381
474
 
382
475
  Unsealer(context, recurse_task=True)(self.pyobject)
383
476
 
384
- def collect_pre_tasks(self) -> Iterator["Config"]:
385
- context = ConfigWalkContext()
386
- pre_tasks: Dict[int, "Config"] = {}
387
-
388
- class PreTaskCollect(ConfigWalk):
389
- def preprocess(self, config: Config):
390
- # Do not cross tasks
391
- return not isinstance(config.__xpm__, Task), config
392
-
393
- def postprocess(self, stub, config: Config, values):
394
- pre_tasks.update(
395
- {id(pre_task): pre_task for pre_task in config.__xpm__.pre_tasks}
396
- )
397
-
398
- PreTaskCollect(context, recurse_task=True)(self.pyobject)
399
- return pre_tasks.values()
400
-
401
- def identifiers(self, only_raw: bool):
477
+ @property
478
+ def identifier(self):
402
479
  """Computes the unique identifier"""
403
- from ..identifier import IdentifierComputer, Identifier
404
-
405
- raw_identifier = self._raw_identifier
406
- full_identifier = self._full_identifier
480
+ from ..identifier import IdentifierComputer
407
481
 
408
482
  # Computes raw identifier if needed
409
- if raw_identifier is None or not self._sealed:
410
- # Get the main identifier
411
- raw_identifier = IdentifierComputer.compute(self.pyobject)
412
- if self._sealed:
413
- self._raw_identifier = raw_identifier
414
-
415
- if only_raw:
416
- return raw_identifier, full_identifier
417
-
418
- # OK, let's compute the full identifier
419
- if full_identifier is None or not self._sealed:
420
- # Compute the full identifier by including the pre-tasks
421
- hasher = hashlib.sha256()
422
- hasher.update(raw_identifier.all)
423
- pre_tasks_ids = [
424
- pre_task.__xpm__.raw_identifier.all
425
- for pre_task in self.collect_pre_tasks()
426
- ]
427
- for task_id in sorted(pre_tasks_ids):
428
- hasher.update(task_id)
429
-
430
- # Adds init tasks
431
- if self.init_tasks:
432
- hasher.update(IdentifierComputer.INIT_TASKS)
433
- for init_task in self.init_tasks:
434
- hasher.update(init_task.__xpm__.raw_identifier.all)
435
-
436
- full_identifier = Identifier(hasher.digest())
437
- full_identifier.has_loops = raw_identifier.has_loops
438
-
439
- # Only cache the identifier if sealed
440
- if self._sealed:
441
- self._full_identifier = full_identifier
483
+ if self._identifier is not None:
484
+ return self._identifier
442
485
 
443
- return raw_identifier, full_identifier
444
-
445
- @property
446
- def raw_identifier(self) -> "Identifier":
447
- """Computes the unique identifier (without task modifiers)"""
448
- raw_identifier, _ = self.identifiers(True)
449
- return raw_identifier
450
-
451
- @property
452
- def full_identifier(self) -> "Identifier":
453
- """Computes the unique identifier (with task modifiers)"""
454
- _, full_identifier = self.identifiers(False)
455
- return full_identifier
456
-
457
- identifier = full_identifier
458
- """Deprecated: use full_identifier"""
486
+ # Get the main identifier
487
+ identifier = IdentifierComputer.compute(self.pyobject)
488
+ if self._sealed:
489
+ self._identifier = identifier
490
+ return identifier
459
491
 
460
492
  def dependency(self):
461
493
  """Returns a dependency"""
@@ -470,12 +502,6 @@ class ConfigInformation:
470
502
  path: List[str],
471
503
  taskids: Set[int],
472
504
  ):
473
- # Add pre-tasks
474
- for pre_task in self.pre_tasks:
475
- pre_task.__xpm__.updatedependencies(
476
- dependencies, path + ["__pre_tasks__"], taskids
477
- )
478
-
479
505
  # Add initialization tasks
480
506
  for init_task in self.init_tasks:
481
507
  init_task.__xpm__.updatedependencies(
@@ -618,10 +644,11 @@ class ConfigInformation:
618
644
  ) or RunMode.NORMAL
619
645
  if run_mode == RunMode.NORMAL:
620
646
  TaskEventListener.connect(experiment.CURRENT)
647
+ experiment.CURRENT.submit(self.job)
621
648
  other = experiment.CURRENT.submit(self.job)
622
649
  if other:
623
- # Just returns the other task
624
- return other.config.__xpm__._taskoutput
650
+ # Our job = previously submitted job
651
+ self.job = other
625
652
  else:
626
653
  # Show a warning
627
654
  if run_mode == RunMode.GENERATE_ONLY:
@@ -657,13 +684,6 @@ class ConfigInformation:
657
684
 
658
685
  print(file=sys.stderr) # noqa: T201
659
686
 
660
- # Handle an output configuration # FIXME: remove
661
- def mark_output(config: "Config"):
662
- """Sets a dependency on the job"""
663
- assert not isinstance(config, Task), "Cannot set a dependency on a task"
664
- config.__xpm__.task = self.pyobject
665
- return config
666
-
667
687
  # Mark this configuration also
668
688
  self.task = self.pyobject
669
689
 
@@ -677,6 +697,9 @@ class ConfigInformation:
677
697
  def mark_output(self, config: "Config"):
678
698
  """Sets a dependency on the job"""
679
699
  assert not isinstance(config, Task), "Cannot set a dependency on a task"
700
+ assert isinstance(
701
+ config, ConfigMixin
702
+ ), "Only configurations can be marked as dependent on a task"
680
703
  config.__xpm__.task = self.pyobject
681
704
  return config
682
705
 
@@ -752,9 +775,6 @@ class ConfigInformation:
752
775
  if self.task is not None and self.task is not self:
753
776
  ConfigInformation.__collect_objects__(self.task, objects, context)
754
777
 
755
- # Serialize pre-tasks
756
- ConfigInformation.__collect_objects__(self.pre_tasks, objects, context)
757
-
758
778
  # Serialize initialization tasks
759
779
  ConfigInformation.__collect_objects__(self.init_tasks, objects, context)
760
780
 
@@ -762,14 +782,12 @@ class ConfigInformation:
762
782
  state_dict = {
763
783
  "id": id(self.pyobject),
764
784
  "module": self.xpmtype._module,
765
- "type": self.xpmtype.basetype.__qualname__,
785
+ "type": self.xpmtype.value_type.__qualname__,
766
786
  "typename": self.xpmtype.name(),
767
787
  "identifier": self.identifier.state_dict(),
768
788
  }
769
789
 
770
790
  # Add pre/init tasks
771
- if self.pre_tasks:
772
- state_dict["pre-tasks"] = [id(pre_task) for pre_task in self.pre_tasks]
773
791
  if self.init_tasks:
774
792
  state_dict["init-tasks"] = [id(init_task) for init_task in self.init_tasks]
775
793
 
@@ -855,6 +873,7 @@ class ConfigInformation:
855
873
  "workspace": str(context.workspace.path.absolute()),
856
874
  "tags": {key: value for key, value in self.tags().items()},
857
875
  "version": 2,
876
+ "experimaestro": experimaestro.__version__,
858
877
  "objects": self.__get_objects__([], context),
859
878
  },
860
879
  out,
@@ -949,34 +968,31 @@ class ConfigInformation:
949
968
 
950
969
  @overload
951
970
  @staticmethod
952
- def fromParameters(
971
+ def fromParameters( # noqa: E704
953
972
  definitions: List[Dict],
954
973
  as_instance=True,
955
974
  save_directory: Optional[Path] = None,
956
975
  discard_id: bool = False,
957
- ) -> "ConfigMixin":
958
- ...
976
+ ) -> "ConfigMixin": ...
959
977
 
960
978
  @overload
961
979
  @staticmethod
962
- def fromParameters(
980
+ def fromParameters( # noqa: E704
963
981
  definitions: List[Dict],
964
982
  as_instance=False,
965
983
  return_tasks=True,
966
984
  save_directory: Optional[Path] = None,
967
985
  discard_id: bool = False,
968
- ) -> Tuple["Config", List["LightweightTask"]]:
969
- ...
986
+ ) -> Tuple["Config", List["LightweightTask"]]: ...
970
987
 
971
988
  @overload
972
989
  @staticmethod
973
- def fromParameters(
990
+ def fromParameters( # noqa: E704
974
991
  definitions: List[Dict],
975
992
  as_instance=False,
976
993
  save_directory: Optional[Path] = None,
977
994
  discard_id: bool = False,
978
- ) -> "Config":
979
- ...
995
+ ) -> "Config": ...
980
996
 
981
997
  @staticmethod
982
998
  def load_objects( # noqa: C901
@@ -1022,7 +1038,7 @@ class ConfigInformation:
1022
1038
 
1023
1039
  # Creates an object (or a config)
1024
1040
  if as_instance:
1025
- o = cls.XPMValue.__new__(cls.XPMValue)
1041
+ o = cls.__new__(cls)
1026
1042
  else:
1027
1043
  o = cls.XPMConfig.__new__(cls.XPMConfig)
1028
1044
  assert definition["id"] not in objects, "Duplicate id %s" % definition["id"]
@@ -1101,12 +1117,6 @@ class ConfigInformation:
1101
1117
  o.__post_init__()
1102
1118
 
1103
1119
  else:
1104
- # Sets pre-tasks
1105
- o.__xpm__.pre_tasks = [
1106
- objects[pre_task_id]
1107
- for pre_task_id in definition.get("pre-tasks", [])
1108
- ]
1109
-
1110
1120
  if task_id := definition.get("task", None):
1111
1121
  o.__xpm__.task = objects[task_id]
1112
1122
 
@@ -1140,15 +1150,6 @@ class ConfigInformation:
1140
1150
 
1141
1151
  # Run pre-task (or returns them)
1142
1152
  if as_instance or return_tasks:
1143
- # Collect pre-tasks (just once)
1144
- completed_pretasks = set()
1145
- pre_tasks = []
1146
- for definition in definitions:
1147
- for pre_task_id in definition.get("pre-tasks", []):
1148
- if pre_task_id not in completed_pretasks:
1149
- completed_pretasks.add(pre_task_id)
1150
- pre_tasks.append(objects[pre_task_id])
1151
-
1152
1153
  # Collect init tasks
1153
1154
  init_tasks = []
1154
1155
  for init_task_id in definitions[-1].get("init-tasks", []):
@@ -1156,14 +1157,11 @@ class ConfigInformation:
1156
1157
  init_tasks.append(init_task)
1157
1158
 
1158
1159
  if as_instance:
1159
- for pre_task in pre_tasks:
1160
- logger.info("Executing pre-task %s", type(pre_task))
1161
- pre_task.execute()
1162
1160
  for init_task in init_tasks:
1163
1161
  logger.info("Executing init task %s", type(init_task))
1164
1162
  init_task.execute()
1165
1163
  else:
1166
- return o, pre_tasks, pre_task + init_tasks
1164
+ return o, init_tasks
1167
1165
 
1168
1166
  return o
1169
1167
 
@@ -1171,7 +1169,6 @@ class ConfigInformation:
1171
1169
  def __init__(self, context: ConfigWalkContext, *, objects: ObjectStore = None):
1172
1170
  super().__init__(context)
1173
1171
  self.objects = ObjectStore() if objects is None else objects
1174
- self.pre_tasks = {}
1175
1172
 
1176
1173
  def preprocess(self, config: "Config"):
1177
1174
  if self.objects.is_constructed(id(config)):
@@ -1183,7 +1180,7 @@ class ConfigInformation:
1183
1180
 
1184
1181
  if o is None:
1185
1182
  # Creates an object (and not a config)
1186
- o = config.XPMValue()
1183
+ o = config.__xpmtype__.value_type()
1187
1184
 
1188
1185
  # Store in cache
1189
1186
  self.objects.add_stub(id(config), o)
@@ -1198,10 +1195,6 @@ class ConfigInformation:
1198
1195
  # Call __post_init__
1199
1196
  stub.__post_init__()
1200
1197
 
1201
- # Gather pre-tasks
1202
- for pre_task in config.__xpm__.pre_tasks:
1203
- self.pre_tasks[id(pre_task)] = self.stub(pre_task)
1204
-
1205
1198
  self.objects.set_constructed(id(config))
1206
1199
  return stub
1207
1200
 
@@ -1215,10 +1208,6 @@ class ConfigInformation:
1215
1208
  processor = ConfigInformation.FromPython(context, objects=objects)
1216
1209
  last_object = processor(self.pyobject)
1217
1210
 
1218
- # Execute pre-tasks
1219
- for pre_task in processor.pre_tasks.values():
1220
- pre_task.execute()
1221
-
1222
1211
  return last_object
1223
1212
 
1224
1213
  def add_dependencies(self, *dependencies):
@@ -1242,6 +1231,9 @@ def clone(v):
1242
1231
  if isinstance(v, Enum):
1243
1232
  return v
1244
1233
 
1234
+ if isinstance(v, tuple):
1235
+ return tuple(clone(x) for x in v)
1236
+
1245
1237
  if isinstance(v, Config):
1246
1238
  # Create a new instance
1247
1239
  kwargs = {
@@ -1260,6 +1252,11 @@ class ConfigMixin:
1260
1252
  """Class for configuration objects"""
1261
1253
 
1262
1254
  __xpmtype__: ObjectType
1255
+ """The associated XPM type"""
1256
+
1257
+ __xpm__: ConfigInformation
1258
+ """The __xpm__ object contains all instance specific information about a
1259
+ configuration/task"""
1263
1260
 
1264
1261
  def __init__(self, **kwargs):
1265
1262
  """Initialize the configuration with the given parameters"""
@@ -1310,8 +1307,8 @@ class ConfigMixin:
1310
1307
  [f"{key}={value}" for key, value in self.__xpm__.values.items()]
1311
1308
  )
1312
1309
  return (
1313
- f"{self.__xpmtype__.basetype.__module__}."
1314
- f"{self.__xpmtype__.basetype.__qualname__}({params})"
1310
+ f"{self.__xpmtype__.value_type.__module__}."
1311
+ f"{self.__xpmtype__.value_type.__qualname__}({params})"
1315
1312
  )
1316
1313
 
1317
1314
  def tag(self, name, value):
@@ -1340,9 +1337,20 @@ class ConfigMixin:
1340
1337
  return self
1341
1338
 
1342
1339
  def instance(
1343
- self, context: ConfigWalkContext = None, *, objects: ObjectStore = None
1340
+ self,
1341
+ context: ConfigWalkContext = None,
1342
+ *,
1343
+ objects: ObjectStore = None,
1344
+ keep: bool = True,
1344
1345
  ) -> T:
1345
- """Return an instance with the current values"""
1346
+ """Return an instance with the current values
1347
+
1348
+ :param context: The context when computing the instance
1349
+ :param objects: The previously built objects (so that we avoid
1350
+ re-creating instances of past configurations)
1351
+ :param keep: register a configuration in the __config__ field of the
1352
+ instance
1353
+ """
1346
1354
  if context is None:
1347
1355
  from experimaestro.xpmutils import EmptyContext
1348
1356
 
@@ -1351,7 +1359,11 @@ class ConfigMixin:
1351
1359
  assert isinstance(
1352
1360
  context, ConfigWalkContext
1353
1361
  ), f"{context.__class__} is not an instance of ConfigWalkContext"
1354
- return self.__xpm__.fromConfig(context, objects=objects) # type: ignore
1362
+
1363
+ instance = self.__xpm__.fromConfig(context, objects=objects) # type: ignore
1364
+ if keep:
1365
+ object.__setattr__(instance, "__config__", self)
1366
+ return instance
1355
1367
 
1356
1368
  def submit(
1357
1369
  self,
@@ -1396,29 +1408,7 @@ class ConfigMixin:
1396
1408
  attributes)"""
1397
1409
  return clone(self)
1398
1410
 
1399
- def add_pretasks(self, *tasks: "LightweightTask"):
1400
- assert all(
1401
- [isinstance(task, LightweightTask) for task in tasks]
1402
- ), "One of the pre-tasks are not lightweight tasks"
1403
- if self.__xpm__._sealed:
1404
- raise SealedError("Cannot add pre-tasks to a sealed configuration")
1405
- self.__xpm__.pre_tasks.extend(tasks)
1406
- return self
1407
-
1408
- def add_pretasks_from(self, *configs: "Config"):
1409
- assert all(
1410
- [isinstance(config, ConfigMixin) for config in configs]
1411
- ), "One of the parameters is not a configuration object"
1412
- for config in configs:
1413
- self.add_pretasks(*config.__xpm__.pre_tasks)
1414
- return self
1415
-
1416
- @property
1417
- def pre_tasks(self) -> List["LightweightTask"]:
1418
- """Access pre-tasks"""
1419
- return self.__xpm__.pre_tasks
1420
-
1421
- def copy_dependencies(self, other: "Config"):
1411
+ def copy_dependencies(self, other: "ConfigMixin"):
1422
1412
  """Add all the dependencies from other configuration"""
1423
1413
 
1424
1414
  # Add task dependency
@@ -1441,52 +1431,17 @@ class Config:
1441
1431
  """The object type holds all the information about a specific subclass
1442
1432
  experimaestro metadata"""
1443
1433
 
1444
- __xpm__: ConfigInformation
1445
- """The __xpm__ object contains all instance specific information about a
1446
- configuration/task"""
1447
-
1448
1434
  @classproperty
1449
1435
  def XPMConfig(cls):
1450
1436
  if issubclass(cls, ConfigMixin):
1451
1437
  return cls
1452
- return cls.__getxpmtype__().configtype
1453
-
1454
- @classproperty
1455
- def XPMValue(cls):
1456
- """Returns the value object for this configuration"""
1457
- if issubclass(cls, ConfigMixin):
1458
- return cls.__xpmtype__.objecttype
1459
-
1460
- if value_cls := cls.__dict__.get("__XPMValue__", None):
1461
- pass
1462
- else:
1463
- from ..types import XPMValue
1464
-
1465
- __objectbases__ = tuple(
1466
- s.XPMValue
1467
- for s in cls.__bases__
1468
- if issubclass(s, Config) and (s is not Config)
1469
- ) or (XPMValue,)
1470
-
1471
- *tp_qual, tp_name = cls.__qualname__.split(".")
1472
- value_cls = type(f"{tp_name}.XPMValue", (cls,) + __objectbases__, {})
1473
- value_cls.__qualname__ = ".".join(tp_qual + [value_cls.__name__])
1474
- value_cls.__module__ = cls.__module__
1475
-
1476
- setattr(cls, "__XPMValue__", value_cls)
1477
-
1478
- return value_cls
1438
+ return cls.__getxpmtype__().config_type
1479
1439
 
1480
1440
  @classproperty
1481
1441
  def C(cls):
1482
1442
  """Alias for XPMConfig"""
1483
1443
  return cls.XPMConfig
1484
1444
 
1485
- @classproperty
1486
- def V(cls):
1487
- """Alias for XPMValue"""
1488
- return cls.XPMValue
1489
-
1490
1445
  @classmethod
1491
1446
  def __getxpmtype__(cls) -> "ObjectType":
1492
1447
  """Get (and create if necessary) the Object type associated
@@ -1503,46 +1458,6 @@ class Config:
1503
1458
  raise
1504
1459
  return xpmtype
1505
1460
 
1506
- def __new__(cls: Type[T], *args, **kwargs) -> T:
1507
- """Returns an instance of a ConfigMixin (for compatibility, use XPMConfig
1508
- or C if possible)
1509
-
1510
- :deprecated: Use Config.C or Config.XPMConfig to construct a new
1511
- configuration, and Config.V (or Config.XPMValue) for a new value
1512
- """
1513
- # If this is an XPMValue, just return a new instance
1514
- from experimaestro.core.types import XPMValue
1515
-
1516
- if issubclass(cls, XPMValue):
1517
- return object.__new__(cls)
1518
-
1519
- # If this is the XPMConfig, just return a new instance
1520
- # __init__ will be called
1521
- if issubclass(cls, ConfigMixin):
1522
- return object.__new__(cls)
1523
-
1524
- # Log a deprecation warning for this way of creating a configuration
1525
- caller = inspect.getframeinfo(inspect.stack()[1][0])
1526
- logger.warning(
1527
- "Creating a configuration using Config.__new__ is deprecated, and will be removed in a future version. "
1528
- "Use Config.C or Config.XPMConfig to create a new configuration. "
1529
- "Issue created at %s:%s",
1530
- str(Path(caller.filename).absolute()),
1531
- caller.lineno,
1532
- )
1533
-
1534
- # otherwise, we use the configuration type
1535
- o: ConfigMixin = object.__new__(cls.__getxpmtype__().configtype)
1536
- try:
1537
- o.__init__(*args, **kwargs)
1538
- except Exception:
1539
- logger.error(
1540
- "Init error in %s:%s"
1541
- % (str(Path(caller.filename).absolute()), caller.lineno)
1542
- )
1543
- raise
1544
- return o
1545
-
1546
1461
  def __validate__(self):
1547
1462
  """Validate the values"""
1548
1463
  pass
@@ -1557,17 +1472,7 @@ class Config:
1557
1472
  return self.__xpm__.__json__()
1558
1473
 
1559
1474
  def __identifier__(self) -> "Identifier":
1560
- return self.__xpm__.full_identifier
1561
-
1562
- def add_pretasks(self, *tasks: "LightweightTask"):
1563
- """Add pre-tasks"""
1564
- raise AssertionError("This method can only be used during configuration")
1565
-
1566
- def add_pretasks_from(self, *configs: "Config"):
1567
- """Add pre-tasks from the listed configurations"""
1568
- raise AssertionError(
1569
- "The 'add_pretasks_from' can only be used during configuration"
1570
- )
1475
+ return self.__xpm__.identifier
1571
1476
 
1572
1477
  def copy_dependencies(self, other: "Config"):
1573
1478
  """Add pre-tasks from the listed configurations"""
@@ -1575,11 +1480,6 @@ class Config:
1575
1480
  "The 'copy_dependencies' method can only be used during configuration"
1576
1481
  )
1577
1482
 
1578
- @property
1579
- def pre_tasks(self) -> List["LightweightTask"]:
1580
- """Access pre-tasks"""
1581
- raise AssertionError("Pre-tasks can be accessed only during configuration")
1582
-
1583
1483
  def register_task_output(self, method, *args, **kwargs):
1584
1484
  # Determine the path for this...
1585
1485
  path = taskglobals.Env.instance().xpm_path / "task-outputs.jsonl"