experimaestro 1.10.0__py3-none-any.whl → 1.16.0__py3-none-any.whl

experimaestro/core/objects/config.py

@@ -9,7 +9,6 @@ from experimaestro import taskglobals
 
 from termcolor import cprint
 from pathlib import Path
-import hashlib
 import logging
 import io
 from enum import Enum
@@ -20,7 +19,6 @@ from typing import (
     Callable,
     ClassVar,
     Dict,
-    Iterator,
     List,
     Optional,
     Set,
@@ -49,7 +47,6 @@ from .config_walk import ConfigWalk, ConfigWalkContext
 from .config_utils import (
     getqualattr,
     add_to_path,
-    SealedError,
     TaggedValue,
     ObjectStore,
     classproperty,
@@ -122,11 +119,11 @@ class ConfigInformation:
     def __init__(self, pyobject: "ConfigMixin"):
         # The underlying pyobject and XPM type
         self.pyobject = pyobject
-        self.xpmtype = pyobject.__xpmtype__  # type: ObjectType
+        self.xpmtype: "ObjectType" = pyobject.__xpmtype__
         self.values = {}
 
         # Meta-informations
-        self._tags = {}
+        self._tags: dict[str, Any] = {}
         self._initinfo = ""
 
         self._taskoutput = None
@@ -142,16 +139,13 @@ class ConfigInformation:
         #: True when this configuration was loaded from disk
         self.loaded = False
 
-        # Explicitely added dependencies
+        # Explicitly added dependencies
         self.dependencies = []
 
         # Concrete type variables resolutions
         # This is used to check typevars coherence
         self.concrete_typevars: Dict[TypeVar, type] = {}
 
-        # Lightweight tasks
-        self.pre_tasks: List["LightweightTask"] = []
-
         # Initialization tasks
         self.init_tasks: List["LightweightTask"] = []
 
@@ -160,16 +154,41 @@ class ConfigInformation:
 
         # Cached information
 
-        self._full_identifier = None
-        """The full identifier (with pre-tasks)"""
-
-        self._raw_identifier = None
-        """The identifier without taking into account pre-tasks"""
+        self._identifier = None
+        """The configuration identifier (cached when sealed)"""
 
         self._validated = False
         self._sealed = False
         self._meta = None
 
+        # This contains the list of generated values (using context) in this
+        # configuration or any sub-configuration, is generated. This prevents
+        # problem when a configuration with generated values is re-used.
+        self._generated_values = []
+
+    def get_generated_paths(
+        self, path: list[str] = None, paths: list[str] = None
+    ) -> list[str]:
+        """Get the list of generated paths, useful to track down those
+
+        :param path: The current path
+        :param paths: The list of generated paths so far, defaults to None
+        :return: The full list of generated paths
+        """
+        paths = [] if paths is None else paths
+        path = [] if path is None else path
+
+        for key in self._generated_values:
+            value = self.values[key]
+            if isinstance(value, ConfigMixin) and value.__xpm__._generated_values:
+                path.append(key)
+                value.__xpm__.get_generated_paths(path, paths)
+                path.pop()
+            else:
+                paths.append(".".join(path + [key]))
+
+        return paths
+
     def set_meta(self, value: Optional[bool]):
         """Sets the meta flag"""
         assert not self._sealed, "Configuration is sealed"
@@ -187,6 +206,31 @@ class ConfigInformation:
         # Not an argument, bypass
         return object.__getattribute__(self.pyobject, name)
 
+    @staticmethod
+    def is_generated_value(argument, value):
+        if argument.ignore_generated:
+            return False
+
+        if value is None:
+            return False
+
+        if isinstance(value, (int, str, float, bool, Path)):
+            return False
+
+        if isinstance(value, ConfigMixin):
+            return value.__xpm__._generated_values and value.__xpm__.task is None
+
+        if isinstance(value, list):
+            return any(ConfigInformation.is_generated_value(argument, x) for x in value)
+
+        if isinstance(value, dict):
+            return any(
+                ConfigInformation.is_generated_value(argument, x)
+                for x in value.values()
+            )
+
+        return False
+
     def set(self, k, v, bypass=False):
         from experimaestro.generators import Generator
 
@@ -198,9 +242,21 @@ class ConfigInformation:
         if self._sealed and not bypass:
             raise AttributeError(f"Object is read-only (trying to set {k})")
 
+        if not isinstance(v, ConfigMixin) and isinstance(v, Config):
+            raise AttributeError(
+                "Configuration (and not objects) should be used. Consider using .C(...)"
+            )
+
         try:
             argument = self.xpmtype.arguments.get(k, None)
             if argument:
+                if ConfigInformation.is_generated_value(argument, v):
+                    raise AttributeError(
+                        f"Cannot set {k} to a configuration with generated values. "
+                        "Here is the list of paths to help you: "
+                        f"""{', '.join(v.__xpm__.get_generated_paths([k]))}"""
+                    )
+
                 if not bypass and (
                     (isinstance(argument.generator, Generator)) or argument.constant
                 ):
@@ -302,10 +358,6 @@ class ConfigInformation:
                             % (k, self.xpmtype, self._initinfo)
                         )
 
-            # Validate pre-tasks
-            for pre_task in self.pre_tasks:
-                pre_task.__xpm__.validate()
-
             # Validate init tasks
             for init_task in self.init_tasks:
                 init_task.__xpm__.validate()
@@ -326,12 +378,21 @@ class ConfigInformation:
         Arguments:
             - context: the generation context
         """
+        if generated_keys := [
+            k
+            for k, v in self.values.items()
+            if ConfigInformation.is_generated_value(self.xpmtype.arguments[k], v)
+        ]:
+            raise AttributeError(
+                "Cannot seal a configuration with generated values:"
+                f"""{",".join(generated_keys)} in {context.currentpath}"""
+            )
 
         class Sealer(ConfigWalk):
-            def preprocess(self, config: Config):
+            def preprocess(self, config: ConfigMixin):
                 return not config.__xpm__._sealed, config
 
-            def postprocess(self, stub, config: Config, values):
+            def postprocess(self, stub, config: ConfigMixin, values):
                 # Generate values
                 from experimaestro.generators import Generator
 
@@ -344,22 +405,36 @@ class ConfigInformation:
                                     continue
                                 value = argument.generator()
                             else:
+                                # Generate a value
                                 sig = inspect.signature(argument.generator)
                                 if len(sig.parameters) == 0:
                                     value = argument.generator()
                                 elif len(sig.parameters) == 2:
+                                    # Only in that case do we need to flag this configuration
+                                    # as containing generated values
+                                    config.__xpm__._generated_values.append(k)
                                     value = argument.generator(self.context, config)
                                 else:
                                     assert (
                                         False
                                     ), "generator has either two parameters (context and config), or none"
                             config.__xpm__.set(k, value, bypass=True)
+                        else:
+                            value = config.__xpm__.values.get(k)
                     except Exception:
                         logger.error(
                             "While setting %s of %s", argument.name, config.__xpmtype__
                         )
                         raise
 
+                    # Propagate the generated value flag
+                    if (
+                        (value is not None)
+                        and isinstance(value, ConfigMixin)
+                        and value.__xpm__._generated_values
+                    ):
+                        config.__xpm__._generated_values.append(k)
+
                 config.__xpm__._sealed = True
 
         Sealer(context, recurse_task=True)(self.pyobject)
@@ -372,90 +447,29 @@ class ConfigInformation:
         context = ConfigWalkContext()
 
         class Unsealer(ConfigWalk):
-            def preprocess(self, config: Config):
+            def preprocess(self, config: ConfigMixin):
                 return config.__xpm__._sealed, config
 
-            def postprocess(self, stub, config: Config, values):
+            def postprocess(self, stub, config: ConfigMixin, values):
                 config.__xpm__._sealed = False
                 config.__xpm__._identifier = None
 
         Unsealer(context, recurse_task=True)(self.pyobject)
 
-    def collect_pre_tasks(self) -> Iterator["Config"]:
-        context = ConfigWalkContext()
-        pre_tasks: Dict[int, "Config"] = {}
-
-        class PreTaskCollect(ConfigWalk):
-            def preprocess(self, config: Config):
-                # Do not cross tasks
-                return not isinstance(config.__xpm__, Task), config
-
-            def postprocess(self, stub, config: Config, values):
-                pre_tasks.update(
-                    {id(pre_task): pre_task for pre_task in config.__xpm__.pre_tasks}
-                )
-
-        PreTaskCollect(context, recurse_task=True)(self.pyobject)
-        return pre_tasks.values()
-
-    def identifiers(self, only_raw: bool):
+    @property
+    def identifier(self):
         """Computes the unique identifier"""
-        from ..identifier import IdentifierComputer, Identifier
-
-        raw_identifier = self._raw_identifier
-        full_identifier = self._full_identifier
+        from ..identifier import IdentifierComputer
 
         # Computes raw identifier if needed
-        if raw_identifier is None or not self._sealed:
-            # Get the main identifier
-            raw_identifier = IdentifierComputer.compute(self.pyobject)
-            if self._sealed:
-                self._raw_identifier = raw_identifier
-
-        if only_raw:
-            return raw_identifier, full_identifier
-
-        # OK, let's compute the full identifier
-        if full_identifier is None or not self._sealed:
-            # Compute the full identifier by including the pre-tasks
-            hasher = hashlib.sha256()
-            hasher.update(raw_identifier.all)
-            pre_tasks_ids = [
-                pre_task.__xpm__.raw_identifier.all
-                for pre_task in self.collect_pre_tasks()
-            ]
-            for task_id in sorted(pre_tasks_ids):
-                hasher.update(task_id)
-
-            # Adds init tasks
-            if self.init_tasks:
-                hasher.update(IdentifierComputer.INIT_TASKS)
-                for init_task in self.init_tasks:
-                    hasher.update(init_task.__xpm__.raw_identifier.all)
-
-            full_identifier = Identifier(hasher.digest())
-            full_identifier.has_loops = raw_identifier.has_loops
+        if self._identifier is not None:
+            return self._identifier
 
-            # Only cache the identifier if sealed
-            if self._sealed:
-                self._full_identifier = full_identifier
-
-        return raw_identifier, full_identifier
-
-    @property
-    def raw_identifier(self) -> "Identifier":
-        """Computes the unique identifier (without task modifiers)"""
-        raw_identifier, _ = self.identifiers(True)
-        return raw_identifier
-
-    @property
-    def full_identifier(self) -> "Identifier":
-        """Computes the unique identifier (with task modifiers)"""
-        _, full_identifier = self.identifiers(False)
-        return full_identifier
-
-    identifier = full_identifier
-    """Deprecated: use full_identifier"""
+        # Get the main identifier
+        identifier = IdentifierComputer.compute(self.pyobject)
+        if self._sealed:
+            self._identifier = identifier
+        return identifier
 
     def dependency(self):
         """Returns a dependency"""
@@ -470,12 +484,6 @@ class ConfigInformation:
         path: List[str],
         taskids: Set[int],
     ):
-        # Add pre-tasks
-        for pre_task in self.pre_tasks:
-            pre_task.__xpm__.updatedependencies(
-                dependencies, path + ["__pre_tasks__"], taskids
-            )
-
         # Add initialization tasks
         for init_task in self.init_tasks:
             init_task.__xpm__.updatedependencies(
@@ -618,10 +626,11 @@ class ConfigInformation:
         ) or RunMode.NORMAL
         if run_mode == RunMode.NORMAL:
             TaskEventListener.connect(experiment.CURRENT)
+            experiment.CURRENT.submit(self.job)
             other = experiment.CURRENT.submit(self.job)
             if other:
-                # Just returns the other task
-                return other.config.__xpm__._taskoutput
+                # Our job = previously submitted job
+                self.job = other
         else:
             # Show a warning
             if run_mode == RunMode.GENERATE_ONLY:
@@ -657,13 +666,6 @@ class ConfigInformation:
 
                 print(file=sys.stderr)  # noqa: T201
 
-        # Handle an output configuration # FIXME: remove
-        def mark_output(config: "Config"):
-            """Sets a dependency on the job"""
-            assert not isinstance(config, Task), "Cannot set a dependency on a task"
-            config.__xpm__.task = self.pyobject
-            return config
-
         # Mark this configuration also
         self.task = self.pyobject
 
@@ -752,9 +754,6 @@ class ConfigInformation:
         if self.task is not None and self.task is not self:
             ConfigInformation.__collect_objects__(self.task, objects, context)
 
-        # Serialize pre-tasks
-        ConfigInformation.__collect_objects__(self.pre_tasks, objects, context)
-
         # Serialize initialization tasks
         ConfigInformation.__collect_objects__(self.init_tasks, objects, context)
 
@@ -768,8 +767,6 @@ class ConfigInformation:
         }
 
         # Add pre/init tasks
-        if self.pre_tasks:
-            state_dict["pre-tasks"] = [id(pre_task) for pre_task in self.pre_tasks]
         if self.init_tasks:
             state_dict["init-tasks"] = [id(init_task) for init_task in self.init_tasks]
 
@@ -949,34 +946,31 @@ class ConfigInformation:
 
     @overload
     @staticmethod
-    def fromParameters(
+    def fromParameters(  # noqa: E704
         definitions: List[Dict],
         as_instance=True,
         save_directory: Optional[Path] = None,
         discard_id: bool = False,
-    ) -> "ConfigMixin":
-        ...
+    ) -> "ConfigMixin": ...
 
     @overload
     @staticmethod
-    def fromParameters(
+    def fromParameters(  # noqa: E704
         definitions: List[Dict],
         as_instance=False,
         return_tasks=True,
         save_directory: Optional[Path] = None,
         discard_id: bool = False,
-    ) -> Tuple["Config", List["LightweightTask"]]:
-        ...
+    ) -> Tuple["Config", List["LightweightTask"]]: ...
 
     @overload
     @staticmethod
-    def fromParameters(
+    def fromParameters(  # noqa: E704
         definitions: List[Dict],
         as_instance=False,
         save_directory: Optional[Path] = None,
         discard_id: bool = False,
-    ) -> "Config":
-        ...
+    ) -> "Config": ...
 
     @staticmethod
     def load_objects(  # noqa: C901
@@ -1101,12 +1095,6 @@ class ConfigInformation:
                 o.__post_init__()
 
             else:
-                # Sets pre-tasks
-                o.__xpm__.pre_tasks = [
-                    objects[pre_task_id]
-                    for pre_task_id in definition.get("pre-tasks", [])
-                ]
-
                 if task_id := definition.get("task", None):
                     o.__xpm__.task = objects[task_id]
 
@@ -1140,15 +1128,6 @@ class ConfigInformation:
 
         # Run pre-task (or returns them)
         if as_instance or return_tasks:
-            # Collect pre-tasks (just once)
-            completed_pretasks = set()
-            pre_tasks = []
-            for definition in definitions:
-                for pre_task_id in definition.get("pre-tasks", []):
-                    if pre_task_id not in completed_pretasks:
-                        completed_pretasks.add(pre_task_id)
-                        pre_tasks.append(objects[pre_task_id])
-
             # Collect init tasks
             init_tasks = []
             for init_task_id in definitions[-1].get("init-tasks", []):
@@ -1156,14 +1135,11 @@ class ConfigInformation:
                 init_tasks.append(init_task)
 
             if as_instance:
-                for pre_task in pre_tasks:
-                    logger.info("Executing pre-task %s", type(pre_task))
-                    pre_task.execute()
                 for init_task in init_tasks:
                     logger.info("Executing init task %s", type(init_task))
                     init_task.execute()
             else:
-                return o, pre_tasks, pre_task + init_tasks
+                return o, init_tasks
 
         return o
 
@@ -1171,7 +1147,6 @@ class ConfigInformation:
         def __init__(self, context: ConfigWalkContext, *, objects: ObjectStore = None):
             super().__init__(context)
             self.objects = ObjectStore() if objects is None else objects
-            self.pre_tasks = {}
 
         def preprocess(self, config: "Config"):
             if self.objects.is_constructed(id(config)):
@@ -1198,10 +1173,6 @@ class ConfigInformation:
             # Call __post_init__
             stub.__post_init__()
 
-            # Gather pre-tasks
-            for pre_task in config.__xpm__.pre_tasks:
-                self.pre_tasks[id(pre_task)] = self.stub(pre_task)
-
             self.objects.set_constructed(id(config))
             return stub
 
@@ -1215,10 +1186,6 @@ class ConfigInformation:
         processor = ConfigInformation.FromPython(context, objects=objects)
         last_object = processor(self.pyobject)
 
-        # Execute pre-tasks
-        for pre_task in processor.pre_tasks.values():
-            pre_task.execute()
-
         return last_object
 
     def add_dependencies(self, *dependencies):
@@ -1242,6 +1209,9 @@ def clone(v):
     if isinstance(v, Enum):
         return v
 
+    if isinstance(v, tuple):
+        return tuple(clone(x) for x in v)
+
     if isinstance(v, Config):
         # Create a new instance
         kwargs = {
@@ -1260,6 +1230,11 @@ class ConfigMixin:
     """Class for configuration objects"""
 
     __xpmtype__: ObjectType
+    """The associated XPM type"""
+
+    __xpm__: ConfigInformation
+    """The __xpm__ object contains all instance specific information about a
+    configuration/task"""
 
     def __init__(self, **kwargs):
         """Initialize the configuration with the given parameters"""
@@ -1396,29 +1371,7 @@ class ConfigMixin:
         attributes)"""
         return clone(self)
 
-    def add_pretasks(self, *tasks: "LightweightTask"):
-        assert all(
-            [isinstance(task, LightweightTask) for task in tasks]
-        ), "One of the pre-tasks are not lightweight tasks"
-        if self.__xpm__._sealed:
-            raise SealedError("Cannot add pre-tasks to a sealed configuration")
-        self.__xpm__.pre_tasks.extend(tasks)
-        return self
-
-    def add_pretasks_from(self, *configs: "Config"):
-        assert all(
-            [isinstance(config, ConfigMixin) for config in configs]
-        ), "One of the parameters is not a configuration object"
-        for config in configs:
-            self.add_pretasks(*config.__xpm__.pre_tasks)
-        return self
-
-    @property
-    def pre_tasks(self) -> List["LightweightTask"]:
-        """Access pre-tasks"""
-        return self.__xpm__.pre_tasks
-
-    def copy_dependencies(self, other: "Config"):
+    def copy_dependencies(self, other: "ConfigMixin"):
         """Add all the dependencies from other configuration"""
 
         # Add task dependency
@@ -1441,10 +1394,6 @@ class Config:
     """The object type holds all the information about a specific subclass
     experimaestro metadata"""
 
-    __xpm__: ConfigInformation
-    """The __xpm__ object contains all instance specific information about a
-    configuration/task"""
-
     @classproperty
     def XPMConfig(cls):
         if issubclass(cls, ConfigMixin):
@@ -1557,17 +1506,7 @@ class Config:
         return self.__xpm__.__json__()
 
     def __identifier__(self) -> "Identifier":
-        return self.__xpm__.full_identifier
-
-    def add_pretasks(self, *tasks: "LightweightTask"):
-        """Add pre-tasks"""
-        raise AssertionError("This method can only be used during configuration")
-
-    def add_pretasks_from(self, *configs: "Config"):
-        """Add pre-tasks from the listed configurations"""
-        raise AssertionError(
-            "The 'add_pretasks_from' can only be used during configuration"
-        )
+        return self.__xpm__.identifier
 
     def copy_dependencies(self, other: "Config"):
         """Add pre-tasks from the listed configurations"""
@@ -1575,11 +1514,6 @@ class Config:
             "The 'copy_dependencies' method can only be used during configuration"
         )
 
-    @property
-    def pre_tasks(self) -> List["LightweightTask"]:
-        """Access pre-tasks"""
-        raise AssertionError("Pre-tasks can be accessed only during configuration")
-
     def register_task_output(self, method, *args, **kwargs):
         # Determine the path for this...
         path = taskglobals.Env.instance().xpm_path / "task-outputs.jsonl"

experimaestro/core/objects/config_walk.py

@@ -71,6 +71,7 @@ class ConfigWalk:
         return self.context.push(str(i))
 
     def map(self, k: str):
+        """Provides a path context when processing a tree"""
         return self.context.push(k)
 
     def stub(self, config):
@@ -108,11 +109,7 @@ class ConfigWalk:
                 else:
                     result[arg.name] = None
 
-            # Deals with pre-tasks
-            if info.pre_tasks:
-                with self.map("__pre_tasks__"):
-                    self(info.pre_tasks)
-
+            # Deals with init tasks
             if info.init_tasks:
                 with self.map("__init_tasks__"):
                     self(info.init_tasks)
@@ -123,7 +120,8 @@ class ConfigWalk:
                 and self.recurse_task
                 and x.__xpm__.task is not x
             ):
-                self(x.__xpm__.task)
+                with self.map("__task__"):
+                    self(x.__xpm__.task)
 
             processed = self.postprocess(stub, x, result)
             self.visited[xid] = processed

experimaestro/core/objects.pyi

@@ -168,7 +168,7 @@ class ConfigMixin:
         *,
         workspace: Incomplete | None = ...,
         launcher: Incomplete | None = ...,
-        run_mode: RunMode = ...
+        run_mode: RunMode = ...,
     ): ...
     def stdout(self): ...
     def stderr(self): ...
@@ -195,11 +195,7 @@ class Config:
     def __post_init__(self) -> None: ...
     def __json__(self): ...
     def __identifier__(self) -> Identifier: ...
-    def add_pretasks(self, *tasks: "LightweightTask"): ...
-    def add_pretasks_from(self, configs: "Config"): ...
     def copy_dependencies(self, other: "Config"): ...
-    @property
-    def pre_tasks(self) -> List["LightweightTask"]: ...
 
 class LightweightTask(Config):
     def execute(self) -> None: ...
@@ -213,7 +209,7 @@ class Task(LightweightTask):
         workspace: Incomplete | None = ...,
         launcher: Incomplete | None = ...,
         run_mode: RunMode = ...,
-        init_tasks: List["LightweightTask"] = []
+        init_tasks: List["LightweightTask"] = [],
     ): ...
     def task_outputs(self, dep: Callable[[Config], None]) -> Any: ...
 

experimaestro/core/serializers.py

@@ -1,10 +1,8 @@
-from typing import List, TypeVar, Callable, Any
-from pathlib import Path
+from typing import List, TypeVar
 from experimaestro import Param
 
 from .objects import Config, LightweightTask
 from .arguments import DataPath
-from experimaestro import copyconfig
 
 
 class SerializationLWTask(LightweightTask):
@@ -39,8 +37,3 @@ class PathSerializationLWTask(SerializationLWTask):
 
     path: DataPath
     """Path containing the data"""
-
-    @classmethod
-    def construct(cls, value: T, path: Path, dep: Callable[[Config], Any]) -> T:
-        value = copyconfig(value)
-        return value.add_pretasks(dep(cls(value=value, path=path)))

experimaestro/core/types.py

@@ -13,10 +13,7 @@ from enum import Enum
 import ast
 import textwrap
 
-if sys.version_info.major == 3 and sys.version_info.minor < 9:
-    from typing_extensions import _AnnotatedAlias, get_type_hints
-else:
-    from typing import _AnnotatedAlias, get_type_hints
+from typing import _AnnotatedAlias, get_type_hints
 
 if typing.TYPE_CHECKING:
     from experimaestro.scheduler.base import Job