experimaestro 1.6.1__py3-none-any.whl → 1.15.2__py3-none-any.whl

experimaestro/core/{objects.py → objects/config.py}

@@ -1,29 +1,24 @@
 """Configuration and tasks"""
 
-from functools import cached_property
 import json
 
-try:
-    from types import NoneType
-except Exception:
-    # compatibility: python-3.8
-    NoneType = type(None)
+from attr import define
+import fasteners
+
+from experimaestro import taskglobals
+
 from termcolor import cprint
-import os
 from pathlib import Path
-import hashlib
 import logging
-import struct
 import io
-import fasteners
 from enum import Enum
 import inspect
 import importlib
 from typing import (
     Any,
+    Callable,
     ClassVar,
     Dict,
-    Iterator,
     List,
     Optional,
     Set,
@@ -37,301 +32,30 @@ from typing import (
 import sys
 import experimaestro
 from experimaestro.utils import logger
-from contextlib import contextmanager
-from experimaestro.core.types import DeprecatedAttribute, ObjectType
-from .context import SerializationContext, SerializedPath, SerializedPathLoader
+from experimaestro.core.types import DeprecatedAttribute, ObjectType, TypeVarType
+from ..context import SerializationContext, SerializedPath, SerializedPathLoader
 
 if TYPE_CHECKING:
+    from ..callbacks import TaskEventListener
+    from ..identifier import Identifier
     from experimaestro.scheduler.base import Job
     from experimaestro.scheduler.workspace import RunMode
     from experimaestro.launchers import Launcher
     from experimaestro.scheduler import Workspace
 
-T = TypeVar("T", bound="Config")
-
-
-class Identifier:
-    def __init__(self, main: bytes):
-        self.main = main
-        self.has_loops = False
-
-    @cached_property
-    def all(self):
-        """Returns the overall identifier"""
-        return self.main
-
-    def __hash__(self) -> int:
-        return hash(self.main)
-
-    def state_dict(self):
-        return self.main.hex()
-
-    def __eq__(self, other: "Identifier"):
-        return self.main == other.main
-
-    @staticmethod
-    def from_state_dict(data: Union[Dict[str, str], str]):
-        if isinstance(data, str):
-            return Identifier(bytes.fromhex(data))
-
-        return Identifier(bytes.fromhex(data["main"]))
-
-    def __repr__(self):
-        return self.main.hex()
-
-
-def is_ignored(value):
-    """Returns True if the value should be ignored by itself"""
-    return value is not None and isinstance(value, Config) and (value.__xpm__.meta)
-
-
-def remove_meta(value):
-    """Cleanup a dict/list by removing ignored values"""
-    if isinstance(value, list):
-        return [el for el in value if not is_ignored(el)]
-    if isinstance(value, dict):
-        return {key: value for key, value in value.items() if not is_ignored(value)}
-    return value
-
-
-class ConfigPath:
-    """Used to keep track of cycles when computing a hash"""
-
-    def __init__(self):
-        self.loops: List[bool] = []
-        """Indicates whether a loop was detected up to this node"""
-
-        self.config2index = {}
-        """Associate an index in the list with a configuration"""
-
-    def detect_loop(self, config) -> Optional[int]:
-        """If there is a loop, return the relative index and update the path"""
-        index = self.config2index.get(id(config), None)
-        if index is not None:
-            for i in range(index, self.depth):
-                self.loops[i] = True
-            return self.depth - index
-
-    def has_loop(self):
-        return self.loops[-1]
-
-    @property
-    def depth(self):
-        return len(self.loops)
-
-    @contextmanager
-    def push(self, config):
-        config_id = id(config)
-        assert config_id not in self.config2index
-
-        self.config2index[config_id] = self.depth
-        self.loops.append(False)
-
-        try:
-            yield
-        finally:
-            self.loops.pop()
-            del self.config2index[config_id]
-
-
-hash_logger = logging.getLogger("xpm.hash")
-
-
-class HashComputer:
-    """This class is in charge of computing a config/task identifier"""
-
-    OBJECT_ID = b"\x00"
-    INT_ID = b"\x01"
-    FLOAT_ID = b"\x02"
-    STR_ID = b"\x03"
-    PATH_ID = b"\x04"
-    NAME_ID = b"\x05"
-    NONE_ID = b"\x06"
-    LIST_ID = b"\x07"
-    TASK_ID = b"\x08"
-    DICT_ID = b"\x09"
-    ENUM_ID = b"\x0a"
-    CYCLE_REFERENCE = b"\x0b"
-    INIT_TASKS = b"\x0c"
-
-    def __init__(self, config: "Config", config_path: ConfigPath, *, version=None):
-        # Hasher for parameters
-        self._hasher = hashlib.sha256()
-        self.config = config
-        self.config_path = config_path
-        self.version = version or int(os.environ.get("XPM_HASH_COMPUTER", 2))
-        if hash_logger.isEnabledFor(logging.DEBUG):
-            hash_logger.debug(
-                "starting hash (%s): %s", hash(str(self.config)), self.config
-            )
-
-    def identifier(self) -> Identifier:
-        main = self._hasher.digest()
-        if hash_logger.isEnabledFor(logging.DEBUG):
-            hash_logger.debug("hash (%s): %s", hash(str(self.config)), str(main))
-        return Identifier(main)
-
-    def _hashupdate(self, bytes: bytes):
-        """Update the hash computers with some bytes"""
-        if hash_logger.isEnabledFor(logging.DEBUG):
-            hash_logger.debug(
-                "updating hash (%s): %s", hash(str(self.config)), str(bytes)
-            )
-        self._hasher.update(bytes)
-
-    def update(self, value, *, myself=False):  # noqa: C901
-        """Update the hash
-
-        :param value: The value to add to the hash
-        :param myself: True if the value is the configuration for which we wish
-            to compute the identifier, defaults to False
-        :raises NotImplementedError: If the value cannot be processed
-        """
-        if value is None:
-            self._hashupdate(HashComputer.NONE_ID)
-        elif isinstance(value, float):
-            self._hashupdate(HashComputer.FLOAT_ID)
-            self._hashupdate(struct.pack("!d", value))
-        elif isinstance(value, int):
-            self._hashupdate(HashComputer.INT_ID)
-            self._hashupdate(struct.pack("!q", value))
-        elif isinstance(value, str):
-            self._hashupdate(HashComputer.STR_ID)
-            self._hashupdate(value.encode("utf-8"))
-        elif isinstance(value, list):
-            values = [el for el in value if not is_ignored(el)]
-            self._hashupdate(HashComputer.LIST_ID)
-            self._hashupdate(struct.pack("!d", len(values)))
-            for x in values:
-                self.update(x)
-        elif isinstance(value, Enum):
-            self._hashupdate(HashComputer.ENUM_ID)
-            k = value.__class__
-            self._hashupdate(
-                f"{k.__module__}.{k.__qualname__ }:{value.name}".encode("utf-8"),
-            )
-        elif isinstance(value, dict):
-            self._hashupdate(HashComputer.DICT_ID)
-            items = [
-                (key, value) for key, value in value.items() if not is_ignored(value)
-            ]
-            items.sort(key=lambda x: x[0])
-            for key, value in items:
-                self.update(key)
-                self.update(value)
-
-        # Handles configurations
-        elif isinstance(value, Config):
-            # Encodes the identifier
-            self._hashupdate(HashComputer.OBJECT_ID)
-
-            # If we encode another config, then
-            if not myself:
-                if loop_ix := self.config_path.detect_loop(value):
-                    # Loop detected: use cycle reference
-                    self._hashupdate(HashComputer.CYCLE_REFERENCE)
-                    self._hashupdate(struct.pack("!q", loop_ix))
-
-                else:
-                    # Just use the object identifier
-                    value_id = HashComputer.compute(
-                        value, version=self.version, config_path=self.config_path
-                    )
-                    self._hashupdate(value_id.all)
-
-                # And that's it!
-                return
-
-            # Process tasks
-            if value.__xpm__.task is not None and (value.__xpm__.task is not value):
-                hash_logger.debug("Computing hash for task %s", value.__xpm__.task)
-                self._hashupdate(HashComputer.TASK_ID)
-                self.update(value.__xpm__.task)
-
-            xpmtype = value.__xpmtype__
-            self._hashupdate(xpmtype.identifier.name.encode("utf-8"))
-
-            # Process arguments (sort by name to ensure uniqueness)
-            arguments = sorted(xpmtype.arguments.values(), key=lambda a: a.name)
-            for argument in arguments:
-                # Ignored argument
-                if argument.ignored:
-                    argvalue = value.__xpm__.values.get(argument.name, None)
-
-                    # ... unless meta is set to false
-                    if (
-                        argvalue is None
-                        or not isinstance(argvalue, Config)
-                        or (argvalue.__xpm__.meta is not False)
-                    ):
-                        continue
-
-                if argument.generator:
-                    continue
-
-                # Argument value
-                # Skip if the argument is not a constant, and
-                # - optional argument: both value and default are None
-                # - the argument value is equal to the default value
-                argvalue = getattr(value, argument.name, None)
-                if not argument.constant and (
-                    (
-                        not argument.required
-                        and argument.default is None
-                        and argvalue is None
-                    )
-                    or (
-                        argument.default is not None
-                        and argument.default == remove_meta(argvalue)
-                    )
-                ):
-                    # No update if same value (and not constant)
-                    continue
-
-                if (
-                    argvalue is not None
-                    and isinstance(argvalue, Config)
-                    and argvalue.__xpm__.meta
-                ):
-                    continue
-
-                # Hash name
-                self.update(argument.name)
-
-                # Hash value
-                self._hashupdate(HashComputer.NAME_ID)
-                self.update(argvalue)
-
-        else:
-            raise NotImplementedError("Cannot compute hash of type %s" % type(value))
-
-    @staticmethod
-    def compute(
-        config: "Config", config_path: ConfigPath = None, version=None
-    ) -> Identifier:
-        """Compute the identifier for a configuration
-
-        :param config: the configuration for which we compute the identifier
-        :param config_path: used to track down cycles between configurations
-        :param version: version for the hash computation (None for the last one)
-        """
-
-        # Try to use the cached value first
-        # (if there are no loops)
-        if config.__xpm__._sealed:
-            identifier = config.__xpm__._raw_identifier
-            if identifier is not None and not identifier.has_loops:
-                return identifier
+from .config_walk import ConfigWalk, ConfigWalkContext
+from .config_utils import (
+    getqualattr,
+    add_to_path,
+    TaggedValue,
+    ObjectStore,
+    classproperty,
+)
 
-        config_path = config_path or ConfigPath()
+T = TypeVar("T", bound="Config")
 
-        with config_path.push(config):
-            self = HashComputer(config, config_path, version=version)
-            self.update(config, myself=True)
-            identifier = self.identifier()
-            identifier.has_loop = config_path.has_loop()
 
-        return identifier
+DependentMarker = Callable[["Config"], None]
 
 
 def updatedependencies(
@@ -362,197 +86,25 @@ def updatedependencies(
         raise NotImplementedError("update dependencies for type %s" % type(value))
 
 
-class SealedError(Exception):
-    """Exception when trying to modify a sealed configuration"""
-
-    pass
-
-
-class TaggedValue:
-    def __init__(self, value):
-        self.value = value
-
-
-@contextmanager
-def add_to_path(p):
-    """Temporarily add a path to sys.path"""
-    import sys
-
-    old_path = sys.path
-    sys.path = sys.path[:]
-    sys.path.insert(0, p)
-    try:
-        yield
-    finally:
-        sys.path = old_path
-
-
-class ConfigWalkContext:
-    """Context when generating values in configurations"""
-
-    @property
-    def path(self):
-        """Returns the path of the job directory"""
-        raise NotImplementedError()
-
-    def __init__(self):
-        self._configpath = None
-
-    @property
-    def task(self):
-        return None
-
-    def currentpath(self) -> Path:
-        """Returns the configuration folder"""
-        if self._configpath:
-            return self.path / self._configpath
-        return self.path
-
-    @contextmanager
-    def push(self, key: str):
-        """Push a new key to contextualize paths"""
-        p = self._configpath
-        try:
-            self._configpath = (Path("out") if p is None else p) / key
-            yield key
-        finally:
-            self._configpath = p
-
-
 NOT_SET = object()
 
 
-class ConfigWalk:
-    """Allows to perform an operation on all nested configurations"""
-
-    def __init__(self, context: ConfigWalkContext = None, recurse_task=False):
-        """
-
-        :param recurse_task: Recurse into linked tasks
-        :param context: The context, by default only tracks the position in the
-            config tree
-        """
-        self.recurse_task = recurse_task
-        self.context = ConfigWalkContext() if context is None else context
-
-        # Stores already visited nodes
-        self.visited = {}
-
-    def preprocess(self, config: "Config") -> Tuple[bool, Any]:
-        """Returns a tuple boolean/value
-
-        The boolean value is used to stop the processing if False.
-        The value is returned
-        """
-        return True, None
-
-    def postprocess(self, stub, config: "Config", values: Dict[str, Any]):
-        return stub
-
-    def list(self, i: int):
-        return self.context.push(str(i))
-
-    def map(self, k: str):
-        return self.context.push(k)
-
-    def stub(self, config: "Config"):
-        return config
-
-    def __call__(self, x):
-        if isinstance(x, Config):
-            info = x.__xpm__  # type: ConfigInformation
-
-            # Avoid loops
-            xid = id(x)
-            if xid in self.visited:
-                return self.visited[xid]
-
-            # Get a stub
-            stub = self.stub(x)
-            self.visited[xid] = stub
-
-            # Pre-process
-            flag, value = self.preprocess(x)
-
-            if not flag:
-                # Stop processing and returns value
-                return value
-
-            # Process all the arguments
-            result = {}
-            for arg, v in info.xpmvalues():
-                if v is not None:
-                    with self.map(arg.name):
-                        result[arg.name] = self(v)
-                else:
-                    result[arg.name] = None
-
-            # Deals with pre-tasks
-            if info.pre_tasks:
-                with self.map("__pre_tasks__"):
-                    self(info.pre_tasks)
+@define()
+class WatchedOutput:
+    #: The enclosing job
+    job: "Job"
 
-            if info.init_tasks:
-                with self.map("__init_tasks__"):
-                    self(info.init_tasks)
+    #: The configuration containing the watched output
+    config: "ConfigInformation"
 
-            # Process task if different
-            if (
-                x.__xpm__.task is not None
-                and self.recurse_task
-                and x.__xpm__.task is not x
-            ):
-                self(x.__xpm__.task)
+    #: The watched output (name)
+    method_name: str
 
-            processed = self.postprocess(stub, x, result)
-            self.visited[xid] = processed
-            return processed
+    #: The watched output method (called with the JSON event)
+    method: Callable
 
-        if isinstance(x, list):
-            result = []
-            for i, sv in enumerate(x):
-                with self.list(i):
-                    result.append(self(sv))
-            return result
-
-        if isinstance(x, dict):
-            result = {}
-            for key, value in x.items():
-                assert isinstance(key, (str, float, int))
-                with self.map(key):
-                    result[key] = self(value)
-            return result
-
-        if isinstance(x, (float, int, str, Path, Enum)):
-            return x
-
-        raise NotImplementedError(f"Cannot handle a value of type {type(x)}")
-
-
-def getqualattr(module, qualname):
-    """Get a qualified attributed value"""
-    cls = module
-    for part in qualname.split("."):
-        cls = getattr(cls, part)
-    return cls
-
-
-class ObjectStore:
-    def __init__(self):
-        self.store: Dict[int, Any] = {}
-        self.constructed: Set[int] = set()
-
-    def set_constructed(self, identifier: int):
-        self.constructed.add(identifier)
-
-    def is_constructed(self, identifier: int):
-        return identifier in self.constructed
-
-    def retrieve(self, identifier: int):
-        return self.store.get(identifier, None)
-
-    def add_stub(self, identifier: int, stub: Any):
-        self.store[identifier] = stub
+    #: The callback to call (with the output of the previous method)
+    callback: Callable
 
 
 class ConfigInformation:
@@ -564,14 +116,14 @@ class ConfigInformation:
     # Set to true when loading from JSON
     LOADING: ClassVar[bool] = False
 
-    def __init__(self, pyobject: "TypeConfig"):
+    def __init__(self, pyobject: "ConfigMixin"):
         # The underlying pyobject and XPM type
         self.pyobject = pyobject
-        self.xpmtype = pyobject.__xpmtype__  # type: ObjectType
+        self.xpmtype: "ObjectType" = pyobject.__xpmtype__
         self.values = {}
 
         # Meta-informations
-        self._tags = {}
+        self._tags: dict[str, Any] = {}
         self._initinfo = ""
 
         self._taskoutput = None
@@ -582,31 +134,61 @@ class ConfigInformation:
 
         # State information
         self.job = None
+        self._job_listener: "TaskEventListener" | None = None
 
         #: True when this configuration was loaded from disk
         self.loaded = False
 
-        # Explicitely added dependencies
+        # Explicitly added dependencies
         self.dependencies = []
 
-        # Lightweight tasks
-        self.pre_tasks: List["LightweightTask"] = []
+        # Concrete type variables resolutions
+        # This is used to check typevars coherence
+        self.concrete_typevars: Dict[TypeVar, type] = {}
 
         # Initialization tasks
         self.init_tasks: List["LightweightTask"] = []
 
-        # Cached information
+        # Watched outputs
+        self.watched_outputs: List[WatchedOutput] = []
 
-        self._full_identifier = None
-        """The full identifier (with pre-tasks)"""
+        # Cached information
 
-        self._raw_identifier = None
-        """The identifier without taking into account pre-tasks"""
+        self._identifier = None
+        """The configuration identifier (cached when sealed)"""
 
         self._validated = False
         self._sealed = False
         self._meta = None
 
+        # This contains the list of generated values (using context) in this
+        # configuration or any sub-configuration, is generated. This prevents
+        # problem when a configuration with generated values is re-used.
+        self._generated_values = []
+
+    def get_generated_paths(
+        self, path: list[str] = None, paths: list[str] = None
+    ) -> list[str]:
+        """Get the list of generated paths, useful to track down those
+
+        :param path: The current path
+        :param paths: The list of generated paths so far, defaults to None
+        :return: The full list of generated paths
+        """
+        paths = [] if paths is None else paths
+        path = [] if path is None else path
+
+        for key in self._generated_values:
+            value = self.values[key]
+            if isinstance(value, ConfigMixin) and value.__xpm__._generated_values:
+                path.append(key)
+                value.__xpm__.get_generated_paths(path, paths)
+                path.pop()
+            else:
+                paths.append(".".join(path + [key]))
+
+        return paths
+
     def set_meta(self, value: Optional[bool]):
         """Sets the meta flag"""
         assert not self._sealed, "Configuration is sealed"
@@ -624,7 +206,34 @@ class ConfigInformation:
         # Not an argument, bypass
         return object.__getattribute__(self.pyobject, name)
 
+    @staticmethod
+    def is_generated_value(argument, value):
+        if argument.ignore_generated:
+            return False
+
+        if value is None:
+            return False
+
+        if isinstance(value, (int, str, float, bool, Path)):
+            return False
+
+        if isinstance(value, ConfigMixin):
+            return value.__xpm__._generated_values and value.__xpm__.task is None
+
+        if isinstance(value, list):
+            return any(ConfigInformation.is_generated_value(argument, x) for x in value)
+
+        if isinstance(value, dict):
+            return any(
+                ConfigInformation.is_generated_value(argument, x)
+                for x in value.values()
+            )
+
+        return False
+
     def set(self, k, v, bypass=False):
+        from experimaestro.generators import Generator
+
         # Not an argument, bypass
         if k not in self.xpmtype.arguments:
             setattr(self.pyobject, k, v)
@@ -633,13 +242,34 @@ class ConfigInformation:
         if self._sealed and not bypass:
             raise AttributeError(f"Object is read-only (trying to set {k})")
 
+        if not isinstance(v, ConfigMixin) and isinstance(v, Config):
+            raise AttributeError(
+                "Configuration (and not objects) should be used. Consider using .C(...)"
+            )
+
         try:
             argument = self.xpmtype.arguments.get(k, None)
             if argument:
-                if not bypass and (argument.generator or argument.constant):
+                if ConfigInformation.is_generated_value(argument, v):
+                    raise AttributeError(
+                        f"Cannot set {k} to a configuration with generated values. "
+                        "Here is the list of paths to help you: "
+                        f"""{', '.join(v.__xpm__.get_generated_paths([k]))}"""
+                    )
+
+                if not bypass and (
+                    (isinstance(argument.generator, Generator)) or argument.constant
+                ):
                     raise AttributeError("Property %s is read-only" % (k))
                 if v is not None:
                     self.values[k] = argument.validate(v)
+                    # Check for type variables
+                    if type(argument.type) is TypeVarType:
+                        self.check_typevar(argument.type.typevar, type(v))
+                    if isinstance(v, Config):
+                        # If the value is a Config, fuse type variables
+                        v.__xpm__.fuse_concrete_typevars(self.concrete_typevars)
+                        self.fuse_concrete_typevars(v.__xpm__.concrete_typevars)
                 elif argument.required:
                     raise AttributeError("Cannot set required attribute to None")
                 else:
@@ -652,6 +282,43 @@ class ConfigInformation:
             logger.error("Error while setting value %s in %s", k, self.xpmtype)
             raise
 
+    def fuse_concrete_typevars(self, typevars: Dict[TypeVar, type]):
+        """Fuses concrete type variables with the current ones"""
+        for typevar, v in typevars.items():
+            self.check_typevar(typevar, v)
+
+    def check_typevar(self, typevar: TypeVar, v: type):
+        """Check if a type variable is coherent with the current typevars bindings,
+        updates the bindings if necessary"""
+        if typevar not in self.concrete_typevars:
+            self.concrete_typevars[typevar] = v
+            return
+
+        concrete_typevar = self.concrete_typevars[typevar]
+        bound = typevar.__bound__
+        # Check that v is a subclass of the typevar OR that typevar is a subclass of v
+        # Then set the concrete type variable to the most generic type
+
+        # First, limiting to the specified bound
+        if bound is not None:
+            if not issubclass(v, bound):
+                raise TypeError(
+                    f"Type variable {typevar} is bound to {bound}, but tried to set it to {v}"
+                )
+
+        if issubclass(v, concrete_typevar):
+            # v is a subclass of the typevar, keep the typevar
+            return
+        if issubclass(concrete_typevar, v):
+            # typevar is a subclass of v, keep v
+            self.concrete_typevars[typevar] = v
+            return
+        raise TypeError(
+            f"Type variable {typevar} is already set to {self.concrete_typevars[typevar]}, "
+            f"but tried to set it to {v}"
+            f" (current typevars bindings: {self.concrete_typevars})"
+        )
+
     def addtag(self, name, value):
         self._tags[name] = value
 
@@ -691,10 +358,6 @@ class ConfigInformation:
                             % (k, self.xpmtype, self._initinfo)
                         )
 
-            # Validate pre-tasks
-            for pre_task in self.pre_tasks:
-                pre_task.__xpm__.validate()
-
             # Validate init tasks
             for init_task in self.init_tasks:
                 init_task.__xpm__.validate()
@@ -715,18 +378,62 @@ class ConfigInformation:
         Arguments:
             - context: the generation context
         """
+        if generated_keys := [
+            k
+            for k, v in self.values.items()
+            if ConfigInformation.is_generated_value(self.xpmtype.arguments[k], v)
+        ]:
+            raise AttributeError(
+                "Cannot seal a configuration with generated values:"
+                f"""{",".join(generated_keys)} in {context.currentpath}"""
+            )
 
         class Sealer(ConfigWalk):
-            def preprocess(self, config: Config):
+            def preprocess(self, config: ConfigMixin):
                 return not config.__xpm__._sealed, config
 
-            def postprocess(self, stub, config: Config, values):
+            def postprocess(self, stub, config: ConfigMixin, values):
                 # Generate values
+                from experimaestro.generators import Generator
+
                 for k, argument in config.__xpmtype__.arguments.items():
-                    if argument.generator:
-                        config.__xpm__.set(
-                            k, argument.generator(self.context, config), bypass=True
+                    try:
+                        if argument.generator:
+                            if not isinstance(argument.generator, Generator):
+                                # Don't set if already set
+                                if config.__xpm__.values.get(k) is not None:
+                                    continue
+                                value = argument.generator()
+                            else:
+                                # Generate a value
+                                sig = inspect.signature(argument.generator)
+                                if len(sig.parameters) == 0:
+                                    value = argument.generator()
+                                elif len(sig.parameters) == 2:
+                                    # Only in that case do we need to flag this configuration
+                                    # as containing generated values
+                                    config.__xpm__._generated_values.append(k)
+                                    value = argument.generator(self.context, config)
+                                else:
+                                    assert (
+                                        False
+                                    ), "generator has either two parameters (context and config), or none"
+                            config.__xpm__.set(k, value, bypass=True)
+                        else:
+                            value = config.__xpm__.values.get(k)
+                    except Exception:
+                        logger.error(
+                            "While setting %s of %s", argument.name, config.__xpmtype__
                         )
+                        raise
+
+                    # Propagate the generated value flag
+                    if (
+                        (value is not None)
+                        and isinstance(value, ConfigMixin)
+                        and value.__xpm__._generated_values
+                    ):
+                        config.__xpm__._generated_values.append(k)
 
                 config.__xpm__._sealed = True
 
@@ -740,89 +447,29 @@ class ConfigInformation:
         context = ConfigWalkContext()
 
         class Unsealer(ConfigWalk):
-            def preprocess(self, config: Config):
+            def preprocess(self, config: ConfigMixin):
                 return config.__xpm__._sealed, config
 
-            def postprocess(self, stub, config: Config, values):
+            def postprocess(self, stub, config: ConfigMixin, values):
                 config.__xpm__._sealed = False
                 config.__xpm__._identifier = None
 
         Unsealer(context, recurse_task=True)(self.pyobject)
 
-    def collect_pre_tasks(self) -> Iterator["Config"]:
-        context = ConfigWalkContext()
-        pre_tasks: Dict[int, "Config"] = {}
-
-        class PreTaskCollect(ConfigWalk):
-            def preprocess(self, config: Config):
-                # Do not cross tasks
-                return not isinstance(config.__xpm__, Task), config
-
-            def postprocess(self, stub, config: Config, values):
-                pre_tasks.update(
-                    {id(pre_task): pre_task for pre_task in config.__xpm__.pre_tasks}
-                )
-
-        PreTaskCollect(context, recurse_task=True)(self.pyobject)
-        return pre_tasks.values()
-
-    def identifiers(self, only_raw: bool):
+    @property
+    def identifier(self):
         """Computes the unique identifier"""
-
-        raw_identifier = self._raw_identifier
-        full_identifier = self._full_identifier
+        from ..identifier import IdentifierComputer
 
         # Computes raw identifier if needed
-        if raw_identifier is None or not self._sealed:
-            # Get the main identifier
-            raw_identifier = HashComputer.compute(self.pyobject)
-            if self._sealed:
-                self._raw_identifier = raw_identifier
-
-        if only_raw:
-            return raw_identifier, full_identifier
-
-        # OK, let's compute the full identifier
-        if full_identifier is None or not self._sealed:
-            # Compute the full identifier by including the pre-tasks
-            hasher = hashlib.sha256()
-            hasher.update(raw_identifier.all)
-            pre_tasks_ids = [
-                pre_task.__xpm__.raw_identifier.all
-                for pre_task in self.collect_pre_tasks()
-            ]
-            for task_id in sorted(pre_tasks_ids):
-                hasher.update(task_id)
-
-            # Adds init tasks
-            if self.init_tasks:
-                hasher.update(HashComputer.INIT_TASKS)
-                for init_task in self.init_tasks:
-                    hasher.update(init_task.__xpm__.raw_identifier.all)
-
-            full_identifier = Identifier(hasher.digest())
-            full_identifier.has_loops = raw_identifier.has_loops
-
-            # Only cache the identifier if sealed
-            if self._sealed:
-                self._full_identifier = full_identifier
-
-        return raw_identifier, full_identifier
-
-    @property
-    def raw_identifier(self) -> Identifier:
-        """Computes the unique identifier (without task modifiers)"""
-        raw_identifier, _ = self.identifiers(True)
-        return raw_identifier
-
-    @property
-    def full_identifier(self) -> Identifier:
-        """Computes the unique identifier (with task modifiers)"""
-        _, full_identifier = self.identifiers(False)
-        return full_identifier
+        if self._identifier is not None:
+            return self._identifier
 
-    identifier = full_identifier
-    """Deprecated: use full_identifier"""
+        # Get the main identifier
+        identifier = IdentifierComputer.compute(self.pyobject)
+        if self._sealed:
+            self._identifier = identifier
+        return identifier
 
     def dependency(self):
         """Returns a dependency"""
@@ -837,12 +484,6 @@ class ConfigInformation:
         path: List[str],
         taskids: Set[int],
     ):
-        # Add pre-tasks
-        for pre_task in self.pre_tasks:
-            pre_task.__xpm__.updatedependencies(
-                dependencies, path + ["__pre_tasks__"], taskids
-            )
-
         # Add initialization tasks
         for init_task in self.init_tasks:
             init_task.__xpm__.updatedependencies(
@@ -899,6 +540,28 @@ class ConfigInformation:
         # Now, seal the object
         self.seal(context)
 
+    def watch_output(self, method, callback):
+        """Watch the task output linked with a given method
+
+        :param method: The method to watch
+        :param callback: The callback
+        """
+        watched = WatchedOutput(
+            self, method.__self__, method.__name__, method, callback
+        )
+        self.watched_outputs.append(watched)
+        if self.job:
+            self.job.watch_output(watched)
+
+    def on_completed(self, callback: Callable[[], None]):
+        """Call a method when the task is completed successfully
+
+        :param callback: _description_
+        """
+        from ..callbacks import TaskEventListener
+
+        TaskEventListener.on_completed(self, callback)
+
     def submit(
         self,
         workspace: "Workspace",
@@ -909,6 +572,7 @@ class ConfigInformation:
     ):
         from experimaestro.scheduler import experiment, JobContext
         from experimaestro.scheduler.workspace import RunMode
+        from ..callbacks import TaskEventListener
 
         # --- Prepare the object
 
@@ -961,10 +625,12 @@ class ConfigInformation:
             workspace.run_mode if run_mode is None else run_mode
         ) or RunMode.NORMAL
         if run_mode == RunMode.NORMAL:
+            TaskEventListener.connect(experiment.CURRENT)
+            experiment.CURRENT.submit(self.job)
             other = experiment.CURRENT.submit(self.job)
             if other:
-                # Just returns the other task
-                return other.config.__xpm__._taskoutput
+                # Our job = previously submitted job
+                self.job = other
         else:
             # Show a warning
             if run_mode == RunMode.GENERATE_ONLY:
@@ -1000,23 +666,22 @@ class ConfigInformation:
 
                 print(file=sys.stderr)  # noqa: T201
 
-        # Handle an output configuration
-        def mark_output(config: "Config"):
-            """Sets a dependency on the job"""
-            assert not isinstance(config, Task), "Cannot set a dependency on a task"
-            config.__xpm__.task = self.pyobject
-            return config
-
         # Mark this configuration also
         self.task = self.pyobject
 
         if hasattr(self.pyobject, "task_outputs"):
-            self._taskoutput = self.pyobject.task_outputs(mark_output)
+            self._taskoutput = self.pyobject.task_outputs(self.mark_output)
         else:
             self._taskoutput = self.task = self.pyobject
 
         return self._taskoutput
 
+    def mark_output(self, config: "Config"):
+        """Sets a dependency on the job"""
+        assert not isinstance(config, Task), "Cannot set a dependency on a task"
+        config.__xpm__.task = self.pyobject
+        return config
+
     # --- Serialization
 
     @staticmethod
@@ -1025,7 +690,7 @@ class ConfigInformation:
         if value is None:
             return None
 
-        elif isinstance(value, list):
+        elif isinstance(value, (list, tuple)):
             return [ConfigInformation._outputjsonvalue(el, context) for el in value]
 
         elif isinstance(value, dict):
@@ -1089,9 +754,6 @@ class ConfigInformation:
         if self.task is not None and self.task is not self:
             ConfigInformation.__collect_objects__(self.task, objects, context)
 
-        # Serialize pre-tasks
-        ConfigInformation.__collect_objects__(self.pre_tasks, objects, context)
-
         # Serialize initialization tasks
         ConfigInformation.__collect_objects__(self.init_tasks, objects, context)
 
@@ -1105,8 +767,6 @@ class ConfigInformation:
         }
 
         # Add pre/init tasks
-        if self.pre_tasks:
-            state_dict["pre-tasks"] = [id(pre_task) for pre_task in self.pre_tasks]
         if self.init_tasks:
             state_dict["init-tasks"] = [id(init_task) for init_task in self.init_tasks]
 
@@ -1140,9 +800,12 @@ class ConfigInformation:
     def __collect_objects__(value, objects: List[Dict], context: SerializationContext):
         """Serialize all needed configuration objects, looking at sub
         configurations if necessary"""
+        if value is None:
+            return
+
         if isinstance(value, Config):
             value.__xpm__.__get_objects__(objects, context)
-        elif isinstance(value, list):
+        elif isinstance(value, (list, tuple)):
             for el in value:
                 ConfigInformation.__collect_objects__(el, objects, context)
         elif isinstance(value, dict):
@@ -1283,34 +946,31 @@ class ConfigInformation:
 
     @overload
     @staticmethod
-    def fromParameters(
+    def fromParameters(  # noqa: E704
         definitions: List[Dict],
         as_instance=True,
         save_directory: Optional[Path] = None,
         discard_id: bool = False,
-    ) -> "TypeConfig":
-        ...
+    ) -> "ConfigMixin": ...
 
     @overload
     @staticmethod
-    def fromParameters(
+    def fromParameters(  # noqa: E704
         definitions: List[Dict],
         as_instance=False,
         return_tasks=True,
         save_directory: Optional[Path] = None,
         discard_id: bool = False,
-    ) -> Tuple["Config", List["LightweightTask"]]:
-        ...
+    ) -> Tuple["Config", List["LightweightTask"]]: ...
 
     @overload
     @staticmethod
-    def fromParameters(
+    def fromParameters(  # noqa: E704
         definitions: List[Dict],
         as_instance=False,
         save_directory: Optional[Path] = None,
         discard_id: bool = False,
-    ) -> "Config":
-        ...
+    ) -> "Config": ...
 
     @staticmethod
     def load_objects(  # noqa: C901
@@ -1323,6 +983,7 @@ class ConfigInformation:
         o = None
         objects = {}
         import experimaestro.taskglobals as taskglobals
+        from ..identifier import Identifier
 
         # Loop over all the definitions and create objects
         for definition in definitions:
@@ -1345,7 +1006,9 @@ class ConfigInformation:
                     mod = importlib.import_module(module_name)
                 except ModuleNotFoundError:
                     # More hints on the nature of the error
-                    logging.warning("(1) Either the python path is wrong – %s", ":".join(sys.path))
+                    logging.warning(
+                        "(1) Either the python path is wrong – %s", ":".join(sys.path)
+                    )
                     logging.warning("(2) There is not __init__.py in your module")
                     raise
 
@@ -1432,12 +1095,6 @@ class ConfigInformation:
                 o.__post_init__()
 
             else:
-                # Sets pre-tasks
-                o.__xpm__.pre_tasks = [
-                    objects[pre_task_id]
-                    for pre_task_id in definition.get("pre-tasks", [])
-                ]
-
                 if task_id := definition.get("task", None):
                     o.__xpm__.task = objects[task_id]
 
@@ -1471,15 +1128,6 @@ class ConfigInformation:
 
         # Run pre-task (or returns them)
         if as_instance or return_tasks:
-            # Collect pre-tasks (just once)
-            completed_pretasks = set()
-            pre_tasks = []
-            for definition in definitions:
-                for pre_task_id in definition.get("pre-tasks", []):
-                    if pre_task_id not in completed_pretasks:
-                        completed_pretasks.add(pre_task_id)
-                        pre_tasks.append(objects[pre_task_id])
-
             # Collect init tasks
             init_tasks = []
             for init_task_id in definitions[-1].get("init-tasks", []):
@@ -1487,14 +1135,11 @@ class ConfigInformation:
                 init_tasks.append(init_task)
 
             if as_instance:
-                for pre_task in pre_tasks:
-                    logger.info("Executing pre-task %s", type(pre_task))
-                    pre_task.execute()
                 for init_task in init_tasks:
                     logger.info("Executing init task %s", type(init_task))
                     init_task.execute()
             else:
-                return o, pre_tasks, pre_task + init_tasks
+                return o, init_tasks
 
         return o
 
@@ -1502,7 +1147,6 @@ class ConfigInformation:
         def __init__(self, context: ConfigWalkContext, *, objects: ObjectStore = None):
             super().__init__(context)
             self.objects = ObjectStore() if objects is None else objects
-            self.pre_tasks = {}
 
         def preprocess(self, config: "Config"):
             if self.objects.is_constructed(id(config)):
@@ -1529,10 +1173,6 @@ class ConfigInformation:
             # Call __post_init__
             stub.__post_init__()
 
-            # Gather pre-tasks
-            for pre_task in config.__xpm__.pre_tasks:
-                self.pre_tasks[id(pre_task)] = self.stub(pre_task)
-
             self.objects.set_constructed(id(config))
             return stub
 
@@ -1546,10 +1186,6 @@ class ConfigInformation:
         processor = ConfigInformation.FromPython(context, objects=objects)
         last_object = processor(self.pyobject)
 
-        # Execute pre-tasks
-        for pre_task in processor.pre_tasks.values():
-            pre_task.execute()
-
         return last_object
 
     def add_dependencies(self, *dependencies):
@@ -1573,6 +1209,9 @@ def clone(v):
     if isinstance(v, Enum):
         return v
 
+    if isinstance(v, tuple):
+        return tuple(clone(x) for x in v)
+
     if isinstance(v, Config):
         # Create a new instance
         kwargs = {
@@ -1587,31 +1226,15 @@ def clone(v):
     raise NotImplementedError("Clone not implemented for type %s" % type(v))
 
 
-def cache(fn, name: str):
-    def __call__(config, *args, **kwargs):
-        import experimaestro.taskglobals as taskglobals
-
-        # Get path and create directory if needed
-        hexid = config.__xpmidentifier__  # type: Identifier
-        typename = config.__xpmtypename__  # type: str
-        dir = taskglobals.Env.instance().wspath / "config" / typename / hexid.all.hex()
-
-        if not dir.exists():
-            dir.mkdir(parents=True, exist_ok=True)
-
-        path = dir / name
-        ipc_lock = fasteners.InterProcessLock(path.with_suffix(path.suffix + ".lock"))
-        with ipc_lock:
-            r = fn(config, path, *args, **kwargs)
-            return r
-
-    return __call__
-
-
-class TypeConfig:
+class ConfigMixin:
     """Class for configuration objects"""
 
     __xpmtype__: ObjectType
+    """The associated XPM type"""
+
+    __xpm__: ConfigInformation
+    """The __xpm__ object contains all instance specific information about a
+    configuration/task"""
 
     def __init__(self, **kwargs):
         """Initialize the configuration with the given parameters"""
@@ -1748,29 +1371,7 @@ class TypeConfig:
         attributes)"""
         return clone(self)
 
-    def add_pretasks(self, *tasks: "LightweightTask"):
-        assert all(
-            [isinstance(task, LightweightTask) for task in tasks]
-        ), "One of the pre-tasks are not lightweight tasks"
-        if self.__xpm__._sealed:
-            raise SealedError("Cannot add pre-tasks to a sealed configuration")
-        self.__xpm__.pre_tasks.extend(tasks)
-        return self
-
-    def add_pretasks_from(self, *configs: "Config"):
-        assert all(
-            [isinstance(config, TypeConfig) for config in configs]
-        ), "One of the parameters is not a configuration object"
-        for config in configs:
-            self.add_pretasks(*config.__xpm__.pre_tasks)
-        return self
-
-    @property
-    def pre_tasks(self) -> List["LightweightTask"]:
-        """Access pre-tasks"""
-        return self.__xpm__.pre_tasks
-
-    def copy_dependencies(self, other: "Config"):
+    def copy_dependencies(self, other: "ConfigMixin"):
         """Add all the dependencies from other configuration"""
 
         # Add task dependency
@@ -1782,42 +1383,33 @@ class TypeConfig:
         self.__xpm__.add_dependencies(*other.__xpm__.dependencies)
 
 
-class classproperty(property):
-    def __get__(self, owner_self, owner_cls):
-        return self.fget(owner_cls)
-
-
 class Config:
     """Base type for all objects in python interface"""
 
+    __xpmid__: ClassVar[Optional[str]]
+    """Optional configuration ID, mostly useful when moving a class to another
+    package to avoid changes in computed task identifiers"""
+
     __xpmtype__: ClassVar[ObjectType]
     """The object type holds all the information about a specific subclass
     experimaestro metadata"""
 
-    __xpm__: ConfigInformation
-    """The __xpm__ object contains all instance specific information about a
-    configuration/task"""
-
     @classproperty
     def XPMConfig(cls):
-        if issubclass(cls, TypeConfig):
+        if issubclass(cls, ConfigMixin):
             return cls
         return cls.__getxpmtype__().configtype
 
-    @classproperty
-    def C(cls):
-        return cls.XPMConfig
-
     @classproperty
     def XPMValue(cls):
         """Returns the value object for this configuration"""
-        if issubclass(cls, TypeConfig):
+        if issubclass(cls, ConfigMixin):
             return cls.__xpmtype__.objecttype
 
         if value_cls := cls.__dict__.get("__XPMValue__", None):
             pass
         else:
-            from .types import XPMValue
+            from ..types import XPMValue
 
             __objectbases__ = tuple(
                 s.XPMValue
@@ -1834,9 +1426,20 @@ class Config:
 
         return value_cls
 
+    @classproperty
+    def C(cls):
+        """Alias for XPMConfig"""
+        return cls.XPMConfig
+
+    @classproperty
+    def V(cls):
+        """Alias for XPMValue"""
+        return cls.XPMValue
+
     @classmethod
     def __getxpmtype__(cls) -> "ObjectType":
-        """Get (and create if necessary) the Object type of this"""
+        """Get (and create if necessary) the Object type associated
+        with thie Config object"""
         xpmtype = cls.__dict__.get("__xpmtype__", None)
         if xpmtype is None:
             from experimaestro.core.types import ObjectType
@@ -1850,9 +1453,12 @@ class Config:
         return xpmtype
 
     def __new__(cls: Type[T], *args, **kwargs) -> T:
-        """Returns an instance of a TypeConfig (for compatibility, use XPMConfig
-        or C if possible)"""
+        """Returns an instance of a ConfigMixin (for compatibility, use XPMConfig
+        or C if possible)
 
+        :deprecated: Use Config.C or Config.XPMConfig to construct a new
+            configuration, and Config.V (or Config.XPMValue) for a new value
+        """
         # If this is an XPMValue, just return a new instance
         from experimaestro.core.types import XPMValue
 
@@ -1861,15 +1467,24 @@ class Config:
 
         # If this is the XPMConfig, just return a new instance
         # __init__ will be called
-        if issubclass(cls, TypeConfig):
+        if issubclass(cls, ConfigMixin):
             return object.__new__(cls)
 
+        # Log a deprecation warning for this way of creating a configuration
+        caller = inspect.getframeinfo(inspect.stack()[1][0])
+        logger.warning(
+            "Creating a configuration using Config.__new__ is deprecated, and will be removed in a future version. "
+            "Use Config.C or Config.XPMConfig to create a new configuration. "
+            "Issue created at %s:%s",
+            str(Path(caller.filename).absolute()),
+            caller.lineno,
+        )
+
         # otherwise, we use the configuration type
-        o: TypeConfig = object.__new__(cls.__getxpmtype__().configtype)
+        o: ConfigMixin = object.__new__(cls.__getxpmtype__().configtype)
         try:
             o.__init__(*args, **kwargs)
         except Exception:
-            caller = inspect.getframeinfo(inspect.stack()[1][0])
             logger.error(
                 "Init error in %s:%s"
                 % (str(Path(caller.filename).absolute()), caller.lineno)
@@ -1890,29 +1505,30 @@ class Config:
         """Returns a JSON version of the object (if possible)"""
         return self.__xpm__.__json__()
 
-    def __identifier__(self) -> Identifier:
+    def __identifier__(self) -> "Identifier":
         return self.__xpm__.identifier
 
-    def add_pretasks(self, *tasks: "LightweightTask"):
-        """Add pre-tasks"""
-        raise AssertionError("This method can only be used during configuration")
-
-    def add_pretasks_from(self, *configs: "Config"):
-        """Add pre-tasks from the listed configurations"""
-        raise AssertionError(
-            "The 'add_pretasks_from' can only be used during configuration"
-        )
-
     def copy_dependencies(self, other: "Config"):
         """Add pre-tasks from the listed configurations"""
         raise AssertionError(
             "The 'copy_dependencies' method can only be used during configuration"
         )
 
-    @property
-    def pre_tasks(self) -> List["LightweightTask"]:
-        """Access pre-tasks"""
-        raise AssertionError("Pre-tasks can be accessed only during configuration")
+    def register_task_output(self, method, *args, **kwargs):
+        # Determine the path for this...
+        path = taskglobals.Env.instance().xpm_path / "task-outputs.jsonl"
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        data = json.dumps(
+            {
+                "key": f"{self.__xpmidentifier__}/{method.__name__}",
+                "args": args,
+                "kwargs": kwargs,
+            }
+        )
+        with path.open("at") as fp:
+            fp.writelines([data, "\n"])
+            fp.flush()
 
 
 class LightweightTask(Config):
@@ -1931,6 +1547,17 @@ class Task(LightweightTask):
     def submit(self):
         raise AssertionError("This method can only be used during configuration")
 
+    def watch_output(self, method, callback):
+        """Sets up a callback
+
+        :param method: a method within a configuration
+        :param callback: the callback
+        """
+        self.__xpm__.watch_output(method, callback)
+
+    def on_completed(self, callback: Callable[[], None]):
+        self.__xpm__.on_completed(callback)
+
 
 # --- Utility functions
 
@@ -1969,3 +1596,24 @@ def setmeta(config: Config, flag: bool):
     """Flags the configuration as a meta-parameter"""
     config.__xpm__.set_meta(flag)
     return config
+
+
+def cache(fn, name: str):
+    def __call__(config, *args, **kwargs):
+        import experimaestro.taskglobals as taskglobals
+
+        # Get path and create directory if needed
+        hexid = config.__xpmidentifier__  # type: Identifier
+        typename = config.__xpmtypename__  # type: str
+        dir = taskglobals.Env.instance().wspath / "config" / typename / hexid.all.hex()
+
+        if not dir.exists():
+            dir.mkdir(parents=True, exist_ok=True)
+
+        path = dir / name
+        ipc_lock = fasteners.InterProcessLock(path.with_suffix(path.suffix + ".lock"))
+        with ipc_lock:
+            r = fn(config, path, *args, **kwargs)
+            return r
+
+    return __call__