experimaestro 1.13.0__tar.gz → 1.15.0__tar.gz

{experimaestro-1.13.0 → experimaestro-1.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: experimaestro
-Version: 1.13.0
+Version: 1.15.0
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 License-File: LICENSE

{experimaestro-1.13.0 → experimaestro-1.15.0}/pyproject.toml

@@ -48,7 +48,7 @@ dependencies = [
     "typing-extensions >=4.2; python_version < \"3.12\"",
     "watchdog >=2"
 ]
-version = "1.13.0"
+version = "1.15.0"
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -140,7 +140,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.13.0"
+version = "1.15.0"
 changelog_start_rev = "v1.0.0"
 tag_format = "v$major.$minor.$patch$prerelease"
 # update_changelog_on_bump = true

{experimaestro-1.13.0 → experimaestro-1.15.0}/src/experimaestro/core/arguments.py

@@ -80,10 +80,12 @@ class Argument:
 
         self.generator = generator
         self.default = None
+        self.ignore_generated = False
 
         if default is not None:
             assert self.generator is None, "generator and default are exclusive options"
             if isinstance(default, field):
+                self.ignore_generated = default.ignore_generated
                 if default.default is not None:
                     self.default = default.default
                 elif default.default_factory is not None:
@@ -184,13 +186,20 @@ DataPath = Annotated[Path, dataHint]
 class field:
     """Extra information for a given experimaestro field (param or meta)"""
 
-    def __init__(self, *, default: Any = None, default_factory: Callable = None):
+    def __init__(
+        self,
+        *,
+        default: Any = None,
+        default_factory: Callable = None,
+        ignore_generated=False,
+    ):
         assert not (
             (default is not None) and (default_factory is not None)
         ), "default and default_factory are mutually exclusive options"
 
         self.default_factory = default_factory
         self.default = default
+        self.ignore_generated = ignore_generated
 
 
 class help(TypeAnnotation):

{experimaestro-1.13.0 → experimaestro-1.15.0}/src/experimaestro/core/identifier.py

@@ -6,7 +6,7 @@ import logging
 import os
 import struct
 from typing import Optional
-from experimaestro.core.objects import Config
+from experimaestro.core.objects import Config, ConfigMixin
 
 
 class ConfigPath:
@@ -116,7 +116,7 @@ class IdentifierComputer:
     CYCLE_REFERENCE = b"\x0b"
     INIT_TASKS = b"\x0c"
 
-    def __init__(self, config: "Config", config_path: ConfigPath, *, version=None):
+    def __init__(self, config: "ConfigMixin", config_path: ConfigPath, *, version=None):
         # Hasher for parameters
         self._hasher = hashlib.sha256()
         self.config = config
@@ -170,7 +170,7 @@ class IdentifierComputer:
             self._hashupdate(IdentifierComputer.ENUM_ID)
             k = value.__class__
             self._hashupdate(
-                f"{k.__module__}.{k.__qualname__ }:{value.name}".encode("utf-8"),
+                f"{k.__module__}.{k.__qualname__}:{value.name}".encode("utf-8"),
             )
         elif isinstance(value, dict):
             self._hashupdate(IdentifierComputer.DICT_ID)
@@ -183,7 +183,7 @@ class IdentifierComputer:
                 self.update(value)
 
         # Handles configurations
-        elif isinstance(value, Config):
+        elif isinstance(value, ConfigMixin):
             # Encodes the identifier
             self._hashupdate(IdentifierComputer.OBJECT_ID)
 
@@ -264,12 +264,17 @@ class IdentifierComputer:
                 self._hashupdate(IdentifierComputer.NAME_ID)
                 self.update(argvalue)
 
+            # Add init tasks
+            if value.__xpm__.init_tasks:
+                self._hashupdate(IdentifierComputer.INIT_TASKS)
+                for init_task in value.__xpm__.init_tasks:
+                    self.update(init_task)
         else:
             raise NotImplementedError("Cannot compute hash of type %s" % type(value))
 
     @staticmethod
     def compute(
-        config: "Config", config_path: ConfigPath | None = None, version=None
+        config: "ConfigMixin", config_path: ConfigPath | None = None, version=None
     ) -> Identifier:
         """Compute the identifier for a configuration
 
@@ -281,7 +286,7 @@ class IdentifierComputer:
         # Try to use the cached value first
         # (if there are no loops)
         if config.__xpm__._sealed:
-            identifier = config.__xpm__._raw_identifier
+            identifier = config.__xpm__._identifier
             if identifier is not None and not identifier.has_loops:
                 return identifier
 

{experimaestro-1.13.0 → experimaestro-1.15.0}/src/experimaestro/core/objects/config.py

@@ -9,7 +9,6 @@ from experimaestro import taskglobals
 
 from termcolor import cprint
 from pathlib import Path
-import hashlib
 import logging
 import io
 from enum import Enum
@@ -20,7 +19,6 @@ from typing import (
     Callable,
     ClassVar,
     Dict,
-    Iterator,
     List,
     Optional,
     Set,
@@ -49,7 +47,6 @@ from .config_walk import ConfigWalk, ConfigWalkContext
 from .config_utils import (
     getqualattr,
     add_to_path,
-    SealedError,
     TaggedValue,
     ObjectStore,
     classproperty,
@@ -149,9 +146,6 @@ class ConfigInformation:
         # This is used to check typevars coherence
         self.concrete_typevars: Dict[TypeVar, type] = {}
 
-        # Lightweight tasks
-        self.pre_tasks: List["LightweightTask"] = []
-
         # Initialization tasks
         self.init_tasks: List["LightweightTask"] = []
 
@@ -160,20 +154,40 @@ class ConfigInformation:
 
         # Cached information
 
-        self._full_identifier = None
-        """The full identifier (with pre-tasks)"""
-
-        self._raw_identifier = None
-        """The identifier without taking into account pre-tasks"""
+        self._identifier = None
+        """The configuration identifier (cached when sealed)"""
 
         self._validated = False
         self._sealed = False
         self._meta = None
 
-        #: This flags is True when a value in this configuration,
-        #: or any sub-configuration, is generated. This prevents problem
-        #: when a configuration with generated values is re-used
-        self._has_generated_value = False
+        # This contains the list of generated values (using context) in this
+        # configuration or any sub-configuration, is generated. This prevents
+        # problem when a configuration with generated values is re-used.
+        self._generated_values = []
+
+    def get_generated_paths(
+        self, path: list[str] = None, paths: list[str] = None
+    ) -> list[str]:
+        """Get the list of generated paths, useful to track down those
+
+        :param path: The current path
+        :param paths: The list of generated paths so far, defaults to None
+        :return: The full list of generated paths
+        """
+        paths = [] if paths is None else paths
+        path = [] if path is None else path
+
+        for key in self._generated_values:
+            value = self.values[key]
+            if isinstance(value, ConfigMixin) and value.__xpm__._generated_values:
+                path.append(key)
+                value.__xpm__.get_generated_paths(path, paths)
+                path.pop()
+            else:
+                paths.append(".".join(path + [key]))
+
+        return paths
 
     def set_meta(self, value: Optional[bool]):
         """Sets the meta flag"""
@@ -192,6 +206,31 @@ class ConfigInformation:
         # Not an argument, bypass
         return object.__getattribute__(self.pyobject, name)
 
+    @staticmethod
+    def is_generated_value(argument, value):
+        if argument.ignore_generated:
+            return False
+
+        if value is None:
+            return False
+
+        if isinstance(value, (int, str, float, bool, Path)):
+            return False
+
+        if isinstance(value, ConfigMixin):
+            return value.__xpm__._generated_values and value.__xpm__.task is None
+
+        if isinstance(value, list):
+            return any(ConfigInformation.is_generated_value(argument, x) for x in value)
+
+        if isinstance(value, dict):
+            return any(
+                ConfigInformation.is_generated_value(argument, x)
+                for x in value.values()
+            )
+
+        return False
+
     def set(self, k, v, bypass=False):
         from experimaestro.generators import Generator
 
@@ -208,18 +247,16 @@ class ConfigInformation:
                 "Configuration (and not objects) should be used. Consider using .C(...)"
             )
 
-        if (
-            isinstance(v, ConfigMixin)
-            and v.__xpm__._has_generated_value
-            and v.__xpm__.task is None
-        ):
-            raise AttributeError(
-                f"Cannot set {k} to a configuration with generated values"
-            )
-
         try:
             argument = self.xpmtype.arguments.get(k, None)
             if argument:
+                if ConfigInformation.is_generated_value(argument, v):
+                    raise AttributeError(
+                        f"Cannot set {k} to a configuration with generated values. "
+                        "Here is the list of paths to help you: "
+                        f"""{', '.join(v.__xpm__.get_generated_paths([k]))}"""
+                    )
+
                 if not bypass and (
                     (isinstance(argument.generator, Generator)) or argument.constant
                 ):
@@ -321,10 +358,6 @@ class ConfigInformation:
                             % (k, self.xpmtype, self._initinfo)
                         )
 
-            # Validate pre-tasks
-            for pre_task in self.pre_tasks:
-                pre_task.__xpm__.validate()
-
             # Validate init tasks
             for init_task in self.init_tasks:
                 init_task.__xpm__.validate()
@@ -345,6 +378,15 @@ class ConfigInformation:
         Arguments:
             - context: the generation context
         """
+        if generated_keys := [
+            k
+            for k, v in self.values.items()
+            if ConfigInformation.is_generated_value(self.xpmtype.arguments[k], v)
+        ]:
+            raise AttributeError(
+                "Cannot seal a configuration with generated values:"
+                f"""{",".join(generated_keys)} in {context.currentpath}"""
+            )
 
         class Sealer(ConfigWalk):
             def preprocess(self, config: ConfigMixin):
@@ -368,13 +410,15 @@ class ConfigInformation:
                                 if len(sig.parameters) == 0:
                                     value = argument.generator()
                                 elif len(sig.parameters) == 2:
+                                    # Only in that case do we need to flag this configuration
+                                    # as containing generated values
+                                    config.__xpm__._generated_values.append(k)
                                     value = argument.generator(self.context, config)
                                 else:
                                     assert (
                                         False
                                     ), "generator has either two parameters (context and config), or none"
                             config.__xpm__.set(k, value, bypass=True)
-                            config.__xpm__._has_generated_value = True
                         else:
                             value = config.__xpm__.values.get(k)
                     except Exception:
@@ -387,9 +431,9 @@ class ConfigInformation:
                     if (
                         (value is not None)
                         and isinstance(value, ConfigMixin)
-                        and value.__xpm__._has_generated_value
+                        and value.__xpm__._generated_values
                     ):
-                        self._has_generated_value = True
+                        config.__xpm__._generated_values.append(k)
 
                 config.__xpm__._sealed = True
 
@@ -403,90 +447,29 @@ class ConfigInformation:
         context = ConfigWalkContext()
 
         class Unsealer(ConfigWalk):
-            def preprocess(self, config: Config):
+            def preprocess(self, config: ConfigMixin):
                 return config.__xpm__._sealed, config
 
-            def postprocess(self, stub, config: Config, values):
+            def postprocess(self, stub, config: ConfigMixin, values):
                 config.__xpm__._sealed = False
                 config.__xpm__._identifier = None
 
         Unsealer(context, recurse_task=True)(self.pyobject)
 
-    def collect_pre_tasks(self) -> Iterator["Config"]:
-        context = ConfigWalkContext()
-        pre_tasks: Dict[int, "Config"] = {}
-
-        class PreTaskCollect(ConfigWalk):
-            def preprocess(self, config: Config):
-                # Do not cross tasks
-                return not isinstance(config.__xpm__, Task), config
-
-            def postprocess(self, stub, config: Config, values):
-                pre_tasks.update(
-                    {id(pre_task): pre_task for pre_task in config.__xpm__.pre_tasks}
-                )
-
-        PreTaskCollect(context, recurse_task=True)(self.pyobject)
-        return pre_tasks.values()
-
-    def identifiers(self, only_raw: bool):
+    @property
+    def identifier(self):
         """Computes the unique identifier"""
-        from ..identifier import IdentifierComputer, Identifier
-
-        raw_identifier = self._raw_identifier
-        full_identifier = self._full_identifier
+        from ..identifier import IdentifierComputer
 
         # Computes raw identifier if needed
-        if raw_identifier is None or not self._sealed:
-            # Get the main identifier
-            raw_identifier = IdentifierComputer.compute(self.pyobject)
-            if self._sealed:
-                self._raw_identifier = raw_identifier
-
-        if only_raw:
-            return raw_identifier, full_identifier
-
-        # OK, let's compute the full identifier
-        if full_identifier is None or not self._sealed:
-            # Compute the full identifier by including the pre-tasks
-            hasher = hashlib.sha256()
-            hasher.update(raw_identifier.all)
-            pre_tasks_ids = [
-                pre_task.__xpm__.raw_identifier.all
-                for pre_task in self.collect_pre_tasks()
-            ]
-            for task_id in sorted(pre_tasks_ids):
-                hasher.update(task_id)
-
-            # Adds init tasks
-            if self.init_tasks:
-                hasher.update(IdentifierComputer.INIT_TASKS)
-                for init_task in self.init_tasks:
-                    hasher.update(init_task.__xpm__.raw_identifier.all)
-
-            full_identifier = Identifier(hasher.digest())
-            full_identifier.has_loops = raw_identifier.has_loops
-
-            # Only cache the identifier if sealed
-            if self._sealed:
-                self._full_identifier = full_identifier
+        if self._identifier is not None:
+            return self._identifier
 
-        return raw_identifier, full_identifier
-
-    @property
-    def raw_identifier(self) -> "Identifier":
-        """Computes the unique identifier (without task modifiers)"""
-        raw_identifier, _ = self.identifiers(True)
-        return raw_identifier
-
-    @property
-    def full_identifier(self) -> "Identifier":
-        """Computes the unique identifier (with task modifiers)"""
-        _, full_identifier = self.identifiers(False)
-        return full_identifier
-
-    identifier = full_identifier
-    """Deprecated: use full_identifier"""
+        # Get the main identifier
+        identifier = IdentifierComputer.compute(self.pyobject)
+        if self._sealed:
+            self._identifier = identifier
+        return identifier
 
     def dependency(self):
         """Returns a dependency"""
@@ -501,12 +484,6 @@ class ConfigInformation:
         path: List[str],
         taskids: Set[int],
     ):
-        # Add pre-tasks
-        for pre_task in self.pre_tasks:
-            pre_task.__xpm__.updatedependencies(
-                dependencies, path + ["__pre_tasks__"], taskids
-            )
-
         # Add initialization tasks
         for init_task in self.init_tasks:
             init_task.__xpm__.updatedependencies(
@@ -776,9 +753,6 @@ class ConfigInformation:
         if self.task is not None and self.task is not self:
             ConfigInformation.__collect_objects__(self.task, objects, context)
 
-        # Serialize pre-tasks
-        ConfigInformation.__collect_objects__(self.pre_tasks, objects, context)
-
         # Serialize initialization tasks
         ConfigInformation.__collect_objects__(self.init_tasks, objects, context)
 
@@ -792,8 +766,6 @@ class ConfigInformation:
         }
 
         # Add pre/init tasks
-        if self.pre_tasks:
-            state_dict["pre-tasks"] = [id(pre_task) for pre_task in self.pre_tasks]
         if self.init_tasks:
             state_dict["init-tasks"] = [id(init_task) for init_task in self.init_tasks]
 
@@ -1125,12 +1097,6 @@ class ConfigInformation:
                 o.__post_init__()
 
             else:
-                # Sets pre-tasks
-                o.__xpm__.pre_tasks = [
-                    objects[pre_task_id]
-                    for pre_task_id in definition.get("pre-tasks", [])
-                ]
-
                 if task_id := definition.get("task", None):
                     o.__xpm__.task = objects[task_id]
 
@@ -1164,15 +1130,6 @@ class ConfigInformation:
 
         # Run pre-task (or returns them)
         if as_instance or return_tasks:
-            # Collect pre-tasks (just once)
-            completed_pretasks = set()
-            pre_tasks = []
-            for definition in definitions:
-                for pre_task_id in definition.get("pre-tasks", []):
-                    if pre_task_id not in completed_pretasks:
-                        completed_pretasks.add(pre_task_id)
-                        pre_tasks.append(objects[pre_task_id])
-
             # Collect init tasks
             init_tasks = []
             for init_task_id in definitions[-1].get("init-tasks", []):
@@ -1180,14 +1137,11 @@ class ConfigInformation:
                 init_tasks.append(init_task)
 
             if as_instance:
-                for pre_task in pre_tasks:
-                    logger.info("Executing pre-task %s", type(pre_task))
-                    pre_task.execute()
                 for init_task in init_tasks:
                     logger.info("Executing init task %s", type(init_task))
                     init_task.execute()
             else:
-                return o, pre_tasks, pre_task + init_tasks
+                return o, init_tasks
 
         return o
 
@@ -1195,7 +1149,6 @@ class ConfigInformation:
         def __init__(self, context: ConfigWalkContext, *, objects: ObjectStore = None):
             super().__init__(context)
             self.objects = ObjectStore() if objects is None else objects
-            self.pre_tasks = {}
 
         def preprocess(self, config: "Config"):
             if self.objects.is_constructed(id(config)):
@@ -1222,10 +1175,6 @@ class ConfigInformation:
             # Call __post_init__
             stub.__post_init__()
 
-            # Gather pre-tasks
-            for pre_task in config.__xpm__.pre_tasks:
-                self.pre_tasks[id(pre_task)] = self.stub(pre_task)
-
             self.objects.set_constructed(id(config))
             return stub
 
@@ -1239,10 +1188,6 @@ class ConfigInformation:
         processor = ConfigInformation.FromPython(context, objects=objects)
         last_object = processor(self.pyobject)
 
-        # Execute pre-tasks
-        for pre_task in processor.pre_tasks.values():
-            pre_task.execute()
-
         return last_object
 
     def add_dependencies(self, *dependencies):
@@ -1428,29 +1373,7 @@ class ConfigMixin:
         attributes)"""
         return clone(self)
 
-    def add_pretasks(self, *tasks: "LightweightTask"):
-        assert all(
-            [isinstance(task, LightweightTask) for task in tasks]
-        ), "One of the pre-tasks are not lightweight tasks"
-        if self.__xpm__._sealed:
-            raise SealedError("Cannot add pre-tasks to a sealed configuration")
-        self.__xpm__.pre_tasks.extend(tasks)
-        return self
-
-    def add_pretasks_from(self, *configs: "Config"):
-        assert all(
-            [isinstance(config, ConfigMixin) for config in configs]
-        ), "One of the parameters is not a configuration object"
-        for config in configs:
-            self.add_pretasks(*config.__xpm__.pre_tasks)
-        return self
-
-    @property
-    def pre_tasks(self) -> List["LightweightTask"]:
-        """Access pre-tasks"""
-        return self.__xpm__.pre_tasks
-
-    def copy_dependencies(self, other: "Config"):
+    def copy_dependencies(self, other: "ConfigMixin"):
         """Add all the dependencies from other configuration"""
 
         # Add task dependency
@@ -1587,27 +1510,12 @@ class Config:
     def __identifier__(self) -> "Identifier":
         return self.__xpm__.full_identifier
 
-    def add_pretasks(self, *tasks: "LightweightTask"):
-        """Add pre-tasks"""
-        raise AssertionError("This method can only be used during configuration")
-
-    def add_pretasks_from(self, *configs: "Config"):
-        """Add pre-tasks from the listed configurations"""
-        raise AssertionError(
-            "The 'add_pretasks_from' can only be used during configuration"
-        )
-
     def copy_dependencies(self, other: "Config"):
         """Add pre-tasks from the listed configurations"""
         raise AssertionError(
             "The 'copy_dependencies' method can only be used during configuration"
         )
 
-    @property
-    def pre_tasks(self) -> List["LightweightTask"]:
-        """Access pre-tasks"""
-        raise AssertionError("Pre-tasks can be accessed only during configuration")
-
     def register_task_output(self, method, *args, **kwargs):
         # Determine the path for this...
         path = taskglobals.Env.instance().xpm_path / "task-outputs.jsonl"

{experimaestro-1.13.0 → experimaestro-1.15.0}/src/experimaestro/core/objects/config_walk.py

@@ -71,6 +71,7 @@ class ConfigWalk:
         return self.context.push(str(i))
 
     def map(self, k: str):
+        """Provides a path context when processing a tree"""
         return self.context.push(k)
 
     def stub(self, config):
@@ -108,11 +109,7 @@ class ConfigWalk:
                 else:
                     result[arg.name] = None
 
-            # Deals with pre-tasks
-            if info.pre_tasks:
-                with self.map("__pre_tasks__"):
-                    self(info.pre_tasks)
-
+            # Deals with init tasks
             if info.init_tasks:
                 with self.map("__init_tasks__"):
                     self(info.init_tasks)
@@ -123,7 +120,8 @@ class ConfigWalk:
                 and self.recurse_task
                 and x.__xpm__.task is not x
             ):
-                self(x.__xpm__.task)
+                with self.map("__task__"):
+                    self(x.__xpm__.task)
 
             processed = self.postprocess(stub, x, result)
             self.visited[xid] = processed

{experimaestro-1.13.0 → experimaestro-1.15.0}/src/experimaestro/core/objects.pyi

@@ -168,7 +168,7 @@ class ConfigMixin:
         *,
         workspace: Incomplete | None = ...,
         launcher: Incomplete | None = ...,
-        run_mode: RunMode = ...
+        run_mode: RunMode = ...,
     ): ...
     def stdout(self): ...
     def stderr(self): ...
@@ -195,11 +195,7 @@ class Config:
     def __post_init__(self) -> None: ...
     def __json__(self): ...
     def __identifier__(self) -> Identifier: ...
-    def add_pretasks(self, *tasks: "LightweightTask"): ...
-    def add_pretasks_from(self, configs: "Config"): ...
     def copy_dependencies(self, other: "Config"): ...
-    @property
-    def pre_tasks(self) -> List["LightweightTask"]: ...
 
 class LightweightTask(Config):
     def execute(self) -> None: ...
@@ -213,7 +209,7 @@ class Task(LightweightTask):
         workspace: Incomplete | None = ...,
         launcher: Incomplete | None = ...,
         run_mode: RunMode = ...,
-        init_tasks: List["LightweightTask"] = []
+        init_tasks: List["LightweightTask"] = [],
     ): ...
     def task_outputs(self, dep: Callable[[Config], None]) -> Any: ...
 

{experimaestro-1.13.0 → experimaestro-1.15.0}/src/experimaestro/core/serializers.py

@@ -1,10 +1,8 @@
-from typing import List, TypeVar, Callable, Any
-from pathlib import Path
+from typing import List, TypeVar
 from experimaestro import Param
 
 from .objects import Config, LightweightTask
 from .arguments import DataPath
-from experimaestro import copyconfig
 
 
 class SerializationLWTask(LightweightTask):
@@ -39,8 +37,3 @@ class PathSerializationLWTask(SerializationLWTask):
 
     path: DataPath
     """Path containing the data"""
-
-    @classmethod
-    def construct(cls, value: T, path: Path, dep: Callable[[Config], Any]) -> T:
-        value = copyconfig(value)
-        return value.add_pretasks(dep(cls(value=value, path=path)))

{experimaestro-1.13.0 → experimaestro-1.15.0}/src/experimaestro/tests/test_dependencies.py

@@ -85,9 +85,3 @@ def test_dependencies_inner_task_output(xp):
     a = task_a.submit()
     b = Inner_TaskB(param_a=a).submit()
     check_dependencies(b, task_a)
-
-
-def test_dependencies_pre_task(xp):
-    a = TaskA().submit()
-    a2 = TaskA().add_pretasks(a).submit()
-    check_dependencies(a2, a)