experimaestro 1.11.1__py3-none-any.whl → 2.0.0a3__py3-none-any.whl

experimaestro/core/objects/config.py

@@ -25,7 +25,6 @@ from typing import (
     Optional,
     Set,
     Tuple,
-    Type,
     TypeVar,
     Union,
     overload,
@@ -122,11 +121,11 @@ class ConfigInformation:
     def __init__(self, pyobject: "ConfigMixin"):
         # The underlying pyobject and XPM type
         self.pyobject = pyobject
-        self.xpmtype = pyobject.__xpmtype__  # type: ObjectType
+        self.xpmtype: "ObjectType" = pyobject.__xpmtype__
         self.values = {}
 
         # Meta-informations
-        self._tags = {}
+        self._tags: dict[str, Any] = {}
         self._initinfo = ""
 
         self._taskoutput = None
@@ -142,7 +141,7 @@ class ConfigInformation:
         #: True when this configuration was loaded from disk
         self.loaded = False
 
-        # Explicitely added dependencies
+        # Explicitly added dependencies
         self.dependencies = []
 
         # Concrete type variables resolutions
@@ -170,6 +169,34 @@ class ConfigInformation:
         self._sealed = False
         self._meta = None
 
+        # This contains the list of generated values (using context) in this
+        # configuration or any sub-configuration, is generated. This prevents
+        # problem when a configuration with generated values is re-used.
+        self._generated_values = []
+
+    def get_generated_paths(
+        self, path: list[str] = None, paths: list[str] = None
+    ) -> list[str]:
+        """Get the list of generated paths, useful to track down those
+
+        :param path: The current path
+        :param paths: The list of generated paths so far, defaults to None
+        :return: The full list of generated paths
+        """
+        paths = [] if paths is None else paths
+        path = [] if path is None else path
+
+        for key in self._generated_values:
+            value = self.values[key]
+            if isinstance(value, ConfigMixin) and value.__xpm__._generated_values:
+                path.append(key)
+                value.__xpm__.get_generated_paths(path, paths)
+                path.pop()
+            else:
+                paths.append(".".join(path + [key]))
+
+        return paths
+
     def set_meta(self, value: Optional[bool]):
         """Sets the meta flag"""
         assert not self._sealed, "Configuration is sealed"
@@ -198,9 +225,26 @@ class ConfigInformation:
         if self._sealed and not bypass:
             raise AttributeError(f"Object is read-only (trying to set {k})")
 
+        if not isinstance(v, ConfigMixin) and isinstance(v, Config):
+            raise AttributeError(
+                "Configuration (and not objects) should be used. Consider using .C(...)"
+            )
+
         try:
             argument = self.xpmtype.arguments.get(k, None)
             if argument:
+                if (
+                    isinstance(v, ConfigMixin)
+                    and v.__xpm__._generated_values
+                    and v.__xpm__.task is None
+                    and not argument.ignore_generated
+                ):
+                    raise AttributeError(
+                        f"Cannot set {k} to a configuration with generated values. "
+                        "Here is the list of paths to help you: "
+                        f"""{', '.join(v.__xpm__.get_generated_paths([k]))}"""
+                    )
+
                 if not bypass and (
                     (isinstance(argument.generator, Generator)) or argument.constant
                 ):
@@ -326,12 +370,23 @@ class ConfigInformation:
         Arguments:
             - context: the generation context
         """
+        if generated_keys := [
+            k
+            for k, v in self.values.items()
+            if isinstance(v, Config)
+            and v.__xpm__.task is None
+            and v.__xpm__._generated_values
+        ]:
+            raise AttributeError(
+                "Cannot seal a configuration with generated values:"
+                f"""{",".join(generated_keys)} in {context.currentpath}"""
+            )
 
         class Sealer(ConfigWalk):
-            def preprocess(self, config: Config):
+            def preprocess(self, config: ConfigMixin):
                 return not config.__xpm__._sealed, config
 
-            def postprocess(self, stub, config: Config, values):
+            def postprocess(self, stub, config: ConfigMixin, values):
                 # Generate values
                 from experimaestro.generators import Generator
 
@@ -344,22 +399,42 @@ class ConfigInformation:
                                     continue
                                 value = argument.generator()
                             else:
+                                # Generate a value
                                 sig = inspect.signature(argument.generator)
                                 if len(sig.parameters) == 0:
                                     value = argument.generator()
                                 elif len(sig.parameters) == 2:
+                                    # Only in that case do we need to flag this configuration
+                                    # as containing generated values
+                                    if not argument.ignore_generated:
+                                        config.__xpm__._generated_values.append(k)
+                                    else:
+                                        logging.warning("Ignoring %s", k)
                                     value = argument.generator(self.context, config)
                                 else:
                                     assert (
                                         False
                                     ), "generator has either two parameters (context and config), or none"
                             config.__xpm__.set(k, value, bypass=True)
+                        else:
+                            value = config.__xpm__.values.get(k)
                     except Exception:
                         logger.error(
                             "While setting %s of %s", argument.name, config.__xpmtype__
                         )
                         raise
 
+                    # Propagate the generated value flag
+                    if (
+                        value is not None
+                        and isinstance(value, ConfigMixin)
+                        and value.__xpm__._generated_values
+                    ):
+                        if not argument.ignore_generated:
+                            config.__xpm__._generated_values.append(k)
+                        else:
+                            logging.warning("Ignoring %s", k)
+
                 config.__xpm__._sealed = True
 
         Sealer(context, recurse_task=True)(self.pyobject)
@@ -657,13 +732,6 @@ class ConfigInformation:
 
                 print(file=sys.stderr)  # noqa: T201
 
-        # Handle an output configuration # FIXME: remove
-        def mark_output(config: "Config"):
-            """Sets a dependency on the job"""
-            assert not isinstance(config, Task), "Cannot set a dependency on a task"
-            config.__xpm__.task = self.pyobject
-            return config
-
         # Mark this configuration also
         self.task = self.pyobject
 
@@ -677,6 +745,9 @@ class ConfigInformation:
     def mark_output(self, config: "Config"):
         """Sets a dependency on the job"""
         assert not isinstance(config, Task), "Cannot set a dependency on a task"
+        assert isinstance(
+            config, ConfigMixin
+        ), "Only configurations can be marked as dependent on a task"
         config.__xpm__.task = self.pyobject
         return config
 
@@ -762,7 +833,7 @@ class ConfigInformation:
         state_dict = {
             "id": id(self.pyobject),
             "module": self.xpmtype._module,
-            "type": self.xpmtype.basetype.__qualname__,
+            "type": self.xpmtype.value_type.__qualname__,
             "typename": self.xpmtype.name(),
             "identifier": self.identifier.state_dict(),
         }
@@ -1022,7 +1093,7 @@ class ConfigInformation:
 
             # Creates an object (or a config)
             if as_instance:
-                o = cls.XPMValue.__new__(cls.XPMValue)
+                o = cls.__new__(cls)
             else:
                 o = cls.XPMConfig.__new__(cls.XPMConfig)
             assert definition["id"] not in objects, "Duplicate id %s" % definition["id"]
@@ -1183,7 +1254,7 @@ class ConfigInformation:
 
             if o is None:
                 # Creates an object (and not a config)
-                o = config.XPMValue()
+                o = config.__xpmtype__.value_type()
 
                 # Store in cache
                 self.objects.add_stub(id(config), o)
@@ -1242,6 +1313,9 @@ def clone(v):
     if isinstance(v, Enum):
         return v
 
+    if isinstance(v, tuple):
+        return tuple(clone(x) for x in v)
+
     if isinstance(v, Config):
         # Create a new instance
         kwargs = {
@@ -1260,6 +1334,11 @@ class ConfigMixin:
     """Class for configuration objects"""
 
     __xpmtype__: ObjectType
+    """The associated XPM type"""
+
+    __xpm__: ConfigInformation
+    """The __xpm__ object contains all instance specific information about a
+    configuration/task"""
 
     def __init__(self, **kwargs):
         """Initialize the configuration with the given parameters"""
@@ -1310,8 +1389,8 @@ class ConfigMixin:
             [f"{key}={value}" for key, value in self.__xpm__.values.items()]
         )
         return (
-            f"{self.__xpmtype__.basetype.__module__}."
-            f"{self.__xpmtype__.basetype.__qualname__}({params})"
+            f"{self.__xpmtype__.value_type.__module__}."
+            f"{self.__xpmtype__.value_type.__qualname__}({params})"
         )
 
     def tag(self, name, value):
@@ -1340,9 +1419,20 @@ class ConfigMixin:
         return self
 
     def instance(
-        self, context: ConfigWalkContext = None, *, objects: ObjectStore = None
+        self,
+        context: ConfigWalkContext = None,
+        *,
+        objects: ObjectStore = None,
+        keep: bool = True,
     ) -> T:
-        """Return an instance with the current values"""
+        """Return an instance with the current values
+
+        :param context: The context when computing the instance
+        :param objects: The previously built objects (so that we avoid
+            re-creating instances of past configurations)
+        :param keep: register a configuration in the __config__ field of the
+            instance
+        """
         if context is None:
             from experimaestro.xpmutils import EmptyContext
 
@@ -1351,7 +1441,11 @@ class ConfigMixin:
             assert isinstance(
                 context, ConfigWalkContext
             ), f"{context.__class__} is not an instance of ConfigWalkContext"
-        return self.__xpm__.fromConfig(context, objects=objects)  # type: ignore
+
+        instance = self.__xpm__.fromConfig(context, objects=objects)  # type: ignore
+        if keep:
+            object.__setattr__(instance, "__config__", self)
+        return instance
 
     def submit(
         self,
@@ -1397,6 +1491,9 @@ class ConfigMixin:
         return clone(self)
 
     def add_pretasks(self, *tasks: "LightweightTask"):
+        assert all(
+            [isinstance(task, ConfigMixin) for task in tasks]
+        ), "One of the parameters is not a configuration object"
         assert all(
             [isinstance(task, LightweightTask) for task in tasks]
         ), "One of the pre-tasks are not lightweight tasks"
@@ -1441,52 +1538,17 @@ class Config:
     """The object type holds all the information about a specific subclass
     experimaestro metadata"""
 
-    __xpm__: ConfigInformation
-    """The __xpm__ object contains all instance specific information about a
-    configuration/task"""
-
     @classproperty
     def XPMConfig(cls):
         if issubclass(cls, ConfigMixin):
             return cls
-        return cls.__getxpmtype__().configtype
-
-    @classproperty
-    def XPMValue(cls):
-        """Returns the value object for this configuration"""
-        if issubclass(cls, ConfigMixin):
-            return cls.__xpmtype__.objecttype
-
-        if value_cls := cls.__dict__.get("__XPMValue__", None):
-            pass
-        else:
-            from ..types import XPMValue
-
-            __objectbases__ = tuple(
-                s.XPMValue
-                for s in cls.__bases__
-                if issubclass(s, Config) and (s is not Config)
-            ) or (XPMValue,)
-
-            *tp_qual, tp_name = cls.__qualname__.split(".")
-            value_cls = type(f"{tp_name}.XPMValue", (cls,) + __objectbases__, {})
-            value_cls.__qualname__ = ".".join(tp_qual + [value_cls.__name__])
-            value_cls.__module__ = cls.__module__
-
-            setattr(cls, "__XPMValue__", value_cls)
-
-        return value_cls
+        return cls.__getxpmtype__().config_type
 
     @classproperty
     def C(cls):
         """Alias for XPMConfig"""
         return cls.XPMConfig
 
-    @classproperty
-    def V(cls):
-        """Alias for XPMValue"""
-        return cls.XPMValue
-
     @classmethod
     def __getxpmtype__(cls) -> "ObjectType":
         """Get (and create if necessary) the Object type associated
@@ -1503,46 +1565,6 @@ class Config:
                 raise
         return xpmtype
 
-    def __new__(cls: Type[T], *args, **kwargs) -> T:
-        """Returns an instance of a ConfigMixin (for compatibility, use XPMConfig
-        or C if possible)
-
-        :deprecated: Use Config.C or Config.XPMConfig to construct a new
-            configuration, and Config.V (or Config.XPMValue) for a new value
-        """
-        # If this is an XPMValue, just return a new instance
-        from experimaestro.core.types import XPMValue
-
-        if issubclass(cls, XPMValue):
-            return object.__new__(cls)
-
-        # If this is the XPMConfig, just return a new instance
-        # __init__ will be called
-        if issubclass(cls, ConfigMixin):
-            return object.__new__(cls)
-
-        # Log a deprecation warning for this way of creating a configuration
-        caller = inspect.getframeinfo(inspect.stack()[1][0])
-        logger.warning(
-            "Creating a configuration using Config.__new__ is deprecated, and will be removed in a future version. "
-            "Use Config.C or Config.XPMConfig to create a new configuration. "
-            "Issue created at %s:%s",
-            str(Path(caller.filename).absolute()),
-            caller.lineno,
-        )
-
-        # otherwise, we use the configuration type
-        o: ConfigMixin = object.__new__(cls.__getxpmtype__().configtype)
-        try:
-            o.__init__(*args, **kwargs)
-        except Exception:
-            logger.error(
-                "Init error in %s:%s"
-                % (str(Path(caller.filename).absolute()), caller.lineno)
-            )
-            raise
-        return o
-
     def __validate__(self):
         """Validate the values"""
         pass

experimaestro/core/objects/config_walk.py

@@ -71,6 +71,7 @@ class ConfigWalk:
         return self.context.push(str(i))
 
     def map(self, k: str):
+        """Provides a path context when processing a tree"""
         return self.context.push(k)
 
     def stub(self, config):
@@ -123,7 +124,8 @@ class ConfigWalk:
                 and self.recurse_task
                 and x.__xpm__.task is not x
             ):
-                self(x.__xpm__.task)
+                with self.map("__task__"):
+                    self(x.__xpm__.task)
 
             processed = self.postprocess(stub, x, result)
             self.visited[xid] = processed

experimaestro/core/types.py

@@ -203,18 +203,14 @@ class ObjectType(Type):
     """ObjectType contains class-level information about
     experimaestro configurations and tasks
 
-    :param objecttype: The python Type of the associated object
-    :param configtype: The python Type of the configuration object that uses
-        property for arguments
+    :param value_type: The Python type of the associated object
+    :param config_type: The Python type of the configuration object
     """
 
-    # Those entries should not be copied in the __dict__
-    FORBIDDEN_KEYS = set(("__dict__", "__weakref__"))
-
     def __init__(
         self,
         tp: type,
-        identifier: Union[str, Identifier] = None,
+        identifier: Union[str, Identifier, None] = None,
     ):
         """Creates a type"""
         from .objects import Config, ConfigMixin
@@ -225,7 +221,7 @@ class ObjectType(Type):
         self._title = None
         self.submit_hooks = set()
 
-        # Get the identifier
+        # --- Get the identifier
         if identifier is None and hasattr(tp, "__xpmid__"):
             __xpmid__ = getattr(tp, "__xpmid__")
             if isinstance(__xpmid__, Identifier):
@@ -250,59 +246,41 @@ class ObjectType(Type):
         # --- Creates the config type and not config type
 
         self.originaltype = tp
-        if not issubclass(tp, Config):
-            # Adds Config as a base class if not present
-            __bases__ = () if tp.__bases__ == (object,) else tp.__bases__
-            __dict__ = dict(tp.__dict__)
-
-            __dict__ = {
-                key: value
-                for key, value in tp.__dict__.items()
-                if key not in ObjectType.FORBIDDEN_KEYS
-            }
-            self.basetype = type(tp.__name__, (Config,) + __bases__, __dict__)
-            self.basetype.__module__ = tp.__module__
-            self.basetype.__qualname__ = tp.__qualname__
-        else:
-            self.basetype = tp
+        assert issubclass(tp, Config)
+        self.value_type = tp
 
         # --- Create the type-specific configuration class (XPMConfig)
         __configbases__ = tuple(
-            s.__getxpmtype__().configtype
+            s.__getxpmtype__().config_type
             for s in tp.__bases__
             if issubclass(s, Config) and (s is not Config)
         ) or (ConfigMixin,)
 
-        *tp_qual, tp_name = self.basetype.__qualname__.split(".")
-        self.configtype = type(
-            f"{tp_name}.XPMConfig", __configbases__ + (self.basetype,), {}
+        *tp_qual, tp_name = self.value_type.__qualname__.split(".")
+        self.config_type = type(
+            f"{tp_name}.XPMConfig", __configbases__ + (self.value_type,), {}
         )
-        self.configtype.__qualname__ = ".".join(tp_qual + [self.configtype.__name__])
-        self.configtype.__module__ = tp.__module__
+        self.config_type.__qualname__ = ".".join(tp_qual + [self.config_type.__name__])
+        self.config_type.__module__ = tp.__module__
 
-        # Return type is used by tasks to change the output
-        if hasattr(self.basetype, "task_outputs") or False:
+        # --- Get the return type
+        if hasattr(self.value_type, "task_outputs") or False:
             self.returntype = get_type_hints(
-                getattr(self.basetype, "task_outputs")
+                getattr(self.value_type, "task_outputs")
             ).get("return", typing.Any)
         else:
-            self.returntype = self.basetype
+            self.returntype = self.value_type
 
-        # Registers ourselves
-        self.basetype.__xpmtype__ = self
-        self.configtype.__xpmtype__ = self
+        # --- Registers ourselves
+        self.value_type.__xpmtype__ = self
+        self.config_type.__xpmtype__ = self
 
-        # Other initializations
+        # --- Other initializations
         self.__initialized__ = False
         self._runtype = None
         self.annotations = []
         self._deprecated = False
 
-    @property
-    def objecttype(self):
-        """Returns the object type"""
-        return self.basetype.XPMValue
-
     def addAnnotation(self, annotation):
         assert not self.__initialized__
         self.annotations.append(annotation)
@@ -357,15 +335,15 @@ class ObjectType(Type):
         # Add task
         if self.taskcommandfactory is not None:
             self.task = self.taskcommandfactory(self)
-        elif issubclass(self.basetype, Task):
+        elif issubclass(self.value_type, Task):
             self.task = self.getpythontaskcommand()
 
         # Add arguments from type hints
         from .arguments import TypeAnnotation
 
-        if hasattr(self.basetype, "__annotations__"):
-            typekeys = set(self.basetype.__dict__.get("__annotations__", {}).keys())
-            hints = get_type_hints(self.basetype, include_extras=True)
+        if hasattr(self.value_type, "__annotations__"):
+            typekeys = set(self.value_type.__dict__.get("__annotations__", {}).keys())
+            hints = get_type_hints(self.value_type, include_extras=True)
             for key, typehint in hints.items():
                 # Filter out hints from parent classes
                 if key in typekeys:
@@ -378,19 +356,19 @@ class ObjectType(Type):
                             try:
                                 self.addArgument(
                                     options.create(
-                                        key, self.objecttype, typehint.__args__[0]
+                                        key, self.value_type, typehint.__args__[0]
                                     )
                                 )
                             except Exception:
                                 logger.error(
                                     "while adding argument %s of %s",
                                     key,
-                                    self.objecttype,
+                                    self.value_type,
                                 )
                                 raise
 
     def name(self):
-        return f"{self.basetype.__module__}.{self.basetype.__qualname__}"
+        return f"{self.value_type.__module__}.{self.value_type.__qualname__}"
 
     def __parsedoc__(self):
         """Parse the documentation"""
@@ -400,7 +378,7 @@ class ObjectType(Type):
         self.__initialize__()
 
         # Get description from documentation
-        __doc__ = self.basetype.__dict__.get("__doc__", None)
+        __doc__ = self.value_type.__dict__.get("__doc__", None)
         if __doc__:
             parseddoc = parse(__doc__)
             self._title = parseddoc.short_description
@@ -430,7 +408,7 @@ class ObjectType(Type):
                 argname = None
 
     def deprecate(self):
-        if len(self.basetype.__bases__) != 1:
+        if len(self.value_type.__bases__) != 1:
             raise RuntimeError(
                 "Deprecated configurations must have "
                 "only one parent (the new configuration)"
@@ -439,7 +417,7 @@ class ObjectType(Type):
 
         # Uses the parent identifier (and saves the deprecated one for path updates)
         self._deprecated_identifier = self.identifier
-        parent = self.basetype.__bases__[0].__getxpmtype__()
+        parent = self.value_type.__bases__[0].__getxpmtype__()
         self.identifier = parent.identifier
         self._deprecated = True
 
@@ -454,7 +432,7 @@ class ObjectType(Type):
         return self._description
 
     @property
-    def title(self) -> Dict[str, Argument]:
+    def title(self) -> str:
         self.__parsedoc__()
         return self._title or str(self.identifier)
 
@@ -469,7 +447,7 @@ class ObjectType(Type):
 
         # The the attribute for the config type
         setattr(
-            self.configtype,
+            self.config_type,
             argument.name,
             property(
                 lambda _self: _self.__xpm__.get(argument.name),
@@ -488,7 +466,7 @@ class ObjectType(Type):
     def parents(self) -> Iterator["ObjectType"]:
         from .objects import Config, Task
 
-        for tp in self.basetype.__bases__:
+        for tp in self.value_type.__bases__:
             if issubclass(tp, Config) and tp not in [Config, Task]:
                 yield tp.__xpmtype__
 
@@ -504,7 +482,7 @@ class ObjectType(Type):
         if not isinstance(value, Config):
             raise ValueError(f"{value} is not an experimaestro type or task")
 
-        types = self.basetype
+        types = self.value_type
 
         if not isinstance(value, types):
             raise ValueError(
@@ -519,7 +497,7 @@ class ObjectType(Type):
 
     def fullyqualifiedname(self) -> str:
         """Returns the fully qualified (Python) name"""
-        return f"{self.basetype.__module__}.{self.basetype.__qualname__}"
+        return f"{self.value_type.__module__}.{self.value_type.__qualname__}"
 
 
 class TypeProxy:

experimaestro/launcherfinder/registry.py

@@ -6,7 +6,7 @@ from typing import ClassVar, Dict, Optional, Set, Type, Union
 from pathlib import Path
 import typing
 from omegaconf import DictConfig, OmegaConf, SCMode
-import pkg_resources
+from importlib.metadata import entry_points
 from experimaestro.utils import logger
 from .base import ConnectorConfiguration, TokenConfiguration
 from .specs import HostRequirement, RequirementUnion
@@ -75,10 +75,10 @@ class LauncherRegistry:
         self.find_launcher_fn = None
 
         # Use entry points for connectors and launchers
-        for entry_point in pkg_resources.iter_entry_points("experimaestro.connectors"):
+        for entry_point in entry_points(group="experimaestro.connectors"):
             entry_point.load().init_registry(self)
 
-        for entry_point in pkg_resources.iter_entry_points("experimaestro.tokens"):
+        for entry_point in entry_points(group="experimaestro.tokens"):
             entry_point.load().init_registry(self)
 
         # Register the find launcher function if it exists