experimaestro 1.11.1__py3-none-any.whl → 2.0.0b4__py3-none-any.whl

experimaestro/core/arguments.py

@@ -1,18 +1,18 @@
 """Management of the arguments (params, options, etc) associated with the XPM objects"""
 
+import warnings
 from typing import Optional, TypeVar, TYPE_CHECKING, Callable, Any
 from experimaestro.typingutils import get_optional
 from pathlib import Path
-import sys
+from typing import Annotated
 
 if TYPE_CHECKING:
-    from typing_extensions import Annotated
     import experimaestro.core.types
-else:
-    if sys.version_info.major == 3 and sys.version_info.minor < 9:
-        from typing_extensions import Annotated
-    else:
-        from typing import Annotated
+    from experimaestro.core.subparameters import ParameterGroup
+
+# Track deprecation warnings per module (max 10 per module)
+_deprecation_warning_counts: dict[str, int] = {}
+_MAX_WARNINGS_PER_MODULE = 10
 
 
 class Argument:
@@ -29,10 +29,12 @@ class Argument:
         help=None,
         generator=None,
         ignored=None,
-        default=None,
+        field_or_default=None,
         checker=None,
         constant=False,
         is_data=False,
+        overrides=False,
+        groups: set["ParameterGroup"] = None,
     ):
         """Creates a new argument
 
@@ -52,7 +54,8 @@ class Argument:
             ignored (bool, optional): True if ignored (if None, computed
             automatically). Defaults to None.
 
-            default (any, optional): . Defaults to None.
+            field_or_default (any | field, optional): Default value or field
+            object containing default information. Defaults to None.
 
             checker (any, optional): Value checker. Defaults to None.
 
@@ -61,9 +64,17 @@ class Argument:
 
             is_data (bool, optional): Flag for paths that are data path (to be
             serialized). Defaults to False.
+
+            overrides (bool, optional): If True, this argument intentionally
+            overrides a parent argument. Suppresses the warning that would
+            otherwise be issued. Defaults to False.
+
+            groups (set[ParameterGroup], optional): Set of groups this parameter
+            belongs to. Used with subparameters to compute partial identifiers.
+            Defaults to None (empty set).
         """
-        required = (default is None) if required is None else required
-        if default is not None and required is not None and required:
+        required = (field_or_default is None) if required is None else required
+        if field_or_default is not None and required is not None and required:
             raise Exception(
                 "argument '%s' is required but default value is given" % name
             )
@@ -77,19 +88,42 @@ class Argument:
         self.required = required
         self.objecttype = None
         self.is_data = is_data
+        self.overrides = overrides
 
         self.generator = generator
         self.default = None
-
-        if default is not None:
-            assert self.generator is None, "generator and default are exclusive options"
-            if isinstance(default, field):
-                if default.default is not None:
-                    self.default = default.default
-                elif default.default_factory is not None:
-                    self.generator = default.default_factory
+        self.ignore_generated = False
+        self.ignore_default_in_identifier = False
+        self.groups = groups if groups else set()
+
+        if field_or_default is not None:
+            assert (
+                self.generator is None
+            ), "generator and field_or_default are exclusive options"
+            if isinstance(field_or_default, field):
+                self.ignore_generated = field_or_default.ignore_generated
+                # Allow field to override the overrides flag
+                if field_or_default.overrides:
+                    self.overrides = True
+
+                # Process groups from field
+                if field_or_default.groups:
+                    self.groups = field_or_default.groups
+
+                if field_or_default.ignore_default is not None:
+                    # ignore_default: value is default AND ignored in identifier
+                    self.default = field_or_default.ignore_default
+                    self.ignore_default_in_identifier = True
+                elif field_or_default.default is not None:
+                    # default: value is default but NOT ignored in identifier
+                    self.default = field_or_default.default
+                    self.ignore_default_in_identifier = False
+                elif field_or_default.default_factory is not None:
+                    self.generator = field_or_default.default_factory
             else:
-                self.default = default
+                # Bare default: backwards compatible, ignore in identifier
+                self.default = field_or_default
+                self.ignore_default_in_identifier = True
 
         assert (
             not self.constant or self.default is not None
@@ -134,12 +168,44 @@ class ArgumentOptions:
         optionaltype = get_optional(typehint)
         type = Type.fromType(optionaltype or typehint)
 
-        if "default" not in self.kwargs or self.kwargs["default"] is None:
+        if (
+            "field_or_default" not in self.kwargs
+            or self.kwargs["field_or_default"] is None
+        ):
             defaultvalue = getattr(originaltype, name, None)
-            self.kwargs["default"] = defaultvalue
+            self.kwargs["field_or_default"] = defaultvalue
+
+            # Emit deprecation warning for bare default values (not wrapped in field)
+            # Skip warning for Constant parameters - they are inherently constant, not defaults
+            if (
+                defaultvalue is not None
+                and not isinstance(defaultvalue, field)
+                and not self.kwargs.get("constant")
+            ):
+                module = originaltype.__module__
+                count = _deprecation_warning_counts.get(module, 0)
+                if count < _MAX_WARNINGS_PER_MODULE:
+                    _deprecation_warning_counts[module] = count + 1
+                    class_name = originaltype.__qualname__
+                    remaining = _MAX_WARNINGS_PER_MODULE - count - 1
+                    limit_msg = (
+                        f" ({remaining} more warnings for this module)"
+                        if remaining > 0
+                        else " (no more warnings for this module)"
+                    )
+                    warnings.warn(
+                        f"Deprecated: parameter `{name}` in {module}.{class_name} "
+                        f"has an ambiguous default value. Use `field(ignore_default=...)` "
+                        f"to keep current behavior (default ignored in identifier) or "
+                        f"`field(default=...)` to include default in identifier. "
+                        f"Run `experimaestro refactor default-values` to fix automatically."
+                        f"{limit_msg}",
+                        DeprecationWarning,
+                        stacklevel=6,
+                    )
 
         self.kwargs["required"] = (optionaltype is None) and (
-            self.kwargs["default"] is None
+            self.kwargs["field_or_default"] is None
         )
 
         return Argument(name, type, **self.kwargs)
@@ -171,26 +237,116 @@ T = TypeVar("T")
 
 paramHint = _Param()
 Param = Annotated[T, paramHint]
+"""Type annotation for configuration parameters.
+
+Parameters annotated with ``Param[T]`` are included in the configuration
+identifier computation and must be set before the configuration is sealed.
+
+Example::
+
+    class MyConfig(Config):
+        name: Param[str]
+        count: Param[int] = field(default=10)
+        threshold: Param[float] = field(ignore_default=0.5)
+"""
 
 optionHint = _Param(ignored=True)
 Option = Annotated[T, optionHint]
+"""Deprecated alias for Meta. Use Meta instead."""
+
 Meta = Annotated[T, optionHint]
+"""Type annotation for meta-parameters (ignored in identifier computation).
+
+Use ``Meta[T]`` for parameters that should not affect the task identity,
+such as output paths or runtime configuration.
+
+Example::
+
+    class MyTask(Task):
+        # This affects the task identity
+        learning_rate: Param[float]
+
+        # This does not affect the identity
+        checkpoint_path: Meta[Path] = field(default_factory=PathGenerator("model.pt"))
+"""
 
 dataHint = _Param(ignored=True, is_data=True)
 DataPath = Annotated[Path, dataHint]
-"""Annotates a path that should be kept to restore an object to its state"""
+"""Type annotation for data paths that should be serialized.
+
+Use ``DataPath`` for paths that point to data files that should be
+preserved when serializing/deserializing a configuration. The path
+is copied during serialization.
+
+Example::
+
+    class MyConfig(Config):
+        model_weights: DataPath
+"""
 
 
 class field:
-    """Extra information for a given experimaestro field (param or meta)"""
+    """Specify additional properties for a configuration parameter.
+
+    Use ``field()`` to control default value behavior and parameter grouping.
+
+    Example::
+
+        class MyConfig(Config):
+            # Default included in identifier
+            count: Param[int] = field(default=10)
+
+            # Default ignored in identifier (backwards compatible)
+            threshold: Param[float] = field(ignore_default=0.5)
 
-    def __init__(self, *, default: Any = None, default_factory: Callable = None):
+            # Generated path
+            output: Meta[Path] = field(default_factory=PathGenerator("out.txt"))
+
+            # Parameter in a group (for partial identifiers)
+            lr: Param[float] = field(groups=[training_group])
+    """
+
+    def __init__(
+        self,
+        *,
+        default: Any = None,
+        default_factory: Callable = None,
+        ignore_default: Any = None,
+        ignore_generated=False,
+        overrides=False,
+        groups: list["ParameterGroup"] = None,
+    ):
+        """Create a field specification.
+
+        :param default: Default value (included in identifier computation)
+        :param default_factory: Callable that generates the default value
+        :param ignore_default: Default value that is ignored in identifier computation
+            when the actual value equals this default. Use for backwards-compatible
+            behavior with bare default values.
+        :param ignore_generated: If True, the generated value is hidden from tasks.
+            Useful for adding a field that changes the identifier but won't be used.
+        :param overrides: If True, suppress warning when overriding parent parameter
+        :param groups: List of ParameterGroup objects for partial identifiers.
+            Used with subparameters to compute identifiers that exclude certain groups.
+        """
         assert not (
             (default is not None) and (default_factory is not None)
         ), "default and default_factory are mutually exclusive options"
 
+        assert not (
+            (default is not None) and (ignore_default is not None)
+        ), "default and ignore_default are mutually exclusive options"
+
+        assert not (
+            (ignore_default is not None) and (default_factory is not None)
+        ), "ignore_default and default_factory are mutually exclusive options"
+
         self.default_factory = default_factory
         self.default = default
+        self.ignore_default = ignore_default
+        self.ignore_generated = ignore_generated
+        self.overrides = overrides
+        self.groups = set(groups) if groups else set()
 
 
 class help(TypeAnnotation):
@@ -201,17 +357,6 @@ class help(TypeAnnotation):
         options.kwargs["help"] = self.text
 
 
-class default(TypeAnnotation):
-    """Adds a default value (useful when we have problems with setattr and class
-    properties)"""
-
-    def __init__(self, value):
-        self.value = value
-
-    def annotate(self, options: ArgumentOptions):
-        options.kwargs["default"] = self.value
-
-
 class ConstantHint(TypeAnnotation):
     def annotate(self, options: ArgumentOptions):
         options.kwargs["constant"] = True
@@ -219,3 +364,13 @@ class ConstantHint(TypeAnnotation):
 
 constantHint = ConstantHint()
 Constant = Annotated[T, constantHint]
+"""Type annotation for constant (read-only) parameters.
+
+Constants must have a default value and cannot be modified after creation.
+They are included in the identifier computation.
+
+Example::
+
+    class MyConfig(Config):
+        version: Constant[str] = "1.0"
+"""

experimaestro/core/identifier.py

@@ -5,8 +5,11 @@ import hashlib
 import logging
 import os
 import struct
-from typing import Optional
-from experimaestro.core.objects import Config
+from typing import Optional, TYPE_CHECKING
+from experimaestro.core.objects import Config, ConfigMixin
+
+if TYPE_CHECKING:
+    from experimaestro.core.subparameters import Subparameters
 
 
 class ConfigPath:
@@ -19,6 +22,9 @@ class ConfigPath:
         self.config2index = {}
         """Associate an index in the list with a configuration"""
 
+        self.instance_order: dict[bytes, list[int]] = {}
+        """Maps InstanceConfig base identifier to list of object ids in encounter order"""
+
     def detect_loop(self, config) -> Optional[int]:
         """If there is a loop, return the relative index and update the path"""
         index = self.config2index.get(id(config), None)
@@ -100,7 +106,15 @@ class Identifier:
 
 
 class IdentifierComputer:
-    """This class is in charge of computing a config/task identifier"""
+    """This class is in charge of computing a config/task identifier
+
+    Args:
+        config: The configuration to compute the identifier for
+        config_path: Used to track cycles when computing identifiers
+        version: Hash computation version (defaults to XPM_HASH_COMPUTER env var or 2)
+        subparameters: If provided, only include parameters that are not excluded
+            by this Subparameters instance (for partial identifier computation)
+    """
 
     OBJECT_ID = b"\x00"
     INT_ID = b"\x01"
@@ -115,13 +129,22 @@ class IdentifierComputer:
     ENUM_ID = b"\x0a"
     CYCLE_REFERENCE = b"\x0b"
     INIT_TASKS = b"\x0c"
-
-    def __init__(self, config: "Config", config_path: ConfigPath, *, version=None):
+    INSTANCE_ID = b"\x0d"
+
+    def __init__(
+        self,
+        config: "ConfigMixin",
+        config_path: ConfigPath,
+        *,
+        version=None,
+        subparameters: "Subparameters" = None,
+    ):
         # Hasher for parameters
         self._hasher = hashlib.sha256()
         self.config = config
         self.config_path = config_path
         self.version = version or int(os.environ.get("XPM_HASH_COMPUTER", 2))
+        self.subparameters = subparameters
         if hash_logger.isEnabledFor(logging.DEBUG):
             hash_logger.debug(
                 "starting hash (%s): %s", hash(str(self.config)), self.config
@@ -170,7 +193,7 @@ class IdentifierComputer:
             self._hashupdate(IdentifierComputer.ENUM_ID)
             k = value.__class__
             self._hashupdate(
-                f"{k.__module__}.{k.__qualname__ }:{value.name}".encode("utf-8"),
+                f"{k.__module__}.{k.__qualname__}:{value.name}".encode("utf-8"),
             )
         elif isinstance(value, dict):
             self._hashupdate(IdentifierComputer.DICT_ID)
@@ -183,7 +206,20 @@ class IdentifierComputer:
                 self.update(value)
 
         # Handles configurations
-        elif isinstance(value, Config):
+        elif isinstance(value, ConfigMixin):
+            # For deprecated configs with __convert__, compute identifier from converted config
+            # This must happen BEFORE hashing OBJECT_ID to ensure identical identifiers
+            if myself:
+                xpmtype = value.__xpmtype__
+                if xpmtype.deprecated and hasattr(value, "__convert__"):
+                    hash_logger.debug(
+                        "Computing hash via __convert__ for deprecated %s", xpmtype
+                    )
+                    converted = value.__convert__()
+                    # Delegate entirely to the converted config
+                    self.update(converted, myself=True)
+                    return
+
             # Encodes the identifier
             self._hashupdate(IdentifierComputer.OBJECT_ID)
 
@@ -201,6 +237,33 @@ class IdentifierComputer:
                     )
                     self._hashupdate(value_id.all)
 
+                    # Handle InstanceConfig: add instance order only if duplicate detected
+                    from experimaestro.core.objects import InstanceConfig
+
+                    if isinstance(value, InstanceConfig):
+                        # Use the base identifier to track instances
+                        base_id = value_id.all
+                        obj_id = id(value)
+
+                        # Track instances with this base identifier
+                        if base_id not in self.config_path.instance_order:
+                            # First occurrence - just record it (NO-OP for backwards compatibility)
+                            self.config_path.instance_order[base_id] = [obj_id]
+                        else:
+                            # Check if this specific object was already seen
+                            instances = self.config_path.instance_order[base_id]
+                            if obj_id not in instances:
+                                # New instance with same params - add to list
+                                instances.append(obj_id)
+
+                            # Get position in the list
+                            position = instances.index(obj_id)
+
+                            # Only add instance order if not the first (position > 0)
+                            if position > 0:
+                                self._hashupdate(IdentifierComputer.INSTANCE_ID)
+                                self._hashupdate(struct.pack("!q", position))
+
                 # And that's it!
                 return
 
@@ -216,6 +279,11 @@ class IdentifierComputer:
             # Process arguments (sort by name to ensure uniqueness)
             arguments = sorted(xpmtype.arguments.values(), key=lambda a: a.name)
             for argument in arguments:
+                # Skip arguments excluded by subparameters (for partial identifiers)
+                if self.subparameters is not None:
+                    if self.subparameters.is_excluded(argument.groups):
+                        continue
+
                 # Ignored argument
                 if argument.ignored:
                     argvalue = value.__xpm__.values.get(argument.name, None)
@@ -234,8 +302,18 @@ class IdentifierComputer:
                 # Argument value
                 # Skip if the argument is not a constant, and
                 # - optional argument: both value and default are None
-                # - the argument value is equal to the default value
-                argvalue = getattr(value, argument.name, None)
+                # - the argument value is equal to the default value AND
+                #   ignore_default_in_identifier is True
+                try:
+                    argvalue = getattr(value, argument.name, None)
+                except KeyError:
+                    logging.warning(
+                        "Parameter %s has not been set in %s created at %s",
+                        argument.name,
+                        value,
+                        value.__xpm__._initinfo,
+                    )
+                    raise
                 if not argument.constant and (
                     (
                         not argument.required
@@ -243,7 +321,8 @@ class IdentifierComputer:
                         and argvalue is None
                     )
                     or (
-                        argument.default is not None
+                        argument.ignore_default_in_identifier
+                        and argument.default is not None
                         and argument.default == remove_meta(argvalue)
                     )
                 ):
@@ -264,12 +343,17 @@ class IdentifierComputer:
                 self._hashupdate(IdentifierComputer.NAME_ID)
                 self.update(argvalue)
 
+            # Add init tasks
+            if value.__xpm__.init_tasks:
+                self._hashupdate(IdentifierComputer.INIT_TASKS)
+                for init_task in value.__xpm__.init_tasks:
+                    self.update(init_task)
         else:
             raise NotImplementedError("Cannot compute hash of type %s" % type(value))
 
     @staticmethod
     def compute(
-        config: "Config", config_path: ConfigPath | None = None, version=None
+        config: "ConfigMixin", config_path: ConfigPath | None = None, version=None
     ) -> Identifier:
         """Compute the identifier for a configuration
 
@@ -281,7 +365,7 @@ class IdentifierComputer:
         # Try to use the cached value first
         # (if there are no loops)
         if config.__xpm__._sealed:
-            identifier = config.__xpm__._raw_identifier
+            identifier = config.__xpm__._identifier
             if identifier is not None and not identifier.has_loops:
                 return identifier
 
@@ -294,3 +378,34 @@ class IdentifierComputer:
             identifier.has_loops = config_path.has_loop()
 
         return identifier
+
+    @staticmethod
+    def compute_partial(
+        config: "ConfigMixin",
+        subparameters: "Subparameters",
+        config_path: ConfigPath | None = None,
+        version=None,
+    ) -> Identifier:
+        """Compute a partial identifier for a configuration
+
+        A partial identifier excludes certain parameter groups, allowing
+        configurations that differ only in those groups to share the same
+        partial identifier (and thus the same partial directory).
+
+        :param config: the configuration for which we compute the identifier
+        :param subparameters: the Subparameters instance defining which groups
+            to include/exclude
+        :param config_path: used to track down cycles between configurations
+        :param version: version for the hash computation (None for the last one)
+        """
+        config_path = config_path or ConfigPath()
+
+        with config_path.push(config):
+            computer = IdentifierComputer(
+                config, config_path, version=version, subparameters=subparameters
+            )
+            computer.update(config, myself=True)
+            identifier = computer.identifier()
+            identifier.has_loops = config_path.has_loop()
+
+        return identifier

experimaestro/core/objects/__init__.py

@@ -2,8 +2,11 @@ from .config_walk import ConfigWalkContext, ConfigWalk
 from .config import (
     ConfigMixin,
     Config,
+    InstanceConfig,
     ConfigInformation,
     Task,
+    TaskStub,
+    ResumableTask,
     LightweightTask,
     WatchedOutput,
     DependentMarker,
@@ -25,10 +28,13 @@ from .config_utils import (
 __all__ = [
     "ConfigMixin",
     "Config",
+    "InstanceConfig",
     "ConfigInformation",
     "ConfigWalkContext",
     "ConfigWalk",
     "Task",
+    "TaskStub",
+    "ResumableTask",
     "LightweightTask",
     "ObjectStore",
     "WatchedOutput",