experimaestro 1.11.1__py3-none-any.whl → 2.0.0b4__py3-none-any.whl

experimaestro/__init__.py

@@ -4,15 +4,7 @@ from pathlib import Path
 
 # Annotations
 from .annotations import (
-    config,
-    task,
-    param,
-    ConstantParam,
-    constant,
-    option,
-    pathoption,
     cache,
-    Identifier,
     Array,
     TagDict,
     tag,
@@ -21,12 +13,11 @@ from .annotations import (
     STDOUT,
     STDERR,
     deprecate,
-    # deprecated
-    argument,
     initializer,
     # Method
     config_only,
 )
+from .core.types import Identifier
 from .core.serialization import (
     load,
     save,
@@ -47,15 +38,22 @@ from .core.arguments import (
     field,
     # Annotations helpers
     help,
-    default,
 )
 from .generators import pathgenerator, PathGenerator
+from .core.subparameters import (
+    subparameters,
+    param_group,
+    ParameterGroup,
+    Subparameters,
+)
 from .core.objects import (
     Config,
+    InstanceConfig,
     copyconfig,
     setmeta,
     DependentMarker,
     Task,
+    ResumableTask,
     LightweightTask,
     ObjectStore,
 )
@@ -64,6 +62,7 @@ from .core.serializers import SerializationLWTask, PathSerializationLWTask
 from .core.types import Any, SubmitHook
 from .launchers import Launcher
 from .scheduler import Scheduler, experiment, FailedExperiment
+from .exceptions import GracefulTimeout
 from .scheduler.workspace import Workspace, RunMode
 from .scheduler.state import get_experiment
 from .notifications import progress, tqdm

experimaestro/annotations.py

@@ -1,18 +1,14 @@
 # Import Python modules
 
 import inspect
-from pathlib import Path
-from typing import Callable, Type as TypingType, Optional, TypeVar, Union
+from typing import Callable, Type as TypingType, TypeVar, Union
 from sortedcontainers import SortedDict
 import experimaestro.core.objects as objects
 import experimaestro.core.types as types
-from experimaestro.generators import PathGenerator
 
-from .core.arguments import Argument as CoreArgument, field
-from .core.objects import Config
-from .core.types import Any, Identifier, TypeProxy, Type, ObjectType
+from .core.objects import Config, Task
+from .core.types import Any, TypeProxy, Type
 from .utils import logger
-from .checkers import Checker
 
 # --- Annotations to define tasks and types
 
@@ -24,56 +20,6 @@ def configmethod(method):
     return method
 
 
-class config:
-    """Annotations for experimaestro types"""
-
-    def __init__(self, identifier=None, description=None):
-        """[summary]
-
-        Keyword Arguments:
-            identifier {Identifier, str} -- Unique identifier of the type,
-            generate by default (None)
-
-            description {str} -- (deprecated, use
-            comments) Description of the config/task, use comments with
-            (default) None
-
-            register {bool} -- False if the type should not be
-            registered (debug only)
-
-        The identifier, if not specified, will be set to `X.CLASSNAME`(by order
-        of priority), where X is:
-            - the parent identifier
-            - the module qualified name
-        """
-        super().__init__()
-        self.identifier = identifier
-        if isinstance(self.identifier, str):
-            self.identifier = Identifier(self.identifier)
-
-        self.description = description
-
-    def __call__(self, tp: T) -> T:
-        """Annotate the class
-
-        Depending on whether we are running or configuring,
-        the behavior is different:
-
-        - when configuring, we return a proxy class
-        - when running, we return the same class
-
-        Arguments:
-            tp {type} -- The type
-
-        Keyword Arguments:
-            basetype {type} -- The base type of the class (Task or Config)
-        """
-        assert inspect.isclass(tp), f"{tp} is not a class"
-
-        # Adds to xpminfo for on demand creation of information
-        return ObjectType(tp, identifier=self.identifier).basetype
-
-
 class Array(TypeProxy):
     """Array of object"""
 
@@ -94,121 +40,12 @@ class Choice(TypeProxy):
         return types.StringType
 
 
-class task(config):
-    """Register a task"""
-
-    def __init__(self, identifier=None, pythonpath=None, description=None):
-        super().__init__(identifier, description)
-        self.pythonpath = pythonpath
-
-    def __call__(self, tp) -> type:
-        # Register the type
-        tp = super().__call__(tp)
-
-        def factory(xpmtype):
-            return xpmtype.getpythontaskcommand(self.pythonpath)
-
-        tp.__getxpmtype__().taskcommandfactory = factory
-        return tp
-
-
-# --- argument related annotations
-
-
-class param:
-    """Defines an argument for an experimaestro type"""
-
-    def __init__(
-        self,
-        name,
-        type=None,
-        default=None,
-        required: bool = None,
-        ignored: Optional[bool] = None,
-        help: Optional[str] = None,
-        checker: Optional[Checker] = None,
-        constant: bool = False,
-    ):
-        # Determine if required
-        self.name = name
-        self.type = Type.fromType(type) if type else None
-        self.help = help
-        self.ignored = ignored
-        self.required = required
-        self.checker = checker
-        self.constant = constant
-
-        self.generator = None
-        self.default = None
-
-        # Set default or generator
-        if isinstance(default, field):
-            if default.default is not None:
-                self.default = default
-            elif default.default_factory is not None:
-                self.generator = default.default_factory
-        else:
-            self.default = default
-
-    def __call__(self, tp):
-        # Don't annotate in task mode
-        tp.__getxpmtype__().addAnnotation(self)
-        return tp
-
-    def process(self, xpmtype):
-        # Get type from default if needed
-        if self.type is None:
-            if self.default is not None:
-                self.type = Type.fromType(type(self.default))
-
-        # Type = any if no type
-        if self.type is None:
-            self.type = types.Any
-
-        argument = CoreArgument(
-            self.name,
-            self.type,
-            help=self.help,
-            required=self.required,
-            ignored=self.ignored,
-            generator=self.generator,
-            default=self.default,
-            checker=self.checker,
-            constant=self.constant,
-        )
-        xpmtype.addArgument(argument)
-
-
-# Just a rebind (back-compatibility)
-argument = param
-
-
-class option(param):
-    """An argument which is ignored
-
-    See argument
-    """
-
-    def __init__(self, *args, **kwargs):
-        kwargs["ignored"] = True
-        super().__init__(*args, **kwargs)
-
-
-class pathoption(param):
-    """Defines a an argument that will be a relative path (automatically
-    set by experimaestro)"""
-
-    def __init__(self, name: str, path=None, help=""):
-        """
-        :param name: The name of argument (in python)
-        :param path: The relative path or a function
-        """
-        super().__init__(name, type=Path, help=help)
+def config_only(method):
+    """Marks a configuration-only method"""
+    assert inspect.ismethod(method)
 
-        if path is None:
-            path = name
 
-        self.generator = PathGenerator(path)
+# --- Path generators for task stdout/stderr
 
 
 def STDERR(jobcontext, config):
@@ -219,28 +56,31 @@ def STDOUT(jobcontext, config):
     return "%s.out" % jobcontext.name
 
 
-class constant(param):
-    """
-    A constant argument (useful for versionning tasks)
-    """
-
-    def __init__(self, name: str, value, type=None, help=""):
-        super().__init__(name, default=value, constant=True, type=type, help=help)
-
+# --- Cache
 
-ConstantParam = constant
 
+def cache(name: str):
+    """Decorator for caching method results to disk.
 
-def config_only(method):
-    """Marks a configuration-only method"""
-    assert inspect.ismethod(method)
+    The cache is stored in the workspace's config directory, keyed by the
+    configuration's identifier.
 
+    Example::
 
-# --- Cache
+        class MyConfig(Config):
+            data_path: Param[Path]
 
+            @cache("processed.pkl")
+            def process(self, cache_path: Path):
+                if cache_path.exists():
+                    return pickle.load(cache_path.open("rb"))
+                result = expensive_computation(self.data_path)
+                pickle.dump(result, cache_path.open("wb"))
+                return result
 
-def cache(name: str):
-    """Use a cache path for a given config"""
+    :param name: Filename for the cache file
+    :return: A decorator that wraps the method with caching logic
+    """
 
     def annotate(method):
         return objects.cache(method, name)
@@ -252,7 +92,22 @@ def cache(name: str):
 
 
 def tag(value):
-    """Tag a value"""
+    """Tag a parameter value for tracking in experiments.
+
+    Tagged values appear in experiment logs and can be used for filtering
+    and organizing results. Tags are included in the task's ``__tags__``
+    dictionary.
+
+    Example::
+
+        task = MyTask.C(
+            learning_rate=tag(0.001),  # Will appear in task tags
+            batch_size=32,
+        ).submit()
+
+    :param value: The value to tag (str, int, float, or bool)
+    :return: A tagged value wrapper that preserves the original value
+    """
     return objects.TaggedValue(value)
 
 
@@ -267,7 +122,19 @@ class TagDict(SortedDict):
 
 
 def tags(value) -> TagDict:
-    """Return the tags associated with a value"""
+    """Return the tags associated with a configuration.
+
+    Returns a dictionary of all tagged parameter values from this configuration
+    and its nested configurations.
+
+    Example::
+
+        config = MyTask.C(learning_rate=tag(0.001), epochs=tag(100))
+        task_tags = tags(config)  # {"learning_rate": 0.001, "epochs": 100}
+
+    :param value: A configuration object
+    :return: A TagDict with tag names as keys and tagged values as values
+    """
     return TagDict(value.__xpm__.tags())
 
 
@@ -277,8 +144,20 @@ def _normalizepathcomponent(v: Any):
     return v
 
 
-def tagspath(value: Config):
-    """Return a unique path made of tags and their values"""
+def tagspath(value: Config) -> str:
+    """Generate a unique path string from a configuration's tags.
+
+    Useful for creating tag-based directory structures. Tags are sorted
+    alphabetically and joined with underscores.
+
+    Example::
+
+        config = MyTask.C(learning_rate=tag(0.001), epochs=tag(100))
+        path = tagspath(config)  # "epochs=100_learning_rate=0.001"
+
+    :param value: A configuration object
+    :return: A string with sorted tags in ``key=value`` format, joined by ``_``
+    """
     return "_".join(
         f"""{_normalizepathcomponent(key)}={_normalizepathcomponent(value)}"""
         for key, value in tags(value).items()
@@ -288,33 +167,101 @@ def tagspath(value: Config):
 # --- Deprecated
 
 
-def deprecate(config: Union[TypingType[Config], Callable]):
-    """Deprecate a configuration / task or
-    an attribute (via a method)
+def deprecate(
+    config_or_target: Union[TypingType[Config], Callable, None] = None,
+    *,
+    replace: bool = False,
+):
+    """Deprecate a configuration/task class or a parameter.
+
+    Deprecated configurations maintain backwards compatibility while allowing
+    migration to new structures. The identifier is computed from the converted
+    configuration, ensuring consistency.
+
+    **Usage patterns:**
 
-    Usage:
+    1. Simple deprecation (class inherits from new class)::
 
         @deprecate
         class OldConfig(NewConfig):
             pass
 
-        # Or only a parameter
-        class MyConfig():
+    2. Deprecation with conversion::
+
+        @deprecate(NewConfig)
+        class OldConfig(Config):
+            value: Param[int]
+
+            def __convert__(self):
+                return NewConfig.C(values=[self.value])
+
+    3. Immediate replacement::
+
+        @deprecate(NewConfig, replace=True)
+        class OldConfig(Config):
+            value: Param[int]
+
+            def __convert__(self):
+                return NewConfig.C(values=[self.value])
+
+    4. Deprecate a parameter::
+
+        class MyConfig(Config):
+            new_param: Param[list[int]]
+
             @deprecate
-            def oldattribute(self, value):
-                # Do something with the value
-                pass
-    """
-    if inspect.isclass(config):
-        config.__getxpmtype__().deprecate()
-        return config
+            def old_param(self, value: int):
+                self.new_param = [value]
 
-    if inspect.isfunction(config):
+    :param config_or_target: Target class for conversion, or the deprecated
+        class/method when used as a simple decorator
+    :param replace: If True, creating the deprecated class immediately returns
+        the converted instance
+    """
+    # Case 1: @deprecate on a function (deprecated attribute)
+    if inspect.isfunction(config_or_target):
         from experimaestro.core.types import DeprecatedAttribute
 
-        return DeprecatedAttribute(config)
+        return DeprecatedAttribute(config_or_target)
+
+    # Case 2: @deprecate (no parens) on a class - legacy pattern
+    # The class inherits from its target (NewConfig), not directly from Config
+    if config_or_target is not None and inspect.isclass(config_or_target):
+        # Check if this looks like a deprecated class (legacy pattern)
+        # Legacy pattern: @deprecate class OldConfig(NewConfig) where NewConfig is a Config subclass
+        # The deprecated class inherits from exactly one Config subclass (the target)
+        # We exclude Config and Task as base classes since those indicate the new pattern
+        base_classes_for_new_pattern = (Config, Task)
+        if (
+            not replace
+            and len(config_or_target.__bases__) == 1
+            and config_or_target.__bases__[0] not in base_classes_for_new_pattern
+            and issubclass(config_or_target.__bases__[0], Config)
+        ):
+            # This is the legacy pattern: @deprecate on a class
+            deprecated_class = config_or_target
+            deprecated_class.__getxpmtype__().deprecate()
+            return deprecated_class
+
+        # Otherwise, this is the new pattern: @deprecate(TargetConfig)
+        target = config_or_target
 
-    raise NotImplementedError("Cannot deprecate %s", config)
+        def decorator(deprecated_class: TypingType[Config]):
+            deprecated_class.__getxpmtype__().deprecate(target=target, replace=replace)
+            return deprecated_class
+
+        return decorator
+
+    # Case 3: @deprecate() with parentheses but no arguments (legacy, uses parent class)
+    if config_or_target is None:
+
+        def decorator(deprecated_class: TypingType[Config]):
+            deprecated_class.__getxpmtype__().deprecate()
+            return deprecated_class
+
+        return decorator
+
+    raise NotImplementedError("Cannot deprecate %s" % config_or_target)
 
 
 def deprecateClass(klass):
@@ -337,7 +284,21 @@ def deprecateClass(klass):
 
 
 def initializer(method):
-    """Defines a method as an initializer that can only be called once"""
+    """Decorator for methods that should only execute once.
+
+    After the first call, subsequent calls return the cached result.
+    This is useful for lazy initialization of expensive resources.
+
+    Example::
+
+        class MyConfig(Config):
+            @initializer
+            def model(self):
+                return load_expensive_model()
+
+    :param method: The method to wrap
+    :return: A wrapper that caches the result after first execution
+    """
 
     def wrapper(self, *args, **kwargs):
         value = method(self, *args, **kwargs)