experimaestro 2.0.0a8__py3-none-any.whl → 2.0.0b4__py3-none-any.whl

experimaestro/__init__.py

@@ -4,15 +4,7 @@ from pathlib import Path
 
 # Annotations
 from .annotations import (
-    config,
-    task,
-    param,
-    ConstantParam,
-    constant,
-    option,
-    pathoption,
     cache,
-    Identifier,
     Array,
     TagDict,
     tag,
@@ -21,12 +13,11 @@ from .annotations import (
     STDOUT,
     STDERR,
     deprecate,
-    # deprecated
-    argument,
     initializer,
     # Method
     config_only,
 )
+from .core.types import Identifier
 from .core.serialization import (
     load,
     save,
@@ -47,15 +38,22 @@ from .core.arguments import (
     field,
     # Annotations helpers
     help,
-    default,
 )
 from .generators import pathgenerator, PathGenerator
+from .core.subparameters import (
+    subparameters,
+    param_group,
+    ParameterGroup,
+    Subparameters,
+)
 from .core.objects import (
     Config,
+    InstanceConfig,
     copyconfig,
     setmeta,
     DependentMarker,
     Task,
+    ResumableTask,
     LightweightTask,
     ObjectStore,
 )
@@ -64,6 +62,7 @@ from .core.serializers import SerializationLWTask, PathSerializationLWTask
 from .core.types import Any, SubmitHook
 from .launchers import Launcher
 from .scheduler import Scheduler, experiment, FailedExperiment
+from .exceptions import GracefulTimeout
 from .scheduler.workspace import Workspace, RunMode
 from .scheduler.state import get_experiment
 from .notifications import progress, tqdm

experimaestro/annotations.py

@@ -1,18 +1,14 @@
 # Import Python modules
 
 import inspect
-from pathlib import Path
-from typing import Callable, Type as TypingType, Optional, TypeVar, Union
+from typing import Callable, Type as TypingType, TypeVar, Union
 from sortedcontainers import SortedDict
 import experimaestro.core.objects as objects
 import experimaestro.core.types as types
-from experimaestro.generators import PathGenerator
 
-from .core.arguments import Argument as CoreArgument, field
-from .core.objects import Config
-from .core.types import Any, Identifier, TypeProxy, Type, ObjectType
+from .core.objects import Config, Task
+from .core.types import Any, TypeProxy, Type
 from .utils import logger
-from .checkers import Checker
 
 # --- Annotations to define tasks and types
 
@@ -24,56 +20,6 @@ def configmethod(method):
     return method
 
 
-class config:
-    """Annotations for experimaestro types"""
-
-    def __init__(self, identifier=None, description=None):
-        """[summary]
-
-        Keyword Arguments:
-            identifier {Identifier, str} -- Unique identifier of the type,
-            generate by default (None)
-
-            description {str} -- (deprecated, use
-            comments) Description of the config/task, use comments with
-            (default) None
-
-            register {bool} -- False if the type should not be
-            registered (debug only)
-
-        The identifier, if not specified, will be set to `X.CLASSNAME`(by order
-        of priority), where X is:
-            - the parent identifier
-            - the module qualified name
-        """
-        super().__init__()
-        self.identifier = identifier
-        if isinstance(self.identifier, str):
-            self.identifier = Identifier(self.identifier)
-
-        self.description = description
-
-    def __call__(self, tp: T) -> T:
-        """Annotate the class
-
-        Depending on whether we are running or configuring,
-        the behavior is different:
-
-        - when configuring, we return a proxy class
-        - when running, we return the same class
-
-        Arguments:
-            tp {type} -- The type
-
-        Keyword Arguments:
-            basetype {type} -- The base type of the class (Task or Config)
-        """
-        assert inspect.isclass(tp), f"{tp} is not a class"
-
-        # Adds to xpminfo for on demand creation of information
-        return ObjectType(tp, identifier=self.identifier).valuetype
-
-
 class Array(TypeProxy):
     """Array of object"""
 
@@ -94,121 +40,12 @@ class Choice(TypeProxy):
         return types.StringType
 
 
-class task(config):
-    """Register a task"""
-
-    def __init__(self, identifier=None, pythonpath=None, description=None):
-        super().__init__(identifier, description)
-        self.pythonpath = pythonpath
-
-    def __call__(self, tp) -> type:
-        # Register the type
-        tp = super().__call__(tp)
-
-        def factory(xpmtype):
-            return xpmtype.getpythontaskcommand(self.pythonpath)
-
-        tp.__getxpmtype__().taskcommandfactory = factory
-        return tp
-
-
-# --- argument related annotations
-
-
-class param:
-    """Defines an argument for an experimaestro type"""
-
-    def __init__(
-        self,
-        name,
-        type=None,
-        default=None,
-        required: bool = None,
-        ignored: Optional[bool] = None,
-        help: Optional[str] = None,
-        checker: Optional[Checker] = None,
-        constant: bool = False,
-    ):
-        # Determine if required
-        self.name = name
-        self.type = Type.fromType(type) if type else None
-        self.help = help
-        self.ignored = ignored
-        self.required = required
-        self.checker = checker
-        self.constant = constant
-
-        self.generator = None
-        self.default = None
-
-        # Set default or generator
-        if isinstance(default, field):
-            if default.default is not None:
-                self.default = default
-            elif default.default_factory is not None:
-                self.generator = default.default_factory
-        else:
-            self.default = default
-
-    def __call__(self, tp):
-        # Don't annotate in task mode
-        tp.__getxpmtype__().addAnnotation(self)
-        return tp
-
-    def process(self, xpmtype):
-        # Get type from default if needed
-        if self.type is None:
-            if self.default is not None:
-                self.type = Type.fromType(type(self.default))
-
-        # Type = any if no type
-        if self.type is None:
-            self.type = types.Any
-
-        argument = CoreArgument(
-            self.name,
-            self.type,
-            help=self.help,
-            required=self.required,
-            ignored=self.ignored,
-            generator=self.generator,
-            default=self.default,
-            checker=self.checker,
-            constant=self.constant,
-        )
-        xpmtype.addArgument(argument)
-
-
-# Just a rebind (back-compatibility)
-argument = param
-
-
-class option(param):
-    """An argument which is ignored
-
-    See argument
-    """
-
-    def __init__(self, *args, **kwargs):
-        kwargs["ignored"] = True
-        super().__init__(*args, **kwargs)
-
-
-class pathoption(param):
-    """Defines a an argument that will be a relative path (automatically
-    set by experimaestro)"""
-
-    def __init__(self, name: str, path=None, help=""):
-        """
-        :param name: The name of argument (in python)
-        :param path: The relative path or a function
-        """
-        super().__init__(name, type=Path, help=help)
+def config_only(method):
+    """Marks a configuration-only method"""
+    assert inspect.ismethod(method)
 
-        if path is None:
-            path = name
 
-        self.generator = PathGenerator(path)
+# --- Path generators for task stdout/stderr
 
 
 def STDERR(jobcontext, config):
@@ -219,28 +56,31 @@ def STDOUT(jobcontext, config):
     return "%s.out" % jobcontext.name
 
 
-class constant(param):
-    """
-    A constant argument (useful for versionning tasks)
-    """
-
-    def __init__(self, name: str, value, type=None, help=""):
-        super().__init__(name, default=value, constant=True, type=type, help=help)
-
+# --- Cache
 
-ConstantParam = constant
 
+def cache(name: str):
+    """Decorator for caching method results to disk.
 
-def config_only(method):
-    """Marks a configuration-only method"""
-    assert inspect.ismethod(method)
+    The cache is stored in the workspace's config directory, keyed by the
+    configuration's identifier.
 
+    Example::
 
-# --- Cache
+        class MyConfig(Config):
+            data_path: Param[Path]
 
+            @cache("processed.pkl")
+            def process(self, cache_path: Path):
+                if cache_path.exists():
+                    return pickle.load(cache_path.open("rb"))
+                result = expensive_computation(self.data_path)
+                pickle.dump(result, cache_path.open("wb"))
+                return result
 
-def cache(name: str):
-    """Use a cache path for a given config"""
+    :param name: Filename for the cache file
+    :return: A decorator that wraps the method with caching logic
+    """
 
     def annotate(method):
         return objects.cache(method, name)
@@ -252,7 +92,22 @@ def cache(name: str):
 
 
 def tag(value):
-    """Tag a value"""
+    """Tag a parameter value for tracking in experiments.
+
+    Tagged values appear in experiment logs and can be used for filtering
+    and organizing results. Tags are included in the task's ``__tags__``
+    dictionary.
+
+    Example::
+
+        task = MyTask.C(
+            learning_rate=tag(0.001),  # Will appear in task tags
+            batch_size=32,
+        ).submit()
+
+    :param value: The value to tag (str, int, float, or bool)
+    :return: A tagged value wrapper that preserves the original value
+    """
     return objects.TaggedValue(value)
 
 
@@ -267,7 +122,19 @@ class TagDict(SortedDict):
 
 
 def tags(value) -> TagDict:
-    """Return the tags associated with a value"""
+    """Return the tags associated with a configuration.
+
+    Returns a dictionary of all tagged parameter values from this configuration
+    and its nested configurations.
+
+    Example::
+
+        config = MyTask.C(learning_rate=tag(0.001), epochs=tag(100))
+        task_tags = tags(config)  # {"learning_rate": 0.001, "epochs": 100}
+
+    :param value: A configuration object
+    :return: A TagDict with tag names as keys and tagged values as values
+    """
     return TagDict(value.__xpm__.tags())
 
 
@@ -277,8 +144,20 @@ def _normalizepathcomponent(v: Any):
     return v
 
 
-def tagspath(value: Config):
-    """Return a unique path made of tags and their values"""
+def tagspath(value: Config) -> str:
+    """Generate a unique path string from a configuration's tags.
+
+    Useful for creating tag-based directory structures. Tags are sorted
+    alphabetically and joined with underscores.
+
+    Example::
+
+        config = MyTask.C(learning_rate=tag(0.001), epochs=tag(100))
+        path = tagspath(config)  # "epochs=100_learning_rate=0.001"
+
+    :param value: A configuration object
+    :return: A string with sorted tags in ``key=value`` format, joined by ``_``
+    """
     return "_".join(
         f"""{_normalizepathcomponent(key)}={_normalizepathcomponent(value)}"""
         for key, value in tags(value).items()
@@ -288,33 +167,101 @@ def tagspath(value: Config):
 # --- Deprecated
 
 
-def deprecate(config: Union[TypingType[Config], Callable]):
-    """Deprecate a configuration / task or
-    an attribute (via a method)
+def deprecate(
+    config_or_target: Union[TypingType[Config], Callable, None] = None,
+    *,
+    replace: bool = False,
+):
+    """Deprecate a configuration/task class or a parameter.
+
+    Deprecated configurations maintain backwards compatibility while allowing
+    migration to new structures. The identifier is computed from the converted
+    configuration, ensuring consistency.
+
+    **Usage patterns:**
 
-    Usage:
+    1. Simple deprecation (class inherits from new class)::
 
         @deprecate
         class OldConfig(NewConfig):
             pass
 
-        # Or only a parameter
-        class MyConfig():
+    2. Deprecation with conversion::
+
+        @deprecate(NewConfig)
+        class OldConfig(Config):
+            value: Param[int]
+
+            def __convert__(self):
+                return NewConfig.C(values=[self.value])
+
+    3. Immediate replacement::
+
+        @deprecate(NewConfig, replace=True)
+        class OldConfig(Config):
+            value: Param[int]
+
+            def __convert__(self):
+                return NewConfig.C(values=[self.value])
+
+    4. Deprecate a parameter::
+
+        class MyConfig(Config):
+            new_param: Param[list[int]]
+
             @deprecate
-            def oldattribute(self, value):
-                # Do something with the value
-                pass
-    """
-    if inspect.isclass(config):
-        config.__getxpmtype__().deprecate()
-        return config
+            def old_param(self, value: int):
+                self.new_param = [value]
 
-    if inspect.isfunction(config):
+    :param config_or_target: Target class for conversion, or the deprecated
+        class/method when used as a simple decorator
+    :param replace: If True, creating the deprecated class immediately returns
+        the converted instance
+    """
+    # Case 1: @deprecate on a function (deprecated attribute)
+    if inspect.isfunction(config_or_target):
         from experimaestro.core.types import DeprecatedAttribute
 
-        return DeprecatedAttribute(config)
+        return DeprecatedAttribute(config_or_target)
+
+    # Case 2: @deprecate (no parens) on a class - legacy pattern
+    # The class inherits from its target (NewConfig), not directly from Config
+    if config_or_target is not None and inspect.isclass(config_or_target):
+        # Check if this looks like a deprecated class (legacy pattern)
+        # Legacy pattern: @deprecate class OldConfig(NewConfig) where NewConfig is a Config subclass
+        # The deprecated class inherits from exactly one Config subclass (the target)
+        # We exclude Config and Task as base classes since those indicate the new pattern
+        base_classes_for_new_pattern = (Config, Task)
+        if (
+            not replace
+            and len(config_or_target.__bases__) == 1
+            and config_or_target.__bases__[0] not in base_classes_for_new_pattern
+            and issubclass(config_or_target.__bases__[0], Config)
+        ):
+            # This is the legacy pattern: @deprecate on a class
+            deprecated_class = config_or_target
+            deprecated_class.__getxpmtype__().deprecate()
+            return deprecated_class
+
+        # Otherwise, this is the new pattern: @deprecate(TargetConfig)
+        target = config_or_target
 
-    raise NotImplementedError("Cannot deprecate %s", config)
+        def decorator(deprecated_class: TypingType[Config]):
+            deprecated_class.__getxpmtype__().deprecate(target=target, replace=replace)
+            return deprecated_class
+
+        return decorator
+
+    # Case 3: @deprecate() with parentheses but no arguments (legacy, uses parent class)
+    if config_or_target is None:
+
+        def decorator(deprecated_class: TypingType[Config]):
+            deprecated_class.__getxpmtype__().deprecate()
+            return deprecated_class
+
+        return decorator
+
+    raise NotImplementedError("Cannot deprecate %s" % config_or_target)
 
 
 def deprecateClass(klass):
@@ -337,7 +284,21 @@ def deprecateClass(klass):
 
 
 def initializer(method):
-    """Defines a method as an initializer that can only be called once"""
+    """Decorator for methods that should only execute once.
+
+    After the first call, subsequent calls return the cached result.
+    This is useful for lazy initialization of expensive resources.
+
+    Example::
+
+        class MyConfig(Config):
+            @initializer
+            def model(self):
+                return load_expensive_model()
+
+    :param method: The method to wrap
+    :return: A wrapper that caches the result after first execution
+    """
 
     def wrapper(self, *args, **kwargs):
         value = method(self, *args, **kwargs)

experimaestro/cli/__init__.py

@@ -1,7 +1,6 @@
 # flake8: noqa: T201
 import sys
 from typing import Set, Optional
-import pkg_resources
 from itertools import chain
 from shutil import rmtree
 import click
@@ -10,11 +9,12 @@ from functools import cached_property, update_wrapper
 from pathlib import Path
 import subprocess
 from termcolor import cprint
+from importlib.metadata import entry_points
 
 import experimaestro
 from experimaestro.experiments.cli import experiments_cli
 import experimaestro.launcherfinder.registry as launcher_registry
-from experimaestro.settings import find_workspace
+from experimaestro.settings import ServerSettings, find_workspace
 
 # --- Command line main options
 logging.basicConfig(level=logging.INFO)
@@ -257,13 +257,13 @@ def find_launchers(config: Optional[Path], spec: str):
     print(launcher_registry.find_launcher(spec))
 
 
-class Launchers(click.MultiCommand):
-    """Connectors commands"""
+class Launchers(click.Group):
+    """Dynamic command group for entry point discovery"""
 
     @cached_property
     def commands(self):
         map = {}
-        for ep in pkg_resources.iter_entry_points(f"experimaestro.{self.name}"):
+        for ep in entry_points(group=f"experimaestro.{self.name}"):
             if get_cli := getattr(ep.load(), "get_cli", None):
                 map[ep.name] = get_cli()
         return map
@@ -289,6 +289,11 @@ from .jobs import jobs as jobs_cli
 
 cli.add_command(jobs_cli)
 
+# Import and add refactor commands
+from .refactor import refactor as refactor_cli
+
+cli.add_command(refactor_cli)
+
 
 @cli.group()
 @click.option("--workdir", type=Path, default=None)
@@ -309,3 +314,123 @@ def list(workdir: Path):
             cprint(f"[unfinished] {p.name}", "yellow")
         else:
             cprint(p.name, "cyan")
+
+
+@experiments.command()
+@click.option("--console", is_flag=True, help="Use console TUI instead of web UI")
+@click.option(
+    "--port", type=int, default=12345, help="Port for web server (default: 12345)"
+)
+@click.option(
+    "--sync", is_flag=True, help="Force sync from disk before starting monitor"
+)
+@pass_cfg
+def monitor(workdir: Path, console: bool, port: int, sync: bool):
+    """Monitor experiments with web UI or console TUI"""
+    # Force sync from disk if requested
+    if sync:
+        from experimaestro.scheduler.state_sync import sync_workspace_from_disk
+
+        cprint("Syncing workspace from disk...", "yellow")
+        sync_workspace_from_disk(workdir, write_mode=True, force=True)
+        cprint("Sync complete", "green")
+
+    if console:
+        # Use Textual TUI
+        from experimaestro.tui import ExperimentTUI
+
+        app = ExperimentTUI(workdir, watch=True)
+        app.run()
+    else:
+        # Use React web server
+        from experimaestro.scheduler.state_provider import WorkspaceStateProvider
+        from experimaestro.server import Server
+
+        cprint(f"Starting experiment monitor on http://localhost:{port}", "green")
+        cprint("Press Ctrl+C to stop", "yellow")
+
+        state_provider = WorkspaceStateProvider.get_instance(
+            workdir,
+            sync_on_start=not sync,  # Skip auto-sync if we just did a forced one
+        )
+        settings = ServerSettings()
+        settings.port = port
+        server = Server.instance(settings, state_provider=state_provider)
+        server.start()
+
+        try:
+            import time
+
+            while True:
+                time.sleep(1)
+        except KeyboardInterrupt:
+            cprint("\nShutting down...", "yellow")
+            state_provider.close()
+
+
+@experiments.command()
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help="Don't write to database, only show what would be synced",
+)
+@click.option(
+    "--force",
+    is_flag=True,
+    help="Force sync even if recently synced (bypasses time throttling)",
+)
+@click.option(
+    "--no-wait",
+    is_flag=True,
+    help="Don't wait for lock, fail immediately if unavailable",
+)
+@pass_cfg
+def sync(workdir: Path, dry_run: bool, force: bool, no_wait: bool):
+    """Synchronize workspace database from disk state
+
+    Scans experiment directories and job marker files to update the workspace
+    database. Uses exclusive locking to prevent conflicts with running experiments.
+    """
+    from experimaestro.scheduler.state_sync import sync_workspace_from_disk
+    from experimaestro.scheduler.workspace import Workspace
+    from experimaestro.settings import Settings
+
+    # Get settings and workspace settings
+    settings = Settings.instance()
+    ws_settings = find_workspace(workdir=workdir)
+
+    # Create workspace instance (manages database lifecycle)
+    workspace = Workspace(
+        settings=settings,
+        workspace_settings=ws_settings,
+        sync_on_init=False,  # Don't sync on init since we're explicitly syncing
+    )
+
+    try:
+        # Enter workspace context to initialize database
+        with workspace:
+            cprint(f"Syncing workspace: {workspace.path}", "cyan")
+            if dry_run:
+                cprint("DRY RUN MODE: No changes will be written", "yellow")
+            if force:
+                cprint("FORCE MODE: Bypassing time throttling", "yellow")
+
+            # Run sync
+            sync_workspace_from_disk(
+                workspace=workspace,
+                write_mode=not dry_run,
+                force=force,
+                blocking=not no_wait,
+            )
+
+            cprint("Sync completed successfully", "green")
+
+    except RuntimeError as e:
+        cprint(f"Sync failed: {e}", "red")
+        sys.exit(1)
+    except Exception as e:
+        cprint(f"Unexpected error during sync: {e}", "red")
+        import traceback
+
+        traceback.print_exc()
+        sys.exit(1)