experimaestro 2.0.0b4__py3-none-any.whl → 2.0.0b17__py3-none-any.whl

experimaestro/connectors/local.py

@@ -1,5 +1,6 @@
 """All classes related to localhost management"""
 
+import asyncio
 import subprocess
 from typing import Optional
 from pathlib import Path, WindowsPath, PosixPath
@@ -39,6 +40,26 @@ class PsutilProcess(Process):
         )
         return code
 
+    async def aio_wait(self) -> int:
+        """Asynchronously wait for process to finish"""
+        logger.debug(
+            "Async waiting (psutil) for process with PID %s", self._process.pid
+        )
+        poll_interval = 0.01  # start at 0.01 seconds, max 10 seconds
+
+        while self._process.is_running():
+            await asyncio.sleep(poll_interval)
+            poll_interval = min(poll_interval * 1.5, 10.0)
+
+        # Process has finished, wait() returns immediately
+        code = self._process.wait()
+        logger.debug(
+            "Finished async wait (psutil) for process with PID %s: code %s",
+            self._process.pid,
+            code,
+        )
+        return code
+
     async def aio_state(self, timeout: float | None = None) -> ProcessState:
         if self._process.is_running():
             return ProcessState.RUNNING
@@ -65,6 +86,25 @@ class LocalProcess(Process):
         )
         return code
 
+    async def aio_wait(self) -> int:
+        """Asynchronously wait for process to finish"""
+        logger.debug(
+            "Async waiting (python) for process with PID %s", self._process.pid
+        )
+        poll_interval = 0.01  # start at 0.01 seconds, max 10 seconds
+
+        while self._process.poll() is None:
+            await asyncio.sleep(poll_interval)
+            poll_interval = min(poll_interval * 1.5, 10.0)
+
+        code = self._process.returncode
+        logger.debug(
+            "Finished async wait (python) for process with PID %s: %s",
+            self._process.pid,
+            code,
+        )
+        return code
+
     async def aio_state(self, timeout: float | None = None) -> ProcessState:
         code = self._process.poll()
         if code is None:
@@ -213,9 +253,9 @@ class LocalConnector(Connector):
         return LocalProcessBuilder()
 
     def resolve(self, path: Path, basepath: Path = None) -> str:
-        assert isinstance(path, PosixPath) or isinstance(
-            path, WindowsPath
-        ), f"Unrecognized path {type(path)}"
+        assert isinstance(path, PosixPath) or isinstance(path, WindowsPath), (
+            f"Unrecognized path {type(path)}"
+        )
         if not basepath:
             return str(path.absolute())
         try:

experimaestro/core/arguments.py

@@ -8,7 +8,7 @@ from typing import Annotated
 
 if TYPE_CHECKING:
     import experimaestro.core.types
-    from experimaestro.core.subparameters import ParameterGroup
+    from experimaestro.core.partial import ParameterGroup
 
 # Track deprecation warnings per module (max 10 per module)
 _deprecation_warning_counts: dict[str, int] = {}
@@ -70,7 +70,7 @@ class Argument:
             otherwise be issued. Defaults to False.
 
             groups (set[ParameterGroup], optional): Set of groups this parameter
-            belongs to. Used with subparameters to compute partial identifiers.
+            belongs to. Used with partial to compute partial identifiers.
             Defaults to None (empty set).
         """
         required = (field_or_default is None) if required is None else required
@@ -97,9 +97,9 @@ class Argument:
         self.groups = groups if groups else set()
 
         if field_or_default is not None:
-            assert (
-                self.generator is None
-            ), "generator and field_or_default are exclusive options"
+            assert self.generator is None, (
+                "generator and field_or_default are exclusive options"
+            )
             if isinstance(field_or_default, field):
                 self.ignore_generated = field_or_default.ignore_generated
                 # Allow field to override the overrides flag
@@ -125,9 +125,9 @@ class Argument:
                 self.default = field_or_default
                 self.ignore_default_in_identifier = True
 
-        assert (
-            not self.constant or self.default is not None
-        ), "Cannot be constant without default"
+        assert not self.constant or self.default is not None, (
+            "Cannot be constant without default"
+        )
 
     def __repr__(self):
         return "Param[{name}:{type}]".format(**self.__dict__)
@@ -327,19 +327,19 @@ class field:
             Useful for adding a field that changes the identifier but won't be used.
         :param overrides: If True, suppress warning when overriding parent parameter
         :param groups: List of ParameterGroup objects for partial identifiers.
-            Used with subparameters to compute identifiers that exclude certain groups.
+            Used with partial to compute identifiers that exclude certain groups.
         """
-        assert not (
-            (default is not None) and (default_factory is not None)
-        ), "default and default_factory are mutually exclusive options"
+        assert not ((default is not None) and (default_factory is not None)), (
+            "default and default_factory are mutually exclusive options"
+        )
 
-        assert not (
-            (default is not None) and (ignore_default is not None)
-        ), "default and ignore_default are mutually exclusive options"
+        assert not ((default is not None) and (ignore_default is not None)), (
+            "default and ignore_default are mutually exclusive options"
+        )
 
-        assert not (
-            (ignore_default is not None) and (default_factory is not None)
-        ), "ignore_default and default_factory are mutually exclusive options"
+        assert not ((ignore_default is not None) and (default_factory is not None)), (
+            "ignore_default and default_factory are mutually exclusive options"
+        )
 
         self.default_factory = default_factory
         self.default = default

experimaestro/core/identifier.py

@@ -9,7 +9,7 @@ from typing import Optional, TYPE_CHECKING
 from experimaestro.core.objects import Config, ConfigMixin
 
 if TYPE_CHECKING:
-    from experimaestro.core.subparameters import Subparameters
+    from experimaestro.core.partial import Partial
 
 
 class ConfigPath:
@@ -112,8 +112,8 @@ class IdentifierComputer:
         config: The configuration to compute the identifier for
         config_path: Used to track cycles when computing identifiers
         version: Hash computation version (defaults to XPM_HASH_COMPUTER env var or 2)
-        subparameters: If provided, only include parameters that are not excluded
-            by this Subparameters instance (for partial identifier computation)
+        partial: If provided, only include parameters that are not excluded
+            by this Partial instance (for partial identifier computation)
     """
 
     OBJECT_ID = b"\x00"
@@ -137,14 +137,14 @@ class IdentifierComputer:
         config_path: ConfigPath,
         *,
         version=None,
-        subparameters: "Subparameters" = None,
+        partial: "Partial" = None,
     ):
         # Hasher for parameters
         self._hasher = hashlib.sha256()
         self.config = config
         self.config_path = config_path
         self.version = version or int(os.environ.get("XPM_HASH_COMPUTER", 2))
-        self.subparameters = subparameters
+        self.partial = partial
         if hash_logger.isEnabledFor(logging.DEBUG):
             hash_logger.debug(
                 "starting hash (%s): %s", hash(str(self.config)), self.config
@@ -279,9 +279,9 @@ class IdentifierComputer:
             # Process arguments (sort by name to ensure uniqueness)
             arguments = sorted(xpmtype.arguments.values(), key=lambda a: a.name)
             for argument in arguments:
-                # Skip arguments excluded by subparameters (for partial identifiers)
-                if self.subparameters is not None:
-                    if self.subparameters.is_excluded(argument.groups):
+                # Skip arguments excluded by partial (for partial identifiers)
+                if self.partial is not None:
+                    if self.partial.is_excluded(argument.groups):
                         continue
 
                 # Ignored argument
@@ -382,7 +382,7 @@ class IdentifierComputer:
     @staticmethod
     def compute_partial(
         config: "ConfigMixin",
-        subparameters: "Subparameters",
+        partial: "Partial",
         config_path: ConfigPath | None = None,
         version=None,
     ) -> Identifier:
@@ -393,7 +393,7 @@ class IdentifierComputer:
         partial identifier (and thus the same partial directory).
 
         :param config: the configuration for which we compute the identifier
-        :param subparameters: the Subparameters instance defining which groups
+        :param partial: the Partial instance defining which groups
             to include/exclude
         :param config_path: used to track down cycles between configurations
         :param version: version for the hash computation (None for the last one)
@@ -402,7 +402,7 @@ class IdentifierComputer:
 
         with config_path.push(config):
             computer = IdentifierComputer(
-                config, config_path, version=version, subparameters=subparameters
+                config, config_path, version=version, partial=partial
             )
             computer.update(config, myself=True)
             identifier = computer.identifier()

experimaestro/core/objects/config.py

@@ -37,9 +37,10 @@ from ..context import SerializationContext, SerializedPath, SerializedPathLoader
 if TYPE_CHECKING:
     from ..callbacks import TaskEventListener
     from ..identifier import Identifier
-    from ..subparameters import Subparameters
+    from ..partial import Partial
     from experimaestro.scheduler.base import Job
     from experimaestro.scheduler.workspace import RunMode
+    from experimaestro.scheduler.transient import TransientMode
     from experimaestro.launchers import Launcher
     from experimaestro.scheduler import Workspace
 
@@ -232,7 +233,7 @@ class ConfigInformation:
         """The configuration identifier (cached when sealed)"""
 
         self._partial_identifiers: Dict[str, "Identifier"] = {}
-        """Cached partial identifiers (keyed by subparameters name)"""
+        """Cached partial identifiers (keyed by partial name)"""
 
         self._validated = False
         self._sealed = False
@@ -308,7 +309,7 @@ class ConfigInformation:
                     raise AttributeError(
                         f"Cannot set {k} to a configuration with generated values. "
                         "Here is the list of paths to help you: "
-                        f"""{', '.join(get_generated_paths(v, [k]))}"""
+                        f"""{", ".join(get_generated_paths(v, [k]))}"""
                     )
 
                 if not bypass and (
@@ -499,9 +500,9 @@ class ConfigInformation:
                                         logging.warning("Ignoring %s", k)
                                     value = argument.generator(self.context, config)
                                 else:
-                                    assert (
-                                        False
-                                    ), "generator has either two parameters (context and config), or none"
+                                    assert False, (
+                                        "generator has either two parameters (context and config), or none"
+                                    )
                             config.__xpm__.set(k, value, bypass=True)
                         else:
                             value = config.__xpm__.values.get(k)
@@ -558,15 +559,15 @@ class ConfigInformation:
             self._identifier = identifier
         return identifier
 
-    def get_partial_identifier(self, subparameters: "Subparameters") -> "Identifier":
-        """Get the partial identifier for a given subparameters instance.
+    def get_partial_identifier(self, partial: "Partial") -> "Identifier":
+        """Get the partial identifier for a given partial instance.
 
         Partial identifiers exclude certain parameter groups, allowing
         configurations that differ only in those groups to share the same
         partial identifier (and thus the same partial directory).
 
         Args:
-            subparameters: The Subparameters instance defining which groups
+            partial: The Partial instance defining which groups
                 to include/exclude.
 
         Returns:
@@ -574,11 +575,11 @@ class ConfigInformation:
         """
         from ..identifier import IdentifierComputer
 
-        name = subparameters.name
+        name = partial.name
         if name in self._partial_identifiers:
             return self._partial_identifiers[name]
 
-        identifier = IdentifierComputer.compute_partial(self.pyobject, subparameters)
+        identifier = IdentifierComputer.compute_partial(self.pyobject, partial)
 
         if self._sealed:
             self._partial_identifiers[name] = identifier
@@ -694,11 +695,17 @@ class ConfigInformation:
         run_mode=None,
         init_tasks: List["LightweightTask"] = [],
         max_retries: Optional[int] = None,
+        transient: "TransientMode" = None,
     ):
         from experimaestro.scheduler import experiment, JobContext
         from experimaestro.scheduler.workspace import RunMode
+        from experimaestro.scheduler.transient import TransientMode
         from ..callbacks import TaskEventListener
 
+        # Use default transient mode if not specified
+        if transient is None:
+            transient = TransientMode.NONE
+
         # --- Prepare the object
 
         if self.job:
@@ -718,6 +725,7 @@ class ConfigInformation:
             workspace=workspace,
             run_mode=run_mode,
             max_retries=max_retries,
+            transient=transient,
         )
 
         # Validate the object
@@ -750,6 +758,15 @@ class ConfigInformation:
         # Add predefined dependencies
         self.job.dependencies.update(self.dependencies)
 
+        # Add partial dependencies for scheduler-level locking
+        from experimaestro.core.partial_lock import PartialJobResource
+
+        for name, partial_spec in self.xpmtype._partials.items():
+            partial_path = job_context.partial_path(partial_spec, self.pyobject)
+            resource = PartialJobResource.create(partial_path)
+            partial_dep = resource.dependency(name)
+            self.job.dependencies.add(partial_dep)
+
         run_mode = (
             workspace.run_mode if run_mode is None else run_mode
         ) or RunMode.NORMAL
@@ -807,9 +824,9 @@ class ConfigInformation:
     def mark_output(self, config: "Config"):
         """Sets a dependency on the job"""
         assert not isinstance(config, Task), "Cannot set a dependency on a task"
-        assert isinstance(
-            config, ConfigMixin
-        ), "Only configurations can be marked as dependent on a task"
+        assert isinstance(config, ConfigMixin), (
+            "Only configurations can be marked as dependent on a task"
+        )
         config.__xpm__.task = self.pyobject
         return config
 
@@ -915,9 +932,9 @@ class ConfigInformation:
         for argument, value in self.xpmvalues():
             with context.push(argument.name) as var_path:
                 if argument.is_data and value is not None:
-                    assert isinstance(
-                        value, Path
-                    ), f"Data arguments should be paths (type is {type(value)})"
+                    assert isinstance(value, Path), (
+                        f"Data arguments should be paths (type is {type(value)})"
+                    )
                     value = context.serialize(var_path, value)
 
                 jsonfields[argument.name] = ConfigInformation._outputjsonvalue(
@@ -975,7 +992,7 @@ class ConfigInformation:
 
         Note: Tags are no longer stored in params.json. They are managed by the
         experiment state provider (scoped to job_id, experiment_id, run_id) and
-        also stored in experiment state.json for full experiment details.
+        also stored in experiment status.json.
 
         Arguments:
             out {io.TextIOBase} -- The output stream
@@ -1023,9 +1040,9 @@ class ConfigInformation:
         :return: a Config object, its instance or a tuple (instance, init_tasks) is return_tasks is True
         """
         # Load
-        assert not (
-            as_instance and return_tasks
-        ), "Cannot set as_instance and return_tasks to True"
+        assert not (as_instance and return_tasks), (
+            "Cannot set as_instance and return_tasks to True"
+        )
         if callable(path):
             data_loader = path
         else:
@@ -1087,7 +1104,7 @@ class ConfigInformation:
 
     @overload
     @staticmethod
-    def fromParameters(  # noqa: E704
+    def fromParameters(
         definitions: List[Dict],
         as_instance=True,
         save_directory: Optional[Path] = None,
@@ -1097,7 +1114,7 @@ class ConfigInformation:
 
     @overload
     @staticmethod
-    def fromParameters(  # noqa: E704
+    def fromParameters(
         definitions: List[Dict],
         as_instance=False,
         return_tasks=True,
@@ -1108,7 +1125,7 @@ class ConfigInformation:
 
     @overload
     @staticmethod
-    def fromParameters(  # noqa: E704
+    def fromParameters(
         definitions: List[Dict],
         as_instance=False,
         save_directory: Optional[Path] = None,
@@ -1325,9 +1342,9 @@ class ConfigInformation:
                     # Unwrap the value if needed
                     setattr(o, name, v)
 
-                    assert (
-                        getattr(o, name) is v
-                    ), f"Problem with deserialization {name} of {o.__class__}"
+                    assert getattr(o, name) is v, (
+                        f"Problem with deserialization {name} of {o.__class__}"
+                    )
                 else:
                     o.__xpm__.set(name, v, bypass=True)
 
@@ -1619,13 +1636,51 @@ class ConfigMixin:
         return f"Config[{self.__xpmtype__.identifier}]"
 
     def __str__(self):
-        params = ", ".join(
-            [f"{key}={value}" for key, value in self.__xpm__.values.items()]
-        )
-        return (
-            f"{self.__xpmtype__.value_type.__module__}."
-            f"{self.__xpmtype__.value_type.__qualname__}({params})"
-        )
+        return self.to_str()
+
+    def to_str(self, max_depth: int = 3, _visited: set = None) -> str:
+        """Convert config to string with cycle detection and depth limiting.
+
+        Args:
+            max_depth: Maximum depth for nested configs (default: 3)
+            _visited: Internal set to track visited objects for cycle detection
+        """
+        if _visited is None:
+            _visited = set()
+
+        obj_id = id(self)
+        if obj_id in _visited:
+            return "..."
+
+        _visited.add(obj_id)
+        try:
+            if max_depth <= 0:
+                return f"{self.__xpmtype__.value_type.__qualname__}(...)"
+
+            def format_value(value):
+                if isinstance(value, Config):
+                    return value.to_str(max_depth - 1, _visited)
+                elif isinstance(value, (list, tuple)):
+                    formatted = [format_value(v) for v in value]
+                    return (
+                        f"[{', '.join(formatted)}]"
+                        if isinstance(value, list)
+                        else f"({', '.join(formatted)})"
+                    )
+                elif isinstance(value, dict):
+                    items = [f"{k}: {format_value(v)}" for k, v in value.items()]
+                    return "{" + ", ".join(items) + "}"
+                return str(value)
+
+            params = ", ".join(
+                [
+                    f"{key}={format_value(value)}"
+                    for key, value in self.__xpm__.values.items()
+                ]
+            )
+            return f"{self.__xpmtype__.value_type.__module__}.{self.__xpmtype__.value_type.__qualname__}({params})"
+        finally:
+            _visited.discard(obj_id)
 
     def tag(self, name, value):
         # Capture caller's location and pass to addtag
@@ -1674,9 +1729,9 @@ class ConfigMixin:
 
             context = EmptyContext()
         else:
-            assert isinstance(
-                context, ConfigWalkContext
-            ), f"{context.__class__} is not an instance of ConfigWalkContext"
+            assert isinstance(context, ConfigWalkContext), (
+                f"{context.__class__} is not an instance of ConfigWalkContext"
+            )
 
         instance = self.__xpm__.fromConfig(context, objects=objects)  # type: ignore
         if keep:
@@ -1691,6 +1746,7 @@ class ConfigMixin:
         run_mode: "RunMode" = None,
         init_tasks: List["LightweightTask"] = [],
         max_retries: Optional[int] = None,
+        transient: "TransientMode" = None,
     ):
         """Submit this task
 
@@ -1698,6 +1754,7 @@ class ConfigMixin:
         :param launcher: The launcher, defaults to None
         :param run_mode: Run mode (if None, uses the workspace default)
         :param max_retries: Maximum number of retries for resumable tasks that timeout (default: from workspace settings or 3)
+        :param transient: Transient mode for intermediary tasks (see TransientMode)
         :return: an object object
         """
         return self.__xpm__.submit(
@@ -1706,6 +1763,7 @@ class ConfigMixin:
             run_mode=run_mode,
             init_tasks=init_tasks,
             max_retries=max_retries,
+            transient=transient,
         )
 
     def stdout(self):
@@ -1857,8 +1915,7 @@ class Config:
             # Check that value class is a subclass of the config class
             if not issubclass(value_class, cls):
                 raise TypeError(
-                    f"Value class {value_class.__name__} must be a subclass of "
-                    f"{cls.__name__}"
+                    f"Value class {value_class.__name__} must be a subclass of {cls.__name__}"
                 )
 
             # Check that value class inherits from parent value classes

experimaestro/core/objects/config_walk.py

@@ -29,14 +29,14 @@ class ConfigWalkContext:
             return self.path / self._configpath
         return self.path
 
-    def partial_path(self, subparameters, config) -> Path:
-        """Returns the partial directory path for a given subparameters instance.
+    def partial_path(self, partial, config) -> Path:
+        """Returns the partial directory path for a given partial instance.
 
         This method should be overridden in subclasses that have access to
         workspace information (like JobContext).
 
         Args:
-            subparameters: The Subparameters instance defining which groups to exclude
+            partial: The Partial instance defining which groups to exclude
             config: The configuration to compute the partial identifier for
 
         Returns:

experimaestro/core/{subparameters.py → partial.py}

@@ -1,15 +1,15 @@
-"""Subparameters for partial identifier computation.
+"""Partial identifier computation.
 
-This module provides the `subparameters` function and `Subparameters` class
-for defining parameter subsets that compute partial identifiers. This enables
-sharing directories (like checkpoints) across tasks that differ only in
-excluded parameter groups.
+This module provides the `partial` function and `Partial` class for defining
+parameter subsets that compute partial identifiers. This enables sharing
+directories (like checkpoints) across tasks that differ only in excluded
+parameter groups.
 
 Example:
     iter_group = param_group("iter")
 
     class Learn(Task):
-        checkpoints = subparameters(exclude_groups=[iter_group])
+        checkpoints = partial(exclude_groups=[iter_group])
 
         max_iter: Param[int] = field(groups=[iter_group])
         learning_rate: Param[float]
@@ -36,7 +36,7 @@ class ParameterGroup:
 
 
 def param_group(name: str) -> ParameterGroup:
-    """Create a parameter group for use with subparameters.
+    """Create a parameter group for use with partial identifiers.
 
     Parameter groups allow computing partial identifiers that exclude
     certain parameters, enabling shared directories across related tasks.
@@ -56,10 +56,10 @@ def param_group(name: str) -> ParameterGroup:
 
 
 @dataclass
-class Subparameters:
+class Partial:
     """Defines a subset of parameters for partial identifier computation.
 
-    A Subparameters instance defines which parameter groups to include or exclude
+    A Partial instance defines which parameter groups to include or exclude
     when computing a partial identifier. This enables sharing directories
     (like checkpoints) across experiments that only differ in excluded groups.
 
@@ -123,16 +123,16 @@ class Subparameters:
         return False
 
 
-def subparameters(
+def partial(
     *,
     exclude_groups: list[ParameterGroup] | None = None,
     include_groups: list[ParameterGroup] | None = None,
     exclude_no_group: bool = False,
     exclude_all: bool = False,
-) -> Subparameters:
-    """Create a subparameters specification for partial identifier computation.
+) -> Partial:
+    """Create a partial specification for partial identifier computation.
 
-    Subparameters allow tasks to share directories when they differ only
+    Partials allow tasks to share directories when they differ only
     in certain parameter groups (e.g., training hyperparameters).
 
     Example::
@@ -146,7 +146,7 @@ def subparameters(
             checkpoint: Meta[Path] = field(
                 default_factory=PathGenerator(
                     "model.pt",
-                    subparameters=subparameters(exclude=[training_group])
+                    partial=partial(exclude_groups=[training_group])
                 )
             )
 
@@ -154,9 +154,9 @@ def subparameters(
     :param include_groups: Parameter groups to always include (overrides exclusion)
     :param exclude_no_group: If True, exclude parameters with no group assigned
     :param exclude_all: If True, exclude all parameters by default
-    :return: A Subparameters object
+    :return: A Partial object
     """
-    return Subparameters(
+    return Partial(
         exclude_groups=set(exclude_groups or []),
         include_groups=set(include_groups or []),
         exclude_no_group=exclude_no_group,