experimaestro 1.5.1__py3-none-any.whl → 2.0.0a8__py3-none-any.whl

experimaestro/core/callbacks.py

@@ -0,0 +1,52 @@
+from collections import defaultdict
+import threading
+from typing import Callable, ClassVar, Optional
+from experimaestro.core.objects import ConfigInformation
+from experimaestro.scheduler import Listener, Job, JobState, experiment
+
+
+class TaskEventListener(Listener):
+    INSTANCE: ClassVar[Optional["TaskEventListener"]] = None
+    """The general instance"""
+
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.experiments: set[int] = set()
+
+        self._on_completed: defaultdict[int, Callable] = defaultdict(list)
+
+    @staticmethod
+    def connect(xp: experiment):
+        _self = TaskEventListener.instance()
+        with _self.lock:
+            if id(xp) not in _self.experiments:
+                _self.experiments.add(id(xp))
+                xp.scheduler.addlistener(_self)
+
+    @staticmethod
+    def instance():
+        if TaskEventListener.INSTANCE is None:
+            TaskEventListener.INSTANCE = TaskEventListener()
+
+        return TaskEventListener.INSTANCE
+
+    def job_state(self, job: Job):
+        if job.state == JobState.DONE:
+            with self.lock:
+                for callback in self._on_completed.get(id(job.config.__xpm__), []):
+                    callback()
+
+    @staticmethod
+    def on_completed(
+        config_information: ConfigInformation, callback: Callable[[], None]
+    ):
+        instance = TaskEventListener.instance()
+
+        with instance.lock:
+            instance._on_completed[id(config_information)].append(callback)
+
+            if (
+                config_information.job is not None
+                and config_information.job == JobState.DONE
+            ):
+                callback()

experimaestro/core/context.py

@@ -1,22 +1,21 @@
 from contextlib import contextmanager
 from pathlib import Path
+
+try:
+    from pathlib import UnsupportedOperation
+except ImportError:
+    UnsupportedOperation = OSError
 import shutil
 from typing import List, Optional, Protocol, Set, Union
 import os
-import sys
-
-has_hardlink_to = sys.version_info.major == 3 and sys.version_info.minor >= 10
 
 
 def shallow_copy(src_path: Path, dest_path: Path):
     """Copy a directory or file, trying to use hard links if possible"""
     if src_path.is_file():
         try:
-            if has_hardlink_to:
-                dest_path.hardlink_to(src_path)
-            else:
-                dest_path.link_to(src_path)
-        except OSError:
+            dest_path.hardlink_to(src_path)
+        except (NotImplementedError, UnsupportedOperation, OSError):
             shutil.copy(src_path, dest_path)
     else:
         if dest_path.exists():
@@ -74,7 +73,7 @@ class SerializationContext:
 
 
 class SerializedPathLoader(Protocol):
-    def __call__(path: Union[Path, str, SerializedPath]) -> Path:
+    def __call__(self, path: Union[Path, str, SerializedPath]) -> Path:
         """Get a filesystem path from a relative path
 
         :param path: The relative path

experimaestro/core/identifier.py

@@ -0,0 +1,310 @@
+from contextlib import contextmanager
+from enum import Enum
+from functools import cached_property
+import hashlib
+import logging
+import os
+import struct
+from typing import Optional
+from experimaestro.core.objects import Config, ConfigMixin
+
+
+class ConfigPath:
+    """Used to keep track of cycles when computing a hash"""
+
+    def __init__(self):
+        self.loops: list[bool] = []
+        """Indicates whether a loop was detected up to this node"""
+
+        self.config2index = {}
+        """Associate an index in the list with a configuration"""
+
+    def detect_loop(self, config) -> Optional[int]:
+        """If there is a loop, return the relative index and update the path"""
+        index = self.config2index.get(id(config), None)
+        if index is not None:
+            for i in range(index, self.depth):
+                self.loops[i] = True
+            return self.depth - index
+        return None
+
+    def has_loop(self):
+        return self.loops[-1]
+
+    @property
+    def depth(self):
+        return len(self.loops)
+
+    @contextmanager
+    def push(self, config):
+        config_id = id(config)
+        assert config_id not in self.config2index
+
+        self.config2index[config_id] = self.depth
+        self.loops.append(False)
+
+        try:
+            yield
+        finally:
+            self.loops.pop()
+            del self.config2index[config_id]
+
+
+hash_logger = logging.getLogger("xpm.hash")
+
+
+def is_ignored(value):
+    """Returns True if the value should be ignored by itself"""
+    return value is not None and isinstance(value, Config) and (value.__xpm__.meta)
+
+
+def remove_meta(value):
+    """Cleanup a dict/list by removing ignored values"""
+    if isinstance(value, list):
+        return [el for el in value if not is_ignored(el)]
+    if isinstance(value, dict):
+        return {key: value for key, value in value.items() if not is_ignored(value)}
+    return value
+
+
+class Identifier:
+    def __init__(self, main: bytes):
+        self.main = main
+        self.has_loops = False
+
+    @cached_property
+    def all(self):
+        """Returns the overall identifier"""
+        return self.main
+
+    def __hash__(self) -> int:
+        return hash(self.main)
+
+    def state_dict(self):
+        return self.main.hex()
+
+    def __eq__(self, other: object):
+        if not isinstance(other, Identifier):
+            return False
+        return self.main == other.main
+
+    @staticmethod
+    def from_state_dict(data: dict[str, str] | str):
+        if isinstance(data, str):
+            return Identifier(bytes.fromhex(data))
+
+        return Identifier(bytes.fromhex(data["main"]))
+
+    def __repr__(self):
+        return self.main.hex()
+
+
+class IdentifierComputer:
+    """This class is in charge of computing a config/task identifier"""
+
+    OBJECT_ID = b"\x00"
+    INT_ID = b"\x01"
+    FLOAT_ID = b"\x02"
+    STR_ID = b"\x03"
+    PATH_ID = b"\x04"
+    NAME_ID = b"\x05"
+    NONE_ID = b"\x06"
+    LIST_ID = b"\x07"
+    TASK_ID = b"\x08"
+    DICT_ID = b"\x09"
+    ENUM_ID = b"\x0a"
+    CYCLE_REFERENCE = b"\x0b"
+    INIT_TASKS = b"\x0c"
+
+    def __init__(self, config: "ConfigMixin", config_path: ConfigPath, *, version=None):
+        # Hasher for parameters
+        self._hasher = hashlib.sha256()
+        self.config = config
+        self.config_path = config_path
+        self.version = version or int(os.environ.get("XPM_HASH_COMPUTER", 2))
+        if hash_logger.isEnabledFor(logging.DEBUG):
+            hash_logger.debug(
+                "starting hash (%s): %s", hash(str(self.config)), self.config
+            )
+
+    def identifier(self) -> Identifier:
+        main = self._hasher.digest()
+        if hash_logger.isEnabledFor(logging.DEBUG):
+            hash_logger.debug("hash (%s): %s", hash(str(self.config)), str(main))
+        return Identifier(main)
+
+    def _hashupdate(self, bytes: bytes):
+        """Update the hash computers with some bytes"""
+        if hash_logger.isEnabledFor(logging.DEBUG):
+            hash_logger.debug(
+                "updating hash (%s): %s", hash(str(self.config)), str(bytes)
+            )
+        self._hasher.update(bytes)
+
+    def update(self, value, *, myself=False):  # noqa: C901
+        """Update the hash
+
+        :param value: The value to add to the hash
+        :param myself: True if the value is the configuration for which we wish
+            to compute the identifier, defaults to False
+        :raises NotImplementedError: If the value cannot be processed
+        """
+        if value is None:
+            self._hashupdate(IdentifierComputer.NONE_ID)
+        elif isinstance(value, float):
+            self._hashupdate(IdentifierComputer.FLOAT_ID)
+            self._hashupdate(struct.pack("!d", value))
+        elif isinstance(value, int):
+            self._hashupdate(IdentifierComputer.INT_ID)
+            self._hashupdate(struct.pack("!q", value))
+        elif isinstance(value, str):
+            self._hashupdate(IdentifierComputer.STR_ID)
+            self._hashupdate(value.encode("utf-8"))
+        elif isinstance(value, list):
+            values = [el for el in value if not is_ignored(el)]
+            self._hashupdate(IdentifierComputer.LIST_ID)
+            self._hashupdate(struct.pack("!d", len(values)))
+            for x in values:
+                self.update(x)
+        elif isinstance(value, Enum):
+            self._hashupdate(IdentifierComputer.ENUM_ID)
+            k = value.__class__
+            self._hashupdate(
+                f"{k.__module__}.{k.__qualname__}:{value.name}".encode("utf-8"),
+            )
+        elif isinstance(value, dict):
+            self._hashupdate(IdentifierComputer.DICT_ID)
+            items = [
+                (key, value) for key, value in value.items() if not is_ignored(value)
+            ]
+            items.sort(key=lambda x: x[0])
+            for key, value in items:
+                self.update(key)
+                self.update(value)
+
+        # Handles configurations
+        elif isinstance(value, ConfigMixin):
+            # Encodes the identifier
+            self._hashupdate(IdentifierComputer.OBJECT_ID)
+
+            # If we encode another config, then
+            if not myself:
+                if loop_ix := self.config_path.detect_loop(value):
+                    # Loop detected: use cycle reference
+                    self._hashupdate(IdentifierComputer.CYCLE_REFERENCE)
+                    self._hashupdate(struct.pack("!q", loop_ix))
+
+                else:
+                    # Just use the object identifier
+                    value_id = IdentifierComputer.compute(
+                        value, version=self.version, config_path=self.config_path
+                    )
+                    self._hashupdate(value_id.all)
+
+                # And that's it!
+                return
+
+            # Process tasks
+            if value.__xpm__.task is not None and (value.__xpm__.task is not value):
+                hash_logger.debug("Computing hash for task %s", value.__xpm__.task)
+                self._hashupdate(IdentifierComputer.TASK_ID)
+                self.update(value.__xpm__.task)
+
+            xpmtype = value.__xpmtype__
+            self._hashupdate(xpmtype.identifier.name.encode("utf-8"))
+
+            # Process arguments (sort by name to ensure uniqueness)
+            arguments = sorted(xpmtype.arguments.values(), key=lambda a: a.name)
+            for argument in arguments:
+                # Ignored argument
+                if argument.ignored:
+                    argvalue = value.__xpm__.values.get(argument.name, None)
+
+                    # ... unless meta is set to false
+                    if (
+                        argvalue is None
+                        or not isinstance(argvalue, Config)
+                        or (argvalue.__xpm__.meta is not False)
+                    ):
+                        continue
+
+                if argument.generator:
+                    continue
+
+                # Argument value
+                # Skip if the argument is not a constant, and
+                # - optional argument: both value and default are None
+                # - the argument value is equal to the default value
+                try:
+                    argvalue = getattr(value, argument.name, None)
+                except KeyError:
+                    logging.warning(
+                        "Parameter %s has not been set in %s created at %s",
+                        argument.name,
+                        value,
+                        value.__xpm__._initinfo,
+                    )
+                    raise
+                if not argument.constant and (
+                    (
+                        not argument.required
+                        and argument.default is None
+                        and argvalue is None
+                    )
+                    or (
+                        argument.default is not None
+                        and argument.default == remove_meta(argvalue)
+                    )
+                ):
+                    # No update if same value (and not constant)
+                    continue
+
+                if (
+                    argvalue is not None
+                    and isinstance(argvalue, Config)
+                    and argvalue.__xpm__.meta
+                ):
+                    continue
+
+                # Hash name
+                self.update(argument.name)
+
+                # Hash value
+                self._hashupdate(IdentifierComputer.NAME_ID)
+                self.update(argvalue)
+
+            # Add init tasks
+            if value.__xpm__.init_tasks:
+                self._hashupdate(IdentifierComputer.INIT_TASKS)
+                for init_task in value.__xpm__.init_tasks:
+                    self.update(init_task)
+        else:
+            raise NotImplementedError("Cannot compute hash of type %s" % type(value))
+
+    @staticmethod
+    def compute(
+        config: "ConfigMixin", config_path: ConfigPath | None = None, version=None
+    ) -> Identifier:
+        """Compute the identifier for a configuration
+
+        :param config: the configuration for which we compute the identifier
+        :param config_path: used to track down cycles between configurations
+        :param version: version for the hash computation (None for the last one)
+        """
+
+        # Try to use the cached value first
+        # (if there are no loops)
+        if config.__xpm__._sealed:
+            identifier = config.__xpm__._identifier
+            if identifier is not None and not identifier.has_loops:
+                return identifier
+
+        config_path = config_path or ConfigPath()
+
+        with config_path.push(config):
+            self = IdentifierComputer(config, config_path, version=version)
+            self.update(config, myself=True)
+            identifier = self.identifier()
+            identifier.has_loops = config_path.has_loop()
+
+        return identifier

experimaestro/core/objects/__init__.py

@@ -0,0 +1,44 @@
+from .config_walk import ConfigWalkContext, ConfigWalk
+from .config import (
+    ConfigMixin,
+    Config,
+    ConfigInformation,
+    Task,
+    LightweightTask,
+    WatchedOutput,
+    DependentMarker,
+    copyconfig,
+    setmeta,
+    cache,
+    logger,
+)
+
+from .config_utils import (
+    getqualattr,
+    add_to_path,
+    ObjectStore,
+    SealedError,
+    TaggedValue,
+)
+
+
+__all__ = [
+    "ConfigMixin",
+    "Config",
+    "ConfigInformation",
+    "ConfigWalkContext",
+    "ConfigWalk",
+    "Task",
+    "LightweightTask",
+    "ObjectStore",
+    "WatchedOutput",
+    "SealedError",
+    "DependentMarker",
+    "TaggedValue",
+    "getqualattr",
+    "copyconfig",
+    "setmeta",
+    "cache",
+    "add_to_path",
+    "logger",
+]