experimaestro 1.6.1__py3-none-any.whl → 1.15.2__py3-none-any.whl

experimaestro/core/objects/config_utils.py

@@ -0,0 +1,58 @@
+from contextlib import contextmanager
+from typing import Any, Dict, Set
+
+
+def getqualattr(module, qualname):
+    """Get a qualified attributed value"""
+    cls = module
+    for part in qualname.split("."):
+        cls = getattr(cls, part)
+    return cls
+
+
+@contextmanager
+def add_to_path(p):
+    """Temporarily add a path to sys.path"""
+    import sys
+
+    old_path = sys.path
+    sys.path = sys.path[:]
+    sys.path.insert(0, p)
+    try:
+        yield
+    finally:
+        sys.path = old_path
+
+
+class ObjectStore:
+    def __init__(self):
+        self.store: Dict[int, Any] = {}
+        self.constructed: Set[int] = set()
+
+    def set_constructed(self, identifier: int):
+        self.constructed.add(identifier)
+
+    def is_constructed(self, identifier: int):
+        return identifier in self.constructed
+
+    def retrieve(self, identifier: int):
+        return self.store.get(identifier, None)
+
+    def add_stub(self, identifier: int, stub: Any):
+        self.store[identifier] = stub
+
+
+class SealedError(Exception):
+    """Exception when trying to modify a sealed configuration"""
+
+    pass
+
+
+class TaggedValue:
+    def __init__(self, value):
+        self.value = value
+
+
+class classproperty(property):
+    def __get__(self, owner_self, owner_cls):
+        return self.fget(owner_cls)

experimaestro/core/objects/config_walk.py

@@ -0,0 +1,151 @@
+from pathlib import Path
+from enum import Enum
+from typing import (
+    Any,
+    Dict,
+    Tuple,
+)
+from contextlib import contextmanager
+
+
+class ConfigWalkContext:
+    """Context when generating values in configurations"""
+
+    @property
+    def path(self):
+        """Returns the path of the job directory"""
+        raise NotImplementedError()
+
+    def __init__(self):
+        self._configpath = None
+
+    @property
+    def task(self):
+        return None
+
+    def currentpath(self) -> Path:
+        """Returns the configuration folder"""
+        if self._configpath:
+            return self.path / self._configpath
+        return self.path
+
+    @contextmanager
+    def push(self, key: str):
+        """Push a new key to contextualize paths"""
+        p = self._configpath
+        try:
+            self._configpath = (Path("out") if p is None else p) / key
+            yield key
+        finally:
+            self._configpath = p
+
+
+class ConfigWalk:
+    """Allows to perform an operation on all nested configurations"""
+
+    def __init__(self, context: ConfigWalkContext = None, recurse_task=False):
+        """
+
+        :param recurse_task: Recurse into linked tasks
+        :param context: The context, by default only tracks the position in the
+            config tree
+        """
+        self.recurse_task = recurse_task
+        self.context = ConfigWalkContext() if context is None else context
+
+        # Stores already visited nodes
+        self.visited = {}
+
+    def preprocess(self, config) -> Tuple[bool, Any]:
+        """Returns a tuple boolean/value
+
+        The boolean value is used to stop the processing if False.
+        The value is returned
+        """
+        return True, None
+
+    def postprocess(self, stub, config, values: Dict[str, Any]):
+        return stub
+
+    def list(self, i: int):
+        return self.context.push(str(i))
+
+    def map(self, k: str):
+        """Provides a path context when processing a tree"""
+        return self.context.push(k)
+
+    def stub(self, config):
+        return config
+
+    def __call__(self, x):
+        from experimaestro.core.objects import Config
+        from experimaestro.core.objects import ConfigInformation  # noqa: F401
+
+        if isinstance(x, Config):
+            info = x.__xpm__  # type: ConfigInformation
+
+            # Avoid loops
+            xid = id(x)
+            if xid in self.visited:
+                return self.visited[xid]
+
+            # Get a stub
+            stub = self.stub(x)
+            self.visited[xid] = stub
+
+            # Pre-process
+            flag, value = self.preprocess(x)
+
+            if not flag:
+                # Stop processing and returns value
+                return value
+
+            # Process all the arguments
+            result = {}
+            for arg, v in info.xpmvalues():
+                if v is not None:
+                    with self.map(arg.name):
+                        result[arg.name] = self(v)
+                else:
+                    result[arg.name] = None
+
+            # Deals with init tasks
+            if info.init_tasks:
+                with self.map("__init_tasks__"):
+                    self(info.init_tasks)
+
+            # Process task if different
+            if (
+                x.__xpm__.task is not None
+                and self.recurse_task
+                and x.__xpm__.task is not x
+            ):
+                with self.map("__task__"):
+                    self(x.__xpm__.task)
+
+            processed = self.postprocess(stub, x, result)
+            self.visited[xid] = processed
+            return processed
+
+        if x is None:
+            return None
+
+        if isinstance(x, list):
+            result = []
+            for i, sv in enumerate(x):
+                with self.list(i):
+                    result.append(self(sv))
+            return result
+
+        if isinstance(x, dict):
+            result = {}
+            for key, value in x.items():
+                assert isinstance(key, (str, float, int))
+                with self.map(key):
+                    result[key] = self(value)
+            return result
+
+        if isinstance(x, (float, int, str, Path, Enum)):
+            return x
+
+        raise NotImplementedError(f"Cannot handle a value of type {type(x)}")

experimaestro/core/objects.pyi

@@ -1,4 +1,5 @@
 from abc import ABC
+from attrs import define
 import typing_extensions
 
 from experimaestro.core.types import ObjectType
@@ -40,40 +41,8 @@ from typing_extensions import Self
 
 TConfig = TypeVar("TConfig", bound="Config")
 
-class Identifier:
-    main: Incomplete
-    sub: Incomplete
-    def __init__(self, main: bytes, sub: Optional[bytes] = ...) -> None: ...
-    def all(self): ...
-    def state_dict(self): ...
-    @staticmethod
-    def from_state_dict(data: Union[Dict[str, str], str]): ...
-
-def is_ignored(value): ...
-def remove_meta(value): ...
-
 class ObjectStore: ...
 
-class HashComputer:
-    OBJECT_ID: bytes
-    INT_ID: bytes
-    FLOAT_ID: bytes
-    STR_ID: bytes
-    PATH_ID: bytes
-    NAME_ID: bytes
-    NONE_ID: bytes
-    LIST_ID: bytes
-    TASK_ID: bytes
-    DICT_ID: bytes
-    ENUM_ID: bytes
-    def __init__(self) -> None: ...
-    def identifier(self) -> Identifier: ...
-    def update(self, value, myself: bool = ...): ...
-
-def updatedependencies(
-    dependencies, value: Config, path: List[str], taskids: Set[int]
-): ...
-
 class TaggedValue:
     value: Incomplete
     def __init__(self, value) -> None: ...
@@ -108,6 +77,10 @@ class ConfigWalk(ConfigProcessing):
     def map(self, k: str): ...
 
 def getqualattr(module, qualname): ...
+@define(frozen=True)
+class WatchedOutput:
+    config: "Config"
+    method_name: str
 
 class ConfigInformation:
     LOADING: bool
@@ -116,7 +89,8 @@ class ConfigInformation:
     values: Dict[str, Any]
     job: Job
     dependencies: Incomplete
-    def __init__(self, pyobject: TypeConfig) -> None: ...
+    watched_outputs: List[WatchedOutput]
+    def __init__(self, pyobject: ConfigMixin) -> None: ...
     def set_meta(self, value: Optional[bool]): ...
     @property
     def meta(self): ...
@@ -156,7 +130,7 @@ class ConfigInformation:
         definitions: List[Dict],
         as_instance: bool = ...,
         save_directory: Optional[Path] = ...,
-    ) -> TypeConfig: ...
+    ) -> ConfigMixin: ...
     @overload
     @staticmethod
     def fromParameters(
@@ -177,7 +151,7 @@ class ConfigInformation:
 def clone(v): ...
 def cache(fn, name: str): ...
 
-class TypeConfig:
+class ConfigMixin:
     __xpmtype__: ObjectType
     __xpm__: Incomplete
     def __init__(self, **kwargs) -> None: ...
@@ -194,7 +168,7 @@ class TypeConfig:
         *,
         workspace: Incomplete | None = ...,
         launcher: Incomplete | None = ...,
-        run_mode: RunMode = ...
+        run_mode: RunMode = ...,
     ): ...
     def stdout(self): ...
     def stderr(self): ...
@@ -211,8 +185,8 @@ class Config:
     __use_xpmobject__: ClassVar[bool]
 
     XPMValue: Type[Self]
-    XPMConfig: Union[Type[Self], Type[TypeConfig[Self]]]
-    C: Union[Type[Self], Type[TypeConfig[Self]]]
+    XPMConfig: Union[Type[Self], Type[ConfigMixin[Self]]]
+    C: Union[Type[Self], Type[ConfigMixin[Self]]]
 
     @classmethod
     def __getxpmtype__(cls) -> ObjectType: ...
@@ -221,17 +195,13 @@ class Config:
     def __post_init__(self) -> None: ...
     def __json__(self): ...
     def __identifier__(self) -> Identifier: ...
-    def add_pretasks(self, *tasks: "LightweightTask"): ...
-    def add_pretasks_from(self, configs: "Config"): ...
     def copy_dependencies(self, other: "Config"): ...
-    @property
-    def pre_tasks(self) -> List["LightweightTask"]: ...
 
 class LightweightTask(Config):
     def execute(self) -> None: ...
 
 class Task(LightweightTask):
-    __tags__: Dict[str, str]
+    # __tags__: Dict[str, str]
 
     def submit(
         self,
@@ -239,13 +209,13 @@ class Task(LightweightTask):
         workspace: Incomplete | None = ...,
         launcher: Incomplete | None = ...,
         run_mode: RunMode = ...,
-        init_tasks: List["LightweightTask"] = []
+        init_tasks: List["LightweightTask"] = [],
     ): ...
     def task_outputs(self, dep: Callable[[Config], None]) -> Any: ...
 
 def copyconfig(config_or_output: TConfig, **kwargs) -> TConfig: ...
 def setmeta(config: TConfig, flag: bool) -> TConfig: ...
 
-class TypeConfig(Generic[T]):
+class ConfigMixin(Generic[T]):
     def __validate__(self):
         pass

experimaestro/core/serialization.py

@@ -1,6 +1,6 @@
 import json
 from pathlib import Path
-from typing import Any, Dict, List, Tuple, Union, TYPE_CHECKING
+from typing import Any, Dict, List, Optional, Tuple, Union, TYPE_CHECKING
 from experimaestro.core.context import (
     SerializationContext,
     SerializedPath,
@@ -17,7 +17,7 @@ def json_object(context: SerializationContext, value: Any, objects=[]):
 
     if isinstance(value, Config):
         value.__xpm__.__get_objects__(objects, context)
-    elif isinstance(value, list):
+    elif isinstance(value, (list, tuple)):
         for el in value:
             ConfigInformation.__collect_objects__(el, objects, context)
     elif isinstance(value, dict):
@@ -36,28 +36,38 @@ def state_dict(context: SerializationContext, obj: Any):
     :param context: The serialization context
     :param obj: the object to serialize
     """
-    objects = []
+    objects: list[Any] = []
     data = json_object(context, obj, objects)
     return {"objects": objects, "data": data}
 
 
-def save(obj: Any, save_directory: Path):
+def save_definition(obj: Any, context: SerializationContext, path: Path):
+    data = state_dict(context, obj)
+    with path.open("wt") as out:
+        json.dump(data, out)
+
+
+def save(obj: Any, save_directory: Optional[Path]):
     """Saves an object into a disk file
 
+    The serialization process also stores in the given folder the different
+    files or folders that are registered as Path parameters (or
+    meta-parameters).
+
     :param save_directory: The directory in which the object and its data will
         be saved (by default, the object is saved in "definition.json")
     """
     context = SerializationContext(save_directory=save_directory)
-
-    data = state_dict(context, obj)
-    with (save_directory / "definition.json").open("wt") as out:
-        json.dump(data, out)
+    save_definition(
+        obj, context, save_directory / "definition.json" if save_directory else None
+    )
 
 
 def get_data_loader(path: Union[str, Path, SerializedPathLoader]):
     if path is None:
 
-        def data_loader():
+        def data_loader(_: Union[str, Path, SerializedPathLoader]):
+            # Just raise an exception
             raise RuntimeError("No serialization path was given")
 
         return data_loader
@@ -126,3 +136,47 @@ def from_task_dir(
     content["data"] = {"type": "python", "value": content["objects"][-1]["id"]}
 
     return from_state_dict(content, as_instance=as_instance)
+
+
+def serialize(
+    obj: Any, save_directory: Path, *, init_tasks: list["LightweightTask"] = []
+):
+    """Saves an object into a disk file, including initialization tasks
+
+    The serialization process also stores in the given folder the different
+    files or folders that are registered as Path parameters (or
+    meta-parameters).
+
+    :param save_directory: The directory in which the object and its data will
+        be saved (by default, the object is saved in "definition.json")
+    :param init_tasks: The optional
+    """
+    context = SerializationContext(save_directory=save_directory)
+    save_definition((obj, init_tasks), context, save_directory / "definition.json")
+
+
+def deserialize(
+    path: Union[str, Path, SerializedPathLoader],
+    as_instance: bool = False,
+) -> tuple[Any, List["LightweightTask"]] | Any:
+    """Load data from disk, and initialize the object
+
+    :param path: A directory or a function that transforms relative file path
+        into absolute ones
+    :param as_instance: returns instances instead of configuration objects
+    :returns: either the object (as_instance is true), or a tuple
+    """
+    data_loader = get_data_loader(path)
+
+    with data_loader("definition.json").open("rt") as fh:
+        content = json.load(fh)
+
+    object, init_tasks = from_state_dict(content, data_loader, as_instance=as_instance)
+
+    if as_instance:
+        for init_task in init_tasks:
+            init_task.execute()
+
+        return object
+
+    return object, init_tasks

experimaestro/core/serializers.py

@@ -1,10 +1,8 @@
-from typing import List, TypeVar, Callable, Any
-from pathlib import Path
+from typing import List, TypeVar
 from experimaestro import Param
 
 from .objects import Config, LightweightTask
 from .arguments import DataPath
-from experimaestro import copyconfig
 
 
 class SerializationLWTask(LightweightTask):
@@ -39,8 +37,3 @@ class PathSerializationLWTask(SerializationLWTask):
 
     path: DataPath
     """Path containing the data"""
-
-    @classmethod
-    def construct(cls, value: T, path: Path, dep: Callable[[Config], Any]) -> T:
-        value = copyconfig(value)
-        return value.add_pretasks(dep(cls(value=value, path=path)))

experimaestro/core/types.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 import inspect
 import sys
-from typing import Set, Union, Dict, Iterator, List, get_args, get_origin
+from typing import Set, TypeVar, Union, Dict, Iterator, List, get_args, get_origin
 from collections import ChainMap
 from pathlib import Path
 import typing
@@ -26,6 +26,7 @@ if typing.TYPE_CHECKING:
 
 class Identifier:
     def __init__(self, name: str):
+        assert isinstance(name, str)
         self.name = name
 
     def __hash__(self):
@@ -126,10 +127,16 @@ class Type:
         if t:
             return DictType(Type.fromType(t[0]), Type.fromType(t[1]))
 
-        # Takes care of generics
+        if union_t := typingutils.get_union(key):
+            return UnionType([Type.fromType(t) for t in union_t])
+
+        # Takes care of generics, like List[int], not List
         if get_origin(key):
             return GenericType(key)
 
+        if isinstance(key, TypeVar):
+            return TypeVarType(key)
+
         raise Exception("No type found for %s", key)
 
 
@@ -210,7 +217,7 @@ class ObjectType(Type):
         identifier: Union[str, Identifier] = None,
     ):
         """Creates a type"""
-        from .objects import Config, TypeConfig
+        from .objects import Config, ConfigMixin
 
         # Task related attributes
         self.taskcommandfactory = None
@@ -223,7 +230,7 @@ class ObjectType(Type):
             __xpmid__ = getattr(tp, "__xpmid__")
             if isinstance(__xpmid__, Identifier):
                 identifier = __xpmid__
-            if inspect.ismethod(__xpmid__):
+            elif inspect.ismethod(__xpmid__):
                 identifier = Identifier(__xpmid__())
             elif "__xpmid__" in tp.__dict__:
                 identifier = Identifier(__xpmid__)
@@ -264,7 +271,7 @@ class ObjectType(Type):
             s.__getxpmtype__().configtype
             for s in tp.__bases__
             if issubclass(s, Config) and (s is not Config)
-        ) or (TypeConfig,)
+        ) or (ConfigMixin,)
 
         *tp_qual, tp_name = self.basetype.__qualname__.split(".")
         self.configtype = type(
@@ -576,7 +583,7 @@ class PathType(Type):
             return Path(value.get("$value"))
 
         if not isinstance(value, (str, Path)):
-            raise TypeError("value is not a pathlike value")
+            raise TypeError(f"value is not a pathlike value ({type(value)})")
         return Path(value)
 
     @property
@@ -593,6 +600,23 @@ class AnyType(Type):
         return value
 
 
+class TypeVarType(Type):
+    def __init__(self, typevar: TypeVar):
+        self.typevar = typevar
+
+    def name(self):
+        return str(self.typevar)
+
+    def validate(self, value):
+        return value
+
+    def __str__(self):
+        return f"TypeVar({self.typevar})"
+
+    def __repr__(self):
+        return f"TypeVar({self.typevar})"
+
+
 Any = AnyType()
 
 
@@ -631,6 +655,32 @@ class EnumType(Type):
         return f"Enum({self.type})"
 
 
+class UnionType(Type):
+    def __init__(self, types: List[Type]):
+        self.types = types
+
+    def name(self):
+        return "Union[" + ", ".join(t.name() for t in self.types) + "]"
+
+    def __str__(self):
+        return "[" + " | ".join(t.name() for t in self.types) + " ]"
+
+    def __repr__(self):
+        return str(self)
+
+    def validate(self, value):
+        for subtype in self.types:
+            try:
+                return subtype.validate(value)
+            except ValueError:
+                pass
+            except TypeError:
+                pass
+
+        if not isinstance(value, dict):
+            raise ValueError(f"value is not within the types {self}")
+
+
 class DictType(Type):
     def __init__(self, keytype: Type, valuetype: Type):
         self.keytype = keytype
@@ -668,6 +718,10 @@ class GenericType(Type):
     def __repr__(self):
         return repr(self.type)
 
+    def identifier(self):
+        """Returns the identifier of the type"""
+        return Identifier(f"{self.origin}.{self.type}")
+
     def validate(self, value):
         # Now, let's check generics...
         mros = typingutils.generic_mro(type(value))
@@ -675,6 +729,7 @@ class GenericType(Type):
             (mro for mro in mros if (get_origin(mro) or mro) is self.origin), None
         )
         target = get_origin(self.type) or self.type
+
         if matching is None:
             raise ValueError(
                 f"{type(value)} is not of type {target} ({type(value).__mro__})"