experimaestro 1.4.2__tar.gz → 1.5.0__tar.gz

{experimaestro-1.4.2 → experimaestro-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: experimaestro
-Version: 1.4.2
+Version: 1.5.0
 Summary: "Experimaestro is a computer science experiment manager"
 Home-page: https://github.com/experimaestro/experimaestro-python
 License: GPL-3
@@ -23,7 +23,6 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: arpeggio (>=2,<3)
 Requires-Dist: attrs (>=23.1.0,<24.0.0)
-Requires-Dist: cached-property (>=1.5.2,<2.0.0) ; python_version < "3.9"
 Requires-Dist: click (>=8)
 Requires-Dist: decorator (>=5,<6)
 Requires-Dist: docstring-parser (>=0.15,<0.16)
@@ -46,7 +45,7 @@ Requires-Dist: rpyc (>=5,<6)
 Requires-Dist: sortedcontainers (>=2.4,<3.0)
 Requires-Dist: termcolor (>=2.3)
 Requires-Dist: tqdm (>=4.66.1,<5.0.0)
-Requires-Dist: typing-extensions (>=4.2) ; python_version < "3.11"
+Requires-Dist: typing-extensions (>=4.2) ; python_version < "3.12"
 Requires-Dist: watchdog (>=2,<3)
 Project-URL: Documentation, https://experimaestro-python.readthedocs.io/
 Project-URL: Repository, https://github.com/experimaestro/experimaestro-python

{experimaestro-1.4.2 → experimaestro-1.5.0}/pyproject.toml

@@ -19,7 +19,7 @@ include = [
     "src/experimaestro/sphinx/static/experimaestro.css",
     "src/experimaestro/mkdocs/style.css"
 ]
-version = "1.4.2"
+version = "1.5.0"
 repository = "https://github.com/experimaestro/experimaestro-python"
 documentation = "https://experimaestro-python.readthedocs.io/"
 
@@ -38,8 +38,7 @@ build-backend = "poetry_dynamic_versioning.backend"
 python = "^3.8"
 click = ">=8"
 omegaconf = "^2.3"
-typing-extensions = {version = ">=4.2", markers = "python_version < \"3.11\""}
-cached-property = {markers = "python_version < \"3.9\"", version = "^1.5.2"}
+typing-extensions = {version = ">=4.2", markers = "python_version < \"3.12\""}
 attrs = "^23.1.0"
 fasteners = "^0.19"
 pyyaml = "^6.0.1"
@@ -119,12 +118,12 @@ max-line-length = "88"
 extend-ignore = "E203"
 
 [tool.mypy]
-python_version = "3.8"
+python_version = "3.9"
 warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.4.2"
+version = "1.5.0"
 changelog_start_rev = "0.15.0"
 tag_format = "v$version"
 update_changelog_on_bump = true

{experimaestro-1.4.2 → experimaestro-1.5.0}/src/experimaestro/connectors/ssh.py

@@ -1,7 +1,8 @@
 from dataclasses import dataclass
-from pathlib import Path, PurePosixPath
+from pathlib import Path, _posix_flavour
 import io
 import os
+import re
 from experimaestro.launcherfinder import LauncherRegistry, YAMLDataClass
 from fabric import Connection
 from invoke import Promise
@@ -21,7 +22,7 @@ from experimaestro.tokens import Token
 # Might be wise to switch to https://github.com/marian-code/ssh-utilities
 
 
-class SshPath(Path, PurePosixPath):
+class SshPath(Path):
     """SSH path
 
     Absolute:
@@ -31,16 +32,31 @@ class SshPath(Path, PurePosixPath):
     ssh://[user@]host[:port]/relative/path
     """
 
+    _flavour = _posix_flavour
+
+    def __new__(cls, *args, **kwargs):
+        return object.__new__(cls)
+
+    def __init__(self, url: str):
+        parsed = urlparse(url)
+        assert parsed.scheme == "ssh"
+        self._host = parsed.hostname
+
+        self._parts = re.split(r"/+", parsed.path)
+        if parsed.path.startswith("//"):
+            self._parts[0] = "/"
+        else:
+            self._parts = self._parts[1:]
+
     @property
     def hostpath(self):
-        # path = "/" if self._parts[:0]  + "/".join(self._parts[1:])
         if self.is_absolute():
-            return "/" + self._flavour.join(self._parts[1:])
-        return self._flavour.join(self._parts)
+            return "/" + "/".join(self._parts[1:])
+        return "/".join(self._parts)
 
     @property
     def host(self):
-        return self._drv
+        return self._host
 
     def is_absolute(self):
         return self._parts and self._parts[0] == "/"
@@ -66,11 +82,15 @@ class SshPath(Path, PurePosixPath):
 
     def _make_child(self, args):
         drv, root, parts = self._parse_args(args)
-        assert self._drv == drv or drv == "", f"{self._drv} and {drv}"
+        assert self._host == drv or drv == "", f"{self._host} and {drv}"
         drv, root, parts = self._flavour.join_parsed_parts(
-            "", self._root, self._parts, "", root, parts
+            "", "", self._parts, "", root, parts
         )
-        return self._from_parsed_parts(self._drv, root, parts)
+
+        child = object.__new__(SshPath)
+        child._parts = parts
+        child._host = self._host
+        return child
 
     def open(self, mode="r", buffering=-1, encoding=None, errors=None, newline=None):
         # FIXME: should probably be wiser
@@ -105,10 +125,10 @@ class SshPath(Path, PurePosixPath):
         return obj
 
     def __repr__(self):
-        return "SshPath(%s,%s)" % (self._drv, self._flavour.join(self._parts[1:]))
+        return "SshPath(%s,%s)" % (self._host, self._flavour.join(self._parts[1:]))
 
     def __str__(self):
-        return "ssh://%s/%s" % (self._drv, self._flavour.join(self._parts[1:]))
+        return "ssh://%s/%s" % (self._host, self._flavour.join(self._parts[1:]))
 
 
 @dataclass

{experimaestro-1.4.2 → experimaestro-1.5.0}/src/experimaestro/core/objects.py

@@ -646,7 +646,7 @@ class ConfigInformation:
                     "Cannot set non existing attribute %s in %s" % (k, self.xpmtype)
                 )
         except Exception:
-            logger.exception("Error while setting value %s" % k)
+            logger.error("Error while setting value %s in %s", k, self.xpmtype)
             raise
 
     def addtag(self, name, value):
@@ -1093,7 +1093,7 @@ class ConfigInformation:
         state_dict = {
             "id": id(self.pyobject),
             "module": self.xpmtype._module,
-            "type": self.xpmtype.objecttype.__qualname__,
+            "type": self.xpmtype.basetype.__qualname__,
             "typename": self.xpmtype.name(),
             "identifier": self.identifier.state_dict(),
         }
@@ -1340,7 +1340,10 @@ class ConfigInformation:
             cls = getqualattr(mod, definition["type"])
 
             # Creates an object (or a config)
-            o = cls.__new__(cls, __xpmobject__=as_instance)
+            if as_instance:
+                o = cls.XPMValue.__new__(cls.XPMValue)
+            else:
+                o = cls.XPMConfig.__new__(cls.XPMConfig)
             assert definition["id"] not in objects, "Duplicate id %s" % definition["id"]
             objects[definition["id"]] = o
 
@@ -1377,6 +1380,7 @@ class ConfigInformation:
                         o.__xpm_stdout__ = basepath / f"{name.lower()}.out"
                         o.__xpm_stderr__ = basepath / f"{name.lower()}.err"
             else:
+                o.__init__()
                 xpminfo = o.__xpm__  # type: ConfigInformation
 
                 meta = definition.get("meta", None)
@@ -1497,12 +1501,7 @@ class ConfigInformation:
 
             if o is None:
                 # Creates an object (and not a config)
-                o = config.__xpmtype__.objecttype.__new__(
-                    config.__xpmtype__.objecttype, __xpmobject__=True
-                )
-
-                # And calls the parameter-less initialization
-                o.__init__()
+                o = config.XPMValue()
 
                 # Store in cache
                 self.objects.add_stub(id(config), o)
@@ -1650,8 +1649,8 @@ class TypeConfig:
             [f"{key}={value}" for key, value in self.__xpm__.values.items()]
         )
         return (
-            f"{self.__xpmtype__.objecttype.__module__}."
-            f"{self.__xpmtype__.objecttype.__qualname__}({params})"
+            f"{self.__xpmtype__.basetype.__module__}."
+            f"{self.__xpmtype__.basetype.__qualname__}({params})"
         )
 
     def tag(self, name, value):
@@ -1770,6 +1769,11 @@ class TypeConfig:
         self.__xpm__.add_dependencies(*other.__xpm__.dependencies)
 
 
+class classproperty(property):
+    def __get__(self, owner_self, owner_cls):
+        return self.fget(owner_cls)
+
+
 class Config:
     """Base type for all objects in python interface"""
 
@@ -1781,6 +1785,42 @@ class Config:
     """The __xpm__ object contains all instance specific information about a
     configuration/task"""
 
+    @classproperty
+    def XPMConfig(cls):
+        if issubclass(cls, TypeConfig):
+            return cls
+        return cls.__getxpmtype__().configtype
+
+    @classproperty
+    def C(cls):
+        return cls.XPMConfig
+
+    @classproperty
+    def XPMValue(cls):
+        """Returns the value object for this configuration"""
+        if issubclass(cls, TypeConfig):
+            return cls.__xpmtype__.objecttype
+
+        if value_cls := cls.__dict__.get("__XPMValue__", None):
+            pass
+        else:
+            from .types import XPMValue
+
+            __objectbases__ = tuple(
+                s.XPMValue
+                for s in cls.__bases__
+                if issubclass(s, Config) and (s is not Config)
+            ) or (XPMValue,)
+
+            *tp_qual, tp_name = cls.__qualname__.split(".")
+            value_cls = type(f"{tp_name}.XPMValue", (cls,) + __objectbases__, {})
+            value_cls.__qualname__ = ".".join(tp_qual + [value_cls.__name__])
+            value_cls.__module__ = cls.__module__
+
+            setattr(cls, "__XPMValue__", value_cls)
+
+        return value_cls
+
     @classmethod
     def __getxpmtype__(cls) -> "ObjectType":
         """Get (and create if necessary) the Object type of this"""
@@ -1796,27 +1836,32 @@ class Config:
                 raise
         return xpmtype
 
-    def __getnewargs_ex__(self):
-        # __new__ will be called with those arguments when unserializing
-        return ((), {"__xpmobject__": True})
+    def __new__(cls: Type[T], *args, **kwargs) -> T:
+        """Returns an instance of a TypeConfig (for compatibility, use XPMConfig
+        or C if possible)"""
 
-    @classmethod
-    def c(cls: Type[T], **kwargs) -> T:
-        """Allows typing to process easily"""
-        return cls.__new__(cls, **kwargs)
+        # If this is an XPMValue, just return a new instance
+        from experimaestro.core.types import XPMValue
 
-    def __new__(cls: Type[T], *args, __xpmobject__=False, **kwargs) -> T:
-        """Returns an instance of a TypeConfig when called __xpmobject__ is False,
-        and otherwise the real object
-        """
+        if issubclass(cls, XPMValue):
+            return object.__new__(cls)
 
-        if __xpmobject__:
-            # __init__ is  called directly
+        # If this is the XPMConfig, just return a new instance
+        # __init__ will be called
+        if issubclass(cls, TypeConfig):
             return object.__new__(cls)
 
-        # We use the configuration type
-        o = object.__new__(cls.__getxpmtype__().configtype)
-        o.__init__(*args, **kwargs)
+        # otherwise, we use the configuration type
+        o: TypeConfig = object.__new__(cls.__getxpmtype__().configtype)
+        try:
+            o.__init__(*args, **kwargs)
+        except Exception:
+            caller = inspect.getframeinfo(inspect.stack()[1][0])
+            logger.error(
+                "Init error in %s:%s"
+                % (str(Path(caller.filename).absolute()), caller.lineno)
+            )
+            raise
         return o
 
     def __validate__(self):

{experimaestro-1.4.2 → experimaestro-1.5.0}/src/experimaestro/core/objects.pyi

@@ -27,13 +27,16 @@ from typing import (
     Callable,
     ClassVar,
     Dict,
+    Generic,
     List,
     Optional,
     Set,
+    Type,
     TypeVar,
     Union,
     overload,
 )
+from typing_extensions import Self
 
 TConfig = TypeVar("TConfig", bound="Config")
 
@@ -205,12 +208,15 @@ class TypeConfig:
 class Config:
     __xpmtype__: ClassVar[ObjectType]
     __xpm__: ConfigInformation
+    __use_xpmobject__: ClassVar[bool]
+
+    XPMValue: Type[Self]
+    XPMConfig: Union[Type[Self], Type[TypeConfig[Self]]]
+    C: Union[Type[Self], Type[TypeConfig[Self]]]
+
     @classmethod
     def __getxpmtype__(cls) -> ObjectType: ...
-    def __getnewargs_ex__(self): ...
-    @classmethod
-    def c(cls, **kwargs) -> T: ...
-    def __new__(cls, *args, __xpmobject__: bool = ..., **kwargs) -> T: ...
+    def __new__(cls, *args, **kwargs) -> Self: ...
     def __validate__(self) -> None: ...
     def __post_init__(self) -> None: ...
     def __json__(self): ...
@@ -240,6 +246,6 @@ class Task(LightweightTask):
 def copyconfig(config_or_output: TConfig, **kwargs) -> TConfig: ...
 def setmeta(config: TConfig, flag: bool) -> TConfig: ...
 
-class TypeConfig:
+class TypeConfig(Generic[T]):
     def __validate__(self):
         pass

{experimaestro-1.4.2 → experimaestro-1.5.0}/src/experimaestro/core/types.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 import inspect
 import sys
-from typing import Set, Union, Dict, Iterator, List
+from typing import Set, Union, Dict, Iterator, List, get_args, get_origin
 from collections import ChainMap
 from pathlib import Path
 import typing
@@ -126,6 +126,10 @@ class Type:
         if t:
             return DictType(Type.fromType(t[0]), Type.fromType(t[1]))
 
+        # Takes care of generics
+        if get_origin(key):
+            return GenericType(key)
+
         raise Exception("No type found for %s", key)
 
 
@@ -179,6 +183,12 @@ class SubmitHook(ABC):
         return hash((self.__class__, self.spec()))
 
 
+class XPMValue:
+    """Jut marks a XPMValue"""
+
+    pass
+
+
 class ObjectType(Type):
     submit_hooks: Set[SubmitHook]
     """Hooks associated with this configuration"""
@@ -211,6 +221,8 @@ class ObjectType(Type):
         # Get the identifier
         if identifier is None and hasattr(tp, "__xpmid__"):
             __xpmid__ = getattr(tp, "__xpmid__")
+            if isinstance(__xpmid__, Identifier):
+                identifier = __xpmid__
             if inspect.ismethod(__xpmid__):
                 identifier = Identifier(__xpmid__())
             elif "__xpmid__" in tp.__dict__:
@@ -247,7 +259,7 @@ class ObjectType(Type):
         else:
             self.basetype = tp
 
-        # Create the type-specific configuration class
+        # --- Create the type-specific configuration class (XPMConfig)
         __configbases__ = tuple(
             s.__getxpmtype__().configtype
             for s in tp.__bases__
@@ -256,23 +268,18 @@ class ObjectType(Type):
 
         *tp_qual, tp_name = self.basetype.__qualname__.split(".")
         self.configtype = type(
-            f"{tp_name}_XPMConfig", __configbases__ + (self.basetype,), {}
+            f"{tp_name}.XPMConfig", __configbases__ + (self.basetype,), {}
         )
         self.configtype.__qualname__ = ".".join(tp_qual + [self.configtype.__name__])
         self.configtype.__module__ = tp.__module__
 
-        # Create the type-specific object class
-        # (now, the same as basetype - but in the future, remove references)
-        self.objecttype = self.basetype  # type: type
-        self.basetype._ = self.configtype
-
         # Return type is used by tasks to change the output
         if hasattr(self.basetype, "task_outputs") or False:
             self.returntype = get_type_hints(
                 getattr(self.basetype, "task_outputs")
             ).get("return", typing.Any)
         else:
-            self.returntype = self.objecttype
+            self.returntype = self.basetype
 
         # Registers ourselves
         self.basetype.__xpmtype__ = self
@@ -284,6 +291,11 @@ class ObjectType(Type):
         self.annotations = []
         self._deprecated = False
 
+    @property
+    def objecttype(self):
+        """Returns the object type"""
+        return self.basetype.XPMValue
+
     def addAnnotation(self, annotation):
         assert not self.__initialized__
         self.annotations.append(annotation)
@@ -636,3 +648,31 @@ class DictType(Type):
 
     def __repr__(self):
         return str(self)
+
+
+class GenericType(Type):
+    def __init__(self, type: typing.Type):
+        self.type = type
+        self.origin = get_origin(type)
+
+        self.args = get_args(type)
+
+    def name(self):
+        return str(self.type)
+
+    def __repr__(self):
+        return repr(self.type)
+
+    def validate(self, value):
+        # Now, let's check generics...
+        mros = typingutils.generic_mro(type(value))
+        matching = next(
+            (mro for mro in mros if (get_origin(mro) or mro) is self.origin), None
+        )
+        target = get_origin(self.type) or self.type
+        if matching is None:
+            raise ValueError(
+                f"{type(value)} is not of type {target} ({type(value).__mro__})"
+            )
+
+        return value

experimaestro-1.5.0/src/experimaestro/exceptions.py

@@ -0,0 +1,2 @@
+class HandledException(Exception):
+    pass

{experimaestro-1.4.2 → experimaestro-1.5.0}/src/experimaestro/experiments/cli.py

@@ -10,6 +10,7 @@ import omegaconf
 import yaml
 from experimaestro import LauncherRegistry, RunMode, experiment
 from experimaestro.experiments.configuration import ConfigurationBase
+from experimaestro.exceptions import HandledException
 from experimaestro.settings import get_workspace
 from omegaconf import OmegaConf, SCMode
 from termcolor import cprint
@@ -63,7 +64,7 @@ def load(yaml_file: Path):
     if parent := _data.get("parent", None):
         data.extend(load(yaml_file.parent / parent))
 
-    return data
+    return data[::-1]
 
 
 @click.option("--debug", is_flag=True, help="Print debug information")
@@ -110,10 +111,16 @@ def load(yaml_file: Path):
     help="Working directory - if None, uses the default XPM " "working directory",
 )
 @click.option("--conf", "-c", "extra_conf", type=str, multiple=True)
+@click.option(
+    "--pre-yaml", type=str, multiple=True, help="Add YAML file after the main one"
+)
+@click.option(
+    "--post-yaml", type=str, multiple=True, help="Add YAML file before the main one"
+)
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 @click.argument("yaml_file", metavar="YAML file", type=str)
 @click.command()
-def experiments_cli(
+def experiments_cli(  # noqa: C901
     yaml_file: str,
     xp_file: str,
     host: str,
@@ -123,6 +130,8 @@ def experiments_cli(
     env: List[Tuple[str, str]],
     run_mode: RunMode,
     extra_conf: List[str],
+    pre_yaml: List[str],
+    post_yaml: List[str],
     args: List[str],
     show: bool,
     debug: bool,
@@ -133,13 +142,17 @@ def experiments_cli(
     logging.getLogger("xpm.hash").setLevel(logging.INFO)
 
     # --- Loads the YAML
-    yamls = load(Path(yaml_file))
+    yamls = []
+    for y in pre_yaml:
+        yamls.extend(load(Path(y)))
+    yamls.extend(load(Path(yaml_file)))
+    for y in post_yaml:
+        yamls.extend(load(Path(y)))
 
     # --- Get the XP file
     if xp_file is None:
-        for data in yamls:
-            if xp_file := data.get("file"):
-                del data["file"]
+        for data in yamls[::-1]:
+            if xp_file := data.get("file", None):
                 break
 
         if xp_file is None:
@@ -201,15 +214,15 @@ def experiments_cli(
             cprint(f"Error in configuration:\n\n{e}", "red", file=sys.stderr)
             sys.exit(1)
 
+    if show:
+        print(json.dumps(OmegaConf.to_container(configuration)))  # noqa: T201
+        sys.exit(0)
+
     # Move to an object container
     configuration: schema = OmegaConf.to_container(
         configuration, structured_config_mode=SCMode.INSTANTIATE
     )
 
-    if show:
-        print(json.dumps(OmegaConf.to_container(configuration)))  # noqa: T201
-        sys.exit(0)
-
     # Get the working directory
     if workdir is None or not Path(workdir).is_dir():
         workdir = get_workspace(workdir).path.expanduser().resolve()
@@ -223,9 +236,13 @@ def experiments_cli(
         for key, value in env:
             xp.setenv(key, value)
 
-        # Run the experiment
-        helper.xp = xp
-        helper.run(list(args), configuration)
+        try:
+            # Run the experiment
+            helper.xp = xp
+            helper.run(list(args), configuration)
+
+            # ... and wait
+            xp.wait()
 
-        # ... and wait
-        xp.wait()
+        except HandledException:
+            sys.exit(1)

{experimaestro-1.4.2 → experimaestro-1.5.0}/src/experimaestro/experiments/configuration.py

@@ -1,3 +1,4 @@
+from omegaconf import MISSING
 from typing import Optional
 import attr
 
@@ -19,14 +20,28 @@ def configuration(*args, **kwargs):
 
 @configuration()
 class ConfigurationBase:
-    id: str
-    """ID of the experiment"""
+    """Base configuration for any experiment"""
 
-    description: str = ""
-    """Description of the experiment"""
+    id: str = MISSING
+    """ID of the experiment
+
+    This ID is used by experimaestro when running as the experiment.
+    """
 
     file: str = "experiment"
-    """qualified name (relative to the module) for the file containing a run function"""
+    """Relative path of the file containing a run function"""
 
     parent: Optional[str] = None
     """Relative path of a YAML file that should be merged"""
+
+    title: str = ""
+    """Short description of the experiment"""
+
+    subtitle: str = ""
+    """Allows to give some more details about the experiment"""
+
+    paper: str = ""
+    """Source paper for this experiment"""
+
+    description: str = ""
+    """Description of the experiment"""

{experimaestro-1.4.2 → experimaestro-1.5.0}/src/experimaestro/launcherfinder/registry.py

@@ -85,6 +85,10 @@ def load_yaml(loader_cls: Type[Loader], path: Path):
     if not path.is_file():
         return None
 
+    logger.warning(
+        "Using YAML file to configure launchers is deprecated. Please remove %s using launchers.py",
+        path,
+    )
     logger.debug("Loading %s", path)
     with path.open("rt") as fp:
         loader = loader_cls(fp)

{experimaestro-1.4.2 → experimaestro-1.5.0}/src/experimaestro/launcherfinder/specs.py

@@ -51,6 +51,13 @@ class CPUSpecification:
     def __lt__(self, other: "CPUSpecification"):
         return self.memory < other.memory and self.cores < other.cores
 
+    def total_memory(self, gpus: int = 0):
+        return max(
+            self.memory,
+            self.mem_per_cpu * self.cores,
+            self.cpu_per_gpu * self.mem_per_cpu * gpus,
+        )
+
 
 @define(kw_only=True)
 class HostSpecification: