experimaestro 1.5.14__py3-none-any.whl → 1.6.1__py3-none-any.whl

experimaestro/core/objects.py

@@ -1340,8 +1340,14 @@ class ConfigInformation:
                     sys.modules[module_name] = mod
                     spec.loader.exec_module(mod)
             else:
-                logger.debug("Importing module %s", definition["module"])
-                mod = importlib.import_module(module_name)
+                try:
+                    logger.debug("Importing module %s", definition["module"])
+                    mod = importlib.import_module(module_name)
+                except ModuleNotFoundError:
+                    # More hints on the nature of the error
+                    logging.warning("(1) Either the python path is wrong – %s", ":".join(sys.path))
+                    logging.warning("(2) There is not __init__.py in your module")
+                    raise
 
             cls = getqualattr(mod, definition["type"])
 

experimaestro/experiments/cli.py

@@ -1,4 +1,5 @@
 import imp
+import importlib
 import inspect
 import json
 import logging
@@ -55,18 +56,29 @@ class ExperimentCallable(Protocol):
         ...
 
 
-def load(yaml_file: Path):
-    """Loads a YAML file, and parents one if they exist"""
-    if not yaml_file.exists() and yaml_file.suffix != ".yaml":
-        yaml_file = yaml_file.with_suffix(".yaml")
+class ConfigurationLoader:
+    def __init__(self):
+        self.yamls = []
+        self.pythonpath = set()
 
-    with yaml_file.open("rt") as fp:
-        _data = yaml.full_load(fp)
-    data = [_data]
-    if parent := _data.get("parent", None):
-        data.extend(load(yaml_file.parent / parent))
+    def load(self, yaml_file: Path):
+        """Loads a YAML file, and parents one if they exist"""
+        if not yaml_file.exists() and yaml_file.suffix != ".yaml":
+            yaml_file = yaml_file.with_suffix(".yaml")
 
-    return data[::-1]
+        with yaml_file.open("rt") as fp:
+            _data = yaml.full_load(fp)
+        if parent := _data.get("parent", None):
+            self.load(yaml_file.parent / parent)
+
+        self.yamls.append(_data)
+
+        for path in _data.get("pythonpath", []):
+            path = Path(path)
+            if path.is_absolute():
+                self.pythonpath.add(path.resolve())
+            else:
+                self.pythonpath.add((yaml_file.parent / path).resolve())
 
 
 @click.option("--debug", is_flag=True, help="Print debug information")
@@ -106,6 +118,9 @@ def load(yaml_file: Path):
 @click.option(
     "--file", "xp_file", help="The file containing the main experimental code"
 )
+@click.option(
+    "--module-name", "module_name", help="Module containing the experimental code"
+)
 @click.option(
     "--workspace",
     type=str,
@@ -141,31 +156,49 @@ def experiments_cli(  # noqa: C901
     extra_conf: List[str],
     pre_yaml: List[str],
     post_yaml: List[str],
+    module_name: Optional[str],
     args: List[str],
     show: bool,
     debug: bool,
 ):
     """Run an experiment"""
+
     # --- Set the logger
     logging.getLogger().setLevel(logging.DEBUG if debug else logging.INFO)
     logging.getLogger("xpm.hash").setLevel(logging.INFO)
 
     # --- Loads the YAML
-    yamls = []
+    conf_loader = ConfigurationLoader()
     for y in pre_yaml:
-        yamls.extend(load(Path(y)))
-    yamls.extend(load(Path(yaml_file)))
+        conf_loader.load(Path(y))
+    conf_loader.load(Path(yaml_file))
     for y in post_yaml:
-        yamls.extend(load(Path(y)))
+        conf_loader.load(Path(y))
+
+    # --- Merge the YAMLs
+    configuration = OmegaConf.merge(*conf_loader.yamls)
+    if extra_conf:
+        configuration.merge_with(OmegaConf.from_dotlist(extra_conf))
 
     # --- Get the XP file
-    if xp_file is None:
-        for data in yamls[::-1]:
-            if xp_file := data.get("file", None):
-                break
+    pythonpath = list(conf_loader.pythonpath)
+    if module_name is None:
+        module_name = configuration.get("module", None)
 
-        if xp_file is None:
-            raise ValueError("No experiment file given")
+    if xp_file is None:
+        xp_file = configuration.get("file", None)
+        if xp_file:
+            assert (
+                not module_name
+            ), "Module name and experiment file are mutually exclusive options"
+            xp_file = Path(xp_file)
+            if not pythonpath:
+                pythonpath.append(xp_file.parent)
+            logging.info("Using python path: %s", ", ".join(str(s) for s in pythonpath))
+
+    assert (
+        module_name or xp_file
+    ), "Either the module name or experiment file should be given"
 
     # --- Set some options
 
@@ -173,23 +206,29 @@ def experiments_cli(  # noqa: C901
         assert xpm_config_dir.is_dir()
         LauncherRegistry.set_config_dir(xpm_config_dir)
 
-    # --- Loads the XP file
-    xp_file = Path(xp_file)
-    if not xp_file.exists() and xp_file.suffix != ".py":
-        xp_file = xp_file.with_suffix(".py")
-    xp_file: Path = Path(yaml_file).parent / xp_file
-
     # --- Finds the "run" function
-    try:
-        sys.path.append(str(xp_file.parent.absolute()))
+
+    # Modifies the Python path
+    for path in pythonpath:
+        sys.path.append(str(path))
+
+    if xp_file:
+        if not xp_file.exists() and xp_file.suffix != ".py":
+            xp_file = xp_file.with_suffix(".py")
+        xp_file: Path = Path(yaml_file).parent / xp_file
         with open(xp_file) as src:
             module_name = xp_file.with_suffix("").name
             mod = imp.load_module(
-                module_name, src, str(xp_file.absolute()), (".py", "r", imp.PY_SOURCE)
+                module_name,
+                src,
+                str(xp_file.absolute()),
+                (".py", "r", imp.PY_SOURCE),
             )
-            helper = getattr(mod, "run", None)
-    finally:
-        sys.path.pop()
+    else:
+        # Module
+        mod = importlib.import_module(module_name)
+
+    helper = getattr(mod, "run", None)
 
     # --- ... and runs it
     if helper is None:
@@ -208,9 +247,6 @@ def experiments_cli(  # noqa: C901
     schema = list_parameters[1].annotation
     omegaconf_schema = OmegaConf.structured(schema())
 
-    configuration = OmegaConf.merge(*yamls)
-    if extra_conf:
-        configuration.merge_with(OmegaConf.from_dotlist(extra_conf))
     if omegaconf_schema is not None:
         try:
             configuration = OmegaConf.merge(omegaconf_schema, configuration)

experimaestro/experiments/configuration.py

@@ -1,5 +1,5 @@
 from omegaconf import MISSING
-from typing import Optional
+from typing import Optional, List
 import attr
 
 try:
@@ -31,6 +31,12 @@ class ConfigurationBase:
     file: str = "experiment"
     """Relative path of the file containing a run function"""
 
+    module: Optional[str] = None
+    """Relative path of the file containing a run function"""
+
+    pythonpath: Optional[List[str]] = None
+    """Python path relative to the parent directory of the YAML file"""
+
     parent: Optional[str] = None
     """Relative path of a YAML file that should be merged"""
 

{experimaestro-1.5.14.dist-info → experimaestro-1.6.1.dist-info}/METADATA

@@ -1,8 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.5.14
+Version: 1.6.1
 Summary: "Experimaestro is a computer science experiment manager"
-Home-page: https://github.com/experimaestro/experimaestro-python
 License: GPL-3
 Keywords: experiment manager
 Author: Benjamin Piwowarski

{experimaestro-1.5.14.dist-info → experimaestro-1.6.1.dist-info}/RECORD

@@ -14,7 +14,7 @@ experimaestro/connectors/ssh.py,sha256=P6XfCdC4mQTsFzgI-dEdx6AdVhwd0T6VrQpPmNpmi
 experimaestro/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/core/arguments.py,sha256=dW32opqNEsULYr6nR7Zk8PqHsSCbLPclfXofw27GTpI,5620
 experimaestro/core/context.py,sha256=Q8_ngiHRBZ0laavXRJNiDvdCprrnROVTWaHfrwMdlG4,2638
-experimaestro/core/objects.py,sha256=M6g2f3klT0Z913yniwKBakChUKGnb2dOoW3hJHZi7do,64305
+experimaestro/core/objects.py,sha256=Oh9S85enB_cuQhFxZqwNoJMcrbuPoESdTdEDcO4tAqM,64649
 experimaestro/core/objects.pyi,sha256=Adi2OKCW0-B9GROlEUgxxBDK6SHUqccTBHTofRRJE9M,7131
 experimaestro/core/serialization.py,sha256=9tg5ebLF3YeZ_zG9DiTHPLthppvo7io710ohD_dcLTo,3836
 experimaestro/core/serializers.py,sha256=R_CAMyjjfU1oi-eHU6VlEUixJpFayGqEPaYu7VsD9xA,1197
@@ -22,8 +22,8 @@ experimaestro/core/types.py,sha256=8alVqTA7aeIZ3FAaDux8IwoRPDbwVxzONVqnAL1A6QI,2
 experimaestro/core/utils.py,sha256=JfC3qGUS9b6FUHc2VxIYUI9ysNpXSQ1LjOBkjfZ8n7o,495
 experimaestro/exceptions.py,sha256=cUy83WHM3GeynxmMk6QRr5xsnpqUAdAoc-m3KQVrE2o,44
 experimaestro/experiments/__init__.py,sha256=GcpDUIbCvhnv6rxFdAp4wTffCVNTv-InY6fbQAlTy-o,159
-experimaestro/experiments/cli.py,sha256=tMMtfAkewj8jAPseXD1RBePBt10XWRItQ9TkKhV18yY,7228
-experimaestro/experiments/configuration.py,sha256=8GRqyLG1leF_NbvbFzqpm0yM24O0WjSNmQzvnuLnxxw,1150
+experimaestro/experiments/cli.py,sha256=AS6LcbY3O8tkQ8X-ushZcRVdLt1WbykwGP8CZ2BfgMw,8371
+experimaestro/experiments/configuration.py,sha256=cFDiUHnUGblJsctAUxAqx0jlM7_Ja_527lzk-4G-44k,1368
 experimaestro/generators.py,sha256=9NQ_TfDfASkArLnO4PF7s5Yoo9KWjlna2DCPzk5gJOI,1230
 experimaestro/huggingface.py,sha256=gnVlr6SZnbutYz4PLH0Q77n1TRF-uk-dR-3UFzFqAY0,2956
 experimaestro/ipc.py,sha256=ltYqybPm_XfcQC3yiskMfhfI_1dREs-XRu0F83YsNws,1490
@@ -55,30 +55,6 @@ experimaestro/scheduler/services.py,sha256=aCKkNZMULlceabqf-kOs_-C7KPINnjU3Q-I00
 experimaestro/scheduler/workspace.py,sha256=vyVbYZML28zvmgxfWc2PsKKHGlrAejY43UYlttbm9AU,2280
 experimaestro/scriptbuilder.py,sha256=WokCsXjmsVsx9mD6-sJdsBI6lD6xNMXUz1YvAnb7e10,4533
 experimaestro/server/__init__.py,sha256=F2bzLf2q29Haj2OIbPA26r5WVbaipBNylIozg-As758,10854
-experimaestro/server/data/016b4a6cdced82ab3aa1.ttf,sha256=AD8RVBhWpkmmyCNcYmbIk2IkxdYJ5RRC2iTcVVbRT78,189684
-experimaestro/server/data/0c35d18bf06992036b69.woff2,sha256=gmX2R4Y5fWuDLRygqv3xSa2E5ydZ__qfcnLpGg-wFdE,128352
-experimaestro/server/data/219aa9140e099e6c72ed.woff2,sha256=0xv7gdVueQ4Nni-gC4Pfj3FZ-QYxFM3AFIWbHUg5Vsg,135984
-experimaestro/server/data/3a4004a46a653d4b2166.woff,sha256=-jiI7_pbxjPGuygaSdiSf-eqACEsn8D3M4yBnsXMFu4,156236
-experimaestro/server/data/3baa5b8f3469222b822d.woff,sha256=PTTzCsa7Ivpb9Y0-Uue7bezZnFcdanfV36e-eL7PK1A,339600
-experimaestro/server/data/4d73cb90e394b34b7670.woff,sha256=_YT4i0lwQNT31ejJ-GNa740-cGwPpS4rb6zxTu6H5SI,164912
-experimaestro/server/data/4ef4218c522f1eb6b5b1.woff2,sha256=Hmc7qFiWVHlNKtN1woc_GHCkV48rPoR9zEB63QO1esI,215704
-experimaestro/server/data/50701fbb8177c2dde530.ttf,sha256=fYGhp8wH4asZbkBJbT9DWel1n3nY7Ig6Rmde5pkSlQs,63348
-experimaestro/server/data/5d681e2edae8c60630db.woff,sha256=HBNbFRpB4jE4sy0wZePw6rIwpnrv6SXplwnL4QK9YEw,206260
-experimaestro/server/data/6f420cf17cc0d7676fad.woff2,sha256=NdyopxRaF8jRMG8lo8oJFXjhU5bwsi1h645zJirHVXc,155276
-experimaestro/server/data/878f31251d960bd6266f.woff2,sha256=-q5vwKqUzFveUHZkfIF6IyBglqHL7aENHG89idYWPtE,109808
-experimaestro/server/data/b041b1fa4fe241b23445.woff2,sha256=kWnYvnqBd-WpKk0Ett5_ZQS5OFc79NpYiYccTzdtOEk,24488
-experimaestro/server/data/b6879d41b0852f01ed5b.woff2,sha256=iGyGESqATvHd0csgavTIxA40tzwmZSyiMUBKo1prMNk,150020
-experimaestro/server/data/c380809fd3677d7d6903.woff2,sha256=yUjxJjNBaZs8HpxV2NDz5EZmnQ8rnVVJTGFpIiwCQ6Y,173620
-experimaestro/server/data/d75e3fd1eb12e9bd6655.ttf,sha256=zqebNFyvSdYiMJi4orBNcCc-jXxrnj0Dz396JneT6ao,394668
-experimaestro/server/data/f882956fd323fd322f31.woff,sha256=jpR1jFTCboWqzy09yhrXqtpZBKRgI4-uSEPrxEvELtw,182028
-experimaestro/server/data/favicon.ico,sha256=PRD32mxgMXg0AIFmjErFs66XQ8qaJiqw_NMS-7n0i90,3870
-experimaestro/server/data/index.css,sha256=JTjLM7vCHt6ShbNYXZ27MJsYcX_KNuVVwg05OcTfqmU,418843
-experimaestro/server/data/index.css.map,sha256=ieCWV1uODCJdH3Lgoekp2WGaphqZird27UsGqFlzEI0,576970
-experimaestro/server/data/index.html,sha256=LcXSo2QU5jcmqgmFhVWWOG_CfGyrqdm3H8jwkdRgdGU,706
-experimaestro/server/data/index.js,sha256=f0GvRsfsQ4ayP4en7Q-raZ6buwRXLCswCbzVaxQEDWY,3734128
-experimaestro/server/data/index.js.map,sha256=za3MUIjzyyGRI6F5KuBFMTgrFU55xgt0LBrw-4YPHag,3904832
-experimaestro/server/data/login.html,sha256=4dvhSOn6DHp_tbmzqIKrqq2uAo0sAUbgLVD0lTnPp4s,511
-experimaestro/server/data/manifest.json,sha256=EpzHQZzrGh9c1Kf63nrqvI33H1cm0nLYfdh5lDm8ijI,318
 experimaestro/settings.py,sha256=UsfKIA4Jx-32hm3xGjBs_b9uvw8M7sOYtzFyUsYSJ8s,3145
 experimaestro/sphinx/__init__.py,sha256=heovvtwbYToZM-b6HNi4pJdBoo_97usdEawhMGSK3bk,9560
 experimaestro/sphinx/static/experimaestro.css,sha256=0rEgt1LoDdD-a_R5rVfWZ19zD1gR-1L7q3f4UibIB58,294
@@ -139,8 +115,8 @@ experimaestro/utils/jupyter.py,sha256=JcEo2yQK7x3Cr1tNl5FqGMZOICxCv9DwMvL5xsWdQP
 experimaestro/utils/resources.py,sha256=MaCQL9dLfze3lwyuBVeWF7Ki5fFSE1F0BGWrfaaHi1I,1135
 experimaestro/utils/settings.py,sha256=jpFMqF0DLL4_P1xGal0zVR5cOrdD8O0Y2IOYvnRgN3k,793
 experimaestro/xpmutils.py,sha256=S21eMbDYsHfvmZ1HmKpq5Pz5O-1HnCLYxKbyTBbASyQ,638
-experimaestro-1.5.14.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-experimaestro-1.5.14.dist-info/METADATA,sha256=jEbKxGTumDrr8CB6aAz50WUx7yel7Kvp3xQiSBX14aw,6286
-experimaestro-1.5.14.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-experimaestro-1.5.14.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
-experimaestro-1.5.14.dist-info/RECORD,,
+experimaestro-1.6.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+experimaestro-1.6.1.dist-info/METADATA,sha256=23LdSFk3UzGvwsUqh3GLk_g-VTqZ_qvJa6SAYHXDGiI,6220
+experimaestro-1.6.1.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+experimaestro-1.6.1.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
+experimaestro-1.6.1.dist-info/RECORD,,

{experimaestro-1.5.14.dist-info → experimaestro-1.6.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.1
+Generator: poetry-core 2.0.1
 Root-Is-Purelib: true
 Tag: py3-none-any