experimaestro 1.6.0__py3-none-any.whl → 1.6.2__py3-none-any.whl

experimaestro/core/objects.py

@@ -1340,8 +1340,14 @@ class ConfigInformation:
                     sys.modules[module_name] = mod
                     spec.loader.exec_module(mod)
             else:
-                logger.debug("Importing module %s", definition["module"])
-                mod = importlib.import_module(module_name)
+                try:
+                    logger.debug("Importing module %s", definition["module"])
+                    mod = importlib.import_module(module_name)
+                except ModuleNotFoundError:
+                    # More hints on the nature of the error
+                    logging.warning("(1) Either the python path is wrong – %s", ":".join(sys.path))
+                    logging.warning("(2) There is not __init__.py in your module")
+                    raise
 
             cls = getqualattr(mod, definition["type"])
 

experimaestro/experiments/cli.py

@@ -59,7 +59,7 @@ class ExperimentCallable(Protocol):
 class ConfigurationLoader:
     def __init__(self):
         self.yamls = []
-        self.pythonpath = set()
+        self.python_path = set()
 
     def load(self, yaml_file: Path):
         """Loads a YAML file, and parents one if they exist"""
@@ -76,9 +76,9 @@ class ConfigurationLoader:
         for path in _data.get("pythonpath", []):
             path = Path(path)
             if path.is_absolute():
-                self.pythonpath.add(path.resolve())
+                self.python_path.add(path.resolve())
             else:
-                self.pythonpath.add((yaml_file.parent / path).resolve())
+                self.python_path.add((yaml_file.parent / path).resolve())
 
 
 @click.option("--debug", is_flag=True, help="Print debug information")
@@ -181,7 +181,7 @@ def experiments_cli(  # noqa: C901
         configuration.merge_with(OmegaConf.from_dotlist(extra_conf))
 
     # --- Get the XP file
-    pythonpath = list(conf_loader.pythonpath)
+    python_path = list(conf_loader.python_path)
     if module_name is None:
         module_name = configuration.get("module", None)
 
@@ -192,9 +192,9 @@ def experiments_cli(  # noqa: C901
                 not module_name
             ), "Module name and experiment file are mutually exclusive options"
             xp_file = Path(xp_file)
-            if not pythonpath:
-                pythonpath.append(xp_file.parent)
-            logging.info("Using python path: %s", ", ".join(str(s) for s in pythonpath))
+            if not python_path:
+                python_path.append(xp_file.parent)
+            logging.info("Using python path: %s", ", ".join(str(s) for s in python_path))
 
     assert (
         module_name or xp_file
@@ -208,29 +208,31 @@ def experiments_cli(  # noqa: C901
 
     # --- Finds the "run" function
 
-    try:
-        for path in pythonpath:
-            sys.path.append(str(path))
-
-        if xp_file:
-            if not xp_file.exists() and xp_file.suffix != ".py":
-                xp_file = xp_file.with_suffix(".py")
-            xp_file: Path = Path(yaml_file).parent / xp_file
-            with open(xp_file) as src:
-                module_name = xp_file.with_suffix("").name
-                mod = imp.load_module(
-                    module_name,
-                    src,
-                    str(xp_file.absolute()),
-                    (".py", "r", imp.PY_SOURCE),
-                )
-        else:
-            # Module
+    # Modifies the Python path
+    for path in python_path:
+        sys.path.append(str(path))
+
+    if xp_file:
+        if not xp_file.exists() and xp_file.suffix != ".py":
+            xp_file = xp_file.with_suffix(".py")
+        xp_file: Path = Path(yaml_file).parent / xp_file
+        with open(xp_file) as src:
+            module_name = xp_file.with_suffix("").name
+            mod = imp.load_module(
+                module_name,
+                src,
+                str(xp_file.absolute()),
+                (".py", "r", imp.PY_SOURCE),
+            )
+    else:
+        # Module
+        try:
             mod = importlib.import_module(module_name)
+        except ModuleNotFoundError as e:
+            logging.error("Module not found: %s with python path %s", e, sys.path)
+            raise
 
-        helper = getattr(mod, "run", None)
-    finally:
-        pass
+    helper = getattr(mod, "run", None)
 
     # --- ... and runs it
     if helper is None:
@@ -267,10 +269,11 @@ def experiments_cli(  # noqa: C901
 
     # Define the workspace
     ws_env = find_workspace(workdir=workdir, workspace=workspace)
+    
     workdir = ws_env.path
 
     logging.info("Using working directory %s", str(workdir.resolve()))
-
+    
     # --- Runs the experiment
     with experiment(
         ws_env, configuration.id, host=host, port=port, run_mode=run_mode
@@ -280,6 +283,9 @@ def experiments_cli(  # noqa: C901
         for key, value in env:
             xp.setenv(key, value)
 
+        # Sets the python path
+        xp.workspace.python_path.extend(python_path)
+
         try:
             # Run the experiment
             helper.xp = xp

experimaestro/scheduler/base.py

@@ -1,12 +1,13 @@
 from collections import ChainMap
 from functools import cached_property
+import itertools
 import logging
 import os
 from pathlib import Path
 from shutil import rmtree
 import threading
 import time
-from typing import Any, List, Optional, Set, TypeVar, Union, TYPE_CHECKING
+from typing import Any, Iterator, List, Optional, Set, TypeVar, Union, TYPE_CHECKING
 import enum
 import signal
 import asyncio
@@ -166,6 +167,13 @@ class Job(Resource):
         assert self._future, "Cannot wait a not submitted job"
         return self._future.result()
 
+    @cached_property
+    def python_path(self) -> Iterator[str]:
+        """Returns an iterator over python path"""
+        return itertools.chain(
+            self.workspace.python_path
+        )
+        
     @cached_property
     def environ(self):
         """Returns the job environment

experimaestro/scheduler/workspace.py

@@ -2,7 +2,7 @@ from collections import ChainMap
 from enum import Enum
 from functools import cached_property
 from pathlib import Path
-from typing import Optional
+from typing import Iterator, Optional
 from experimaestro.settings import WorkspaceSettings, Settings
 
 
@@ -46,6 +46,7 @@ class Workspace:
             path = path.absolute()
         self.path = path
         self.run_mode = run_mode
+        self.python_path = []
         from ..launchers import Launcher
 
         self.launcher = launcher or Launcher.get(path)

experimaestro/scriptbuilder.py

@@ -94,6 +94,7 @@ class PythonScriptBuilder:
             out.write("# Experimaestro generated task\n\n")
             out.write(
                 """import logging\n"""
+                """import sys\n"""
                 """logging.basicConfig(level=logging.INFO, """
                 """format='%(levelname)s:%(process)d:%(asctime)s [%(name)s] %(message)s', datefmt='%y-%m-%d %H:%M:%S')\n\n"""
             )
@@ -112,9 +113,17 @@ class PythonScriptBuilder:
             out.write("    ]\n")
 
             for name, value in job.environ.items():
-                out.write(f"""    os.environ["{name}"] = "{shquote(value)}"\n""")
+                if name == "PYTHONPATH":
+                    # Handles properly python path
+                    for path in value.split(":"):
+                        out.write(f"""    sys.path.insert(0, "{shquote(path)}")\n""")
+                else:
+                    out.write(f"""    os.environ["{name}"] = "{shquote(value)}"\n""")
             out.write("\n")
 
+            for path in job.python_path:
+                out.write(f"""    sys.path.insert(0, "{shquote(str(path))}")\n""")
+
             out.write(
                 f"""    TaskRunner("{shquote(connector.resolve(scriptpath))}","""
                 """ lockfiles).run()\n"""

experimaestro/settings.py

@@ -37,7 +37,7 @@ class WorkspaceSettings:
 
     alt_workspaces: List[str] = field(default_factory=list)
     """Alternative workspaces to find jobs or experiments"""
-
+    
     def __post_init__(self):
         self.path = self.path.expanduser().resolve()
 
@@ -83,7 +83,7 @@ def get_workspace(id: Optional[str] = None) -> Optional[WorkspaceSettings]:
     return None
 
 
-def find_workspace(*, workspace: Optional[str] = None, workdir: Optional[Path] = None):
+def find_workspace(*, workspace: Optional[str] = None, workdir: Optional[Path] = None) -> WorkspaceSettings:
     """Find workspace"""
     workdir = Path(workdir) if workdir else None
 

{experimaestro-1.6.0.dist-info → experimaestro-1.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.6.0
+Version: 1.6.2
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager

{experimaestro-1.6.0.dist-info → experimaestro-1.6.2.dist-info}/RECORD

@@ -14,7 +14,7 @@ experimaestro/connectors/ssh.py,sha256=P6XfCdC4mQTsFzgI-dEdx6AdVhwd0T6VrQpPmNpmi
 experimaestro/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/core/arguments.py,sha256=dW32opqNEsULYr6nR7Zk8PqHsSCbLPclfXofw27GTpI,5620
 experimaestro/core/context.py,sha256=Q8_ngiHRBZ0laavXRJNiDvdCprrnROVTWaHfrwMdlG4,2638
-experimaestro/core/objects.py,sha256=M6g2f3klT0Z913yniwKBakChUKGnb2dOoW3hJHZi7do,64305
+experimaestro/core/objects.py,sha256=Oh9S85enB_cuQhFxZqwNoJMcrbuPoESdTdEDcO4tAqM,64649
 experimaestro/core/objects.pyi,sha256=Adi2OKCW0-B9GROlEUgxxBDK6SHUqccTBHTofRRJE9M,7131
 experimaestro/core/serialization.py,sha256=9tg5ebLF3YeZ_zG9DiTHPLthppvo7io710ohD_dcLTo,3836
 experimaestro/core/serializers.py,sha256=R_CAMyjjfU1oi-eHU6VlEUixJpFayGqEPaYu7VsD9xA,1197
@@ -22,7 +22,7 @@ experimaestro/core/types.py,sha256=8alVqTA7aeIZ3FAaDux8IwoRPDbwVxzONVqnAL1A6QI,2
 experimaestro/core/utils.py,sha256=JfC3qGUS9b6FUHc2VxIYUI9ysNpXSQ1LjOBkjfZ8n7o,495
 experimaestro/exceptions.py,sha256=cUy83WHM3GeynxmMk6QRr5xsnpqUAdAoc-m3KQVrE2o,44
 experimaestro/experiments/__init__.py,sha256=GcpDUIbCvhnv6rxFdAp4wTffCVNTv-InY6fbQAlTy-o,159
-experimaestro/experiments/cli.py,sha256=Uu9GlCrwBvyp_sxmO4VLLcSwVVquKfdlYrxCXhlN7Kc,8447
+experimaestro/experiments/cli.py,sha256=IyoYL2tBR33tI2bxLD0Z5fwXlyTQH626eFE8NAws-VE,8633
 experimaestro/experiments/configuration.py,sha256=cFDiUHnUGblJsctAUxAqx0jlM7_Ja_527lzk-4G-44k,1368
 experimaestro/generators.py,sha256=9NQ_TfDfASkArLnO4PF7s5Yoo9KWjlna2DCPzk5gJOI,1230
 experimaestro/huggingface.py,sha256=gnVlr6SZnbutYz4PLH0Q77n1TRF-uk-dR-3UFzFqAY0,2956
@@ -49,13 +49,13 @@ experimaestro/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/rpyc.py,sha256=ZRKol-3tVoeoUITLNFenLF4dhWBLW_FvSV_GvsypmeI,3605
 experimaestro/run.py,sha256=NTFORDb_RlEK6tWKa7K-_2_bGCdHzzjBJVH5C1ReYtw,5222
 experimaestro/scheduler/__init__.py,sha256=ERmmOxz_9mUkIuccNbzUa5Y6gVLLVDdyc4cCxbCCUbY,20
-experimaestro/scheduler/base.py,sha256=XGq8kokNycba5D-KYKrNZ3WcXWKDUxusTkgR-X9S0E0,31941
+experimaestro/scheduler/base.py,sha256=z2npreB_MiE-mjZgxLSYFSMh8Ugns_AlyaNPY-BRw8U,32174
 experimaestro/scheduler/dependencies.py,sha256=n9XegwrmjayOIxt3xhuTEBVEBGSq4oeVdzz-FviDGXo,1994
 experimaestro/scheduler/services.py,sha256=aCKkNZMULlceabqf-kOs_-C7KPINnjU3Q-I00o5x6iY,2189
-experimaestro/scheduler/workspace.py,sha256=vyVbYZML28zvmgxfWc2PsKKHGlrAejY43UYlttbm9AU,2280
-experimaestro/scriptbuilder.py,sha256=WokCsXjmsVsx9mD6-sJdsBI6lD6xNMXUz1YvAnb7e10,4533
+experimaestro/scheduler/workspace.py,sha256=KNdxPBwUk7gO8h2utMCrlIVKB-f2Ylqg_IxLc4okp_8,2320
+experimaestro/scriptbuilder.py,sha256=nLtwZG136mcji3UhbUDNw4Wj8nbHcj1XysZ2f0_A8eQ,4947
 experimaestro/server/__init__.py,sha256=F2bzLf2q29Haj2OIbPA26r5WVbaipBNylIozg-As758,10854
-experimaestro/settings.py,sha256=UsfKIA4Jx-32hm3xGjBs_b9uvw8M7sOYtzFyUsYSJ8s,3145
+experimaestro/settings.py,sha256=U6gTVBL5Z4Rk0_7BAVoavVJKN2sQNRpspE-601Elfys,3170
 experimaestro/sphinx/__init__.py,sha256=heovvtwbYToZM-b6HNi4pJdBoo_97usdEawhMGSK3bk,9560
 experimaestro/sphinx/static/experimaestro.css,sha256=0rEgt1LoDdD-a_R5rVfWZ19zD1gR-1L7q3f4UibIB58,294
 experimaestro/taskglobals.py,sha256=aBjPpo4HQp6E6M3GQ8L6PR4rK2Lu0kD5dS1WjnaGgDc,499
@@ -115,8 +115,8 @@ experimaestro/utils/jupyter.py,sha256=JcEo2yQK7x3Cr1tNl5FqGMZOICxCv9DwMvL5xsWdQP
 experimaestro/utils/resources.py,sha256=MaCQL9dLfze3lwyuBVeWF7Ki5fFSE1F0BGWrfaaHi1I,1135
 experimaestro/utils/settings.py,sha256=jpFMqF0DLL4_P1xGal0zVR5cOrdD8O0Y2IOYvnRgN3k,793
 experimaestro/xpmutils.py,sha256=S21eMbDYsHfvmZ1HmKpq5Pz5O-1HnCLYxKbyTBbASyQ,638
-experimaestro-1.6.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-experimaestro-1.6.0.dist-info/METADATA,sha256=ls8yp4BJZ_021MzRsuWAmIWLbKWz8apcxCwzKT8H7pM,6220
-experimaestro-1.6.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-experimaestro-1.6.0.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
-experimaestro-1.6.0.dist-info/RECORD,,
+experimaestro-1.6.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+experimaestro-1.6.2.dist-info/METADATA,sha256=3UbntC-D10JVbARfMQQKenXFCYeSZn6lkM9zkIbz_pY,6220
+experimaestro-1.6.2.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+experimaestro-1.6.2.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
+experimaestro-1.6.2.dist-info/RECORD,,