hydra-submitit-launcher 1.4.0.dev5__tar.gz → 1.4.0.dev8__tar.gz

{hydra_submitit_launcher-1.4.0.dev5/hydra_submitit_launcher.egg-info → hydra_submitit_launcher-1.4.0.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydra-submitit-launcher
-Version: 1.4.0.dev5
+Version: 1.4.0.dev8
 Summary: Submitit Launcher for Hydra apps
 Home-page: https://github.com/facebookincubator/submitit
 Author: Jeremy Rapin, Jieru Hu, Omry Yadan
@@ -17,7 +17,7 @@ Classifier: Development Status :: 4 - Beta
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: hydra-core>=1.1.0.dev7
-Requires-Dist: submitit>=1.3.3
+Requires-Dist: submitit>=1.4.6
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{hydra_submitit_launcher-1.4.0.dev5 → hydra_submitit_launcher-1.4.0.dev8}/hydra_plugins/hydra_submitit_launcher/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 
-__version__ = "1.4.0.dev5"
+__version__ = "1.4.0.dev8"

{hydra_submitit_launcher-1.4.0.dev5 → hydra_submitit_launcher-1.4.0.dev8}/hydra_plugins/hydra_submitit_launcher/config.py

@@ -18,7 +18,7 @@ class BaseQueueConf:
     # number of gpus to use on each node
     gpus_per_node: Optional[int] = None
     # number of tasks to spawn on each node
-    tasks_per_node: int = 1
+    tasks_per_node: Optional[int] = 1
     # memory to reserve for the job on each node (in GB)
     mem_gb: Optional[int] = None
     # number of nodes to use for the job
@@ -66,6 +66,8 @@ class SlurmQueueConf(BaseQueueConf):
     # check the following for more info on slurm_max_num_timeout
     # https://github.com/facebookincubator/submitit/blob/main/docs/checkpointing.md
     max_num_timeout: int = 0
+    # Python executable to use instead of the default sys.executable
+    python: Optional[str] = None
     # Useful to add parameters which are not currently available in the plugin.
     # Eg: {"mail-user": "blublu@fb.com", "mail-type": "BEGIN"}
     additional_parameters: Dict[str, Any] = field(default_factory=dict)

{hydra_submitit_launcher-1.4.0.dev5 → hydra_submitit_launcher-1.4.0.dev8}/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py

@@ -95,7 +95,7 @@ class BaseSubmititLauncher(Launcher):
         params = self.params
         # build executor
         init_params = {"folder": self.params["submitit_folder"]}
-        specific_init_keys = {"max_num_timeout"}
+        specific_init_keys = {"max_num_timeout", "python"}
 
         init_params.update(
             **{

{hydra_submitit_launcher-1.4.0.dev5 → hydra_submitit_launcher-1.4.0.dev8/hydra_submitit_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydra-submitit-launcher
-Version: 1.4.0.dev5
+Version: 1.4.0.dev8
 Summary: Submitit Launcher for Hydra apps
 Home-page: https://github.com/facebookincubator/submitit
 Author: Jeremy Rapin, Jieru Hu, Omry Yadan
@@ -17,7 +17,7 @@ Classifier: Development Status :: 4 - Beta
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: hydra-core>=1.1.0.dev7
-Requires-Dist: submitit>=1.3.3
+Requires-Dist: submitit>=1.4.6
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{hydra_submitit_launcher-1.4.0.dev5 → hydra_submitit_launcher-1.4.0.dev8}/hydra_submitit_launcher.egg-info/requires.txt

@@ -1,2 +1,2 @@
 hydra-core>=1.1.0.dev7
-submitit>=1.3.3
+submitit>=1.4.6

{hydra_submitit_launcher-1.4.0.dev5 → hydra_submitit_launcher-1.4.0.dev8}/setup.py

@@ -29,7 +29,7 @@ setup(
     python_requires=">=3.10",
     install_requires=[
         "hydra-core>=1.1.0.dev7",
-        "submitit>=1.3.3",
+        "submitit>=1.4.6",
     ],
     include_package_data=True,
 )

hydra_submitit_launcher-1.4.0.dev8/tests/test_submitit_launcher.py

@@ -0,0 +1,140 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from pathlib import Path
+from typing import Optional, Type
+from unittest.mock import MagicMock, patch
+
+import submitit
+from hydra.core.plugins import Plugins
+from hydra.plugins.launcher import Launcher
+from hydra.test_utils.launcher_common_tests import (
+    IntegrationTestSuite,
+    LauncherTestSuite,
+)
+from hydra.test_utils.test_utils import chdir_plugin_root, run_python_script
+from hydra.utils import instantiate
+from omegaconf import OmegaConf
+from pytest import mark
+
+from hydra_plugins.hydra_submitit_launcher import submitit_launcher
+from hydra_plugins.hydra_submitit_launcher.config import SlurmQueueConf
+
+chdir_plugin_root()
+
+pytestmark = mark.filterwarnings("ignore:pkg_resources is deprecated as an API:Warning")
+
+
+@mark.parametrize(
+    "cls", [submitit_launcher.LocalLauncher, submitit_launcher.SlurmLauncher]
+)
+def test_discovery(cls: Type[Launcher]) -> None:
+    # Tests that this plugin can be discovered via the plugins subsystem when looking for Launchers
+    assert cls.__name__ in [x.__name__ for x in Plugins.instance().discover(Launcher)]
+
+
+@mark.parametrize(
+    "launcher_name, overrides", [("submitit_local", ["hydra.launcher.timeout_min=2"])]
+)
+class TestSubmititLauncher(LauncherTestSuite):
+    pass
+
+
+@mark.parametrize(
+    "task_launcher_cfg, extra_flags",
+    [
+        (
+            {},
+            [
+                "-m",
+                "hydra/hydra_logging=disabled",
+                "hydra/job_logging=disabled",
+                "hydra/launcher=submitit_local",
+                "hydra.launcher.gpus_per_node=0",
+                "hydra.launcher.timeout_min=1",
+            ],
+        ),
+    ],
+)
+class TestSubmititLauncherIntegration(IntegrationTestSuite):
+    pass
+
+
+@mark.parametrize("python", [None, "/opt/venv/bin/python"])
+def test_slurm_python_parameter(tmp_path: Path, python: Optional[str]) -> None:
+    executor = MagicMock()
+    executor.map_array.return_value = []
+    auto_executor = MagicMock(return_value=executor)
+
+    config = OmegaConf.create(
+        {
+            "hydra": {
+                "job": {"name": "test"},
+                "sweep": {"dir": str(tmp_path / "sweep")},
+                "launcher": OmegaConf.structured(SlurmQueueConf),
+            }
+        }
+    )
+    config.hydra.launcher.python = python
+    launcher = instantiate(
+        config.hydra.launcher,
+        _target_whitelist_=(
+            "hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher"
+        ),
+    )
+    launcher.config = config
+
+    with patch.object(submitit, "AutoExecutor", auto_executor):
+        assert launcher.launch([[]], initial_job_idx=0) == []
+
+    auto_executor.assert_called_once_with(
+        cluster="slurm",
+        folder=str(tmp_path / "sweep" / ".submitit" / "%j"),
+        slurm_max_num_timeout=0,
+        slurm_python=python,
+    )
+    assert "slurm_python" not in executor.update_parameters.call_args.kwargs
+
+
+@mark.parametrize("tasks_per_node", [1, None])
+def test_slurm_tasks_per_node_is_optional_with_compatible_default(
+    tmp_path: Path, tasks_per_node: Optional[int]
+) -> None:
+    assert SlurmQueueConf().tasks_per_node == 1
+
+    executor = MagicMock()
+    executor.map_array.return_value = []
+    config = OmegaConf.create(
+        {
+            "hydra": {
+                "job": {"name": "test"},
+                "sweep": {"dir": str(tmp_path / "sweep")},
+                "launcher": OmegaConf.structured(SlurmQueueConf),
+            }
+        }
+    )
+    config.hydra.launcher.tasks_per_node = tasks_per_node
+    launcher = instantiate(
+        config.hydra.launcher,
+        _target_whitelist_=(
+            "hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher"
+        ),
+    )
+    launcher.config = config
+
+    with patch.object(submitit, "AutoExecutor", return_value=executor):
+        assert launcher.launch([[]], initial_job_idx=0) == []
+
+    assert (
+        executor.update_parameters.call_args.kwargs["tasks_per_node"] == tasks_per_node
+    )
+
+
+def test_example(tmpdir: Path) -> None:
+    run_python_script(
+        [
+            "example/my_app.py",
+            "-m",
+            f"hydra.sweep.dir={tmpdir}",
+            "hydra/launcher=submitit_local",
+        ],
+        allow_warnings=True,
+    )

hydra_submitit_launcher-1.4.0.dev5/tests/test_submitit_launcher.py

@@ -1,65 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-from pathlib import Path
-from typing import Type
-
-from hydra.core.plugins import Plugins
-from hydra.plugins.launcher import Launcher
-from hydra.test_utils.launcher_common_tests import (
-    IntegrationTestSuite,
-    LauncherTestSuite,
-)
-from hydra.test_utils.test_utils import chdir_plugin_root, run_python_script
-from pytest import mark
-
-from hydra_plugins.hydra_submitit_launcher import submitit_launcher
-
-chdir_plugin_root()
-
-pytestmark = mark.filterwarnings("ignore:pkg_resources is deprecated as an API:Warning")
-
-
-@mark.parametrize(
-    "cls", [submitit_launcher.LocalLauncher, submitit_launcher.SlurmLauncher]
-)
-def test_discovery(cls: Type[Launcher]) -> None:
-    # Tests that this plugin can be discovered via the plugins subsystem when looking for Launchers
-    assert cls.__name__ in [x.__name__ for x in Plugins.instance().discover(Launcher)]
-
-
-@mark.parametrize(
-    "launcher_name, overrides", [("submitit_local", ["hydra.launcher.timeout_min=2"])]
-)
-class TestSubmititLauncher(LauncherTestSuite):
-    pass
-
-
-@mark.parametrize(
-    "task_launcher_cfg, extra_flags",
-    [
-        (
-            {},
-            [
-                "-m",
-                "hydra/hydra_logging=disabled",
-                "hydra/job_logging=disabled",
-                "hydra/launcher=submitit_local",
-                "hydra.launcher.gpus_per_node=0",
-                "hydra.launcher.timeout_min=1",
-            ],
-        ),
-    ],
-)
-class TestSubmititLauncherIntegration(IntegrationTestSuite):
-    pass
-
-
-def test_example(tmpdir: Path) -> None:
-    run_python_script(
-        [
-            "example/my_app.py",
-            "-m",
-            f"hydra.sweep.dir={tmpdir}",
-            "hydra/launcher=submitit_local",
-        ],
-        allow_warnings=True,
-    )