resubmit 0.0.4__tar.gz → 0.0.6__tar.gz

{resubmit-0.0.4/src/resubmit.egg-info → resubmit-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: resubmit
-Version: 0.0.4
+Version: 0.0.6
 Summary: Small wrapper around submitit to simplify cluster submissions
 Author: Amir Mehrpanah
 License: MIT

{resubmit-0.0.4 → resubmit-0.0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "resubmit"
-version = "0.0.4"
+version = "0.0.6"
 description = "Small wrapper around submitit to simplify cluster submissions"
 readme = "README.md"
 license = { text = "MIT" }

{resubmit-0.0.4 → resubmit-0.0.6}/src/resubmit/__bookkeeping.py

@@ -5,27 +5,6 @@ from itertools import product
 import logging
 
 
-def _is_regex_spec(val: Any) -> bool:
-    """Return True if val looks like a regex specifier.
-
-    Accepted forms:
-    - compiled `re.Pattern`
-    - tuple (`re.Pattern`, exclude: bool)
-    - dict with keys `pattern` (re.Pattern) and optional `exclude` (bool)
-    - string starting with 're:' (e.g. 're:^foo.*') meaning include matches
-    - string starting with '!re:' meaning exclude matches
-    """
-    if hasattr(val, "search") and callable(val.search):
-        return True
-    if isinstance(val, tuple) and len(val) >= 1 and hasattr(val[0], "search"):
-        return True
-    if isinstance(val, dict) and "pattern" in val:
-        return True
-    if isinstance(val, str) and (val.startswith("re:") or val.startswith("!re:")):
-        return True
-    return False
-
-
 def _normalize_regex_spec(val: Any) -> Tuple[re.Pattern, bool]:
     """Return (compiled_pattern, exclude_flag) for a given regex spec.
 
@@ -115,11 +94,6 @@ def create_jobs_dataframe(params: Dict[str, Any]) -> pd.DataFrame:
                 base = k[: -len("_unique")]
             unique_items[base] = v
             continue
-        elif callable(v):
-            callables[k] = v
-        elif _is_regex_spec(v):
-            # treat a regex spec provided under the same key as a filter for that column
-            regex_specs[k] = v
         else:
             static_items[k] = v
 
@@ -206,7 +180,7 @@ def submit_jobs(
 
     jobs_df = create_jobs_dataframe(jobs_args)
     records = jobs_df.to_dict(orient="records")
-    from .__submit import submit_jobs as _submit_jobs
+    from .__submit import _submit_jobs
 
     return _submit_jobs(
         records,

{resubmit-0.0.4 → resubmit-0.0.6}/src/resubmit/__submit.py

@@ -3,7 +3,7 @@
 from typing import Any, Callable, Iterable, List, Optional, Dict
 
 
-def submit_jobs(
+def _submit_jobs(
     jobs_args: Iterable[dict],
     func: Callable[[List[dict]], Any],
     *,
@@ -15,6 +15,7 @@ def submit_jobs(
     block: bool,
     prompt: bool,
     local_run: bool,
+    job_name: Optional[str] = "resubmit",
     slurm_additional_parameters: Optional[Dict] = None,
 ):
     """Submit jobs described by `jobs_args` where each entry is a dict of kwargs for `func`.
@@ -48,18 +49,13 @@ def submit_jobs(
     print("submitting jobs")
     executor = submitit.AutoExecutor(folder=folder)
 
-    # default slurm params (keep cluster-specific options out unless explicitly set)
-    if slurm_additional_parameters is None:
-        slurm_additional_parameters = {"gpus": num_gpus}
-    else:
-        slurm_additional_parameters = dict(slurm_additional_parameters)
-        slurm_additional_parameters.setdefault("gpus", num_gpus)
-
     print("Slurm additional parameters:", slurm_additional_parameters)
 
     executor.update_parameters(
+        name=job_name,
         timeout_min=timeout_min,
         cpus_per_task=cpus_per_task,
+        gpus_per_node=num_gpus,
         mem_gb=mem_gb,
         slurm_additional_parameters=slurm_additional_parameters,
     )

{resubmit-0.0.4 → resubmit-0.0.6/src/resubmit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: resubmit
-Version: 0.0.4
+Version: 0.0.6
 Summary: Small wrapper around submitit to simplify cluster submissions
 Author: Amir Mehrpanah
 License: MIT

{resubmit-0.0.4 → resubmit-0.0.6}/tests/test_bookkeeping.py

@@ -1,6 +1,6 @@
 import re
 import pandas as pd
-from src.resubmit.__bookkeeping import create_jobs_dataframe, ensure_unique_combinations
+from resubmit.__bookkeeping import create_jobs_dataframe, ensure_unique_combinations
 
 
 def test_create_jobs_basic():
@@ -11,7 +11,7 @@ def test_create_jobs_basic():
 
 
 def test_create_jobs_callable():
-    params = {"a": [1, 2], "b": lambda df: df["a"] * 10}
+    params = {"a": [1, 2], "b__callable": lambda df: df["a"] * 10}
     df = create_jobs_dataframe(params)
     assert list(df["b"]) == [10, 20]
 

resubmit-0.0.6/tests/test_resubmit.py

@@ -0,0 +1,115 @@
+import pytest
+from resubmit import maybe_attach_debugger
+from resubmit.__submit import _submit_jobs
+
+
+def dummy_func(jobs):
+    # return a list of strings to show behavior
+    return [f"ok-{j['id']}" for j in jobs]
+
+
+def test_submit_local_run():
+    jobs = [{"id": 1}, {"id": 2}]
+    res = _submit_jobs(
+        jobs,
+        dummy_func,
+        timeout_min=1,
+        local_run=True,
+        num_gpus=0,
+        cpus_per_task=1,
+        mem_gb=8,
+        folder="dummy/%j",
+        block=False,
+        prompt=False,
+    )
+    assert res == ["ok-1", "ok-2"]
+
+
+def test_maybe_attach_debugger_noop():
+    # should not raise when port is None or 0
+    maybe_attach_debugger(None)
+    maybe_attach_debugger(0)
+
+
+def test_slurm_parameters_optional(monkeypatch):
+    events = {}
+
+    class DummyExecutor:
+        def __init__(self, folder):
+            events["folder"] = folder
+
+        def update_parameters(self, **kwargs):
+            # capture the parameters passed to the executor
+            events["update"] = kwargs
+
+        def map_array(self, func, jobs_list):
+            return []
+
+    class DummyModule:
+        AutoExecutor = DummyExecutor
+
+    import sys
+
+    monkeypatch.setitem(sys.modules, "submitit", DummyModule)
+
+    jobs = [{"id": 1}]
+    # default: no constraint/reservation keys
+    _submit_jobs(
+        jobs,
+        dummy_func,
+        timeout_min=1,
+        local_run=False,
+        num_gpus=2,
+        prompt=False,
+        cpus_per_task=4,
+        mem_gb=16,
+        folder="logs/%j",
+        block=False,
+    )
+    slurm = events["update"]["slurm_additional_parameters"]
+    assert slurm["gpus"] == 2
+    assert "constraint" not in slurm
+    assert "reservation" not in slurm
+
+
+def test_slurm_parameters_settable(monkeypatch):
+    events = {}
+
+    class DummyExecutor:
+        def __init__(self, folder):
+            events["folder"] = folder
+
+        def update_parameters(self, **kwargs):
+            events["update"] = kwargs
+
+        def map_array(self, func, jobs_list):
+            return []
+
+    class DummyModule:
+        AutoExecutor = DummyExecutor
+
+    import sys
+
+    monkeypatch.setitem(sys.modules, "submitit", DummyModule)
+
+    jobs = [{"id": 1}]
+    _submit_jobs(
+        jobs,
+        dummy_func,
+        timeout_min=1,
+        local_run=False,
+        prompt=False,
+        slurm_additional_parameters={
+            "constraint": "thin",
+            "reservation": "safe",
+        },
+        cpus_per_task=4,
+        mem_gb=16,
+        folder="logs/%j",
+        block=False,
+        num_gpus=1,
+    )
+    slurm = events["update"]["slurm_additional_parameters"]
+    assert slurm["constraint"] == "thin"
+    assert slurm["reservation"] == "safe"
+

resubmit-0.0.4/tests/test_resubmit.py

@@ -1,117 +0,0 @@
-import pytest
-from resubmit import maybe_attach_debugger
-from resubmit.__submit import submit_jobs 
-
-
-def dummy_func(jobs):
-    # return a list of strings to show behavior
-    return [f"ok-{j['id']}" for j in jobs]
-
-
-def test_submit_local_run():
-    jobs = [{"id": 1}, {"id": 2}]
-    res = submit_jobs(jobs, dummy_func, timeout_min=1, local_run=True)
-    assert res == ["ok-1", "ok-2"]
-
-
-def test_maybe_attach_debugger_noop():
-    # should not raise when port is None or 0
-    maybe_attach_debugger(None)
-    maybe_attach_debugger(0)
-
-
-def test_slurm_parameters_optional(monkeypatch):
-    events = {}
-
-    class DummyExecutor:
-        def __init__(self, folder):
-            events['folder'] = folder
-
-        def update_parameters(self, **kwargs):
-            # capture the parameters passed to the executor
-            events['update'] = kwargs
-
-        def map_array(self, func, jobs_list):
-            return []
-
-    class DummyModule:
-        AutoExecutor = DummyExecutor
-
-    import sys
-    monkeypatch.setitem(sys.modules, 'submitit', DummyModule)
-
-    jobs = [{"id": 1}]
-    # default: no constraint/reservation keys
-    submit_jobs(jobs, dummy_func, timeout_min=1, local_run=False, num_gpus=2, prompt=False)
-    slurm = events['update']['slurm_additional_parameters']
-    assert slurm['gpus'] == 2
-    assert 'constraint' not in slurm
-    assert 'reservation' not in slurm
-
-
-def test_slurm_parameters_settable(monkeypatch):
-    events = {}
-
-    class DummyExecutor:
-        def __init__(self, folder):
-            events['folder'] = folder
-
-        def update_parameters(self, **kwargs):
-            events['update'] = kwargs
-
-        def map_array(self, func, jobs_list):
-            return []
-
-    class DummyModule:
-        AutoExecutor = DummyExecutor
-
-    import sys
-    monkeypatch.setitem(sys.modules, 'submitit', DummyModule)
-
-    jobs = [{"id": 1}]
-    submit_jobs(
-        jobs,
-        dummy_func,
-        timeout_min=1,
-        local_run=False,
-        constraint='thin',
-        reservation='safe',
-        prompt=False,
-    )
-    slurm = events['update']['slurm_additional_parameters']
-    assert slurm['constraint'] == 'thin'
-    assert slurm['reservation'] == 'safe'
-
-
-def test_slurm_parameters_arg_precedence(monkeypatch):
-    events = {}
-
-    class DummyExecutor:
-        def __init__(self, folder):
-            events['folder'] = folder
-
-        def update_parameters(self, **kwargs):
-            events['update'] = kwargs
-
-        def map_array(self, func, jobs_list):
-            return []
-
-    class DummyModule:
-        AutoExecutor = DummyExecutor
-
-    import sys
-    monkeypatch.setitem(sys.modules, 'submitit', DummyModule)
-
-    jobs = [{"id": 1}]
-    # slurm_additional_parameters has constraint='foo' but explicit arg should override
-    submit_jobs(
-        jobs,
-        dummy_func,
-        timeout_min=1,
-        local_run=False,
-        slurm_additional_parameters={'constraint': 'foo'},
-        constraint='bar',
-        prompt=False,
-    )
-    slurm = events['update']['slurm_additional_parameters']
-    assert slurm['constraint'] == 'bar'