lsst-ctrl-bps-htcondor 29.2025.1500__tar.gz → 29.2025.1700__tar.gz

{lsst_ctrl_bps_htcondor-29.2025.1500/python/lsst_ctrl_bps_htcondor.egg-info → lsst_ctrl_bps_htcondor-29.2025.1700}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.1500
+Version: 29.2025.1700
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_htcondor-29.2025.1500 → lsst_ctrl_bps_htcondor-29.2025.1700}/python/lsst/ctrl/bps/htcondor/lssthtc.py

@@ -703,7 +703,7 @@ def htc_submit_dag(sub):
     return schedd_dag_info
 
 
-def htc_create_submit_from_dag(dag_filename, submit_options=None):
+def htc_create_submit_from_dag(dag_filename: str, submit_options: dict[str, Any]) -> htcondor.Submit:
     """Create a DAGMan job submit description.
 
     Parameters
@@ -730,11 +730,26 @@ def htc_create_submit_from_dag(dag_filename, submit_options=None):
     var_name = "_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"
     if var_name not in os.environ:
         os.environ[var_name] = "True"
-    do_recurse = submit_options.get("do_recurse", None)
-    if do_recurse:
+
+    if "do_recurse" in submit_options:
         var_name = "_CONDOR_DAGMAN_GENERATE_SUBDAG_SUBMITS"
         if var_name not in os.environ:
-            os.environ[var_name] = str(do_recurse)
+            os.environ[var_name] = str(submit_options["do_recurse"])
+
+    # Config and environment variables do not seem to override -MaxIdle
+    # on the .dag.condor.sub's command line (broken in some 24.0.x versions).
+    # Explicitly forward them as a submit_option if either exists.
+    # Note: auto generated subdag submit files are still the -MaxIdle=1000
+    # in the broken versions.
+    if "MaxIdle" not in submit_options:
+        max_jobs_idle: int | None = None
+        config_var_name = "DAGMAN_MAX_JOBS_IDLE"
+        if f"_CONDOR_{config_var_name}" in os.environ:
+            max_jobs_idle = int(os.environ[f"_CONDOR_{config_var_name}"])
+        elif config_var_name in htcondor.param:
+            max_jobs_idle = htcondor.param[config_var_name]
+        if max_jobs_idle:
+            submit_options["MaxIdle"] = max_jobs_idle
 
     return htcondor.Submit.from_dag(dag_filename, submit_options)
 

lsst_ctrl_bps_htcondor-29.2025.1700/python/lsst/ctrl/bps/htcondor/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.1700"

{lsst_ctrl_bps_htcondor-29.2025.1500 → lsst_ctrl_bps_htcondor-29.2025.1700/python/lsst_ctrl_bps_htcondor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.1500
+Version: 29.2025.1700
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_htcondor-29.2025.1500 → lsst_ctrl_bps_htcondor-29.2025.1700}/tests/test_lssthtc.py

@@ -30,10 +30,11 @@ import io
 import logging
 import os
 import pathlib
+import stat
 import sys
 import tempfile
 import unittest
-from shutil import copy2, copytree, ignore_patterns, rmtree
+from shutil import copy2, copytree, ignore_patterns, rmtree, which
 
 import htcondor
 
@@ -1139,5 +1140,92 @@ class HtcWriteCondorFileTestCase(unittest.TestCase):
             _ = lssthtc.htc_create_submit_from_file(filename)
 
 
+class HtcCreateSubmitFromDagTestCase(unittest.TestCase):
+    """Test htc_create_submit_from_dag function."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.bindir = None
+        # htcondor.Submit.from_dag requires condor_dagman executable in path.
+        if not which("condor_dagman"):  # pragma: no cover
+            cls.bindir = tempfile.TemporaryDirectory()
+            fake_dagman_exec = pathlib.Path(cls.bindir.name) / "condor_dagman"
+            with open(fake_dagman_exec, "w") as fh:
+                print("#!/bin/bash", file=fh)
+                print("echo fake_condor_dagman $@", file=fh)
+                print("exit 0", file=fh)
+            fake_dagman_exec.chmod(fake_dagman_exec.stat().st_mode | stat.S_IEXEC)
+            os.environ["PATH"] = f"{os.environ['PATH']}:{cls.bindir.name}"
+
+    @classmethod
+    def tearDownClass(cls):
+        if cls.bindir:
+            cls.bindir.cleanup()
+
+    @unittest.mock.patch.dict(os.environ, {"_CONDOR_DAGMAN_MAX_JOBS_IDLE": "42"})
+    def testMaxIdleEnvVar(self):
+        with temporaryDirectory() as tmp_dir:
+            copy2(f"{TESTDIR}/data/tiny_success/tiny_success.dag", tmp_dir)
+            dag_filename = pathlib.Path(tmp_dir) / "tiny_success.dag"
+            submit = lssthtc.htc_create_submit_from_dag(str(dag_filename), {})
+            self.assertIn("-MaxIdle 42", submit["arguments"])
+            self.assertEqual("true", os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"].lower())
+
+    @unittest.mock.patch.dict(os.environ, {})
+    def testMaxIdleGiven(self):
+        with temporaryDirectory() as tmp_dir:
+            copy2(f"{TESTDIR}/data/tiny_success/tiny_success.dag", tmp_dir)
+            dag_filename = pathlib.Path(tmp_dir) / "tiny_success.dag"
+            submit = lssthtc.htc_create_submit_from_dag(str(dag_filename), {"MaxIdle": 37})
+            self.assertIn("-MaxIdle 37", submit["arguments"])
+            self.assertEqual("true", os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"].lower())
+
+    @unittest.mock.patch.dict(os.environ, {})
+    def testNoMaxJobsIdle(self):
+        """Note: Since the produced arguments differ depending on
+        HTCondor version when no MaxIdle passed to from_dag, not
+        checking arguments string here.  Instead just making sure
+        lssthtc code doesn't pass MaxIdle value to from_dag.
+        """
+        with temporaryDirectory() as tmp_dir:
+            copy2(f"{TESTDIR}/data/tiny_success/tiny_success.dag", tmp_dir)
+            dag_filename = pathlib.Path(tmp_dir) / "tiny_success.dag"
+            with unittest.mock.patch("htcondor.Submit.from_dag") as submit_mock:
+                with unittest.mock.patch("htcondor.param") as mock_param:
+                    mock_param.__contains__.return_value = False
+                    _ = lssthtc.htc_create_submit_from_dag(str(dag_filename), {})
+                    self.assertEqual("true", os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"].lower())
+                    submit_mock.assert_called_once_with(str(dag_filename), {})
+
+    @unittest.mock.patch.dict(os.environ, {})
+    def testDoRecurseGivenWithNoEnv(self):
+        with temporaryDirectory() as tmp_dir:
+            copy2(f"{TESTDIR}/data/tiny_success/tiny_success.dag", tmp_dir)
+            dag_filename = pathlib.Path(tmp_dir) / "tiny_success.dag"
+            submit = lssthtc.htc_create_submit_from_dag(str(dag_filename), {"do_recurse": True})
+            self.assertIn("-do_recurse", submit["arguments"])
+            self.assertEqual("true", os.environ["_CONDOR_DAGMAN_GENERATE_SUBDAG_SUBMITS"].lower())
+            self.assertEqual("true", os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"].lower())
+
+    @unittest.mock.patch.dict(os.environ, {"_CONDOR_DAGMAN_GENERATE_SUBDAG_SUBMITS": "False"})
+    def testDoRecurseGivenWithEnv(self):
+        with temporaryDirectory() as tmp_dir:
+            copy2(f"{TESTDIR}/data/tiny_success/tiny_success.dag", tmp_dir)
+            dag_filename = pathlib.Path(tmp_dir) / "tiny_success.dag"
+            submit = lssthtc.htc_create_submit_from_dag(str(dag_filename), {"do_recurse": True})
+            self.assertIn("-do_recurse", submit["arguments"])
+            self.assertEqual("true", os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"].lower())
+            self.assertEqual("false", os.environ["_CONDOR_DAGMAN_GENERATE_SUBDAG_SUBMITS"].lower())
+
+    @unittest.mock.patch.dict(os.environ, {"_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV": "*_DIR"})
+    def testGetEnvOverridden(self):
+        with temporaryDirectory() as tmp_dir:
+            copy2(f"{TESTDIR}/data/tiny_success/tiny_success.dag", tmp_dir)
+            dag_filename = pathlib.Path(tmp_dir) / "tiny_success.dag"
+            submit = lssthtc.htc_create_submit_from_dag(str(dag_filename), {"do_recurse": True})
+            self.assertIn("-do_recurse", submit["arguments"])
+            self.assertEqual(os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"], "*_DIR")
+
+
 if __name__ == "__main__":
     unittest.main()

lsst_ctrl_bps_htcondor-29.2025.1500/python/lsst/ctrl/bps/htcondor/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.1500"