lsst-ctrl-bps-htcondor 29.2025.4500__tar.gz → 29.2025.4600__tar.gz

{lsst_ctrl_bps_htcondor-29.2025.4500/python/lsst_ctrl_bps_htcondor.egg-info → lsst_ctrl_bps_htcondor-29.2025.4600}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.4500
+Version: 29.2025.4600
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later
@@ -23,7 +23,10 @@ License-File: gpl-v3.0.txt
 Requires-Dist: htcondor>=8.8
 Requires-Dist: lsst-ctrl-bps
 Requires-Dist: lsst-daf-butler
+Requires-Dist: lsst-pipe-base
 Requires-Dist: lsst-utils
+Requires-Dist: packaging
+Requires-Dist: pydantic<3.0,>=2
 Provides-Extra: test
 Requires-Dist: pytest>=3.2; extra == "test"
 Requires-Dist: pytest-openfiles>=0.5.0; extra == "test"

{lsst_ctrl_bps_htcondor-29.2025.4500 → lsst_ctrl_bps_htcondor-29.2025.4600}/doc/lsst.ctrl.bps.htcondor/userguide.rst

@@ -150,6 +150,46 @@ available in your BPS configuration file. For example:
 .. __: https://pipelines.lsst.io/v/weekly/modules/lsst.ctrl.bps/quickstart.html#bps-configuration-file
 .. __: https://pipelines.lsst.io/v/weekly/modules/lsst.ctrl.bps/quickstart.html#supported-settings
 
+Configuring DAGMan
+^^^^^^^^^^^^^^^^^^
+
+`DAGMan`_ is a `HTCondor`_ tool that allows multiple jobs to be organized in
+workflows.  It orchestrates the execution of jobs in a workflow to satisfy their
+data dependencies. DAGman workflows are described in the DAG description files.
+
+`HTCondor`_ has many settings that affect the operation of `DAGMan`_. Any of
+these settings can be managed via the submit YAML by specifying their values
+in the ``wmsConfig`` section. For example, including the lines below in your
+submit YAML will instruct DAGMan to throttle the number of jobs DAGMan will
+submit at once for execution to 256:
+
+.. code-block:: YAML
+
+   wmsConfig:
+     DAGMAN_MAX_JOBS_IDLE: 256
+
+A complete list of the supported settings, their descriptions, and default
+values can be found `here`__.
+
+.. note::
+
+   Make sure to select the version of the documentation that corresponds to the
+   version of the HTCondor you're using.  Supported settings may vary between
+   different versions.
+
+When customizing DAGMan's settings, make sure the value you provide has the
+appropriate type.  Using incorrect value type will result in an error during
+the BPS submission.
+
+The settings are for the entire workflow, so the ``wmsConfig`` section can go
+at the root level or inside a ``site`` section, but not inside a ``pipetask``,
+``clusterorfinalJob`` section.
+
+If your main workflow contains sub-workflow defined in individual DAG
+description files, they will use the same configuration as the main workflow.
+
+.. __: https://htcondor.readthedocs.io/en/latest/admin-manual/configuration-macros.html#dagman-configuration-file-entries
+
 .. .. _htc-plugin-authenticating:
 
 .. Authenticating
@@ -629,6 +669,7 @@ complete your run.
 
 .. __: https://developer.lsst.io/usdf/batch.html#ctrl-bps-htcondor
 
+.. _DAGMan: https://htcondor.readthedocs.io/en/latest/automated-workflows/index.html#dagman-workflows
 .. _HTCondor: https://htcondor.readthedocs.io/en/latest/
 .. _Slurm: https://slurm.schedmd.com/overview.html
 .. _bps cancel: https://pipelines.lsst.io/v/weekly/modules/lsst.ctrl.bps/quickstart.html#canceling-submitted-jobs

{lsst_ctrl_bps_htcondor-29.2025.4500 → lsst_ctrl_bps_htcondor-29.2025.4600}/pyproject.toml

@@ -27,7 +27,10 @@ dependencies = [
     "htcondor >= 8.8",
     "lsst-ctrl-bps",
     "lsst-daf-butler",
-    "lsst-utils"
+    "lsst-pipe-base",
+    "lsst-utils",
+    "packaging",
+    "pydantic >= 2, < 3.0",
 ]
 dynamic = ["version"]
 

lsst_ctrl_bps_htcondor-29.2025.4600/python/lsst/ctrl/bps/htcondor/dagman_configurator.py

@@ -0,0 +1,196 @@
+# This file is part of ctrl_bps_htcondor.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Module enabling configuring DAGMan via submit YAML."""
+
+__all__ = ["DagmanConfigurator"]
+
+import logging
+import os
+from pathlib import Path
+from typing import Any
+
+import htcondor
+from pydantic import AliasGenerator, ConfigDict, create_model
+
+from lsst.ctrl.bps import BpsConfig
+
+from .lssthtc import HTCDag
+
+_LOG = logging.getLogger(__name__)
+
+# Extract DAGMan configuration options with their types and default values from
+# the local HTCondor configuration.
+#
+# Notes
+# -----
+# There are some DAGMan configuration options that names do not start with
+# ``DAGMAN_`` (e.g., ``MAX_DAGMAN_LOG``). Hence, do not use
+# ``key.startswith("DAGMAN_")``.
+_fields = {key.lower(): (type(val), val) for key, val in htcondor.param.items() if "DAGMAN_" in key}
+
+# Add some valid configuration options are not set by default by HTCondor and
+# are missing from ``htcondor.param``.
+#
+# Notes
+# -----
+# A complete list of configuration options HTCondor supports can be found in
+# ``src/condor_utils/param_info.in`` in
+# `HTCondor GitHub repository <https://github.com/htcondor/htcondor>`_.
+_fields.update(
+    {
+        "dagman_debug": (str, ""),
+        "dagman_node_record_info": (str, ""),
+        "dagman_record_machine_attrs": (str, ""),
+    }
+)
+
+# Dynamically create a Pydantic model encapsulating the DAGMan configuration
+# options gathered above.
+_DagmanOptions = create_model(
+    "DagmanOptions",
+    __config__=ConfigDict(
+        alias_generator=AliasGenerator(
+            serialization_alias=lambda name: name.upper(),
+        ),
+        extra="allow",
+        serialize_by_alias=True,
+    ),
+    **_fields,
+)
+
+
+class DagmanConfigurator:
+    """Class responsible for setting WMS-specific configuration options.
+
+    Parameters
+    ----------
+    config : `lsst.ctrl.bps.BpsConfig`
+        BPS configuration.
+    search_opts : `dict` [`str`, `Any`], optional
+        Options to use while searching the BPS configuration for values.
+
+    Raises
+    ------
+    KeyError
+        Raised if DAGMan configuration is missing from the BPS configuration.
+    """
+
+    def __init__(self, config: BpsConfig, search_opts: dict[str, Any] | None = None) -> None:
+        if search_opts is None:
+            search_opts = {}
+        _, site = config.search("computeSite", search_opts)
+        if site:
+            search_opts["curvals"] = {"curr_site": site}
+        _, wms_config = config.search("wmsConfig", search_opts)
+        if not wms_config:
+            raise KeyError("WMS-specific configuration not found")
+        self._options = _DagmanOptions.model_validate({key.lower(): val for key, val in wms_config.items()})
+        if self._options.model_extra:
+            unknown_opts = [key.upper() for key in self._options.model_extra]
+            _LOG.warning(
+                "The following WMS-specific config options were not recognized and will be ignored: %s.",
+                ", ".join(unknown_opts),
+            )
+        self.config_path: Path | None = None
+        self.prefix: Path | None = None
+
+    @property
+    def options(self) -> dict[str, Any]:
+        """DAGMan configuration options set via BPS (`dict` [`str`, `Any`])."""
+        return {
+            key: val
+            for key, val in self._options.model_dump(exclude_unset=True).items()
+            if key not in self._options.model_extra
+        }
+
+    def prepare(self, filename: os.PathLike | str, prefix: os.PathLike | str | None) -> None:
+        """Write WMS-specific configuration to a file.
+
+        Parameters
+        ----------
+        filename : `str`, optional
+            Name of the file to use when creating the DAG configuration.
+        prefix : `pathlib.Path` | `str`, optional
+            Directory in which to output the DAG configuration file. If not
+            provided, the script will be written to the current directory.
+
+        Raises
+        ------
+        OSError
+            Raised if the configuration file cannot be created.
+        """
+        if prefix:
+            self.prefix = Path(prefix)
+        self.config_path = self.prefix / filename if self.prefix else Path(filename)
+        try:
+            self.config_path.parent.mkdir(parents=True, exist_ok=True)
+        except OSError as exc:
+            _LOG.error(
+                "Could not write WMS-specific configuration file '%s': %s",
+                self.config_path,
+                exc.strerror,
+            )
+            raise
+
+        # Populate the DAG configuration file only with options that were
+        # explicitly set in the BPS configuration.
+        #
+        # Notes
+        # -----
+        # The Pydantic model we are using to represent the DAGMan configuration
+        # options allows for extra fields. However, it seems that
+        # BaseModel.model_dump() does not support excluding these fields during
+        # serialization at the moment (Pydantic ver. 2.12), so we have to do it
+        # manually.
+        self.config_path.write_text("\n".join(f"{key} = {val}" for key, val in self.options.items()))
+
+    def configure(self, dag: HTCDag) -> None:
+        """Add DAG configuration file to the workflow.
+
+        Parameters
+        ----------
+        dag : `lsst.ctrl.bps.htcondor.HTCDag`
+            HTCondor DAG.
+
+        Raises
+        ------
+        RuntimeError
+            Raised if the prepare step was omitted.
+
+        Notes
+        -----
+        The path to the DAG configuration is added as a DAG attribute named
+        ``bps_wms_config_path``. The stored path is relative to the prefix.
+        """
+        if self.config_path is None:
+            raise RuntimeError(
+                f"cannot add WMS-specific configuration to the workflow: file does not exist. "
+                f"(hint: run {type(self).__qualname__}.prepare() to create it)"
+            )
+        config_path = self.config_path.relative_to(self.prefix) if self.prefix else self.config_path
+        dag.add_attribs({"bps_wms_config_path": str(config_path)})

{lsst_ctrl_bps_htcondor-29.2025.4500 → lsst_ctrl_bps_htcondor-29.2025.4600}/python/lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml

@@ -43,3 +43,15 @@ provisionResources: false
 overwriteJobFiles: true
 finalJob:
   overwriteJobFiles: false
+
+# Define the default DAGMan configuration.
+wmsConfig:
+  # A comma separated list of variable names to add to the DAGMan *.condor.sub
+  # file’s getenv option. If set to "TRUE" (a string, not a boolean!), DAGMan
+  # will effectively lift all environmental variables from the user's shell
+  # environment.
+  DAGMAN_MANAGER_JOB_APPEND_GETENV: "TRUE"
+
+  # A boolean flag controlling whether DAGMan should generate submit files for
+  # nested DAGs automatically.
+  DAGMAN_GENERATE_SUBDAG_SUBMITS: true

{lsst_ctrl_bps_htcondor-29.2025.4500 → lsst_ctrl_bps_htcondor-29.2025.4600}/python/lsst/ctrl/bps/htcondor/htcondor_service.py

@@ -46,6 +46,7 @@ from lsst.daf.butler import Config
 from lsst.utils.timer import time_this
 
 from .common_utils import WmsIdType, _wms_id_to_cluster, _wms_id_to_dir, _wms_id_type
+from .dagman_configurator import DagmanConfigurator
 from .htcondor_config import HTC_DEFAULTS_URI
 from .htcondor_workflow import HTCondorWorkflow
 from .lssthtc import (
@@ -118,6 +119,17 @@ class HTCondorService(BaseWmsService):
                 provisioner.prepare("provisioningJob.bash", prefix=out_prefix)
                 provisioner.provision(workflow.dag)
 
+            try:
+                configurator = DagmanConfigurator(config)
+            except KeyError:
+                _LOG.debug(
+                    "No DAGMan-specific settings were found in BPS config; "
+                    "skipping writing DAG-specific configuration file."
+                )
+            else:
+                configurator.prepare("dagman.conf", prefix=out_prefix)
+                configurator.configure(workflow.dag)
+
         with time_this(
             log=_LOG, level=logging.INFO, prefix=None, msg="Completed writing out HTCondor workflow"
         ):

{lsst_ctrl_bps_htcondor-29.2025.4500 → lsst_ctrl_bps_htcondor-29.2025.4600}/python/lsst/ctrl/bps/htcondor/lssthtc.py

@@ -725,19 +725,6 @@ def htc_create_submit_from_dag(dag_filename: str, submit_options: dict[str, Any]
     Use with HTCondor versions which support htcondor.Submit.from_dag(),
     i.e., 8.9.3 or newer.
     """
-    # Passing do_recurse as submit_option does not seem to
-    # override DAGMAN_GENERATE_SUBDAG_SUBMITS as manual implies.
-    # So setting it and the other bps required setting here as
-    # environment variables if they don't exist.
-    var_name = "_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"
-    if var_name not in os.environ:
-        os.environ[var_name] = "True"
-
-    if "do_recurse" in submit_options:
-        var_name = "_CONDOR_DAGMAN_GENERATE_SUBDAG_SUBMITS"
-        if var_name not in os.environ:
-            os.environ[var_name] = str(submit_options["do_recurse"])
-
     # Config and environment variables do not seem to override -MaxIdle
     # on the .dag.condor.sub's command line (broken in some 24.0.x versions).
     # Explicitly forward them as a submit_option if either exists.
@@ -1164,7 +1151,15 @@ class HTCDag(networkx.DiGraph):
         self.graph["dag_filename"] = os.path.join(dag_subdir, f"{self.graph['name']}.dag")
         full_filename = os.path.join(submit_path, self.graph["dag_filename"])
         os.makedirs(os.path.dirname(full_filename), exist_ok=True)
+
+        try:
+            dagman_config_path = Path(self.graph["attr"]["bps_wms_config_path"])
+        except KeyError:
+            dagman_config_path = None
         with open(full_filename, "w") as fh:
+            if dagman_config_path is not None:
+                fh.write(f"CONFIG {dag_rel_path / dagman_config_path}\n")
+
             for name, nodeval in self.nodes().items():
                 try:
                     job = nodeval["data"]
@@ -1177,6 +1172,8 @@ class HTCDag(networkx.DiGraph):
                         subdir = job.dagcmds["dir"]
                     else:
                         subdir = job_subdir
+                    if dagman_config_path is not None:
+                        job.subdag.add_attribs({"bps_wms_config_path": str(dagman_config_path)})
                     job.subdag.write(submit_path, subdir, dag_subdir, "../..")
                     fh.write(
                         f"SUBDAG EXTERNAL {job.name} {Path(job.subdag.graph['dag_filename']).name} "
@@ -1468,7 +1465,17 @@ def count_jobs_in_single_dag(
     job_name_to_type: dict[str, WmsNodeType] = {}
     with open(filename) as fh:
         for line in fh:
-            job_name = ""
+            # Skip any line that contains commands irrelevant to job counting.
+            if not line.startswith(
+                (
+                    "JOB",
+                    "FINAL",
+                    "SERVICE",
+                    "SUBDAG EXTERNAL",
+                )
+            ):
+                continue
+
             m = re.match(
                 r"(?P<command>JOB|FINAL|SERVICE|SUBDAG EXTERNAL)\s+"
                 r'(?P<jobname>(?P<wms>wms_)?\S+)\s+"?(?P<subfile>\S+)"?\s*'
@@ -1524,9 +1531,9 @@ def count_jobs_in_single_dag(
 
                 job_name_to_label[job_name] = label
                 job_name_to_type[job_name] = job_type
-            elif not line.startswith(("VARS", "PARENT", "DOT", "NODE_STATUS_FILE", "SET_JOB_ATTR", "SCRIPT")):
-                # Only print warning if not a line wanting to skip
-                # Probably means problem with regex in above match pattern.
+            else:
+                # The line should, but didn't match the pattern above. Probably
+                # problems with regex.
                 _LOG.warning("Unexpected skipping of dag line: %s", line)
 
     return counts, job_name_to_label, job_name_to_type

{lsst_ctrl_bps_htcondor-29.2025.4500 → lsst_ctrl_bps_htcondor-29.2025.4600}/python/lsst/ctrl/bps/htcondor/prepare_utils.py

@@ -897,8 +897,6 @@ def _generic_workflow_to_htcondor_dag(
         elif gwjob.node_type == GenericWorkflowNodeType.GROUP:
             gwjob = cast(GenericWorkflowGroup, gwjob)
             htc_job = _group_to_subdag(config, gwjob, out_prefix)
-            # In case DAGMAN_GENERATE_SUBDAG_SUBMITS is False,
-            dag.graph["submit_options"]["do_recurse"] = True
         else:
             raise RuntimeError(f"Unsupported generic workflow node type {gwjob.node_type} ({gwjob.name})")
         _LOG.debug("Calling adding job %s %s", htc_job.name, htc_job.label)

lsst_ctrl_bps_htcondor-29.2025.4600/python/lsst/ctrl/bps/htcondor/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.4600"

{lsst_ctrl_bps_htcondor-29.2025.4500 → lsst_ctrl_bps_htcondor-29.2025.4600/python/lsst_ctrl_bps_htcondor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.4500
+Version: 29.2025.4600
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later
@@ -23,7 +23,10 @@ License-File: gpl-v3.0.txt
 Requires-Dist: htcondor>=8.8
 Requires-Dist: lsst-ctrl-bps
 Requires-Dist: lsst-daf-butler
+Requires-Dist: lsst-pipe-base
 Requires-Dist: lsst-utils
+Requires-Dist: packaging
+Requires-Dist: pydantic<3.0,>=2
 Provides-Extra: test
 Requires-Dist: pytest>=3.2; extra == "test"
 Requires-Dist: pytest-openfiles>=0.5.0; extra == "test"

{lsst_ctrl_bps_htcondor-29.2025.4500 → lsst_ctrl_bps_htcondor-29.2025.4600}/python/lsst_ctrl_bps_htcondor.egg-info/SOURCES.txt

@@ -11,6 +11,7 @@ doc/lsst.ctrl.bps.htcondor/index.rst
 doc/lsst.ctrl.bps.htcondor/userguide.rst
 python/lsst/ctrl/bps/htcondor/__init__.py
 python/lsst/ctrl/bps/htcondor/common_utils.py
+python/lsst/ctrl/bps/htcondor/dagman_configurator.py
 python/lsst/ctrl/bps/htcondor/final_post.sh
 python/lsst/ctrl/bps/htcondor/handlers.py
 python/lsst/ctrl/bps/htcondor/htcondor_config.py
@@ -30,6 +31,7 @@ python/lsst_ctrl_bps_htcondor.egg-info/requires.txt
 python/lsst_ctrl_bps_htcondor.egg-info/top_level.txt
 python/lsst_ctrl_bps_htcondor.egg-info/zip-safe
 tests/test_common_utils.py
+tests/test_dagman_configurator.py
 tests/test_handlers.py
 tests/test_htcondor_service.py
 tests/test_lssthtc.py

{lsst_ctrl_bps_htcondor-29.2025.4500 → lsst_ctrl_bps_htcondor-29.2025.4600}/python/lsst_ctrl_bps_htcondor.egg-info/requires.txt

@@ -1,7 +1,10 @@
 htcondor>=8.8
 lsst-ctrl-bps
 lsst-daf-butler
+lsst-pipe-base
 lsst-utils
+packaging
+pydantic<3.0,>=2
 
 [test]
 pytest>=3.2

lsst_ctrl_bps_htcondor-29.2025.4600/tests/test_dagman_configurator.py

@@ -0,0 +1,143 @@
+# This file is part of ctrl_bps_htcondor.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Unit tests for DagmanConfigurator class."""
+
+import logging
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from pydantic import ValidationError
+
+from lsst.ctrl.bps import BpsConfig
+from lsst.ctrl.bps.htcondor import HTCDag
+from lsst.ctrl.bps.htcondor.dagman_configurator import DagmanConfigurator
+
+logger = logging.getLogger("lsst.ctrl.bps.htcondor")
+
+
+class DagmanConfiguratorTestCase(unittest.TestCase):
+    """Unit tests for DagmanConfigurator class."""
+
+    def setUp(self):
+        self.config = BpsConfig(
+            {
+                "site": {
+                    "foo": {
+                        "wmsConfig": {"DAGMAN_USE_STRICT": 1},
+                    },
+                },
+                "wmsConfig": {"DAGMAN_USE_STRICT": 0},
+            }
+        )
+
+    def tearDown(self):
+        pass
+
+    def testInitDefaultSearchOptions(self):
+        """Test object instantiation with default search options."""
+        configurator = DagmanConfigurator(self.config)
+        self.assertIn("DAGMAN_USE_STRICT", configurator.options)
+        self.assertEqual(configurator.options["DAGMAN_USE_STRICT"], 0)
+        self.assertIsNone(configurator.config_path)
+        self.assertIsNone(configurator.prefix)
+
+    def testInitCustomSearchOptions(self):
+        """Test object instantiation with custom search options."""
+        configurator = DagmanConfigurator(self.config, search_opts={"curvals": {"computeSite": "foo"}})
+        self.assertIn("DAGMAN_USE_STRICT", configurator.options)
+        self.assertEqual(configurator.options["DAGMAN_USE_STRICT"], 1)
+        self.assertIsNone(configurator.config_path)
+        self.assertIsNone(configurator.prefix)
+
+    def testInitWrongOptionType(self):
+        self.config[".wmsConfig.DAGMAN_USE_STRICT"] = "foo"
+        with self.assertRaisesRegex(ValidationError, "DAGMAN_USE_STRICT".lower()):
+            DagmanConfigurator(self.config)
+
+    def testInitUnsupportedDagmanOption(self):
+        """Test object instantiation with unsupported DAGMAN options."""
+        self.config[".wmsConfig.DAGMAN_UNSUPPORTED_OPTION"] = "foo"
+        with self.assertLogs(logger=logger, level="WARNING") as cm:
+            configurator = DagmanConfigurator(self.config)
+            self.assertIn("DAGMAN_UNSUPPORTED_OPTION", cm.output[0])
+            self.assertNotIn("DAGMAN_UNSUPPORTED_OPTION", configurator.options)
+
+    def testInitNoWmsConfig(self):
+        """Test object instantiation fails when no WMS-specific options."""
+        del self.config["wmsConfig"]
+        with self.assertRaisesRegex(KeyError, "not found"):
+            DagmanConfigurator(self.config)
+
+    def testPrepare(self):
+        """Test if the method creates the configuration file."""
+        configurator = DagmanConfigurator(self.config)
+        with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
+            configurator.prepare("dagman.conf", prefix=tmpdir)
+            self.assertIn(Path(tmpdir), list(configurator.config_path.parents))
+            self.assertTrue(configurator.config_path.is_file())
+            self.assertEqual(configurator.config_path.read_text(), "DAGMAN_USE_STRICT = 0")
+
+    def testPrepareWithUnsupportedOption(self):
+        """Test if the method does not include unsupported options."""
+        self.config[".wmsConfig.DAGMAN_UNSUPPORTED_OPTION"] = "foo"
+        configurator = DagmanConfigurator(self.config)
+        with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
+            configurator.prepare("dagman.conf", prefix=tmpdir)
+            self.assertIn(Path(tmpdir), list(configurator.config_path.parents))
+            self.assertTrue(configurator.config_path.is_file())
+            self.assertEqual(configurator.config_path.read_text(), "DAGMAN_USE_STRICT = 0")
+
+    def testPrepareConfigWriteFailure(self):
+        """Test if the method raises when it can't create the configuration."""
+        with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
+            os.chmod(tmpdir, 0o500)
+
+            configurator = DagmanConfigurator(self.config)
+            with self.assertLogs(logger=logger, level="ERROR") as cm, self.assertRaises(OSError):
+                configurator.prepare("dagman.conf", f"{tmpdir}/subdir")
+            self.assertIn("Could not write", cm.output[0])
+
+            os.chmod(tmpdir, 0o700)
+
+    def testConfigure(self):
+        with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
+            dag = HTCDag(name="test_configure")
+            configurator = DagmanConfigurator(self.config)
+            configurator.prepare("dagman.conf", prefix=tmpdir)
+            configurator.configure(dag)
+        self.assertIn("bps_wms_config_path", dag.graph["attr"])
+        self.assertEqual(dag.graph["attr"]["bps_wms_config_path"], "dagman.conf")
+
+    def testConfigureIfNotPrepared(self):
+        """Test if the method raises when prepare step was skipped."""
+        dag = HTCDag(name="test_configure_not_prepared")
+        configurator = DagmanConfigurator(self.config)
+        with self.assertRaisesRegex(RuntimeError, "file does not exist"):
+            configurator.configure(dag)

{lsst_ctrl_bps_htcondor-29.2025.4500 → lsst_ctrl_bps_htcondor-29.2025.4600}/tests/test_lssthtc.py

@@ -38,7 +38,9 @@ from shutil import copy2, copytree, ignore_patterns, rmtree, which
 
 import htcondor
 
-from lsst.ctrl.bps.htcondor import lssthtc
+from lsst.ctrl.bps import BpsConfig
+from lsst.ctrl.bps.htcondor import dagman_configurator, htcondor_config, lssthtc
+from lsst.daf.butler import Config
 from lsst.utils.tests import temporaryDirectory
 
 logger = logging.getLogger("lsst.ctrl.bps.htcondor")
@@ -1190,7 +1192,6 @@ class HtcCreateSubmitFromDagTestCase(unittest.TestCase):
             dag_filename = pathlib.Path(tmp_dir) / "tiny_success.dag"
             submit = lssthtc.htc_create_submit_from_dag(str(dag_filename), {})
             self.assertIn("-MaxIdle 42", submit["arguments"])
-            self.assertEqual("true", os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"].lower())
 
     @unittest.mock.patch.dict(os.environ, {})
     def testMaxIdleGiven(self):
@@ -1199,7 +1200,6 @@ class HtcCreateSubmitFromDagTestCase(unittest.TestCase):
             dag_filename = pathlib.Path(tmp_dir) / "tiny_success.dag"
             submit = lssthtc.htc_create_submit_from_dag(str(dag_filename), {"MaxIdle": 37})
             self.assertIn("-MaxIdle 37", submit["arguments"])
-            self.assertEqual("true", os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"].lower())
 
     @unittest.mock.patch.dict(os.environ, {})
     def testNoMaxJobsIdle(self):
@@ -1215,37 +1215,90 @@ class HtcCreateSubmitFromDagTestCase(unittest.TestCase):
                 with unittest.mock.patch("htcondor.param") as mock_param:
                     mock_param.__contains__.return_value = False
                     _ = lssthtc.htc_create_submit_from_dag(str(dag_filename), {})
-                    self.assertEqual("true", os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"].lower())
                     submit_mock.assert_called_once_with(str(dag_filename), {})
 
-    @unittest.mock.patch.dict(os.environ, {})
-    def testDoRecurseGivenWithNoEnv(self):
-        with temporaryDirectory() as tmp_dir:
-            copy2(f"{TESTDIR}/data/tiny_success/tiny_success.dag", tmp_dir)
-            dag_filename = pathlib.Path(tmp_dir) / "tiny_success.dag"
-            submit = lssthtc.htc_create_submit_from_dag(str(dag_filename), {"do_recurse": True})
-            self.assertIn("-do_recurse", submit["arguments"])
-            self.assertEqual("true", os.environ["_CONDOR_DAGMAN_GENERATE_SUBDAG_SUBMITS"].lower())
-            self.assertEqual("true", os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"].lower())
 
-    @unittest.mock.patch.dict(os.environ, {"_CONDOR_DAGMAN_GENERATE_SUBDAG_SUBMITS": "False"})
-    def testDoRecurseGivenWithEnv(self):
+class HtcDagTestCase(unittest.TestCase):
+    """Test for HTCDag class."""
+
+    def setUp(self):
+        job = lssthtc.HTCJob(name="test_job")
+        job.add_job_cmds(
+            {
+                "executable": "/usr/bin/echo",
+                "arguments": "foo",
+                "output": "test_job.$(Cluster).out",
+                "error": "test_job.$(Cluster).out",
+                "log": "test_job.$(Cluster).log",
+            }
+        )
+        job.subfile = f"{job.name}.sub"
+
+        self.dag = lssthtc.HTCDag(name="test_workflow")
+        self.dag.add_job(job)
+
+        self.subfile_expected = [
+            "executable=/usr/bin/echo\n",
+            "arguments=foo\n",
+            "output=test_job.$(Cluster).out\n",
+            "error=test_job.$(Cluster).out\n",
+            "log=test_job.$(Cluster).log\n",
+            "queue\n",
+        ]
+
+    def tearDown(self):
+        pass
+
+    def testWriteWithDagConfig(self):
         with temporaryDirectory() as tmp_dir:
-            copy2(f"{TESTDIR}/data/tiny_success/tiny_success.dag", tmp_dir)
-            dag_filename = pathlib.Path(tmp_dir) / "tiny_success.dag"
-            submit = lssthtc.htc_create_submit_from_dag(str(dag_filename), {"do_recurse": True})
-            self.assertIn("-do_recurse", submit["arguments"])
-            self.assertEqual("true", os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"].lower())
-            self.assertEqual("false", os.environ["_CONDOR_DAGMAN_GENERATE_SUBDAG_SUBMITS"].lower())
+            config = BpsConfig(Config(htcondor_config.HTC_DEFAULTS_URI))
+            job = self.dag.nodes["test_job"]["data"]
+            wms_config_filename = "dagman.conf"
+            wms_configurator = dagman_configurator.DagmanConfigurator(config)
+            wms_configurator.prepare(wms_config_filename, prefix=tmp_dir)
+            wms_configurator.configure(self.dag)
+            dagfile_expected = [
+                f"CONFIG {wms_config_filename}\n",
+                f'JOB {job.name} "{job.subfile}"\n',
+                f"DOT {self.dag.name}.dot\n",
+                f"NODE_STATUS_FILE {self.dag.name}.node_status\n",
+                f'SET_JOB_ATTR bps_wms_config_path= "{wms_config_filename}"\n',
+            ]
+
+            self.dag.write(tmp_dir, "", "")
 
-    @unittest.mock.patch.dict(os.environ, {"_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV": "*_DIR"})
-    def testGetEnvOverridden(self):
+            self.assertIn("submit_path", self.dag.graph)
+            self.assertEqual(self.dag.graph["submit_path"], tmp_dir)
+            self.assertIn("dag_filename", self.dag.graph)
+            self.assertEqual(self.dag.graph["dag_filename"], f"{self.dag.graph['name']}.dag")
+            with open(os.path.join(tmp_dir, self.dag.graph["dag_filename"]), encoding="utf-8") as f:
+                dagfile_actual = f.readlines()
+                self.assertEqual(dagfile_actual, dagfile_expected)
+            with open(os.path.join(tmp_dir, job.subfile), encoding="utf-8") as f:
+                subfile_actual = f.readlines()
+                self.assertEqual(subfile_actual, self.subfile_expected)
+
+    def testWriteWithoutDagConfig(self):
         with temporaryDirectory() as tmp_dir:
-            copy2(f"{TESTDIR}/data/tiny_success/tiny_success.dag", tmp_dir)
-            dag_filename = pathlib.Path(tmp_dir) / "tiny_success.dag"
-            submit = lssthtc.htc_create_submit_from_dag(str(dag_filename), {"do_recurse": True})
-            self.assertIn("-do_recurse", submit["arguments"])
-            self.assertEqual(os.environ["_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"], "*_DIR")
+            job = self.dag.nodes["test_job"]["data"]
+            dagfile_expected = [
+                f'JOB {job.name} "{job.subfile}"\n',
+                f"DOT {self.dag.name}.dot\n",
+                f"NODE_STATUS_FILE {self.dag.name}.node_status\n",
+            ]
+
+            self.dag.write(tmp_dir, "", "")
+
+            self.assertIn("submit_path", self.dag.graph)
+            self.assertEqual(self.dag.graph["submit_path"], tmp_dir)
+            self.assertIn("dag_filename", self.dag.graph)
+            self.assertEqual(self.dag.graph["dag_filename"], f"{self.dag.graph['name']}.dag")
+            with open(os.path.join(tmp_dir, self.dag.graph["dag_filename"]), encoding="utf-8") as f:
+                dagfile_actual = f.readlines()
+                self.assertEqual(dagfile_actual, dagfile_expected)
+            with open(os.path.join(tmp_dir, job.subfile), encoding="utf-8") as f:
+                subfile_actual = f.readlines()
+                self.assertEqual(subfile_actual, self.subfile_expected)
 
 
 if __name__ == "__main__":

lsst_ctrl_bps_htcondor-29.2025.4500/python/lsst/ctrl/bps/htcondor/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.4500"