lsst-ctrl-bps-htcondor 29.2025.4500__py3-none-any.whl → 29.2025.4700__py3-none-any.whl

lsst/ctrl/bps/htcondor/dagman_configurator.py

@@ -0,0 +1,196 @@
+# This file is part of ctrl_bps_htcondor.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Module enabling configuring DAGMan via submit YAML."""
+
+__all__ = ["DagmanConfigurator"]
+
+import logging
+import os
+from pathlib import Path
+from typing import Any
+
+import htcondor
+from pydantic import AliasGenerator, ConfigDict, create_model
+
+from lsst.ctrl.bps import BpsConfig
+
+from .lssthtc import HTCDag
+
+_LOG = logging.getLogger(__name__)
+
+# Extract DAGMan configuration options with their types and default values from
+# the local HTCondor configuration.
+#
+# Notes
+# -----
+# There are some DAGMan configuration options that names do not start with
+# ``DAGMAN_`` (e.g., ``MAX_DAGMAN_LOG``). Hence, do not use
+# ``key.startswith("DAGMAN_")``.
+_fields = {key.lower(): (type(val), val) for key, val in htcondor.param.items() if "DAGMAN_" in key}
+
+# Add some valid configuration options are not set by default by HTCondor and
+# are missing from ``htcondor.param``.
+#
+# Notes
+# -----
+# A complete list of configuration options HTCondor supports can be found in
+# ``src/condor_utils/param_info.in`` in
+# `HTCondor GitHub repository <https://github.com/htcondor/htcondor>`_.
+_fields.update(
+    {
+        "dagman_debug": (str, ""),
+        "dagman_node_record_info": (str, ""),
+        "dagman_record_machine_attrs": (str, ""),
+    }
+)
+
+# Dynamically create a Pydantic model encapsulating the DAGMan configuration
+# options gathered above.
+_DagmanOptions = create_model(
+    "DagmanOptions",
+    __config__=ConfigDict(
+        alias_generator=AliasGenerator(
+            serialization_alias=lambda name: name.upper(),
+        ),
+        extra="allow",
+        serialize_by_alias=True,
+    ),
+    **_fields,
+)
+
+
+class DagmanConfigurator:
+    """Class responsible for setting WMS-specific configuration options.
+
+    Parameters
+    ----------
+    config : `lsst.ctrl.bps.BpsConfig`
+        BPS configuration.
+    search_opts : `dict` [`str`, `Any`], optional
+        Options to use while searching the BPS configuration for values.
+
+    Raises
+    ------
+    KeyError
+        Raised if DAGMan configuration is missing from the BPS configuration.
+    """
+
+    def __init__(self, config: BpsConfig, search_opts: dict[str, Any] | None = None) -> None:
+        if search_opts is None:
+            search_opts = {}
+        _, site = config.search("computeSite", search_opts)
+        if site:
+            search_opts["curvals"] = {"curr_site": site}
+        _, wms_config = config.search("wmsConfig", search_opts)
+        if not wms_config:
+            raise KeyError("WMS-specific configuration not found")
+        self._options = _DagmanOptions.model_validate({key.lower(): val for key, val in wms_config.items()})
+        if self._options.model_extra:
+            unknown_opts = [key.upper() for key in self._options.model_extra]
+            _LOG.warning(
+                "The following WMS-specific config options were not recognized and will be ignored: %s.",
+                ", ".join(unknown_opts),
+            )
+        self.config_path: Path | None = None
+        self.prefix: Path | None = None
+
+    @property
+    def options(self) -> dict[str, Any]:
+        """DAGMan configuration options set via BPS (`dict` [`str`, `Any`])."""
+        return {
+            key: val
+            for key, val in self._options.model_dump(exclude_unset=True).items()
+            if key not in self._options.model_extra
+        }
+
+    def prepare(self, filename: os.PathLike | str, prefix: os.PathLike | str | None) -> None:
+        """Write WMS-specific configuration to a file.
+
+        Parameters
+        ----------
+        filename : `str`, optional
+            Name of the file to use when creating the DAG configuration.
+        prefix : `pathlib.Path` | `str`, optional
+            Directory in which to output the DAG configuration file. If not
+            provided, the script will be written to the current directory.
+
+        Raises
+        ------
+        OSError
+            Raised if the configuration file cannot be created.
+        """
+        if prefix:
+            self.prefix = Path(prefix)
+        self.config_path = self.prefix / filename if self.prefix else Path(filename)
+        try:
+            self.config_path.parent.mkdir(parents=True, exist_ok=True)
+        except OSError as exc:
+            _LOG.error(
+                "Could not write WMS-specific configuration file '%s': %s",
+                self.config_path,
+                exc.strerror,
+            )
+            raise
+
+        # Populate the DAG configuration file only with options that were
+        # explicitly set in the BPS configuration.
+        #
+        # Notes
+        # -----
+        # The Pydantic model we are using to represent the DAGMan configuration
+        # options allows for extra fields. However, it seems that
+        # BaseModel.model_dump() does not support excluding these fields during
+        # serialization at the moment (Pydantic ver. 2.12), so we have to do it
+        # manually.
+        self.config_path.write_text("\n".join(f"{key} = {val}" for key, val in self.options.items()))
+
+    def configure(self, dag: HTCDag) -> None:
+        """Add DAG configuration file to the workflow.
+
+        Parameters
+        ----------
+        dag : `lsst.ctrl.bps.htcondor.HTCDag`
+            HTCondor DAG.
+
+        Raises
+        ------
+        RuntimeError
+            Raised if the prepare step was omitted.
+
+        Notes
+        -----
+        The path to the DAG configuration is added as a DAG attribute named
+        ``bps_wms_config_path``. The stored path is relative to the prefix.
+        """
+        if self.config_path is None:
+            raise RuntimeError(
+                f"cannot add WMS-specific configuration to the workflow: file does not exist. "
+                f"(hint: run {type(self).__qualname__}.prepare() to create it)"
+            )
+        config_path = self.config_path.relative_to(self.prefix) if self.prefix else self.config_path
+        dag.add_attribs({"bps_wms_config_path": str(config_path)})

lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml

@@ -43,3 +43,15 @@ provisionResources: false
 overwriteJobFiles: true
 finalJob:
   overwriteJobFiles: false
+
+# Define the default DAGMan configuration.
+wmsConfig:
+  # A comma separated list of variable names to add to the DAGMan *.condor.sub
+  # file’s getenv option. If set to "TRUE" (a string, not a boolean!), DAGMan
+  # will effectively lift all environmental variables from the user's shell
+  # environment.
+  DAGMAN_MANAGER_JOB_APPEND_GETENV: "TRUE"
+
+  # A boolean flag controlling whether DAGMan should generate submit files for
+  # nested DAGs automatically.
+  DAGMAN_GENERATE_SUBDAG_SUBMITS: true

lsst/ctrl/bps/htcondor/htcondor_service.py

@@ -46,6 +46,7 @@ from lsst.daf.butler import Config
 from lsst.utils.timer import time_this
 
 from .common_utils import WmsIdType, _wms_id_to_cluster, _wms_id_to_dir, _wms_id_type
+from .dagman_configurator import DagmanConfigurator
 from .htcondor_config import HTC_DEFAULTS_URI
 from .htcondor_workflow import HTCondorWorkflow
 from .lssthtc import (
@@ -118,6 +119,17 @@ class HTCondorService(BaseWmsService):
                 provisioner.prepare("provisioningJob.bash", prefix=out_prefix)
                 provisioner.provision(workflow.dag)
 
+            try:
+                configurator = DagmanConfigurator(config)
+            except KeyError:
+                _LOG.debug(
+                    "No DAGMan-specific settings were found in BPS config; "
+                    "skipping writing DAG-specific configuration file."
+                )
+            else:
+                configurator.prepare("dagman.conf", prefix=out_prefix)
+                configurator.configure(workflow.dag)
+
         with time_this(
             log=_LOG, level=logging.INFO, prefix=None, msg="Completed writing out HTCondor workflow"
         ):

lsst/ctrl/bps/htcondor/lssthtc.py

@@ -725,19 +725,6 @@ def htc_create_submit_from_dag(dag_filename: str, submit_options: dict[str, Any]
     Use with HTCondor versions which support htcondor.Submit.from_dag(),
     i.e., 8.9.3 or newer.
     """
-    # Passing do_recurse as submit_option does not seem to
-    # override DAGMAN_GENERATE_SUBDAG_SUBMITS as manual implies.
-    # So setting it and the other bps required setting here as
-    # environment variables if they don't exist.
-    var_name = "_CONDOR_DAGMAN_MANAGER_JOB_APPEND_GETENV"
-    if var_name not in os.environ:
-        os.environ[var_name] = "True"
-
-    if "do_recurse" in submit_options:
-        var_name = "_CONDOR_DAGMAN_GENERATE_SUBDAG_SUBMITS"
-        if var_name not in os.environ:
-            os.environ[var_name] = str(submit_options["do_recurse"])
-
     # Config and environment variables do not seem to override -MaxIdle
     # on the .dag.condor.sub's command line (broken in some 24.0.x versions).
     # Explicitly forward them as a submit_option if either exists.
@@ -1164,7 +1151,15 @@ class HTCDag(networkx.DiGraph):
         self.graph["dag_filename"] = os.path.join(dag_subdir, f"{self.graph['name']}.dag")
         full_filename = os.path.join(submit_path, self.graph["dag_filename"])
         os.makedirs(os.path.dirname(full_filename), exist_ok=True)
+
+        try:
+            dagman_config_path = Path(self.graph["attr"]["bps_wms_config_path"])
+        except KeyError:
+            dagman_config_path = None
         with open(full_filename, "w") as fh:
+            if dagman_config_path is not None:
+                fh.write(f"CONFIG {dag_rel_path / dagman_config_path}\n")
+
             for name, nodeval in self.nodes().items():
                 try:
                     job = nodeval["data"]
@@ -1177,6 +1172,8 @@ class HTCDag(networkx.DiGraph):
                         subdir = job.dagcmds["dir"]
                     else:
                         subdir = job_subdir
+                    if dagman_config_path is not None:
+                        job.subdag.add_attribs({"bps_wms_config_path": str(dagman_config_path)})
                     job.subdag.write(submit_path, subdir, dag_subdir, "../..")
                     fh.write(
                         f"SUBDAG EXTERNAL {job.name} {Path(job.subdag.graph['dag_filename']).name} "
@@ -1468,7 +1465,17 @@ def count_jobs_in_single_dag(
     job_name_to_type: dict[str, WmsNodeType] = {}
     with open(filename) as fh:
         for line in fh:
-            job_name = ""
+            # Skip any line that contains commands irrelevant to job counting.
+            if not line.startswith(
+                (
+                    "JOB",
+                    "FINAL",
+                    "SERVICE",
+                    "SUBDAG EXTERNAL",
+                )
+            ):
+                continue
+
             m = re.match(
                 r"(?P<command>JOB|FINAL|SERVICE|SUBDAG EXTERNAL)\s+"
                 r'(?P<jobname>(?P<wms>wms_)?\S+)\s+"?(?P<subfile>\S+)"?\s*'
@@ -1524,9 +1531,9 @@ def count_jobs_in_single_dag(
 
                 job_name_to_label[job_name] = label
                 job_name_to_type[job_name] = job_type
-            elif not line.startswith(("VARS", "PARENT", "DOT", "NODE_STATUS_FILE", "SET_JOB_ATTR", "SCRIPT")):
-                # Only print warning if not a line wanting to skip
-                # Probably means problem with regex in above match pattern.
+            else:
+                # The line should, but didn't match the pattern above. Probably
+                # problems with regex.
                 _LOG.warning("Unexpected skipping of dag line: %s", line)
 
     return counts, job_name_to_label, job_name_to_type

lsst/ctrl/bps/htcondor/prepare_utils.py

@@ -897,8 +897,6 @@ def _generic_workflow_to_htcondor_dag(
         elif gwjob.node_type == GenericWorkflowNodeType.GROUP:
             gwjob = cast(GenericWorkflowGroup, gwjob)
             htc_job = _group_to_subdag(config, gwjob, out_prefix)
-            # In case DAGMAN_GENERATE_SUBDAG_SUBMITS is False,
-            dag.graph["submit_options"]["do_recurse"] = True
         else:
             raise RuntimeError(f"Unsupported generic workflow node type {gwjob.node_type} ({gwjob.name})")
         _LOG.debug("Calling adding job %s %s", htc_job.name, htc_job.label)

lsst/ctrl/bps/htcondor/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.4500"
+__version__ = "29.2025.4700"

{lsst_ctrl_bps_htcondor-29.2025.4500.dist-info → lsst_ctrl_bps_htcondor-29.2025.4700.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.4500
+Version: 29.2025.4700
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later
@@ -23,7 +23,10 @@ License-File: gpl-v3.0.txt
 Requires-Dist: htcondor>=8.8
 Requires-Dist: lsst-ctrl-bps
 Requires-Dist: lsst-daf-butler
+Requires-Dist: lsst-pipe-base
 Requires-Dist: lsst-utils
+Requires-Dist: packaging
+Requires-Dist: pydantic<3.0,>=2
 Provides-Extra: test
 Requires-Dist: pytest>=3.2; extra == "test"
 Requires-Dist: pytest-openfiles>=0.5.0; extra == "test"

{lsst_ctrl_bps_htcondor-29.2025.4500.dist-info → lsst_ctrl_bps_htcondor-29.2025.4700.dist-info}/RECORD

@@ -1,23 +1,24 @@
 lsst/ctrl/bps/htcondor/__init__.py,sha256=8PVRtHS2tn_BDH9gnWQ-Fbg7o7wTbZoW7f41RGf0g7A,1450
 lsst/ctrl/bps/htcondor/common_utils.py,sha256=cHdPaOwS2I5T4RpGChdMAaqis7T3B2dR45zu8cVzpto,9744
+lsst/ctrl/bps/htcondor/dagman_configurator.py,sha256=YKqQr7_8sTMvrhorsQQEShxPNQTBFYI0vo62tpasihs,7231
 lsst/ctrl/bps/htcondor/final_post.sh,sha256=chfaQV6Q7rGsK-8Hx58ch52m-PofvBanrl7VwCssHec,248
 lsst/ctrl/bps/htcondor/handlers.py,sha256=fkTEKulfwYOMofya9PzbvCiI9WNLfj_yTnno8Sm3srQ,14860
 lsst/ctrl/bps/htcondor/htcondor_config.py,sha256=c4lCiYEwEXFdxgbMfEkbDm4LrvkRMF31SqLtQqzqIV4,1523
-lsst/ctrl/bps/htcondor/htcondor_service.py,sha256=dnpxje5XRI0TEui-oXdp9kKlnTMiOZZNk0jDJIjNFDE,22177
+lsst/ctrl/bps/htcondor/htcondor_service.py,sha256=raGVthF4Q92B0VKeE5T1eNzOnmL6u4ni2Dcr3gThvd8,22671
 lsst/ctrl/bps/htcondor/htcondor_workflow.py,sha256=wkANkAA4Ciq9WP_DWkjH2k0xWz9_i6gaNHWcxWQ4zkM,3071
-lsst/ctrl/bps/htcondor/lssthtc.py,sha256=c7eecmDgEkfxASwPJ645rireM0Pe6WdMImnUT_3y-SA,81454
-lsst/ctrl/bps/htcondor/prepare_utils.py,sha256=rkh41PGijZuO-LbxBpM-XQOAn9srK_rjVWPIj4n9j9c,36738
+lsst/ctrl/bps/htcondor/lssthtc.py,sha256=2XVwlDByOHvOyB9yrjY_UJeoZgmh_OLVhg782PLC-fg,81485
+lsst/ctrl/bps/htcondor/prepare_utils.py,sha256=yjd31cG28zpvkDbyrI6gtRnkF2Wnzz_HzBEAyH_3zbs,36614
 lsst/ctrl/bps/htcondor/provisioner.py,sha256=DxhCOCpqyBXIBR2m8VL_FwaDMr2scQIOe8ArWjgQ_Ls,7929
 lsst/ctrl/bps/htcondor/report_utils.py,sha256=7idf6GRfMhCt5OuuCzHnnhPq5YVYm8k0bqS301MEjC8,31123
-lsst/ctrl/bps/htcondor/version.py,sha256=ec8d6sw9aRuINU5Qs4LjOV9LhqSISytoOfEoRYO2XzU,55
+lsst/ctrl/bps/htcondor/version.py,sha256=xCUcvtDm7ffryaF3_oc2AhBXCqIFOubJb8jOIg_6D6E,55
 lsst/ctrl/bps/htcondor/etc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml,sha256=C6DKJKmKFKczukpXVXev9u1-vmv2IcgcdtjTtgJWDQM,1561
-lsst_ctrl_bps_htcondor-29.2025.4500.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
-lsst_ctrl_bps_htcondor-29.2025.4500.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps_htcondor-29.2025.4500.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps_htcondor-29.2025.4500.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps_htcondor-29.2025.4500.dist-info/METADATA,sha256=oUkqyydYwm4_SrWvV65WD51DDCOD-NXUpUs90FGoul0,2213
-lsst_ctrl_bps_htcondor-29.2025.4500.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_ctrl_bps_htcondor-29.2025.4500.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps_htcondor-29.2025.4500.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps_htcondor-29.2025.4500.dist-info/RECORD,,
+lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml,sha256=0mIBmXBy5TBBtw1g6RL5TNjbnWM_ADeLp94r1-gfwAg,2061
+lsst_ctrl_bps_htcondor-29.2025.4700.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
+lsst_ctrl_bps_htcondor-29.2025.4700.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps_htcondor-29.2025.4700.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps_htcondor-29.2025.4700.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps_htcondor-29.2025.4700.dist-info/METADATA,sha256=NyRVqSZBmYe6ooKa84VMEUIpGPB6pV3LR9KN9JX3Ddk,2300
+lsst_ctrl_bps_htcondor-29.2025.4700.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_ctrl_bps_htcondor-29.2025.4700.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps_htcondor-29.2025.4700.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps_htcondor-29.2025.4700.dist-info/RECORD,,