lsst-ctrl-bps-htcondor 29.2025.2000__py3-none-any.whl → 29.2025.2200__py3-none-any.whl

lsst/ctrl/bps/htcondor/htcondor_service.py

@@ -34,6 +34,7 @@ import logging
 import os
 import re
 from collections import defaultdict
+from copy import deepcopy
 from enum import IntEnum, auto
 from pathlib import Path
 from typing import Any, cast
@@ -331,7 +332,7 @@ class HTCondorService(BaseWmsService):
 
         Returns
         -------
-        job_ids : `list` [`Any`]
+        job_ids : `list` [`~typing.Any`]
             Only job ids to be used by cancel and other functions.  Typically
             this means top-level jobs (i.e., not children jobs).
         """
@@ -400,6 +401,54 @@ class HTCondorService(BaseWmsService):
         _LOG.debug("job_ids = %s", job_ids)
         return job_ids
 
+    def get_status(
+        self,
+        wms_workflow_id: str,
+        hist: float = 1,
+        is_global: bool = False,
+    ) -> tuple[WmsStates, str]:
+        """Return status of run based upon given constraints.
+
+        Parameters
+        ----------
+        wms_workflow_id : `str`
+            Limit to specific run based on id (queue id or path).
+        hist : `float`, optional
+            Limit history search to this many days. Defaults to 1.
+        is_global : `bool`, optional
+            If set, all job queues (and their histories) will be queried for
+            job information. Defaults to False which means that only the local
+            job queue will be queried.
+
+        Returns
+        -------
+        state : `lsst.ctrl.bps.WmsStates`
+            Status of single run from given information.
+        message : `str`
+            Extra message for status command to print.  This could be pointers
+            to documentation or to WMS specific commands.
+        """
+        _LOG.debug("get_status: id=%s, hist=%s, is_global=%s", wms_workflow_id, hist, is_global)
+
+        id_type = _wms_id_type(wms_workflow_id)
+        _LOG.debug("id_type = %s", id_type.name)
+
+        if id_type == WmsIdType.LOCAL:
+            schedulers = _locate_schedds(locate_all=is_global)
+            _LOG.debug("schedulers = %s", schedulers)
+            state, message = _get_status_from_id(wms_workflow_id, hist, schedds=schedulers)
+        elif id_type == WmsIdType.GLOBAL:
+            schedulers = _locate_schedds(locate_all=True)
+            _LOG.debug("schedulers = %s", schedulers)
+            state, message = _get_status_from_id(wms_workflow_id, hist, schedds=schedulers)
+        elif id_type == WmsIdType.PATH:
+            state, message = _get_status_from_path(wms_workflow_id)
+        else:
+            state, message = WmsStates.UNKNOWN, "Invalid job id"
+        _LOG.debug("state: %s, %s", state, message)
+
+        return state, message
+
     def report(
         self,
         wms_workflow_id=None,
@@ -604,15 +653,15 @@ class HTCondorWorkflow(BaseWmsWorkflow):
         self.dag.write(out_prefix, job_subdir="jobs/{self.label}")
 
 
-def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefix):
+def _create_job(subdir_template, cached_values, generic_workflow, gwjob, out_prefix):
     """Convert GenericWorkflow job nodes to DAG jobs.
 
     Parameters
     ----------
     subdir_template : `str`
         Template for making subdirs.
-    site_values : `dict`
-        Site specific values
+    cached_values : `dict`
+        Site and label specific values.
     generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
         Generic workflow that is being converted.
     gwjob : `lsst.ctrl.bps.GenericWorkflowJob`
@@ -654,7 +703,7 @@ def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefi
         "on_exit_hold_subcode": "34",
     }
 
-    htc_job_cmds.update(_translate_job_cmds(site_values, generic_workflow, gwjob))
+    htc_job_cmds.update(_translate_job_cmds(cached_values, generic_workflow, gwjob))
 
     # job stdout, stderr, htcondor user log.
     for key in ("output", "error", "log"):
@@ -662,7 +711,7 @@ def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefi
         _LOG.debug("HTCondor %s = %s", key, htc_job_cmds[key])
 
     htc_job_cmds.update(
-        _handle_job_inputs(generic_workflow, gwjob.name, site_values["bpsUseShared"], out_prefix)
+        _handle_job_inputs(generic_workflow, gwjob.name, cached_values["bpsUseShared"], out_prefix)
     )
 
     # Add the job cmds dict to the job object.
@@ -673,7 +722,7 @@ def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefi
     # Add job attributes to job.
     _LOG.debug("gwjob.attrs = %s", gwjob.attrs)
     htc_job.add_job_attrs(gwjob.attrs)
-    htc_job.add_job_attrs(site_values["attrs"])
+    htc_job.add_job_attrs(cached_values["attrs"])
     htc_job.add_job_attrs({"bps_job_quanta": create_count_summary(gwjob.quanta_counts)})
     htc_job.add_job_attrs({"bps_job_name": gwjob.name, "bps_job_label": gwjob.label})
 
@@ -685,8 +734,8 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
 
     Parameters
     ----------
-    cached_vals : `dict` [`str`, `Any`]
-        Config values common to jobs with same label.
+    cached_vals : `dict` [`str`, `~typing.Any`]
+        Config values common to jobs with same site or label.
     generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
         Generic workflow that contains job to being converted.
     gwjob : `lsst.ctrl.bps.GenericWorkflowJob`
@@ -694,7 +743,7 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
 
     Returns
     -------
-    htc_job_commands : `dict` [`str`, `Any`]
+    htc_job_commands : `dict` [`str`, `~typing.Any`]
         Contains commands which can appear in the HTCondor submit description
         file.
     """
@@ -720,9 +769,6 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
         jobcmds["accounting_group_user"] = cached_vals.get("accountingUser")
 
     # job commands that need modification
-    if gwjob.number_of_retries:
-        jobcmds["max_retries"] = f"{gwjob.number_of_retries}"
-
     if gwjob.retry_unless_exit:
         if isinstance(gwjob.retry_unless_exit, int):
             jobcmds["retry_until"] = f"{gwjob.retry_unless_exit}"
@@ -739,6 +785,7 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
     if gwjob.request_memory:
         jobcmds["request_memory"] = f"{gwjob.request_memory}"
 
+    memory_max = 0
     if gwjob.memory_multiplier:
         # Do not use try-except! At the moment, BpsConfig returns an empty
         # string if it does not contain the key.
@@ -765,13 +812,18 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
             gwjob.request_memory, gwjob.memory_multiplier, memory_max
         )
 
-        # Periodically release jobs which are being held due to exceeding
-        # memory. Stop doing that (by removing the job from the HTCondor queue)
-        # after the maximal number of retries has been reached or the job was
-        # already run at maximal allowed memory.
-        jobcmds["periodic_release"] = _create_periodic_release_expr(
-            gwjob.request_memory, gwjob.memory_multiplier, memory_max
-        )
+    user_release_expr = cached_vals.get("releaseExpr", "")
+    if gwjob.number_of_retries is not None and gwjob.number_of_retries >= 0:
+        jobcmds["max_retries"] = gwjob.number_of_retries
+
+        # No point in adding periodic_release if 0 retries
+        if gwjob.number_of_retries > 0:
+            periodic_release = _create_periodic_release_expr(
+                gwjob.request_memory, gwjob.memory_multiplier, memory_max, user_release_expr
+            )
+            if periodic_release:
+                jobcmds["periodic_release"] = periodic_release
+
         jobcmds["periodic_remove"] = _create_periodic_remove_expr(
             gwjob.request_memory, gwjob.memory_multiplier, memory_max
         )
@@ -830,7 +882,7 @@ def _translate_dag_cmds(gwjob):
 
     Returns
     -------
-    dagcmds : `dict` [`str`, `Any`]
+    dagcmds : `dict` [`str`, `~typing.Any`]
         DAGMan commands for the job.
     """
     # Values in the dag script that just are name mappings.
@@ -1010,6 +1062,77 @@ def _handle_job_inputs(generic_workflow: GenericWorkflow, job_name: str, use_sha
     return htc_commands
 
 
+def _get_status_from_id(
+    wms_workflow_id: str, hist: float, schedds: dict[str, htcondor.Schedd]
+) -> tuple[WmsStates, str]:
+    """Gather run information using workflow id.
+
+    Parameters
+    ----------
+    wms_workflow_id : `str`
+        Limit to specific run based on id.
+    hist : `float`
+        Limit history search to this many days.
+    schedds : `dict` [ `str`, `htcondor.Schedd` ]
+        HTCondor schedulers which to query for job information. If empty
+        dictionary, all queries will be run against the local scheduler only.
+
+    Returns
+    -------
+    state : `lsst.ctrl.bps.WmsStates`
+        Status for the corresponding run.
+    message : `str`
+        Message with extra error information.
+    """
+    _LOG.debug("_get_status_from_id: id=%s, hist=%s, schedds=%s", wms_workflow_id, hist, schedds)
+
+    message = ""
+
+    # Collect information about the job by querying HTCondor schedd and
+    # HTCondor history.
+    schedd_dag_info = _get_info_from_schedd(wms_workflow_id, hist, schedds)
+    if len(schedd_dag_info) == 1:
+        schedd_name = next(iter(schedd_dag_info))
+        dag_id = next(iter(schedd_dag_info[schedd_name]))
+        dag_ad = schedd_dag_info[schedd_name][dag_id]
+        state = _htc_status_to_wms_state(dag_ad)
+    else:
+        state = WmsStates.UNKNOWN
+        message = f"DAGMan job {wms_workflow_id} not found in queue or history.  Check id or try path."
+    return state, message
+
+
+def _get_status_from_path(wms_path: str | os.PathLike) -> tuple[WmsStates, str]:
+    """Gather run status from a given run directory.
+
+    Parameters
+    ----------
+    wms_path : `str` | `os.PathLike`
+        The directory containing the submit side files (e.g., HTCondor files).
+
+    Returns
+    -------
+    state : `lsst.ctrl.bps.WmsStates`
+        Status for the run.
+    message : `str`
+        Message to be printed.
+    """
+    wms_path = Path(wms_path).resolve()
+    message = ""
+    try:
+        wms_workflow_id, dag_ad = read_dag_log(wms_path)
+    except FileNotFoundError:
+        wms_workflow_id = MISSING_ID
+        message = f"DAGMan log not found in {wms_path}.  Check path."
+
+    if wms_workflow_id == MISSING_ID:
+        state = WmsStates.UNKNOWN
+    else:
+        state = _htc_status_to_wms_state(dag_ad[wms_workflow_id])
+
+    return state, message
+
+
 def _report_from_path(wms_path):
     """Gather run information from a given run directory.
 
@@ -1135,11 +1258,11 @@ def _get_info_from_schedd(
     ----------
     wms_workflow_id : `str`
         Limit to specific run based on id.
-    hist : `int`
+    hist : `float`
         Limit history search to this many days.
-    schedds : `dict` [ `str`, `htcondor.Schedd` ], optional
-        HTCondor schedulers which to query for job information. If None
-        (default), all queries will be run against the local scheduler only.
+    schedds : `dict` [ `str`, `htcondor.Schedd` ]
+        HTCondor schedulers which to query for job information. If empty
+        dictionary, all queries will be run against the local scheduler only.
 
     Returns
     -------
@@ -1148,6 +1271,8 @@ def _get_info_from_schedd(
         Scheduler, local HTCondor job ids are mapped to their respective
         classads.
     """
+    _LOG.debug("_get_info_from_schedd: id=%s, hist=%s, schedds=%s", wms_workflow_id, hist, schedds)
+
     dag_constraint = 'regexp("dagman$", Cmd)'
     try:
         cluster_id = int(float(wms_workflow_id))
@@ -1180,7 +1305,7 @@ def _get_info_from_path(wms_path: str | os.PathLike) -> tuple[str, dict[str, dic
     -------
     wms_workflow_id : `str`
         The run id which is a DAGman job id.
-    jobs : `dict` [`str`, `dict` [`str`, `Any`]]
+    jobs : `dict` [`str`, `dict` [`str`, `~typing.Any`]]
         Information about jobs read from files in the given directory.
         The key is the HTCondor id and the value is a dictionary of HTCondor
         keys and values.
@@ -1340,7 +1465,7 @@ def _add_service_job_specific_info(job_ad: dict[str, Any], specific_info: WmsSpe
 
     Parameters
     ----------
-    job_ad : `dict` [`str`, `Any`]
+    job_ad : `dict` [`str`, `~typing.Any`]
         Provisioning job information.
     specific_info : `lsst.ctrl.bps.WmsSpecificInfo`
         Where to add message.
@@ -1466,7 +1591,7 @@ def _add_run_info(wms_path, job):
     ----------
     wms_path : `str`
         Path to submit files for the run.
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Raises
@@ -1502,7 +1627,7 @@ def _get_owner(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Returns
@@ -1524,7 +1649,7 @@ def _get_run_summary(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Returns
@@ -1600,7 +1725,7 @@ def _get_state_counts_from_jobs(
     ----------
     wms_workflow_id : `str`
         HTCondor job id.
-    jobs : `dict [`dict` [`str`, `Any`]]
+    jobs : `dict [`dict` [`str`, `~typing.Any`]]
         HTCondor dag job information.
 
     Returns
@@ -1628,7 +1753,7 @@ def _get_state_counts_from_dag_job(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Returns
@@ -1684,7 +1809,7 @@ def _htc_status_to_wms_state(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor job information.
 
     Returns
@@ -1706,7 +1831,7 @@ def _htc_job_status_to_wms_state(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor job information.
 
     Returns
@@ -1748,7 +1873,7 @@ def _htc_node_status_to_wms_state(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor job information.
 
     Returns
@@ -1795,9 +1920,9 @@ def _update_jobs(jobs1, jobs2):
 
     Parameters
     ----------
-    jobs1 : `dict` [`str`, `dict` [`str`, `Any`]]
+    jobs1 : `dict` [`str`, `dict` [`str`, `~typing.Any`]]
         HTCondor job information to be updated.
-    jobs2 : `dict` [`str`, `dict` [`str`, `Any`]]
+    jobs2 : `dict` [`str`, `dict` [`str`, `~typing.Any`]]
         Additional HTCondor job information.
     """
     for job_id, job_ad in jobs2.items():
@@ -1937,34 +2062,39 @@ def _wms_id_to_dir(wms_id):
     return wms_path, id_type
 
 
-def _create_periodic_release_expr(memory, multiplier, limit):
+def _create_periodic_release_expr(
+    memory: int, multiplier: float | None, limit: int, additional_expr: str = ""
+) -> str:
     """Construct an HTCondorAd expression for releasing held jobs.
 
-    The expression instruct HTCondor to release any job which was put on hold
-    due to exceeding memory requirements back to the job queue providing it
-    satisfies all of the conditions below:
-
-    * number of run attempts did not reach allowable number of retries,
-    * the memory requirements in the last failed run attempt did not reach
-      the specified memory limit.
-
     Parameters
     ----------
     memory : `int`
         Requested memory in MB.
-    multiplier : `float`
-        Memory growth rate between retires.
+    multiplier : `float` or None
+        Memory growth rate between retries.
     limit : `int`
         Memory limit.
+    additional_expr : `str`, optional
+        Expression to add to periodic_release.  Defaults to empty string.
 
     Returns
     -------
     expr : `str`
-        A string representing an HTCondor ClassAd expression for releasing jobs
-        which have been held due to exceeding the memory requirements.
+        A string representing an HTCondor ClassAd expression for releasing job.
     """
-    is_retry_allowed = "NumJobStarts <= JobMaxRetries"
-    was_below_limit = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) < {limit}"
+    _LOG.debug(
+        "periodic_release: memory: %s, multiplier: %s, limit: %s, additional_expr: %s",
+        memory,
+        multiplier,
+        limit,
+        additional_expr,
+    )
+
+    # ctrl_bps sets multiplier to None in the GenericWorkflow if
+    # memoryMultiplier <= 1, but checking value just in case.
+    if (not multiplier or multiplier <= 1) and not additional_expr:
+        return ""
 
     # Job ClassAds attributes 'HoldReasonCode' and 'HoldReasonSubCode' are
     # UNDEFINED if job is not HELD (i.e. when 'JobStatus' is not 5).
@@ -1976,63 +2106,74 @@ def _create_periodic_release_expr(memory, multiplier, limit):
     # the entire expression should evaluate to FALSE when the job is not HELD.
     # According to ClassAd evaluation semantics FALSE && UNDEFINED is FALSE,
     # but better safe than sorry.
-    was_mem_exceeded = (
-        "JobStatus == 5 "
-        "&& (HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
-        "|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34)"
-    )
+    is_held = "JobStatus == 5"
+    is_retry_allowed = "NumJobStarts <= JobMaxRetries"
+
+    mem_expr = ""
+    if memory and multiplier and multiplier > 1 and limit:
+        was_mem_exceeded = (
+            "(HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
+            "|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34)"
+        )
+        was_below_limit = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) < {limit}"
+        mem_expr = f"{was_mem_exceeded} && {was_below_limit}"
+
+    user_expr = ""
+    if additional_expr:
+        # Never auto release a job held by user.
+        user_expr = f"HoldReasonCode =!= 1 && {additional_expr}"
+
+    expr = f"{is_held} && {is_retry_allowed}"
+    if user_expr and mem_expr:
+        expr += f" && ({mem_expr} || {user_expr})"
+    elif user_expr:
+        expr += f" && {user_expr}"
+    elif mem_expr:
+        expr += f" && {mem_expr}"
 
-    expr = f"{was_mem_exceeded} && {is_retry_allowed} && {was_below_limit}"
     return expr
 
 
 def _create_periodic_remove_expr(memory, multiplier, limit):
     """Construct an HTCondorAd expression for removing jobs from the queue.
 
-    The expression instruct HTCondor to remove any job which was put on hold
-    due to exceeding memory requirements from the job queue providing it
-    satisfies any of the conditions below:
-
-    * allowable number of retries was reached,
-    * the memory requirements during the last failed run attempt reached
-      the specified memory limit.
-
     Parameters
     ----------
     memory : `int`
         Requested memory in MB.
     multiplier : `float`
-        Memory growth rate between retires.
+        Memory growth rate between retries.
     limit : `int`
         Memory limit.
 
     Returns
     -------
     expr : `str`
-        A string representing an HTCondor ClassAd expression for removing jobs
-        which were run at the maximal allowable memory and still exceeded
-        the memory requirements.
+        A string representing an HTCondor ClassAd expression for removing jobs.
     """
-    is_retry_disallowed = "NumJobStarts > JobMaxRetries"
-    was_limit_reached = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) == {limit}"
-
-    # Job ClassAds attributes 'HoldReasonCode' and 'HoldReasonSubCode' are
-    # UNDEFINED if job is not HELD (i.e. when 'JobStatus' is not 5).
-    # The special comparison operators ensure that all comparisons below will
-    # evaluate to FALSE in this case.
+    # Job ClassAds attributes 'HoldReasonCode' and 'HoldReasonSubCode'
+    # are UNDEFINED if job is not HELD (i.e. when 'JobStatus' is not 5).
+    # The special comparison operators ensure that all comparisons below
+    # will evaluate to FALSE in this case.
     #
     # Note:
-    # May not be strictly necessary. Operators '&&' and '||' are not strict so
-    # the entire expression should evaluate to FALSE when the job is not HELD.
-    # According to ClassAd evaluation semantics FALSE && UNDEFINED is FALSE,
-    # but better safe than sorry.
-    was_mem_exceeded = (
-        "JobStatus == 5 "
-        "&& (HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
-        "|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34)"
-    )
+    # May not be strictly necessary. Operators '&&' and '||' are not
+    # strict so the entire expression should evaluate to FALSE when the
+    # job is not HELD. According to ClassAd evaluation semantics
+    # FALSE && UNDEFINED is FALSE, but better safe than sorry.
+    is_held = "JobStatus == 5"
+    is_retry_disallowed = "NumJobStarts > JobMaxRetries"
+
+    mem_expr = ""
+    if memory and multiplier and multiplier > 1 and limit:
+        mem_limit_expr = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) == {limit}"
+
+        mem_expr = (  # Add || here so only added if adding memory expr
+            " || ((HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
+            f"|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34) && {mem_limit_expr})"
+        )
 
-    expr = f"{was_mem_exceeded} && ({is_retry_disallowed} || {was_limit_reached})"
+    expr = f"{is_held} && ({is_retry_disallowed}{mem_expr})"
     return expr
 
 
@@ -2044,7 +2185,7 @@ def _create_request_memory_expr(memory, multiplier, limit):
     memory : `int`
         Requested memory in MB.
     multiplier : `float`
-        Memory growth rate between retires.
+        Memory growth rate between retries.
     limit : `int`
         Memory limit.
 
@@ -2119,7 +2260,7 @@ def _gather_site_values(config, compute_site):
 
     Returns
     -------
-    site_values : `dict` [`str`, `Any`]
+    site_values : `dict` [`str`, `~typing.Any`]
         Values specific to the given site.
     """
     site_values = {"attrs": {}, "profile": {}}
@@ -2167,6 +2308,50 @@ def _gather_site_values(config, compute_site):
     return site_values
 
 
+def _gather_label_values(config: BpsConfig, label: str) -> dict[str, Any]:
+    """Gather values specific to given job label.
+
+    Parameters
+    ----------
+    config : `lsst.ctrl.bps.BpsConfig`
+        BPS configuration that includes necessary submit/runtime
+        information.
+    label : `str`
+        GenericWorkflowJob label.
+
+    Returns
+    -------
+    values : `dict` [`str`, `~typing.Any`]
+        Values specific to the given job label.
+    """
+    values: dict[str, Any] = {"attrs": {}, "profile": {}}
+
+    search_opts = {}
+    profile_key = ""
+    if label == "finalJob":
+        search_opts["searchobj"] = config["finalJob"]
+        profile_key = ".finalJob.profile.condor"
+    elif label in config["cluster"]:
+        search_opts["curvals"] = {"curr_cluster": label}
+        profile_key = f".cluster.{label}.profile.condor"
+    elif label in config["pipetask"]:
+        search_opts["curvals"] = {"curr_pipetask": label}
+        profile_key = f".pipetask.{label}.profile.condor"
+
+    found, value = config.search("releaseExpr", opt=search_opts)
+    if found:
+        values["releaseExpr"] = value
+
+    if profile_key and profile_key in config:
+        for subkey, val in config[profile_key].items():
+            if subkey.startswith("+"):
+                values["attrs"][subkey[1:]] = val
+            else:
+                values["profile"][subkey] = val
+
+    return values
+
+
 def is_service_job(job_ad: dict[str, Any]) -> bool:
     """Determine if a job is a service one.
 
@@ -2280,16 +2465,22 @@ def _generic_workflow_to_htcondor_dag(
         subdir_template = tmp_template
 
     # Create all DAG jobs
-    site_values = {}  # cache compute site specific values to reduce config lookups
+    site_values = {}  # Cache compute site specific values to reduce config lookups.
+    cached_values = {}  # Cache label-specific values to reduce config lookups.
+    # Note: Can't use get_job_by_label because those only include payload jobs.
     for job_name in generic_workflow:
         gwjob = generic_workflow.get_job(job_name)
         if gwjob.node_type == GenericWorkflowNodeType.PAYLOAD:
             gwjob = cast(GenericWorkflowJob, gwjob)
             if gwjob.compute_site not in site_values:
                 site_values[gwjob.compute_site] = _gather_site_values(config, gwjob.compute_site)
+            if gwjob.label not in cached_values:
+                cached_values[gwjob.label] = deepcopy(site_values[gwjob.compute_site])
+                cached_values[gwjob.label].update(_gather_label_values(config, gwjob.label))
+                _LOG.debug("cached: %s= %s", gwjob.label, cached_values[gwjob.label])
             htc_job = _create_job(
                 subdir_template[gwjob.label],
-                site_values[gwjob.compute_site],
+                cached_values[gwjob.label],
                 generic_workflow,
                 gwjob,
                 out_prefix,
@@ -2351,8 +2542,15 @@ def _generic_workflow_to_htcondor_dag(
     if final and isinstance(final, GenericWorkflowJob):
         if final.compute_site and final.compute_site not in site_values:
             site_values[final.compute_site] = _gather_site_values(config, final.compute_site)
+        if final.label not in cached_values:
+            cached_values[final.label] = deepcopy(site_values[final.compute_site])
+            cached_values[final.label].update(_gather_label_values(config, final.label))
         final_htjob = _create_job(
-            subdir_template[final.label], site_values[final.compute_site], generic_workflow, final, out_prefix
+            subdir_template[final.label],
+            cached_values[final.label],
+            generic_workflow,
+            final,
+            out_prefix,
         )
         if "post" not in final_htjob.dagcmds:
             final_htjob.dagcmds["post"] = {

lsst/ctrl/bps/htcondor/lssthtc.py

@@ -1376,6 +1376,7 @@ def condor_search(constraint=None, hist=None, schedds=None):
 
     job_info = condor_q(constraint=constraint, schedds=schedds)
     if hist is not None:
+        _LOG.debug("Searching history going back %s days", hist)
         epoch = (datetime.now() - timedelta(days=hist)).timestamp()
         constraint += f" && (CompletionDate >= {epoch} || JobFinishedHookDone >= {epoch})"
         hist_info = condor_history(constraint, schedds=schedds)

lsst/ctrl/bps/htcondor/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.2000"
+__version__ = "29.2025.2200"

{lsst_ctrl_bps_htcondor-29.2025.2000.dist-info → lsst_ctrl_bps_htcondor-29.2025.2200.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.2000
+Version: 29.2025.2200
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_htcondor-29.2025.2000.dist-info → lsst_ctrl_bps_htcondor-29.2025.2200.dist-info}/RECORD

@@ -2,18 +2,18 @@ lsst/ctrl/bps/htcondor/__init__.py,sha256=1gTmOVLJILvBqgqHVECo8uqoX8e4fiTeH_dHBU
 lsst/ctrl/bps/htcondor/final_post.sh,sha256=chfaQV6Q7rGsK-8Hx58ch52m-PofvBanrl7VwCssHec,248
 lsst/ctrl/bps/htcondor/handlers.py,sha256=2gM3Ac00in4ob9ckcP331W1LSEjs9UDKIqt4MULA4bg,11196
 lsst/ctrl/bps/htcondor/htcondor_config.py,sha256=c4lCiYEwEXFdxgbMfEkbDm4LrvkRMF31SqLtQqzqIV4,1523
-lsst/ctrl/bps/htcondor/htcondor_service.py,sha256=-qf7FG1yvxawf9vHfUMZtofVrrK06CMbviarLRXGg_0,88308
-lsst/ctrl/bps/htcondor/lssthtc.py,sha256=Rsfr7ZehZHiRWmF-8FMDReZQrGMtKii7CO2O8Vu9hYg,80420
+lsst/ctrl/bps/htcondor/htcondor_service.py,sha256=4_jm0lIZw3mYXyays1IWyg3pFwpODR_-g6CLIepXu7w,95330
+lsst/ctrl/bps/htcondor/lssthtc.py,sha256=pYxcA5jicuJs1RnhusSuMFrOU92Xy1fb-tleZ9m784Y,80485
 lsst/ctrl/bps/htcondor/provisioner.py,sha256=hPN8YJUtwNHQylw68kfskF1S2vCeQvztF8W0d_QKqqM,7851
-lsst/ctrl/bps/htcondor/version.py,sha256=bT9RvvieKvOl5VImh838xbe0jWwzezTNwq19FWhF5NU,55
+lsst/ctrl/bps/htcondor/version.py,sha256=RaoK8ADNKLNvY-bXCQnWhCS6HwZ0pWKOeKdmgzS7d6Y,55
 lsst/ctrl/bps/htcondor/etc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml,sha256=xDRts4vHKov2PE_JRh-0nF3jfuNJXtKBXZqveASp_iA,1422
-lsst_ctrl_bps_htcondor-29.2025.2000.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
-lsst_ctrl_bps_htcondor-29.2025.2000.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps_htcondor-29.2025.2000.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps_htcondor-29.2025.2000.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps_htcondor-29.2025.2000.dist-info/METADATA,sha256=6brPWGgVcrgiUBsCbbloAFpbz6zzOaPIQ8LdXM3JQCU,2139
-lsst_ctrl_bps_htcondor-29.2025.2000.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-lsst_ctrl_bps_htcondor-29.2025.2000.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps_htcondor-29.2025.2000.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps_htcondor-29.2025.2000.dist-info/RECORD,,
+lsst_ctrl_bps_htcondor-29.2025.2200.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
+lsst_ctrl_bps_htcondor-29.2025.2200.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps_htcondor-29.2025.2200.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps_htcondor-29.2025.2200.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps_htcondor-29.2025.2200.dist-info/METADATA,sha256=Ls4Ipu4B4iFbZ4rozJhOoBeuOOR6Jg6E3X31dWa2qU0,2139
+lsst_ctrl_bps_htcondor-29.2025.2200.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_ctrl_bps_htcondor-29.2025.2200.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps_htcondor-29.2025.2200.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps_htcondor-29.2025.2200.dist-info/RECORD,,

{lsst_ctrl_bps_htcondor-29.2025.2000.dist-info → lsst_ctrl_bps_htcondor-29.2025.2200.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.1)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any