lsst-ctrl-bps-htcondor 29.2025.2000__tar.gz → 29.2025.2100__tar.gz

{lsst_ctrl_bps_htcondor-29.2025.2000/python/lsst_ctrl_bps_htcondor.egg-info → lsst_ctrl_bps_htcondor-29.2025.2100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.2000
+Version: 29.2025.2100
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_htcondor-29.2025.2000 → lsst_ctrl_bps_htcondor-29.2025.2100}/doc/lsst.ctrl.bps.htcondor/userguide.rst

@@ -476,12 +476,42 @@ for a specific reason, here, the memory usage exceeded memory limits
 .. note::
 
    By default, BPS will automatically retry jobs that failed due to the out of
-   memory error (see `Automatic memory scaling`__ section in **ctrl_bps**
+   memory error (see `Automatic memory scaling`_ section in **ctrl_bps**
    documentation for more information regarding this topic) and the issues
    illustrated by the above examples should only occur if automatic memory
    scalling was explicitly disabled in the submit YAML file.
 
-.. __: https://pipelines.lsst.io/v/weekly/modules/lsst.ctrl.bps/quickstart.html#automatic-memory-scaling
+
+Automatic Releasing of Held Jobs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Many times releasing the jobs to just try again is successful because the
+system issues are transient.
+
+``releaseExpr`` can be set in the submit yaml to add automatic release
+conditions.  Like other BPS config values, this can be set globally or
+set for a specific cluster or pipetask.  The number of retries is still
+limited by the ``numberOfRetries``.  All held jobs count towards this
+limit no matter what the reason.  The plugin prohibits the automatic
+release of jobs held by user.
+
+Example expressions:
+
+* ``releaseExpr: "True"`` - will always release held job unless held by user.
+* ``releaseExpr: "HoldReasonCode =?= 7"`` - release jobs where the standard
+  output file for the job could not be opened.
+
+For more information about expressions, see HTCondor documentation:
+
+* HTCondor `ClassAd expressions`_
+* list of `HoldReasonCodes`_
+
+.. warning::
+
+   System problems should still be tracked and reported.  All of the
+   hold reasons for a single completed run can be found via ``grep -A
+   2 held <submit dir>/*.nodes.log``.
+
 
 .. _htc-plugin-troubleshooting:
 
@@ -535,3 +565,6 @@ complete your run.
 .. _condor_release: https://htcondor.readthedocs.io/en/latest/man-pages/condor_release.html
 .. _condor_rm: https://htcondor.readthedocs.io/en/latest/man-pages/condor_rm.html
 .. _lsst_distrib: https://github.com/lsst/lsst_distrib.git
+.. _Automatic memory scaling: https://pipelines.lsst.io/v/weekly/modules/lsst.ctrl.bps/quickstart.html#automatic-memory-scaling
+.. _HoldReasonCodes: https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html#HoldReasonCode
+.. _ClassAd expressions: https://htcondor.readthedocs.io/en/latest/classads/classad-mechanism.html#classad-evaluation-semantics

{lsst_ctrl_bps_htcondor-29.2025.2000 → lsst_ctrl_bps_htcondor-29.2025.2100}/python/lsst/ctrl/bps/htcondor/htcondor_service.py

@@ -34,6 +34,7 @@ import logging
 import os
 import re
 from collections import defaultdict
+from copy import deepcopy
 from enum import IntEnum, auto
 from pathlib import Path
 from typing import Any, cast
@@ -331,7 +332,7 @@ class HTCondorService(BaseWmsService):
 
         Returns
         -------
-        job_ids : `list` [`Any`]
+        job_ids : `list` [`~typing.Any`]
             Only job ids to be used by cancel and other functions.  Typically
             this means top-level jobs (i.e., not children jobs).
         """
@@ -604,15 +605,15 @@ class HTCondorWorkflow(BaseWmsWorkflow):
         self.dag.write(out_prefix, job_subdir="jobs/{self.label}")
 
 
-def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefix):
+def _create_job(subdir_template, cached_values, generic_workflow, gwjob, out_prefix):
     """Convert GenericWorkflow job nodes to DAG jobs.
 
     Parameters
     ----------
     subdir_template : `str`
         Template for making subdirs.
-    site_values : `dict`
-        Site specific values
+    cached_values : `dict`
+        Site and label specific values.
     generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
         Generic workflow that is being converted.
     gwjob : `lsst.ctrl.bps.GenericWorkflowJob`
@@ -654,7 +655,7 @@ def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefi
         "on_exit_hold_subcode": "34",
     }
 
-    htc_job_cmds.update(_translate_job_cmds(site_values, generic_workflow, gwjob))
+    htc_job_cmds.update(_translate_job_cmds(cached_values, generic_workflow, gwjob))
 
     # job stdout, stderr, htcondor user log.
     for key in ("output", "error", "log"):
@@ -662,7 +663,7 @@ def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefi
         _LOG.debug("HTCondor %s = %s", key, htc_job_cmds[key])
 
     htc_job_cmds.update(
-        _handle_job_inputs(generic_workflow, gwjob.name, site_values["bpsUseShared"], out_prefix)
+        _handle_job_inputs(generic_workflow, gwjob.name, cached_values["bpsUseShared"], out_prefix)
     )
 
     # Add the job cmds dict to the job object.
@@ -673,7 +674,7 @@ def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefi
     # Add job attributes to job.
     _LOG.debug("gwjob.attrs = %s", gwjob.attrs)
     htc_job.add_job_attrs(gwjob.attrs)
-    htc_job.add_job_attrs(site_values["attrs"])
+    htc_job.add_job_attrs(cached_values["attrs"])
     htc_job.add_job_attrs({"bps_job_quanta": create_count_summary(gwjob.quanta_counts)})
     htc_job.add_job_attrs({"bps_job_name": gwjob.name, "bps_job_label": gwjob.label})
 
@@ -685,8 +686,8 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
 
     Parameters
     ----------
-    cached_vals : `dict` [`str`, `Any`]
-        Config values common to jobs with same label.
+    cached_vals : `dict` [`str`, `~typing.Any`]
+        Config values common to jobs with same site or label.
     generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
         Generic workflow that contains job to being converted.
     gwjob : `lsst.ctrl.bps.GenericWorkflowJob`
@@ -694,7 +695,7 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
 
     Returns
     -------
-    htc_job_commands : `dict` [`str`, `Any`]
+    htc_job_commands : `dict` [`str`, `~typing.Any`]
         Contains commands which can appear in the HTCondor submit description
         file.
     """
@@ -720,9 +721,6 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
         jobcmds["accounting_group_user"] = cached_vals.get("accountingUser")
 
     # job commands that need modification
-    if gwjob.number_of_retries:
-        jobcmds["max_retries"] = f"{gwjob.number_of_retries}"
-
     if gwjob.retry_unless_exit:
         if isinstance(gwjob.retry_unless_exit, int):
             jobcmds["retry_until"] = f"{gwjob.retry_unless_exit}"
@@ -739,6 +737,7 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
     if gwjob.request_memory:
         jobcmds["request_memory"] = f"{gwjob.request_memory}"
 
+    memory_max = 0
     if gwjob.memory_multiplier:
         # Do not use try-except! At the moment, BpsConfig returns an empty
         # string if it does not contain the key.
@@ -765,13 +764,18 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
             gwjob.request_memory, gwjob.memory_multiplier, memory_max
         )
 
-        # Periodically release jobs which are being held due to exceeding
-        # memory. Stop doing that (by removing the job from the HTCondor queue)
-        # after the maximal number of retries has been reached or the job was
-        # already run at maximal allowed memory.
-        jobcmds["periodic_release"] = _create_periodic_release_expr(
-            gwjob.request_memory, gwjob.memory_multiplier, memory_max
-        )
+    user_release_expr = cached_vals.get("releaseExpr", "")
+    if gwjob.number_of_retries is not None and gwjob.number_of_retries >= 0:
+        jobcmds["max_retries"] = gwjob.number_of_retries
+
+        # No point in adding periodic_release if 0 retries
+        if gwjob.number_of_retries > 0:
+            periodic_release = _create_periodic_release_expr(
+                gwjob.request_memory, gwjob.memory_multiplier, memory_max, user_release_expr
+            )
+            if periodic_release:
+                jobcmds["periodic_release"] = periodic_release
+
         jobcmds["periodic_remove"] = _create_periodic_remove_expr(
             gwjob.request_memory, gwjob.memory_multiplier, memory_max
         )
@@ -830,7 +834,7 @@ def _translate_dag_cmds(gwjob):
 
     Returns
     -------
-    dagcmds : `dict` [`str`, `Any`]
+    dagcmds : `dict` [`str`, `~typing.Any`]
         DAGMan commands for the job.
     """
     # Values in the dag script that just are name mappings.
@@ -1180,7 +1184,7 @@ def _get_info_from_path(wms_path: str | os.PathLike) -> tuple[str, dict[str, dic
     -------
     wms_workflow_id : `str`
         The run id which is a DAGman job id.
-    jobs : `dict` [`str`, `dict` [`str`, `Any`]]
+    jobs : `dict` [`str`, `dict` [`str`, `~typing.Any`]]
         Information about jobs read from files in the given directory.
         The key is the HTCondor id and the value is a dictionary of HTCondor
         keys and values.
@@ -1340,7 +1344,7 @@ def _add_service_job_specific_info(job_ad: dict[str, Any], specific_info: WmsSpe
 
     Parameters
     ----------
-    job_ad : `dict` [`str`, `Any`]
+    job_ad : `dict` [`str`, `~typing.Any`]
         Provisioning job information.
     specific_info : `lsst.ctrl.bps.WmsSpecificInfo`
         Where to add message.
@@ -1466,7 +1470,7 @@ def _add_run_info(wms_path, job):
     ----------
     wms_path : `str`
         Path to submit files for the run.
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Raises
@@ -1502,7 +1506,7 @@ def _get_owner(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Returns
@@ -1524,7 +1528,7 @@ def _get_run_summary(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Returns
@@ -1600,7 +1604,7 @@ def _get_state_counts_from_jobs(
     ----------
     wms_workflow_id : `str`
         HTCondor job id.
-    jobs : `dict [`dict` [`str`, `Any`]]
+    jobs : `dict [`dict` [`str`, `~typing.Any`]]
         HTCondor dag job information.
 
     Returns
@@ -1628,7 +1632,7 @@ def _get_state_counts_from_dag_job(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Returns
@@ -1684,7 +1688,7 @@ def _htc_status_to_wms_state(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor job information.
 
     Returns
@@ -1706,7 +1710,7 @@ def _htc_job_status_to_wms_state(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor job information.
 
     Returns
@@ -1748,7 +1752,7 @@ def _htc_node_status_to_wms_state(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor job information.
 
     Returns
@@ -1795,9 +1799,9 @@ def _update_jobs(jobs1, jobs2):
 
     Parameters
     ----------
-    jobs1 : `dict` [`str`, `dict` [`str`, `Any`]]
+    jobs1 : `dict` [`str`, `dict` [`str`, `~typing.Any`]]
         HTCondor job information to be updated.
-    jobs2 : `dict` [`str`, `dict` [`str`, `Any`]]
+    jobs2 : `dict` [`str`, `dict` [`str`, `~typing.Any`]]
         Additional HTCondor job information.
     """
     for job_id, job_ad in jobs2.items():
@@ -1937,34 +1941,39 @@ def _wms_id_to_dir(wms_id):
     return wms_path, id_type
 
 
-def _create_periodic_release_expr(memory, multiplier, limit):
+def _create_periodic_release_expr(
+    memory: int, multiplier: float | None, limit: int, additional_expr: str = ""
+) -> str:
     """Construct an HTCondorAd expression for releasing held jobs.
 
-    The expression instruct HTCondor to release any job which was put on hold
-    due to exceeding memory requirements back to the job queue providing it
-    satisfies all of the conditions below:
-
-    * number of run attempts did not reach allowable number of retries,
-    * the memory requirements in the last failed run attempt did not reach
-      the specified memory limit.
-
     Parameters
     ----------
     memory : `int`
         Requested memory in MB.
-    multiplier : `float`
-        Memory growth rate between retires.
+    multiplier : `float` or None
+        Memory growth rate between retries.
     limit : `int`
         Memory limit.
+    additional_expr : `str`, optional
+        Expression to add to periodic_release.  Defaults to empty string.
 
     Returns
     -------
     expr : `str`
-        A string representing an HTCondor ClassAd expression for releasing jobs
-        which have been held due to exceeding the memory requirements.
+        A string representing an HTCondor ClassAd expression for releasing job.
     """
-    is_retry_allowed = "NumJobStarts <= JobMaxRetries"
-    was_below_limit = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) < {limit}"
+    _LOG.debug(
+        "periodic_release: memory: %s, multiplier: %s, limit: %s, additional_expr: %s",
+        memory,
+        multiplier,
+        limit,
+        additional_expr,
+    )
+
+    # ctrl_bps sets multiplier to None in the GenericWorkflow if
+    # memoryMultiplier <= 1, but checking value just in case.
+    if (not multiplier or multiplier <= 1) and not additional_expr:
+        return ""
 
     # Job ClassAds attributes 'HoldReasonCode' and 'HoldReasonSubCode' are
     # UNDEFINED if job is not HELD (i.e. when 'JobStatus' is not 5).
@@ -1976,63 +1985,74 @@ def _create_periodic_release_expr(memory, multiplier, limit):
     # the entire expression should evaluate to FALSE when the job is not HELD.
     # According to ClassAd evaluation semantics FALSE && UNDEFINED is FALSE,
     # but better safe than sorry.
-    was_mem_exceeded = (
-        "JobStatus == 5 "
-        "&& (HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
-        "|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34)"
-    )
+    is_held = "JobStatus == 5"
+    is_retry_allowed = "NumJobStarts <= JobMaxRetries"
+
+    mem_expr = ""
+    if memory and multiplier and multiplier > 1 and limit:
+        was_mem_exceeded = (
+            "(HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
+            "|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34)"
+        )
+        was_below_limit = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) < {limit}"
+        mem_expr = f"{was_mem_exceeded} && {was_below_limit}"
+
+    user_expr = ""
+    if additional_expr:
+        # Never auto release a job held by user.
+        user_expr = f"HoldReasonCode =!= 1 && {additional_expr}"
+
+    expr = f"{is_held} && {is_retry_allowed}"
+    if user_expr and mem_expr:
+        expr += f" && ({mem_expr} || {user_expr})"
+    elif user_expr:
+        expr += f" && {user_expr}"
+    elif mem_expr:
+        expr += f" && {mem_expr}"
 
-    expr = f"{was_mem_exceeded} && {is_retry_allowed} && {was_below_limit}"
     return expr
 
 
 def _create_periodic_remove_expr(memory, multiplier, limit):
     """Construct an HTCondorAd expression for removing jobs from the queue.
 
-    The expression instruct HTCondor to remove any job which was put on hold
-    due to exceeding memory requirements from the job queue providing it
-    satisfies any of the conditions below:
-
-    * allowable number of retries was reached,
-    * the memory requirements during the last failed run attempt reached
-      the specified memory limit.
-
     Parameters
     ----------
     memory : `int`
         Requested memory in MB.
     multiplier : `float`
-        Memory growth rate between retires.
+        Memory growth rate between retries.
     limit : `int`
         Memory limit.
 
     Returns
     -------
     expr : `str`
-        A string representing an HTCondor ClassAd expression for removing jobs
-        which were run at the maximal allowable memory and still exceeded
-        the memory requirements.
+        A string representing an HTCondor ClassAd expression for removing jobs.
     """
-    is_retry_disallowed = "NumJobStarts > JobMaxRetries"
-    was_limit_reached = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) == {limit}"
-
-    # Job ClassAds attributes 'HoldReasonCode' and 'HoldReasonSubCode' are
-    # UNDEFINED if job is not HELD (i.e. when 'JobStatus' is not 5).
-    # The special comparison operators ensure that all comparisons below will
-    # evaluate to FALSE in this case.
+    # Job ClassAds attributes 'HoldReasonCode' and 'HoldReasonSubCode'
+    # are UNDEFINED if job is not HELD (i.e. when 'JobStatus' is not 5).
+    # The special comparison operators ensure that all comparisons below
+    # will evaluate to FALSE in this case.
     #
     # Note:
-    # May not be strictly necessary. Operators '&&' and '||' are not strict so
-    # the entire expression should evaluate to FALSE when the job is not HELD.
-    # According to ClassAd evaluation semantics FALSE && UNDEFINED is FALSE,
-    # but better safe than sorry.
-    was_mem_exceeded = (
-        "JobStatus == 5 "
-        "&& (HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
-        "|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34)"
-    )
+    # May not be strictly necessary. Operators '&&' and '||' are not
+    # strict so the entire expression should evaluate to FALSE when the
+    # job is not HELD. According to ClassAd evaluation semantics
+    # FALSE && UNDEFINED is FALSE, but better safe than sorry.
+    is_held = "JobStatus == 5"
+    is_retry_disallowed = "NumJobStarts > JobMaxRetries"
+
+    mem_expr = ""
+    if memory and multiplier and multiplier > 1 and limit:
+        mem_limit_expr = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) == {limit}"
+
+        mem_expr = (  # Add || here so only added if adding memory expr
+            " || ((HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
+            f"|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34) && {mem_limit_expr})"
+        )
 
-    expr = f"{was_mem_exceeded} && ({is_retry_disallowed} || {was_limit_reached})"
+    expr = f"{is_held} && ({is_retry_disallowed}{mem_expr})"
     return expr
 
 
@@ -2044,7 +2064,7 @@ def _create_request_memory_expr(memory, multiplier, limit):
     memory : `int`
         Requested memory in MB.
     multiplier : `float`
-        Memory growth rate between retires.
+        Memory growth rate between retries.
     limit : `int`
         Memory limit.
 
@@ -2119,7 +2139,7 @@ def _gather_site_values(config, compute_site):
 
     Returns
     -------
-    site_values : `dict` [`str`, `Any`]
+    site_values : `dict` [`str`, `~typing.Any`]
         Values specific to the given site.
     """
     site_values = {"attrs": {}, "profile": {}}
@@ -2167,6 +2187,50 @@ def _gather_site_values(config, compute_site):
     return site_values
 
 
+def _gather_label_values(config: BpsConfig, label: str) -> dict[str, Any]:
+    """Gather values specific to given job label.
+
+    Parameters
+    ----------
+    config : `lsst.ctrl.bps.BpsConfig`
+        BPS configuration that includes necessary submit/runtime
+        information.
+    label : `str`
+        GenericWorkflowJob label.
+
+    Returns
+    -------
+    values : `dict` [`str`, `~typing.Any`]
+        Values specific to the given job label.
+    """
+    values: dict[str, Any] = {"attrs": {}, "profile": {}}
+
+    search_opts = {}
+    profile_key = ""
+    if label == "finalJob":
+        search_opts["searchobj"] = config["finalJob"]
+        profile_key = ".finalJob.profile.condor"
+    elif label in config["cluster"]:
+        search_opts["curvals"] = {"curr_cluster": label}
+        profile_key = f".cluster.{label}.profile.condor"
+    elif label in config["pipetask"]:
+        search_opts["curvals"] = {"curr_pipetask": label}
+        profile_key = f".pipetask.{label}.profile.condor"
+
+    found, value = config.search("releaseExpr", opt=search_opts)
+    if found:
+        values["releaseExpr"] = value
+
+    if profile_key and profile_key in config:
+        for subkey, val in config[profile_key].items():
+            if subkey.startswith("+"):
+                values["attrs"][subkey[1:]] = val
+            else:
+                values["profile"][subkey] = val
+
+    return values
+
+
 def is_service_job(job_ad: dict[str, Any]) -> bool:
     """Determine if a job is a service one.
 
@@ -2280,16 +2344,22 @@ def _generic_workflow_to_htcondor_dag(
         subdir_template = tmp_template
 
     # Create all DAG jobs
-    site_values = {}  # cache compute site specific values to reduce config lookups
+    site_values = {}  # Cache compute site specific values to reduce config lookups.
+    cached_values = {}  # Cache label-specific values to reduce config lookups.
+    # Note: Can't use get_job_by_label because those only include payload jobs.
     for job_name in generic_workflow:
         gwjob = generic_workflow.get_job(job_name)
         if gwjob.node_type == GenericWorkflowNodeType.PAYLOAD:
             gwjob = cast(GenericWorkflowJob, gwjob)
             if gwjob.compute_site not in site_values:
                 site_values[gwjob.compute_site] = _gather_site_values(config, gwjob.compute_site)
+            if gwjob.label not in cached_values:
+                cached_values[gwjob.label] = deepcopy(site_values[gwjob.compute_site])
+                cached_values[gwjob.label].update(_gather_label_values(config, gwjob.label))
+                _LOG.debug("cached: %s= %s", gwjob.label, cached_values[gwjob.label])
             htc_job = _create_job(
                 subdir_template[gwjob.label],
-                site_values[gwjob.compute_site],
+                cached_values[gwjob.label],
                 generic_workflow,
                 gwjob,
                 out_prefix,
@@ -2351,8 +2421,15 @@ def _generic_workflow_to_htcondor_dag(
     if final and isinstance(final, GenericWorkflowJob):
         if final.compute_site and final.compute_site not in site_values:
             site_values[final.compute_site] = _gather_site_values(config, final.compute_site)
+        if final.label not in cached_values:
+            cached_values[final.label] = deepcopy(site_values[final.compute_site])
+            cached_values[final.label].update(_gather_label_values(config, final.label))
         final_htjob = _create_job(
-            subdir_template[final.label], site_values[final.compute_site], generic_workflow, final, out_prefix
+            subdir_template[final.label],
+            cached_values[final.label],
+            generic_workflow,
+            final,
+            out_prefix,
         )
         if "post" not in final_htjob.dagcmds:
             final_htjob.dagcmds["post"] = {

lsst_ctrl_bps_htcondor-29.2025.2100/python/lsst/ctrl/bps/htcondor/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2100"

{lsst_ctrl_bps_htcondor-29.2025.2000 → lsst_ctrl_bps_htcondor-29.2025.2100/python/lsst_ctrl_bps_htcondor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.2000
+Version: 29.2025.2100
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_htcondor-29.2025.2000 → lsst_ctrl_bps_htcondor-29.2025.2100}/tests/test_htcondor_service.py

@@ -352,7 +352,7 @@ class TranslateJobCmdsTestCase(unittest.TestCase):
 
     def setUp(self):
         self.gw_exec = GenericWorkflowExec("test_exec", "/dummy/dir/pipetask")
-        self.cached_vals = {"profile": {}, "bpsUseShared": True}
+        self.cached_vals = {"profile": {}, "bpsUseShared": True, "memoryLimit": 32768}
 
     def testRetryUnlessNone(self):
         gwjob = GenericWorkflowJob("retryUnless", "label1", executable=self.gw_exec)
@@ -409,6 +409,30 @@ class TranslateJobCmdsTestCase(unittest.TestCase):
         htc_commands = htcondor_service._translate_job_cmds(self.cached_vals, None, gwjob)
         self.assertEqual(htc_commands["environment"], "TEST_ENV_VAR='$ENV(CTRL_BPS_DIR)/tests'")
 
+    def testPeriodicRelease(self):
+        gwjob = GenericWorkflowJob("periodicRelease", "label1", executable=self.gw_exec)
+        gwjob.request_memory = 2048
+        gwjob.memory_multiplier = 2
+        gwjob.number_of_retries = 3
+        htc_commands = htcondor_service._translate_job_cmds(self.cached_vals, None, gwjob)
+        release = (
+            "JobStatus == 5 && NumJobStarts <= JobMaxRetries && "
+            "(HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 || "
+            "HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34) && "
+            "min({int(2048 * pow(2, NumJobStarts - 1)), 32768}) < 32768"
+        )
+        self.assertEqual(htc_commands["periodic_release"], release)
+
+    def testPeriodicRemoveNoRetries(self):
+        gwjob = GenericWorkflowJob("periodicRelease", "label1", executable=self.gw_exec)
+        gwjob.request_memory = 2048
+        gwjob.memory_multiplier = 1
+        gwjob.number_of_retries = 0
+        htc_commands = htcondor_service._translate_job_cmds(self.cached_vals, None, gwjob)
+        remove = "JobStatus == 5 && (NumJobStarts > JobMaxRetries)"
+        self.assertEqual(htc_commands["periodic_remove"], remove)
+        self.assertEqual(htc_commands["max_retries"], 0)
+
 
 class GetStateCountsFromDagJobTestCase(unittest.TestCase):
     """Test counting number of jobs per WMS state."""
@@ -1135,6 +1159,53 @@ class GatherSiteValuesTestCase(unittest.TestCase):
         self.assertEqual(results["memoryLimit"], BPS_DEFAULTS["memoryLimit"])
 
 
+class GatherLabelValuesTestCase(unittest.TestCase):
+    """Test _gather_labels_values function."""
+
+    def testClusterLabel(self):
+        # Test cluster value overrides pipetask.
+        label = "label1"
+        config = BpsConfig(
+            {
+                "cluster": {
+                    "label1": {"releaseExpr": "cluster_val", "profile": {"condor": {"prof_val1": 3}}}
+                },
+                "pipetask": {"label1": {"releaseExpr": "pipetask_val"}},
+            },
+            search_order=BPS_SEARCH_ORDER,
+            defaults=BPS_DEFAULTS,
+            wms_service_class_fqn="lsst.ctrl.bps.htcondor.HTCondorService",
+        )
+        results = htcondor_service._gather_label_values(config, label)
+        self.assertEqual(results, {"attrs": {}, "profile": {"prof_val1": 3}, "releaseExpr": "cluster_val"})
+
+    def testPipetaskLabel(self):
+        label = "label1"
+        config = BpsConfig(
+            {
+                "pipetask": {
+                    "label1": {"releaseExpr": "pipetask_val", "profile": {"condor": {"prof_val1": 3}}}
+                }
+            },
+            search_order=BPS_SEARCH_ORDER,
+            defaults=BPS_DEFAULTS,
+            wms_service_class_fqn="lsst.ctrl.bps.htcondor.HTCondorService",
+        )
+        results = htcondor_service._gather_label_values(config, label)
+        self.assertEqual(results, {"attrs": {}, "profile": {"prof_val1": 3}, "releaseExpr": "pipetask_val"})
+
+    def testNoSection(self):
+        label = "notThere"
+        config = BpsConfig(
+            {},
+            search_order=BPS_SEARCH_ORDER,
+            defaults=BPS_DEFAULTS,
+            wms_service_class_fqn="lsst.ctrl.bps.htcondor.HTCondorService",
+        )
+        results = htcondor_service._gather_label_values(config, label)
+        self.assertEqual(results, {"attrs": {}, "profile": {}})
+
+
 class CreateCheckJobTestCase(unittest.TestCase):
     """Test _create_check_job function."""
 
@@ -1147,5 +1218,77 @@ class CreateCheckJobTestCase(unittest.TestCase):
         self.assertIn("check_group_status.sub", job.subfile)
 
 
+class CreatePeriodicReleaseExprTestCase(unittest.TestCase):
+    """Test _create_periodic_release_expr function."""
+
+    def testNoReleaseExpr(self):
+        results = htcondor_service._create_periodic_release_expr(2048, 1, 32768, "")
+        self.assertEqual(results, "")
+
+    def testMultiplierNone(self):
+        results = htcondor_service._create_periodic_release_expr(2048, None, 32768, "")
+        self.assertEqual(results, "")
+
+    def testJustMemoryReleaseExpr(self):
+        self.maxDiff = None  # so test error shows entire strings
+        results = htcondor_service._create_periodic_release_expr(2048, 2, 32768, "")
+        truth = (
+            "JobStatus == 5 && NumJobStarts <= JobMaxRetries && "
+            "(HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 || "
+            "HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34) && "
+            "min({int(2048 * pow(2, NumJobStarts - 1)), 32768}) < 32768"
+        )
+        self.assertEqual(results, truth)
+
+    def testJustUserReleaseExpr(self):
+        results = htcondor_service._create_periodic_release_expr(2048, 1, 32768, "True")
+        truth = "JobStatus == 5 && NumJobStarts <= JobMaxRetries && HoldReasonCode =!= 1 && True"
+        self.assertEqual(results, truth)
+
+    def testJustUserReleaseExprMultiplierNone(self):
+        results = htcondor_service._create_periodic_release_expr(2048, None, 32768, "True")
+        truth = "JobStatus == 5 && NumJobStarts <= JobMaxRetries && HoldReasonCode =!= 1 && True"
+        self.assertEqual(results, truth)
+
+    def testMemoryAndUserReleaseExpr(self):
+        self.maxDiff = None  # so test error shows entire strings
+        results = htcondor_service._create_periodic_release_expr(2048, 2, 32768, "True")
+        truth = (
+            "JobStatus == 5 && NumJobStarts <= JobMaxRetries && "
+            "((HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 || "
+            "HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34) && "
+            "min({int(2048 * pow(2, NumJobStarts - 1)), 32768}) < 32768 || "
+            "HoldReasonCode =!= 1 && True)"
+        )
+        self.assertEqual(results, truth)
+
+
+class CreatePeriodicRemoveExprTestCase(unittest.TestCase):
+    """Test _create_periodic_release_expr function."""
+
+    def testBasicRemoveExpr(self):
+        """Function assumes only called if max_retries >= 0."""
+        results = htcondor_service._create_periodic_remove_expr(2048, 1, 32768)
+        truth = "JobStatus == 5 && (NumJobStarts > JobMaxRetries)"
+        self.assertEqual(results, truth)
+
+    def testBasicRemoveExprMultiplierNone(self):
+        """Function assumes only called if max_retries >= 0."""
+        results = htcondor_service._create_periodic_remove_expr(2048, None, 32768)
+        truth = "JobStatus == 5 && (NumJobStarts > JobMaxRetries)"
+        self.assertEqual(results, truth)
+
+    def testMemoryRemoveExpr(self):
+        self.maxDiff = None  # so test error shows entire strings
+        results = htcondor_service._create_periodic_remove_expr(2048, 2, 32768)
+        truth = (
+            "JobStatus == 5 && (NumJobStarts > JobMaxRetries || "
+            "((HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 || "
+            "HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34) && "
+            "min({int(2048 * pow(2, NumJobStarts - 1)), 32768}) == 32768))"
+        )
+        self.assertEqual(results, truth)
+
+
 if __name__ == "__main__":
     unittest.main()

lsst_ctrl_bps_htcondor-29.2025.2000/python/lsst/ctrl/bps/htcondor/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2000"