lsst-ctrl-bps-htcondor 29.0.1rc1__py3-none-any.whl → 29.1.0rc1__py3-none-any.whl

lsst/ctrl/bps/htcondor/htcondor_service.py

@@ -34,9 +34,10 @@ import logging
 import os
 import re
 from collections import defaultdict
+from copy import deepcopy
 from enum import IntEnum, auto
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 import htcondor
 from packaging import version
@@ -44,8 +45,12 @@ from packaging import version
 from lsst.ctrl.bps import (
     BaseWmsService,
     BaseWmsWorkflow,
+    BpsConfig,
     GenericWorkflow,
+    GenericWorkflowGroup,
     GenericWorkflowJob,
+    GenericWorkflowNodeType,
+    GenericWorkflowNoopJob,
     WmsJobReport,
     WmsRunReport,
     WmsSpecificInfo,
@@ -60,8 +65,9 @@ from .lssthtc import (
     MISSING_ID,
     HTCDag,
     HTCJob,
-    JobStatus,
     NodeStatus,
+    WmsNodeType,
+    _update_rescue_file,
     condor_history,
     condor_q,
     condor_search,
@@ -175,17 +181,23 @@ class HTCondorService(BaseWmsService):
             Keyword arguments for the options.
         """
         dag = workflow.dag
-
         ver = version.parse(htc_version())
-        if ver >= version.parse("8.9.3"):
-            sub = htc_create_submit_from_dag(dag.graph["dag_filename"], {})
-        else:
-            sub = htc_create_submit_from_cmd(dag.graph["dag_filename"], {})
 
         # For workflow portability, internal paths are all relative. Hence
         # the DAG needs to be submitted to HTCondor from inside the submit
         # directory.
         with chdir(workflow.submit_path):
+            try:
+                if ver >= version.parse("8.9.3"):
+                    sub = htc_create_submit_from_dag(dag.graph["dag_filename"], dag.graph["submit_options"])
+                else:
+                    sub = htc_create_submit_from_cmd(dag.graph["dag_filename"], dag.graph["submit_options"])
+            except Exception:
+                _LOG.error(
+                    "Problems creating HTCondor submit object from filename: %s", dag.graph["dag_filename"]
+                )
+                raise
+
             _LOG.info("Submitting from directory: %s", os.getcwd())
             schedd_dag_info = htc_submit_dag(sub)
             if schedd_dag_info:
@@ -226,7 +238,7 @@ class HTCondorService(BaseWmsService):
                 None,
                 (
                     f"workflow with run id '{wms_workflow_id}' not found. "
-                    f"Hint: use run's submit directory as the id instead"
+                    "Hint: use run's submit directory as the id instead"
                 ),
             )
 
@@ -266,7 +278,9 @@ class HTCondorService(BaseWmsService):
             )
 
         _LOG.info("Backing up select HTCondor files from previous run attempt")
-        htc_backup_files(wms_path, subdir="backups")
+        rescue_file = htc_backup_files(wms_path, subdir="backups")
+        if (wms_path / "subdags").exists():
+            _update_rescue_file(rescue_file)
 
         # For workflow portability, internal paths are all relative. Hence
         # the DAG needs to be resubmitted to HTCondor from inside the submit
@@ -318,7 +332,7 @@ class HTCondorService(BaseWmsService):
 
         Returns
         -------
-        job_ids : `list` [`Any`]
+        job_ids : `list` [`~typing.Any`]
             Only job ids to be used by cancel and other functions.  Typically
             this means top-level jobs (i.e., not children jobs).
         """
@@ -563,66 +577,17 @@ class HTCondorWorkflow(BaseWmsWorkflow):
     def from_generic_workflow(cls, config, generic_workflow, out_prefix, service_class):
         # Docstring inherited
         htc_workflow = cls(generic_workflow.name, config)
-        htc_workflow.dag = HTCDag(name=generic_workflow.name)
+        htc_workflow.dag = _generic_workflow_to_htcondor_dag(config, generic_workflow, out_prefix)
 
         _LOG.debug("htcondor dag attribs %s", generic_workflow.run_attrs)
-        htc_workflow.dag.add_attribs(generic_workflow.run_attrs)
+        # Add extra attributes to top most DAG.
         htc_workflow.dag.add_attribs(
             {
                 "bps_wms_service": service_class,
                 "bps_wms_workflow": f"{cls.__module__}.{cls.__name__}",
-                "bps_run_quanta": create_count_summary(generic_workflow.quanta_counts),
-                "bps_job_summary": create_count_summary(generic_workflow.job_counts),
             }
         )
 
-        _, tmp_template = config.search("subDirTemplate", opt={"replaceVars": False, "default": ""})
-        if isinstance(tmp_template, str):
-            subdir_template = defaultdict(lambda: tmp_template)
-        else:
-            subdir_template = tmp_template
-
-        # Create all DAG jobs
-        site_values = {}  # cache compute site specific values to reduce config lookups
-        for job_name in generic_workflow:
-            gwjob = generic_workflow.get_job(job_name)
-            if gwjob.compute_site not in site_values:
-                site_values[gwjob.compute_site] = _gather_site_values(config, gwjob.compute_site)
-            htc_job = _create_job(
-                subdir_template[gwjob.label],
-                site_values[gwjob.compute_site],
-                generic_workflow,
-                gwjob,
-                out_prefix,
-            )
-            htc_workflow.dag.add_job(htc_job)
-
-        # Add job dependencies to the DAG
-        for job_name in generic_workflow:
-            htc_workflow.dag.add_job_relationships([job_name], generic_workflow.successors(job_name))
-
-        # If final job exists in generic workflow, create DAG final job
-        final = generic_workflow.get_final()
-        if final and isinstance(final, GenericWorkflowJob):
-            if final.compute_site and final.compute_site not in site_values:
-                site_values[final.compute_site] = _gather_site_values(config, final.compute_site)
-            final_htjob = _create_job(
-                subdir_template[final.label],
-                site_values[final.compute_site],
-                generic_workflow,
-                final,
-                out_prefix,
-            )
-            if "post" not in final_htjob.dagcmds:
-                final_htjob.dagcmds["post"] = (
-                    f"{os.path.dirname(__file__)}/final_post.sh {final.name} $DAG_STATUS $RETURN"
-                )
-            htc_workflow.dag.add_final_job(final_htjob)
-        elif final and isinstance(final, GenericWorkflow):
-            raise NotImplementedError("HTCondor plugin does not support a workflow as the final job")
-        elif final:
-            return TypeError(f"Invalid type for GenericWorkflow.get_final() results ({type(final)})")
-
         return htc_workflow
 
     def write(self, out_prefix):
@@ -637,18 +602,18 @@ class HTCondorWorkflow(BaseWmsWorkflow):
         os.makedirs(out_prefix, exist_ok=True)
 
         # Write down the workflow in HTCondor format.
-        self.dag.write(out_prefix, "jobs/{self.label}")
+        self.dag.write(out_prefix, job_subdir="jobs/{self.label}")
 
 
-def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefix):
+def _create_job(subdir_template, cached_values, generic_workflow, gwjob, out_prefix):
     """Convert GenericWorkflow job nodes to DAG jobs.
 
     Parameters
     ----------
     subdir_template : `str`
         Template for making subdirs.
-    site_values : `dict`
-        Site specific values
+    cached_values : `dict`
+        Site and label specific values.
     generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
         Generic workflow that is being converted.
     gwjob : `lsst.ctrl.bps.GenericWorkflowJob`
@@ -668,8 +633,10 @@ def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefi
     if gwjob.tags:
         curvals.update(gwjob.tags)
 
-    subdir = subdir_template.format_map(curvals)
-    htc_job.subfile = Path("jobs") / subdir / f"{gwjob.name}.sub"
+    subdir = Path("jobs") / subdir_template.format_map(curvals)
+    htc_job.subdir = subdir
+    htc_job.subfile = f"{gwjob.name}.sub"
+    htc_job.add_dag_cmds({"dir": subdir})
 
     htc_job_cmds = {
         "universe": "vanilla",
@@ -681,20 +648,22 @@ def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefi
         # Exceeding memory sometimes triggering SIGBUS or SIGSEGV error. Tell
         # htcondor to put on hold any jobs which exited by a signal.
         "on_exit_hold": "ExitBySignal == true",
-        "on_exit_hold_reason": 'strcat("Job raised a signal ", string(ExitSignal), ". ", '
-        '"Handling signal as if job has gone over memory limit.")',
+        "on_exit_hold_reason": (
+            'strcat("Job raised a signal ", string(ExitSignal), ". ", '
+            '"Handling signal as if job has gone over memory limit.")'
+        ),
         "on_exit_hold_subcode": "34",
     }
 
-    htc_job_cmds.update(_translate_job_cmds(site_values, generic_workflow, gwjob))
+    htc_job_cmds.update(_translate_job_cmds(cached_values, generic_workflow, gwjob))
 
     # job stdout, stderr, htcondor user log.
     for key in ("output", "error", "log"):
-        htc_job_cmds[key] = htc_job.subfile.with_suffix(f".$(Cluster).{key[:3]}")
+        htc_job_cmds[key] = f"{gwjob.name}.$(Cluster).{key[:3]}"
         _LOG.debug("HTCondor %s = %s", key, htc_job_cmds[key])
 
     htc_job_cmds.update(
-        _handle_job_inputs(generic_workflow, gwjob.name, site_values["bpsUseShared"], out_prefix)
+        _handle_job_inputs(generic_workflow, gwjob.name, cached_values["bpsUseShared"], out_prefix)
     )
 
     # Add the job cmds dict to the job object.
@@ -705,7 +674,7 @@ def _create_job(subdir_template, site_values, generic_workflow, gwjob, out_prefi
     # Add job attributes to job.
     _LOG.debug("gwjob.attrs = %s", gwjob.attrs)
     htc_job.add_job_attrs(gwjob.attrs)
-    htc_job.add_job_attrs(site_values["attrs"])
+    htc_job.add_job_attrs(cached_values["attrs"])
     htc_job.add_job_attrs({"bps_job_quanta": create_count_summary(gwjob.quanta_counts)})
     htc_job.add_job_attrs({"bps_job_name": gwjob.name, "bps_job_label": gwjob.label})
 
@@ -717,8 +686,8 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
 
     Parameters
     ----------
-    cached_vals : `dict` [`str`, `Any`]
-        Config values common to jobs with same label.
+    cached_vals : `dict` [`str`, `~typing.Any`]
+        Config values common to jobs with same site or label.
     generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
         Generic workflow that contains job to being converted.
     gwjob : `lsst.ctrl.bps.GenericWorkflowJob`
@@ -726,7 +695,7 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
 
     Returns
     -------
-    htc_job_commands : `dict` [`str`, `Any`]
+    htc_job_commands : `dict` [`str`, `~typing.Any`]
         Contains commands which can appear in the HTCondor submit description
         file.
     """
@@ -752,9 +721,6 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
         jobcmds["accounting_group_user"] = cached_vals.get("accountingUser")
 
     # job commands that need modification
-    if gwjob.number_of_retries:
-        jobcmds["max_retries"] = f"{gwjob.number_of_retries}"
-
     if gwjob.retry_unless_exit:
         if isinstance(gwjob.retry_unless_exit, int):
             jobcmds["retry_until"] = f"{gwjob.retry_unless_exit}"
@@ -771,6 +737,7 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
     if gwjob.request_memory:
         jobcmds["request_memory"] = f"{gwjob.request_memory}"
 
+    memory_max = 0
     if gwjob.memory_multiplier:
         # Do not use try-except! At the moment, BpsConfig returns an empty
         # string if it does not contain the key.
@@ -797,13 +764,18 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
             gwjob.request_memory, gwjob.memory_multiplier, memory_max
         )
 
-        # Periodically release jobs which are being held due to exceeding
-        # memory. Stop doing that (by removing the job from the HTCondor queue)
-        # after the maximal number of retries has been reached or the job was
-        # already run at maximal allowed memory.
-        jobcmds["periodic_release"] = _create_periodic_release_expr(
-            gwjob.request_memory, gwjob.memory_multiplier, memory_max
-        )
+    user_release_expr = cached_vals.get("releaseExpr", "")
+    if gwjob.number_of_retries is not None and gwjob.number_of_retries >= 0:
+        jobcmds["max_retries"] = gwjob.number_of_retries
+
+        # No point in adding periodic_release if 0 retries
+        if gwjob.number_of_retries > 0:
+            periodic_release = _create_periodic_release_expr(
+                gwjob.request_memory, gwjob.memory_multiplier, memory_max, user_release_expr
+            )
+            if periodic_release:
+                jobcmds["periodic_release"] = periodic_release
+
         jobcmds["periodic_remove"] = _create_periodic_remove_expr(
             gwjob.request_memory, gwjob.memory_multiplier, memory_max
         )
@@ -817,7 +789,7 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
     # Handle command line
     if gwjob.executable.transfer_executable:
         jobcmds["transfer_executable"] = "True"
-        jobcmds["executable"] = os.path.basename(gwjob.executable.src_uri)
+        jobcmds["executable"] = gwjob.executable.src_uri
     else:
         jobcmds["executable"] = _fix_env_var_syntax(gwjob.executable.src_uri)
 
@@ -862,7 +834,7 @@ def _translate_dag_cmds(gwjob):
 
     Returns
     -------
-    dagcmds : `dict` [`str`, `Any`]
+    dagcmds : `dict` [`str`, `~typing.Any`]
         DAGMan commands for the job.
     """
     # Values in the dag script that just are name mappings.
@@ -974,7 +946,7 @@ def _replace_cmd_vars(arguments, gwjob):
     replacements = gwjob.cmdvals if gwjob.cmdvals is not None else {}
     try:
         arguments = arguments.format(**replacements)
-    except KeyError as exc:
+    except (KeyError, TypeError) as exc:  # TypeError in case None instead of {}
         _LOG.error("Could not replace command variables: replacement for %s not provided", str(exc))
         _LOG.debug("arguments: %s\ncmdvals: %s", arguments, replacements)
         raise
@@ -1200,19 +1172,19 @@ def _get_info_from_schedd(
     return schedd_dag_info
 
 
-def _get_info_from_path(wms_path: str) -> tuple[str, dict[str, dict[str, Any]], str]:
+def _get_info_from_path(wms_path: str | os.PathLike) -> tuple[str, dict[str, dict[str, Any]], str]:
     """Gather run information from a given run directory.
 
     Parameters
     ----------
-    wms_path : `str`
+    wms_path : `str` or `os.PathLike`
         Directory containing HTCondor files.
 
     Returns
     -------
     wms_workflow_id : `str`
         The run id which is a DAGman job id.
-    jobs : `dict` [`str`, `dict` [`str`, `Any`]]
+    jobs : `dict` [`str`, `dict` [`str`, `~typing.Any`]]
         Information about jobs read from files in the given directory.
         The key is the HTCondor id and the value is a dictionary of HTCondor
         keys and values.
@@ -1263,9 +1235,9 @@ def _get_info_from_path(wms_path: str) -> tuple[str, dict[str, dict[str, Any]],
             schedd_name = next(iter(job_info))
             job_ad = next(iter(job_info[schedd_name].values()))
             job.update(job_ad)
-    except FileNotFoundError:
-        message = f"Could not find HTCondor files in '{wms_path}'"
-        _LOG.warning(message)
+    except FileNotFoundError as err:
+        message = f"Could not find HTCondor files in '{wms_path}' ({err})"
+        _LOG.debug(message)
         messages.append(message)
         message = htc_check_dagman_output(wms_path)
         if message:
@@ -1298,8 +1270,9 @@ def _create_detailed_report_from_jobs(
         id and the value is a collection of report information for that run.
     """
     _LOG.debug("_create_detailed_report: id = %s, job = %s", wms_workflow_id, jobs[wms_workflow_id])
-    dag_ad = jobs.pop(wms_workflow_id)
-    total_jobs, state_counts = _get_state_counts_from_dag_job(dag_ad)
+
+    dag_ad = jobs[wms_workflow_id]
+
     report = WmsRunReport(
         wms_id=f"{dag_ad['ClusterId']}.{dag_ad['ProcId']}",
         global_wms_id=dag_ad.get("GlobalJobId", "MISS"),
@@ -1312,28 +1285,34 @@ def _create_detailed_report_from_jobs(
         operator=_get_owner(dag_ad),
         run_summary=_get_run_summary(dag_ad),
         state=_htc_status_to_wms_state(dag_ad),
+        total_number_jobs=0,
         jobs=[],
-        total_number_jobs=dag_ad.get("total_jobs", total_jobs),
-        job_state_counts=dag_ad.get("state_counts", state_counts),
-        exit_code_summary=_get_exit_code_summary(jobs),
+        job_state_counts=dict.fromkeys(WmsStates, 0),
+        exit_code_summary={},
     )
+
+    payload_jobs = {}  # keep track for later processing
     specific_info = WmsSpecificInfo()
     for job_id, job_ad in jobs.items():
-        if not is_service_job(job_ad):
+        if job_ad.get("wms_node_type", WmsNodeType.UNKNOWN) in [WmsNodeType.PAYLOAD, WmsNodeType.FINAL]:
             try:
+                name = job_ad.get("DAGNodeName", job_id)
+                wms_state = _htc_status_to_wms_state(job_ad)
                 job_report = WmsJobReport(
                     wms_id=job_id,
-                    name=job_ad.get("DAGNodeName", job_id),
-                    label=job_ad.get("bps_job_label", pegasus_name_to_label(job_ad["DAGNodeName"])),
-                    state=_htc_status_to_wms_state(job_ad),
+                    name=name,
+                    label=job_ad.get("bps_job_label", pegasus_name_to_label(name)),
+                    state=wms_state,
                 )
                 if job_report.label == "init":
                     job_report.label = "pipetaskInit"
+                report.job_state_counts[wms_state] += 1
                 report.jobs.append(job_report)
+                payload_jobs[job_id] = job_ad
             except KeyError as ex:
                 _LOG.error("Job missing key '%s': %s", str(ex), job_ad)
                 raise
-        else:
+        elif is_service_job(job_ad):
             _LOG.debug(
                 "Found service job: id='%s', name='%s', label='%s', NodeStatus='%s', JobStatus='%s'",
                 job_id,
@@ -1344,13 +1323,11 @@ def _create_detailed_report_from_jobs(
             )
             _add_service_job_specific_info(job_ad, specific_info)
 
+    report.total_number_jobs = len(payload_jobs)
+    report.exit_code_summary = _get_exit_code_summary(payload_jobs)
     if specific_info:
         report.specific_info = specific_info
 
-    # Add the removed entry to restore the original content of the dictionary.
-    # The ordering of keys will be change permanently though.
-    jobs.update({wms_workflow_id: dag_ad})
-
     # Workflow will exit with non-zero DAG_STATUS if problem with
     # any of the wms jobs.  So change FAILED to SUCCEEDED if all
     # payload jobs SUCCEEDED.
@@ -1367,7 +1344,7 @@ def _add_service_job_specific_info(job_ad: dict[str, Any], specific_info: WmsSpe
 
     Parameters
     ----------
-    job_ad : `dict` [`str`, `Any`]
+    job_ad : `dict` [`str`, `~typing.Any`]
         Provisioning job information.
     specific_info : `lsst.ctrl.bps.WmsSpecificInfo`
         Where to add message.
@@ -1450,6 +1427,7 @@ def _summary_report(user, hist, pass_thru, schedds=None):
 
     # Have list of DAGMan jobs, need to get run_report info.
     run_reports = {}
+    msg = ""
     for jobs in job_info.values():
         for job_id, job in jobs.items():
             total_jobs, state_counts = _get_state_counts_from_dag_job(job)
@@ -1482,7 +1460,7 @@ def _summary_report(user, hist, pass_thru, schedds=None):
             )
             run_reports[report.global_wms_id] = report
 
-    return run_reports, ""
+    return run_reports, msg
 
 
 def _add_run_info(wms_path, job):
@@ -1492,7 +1470,7 @@ def _add_run_info(wms_path, job):
     ----------
     wms_path : `str`
         Path to submit files for the run.
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Raises
@@ -1528,7 +1506,7 @@ def _get_owner(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Returns
@@ -1550,7 +1528,7 @@ def _get_run_summary(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Returns
@@ -1596,14 +1574,14 @@ def _get_exit_code_summary(jobs):
             exit_code = 0
             job_status = job_ad["JobStatus"]
             match job_status:
-                case JobStatus.COMPLETED | JobStatus.HELD:
+                case htcondor.JobStatus.COMPLETED | htcondor.JobStatus.HELD:
                     exit_code = job_ad["ExitSignal"] if job_ad["ExitBySignal"] else job_ad["ExitCode"]
                 case (
-                    JobStatus.IDLE
-                    | JobStatus.RUNNING
-                    | JobStatus.REMOVED
-                    | JobStatus.TRANSFERRING_OUTPUT
-                    | JobStatus.SUSPENDED
+                    htcondor.JobStatus.IDLE
+                    | htcondor.JobStatus.RUNNING
+                    | htcondor.JobStatus.REMOVED
+                    | htcondor.JobStatus.TRANSFERRING_OUTPUT
+                    | htcondor.JobStatus.SUSPENDED
                 ):
                     pass
                 case _:
@@ -1626,7 +1604,7 @@ def _get_state_counts_from_jobs(
     ----------
     wms_workflow_id : `str`
         HTCondor job id.
-    jobs : `dict [`dict` [`str`, `Any`]]
+    jobs : `dict [`dict` [`str`, `~typing.Any`]]
         HTCondor dag job information.
 
     Returns
@@ -1639,16 +1617,13 @@ def _get_state_counts_from_jobs(
     """
     state_counts = dict.fromkeys(WmsStates, 0)
     for job_id, job_ad in jobs.items():
-        if job_id != wms_workflow_id and not is_service_job(job_ad):
+        if job_id != wms_workflow_id and job_ad.get("wms_node_type", WmsNodeType.UNKNOWN) in [
+            WmsNodeType.PAYLOAD,
+            WmsNodeType.FINAL,
+        ]:
             state_counts[_htc_status_to_wms_state(job_ad)] += 1
-    total_counted = sum(state_counts.values())
-
-    if "NodesTotal" in jobs[wms_workflow_id]:
-        total_count = jobs[wms_workflow_id]["NodesTotal"]
-    else:
-        total_count = total_counted
+    total_count = sum(state_counts.values())
 
-    state_counts[WmsStates.UNREADY] += total_count - total_counted
     return total_count, state_counts
 
 
@@ -1657,7 +1632,7 @@ def _get_state_counts_from_dag_job(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor dag job information.
 
     Returns
@@ -1713,7 +1688,7 @@ def _htc_status_to_wms_state(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor job information.
 
     Returns
@@ -1735,7 +1710,7 @@ def _htc_job_status_to_wms_state(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor job information.
 
     Returns
@@ -1746,27 +1721,28 @@ def _htc_job_status_to_wms_state(job):
     _LOG.debug(
         "htc_job_status_to_wms_state: %s=%s, %s", job["ClusterId"], job["JobStatus"], type(job["JobStatus"])
     )
-    job_status = int(job["JobStatus"])
     wms_state = WmsStates.MISFIT
-
-    _LOG.debug("htc_job_status_to_wms_state: job_status = %s", job_status)
-    if job_status == JobStatus.IDLE:
-        wms_state = WmsStates.PENDING
-    elif job_status == JobStatus.RUNNING:
-        wms_state = WmsStates.RUNNING
-    elif job_status == JobStatus.REMOVED:
-        wms_state = WmsStates.DELETED
-    elif job_status == JobStatus.COMPLETED:
-        if (
-            (job.get("ExitBySignal", False) and job.get("ExitSignal", 0))
-            or job.get("ExitCode", 0)
-            or job.get("DAG_Status", 0)
-        ):
-            wms_state = WmsStates.FAILED
-        else:
-            wms_state = WmsStates.SUCCEEDED
-    elif job_status == JobStatus.HELD:
-        wms_state = WmsStates.HELD
+    if "JobStatus" in job and job["JobStatus"]:
+        job_status = int(job["JobStatus"])
+
+        _LOG.debug("htc_job_status_to_wms_state: job_status = %s", job_status)
+        if job_status == htcondor.JobStatus.IDLE:
+            wms_state = WmsStates.PENDING
+        elif job_status == htcondor.JobStatus.RUNNING:
+            wms_state = WmsStates.RUNNING
+        elif job_status == htcondor.JobStatus.REMOVED:
+            wms_state = WmsStates.DELETED
+        elif job_status == htcondor.JobStatus.COMPLETED:
+            if (
+                (job.get("ExitBySignal", False) and job.get("ExitSignal", 0))
+                or job.get("ExitCode", 0)
+                or job.get("DAG_Status", 0)
+            ):
+                wms_state = WmsStates.FAILED
+            else:
+                wms_state = WmsStates.SUCCEEDED
+        elif job_status == htcondor.JobStatus.HELD:
+            wms_state = WmsStates.HELD
 
     return wms_state
 
@@ -1776,7 +1752,7 @@ def _htc_node_status_to_wms_state(job):
 
     Parameters
     ----------
-    job : `dict` [`str`, `Any`]
+    job : `dict` [`str`, `~typing.Any`]
         HTCondor job information.
 
     Returns
@@ -1823,9 +1799,9 @@ def _update_jobs(jobs1, jobs2):
 
     Parameters
     ----------
-    jobs1 : `dict` [`str`, `dict` [`str`, `Any`]]
+    jobs1 : `dict` [`str`, `dict` [`str`, `~typing.Any`]]
         HTCondor job information to be updated.
-    jobs2 : `dict` [`str`, `dict` [`str`, `Any`]]
+    jobs2 : `dict` [`str`, `dict` [`str`, `~typing.Any`]]
         Additional HTCondor job information.
     """
     for job_id, job_ad in jobs2.items():
@@ -1965,34 +1941,39 @@ def _wms_id_to_dir(wms_id):
     return wms_path, id_type
 
 
-def _create_periodic_release_expr(memory, multiplier, limit):
+def _create_periodic_release_expr(
+    memory: int, multiplier: float | None, limit: int, additional_expr: str = ""
+) -> str:
     """Construct an HTCondorAd expression for releasing held jobs.
 
-    The expression instruct HTCondor to release any job which was put on hold
-    due to exceeding memory requirements back to the job queue providing it
-    satisfies all of the conditions below:
-
-    * number of run attempts did not reach allowable number of retries,
-    * the memory requirements in the last failed run attempt did not reach
-      the specified memory limit.
-
     Parameters
     ----------
     memory : `int`
         Requested memory in MB.
-    multiplier : `float`
-        Memory growth rate between retires.
+    multiplier : `float` or None
+        Memory growth rate between retries.
     limit : `int`
         Memory limit.
+    additional_expr : `str`, optional
+        Expression to add to periodic_release.  Defaults to empty string.
 
     Returns
     -------
     expr : `str`
-        A string representing an HTCondor ClassAd expression for releasing jobs
-        which have been held due to exceeding the memory requirements.
+        A string representing an HTCondor ClassAd expression for releasing job.
     """
-    is_retry_allowed = "NumJobStarts <= JobMaxRetries"
-    was_below_limit = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) < {limit}"
+    _LOG.debug(
+        "periodic_release: memory: %s, multiplier: %s, limit: %s, additional_expr: %s",
+        memory,
+        multiplier,
+        limit,
+        additional_expr,
+    )
+
+    # ctrl_bps sets multiplier to None in the GenericWorkflow if
+    # memoryMultiplier <= 1, but checking value just in case.
+    if (not multiplier or multiplier <= 1) and not additional_expr:
+        return ""
 
     # Job ClassAds attributes 'HoldReasonCode' and 'HoldReasonSubCode' are
     # UNDEFINED if job is not HELD (i.e. when 'JobStatus' is not 5).
@@ -2004,63 +1985,74 @@ def _create_periodic_release_expr(memory, multiplier, limit):
     # the entire expression should evaluate to FALSE when the job is not HELD.
     # According to ClassAd evaluation semantics FALSE && UNDEFINED is FALSE,
     # but better safe than sorry.
-    was_mem_exceeded = (
-        "JobStatus == 5 "
-        "&& (HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
-        "|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34)"
-    )
+    is_held = "JobStatus == 5"
+    is_retry_allowed = "NumJobStarts <= JobMaxRetries"
+
+    mem_expr = ""
+    if memory and multiplier and multiplier > 1 and limit:
+        was_mem_exceeded = (
+            "(HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
+            "|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34)"
+        )
+        was_below_limit = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) < {limit}"
+        mem_expr = f"{was_mem_exceeded} && {was_below_limit}"
+
+    user_expr = ""
+    if additional_expr:
+        # Never auto release a job held by user.
+        user_expr = f"HoldReasonCode =!= 1 && {additional_expr}"
+
+    expr = f"{is_held} && {is_retry_allowed}"
+    if user_expr and mem_expr:
+        expr += f" && ({mem_expr} || {user_expr})"
+    elif user_expr:
+        expr += f" && {user_expr}"
+    elif mem_expr:
+        expr += f" && {mem_expr}"
 
-    expr = f"{was_mem_exceeded} && {is_retry_allowed} && {was_below_limit}"
     return expr
 
 
 def _create_periodic_remove_expr(memory, multiplier, limit):
     """Construct an HTCondorAd expression for removing jobs from the queue.
 
-    The expression instruct HTCondor to remove any job which was put on hold
-    due to exceeding memory requirements from the job queue providing it
-    satisfies any of the conditions below:
-
-    * allowable number of retries was reached,
-    * the memory requirements during the last failed run attempt reached
-      the specified memory limit.
-
     Parameters
     ----------
     memory : `int`
         Requested memory in MB.
     multiplier : `float`
-        Memory growth rate between retires.
+        Memory growth rate between retries.
     limit : `int`
         Memory limit.
 
     Returns
     -------
     expr : `str`
-        A string representing an HTCondor ClassAd expression for removing jobs
-        which were run at the maximal allowable memory and still exceeded
-        the memory requirements.
+        A string representing an HTCondor ClassAd expression for removing jobs.
     """
-    is_retry_disallowed = "NumJobStarts > JobMaxRetries"
-    was_limit_reached = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) == {limit}"
-
-    # Job ClassAds attributes 'HoldReasonCode' and 'HoldReasonSubCode' are
-    # UNDEFINED if job is not HELD (i.e. when 'JobStatus' is not 5).
-    # The special comparison operators ensure that all comparisons below will
-    # evaluate to FALSE in this case.
+    # Job ClassAds attributes 'HoldReasonCode' and 'HoldReasonSubCode'
+    # are UNDEFINED if job is not HELD (i.e. when 'JobStatus' is not 5).
+    # The special comparison operators ensure that all comparisons below
+    # will evaluate to FALSE in this case.
     #
     # Note:
-    # May not be strictly necessary. Operators '&&' and '||' are not strict so
-    # the entire expression should evaluate to FALSE when the job is not HELD.
-    # According to ClassAd evaluation semantics FALSE && UNDEFINED is FALSE,
-    # but better safe than sorry.
-    was_mem_exceeded = (
-        "JobStatus == 5 "
-        "&& (HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
-        "|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34)"
-    )
+    # May not be strictly necessary. Operators '&&' and '||' are not
+    # strict so the entire expression should evaluate to FALSE when the
+    # job is not HELD. According to ClassAd evaluation semantics
+    # FALSE && UNDEFINED is FALSE, but better safe than sorry.
+    is_held = "JobStatus == 5"
+    is_retry_disallowed = "NumJobStarts > JobMaxRetries"
 
-    expr = f"{was_mem_exceeded} && ({is_retry_disallowed} || {was_limit_reached})"
+    mem_expr = ""
+    if memory and multiplier and multiplier > 1 and limit:
+        mem_limit_expr = f"min({{int({memory} * pow({multiplier}, NumJobStarts - 1)), {limit}}}) == {limit}"
+
+        mem_expr = (  # Add || here so only added if adding memory expr
+            " || ((HoldReasonCode =?= 34 && HoldReasonSubCode =?= 0 "
+            f"|| HoldReasonCode =?= 3 && HoldReasonSubCode =?= 34) && {mem_limit_expr})"
+        )
+
+    expr = f"{is_held} && ({is_retry_disallowed}{mem_expr})"
     return expr
 
 
@@ -2072,7 +2064,7 @@ def _create_request_memory_expr(memory, multiplier, limit):
     memory : `int`
         Requested memory in MB.
     multiplier : `float`
-        Memory growth rate between retires.
+        Memory growth rate between retries.
     limit : `int`
         Memory limit.
 
@@ -2147,7 +2139,7 @@ def _gather_site_values(config, compute_site):
 
     Returns
     -------
-    site_values : `dict` [`str`, `Any`]
+    site_values : `dict` [`str`, `~typing.Any`]
         Values specific to the given site.
     """
     site_values = {"attrs": {}, "profile": {}}
@@ -2195,6 +2187,50 @@ def _gather_site_values(config, compute_site):
     return site_values
 
 
+def _gather_label_values(config: BpsConfig, label: str) -> dict[str, Any]:
+    """Gather values specific to given job label.
+
+    Parameters
+    ----------
+    config : `lsst.ctrl.bps.BpsConfig`
+        BPS configuration that includes necessary submit/runtime
+        information.
+    label : `str`
+        GenericWorkflowJob label.
+
+    Returns
+    -------
+    values : `dict` [`str`, `~typing.Any`]
+        Values specific to the given job label.
+    """
+    values: dict[str, Any] = {"attrs": {}, "profile": {}}
+
+    search_opts = {}
+    profile_key = ""
+    if label == "finalJob":
+        search_opts["searchobj"] = config["finalJob"]
+        profile_key = ".finalJob.profile.condor"
+    elif label in config["cluster"]:
+        search_opts["curvals"] = {"curr_cluster": label}
+        profile_key = f".cluster.{label}.profile.condor"
+    elif label in config["pipetask"]:
+        search_opts["curvals"] = {"curr_pipetask": label}
+        profile_key = f".pipetask.{label}.profile.condor"
+
+    found, value = config.search("releaseExpr", opt=search_opts)
+    if found:
+        values["releaseExpr"] = value
+
+    if profile_key and profile_key in config:
+        for subkey, val in config[profile_key].items():
+            if subkey.startswith("+"):
+                values["attrs"][subkey[1:]] = val
+            else:
+                values["profile"][subkey] = val
+
+    return values
+
+
 def is_service_job(job_ad: dict[str, Any]) -> bool:
     """Determine if a job is a service one.
 
@@ -2212,6 +2248,199 @@ def is_service_job(job_ad: dict[str, Any]) -> bool:
     -----
     At the moment, HTCondor does not provide a native way to distinguish
     between payload and service jobs in the workflow.  This code depends
-    on read_node_status adding bps_job_type.
+    on read_node_status adding wms_node_type.
+    """
+    return job_ad.get("wms_node_type", WmsNodeType.UNKNOWN) == WmsNodeType.SERVICE
+
+
+def _group_to_subdag(
+    config: BpsConfig, generic_workflow_group: GenericWorkflowGroup, out_prefix: str
+) -> HTCJob:
+    """Convert a generic workflow group to an HTCondor dag.
+
+    Parameters
+    ----------
+    config : `lsst.ctrl.bps.BpsConfig`
+        Workflow configuration.
+    generic_workflow_group : `lsst.ctrl.bps.GenericWorkflowGroup`
+        The generic workflow group to convert.
+    out_prefix : `str`
+        Location prefix to be used when creating jobs.
+
+    Returns
+    -------
+    htc_job : `lsst.ctrl.bps.htcondor.HTCJob`
+        Job for running the HTCondor dag.
+    """
+    jobname = f"wms_{generic_workflow_group.name}"
+    htc_job = HTCJob(name=jobname, label=generic_workflow_group.label)
+    htc_job.add_dag_cmds({"dir": f"subdags/{jobname}"})
+    htc_job.subdag = _generic_workflow_to_htcondor_dag(config, generic_workflow_group, out_prefix)
+    if not generic_workflow_group.blocking:
+        htc_job.dagcmds["post"] = {
+            "defer": "",
+            "executable": f"{os.path.dirname(__file__)}/subdag_post.sh",
+            "arguments": f"{jobname} $RETURN",
+        }
+    return htc_job
+
+
+def _create_check_job(group_job_name: str, job_label: str) -> HTCJob:
+    """Create a job to check status of a group job.
+
+    Parameters
+    ----------
+    group_job_name : `str`
+        Name of the group job.
+    job_label : `str`
+        Label to use for the check status job.
+
+    Returns
+    -------
+    htc_job : `lsst.ctrl.bps.htcondor.HTCJob`
+        Job description for the job to check group job status.
     """
-    return job_ad.get("bps_job_type", "MISSING") == "service"
+    htc_job = HTCJob(name=f"wms_check_status_{group_job_name}", label=job_label)
+    htc_job.subfile = "${CTRL_BPS_HTCONDOR_DIR}/python/lsst/ctrl/bps/htcondor/check_group_status.sub"
+    htc_job.add_dag_cmds({"dir": f"subdags/{group_job_name}", "vars": {"group_job_name": group_job_name}})
+
+    return htc_job
+
+
+def _generic_workflow_to_htcondor_dag(
+    config: BpsConfig, generic_workflow: GenericWorkflow, out_prefix: str
+) -> HTCDag:
+    """Convert a GenericWorkflow to a HTCDag.
+
+    Parameters
+    ----------
+    config : `lsst.ctrl.bps.BpsConfig`
+        Workflow configuration.
+    generic_workflow : `lsst.ctrl.bps.GenericWorkflow`
+        The GenericWorkflow to convert.
+    out_prefix : `str`
+        Location prefix where the HTCondor files will be written.
+
+    Returns
+    -------
+    dag : `lsst.ctrl.bps.htcondor.HTCDag`
+        The HTCDag representation of the given GenericWorkflow.
+    """
+    dag = HTCDag(name=generic_workflow.name)
+
+    _LOG.debug("htcondor dag attribs %s", generic_workflow.run_attrs)
+    dag.add_attribs(generic_workflow.run_attrs)
+    dag.add_attribs(
+        {
+            "bps_run_quanta": create_count_summary(generic_workflow.quanta_counts),
+            "bps_job_summary": create_count_summary(generic_workflow.job_counts),
+        }
+    )
+
+    _, tmp_template = config.search("subDirTemplate", opt={"replaceVars": False, "default": ""})
+    if isinstance(tmp_template, str):
+        subdir_template = defaultdict(lambda: tmp_template)
+    else:
+        subdir_template = tmp_template
+
+    # Create all DAG jobs
+    site_values = {}  # Cache compute site specific values to reduce config lookups.
+    cached_values = {}  # Cache label-specific values to reduce config lookups.
+    # Note: Can't use get_job_by_label because those only include payload jobs.
+    for job_name in generic_workflow:
+        gwjob = generic_workflow.get_job(job_name)
+        if gwjob.node_type == GenericWorkflowNodeType.PAYLOAD:
+            gwjob = cast(GenericWorkflowJob, gwjob)
+            if gwjob.compute_site not in site_values:
+                site_values[gwjob.compute_site] = _gather_site_values(config, gwjob.compute_site)
+            if gwjob.label not in cached_values:
+                cached_values[gwjob.label] = deepcopy(site_values[gwjob.compute_site])
+                cached_values[gwjob.label].update(_gather_label_values(config, gwjob.label))
+                _LOG.debug("cached: %s= %s", gwjob.label, cached_values[gwjob.label])
+            htc_job = _create_job(
+                subdir_template[gwjob.label],
+                cached_values[gwjob.label],
+                generic_workflow,
+                gwjob,
+                out_prefix,
+            )
+        elif gwjob.node_type == GenericWorkflowNodeType.NOOP:
+            gwjob = cast(GenericWorkflowNoopJob, gwjob)
+            htc_job = HTCJob(f"wms_{gwjob.name}", label=gwjob.label)
+            htc_job.subfile = "${CTRL_BPS_HTCONDOR_DIR}/python/lsst/ctrl/bps/htcondor/noop.sub"
+            htc_job.add_job_attrs({"bps_job_name": gwjob.name, "bps_job_label": gwjob.label})
+            htc_job.add_dag_cmds({"noop": True})
+        elif gwjob.node_type == GenericWorkflowNodeType.GROUP:
+            gwjob = cast(GenericWorkflowGroup, gwjob)
+            htc_job = _group_to_subdag(config, gwjob, out_prefix)
+            # In case DAGMAN_GENERATE_SUBDAG_SUBMITS is False,
+            dag.graph["submit_options"]["do_recurse"] = True
+        else:
+            raise RuntimeError(f"Unsupported generic workflow node type {gwjob.node_type} ({gwjob.name})")
+        _LOG.debug("Calling adding job %s %s", htc_job.name, htc_job.label)
+        dag.add_job(htc_job)
+
+    # Add job dependencies to the DAG (be careful with wms_ jobs)
+    for job_name in generic_workflow:
+        gwjob = generic_workflow.get_job(job_name)
+        parent_name = (
+            gwjob.name if gwjob.node_type == GenericWorkflowNodeType.PAYLOAD else f"wms_{gwjob.name}"
+        )
+        successor_jobs = [generic_workflow.get_job(j) for j in generic_workflow.successors(job_name)]
+        children_names = []
+        if gwjob.node_type == GenericWorkflowNodeType.GROUP:
+            gwjob = cast(GenericWorkflowGroup, gwjob)
+            group_children = []  # Dependencies between same group jobs
+            for sjob in successor_jobs:
+                if sjob.node_type == GenericWorkflowNodeType.GROUP and sjob.label == gwjob.label:
+                    group_children.append(f"wms_{sjob.name}")
+                elif sjob.node_type == GenericWorkflowNodeType.PAYLOAD:
+                    children_names.append(sjob.name)
+                else:
+                    children_names.append(f"wms_{sjob.name}")
+            if group_children:
+                dag.add_job_relationships([parent_name], group_children)
+            if not gwjob.blocking:
+                # Since subdag will always succeed, need to add a special
+                # job that fails if group failed to block payload children.
+                check_job = _create_check_job(f"wms_{gwjob.name}", gwjob.label)
+                dag.add_job(check_job)
+                dag.add_job_relationships([f"wms_{gwjob.name}"], [check_job.name])
+                parent_name = check_job.name
+        else:
+            for sjob in successor_jobs:
+                if sjob.node_type == GenericWorkflowNodeType.PAYLOAD:
+                    children_names.append(sjob.name)
+                else:
+                    children_names.append(f"wms_{sjob.name}")
+
+        dag.add_job_relationships([parent_name], children_names)
+
+    # If final job exists in generic workflow, create DAG final job
+    final = generic_workflow.get_final()
+    if final and isinstance(final, GenericWorkflowJob):
+        if final.compute_site and final.compute_site not in site_values:
+            site_values[final.compute_site] = _gather_site_values(config, final.compute_site)
+        if final.label not in cached_values:
+            cached_values[final.label] = deepcopy(site_values[final.compute_site])
+            cached_values[final.label].update(_gather_label_values(config, final.label))
+        final_htjob = _create_job(
+            subdir_template[final.label],
+            cached_values[final.label],
+            generic_workflow,
+            final,
+            out_prefix,
+        )
+        if "post" not in final_htjob.dagcmds:
+            final_htjob.dagcmds["post"] = {
+                "defer": "",
+                "executable": f"{os.path.dirname(__file__)}/final_post.sh",
+                "arguments": f"{final.name} $DAG_STATUS $RETURN",
+            }
+        dag.add_final_job(final_htjob)
+    elif final and isinstance(final, GenericWorkflow):
+        raise NotImplementedError("HTCondor plugin does not support a workflow as the final job")
+    elif final:
+        raise TypeError(f"Invalid type for GenericWorkflow.get_final() results ({type(final)})")
+
+    return dag