lsst-ctrl-bps-htcondor 28.2025.800__tar.gz → 29.0.0rc1__tar.gz

{lsst_ctrl_bps_htcondor-28.2025.800/python/lsst_ctrl_bps_htcondor.egg-info → lsst_ctrl_bps_htcondor-29.0.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lsst-ctrl-bps-htcondor
-Version: 28.2025.800
+Version: 29.0.0rc1
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_htcondor-28.2025.800 → lsst_ctrl_bps_htcondor-29.0.0rc1}/doc/lsst.ctrl.bps.htcondor/userguide.rst

@@ -148,6 +148,17 @@ from files.  So, the detailed report can distinguish between failed and
 deleted jobs, and thus will show ``D`` in the flag column for a running
 workflow if there is a deleted job.
 
+Rarely, a detailed report may warn about job submission issues.  For example:
+
+.. code-block:: bash
+
+   Warn: Job submission issues (last: 01/30/25 10:36:57)
+
+A job submission issue could be intermittent or not.  It may cause
+problems with the status or counts in the reports.  To get more information
+about the submission issue, look in the ``*.dag.dagman.out`` file for
+errors, in particular lines containing ``submit attempt failed``.
+
 Occasionally, some jobs are put on hold by HTCondor.  To see the reason why
 jobs are being held, use
 
@@ -276,12 +287,27 @@ Look for the line starting with "Provisioning job status".  For example
    calibrate               0      0       1     0       0       0       0    0         0      0      0        1
    finalJob                0      0       1     0       0       0       0    0         0      0      0        1
 
+If the provisioning job status is UNREADY, check the end of the report to see
+if there is a warning about submission issues.  There may be a temporary problem.
+Check the ``*.dag.dagman.out`` in run submit directory for errors, in
+particular for ``ERROR: submit attempt failed``.
+
+If the provisioning job status is HELD, the hold reason will appear in parentheses.
+
 The service job managing the glideins will be automatically canceled once the
 workflow is completed.  However, the existing glideins will be left for
 HTCondor to shut them down once they remain inactive for the period specified
 by ``provisioningMaxIdleTime`` (default value: 15 min., see below) or maximum
 wall time is reached.
 
+The provisioning job is expected to run as long as the workflow.  If the job
+dies, the job status will be `FAILED`.  If the job just completed successfully,
+the job status will be `SUCCEEDED` with a message saying it ended early (which
+may or may not cause a problem since existing glideins could remain running).
+To get more information about either of these cases, check the job output
+and error files in the `jobs/provisioningJob` subdirectory.
+
+
 If the automatic provisioning of the resources is enabled, the script that the
 service job is supposed to run in order to provide the required resources *must
 be* defined by the ``provisioningScript`` setting in the ``provisioning``

{lsst_ctrl_bps_htcondor-28.2025.800 → lsst_ctrl_bps_htcondor-29.0.0rc1}/pyproject.toml

@@ -113,6 +113,15 @@ convention = "numpy"
 # not fit on one line.
 add-ignore = ["D107", "D105", "D102", "D100", "D200", "D205", "D400", "D104"]
 
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+]
+
 [tool.ruff]
 target-version = "py311"
 line-length = 110

{lsst_ctrl_bps_htcondor-28.2025.800 → lsst_ctrl_bps_htcondor-29.0.0rc1}/python/lsst/ctrl/bps/htcondor/htcondor_service.py

@@ -79,7 +79,7 @@ from .lssthtc import (
     read_dag_log,
     read_dag_status,
     read_node_status,
-    summary_from_dag,
+    summarize_dag,
     write_dag_info,
 )
 from .provisioner import Provisioner
@@ -154,7 +154,7 @@ class HTCondorService(BaseWmsService):
             if enable_provisioning:
                 provisioner = Provisioner(config)
                 provisioner.configure()
-                provisioner.prepare("provisioning_job.bash", prefix=out_prefix)
+                provisioner.prepare("provisioningJob.bash", prefix=out_prefix)
                 provisioner.provision(workflow.dag)
 
         with time_this(
@@ -1317,9 +1317,9 @@ def _create_detailed_report_from_jobs(
         job_state_counts=dag_ad.get("state_counts", state_counts),
         exit_code_summary=_get_exit_code_summary(jobs),
     )
-
+    specific_info = WmsSpecificInfo()
     for job_id, job_ad in jobs.items():
-        if not is_service_job(job_id):
+        if not is_service_job(job_ad):
             try:
                 job_report = WmsJobReport(
                     wms_id=job_id,
@@ -1334,33 +1334,85 @@ def _create_detailed_report_from_jobs(
                 _LOG.error("Job missing key '%s': %s", str(ex), job_ad)
                 raise
         else:
-            job_label = job_ad.get("bps_job_label")
-            if job_label is None:
-                _LOG.warning("Service job with id '%s': missing label, no action taken", job_id)
-            elif job_label == dag_ad.get("bps_provisioning_job", "MISS"):
-                report.specific_info = WmsSpecificInfo()
-                job_status = _htc_status_to_wms_state(job_ad)
-                if job_status == WmsStates.DELETED:
-                    if "Reason" in job_ad and "Removed by DAGMan" in job_ad["Reason"]:
-                        job_status = WmsStates.SUCCEEDED
-                report.specific_info.add_message(
-                    template="Provisioning job status: {status}",
-                    context={"status": job_status.name},
-                )
-            else:
-                _LOG.warning(
-                    "Service job with id '%s' (label '%s'): no handler, no action taken", job_id, job_label
-                )
+            _LOG.debug(
+                "Found service job: id='%s', name='%s', label='%s', NodeStatus='%s', JobStatus='%s'",
+                job_id,
+                job_ad["DAGNodeName"],
+                job_ad.get("bps_job_label", "MISS"),
+                job_ad.get("NodeStatus", "MISS"),
+                job_ad.get("JobStatus", "MISS"),
+            )
+            _add_service_job_specific_info(job_ad, specific_info)
+
+    if specific_info:
+        report.specific_info = specific_info
 
     # Add the removed entry to restore the original content of the dictionary.
     # The ordering of keys will be change permanently though.
     jobs.update({wms_workflow_id: dag_ad})
 
+    # Workflow will exit with non-zero DAG_STATUS if problem with
+    # any of the wms jobs.  So change FAILED to SUCCEEDED if all
+    # payload jobs SUCCEEDED.
+    if report.total_number_jobs == report.job_state_counts[WmsStates.SUCCEEDED]:
+        report.state = WmsStates.SUCCEEDED
+
     run_reports = {report.wms_id: report}
     _LOG.debug("_create_detailed_report: run_reports = %s", run_reports)
     return run_reports
 
 
+def _add_service_job_specific_info(job_ad: dict[str, Any], specific_info: WmsSpecificInfo) -> None:
+    """Generate report information for service job.
+
+    Parameters
+    ----------
+    job_ad : `dict` [`str`, `Any`]
+        Provisioning job information.
+    specific_info : `lsst.ctrl.bps.WmsSpecificInfo`
+        Where to add message.
+    """
+    status_details = ""
+    job_status = _htc_status_to_wms_state(job_ad)
+
+    # Service jobs in queue are deleted when DAG is done.
+    # To get accurate status, need to check other info.
+    if (
+        job_status == WmsStates.DELETED
+        and "Reason" in job_ad
+        and (
+            "Removed by DAGMan" in job_ad["Reason"]
+            or "removed because <OtherJobRemoveRequirements = DAGManJobId =?=" in job_ad["Reason"]
+            or "DAG is exiting and writing rescue file." in job_ad["Reason"]
+        )
+    ):
+        if "HoldReason" in job_ad:
+            # HoldReason exists even if released, so check.
+            if "job_released_time" in job_ad and job_ad["job_held_time"] < job_ad["job_released_time"]:
+                # If released, assume running until deleted.
+                job_status = WmsStates.SUCCEEDED
+                status_details = ""
+            else:
+                # If job held when deleted by DAGMan, still want to
+                # report hold reason
+                status_details = f"(Job was held for the following reason: {job_ad['HoldReason']})"
+
+        else:
+            job_status = WmsStates.SUCCEEDED
+    elif job_status == WmsStates.SUCCEEDED:
+        status_details = "(Note: Finished before workflow.)"
+    elif job_status == WmsStates.HELD:
+        status_details = f"({job_ad['HoldReason']})"
+
+    template = "Status of {job_name}: {status} {status_details}"
+    context = {
+        "job_name": job_ad["DAGNodeName"],
+        "status": job_status.name,
+        "status_details": status_details,
+    }
+    specific_info.add_message(template=template, context=context)
+
+
 def _summary_report(user, hist, pass_thru, schedds=None):
     """Gather run information to be used in generating summary reports.
 
@@ -1509,7 +1561,7 @@ def _get_run_summary(job):
     """
     summary = job.get("bps_job_summary", job.get("bps_run_summary", None))
     if not summary:
-        summary, _ = summary_from_dag(job["Iwd"])
+        summary, _, _ = summarize_dag(job["Iwd"])
         if not summary:
             _LOG.warning("Could not get run summary for htcondor job: %s", job)
     _LOG.debug("_get_run_summary: summary=%s", summary)
@@ -1587,7 +1639,7 @@ def _get_state_counts_from_jobs(
     """
     state_counts = dict.fromkeys(WmsStates, 0)
     for job_id, job_ad in jobs.items():
-        if job_id != wms_workflow_id and not is_service_job(job_id):
+        if job_id != wms_workflow_id and not is_service_job(job_ad):
             state_counts[_htc_status_to_wms_state(job_ad)] += 1
     total_counted = sum(state_counts.values())
 
@@ -2143,13 +2195,13 @@ def _gather_site_values(config, compute_site):
     return site_values
 
 
-def is_service_job(job_id: str) -> bool:
+def is_service_job(job_ad: dict[str, Any]) -> bool:
     """Determine if a job is a service one.
 
     Parameters
     ----------
-    job_id : str
-        HTCondor job id.
+    job_ad : `dict` [`str`, Any]
+        Information about an HTCondor job.
 
     Returns
     -------
@@ -2159,10 +2211,7 @@ def is_service_job(job_id: str) -> bool:
     Notes
     -----
     At the moment, HTCondor does not provide a native way to distinguish
-    between payload and service jobs in the workflow. As a result, the current
-    implementation depends entirely on the logic that is used in
-    :py:func:`read_node_status()` (service jobs are given ids with ClusterId=0
-    and ProcId=some integer). If it changes, this function needs to be
-    updated too.
+    between payload and service jobs in the workflow.  This code depends
+    on read_node_status adding bps_job_type.
     """
-    return int(float(job_id)) == 0
+    return job_ad.get("bps_job_type", "MISSING") == "service"

{lsst_ctrl_bps_htcondor-28.2025.800 → lsst_ctrl_bps_htcondor-29.0.0rc1}/python/lsst/ctrl/bps/htcondor/lssthtc.py

@@ -63,7 +63,8 @@ __all__ = [
     "read_dag_nodes_log",
     "read_dag_status",
     "read_node_status",
-    "summary_from_dag",
+    "summarize_dag",
+    "update_job_info",
     "update_job_info",
     "write_dag_info",
 ]
@@ -1245,7 +1246,7 @@ def update_job_info(job_info, other_info):
     return job_info
 
 
-def summary_from_dag(dir_name):
+def summarize_dag(dir_name: str) -> tuple[str, dict[str, str], dict[str, str]]:
     """Build bps_run_summary string from dag file.
 
     Parameters
@@ -1256,51 +1257,64 @@ def summary_from_dag(dir_name):
     Returns
     -------
     summary : `str`
-        Semi-colon separated list of job labels and counts.
+        Semi-colon separated list of job labels and counts
         (Same format as saved in dag classad).
     job_name_to_label : `dict` [`str`, `str`]
         Mapping of job names to job labels.
+    job_name_to_type : `dict` [`str`, `str`]
+        Mapping of job names to job types
+        (e.g., payload, final, service).
     """
     # Later code depends upon insertion order
-    counts = defaultdict(int)
+    counts: defaultdict[str, int] = defaultdict(int)  # counts of payload jobs per label
     job_name_to_label = {}
+    job_name_to_type = {}
     try:
         dag = next(Path(dir_name).glob("*.dag"))
         with open(dag) as fh:
             for line in fh:
+                job_name = ""
                 if line.startswith("JOB"):
-                    m = re.match(r'JOB (\S+) "jobs/([^/]+)/', line)
+                    m = re.match(r'JOB (\S+) "?jobs/([^/]+)/', line)
                     if m:
+                        job_name = m.group(1)
                         label = m.group(2)
                         if label == "init":
                             label = "pipetaskInit"
-                        job_name_to_label[m.group(1)] = label
                         counts[label] += 1
                     else:  # Check if Pegasus submission
                         m = re.match(r"JOB (\S+) (\S+)", line)
                         if m:
+                            job_name = m.group(1)
                             label = pegasus_name_to_label(m.group(1))
-                            job_name_to_label[m.group(1)] = label
                             counts[label] += 1
                         else:
                             _LOG.warning("Parse DAG: unmatched job line: %s", line)
+                    job_type = "payload"
                 elif line.startswith("FINAL"):
                     m = re.match(r"FINAL (\S+) jobs/([^/]+)/", line)
                     if m:
+                        job_name = m.group(1)
                         label = m.group(2)
-                        job_name_to_label[m.group(1)] = label
-                        counts[label] += 1
+                        counts[label] += 1  # final counts a payload job.
+                        job_type = "final"
                 elif line.startswith("SERVICE"):
                     m = re.match(r"SERVICE (\S+) jobs/([^/]+)/", line)
                     if m:
+                        job_name = m.group(1)
                         label = m.group(2)
-                        job_name_to_label[m.group(1)] = label
+                        job_type = "service"
+
+                if job_name:
+                    job_name_to_label[job_name] = label
+                    job_name_to_type[job_name] = job_type
+
     except (OSError, PermissionError, StopIteration):
         pass
 
     summary = ";".join([f"{name}:{counts[name]}" for name in counts])
-    _LOG.debug("summary_from_dag: %s %s", summary, job_name_to_label)
-    return summary, job_name_to_label
+    _LOG.debug("summarize_dag: %s %s %s", summary, job_name_to_label, job_name_to_type)
+    return summary, job_name_to_label, job_name_to_type
 
 
 def pegasus_name_to_label(name):
@@ -1400,7 +1414,7 @@ def read_node_status(wms_path):
         file.
     """
     # Get jobid info from other places to fill in gaps in info from node_status
-    _, job_name_to_label = summary_from_dag(wms_path)
+    _, job_name_to_label, job_name_to_type = summarize_dag(wms_path)
     wms_workflow_id, loginfo = read_dag_log(wms_path)
     loginfo = read_dag_nodes_log(wms_path)
     _LOG.debug("loginfo = %s", loginfo)
@@ -1409,17 +1423,17 @@ def read_node_status(wms_path):
         if "LogNotes" in job_info:
             m = re.match(r"DAG Node: (\S+)", job_info["LogNotes"])
             if m:
-                job_name_to_id[m.group(1)] = job_id
-                job_info["DAGNodeName"] = m.group(1)
+                job_name = m.group(1)
+                job_name_to_id[job_name] = job_id
+                job_info["DAGNodeName"] = job_name
+                job_info["bps_job_type"] = job_name_to_type[job_name]
+                job_info["bps_job_label"] = job_name_to_label[job_name]
 
+    jobs = loginfo
+    fake_id = -1.0  # For nodes that do not yet have a job id, give fake one
     try:
         node_status = next(Path(wms_path).glob("*.node_status"))
-    except StopIteration:
-        return loginfo
 
-    jobs = {}
-    fake_id = -1.0  # For nodes that do not yet have a job id, give fake one
-    try:
         with open(node_status) as fh:
             for ad in classad.parseAds(fh):
                 match ad["Type"]:
@@ -1438,22 +1452,19 @@ def read_node_status(wms_path):
                         # Make job info as if came from condor_q.
                         if job_name in job_name_to_id:
                             job_id = str(job_name_to_id[job_name])
+                            job = jobs[job_id]
                         else:
                             job_id = str(fake_id)
+                            job_name_to_id[job_name] = job_id
+                            job = dict(ad)
+                            jobs[job_id] = job
                             fake_id -= 1
-                        job = dict(ad)
                         job["ClusterId"] = int(float(job_id))
                         job["DAGManJobID"] = wms_workflow_id
                         job["DAGNodeName"] = job_name
                         job["bps_job_label"] = job_label
+                        job["bps_job_type"] = job_name_to_type[job_name]
 
-                        # Include information retrieved from the event log
-                        # if available.
-                        jobs[job_id] = job
-                        try:
-                            jobs[job_id] |= loginfo[job_id]
-                        except KeyError:
-                            pass
                     case "StatusEnd":
                         # Skip node status file "epilog".
                         pass
@@ -1463,24 +1474,22 @@ def read_node_status(wms_path):
                             ad["Type"],
                             wms_path,
                         )
-    except (OSError, PermissionError):
+    except (StopIteration, OSError, PermissionError):
         pass
-    else:
-        # Assume that the jobs found in the event log, but *not* in the node
-        # status file are the service jobs as HTCondor does not include
-        # information about these jobs in the node status file at the moment.
-        #
-        # Note: To be able to easily identify the service jobs downstream,
-        # we reverse the ClusterId and ProcId in their HTCondor ids in internal
-        # use. For example, if HTCondor id of a service job is '1.0', we will
-        # use '0.1' instead.
-        service_jobs = {job_id: loginfo[job_id] for job_id in set(loginfo) - set(jobs)}
-        job_id_to_name = {
-            job_id: job_name for job_name, job_id in job_name_to_id.items() if job_id in service_jobs
-        }
-        for job_id, job_info in service_jobs.items():
-            job_info["bps_job_label"] = job_name_to_label[job_id_to_name[job_id]]
-            jobs[f"{job_info['ProcId']}.{job_info['ClusterId']}"] = job_info
+
+    # Check for missing jobs (e.g., submission failure or not submitted yet)
+    # Use dag info to create job placeholders
+    for name in set(job_name_to_label) - set(job_name_to_id):
+        job = {}
+        job["ClusterId"] = int(float(fake_id))
+        job["ProcId"] = 0
+        job["DAGManJobID"] = wms_workflow_id
+        job["DAGNodeName"] = name
+        job["bps_job_label"] = job_name_to_label[name]
+        job["bps_job_type"] = job_name_to_type[name]
+        job["NodeStatus"] = NodeStatus.NOT_READY
+        jobs[f"{job['ClusterId']}.{job['ProcId']}"] = job
+        fake_id -= 1
 
     return jobs
 

lsst_ctrl_bps_htcondor-29.0.0rc1/python/lsst/ctrl/bps/htcondor/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.0.0rc1"

{lsst_ctrl_bps_htcondor-28.2025.800 → lsst_ctrl_bps_htcondor-29.0.0rc1/python/lsst_ctrl_bps_htcondor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lsst-ctrl-bps-htcondor
-Version: 28.2025.800
+Version: 29.0.0rc1
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_htcondor-28.2025.800 → lsst_ctrl_bps_htcondor-29.0.0rc1}/tests/test_htcondor_service.py

@@ -31,24 +31,28 @@ import logging
 import os
 import unittest
 from pathlib import Path
-from shutil import copy2
+from shutil import copy2, copytree
 
 import htcondor
 
-from lsst.ctrl.bps import BpsConfig, GenericWorkflowExec, GenericWorkflowJob, WmsStates
+from lsst.ctrl.bps import BpsConfig, GenericWorkflowExec, GenericWorkflowJob, WmsSpecificInfo, WmsStates
 from lsst.ctrl.bps.htcondor.htcondor_config import HTC_DEFAULTS_URI
 from lsst.ctrl.bps.htcondor.htcondor_service import (
     HTCondorService,
     JobStatus,
     NodeStatus,
     WmsIdType,
+    _add_service_job_specific_info,
+    _create_detailed_report_from_jobs,
     _get_exit_code_summary,
     _get_info_from_path,
+    _get_run_summary,
     _get_state_counts_from_dag_job,
     _htc_node_status_to_wms_state,
     _htc_status_to_wms_state,
     _translate_job_cmds,
     _wms_id_to_dir,
+    is_service_job,
 )
 from lsst.ctrl.bps.htcondor.lssthtc import MISSING_ID
 from lsst.utils.tests import temporaryDirectory
@@ -532,3 +536,491 @@ class WmsIdToDirTestCase(unittest.TestCase):
             self.assertEqual(id_type, WmsIdType.PATH)
             self.assertEqual(abs_path.resolve(), wms_path)
             os.chdir(orig_dir)
+
+
+class AddServiceJobSpecificInfoTestCase(unittest.TestCase):
+    """Test _add_service_job_specific_info function.
+
+    Note: The job_ad's are hardcoded in these tests.  The
+    values in the dictionaries come from plugin code as
+    well as HTCondor.  Changes in either of those codes
+    that produce data for the job_ad can break this
+    function without breaking these unit tests.
+
+    Also, since hold status/messages stick around, testing
+    various cases with and without job being held just to
+    ensure get right status in both cases.
+    """
+
+    def testNotSubmitted(self):
+        # Service job not submitted yet or can't be submitted.
+        # (Typically an plugin bug.)
+        # At this function level, can't tell if not submitted
+        # yet or problem so it never will.
+        job_ad = {
+            "ClusterId": -64,
+            "DAGManJobID": "8997.0",
+            "DAGNodeName": "provisioningJob",
+            "NodeStatus": NodeStatus.NOT_READY,
+            "ProcId": 0,
+            "bps_job_label": "service_provisioningJob",
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context, {"job_name": "provisioningJob", "status": "UNREADY", "status_details": ""}
+        )
+
+    def testRunning(self):
+        # DAG hasn't completed (Running or held),
+        # Service job is running.
+        job_ad = {
+            "ClusterId": 8523,
+            "ProcId": 0,
+            "DAGNodeName": "provisioningJob",
+            "JobStatus": JobStatus.RUNNING,
+        }
+
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context, {"job_name": "provisioningJob", "status": "RUNNING", "status_details": ""}
+        )
+
+    def testDied(self):
+        # DAG hasn't completed (Running or held),
+        # Service job failed (completed non-zero exit code)
+        job_ad = {
+            "ClusterId": 8761,
+            "ProcId": 0,
+            "DAGNodeName": "provisioningJob",
+            "JobStatus": JobStatus.COMPLETED,
+            "ExitCode": 4,
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context, {"job_name": "provisioningJob", "status": "FAILED", "status_details": ""}
+        )
+
+    def testDeleted(self):
+        # Deleted by user (never held)
+        job_ad = {
+            "ClusterId": 9086,
+            "DAGNodeName": "provisioningJob",
+            "JobStatus": JobStatus.REMOVED,
+            "ProcId": 0,
+            "Reason": "via condor_rm (by user mgower)",
+            "job_evicted_time": "2025-02-11T11:35:04",
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context, {"job_name": "provisioningJob", "status": "DELETED", "status_details": ""}
+        )
+
+    def testSucceedEarly(self):
+        # DAG hasn't completed (Running or held),
+        # Service job completed with exit code 0
+        job_ad = {
+            "ClusterId": 8761,
+            "ProcId": 0,
+            "DAGNodeName": "provisioningJob",
+            "JobStatus": JobStatus.COMPLETED,
+            "ExitCode": 0,
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context,
+            {
+                "job_name": "provisioningJob",
+                "status": "SUCCEEDED",
+                "status_details": "(Note: Finished before workflow.)",
+            },
+        )
+
+    def testSucceedOldRemoveMessage(self):
+        # DAG completed, job was in running state when removed.
+        job_ad = {
+            "ClusterId": 8761,
+            "ProcId": 0,
+            "DAGNodeName": "provisioningJob",
+            "JobStatus": JobStatus.REMOVED,
+            "Reason": "Removed by DAGMan (by user mgower)",
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context, {"job_name": "provisioningJob", "status": "SUCCEEDED", "status_details": ""}
+        )
+
+    def testSucceed(self):
+        # DAG completed, job was in running state when removed.
+        job_ad = {
+            "ClusterId": 8761,
+            "ProcId": 0,
+            "DAGNodeName": "provisioningJob",
+            "JobStatus": JobStatus.REMOVED,
+            "Reason": (
+                "removed because <OtherJobRemoveRequirements = DAGManJobId =?= 8556>"
+                " fired when job (8556.0) was removed"
+            ),
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context, {"job_name": "provisioningJob", "status": "SUCCEEDED", "status_details": ""}
+        )
+
+    def testUserHeldWhileRunning(self):
+        # DAG hasn't completed (Running or held),
+        # user put at least service job on hold
+        job_ad = {
+            "ClusterId": 8523,
+            "ProcId": 0,
+            "DAGNodeName": "provisioningJob",
+            "JobStatus": JobStatus.HELD,
+            "HoldReason": "via condor_hold (by user mgower)",
+            "HoldReasonCode": 1,
+            "HoldReasonSubCode": 0,
+        }
+
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context,
+            {
+                "job_name": "provisioningJob",
+                "status": "HELD",
+                "status_details": "(via condor_hold (by user mgower))",
+            },
+        )
+
+    def testHeldByHTC(self):
+        # Job put on hold by HTCondor, removed when DAG ends
+        job_ad = {
+            "ClusterId": 8693,
+            "DAGNodeName": "provisioningJob",
+            "HoldReason": "Failed to execute",
+            "HoldReasonCode": 6,
+            "HoldReasonSubCode": 2,
+            "JobStatus": JobStatus.REMOVED,
+            "ProcId": 0,
+            "Reason": "Removed by DAGMan (by user mgower)",
+            "job_held_time": "2025-02-07T12:50:07",
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context,
+            {
+                "job_name": "provisioningJob",
+                "status": "DELETED",
+                "status_details": "(Job was held for the following reason: Failed to execute)",
+            },
+        )
+
+    def testHeldReleasedRunning(self):
+        # DAG hasn't completed (Running or held),
+        # Since held info will be in job_ad, make sure knows released.
+        job_ad = {
+            "ClusterId": 8625,
+            "DAGNodeName": "provisioningJob",
+            "HoldReason": "via condor_hold (by user mgower)",
+            "HoldReasonCode": 1,
+            "HoldReasonSubCode": 0,
+            "JobStatus": JobStatus.RUNNING,
+            "LogNotes": "DAG Node: provisioningJob",
+            "ProcId": 0,
+            "job_held_time": "2025-02-07T12:33:34",
+            "job_released_time": "2025-02-07T12:33:47",
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context, {"job_name": "provisioningJob", "status": "RUNNING", "status_details": ""}
+        )
+
+    def testHeldReleasedDied(self):
+        # Since held info will be in job_ad,
+        # make sure knows status after released.
+        job_ad = {
+            "ClusterId": 9120,
+            "DAGNodeName": "provisioningJob",
+            "ExitBySignal": False,
+            "ExitCode": 4,
+            "HoldReason": "via condor_hold (by user mgower)",
+            "HoldReasonCode": 1,
+            "HoldReasonSubCode": 0,
+            "JobStatus": JobStatus.COMPLETED,
+            "ProcId": 0,
+            "Reason": "via condor_release (by user mgower)",
+            "ReturnValue": 4,
+            "TerminatedNormally": True,
+            "job_held_time": "2025-02-11T11:46:40",
+            "job_released_time": "2025-02-11T11:46:47",
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context, {"job_name": "provisioningJob", "status": "FAILED", "status_details": ""}
+        )
+
+    def testHeldReleasedSuccessEarly(self):
+        # Since held info will be in job_ad,
+        # make sure knows status after released.
+        job_ad = {
+            "ClusterId": 9154,
+            "DAGNodeName": "provisioningJob",
+            "ExitBySignal": False,
+            "ExitCode": 0,
+            "HoldReason": "via condor_hold (by user mgower)",
+            "HoldReasonCode": 1,
+            "HoldReasonSubCode": 0,
+            "JobStatus": JobStatus.COMPLETED,
+            "ProcId": 0,
+            "Reason": "via condor_release (by user mgower)",
+            "TerminatedNormally": True,
+            "job_held_time": "2025-02-11T11:55:20",
+            "job_released_time": "2025-02-11T11:55:25",
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context,
+            {
+                "job_name": "provisioningJob",
+                "status": "SUCCEEDED",
+                "status_details": "(Note: Finished before workflow.)",
+            },
+        )
+
+    def testHeldReleasedSuccess(self):
+        # DAG has completed.
+        # Since held info will be in job_ad,
+        # make sure knows status after released.
+        job_ad = {
+            "ClusterId": 8625,
+            "DAGNodeName": "provisioningJob",
+            "HoldReason": "via condor_hold (by user mgower)",
+            "HoldReasonCode": 1,
+            "HoldReasonSubCode": 0,
+            "JobStatus": JobStatus.REMOVED,
+            "ProcId": 0,
+            "Reason": "removed because <OtherJobRemoveRequirements = DAGManJobId =?= "
+            "8624> fired when job (8624.0) was removed",
+            "job_held_time": "2025-02-07T12:33:34",
+            "job_released_time": "2025-02-07T12:33:47",
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context, {"job_name": "provisioningJob", "status": "SUCCEEDED", "status_details": ""}
+        )
+
+    def testHeldReleasedDeleted(self):
+        # Since held info will be in job_ad,
+        # make sure knows status after released.
+        job_ad = {
+            "ClusterId": 9086,
+            "DAGNodeName": "provisioningJob",
+            "HoldReason": "via condor_hold (by user mgower)",
+            "HoldReasonCode": 1,
+            "HoldReasonSubCode": 0,
+            "JobStatus": JobStatus.REMOVED,
+            "ProcId": 0,
+            "Reason": "via condor_rm (by user mgower)",
+            "job_evicted_time": "2025-02-11T11:35:04",
+            "job_held_time": "2025-02-11T11:35:04",
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context, {"job_name": "provisioningJob", "status": "DELETED", "status_details": ""}
+        )
+
+    def testHeldReleasedHeld(self):
+        # Since release info will be in job_ad,
+        # make sure knows held after release.
+        job_ad = {
+            "ClusterId": 8659,
+            "DAGNodeName": "provisioningJob",
+            "HoldReason": "via condor_hold (by user mgower)",
+            "HoldReasonCode": 1,
+            "HoldReasonSubCode": 0,
+            "JobStatus": JobStatus.REMOVED,
+            "ProcId": 0,
+            "Reason": "Removed by DAGMan (by user mgower)",
+            "TerminatedNormally": False,
+            "job_held_time": "2025-02-07T12:36:15",
+            "job_released_time": "2025-02-07T12:36:07",
+        }
+        results = WmsSpecificInfo()
+        _add_service_job_specific_info(job_ad, results)
+        self.assertEqual(
+            results.context,
+            {
+                "job_name": "provisioningJob",
+                "status": "DELETED",
+                "status_details": "(Job was held for the following reason: via condor_hold (by user mgower))",
+            },
+        )
+
+
+class GetRunSummaryTestCase(unittest.TestCase):
+    """Test _get_run_summary function."""
+
+    def testJobSummaryInJobAd(self):
+        summary = "pipetaskInit:1;label1:2;label2:2;finalJob:1"
+        job_ad = {"ClusterId": 8659, "DAGNodeName": "testJob", "bps_job_summary": summary}
+        results = _get_run_summary(job_ad)
+        self.assertEqual(results, summary)
+
+    def testRunSummaryInJobAd(self):
+        summary = "pipetaskInit:1;label1:2;label2:2;finalJob:1"
+        job_ad = {"ClusterId": 8659, "DAGNodeName": "testJob", "bps_run_summary": summary}
+        results = _get_run_summary(job_ad)
+        self.assertEqual(results, summary)
+
+    def testSummaryFromDag(self):
+        with temporaryDirectory() as tmp_dir:
+            copy2(f"{TESTDIR}/data/good.dag", tmp_dir)
+            job_ad = {"ClusterId": 8659, "DAGNodeName": "testJob", "Iwd": tmp_dir}
+            results = _get_run_summary(job_ad)
+            self.assertEqual(results, "pipetaskInit:1;label1:1;label2:1;label3:1;finalJob:1")
+
+    def testSummaryNoDag(self):
+        with self.assertLogs(logger=logger, level="WARNING") as cm:
+            with temporaryDirectory() as tmp_dir:
+                job_ad = {"ClusterId": 8659, "DAGNodeName": "testJob", "Iwd": tmp_dir}
+                results = _get_run_summary(job_ad)
+                self.assertEqual(results, "")
+            self.assertIn("lsst.ctrl.bps.htcondor", cm.records[0].name)
+            self.assertIn("Could not get run summary for htcondor job", cm.output[0])
+
+
+class IsServiceJobTestCase(unittest.TestCase):
+    """Test is_service_job function."""
+
+    def testNotServiceJob(self):
+        job_ad = {"ClusterId": 8659, "DAGNodeName": "testJob", "bps_job_type": "payload"}
+        self.assertFalse(is_service_job(job_ad))
+
+    def testIsServiceJob(self):
+        job_ad = {"ClusterId": 8659, "DAGNodeName": "testJob", "bps_job_type": "service"}
+        self.assertTrue(is_service_job(job_ad))
+
+    def testMissingBpsType(self):
+        job_ad = {
+            "ClusterId": 8659,
+            "DAGNodeName": "testJob",
+        }
+        self.assertFalse(is_service_job(job_ad))
+
+
+class CreateDetailedReportFromJobsTestCase(unittest.TestCase):
+    """Test _create_detailed_report_from_jobs function."""
+
+    def testTinySuccess(self):
+        with temporaryDirectory() as tmp_dir:
+            test_submit_dir = os.path.join(tmp_dir, "tiny_success")
+            copytree(f"{TESTDIR}/data/tiny_success", test_submit_dir)
+            wms_workflow_id, jobs, message = _get_info_from_path(test_submit_dir)
+            run_reports = _create_detailed_report_from_jobs(wms_workflow_id, jobs)
+            self.assertEqual(len(run_reports), 1)
+            report = run_reports[wms_workflow_id]
+            self.assertEqual(report.wms_id, wms_workflow_id)
+            self.assertEqual(report.state, WmsStates.SUCCEEDED)
+            self.assertTrue(os.path.samefile(report.path, test_submit_dir))
+            self.assertEqual(report.run_summary, "pipetaskInit:1;label1:1;label2:1;finalJob:1")
+            self.assertEqual(
+                report.job_state_counts,
+                {
+                    WmsStates.UNKNOWN: 0,
+                    WmsStates.MISFIT: 0,
+                    WmsStates.UNREADY: 0,
+                    WmsStates.READY: 0,
+                    WmsStates.PENDING: 0,
+                    WmsStates.RUNNING: 0,
+                    WmsStates.DELETED: 0,
+                    WmsStates.HELD: 0,
+                    WmsStates.SUCCEEDED: 4,
+                    WmsStates.FAILED: 0,
+                    WmsStates.PRUNED: 0,
+                },
+            )
+            self.assertEqual(
+                report.specific_info.context,
+                {"job_name": "provisioningJob", "status": "SUCCEEDED", "status_details": ""},
+            )
+
+    def testTinyProblems(self):
+        with temporaryDirectory() as tmp_dir:
+            test_submit_dir = os.path.join(tmp_dir, "tiny_problems")
+            copytree(f"{TESTDIR}/data/tiny_problems", test_submit_dir)
+            wms_workflow_id, jobs, message = _get_info_from_path(test_submit_dir)
+            run_reports = _create_detailed_report_from_jobs(wms_workflow_id, jobs)
+            self.assertEqual(len(run_reports), 1)
+            report = run_reports[wms_workflow_id]
+            self.assertEqual(report.wms_id, wms_workflow_id)
+            self.assertEqual(report.state, WmsStates.FAILED)
+            self.assertTrue(os.path.samefile(report.path, test_submit_dir))
+            self.assertEqual(report.run_summary, "pipetaskInit:1;label1:2;label2:2;finalJob:1")
+            self.assertEqual(
+                report.job_state_counts,
+                {
+                    WmsStates.UNKNOWN: 0,
+                    WmsStates.MISFIT: 0,
+                    WmsStates.UNREADY: 0,
+                    WmsStates.READY: 0,
+                    WmsStates.PENDING: 0,
+                    WmsStates.RUNNING: 0,
+                    WmsStates.DELETED: 0,
+                    WmsStates.HELD: 0,
+                    WmsStates.SUCCEEDED: 4,
+                    WmsStates.FAILED: 1,
+                    WmsStates.PRUNED: 1,
+                },
+            )
+            self.assertEqual(
+                run_reports[wms_workflow_id].specific_info.context,
+                {"job_name": "provisioningJob", "status": "SUCCEEDED", "status_details": ""},
+            )
+
+    def testTinyRunning(self):
+        with temporaryDirectory() as tmp_dir:
+            test_submit_dir = os.path.join(tmp_dir, "tiny_running")
+            copytree(f"{TESTDIR}/data/tiny_running", test_submit_dir)
+            wms_workflow_id, jobs, message = _get_info_from_path(test_submit_dir)
+            run_reports = _create_detailed_report_from_jobs(wms_workflow_id, jobs)
+            self.assertEqual(len(run_reports), 1)
+            report = run_reports[wms_workflow_id]
+            self.assertEqual(report.wms_id, wms_workflow_id)
+            self.assertEqual(report.state, WmsStates.RUNNING)
+            self.assertTrue(os.path.samefile(report.path, test_submit_dir))
+            self.assertEqual(report.run_summary, "pipetaskInit:1;label1:1;label2:1;finalJob:1")
+            self.assertEqual(
+                report.job_state_counts,
+                {
+                    WmsStates.UNKNOWN: 0,
+                    WmsStates.MISFIT: 0,
+                    WmsStates.UNREADY: 2,
+                    WmsStates.READY: 0,
+                    WmsStates.PENDING: 0,
+                    WmsStates.RUNNING: 1,
+                    WmsStates.DELETED: 0,
+                    WmsStates.HELD: 0,
+                    WmsStates.SUCCEEDED: 1,
+                    WmsStates.FAILED: 0,
+                    WmsStates.PRUNED: 0,
+                },
+            )
+            self.assertEqual(
+                report.specific_info.context,
+                {"job_name": "provisioningJob", "status": "RUNNING", "status_details": ""},
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

{lsst_ctrl_bps_htcondor-28.2025.800 → lsst_ctrl_bps_htcondor-29.0.0rc1}/tests/test_lssthtc.py

@@ -31,7 +31,7 @@ import os
 import pathlib
 import tempfile
 import unittest
-from shutil import copy2
+from shutil import copy2, rmtree
 
 import htcondor
 
@@ -197,22 +197,23 @@ class HtcCheckDagmanOutputTestCase(unittest.TestCase):
             self.assertEqual("", results)
 
 
-class SummaryFromDagTestCase(unittest.TestCase):
-    """Test summary_from_dag function."""
+class SummarizeDagTestCase(unittest.TestCase):
+    """Test summarize_dag function."""
 
     def test_no_dag_file(self):
         with temporaryDirectory() as tmp_dir:
-            summary, job_name_to_pipetask = lssthtc.summary_from_dag(tmp_dir)
+            summary, job_name_to_pipetask, job_name_to_type = lssthtc.summarize_dag(tmp_dir)
             self.assertFalse(len(job_name_to_pipetask))
+            self.assertFalse(len(job_name_to_type))
             self.assertFalse(summary)
 
     def test_success(self):
         with temporaryDirectory() as tmp_dir:
             copy2(f"{TESTDIR}/data/good.dag", tmp_dir)
-            summary, job_name_to_pipetask = lssthtc.summary_from_dag(tmp_dir)
+            summary, job_name_to_label, job_name_to_type = lssthtc.summarize_dag(tmp_dir)
             self.assertEqual(summary, "pipetaskInit:1;label1:1;label2:1;label3:1;finalJob:1")
             self.assertEqual(
-                job_name_to_pipetask,
+                job_name_to_label,
                 {
                     "pipetaskInit": "pipetaskInit",
                     "0682f8f9-12f0-40a5-971e-8b30c7231e5c_label1_val1_val2": "label1",
@@ -221,6 +222,98 @@ class SummaryFromDagTestCase(unittest.TestCase):
                     "finalJob": "finalJob",
                 },
             )
+            self.assertEqual(
+                job_name_to_type,
+                {
+                    "pipetaskInit": "payload",
+                    "0682f8f9-12f0-40a5-971e-8b30c7231e5c_label1_val1_val2": "payload",
+                    "d0305e2d-f164-4a85-bd24-06afe6c84ed9_label2_val1_val2": "payload",
+                    "2806ecc9-1bba-4362-8fff-ab4e6abb9f83_label3_val1_val2": "payload",
+                    "finalJob": "final",
+                },
+            )
+
+    def test_service(self):
+        with temporaryDirectory() as tmp_dir:
+            copy2(f"{TESTDIR}/data/tiny_problems/tiny_problems.dag", tmp_dir)
+            summary, job_name_to_label, job_name_to_type = lssthtc.summarize_dag(tmp_dir)
+            self.assertEqual(summary, "pipetaskInit:1;label1:2;label2:2;finalJob:1")
+            self.assertEqual(
+                job_name_to_label,
+                {
+                    "pipetaskInit": "pipetaskInit",
+                    "4a7f478b-2e9b-435c-a730-afac3f621658_label1_val1_val2a": "label1",
+                    "057c8caf-66f6-4612-abf7-cdea5b666b1b_label1_val1_val2b": "label1",
+                    "696ee50d-e711-40d6-9caf-ee29ae4a656d_label2_val1_val2a": "label2",
+                    "40040b97-606d-4997-98d3-e0493055fe7e_label2_val1_val2b": "label2",
+                    "finalJob": "finalJob",
+                    "provisioningJob": "provisioningJob",
+                },
+            )
+            self.assertEqual(
+                job_name_to_type,
+                {
+                    "pipetaskInit": "payload",
+                    "4a7f478b-2e9b-435c-a730-afac3f621658_label1_val1_val2a": "payload",
+                    "057c8caf-66f6-4612-abf7-cdea5b666b1b_label1_val1_val2b": "payload",
+                    "696ee50d-e711-40d6-9caf-ee29ae4a656d_label2_val1_val2a": "payload",
+                    "40040b97-606d-4997-98d3-e0493055fe7e_label2_val1_val2b": "payload",
+                    "finalJob": "final",
+                    "provisioningJob": "service",
+                },
+            )
+
+
+class ReadDagNodesLogTestCase(unittest.TestCase):
+    """Test read_dag_nodes_log function."""
+
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        rmtree(self.tmpdir, ignore_errors=True)
+
+    def testFileMissing(self):
+        with self.assertRaisesRegex(FileNotFoundError, "DAGMan node log not found in"):
+            _, _ = lssthtc.read_dag_nodes_log(self.tmpdir)
+
+
+class ReadNodeStatusTestCase(unittest.TestCase):
+    """Test read_node_status function."""
+
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        rmtree(self.tmpdir, ignore_errors=True)
+
+    def testServiceJobNotSubmitted(self):
+        # tiny_prov_no_submit files have successful workflow
+        # but provisioningJob could not submit.
+        copy2(f"{TESTDIR}/data/tiny_prov_no_submit/tiny_prov_no_submit.dag.nodes.log", self.tmpdir)
+        copy2(f"{TESTDIR}/data/tiny_prov_no_submit/tiny_prov_no_submit.dag.dagman.log", self.tmpdir)
+        copy2(f"{TESTDIR}/data/tiny_prov_no_submit/tiny_prov_no_submit.node_status", self.tmpdir)
+        copy2(f"{TESTDIR}/data/tiny_prov_no_submit/tiny_prov_no_submit.dag", self.tmpdir)
+
+        jobs = lssthtc.read_node_status(self.tmpdir)
+        found = [id_ for id_ in jobs if jobs[id_].get("bps_job_type", "MISS") == "service"]
+        self.assertEqual(len(found), 1)
+        self.assertEqual(jobs[found[0]]["DAGNodeName"], "provisioningJob")
+        self.assertEqual(jobs[found[0]]["NodeStatus"], lssthtc.NodeStatus.NOT_READY)
+
+    def testMissingStatusFile(self):
+        copy2(f"{TESTDIR}/data/tiny_problems/tiny_problems.dag.nodes.log", self.tmpdir)
+        copy2(f"{TESTDIR}/data/tiny_problems/tiny_problems.dag.dagman.log", self.tmpdir)
+        copy2(f"{TESTDIR}/data/tiny_problems/tiny_problems.dag", self.tmpdir)
+
+        jobs = lssthtc.read_node_status(self.tmpdir)
+        self.assertEqual(len(jobs), 7)
+        self.assertEqual(jobs["9230.0"]["DAGNodeName"], "pipetaskInit")
+        self.assertEqual(jobs["9230.0"]["bps_job_type"], "payload")
+        self.assertEqual(jobs["9230.0"]["JobStatus"], lssthtc.JobStatus.COMPLETED)
+        found = [id_ for id_ in jobs if jobs[id_].get("bps_job_type", "MISS") == "service"]
+        self.assertEqual(len(found), 1)
+        self.assertEqual(jobs[found[0]]["DAGNodeName"], "provisioningJob")
 
 
 if __name__ == "__main__":

lsst_ctrl_bps_htcondor-28.2025.800/python/lsst/ctrl/bps/htcondor/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "28.2025.800"