lsst-ctrl-bps-htcondor 28.2025.700__py3-none-any.whl → 28.2025.900__py3-none-any.whl

lsst/ctrl/bps/htcondor/htcondor_service.py

@@ -79,7 +79,7 @@ from .lssthtc import (
     read_dag_log,
     read_dag_status,
     read_node_status,
-    summary_from_dag,
+    summarize_dag,
     write_dag_info,
 )
 from .provisioner import Provisioner
@@ -154,7 +154,7 @@ class HTCondorService(BaseWmsService):
             if enable_provisioning:
                 provisioner = Provisioner(config)
                 provisioner.configure()
-                provisioner.prepare("provisioning_job.bash", prefix=out_prefix)
+                provisioner.prepare("provisioningJob.bash", prefix=out_prefix)
                 provisioner.provision(workflow.dag)
 
         with time_this(
@@ -1317,9 +1317,9 @@ def _create_detailed_report_from_jobs(
         job_state_counts=dag_ad.get("state_counts", state_counts),
         exit_code_summary=_get_exit_code_summary(jobs),
     )
-
+    specific_info = WmsSpecificInfo()
     for job_id, job_ad in jobs.items():
-        if not is_service_job(job_id):
+        if not is_service_job(job_ad):
             try:
                 job_report = WmsJobReport(
                     wms_id=job_id,
@@ -1334,33 +1334,85 @@ def _create_detailed_report_from_jobs(
                 _LOG.error("Job missing key '%s': %s", str(ex), job_ad)
                 raise
         else:
-            job_label = job_ad.get("bps_job_label")
-            if job_label is None:
-                _LOG.warning("Service job with id '%s': missing label, no action taken", job_id)
-            elif job_label == dag_ad.get("bps_provisioning_job", "MISS"):
-                report.specific_info = WmsSpecificInfo()
-                job_status = _htc_status_to_wms_state(job_ad)
-                if job_status == WmsStates.DELETED:
-                    if "Reason" in job_ad and "Removed by DAGMan" in job_ad["Reason"]:
-                        job_status = WmsStates.SUCCEEDED
-                report.specific_info.add_message(
-                    template="Provisioning job status: {status}",
-                    context={"status": job_status.name},
-                )
-            else:
-                _LOG.warning(
-                    "Service job with id '%s' (label '%s'): no handler, no action taken", job_id, job_label
-                )
+            _LOG.debug(
+                "Found service job: id='%s', name='%s', label='%s', NodeStatus='%s', JobStatus='%s'",
+                job_id,
+                job_ad["DAGNodeName"],
+                job_ad.get("bps_job_label", "MISS"),
+                job_ad.get("NodeStatus", "MISS"),
+                job_ad.get("JobStatus", "MISS"),
+            )
+            _add_service_job_specific_info(job_ad, specific_info)
+
+    if specific_info:
+        report.specific_info = specific_info
 
     # Add the removed entry to restore the original content of the dictionary.
     # The ordering of keys will be change permanently though.
     jobs.update({wms_workflow_id: dag_ad})
 
+    # Workflow will exit with non-zero DAG_STATUS if problem with
+    # any of the wms jobs.  So change FAILED to SUCCEEDED if all
+    # payload jobs SUCCEEDED.
+    if report.total_number_jobs == report.job_state_counts[WmsStates.SUCCEEDED]:
+        report.state = WmsStates.SUCCEEDED
+
     run_reports = {report.wms_id: report}
     _LOG.debug("_create_detailed_report: run_reports = %s", run_reports)
     return run_reports
 
 
+def _add_service_job_specific_info(job_ad: dict[str, Any], specific_info: WmsSpecificInfo) -> None:
+    """Generate report information for service job.
+
+    Parameters
+    ----------
+    job_ad : `dict` [`str`, `Any`]
+        Provisioning job information.
+    specific_info : `lsst.ctrl.bps.WmsSpecificInfo`
+        Where to add message.
+    """
+    status_details = ""
+    job_status = _htc_status_to_wms_state(job_ad)
+
+    # Service jobs in queue are deleted when DAG is done.
+    # To get accurate status, need to check other info.
+    if (
+        job_status == WmsStates.DELETED
+        and "Reason" in job_ad
+        and (
+            "Removed by DAGMan" in job_ad["Reason"]
+            or "removed because <OtherJobRemoveRequirements = DAGManJobId =?=" in job_ad["Reason"]
+            or "DAG is exiting and writing rescue file." in job_ad["Reason"]
+        )
+    ):
+        if "HoldReason" in job_ad:
+            # HoldReason exists even if released, so check.
+            if "job_released_time" in job_ad and job_ad["job_held_time"] < job_ad["job_released_time"]:
+                # If released, assume running until deleted.
+                job_status = WmsStates.SUCCEEDED
+                status_details = ""
+            else:
+                # If job held when deleted by DAGMan, still want to
+                # report hold reason
+                status_details = f"(Job was held for the following reason: {job_ad['HoldReason']})"
+
+        else:
+            job_status = WmsStates.SUCCEEDED
+    elif job_status == WmsStates.SUCCEEDED:
+        status_details = "(Note: Finished before workflow.)"
+    elif job_status == WmsStates.HELD:
+        status_details = f"({job_ad['HoldReason']})"
+
+    template = "Status of {job_name}: {status} {status_details}"
+    context = {
+        "job_name": job_ad["DAGNodeName"],
+        "status": job_status.name,
+        "status_details": status_details,
+    }
+    specific_info.add_message(template=template, context=context)
+
+
 def _summary_report(user, hist, pass_thru, schedds=None):
     """Gather run information to be used in generating summary reports.
 
@@ -1509,7 +1561,7 @@ def _get_run_summary(job):
     """
     summary = job.get("bps_job_summary", job.get("bps_run_summary", None))
     if not summary:
-        summary, _ = summary_from_dag(job["Iwd"])
+        summary, _, _ = summarize_dag(job["Iwd"])
         if not summary:
             _LOG.warning("Could not get run summary for htcondor job: %s", job)
     _LOG.debug("_get_run_summary: summary=%s", summary)
@@ -1587,7 +1639,7 @@ def _get_state_counts_from_jobs(
     """
     state_counts = dict.fromkeys(WmsStates, 0)
     for job_id, job_ad in jobs.items():
-        if job_id != wms_workflow_id and not is_service_job(job_id):
+        if job_id != wms_workflow_id and not is_service_job(job_ad):
             state_counts[_htc_status_to_wms_state(job_ad)] += 1
     total_counted = sum(state_counts.values())
 
@@ -2143,13 +2195,13 @@ def _gather_site_values(config, compute_site):
     return site_values
 
 
-def is_service_job(job_id: str) -> bool:
+def is_service_job(job_ad: dict[str, Any]) -> bool:
     """Determine if a job is a service one.
 
     Parameters
     ----------
-    job_id : str
-        HTCondor job id.
+    job_ad : `dict` [`str`, Any]
+        Information about an HTCondor job.
 
     Returns
     -------
@@ -2159,10 +2211,7 @@ def is_service_job(job_id: str) -> bool:
     Notes
     -----
     At the moment, HTCondor does not provide a native way to distinguish
-    between payload and service jobs in the workflow. As a result, the current
-    implementation depends entirely on the logic that is used in
-    :py:func:`read_node_status()` (service jobs are given ids with ClusterId=0
-    and ProcId=some integer). If it changes, this function needs to be
-    updated too.
+    between payload and service jobs in the workflow.  This code depends
+    on read_node_status adding bps_job_type.
     """
-    return int(float(job_id)) == 0
+    return job_ad.get("bps_job_type", "MISSING") == "service"

lsst/ctrl/bps/htcondor/lssthtc.py

@@ -63,7 +63,8 @@ __all__ = [
     "read_dag_nodes_log",
     "read_dag_status",
     "read_node_status",
-    "summary_from_dag",
+    "summarize_dag",
+    "update_job_info",
     "update_job_info",
     "write_dag_info",
 ]
@@ -1245,7 +1246,7 @@ def update_job_info(job_info, other_info):
     return job_info
 
 
-def summary_from_dag(dir_name):
+def summarize_dag(dir_name: str) -> tuple[str, dict[str, str], dict[str, str]]:
     """Build bps_run_summary string from dag file.
 
     Parameters
@@ -1256,51 +1257,64 @@ def summary_from_dag(dir_name):
     Returns
     -------
     summary : `str`
-        Semi-colon separated list of job labels and counts.
+        Semi-colon separated list of job labels and counts
         (Same format as saved in dag classad).
     job_name_to_label : `dict` [`str`, `str`]
         Mapping of job names to job labels.
+    job_name_to_type : `dict` [`str`, `str`]
+        Mapping of job names to job types
+        (e.g., payload, final, service).
     """
     # Later code depends upon insertion order
-    counts = defaultdict(int)
+    counts: defaultdict[str, int] = defaultdict(int)  # counts of payload jobs per label
     job_name_to_label = {}
+    job_name_to_type = {}
     try:
         dag = next(Path(dir_name).glob("*.dag"))
         with open(dag) as fh:
             for line in fh:
+                job_name = ""
                 if line.startswith("JOB"):
-                    m = re.match(r'JOB (\S+) "jobs/([^/]+)/', line)
+                    m = re.match(r'JOB (\S+) "?jobs/([^/]+)/', line)
                     if m:
+                        job_name = m.group(1)
                         label = m.group(2)
                         if label == "init":
                             label = "pipetaskInit"
-                        job_name_to_label[m.group(1)] = label
                         counts[label] += 1
                     else:  # Check if Pegasus submission
                         m = re.match(r"JOB (\S+) (\S+)", line)
                         if m:
+                            job_name = m.group(1)
                             label = pegasus_name_to_label(m.group(1))
-                            job_name_to_label[m.group(1)] = label
                             counts[label] += 1
                         else:
                             _LOG.warning("Parse DAG: unmatched job line: %s", line)
+                    job_type = "payload"
                 elif line.startswith("FINAL"):
                     m = re.match(r"FINAL (\S+) jobs/([^/]+)/", line)
                     if m:
+                        job_name = m.group(1)
                         label = m.group(2)
-                        job_name_to_label[m.group(1)] = label
-                        counts[label] += 1
+                        counts[label] += 1  # final counts a payload job.
+                        job_type = "final"
                 elif line.startswith("SERVICE"):
                     m = re.match(r"SERVICE (\S+) jobs/([^/]+)/", line)
                     if m:
+                        job_name = m.group(1)
                         label = m.group(2)
-                        job_name_to_label[m.group(1)] = label
+                        job_type = "service"
+
+                if job_name:
+                    job_name_to_label[job_name] = label
+                    job_name_to_type[job_name] = job_type
+
     except (OSError, PermissionError, StopIteration):
         pass
 
     summary = ";".join([f"{name}:{counts[name]}" for name in counts])
-    _LOG.debug("summary_from_dag: %s %s", summary, job_name_to_label)
-    return summary, job_name_to_label
+    _LOG.debug("summarize_dag: %s %s %s", summary, job_name_to_label, job_name_to_type)
+    return summary, job_name_to_label, job_name_to_type
 
 
 def pegasus_name_to_label(name):
@@ -1400,7 +1414,7 @@ def read_node_status(wms_path):
         file.
     """
     # Get jobid info from other places to fill in gaps in info from node_status
-    _, job_name_to_label = summary_from_dag(wms_path)
+    _, job_name_to_label, job_name_to_type = summarize_dag(wms_path)
     wms_workflow_id, loginfo = read_dag_log(wms_path)
     loginfo = read_dag_nodes_log(wms_path)
     _LOG.debug("loginfo = %s", loginfo)
@@ -1409,17 +1423,17 @@ def read_node_status(wms_path):
         if "LogNotes" in job_info:
             m = re.match(r"DAG Node: (\S+)", job_info["LogNotes"])
             if m:
-                job_name_to_id[m.group(1)] = job_id
-                job_info["DAGNodeName"] = m.group(1)
+                job_name = m.group(1)
+                job_name_to_id[job_name] = job_id
+                job_info["DAGNodeName"] = job_name
+                job_info["bps_job_type"] = job_name_to_type[job_name]
+                job_info["bps_job_label"] = job_name_to_label[job_name]
 
+    jobs = loginfo
+    fake_id = -1.0  # For nodes that do not yet have a job id, give fake one
     try:
         node_status = next(Path(wms_path).glob("*.node_status"))
-    except StopIteration:
-        return loginfo
 
-    jobs = {}
-    fake_id = -1.0  # For nodes that do not yet have a job id, give fake one
-    try:
         with open(node_status) as fh:
             for ad in classad.parseAds(fh):
                 match ad["Type"]:
@@ -1438,22 +1452,19 @@ def read_node_status(wms_path):
                         # Make job info as if came from condor_q.
                         if job_name in job_name_to_id:
                             job_id = str(job_name_to_id[job_name])
+                            job = jobs[job_id]
                         else:
                             job_id = str(fake_id)
+                            job_name_to_id[job_name] = job_id
+                            job = dict(ad)
+                            jobs[job_id] = job
                             fake_id -= 1
-                        job = dict(ad)
                         job["ClusterId"] = int(float(job_id))
                         job["DAGManJobID"] = wms_workflow_id
                         job["DAGNodeName"] = job_name
                         job["bps_job_label"] = job_label
+                        job["bps_job_type"] = job_name_to_type[job_name]
 
-                        # Include information retrieved from the event log
-                        # if available.
-                        jobs[job_id] = job
-                        try:
-                            jobs[job_id] |= loginfo[job_id]
-                        except KeyError:
-                            pass
                     case "StatusEnd":
                         # Skip node status file "epilog".
                         pass
@@ -1463,24 +1474,22 @@ def read_node_status(wms_path):
                             ad["Type"],
                             wms_path,
                         )
-    except (OSError, PermissionError):
+    except (StopIteration, OSError, PermissionError):
         pass
-    else:
-        # Assume that the jobs found in the event log, but *not* in the node
-        # status file are the service jobs as HTCondor does not include
-        # information about these jobs in the node status file at the moment.
-        #
-        # Note: To be able to easily identify the service jobs downstream,
-        # we reverse the ClusterId and ProcId in their HTCondor ids in internal
-        # use. For example, if HTCondor id of a service job is '1.0', we will
-        # use '0.1' instead.
-        service_jobs = {job_id: loginfo[job_id] for job_id in set(loginfo) - set(jobs)}
-        job_id_to_name = {
-            job_id: job_name for job_name, job_id in job_name_to_id.items() if job_id in service_jobs
-        }
-        for job_id, job_info in service_jobs.items():
-            job_info["bps_job_label"] = job_name_to_label[job_id_to_name[job_id]]
-            jobs[f"{job_info['ProcId']}.{job_info['ClusterId']}"] = job_info
+
+    # Check for missing jobs (e.g., submission failure or not submitted yet)
+    # Use dag info to create job placeholders
+    for name in set(job_name_to_label) - set(job_name_to_id):
+        job = {}
+        job["ClusterId"] = int(float(fake_id))
+        job["ProcId"] = 0
+        job["DAGManJobID"] = wms_workflow_id
+        job["DAGNodeName"] = name
+        job["bps_job_label"] = job_name_to_label[name]
+        job["bps_job_type"] = job_name_to_type[name]
+        job["NodeStatus"] = NodeStatus.NOT_READY
+        jobs[f"{job['ClusterId']}.{job['ProcId']}"] = job
+        fake_id -= 1
 
     return jobs
 

lsst/ctrl/bps/htcondor/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "28.2025.700"
+__version__ = "28.2025.900"

{lsst_ctrl_bps_htcondor-28.2025.700.dist-info → lsst_ctrl_bps_htcondor-28.2025.900.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lsst-ctrl-bps-htcondor
-Version: 28.2025.700
+Version: 28.2025.900
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_htcondor-28.2025.700.dist-info → lsst_ctrl_bps_htcondor-28.2025.900.dist-info}/RECORD

@@ -2,18 +2,18 @@ lsst/ctrl/bps/htcondor/__init__.py,sha256=1gTmOVLJILvBqgqHVECo8uqoX8e4fiTeH_dHBU
 lsst/ctrl/bps/htcondor/final_post.sh,sha256=chfaQV6Q7rGsK-8Hx58ch52m-PofvBanrl7VwCssHec,248
 lsst/ctrl/bps/htcondor/handlers.py,sha256=2gM3Ac00in4ob9ckcP331W1LSEjs9UDKIqt4MULA4bg,11196
 lsst/ctrl/bps/htcondor/htcondor_config.py,sha256=c4lCiYEwEXFdxgbMfEkbDm4LrvkRMF31SqLtQqzqIV4,1523
-lsst/ctrl/bps/htcondor/htcondor_service.py,sha256=UGRxT4cT5giQbEmQQfU546oEm2dxDXQRr46YajhRqh4,80335
-lsst/ctrl/bps/htcondor/lssthtc.py,sha256=kouYnXE9tiWGNupkNki0kU26r8p_PhFTOT4w9nNCsXs,56978
+lsst/ctrl/bps/htcondor/htcondor_service.py,sha256=nWStzzQVO-Th_t8AEbrUIMN8qyKGdfYrKGjdWS8r66A,82060
+lsst/ctrl/bps/htcondor/lssthtc.py,sha256=Um7xycgLLJO9EAZiTHBpFQ9xPTpWQzLf1rZaUvbhGdU,57273
 lsst/ctrl/bps/htcondor/provisioner.py,sha256=hPN8YJUtwNHQylw68kfskF1S2vCeQvztF8W0d_QKqqM,7851
-lsst/ctrl/bps/htcondor/version.py,sha256=2cqkmTChym3nVzfo4P4IJHnN6xCQuoQRtosac3HQaWA,54
+lsst/ctrl/bps/htcondor/version.py,sha256=SMLiEYE3TidCx-PaKCULel49al2Cp64_dNg6h9nCkUc,54
 lsst/ctrl/bps/htcondor/etc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml,sha256=xDRts4vHKov2PE_JRh-0nF3jfuNJXtKBXZqveASp_iA,1422
-lsst_ctrl_bps_htcondor-28.2025.700.dist-info/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
-lsst_ctrl_bps_htcondor-28.2025.700.dist-info/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps_htcondor-28.2025.700.dist-info/METADATA,sha256=fSyyHtHkKUofg48ciI4Vy2_J3ZaRGamdnpoZ7gZ-9dY,2116
-lsst_ctrl_bps_htcondor-28.2025.700.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-lsst_ctrl_bps_htcondor-28.2025.700.dist-info/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps_htcondor-28.2025.700.dist-info/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps_htcondor-28.2025.700.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps_htcondor-28.2025.700.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps_htcondor-28.2025.700.dist-info/RECORD,,
+lsst_ctrl_bps_htcondor-28.2025.900.dist-info/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
+lsst_ctrl_bps_htcondor-28.2025.900.dist-info/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps_htcondor-28.2025.900.dist-info/METADATA,sha256=DN87N3GKWII4gc5F9eQpSqbPyNQGhgp2yAAfyJ5Oc1Y,2116
+lsst_ctrl_bps_htcondor-28.2025.900.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+lsst_ctrl_bps_htcondor-28.2025.900.dist-info/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps_htcondor-28.2025.900.dist-info/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps_htcondor-28.2025.900.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps_htcondor-28.2025.900.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps_htcondor-28.2025.900.dist-info/RECORD,,

{lsst_ctrl_bps_htcondor-28.2025.700.dist-info → lsst_ctrl_bps_htcondor-28.2025.900.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any