lsst-ctrl-bps-htcondor 29.2025.4100__py3-none-any.whl → 29.2025.4300__py3-none-any.whl

lsst/ctrl/bps/htcondor/common_utils.py

@@ -115,7 +115,10 @@ def _htc_job_status_to_wms_state(job):
         elif job_status == htcondor.JobStatus.RUNNING:
             wms_state = WmsStates.RUNNING
         elif job_status == htcondor.JobStatus.REMOVED:
-            wms_state = WmsStates.DELETED
+            if (job.get("ExitBySignal", False) and job.get("ExitSignal", 0)) or job.get("ExitCode", 0):
+                wms_state = WmsStates.FAILED
+            else:
+                wms_state = WmsStates.DELETED
         elif job_status == htcondor.JobStatus.COMPLETED:
             if (
                 (job.get("ExitBySignal", False) and job.get("ExitSignal", 0))

lsst/ctrl/bps/htcondor/handlers.py

@@ -31,6 +31,8 @@ __all__ = [
     "HTC_JOB_AD_HANDLERS",
     "Chain",
     "Handler",
+    "JobAbortedByPeriodicRemoveHandler",
+    "JobAbortedByUserHandler",
     "JobCompletedWithExecTicketHandler",
     "JobCompletedWithoutExecTicketHandler",
     "JobHeldByOtherHandler",
@@ -43,7 +45,7 @@ import abc
 import logging
 import re
 from collections.abc import Sequence
-from typing import Any
+from typing import Any, overload
 
 _LOG = logging.getLogger(__name__)
 
@@ -84,13 +86,17 @@ class Chain(Sequence):
         List of handlers that will be used to initialize the chain.
     """
 
-    def __init__(self, handlers: Sequence[Handler] = None) -> None:
-        self._handlers = []
+    def __init__(self, handlers: Sequence[Handler] | None = None) -> None:
+        self._handlers: list[Handler] = []
         if handlers is not None:
             for handler in handlers:
                 self.append(handler)
 
-    def __getitem__(self, index: int) -> Handler:
+    @overload
+    def __getitem__(self, index: int) -> Handler: ...
+    @overload
+    def __getitem__(self, index: slice) -> Sequence[Handler]: ...
+    def __getitem__(self, index):
         return self._handlers[index]
 
     def __len__(self) -> int:
@@ -207,7 +213,7 @@ class JobCompletedWithoutExecTicketHandler(Handler):
                 ad["ExitCode"] = ad["ReturnValue"]
         else:
             _LOG.debug(
-                "%s: refusing to process the ad for the job '%s.%s': ticket of execution found",
+                "Handler %s: refusing to process the ad for the job '%s.%s': ticket of execution found",
                 self.__class__.__name__,
                 ad["ClusterId"],
                 ad["ProcId"],
@@ -314,7 +320,98 @@ class JobHeldByUserHandler(Handler):
         return ad
 
 
+class JobAbortedByPeriodicRemoveHandler(Handler):
+    """Handler of ClassAds for jobs deleted by periodic remove policy."""
+
+    def handle(self, ad: dict[str, Any]) -> dict[str, Any] | None:
+        if not ad["MyType"].endswith("AbortedEvent"):
+            _LOG.debug(
+                "Handler '%s': refusing to process the ad for the job '%s.%s': job not removed",
+                self.__class__.__name__,
+                ad["ClusterId"],
+                ad["ProcId"],
+            )
+            return None
+        if "Reason" in ad:
+            if "PeriodicRemove" in ad["Reason"]:
+                ad["ExitBySignal"] = True
+
+                ad["ExitSignal"] = -1
+                if "HoldReason" in ad:
+                    match = re.search(r"signal (\d+)", ad["HoldReason"])
+                    if match is not None:
+                        ad["ExitSignal"] = int(match.group(1))
+
+            else:
+                _LOG.debug(
+                    "Handler '%s': refusing to process the ad for the job '%s.%s': "
+                    "job was not removed by the periodic removal policy: Reason = %s",
+                    self.__class__.__name__,
+                    ad["ClusterId"],
+                    ad["ProcId"],
+                    ad["Reason"],
+                )
+                return None
+        else:
+            _LOG.debug(
+                "Handler '%s': refusing to process the ad for the job '%s.%s': "
+                "unable to determine the reason for the removal.",
+                self.__class__.__name__,
+                ad["ClusterId"],
+                ad["ProcId"],
+            )
+            return None
+        return ad
+
+
+class JobAbortedByUserHandler(Handler):
+    """Handler of ClassAds for jobs deleted by the user."""
+
+    def handle(self, ad: dict[str, Any]) -> dict[str, Any] | None:
+        if not ad["MyType"].endswith("AbortedEvent"):
+            _LOG.debug(
+                "Handler '%s': refusing to process the ad for the job '%s.%s': job not removed",
+                self.__class__.__name__,
+                ad["ClusterId"],
+                ad["ProcId"],
+            )
+            return None
+        if "Reason" in ad:
+            patterns = (
+                "Python-initiated action",  # DAGMan job removed by the user
+                "DAG Removed",  # payload job removed by the user
+                "OtherJobRemoveRequirements",  # a subdag job removed by the user
+            )
+            for patt in patterns:
+                if patt in ad["Reason"]:
+                    ad["ExitBySignal"] = False
+                    ad["ExitCode"] = 0
+                    break
+            else:
+                _LOG.debug(
+                    "Handler '%s': refusing to process the ad for the job '%s.%s': "
+                    "job not removed by the user: Reason = %s",
+                    self.__class__.__name__,
+                    ad["ClusterId"],
+                    ad["ProcId"],
+                    ad["Reason"],
+                )
+                return None
+        else:
+            _LOG.debug(
+                "Handler '%s': refusing to process the ad for the job '%s.%s': "
+                "unable to determine the reason for the removal.",
+                self.__class__.__name__,
+                ad["ClusterId"],
+                ad["ProcId"],
+            )
+            return None
+        return ad
+
+
 _handlers = [
+    JobAbortedByPeriodicRemoveHandler(),
+    JobAbortedByUserHandler(),
     JobHeldByUserHandler(),
     JobHeldBySignalHandler(),
     JobHeldByOtherHandler(),

lsst/ctrl/bps/htcondor/lssthtc.py

@@ -54,6 +54,7 @@ __all__ = [
     "htc_query_history",
     "htc_query_present",
     "htc_submit_dag",
+    "htc_tweak_log_info",
     "htc_version",
     "htc_write_attribs",
     "htc_write_condor_file",
@@ -1890,8 +1891,6 @@ def read_single_dag_log(log_filename: str | os.PathLike) -> tuple[str, dict[str,
 
         # only save latest DAG job
         dag_info = {wms_workflow_id: info[wms_workflow_id]}
-        for job in dag_info.values():
-            _tweak_log_info(filename, job)
 
     return wms_workflow_id, dag_info
 
@@ -1992,10 +1991,6 @@ def read_single_dag_nodes_log(filename: str | os.PathLike) -> dict[str, dict[str
                 _update_dicts(info[id_], event)
                 info[id_][f"{event.type.name.lower()}_time"] = event["EventTime"]
 
-    # Add more condor_q-like info to info parsed from log file.
-    for job in info.values():
-        _tweak_log_info(filename, job)
-
     return info
 
 
@@ -2091,60 +2086,69 @@ def write_dag_info(filename, dag_info):
         _LOG.debug("Persisting DAGMan job information failed: %s", exc)
 
 
-def _tweak_log_info(filename, job):
+def htc_tweak_log_info(wms_path: str | Path, job: dict[str, Any]) -> None:
     """Massage the given job info has same structure as if came from condor_q.
 
     Parameters
     ----------
-    filename : `pathlib.Path`
-        Name of the DAGMan log.
+    wms_path : `str` | `os.PathLike`
+        Path containing an HTCondor event log file.
     job : `dict` [ `str`, `~typing.Any` ]
         A mapping between HTCondor job id and job information read from
         the log.
     """
-    _LOG.debug("_tweak_log_info: %s %s", filename, job)
+    _LOG.debug("htc_tweak_log_info: %s %s", wms_path, job)
+
+    # Use the presence of 'MyType' key as a proxy to determine if the job ad
+    # contains the info extracted from the event log. Exit early if it doesn't
+    # (e.g. it is a job ad for a pruned job).
+    if "MyType" not in job:
+        return
 
     try:
         job["ClusterId"] = job["Cluster"]
         job["ProcId"] = job["Proc"]
-        job["Iwd"] = str(filename.parent)
-        job["Owner"] = filename.owner()
-
-        match job["MyType"]:
-            case "ExecuteEvent":
-                job["JobStatus"] = htcondor.JobStatus.RUNNING
-            case "JobTerminatedEvent" | "PostScriptTerminatedEvent":
-                job["JobStatus"] = htcondor.JobStatus.COMPLETED
-            case "SubmitEvent":
-                job["JobStatus"] = htcondor.JobStatus.IDLE
-            case "JobAbortedEvent":
-                job["JobStatus"] = htcondor.JobStatus.REMOVED
-            case "JobHeldEvent":
-                job["JobStatus"] = htcondor.JobStatus.HELD
-            case "JobReleaseEvent":
-                # Shows up as last event if a DAG job was held and released
-                # so assume job is running.  If regular job is released, there
-                # will be other events so JobReleaseEvent won't be the last
-                job["JobStatus"] = htcondor.JobStatus.RUNNING
-            case _:
-                _LOG.debug("Unknown log event type: %s", job["MyType"])
-                job["JobStatus"] = None
-
-        if job["JobStatus"] in {htcondor.JobStatus.COMPLETED, htcondor.JobStatus.HELD}:
-            new_job = HTC_JOB_AD_HANDLERS.handle(job)
-            if new_job is not None:
-                job = new_job
-            else:
-                _LOG.error("Could not determine exit status for job '%s.%s'", job["ClusterId"], job["ProcId"])
-
-        if "LogNotes" in job:
-            m = re.match(r"DAG Node: (\S+)", job["LogNotes"])
-            if m:
-                job["DAGNodeName"] = m.group(1)
-
     except KeyError as e:
         _LOG.error("Missing key %s in job: %s", str(e), job)
         raise
+    job["Iwd"] = str(wms_path)
+    job["Owner"] = Path(wms_path).owner()
+
+    match job["MyType"]:
+        case "ExecuteEvent":
+            job["JobStatus"] = htcondor.JobStatus.RUNNING
+        case "JobTerminatedEvent" | "PostScriptTerminatedEvent":
+            job["JobStatus"] = htcondor.JobStatus.COMPLETED
+        case "SubmitEvent":
+            job["JobStatus"] = htcondor.JobStatus.IDLE
+        case "JobAbortedEvent":
+            job["JobStatus"] = htcondor.JobStatus.REMOVED
+        case "JobHeldEvent":
+            job["JobStatus"] = htcondor.JobStatus.HELD
+        case "JobReleaseEvent":
+            # Shows up as the last event if a DAG job was held and released,
+            # so assume the job is running.  If a regular job is released,
+            # there will be other events, so JobReleaseEvent won't be the last.
+            job["JobStatus"] = htcondor.JobStatus.RUNNING
+        case _:
+            _LOG.debug("Unknown log event type: %s", job["MyType"])
+            job["JobStatus"] = None
+
+    if job["JobStatus"] in {
+        htcondor.JobStatus.COMPLETED,
+        htcondor.JobStatus.HELD,
+        htcondor.JobStatus.REMOVED,
+    }:
+        new_job = HTC_JOB_AD_HANDLERS.handle(job)
+        if new_job is not None:
+            job = new_job
+        else:
+            _LOG.error("Could not determine exit status for job '%s.%s'", job["ClusterId"], job["ProcId"])
+
+    if "LogNotes" in job:
+        m = re.match(r"DAG Node: (\S+)", job["LogNotes"])
+        if m:
+            job["DAGNodeName"] = m.group(1)
 
 
 def htc_check_dagman_output(wms_path: str | os.PathLike) -> str:

lsst/ctrl/bps/htcondor/report_utils.py

@@ -48,6 +48,7 @@ from .lssthtc import (
     WmsNodeType,
     condor_search,
     htc_check_dagman_output,
+    htc_tweak_log_info,
     pegasus_name_to_label,
     read_dag_info,
     read_dag_log,
@@ -125,6 +126,7 @@ def _get_status_from_path(wms_path: str | os.PathLike) -> tuple[WmsStates, str]:
     if wms_workflow_id == MISSING_ID:
         state = WmsStates.UNKNOWN
     else:
+        htc_tweak_log_info(wms_path, dag_ad[wms_workflow_id])
         state = _htc_status_to_wms_state(dag_ad[wms_workflow_id])
 
     return state, message
@@ -363,6 +365,10 @@ def _get_info_from_path(wms_path: str | os.PathLike) -> tuple[str, dict[str, dic
         wms_workflow_id = MISSING_ID
         jobs = {}
 
+    # Add more condor_q-like info.
+    for job in jobs.values():
+        htc_tweak_log_info(wms_path, job)
+
     message = "\n".join([msg for msg in messages if msg])
     _LOG.debug("wms_workflow_id = %s, jobs = %s", wms_workflow_id, jobs.keys())
     _LOG.debug("message = %s", message)
@@ -692,12 +698,11 @@ def _get_exit_code_summary(jobs):
             exit_code = 0
             job_status = job_ad["JobStatus"]
             match job_status:
-                case htcondor.JobStatus.COMPLETED | htcondor.JobStatus.HELD:
+                case htcondor.JobStatus.COMPLETED | htcondor.JobStatus.HELD | htcondor.JobStatus.REMOVED:
                     exit_code = job_ad["ExitSignal"] if job_ad["ExitBySignal"] else job_ad["ExitCode"]
                 case (
                     htcondor.JobStatus.IDLE
                     | htcondor.JobStatus.RUNNING
-                    | htcondor.JobStatus.REMOVED
                     | htcondor.JobStatus.TRANSFERRING_OUTPUT
                     | htcondor.JobStatus.SUSPENDED
                 ):

lsst/ctrl/bps/htcondor/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.4100"
+__version__ = "29.2025.4300"

{lsst_ctrl_bps_htcondor-29.2025.4100.dist-info → lsst_ctrl_bps_htcondor-29.2025.4300.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps-htcondor
-Version: 29.2025.4100
+Version: 29.2025.4300
 Summary: HTCondor plugin for lsst-ctrl-bps.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps_htcondor-29.2025.4100.dist-info → lsst_ctrl_bps_htcondor-29.2025.4300.dist-info}/RECORD

@@ -1,23 +1,23 @@
 lsst/ctrl/bps/htcondor/__init__.py,sha256=8PVRtHS2tn_BDH9gnWQ-Fbg7o7wTbZoW7f41RGf0g7A,1450
-lsst/ctrl/bps/htcondor/common_utils.py,sha256=IgONfw1CRGn1vKJO4LLQDcY3FgH7COSVgquSlYF76Qg,9573
+lsst/ctrl/bps/htcondor/common_utils.py,sha256=cHdPaOwS2I5T4RpGChdMAaqis7T3B2dR45zu8cVzpto,9744
 lsst/ctrl/bps/htcondor/final_post.sh,sha256=chfaQV6Q7rGsK-8Hx58ch52m-PofvBanrl7VwCssHec,248
-lsst/ctrl/bps/htcondor/handlers.py,sha256=2gM3Ac00in4ob9ckcP331W1LSEjs9UDKIqt4MULA4bg,11196
+lsst/ctrl/bps/htcondor/handlers.py,sha256=fkTEKulfwYOMofya9PzbvCiI9WNLfj_yTnno8Sm3srQ,14860
 lsst/ctrl/bps/htcondor/htcondor_config.py,sha256=c4lCiYEwEXFdxgbMfEkbDm4LrvkRMF31SqLtQqzqIV4,1523
 lsst/ctrl/bps/htcondor/htcondor_service.py,sha256=dnpxje5XRI0TEui-oXdp9kKlnTMiOZZNk0jDJIjNFDE,22177
 lsst/ctrl/bps/htcondor/htcondor_workflow.py,sha256=wkANkAA4Ciq9WP_DWkjH2k0xWz9_i6gaNHWcxWQ4zkM,3071
-lsst/ctrl/bps/htcondor/lssthtc.py,sha256=d6rXXRAuX3SUVTXvzD8Qw9zNxFWhCnniK32AwKg42_s,81382
+lsst/ctrl/bps/htcondor/lssthtc.py,sha256=c7eecmDgEkfxASwPJ645rireM0Pe6WdMImnUT_3y-SA,81454
 lsst/ctrl/bps/htcondor/prepare_utils.py,sha256=XdNAuEY72pzxpU01kmZz9LYc_VU_yw3Trl9-PTXBsVg,36449
 lsst/ctrl/bps/htcondor/provisioner.py,sha256=DxhCOCpqyBXIBR2m8VL_FwaDMr2scQIOe8ArWjgQ_Ls,7929
-lsst/ctrl/bps/htcondor/report_utils.py,sha256=5MDB_AH11FH_k0OwOIWy1lqsWYce3qKUVaA8KR-oI48,30949
-lsst/ctrl/bps/htcondor/version.py,sha256=imQpc8l8n1Xj3guWKoFaXrBR3HnAY8jkMITBuZx-5DA,55
+lsst/ctrl/bps/htcondor/report_utils.py,sha256=7idf6GRfMhCt5OuuCzHnnhPq5YVYm8k0bqS301MEjC8,31123
+lsst/ctrl/bps/htcondor/version.py,sha256=pmH-QwBOGm7hpNEwNmgwgFuJ9J8NNZQ81N3RGqBJ3YU,55
 lsst/ctrl/bps/htcondor/etc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml,sha256=C6DKJKmKFKczukpXVXev9u1-vmv2IcgcdtjTtgJWDQM,1561
-lsst_ctrl_bps_htcondor-29.2025.4100.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
-lsst_ctrl_bps_htcondor-29.2025.4100.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps_htcondor-29.2025.4100.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps_htcondor-29.2025.4100.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps_htcondor-29.2025.4100.dist-info/METADATA,sha256=D1NeD0wVzRrC-CW49LF6Cerni8YUl4svOEDOm4sttDo,2139
-lsst_ctrl_bps_htcondor-29.2025.4100.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_ctrl_bps_htcondor-29.2025.4100.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps_htcondor-29.2025.4100.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps_htcondor-29.2025.4100.dist-info/RECORD,,
+lsst_ctrl_bps_htcondor-29.2025.4300.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
+lsst_ctrl_bps_htcondor-29.2025.4300.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps_htcondor-29.2025.4300.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps_htcondor-29.2025.4300.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps_htcondor-29.2025.4300.dist-info/METADATA,sha256=j5skk3s8N-lpAQoAOuHwyThCrg4C-cavV4zX00zv15I,2139
+lsst_ctrl_bps_htcondor-29.2025.4300.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_ctrl_bps_htcondor-29.2025.4300.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps_htcondor-29.2025.4300.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps_htcondor-29.2025.4300.dist-info/RECORD,,