lsst-ctrl-bps 29.2025.3700__py3-none-any.whl → 29.2025.3900__py3-none-any.whl

lsst/ctrl/bps/bps_reports.py

@@ -27,7 +27,14 @@
 
 """Classes and functions used in reporting run status."""
 
-__all__ = ["BaseRunReport", "DetailedRunReport", "ExitCodesReport", "SummaryRunReport", "compile_job_summary"]
+__all__ = [
+    "BaseRunReport",
+    "DetailedRunReport",
+    "ExitCodesReport",
+    "SummaryRunReport",
+    "compile_code_summary",
+    "compile_job_summary",
+]
 
 import abc
 import logging
@@ -55,7 +62,7 @@ class BaseRunReport(abc.ABC):
 
     def __eq__(self, other):
         if isinstance(other, BaseRunReport):
-            return all(self._table == other._table)
+            return self._table.pformat() == other._table.pformat()
         return False
 
     def __len__(self):
@@ -195,7 +202,7 @@ class DetailedRunReport(BaseRunReport):
         job_summary = run_report.job_summary
         if job_summary is None:
             id_ = run_report.global_wms_id if use_global_id else run_report.wms_id
-            self._msg = f"WARNING: Job summary for run '{id_}' not available, report maybe incomplete."
+            self._msg = f"WARNING: Job summary for run '{id_}' not available, report may be incomplete."
             return
 
         if by_label_expected:
@@ -231,44 +238,60 @@ class ExitCodesReport(BaseRunReport):
     error handling from the wms service.
     """
 
-    def add(self, run_report, use_global_id=False):
+    def add(self, run_report: WmsRunReport, use_global_id: bool = False) -> None:
         # Docstring inherited from the base class.
 
-        # Use label ordering from the run summary as it should reflect
-        # the ordering of the pipetasks in the pipeline.
+        exit_code_summary = run_report.exit_code_summary
+        if not exit_code_summary:
+            id_ = run_report.global_wms_id if use_global_id else run_report.wms_id
+            self._msg = f"WARNING: Exit code summary for run '{id_}' not available, report may be incomplete."
+            return
+
+        warnings = []
+
+        # If available, use label ordering from the run summary as it should
+        # reflect the ordering of the pipetasks in the pipeline.
         labels = []
         if run_report.run_summary:
             for part in run_report.run_summary.split(";"):
                 label, _ = part.split(":")
                 labels.append(label)
-        else:
-            id_ = run_report.global_wms_id if use_global_id else run_report.wms_id
-            self._msg = f"WARNING: Job summary for run '{id_}' not available, report maybe incomplete."
-            return
+        if not labels:
+            labels = sorted(exit_code_summary)
+            warnings.append("WARNING: Could not determine order of pipeline, instead sorted alphabetically.")
 
         # Payload (e.g. pipetask) error codes:
         # * 1: general failure,
         # * 2: command line error (e.g. unknown command and/or option).
         pyld_error_codes = {1, 2}
 
-        exit_code_summary = run_report.exit_code_summary
+        missing_labels = set()
         for label in labels:
-            exit_codes = exit_code_summary[label]
-
-            pyld_errors = [code for code in exit_codes if code in pyld_error_codes]
-            pyld_error_count = len(pyld_errors)
-            pyld_error_summary = (
-                ", ".join(sorted(str(code) for code in set(pyld_errors))) if pyld_errors else "None"
-            )
-
-            infra_errors = [code for code in exit_codes if code not in pyld_error_codes]
-            infra_error_count = len(infra_errors)
-            infra_error_summary = (
-                ", ".join(sorted(str(code) for code in set(infra_errors))) if infra_errors else "None"
+            try:
+                exit_codes = exit_code_summary[label]
+            except KeyError:
+                missing_labels.add(label)
+            else:
+                pyld_errors = [code for code in exit_codes if code in pyld_error_codes]
+                pyld_error_count = len(pyld_errors)
+                pyld_error_summary = (
+                    ", ".join(sorted(str(code) for code in set(pyld_errors))) if pyld_errors else "None"
+                )
+
+                infra_errors = [code for code in exit_codes if code not in pyld_error_codes]
+                infra_error_count = len(infra_errors)
+                infra_error_summary = (
+                    ", ".join(sorted(str(code) for code in set(infra_errors))) if infra_errors else "None"
+                )
+
+                run = [label, pyld_error_count, pyld_error_summary, infra_error_count, infra_error_summary]
+                self._table.add_row(run)
+        if missing_labels:
+            warnings.append(
+                f"WARNING: Exit code summary was not available for job labels: {', '.join(missing_labels)}"
             )
-
-            run = [label, pyld_error_count, pyld_error_summary, infra_error_count, infra_error_summary]
-            self._table.add_row(run)
+        if warnings:
+            self._msg = "\n".join(warnings)
 
     def __str__(self):
         alignments = ["<"] + [">"] * (len(self._table.colnames) - 1)
@@ -276,7 +299,7 @@ class ExitCodesReport(BaseRunReport):
         return str("\n".join(lines))
 
 
-def compile_job_summary(report: WmsRunReport) -> None:
+def compile_job_summary(report: WmsRunReport) -> list[str]:
     """Add a job summary to the run report if necessary.
 
     If the job summary is not provided, the function will attempt to compile
@@ -289,24 +312,89 @@ def compile_job_summary(report: WmsRunReport) -> None:
     report : `lsst.ctrl.bps.WmsRunReport`
         Information about a single run.
 
-    Raises
-    ------
-    ValueError
-        Raised if the job summary *and* information about individual jobs
-        is not available.
+    Returns
+    -------
+    warnings : `list` [`str`]
+        List of messages describing any non-critical issues encountered during
+        processing. Empty if none.
     """
+    warnings: list[str] = []
+
+    # If the job summary already exists, exit early.
     if report.job_summary:
-        return
-    if not report.jobs:
-        raise ValueError("job summary cannot be compiled: information about individual jobs not available.")
-    job_summary = {}
-    by_label = group_jobs_by_label(report.jobs)
-    for label, job_group in by_label.items():
-        by_label_state = group_jobs_by_state(job_group)
-        _LOG.debug("by_label_state = %s", by_label_state)
-        counts = {state: len(jobs) for state, jobs in by_label_state.items()}
-        job_summary[label] = counts
-    report.job_summary = job_summary
+        return warnings
+
+    if report.jobs:
+        job_summary = {}
+        by_label = group_jobs_by_label(report.jobs)
+        for label, job_group in by_label.items():
+            by_label_state = group_jobs_by_state(job_group)
+            _LOG.debug("by_label_state = %s", by_label_state)
+            counts = {state: len(jobs) for state, jobs in by_label_state.items()}
+            job_summary[label] = counts
+        report.job_summary = job_summary
+    else:
+        warnings.append("information about individual jobs not available")
+
+    return warnings
+
+
+def compile_code_summary(report: WmsRunReport) -> list[str]:
+    """Add missing entries to the exit code summary if necessary.
+
+    A WMS plugin may exclude job labels for which there are no failures from
+    the exit code summary. The function will attempt to use the job summary,
+    if available, to add missing entries for these labels.
+
+    Parameters
+    ----------
+    report : `lsst.ctrl.bps.WmsRunReport`
+        Information about a single run.
+
+    Returns
+    -------
+    warnings : `list` [`str`]
+        List of messages describing any non-critical issues encountered during
+        processing. Empty if none.
+    """
+    warnings: list[str] = []
+
+    # If the job summary is not available, exit early.
+    if not report.job_summary:
+        return warnings
+
+    # A shallow copy is enough here because we won't be modifying the existing
+    # entries, only adding new ones if necessary.
+    exit_code_summary = dict(report.exit_code_summary) if report.exit_code_summary else {}
+
+    # Use the job summary to add the entries for labels with no failures
+    # *without* modifying already existing entries.
+    failure_summary = {label: states[WmsStates.FAILED] for label, states in report.job_summary.items()}
+    for label, count in failure_summary.items():
+        if count == 0:
+            exit_code_summary.setdefault(label, [])
+
+    # Check if there are any discrepancies between the data in the exit code
+    # summary and the job summary.
+    code_summary_labels = set(exit_code_summary)
+    failure_summary_labels = set(failure_summary)
+    mismatches = {
+        label
+        for label in failure_summary_labels & code_summary_labels
+        if len(exit_code_summary[label]) != failure_summary[label]
+    }
+    if mismatches:
+        warnings.append(
+            f"number of exit codes differs from number of failures for job labels: {', '.join(mismatches)}"
+        )
+    missing = failure_summary_labels - code_summary_labels
+    if missing:
+        warnings.append(f"exit codes not available for job labels: {', '.join(missing)}")
+
+    if exit_code_summary:
+        report.exit_code_summary = exit_code_summary
+
+    return warnings
 
 
 def group_jobs_by_state(jobs):

lsst/ctrl/bps/report.py

@@ -38,12 +38,18 @@ import sys
 from collections.abc import Callable, Sequence
 from typing import TextIO
 
-from lsst.utils import doImport
-
-from .bps_reports import DetailedRunReport, ExitCodesReport, SummaryRunReport, compile_job_summary
-from .wms_service import WmsRunReport, WmsStates
-
-BPS_POSTPROCESSORS = (compile_job_summary,)
+from lsst.utils import doImportType
+
+from .bps_reports import (
+    DetailedRunReport,
+    ExitCodesReport,
+    SummaryRunReport,
+    compile_code_summary,
+    compile_job_summary,
+)
+from .wms_service import BaseWmsService, WmsRunReport, WmsStates
+
+BPS_POSTPROCESSORS = (compile_job_summary, compile_code_summary)
 """Postprocessors for massaging run reports
 (`tuple` [`Callable` [[`WmsRunReport`], None]).
 """
@@ -111,7 +117,7 @@ def display_report(
 
             run_report.add(run, use_global_id=is_global)
             if run_report.message:
-                print(run_report.message, file=file)
+                messages.append(run_report.message)
 
             print(run_brief, file=file)
             print("\n", file=file)
@@ -132,6 +138,8 @@ def display_report(
                 ]
                 run_exits_report = ExitCodesReport(fields)
                 run_exits_report.add(run, use_global_id=is_global)
+                if run_exits_report.message:
+                    messages.append(run_exits_report.message)
                 print("\n", file=file)
                 print(run_exits_report, file=file)
                 run_exits_report.clear()
@@ -145,12 +153,13 @@ def display_report(
         print(run_brief, file=file)
 
     if messages:
-        print("\n".join(messages), file=file)
+        uniques = list(dict.fromkeys(messages))
+        print("\n".join(uniques), file=file)
         print("\n", file=file)
 
 
 def retrieve_report(
-    wms_service: str,
+    wms_service_fqn: str,
     *,
     run_id: str | None = None,
     user: str | None = None,
@@ -163,7 +172,7 @@ def retrieve_report(
 
     Parameters
     ----------
-    wms_service : `str`
+    wms_service_fqn : `str`
         Name of the WMS service class.
     run_id : `str`, optional
         A run id the report will be restricted to.
@@ -196,11 +205,21 @@ def retrieve_report(
     messages : `list` [`str`]
         Errors that happened during report retrieval and/or processing.
         Empty if no issues were encountered.
+
+    Raises
+    ------
+    TypeError
+        Raised if the WMS service class is not a subclass of BaseWmsService.
     """
-    messages = []
+    messages: list[str] = []
 
-    wms_service_class = doImport(wms_service)
+    wms_service_class = doImportType(wms_service_fqn)
+    if not issubclass(wms_service_class, BaseWmsService):
+        raise TypeError(
+            f"Invalid WMS service class '{wms_service_fqn}'; must be a subclass of BaseWmsService"
+        )
     wms_service = wms_service_class({})
+
     reports, message = wms_service.report(
         wms_workflow_id=run_id, user=user, hist=hist, pass_thru=pass_thru, is_global=is_global
     )
@@ -210,12 +229,12 @@ def retrieve_report(
     if postprocessors:
         for report in reports:
             for postprocessor in postprocessors:
-                try:
-                    postprocessor(report)
-                except Exception as exc:
-                    messages.append(
-                        f"Postprocessing error for '{report.wms_id}': {str(exc)} "
-                        f"(origin: {postprocessor.__name__})"
-                    )
+                if warnings := postprocessor(report):
+                    for warning in warnings:
+                        messages.append(
+                            f"WARNING: Report may be incomplete. "
+                            f"There was an issue with report postprocessing for '{report.wms_id}': "
+                            f"{warning} (origin: {postprocessor.__name__})"
+                        )
 
     return reports, messages

lsst/ctrl/bps/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.3700"
+__version__ = "29.2025.3900"

{lsst_ctrl_bps-29.2025.3700.dist-info → lsst_ctrl_bps-29.2025.3900.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps
-Version: 29.2025.3700
+Version: 29.2025.3900
 Summary: Pluggable execution of workflow graphs from Rubin pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps-29.2025.3700.dist-info → lsst_ctrl_bps-29.2025.3900.dist-info}/RECORD

@@ -4,7 +4,7 @@ lsst/ctrl/bps/__init__.py,sha256=qNei3h4EwqUnyyCsBUkYXcRNNlSvzUzbM6Zdkd4eklE,166
 lsst/ctrl/bps/_exceptions.py,sha256=qNftFVE8yhiRD8yvACKEf5iHhi1o2eaa69ZCBvw6lDo,1974
 lsst/ctrl/bps/bps_config.py,sha256=4t1jyIBiw9leAySjkcksCaZ3CPJMOt8afxGxTEWRSyQ,18460
 lsst/ctrl/bps/bps_draw.py,sha256=Sl04jpbm6jB_5lBzfJprqnDes9t_aqQQhr0Ag5RSv-Y,1984
-lsst/ctrl/bps/bps_reports.py,sha256=j6HdAvkOBTSxSHmkx_vd04FvifToTaZVGpwfF-LIUu0,12487
+lsst/ctrl/bps/bps_reports.py,sha256=73l5Gz00VQq8NgoBrGkvCaUQJE5kw2emhQxxzthMQzs,15574
 lsst/ctrl/bps/bps_utils.py,sha256=-w0HSUXKw8jgJFl28yzwJuuqLqT_UOw12_XQBELUWrE,12206
 lsst/ctrl/bps/cancel.py,sha256=mAdBi-oUpepyo-1MCqx_I34dbm6cqT0VJu3d2-y9T2Y,3317
 lsst/ctrl/bps/clustered_quantum_graph.py,sha256=mBf8s_DlTzGCFq7aAKmD1cAXq6Cqr8QxjwpgXPOQcRc,18765
@@ -17,12 +17,12 @@ lsst/ctrl/bps/ping.py,sha256=orwTZUNFtlexMYFcNWW_48jaa7Jo1oK4_eb_HuC-p5E,2235
 lsst/ctrl/bps/pre_transform.py,sha256=mPUW1QuHwgkSgZGVK6xO4RN590z4g1zf3p4kNsx_yOg,10222
 lsst/ctrl/bps/prepare.py,sha256=Fa2OEQIo4Pa8R5WmRo0PvJgXWNjynRijATvu1x80qlw,3129
 lsst/ctrl/bps/quantum_clustering_funcs.py,sha256=8fidDw53MpcqeWVXs0F0jIvSU0fLV_-1fEbC-Jhqg28,31816
-lsst/ctrl/bps/report.py,sha256=6qCt6-0apuVZrevtQzJLRmQc_ve_uPyNH3HkgC5KSjc,7898
+lsst/ctrl/bps/report.py,sha256=893m2KBBaQouAml8DKmmVxBo-Q7fFizxl_OT6wyFBXk,8575
 lsst/ctrl/bps/restart.py,sha256=yVwxeviLiehyIfPmwU-H3tJ9ou7OWZZcrNf8PMxjr8o,2298
 lsst/ctrl/bps/status.py,sha256=Lrt0cAqROv77B8UvYXGimCa4cDHBD1N0K2Xx7oS6fXk,3362
 lsst/ctrl/bps/submit.py,sha256=Ev-yhcoZwtBPIo5bRt_4XFJRtgQBd8JHUurEfn01HpU,2880
 lsst/ctrl/bps/transform.py,sha256=P3dGwKStoCLOn7F2Oez6B__f7bGV5jlUF_Vj9lJQcU0,35256
-lsst/ctrl/bps/version.py,sha256=AQOxvPBwPILItheZOADktYgvye1bfruQzhiAhgZby0A,55
+lsst/ctrl/bps/version.py,sha256=qaaoasFkyU8Kx3tKT5jPh3H-bKD5Y8pAmsL3Scaq2UU,55
 lsst/ctrl/bps/wms_service.py,sha256=l3T6i1MG72dhHY0KXMUlBjWUpCLOfaySs7o2W8oCwhs,18891
 lsst/ctrl/bps/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/ctrl/bps/cli/bps.py,sha256=CHfAL-U4mSi-FqeGKtmkX5nI5H9gkRre4XEWNVdeMRk,2559
@@ -35,13 +35,13 @@ lsst/ctrl/bps/cli/opt/options.py,sha256=pZQpjJ2Vrx6kYJGs5vVERBMFstPocwBXHffxkWkm
 lsst/ctrl/bps/etc/bps_defaults.yaml,sha256=UwKAKWl09-Xjtl6kiz-AiXA9SDczeCvPGRXOPG3fzb8,4828
 lsst/ctrl/bps/tests/config_test_utils.py,sha256=WM8Vrigk4OO0TBoL1A73a6hLhf2a6-ACD20fROJ0U7A,3537
 lsst/ctrl/bps/tests/gw_test_utils.py,sha256=zVVQqzwSiQgPgk9TnqDzgR7uDnaTMeuBLYKA8vOp5RI,22452
-lsst_ctrl_bps-29.2025.3700.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
-lsst_ctrl_bps-29.2025.3700.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_bps-29.2025.3700.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_bps-29.2025.3700.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_bps-29.2025.3700.dist-info/METADATA,sha256=1aCeZeV9asx8ORK4RKR_L-7BNEHXwJwsgq4Bd26750U,2190
-lsst_ctrl_bps-29.2025.3700.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_ctrl_bps-29.2025.3700.dist-info/entry_points.txt,sha256=d6FhN79h7s9frdBI7YkScsGEInwpGGub49pAjXWbIbI,51
-lsst_ctrl_bps-29.2025.3700.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_bps-29.2025.3700.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_bps-29.2025.3700.dist-info/RECORD,,
+lsst_ctrl_bps-29.2025.3900.dist-info/licenses/COPYRIGHT,sha256=Lc6NoAEFQ65v_SmtS9NwfHTOuSUtC2Umbjv5zyowiQM,61
+lsst_ctrl_bps-29.2025.3900.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_bps-29.2025.3900.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_bps-29.2025.3900.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_bps-29.2025.3900.dist-info/METADATA,sha256=Etf24NWJ8pPnq6jmRV-0n55ZhvJ29kDoSXiYFsRHzso,2190
+lsst_ctrl_bps-29.2025.3900.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_ctrl_bps-29.2025.3900.dist-info/entry_points.txt,sha256=d6FhN79h7s9frdBI7YkScsGEInwpGGub49pAjXWbIbI,51
+lsst_ctrl_bps-29.2025.3900.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_bps-29.2025.3900.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_bps-29.2025.3900.dist-info/RECORD,,