lsst-ctrl-bps 29.1.0rc5__tar.gz → 30.0.0rc3__tar.gz

{lsst_ctrl_bps-29.1.0rc5/python/lsst_ctrl_bps.egg-info → lsst_ctrl_bps-30.0.0rc3}/PKG-INFO

@@ -1,18 +1,18 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps
-Version: 29.1.0rc5
+Version: 30.0.0rc3
 Summary: Pluggable execution of workflow graphs from Rubin pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
-License: BSD 3-Clause License
+License-Expression: BSD-3-Clause OR GPL-3.0-or-later
 Project-URL: Homepage, https://github.com/lsst/ctrl_bps
 Keywords: lsst
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Astronomy
 Requires-Python: >=3.11.0
 Description-Content-Type: text/markdown

{lsst_ctrl_bps-29.1.0rc5 → lsst_ctrl_bps-30.0.0rc3}/doc/lsst.ctrl.bps/CHANGES.rst

@@ -1,3 +1,33 @@
+lsst-ctrl-bps v30.0.0 (2026-01-16)
+==================================
+
+New Features
+------------
+
+- Added support for transferring input files to the execution site and bringing the produced output files back to the submit site when using ``bps submitcmd``. (`DM-48479 <https://rubinobs.atlassian.net/browse/DM-48479>`_)
+- Added ``bpsGenerateConfig`` and ``bpsEval`` to run functions to produce config values.  ``bpsGenerateConfig`` is used when inserting or updating one or more key/value pairs.  ``bpsEval`` is for replacing part of a string value. (`DM-50616 <https://rubinobs.atlassian.net/browse/DM-50616>`_)
+
+Bug Fixes
+---------
+
+- Fixed bug where the ``return_exit_codes`` command line value was not passed to the plugin's report function.  While it was correctly used when displaying the report, not having the value didn't allow the plugin to optimize its report function. (`DM-52791 <https://rubinobs.atlassian.net/browse/DM-52791>`_)
+
+Performance Enhancement
+-----------------------
+
+- Switched the default ``finalJob`` implementation to the new ``aggregate-graph`` command, which makes use of multiple cores much more effectively than ``transfer-from-graph``. (`DM-52360 <https://rubinobs.atlassian.net/browse/DM-52360>`_)
+
+
+Other Changes and Additions
+---------------------------
+
+- Removed exit code 1 from the default for ``finalJob``\ 's ``retryUnlessExit`` as the majority of these are related to system issues which could be transient. (`DM-51313 <https://rubinobs.atlassian.net/browse/DM-51313>`_)
+- Modified the BPS report driver so it compiles exit code summary only when necessary, i.e., when ``--return-exit-codes`` option was used with ``bps report``. (`DM-52898 <https://rubinobs.atlassian.net/browse/DM-52898>`_)
+- Added a custom __new__ method to the GenericWorkflow class so **ctrl_bps** can work when using NetworkX 3.6. (`DM-53492 <https://rubinobs.atlassian.net/browse/DM-53492>`_)
+- Made the BPS reporting mechanism a bit more robust. If the BPS plugin does not explicitly includes jobs labels for which there are no failures in the run's exit code summary, it will try to use the run's job summary to do that. (`DM-51261 <https://rubinobs.atlassian.net/browse/DM-51261>`_)
+- Used the new ``PredictedQuantumGraph`` class internally and default to the new QG file format via the "``.qg``" extension. (`DM-52339 <https://rubinobs.atlassian.net/browse/DM-52339>`_)
+
+
 lsst-ctrl-bps v29.1.0 (2025-06-13)
 ==================================
 

{lsst_ctrl_bps-29.1.0rc5 → lsst_ctrl_bps-30.0.0rc3}/doc/lsst.ctrl.bps/quickstart.rst

@@ -157,7 +157,7 @@ or a pre-made file containing a serialized QuantumGraph, for example
 
 .. code-block:: YAML
 
-   qgraphFile: pipelines_check_w_2020_45.qgraph
+   qgraphFile: pipelines_check_w_2020_45.qg
 
 .. warning::
 
@@ -265,7 +265,7 @@ arguments, e.g.:
 .. code-block:: yaml
 
    customJob:
-     executable: "${HOME}/scripts/sleep.sh"
+     executable: "${HOME}/scripts/do_stuff.sh"
      arguments: "2"
 
    # Uncomment settings below to disable automatic memory scaling and retries
@@ -276,16 +276,62 @@ arguments, e.g.:
 
 where ``executable`` specifies the path to the executable to run and
 ``arguments`` is a list of arguments to be supplied to the executable as part
-of the command line.
+of the command line.  If your executable does not take any command line
+arguments set ``arguments`` to an empty string.
 
 .. note::
 
-   If your executable does not take any command line arguments set
-   ``arguments`` to an empty string.
+   The script specified by ``customJob.executable`` is copied to the run's
+   submit directory and this copy (not the original script) is being submitted
+   for execution.  As a result, making any changes to the original script after
+   the run has been submitted will have no effect even if the run is still
+   in the WMS work queue waiting for execution.
+
+If the script requires any input files that should be transferred to the
+execution site as well and/or produces output files that should be brought back
+specify them as follows:
+
+.. code-block:: yaml
+
+   customJob:
+     executable: "${HOME}/scripts/do_stuff.sh"
+     arguments: "-o {outfile} {infile}"
+     inputs:
+       infile: path/to/input/file
+     outputs:
+       outfile: path/to/output/file
+
+   # Uncomment settings below to disable automatic memory scaling and retries
+   # which BPS enables by default.
+   #
+   # memoryMultiplier: 1
+   # numberOfRetries: 1
+
+The paths in ``inputs`` specify files as they are accessed on the submit site.
+They can be absolute or relative to the current working directory as the run is
+submitted.  The input files will be copied to the run's submit directory.
+These copies (not the original files) will be submitted along with the script's
+copy for execution.  During the execution BPS will transfer these copies into a
+single flat directory -- job's scratch directory on the execute machine.
+
+The paths in ``outputs`` specifies the paths on the submit site the output
+files will be copied to after job's completion.  As input paths they can be
+either absolute or relative.  However, on the execution site, the script is
+expected to write all its output files directly to job's scratch directory
+(assumed to be the current working directory when the job starts unless stated
+otherwise in the WMS-specific documentation).
+
+As a result, both input and output files base names *must* be unique.
+
+.. note::
+
+   Currently, BPS itself doesn't verify if the file declared in ``outputs`` was
+   produced by the script.  Whether a missing output file will be considered an
+   error depends entirely on WMS in use.
 
-This config file will instruct BPS to create a special single-job *workflow* to
-run your script.  That workflow will be submitted for execution as any other
-workflow.
+The config files shown above will instruct BPS to create a special single-job
+*workflow* to run your script.  That workflow will be submitted for execution
+as any other workflow.
 
 As a result, the submission process for a custom script looks quite similar
 to the submission process of regular payload jobs (i.e. jobs running
@@ -305,15 +351,6 @@ There are few things you need to keep in mind though:
    instructions exist in the submit YAML.  If you need the quantum graph, use
    ``bps submit``.
 
-#. At the moment, the mechanism does not support transferring files other than
-   executable.
-
-#. The script specified by ``customJob.executable`` is copied to the run's
-   submit directory and this copy (not the original script) is being submitted
-   for execution.  As a result, making any changes to the original script after
-   the run has been submitted will have no effect even if the run is still in
-   the WMS work queue waiting for execution.
-
 #. Some BPS plugins may require inclusion of plugin-specific settings for this
    mechanism to work.  Consult the documentation of the plugin you use for
    details.
@@ -809,7 +846,7 @@ Supported settings
     When to output job QuantumGraph files (default = TRANSFORM).
 
     * NEVER = all jobs will use full QuantumGraph file. (Warning: make sure
-      runQuantumCommand has ``--qgraph-id {qgraphId} --qgraph-node-id {qgraphNodeId}``.)
+      runQuantumCommand has ``--qgraph-node-id {qgraphNodeId}``.)
     * TRANSFORM = Output QuantumGraph files after creating GenericWorkflow.
     * PREPARE = QuantumGraph files are output after creating WMS submission.
 
@@ -867,7 +904,7 @@ Reserved keywords
     However, contrary to YAML specification, it is currently not portable.
 
 **qgraphId**
-    Internal ID for the full QuantumGraph (passed as ``--qgraph-id`` on pipetask command line).
+    Ignored; accepted for backwards compatibility.
 
 **qgraphNodeId**
     Comma-separated list of internal QuantumGraph node numbers to be
@@ -1042,13 +1079,12 @@ single full QuantumGraph file plus node numbers for each job. The default is
 using per-job QuantumGraph files.
 
 To use full QuantumGraph file, the submit YAML must set ``whenSaveJobQgraph`` to
-"NEVER" and the ``pipetask run`` command must include ``--qgraph-id {qgraphId}
---qgraph-node-id {qgraphNodeId}``.  For example:
+"NEVER" and the ``pipetask run`` command must include ``--qgraph-node-id {qgraphNodeId}``.  For example:
 
 .. code::
 
     whenSaveJobQgraph: "NEVER"
-    runQuantumCommand: "${CTRL_MPEXEC_DIR}/bin/pipetask --long-log run -b {butlerConfig} --output {output} --output-run {outputRun} --qgraph {qgraphFile} --qgraph-id {qgraphId} --qgraph-node-id {qgraphNodeId} --skip-init-writes --extend-run --clobber-outputs --skip-existing"
+    runQuantumCommand: "${CTRL_MPEXEC_DIR}/bin/pipetask --long-log run -b {butlerConfig} --output {output} --output-run {outputRun} --qgraph {qgraphFile} --qgraph-node-id {qgraphNodeId} --skip-init-writes --extend-run --clobber-outputs --skip-existing"
 
 
 .. warning::
@@ -1125,7 +1161,7 @@ New YAML Section
      implementation: JOB
      concurrencyLimit: db_limit
      command1: >-
-       ${DAF_BUTLER_DIR}/bin/butler transfer-from-graph
+       ${DAF_BUTLER_DIR}/bin/butler aggregate-graph
        {fileDistributionEndPoint}{qgraphFile}
        {butlerConfig}
        --register-dataset-types
@@ -1207,7 +1243,7 @@ The major differences to users are:
     the output run in the provided pre-existing quantum graph.
   - ``final_post_finalJob.out``: An internal file for debugging incorrect
     reporting of final run status.
-  - ``<qgraph_filename>_orig.qgraph``: A backup copy of the original
+  - ``<qgraph_filename>_orig.qg``: A backup copy of the original
     pre-existing quantum graph file that was used for submitting the run.  Note
     that this file will *not* be present in the submit directory if the
     pipeline YAML specification was used during the submission instead.
@@ -1497,6 +1533,49 @@ invisible to the user.  ``bps report`` will still show same labels and
 total counts as without ordering.  ``cancel`` and ``restart`` will still
 work the same.
 
+.. _bps-config-generation:
+
+Config Generation
+-----------------
+
+In some rare use cases, the submit yaml depends upon what happened in
+previous runs (e.g., passing pipeline configuration values to the HiPS
+QuantumGraph generation command depending upon colors of generated outputs
+of previous run).  One can wait until a run finishes, query the results,
+and then manually modify the submit yaml for the next run.  To help make
+this easier to automate, two special mechanisms, ``bpsGenerateConfig`` and
+``bpsEval`` have been added to ``bps``.  While different syntax, both take
+two pieces of information.  The first piece describes what to import and
+execute and the second the parameters to pass which typically will
+be config variables (e.g., "{butlerConfig}").
+
+``bpsGenerateConfig`` is a key/value pair where the function returns a
+Mapping to update the config.  It can be used at the root level and can
+return nested dictionaries to replace values across multiple sections.
+It can also be used inside sections (e.g., inside a specific pipetask
+section).  The function is not run when loading the config.  Instead it
+is run during the initialization in the ``bps submit`` (before saving
+the config yaml to the submit directory and before running QuantumGraph
+generation).  Example:
+
+..code::
+
+  bpsGenerateConfig: "lsst.my.package.my_func_1('{butlerConfig}', param3='{output}')"
+
+
+``bpsEval`` is a placeholder in an submit yaml value.  It is executed when
+the corresponding key is requested from the config.  Its function needs
+to return a value whose string representation can replace ``bpsEval``.  Example:
+
+..code::
+
+  extraQgraphOptions: "--dataset-query-constraint off bpsEval(lsst.my.package.my_func_2, '{butlerConfig}', '{output}')
+
+.. warning::
+
+   Quotes must be placed around variables that return strings (e.g., '{butlerConfig}').
+   Forgetting the quotes typically results in an invalid syntax error.
+
 .. _bps-softlink:
 
 WMS-id softlink

{lsst_ctrl_bps-29.1.0rc5 → lsst_ctrl_bps-30.0.0rc3}/pyproject.toml

@@ -6,19 +6,20 @@ build-backend = "setuptools.build_meta"
 name = "lsst-ctrl-bps"
 requires-python = ">=3.11.0"
 description = "Pluggable execution of workflow graphs from Rubin pipelines."
-license = {text = "BSD 3-Clause License"}
+license = "BSD-3-Clause OR GPL-3.0-or-later"
+license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
 readme = "README.md"
 authors = [
     {name="Rubin Observatory Data Management", email="dm-admin@lists.lsst.org"},
 ]
 classifiers = [
     "Intended Audience :: Science/Research",
-    "License :: OSI Approved :: BSD License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Scientific/Engineering :: Astronomy",
 ]
 keywords = ["lsst"]
@@ -49,7 +50,6 @@ where = ["python"]
 
 [tool.setuptools]
 zip-safe = true
-license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
 
 [tool.setuptools.package-data]
 "lsst.ctrl.bps" = ["etc/*.yaml"]

{lsst_ctrl_bps-29.1.0rc5 → lsst_ctrl_bps-30.0.0rc3}/python/lsst/ctrl/bps/bps_config.py

@@ -44,6 +44,8 @@ from lsst.daf.butler import Config
 from lsst.resources import ResourcePath
 from lsst.utils import doImport
 
+from .bps_utils import bps_eval
+
 _LOG = logging.getLogger(__name__)
 
 # Using lsst.daf.butler.Config to resolve possible includes.
@@ -417,4 +419,63 @@ class BpsConfig(Config):
         if default != _NO_SEARCH_DEFAULT_VALUE:
             opt["default"] = default
 
+        # check for bpsEval
+        value = re.sub(
+            r"bpsEval\(([^,)]+), ([^)]+)\)", lambda m: str(bps_eval(m.group(1), m.group(2))), value
+        )
+        if "bpsEval" in value:
+            raise ValueError(f"Unparsable bpsEval in '{value}'")
+
         return value
+
+    def generate_config(self) -> None:
+        """Update config with values generated by bpsGenerateConfig
+        entries.
+        """
+        _LOG.debug("generate_config before: %s", self)
+        self._recursive_generate_config("", self)
+        _LOG.debug("generate_config after: %s", self)
+
+    def _recursive_generate_config(self, recursive_key: str, sub_config: Config) -> None:
+        """Update config with values generated by bpsGenerateConfig
+        entries.
+
+        Parameters
+        ----------
+        recursive_key : `str`
+            Corresponds to a new subconfig in which to search
+            and replace bpsGenerateConfig.
+
+        sub_config : `lsst.daf.butler.Config`
+            The nested config corresponding to the recursive_key.
+
+        Raises
+        ------
+        ValueError
+            If bpsGenerateConfig value isn't parseable.
+        ImportError
+            If problems importing bpsGenerateConfig's method.
+        """
+        _LOG.debug("recursive_key = '%s'", recursive_key)
+        genkey = "bpsGenerateConfig"  # to make it easier to change
+
+        # Save to avoid dictionary changed size during iteration error.
+        orig_keys = list(sub_config)
+        for key in orig_keys:
+            value = Config.__getitem__(sub_config, key)
+            _LOG.debug("key = %s, type(value) = %s", key, type(value))
+            if isinstance(value, Config):
+                self._recursive_generate_config(f"{recursive_key}.{key}", value)
+            elif key == genkey:
+                value = self.replace_vars(value, {"searchobj": sub_config})
+
+                m = re.match(r"(\S+)\((.+)\)", value)
+                if m:
+                    results = bps_eval(m.group(1), m.group(2))
+                    del sub_config[genkey]
+                    sub_config.update(results)
+                    if recursive_key:
+                        self[recursive_key] = sub_config
+                    _LOG.debug("After config = %s", self)
+                else:
+                    raise ValueError(f"Unparsable {genkey} value='{value}'")

{lsst_ctrl_bps-29.1.0rc5 → lsst_ctrl_bps-30.0.0rc3}/python/lsst/ctrl/bps/bps_reports.py

@@ -27,7 +27,14 @@
 
 """Classes and functions used in reporting run status."""
 
-__all__ = ["BaseRunReport", "DetailedRunReport", "ExitCodesReport", "SummaryRunReport", "compile_job_summary"]
+__all__ = [
+    "BaseRunReport",
+    "DetailedRunReport",
+    "ExitCodesReport",
+    "SummaryRunReport",
+    "compile_code_summary",
+    "compile_job_summary",
+]
 
 import abc
 import logging
@@ -55,7 +62,7 @@ class BaseRunReport(abc.ABC):
 
     def __eq__(self, other):
         if isinstance(other, BaseRunReport):
-            return all(self._table == other._table)
+            return self._table.pformat() == other._table.pformat()
         return False
 
     def __len__(self):
@@ -195,7 +202,7 @@ class DetailedRunReport(BaseRunReport):
         job_summary = run_report.job_summary
         if job_summary is None:
             id_ = run_report.global_wms_id if use_global_id else run_report.wms_id
-            self._msg = f"WARNING: Job summary for run '{id_}' not available, report maybe incomplete."
+            self._msg = f"WARNING: Job summary for run '{id_}' not available, report may be incomplete."
             return
 
         if by_label_expected:
@@ -231,44 +238,60 @@ class ExitCodesReport(BaseRunReport):
     error handling from the wms service.
     """
 
-    def add(self, run_report, use_global_id=False):
+    def add(self, run_report: WmsRunReport, use_global_id: bool = False) -> None:
         # Docstring inherited from the base class.
 
-        # Use label ordering from the run summary as it should reflect
-        # the ordering of the pipetasks in the pipeline.
+        exit_code_summary = run_report.exit_code_summary
+        if not exit_code_summary:
+            id_ = run_report.global_wms_id if use_global_id else run_report.wms_id
+            self._msg = f"WARNING: Exit code summary for run '{id_}' not available, report may be incomplete."
+            return
+
+        warnings = []
+
+        # If available, use label ordering from the run summary as it should
+        # reflect the ordering of the pipetasks in the pipeline.
         labels = []
         if run_report.run_summary:
             for part in run_report.run_summary.split(";"):
                 label, _ = part.split(":")
                 labels.append(label)
-        else:
-            id_ = run_report.global_wms_id if use_global_id else run_report.wms_id
-            self._msg = f"WARNING: Job summary for run '{id_}' not available, report maybe incomplete."
-            return
+        if not labels:
+            labels = sorted(exit_code_summary)
+            warnings.append("WARNING: Could not determine order of pipeline, instead sorted alphabetically.")
 
         # Payload (e.g. pipetask) error codes:
         # * 1: general failure,
         # * 2: command line error (e.g. unknown command and/or option).
         pyld_error_codes = {1, 2}
 
-        exit_code_summary = run_report.exit_code_summary
+        missing_labels = set()
         for label in labels:
-            exit_codes = exit_code_summary[label]
-
-            pyld_errors = [code for code in exit_codes if code in pyld_error_codes]
-            pyld_error_count = len(pyld_errors)
-            pyld_error_summary = (
-                ", ".join(sorted(str(code) for code in set(pyld_errors))) if pyld_errors else "None"
-            )
-
-            infra_errors = [code for code in exit_codes if code not in pyld_error_codes]
-            infra_error_count = len(infra_errors)
-            infra_error_summary = (
-                ", ".join(sorted(str(code) for code in set(infra_errors))) if infra_errors else "None"
+            try:
+                exit_codes = exit_code_summary[label]
+            except KeyError:
+                missing_labels.add(label)
+            else:
+                pyld_errors = [code for code in exit_codes if code in pyld_error_codes]
+                pyld_error_count = len(pyld_errors)
+                pyld_error_summary = (
+                    ", ".join(sorted(str(code) for code in set(pyld_errors))) if pyld_errors else "None"
+                )
+
+                infra_errors = [code for code in exit_codes if code not in pyld_error_codes]
+                infra_error_count = len(infra_errors)
+                infra_error_summary = (
+                    ", ".join(sorted(str(code) for code in set(infra_errors))) if infra_errors else "None"
+                )
+
+                run = [label, pyld_error_count, pyld_error_summary, infra_error_count, infra_error_summary]
+                self._table.add_row(run)
+        if missing_labels:
+            warnings.append(
+                f"WARNING: Exit code summary was not available for job labels: {', '.join(missing_labels)}"
             )
-
-            run = [label, pyld_error_count, pyld_error_summary, infra_error_count, infra_error_summary]
-            self._table.add_row(run)
+        if warnings:
+            self._msg = "\n".join(warnings)
 
     def __str__(self):
         alignments = ["<"] + [">"] * (len(self._table.colnames) - 1)
@@ -276,7 +299,7 @@ class ExitCodesReport(BaseRunReport):
         return str("\n".join(lines))
 
 
-def compile_job_summary(report: WmsRunReport) -> None:
+def compile_job_summary(report: WmsRunReport) -> list[str]:
     """Add a job summary to the run report if necessary.
 
     If the job summary is not provided, the function will attempt to compile
@@ -289,24 +312,89 @@ def compile_job_summary(report: WmsRunReport) -> None:
     report : `lsst.ctrl.bps.WmsRunReport`
         Information about a single run.
 
-    Raises
-    ------
-    ValueError
-        Raised if the job summary *and* information about individual jobs
-        is not available.
+    Returns
+    -------
+    warnings : `list` [`str`]
+        List of messages describing any non-critical issues encountered during
+        processing. Empty if none.
     """
+    warnings: list[str] = []
+
+    # If the job summary already exists, exit early.
     if report.job_summary:
-        return
-    if not report.jobs:
-        raise ValueError("job summary cannot be compiled: information about individual jobs not available.")
-    job_summary = {}
-    by_label = group_jobs_by_label(report.jobs)
-    for label, job_group in by_label.items():
-        by_label_state = group_jobs_by_state(job_group)
-        _LOG.debug("by_label_state = %s", by_label_state)
-        counts = {state: len(jobs) for state, jobs in by_label_state.items()}
-        job_summary[label] = counts
-    report.job_summary = job_summary
+        return warnings
+
+    if report.jobs:
+        job_summary = {}
+        by_label = group_jobs_by_label(report.jobs)
+        for label, job_group in by_label.items():
+            by_label_state = group_jobs_by_state(job_group)
+            _LOG.debug("by_label_state = %s", by_label_state)
+            counts = {state: len(jobs) for state, jobs in by_label_state.items()}
+            job_summary[label] = counts
+        report.job_summary = job_summary
+    else:
+        warnings.append("information about individual jobs not available")
+
+    return warnings
+
+
+def compile_code_summary(report: WmsRunReport) -> list[str]:
+    """Add missing entries to the exit code summary if necessary.
+
+    A WMS plugin may exclude job labels for which there are no failures from
+    the exit code summary. The function will attempt to use the job summary,
+    if available, to add missing entries for these labels.
+
+    Parameters
+    ----------
+    report : `lsst.ctrl.bps.WmsRunReport`
+        Information about a single run.
+
+    Returns
+    -------
+    warnings : `list` [`str`]
+        List of messages describing any non-critical issues encountered during
+        processing. Empty if none.
+    """
+    warnings: list[str] = []
+
+    # If the job summary is not available, exit early.
+    if not report.job_summary:
+        return warnings
+
+    # A shallow copy is enough here because we won't be modifying the existing
+    # entries, only adding new ones if necessary.
+    exit_code_summary = dict(report.exit_code_summary) if report.exit_code_summary else {}
+
+    # Use the job summary to add the entries for labels with no failures
+    # *without* modifying already existing entries.
+    failure_summary = {label: states[WmsStates.FAILED] for label, states in report.job_summary.items()}
+    for label, count in failure_summary.items():
+        if count == 0:
+            exit_code_summary.setdefault(label, [])
+
+    # Check if there are any discrepancies between the data in the exit code
+    # summary and the job summary.
+    code_summary_labels = set(exit_code_summary)
+    failure_summary_labels = set(failure_summary)
+    mismatches = {
+        label
+        for label in failure_summary_labels & code_summary_labels
+        if len(exit_code_summary[label]) != failure_summary[label]
+    }
+    if mismatches:
+        warnings.append(
+            f"number of exit codes differs from number of failures for job labels: {', '.join(mismatches)}"
+        )
+    missing = failure_summary_labels - code_summary_labels
+    if missing:
+        warnings.append(f"exit codes not available for job labels: {', '.join(missing)}")
+
+    if exit_code_summary:
+        report.exit_code_summary = exit_code_summary
+
+    return warnings
 
 
 def group_jobs_by_state(jobs):

{lsst_ctrl_bps-29.1.0rc5 → lsst_ctrl_bps-30.0.0rc3}/python/lsst/ctrl/bps/bps_utils.py

@@ -31,6 +31,7 @@ __all__ = [
     "_dump_env_info",
     "_dump_pkg_info",
     "_make_id_link",
+    "bps_eval",
     "chdir",
     "create_count_summary",
     "create_job_quantum_graph_filename",
@@ -52,6 +53,7 @@ from typing import Any
 
 import yaml
 
+from lsst.utils import doImport
 from lsst.utils.packages import Packages
 
 _LOG = logging.getLogger(__name__)
@@ -144,7 +146,7 @@ def create_job_quantum_graph_filename(config, job, out_prefix=None):
     found, subdir = config.search("subDirTemplate", opt={"curvals": curvals})
     if not found:
         subdir = "{job.label}"
-    full_filename = Path("inputs") / subdir / f"quantum_{job.name}.qgraph"
+    full_filename = Path("inputs") / subdir / f"quantum_{job.name}.qg"
 
     if out_prefix is not None:
         full_filename = Path(out_prefix) / full_filename
@@ -353,3 +355,35 @@ def subset_dimension_values(
             f"{desc_what} missing dimensions ({', '.join(sorted(missing_dims))}) required for {desc_for}"
         )
     return dim_values
+
+
+def bps_eval(func: str, args: str) -> Any:
+    """Evaluate user provided expression/function.
+
+    Parameters
+    ----------
+    func : `str`
+        Importable string or built-in function name.
+    args : `str`
+        Parameters to pass to the function.
+
+    Returns
+    -------
+    results : `~typing.Any`
+        Results of running eval.
+
+    Raises
+    ------
+    ImportError
+        If problems importing.
+    """
+    if "." in func:
+        genfunc = doImport(func)  # noqa: F841
+        func_reference = "genfunc"
+    else:
+        func_reference = func
+    eval_str = f"{func_reference}({args})"
+    _LOG.debug("String passed to eval: '%s'", eval_str)
+    results = eval(eval_str)
+
+    return results