lsst-ctrl-mpexec 29.2025.1100__py3-none-any.whl → 29.2025.1300__py3-none-any.whl

lsst/ctrl/mpexec/cli/cmd/commands.py

@@ -382,6 +382,18 @@ def update_graph_run(
     "even when there is only one qgraph. Otherwise, the `QuantumGraphExecutionReport` "
     "will run on one graph by default.",
 )
+@click.option(
+    "--read-caveats",
+    type=click.Choice(["exhaustive", "lazy", "none"], case_sensitive=False),
+    default="lazy",
+)
+@click.option(
+    "--use-qbb/--no-use-qbb",
+    is_flag=True,
+    default=True,
+    help="Whether to use a quantum-backed butler for metadata and log reads.",
+)
+@processes_option()
 def report(
     repo: str,
     qgraphs: Sequence[str],
@@ -392,6 +404,9 @@ def report(
     brief: bool = False,
     curse_failed_logs: bool = False,
     force_v2: bool = False,
+    read_caveats: str = "lazy",
+    use_qbb: bool = True,
+    processes: int = 1,
 ) -> None:
     """Summarize the state of executed quantum graph(s), with counts of failed,
     successful and expected quanta, as well as counts of output datasets and
@@ -417,7 +432,17 @@ def report(
     """
     if any([force_v2, len(qgraphs) > 1, collections, where, curse_failed_logs]):
         script.report_v2(
-            repo, qgraphs, collections, where, full_output_filename, logs, brief, curse_failed_logs
+            repo,
+            qgraphs,
+            collections,
+            where,
+            full_output_filename,
+            logs,
+            brief,
+            curse_failed_logs,
+            read_caveats=(read_caveats if read_caveats != "none" else None),  # type: ignore[arg-type]
+            use_qbb=use_qbb,
+            n_cores=processes,
         )
     else:
         assert len(qgraphs) == 1, "Cannot make a report without a quantum graph."

lsst/ctrl/mpexec/cli/script/report.py

@@ -26,6 +26,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import pprint
 from collections.abc import Sequence
+from typing import Literal
 
 from astropy.table import Table
 
@@ -48,23 +49,22 @@ def report(
 
     Parameters
     ----------
-        butler_config : `str`
-            The Butler used for this report. This should match the Butler used
-            for the run associated with the executed quantum graph.
-        qgraph_uri : `str`
-            The uri of the location of said quantum graph.
-        full_output_filename : `str`
-            Output the full summary report to a yaml file (named herein).
-            Each data id and error message is keyed to a quantum graph node id.
-            A convenient output format for error-matching and cataloguing tools
-            such as the ones in the Campaign Management database. If this is
-            not included, quanta and dataset information will be printed to the
-            command-line instead.
-        logs : `bool`
-            Get butler log datasets for extra information (error messages).
-        brief : `bool`
-            List only the counts (or data_ids if number of failures < 5). This
-            option is good for those who just want to see totals.
+    butler_config : `str`
+        The Butler used for this report. This should match the Butler used for
+        the run associated with the executed quantum graph.
+    qgraph_uri : `str`
+        The uri of the location of said quantum graph.
+    full_output_filename : `str`
+        Output the full summary report to a yaml file (named herein). Each data
+        id and error message is keyed to a quantum graph node id. A convenient
+        output format for error-matching and cataloguing tools such as the ones
+        in the Campaign Management database. If this is not included, quanta
+        and dataset information will be printed to the command-line instead.
+    logs : `bool`
+        Get butler log datasets for extra information (error messages).
+    brief : `bool`
+        List only the counts (or data_ids if number of failures < 5). This
+        option is good for those who just want to see totals.
     """
     butler = Butler.from_config(butler_config, writeable=False)
     qgraph = QuantumGraph.loadUri(qgraph_uri)
@@ -125,6 +125,9 @@ def report_v2(
     logs: bool = True,
     brief: bool = False,
     curse_failed_logs: bool = False,
+    read_caveats: Literal["lazy", "exhaustive"] | None = "lazy",
+    use_qbb: bool = True,
+    n_cores: int = 1,
 ) -> None:
     """Summarize the state of executed quantum graph(s), with counts of failed,
     successful and expected quanta, as well as counts of output datasets and
@@ -135,62 +138,62 @@ def report_v2(
     Parameters
     ----------
     butler_config : `str`
-            The Butler used for this report. This should match the Butler used
-            for the run associated with the executed quantum graph.
+        The Butler used for this report. This should match the Butler used for
+        the run associated with the executed quantum graph.
     qgraph_uris : `Sequence` [`str`]
-            One or more uris to the serialized Quantum Graph(s).
+        One or more uris to the serialized Quantum Graph(s).
     collections : `Sequence` [`str`] | None`
-            Collection(s) associated with said graphs/processing. For use in
-            `lsst.daf.butler.registry.queryDatasets` if paring down the query
-            would be useful.
+        Collection(s) associated with said graphs/processing. For use in
+        `lsst.daf.butler.registry.queryDatasets` if paring down the query would
+        be useful.
     where : `str`
-            A "where" string to use to constrain the collections, if passed.
+        A "where" string to use to constrain the collections, if passed.
     full_output_filename : `str`
-            Output the full pydantic model `QuantumProvenanceGraph.Summary`
-            object into a JSON file. This is ideal for error-matching and
-            cataloguing tools such as the ones used by Campaign Management
-            software and pilots, and for searching and counting specific kinds
-            or instances of failures. This option will also print a "brief"
-            (counts-only) summary to stdout.
+        Output the full pydantic model `QuantumProvenanceGraph.Summary` object
+        into a JSON file. This is ideal for error-matching and cataloguing
+        tools such as the ones used by Campaign Management software and pilots,
+        and for searching and counting specific kinds or instances of failures.
+        This option will also print a "brief" (counts-only) summary to stdout.
     logs : `bool`
-            Store error messages from Butler logs associated with failed quanta
-            if `True`.
+        Store error messages from Butler logs associated with failed quanta if
+        `True`.
     brief : `bool`
-            Only display short (counts-only) summary on stdout. This includes
-            counts and not error messages or data_ids (similar to BPS report).
-            This option will still report all `cursed` datasets and `wonky`
-            quanta.
+        Only display short (counts-only) summary on stdout. This includes
+        counts and not error messages or data_ids (similar to BPS report). This
+        option will still report all `cursed` datasets and `wonky` quanta.
     curse_failed_logs : `bool`
-            Mark log datasets as `cursed` if they are published in the final
-            output collection. Note that a campaign-level collection must be
-            used here for `collections` if `curse_failed_logs` is `True`; if
-            `lsst.pipe.base.QuantumProvenanceGraph.__resolve_duplicates`
-            is run on a list of group-level collections, then each will only
-            show log datasets from their own failures as visible and datasets
-            from others will be marked as cursed.
+        Mark log datasets as `cursed` if they are published in the final output
+        collection. Note that a campaign-level collection must be used here for
+        `collections` if `curse_failed_logs` is `True`; if
+        `lsst.pipe.base.QuantumProvenanceGraph.__resolve_duplicates` is run on
+        a list of group-level collections, then each will only show log
+        datasets from their own failures as visible and datasets from others
+        will be marked as cursed.
+    read_caveats : `str`, optional
+        Whether and how to read success caveats from metadata datasets:
+
+        - "exhaustive": read all metadata datasets;
+        - "lazy": read metadata datasets only for quanta that had predicted
+          outputs that were not produced (will not pick up exceptions raised
+          after all datasets were written);
+        - `None`: do not read metadata datasets at all.
+    use_qbb : `bool`, optional
+        Whether to use a quantum-backed butler for metadata and log reads.
+        This should reduce the number of database operations.
+    n_cores : `int`, optional
+        Number of cores for metadata and log reads.
     """
     butler = Butler.from_config(butler_config, writeable=False)
-    qpg = QuantumProvenanceGraph()
-    qgraphs = []
-    for qgraph_uri in qgraph_uris:
-        qgraph = QuantumGraph.loadUri(qgraph_uri)
-        assert qgraph.metadata is not None, "Saved QGs always have metadata."
-        qgraphs.append(qgraph)
-    # If the most recent graph's timestamp was earlier than any of the
-    # previous graphs, raise a RuntimeError.
-    for count, qgraph in enumerate(qgraphs):
-        if len(qgraphs) > 1:
-            previous_graph = qgraphs[count - 1]
-            if count > 0 and qgraph.metadata["time"] < previous_graph.metadata["time"]:
-                raise RuntimeError(
-                    f"""add_new_graph may only be called on graphs
-                    which are passed in the order they were
-                    created. Please call again, passing your
-                    graphs in order. Time of second graph:
-                    {qgraph.metadata["time"]} >
-                    time of first graph: {previous_graph.metadata["time"]}"""
-                )
-    qpg.assemble_quantum_provenance_graph(butler, qgraphs, collections, where, curse_failed_logs)
+    qpg = QuantumProvenanceGraph(
+        butler,
+        qgraph_uris,
+        collections=collections,
+        where=where,
+        curse_failed_logs=curse_failed_logs,
+        read_caveats=read_caveats,
+        use_qbb=use_qbb,
+        n_cores=n_cores,
+    )
     summary = qpg.to_summary(butler, do_store_logs=logs)
     print_summary(summary, full_output_filename, brief)
 

lsst/ctrl/mpexec/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.1100"
+__version__ = "29.2025.1300"

{lsst_ctrl_mpexec-29.2025.1100.dist-info → lsst_ctrl_mpexec-29.2025.1300.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: lsst-ctrl-mpexec
-Version: 29.2025.1100
+Version: 29.2025.1300
 Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License
@@ -33,6 +33,7 @@ Provides-Extra: coverage
 Requires-Dist: coverage; extra == "coverage"
 Provides-Extra: test
 Requires-Dist: pytest>=3.2; extra == "test"
+Dynamic: license-file
 
 ################
 lsst-ctrl-mpexec

{lsst_ctrl_mpexec-29.2025.1100.dist-info → lsst_ctrl_mpexec-29.2025.1300.dist-info}/RECORD

@@ -17,12 +17,12 @@ lsst/ctrl/mpexec/simple_pipeline_executor.py,sha256=scqQ1MDS6FcZvFTwxtCkmTSg0zJm
 lsst/ctrl/mpexec/singleQuantumExecutor.py,sha256=mlv3nF29N1meiHie3r072Ynz5k6XTpFO8jfhq7BufF8,28036
 lsst/ctrl/mpexec/taskFactory.py,sha256=c4xj8cR_Ts5uzzGovh87ZKdVeeXy5E3lIjCwJnDuOqg,2720
 lsst/ctrl/mpexec/util.py,sha256=y2Rw5PL40_EuLtVxiqSVX0JfPV4IrFl1LfOMUWx2u30,4236
-lsst/ctrl/mpexec/version.py,sha256=sHiUhHRfi72yQHyY-cFkHWLqVyELO1VeQ3Uo0NmIBCc,55
+lsst/ctrl/mpexec/version.py,sha256=tryL-NT2fHNIinBXQOOXxLFsVaDOMR53wZgDt_5tYb0,55
 lsst/ctrl/mpexec/cli/__init__.py,sha256=6dpDHNBzyicVpFi1fsaiYVbYEMeoL57IHKkPaej24gs,1301
 lsst/ctrl/mpexec/cli/pipetask.py,sha256=4HnhX9dCizCihVbpHVJX5WXO9TEli9oL6wA-tPh1_vA,2209
 lsst/ctrl/mpexec/cli/utils.py,sha256=5iOrlj5jJTWtZS0BMLsuiCGAvxbfrjd1MSyXxBthWcc,6503
 lsst/ctrl/mpexec/cli/cmd/__init__.py,sha256=nRmwwW5d55gAEkyE7NpSK8mxa56HcfEta2r-Y9I07F8,1661
-lsst/ctrl/mpexec/cli/cmd/commands.py,sha256=VKfOoGem15I3JErBY8Iic9dmcuQkzPHwc4MCsbD9nsM,16891
+lsst/ctrl/mpexec/cli/cmd/commands.py,sha256=7ohYuNeUhnnyNvWYWYBdHLkNm5VWsOQiOh6HjzBBYXc,17539
 lsst/ctrl/mpexec/cli/opt/__init__.py,sha256=IzUInuJj9igiaNcEqMx0adelgJtQC5_XMYnaiizBn0A,1378
 lsst/ctrl/mpexec/cli/opt/arguments.py,sha256=vjUw0ZN_4HStp-_3ne6AT5S_eH7sly3OVfL07tgrJnY,1572
 lsst/ctrl/mpexec/cli/opt/optionGroups.py,sha256=z1vrRsNlk11cGulbD9qsBOuPhZgryPrka_IxIeVW9TM,8031
@@ -34,17 +34,17 @@ lsst/ctrl/mpexec/cli/script/confirmable.py,sha256=Bo1GSTZQn44d_TRj6N3YfpYcZiuHEY
 lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py,sha256=DGNseiavrI1VxR_o6vCelbPvr9kCQZzroZHHSQe-RTE,2479
 lsst/ctrl/mpexec/cli/script/purge.py,sha256=gYwSsZfTBP6oDcDp_YdqQEKGvAStvsj5hwNw42S8ptE,10637
 lsst/ctrl/mpexec/cli/script/qgraph.py,sha256=p5z84bmRBbZXIEql76nXYnRQcLau-VNTUVkfOhdqjfU,9805
-lsst/ctrl/mpexec/cli/script/report.py,sha256=IazYZT-aOtkYpCpGqslMazjXYVma5GajiEbJP_OTVUU,11656
+lsst/ctrl/mpexec/cli/script/report.py,sha256=AssS4S99LFrPZtATsu7lsXVdxk7mOwfrdM9LXmUxgZ0,11427
 lsst/ctrl/mpexec/cli/script/run.py,sha256=22g7vBhGDGsFSQq7CtruTf-bivo3t53decBAV0wAg1g,9767
 lsst/ctrl/mpexec/cli/script/run_qbb.py,sha256=oCEydg13zNAtci8XB0IwXx_D5PPsHbjCPATWoBlhcPk,5294
 lsst/ctrl/mpexec/cli/script/update_graph_run.py,sha256=v_EdOaD6jR_vSlgm_5-pwUjoNEFMrAuYFM1xIaHVU3Q,2597
-lsst_ctrl_mpexec-29.2025.1100.dist-info/COPYRIGHT,sha256=pGCjnRAnyt02a6_9PLzXQikpvYmvMmK9fCdOKlRSV6k,369
-lsst_ctrl_mpexec-29.2025.1100.dist-info/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_mpexec-29.2025.1100.dist-info/METADATA,sha256=PJ-h2IwjqTbxG3DaHgp6YvyLg3wJbkmV71zchYxH5lc,2280
-lsst_ctrl_mpexec-29.2025.1100.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-lsst_ctrl_mpexec-29.2025.1100.dist-info/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_mpexec-29.2025.1100.dist-info/entry_points.txt,sha256=aYE38yqZU8qvpLUUkXzgmUxDJYYknEqPxgxYkowrL4s,64
-lsst_ctrl_mpexec-29.2025.1100.dist-info/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_mpexec-29.2025.1100.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_mpexec-29.2025.1100.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_mpexec-29.2025.1100.dist-info/RECORD,,
+lsst_ctrl_mpexec-29.2025.1300.dist-info/licenses/COPYRIGHT,sha256=pGCjnRAnyt02a6_9PLzXQikpvYmvMmK9fCdOKlRSV6k,369
+lsst_ctrl_mpexec-29.2025.1300.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_mpexec-29.2025.1300.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_mpexec-29.2025.1300.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_mpexec-29.2025.1300.dist-info/METADATA,sha256=FlKQIJZKX8EfRF1yM6xXCbtIAHE5XfFB2ULaqZSw4dE,2302
+lsst_ctrl_mpexec-29.2025.1300.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+lsst_ctrl_mpexec-29.2025.1300.dist-info/entry_points.txt,sha256=aYE38yqZU8qvpLUUkXzgmUxDJYYknEqPxgxYkowrL4s,64
+lsst_ctrl_mpexec-29.2025.1300.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_mpexec-29.2025.1300.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_mpexec-29.2025.1300.dist-info/RECORD,,

{lsst_ctrl_mpexec-29.2025.1100.dist-info → lsst_ctrl_mpexec-29.2025.1300.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.0.0)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any