lsst-ctrl-mpexec 29.2025.1000__py3-none-any.whl → 29.2025.1200__py3-none-any.whl

lsst/ctrl/mpexec/cli/cmd/commands.py

@@ -382,6 +382,18 @@ def update_graph_run(
     "even when there is only one qgraph. Otherwise, the `QuantumGraphExecutionReport` "
     "will run on one graph by default.",
 )
+@click.option(
+    "--read-caveats",
+    type=click.Choice(["exhaustive", "lazy", "none"], case_sensitive=False),
+    default="lazy",
+)
+@click.option(
+    "--use-qbb/--no-use-qbb",
+    is_flag=True,
+    default=True,
+    help="Whether to use a quantum-backed butler for metadata and log reads.",
+)
+@processes_option()
 def report(
     repo: str,
     qgraphs: Sequence[str],
@@ -392,6 +404,9 @@ def report(
     brief: bool = False,
     curse_failed_logs: bool = False,
     force_v2: bool = False,
+    read_caveats: str = "lazy",
+    use_qbb: bool = True,
+    processes: int = 1,
 ) -> None:
     """Summarize the state of executed quantum graph(s), with counts of failed,
     successful and expected quanta, as well as counts of output datasets and
@@ -417,7 +432,17 @@ def report(
     """
     if any([force_v2, len(qgraphs) > 1, collections, where, curse_failed_logs]):
         script.report_v2(
-            repo, qgraphs, collections, where, full_output_filename, logs, brief, curse_failed_logs
+            repo,
+            qgraphs,
+            collections,
+            where,
+            full_output_filename,
+            logs,
+            brief,
+            curse_failed_logs,
+            read_caveats=(read_caveats if read_caveats != "none" else None),  # type: ignore[arg-type]
+            use_qbb=use_qbb,
+            n_cores=processes,
         )
     else:
         assert len(qgraphs) == 1, "Cannot make a report without a quantum graph."

lsst/ctrl/mpexec/cli/script/build.py

@@ -129,10 +129,23 @@ def build(  # type: ignore
             )
 
     if pipeline_mermaid:
-        with open(pipeline_mermaid, "w") as stream:
+        # Determine output format based on file extension.
+        if pipeline_mermaid.endswith(".svg"):
+            output_format = "svg"
+            file_mode = "wb"
+        elif pipeline_mermaid.endswith(".png"):
+            output_format = "png"
+            file_mode = "wb"
+        else:  # Default to the text-based mmd format.
+            output_format = "mmd"
+            file_mode = "w"
+
+        with open(pipeline_mermaid, file_mode) as stream:
             visualization.show_mermaid(
                 pipeline.to_graph(butler.registry if butler is not None else None, visualization_only=True),
                 stream,
+                output_format=output_format,
+                width=4500 if output_format != "mmd" else None,
                 dataset_types=True,
                 task_classes="full",
             )

lsst/ctrl/mpexec/cli/script/report.py

@@ -26,6 +26,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import pprint
 from collections.abc import Sequence
+from typing import Literal
 
 from astropy.table import Table
 
@@ -48,23 +49,22 @@ def report(
 
     Parameters
     ----------
-        butler_config : `str`
-            The Butler used for this report. This should match the Butler used
-            for the run associated with the executed quantum graph.
-        qgraph_uri : `str`
-            The uri of the location of said quantum graph.
-        full_output_filename : `str`
-            Output the full summary report to a yaml file (named herein).
-            Each data id and error message is keyed to a quantum graph node id.
-            A convenient output format for error-matching and cataloguing tools
-            such as the ones in the Campaign Management database. If this is
-            not included, quanta and dataset information will be printed to the
-            command-line instead.
-        logs : `bool`
-            Get butler log datasets for extra information (error messages).
-        brief : `bool`
-            List only the counts (or data_ids if number of failures < 5). This
-            option is good for those who just want to see totals.
+    butler_config : `str`
+        The Butler used for this report. This should match the Butler used for
+        the run associated with the executed quantum graph.
+    qgraph_uri : `str`
+        The uri of the location of said quantum graph.
+    full_output_filename : `str`
+        Output the full summary report to a yaml file (named herein). Each data
+        id and error message is keyed to a quantum graph node id. A convenient
+        output format for error-matching and cataloguing tools such as the ones
+        in the Campaign Management database. If this is not included, quanta
+        and dataset information will be printed to the command-line instead.
+    logs : `bool`
+        Get butler log datasets for extra information (error messages).
+    brief : `bool`
+        List only the counts (or data_ids if number of failures < 5). This
+        option is good for those who just want to see totals.
     """
     butler = Butler.from_config(butler_config, writeable=False)
     qgraph = QuantumGraph.loadUri(qgraph_uri)
@@ -125,6 +125,9 @@ def report_v2(
     logs: bool = True,
     brief: bool = False,
     curse_failed_logs: bool = False,
+    read_caveats: Literal["lazy", "exhaustive"] | None = "lazy",
+    use_qbb: bool = True,
+    n_cores: int = 1,
 ) -> None:
     """Summarize the state of executed quantum graph(s), with counts of failed,
     successful and expected quanta, as well as counts of output datasets and
@@ -135,62 +138,62 @@ def report_v2(
     Parameters
     ----------
     butler_config : `str`
-            The Butler used for this report. This should match the Butler used
-            for the run associated with the executed quantum graph.
+        The Butler used for this report. This should match the Butler used for
+        the run associated with the executed quantum graph.
     qgraph_uris : `Sequence` [`str`]
-            One or more uris to the serialized Quantum Graph(s).
+        One or more uris to the serialized Quantum Graph(s).
     collections : `Sequence` [`str`] | None`
-            Collection(s) associated with said graphs/processing. For use in
-            `lsst.daf.butler.registry.queryDatasets` if paring down the query
-            would be useful.
+        Collection(s) associated with said graphs/processing. For use in
+        `lsst.daf.butler.registry.queryDatasets` if paring down the query would
+        be useful.
     where : `str`
-            A "where" string to use to constrain the collections, if passed.
+        A "where" string to use to constrain the collections, if passed.
     full_output_filename : `str`
-            Output the full pydantic model `QuantumProvenanceGraph.Summary`
-            object into a JSON file. This is ideal for error-matching and
-            cataloguing tools such as the ones used by Campaign Management
-            software and pilots, and for searching and counting specific kinds
-            or instances of failures. This option will also print a "brief"
-            (counts-only) summary to stdout.
+        Output the full pydantic model `QuantumProvenanceGraph.Summary` object
+        into a JSON file. This is ideal for error-matching and cataloguing
+        tools such as the ones used by Campaign Management software and pilots,
+        and for searching and counting specific kinds or instances of failures.
+        This option will also print a "brief" (counts-only) summary to stdout.
     logs : `bool`
-            Store error messages from Butler logs associated with failed quanta
-            if `True`.
+        Store error messages from Butler logs associated with failed quanta if
+        `True`.
     brief : `bool`
-            Only display short (counts-only) summary on stdout. This includes
-            counts and not error messages or data_ids (similar to BPS report).
-            This option will still report all `cursed` datasets and `wonky`
-            quanta.
+        Only display short (counts-only) summary on stdout. This includes
+        counts and not error messages or data_ids (similar to BPS report). This
+        option will still report all `cursed` datasets and `wonky` quanta.
     curse_failed_logs : `bool`
-            Mark log datasets as `cursed` if they are published in the final
-            output collection. Note that a campaign-level collection must be
-            used here for `collections` if `curse_failed_logs` is `True`; if
-            `lsst.pipe.base.QuantumProvenanceGraph.__resolve_duplicates`
-            is run on a list of group-level collections, then each will only
-            show log datasets from their own failures as visible and datasets
-            from others will be marked as cursed.
+        Mark log datasets as `cursed` if they are published in the final output
+        collection. Note that a campaign-level collection must be used here for
+        `collections` if `curse_failed_logs` is `True`; if
+        `lsst.pipe.base.QuantumProvenanceGraph.__resolve_duplicates` is run on
+        a list of group-level collections, then each will only show log
+        datasets from their own failures as visible and datasets from others
+        will be marked as cursed.
+    read_caveats : `str`, optional
+        Whether and how to read success caveats from metadata datasets:
+
+        - "exhaustive": read all metadata datasets;
+        - "lazy": read metadata datasets only for quanta that had predicted
+          outputs that were not produced (will not pick up exceptions raised
+          after all datasets were written);
+        - `None`: do not read metadata datasets at all.
+    use_qbb : `bool`, optional
+        Whether to use a quantum-backed butler for metadata and log reads.
+        This should reduce the number of database operations.
+    n_cores : `int`, optional
+        Number of cores for metadata and log reads.
     """
     butler = Butler.from_config(butler_config, writeable=False)
-    qpg = QuantumProvenanceGraph()
-    qgraphs = []
-    for qgraph_uri in qgraph_uris:
-        qgraph = QuantumGraph.loadUri(qgraph_uri)
-        assert qgraph.metadata is not None, "Saved QGs always have metadata."
-        qgraphs.append(qgraph)
-    # If the most recent graph's timestamp was earlier than any of the
-    # previous graphs, raise a RuntimeError.
-    for count, qgraph in enumerate(qgraphs):
-        if len(qgraphs) > 1:
-            previous_graph = qgraphs[count - 1]
-            if count > 0 and qgraph.metadata["time"] < previous_graph.metadata["time"]:
-                raise RuntimeError(
-                    f"""add_new_graph may only be called on graphs
-                    which are passed in the order they were
-                    created. Please call again, passing your
-                    graphs in order. Time of second graph:
-                    {qgraph.metadata["time"]} >
-                    time of first graph: {previous_graph.metadata["time"]}"""
-                )
-    qpg.assemble_quantum_provenance_graph(butler, qgraphs, collections, where, curse_failed_logs)
+    qpg = QuantumProvenanceGraph(
+        butler,
+        qgraph_uris,
+        collections=collections,
+        where=where,
+        curse_failed_logs=curse_failed_logs,
+        read_caveats=read_caveats,
+        use_qbb=use_qbb,
+        n_cores=n_cores,
+    )
     summary = qpg.to_summary(butler, do_store_logs=logs)
     print_summary(summary, full_output_filename, brief)
 

lsst/ctrl/mpexec/cmdLineFwk.py

@@ -71,6 +71,7 @@ from lsst.pipe.base import (
 )
 from lsst.pipe.base.all_dimensions_quantum_graph_builder import AllDimensionsQuantumGraphBuilder
 from lsst.pipe.base.dot_tools import graph2dot
+from lsst.pipe.base.mermaid_tools import graph2mermaid
 from lsst.pipe.base.pipeline_graph import NodeType
 from lsst.utils import doImportType
 from lsst.utils.logging import getLogger
@@ -704,6 +705,10 @@ class CmdLineFwk:
             _LOG.verbose("Writing quantum graph DOT visualization to %r.", args.qgraph_dot)
             graph2dot(qgraph, args.qgraph_dot)
 
+        if args.qgraph_mermaid:
+            _LOG.verbose("Writing quantum graph Mermaid visualization to %r.", args.qgraph_mermaid)
+            graph2mermaid(qgraph, args.qgraph_mermaid)
+
         if args.execution_butler_location:
             _LOG.verbose("Writing execution butler to %r.", args.execution_butler_location)
             butler = Butler.from_config(args.butler_config)

lsst/ctrl/mpexec/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.1000"
+__version__ = "29.2025.1200"

{lsst_ctrl_mpexec-29.2025.1000.dist-info → lsst_ctrl_mpexec-29.2025.1200.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: lsst-ctrl-mpexec
-Version: 29.2025.1000
+Version: 29.2025.1200
 Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License
@@ -33,6 +33,7 @@ Provides-Extra: coverage
 Requires-Dist: coverage; extra == "coverage"
 Provides-Extra: test
 Requires-Dist: pytest>=3.2; extra == "test"
+Dynamic: license-file
 
 ################
 lsst-ctrl-mpexec

{lsst_ctrl_mpexec-29.2025.1000.dist-info → lsst_ctrl_mpexec-29.2025.1200.dist-info}/RECORD

@@ -1,7 +1,7 @@
 lsst/__init__.py,sha256=aXdEOZVrBQISQi6XPS9s1NhBjIJaIwNNxCFRiGchRAw,1369
 lsst/ctrl/__init__.py,sha256=aXdEOZVrBQISQi6XPS9s1NhBjIJaIwNNxCFRiGchRAw,1369
 lsst/ctrl/mpexec/__init__.py,sha256=34loiOJc5I_65Npo6zsO1Bf8dB5zV1y1MQid6W65cV8,1699
-lsst/ctrl/mpexec/cmdLineFwk.py,sha256=93q-HsMjbItXmmdpgoFR-vCYjFUkbS7YbmWpvXP0FbE,41500
+lsst/ctrl/mpexec/cmdLineFwk.py,sha256=4h3qqzUjYRuI8oJehjwa5_ytrxhf0JlBLdDeQYOTGx4,41743
 lsst/ctrl/mpexec/dotTools.py,sha256=nx31DC7NkQM766sC97lPQMI22snVaWS7USFMFVLChqk,3421
 lsst/ctrl/mpexec/execFixupDataId.py,sha256=MUirmuoQiWj7C7wQ23QGkgMqx_5_CP4_UMV6h_gVfxk,5056
 lsst/ctrl/mpexec/executionGraphFixup.py,sha256=1z4P4YmtLTMPoMxmFiOnzljLcP9B5V1SCG9haqR_TyI,2830
@@ -17,34 +17,34 @@ lsst/ctrl/mpexec/simple_pipeline_executor.py,sha256=scqQ1MDS6FcZvFTwxtCkmTSg0zJm
 lsst/ctrl/mpexec/singleQuantumExecutor.py,sha256=mlv3nF29N1meiHie3r072Ynz5k6XTpFO8jfhq7BufF8,28036
 lsst/ctrl/mpexec/taskFactory.py,sha256=c4xj8cR_Ts5uzzGovh87ZKdVeeXy5E3lIjCwJnDuOqg,2720
 lsst/ctrl/mpexec/util.py,sha256=y2Rw5PL40_EuLtVxiqSVX0JfPV4IrFl1LfOMUWx2u30,4236
-lsst/ctrl/mpexec/version.py,sha256=J0EmBUnWW_L_z5vGYTLK_7oa_NUvkGOs1LJlKYkBXtk,55
+lsst/ctrl/mpexec/version.py,sha256=Q55pqojE5GgOP1ODSUiZGmFU6sD87FAaQ-iGrKq2N5Y,55
 lsst/ctrl/mpexec/cli/__init__.py,sha256=6dpDHNBzyicVpFi1fsaiYVbYEMeoL57IHKkPaej24gs,1301
 lsst/ctrl/mpexec/cli/pipetask.py,sha256=4HnhX9dCizCihVbpHVJX5WXO9TEli9oL6wA-tPh1_vA,2209
 lsst/ctrl/mpexec/cli/utils.py,sha256=5iOrlj5jJTWtZS0BMLsuiCGAvxbfrjd1MSyXxBthWcc,6503
 lsst/ctrl/mpexec/cli/cmd/__init__.py,sha256=nRmwwW5d55gAEkyE7NpSK8mxa56HcfEta2r-Y9I07F8,1661
-lsst/ctrl/mpexec/cli/cmd/commands.py,sha256=VKfOoGem15I3JErBY8Iic9dmcuQkzPHwc4MCsbD9nsM,16891
+lsst/ctrl/mpexec/cli/cmd/commands.py,sha256=7ohYuNeUhnnyNvWYWYBdHLkNm5VWsOQiOh6HjzBBYXc,17539
 lsst/ctrl/mpexec/cli/opt/__init__.py,sha256=IzUInuJj9igiaNcEqMx0adelgJtQC5_XMYnaiizBn0A,1378
 lsst/ctrl/mpexec/cli/opt/arguments.py,sha256=vjUw0ZN_4HStp-_3ne6AT5S_eH7sly3OVfL07tgrJnY,1572
 lsst/ctrl/mpexec/cli/opt/optionGroups.py,sha256=z1vrRsNlk11cGulbD9qsBOuPhZgryPrka_IxIeVW9TM,8031
 lsst/ctrl/mpexec/cli/opt/options.py,sha256=TyTYxDrFqhUqfnLPfjr_2kJLPNlBbQ86nX2SuOlqAts,19911
 lsst/ctrl/mpexec/cli/script/__init__.py,sha256=eCuF4FAI5D3pl05IMJj7TCkZq-hireua2mA5Ui-mKSI,1624
-lsst/ctrl/mpexec/cli/script/build.py,sha256=F8EcD6u4EIb8oUkl4KjddhF_HxsIlkgEOq0ufetlDVE,5348
+lsst/ctrl/mpexec/cli/script/build.py,sha256=fTa4veqSfUR5nONedmS14q4N5Y_E3nDP1Q-chJvC5IA,5860
 lsst/ctrl/mpexec/cli/script/cleanup.py,sha256=D7W-azf4mNJcIWhbU5uCRCi94mkb8-Q2ksRFblQGrUw,4990
 lsst/ctrl/mpexec/cli/script/confirmable.py,sha256=Bo1GSTZQn44d_TRj6N3YfpYcZiuHEYoz26WZQwMyb4A,3918
 lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py,sha256=DGNseiavrI1VxR_o6vCelbPvr9kCQZzroZHHSQe-RTE,2479
 lsst/ctrl/mpexec/cli/script/purge.py,sha256=gYwSsZfTBP6oDcDp_YdqQEKGvAStvsj5hwNw42S8ptE,10637
 lsst/ctrl/mpexec/cli/script/qgraph.py,sha256=p5z84bmRBbZXIEql76nXYnRQcLau-VNTUVkfOhdqjfU,9805
-lsst/ctrl/mpexec/cli/script/report.py,sha256=IazYZT-aOtkYpCpGqslMazjXYVma5GajiEbJP_OTVUU,11656
+lsst/ctrl/mpexec/cli/script/report.py,sha256=AssS4S99LFrPZtATsu7lsXVdxk7mOwfrdM9LXmUxgZ0,11427
 lsst/ctrl/mpexec/cli/script/run.py,sha256=22g7vBhGDGsFSQq7CtruTf-bivo3t53decBAV0wAg1g,9767
 lsst/ctrl/mpexec/cli/script/run_qbb.py,sha256=oCEydg13zNAtci8XB0IwXx_D5PPsHbjCPATWoBlhcPk,5294
 lsst/ctrl/mpexec/cli/script/update_graph_run.py,sha256=v_EdOaD6jR_vSlgm_5-pwUjoNEFMrAuYFM1xIaHVU3Q,2597
-lsst_ctrl_mpexec-29.2025.1000.dist-info/COPYRIGHT,sha256=pGCjnRAnyt02a6_9PLzXQikpvYmvMmK9fCdOKlRSV6k,369
-lsst_ctrl_mpexec-29.2025.1000.dist-info/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_mpexec-29.2025.1000.dist-info/METADATA,sha256=5m55RyOiPvT7LbQE4cAT9U_ienv_asYLu6nnT2MfuKk,2280
-lsst_ctrl_mpexec-29.2025.1000.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-lsst_ctrl_mpexec-29.2025.1000.dist-info/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_mpexec-29.2025.1000.dist-info/entry_points.txt,sha256=aYE38yqZU8qvpLUUkXzgmUxDJYYknEqPxgxYkowrL4s,64
-lsst_ctrl_mpexec-29.2025.1000.dist-info/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_mpexec-29.2025.1000.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_mpexec-29.2025.1000.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_mpexec-29.2025.1000.dist-info/RECORD,,
+lsst_ctrl_mpexec-29.2025.1200.dist-info/licenses/COPYRIGHT,sha256=pGCjnRAnyt02a6_9PLzXQikpvYmvMmK9fCdOKlRSV6k,369
+lsst_ctrl_mpexec-29.2025.1200.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_mpexec-29.2025.1200.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_mpexec-29.2025.1200.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_mpexec-29.2025.1200.dist-info/METADATA,sha256=Jpvf0NjiVh3olU70ZVmnolX3oS1H3CJuHe1XwIAg3GM,2302
+lsst_ctrl_mpexec-29.2025.1200.dist-info/WHEEL,sha256=tTnHoFhvKQHCh4jz3yCn0WPTYIy7wXx3CJtJ7SJGV7c,91
+lsst_ctrl_mpexec-29.2025.1200.dist-info/entry_points.txt,sha256=aYE38yqZU8qvpLUUkXzgmUxDJYYknEqPxgxYkowrL4s,64
+lsst_ctrl_mpexec-29.2025.1200.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_mpexec-29.2025.1200.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_mpexec-29.2025.1200.dist-info/RECORD,,

{lsst_ctrl_mpexec-29.2025.1000.dist-info → lsst_ctrl_mpexec-29.2025.1200.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (77.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any