lsst-pipe-base 30.2026.300__py3-none-any.whl → 30.2026.500__py3-none-any.whl

lsst/pipe/base/quantum_graph/aggregator/__init__.py

@@ -140,4 +140,3 @@ from ._communicators import FatalWorkerError
 # - having the worker logs go to separate files is actually very nice, and it's
 #   more efficient if they just do that themselves, and that's not something
 #   our logging CLI can actually do, AFAICT.
-

lsst/pipe/base/quantum_graph/aggregator/_communicators.py

@@ -318,6 +318,12 @@ Report: TypeAlias = (
 )
 
 
+def _disable_resources_parallelism() -> None:
+    os.environ["LSST_RESOURCES_NUM_WORKERS"] = "1"
+    os.environ.pop("LSST_RESOURCES_EXECUTOR", None)
+    os.environ["LSST_S3_USE_THREADS"] = "False"
+
+
 class SupervisorCommunicator:
     """A helper object that lets the supervisor direct the other workers.
 
@@ -364,7 +370,7 @@ class SupervisorCommunicator:
         # starts its shutdown.
         self._write_requests: (
             Queue[ProvenanceQuantumScanData | Literal[_Sentinel.NO_MORE_WRITE_REQUESTS]] | None
-        ) = context.make_queue() if config.output_path is not None else None
+        ) = context.make_queue() if config.is_writing_provenance else None
         # All other workers use this queue to send many different kinds of
         # reports the supervisor.  The supervisor waits for a _DONE sentinal
         # from each worker before it finishes its shutdown.
@@ -433,6 +439,7 @@ class SupervisorCommunicator:
         self._expect_empty_queue(self._compression_dict)
 
     def __enter__(self) -> Self:
+        _disable_resources_parallelism()
         self.progress.__enter__()
         # We make the low-level logger in __enter__ instead of __init__ only
         # because that's the pattern used by true workers (where it matters).
@@ -581,6 +588,7 @@ class WorkerCommunicator:
         self._cancel_event = supervisor._cancel_event
 
     def __enter__(self) -> Self:
+        _disable_resources_parallelism()
         self.log = make_worker_log(self.name, self.config)
         self.log.verbose("%s has PID %s (parent is %s).", self.name, os.getpid(), os.getppid())
         self._exit_stack = ExitStack().__enter__()

lsst/pipe/base/quantum_graph/aggregator/_config.py

@@ -29,6 +29,8 @@ from __future__ import annotations
 
 __all__ = ("AggregatorConfig",)
 
+import sys
+from typing import TYPE_CHECKING, Any
 
 import pydantic
 
@@ -60,11 +62,13 @@ class AggregatorConfig(pydantic.BaseModel):
     n_processes: int = 1
     """Number of processes the scanner should use."""
 
-    assume_complete: bool = True
-    """If `True`, the aggregator can assume all quanta have run to completion
-    (including any automatic retries).  If `False`, only successes can be
-    considered final, and quanta that appear to have failed or to have not been
-    executed are ignored.
+    incomplete: bool = False
+    """If `True`, do not expect the graph to have been executed to completion
+    yet, and only ingest the outputs of successful quanta.
+
+    This disables writing the provenance quantum graph, since this is likely to
+    be wasted effort that just complicates a follow-up run with
+    ``incomplete=False`` later.
     """
 
     defensive_ingest: bool = False
@@ -95,11 +99,10 @@ class AggregatorConfig(pydantic.BaseModel):
     """
 
     dry_run: bool = False
-    """If `True`, do not actually perform any deletions or central butler
-    ingests.
+    """If `True`, do not actually perform any central butler ingests.
 
-    Most log messages concerning deletions and ingests will still be emitted in
-    order to provide a better emulation of a real run.
+    Most log messages concerning ingests will still be emitted in order to
+    provide a better emulation of a real run.
     """
 
     interactive_status: bool = False
@@ -137,3 +140,69 @@ class AggregatorConfig(pydantic.BaseModel):
     """Enable support for storage classes by created by the
     lsst.pipe.base.tests.mocks package.
     """
+
+    promise_ingest_graph: bool = False
+    """If `True`, the aggregator will assume that `~.ingest_graph.ingest_graph`
+    will be run later to ingest metadata/log/config datasets, and will not
+    ingest them itself.  This means that if `~.ingest_graph.ingest_graph` is
+    not run, those files will be abandoned in the butler storage root without
+    being present in the butler database, but it will speed up both processes.
+
+    It is *usually* safe to build a quantum graph for downstream processing
+    before or while running `~.ingest_graph.ingest_graph`, because
+    metadata/log/config datasets are rarely used as inputs.  To check, use
+    ``pipetask build ... --show inputs`` to show the overall-inputs to the
+    graph and scan for these dataset types.
+    """
+
+    @property
+    def is_writing_provenance(self) -> bool:
+        """Whether the aggregator is configured to write the provenance quantum
+        graph.
+        """
+        return self.output_path is not None and not self.incomplete
+
+    # Work around the fact that Sphinx chokes on Pydantic docstring formatting,
+    # when we inherit those docstrings in our public classes.
+    if "sphinx" in sys.modules and not TYPE_CHECKING:
+
+        def copy(self, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.copy`."""
+            return super().copy(*args, **kwargs)
+
+        def model_dump(self, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_dump`."""
+            return super().model_dump(*args, **kwargs)
+
+        def model_dump_json(self, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_dump_json`."""
+            return super().model_dump(*args, **kwargs)
+
+        def model_copy(self, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_copy`."""
+            return super().model_copy(*args, **kwargs)
+
+        @classmethod
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc, override]
+            """See `pydantic.BaseModel.model_construct`."""
+            return super().model_construct(*args, **kwargs)
+
+        @classmethod
+        def model_json_schema(cls, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_json_schema`."""
+            return super().model_json_schema(*args, **kwargs)
+
+        @classmethod
+        def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_validate`."""
+            return super().model_validate(*args, **kwargs)
+
+        @classmethod
+        def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_validate_json`."""
+            return super().model_validate_json(*args, **kwargs)
+
+        @classmethod
+        def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_validate_strings`."""
+            return super().model_validate_strings(*args, **kwargs)

lsst/pipe/base/quantum_graph/aggregator/_ingester.py

@@ -43,7 +43,7 @@ from lsst.daf.butler.registry import ConflictingDefinitionError
 
 from ...pipeline_graph import TaskImportMode
 from .._common import DatastoreName
-from .._predicted import PredictedDatasetModel, PredictedQuantumGraphComponents, PredictedQuantumGraphReader
+from .._predicted import PredictedQuantumGraphComponents, PredictedQuantumGraphReader
 from ._communicators import IngesterCommunicator
 
 
@@ -170,7 +170,7 @@ class Ingester(AbstractContextManager):
             for ingest_request in self.comms.poll():
                 self.n_producers_pending += 1
                 self.comms.log.debug(f"Got ingest request for producer {ingest_request.producer_id}.")
-                self.update_pending(ingest_request.datasets, ingest_request.records)
+                self.update_outputs_pending(refs=ingest_request.refs, records=ingest_request.records)
                 if self.n_datasets_pending > self.comms.config.ingest_batch_size:
                     self.ingest()
             self.comms.log.info("All ingest requests received.")
@@ -266,31 +266,32 @@ class Ingester(AbstractContextManager):
             else:
                 del self.records_pending[datastore_name]
 
-    def update_pending(
-        self, datasets: list[PredictedDatasetModel], records: dict[DatastoreName, DatastoreRecordData]
+    def update_outputs_pending(
+        self,
+        refs: list[DatasetRef],
+        records: dict[DatastoreName, DatastoreRecordData],
     ) -> None:
         """Add an ingest request to the pending-ingest data structures.
 
         Parameters
         ----------
-        datasets : `list` [ `PredictedDatasetModel` ]
-            Registry information about the datasets.
+        refs : `list` [ `lsst.daf.butler.DatasetRef` ]
+            Registry information about regular quantum-output datasets.
         records : `dict` [ `str`, \
                 `lsst.daf.butler.datastore.record_data.DatastoreRecordData` ]
             Datastore information about the datasets.
         """
-        n_given = len(datasets)
+        n_given = len(refs)
         if self.already_ingested is not None:
-            datasets = [d for d in datasets if d.dataset_id not in self.already_ingested]
-            kept = {d.dataset_id for d in datasets}
+            refs = [ref for ref in refs if ref.id not in self.already_ingested]
+            kept = {ref.id for ref in refs}
             self.n_datasets_skipped += n_given - len(kept)
             records = {
                 datastore_name: filtered_records
                 for datastore_name, original_records in records.items()
                 if (filtered_records := original_records.subset(kept)) is not None
             }
-        for dataset in datasets:
-            ref = self.predicted.make_dataset_ref(dataset)
+        for ref in refs:
             self.refs_pending[ref.datasetType.dimensions].append(ref)
         for datastore_name, datastore_records in records.items():
             if (existing_records := self.records_pending.get(datastore_name)) is not None:

lsst/pipe/base/quantum_graph/aggregator/_scanner.py

@@ -223,7 +223,7 @@ class Scanner(AbstractContextManager):
             logs = self._read_log(predicted_quantum)
             metadata = self._read_metadata(predicted_quantum)
             result = ProvenanceQuantumScanModels.from_metadata_and_logs(
-                predicted_quantum, metadata, logs, assume_complete=self.comms.config.assume_complete
+                predicted_quantum, metadata, logs, incomplete=self.comms.config.incomplete
             )
             if result.status is ProvenanceQuantumScanStatus.ABANDONED:
                 self.comms.log.debug("Abandoning scan for failed quantum %s.", quantum_id)
@@ -233,7 +233,7 @@ class Scanner(AbstractContextManager):
             if predicted_output.dataset_id not in result.output_existence:
                 result.output_existence[predicted_output.dataset_id] = self.scan_dataset(predicted_output)
         to_ingest = self._make_ingest_request(predicted_quantum, result)
-        if self.comms.config.output_path is not None:
+        if self.comms.config.is_writing_provenance:
             to_write = result.to_scan_data(predicted_quantum, compressor=self.compressor)
             self.comms.request_write(to_write)
         self.comms.request_ingest(to_ingest)
@@ -261,15 +261,23 @@ class Scanner(AbstractContextManager):
         predicted_outputs_by_id = {
             d.dataset_id: d for d in itertools.chain.from_iterable(predicted_quantum.outputs.values())
         }
-        to_ingest_predicted: list[PredictedDatasetModel] = []
         to_ingest_refs: list[DatasetRef] = []
+        to_ignore: set[uuid.UUID] = set()
+        if self.comms.config.promise_ingest_graph:
+            if result.status is ProvenanceQuantumScanStatus.INIT:
+                if predicted_quantum.task_label:  # i.e. not the 'packages' producer
+                    to_ignore.add(
+                        predicted_quantum.outputs[acc.CONFIG_INIT_OUTPUT_CONNECTION_NAME][0].dataset_id
+                    )
+            else:
+                to_ignore.add(predicted_quantum.outputs[acc.METADATA_OUTPUT_CONNECTION_NAME][0].dataset_id)
+                to_ignore.add(predicted_quantum.outputs[acc.LOG_OUTPUT_CONNECTION_NAME][0].dataset_id)
         for dataset_id, was_produced in result.output_existence.items():
-            if was_produced:
+            if was_produced and dataset_id not in to_ignore:
                 predicted_output = predicted_outputs_by_id[dataset_id]
-                to_ingest_predicted.append(predicted_output)
                 to_ingest_refs.append(self.reader.components.make_dataset_ref(predicted_output))
         to_ingest_records = self.qbb._datastore.export_predicted_records(to_ingest_refs)
-        return IngestRequest(result.quantum_id, to_ingest_predicted, to_ingest_records)
+        return IngestRequest(result.quantum_id, to_ingest_refs, to_ingest_records)
 
     def _read_metadata(self, predicted_quantum: PredictedQuantumDatasetsModel) -> TaskMetadata | None:
         """Attempt to read the metadata dataset for a quantum.

lsst/pipe/base/quantum_graph/aggregator/_structs.py

@@ -32,10 +32,10 @@ __all__ = ("IngestRequest", "ScanReport")
 import dataclasses
 import uuid
 
+from lsst.daf.butler import DatasetRef
 from lsst.daf.butler.datastore.record_data import DatastoreRecordData
 
 from .._common import DatastoreName
-from .._predicted import PredictedDatasetModel
 from .._provenance import ProvenanceQuantumScanStatus
 
 
@@ -57,11 +57,11 @@ class IngestRequest:
     producer_id: uuid.UUID
     """ID of the quantum that produced these datasets."""
 
-    datasets: list[PredictedDatasetModel]
+    refs: list[DatasetRef]
     """Registry information about the datasets."""
 
     records: dict[DatastoreName, DatastoreRecordData]
     """Datastore information about the datasets."""
 
     def __bool__(self) -> bool:
-        return bool(self.datasets or self.records)
+        return bool(self.refs or self.records)

lsst/pipe/base/quantum_graph/aggregator/_supervisor.py

@@ -117,6 +117,18 @@ class Supervisor:
                 self.comms.request_scan(ready_set.pop())
             for scan_return in self.comms.poll():
                 self.handle_report(scan_return)
+        if self.comms.config.incomplete:
+            quantum_or_quanta = "quanta" if self.n_abandoned != 1 else "quantum"
+            self.comms.progress.log.info(
+                "%d %s incomplete/failed abandoned; re-run with incomplete=False to finish.",
+                self.n_abandoned,
+                quantum_or_quanta,
+            )
+        self.comms.progress.log.info(
+            "Scanning complete after %0.1fs; waiting for workers to finish.",
+            self.comms.progress.elapsed_time,
+        )
+        self.comms.wait_for_workers_to_finish()
 
     def handle_report(self, scan_report: ScanReport) -> None:
         """Handle a report from a scanner.
@@ -134,7 +146,7 @@ class Supervisor:
                 self.comms.log.debug("Scan complete for %s: quantum failed.", scan_report.quantum_id)
                 blocked_quanta = self.walker.fail(scan_report.quantum_id)
                 for blocked_quantum_id in blocked_quanta:
-                    if self.comms.config.output_path is not None:
+                    if self.comms.config.is_writing_provenance:
                         self.comms.request_write(
                             ProvenanceQuantumScanData(
                                 blocked_quantum_id, status=ProvenanceQuantumScanStatus.BLOCKED
@@ -172,7 +184,7 @@ def aggregate_graph(predicted_path: str, butler_path: str, config: AggregatorCon
     writer: Worker | None = None
     with SupervisorCommunicator(log, config.n_processes, ctx, config) as comms:
         comms.progress.log.verbose("Starting workers.")
-        if config.output_path is not None:
+        if config.is_writing_provenance:
             writer_comms = WriterCommunicator(comms)
             writer = ctx.make_worker(
                 target=Writer.run,
@@ -198,17 +210,6 @@ def aggregate_graph(predicted_path: str, butler_path: str, config: AggregatorCon
         ingester.start()
         supervisor = Supervisor(predicted_path, comms)
         supervisor.loop()
-        log.info(
-            "Scanning complete after %0.1fs; waiting for workers to finish.",
-            comms.progress.elapsed_time,
-        )
-        comms.wait_for_workers_to_finish()
-        if supervisor.n_abandoned:
-            raise RuntimeError(
-                f"{supervisor.n_abandoned} {'quanta' if supervisor.n_abandoned > 1 else 'quantum'} "
-                "abandoned because they did not succeed.  Re-run with assume_complete=True after all retry "
-                "attempts have been exhausted."
-            )
     for w in scanners:
         w.join()
     ingester.join()

lsst/pipe/base/quantum_graph/aggregator/_writer.py

@@ -61,7 +61,7 @@ class Writer:
     """
 
     def __post_init__(self) -> None:
-        assert self.comms.config.output_path is not None, "Writer should not be used if writing is disabled."
+        assert self.comms.config.is_writing_provenance, "Writer should not be used if writing is disabled."
         self.comms.log.info("Reading predicted quantum graph.")
         with PredictedQuantumGraphReader.open(
             self.predicted_path, import_mode=TaskImportMode.DO_NOT_IMPORT
@@ -123,7 +123,7 @@ class Writer:
         """
         cdict = self.make_compression_dictionary()
         self.comms.send_compression_dict(cdict.as_bytes())
-        assert self.comms.config.output_path is not None
+        assert self.comms.config.is_writing_provenance and self.comms.config.output_path is not None
         self.comms.log.info("Opening output files and processing predicted graph.")
         qg_writer = ProvenanceQuantumGraphWriter(
             self.comms.config.output_path,

lsst/pipe/base/quantum_graph/formatter.py

@@ -35,9 +35,13 @@ from typing import Any, ClassVar
 import pydantic
 
 from lsst.daf.butler import FormatterV2
+from lsst.daf.butler.logging import ButlerLogRecords
+from lsst.pex.config import Config
 from lsst.resources import ResourcePath
 from lsst.utils.logging import getLogger
+from lsst.utils.packages import Packages
 
+from .._task_metadata import TaskMetadata
 from ..pipeline_graph import TaskImportMode
 from ._provenance import ProvenanceQuantumGraphReader
 
@@ -56,13 +60,13 @@ class _ProvenanceFormatterParameters(pydantic.BaseModel):
 
     @pydantic.field_validator("quanta", mode="before")
     @classmethod
-    def quanta_to_list(cls, v: Any) -> list[uuid.UUID]:
-        return list(v)
+    def quanta_to_list(cls, v: Any) -> list[uuid.UUID] | None:
+        return list(v) if v is not None else None
 
     @pydantic.field_validator("datasets", mode="before")
     @classmethod
-    def datasets_to_list(cls, v: Any) -> list[uuid.UUID]:
-        return list(v)
+    def datasets_to_list(cls, v: Any) -> list[uuid.UUID] | None:
+        return list(v) if v is not None else None
 
     @property
     def nodes(self) -> list[uuid.UUID]:
@@ -83,6 +87,17 @@ class ProvenanceFormatter(FormatterV2):
     can_read_from_uri: ClassVar[bool] = True
 
     def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
+        match self._dataset_ref.datasetType.storageClass_name:
+            case "TaskMetadata" | "PropertySet":
+                return self._read_metadata(uri)
+            case "ButlerLogRecords":
+                return self._read_log(uri)
+            case "Config":
+                return self._read_config(uri)
+            case "ProvenanceQuantumGraph":
+                pass
+            case unexpected:
+                raise ValueError(f"Unsupported storage class {unexpected!r} for ProvenanceFormatter.")
         parameters = _ProvenanceFormatterParameters.model_validate(self.file_descriptor.parameters or {})
         with ProvenanceQuantumGraphReader.open(uri, import_mode=parameters.import_mode) as reader:
             match component:
@@ -99,3 +114,58 @@ class ProvenanceFormatter(FormatterV2):
                 case "packages":
                     return reader.fetch_packages()
         raise AssertionError(f"Unexpected component {component!r}.")
+
+    def _read_metadata(self, uri: ResourcePath) -> TaskMetadata:
+        with ProvenanceQuantumGraphReader.open(uri, import_mode=TaskImportMode.DO_NOT_IMPORT) as reader:
+            try:
+                attempts = reader.fetch_metadata([self._dataset_ref.id])[self._dataset_ref.id]
+            except LookupError:
+                raise FileNotFoundError(
+                    f"No dataset with ID {self._dataset_ref.id} present in this graph."
+                ) from None
+        if not attempts:
+            raise FileNotFoundError(
+                f"No metadata dataset {self._dataset_ref} stored in this graph "
+                "(no attempts for this quantum)."
+            )
+        if attempts[-1] is None:
+            raise FileNotFoundError(
+                f"No metadata dataset {self._dataset_ref} stored in this graph "
+                "(most recent attempt failed and did not write metadata)."
+            )
+        return attempts[-1]
+
+    def _read_log(self, uri: ResourcePath) -> ButlerLogRecords:
+        with ProvenanceQuantumGraphReader.open(uri, import_mode=TaskImportMode.DO_NOT_IMPORT) as reader:
+            try:
+                attempts = reader.fetch_logs([self._dataset_ref.id])[self._dataset_ref.id]
+            except LookupError:
+                raise FileNotFoundError(
+                    f"No dataset with ID {self._dataset_ref.id} present in this graph."
+                ) from None
+        if not attempts:
+            raise FileNotFoundError(
+                f"No log dataset {self._dataset_ref} stored in this graph (no attempts for this quantum)."
+            )
+        if attempts[-1] is None:
+            raise FileNotFoundError(
+                f"No log dataset {self._dataset_ref} stored in this graph "
+                "(most recent attempt failed and did not write logs)."
+            )
+        return attempts[-1]
+
+    def _read_packages(self, uri: ResourcePath) -> Packages:
+        with ProvenanceQuantumGraphReader.open(uri, import_mode=TaskImportMode.DO_NOT_IMPORT) as reader:
+            return reader.fetch_packages()
+
+    def _read_config(self, uri: ResourcePath) -> Config:
+        task_label = self._dataset_ref.datasetType.name.removesuffix("_config")
+        with ProvenanceQuantumGraphReader.open(
+            uri, import_mode=TaskImportMode.ASSUME_CONSISTENT_EDGES
+        ) as reader:
+            try:
+                return reader.pipeline_graph.tasks[task_label].config.copy()
+            except KeyError:
+                raise FileNotFoundError(
+                    f"No task with label {task_label!r} found in the pipeline graph."
+                ) from None