lsst-pipe-base 30.0.1rc1__py3-none-any.whl → 30.2025.5100__py3-none-any.whl

lsst/pipe/base/_instrument.py

@@ -31,14 +31,14 @@ __all__ = ("Instrument",)
 
 import contextlib
 import datetime
+import os.path
 from abc import ABCMeta, abstractmethod
 from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any, Self, cast, final
 
-from lsst.daf.butler import DataCoordinate, DataId, DimensionPacker, DimensionRecord, Formatter, FormatterV2
+from lsst.daf.butler import DataCoordinate, DataId, DimensionPacker, DimensionRecord, Formatter
 from lsst.daf.butler.registry import DataIdError
 from lsst.pex.config import Config, RegistryField
-from lsst.resources import ResourcePath, ResourcePathExpression
 from lsst.utils import doImportType
 from lsst.utils.introspection import get_full_type_name
 
@@ -65,10 +65,10 @@ class Instrument(metaclass=ABCMeta):
     the base class.
     """
 
-    configPaths: Sequence[ResourcePathExpression] = ()
+    configPaths: Sequence[str] = ()
     """Paths to config files to read for specific Tasks.
 
-    The paths in this list should contain files of the form ``task.py``, for
+    The paths in this list should contain files of the form `task.py`, for
     each of the Tasks that requires special configuration.
     """
 
@@ -99,8 +99,7 @@ class Instrument(metaclass=ABCMeta):
 
     @abstractmethod
     def register(self, registry: Registry, *, update: bool = False) -> None:
-        """Insert instrument, and other relevant records into a butler
-        registry.
+        """Insert instrument, and other relevant records into `Registry`.
 
         Parameters
         ----------
@@ -110,10 +109,6 @@ class Instrument(metaclass=ABCMeta):
             If `True` (`False` is default), update existing records if they
             differ from the new ones.
 
-        Returns
-        -------
-        None
-
         Raises
         ------
         lsst.daf.butler.registry.ConflictingDefinitionError
@@ -132,6 +127,13 @@ class Instrument(metaclass=ABCMeta):
         the level of individual dimension entries; new detectors and filters
         should be added, but changes to any existing record should not be.
         This can generally be achieved via a block like
+
+        .. code-block:: python
+
+            with registry.transaction():
+                registry.syncDimensionData("instrument", ...)
+                registry.syncDimensionData("detector", ...)
+                self.registerFilters(registry)
         """
         raise NotImplementedError()
 
@@ -312,7 +314,7 @@ class Instrument(metaclass=ABCMeta):
         return instrument_cls(collection_prefix=collection_prefix)
 
     @staticmethod
-    def importAll(registry: Registry) -> dict[str, type[Instrument]]:
+    def importAll(registry: Registry) -> None:
         """Import all the instruments known to this registry.
 
         This will ensure that all metadata translators have been registered.
@@ -322,43 +324,31 @@ class Instrument(metaclass=ABCMeta):
         registry : `lsst.daf.butler.Registry`
             Butler registry to query to find the information.
 
-        Returns
-        -------
-        imported : `dict` [`str`, `type` [`Instrument`]]
-            A mapping containing all the instrument classes that were loaded
-            successfully, keyed by their butler names.
-
         Notes
         -----
         It is allowed for a particular instrument class to fail on import.
         This might simply indicate that a particular obs package has
         not been setup.
         """
-        imported: dict[str, type[Instrument]] = {}
         records = list(registry.queryDimensionRecords("instrument"))
         for record in records:
             cls = record.class_name
-            instrument_name: str = cast(str, record.name)
             with contextlib.suppress(Exception):
-                instr = doImportType(cls)
-                assert issubclass(instr, Instrument)
-                imported[instrument_name] = instr
-        return imported
+                doImportType(cls)
 
     @abstractmethod
-    def getRawFormatter(self, dataId: DataId) -> type[Formatter | FormatterV2]:
+    def getRawFormatter(self, dataId: DataId) -> type[Formatter]:
         """Return the Formatter class that should be used to read a particular
         raw file.
 
         Parameters
         ----------
-        dataId : `lsst.daf.butler.DataId`
+        dataId : `DataId`
             Dimension-based ID for the raw file or files being ingested.
 
         Returns
         -------
-        formatter : `type` \
-                [`lsst.daf.butler.Formatter` | `lsst.daf.butler.FormatterV2` ]
+        formatter : `lsst.daf.butler.Formatter` class
             Class to be used that reads the file into the correct
             Python object for the raw data.
         """
@@ -376,10 +366,9 @@ class Instrument(metaclass=ABCMeta):
             Config instance to which overrides should be applied.
         """
         for root in self.configPaths:
-            resource = ResourcePath(root, forceDirectory=True, forceAbsolute=True)
-            uri = resource.join(f"{name}.py", forceDirectory=False)
-            if uri.exists():
-                config.load(uri)
+            path = os.path.join(root, f"{name}.py")
+            if os.path.exists(path):
+                config.load(path)
 
     @staticmethod
     def formatCollectionTimestamp(timestamp: str | datetime.datetime) -> str:

lsst/pipe/base/_quantumContext.py

@@ -380,8 +380,8 @@ class QuantumContext:
             if dataset is directly a `list` of `~lsst.daf.butler.DatasetRef`
             or a single `~lsst.daf.butler.DatasetRef`. If ``values.NAME`` is
             None, no output is written.
-        dataset : `OutputQuantizedConnection` or `list` \
-                [`lsst.daf.butler.DatasetRef`] or `lsst.daf.butler.DatasetRef`
+        dataset : `OutputQuantizedConnection` or `list`[`DatasetRef`] \
+                or `DatasetRef`
             This argument may either be an `InputQuantizedConnection` which
             describes all the inputs of a quantum, a list of
             `lsst.daf.butler.DatasetRef`, or a single
@@ -460,7 +460,7 @@ class QuantumContext:
 
         Parameters
         ----------
-        ref : `lsst.daf.butler.DatasetRef`
+        ref : `DatasetRef`
             The dataset to attach provenance to. This dataset must have been
             retrieved by this quantum context.
         extra : `dict` [ `str`, `int` | `float` | `str` | `bool` ]

lsst/pipe/base/_status.py

@@ -275,23 +275,15 @@ class ExceptionInfo(pydantic.BaseModel):
 class QuantumAttemptStatus(enum.Enum):
     """Enum summarizing an attempt to run a quantum."""
 
-    ABORTED = -4
-    """The quantum failed with a hard error that prevented both logs and
-    metadata from being written.
-
-    This state is only set if information from higher-level tooling (e.g. BPS)
-    is available to distinguish it from ``UNKNOWN``.
-    """
-
     UNKNOWN = -3
     """The status of this attempt is unknown.
 
-    This means no logs or metadata were written, and it at least could not be
-    determined whether the quantum was blocked by an upstream failure (if it
-    was definitely blocked, `BLOCKED` is set instead).
+    This usually means no logs or metadata were written, and it at least could
+    not be determined whether the quantum was blocked by an upstream failure
+    (if it was definitely blocked, `BLOCKED` is set instead).
     """
 
-    ABORTED_SUCCESS = -2
+    LOGS_MISSING = -2
     """Task metadata was written for this attempt but logs were not.
 
     This is a rare condition that requires a hard failure (i.e. the kind that
@@ -300,21 +292,20 @@ class QuantumAttemptStatus(enum.Enum):
     """
 
     FAILED = -1
-    """Execution of the quantum failed gracefully.
+    """Execution of the quantum failed.
 
     This is always set if the task metadata dataset was not written but logs
     were, as is the case when a Python exception is caught and handled by the
-    execution system.
-
-    This status guarantees that the task log dataset was produced but the
-    metadata dataset was not.
+    execution system.  It may also be set in cases where logs were not written
+    either, but other information was available (e.g. from higher-level
+    orchestration tooling) to mark it as a failure.
     """
 
     BLOCKED = 0
     """This quantum was not executed because an upstream quantum failed.
 
-    Upstream quanta with status `UNKNOWN`, `FAILED`, or `ABORTED` are
-    considered blockers; `ABORTED_SUCCESS` is not.
+    Upstream quanta with status `UNKNOWN` or `FAILED` are considered blockers;
+    `LOGS_MISSING` is not.
     """
 
     SUCCESSFUL = 1
@@ -328,30 +319,6 @@ class QuantumAttemptStatus(enum.Enum):
     these "successes with caveats" are reported.
     """
 
-    @property
-    def has_metadata(self) -> bool:
-        """Whether the task metadata dataset was produced."""
-        return self is self.SUCCESSFUL or self is self.ABORTED_SUCCESS
-
-    @property
-    def has_log(self) -> bool:
-        """Whether the log dataset was produced."""
-        return self is self.SUCCESSFUL or self is self.FAILED
-
-    @property
-    def title(self) -> str:
-        """A version of this status' name suitable for use as a title in a plot
-        or table.
-        """
-        return self.name.capitalize().replace("_", " ")
-
-    @property
-    def is_rare(self) -> bool:
-        """Whether this status is rare enough that it should only be listed
-        when it actually occurs.
-        """
-        return self in (self.ABORTED, self.ABORTED_SUCCESS, self.UNKNOWN)
-
 
 class GetSetDictMetadataHolder(Protocol):
     """Protocol for objects that have a ``metadata`` attribute that satisfies

lsst/pipe/base/_task_metadata.py

@@ -37,7 +37,7 @@ import itertools
 import numbers
 import sys
 from collections.abc import Collection, Iterator, Mapping, Sequence
-from typing import Any, Protocol
+from typing import Any, Protocol, TypeAlias, Union
 
 from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictFloat, StrictInt, StrictStr
 
@@ -47,7 +47,7 @@ _ALLOWED_PRIMITIVE_TYPES = (str, float, int, bool)
 
 # Note that '|' syntax for unions doesn't work when we have to use a string
 # literal (and we do since it's recursive and not an annotation).
-type NestedMetadataDict = Mapping[str, str | float | int | bool | "NestedMetadataDict"]
+NestedMetadataDict: TypeAlias = Mapping[str, Union[str, float, int, bool, "NestedMetadataDict"]]
 
 
 class PropertySetLike(Protocol):

lsst/pipe/base/all_dimensions_quantum_graph_builder.py

@@ -574,7 +574,7 @@ class _DimensionGroupBranch:
 
         Parameters
         ----------
-        log : `lsst.utils.logging.LsstLogAdapter`
+        log : `lsst.logging.LsstLogAdapter`
             Logger to use for status reporting.
         log_indent : `str`, optional
             Indentation to prefix the log message.  This is used when recursing
@@ -739,7 +739,7 @@ class _DimensionGroupTree:
             Query constraint specified by the user.
         data_id_tables : `~collections.abc.Iterable` [ `astropy.table.Table` ]
             Data ID tables being joined into the query.
-        log : `lsst.utils.logging.LsstLogAdapter`
+        log : `lsst.log.LsstLogAdapter`
             Logger that supports ``verbose`` output.
         """
         universe = self.all_dimensions.universe
@@ -805,7 +805,7 @@ class _DimensionGroupTree:
         ----------
         requested : `DatasetQueryConstraintVariant`
             Query constraint specified by the user.
-        log : `lsst.utils.logging.LsstLogAdapter`
+        log : `lsst.log.LsstLogAdapter`
             Logger that supports ``verbose`` output.
         """
         overall_inputs: dict[str, DatasetTypeNode] = {
@@ -834,11 +834,6 @@ class _DimensionGroupTree:
                         remainder,
                     )
                 self.dataset_constraint.intersection_update(inputs)
-                if not self.dataset_constraint:
-                    raise QuantumGraphBuilderError(
-                        "An explicit dataset query constraint was provided, but it does not include any "
-                        f"inputs to the pipeline subset with tasks {list(self.subgraph.tasks.keys())}."
-                    )
             case _:
                 raise QuantumGraphBuilderError(
                     f"Unable to handle type {requested} given as dataset query constraint."

lsst/pipe/base/automatic_connection_constants.py

@@ -26,7 +26,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """Constants used to define the connections automatically added for each
-PipelineTask by the execution system, as well as other special dataset types.
+PipelineTask by the execution system.
 """
 
 from __future__ import annotations
@@ -43,10 +43,6 @@ __all__ = (
     "METADATA_OUTPUT_TEMPLATE",
     "PACKAGES_INIT_OUTPUT_NAME",
     "PACKAGES_INIT_OUTPUT_STORAGE_CLASS",
-    "PROVENANCE_DATASET_TYPE_NAME",
-    "PROVENANCE_STORAGE_CLASS",
-    "RESOURCE_USAGE_STORAGE_CLASS",
-    "RESOURCE_USAGE_TEMPLATE",
 )
 
 
@@ -95,18 +91,3 @@ type names.
 METADATA_OUTPUT_STORAGE_CLASS: str = "TaskMetadata"
 """Name of the storage class for task metadata output datasets.
 """
-
-PROVENANCE_DATASET_TYPE_NAME: str = "run_provenance"
-"""Name of the dataset used to store per-RUN provenance."""
-
-PROVENANCE_STORAGE_CLASS: str = "ProvenanceQuantumGraph"
-"""Name of the storage class used to store provenance."""
-
-RESOURCE_USAGE_TEMPLATE: str = "{label}_resource_usage"
-"""String template used to form the name of the resource usage dataset type for
-a task.
-"""
-
-RESOURCE_USAGE_STORAGE_CLASS: str = "ArrowAstropy"
-"""Storage class of the resource usage dataset type for a task.
-"""

lsst/pipe/base/cli/cmd/__init__.py

@@ -25,22 +25,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-__all__ = [
-    "register_instrument",
-    "transfer_from_graph",
-    "zip_from_graph",
-    "retrieve_artifacts_for_quanta",
-    "aggregate_graph",
-    "ingest_graph",
-    "provenance_report",
-]
+__all__ = ["register_instrument", "transfer_from_graph", "zip_from_graph", "retrieve_artifacts_for_quanta", "aggregate_graph"]
 
-from .commands import (
-    register_instrument,
-    retrieve_artifacts_for_quanta,
-    transfer_from_graph,
-    zip_from_graph,
-    aggregate_graph,
-    ingest_graph,
-    provenance_report,
-)
+from .commands import (register_instrument, retrieve_artifacts_for_quanta, transfer_from_graph, zip_from_graph, aggregate_graph)

lsst/pipe/base/cli/cmd/commands.py

@@ -25,9 +25,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import functools
-import operator
-from collections.abc import Iterable
 from typing import Any
 
 import click
@@ -43,7 +40,6 @@ from lsst.daf.butler.cli.opt import (
 from lsst.daf.butler.cli.utils import ButlerCommand, split_commas, unwrap
 
 from ... import script
-from ..._status import QuantumAttemptStatus, QuantumSuccessCaveats
 from ...quantum_graph import aggregator
 from ..opt import instrument_argument, update_output_chain_option
 
@@ -165,7 +161,7 @@ _AGGREGATOR_DEFAULTS = aggregator.AggregatorConfig()
 
 @click.command(short_help="Scan for the outputs of an active or completed quantum graph.", cls=ButlerCommand)
 @click.argument("predicted_graph", required=True)
-@repo_argument(required=True, help="Path or alias for the butler repository.")
+@repo_argument(required=True, help="Path to the central butler repository.")
 @click.option(
     "-o",
     "--output",
@@ -185,9 +181,9 @@ _AGGREGATOR_DEFAULTS = aggregator.AggregatorConfig()
     help="Number of processes to use.",
 )
 @click.option(
-    "--incomplete/--complete",
-    "incomplete",
-    default=_AGGREGATOR_DEFAULTS.incomplete,
+    "--complete/--incomplete",
+    "assume_complete",
+    default=_AGGREGATOR_DEFAULTS.assume_complete,
     help="Whether execution has completed (and failures cannot be retried).",
 )
 @click.option(
@@ -253,14 +249,6 @@ _AGGREGATOR_DEFAULTS = aggregator.AggregatorConfig()
     default=_AGGREGATOR_DEFAULTS.mock_storage_classes,
     help="Enable support for storage classes created by the lsst.pipe.base.tests.mocks package.",
 )
-@click.option(
-    "--promise-ingest-graph/--no-promise-ingest-graph",
-    default=_AGGREGATOR_DEFAULTS.promise_ingest_graph,
-    help=(
-        "Promise to run 'butler ingest-graph' later, allowing aggregate-graph "
-        "to skip metadata/log/config ingestion for now."
-    ),
-)
 def aggregate_graph(predicted_graph: str, repo: str, **kwargs: Any) -> None:
     """Scan for quantum graph's outputs to gather provenance, ingest datasets
     into the central butler repository, and delete datasets that are no
@@ -280,136 +268,3 @@ def aggregate_graph(predicted_graph: str, repo: str, **kwargs: Any) -> None:
         # When this exception is raised, we'll have already logged the relevant
         # traceback from a separate worker.
         raise click.ClickException(str(err)) from None
-
-
-@click.command(
-    short_help="Ingest a provenance quantum graph into a butler.",
-    cls=ButlerCommand,
-)
-@repo_argument(required=True, help="Path or alias for the butler repository.")
-@click.argument("provenance_graph", required=False)
-@transfer_option(default="move")
-@click.option("--batch-size", default=10000, help="How many datasets to process in each transaction.")
-@click.option(
-    "--output-run",
-    default=None,
-    help=(
-        "Name of the output RUN collection.  Must be provided if the provenance graph is not"
-        " provided (so the graph can be found in the butler)."
-    ),
-)
-def ingest_graph(
-    *,
-    repo: str,
-    provenance_graph: str | None,
-    transfer: str | None,
-    batch_size: int,
-    output_run: str | None,
-) -> None:
-    """Ingest a provenance graph into a butler repository."""
-    from ...quantum_graph.ingest_graph import ingest_graph as ingest_graph_py
-
-    ingest_graph_py(repo, provenance_graph, transfer=transfer, batch_size=batch_size, output_run=output_run)
-
-
-@click.command(
-    short_help="Print and write provenance reports.",
-    cls=ButlerCommand,
-)
-@click.argument("repo_or_qg")
-@click.argument("collection", required=False, default=None)
-@click.option(
-    "--state",
-    multiple=True,
-    type=click.Choice(QuantumAttemptStatus),
-    help=(
-        "Additional quantum state to include in the status report and data ID tables "
-        "(FAILED, ABORTED, and ABORTED_SUCCESS are included by default)."
-    ),
-)
-@click.option(
-    "--no-state",
-    multiple=True,
-    type=str,
-    metavar="STATE",
-    help="Quantum state to drop from in status report and data ID tables (same options as --state).",
-)
-@click.option(
-    "--status-report",
-    default=None,
-    metavar="URI",
-    help="File or URI (.json) for a detailed report (with data IDs) on quanta with certain states.",
-)
-@click.option(
-    "--quantum-table/--no-quantum-table",
-    default=True,
-    help="Whether to print summary of quantum status counts to STDOUT.",
-)
-@click.option(
-    "--exception-table/--no-exception-table",
-    default=True,
-    help="Whether to print summary of exception type counts STDOUT.",
-)
-@click.option(
-    "--caveat",
-    multiple=True,
-    type=click.Choice(QuantumSuccessCaveats),
-    help=(
-        "Include successful quanta in the status report if they have this caveat. "
-        "May be passed multiple times; any matching caveat is included. "
-        "Passing this option implicitly adds '--state SUCCESSFUL'."
-    ),
-)
-@click.option(
-    "--data-id-table-dir",
-    default=None,
-    metavar="URI",
-    help=(
-        "Directory (may be a URI) for a tree of data ID tables for each "
-        "task label, status, and exception type combination in the status report."
-    ),
-)
-def provenance_report(
-    *,
-    repo_or_qg: str,
-    collection: str | None,
-    state: Iterable[QuantumAttemptStatus],
-    no_state: Iterable[str],
-    status_report: str | None,
-    quantum_table: bool = False,
-    exception_table: bool = False,
-    caveat: Iterable[QuantumSuccessCaveats],
-    data_id_table_dir: str | None,
-) -> None:
-    """Read a provenance quantum graph from a butler or file and use it to
-    generate reports.
-
-    REPO_OR_QG is a path or alias for the butler repository (if reading an
-    ingested graph, as indicated by passing COLLECTION), or the path to a
-    provenance quantum graph file.
-    """
-    from ...quantum_graph import ProvenanceQuantumGraph
-
-    states = set(state)
-    states.add(QuantumAttemptStatus.FAILED)
-    states.add(QuantumAttemptStatus.ABORTED)
-    states.add(QuantumAttemptStatus.ABORTED_SUCCESS)
-    for state_name in no_state:
-        states.discard(QuantumAttemptStatus.__members__[state_name])
-    with_caveats: QuantumSuccessCaveats | None = None
-    if caveat:
-        states.add(QuantumAttemptStatus.SUCCESSFUL)
-        with_caveats = functools.reduce(
-            operator.__or__,
-            caveat,
-            QuantumSuccessCaveats.NO_CAVEATS,
-        )
-    with ProvenanceQuantumGraph.from_args(repo_or_qg, collection=collection, datasets=()) as (graph, _):
-        graph.make_many_reports(
-            status_report_file=status_report,
-            states=states,
-            print_quantum_table=quantum_table,
-            print_exception_table=exception_table,
-            with_caveats=with_caveats,
-            data_id_table_dir=data_id_table_dir,
-        )