lsst-ctrl-mpexec 29.2025.2400__py3-none-any.whl → 29.2025.3200__py3-none-any.whl

lsst/ctrl/mpexec/quantumGraphExecutor.py

@@ -25,100 +25,30 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from __future__ import annotations
+__all__ = ("QuantumExecutor", "QuantumGraphExecutor")
 
-__all__ = ["QuantumExecutor", "QuantumGraphExecutor"]
+from deprecated.sphinx import deprecated
 
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING
+import lsst.pipe.base.quantum_graph_executor
 
-from .reports import QuantumReport, Report
+# TODO[DM-51962]: Remove this module.
 
-if TYPE_CHECKING:
-    import uuid
 
-    from lsst.daf.butler import Quantum
-    from lsst.pipe.base import QuantumGraph
-    from lsst.pipe.base.pipeline_graph import TaskNode
+@deprecated(
+    "The QuantumExecutor class has moved to lsst.pipe.base.quantum_graph_executor. "
+    "This forwarding shim will be removed after v30.",
+    version="v30",
+    category=FutureWarning,
+)
+class QuantumExecutor(lsst.pipe.base.quantum_graph_executor.QuantumExecutor):  # noqa: D101
+    pass
 
 
-class QuantumExecutor(ABC):
-    """Class which abstracts execution of a single Quantum.
-
-    In general implementation should not depend on execution model and
-    execution should always happen in-process. Main reason for existence
-    of this class is to provide do-nothing implementation that can be used
-    in the unit tests.
-    """
-
-    @abstractmethod
-    def execute(
-        self, task_node: TaskNode, /, quantum: Quantum, quantum_id: uuid.UUID | None = None
-    ) -> tuple[Quantum, QuantumReport | None]:
-        """Execute single quantum.
-
-        Parameters
-        ----------
-        task_node : `~lsst.pipe.base.pipeline_graph.TaskNode`
-            Task definition structure.
-        quantum : `~lsst.daf.butler.Quantum`
-            Quantum for this execution.
-        quantum_id : `uuid.UUID` or `None`, optional
-            The ID of the quantum to be executed.
-
-        Returns
-        -------
-        quantum : `~lsst.daf.butler.Quantum`
-            The quantum actually executed.
-        report : `~lsst.ctrl.mpexec.QuantumReport`
-            Structure describing the status of the execution of a quantum.
-            `None` is returned if implementation does not support this
-            feature.
-
-        Notes
-        -----
-        Any exception raised by the task or code that wraps task execution is
-        propagated to the caller of this method.
-        """
-        raise NotImplementedError()
-
-
-class QuantumGraphExecutor(ABC):
-    """Class which abstracts QuantumGraph execution.
-
-    Any specific execution model is implemented in sub-class by overriding
-    the `execute` method.
-    """
-
-    @abstractmethod
-    def execute(self, graph: QuantumGraph) -> None:
-        """Execute whole graph.
-
-        Implementation of this method depends on particular execution model
-        and it has to be provided by a subclass. Execution model determines
-        what happens here; it can be either actual running of the task or,
-        for example, generation of the scripts for delayed batch execution.
-
-        Parameters
-        ----------
-        graph : `~lsst.pipe.base.QuantumGraph`
-            Execution graph.
-        """
-        raise NotImplementedError()
-
-    def getReport(self) -> Report | None:
-        """Return execution report from last call to `execute`.
-
-        Returns
-        -------
-        report : `~lsst.ctrl.mpexec.Report`, optional
-            Structure describing the status of the execution of a quantum
-            graph. `None` is returned if implementation does not support
-            this feature.
-
-        Raises
-        ------
-        RuntimeError
-            Raised if this method is called before `execute`.
-        """
-        return None
+@deprecated(
+    "The QuantumGraphExecutor class has moved to lsst.pipe.base.quantum_graph_executor. "
+    "This forwarding shim will be removed after v30.",
+    version="v30",
+    category=FutureWarning,
+)
+class QuantumGraphExecutor(lsst.pipe.base.quantum_graph_executor.QuantumGraphExecutor):  # noqa: D101
+    pass

lsst/ctrl/mpexec/reports.py

@@ -25,219 +25,43 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from __future__ import annotations
+__all__ = ("ExceptionInfo", "ExecutionStatus", "QuantumReport", "Report")
 
-__all__ = ["ExceptionInfo", "ExecutionStatus", "QuantumReport", "Report"]
+from deprecated.sphinx import deprecated
 
-import enum
-import sys
-from typing import Any
+import lsst.pipe.base.quantum_reports
 
-import pydantic
+# TODO[DM-51962]: Remove this module.
 
-from lsst.daf.butler import DataCoordinate, DataId, DataIdValue
-from lsst.pipe.base import QgraphSummary
-from lsst.utils.introspection import get_full_type_name
+# We can't make this shim warn because enums can't be subclassed.
+ExecutionStatus = lsst.pipe.base.quantum_reports.ExecutionStatus
 
 
-def _serializeDataId(dataId: DataId) -> dict[str, DataIdValue]:
-    if isinstance(dataId, DataCoordinate):
-        return dict(dataId.required)
-    else:
-        return dataId  # type: ignore
+@deprecated(
+    "The ExceptionInfo class has moved to lsst.pipe.base.quantum_reports. "
+    "This forwarding shim will be removed after v30.",
+    version="v30",
+    category=FutureWarning,
+)
+class ExceptionInfo(lsst.pipe.base.quantum_reports.ExceptionInfo):  # noqa: D101
+    pass
 
 
-class ExecutionStatus(enum.Enum):
-    """Possible values for job execution status.
+@deprecated(
+    "The QuantumReport class has moved to lsst.pipe.base.quantum_reports. "
+    "This forwarding shim will be removed after v30.",
+    version="v30",
+    category=FutureWarning,
+)
+class QuantumReport(lsst.pipe.base.quantum_reports.QuantumReport):  # noqa: D101
+    pass
 
-    Status `FAILURE` is set if one or more tasks failed. Status `TIMEOUT` is
-    set if there are no failures but one or more tasks timed out. Timeouts can
-    only be detected in multi-process mode, child task is killed on timeout
-    and usually should have non-zero exit code.
-    """
 
-    SUCCESS = "success"
-    FAILURE = "failure"
-    TIMEOUT = "timeout"
-    SKIPPED = "skipped"
-
-
-class ExceptionInfo(pydantic.BaseModel):
-    """Information about exception."""
-
-    className: str
-    """Name of the exception class if exception was raised."""
-
-    message: str
-    """Exception message for in-process quantum execution, None if
-    quantum was executed in sub-process.
-    """
-
-    @classmethod
-    def from_exception(cls, exception: Exception) -> ExceptionInfo:
-        """Construct instance from an exception.
-
-        Parameters
-        ----------
-        exception : `Exception`
-            Exception to wrap.
-
-        Returns
-        -------
-        info : `ExceptionInfo`
-            Information about the exception.
-        """
-        return cls(className=get_full_type_name(exception), message=str(exception))
-
-
-class QuantumReport(pydantic.BaseModel):
-    """Task execution report for a single Quantum.
-
-    Parameters
-    ----------
-    dataId : `~lsst.daf.butler.DataId`
-        Quantum data ID.
-    taskLabel : `str`
-        Label for task executing this Quantum.
-    status : `ExecutionStatus`
-        Status of this quantum execution.
-    exitCode : `int` or `None`, optional
-        Exit code for sub-process executing this Quantum. `None` for
-        in-process execution. Negative if process was killed by a signal.
-    exceptionInfo : `ExceptionInfo` or `None`, optional
-        Exception information if an exception was raised.
-    """
-
-    status: ExecutionStatus = ExecutionStatus.SUCCESS
-    """Execution status, one of the values in `ExecutionStatus` enum."""
-
-    dataId: dict[str, DataIdValue]
-    """Quantum DataId."""
-
-    taskLabel: str | None
-    """Label for a task executing this Quantum."""
-
-    exitCode: int | None = None
-    """Exit code for a sub-process executing Quantum, None for in-process
-    Quantum execution. Negative if process was killed by a signal.
-    """
-
-    exceptionInfo: ExceptionInfo | None = None
-    """Exception information if exception was raised."""
-
-    def __init__(
-        self,
-        dataId: DataId,
-        taskLabel: str,
-        status: ExecutionStatus = ExecutionStatus.SUCCESS,
-        exitCode: int | None = None,
-        exceptionInfo: ExceptionInfo | None = None,
-    ):
-        super().__init__(
-            status=status,
-            dataId=_serializeDataId(dataId),
-            taskLabel=taskLabel,
-            exitCode=exitCode,
-            exceptionInfo=exceptionInfo,
-        )
-
-    @classmethod
-    def from_exception(
-        cls,
-        exception: Exception,
-        dataId: DataId,
-        taskLabel: str,
-        *,
-        exitCode: int | None = None,
-    ) -> QuantumReport:
-        """Construct report instance from an exception and other pieces of
-        data.
-
-        Parameters
-        ----------
-        exception : `Exception`
-            Exception caught from processing quantum.
-        dataId : `~lsst.daf.butler.DataId`
-            Data ID of quantum.
-        taskLabel : `str`
-            Label of task.
-        exitCode : `int`, optional
-            Exit code for the process, used when it is known that the process
-            will exit with that exit code.
-        """
-        return cls(
-            status=ExecutionStatus.FAILURE,
-            dataId=dataId,
-            taskLabel=taskLabel,
-            exitCode=exitCode,
-            exceptionInfo=ExceptionInfo.from_exception(exception),
-        )
-
-    @classmethod
-    def from_exit_code(
-        cls,
-        exitCode: int,
-        dataId: DataId,
-        taskLabel: str,
-    ) -> QuantumReport:
-        """Construct report instance from an exit code and other pieces of
-        data.
-
-        Parameters
-        ----------
-        exitCode : `int`
-            The exit code of the subprocess.
-        dataId : `~lsst.daf.butler.DataId`
-            The quantum Data ID.
-        taskLabel : `str`
-            The task label.
-        """
-        return cls(
-            status=ExecutionStatus.SUCCESS if exitCode == 0 else ExecutionStatus.FAILURE,
-            dataId=dataId,
-            taskLabel=taskLabel,
-            exitCode=exitCode,
-        )
-
-
-class Report(pydantic.BaseModel):
-    """Execution report for the whole job with one or few quanta."""
-
-    qgraphSummary: QgraphSummary
-    """Summary report about QuantumGraph."""
-
-    status: ExecutionStatus = ExecutionStatus.SUCCESS
-    """Job status."""
-
-    cmdLine: list[str] | None = None
-    """Command line for the whole job."""
-
-    exitCode: int | None = None
-    """Job exit code, this obviously cannot be set in pipetask."""
-
-    exceptionInfo: ExceptionInfo | None = None
-    """Exception information if exception was raised."""
-
-    quantaReports: list[QuantumReport] = []
-    """List of per-quantum reports, ordering is not specified. Some or all
-    quanta may not produce a report.
-    """
-
-    # Always want to validate the default value for cmdLine so
-    # use a model_validator.
-    @pydantic.model_validator(mode="before")
-    @classmethod
-    def _set_cmdLine(cls, data: Any) -> Any:
-        if data.get("cmdLine") is None:
-            data["cmdLine"] = sys.argv
-        return data
-
-    def set_exception(self, exception: Exception) -> None:
-        """Update exception information from an exception object.
-
-        Parameters
-        ----------
-        exception : `Exception`
-            Exception to use to extract information from.
-        """
-        self.exceptionInfo = ExceptionInfo.from_exception(exception)
+@deprecated(
+    "The Report class has moved to lsst.pipe.base.quantum_reports. "
+    "This forwarding shim will be removed after v30.",
+    version="v30",
+    category=FutureWarning,
+)
+class Report(lsst.pipe.base.quantum_reports.Report):  # noqa: D101
+    pass

lsst/ctrl/mpexec/separablePipelineExecutor.py

@@ -25,271 +25,20 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+__all__ = ("SeparablePipelineExecutor",)
 
-from __future__ import annotations
+from deprecated.sphinx import deprecated
 
-__all__ = [
-    "SeparablePipelineExecutor",
-]
+import lsst.pipe.base.separable_pipeline_executor
 
+# TODO[DM-51962]: Remove this module.
 
-import datetime
-import getpass
-import logging
-from collections.abc import Iterable
-from typing import Any
 
-import lsst.pipe.base
-import lsst.resources
-from lsst.daf.butler import Butler
-from lsst.pipe.base.all_dimensions_quantum_graph_builder import AllDimensionsQuantumGraphBuilder
-from lsst.pipe.base.quantum_graph_builder import QuantumGraphBuilder
-
-from .mpGraphExecutor import MPGraphExecutor
-from .preExecInit import PreExecInit
-from .quantumGraphExecutor import QuantumGraphExecutor
-from .singleQuantumExecutor import SingleQuantumExecutor
-from .taskFactory import TaskFactory
-
-_LOG = logging.getLogger(__name__)
-
-
-class SeparablePipelineExecutor:
-    """An executor that allows each step of pipeline execution to be
-    run independently.
-
-    The executor can run any or all of the following steps:
-
-        * pre-execution initialization
-        * pipeline building
-        * quantum graph generation
-        * quantum graph execution
-
-    Any of these steps can also be handed off to external code without
-    compromising the remaining ones.
-
-    Parameters
-    ----------
-    butler : `lsst.daf.butler.Butler`
-        A Butler whose ``collections`` and ``run`` attributes contain the input
-        and output collections to use for processing.
-    clobber_output : `bool`, optional
-        If set, the pipeline execution overwrites existing output files.
-        Otherwise, any conflict between existing and new outputs is an error.
-    skip_existing_in : iterable [`str`], optional
-        If not empty, the pipeline execution searches the listed collections
-        for existing outputs, and skips any quanta that have run to completion
-        (or have no work to do). Otherwise, all tasks are attempted (subject to
-        ``clobber_output``).
-    task_factory : `lsst.pipe.base.TaskFactory`, optional
-        A custom task factory for use in pre-execution and execution. By
-        default, a new instance of `lsst.ctrl.mpexec.TaskFactory` is used.
-    resources : `~lsst.pipe.base.ExecutionResources`
-        The resources available to each quantum being executed.
-    raise_on_partial_outputs : `bool`, optional
-        If `True` raise exceptions chained by
-        `lsst.pipe.base.AnnotatedPartialOutputError` immediately, instead of
-        considering the partial result a success and continuing to run
-        downstream tasks.
-    """
-
-    def __init__(
-        self,
-        butler: Butler,
-        clobber_output: bool = False,
-        skip_existing_in: Iterable[str] | None = None,
-        task_factory: lsst.pipe.base.TaskFactory | None = None,
-        resources: lsst.pipe.base.ExecutionResources | None = None,
-        raise_on_partial_outputs: bool = True,
-    ):
-        self._butler = Butler.from_config(
-            butler=butler, collections=butler.collections.defaults, run=butler.run
-        )
-        if not self._butler.collections.defaults:
-            raise ValueError("Butler must specify input collections for pipeline.")
-        if not self._butler.run:
-            raise ValueError("Butler must specify output run for pipeline.")
-
-        self._clobber_output = clobber_output
-        self._skip_existing_in = list(skip_existing_in) if skip_existing_in else []
-
-        self._task_factory = task_factory if task_factory else TaskFactory()
-        self.resources = resources
-        self.raise_on_partial_outputs = raise_on_partial_outputs
-
-    def pre_execute_qgraph(
-        self,
-        graph: lsst.pipe.base.QuantumGraph,
-        register_dataset_types: bool = False,
-        save_init_outputs: bool = True,
-        save_versions: bool = True,
-    ) -> None:
-        """Run pre-execution initialization.
-
-        This method will be deprecated after DM-38041, to be replaced with a
-        method that takes either a `~lsst.pipe.base.Pipeline` or a
-        ``ResolvedPipelineGraph`` instead of a `~lsst.pipe.base.QuantumGraph`.
-
-        Parameters
-        ----------
-        graph : `lsst.pipe.base.QuantumGraph`
-            The quantum graph defining the pipeline and datasets to
-            be initialized.
-        register_dataset_types : `bool`, optional
-            If `True`, register all output dataset types from the pipeline
-            represented by ``graph``.
-        save_init_outputs : `bool`, optional
-            If `True`, create init-output datasets in this object's output run.
-        save_versions : `bool`, optional
-            If `True`, save a package versions dataset.
-        """
-        pre_exec_init = PreExecInit(self._butler, self._task_factory, extendRun=self._clobber_output)
-        pre_exec_init.initialize(
-            graph=graph,
-            saveInitOutputs=save_init_outputs,
-            registerDatasetTypes=register_dataset_types,
-            saveVersions=save_versions,
-        )
-
-    def make_pipeline(self, pipeline_uri: str | lsst.resources.ResourcePath) -> lsst.pipe.base.Pipeline:
-        """Build a pipeline from pipeline and configuration information.
-
-        Parameters
-        ----------
-        pipeline_uri : `str` or `lsst.resources.ResourcePath`
-            URI to a file containing a pipeline definition. A URI fragment may
-            be used to specify a subset of the pipeline, as described in
-            :ref:`pipeline-running-intro`.
-
-        Returns
-        -------
-        pipeline : `lsst.pipe.base.Pipeline`
-            The fully-built pipeline.
-        """
-        return lsst.pipe.base.Pipeline.from_uri(pipeline_uri)
-
-    def make_quantum_graph(
-        self,
-        pipeline: lsst.pipe.base.Pipeline,
-        where: str = "",
-        *,
-        builder_class: type[QuantumGraphBuilder] = AllDimensionsQuantumGraphBuilder,
-        attach_datastore_records: bool = False,
-        **kwargs: Any,
-    ) -> lsst.pipe.base.QuantumGraph:
-        """Build a quantum graph from a pipeline and input datasets.
-
-        Parameters
-        ----------
-        pipeline : `lsst.pipe.base.Pipeline`
-            The pipeline for which to generate a quantum graph.
-        where : `str`, optional
-            A data ID query that constrains the quanta generated.  Must not be
-            provided if a custom ``builder_class`` is given and that class does
-            not accept ``where`` as a construction argument.
-        builder_class : `type` [ \
-                `lsst.pipe.base.quantum_graph_builder.QuantumGraphBuilder` ], \
-                optional
-            Quantum graph builder implementation.  Ignored if ``builder`` is
-            provided.
-        attach_datastore_records : `bool`, optional
-            Whether to attach datastore records.  These are currently used only
-            by `lsst.daf.butler.QuantumBackedButler`, which is not used by
-            `SeparablePipelineExecutor` for execution.
-        **kwargs
-            Additional keyword arguments are forwarded to ``builder_class``
-            when a quantum graph builder instance is constructed.  All
-            arguments accepted by the
-            `~lsst.pipe.base.quantum_graph_builder.QuantumGraphBuilder` base
-            class are provided automatically (from explicit arguments to this
-            method and executor attributes) and do not need to be included
-            as keyword arguments.
-
-        Returns
-        -------
-        graph : `lsst.pipe.base.QuantumGraph`
-            The quantum graph for ``pipeline`` as run on the datasets
-            identified by ``where``.
-
-        Notes
-        -----
-        This method does no special handling of empty quantum graphs. If
-        needed, clients can use `len` to test if the returned graph is empty.
-        """
-        metadata = {
-            "input": self._butler.collections.defaults,
-            "output_run": self._butler.run,
-            "skip_existing_in": self._skip_existing_in,
-            "skip_existing": bool(self._skip_existing_in),
-            "data_query": where,
-            "user": getpass.getuser(),
-            "time": str(datetime.datetime.now()),
-        }
-        if where:
-            # Only pass 'where' if it's actually provided, since some
-            # QuantumGraphBuilder subclasses may not accept it.
-            kwargs["where"] = where
-        qg_builder = builder_class(
-            pipeline.to_graph(),
-            self._butler,
-            skip_existing_in=self._skip_existing_in,
-            clobber=self._clobber_output,
-            **kwargs,
-        )
-        graph = qg_builder.build(metadata=metadata, attach_datastore_records=attach_datastore_records)
-        _LOG.info(
-            "QuantumGraph contains %d quanta for %d tasks, graph ID: %r",
-            len(graph),
-            len(graph.taskGraph),
-            graph.graphID,
-        )
-        return graph
-
-    def run_pipeline(
-        self,
-        graph: lsst.pipe.base.QuantumGraph,
-        fail_fast: bool = False,
-        graph_executor: QuantumGraphExecutor | None = None,
-        num_proc: int = 1,
-    ) -> None:
-        """Run a pipeline in the form of a prepared quantum graph.
-
-        Pre-execution initialization must have already been run;
-        see `pre_execute_qgraph`.
-
-        Parameters
-        ----------
-        graph : `lsst.pipe.base.QuantumGraph`
-            The pipeline and datasets to execute.
-        fail_fast : `bool`, optional
-            If `True`, abort all execution if any task fails when
-            running with multiple processes. Only used with the default graph
-            executor).
-        graph_executor : `lsst.ctrl.mpexec.QuantumGraphExecutor`, optional
-            A custom graph executor. By default, a new instance of
-            `lsst.ctrl.mpexec.MPGraphExecutor` is used.
-        num_proc : `int`, optional
-            The number of processes that can be used to run the pipeline. The
-            default value ensures that no subprocess is created. Only used with
-            the default graph executor.
-        """
-        if not graph_executor:
-            quantum_executor = SingleQuantumExecutor(
-                self._butler,
-                self._task_factory,
-                skipExistingIn=self._skip_existing_in,
-                clobberOutputs=self._clobber_output,
-                resources=self.resources,
-                raise_on_partial_outputs=self.raise_on_partial_outputs,
-            )
-            graph_executor = MPGraphExecutor(
-                numProc=num_proc,
-                timeout=2_592_000.0,  # In practice, timeout is never helpful; set to 30 days.
-                quantumExecutor=quantum_executor,
-                failFast=fail_fast,
-            )
-            # Have to reset connection pool to avoid sharing connections with
-            # forked processes.
-            self._butler.registry.resetConnectionPool()
-
-        graph_executor.execute(graph)
+@deprecated(
+    "The SeparablePipelineExecutor class has moved to lsst.pipe.base.separable_pipeline_executor. "
+    "This forwarding shim will be removed after v30.",
+    version="v30",
+    category=FutureWarning,
+)
+class SeparablePipelineExecutor(lsst.pipe.base.separable_pipeline_executor.SeparablePipelineExecutor):  # noqa: D101
+    pass

lsst/ctrl/mpexec/showInfo.py

@@ -46,7 +46,7 @@ from lsst.pipe.base.pipeline_graph import visualization
 
 from . import util
 from ._pipeline_graph_factory import PipelineGraphFactory
-from .cmdLineFwk import _ButlerFactory
+from .cli.butler_factory import ButlerFactory
 
 
 class _FilteredStream:
@@ -385,7 +385,7 @@ class ShowInfo:
                 for compName, compUri in components.items():
                     print(f"        {compName}: {compUri}", file=self.stream)
 
-        butler = _ButlerFactory.makeReadButler(args)
+        butler = ButlerFactory.make_read_butler(args)
         for node in graph:
             print(f"Quantum {node.nodeId}: {node.taskDef.taskName}", file=self.stream)
             print("  inputs:", file=self.stream)