lsst-ctrl-mpexec 29.2025.2400__py3-none-any.whl → 29.2025.3100__py3-none-any.whl

lsst/ctrl/mpexec/__init__.py

@@ -27,14 +27,13 @@
 
 from ._pipeline_graph_factory import PipelineGraphFactory
 from .cmdLineFwk import *
-from .dotTools import *
 from .executionGraphFixup import *
 from .mpGraphExecutor import *
-from .preExecInit import *
 from .quantumGraphExecutor import *
 from .reports import *
 from .separablePipelineExecutor import *
 from .simple_pipeline_executor import *
 from .singleQuantumExecutor import *
+from .preExecInit import *
 from .taskFactory import *
 from .version import *  # Generated by sconsUtils

lsst/ctrl/mpexec/cli/cmd/commands.py

@@ -36,7 +36,6 @@ from typing import Any
 import click
 
 import lsst.pipe.base.cli.opt as pipeBaseOpts
-from lsst.ctrl.mpexec import Report
 from lsst.ctrl.mpexec.showInfo import ShowInfo
 from lsst.daf.butler.cli.opt import (
     collections_option,
@@ -49,6 +48,7 @@ from lsst.daf.butler.cli.opt import (
     where_option,
 )
 from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap
+from lsst.pipe.base.quantum_reports import Report
 
 from .. import opt as ctrlMpExecOpts
 from .. import script

lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py

@@ -27,7 +27,9 @@
 
 from types import SimpleNamespace
 
-from ... import CmdLineFwk, TaskFactory
+from lsst.pipe.base import TaskFactory
+
+from ... import CmdLineFwk
 
 
 def pre_exec_init_qbb(

lsst/ctrl/mpexec/cli/script/run.py

@@ -28,9 +28,10 @@
 import logging
 from types import SimpleNamespace
 
+from lsst.pipe.base import TaskFactory
 from lsst.utils.threads import disable_implicit_threading
 
-from ... import CmdLineFwk, TaskFactory
+from ... import CmdLineFwk
 
 _log = logging.getLogger(__name__)
 

lsst/ctrl/mpexec/cli/script/run_qbb.py

@@ -28,9 +28,10 @@
 import logging
 from types import SimpleNamespace
 
+from lsst.pipe.base import TaskFactory
 from lsst.utils.threads import disable_implicit_threading
 
-from ... import CmdLineFwk, TaskFactory
+from ... import CmdLineFwk
 
 _log = logging.getLogger(__name__)
 

lsst/ctrl/mpexec/cmdLineFwk.py

@@ -71,19 +71,19 @@ from lsst.pipe.base import (
 )
 from lsst.pipe.base.all_dimensions_quantum_graph_builder import AllDimensionsQuantumGraphBuilder
 from lsst.pipe.base.dot_tools import graph2dot
+from lsst.pipe.base.execution_graph_fixup import ExecutionGraphFixup
 from lsst.pipe.base.mermaid_tools import graph2mermaid
+from lsst.pipe.base.mp_graph_executor import MPGraphExecutor
 from lsst.pipe.base.pipeline_graph import NodeType
+from lsst.pipe.base.quantum_reports import Report
+from lsst.pipe.base.single_quantum_executor import SingleQuantumExecutor
 from lsst.resources import ResourcePath
 from lsst.utils import doImportType
 from lsst.utils.logging import VERBOSE, getLogger
 from lsst.utils.threads import disable_implicit_threading
 
 from ._pipeline_graph_factory import PipelineGraphFactory
-from .executionGraphFixup import ExecutionGraphFixup
-from .mpGraphExecutor import MPGraphExecutor
 from .preExecInit import PreExecInit, PreExecInitLimited
-from .reports import Report
-from .singleQuantumExecutor import SingleQuantumExecutor
 
 # ----------------------------------
 #  Local non-exported definitions --
@@ -859,24 +859,24 @@ class CmdLineFwk:
             graphFixup = self._importGraphFixup(args)
             resources = self._make_execution_resources(args)
             quantumExecutor = SingleQuantumExecutor(
-                butler,
-                taskFactory,
-                skipExistingIn=args.skip_existing_in,
-                clobberOutputs=args.clobber_outputs,
-                enableLsstDebug=args.enableLsstDebug,
+                butler=butler,
+                task_factory=taskFactory,
+                skip_existing_in=args.skip_existing_in,
+                clobber_outputs=args.clobber_outputs,
+                enable_lsst_debug=args.enableLsstDebug,
                 resources=resources,
                 raise_on_partial_outputs=args.raise_on_partial_outputs,
             )
 
             timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout
             executor = MPGraphExecutor(
-                numProc=args.processes,
+                num_proc=args.processes,
                 timeout=timeout,
-                startMethod=args.start_method,
-                quantumExecutor=quantumExecutor,
-                failFast=args.fail_fast,
+                start_method=args.start_method,
+                quantum_executor=quantumExecutor,
+                fail_fast=args.fail_fast,
                 pdb=args.pdb,
-                executionGraphFixup=graphFixup,
+                execution_graph_fixup=graphFixup,
             )
             # Have to reset connection pool to avoid sharing connections with
             # forked processes.
@@ -1028,24 +1028,24 @@ class CmdLineFwk:
         resources = self._make_execution_resources(args)
         quantumExecutor = SingleQuantumExecutor(
             butler=None,
-            taskFactory=task_factory,
-            enableLsstDebug=args.enableLsstDebug,
+            task_factory=task_factory,
+            enable_lsst_debug=args.enableLsstDebug,
             limited_butler_factory=_butler_factory,
             resources=resources,
-            assumeNoExistingOutputs=args.no_existing_outputs,
-            skipExisting=True,
-            clobberOutputs=True,
+            assume_no_existing_outputs=args.no_existing_outputs,
+            skip_existing=True,
+            clobber_outputs=True,
             raise_on_partial_outputs=args.raise_on_partial_outputs,
             job_metadata=job_metadata,
         )
 
         timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout
         executor = MPGraphExecutor(
-            numProc=args.processes,
+            num_proc=args.processes,
             timeout=timeout,
-            startMethod=args.start_method,
-            quantumExecutor=quantumExecutor,
-            failFast=args.fail_fast,
+            start_method=args.start_method,
+            quantum_executor=quantumExecutor,
+            fail_fast=args.fail_fast,
             pdb=args.pdb,
         )
         try:

lsst/ctrl/mpexec/execFixupDataId.py

@@ -25,108 +25,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-__all__ = ["ExecutionGraphFixup"]
+__all__ = ("ExecFixupDataId",)
 
-import contextlib
-import itertools
-from collections import defaultdict
-from collections.abc import Sequence
-from typing import Any
+import warnings
 
-import networkx as nx
+from lsst.pipe.base.exec_fixup_data_id import ExecFixupDataId
 
-from lsst.pipe.base import QuantumGraph, QuantumNode
+# TODO[DM-51962]: Remove this module.
 
-from .executionGraphFixup import ExecutionGraphFixup
-
-
-class ExecFixupDataId(ExecutionGraphFixup):
-    """Implementation of ExecutionGraphFixup for ordering of tasks based
-    on DataId values.
-
-    This class is a trivial implementation mostly useful as an example,
-    though it can be used to make actual fixup instances by defining
-    a method that instantiates it, e.g.::
-
-        # lsst/ap/verify/ci_fixup.py
-
-        from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
-
-
-        def assoc_fixup():
-            return ExecFixupDataId(
-                taskLabel="ap_assoc", dimensions=("visit", "detector")
-            )
-
-    and then executing pipetask::
-
-        pipetask run --graph-fixup=lsst.ap.verify.ci_fixup.assoc_fixup ...
-
-    This will add new dependencies between quanta executed by the task with
-    label "ap_assoc". Quanta with higher visit number will depend on quanta
-    with lower visit number and their execution will wait until lower visit
-    number finishes.
-
-    Parameters
-    ----------
-    taskLabel : `str`
-        The label of the task for which to add dependencies.
-    dimensions : `str` or sequence [`str`]
-        One or more dimension names, quanta execution will be ordered
-        according to values of these dimensions.
-    reverse : `bool`, optional
-        If `False` (default) then quanta with higher values of dimensions
-        will be executed after quanta with lower values, otherwise the order
-        is reversed.
-    """
-
-    def __init__(self, taskLabel: str, dimensions: str | Sequence[str], reverse: bool = False):
-        self.taskLabel = taskLabel
-        self.dimensions = dimensions
-        self.reverse = reverse
-        if isinstance(self.dimensions, str):
-            self.dimensions = (self.dimensions,)
-        else:
-            self.dimensions = tuple(self.dimensions)
-
-    def _key(self, qnode: QuantumNode) -> tuple[Any, ...]:
-        """Produce comparison key for quantum data.
-
-        Parameters
-        ----------
-        qnode : `QuantumNode`
-            An individual node in a `~lsst.pipe.base.QuantumGraph`
-
-        Returns
-        -------
-        key : `tuple`
-        """
-        dataId = qnode.quantum.dataId
-        assert dataId is not None, "Quantum DataId cannot be None"
-        key = tuple(dataId[dim] for dim in self.dimensions)
-        return key
-
-    def fixupQuanta(self, graph: QuantumGraph) -> QuantumGraph:
-        taskDef = graph.findTaskDefByLabel(self.taskLabel)
-        if taskDef is None:
-            raise ValueError(f"Cannot find task with label {self.taskLabel}")
-        quanta = list(graph.getNodesForTask(taskDef))
-        keyQuanta = defaultdict(list)
-        for q in quanta:
-            key = self._key(q)
-            keyQuanta[key].append(q)
-        keys = sorted(keyQuanta.keys(), reverse=self.reverse)
-        networkGraph = graph.graph
-
-        for prev_key, key in itertools.pairwise(keys):
-            for prev_node in keyQuanta[prev_key]:
-                for node in keyQuanta[key]:
-                    # remove any existing edges between the two nodes, but
-                    # don't fail if there are not any. Both directions need
-                    # tried because in a directed graph, order maters
-                    for edge in ((node, prev_node), (prev_node, node)):
-                        with contextlib.suppress(nx.NetworkXException):
-                            networkGraph.remove_edge(*edge)
-
-                    networkGraph.add_edge(prev_node, node)
-        return graph
+warnings.warn(
+    "The execFixupDataId module has moved to lsst.pipe.base.exec_fixup_data_id. "
+    "This forwarding shim will be removed after v30.",
+    category=FutureWarning,
+)

lsst/ctrl/mpexec/executionGraphFixup.py

@@ -25,45 +25,20 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-__all__ = ["ExecutionGraphFixup"]
+__all__ = ("ExecutionGraphFixup",)
 
-from abc import ABC, abstractmethod
+from deprecated.sphinx import deprecated
 
-from lsst.pipe.base import QuantumGraph
+import lsst.pipe.base.execution_graph_fixup
 
+# TODO[DM-51962]: Remove this module.
 
-class ExecutionGraphFixup(ABC):
-    """Interface for classes which update quantum graphs before execution.
 
-    Primary goal of this class is to modify quanta dependencies which may
-    not be possible to reflect in a quantum graph using standard tools.
-    One known use case for that is to guarantee particular execution order
-    of visits in CI jobs for cases when outcome depends on the processing
-    order of visits (e.g. AP association pipeline).
-
-    Instances of this class receive pre-ordered sequence of quanta
-    (`~lsst.pipe.base.QuantumGraph` instances) and they are allowed to
-    modify quanta data in place, for example update ``dependencies`` field to
-    add additional dependencies. Returned list of quanta will be re-ordered
-    once again by the graph executor to reflect new dependencies.
-    """
-
-    @abstractmethod
-    def fixupQuanta(self, graph: QuantumGraph) -> QuantumGraph:
-        """Update quanta in a graph.
-
-        Potentially anything in the graph could be changed if it does not
-        break executor assumptions. If modifications result in a dependency
-        cycle the executor will raise an exception.
-
-        Parameters
-        ----------
-        graph : QuantumGraph
-            Quantum Graph that will be executed by the executor.
-
-        Returns
-        -------
-        graph : QuantumGraph
-            Modified graph.
-        """
-        raise NotImplementedError
+@deprecated(
+    "The ExecutionGraphFixup class has moved to lsst.pipe.base.execution_graph_fixup. "
+    "This forwarding shim will be removed after v30.",
+    version="v30",
+    category=FutureWarning,
+)
+class ExecutionGraphFixup(lsst.pipe.base.execution_graph_fixup.ExecutionGraphFixup):  # noqa: D101
+    pass

lsst/ctrl/mpexec/log_capture.py

@@ -25,202 +25,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from __future__ import annotations
+__all__ = ("LogCapture",)
 
-__all__ = ["LogCapture"]
+import warnings
 
-import logging
-import os
-import shutil
-import tempfile
-from collections.abc import Iterator
-from contextlib import contextmanager, suppress
-from logging import FileHandler
+from lsst.pipe.base.log_capture import LogCapture
 
-from lsst.daf.butler import Butler, FileDataset, LimitedButler, Quantum
-from lsst.daf.butler.logging import ButlerLogRecordHandler, ButlerLogRecords, ButlerMDC, JsonLogFormatter
-from lsst.pipe.base import InvalidQuantumError
-from lsst.pipe.base.pipeline_graph import TaskNode
+# TODO[DM-51962]: Remove this module.
 
-_LOG = logging.getLogger(__name__)
-
-
-class _LogCaptureFlag:
-    """Simple flag to enable/disable log-to-butler saving."""
-
-    store: bool = True
-
-
-class LogCapture:
-    """Class handling capture of logging messages and their export to butler.
-
-    Parameters
-    ----------
-    butler : `~lsst.daf.butler.LimitedButler`
-        Data butler with limited API.
-    full_butler : `~lsst.daf.butler.Butler` or `None`
-        Data butler with full API, or `None` if full Butler is not available.
-        If not none, then this must be the same instance as ``butler``.
-    """
-
-    stream_json_logs = True
-    """If True each log record is written to a temporary file and ingested
-    when quantum completes. If False the records are accumulated in memory
-    and stored in butler on quantum completion. If full butler is not available
-    then temporary file is not used."""
-
-    def __init__(
-        self,
-        butler: LimitedButler,
-        full_butler: Butler | None,
-    ):
-        self.butler = butler
-        self.full_butler = full_butler
-
-    @classmethod
-    def from_limited(cls, butler: LimitedButler) -> LogCapture:
-        return cls(butler, None)
-
-    @classmethod
-    def from_full(cls, butler: Butler) -> LogCapture:
-        return cls(butler, butler)
-
-    @contextmanager
-    def capture_logging(self, task_node: TaskNode, /, quantum: Quantum) -> Iterator[_LogCaptureFlag]:
-        """Configure logging system to capture logs for execution of this task.
-
-        Parameters
-        ----------
-        task_node : `~lsst.pipe.base.pipeline_graph.TaskNode`
-            The task definition.
-        quantum : `~lsst.daf.butler.Quantum`
-            Single Quantum instance.
-
-        Notes
-        -----
-        Expected to be used as a context manager to ensure that logging
-        records are inserted into the butler once the quantum has been
-        executed:
-
-        .. code-block:: py
-
-           with self.capture_logging(task_node, quantum):
-               # Run quantum and capture logs.
-
-        Ths method can also setup logging to attach task- or
-        quantum-specific information to log messages. Potentially this can
-        take into account some info from task configuration as well.
-        """
-        # include quantum dataId and task label into MDC
-        mdc = {"LABEL": task_node.label, "RUN": ""}
-        if quantum.dataId:
-            mdc["LABEL"] += f":{quantum.dataId}"
-        if self.full_butler is not None:
-            mdc["RUN"] = self.full_butler.run or ""
-        ctx = _LogCaptureFlag()
-        log_dataset_name = (
-            task_node.log_output.dataset_type_name if task_node.log_output is not None else None
-        )
-
-        # Add a handler to the root logger to capture execution log output.
-        if log_dataset_name is not None:
-            # Either accumulate into ButlerLogRecords or stream JSON records to
-            # file and ingest that (ingest is possible only with full butler).
-            if self.stream_json_logs and self.full_butler is not None:
-                # Create the log file in a temporary directory rather than
-                # creating a temporary file. This is necessary because
-                # temporary files are created with restrictive permissions
-                # and during file ingest these permissions persist in the
-                # datastore. Using a temp directory allows us to create
-                # a file with umask default permissions.
-                tmpdir = tempfile.mkdtemp(prefix="butler-temp-logs-")
-
-                # Construct a file to receive the log records and "touch" it.
-                log_file = os.path.join(tmpdir, f"butler-log-{task_node.label}.json")
-                with open(log_file, "w"):
-                    pass
-                log_handler_file = FileHandler(log_file)
-                log_handler_file.setFormatter(JsonLogFormatter())
-                logging.getLogger().addHandler(log_handler_file)
-
-                try:
-                    with ButlerMDC.set_mdc(mdc):
-                        yield ctx
-                finally:
-                    # Ensure that the logs are stored in butler.
-                    logging.getLogger().removeHandler(log_handler_file)
-                    log_handler_file.close()
-                    if ctx.store:
-                        self._ingest_log_records(quantum, log_dataset_name, log_file)
-                    shutil.rmtree(tmpdir, ignore_errors=True)
-
-            else:
-                log_handler_memory = ButlerLogRecordHandler()
-                logging.getLogger().addHandler(log_handler_memory)
-
-                try:
-                    with ButlerMDC.set_mdc(mdc):
-                        yield ctx
-                finally:
-                    # Ensure that the logs are stored in butler.
-                    logging.getLogger().removeHandler(log_handler_memory)
-                    if ctx.store:
-                        self._store_log_records(quantum, log_dataset_name, log_handler_memory)
-                    log_handler_memory.records.clear()
-
-        else:
-            with ButlerMDC.set_mdc(mdc):
-                yield ctx
-
-    def _store_log_records(
-        self, quantum: Quantum, dataset_type: str, log_handler: ButlerLogRecordHandler
-    ) -> None:
-        # DatasetRef has to be in the Quantum outputs, can lookup by name.
-        try:
-            [ref] = quantum.outputs[dataset_type]
-        except LookupError as exc:
-            raise InvalidQuantumError(
-                f"Quantum outputs is missing log output dataset type {dataset_type};"
-                " this could happen due to inconsistent options between QuantumGraph generation"
-                " and execution"
-            ) from exc
-
-        self.butler.put(log_handler.records, ref)
-
-    def _ingest_log_records(self, quantum: Quantum, dataset_type: str, filename: str) -> None:
-        # If we are logging to an external file we must always try to
-        # close it.
-        assert self.full_butler is not None, "Expected to have full butler for ingest"
-        ingested = False
-        try:
-            # DatasetRef has to be in the Quantum outputs, can lookup by name.
-            try:
-                [ref] = quantum.outputs[dataset_type]
-            except LookupError as exc:
-                raise InvalidQuantumError(
-                    f"Quantum outputs is missing log output dataset type {dataset_type};"
-                    " this could happen due to inconsistent options between QuantumGraph generation"
-                    " and execution"
-                ) from exc
-
-            # Need to ingest this file directly into butler.
-            dataset = FileDataset(path=filename, refs=ref)
-            try:
-                self.full_butler.ingest(dataset, transfer="move")
-                ingested = True
-            except NotImplementedError:
-                # Some datastores can't receive files (e.g. in-memory datastore
-                # when testing), we store empty list for those just to have a
-                # dataset. Alternative is to read the file as a
-                # ButlerLogRecords object and put it.
-                _LOG.info(
-                    "Log records could not be stored in this butler because the"
-                    " datastore can not ingest files, empty record list is stored instead."
-                )
-                records = ButlerLogRecords.from_records([])
-                self.full_butler.put(records, ref)
-        finally:
-            # remove file if it is not ingested
-            if not ingested:
-                with suppress(OSError):
-                    os.remove(filename)
+warnings.warn(
+    "The log_capture module has moved to lsst.pipe.base.log_capture. "
+    "This forwarding shim will be removed after v30.",
+    category=FutureWarning,
+)