lsst-ctrl-mpexec 29.2025.2400__py3-none-any.whl → 29.2025.3200__py3-none-any.whl

lsst/ctrl/mpexec/execFixupDataId.py

@@ -25,108 +25,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-__all__ = ["ExecutionGraphFixup"]
+__all__ = ("ExecFixupDataId",)
 
-import contextlib
-import itertools
-from collections import defaultdict
-from collections.abc import Sequence
-from typing import Any
+import warnings
 
-import networkx as nx
+from lsst.pipe.base.exec_fixup_data_id import ExecFixupDataId
 
-from lsst.pipe.base import QuantumGraph, QuantumNode
+# TODO[DM-51962]: Remove this module.
 
-from .executionGraphFixup import ExecutionGraphFixup
-
-
-class ExecFixupDataId(ExecutionGraphFixup):
-    """Implementation of ExecutionGraphFixup for ordering of tasks based
-    on DataId values.
-
-    This class is a trivial implementation mostly useful as an example,
-    though it can be used to make actual fixup instances by defining
-    a method that instantiates it, e.g.::
-
-        # lsst/ap/verify/ci_fixup.py
-
-        from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
-
-
-        def assoc_fixup():
-            return ExecFixupDataId(
-                taskLabel="ap_assoc", dimensions=("visit", "detector")
-            )
-
-    and then executing pipetask::
-
-        pipetask run --graph-fixup=lsst.ap.verify.ci_fixup.assoc_fixup ...
-
-    This will add new dependencies between quanta executed by the task with
-    label "ap_assoc". Quanta with higher visit number will depend on quanta
-    with lower visit number and their execution will wait until lower visit
-    number finishes.
-
-    Parameters
-    ----------
-    taskLabel : `str`
-        The label of the task for which to add dependencies.
-    dimensions : `str` or sequence [`str`]
-        One or more dimension names, quanta execution will be ordered
-        according to values of these dimensions.
-    reverse : `bool`, optional
-        If `False` (default) then quanta with higher values of dimensions
-        will be executed after quanta with lower values, otherwise the order
-        is reversed.
-    """
-
-    def __init__(self, taskLabel: str, dimensions: str | Sequence[str], reverse: bool = False):
-        self.taskLabel = taskLabel
-        self.dimensions = dimensions
-        self.reverse = reverse
-        if isinstance(self.dimensions, str):
-            self.dimensions = (self.dimensions,)
-        else:
-            self.dimensions = tuple(self.dimensions)
-
-    def _key(self, qnode: QuantumNode) -> tuple[Any, ...]:
-        """Produce comparison key for quantum data.
-
-        Parameters
-        ----------
-        qnode : `QuantumNode`
-            An individual node in a `~lsst.pipe.base.QuantumGraph`
-
-        Returns
-        -------
-        key : `tuple`
-        """
-        dataId = qnode.quantum.dataId
-        assert dataId is not None, "Quantum DataId cannot be None"
-        key = tuple(dataId[dim] for dim in self.dimensions)
-        return key
-
-    def fixupQuanta(self, graph: QuantumGraph) -> QuantumGraph:
-        taskDef = graph.findTaskDefByLabel(self.taskLabel)
-        if taskDef is None:
-            raise ValueError(f"Cannot find task with label {self.taskLabel}")
-        quanta = list(graph.getNodesForTask(taskDef))
-        keyQuanta = defaultdict(list)
-        for q in quanta:
-            key = self._key(q)
-            keyQuanta[key].append(q)
-        keys = sorted(keyQuanta.keys(), reverse=self.reverse)
-        networkGraph = graph.graph
-
-        for prev_key, key in itertools.pairwise(keys):
-            for prev_node in keyQuanta[prev_key]:
-                for node in keyQuanta[key]:
-                    # remove any existing edges between the two nodes, but
-                    # don't fail if there are not any. Both directions need
-                    # tried because in a directed graph, order maters
-                    for edge in ((node, prev_node), (prev_node, node)):
-                        with contextlib.suppress(nx.NetworkXException):
-                            networkGraph.remove_edge(*edge)
-
-                    networkGraph.add_edge(prev_node, node)
-        return graph
+warnings.warn(
+    "The execFixupDataId module has moved to lsst.pipe.base.exec_fixup_data_id. "
+    "This forwarding shim will be removed after v30.",
+    category=FutureWarning,
+)

lsst/ctrl/mpexec/executionGraphFixup.py

@@ -25,45 +25,20 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-__all__ = ["ExecutionGraphFixup"]
+__all__ = ("ExecutionGraphFixup",)
 
-from abc import ABC, abstractmethod
+from deprecated.sphinx import deprecated
 
-from lsst.pipe.base import QuantumGraph
+import lsst.pipe.base.execution_graph_fixup
 
+# TODO[DM-51962]: Remove this module.
 
-class ExecutionGraphFixup(ABC):
-    """Interface for classes which update quantum graphs before execution.
 
-    Primary goal of this class is to modify quanta dependencies which may
-    not be possible to reflect in a quantum graph using standard tools.
-    One known use case for that is to guarantee particular execution order
-    of visits in CI jobs for cases when outcome depends on the processing
-    order of visits (e.g. AP association pipeline).
-
-    Instances of this class receive pre-ordered sequence of quanta
-    (`~lsst.pipe.base.QuantumGraph` instances) and they are allowed to
-    modify quanta data in place, for example update ``dependencies`` field to
-    add additional dependencies. Returned list of quanta will be re-ordered
-    once again by the graph executor to reflect new dependencies.
-    """
-
-    @abstractmethod
-    def fixupQuanta(self, graph: QuantumGraph) -> QuantumGraph:
-        """Update quanta in a graph.
-
-        Potentially anything in the graph could be changed if it does not
-        break executor assumptions. If modifications result in a dependency
-        cycle the executor will raise an exception.
-
-        Parameters
-        ----------
-        graph : QuantumGraph
-            Quantum Graph that will be executed by the executor.
-
-        Returns
-        -------
-        graph : QuantumGraph
-            Modified graph.
-        """
-        raise NotImplementedError
+@deprecated(
+    "The ExecutionGraphFixup class has moved to lsst.pipe.base.execution_graph_fixup. "
+    "This forwarding shim will be removed after v30.",
+    version="v30",
+    category=FutureWarning,
+)
+class ExecutionGraphFixup(lsst.pipe.base.execution_graph_fixup.ExecutionGraphFixup):  # noqa: D101
+    pass

lsst/ctrl/mpexec/log_capture.py

@@ -25,202 +25,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from __future__ import annotations
+__all__ = ("LogCapture",)
 
-__all__ = ["LogCapture"]
+import warnings
 
-import logging
-import os
-import shutil
-import tempfile
-from collections.abc import Iterator
-from contextlib import contextmanager, suppress
-from logging import FileHandler
+from lsst.pipe.base.log_capture import LogCapture
 
-from lsst.daf.butler import Butler, FileDataset, LimitedButler, Quantum
-from lsst.daf.butler.logging import ButlerLogRecordHandler, ButlerLogRecords, ButlerMDC, JsonLogFormatter
-from lsst.pipe.base import InvalidQuantumError
-from lsst.pipe.base.pipeline_graph import TaskNode
+# TODO[DM-51962]: Remove this module.
 
-_LOG = logging.getLogger(__name__)
-
-
-class _LogCaptureFlag:
-    """Simple flag to enable/disable log-to-butler saving."""
-
-    store: bool = True
-
-
-class LogCapture:
-    """Class handling capture of logging messages and their export to butler.
-
-    Parameters
-    ----------
-    butler : `~lsst.daf.butler.LimitedButler`
-        Data butler with limited API.
-    full_butler : `~lsst.daf.butler.Butler` or `None`
-        Data butler with full API, or `None` if full Butler is not available.
-        If not none, then this must be the same instance as ``butler``.
-    """
-
-    stream_json_logs = True
-    """If True each log record is written to a temporary file and ingested
-    when quantum completes. If False the records are accumulated in memory
-    and stored in butler on quantum completion. If full butler is not available
-    then temporary file is not used."""
-
-    def __init__(
-        self,
-        butler: LimitedButler,
-        full_butler: Butler | None,
-    ):
-        self.butler = butler
-        self.full_butler = full_butler
-
-    @classmethod
-    def from_limited(cls, butler: LimitedButler) -> LogCapture:
-        return cls(butler, None)
-
-    @classmethod
-    def from_full(cls, butler: Butler) -> LogCapture:
-        return cls(butler, butler)
-
-    @contextmanager
-    def capture_logging(self, task_node: TaskNode, /, quantum: Quantum) -> Iterator[_LogCaptureFlag]:
-        """Configure logging system to capture logs for execution of this task.
-
-        Parameters
-        ----------
-        task_node : `~lsst.pipe.base.pipeline_graph.TaskNode`
-            The task definition.
-        quantum : `~lsst.daf.butler.Quantum`
-            Single Quantum instance.
-
-        Notes
-        -----
-        Expected to be used as a context manager to ensure that logging
-        records are inserted into the butler once the quantum has been
-        executed:
-
-        .. code-block:: py
-
-           with self.capture_logging(task_node, quantum):
-               # Run quantum and capture logs.
-
-        Ths method can also setup logging to attach task- or
-        quantum-specific information to log messages. Potentially this can
-        take into account some info from task configuration as well.
-        """
-        # include quantum dataId and task label into MDC
-        mdc = {"LABEL": task_node.label, "RUN": ""}
-        if quantum.dataId:
-            mdc["LABEL"] += f":{quantum.dataId}"
-        if self.full_butler is not None:
-            mdc["RUN"] = self.full_butler.run or ""
-        ctx = _LogCaptureFlag()
-        log_dataset_name = (
-            task_node.log_output.dataset_type_name if task_node.log_output is not None else None
-        )
-
-        # Add a handler to the root logger to capture execution log output.
-        if log_dataset_name is not None:
-            # Either accumulate into ButlerLogRecords or stream JSON records to
-            # file and ingest that (ingest is possible only with full butler).
-            if self.stream_json_logs and self.full_butler is not None:
-                # Create the log file in a temporary directory rather than
-                # creating a temporary file. This is necessary because
-                # temporary files are created with restrictive permissions
-                # and during file ingest these permissions persist in the
-                # datastore. Using a temp directory allows us to create
-                # a file with umask default permissions.
-                tmpdir = tempfile.mkdtemp(prefix="butler-temp-logs-")
-
-                # Construct a file to receive the log records and "touch" it.
-                log_file = os.path.join(tmpdir, f"butler-log-{task_node.label}.json")
-                with open(log_file, "w"):
-                    pass
-                log_handler_file = FileHandler(log_file)
-                log_handler_file.setFormatter(JsonLogFormatter())
-                logging.getLogger().addHandler(log_handler_file)
-
-                try:
-                    with ButlerMDC.set_mdc(mdc):
-                        yield ctx
-                finally:
-                    # Ensure that the logs are stored in butler.
-                    logging.getLogger().removeHandler(log_handler_file)
-                    log_handler_file.close()
-                    if ctx.store:
-                        self._ingest_log_records(quantum, log_dataset_name, log_file)
-                    shutil.rmtree(tmpdir, ignore_errors=True)
-
-            else:
-                log_handler_memory = ButlerLogRecordHandler()
-                logging.getLogger().addHandler(log_handler_memory)
-
-                try:
-                    with ButlerMDC.set_mdc(mdc):
-                        yield ctx
-                finally:
-                    # Ensure that the logs are stored in butler.
-                    logging.getLogger().removeHandler(log_handler_memory)
-                    if ctx.store:
-                        self._store_log_records(quantum, log_dataset_name, log_handler_memory)
-                    log_handler_memory.records.clear()
-
-        else:
-            with ButlerMDC.set_mdc(mdc):
-                yield ctx
-
-    def _store_log_records(
-        self, quantum: Quantum, dataset_type: str, log_handler: ButlerLogRecordHandler
-    ) -> None:
-        # DatasetRef has to be in the Quantum outputs, can lookup by name.
-        try:
-            [ref] = quantum.outputs[dataset_type]
-        except LookupError as exc:
-            raise InvalidQuantumError(
-                f"Quantum outputs is missing log output dataset type {dataset_type};"
-                " this could happen due to inconsistent options between QuantumGraph generation"
-                " and execution"
-            ) from exc
-
-        self.butler.put(log_handler.records, ref)
-
-    def _ingest_log_records(self, quantum: Quantum, dataset_type: str, filename: str) -> None:
-        # If we are logging to an external file we must always try to
-        # close it.
-        assert self.full_butler is not None, "Expected to have full butler for ingest"
-        ingested = False
-        try:
-            # DatasetRef has to be in the Quantum outputs, can lookup by name.
-            try:
-                [ref] = quantum.outputs[dataset_type]
-            except LookupError as exc:
-                raise InvalidQuantumError(
-                    f"Quantum outputs is missing log output dataset type {dataset_type};"
-                    " this could happen due to inconsistent options between QuantumGraph generation"
-                    " and execution"
-                ) from exc
-
-            # Need to ingest this file directly into butler.
-            dataset = FileDataset(path=filename, refs=ref)
-            try:
-                self.full_butler.ingest(dataset, transfer="move")
-                ingested = True
-            except NotImplementedError:
-                # Some datastores can't receive files (e.g. in-memory datastore
-                # when testing), we store empty list for those just to have a
-                # dataset. Alternative is to read the file as a
-                # ButlerLogRecords object and put it.
-                _LOG.info(
-                    "Log records could not be stored in this butler because the"
-                    " datastore can not ingest files, empty record list is stored instead."
-                )
-                records = ButlerLogRecords.from_records([])
-                self.full_butler.put(records, ref)
-        finally:
-            # remove file if it is not ingested
-            if not ingested:
-                with suppress(OSError):
-                    os.remove(filename)
+warnings.warn(
+    "The log_capture module has moved to lsst.pipe.base.log_capture. "
+    "This forwarding shim will be removed after v30.",
+    category=FutureWarning,
+)