lsst-pipe-base 30.2025.5100__py3-none-any.whl → 30.2026.100__py3-none-any.whl

lsst/pipe/base/log_capture.py

@@ -31,17 +31,15 @@ __all__ = ["LogCapture"]
 
 import dataclasses
 import logging
-import os
-import shutil
-import tempfile
 import uuid
 from collections.abc import Iterator
-from contextlib import contextmanager, suppress
+from contextlib import contextmanager
 from logging import FileHandler
 
 import pydantic
 
-from lsst.daf.butler import Butler, FileDataset, LimitedButler, Quantum
+from lsst.daf.butler import Butler, LimitedButler, Quantum
+from lsst.daf.butler._rubin.temporary_for_ingest import TemporaryForIngest
 from lsst.daf.butler.logging import (
     ButlerLogRecord,
     ButlerLogRecordHandler,
@@ -205,41 +203,37 @@ class LogCapture:
 
         # Add a handler to the root logger to capture execution log output.
         if log_dataset_name is not None:
+            try:
+                [ref] = quantum.outputs[log_dataset_name]
+            except LookupError as exc:
+                raise InvalidQuantumError(
+                    f"Quantum outputs is missing log output dataset type {log_dataset_name};"
+                    " this could happen due to inconsistent options between QuantumGraph generation"
+                    " and execution"
+                ) from exc
             # Either accumulate into ButlerLogRecords or stream JSON records to
             # file and ingest that (ingest is possible only with full butler).
             if self.stream_json_logs and self.full_butler is not None:
-                # Create the log file in a temporary directory rather than
-                # creating a temporary file. This is necessary because
-                # temporary files are created with restrictive permissions
-                # and during file ingest these permissions persist in the
-                # datastore. Using a temp directory allows us to create
-                # a file with umask default permissions.
-                tmpdir = tempfile.mkdtemp(prefix="butler-temp-logs-")
-
-                # Construct a file to receive the log records and "touch" it.
-                log_file = os.path.join(tmpdir, f"butler-log-{task_node.label}.json")
-                with open(log_file, "w"):
-                    pass
-                log_handler_file = FileHandler(log_file)
-                log_handler_file.setFormatter(JsonLogFormatter())
-                logging.getLogger().addHandler(log_handler_file)
-
-                try:
-                    with ButlerMDC.set_mdc(mdc):
-                        yield ctx
-                finally:
-                    # Ensure that the logs are stored in butler.
-                    logging.getLogger().removeHandler(log_handler_file)
-                    log_handler_file.close()
-                    if ctx.extra:
-                        with open(log_file, "a") as log_stream:
-                            ButlerLogRecords.write_streaming_extra(
-                                log_stream,
-                                ctx.extra.model_dump_json(exclude_unset=True, exclude_defaults=True),
-                            )
-                    if ctx.store:
-                        self._ingest_log_records(quantum, log_dataset_name, log_file)
-                    shutil.rmtree(tmpdir, ignore_errors=True)
+                with TemporaryForIngest(self.full_butler, ref) as temporary:
+                    log_handler_file = FileHandler(temporary.ospath)
+                    log_handler_file.setFormatter(JsonLogFormatter())
+                    logging.getLogger().addHandler(log_handler_file)
+
+                    try:
+                        with ButlerMDC.set_mdc(mdc):
+                            yield ctx
+                    finally:
+                        # Ensure that the logs are stored in butler.
+                        logging.getLogger().removeHandler(log_handler_file)
+                        log_handler_file.close()
+                        if ctx.extra:
+                            with open(temporary.ospath, "a") as log_stream:
+                                ButlerLogRecords.write_streaming_extra(
+                                    log_stream,
+                                    ctx.extra.model_dump_json(exclude_unset=True, exclude_defaults=True),
+                                )
+                        if ctx.store:
+                            temporary.ingest()
 
             else:
                 log_handler_memory = ButlerLogRecordHandler()
@@ -281,41 +275,3 @@ class LogCapture:
             ) from exc
 
         self.butler.put(log_handler.records, ref)
-
-    def _ingest_log_records(self, quantum: Quantum, dataset_type: str, filename: str) -> None:
-        # If we are logging to an external file we must always try to
-        # close it.
-        assert self.full_butler is not None, "Expected to have full butler for ingest"
-        ingested = False
-        try:
-            # DatasetRef has to be in the Quantum outputs, can lookup by name.
-            try:
-                [ref] = quantum.outputs[dataset_type]
-            except LookupError as exc:
-                raise InvalidQuantumError(
-                    f"Quantum outputs is missing log output dataset type {dataset_type};"
-                    " this could happen due to inconsistent options between QuantumGraph generation"
-                    " and execution"
-                ) from exc
-
-            # Need to ingest this file directly into butler.
-            dataset = FileDataset(path=filename, refs=ref)
-            try:
-                self.full_butler.ingest(dataset, transfer="move")
-                ingested = True
-            except NotImplementedError:
-                # Some datastores can't receive files (e.g. in-memory datastore
-                # when testing), we store empty list for those just to have a
-                # dataset. Alternative is to read the file as a
-                # ButlerLogRecords object and put it.
-                _LOG.info(
-                    "Log records could not be stored in this butler because the"
-                    " datastore can not ingest files, empty record list is stored instead."
-                )
-                records = ButlerLogRecords.from_records([])
-                self.full_butler.put(records, ref)
-        finally:
-            # remove file if it is not ingested
-            if not ingested:
-                with suppress(OSError):
-                    os.remove(filename)

lsst/pipe/base/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "30.2025.5100"
+__version__ = "30.2026.100"

{lsst_pipe_base-30.2025.5100.dist-info → lsst_pipe_base-30.2026.100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 30.2025.5100
+Version: 30.2026.100
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_pipe_base-30.2025.5100.dist-info → lsst_pipe_base-30.2026.100.dist-info}/RECORD

@@ -20,7 +20,7 @@ lsst/pipe/base/exec_fixup_data_id.py,sha256=9OjOcH-6AHZ1JnD_CemieI0wWX90J_VdaY9v
 lsst/pipe/base/execution_graph_fixup.py,sha256=ND0x4hlpeEW-gudo-i2K7HT7MoM5sp_mcoqRMCopSqQ,3815
 lsst/pipe/base/execution_reports.py,sha256=jYtWCD4PkEAeVUpKIxuiJJVgsCm7qiwCorWVgNHkVgU,17270
 lsst/pipe/base/graph_walker.py,sha256=Ij7JfYF0srA29VgM_DhbNBxUBeOHDOnujrTQPjVNha0,4694
-lsst/pipe/base/log_capture.py,sha256=5r99_Ek2A75vYOMo-z52ltWLdfYfWExm55UU9a4nqmM,12909
+lsst/pipe/base/log_capture.py,sha256=2kuapttiMwpvevOleOMDcjEorXBCiVNSzsKj9fMDhI4,10818
 lsst/pipe/base/mermaid_tools.py,sha256=cdlDJQ1x8k7-VvCLEUqvSC3GR1zCsB-aUTxOjYejNWc,5216
 lsst/pipe/base/mp_graph_executor.py,sha256=FKlFxjtU2-6SFzX_wsdtMMAj5P09ZE8V65-Sg585g2Y,35501
 lsst/pipe/base/pipeline.py,sha256=FVaiLhgw9Pzo-nzXKS0dLNafegP0AMZKLtPlSvOSkRU,37563
@@ -42,7 +42,7 @@ lsst/pipe/base/task.py,sha256=XHBd-7m1a4-6LgobBYA1DgY4H7EV-_RWKfxbhZbMmD4,15145
 lsst/pipe/base/taskFactory.py,sha256=MsDGECJqZLSZk8SGhpuVhNaP32UWuNvxZiDcZExPFG8,3412
 lsst/pipe/base/testUtils.py,sha256=lSBKMhoKflbi8JkMNYfEqqHNl-rtFI8UYT3QneDYpLo,18477
 lsst/pipe/base/utils.py,sha256=JmEt3l0xrh9uayKrSXuQEq12aXOhDr2YXmbYduaxCko,1940
-lsst/pipe/base/version.py,sha256=T_ii-AmyfLEzX_XiWTNw5GEFMy94NcXrbg2rkzKZg7g,55
+lsst/pipe/base/version.py,sha256=BC5axNzZNKnyS8niPRP5v8FOY10ThRUTtQL-d2K4BdU,54
 lsst/pipe/base/cli/__init__.py,sha256=861tXIAW7SqtqNUYkjbeEdfg8lDswXsjJQca0gVCFz4,54
 lsst/pipe/base/cli/_get_cli_subcommands.py,sha256=g_af64klRybBGKAg7fmBSZBdw2LYBAsFON_yQIMZON0,1289
 lsst/pipe/base/cli/cmd/__init__.py,sha256=3UF2IQEEBor4YMGRNPdcZAVCAI5yFyeHp5nGul4IoyM,1557
@@ -113,13 +113,13 @@ lsst/pipe/base/tests/mocks/_data_id_match.py,sha256=jVekStcrItC0tqOCc01VjYaiE9ex
 lsst/pipe/base/tests/mocks/_pipeline_task.py,sha256=N3fC4OMAMWWnYtyLkVdMfb9ZiFse39HniRDvlAOofOY,30691
 lsst/pipe/base/tests/mocks/_repo.py,sha256=SH-jzynS-H2xc_3GLjF7ln-kHdRoSeVVaal5qLd2hXI,28359
 lsst/pipe/base/tests/mocks/_storage_class.py,sha256=12IFfJMbZ5GkYlMX6ZMWiG8pMZc2Jlxke3qQW-bljdU,27434
-lsst_pipe_base-30.2025.5100.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
-lsst_pipe_base-30.2025.5100.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_pipe_base-30.2025.5100.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_pipe_base-30.2025.5100.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_pipe_base-30.2025.5100.dist-info/METADATA,sha256=AZu2-D91gCmfRmP0m2ETbsiRKZwX8DXqBiO-rvyDzFo,2257
-lsst_pipe_base-30.2025.5100.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_pipe_base-30.2025.5100.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
-lsst_pipe_base-30.2025.5100.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_pipe_base-30.2025.5100.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_pipe_base-30.2025.5100.dist-info/RECORD,,
+lsst_pipe_base-30.2026.100.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
+lsst_pipe_base-30.2026.100.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_pipe_base-30.2026.100.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_pipe_base-30.2026.100.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_pipe_base-30.2026.100.dist-info/METADATA,sha256=TTpe8bR7EgNLhCX02RnYtBxB9WOIAsmdJSbrUpS-XUA,2256
+lsst_pipe_base-30.2026.100.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_pipe_base-30.2026.100.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
+lsst_pipe_base-30.2026.100.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_pipe_base-30.2026.100.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_pipe_base-30.2026.100.dist-info/RECORD,,