lsst-daf-butler 30.2025.5100__tar.gz → 30.2026.100__tar.gz

{lsst_daf_butler-30.2025.5100/python/lsst_daf_butler.egg-info → lsst_daf_butler-30.2026.100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 30.2025.5100
+Version: 30.2026.100
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_daf_butler-30.2025.5100 → lsst_daf_butler-30.2026.100}/python/lsst/daf/butler/_formatter.py

@@ -54,6 +54,7 @@ from ._config import Config
 from ._config_support import LookupKey, processLookupConfigs
 from ._file_descriptor import FileDescriptor
 from ._location import Location
+from ._rubin.temporary_for_ingest import TemporaryForIngest
 from .dimensions import DataCoordinate, DimensionUniverse
 from .mapping_factory import MappingFactory
 
@@ -1031,15 +1032,7 @@ class FormatterV2:
         """
         cache_manager = self._ensure_cache(cache_manager)
 
-        # Always write to a temporary even if
-        # using a local file system -- that gives us atomic writes.
-        # If a process is killed as the file is being written we do not
-        # want it to remain in the correct place but in corrupt state.
-        # For local files write to the output directory not temporary dir.
-        prefix = uri.dirname() if uri.isLocal else None
-        if prefix is not None:
-            prefix.mkdir()
-        with ResourcePath.temporary_uri(suffix=uri.getExtension(), prefix=prefix) as temporary_uri:
+        with TemporaryForIngest.make_path(uri) as temporary_uri:
             # Need to configure the formatter to write to a different
             # location and that needs us to overwrite internals
             log.debug("Writing dataset to temporary location at %s", temporary_uri)

lsst_daf_butler-30.2026.100/python/lsst/daf/butler/_rubin/temporary_for_ingest.py

@@ -0,0 +1,207 @@
+# This file is part of daf_butler.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+__all__ = ("TemporaryForIngest",)
+
+import dataclasses
+import glob
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Self, cast
+
+from lsst.resources import ResourcePath
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+    from types import TracebackType
+
+    from .._butler import Butler
+    from .._dataset_ref import DatasetRef
+    from .._file_dataset import FileDataset
+    from .._limited_butler import LimitedButler
+
+
+@dataclasses.dataclass
+class TemporaryForIngest:
+    """A context manager for generating temporary paths that will be ingested
+    as butler datasets.
+
+    Notes
+    -----
+    Neither this class nor its `make_path` method run ingest automatically when
+    their context manager is exited; the `ingest` method must always be called
+    explicitly.
+    """
+
+    butler: Butler
+    """Full butler to obtain a predicted path from and ingest into."""
+
+    ref: DatasetRef
+    """Description of the dataset to ingest."""
+
+    dataset: FileDataset = dataclasses.field(init=False)
+    """The dataset that will be passed to `Butler.ingest`."""
+
+    @property
+    def path(self) -> ResourcePath:
+        """The temporary path.
+
+        Guaranteed to be a local POSIX path.
+        """
+        return cast(ResourcePath, self.dataset.path)
+
+    @property
+    def ospath(self) -> str:
+        """The temporary path as a complete filename."""
+        return self.path.ospath
+
+    @classmethod
+    @contextmanager
+    def make_path(cls, final_path: ResourcePath) -> Iterator[ResourcePath]:
+        """Return a temporary path context manager given the predicted final
+        path.
+
+        Parameters
+        ----------
+        final_path : `lsst.resources.ResourcePath`
+            Predicted final path.
+
+        Returns
+        -------
+        context : `contextlib.AbstractContextManager`
+            A context manager that yields the temporary
+            `~lsst.resources.ResourcePath` when entered and deletes that file
+            when exited.
+        """
+        # Always write to a temporary even if using a local file system -- that
+        # gives us atomic writes. If a process is killed as the file is being
+        # written we do not want it to remain in the correct place but in
+        # corrupt state. For local files write to the output directory not
+        # temporary dir.
+        prefix = final_path.dirname() if final_path.isLocal else None
+        if prefix is not None:
+            prefix.mkdir()
+        with ResourcePath.temporary_uri(
+            suffix=cls._get_temporary_suffix(final_path), prefix=prefix
+        ) as temporary_path:
+            yield temporary_path
+
+    def ingest(self, record_validation_info: bool = True) -> None:
+        """Ingest the file into the butler.
+
+        Parameters
+        ----------
+        record_validation_info : `bool`, optional
+            Whether to- record the file size and checksum upon ingest.
+        """
+        self.butler.ingest(self.dataset, transfer="move", record_validation_info=record_validation_info)
+
+    def __enter__(self) -> Self:
+        from .._file_dataset import FileDataset
+
+        final_path = self.butler.getURI(self.ref, predict=True).replace(fragment="")
+        prefix = final_path.dirname() if final_path.isLocal else None
+        if prefix is not None:
+            prefix.mkdir()
+        self._temporary_path_context = self.make_path(final_path)
+        temporary_path = self._temporary_path_context.__enter__()
+        self.dataset = FileDataset(temporary_path, [self.ref], formatter=None)
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> bool | None:
+        return self._temporary_path_context.__exit__(exc_type, exc_value, traceback)
+
+    @classmethod
+    def find_orphaned_temporaries_by_path(cls, final_path: ResourcePath) -> list[ResourcePath]:
+        """Search for temporary files that were not successfully ingested.
+
+        Parameters
+        ----------
+        final_path : `lsst.resources.ResourcePath`
+            Final path a successfully-ingested file would have.
+
+        Returns
+        -------
+        paths : `list` [ `lsst.resources.ResourcePath` ]
+            Files that look like temporaries that might have been created while
+            trying to write the target dataset.
+
+        Notes
+        -----
+        Orphaned files are only possible when a context manager is interrupted
+        by a hard error that prevents any cleanup code from running (e.g.
+        sudden loss of power).
+        """
+        if not final_path.isLocal:
+            # We return true tempfile for non-local predicted paths, so orphans
+            # are not our problem (the OS etc. will take care of them).
+            return []
+        return [
+            ResourcePath(filename)
+            for filename in glob.glob(
+                f"{glob.escape(final_path.dirname().ospath)}*{glob.escape(cls._get_temporary_suffix(final_path))}"
+            )
+            if filename != final_path.ospath
+        ]
+
+    @classmethod
+    def find_orphaned_temporaries_by_ref(cls, ref: DatasetRef, butler: LimitedButler) -> list[ResourcePath]:
+        """Search for temporary files that were not successfully ingested.
+
+        Parameters
+        ----------
+        ref : `..DatasetRef`
+            A dataset reference the temporaries correspond to.
+        butler : `lsst.daf.butler.LimitedButler`
+            Butler that can be used to obtain a predicted URI for a dataset.
+
+        Returns
+        -------
+        paths : `list` [ `lsst.resources.ResourcePath` ]
+            Files that look like temporaries that might have been created while
+            trying to write the target dataset.
+
+        Notes
+        -----
+        Orphaned files are only possible when a context manager is interrupted
+        by a hard error that prevents any cleanup code from running (e.g.
+        sudden loss of power).
+        """
+        final_path = butler.getURI(ref, predict=True).replace(fragment="")
+        return cls.find_orphaned_temporaries_by_path(final_path)
+
+    @staticmethod
+    def _get_temporary_suffix(path: ResourcePath) -> str:
+        ext = path.getExtension()
+        basename = path.basename().removesuffix(ext)
+        return f"{basename}.tmp{ext}"

lsst_daf_butler-30.2026.100/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "30.2026.100"

{lsst_daf_butler-30.2025.5100 → lsst_daf_butler-30.2026.100/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 30.2025.5100
+Version: 30.2026.100
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License-Expression: BSD-3-Clause OR GPL-3.0-or-later

{lsst_daf_butler-30.2025.5100 → lsst_daf_butler-30.2026.100}/python/lsst_daf_butler.egg-info/SOURCES.txt

@@ -73,6 +73,7 @@ python/lsst/daf/butler/utils.py
 python/lsst/daf/butler/version.py
 python/lsst/daf/butler/_rubin/__init__.py
 python/lsst/daf/butler/_rubin/file_datasets.py
+python/lsst/daf/butler/_rubin/temporary_for_ingest.py
 python/lsst/daf/butler/_utilities/__init__.py
 python/lsst/daf/butler/_utilities/locked_object.py
 python/lsst/daf/butler/_utilities/named_locks.py

{lsst_daf_butler-30.2025.5100 → lsst_daf_butler-30.2026.100}/tests/test_butler.py

@@ -90,6 +90,7 @@ from lsst.daf.butler import (
     script,
 )
 from lsst.daf.butler._rubin.file_datasets import transfer_datasets_to_datastore
+from lsst.daf.butler._rubin.temporary_for_ingest import TemporaryForIngest
 from lsst.daf.butler.datastore import NullDatastore
 from lsst.daf.butler.datastore.file_templates import FileTemplate, FileTemplateValidationError
 from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import ZipIndex
@@ -2441,6 +2442,21 @@ class PosixDatastoreButlerTestCase(FileDatastoreButlerTests, unittest.TestCase):
             self.assertIsNotNone(target_butler.get(repo.ref2))
             self.assertIsNotNone(target_butler.get(other_ref))
 
+    def test_temporary_for_ingest(self) -> None:
+        """Test the `lsst.daf.butler._rubin.ingest_from_temporary` module."""
+        with self.create_empty_butler("example_run") as butler:
+            dataset_type = DatasetType("example", butler.dimensions.empty, "StructuredDataDict")
+            butler.registry.registerDatasetType(dataset_type)
+            ref = DatasetRef(dataset_type, DataCoordinate.make_empty(butler.dimensions), "example_run")
+            with TemporaryForIngest(butler, ref) as temporary:
+                temporary.path.write(b"three: 3")
+                found = TemporaryForIngest.find_orphaned_temporaries_by_ref(ref, butler)
+                self.assertEqual(found, [temporary.path])
+                self.assertIn(".tmp", temporary.ospath)
+                temporary.ingest()
+            loaded = butler.get(ref)
+            self.assertEqual(loaded, {"three": 3})
+
 
 class PostgresPosixDatastoreButlerTestCase(FileDatastoreButlerTests, unittest.TestCase):
     """PosixDatastore specialization of a butler using Postgres"""

lsst_daf_butler-30.2025.5100/python/lsst/daf/butler/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "30.2025.5100"