lsst-daf-butler 29.2025.3500__tar.gz → 29.2025.3700__tar.gz

{lsst_daf_butler-29.2025.3500/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.3700}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.3500
+Version: 29.2025.3700
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.3500 → lsst_daf_butler-29.2025.3700}/python/lsst/daf/butler/_butler.py

@@ -1333,6 +1333,7 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
         *datasets: FileDataset,
         transfer: str | None = "auto",
         record_validation_info: bool = True,
+        skip_existing: bool = False,
     ) -> None:
         """Store and register one or more datasets that already exist on disk.
 
@@ -1360,6 +1361,12 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
             or file sizes. This can be useful if such information is tracked
             in an external system or if the file is to be compressed in place.
             It is up to the datastore whether this parameter is relevant.
+        skip_existing : `bool`, optional
+            If `True`, a dataset will not be ingested if a dataset with the
+            same dataset ID already exists in the datastore.
+            If `False` (the default), a `ConflictingDefinitionError` will be
+            raised if any datasets with the same dataset ID already exist
+            in the datastore.
 
         Raises
         ------
@@ -1375,6 +1382,9 @@ class Butler(LimitedButler):  # numpydoc ignore=PR02
         FileExistsError
             Raised if transfer is not `None` but the (internal) location the
             file would be moved to is already occupied.
+        ConflictingDefinitionError
+            Raised if a dataset already exists in the repository and
+            ``skip_existing`` is `False`.
 
         Notes
         -----

{lsst_daf_butler-29.2025.3500 → lsst_daf_butler-29.2025.3700}/python/lsst/daf/butler/_quantum_backed.py

@@ -27,8 +27,6 @@
 
 from __future__ import annotations
 
-from . import ddl
-
 __all__ = ("QuantumBackedButler", "QuantumProvenanceData")
 
 import itertools
@@ -51,15 +49,13 @@ from ._dataset_type import DatasetType
 from ._deferredDatasetHandle import DeferredDatasetHandle
 from ._limited_butler import LimitedButler
 from ._quantum import Quantum
+from ._standalone_datastore import instantiate_standalone_datastore
 from ._storage_class import StorageClass, StorageClassFactory
 from .datastore import Datastore
 from .datastore.record_data import DatastoreRecordData, SerializedDatastoreRecordData
 from .datastores.file_datastore.retrieve_artifacts import retrieve_and_zip
 from .dimensions import DimensionUniverse
-from .registry.bridge.monolithic import MonolithicDatastoreRegistryBridgeManager
-from .registry.databases.sqlite import SqliteDatabase
 from .registry.interfaces import DatastoreRegistryBridgeManager, OpaqueTableStorageManager
-from .registry.opaque import ByNameOpaqueTableStorageManager
 
 if TYPE_CHECKING:
     from ._butler import Butler
@@ -67,40 +63,6 @@ if TYPE_CHECKING:
 _LOG = logging.getLogger(__name__)
 
 
-class _DatasetRecordStorageManagerDatastoreConstructionMimic:
-    """A partial implementation of `DatasetRecordStorageManager` that exists
-    only to allow a `DatastoreRegistryBridgeManager` (and hence a `Datastore`)
-    to be constructed without a full `Registry`.
-
-    Notes
-    -----
-    The interface implemented by this class should probably be its own ABC,
-    and that ABC should probably be used in the definition of
-    `DatastoreRegistryBridgeManager`, but while prototyping I'm trying to keep
-    changes minimal.
-    """
-
-    @classmethod
-    def getIdColumnType(cls) -> type:
-        # Docstring inherited.
-        return ddl.GUID
-
-    @classmethod
-    def addDatasetForeignKey(
-        cls,
-        tableSpec: ddl.TableSpec,
-        *,
-        name: str = "dataset",
-        constraint: bool = True,
-        onDelete: str | None = None,
-        **kwargs: Any,
-    ) -> ddl.FieldSpec:
-        # Docstring inherited.
-        idFieldSpec = ddl.FieldSpec(f"{name}_id", dtype=ddl.GUID, **kwargs)
-        tableSpec.fields.add(idFieldSpec)
-        return idFieldSpec
-
-
 class QuantumBackedButler(LimitedButler):
     """An implementation of `LimitedButler` intended to back execution of a
     single `Quantum`.
@@ -190,9 +152,9 @@ class QuantumBackedButler(LimitedButler):
         config: Config | ResourcePathExpression,
         quantum: Quantum,
         dimensions: DimensionUniverse,
-        filename: str = ":memory:",
-        OpaqueManagerClass: type[OpaqueTableStorageManager] = ByNameOpaqueTableStorageManager,
-        BridgeManagerClass: type[DatastoreRegistryBridgeManager] = MonolithicDatastoreRegistryBridgeManager,
+        filename: str | None = None,
+        OpaqueManagerClass: type[OpaqueTableStorageManager] | None = None,
+        BridgeManagerClass: type[DatastoreRegistryBridgeManager] | None = None,
         search_paths: list[str] | None = None,
         dataset_types: Mapping[str, DatasetType] | None = None,
         metrics: ButlerMetrics | None = None,
@@ -253,9 +215,9 @@ class QuantumBackedButler(LimitedButler):
         predicted_outputs: Iterable[DatasetId],
         dimensions: DimensionUniverse,
         datastore_records: Mapping[str, DatastoreRecordData],
-        filename: str = ":memory:",
-        OpaqueManagerClass: type[OpaqueTableStorageManager] = ByNameOpaqueTableStorageManager,
-        BridgeManagerClass: type[DatastoreRegistryBridgeManager] = MonolithicDatastoreRegistryBridgeManager,
+        filename: str | None = None,
+        OpaqueManagerClass: type[OpaqueTableStorageManager] | None = None,
+        BridgeManagerClass: type[DatastoreRegistryBridgeManager] | None = None,
         search_paths: list[str] | None = None,
         dataset_types: Mapping[str, DatasetType] | None = None,
         metrics: ButlerMetrics | None = None,
@@ -316,10 +278,10 @@ class QuantumBackedButler(LimitedButler):
         predicted_inputs: Iterable[DatasetId],
         predicted_outputs: Iterable[DatasetId],
         dimensions: DimensionUniverse,
-        filename: str = ":memory:",
+        filename: str | None = None,
         datastore_records: Mapping[str, DatastoreRecordData] | None = None,
-        OpaqueManagerClass: type[OpaqueTableStorageManager] = ByNameOpaqueTableStorageManager,
-        BridgeManagerClass: type[DatastoreRegistryBridgeManager] = MonolithicDatastoreRegistryBridgeManager,
+        OpaqueManagerClass: type[OpaqueTableStorageManager] | None = None,
+        BridgeManagerClass: type[DatastoreRegistryBridgeManager] | None = None,
         search_paths: list[str] | None = None,
         dataset_types: Mapping[str, DatasetType] | None = None,
         metrics: ButlerMetrics | None = None,
@@ -359,19 +321,9 @@ class QuantumBackedButler(LimitedButler):
             Metrics object for gathering butler statistics.
         """
         butler_config = ButlerConfig(config, searchPaths=search_paths)
-        butler_root = butler_config.get("root", butler_config.configDir)
-        db = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0)
-        with db.declareStaticTables(create=True) as context:
-            opaque_manager = OpaqueManagerClass.initialize(db, context)
-            bridge_manager = BridgeManagerClass.initialize(
-                db,
-                context,
-                opaque=opaque_manager,
-                # MyPy can tell it's a fake, but we know it shouldn't care.
-                datasets=_DatasetRecordStorageManagerDatastoreConstructionMimic,  # type: ignore
-                universe=dimensions,
-            )
-        datastore = Datastore.fromConfig(butler_config, bridge_manager, butler_root)
+        datastore, _ = instantiate_standalone_datastore(
+            butler_config, dimensions, filename, OpaqueManagerClass, BridgeManagerClass
+        )
 
         # TODO: We need to inform `Datastore` here that it needs to support
         # predictive reads; This only really works for file datastore but

lsst_daf_butler-29.2025.3700/python/lsst/daf/butler/_rubin/__init__.py

@@ -0,0 +1,31 @@
+# This file is part of daf_butler.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Modules in the ``_rubin`` package contain functions that are used internally
+by other LSST packages.  The interfaces and behavior of these functions are
+subject to change at any time.
+"""

lsst_daf_butler-29.2025.3700/python/lsst/daf/butler/_rubin/file_datasets.py

@@ -0,0 +1,88 @@
+# This file is part of daf_butler.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+__all__ = ("transfer_datasets_to_datastore",)
+
+from collections.abc import Iterable
+
+from .._butler import Butler
+from .._butler_config import ButlerConfig
+from .._dataset_ref import DatasetRef
+from .._file_dataset import FileDataset
+from .._standalone_datastore import instantiate_standalone_datastore
+from ..direct_butler import DirectButler
+
+
+def transfer_datasets_to_datastore(
+    source_butler: Butler, output_config: ButlerConfig, refs: Iterable[DatasetRef]
+) -> list[FileDataset]:
+    """Copy datasets into a `Datastore` without writing entries into the
+    associated registry database.
+
+    Parameters
+    ----------
+    source_butler : `Butler`
+        Butler instance from which to copy datasets.
+    output_config : `ButlerConfig`
+        Butler configuration specifying the configuration for the target
+        datastore.
+    refs : `~collections.abc.Iterable` [ `DatasetRef` ]
+        List of datasets to copy into the target datastore.
+
+    Returns
+    -------
+    file_datasets : `list` [ `FileDataset` ]
+        `FileDataset` instances representing the artifacts copied to
+        the target datastore.
+    """
+    refs = list(refs)
+    if len(refs) == 0:
+        return []
+
+    if not isinstance(source_butler, DirectButler):
+        raise ValueError("Butler must be an instance of DirectButler")
+
+    # Refs must contain dimension records to allow for filename template
+    # expansion in the datastore.
+    refs = source_butler._registry.expand_refs(list(refs))
+    # Read out the absolute URLs of the datasets we are about to transfer.
+    datasets = list(source_butler._datastore.export(refs, directory=None, transfer="direct"))
+
+    # Set up a temporary, initially empty, in-memory DB for the target
+    # Datastore.
+    dimension_universe = datasets[0].refs[0].dimensions.universe
+    datastore, db = instantiate_standalone_datastore(output_config, dimension_universe)
+    try:
+        # Write the files to their destination in the target datastore.
+        datastore.ingest(*datasets, transfer="copy", record_validation_info=False)
+        # Read out the relative URLs of the files, for their location
+        # in the target datastore.
+        return list(datastore.export(refs, directory=None, transfer=None))
+    finally:
+        db.dispose()

lsst_daf_butler-29.2025.3700/python/lsst/daf/butler/_standalone_datastore.py

@@ -0,0 +1,129 @@
+# This file is part of daf_butler.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+__all__ = ("instantiate_standalone_datastore",)
+
+from typing import Any
+
+from . import ddl
+from ._butler_config import ButlerConfig
+from .datastore import Datastore
+from .dimensions import DimensionUniverse
+from .registry.bridge.monolithic import MonolithicDatastoreRegistryBridgeManager
+from .registry.databases.sqlite import SqliteDatabase
+from .registry.interfaces import Database, DatastoreRegistryBridgeManager, OpaqueTableStorageManager
+from .registry.opaque import ByNameOpaqueTableStorageManager
+
+
+def instantiate_standalone_datastore(
+    butler_config: ButlerConfig,
+    dimensions: DimensionUniverse,
+    filename: str | None = None,
+    OpaqueManagerClass: type[OpaqueTableStorageManager] | None = None,
+    BridgeManagerClass: type[DatastoreRegistryBridgeManager] | None = None,
+) -> tuple[Datastore, Database]:
+    """Initialize a `Datastore` instance without an associated `Registry`.
+
+    Parameters
+    ----------
+    butler_config : `ButlerConfig`
+        Butler configuration that defines the configuration for the `Datastore`
+        to be initialized.
+    dimensions : `DimensionUniverse`
+        Dimension universe used by the Butler repository backing the
+        `Datastore`.
+    filename : `str`, optional
+        Name for the SQLite database that will back the `Datastore`; defaults
+        to an in-memory database.
+    OpaqueManagerClass : `type`, optional
+        A subclass of `OpaqueTableStorageManager` to use for datastore
+        opaque records.  Default is a SQL-backed implementation.
+    BridgeManagerClass : `type`, optional
+        A subclass of `DatastoreRegistryBridgeManager` to use for datastore
+        location records.  Default is a SQL-backed implementation.
+
+    Returns
+    -------
+    datastore_and_database : `tuple` [ `Datastore` , `Database` ]
+        The temporary datastore, and the database instance backing it.
+
+    Notes
+    -----
+    This allows files to be written and read without access to the Butler
+    database.  It is primarily used by `QuantumBackedButler`.
+    """
+    if filename is None:
+        filename = ":memory:"
+    if OpaqueManagerClass is None:
+        OpaqueManagerClass = ByNameOpaqueTableStorageManager
+    if BridgeManagerClass is None:
+        BridgeManagerClass = MonolithicDatastoreRegistryBridgeManager
+
+    butler_root = butler_config.get("root", butler_config.configDir)
+    db = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0)
+    with db.declareStaticTables(create=True) as context:
+        opaque_manager = OpaqueManagerClass.initialize(db, context)
+        bridge_manager = BridgeManagerClass.initialize(
+            db,
+            context,
+            opaque=opaque_manager,
+            # MyPy can tell it's a fake, but we know it shouldn't care.
+            datasets=_DatasetRecordStorageManagerDatastoreConstructionMimic,  # type: ignore
+            universe=dimensions,
+        )
+
+    datastore = Datastore.fromConfig(butler_config, bridge_manager, butler_root)
+    return (datastore, db)
+
+
+class _DatasetRecordStorageManagerDatastoreConstructionMimic:
+    """A partial implementation of `DatasetRecordStorageManager` that exists
+    only to allow a `DatastoreRegistryBridgeManager` (and hence a `Datastore`)
+    to be constructed without a full `Registry`.
+    """
+
+    @classmethod
+    def getIdColumnType(cls) -> type:
+        # Docstring inherited.
+        return ddl.GUID
+
+    @classmethod
+    def addDatasetForeignKey(
+        cls,
+        tableSpec: ddl.TableSpec,
+        *,
+        name: str = "dataset",
+        constraint: bool = True,
+        onDelete: str | None = None,
+        **kwargs: Any,
+    ) -> ddl.FieldSpec:
+        # Docstring inherited.
+        idFieldSpec = ddl.FieldSpec(f"{name}_id", dtype=ddl.GUID, **kwargs)
+        tableSpec.fields.add(idFieldSpec)
+        return idFieldSpec

{lsst_daf_butler-29.2025.3500 → lsst_daf_butler-29.2025.3700}/python/lsst/daf/butler/datastores/chainedDatastore.py

@@ -562,9 +562,6 @@ class ChainedDatastore(Datastore):
 
     def _prepIngest(self, *datasets: FileDataset, transfer: str | None = None) -> _IngestPrepData:
         # Docstring inherited from Datastore._prepIngest.
-        if transfer is None:
-            raise NotImplementedError("ChainedDatastore does not support transfer=None.")
-
         def isDatasetAcceptable(dataset: FileDataset, *, name: str, constraints: Constraints) -> bool:
             acceptable = [ref for ref in dataset.refs if constraints.isAcceptable(ref)]
             if not acceptable:

{lsst_daf_butler-29.2025.3500 → lsst_daf_butler-29.2025.3700}/python/lsst/daf/butler/datastores/fileDatastore.py

@@ -1385,6 +1385,7 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
 
     def knows_these(self, refs: Iterable[DatasetRef]) -> dict[DatasetRef, bool]:
         # Docstring inherited from the base class.
+        refs = list(refs)
 
         # The records themselves. Could be missing some entries.
         records = self._get_stored_records_associated_with_refs(refs, ignore_datastore_records=True)
@@ -3050,17 +3051,11 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
         if transfer == "auto" and directory is None:
             transfer = None
 
-        if transfer is not None and directory is None:
+        if transfer is not None and transfer != "direct" and directory is None:
             raise TypeError(f"Cannot export using transfer mode {transfer} with no export directory given")
 
         if transfer == "move":
             raise TypeError("Can not export by moving files out of datastore.")
-        elif transfer == "direct":
-            # For an export, treat this as equivalent to None. We do not
-            # want an import to risk using absolute URIs to datasets owned
-            # by another datastore.
-            log.info("Treating 'direct' transfer mode as in-place export.")
-            transfer = None
 
         # Force the directory to be a URI object
         directoryUri: ResourcePath | None = None

{lsst_daf_butler-29.2025.3500 → lsst_daf_butler-29.2025.3700}/python/lsst/daf/butler/direct_butler/_direct_butler.py

@@ -1797,6 +1797,7 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
         *datasets: FileDataset,
         transfer: str | None = "auto",
         record_validation_info: bool = True,
+        skip_existing: bool = False,
     ) -> None:
         # Docstring inherited.
         if not datasets:
@@ -1806,6 +1807,21 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
         _LOG.verbose("Ingesting %d file dataset%s.", len(datasets), "" if len(datasets) == 1 else "s")
         progress = Progress("lsst.daf.butler.Butler.ingest", level=VERBOSE)
 
+        new_datasets, existing_datasets = self._partition_datasets_by_known(datasets)
+        if existing_datasets:
+            if skip_existing:
+                _LOG.info(
+                    "Skipping %d datasets which already exist in the repository.", len(existing_datasets)
+                )
+            else:
+                raise ConflictingDefinitionError(
+                    f"Datastore already contains {len(existing_datasets)} of the given datasets."
+                    f" Example: {existing_datasets[0]}"
+                )
+
+        # We use `datasets` rather `new_datasets` for the Registry
+        # portion of this, to let it confirm that everything matches the
+        # existing datasets.
         import_info = self._prepare_ingest_file_datasets(datasets, progress)
 
         with self.transaction():
@@ -1813,17 +1829,41 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
 
             # Bulk-insert everything into Datastore.
             # We do not know if any of the registry entries already existed
-            # (_importDatasets only complains if they exist but differ) so
-            # we have to catch IntegrityError explicitly.
+            # (_importDatasets only complains if they exist but differ).
+            # The _partition_datasets_by_known logic above should catch most
+            # instances where we attempt to re-ingest files that were already
+            # ingested, but a concurrent writer could cause a unique constraint
+            # violation here.
             try:
                 self._datastore.ingest(
-                    *datasets, transfer=transfer, record_validation_info=record_validation_info
+                    *new_datasets, transfer=transfer, record_validation_info=record_validation_info
                 )
             except IntegrityError as e:
                 raise ConflictingDefinitionError(
                     f"Datastore already contains one or more datasets: {e}"
                 ) from e
 
+    def _partition_datasets_by_known(
+        self, datasets: Iterable[FileDataset]
+    ) -> tuple[list[FileDataset], list[FileDataset]]:
+        """Divides the given `FileDataset` objects into two groups: those for
+        which the Datastore already has an entry, and those for which it does
+        not.
+        """
+        new_datasets = []
+        existing_datasets = []
+
+        refs = itertools.chain.from_iterable(dataset.refs for dataset in datasets)
+        known_refs = self._datastore.knows_these(refs)
+
+        for dataset in datasets:
+            if any(known_refs[ref] for ref in dataset.refs):
+                existing_datasets.append(dataset)
+            else:
+                new_datasets.append(dataset)
+
+        return new_datasets, existing_datasets
+
     @contextlib.contextmanager
     def export(
         self,

{lsst_daf_butler-29.2025.3500 → lsst_daf_butler-29.2025.3700}/python/lsst/daf/butler/logging.py

@@ -35,6 +35,7 @@ import traceback
 from collections.abc import Callable, Generator, Iterable, Iterator
 from contextlib import contextmanager
 from logging import Formatter, LogRecord, StreamHandler
+from types import TracebackType
 from typing import IO, Any, ClassVar, overload
 
 from pydantic import BaseModel, ConfigDict, PrivateAttr, RootModel
@@ -127,6 +128,9 @@ class ButlerMDC:
     def set_mdc(cls, mdc: dict[str, str]) -> Generator[None, None, None]:
         """Set the MDC key for this context.
 
+        This context manager also adds a mapping named ``mdc`` to any
+        exceptions that escape it.
+
         Parameters
         ----------
         mdc : `dict` of `str`, `str`
@@ -143,6 +147,12 @@ class ButlerMDC:
 
         try:
             yield
+        except BaseException as e:
+            # In logging, inner context overrules outer context. Need the same
+            # for exceptions.
+            inner_context: MDCDict = e.mdc if hasattr(e, "mdc") else MDCDict()
+            e.mdc = cls._MDC.copy() | inner_context  # type: ignore[attr-defined]
+            raise
         finally:
             for k, v in previous.items():
                 if not v:
@@ -155,10 +165,31 @@ class ButlerMDC:
         """Add a log record factory that adds a MDC record to `LogRecord`."""
         old_factory = logging.getLogRecordFactory()
 
-        def record_factory(*args: Any, **kwargs: Any) -> LogRecord:
-            record = old_factory(*args, **kwargs)
+        # Need explicit args in case exc_info is called as an arg (and because
+        # the factory API has a parameter literally called `args`).
+        def record_factory(
+            name: str,
+            level: int,
+            fn: str,
+            lno: int,
+            msg: str,
+            args: tuple,
+            exc_info: tuple | None,
+            func: str | None = None,
+            sinfo: TracebackType | None = None,
+            **kwargs: Any,
+        ) -> LogRecord:
+            record = old_factory(name, level, fn, lno, msg, args, exc_info, func, sinfo, **kwargs)
             # Make sure we send a copy of the global dict in the record.
-            record.MDC = MDCDict(cls._MDC)
+            mdc = MDCDict(cls._MDC)
+            if exc_info is not None:
+                _, ex, _ = exc_info
+                # TODO: this doesn't handle chained exceptions, fix on DM-47546
+                if hasattr(ex, "mdc"):
+                    # Context at the point where the exception was raised
+                    # takes precedence.
+                    mdc.update(ex.mdc)
+            record.MDC = mdc
             return record
 
         cls._old_factory = old_factory

{lsst_daf_butler-29.2025.3500 → lsst_daf_butler-29.2025.3700}/python/lsst/daf/butler/registry/interfaces/_database.py

@@ -287,6 +287,12 @@ class Database(ABC):
         self._metadata = metadata
         self._allow_temporary_tables = allow_temporary_tables
 
+    def dispose(self) -> None:
+        """Close all open database connections held by this `Database`
+        instance.
+        """
+        self._engine.dispose()
+
     def __repr__(self) -> str:
         # Rather than try to reproduce all the parameters used to create
         # the object, instead report the more useful information of the

{lsst_daf_butler-29.2025.3500 → lsst_daf_butler-29.2025.3700}/python/lsst/daf/butler/remote_butler/_remote_butler.py

@@ -576,6 +576,7 @@ class RemoteButler(Butler):  # numpydoc ignore=PR02
         *datasets: FileDataset,
         transfer: str | None = "auto",
         record_validation_info: bool = True,
+        skip_existing: bool = False,
     ) -> None:
         # Docstring inherited.
         raise NotImplementedError()

{lsst_daf_butler-29.2025.3500 → lsst_daf_butler-29.2025.3700}/python/lsst/daf/butler/tests/hybrid_butler.py

@@ -292,9 +292,13 @@ class HybridButler(Butler):
         *datasets: FileDataset,
         transfer: str | None = "auto",
         record_validation_info: bool = True,
+        skip_existing: bool = False,
     ) -> None:
         return self._direct_butler.ingest(
-            *datasets, transfer=transfer, record_validation_info=record_validation_info
+            *datasets,
+            transfer=transfer,
+            record_validation_info=record_validation_info,
+            skip_existing=skip_existing,
         )
 
     def export(

{lsst_daf_butler-29.2025.3500 → lsst_daf_butler-29.2025.3700}/python/lsst/daf/butler/tests/utils.py

@@ -381,8 +381,8 @@ class MetricTestRepo:
             ),
         )
 
-        self.addDataset({"instrument": "DummyCamComp", "visit": 423})
-        self.addDataset({"instrument": "DummyCamComp", "visit": 424})
+        self.ref1 = self.addDataset({"instrument": "DummyCamComp", "visit": 423})
+        self.ref2 = self.addDataset({"instrument": "DummyCamComp", "visit": 424})
 
     def addDataset(
         self, dataId: dict[str, Any], run: str | None = None, datasetType: DatasetType | None = None

lsst_daf_butler-29.2025.3700/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.3700"

{lsst_daf_butler-29.2025.3500 → lsst_daf_butler-29.2025.3700/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.3500
+Version: 29.2025.3700
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License