lsst-daf-butler 29.2025.1600__tar.gz → 29.2025.1700__tar.gz

{lsst_daf_butler-29.2025.1600/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.1700}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.1600
+Version: 29.2025.1700
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_daf_butler-29.2025.1600 → lsst_daf_butler-29.2025.1700}/python/lsst/daf/butler/dimensions/_universe.py

@@ -31,6 +31,7 @@ __all__ = ["DimensionUniverse"]
 
 import logging
 import pickle
+import warnings
 from collections import defaultdict
 from collections.abc import Iterable, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, overload
@@ -117,6 +118,13 @@ class DimensionUniverse:  # numpydoc ignore=PR02
             else:
                 version = builder.version
 
+        if use_cache is not True:
+            warnings.warn(
+                "use_cache parameter is no longer supported and is ignored. Will be removed after v30.",
+                category=FutureWarning,
+                stacklevel=2,
+            )
+
         # Then a namespace.
         if namespace is None:
             if builder is None:
@@ -129,10 +137,9 @@ class DimensionUniverse:  # numpydoc ignore=PR02
             namespace = _DEFAULT_NAMESPACE
 
         # See if an equivalent instance already exists.
-        if use_cache:
-            existing_instance = cls._instances.get((version, namespace))
-            if existing_instance is not None:
-                return existing_instance
+        existing_instance = cls._instances.get((version, namespace))
+        if existing_instance is not None:
+            return existing_instance
 
         # Ensure we have a builder, building one from config if necessary.
         if builder is None:
@@ -183,10 +190,7 @@ class DimensionUniverse:  # numpydoc ignore=PR02
             if element.populated_by is not None:
                 self._populates[element.populated_by.name].add(element)
 
-        if use_cache:
-            return cls._instances.set_or_get((self._version, self._namespace), self)
-        else:
-            return self
+        return cls._instances.set_or_get((self._version, self._namespace), self)
 
     @property
     def version(self) -> int:

{lsst_daf_butler-29.2025.1600 → lsst_daf_butler-29.2025.1700}/python/lsst/daf/butler/direct_butler/_direct_butler.py

@@ -1587,30 +1587,24 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
     def _ingest_file_datasets(
         self,
         datasets: Sequence[FileDataset],
+        *,
+        dry_run: bool = False,
     ) -> None:
-        # Docstring inherited.
         if not datasets:
             return
 
         progress = Progress("lsst.daf.butler.Butler.ingest", level=logging.DEBUG)
 
-        # We need to reorganize all the inputs so that they are grouped
-        # by dataset type and run. Multiple refs in a single FileDataset
-        # are required to share the run and dataset type.
-        groupedData: MutableMapping[tuple[DatasetType, str], list[FileDataset]] = defaultdict(list)
-
         # Track DataIDs that are being ingested so we can spot issues early
         # with duplication. Retain previous FileDataset so we can report it.
         groupedDataIds: MutableMapping[tuple[DatasetType, str], dict[DataCoordinate, FileDataset]] = (
             defaultdict(dict)
         )
 
-        # And the nested loop that populates it:
-        for dataset in progress.wrap(datasets, desc="Grouping by dataset type"):
-            # Somewhere to store pre-existing refs if we have an
-            # execution butler.
-            existingRefs: list[DatasetRef] = []
+        # All the refs we need to import.
+        refs: list[DatasetRef] = []
 
+        for dataset in progress.wrap(datasets, desc="Validating dataIDs"):
             for ref in dataset.refs:
                 group_key = (ref.datasetType, ref.run)
 
@@ -1623,54 +1617,29 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
                     )
 
                 groupedDataIds[group_key][ref.dataId] = dataset
+            refs.extend(dataset.refs)
 
-            if existingRefs:
-                if len(dataset.refs) != len(existingRefs):
-                    # Keeping track of partially pre-existing datasets is hard
-                    # and should generally never happen. For now don't allow
-                    # it.
-                    raise ConflictingDefinitionError(
-                        f"For dataset {dataset.path} some dataIds already exist"
-                        " in registry but others do not. This is not supported."
-                    )
-
-                # Store expanded form in the original FileDataset.
-                dataset.refs = existingRefs
-            else:
-                groupedData[group_key].append(dataset)
-
-        # Now we can bulk-insert into Registry for each DatasetType.
-        for (datasetType, this_run), grouped_datasets in progress.iter_item_chunks(
-            groupedData.items(), desc="Bulk-inserting datasets by type"
-        ):
-            refs_to_import = []
-            for dataset in grouped_datasets:
-                refs_to_import.extend(dataset.refs)
-
-            n_refs = len(refs_to_import)
-            _LOG.verbose(
-                "Importing %d ref%s of dataset type %r into run %r",
-                n_refs,
-                "" if n_refs == 1 else "s",
-                datasetType.name,
-                this_run,
-            )
+        # Bulk insert.
+        import_info = self._prepare_for_import_refs(
+            self,
+            refs,
+            skip_missing=False,
+            register_dataset_types=True,
+            dry_run=dry_run,
+            transfer_dimensions=False,
+        )
 
-            # Import the refs and expand the DataCoordinates since we can't
-            # guarantee that they are expanded and Datastore will need
-            # the records.
-            imported_refs = self._registry._importDatasets(refs_to_import, expand=True)
+        self._import_dimension_records(import_info.dimension_records, dry_run=dry_run)
+        imported_refs = self._import_grouped_refs(
+            import_info.grouped_refs, self, progress, dry_run=dry_run, expand_refs=True
+        )
 
-            # Since refs can come from an external butler, there is no
-            # guarantee that the DatasetType values match. Compare IDs.
-            assert {ref.id for ref in imported_refs} == {ref.id for ref in refs_to_import}
+        # The expanded refs need to be attached back to the original
+        # FileDatasets for datastore to use.
+        id_to_ref = {ref.id: ref for ref in imported_refs}
 
-            # Replace all the refs in the FileDataset with expanded versions.
-            # Pull them off in the order we put them on the list.
-            for dataset in grouped_datasets:
-                n_dataset_refs = len(dataset.refs)
-                dataset.refs = imported_refs[:n_dataset_refs]
-                del imported_refs[:n_dataset_refs]
+        for dataset in progress.wrap(datasets, desc="Re-attaching expanded refs"):
+            dataset.refs = [id_to_ref[ref.id] for ref in dataset.refs]
 
     @transactional
     def ingest(
@@ -1933,20 +1902,19 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
 
         return dimension_records
 
-    def transfer_from(
+    def _prepare_for_import_refs(
         self,
         source_butler: LimitedButler,
         source_refs: Iterable[DatasetRef],
-        transfer: str = "auto",
+        *,
         skip_missing: bool = True,
         register_dataset_types: bool = False,
         transfer_dimensions: bool = False,
         dry_run: bool = False,
-    ) -> collections.abc.Collection[DatasetRef]:
+    ) -> _ImportDatasetsInfo:
         # Docstring inherited.
         if not self.isWriteable():
             raise TypeError("Butler is read-only.")
-        progress = Progress("lsst.daf.butler.Butler.transfer_from", level=VERBOSE)
 
         # Will iterate through the refs multiple times so need to convert
         # to a list if this isn't a collection.
@@ -1954,7 +1922,7 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
             source_refs = list(source_refs)
 
         original_count = len(source_refs)
-        _LOG.info("Transferring %d datasets into %s", original_count, str(self))
+        _LOG.info("Importing %d datasets into %s", original_count, str(self))
 
         # In some situations the datastore artifact may be missing
         # and we do not want that registry entry to be imported.
@@ -1996,7 +1964,7 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
                 # might result in additional unwanted dataset types being
                 # registered.
                 try:
-                    if self._registry.registerDatasetType(datasetType):
+                    if not dry_run and self._registry.registerDatasetType(datasetType):
                         newly_registered_dataset_types.add(datasetType)
                 except ConflictingDefinitionError as e:
                     # Be safe and require that conversions be bidirectional
@@ -2076,86 +2044,132 @@ class DirectButler(Butler):  # numpydoc ignore=PR02
             dimension_records = self._extract_all_dimension_records_from_data_ids(
                 source_butler, dataIds, elements
             )
+        return _ImportDatasetsInfo(grouped_refs, dimension_records, artifact_existence)
 
+    def _import_dimension_records(
+        self,
+        dimension_records: dict[DimensionElement, dict[DataCoordinate, DimensionRecord]],
+        *,
+        dry_run: bool,
+    ) -> None:
+        """Import dimension records collected during import pre-process."""
+        if dimension_records and not dry_run:
+            _LOG.verbose("Ensuring that dimension records exist for transferred datasets.")
+            # Order matters.
+            for element in self.dimensions.sorted(dimension_records.keys()):
+                records = list(dimension_records[element].values())
+                # Assume that if the record is already present that we can
+                # use it without having to check that the record metadata
+                # is consistent.
+                self._registry.insertDimensionData(element, *records, skip_existing=True)
+
+    def _import_grouped_refs(
+        self,
+        grouped_refs: defaultdict[_RefGroup, list[DatasetRef]],
+        source_butler: LimitedButler,
+        progress: Progress,
+        *,
+        dry_run: bool = False,
+        expand_refs: bool = False,
+    ) -> list[DatasetRef]:
         handled_collections: set[str] = set()
-
-        # Do all the importing in a single transaction.
-        with self.transaction():
-            if dimension_records and not dry_run:
-                _LOG.verbose("Ensuring that dimension records exist for transferred datasets.")
-                # Order matters.
-                for element in self.dimensions.sorted(dimension_records.keys()):
-                    records = [r for r in dimension_records[element].values()]
-                    # Assume that if the record is already present that we can
-                    # use it without having to check that the record metadata
-                    # is consistent.
-                    self._registry.insertDimensionData(element, *records, skip_existing=True)
-
-            n_imported = 0
-
-            # Sort by run collection name to ensure Postgres takes locks in the
-            # same order between different processes, to mitigate an issue
-            # where Postgres can deadlock due to the unique index on collection
-            # name. (See DM-47543).
-            groups = sorted(grouped_refs.items(), key=lambda item: item[0].run)
-            for (dimension_group, run), refs_to_import in progress.iter_item_chunks(
-                groups, desc="Importing to registry by run and dataset type"
-            ):
-                if run not in handled_collections:
-                    # May need to create output collection. If source butler
-                    # has a registry, ask for documentation string.
-                    run_doc = None
-                    if registry := getattr(source_butler, "registry", None):
-                        run_doc = registry.getCollectionDocumentation(run)
-                    if not dry_run:
-                        registered = self.collections.register(run, doc=run_doc)
-                    else:
-                        registered = True
-                    handled_collections.add(run)
-                    if registered:
-                        _LOG.verbose("Creating output run %s", run)
-
-                n_refs = len(refs_to_import)
-                _LOG.verbose(
-                    "Importing %d ref%s with dimensions %s into run %s",
-                    n_refs,
-                    "" if n_refs == 1 else "s",
-                    dimension_group.names,
-                    run,
-                )
-
-                # Assume we are using UUIDs and the source refs will match
-                # those imported.
+        n_to_import = 0
+        all_imported_refs: list[DatasetRef] = []
+        # Sort by run collection name to ensure Postgres takes locks in the
+        # same order between different processes, to mitigate an issue
+        # where Postgres can deadlock due to the unique index on collection
+        # name. (See DM-47543).
+        groups = sorted(grouped_refs.items(), key=lambda item: item[0].run)
+        for (dimension_group, run), refs_to_import in progress.iter_item_chunks(
+            groups, desc="Importing to registry by run and dataset type"
+        ):
+            if run not in handled_collections:
+                # May need to create output collection. If source butler
+                # has a registry, ask for documentation string.
+                run_doc = None
+                if registry := getattr(source_butler, "registry", None):
+                    run_doc = registry.getCollectionDocumentation(run)
                 if not dry_run:
-                    imported_refs = self._registry._importDatasets(refs_to_import)
+                    registered = self.collections.register(run, doc=run_doc)
                 else:
-                    imported_refs = refs_to_import
+                    registered = True
+                handled_collections.add(run)
+                if registered:
+                    _LOG.verbose("Creating output run %s", run)
+
+            n_refs = len(refs_to_import)
+            n_to_import += n_refs
+            _LOG.verbose(
+                "Importing %d ref%s with dimensions %s into run %s",
+                n_refs,
+                "" if n_refs == 1 else "s",
+                dimension_group.names,
+                run,
+            )
+
+            # Assume we are using UUIDs and the source refs will match
+            # those imported.
+            if not dry_run:
+                imported_refs = self._registry._importDatasets(refs_to_import, expand=expand_refs)
+            else:
+                imported_refs = refs_to_import
+
+            all_imported_refs.extend(imported_refs)
+
+        assert n_to_import == len(all_imported_refs)
+        _LOG.verbose("Imported %d datasets into destination butler", n_to_import)
+        return all_imported_refs
+
+    def transfer_from(
+        self,
+        source_butler: LimitedButler,
+        source_refs: Iterable[DatasetRef],
+        transfer: str = "auto",
+        skip_missing: bool = True,
+        register_dataset_types: bool = False,
+        transfer_dimensions: bool = False,
+        dry_run: bool = False,
+    ) -> collections.abc.Collection[DatasetRef]:
+        # Docstring inherited.
+        if not self.isWriteable():
+            raise TypeError("Butler is read-only.")
 
-                n_imported += len(imported_refs)
+        progress = Progress("lsst.daf.butler.Butler.transfer_from", level=VERBOSE)
 
-            assert len(source_refs) == n_imported
-            _LOG.verbose("Imported %d datasets into destination butler", n_imported)
+        import_info = self._prepare_for_import_refs(
+            source_butler,
+            source_refs,
+            skip_missing=skip_missing,
+            register_dataset_types=register_dataset_types,
+            dry_run=dry_run,
+            transfer_dimensions=transfer_dimensions,
+        )
+
+        # Do all the importing in a single transaction.
+        with self.transaction():
+            self._import_dimension_records(import_info.dimension_records, dry_run=dry_run)
+            imported_refs = self._import_grouped_refs(
+                import_info.grouped_refs, source_butler, progress, dry_run=dry_run
+            )
 
             # Ask the datastore to transfer. The datastore has to check that
             # the source datastore is compatible with the target datastore.
             accepted, rejected = self._datastore.transfer_from(
                 source_butler._datastore,
-                source_refs,
+                imported_refs,
                 transfer=transfer,
-                artifact_existence=artifact_existence,
+                artifact_existence=import_info.artifact_existence,
                 dry_run=dry_run,
             )
             if rejected:
                 # For now, accept the registry entries but not the files.
                 _LOG.warning(
-                    "%d datasets were rejected and %d accepted for dataset type %s in run %r.",
+                    "%d datasets were rejected and %d accepted for transfer.",
                     len(rejected),
                     len(accepted),
-                    datasetType,
-                    run,
                 )
 
-        return source_refs
+        return imported_refs
 
     def validateConfiguration(
         self,
@@ -2382,3 +2396,11 @@ class _RefGroup(NamedTuple):
 
     dimensions: DimensionGroup
     run: str
+
+
+class _ImportDatasetsInfo(NamedTuple):
+    """Information extracted from datasets to be imported."""
+
+    grouped_refs: defaultdict[_RefGroup, list[DatasetRef]]
+    dimension_records: dict[DimensionElement, dict[DataCoordinate, DimensionRecord]]
+    artifact_existence: dict[ResourcePath, bool]

{lsst_daf_butler-29.2025.1600 → lsst_daf_butler-29.2025.1700}/python/lsst/daf/butler/remote_butler/_remote_butler.py

@@ -187,14 +187,7 @@ class RemoteButler(Butler):  # numpydoc ignore=PR02
         model = parse_model(response, GetUniverseResponseModel)
 
         config = DimensionConfig.from_simple(model.universe)
-        universe = DimensionUniverse(
-            config,
-            # The process-global cache internal to DimensionUniverse can mask
-            # problems in unit tests, since client and server live in the same
-            # process for these tests.  We are doing our own caching, so we
-            # don't benefit from it.  So just disable it.
-            use_cache=False,
-        )
+        universe = DimensionUniverse(config)
         with self._cache.access() as cache:
             if cache.dimensions is None:
                 cache.dimensions = universe

lsst_daf_butler-29.2025.1700/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.1700"

{lsst_daf_butler-29.2025.1600 → lsst_daf_butler-29.2025.1700/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.1600
+Version: 29.2025.1700
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

lsst_daf_butler-29.2025.1600/python/lsst/daf/butler/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.1600"