lsst-daf-butler 29.2025.3900__tar.gz → 29.2025.4000__tar.gz

{lsst_daf_butler-29.2025.3900/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.4000}/PKG-INFO

@@ -1,13 +1,12 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.3900
+Version: 29.2025.4000
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
-License: BSD 3-Clause License
+License-Expression: BSD-3-Clause OR GPL-3.0-or-later
 Project-URL: Homepage, https://github.com/lsst/daf_butler
 Keywords: lsst
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/pyproject.toml

@@ -6,14 +6,14 @@ build-backend = "setuptools.build_meta"
 name = "lsst-daf-butler"
 requires-python = ">=3.11.0"
 description = "An abstraction layer for reading and writing astronomical data to datastores."
-license = {text = "BSD 3-Clause License"}
+license = "BSD-3-Clause OR GPL-3.0-or-later"
+license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
 readme = "README.md"
 authors = [
     {name="Rubin Observatory Data Management", email="dm-admin@lists.lsst.org"},
 ]
 classifiers = [
     "Intended Audience :: Science/Research",
-    "License :: OSI Approved :: BSD License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.12",
@@ -74,7 +74,6 @@ where = ["python"]
 
 [tool.setuptools]
 zip-safe = true
-license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
 
 [tool.setuptools.package-data]
 "lsst.daf.butler" = ["py.typed", "configs/*.yaml", "configs/*/*.yaml", "tests/registry_data/*.yaml"]

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/_quantum_backed.py

@@ -614,6 +614,34 @@ class QuantumBackedButler(LimitedButler):
             datastore_records=provenance_records,
         )
 
+    def export_predicted_datastore_records(
+        self, refs: Iterable[DatasetRef]
+    ) -> dict[str, DatastoreRecordData]:
+        """Export datastore records for a set of predicted output dataset
+        references.
+
+        Parameters
+        ----------
+        refs : `~collections.abc.Iterable` [ `DatasetRef` ]
+            Dataset references for which to export datastore records. These
+            refs must be known to this butler.
+
+        Returns
+        -------
+        records : `dict` [ `str`, `DatastoreRecordData` ]
+            Predicted datastore records indexed by datastore name. No attempt
+            is made to ensure that the associated datasets exist on disk.
+        """
+        unknowns = [
+            ref
+            for ref in refs
+            if (ref.id not in self._predicted_outputs and ref.id not in self._predicted_inputs)
+        ]
+        if unknowns:
+            raise ValueError(f"Cannot export datastore records for unknown outputs: {unknowns}")
+
+        return self._datastore.export_predicted_records(refs)
+
 
 class QuantumProvenanceData(pydantic.BaseModel):
     """A serializable struct for per-quantum provenance information and

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/configs/datastores/formatters.yaml

@@ -95,3 +95,4 @@ RegionTimeInfo: lsst.daf.butler.formatters.json.JsonFormatter
 QPEnsemble: lsst.meas.pz.qp_formatter.QPFormatter
 PZModel: lsst.meas.pz.model_formatter.ModelFormatter
 VisitBackgroundModel: lsst.daf.butler.formatters.json.JsonFormatter
+VignettingCorrection: lsst.ts.observatory.control.utils.extras.vignetting_storage.VignettingCorrectionFormatter

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/configs/storageClasses.yaml

@@ -428,3 +428,5 @@ storageClasses:
     pytype: rail.core.model.Model
   VisitBackgroundModel:
     pytype: lsst.drp.tasks.fit_visit_background.VisitBackgroundModel
+  VignettingCorrection:
+    pytype: lsst.ts.observatory.control.utils.extras.vignetting_correction.VignettingCorrection

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/datastore/_datastore.py

@@ -1381,6 +1381,24 @@ class Datastore(FileTransferSource, metaclass=ABCMeta):
         """
         raise NotImplementedError()
 
+    def export_predicted_records(self, refs: Iterable[DatasetRef]) -> dict[str, DatastoreRecordData]:
+        """Export predicted datastore records and locations to an in-memory
+        data structure.
+
+        Parameters
+        ----------
+        refs : `~collections.abc.Iterable` [ `DatasetRef` ]
+            Datastore records that would be used if the given refs were to
+            exist in this datastore.  No attempt is made to determine if these
+            datasets actually exist.
+
+        Returns
+        -------
+        data : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ]
+            Exported datastore records indexed by datastore name.
+        """
+        raise NotImplementedError()
+
     def set_retrieve_dataset_type_method(self, method: Callable[[str], DatasetType | None] | None) -> None:
         """Specify a method that can be used by datastore to retrieve
         registry-defined dataset type.

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/datastores/chainedDatastore.py

@@ -1165,6 +1165,25 @@ class ChainedDatastore(Datastore):
 
         return all_records
 
+    def export_predicted_records(self, refs: Iterable[DatasetRef]) -> dict[str, DatastoreRecordData]:
+        # Docstring inherited from the base class.
+
+        all_records: dict[str, DatastoreRecordData] = {}
+
+        # Filter out datasets that this datastore is not allowed to contain.
+        refs = [ref for ref in refs if self.constraints.isAcceptable(ref)]
+
+        # Merge all sub-datastore records into one structure
+        for datastore in self.datastores:
+            sub_records = datastore.export_predicted_records(refs)
+            for name, record_data in sub_records.items():
+                # All datastore names must be unique in a chain.
+                if name in all_records:
+                    raise ValueError("Non-unique datastore name found in datastore {datastore}")
+                all_records[name] = record_data
+
+        return all_records
+
     def export(
         self,
         refs: Iterable[DatasetRef],

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/datastores/fileDatastore.py

@@ -3188,6 +3188,24 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
         record_data = DatastoreRecordData(records=records)
         return {self.name: record_data}
 
+    def export_predicted_records(self, refs: Iterable[DatasetRef]) -> dict[str, DatastoreRecordData]:
+        # Docstring inherited from the base class.
+        refs = [self._cast_storage_class(ref) for ref in refs]
+        records: dict[DatasetId, dict[str, list[StoredDatastoreItemInfo]]] = {}
+        for ref in refs:
+            if not self.constraints.isAcceptable(ref):
+                continue
+            fileLocations = self._get_expected_dataset_locations_info(ref)
+            if not fileLocations:
+                continue
+            dataset_records = records.setdefault(ref.id, {})
+            dataset_records.setdefault(self._table.name, [])
+            for _, storedFileInfo in fileLocations:
+                dataset_records[self._table.name].append(storedFileInfo)
+
+        record_data = DatastoreRecordData(records=records)
+        return {self.name: record_data}
+
     def set_retrieve_dataset_type_method(self, method: Callable[[str], DatasetType | None] | None) -> None:
         # Docstring inherited from the base class.
         self._retrieve_dataset_method = method

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/datastores/inMemoryDatastore.py

@@ -763,6 +763,12 @@ class InMemoryDatastore(GenericBaseDatastore[StoredMemoryItemInfo]):
         # In-memory Datastore records cannot be exported or imported
         return {}
 
+    def export_predicted_records(self, refs: Iterable[DatasetIdRef]) -> dict[str, DatastoreRecordData]:
+        # Docstring inherited from the base class.
+
+        # In-memory Datastore records cannot be exported or imported
+        return {}
+
     def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]:
         # Docstring inherited from the base class.
         return {}

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/queries/_expression_strings.py

@@ -204,8 +204,6 @@ class _ConversionVisitor(TreeVisitor[_VisitorResult]):
         return result
 
     def visitIdentifier(self, name: str, node: Node) -> _VisitorResult:
-        name = name.lower()
-
         if name in self.context.bind:
             value = self.context.bind[name]
             # Lists of values do not have a direct representation in the new
@@ -242,7 +240,6 @@ class _ConversionVisitor(TreeVisitor[_VisitorResult]):
             return _ColExpr(column_expression)
 
     def visitBind(self, name: str, node: Node) -> _VisitorResult:
-        name = name.lower()
         if name not in self.context.bind:
             raise InvalidQueryError("Name {name!r} is not in the bind map.")
         # Logic in visitIdentifier handles binds.

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/queries/_identifiers.py

@@ -71,8 +71,7 @@ class IdentifierContext:  # numpydoc ignore=PR01
         if bind is None:
             self.bind = {}
         else:
-            # Make bind names case-insensitive.
-            self.bind = {k.lower(): v for k, v in bind.items()}
+            self.bind = dict(bind)
             if len(self.bind.keys()) != len(bind.keys()):
                 raise ValueError(f"Duplicate keys present in bind: {bind.keys()}")
 
@@ -96,9 +95,6 @@ def interpret_identifier(context: IdentifierContext, identifier: str) -> ColumnE
     dimensions = context.dimensions
     datasets = context.datasets
     bind = context.bind
-    # Make identifiers case-insensitive.
-    identifier = identifier.lower()
-
     if identifier in bind:
         return make_column_literal(bind[identifier])
     terms = identifier.split(".")

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/queries/tree/_base.py

@@ -110,7 +110,7 @@ def is_dataset_field(s: str) -> TypeGuard[DatasetFieldName]:
 class QueryTreeBase(pydantic.BaseModel):
     """Base class for all non-primitive types in a query tree."""
 
-    model_config = pydantic.ConfigDict(frozen=True, extra="forbid", strict=True)
+    model_config = pydantic.ConfigDict(frozen=True, strict=True)
 
 
 class ColumnExpressionBase(QueryTreeBase, ABC):

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/tests/utils.py

@@ -46,6 +46,7 @@ from lsst.resources import ResourcePathExpression
 from .. import Butler, ButlerConfig, Config, DatasetRef, StorageClassFactory, Timespan
 from .._collection_type import CollectionType
 from ..datastore import NullDatastore
+from ..dimensions import DimensionConfig
 from ..direct_butler import DirectButler
 from ..registry.sql_registry import RegistryConfig, SqlRegistry
 from ..tests import MetricsExample, addDatasetType
@@ -130,7 +131,9 @@ def safeTestTempDir(default_base: str) -> Iterator[str]:
 
 
 def create_populated_sqlite_registry(
-    *args: ResourcePathExpression, registry_config: RegistryConfig | None = None
+    *args: ResourcePathExpression,
+    registry_config: RegistryConfig | None = None,
+    dimension_config: DimensionConfig | None = None,
 ) -> Butler:
     """Create an in-memory registry-only sqlite butler and populate it.
 
@@ -141,6 +144,8 @@ def create_populated_sqlite_registry(
     registry_config : ``RegistryConfig``, optional
         Registry configuration to use as the basis for the Butler
         configuration.
+    dimension_config : ``DimensionConfig``, optional
+        Dimension universe configuration.
 
     Returns
     -------
@@ -151,7 +156,7 @@ def create_populated_sqlite_registry(
     if registry_config is not None:
         config["registry"] = registry_config
     config[".registry.db"] = "sqlite://"
-    registry = SqlRegistry.createFromConfig(config["registry"])
+    registry = SqlRegistry.createFromConfig(config["registry"], dimension_config)
     butler = DirectButler(
         config=config,
         registry=registry,

lsst_daf_butler-29.2025.4000/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.4000"

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,13 +1,12 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.3900
+Version: 29.2025.4000
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
-License: BSD 3-Clause License
+License-Expression: BSD-3-Clause OR GPL-3.0-or-later
 Project-URL: Homepage, https://github.com/lsst/daf_butler
 Keywords: lsst
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/tests/test_datastore.py

@@ -988,6 +988,30 @@ class DatastoreTests(DatastoreTestsBase):
         data = datastore2.get(refs[2])
         self.assertIsNotNone(data)
 
+    def testExportPredictedRecords(self):
+        if self.isEphemeral:
+            raise unittest.SkipTest("in-memory datastore does not support record export/import")
+        sc = self.storageClassFactory.getStorageClass("ThingOne")
+        dimensions = self.universe.conform(("visit", "physical_filter"))
+        dataId = {
+            "instrument": "dummy",
+            "visit": 52,
+            "physical_filter": "V",
+            "band": "v",
+            "day_obs": 20250101,
+        }
+        ref = self.makeDatasetRef("metric", dimensions, sc, dataId)
+
+        datastore = self.makeDatastore("test_datastore")
+        names = {n for n in datastore.names if not n.startswith("InMemory")}
+        records = datastore.export_predicted_records([ref])
+
+        # Expect predicted records from all datastores.
+        self.assertEqual(set(records.keys()), names)
+
+        for record_data in records.values():
+            self.assertEqual(len(record_data.records), 1)
+
     def testExport(self) -> None:
         datastore, refs = self._populate_export_datastore("test_datastore")
 
@@ -1414,6 +1438,7 @@ class ChainedDatastoreMemoryTestCase(InMemoryDatastoreTestCase):
 
     configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2.yaml")
     validationCanFail = False
+    isEphemeral = True
 
 
 class DatastoreConstraintsTests(DatastoreTestsBase):
@@ -2124,6 +2149,8 @@ class NullDatastoreTestCase(DatasetTestHelper, unittest.TestCase):
             null.import_records({})
         with self.assertRaises(NotImplementedError):
             null.export_records([])
+        with self.assertRaises(NotImplementedError):
+            null.export_predicted_records([])
         with self.assertRaises(NotImplementedError):
             null.export([ref])
         with self.assertRaises(NotImplementedError):

{lsst_daf_butler-29.2025.3900 → lsst_daf_butler-29.2025.4000}/tests/test_quantumBackedButler.py

@@ -416,6 +416,19 @@ class QuantumBackedButlerTestCase(unittest.TestCase):
             for dataset_data in datastore_records.records[class_name].values():
                 self.assertEqual(set(dataset_data), {table_name})
 
+    def test_export_predicted_datastore_records(self) -> None:
+        """Test for export_predicted_datastore_records method"""
+        quantum = self.make_quantum()
+        qbb = QuantumBackedButler.initialize(
+            config=self.config, quantum=quantum, dimensions=self.universe, dataset_types=self.dataset_types
+        )
+
+        records = qbb.export_predicted_datastore_records(self.output_refs)
+        self.assertEqual(len(records["FileDatastore@<butlerRoot>/datastore"].records), len(self.output_refs))
+
+        with self.assertRaises(ValueError):
+            qbb.export_predicted_datastore_records(self.missing_refs)
+
     def test_collect_and_transfer(self) -> None:
         """Test for collect_and_transfer method"""
         quantum1 = self.make_quantum(1)

lsst_daf_butler-29.2025.3900/python/lsst/daf/butler/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.3900"