lsst-daf-butler 29.2025.3800__tar.gz → 29.2025.4000__tar.gz

{lsst_daf_butler-29.2025.3800/python/lsst_daf_butler.egg-info → lsst_daf_butler-29.2025.4000}/PKG-INFO

@@ -1,13 +1,12 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.3800
+Version: 29.2025.4000
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
-License: BSD 3-Clause License
+License-Expression: BSD-3-Clause OR GPL-3.0-or-later
 Project-URL: Homepage, https://github.com/lsst/daf_butler
 Keywords: lsst
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/pyproject.toml

@@ -6,14 +6,14 @@ build-backend = "setuptools.build_meta"
 name = "lsst-daf-butler"
 requires-python = ">=3.11.0"
 description = "An abstraction layer for reading and writing astronomical data to datastores."
-license = {text = "BSD 3-Clause License"}
+license = "BSD-3-Clause OR GPL-3.0-or-later"
+license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
 readme = "README.md"
 authors = [
     {name="Rubin Observatory Data Management", email="dm-admin@lists.lsst.org"},
 ]
 classifiers = [
     "Intended Audience :: Science/Research",
-    "License :: OSI Approved :: BSD License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.12",
@@ -74,7 +74,6 @@ where = ["python"]
 
 [tool.setuptools]
 zip-safe = true
-license-files = ["COPYRIGHT", "LICENSE", "bsd_license.txt", "gpl-v3.0.txt"]
 
 [tool.setuptools.package-data]
 "lsst.daf.butler" = ["py.typed", "configs/*.yaml", "configs/*/*.yaml", "tests/registry_data/*.yaml"]

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/_quantum_backed.py

@@ -614,6 +614,34 @@ class QuantumBackedButler(LimitedButler):
             datastore_records=provenance_records,
         )
 
+    def export_predicted_datastore_records(
+        self, refs: Iterable[DatasetRef]
+    ) -> dict[str, DatastoreRecordData]:
+        """Export datastore records for a set of predicted output dataset
+        references.
+
+        Parameters
+        ----------
+        refs : `~collections.abc.Iterable` [ `DatasetRef` ]
+            Dataset references for which to export datastore records. These
+            refs must be known to this butler.
+
+        Returns
+        -------
+        records : `dict` [ `str`, `DatastoreRecordData` ]
+            Predicted datastore records indexed by datastore name. No attempt
+            is made to ensure that the associated datasets exist on disk.
+        """
+        unknowns = [
+            ref
+            for ref in refs
+            if (ref.id not in self._predicted_outputs and ref.id not in self._predicted_inputs)
+        ]
+        if unknowns:
+            raise ValueError(f"Cannot export datastore records for unknown outputs: {unknowns}")
+
+        return self._datastore.export_predicted_records(refs)
+
 
 class QuantumProvenanceData(pydantic.BaseModel):
     """A serializable struct for per-quantum provenance information and

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/cli/utils.py

@@ -74,6 +74,7 @@ from typing import TYPE_CHECKING, Any, cast
 from unittest.mock import patch
 
 import click
+import click.core
 import click.exceptions
 import click.testing
 import yaml
@@ -95,6 +96,11 @@ if _click_version >= Version("8.2.0"):
 else:
     _click_make_metavar_has_context = False
 
+# Starting from Click 8.3.0, a special `UNSET` sentinel value is used to
+# indicate the absence of a default value for a parameter.  Prior to 8.3.0,
+# they just used `None`.
+_CLICK_UNSET_SENTINEL = getattr(click.core, "UNSET", None)
+
 log = logging.getLogger(__name__)
 
 # This is used as the metavar argument to Options that accept multiple string
@@ -554,7 +560,7 @@ def split_kv(
 
     if add_to_default:
         default = param.get_default(context)
-        if default:
+        if default and default != _CLICK_UNSET_SENTINEL:
             vals = tuple(v for v in itertools.chain(default, vals))  # Convert to tuple for mypy
 
     ret: RetDict | RetTuple
@@ -1013,7 +1019,8 @@ class MWCommand(click.Command):
                     captured_args.append(val)
             elif isinstance(param, click.Argument):
                 param_name = cast(str, param.name)
-                if (opt := opts[param_name]) is not None:
+                opt = opts[param_name]
+                if opt is not None and opt != _CLICK_UNSET_SENTINEL:
                     captured_args.append(opt)
             else:
                 raise AssertionError("All parameters should be an Option or an Argument")

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/configs/datastores/formatters.yaml

@@ -95,3 +95,4 @@ RegionTimeInfo: lsst.daf.butler.formatters.json.JsonFormatter
 QPEnsemble: lsst.meas.pz.qp_formatter.QPFormatter
 PZModel: lsst.meas.pz.model_formatter.ModelFormatter
 VisitBackgroundModel: lsst.daf.butler.formatters.json.JsonFormatter
+VignettingCorrection: lsst.ts.observatory.control.utils.extras.vignetting_storage.VignettingCorrectionFormatter

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/configs/storageClasses.yaml

@@ -428,3 +428,5 @@ storageClasses:
     pytype: rail.core.model.Model
   VisitBackgroundModel:
     pytype: lsst.drp.tasks.fit_visit_background.VisitBackgroundModel
+  VignettingCorrection:
+    pytype: lsst.ts.observatory.control.utils.extras.vignetting_correction.VignettingCorrection

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/datastore/_datastore.py

@@ -1381,6 +1381,24 @@ class Datastore(FileTransferSource, metaclass=ABCMeta):
         """
         raise NotImplementedError()
 
+    def export_predicted_records(self, refs: Iterable[DatasetRef]) -> dict[str, DatastoreRecordData]:
+        """Export predicted datastore records and locations to an in-memory
+        data structure.
+
+        Parameters
+        ----------
+        refs : `~collections.abc.Iterable` [ `DatasetRef` ]
+            Datastore records that would be used if the given refs were to
+            exist in this datastore.  No attempt is made to determine if these
+            datasets actually exist.
+
+        Returns
+        -------
+        data : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ]
+            Exported datastore records indexed by datastore name.
+        """
+        raise NotImplementedError()
+
     def set_retrieve_dataset_type_method(self, method: Callable[[str], DatasetType | None] | None) -> None:
         """Specify a method that can be used by datastore to retrieve
         registry-defined dataset type.

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/datastores/chainedDatastore.py

@@ -1165,6 +1165,25 @@ class ChainedDatastore(Datastore):
 
         return all_records
 
+    def export_predicted_records(self, refs: Iterable[DatasetRef]) -> dict[str, DatastoreRecordData]:
+        # Docstring inherited from the base class.
+
+        all_records: dict[str, DatastoreRecordData] = {}
+
+        # Filter out datasets that this datastore is not allowed to contain.
+        refs = [ref for ref in refs if self.constraints.isAcceptable(ref)]
+
+        # Merge all sub-datastore records into one structure
+        for datastore in self.datastores:
+            sub_records = datastore.export_predicted_records(refs)
+            for name, record_data in sub_records.items():
+                # All datastore names must be unique in a chain.
+                if name in all_records:
+                    raise ValueError("Non-unique datastore name found in datastore {datastore}")
+                all_records[name] = record_data
+
+        return all_records
+
     def export(
         self,
         refs: Iterable[DatasetRef],

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/datastores/fileDatastore.py

@@ -3188,6 +3188,24 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
         record_data = DatastoreRecordData(records=records)
         return {self.name: record_data}
 
+    def export_predicted_records(self, refs: Iterable[DatasetRef]) -> dict[str, DatastoreRecordData]:
+        # Docstring inherited from the base class.
+        refs = [self._cast_storage_class(ref) for ref in refs]
+        records: dict[DatasetId, dict[str, list[StoredDatastoreItemInfo]]] = {}
+        for ref in refs:
+            if not self.constraints.isAcceptable(ref):
+                continue
+            fileLocations = self._get_expected_dataset_locations_info(ref)
+            if not fileLocations:
+                continue
+            dataset_records = records.setdefault(ref.id, {})
+            dataset_records.setdefault(self._table.name, [])
+            for _, storedFileInfo in fileLocations:
+                dataset_records[self._table.name].append(storedFileInfo)
+
+        record_data = DatastoreRecordData(records=records)
+        return {self.name: record_data}
+
     def set_retrieve_dataset_type_method(self, method: Callable[[str], DatasetType | None] | None) -> None:
         # Docstring inherited from the base class.
         self._retrieve_dataset_method = method

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/datastores/inMemoryDatastore.py

@@ -763,6 +763,12 @@ class InMemoryDatastore(GenericBaseDatastore[StoredMemoryItemInfo]):
         # In-memory Datastore records cannot be exported or imported
         return {}
 
+    def export_predicted_records(self, refs: Iterable[DatasetIdRef]) -> dict[str, DatastoreRecordData]:
+        # Docstring inherited from the base class.
+
+        # In-memory Datastore records cannot be exported or imported
+        return {}
+
     def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]:
         # Docstring inherited from the base class.
         return {}

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/queries/_expression_strings.py

@@ -204,8 +204,6 @@ class _ConversionVisitor(TreeVisitor[_VisitorResult]):
         return result
 
     def visitIdentifier(self, name: str, node: Node) -> _VisitorResult:
-        name = name.lower()
-
         if name in self.context.bind:
             value = self.context.bind[name]
             # Lists of values do not have a direct representation in the new
@@ -242,7 +240,6 @@ class _ConversionVisitor(TreeVisitor[_VisitorResult]):
             return _ColExpr(column_expression)
 
     def visitBind(self, name: str, node: Node) -> _VisitorResult:
-        name = name.lower()
         if name not in self.context.bind:
             raise InvalidQueryError("Name {name!r} is not in the bind map.")
         # Logic in visitIdentifier handles binds.

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/queries/_identifiers.py

@@ -71,8 +71,7 @@ class IdentifierContext:  # numpydoc ignore=PR01
         if bind is None:
             self.bind = {}
         else:
-            # Make bind names case-insensitive.
-            self.bind = {k.lower(): v for k, v in bind.items()}
+            self.bind = dict(bind)
             if len(self.bind.keys()) != len(bind.keys()):
                 raise ValueError(f"Duplicate keys present in bind: {bind.keys()}")
 
@@ -96,9 +95,6 @@ def interpret_identifier(context: IdentifierContext, identifier: str) -> ColumnE
     dimensions = context.dimensions
     datasets = context.datasets
     bind = context.bind
-    # Make identifiers case-insensitive.
-    identifier = identifier.lower()
-
     if identifier in bind:
         return make_column_literal(bind[identifier])
     terms = identifier.split(".")

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/queries/tree/_base.py

@@ -110,7 +110,7 @@ def is_dataset_field(s: str) -> TypeGuard[DatasetFieldName]:
 class QueryTreeBase(pydantic.BaseModel):
     """Base class for all non-primitive types in a query tree."""
 
-    model_config = pydantic.ConfigDict(frozen=True, extra="forbid", strict=True)
+    model_config = pydantic.ConfigDict(frozen=True, strict=True)
 
 
 class ColumnExpressionBase(QueryTreeBase, ABC):

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/remote_butler/server/_dependencies.py

@@ -29,6 +29,8 @@ from typing import Annotated
 
 from fastapi import Depends, Header, HTTPException
 from safir.dependencies.gafaelfawr import auth_delegated_token_dependency
+from safir.dependencies.logger import logger_dependency as safir_logger_dependency
+from structlog.stdlib import BoundLogger
 
 from lsst.daf.butler import LabeledButlerFactory
 
@@ -117,6 +119,31 @@ async def repository_authorization_dependency(
         )
 
 
+async def logger_dependency(
+    logger: Annotated[BoundLogger, Depends(safir_logger_dependency)],
+    repository: str,
+    user_name: Annotated[str | None, Depends(user_name_dependency)],
+    x_auth_request_service: Annotated[str | None, Header()] = None,
+) -> BoundLogger:
+    """Return a logger with additional bound context information.
+
+    Parameters
+    ----------
+    logger : `structlog.stdlib.BoundLogger`
+        Logger provided by Safir.
+    repository : `str`
+        Butler repository that is being accessed.
+    user_name : `str` or `None`
+        Name of the user accessing the repository, from Gafaelfawr headers.
+    x_auth_request_service : `str` or `None`
+        Name of the service being used to access the repository, from
+        Gafaelfawr headers.
+    """
+    return logger.bind(
+        butler_repo=repository, requester={"username": user_name, "service": x_auth_request_service}
+    )
+
+
 async def factory_dependency(
     repository: str,
     butler_factory: Annotated[LabeledButlerFactory, Depends(butler_factory_dependency)],

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/remote_butler/server/handlers/_external.py

@@ -30,10 +30,11 @@ from __future__ import annotations
 __all__ = ()
 
 import uuid
-from typing import Any
+from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, HTTPException, Request
 from fastapi.responses import RedirectResponse
+from structlog.stdlib import BoundLogger
 
 from ...._butler import Butler
 from ...._collection_type import CollectionType
@@ -64,7 +65,7 @@ from ...server_models import (
     QueryDatasetTypesResponseModel,
 )
 from .._config import load_config
-from .._dependencies import factory_dependency
+from .._dependencies import factory_dependency, logger_dependency
 from .._factory import Factory
 from ._file_info import get_file_info_payload
 from ._utils import generate_file_download_uri, set_default_data_id
@@ -184,12 +185,14 @@ def expand_data_id(
 )
 def get_file(
     dataset_id: uuid.UUID,
+    logger: Annotated[BoundLogger, Depends(logger_dependency)],
     factory: Factory = Depends(factory_dependency),
 ) -> GetFileResponseModel:
     butler = factory.create_butler()
     ref = butler.get_dataset(dataset_id, datastore_records=True)
     if ref is None:
         raise DatasetNotFoundError(f"Dataset ID {dataset_id} not found")
+    _log_file_access(logger, ref, "uuid")
     return _get_file_info_response(butler, ref)
 
 
@@ -200,6 +203,7 @@ def get_file(
 )
 def get_file_by_data_id(
     request: GetFileByDataIdRequestModel,
+    logger: Annotated[BoundLogger, Depends(logger_dependency)],
     factory: Factory = Depends(factory_dependency),
 ) -> GetFileResponseModel:
     butler = factory.create_butler()
@@ -211,6 +215,7 @@ def get_file_by_data_id(
         datastore_records=True,
         timespan=request.timespan,
     )
+    _log_file_access(logger, ref, "data-id")
     return _get_file_info_response(butler, ref)
 
 
@@ -220,12 +225,21 @@ def _get_file_info_response(butler: Butler, ref: DatasetRef) -> GetFileResponseM
     )
 
 
+def _log_file_access(logger: BoundLogger, ref: DatasetRef, access_method: str) -> None:
+    logger.info(
+        "file-access",
+        dataset={"uuid": str(ref.id), "type": ref.datasetType.name, "data_id": dict(ref.dataId.required)},
+        file_access_endpoint=access_method,
+    )
+
+
 @external_router.get(
     "/v1/dataset/{dataset_id}/download",
     summary="Return an HTTP redirect to a location where you can download the artifact file for the dataset.",
 )
 def redirect_to_dataset_download(
     dataset_id: uuid.UUID,
+    logger: Annotated[BoundLogger, Depends(logger_dependency)],
     component: str | None = None,
     factory: Factory = Depends(factory_dependency),
 ) -> RedirectResponse:
@@ -245,6 +259,7 @@ def redirect_to_dataset_download(
         # files, and you must specify the component to pick which one you want
         # to download.
         if info.datastoreRecords.component == component:
+            _log_file_access(logger, ref, "uuid-redirect")
             return RedirectResponse(status_code=307, url=str(info.url))
 
     found_components = [info.datastoreRecords.component for info in payload.file_info]

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/python/lsst/daf/butler/tests/utils.py

@@ -46,6 +46,7 @@ from lsst.resources import ResourcePathExpression
 from .. import Butler, ButlerConfig, Config, DatasetRef, StorageClassFactory, Timespan
 from .._collection_type import CollectionType
 from ..datastore import NullDatastore
+from ..dimensions import DimensionConfig
 from ..direct_butler import DirectButler
 from ..registry.sql_registry import RegistryConfig, SqlRegistry
 from ..tests import MetricsExample, addDatasetType
@@ -130,7 +131,9 @@ def safeTestTempDir(default_base: str) -> Iterator[str]:
 
 
 def create_populated_sqlite_registry(
-    *args: ResourcePathExpression, registry_config: RegistryConfig | None = None
+    *args: ResourcePathExpression,
+    registry_config: RegistryConfig | None = None,
+    dimension_config: DimensionConfig | None = None,
 ) -> Butler:
     """Create an in-memory registry-only sqlite butler and populate it.
 
@@ -141,6 +144,8 @@ def create_populated_sqlite_registry(
     registry_config : ``RegistryConfig``, optional
         Registry configuration to use as the basis for the Butler
         configuration.
+    dimension_config : ``DimensionConfig``, optional
+        Dimension universe configuration.
 
     Returns
     -------
@@ -151,7 +156,7 @@ def create_populated_sqlite_registry(
     if registry_config is not None:
         config["registry"] = registry_config
     config[".registry.db"] = "sqlite://"
-    registry = SqlRegistry.createFromConfig(config["registry"])
+    registry = SqlRegistry.createFromConfig(config["registry"], dimension_config)
     butler = DirectButler(
         config=config,
         registry=registry,

lsst_daf_butler-29.2025.4000/python/lsst/daf/butler/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.4000"

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000/python/lsst_daf_butler.egg-info}/PKG-INFO

@@ -1,13 +1,12 @@
 Metadata-Version: 2.4
 Name: lsst-daf-butler
-Version: 29.2025.3800
+Version: 29.2025.4000
 Summary: An abstraction layer for reading and writing astronomical data to datastores.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
-License: BSD 3-Clause License
+License-Expression: BSD-3-Clause OR GPL-3.0-or-later
 Project-URL: Homepage, https://github.com/lsst/daf_butler
 Keywords: lsst
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/tests/test_datastore.py

@@ -988,6 +988,30 @@ class DatastoreTests(DatastoreTestsBase):
         data = datastore2.get(refs[2])
         self.assertIsNotNone(data)
 
+    def testExportPredictedRecords(self):
+        if self.isEphemeral:
+            raise unittest.SkipTest("in-memory datastore does not support record export/import")
+        sc = self.storageClassFactory.getStorageClass("ThingOne")
+        dimensions = self.universe.conform(("visit", "physical_filter"))
+        dataId = {
+            "instrument": "dummy",
+            "visit": 52,
+            "physical_filter": "V",
+            "band": "v",
+            "day_obs": 20250101,
+        }
+        ref = self.makeDatasetRef("metric", dimensions, sc, dataId)
+
+        datastore = self.makeDatastore("test_datastore")
+        names = {n for n in datastore.names if not n.startswith("InMemory")}
+        records = datastore.export_predicted_records([ref])
+
+        # Expect predicted records from all datastores.
+        self.assertEqual(set(records.keys()), names)
+
+        for record_data in records.values():
+            self.assertEqual(len(record_data.records), 1)
+
     def testExport(self) -> None:
         datastore, refs = self._populate_export_datastore("test_datastore")
 
@@ -1414,6 +1438,7 @@ class ChainedDatastoreMemoryTestCase(InMemoryDatastoreTestCase):
 
     configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2.yaml")
     validationCanFail = False
+    isEphemeral = True
 
 
 class DatastoreConstraintsTests(DatastoreTestsBase):
@@ -2124,6 +2149,8 @@ class NullDatastoreTestCase(DatasetTestHelper, unittest.TestCase):
             null.import_records({})
         with self.assertRaises(NotImplementedError):
             null.export_records([])
+        with self.assertRaises(NotImplementedError):
+            null.export_predicted_records([])
         with self.assertRaises(NotImplementedError):
             null.export([ref])
         with self.assertRaises(NotImplementedError):

{lsst_daf_butler-29.2025.3800 → lsst_daf_butler-29.2025.4000}/tests/test_quantumBackedButler.py

@@ -416,6 +416,19 @@ class QuantumBackedButlerTestCase(unittest.TestCase):
             for dataset_data in datastore_records.records[class_name].values():
                 self.assertEqual(set(dataset_data), {table_name})
 
+    def test_export_predicted_datastore_records(self) -> None:
+        """Test for export_predicted_datastore_records method"""
+        quantum = self.make_quantum()
+        qbb = QuantumBackedButler.initialize(
+            config=self.config, quantum=quantum, dimensions=self.universe, dataset_types=self.dataset_types
+        )
+
+        records = qbb.export_predicted_datastore_records(self.output_refs)
+        self.assertEqual(len(records["FileDatastore@<butlerRoot>/datastore"].records), len(self.output_refs))
+
+        with self.assertRaises(ValueError):
+            qbb.export_predicted_datastore_records(self.missing_refs)
+
     def test_collect_and_transfer(self) -> None:
         """Test for collect_and_transfer method"""
         quantum1 = self.make_quantum(1)

lsst_daf_butler-29.2025.3800/python/lsst/daf/butler/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.3800"