lsst-daf-butler 29.2025.3000__py3-none-any.whl → 29.2025.3200__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. lsst/daf/butler/_butler.py +2 -2
  2. lsst/daf/butler/_labeled_butler_factory.py +2 -2
  3. lsst/daf/butler/_limited_butler.py +1 -6
  4. lsst/daf/butler/_storage_class.py +66 -174
  5. lsst/daf/butler/datastore/_datastore.py +6 -8
  6. lsst/daf/butler/datastore/record_data.py +22 -0
  7. lsst/daf/butler/datastore/stored_file_info.py +2 -4
  8. lsst/daf/butler/datastores/chainedDatastore.py +2 -3
  9. lsst/daf/butler/datastores/fileDatastore.py +2 -34
  10. lsst/daf/butler/dimensions/_record_set.py +49 -2
  11. lsst/daf/butler/remote_butler/__init__.py +0 -1
  12. lsst/daf/butler/remote_butler/_config.py +12 -4
  13. lsst/daf/butler/remote_butler/_factory.py +51 -27
  14. lsst/daf/butler/remote_butler/_get.py +98 -0
  15. lsst/daf/butler/remote_butler/_http_connection.py +11 -17
  16. lsst/daf/butler/remote_butler/_remote_butler.py +12 -7
  17. lsst/daf/butler/remote_butler/_remote_file_transfer_source.py +3 -10
  18. lsst/daf/butler/remote_butler/authentication/__init__.py +0 -0
  19. lsst/daf/butler/remote_butler/authentication/cadc.py +54 -0
  20. lsst/daf/butler/remote_butler/authentication/interface.py +46 -0
  21. lsst/daf/butler/remote_butler/{_authentication.py → authentication/rubin.py} +36 -20
  22. lsst/daf/butler/remote_butler/server/_config.py +5 -0
  23. lsst/daf/butler/remote_butler/server/_gafaelfawr.py +4 -2
  24. lsst/daf/butler/remote_butler/server/handlers/_external.py +29 -12
  25. lsst/daf/butler/remote_butler/server/handlers/_file_info.py +72 -0
  26. lsst/daf/butler/remote_butler/server_models.py +38 -3
  27. lsst/daf/butler/tests/_examplePythonTypes.py +1 -6
  28. lsst/daf/butler/tests/server.py +13 -6
  29. lsst/daf/butler/version.py +1 -1
  30. {lsst_daf_butler-29.2025.3000.dist-info → lsst_daf_butler-29.2025.3200.dist-info}/METADATA +1 -1
  31. {lsst_daf_butler-29.2025.3000.dist-info → lsst_daf_butler-29.2025.3200.dist-info}/RECORD +39 -35
  32. lsst/daf/butler/datastores/fileDatastoreClient.py +0 -90
  33. {lsst_daf_butler-29.2025.3000.dist-info → lsst_daf_butler-29.2025.3200.dist-info}/WHEEL +0 -0
  34. {lsst_daf_butler-29.2025.3000.dist-info → lsst_daf_butler-29.2025.3200.dist-info}/entry_points.txt +0 -0
  35. {lsst_daf_butler-29.2025.3000.dist-info → lsst_daf_butler-29.2025.3200.dist-info}/licenses/COPYRIGHT +0 -0
  36. {lsst_daf_butler-29.2025.3000.dist-info → lsst_daf_butler-29.2025.3200.dist-info}/licenses/LICENSE +0 -0
  37. {lsst_daf_butler-29.2025.3000.dist-info → lsst_daf_butler-29.2025.3200.dist-info}/licenses/bsd_license.txt +0 -0
  38. {lsst_daf_butler-29.2025.3000.dist-info → lsst_daf_butler-29.2025.3200.dist-info}/licenses/gpl-v3.0.txt +0 -0
  39. {lsst_daf_butler-29.2025.3000.dist-info → lsst_daf_butler-29.2025.3200.dist-info}/top_level.txt +0 -0
  40. {lsst_daf_butler-29.2025.3000.dist-info → lsst_daf_butler-29.2025.3200.dist-info}/zip-safe +0 -0
@@ -347,13 +347,13 @@ class Butler(LimitedButler): # numpydoc ignore=PR02
347
347
  without_datastore=without_datastore,
348
348
  )
349
349
  case ButlerType.REMOTE:
350
- from .remote_butler import RemoteButlerFactory
350
+ from .remote_butler._factory import RemoteButlerFactory
351
351
 
352
352
  # Assume this is being created by a client who would like
353
353
  # default caching of remote datasets.
354
354
  factory = RemoteButlerFactory.create_factory_from_config(butler_config)
355
355
  return factory.create_butler_with_credentials_from_environment(
356
- butler_options=options, use_disabled_datastore_cache=False
356
+ butler_options=options, enable_datastore_cache=True
357
357
  )
358
358
  case _:
359
359
  raise TypeError(f"Unknown Butler type '{butler_type}'")
@@ -198,9 +198,9 @@ def _create_direct_butler_factory(config: ButlerConfig, preload_unsafe_caches: b
198
198
 
199
199
 
200
200
  def _create_remote_butler_factory(config: ButlerConfig) -> _FactoryFunction:
201
- import lsst.daf.butler.remote_butler
201
+ import lsst.daf.butler.remote_butler._factory
202
202
 
203
- factory = lsst.daf.butler.remote_butler.RemoteButlerFactory.create_factory_from_config(config)
203
+ factory = lsst.daf.butler.remote_butler._factory.RemoteButlerFactory.create_factory_from_config(config)
204
204
 
205
205
  def create_butler(access_token: str | None) -> Butler:
206
206
  if access_token is None:
@@ -129,11 +129,6 @@ class LimitedButler(ABC):
129
129
  obj : `object`
130
130
  The dataset.
131
131
 
132
- Raises
133
- ------
134
- AmbiguousDatasetError
135
- Raised if the supplied `DatasetRef` is unresolved.
136
-
137
132
  Notes
138
133
  -----
139
134
  In a `LimitedButler` the only allowable way to specify a dataset is
@@ -326,7 +321,7 @@ class LimitedButler(ABC):
326
321
  Whether the dataset artifact exists in the datastore and can be
327
322
  retrieved.
328
323
  """
329
- return self._datastore.exists(ref)
324
+ return self.stored_many([ref])[ref]
330
325
 
331
326
  def stored_many(
332
327
  self,
@@ -39,6 +39,8 @@ from collections.abc import Callable, Collection, Mapping, Sequence, Set
39
39
  from threading import RLock
40
40
  from typing import Any
41
41
 
42
+ import pydantic
43
+
42
44
  from lsst.utils import doImportType
43
45
  from lsst.utils.classes import Singleton
44
46
  from lsst.utils.introspection import get_full_type_name
@@ -57,6 +59,18 @@ class StorageClassConfig(ConfigSubset):
57
59
  defaultConfigFile = "storageClasses.yaml"
58
60
 
59
61
 
62
+ class _StorageClassModel(pydantic.BaseModel):
63
+ """Model class used to validate storage class configuration."""
64
+
65
+ pytype: str | None = None
66
+ inheritsFrom: str | None = None
67
+ components: dict[str, str] = pydantic.Field(default_factory=dict)
68
+ derivedComponents: dict[str, str] = pydantic.Field(default_factory=dict)
69
+ parameters: list[str] = pydantic.Field(default_factory=list)
70
+ delegate: str | None = None
71
+ converters: dict[str, str] = pydantic.Field(default_factory=dict)
72
+
73
+
60
74
  class StorageClass:
61
75
  """Class describing how a label maps to a particular Python type.
62
76
 
@@ -81,17 +95,9 @@ class StorageClass:
81
95
  that python type to the valid type of this storage class.
82
96
  """
83
97
 
84
- _cls_name: str = "BaseStorageClass"
85
- _cls_components: dict[str, StorageClass] | None = None
86
- _cls_derivedComponents: dict[str, StorageClass] | None = None
87
- _cls_parameters: Set[str] | Sequence[str] | None = None
88
- _cls_delegate: str | None = None
89
- _cls_pytype: type | str | None = None
90
- _cls_converters: dict[str, str] | None = None
91
-
92
98
  def __init__(
93
99
  self,
94
- name: str | None = None,
100
+ name: str = "",
95
101
  pytype: type | str | None = None,
96
102
  components: dict[str, StorageClass] | None = None,
97
103
  derivedComponents: dict[str, StorageClass] | None = None,
@@ -99,23 +105,8 @@ class StorageClass:
99
105
  delegate: str | None = None,
100
106
  converters: dict[str, str] | None = None,
101
107
  ):
102
- if name is None:
103
- name = self._cls_name
104
- if pytype is None:
105
- pytype = self._cls_pytype
106
- if components is None:
107
- components = self._cls_components
108
- if derivedComponents is None:
109
- derivedComponents = self._cls_derivedComponents
110
- if parameters is None:
111
- parameters = self._cls_parameters
112
- if delegate is None:
113
- delegate = self._cls_delegate
114
-
115
108
  # Merge converters with class defaults.
116
109
  self._converters = {}
117
- if self._cls_converters is not None:
118
- self._converters.update(self._cls_converters)
119
110
  if converters:
120
111
  self._converters.update(converters)
121
112
 
@@ -634,7 +625,6 @@ class StorageClassFactory(metaclass=Singleton):
634
625
 
635
626
  def __init__(self, config: StorageClassConfig | str | None = None):
636
627
  self._storageClasses: dict[str, StorageClass] = {}
637
- self._configs: list[StorageClassConfig] = []
638
628
  self._lock = RLock()
639
629
 
640
630
  # Always seed with the default config
@@ -657,40 +647,15 @@ class StorageClassFactory(metaclass=Singleton):
657
647
 
658
648
  StorageClasses
659
649
  --------------
660
- {sep.join(f"{s}: {self._storageClasses[s]!r}" for s in sorted(self._storageClasses))}
650
+ {sep.join(f"{self._storageClasses[s]!r}" for s in sorted(self._storageClasses))}
661
651
  """
662
652
 
663
- def __contains__(self, storageClassOrName: StorageClass | str) -> bool:
664
- """Indicate whether the storage class exists in the factory.
665
-
666
- Parameters
667
- ----------
668
- storageClassOrName : `str` or `StorageClass`
669
- If `str` is given existence of the named StorageClass
670
- in the factory is checked. If `StorageClass` is given
671
- existence and equality are checked.
672
-
673
- Returns
674
- -------
675
- in : `bool`
676
- True if the supplied string is present, or if the supplied
677
- `StorageClass` is present and identical.
678
-
679
- Notes
680
- -----
681
- The two different checks (one for "key" and one for "value") based on
682
- the type of the given argument mean that it is possible for
683
- StorageClass.name to be in the factory but StorageClass to not be
684
- in the factory.
685
- """
653
+ def __contains__(self, storageClassOrName: object) -> bool:
686
654
  with self._lock:
687
655
  if isinstance(storageClassOrName, str):
688
656
  return storageClassOrName in self._storageClasses
689
- elif (
690
- isinstance(storageClassOrName, StorageClass)
691
- and storageClassOrName.name in self._storageClasses
692
- ):
693
- return storageClassOrName == self._storageClasses[storageClassOrName.name]
657
+ elif isinstance(storageClassOrName, StorageClass):
658
+ return storageClassOrName.name in self._storageClasses
694
659
  return False
695
660
 
696
661
  def addFromConfig(self, config: StorageClassConfig | Config | str) -> None:
@@ -708,68 +673,54 @@ StorageClasses
708
673
  # components or parents before their classes are defined
709
674
  # we have a helper function that we can call recursively
710
675
  # to extract definitions from the configuration.
711
- def processStorageClass(name: str, _sconfig: StorageClassConfig, msg: str = "") -> None:
712
- # Maybe we've already processed this through recursion
676
+ def processStorageClass(name: str, _sconfig: StorageClassConfig, msg: str = "") -> StorageClass:
677
+ # This might have already been processed through recursion, or
678
+ # already present in the factory.
713
679
  if name not in _sconfig:
714
- return
715
- info = _sconfig.pop(name)
716
-
717
- # Always create the storage class so we can ensure that
718
- # we are not trying to overwrite with a different definition
719
- components = None
720
-
721
- # Extract scalar items from dict that are needed for
722
- # StorageClass Constructor
723
- storageClassKwargs = {k: info[k] for k in ("pytype", "delegate", "parameters") if k in info}
724
-
725
- if "converters" in info:
726
- storageClassKwargs["converters"] = info["converters"].toDict()
727
-
728
- for compName in ("components", "derivedComponents"):
729
- if compName not in info:
730
- continue
731
- components = {}
732
- for cname, ctype in info[compName].items():
733
- if ctype not in self:
734
- processStorageClass(ctype, sconfig, msg)
735
- components[cname] = self.getStorageClass(ctype)
736
-
737
- # Fill in other items
738
- storageClassKwargs[compName] = components
739
-
740
- # Create the new storage class and register it
741
- baseClass = None
742
- if "inheritsFrom" in info:
743
- baseName = info["inheritsFrom"]
744
-
745
- # The inheritsFrom feature requires that the storage class
746
- # being inherited from is itself a subclass of StorageClass
747
- # that was created with makeNewStorageClass. If it was made
748
- # and registered with a simple StorageClass constructor it
749
- # cannot be used here and we try to recreate it.
750
- if baseName in self:
751
- baseClass = type(self.getStorageClass(baseName))
752
- if baseClass is StorageClass:
753
- log.warning(
754
- "Storage class %s is requested to inherit from %s but that storage class "
755
- "has not been defined to be a subclass of StorageClass and so can not "
756
- "be used. Attempting to recreate parent class from current configuration.",
757
- name,
758
- baseName,
759
- )
760
- processStorageClass(baseName, sconfig, msg)
761
- else:
762
- processStorageClass(baseName, sconfig, msg)
763
- baseClass = type(self.getStorageClass(baseName))
764
- if baseClass is StorageClass:
765
- raise TypeError(
766
- f"Configuration for storage class {name} requests to inherit from "
767
- f" storage class {baseName} but that class is not defined correctly."
768
- )
769
-
770
- newStorageClassType = self.makeNewStorageClass(name, baseClass, **storageClassKwargs)
771
- newStorageClass = newStorageClassType()
772
- self.registerStorageClass(newStorageClass, msg=msg)
680
+ return self.getStorageClass(name)
681
+ try:
682
+ model = _StorageClassModel.model_validate(_sconfig.pop(name))
683
+ except Exception as err:
684
+ err.add_note(msg)
685
+ raise
686
+ components: dict[str, StorageClass] = {}
687
+ derivedComponents: dict[str, StorageClass] = {}
688
+ parameters: set[str] = set()
689
+ delegate: str | None = None
690
+ converters: dict[str, str] = {}
691
+ if model.inheritsFrom is not None:
692
+ base = processStorageClass(model.inheritsFrom, _sconfig, msg + f"; processing base of {name}")
693
+ pytype = base._pytypeName
694
+ components.update(base.components)
695
+ derivedComponents.update(base.derivedComponents)
696
+ parameters.update(base.parameters)
697
+ delegate = base._delegateClassName
698
+ converters.update(base.converters)
699
+ if model.pytype is not None:
700
+ pytype = model.pytype
701
+ for k, v in model.components.items():
702
+ components[k] = processStorageClass(
703
+ v, _sconfig, msg + f"; processing component {k} of {name}"
704
+ )
705
+ for k, v in model.derivedComponents.items():
706
+ derivedComponents[k] = processStorageClass(
707
+ v, _sconfig, msg + f"; processing derivedCmponent {k} of {name}"
708
+ )
709
+ parameters.update(model.parameters)
710
+ if model.delegate is not None:
711
+ delegate = model.delegate
712
+ converters.update(model.converters)
713
+ result = StorageClass(
714
+ name=name,
715
+ pytype=pytype,
716
+ components=components,
717
+ derivedComponents=derivedComponents,
718
+ parameters=parameters,
719
+ delegate=delegate,
720
+ converters=converters,
721
+ )
722
+ self.registerStorageClass(result, msg=msg)
723
+ return result
773
724
 
774
725
  # In case there is a problem, construct a context message for any
775
726
  # error reporting.
@@ -778,68 +729,9 @@ StorageClasses
778
729
  log.debug("Adding definitions from config %s", ", ".join(files))
779
730
 
780
731
  with self._lock:
781
- self._configs.append(sconfig)
782
732
  for name in list(sconfig.keys()):
783
733
  processStorageClass(name, sconfig, context)
784
734
 
785
- @staticmethod
786
- def makeNewStorageClass(
787
- name: str, baseClass: type[StorageClass] | None = StorageClass, **kwargs: Any
788
- ) -> type[StorageClass]:
789
- """Create a new Python class as a subclass of `StorageClass`.
790
-
791
- Parameters
792
- ----------
793
- name : `str`
794
- Name to use for this class.
795
- baseClass : `type`, optional
796
- Base class for this `StorageClass`. Must be either `StorageClass`
797
- or a subclass of `StorageClass`. If `None`, `StorageClass` will
798
- be used.
799
- **kwargs
800
- Additional parameter values to use as defaults for this class.
801
- This can include ``components``, ``parameters``,
802
- ``derivedComponents``, and ``converters``.
803
-
804
- Returns
805
- -------
806
- newtype : `type` subclass of `StorageClass`
807
- Newly created Python type.
808
- """
809
- if baseClass is None:
810
- baseClass = StorageClass
811
- if not issubclass(baseClass, StorageClass):
812
- raise ValueError(f"Base class must be a StorageClass not {baseClass}")
813
-
814
- # convert the arguments to use different internal names
815
- clsargs = {f"_cls_{k}": v for k, v in kwargs.items() if v is not None}
816
- clsargs["_cls_name"] = name
817
-
818
- # Some container items need to merge with the base class values
819
- # so that a child can inherit but override one bit.
820
- # lists (which you get from configs) are treated as sets for this to
821
- # work consistently.
822
- for k in ("components", "parameters", "derivedComponents", "converters"):
823
- classKey = f"_cls_{k}"
824
- if classKey in clsargs:
825
- baseValue = getattr(baseClass, classKey, None)
826
- if baseValue is not None:
827
- currentValue = clsargs[classKey]
828
- if isinstance(currentValue, dict):
829
- newValue = baseValue.copy()
830
- else:
831
- newValue = set(baseValue)
832
- newValue.update(currentValue)
833
- clsargs[classKey] = newValue
834
-
835
- # If we have parameters they should be a frozen set so that the
836
- # parameters in the class can not be modified.
837
- pk = "_cls_parameters"
838
- if pk in clsargs:
839
- clsargs[pk] = frozenset(clsargs[pk])
840
-
841
- return type(f"StorageClass{name}", (baseClass,), clsargs)
842
-
843
735
  def getStorageClass(self, storageClassName: str) -> StorageClass:
844
736
  """Get a StorageClass instance associated with the supplied name.
845
737
 
@@ -66,8 +66,8 @@ if TYPE_CHECKING:
66
66
  from .._dataset_ref import DatasetId, DatasetRef
67
67
  from .._dataset_type import DatasetType
68
68
  from .._storage_class import StorageClass
69
+ from ..datastores.file_datastore.get import DatasetLocationInformation
69
70
  from ..datastores.file_datastore.retrieve_artifacts import ArtifactIndexInfo
70
- from ..datastores.fileDatastoreClient import FileDatastoreGetPayload
71
71
  from ..registry.interfaces import DatasetIdRef, DatastoreRegistryBridgeManager
72
72
  from .record_data import DatastoreRecordData
73
73
  from .stored_file_info import StoredDatastoreItemInfo
@@ -614,8 +614,8 @@ class Datastore(FileTransferSource, metaclass=ABCMeta):
614
614
  """
615
615
  raise NotImplementedError("Must be implemented by subclass")
616
616
 
617
- def prepare_get_for_external_client(self, ref: DatasetRef) -> FileDatastoreGetPayload | None:
618
- """Retrieve serializable data that can be used to execute a ``get()``.
617
+ def prepare_get_for_external_client(self, ref: DatasetRef) -> list[DatasetLocationInformation] | None:
618
+ """Retrieve data that can be used to execute a ``get()``.
619
619
 
620
620
  Parameters
621
621
  ----------
@@ -624,11 +624,9 @@ class Datastore(FileTransferSource, metaclass=ABCMeta):
624
624
 
625
625
  Returns
626
626
  -------
627
- payload : `object` | `None`
628
- Serializable payload containing the information needed to perform a
629
- get() operation. This payload may be sent over the wire to another
630
- system to perform the get(). Returns `None` if the dataset is not
631
- known to this datastore.
627
+ payload : `list` [ `DatasetLocationInformation` ] | `None`
628
+ Information needed to perform a get() operation. Returns `None` if
629
+ the dataset is not known to this datastore.
632
630
  """
633
631
  raise NotImplementedError()
634
632
 
@@ -111,6 +111,28 @@ class DatastoreRecordData:
111
111
  """Opaque table data, indexed by dataset ID and grouped by opaque table
112
112
  name."""
113
113
 
114
+ @staticmethod
115
+ def merge_mappings(*args: Mapping[str, DatastoreRecordData]) -> dict[str, DatastoreRecordData]:
116
+ """Merge mappings of datastore record data.
117
+
118
+ Parameters
119
+ ----------
120
+ *args : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ]
121
+ Mappings of record data, keyed by datastore name.
122
+
123
+ Returns
124
+ -------
125
+ merged : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ]
126
+ Merged mapping of record data, keyed by datastore name.
127
+ """
128
+ result: dict[str, DatastoreRecordData] = {}
129
+ for arg in args:
130
+ for datastore_name, record_data in arg.items():
131
+ if datastore_name not in result:
132
+ result[datastore_name] = DatastoreRecordData()
133
+ result[datastore_name].update(record_data)
134
+ return result
135
+
114
136
  def update(self, other: DatastoreRecordData) -> None:
115
137
  """Update contents of this instance with data from another instance.
116
138
 
@@ -32,7 +32,7 @@ __all__ = ("SerializedStoredFileInfo", "StoredDatastoreItemInfo", "StoredFileInf
32
32
  import inspect
33
33
  from collections.abc import Iterable, Mapping
34
34
  from dataclasses import dataclass
35
- from typing import TYPE_CHECKING, Any, ClassVar
35
+ from typing import TYPE_CHECKING, Any
36
36
 
37
37
  import pydantic
38
38
 
@@ -209,8 +209,6 @@ class StoredFileInfo(StoredDatastoreItemInfo):
209
209
  compatibility, it remains a positional argument with no default).
210
210
  """
211
211
 
212
- storageClassFactory: ClassVar[StorageClassFactory] = StorageClassFactory()
213
-
214
212
  def __init__(
215
213
  self,
216
214
  formatter: FormatterParameter,
@@ -268,7 +266,7 @@ class StoredFileInfo(StoredDatastoreItemInfo):
268
266
  @property
269
267
  def storageClass(self) -> StorageClass:
270
268
  """Storage class associated with this dataset."""
271
- return self.storageClassFactory.getStorageClass(self.storage_class_name)
269
+ return StorageClassFactory().getStorageClass(self.storage_class_name)
272
270
 
273
271
  def rebase(self, ref: DatasetRef) -> StoredFileInfo:
274
272
  """Return a copy of the record suitable for a specified reference.
@@ -47,6 +47,7 @@ from lsst.daf.butler.datastore import (
47
47
  )
48
48
  from lsst.daf.butler.datastore.constraints import Constraints
49
49
  from lsst.daf.butler.datastore.record_data import DatastoreRecordData
50
+ from lsst.daf.butler.datastores.file_datastore.get import DatasetLocationInformation
50
51
  from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import ArtifactIndexInfo, ZipIndex
51
52
  from lsst.resources import ResourcePath
52
53
  from lsst.utils import doImportType
@@ -61,8 +62,6 @@ if TYPE_CHECKING:
61
62
  from lsst.daf.butler.registry.interfaces import DatasetIdRef, DatastoreRegistryBridgeManager
62
63
  from lsst.resources import ResourcePathExpression
63
64
 
64
- from .fileDatastoreClient import FileDatastoreGetPayload
65
-
66
65
  log = getLogger(__name__)
67
66
 
68
67
 
@@ -411,7 +410,7 @@ class ChainedDatastore(Datastore):
411
410
 
412
411
  raise FileNotFoundError(f"Dataset {ref} could not be found in any of the datastores")
413
412
 
414
- def prepare_get_for_external_client(self, ref: DatasetRef) -> FileDatastoreGetPayload | None:
413
+ def prepare_get_for_external_client(self, ref: DatasetRef) -> list[DatasetLocationInformation] | None:
415
414
  datastore = self._get_matching_datastore(ref)
416
415
  if datastore is None:
417
416
  return None
@@ -89,10 +89,6 @@ from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import (
89
89
  determine_destination_for_retrieved_artifact,
90
90
  unpack_zips,
91
91
  )
92
- from lsst.daf.butler.datastores.fileDatastoreClient import (
93
- FileDatastoreGetPayload,
94
- FileDatastoreGetPayloadFileInfo,
95
- )
96
92
  from lsst.daf.butler.registry.interfaces import (
97
93
  DatabaseInsertMode,
98
94
  DatastoreRegistryBridge,
@@ -110,7 +106,6 @@ from lsst.utils.logging import VERBOSE, getLogger
110
106
  from lsst.utils.timer import time_this
111
107
 
112
108
  from ..datastore import FileTransferMap, FileTransferRecord
113
- from ..datastore.stored_file_info import make_datastore_path_relative
114
109
 
115
110
  if TYPE_CHECKING:
116
111
  from lsst.daf.butler import DatasetProvenance, LookupKey
@@ -2304,27 +2299,14 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
2304
2299
  allGetInfo, ref=ref, parameters=parameters, cache_manager=self.cacheManager
2305
2300
  )
2306
2301
 
2307
- def prepare_get_for_external_client(self, ref: DatasetRef) -> FileDatastoreGetPayload | None:
2302
+ def prepare_get_for_external_client(self, ref: DatasetRef) -> list[DatasetLocationInformation] | None:
2308
2303
  # Docstring inherited
2309
2304
 
2310
- # 1 hour. Chosen somewhat arbitrarily -- this is long enough that the
2311
- # client should have time to download a large file with retries if
2312
- # needed, but short enough that it will become obvious quickly that
2313
- # these URLs expire.
2314
- # From a strictly technical standpoint there is no reason this
2315
- # shouldn't be a day or more, but there seems to be a political issue
2316
- # where people think there is a risk of end users posting presigned
2317
- # URLs for people without access rights to download.
2318
- url_expiration_time_seconds = 1 * 60 * 60
2319
-
2320
2305
  locations = self._get_dataset_locations_info(ref)
2321
2306
  if len(locations) == 0:
2322
2307
  return None
2323
2308
 
2324
- return FileDatastoreGetPayload(
2325
- datastore_type="file",
2326
- file_info=[_to_file_info_payload(info, url_expiration_time_seconds) for info in locations],
2327
- )
2309
+ return locations
2328
2310
 
2329
2311
  @transactional
2330
2312
  def put(self, inMemoryDataset: Any, ref: DatasetRef, provenance: DatasetProvenance | None = None) -> None:
@@ -3229,17 +3211,3 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
3229
3211
  def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]:
3230
3212
  # Docstring inherited from the base class.
3231
3213
  return {self._opaque_table_name: DatastoreOpaqueTable(self.makeTableSpec(), StoredFileInfo)}
3232
-
3233
-
3234
- def _to_file_info_payload(
3235
- info: DatasetLocationInformation, url_expiration_time_seconds: int
3236
- ) -> FileDatastoreGetPayloadFileInfo:
3237
- location, file_info = info
3238
-
3239
- datastoreRecords = file_info.to_simple()
3240
- datastoreRecords.path = make_datastore_path_relative(datastoreRecords.path)
3241
-
3242
- return FileDatastoreGetPayloadFileInfo(
3243
- url=location.uri.generate_presigned_get_url(expiration_time_seconds=url_expiration_time_seconds),
3244
- datastoreRecords=datastoreRecords,
3245
- )
@@ -753,7 +753,6 @@ class SerializableDimensionData(pydantic.RootModel):
753
753
  ]
754
754
 
755
755
 
756
- @dataclasses.dataclass
757
756
  class DimensionDataAttacher:
758
757
  """A helper class for attaching dimension records to data IDs.
759
758
 
@@ -786,7 +785,7 @@ class DimensionDataAttacher:
786
785
  dimensions: DimensionGroup | None = None,
787
786
  ):
788
787
  self.records = {record_set.element.name: record_set for record_set in records}
789
- self.deserializers = {}
788
+ self.deserializers: dict[str, DimensionRecordSetDeserializer] = {}
790
789
  for deserializer in deserializers:
791
790
  self.deserializers[deserializer.element.name] = deserializer
792
791
  if deserializer.element.name not in self.records:
@@ -851,6 +850,54 @@ class DimensionDataAttacher:
851
850
 
852
851
  return [r.data_id.expanded(r.done) for r in records]
853
852
 
853
+ def serialized(
854
+ self, *, ignore: Iterable[str] = (), ignore_cached: bool = False, include_skypix: bool = False
855
+ ) -> SerializableDimensionData:
856
+ """Serialize all dimension data in this attacher, with deduplication
857
+ across fully- and partially-deserialized records.
858
+
859
+ Parameters
860
+ ----------
861
+ ignore : `~collections.abc.Iterable` [ `str` ], optional
862
+ Names of dimension elements that should not be serialized.
863
+ ignore_cached : `bool`, optional
864
+ If `True`, ignore all dimension elements for which
865
+ `DimensionElement.is_cached` is `True`.
866
+ include_skypix : `bool`, optional
867
+ If `True`, include skypix dimensions. These are ignored by default
868
+ because they can always be recomputed from their IDs on-the-fly.
869
+
870
+ Returns
871
+ -------
872
+ serialized : `SerializedDimensionData`
873
+ Serialized dimension records.
874
+ """
875
+ from ._skypix import SkyPixDimension
876
+
877
+ ignore = set(ignore)
878
+ result = SerializableDimensionData()
879
+ for record_set in self.records.values():
880
+ if record_set.element.name in ignore:
881
+ continue
882
+ if not include_skypix and isinstance(record_set.element, SkyPixDimension):
883
+ continue
884
+ if ignore_cached and record_set.element.is_cached:
885
+ continue
886
+ serialized_records: dict[tuple[DataIdValue, ...], SerializedKeyValueDimensionRecord] = {}
887
+ if (deserializer := self.deserializers.get(record_set.element.name)) is not None:
888
+ for key, value in deserializer._mapping.items():
889
+ serialized_record = list(key)
890
+ serialized_record.extend(value)
891
+ serialized_records[key] = serialized_record
892
+ for key, record in record_set._by_required_values.items():
893
+ if key not in serialized_records:
894
+ serialized_records[key] = record.serialize_key_value()
895
+ result.root[record_set.element.name] = list(serialized_records.values())
896
+ if self.cache is not None and not ignore_cached:
897
+ for record_set in self.cache.values():
898
+ result.root[record_set.element.name] = record_set.serialize_records()
899
+ return result
900
+
854
901
 
855
902
  @dataclasses.dataclass
856
903
  class _InProgressRecordDicts:
@@ -25,6 +25,5 @@
25
25
  # You should have received a copy of the GNU General Public License
26
26
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
27
 
28
- from ._factory import *
29
28
  from ._http_connection import ButlerServerError
30
29
  from ._remote_butler import *
@@ -25,16 +25,24 @@
25
25
  # You should have received a copy of the GNU General Public License
26
26
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
27
 
28
+ from __future__ import annotations
29
+
30
+ from typing import Literal, TypeAlias
31
+
28
32
  from pydantic import AnyHttpUrl, BaseModel
29
33
 
30
34
 
35
+ class RemoteButlerConfigModel(BaseModel):
36
+ """ButlerConfig properties for RemoteButler."""
37
+
38
+ remote_butler: RemoteButlerOptionsModel
39
+
40
+
31
41
  class RemoteButlerOptionsModel(BaseModel):
32
42
  """Model representing the remote server connection."""
33
43
 
34
44
  url: AnyHttpUrl
45
+ authentication: AuthenticationMode = "rubin_science_platform"
35
46
 
36
47
 
37
- class RemoteButlerConfigModel(BaseModel):
38
- """Configuration representing a remote butler."""
39
-
40
- remote_butler: RemoteButlerOptionsModel
48
+ AuthenticationMode: TypeAlias = Literal["rubin_science_platform", "cadc"]