lsst-daf-butler 29.2025.3900__py3-none-any.whl → 29.2025.4100__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. lsst/daf/butler/_dataset_ref.py +10 -3
  2. lsst/daf/butler/_dataset_type.py +2 -2
  3. lsst/daf/butler/_quantum_backed.py +28 -0
  4. lsst/daf/butler/_registry_shim.py +9 -96
  5. lsst/daf/butler/_rubin/__init__.py +3 -0
  6. lsst/daf/butler/_uuid.py +79 -0
  7. lsst/daf/butler/configs/datastores/formatters.yaml +1 -0
  8. lsst/daf/butler/configs/storageClasses.yaml +2 -0
  9. lsst/daf/butler/datastore/_datastore.py +18 -0
  10. lsst/daf/butler/datastores/chainedDatastore.py +19 -0
  11. lsst/daf/butler/datastores/fileDatastore.py +18 -0
  12. lsst/daf/butler/datastores/inMemoryDatastore.py +6 -0
  13. lsst/daf/butler/dimensions/_coordinate.py +3 -3
  14. lsst/daf/butler/dimensions/_elements.py +3 -2
  15. lsst/daf/butler/dimensions/_records.py +3 -3
  16. lsst/daf/butler/direct_query_driver/_driver.py +5 -1
  17. lsst/daf/butler/queries/_expression_strings.py +0 -3
  18. lsst/daf/butler/queries/_identifiers.py +1 -5
  19. lsst/daf/butler/queries/_query.py +4 -0
  20. lsst/daf/butler/queries/predicate_constraints_summary.py +8 -3
  21. lsst/daf/butler/queries/tree/_base.py +1 -1
  22. lsst/daf/butler/queries/tree/_query_tree.py +5 -0
  23. lsst/daf/butler/registry/_registry.py +6 -3
  24. lsst/daf/butler/registry/_registry_base.py +265 -0
  25. lsst/daf/butler/registry/dimensions/static.py +0 -120
  26. lsst/daf/butler/registry/interfaces/_dimensions.py +0 -40
  27. lsst/daf/butler/registry/queries/__init__.py +0 -3
  28. lsst/daf/butler/registry/queries/_query_backend.py +0 -65
  29. lsst/daf/butler/{remote_butler/registry → registry/queries}/_query_common.py +73 -31
  30. lsst/daf/butler/{remote_butler/registry → registry/queries}/_query_data_coordinates.py +51 -14
  31. lsst/daf/butler/{remote_butler/registry → registry/queries}/_query_datasets.py +8 -11
  32. lsst/daf/butler/{remote_butler/registry → registry/queries}/_query_dimension_records.py +7 -8
  33. lsst/daf/butler/registry/queries/_results.py +2 -316
  34. lsst/daf/butler/registry/queries/_sql_query_backend.py +1 -147
  35. lsst/daf/butler/registry/sql_registry.py +1 -606
  36. lsst/daf/butler/registry/tests/_registry.py +109 -451
  37. lsst/daf/butler/remote_butler/_registry.py +3 -219
  38. lsst/daf/butler/tests/hybrid_butler_registry.py +3 -94
  39. lsst/daf/butler/tests/utils.py +7 -2
  40. lsst/daf/butler/version.py +1 -1
  41. {lsst_daf_butler-29.2025.3900.dist-info → lsst_daf_butler-29.2025.4100.dist-info}/METADATA +2 -3
  42. {lsst_daf_butler-29.2025.3900.dist-info → lsst_daf_butler-29.2025.4100.dist-info}/RECORD +50 -51
  43. lsst/daf/butler/registry/queries/_builder.py +0 -276
  44. lsst/daf/butler/registry/queries/_query.py +0 -1087
  45. lsst/daf/butler/registry/queries/_structs.py +0 -525
  46. {lsst_daf_butler-29.2025.3900.dist-info → lsst_daf_butler-29.2025.4100.dist-info}/WHEEL +0 -0
  47. {lsst_daf_butler-29.2025.3900.dist-info → lsst_daf_butler-29.2025.4100.dist-info}/entry_points.txt +0 -0
  48. {lsst_daf_butler-29.2025.3900.dist-info → lsst_daf_butler-29.2025.4100.dist-info}/licenses/COPYRIGHT +0 -0
  49. {lsst_daf_butler-29.2025.3900.dist-info → lsst_daf_butler-29.2025.4100.dist-info}/licenses/LICENSE +0 -0
  50. {lsst_daf_butler-29.2025.3900.dist-info → lsst_daf_butler-29.2025.4100.dist-info}/licenses/bsd_license.txt +0 -0
  51. {lsst_daf_butler-29.2025.3900.dist-info → lsst_daf_butler-29.2025.4100.dist-info}/licenses/gpl-v3.0.txt +0 -0
  52. {lsst_daf_butler-29.2025.3900.dist-info → lsst_daf_butler-29.2025.4100.dist-info}/top_level.txt +0 -0
  53. {lsst_daf_butler-29.2025.3900.dist-info → lsst_daf_butler-29.2025.4100.dist-info}/zip-safe +0 -0
@@ -42,7 +42,7 @@ import enum
42
42
  import logging
43
43
  import sys
44
44
  import uuid
45
- from collections.abc import Iterable, Mapping
45
+ from collections.abc import Callable, Iterable, Mapping
46
46
  from typing import (
47
47
  TYPE_CHECKING,
48
48
  Annotated,
@@ -52,6 +52,7 @@ from typing import (
52
52
  Protocol,
53
53
  Self,
54
54
  TypeAlias,
55
+ cast,
55
56
  runtime_checkable,
56
57
  )
57
58
 
@@ -63,6 +64,7 @@ from lsst.utils.classes import immutable
63
64
  from ._config_support import LookupKey
64
65
  from ._dataset_type import DatasetType, SerializedDatasetType
65
66
  from ._named import NamedKeyDict
67
+ from ._uuid import generate_uuidv7
66
68
  from .datastore.stored_file_info import StoredDatastoreItemInfo
67
69
  from .dimensions import (
68
70
  DataCoordinate,
@@ -181,7 +183,12 @@ class DatasetIdFactory:
181
183
  Dataset identifier.
182
184
  """
183
185
  if idGenerationMode is DatasetIdGenEnum.UNIQUE:
184
- return uuid.uuid4()
186
+ # Earlier versions of this code used UUIDv4. However, totally
187
+ # random IDs create problems for Postgres insert performance,
188
+ # because it scatters index updates randomly around the disk.
189
+ # UUIDv7 has similar uniqueness properties to v4, but IDs generated
190
+ # at the same time are close together in the index.
191
+ return generate_uuidv7()
185
192
  else:
186
193
  # WARNING: If you modify this code make sure that the order of
187
194
  # items in the `items` list below never changes.
@@ -559,7 +566,7 @@ class DatasetRef:
559
566
  return ref
560
567
 
561
568
  to_json = to_json_pydantic
562
- from_json: ClassVar = classmethod(from_json_pydantic)
569
+ from_json: ClassVar[Callable[..., Self]] = cast(Callable[..., Self], classmethod(from_json_pydantic))
563
570
 
564
571
  @classmethod
565
572
  def _unpickle(
@@ -33,7 +33,7 @@ import re
33
33
  from collections.abc import Callable, Iterable, Mapping
34
34
  from copy import deepcopy
35
35
  from types import MappingProxyType
36
- from typing import TYPE_CHECKING, Any, ClassVar
36
+ from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
37
37
 
38
38
  from pydantic import BaseModel, StrictBool, StrictStr
39
39
 
@@ -756,7 +756,7 @@ class DatasetType:
756
756
  return newType
757
757
 
758
758
  to_json = to_json_pydantic
759
- from_json: ClassVar = classmethod(from_json_pydantic)
759
+ from_json: ClassVar[Callable[..., Self]] = cast(Callable[..., Self], classmethod(from_json_pydantic))
760
760
 
761
761
  def __reduce__(
762
762
  self,
@@ -614,6 +614,34 @@ class QuantumBackedButler(LimitedButler):
614
614
  datastore_records=provenance_records,
615
615
  )
616
616
 
617
+ def export_predicted_datastore_records(
618
+ self, refs: Iterable[DatasetRef]
619
+ ) -> dict[str, DatastoreRecordData]:
620
+ """Export datastore records for a set of predicted output dataset
621
+ references.
622
+
623
+ Parameters
624
+ ----------
625
+ refs : `~collections.abc.Iterable` [ `DatasetRef` ]
626
+ Dataset references for which to export datastore records. These
627
+ refs must be known to this butler.
628
+
629
+ Returns
630
+ -------
631
+ records : `dict` [ `str`, `DatastoreRecordData` ]
632
+ Predicted datastore records indexed by datastore name. No attempt
633
+ is made to ensure that the associated datasets exist on disk.
634
+ """
635
+ unknowns = [
636
+ ref
637
+ for ref in refs
638
+ if (ref.id not in self._predicted_outputs and ref.id not in self._predicted_inputs)
639
+ ]
640
+ if unknowns:
641
+ raise ValueError(f"Cannot export datastore records for unknown outputs: {unknowns}")
642
+
643
+ return self._datastore.export_predicted_records(refs)
644
+
617
645
 
618
646
  class QuantumProvenanceData(pydantic.BaseModel):
619
647
  """A serializable struct for per-quantum provenance information and
@@ -27,16 +27,16 @@
27
27
 
28
28
  from __future__ import annotations
29
29
 
30
- __all__ = ("Registry",)
30
+ __all__ = ("RegistryShim",)
31
31
 
32
32
  import contextlib
33
33
  from collections.abc import Iterable, Iterator, Mapping, Sequence
34
34
  from typing import TYPE_CHECKING, Any
35
35
 
36
36
  from ._collection_type import CollectionType
37
- from ._dataset_association import DatasetAssociation
38
37
  from ._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef
39
38
  from ._dataset_type import DatasetType
39
+ from ._storage_class import StorageClassFactory
40
40
  from ._timespan import Timespan
41
41
  from .dimensions import (
42
42
  DataCoordinate,
@@ -46,10 +46,9 @@ from .dimensions import (
46
46
  DimensionRecord,
47
47
  DimensionUniverse,
48
48
  )
49
- from .registry import Registry
50
49
  from .registry._collection_summary import CollectionSummary
51
50
  from .registry._defaults import RegistryDefaults
52
- from .registry.queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults
51
+ from .registry._registry_base import RegistryBase
53
52
 
54
53
  if TYPE_CHECKING:
55
54
  from .direct_butler import DirectButler
@@ -57,7 +56,7 @@ if TYPE_CHECKING:
57
56
  from .registry.interfaces import ObsCoreTableManager
58
57
 
59
58
 
60
- class RegistryShim(Registry):
59
+ class RegistryShim(RegistryBase):
61
60
  """Implementation of `Registry` interface exposed to clients by `Butler`.
62
61
 
63
62
  Parameters
@@ -74,7 +73,7 @@ class RegistryShim(Registry):
74
73
  """
75
74
 
76
75
  def __init__(self, butler: DirectButler):
77
- self._butler = butler
76
+ super().__init__(butler)
78
77
  self._registry = butler._registry
79
78
 
80
79
  def isWriteable(self) -> bool:
@@ -299,97 +298,11 @@ class RegistryShim(Registry):
299
298
  expression, datasetType, collectionTypes, flattenChains, includeChains
300
299
  )
301
300
 
302
- def queryDatasets(
303
- self,
304
- datasetType: Any,
305
- *,
306
- collections: CollectionArgType | None = None,
307
- dimensions: Iterable[str] | None = None,
308
- dataId: DataId | None = None,
309
- where: str = "",
310
- findFirst: bool = False,
311
- bind: Mapping[str, Any] | None = None,
312
- check: bool = True,
313
- **kwargs: Any,
314
- ) -> DatasetQueryResults:
315
- # Docstring inherited from a base class.
316
- return self._registry.queryDatasets(
317
- datasetType,
318
- collections=collections,
319
- dimensions=dimensions,
320
- dataId=dataId,
321
- where=where,
322
- findFirst=findFirst,
323
- bind=bind,
324
- check=check,
325
- **kwargs,
326
- )
327
-
328
- def queryDataIds(
329
- self,
330
- dimensions: DimensionGroup | Iterable[str] | str,
331
- *,
332
- dataId: DataId | None = None,
333
- datasets: Any = None,
334
- collections: CollectionArgType | None = None,
335
- where: str = "",
336
- bind: Mapping[str, Any] | None = None,
337
- check: bool = True,
338
- **kwargs: Any,
339
- ) -> DataCoordinateQueryResults:
340
- # Docstring inherited from a base class.
341
- return self._registry.queryDataIds(
342
- dimensions,
343
- dataId=dataId,
344
- datasets=datasets,
345
- collections=collections,
346
- where=where,
347
- bind=bind,
348
- check=check,
349
- **kwargs,
350
- )
351
-
352
- def queryDimensionRecords(
353
- self,
354
- element: DimensionElement | str,
355
- *,
356
- dataId: DataId | None = None,
357
- datasets: Any = None,
358
- collections: CollectionArgType | None = None,
359
- where: str = "",
360
- bind: Mapping[str, Any] | None = None,
361
- check: bool = True,
362
- **kwargs: Any,
363
- ) -> DimensionRecordQueryResults:
364
- # Docstring inherited from a base class.
365
- return self._registry.queryDimensionRecords(
366
- element,
367
- dataId=dataId,
368
- datasets=datasets,
369
- collections=collections,
370
- where=where,
371
- bind=bind,
372
- check=check,
373
- **kwargs,
374
- )
375
-
376
- def queryDatasetAssociations(
377
- self,
378
- datasetType: str | DatasetType,
379
- collections: CollectionArgType | None = ...,
380
- *,
381
- collectionTypes: Iterable[CollectionType] = CollectionType.all(),
382
- flattenChains: bool = False,
383
- ) -> Iterator[DatasetAssociation]:
384
- # Docstring inherited from a base class.
385
- return self._registry.queryDatasetAssociations(
386
- datasetType,
387
- collections,
388
- collectionTypes=collectionTypes,
389
- flattenChains=flattenChains,
390
- )
391
-
392
301
  @property
393
302
  def obsCoreTableManager(self) -> ObsCoreTableManager | None:
394
303
  # Docstring inherited from a base class.
395
304
  return self._registry.obsCoreTableManager
305
+
306
+ @property
307
+ def storageClasses(self) -> StorageClassFactory:
308
+ return self._registry.storageClasses
@@ -29,3 +29,6 @@
29
29
  by other LSST packages. The interfaces and behavior of these functions are
30
30
  subject to change at any time.
31
31
  """
32
+
33
+ # UUIDv7 generation is used by pipe_base.
34
+ from .._uuid import generate_uuidv7
@@ -0,0 +1,79 @@
1
+ # This file is part of daf_butler.
2
+ #
3
+ # Developed for the LSST Data Management System.
4
+ # This product includes software developed by the LSST Project
5
+ # (http://www.lsst.org).
6
+ # See the COPYRIGHT file at the top-level directory of this distribution
7
+ # for details of code ownership.
8
+ #
9
+ # This software is dual licensed under the GNU General Public License and also
10
+ # under a 3-clause BSD license. Recipients may choose which of these licenses
11
+ # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
+ # respectively. If you choose the GPL option then the following text applies
13
+ # (but note that there is still no warranty even if you opt for BSD instead):
14
+ #
15
+ # This program is free software: you can redistribute it and/or modify
16
+ # it under the terms of the GNU General Public License as published by
17
+ # the Free Software Foundation, either version 3 of the License, or
18
+ # (at your option) any later version.
19
+ #
20
+ # This program is distributed in the hope that it will be useful,
21
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
+ # GNU General Public License for more details.
24
+ #
25
+ # You should have received a copy of the GNU General Public License
26
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
+
28
+ from __future__ import annotations
29
+
30
+ __all__ = ("generate_uuidv7",)
31
+
32
+ import time
33
+ from uuid import UUID, uuid4
34
+
35
+
36
+ def generate_uuidv7() -> UUID:
37
+ """Generate a v7 UUID compliant with IETF RFC-9562.
38
+
39
+ Returns
40
+ -------
41
+ uuid : `uuid.UUID`
42
+ Version 7 UUID.
43
+ """
44
+ # Python 3.14 will include an implementation of UUIDv7 that can replace
45
+ # this, but at the time of writing 3.14 hasn't been released and we're
46
+ # still supporting 3.12. There are a few libraries on PyPI with an
47
+ # implementation of v7 UUIDs, but none of them look well-maintained.
48
+ #
49
+ # This is the format of a v7 UUID:
50
+ # 0 1 2 3
51
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
52
+ # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
53
+ # | unix_ts_ms |
54
+ # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
55
+ # | unix_ts_ms | ver | rand_a |
56
+ # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
57
+ # |var| rand_b |
58
+ # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
59
+ # | rand_b |
60
+ # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
61
+ #
62
+ # It's basically identical to a UUID v4, but with the top 6 bytes
63
+ # replaced with a millisecond UNIX timestamp.
64
+
65
+ # Generate a UUIDv4 for the random portion of the data.
66
+ # A little wasteful, but means we inherit the best practices from
67
+ # the standard library for generating this random data.
68
+ byte_data = bytearray(uuid4().bytes)
69
+
70
+ # Replace high 6 bytes with millisecond UNIX timestamp.
71
+ timestamp = time.time_ns() // 1_000_000
72
+ timestamp_bytes = timestamp.to_bytes(length=6, byteorder="big")
73
+ byte_data[0:6] = timestamp_bytes
74
+
75
+ # Set 4-bit version field to 7.
76
+ byte_data[6] &= 0x0F
77
+ byte_data[6] |= 0x70
78
+
79
+ return UUID(bytes=bytes(byte_data))
@@ -95,3 +95,4 @@ RegionTimeInfo: lsst.daf.butler.formatters.json.JsonFormatter
95
95
  QPEnsemble: lsst.meas.pz.qp_formatter.QPFormatter
96
96
  PZModel: lsst.meas.pz.model_formatter.ModelFormatter
97
97
  VisitBackgroundModel: lsst.daf.butler.formatters.json.JsonFormatter
98
+ VignettingCorrection: lsst.ts.observatory.control.utils.extras.vignetting_storage.VignettingCorrectionFormatter
@@ -428,3 +428,5 @@ storageClasses:
428
428
  pytype: rail.core.model.Model
429
429
  VisitBackgroundModel:
430
430
  pytype: lsst.drp.tasks.fit_visit_background.VisitBackgroundModel
431
+ VignettingCorrection:
432
+ pytype: lsst.ts.observatory.control.utils.extras.vignetting_correction.VignettingCorrection
@@ -1381,6 +1381,24 @@ class Datastore(FileTransferSource, metaclass=ABCMeta):
1381
1381
  """
1382
1382
  raise NotImplementedError()
1383
1383
 
1384
+ def export_predicted_records(self, refs: Iterable[DatasetRef]) -> dict[str, DatastoreRecordData]:
1385
+ """Export predicted datastore records and locations to an in-memory
1386
+ data structure.
1387
+
1388
+ Parameters
1389
+ ----------
1390
+ refs : `~collections.abc.Iterable` [ `DatasetRef` ]
1391
+ Datastore records that would be used if the given refs were to
1392
+ exist in this datastore. No attempt is made to determine if these
1393
+ datasets actually exist.
1394
+
1395
+ Returns
1396
+ -------
1397
+ data : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ]
1398
+ Exported datastore records indexed by datastore name.
1399
+ """
1400
+ raise NotImplementedError()
1401
+
1384
1402
  def set_retrieve_dataset_type_method(self, method: Callable[[str], DatasetType | None] | None) -> None:
1385
1403
  """Specify a method that can be used by datastore to retrieve
1386
1404
  registry-defined dataset type.
@@ -1165,6 +1165,25 @@ class ChainedDatastore(Datastore):
1165
1165
 
1166
1166
  return all_records
1167
1167
 
1168
+ def export_predicted_records(self, refs: Iterable[DatasetRef]) -> dict[str, DatastoreRecordData]:
1169
+ # Docstring inherited from the base class.
1170
+
1171
+ all_records: dict[str, DatastoreRecordData] = {}
1172
+
1173
+ # Filter out datasets that this datastore is not allowed to contain.
1174
+ refs = [ref for ref in refs if self.constraints.isAcceptable(ref)]
1175
+
1176
+ # Merge all sub-datastore records into one structure
1177
+ for datastore in self.datastores:
1178
+ sub_records = datastore.export_predicted_records(refs)
1179
+ for name, record_data in sub_records.items():
1180
+ # All datastore names must be unique in a chain.
1181
+ if name in all_records:
1182
+ raise ValueError("Non-unique datastore name found in datastore {datastore}")
1183
+ all_records[name] = record_data
1184
+
1185
+ return all_records
1186
+
1168
1187
  def export(
1169
1188
  self,
1170
1189
  refs: Iterable[DatasetRef],
@@ -3188,6 +3188,24 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
3188
3188
  record_data = DatastoreRecordData(records=records)
3189
3189
  return {self.name: record_data}
3190
3190
 
3191
+ def export_predicted_records(self, refs: Iterable[DatasetRef]) -> dict[str, DatastoreRecordData]:
3192
+ # Docstring inherited from the base class.
3193
+ refs = [self._cast_storage_class(ref) for ref in refs]
3194
+ records: dict[DatasetId, dict[str, list[StoredDatastoreItemInfo]]] = {}
3195
+ for ref in refs:
3196
+ if not self.constraints.isAcceptable(ref):
3197
+ continue
3198
+ fileLocations = self._get_expected_dataset_locations_info(ref)
3199
+ if not fileLocations:
3200
+ continue
3201
+ dataset_records = records.setdefault(ref.id, {})
3202
+ dataset_records.setdefault(self._table.name, [])
3203
+ for _, storedFileInfo in fileLocations:
3204
+ dataset_records[self._table.name].append(storedFileInfo)
3205
+
3206
+ record_data = DatastoreRecordData(records=records)
3207
+ return {self.name: record_data}
3208
+
3191
3209
  def set_retrieve_dataset_type_method(self, method: Callable[[str], DatasetType | None] | None) -> None:
3192
3210
  # Docstring inherited from the base class.
3193
3211
  self._retrieve_dataset_method = method
@@ -763,6 +763,12 @@ class InMemoryDatastore(GenericBaseDatastore[StoredMemoryItemInfo]):
763
763
  # In-memory Datastore records cannot be exported or imported
764
764
  return {}
765
765
 
766
+ def export_predicted_records(self, refs: Iterable[DatasetIdRef]) -> dict[str, DatastoreRecordData]:
767
+ # Docstring inherited from the base class.
768
+
769
+ # In-memory Datastore records cannot be exported or imported
770
+ return {}
771
+
766
772
  def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]:
767
773
  # Docstring inherited from the base class.
768
774
  return {}
@@ -43,8 +43,8 @@ __all__ = (
43
43
 
44
44
  import numbers
45
45
  from abc import abstractmethod
46
- from collections.abc import Iterable, Iterator, Mapping
47
- from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, overload
46
+ from collections.abc import Callable, Iterable, Iterator, Mapping
47
+ from typing import TYPE_CHECKING, Any, ClassVar, Self, TypeAlias, cast, overload
48
48
 
49
49
  import pydantic
50
50
 
@@ -753,7 +753,7 @@ class DataCoordinate:
753
753
  return dataId
754
754
 
755
755
  to_json = to_json_pydantic
756
- from_json: ClassVar = classmethod(from_json_pydantic)
756
+ from_json: ClassVar[Callable[..., Self]] = cast(Callable[..., Self], classmethod(from_json_pydantic))
757
757
 
758
758
 
759
759
  DataId = DataCoordinate | Mapping[str, Any]
@@ -34,7 +34,8 @@ __all__ = (
34
34
  )
35
35
 
36
36
  from abc import abstractmethod
37
- from typing import TYPE_CHECKING, Annotated, Any, ClassVar, TypeAlias, Union, cast
37
+ from collections.abc import Callable
38
+ from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Self, TypeAlias, Union, cast
38
39
 
39
40
  import pydantic
40
41
  from pydantic_core import core_schema
@@ -212,7 +213,7 @@ class DimensionElement(TopologicalRelationshipEndpoint):
212
213
  return universe[simple]
213
214
 
214
215
  to_json = to_json_generic
215
- from_json: ClassVar = classmethod(from_json_generic)
216
+ from_json: ClassVar[Callable[..., Self]] = cast(Callable[..., Self], classmethod(from_json_generic))
216
217
 
217
218
  def hasTable(self) -> bool:
218
219
  """Indicate if this element is associated with a table.
@@ -30,8 +30,8 @@ from __future__ import annotations
30
30
  __all__ = ("DimensionRecord", "SerializedDimensionRecord", "SerializedKeyValueDimensionRecord")
31
31
 
32
32
  import itertools
33
- from collections.abc import Hashable
34
- from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias
33
+ from collections.abc import Callable, Hashable
34
+ from typing import TYPE_CHECKING, Any, ClassVar, Self, TypeAlias, cast
35
35
 
36
36
  import pydantic
37
37
  from pydantic import BaseModel, Field, StrictBool, StrictFloat, StrictInt, StrictStr, create_model
@@ -505,7 +505,7 @@ class DimensionRecord:
505
505
  return dimRec
506
506
 
507
507
  to_json = to_json_pydantic
508
- from_json: ClassVar = classmethod(from_json_pydantic)
508
+ from_json: ClassVar[Callable[..., Self]] = cast(Callable[..., Self], classmethod(from_json_pydantic))
509
509
 
510
510
  def toDict(self, splitTimespan: bool = False) -> dict[str, Any]:
511
511
  """Return a vanilla `dict` representation of this record.
@@ -641,7 +641,11 @@ class DirectQueryDriver(QueryDriver):
641
641
  # datasets later.
642
642
  predicate_constraints = PredicateConstraintsSummary(tree.predicate)
643
643
  # Use the default data ID to apply additional constraints where needed.
644
- predicate_constraints.apply_default_data_id(self._default_data_id, tree.dimensions)
644
+ predicate_constraints.apply_default_data_id(
645
+ self._default_data_id,
646
+ tree.dimensions,
647
+ validate_governor_constraints=tree.validateGovernorConstraints,
648
+ )
645
649
  predicate = predicate_constraints.predicate
646
650
  # Delegate to the dimensions manager to rewrite the predicate and start
647
651
  # a SqlSelectBuilder to cover any spatial overlap joins or constraints.
@@ -204,8 +204,6 @@ class _ConversionVisitor(TreeVisitor[_VisitorResult]):
204
204
  return result
205
205
 
206
206
  def visitIdentifier(self, name: str, node: Node) -> _VisitorResult:
207
- name = name.lower()
208
-
209
207
  if name in self.context.bind:
210
208
  value = self.context.bind[name]
211
209
  # Lists of values do not have a direct representation in the new
@@ -242,7 +240,6 @@ class _ConversionVisitor(TreeVisitor[_VisitorResult]):
242
240
  return _ColExpr(column_expression)
243
241
 
244
242
  def visitBind(self, name: str, node: Node) -> _VisitorResult:
245
- name = name.lower()
246
243
  if name not in self.context.bind:
247
244
  raise InvalidQueryError("Name {name!r} is not in the bind map.")
248
245
  # Logic in visitIdentifier handles binds.
@@ -71,8 +71,7 @@ class IdentifierContext: # numpydoc ignore=PR01
71
71
  if bind is None:
72
72
  self.bind = {}
73
73
  else:
74
- # Make bind names case-insensitive.
75
- self.bind = {k.lower(): v for k, v in bind.items()}
74
+ self.bind = dict(bind)
76
75
  if len(self.bind.keys()) != len(bind.keys()):
77
76
  raise ValueError(f"Duplicate keys present in bind: {bind.keys()}")
78
77
 
@@ -96,9 +95,6 @@ def interpret_identifier(context: IdentifierContext, identifier: str) -> ColumnE
96
95
  dimensions = context.dimensions
97
96
  datasets = context.datasets
98
97
  bind = context.bind
99
- # Make identifiers case-insensitive.
100
- identifier = identifier.lower()
101
-
102
98
  if identifier in bind:
103
99
  return make_column_literal(bind[identifier])
104
100
  terms = identifier.split(".")
@@ -746,6 +746,10 @@ class Query(QueryBase):
746
746
  driver=self._driver,
747
747
  )
748
748
 
749
+ def _skip_governor_validation(self) -> Query:
750
+ tree = self._tree.model_copy(update={"validateGovernorConstraints": False})
751
+ return Query(tree=tree, driver=self._driver)
752
+
749
753
  def _join_dataset_search_impl(
750
754
  self,
751
755
  dataset_type: str | DatasetType,
@@ -71,7 +71,10 @@ class PredicateConstraintsSummary:
71
71
  )
72
72
 
73
73
  def apply_default_data_id(
74
- self, default_data_id: DataCoordinate, query_dimensions: DimensionGroup
74
+ self,
75
+ default_data_id: DataCoordinate,
76
+ query_dimensions: DimensionGroup,
77
+ validate_governor_constraints: bool,
75
78
  ) -> None:
76
79
  """Augment the predicate and summary by adding missing constraints for
77
80
  governor dimensions using a default data ID.
@@ -81,9 +84,11 @@ class PredicateConstraintsSummary:
81
84
  default_data_id : `DataCoordinate`
82
85
  Data ID values that will be used to constrain the query if governor
83
86
  dimensions have not already been constrained by the predicate.
84
-
85
87
  query_dimensions : `DimensionGroup`
86
88
  The set of dimensions returned in result rows from the query.
89
+ validate_governor_constraints : `bool`
90
+ If `True`, enforce the requirement that governor dimensions must be
91
+ constrained if any dimensions that depend on them have constraints.
87
92
  """
88
93
  # Find governor dimensions required by the predicate.
89
94
  # If these are not constrained by the predicate or the default data ID,
@@ -108,7 +113,7 @@ class PredicateConstraintsSummary:
108
113
  self.predicate = self.predicate.logical_and(
109
114
  _create_data_id_predicate(governor, data_id_value, query_dimensions.universe)
110
115
  )
111
- elif governor in where_governors:
116
+ elif governor in where_governors and validate_governor_constraints:
112
117
  # Check that the predicate doesn't reference any dimensions
113
118
  # without constraining their governor dimensions, since
114
119
  # that's a particularly easy mistake to make and it's
@@ -110,7 +110,7 @@ def is_dataset_field(s: str) -> TypeGuard[DatasetFieldName]:
110
110
  class QueryTreeBase(pydantic.BaseModel):
111
111
  """Base class for all non-primitive types in a query tree."""
112
112
 
113
- model_config = pydantic.ConfigDict(frozen=True, extra="forbid", strict=True)
113
+ model_config = pydantic.ConfigDict(frozen=True, strict=True)
114
114
 
115
115
 
116
116
  class ColumnExpressionBase(QueryTreeBase, ABC):
@@ -146,6 +146,11 @@ class QueryTree(QueryTreeBase):
146
146
  predicate: Predicate = Predicate.from_bool(True)
147
147
  """Boolean expression trees whose logical AND defines a row filter."""
148
148
 
149
+ validateGovernorConstraints: bool = True
150
+ """If True, enforce the requirement that governor dimensions must be
151
+ constrained if any dimensions that depend on them have constraints.
152
+ """
153
+
149
154
  def iter_all_dataset_searches(self) -> Iterator[tuple[str | AnyDatasetType, DatasetSearch]]:
150
155
  yield from self.datasets.items()
151
156
  if self.any_dataset is not None:
@@ -1449,6 +1449,9 @@ class Registry(ABC):
1449
1449
  """
1450
1450
  return None
1451
1451
 
1452
- storageClasses: StorageClassFactory
1453
- """All storage classes known to the registry (`StorageClassFactory`).
1454
- """
1452
+ @property
1453
+ def storageClasses(self) -> StorageClassFactory:
1454
+ """All storage classes known to the registry
1455
+ (`StorageClassFactory`).
1456
+ """
1457
+ raise NotImplementedError()