lsst-daf-butler 30.2025.5000__py3-none-any.whl → 30.2025.5200__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/daf/butler/_formatter.py +2 -9
- lsst/daf/butler/_rubin/temporary_for_ingest.py +207 -0
- lsst/daf/butler/datastore/record_data.py +1 -1
- lsst/daf/butler/datastores/fileDatastore.py +14 -0
- lsst/daf/butler/registry/bridge/monolithic.py +17 -13
- lsst/daf/butler/registry/datasets/byDimensions/_manager.py +49 -45
- lsst/daf/butler/registry/interfaces/_database.py +6 -1
- lsst/daf/butler/version.py +1 -1
- {lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/METADATA +1 -1
- {lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/RECORD +18 -17
- {lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/WHEEL +0 -0
- {lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/entry_points.txt +0 -0
- {lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/licenses/LICENSE +0 -0
- {lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/licenses/bsd_license.txt +0 -0
- {lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/licenses/gpl-v3.0.txt +0 -0
- {lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/top_level.txt +0 -0
- {lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/zip-safe +0 -0
lsst/daf/butler/_formatter.py
CHANGED
|
@@ -54,6 +54,7 @@ from ._config import Config
|
|
|
54
54
|
from ._config_support import LookupKey, processLookupConfigs
|
|
55
55
|
from ._file_descriptor import FileDescriptor
|
|
56
56
|
from ._location import Location
|
|
57
|
+
from ._rubin.temporary_for_ingest import TemporaryForIngest
|
|
57
58
|
from .dimensions import DataCoordinate, DimensionUniverse
|
|
58
59
|
from .mapping_factory import MappingFactory
|
|
59
60
|
|
|
@@ -1031,15 +1032,7 @@ class FormatterV2:
|
|
|
1031
1032
|
"""
|
|
1032
1033
|
cache_manager = self._ensure_cache(cache_manager)
|
|
1033
1034
|
|
|
1034
|
-
|
|
1035
|
-
# using a local file system -- that gives us atomic writes.
|
|
1036
|
-
# If a process is killed as the file is being written we do not
|
|
1037
|
-
# want it to remain in the correct place but in corrupt state.
|
|
1038
|
-
# For local files write to the output directory not temporary dir.
|
|
1039
|
-
prefix = uri.dirname() if uri.isLocal else None
|
|
1040
|
-
if prefix is not None:
|
|
1041
|
-
prefix.mkdir()
|
|
1042
|
-
with ResourcePath.temporary_uri(suffix=uri.getExtension(), prefix=prefix) as temporary_uri:
|
|
1035
|
+
with TemporaryForIngest.make_path(uri) as temporary_uri:
|
|
1043
1036
|
# Need to configure the formatter to write to a different
|
|
1044
1037
|
# location and that needs us to overwrite internals
|
|
1045
1038
|
log.debug("Writing dataset to temporary location at %s", temporary_uri)
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
# This file is part of daf_butler.
|
|
2
|
+
#
|
|
3
|
+
# Developed for the LSST Data Management System.
|
|
4
|
+
# This product includes software developed by the LSST Project
|
|
5
|
+
# (http://www.lsst.org).
|
|
6
|
+
# See the COPYRIGHT file at the top-level directory of this distribution
|
|
7
|
+
# for details of code ownership.
|
|
8
|
+
#
|
|
9
|
+
# This software is dual licensed under the GNU General Public License and also
|
|
10
|
+
# under a 3-clause BSD license. Recipients may choose which of these licenses
|
|
11
|
+
# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
|
|
12
|
+
# respectively. If you choose the GPL option then the following text applies
|
|
13
|
+
# (but note that there is still no warranty even if you opt for BSD instead):
|
|
14
|
+
#
|
|
15
|
+
# This program is free software: you can redistribute it and/or modify
|
|
16
|
+
# it under the terms of the GNU General Public License as published by
|
|
17
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
18
|
+
# (at your option) any later version.
|
|
19
|
+
#
|
|
20
|
+
# This program is distributed in the hope that it will be useful,
|
|
21
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
22
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
23
|
+
# GNU General Public License for more details.
|
|
24
|
+
#
|
|
25
|
+
# You should have received a copy of the GNU General Public License
|
|
26
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
__all__ = ("TemporaryForIngest",)
|
|
31
|
+
|
|
32
|
+
import dataclasses
|
|
33
|
+
import glob
|
|
34
|
+
from contextlib import contextmanager
|
|
35
|
+
from typing import TYPE_CHECKING, Self, cast
|
|
36
|
+
|
|
37
|
+
from lsst.resources import ResourcePath
|
|
38
|
+
|
|
39
|
+
if TYPE_CHECKING:
|
|
40
|
+
from collections.abc import Iterator
|
|
41
|
+
from types import TracebackType
|
|
42
|
+
|
|
43
|
+
from .._butler import Butler
|
|
44
|
+
from .._dataset_ref import DatasetRef
|
|
45
|
+
from .._file_dataset import FileDataset
|
|
46
|
+
from .._limited_butler import LimitedButler
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclasses.dataclass
|
|
50
|
+
class TemporaryForIngest:
|
|
51
|
+
"""A context manager for generating temporary paths that will be ingested
|
|
52
|
+
as butler datasets.
|
|
53
|
+
|
|
54
|
+
Notes
|
|
55
|
+
-----
|
|
56
|
+
Neither this class nor its `make_path` method run ingest automatically when
|
|
57
|
+
their context manager is exited; the `ingest` method must always be called
|
|
58
|
+
explicitly.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
butler: Butler
|
|
62
|
+
"""Full butler to obtain a predicted path from and ingest into."""
|
|
63
|
+
|
|
64
|
+
ref: DatasetRef
|
|
65
|
+
"""Description of the dataset to ingest."""
|
|
66
|
+
|
|
67
|
+
dataset: FileDataset = dataclasses.field(init=False)
|
|
68
|
+
"""The dataset that will be passed to `Butler.ingest`."""
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def path(self) -> ResourcePath:
|
|
72
|
+
"""The temporary path.
|
|
73
|
+
|
|
74
|
+
Guaranteed to be a local POSIX path.
|
|
75
|
+
"""
|
|
76
|
+
return cast(ResourcePath, self.dataset.path)
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def ospath(self) -> str:
|
|
80
|
+
"""The temporary path as a complete filename."""
|
|
81
|
+
return self.path.ospath
|
|
82
|
+
|
|
83
|
+
@classmethod
|
|
84
|
+
@contextmanager
|
|
85
|
+
def make_path(cls, final_path: ResourcePath) -> Iterator[ResourcePath]:
|
|
86
|
+
"""Return a temporary path context manager given the predicted final
|
|
87
|
+
path.
|
|
88
|
+
|
|
89
|
+
Parameters
|
|
90
|
+
----------
|
|
91
|
+
final_path : `lsst.resources.ResourcePath`
|
|
92
|
+
Predicted final path.
|
|
93
|
+
|
|
94
|
+
Returns
|
|
95
|
+
-------
|
|
96
|
+
context : `contextlib.AbstractContextManager`
|
|
97
|
+
A context manager that yields the temporary
|
|
98
|
+
`~lsst.resources.ResourcePath` when entered and deletes that file
|
|
99
|
+
when exited.
|
|
100
|
+
"""
|
|
101
|
+
# Always write to a temporary even if using a local file system -- that
|
|
102
|
+
# gives us atomic writes. If a process is killed as the file is being
|
|
103
|
+
# written we do not want it to remain in the correct place but in
|
|
104
|
+
# corrupt state. For local files write to the output directory not
|
|
105
|
+
# temporary dir.
|
|
106
|
+
prefix = final_path.dirname() if final_path.isLocal else None
|
|
107
|
+
if prefix is not None:
|
|
108
|
+
prefix.mkdir()
|
|
109
|
+
with ResourcePath.temporary_uri(
|
|
110
|
+
suffix=cls._get_temporary_suffix(final_path), prefix=prefix
|
|
111
|
+
) as temporary_path:
|
|
112
|
+
yield temporary_path
|
|
113
|
+
|
|
114
|
+
def ingest(self, record_validation_info: bool = True) -> None:
|
|
115
|
+
"""Ingest the file into the butler.
|
|
116
|
+
|
|
117
|
+
Parameters
|
|
118
|
+
----------
|
|
119
|
+
record_validation_info : `bool`, optional
|
|
120
|
+
Whether to- record the file size and checksum upon ingest.
|
|
121
|
+
"""
|
|
122
|
+
self.butler.ingest(self.dataset, transfer="move", record_validation_info=record_validation_info)
|
|
123
|
+
|
|
124
|
+
def __enter__(self) -> Self:
|
|
125
|
+
from .._file_dataset import FileDataset
|
|
126
|
+
|
|
127
|
+
final_path = self.butler.getURI(self.ref, predict=True).replace(fragment="")
|
|
128
|
+
prefix = final_path.dirname() if final_path.isLocal else None
|
|
129
|
+
if prefix is not None:
|
|
130
|
+
prefix.mkdir()
|
|
131
|
+
self._temporary_path_context = self.make_path(final_path)
|
|
132
|
+
temporary_path = self._temporary_path_context.__enter__()
|
|
133
|
+
self.dataset = FileDataset(temporary_path, [self.ref], formatter=None)
|
|
134
|
+
return self
|
|
135
|
+
|
|
136
|
+
def __exit__(
|
|
137
|
+
self,
|
|
138
|
+
exc_type: type[BaseException] | None,
|
|
139
|
+
exc_value: BaseException | None,
|
|
140
|
+
traceback: TracebackType | None,
|
|
141
|
+
) -> bool | None:
|
|
142
|
+
return self._temporary_path_context.__exit__(exc_type, exc_value, traceback)
|
|
143
|
+
|
|
144
|
+
@classmethod
|
|
145
|
+
def find_orphaned_temporaries_by_path(cls, final_path: ResourcePath) -> list[ResourcePath]:
|
|
146
|
+
"""Search for temporary files that were not successfully ingested.
|
|
147
|
+
|
|
148
|
+
Parameters
|
|
149
|
+
----------
|
|
150
|
+
final_path : `lsst.resources.ResourcePath`
|
|
151
|
+
Final path a successfully-ingested file would have.
|
|
152
|
+
|
|
153
|
+
Returns
|
|
154
|
+
-------
|
|
155
|
+
paths : `list` [ `lsst.resources.ResourcePath` ]
|
|
156
|
+
Files that look like temporaries that might have been created while
|
|
157
|
+
trying to write the target dataset.
|
|
158
|
+
|
|
159
|
+
Notes
|
|
160
|
+
-----
|
|
161
|
+
Orphaned files are only possible when a context manager is interrupted
|
|
162
|
+
by a hard error that prevents any cleanup code from running (e.g.
|
|
163
|
+
sudden loss of power).
|
|
164
|
+
"""
|
|
165
|
+
if not final_path.isLocal:
|
|
166
|
+
# We return true tempfile for non-local predicted paths, so orphans
|
|
167
|
+
# are not our problem (the OS etc. will take care of them).
|
|
168
|
+
return []
|
|
169
|
+
return [
|
|
170
|
+
ResourcePath(filename)
|
|
171
|
+
for filename in glob.glob(
|
|
172
|
+
f"{glob.escape(final_path.dirname().ospath)}*{glob.escape(cls._get_temporary_suffix(final_path))}"
|
|
173
|
+
)
|
|
174
|
+
if filename != final_path.ospath
|
|
175
|
+
]
|
|
176
|
+
|
|
177
|
+
@classmethod
|
|
178
|
+
def find_orphaned_temporaries_by_ref(cls, ref: DatasetRef, butler: LimitedButler) -> list[ResourcePath]:
|
|
179
|
+
"""Search for temporary files that were not successfully ingested.
|
|
180
|
+
|
|
181
|
+
Parameters
|
|
182
|
+
----------
|
|
183
|
+
ref : `..DatasetRef`
|
|
184
|
+
A dataset reference the temporaries correspond to.
|
|
185
|
+
butler : `lsst.daf.butler.LimitedButler`
|
|
186
|
+
Butler that can be used to obtain a predicted URI for a dataset.
|
|
187
|
+
|
|
188
|
+
Returns
|
|
189
|
+
-------
|
|
190
|
+
paths : `list` [ `lsst.resources.ResourcePath` ]
|
|
191
|
+
Files that look like temporaries that might have been created while
|
|
192
|
+
trying to write the target dataset.
|
|
193
|
+
|
|
194
|
+
Notes
|
|
195
|
+
-----
|
|
196
|
+
Orphaned files are only possible when a context manager is interrupted
|
|
197
|
+
by a hard error that prevents any cleanup code from running (e.g.
|
|
198
|
+
sudden loss of power).
|
|
199
|
+
"""
|
|
200
|
+
final_path = butler.getURI(ref, predict=True).replace(fragment="")
|
|
201
|
+
return cls.find_orphaned_temporaries_by_path(final_path)
|
|
202
|
+
|
|
203
|
+
@staticmethod
|
|
204
|
+
def _get_temporary_suffix(path: ResourcePath) -> str:
|
|
205
|
+
ext = path.getExtension()
|
|
206
|
+
basename = path.basename().removesuffix(ext)
|
|
207
|
+
return f"{basename}.tmp{ext}"
|
|
@@ -49,7 +49,7 @@ if TYPE_CHECKING:
|
|
|
49
49
|
# Pydantic requires the possible value types to be explicitly enumerated in
|
|
50
50
|
# order for `uuid.UUID` in particular to work. `typing.Any` does not work
|
|
51
51
|
# here.
|
|
52
|
-
_Record: TypeAlias = dict[str, int | str |
|
|
52
|
+
_Record: TypeAlias = dict[str, int | str | None]
|
|
53
53
|
|
|
54
54
|
|
|
55
55
|
class SerializedDatastoreRecordData(pydantic.BaseModel):
|
|
@@ -3166,6 +3166,20 @@ class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
|
|
|
3166
3166
|
|
|
3167
3167
|
def export_records(self, refs: Iterable[DatasetIdRef]) -> Mapping[str, DatastoreRecordData]:
|
|
3168
3168
|
# Docstring inherited from the base class.
|
|
3169
|
+
|
|
3170
|
+
# This call to 'bridge.check' filters out "partially deleted" datasets.
|
|
3171
|
+
# Specifically, ones in the unusual edge state that:
|
|
3172
|
+
# 1. They have an entry in the registry dataset tables
|
|
3173
|
+
# 2. They were "trashed" from the datastore, so they are not
|
|
3174
|
+
# present in the "dataset_location" table.)
|
|
3175
|
+
# 3. But the trash has not been "emptied", so there are still entries
|
|
3176
|
+
# in the "opaque" datastore records table.
|
|
3177
|
+
#
|
|
3178
|
+
# As far as I can tell, this can only occur in the case of a concurrent
|
|
3179
|
+
# or aborted call to `Butler.pruneDatasets(unstore=True, purge=False)`.
|
|
3180
|
+
# Datasets (with or without files existing on disk) can persist in
|
|
3181
|
+
# this zombie state indefinitely, until someone manually empties
|
|
3182
|
+
# the trash.
|
|
3169
3183
|
exported_refs = list(self._bridge.check(refs))
|
|
3170
3184
|
ids = {ref.id for ref in exported_refs}
|
|
3171
3185
|
records: dict[DatasetId, dict[str, list[StoredDatastoreItemInfo]]] = {id: {} for id in ids}
|
|
@@ -215,20 +215,24 @@ class MonolithicDatastoreRegistryBridge(DatastoreRegistryBridge):
|
|
|
215
215
|
def check(self, refs: Iterable[DatasetIdRef]) -> Iterable[DatasetIdRef]:
|
|
216
216
|
# Docstring inherited from DatastoreRegistryBridge
|
|
217
217
|
byId = {ref.id: ref for ref in refs}
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
.
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
self._tables.dataset_location
|
|
224
|
-
|
|
218
|
+
found: list[DatasetIdRef] = []
|
|
219
|
+
with self._db.session():
|
|
220
|
+
for batch in chunk_iterable(byId.keys(), 50000):
|
|
221
|
+
sql = (
|
|
222
|
+
sqlalchemy.sql.select(self._tables.dataset_location.columns.dataset_id)
|
|
223
|
+
.select_from(self._tables.dataset_location)
|
|
224
|
+
.where(
|
|
225
|
+
sqlalchemy.sql.and_(
|
|
226
|
+
self._tables.dataset_location.columns.datastore_name == self.datastoreName,
|
|
227
|
+
self._tables.dataset_location.columns.dataset_id.in_(batch),
|
|
228
|
+
)
|
|
229
|
+
)
|
|
225
230
|
)
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
yield byId[row.dataset_id]
|
|
231
|
+
with self._db.query(sql) as sql_result:
|
|
232
|
+
sql_ids = sql_result.scalars().all()
|
|
233
|
+
found.extend(byId[id] for id in sql_ids)
|
|
234
|
+
|
|
235
|
+
return found
|
|
232
236
|
|
|
233
237
|
@contextmanager
|
|
234
238
|
def emptyTrash(
|
|
@@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
|
|
12
12
|
import astropy.time
|
|
13
13
|
import sqlalchemy
|
|
14
14
|
|
|
15
|
+
from lsst.utils.iteration import chunk_iterable
|
|
16
|
+
|
|
15
17
|
from .... import ddl
|
|
16
18
|
from ...._collection_type import CollectionType
|
|
17
19
|
from ...._dataset_ref import DatasetId, DatasetIdFactory, DatasetIdGenEnum, DatasetRef
|
|
@@ -424,17 +426,18 @@ class ByDimensionsDatasetRecordStorageManagerUUID(DatasetRecordStorageManager):
|
|
|
424
426
|
return result
|
|
425
427
|
|
|
426
428
|
def get_dataset_refs(self, ids: list[DatasetId]) -> list[DatasetRef]:
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
id_col
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
429
|
+
dataset_type_map: dict[DatasetId, DatasetType] = {}
|
|
430
|
+
for batch in chunk_iterable(set(ids), 50000):
|
|
431
|
+
# Look up the dataset types corresponding to the given Dataset IDs.
|
|
432
|
+
id_col = self._static.dataset.columns["id"]
|
|
433
|
+
sql = sqlalchemy.sql.select(
|
|
434
|
+
id_col,
|
|
435
|
+
self._static.dataset.columns["dataset_type_id"],
|
|
436
|
+
).where(id_col.in_(batch))
|
|
437
|
+
with self._db.query(sql) as sql_result:
|
|
438
|
+
dataset_rows = sql_result.mappings().all()
|
|
439
|
+
for row in dataset_rows:
|
|
440
|
+
dataset_type_map[row["id"]] = self._get_dataset_type_by_id(row["dataset_type_id"])
|
|
438
441
|
|
|
439
442
|
# Group the given dataset IDs by the DimensionGroup of their dataset
|
|
440
443
|
# types -- there is a separate tags table for each DimensionGroup.
|
|
@@ -448,40 +451,41 @@ class ByDimensionsDatasetRecordStorageManagerUUID(DatasetRecordStorageManager):
|
|
|
448
451
|
# data IDs corresponding to the UUIDs found from the dataset table.
|
|
449
452
|
dynamic_tables = self._get_dynamic_tables(dimension_group)
|
|
450
453
|
tags_table = self._get_tags_table(dynamic_tables)
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
454
|
+
for batch in chunk_iterable(datasets, 50000):
|
|
455
|
+
tags_sql = tags_table.select().where(tags_table.columns["dataset_id"].in_(batch))
|
|
456
|
+
# Join in the collection table to fetch the run name.
|
|
457
|
+
collection_column = tags_table.columns[self._collections.getCollectionForeignKeyName()]
|
|
458
|
+
joined_collections = self._collections.join_collections_sql(collection_column, tags_sql)
|
|
459
|
+
tags_sql = joined_collections.joined_sql
|
|
460
|
+
run_name_column = joined_collections.name_column
|
|
461
|
+
tags_sql = tags_sql.add_columns(run_name_column)
|
|
462
|
+
# Tags table includes run collections and tagged
|
|
463
|
+
# collections.
|
|
464
|
+
# In theory the data ID for a given dataset should be the
|
|
465
|
+
# same in both, but nothing actually guarantees this.
|
|
466
|
+
# So skip any tagged collections, using the run collection
|
|
467
|
+
# as the definitive definition.
|
|
468
|
+
tags_sql = tags_sql.where(joined_collections.type_column == int(CollectionType.RUN))
|
|
469
|
+
|
|
470
|
+
with self._db.query(tags_sql) as sql_result:
|
|
471
|
+
data_id_rows = sql_result.mappings().all()
|
|
472
|
+
|
|
473
|
+
assert run_name_column.key is not None
|
|
474
|
+
for data_id_row in data_id_rows:
|
|
475
|
+
id = data_id_row["dataset_id"]
|
|
476
|
+
dataset_type = dataset_type_map[id]
|
|
477
|
+
run_name = data_id_row[run_name_column.key]
|
|
478
|
+
data_id = DataCoordinate.from_required_values(
|
|
479
|
+
dimension_group,
|
|
480
|
+
tuple(data_id_row[dimension] for dimension in dimension_group.required),
|
|
481
|
+
)
|
|
482
|
+
ref = DatasetRef(
|
|
483
|
+
datasetType=dataset_type,
|
|
484
|
+
dataId=data_id,
|
|
485
|
+
id=id,
|
|
486
|
+
run=run_name,
|
|
487
|
+
)
|
|
488
|
+
output_refs.append(ref)
|
|
485
489
|
|
|
486
490
|
return output_refs
|
|
487
491
|
|
|
@@ -1562,7 +1562,12 @@ class Database(ABC):
|
|
|
1562
1562
|
return None
|
|
1563
1563
|
else:
|
|
1564
1564
|
sql = table.insert()
|
|
1565
|
-
|
|
1565
|
+
ids = []
|
|
1566
|
+
for row in rows:
|
|
1567
|
+
key = connection.execute(sql, row).inserted_primary_key
|
|
1568
|
+
assert key is not None
|
|
1569
|
+
ids.append(key[0])
|
|
1570
|
+
return ids
|
|
1566
1571
|
|
|
1567
1572
|
@abstractmethod
|
|
1568
1573
|
def replace(self, table: sqlalchemy.schema.Table, *rows: dict) -> None:
|
lsst/daf/butler/version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
__all__ = ["__version__"]
|
|
2
|
-
__version__ = "30.2025.
|
|
2
|
+
__version__ = "30.2025.5200"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-daf-butler
|
|
3
|
-
Version: 30.2025.
|
|
3
|
+
Version: 30.2025.5200
|
|
4
4
|
Summary: An abstraction layer for reading and writing astronomical data to datastores.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License-Expression: BSD-3-Clause OR GPL-3.0-or-later
|
|
@@ -20,7 +20,7 @@ lsst/daf/butler/_exceptions.py,sha256=XPNeWQ4YUOHCIKHc2cL4RrANQf03fAYyQsN7hItnBJ
|
|
|
20
20
|
lsst/daf/butler/_exceptions_legacy.py,sha256=IBE66bI2Tn1Cy-Kz2g1R-8PFxY6xKtbnaLH3rWvqv8k,2577
|
|
21
21
|
lsst/daf/butler/_file_dataset.py,sha256=KF_o5zi01L0gK4t_Ncb3BSbCTiyf9zmwwTpWoNgasRI,6051
|
|
22
22
|
lsst/daf/butler/_file_descriptor.py,sha256=PIYT9O2NRbNndtI2rMXfx7tP6nUcTnd9fxFVsq-Hu1s,3841
|
|
23
|
-
lsst/daf/butler/_formatter.py,sha256=
|
|
23
|
+
lsst/daf/butler/_formatter.py,sha256=ZTfYUvRRHMi9u1qohlJ0G-jcgeO7vGDzt3cV95Lze2U,82793
|
|
24
24
|
lsst/daf/butler/_labeled_butler_factory.py,sha256=Ly1oaSBDTNDaDtEJFEEys4KRc359Gcr9KLLwi-ggZHE,9052
|
|
25
25
|
lsst/daf/butler/_limited_butler.py,sha256=GTsAzymQf11daCtDhI4WBqee8zyBzKN9CD85gOeG-WU,17514
|
|
26
26
|
lsst/daf/butler/_location.py,sha256=oX-AKZeSdToeKBxvB6y-pxLMiFwF-3nz3YleH5SiA3A,10451
|
|
@@ -51,9 +51,10 @@ lsst/daf/butler/repo_relocation.py,sha256=Ivhx2xU4slc53Z6RExhNnquMr2Hx-S8h62emml
|
|
|
51
51
|
lsst/daf/butler/time_utils.py,sha256=MVTfOFI2xt3IeA46pa-fWY2kJRwSzaQyq1uzeUABcTM,11805
|
|
52
52
|
lsst/daf/butler/timespan_database_representation.py,sha256=rYeQ_vp6gneRjboqV-gvNW0DWhm1QJM-KnVzFTDVZ0I,24550
|
|
53
53
|
lsst/daf/butler/utils.py,sha256=5u50COK5z4u31grOhmQF7mFz55biNLOvSMRdQjEdsjo,5140
|
|
54
|
-
lsst/daf/butler/version.py,sha256=
|
|
54
|
+
lsst/daf/butler/version.py,sha256=ViUKxGsTVoNOYgWzjL3nVBfNBOiNf71erkUgj_xAp8o,55
|
|
55
55
|
lsst/daf/butler/_rubin/__init__.py,sha256=9z5kmc6LJ3C_iPFV46cvdlQ2qOGJbZh-2Ft5Z-rbE28,1569
|
|
56
56
|
lsst/daf/butler/_rubin/file_datasets.py,sha256=P5_BIhxpVj9qfLuLiI2_dClMHsjO5Qm5oDXVr3WntNU,3607
|
|
57
|
+
lsst/daf/butler/_rubin/temporary_for_ingest.py,sha256=nmNR3XM0HSA_vlqx8nl13pfr_53HJKUVo0iWhcOX3To,7672
|
|
57
58
|
lsst/daf/butler/_utilities/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
|
|
58
59
|
lsst/daf/butler/_utilities/locked_object.py,sha256=3RQf0Ish55mfQAfBy3V4Tfnfq5Q7-cxrwTlQMUhrIno,1931
|
|
59
60
|
lsst/daf/butler/_utilities/named_locks.py,sha256=Zj_u1rZELaiWec3wJfkgmGD_YiZMLVxbMQmdbbVgk5E,2286
|
|
@@ -96,11 +97,11 @@ lsst/daf/butler/datastore/composites.py,sha256=NZ7rBK5yH-hrtpqZxC8d49UwwQqWZlEmm
|
|
|
96
97
|
lsst/daf/butler/datastore/constraints.py,sha256=OcUXuXZq1UBnuQqq8U7Hp3Ezqu0RBN8pIo93BEq7lyI,5921
|
|
97
98
|
lsst/daf/butler/datastore/file_templates.py,sha256=xnb4ZheW6NqeCE__vkIvLF91d57nhfcX3ynGWI0rX_0,35095
|
|
98
99
|
lsst/daf/butler/datastore/generic_base.py,sha256=C15FN1fDVxs-XjeDc1hw5un3MMMVIaZN4QdFbjqQ168,5176
|
|
99
|
-
lsst/daf/butler/datastore/record_data.py,sha256=
|
|
100
|
+
lsst/daf/butler/datastore/record_data.py,sha256=kY-fqXrhdffpIHgczgBDvGYb4VnDoFtIeNfqUpteR1w,10416
|
|
100
101
|
lsst/daf/butler/datastore/stored_file_info.py,sha256=s_9LsLZgIF7TwjiXEXgNPz3OaaRiUyzmf5SOdJ1elhk,15838
|
|
101
102
|
lsst/daf/butler/datastores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
102
103
|
lsst/daf/butler/datastores/chainedDatastore.py,sha256=3DUKoGLVbmqyqm2K4-De3GKswE4WALFp0P9ul6OF-Qk,56220
|
|
103
|
-
lsst/daf/butler/datastores/fileDatastore.py,sha256=
|
|
104
|
+
lsst/daf/butler/datastores/fileDatastore.py,sha256=AN4IKz5hcz2jcfKQUhEnT4DxiPDs1nf5_WF5P_t-u68,136876
|
|
104
105
|
lsst/daf/butler/datastores/inMemoryDatastore.py,sha256=ZLgJuPnKJPsnaZQ_8rTZn02YHl2MxsQ_kcxCXhcc5dk,30213
|
|
105
106
|
lsst/daf/butler/datastores/file_datastore/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
|
|
106
107
|
lsst/daf/butler/datastores/file_datastore/get.py,sha256=w-5tuXxdthsSiOlI7MBFrFO6zWM_f1Uqqoy8VGAdvcQ,16964
|
|
@@ -203,7 +204,7 @@ lsst/daf/butler/registry/versions.py,sha256=egvrctt_1wBzZgh8iSfySaQJQ9bkx_9bUJWk
|
|
|
203
204
|
lsst/daf/butler/registry/wildcards.py,sha256=akMGgqDkVM0mQ9RAFENv0IrnoUyMP3mhODYXDaWIQ8o,20277
|
|
204
205
|
lsst/daf/butler/registry/bridge/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
|
|
205
206
|
lsst/daf/butler/registry/bridge/ephemeral.py,sha256=QSKMRwQTAHnMOwdigH5tSNrQaM2dK-IXymb7x87bY-w,5845
|
|
206
|
-
lsst/daf/butler/registry/bridge/monolithic.py,sha256=
|
|
207
|
+
lsst/daf/butler/registry/bridge/monolithic.py,sha256=NLMLPh9ORChrtHT0be8PtmIlnnDQlmeAsM2MPmTeZZM,18386
|
|
207
208
|
lsst/daf/butler/registry/collections/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
|
|
208
209
|
lsst/daf/butler/registry/collections/_base.py,sha256=MxrvTh81aHUULXtf5hHzK6iAjwkes5NckouHMhTFdPM,37480
|
|
209
210
|
lsst/daf/butler/registry/collections/nameKey.py,sha256=UZCwiY0hOyB1NXA_1ZzjD4tuKnQ_jvrMmPQlUN6tWuk,12890
|
|
@@ -214,7 +215,7 @@ lsst/daf/butler/registry/databases/sqlite.py,sha256=xW82mdbOoOOTBeuSYDQ9DiDPhnMG
|
|
|
214
215
|
lsst/daf/butler/registry/datasets/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
|
|
215
216
|
lsst/daf/butler/registry/datasets/byDimensions/__init__.py,sha256=BG4C7mhKFbCzvfQSI31CIV_iTMc1gYL_LT4Plyu6LdE,1323
|
|
216
217
|
lsst/daf/butler/registry/datasets/byDimensions/_dataset_type_cache.py,sha256=VmpKsw7PfHTu54cChAUH8qcOC2jizBLNJX_SE_DQ1yA,8315
|
|
217
|
-
lsst/daf/butler/registry/datasets/byDimensions/_manager.py,sha256=
|
|
218
|
+
lsst/daf/butler/registry/datasets/byDimensions/_manager.py,sha256=DeJhLIc7hqw19j_HYh-boQ8jow3NMurbhUegrQ8GO8s,71720
|
|
218
219
|
lsst/daf/butler/registry/datasets/byDimensions/summaries.py,sha256=MuRk2p6fAKhvjId3jWnuFzhqMnKNF31ugExD7a2g48k,18534
|
|
219
220
|
lsst/daf/butler/registry/datasets/byDimensions/tables.py,sha256=9MXcpRGuXQonEMmh-zQOUcYJVA9pvUuNXyZRZtP-KH8,25768
|
|
220
221
|
lsst/daf/butler/registry/dimensions/__init__.py,sha256=vLzPZYAJ-9r1cnqsP64MVpFgSw2166yOpq0iPMSdAvw,1298
|
|
@@ -223,7 +224,7 @@ lsst/daf/butler/registry/interfaces/__init__.py,sha256=IBMBBb1gyAx3o9uTufhQHtMrh
|
|
|
223
224
|
lsst/daf/butler/registry/interfaces/_attributes.py,sha256=z-njEpWLhmKU4S0KOCplrY4QeBGoKUhlPRtSdNS_4uw,7258
|
|
224
225
|
lsst/daf/butler/registry/interfaces/_bridge.py,sha256=pds0AhEXZVO3YguU-tkZ8J_NZrBj55HCGkHpBTXbBFQ,15924
|
|
225
226
|
lsst/daf/butler/registry/interfaces/_collections.py,sha256=q0xRy7gA4KacPvJb1Wtk-uDSHrjmv4xpD2qGmzpvdKM,28485
|
|
226
|
-
lsst/daf/butler/registry/interfaces/_database.py,sha256=
|
|
227
|
+
lsst/daf/butler/registry/interfaces/_database.py,sha256=YmbGySh0qXf8HJe_FQd69DgBMdx0HuytbhnPVIpYCJw,84706
|
|
227
228
|
lsst/daf/butler/registry/interfaces/_database_explain.py,sha256=CkALWwNeyrjRvKizWrxvcGDunIhB77kLtEuXscrXVOY,3052
|
|
228
229
|
lsst/daf/butler/registry/interfaces/_datasets.py,sha256=_vvEStau34AdZ45NdnrrgBh_9uSHmth7amnH538gj7A,24975
|
|
229
230
|
lsst/daf/butler/registry/interfaces/_dimensions.py,sha256=peUPhmQEdwvgZcSsEsR80OZBcKEyjBRvmAbEkvMKlKY,14504
|
|
@@ -337,13 +338,13 @@ lsst/daf/butler/transfers/__init__.py,sha256=M1YcFszSkNB5hB2pZwwGXqbJE2dKt4YXDin
|
|
|
337
338
|
lsst/daf/butler/transfers/_context.py,sha256=Ro_nf9NDw9IAr-Pw_NtcdotQKx34RbBbNubt20zwRXU,16449
|
|
338
339
|
lsst/daf/butler/transfers/_interfaces.py,sha256=Ia1NqcFR5E-Ik4zsXEe2fuMtNCJj5Yfe_gVHLTBtJDw,7490
|
|
339
340
|
lsst/daf/butler/transfers/_yaml.py,sha256=w_0GmrueuHVLfOfAXGHFBbWAl18tX6eSElbTC-2jRoc,32632
|
|
340
|
-
lsst_daf_butler-30.2025.
|
|
341
|
-
lsst_daf_butler-30.2025.
|
|
342
|
-
lsst_daf_butler-30.2025.
|
|
343
|
-
lsst_daf_butler-30.2025.
|
|
344
|
-
lsst_daf_butler-30.2025.
|
|
345
|
-
lsst_daf_butler-30.2025.
|
|
346
|
-
lsst_daf_butler-30.2025.
|
|
347
|
-
lsst_daf_butler-30.2025.
|
|
348
|
-
lsst_daf_butler-30.2025.
|
|
349
|
-
lsst_daf_butler-30.2025.
|
|
341
|
+
lsst_daf_butler-30.2025.5200.dist-info/licenses/COPYRIGHT,sha256=k1Vq0-Be_K-puaeW4UZnckPjksEL-MJh4XKiWcjMxJE,312
|
|
342
|
+
lsst_daf_butler-30.2025.5200.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
|
|
343
|
+
lsst_daf_butler-30.2025.5200.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
|
|
344
|
+
lsst_daf_butler-30.2025.5200.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
345
|
+
lsst_daf_butler-30.2025.5200.dist-info/METADATA,sha256=6Tv2auxb8OH06kwe8MiqdabMbq3hVqzM4RuFVtVxP2E,3813
|
|
346
|
+
lsst_daf_butler-30.2025.5200.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
347
|
+
lsst_daf_butler-30.2025.5200.dist-info/entry_points.txt,sha256=XsRxyTK3c-jGlKVuVnbpch3gtaO0lAA_fS3i2NGS5rw,59
|
|
348
|
+
lsst_daf_butler-30.2025.5200.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
|
|
349
|
+
lsst_daf_butler-30.2025.5200.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
350
|
+
lsst_daf_butler-30.2025.5200.dist-info/RECORD,,
|
|
File without changes
|
{lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/licenses/COPYRIGHT
RENAMED
|
File without changes
|
{lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{lsst_daf_butler-30.2025.5000.dist-info → lsst_daf_butler-30.2025.5200.dist-info}/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|