lsst-pipe-base 29.2025.4000__py3-none-any.whl → 29.2025.4200__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/pipe/base/_task_metadata.py +15 -0
- lsst/pipe/base/graph/_versionDeserializers.py +6 -5
- lsst/pipe/base/graph/graph.py +2 -1
- lsst/pipe/base/graph/graphSummary.py +30 -0
- lsst/pipe/base/pipeline_graph/io.py +1 -1
- lsst/pipe/base/quantum_graph/_common.py +23 -1
- lsst/pipe/base/quantum_graph/_multiblock.py +174 -103
- lsst/pipe/base/quantum_graph/_predicted.py +82 -6
- lsst/pipe/base/quantum_graph_builder.py +4 -4
- lsst/pipe/base/quantum_reports.py +45 -0
- lsst/pipe/base/simple_pipeline_executor.py +4 -1
- lsst/pipe/base/tests/mocks/_storage_class.py +45 -0
- lsst/pipe/base/version.py +1 -1
- {lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/METADATA +1 -1
- {lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/RECORD +23 -23
- {lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/WHEEL +0 -0
- {lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/entry_points.txt +0 -0
- {lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/licenses/LICENSE +0 -0
- {lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/licenses/bsd_license.txt +0 -0
- {lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/licenses/gpl-v3.0.txt +0 -0
- {lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/top_level.txt +0 -0
- {lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/zip-safe +0 -0
lsst/pipe/base/_task_metadata.py
CHANGED
|
@@ -686,6 +686,21 @@ class TaskMetadata(BaseModel):
|
|
|
686
686
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
687
687
|
return super().model_json_schema(*args, **kwargs)
|
|
688
688
|
|
|
689
|
+
@classmethod
|
|
690
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
691
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
692
|
+
return super().model_validate(*args, **kwargs)
|
|
693
|
+
|
|
694
|
+
@classmethod
|
|
695
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
696
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
697
|
+
return super().model_validate_json(*args, **kwargs)
|
|
698
|
+
|
|
699
|
+
@classmethod
|
|
700
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
701
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
702
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
703
|
+
|
|
689
704
|
|
|
690
705
|
# Needed because a TaskMetadata can contain a TaskMetadata.
|
|
691
706
|
TaskMetadata.model_rebuild()
|
|
@@ -38,7 +38,7 @@ from collections import defaultdict
|
|
|
38
38
|
from collections.abc import Callable
|
|
39
39
|
from dataclasses import dataclass
|
|
40
40
|
from types import SimpleNamespace
|
|
41
|
-
from typing import TYPE_CHECKING, ClassVar
|
|
41
|
+
from typing import TYPE_CHECKING, ClassVar
|
|
42
42
|
|
|
43
43
|
import networkx as nx
|
|
44
44
|
|
|
@@ -50,6 +50,7 @@ from lsst.daf.butler import (
|
|
|
50
50
|
Quantum,
|
|
51
51
|
SerializedDimensionRecord,
|
|
52
52
|
)
|
|
53
|
+
from lsst.daf.butler._rubin import generate_uuidv7
|
|
53
54
|
from lsst.utils import doImportType
|
|
54
55
|
|
|
55
56
|
from ..config import PipelineTaskConfig
|
|
@@ -242,7 +243,7 @@ class DeserializerV1(DeserializerBase):
|
|
|
242
243
|
|
|
243
244
|
# reconstruct node
|
|
244
245
|
qNode = pickle.loads(dump)
|
|
245
|
-
object.__setattr__(qNode, "nodeId",
|
|
246
|
+
object.__setattr__(qNode, "nodeId", generate_uuidv7())
|
|
246
247
|
|
|
247
248
|
# read the saved node, name. If it has been loaded, attach it, if
|
|
248
249
|
# not read in the taskDef first, and then load it
|
|
@@ -376,7 +377,7 @@ class DeserializerV2(DeserializerBase):
|
|
|
376
377
|
|
|
377
378
|
# reconstruct node
|
|
378
379
|
qNode = pickle.loads(dump)
|
|
379
|
-
object.__setattr__(qNode, "nodeId",
|
|
380
|
+
object.__setattr__(qNode, "nodeId", generate_uuidv7())
|
|
380
381
|
|
|
381
382
|
# read the saved node, name. If it has been loaded, attach it, if
|
|
382
383
|
# not read in the taskDef first, and then load it
|
|
@@ -599,11 +600,11 @@ class DeserializerV3(DeserializerBase):
|
|
|
599
600
|
# initInputRefs and initOutputRefs are optional
|
|
600
601
|
if (refs := taskDefDump.get("initInputRefs")) is not None:
|
|
601
602
|
initInputRefs[recreatedTaskDef.label] = [
|
|
602
|
-
|
|
603
|
+
DatasetRef.from_json(ref, universe=universe) for ref in refs
|
|
603
604
|
]
|
|
604
605
|
if (refs := taskDefDump.get("initOutputRefs")) is not None:
|
|
605
606
|
initOutputRefs[recreatedTaskDef.label] = [
|
|
606
|
-
|
|
607
|
+
DatasetRef.from_json(ref, universe=universe) for ref in refs
|
|
607
608
|
]
|
|
608
609
|
|
|
609
610
|
# rebuild the mappings that associate dataset type names with
|
lsst/pipe/base/graph/graph.py
CHANGED
|
@@ -59,6 +59,7 @@ from lsst.daf.butler import (
|
|
|
59
59
|
Quantum,
|
|
60
60
|
QuantumBackedButler,
|
|
61
61
|
)
|
|
62
|
+
from lsst.daf.butler._rubin import generate_uuidv7
|
|
62
63
|
from lsst.daf.butler.datastore.record_data import DatastoreRecordData
|
|
63
64
|
from lsst.daf.butler.persistence_context import PersistenceContextVars
|
|
64
65
|
from lsst.daf.butler.registry import ConflictingDefinitionError
|
|
@@ -246,7 +247,7 @@ class QuantumGraph:
|
|
|
246
247
|
"associated value in the mapping"
|
|
247
248
|
)
|
|
248
249
|
else:
|
|
249
|
-
nodeId =
|
|
250
|
+
nodeId = generate_uuidv7()
|
|
250
251
|
|
|
251
252
|
inits = quantum.initInputs.values()
|
|
252
253
|
inputs = quantum.inputs.values()
|
|
@@ -75,6 +75,21 @@ class QgraphTaskSummary(pydantic.BaseModel):
|
|
|
75
75
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
76
76
|
return super().model_json_schema(*args, **kwargs)
|
|
77
77
|
|
|
78
|
+
@classmethod
|
|
79
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
80
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
81
|
+
return super().model_validate(*args, **kwargs)
|
|
82
|
+
|
|
83
|
+
@classmethod
|
|
84
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
85
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
86
|
+
return super().model_validate_json(*args, **kwargs)
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
90
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
91
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
92
|
+
|
|
78
93
|
|
|
79
94
|
class QgraphSummary(pydantic.BaseModel):
|
|
80
95
|
"""Report for the QuantumGraph creation or reading."""
|
|
@@ -129,3 +144,18 @@ class QgraphSummary(pydantic.BaseModel):
|
|
|
129
144
|
def model_json_schema(cls, *args: Any, **kwargs: Any) -> Any:
|
|
130
145
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
131
146
|
return super().model_json_schema(*args, **kwargs)
|
|
147
|
+
|
|
148
|
+
@classmethod
|
|
149
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
150
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
151
|
+
return super().model_validate(*args, **kwargs)
|
|
152
|
+
|
|
153
|
+
@classmethod
|
|
154
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
155
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
156
|
+
return super().model_validate_json(*args, **kwargs)
|
|
157
|
+
|
|
158
|
+
@classmethod
|
|
159
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
160
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
161
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
@@ -650,7 +650,7 @@ class SerializedTaskSubset(pydantic.BaseModel):
|
|
|
650
650
|
"""
|
|
651
651
|
members = set(self.tasks)
|
|
652
652
|
if label in steps:
|
|
653
|
-
steps.
|
|
653
|
+
steps._dimensions_by_label[label] = frozenset(self.dimensions)
|
|
654
654
|
return TaskSubset(xgraph, label, members, self.description, steps)
|
|
655
655
|
|
|
656
656
|
|
|
@@ -229,6 +229,21 @@ class HeaderModel(pydantic.BaseModel):
|
|
|
229
229
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
230
230
|
return super().model_json_schema(*args, **kwargs)
|
|
231
231
|
|
|
232
|
+
@classmethod
|
|
233
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
234
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
235
|
+
return super().model_validate(*args, **kwargs)
|
|
236
|
+
|
|
237
|
+
@classmethod
|
|
238
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
239
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
240
|
+
return super().model_validate_json(*args, **kwargs)
|
|
241
|
+
|
|
242
|
+
@classmethod
|
|
243
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
244
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
245
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
246
|
+
|
|
232
247
|
|
|
233
248
|
class QuantumInfo(TypedDict):
|
|
234
249
|
"""A typed dictionary that annotates the attributes of the NetworkX graph
|
|
@@ -493,6 +508,7 @@ class BaseQuantumGraphReader:
|
|
|
493
508
|
*,
|
|
494
509
|
address_filename: str,
|
|
495
510
|
graph_type: str,
|
|
511
|
+
n_addresses: int,
|
|
496
512
|
page_size: int | None = None,
|
|
497
513
|
import_mode: TaskImportMode = TaskImportMode.ASSUME_CONSISTENT_EDGES,
|
|
498
514
|
) -> Iterator[Self]:
|
|
@@ -506,6 +522,8 @@ class BaseQuantumGraphReader:
|
|
|
506
522
|
Base filename for the address file.
|
|
507
523
|
graph_type : `str`
|
|
508
524
|
Value to expect for `HeaderModel.graph_type`.
|
|
525
|
+
n_addresses : `int`
|
|
526
|
+
Number of addresses to expect per row in the address file.
|
|
509
527
|
page_size : `int`, optional
|
|
510
528
|
Approximate number of bytes to read at once from address files.
|
|
511
529
|
Note that this does not set a page size for *all* reads, but it
|
|
@@ -539,7 +557,11 @@ class BaseQuantumGraphReader:
|
|
|
539
557
|
)
|
|
540
558
|
pipeline_graph = serialized_pipeline_graph.deserialize(import_mode)
|
|
541
559
|
with AddressReader.open_in_zip(
|
|
542
|
-
zf,
|
|
560
|
+
zf,
|
|
561
|
+
address_filename,
|
|
562
|
+
page_size=page_size,
|
|
563
|
+
int_size=header.int_size,
|
|
564
|
+
n_addresses=n_addresses,
|
|
543
565
|
) as address_reader:
|
|
544
566
|
yield cls(
|
|
545
567
|
header=header,
|
|
@@ -40,14 +40,13 @@ __all__ = (
|
|
|
40
40
|
)
|
|
41
41
|
|
|
42
42
|
import dataclasses
|
|
43
|
-
import itertools
|
|
44
43
|
import logging
|
|
45
44
|
import uuid
|
|
46
45
|
from collections.abc import Iterator
|
|
47
46
|
from contextlib import contextmanager
|
|
48
47
|
from io import BufferedReader, BytesIO
|
|
49
48
|
from operator import attrgetter
|
|
50
|
-
from typing import IO, TYPE_CHECKING,
|
|
49
|
+
from typing import IO, TYPE_CHECKING, Protocol, TypeAlias, TypeVar
|
|
51
50
|
|
|
52
51
|
import pydantic
|
|
53
52
|
|
|
@@ -61,6 +60,11 @@ _LOG = logging.getLogger(__name__)
|
|
|
61
60
|
_T = TypeVar("_T", bound=pydantic.BaseModel)
|
|
62
61
|
|
|
63
62
|
|
|
63
|
+
UUID_int: TypeAlias = int
|
|
64
|
+
|
|
65
|
+
MAX_UUID_INT: UUID_int = 2**128
|
|
66
|
+
|
|
67
|
+
|
|
64
68
|
DEFAULT_PAGE_SIZE: int = 5_000_000
|
|
65
69
|
"""Default page size for reading chunks of quantum graph files.
|
|
66
70
|
|
|
@@ -255,13 +259,51 @@ class AddressWriter:
|
|
|
255
259
|
self.write(stream, int_size=int_size)
|
|
256
260
|
|
|
257
261
|
|
|
262
|
+
@dataclasses.dataclass
|
|
263
|
+
class AddressPage:
|
|
264
|
+
"""A page of addresses in the `AddressReader`."""
|
|
265
|
+
|
|
266
|
+
file_offset: int
|
|
267
|
+
"""Offset in bytes to this page from the beginning of the file."""
|
|
268
|
+
|
|
269
|
+
begin: int
|
|
270
|
+
"""Index of the first row in this page."""
|
|
271
|
+
|
|
272
|
+
n_rows: int
|
|
273
|
+
"""Number of rows in this page."""
|
|
274
|
+
|
|
275
|
+
read: bool = False
|
|
276
|
+
"""Whether this page has already been read."""
|
|
277
|
+
|
|
278
|
+
@property
|
|
279
|
+
def end(self) -> int:
|
|
280
|
+
"""One past the last row index in this page."""
|
|
281
|
+
return self.begin + self.n_rows
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
@dataclasses.dataclass
|
|
285
|
+
class PageBounds:
|
|
286
|
+
"""A page index and the UUID interval that page covers."""
|
|
287
|
+
|
|
288
|
+
page_index: int
|
|
289
|
+
"""Index into the page array."""
|
|
290
|
+
|
|
291
|
+
uuid_int_begin: UUID_int
|
|
292
|
+
"""Integer representation of the smallest UUID in this page."""
|
|
293
|
+
|
|
294
|
+
uuid_int_end: UUID_int
|
|
295
|
+
"""One larger than the integer representation of the largest UUID in this
|
|
296
|
+
page.
|
|
297
|
+
"""
|
|
298
|
+
|
|
299
|
+
def __str__(self) -> str:
|
|
300
|
+
return f"{self.page_index} [{self.uuid_int_begin:x}:{self.uuid_int_end:x}]"
|
|
301
|
+
|
|
302
|
+
|
|
258
303
|
@dataclasses.dataclass
|
|
259
304
|
class AddressReader:
|
|
260
305
|
"""A helper object for reading address files for multi-block files."""
|
|
261
306
|
|
|
262
|
-
MAX_UUID_INT: ClassVar[int] = 2**128
|
|
263
|
-
"""The maximum value of a UUID's integer form."""
|
|
264
|
-
|
|
265
307
|
stream: IO[bytes]
|
|
266
308
|
"""Stream to read from."""
|
|
267
309
|
|
|
@@ -269,30 +311,28 @@ class AddressReader:
|
|
|
269
311
|
"""Size of each integer in bytes."""
|
|
270
312
|
|
|
271
313
|
n_rows: int
|
|
272
|
-
"""Number of
|
|
314
|
+
"""Number of rows in the file."""
|
|
273
315
|
|
|
274
316
|
n_addresses: int
|
|
275
317
|
"""Number of addresses in each row."""
|
|
276
318
|
|
|
277
|
-
|
|
278
|
-
"""
|
|
319
|
+
rows_per_page: int
|
|
320
|
+
"""Number of addresses in each page."""
|
|
279
321
|
|
|
280
|
-
rows: dict[uuid.UUID, AddressRow]
|
|
322
|
+
rows: dict[uuid.UUID, AddressRow] = dataclasses.field(default_factory=dict)
|
|
281
323
|
"""Rows that have already been read."""
|
|
282
324
|
|
|
283
|
-
|
|
284
|
-
"""
|
|
285
|
-
|
|
286
|
-
unread_pages: dict[int, int]
|
|
287
|
-
"""Pages that have not yet been read, as a mapping from page index to the
|
|
288
|
-
number of rows in that page.
|
|
325
|
+
rows_by_index: dict[int, AddressRow] = dataclasses.field(default_factory=dict)
|
|
326
|
+
"""Rows that have already been read, keyed by integer index."""
|
|
289
327
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
"""
|
|
328
|
+
pages: list[AddressPage] = dataclasses.field(default_factory=list)
|
|
329
|
+
page_bounds: dict[int, PageBounds] = dataclasses.field(default_factory=dict)
|
|
330
|
+
"""Mapping from page index to page boundary information."""
|
|
293
331
|
|
|
294
332
|
@classmethod
|
|
295
|
-
def from_stream(
|
|
333
|
+
def from_stream(
|
|
334
|
+
cls, stream: IO[bytes], *, page_size: int, n_addresses: int, int_size: int
|
|
335
|
+
) -> AddressReader:
|
|
296
336
|
"""Construct from a stream by reading the header.
|
|
297
337
|
|
|
298
338
|
Parameters
|
|
@@ -302,27 +342,47 @@ class AddressReader:
|
|
|
302
342
|
page_size : `int`
|
|
303
343
|
Approximate number of bytes to read at a time when searching for an
|
|
304
344
|
address.
|
|
305
|
-
|
|
306
|
-
|
|
345
|
+
n_addresses : `int`
|
|
346
|
+
Number of addresses to expect per row. This is checked against
|
|
347
|
+
the size embedded in the file.
|
|
348
|
+
int_size : `int`
|
|
349
|
+
Number of bytes to use for all integers. This is checked against
|
|
350
|
+
the size embedded in the file.
|
|
307
351
|
"""
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
352
|
+
header_size = cls.compute_header_size(int_size)
|
|
353
|
+
row_size = cls.compute_row_size(int_size, n_addresses)
|
|
354
|
+
# Read the raw header page.
|
|
355
|
+
header_page_data = stream.read(header_size)
|
|
356
|
+
if len(header_page_data) < header_size:
|
|
357
|
+
raise InvalidQuantumGraphFileError("Address file unexpectedly truncated.")
|
|
358
|
+
# Interpret the raw header data and initialize the reader instance.
|
|
359
|
+
header_page_stream = BytesIO(header_page_data)
|
|
360
|
+
file_int_size = int.from_bytes(header_page_stream.read(1))
|
|
361
|
+
if file_int_size != int_size:
|
|
362
|
+
raise InvalidQuantumGraphFileError(
|
|
363
|
+
f"int size in address file ({file_int_size}) does not match int size in header ({int_size})."
|
|
364
|
+
)
|
|
365
|
+
n_rows = int.from_bytes(header_page_stream.read(int_size))
|
|
366
|
+
file_n_addresses = int.from_bytes(header_page_stream.read(int_size))
|
|
367
|
+
if file_n_addresses != n_addresses:
|
|
368
|
+
raise InvalidQuantumGraphFileError(
|
|
369
|
+
f"Incorrect number of addresses per row: expected {n_addresses}, got {file_n_addresses}."
|
|
370
|
+
)
|
|
371
|
+
rows_per_page = max(page_size // row_size, 1)
|
|
372
|
+
# Construct an instance.
|
|
373
|
+
self = cls(stream, int_size, n_rows, n_addresses, rows_per_page=rows_per_page)
|
|
374
|
+
# Calculate positions of each page of rows.
|
|
375
|
+
row_index = 0
|
|
376
|
+
file_offset = header_size
|
|
377
|
+
while row_index < n_rows:
|
|
378
|
+
self.pages.append(AddressPage(file_offset=file_offset, begin=row_index, n_rows=rows_per_page))
|
|
379
|
+
row_index += rows_per_page
|
|
380
|
+
file_offset += rows_per_page * row_size
|
|
381
|
+
if row_index != n_rows:
|
|
382
|
+
# Last page was too big.
|
|
383
|
+
self.pages[-1].n_rows -= row_index - n_rows
|
|
384
|
+
assert sum(p.n_rows for p in self.pages) == n_rows, "Bad logic setting page row counts."
|
|
385
|
+
return self
|
|
326
386
|
|
|
327
387
|
@classmethod
|
|
328
388
|
@contextmanager
|
|
@@ -330,9 +390,10 @@ class AddressReader:
|
|
|
330
390
|
cls,
|
|
331
391
|
zf: zipfile.ZipFile,
|
|
332
392
|
name: str,
|
|
393
|
+
*,
|
|
333
394
|
page_size: int,
|
|
334
|
-
|
|
335
|
-
|
|
395
|
+
n_addresses: int,
|
|
396
|
+
int_size: int,
|
|
336
397
|
) -> Iterator[AddressReader]:
|
|
337
398
|
"""Make a reader for an address file in a zip archive.
|
|
338
399
|
|
|
@@ -345,11 +406,12 @@ class AddressReader:
|
|
|
345
406
|
page_size : `int`
|
|
346
407
|
Approximate number of bytes to read at a time when searching for an
|
|
347
408
|
address.
|
|
348
|
-
|
|
409
|
+
n_addresses : `int`
|
|
410
|
+
Number of addresses to expect per row. This is checked against
|
|
411
|
+
the size embedded in the file.
|
|
412
|
+
int_size : `int`
|
|
349
413
|
Number of bytes to use for all integers. This is checked against
|
|
350
414
|
the size embedded in the file.
|
|
351
|
-
start_index : `int`, optional
|
|
352
|
-
Value of the first index in the file.
|
|
353
415
|
|
|
354
416
|
Returns
|
|
355
417
|
-------
|
|
@@ -357,12 +419,7 @@ class AddressReader:
|
|
|
357
419
|
Context manager that returns a reader when entered.
|
|
358
420
|
"""
|
|
359
421
|
with zf.open(f"{name}.addr", mode="r") as stream:
|
|
360
|
-
|
|
361
|
-
if int_size is not None and result.int_size != int_size:
|
|
362
|
-
raise InvalidQuantumGraphFileError(
|
|
363
|
-
"int size in address file does not match int size in header."
|
|
364
|
-
)
|
|
365
|
-
yield result
|
|
422
|
+
yield cls.from_stream(stream, page_size=page_size, n_addresses=n_addresses, int_size=int_size)
|
|
366
423
|
|
|
367
424
|
@staticmethod
|
|
368
425
|
def compute_header_size(int_size: int) -> int:
|
|
@@ -409,11 +466,6 @@ class AddressReader:
|
|
|
409
466
|
)
|
|
410
467
|
)
|
|
411
468
|
|
|
412
|
-
@property
|
|
413
|
-
def header_size(self) -> int:
|
|
414
|
-
"""The size (in bytes) of the header of this address file."""
|
|
415
|
-
return self.compute_header_size(self.int_size)
|
|
416
|
-
|
|
417
469
|
@property
|
|
418
470
|
def row_size(self) -> int:
|
|
419
471
|
"""The size (in bytes) of each row of this address file."""
|
|
@@ -427,16 +479,26 @@ class AddressReader:
|
|
|
427
479
|
rows : `dict` [ `uuid.UUID`, `AddressRow` ]
|
|
428
480
|
Mapping of loaded address rows, keyed by UUID.
|
|
429
481
|
"""
|
|
482
|
+
# Skip any pages from the beginning that have already been read; this
|
|
483
|
+
# nicely handles both the case where we already read everything (or
|
|
484
|
+
# there was nothing to read) while giving us a page with a file offset
|
|
485
|
+
# to start from.
|
|
486
|
+
for page in self.pages:
|
|
487
|
+
if not page.read:
|
|
488
|
+
break
|
|
489
|
+
else:
|
|
490
|
+
return self.rows
|
|
491
|
+
# Read the entire rest of the file into memory.
|
|
492
|
+
self.stream.seek(page.file_offset)
|
|
493
|
+
data = self.stream.read()
|
|
494
|
+
buffer = BytesIO(data)
|
|
430
495
|
# Shortcut out if we've already read everything, but don't bother
|
|
431
496
|
# optimizing previous partial reads.
|
|
432
|
-
|
|
433
|
-
self.
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
for _ in range(self.n_rows):
|
|
438
|
-
self._read_row(buffer)
|
|
439
|
-
self.unread_pages.clear()
|
|
497
|
+
while len(self.rows) < self.n_rows:
|
|
498
|
+
self._read_row(buffer)
|
|
499
|
+
# Delete all pages; they don't matter anymore, and that's easier than
|
|
500
|
+
# updating them to reflect the reads we've done.
|
|
501
|
+
self.pages.clear()
|
|
440
502
|
return self.rows
|
|
441
503
|
|
|
442
504
|
def find(self, key: uuid.UUID) -> AddressRow:
|
|
@@ -452,54 +514,63 @@ class AddressReader:
|
|
|
452
514
|
row : `AddressRow`
|
|
453
515
|
Addresses for the given UUID.
|
|
454
516
|
"""
|
|
455
|
-
|
|
517
|
+
match key:
|
|
518
|
+
case uuid.UUID():
|
|
519
|
+
return self._find_uuid(key)
|
|
520
|
+
case _:
|
|
521
|
+
raise TypeError(f"Invalid argument: {key}.")
|
|
522
|
+
|
|
523
|
+
def _find_uuid(self, target: uuid.UUID) -> AddressRow:
|
|
524
|
+
if (row := self.rows.get(target)) is not None:
|
|
456
525
|
return row
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
self.
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
526
|
+
if self.n_rows == 0 or not self.pages:
|
|
527
|
+
raise LookupError(f"Address for {target} not found.")
|
|
528
|
+
|
|
529
|
+
# Use a binary search to find the page containing the target UUID.
|
|
530
|
+
left = 0
|
|
531
|
+
right = len(self.pages) - 1
|
|
532
|
+
while left <= right:
|
|
533
|
+
mid = left + ((right - left) // 2)
|
|
534
|
+
self._read_page(mid)
|
|
535
|
+
if (row := self.rows.get(target)) is not None:
|
|
536
|
+
return row
|
|
537
|
+
bounds = self.page_bounds[mid]
|
|
538
|
+
if target.int < bounds.uuid_int_begin:
|
|
539
|
+
right = mid - 1
|
|
540
|
+
elif target.int > bounds.uuid_int_end:
|
|
541
|
+
left = mid + 1
|
|
542
|
+
else:
|
|
543
|
+
# Should have been on this page, but it wasn't.
|
|
544
|
+
raise LookupError(f"Address for {target} not found.")
|
|
545
|
+
|
|
546
|
+
# Ran out of pages to search.
|
|
547
|
+
raise LookupError(f"Address for {target} not found.")
|
|
548
|
+
|
|
549
|
+
def _read_page(self, page_index: int, page_stream: BytesIO | None = None) -> bool:
|
|
550
|
+
page = self.pages[page_index]
|
|
551
|
+
if page.read:
|
|
552
|
+
return False
|
|
553
|
+
if page_stream is None:
|
|
554
|
+
self.stream.seek(page.file_offset)
|
|
555
|
+
page_stream = BytesIO(self.stream.read(page.n_rows * self.row_size))
|
|
556
|
+
row = self._read_row(page_stream)
|
|
557
|
+
uuid_int_begin = row.key.int
|
|
558
|
+
for _ in range(1, page.n_rows):
|
|
559
|
+
row = self._read_row(page_stream)
|
|
560
|
+
uuid_int_end = row.key.int + 1 # Python's loop scoping rules are actually useful here!
|
|
561
|
+
page.read = True
|
|
562
|
+
bounds = PageBounds(page_index=page_index, uuid_int_begin=uuid_int_begin, uuid_int_end=uuid_int_end)
|
|
563
|
+
self.page_bounds[page_index] = bounds
|
|
564
|
+
_LOG.debug("Read page %s with rows [%s:%s].", bounds, page.begin, page.end)
|
|
565
|
+
return True
|
|
490
566
|
|
|
491
567
|
def _read_row(self, page_stream: BytesIO) -> AddressRow:
|
|
492
568
|
row = AddressRow.read(page_stream, self.n_addresses, self.int_size)
|
|
493
569
|
self.rows[row.key] = row
|
|
570
|
+
self.rows_by_index[row.index] = row
|
|
494
571
|
_LOG.debug("Read address row %s.", row)
|
|
495
572
|
return row
|
|
496
573
|
|
|
497
|
-
def _page_search_path(self, mid: int) -> Iterator[int]:
|
|
498
|
-
yield mid
|
|
499
|
-
for abs_offset in itertools.count(1):
|
|
500
|
-
yield mid + abs_offset
|
|
501
|
-
yield mid - abs_offset
|
|
502
|
-
|
|
503
574
|
|
|
504
575
|
@dataclasses.dataclass
|
|
505
576
|
class MultiblockWriter:
|
|
@@ -72,6 +72,7 @@ from lsst.daf.butler import (
|
|
|
72
72
|
QuantumBackedButler,
|
|
73
73
|
SerializableDimensionData,
|
|
74
74
|
)
|
|
75
|
+
from lsst.daf.butler._rubin import generate_uuidv7
|
|
75
76
|
from lsst.daf.butler.datastore.record_data import DatastoreRecordData, SerializedDatastoreRecordData
|
|
76
77
|
from lsst.daf.butler.registry import ConflictingDefinitionError
|
|
77
78
|
from lsst.resources import ResourcePath, ResourcePathExpression
|
|
@@ -155,6 +156,21 @@ class PredictedThinQuantumModel(pydantic.BaseModel):
|
|
|
155
156
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
156
157
|
return super().model_json_schema(*args, **kwargs)
|
|
157
158
|
|
|
159
|
+
@classmethod
|
|
160
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
161
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
162
|
+
return super().model_validate(*args, **kwargs)
|
|
163
|
+
|
|
164
|
+
@classmethod
|
|
165
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
166
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
167
|
+
return super().model_validate_json(*args, **kwargs)
|
|
168
|
+
|
|
169
|
+
@classmethod
|
|
170
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
171
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
172
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
173
|
+
|
|
158
174
|
|
|
159
175
|
class PredictedThinGraphModel(pydantic.BaseModel):
|
|
160
176
|
"""Data model for the predicted quantum graph component that maps each
|
|
@@ -197,6 +213,21 @@ class PredictedThinGraphModel(pydantic.BaseModel):
|
|
|
197
213
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
198
214
|
return super().model_json_schema(*args, **kwargs)
|
|
199
215
|
|
|
216
|
+
@classmethod
|
|
217
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
218
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
219
|
+
return super().model_validate(*args, **kwargs)
|
|
220
|
+
|
|
221
|
+
@classmethod
|
|
222
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
223
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
224
|
+
return super().model_validate_json(*args, **kwargs)
|
|
225
|
+
|
|
226
|
+
@classmethod
|
|
227
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
228
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
229
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
230
|
+
|
|
200
231
|
|
|
201
232
|
class PredictedDatasetModel(pydantic.BaseModel):
|
|
202
233
|
"""Data model for the datasets in a predicted quantum graph file."""
|
|
@@ -270,6 +301,21 @@ class PredictedDatasetModel(pydantic.BaseModel):
|
|
|
270
301
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
271
302
|
return super().model_json_schema(*args, **kwargs)
|
|
272
303
|
|
|
304
|
+
@classmethod
|
|
305
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
306
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
307
|
+
return super().model_validate(*args, **kwargs)
|
|
308
|
+
|
|
309
|
+
@classmethod
|
|
310
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
311
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
312
|
+
return super().model_validate_json(*args, **kwargs)
|
|
313
|
+
|
|
314
|
+
@classmethod
|
|
315
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
316
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
317
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
318
|
+
|
|
273
319
|
|
|
274
320
|
class PredictedQuantumDatasetsModel(pydantic.BaseModel):
|
|
275
321
|
"""Data model for a description of a single predicted quantum that includes
|
|
@@ -390,7 +436,7 @@ class PredictedQuantumDatasetsModel(pydantic.BaseModel):
|
|
|
390
436
|
}
|
|
391
437
|
init_input_ids = {ref.id for ref in init_input_refs.values()}
|
|
392
438
|
result: PredictedQuantumDatasetsModel = cls.model_construct(
|
|
393
|
-
quantum_id=
|
|
439
|
+
quantum_id=generate_uuidv7(), task_label=task_init_node.label
|
|
394
440
|
)
|
|
395
441
|
for read_edge in task_init_node.iter_all_inputs():
|
|
396
442
|
ref = init_input_refs[read_edge.dataset_type_name]
|
|
@@ -440,6 +486,21 @@ class PredictedQuantumDatasetsModel(pydantic.BaseModel):
|
|
|
440
486
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
441
487
|
return super().model_json_schema(*args, **kwargs)
|
|
442
488
|
|
|
489
|
+
@classmethod
|
|
490
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
491
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
492
|
+
return super().model_validate(*args, **kwargs)
|
|
493
|
+
|
|
494
|
+
@classmethod
|
|
495
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
496
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
497
|
+
return super().model_validate_json(*args, **kwargs)
|
|
498
|
+
|
|
499
|
+
@classmethod
|
|
500
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
501
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
502
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
503
|
+
|
|
443
504
|
|
|
444
505
|
class PredictedInitQuantaModel(pydantic.RootModel):
|
|
445
506
|
"""Data model for the init-inputs and init-outputs of a predicted quantum
|
|
@@ -461,7 +522,7 @@ class PredictedInitQuantaModel(pydantic.RootModel):
|
|
|
461
522
|
Quantum graph.
|
|
462
523
|
"""
|
|
463
524
|
global_init_quantum = PredictedQuantumDatasetsModel.model_construct(
|
|
464
|
-
quantum_id=
|
|
525
|
+
quantum_id=generate_uuidv7(), task_label=""
|
|
465
526
|
)
|
|
466
527
|
for ref in old_quantum_graph.globalInitOutputRefs():
|
|
467
528
|
global_init_quantum.outputs[ref.datasetType.name] = [PredictedDatasetModel.from_dataset_ref(ref)]
|
|
@@ -501,6 +562,21 @@ class PredictedInitQuantaModel(pydantic.RootModel):
|
|
|
501
562
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
502
563
|
return super().model_json_schema(*args, **kwargs)
|
|
503
564
|
|
|
565
|
+
@classmethod
|
|
566
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
567
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
568
|
+
return super().model_validate(*args, **kwargs)
|
|
569
|
+
|
|
570
|
+
@classmethod
|
|
571
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
572
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
573
|
+
return super().model_validate_json(*args, **kwargs)
|
|
574
|
+
|
|
575
|
+
@classmethod
|
|
576
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
577
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
578
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
579
|
+
|
|
504
580
|
|
|
505
581
|
class PredictedQuantumInfo(QuantumInfo):
|
|
506
582
|
"""A typed dictionary that annotates the attributes of the NetworkX graph
|
|
@@ -1486,14 +1562,13 @@ class PredictedQuantumGraphComponents:
|
|
|
1486
1562
|
# Do all outputs and then all inputs in separate passes so we don't
|
|
1487
1563
|
# need to rely on topological ordering of anything.
|
|
1488
1564
|
for quantum_datasets in itertools.chain(self.init_quanta.root, self.quantum_datasets.values()):
|
|
1489
|
-
new_quantum_id =
|
|
1490
|
-
|
|
1491
|
-
quantum_datasets.quantum_id = uuid.uuid4()
|
|
1565
|
+
new_quantum_id = generate_uuidv7()
|
|
1566
|
+
quantum_datasets.quantum_id = new_quantum_id
|
|
1492
1567
|
for output_dataset in itertools.chain.from_iterable(quantum_datasets.outputs.values()):
|
|
1493
1568
|
assert output_dataset.run == self.header.output_run, (
|
|
1494
1569
|
f"Incorrect run {output_dataset.run} for output dataset {output_dataset.dataset_id}."
|
|
1495
1570
|
)
|
|
1496
|
-
new_dataset_id =
|
|
1571
|
+
new_dataset_id = generate_uuidv7()
|
|
1497
1572
|
uuid_map[output_dataset.dataset_id] = new_dataset_id
|
|
1498
1573
|
output_dataset.dataset_id = new_dataset_id
|
|
1499
1574
|
output_dataset.run = output_run
|
|
@@ -1754,6 +1829,7 @@ class PredictedQuantumGraphReader(BaseQuantumGraphReader):
|
|
|
1754
1829
|
address_filename="quanta",
|
|
1755
1830
|
page_size=page_size,
|
|
1756
1831
|
import_mode=import_mode,
|
|
1832
|
+
n_addresses=1,
|
|
1757
1833
|
) as self:
|
|
1758
1834
|
yield self
|
|
1759
1835
|
|
|
@@ -41,7 +41,6 @@ __all__ = (
|
|
|
41
41
|
|
|
42
42
|
import dataclasses
|
|
43
43
|
import operator
|
|
44
|
-
import uuid
|
|
45
44
|
from abc import ABC, abstractmethod
|
|
46
45
|
from collections import defaultdict
|
|
47
46
|
from collections.abc import Iterable, Mapping, Sequence
|
|
@@ -59,6 +58,7 @@ from lsst.daf.butler import (
|
|
|
59
58
|
NamedKeyMapping,
|
|
60
59
|
Quantum,
|
|
61
60
|
)
|
|
61
|
+
from lsst.daf.butler._rubin import generate_uuidv7
|
|
62
62
|
from lsst.daf.butler.datastore.record_data import DatastoreRecordData
|
|
63
63
|
from lsst.daf.butler.registry import MissingCollectionError, MissingDatasetTypeError
|
|
64
64
|
from lsst.utils.logging import LsstLogAdapter, getLogger
|
|
@@ -1255,7 +1255,7 @@ class QuantumGraphBuilder(ABC):
|
|
|
1255
1255
|
)
|
|
1256
1256
|
components.init_quanta.root = [
|
|
1257
1257
|
PredictedQuantumDatasetsModel.model_construct(
|
|
1258
|
-
quantum_id=
|
|
1258
|
+
quantum_id=generate_uuidv7(),
|
|
1259
1259
|
task_label="",
|
|
1260
1260
|
outputs={
|
|
1261
1261
|
dataset_key.parent_dataset_type_name: [
|
|
@@ -1272,7 +1272,7 @@ class QuantumGraphBuilder(ABC):
|
|
|
1272
1272
|
continue
|
|
1273
1273
|
task_init_key = TaskInitKey(task_node.label)
|
|
1274
1274
|
init_quantum_datasets = PredictedQuantumDatasetsModel.model_construct(
|
|
1275
|
-
quantum_id=
|
|
1275
|
+
quantum_id=generate_uuidv7(),
|
|
1276
1276
|
task_label=task_node.label,
|
|
1277
1277
|
inputs=self._make_predicted_datasets(
|
|
1278
1278
|
skeleton,
|
|
@@ -1294,7 +1294,7 @@ class QuantumGraphBuilder(ABC):
|
|
|
1294
1294
|
components.init_quanta.root.append(init_quantum_datasets)
|
|
1295
1295
|
for quantum_key in skeleton.get_quanta(task_node.label):
|
|
1296
1296
|
quantum_datasets = PredictedQuantumDatasetsModel.model_construct(
|
|
1297
|
-
quantum_id=
|
|
1297
|
+
quantum_id=generate_uuidv7(),
|
|
1298
1298
|
task_label=task_node.label,
|
|
1299
1299
|
data_coordinate=list(skeleton.get_data_id(quantum_key).full_values),
|
|
1300
1300
|
inputs=self._make_predicted_datasets(
|
|
@@ -121,6 +121,21 @@ class ExceptionInfo(pydantic.BaseModel):
|
|
|
121
121
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
122
122
|
return super().model_json_schema(*args, **kwargs)
|
|
123
123
|
|
|
124
|
+
@classmethod
|
|
125
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
126
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
127
|
+
return super().model_validate(*args, **kwargs)
|
|
128
|
+
|
|
129
|
+
@classmethod
|
|
130
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
131
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
132
|
+
return super().model_validate_json(*args, **kwargs)
|
|
133
|
+
|
|
134
|
+
@classmethod
|
|
135
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
136
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
137
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
138
|
+
|
|
124
139
|
|
|
125
140
|
class QuantumReport(pydantic.BaseModel):
|
|
126
141
|
"""Task execution report for a single Quantum.
|
|
@@ -276,6 +291,21 @@ class QuantumReport(pydantic.BaseModel):
|
|
|
276
291
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
277
292
|
return super().model_json_schema(*args, **kwargs)
|
|
278
293
|
|
|
294
|
+
@classmethod
|
|
295
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
296
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
297
|
+
return super().model_validate(*args, **kwargs)
|
|
298
|
+
|
|
299
|
+
@classmethod
|
|
300
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
301
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
302
|
+
return super().model_validate_json(*args, **kwargs)
|
|
303
|
+
|
|
304
|
+
@classmethod
|
|
305
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
306
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
307
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
308
|
+
|
|
279
309
|
|
|
280
310
|
class Report(pydantic.BaseModel):
|
|
281
311
|
"""Execution report for the whole job with one or few quanta."""
|
|
@@ -348,3 +378,18 @@ class Report(pydantic.BaseModel):
|
|
|
348
378
|
def model_json_schema(cls, *args: Any, **kwargs: Any) -> Any:
|
|
349
379
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
350
380
|
return super().model_json_schema(*args, **kwargs)
|
|
381
|
+
|
|
382
|
+
@classmethod
|
|
383
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
384
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
385
|
+
return super().model_validate(*args, **kwargs)
|
|
386
|
+
|
|
387
|
+
@classmethod
|
|
388
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
389
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
390
|
+
return super().model_validate_json(*args, **kwargs)
|
|
391
|
+
|
|
392
|
+
@classmethod
|
|
393
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
394
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
395
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
@@ -667,7 +667,10 @@ class SimplePipelineExecutor:
|
|
|
667
667
|
Butler to transfer records to.
|
|
668
668
|
"""
|
|
669
669
|
assert self.predicted.dimension_data is not None, "Dimension data must be present for execution."
|
|
670
|
-
|
|
670
|
+
records = self.predicted.dimension_data.records
|
|
671
|
+
dimensions = out_butler.dimensions.sorted(records.keys())
|
|
672
|
+
for dimension in dimensions:
|
|
673
|
+
record_set = records[dimension.name]
|
|
671
674
|
if record_set and record_set.element.has_own_table:
|
|
672
675
|
out_butler.registry.insertDimensionData(
|
|
673
676
|
record_set.element,
|
|
@@ -242,6 +242,21 @@ class MockDataset(pydantic.BaseModel):
|
|
|
242
242
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
243
243
|
return super().model_json_schema(*args, **kwargs)
|
|
244
244
|
|
|
245
|
+
@classmethod
|
|
246
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
247
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
248
|
+
return super().model_validate(*args, **kwargs)
|
|
249
|
+
|
|
250
|
+
@classmethod
|
|
251
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
252
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
253
|
+
return super().model_validate_json(*args, **kwargs)
|
|
254
|
+
|
|
255
|
+
@classmethod
|
|
256
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
257
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
258
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
259
|
+
|
|
245
260
|
|
|
246
261
|
class ConvertedUnmockedDataset(pydantic.BaseModel):
|
|
247
262
|
"""A marker class that represents a conversion from a regular in-memory
|
|
@@ -281,6 +296,21 @@ class ConvertedUnmockedDataset(pydantic.BaseModel):
|
|
|
281
296
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
282
297
|
return super().model_json_schema(*args, **kwargs)
|
|
283
298
|
|
|
299
|
+
@classmethod
|
|
300
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
301
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
302
|
+
return super().model_validate(*args, **kwargs)
|
|
303
|
+
|
|
304
|
+
@classmethod
|
|
305
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
306
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
307
|
+
return super().model_validate_json(*args, **kwargs)
|
|
308
|
+
|
|
309
|
+
@classmethod
|
|
310
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
311
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
312
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
313
|
+
|
|
284
314
|
|
|
285
315
|
class MockDatasetQuantum(pydantic.BaseModel):
|
|
286
316
|
"""Description of the quantum that produced a mock dataset.
|
|
@@ -331,6 +361,21 @@ class MockDatasetQuantum(pydantic.BaseModel):
|
|
|
331
361
|
"""See `pydantic.BaseModel.model_json_schema`."""
|
|
332
362
|
return super().model_json_schema(*args, **kwargs)
|
|
333
363
|
|
|
364
|
+
@classmethod
|
|
365
|
+
def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
|
|
366
|
+
"""See `pydantic.BaseModel.model_validate`."""
|
|
367
|
+
return super().model_validate(*args, **kwargs)
|
|
368
|
+
|
|
369
|
+
@classmethod
|
|
370
|
+
def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
|
|
371
|
+
"""See `pydantic.BaseModel.model_validate_json`."""
|
|
372
|
+
return super().model_validate_json(*args, **kwargs)
|
|
373
|
+
|
|
374
|
+
@classmethod
|
|
375
|
+
def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
|
|
376
|
+
"""See `pydantic.BaseModel.model_validate_strings`."""
|
|
377
|
+
return super().model_validate_strings(*args, **kwargs)
|
|
378
|
+
|
|
334
379
|
|
|
335
380
|
MockDataset.model_rebuild()
|
|
336
381
|
|
lsst/pipe/base/version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
__all__ = ["__version__"]
|
|
2
|
-
__version__ = "29.2025.
|
|
2
|
+
__version__ = "29.2025.4200"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-pipe-base
|
|
3
|
-
Version: 29.2025.
|
|
3
|
+
Version: 29.2025.4200
|
|
4
4
|
Summary: Pipeline infrastructure for the Rubin Science Pipelines.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License: BSD 3-Clause License
|
|
@@ -7,7 +7,7 @@ lsst/pipe/base/_instrument.py,sha256=I9UTaj81krR1zkTZ1owfOPBzHN29PY3Egg7fIE5obxQ
|
|
|
7
7
|
lsst/pipe/base/_observation_dimension_packer.py,sha256=78Jg2OVFOdXIK62TS2Y3X4095xqCzmiIx9o4TXyADYA,8027
|
|
8
8
|
lsst/pipe/base/_quantumContext.py,sha256=gb60mTHbgOIEptYvJ64SaChvViXyeKJlG6kEHq4nYVw,19345
|
|
9
9
|
lsst/pipe/base/_status.py,sha256=tvKm-z_haZGksOR4nQ-ePJgbLag-e3t4nQY47yLFP2M,15741
|
|
10
|
-
lsst/pipe/base/_task_metadata.py,sha256=
|
|
10
|
+
lsst/pipe/base/_task_metadata.py,sha256=Y4rjrYWvYxYCJwy86VvzxKMkNxEJbYVgLVuXo6KiXac,25638
|
|
11
11
|
lsst/pipe/base/all_dimensions_quantum_graph_builder.py,sha256=nazY74jrdSCr6CFfPp78JecM_-udW95EYP7grLPO2hg,70830
|
|
12
12
|
lsst/pipe/base/automatic_connection_constants.py,sha256=H5uuh1rYRpjndgPdb0dh1L_-OyLKdT6VWOZTAb__xCU,3298
|
|
13
13
|
lsst/pipe/base/caching_limited_butler.py,sha256=x_e4EXYODcVJV8BkLzvUTu2yCd6dqM1rJ-xwLISadeg,7798
|
|
@@ -28,20 +28,20 @@ lsst/pipe/base/pipelineIR.py,sha256=DDOAYHnMP-iw021RDMYsZnvb21tWumLjYqO5d38q_Zk,
|
|
|
28
28
|
lsst/pipe/base/pipelineTask.py,sha256=K3GdjJLvy8A7I-jzQiERQZaYF7mC1LM3iB5TmUtbOCI,8394
|
|
29
29
|
lsst/pipe/base/prerequisite_helpers.py,sha256=bmiebQ4veSrypZgAXjmCBFfj8fUtPW9eRQaVShhxdBQ,28446
|
|
30
30
|
lsst/pipe/base/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
31
|
-
lsst/pipe/base/quantum_graph_builder.py,sha256=
|
|
31
|
+
lsst/pipe/base/quantum_graph_builder.py,sha256=SwfZny05lJLZQHG69550GoafKWWZhX5Glgu345-VAsE,67981
|
|
32
32
|
lsst/pipe/base/quantum_graph_executor.py,sha256=WP41iQmihy1jfgaHV6eu2aSrqQx_Fydq3mbEF6CLQ-s,4419
|
|
33
33
|
lsst/pipe/base/quantum_graph_skeleton.py,sha256=GhSQjRHaErneGY4A4E0tERqg9QPEeYrlpmdLzqFXy6E,28586
|
|
34
34
|
lsst/pipe/base/quantum_provenance_graph.py,sha256=SChke6lcSzQQQMbVN_mCZ-RxDj7chehjJrSMvXRuhAI,93372
|
|
35
|
-
lsst/pipe/base/quantum_reports.py,sha256=
|
|
35
|
+
lsst/pipe/base/quantum_reports.py,sha256=ut235L88v7SXaeVUvMA9qFl7tpeMwGnzob3X0QoOI_s,14210
|
|
36
36
|
lsst/pipe/base/separable_pipeline_executor.py,sha256=vXqJrRI5GNezzGV9QsiaRHEhioDF2Y_W7JQYQCzHR7A,16720
|
|
37
|
-
lsst/pipe/base/simple_pipeline_executor.py,sha256=
|
|
37
|
+
lsst/pipe/base/simple_pipeline_executor.py,sha256=QjgX5seueopxPpnJz7H2hBNx-8ucxeT-3BXj6ET_vww,29543
|
|
38
38
|
lsst/pipe/base/single_quantum_executor.py,sha256=YWzLJDTRlvMwmtjsIW7XXZEUlLlv7ize-Oxb_0eKTrI,26667
|
|
39
39
|
lsst/pipe/base/struct.py,sha256=Fa-UkpuXOxdzKWbHrMUkJYOszZuBXCm2NesXNR0IOPQ,5048
|
|
40
40
|
lsst/pipe/base/task.py,sha256=XHBd-7m1a4-6LgobBYA1DgY4H7EV-_RWKfxbhZbMmD4,15145
|
|
41
41
|
lsst/pipe/base/taskFactory.py,sha256=MsDGECJqZLSZk8SGhpuVhNaP32UWuNvxZiDcZExPFG8,3412
|
|
42
42
|
lsst/pipe/base/testUtils.py,sha256=lSBKMhoKflbi8JkMNYfEqqHNl-rtFI8UYT3QneDYpLo,18477
|
|
43
43
|
lsst/pipe/base/utils.py,sha256=JmEt3l0xrh9uayKrSXuQEq12aXOhDr2YXmbYduaxCko,1940
|
|
44
|
-
lsst/pipe/base/version.py,sha256=
|
|
44
|
+
lsst/pipe/base/version.py,sha256=loN_SewEappJtRr1bfMkUAloXYiZtM4w0O03FF5yeZQ,55
|
|
45
45
|
lsst/pipe/base/cli/__init__.py,sha256=861tXIAW7SqtqNUYkjbeEdfg8lDswXsjJQca0gVCFz4,54
|
|
46
46
|
lsst/pipe/base/cli/_get_cli_subcommands.py,sha256=g_af64klRybBGKAg7fmBSZBdw2LYBAsFON_yQIMZON0,1289
|
|
47
47
|
lsst/pipe/base/cli/cmd/__init__.py,sha256=BGicstnryQ48rYcNRh4fa6Vy63ZIlZ_pPAEa17jhkwY,1519
|
|
@@ -54,9 +54,9 @@ lsst/pipe/base/formatters/pexConfig.py,sha256=MA-08FIDV7PmpcV-VtDaBegR6YO6_pEhMB
|
|
|
54
54
|
lsst/pipe/base/graph/__init__.py,sha256=Zs2vwSFNiu1bYDsgrWQZ0qegG5F6PIjiQ5ZGT3EqcfA,118
|
|
55
55
|
lsst/pipe/base/graph/_implDetails.py,sha256=QQHVnCW78UnIbALXX_v7EW7g6MTUTuuR1Q_Ss_squUw,6784
|
|
56
56
|
lsst/pipe/base/graph/_loadHelpers.py,sha256=qUfjIgFezaXZRCFV7PFzmz1SSKFjRWOMWJePuyKiD24,12064
|
|
57
|
-
lsst/pipe/base/graph/_versionDeserializers.py,sha256=
|
|
58
|
-
lsst/pipe/base/graph/graph.py,sha256=
|
|
59
|
-
lsst/pipe/base/graph/graphSummary.py,sha256=
|
|
57
|
+
lsst/pipe/base/graph/_versionDeserializers.py,sha256=Xwq-MHfxaml2bL5cxSF8qmb6rjEHHZBuSengX8iggRg,28011
|
|
58
|
+
lsst/pipe/base/graph/graph.py,sha256=AZzq8HXUbWLcRKIGnKpwcCA8q0Fz-BT2pn0tFU1zulo,75148
|
|
59
|
+
lsst/pipe/base/graph/graphSummary.py,sha256=F0ET5H4cBFYNPXvHuUBa3nTCj99rpf0JwxPG5Kfi7iw,6352
|
|
60
60
|
lsst/pipe/base/graph/quantumNode.py,sha256=l4mslxBgyUzBAqwjpx6XRP-UPxe-oRMxHJWt-_y3Dm0,7196
|
|
61
61
|
lsst/pipe/base/pipeline_graph/__init__.py,sha256=yTEuvlzbeKIHIm7GeRmGSsma1wpZFNv8j12WfSH-deY,1516
|
|
62
62
|
lsst/pipe/base/pipeline_graph/__main__.py,sha256=E6ugEwJbds22wjgcfcgzeyO04JofQwVhn_Y8kZYY1lQ,20769
|
|
@@ -69,7 +69,7 @@ lsst/pipe/base/pipeline_graph/_pipeline_graph.py,sha256=G-P3r-AHBM1cMP7ex75M-Xtu
|
|
|
69
69
|
lsst/pipe/base/pipeline_graph/_task_subsets.py,sha256=lLvcndSGcZigteWd4eeAM8LxQ1lHPBoysY8PjJTxx1c,13244
|
|
70
70
|
lsst/pipe/base/pipeline_graph/_tasks.py,sha256=jTLpm5dZMXRNrGi3L45-3DtF95PGwhmejWLZ-zcSTzo,42802
|
|
71
71
|
lsst/pipe/base/pipeline_graph/expressions.py,sha256=MZ0qxGA4ctu_WqVjdjjezZF8Jd5174PWbio7EF2wdl0,7717
|
|
72
|
-
lsst/pipe/base/pipeline_graph/io.py,sha256=
|
|
72
|
+
lsst/pipe/base/pipeline_graph/io.py,sha256=zVIybq5JyR1u1FwqF60wG0bIhz_SkgzzQiw2A7a1oNk,30943
|
|
73
73
|
lsst/pipe/base/pipeline_graph/visualization/__init__.py,sha256=qQctfWuFpcmgRdgu8Y6OsJ_pXpLKrCK-alqfVtIecls,1551
|
|
74
74
|
lsst/pipe/base/pipeline_graph/visualization/_dot.py,sha256=hgy5Wk4GXptb9GbjPn8-0D9EjWsXKBEEVs1ocHLh_MA,13535
|
|
75
75
|
lsst/pipe/base/pipeline_graph/visualization/_formatting.py,sha256=NsBxXwdmISitr8_4wPc-T8CqVB-Mq4pv7DmUefFm3JU,17845
|
|
@@ -81,9 +81,9 @@ lsst/pipe/base/pipeline_graph/visualization/_printer.py,sha256=yJMRJ-aXd3nYDgs1F
|
|
|
81
81
|
lsst/pipe/base/pipeline_graph/visualization/_show.py,sha256=lPRjO1To2n5r3f_Wgcwy-7TmyJ7UszGGFXAlOtN1wDs,10510
|
|
82
82
|
lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py,sha256=dp7PXl9Cu7GfWjBi5g8KjXZgnF1KGg_idKKxtICL53Q,8679
|
|
83
83
|
lsst/pipe/base/quantum_graph/__init__.py,sha256=rPvhjKcnPisHsJ1-PbyASSmyaxzw55nGYi4ZKI5LO84,1410
|
|
84
|
-
lsst/pipe/base/quantum_graph/_common.py,sha256=
|
|
85
|
-
lsst/pipe/base/quantum_graph/_multiblock.py,sha256=
|
|
86
|
-
lsst/pipe/base/quantum_graph/_predicted.py,sha256=
|
|
84
|
+
lsst/pipe/base/quantum_graph/_common.py,sha256=PKYd7YsFCB7dW-WBlUVRmQEpt4rNKEfJmKL0SgFwd-M,22138
|
|
85
|
+
lsst/pipe/base/quantum_graph/_multiblock.py,sha256=Y99r4yDAuxmf_YfXbHfBXVmMnKxmIBvOp9NPztlZqE8,27488
|
|
86
|
+
lsst/pipe/base/quantum_graph/_predicted.py,sha256=_yp2mth3j0BC6lB4s_wk-atQ-scenoGxhiHD5qSb5e0,86013
|
|
87
87
|
lsst/pipe/base/quantum_graph/visualization.py,sha256=EbTWhk9aPq7sX6bcHmnEIsr2xuuR6d1SxspQbRe8D0Q,12235
|
|
88
88
|
lsst/pipe/base/script/__init__.py,sha256=cLEXE7aq5UZ0juL_ScmRw0weFgp4tDgwEX_ts-NEYic,1522
|
|
89
89
|
lsst/pipe/base/script/register_instrument.py,sha256=TRC2r2tSoYBNWNVQya01ELxAtGH8WVk9Ya-uNgCIL5U,2426
|
|
@@ -101,14 +101,14 @@ lsst/pipe/base/tests/mocks/__init__.py,sha256=fDy9H9vRAIBpKDJEXNZuDWJMzWZfpcBT4T
|
|
|
101
101
|
lsst/pipe/base/tests/mocks/_data_id_match.py,sha256=v33QZhZm-srXZAXZ8NbNKGN-_ql4AzaArBUk1lxhyss,7474
|
|
102
102
|
lsst/pipe/base/tests/mocks/_pipeline_task.py,sha256=_n16lDsk3ItWi2J28Qheuqphr4aaCK6CN9acmJ1hAqI,30692
|
|
103
103
|
lsst/pipe/base/tests/mocks/_repo.py,sha256=OTJw_fi37w7bkZbbLa7z51W-45zxySAnLbV7Qv_aSB4,27423
|
|
104
|
-
lsst/pipe/base/tests/mocks/_storage_class.py,sha256=
|
|
105
|
-
lsst_pipe_base-29.2025.
|
|
106
|
-
lsst_pipe_base-29.2025.
|
|
107
|
-
lsst_pipe_base-29.2025.
|
|
108
|
-
lsst_pipe_base-29.2025.
|
|
109
|
-
lsst_pipe_base-29.2025.
|
|
110
|
-
lsst_pipe_base-29.2025.
|
|
111
|
-
lsst_pipe_base-29.2025.
|
|
112
|
-
lsst_pipe_base-29.2025.
|
|
113
|
-
lsst_pipe_base-29.2025.
|
|
114
|
-
lsst_pipe_base-29.2025.
|
|
104
|
+
lsst/pipe/base/tests/mocks/_storage_class.py,sha256=12IFfJMbZ5GkYlMX6ZMWiG8pMZc2Jlxke3qQW-bljdU,27434
|
|
105
|
+
lsst_pipe_base-29.2025.4200.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
|
|
106
|
+
lsst_pipe_base-29.2025.4200.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
|
|
107
|
+
lsst_pipe_base-29.2025.4200.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
|
|
108
|
+
lsst_pipe_base-29.2025.4200.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
109
|
+
lsst_pipe_base-29.2025.4200.dist-info/METADATA,sha256=EmpWwNeiAz6QRhEheCht9pC658N6j-wkWnD9Nl2NeMQ,2234
|
|
110
|
+
lsst_pipe_base-29.2025.4200.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
111
|
+
lsst_pipe_base-29.2025.4200.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
|
|
112
|
+
lsst_pipe_base-29.2025.4200.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
|
|
113
|
+
lsst_pipe_base-29.2025.4200.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
114
|
+
lsst_pipe_base-29.2025.4200.dist-info/RECORD,,
|
|
File without changes
|
{lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/licenses/COPYRIGHT
RENAMED
|
File without changes
|
{lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{lsst_pipe_base-29.2025.4000.dist-info → lsst_pipe_base-29.2025.4200.dist-info}/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|