lsst-pipe-base 29.2025.4600__py3-none-any.whl → 29.2025.4700__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/pipe/base/quantum_graph/_common.py +15 -1
- lsst/pipe/base/quantum_graph/_multiblock.py +14 -39
- lsst/pipe/base/quantum_graph/_predicted.py +77 -73
- lsst/pipe/base/quantum_graph/_provenance.py +73 -144
- lsst/pipe/base/quantum_graph/aggregator/_communicators.py +10 -10
- lsst/pipe/base/quantum_graph/aggregator/_scanner.py +88 -60
- lsst/pipe/base/quantum_graph/aggregator/_structs.py +36 -19
- lsst/pipe/base/quantum_graph/aggregator/_supervisor.py +7 -10
- lsst/pipe/base/quantum_graph/aggregator/_writer.py +55 -144
- lsst/pipe/base/quantum_graph_builder.py +0 -1
- lsst/pipe/base/version.py +1 -1
- {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/METADATA +1 -1
- {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/RECORD +21 -21
- {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/WHEEL +0 -0
- {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/entry_points.txt +0 -0
- {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/licenses/LICENSE +0 -0
- {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/licenses/bsd_license.txt +0 -0
- {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/licenses/gpl-v3.0.txt +0 -0
- {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/top_level.txt +0 -0
- {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/zip-safe +0 -0
|
@@ -47,7 +47,7 @@ import uuid
|
|
|
47
47
|
from collections import Counter
|
|
48
48
|
from collections.abc import Iterable, Iterator, Mapping
|
|
49
49
|
from contextlib import contextmanager
|
|
50
|
-
from typing import TYPE_CHECKING, Any,
|
|
50
|
+
from typing import TYPE_CHECKING, Any, TypedDict, TypeVar
|
|
51
51
|
|
|
52
52
|
import astropy.table
|
|
53
53
|
import networkx
|
|
@@ -68,15 +68,13 @@ from ._common import (
|
|
|
68
68
|
BaseQuantumGraphReader,
|
|
69
69
|
ConnectionName,
|
|
70
70
|
DataCoordinateValues,
|
|
71
|
-
DatasetIndex,
|
|
72
71
|
DatasetInfo,
|
|
73
72
|
DatasetTypeName,
|
|
74
73
|
HeaderModel,
|
|
75
|
-
QuantumIndex,
|
|
76
74
|
QuantumInfo,
|
|
77
75
|
TaskLabel,
|
|
78
76
|
)
|
|
79
|
-
from ._multiblock import
|
|
77
|
+
from ._multiblock import MultiblockReader
|
|
80
78
|
from ._predicted import PredictedDatasetModel, PredictedQuantumDatasetsModel
|
|
81
79
|
|
|
82
80
|
DATASET_ADDRESS_INDEX = 0
|
|
@@ -202,16 +200,14 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
|
|
|
202
200
|
existence of the dataset.
|
|
203
201
|
"""
|
|
204
202
|
|
|
205
|
-
producer:
|
|
206
|
-
"""
|
|
203
|
+
producer: uuid.UUID | None = None
|
|
204
|
+
"""ID of the quantum that produced this dataset.
|
|
207
205
|
|
|
208
206
|
This is `None` for overall inputs to the graph.
|
|
209
207
|
"""
|
|
210
208
|
|
|
211
|
-
consumers: list[
|
|
212
|
-
"""
|
|
213
|
-
dataset.
|
|
214
|
-
"""
|
|
209
|
+
consumers: list[uuid.UUID] = pydantic.Field(default_factory=list)
|
|
210
|
+
"""IDs of quanta that were predicted to consume this dataset."""
|
|
215
211
|
|
|
216
212
|
@property
|
|
217
213
|
def node_id(self) -> uuid.UUID:
|
|
@@ -222,8 +218,8 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
|
|
|
222
218
|
def from_predicted(
|
|
223
219
|
cls,
|
|
224
220
|
predicted: PredictedDatasetModel,
|
|
225
|
-
producer:
|
|
226
|
-
consumers: Iterable[
|
|
221
|
+
producer: uuid.UUID | None = None,
|
|
222
|
+
consumers: Iterable[uuid.UUID] = (),
|
|
227
223
|
) -> ProvenanceDatasetModel:
|
|
228
224
|
"""Construct from a predicted dataset model.
|
|
229
225
|
|
|
@@ -231,12 +227,10 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
|
|
|
231
227
|
----------
|
|
232
228
|
predicted : `PredictedDatasetModel`
|
|
233
229
|
Information about the dataset from the predicted graph.
|
|
234
|
-
producer : `
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
Internal IDs of the quanta that were predicted to consume this
|
|
239
|
-
dataset.
|
|
230
|
+
producer : `uuid.UUID` or `None`, optional
|
|
231
|
+
ID of the quantum that was predicted to produce this dataset.
|
|
232
|
+
consumers : `~collections.abc.Iterable` [`uuid.UUID`], optional
|
|
233
|
+
IDs of the quanta that were predicted to consume this dataset.
|
|
240
234
|
|
|
241
235
|
Returns
|
|
242
236
|
-------
|
|
@@ -258,16 +252,13 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
|
|
|
258
252
|
consumers=list(consumers),
|
|
259
253
|
)
|
|
260
254
|
|
|
261
|
-
def _add_to_graph(self, graph: ProvenanceQuantumGraph
|
|
255
|
+
def _add_to_graph(self, graph: ProvenanceQuantumGraph) -> None:
|
|
262
256
|
"""Add this dataset and its edges to quanta to a provenance graph.
|
|
263
257
|
|
|
264
258
|
Parameters
|
|
265
259
|
----------
|
|
266
260
|
graph : `ProvenanceQuantumGraph`
|
|
267
261
|
Graph to update in place.
|
|
268
|
-
address_reader : `AddressReader`
|
|
269
|
-
Reader object that can be used to look up UUIDs from integer
|
|
270
|
-
indexes.
|
|
271
262
|
|
|
272
263
|
Notes
|
|
273
264
|
-----
|
|
@@ -290,15 +281,12 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
|
|
|
290
281
|
run=self.run,
|
|
291
282
|
produced=self.produced,
|
|
292
283
|
)
|
|
293
|
-
producer_id: uuid.UUID | None = None
|
|
294
284
|
if self.producer is not None:
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
for consumer_index in self.consumers:
|
|
298
|
-
consumer_id = address_reader.find(consumer_index).key
|
|
285
|
+
graph._bipartite_xgraph.add_edge(self.producer, self.dataset_id)
|
|
286
|
+
for consumer_id in self.consumers:
|
|
299
287
|
graph._bipartite_xgraph.add_edge(self.dataset_id, consumer_id)
|
|
300
|
-
if
|
|
301
|
-
graph._quantum_only_xgraph.add_edge(
|
|
288
|
+
if self.producer is not None:
|
|
289
|
+
graph._quantum_only_xgraph.add_edge(self.producer, consumer_id)
|
|
302
290
|
graph._datasets_by_type[self.dataset_type_name][data_id] = self.dataset_id
|
|
303
291
|
|
|
304
292
|
# Work around the fact that Sphinx chokes on Pydantic docstring formatting,
|
|
@@ -347,7 +335,7 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
|
|
|
347
335
|
return super().model_validate_strings(*args, **kwargs)
|
|
348
336
|
|
|
349
337
|
|
|
350
|
-
class
|
|
338
|
+
class ProvenanceQuantumAttemptModel(pydantic.BaseModel):
|
|
351
339
|
"""Data model for a now-superseded attempt to run a quantum in a
|
|
352
340
|
provenance quantum graph file.
|
|
353
341
|
"""
|
|
@@ -367,35 +355,11 @@ class _GenericProvenanceQuantumAttemptModel(pydantic.BaseModel, Generic[_I]):
|
|
|
367
355
|
resource_usage: QuantumResourceUsage | None = None
|
|
368
356
|
"""Resource usage information (timing, memory use) for this quantum."""
|
|
369
357
|
|
|
370
|
-
previous_process_quanta: list[
|
|
358
|
+
previous_process_quanta: list[uuid.UUID] = pydantic.Field(default_factory=list)
|
|
371
359
|
"""The IDs of other quanta previously executed in the same process as this
|
|
372
360
|
one.
|
|
373
361
|
"""
|
|
374
362
|
|
|
375
|
-
def remap_uuids(
|
|
376
|
-
self: ProvenanceQuantumAttemptModel, indices: Mapping[uuid.UUID, QuantumIndex]
|
|
377
|
-
) -> StorageProvenanceQuantumAttemptModel:
|
|
378
|
-
return StorageProvenanceQuantumAttemptModel(
|
|
379
|
-
attempt=self.attempt,
|
|
380
|
-
status=self.status,
|
|
381
|
-
caveats=self.caveats,
|
|
382
|
-
exception=self.exception,
|
|
383
|
-
resource_usage=self.resource_usage,
|
|
384
|
-
previous_process_quanta=[indices[q] for q in self.previous_process_quanta],
|
|
385
|
-
)
|
|
386
|
-
|
|
387
|
-
def remap_indices(
|
|
388
|
-
self: StorageProvenanceQuantumAttemptModel, address_reader: AddressReader
|
|
389
|
-
) -> ProvenanceQuantumAttemptModel:
|
|
390
|
-
return ProvenanceQuantumAttemptModel(
|
|
391
|
-
attempt=self.attempt,
|
|
392
|
-
status=self.status,
|
|
393
|
-
caveats=self.caveats,
|
|
394
|
-
exception=self.exception,
|
|
395
|
-
resource_usage=self.resource_usage,
|
|
396
|
-
previous_process_quanta=[address_reader.find(q).key for q in self.previous_process_quanta],
|
|
397
|
-
)
|
|
398
|
-
|
|
399
363
|
# Work around the fact that Sphinx chokes on Pydantic docstring formatting,
|
|
400
364
|
# when we inherit those docstrings in our public classes.
|
|
401
365
|
if "sphinx" in sys.modules and not TYPE_CHECKING:
|
|
@@ -442,10 +406,6 @@ class _GenericProvenanceQuantumAttemptModel(pydantic.BaseModel, Generic[_I]):
|
|
|
442
406
|
return super().model_validate_strings(*args, **kwargs)
|
|
443
407
|
|
|
444
408
|
|
|
445
|
-
StorageProvenanceQuantumAttemptModel: TypeAlias = _GenericProvenanceQuantumAttemptModel[QuantumIndex]
|
|
446
|
-
ProvenanceQuantumAttemptModel: TypeAlias = _GenericProvenanceQuantumAttemptModel[uuid.UUID]
|
|
447
|
-
|
|
448
|
-
|
|
449
409
|
class ProvenanceLogRecordsModel(pydantic.BaseModel):
|
|
450
410
|
"""Data model for storing execution logs in a provenance quantum graph
|
|
451
411
|
file.
|
|
@@ -570,17 +530,17 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
|
|
|
570
530
|
data_coordinate: DataCoordinateValues = pydantic.Field(default_factory=list)
|
|
571
531
|
"""The full values (required and implied) of this dataset's data ID."""
|
|
572
532
|
|
|
573
|
-
inputs: dict[ConnectionName, list[
|
|
574
|
-
"""
|
|
575
|
-
|
|
533
|
+
inputs: dict[ConnectionName, list[uuid.UUID]] = pydantic.Field(default_factory=dict)
|
|
534
|
+
"""IDs of the datasets predicted to be consumed by this quantum, grouped by
|
|
535
|
+
connection name.
|
|
576
536
|
"""
|
|
577
537
|
|
|
578
|
-
outputs: dict[ConnectionName, list[
|
|
579
|
-
"""
|
|
580
|
-
|
|
538
|
+
outputs: dict[ConnectionName, list[uuid.UUID]] = pydantic.Field(default_factory=dict)
|
|
539
|
+
"""IDs of the datasets predicted to be produced by this quantum, grouped by
|
|
540
|
+
connection name.
|
|
581
541
|
"""
|
|
582
542
|
|
|
583
|
-
attempts: list[
|
|
543
|
+
attempts: list[ProvenanceQuantumAttemptModel] = pydantic.Field(default_factory=list)
|
|
584
544
|
"""Provenance for all attempts to execute this quantum, ordered
|
|
585
545
|
chronologically from first to last.
|
|
586
546
|
|
|
@@ -595,17 +555,13 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
|
|
|
595
555
|
return self.quantum_id
|
|
596
556
|
|
|
597
557
|
@classmethod
|
|
598
|
-
def from_predicted(
|
|
599
|
-
cls, predicted: PredictedQuantumDatasetsModel, indices: Mapping[uuid.UUID, int]
|
|
600
|
-
) -> ProvenanceQuantumModel:
|
|
558
|
+
def from_predicted(cls, predicted: PredictedQuantumDatasetsModel) -> ProvenanceQuantumModel:
|
|
601
559
|
"""Construct from a predicted quantum model.
|
|
602
560
|
|
|
603
561
|
Parameters
|
|
604
562
|
----------
|
|
605
563
|
predicted : `PredictedQuantumDatasetsModel`
|
|
606
564
|
Information about the quantum from the predicted graph.
|
|
607
|
-
indices : `~collections.abc.Mapping [`uuid.UUID`, `int`]
|
|
608
|
-
Mapping from quantum or dataset UUID to internal integer ID.
|
|
609
565
|
|
|
610
566
|
Returns
|
|
611
567
|
-------
|
|
@@ -613,11 +569,11 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
|
|
|
613
569
|
Provenance quantum model.
|
|
614
570
|
"""
|
|
615
571
|
inputs = {
|
|
616
|
-
connection_name: [
|
|
572
|
+
connection_name: [d.dataset_id for d in predicted_inputs]
|
|
617
573
|
for connection_name, predicted_inputs in predicted.inputs.items()
|
|
618
574
|
}
|
|
619
575
|
outputs = {
|
|
620
|
-
connection_name: [
|
|
576
|
+
connection_name: [d.dataset_id for d in predicted_outputs]
|
|
621
577
|
for connection_name, predicted_outputs in predicted.outputs.items()
|
|
622
578
|
}
|
|
623
579
|
return cls(
|
|
@@ -628,16 +584,13 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
|
|
|
628
584
|
outputs=outputs,
|
|
629
585
|
)
|
|
630
586
|
|
|
631
|
-
def _add_to_graph(self, graph: ProvenanceQuantumGraph
|
|
587
|
+
def _add_to_graph(self, graph: ProvenanceQuantumGraph) -> None:
|
|
632
588
|
"""Add this quantum and its edges to datasets to a provenance graph.
|
|
633
589
|
|
|
634
590
|
Parameters
|
|
635
591
|
----------
|
|
636
592
|
graph : `ProvenanceQuantumGraph`
|
|
637
593
|
Graph to update in place.
|
|
638
|
-
address_reader : `AddressReader`
|
|
639
|
-
Reader object that can be used to look up UUIDs from integer
|
|
640
|
-
indexes.
|
|
641
594
|
|
|
642
595
|
Notes
|
|
643
596
|
-----
|
|
@@ -655,7 +608,7 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
|
|
|
655
608
|
last_attempt = (
|
|
656
609
|
self.attempts[-1]
|
|
657
610
|
if self.attempts
|
|
658
|
-
else
|
|
611
|
+
else ProvenanceQuantumAttemptModel(status=QuantumAttemptStatus.BLOCKED)
|
|
659
612
|
)
|
|
660
613
|
graph._bipartite_xgraph.add_node(
|
|
661
614
|
self.quantum_id,
|
|
@@ -666,20 +619,18 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
|
|
|
666
619
|
caveats=last_attempt.caveats,
|
|
667
620
|
exception=last_attempt.exception,
|
|
668
621
|
resource_usage=last_attempt.resource_usage,
|
|
669
|
-
attempts=
|
|
622
|
+
attempts=self.attempts,
|
|
670
623
|
)
|
|
671
|
-
for connection_name,
|
|
624
|
+
for connection_name, dataset_ids in self.inputs.items():
|
|
672
625
|
read_edge = task_node.get_input_edge(connection_name)
|
|
673
|
-
for
|
|
674
|
-
dataset_id = address_reader.find(dataset_index).key
|
|
626
|
+
for dataset_id in dataset_ids:
|
|
675
627
|
graph._bipartite_xgraph.add_edge(dataset_id, self.quantum_id, is_read=True)
|
|
676
628
|
graph._bipartite_xgraph.edges[dataset_id, self.quantum_id].setdefault(
|
|
677
629
|
"pipeline_edges", []
|
|
678
630
|
).append(read_edge)
|
|
679
|
-
for connection_name,
|
|
631
|
+
for connection_name, dataset_ids in self.outputs.items():
|
|
680
632
|
write_edge = task_node.get_output_edge(connection_name)
|
|
681
|
-
for
|
|
682
|
-
dataset_id = address_reader.find(dataset_index).key
|
|
633
|
+
for dataset_id in dataset_ids:
|
|
683
634
|
graph._bipartite_xgraph.add_edge(
|
|
684
635
|
self.quantum_id,
|
|
685
636
|
dataset_id,
|
|
@@ -758,28 +709,24 @@ class ProvenanceInitQuantumModel(pydantic.BaseModel):
|
|
|
758
709
|
Note that full dataset type definitions are stored in the pipeline graph.
|
|
759
710
|
"""
|
|
760
711
|
|
|
761
|
-
inputs: dict[ConnectionName,
|
|
762
|
-
"""
|
|
763
|
-
|
|
712
|
+
inputs: dict[ConnectionName, uuid.UUID] = pydantic.Field(default_factory=dict)
|
|
713
|
+
"""IDs of the datasets predicted to be consumed by this quantum, grouped by
|
|
714
|
+
connection name.
|
|
764
715
|
"""
|
|
765
716
|
|
|
766
|
-
outputs: dict[ConnectionName,
|
|
767
|
-
"""
|
|
768
|
-
|
|
717
|
+
outputs: dict[ConnectionName, uuid.UUID] = pydantic.Field(default_factory=dict)
|
|
718
|
+
"""IDs of the datasets predicted to be produced by this quantum, grouped by
|
|
719
|
+
connection name.
|
|
769
720
|
"""
|
|
770
721
|
|
|
771
722
|
@classmethod
|
|
772
|
-
def from_predicted(
|
|
773
|
-
cls, predicted: PredictedQuantumDatasetsModel, indices: Mapping[uuid.UUID, int]
|
|
774
|
-
) -> ProvenanceInitQuantumModel:
|
|
723
|
+
def from_predicted(cls, predicted: PredictedQuantumDatasetsModel) -> ProvenanceInitQuantumModel:
|
|
775
724
|
"""Construct from a predicted quantum model.
|
|
776
725
|
|
|
777
726
|
Parameters
|
|
778
727
|
----------
|
|
779
728
|
predicted : `PredictedQuantumDatasetsModel`
|
|
780
729
|
Information about the quantum from the predicted graph.
|
|
781
|
-
indices : `~collections.abc.Mapping [`uuid.UUID`, `int`]
|
|
782
|
-
Mapping from quantum or dataset UUID to internal integer ID.
|
|
783
730
|
|
|
784
731
|
Returns
|
|
785
732
|
-------
|
|
@@ -787,11 +734,11 @@ class ProvenanceInitQuantumModel(pydantic.BaseModel):
|
|
|
787
734
|
Provenance init quantum model.
|
|
788
735
|
"""
|
|
789
736
|
inputs = {
|
|
790
|
-
connection_name:
|
|
737
|
+
connection_name: predicted_inputs[0].dataset_id
|
|
791
738
|
for connection_name, predicted_inputs in predicted.inputs.items()
|
|
792
739
|
}
|
|
793
740
|
outputs = {
|
|
794
|
-
connection_name:
|
|
741
|
+
connection_name: predicted_outputs[0].dataset_id
|
|
795
742
|
for connection_name, predicted_outputs in predicted.outputs.items()
|
|
796
743
|
}
|
|
797
744
|
return cls(
|
|
@@ -801,21 +748,13 @@ class ProvenanceInitQuantumModel(pydantic.BaseModel):
|
|
|
801
748
|
outputs=outputs,
|
|
802
749
|
)
|
|
803
750
|
|
|
804
|
-
def _add_to_graph(
|
|
805
|
-
self,
|
|
806
|
-
graph: ProvenanceQuantumGraph,
|
|
807
|
-
address_reader: AddressReader,
|
|
808
|
-
empty_data_id: DataCoordinate,
|
|
809
|
-
) -> None:
|
|
751
|
+
def _add_to_graph(self, graph: ProvenanceQuantumGraph, empty_data_id: DataCoordinate) -> None:
|
|
810
752
|
"""Add this quantum and its edges to datasets to a provenance graph.
|
|
811
753
|
|
|
812
754
|
Parameters
|
|
813
755
|
----------
|
|
814
756
|
graph : `ProvenanceQuantumGraph`
|
|
815
757
|
Graph to update in place.
|
|
816
|
-
address_reader : `AddressReader`
|
|
817
|
-
Reader object that can be used to look up UUIDs from integer
|
|
818
|
-
indexes.
|
|
819
758
|
empty_data_id : `lsst.daf.butler.DataCoordinate`
|
|
820
759
|
The empty data ID for the appropriate dimension universe.
|
|
821
760
|
|
|
@@ -831,16 +770,14 @@ class ProvenanceInitQuantumModel(pydantic.BaseModel):
|
|
|
831
770
|
graph._bipartite_xgraph.add_node(
|
|
832
771
|
self.quantum_id, data_id=empty_data_id, task_label=self.task_label, pipeline_node=task_init_node
|
|
833
772
|
)
|
|
834
|
-
for connection_name,
|
|
773
|
+
for connection_name, dataset_id in self.inputs.items():
|
|
835
774
|
read_edge = task_init_node.get_input_edge(connection_name)
|
|
836
|
-
dataset_id = address_reader.find(dataset_index).key
|
|
837
775
|
graph._bipartite_xgraph.add_edge(dataset_id, self.quantum_id, is_read=True)
|
|
838
776
|
graph._bipartite_xgraph.edges[dataset_id, self.quantum_id].setdefault(
|
|
839
777
|
"pipeline_edges", []
|
|
840
778
|
).append(read_edge)
|
|
841
|
-
for connection_name,
|
|
779
|
+
for connection_name, dataset_id in self.outputs.items():
|
|
842
780
|
write_edge = task_init_node.get_output_edge(connection_name)
|
|
843
|
-
dataset_id = address_reader.find(dataset_index).key
|
|
844
781
|
graph._bipartite_xgraph.add_edge(
|
|
845
782
|
self.quantum_id,
|
|
846
783
|
dataset_id,
|
|
@@ -902,20 +839,17 @@ class ProvenanceInitQuantaModel(pydantic.RootModel):
|
|
|
902
839
|
root: list[ProvenanceInitQuantumModel] = pydantic.Field(default_factory=list)
|
|
903
840
|
"""List of special "init" quanta, one for each task."""
|
|
904
841
|
|
|
905
|
-
def _add_to_graph(self, graph: ProvenanceQuantumGraph
|
|
842
|
+
def _add_to_graph(self, graph: ProvenanceQuantumGraph) -> None:
|
|
906
843
|
"""Add this quantum and its edges to datasets to a provenance graph.
|
|
907
844
|
|
|
908
845
|
Parameters
|
|
909
846
|
----------
|
|
910
847
|
graph : `ProvenanceQuantumGraph`
|
|
911
848
|
Graph to update in place.
|
|
912
|
-
address_reader : `AddressReader`
|
|
913
|
-
Reader object that can be used to look up UUIDs from integer
|
|
914
|
-
indexes.
|
|
915
849
|
"""
|
|
916
850
|
empty_data_id = DataCoordinate.make_empty(graph.pipeline_graph.universe)
|
|
917
851
|
for init_quantum in self.root:
|
|
918
|
-
init_quantum._add_to_graph(graph,
|
|
852
|
+
init_quantum._add_to_graph(graph, empty_data_id=empty_data_id)
|
|
919
853
|
|
|
920
854
|
# Work around the fact that Sphinx chokes on Pydantic docstring formatting,
|
|
921
855
|
# when we inherit those docstrings in our public classes.
|
|
@@ -1273,7 +1207,7 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
|
|
|
1273
1207
|
init_quanta = self._read_single_block("init_quanta", ProvenanceInitQuantaModel)
|
|
1274
1208
|
for init_quantum in init_quanta.root:
|
|
1275
1209
|
self.graph._init_quanta[init_quantum.task_label] = init_quantum.quantum_id
|
|
1276
|
-
init_quanta._add_to_graph(self.graph
|
|
1210
|
+
init_quanta._add_to_graph(self.graph)
|
|
1277
1211
|
|
|
1278
1212
|
def read_full_graph(self) -> None:
|
|
1279
1213
|
"""Read all bipartite edges and all quantum and dataset node
|
|
@@ -1288,33 +1222,32 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
|
|
|
1288
1222
|
self.read_datasets()
|
|
1289
1223
|
self.read_quanta()
|
|
1290
1224
|
|
|
1291
|
-
def read_datasets(self, datasets: Iterable[uuid.UUID
|
|
1225
|
+
def read_datasets(self, datasets: Iterable[uuid.UUID] | None = None) -> None:
|
|
1292
1226
|
"""Read information about the given datasets.
|
|
1293
1227
|
|
|
1294
1228
|
Parameters
|
|
1295
1229
|
----------
|
|
1296
|
-
datasets : `~collections.abc.Iterable` [`uuid.UUID`
|
|
1297
|
-
Iterable of dataset IDs
|
|
1298
|
-
|
|
1299
|
-
ignored.
|
|
1230
|
+
datasets : `~collections.abc.Iterable` [`uuid.UUID`], optional
|
|
1231
|
+
Iterable of dataset IDs to load. If not provided, all datasets
|
|
1232
|
+
will be loaded. The UUIDs and indices of quanta will be ignored.
|
|
1300
1233
|
"""
|
|
1301
1234
|
self._read_nodes(datasets, DATASET_ADDRESS_INDEX, DATASET_MB_NAME, ProvenanceDatasetModel)
|
|
1302
1235
|
|
|
1303
|
-
def read_quanta(self, quanta: Iterable[uuid.UUID
|
|
1236
|
+
def read_quanta(self, quanta: Iterable[uuid.UUID] | None = None) -> None:
|
|
1304
1237
|
"""Read information about the given quanta.
|
|
1305
1238
|
|
|
1306
1239
|
Parameters
|
|
1307
1240
|
----------
|
|
1308
|
-
quanta : `~collections.abc.Iterable` [`uuid.UUID`
|
|
1309
|
-
Iterable of quantum IDs
|
|
1310
|
-
|
|
1311
|
-
|
|
1241
|
+
quanta : `~collections.abc.Iterable` [`uuid.UUID`], optional
|
|
1242
|
+
Iterable of quantum IDs to load. If not provided, all quanta will
|
|
1243
|
+
be loaded. The UUIDs and indices of datasets and special init
|
|
1244
|
+
quanta will be ignored.
|
|
1312
1245
|
"""
|
|
1313
1246
|
self._read_nodes(quanta, QUANTUM_ADDRESS_INDEX, QUANTUM_MB_NAME, ProvenanceQuantumModel)
|
|
1314
1247
|
|
|
1315
1248
|
def _read_nodes(
|
|
1316
1249
|
self,
|
|
1317
|
-
nodes: Iterable[uuid.UUID
|
|
1250
|
+
nodes: Iterable[uuid.UUID] | None,
|
|
1318
1251
|
address_index: int,
|
|
1319
1252
|
mb_name: str,
|
|
1320
1253
|
model_type: type[ProvenanceDatasetModel] | type[ProvenanceQuantumModel],
|
|
@@ -1335,7 +1268,7 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
|
|
|
1335
1268
|
# Use the old node to reduce memory usage (since it might
|
|
1336
1269
|
# also have other outstanding reference holders).
|
|
1337
1270
|
continue
|
|
1338
|
-
node._add_to_graph(self.graph
|
|
1271
|
+
node._add_to_graph(self.graph)
|
|
1339
1272
|
return
|
|
1340
1273
|
with MultiblockReader.open_in_zip(self.zf, mb_name, int_size=self.header.int_size) as mb_reader:
|
|
1341
1274
|
for node_id_or_index in nodes:
|
|
@@ -1348,29 +1281,27 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
|
|
|
1348
1281
|
address_row.addresses[address_index], model_type, self.decompressor
|
|
1349
1282
|
)
|
|
1350
1283
|
if node is not None:
|
|
1351
|
-
node._add_to_graph(self.graph
|
|
1284
|
+
node._add_to_graph(self.graph)
|
|
1352
1285
|
|
|
1353
|
-
def fetch_logs(
|
|
1354
|
-
self, nodes: Iterable[uuid.UUID | DatasetIndex | QuantumIndex]
|
|
1355
|
-
) -> dict[uuid.UUID | DatasetIndex | QuantumIndex, list[ButlerLogRecords | None]]:
|
|
1286
|
+
def fetch_logs(self, nodes: Iterable[uuid.UUID]) -> dict[uuid.UUID, list[ButlerLogRecords | None]]:
|
|
1356
1287
|
"""Fetch log datasets.
|
|
1357
1288
|
|
|
1358
1289
|
Parameters
|
|
1359
1290
|
----------
|
|
1360
1291
|
nodes : `~collections.abc.Iterable` [ `uuid.UUID` ]
|
|
1361
|
-
UUIDs
|
|
1362
|
-
|
|
1292
|
+
UUIDs of the log datasets themselves or of the quanta they
|
|
1293
|
+
correspond to.
|
|
1363
1294
|
|
|
1364
1295
|
Returns
|
|
1365
1296
|
-------
|
|
1366
|
-
logs : `dict` [ `uuid.UUID
|
|
1297
|
+
logs : `dict` [ `uuid.UUID`, `list` [\
|
|
1367
1298
|
`lsst.daf.butler.ButlerLogRecords` or `None`] ]
|
|
1368
1299
|
Logs for the given IDs. Each value is a list of
|
|
1369
1300
|
`lsst.daf.butler.ButlerLogRecords` instances representing different
|
|
1370
1301
|
execution attempts, ordered chronologically from first to last.
|
|
1371
1302
|
Attempts where logs were missing will have `None` in this list.
|
|
1372
1303
|
"""
|
|
1373
|
-
result: dict[uuid.UUID
|
|
1304
|
+
result: dict[uuid.UUID, list[ButlerLogRecords | None]] = {}
|
|
1374
1305
|
with MultiblockReader.open_in_zip(self.zf, LOG_MB_NAME, int_size=self.header.int_size) as mb_reader:
|
|
1375
1306
|
for node_id_or_index in nodes:
|
|
1376
1307
|
address_row = self.address_reader.find(node_id_or_index)
|
|
@@ -1384,27 +1315,25 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
|
|
|
1384
1315
|
]
|
|
1385
1316
|
return result
|
|
1386
1317
|
|
|
1387
|
-
def fetch_metadata(
|
|
1388
|
-
self, nodes: Iterable[uuid.UUID | DatasetIndex | QuantumIndex]
|
|
1389
|
-
) -> dict[uuid.UUID | DatasetIndex | QuantumIndex, list[TaskMetadata | None]]:
|
|
1318
|
+
def fetch_metadata(self, nodes: Iterable[uuid.UUID]) -> dict[uuid.UUID, list[TaskMetadata | None]]:
|
|
1390
1319
|
"""Fetch metadata datasets.
|
|
1391
1320
|
|
|
1392
1321
|
Parameters
|
|
1393
1322
|
----------
|
|
1394
1323
|
nodes : `~collections.abc.Iterable` [ `uuid.UUID` ]
|
|
1395
|
-
UUIDs
|
|
1396
|
-
|
|
1324
|
+
UUIDs of the metadata datasets themselves or of the quanta they
|
|
1325
|
+
correspond to.
|
|
1397
1326
|
|
|
1398
1327
|
Returns
|
|
1399
1328
|
-------
|
|
1400
|
-
metadata : `dict` [ `uuid.UUID
|
|
1329
|
+
metadata : `dict` [ `uuid.UUID`, `list` [`.TaskMetadata`] ]
|
|
1401
1330
|
Metadata for the given IDs. Each value is a list of
|
|
1402
1331
|
`.TaskMetadata` instances representing different execution
|
|
1403
1332
|
attempts, ordered chronologically from first to last. Attempts
|
|
1404
1333
|
where metadata was missing (not written even in the fallback extra
|
|
1405
1334
|
provenance in the logs) will have `None` in this list.
|
|
1406
1335
|
"""
|
|
1407
|
-
result: dict[uuid.UUID
|
|
1336
|
+
result: dict[uuid.UUID, list[TaskMetadata | None]] = {}
|
|
1408
1337
|
with MultiblockReader.open_in_zip(
|
|
1409
1338
|
self.zf, METADATA_MB_NAME, int_size=self.header.int_size
|
|
1410
1339
|
) as mb_reader:
|
|
@@ -60,7 +60,7 @@ from lsst.utils.logging import VERBOSE, LsstLogAdapter
|
|
|
60
60
|
|
|
61
61
|
from ._config import AggregatorConfig
|
|
62
62
|
from ._progress import ProgressManager, make_worker_log
|
|
63
|
-
from ._structs import IngestRequest, ScanReport,
|
|
63
|
+
from ._structs import IngestRequest, ScanReport, WriteRequest
|
|
64
64
|
|
|
65
65
|
_T = TypeVar("_T")
|
|
66
66
|
|
|
@@ -361,7 +361,7 @@ class SupervisorCommunicator:
|
|
|
361
361
|
# scanner and the supervisor send one sentinal when done, and the
|
|
362
362
|
# writer waits for (n_scanners + 1) sentinals to arrive before it
|
|
363
363
|
# starts its shutdown.
|
|
364
|
-
self._write_requests: Queue[
|
|
364
|
+
self._write_requests: Queue[WriteRequest | Literal[_Sentinel.NO_MORE_WRITE_REQUESTS]] | None = (
|
|
365
365
|
context.make_queue() if config.output_path is not None else None
|
|
366
366
|
)
|
|
367
367
|
# All other workers use this queue to send many different kinds of
|
|
@@ -461,17 +461,17 @@ class SupervisorCommunicator:
|
|
|
461
461
|
"""
|
|
462
462
|
self._scan_requests.put(_ScanRequest(quantum_id), block=False)
|
|
463
463
|
|
|
464
|
-
def request_write(self,
|
|
464
|
+
def request_write(self, request: WriteRequest) -> None:
|
|
465
465
|
"""Send a request to the writer to write provenance for the given scan.
|
|
466
466
|
|
|
467
467
|
Parameters
|
|
468
468
|
----------
|
|
469
|
-
|
|
469
|
+
request : `WriteRequest`
|
|
470
470
|
Information from scanning a quantum (or knowing you don't have to,
|
|
471
471
|
in the case of blocked quanta).
|
|
472
472
|
"""
|
|
473
473
|
assert self._write_requests is not None, "Writer should not be used if writing is disabled."
|
|
474
|
-
self._write_requests.put(
|
|
474
|
+
self._write_requests.put(request, block=False)
|
|
475
475
|
|
|
476
476
|
def poll(self) -> Iterator[ScanReport]:
|
|
477
477
|
"""Poll for reports from workers while sending scan requests.
|
|
@@ -728,16 +728,16 @@ class ScannerCommunicator(WorkerCommunicator):
|
|
|
728
728
|
else:
|
|
729
729
|
self._reports.put(_IngestReport(1), block=False)
|
|
730
730
|
|
|
731
|
-
def request_write(self,
|
|
731
|
+
def request_write(self, request: WriteRequest) -> None:
|
|
732
732
|
"""Ask the writer to write provenance for a quantum.
|
|
733
733
|
|
|
734
734
|
Parameters
|
|
735
735
|
----------
|
|
736
|
-
|
|
736
|
+
request : `WriteRequest`
|
|
737
737
|
Result of scanning a quantum.
|
|
738
738
|
"""
|
|
739
739
|
assert self._write_requests is not None, "Writer should not be used if writing is disabled."
|
|
740
|
-
self._write_requests.put(
|
|
740
|
+
self._write_requests.put(request, block=False)
|
|
741
741
|
|
|
742
742
|
def get_compression_dict(self) -> bytes | None:
|
|
743
743
|
"""Attempt to get the compression dict from the writer.
|
|
@@ -913,12 +913,12 @@ class WriterCommunicator(WorkerCommunicator):
|
|
|
913
913
|
self._reports.put(_Sentinel.WRITER_DONE, block=False)
|
|
914
914
|
return result
|
|
915
915
|
|
|
916
|
-
def poll(self) -> Iterator[
|
|
916
|
+
def poll(self) -> Iterator[WriteRequest]:
|
|
917
917
|
"""Poll for writer requests from the scanner workers and supervisor.
|
|
918
918
|
|
|
919
919
|
Yields
|
|
920
920
|
------
|
|
921
|
-
request : `
|
|
921
|
+
request : `WriteRequest`
|
|
922
922
|
The result of a quantum scan.
|
|
923
923
|
|
|
924
924
|
Notes
|