lsst-pipe-base 29.2025.4600__py3-none-any.whl → 29.2025.4700__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. lsst/pipe/base/quantum_graph/_common.py +15 -1
  2. lsst/pipe/base/quantum_graph/_multiblock.py +14 -39
  3. lsst/pipe/base/quantum_graph/_predicted.py +77 -73
  4. lsst/pipe/base/quantum_graph/_provenance.py +73 -144
  5. lsst/pipe/base/quantum_graph/aggregator/_communicators.py +10 -10
  6. lsst/pipe/base/quantum_graph/aggregator/_scanner.py +88 -60
  7. lsst/pipe/base/quantum_graph/aggregator/_structs.py +36 -19
  8. lsst/pipe/base/quantum_graph/aggregator/_supervisor.py +7 -10
  9. lsst/pipe/base/quantum_graph/aggregator/_writer.py +55 -144
  10. lsst/pipe/base/quantum_graph_builder.py +0 -1
  11. lsst/pipe/base/version.py +1 -1
  12. {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/METADATA +1 -1
  13. {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/RECORD +21 -21
  14. {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/WHEEL +0 -0
  15. {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/entry_points.txt +0 -0
  16. {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/licenses/COPYRIGHT +0 -0
  17. {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/licenses/LICENSE +0 -0
  18. {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/licenses/bsd_license.txt +0 -0
  19. {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/licenses/gpl-v3.0.txt +0 -0
  20. {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/top_level.txt +0 -0
  21. {lsst_pipe_base-29.2025.4600.dist-info → lsst_pipe_base-29.2025.4700.dist-info}/zip-safe +0 -0
@@ -47,7 +47,7 @@ import uuid
47
47
  from collections import Counter
48
48
  from collections.abc import Iterable, Iterator, Mapping
49
49
  from contextlib import contextmanager
50
- from typing import TYPE_CHECKING, Any, Generic, TypeAlias, TypedDict, TypeVar
50
+ from typing import TYPE_CHECKING, Any, TypedDict, TypeVar
51
51
 
52
52
  import astropy.table
53
53
  import networkx
@@ -68,15 +68,13 @@ from ._common import (
68
68
  BaseQuantumGraphReader,
69
69
  ConnectionName,
70
70
  DataCoordinateValues,
71
- DatasetIndex,
72
71
  DatasetInfo,
73
72
  DatasetTypeName,
74
73
  HeaderModel,
75
- QuantumIndex,
76
74
  QuantumInfo,
77
75
  TaskLabel,
78
76
  )
79
- from ._multiblock import AddressReader, MultiblockReader
77
+ from ._multiblock import MultiblockReader
80
78
  from ._predicted import PredictedDatasetModel, PredictedQuantumDatasetsModel
81
79
 
82
80
  DATASET_ADDRESS_INDEX = 0
@@ -202,16 +200,14 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
202
200
  existence of the dataset.
203
201
  """
204
202
 
205
- producer: QuantumIndex | None = None
206
- """Internal integer ID of the quantum that produced this dataset.
203
+ producer: uuid.UUID | None = None
204
+ """ID of the quantum that produced this dataset.
207
205
 
208
206
  This is `None` for overall inputs to the graph.
209
207
  """
210
208
 
211
- consumers: list[QuantumIndex] = pydantic.Field(default_factory=list)
212
- """Internal integer IDs of quanta that were predicted to consume this
213
- dataset.
214
- """
209
+ consumers: list[uuid.UUID] = pydantic.Field(default_factory=list)
210
+ """IDs of quanta that were predicted to consume this dataset."""
215
211
 
216
212
  @property
217
213
  def node_id(self) -> uuid.UUID:
@@ -222,8 +218,8 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
222
218
  def from_predicted(
223
219
  cls,
224
220
  predicted: PredictedDatasetModel,
225
- producer: QuantumIndex | None = None,
226
- consumers: Iterable[QuantumIndex] = (),
221
+ producer: uuid.UUID | None = None,
222
+ consumers: Iterable[uuid.UUID] = (),
227
223
  ) -> ProvenanceDatasetModel:
228
224
  """Construct from a predicted dataset model.
229
225
 
@@ -231,12 +227,10 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
231
227
  ----------
232
228
  predicted : `PredictedDatasetModel`
233
229
  Information about the dataset from the predicted graph.
234
- producer : `int` or `None`, optional
235
- Internal ID of the quantum that was predicted to produce this
236
- dataset.
237
- consumers : `~collections.abc.Iterable` [`int`], optional
238
- Internal IDs of the quanta that were predicted to consume this
239
- dataset.
230
+ producer : `uuid.UUID` or `None`, optional
231
+ ID of the quantum that was predicted to produce this dataset.
232
+ consumers : `~collections.abc.Iterable` [`uuid.UUID`], optional
233
+ IDs of the quanta that were predicted to consume this dataset.
240
234
 
241
235
  Returns
242
236
  -------
@@ -258,16 +252,13 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
258
252
  consumers=list(consumers),
259
253
  )
260
254
 
261
- def _add_to_graph(self, graph: ProvenanceQuantumGraph, address_reader: AddressReader) -> None:
255
+ def _add_to_graph(self, graph: ProvenanceQuantumGraph) -> None:
262
256
  """Add this dataset and its edges to quanta to a provenance graph.
263
257
 
264
258
  Parameters
265
259
  ----------
266
260
  graph : `ProvenanceQuantumGraph`
267
261
  Graph to update in place.
268
- address_reader : `AddressReader`
269
- Reader object that can be used to look up UUIDs from integer
270
- indexes.
271
262
 
272
263
  Notes
273
264
  -----
@@ -290,15 +281,12 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
290
281
  run=self.run,
291
282
  produced=self.produced,
292
283
  )
293
- producer_id: uuid.UUID | None = None
294
284
  if self.producer is not None:
295
- producer_id = address_reader.find(self.producer).key
296
- graph._bipartite_xgraph.add_edge(producer_id, self.dataset_id)
297
- for consumer_index in self.consumers:
298
- consumer_id = address_reader.find(consumer_index).key
285
+ graph._bipartite_xgraph.add_edge(self.producer, self.dataset_id)
286
+ for consumer_id in self.consumers:
299
287
  graph._bipartite_xgraph.add_edge(self.dataset_id, consumer_id)
300
- if producer_id is not None:
301
- graph._quantum_only_xgraph.add_edge(producer_id, consumer_id)
288
+ if self.producer is not None:
289
+ graph._quantum_only_xgraph.add_edge(self.producer, consumer_id)
302
290
  graph._datasets_by_type[self.dataset_type_name][data_id] = self.dataset_id
303
291
 
304
292
  # Work around the fact that Sphinx chokes on Pydantic docstring formatting,
@@ -347,7 +335,7 @@ class ProvenanceDatasetModel(PredictedDatasetModel):
347
335
  return super().model_validate_strings(*args, **kwargs)
348
336
 
349
337
 
350
- class _GenericProvenanceQuantumAttemptModel(pydantic.BaseModel, Generic[_I]):
338
+ class ProvenanceQuantumAttemptModel(pydantic.BaseModel):
351
339
  """Data model for a now-superseded attempt to run a quantum in a
352
340
  provenance quantum graph file.
353
341
  """
@@ -367,35 +355,11 @@ class _GenericProvenanceQuantumAttemptModel(pydantic.BaseModel, Generic[_I]):
367
355
  resource_usage: QuantumResourceUsage | None = None
368
356
  """Resource usage information (timing, memory use) for this quantum."""
369
357
 
370
- previous_process_quanta: list[_I] = pydantic.Field(default_factory=list)
358
+ previous_process_quanta: list[uuid.UUID] = pydantic.Field(default_factory=list)
371
359
  """The IDs of other quanta previously executed in the same process as this
372
360
  one.
373
361
  """
374
362
 
375
- def remap_uuids(
376
- self: ProvenanceQuantumAttemptModel, indices: Mapping[uuid.UUID, QuantumIndex]
377
- ) -> StorageProvenanceQuantumAttemptModel:
378
- return StorageProvenanceQuantumAttemptModel(
379
- attempt=self.attempt,
380
- status=self.status,
381
- caveats=self.caveats,
382
- exception=self.exception,
383
- resource_usage=self.resource_usage,
384
- previous_process_quanta=[indices[q] for q in self.previous_process_quanta],
385
- )
386
-
387
- def remap_indices(
388
- self: StorageProvenanceQuantumAttemptModel, address_reader: AddressReader
389
- ) -> ProvenanceQuantumAttemptModel:
390
- return ProvenanceQuantumAttemptModel(
391
- attempt=self.attempt,
392
- status=self.status,
393
- caveats=self.caveats,
394
- exception=self.exception,
395
- resource_usage=self.resource_usage,
396
- previous_process_quanta=[address_reader.find(q).key for q in self.previous_process_quanta],
397
- )
398
-
399
363
  # Work around the fact that Sphinx chokes on Pydantic docstring formatting,
400
364
  # when we inherit those docstrings in our public classes.
401
365
  if "sphinx" in sys.modules and not TYPE_CHECKING:
@@ -442,10 +406,6 @@ class _GenericProvenanceQuantumAttemptModel(pydantic.BaseModel, Generic[_I]):
442
406
  return super().model_validate_strings(*args, **kwargs)
443
407
 
444
408
 
445
- StorageProvenanceQuantumAttemptModel: TypeAlias = _GenericProvenanceQuantumAttemptModel[QuantumIndex]
446
- ProvenanceQuantumAttemptModel: TypeAlias = _GenericProvenanceQuantumAttemptModel[uuid.UUID]
447
-
448
-
449
409
  class ProvenanceLogRecordsModel(pydantic.BaseModel):
450
410
  """Data model for storing execution logs in a provenance quantum graph
451
411
  file.
@@ -570,17 +530,17 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
570
530
  data_coordinate: DataCoordinateValues = pydantic.Field(default_factory=list)
571
531
  """The full values (required and implied) of this dataset's data ID."""
572
532
 
573
- inputs: dict[ConnectionName, list[DatasetIndex]] = pydantic.Field(default_factory=dict)
574
- """Internal integer IDs of the datasets predicted to be consumed by this
575
- quantum, grouped by connection name.
533
+ inputs: dict[ConnectionName, list[uuid.UUID]] = pydantic.Field(default_factory=dict)
534
+ """IDs of the datasets predicted to be consumed by this quantum, grouped by
535
+ connection name.
576
536
  """
577
537
 
578
- outputs: dict[ConnectionName, list[DatasetIndex]] = pydantic.Field(default_factory=dict)
579
- """Internal integer IDs of the datasets predicted to be produced by this
580
- quantum, grouped by connection name.
538
+ outputs: dict[ConnectionName, list[uuid.UUID]] = pydantic.Field(default_factory=dict)
539
+ """IDs of the datasets predicted to be produced by this quantum, grouped by
540
+ connection name.
581
541
  """
582
542
 
583
- attempts: list[StorageProvenanceQuantumAttemptModel] = pydantic.Field(default_factory=list)
543
+ attempts: list[ProvenanceQuantumAttemptModel] = pydantic.Field(default_factory=list)
584
544
  """Provenance for all attempts to execute this quantum, ordered
585
545
  chronologically from first to last.
586
546
 
@@ -595,17 +555,13 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
595
555
  return self.quantum_id
596
556
 
597
557
  @classmethod
598
- def from_predicted(
599
- cls, predicted: PredictedQuantumDatasetsModel, indices: Mapping[uuid.UUID, int]
600
- ) -> ProvenanceQuantumModel:
558
+ def from_predicted(cls, predicted: PredictedQuantumDatasetsModel) -> ProvenanceQuantumModel:
601
559
  """Construct from a predicted quantum model.
602
560
 
603
561
  Parameters
604
562
  ----------
605
563
  predicted : `PredictedQuantumDatasetsModel`
606
564
  Information about the quantum from the predicted graph.
607
- indices : `~collections.abc.Mapping [`uuid.UUID`, `int`]
608
- Mapping from quantum or dataset UUID to internal integer ID.
609
565
 
610
566
  Returns
611
567
  -------
@@ -613,11 +569,11 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
613
569
  Provenance quantum model.
614
570
  """
615
571
  inputs = {
616
- connection_name: [indices[d.dataset_id] for d in predicted_inputs]
572
+ connection_name: [d.dataset_id for d in predicted_inputs]
617
573
  for connection_name, predicted_inputs in predicted.inputs.items()
618
574
  }
619
575
  outputs = {
620
- connection_name: [indices[d.dataset_id] for d in predicted_outputs]
576
+ connection_name: [d.dataset_id for d in predicted_outputs]
621
577
  for connection_name, predicted_outputs in predicted.outputs.items()
622
578
  }
623
579
  return cls(
@@ -628,16 +584,13 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
628
584
  outputs=outputs,
629
585
  )
630
586
 
631
- def _add_to_graph(self, graph: ProvenanceQuantumGraph, address_reader: AddressReader) -> None:
587
+ def _add_to_graph(self, graph: ProvenanceQuantumGraph) -> None:
632
588
  """Add this quantum and its edges to datasets to a provenance graph.
633
589
 
634
590
  Parameters
635
591
  ----------
636
592
  graph : `ProvenanceQuantumGraph`
637
593
  Graph to update in place.
638
- address_reader : `AddressReader`
639
- Reader object that can be used to look up UUIDs from integer
640
- indexes.
641
594
 
642
595
  Notes
643
596
  -----
@@ -655,7 +608,7 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
655
608
  last_attempt = (
656
609
  self.attempts[-1]
657
610
  if self.attempts
658
- else StorageProvenanceQuantumAttemptModel(status=QuantumAttemptStatus.BLOCKED)
611
+ else ProvenanceQuantumAttemptModel(status=QuantumAttemptStatus.BLOCKED)
659
612
  )
660
613
  graph._bipartite_xgraph.add_node(
661
614
  self.quantum_id,
@@ -666,20 +619,18 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
666
619
  caveats=last_attempt.caveats,
667
620
  exception=last_attempt.exception,
668
621
  resource_usage=last_attempt.resource_usage,
669
- attempts=[a.remap_indices(address_reader) for a in self.attempts],
622
+ attempts=self.attempts,
670
623
  )
671
- for connection_name, dataset_indices in self.inputs.items():
624
+ for connection_name, dataset_ids in self.inputs.items():
672
625
  read_edge = task_node.get_input_edge(connection_name)
673
- for dataset_index in dataset_indices:
674
- dataset_id = address_reader.find(dataset_index).key
626
+ for dataset_id in dataset_ids:
675
627
  graph._bipartite_xgraph.add_edge(dataset_id, self.quantum_id, is_read=True)
676
628
  graph._bipartite_xgraph.edges[dataset_id, self.quantum_id].setdefault(
677
629
  "pipeline_edges", []
678
630
  ).append(read_edge)
679
- for connection_name, dataset_indices in self.outputs.items():
631
+ for connection_name, dataset_ids in self.outputs.items():
680
632
  write_edge = task_node.get_output_edge(connection_name)
681
- for dataset_index in dataset_indices:
682
- dataset_id = address_reader.find(dataset_index).key
633
+ for dataset_id in dataset_ids:
683
634
  graph._bipartite_xgraph.add_edge(
684
635
  self.quantum_id,
685
636
  dataset_id,
@@ -758,28 +709,24 @@ class ProvenanceInitQuantumModel(pydantic.BaseModel):
758
709
  Note that full dataset type definitions are stored in the pipeline graph.
759
710
  """
760
711
 
761
- inputs: dict[ConnectionName, DatasetIndex] = pydantic.Field(default_factory=dict)
762
- """Internal integer IDs of the datasets predicted to be consumed by this
763
- quantum, grouped by connection name.
712
+ inputs: dict[ConnectionName, uuid.UUID] = pydantic.Field(default_factory=dict)
713
+ """IDs of the datasets predicted to be consumed by this quantum, grouped by
714
+ connection name.
764
715
  """
765
716
 
766
- outputs: dict[ConnectionName, DatasetIndex] = pydantic.Field(default_factory=dict)
767
- """Internal integer IDs of the datasets predicted to be produced by this
768
- quantum, grouped by connection name.
717
+ outputs: dict[ConnectionName, uuid.UUID] = pydantic.Field(default_factory=dict)
718
+ """IDs of the datasets predicted to be produced by this quantum, grouped by
719
+ connection name.
769
720
  """
770
721
 
771
722
  @classmethod
772
- def from_predicted(
773
- cls, predicted: PredictedQuantumDatasetsModel, indices: Mapping[uuid.UUID, int]
774
- ) -> ProvenanceInitQuantumModel:
723
+ def from_predicted(cls, predicted: PredictedQuantumDatasetsModel) -> ProvenanceInitQuantumModel:
775
724
  """Construct from a predicted quantum model.
776
725
 
777
726
  Parameters
778
727
  ----------
779
728
  predicted : `PredictedQuantumDatasetsModel`
780
729
  Information about the quantum from the predicted graph.
781
- indices : `~collections.abc.Mapping [`uuid.UUID`, `int`]
782
- Mapping from quantum or dataset UUID to internal integer ID.
783
730
 
784
731
  Returns
785
732
  -------
@@ -787,11 +734,11 @@ class ProvenanceInitQuantumModel(pydantic.BaseModel):
787
734
  Provenance init quantum model.
788
735
  """
789
736
  inputs = {
790
- connection_name: indices[predicted_inputs[0].dataset_id]
737
+ connection_name: predicted_inputs[0].dataset_id
791
738
  for connection_name, predicted_inputs in predicted.inputs.items()
792
739
  }
793
740
  outputs = {
794
- connection_name: indices[predicted_outputs[0].dataset_id]
741
+ connection_name: predicted_outputs[0].dataset_id
795
742
  for connection_name, predicted_outputs in predicted.outputs.items()
796
743
  }
797
744
  return cls(
@@ -801,21 +748,13 @@ class ProvenanceInitQuantumModel(pydantic.BaseModel):
801
748
  outputs=outputs,
802
749
  )
803
750
 
804
- def _add_to_graph(
805
- self,
806
- graph: ProvenanceQuantumGraph,
807
- address_reader: AddressReader,
808
- empty_data_id: DataCoordinate,
809
- ) -> None:
751
+ def _add_to_graph(self, graph: ProvenanceQuantumGraph, empty_data_id: DataCoordinate) -> None:
810
752
  """Add this quantum and its edges to datasets to a provenance graph.
811
753
 
812
754
  Parameters
813
755
  ----------
814
756
  graph : `ProvenanceQuantumGraph`
815
757
  Graph to update in place.
816
- address_reader : `AddressReader`
817
- Reader object that can be used to look up UUIDs from integer
818
- indexes.
819
758
  empty_data_id : `lsst.daf.butler.DataCoordinate`
820
759
  The empty data ID for the appropriate dimension universe.
821
760
 
@@ -831,16 +770,14 @@ class ProvenanceInitQuantumModel(pydantic.BaseModel):
831
770
  graph._bipartite_xgraph.add_node(
832
771
  self.quantum_id, data_id=empty_data_id, task_label=self.task_label, pipeline_node=task_init_node
833
772
  )
834
- for connection_name, dataset_index in self.inputs.items():
773
+ for connection_name, dataset_id in self.inputs.items():
835
774
  read_edge = task_init_node.get_input_edge(connection_name)
836
- dataset_id = address_reader.find(dataset_index).key
837
775
  graph._bipartite_xgraph.add_edge(dataset_id, self.quantum_id, is_read=True)
838
776
  graph._bipartite_xgraph.edges[dataset_id, self.quantum_id].setdefault(
839
777
  "pipeline_edges", []
840
778
  ).append(read_edge)
841
- for connection_name, dataset_index in self.outputs.items():
779
+ for connection_name, dataset_id in self.outputs.items():
842
780
  write_edge = task_init_node.get_output_edge(connection_name)
843
- dataset_id = address_reader.find(dataset_index).key
844
781
  graph._bipartite_xgraph.add_edge(
845
782
  self.quantum_id,
846
783
  dataset_id,
@@ -902,20 +839,17 @@ class ProvenanceInitQuantaModel(pydantic.RootModel):
902
839
  root: list[ProvenanceInitQuantumModel] = pydantic.Field(default_factory=list)
903
840
  """List of special "init" quanta, one for each task."""
904
841
 
905
- def _add_to_graph(self, graph: ProvenanceQuantumGraph, address_reader: AddressReader) -> None:
842
+ def _add_to_graph(self, graph: ProvenanceQuantumGraph) -> None:
906
843
  """Add this quantum and its edges to datasets to a provenance graph.
907
844
 
908
845
  Parameters
909
846
  ----------
910
847
  graph : `ProvenanceQuantumGraph`
911
848
  Graph to update in place.
912
- address_reader : `AddressReader`
913
- Reader object that can be used to look up UUIDs from integer
914
- indexes.
915
849
  """
916
850
  empty_data_id = DataCoordinate.make_empty(graph.pipeline_graph.universe)
917
851
  for init_quantum in self.root:
918
- init_quantum._add_to_graph(graph, address_reader, empty_data_id=empty_data_id)
852
+ init_quantum._add_to_graph(graph, empty_data_id=empty_data_id)
919
853
 
920
854
  # Work around the fact that Sphinx chokes on Pydantic docstring formatting,
921
855
  # when we inherit those docstrings in our public classes.
@@ -1273,7 +1207,7 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
1273
1207
  init_quanta = self._read_single_block("init_quanta", ProvenanceInitQuantaModel)
1274
1208
  for init_quantum in init_quanta.root:
1275
1209
  self.graph._init_quanta[init_quantum.task_label] = init_quantum.quantum_id
1276
- init_quanta._add_to_graph(self.graph, self.address_reader)
1210
+ init_quanta._add_to_graph(self.graph)
1277
1211
 
1278
1212
  def read_full_graph(self) -> None:
1279
1213
  """Read all bipartite edges and all quantum and dataset node
@@ -1288,33 +1222,32 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
1288
1222
  self.read_datasets()
1289
1223
  self.read_quanta()
1290
1224
 
1291
- def read_datasets(self, datasets: Iterable[uuid.UUID | DatasetIndex] | None = None) -> None:
1225
+ def read_datasets(self, datasets: Iterable[uuid.UUID] | None = None) -> None:
1292
1226
  """Read information about the given datasets.
1293
1227
 
1294
1228
  Parameters
1295
1229
  ----------
1296
- datasets : `~collections.abc.Iterable` [`uuid.UUID` or `int`], optional
1297
- Iterable of dataset IDs or indices to load. If not provided, all
1298
- datasets will be loaded. The UUIDs and indices of quanta will be
1299
- ignored.
1230
+ datasets : `~collections.abc.Iterable` [`uuid.UUID`], optional
1231
+ Iterable of dataset IDs to load. If not provided, all datasets
1232
+ will be loaded. The UUIDs and indices of quanta will be ignored.
1300
1233
  """
1301
1234
  self._read_nodes(datasets, DATASET_ADDRESS_INDEX, DATASET_MB_NAME, ProvenanceDatasetModel)
1302
1235
 
1303
- def read_quanta(self, quanta: Iterable[uuid.UUID | QuantumIndex] | None = None) -> None:
1236
+ def read_quanta(self, quanta: Iterable[uuid.UUID] | None = None) -> None:
1304
1237
  """Read information about the given quanta.
1305
1238
 
1306
1239
  Parameters
1307
1240
  ----------
1308
- quanta : `~collections.abc.Iterable` [`uuid.UUID` or `int`], optional
1309
- Iterable of quantum IDs or indices to load. If not provided, all
1310
- quanta will be loaded. The UUIDs and indices of datasets and
1311
- special init quanta will be ignored.
1241
+ quanta : `~collections.abc.Iterable` [`uuid.UUID`], optional
1242
+ Iterable of quantum IDs to load. If not provided, all quanta will
1243
+ be loaded. The UUIDs and indices of datasets and special init
1244
+ quanta will be ignored.
1312
1245
  """
1313
1246
  self._read_nodes(quanta, QUANTUM_ADDRESS_INDEX, QUANTUM_MB_NAME, ProvenanceQuantumModel)
1314
1247
 
1315
1248
  def _read_nodes(
1316
1249
  self,
1317
- nodes: Iterable[uuid.UUID | int] | None,
1250
+ nodes: Iterable[uuid.UUID] | None,
1318
1251
  address_index: int,
1319
1252
  mb_name: str,
1320
1253
  model_type: type[ProvenanceDatasetModel] | type[ProvenanceQuantumModel],
@@ -1335,7 +1268,7 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
1335
1268
  # Use the old node to reduce memory usage (since it might
1336
1269
  # also have other outstanding reference holders).
1337
1270
  continue
1338
- node._add_to_graph(self.graph, self.address_reader)
1271
+ node._add_to_graph(self.graph)
1339
1272
  return
1340
1273
  with MultiblockReader.open_in_zip(self.zf, mb_name, int_size=self.header.int_size) as mb_reader:
1341
1274
  for node_id_or_index in nodes:
@@ -1348,29 +1281,27 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
1348
1281
  address_row.addresses[address_index], model_type, self.decompressor
1349
1282
  )
1350
1283
  if node is not None:
1351
- node._add_to_graph(self.graph, self.address_reader)
1284
+ node._add_to_graph(self.graph)
1352
1285
 
1353
- def fetch_logs(
1354
- self, nodes: Iterable[uuid.UUID | DatasetIndex | QuantumIndex]
1355
- ) -> dict[uuid.UUID | DatasetIndex | QuantumIndex, list[ButlerLogRecords | None]]:
1286
+ def fetch_logs(self, nodes: Iterable[uuid.UUID]) -> dict[uuid.UUID, list[ButlerLogRecords | None]]:
1356
1287
  """Fetch log datasets.
1357
1288
 
1358
1289
  Parameters
1359
1290
  ----------
1360
1291
  nodes : `~collections.abc.Iterable` [ `uuid.UUID` ]
1361
- UUIDs or internal integer IDS of the log datasets themselves or of
1362
- the quanta they correspond to.
1292
+ UUIDs of the log datasets themselves or of the quanta they
1293
+ correspond to.
1363
1294
 
1364
1295
  Returns
1365
1296
  -------
1366
- logs : `dict` [ `uuid.UUID` or `int`, `list` [\
1297
+ logs : `dict` [ `uuid.UUID`, `list` [\
1367
1298
  `lsst.daf.butler.ButlerLogRecords` or `None`] ]
1368
1299
  Logs for the given IDs. Each value is a list of
1369
1300
  `lsst.daf.butler.ButlerLogRecords` instances representing different
1370
1301
  execution attempts, ordered chronologically from first to last.
1371
1302
  Attempts where logs were missing will have `None` in this list.
1372
1303
  """
1373
- result: dict[uuid.UUID | DatasetIndex | QuantumIndex, list[ButlerLogRecords | None]] = {}
1304
+ result: dict[uuid.UUID, list[ButlerLogRecords | None]] = {}
1374
1305
  with MultiblockReader.open_in_zip(self.zf, LOG_MB_NAME, int_size=self.header.int_size) as mb_reader:
1375
1306
  for node_id_or_index in nodes:
1376
1307
  address_row = self.address_reader.find(node_id_or_index)
@@ -1384,27 +1315,25 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
1384
1315
  ]
1385
1316
  return result
1386
1317
 
1387
- def fetch_metadata(
1388
- self, nodes: Iterable[uuid.UUID | DatasetIndex | QuantumIndex]
1389
- ) -> dict[uuid.UUID | DatasetIndex | QuantumIndex, list[TaskMetadata | None]]:
1318
+ def fetch_metadata(self, nodes: Iterable[uuid.UUID]) -> dict[uuid.UUID, list[TaskMetadata | None]]:
1390
1319
  """Fetch metadata datasets.
1391
1320
 
1392
1321
  Parameters
1393
1322
  ----------
1394
1323
  nodes : `~collections.abc.Iterable` [ `uuid.UUID` ]
1395
- UUIDs or internal integer IDs of the metadata datasets themselves
1396
- or of the quanta they correspond to.
1324
+ UUIDs of the metadata datasets themselves or of the quanta they
1325
+ correspond to.
1397
1326
 
1398
1327
  Returns
1399
1328
  -------
1400
- metadata : `dict` [ `uuid.UUID` or `int`, `list` [`.TaskMetadata`] ]
1329
+ metadata : `dict` [ `uuid.UUID`, `list` [`.TaskMetadata`] ]
1401
1330
  Metadata for the given IDs. Each value is a list of
1402
1331
  `.TaskMetadata` instances representing different execution
1403
1332
  attempts, ordered chronologically from first to last. Attempts
1404
1333
  where metadata was missing (not written even in the fallback extra
1405
1334
  provenance in the logs) will have `None` in this list.
1406
1335
  """
1407
- result: dict[uuid.UUID | DatasetIndex | QuantumIndex, list[TaskMetadata | None]] = {}
1336
+ result: dict[uuid.UUID, list[TaskMetadata | None]] = {}
1408
1337
  with MultiblockReader.open_in_zip(
1409
1338
  self.zf, METADATA_MB_NAME, int_size=self.header.int_size
1410
1339
  ) as mb_reader:
@@ -60,7 +60,7 @@ from lsst.utils.logging import VERBOSE, LsstLogAdapter
60
60
 
61
61
  from ._config import AggregatorConfig
62
62
  from ._progress import ProgressManager, make_worker_log
63
- from ._structs import IngestRequest, ScanReport, ScanResult
63
+ from ._structs import IngestRequest, ScanReport, WriteRequest
64
64
 
65
65
  _T = TypeVar("_T")
66
66
 
@@ -361,7 +361,7 @@ class SupervisorCommunicator:
361
361
  # scanner and the supervisor send one sentinal when done, and the
362
362
  # writer waits for (n_scanners + 1) sentinals to arrive before it
363
363
  # starts its shutdown.
364
- self._write_requests: Queue[ScanResult | Literal[_Sentinel.NO_MORE_WRITE_REQUESTS]] | None = (
364
+ self._write_requests: Queue[WriteRequest | Literal[_Sentinel.NO_MORE_WRITE_REQUESTS]] | None = (
365
365
  context.make_queue() if config.output_path is not None else None
366
366
  )
367
367
  # All other workers use this queue to send many different kinds of
@@ -461,17 +461,17 @@ class SupervisorCommunicator:
461
461
  """
462
462
  self._scan_requests.put(_ScanRequest(quantum_id), block=False)
463
463
 
464
- def request_write(self, scan_result: ScanResult) -> None:
464
+ def request_write(self, request: WriteRequest) -> None:
465
465
  """Send a request to the writer to write provenance for the given scan.
466
466
 
467
467
  Parameters
468
468
  ----------
469
- scan_result : `ScanResult`
469
+ request : `WriteRequest`
470
470
  Information from scanning a quantum (or knowing you don't have to,
471
471
  in the case of blocked quanta).
472
472
  """
473
473
  assert self._write_requests is not None, "Writer should not be used if writing is disabled."
474
- self._write_requests.put(scan_result, block=False)
474
+ self._write_requests.put(request, block=False)
475
475
 
476
476
  def poll(self) -> Iterator[ScanReport]:
477
477
  """Poll for reports from workers while sending scan requests.
@@ -728,16 +728,16 @@ class ScannerCommunicator(WorkerCommunicator):
728
728
  else:
729
729
  self._reports.put(_IngestReport(1), block=False)
730
730
 
731
- def request_write(self, scan_result: ScanResult) -> None:
731
+ def request_write(self, request: WriteRequest) -> None:
732
732
  """Ask the writer to write provenance for a quantum.
733
733
 
734
734
  Parameters
735
735
  ----------
736
- scan_result : `ScanResult`
736
+ request : `WriteRequest`
737
737
  Result of scanning a quantum.
738
738
  """
739
739
  assert self._write_requests is not None, "Writer should not be used if writing is disabled."
740
- self._write_requests.put(scan_result, block=False)
740
+ self._write_requests.put(request, block=False)
741
741
 
742
742
  def get_compression_dict(self) -> bytes | None:
743
743
  """Attempt to get the compression dict from the writer.
@@ -913,12 +913,12 @@ class WriterCommunicator(WorkerCommunicator):
913
913
  self._reports.put(_Sentinel.WRITER_DONE, block=False)
914
914
  return result
915
915
 
916
- def poll(self) -> Iterator[ScanResult]:
916
+ def poll(self) -> Iterator[WriteRequest]:
917
917
  """Poll for writer requests from the scanner workers and supervisor.
918
918
 
919
919
  Yields
920
920
  ------
921
- request : `ScanResult`
921
+ request : `WriteRequest`
922
922
  The result of a quantum scan.
923
923
 
924
924
  Notes