lsst-pipe-base 30.0.0rc3__py3-none-any.whl → 30.0.1rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/pipe/base/_instrument.py +25 -15
- lsst/pipe/base/_quantumContext.py +3 -3
- lsst/pipe/base/_status.py +43 -10
- lsst/pipe/base/_task_metadata.py +2 -2
- lsst/pipe/base/all_dimensions_quantum_graph_builder.py +8 -3
- lsst/pipe/base/automatic_connection_constants.py +20 -1
- lsst/pipe/base/cli/cmd/__init__.py +18 -2
- lsst/pipe/base/cli/cmd/commands.py +149 -4
- lsst/pipe/base/connectionTypes.py +72 -160
- lsst/pipe/base/connections.py +6 -9
- lsst/pipe/base/execution_reports.py +0 -5
- lsst/pipe/base/graph/graph.py +11 -10
- lsst/pipe/base/graph/quantumNode.py +4 -4
- lsst/pipe/base/graph_walker.py +8 -10
- lsst/pipe/base/log_capture.py +1 -1
- lsst/pipe/base/log_on_close.py +4 -7
- lsst/pipe/base/pipeline.py +5 -6
- lsst/pipe/base/pipelineIR.py +2 -8
- lsst/pipe/base/pipelineTask.py +5 -7
- lsst/pipe/base/pipeline_graph/_dataset_types.py +2 -2
- lsst/pipe/base/pipeline_graph/_edges.py +32 -22
- lsst/pipe/base/pipeline_graph/_mapping_views.py +4 -7
- lsst/pipe/base/pipeline_graph/_pipeline_graph.py +14 -7
- lsst/pipe/base/pipeline_graph/expressions.py +2 -2
- lsst/pipe/base/pipeline_graph/io.py +7 -10
- lsst/pipe/base/pipeline_graph/visualization/_dot.py +13 -12
- lsst/pipe/base/pipeline_graph/visualization/_layout.py +16 -18
- lsst/pipe/base/pipeline_graph/visualization/_merge.py +4 -7
- lsst/pipe/base/pipeline_graph/visualization/_printer.py +10 -10
- lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py +7 -0
- lsst/pipe/base/prerequisite_helpers.py +2 -1
- lsst/pipe/base/quantum_graph/_common.py +15 -17
- lsst/pipe/base/quantum_graph/_multiblock.py +36 -20
- lsst/pipe/base/quantum_graph/_predicted.py +7 -3
- lsst/pipe/base/quantum_graph/_provenance.py +501 -61
- lsst/pipe/base/quantum_graph/aggregator/__init__.py +0 -1
- lsst/pipe/base/quantum_graph/aggregator/_communicators.py +187 -240
- lsst/pipe/base/quantum_graph/aggregator/_config.py +87 -9
- lsst/pipe/base/quantum_graph/aggregator/_ingester.py +13 -12
- lsst/pipe/base/quantum_graph/aggregator/_scanner.py +15 -7
- lsst/pipe/base/quantum_graph/aggregator/_structs.py +3 -3
- lsst/pipe/base/quantum_graph/aggregator/_supervisor.py +19 -34
- lsst/pipe/base/quantum_graph/aggregator/_workers.py +303 -0
- lsst/pipe/base/quantum_graph/aggregator/_writer.py +3 -3
- lsst/pipe/base/quantum_graph/formatter.py +74 -4
- lsst/pipe/base/quantum_graph/ingest_graph.py +413 -0
- lsst/pipe/base/quantum_graph/visualization.py +5 -1
- lsst/pipe/base/quantum_graph_builder.py +21 -8
- lsst/pipe/base/quantum_graph_skeleton.py +31 -29
- lsst/pipe/base/quantum_provenance_graph.py +29 -12
- lsst/pipe/base/separable_pipeline_executor.py +1 -1
- lsst/pipe/base/single_quantum_executor.py +15 -8
- lsst/pipe/base/struct.py +4 -0
- lsst/pipe/base/testUtils.py +3 -3
- lsst/pipe/base/tests/mocks/_storage_class.py +2 -1
- lsst/pipe/base/version.py +1 -1
- {lsst_pipe_base-30.0.0rc3.dist-info → lsst_pipe_base-30.0.1rc1.dist-info}/METADATA +3 -3
- {lsst_pipe_base-30.0.0rc3.dist-info → lsst_pipe_base-30.0.1rc1.dist-info}/RECORD +66 -64
- {lsst_pipe_base-30.0.0rc3.dist-info → lsst_pipe_base-30.0.1rc1.dist-info}/WHEEL +1 -1
- {lsst_pipe_base-30.0.0rc3.dist-info → lsst_pipe_base-30.0.1rc1.dist-info}/entry_points.txt +0 -0
- {lsst_pipe_base-30.0.0rc3.dist-info → lsst_pipe_base-30.0.1rc1.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_pipe_base-30.0.0rc3.dist-info → lsst_pipe_base-30.0.1rc1.dist-info}/licenses/LICENSE +0 -0
- {lsst_pipe_base-30.0.0rc3.dist-info → lsst_pipe_base-30.0.1rc1.dist-info}/licenses/bsd_license.txt +0 -0
- {lsst_pipe_base-30.0.0rc3.dist-info → lsst_pipe_base-30.0.1rc1.dist-info}/licenses/gpl-v3.0.txt +0 -0
- {lsst_pipe_base-30.0.0rc3.dist-info → lsst_pipe_base-30.0.1rc1.dist-info}/top_level.txt +0 -0
- {lsst_pipe_base-30.0.0rc3.dist-info → lsst_pipe_base-30.0.1rc1.dist-info}/zip-safe +0 -0
|
@@ -106,8 +106,8 @@ class DatasetTypeNode:
|
|
|
106
106
|
The internal networkx graph.
|
|
107
107
|
get_registered : `~collections.abc.Callable` or `None`
|
|
108
108
|
Callable that takes a dataset type name and returns the
|
|
109
|
-
|
|
110
|
-
not registered.
|
|
109
|
+
`~lsst.daf.butler.DatasetType` registered in the data repository,
|
|
110
|
+
or `None` if it is not registered.
|
|
111
111
|
dimensions : `lsst.daf.butler.DimensionUniverse`
|
|
112
112
|
Definitions of all dimensions.
|
|
113
113
|
previous : `DatasetTypeNode` or `None`
|
|
@@ -30,7 +30,7 @@ __all__ = ("Edge", "ReadEdge", "WriteEdge")
|
|
|
30
30
|
|
|
31
31
|
from abc import ABC, abstractmethod
|
|
32
32
|
from collections.abc import Callable, Mapping, Sequence
|
|
33
|
-
from typing import Any, ClassVar, Self
|
|
33
|
+
from typing import Any, ClassVar, Self
|
|
34
34
|
|
|
35
35
|
from lsst.daf.butler import DatasetRef, DatasetType, DimensionUniverse, StorageClassFactory
|
|
36
36
|
from lsst.daf.butler.registry import MissingDatasetTypeError
|
|
@@ -40,8 +40,6 @@ from ..connectionTypes import BaseConnection
|
|
|
40
40
|
from ._exceptions import ConnectionTypeConsistencyError, IncompatibleDatasetTypeError
|
|
41
41
|
from ._nodes import NodeKey, NodeType
|
|
42
42
|
|
|
43
|
-
_S = TypeVar("_S", bound="Edge")
|
|
44
|
-
|
|
45
43
|
|
|
46
44
|
@immutable
|
|
47
45
|
class Edge(ABC):
|
|
@@ -172,7 +170,7 @@ class Edge(ABC):
|
|
|
172
170
|
"""
|
|
173
171
|
return self.parent_dataset_type_name
|
|
174
172
|
|
|
175
|
-
def diff(self:
|
|
173
|
+
def diff[S: Edge](self: S, other: S, connection_type: str = "connection") -> list[str]:
|
|
176
174
|
"""Compare this edge to another one from a possibly-different
|
|
177
175
|
configuration of the same task label.
|
|
178
176
|
|
|
@@ -480,11 +478,11 @@ class ReadEdge(Edge):
|
|
|
480
478
|
Parameters
|
|
481
479
|
----------
|
|
482
480
|
current : `lsst.daf.butler.DatasetType` or `None`
|
|
483
|
-
The current graph-wide
|
|
484
|
-
be the registry's definition of the parent dataset
|
|
485
|
-
exists. If not, it will be the dataset type
|
|
486
|
-
task in the graph that writes it, if there is
|
|
487
|
-
such task, this will be `None`.
|
|
481
|
+
The current graph-wide `~lsst.daf.butler.DatasetType`, or `None`.
|
|
482
|
+
This will always be the registry's definition of the parent dataset
|
|
483
|
+
type, if one exists. If not, it will be the dataset type
|
|
484
|
+
definition from the task in the graph that writes it, if there is
|
|
485
|
+
one. If there is no such task, this will be `None`.
|
|
488
486
|
is_initial_query_constraint : `bool`
|
|
489
487
|
Whether this dataset type is currently marked as a constraint on
|
|
490
488
|
the initial data ID query in QuantumGraph generation.
|
|
@@ -496,7 +494,7 @@ class ReadEdge(Edge):
|
|
|
496
494
|
producer : `str` or `None`
|
|
497
495
|
The label of the task that produces this dataset type in the
|
|
498
496
|
pipeline, or `None` if it is an overall input.
|
|
499
|
-
consumers :
|
|
497
|
+
consumers : `~collections.abc.Sequence` [ `str` ]
|
|
500
498
|
Labels for other consuming tasks that have already participated in
|
|
501
499
|
this dataset type's resolution.
|
|
502
500
|
is_registered : `bool`
|
|
@@ -512,7 +510,7 @@ class ReadEdge(Edge):
|
|
|
512
510
|
|
|
513
511
|
Returns
|
|
514
512
|
-------
|
|
515
|
-
dataset_type :
|
|
513
|
+
dataset_type : `~lsst.daf.butler.DatasetType`
|
|
516
514
|
The updated graph-wide dataset type. If ``current`` was provided,
|
|
517
515
|
this must be equal to it.
|
|
518
516
|
is_initial_query_constraint : `bool`
|
|
@@ -659,13 +657,25 @@ class ReadEdge(Edge):
|
|
|
659
657
|
# compatible), since neither connection should take
|
|
660
658
|
# precedence.
|
|
661
659
|
if dataset_type != current:
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
"
|
|
667
|
-
|
|
668
|
-
|
|
660
|
+
if visualization_only and dataset_type.dimensions == current.dimensions:
|
|
661
|
+
# Make a visualization-only ambiguous storage class
|
|
662
|
+
# "name".
|
|
663
|
+
all_storage_classes = set(current.storageClass_name.split("/"))
|
|
664
|
+
all_storage_classes.update(dataset_type.storageClass_name.split("/"))
|
|
665
|
+
current = DatasetType(
|
|
666
|
+
current.name,
|
|
667
|
+
current.dimensions,
|
|
668
|
+
"/".join(sorted(all_storage_classes)),
|
|
669
|
+
)
|
|
670
|
+
else:
|
|
671
|
+
raise MissingDatasetTypeError(
|
|
672
|
+
f"Definitions differ for input dataset type "
|
|
673
|
+
f"{self.parent_dataset_type_name!r}; task {self.task_label!r} has "
|
|
674
|
+
f"{dataset_type}, but the definition from {report_current_origin()} is "
|
|
675
|
+
f"{current}. If the storage classes are compatible but different, "
|
|
676
|
+
"registering the dataset type in the data repository in advance will avoid "
|
|
677
|
+
"this error."
|
|
678
|
+
)
|
|
669
679
|
elif not visualization_only and not dataset_type.is_compatible_with(current):
|
|
670
680
|
raise IncompatibleDatasetTypeError(
|
|
671
681
|
f"Incompatible definition for input dataset type {self.parent_dataset_type_name!r}; "
|
|
@@ -788,15 +798,15 @@ class WriteEdge(Edge):
|
|
|
788
798
|
Parameters
|
|
789
799
|
----------
|
|
790
800
|
current : `lsst.daf.butler.DatasetType` or `None`
|
|
791
|
-
The current graph-wide
|
|
792
|
-
be the registry's definition of the parent dataset
|
|
793
|
-
exists.
|
|
801
|
+
The current graph-wide `~lsst.daf.butler.DatasetType`, or `None`.
|
|
802
|
+
This will always be the registry's definition of the parent dataset
|
|
803
|
+
type, if one exists.
|
|
794
804
|
universe : `lsst.daf.butler.DimensionUniverse`
|
|
795
805
|
Object that holds all dimension definitions.
|
|
796
806
|
|
|
797
807
|
Returns
|
|
798
808
|
-------
|
|
799
|
-
dataset_type :
|
|
809
|
+
dataset_type : `~lsst.daf.butler.DatasetType`
|
|
800
810
|
A dataset type compatible with this edge. If ``current`` was
|
|
801
811
|
provided, this must be equal to it.
|
|
802
812
|
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
from __future__ import annotations
|
|
28
28
|
|
|
29
29
|
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
|
30
|
-
from typing import Any, ClassVar,
|
|
30
|
+
from typing import Any, ClassVar, cast, overload
|
|
31
31
|
|
|
32
32
|
import networkx
|
|
33
33
|
|
|
@@ -36,11 +36,8 @@ from ._exceptions import UnresolvedGraphError
|
|
|
36
36
|
from ._nodes import NodeKey, NodeType
|
|
37
37
|
from ._tasks import TaskInitNode, TaskNode
|
|
38
38
|
|
|
39
|
-
_N = TypeVar("_N", covariant=True)
|
|
40
|
-
_T = TypeVar("_T")
|
|
41
39
|
|
|
42
|
-
|
|
43
|
-
class MappingView(Mapping[str, _N]):
|
|
40
|
+
class MappingView[N](Mapping[str, N]):
|
|
44
41
|
"""Base class for mapping views into nodes of certain types in a
|
|
45
42
|
`PipelineGraph`.
|
|
46
43
|
|
|
@@ -74,7 +71,7 @@ class MappingView(Mapping[str, _N]):
|
|
|
74
71
|
self._keys = self._make_keys(self._parent_xgraph)
|
|
75
72
|
return iter(self._keys)
|
|
76
73
|
|
|
77
|
-
def __getitem__(self, key: str) ->
|
|
74
|
+
def __getitem__(self, key: str) -> N:
|
|
78
75
|
return self._parent_xgraph.nodes[NodeKey(self._NODE_TYPE, key)]["instance"]
|
|
79
76
|
|
|
80
77
|
def __len__(self) -> int:
|
|
@@ -230,7 +227,7 @@ class DatasetTypeMappingView(MappingView[DatasetTypeNode]):
|
|
|
230
227
|
def get_if_resolved(self, key: str) -> DatasetTypeNode | None: ... # pragma: nocover
|
|
231
228
|
|
|
232
229
|
@overload
|
|
233
|
-
def get_if_resolved(self, key: str, default:
|
|
230
|
+
def get_if_resolved[T](self, key: str, default: T) -> DatasetTypeNode | T: ... # pragma: nocover
|
|
234
231
|
|
|
235
232
|
def get_if_resolved(self, key: str, default: Any = None) -> DatasetTypeNode | Any:
|
|
236
233
|
"""Get a node or return a default if it has not been resolved.
|
|
@@ -33,7 +33,7 @@ import itertools
|
|
|
33
33
|
import json
|
|
34
34
|
import logging
|
|
35
35
|
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set
|
|
36
|
-
from typing import TYPE_CHECKING, Any, BinaryIO, Literal,
|
|
36
|
+
from typing import TYPE_CHECKING, Any, BinaryIO, Literal, cast
|
|
37
37
|
|
|
38
38
|
import networkx
|
|
39
39
|
import networkx.algorithms.bipartite
|
|
@@ -79,9 +79,6 @@ if TYPE_CHECKING:
|
|
|
79
79
|
from ..pipeline import TaskDef
|
|
80
80
|
from ..pipelineTask import PipelineTask
|
|
81
81
|
|
|
82
|
-
|
|
83
|
-
_G = TypeVar("_G", bound=networkx.DiGraph | networkx.MultiDiGraph)
|
|
84
|
-
|
|
85
82
|
_LOG = logging.getLogger("lsst.pipe.base.pipeline_graph")
|
|
86
83
|
|
|
87
84
|
|
|
@@ -897,6 +894,10 @@ class PipelineGraph:
|
|
|
897
894
|
New config objects or overrides to apply to copies of the current
|
|
898
895
|
config objects, with task labels as the keywords.
|
|
899
896
|
|
|
897
|
+
Returns
|
|
898
|
+
-------
|
|
899
|
+
None
|
|
900
|
+
|
|
900
901
|
Raises
|
|
901
902
|
------
|
|
902
903
|
ValueError
|
|
@@ -1632,7 +1633,7 @@ class PipelineGraph:
|
|
|
1632
1633
|
|
|
1633
1634
|
Returns
|
|
1634
1635
|
-------
|
|
1635
|
-
subgraphs :
|
|
1636
|
+
subgraphs : `~collections.abc.Iterable` [ `PipelineGraph` ]
|
|
1636
1637
|
An iterable over component subgraphs that could be run
|
|
1637
1638
|
independently (they have only overall inputs in common). May be a
|
|
1638
1639
|
lazy iterator.
|
|
@@ -1755,6 +1756,10 @@ class PipelineGraph:
|
|
|
1755
1756
|
not considered part of the pipeline graph in other respects, but it
|
|
1756
1757
|
does get written with other provenance datasets).
|
|
1757
1758
|
|
|
1759
|
+
Returns
|
|
1760
|
+
-------
|
|
1761
|
+
None
|
|
1762
|
+
|
|
1758
1763
|
Raises
|
|
1759
1764
|
------
|
|
1760
1765
|
lsst.daf.butler.MissingDatasetTypeError
|
|
@@ -2179,7 +2184,9 @@ class PipelineGraph:
|
|
|
2179
2184
|
]
|
|
2180
2185
|
return networkx.algorithms.bipartite.projected_graph(networkx.DiGraph(bipartite_xgraph), task_keys)
|
|
2181
2186
|
|
|
2182
|
-
def _transform_xgraph_state
|
|
2187
|
+
def _transform_xgraph_state[G: networkx.DiGraph | networkx.MultiDiGraph](
|
|
2188
|
+
self, xgraph: G, skip_edges: bool
|
|
2189
|
+
) -> G:
|
|
2183
2190
|
"""Transform networkx graph attributes in-place from the internal
|
|
2184
2191
|
"instance" attributes to the documented exported attributes.
|
|
2185
2192
|
|
|
@@ -2228,7 +2235,7 @@ class PipelineGraph:
|
|
|
2228
2235
|
|
|
2229
2236
|
Parameters
|
|
2230
2237
|
----------
|
|
2231
|
-
updates :
|
|
2238
|
+
updates : `~collections.abc.Mapping` [ `str`, `TaskNode` ]
|
|
2232
2239
|
New task nodes with task label keys. All keys must be task labels
|
|
2233
2240
|
that are already present in the graph.
|
|
2234
2241
|
check_edges_unchanged : `bool`, optional
|
|
@@ -43,7 +43,7 @@ __all__ = (
|
|
|
43
43
|
|
|
44
44
|
import dataclasses
|
|
45
45
|
import functools
|
|
46
|
-
from typing import TYPE_CHECKING, Any, Literal
|
|
46
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
47
47
|
|
|
48
48
|
from lsst.daf.butler.queries.expressions.parser.ply import lex, yacc
|
|
49
49
|
|
|
@@ -268,4 +268,4 @@ def parse(expression: str) -> Node:
|
|
|
268
268
|
return _ParserYacc().parse(expression)
|
|
269
269
|
|
|
270
270
|
|
|
271
|
-
Node
|
|
271
|
+
type Node = IdentifierNode | DirectionNode | NotNode | UnionNode | IntersectionNode
|
|
@@ -33,11 +33,10 @@ __all__ = (
|
|
|
33
33
|
"SerializedTaskInitNode",
|
|
34
34
|
"SerializedTaskNode",
|
|
35
35
|
"SerializedTaskSubset",
|
|
36
|
-
"expect_not_none",
|
|
37
36
|
)
|
|
38
37
|
|
|
39
38
|
from collections.abc import Mapping
|
|
40
|
-
from typing import Any
|
|
39
|
+
from typing import Any
|
|
41
40
|
|
|
42
41
|
import networkx
|
|
43
42
|
import pydantic
|
|
@@ -53,14 +52,12 @@ from ._pipeline_graph import PipelineGraph
|
|
|
53
52
|
from ._task_subsets import StepDefinitions, TaskSubset
|
|
54
53
|
from ._tasks import TaskImportMode, TaskInitNode, TaskNode
|
|
55
54
|
|
|
56
|
-
_U = TypeVar("_U")
|
|
57
|
-
|
|
58
55
|
_IO_VERSION_INFO = (0, 0, 1)
|
|
59
56
|
"""Version tuple embedded in saved PipelineGraphs.
|
|
60
57
|
"""
|
|
61
58
|
|
|
62
59
|
|
|
63
|
-
def
|
|
60
|
+
def _expect_not_none[U](value: U | None, msg: str) -> U:
|
|
64
61
|
"""Check that a value is not `None` and return it.
|
|
65
62
|
|
|
66
63
|
Parameters
|
|
@@ -418,7 +415,7 @@ class SerializedTaskNode(pydantic.BaseModel):
|
|
|
418
415
|
init = self.init.deserialize(
|
|
419
416
|
init_key,
|
|
420
417
|
task_class_name=self.task_class,
|
|
421
|
-
config_str=
|
|
418
|
+
config_str=_expect_not_none(
|
|
422
419
|
self.config_str, f"No serialized config file for task with label {key.name!r}."
|
|
423
420
|
),
|
|
424
421
|
dataset_type_keys=dataset_type_keys,
|
|
@@ -547,16 +544,16 @@ class SerializedDatasetTypeNode(pydantic.BaseModel):
|
|
|
547
544
|
if self.dimensions is not None:
|
|
548
545
|
dataset_type = DatasetType(
|
|
549
546
|
key.name,
|
|
550
|
-
|
|
547
|
+
_expect_not_none(
|
|
551
548
|
self.dimensions,
|
|
552
549
|
f"Serialized dataset type {key.name!r} has no dimensions.",
|
|
553
550
|
),
|
|
554
|
-
storageClass=
|
|
551
|
+
storageClass=_expect_not_none(
|
|
555
552
|
self.storage_class,
|
|
556
553
|
f"Serialized dataset type {key.name!r} has no storage class.",
|
|
557
554
|
),
|
|
558
555
|
isCalibration=self.is_calibration,
|
|
559
|
-
universe=
|
|
556
|
+
universe=_expect_not_none(
|
|
560
557
|
universe,
|
|
561
558
|
f"Serialized dataset type {key.name!r} has dimensions, "
|
|
562
559
|
"but no dimension universe was stored.",
|
|
@@ -747,7 +744,7 @@ class SerializedPipelineGraph(pydantic.BaseModel):
|
|
|
747
744
|
if self.dimensions is not None:
|
|
748
745
|
universe = DimensionUniverse(
|
|
749
746
|
config=DimensionConfig(
|
|
750
|
-
|
|
747
|
+
_expect_not_none(
|
|
751
748
|
self.dimensions,
|
|
752
749
|
"Serialized pipeline graph has not been resolved; "
|
|
753
750
|
"load it is a MutablePipelineGraph instead.",
|
|
@@ -66,7 +66,7 @@ def show_dot(
|
|
|
66
66
|
----------
|
|
67
67
|
pipeline_graph : `PipelineGraph`
|
|
68
68
|
Pipeline graph to show.
|
|
69
|
-
stream : `TextIO`, optional
|
|
69
|
+
stream : `io.TextIO`, optional
|
|
70
70
|
Stream to write the DOT representation to.
|
|
71
71
|
label_edge_connections : `bool`, optional
|
|
72
72
|
If `True`, label edges with their connection names.
|
|
@@ -167,21 +167,22 @@ def _render_dataset_type_node(
|
|
|
167
167
|
|
|
168
168
|
Parameters
|
|
169
169
|
----------
|
|
170
|
-
node_key : NodeKey
|
|
171
|
-
The key for the node
|
|
172
|
-
node_data : Mapping[str
|
|
173
|
-
The data associated with the node
|
|
174
|
-
options : NodeAttributeOptions
|
|
175
|
-
Options for rendering the node
|
|
176
|
-
stream : TextIO
|
|
177
|
-
The stream to write the node to
|
|
170
|
+
node_key : `NodeKey`
|
|
171
|
+
The key for the node.
|
|
172
|
+
node_data : `~collections.abc.Mapping` [`str`, `typing.Any`]
|
|
173
|
+
The data associated with the node.
|
|
174
|
+
options : `NodeAttributeOptions`
|
|
175
|
+
Options for rendering the node.
|
|
176
|
+
stream : `io.TextIO`
|
|
177
|
+
The stream to write the node to.
|
|
178
|
+
overflow_ref : `int`, optional
|
|
178
179
|
|
|
179
180
|
Returns
|
|
180
181
|
-------
|
|
181
182
|
overflow_ref : int
|
|
182
|
-
The reference number for the next overflow node
|
|
183
|
+
The reference number for the next overflow node.
|
|
183
184
|
overflow_ids : str | None
|
|
184
|
-
The ID of the overflow node, if any
|
|
185
|
+
The ID of the overflow node, if any.
|
|
185
186
|
"""
|
|
186
187
|
labels, label_extras, common_prefix = _format_label(str(node_key), _LABEL_MAX_LINES_SOFT)
|
|
187
188
|
if len(labels) + len(label_extras) <= _LABEL_MAX_LINES_HARD:
|
|
@@ -271,7 +272,7 @@ def _render_edge(from_node_id: str, to_node_id: str, stream: TextIO, **kwargs: A
|
|
|
271
272
|
The unique ID of the node the edge is going to
|
|
272
273
|
stream : TextIO
|
|
273
274
|
The stream to write the edge to
|
|
274
|
-
kwargs : Any
|
|
275
|
+
**kwargs : Any
|
|
275
276
|
Additional keyword arguments to pass to the edge
|
|
276
277
|
"""
|
|
277
278
|
if kwargs:
|
|
@@ -30,7 +30,7 @@ __all__ = ("ColumnSelector", "Layout", "LayoutRow")
|
|
|
30
30
|
|
|
31
31
|
import dataclasses
|
|
32
32
|
from collections.abc import Iterable, Iterator, Mapping, Set
|
|
33
|
-
from typing import
|
|
33
|
+
from typing import TextIO
|
|
34
34
|
|
|
35
35
|
import networkx
|
|
36
36
|
import networkx.algorithms.components
|
|
@@ -38,10 +38,8 @@ import networkx.algorithms.dag
|
|
|
38
38
|
import networkx.algorithms.shortest_paths
|
|
39
39
|
import networkx.algorithms.traversal
|
|
40
40
|
|
|
41
|
-
_K = TypeVar("_K")
|
|
42
41
|
|
|
43
|
-
|
|
44
|
-
class Layout(Generic[_K]):
|
|
42
|
+
class Layout[K]:
|
|
45
43
|
"""A class that positions nodes and edges in text-art graph visualizations.
|
|
46
44
|
|
|
47
45
|
Parameters
|
|
@@ -73,9 +71,9 @@ class Layout(Generic[_K]):
|
|
|
73
71
|
# to be close to that text when possible (or maybe it's historical, and
|
|
74
72
|
# it's just a lot of work to re-invert the algorithm now that it's
|
|
75
73
|
# written).
|
|
76
|
-
self._active_columns: dict[int, set[
|
|
74
|
+
self._active_columns: dict[int, set[K]] = {}
|
|
77
75
|
# Mapping from node key to its column.
|
|
78
|
-
self._locations: dict[
|
|
76
|
+
self._locations: dict[K, int] = {}
|
|
79
77
|
# Minimum and maximum column (may go negative; will be shifted as
|
|
80
78
|
# needed before actual display).
|
|
81
79
|
self._x_min = 0
|
|
@@ -116,7 +114,7 @@ class Layout(Generic[_K]):
|
|
|
116
114
|
for component_xgraph, component_order in component_xgraphs_and_orders:
|
|
117
115
|
self._add_connected_graph(component_xgraph, component_order)
|
|
118
116
|
|
|
119
|
-
def _add_single_node(self, node:
|
|
117
|
+
def _add_single_node(self, node: K) -> None:
|
|
120
118
|
"""Add a single node to the layout."""
|
|
121
119
|
assert node not in self._locations
|
|
122
120
|
if not self._locations:
|
|
@@ -184,7 +182,7 @@ class Layout(Generic[_K]):
|
|
|
184
182
|
return x + 1
|
|
185
183
|
|
|
186
184
|
def _add_connected_graph(
|
|
187
|
-
self, xgraph: networkx.DiGraph | networkx.MultiDiGraph, order: list[
|
|
185
|
+
self, xgraph: networkx.DiGraph | networkx.MultiDiGraph, order: list[K] | None = None
|
|
188
186
|
) -> None:
|
|
189
187
|
"""Add a subgraph whose nodes are connected.
|
|
190
188
|
|
|
@@ -202,7 +200,7 @@ class Layout(Generic[_K]):
|
|
|
202
200
|
# "backbone" of our layout; we'll step through this path and add
|
|
203
201
|
# recurse via calls to `_add_graph` on the nodes that we think should
|
|
204
202
|
# go between the backbone nodes.
|
|
205
|
-
backbone: list[
|
|
203
|
+
backbone: list[K] = networkx.algorithms.dag.dag_longest_path(xgraph, topo_order=order)
|
|
206
204
|
# Add the first backbone node and any ancestors according to the full
|
|
207
205
|
# graph (it can't have ancestors in this _subgraph_ because they'd have
|
|
208
206
|
# been part of the longest path themselves, but the subgraph doesn't
|
|
@@ -237,7 +235,7 @@ class Layout(Generic[_K]):
|
|
|
237
235
|
remaining.remove_nodes_from(self._locations.keys())
|
|
238
236
|
self._add_graph(remaining)
|
|
239
237
|
|
|
240
|
-
def _add_blockers_of(self, node:
|
|
238
|
+
def _add_blockers_of(self, node: K) -> None:
|
|
241
239
|
"""Add all nodes that are ancestors of the given node according to the
|
|
242
240
|
full graph.
|
|
243
241
|
"""
|
|
@@ -251,7 +249,7 @@ class Layout(Generic[_K]):
|
|
|
251
249
|
return (self._x_max - self._x_min) // 2
|
|
252
250
|
|
|
253
251
|
@property
|
|
254
|
-
def nodes(self) -> Iterable[
|
|
252
|
+
def nodes(self) -> Iterable[K]:
|
|
255
253
|
"""The graph nodes in the order they appear in the layout."""
|
|
256
254
|
return self._locations.keys()
|
|
257
255
|
|
|
@@ -277,7 +275,7 @@ class Layout(Generic[_K]):
|
|
|
277
275
|
return (self._x_max - x) // 2
|
|
278
276
|
|
|
279
277
|
def __iter__(self) -> Iterator[LayoutRow]:
|
|
280
|
-
active_edges: dict[
|
|
278
|
+
active_edges: dict[K, set[K]] = {}
|
|
281
279
|
for node, node_x in self._locations.items():
|
|
282
280
|
row = LayoutRow(node, self._external_location(node_x))
|
|
283
281
|
for origin, destinations in active_edges.items():
|
|
@@ -295,20 +293,20 @@ class Layout(Generic[_K]):
|
|
|
295
293
|
|
|
296
294
|
|
|
297
295
|
@dataclasses.dataclass
|
|
298
|
-
class LayoutRow
|
|
296
|
+
class LayoutRow[K]:
|
|
299
297
|
"""Information about a single text-art row in a graph."""
|
|
300
298
|
|
|
301
|
-
node:
|
|
299
|
+
node: K
|
|
302
300
|
"""Key for the node in the exported NetworkX graph."""
|
|
303
301
|
|
|
304
302
|
x: int
|
|
305
303
|
"""Column of the node's symbol and its outgoing edges."""
|
|
306
304
|
|
|
307
|
-
connecting: list[tuple[int,
|
|
305
|
+
connecting: list[tuple[int, K]] = dataclasses.field(default_factory=list)
|
|
308
306
|
"""The columns and node keys of edges that terminate at this row.
|
|
309
307
|
"""
|
|
310
308
|
|
|
311
|
-
continuing: list[tuple[int,
|
|
309
|
+
continuing: list[tuple[int, K, frozenset[K]]] = dataclasses.field(default_factory=list)
|
|
312
310
|
"""The columns and node keys of edges that continue through this row.
|
|
313
311
|
"""
|
|
314
312
|
|
|
@@ -337,11 +335,11 @@ class ColumnSelector:
|
|
|
337
335
|
out in that case because it's applied to all candidate columns.
|
|
338
336
|
"""
|
|
339
337
|
|
|
340
|
-
def __call__(
|
|
338
|
+
def __call__[K](
|
|
341
339
|
self,
|
|
342
340
|
connecting_x: list[int],
|
|
343
341
|
node_x: int,
|
|
344
|
-
active_columns: Mapping[int, Set[
|
|
342
|
+
active_columns: Mapping[int, Set[K]],
|
|
345
343
|
x_min: int,
|
|
346
344
|
x_max: int,
|
|
347
345
|
) -> int:
|
|
@@ -38,7 +38,7 @@ import hashlib
|
|
|
38
38
|
from collections import defaultdict
|
|
39
39
|
from collections.abc import Iterable
|
|
40
40
|
from functools import cached_property
|
|
41
|
-
from typing import Any
|
|
41
|
+
from typing import Any
|
|
42
42
|
|
|
43
43
|
import networkx
|
|
44
44
|
import networkx.algorithms.dag
|
|
@@ -49,9 +49,6 @@ from lsst.daf.butler import DimensionGroup
|
|
|
49
49
|
from .._nodes import NodeKey, NodeType
|
|
50
50
|
from ._options import NodeAttributeOptions
|
|
51
51
|
|
|
52
|
-
_P = TypeVar("_P")
|
|
53
|
-
_C = TypeVar("_C")
|
|
54
|
-
|
|
55
52
|
|
|
56
53
|
class MergedNodeKey(frozenset[NodeKey]):
|
|
57
54
|
"""A key for NetworkX graph nodes that represent multiple similar tasks
|
|
@@ -225,11 +222,11 @@ class _MergeKey:
|
|
|
225
222
|
"""
|
|
226
223
|
|
|
227
224
|
@classmethod
|
|
228
|
-
def from_node_state(
|
|
225
|
+
def from_node_state[P, C](
|
|
229
226
|
cls,
|
|
230
227
|
state: dict[str, Any],
|
|
231
|
-
parents: Iterable[
|
|
232
|
-
children: Iterable[
|
|
228
|
+
parents: Iterable[P],
|
|
229
|
+
children: Iterable[C],
|
|
233
230
|
options: NodeAttributeOptions,
|
|
234
231
|
) -> _MergeKey:
|
|
235
232
|
"""Construct from a NetworkX node attribute state dictionary.
|
|
@@ -30,9 +30,9 @@ __all__ = ("Printer", "make_colorama_printer", "make_default_printer", "make_sim
|
|
|
30
30
|
|
|
31
31
|
import sys
|
|
32
32
|
from collections.abc import Callable, Sequence
|
|
33
|
-
from typing import
|
|
33
|
+
from typing import TextIO
|
|
34
34
|
|
|
35
|
-
from ._layout import
|
|
35
|
+
from ._layout import Layout, LayoutRow
|
|
36
36
|
|
|
37
37
|
_CHAR_DECOMPOSITION = {
|
|
38
38
|
# This mapping provides the "logic" for how to decompose the relevant
|
|
@@ -170,7 +170,7 @@ class PrintRow:
|
|
|
170
170
|
return "".join(self._cells)
|
|
171
171
|
|
|
172
172
|
|
|
173
|
-
def _default_get_text(node:
|
|
173
|
+
def _default_get_text[K](node: K, x: int, style: tuple[str, str]) -> str:
|
|
174
174
|
"""Return the default text to associate with a node.
|
|
175
175
|
|
|
176
176
|
This function is the default value for the ``get_text`` argument to
|
|
@@ -179,7 +179,7 @@ def _default_get_text(node: _K, x: int, style: tuple[str, str]) -> str:
|
|
|
179
179
|
return str(node)
|
|
180
180
|
|
|
181
181
|
|
|
182
|
-
def _default_get_symbol(node:
|
|
182
|
+
def _default_get_symbol[K](node: K, x: int) -> str:
|
|
183
183
|
"""Return the default symbol for a node.
|
|
184
184
|
|
|
185
185
|
This function is the default value for the ``get_symbol`` argument to
|
|
@@ -188,7 +188,7 @@ def _default_get_symbol(node: _K, x: int) -> str:
|
|
|
188
188
|
return "⬤"
|
|
189
189
|
|
|
190
190
|
|
|
191
|
-
def _default_get_style(node:
|
|
191
|
+
def _default_get_style[K](node: K, x: int) -> tuple[str, str]:
|
|
192
192
|
"""Get the default styling suffix/prefix strings.
|
|
193
193
|
|
|
194
194
|
This function is the default value for the ``get_style`` argument to
|
|
@@ -197,7 +197,7 @@ def _default_get_style(node: _K, x: int) -> tuple[str, str]:
|
|
|
197
197
|
return "", ""
|
|
198
198
|
|
|
199
199
|
|
|
200
|
-
class Printer
|
|
200
|
+
class Printer[K]:
|
|
201
201
|
"""High-level tool for drawing a text-based DAG visualization.
|
|
202
202
|
|
|
203
203
|
Parameters
|
|
@@ -231,9 +231,9 @@ class Printer(Generic[_K]):
|
|
|
231
231
|
*,
|
|
232
232
|
pad: str = " ",
|
|
233
233
|
make_blank_row: Callable[[int, str], PrintRow] = PrintRow,
|
|
234
|
-
get_text: Callable[[
|
|
235
|
-
get_symbol: Callable[[
|
|
236
|
-
get_style: Callable[[
|
|
234
|
+
get_text: Callable[[K, int, tuple[str, str]], str] = _default_get_text,
|
|
235
|
+
get_symbol: Callable[[K, int], str] = _default_get_symbol,
|
|
236
|
+
get_style: Callable[[K, int], tuple[str, str]] = _default_get_style,
|
|
237
237
|
):
|
|
238
238
|
self.width = layout_width * 2 + 1
|
|
239
239
|
self.pad = pad
|
|
@@ -245,7 +245,7 @@ class Printer(Generic[_K]):
|
|
|
245
245
|
def print_row(
|
|
246
246
|
self,
|
|
247
247
|
stream: TextIO,
|
|
248
|
-
layout_row: LayoutRow[
|
|
248
|
+
layout_row: LayoutRow[K],
|
|
249
249
|
) -> None:
|
|
250
250
|
"""Print a single row of the DAG visualization to a file-like object.
|
|
251
251
|
|
|
@@ -200,6 +200,13 @@ class QuantumGraphExecutionStatusAnnotator:
|
|
|
200
200
|
"""Annotates a networkx graph with task and dataset status information from
|
|
201
201
|
a quantum graph execution summary, implementing the StatusAnnotator
|
|
202
202
|
protocol to update the graph with status data.
|
|
203
|
+
|
|
204
|
+
Parameters
|
|
205
|
+
----------
|
|
206
|
+
*args : `typing.Any`
|
|
207
|
+
Arbitrary arguments.
|
|
208
|
+
**kwargs : `typing.Any`
|
|
209
|
+
Arbitrary keyword arguments.
|
|
203
210
|
"""
|
|
204
211
|
|
|
205
212
|
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
@@ -252,7 +252,8 @@ class PrerequisiteFinder:
|
|
|
252
252
|
Sequence of collections to search, in order.
|
|
253
253
|
data_id : `lsst.daf.butler.DataCoordinate`
|
|
254
254
|
Data ID for the quantum.
|
|
255
|
-
skypix_bounds :
|
|
255
|
+
skypix_bounds : `~collections.abc.Mapping` \
|
|
256
|
+
[ `str`, `lsst.sphgeom.RangeSet` ]
|
|
256
257
|
The spatial bounds of this quantum in various skypix dimensions.
|
|
257
258
|
Keys are skypix dimension names (a superset of those in
|
|
258
259
|
`dataset_skypix`) and values are sets of integer pixel ID ranges.
|