lsst-pipe-base 30.2026.300__py3-none-any.whl → 30.2026.500__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/pipe/base/_instrument.py +21 -12
- lsst/pipe/base/_quantumContext.py +3 -3
- lsst/pipe/base/_status.py +43 -10
- lsst/pipe/base/automatic_connection_constants.py +20 -1
- lsst/pipe/base/cli/cmd/__init__.py +18 -2
- lsst/pipe/base/cli/cmd/commands.py +149 -4
- lsst/pipe/base/connectionTypes.py +72 -160
- lsst/pipe/base/connections.py +3 -6
- lsst/pipe/base/execution_reports.py +0 -5
- lsst/pipe/base/graph/graph.py +9 -8
- lsst/pipe/base/log_capture.py +1 -1
- lsst/pipe/base/pipeline.py +5 -6
- lsst/pipe/base/pipelineIR.py +1 -7
- lsst/pipe/base/pipelineTask.py +5 -7
- lsst/pipe/base/pipeline_graph/_dataset_types.py +2 -2
- lsst/pipe/base/pipeline_graph/_edges.py +30 -18
- lsst/pipe/base/pipeline_graph/_pipeline_graph.py +10 -2
- lsst/pipe/base/pipeline_graph/visualization/_dot.py +13 -12
- lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py +7 -0
- lsst/pipe/base/prerequisite_helpers.py +2 -1
- lsst/pipe/base/quantum_graph/_common.py +3 -1
- lsst/pipe/base/quantum_graph/_multiblock.py +29 -13
- lsst/pipe/base/quantum_graph/_predicted.py +7 -0
- lsst/pipe/base/quantum_graph/_provenance.py +498 -56
- lsst/pipe/base/quantum_graph/aggregator/__init__.py +0 -1
- lsst/pipe/base/quantum_graph/aggregator/_communicators.py +9 -1
- lsst/pipe/base/quantum_graph/aggregator/_config.py +78 -9
- lsst/pipe/base/quantum_graph/aggregator/_ingester.py +12 -11
- lsst/pipe/base/quantum_graph/aggregator/_scanner.py +14 -6
- lsst/pipe/base/quantum_graph/aggregator/_structs.py +3 -3
- lsst/pipe/base/quantum_graph/aggregator/_supervisor.py +14 -13
- lsst/pipe/base/quantum_graph/aggregator/_writer.py +2 -2
- lsst/pipe/base/quantum_graph/formatter.py +74 -4
- lsst/pipe/base/quantum_graph/ingest_graph.py +413 -0
- lsst/pipe/base/quantum_graph_builder.py +1 -8
- lsst/pipe/base/quantum_graph_skeleton.py +29 -27
- lsst/pipe/base/quantum_provenance_graph.py +29 -12
- lsst/pipe/base/separable_pipeline_executor.py +6 -7
- lsst/pipe/base/single_quantum_executor.py +7 -7
- lsst/pipe/base/struct.py +4 -0
- lsst/pipe/base/tests/mocks/_storage_class.py +2 -1
- lsst/pipe/base/version.py +1 -1
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/METADATA +2 -1
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/RECORD +52 -51
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/WHEEL +1 -1
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/entry_points.txt +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/LICENSE +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/bsd_license.txt +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/gpl-v3.0.txt +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/top_level.txt +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/zip-safe +0 -0
|
@@ -41,35 +41,36 @@ from lsst.utils.introspection import find_outside_stacklevel
|
|
|
41
41
|
|
|
42
42
|
@dataclasses.dataclass(frozen=True)
|
|
43
43
|
class BaseConnection:
|
|
44
|
-
"""Base class used for declaring `PipelineTask` connections.
|
|
45
|
-
|
|
46
|
-
Attributes
|
|
47
|
-
----------
|
|
48
|
-
name : `str`
|
|
49
|
-
The name used to identify the dataset type.
|
|
50
|
-
storageClass : `str`
|
|
51
|
-
The storage class used when (un)/persisting the dataset type.
|
|
52
|
-
multiple : `bool`
|
|
53
|
-
Indicates if this connection should expect to contain multiple objects
|
|
54
|
-
of the given dataset type. Tasks with more than one connection with
|
|
55
|
-
``multiple=True`` with the same dimensions may want to implement
|
|
56
|
-
`.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
|
|
57
|
-
consistent (i.e. zip-iterable) in `PipelineTask.runQuantum()` and
|
|
58
|
-
notify the execution system as early as possible of outputs that will
|
|
59
|
-
not be produced because the corresponding input is missing.
|
|
60
|
-
deprecated : `str`, optional
|
|
61
|
-
A description of why this connection is deprecated, including the
|
|
62
|
-
version after which it may be removed.
|
|
63
|
-
|
|
64
|
-
If not `None`, the string is appended to the docstring for this
|
|
65
|
-
connection and the corresponding config Field.
|
|
66
|
-
"""
|
|
44
|
+
"""Base class used for declaring `PipelineTask` connections."""
|
|
67
45
|
|
|
68
46
|
name: str
|
|
47
|
+
"""The name used to identify the dataset type."""
|
|
48
|
+
|
|
69
49
|
storageClass: str
|
|
50
|
+
"""The storage class used when (un)/persisting the dataset type."""
|
|
51
|
+
|
|
70
52
|
doc: str = ""
|
|
53
|
+
"""Documentation for this connection."""
|
|
54
|
+
|
|
71
55
|
multiple: bool = False
|
|
56
|
+
"""Indicates if this connection should expect to contain multiple objects
|
|
57
|
+
of the given dataset type.
|
|
58
|
+
|
|
59
|
+
Tasks with more than one connection with ``multiple=True`` with the same
|
|
60
|
+
dimensions may want to implement `.PipelineTaskConnections.adjustQuantum`
|
|
61
|
+
to ensure those datasets are consistent (i.e. zip-iterable) in
|
|
62
|
+
`PipelineTask.runQuantum()` and notify the execution system as early as
|
|
63
|
+
possible of outputs that will not be produced because the corresponding
|
|
64
|
+
input is missing.
|
|
65
|
+
"""
|
|
66
|
+
|
|
72
67
|
deprecated: str | None = dataclasses.field(default=None, kw_only=True)
|
|
68
|
+
"""A description of why this connection is deprecated, including the
|
|
69
|
+
version after which it may be removed.
|
|
70
|
+
|
|
71
|
+
If not `None`, the string is appended to the docstring for this
|
|
72
|
+
connection and the corresponding config Field.
|
|
73
|
+
"""
|
|
73
74
|
|
|
74
75
|
_connection_type_set: ClassVar[str]
|
|
75
76
|
_deprecation_context: str = ""
|
|
@@ -110,32 +111,15 @@ class BaseConnection:
|
|
|
110
111
|
class DimensionedConnection(BaseConnection):
|
|
111
112
|
"""Class used for declaring PipelineTask connections that includes
|
|
112
113
|
dimensions.
|
|
113
|
-
|
|
114
|
-
Attributes
|
|
115
|
-
----------
|
|
116
|
-
name : `str`
|
|
117
|
-
The name used to identify the dataset type.
|
|
118
|
-
storageClass : `str`
|
|
119
|
-
The storage class used when (un)/persisting the dataset type.
|
|
120
|
-
multiple : `bool`
|
|
121
|
-
Indicates if this connection should expect to contain multiple objects
|
|
122
|
-
of the given dataset type. Tasks with more than one connection with
|
|
123
|
-
``multiple=True`` with the same dimensions may want to implement
|
|
124
|
-
`.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
|
|
125
|
-
consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
|
|
126
|
-
the execution system as early as possible of outputs that will not be
|
|
127
|
-
produced because the corresponding input is missing.
|
|
128
|
-
dimensions : iterable of `str`
|
|
129
|
-
The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
|
|
130
|
-
to identify the dataset type identified by the specified name.
|
|
131
|
-
isCalibration : `bool`, optional
|
|
132
|
-
`True` if this dataset type may be included in CALIBRATION-type
|
|
133
|
-
collections to associate it with a validity range, `False` (default)
|
|
134
|
-
otherwise.
|
|
135
114
|
"""
|
|
136
115
|
|
|
137
116
|
dimensions: Iterable[str] = ()
|
|
117
|
+
"""The keys of the butler data coordinates for this dataset type."""
|
|
118
|
+
|
|
138
119
|
isCalibration: bool = False
|
|
120
|
+
""" `True` if this dataset type may be included in
|
|
121
|
+
`~lsst.daf.butler.CollectionType.CALIBRATION` collections to associate it
|
|
122
|
+
with a validity range, `False` (default) otherwise."""
|
|
139
123
|
|
|
140
124
|
def __post_init__(self):
|
|
141
125
|
super().__post_init__()
|
|
@@ -151,39 +135,6 @@ class DimensionedConnection(BaseConnection):
|
|
|
151
135
|
class BaseInput(DimensionedConnection):
|
|
152
136
|
"""Class used for declaring PipelineTask input connections.
|
|
153
137
|
|
|
154
|
-
Attributes
|
|
155
|
-
----------
|
|
156
|
-
name : `str`
|
|
157
|
-
The default name used to identify the dataset type.
|
|
158
|
-
storageClass : `str`
|
|
159
|
-
The storage class used when (un)/persisting the dataset type.
|
|
160
|
-
multiple : `bool`
|
|
161
|
-
Indicates if this connection should expect to contain multiple objects
|
|
162
|
-
of the given dataset type. Tasks with more than one connection with
|
|
163
|
-
``multiple=True`` with the same dimensions may want to implement
|
|
164
|
-
`.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
|
|
165
|
-
consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
|
|
166
|
-
the execution system as early as possible of outputs that will not be
|
|
167
|
-
produced because the corresponding input is missing.
|
|
168
|
-
dimensions : iterable of `str`
|
|
169
|
-
The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
|
|
170
|
-
to identify the dataset type identified by the specified name.
|
|
171
|
-
deferLoad : `bool`
|
|
172
|
-
Indicates that this dataset type will be loaded as a
|
|
173
|
-
`lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
|
|
174
|
-
object to load the object at a later time.
|
|
175
|
-
minimum : `bool`
|
|
176
|
-
Minimum number of datasets required for this connection, per quantum.
|
|
177
|
-
This is checked in the base implementation of
|
|
178
|
-
`.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if
|
|
179
|
-
the minimum is not met for `Input` connections (causing the quantum to
|
|
180
|
-
be pruned, skipped, or never created, depending on the context), and
|
|
181
|
-
`FileNotFoundError` for `PrerequisiteInput` connections (causing
|
|
182
|
-
QuantumGraph generation to fail). `PipelineTask` implementations may
|
|
183
|
-
provide custom `~.PipelineTaskConnections.adjustQuantum`
|
|
184
|
-
implementations for more fine-grained or configuration-driven
|
|
185
|
-
constraints, as long as they are compatible with this minium.
|
|
186
|
-
|
|
187
138
|
Raises
|
|
188
139
|
------
|
|
189
140
|
TypeError
|
|
@@ -194,7 +145,24 @@ class BaseInput(DimensionedConnection):
|
|
|
194
145
|
"""
|
|
195
146
|
|
|
196
147
|
deferLoad: bool = False
|
|
148
|
+
"""Whether this dataset type will be loaded as a
|
|
149
|
+
`lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
|
|
150
|
+
object to load the object at a later time.
|
|
151
|
+
"""
|
|
152
|
+
|
|
197
153
|
minimum: int = 1
|
|
154
|
+
"""Minimum number of datasets required for this connection, per quantum.
|
|
155
|
+
|
|
156
|
+
This is checked in the base implementation of
|
|
157
|
+
`.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if the
|
|
158
|
+
minimum is not met for `Input` connections (causing the quantum to be
|
|
159
|
+
pruned, skipped, or never created, depending on the context), and
|
|
160
|
+
`FileNotFoundError` for `PrerequisiteInput` connections (causing
|
|
161
|
+
QuantumGraph generation to fail). `PipelineTask` implementations may
|
|
162
|
+
provide custom `~.PipelineTaskConnections.adjustQuantum` implementations
|
|
163
|
+
for more fine-grained or configuration-driven constraints, as long as they
|
|
164
|
+
are compatible with this minimum.
|
|
165
|
+
"""
|
|
198
166
|
|
|
199
167
|
def __post_init__(self) -> None:
|
|
200
168
|
super().__post_init__()
|
|
@@ -206,56 +174,6 @@ class BaseInput(DimensionedConnection):
|
|
|
206
174
|
class Input(BaseInput):
|
|
207
175
|
"""Class used for declaring PipelineTask input connections.
|
|
208
176
|
|
|
209
|
-
Attributes
|
|
210
|
-
----------
|
|
211
|
-
name : `str`
|
|
212
|
-
The default name used to identify the dataset type.
|
|
213
|
-
storageClass : `str`
|
|
214
|
-
The storage class used when (un)/persisting the dataset type.
|
|
215
|
-
multiple : `bool`
|
|
216
|
-
Indicates if this connection should expect to contain multiple objects
|
|
217
|
-
of the given dataset type. Tasks with more than one connection with
|
|
218
|
-
``multiple=True`` with the same dimensions may want to implement
|
|
219
|
-
`.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
|
|
220
|
-
consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
|
|
221
|
-
the execution system as early as possible of outputs that will not be
|
|
222
|
-
produced because the corresponding input is missing.
|
|
223
|
-
dimensions : iterable of `str`
|
|
224
|
-
The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
|
|
225
|
-
to identify the dataset type identified by the specified name.
|
|
226
|
-
deferLoad : `bool`
|
|
227
|
-
Indicates that this dataset type will be loaded as a
|
|
228
|
-
`lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
|
|
229
|
-
object to load the object at a later time.
|
|
230
|
-
minimum : `bool`
|
|
231
|
-
Minimum number of datasets required for this connection, per quantum.
|
|
232
|
-
This is checked in the base implementation of
|
|
233
|
-
`.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if
|
|
234
|
-
the minimum is not met for `Input` connections (causing the quantum to
|
|
235
|
-
be pruned, skipped, or never created, depending on the context), and
|
|
236
|
-
`FileNotFoundError` for `PrerequisiteInput` connections (causing
|
|
237
|
-
QuantumGraph generation to fail). `PipelineTask` implementations may
|
|
238
|
-
provide custom `~.PipelineTaskConnections.adjustQuantum`
|
|
239
|
-
implementations for more fine-grained or configuration-driven
|
|
240
|
-
constraints, as long as they are compatible with this minium.
|
|
241
|
-
deferGraphConstraint : `bool`, optional
|
|
242
|
-
If `True`, do not include this dataset type's existence in the initial
|
|
243
|
-
query that starts the QuantumGraph generation process. This can be
|
|
244
|
-
used to make QuantumGraph generation faster by avoiding redundant
|
|
245
|
-
datasets, and in certain cases it can (along with careful attention to
|
|
246
|
-
which tasks are included in the same QuantumGraph) be used to work
|
|
247
|
-
around the QuantumGraph generation algorithm's inflexible handling of
|
|
248
|
-
spatial overlaps. This option has no effect when the connection is not
|
|
249
|
-
an overall input of the pipeline (or subset thereof) for which a graph
|
|
250
|
-
is being created, and it never affects the ordering of quanta.
|
|
251
|
-
deferBinding : `bool`, optional
|
|
252
|
-
If `True`, the dataset will not be automatically included in
|
|
253
|
-
the pipeline graph, ``deferGraphConstraint`` is implied.
|
|
254
|
-
The custom QuantumGraphBuilder is required to bind it and add a
|
|
255
|
-
corresponding edge to the pipeline graph.
|
|
256
|
-
This option allows to have the same dataset type as both
|
|
257
|
-
input and output of a quantum.
|
|
258
|
-
|
|
259
177
|
Raises
|
|
260
178
|
------
|
|
261
179
|
TypeError
|
|
@@ -266,8 +184,27 @@ class Input(BaseInput):
|
|
|
266
184
|
"""
|
|
267
185
|
|
|
268
186
|
deferGraphConstraint: bool = False
|
|
187
|
+
"""If `True`, do not include this dataset type's existence in the initial
|
|
188
|
+
query that starts the QuantumGraph generation process.
|
|
189
|
+
|
|
190
|
+
This can be used to make QuantumGraph generation faster by avoiding
|
|
191
|
+
redundant datasets, and in certain cases it can (along with careful
|
|
192
|
+
attention to which tasks are included in the same QuantumGraph) be used to
|
|
193
|
+
work around the QuantumGraph generation algorithm's inflexible handling of
|
|
194
|
+
spatial overlaps. This option has no effect when the connection is not an
|
|
195
|
+
overall input of the pipeline (or subset thereof) for which a graph is
|
|
196
|
+
being created, and it never affects the ordering of quanta.
|
|
197
|
+
"""
|
|
269
198
|
|
|
270
199
|
deferBinding: bool = False
|
|
200
|
+
"""If `True`, the dataset will not be automatically included in the
|
|
201
|
+
pipeline graph (``deferGraphConstraint=True`` is implied).
|
|
202
|
+
|
|
203
|
+
A custom `~.quantum_graph_builder.QuantumGraphBuilder` is required to bind
|
|
204
|
+
it and add a corresponding edge to the pipeline graph. This option allows
|
|
205
|
+
the same dataset type to be used as both an input and an output of a
|
|
206
|
+
quantum.
|
|
207
|
+
"""
|
|
271
208
|
|
|
272
209
|
_connection_type_set: ClassVar[str] = "inputs"
|
|
273
210
|
|
|
@@ -276,38 +213,6 @@ class Input(BaseInput):
|
|
|
276
213
|
class PrerequisiteInput(BaseInput):
|
|
277
214
|
"""Class used for declaring PipelineTask prerequisite connections.
|
|
278
215
|
|
|
279
|
-
Attributes
|
|
280
|
-
----------
|
|
281
|
-
name : `str`
|
|
282
|
-
The default name used to identify the dataset type.
|
|
283
|
-
storageClass : `str`
|
|
284
|
-
The storage class used when (un)/persisting the dataset type.
|
|
285
|
-
multiple : `bool`
|
|
286
|
-
Indicates if this connection should expect to contain multiple objects
|
|
287
|
-
of the given dataset type. Tasks with more than one connection with
|
|
288
|
-
``multiple=True`` with the same dimensions may want to implement
|
|
289
|
-
`.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
|
|
290
|
-
consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
|
|
291
|
-
the execution system as early as possible of outputs that will not be
|
|
292
|
-
produced because the corresponding input is missing.
|
|
293
|
-
dimensions : iterable of `str`
|
|
294
|
-
The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
|
|
295
|
-
to identify the dataset type identified by the specified name.
|
|
296
|
-
minimum : `bool`
|
|
297
|
-
Minimum number of datasets required for this connection, per quantum.
|
|
298
|
-
This is checked in the base implementation of
|
|
299
|
-
`.PipelineTaskConnections.adjustQuantum`, which raises
|
|
300
|
-
`FileNotFoundError` (causing QuantumGraph generation to fail).
|
|
301
|
-
`PipelineTask` implementations may provide custom
|
|
302
|
-
`~.PipelineTaskConnections.adjustQuantum` implementations for more
|
|
303
|
-
fine-grained or configuration-driven constraints, as long as they are
|
|
304
|
-
compatible with this minium.
|
|
305
|
-
lookupFunction : `typing.Callable`, optional
|
|
306
|
-
An optional callable function that will look up PrerequisiteInputs
|
|
307
|
-
using the DatasetType, registry, quantum dataId, and input collections
|
|
308
|
-
passed to it. If no function is specified, the default temporal spatial
|
|
309
|
-
lookup will be used.
|
|
310
|
-
|
|
311
216
|
Raises
|
|
312
217
|
------
|
|
313
218
|
TypeError
|
|
@@ -342,6 +247,13 @@ class PrerequisiteInput(BaseInput):
|
|
|
342
247
|
lookupFunction: (
|
|
343
248
|
Callable[[DatasetType, Registry, DataCoordinate, Sequence[str]], Iterable[DatasetRef]] | None
|
|
344
249
|
) = None
|
|
250
|
+
"""An optional callable function that will look up PrerequisiteInputs
|
|
251
|
+
using the DatasetType, registry, quantum dataId, and input collections
|
|
252
|
+
passed to it.
|
|
253
|
+
|
|
254
|
+
If no function is specified, the default temporal/spatial lookup will be
|
|
255
|
+
used.
|
|
256
|
+
"""
|
|
345
257
|
|
|
346
258
|
_connection_type_set: ClassVar[str] = "prerequisiteInputs"
|
|
347
259
|
|
lsst/pipe/base/connections.py
CHANGED
|
@@ -495,15 +495,12 @@ class DeferredDatasetRef:
|
|
|
495
495
|
"""A wrapper class for `~lsst.daf.butler.DatasetRef` that indicates that a
|
|
496
496
|
`PipelineTask` should receive a `~lsst.daf.butler.DeferredDatasetHandle`
|
|
497
497
|
instead of an in-memory dataset.
|
|
498
|
-
|
|
499
|
-
Attributes
|
|
500
|
-
----------
|
|
501
|
-
datasetRef : `lsst.daf.butler.DatasetRef`
|
|
502
|
-
The `lsst.daf.butler.DatasetRef` that will be eventually used to
|
|
503
|
-
resolve a dataset.
|
|
504
498
|
"""
|
|
505
499
|
|
|
506
500
|
datasetRef: DatasetRef
|
|
501
|
+
"""The `lsst.daf.butler.DatasetRef` that will be eventually used to
|
|
502
|
+
resolve a dataset.
|
|
503
|
+
"""
|
|
507
504
|
|
|
508
505
|
def __getattr__(self, name: str) -> Any:
|
|
509
506
|
# make sure reduce is called on DeferredDatasetRef and not on
|
|
@@ -299,11 +299,6 @@ class QuantumGraphExecutionReport:
|
|
|
299
299
|
produced DatasetTypes for each task. This report can be output as a
|
|
300
300
|
dictionary or a yaml file.
|
|
301
301
|
|
|
302
|
-
Attributes
|
|
303
|
-
----------
|
|
304
|
-
tasks : `dict`
|
|
305
|
-
A dictionary of TaskExecutionReports by task label.
|
|
306
|
-
|
|
307
302
|
See Also
|
|
308
303
|
--------
|
|
309
304
|
TaskExecutionReport : A task report.
|
lsst/pipe/base/graph/graph.py
CHANGED
|
@@ -136,13 +136,14 @@ class QuantumGraph:
|
|
|
136
136
|
Maps tasks to their InitOutput dataset refs. Dataset refs can be either
|
|
137
137
|
resolved or non-resolved. For intermediate resolved refs their dataset
|
|
138
138
|
ID must match ``initInputs`` and Quantum ``initInputs``.
|
|
139
|
-
globalInitOutputs :
|
|
139
|
+
globalInitOutputs : `~collections.abc.Iterable` \
|
|
140
|
+
[ `~lsst.daf.butler.DatasetRef` ], optional
|
|
140
141
|
Dataset refs for some global objects produced by pipeline. These
|
|
141
142
|
objects include task configurations and package versions. Typically
|
|
142
143
|
they have an empty DataId, but there is no real restriction on what
|
|
143
144
|
can appear here.
|
|
144
|
-
registryDatasetTypes :
|
|
145
|
-
optional
|
|
145
|
+
registryDatasetTypes : `~collections.abc.Iterable` \
|
|
146
|
+
[ `~lsst.daf.butler.DatasetType` ], optional
|
|
146
147
|
Dataset types which are used by this graph, their definitions must
|
|
147
148
|
match registry. If registry does not define dataset type yet, then
|
|
148
149
|
it should match one that will be created later.
|
|
@@ -488,7 +489,7 @@ class QuantumGraph:
|
|
|
488
489
|
|
|
489
490
|
Returns
|
|
490
491
|
-------
|
|
491
|
-
tasks :
|
|
492
|
+
tasks : `~collections.abc.Iterable` [ `TaskDef` ]
|
|
492
493
|
`TaskDef` objects that have the specified `DatasetTypeName` as an
|
|
493
494
|
input, list will be empty if no tasks use specified
|
|
494
495
|
`DatasetTypeName` as an input.
|
|
@@ -537,7 +538,7 @@ class QuantumGraph:
|
|
|
537
538
|
|
|
538
539
|
Returns
|
|
539
540
|
-------
|
|
540
|
-
result :
|
|
541
|
+
result : `~collections.abc.Iterable` [`TaskDef`]
|
|
541
542
|
`TaskDef` objects that are associated with the specified
|
|
542
543
|
`DatasetTypeName`.
|
|
543
544
|
|
|
@@ -935,7 +936,7 @@ class QuantumGraph:
|
|
|
935
936
|
saved structure. If supplied, the
|
|
936
937
|
`~lsst.daf.butler.DimensionUniverse` from the loaded `QuantumGraph`
|
|
937
938
|
will be validated against the supplied argument for compatibility.
|
|
938
|
-
nodes :
|
|
939
|
+
nodes : `~collections.abc.Iterable` [ `uuid.UUID` | `str` ] or `None`
|
|
939
940
|
UUIDs that correspond to nodes in the graph. If specified, only
|
|
940
941
|
these nodes will be loaded. Defaults to None, in which case all
|
|
941
942
|
nodes will be loaded.
|
|
@@ -1220,7 +1221,7 @@ class QuantumGraph:
|
|
|
1220
1221
|
saved structure. If supplied, the
|
|
1221
1222
|
`~lsst.daf.butler.DimensionUniverse` from the loaded `QuantumGraph`
|
|
1222
1223
|
will be validated against the supplied argument for compatibility.
|
|
1223
|
-
nodes :
|
|
1224
|
+
nodes : `~collections.abc.Iterable` [`uuid.UUID`] or `None`
|
|
1224
1225
|
UUIDs that correspond to nodes in the graph. If specified, only
|
|
1225
1226
|
these nodes will be loaded. Defaults to None, in which case all
|
|
1226
1227
|
nodes will be loaded.
|
|
@@ -1438,7 +1439,7 @@ class QuantumGraph:
|
|
|
1438
1439
|
Returns
|
|
1439
1440
|
-------
|
|
1440
1441
|
summary : `QgraphSummary`
|
|
1441
|
-
|
|
1442
|
+
Summary of QuantumGraph.
|
|
1442
1443
|
"""
|
|
1443
1444
|
inCollection = self.metadata.get("input", None)
|
|
1444
1445
|
if isinstance(inCollection, str):
|
lsst/pipe/base/log_capture.py
CHANGED
|
@@ -103,7 +103,7 @@ class _ExecutionLogRecordsExtra(pydantic.BaseModel):
|
|
|
103
103
|
|
|
104
104
|
Parameters
|
|
105
105
|
----------
|
|
106
|
-
log_records : `ButlerLogRecords`
|
|
106
|
+
log_records : `lsst.daf.butler.ButlerLogRecords`
|
|
107
107
|
Logs from a past attempt to run a quantum.
|
|
108
108
|
"""
|
|
109
109
|
previous = self.model_validate(log_records.extra)
|
lsst/pipe/base/pipeline.py
CHANGED
|
@@ -54,13 +54,12 @@ from lsst.utils.introspection import get_full_type_name
|
|
|
54
54
|
|
|
55
55
|
from . import automatic_connection_constants as acc
|
|
56
56
|
from . import pipeline_graph, pipelineIR
|
|
57
|
-
from ._instrument import Instrument as
|
|
57
|
+
from ._instrument import Instrument as Instrument
|
|
58
58
|
from .config import PipelineTaskConfig
|
|
59
59
|
from .connections import PipelineTaskConnections
|
|
60
60
|
from .pipelineTask import PipelineTask
|
|
61
61
|
|
|
62
62
|
if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.
|
|
63
|
-
from lsst.obs.base import Instrument
|
|
64
63
|
from lsst.pex.config import Config
|
|
65
64
|
|
|
66
65
|
# ----------------------------------
|
|
@@ -496,7 +495,7 @@ class Pipeline:
|
|
|
496
495
|
Returns
|
|
497
496
|
-------
|
|
498
497
|
pipeline: `Pipeline`
|
|
499
|
-
|
|
498
|
+
The new pipeline.
|
|
500
499
|
"""
|
|
501
500
|
return cls.fromIR(copy.deepcopy(pipeline._pipelineIR))
|
|
502
501
|
|
|
@@ -606,7 +605,7 @@ class Pipeline:
|
|
|
606
605
|
|
|
607
606
|
@property
|
|
608
607
|
def subsets(self) -> MappingProxyType[str, set]:
|
|
609
|
-
"""Returns a `MappingProxyType` where the keys are the labels of
|
|
608
|
+
"""Returns a `types.MappingProxyType` where the keys are the labels of
|
|
610
609
|
labeled subsets in the `Pipeline` and the values are the set of task
|
|
611
610
|
labels contained within that subset.
|
|
612
611
|
"""
|
|
@@ -702,7 +701,7 @@ class Pipeline:
|
|
|
702
701
|
"""
|
|
703
702
|
instrument_class_name = self._pipelineIR.instrument
|
|
704
703
|
if instrument_class_name is not None:
|
|
705
|
-
instrument_class = cast(
|
|
704
|
+
instrument_class = cast(Instrument, doImportType(instrument_class_name))
|
|
706
705
|
if instrument_class is not None:
|
|
707
706
|
return DataCoordinate.standardize(instrument=instrument_class.getName(), universe=universe)
|
|
708
707
|
return DataCoordinate.make_empty(universe)
|
|
@@ -893,7 +892,7 @@ class Pipeline:
|
|
|
893
892
|
raise NameError(f"Label {label} does not appear in this pipeline")
|
|
894
893
|
taskClass: type[PipelineTask] = doImportType(taskIR.klass)
|
|
895
894
|
config = taskClass.ConfigClass()
|
|
896
|
-
instrument:
|
|
895
|
+
instrument: Instrument | None = None
|
|
897
896
|
if (instrumentName := self._pipelineIR.instrument) is not None:
|
|
898
897
|
instrument_cls: type = doImportType(instrumentName)
|
|
899
898
|
instrument = instrument_cls()
|
lsst/pipe/base/pipelineIR.py
CHANGED
|
@@ -220,12 +220,6 @@ class LabeledSubset:
|
|
|
220
220
|
class ParametersIR:
|
|
221
221
|
"""Intermediate representation of parameters that are global to a pipeline.
|
|
222
222
|
|
|
223
|
-
Attributes
|
|
224
|
-
----------
|
|
225
|
-
mapping : `dict` [`str`, `str`]
|
|
226
|
-
A mutable mapping of identifiers as keys, and shared configuration
|
|
227
|
-
as values.
|
|
228
|
-
|
|
229
223
|
Notes
|
|
230
224
|
-----
|
|
231
225
|
These parameters are specified under a top level key named ``parameters``
|
|
@@ -706,7 +700,7 @@ class PipelineIR:
|
|
|
706
700
|
|
|
707
701
|
Parameters
|
|
708
702
|
----------
|
|
709
|
-
loaded_yaml: `dict`
|
|
703
|
+
loaded_yaml : `dict`
|
|
710
704
|
A dictionary which matches the structure that would be produced
|
|
711
705
|
by a yaml reader which parses a pipeline definition document
|
|
712
706
|
"""
|
lsst/pipe/base/pipelineTask.py
CHANGED
|
@@ -55,7 +55,7 @@ class PipelineTask(Task):
|
|
|
55
55
|
resulting data is also stored in a data butler.
|
|
56
56
|
|
|
57
57
|
PipelineTask inherits from a `~lsst.pipe.base.Task` and uses the same
|
|
58
|
-
configuration mechanism based on
|
|
58
|
+
configuration mechanism based on `lsst.pex.config`. `PipelineTask`
|
|
59
59
|
classes also have a `PipelineTaskConnections` class associated with their
|
|
60
60
|
config which defines all of the IO a `PipelineTask` will need to do.
|
|
61
61
|
PipelineTask sub-class typically implements `run()` method which receives
|
|
@@ -75,12 +75,6 @@ class PipelineTask(Task):
|
|
|
75
75
|
PipelineTask base class constructor, but may support other signatures as
|
|
76
76
|
well.
|
|
77
77
|
|
|
78
|
-
Attributes
|
|
79
|
-
----------
|
|
80
|
-
canMultiprocess : bool, True by default (class attribute)
|
|
81
|
-
This class attribute is checked by execution framework, sub-classes
|
|
82
|
-
can set it to ``False`` in case task does not support multiprocessing.
|
|
83
|
-
|
|
84
78
|
Parameters
|
|
85
79
|
----------
|
|
86
80
|
config : `~lsst.pex.config.Config`, optional
|
|
@@ -102,7 +96,11 @@ class PipelineTask(Task):
|
|
|
102
96
|
"""
|
|
103
97
|
|
|
104
98
|
ConfigClass: ClassVar[type[PipelineTaskConfig]]
|
|
99
|
+
|
|
105
100
|
canMultiprocess: ClassVar[bool] = True
|
|
101
|
+
"""Whether this task can be run by an executor that uses subprocesses for
|
|
102
|
+
parallelism.
|
|
103
|
+
"""
|
|
106
104
|
|
|
107
105
|
def __init__(
|
|
108
106
|
self,
|
|
@@ -106,8 +106,8 @@ class DatasetTypeNode:
|
|
|
106
106
|
The internal networkx graph.
|
|
107
107
|
get_registered : `~collections.abc.Callable` or `None`
|
|
108
108
|
Callable that takes a dataset type name and returns the
|
|
109
|
-
|
|
110
|
-
not registered.
|
|
109
|
+
`~lsst.daf.butler.DatasetType` registered in the data repository,
|
|
110
|
+
or `None` if it is not registered.
|
|
111
111
|
dimensions : `lsst.daf.butler.DimensionUniverse`
|
|
112
112
|
Definitions of all dimensions.
|
|
113
113
|
previous : `DatasetTypeNode` or `None`
|
|
@@ -480,11 +480,11 @@ class ReadEdge(Edge):
|
|
|
480
480
|
Parameters
|
|
481
481
|
----------
|
|
482
482
|
current : `lsst.daf.butler.DatasetType` or `None`
|
|
483
|
-
The current graph-wide
|
|
484
|
-
be the registry's definition of the parent dataset
|
|
485
|
-
exists. If not, it will be the dataset type
|
|
486
|
-
task in the graph that writes it, if there is
|
|
487
|
-
such task, this will be `None`.
|
|
483
|
+
The current graph-wide `~lsst.daf.butler.DatasetType`, or `None`.
|
|
484
|
+
This will always be the registry's definition of the parent dataset
|
|
485
|
+
type, if one exists. If not, it will be the dataset type
|
|
486
|
+
definition from the task in the graph that writes it, if there is
|
|
487
|
+
one. If there is no such task, this will be `None`.
|
|
488
488
|
is_initial_query_constraint : `bool`
|
|
489
489
|
Whether this dataset type is currently marked as a constraint on
|
|
490
490
|
the initial data ID query in QuantumGraph generation.
|
|
@@ -496,7 +496,7 @@ class ReadEdge(Edge):
|
|
|
496
496
|
producer : `str` or `None`
|
|
497
497
|
The label of the task that produces this dataset type in the
|
|
498
498
|
pipeline, or `None` if it is an overall input.
|
|
499
|
-
consumers :
|
|
499
|
+
consumers : `~collections.abc.Sequence` [ `str` ]
|
|
500
500
|
Labels for other consuming tasks that have already participated in
|
|
501
501
|
this dataset type's resolution.
|
|
502
502
|
is_registered : `bool`
|
|
@@ -512,7 +512,7 @@ class ReadEdge(Edge):
|
|
|
512
512
|
|
|
513
513
|
Returns
|
|
514
514
|
-------
|
|
515
|
-
dataset_type :
|
|
515
|
+
dataset_type : `~lsst.daf.butler.DatasetType`
|
|
516
516
|
The updated graph-wide dataset type. If ``current`` was provided,
|
|
517
517
|
this must be equal to it.
|
|
518
518
|
is_initial_query_constraint : `bool`
|
|
@@ -659,13 +659,25 @@ class ReadEdge(Edge):
|
|
|
659
659
|
# compatible), since neither connection should take
|
|
660
660
|
# precedence.
|
|
661
661
|
if dataset_type != current:
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
"
|
|
667
|
-
|
|
668
|
-
|
|
662
|
+
if visualization_only and dataset_type.dimensions == current.dimensions:
|
|
663
|
+
# Make a visualization-only ambiguous storage class
|
|
664
|
+
# "name".
|
|
665
|
+
all_storage_classes = set(current.storageClass_name.split("/"))
|
|
666
|
+
all_storage_classes.update(dataset_type.storageClass_name.split("/"))
|
|
667
|
+
current = DatasetType(
|
|
668
|
+
current.name,
|
|
669
|
+
current.dimensions,
|
|
670
|
+
"/".join(sorted(all_storage_classes)),
|
|
671
|
+
)
|
|
672
|
+
else:
|
|
673
|
+
raise MissingDatasetTypeError(
|
|
674
|
+
f"Definitions differ for input dataset type "
|
|
675
|
+
f"{self.parent_dataset_type_name!r}; task {self.task_label!r} has "
|
|
676
|
+
f"{dataset_type}, but the definition from {report_current_origin()} is "
|
|
677
|
+
f"{current}. If the storage classes are compatible but different, "
|
|
678
|
+
"registering the dataset type in the data repository in advance will avoid "
|
|
679
|
+
"this error."
|
|
680
|
+
)
|
|
669
681
|
elif not visualization_only and not dataset_type.is_compatible_with(current):
|
|
670
682
|
raise IncompatibleDatasetTypeError(
|
|
671
683
|
f"Incompatible definition for input dataset type {self.parent_dataset_type_name!r}; "
|
|
@@ -788,15 +800,15 @@ class WriteEdge(Edge):
|
|
|
788
800
|
Parameters
|
|
789
801
|
----------
|
|
790
802
|
current : `lsst.daf.butler.DatasetType` or `None`
|
|
791
|
-
The current graph-wide
|
|
792
|
-
be the registry's definition of the parent dataset
|
|
793
|
-
exists.
|
|
803
|
+
The current graph-wide `~lsst.daf.butler.DatasetType`, or `None`.
|
|
804
|
+
This will always be the registry's definition of the parent dataset
|
|
805
|
+
type, if one exists.
|
|
794
806
|
universe : `lsst.daf.butler.DimensionUniverse`
|
|
795
807
|
Object that holds all dimension definitions.
|
|
796
808
|
|
|
797
809
|
Returns
|
|
798
810
|
-------
|
|
799
|
-
dataset_type :
|
|
811
|
+
dataset_type : `~lsst.daf.butler.DatasetType`
|
|
800
812
|
A dataset type compatible with this edge. If ``current`` was
|
|
801
813
|
provided, this must be equal to it.
|
|
802
814
|
|
|
@@ -897,6 +897,10 @@ class PipelineGraph:
|
|
|
897
897
|
New config objects or overrides to apply to copies of the current
|
|
898
898
|
config objects, with task labels as the keywords.
|
|
899
899
|
|
|
900
|
+
Returns
|
|
901
|
+
-------
|
|
902
|
+
None
|
|
903
|
+
|
|
900
904
|
Raises
|
|
901
905
|
------
|
|
902
906
|
ValueError
|
|
@@ -1632,7 +1636,7 @@ class PipelineGraph:
|
|
|
1632
1636
|
|
|
1633
1637
|
Returns
|
|
1634
1638
|
-------
|
|
1635
|
-
subgraphs :
|
|
1639
|
+
subgraphs : `~collections.abc.Iterable` [ `PipelineGraph` ]
|
|
1636
1640
|
An iterable over component subgraphs that could be run
|
|
1637
1641
|
independently (they have only overall inputs in common). May be a
|
|
1638
1642
|
lazy iterator.
|
|
@@ -1755,6 +1759,10 @@ class PipelineGraph:
|
|
|
1755
1759
|
not considered part of the pipeline graph in other respects, but it
|
|
1756
1760
|
does get written with other provenance datasets).
|
|
1757
1761
|
|
|
1762
|
+
Returns
|
|
1763
|
+
-------
|
|
1764
|
+
None
|
|
1765
|
+
|
|
1758
1766
|
Raises
|
|
1759
1767
|
------
|
|
1760
1768
|
lsst.daf.butler.MissingDatasetTypeError
|
|
@@ -2228,7 +2236,7 @@ class PipelineGraph:
|
|
|
2228
2236
|
|
|
2229
2237
|
Parameters
|
|
2230
2238
|
----------
|
|
2231
|
-
updates :
|
|
2239
|
+
updates : `~collections.abc.Mapping` [ `str`, `TaskNode` ]
|
|
2232
2240
|
New task nodes with task label keys. All keys must be task labels
|
|
2233
2241
|
that are already present in the graph.
|
|
2234
2242
|
check_edges_unchanged : `bool`, optional
|