lsst-pipe-base 30.2026.300__py3-none-any.whl → 30.2026.500__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. lsst/pipe/base/_instrument.py +21 -12
  2. lsst/pipe/base/_quantumContext.py +3 -3
  3. lsst/pipe/base/_status.py +43 -10
  4. lsst/pipe/base/automatic_connection_constants.py +20 -1
  5. lsst/pipe/base/cli/cmd/__init__.py +18 -2
  6. lsst/pipe/base/cli/cmd/commands.py +149 -4
  7. lsst/pipe/base/connectionTypes.py +72 -160
  8. lsst/pipe/base/connections.py +3 -6
  9. lsst/pipe/base/execution_reports.py +0 -5
  10. lsst/pipe/base/graph/graph.py +9 -8
  11. lsst/pipe/base/log_capture.py +1 -1
  12. lsst/pipe/base/pipeline.py +5 -6
  13. lsst/pipe/base/pipelineIR.py +1 -7
  14. lsst/pipe/base/pipelineTask.py +5 -7
  15. lsst/pipe/base/pipeline_graph/_dataset_types.py +2 -2
  16. lsst/pipe/base/pipeline_graph/_edges.py +30 -18
  17. lsst/pipe/base/pipeline_graph/_pipeline_graph.py +10 -2
  18. lsst/pipe/base/pipeline_graph/visualization/_dot.py +13 -12
  19. lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py +7 -0
  20. lsst/pipe/base/prerequisite_helpers.py +2 -1
  21. lsst/pipe/base/quantum_graph/_common.py +3 -1
  22. lsst/pipe/base/quantum_graph/_multiblock.py +29 -13
  23. lsst/pipe/base/quantum_graph/_predicted.py +7 -0
  24. lsst/pipe/base/quantum_graph/_provenance.py +498 -56
  25. lsst/pipe/base/quantum_graph/aggregator/__init__.py +0 -1
  26. lsst/pipe/base/quantum_graph/aggregator/_communicators.py +9 -1
  27. lsst/pipe/base/quantum_graph/aggregator/_config.py +78 -9
  28. lsst/pipe/base/quantum_graph/aggregator/_ingester.py +12 -11
  29. lsst/pipe/base/quantum_graph/aggregator/_scanner.py +14 -6
  30. lsst/pipe/base/quantum_graph/aggregator/_structs.py +3 -3
  31. lsst/pipe/base/quantum_graph/aggregator/_supervisor.py +14 -13
  32. lsst/pipe/base/quantum_graph/aggregator/_writer.py +2 -2
  33. lsst/pipe/base/quantum_graph/formatter.py +74 -4
  34. lsst/pipe/base/quantum_graph/ingest_graph.py +413 -0
  35. lsst/pipe/base/quantum_graph_builder.py +1 -8
  36. lsst/pipe/base/quantum_graph_skeleton.py +29 -27
  37. lsst/pipe/base/quantum_provenance_graph.py +29 -12
  38. lsst/pipe/base/separable_pipeline_executor.py +6 -7
  39. lsst/pipe/base/single_quantum_executor.py +7 -7
  40. lsst/pipe/base/struct.py +4 -0
  41. lsst/pipe/base/tests/mocks/_storage_class.py +2 -1
  42. lsst/pipe/base/version.py +1 -1
  43. {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/METADATA +2 -1
  44. {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/RECORD +52 -51
  45. {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/WHEEL +1 -1
  46. {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/entry_points.txt +0 -0
  47. {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/COPYRIGHT +0 -0
  48. {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/LICENSE +0 -0
  49. {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/bsd_license.txt +0 -0
  50. {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/gpl-v3.0.txt +0 -0
  51. {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/top_level.txt +0 -0
  52. {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.500.dist-info}/zip-safe +0 -0
@@ -41,35 +41,36 @@ from lsst.utils.introspection import find_outside_stacklevel
41
41
 
42
42
  @dataclasses.dataclass(frozen=True)
43
43
  class BaseConnection:
44
- """Base class used for declaring `PipelineTask` connections.
45
-
46
- Attributes
47
- ----------
48
- name : `str`
49
- The name used to identify the dataset type.
50
- storageClass : `str`
51
- The storage class used when (un)/persisting the dataset type.
52
- multiple : `bool`
53
- Indicates if this connection should expect to contain multiple objects
54
- of the given dataset type. Tasks with more than one connection with
55
- ``multiple=True`` with the same dimensions may want to implement
56
- `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
57
- consistent (i.e. zip-iterable) in `PipelineTask.runQuantum()` and
58
- notify the execution system as early as possible of outputs that will
59
- not be produced because the corresponding input is missing.
60
- deprecated : `str`, optional
61
- A description of why this connection is deprecated, including the
62
- version after which it may be removed.
63
-
64
- If not `None`, the string is appended to the docstring for this
65
- connection and the corresponding config Field.
66
- """
44
+ """Base class used for declaring `PipelineTask` connections."""
67
45
 
68
46
  name: str
47
+ """The name used to identify the dataset type."""
48
+
69
49
  storageClass: str
50
+ """The storage class used when (un)/persisting the dataset type."""
51
+
70
52
  doc: str = ""
53
+ """Documentation for this connection."""
54
+
71
55
  multiple: bool = False
56
+ """Indicates if this connection should expect to contain multiple objects
57
+ of the given dataset type.
58
+
59
+ Tasks with more than one connection with ``multiple=True`` with the same
60
+ dimensions may want to implement `.PipelineTaskConnections.adjustQuantum`
61
+ to ensure those datasets are consistent (i.e. zip-iterable) in
62
+ `PipelineTask.runQuantum()` and notify the execution system as early as
63
+ possible of outputs that will not be produced because the corresponding
64
+ input is missing.
65
+ """
66
+
72
67
  deprecated: str | None = dataclasses.field(default=None, kw_only=True)
68
+ """A description of why this connection is deprecated, including the
69
+ version after which it may be removed.
70
+
71
+ If not `None`, the string is appended to the docstring for this
72
+ connection and the corresponding config Field.
73
+ """
73
74
 
74
75
  _connection_type_set: ClassVar[str]
75
76
  _deprecation_context: str = ""
@@ -110,32 +111,15 @@ class BaseConnection:
110
111
  class DimensionedConnection(BaseConnection):
111
112
  """Class used for declaring PipelineTask connections that includes
112
113
  dimensions.
113
-
114
- Attributes
115
- ----------
116
- name : `str`
117
- The name used to identify the dataset type.
118
- storageClass : `str`
119
- The storage class used when (un)/persisting the dataset type.
120
- multiple : `bool`
121
- Indicates if this connection should expect to contain multiple objects
122
- of the given dataset type. Tasks with more than one connection with
123
- ``multiple=True`` with the same dimensions may want to implement
124
- `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
125
- consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
126
- the execution system as early as possible of outputs that will not be
127
- produced because the corresponding input is missing.
128
- dimensions : iterable of `str`
129
- The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
130
- to identify the dataset type identified by the specified name.
131
- isCalibration : `bool`, optional
132
- `True` if this dataset type may be included in CALIBRATION-type
133
- collections to associate it with a validity range, `False` (default)
134
- otherwise.
135
114
  """
136
115
 
137
116
  dimensions: Iterable[str] = ()
117
+ """The keys of the butler data coordinates for this dataset type."""
118
+
138
119
  isCalibration: bool = False
120
+ """ `True` if this dataset type may be included in
121
+ `~lsst.daf.butler.CollectionType.CALIBRATION` collections to associate it
122
+ with a validity range, `False` (default) otherwise."""
139
123
 
140
124
  def __post_init__(self):
141
125
  super().__post_init__()
@@ -151,39 +135,6 @@ class DimensionedConnection(BaseConnection):
151
135
  class BaseInput(DimensionedConnection):
152
136
  """Class used for declaring PipelineTask input connections.
153
137
 
154
- Attributes
155
- ----------
156
- name : `str`
157
- The default name used to identify the dataset type.
158
- storageClass : `str`
159
- The storage class used when (un)/persisting the dataset type.
160
- multiple : `bool`
161
- Indicates if this connection should expect to contain multiple objects
162
- of the given dataset type. Tasks with more than one connection with
163
- ``multiple=True`` with the same dimensions may want to implement
164
- `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
165
- consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
166
- the execution system as early as possible of outputs that will not be
167
- produced because the corresponding input is missing.
168
- dimensions : iterable of `str`
169
- The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
170
- to identify the dataset type identified by the specified name.
171
- deferLoad : `bool`
172
- Indicates that this dataset type will be loaded as a
173
- `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
174
- object to load the object at a later time.
175
- minimum : `bool`
176
- Minimum number of datasets required for this connection, per quantum.
177
- This is checked in the base implementation of
178
- `.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if
179
- the minimum is not met for `Input` connections (causing the quantum to
180
- be pruned, skipped, or never created, depending on the context), and
181
- `FileNotFoundError` for `PrerequisiteInput` connections (causing
182
- QuantumGraph generation to fail). `PipelineTask` implementations may
183
- provide custom `~.PipelineTaskConnections.adjustQuantum`
184
- implementations for more fine-grained or configuration-driven
185
- constraints, as long as they are compatible with this minium.
186
-
187
138
  Raises
188
139
  ------
189
140
  TypeError
@@ -194,7 +145,24 @@ class BaseInput(DimensionedConnection):
194
145
  """
195
146
 
196
147
  deferLoad: bool = False
148
+ """Whether this dataset type will be loaded as a
149
+ `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
150
+ object to load the object at a later time.
151
+ """
152
+
197
153
  minimum: int = 1
154
+ """Minimum number of datasets required for this connection, per quantum.
155
+
156
+ This is checked in the base implementation of
157
+ `.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if the
158
+ minimum is not met for `Input` connections (causing the quantum to be
159
+ pruned, skipped, or never created, depending on the context), and
160
+ `FileNotFoundError` for `PrerequisiteInput` connections (causing
161
+ QuantumGraph generation to fail). `PipelineTask` implementations may
162
+ provide custom `~.PipelineTaskConnections.adjustQuantum` implementations
163
+ for more fine-grained or configuration-driven constraints, as long as they
164
+ are compatible with this minimum.
165
+ """
198
166
 
199
167
  def __post_init__(self) -> None:
200
168
  super().__post_init__()
@@ -206,56 +174,6 @@ class BaseInput(DimensionedConnection):
206
174
  class Input(BaseInput):
207
175
  """Class used for declaring PipelineTask input connections.
208
176
 
209
- Attributes
210
- ----------
211
- name : `str`
212
- The default name used to identify the dataset type.
213
- storageClass : `str`
214
- The storage class used when (un)/persisting the dataset type.
215
- multiple : `bool`
216
- Indicates if this connection should expect to contain multiple objects
217
- of the given dataset type. Tasks with more than one connection with
218
- ``multiple=True`` with the same dimensions may want to implement
219
- `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
220
- consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
221
- the execution system as early as possible of outputs that will not be
222
- produced because the corresponding input is missing.
223
- dimensions : iterable of `str`
224
- The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
225
- to identify the dataset type identified by the specified name.
226
- deferLoad : `bool`
227
- Indicates that this dataset type will be loaded as a
228
- `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
229
- object to load the object at a later time.
230
- minimum : `bool`
231
- Minimum number of datasets required for this connection, per quantum.
232
- This is checked in the base implementation of
233
- `.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if
234
- the minimum is not met for `Input` connections (causing the quantum to
235
- be pruned, skipped, or never created, depending on the context), and
236
- `FileNotFoundError` for `PrerequisiteInput` connections (causing
237
- QuantumGraph generation to fail). `PipelineTask` implementations may
238
- provide custom `~.PipelineTaskConnections.adjustQuantum`
239
- implementations for more fine-grained or configuration-driven
240
- constraints, as long as they are compatible with this minium.
241
- deferGraphConstraint : `bool`, optional
242
- If `True`, do not include this dataset type's existence in the initial
243
- query that starts the QuantumGraph generation process. This can be
244
- used to make QuantumGraph generation faster by avoiding redundant
245
- datasets, and in certain cases it can (along with careful attention to
246
- which tasks are included in the same QuantumGraph) be used to work
247
- around the QuantumGraph generation algorithm's inflexible handling of
248
- spatial overlaps. This option has no effect when the connection is not
249
- an overall input of the pipeline (or subset thereof) for which a graph
250
- is being created, and it never affects the ordering of quanta.
251
- deferBinding : `bool`, optional
252
- If `True`, the dataset will not be automatically included in
253
- the pipeline graph, ``deferGraphConstraint`` is implied.
254
- The custom QuantumGraphBuilder is required to bind it and add a
255
- corresponding edge to the pipeline graph.
256
- This option allows to have the same dataset type as both
257
- input and output of a quantum.
258
-
259
177
  Raises
260
178
  ------
261
179
  TypeError
@@ -266,8 +184,27 @@ class Input(BaseInput):
266
184
  """
267
185
 
268
186
  deferGraphConstraint: bool = False
187
+ """If `True`, do not include this dataset type's existence in the initial
188
+ query that starts the QuantumGraph generation process.
189
+
190
+ This can be used to make QuantumGraph generation faster by avoiding
191
+ redundant datasets, and in certain cases it can (along with careful
192
+ attention to which tasks are included in the same QuantumGraph) be used to
193
+ work around the QuantumGraph generation algorithm's inflexible handling of
194
+ spatial overlaps. This option has no effect when the connection is not an
195
+ overall input of the pipeline (or subset thereof) for which a graph is
196
+ being created, and it never affects the ordering of quanta.
197
+ """
269
198
 
270
199
  deferBinding: bool = False
200
+ """If `True`, the dataset will not be automatically included in the
201
+ pipeline graph (``deferGraphConstraint=True`` is implied).
202
+
203
+ A custom `~.quantum_graph_builder.QuantumGraphBuilder` is required to bind
204
+ it and add a corresponding edge to the pipeline graph. This option allows
205
+ the same dataset type to be used as both an input and an output of a
206
+ quantum.
207
+ """
271
208
 
272
209
  _connection_type_set: ClassVar[str] = "inputs"
273
210
 
@@ -276,38 +213,6 @@ class Input(BaseInput):
276
213
  class PrerequisiteInput(BaseInput):
277
214
  """Class used for declaring PipelineTask prerequisite connections.
278
215
 
279
- Attributes
280
- ----------
281
- name : `str`
282
- The default name used to identify the dataset type.
283
- storageClass : `str`
284
- The storage class used when (un)/persisting the dataset type.
285
- multiple : `bool`
286
- Indicates if this connection should expect to contain multiple objects
287
- of the given dataset type. Tasks with more than one connection with
288
- ``multiple=True`` with the same dimensions may want to implement
289
- `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
290
- consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
291
- the execution system as early as possible of outputs that will not be
292
- produced because the corresponding input is missing.
293
- dimensions : iterable of `str`
294
- The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
295
- to identify the dataset type identified by the specified name.
296
- minimum : `bool`
297
- Minimum number of datasets required for this connection, per quantum.
298
- This is checked in the base implementation of
299
- `.PipelineTaskConnections.adjustQuantum`, which raises
300
- `FileNotFoundError` (causing QuantumGraph generation to fail).
301
- `PipelineTask` implementations may provide custom
302
- `~.PipelineTaskConnections.adjustQuantum` implementations for more
303
- fine-grained or configuration-driven constraints, as long as they are
304
- compatible with this minium.
305
- lookupFunction : `typing.Callable`, optional
306
- An optional callable function that will look up PrerequisiteInputs
307
- using the DatasetType, registry, quantum dataId, and input collections
308
- passed to it. If no function is specified, the default temporal spatial
309
- lookup will be used.
310
-
311
216
  Raises
312
217
  ------
313
218
  TypeError
@@ -342,6 +247,13 @@ class PrerequisiteInput(BaseInput):
342
247
  lookupFunction: (
343
248
  Callable[[DatasetType, Registry, DataCoordinate, Sequence[str]], Iterable[DatasetRef]] | None
344
249
  ) = None
250
+ """An optional callable function that will look up PrerequisiteInputs
251
+ using the DatasetType, registry, quantum dataId, and input collections
252
+ passed to it.
253
+
254
+ If no function is specified, the default temporal/spatial lookup will be
255
+ used.
256
+ """
345
257
 
346
258
  _connection_type_set: ClassVar[str] = "prerequisiteInputs"
347
259
 
@@ -495,15 +495,12 @@ class DeferredDatasetRef:
495
495
  """A wrapper class for `~lsst.daf.butler.DatasetRef` that indicates that a
496
496
  `PipelineTask` should receive a `~lsst.daf.butler.DeferredDatasetHandle`
497
497
  instead of an in-memory dataset.
498
-
499
- Attributes
500
- ----------
501
- datasetRef : `lsst.daf.butler.DatasetRef`
502
- The `lsst.daf.butler.DatasetRef` that will be eventually used to
503
- resolve a dataset.
504
498
  """
505
499
 
506
500
  datasetRef: DatasetRef
501
+ """The `lsst.daf.butler.DatasetRef` that will be eventually used to
502
+ resolve a dataset.
503
+ """
507
504
 
508
505
  def __getattr__(self, name: str) -> Any:
509
506
  # make sure reduce is called on DeferredDatasetRef and not on
@@ -299,11 +299,6 @@ class QuantumGraphExecutionReport:
299
299
  produced DatasetTypes for each task. This report can be output as a
300
300
  dictionary or a yaml file.
301
301
 
302
- Attributes
303
- ----------
304
- tasks : `dict`
305
- A dictionary of TaskExecutionReports by task label.
306
-
307
302
  See Also
308
303
  --------
309
304
  TaskExecutionReport : A task report.
@@ -136,13 +136,14 @@ class QuantumGraph:
136
136
  Maps tasks to their InitOutput dataset refs. Dataset refs can be either
137
137
  resolved or non-resolved. For intermediate resolved refs their dataset
138
138
  ID must match ``initInputs`` and Quantum ``initInputs``.
139
- globalInitOutputs : iterable [ `~lsst.daf.butler.DatasetRef` ], optional
139
+ globalInitOutputs : `~collections.abc.Iterable` \
140
+ [ `~lsst.daf.butler.DatasetRef` ], optional
140
141
  Dataset refs for some global objects produced by pipeline. These
141
142
  objects include task configurations and package versions. Typically
142
143
  they have an empty DataId, but there is no real restriction on what
143
144
  can appear here.
144
- registryDatasetTypes : iterable [ `~lsst.daf.butler.DatasetType` ], \
145
- optional
145
+ registryDatasetTypes : `~collections.abc.Iterable` \
146
+ [ `~lsst.daf.butler.DatasetType` ], optional
146
147
  Dataset types which are used by this graph, their definitions must
147
148
  match registry. If registry does not define dataset type yet, then
148
149
  it should match one that will be created later.
@@ -488,7 +489,7 @@ class QuantumGraph:
488
489
 
489
490
  Returns
490
491
  -------
491
- tasks : iterable of `TaskDef`
492
+ tasks : `~collections.abc.Iterable` [ `TaskDef` ]
492
493
  `TaskDef` objects that have the specified `DatasetTypeName` as an
493
494
  input, list will be empty if no tasks use specified
494
495
  `DatasetTypeName` as an input.
@@ -537,7 +538,7 @@ class QuantumGraph:
537
538
 
538
539
  Returns
539
540
  -------
540
- result : iterable of `TaskDef`
541
+ result : `~collections.abc.Iterable` [`TaskDef`]
541
542
  `TaskDef` objects that are associated with the specified
542
543
  `DatasetTypeName`.
543
544
 
@@ -935,7 +936,7 @@ class QuantumGraph:
935
936
  saved structure. If supplied, the
936
937
  `~lsst.daf.butler.DimensionUniverse` from the loaded `QuantumGraph`
937
938
  will be validated against the supplied argument for compatibility.
938
- nodes : iterable of [ `uuid.UUID` | `str` ] or `None`
939
+ nodes : `~collections.abc.Iterable` [ `uuid.UUID` | `str` ] or `None`
939
940
  UUIDs that correspond to nodes in the graph. If specified, only
940
941
  these nodes will be loaded. Defaults to None, in which case all
941
942
  nodes will be loaded.
@@ -1220,7 +1221,7 @@ class QuantumGraph:
1220
1221
  saved structure. If supplied, the
1221
1222
  `~lsst.daf.butler.DimensionUniverse` from the loaded `QuantumGraph`
1222
1223
  will be validated against the supplied argument for compatibility.
1223
- nodes : iterable of `uuid.UUID` or `None`
1224
+ nodes : `~collections.abc.Iterable` [`uuid.UUID`] or `None`
1224
1225
  UUIDs that correspond to nodes in the graph. If specified, only
1225
1226
  these nodes will be loaded. Defaults to None, in which case all
1226
1227
  nodes will be loaded.
@@ -1438,7 +1439,7 @@ class QuantumGraph:
1438
1439
  Returns
1439
1440
  -------
1440
1441
  summary : `QgraphSummary`
1441
- Summary of QuantumGraph.
1442
+ Summary of QuantumGraph.
1442
1443
  """
1443
1444
  inCollection = self.metadata.get("input", None)
1444
1445
  if isinstance(inCollection, str):
@@ -103,7 +103,7 @@ class _ExecutionLogRecordsExtra(pydantic.BaseModel):
103
103
 
104
104
  Parameters
105
105
  ----------
106
- log_records : `ButlerLogRecords`
106
+ log_records : `lsst.daf.butler.ButlerLogRecords`
107
107
  Logs from a past attempt to run a quantum.
108
108
  """
109
109
  previous = self.model_validate(log_records.extra)
@@ -54,13 +54,12 @@ from lsst.utils.introspection import get_full_type_name
54
54
 
55
55
  from . import automatic_connection_constants as acc
56
56
  from . import pipeline_graph, pipelineIR
57
- from ._instrument import Instrument as PipeBaseInstrument
57
+ from ._instrument import Instrument as Instrument
58
58
  from .config import PipelineTaskConfig
59
59
  from .connections import PipelineTaskConnections
60
60
  from .pipelineTask import PipelineTask
61
61
 
62
62
  if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.
63
- from lsst.obs.base import Instrument
64
63
  from lsst.pex.config import Config
65
64
 
66
65
  # ----------------------------------
@@ -496,7 +495,7 @@ class Pipeline:
496
495
  Returns
497
496
  -------
498
497
  pipeline: `Pipeline`
499
- The new pipeline.
498
+ The new pipeline.
500
499
  """
501
500
  return cls.fromIR(copy.deepcopy(pipeline._pipelineIR))
502
501
 
@@ -606,7 +605,7 @@ class Pipeline:
606
605
 
607
606
  @property
608
607
  def subsets(self) -> MappingProxyType[str, set]:
609
- """Returns a `MappingProxyType` where the keys are the labels of
608
+ """Returns a `types.MappingProxyType` where the keys are the labels of
610
609
  labeled subsets in the `Pipeline` and the values are the set of task
611
610
  labels contained within that subset.
612
611
  """
@@ -702,7 +701,7 @@ class Pipeline:
702
701
  """
703
702
  instrument_class_name = self._pipelineIR.instrument
704
703
  if instrument_class_name is not None:
705
- instrument_class = cast(PipeBaseInstrument, doImportType(instrument_class_name))
704
+ instrument_class = cast(Instrument, doImportType(instrument_class_name))
706
705
  if instrument_class is not None:
707
706
  return DataCoordinate.standardize(instrument=instrument_class.getName(), universe=universe)
708
707
  return DataCoordinate.make_empty(universe)
@@ -893,7 +892,7 @@ class Pipeline:
893
892
  raise NameError(f"Label {label} does not appear in this pipeline")
894
893
  taskClass: type[PipelineTask] = doImportType(taskIR.klass)
895
894
  config = taskClass.ConfigClass()
896
- instrument: PipeBaseInstrument | None = None
895
+ instrument: Instrument | None = None
897
896
  if (instrumentName := self._pipelineIR.instrument) is not None:
898
897
  instrument_cls: type = doImportType(instrumentName)
899
898
  instrument = instrument_cls()
@@ -220,12 +220,6 @@ class LabeledSubset:
220
220
  class ParametersIR:
221
221
  """Intermediate representation of parameters that are global to a pipeline.
222
222
 
223
- Attributes
224
- ----------
225
- mapping : `dict` [`str`, `str`]
226
- A mutable mapping of identifiers as keys, and shared configuration
227
- as values.
228
-
229
223
  Notes
230
224
  -----
231
225
  These parameters are specified under a top level key named ``parameters``
@@ -706,7 +700,7 @@ class PipelineIR:
706
700
 
707
701
  Parameters
708
702
  ----------
709
- loaded_yaml: `dict`
703
+ loaded_yaml : `dict`
710
704
  A dictionary which matches the structure that would be produced
711
705
  by a yaml reader which parses a pipeline definition document
712
706
  """
@@ -55,7 +55,7 @@ class PipelineTask(Task):
55
55
  resulting data is also stored in a data butler.
56
56
 
57
57
  PipelineTask inherits from a `~lsst.pipe.base.Task` and uses the same
58
- configuration mechanism based on :ref:`lsst.pex.config`. `PipelineTask`
58
+ configuration mechanism based on `lsst.pex.config`. `PipelineTask`
59
59
  classes also have a `PipelineTaskConnections` class associated with their
60
60
  config which defines all of the IO a `PipelineTask` will need to do.
61
61
  PipelineTask sub-class typically implements `run()` method which receives
@@ -75,12 +75,6 @@ class PipelineTask(Task):
75
75
  PipelineTask base class constructor, but may support other signatures as
76
76
  well.
77
77
 
78
- Attributes
79
- ----------
80
- canMultiprocess : bool, True by default (class attribute)
81
- This class attribute is checked by execution framework, sub-classes
82
- can set it to ``False`` in case task does not support multiprocessing.
83
-
84
78
  Parameters
85
79
  ----------
86
80
  config : `~lsst.pex.config.Config`, optional
@@ -102,7 +96,11 @@ class PipelineTask(Task):
102
96
  """
103
97
 
104
98
  ConfigClass: ClassVar[type[PipelineTaskConfig]]
99
+
105
100
  canMultiprocess: ClassVar[bool] = True
101
+ """Whether this task can be run by an executor that uses subprocesses for
102
+ parallelism.
103
+ """
106
104
 
107
105
  def __init__(
108
106
  self,
@@ -106,8 +106,8 @@ class DatasetTypeNode:
106
106
  The internal networkx graph.
107
107
  get_registered : `~collections.abc.Callable` or `None`
108
108
  Callable that takes a dataset type name and returns the
109
- `DatasetType` registered in the data repository, or `None` if it is
110
- not registered.
109
+ `~lsst.daf.butler.DatasetType` registered in the data repository,
110
+ or `None` if it is not registered.
111
111
  dimensions : `lsst.daf.butler.DimensionUniverse`
112
112
  Definitions of all dimensions.
113
113
  previous : `DatasetTypeNode` or `None`
@@ -480,11 +480,11 @@ class ReadEdge(Edge):
480
480
  Parameters
481
481
  ----------
482
482
  current : `lsst.daf.butler.DatasetType` or `None`
483
- The current graph-wide `DatasetType`, or `None`. This will always
484
- be the registry's definition of the parent dataset type, if one
485
- exists. If not, it will be the dataset type definition from the
486
- task in the graph that writes it, if there is one. If there is no
487
- such task, this will be `None`.
483
+ The current graph-wide `~lsst.daf.butler.DatasetType`, or `None`.
484
+ This will always be the registry's definition of the parent dataset
485
+ type, if one exists. If not, it will be the dataset type
486
+ definition from the task in the graph that writes it, if there is
487
+ one. If there is no such task, this will be `None`.
488
488
  is_initial_query_constraint : `bool`
489
489
  Whether this dataset type is currently marked as a constraint on
490
490
  the initial data ID query in QuantumGraph generation.
@@ -496,7 +496,7 @@ class ReadEdge(Edge):
496
496
  producer : `str` or `None`
497
497
  The label of the task that produces this dataset type in the
498
498
  pipeline, or `None` if it is an overall input.
499
- consumers : `Sequence` [ `str` ]
499
+ consumers : `~collections.abc.Sequence` [ `str` ]
500
500
  Labels for other consuming tasks that have already participated in
501
501
  this dataset type's resolution.
502
502
  is_registered : `bool`
@@ -512,7 +512,7 @@ class ReadEdge(Edge):
512
512
 
513
513
  Returns
514
514
  -------
515
- dataset_type : `DatasetType`
515
+ dataset_type : `~lsst.daf.butler.DatasetType`
516
516
  The updated graph-wide dataset type. If ``current`` was provided,
517
517
  this must be equal to it.
518
518
  is_initial_query_constraint : `bool`
@@ -659,13 +659,25 @@ class ReadEdge(Edge):
659
659
  # compatible), since neither connection should take
660
660
  # precedence.
661
661
  if dataset_type != current:
662
- raise MissingDatasetTypeError(
663
- f"Definitions differ for input dataset type {self.parent_dataset_type_name!r}; "
664
- f"task {self.task_label!r} has {dataset_type}, but the definition "
665
- f"from {report_current_origin()} is {current}. If the storage classes are "
666
- "compatible but different, registering the dataset type in the data repository "
667
- "in advance will avoid this error."
668
- )
662
+ if visualization_only and dataset_type.dimensions == current.dimensions:
663
+ # Make a visualization-only ambiguous storage class
664
+ # "name".
665
+ all_storage_classes = set(current.storageClass_name.split("/"))
666
+ all_storage_classes.update(dataset_type.storageClass_name.split("/"))
667
+ current = DatasetType(
668
+ current.name,
669
+ current.dimensions,
670
+ "/".join(sorted(all_storage_classes)),
671
+ )
672
+ else:
673
+ raise MissingDatasetTypeError(
674
+ f"Definitions differ for input dataset type "
675
+ f"{self.parent_dataset_type_name!r}; task {self.task_label!r} has "
676
+ f"{dataset_type}, but the definition from {report_current_origin()} is "
677
+ f"{current}. If the storage classes are compatible but different, "
678
+ "registering the dataset type in the data repository in advance will avoid "
679
+ "this error."
680
+ )
669
681
  elif not visualization_only and not dataset_type.is_compatible_with(current):
670
682
  raise IncompatibleDatasetTypeError(
671
683
  f"Incompatible definition for input dataset type {self.parent_dataset_type_name!r}; "
@@ -788,15 +800,15 @@ class WriteEdge(Edge):
788
800
  Parameters
789
801
  ----------
790
802
  current : `lsst.daf.butler.DatasetType` or `None`
791
- The current graph-wide `DatasetType`, or `None`. This will always
792
- be the registry's definition of the parent dataset type, if one
793
- exists.
803
+ The current graph-wide `~lsst.daf.butler.DatasetType`, or `None`.
804
+ This will always be the registry's definition of the parent dataset
805
+ type, if one exists.
794
806
  universe : `lsst.daf.butler.DimensionUniverse`
795
807
  Object that holds all dimension definitions.
796
808
 
797
809
  Returns
798
810
  -------
799
- dataset_type : `DatasetType`
811
+ dataset_type : `~lsst.daf.butler.DatasetType`
800
812
  A dataset type compatible with this edge. If ``current`` was
801
813
  provided, this must be equal to it.
802
814
 
@@ -897,6 +897,10 @@ class PipelineGraph:
897
897
  New config objects or overrides to apply to copies of the current
898
898
  config objects, with task labels as the keywords.
899
899
 
900
+ Returns
901
+ -------
902
+ None
903
+
900
904
  Raises
901
905
  ------
902
906
  ValueError
@@ -1632,7 +1636,7 @@ class PipelineGraph:
1632
1636
 
1633
1637
  Returns
1634
1638
  -------
1635
- subgraphs : `Iterable` [ `PipelineGraph` ]
1639
+ subgraphs : `~collections.abc.Iterable` [ `PipelineGraph` ]
1636
1640
  An iterable over component subgraphs that could be run
1637
1641
  independently (they have only overall inputs in common). May be a
1638
1642
  lazy iterator.
@@ -1755,6 +1759,10 @@ class PipelineGraph:
1755
1759
  not considered part of the pipeline graph in other respects, but it
1756
1760
  does get written with other provenance datasets).
1757
1761
 
1762
+ Returns
1763
+ -------
1764
+ None
1765
+
1758
1766
  Raises
1759
1767
  ------
1760
1768
  lsst.daf.butler.MissingDatasetTypeError
@@ -2228,7 +2236,7 @@ class PipelineGraph:
2228
2236
 
2229
2237
  Parameters
2230
2238
  ----------
2231
- updates : `Mapping` [ `str`, `TaskNode` ]
2239
+ updates : `~collections.abc.Mapping` [ `str`, `TaskNode` ]
2232
2240
  New task nodes with task label keys. All keys must be task labels
2233
2241
  that are already present in the graph.
2234
2242
  check_edges_unchanged : `bool`, optional