lsst-pipe-base 30.0.0__py3-none-any.whl → 30.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/pipe/base/_instrument.py +5 -6
- lsst/pipe/base/log_capture.py +79 -39
- lsst/pipe/base/mp_graph_executor.py +15 -51
- lsst/pipe/base/quantum_graph/_common.py +3 -4
- lsst/pipe/base/quantum_graph/_multiblock.py +16 -6
- lsst/pipe/base/quantum_graph/_predicted.py +12 -106
- lsst/pipe/base/quantum_graph/_provenance.py +6 -657
- lsst/pipe/base/quantum_graph/aggregator/_communicators.py +50 -18
- lsst/pipe/base/quantum_graph/aggregator/_scanner.py +229 -35
- lsst/pipe/base/quantum_graph/aggregator/_structs.py +113 -3
- lsst/pipe/base/quantum_graph/aggregator/_supervisor.py +5 -10
- lsst/pipe/base/quantum_graph/aggregator/_writer.py +348 -31
- lsst/pipe/base/quantum_graph_builder.py +1 -12
- lsst/pipe/base/quantum_graph_executor.py +13 -116
- lsst/pipe/base/quantum_graph_skeleton.py +7 -1
- lsst/pipe/base/separable_pipeline_executor.py +2 -18
- lsst/pipe/base/single_quantum_executor.py +35 -53
- lsst/pipe/base/version.py +1 -1
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/METADATA +1 -1
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/RECORD +28 -30
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/WHEEL +1 -1
- lsst/pipe/base/log_on_close.py +0 -79
- lsst/pipe/base/quantum_graph/formatter.py +0 -101
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/entry_points.txt +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/LICENSE +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/bsd_license.txt +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/gpl-v3.0.txt +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/top_level.txt +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/zip-safe +0 -0
|
@@ -27,113 +27,23 @@
|
|
|
27
27
|
|
|
28
28
|
from __future__ import annotations
|
|
29
29
|
|
|
30
|
-
__all__ = ["
|
|
30
|
+
__all__ = ["QuantumExecutor", "QuantumGraphExecutor"]
|
|
31
31
|
|
|
32
32
|
from abc import ABC, abstractmethod
|
|
33
|
-
from typing import TYPE_CHECKING
|
|
34
|
-
|
|
35
|
-
from lsst.daf.butler import Quantum
|
|
33
|
+
from typing import TYPE_CHECKING
|
|
36
34
|
|
|
37
35
|
from .quantum_reports import QuantumReport, Report
|
|
38
36
|
|
|
39
37
|
if TYPE_CHECKING:
|
|
40
38
|
import uuid
|
|
41
39
|
|
|
42
|
-
from lsst.daf.butler
|
|
40
|
+
from lsst.daf.butler import Quantum
|
|
43
41
|
|
|
44
|
-
from ._task_metadata import TaskMetadata
|
|
45
42
|
from .graph import QuantumGraph
|
|
46
43
|
from .pipeline_graph import TaskNode
|
|
47
44
|
from .quantum_graph import PredictedQuantumGraph
|
|
48
45
|
|
|
49
46
|
|
|
50
|
-
class QuantumExecutionResult(tuple[Quantum, QuantumReport | None]):
|
|
51
|
-
"""A result struct that captures information about a single quantum's
|
|
52
|
-
execution.
|
|
53
|
-
|
|
54
|
-
Parameters
|
|
55
|
-
----------
|
|
56
|
-
quantum : `lsst.daf.butler.Quantum`
|
|
57
|
-
Quantum that was executed.
|
|
58
|
-
report : `.quantum_reports.QuantumReport`
|
|
59
|
-
Report with basic information about the execution.
|
|
60
|
-
task_metadata : `TaskMetadata`, optional
|
|
61
|
-
Metadata saved by the task and executor during execution.
|
|
62
|
-
skipped_existing : `bool`, optional
|
|
63
|
-
If `True`, this quantum was not executed because it appeared to have
|
|
64
|
-
already been executed successfully.
|
|
65
|
-
adjusted_no_work : `bool`, optional
|
|
66
|
-
If `True`, this quantum was not executed because the
|
|
67
|
-
`PipelineTaskConnections.adjustQuanta` hook raised `NoWorkFound`.
|
|
68
|
-
|
|
69
|
-
Notes
|
|
70
|
-
-----
|
|
71
|
-
For backwards compatibility, this class is a two-element tuple that allows
|
|
72
|
-
the ``quantum`` and ``report`` attributes to be unpacked. Additional
|
|
73
|
-
regular attributes may be added by executors (but the tuple must remain
|
|
74
|
-
only two elements to enable the current unpacking interface).
|
|
75
|
-
"""
|
|
76
|
-
|
|
77
|
-
def __new__(
|
|
78
|
-
cls,
|
|
79
|
-
quantum: Quantum,
|
|
80
|
-
report: QuantumReport | None,
|
|
81
|
-
*,
|
|
82
|
-
task_metadata: TaskMetadata | None = None,
|
|
83
|
-
skipped_existing: bool | None = None,
|
|
84
|
-
adjusted_no_work: bool | None = None,
|
|
85
|
-
) -> Self:
|
|
86
|
-
return super().__new__(cls, (quantum, report))
|
|
87
|
-
|
|
88
|
-
# We need to define both __init__ and __new__ because tuple inheritance
|
|
89
|
-
# requires __new__ and numpydoc requires __init__.
|
|
90
|
-
|
|
91
|
-
def __init__(
|
|
92
|
-
self,
|
|
93
|
-
quantum: Quantum,
|
|
94
|
-
report: QuantumReport | None,
|
|
95
|
-
*,
|
|
96
|
-
task_metadata: TaskMetadata | None = None,
|
|
97
|
-
skipped_existing: bool | None = None,
|
|
98
|
-
adjusted_no_work: bool | None = None,
|
|
99
|
-
):
|
|
100
|
-
self._task_metadata = task_metadata
|
|
101
|
-
self._skipped_existing = skipped_existing
|
|
102
|
-
self._adjusted_no_work = adjusted_no_work
|
|
103
|
-
|
|
104
|
-
@property
|
|
105
|
-
def quantum(self) -> Quantum:
|
|
106
|
-
"""The quantum actually executed."""
|
|
107
|
-
return self[0]
|
|
108
|
-
|
|
109
|
-
@property
|
|
110
|
-
def report(self) -> QuantumReport | None:
|
|
111
|
-
"""Structure describing the status of the execution of a quantum.
|
|
112
|
-
|
|
113
|
-
This is `None` if the implementation does not support this feature.
|
|
114
|
-
"""
|
|
115
|
-
return self[1]
|
|
116
|
-
|
|
117
|
-
@property
|
|
118
|
-
def task_metadata(self) -> TaskMetadata | None:
|
|
119
|
-
"""Metadata saved by the task and executor during execution."""
|
|
120
|
-
return self._task_metadata
|
|
121
|
-
|
|
122
|
-
@property
|
|
123
|
-
def skipped_existing(self) -> bool | None:
|
|
124
|
-
"""If `True`, this quantum was not executed because it appeared to have
|
|
125
|
-
already been executed successfully.
|
|
126
|
-
"""
|
|
127
|
-
return self._skipped_existing
|
|
128
|
-
|
|
129
|
-
@property
|
|
130
|
-
def adjusted_no_work(self) -> bool | None:
|
|
131
|
-
"""If `True`, this quantum was not executed because the
|
|
132
|
-
`PipelineTaskConnections.adjustQuanta` hook raised `NoWorkFound`.
|
|
133
|
-
"""
|
|
134
|
-
return self._adjusted_no_work
|
|
135
|
-
|
|
136
|
-
|
|
137
47
|
class QuantumExecutor(ABC):
|
|
138
48
|
"""Class which abstracts execution of a single Quantum.
|
|
139
49
|
|
|
@@ -145,14 +55,8 @@ class QuantumExecutor(ABC):
|
|
|
145
55
|
|
|
146
56
|
@abstractmethod
|
|
147
57
|
def execute(
|
|
148
|
-
self,
|
|
149
|
-
|
|
150
|
-
/,
|
|
151
|
-
quantum: Quantum,
|
|
152
|
-
quantum_id: uuid.UUID | None = None,
|
|
153
|
-
*,
|
|
154
|
-
log_records: ButlerLogRecords | None = None,
|
|
155
|
-
) -> QuantumExecutionResult:
|
|
58
|
+
self, task_node: TaskNode, /, quantum: Quantum, quantum_id: uuid.UUID | None = None
|
|
59
|
+
) -> tuple[Quantum, QuantumReport | None]:
|
|
156
60
|
"""Execute single quantum.
|
|
157
61
|
|
|
158
62
|
Parameters
|
|
@@ -163,18 +67,15 @@ class QuantumExecutor(ABC):
|
|
|
163
67
|
Quantum for this execution.
|
|
164
68
|
quantum_id : `uuid.UUID` or `None`, optional
|
|
165
69
|
The ID of the quantum to be executed.
|
|
166
|
-
log_records : `lsst.daf.butler.ButlerLogRecords`, optional
|
|
167
|
-
Container that should be used to store logs in memory before
|
|
168
|
-
writing them to the butler. This disables streaming log (since
|
|
169
|
-
we'd have to store them in memory anyway), but it permits the
|
|
170
|
-
caller to prepend logs to be stored in the butler and allows task
|
|
171
|
-
logs to be inspected by the caller after execution is complete.
|
|
172
70
|
|
|
173
71
|
Returns
|
|
174
72
|
-------
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
73
|
+
quantum : `~lsst.daf.butler.Quantum`
|
|
74
|
+
The quantum actually executed.
|
|
75
|
+
report : `~.quantum_reports.QuantumReport`
|
|
76
|
+
Structure describing the status of the execution of a quantum.
|
|
77
|
+
`None` is returned if implementation does not support this
|
|
78
|
+
feature.
|
|
178
79
|
|
|
179
80
|
Notes
|
|
180
81
|
-----
|
|
@@ -192,9 +93,7 @@ class QuantumGraphExecutor(ABC):
|
|
|
192
93
|
"""
|
|
193
94
|
|
|
194
95
|
@abstractmethod
|
|
195
|
-
def execute(
|
|
196
|
-
self, graph: QuantumGraph | PredictedQuantumGraph, *, provenance_graph_file: str | None = None
|
|
197
|
-
) -> None:
|
|
96
|
+
def execute(self, graph: QuantumGraph | PredictedQuantumGraph) -> None:
|
|
198
97
|
"""Execute whole graph.
|
|
199
98
|
|
|
200
99
|
Implementation of this method depends on particular execution model
|
|
@@ -204,10 +103,8 @@ class QuantumGraphExecutor(ABC):
|
|
|
204
103
|
|
|
205
104
|
Parameters
|
|
206
105
|
----------
|
|
207
|
-
graph : `.QuantumGraph`
|
|
106
|
+
graph : `.QuantumGraph`
|
|
208
107
|
Execution graph.
|
|
209
|
-
provenance_graph_file : `str`, optional
|
|
210
|
-
A filename to write provenance to.
|
|
211
108
|
"""
|
|
212
109
|
raise NotImplementedError()
|
|
213
110
|
|
|
@@ -383,6 +383,12 @@ class QuantumGraphSkeleton:
|
|
|
383
383
|
The dataset ref of the prerequisite.
|
|
384
384
|
**attrs : `~typing.Any`
|
|
385
385
|
Additional attributes for the node.
|
|
386
|
+
|
|
387
|
+
Notes
|
|
388
|
+
-----
|
|
389
|
+
This automatically sets the 'existing_input' ref attribute (see
|
|
390
|
+
`set_existing_input_ref`), since prerequisites are always overall
|
|
391
|
+
inputs.
|
|
386
392
|
"""
|
|
387
393
|
key = PrerequisiteDatasetKey(ref.datasetType.name, ref.id.bytes)
|
|
388
394
|
self._xgraph.add_node(key, data_id=ref.dataId, ref=ref, **attrs)
|
|
@@ -600,7 +606,7 @@ class QuantumGraphSkeleton:
|
|
|
600
606
|
|
|
601
607
|
def set_output_in_the_way(self, ref: DatasetRef) -> None:
|
|
602
608
|
"""Associate a dataset node with a `DatasetRef` that represents an
|
|
603
|
-
existing output in the output RUN
|
|
609
|
+
existing output in the output RUN collectoin.
|
|
604
610
|
|
|
605
611
|
Parameters
|
|
606
612
|
----------
|
|
@@ -40,8 +40,7 @@ from collections.abc import Iterable
|
|
|
40
40
|
from typing import Any
|
|
41
41
|
|
|
42
42
|
import lsst.resources
|
|
43
|
-
from lsst.daf.butler import Butler
|
|
44
|
-
from lsst.daf.butler._rubin.temporary_for_ingest import TemporaryForIngest
|
|
43
|
+
from lsst.daf.butler import Butler
|
|
45
44
|
|
|
46
45
|
from ._quantumContext import ExecutionResources
|
|
47
46
|
from .all_dimensions_quantum_graph_builder import AllDimensionsQuantumGraphBuilder
|
|
@@ -363,8 +362,6 @@ class SeparablePipelineExecutor:
|
|
|
363
362
|
fail_fast: bool = False,
|
|
364
363
|
graph_executor: QuantumGraphExecutor | None = None,
|
|
365
364
|
num_proc: int = 1,
|
|
366
|
-
*,
|
|
367
|
-
provenance_dataset_ref: DatasetRef | None = None,
|
|
368
365
|
) -> None:
|
|
369
366
|
"""Run a pipeline in the form of a prepared quantum graph.
|
|
370
367
|
|
|
@@ -387,14 +384,6 @@ class SeparablePipelineExecutor:
|
|
|
387
384
|
The number of processes that can be used to run the pipeline. The
|
|
388
385
|
default value ensures that no subprocess is created. Only used with
|
|
389
386
|
the default graph executor.
|
|
390
|
-
provenance_dataset_ref : `lsst.daf.butler.DatasetRef`, optional
|
|
391
|
-
Dataset that should be used to save provenance. Provenance is only
|
|
392
|
-
supported when running in a single process (at least for the
|
|
393
|
-
default quantum executor), and should not be used with
|
|
394
|
-
``skip_existing_in=[output_run]`` when retrying a previous
|
|
395
|
-
execution attempt. The caller is responsible for registering the
|
|
396
|
-
dataset type and for ensuring that the dimensions of this dataset
|
|
397
|
-
do not lead to uniqueness conflicts.
|
|
398
387
|
"""
|
|
399
388
|
if not graph_executor:
|
|
400
389
|
quantum_executor = SingleQuantumExecutor(
|
|
@@ -415,9 +404,4 @@ class SeparablePipelineExecutor:
|
|
|
415
404
|
# forked processes.
|
|
416
405
|
self._butler.registry.resetConnectionPool()
|
|
417
406
|
|
|
418
|
-
|
|
419
|
-
with TemporaryForIngest(self._butler, provenance_dataset_ref) as temporary:
|
|
420
|
-
graph_executor.execute(graph, provenance_graph_file=temporary.ospath)
|
|
421
|
-
temporary.ingest()
|
|
422
|
-
else:
|
|
423
|
-
graph_executor.execute(graph)
|
|
407
|
+
graph_executor.execute(graph)
|
|
@@ -37,13 +37,13 @@ from typing import Any
|
|
|
37
37
|
|
|
38
38
|
from lsst.daf.butler import (
|
|
39
39
|
Butler,
|
|
40
|
+
ButlerMetrics,
|
|
40
41
|
DatasetRef,
|
|
41
42
|
DatasetType,
|
|
42
43
|
LimitedButler,
|
|
43
44
|
NamedKeyDict,
|
|
44
45
|
Quantum,
|
|
45
46
|
)
|
|
46
|
-
from lsst.daf.butler.logging import ButlerLogRecords
|
|
47
47
|
from lsst.utils.introspection import get_full_type_name
|
|
48
48
|
from lsst.utils.timer import logInfo
|
|
49
49
|
|
|
@@ -59,7 +59,7 @@ from .connections import AdjustQuantumHelper
|
|
|
59
59
|
from .log_capture import LogCapture, _ExecutionLogRecordsExtra
|
|
60
60
|
from .pipeline_graph import TaskNode
|
|
61
61
|
from .pipelineTask import PipelineTask
|
|
62
|
-
from .quantum_graph_executor import
|
|
62
|
+
from .quantum_graph_executor import QuantumExecutor
|
|
63
63
|
from .quantum_reports import QuantumReport
|
|
64
64
|
from .task import _TASK_FULL_METADATA_TYPE, _TASK_METADATA_TYPE
|
|
65
65
|
from .taskFactory import TaskFactory
|
|
@@ -157,28 +157,21 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
157
157
|
self._previous_process_quanta: list[uuid.UUID] = []
|
|
158
158
|
|
|
159
159
|
def execute(
|
|
160
|
-
self,
|
|
161
|
-
|
|
162
|
-
/,
|
|
163
|
-
quantum: Quantum,
|
|
164
|
-
quantum_id: uuid.UUID | None = None,
|
|
165
|
-
*,
|
|
166
|
-
log_records: ButlerLogRecords | None = None,
|
|
167
|
-
) -> QuantumExecutionResult:
|
|
160
|
+
self, task_node: TaskNode, /, quantum: Quantum, quantum_id: uuid.UUID | None = None
|
|
161
|
+
) -> tuple[Quantum, QuantumReport | None]:
|
|
168
162
|
# Docstring inherited from QuantumExecutor.execute
|
|
163
|
+
assert quantum.dataId is not None, "Quantum DataId cannot be None"
|
|
164
|
+
|
|
169
165
|
if self._butler is not None:
|
|
170
166
|
self._butler.registry.refresh()
|
|
171
|
-
|
|
167
|
+
|
|
168
|
+
result = self._execute(task_node, quantum, quantum_id=quantum_id)
|
|
169
|
+
report = QuantumReport(quantumId=quantum_id, dataId=quantum.dataId, taskLabel=task_node.label)
|
|
170
|
+
return result, report
|
|
172
171
|
|
|
173
172
|
def _execute(
|
|
174
|
-
self,
|
|
175
|
-
|
|
176
|
-
/,
|
|
177
|
-
quantum: Quantum,
|
|
178
|
-
quantum_id: uuid.UUID | None = None,
|
|
179
|
-
*,
|
|
180
|
-
log_records: ButlerLogRecords | None = None,
|
|
181
|
-
) -> QuantumExecutionResult:
|
|
173
|
+
self, task_node: TaskNode, /, quantum: Quantum, quantum_id: uuid.UUID | None = None
|
|
174
|
+
) -> Quantum:
|
|
182
175
|
"""Execute the quantum.
|
|
183
176
|
|
|
184
177
|
Internal implementation of `execute()`.
|
|
@@ -196,7 +189,7 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
196
189
|
|
|
197
190
|
try:
|
|
198
191
|
return self._execute_with_limited_butler(
|
|
199
|
-
task_node, limited_butler, quantum=quantum, quantum_id=quantum_id
|
|
192
|
+
task_node, limited_butler, quantum=quantum, quantum_id=quantum_id
|
|
200
193
|
)
|
|
201
194
|
finally:
|
|
202
195
|
if used_butler_factory:
|
|
@@ -209,17 +202,14 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
209
202
|
/,
|
|
210
203
|
quantum: Quantum,
|
|
211
204
|
quantum_id: uuid.UUID | None = None,
|
|
212
|
-
|
|
213
|
-
log_records: ButlerLogRecords | None = None,
|
|
214
|
-
) -> QuantumExecutionResult:
|
|
205
|
+
) -> Quantum:
|
|
215
206
|
startTime = time.time()
|
|
216
|
-
|
|
217
|
-
report = QuantumReport(quantumId=quantum_id, dataId=quantum.dataId, taskLabel=task_node.label)
|
|
207
|
+
|
|
218
208
|
if self._butler is not None:
|
|
219
209
|
log_capture = LogCapture.from_full(self._butler)
|
|
220
210
|
else:
|
|
221
211
|
log_capture = LogCapture.from_limited(limited_butler)
|
|
222
|
-
with log_capture.capture_logging(task_node, quantum
|
|
212
|
+
with log_capture.capture_logging(task_node, quantum) as captureLog:
|
|
223
213
|
# Save detailed resource usage before task start to metadata.
|
|
224
214
|
quantumMetadata = _TASK_METADATA_TYPE()
|
|
225
215
|
logInfo(None, "prep", metadata=quantumMetadata) # type: ignore[arg-type]
|
|
@@ -238,7 +228,7 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
238
228
|
task_node.label,
|
|
239
229
|
quantum.dataId,
|
|
240
230
|
)
|
|
241
|
-
return
|
|
231
|
+
return quantum
|
|
242
232
|
captureLog.store = True
|
|
243
233
|
|
|
244
234
|
captureLog.extra.previous_process_quanta.extend(self._previous_process_quanta)
|
|
@@ -266,7 +256,7 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
266
256
|
if self._job_metadata is not None:
|
|
267
257
|
fullMetadata["job"] = self._job_metadata
|
|
268
258
|
self._write_metadata(quantum, fullMetadata, task_node, limited_butler)
|
|
269
|
-
return
|
|
259
|
+
return quantum
|
|
270
260
|
|
|
271
261
|
# enable lsstDebug debugging
|
|
272
262
|
if self._enable_lsst_debug:
|
|
@@ -288,12 +278,10 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
288
278
|
)
|
|
289
279
|
task = self._task_factory.makeTask(task_node, limited_butler, init_input_refs)
|
|
290
280
|
logInfo(None, "start", metadata=quantumMetadata) # type: ignore[arg-type]
|
|
291
|
-
outputs_put: list[uuid.UUID] = []
|
|
292
281
|
try:
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
)
|
|
282
|
+
caveats, outputsPut, butler_metrics = self._run_quantum(
|
|
283
|
+
task, quantum, task_node, limited_butler, quantum_id=quantum_id
|
|
284
|
+
)
|
|
297
285
|
except Exception as e:
|
|
298
286
|
_LOG.error(
|
|
299
287
|
"Execution of task '%s' on quantum %s failed. Exception %s: %s",
|
|
@@ -313,10 +301,10 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
313
301
|
quantumMetadata["caveats"] = caveats.value
|
|
314
302
|
# Stringify the UUID for easier compatibility with
|
|
315
303
|
# PropertyList.
|
|
304
|
+
quantumMetadata["outputs"] = [str(output) for output in outputsPut]
|
|
316
305
|
finally:
|
|
317
306
|
logInfo(None, "end", metadata=quantumMetadata) # type: ignore[arg-type]
|
|
318
307
|
fullMetadata = task.getFullMetadata()
|
|
319
|
-
quantumMetadata["outputs"] = [str(output) for output in outputs_put]
|
|
320
308
|
fullMetadata["quantum"] = quantumMetadata
|
|
321
309
|
if self._job_metadata is not None:
|
|
322
310
|
fullMetadata["job"] = self._job_metadata
|
|
@@ -329,13 +317,7 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
329
317
|
quantum.dataId,
|
|
330
318
|
stopTime - startTime,
|
|
331
319
|
)
|
|
332
|
-
return
|
|
333
|
-
quantum,
|
|
334
|
-
report,
|
|
335
|
-
skipped_existing=False,
|
|
336
|
-
adjusted_no_work=False,
|
|
337
|
-
task_metadata=fullMetadata,
|
|
338
|
-
)
|
|
320
|
+
return quantum
|
|
339
321
|
|
|
340
322
|
def _check_existing_outputs(
|
|
341
323
|
self,
|
|
@@ -537,9 +519,8 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
537
519
|
task_node: TaskNode,
|
|
538
520
|
/,
|
|
539
521
|
limited_butler: LimitedButler,
|
|
540
|
-
quantum_id: uuid.UUID | None,
|
|
541
|
-
|
|
542
|
-
) -> QuantumSuccessCaveats:
|
|
522
|
+
quantum_id: uuid.UUID | None = None,
|
|
523
|
+
) -> tuple[QuantumSuccessCaveats, list[uuid.UUID], ButlerMetrics]:
|
|
543
524
|
"""Execute task on a single quantum.
|
|
544
525
|
|
|
545
526
|
Parameters
|
|
@@ -552,17 +533,18 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
552
533
|
Task definition structure.
|
|
553
534
|
limited_butler : `~lsst.daf.butler.LimitedButler`
|
|
554
535
|
Butler to use for dataset I/O.
|
|
555
|
-
quantum_id : `uuid.UUID` or `None
|
|
536
|
+
quantum_id : `uuid.UUID` or `None`, optional
|
|
556
537
|
ID of the quantum being executed.
|
|
557
|
-
ids_put : list[ `uuid.UUID` ]
|
|
558
|
-
List to be populated with the dataset IDs that were written by this
|
|
559
|
-
quantum. This is an output parameter in order to allow it to be
|
|
560
|
-
populated even when an exception is raised.
|
|
561
538
|
|
|
562
539
|
Returns
|
|
563
540
|
-------
|
|
564
541
|
flags : `QuantumSuccessCaveats`
|
|
565
542
|
Flags that describe qualified successes.
|
|
543
|
+
ids_put : list[ `uuid.UUID` ]
|
|
544
|
+
Record of all the dataset IDs that were written by this quantum
|
|
545
|
+
being executed.
|
|
546
|
+
metrics : `lsst.daf.butler.ButlerMetrics`
|
|
547
|
+
Butler metrics recorded for this quantum.
|
|
566
548
|
"""
|
|
567
549
|
flags = QuantumSuccessCaveats.NO_CAVEATS
|
|
568
550
|
|
|
@@ -574,7 +556,8 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
574
556
|
|
|
575
557
|
# Call task runQuantum() method.
|
|
576
558
|
try:
|
|
577
|
-
|
|
559
|
+
with limited_butler.record_metrics() as butler_metrics:
|
|
560
|
+
task.runQuantum(butlerQC, inputRefs, outputRefs)
|
|
578
561
|
except NoWorkFound as err:
|
|
579
562
|
# Not an error, just an early exit.
|
|
580
563
|
_LOG.info(
|
|
@@ -612,13 +595,12 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
612
595
|
)
|
|
613
596
|
_LOG.error(error, exc_info=error)
|
|
614
597
|
flags |= caught.FLAGS
|
|
615
|
-
finally:
|
|
616
|
-
ids_put.extend(output[2] for output in butlerQC.outputsPut)
|
|
617
598
|
if not butlerQC.outputsPut:
|
|
618
599
|
flags |= QuantumSuccessCaveats.ALL_OUTPUTS_MISSING
|
|
619
600
|
if not butlerQC.outputsPut == butlerQC.allOutputs:
|
|
620
601
|
flags |= QuantumSuccessCaveats.ANY_OUTPUTS_MISSING
|
|
621
|
-
|
|
602
|
+
ids_put = [output[2] for output in butlerQC.outputsPut]
|
|
603
|
+
return flags, ids_put, butler_metrics
|
|
622
604
|
|
|
623
605
|
def _write_metadata(
|
|
624
606
|
self, quantum: Quantum, metadata: Any, task_node: TaskNode, /, limited_butler: LimitedButler
|
lsst/pipe/base/version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
__all__ = ["__version__"]
|
|
2
|
-
__version__ = "30.0.
|
|
2
|
+
__version__ = "30.0.0rc1"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-pipe-base
|
|
3
|
-
Version: 30.0.
|
|
3
|
+
Version: 30.0.0rc1
|
|
4
4
|
Summary: Pipeline infrastructure for the Rubin Science Pipelines.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License-Expression: BSD-3-Clause OR GPL-3.0-or-later
|
|
@@ -3,7 +3,7 @@ lsst/pipe/__init__.py,sha256=_2bZAHuDVAx7MM7KA7pt3DYp641NY4RzSoRAwesWKfU,67
|
|
|
3
3
|
lsst/pipe/base/__init__.py,sha256=qBLN0yYQjIcLBLb4jFKM_ppopuqTnCehcUdFcEe69Js,970
|
|
4
4
|
lsst/pipe/base/_datasetQueryConstraints.py,sha256=bFH0_lVc49NS2_4v_i6r9POr500c0K-OHLMhMX5FjkQ,6373
|
|
5
5
|
lsst/pipe/base/_dataset_handle.py,sha256=ft_ke1LbhLLndDPARsHSQJUA05LgUFnfWOq2vbwH3wI,11353
|
|
6
|
-
lsst/pipe/base/_instrument.py,sha256=
|
|
6
|
+
lsst/pipe/base/_instrument.py,sha256=I9UTaj81krR1zkTZ1owfOPBzHN29PY3Egg7fIE5obxQ,30057
|
|
7
7
|
lsst/pipe/base/_observation_dimension_packer.py,sha256=78Jg2OVFOdXIK62TS2Y3X4095xqCzmiIx9o4TXyADYA,8027
|
|
8
8
|
lsst/pipe/base/_quantumContext.py,sha256=gb60mTHbgOIEptYvJ64SaChvViXyeKJlG6kEHq4nYVw,19345
|
|
9
9
|
lsst/pipe/base/_status.py,sha256=7gJPrqt03t1fO5tREc_sfQaN7XmDBvruihORXPnRqUE,21216
|
|
@@ -20,30 +20,29 @@ lsst/pipe/base/exec_fixup_data_id.py,sha256=9OjOcH-6AHZ1JnD_CemieI0wWX90J_VdaY9v
|
|
|
20
20
|
lsst/pipe/base/execution_graph_fixup.py,sha256=ND0x4hlpeEW-gudo-i2K7HT7MoM5sp_mcoqRMCopSqQ,3815
|
|
21
21
|
lsst/pipe/base/execution_reports.py,sha256=jYtWCD4PkEAeVUpKIxuiJJVgsCm7qiwCorWVgNHkVgU,17270
|
|
22
22
|
lsst/pipe/base/graph_walker.py,sha256=Ij7JfYF0srA29VgM_DhbNBxUBeOHDOnujrTQPjVNha0,4694
|
|
23
|
-
lsst/pipe/base/log_capture.py,sha256=
|
|
24
|
-
lsst/pipe/base/log_on_close.py,sha256=rauQYvsWQF9GuxoaZRjPJJ7VjATGyRUo0RyHEoUVTnQ,2666
|
|
23
|
+
lsst/pipe/base/log_capture.py,sha256=5r99_Ek2A75vYOMo-z52ltWLdfYfWExm55UU9a4nqmM,12909
|
|
25
24
|
lsst/pipe/base/mermaid_tools.py,sha256=cdlDJQ1x8k7-VvCLEUqvSC3GR1zCsB-aUTxOjYejNWc,5216
|
|
26
|
-
lsst/pipe/base/mp_graph_executor.py,sha256=
|
|
25
|
+
lsst/pipe/base/mp_graph_executor.py,sha256=FKlFxjtU2-6SFzX_wsdtMMAj5P09ZE8V65-Sg585g2Y,35501
|
|
27
26
|
lsst/pipe/base/pipeline.py,sha256=FVaiLhgw9Pzo-nzXKS0dLNafegP0AMZKLtPlSvOSkRU,37563
|
|
28
27
|
lsst/pipe/base/pipelineIR.py,sha256=UuZ02NLhVmzzekbuWlyar7cPLCf_4yfzD5qFEmGHs_A,45821
|
|
29
28
|
lsst/pipe/base/pipelineTask.py,sha256=K3GdjJLvy8A7I-jzQiERQZaYF7mC1LM3iB5TmUtbOCI,8394
|
|
30
29
|
lsst/pipe/base/prerequisite_helpers.py,sha256=bmiebQ4veSrypZgAXjmCBFfj8fUtPW9eRQaVShhxdBQ,28446
|
|
31
30
|
lsst/pipe/base/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
|
-
lsst/pipe/base/quantum_graph_builder.py,sha256=
|
|
33
|
-
lsst/pipe/base/quantum_graph_executor.py,sha256
|
|
34
|
-
lsst/pipe/base/quantum_graph_skeleton.py,sha256=
|
|
31
|
+
lsst/pipe/base/quantum_graph_builder.py,sha256=nJsrGx5P4NLTBZo2ZXGpHO8JU05yZkJYxKxWk3ZecEI,67940
|
|
32
|
+
lsst/pipe/base/quantum_graph_executor.py,sha256=WP41iQmihy1jfgaHV6eu2aSrqQx_Fydq3mbEF6CLQ-s,4419
|
|
33
|
+
lsst/pipe/base/quantum_graph_skeleton.py,sha256=GhSQjRHaErneGY4A4E0tERqg9QPEeYrlpmdLzqFXy6E,28586
|
|
35
34
|
lsst/pipe/base/quantum_provenance_graph.py,sha256=33S5iCVxD9Co4oJSU_N8AJXL14Nw0UwGzPEc3gpQiqk,91981
|
|
36
35
|
lsst/pipe/base/quantum_reports.py,sha256=ut235L88v7SXaeVUvMA9qFl7tpeMwGnzob3X0QoOI_s,14210
|
|
37
36
|
lsst/pipe/base/resource_usage.py,sha256=LfH7Qf6taI3lxw0aB90riRMn1UxUTMBSqtBjKPJ-XuY,6759
|
|
38
|
-
lsst/pipe/base/separable_pipeline_executor.py,sha256=
|
|
37
|
+
lsst/pipe/base/separable_pipeline_executor.py,sha256=vXqJrRI5GNezzGV9QsiaRHEhioDF2Y_W7JQYQCzHR7A,16720
|
|
39
38
|
lsst/pipe/base/simple_pipeline_executor.py,sha256=_HGeAOgHUiRdQwsmD_qira-vcHZXlLbf_LmhLJffh2U,29563
|
|
40
|
-
lsst/pipe/base/single_quantum_executor.py,sha256=
|
|
39
|
+
lsst/pipe/base/single_quantum_executor.py,sha256=yCpDS_eHsJTOu34mHYv8DJjj8UG0GjPdCuXdb74A2p8,28543
|
|
41
40
|
lsst/pipe/base/struct.py,sha256=Fa-UkpuXOxdzKWbHrMUkJYOszZuBXCm2NesXNR0IOPQ,5048
|
|
42
41
|
lsst/pipe/base/task.py,sha256=XHBd-7m1a4-6LgobBYA1DgY4H7EV-_RWKfxbhZbMmD4,15145
|
|
43
42
|
lsst/pipe/base/taskFactory.py,sha256=MsDGECJqZLSZk8SGhpuVhNaP32UWuNvxZiDcZExPFG8,3412
|
|
44
43
|
lsst/pipe/base/testUtils.py,sha256=lSBKMhoKflbi8JkMNYfEqqHNl-rtFI8UYT3QneDYpLo,18477
|
|
45
44
|
lsst/pipe/base/utils.py,sha256=JmEt3l0xrh9uayKrSXuQEq12aXOhDr2YXmbYduaxCko,1940
|
|
46
|
-
lsst/pipe/base/version.py,sha256
|
|
45
|
+
lsst/pipe/base/version.py,sha256=vSgpHsF0PG5ox9cJDNTmDJ8a4r5c8EKOc2gxM3iszgs,52
|
|
47
46
|
lsst/pipe/base/cli/__init__.py,sha256=861tXIAW7SqtqNUYkjbeEdfg8lDswXsjJQca0gVCFz4,54
|
|
48
47
|
lsst/pipe/base/cli/_get_cli_subcommands.py,sha256=g_af64klRybBGKAg7fmBSZBdw2LYBAsFON_yQIMZON0,1289
|
|
49
48
|
lsst/pipe/base/cli/cmd/__init__.py,sha256=3UF2IQEEBor4YMGRNPdcZAVCAI5yFyeHp5nGul4IoyM,1557
|
|
@@ -83,21 +82,20 @@ lsst/pipe/base/pipeline_graph/visualization/_printer.py,sha256=yJMRJ-aXd3nYDgs1F
|
|
|
83
82
|
lsst/pipe/base/pipeline_graph/visualization/_show.py,sha256=lPRjO1To2n5r3f_Wgcwy-7TmyJ7UszGGFXAlOtN1wDs,10510
|
|
84
83
|
lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py,sha256=dp7PXl9Cu7GfWjBi5g8KjXZgnF1KGg_idKKxtICL53Q,8679
|
|
85
84
|
lsst/pipe/base/quantum_graph/__init__.py,sha256=-Gp3LihB0AXCvhG387wKAEpHRM-NrHGSXMti8cHee90,1437
|
|
86
|
-
lsst/pipe/base/quantum_graph/_common.py,sha256=
|
|
87
|
-
lsst/pipe/base/quantum_graph/_multiblock.py,sha256=
|
|
88
|
-
lsst/pipe/base/quantum_graph/_predicted.py,sha256=
|
|
89
|
-
lsst/pipe/base/quantum_graph/_provenance.py,sha256=
|
|
90
|
-
lsst/pipe/base/quantum_graph/formatter.py,sha256=PZhn6I7HAdBjhm-_vcWoXdhwm9hDsK5QQ07GnIGJtPU,3948
|
|
85
|
+
lsst/pipe/base/quantum_graph/_common.py,sha256=tgQsIylvs5wSAKs3SVp5_XJ7XEBJwBcRUbBGR2EFuwU,22714
|
|
86
|
+
lsst/pipe/base/quantum_graph/_multiblock.py,sha256=kc2pqezfU7xOAxYZa41mz3G4X5w4GVfovw3ZcjNgK2Y,27950
|
|
87
|
+
lsst/pipe/base/quantum_graph/_predicted.py,sha256=VoJev2xEnNcXA1ar_cM7paUfAP5DrE95FWjgqy4FU78,87901
|
|
88
|
+
lsst/pipe/base/quantum_graph/_provenance.py,sha256=R1SoRxyi6cNi9fXSc7ybrbri9anuYnliJ6SBmVUrh40,55102
|
|
91
89
|
lsst/pipe/base/quantum_graph/visualization.py,sha256=EbTWhk9aPq7sX6bcHmnEIsr2xuuR6d1SxspQbRe8D0Q,12235
|
|
92
90
|
lsst/pipe/base/quantum_graph/aggregator/__init__.py,sha256=4CK8sP_ZjUKmxKS3LnCH1zG7XSk9IEwijrluRBHhEMU,7436
|
|
93
|
-
lsst/pipe/base/quantum_graph/aggregator/_communicators.py,sha256=
|
|
91
|
+
lsst/pipe/base/quantum_graph/aggregator/_communicators.py,sha256=w7hNEQjE1Qjhi6u4Tx4wS_1RrV3-WQRCz1FACd-vmro,36396
|
|
94
92
|
lsst/pipe/base/quantum_graph/aggregator/_config.py,sha256=iV1Ejfk-UnFoQ8TkpJE_jMZYHsmZcdLm5R-FnQEqO7s,5167
|
|
95
93
|
lsst/pipe/base/quantum_graph/aggregator/_ingester.py,sha256=lhu8I0R3IwoLkVcrqF1ypoHAFamNapSoMcG9aYFjbQ4,14158
|
|
96
94
|
lsst/pipe/base/quantum_graph/aggregator/_progress.py,sha256=jiz9Np73uUQ03CtH7pI6TXxXrwFUChh5hSj_gbMGHr0,7207
|
|
97
|
-
lsst/pipe/base/quantum_graph/aggregator/_scanner.py,sha256=
|
|
98
|
-
lsst/pipe/base/quantum_graph/aggregator/_structs.py,sha256=
|
|
99
|
-
lsst/pipe/base/quantum_graph/aggregator/_supervisor.py,sha256=
|
|
100
|
-
lsst/pipe/base/quantum_graph/aggregator/_writer.py,sha256=
|
|
95
|
+
lsst/pipe/base/quantum_graph/aggregator/_scanner.py,sha256=4wg2cgVcAoqyuZZUBogHXWkSr9X6sVwnJLu_o96D204,22758
|
|
96
|
+
lsst/pipe/base/quantum_graph/aggregator/_structs.py,sha256=bM-MejZz5DFUFQCH8kZDykuMBmBJYbQMl15nBm-3dtc,5499
|
|
97
|
+
lsst/pipe/base/quantum_graph/aggregator/_supervisor.py,sha256=5-6CshU9RFyVEgYI3qJwuVEGw8YtT9Ei7cNkcghXzMk,9290
|
|
98
|
+
lsst/pipe/base/quantum_graph/aggregator/_writer.py,sha256=fYQrJGCg3MQQQF1feWXCLyG6m5ucnzFrIINawWgnF8g,21504
|
|
101
99
|
lsst/pipe/base/script/__init__.py,sha256=cLEXE7aq5UZ0juL_ScmRw0weFgp4tDgwEX_ts-NEYic,1522
|
|
102
100
|
lsst/pipe/base/script/register_instrument.py,sha256=neQ2MTPtAiV_Hl2yatQ8-vQC24xHjhpI7VJUHf5kPX4,2445
|
|
103
101
|
lsst/pipe/base/script/retrieve_artifacts_for_quanta.py,sha256=Cr0HpzXm_C3LnIOQg5tNJht02O6xoqtWWIphjugasMA,3957
|
|
@@ -115,13 +113,13 @@ lsst/pipe/base/tests/mocks/_data_id_match.py,sha256=jVekStcrItC0tqOCc01VjYaiE9ex
|
|
|
115
113
|
lsst/pipe/base/tests/mocks/_pipeline_task.py,sha256=N3fC4OMAMWWnYtyLkVdMfb9ZiFse39HniRDvlAOofOY,30691
|
|
116
114
|
lsst/pipe/base/tests/mocks/_repo.py,sha256=SH-jzynS-H2xc_3GLjF7ln-kHdRoSeVVaal5qLd2hXI,28359
|
|
117
115
|
lsst/pipe/base/tests/mocks/_storage_class.py,sha256=12IFfJMbZ5GkYlMX6ZMWiG8pMZc2Jlxke3qQW-bljdU,27434
|
|
118
|
-
lsst_pipe_base-30.0.
|
|
119
|
-
lsst_pipe_base-30.0.
|
|
120
|
-
lsst_pipe_base-30.0.
|
|
121
|
-
lsst_pipe_base-30.0.
|
|
122
|
-
lsst_pipe_base-30.0.
|
|
123
|
-
lsst_pipe_base-30.0.
|
|
124
|
-
lsst_pipe_base-30.0.
|
|
125
|
-
lsst_pipe_base-30.0.
|
|
126
|
-
lsst_pipe_base-30.0.
|
|
127
|
-
lsst_pipe_base-30.0.
|
|
116
|
+
lsst_pipe_base-30.0.0rc1.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
|
|
117
|
+
lsst_pipe_base-30.0.0rc1.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
|
|
118
|
+
lsst_pipe_base-30.0.0rc1.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
|
|
119
|
+
lsst_pipe_base-30.0.0rc1.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
120
|
+
lsst_pipe_base-30.0.0rc1.dist-info/METADATA,sha256=V0tPgd6mwgckUdkx-NVCXHuNJ6H3uYxDXI6DW3YGs9c,2254
|
|
121
|
+
lsst_pipe_base-30.0.0rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
122
|
+
lsst_pipe_base-30.0.0rc1.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
|
|
123
|
+
lsst_pipe_base-30.0.0rc1.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
|
|
124
|
+
lsst_pipe_base-30.0.0rc1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
125
|
+
lsst_pipe_base-30.0.0rc1.dist-info/RECORD,,
|