lsst-ctrl-mpexec 29.2025.3900__py3-none-any.whl → 29.2025.4000__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/ctrl/mpexec/_pipeline_graph_factory.py +21 -3
- lsst/ctrl/mpexec/cli/cmd/commands.py +17 -46
- lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py +25 -6
- lsst/ctrl/mpexec/cli/script/qgraph.py +67 -30
- lsst/ctrl/mpexec/cli/script/run.py +14 -15
- lsst/ctrl/mpexec/cli/script/run_qbb.py +13 -12
- lsst/ctrl/mpexec/cli/script/update_graph_run.py +17 -13
- lsst/ctrl/mpexec/cli/utils.py +64 -13
- lsst/ctrl/mpexec/showInfo.py +36 -32
- lsst/ctrl/mpexec/singleQuantumExecutor.py +1 -1
- lsst/ctrl/mpexec/version.py +1 -1
- {lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/METADATA +1 -1
- {lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/RECORD +21 -21
- {lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/WHEEL +0 -0
- {lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/entry_points.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/licenses/LICENSE +0 -0
- {lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/licenses/bsd_license.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/licenses/gpl-v3.0.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/top_level.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/zip-safe +0 -0
|
@@ -45,18 +45,30 @@ class PipelineGraphFactory:
|
|
|
45
45
|
dimension schema.
|
|
46
46
|
select_tasks : `str`, optional
|
|
47
47
|
String expression that filters the tasks in the pipeline graph.
|
|
48
|
+
pipeline_graph : `lsst.pipe.base.pipeline_graph.PipelineGraph`, optional
|
|
49
|
+
Already-constructed pipeline graph.
|
|
48
50
|
"""
|
|
49
51
|
|
|
50
|
-
def __init__(
|
|
52
|
+
def __init__(
|
|
53
|
+
self,
|
|
54
|
+
pipeline: Pipeline | None = None,
|
|
55
|
+
butler: Butler | None = None,
|
|
56
|
+
select_tasks: str = "",
|
|
57
|
+
*,
|
|
58
|
+
pipeline_graph: PipelineGraph | None = None,
|
|
59
|
+
):
|
|
60
|
+
if pipeline is None and pipeline_graph is None:
|
|
61
|
+
raise TypeError("At least one of 'pipeline' and 'pipeline_graph' must not be `None`.")
|
|
51
62
|
self._pipeline = pipeline
|
|
52
63
|
self._registry = butler.registry if butler is not None else None
|
|
53
64
|
self._select_tasks = select_tasks
|
|
54
|
-
self._pipeline_graph: PipelineGraph | None =
|
|
65
|
+
self._pipeline_graph: PipelineGraph | None = pipeline_graph
|
|
55
66
|
self._resolved: bool = False
|
|
56
67
|
self._for_visualization_only: bool = False
|
|
57
68
|
|
|
58
69
|
def __call__(self, *, resolve: bool = True, visualization_only: bool = False) -> PipelineGraph:
|
|
59
70
|
if self._pipeline_graph is None:
|
|
71
|
+
assert self._pipeline is not None, "Guaranteed at construction."
|
|
60
72
|
self._pipeline_graph = self._pipeline.to_graph()
|
|
61
73
|
if self._select_tasks:
|
|
62
74
|
self._pipeline_graph = self._pipeline_graph.select(self._select_tasks)
|
|
@@ -71,6 +83,8 @@ class PipelineGraphFactory:
|
|
|
71
83
|
@property
|
|
72
84
|
def pipeline(self) -> Pipeline:
|
|
73
85
|
"""The original pipeline definition."""
|
|
86
|
+
if self._pipeline is None:
|
|
87
|
+
raise RuntimeError("Cannot obtain pipeline from pipeline graph.")
|
|
74
88
|
if self._select_tasks:
|
|
75
89
|
raise RuntimeError(
|
|
76
90
|
"The --select-tasks option cannot be used with operations that return or display a "
|
|
@@ -79,4 +93,8 @@ class PipelineGraphFactory:
|
|
|
79
93
|
return self._pipeline
|
|
80
94
|
|
|
81
95
|
def __bool__(self) -> bool:
|
|
82
|
-
|
|
96
|
+
if self._pipeline is not None:
|
|
97
|
+
return bool(self._pipeline)
|
|
98
|
+
else:
|
|
99
|
+
assert self._pipeline_graph is not None, "Guaranteed at construction."
|
|
100
|
+
return bool(self._pipeline_graph.tasks)
|
|
@@ -35,25 +35,22 @@ from typing import Any
|
|
|
35
35
|
|
|
36
36
|
import click
|
|
37
37
|
|
|
38
|
-
import lsst.pipe.base.cli.opt as pipeBaseOpts
|
|
39
38
|
from lsst.ctrl.mpexec.showInfo import ShowInfo
|
|
40
39
|
from lsst.daf.butler.cli.opt import (
|
|
41
40
|
collections_option,
|
|
42
|
-
config_file_option,
|
|
43
|
-
config_option,
|
|
44
41
|
confirm_option,
|
|
45
42
|
options_file_option,
|
|
46
43
|
processes_option,
|
|
47
44
|
repo_argument,
|
|
48
45
|
where_option,
|
|
49
46
|
)
|
|
50
|
-
from lsst.daf.butler.cli.utils import
|
|
47
|
+
from lsst.daf.butler.cli.utils import catch_and_exit, option_section, unwrap
|
|
51
48
|
from lsst.pipe.base.quantum_reports import Report
|
|
52
49
|
|
|
53
50
|
from .. import opt as ctrlMpExecOpts
|
|
54
51
|
from .. import script
|
|
55
52
|
from ..script import confirmable
|
|
56
|
-
from ..utils import PipetaskCommand,
|
|
53
|
+
from ..utils import PipetaskCommand, collect_pipeline_actions
|
|
57
54
|
|
|
58
55
|
epilog = unwrap(
|
|
59
56
|
"""Notes:
|
|
@@ -69,41 +66,6 @@ ignored.)
|
|
|
69
66
|
)
|
|
70
67
|
|
|
71
68
|
|
|
72
|
-
def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]:
|
|
73
|
-
"""Extract pipeline building options, replace them with PipelineActions,
|
|
74
|
-
return updated `kwargs`.
|
|
75
|
-
|
|
76
|
-
Notes
|
|
77
|
-
-----
|
|
78
|
-
The pipeline actions (task, delete, config, config_file, and instrument)
|
|
79
|
-
must be handled in the order they appear on the command line, but the CLI
|
|
80
|
-
specification gives them all different option names. So, instead of using
|
|
81
|
-
the individual action options as they appear in kwargs (because
|
|
82
|
-
invocation order can't be known), we capture the CLI arguments by
|
|
83
|
-
overriding `click.Command.parse_args` and save them in the Context's
|
|
84
|
-
`obj` parameter. We use `makePipelineActions` to create a list of
|
|
85
|
-
pipeline actions from the CLI arguments and pass that list to the script
|
|
86
|
-
function using the `pipeline_actions` kwarg name, and remove the action
|
|
87
|
-
options from kwargs.
|
|
88
|
-
"""
|
|
89
|
-
for pipelineAction in (
|
|
90
|
-
ctrlMpExecOpts.task_option.name(),
|
|
91
|
-
ctrlMpExecOpts.delete_option.name(),
|
|
92
|
-
config_option.name(),
|
|
93
|
-
config_file_option.name(),
|
|
94
|
-
pipeBaseOpts.instrument_option.name(),
|
|
95
|
-
):
|
|
96
|
-
kwargs.pop(pipelineAction)
|
|
97
|
-
|
|
98
|
-
actions = makePipelineActions(MWCtxObj.getFrom(ctx).args)
|
|
99
|
-
pipeline_actions = []
|
|
100
|
-
for action in actions:
|
|
101
|
-
pipeline_actions.append(action)
|
|
102
|
-
|
|
103
|
-
kwargs["pipeline_actions"] = pipeline_actions
|
|
104
|
-
return kwargs
|
|
105
|
-
|
|
106
|
-
|
|
107
69
|
def _unhandledShow(show: ShowInfo, cmd: str) -> None:
|
|
108
70
|
if show.unhandled:
|
|
109
71
|
print(
|
|
@@ -125,7 +87,7 @@ def build(ctx: click.Context, **kwargs: Any) -> None:
|
|
|
125
87
|
|
|
126
88
|
This does not require input data to be specified.
|
|
127
89
|
"""
|
|
128
|
-
kwargs =
|
|
90
|
+
kwargs = collect_pipeline_actions(ctx, **kwargs)
|
|
129
91
|
show = ShowInfo(kwargs.pop("show", []))
|
|
130
92
|
if kwargs.get("butler_config") is not None and (
|
|
131
93
|
{"pipeline-graph", "task-graph"}.isdisjoint(show.commands) and not kwargs.get("pipeline_dot")
|
|
@@ -187,7 +149,7 @@ concurrency = multiprocessing
|
|
|
187
149
|
@catch_and_exit
|
|
188
150
|
def qgraph(ctx: click.Context, **kwargs: Any) -> None:
|
|
189
151
|
"""Build and optionally save quantum graph."""
|
|
190
|
-
kwargs =
|
|
152
|
+
kwargs = collect_pipeline_actions(ctx, **kwargs)
|
|
191
153
|
summary = kwargs.pop("summary", None)
|
|
192
154
|
with coverage_context(kwargs):
|
|
193
155
|
show = ShowInfo(kwargs.pop("show", []))
|
|
@@ -204,12 +166,21 @@ def qgraph(ctx: click.Context, **kwargs: Any) -> None:
|
|
|
204
166
|
file=sys.stderr,
|
|
205
167
|
)
|
|
206
168
|
return
|
|
207
|
-
if (
|
|
169
|
+
if (
|
|
170
|
+
qgraph := script.qgraph(
|
|
171
|
+
pipeline_graph_factory,
|
|
172
|
+
**kwargs,
|
|
173
|
+
show=show,
|
|
174
|
+
# Making a summary report requires that we load the same graph
|
|
175
|
+
# components as execution.
|
|
176
|
+
for_execution=(summary is not None),
|
|
177
|
+
)
|
|
178
|
+
) is None:
|
|
208
179
|
raise click.ClickException("QuantumGraph was empty; ERROR logs above should provide details.")
|
|
209
180
|
# QuantumGraph-only summary call here since script.qgraph also called
|
|
210
181
|
# by run methods.
|
|
211
182
|
if summary:
|
|
212
|
-
report = Report(qgraphSummary=qgraph.
|
|
183
|
+
report = Report(qgraphSummary=qgraph._make_summary())
|
|
213
184
|
with open(summary, "w") as out:
|
|
214
185
|
# Do not save fields that are not set.
|
|
215
186
|
out.write(report.model_dump_json(exclude_none=True, indent=2))
|
|
@@ -222,7 +193,7 @@ def qgraph(ctx: click.Context, **kwargs: Any) -> None:
|
|
|
222
193
|
@catch_and_exit
|
|
223
194
|
def run(ctx: click.Context, **kwargs: Any) -> None:
|
|
224
195
|
"""Build and execute pipeline and quantum graph."""
|
|
225
|
-
kwargs =
|
|
196
|
+
kwargs = collect_pipeline_actions(ctx, **kwargs)
|
|
226
197
|
with coverage_context(kwargs):
|
|
227
198
|
show = ShowInfo(kwargs.pop("show", []))
|
|
228
199
|
pipeline_graph_factory = script.build(**kwargs, show=show)
|
|
@@ -233,7 +204,7 @@ def run(ctx: click.Context, **kwargs: Any) -> None:
|
|
|
233
204
|
file=sys.stderr,
|
|
234
205
|
)
|
|
235
206
|
return
|
|
236
|
-
if (qgraph := script.qgraph(pipeline_graph_factory, **kwargs, show=show)) is None:
|
|
207
|
+
if (qgraph := script.qgraph(pipeline_graph_factory, for_execution=True, **kwargs, show=show)) is None:
|
|
237
208
|
raise click.ClickException("QuantumGraph was empty; ERROR logs above should provide details.")
|
|
238
209
|
_unhandledShow(show, "run")
|
|
239
210
|
if show.handled:
|
|
@@ -28,6 +28,9 @@
|
|
|
28
28
|
from __future__ import annotations
|
|
29
29
|
|
|
30
30
|
from lsst.pipe.base import BuildId, QuantumGraph
|
|
31
|
+
from lsst.pipe.base.pipeline_graph import TaskImportMode
|
|
32
|
+
from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
|
|
33
|
+
from lsst.resources import ResourcePath, ResourcePathExpression
|
|
31
34
|
from lsst.utils.logging import getLogger
|
|
32
35
|
|
|
33
36
|
from ..butler_factory import ButlerFactory
|
|
@@ -37,7 +40,7 @@ _LOG = getLogger(__name__)
|
|
|
37
40
|
|
|
38
41
|
def pre_exec_init_qbb(
|
|
39
42
|
butler_config: str,
|
|
40
|
-
qgraph:
|
|
43
|
+
qgraph: ResourcePathExpression,
|
|
41
44
|
qgraph_id: str | None,
|
|
42
45
|
config_search_path: list[str] | None,
|
|
43
46
|
**kwargs: object,
|
|
@@ -64,11 +67,27 @@ def pre_exec_init_qbb(
|
|
|
64
67
|
function and pass all the option kwargs to each of the script functions
|
|
65
68
|
which ignore these unused kwargs.
|
|
66
69
|
"""
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
70
|
+
qgraph = ResourcePath(qgraph)
|
|
71
|
+
match qgraph.getExtension():
|
|
72
|
+
case ".qgraph":
|
|
73
|
+
_LOG.verbose("Reading full quantum graph from %s.", qgraph)
|
|
74
|
+
qg = PredictedQuantumGraph.from_old_quantum_graph(
|
|
75
|
+
QuantumGraph.loadUri(
|
|
76
|
+
qgraph,
|
|
77
|
+
graphID=BuildId(qgraph_id) if qgraph_id is not None else None,
|
|
78
|
+
)
|
|
79
|
+
)
|
|
80
|
+
case ".qg":
|
|
81
|
+
_LOG.verbose("Reading init quanta from quantum graph from %s.", qgraph)
|
|
82
|
+
if qgraph_id is not None:
|
|
83
|
+
_LOG.warning("--qgraph-id is ignored when loading new '.qg' files.")
|
|
84
|
+
with PredictedQuantumGraph.open(
|
|
85
|
+
qgraph, import_mode=TaskImportMode.ASSUME_CONSISTENT_EDGES
|
|
86
|
+
) as reader:
|
|
87
|
+
reader.read_init_quanta()
|
|
88
|
+
qg = reader.finish()
|
|
89
|
+
case ext:
|
|
90
|
+
raise ValueError(f"Unrecognized extension for quantum graph: {ext!r}")
|
|
72
91
|
|
|
73
92
|
# Ensure that QBB uses shared datastore cache for writes.
|
|
74
93
|
ButlerFactory.define_datastore_cache()
|
|
@@ -42,6 +42,8 @@ from lsst.pipe.base.all_dimensions_quantum_graph_builder import (
|
|
|
42
42
|
)
|
|
43
43
|
from lsst.pipe.base.dot_tools import graph2dot
|
|
44
44
|
from lsst.pipe.base.mermaid_tools import graph2mermaid
|
|
45
|
+
from lsst.pipe.base.pipeline_graph import TaskImportMode
|
|
46
|
+
from lsst.pipe.base.quantum_graph import PredictedQuantumGraph, PredictedQuantumGraphComponents
|
|
45
47
|
from lsst.resources import ResourcePath, ResourcePathExpression
|
|
46
48
|
from lsst.utils.iteration import ensure_iterable
|
|
47
49
|
from lsst.utils.logging import getLogger
|
|
@@ -60,7 +62,7 @@ _LOG = getLogger(__name__)
|
|
|
60
62
|
def qgraph(
|
|
61
63
|
pipeline_graph_factory: PipelineGraphFactory | None,
|
|
62
64
|
*,
|
|
63
|
-
qgraph:
|
|
65
|
+
qgraph: ResourcePathExpression | None,
|
|
64
66
|
qgraph_id: str | None,
|
|
65
67
|
qgraph_node_id: Iterable[uuid.UUID | str] | None,
|
|
66
68
|
qgraph_datastore_records: bool,
|
|
@@ -85,8 +87,10 @@ def qgraph(
|
|
|
85
87
|
mock: bool = False,
|
|
86
88
|
unmocked_dataset_types: Sequence[str],
|
|
87
89
|
mock_failure: Mapping[str, ForcedFailure],
|
|
90
|
+
for_execution: bool = False,
|
|
91
|
+
for_init_output_run: bool = False,
|
|
88
92
|
**kwargs: object,
|
|
89
|
-
) ->
|
|
93
|
+
) -> PredictedQuantumGraph | None:
|
|
90
94
|
"""Implement the command line interface `pipetask qgraph` subcommand.
|
|
91
95
|
|
|
92
96
|
Should only be called by command line tools and unit test code that test
|
|
@@ -97,8 +101,7 @@ def qgraph(
|
|
|
97
101
|
pipeline_graph_factory : `..PipelineGraphFactory` or `None`
|
|
98
102
|
A factory that holds the pipeline and can produce a pipeline graph.
|
|
99
103
|
If this is not `None` then `qgraph` should be `None`.
|
|
100
|
-
qgraph : `lsst.
|
|
101
|
-
`lsst.resources.ResourcePath` or `None`
|
|
104
|
+
qgraph : convertible to `lsst.resources.ResourcePath`, or `None`
|
|
102
105
|
URI location for a serialized quantum graph definition. If this option
|
|
103
106
|
is not `None` then ``pipeline_graph_factory`` should be `None`.
|
|
104
107
|
qgraph_id : `str` or `None`
|
|
@@ -179,15 +182,23 @@ def qgraph(
|
|
|
179
182
|
List of overall-input dataset types that should not be mocked.
|
|
180
183
|
mock_failure : `~collections.abc.Mapping`
|
|
181
184
|
Quanta that should raise exceptions.
|
|
182
|
-
|
|
185
|
+
for_execution : `bool`, optional
|
|
186
|
+
If `True`, the script is being used to feed another that will execute
|
|
187
|
+
the given quanta, and hence all information needed for execution must
|
|
188
|
+
be loaded.
|
|
189
|
+
for_init_output_run : `bool`, optional
|
|
190
|
+
If `True`, the script is being used to feed another that will
|
|
191
|
+
initialize the output run, and hence all information needed to do so
|
|
192
|
+
must be loaded.
|
|
193
|
+
**kwargs : `dict` [`str`, `str`]
|
|
183
194
|
Ignored; click commands may accept options for more than one script
|
|
184
195
|
function and pass all the option kwargs to each of the script functions
|
|
185
196
|
which ignore these unused kwargs.
|
|
186
197
|
|
|
187
198
|
Returns
|
|
188
199
|
-------
|
|
189
|
-
|
|
190
|
-
The
|
|
200
|
+
qg : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
|
|
201
|
+
The quantum graph object that was created or loaded.
|
|
191
202
|
"""
|
|
192
203
|
# make sure that --extend-run always enables --skip-existing
|
|
193
204
|
if extend_run:
|
|
@@ -213,16 +224,43 @@ def qgraph(
|
|
|
213
224
|
if skip_existing and run:
|
|
214
225
|
skip_existing_in += (run,)
|
|
215
226
|
|
|
227
|
+
qgc: PredictedQuantumGraphComponents
|
|
216
228
|
if qgraph is not None:
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
229
|
+
# click passes empty tuple as default value for qgraph_node_id
|
|
230
|
+
quantum_ids = (
|
|
231
|
+
{uuid.UUID(q) if not isinstance(q, uuid.UUID) else q for q in qgraph_node_id}
|
|
232
|
+
if qgraph_node_id
|
|
233
|
+
else None
|
|
234
|
+
)
|
|
235
|
+
qgraph = ResourcePath(qgraph)
|
|
236
|
+
match qgraph.getExtension():
|
|
237
|
+
case ".qgraph":
|
|
238
|
+
qgc = PredictedQuantumGraphComponents.from_old_quantum_graph(
|
|
239
|
+
QuantumGraph.loadUri(
|
|
240
|
+
qgraph,
|
|
241
|
+
butler.dimensions,
|
|
242
|
+
nodes=quantum_ids,
|
|
243
|
+
graphID=BuildId(qgraph_id) if qgraph_id is not None else None,
|
|
244
|
+
)
|
|
245
|
+
)
|
|
246
|
+
case ".qg":
|
|
247
|
+
if qgraph_id is not None:
|
|
248
|
+
_LOG.warning("--qgraph-id is ignored when loading new '.qg' files.")
|
|
249
|
+
if for_execution or for_init_output_run or save_qgraph or show.needs_full_qg:
|
|
250
|
+
import_mode = TaskImportMode.ASSUME_CONSISTENT_EDGES
|
|
251
|
+
else:
|
|
252
|
+
import_mode = TaskImportMode.DO_NOT_IMPORT
|
|
253
|
+
with PredictedQuantumGraph.open(qgraph, import_mode=import_mode) as reader:
|
|
254
|
+
if for_execution or qgraph_dot or qgraph_mermaid or show.needs_full_qg:
|
|
255
|
+
# This reads everything for the given quanta.
|
|
256
|
+
reader.read_execution_quanta(quantum_ids)
|
|
257
|
+
elif for_init_output_run:
|
|
258
|
+
reader.read_init_quanta()
|
|
259
|
+
else:
|
|
260
|
+
reader.read_thin_graph()
|
|
261
|
+
qgc = reader.components
|
|
262
|
+
case ext:
|
|
263
|
+
raise ValueError(f"Unrecognized extension for quantum graph: {ext!r}")
|
|
226
264
|
|
|
227
265
|
# pipeline can not be provided in this case
|
|
228
266
|
if pipeline_graph_factory:
|
|
@@ -265,38 +303,37 @@ def qgraph(
|
|
|
265
303
|
output_run=run,
|
|
266
304
|
data_id_tables=data_id_tables,
|
|
267
305
|
)
|
|
268
|
-
#
|
|
306
|
+
# Accumulate metadata (QB builder adds some of its own).
|
|
269
307
|
metadata = {
|
|
270
|
-
"input": inputs,
|
|
271
|
-
"output": output,
|
|
272
308
|
"butler_argument": str(butler_config),
|
|
273
|
-
"output_run": run,
|
|
274
309
|
"extend_run": extend_run,
|
|
275
310
|
"skip_existing_in": skip_existing_in,
|
|
276
311
|
"skip_existing": skip_existing,
|
|
277
312
|
"data_query": data_query,
|
|
278
313
|
}
|
|
279
314
|
assert run is not None, "Butler output run collection must be defined"
|
|
280
|
-
|
|
315
|
+
qgc = graph_builder.finish(
|
|
316
|
+
output, metadata=metadata, attach_datastore_records=qgraph_datastore_records
|
|
317
|
+
)
|
|
281
318
|
|
|
282
|
-
if
|
|
283
|
-
# Nothing to do.
|
|
319
|
+
if not summarize_quantum_graph(qgc.header):
|
|
284
320
|
return None
|
|
285
|
-
summarize_quantum_graph(qgraph)
|
|
286
321
|
|
|
287
322
|
if save_qgraph:
|
|
288
|
-
_LOG.verbose("Writing
|
|
289
|
-
|
|
323
|
+
_LOG.verbose("Writing quantum graph to %r.", save_qgraph)
|
|
324
|
+
qgc.write(save_qgraph)
|
|
325
|
+
|
|
326
|
+
qg = qgc.assemble()
|
|
290
327
|
|
|
291
328
|
if qgraph_dot:
|
|
292
329
|
_LOG.verbose("Writing quantum graph DOT visualization to %r.", qgraph_dot)
|
|
293
|
-
graph2dot(
|
|
330
|
+
graph2dot(qg, qgraph_dot) # TODO[DM-51850]: make this work
|
|
294
331
|
|
|
295
332
|
if qgraph_mermaid:
|
|
296
333
|
_LOG.verbose("Writing quantum graph Mermaid visualization to %r.", qgraph_mermaid)
|
|
297
|
-
graph2mermaid(
|
|
334
|
+
graph2mermaid(qg, qgraph_mermaid) # TODO[DM-51850]: make this work
|
|
298
335
|
|
|
299
336
|
# optionally dump some info.
|
|
300
|
-
show.show_graph_info(
|
|
337
|
+
show.show_graph_info(qg, butler_config) # TODO[DM-51850]: make this work
|
|
301
338
|
|
|
302
|
-
return
|
|
339
|
+
return qg
|
|
@@ -33,8 +33,9 @@ from typing import TYPE_CHECKING, Literal
|
|
|
33
33
|
import astropy.units as u
|
|
34
34
|
|
|
35
35
|
import lsst.utils.timer
|
|
36
|
-
from lsst.pipe.base import ExecutionResources,
|
|
36
|
+
from lsst.pipe.base import ExecutionResources, TaskFactory
|
|
37
37
|
from lsst.pipe.base.mp_graph_executor import MPGraphExecutor
|
|
38
|
+
from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
|
|
38
39
|
from lsst.pipe.base.single_quantum_executor import SingleQuantumExecutor
|
|
39
40
|
from lsst.resources import ResourcePath, ResourcePathExpression
|
|
40
41
|
from lsst.utils.doImport import doImportType
|
|
@@ -52,7 +53,7 @@ _LOG = getLogger(__name__)
|
|
|
52
53
|
|
|
53
54
|
|
|
54
55
|
def run(
|
|
55
|
-
qg:
|
|
56
|
+
qg: PredictedQuantumGraph,
|
|
56
57
|
*,
|
|
57
58
|
task_factory: TaskFactory | None = None,
|
|
58
59
|
pdb: str | None,
|
|
@@ -93,8 +94,8 @@ def run(
|
|
|
93
94
|
|
|
94
95
|
Parameters
|
|
95
96
|
----------
|
|
96
|
-
qg : `lsst.pipe.base.
|
|
97
|
-
A
|
|
97
|
+
qg : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
|
|
98
|
+
A quantum graph generated by a previous subcommand.
|
|
98
99
|
task_factory : `lsst.pipe.base.TaskFactory`, optional
|
|
99
100
|
A custom task factory to use.
|
|
100
101
|
pdb : `str`, optional
|
|
@@ -211,19 +212,17 @@ def run(
|
|
|
211
212
|
enable_lsst_debug = debug
|
|
212
213
|
del debug
|
|
213
214
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
output_run = output_run
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
graph_output_run = qg.metadata.get("output_run", output_run)
|
|
223
|
-
if graph_output_run != output_run:
|
|
215
|
+
if not output_run:
|
|
216
|
+
# If we have no output run specified, use the one from the graph rather
|
|
217
|
+
# than letting a new timestamped run be created.
|
|
218
|
+
output_run = qg.header.output_run
|
|
219
|
+
else:
|
|
220
|
+
# Check that output run defined on command line is consistent with
|
|
221
|
+
# quantum graph.
|
|
222
|
+
if qg.header.output_run != output_run:
|
|
224
223
|
raise ValueError(
|
|
225
224
|
f"Output run defined on command line ({output_run}) has to be "
|
|
226
|
-
f"identical to graph metadata ({
|
|
225
|
+
f"identical to graph metadata ({qg.header.output_run}). "
|
|
227
226
|
"To update graph metadata run `pipetask update-graph-run` command."
|
|
228
227
|
)
|
|
229
228
|
|
|
@@ -43,8 +43,9 @@ from lsst.daf.butler import (
|
|
|
43
43
|
Quantum,
|
|
44
44
|
QuantumBackedButler,
|
|
45
45
|
)
|
|
46
|
-
from lsst.pipe.base import
|
|
46
|
+
from lsst.pipe.base import ExecutionResources, TaskFactory
|
|
47
47
|
from lsst.pipe.base.mp_graph_executor import MPGraphExecutor
|
|
48
|
+
from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
|
|
48
49
|
from lsst.pipe.base.single_quantum_executor import SingleQuantumExecutor
|
|
49
50
|
from lsst.resources import ResourcePath, ResourcePathExpression
|
|
50
51
|
from lsst.utils.logging import VERBOSE, getLogger
|
|
@@ -147,31 +148,31 @@ def run_qbb(
|
|
|
147
148
|
if not enable_implicit_threading:
|
|
148
149
|
disable_implicit_threading()
|
|
149
150
|
|
|
151
|
+
# click passes empty tuple as default value for qgraph_node_id
|
|
152
|
+
quantum_ids = (
|
|
153
|
+
{uuid.UUID(q) if not isinstance(q, uuid.UUID) else q for q in qgraph_node_id}
|
|
154
|
+
if qgraph_node_id
|
|
155
|
+
else None
|
|
156
|
+
)
|
|
150
157
|
# Load quantum graph.
|
|
151
|
-
nodes = qgraph_node_id or None
|
|
152
158
|
with lsst.utils.timer.time_this(
|
|
153
159
|
_LOG,
|
|
154
|
-
msg=f"Reading {str(len(
|
|
160
|
+
msg=f"Reading {str(len(quantum_ids)) if quantum_ids is not None else 'all'} quanta.",
|
|
155
161
|
level=VERBOSE,
|
|
156
162
|
) as qg_read_time:
|
|
157
|
-
qg =
|
|
158
|
-
qgraph, nodes=nodes, graphID=BuildId(qgraph_id) if qgraph_id is not None else None
|
|
159
|
-
)
|
|
163
|
+
qg = PredictedQuantumGraph.read_execution_quanta(qgraph, quantum_ids=quantum_ids)
|
|
160
164
|
job_metadata = {"qg_read_time": qg_read_time.duration, "qg_size": len(qg)}
|
|
161
165
|
|
|
162
|
-
|
|
163
|
-
raise ValueError("QuantumGraph is missing metadata, cannot continue.")
|
|
164
|
-
|
|
165
|
-
summarize_quantum_graph(qg)
|
|
166
|
+
summarize_quantum_graph(qg.header)
|
|
166
167
|
|
|
167
|
-
dataset_types = {
|
|
168
|
+
dataset_types = {dtn.name: dtn.dataset_type for dtn in qg.pipeline_graph.dataset_types.values()}
|
|
168
169
|
|
|
169
170
|
# Ensure that QBB uses shared datastore cache.
|
|
170
171
|
ButlerFactory.define_datastore_cache()
|
|
171
172
|
|
|
172
173
|
_butler_factory = _QBBFactory(
|
|
173
174
|
butler_config=butler_config,
|
|
174
|
-
dimensions=qg.universe,
|
|
175
|
+
dimensions=qg.pipeline_graph.universe,
|
|
175
176
|
dataset_types=dataset_types,
|
|
176
177
|
config_search_path=config_search_path,
|
|
177
178
|
)
|
|
@@ -25,16 +25,20 @@
|
|
|
25
25
|
# You should have received a copy of the GNU General Public License
|
|
26
26
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
import logging
|
|
29
|
+
|
|
30
|
+
from lsst.pipe.base.quantum_graph import PredictedQuantumGraphComponents
|
|
29
31
|
from lsst.resources import ResourcePathExpression
|
|
30
32
|
|
|
33
|
+
_LOG = logging.getLogger(__name__)
|
|
34
|
+
|
|
31
35
|
|
|
32
36
|
def update_graph_run(
|
|
33
37
|
input_graph: ResourcePathExpression,
|
|
34
38
|
run: str,
|
|
35
39
|
output_graph: ResourcePathExpression,
|
|
36
|
-
metadata_run_key: str,
|
|
37
|
-
update_graph_id: bool,
|
|
40
|
+
metadata_run_key: str = "output_run",
|
|
41
|
+
update_graph_id: bool = False,
|
|
38
42
|
) -> None:
|
|
39
43
|
"""Update quantum graph with new output run name and dataset IDs and save
|
|
40
44
|
updated graph to a file.
|
|
@@ -47,15 +51,15 @@ def update_graph_run(
|
|
|
47
51
|
Collection name, if collection exists it must be of ``RUN`` type.
|
|
48
52
|
output_graph : `~lsst.resources.ResourcePathExpression`
|
|
49
53
|
Location to store updated quantum graph.
|
|
50
|
-
metadata_run_key : `str
|
|
51
|
-
|
|
52
|
-
with new run name. If metadata is missing it is not
|
|
53
|
-
updated. If metadata is present but key is missing, it will be
|
|
54
|
-
added.
|
|
54
|
+
metadata_run_key : `str`, optional
|
|
55
|
+
Ignored (overriding warns).
|
|
55
56
|
update_graph_id : `bool`
|
|
56
|
-
|
|
57
|
+
Ignored (overriding warns).
|
|
57
58
|
"""
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
59
|
+
qgc = PredictedQuantumGraphComponents.read_execution_quanta(input_graph)
|
|
60
|
+
if metadata_run_key and metadata_run_key != "output_run":
|
|
61
|
+
_LOG.warning("--metadata-run-key is now ignored.")
|
|
62
|
+
if update_graph_id:
|
|
63
|
+
_LOG.warning("--update-graph-id is now ignored.")
|
|
64
|
+
qgc.update_output_run(run)
|
|
65
|
+
qgc.write(output_graph)
|
lsst/ctrl/mpexec/cli/utils.py
CHANGED
|
@@ -30,13 +30,15 @@ import collections
|
|
|
30
30
|
import contextlib
|
|
31
31
|
import logging
|
|
32
32
|
import re
|
|
33
|
+
from typing import Any
|
|
33
34
|
|
|
35
|
+
import click
|
|
34
36
|
from astropy.table import Table
|
|
35
37
|
|
|
36
38
|
from lsst.daf.butler.cli.opt import config_file_option, config_option
|
|
37
|
-
from lsst.daf.butler.cli.utils import MWCommand, split_commas
|
|
38
|
-
from lsst.pipe.base import QuantumGraph
|
|
39
|
+
from lsst.daf.butler.cli.utils import MWCommand, MWCtxObj, split_commas
|
|
39
40
|
from lsst.pipe.base.cli.opt import instrument_option
|
|
41
|
+
from lsst.pipe.base.quantum_graph import HeaderModel
|
|
40
42
|
from lsst.utils.logging import getLogger
|
|
41
43
|
|
|
42
44
|
from .opt import delete_option, task_option
|
|
@@ -103,7 +105,7 @@ _ACTION_CONFIG_FILE = _PipelineActionType("configfile", "(?P<label>.+):(?P<value
|
|
|
103
105
|
_ACTION_ADD_INSTRUMENT = _PipelineActionType("add_instrument", "(?P<value>[^:]+)")
|
|
104
106
|
|
|
105
107
|
|
|
106
|
-
def
|
|
108
|
+
def make_pipeline_actions(
|
|
107
109
|
args: list[str],
|
|
108
110
|
taskFlags: list[str] = task_option.opts(),
|
|
109
111
|
deleteFlags: list[str] = delete_option.opts(),
|
|
@@ -160,47 +162,96 @@ def makePipelineActions(
|
|
|
160
162
|
return pipelineActions
|
|
161
163
|
|
|
162
164
|
|
|
165
|
+
def collect_pipeline_actions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]:
|
|
166
|
+
"""Extract pipeline building options, replace them with PipelineActions,
|
|
167
|
+
return updated `kwargs`.
|
|
168
|
+
|
|
169
|
+
Parameters
|
|
170
|
+
----------
|
|
171
|
+
ctx : `click.Context`
|
|
172
|
+
Click context to extract actions from.
|
|
173
|
+
**kwargs : `object`
|
|
174
|
+
Keyword arguments to start from.
|
|
175
|
+
|
|
176
|
+
Returns
|
|
177
|
+
-------
|
|
178
|
+
kwargs : `dict`
|
|
179
|
+
Updated keyword arguments.
|
|
180
|
+
|
|
181
|
+
Notes
|
|
182
|
+
-----
|
|
183
|
+
The pipeline actions (task, delete, config, config_file, and instrument)
|
|
184
|
+
must be handled in the order they appear on the command line, but the CLI
|
|
185
|
+
specification gives them all different option names. So, instead of using
|
|
186
|
+
the individual action options as they appear in kwargs (because
|
|
187
|
+
invocation order can't be known), we capture the CLI arguments by
|
|
188
|
+
overriding `click.Command.parse_args` and save them in the Context's
|
|
189
|
+
`obj` parameter. We use `makePipelineActions` to create a list of
|
|
190
|
+
pipeline actions from the CLI arguments and pass that list to the script
|
|
191
|
+
function using the `pipeline_actions` kwarg name, and remove the action
|
|
192
|
+
options from kwargs.
|
|
193
|
+
"""
|
|
194
|
+
from lsst.pipe.base.cli.opt import instrument_option
|
|
195
|
+
|
|
196
|
+
from .opt import delete_option, task_option
|
|
197
|
+
|
|
198
|
+
for pipelineAction in (
|
|
199
|
+
task_option.name(),
|
|
200
|
+
delete_option.name(),
|
|
201
|
+
config_option.name(),
|
|
202
|
+
config_file_option.name(),
|
|
203
|
+
instrument_option.name(),
|
|
204
|
+
):
|
|
205
|
+
kwargs.pop(pipelineAction)
|
|
206
|
+
|
|
207
|
+
actions = make_pipeline_actions(MWCtxObj.getFrom(ctx).args)
|
|
208
|
+
pipeline_actions = []
|
|
209
|
+
for action in actions:
|
|
210
|
+
pipeline_actions.append(action)
|
|
211
|
+
|
|
212
|
+
kwargs["pipeline_actions"] = pipeline_actions
|
|
213
|
+
return kwargs
|
|
214
|
+
|
|
215
|
+
|
|
163
216
|
class PipetaskCommand(MWCommand):
|
|
164
217
|
"""Command subclass with pipetask-command specific overrides."""
|
|
165
218
|
|
|
166
219
|
extra_epilog = "See 'pipetask --help' for more options."
|
|
167
220
|
|
|
168
221
|
|
|
169
|
-
def summarize_quantum_graph(
|
|
222
|
+
def summarize_quantum_graph(qg_header: HeaderModel) -> int:
|
|
170
223
|
"""Report a summary of the quanta in the graph.
|
|
171
224
|
|
|
172
225
|
Parameters
|
|
173
226
|
----------
|
|
174
|
-
|
|
175
|
-
|
|
227
|
+
qg_header : `lsst.pipe.base.quantum_graph.HeaderModel`
|
|
228
|
+
Header of the quantum graph.
|
|
176
229
|
|
|
177
230
|
Returns
|
|
178
231
|
-------
|
|
179
232
|
n_quanta : `int`
|
|
180
233
|
The number of quanta in the graph.
|
|
181
234
|
"""
|
|
182
|
-
n_quanta =
|
|
235
|
+
n_quanta = qg_header.n_quanta
|
|
183
236
|
if n_quanta == 0:
|
|
184
237
|
_LOG.info("QuantumGraph contains no quanta.")
|
|
185
238
|
else:
|
|
186
|
-
summary = qgraph.getSummary()
|
|
187
239
|
if _LOG.isEnabledFor(logging.INFO):
|
|
188
240
|
qg_quanta, qg_tasks = [], []
|
|
189
|
-
for task_label,
|
|
241
|
+
for task_label, n_quanta_for_task in qg_header.n_task_quanta.items():
|
|
190
242
|
qg_tasks.append(task_label)
|
|
191
|
-
qg_quanta.append(
|
|
243
|
+
qg_quanta.append(n_quanta_for_task)
|
|
192
244
|
qg_task_table = Table(dict(Quanta=qg_quanta, Tasks=qg_tasks))
|
|
193
245
|
qg_task_table_formatted = "\n".join(qg_task_table.pformat())
|
|
194
246
|
quanta_str = "quantum" if n_quanta == 1 else "quanta"
|
|
195
|
-
n_tasks = len(
|
|
247
|
+
n_tasks = len(qg_header.n_task_quanta)
|
|
196
248
|
n_tasks_plural = "" if n_tasks == 1 else "s"
|
|
197
249
|
_LOG.info(
|
|
198
|
-
"QuantumGraph contains %d %s for %d task%s
|
|
250
|
+
"QuantumGraph contains %d %s for %d task%s\n%s",
|
|
199
251
|
n_quanta,
|
|
200
252
|
quanta_str,
|
|
201
253
|
n_tasks,
|
|
202
254
|
n_tasks_plural,
|
|
203
|
-
qgraph.graphID,
|
|
204
255
|
qg_task_table_formatted,
|
|
205
256
|
)
|
|
206
257
|
return n_quanta
|
lsst/ctrl/mpexec/showInfo.py
CHANGED
|
@@ -40,8 +40,9 @@ import lsst.pex.config as pexConfig
|
|
|
40
40
|
import lsst.pex.config.history as pexConfigHistory
|
|
41
41
|
from lsst.daf.butler import Butler, DatasetRef, DatasetType, NamedKeyMapping
|
|
42
42
|
from lsst.daf.butler.datastore.record_data import DatastoreRecordData
|
|
43
|
-
from lsst.pipe.base import PipelineGraph
|
|
43
|
+
from lsst.pipe.base import PipelineGraph
|
|
44
44
|
from lsst.pipe.base.pipeline_graph import visualization
|
|
45
|
+
from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
|
|
45
46
|
from lsst.resources import ResourcePathExpression
|
|
46
47
|
|
|
47
48
|
from . import util
|
|
@@ -135,6 +136,7 @@ class ShowInfo:
|
|
|
135
136
|
raise ValueError(
|
|
136
137
|
f"Unknown value(s) for show: {unknown} (choose from '{', '.join(sorted(known))}')"
|
|
137
138
|
)
|
|
139
|
+
self.needs_full_qg: bool = "graph" in self.commands.keys() or "uri" in self.commands.keys()
|
|
138
140
|
|
|
139
141
|
@property
|
|
140
142
|
def unhandled(self) -> frozenset[str]:
|
|
@@ -190,15 +192,15 @@ class ShowInfo:
|
|
|
190
192
|
|
|
191
193
|
def show_graph_info(
|
|
192
194
|
self,
|
|
193
|
-
|
|
195
|
+
qg: PredictedQuantumGraph,
|
|
194
196
|
butler_config: ResourcePathExpression | None = None,
|
|
195
197
|
) -> None:
|
|
196
198
|
"""Show information associated with this graph.
|
|
197
199
|
|
|
198
200
|
Parameters
|
|
199
201
|
----------
|
|
200
|
-
|
|
201
|
-
|
|
202
|
+
qg : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
|
|
203
|
+
Quantum graph.
|
|
202
204
|
butler_config : convertible to `lsst.resources.ResourcePath`, optional
|
|
203
205
|
Path to configuration for the butler.
|
|
204
206
|
"""
|
|
@@ -207,13 +209,13 @@ class ShowInfo:
|
|
|
207
209
|
continue
|
|
208
210
|
match command:
|
|
209
211
|
case "graph":
|
|
210
|
-
self._showGraph(
|
|
212
|
+
self._showGraph(qg)
|
|
211
213
|
case "uri":
|
|
212
214
|
if butler_config is None:
|
|
213
215
|
raise ValueError("Showing URIs requires the -b option")
|
|
214
|
-
self._showUri(
|
|
216
|
+
self._showUri(qg, butler_config)
|
|
215
217
|
case "workflow":
|
|
216
|
-
self._showWorkflow(
|
|
218
|
+
self._showWorkflow(qg)
|
|
217
219
|
case _:
|
|
218
220
|
raise RuntimeError(f"Unexpectedly tried to process command {command!r}.")
|
|
219
221
|
self.handled.add(command)
|
|
@@ -322,13 +324,13 @@ class ShowInfo:
|
|
|
322
324
|
for configName, taskName in util.subTaskIter(task_node.config):
|
|
323
325
|
print(f"{configName}: {taskName}", file=self.stream)
|
|
324
326
|
|
|
325
|
-
def _showGraph(self,
|
|
327
|
+
def _showGraph(self, qg: PredictedQuantumGraph) -> None:
|
|
326
328
|
"""Print quanta information to stdout
|
|
327
329
|
|
|
328
330
|
Parameters
|
|
329
331
|
----------
|
|
330
|
-
|
|
331
|
-
|
|
332
|
+
qg : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
|
|
333
|
+
Quantum graph.
|
|
332
334
|
"""
|
|
333
335
|
|
|
334
336
|
def _print_refs(
|
|
@@ -351,39 +353,38 @@ class ShowInfo:
|
|
|
351
353
|
else:
|
|
352
354
|
print(f" {key}: []", file=self.stream)
|
|
353
355
|
|
|
354
|
-
for
|
|
355
|
-
print(
|
|
356
|
-
|
|
357
|
-
for
|
|
358
|
-
quantum =
|
|
359
|
-
print(
|
|
360
|
-
f" Quantum {iq} dataId={quantum.dataId} nodeId={quantum_node.nodeId}:", file=self.stream
|
|
361
|
-
)
|
|
356
|
+
for task_label, quanta_for_task in qg.quanta_by_task.items():
|
|
357
|
+
print(f"{task_label} ({qg.pipeline_graph.tasks[task_label].task_class_name})", file=self.stream)
|
|
358
|
+
execution_quanta = qg.build_execution_quanta(task_label=task_label)
|
|
359
|
+
for data_id, quantum_id in quanta_for_task.items():
|
|
360
|
+
quantum = execution_quanta[quantum_id]
|
|
361
|
+
print(f" Quantum {quantum_id} dataId={data_id}:", file=self.stream)
|
|
362
362
|
print(" inputs:", file=self.stream)
|
|
363
363
|
_print_refs(quantum.inputs, quantum.datastore_records)
|
|
364
364
|
print(" outputs:", file=self.stream)
|
|
365
365
|
_print_refs(quantum.outputs, quantum.datastore_records)
|
|
366
366
|
|
|
367
|
-
def _showWorkflow(self,
|
|
367
|
+
def _showWorkflow(self, qg: PredictedQuantumGraph) -> None:
|
|
368
368
|
"""Print quanta information and dependency to stdout
|
|
369
369
|
|
|
370
370
|
Parameters
|
|
371
371
|
----------
|
|
372
|
-
|
|
373
|
-
|
|
372
|
+
qg : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
|
|
373
|
+
Quantum graph.
|
|
374
374
|
"""
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
375
|
+
xgraph = qg.quantum_only_xgraph
|
|
376
|
+
for child_id, child_data in xgraph.nodes.items():
|
|
377
|
+
print(f"Quantum {child_id}: {child_data['pipeline_node'].task_class_name}", file=self.stream)
|
|
378
|
+
for parent_id in xgraph.predecessors(child_id):
|
|
379
|
+
print(f"Parent Quantum {parent_id} - Child Quantum {child_id}", file=self.stream)
|
|
379
380
|
|
|
380
|
-
def _showUri(self,
|
|
381
|
+
def _showUri(self, qg: PredictedQuantumGraph, butler_config: ResourcePathExpression) -> None:
|
|
381
382
|
"""Print input and predicted output URIs to stdout.
|
|
382
383
|
|
|
383
384
|
Parameters
|
|
384
385
|
----------
|
|
385
|
-
|
|
386
|
-
|
|
386
|
+
qg : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
|
|
387
|
+
Quantum graph.
|
|
387
388
|
butler_config : convertible to `lsst.resources.ResourcePath`
|
|
388
389
|
Path to configuration for the butler.
|
|
389
390
|
"""
|
|
@@ -398,13 +399,16 @@ class ShowInfo:
|
|
|
398
399
|
print(f" {compName}: {compUri}", file=self.stream)
|
|
399
400
|
|
|
400
401
|
butler = Butler.from_config(butler_config)
|
|
401
|
-
|
|
402
|
-
|
|
402
|
+
xgraph = qg.quantum_only_xgraph
|
|
403
|
+
execution_quanta = qg.build_execution_quanta()
|
|
404
|
+
for quantum_id, quantum_data in xgraph.nodes.items():
|
|
405
|
+
print(f"Quantum {quantum_id}: {quantum_data['pipeline_node'].task_class_name}", file=self.stream)
|
|
403
406
|
print(" inputs:", file=self.stream)
|
|
404
|
-
|
|
407
|
+
execution_quantum = execution_quanta[quantum_id]
|
|
408
|
+
for refs in execution_quantum.inputs.values():
|
|
405
409
|
for ref in refs:
|
|
406
410
|
dumpURIs(ref)
|
|
407
411
|
print(" outputs:", file=self.stream)
|
|
408
|
-
for refs in
|
|
412
|
+
for refs in execution_quantum.outputs.values():
|
|
409
413
|
for ref in refs:
|
|
410
414
|
dumpURIs(ref)
|
|
@@ -164,7 +164,7 @@ class SingleQuantumExecutor(lsst.pipe.base.single_quantum_executor.SingleQuantum
|
|
|
164
164
|
return super()._write_metadata(quantum, metadata, task_node, limited_butler=limited_butler)
|
|
165
165
|
|
|
166
166
|
def initGlobals(self, quantum: Quantum) -> None:
|
|
167
|
-
|
|
167
|
+
pass
|
|
168
168
|
|
|
169
169
|
@property
|
|
170
170
|
def butler(self) -> Butler | None:
|
lsst/ctrl/mpexec/version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
__all__ = ["__version__"]
|
|
2
|
-
__version__ = "29.2025.
|
|
2
|
+
__version__ = "29.2025.4000"
|
{lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-ctrl-mpexec
|
|
3
|
-
Version: 29.2025.
|
|
3
|
+
Version: 29.2025.4000
|
|
4
4
|
Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License: BSD 3-Clause License
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
lsst/__init__.py,sha256=aXdEOZVrBQISQi6XPS9s1NhBjIJaIwNNxCFRiGchRAw,1369
|
|
2
2
|
lsst/ctrl/__init__.py,sha256=aXdEOZVrBQISQi6XPS9s1NhBjIJaIwNNxCFRiGchRAw,1369
|
|
3
3
|
lsst/ctrl/mpexec/__init__.py,sha256=Ld4bk-SLij104yAdrAzvh54ZblCglYszOa6cLjWnIDE,1707
|
|
4
|
-
lsst/ctrl/mpexec/_pipeline_graph_factory.py,sha256=
|
|
4
|
+
lsst/ctrl/mpexec/_pipeline_graph_factory.py,sha256=HJPjvbsfqsKE0FVru5zje55rgKb9jXvOMVEEGIaTCU8,4337
|
|
5
5
|
lsst/ctrl/mpexec/execFixupDataId.py,sha256=26eFIrZ-I5oTGoBzO35WH2UtwYIPe5aX37IHE9OfAa0,1633
|
|
6
6
|
lsst/ctrl/mpexec/executionGraphFixup.py,sha256=hpkU0cCRcz80c-YQS1eU5G4riRTSUfOfPjCehTkWTMA,1776
|
|
7
7
|
lsst/ctrl/mpexec/log_capture.py,sha256=Y8ExKqrpyboBoi6giY9dlL4OyCrZn-c_6EB8Oh_z6dc,1605
|
|
@@ -11,18 +11,18 @@ lsst/ctrl/mpexec/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
11
11
|
lsst/ctrl/mpexec/quantumGraphExecutor.py,sha256=7LCoqsmNSWUJ1ph1bfgrVrpk5Ug5ArzniFkqcxgjfhI,2106
|
|
12
12
|
lsst/ctrl/mpexec/reports.py,sha256=EvHjW9wdjfN1lj4DQH634oJ9zv3wYKySvJ_UtFDtNHY,2460
|
|
13
13
|
lsst/ctrl/mpexec/separablePipelineExecutor.py,sha256=yHymF9jSthVCl_230TzRqQrcAmohSgR7Me8YAbMazM8,1818
|
|
14
|
-
lsst/ctrl/mpexec/showInfo.py,sha256=
|
|
14
|
+
lsst/ctrl/mpexec/showInfo.py,sha256=WCoZYd_4q6ZIYRH0qv-VK8zCl53aIIeYlma48tVO3ZI,17470
|
|
15
15
|
lsst/ctrl/mpexec/simple_pipeline_executor.py,sha256=aVWgOds0ILCTzXdrWy0KFyjcIuqw6cNq8alAJeZd8AE,1797
|
|
16
|
-
lsst/ctrl/mpexec/singleQuantumExecutor.py,sha256=
|
|
16
|
+
lsst/ctrl/mpexec/singleQuantumExecutor.py,sha256=Y70xs5f06sEeDXF3cKNjxL4gdG69Osu8Xbol-vm-VIo,8616
|
|
17
17
|
lsst/ctrl/mpexec/taskFactory.py,sha256=0TdCM-le7CXhobVbWPTqcq9sOrGceUY6mmkzzWhi0dM,1707
|
|
18
18
|
lsst/ctrl/mpexec/util.py,sha256=y2Rw5PL40_EuLtVxiqSVX0JfPV4IrFl1LfOMUWx2u30,4236
|
|
19
|
-
lsst/ctrl/mpexec/version.py,sha256=
|
|
19
|
+
lsst/ctrl/mpexec/version.py,sha256=OZgl6BR-Rxy62NJlHO2yg_Rq4EhfmRSKDkEEnAyc-3Q,55
|
|
20
20
|
lsst/ctrl/mpexec/cli/__init__.py,sha256=6dpDHNBzyicVpFi1fsaiYVbYEMeoL57IHKkPaej24gs,1301
|
|
21
21
|
lsst/ctrl/mpexec/cli/butler_factory.py,sha256=-Nlm5nVcWK_JbW8oc5qudu9yuZg5Pknf1LmtkxTkHJs,26713
|
|
22
22
|
lsst/ctrl/mpexec/cli/pipetask.py,sha256=4HnhX9dCizCihVbpHVJX5WXO9TEli9oL6wA-tPh1_vA,2209
|
|
23
|
-
lsst/ctrl/mpexec/cli/utils.py,sha256=
|
|
23
|
+
lsst/ctrl/mpexec/cli/utils.py,sha256=Oiw4yOpsR7_kJObhZAZu8hm81NFmaLaRAv86v4smadk,9833
|
|
24
24
|
lsst/ctrl/mpexec/cli/cmd/__init__.py,sha256=nRmwwW5d55gAEkyE7NpSK8mxa56HcfEta2r-Y9I07F8,1661
|
|
25
|
-
lsst/ctrl/mpexec/cli/cmd/commands.py,sha256=
|
|
25
|
+
lsst/ctrl/mpexec/cli/cmd/commands.py,sha256=dO0WMFEwAAGoiNBgkQXu1oW-N89Eah89X2XR84PKQBg,17063
|
|
26
26
|
lsst/ctrl/mpexec/cli/opt/__init__.py,sha256=IzUInuJj9igiaNcEqMx0adelgJtQC5_XMYnaiizBn0A,1378
|
|
27
27
|
lsst/ctrl/mpexec/cli/opt/arguments.py,sha256=vjUw0ZN_4HStp-_3ne6AT5S_eH7sly3OVfL07tgrJnY,1572
|
|
28
28
|
lsst/ctrl/mpexec/cli/opt/optionGroups.py,sha256=v_fbB-frVUdXr82VY_mC8KHdUJSJNaU64qBJO4cA3To,7445
|
|
@@ -31,20 +31,20 @@ lsst/ctrl/mpexec/cli/script/__init__.py,sha256=eCuF4FAI5D3pl05IMJj7TCkZq-hireua2
|
|
|
31
31
|
lsst/ctrl/mpexec/cli/script/build.py,sha256=3_ohOGuope_hCrvBAo1X31nejgTZ73PZv_ew3ykhYIc,6919
|
|
32
32
|
lsst/ctrl/mpexec/cli/script/cleanup.py,sha256=D7W-azf4mNJcIWhbU5uCRCi94mkb8-Q2ksRFblQGrUw,4990
|
|
33
33
|
lsst/ctrl/mpexec/cli/script/confirmable.py,sha256=Bo1GSTZQn44d_TRj6N3YfpYcZiuHEYoz26WZQwMyb4A,3918
|
|
34
|
-
lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py,sha256=
|
|
34
|
+
lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py,sha256=3cNw5BKMT4_YWo1GeuH4VmvKeYOoyDcjH7QgBEoiPmA,4140
|
|
35
35
|
lsst/ctrl/mpexec/cli/script/purge.py,sha256=gYwSsZfTBP6oDcDp_YdqQEKGvAStvsj5hwNw42S8ptE,10637
|
|
36
|
-
lsst/ctrl/mpexec/cli/script/qgraph.py,sha256=
|
|
36
|
+
lsst/ctrl/mpexec/cli/script/qgraph.py,sha256=aaWRxqQHWWg8hRYlMZmKxQMq5arZNIYa4rHvQmENRUs,14905
|
|
37
37
|
lsst/ctrl/mpexec/cli/script/report.py,sha256=ItJitmYmWIDjj7PxRtP4qXLx-z5FAU6nSfI2gx0GS5k,12800
|
|
38
|
-
lsst/ctrl/mpexec/cli/script/run.py,sha256=
|
|
39
|
-
lsst/ctrl/mpexec/cli/script/run_qbb.py,sha256=
|
|
40
|
-
lsst/ctrl/mpexec/cli/script/update_graph_run.py,sha256=
|
|
41
|
-
lsst_ctrl_mpexec-29.2025.
|
|
42
|
-
lsst_ctrl_mpexec-29.2025.
|
|
43
|
-
lsst_ctrl_mpexec-29.2025.
|
|
44
|
-
lsst_ctrl_mpexec-29.2025.
|
|
45
|
-
lsst_ctrl_mpexec-29.2025.
|
|
46
|
-
lsst_ctrl_mpexec-29.2025.
|
|
47
|
-
lsst_ctrl_mpexec-29.2025.
|
|
48
|
-
lsst_ctrl_mpexec-29.2025.
|
|
49
|
-
lsst_ctrl_mpexec-29.2025.
|
|
50
|
-
lsst_ctrl_mpexec-29.2025.
|
|
38
|
+
lsst/ctrl/mpexec/cli/script/run.py,sha256=d_oSGl6t9qEPeFPsZxlPQkUsGLNNl5wB1PXGn71Fw-s,14135
|
|
39
|
+
lsst/ctrl/mpexec/cli/script/run_qbb.py,sha256=lz4PMezfyDjFzyjrOnk0i-BULj_Tn8axHHDPQ20KgVo,10862
|
|
40
|
+
lsst/ctrl/mpexec/cli/script/update_graph_run.py,sha256=2b0Q0j5yPC4bWxq0tsd4et2VmOnu-dQLQa94340z78Y,2623
|
|
41
|
+
lsst_ctrl_mpexec-29.2025.4000.dist-info/licenses/COPYRIGHT,sha256=pGCjnRAnyt02a6_9PLzXQikpvYmvMmK9fCdOKlRSV6k,369
|
|
42
|
+
lsst_ctrl_mpexec-29.2025.4000.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
|
|
43
|
+
lsst_ctrl_mpexec-29.2025.4000.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
|
|
44
|
+
lsst_ctrl_mpexec-29.2025.4000.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
45
|
+
lsst_ctrl_mpexec-29.2025.4000.dist-info/METADATA,sha256=ccuSpT6JGbaAbBp6qCSkQMJYnLPaY_aMKR4drY_2eaU,2302
|
|
46
|
+
lsst_ctrl_mpexec-29.2025.4000.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
47
|
+
lsst_ctrl_mpexec-29.2025.4000.dist-info/entry_points.txt,sha256=aYE38yqZU8qvpLUUkXzgmUxDJYYknEqPxgxYkowrL4s,64
|
|
48
|
+
lsst_ctrl_mpexec-29.2025.4000.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
|
|
49
|
+
lsst_ctrl_mpexec-29.2025.4000.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
50
|
+
lsst_ctrl_mpexec-29.2025.4000.dist-info/RECORD,,
|
|
File without changes
|
{lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
{lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/top_level.txt
RENAMED
|
File without changes
|
{lsst_ctrl_mpexec-29.2025.3900.dist-info → lsst_ctrl_mpexec-29.2025.4000.dist-info}/zip-safe
RENAMED
|
File without changes
|