lsst-ctrl-mpexec 29.2025.3900__tar.gz → 29.2025.4100__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lsst_ctrl_mpexec-29.2025.3900/python/lsst_ctrl_mpexec.egg-info → lsst_ctrl_mpexec-29.2025.4100}/PKG-INFO +1 -1
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/_pipeline_graph_factory.py +21 -3
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/cmd/commands.py +17 -46
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py +25 -6
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/qgraph.py +67 -30
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/run.py +14 -15
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/run_qbb.py +13 -12
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/update_graph_run.py +17 -13
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/utils.py +64 -13
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/showInfo.py +36 -32
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/singleQuantumExecutor.py +1 -1
- lsst_ctrl_mpexec-29.2025.4100/python/lsst/ctrl/mpexec/version.py +2 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100/python/lsst_ctrl_mpexec.egg-info}/PKG-INFO +1 -1
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/SOURCES.txt +4 -4
- lsst_ctrl_mpexec-29.2025.4100/tests/test_build.py +302 -0
- lsst_ctrl_mpexec-29.2025.3900/tests/test_cliCmdQgraph.py → lsst_ctrl_mpexec-29.2025.4100/tests/test_cliCmdUpdateGraphRun.py +21 -27
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/tests/test_cliUtils.py +14 -13
- lsst_ctrl_mpexec-29.2025.4100/tests/test_qgraph.py +144 -0
- lsst_ctrl_mpexec-29.2025.4100/tests/test_run.py +674 -0
- lsst_ctrl_mpexec-29.2025.3900/python/lsst/ctrl/mpexec/version.py +0 -2
- lsst_ctrl_mpexec-29.2025.3900/tests/test_cliCmdUpdateGraphRun.py +0 -112
- lsst_ctrl_mpexec-29.2025.3900/tests/test_cliScript.py +0 -266
- lsst_ctrl_mpexec-29.2025.3900/tests/test_cmdLineFwk.py +0 -1218
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/COPYRIGHT +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/LICENSE +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/MANIFEST.in +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/README.rst +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/bsd_license.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/doc/lsst.ctrl.mpexec/CHANGES.rst +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/doc/lsst.ctrl.mpexec/configuring-pipetask-tasks.rst +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/doc/lsst.ctrl.mpexec/index.rst +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/doc/lsst.ctrl.mpexec/pipetask.rst +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/gpl-v3.0.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/pyproject.toml +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/butler_factory.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/cmd/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/opt/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/opt/arguments.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/opt/options.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/pipetask.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/build.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/cleanup.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/confirmable.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/purge.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/report.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/execFixupDataId.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/executionGraphFixup.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/log_capture.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/mpGraphExecutor.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/preExecInit.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/py.typed +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/quantumGraphExecutor.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/reports.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/separablePipelineExecutor.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/simple_pipeline_executor.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/taskFactory.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/util.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/dependency_links.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/entry_points.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/requires.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/top_level.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/zip-safe +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/setup.cfg +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/tests/test_cliCmdCleanup.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/tests/test_cliCmdPurge.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/tests/test_cliCmdReport.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/tests/test_preExecInit.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-ctrl-mpexec
|
|
3
|
-
Version: 29.2025.
|
|
3
|
+
Version: 29.2025.4100
|
|
4
4
|
Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License: BSD 3-Clause License
|
|
@@ -45,18 +45,30 @@ class PipelineGraphFactory:
|
|
|
45
45
|
dimension schema.
|
|
46
46
|
select_tasks : `str`, optional
|
|
47
47
|
String expression that filters the tasks in the pipeline graph.
|
|
48
|
+
pipeline_graph : `lsst.pipe.base.pipeline_graph.PipelineGraph`, optional
|
|
49
|
+
Already-constructed pipeline graph.
|
|
48
50
|
"""
|
|
49
51
|
|
|
50
|
-
def __init__(
|
|
52
|
+
def __init__(
|
|
53
|
+
self,
|
|
54
|
+
pipeline: Pipeline | None = None,
|
|
55
|
+
butler: Butler | None = None,
|
|
56
|
+
select_tasks: str = "",
|
|
57
|
+
*,
|
|
58
|
+
pipeline_graph: PipelineGraph | None = None,
|
|
59
|
+
):
|
|
60
|
+
if pipeline is None and pipeline_graph is None:
|
|
61
|
+
raise TypeError("At least one of 'pipeline' and 'pipeline_graph' must not be `None`.")
|
|
51
62
|
self._pipeline = pipeline
|
|
52
63
|
self._registry = butler.registry if butler is not None else None
|
|
53
64
|
self._select_tasks = select_tasks
|
|
54
|
-
self._pipeline_graph: PipelineGraph | None =
|
|
65
|
+
self._pipeline_graph: PipelineGraph | None = pipeline_graph
|
|
55
66
|
self._resolved: bool = False
|
|
56
67
|
self._for_visualization_only: bool = False
|
|
57
68
|
|
|
58
69
|
def __call__(self, *, resolve: bool = True, visualization_only: bool = False) -> PipelineGraph:
|
|
59
70
|
if self._pipeline_graph is None:
|
|
71
|
+
assert self._pipeline is not None, "Guaranteed at construction."
|
|
60
72
|
self._pipeline_graph = self._pipeline.to_graph()
|
|
61
73
|
if self._select_tasks:
|
|
62
74
|
self._pipeline_graph = self._pipeline_graph.select(self._select_tasks)
|
|
@@ -71,6 +83,8 @@ class PipelineGraphFactory:
|
|
|
71
83
|
@property
|
|
72
84
|
def pipeline(self) -> Pipeline:
|
|
73
85
|
"""The original pipeline definition."""
|
|
86
|
+
if self._pipeline is None:
|
|
87
|
+
raise RuntimeError("Cannot obtain pipeline from pipeline graph.")
|
|
74
88
|
if self._select_tasks:
|
|
75
89
|
raise RuntimeError(
|
|
76
90
|
"The --select-tasks option cannot be used with operations that return or display a "
|
|
@@ -79,4 +93,8 @@ class PipelineGraphFactory:
|
|
|
79
93
|
return self._pipeline
|
|
80
94
|
|
|
81
95
|
def __bool__(self) -> bool:
|
|
82
|
-
|
|
96
|
+
if self._pipeline is not None:
|
|
97
|
+
return bool(self._pipeline)
|
|
98
|
+
else:
|
|
99
|
+
assert self._pipeline_graph is not None, "Guaranteed at construction."
|
|
100
|
+
return bool(self._pipeline_graph.tasks)
|
|
@@ -35,25 +35,22 @@ from typing import Any
|
|
|
35
35
|
|
|
36
36
|
import click
|
|
37
37
|
|
|
38
|
-
import lsst.pipe.base.cli.opt as pipeBaseOpts
|
|
39
38
|
from lsst.ctrl.mpexec.showInfo import ShowInfo
|
|
40
39
|
from lsst.daf.butler.cli.opt import (
|
|
41
40
|
collections_option,
|
|
42
|
-
config_file_option,
|
|
43
|
-
config_option,
|
|
44
41
|
confirm_option,
|
|
45
42
|
options_file_option,
|
|
46
43
|
processes_option,
|
|
47
44
|
repo_argument,
|
|
48
45
|
where_option,
|
|
49
46
|
)
|
|
50
|
-
from lsst.daf.butler.cli.utils import
|
|
47
|
+
from lsst.daf.butler.cli.utils import catch_and_exit, option_section, unwrap
|
|
51
48
|
from lsst.pipe.base.quantum_reports import Report
|
|
52
49
|
|
|
53
50
|
from .. import opt as ctrlMpExecOpts
|
|
54
51
|
from .. import script
|
|
55
52
|
from ..script import confirmable
|
|
56
|
-
from ..utils import PipetaskCommand,
|
|
53
|
+
from ..utils import PipetaskCommand, collect_pipeline_actions
|
|
57
54
|
|
|
58
55
|
epilog = unwrap(
|
|
59
56
|
"""Notes:
|
|
@@ -69,41 +66,6 @@ ignored.)
|
|
|
69
66
|
)
|
|
70
67
|
|
|
71
68
|
|
|
72
|
-
def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]:
|
|
73
|
-
"""Extract pipeline building options, replace them with PipelineActions,
|
|
74
|
-
return updated `kwargs`.
|
|
75
|
-
|
|
76
|
-
Notes
|
|
77
|
-
-----
|
|
78
|
-
The pipeline actions (task, delete, config, config_file, and instrument)
|
|
79
|
-
must be handled in the order they appear on the command line, but the CLI
|
|
80
|
-
specification gives them all different option names. So, instead of using
|
|
81
|
-
the individual action options as they appear in kwargs (because
|
|
82
|
-
invocation order can't be known), we capture the CLI arguments by
|
|
83
|
-
overriding `click.Command.parse_args` and save them in the Context's
|
|
84
|
-
`obj` parameter. We use `makePipelineActions` to create a list of
|
|
85
|
-
pipeline actions from the CLI arguments and pass that list to the script
|
|
86
|
-
function using the `pipeline_actions` kwarg name, and remove the action
|
|
87
|
-
options from kwargs.
|
|
88
|
-
"""
|
|
89
|
-
for pipelineAction in (
|
|
90
|
-
ctrlMpExecOpts.task_option.name(),
|
|
91
|
-
ctrlMpExecOpts.delete_option.name(),
|
|
92
|
-
config_option.name(),
|
|
93
|
-
config_file_option.name(),
|
|
94
|
-
pipeBaseOpts.instrument_option.name(),
|
|
95
|
-
):
|
|
96
|
-
kwargs.pop(pipelineAction)
|
|
97
|
-
|
|
98
|
-
actions = makePipelineActions(MWCtxObj.getFrom(ctx).args)
|
|
99
|
-
pipeline_actions = []
|
|
100
|
-
for action in actions:
|
|
101
|
-
pipeline_actions.append(action)
|
|
102
|
-
|
|
103
|
-
kwargs["pipeline_actions"] = pipeline_actions
|
|
104
|
-
return kwargs
|
|
105
|
-
|
|
106
|
-
|
|
107
69
|
def _unhandledShow(show: ShowInfo, cmd: str) -> None:
|
|
108
70
|
if show.unhandled:
|
|
109
71
|
print(
|
|
@@ -125,7 +87,7 @@ def build(ctx: click.Context, **kwargs: Any) -> None:
|
|
|
125
87
|
|
|
126
88
|
This does not require input data to be specified.
|
|
127
89
|
"""
|
|
128
|
-
kwargs =
|
|
90
|
+
kwargs = collect_pipeline_actions(ctx, **kwargs)
|
|
129
91
|
show = ShowInfo(kwargs.pop("show", []))
|
|
130
92
|
if kwargs.get("butler_config") is not None and (
|
|
131
93
|
{"pipeline-graph", "task-graph"}.isdisjoint(show.commands) and not kwargs.get("pipeline_dot")
|
|
@@ -187,7 +149,7 @@ concurrency = multiprocessing
|
|
|
187
149
|
@catch_and_exit
|
|
188
150
|
def qgraph(ctx: click.Context, **kwargs: Any) -> None:
|
|
189
151
|
"""Build and optionally save quantum graph."""
|
|
190
|
-
kwargs =
|
|
152
|
+
kwargs = collect_pipeline_actions(ctx, **kwargs)
|
|
191
153
|
summary = kwargs.pop("summary", None)
|
|
192
154
|
with coverage_context(kwargs):
|
|
193
155
|
show = ShowInfo(kwargs.pop("show", []))
|
|
@@ -204,12 +166,21 @@ def qgraph(ctx: click.Context, **kwargs: Any) -> None:
|
|
|
204
166
|
file=sys.stderr,
|
|
205
167
|
)
|
|
206
168
|
return
|
|
207
|
-
if (
|
|
169
|
+
if (
|
|
170
|
+
qgraph := script.qgraph(
|
|
171
|
+
pipeline_graph_factory,
|
|
172
|
+
**kwargs,
|
|
173
|
+
show=show,
|
|
174
|
+
# Making a summary report requires that we load the same graph
|
|
175
|
+
# components as execution.
|
|
176
|
+
for_execution=(summary is not None),
|
|
177
|
+
)
|
|
178
|
+
) is None:
|
|
208
179
|
raise click.ClickException("QuantumGraph was empty; ERROR logs above should provide details.")
|
|
209
180
|
# QuantumGraph-only summary call here since script.qgraph also called
|
|
210
181
|
# by run methods.
|
|
211
182
|
if summary:
|
|
212
|
-
report = Report(qgraphSummary=qgraph.
|
|
183
|
+
report = Report(qgraphSummary=qgraph._make_summary())
|
|
213
184
|
with open(summary, "w") as out:
|
|
214
185
|
# Do not save fields that are not set.
|
|
215
186
|
out.write(report.model_dump_json(exclude_none=True, indent=2))
|
|
@@ -222,7 +193,7 @@ def qgraph(ctx: click.Context, **kwargs: Any) -> None:
|
|
|
222
193
|
@catch_and_exit
|
|
223
194
|
def run(ctx: click.Context, **kwargs: Any) -> None:
|
|
224
195
|
"""Build and execute pipeline and quantum graph."""
|
|
225
|
-
kwargs =
|
|
196
|
+
kwargs = collect_pipeline_actions(ctx, **kwargs)
|
|
226
197
|
with coverage_context(kwargs):
|
|
227
198
|
show = ShowInfo(kwargs.pop("show", []))
|
|
228
199
|
pipeline_graph_factory = script.build(**kwargs, show=show)
|
|
@@ -233,7 +204,7 @@ def run(ctx: click.Context, **kwargs: Any) -> None:
|
|
|
233
204
|
file=sys.stderr,
|
|
234
205
|
)
|
|
235
206
|
return
|
|
236
|
-
if (qgraph := script.qgraph(pipeline_graph_factory, **kwargs, show=show)) is None:
|
|
207
|
+
if (qgraph := script.qgraph(pipeline_graph_factory, for_execution=True, **kwargs, show=show)) is None:
|
|
237
208
|
raise click.ClickException("QuantumGraph was empty; ERROR logs above should provide details.")
|
|
238
209
|
_unhandledShow(show, "run")
|
|
239
210
|
if show.handled:
|
|
@@ -28,6 +28,9 @@
|
|
|
28
28
|
from __future__ import annotations
|
|
29
29
|
|
|
30
30
|
from lsst.pipe.base import BuildId, QuantumGraph
|
|
31
|
+
from lsst.pipe.base.pipeline_graph import TaskImportMode
|
|
32
|
+
from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
|
|
33
|
+
from lsst.resources import ResourcePath, ResourcePathExpression
|
|
31
34
|
from lsst.utils.logging import getLogger
|
|
32
35
|
|
|
33
36
|
from ..butler_factory import ButlerFactory
|
|
@@ -37,7 +40,7 @@ _LOG = getLogger(__name__)
|
|
|
37
40
|
|
|
38
41
|
def pre_exec_init_qbb(
|
|
39
42
|
butler_config: str,
|
|
40
|
-
qgraph:
|
|
43
|
+
qgraph: ResourcePathExpression,
|
|
41
44
|
qgraph_id: str | None,
|
|
42
45
|
config_search_path: list[str] | None,
|
|
43
46
|
**kwargs: object,
|
|
@@ -64,11 +67,27 @@ def pre_exec_init_qbb(
|
|
|
64
67
|
function and pass all the option kwargs to each of the script functions
|
|
65
68
|
which ignore these unused kwargs.
|
|
66
69
|
"""
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
70
|
+
qgraph = ResourcePath(qgraph)
|
|
71
|
+
match qgraph.getExtension():
|
|
72
|
+
case ".qgraph":
|
|
73
|
+
_LOG.verbose("Reading full quantum graph from %s.", qgraph)
|
|
74
|
+
qg = PredictedQuantumGraph.from_old_quantum_graph(
|
|
75
|
+
QuantumGraph.loadUri(
|
|
76
|
+
qgraph,
|
|
77
|
+
graphID=BuildId(qgraph_id) if qgraph_id is not None else None,
|
|
78
|
+
)
|
|
79
|
+
)
|
|
80
|
+
case ".qg":
|
|
81
|
+
_LOG.verbose("Reading init quanta from quantum graph from %s.", qgraph)
|
|
82
|
+
if qgraph_id is not None:
|
|
83
|
+
_LOG.warning("--qgraph-id is ignored when loading new '.qg' files.")
|
|
84
|
+
with PredictedQuantumGraph.open(
|
|
85
|
+
qgraph, import_mode=TaskImportMode.ASSUME_CONSISTENT_EDGES
|
|
86
|
+
) as reader:
|
|
87
|
+
reader.read_init_quanta()
|
|
88
|
+
qg = reader.finish()
|
|
89
|
+
case ext:
|
|
90
|
+
raise ValueError(f"Unrecognized extension for quantum graph: {ext!r}")
|
|
72
91
|
|
|
73
92
|
# Ensure that QBB uses shared datastore cache for writes.
|
|
74
93
|
ButlerFactory.define_datastore_cache()
|
|
@@ -42,6 +42,8 @@ from lsst.pipe.base.all_dimensions_quantum_graph_builder import (
|
|
|
42
42
|
)
|
|
43
43
|
from lsst.pipe.base.dot_tools import graph2dot
|
|
44
44
|
from lsst.pipe.base.mermaid_tools import graph2mermaid
|
|
45
|
+
from lsst.pipe.base.pipeline_graph import TaskImportMode
|
|
46
|
+
from lsst.pipe.base.quantum_graph import PredictedQuantumGraph, PredictedQuantumGraphComponents
|
|
45
47
|
from lsst.resources import ResourcePath, ResourcePathExpression
|
|
46
48
|
from lsst.utils.iteration import ensure_iterable
|
|
47
49
|
from lsst.utils.logging import getLogger
|
|
@@ -60,7 +62,7 @@ _LOG = getLogger(__name__)
|
|
|
60
62
|
def qgraph(
|
|
61
63
|
pipeline_graph_factory: PipelineGraphFactory | None,
|
|
62
64
|
*,
|
|
63
|
-
qgraph:
|
|
65
|
+
qgraph: ResourcePathExpression | None,
|
|
64
66
|
qgraph_id: str | None,
|
|
65
67
|
qgraph_node_id: Iterable[uuid.UUID | str] | None,
|
|
66
68
|
qgraph_datastore_records: bool,
|
|
@@ -85,8 +87,10 @@ def qgraph(
|
|
|
85
87
|
mock: bool = False,
|
|
86
88
|
unmocked_dataset_types: Sequence[str],
|
|
87
89
|
mock_failure: Mapping[str, ForcedFailure],
|
|
90
|
+
for_execution: bool = False,
|
|
91
|
+
for_init_output_run: bool = False,
|
|
88
92
|
**kwargs: object,
|
|
89
|
-
) ->
|
|
93
|
+
) -> PredictedQuantumGraph | None:
|
|
90
94
|
"""Implement the command line interface `pipetask qgraph` subcommand.
|
|
91
95
|
|
|
92
96
|
Should only be called by command line tools and unit test code that test
|
|
@@ -97,8 +101,7 @@ def qgraph(
|
|
|
97
101
|
pipeline_graph_factory : `..PipelineGraphFactory` or `None`
|
|
98
102
|
A factory that holds the pipeline and can produce a pipeline graph.
|
|
99
103
|
If this is not `None` then `qgraph` should be `None`.
|
|
100
|
-
qgraph : `lsst.
|
|
101
|
-
`lsst.resources.ResourcePath` or `None`
|
|
104
|
+
qgraph : convertible to `lsst.resources.ResourcePath`, or `None`
|
|
102
105
|
URI location for a serialized quantum graph definition. If this option
|
|
103
106
|
is not `None` then ``pipeline_graph_factory`` should be `None`.
|
|
104
107
|
qgraph_id : `str` or `None`
|
|
@@ -179,15 +182,23 @@ def qgraph(
|
|
|
179
182
|
List of overall-input dataset types that should not be mocked.
|
|
180
183
|
mock_failure : `~collections.abc.Mapping`
|
|
181
184
|
Quanta that should raise exceptions.
|
|
182
|
-
|
|
185
|
+
for_execution : `bool`, optional
|
|
186
|
+
If `True`, the script is being used to feed another that will execute
|
|
187
|
+
the given quanta, and hence all information needed for execution must
|
|
188
|
+
be loaded.
|
|
189
|
+
for_init_output_run : `bool`, optional
|
|
190
|
+
If `True`, the script is being used to feed another that will
|
|
191
|
+
initialize the output run, and hence all information needed to do so
|
|
192
|
+
must be loaded.
|
|
193
|
+
**kwargs : `dict` [`str`, `str`]
|
|
183
194
|
Ignored; click commands may accept options for more than one script
|
|
184
195
|
function and pass all the option kwargs to each of the script functions
|
|
185
196
|
which ignore these unused kwargs.
|
|
186
197
|
|
|
187
198
|
Returns
|
|
188
199
|
-------
|
|
189
|
-
|
|
190
|
-
The
|
|
200
|
+
qg : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
|
|
201
|
+
The quantum graph object that was created or loaded.
|
|
191
202
|
"""
|
|
192
203
|
# make sure that --extend-run always enables --skip-existing
|
|
193
204
|
if extend_run:
|
|
@@ -213,16 +224,43 @@ def qgraph(
|
|
|
213
224
|
if skip_existing and run:
|
|
214
225
|
skip_existing_in += (run,)
|
|
215
226
|
|
|
227
|
+
qgc: PredictedQuantumGraphComponents
|
|
216
228
|
if qgraph is not None:
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
229
|
+
# click passes empty tuple as default value for qgraph_node_id
|
|
230
|
+
quantum_ids = (
|
|
231
|
+
{uuid.UUID(q) if not isinstance(q, uuid.UUID) else q for q in qgraph_node_id}
|
|
232
|
+
if qgraph_node_id
|
|
233
|
+
else None
|
|
234
|
+
)
|
|
235
|
+
qgraph = ResourcePath(qgraph)
|
|
236
|
+
match qgraph.getExtension():
|
|
237
|
+
case ".qgraph":
|
|
238
|
+
qgc = PredictedQuantumGraphComponents.from_old_quantum_graph(
|
|
239
|
+
QuantumGraph.loadUri(
|
|
240
|
+
qgraph,
|
|
241
|
+
butler.dimensions,
|
|
242
|
+
nodes=quantum_ids,
|
|
243
|
+
graphID=BuildId(qgraph_id) if qgraph_id is not None else None,
|
|
244
|
+
)
|
|
245
|
+
)
|
|
246
|
+
case ".qg":
|
|
247
|
+
if qgraph_id is not None:
|
|
248
|
+
_LOG.warning("--qgraph-id is ignored when loading new '.qg' files.")
|
|
249
|
+
if for_execution or for_init_output_run or save_qgraph or show.needs_full_qg:
|
|
250
|
+
import_mode = TaskImportMode.ASSUME_CONSISTENT_EDGES
|
|
251
|
+
else:
|
|
252
|
+
import_mode = TaskImportMode.DO_NOT_IMPORT
|
|
253
|
+
with PredictedQuantumGraph.open(qgraph, import_mode=import_mode) as reader:
|
|
254
|
+
if for_execution or qgraph_dot or qgraph_mermaid or show.needs_full_qg:
|
|
255
|
+
# This reads everything for the given quanta.
|
|
256
|
+
reader.read_execution_quanta(quantum_ids)
|
|
257
|
+
elif for_init_output_run:
|
|
258
|
+
reader.read_init_quanta()
|
|
259
|
+
else:
|
|
260
|
+
reader.read_thin_graph()
|
|
261
|
+
qgc = reader.components
|
|
262
|
+
case ext:
|
|
263
|
+
raise ValueError(f"Unrecognized extension for quantum graph: {ext!r}")
|
|
226
264
|
|
|
227
265
|
# pipeline can not be provided in this case
|
|
228
266
|
if pipeline_graph_factory:
|
|
@@ -265,38 +303,37 @@ def qgraph(
|
|
|
265
303
|
output_run=run,
|
|
266
304
|
data_id_tables=data_id_tables,
|
|
267
305
|
)
|
|
268
|
-
#
|
|
306
|
+
# Accumulate metadata (QB builder adds some of its own).
|
|
269
307
|
metadata = {
|
|
270
|
-
"input": inputs,
|
|
271
|
-
"output": output,
|
|
272
308
|
"butler_argument": str(butler_config),
|
|
273
|
-
"output_run": run,
|
|
274
309
|
"extend_run": extend_run,
|
|
275
310
|
"skip_existing_in": skip_existing_in,
|
|
276
311
|
"skip_existing": skip_existing,
|
|
277
312
|
"data_query": data_query,
|
|
278
313
|
}
|
|
279
314
|
assert run is not None, "Butler output run collection must be defined"
|
|
280
|
-
|
|
315
|
+
qgc = graph_builder.finish(
|
|
316
|
+
output, metadata=metadata, attach_datastore_records=qgraph_datastore_records
|
|
317
|
+
)
|
|
281
318
|
|
|
282
|
-
if
|
|
283
|
-
# Nothing to do.
|
|
319
|
+
if not summarize_quantum_graph(qgc.header):
|
|
284
320
|
return None
|
|
285
|
-
summarize_quantum_graph(qgraph)
|
|
286
321
|
|
|
287
322
|
if save_qgraph:
|
|
288
|
-
_LOG.verbose("Writing
|
|
289
|
-
|
|
323
|
+
_LOG.verbose("Writing quantum graph to %r.", save_qgraph)
|
|
324
|
+
qgc.write(save_qgraph)
|
|
325
|
+
|
|
326
|
+
qg = qgc.assemble()
|
|
290
327
|
|
|
291
328
|
if qgraph_dot:
|
|
292
329
|
_LOG.verbose("Writing quantum graph DOT visualization to %r.", qgraph_dot)
|
|
293
|
-
graph2dot(
|
|
330
|
+
graph2dot(qg, qgraph_dot) # TODO[DM-51850]: make this work
|
|
294
331
|
|
|
295
332
|
if qgraph_mermaid:
|
|
296
333
|
_LOG.verbose("Writing quantum graph Mermaid visualization to %r.", qgraph_mermaid)
|
|
297
|
-
graph2mermaid(
|
|
334
|
+
graph2mermaid(qg, qgraph_mermaid) # TODO[DM-51850]: make this work
|
|
298
335
|
|
|
299
336
|
# optionally dump some info.
|
|
300
|
-
show.show_graph_info(
|
|
337
|
+
show.show_graph_info(qg, butler_config) # TODO[DM-51850]: make this work
|
|
301
338
|
|
|
302
|
-
return
|
|
339
|
+
return qg
|
|
@@ -33,8 +33,9 @@ from typing import TYPE_CHECKING, Literal
|
|
|
33
33
|
import astropy.units as u
|
|
34
34
|
|
|
35
35
|
import lsst.utils.timer
|
|
36
|
-
from lsst.pipe.base import ExecutionResources,
|
|
36
|
+
from lsst.pipe.base import ExecutionResources, TaskFactory
|
|
37
37
|
from lsst.pipe.base.mp_graph_executor import MPGraphExecutor
|
|
38
|
+
from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
|
|
38
39
|
from lsst.pipe.base.single_quantum_executor import SingleQuantumExecutor
|
|
39
40
|
from lsst.resources import ResourcePath, ResourcePathExpression
|
|
40
41
|
from lsst.utils.doImport import doImportType
|
|
@@ -52,7 +53,7 @@ _LOG = getLogger(__name__)
|
|
|
52
53
|
|
|
53
54
|
|
|
54
55
|
def run(
|
|
55
|
-
qg:
|
|
56
|
+
qg: PredictedQuantumGraph,
|
|
56
57
|
*,
|
|
57
58
|
task_factory: TaskFactory | None = None,
|
|
58
59
|
pdb: str | None,
|
|
@@ -93,8 +94,8 @@ def run(
|
|
|
93
94
|
|
|
94
95
|
Parameters
|
|
95
96
|
----------
|
|
96
|
-
qg : `lsst.pipe.base.
|
|
97
|
-
A
|
|
97
|
+
qg : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
|
|
98
|
+
A quantum graph generated by a previous subcommand.
|
|
98
99
|
task_factory : `lsst.pipe.base.TaskFactory`, optional
|
|
99
100
|
A custom task factory to use.
|
|
100
101
|
pdb : `str`, optional
|
|
@@ -211,19 +212,17 @@ def run(
|
|
|
211
212
|
enable_lsst_debug = debug
|
|
212
213
|
del debug
|
|
213
214
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
output_run = output_run
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
graph_output_run = qg.metadata.get("output_run", output_run)
|
|
223
|
-
if graph_output_run != output_run:
|
|
215
|
+
if not output_run:
|
|
216
|
+
# If we have no output run specified, use the one from the graph rather
|
|
217
|
+
# than letting a new timestamped run be created.
|
|
218
|
+
output_run = qg.header.output_run
|
|
219
|
+
else:
|
|
220
|
+
# Check that output run defined on command line is consistent with
|
|
221
|
+
# quantum graph.
|
|
222
|
+
if qg.header.output_run != output_run:
|
|
224
223
|
raise ValueError(
|
|
225
224
|
f"Output run defined on command line ({output_run}) has to be "
|
|
226
|
-
f"identical to graph metadata ({
|
|
225
|
+
f"identical to graph metadata ({qg.header.output_run}). "
|
|
227
226
|
"To update graph metadata run `pipetask update-graph-run` command."
|
|
228
227
|
)
|
|
229
228
|
|
|
@@ -43,8 +43,9 @@ from lsst.daf.butler import (
|
|
|
43
43
|
Quantum,
|
|
44
44
|
QuantumBackedButler,
|
|
45
45
|
)
|
|
46
|
-
from lsst.pipe.base import
|
|
46
|
+
from lsst.pipe.base import ExecutionResources, TaskFactory
|
|
47
47
|
from lsst.pipe.base.mp_graph_executor import MPGraphExecutor
|
|
48
|
+
from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
|
|
48
49
|
from lsst.pipe.base.single_quantum_executor import SingleQuantumExecutor
|
|
49
50
|
from lsst.resources import ResourcePath, ResourcePathExpression
|
|
50
51
|
from lsst.utils.logging import VERBOSE, getLogger
|
|
@@ -147,31 +148,31 @@ def run_qbb(
|
|
|
147
148
|
if not enable_implicit_threading:
|
|
148
149
|
disable_implicit_threading()
|
|
149
150
|
|
|
151
|
+
# click passes empty tuple as default value for qgraph_node_id
|
|
152
|
+
quantum_ids = (
|
|
153
|
+
{uuid.UUID(q) if not isinstance(q, uuid.UUID) else q for q in qgraph_node_id}
|
|
154
|
+
if qgraph_node_id
|
|
155
|
+
else None
|
|
156
|
+
)
|
|
150
157
|
# Load quantum graph.
|
|
151
|
-
nodes = qgraph_node_id or None
|
|
152
158
|
with lsst.utils.timer.time_this(
|
|
153
159
|
_LOG,
|
|
154
|
-
msg=f"Reading {str(len(
|
|
160
|
+
msg=f"Reading {str(len(quantum_ids)) if quantum_ids is not None else 'all'} quanta.",
|
|
155
161
|
level=VERBOSE,
|
|
156
162
|
) as qg_read_time:
|
|
157
|
-
qg =
|
|
158
|
-
qgraph, nodes=nodes, graphID=BuildId(qgraph_id) if qgraph_id is not None else None
|
|
159
|
-
)
|
|
163
|
+
qg = PredictedQuantumGraph.read_execution_quanta(qgraph, quantum_ids=quantum_ids)
|
|
160
164
|
job_metadata = {"qg_read_time": qg_read_time.duration, "qg_size": len(qg)}
|
|
161
165
|
|
|
162
|
-
|
|
163
|
-
raise ValueError("QuantumGraph is missing metadata, cannot continue.")
|
|
164
|
-
|
|
165
|
-
summarize_quantum_graph(qg)
|
|
166
|
+
summarize_quantum_graph(qg.header)
|
|
166
167
|
|
|
167
|
-
dataset_types = {
|
|
168
|
+
dataset_types = {dtn.name: dtn.dataset_type for dtn in qg.pipeline_graph.dataset_types.values()}
|
|
168
169
|
|
|
169
170
|
# Ensure that QBB uses shared datastore cache.
|
|
170
171
|
ButlerFactory.define_datastore_cache()
|
|
171
172
|
|
|
172
173
|
_butler_factory = _QBBFactory(
|
|
173
174
|
butler_config=butler_config,
|
|
174
|
-
dimensions=qg.universe,
|
|
175
|
+
dimensions=qg.pipeline_graph.universe,
|
|
175
176
|
dataset_types=dataset_types,
|
|
176
177
|
config_search_path=config_search_path,
|
|
177
178
|
)
|
|
@@ -25,16 +25,20 @@
|
|
|
25
25
|
# You should have received a copy of the GNU General Public License
|
|
26
26
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
import logging
|
|
29
|
+
|
|
30
|
+
from lsst.pipe.base.quantum_graph import PredictedQuantumGraphComponents
|
|
29
31
|
from lsst.resources import ResourcePathExpression
|
|
30
32
|
|
|
33
|
+
_LOG = logging.getLogger(__name__)
|
|
34
|
+
|
|
31
35
|
|
|
32
36
|
def update_graph_run(
|
|
33
37
|
input_graph: ResourcePathExpression,
|
|
34
38
|
run: str,
|
|
35
39
|
output_graph: ResourcePathExpression,
|
|
36
|
-
metadata_run_key: str,
|
|
37
|
-
update_graph_id: bool,
|
|
40
|
+
metadata_run_key: str = "output_run",
|
|
41
|
+
update_graph_id: bool = False,
|
|
38
42
|
) -> None:
|
|
39
43
|
"""Update quantum graph with new output run name and dataset IDs and save
|
|
40
44
|
updated graph to a file.
|
|
@@ -47,15 +51,15 @@ def update_graph_run(
|
|
|
47
51
|
Collection name, if collection exists it must be of ``RUN`` type.
|
|
48
52
|
output_graph : `~lsst.resources.ResourcePathExpression`
|
|
49
53
|
Location to store updated quantum graph.
|
|
50
|
-
metadata_run_key : `str
|
|
51
|
-
|
|
52
|
-
with new run name. If metadata is missing it is not
|
|
53
|
-
updated. If metadata is present but key is missing, it will be
|
|
54
|
-
added.
|
|
54
|
+
metadata_run_key : `str`, optional
|
|
55
|
+
Ignored (overriding warns).
|
|
55
56
|
update_graph_id : `bool`
|
|
56
|
-
|
|
57
|
+
Ignored (overriding warns).
|
|
57
58
|
"""
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
59
|
+
qgc = PredictedQuantumGraphComponents.read_execution_quanta(input_graph)
|
|
60
|
+
if metadata_run_key and metadata_run_key != "output_run":
|
|
61
|
+
_LOG.warning("--metadata-run-key is now ignored.")
|
|
62
|
+
if update_graph_id:
|
|
63
|
+
_LOG.warning("--update-graph-id is now ignored.")
|
|
64
|
+
qgc.update_output_run(run)
|
|
65
|
+
qgc.write(output_graph)
|