lsst-ctrl-mpexec 29.2025.3900__tar.gz → 29.2025.4100__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. {lsst_ctrl_mpexec-29.2025.3900/python/lsst_ctrl_mpexec.egg-info → lsst_ctrl_mpexec-29.2025.4100}/PKG-INFO +1 -1
  2. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/_pipeline_graph_factory.py +21 -3
  3. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/cmd/commands.py +17 -46
  4. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py +25 -6
  5. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/qgraph.py +67 -30
  6. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/run.py +14 -15
  7. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/run_qbb.py +13 -12
  8. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/update_graph_run.py +17 -13
  9. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/utils.py +64 -13
  10. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/showInfo.py +36 -32
  11. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/singleQuantumExecutor.py +1 -1
  12. lsst_ctrl_mpexec-29.2025.4100/python/lsst/ctrl/mpexec/version.py +2 -0
  13. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100/python/lsst_ctrl_mpexec.egg-info}/PKG-INFO +1 -1
  14. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/SOURCES.txt +4 -4
  15. lsst_ctrl_mpexec-29.2025.4100/tests/test_build.py +302 -0
  16. lsst_ctrl_mpexec-29.2025.3900/tests/test_cliCmdQgraph.py → lsst_ctrl_mpexec-29.2025.4100/tests/test_cliCmdUpdateGraphRun.py +21 -27
  17. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/tests/test_cliUtils.py +14 -13
  18. lsst_ctrl_mpexec-29.2025.4100/tests/test_qgraph.py +144 -0
  19. lsst_ctrl_mpexec-29.2025.4100/tests/test_run.py +674 -0
  20. lsst_ctrl_mpexec-29.2025.3900/python/lsst/ctrl/mpexec/version.py +0 -2
  21. lsst_ctrl_mpexec-29.2025.3900/tests/test_cliCmdUpdateGraphRun.py +0 -112
  22. lsst_ctrl_mpexec-29.2025.3900/tests/test_cliScript.py +0 -266
  23. lsst_ctrl_mpexec-29.2025.3900/tests/test_cmdLineFwk.py +0 -1218
  24. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/COPYRIGHT +0 -0
  25. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/LICENSE +0 -0
  26. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/MANIFEST.in +0 -0
  27. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/README.rst +0 -0
  28. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/bsd_license.txt +0 -0
  29. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/doc/lsst.ctrl.mpexec/CHANGES.rst +0 -0
  30. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/doc/lsst.ctrl.mpexec/configuring-pipetask-tasks.rst +0 -0
  31. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/doc/lsst.ctrl.mpexec/index.rst +0 -0
  32. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/doc/lsst.ctrl.mpexec/pipetask.rst +0 -0
  33. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/gpl-v3.0.txt +0 -0
  34. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/pyproject.toml +0 -0
  35. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/__init__.py +0 -0
  36. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/__init__.py +0 -0
  37. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/__init__.py +0 -0
  38. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/__init__.py +0 -0
  39. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/butler_factory.py +0 -0
  40. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/cmd/__init__.py +0 -0
  41. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/opt/__init__.py +0 -0
  42. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/opt/arguments.py +0 -0
  43. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py +0 -0
  44. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/opt/options.py +0 -0
  45. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/pipetask.py +0 -0
  46. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/__init__.py +0 -0
  47. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/build.py +0 -0
  48. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/cleanup.py +0 -0
  49. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/confirmable.py +0 -0
  50. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/purge.py +0 -0
  51. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/cli/script/report.py +0 -0
  52. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/execFixupDataId.py +0 -0
  53. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/executionGraphFixup.py +0 -0
  54. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/log_capture.py +0 -0
  55. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/mpGraphExecutor.py +0 -0
  56. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/preExecInit.py +0 -0
  57. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/py.typed +0 -0
  58. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/quantumGraphExecutor.py +0 -0
  59. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/reports.py +0 -0
  60. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/separablePipelineExecutor.py +0 -0
  61. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/simple_pipeline_executor.py +0 -0
  62. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/taskFactory.py +0 -0
  63. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst/ctrl/mpexec/util.py +0 -0
  64. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/dependency_links.txt +0 -0
  65. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/entry_points.txt +0 -0
  66. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/requires.txt +0 -0
  67. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/top_level.txt +0 -0
  68. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/python/lsst_ctrl_mpexec.egg-info/zip-safe +0 -0
  69. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/setup.cfg +0 -0
  70. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/tests/test_cliCmdCleanup.py +0 -0
  71. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/tests/test_cliCmdPurge.py +0 -0
  72. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/tests/test_cliCmdReport.py +0 -0
  73. {lsst_ctrl_mpexec-29.2025.3900 → lsst_ctrl_mpexec-29.2025.4100}/tests/test_preExecInit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lsst-ctrl-mpexec
3
- Version: 29.2025.3900
3
+ Version: 29.2025.4100
4
4
  Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
5
5
  Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
6
6
  License: BSD 3-Clause License
@@ -45,18 +45,30 @@ class PipelineGraphFactory:
45
45
  dimension schema.
46
46
  select_tasks : `str`, optional
47
47
  String expression that filters the tasks in the pipeline graph.
48
+ pipeline_graph : `lsst.pipe.base.pipeline_graph.PipelineGraph`, optional
49
+ Already-constructed pipeline graph.
48
50
  """
49
51
 
50
- def __init__(self, pipeline: Pipeline, butler: Butler | None = None, select_tasks: str = ""):
52
+ def __init__(
53
+ self,
54
+ pipeline: Pipeline | None = None,
55
+ butler: Butler | None = None,
56
+ select_tasks: str = "",
57
+ *,
58
+ pipeline_graph: PipelineGraph | None = None,
59
+ ):
60
+ if pipeline is None and pipeline_graph is None:
61
+ raise TypeError("At least one of 'pipeline' and 'pipeline_graph' must not be `None`.")
51
62
  self._pipeline = pipeline
52
63
  self._registry = butler.registry if butler is not None else None
53
64
  self._select_tasks = select_tasks
54
- self._pipeline_graph: PipelineGraph | None = None
65
+ self._pipeline_graph: PipelineGraph | None = pipeline_graph
55
66
  self._resolved: bool = False
56
67
  self._for_visualization_only: bool = False
57
68
 
58
69
  def __call__(self, *, resolve: bool = True, visualization_only: bool = False) -> PipelineGraph:
59
70
  if self._pipeline_graph is None:
71
+ assert self._pipeline is not None, "Guaranteed at construction."
60
72
  self._pipeline_graph = self._pipeline.to_graph()
61
73
  if self._select_tasks:
62
74
  self._pipeline_graph = self._pipeline_graph.select(self._select_tasks)
@@ -71,6 +83,8 @@ class PipelineGraphFactory:
71
83
  @property
72
84
  def pipeline(self) -> Pipeline:
73
85
  """The original pipeline definition."""
86
+ if self._pipeline is None:
87
+ raise RuntimeError("Cannot obtain pipeline from pipeline graph.")
74
88
  if self._select_tasks:
75
89
  raise RuntimeError(
76
90
  "The --select-tasks option cannot be used with operations that return or display a "
@@ -79,4 +93,8 @@ class PipelineGraphFactory:
79
93
  return self._pipeline
80
94
 
81
95
  def __bool__(self) -> bool:
82
- return bool(self._pipeline)
96
+ if self._pipeline is not None:
97
+ return bool(self._pipeline)
98
+ else:
99
+ assert self._pipeline_graph is not None, "Guaranteed at construction."
100
+ return bool(self._pipeline_graph.tasks)
@@ -35,25 +35,22 @@ from typing import Any
35
35
 
36
36
  import click
37
37
 
38
- import lsst.pipe.base.cli.opt as pipeBaseOpts
39
38
  from lsst.ctrl.mpexec.showInfo import ShowInfo
40
39
  from lsst.daf.butler.cli.opt import (
41
40
  collections_option,
42
- config_file_option,
43
- config_option,
44
41
  confirm_option,
45
42
  options_file_option,
46
43
  processes_option,
47
44
  repo_argument,
48
45
  where_option,
49
46
  )
50
- from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap
47
+ from lsst.daf.butler.cli.utils import catch_and_exit, option_section, unwrap
51
48
  from lsst.pipe.base.quantum_reports import Report
52
49
 
53
50
  from .. import opt as ctrlMpExecOpts
54
51
  from .. import script
55
52
  from ..script import confirmable
56
- from ..utils import PipetaskCommand, makePipelineActions
53
+ from ..utils import PipetaskCommand, collect_pipeline_actions
57
54
 
58
55
  epilog = unwrap(
59
56
  """Notes:
@@ -69,41 +66,6 @@ ignored.)
69
66
  )
70
67
 
71
68
 
72
- def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]:
73
- """Extract pipeline building options, replace them with PipelineActions,
74
- return updated `kwargs`.
75
-
76
- Notes
77
- -----
78
- The pipeline actions (task, delete, config, config_file, and instrument)
79
- must be handled in the order they appear on the command line, but the CLI
80
- specification gives them all different option names. So, instead of using
81
- the individual action options as they appear in kwargs (because
82
- invocation order can't be known), we capture the CLI arguments by
83
- overriding `click.Command.parse_args` and save them in the Context's
84
- `obj` parameter. We use `makePipelineActions` to create a list of
85
- pipeline actions from the CLI arguments and pass that list to the script
86
- function using the `pipeline_actions` kwarg name, and remove the action
87
- options from kwargs.
88
- """
89
- for pipelineAction in (
90
- ctrlMpExecOpts.task_option.name(),
91
- ctrlMpExecOpts.delete_option.name(),
92
- config_option.name(),
93
- config_file_option.name(),
94
- pipeBaseOpts.instrument_option.name(),
95
- ):
96
- kwargs.pop(pipelineAction)
97
-
98
- actions = makePipelineActions(MWCtxObj.getFrom(ctx).args)
99
- pipeline_actions = []
100
- for action in actions:
101
- pipeline_actions.append(action)
102
-
103
- kwargs["pipeline_actions"] = pipeline_actions
104
- return kwargs
105
-
106
-
107
69
  def _unhandledShow(show: ShowInfo, cmd: str) -> None:
108
70
  if show.unhandled:
109
71
  print(
@@ -125,7 +87,7 @@ def build(ctx: click.Context, **kwargs: Any) -> None:
125
87
 
126
88
  This does not require input data to be specified.
127
89
  """
128
- kwargs = _collectActions(ctx, **kwargs)
90
+ kwargs = collect_pipeline_actions(ctx, **kwargs)
129
91
  show = ShowInfo(kwargs.pop("show", []))
130
92
  if kwargs.get("butler_config") is not None and (
131
93
  {"pipeline-graph", "task-graph"}.isdisjoint(show.commands) and not kwargs.get("pipeline_dot")
@@ -187,7 +149,7 @@ concurrency = multiprocessing
187
149
  @catch_and_exit
188
150
  def qgraph(ctx: click.Context, **kwargs: Any) -> None:
189
151
  """Build and optionally save quantum graph."""
190
- kwargs = _collectActions(ctx, **kwargs)
152
+ kwargs = collect_pipeline_actions(ctx, **kwargs)
191
153
  summary = kwargs.pop("summary", None)
192
154
  with coverage_context(kwargs):
193
155
  show = ShowInfo(kwargs.pop("show", []))
@@ -204,12 +166,21 @@ def qgraph(ctx: click.Context, **kwargs: Any) -> None:
204
166
  file=sys.stderr,
205
167
  )
206
168
  return
207
- if (qgraph := script.qgraph(pipeline_graph_factory, **kwargs, show=show)) is None:
169
+ if (
170
+ qgraph := script.qgraph(
171
+ pipeline_graph_factory,
172
+ **kwargs,
173
+ show=show,
174
+ # Making a summary report requires that we load the same graph
175
+ # components as execution.
176
+ for_execution=(summary is not None),
177
+ )
178
+ ) is None:
208
179
  raise click.ClickException("QuantumGraph was empty; ERROR logs above should provide details.")
209
180
  # QuantumGraph-only summary call here since script.qgraph also called
210
181
  # by run methods.
211
182
  if summary:
212
- report = Report(qgraphSummary=qgraph.getSummary())
183
+ report = Report(qgraphSummary=qgraph._make_summary())
213
184
  with open(summary, "w") as out:
214
185
  # Do not save fields that are not set.
215
186
  out.write(report.model_dump_json(exclude_none=True, indent=2))
@@ -222,7 +193,7 @@ def qgraph(ctx: click.Context, **kwargs: Any) -> None:
222
193
  @catch_and_exit
223
194
  def run(ctx: click.Context, **kwargs: Any) -> None:
224
195
  """Build and execute pipeline and quantum graph."""
225
- kwargs = _collectActions(ctx, **kwargs)
196
+ kwargs = collect_pipeline_actions(ctx, **kwargs)
226
197
  with coverage_context(kwargs):
227
198
  show = ShowInfo(kwargs.pop("show", []))
228
199
  pipeline_graph_factory = script.build(**kwargs, show=show)
@@ -233,7 +204,7 @@ def run(ctx: click.Context, **kwargs: Any) -> None:
233
204
  file=sys.stderr,
234
205
  )
235
206
  return
236
- if (qgraph := script.qgraph(pipeline_graph_factory, **kwargs, show=show)) is None:
207
+ if (qgraph := script.qgraph(pipeline_graph_factory, for_execution=True, **kwargs, show=show)) is None:
237
208
  raise click.ClickException("QuantumGraph was empty; ERROR logs above should provide details.")
238
209
  _unhandledShow(show, "run")
239
210
  if show.handled:
@@ -28,6 +28,9 @@
28
28
  from __future__ import annotations
29
29
 
30
30
  from lsst.pipe.base import BuildId, QuantumGraph
31
+ from lsst.pipe.base.pipeline_graph import TaskImportMode
32
+ from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
33
+ from lsst.resources import ResourcePath, ResourcePathExpression
31
34
  from lsst.utils.logging import getLogger
32
35
 
33
36
  from ..butler_factory import ButlerFactory
@@ -37,7 +40,7 @@ _LOG = getLogger(__name__)
37
40
 
38
41
  def pre_exec_init_qbb(
39
42
  butler_config: str,
40
- qgraph: str,
43
+ qgraph: ResourcePathExpression,
41
44
  qgraph_id: str | None,
42
45
  config_search_path: list[str] | None,
43
46
  **kwargs: object,
@@ -64,11 +67,27 @@ def pre_exec_init_qbb(
64
67
  function and pass all the option kwargs to each of the script functions
65
68
  which ignore these unused kwargs.
66
69
  """
67
- _LOG.verbose("Reading full quantum graph from %s.", qgraph)
68
- # Load quantum graph. We do not really need individual Quanta here,
69
- # but we need datastore records for initInputs, and those are only
70
- # available from Quanta, so load the whole thing.
71
- qg = QuantumGraph.loadUri(qgraph, graphID=BuildId(qgraph_id) if qgraph_id is not None else None)
70
+ qgraph = ResourcePath(qgraph)
71
+ match qgraph.getExtension():
72
+ case ".qgraph":
73
+ _LOG.verbose("Reading full quantum graph from %s.", qgraph)
74
+ qg = PredictedQuantumGraph.from_old_quantum_graph(
75
+ QuantumGraph.loadUri(
76
+ qgraph,
77
+ graphID=BuildId(qgraph_id) if qgraph_id is not None else None,
78
+ )
79
+ )
80
+ case ".qg":
81
+ _LOG.verbose("Reading init quanta from quantum graph from %s.", qgraph)
82
+ if qgraph_id is not None:
83
+ _LOG.warning("--qgraph-id is ignored when loading new '.qg' files.")
84
+ with PredictedQuantumGraph.open(
85
+ qgraph, import_mode=TaskImportMode.ASSUME_CONSISTENT_EDGES
86
+ ) as reader:
87
+ reader.read_init_quanta()
88
+ qg = reader.finish()
89
+ case ext:
90
+ raise ValueError(f"Unrecognized extension for quantum graph: {ext!r}")
72
91
 
73
92
  # Ensure that QBB uses shared datastore cache for writes.
74
93
  ButlerFactory.define_datastore_cache()
@@ -42,6 +42,8 @@ from lsst.pipe.base.all_dimensions_quantum_graph_builder import (
42
42
  )
43
43
  from lsst.pipe.base.dot_tools import graph2dot
44
44
  from lsst.pipe.base.mermaid_tools import graph2mermaid
45
+ from lsst.pipe.base.pipeline_graph import TaskImportMode
46
+ from lsst.pipe.base.quantum_graph import PredictedQuantumGraph, PredictedQuantumGraphComponents
45
47
  from lsst.resources import ResourcePath, ResourcePathExpression
46
48
  from lsst.utils.iteration import ensure_iterable
47
49
  from lsst.utils.logging import getLogger
@@ -60,7 +62,7 @@ _LOG = getLogger(__name__)
60
62
  def qgraph(
61
63
  pipeline_graph_factory: PipelineGraphFactory | None,
62
64
  *,
63
- qgraph: QuantumGraph | ResourcePathExpression | None,
65
+ qgraph: ResourcePathExpression | None,
64
66
  qgraph_id: str | None,
65
67
  qgraph_node_id: Iterable[uuid.UUID | str] | None,
66
68
  qgraph_datastore_records: bool,
@@ -85,8 +87,10 @@ def qgraph(
85
87
  mock: bool = False,
86
88
  unmocked_dataset_types: Sequence[str],
87
89
  mock_failure: Mapping[str, ForcedFailure],
90
+ for_execution: bool = False,
91
+ for_init_output_run: bool = False,
88
92
  **kwargs: object,
89
- ) -> QuantumGraph | None:
93
+ ) -> PredictedQuantumGraph | None:
90
94
  """Implement the command line interface `pipetask qgraph` subcommand.
91
95
 
92
96
  Should only be called by command line tools and unit test code that test
@@ -97,8 +101,7 @@ def qgraph(
97
101
  pipeline_graph_factory : `..PipelineGraphFactory` or `None`
98
102
  A factory that holds the pipeline and can produce a pipeline graph.
99
103
  If this is not `None` then `qgraph` should be `None`.
100
- qgraph : `lsst.pipe.base.QuantumGraph`, convertible to \
101
- `lsst.resources.ResourcePath` or `None`
104
+ qgraph : convertible to `lsst.resources.ResourcePath`, or `None`
102
105
  URI location for a serialized quantum graph definition. If this option
103
106
  is not `None` then ``pipeline_graph_factory`` should be `None`.
104
107
  qgraph_id : `str` or `None`
@@ -179,15 +182,23 @@ def qgraph(
179
182
  List of overall-input dataset types that should not be mocked.
180
183
  mock_failure : `~collections.abc.Mapping`
181
184
  Quanta that should raise exceptions.
182
- **kwargs : `object`
185
+ for_execution : `bool`, optional
186
+ If `True`, the script is being used to feed another that will execute
187
+ the given quanta, and hence all information needed for execution must
188
+ be loaded.
189
+ for_init_output_run : `bool`, optional
190
+ If `True`, the script is being used to feed another that will
191
+ initialize the output run, and hence all information needed to do so
192
+ must be loaded.
193
+ **kwargs : `dict` [`str`, `str`]
183
194
  Ignored; click commands may accept options for more than one script
184
195
  function and pass all the option kwargs to each of the script functions
185
196
  which ignore these unused kwargs.
186
197
 
187
198
  Returns
188
199
  -------
189
- qgraph : `lsst.pipe.base.QuantumGraph` or `None`
190
- The qgraph object that was created.
200
+ qg : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
201
+ The quantum graph object that was created or loaded.
191
202
  """
192
203
  # make sure that --extend-run always enables --skip-existing
193
204
  if extend_run:
@@ -213,16 +224,43 @@ def qgraph(
213
224
  if skip_existing and run:
214
225
  skip_existing_in += (run,)
215
226
 
227
+ qgc: PredictedQuantumGraphComponents
216
228
  if qgraph is not None:
217
- if not isinstance(qgraph, QuantumGraph):
218
- # click passes empty tuple as default value for qgraph_node_id
219
- nodes = qgraph_node_id or None
220
- qgraph = QuantumGraph.loadUri(
221
- qgraph,
222
- butler.dimensions,
223
- nodes=nodes,
224
- graphID=BuildId(qgraph_id) if qgraph_id is not None else None,
225
- )
229
+ # click passes empty tuple as default value for qgraph_node_id
230
+ quantum_ids = (
231
+ {uuid.UUID(q) if not isinstance(q, uuid.UUID) else q for q in qgraph_node_id}
232
+ if qgraph_node_id
233
+ else None
234
+ )
235
+ qgraph = ResourcePath(qgraph)
236
+ match qgraph.getExtension():
237
+ case ".qgraph":
238
+ qgc = PredictedQuantumGraphComponents.from_old_quantum_graph(
239
+ QuantumGraph.loadUri(
240
+ qgraph,
241
+ butler.dimensions,
242
+ nodes=quantum_ids,
243
+ graphID=BuildId(qgraph_id) if qgraph_id is not None else None,
244
+ )
245
+ )
246
+ case ".qg":
247
+ if qgraph_id is not None:
248
+ _LOG.warning("--qgraph-id is ignored when loading new '.qg' files.")
249
+ if for_execution or for_init_output_run or save_qgraph or show.needs_full_qg:
250
+ import_mode = TaskImportMode.ASSUME_CONSISTENT_EDGES
251
+ else:
252
+ import_mode = TaskImportMode.DO_NOT_IMPORT
253
+ with PredictedQuantumGraph.open(qgraph, import_mode=import_mode) as reader:
254
+ if for_execution or qgraph_dot or qgraph_mermaid or show.needs_full_qg:
255
+ # This reads everything for the given quanta.
256
+ reader.read_execution_quanta(quantum_ids)
257
+ elif for_init_output_run:
258
+ reader.read_init_quanta()
259
+ else:
260
+ reader.read_thin_graph()
261
+ qgc = reader.components
262
+ case ext:
263
+ raise ValueError(f"Unrecognized extension for quantum graph: {ext!r}")
226
264
 
227
265
  # pipeline can not be provided in this case
228
266
  if pipeline_graph_factory:
@@ -265,38 +303,37 @@ def qgraph(
265
303
  output_run=run,
266
304
  data_id_tables=data_id_tables,
267
305
  )
268
- # accumulate metadata
306
+ # Accumulate metadata (QB builder adds some of its own).
269
307
  metadata = {
270
- "input": inputs,
271
- "output": output,
272
308
  "butler_argument": str(butler_config),
273
- "output_run": run,
274
309
  "extend_run": extend_run,
275
310
  "skip_existing_in": skip_existing_in,
276
311
  "skip_existing": skip_existing,
277
312
  "data_query": data_query,
278
313
  }
279
314
  assert run is not None, "Butler output run collection must be defined"
280
- qgraph = graph_builder.build(metadata, attach_datastore_records=qgraph_datastore_records)
315
+ qgc = graph_builder.finish(
316
+ output, metadata=metadata, attach_datastore_records=qgraph_datastore_records
317
+ )
281
318
 
282
- if len(qgraph) == 0:
283
- # Nothing to do.
319
+ if not summarize_quantum_graph(qgc.header):
284
320
  return None
285
- summarize_quantum_graph(qgraph)
286
321
 
287
322
  if save_qgraph:
288
- _LOG.verbose("Writing QuantumGraph to %r.", save_qgraph)
289
- qgraph.saveUri(save_qgraph)
323
+ _LOG.verbose("Writing quantum graph to %r.", save_qgraph)
324
+ qgc.write(save_qgraph)
325
+
326
+ qg = qgc.assemble()
290
327
 
291
328
  if qgraph_dot:
292
329
  _LOG.verbose("Writing quantum graph DOT visualization to %r.", qgraph_dot)
293
- graph2dot(qgraph, qgraph_dot)
330
+ graph2dot(qg, qgraph_dot) # TODO[DM-51850]: make this work
294
331
 
295
332
  if qgraph_mermaid:
296
333
  _LOG.verbose("Writing quantum graph Mermaid visualization to %r.", qgraph_mermaid)
297
- graph2mermaid(qgraph, qgraph_mermaid)
334
+ graph2mermaid(qg, qgraph_mermaid) # TODO[DM-51850]: make this work
298
335
 
299
336
  # optionally dump some info.
300
- show.show_graph_info(qgraph, butler_config)
337
+ show.show_graph_info(qg, butler_config) # TODO[DM-51850]: make this work
301
338
 
302
- return qgraph
339
+ return qg
@@ -33,8 +33,9 @@ from typing import TYPE_CHECKING, Literal
33
33
  import astropy.units as u
34
34
 
35
35
  import lsst.utils.timer
36
- from lsst.pipe.base import ExecutionResources, QuantumGraph, TaskFactory
36
+ from lsst.pipe.base import ExecutionResources, TaskFactory
37
37
  from lsst.pipe.base.mp_graph_executor import MPGraphExecutor
38
+ from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
38
39
  from lsst.pipe.base.single_quantum_executor import SingleQuantumExecutor
39
40
  from lsst.resources import ResourcePath, ResourcePathExpression
40
41
  from lsst.utils.doImport import doImportType
@@ -52,7 +53,7 @@ _LOG = getLogger(__name__)
52
53
 
53
54
 
54
55
  def run(
55
- qg: QuantumGraph,
56
+ qg: PredictedQuantumGraph,
56
57
  *,
57
58
  task_factory: TaskFactory | None = None,
58
59
  pdb: str | None,
@@ -93,8 +94,8 @@ def run(
93
94
 
94
95
  Parameters
95
96
  ----------
96
- qg : `lsst.pipe.base.QuantumGraph`
97
- A QuantumGraph generated by a previous subcommand.
97
+ qg : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
98
+ A quantum graph generated by a previous subcommand.
98
99
  task_factory : `lsst.pipe.base.TaskFactory`, optional
99
100
  A custom task factory to use.
100
101
  pdb : `str`, optional
@@ -211,19 +212,17 @@ def run(
211
212
  enable_lsst_debug = debug
212
213
  del debug
213
214
 
214
- # If we have no output run specified, use the one from the graph rather
215
- # than letting a new timestamped run be created.
216
- if not output_run and qg.metadata and (output_run := qg.metadata.get("output_run")):
217
- output_run = output_run
218
-
219
- # Check that output run defined on command line is consistent with
220
- # quantum graph.
221
- if output_run and qg.metadata:
222
- graph_output_run = qg.metadata.get("output_run", output_run)
223
- if graph_output_run != output_run:
215
+ if not output_run:
216
+ # If we have no output run specified, use the one from the graph rather
217
+ # than letting a new timestamped run be created.
218
+ output_run = qg.header.output_run
219
+ else:
220
+ # Check that output run defined on command line is consistent with
221
+ # quantum graph.
222
+ if qg.header.output_run != output_run:
224
223
  raise ValueError(
225
224
  f"Output run defined on command line ({output_run}) has to be "
226
- f"identical to graph metadata ({graph_output_run}). "
225
+ f"identical to graph metadata ({qg.header.output_run}). "
227
226
  "To update graph metadata run `pipetask update-graph-run` command."
228
227
  )
229
228
 
@@ -43,8 +43,9 @@ from lsst.daf.butler import (
43
43
  Quantum,
44
44
  QuantumBackedButler,
45
45
  )
46
- from lsst.pipe.base import BuildId, ExecutionResources, QuantumGraph, TaskFactory
46
+ from lsst.pipe.base import ExecutionResources, TaskFactory
47
47
  from lsst.pipe.base.mp_graph_executor import MPGraphExecutor
48
+ from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
48
49
  from lsst.pipe.base.single_quantum_executor import SingleQuantumExecutor
49
50
  from lsst.resources import ResourcePath, ResourcePathExpression
50
51
  from lsst.utils.logging import VERBOSE, getLogger
@@ -147,31 +148,31 @@ def run_qbb(
147
148
  if not enable_implicit_threading:
148
149
  disable_implicit_threading()
149
150
 
151
+ # click passes empty tuple as default value for qgraph_node_id
152
+ quantum_ids = (
153
+ {uuid.UUID(q) if not isinstance(q, uuid.UUID) else q for q in qgraph_node_id}
154
+ if qgraph_node_id
155
+ else None
156
+ )
150
157
  # Load quantum graph.
151
- nodes = qgraph_node_id or None
152
158
  with lsst.utils.timer.time_this(
153
159
  _LOG,
154
- msg=f"Reading {str(len(nodes)) if nodes is not None else 'all'} quanta.",
160
+ msg=f"Reading {str(len(quantum_ids)) if quantum_ids is not None else 'all'} quanta.",
155
161
  level=VERBOSE,
156
162
  ) as qg_read_time:
157
- qg = QuantumGraph.loadUri(
158
- qgraph, nodes=nodes, graphID=BuildId(qgraph_id) if qgraph_id is not None else None
159
- )
163
+ qg = PredictedQuantumGraph.read_execution_quanta(qgraph, quantum_ids=quantum_ids)
160
164
  job_metadata = {"qg_read_time": qg_read_time.duration, "qg_size": len(qg)}
161
165
 
162
- if qg.metadata is None:
163
- raise ValueError("QuantumGraph is missing metadata, cannot continue.")
164
-
165
- summarize_quantum_graph(qg)
166
+ summarize_quantum_graph(qg.header)
166
167
 
167
- dataset_types = {dstype.name: dstype for dstype in qg.registryDatasetTypes()}
168
+ dataset_types = {dtn.name: dtn.dataset_type for dtn in qg.pipeline_graph.dataset_types.values()}
168
169
 
169
170
  # Ensure that QBB uses shared datastore cache.
170
171
  ButlerFactory.define_datastore_cache()
171
172
 
172
173
  _butler_factory = _QBBFactory(
173
174
  butler_config=butler_config,
174
- dimensions=qg.universe,
175
+ dimensions=qg.pipeline_graph.universe,
175
176
  dataset_types=dataset_types,
176
177
  config_search_path=config_search_path,
177
178
  )
@@ -25,16 +25,20 @@
25
25
  # You should have received a copy of the GNU General Public License
26
26
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
27
 
28
- from lsst.pipe.base import QuantumGraph
28
+ import logging
29
+
30
+ from lsst.pipe.base.quantum_graph import PredictedQuantumGraphComponents
29
31
  from lsst.resources import ResourcePathExpression
30
32
 
33
+ _LOG = logging.getLogger(__name__)
34
+
31
35
 
32
36
  def update_graph_run(
33
37
  input_graph: ResourcePathExpression,
34
38
  run: str,
35
39
  output_graph: ResourcePathExpression,
36
- metadata_run_key: str,
37
- update_graph_id: bool,
40
+ metadata_run_key: str = "output_run",
41
+ update_graph_id: bool = False,
38
42
  ) -> None:
39
43
  """Update quantum graph with new output run name and dataset IDs and save
40
44
  updated graph to a file.
@@ -47,15 +51,15 @@ def update_graph_run(
47
51
  Collection name, if collection exists it must be of ``RUN`` type.
48
52
  output_graph : `~lsst.resources.ResourcePathExpression`
49
53
  Location to store updated quantum graph.
50
- metadata_run_key : `str`
51
- Specifies metadata key corresponding to output run name to update
52
- with new run name. If metadata is missing it is not
53
- updated. If metadata is present but key is missing, it will be
54
- added.
54
+ metadata_run_key : `str`, optional
55
+ Ignored (overriding warns).
55
56
  update_graph_id : `bool`
56
- If `True` then also update graph ID with a new unique value.
57
+ Ignored (overriding warns).
57
58
  """
58
- qgraph = QuantumGraph.loadUri(input_graph)
59
- key = metadata_run_key if metadata_run_key else None
60
- qgraph.updateRun(run, metadata_key=key, update_graph_id=update_graph_id)
61
- qgraph.saveUri(output_graph)
59
+ qgc = PredictedQuantumGraphComponents.read_execution_quanta(input_graph)
60
+ if metadata_run_key and metadata_run_key != "output_run":
61
+ _LOG.warning("--metadata-run-key is now ignored.")
62
+ if update_graph_id:
63
+ _LOG.warning("--update-graph-id is now ignored.")
64
+ qgc.update_output_run(run)
65
+ qgc.write(output_graph)