lsst-ctrl-mpexec 29.2025.1500__py3-none-any.whl → 29.2025.1600__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (20) hide show
  1. lsst/ctrl/mpexec/__init__.py +1 -0
  2. lsst/ctrl/mpexec/_pipeline_graph_factory.py +82 -0
  3. lsst/ctrl/mpexec/cli/cmd/commands.py +4 -4
  4. lsst/ctrl/mpexec/cli/opt/optionGroups.py +2 -0
  5. lsst/ctrl/mpexec/cli/opt/options.py +23 -0
  6. lsst/ctrl/mpexec/cli/script/build.py +15 -7
  7. lsst/ctrl/mpexec/cli/script/qgraph.py +12 -6
  8. lsst/ctrl/mpexec/cmdLineFwk.py +35 -6
  9. lsst/ctrl/mpexec/showInfo.py +26 -30
  10. lsst/ctrl/mpexec/version.py +1 -1
  11. {lsst_ctrl_mpexec-29.2025.1500.dist-info → lsst_ctrl_mpexec-29.2025.1600.dist-info}/METADATA +1 -1
  12. {lsst_ctrl_mpexec-29.2025.1500.dist-info → lsst_ctrl_mpexec-29.2025.1600.dist-info}/RECORD +20 -19
  13. {lsst_ctrl_mpexec-29.2025.1500.dist-info → lsst_ctrl_mpexec-29.2025.1600.dist-info}/WHEEL +0 -0
  14. {lsst_ctrl_mpexec-29.2025.1500.dist-info → lsst_ctrl_mpexec-29.2025.1600.dist-info}/entry_points.txt +0 -0
  15. {lsst_ctrl_mpexec-29.2025.1500.dist-info → lsst_ctrl_mpexec-29.2025.1600.dist-info}/licenses/COPYRIGHT +0 -0
  16. {lsst_ctrl_mpexec-29.2025.1500.dist-info → lsst_ctrl_mpexec-29.2025.1600.dist-info}/licenses/LICENSE +0 -0
  17. {lsst_ctrl_mpexec-29.2025.1500.dist-info → lsst_ctrl_mpexec-29.2025.1600.dist-info}/licenses/bsd_license.txt +0 -0
  18. {lsst_ctrl_mpexec-29.2025.1500.dist-info → lsst_ctrl_mpexec-29.2025.1600.dist-info}/licenses/gpl-v3.0.txt +0 -0
  19. {lsst_ctrl_mpexec-29.2025.1500.dist-info → lsst_ctrl_mpexec-29.2025.1600.dist-info}/top_level.txt +0 -0
  20. {lsst_ctrl_mpexec-29.2025.1500.dist-info → lsst_ctrl_mpexec-29.2025.1600.dist-info}/zip-safe +0 -0
@@ -25,6 +25,7 @@
25
25
  # You should have received a copy of the GNU General Public License
26
26
  # along with this program. If not, see <https://www.gnu.org/licenses/>.
27
27
 
28
+ from ._pipeline_graph_factory import PipelineGraphFactory
28
29
  from .cmdLineFwk import *
29
30
  from .dotTools import *
30
31
  from .executionGraphFixup import *
@@ -0,0 +1,82 @@
1
+ # This file is part of ctrl_mpexec.
2
+ #
3
+ # Developed for the LSST Data Management System.
4
+ # This product includes software developed by the LSST Project
5
+ # (http://www.lsst.org).
6
+ # See the COPYRIGHT file at the top-level directory of this distribution
7
+ # for details of code ownership.
8
+ #
9
+ # This software is dual licensed under the GNU General Public License and also
10
+ # under a 3-clause BSD license. Recipients may choose which of these licenses
11
+ # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
+ # respectively. If you choose the GPL option then the following text applies
13
+ # (but note that there is still no warranty even if you opt for BSD instead):
14
+ #
15
+ # This program is free software: you can redistribute it and/or modify
16
+ # it under the terms of the GNU General Public License as published by
17
+ # the Free Software Foundation, either version 3 of the License, or
18
+ # (at your option) any later version.
19
+ #
20
+ # This program is distributed in the hope that it will be useful,
21
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
+ # GNU General Public License for more details.
24
+ #
25
+ # You should have received a copy of the GNU General Public License
26
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
+
28
+ from __future__ import annotations
29
+
30
+ __all__ = ("PipelineGraphFactory",)
31
+
32
+ from lsst.daf.butler import Butler
33
+ from lsst.pipe.base import Pipeline, PipelineGraph
34
+
35
+
36
+ class PipelineGraphFactory:
37
+ """A factory for building and caching a PipelineGraph.
38
+
39
+ Parameters
40
+ ----------
41
+ pipeline : `lsst.pipe.base.Pipeline`
42
+ Pipeline definition to start from.
43
+ butler : `lsst.daf.butler.Butler` or `None`, optional
44
+ Butler that can be used to resolve dataset type definitions and get
45
+ dimension schema.
46
+ select_tasks : `str`, optional
47
+ String expression that filters the tasks in the pipeline graph.
48
+ """
49
+
50
+ def __init__(self, pipeline: Pipeline, butler: Butler | None = None, select_tasks: str = ""):
51
+ self._pipeline = pipeline
52
+ self._registry = butler.registry if butler is not None else None
53
+ self._select_tasks = select_tasks
54
+ self._pipeline_graph: PipelineGraph | None = None
55
+ self._resolved: bool = False
56
+ self._for_visualization_only: bool = False
57
+
58
+ def __call__(self, *, resolve: bool = True, visualization_only: bool = False) -> PipelineGraph:
59
+ if self._pipeline_graph is None:
60
+ self._pipeline_graph = self._pipeline.to_graph()
61
+ if self._select_tasks:
62
+ self._pipeline_graph = self._pipeline_graph.select(self._select_tasks)
63
+ if resolve and not self._resolved:
64
+ self._pipeline_graph.resolve(self._registry, visualization_only=visualization_only)
65
+ self._resolved = True
66
+ self._for_visualization_only = self._registry is None
67
+ elif resolve and not visualization_only and self._for_visualization_only:
68
+ raise RuntimeError("Cannot resolve pipeline graph without butler.")
69
+ return self._pipeline_graph
70
+
71
+ @property
72
+ def pipeline(self) -> Pipeline:
73
+ """The original pipeline definition."""
74
+ if self._select_tasks:
75
+ raise RuntimeError(
76
+ "The --select-tasks option cannot be used with operations that return or display a "
77
+ "pipeline as YAML, since it only operates on the pipeline graph."
78
+ )
79
+ return self._pipeline
80
+
81
+ def __bool__(self) -> bool:
82
+ return bool(self._pipeline)
@@ -191,14 +191,14 @@ def qgraph(ctx: click.Context, **kwargs: Any) -> None:
191
191
  summary = kwargs.pop("summary", None)
192
192
  with coverage_context(kwargs):
193
193
  show = ShowInfo(kwargs.pop("show", []))
194
- pipeline = script.build(**kwargs, show=show)
194
+ pipeline_graph_factory = script.build(**kwargs, show=show)
195
195
  if show.handled and not show.unhandled:
196
196
  print(
197
197
  "No quantum graph generated. The --show option was given and all options were processed.",
198
198
  file=sys.stderr,
199
199
  )
200
200
  return
201
- if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None:
201
+ if (qgraph := script.qgraph(pipeline_graph_factory, **kwargs, show=show)) is None:
202
202
  raise click.ClickException("QuantumGraph was empty; ERROR logs above should provide details.")
203
203
  # QuantumGraph-only summary call here since script.qgraph also called
204
204
  # by run methods.
@@ -219,7 +219,7 @@ def run(ctx: click.Context, **kwargs: Any) -> None:
219
219
  kwargs = _collectActions(ctx, **kwargs)
220
220
  with coverage_context(kwargs):
221
221
  show = ShowInfo(kwargs.pop("show", []))
222
- pipeline = script.build(**kwargs, show=show)
222
+ pipeline_graph_factory = script.build(**kwargs, show=show)
223
223
  if show.handled and not show.unhandled:
224
224
  print(
225
225
  "No quantum graph generated or pipeline executed. "
@@ -227,7 +227,7 @@ def run(ctx: click.Context, **kwargs: Any) -> None:
227
227
  file=sys.stderr,
228
228
  )
229
229
  return
230
- if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None:
230
+ if (qgraph := script.qgraph(pipeline_graph_factory, **kwargs, show=show)) is None:
231
231
  raise click.ClickException("QuantumGraph was empty; ERROR logs above should provide details.")
232
232
  _unhandledShow(show, "run")
233
233
  if show.handled:
@@ -74,6 +74,7 @@ class pipeline_build_options(OptionGroup): # noqa: N801
74
74
  ),
75
75
  ctrlMpExecOpts.order_pipeline_option(),
76
76
  ctrlMpExecOpts.save_pipeline_option(),
77
+ ctrlMpExecOpts.select_tasks_option(),
77
78
  ctrlMpExecOpts.pipeline_dot_option(),
78
79
  ctrlMpExecOpts.pipeline_mermaid_option(),
79
80
  pipeBaseOpts.instrument_option(help=instrumentOptionHelp, metavar="instrument", multiple=True),
@@ -124,6 +125,7 @@ class qgraph_options(OptionGroup): # noqa: N801
124
125
  ),
125
126
  ),
126
127
  ctrlMpExecOpts.dataset_query_constraint(),
128
+ ctrlMpExecOpts.data_id_table_option(),
127
129
  ctrlMpExecOpts.qgraph_header_data_option(),
128
130
  ctrlMpExecOpts.mock_option(),
129
131
  ctrlMpExecOpts.mock_failure_option(),
@@ -51,6 +51,18 @@ data_query_option = MWOptionDecorator(
51
51
  )
52
52
 
53
53
 
54
+ data_id_table_option = MWOptionDecorator(
55
+ "--data-id-table",
56
+ multiple=True,
57
+ default=(),
58
+ help=(
59
+ "URI to table of data IDs to join as a constraint; may be any format accepted by astropy.table. "
60
+ "May be passed multiple times."
61
+ ),
62
+ metavar="URI",
63
+ )
64
+
65
+
54
66
  debug_option = MWOptionDecorator(
55
67
  "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True
56
68
  )
@@ -442,6 +454,17 @@ task_option = MWOptionDecorator(
442
454
  multiple=True,
443
455
  )
444
456
 
457
+ select_tasks_option = MWOptionDecorator(
458
+ "--select-tasks",
459
+ metavar="EXPR",
460
+ default="",
461
+ help=(
462
+ "A string expression that filters the tasks to run from the pipeline. "
463
+ "See https://pipelines.lsst.io/v/weekly/modules/lsst.pipe.base/working-with-pipeline-graphs.html"
464
+ "#pipeline-graph-subset-expressions for details."
465
+ ),
466
+ )
467
+
445
468
 
446
469
  timeout_option = MWOptionDecorator(
447
470
  "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)."
@@ -31,10 +31,12 @@ from lsst.daf.butler import Butler
31
31
  from lsst.pipe.base.pipeline_graph import visualization
32
32
 
33
33
  from ... import CmdLineFwk
34
+ from ..._pipeline_graph_factory import PipelineGraphFactory
34
35
  from ..utils import _PipelineAction
35
36
 
36
37
 
37
38
  def build( # type: ignore
39
+ *,
38
40
  order_pipeline,
39
41
  pipeline,
40
42
  pipeline_actions,
@@ -43,8 +45,9 @@ def build( # type: ignore
43
45
  save_pipeline,
44
46
  show,
45
47
  butler_config=None,
48
+ select_tasks="",
46
49
  **kwargs,
47
- ):
50
+ ) -> PipelineGraphFactory:
48
51
  """Implement the command line interface `pipetask build` subcommand.
49
52
 
50
53
  Should only be called by command line tools and unit test code that tests
@@ -79,6 +82,8 @@ def build( # type: ignore
79
82
  `Config`, it is the object used to configure a Butler.
80
83
  Only used to resolve pipeline graphs for --show pipeline-graph and
81
84
  --show task-graph.
85
+ select_tasks : `str`, optional
86
+ String expression that filters the tasks in the pipeline.
82
87
  **kwargs
83
88
  Ignored; click commands may accept options for more than one script
84
89
  function and pass all the option kwargs to each of the script functions
@@ -86,8 +91,9 @@ def build( # type: ignore
86
91
 
87
92
  Returns
88
93
  -------
89
- pipeline : `lsst.pipe.base.Pipeline`
90
- The pipeline instance that was built.
94
+ pipeline_graph_factory : `..PipelineGraphFactory`
95
+ A helper object that holds the built pipeline and can turn it into a
96
+ pipeline graph.
91
97
 
92
98
  Raises
93
99
  ------
@@ -119,10 +125,12 @@ def build( # type: ignore
119
125
  else:
120
126
  butler = None
121
127
 
128
+ pipeline_graph_factory = PipelineGraphFactory(pipeline, butler, select_tasks)
129
+
122
130
  if pipeline_dot:
123
131
  with open(pipeline_dot, "w") as stream:
124
132
  visualization.show_dot(
125
- pipeline.to_graph(butler.registry if butler is not None else None, visualization_only=True),
133
+ pipeline_graph_factory(visualization_only=True),
126
134
  stream,
127
135
  dataset_types=True,
128
136
  task_classes="full",
@@ -142,7 +150,7 @@ def build( # type: ignore
142
150
 
143
151
  with open(pipeline_mermaid, file_mode) as stream:
144
152
  visualization.show_mermaid(
145
- pipeline.to_graph(butler.registry if butler is not None else None, visualization_only=True),
153
+ pipeline_graph_factory(visualization_only=True),
146
154
  stream,
147
155
  output_format=output_format,
148
156
  width=4500 if output_format != "mmd" else None,
@@ -150,6 +158,6 @@ def build( # type: ignore
150
158
  task_classes="full",
151
159
  )
152
160
 
153
- show.show_pipeline_info(pipeline, butler=butler)
161
+ show.show_pipeline_info(pipeline_graph_factory)
154
162
 
155
- return pipeline
163
+ return pipeline_graph_factory
@@ -36,7 +36,8 @@ _log = logging.getLogger(__name__)
36
36
 
37
37
 
38
38
  def qgraph( # type: ignore
39
- pipelineObj,
39
+ pipeline_graph_factory,
40
+ *,
40
41
  qgraph,
41
42
  qgraph_id,
42
43
  qgraph_node_id,
@@ -55,6 +56,7 @@ def qgraph( # type: ignore
55
56
  replace_run,
56
57
  prune_replaced,
57
58
  data_query,
59
+ data_id_table=(),
58
60
  show,
59
61
  save_execution_butler,
60
62
  clobber_execution_butler,
@@ -76,12 +78,13 @@ def qgraph( # type: ignore
76
78
 
77
79
  Parameters
78
80
  ----------
79
- pipelineObj : `lsst.pipe.base.Pipeline` or None
80
- The pipeline object used to generate a qgraph. If this is not `None`
81
- then `qgraph` should be `None`.
81
+ pipeline_graph_factory : `..PipelineGraphFactory` or None
82
+ A factory that holds the pipeline and can produce a pipeline graph.
83
+ If this is not `None` then `qgraph` should be `None`.
82
84
  qgraph : `str` or `None`
83
85
  URI location for a serialized quantum graph definition as a pickle
84
- file. If this option is not None then `pipeline` should be `None`.
86
+ file. If this option is not None then ``pipeline_graph_factory`` should
87
+ be `None`.
85
88
  qgraph_id : `str` or `None`
86
89
  Quantum graph identifier, if specified must match the identifier of the
87
90
  graph loaded from a file. Ignored if graph is not loaded from a file.
@@ -145,6 +148,8 @@ def qgraph( # type: ignore
145
148
  ``replace_run`` to be `True`.
146
149
  data_query : `str`
147
150
  User query selection expression.
151
+ data_id_table : `~collections.abc.Iterable` [`str`]
152
+ Paths to data ID tables to join in.
148
153
  show : `lsst.ctrl.mpexec.showInfo.ShowInfo`
149
154
  Descriptions of what to dump to stdout.
150
155
  save_execution_butler : `str` or `None`
@@ -206,6 +211,7 @@ def qgraph( # type: ignore
206
211
  replace_run=replace_run,
207
212
  prune_replaced=prune_replaced,
208
213
  data_query=data_query,
214
+ data_id_table=data_id_table,
209
215
  skip_existing_in=skip_existing_in,
210
216
  skip_existing=skip_existing,
211
217
  execution_butler_location=save_execution_butler,
@@ -222,7 +228,7 @@ def qgraph( # type: ignore
222
228
  )
223
229
 
224
230
  f = CmdLineFwk()
225
- qgraph = f.makeGraph(pipelineObj, args)
231
+ qgraph = f.makeGraph(pipeline_graph_factory, args)
226
232
 
227
233
  if qgraph is None:
228
234
  return None
@@ -73,10 +73,12 @@ from lsst.pipe.base.all_dimensions_quantum_graph_builder import AllDimensionsQua
73
73
  from lsst.pipe.base.dot_tools import graph2dot
74
74
  from lsst.pipe.base.mermaid_tools import graph2mermaid
75
75
  from lsst.pipe.base.pipeline_graph import NodeType
76
+ from lsst.resources import ResourcePath
76
77
  from lsst.utils import doImportType
77
78
  from lsst.utils.logging import getLogger
78
79
  from lsst.utils.threads import disable_implicit_threading
79
80
 
81
+ from ._pipeline_graph_factory import PipelineGraphFactory
80
82
  from .executionGraphFixup import ExecutionGraphFixup
81
83
  from .mpGraphExecutor import MPGraphExecutor
82
84
  from .preExecInit import PreExecInit, PreExecInitLimited
@@ -615,13 +617,16 @@ class CmdLineFwk:
615
617
 
616
618
  return pipeline
617
619
 
618
- def makeGraph(self, pipeline: Pipeline, args: SimpleNamespace) -> QuantumGraph | None:
620
+ def makeGraph(
621
+ self, pipeline_graph_factory: PipelineGraphFactory | None, args: SimpleNamespace
622
+ ) -> QuantumGraph | None:
619
623
  """Build a graph from command line arguments.
620
624
 
621
625
  Parameters
622
626
  ----------
623
- pipeline : `~lsst.pipe.base.Pipeline`
624
- Pipeline, can be empty or ``None`` if graph is read from a file.
627
+ pipeline_graph_factory : `PipelineGraphFactory`
628
+ Factory that holds a pipeline and can produce a pipeline graph.
629
+ Must be ``None`` if and only if graph is read from a file.
625
630
  args : `types.SimpleNamespace`
626
631
  Parsed command line.
627
632
 
@@ -645,12 +650,20 @@ class CmdLineFwk:
645
650
  qgraph = QuantumGraph.loadUri(args.qgraph, butler.dimensions, nodes=nodes, graphID=args.qgraph_id)
646
651
 
647
652
  # pipeline can not be provided in this case
648
- if pipeline:
649
- raise ValueError("Pipeline must not be given when quantum graph is read from file.")
653
+ if pipeline_graph_factory:
654
+ raise ValueError(
655
+ "Pipeline must not be given when quantum graph is read from "
656
+ f"file: {bool(pipeline_graph_factory)}"
657
+ )
650
658
  if args.show_qgraph_header:
651
659
  print(QuantumGraph.readHeader(args.qgraph))
652
660
  else:
653
- pipeline_graph = pipeline.to_graph()
661
+ if pipeline_graph_factory is None:
662
+ raise ValueError("Pipeline must be given when quantum graph is not read from file.")
663
+ # We can't resolve the pipeline graph if we're mocking until after
664
+ # we've done the mocking (and the QG build will resolve on its own
665
+ # anyway).
666
+ pipeline_graph = pipeline_graph_factory(resolve=False)
654
667
  if args.mock:
655
668
  from lsst.pipe.base.tests.mocks import mock_pipeline_graph
656
669
 
@@ -659,6 +672,14 @@ class CmdLineFwk:
659
672
  unmocked_dataset_types=args.unmocked_dataset_types,
660
673
  force_failures=args.mock_failure,
661
674
  )
675
+ data_id_tables = []
676
+ for table_file in args.data_id_table:
677
+ with ResourcePath(table_file).as_local() as local_path:
678
+ table = Table.read(local_path.ospath)
679
+ # Add the filename to the metadata for more logging
680
+ # information down in the QG builder.
681
+ table.meta["filename"] = table_file
682
+ data_id_tables.append(table)
662
683
  # make execution plan (a.k.a. DAG) for pipeline
663
684
  graph_builder = AllDimensionsQuantumGraphBuilder(
664
685
  pipeline_graph,
@@ -669,6 +690,7 @@ class CmdLineFwk:
669
690
  dataset_query_constraint=args.dataset_query_constraint,
670
691
  input_collections=collections,
671
692
  output_run=run,
693
+ data_id_tables=data_id_tables,
672
694
  )
673
695
  # accumulate metadata
674
696
  metadata = {
@@ -959,6 +981,10 @@ class CmdLineFwk:
959
981
  # but we need datastore records for initInputs, and those are only
960
982
  # available from Quanta, so load the whole thing.
961
983
  qgraph = QuantumGraph.loadUri(args.qgraph, graphID=args.qgraph_id)
984
+
985
+ # Ensure that QBB uses shared datastore cache for writes.
986
+ _ButlerFactory.defineDatastoreCache()
987
+
962
988
  # Make QBB.
963
989
  butler = qgraph.make_init_qbb(args.butler_config, config_search_paths=args.config_search_path)
964
990
  # Save all InitOutputs, configs, etc.
@@ -980,6 +1006,9 @@ class CmdLineFwk:
980
1006
 
981
1007
  dataset_types = {dstype.name: dstype for dstype in qgraph.registryDatasetTypes()}
982
1008
 
1009
+ # Ensure that QBB uses shared datastore cache.
1010
+ _ButlerFactory.defineDatastoreCache()
1011
+
983
1012
  _butler_factory = _QBBFactory(
984
1013
  butler_config=args.butler_config,
985
1014
  dimensions=qgraph.universe,
@@ -39,12 +39,13 @@ from typing import Any
39
39
 
40
40
  import lsst.pex.config as pexConfig
41
41
  import lsst.pex.config.history as pexConfigHistory
42
- from lsst.daf.butler import Butler, DatasetRef, DatasetType, NamedKeyMapping
42
+ from lsst.daf.butler import DatasetRef, DatasetType, NamedKeyMapping
43
43
  from lsst.daf.butler.datastore.record_data import DatastoreRecordData
44
- from lsst.pipe.base import Pipeline, QuantumGraph
44
+ from lsst.pipe.base import PipelineGraph, QuantumGraph
45
45
  from lsst.pipe.base.pipeline_graph import visualization
46
46
 
47
47
  from . import util
48
+ from ._pipeline_graph_factory import PipelineGraphFactory
48
49
  from .cmdLineFwk import _ButlerFactory
49
50
 
50
51
 
@@ -138,20 +139,15 @@ class ShowInfo:
138
139
  """Return the commands that have not yet been processed."""
139
140
  return frozenset(set(self.commands) - self.handled)
140
141
 
141
- def show_pipeline_info(self, pipeline: Pipeline, butler: Butler | None) -> None:
142
+ def show_pipeline_info(self, pipeline_graph_factory: PipelineGraphFactory) -> None:
142
143
  """Display useful information about the pipeline.
143
144
 
144
145
  Parameters
145
146
  ----------
146
- pipeline : `lsst.pipe.base.Pipeline`
147
- The pipeline to use when reporting information.
148
- butler : `~lsst.daf.butler.Butler`
149
- Butler to use for querying.
147
+ pipeline_graph_factory : `PipelineGraphFactory`
148
+ Factory object that holds the pipeline and can produce a pipeline
149
+ graph.
150
150
  """
151
- if butler is not None:
152
- registry = butler.registry
153
- else:
154
- registry = None
155
151
  for command in self.pipeline_commands:
156
152
  if command not in self.commands:
157
153
  continue
@@ -159,25 +155,25 @@ class ShowInfo:
159
155
 
160
156
  match command:
161
157
  case "pipeline":
162
- print(pipeline, file=self.stream)
158
+ print(pipeline_graph_factory.pipeline, file=self.stream)
163
159
  case "config":
164
160
  for arg in args:
165
- self._showConfig(pipeline, arg, False)
161
+ self._showConfig(pipeline_graph_factory(visualization_only=True), arg, False)
166
162
  case "dump-config":
167
163
  for arg in args:
168
- self._showConfig(pipeline, arg, True)
164
+ self._showConfig(pipeline_graph_factory(visualization_only=True), arg, True)
169
165
  case "history":
170
166
  for arg in args:
171
- self._showConfigHistory(pipeline, arg)
167
+ self._showConfigHistory(pipeline_graph_factory(visualization_only=True), arg)
172
168
  case "tasks":
173
- self._showTaskHierarchy(pipeline)
169
+ self._showTaskHierarchy(pipeline_graph_factory(visualization_only=True))
174
170
  case "pipeline-graph":
175
171
  visualization.show(
176
- pipeline.to_graph(registry, visualization_only=True), self.stream, dataset_types=True
172
+ pipeline_graph_factory(visualization_only=True), self.stream, dataset_types=True
177
173
  )
178
174
  case "task-graph":
179
175
  visualization.show(
180
- pipeline.to_graph(registry, visualization_only=True), self.stream, dataset_types=False
176
+ pipeline_graph_factory(visualization_only=True), self.stream, dataset_types=False
181
177
  )
182
178
  case _:
183
179
  raise RuntimeError(f"Unexpectedly tried to process command {command!r}.")
@@ -210,13 +206,13 @@ class ShowInfo:
210
206
  raise RuntimeError(f"Unexpectedly tried to process command {command!r}.")
211
207
  self.handled.add(command)
212
208
 
213
- def _showConfig(self, pipeline: Pipeline, showArgs: str, dumpFullConfig: bool) -> None:
209
+ def _showConfig(self, pipeline_graph: PipelineGraph, showArgs: str, dumpFullConfig: bool) -> None:
214
210
  """Show task configuration
215
211
 
216
212
  Parameters
217
213
  ----------
218
- pipeline : `lsst.pipe.base.Pipeline`
219
- Pipeline definition
214
+ pipeline : `lsst.pipe.base.pipeline_graph.Pipeline`
215
+ Pipeline definition as a graph.
220
216
  showArgs : `str`
221
217
  Defines what to show
222
218
  dumpFullConfig : `bool`
@@ -235,7 +231,7 @@ class ShowInfo:
235
231
  if pattern:
236
232
  stream = _FilteredStream(pattern, stream=stream)
237
233
 
238
- tasks = util.filterTaskNodes(pipeline.to_graph(), taskName)
234
+ tasks = util.filterTaskNodes(pipeline_graph, taskName)
239
235
  if not tasks:
240
236
  raise ValueError(f"Pipeline has no tasks named {taskName}")
241
237
 
@@ -243,13 +239,13 @@ class ShowInfo:
243
239
  print(f"### Configuration for task `{task_node.label}'", file=self.stream)
244
240
  task_node.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig)
245
241
 
246
- def _showConfigHistory(self, pipeline: Pipeline, showArgs: str) -> None:
242
+ def _showConfigHistory(self, pipeline_graph: PipelineGraph, showArgs: str) -> None:
247
243
  """Show history for task configuration.
248
244
 
249
245
  Parameters
250
246
  ----------
251
- pipeline : `lsst.pipe.base.Pipeline`
252
- Pipeline definition
247
+ pipeline_graph : `lsst.pipe.base.pipeline_graph.PipelineGraph`
248
+ Pipeline definition as a graph.
253
249
  showArgs : `str`
254
250
  Defines what to show
255
251
  """
@@ -262,7 +258,7 @@ class ShowInfo:
262
258
  if not pattern:
263
259
  raise ValueError("Please provide a value with --show history (e.g. history=Task::param)")
264
260
 
265
- tasks = util.filterTaskNodes(pipeline.to_graph(), taskName)
261
+ tasks = util.filterTaskNodes(pipeline_graph, taskName)
266
262
  if not tasks:
267
263
  raise ValueError(f"Pipeline has no tasks named {taskName}")
268
264
 
@@ -300,15 +296,15 @@ class ShowInfo:
300
296
  if not found:
301
297
  raise ValueError(f"None of the tasks has field matching {pattern}")
302
298
 
303
- def _showTaskHierarchy(self, pipeline: Pipeline) -> None:
299
+ def _showTaskHierarchy(self, pipeline_graph: PipelineGraph) -> None:
304
300
  """Print task hierarchy to stdout
305
301
 
306
302
  Parameters
307
303
  ----------
308
- pipeline : `lsst.pipe.base.Pipeline`
309
- Pipeline definition.
304
+ pipeline_graph : `lsst.pipe.base.pipeline_graph.PipelineGraph`
305
+ Pipeline definition as a graph.
310
306
  """
311
- for task_node in pipeline.to_graph().tasks.values():
307
+ for task_node in pipeline_graph.tasks.values():
312
308
  print(f"### Subtasks for task `{task_node.task_class_name}'", file=self.stream)
313
309
 
314
310
  for configName, taskName in util.subTaskIter(task_node.config):
@@ -1,2 +1,2 @@
1
1
  __all__ = ["__version__"]
2
- __version__ = "29.2025.1500"
2
+ __version__ = "29.2025.1600"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lsst-ctrl-mpexec
3
- Version: 29.2025.1500
3
+ Version: 29.2025.1600
4
4
  Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
5
5
  Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
6
6
  License: BSD 3-Clause License
@@ -1,7 +1,8 @@
1
1
  lsst/__init__.py,sha256=aXdEOZVrBQISQi6XPS9s1NhBjIJaIwNNxCFRiGchRAw,1369
2
2
  lsst/ctrl/__init__.py,sha256=aXdEOZVrBQISQi6XPS9s1NhBjIJaIwNNxCFRiGchRAw,1369
3
- lsst/ctrl/mpexec/__init__.py,sha256=34loiOJc5I_65Npo6zsO1Bf8dB5zV1y1MQid6W65cV8,1699
4
- lsst/ctrl/mpexec/cmdLineFwk.py,sha256=4h3qqzUjYRuI8oJehjwa5_ytrxhf0JlBLdDeQYOTGx4,41743
3
+ lsst/ctrl/mpexec/__init__.py,sha256=c2mK9--wemp2AEa81CSMkrqaIjJ1_iOs_M-O8CHJxB4,1757
4
+ lsst/ctrl/mpexec/_pipeline_graph_factory.py,sha256=suzWUn9YGn0CTA_3N1Wd-sUo7TFxuo_6VZ2nO0CJ5a8,3552
5
+ lsst/ctrl/mpexec/cmdLineFwk.py,sha256=L8CaSTLZDe9cz4KGSlelMGDjpTKXxC4DTBEw6u9YPS8,43132
5
6
  lsst/ctrl/mpexec/dotTools.py,sha256=nx31DC7NkQM766sC97lPQMI22snVaWS7USFMFVLChqk,3421
6
7
  lsst/ctrl/mpexec/execFixupDataId.py,sha256=MUirmuoQiWj7C7wQ23QGkgMqx_5_CP4_UMV6h_gVfxk,5056
7
8
  lsst/ctrl/mpexec/executionGraphFixup.py,sha256=1z4P4YmtLTMPoMxmFiOnzljLcP9B5V1SCG9haqR_TyI,2830
@@ -12,39 +13,39 @@ lsst/ctrl/mpexec/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
12
13
  lsst/ctrl/mpexec/quantumGraphExecutor.py,sha256=DV0j_CNjS8CNMR-kiVIWj1vyzLKZzi4wilc_r0z2L0Q,4386
13
14
  lsst/ctrl/mpexec/reports.py,sha256=zxZOkJWhTFdl4eJTOYsM-58tqRbbyF5-cQ-zF-zIR9Y,7677
14
15
  lsst/ctrl/mpexec/separablePipelineExecutor.py,sha256=6QvZ9EFHoqXwYjHbcNe9wWMssFbAg2I9mNpjFnTSiM0,12074
15
- lsst/ctrl/mpexec/showInfo.py,sha256=3GmRcerbKg3zXpEGiIWKY8z2mXoxZ3w6Qs1DCX73slE,16207
16
+ lsst/ctrl/mpexec/showInfo.py,sha256=elrWQ8QPusZguLqYhvmMjYaNmoJgWpQDr3uQ0QhB1HI,16407
16
17
  lsst/ctrl/mpexec/simple_pipeline_executor.py,sha256=scqQ1MDS6FcZvFTwxtCkmTSg0zJmKIfah91lawr7Dhc,20454
17
18
  lsst/ctrl/mpexec/singleQuantumExecutor.py,sha256=mlv3nF29N1meiHie3r072Ynz5k6XTpFO8jfhq7BufF8,28036
18
19
  lsst/ctrl/mpexec/taskFactory.py,sha256=c4xj8cR_Ts5uzzGovh87ZKdVeeXy5E3lIjCwJnDuOqg,2720
19
20
  lsst/ctrl/mpexec/util.py,sha256=y2Rw5PL40_EuLtVxiqSVX0JfPV4IrFl1LfOMUWx2u30,4236
20
- lsst/ctrl/mpexec/version.py,sha256=altncYankWkwG1F2OVHhPl0np72lUhdVRAOSqBtd-kc,55
21
+ lsst/ctrl/mpexec/version.py,sha256=DWDCyavYRMZByGPGMYcnpz6piNv1sQvCKH8yohX_F3c,55
21
22
  lsst/ctrl/mpexec/cli/__init__.py,sha256=6dpDHNBzyicVpFi1fsaiYVbYEMeoL57IHKkPaej24gs,1301
22
23
  lsst/ctrl/mpexec/cli/pipetask.py,sha256=4HnhX9dCizCihVbpHVJX5WXO9TEli9oL6wA-tPh1_vA,2209
23
24
  lsst/ctrl/mpexec/cli/utils.py,sha256=5iOrlj5jJTWtZS0BMLsuiCGAvxbfrjd1MSyXxBthWcc,6503
24
25
  lsst/ctrl/mpexec/cli/cmd/__init__.py,sha256=nRmwwW5d55gAEkyE7NpSK8mxa56HcfEta2r-Y9I07F8,1661
25
- lsst/ctrl/mpexec/cli/cmd/commands.py,sha256=epqrQdu5jQkqs7hesEK_25pSmZc4NaKNkff-qJhid0s,17779
26
+ lsst/ctrl/mpexec/cli/cmd/commands.py,sha256=Wevv7Wu3hgLY2tNA4RJdMpOFur9rbH5JtwgtNLPnqiY,17811
26
27
  lsst/ctrl/mpexec/cli/opt/__init__.py,sha256=IzUInuJj9igiaNcEqMx0adelgJtQC5_XMYnaiizBn0A,1378
27
28
  lsst/ctrl/mpexec/cli/opt/arguments.py,sha256=vjUw0ZN_4HStp-_3ne6AT5S_eH7sly3OVfL07tgrJnY,1572
28
- lsst/ctrl/mpexec/cli/opt/optionGroups.py,sha256=z1vrRsNlk11cGulbD9qsBOuPhZgryPrka_IxIeVW9TM,8031
29
- lsst/ctrl/mpexec/cli/opt/options.py,sha256=TyTYxDrFqhUqfnLPfjr_2kJLPNlBbQ86nX2SuOlqAts,19911
29
+ lsst/ctrl/mpexec/cli/opt/optionGroups.py,sha256=qtpwlgfvEzAs0CNav7t9ylywWBavoeWlZfYhRQEuZTw,8132
30
+ lsst/ctrl/mpexec/cli/opt/options.py,sha256=8gu-QYCblBO_RlPGYFw__teRWxtRCyasGKz7wNhv_dc,20563
30
31
  lsst/ctrl/mpexec/cli/script/__init__.py,sha256=eCuF4FAI5D3pl05IMJj7TCkZq-hireua2mA5Ui-mKSI,1624
31
- lsst/ctrl/mpexec/cli/script/build.py,sha256=fTa4veqSfUR5nONedmS14q4N5Y_E3nDP1Q-chJvC5IA,5860
32
+ lsst/ctrl/mpexec/cli/script/build.py,sha256=Ff_8ffBmxuJi45x7-rUb-f0axucagYs6rNvBowwVmq0,6149
32
33
  lsst/ctrl/mpexec/cli/script/cleanup.py,sha256=D7W-azf4mNJcIWhbU5uCRCi94mkb8-Q2ksRFblQGrUw,4990
33
34
  lsst/ctrl/mpexec/cli/script/confirmable.py,sha256=Bo1GSTZQn44d_TRj6N3YfpYcZiuHEYoz26WZQwMyb4A,3918
34
35
  lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py,sha256=DGNseiavrI1VxR_o6vCelbPvr9kCQZzroZHHSQe-RTE,2479
35
36
  lsst/ctrl/mpexec/cli/script/purge.py,sha256=gYwSsZfTBP6oDcDp_YdqQEKGvAStvsj5hwNw42S8ptE,10637
36
- lsst/ctrl/mpexec/cli/script/qgraph.py,sha256=p5z84bmRBbZXIEql76nXYnRQcLau-VNTUVkfOhdqjfU,9805
37
+ lsst/ctrl/mpexec/cli/script/qgraph.py,sha256=ncjbtJ7AMp-Obo524HAC2E4Hnqyk1Aji4mJPAZUGMDM,10048
37
38
  lsst/ctrl/mpexec/cli/script/report.py,sha256=ItJitmYmWIDjj7PxRtP4qXLx-z5FAU6nSfI2gx0GS5k,12800
38
39
  lsst/ctrl/mpexec/cli/script/run.py,sha256=22g7vBhGDGsFSQq7CtruTf-bivo3t53decBAV0wAg1g,9767
39
40
  lsst/ctrl/mpexec/cli/script/run_qbb.py,sha256=oCEydg13zNAtci8XB0IwXx_D5PPsHbjCPATWoBlhcPk,5294
40
41
  lsst/ctrl/mpexec/cli/script/update_graph_run.py,sha256=v_EdOaD6jR_vSlgm_5-pwUjoNEFMrAuYFM1xIaHVU3Q,2597
41
- lsst_ctrl_mpexec-29.2025.1500.dist-info/licenses/COPYRIGHT,sha256=pGCjnRAnyt02a6_9PLzXQikpvYmvMmK9fCdOKlRSV6k,369
42
- lsst_ctrl_mpexec-29.2025.1500.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
43
- lsst_ctrl_mpexec-29.2025.1500.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
44
- lsst_ctrl_mpexec-29.2025.1500.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
45
- lsst_ctrl_mpexec-29.2025.1500.dist-info/METADATA,sha256=WCtYWWt_yc8YespjVTklDZ2H1ES1YFsrlMngPoli6qQ,2302
46
- lsst_ctrl_mpexec-29.2025.1500.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
47
- lsst_ctrl_mpexec-29.2025.1500.dist-info/entry_points.txt,sha256=aYE38yqZU8qvpLUUkXzgmUxDJYYknEqPxgxYkowrL4s,64
48
- lsst_ctrl_mpexec-29.2025.1500.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
49
- lsst_ctrl_mpexec-29.2025.1500.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
50
- lsst_ctrl_mpexec-29.2025.1500.dist-info/RECORD,,
42
+ lsst_ctrl_mpexec-29.2025.1600.dist-info/licenses/COPYRIGHT,sha256=pGCjnRAnyt02a6_9PLzXQikpvYmvMmK9fCdOKlRSV6k,369
43
+ lsst_ctrl_mpexec-29.2025.1600.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
44
+ lsst_ctrl_mpexec-29.2025.1600.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
45
+ lsst_ctrl_mpexec-29.2025.1600.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
46
+ lsst_ctrl_mpexec-29.2025.1600.dist-info/METADATA,sha256=4w8JjdeOxf8HVCuGdbBxHxYx0DTb-RiKmQZ9rqkLLlk,2302
47
+ lsst_ctrl_mpexec-29.2025.1600.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
48
+ lsst_ctrl_mpexec-29.2025.1600.dist-info/entry_points.txt,sha256=aYE38yqZU8qvpLUUkXzgmUxDJYYknEqPxgxYkowrL4s,64
49
+ lsst_ctrl_mpexec-29.2025.1600.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
50
+ lsst_ctrl_mpexec-29.2025.1600.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
51
+ lsst_ctrl_mpexec-29.2025.1600.dist-info/RECORD,,