lsst-ctrl-mpexec 29.2025.3300__tar.gz → 29.2025.3500__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {lsst_ctrl_mpexec-29.2025.3300/python/lsst_ctrl_mpexec.egg-info → lsst_ctrl_mpexec-29.2025.3500}/PKG-INFO +1 -1
  2. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/__init__.py +0 -1
  3. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/butler_factory.py +253 -95
  4. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/cmd/commands.py +2 -2
  5. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py +0 -1
  6. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/opt/options.py +0 -7
  7. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py +25 -12
  8. lsst_ctrl_mpexec-29.2025.3500/python/lsst/ctrl/mpexec/cli/script/qgraph.py +302 -0
  9. lsst_ctrl_mpexec-29.2025.3500/python/lsst/ctrl/mpexec/cli/script/run.py +352 -0
  10. lsst_ctrl_mpexec-29.2025.3500/python/lsst/ctrl/mpexec/cli/script/run_qbb.py +279 -0
  11. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/utils.py +49 -0
  12. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/showInfo.py +17 -15
  13. lsst_ctrl_mpexec-29.2025.3500/python/lsst/ctrl/mpexec/version.py +2 -0
  14. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500/python/lsst_ctrl_mpexec.egg-info}/PKG-INFO +1 -1
  15. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/SOURCES.txt +0 -1
  16. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cmdLineFwk.py +65 -102
  17. lsst_ctrl_mpexec-29.2025.3300/python/lsst/ctrl/mpexec/cli/script/qgraph.py +0 -214
  18. lsst_ctrl_mpexec-29.2025.3300/python/lsst/ctrl/mpexec/cli/script/run.py +0 -240
  19. lsst_ctrl_mpexec-29.2025.3300/python/lsst/ctrl/mpexec/cli/script/run_qbb.py +0 -144
  20. lsst_ctrl_mpexec-29.2025.3300/python/lsst/ctrl/mpexec/cmdLineFwk.py +0 -534
  21. lsst_ctrl_mpexec-29.2025.3300/python/lsst/ctrl/mpexec/version.py +0 -2
  22. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/COPYRIGHT +0 -0
  23. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/LICENSE +0 -0
  24. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/MANIFEST.in +0 -0
  25. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/README.rst +0 -0
  26. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/bsd_license.txt +0 -0
  27. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/doc/lsst.ctrl.mpexec/CHANGES.rst +0 -0
  28. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/doc/lsst.ctrl.mpexec/configuring-pipetask-tasks.rst +0 -0
  29. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/doc/lsst.ctrl.mpexec/index.rst +0 -0
  30. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/doc/lsst.ctrl.mpexec/pipetask.rst +0 -0
  31. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/gpl-v3.0.txt +0 -0
  32. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/pyproject.toml +0 -0
  33. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/__init__.py +0 -0
  34. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/__init__.py +0 -0
  35. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/_pipeline_graph_factory.py +0 -0
  36. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/__init__.py +0 -0
  37. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/cmd/__init__.py +0 -0
  38. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/opt/__init__.py +0 -0
  39. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/opt/arguments.py +0 -0
  40. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/pipetask.py +0 -0
  41. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/__init__.py +0 -0
  42. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/build.py +0 -0
  43. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/cleanup.py +0 -0
  44. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/confirmable.py +0 -0
  45. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/purge.py +0 -0
  46. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/report.py +0 -0
  47. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/update_graph_run.py +0 -0
  48. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/execFixupDataId.py +0 -0
  49. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/executionGraphFixup.py +0 -0
  50. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/log_capture.py +0 -0
  51. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/mpGraphExecutor.py +0 -0
  52. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/preExecInit.py +0 -0
  53. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/py.typed +0 -0
  54. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/quantumGraphExecutor.py +0 -0
  55. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/reports.py +0 -0
  56. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/separablePipelineExecutor.py +0 -0
  57. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/simple_pipeline_executor.py +0 -0
  58. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/singleQuantumExecutor.py +0 -0
  59. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/taskFactory.py +0 -0
  60. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/util.py +0 -0
  61. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/dependency_links.txt +0 -0
  62. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/entry_points.txt +0 -0
  63. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/requires.txt +0 -0
  64. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/top_level.txt +0 -0
  65. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/zip-safe +0 -0
  66. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/setup.cfg +0 -0
  67. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliCmdCleanup.py +0 -0
  68. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliCmdPurge.py +0 -0
  69. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliCmdQgraph.py +0 -0
  70. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliCmdReport.py +0 -0
  71. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliCmdUpdateGraphRun.py +0 -0
  72. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliScript.py +0 -0
  73. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliUtils.py +0 -0
  74. {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_preExecInit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lsst-ctrl-mpexec
3
- Version: 29.2025.3300
3
+ Version: 29.2025.3500
4
4
  Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
5
5
  Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
6
6
  License: BSD 3-Clause License
@@ -26,7 +26,6 @@
26
26
  # along with this program. If not, see <https://www.gnu.org/licenses/>.
27
27
 
28
28
  from ._pipeline_graph_factory import PipelineGraphFactory
29
- from .cmdLineFwk import *
30
29
  from .executionGraphFixup import *
31
30
  from .mpGraphExecutor import *
32
31
  from .quantumGraphExecutor import *
@@ -35,8 +35,7 @@ __all__ = [
35
35
 
36
36
  import atexit
37
37
  import shutil
38
- from collections.abc import Sequence
39
- from types import SimpleNamespace
38
+ from collections.abc import Iterable, Sequence
40
39
 
41
40
  from lsst.daf.butler import Butler, CollectionType
42
41
  from lsst.daf.butler.datastore.cache_manager import DatastoreCacheManager
@@ -44,6 +43,7 @@ from lsst.daf.butler.registry import MissingCollectionError, RegistryDefaults
44
43
  from lsst.daf.butler.registry.wildcards import CollectionWildcard
45
44
  from lsst.pipe.base import Instrument, PipelineGraph
46
45
  from lsst.pipe.base.pipeline_graph import NodeType
46
+ from lsst.resources import ResourcePathExpression
47
47
  from lsst.utils.logging import getLogger
48
48
 
49
49
  _LOG = getLogger(__name__)
@@ -127,46 +127,23 @@ class ButlerFactory:
127
127
  ----------
128
128
  butler : `lsst.daf.butler.Butler`
129
129
  Butler that collections will be added to and/or queried from.
130
-
131
- args : `types.SimpleNamespace`
132
- Parsed command-line arguments. The following attributes are used,
133
- either at construction or in later methods.
134
-
135
- ``output``
136
- The name of a `~lsst.daf.butler.CollectionType.CHAINED`
137
- input/output collection.
138
-
139
- ``output_run``
140
- The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
141
- collection.
142
-
143
- ``extend_run``
144
- A boolean indicating whether ``output_run`` should already exist
145
- and be extended.
146
-
147
- ``replace_run``
148
- A boolean indicating that (if `True`) ``output_run`` should already
149
- exist but will be removed from the output chained collection and
150
- replaced with a new one.
151
-
152
- ``prune_replaced``
153
- A boolean indicating whether to prune the replaced run (requires
154
- ``replace_run``).
155
-
156
- ``rebase``
157
- A boolean indicating whether to force the ``output`` collection
158
- to be consistent with ``inputs`` and ``output`` run such that the
159
- ``output`` collection has output run collections first (i.e. those
160
- that start with the same prefix), then the new inputs, then any
161
- original inputs not included in the new inputs.
162
-
163
- ``inputs``
164
- Input collections of any type; see
165
- :ref:`daf_butler_ordered_collection_searches` for details.
166
-
167
- ``butler_config``
168
- Path to a data repository root or configuration file.
169
-
130
+ output : `str` or `None`
131
+ The name of a `~lsst.daf.butler.CollectionType.CHAINED` input/output
132
+ collection.
133
+ output_run : `str` or `None`
134
+ The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
135
+ collection.
136
+ inputs : `str` or `~collections.abc.Iterable` [`str`]
137
+ Input collection name or iterable of collection names.
138
+ extend_run : `bool`
139
+ A boolean indicating whether ``output_run`` should already exist and be
140
+ extended.
141
+ rebase : `bool`
142
+ A boolean indicating whether to force the ``output`` collection to be
143
+ consistent with ``inputs`` and ``output`` run such that the ``output``
144
+ collection has output run collections first (i.e. those that start with
145
+ the same prefix), then the new inputs, then any original inputs not
146
+ included in the new inputs.
170
147
  writeable : `bool`
171
148
  If `True`, a `~lsst.daf.butler.Butler` is being initialized in a
172
149
  context where actual writes should happens, and hence no output run
@@ -178,17 +155,27 @@ class ButlerFactory:
178
155
  Raised if ``writeable is True`` but there are no output collections.
179
156
  """
180
157
 
181
- def __init__(self, butler: Butler, args: SimpleNamespace, writeable: bool):
182
- if args.output is not None:
183
- self.output = OutputChainedCollectionInfo(butler, args.output)
158
+ def __init__(
159
+ self,
160
+ butler: Butler,
161
+ *,
162
+ output: str | None,
163
+ output_run: str | None,
164
+ inputs: str | Iterable[str],
165
+ extend_run: bool = False,
166
+ rebase: bool = False,
167
+ writeable: bool,
168
+ ):
169
+ if output is not None:
170
+ self.output = OutputChainedCollectionInfo(butler, output)
184
171
  else:
185
172
  self.output = None
186
- if args.output_run is not None:
187
- if args.rebase and self.output and not args.output_run.startswith(self.output.name):
173
+ if output_run is not None:
174
+ if rebase and self.output and not output_run.startswith(self.output.name):
188
175
  raise ValueError("Cannot rebase if output run does not start with output collection name.")
189
- self.output_run = OutputRunCollectionInfo(butler, args.output_run)
176
+ self.output_run = OutputRunCollectionInfo(butler, output_run)
190
177
  elif self.output is not None:
191
- if args.extend_run:
178
+ if extend_run:
192
179
  if not self.output.chain:
193
180
  raise ValueError("Cannot use --extend-run option with non-existing or empty output chain")
194
181
  run_name = self.output.chain[0]
@@ -204,45 +191,50 @@ class ButlerFactory:
204
191
  # front so we can tell if the user passes the same inputs on subsequent
205
192
  # calls, even though we also flatten when we define the output CHAINED
206
193
  # collection.
207
- self.inputs = tuple(butler.collections.query(args.input, flatten_chains=True)) if args.input else ()
194
+ self.inputs = tuple(butler.collections.query(inputs, flatten_chains=True)) if inputs else ()
208
195
 
209
196
  # If things are inconsistent and user has asked for a rebase then
210
197
  # construct the new output chain.
211
- if args.rebase and self._check_output_input_consistency():
198
+ if rebase and self._check_output_input_consistency():
212
199
  assert self.output is not None
213
200
  newOutputChain = [item for item in self.output.chain if item.startswith(self.output.name)]
214
201
  newOutputChain.extend([item for item in self.inputs if item not in newOutputChain])
215
202
  newOutputChain.extend([item for item in self.output.chain if item not in newOutputChain])
216
203
  self.output.chain = tuple(newOutputChain)
217
204
 
218
- def check(self, args: SimpleNamespace) -> None:
205
+ def check(self, *, extend_run: bool, replace_run: bool, prune_replaced: str | None = None) -> None:
219
206
  """Check command-line options for consistency with each other and the
220
207
  data repository.
221
208
 
222
209
  Parameters
223
210
  ----------
224
- args : `types.SimpleNamespace`
225
- Parsed command-line arguments. See class documentation for the
226
- construction parameter of the same name.
211
+ extend_run : `bool`
212
+ Whether the ``output_run`` should already exist and be extended.
213
+ replace_run : `bool`
214
+ Whether the ``output_run`` should be replaced in the ``output``
215
+ chain.
216
+ prune_replaced : `str` or `None`
217
+ If ``replace_run=True``, whether/how datasets in the old run should
218
+ be removed. Options are ``"purge"``, ``"unstore"``, and `None`.
227
219
  """
228
- assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser."
220
+ assert not (extend_run and replace_run), "In mutually-exclusive group in ArgumentParser."
229
221
  if consistencyError := self._check_output_input_consistency():
230
222
  raise ValueError(consistencyError)
231
223
 
232
- if args.extend_run:
224
+ if extend_run:
233
225
  if self.output_run is None:
234
226
  raise ValueError("Cannot --extend-run when no output collection is given.")
235
227
  elif not self.output_run.exists:
236
228
  raise ValueError(
237
229
  f"Cannot --extend-run; output collection '{self.output_run.name}' does not exist."
238
230
  )
239
- if not args.extend_run and self.output_run is not None and self.output_run.exists:
231
+ if not extend_run and self.output_run is not None and self.output_run.exists:
240
232
  raise ValueError(
241
233
  f"Output run '{self.output_run.name}' already exists, but --extend-run was not given."
242
234
  )
243
- if args.prune_replaced and not args.replace_run:
235
+ if prune_replaced and not replace_run:
244
236
  raise ValueError("--prune-replaced requires --replace-run.")
245
- if args.replace_run and (self.output is None or not self.output.exists):
237
+ if replace_run and (self.output is None or not self.output.exists):
246
238
  raise ValueError("--output must point to an existing CHAINED collection for --replace-run.")
247
239
 
248
240
  def _check_output_input_consistency(self) -> str | None:
@@ -264,15 +256,48 @@ class ButlerFactory:
264
256
  return None
265
257
 
266
258
  @classmethod
267
- def _make_read_parts(cls, args: SimpleNamespace) -> tuple[Butler, Sequence[str], ButlerFactory]:
259
+ def _make_read_parts(
260
+ cls,
261
+ butler_config: ResourcePathExpression,
262
+ *,
263
+ output: str | None,
264
+ output_run: str | None,
265
+ inputs: str | Iterable[str],
266
+ extend_run: bool = False,
267
+ rebase: bool = False,
268
+ replace_run: bool,
269
+ prune_replaced: str | None = None,
270
+ ) -> tuple[Butler, Sequence[str], ButlerFactory]:
268
271
  """Parse arguments to support implementations of `make_read_butler` and
269
272
  `make_butler_and_collections`.
270
273
 
271
274
  Parameters
272
275
  ----------
273
- args : `types.SimpleNamespace`
274
- Parsed command-line arguments. See class documentation for the
275
- construction parameter of the same name.
276
+ butler_config : convertible to `lsst.resources.ResourcePath`
277
+ Path to configuration for the butler.
278
+ output : `str` or `None`
279
+ The name of a `~lsst.daf.butler.CollectionType.CHAINED`
280
+ input/output collection.
281
+ output_run : `str` or `None`
282
+ The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
283
+ collection.
284
+ inputs : `str` or `~collections.abc.Iterable` [`str`]
285
+ Input collection name or iterable of collection names.
286
+ extend_run : `bool`
287
+ A boolean indicating whether ``output_run`` should already exist
288
+ and be extended.
289
+ rebase : `bool`
290
+ A boolean indicating whether to force the ``output`` collection to
291
+ be consistent with ``inputs`` and ``output`` run such that the
292
+ ``output`` collection has output run collections first (i.e. those
293
+ that start with the same prefix), then the new inputs, then any
294
+ original inputs not included in the new inputs.
295
+ replace_run : `bool`
296
+ Whether the ``output_run`` should be replaced in the ``output``
297
+ chain.
298
+ prune_replaced : `str` or `None`
299
+ If ``replace_run=True``, whether/how datasets in the old run should
300
+ be removed. Options are ``"purge"``, ``"unstore"``, and `None`.
276
301
 
277
302
  Returns
278
303
  -------
@@ -285,11 +310,19 @@ class ButlerFactory:
285
310
  A new `ButlerFactory` instance representing the processed version
286
311
  of ``args``.
287
312
  """
288
- butler = Butler.from_config(args.butler_config, writeable=False)
289
- self = cls(butler, args, writeable=False)
290
- self.check(args)
313
+ butler = Butler.from_config(butler_config, writeable=False)
314
+ self = cls(
315
+ butler,
316
+ output=output,
317
+ output_run=output_run,
318
+ inputs=inputs,
319
+ extend_run=extend_run,
320
+ rebase=rebase,
321
+ writeable=False,
322
+ )
323
+ self.check(extend_run=extend_run, replace_run=replace_run, prune_replaced=prune_replaced)
291
324
  if self.output and self.output.exists:
292
- if args.replace_run:
325
+ if replace_run:
293
326
  replaced = self.output.chain[0]
294
327
  inputs = list(self.output.chain[1:])
295
328
  _LOG.debug(
@@ -299,44 +332,118 @@ class ButlerFactory:
299
332
  inputs = [self.output.name]
300
333
  else:
301
334
  inputs = list(self.inputs)
302
- if args.extend_run:
335
+ if extend_run:
303
336
  assert self.output_run is not None, "Output collection has to be specified."
304
337
  inputs.insert(0, self.output_run.name)
305
338
  collSearch = CollectionWildcard.from_expression(inputs).require_ordered()
306
339
  return butler, collSearch, self
307
340
 
308
341
  @classmethod
309
- def make_read_butler(cls, args: SimpleNamespace) -> Butler:
342
+ def make_read_butler(
343
+ cls,
344
+ butler_config: ResourcePathExpression,
345
+ *,
346
+ output: str | None,
347
+ output_run: str | None,
348
+ inputs: str | Iterable[str],
349
+ extend_run: bool = False,
350
+ rebase: bool = False,
351
+ replace_run: bool,
352
+ prune_replaced: str | None = None,
353
+ ) -> Butler:
310
354
  """Construct a read-only butler according to the given command-line
311
355
  arguments.
312
356
 
313
357
  Parameters
314
358
  ----------
315
- args : `types.SimpleNamespace`
316
- Parsed command-line arguments. See class documentation for the
317
- construction parameter of the same name.
359
+ butler_config : convertible to `lsst.resources.ResourcePath`
360
+ Path to configuration for the butler.
361
+ output : `str` or `None`
362
+ The name of a `~lsst.daf.butler.CollectionType.CHAINED`
363
+ input/output collection.
364
+ output_run : `str` or `None`
365
+ The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
366
+ collection.
367
+ inputs : `str` or `~collections.abc.Iterable` [`str`]
368
+ Input collection name or iterable of collection names.
369
+ extend_run : `bool`
370
+ A boolean indicating whether ``output_run`` should already exist
371
+ and be extended.
372
+ rebase : `bool`
373
+ A boolean indicating whether to force the ``output`` collection to
374
+ be consistent with ``inputs`` and ``output`` run such that the
375
+ ``output`` collection has output run collections first (i.e. those
376
+ that start with the same prefix), then the new inputs, then any
377
+ original inputs not included in the new inputs.
378
+ replace_run : `bool`
379
+ Whether the ``output_run`` should be replaced in the ``output``
380
+ chain.
381
+ prune_replaced : `str` or `None`
382
+ If ``replace_run=True``, whether/how datasets in the old run should
383
+ be removed. Options are ``"purge"``, ``"unstore"``, and `None`.
318
384
 
319
385
  Returns
320
386
  -------
321
387
  butler : `lsst.daf.butler.Butler`
322
- A read-only butler initialized with the collections specified by
323
- ``args``.
388
+ A read-only butler initialized with the given collections.
324
389
  """
325
390
  cls.define_datastore_cache() # Ensure that this butler can use a shared cache.
326
- butler, inputs, _ = cls._make_read_parts(args)
391
+ butler, inputs, _ = cls._make_read_parts(
392
+ butler_config,
393
+ output=output,
394
+ output_run=output_run,
395
+ inputs=inputs,
396
+ extend_run=extend_run,
397
+ rebase=rebase,
398
+ replace_run=replace_run,
399
+ prune_replaced=prune_replaced,
400
+ )
327
401
  _LOG.debug("Preparing butler to read from %s.", inputs)
328
402
  return Butler.from_config(butler=butler, collections=inputs)
329
403
 
330
404
  @classmethod
331
- def make_butler_and_collections(cls, args: SimpleNamespace) -> tuple[Butler, Sequence[str], str | None]:
405
+ def make_butler_and_collections(
406
+ cls,
407
+ butler_config: ResourcePathExpression,
408
+ *,
409
+ output: str | None,
410
+ output_run: str | None,
411
+ inputs: str | Iterable[str],
412
+ extend_run: bool = False,
413
+ rebase: bool = False,
414
+ replace_run: bool,
415
+ prune_replaced: str | None = None,
416
+ ) -> tuple[Butler, Sequence[str], str | None]:
332
417
  """Return a read-only butler, a collection search path, and the name
333
418
  of the run to be used for future writes.
334
419
 
335
420
  Parameters
336
421
  ----------
337
- args : `types.SimpleNamespace`
338
- Parsed command-line arguments. See class documentation for the
339
- construction parameter of the same name.
422
+ butler_config : convertible to `lsst.resources.ResourcePath`
423
+ Path to configuration for the butler.
424
+ output : `str` or `None`
425
+ The name of a `~lsst.daf.butler.CollectionType.CHAINED`
426
+ input/output collection.
427
+ output_run : `str` or `None`
428
+ The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
429
+ collection.
430
+ inputs : `str` or `~collections.abc.Iterable` [`str`]
431
+ Input collection name or iterable of collection names.
432
+ extend_run : `bool`
433
+ A boolean indicating whether ``output_run`` should already exist
434
+ and be extended.
435
+ rebase : `bool`
436
+ A boolean indicating whether to force the ``output`` collection to
437
+ be consistent with ``inputs`` and ``output`` run such that the
438
+ ``output`` collection has output run collections first (i.e. those
439
+ that start with the same prefix), then the new inputs, then any
440
+ original inputs not included in the new inputs.
441
+ replace_run : `bool`
442
+ Whether the ``output_run`` should be replaced in the ``output``
443
+ chain.
444
+ prune_replaced : `str` or `None`
445
+ If ``replace_run=True``, whether/how datasets in the old run should
446
+ be removed. Options are ``"purge"``, ``"unstore"``, and `None`.
340
447
 
341
448
  Returns
342
449
  -------
@@ -349,9 +456,18 @@ class ButlerFactory:
349
456
  Name of the output `~lsst.daf.butler.CollectionType.RUN` collection
350
457
  if it already exists, or `None` if it does not.
351
458
  """
352
- butler, inputs, self = cls._make_read_parts(args)
459
+ butler, inputs, self = cls._make_read_parts(
460
+ butler_config,
461
+ output=output,
462
+ output_run=output_run,
463
+ inputs=inputs,
464
+ extend_run=extend_run,
465
+ rebase=rebase,
466
+ replace_run=replace_run,
467
+ prune_replaced=prune_replaced,
468
+ )
353
469
  run: str | None = None
354
- if args.extend_run:
470
+ if extend_run:
355
471
  assert self.output_run is not None, "Output collection has to be specified."
356
472
  if self.output_run is not None:
357
473
  run = self.output_run.name
@@ -374,17 +490,51 @@ class ButlerFactory:
374
490
  _LOG.debug("Defining shared datastore cache directory to %s", cache_dir)
375
491
 
376
492
  @classmethod
377
- def make_write_butler(cls, args: SimpleNamespace, pipeline_graph: PipelineGraph) -> Butler:
493
+ def make_write_butler(
494
+ cls,
495
+ butler_config: ResourcePathExpression,
496
+ pipeline_graph: PipelineGraph,
497
+ *,
498
+ output: str | None,
499
+ output_run: str | None,
500
+ inputs: str | Iterable[str],
501
+ extend_run: bool = False,
502
+ rebase: bool = False,
503
+ replace_run: bool,
504
+ prune_replaced: str | None = None,
505
+ ) -> Butler:
378
506
  """Return a read-write butler initialized to write to and read from
379
507
  the collections specified by the given command-line arguments.
380
508
 
381
509
  Parameters
382
510
  ----------
383
- args : `types.SimpleNamespace`
384
- Parsed command-line arguments. See class documentation for the
385
- construction parameter of the same name.
511
+ butler_config : convertible to `lsst.resources.ResourcePath`
512
+ Path to configuration for the butler.
386
513
  pipeline_graph : `lsst.pipe.base.PipelineGraph`
387
514
  Definitions for tasks in a pipeline.
515
+ output : `str` or `None`
516
+ The name of a `~lsst.daf.butler.CollectionType.CHAINED`
517
+ input/output collection.
518
+ output_run : `str` or `None`
519
+ The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
520
+ collection.
521
+ inputs : `str` or `~collections.abc.Iterable` [`str`]
522
+ Input collection name or iterable of collection names.
523
+ extend_run : `bool`
524
+ A boolean indicating whether ``output_run`` should already exist
525
+ and be extended.
526
+ rebase : `bool`
527
+ A boolean indicating whether to force the ``output`` collection to
528
+ be consistent with ``inputs`` and ``output`` run such that the
529
+ ``output`` collection has output run collections first (i.e. those
530
+ that start with the same prefix), then the new inputs, then any
531
+ original inputs not included in the new inputs.
532
+ replace_run : `bool`
533
+ Whether the ``output_run`` should be replaced in the ``output``
534
+ chain.
535
+ prune_replaced : `str` or `None`
536
+ If ``replace_run=True``, whether/how datasets in the old run should
537
+ be removed. Options are ``"purge"``, ``"unstore"``, and `None`.
388
538
 
389
539
  Returns
390
540
  -------
@@ -392,9 +542,17 @@ class ButlerFactory:
392
542
  A read-write butler initialized according to the given arguments.
393
543
  """
394
544
  cls.define_datastore_cache() # Ensure that this butler can use a shared cache.
395
- butler = Butler.from_config(args.butler_config, writeable=True)
396
- self = cls(butler, args, writeable=True)
397
- self.check(args)
545
+ butler = Butler.from_config(butler_config, writeable=True)
546
+ self = cls(
547
+ butler,
548
+ output=output,
549
+ output_run=output_run,
550
+ inputs=inputs,
551
+ extend_run=extend_run,
552
+ rebase=rebase,
553
+ writeable=True,
554
+ )
555
+ self.check(extend_run=extend_run, replace_run=replace_run, prune_replaced=prune_replaced)
398
556
  assert self.output_run is not None, "Output collection has to be specified." # for mypy
399
557
  if self.output is not None:
400
558
  chain_definition = list(
@@ -404,9 +562,9 @@ class ButlerFactory:
404
562
  include_chains=False,
405
563
  )
406
564
  )
407
- if args.replace_run:
565
+ if replace_run:
408
566
  replaced = chain_definition.pop(0)
409
- if args.prune_replaced == "unstore":
567
+ if prune_replaced == "unstore":
410
568
  # Remove datasets from datastore
411
569
  with butler.transaction():
412
570
  # we want to remove regular outputs from this pipeline,
@@ -420,17 +578,17 @@ class ButlerFactory:
420
578
  )
421
579
  ]
422
580
  butler.pruneDatasets(refs, unstore=True, disassociate=False)
423
- elif args.prune_replaced == "purge":
581
+ elif prune_replaced == "purge":
424
582
  # Erase entire collection and all datasets, need to remove
425
583
  # collection from its chain collection first.
426
584
  with butler.transaction():
427
585
  butler.collections.redefine_chain(self.output.name, chain_definition)
428
586
  butler.removeRuns([replaced], unstore=True)
429
- elif args.prune_replaced is not None:
430
- raise NotImplementedError(f"Unsupported --prune-replaced option '{args.prune_replaced}'.")
587
+ elif prune_replaced is not None:
588
+ raise NotImplementedError(f"Unsupported --prune-replaced option '{prune_replaced}'.")
431
589
  if not self.output.exists:
432
590
  butler.collections.register(self.output.name, CollectionType.CHAINED)
433
- if not args.extend_run:
591
+ if not extend_run:
434
592
  butler.collections.register(self.output_run.name, CollectionType.RUN)
435
593
  chain_definition.insert(0, self.output_run.name)
436
594
  butler.collections.redefine_chain(self.output.name, chain_definition)
@@ -242,7 +242,7 @@ def run(ctx: click.Context, **kwargs: Any) -> None:
242
242
  file=sys.stderr,
243
243
  )
244
244
  return
245
- script.run(qgraphObj=qgraph, **kwargs)
245
+ script.run(qgraph, **kwargs)
246
246
 
247
247
 
248
248
  @click.command(cls=PipetaskCommand)
@@ -326,7 +326,7 @@ def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
326
326
  QGRAPH is the path to a serialized Quantum Graph file.
327
327
  """
328
328
  with coverage_context(kwargs):
329
- script.run_qbb(repo, qgraph, **kwargs)
329
+ script.run_qbb(butler_config=repo, qgraph=qgraph, **kwargs)
330
330
 
331
331
 
332
332
  @click.command(cls=PipetaskCommand)
@@ -113,7 +113,6 @@ class qgraph_options(OptionGroup): # noqa: N801
113
113
  ctrlMpExecOpts.summary_option(),
114
114
  ctrlMpExecOpts.dataset_query_constraint(),
115
115
  ctrlMpExecOpts.data_id_table_option(),
116
- ctrlMpExecOpts.qgraph_header_data_option(),
117
116
  ctrlMpExecOpts.mock_option(),
118
117
  ctrlMpExecOpts.mock_failure_option(),
119
118
  ctrlMpExecOpts.unmocked_dataset_types_option(),
@@ -262,13 +262,6 @@ qgraph_node_id_option = MWOptionDecorator(
262
262
  ),
263
263
  )
264
264
 
265
- qgraph_header_data_option = MWOptionDecorator(
266
- "--show-qgraph-header",
267
- is_flag=True,
268
- default=False,
269
- help="Print the headerData for Quantum Graph to the console",
270
- )
271
-
272
265
  qgraph_dot_option = MWOptionDecorator(
273
266
  "--qgraph-dot",
274
267
  help="Location for storing GraphViz DOT representation of a quantum graph.",
@@ -25,11 +25,14 @@
25
25
  # You should have received a copy of the GNU General Public License
26
26
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
27
 
28
- from types import SimpleNamespace
28
+ from __future__ import annotations
29
29
 
30
- from lsst.pipe.base import TaskFactory
30
+ from lsst.pipe.base import BuildId, QuantumGraph
31
+ from lsst.utils.logging import getLogger
31
32
 
32
- from ... import CmdLineFwk
33
+ from ..butler_factory import ButlerFactory
34
+
35
+ _LOG = getLogger(__name__)
33
36
 
34
37
 
35
38
  def pre_exec_init_qbb(
@@ -37,6 +40,7 @@ def pre_exec_init_qbb(
37
40
  qgraph: str,
38
41
  qgraph_id: str | None,
39
42
  config_search_path: list[str] | None,
43
+ **kwargs: object,
40
44
  ) -> None:
41
45
  """Implement the command line interface ``pipetask pre-exec-init-qbb``
42
46
  subcommand.
@@ -55,14 +59,23 @@ def pre_exec_init_qbb(
55
59
  graph loaded from a file. Ignored if graph is not loaded from a file.
56
60
  config_search_path : `list` [`str`]
57
61
  Additional search paths for butler configuration.
62
+ **kwargs : `object`
63
+ Ignored; click commands may accept options for more than one script
64
+ function and pass all the option kwargs to each of the script functions
65
+ which ignore these unused kwargs.
58
66
  """
59
- args = SimpleNamespace(
60
- butler_config=butler_config,
61
- qgraph=qgraph,
62
- qgraph_id=qgraph_id,
63
- config_search_path=config_search_path,
64
- )
67
+ _LOG.verbose("Reading full quantum graph from %s.", qgraph)
68
+ # Load quantum graph. We do not really need individual Quanta here,
69
+ # but we need datastore records for initInputs, and those are only
70
+ # available from Quanta, so load the whole thing.
71
+ qg = QuantumGraph.loadUri(qgraph, graphID=BuildId(qgraph_id) if qgraph_id is not None else None)
72
+
73
+ # Ensure that QBB uses shared datastore cache for writes.
74
+ ButlerFactory.define_datastore_cache()
65
75
 
66
- f = CmdLineFwk()
67
- task_factory = TaskFactory()
68
- f.preExecInitQBB(task_factory, args)
76
+ # Make QBB.
77
+ _LOG.verbose("Initializing quantum-backed butler.")
78
+ butler = qg.make_init_qbb(butler_config, config_search_paths=config_search_path)
79
+ # Save all InitOutputs, configs, etc.
80
+ _LOG.verbose("Instantiating tasks and saving init-outputs.")
81
+ qg.init_output_run(butler)