lsst-ctrl-mpexec 29.2025.3300__tar.gz → 29.2025.3500__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lsst_ctrl_mpexec-29.2025.3300/python/lsst_ctrl_mpexec.egg-info → lsst_ctrl_mpexec-29.2025.3500}/PKG-INFO +1 -1
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/__init__.py +0 -1
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/butler_factory.py +253 -95
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/cmd/commands.py +2 -2
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py +0 -1
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/opt/options.py +0 -7
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/pre_exec_init_qbb.py +25 -12
- lsst_ctrl_mpexec-29.2025.3500/python/lsst/ctrl/mpexec/cli/script/qgraph.py +302 -0
- lsst_ctrl_mpexec-29.2025.3500/python/lsst/ctrl/mpexec/cli/script/run.py +352 -0
- lsst_ctrl_mpexec-29.2025.3500/python/lsst/ctrl/mpexec/cli/script/run_qbb.py +279 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/utils.py +49 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/showInfo.py +17 -15
- lsst_ctrl_mpexec-29.2025.3500/python/lsst/ctrl/mpexec/version.py +2 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500/python/lsst_ctrl_mpexec.egg-info}/PKG-INFO +1 -1
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/SOURCES.txt +0 -1
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cmdLineFwk.py +65 -102
- lsst_ctrl_mpexec-29.2025.3300/python/lsst/ctrl/mpexec/cli/script/qgraph.py +0 -214
- lsst_ctrl_mpexec-29.2025.3300/python/lsst/ctrl/mpexec/cli/script/run.py +0 -240
- lsst_ctrl_mpexec-29.2025.3300/python/lsst/ctrl/mpexec/cli/script/run_qbb.py +0 -144
- lsst_ctrl_mpexec-29.2025.3300/python/lsst/ctrl/mpexec/cmdLineFwk.py +0 -534
- lsst_ctrl_mpexec-29.2025.3300/python/lsst/ctrl/mpexec/version.py +0 -2
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/COPYRIGHT +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/LICENSE +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/MANIFEST.in +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/README.rst +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/bsd_license.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/doc/lsst.ctrl.mpexec/CHANGES.rst +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/doc/lsst.ctrl.mpexec/configuring-pipetask-tasks.rst +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/doc/lsst.ctrl.mpexec/index.rst +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/doc/lsst.ctrl.mpexec/pipetask.rst +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/gpl-v3.0.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/pyproject.toml +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/_pipeline_graph_factory.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/cmd/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/opt/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/opt/arguments.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/pipetask.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/__init__.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/build.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/cleanup.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/confirmable.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/purge.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/report.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/cli/script/update_graph_run.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/execFixupDataId.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/executionGraphFixup.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/log_capture.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/mpGraphExecutor.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/preExecInit.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/py.typed +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/quantumGraphExecutor.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/reports.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/separablePipelineExecutor.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/simple_pipeline_executor.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/singleQuantumExecutor.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/taskFactory.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/util.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/dependency_links.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/entry_points.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/requires.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/top_level.txt +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst_ctrl_mpexec.egg-info/zip-safe +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/setup.cfg +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliCmdCleanup.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliCmdPurge.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliCmdQgraph.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliCmdReport.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliCmdUpdateGraphRun.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliScript.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_cliUtils.py +0 -0
- {lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/tests/test_preExecInit.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-ctrl-mpexec
|
|
3
|
-
Version: 29.2025.
|
|
3
|
+
Version: 29.2025.3500
|
|
4
4
|
Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License: BSD 3-Clause License
|
{lsst_ctrl_mpexec-29.2025.3300 → lsst_ctrl_mpexec-29.2025.3500}/python/lsst/ctrl/mpexec/__init__.py
RENAMED
|
@@ -26,7 +26,6 @@
|
|
|
26
26
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
27
27
|
|
|
28
28
|
from ._pipeline_graph_factory import PipelineGraphFactory
|
|
29
|
-
from .cmdLineFwk import *
|
|
30
29
|
from .executionGraphFixup import *
|
|
31
30
|
from .mpGraphExecutor import *
|
|
32
31
|
from .quantumGraphExecutor import *
|
|
@@ -35,8 +35,7 @@ __all__ = [
|
|
|
35
35
|
|
|
36
36
|
import atexit
|
|
37
37
|
import shutil
|
|
38
|
-
from collections.abc import Sequence
|
|
39
|
-
from types import SimpleNamespace
|
|
38
|
+
from collections.abc import Iterable, Sequence
|
|
40
39
|
|
|
41
40
|
from lsst.daf.butler import Butler, CollectionType
|
|
42
41
|
from lsst.daf.butler.datastore.cache_manager import DatastoreCacheManager
|
|
@@ -44,6 +43,7 @@ from lsst.daf.butler.registry import MissingCollectionError, RegistryDefaults
|
|
|
44
43
|
from lsst.daf.butler.registry.wildcards import CollectionWildcard
|
|
45
44
|
from lsst.pipe.base import Instrument, PipelineGraph
|
|
46
45
|
from lsst.pipe.base.pipeline_graph import NodeType
|
|
46
|
+
from lsst.resources import ResourcePathExpression
|
|
47
47
|
from lsst.utils.logging import getLogger
|
|
48
48
|
|
|
49
49
|
_LOG = getLogger(__name__)
|
|
@@ -127,46 +127,23 @@ class ButlerFactory:
|
|
|
127
127
|
----------
|
|
128
128
|
butler : `lsst.daf.butler.Butler`
|
|
129
129
|
Butler that collections will be added to and/or queried from.
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
``output_run``
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
``
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
``replace_run``
|
|
148
|
-
A boolean indicating that (if `True`) ``output_run`` should already
|
|
149
|
-
exist but will be removed from the output chained collection and
|
|
150
|
-
replaced with a new one.
|
|
151
|
-
|
|
152
|
-
``prune_replaced``
|
|
153
|
-
A boolean indicating whether to prune the replaced run (requires
|
|
154
|
-
``replace_run``).
|
|
155
|
-
|
|
156
|
-
``rebase``
|
|
157
|
-
A boolean indicating whether to force the ``output`` collection
|
|
158
|
-
to be consistent with ``inputs`` and ``output`` run such that the
|
|
159
|
-
``output`` collection has output run collections first (i.e. those
|
|
160
|
-
that start with the same prefix), then the new inputs, then any
|
|
161
|
-
original inputs not included in the new inputs.
|
|
162
|
-
|
|
163
|
-
``inputs``
|
|
164
|
-
Input collections of any type; see
|
|
165
|
-
:ref:`daf_butler_ordered_collection_searches` for details.
|
|
166
|
-
|
|
167
|
-
``butler_config``
|
|
168
|
-
Path to a data repository root or configuration file.
|
|
169
|
-
|
|
130
|
+
output : `str` or `None`
|
|
131
|
+
The name of a `~lsst.daf.butler.CollectionType.CHAINED` input/output
|
|
132
|
+
collection.
|
|
133
|
+
output_run : `str` or `None`
|
|
134
|
+
The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
|
|
135
|
+
collection.
|
|
136
|
+
inputs : `str` or `~collections.abc.Iterable` [`str`]
|
|
137
|
+
Input collection name or iterable of collection names.
|
|
138
|
+
extend_run : `bool`
|
|
139
|
+
A boolean indicating whether ``output_run`` should already exist and be
|
|
140
|
+
extended.
|
|
141
|
+
rebase : `bool`
|
|
142
|
+
A boolean indicating whether to force the ``output`` collection to be
|
|
143
|
+
consistent with ``inputs`` and ``output`` run such that the ``output``
|
|
144
|
+
collection has output run collections first (i.e. those that start with
|
|
145
|
+
the same prefix), then the new inputs, then any original inputs not
|
|
146
|
+
included in the new inputs.
|
|
170
147
|
writeable : `bool`
|
|
171
148
|
If `True`, a `~lsst.daf.butler.Butler` is being initialized in a
|
|
172
149
|
context where actual writes should happens, and hence no output run
|
|
@@ -178,17 +155,27 @@ class ButlerFactory:
|
|
|
178
155
|
Raised if ``writeable is True`` but there are no output collections.
|
|
179
156
|
"""
|
|
180
157
|
|
|
181
|
-
def __init__(
|
|
182
|
-
|
|
183
|
-
|
|
158
|
+
def __init__(
|
|
159
|
+
self,
|
|
160
|
+
butler: Butler,
|
|
161
|
+
*,
|
|
162
|
+
output: str | None,
|
|
163
|
+
output_run: str | None,
|
|
164
|
+
inputs: str | Iterable[str],
|
|
165
|
+
extend_run: bool = False,
|
|
166
|
+
rebase: bool = False,
|
|
167
|
+
writeable: bool,
|
|
168
|
+
):
|
|
169
|
+
if output is not None:
|
|
170
|
+
self.output = OutputChainedCollectionInfo(butler, output)
|
|
184
171
|
else:
|
|
185
172
|
self.output = None
|
|
186
|
-
if
|
|
187
|
-
if
|
|
173
|
+
if output_run is not None:
|
|
174
|
+
if rebase and self.output and not output_run.startswith(self.output.name):
|
|
188
175
|
raise ValueError("Cannot rebase if output run does not start with output collection name.")
|
|
189
|
-
self.output_run = OutputRunCollectionInfo(butler,
|
|
176
|
+
self.output_run = OutputRunCollectionInfo(butler, output_run)
|
|
190
177
|
elif self.output is not None:
|
|
191
|
-
if
|
|
178
|
+
if extend_run:
|
|
192
179
|
if not self.output.chain:
|
|
193
180
|
raise ValueError("Cannot use --extend-run option with non-existing or empty output chain")
|
|
194
181
|
run_name = self.output.chain[0]
|
|
@@ -204,45 +191,50 @@ class ButlerFactory:
|
|
|
204
191
|
# front so we can tell if the user passes the same inputs on subsequent
|
|
205
192
|
# calls, even though we also flatten when we define the output CHAINED
|
|
206
193
|
# collection.
|
|
207
|
-
self.inputs = tuple(butler.collections.query(
|
|
194
|
+
self.inputs = tuple(butler.collections.query(inputs, flatten_chains=True)) if inputs else ()
|
|
208
195
|
|
|
209
196
|
# If things are inconsistent and user has asked for a rebase then
|
|
210
197
|
# construct the new output chain.
|
|
211
|
-
if
|
|
198
|
+
if rebase and self._check_output_input_consistency():
|
|
212
199
|
assert self.output is not None
|
|
213
200
|
newOutputChain = [item for item in self.output.chain if item.startswith(self.output.name)]
|
|
214
201
|
newOutputChain.extend([item for item in self.inputs if item not in newOutputChain])
|
|
215
202
|
newOutputChain.extend([item for item in self.output.chain if item not in newOutputChain])
|
|
216
203
|
self.output.chain = tuple(newOutputChain)
|
|
217
204
|
|
|
218
|
-
def check(self,
|
|
205
|
+
def check(self, *, extend_run: bool, replace_run: bool, prune_replaced: str | None = None) -> None:
|
|
219
206
|
"""Check command-line options for consistency with each other and the
|
|
220
207
|
data repository.
|
|
221
208
|
|
|
222
209
|
Parameters
|
|
223
210
|
----------
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
211
|
+
extend_run : `bool`
|
|
212
|
+
Whether the ``output_run`` should already exist and be extended.
|
|
213
|
+
replace_run : `bool`
|
|
214
|
+
Whether the ``output_run`` should be replaced in the ``output``
|
|
215
|
+
chain.
|
|
216
|
+
prune_replaced : `str` or `None`
|
|
217
|
+
If ``replace_run=True``, whether/how datasets in the old run should
|
|
218
|
+
be removed. Options are ``"purge"``, ``"unstore"``, and `None`.
|
|
227
219
|
"""
|
|
228
|
-
assert not (
|
|
220
|
+
assert not (extend_run and replace_run), "In mutually-exclusive group in ArgumentParser."
|
|
229
221
|
if consistencyError := self._check_output_input_consistency():
|
|
230
222
|
raise ValueError(consistencyError)
|
|
231
223
|
|
|
232
|
-
if
|
|
224
|
+
if extend_run:
|
|
233
225
|
if self.output_run is None:
|
|
234
226
|
raise ValueError("Cannot --extend-run when no output collection is given.")
|
|
235
227
|
elif not self.output_run.exists:
|
|
236
228
|
raise ValueError(
|
|
237
229
|
f"Cannot --extend-run; output collection '{self.output_run.name}' does not exist."
|
|
238
230
|
)
|
|
239
|
-
if not
|
|
231
|
+
if not extend_run and self.output_run is not None and self.output_run.exists:
|
|
240
232
|
raise ValueError(
|
|
241
233
|
f"Output run '{self.output_run.name}' already exists, but --extend-run was not given."
|
|
242
234
|
)
|
|
243
|
-
if
|
|
235
|
+
if prune_replaced and not replace_run:
|
|
244
236
|
raise ValueError("--prune-replaced requires --replace-run.")
|
|
245
|
-
if
|
|
237
|
+
if replace_run and (self.output is None or not self.output.exists):
|
|
246
238
|
raise ValueError("--output must point to an existing CHAINED collection for --replace-run.")
|
|
247
239
|
|
|
248
240
|
def _check_output_input_consistency(self) -> str | None:
|
|
@@ -264,15 +256,48 @@ class ButlerFactory:
|
|
|
264
256
|
return None
|
|
265
257
|
|
|
266
258
|
@classmethod
|
|
267
|
-
def _make_read_parts(
|
|
259
|
+
def _make_read_parts(
|
|
260
|
+
cls,
|
|
261
|
+
butler_config: ResourcePathExpression,
|
|
262
|
+
*,
|
|
263
|
+
output: str | None,
|
|
264
|
+
output_run: str | None,
|
|
265
|
+
inputs: str | Iterable[str],
|
|
266
|
+
extend_run: bool = False,
|
|
267
|
+
rebase: bool = False,
|
|
268
|
+
replace_run: bool,
|
|
269
|
+
prune_replaced: str | None = None,
|
|
270
|
+
) -> tuple[Butler, Sequence[str], ButlerFactory]:
|
|
268
271
|
"""Parse arguments to support implementations of `make_read_butler` and
|
|
269
272
|
`make_butler_and_collections`.
|
|
270
273
|
|
|
271
274
|
Parameters
|
|
272
275
|
----------
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
+
butler_config : convertible to `lsst.resources.ResourcePath`
|
|
277
|
+
Path to configuration for the butler.
|
|
278
|
+
output : `str` or `None`
|
|
279
|
+
The name of a `~lsst.daf.butler.CollectionType.CHAINED`
|
|
280
|
+
input/output collection.
|
|
281
|
+
output_run : `str` or `None`
|
|
282
|
+
The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
|
|
283
|
+
collection.
|
|
284
|
+
inputs : `str` or `~collections.abc.Iterable` [`str`]
|
|
285
|
+
Input collection name or iterable of collection names.
|
|
286
|
+
extend_run : `bool`
|
|
287
|
+
A boolean indicating whether ``output_run`` should already exist
|
|
288
|
+
and be extended.
|
|
289
|
+
rebase : `bool`
|
|
290
|
+
A boolean indicating whether to force the ``output`` collection to
|
|
291
|
+
be consistent with ``inputs`` and ``output`` run such that the
|
|
292
|
+
``output`` collection has output run collections first (i.e. those
|
|
293
|
+
that start with the same prefix), then the new inputs, then any
|
|
294
|
+
original inputs not included in the new inputs.
|
|
295
|
+
replace_run : `bool`
|
|
296
|
+
Whether the ``output_run`` should be replaced in the ``output``
|
|
297
|
+
chain.
|
|
298
|
+
prune_replaced : `str` or `None`
|
|
299
|
+
If ``replace_run=True``, whether/how datasets in the old run should
|
|
300
|
+
be removed. Options are ``"purge"``, ``"unstore"``, and `None`.
|
|
276
301
|
|
|
277
302
|
Returns
|
|
278
303
|
-------
|
|
@@ -285,11 +310,19 @@ class ButlerFactory:
|
|
|
285
310
|
A new `ButlerFactory` instance representing the processed version
|
|
286
311
|
of ``args``.
|
|
287
312
|
"""
|
|
288
|
-
butler = Butler.from_config(
|
|
289
|
-
self = cls(
|
|
290
|
-
|
|
313
|
+
butler = Butler.from_config(butler_config, writeable=False)
|
|
314
|
+
self = cls(
|
|
315
|
+
butler,
|
|
316
|
+
output=output,
|
|
317
|
+
output_run=output_run,
|
|
318
|
+
inputs=inputs,
|
|
319
|
+
extend_run=extend_run,
|
|
320
|
+
rebase=rebase,
|
|
321
|
+
writeable=False,
|
|
322
|
+
)
|
|
323
|
+
self.check(extend_run=extend_run, replace_run=replace_run, prune_replaced=prune_replaced)
|
|
291
324
|
if self.output and self.output.exists:
|
|
292
|
-
if
|
|
325
|
+
if replace_run:
|
|
293
326
|
replaced = self.output.chain[0]
|
|
294
327
|
inputs = list(self.output.chain[1:])
|
|
295
328
|
_LOG.debug(
|
|
@@ -299,44 +332,118 @@ class ButlerFactory:
|
|
|
299
332
|
inputs = [self.output.name]
|
|
300
333
|
else:
|
|
301
334
|
inputs = list(self.inputs)
|
|
302
|
-
if
|
|
335
|
+
if extend_run:
|
|
303
336
|
assert self.output_run is not None, "Output collection has to be specified."
|
|
304
337
|
inputs.insert(0, self.output_run.name)
|
|
305
338
|
collSearch = CollectionWildcard.from_expression(inputs).require_ordered()
|
|
306
339
|
return butler, collSearch, self
|
|
307
340
|
|
|
308
341
|
@classmethod
|
|
309
|
-
def make_read_butler(
|
|
342
|
+
def make_read_butler(
|
|
343
|
+
cls,
|
|
344
|
+
butler_config: ResourcePathExpression,
|
|
345
|
+
*,
|
|
346
|
+
output: str | None,
|
|
347
|
+
output_run: str | None,
|
|
348
|
+
inputs: str | Iterable[str],
|
|
349
|
+
extend_run: bool = False,
|
|
350
|
+
rebase: bool = False,
|
|
351
|
+
replace_run: bool,
|
|
352
|
+
prune_replaced: str | None = None,
|
|
353
|
+
) -> Butler:
|
|
310
354
|
"""Construct a read-only butler according to the given command-line
|
|
311
355
|
arguments.
|
|
312
356
|
|
|
313
357
|
Parameters
|
|
314
358
|
----------
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
359
|
+
butler_config : convertible to `lsst.resources.ResourcePath`
|
|
360
|
+
Path to configuration for the butler.
|
|
361
|
+
output : `str` or `None`
|
|
362
|
+
The name of a `~lsst.daf.butler.CollectionType.CHAINED`
|
|
363
|
+
input/output collection.
|
|
364
|
+
output_run : `str` or `None`
|
|
365
|
+
The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
|
|
366
|
+
collection.
|
|
367
|
+
inputs : `str` or `~collections.abc.Iterable` [`str`]
|
|
368
|
+
Input collection name or iterable of collection names.
|
|
369
|
+
extend_run : `bool`
|
|
370
|
+
A boolean indicating whether ``output_run`` should already exist
|
|
371
|
+
and be extended.
|
|
372
|
+
rebase : `bool`
|
|
373
|
+
A boolean indicating whether to force the ``output`` collection to
|
|
374
|
+
be consistent with ``inputs`` and ``output`` run such that the
|
|
375
|
+
``output`` collection has output run collections first (i.e. those
|
|
376
|
+
that start with the same prefix), then the new inputs, then any
|
|
377
|
+
original inputs not included in the new inputs.
|
|
378
|
+
replace_run : `bool`
|
|
379
|
+
Whether the ``output_run`` should be replaced in the ``output``
|
|
380
|
+
chain.
|
|
381
|
+
prune_replaced : `str` or `None`
|
|
382
|
+
If ``replace_run=True``, whether/how datasets in the old run should
|
|
383
|
+
be removed. Options are ``"purge"``, ``"unstore"``, and `None`.
|
|
318
384
|
|
|
319
385
|
Returns
|
|
320
386
|
-------
|
|
321
387
|
butler : `lsst.daf.butler.Butler`
|
|
322
|
-
A read-only butler initialized with the collections
|
|
323
|
-
``args``.
|
|
388
|
+
A read-only butler initialized with the given collections.
|
|
324
389
|
"""
|
|
325
390
|
cls.define_datastore_cache() # Ensure that this butler can use a shared cache.
|
|
326
|
-
butler, inputs, _ = cls._make_read_parts(
|
|
391
|
+
butler, inputs, _ = cls._make_read_parts(
|
|
392
|
+
butler_config,
|
|
393
|
+
output=output,
|
|
394
|
+
output_run=output_run,
|
|
395
|
+
inputs=inputs,
|
|
396
|
+
extend_run=extend_run,
|
|
397
|
+
rebase=rebase,
|
|
398
|
+
replace_run=replace_run,
|
|
399
|
+
prune_replaced=prune_replaced,
|
|
400
|
+
)
|
|
327
401
|
_LOG.debug("Preparing butler to read from %s.", inputs)
|
|
328
402
|
return Butler.from_config(butler=butler, collections=inputs)
|
|
329
403
|
|
|
330
404
|
@classmethod
|
|
331
|
-
def make_butler_and_collections(
|
|
405
|
+
def make_butler_and_collections(
|
|
406
|
+
cls,
|
|
407
|
+
butler_config: ResourcePathExpression,
|
|
408
|
+
*,
|
|
409
|
+
output: str | None,
|
|
410
|
+
output_run: str | None,
|
|
411
|
+
inputs: str | Iterable[str],
|
|
412
|
+
extend_run: bool = False,
|
|
413
|
+
rebase: bool = False,
|
|
414
|
+
replace_run: bool,
|
|
415
|
+
prune_replaced: str | None = None,
|
|
416
|
+
) -> tuple[Butler, Sequence[str], str | None]:
|
|
332
417
|
"""Return a read-only butler, a collection search path, and the name
|
|
333
418
|
of the run to be used for future writes.
|
|
334
419
|
|
|
335
420
|
Parameters
|
|
336
421
|
----------
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
422
|
+
butler_config : convertible to `lsst.resources.ResourcePath`
|
|
423
|
+
Path to configuration for the butler.
|
|
424
|
+
output : `str` or `None`
|
|
425
|
+
The name of a `~lsst.daf.butler.CollectionType.CHAINED`
|
|
426
|
+
input/output collection.
|
|
427
|
+
output_run : `str` or `None`
|
|
428
|
+
The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
|
|
429
|
+
collection.
|
|
430
|
+
inputs : `str` or `~collections.abc.Iterable` [`str`]
|
|
431
|
+
Input collection name or iterable of collection names.
|
|
432
|
+
extend_run : `bool`
|
|
433
|
+
A boolean indicating whether ``output_run`` should already exist
|
|
434
|
+
and be extended.
|
|
435
|
+
rebase : `bool`
|
|
436
|
+
A boolean indicating whether to force the ``output`` collection to
|
|
437
|
+
be consistent with ``inputs`` and ``output`` run such that the
|
|
438
|
+
``output`` collection has output run collections first (i.e. those
|
|
439
|
+
that start with the same prefix), then the new inputs, then any
|
|
440
|
+
original inputs not included in the new inputs.
|
|
441
|
+
replace_run : `bool`
|
|
442
|
+
Whether the ``output_run`` should be replaced in the ``output``
|
|
443
|
+
chain.
|
|
444
|
+
prune_replaced : `str` or `None`
|
|
445
|
+
If ``replace_run=True``, whether/how datasets in the old run should
|
|
446
|
+
be removed. Options are ``"purge"``, ``"unstore"``, and `None`.
|
|
340
447
|
|
|
341
448
|
Returns
|
|
342
449
|
-------
|
|
@@ -349,9 +456,18 @@ class ButlerFactory:
|
|
|
349
456
|
Name of the output `~lsst.daf.butler.CollectionType.RUN` collection
|
|
350
457
|
if it already exists, or `None` if it does not.
|
|
351
458
|
"""
|
|
352
|
-
butler, inputs, self = cls._make_read_parts(
|
|
459
|
+
butler, inputs, self = cls._make_read_parts(
|
|
460
|
+
butler_config,
|
|
461
|
+
output=output,
|
|
462
|
+
output_run=output_run,
|
|
463
|
+
inputs=inputs,
|
|
464
|
+
extend_run=extend_run,
|
|
465
|
+
rebase=rebase,
|
|
466
|
+
replace_run=replace_run,
|
|
467
|
+
prune_replaced=prune_replaced,
|
|
468
|
+
)
|
|
353
469
|
run: str | None = None
|
|
354
|
-
if
|
|
470
|
+
if extend_run:
|
|
355
471
|
assert self.output_run is not None, "Output collection has to be specified."
|
|
356
472
|
if self.output_run is not None:
|
|
357
473
|
run = self.output_run.name
|
|
@@ -374,17 +490,51 @@ class ButlerFactory:
|
|
|
374
490
|
_LOG.debug("Defining shared datastore cache directory to %s", cache_dir)
|
|
375
491
|
|
|
376
492
|
@classmethod
|
|
377
|
-
def make_write_butler(
|
|
493
|
+
def make_write_butler(
|
|
494
|
+
cls,
|
|
495
|
+
butler_config: ResourcePathExpression,
|
|
496
|
+
pipeline_graph: PipelineGraph,
|
|
497
|
+
*,
|
|
498
|
+
output: str | None,
|
|
499
|
+
output_run: str | None,
|
|
500
|
+
inputs: str | Iterable[str],
|
|
501
|
+
extend_run: bool = False,
|
|
502
|
+
rebase: bool = False,
|
|
503
|
+
replace_run: bool,
|
|
504
|
+
prune_replaced: str | None = None,
|
|
505
|
+
) -> Butler:
|
|
378
506
|
"""Return a read-write butler initialized to write to and read from
|
|
379
507
|
the collections specified by the given command-line arguments.
|
|
380
508
|
|
|
381
509
|
Parameters
|
|
382
510
|
----------
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
construction parameter of the same name.
|
|
511
|
+
butler_config : convertible to `lsst.resources.ResourcePath`
|
|
512
|
+
Path to configuration for the butler.
|
|
386
513
|
pipeline_graph : `lsst.pipe.base.PipelineGraph`
|
|
387
514
|
Definitions for tasks in a pipeline.
|
|
515
|
+
output : `str` or `None`
|
|
516
|
+
The name of a `~lsst.daf.butler.CollectionType.CHAINED`
|
|
517
|
+
input/output collection.
|
|
518
|
+
output_run : `str` or `None`
|
|
519
|
+
The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
|
|
520
|
+
collection.
|
|
521
|
+
inputs : `str` or `~collections.abc.Iterable` [`str`]
|
|
522
|
+
Input collection name or iterable of collection names.
|
|
523
|
+
extend_run : `bool`
|
|
524
|
+
A boolean indicating whether ``output_run`` should already exist
|
|
525
|
+
and be extended.
|
|
526
|
+
rebase : `bool`
|
|
527
|
+
A boolean indicating whether to force the ``output`` collection to
|
|
528
|
+
be consistent with ``inputs`` and ``output`` run such that the
|
|
529
|
+
``output`` collection has output run collections first (i.e. those
|
|
530
|
+
that start with the same prefix), then the new inputs, then any
|
|
531
|
+
original inputs not included in the new inputs.
|
|
532
|
+
replace_run : `bool`
|
|
533
|
+
Whether the ``output_run`` should be replaced in the ``output``
|
|
534
|
+
chain.
|
|
535
|
+
prune_replaced : `str` or `None`
|
|
536
|
+
If ``replace_run=True``, whether/how datasets in the old run should
|
|
537
|
+
be removed. Options are ``"purge"``, ``"unstore"``, and `None`.
|
|
388
538
|
|
|
389
539
|
Returns
|
|
390
540
|
-------
|
|
@@ -392,9 +542,17 @@ class ButlerFactory:
|
|
|
392
542
|
A read-write butler initialized according to the given arguments.
|
|
393
543
|
"""
|
|
394
544
|
cls.define_datastore_cache() # Ensure that this butler can use a shared cache.
|
|
395
|
-
butler = Butler.from_config(
|
|
396
|
-
self = cls(
|
|
397
|
-
|
|
545
|
+
butler = Butler.from_config(butler_config, writeable=True)
|
|
546
|
+
self = cls(
|
|
547
|
+
butler,
|
|
548
|
+
output=output,
|
|
549
|
+
output_run=output_run,
|
|
550
|
+
inputs=inputs,
|
|
551
|
+
extend_run=extend_run,
|
|
552
|
+
rebase=rebase,
|
|
553
|
+
writeable=True,
|
|
554
|
+
)
|
|
555
|
+
self.check(extend_run=extend_run, replace_run=replace_run, prune_replaced=prune_replaced)
|
|
398
556
|
assert self.output_run is not None, "Output collection has to be specified." # for mypy
|
|
399
557
|
if self.output is not None:
|
|
400
558
|
chain_definition = list(
|
|
@@ -404,9 +562,9 @@ class ButlerFactory:
|
|
|
404
562
|
include_chains=False,
|
|
405
563
|
)
|
|
406
564
|
)
|
|
407
|
-
if
|
|
565
|
+
if replace_run:
|
|
408
566
|
replaced = chain_definition.pop(0)
|
|
409
|
-
if
|
|
567
|
+
if prune_replaced == "unstore":
|
|
410
568
|
# Remove datasets from datastore
|
|
411
569
|
with butler.transaction():
|
|
412
570
|
# we want to remove regular outputs from this pipeline,
|
|
@@ -420,17 +578,17 @@ class ButlerFactory:
|
|
|
420
578
|
)
|
|
421
579
|
]
|
|
422
580
|
butler.pruneDatasets(refs, unstore=True, disassociate=False)
|
|
423
|
-
elif
|
|
581
|
+
elif prune_replaced == "purge":
|
|
424
582
|
# Erase entire collection and all datasets, need to remove
|
|
425
583
|
# collection from its chain collection first.
|
|
426
584
|
with butler.transaction():
|
|
427
585
|
butler.collections.redefine_chain(self.output.name, chain_definition)
|
|
428
586
|
butler.removeRuns([replaced], unstore=True)
|
|
429
|
-
elif
|
|
430
|
-
raise NotImplementedError(f"Unsupported --prune-replaced option '{
|
|
587
|
+
elif prune_replaced is not None:
|
|
588
|
+
raise NotImplementedError(f"Unsupported --prune-replaced option '{prune_replaced}'.")
|
|
431
589
|
if not self.output.exists:
|
|
432
590
|
butler.collections.register(self.output.name, CollectionType.CHAINED)
|
|
433
|
-
if not
|
|
591
|
+
if not extend_run:
|
|
434
592
|
butler.collections.register(self.output_run.name, CollectionType.RUN)
|
|
435
593
|
chain_definition.insert(0, self.output_run.name)
|
|
436
594
|
butler.collections.redefine_chain(self.output.name, chain_definition)
|
|
@@ -242,7 +242,7 @@ def run(ctx: click.Context, **kwargs: Any) -> None:
|
|
|
242
242
|
file=sys.stderr,
|
|
243
243
|
)
|
|
244
244
|
return
|
|
245
|
-
script.run(
|
|
245
|
+
script.run(qgraph, **kwargs)
|
|
246
246
|
|
|
247
247
|
|
|
248
248
|
@click.command(cls=PipetaskCommand)
|
|
@@ -326,7 +326,7 @@ def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
|
|
|
326
326
|
QGRAPH is the path to a serialized Quantum Graph file.
|
|
327
327
|
"""
|
|
328
328
|
with coverage_context(kwargs):
|
|
329
|
-
script.run_qbb(repo, qgraph, **kwargs)
|
|
329
|
+
script.run_qbb(butler_config=repo, qgraph=qgraph, **kwargs)
|
|
330
330
|
|
|
331
331
|
|
|
332
332
|
@click.command(cls=PipetaskCommand)
|
|
@@ -113,7 +113,6 @@ class qgraph_options(OptionGroup): # noqa: N801
|
|
|
113
113
|
ctrlMpExecOpts.summary_option(),
|
|
114
114
|
ctrlMpExecOpts.dataset_query_constraint(),
|
|
115
115
|
ctrlMpExecOpts.data_id_table_option(),
|
|
116
|
-
ctrlMpExecOpts.qgraph_header_data_option(),
|
|
117
116
|
ctrlMpExecOpts.mock_option(),
|
|
118
117
|
ctrlMpExecOpts.mock_failure_option(),
|
|
119
118
|
ctrlMpExecOpts.unmocked_dataset_types_option(),
|
|
@@ -262,13 +262,6 @@ qgraph_node_id_option = MWOptionDecorator(
|
|
|
262
262
|
),
|
|
263
263
|
)
|
|
264
264
|
|
|
265
|
-
qgraph_header_data_option = MWOptionDecorator(
|
|
266
|
-
"--show-qgraph-header",
|
|
267
|
-
is_flag=True,
|
|
268
|
-
default=False,
|
|
269
|
-
help="Print the headerData for Quantum Graph to the console",
|
|
270
|
-
)
|
|
271
|
-
|
|
272
265
|
qgraph_dot_option = MWOptionDecorator(
|
|
273
266
|
"--qgraph-dot",
|
|
274
267
|
help="Location for storing GraphViz DOT representation of a quantum graph.",
|
|
@@ -25,11 +25,14 @@
|
|
|
25
25
|
# You should have received a copy of the GNU General Public License
|
|
26
26
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
27
27
|
|
|
28
|
-
from
|
|
28
|
+
from __future__ import annotations
|
|
29
29
|
|
|
30
|
-
from lsst.pipe.base import
|
|
30
|
+
from lsst.pipe.base import BuildId, QuantumGraph
|
|
31
|
+
from lsst.utils.logging import getLogger
|
|
31
32
|
|
|
32
|
-
from
|
|
33
|
+
from ..butler_factory import ButlerFactory
|
|
34
|
+
|
|
35
|
+
_LOG = getLogger(__name__)
|
|
33
36
|
|
|
34
37
|
|
|
35
38
|
def pre_exec_init_qbb(
|
|
@@ -37,6 +40,7 @@ def pre_exec_init_qbb(
|
|
|
37
40
|
qgraph: str,
|
|
38
41
|
qgraph_id: str | None,
|
|
39
42
|
config_search_path: list[str] | None,
|
|
43
|
+
**kwargs: object,
|
|
40
44
|
) -> None:
|
|
41
45
|
"""Implement the command line interface ``pipetask pre-exec-init-qbb``
|
|
42
46
|
subcommand.
|
|
@@ -55,14 +59,23 @@ def pre_exec_init_qbb(
|
|
|
55
59
|
graph loaded from a file. Ignored if graph is not loaded from a file.
|
|
56
60
|
config_search_path : `list` [`str`]
|
|
57
61
|
Additional search paths for butler configuration.
|
|
62
|
+
**kwargs : `object`
|
|
63
|
+
Ignored; click commands may accept options for more than one script
|
|
64
|
+
function and pass all the option kwargs to each of the script functions
|
|
65
|
+
which ignore these unused kwargs.
|
|
58
66
|
"""
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
67
|
+
_LOG.verbose("Reading full quantum graph from %s.", qgraph)
|
|
68
|
+
# Load quantum graph. We do not really need individual Quanta here,
|
|
69
|
+
# but we need datastore records for initInputs, and those are only
|
|
70
|
+
# available from Quanta, so load the whole thing.
|
|
71
|
+
qg = QuantumGraph.loadUri(qgraph, graphID=BuildId(qgraph_id) if qgraph_id is not None else None)
|
|
72
|
+
|
|
73
|
+
# Ensure that QBB uses shared datastore cache for writes.
|
|
74
|
+
ButlerFactory.define_datastore_cache()
|
|
65
75
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
76
|
+
# Make QBB.
|
|
77
|
+
_LOG.verbose("Initializing quantum-backed butler.")
|
|
78
|
+
butler = qg.make_init_qbb(butler_config, config_search_paths=config_search_path)
|
|
79
|
+
# Save all InitOutputs, configs, etc.
|
|
80
|
+
_LOG.verbose("Instantiating tasks and saving init-outputs.")
|
|
81
|
+
qg.init_output_run(butler)
|