lsst-pipe-base 29.2025.4800__py3-none-any.whl → 30.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/pipe/base/_instrument.py +6 -5
- lsst/pipe/base/caching_limited_butler.py +3 -0
- lsst/pipe/base/log_capture.py +39 -79
- lsst/pipe/base/log_on_close.py +79 -0
- lsst/pipe/base/mp_graph_executor.py +51 -15
- lsst/pipe/base/quantum_graph/_common.py +4 -3
- lsst/pipe/base/quantum_graph/_multiblock.py +6 -16
- lsst/pipe/base/quantum_graph/_predicted.py +106 -12
- lsst/pipe/base/quantum_graph/_provenance.py +657 -6
- lsst/pipe/base/quantum_graph/aggregator/_communicators.py +18 -50
- lsst/pipe/base/quantum_graph/aggregator/_ingester.py +14 -3
- lsst/pipe/base/quantum_graph/aggregator/_scanner.py +49 -232
- lsst/pipe/base/quantum_graph/aggregator/_structs.py +3 -113
- lsst/pipe/base/quantum_graph/aggregator/_supervisor.py +10 -5
- lsst/pipe/base/quantum_graph/aggregator/_writer.py +31 -348
- lsst/pipe/base/quantum_graph/formatter.py +101 -0
- lsst/pipe/base/quantum_graph_builder.py +12 -1
- lsst/pipe/base/quantum_graph_executor.py +116 -13
- lsst/pipe/base/quantum_graph_skeleton.py +1 -7
- lsst/pipe/base/script/register_instrument.py +4 -4
- lsst/pipe/base/script/retrieve_artifacts_for_quanta.py +5 -6
- lsst/pipe/base/script/transfer_from_graph.py +42 -42
- lsst/pipe/base/script/zip_from_graph.py +7 -8
- lsst/pipe/base/separable_pipeline_executor.py +18 -2
- lsst/pipe/base/simple_pipeline_executor.py +4 -3
- lsst/pipe/base/single_quantum_executor.py +70 -34
- lsst/pipe/base/tests/mocks/_repo.py +44 -16
- lsst/pipe/base/tests/simpleQGraph.py +43 -35
- lsst/pipe/base/version.py +1 -1
- {lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/METADATA +1 -1
- {lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/RECORD +39 -37
- {lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/WHEEL +1 -1
- {lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/entry_points.txt +0 -0
- {lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/licenses/LICENSE +0 -0
- {lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/licenses/bsd_license.txt +0 -0
- {lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/licenses/gpl-v3.0.txt +0 -0
- {lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/top_level.txt +0 -0
- {lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/zip-safe +0 -0
|
@@ -37,13 +37,13 @@ from typing import Any
|
|
|
37
37
|
|
|
38
38
|
from lsst.daf.butler import (
|
|
39
39
|
Butler,
|
|
40
|
-
ButlerMetrics,
|
|
41
40
|
DatasetRef,
|
|
42
41
|
DatasetType,
|
|
43
42
|
LimitedButler,
|
|
44
43
|
NamedKeyDict,
|
|
45
44
|
Quantum,
|
|
46
45
|
)
|
|
46
|
+
from lsst.daf.butler.logging import ButlerLogRecords
|
|
47
47
|
from lsst.utils.introspection import get_full_type_name
|
|
48
48
|
from lsst.utils.timer import logInfo
|
|
49
49
|
|
|
@@ -59,7 +59,7 @@ from .connections import AdjustQuantumHelper
|
|
|
59
59
|
from .log_capture import LogCapture, _ExecutionLogRecordsExtra
|
|
60
60
|
from .pipeline_graph import TaskNode
|
|
61
61
|
from .pipelineTask import PipelineTask
|
|
62
|
-
from .quantum_graph_executor import QuantumExecutor
|
|
62
|
+
from .quantum_graph_executor import QuantumExecutionResult, QuantumExecutor
|
|
63
63
|
from .quantum_reports import QuantumReport
|
|
64
64
|
from .task import _TASK_FULL_METADATA_TYPE, _TASK_METADATA_TYPE
|
|
65
65
|
from .taskFactory import TaskFactory
|
|
@@ -157,41 +157,69 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
157
157
|
self._previous_process_quanta: list[uuid.UUID] = []
|
|
158
158
|
|
|
159
159
|
def execute(
|
|
160
|
-
self,
|
|
161
|
-
|
|
160
|
+
self,
|
|
161
|
+
task_node: TaskNode,
|
|
162
|
+
/,
|
|
163
|
+
quantum: Quantum,
|
|
164
|
+
quantum_id: uuid.UUID | None = None,
|
|
165
|
+
*,
|
|
166
|
+
log_records: ButlerLogRecords | None = None,
|
|
167
|
+
) -> QuantumExecutionResult:
|
|
162
168
|
# Docstring inherited from QuantumExecutor.execute
|
|
163
|
-
assert quantum.dataId is not None, "Quantum DataId cannot be None"
|
|
164
|
-
|
|
165
169
|
if self._butler is not None:
|
|
166
170
|
self._butler.registry.refresh()
|
|
167
|
-
|
|
168
|
-
result = self._execute(task_node, quantum, quantum_id=quantum_id)
|
|
169
|
-
report = QuantumReport(quantumId=quantum_id, dataId=quantum.dataId, taskLabel=task_node.label)
|
|
170
|
-
return result, report
|
|
171
|
+
return self._execute(task_node, quantum, quantum_id=quantum_id, log_records=log_records)
|
|
171
172
|
|
|
172
173
|
def _execute(
|
|
173
|
-
self,
|
|
174
|
-
|
|
174
|
+
self,
|
|
175
|
+
task_node: TaskNode,
|
|
176
|
+
/,
|
|
177
|
+
quantum: Quantum,
|
|
178
|
+
quantum_id: uuid.UUID | None = None,
|
|
179
|
+
*,
|
|
180
|
+
log_records: ButlerLogRecords | None = None,
|
|
181
|
+
) -> QuantumExecutionResult:
|
|
175
182
|
"""Execute the quantum.
|
|
176
183
|
|
|
177
184
|
Internal implementation of `execute()`.
|
|
178
185
|
"""
|
|
179
|
-
startTime = time.time()
|
|
180
|
-
|
|
181
186
|
# Make a limited butler instance if needed.
|
|
182
187
|
limited_butler: LimitedButler
|
|
188
|
+
used_butler_factory = False
|
|
183
189
|
if self._butler is not None:
|
|
184
190
|
limited_butler = self._butler
|
|
185
191
|
else:
|
|
186
192
|
# We check this in constructor, but mypy needs this check here.
|
|
187
193
|
assert self._limited_butler_factory is not None
|
|
188
194
|
limited_butler = self._limited_butler_factory(quantum)
|
|
195
|
+
used_butler_factory = True
|
|
189
196
|
|
|
197
|
+
try:
|
|
198
|
+
return self._execute_with_limited_butler(
|
|
199
|
+
task_node, limited_butler, quantum=quantum, quantum_id=quantum_id, log_records=log_records
|
|
200
|
+
)
|
|
201
|
+
finally:
|
|
202
|
+
if used_butler_factory:
|
|
203
|
+
limited_butler.close()
|
|
204
|
+
|
|
205
|
+
def _execute_with_limited_butler(
|
|
206
|
+
self,
|
|
207
|
+
task_node: TaskNode,
|
|
208
|
+
limited_butler: LimitedButler,
|
|
209
|
+
/,
|
|
210
|
+
quantum: Quantum,
|
|
211
|
+
quantum_id: uuid.UUID | None = None,
|
|
212
|
+
*,
|
|
213
|
+
log_records: ButlerLogRecords | None = None,
|
|
214
|
+
) -> QuantumExecutionResult:
|
|
215
|
+
startTime = time.time()
|
|
216
|
+
assert quantum.dataId is not None, "Quantum DataId cannot be None"
|
|
217
|
+
report = QuantumReport(quantumId=quantum_id, dataId=quantum.dataId, taskLabel=task_node.label)
|
|
190
218
|
if self._butler is not None:
|
|
191
219
|
log_capture = LogCapture.from_full(self._butler)
|
|
192
220
|
else:
|
|
193
221
|
log_capture = LogCapture.from_limited(limited_butler)
|
|
194
|
-
with log_capture.capture_logging(task_node, quantum) as captureLog:
|
|
222
|
+
with log_capture.capture_logging(task_node, quantum, records=log_records) as captureLog:
|
|
195
223
|
# Save detailed resource usage before task start to metadata.
|
|
196
224
|
quantumMetadata = _TASK_METADATA_TYPE()
|
|
197
225
|
logInfo(None, "prep", metadata=quantumMetadata) # type: ignore[arg-type]
|
|
@@ -210,7 +238,7 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
210
238
|
task_node.label,
|
|
211
239
|
quantum.dataId,
|
|
212
240
|
)
|
|
213
|
-
return quantum
|
|
241
|
+
return QuantumExecutionResult(quantum, report, skipped_existing=True, adjusted_no_work=False)
|
|
214
242
|
captureLog.store = True
|
|
215
243
|
|
|
216
244
|
captureLog.extra.previous_process_quanta.extend(self._previous_process_quanta)
|
|
@@ -238,7 +266,7 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
238
266
|
if self._job_metadata is not None:
|
|
239
267
|
fullMetadata["job"] = self._job_metadata
|
|
240
268
|
self._write_metadata(quantum, fullMetadata, task_node, limited_butler)
|
|
241
|
-
return quantum
|
|
269
|
+
return QuantumExecutionResult(quantum, report, skipped_existing=False, adjusted_no_work=True)
|
|
242
270
|
|
|
243
271
|
# enable lsstDebug debugging
|
|
244
272
|
if self._enable_lsst_debug:
|
|
@@ -260,10 +288,12 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
260
288
|
)
|
|
261
289
|
task = self._task_factory.makeTask(task_node, limited_butler, init_input_refs)
|
|
262
290
|
logInfo(None, "start", metadata=quantumMetadata) # type: ignore[arg-type]
|
|
291
|
+
outputs_put: list[uuid.UUID] = []
|
|
263
292
|
try:
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
293
|
+
with limited_butler.record_metrics() as butler_metrics:
|
|
294
|
+
caveats = self._run_quantum(
|
|
295
|
+
task, quantum, task_node, limited_butler, quantum_id=quantum_id, ids_put=outputs_put
|
|
296
|
+
)
|
|
267
297
|
except Exception as e:
|
|
268
298
|
_LOG.error(
|
|
269
299
|
"Execution of task '%s' on quantum %s failed. Exception %s: %s",
|
|
@@ -283,10 +313,10 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
283
313
|
quantumMetadata["caveats"] = caveats.value
|
|
284
314
|
# Stringify the UUID for easier compatibility with
|
|
285
315
|
# PropertyList.
|
|
286
|
-
quantumMetadata["outputs"] = [str(output) for output in outputsPut]
|
|
287
316
|
finally:
|
|
288
317
|
logInfo(None, "end", metadata=quantumMetadata) # type: ignore[arg-type]
|
|
289
318
|
fullMetadata = task.getFullMetadata()
|
|
319
|
+
quantumMetadata["outputs"] = [str(output) for output in outputs_put]
|
|
290
320
|
fullMetadata["quantum"] = quantumMetadata
|
|
291
321
|
if self._job_metadata is not None:
|
|
292
322
|
fullMetadata["job"] = self._job_metadata
|
|
@@ -299,7 +329,13 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
299
329
|
quantum.dataId,
|
|
300
330
|
stopTime - startTime,
|
|
301
331
|
)
|
|
302
|
-
return
|
|
332
|
+
return QuantumExecutionResult(
|
|
333
|
+
quantum,
|
|
334
|
+
report,
|
|
335
|
+
skipped_existing=False,
|
|
336
|
+
adjusted_no_work=False,
|
|
337
|
+
task_metadata=fullMetadata,
|
|
338
|
+
)
|
|
303
339
|
|
|
304
340
|
def _check_existing_outputs(
|
|
305
341
|
self,
|
|
@@ -501,8 +537,9 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
501
537
|
task_node: TaskNode,
|
|
502
538
|
/,
|
|
503
539
|
limited_butler: LimitedButler,
|
|
504
|
-
quantum_id: uuid.UUID | None
|
|
505
|
-
|
|
540
|
+
quantum_id: uuid.UUID | None,
|
|
541
|
+
ids_put: list[uuid.UUID],
|
|
542
|
+
) -> QuantumSuccessCaveats:
|
|
506
543
|
"""Execute task on a single quantum.
|
|
507
544
|
|
|
508
545
|
Parameters
|
|
@@ -515,18 +552,17 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
515
552
|
Task definition structure.
|
|
516
553
|
limited_butler : `~lsst.daf.butler.LimitedButler`
|
|
517
554
|
Butler to use for dataset I/O.
|
|
518
|
-
quantum_id : `uuid.UUID` or `None
|
|
555
|
+
quantum_id : `uuid.UUID` or `None`
|
|
519
556
|
ID of the quantum being executed.
|
|
557
|
+
ids_put : list[ `uuid.UUID` ]
|
|
558
|
+
List to be populated with the dataset IDs that were written by this
|
|
559
|
+
quantum. This is an output parameter in order to allow it to be
|
|
560
|
+
populated even when an exception is raised.
|
|
520
561
|
|
|
521
562
|
Returns
|
|
522
563
|
-------
|
|
523
564
|
flags : `QuantumSuccessCaveats`
|
|
524
565
|
Flags that describe qualified successes.
|
|
525
|
-
ids_put : list[ `uuid.UUID` ]
|
|
526
|
-
Record of all the dataset IDs that were written by this quantum
|
|
527
|
-
being executed.
|
|
528
|
-
metrics : `lsst.daf.butler.ButlerMetrics`
|
|
529
|
-
Butler metrics recorded for this quantum.
|
|
530
566
|
"""
|
|
531
567
|
flags = QuantumSuccessCaveats.NO_CAVEATS
|
|
532
568
|
|
|
@@ -538,8 +574,7 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
538
574
|
|
|
539
575
|
# Call task runQuantum() method.
|
|
540
576
|
try:
|
|
541
|
-
|
|
542
|
-
task.runQuantum(butlerQC, inputRefs, outputRefs)
|
|
577
|
+
task.runQuantum(butlerQC, inputRefs, outputRefs)
|
|
543
578
|
except NoWorkFound as err:
|
|
544
579
|
# Not an error, just an early exit.
|
|
545
580
|
_LOG.info(
|
|
@@ -577,12 +612,13 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
577
612
|
)
|
|
578
613
|
_LOG.error(error, exc_info=error)
|
|
579
614
|
flags |= caught.FLAGS
|
|
615
|
+
finally:
|
|
616
|
+
ids_put.extend(output[2] for output in butlerQC.outputsPut)
|
|
580
617
|
if not butlerQC.outputsPut:
|
|
581
618
|
flags |= QuantumSuccessCaveats.ALL_OUTPUTS_MISSING
|
|
582
619
|
if not butlerQC.outputsPut == butlerQC.allOutputs:
|
|
583
620
|
flags |= QuantumSuccessCaveats.ANY_OUTPUTS_MISSING
|
|
584
|
-
|
|
585
|
-
return flags, ids_put, butler_metrics
|
|
621
|
+
return flags
|
|
586
622
|
|
|
587
623
|
def _write_metadata(
|
|
588
624
|
self, quantum: Quantum, metadata: Any, task_node: TaskNode, /, limited_butler: LimitedButler
|
|
@@ -29,11 +29,12 @@ from __future__ import annotations
|
|
|
29
29
|
|
|
30
30
|
__all__ = ("DirectButlerRepo", "InMemoryRepo", "MockRepo")
|
|
31
31
|
|
|
32
|
+
import logging
|
|
32
33
|
import tempfile
|
|
33
|
-
from abc import
|
|
34
|
+
from abc import abstractmethod
|
|
34
35
|
from collections.abc import Iterable, Iterator, Mapping
|
|
35
|
-
from contextlib import contextmanager
|
|
36
|
-
from typing import Any
|
|
36
|
+
from contextlib import AbstractContextManager, contextmanager
|
|
37
|
+
from typing import Any, Literal, Self
|
|
37
38
|
|
|
38
39
|
from lsst.daf.butler import (
|
|
39
40
|
Butler,
|
|
@@ -61,8 +62,10 @@ from ._pipeline_task import (
|
|
|
61
62
|
)
|
|
62
63
|
from ._storage_class import MockDataset, is_mock_name
|
|
63
64
|
|
|
65
|
+
_LOG = logging.getLogger(__name__)
|
|
64
66
|
|
|
65
|
-
|
|
67
|
+
|
|
68
|
+
class MockRepo(AbstractContextManager):
|
|
66
69
|
"""A test helper that populates a butler repository for task execution.
|
|
67
70
|
|
|
68
71
|
Parameters
|
|
@@ -100,6 +103,31 @@ class MockRepo(ABC):
|
|
|
100
103
|
self.last_auto_dataset_type_index = 0
|
|
101
104
|
self.last_auto_task_index = 0
|
|
102
105
|
|
|
106
|
+
def __enter__(self) -> Self:
|
|
107
|
+
return self
|
|
108
|
+
|
|
109
|
+
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Literal[False]:
|
|
110
|
+
try:
|
|
111
|
+
self.close()
|
|
112
|
+
except Exception:
|
|
113
|
+
_LOG.exception("An exception occurred during MockRepo.close()")
|
|
114
|
+
return False
|
|
115
|
+
|
|
116
|
+
def close(self) -> None:
|
|
117
|
+
"""Release all resources associated with this mock instance. The
|
|
118
|
+
instance may no longer be used after this is called.
|
|
119
|
+
|
|
120
|
+
Notes
|
|
121
|
+
-----
|
|
122
|
+
Instead of calling ``close()`` directly, you can use the mock object
|
|
123
|
+
as a context manager. For example::
|
|
124
|
+
|
|
125
|
+
with MockRepo(...) as butler:
|
|
126
|
+
butler.get(...)
|
|
127
|
+
# butler is closed after exiting the block.
|
|
128
|
+
"""
|
|
129
|
+
self.butler.close()
|
|
130
|
+
|
|
103
131
|
def add_task(
|
|
104
132
|
self,
|
|
105
133
|
label: str | None = None,
|
|
@@ -581,18 +609,18 @@ class DirectButlerRepo(MockRepo):
|
|
|
581
609
|
) -> Iterator[tuple[DirectButlerRepo, str]]:
|
|
582
610
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as root:
|
|
583
611
|
config = Butler.makeRepo(root, **kwargs)
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
612
|
+
with Butler.from_config(config, writeable=True) as butler:
|
|
613
|
+
yield (
|
|
614
|
+
cls(
|
|
615
|
+
butler,
|
|
616
|
+
*args,
|
|
617
|
+
input_run=input_run,
|
|
618
|
+
input_chain=input_chain,
|
|
619
|
+
use_import_collections_as_input=use_import_collections_as_input,
|
|
620
|
+
data_root=data_root,
|
|
621
|
+
),
|
|
622
|
+
root,
|
|
623
|
+
)
|
|
596
624
|
|
|
597
625
|
def _insert_datasets_impl(
|
|
598
626
|
self, dataset_type: DatasetType, data_ids: list[DataCoordinate]
|
|
@@ -41,6 +41,7 @@ import lsst.daf.butler.tests as butlerTests
|
|
|
41
41
|
import lsst.pex.config as pexConfig
|
|
42
42
|
from lsst.daf.butler import Butler, Config, DataId, DatasetRef, DatasetType, Formatter, LimitedButler
|
|
43
43
|
from lsst.daf.butler.logging import ButlerLogRecords
|
|
44
|
+
from lsst.daf.butler.registry import RegistryDefaults
|
|
44
45
|
from lsst.resources import ResourcePath
|
|
45
46
|
from lsst.utils import doImportType
|
|
46
47
|
from lsst.utils.introspection import get_full_type_name
|
|
@@ -295,8 +296,8 @@ def makeSimpleButler(
|
|
|
295
296
|
if not inMemory:
|
|
296
297
|
butler_config["registry", "db"] = f"sqlite:///{root_path.ospath}/gen3.sqlite"
|
|
297
298
|
butler_config["datastore", "cls"] = "lsst.daf.butler.datastores.fileDatastore.FileDatastore"
|
|
298
|
-
|
|
299
|
-
butler =
|
|
299
|
+
butler = butlerTests.makeTestRepo(str(root_path), {}, config=butler_config)
|
|
300
|
+
butler.registry.defaults = RegistryDefaults(run=run)
|
|
300
301
|
return butler
|
|
301
302
|
|
|
302
303
|
|
|
@@ -480,44 +481,51 @@ def makeSimpleQGraph(
|
|
|
480
481
|
case _:
|
|
481
482
|
raise TypeError(f"Unexpected pipeline object: {pipeline!r}.")
|
|
482
483
|
|
|
484
|
+
butler_created = False
|
|
483
485
|
if butler is None:
|
|
484
486
|
if root is None:
|
|
485
487
|
raise ValueError("Must provide `root` when `butler` is None")
|
|
486
488
|
if callPopulateButler is False:
|
|
487
489
|
raise ValueError("populateButler can only be False when butler is supplied as an argument")
|
|
488
490
|
butler = makeSimpleButler(root, run=run, inMemory=inMemory)
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
491
|
+
butler_created = True
|
|
492
|
+
|
|
493
|
+
try:
|
|
494
|
+
if callPopulateButler:
|
|
495
|
+
populateButler(pipeline_graph, butler, datasetTypes=datasetTypes, instrument=instrument)
|
|
496
|
+
|
|
497
|
+
# Make the graph
|
|
498
|
+
_LOG.debug(
|
|
499
|
+
"Instantiating QuantumGraphBuilder, "
|
|
500
|
+
"skip_existing_in=%s, input_collections=%r, output_run=%r, where=%r, bind=%s.",
|
|
501
|
+
skipExistingIn,
|
|
502
|
+
butler.collections.defaults,
|
|
503
|
+
run,
|
|
504
|
+
userQuery,
|
|
505
|
+
bind,
|
|
506
|
+
)
|
|
507
|
+
if not run:
|
|
508
|
+
assert butler.run is not None, "Butler must have run defined"
|
|
509
|
+
run = butler.run
|
|
510
|
+
builder = AllDimensionsQuantumGraphBuilder(
|
|
511
|
+
pipeline_graph,
|
|
512
|
+
butler,
|
|
513
|
+
skip_existing_in=skipExistingIn if skipExistingIn is not None else [],
|
|
514
|
+
input_collections=butler.collections.defaults,
|
|
515
|
+
output_run=run,
|
|
516
|
+
where=userQuery,
|
|
517
|
+
bind=bind,
|
|
518
|
+
dataset_query_constraint=datasetQueryConstraint,
|
|
519
|
+
)
|
|
520
|
+
_LOG.debug("Calling QuantumGraphBuilder.build.")
|
|
521
|
+
if not metadata:
|
|
522
|
+
metadata = {}
|
|
523
|
+
metadata["output_run"] = run
|
|
524
|
+
|
|
525
|
+
qgraph = builder.build(metadata=metadata, attach_datastore_records=makeDatastoreRecords)
|
|
526
|
+
except Exception:
|
|
527
|
+
if butler_created:
|
|
528
|
+
butler.close()
|
|
529
|
+
raise
|
|
522
530
|
|
|
523
531
|
return butler, qgraph
|
lsst/pipe/base/version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
__all__ = ["__version__"]
|
|
2
|
-
__version__ = "
|
|
2
|
+
__version__ = "30.0.0"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-pipe-base
|
|
3
|
-
Version:
|
|
3
|
+
Version: 30.0.0
|
|
4
4
|
Summary: Pipeline infrastructure for the Rubin Science Pipelines.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License-Expression: BSD-3-Clause OR GPL-3.0-or-later
|
|
@@ -3,14 +3,14 @@ lsst/pipe/__init__.py,sha256=_2bZAHuDVAx7MM7KA7pt3DYp641NY4RzSoRAwesWKfU,67
|
|
|
3
3
|
lsst/pipe/base/__init__.py,sha256=qBLN0yYQjIcLBLb4jFKM_ppopuqTnCehcUdFcEe69Js,970
|
|
4
4
|
lsst/pipe/base/_datasetQueryConstraints.py,sha256=bFH0_lVc49NS2_4v_i6r9POr500c0K-OHLMhMX5FjkQ,6373
|
|
5
5
|
lsst/pipe/base/_dataset_handle.py,sha256=ft_ke1LbhLLndDPARsHSQJUA05LgUFnfWOq2vbwH3wI,11353
|
|
6
|
-
lsst/pipe/base/_instrument.py,sha256=
|
|
6
|
+
lsst/pipe/base/_instrument.py,sha256=yZTTgEPmGcEKG4WnAkfWGM1K5f3PxALcQRFqrtVp57A,30215
|
|
7
7
|
lsst/pipe/base/_observation_dimension_packer.py,sha256=78Jg2OVFOdXIK62TS2Y3X4095xqCzmiIx9o4TXyADYA,8027
|
|
8
8
|
lsst/pipe/base/_quantumContext.py,sha256=gb60mTHbgOIEptYvJ64SaChvViXyeKJlG6kEHq4nYVw,19345
|
|
9
9
|
lsst/pipe/base/_status.py,sha256=7gJPrqt03t1fO5tREc_sfQaN7XmDBvruihORXPnRqUE,21216
|
|
10
10
|
lsst/pipe/base/_task_metadata.py,sha256=Y4rjrYWvYxYCJwy86VvzxKMkNxEJbYVgLVuXo6KiXac,25638
|
|
11
11
|
lsst/pipe/base/all_dimensions_quantum_graph_builder.py,sha256=nazY74jrdSCr6CFfPp78JecM_-udW95EYP7grLPO2hg,70830
|
|
12
12
|
lsst/pipe/base/automatic_connection_constants.py,sha256=H5uuh1rYRpjndgPdb0dh1L_-OyLKdT6VWOZTAb__xCU,3298
|
|
13
|
-
lsst/pipe/base/caching_limited_butler.py,sha256=
|
|
13
|
+
lsst/pipe/base/caching_limited_butler.py,sha256=FEQK_QesUWG9sK0w9m6_SA5OPcrd6vqyq8oDKE9Wu6I,7858
|
|
14
14
|
lsst/pipe/base/config.py,sha256=yNipVEc6awwhU_O9I01g20OnvQrs28dAwkXuI1hrlYE,11982
|
|
15
15
|
lsst/pipe/base/configOverrides.py,sha256=B0An8EaX76VzWnC5dJxvyZ2AhVzawMtq7qlE9ma5lkc,14661
|
|
16
16
|
lsst/pipe/base/connectionTypes.py,sha256=inUDyzbM1sKMCtHaRkhx3dWSPHPBIDVMHOPhzB13Kdw,16720
|
|
@@ -20,29 +20,30 @@ lsst/pipe/base/exec_fixup_data_id.py,sha256=9OjOcH-6AHZ1JnD_CemieI0wWX90J_VdaY9v
|
|
|
20
20
|
lsst/pipe/base/execution_graph_fixup.py,sha256=ND0x4hlpeEW-gudo-i2K7HT7MoM5sp_mcoqRMCopSqQ,3815
|
|
21
21
|
lsst/pipe/base/execution_reports.py,sha256=jYtWCD4PkEAeVUpKIxuiJJVgsCm7qiwCorWVgNHkVgU,17270
|
|
22
22
|
lsst/pipe/base/graph_walker.py,sha256=Ij7JfYF0srA29VgM_DhbNBxUBeOHDOnujrTQPjVNha0,4694
|
|
23
|
-
lsst/pipe/base/log_capture.py,sha256=
|
|
23
|
+
lsst/pipe/base/log_capture.py,sha256=6fq1L53FUYsbILX-tQixXUhxQjvKxFGDjYELRDAXnsw,11072
|
|
24
|
+
lsst/pipe/base/log_on_close.py,sha256=rauQYvsWQF9GuxoaZRjPJJ7VjATGyRUo0RyHEoUVTnQ,2666
|
|
24
25
|
lsst/pipe/base/mermaid_tools.py,sha256=cdlDJQ1x8k7-VvCLEUqvSC3GR1zCsB-aUTxOjYejNWc,5216
|
|
25
|
-
lsst/pipe/base/mp_graph_executor.py,sha256=
|
|
26
|
+
lsst/pipe/base/mp_graph_executor.py,sha256=bbsJD0i-WpW1Qc6lAvagIIx__jZfeOqN8dyt3IsyEq0,37350
|
|
26
27
|
lsst/pipe/base/pipeline.py,sha256=FVaiLhgw9Pzo-nzXKS0dLNafegP0AMZKLtPlSvOSkRU,37563
|
|
27
28
|
lsst/pipe/base/pipelineIR.py,sha256=UuZ02NLhVmzzekbuWlyar7cPLCf_4yfzD5qFEmGHs_A,45821
|
|
28
29
|
lsst/pipe/base/pipelineTask.py,sha256=K3GdjJLvy8A7I-jzQiERQZaYF7mC1LM3iB5TmUtbOCI,8394
|
|
29
30
|
lsst/pipe/base/prerequisite_helpers.py,sha256=bmiebQ4veSrypZgAXjmCBFfj8fUtPW9eRQaVShhxdBQ,28446
|
|
30
31
|
lsst/pipe/base/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
31
|
-
lsst/pipe/base/quantum_graph_builder.py,sha256=
|
|
32
|
-
lsst/pipe/base/quantum_graph_executor.py,sha256
|
|
33
|
-
lsst/pipe/base/quantum_graph_skeleton.py,sha256=
|
|
32
|
+
lsst/pipe/base/quantum_graph_builder.py,sha256=YM3pvt4ignAZnpO9bpBc_Yl8V-y2r3SQmYJdWnyQQNI,68742
|
|
33
|
+
lsst/pipe/base/quantum_graph_executor.py,sha256=-bbeR9wZ5_Etcqpx6cDS_R-oEvfUVv9gmW0Wc4QuQNc,8127
|
|
34
|
+
lsst/pipe/base/quantum_graph_skeleton.py,sha256=hxSbeNhdCSBO2aqdIMRyQaejiUxKQfE6bSn0S0qWBNo,28395
|
|
34
35
|
lsst/pipe/base/quantum_provenance_graph.py,sha256=33S5iCVxD9Co4oJSU_N8AJXL14Nw0UwGzPEc3gpQiqk,91981
|
|
35
36
|
lsst/pipe/base/quantum_reports.py,sha256=ut235L88v7SXaeVUvMA9qFl7tpeMwGnzob3X0QoOI_s,14210
|
|
36
37
|
lsst/pipe/base/resource_usage.py,sha256=LfH7Qf6taI3lxw0aB90riRMn1UxUTMBSqtBjKPJ-XuY,6759
|
|
37
|
-
lsst/pipe/base/separable_pipeline_executor.py,sha256=
|
|
38
|
-
lsst/pipe/base/simple_pipeline_executor.py,sha256=
|
|
39
|
-
lsst/pipe/base/single_quantum_executor.py,sha256=
|
|
38
|
+
lsst/pipe/base/separable_pipeline_executor.py,sha256=y3ZnQJApocxusHGG1R5Rnpke3B0vren1h4dcuyZLivs,17717
|
|
39
|
+
lsst/pipe/base/simple_pipeline_executor.py,sha256=_HGeAOgHUiRdQwsmD_qira-vcHZXlLbf_LmhLJffh2U,29563
|
|
40
|
+
lsst/pipe/base/single_quantum_executor.py,sha256=BgmRtouSIKMeh44hbihdyWsYG4sH1fvoN1PoyVNjxUI,29283
|
|
40
41
|
lsst/pipe/base/struct.py,sha256=Fa-UkpuXOxdzKWbHrMUkJYOszZuBXCm2NesXNR0IOPQ,5048
|
|
41
42
|
lsst/pipe/base/task.py,sha256=XHBd-7m1a4-6LgobBYA1DgY4H7EV-_RWKfxbhZbMmD4,15145
|
|
42
43
|
lsst/pipe/base/taskFactory.py,sha256=MsDGECJqZLSZk8SGhpuVhNaP32UWuNvxZiDcZExPFG8,3412
|
|
43
44
|
lsst/pipe/base/testUtils.py,sha256=lSBKMhoKflbi8JkMNYfEqqHNl-rtFI8UYT3QneDYpLo,18477
|
|
44
45
|
lsst/pipe/base/utils.py,sha256=JmEt3l0xrh9uayKrSXuQEq12aXOhDr2YXmbYduaxCko,1940
|
|
45
|
-
lsst/pipe/base/version.py,sha256
|
|
46
|
+
lsst/pipe/base/version.py,sha256=--s7nTlXrHcUGZfOJGTF8_DCdIUv7QPWWD7as9kq0iY,49
|
|
46
47
|
lsst/pipe/base/cli/__init__.py,sha256=861tXIAW7SqtqNUYkjbeEdfg8lDswXsjJQca0gVCFz4,54
|
|
47
48
|
lsst/pipe/base/cli/_get_cli_subcommands.py,sha256=g_af64klRybBGKAg7fmBSZBdw2LYBAsFON_yQIMZON0,1289
|
|
48
49
|
lsst/pipe/base/cli/cmd/__init__.py,sha256=3UF2IQEEBor4YMGRNPdcZAVCAI5yFyeHp5nGul4IoyM,1557
|
|
@@ -82,44 +83,45 @@ lsst/pipe/base/pipeline_graph/visualization/_printer.py,sha256=yJMRJ-aXd3nYDgs1F
|
|
|
82
83
|
lsst/pipe/base/pipeline_graph/visualization/_show.py,sha256=lPRjO1To2n5r3f_Wgcwy-7TmyJ7UszGGFXAlOtN1wDs,10510
|
|
83
84
|
lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py,sha256=dp7PXl9Cu7GfWjBi5g8KjXZgnF1KGg_idKKxtICL53Q,8679
|
|
84
85
|
lsst/pipe/base/quantum_graph/__init__.py,sha256=-Gp3LihB0AXCvhG387wKAEpHRM-NrHGSXMti8cHee90,1437
|
|
85
|
-
lsst/pipe/base/quantum_graph/_common.py,sha256=
|
|
86
|
-
lsst/pipe/base/quantum_graph/_multiblock.py,sha256=
|
|
87
|
-
lsst/pipe/base/quantum_graph/_predicted.py,sha256=
|
|
88
|
-
lsst/pipe/base/quantum_graph/_provenance.py,sha256=
|
|
86
|
+
lsst/pipe/base/quantum_graph/_common.py,sha256=jBNmuVK0RaD6nFNk2cmRQY5ej0v_wADRREcAPlhneNs,22841
|
|
87
|
+
lsst/pipe/base/quantum_graph/_multiblock.py,sha256=lw--3Ol-nmQPkC5gzC95xUTkGXOVAfPoa1bb21cFHXE,27412
|
|
88
|
+
lsst/pipe/base/quantum_graph/_predicted.py,sha256=vUbKXfIPLtH4YZmgAlxJBwCqwMJ9X-2dSFh35vgvntg,91216
|
|
89
|
+
lsst/pipe/base/quantum_graph/_provenance.py,sha256=eGCs2K9UYg5bILEPY6x6xsoA1x-mTTgzn5opFq1bL6k,84207
|
|
90
|
+
lsst/pipe/base/quantum_graph/formatter.py,sha256=PZhn6I7HAdBjhm-_vcWoXdhwm9hDsK5QQ07GnIGJtPU,3948
|
|
89
91
|
lsst/pipe/base/quantum_graph/visualization.py,sha256=EbTWhk9aPq7sX6bcHmnEIsr2xuuR6d1SxspQbRe8D0Q,12235
|
|
90
92
|
lsst/pipe/base/quantum_graph/aggregator/__init__.py,sha256=4CK8sP_ZjUKmxKS3LnCH1zG7XSk9IEwijrluRBHhEMU,7436
|
|
91
|
-
lsst/pipe/base/quantum_graph/aggregator/_communicators.py,sha256=
|
|
93
|
+
lsst/pipe/base/quantum_graph/aggregator/_communicators.py,sha256=deLXCdKlSM2GXGW4LdtfzOS7aa3Yd9QVajtyjJE4u3s,35197
|
|
92
94
|
lsst/pipe/base/quantum_graph/aggregator/_config.py,sha256=iV1Ejfk-UnFoQ8TkpJE_jMZYHsmZcdLm5R-FnQEqO7s,5167
|
|
93
|
-
lsst/pipe/base/quantum_graph/aggregator/_ingester.py,sha256=
|
|
95
|
+
lsst/pipe/base/quantum_graph/aggregator/_ingester.py,sha256=lhu8I0R3IwoLkVcrqF1ypoHAFamNapSoMcG9aYFjbQ4,14158
|
|
94
96
|
lsst/pipe/base/quantum_graph/aggregator/_progress.py,sha256=jiz9Np73uUQ03CtH7pI6TXxXrwFUChh5hSj_gbMGHr0,7207
|
|
95
|
-
lsst/pipe/base/quantum_graph/aggregator/_scanner.py,sha256=
|
|
96
|
-
lsst/pipe/base/quantum_graph/aggregator/_structs.py,sha256=
|
|
97
|
-
lsst/pipe/base/quantum_graph/aggregator/_supervisor.py,sha256=
|
|
98
|
-
lsst/pipe/base/quantum_graph/aggregator/_writer.py,sha256=
|
|
97
|
+
lsst/pipe/base/quantum_graph/aggregator/_scanner.py,sha256=HxW8F4smuV3-0g4SFwlQ3SKrktpPDcRkvckz8oVV-s0,12872
|
|
98
|
+
lsst/pipe/base/quantum_graph/aggregator/_structs.py,sha256=tkFh6-XaEoXGqfMH9RQq8rZqXXco1jUtiYCJKPsGus8,2401
|
|
99
|
+
lsst/pipe/base/quantum_graph/aggregator/_supervisor.py,sha256=NuUQiC-ZtLQkK_sJ1bktRiRv4ObdwjeE6vOJLjDrepc,9559
|
|
100
|
+
lsst/pipe/base/quantum_graph/aggregator/_writer.py,sha256=hd3zALTm9uTicQyPukN34fjcUJ3TIogf_EONy5nsywU,7917
|
|
99
101
|
lsst/pipe/base/script/__init__.py,sha256=cLEXE7aq5UZ0juL_ScmRw0weFgp4tDgwEX_ts-NEYic,1522
|
|
100
|
-
lsst/pipe/base/script/register_instrument.py,sha256=
|
|
101
|
-
lsst/pipe/base/script/retrieve_artifacts_for_quanta.py,sha256=
|
|
102
|
-
lsst/pipe/base/script/transfer_from_graph.py,sha256=
|
|
102
|
+
lsst/pipe/base/script/register_instrument.py,sha256=neQ2MTPtAiV_Hl2yatQ8-vQC24xHjhpI7VJUHf5kPX4,2445
|
|
103
|
+
lsst/pipe/base/script/retrieve_artifacts_for_quanta.py,sha256=Cr0HpzXm_C3LnIOQg5tNJht02O6xoqtWWIphjugasMA,3957
|
|
104
|
+
lsst/pipe/base/script/transfer_from_graph.py,sha256=NelxNU_aNEwPHDZnDGXA07oeuSkeXqEZ6KG0ax6VE8c,7254
|
|
103
105
|
lsst/pipe/base/script/utils.py,sha256=zNqpHG3kXA8OaNXnwYIo0Hu_LCie1qoBAARAME3WEjs,3739
|
|
104
|
-
lsst/pipe/base/script/zip_from_graph.py,sha256=
|
|
106
|
+
lsst/pipe/base/script/zip_from_graph.py,sha256=dMyyTGzEBCMMmR6Ts8R7QXUJrbtdrac_i-iy87lFDSI,3242
|
|
105
107
|
lsst/pipe/base/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
106
108
|
lsst/pipe/base/tests/in_memory_limited_butler.py,sha256=UzLh416H67nCUhD9y3cniAAjY7VojvhOLjF3gHHgjA4,8679
|
|
107
109
|
lsst/pipe/base/tests/no_dimensions.py,sha256=58UpyRN8cLAMZtkOmjTm3dJZyRFRekotQ-7-OgEfiAI,4710
|
|
108
110
|
lsst/pipe/base/tests/pipelineStepTester.py,sha256=KGxdB8gdVpSey2RUGURDIzIfPL-4qvQCsBpMrhG4Z2M,7208
|
|
109
|
-
lsst/pipe/base/tests/simpleQGraph.py,sha256=
|
|
111
|
+
lsst/pipe/base/tests/simpleQGraph.py,sha256=V_h1SifQ27vnjz5Fax1KZYyn6yPUDNfc8gRjc7SFbhE,20197
|
|
110
112
|
lsst/pipe/base/tests/util.py,sha256=IXpZOC58fdRnurB5lPcNX-xRgKEV-cPNkWKJDFIr1gs,4772
|
|
111
113
|
lsst/pipe/base/tests/mocks/__init__.py,sha256=fDy9H9vRAIBpKDJEXNZuDWJMzWZfpcBT4TmyOw4o-RY,1572
|
|
112
114
|
lsst/pipe/base/tests/mocks/_data_id_match.py,sha256=jVekStcrItC0tqOCc01VjYaiE9exYm3MRkwB0Gh_3J0,7465
|
|
113
115
|
lsst/pipe/base/tests/mocks/_pipeline_task.py,sha256=N3fC4OMAMWWnYtyLkVdMfb9ZiFse39HniRDvlAOofOY,30691
|
|
114
|
-
lsst/pipe/base/tests/mocks/_repo.py,sha256=
|
|
116
|
+
lsst/pipe/base/tests/mocks/_repo.py,sha256=SH-jzynS-H2xc_3GLjF7ln-kHdRoSeVVaal5qLd2hXI,28359
|
|
115
117
|
lsst/pipe/base/tests/mocks/_storage_class.py,sha256=12IFfJMbZ5GkYlMX6ZMWiG8pMZc2Jlxke3qQW-bljdU,27434
|
|
116
|
-
lsst_pipe_base-
|
|
117
|
-
lsst_pipe_base-
|
|
118
|
-
lsst_pipe_base-
|
|
119
|
-
lsst_pipe_base-
|
|
120
|
-
lsst_pipe_base-
|
|
121
|
-
lsst_pipe_base-
|
|
122
|
-
lsst_pipe_base-
|
|
123
|
-
lsst_pipe_base-
|
|
124
|
-
lsst_pipe_base-
|
|
125
|
-
lsst_pipe_base-
|
|
118
|
+
lsst_pipe_base-30.0.0.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
|
|
119
|
+
lsst_pipe_base-30.0.0.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
|
|
120
|
+
lsst_pipe_base-30.0.0.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
|
|
121
|
+
lsst_pipe_base-30.0.0.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
122
|
+
lsst_pipe_base-30.0.0.dist-info/METADATA,sha256=fcVHOliPfrtjtF4Ppca6NlxzcZlOM9XJLRmzRyTJhpE,2251
|
|
123
|
+
lsst_pipe_base-30.0.0.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
|
|
124
|
+
lsst_pipe_base-30.0.0.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
|
|
125
|
+
lsst_pipe_base-30.0.0.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
|
|
126
|
+
lsst_pipe_base-30.0.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
127
|
+
lsst_pipe_base-30.0.0.dist-info/RECORD,,
|
|
File without changes
|
{lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/licenses/COPYRIGHT
RENAMED
|
File without changes
|
|
File without changes
|
{lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/licenses/bsd_license.txt
RENAMED
|
File without changes
|
{lsst_pipe_base-29.2025.4800.dist-info → lsst_pipe_base-30.0.0.dist-info}/licenses/gpl-v3.0.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|