lsst-pipe-base 30.2026.300__py3-none-any.whl → 30.2026.400__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/pipe/base/_instrument.py +4 -7
- lsst/pipe/base/_status.py +29 -10
- lsst/pipe/base/automatic_connection_constants.py +9 -1
- lsst/pipe/base/cli/cmd/__init__.py +16 -2
- lsst/pipe/base/cli/cmd/commands.py +42 -4
- lsst/pipe/base/connectionTypes.py +72 -160
- lsst/pipe/base/connections.py +3 -6
- lsst/pipe/base/execution_reports.py +0 -5
- lsst/pipe/base/pipeline.py +3 -4
- lsst/pipe/base/pipelineIR.py +0 -6
- lsst/pipe/base/pipelineTask.py +5 -7
- lsst/pipe/base/pipeline_graph/_edges.py +19 -7
- lsst/pipe/base/pipeline_graph/_pipeline_graph.py +8 -0
- lsst/pipe/base/quantum_graph/_common.py +3 -1
- lsst/pipe/base/quantum_graph/_predicted.py +7 -0
- lsst/pipe/base/quantum_graph/_provenance.py +87 -37
- lsst/pipe/base/quantum_graph/aggregator/_communicators.py +9 -1
- lsst/pipe/base/quantum_graph/aggregator/_config.py +78 -9
- lsst/pipe/base/quantum_graph/aggregator/_ingester.py +12 -11
- lsst/pipe/base/quantum_graph/aggregator/_scanner.py +14 -6
- lsst/pipe/base/quantum_graph/aggregator/_structs.py +3 -3
- lsst/pipe/base/quantum_graph/aggregator/_supervisor.py +14 -13
- lsst/pipe/base/quantum_graph/aggregator/_writer.py +2 -2
- lsst/pipe/base/quantum_graph/formatter.py +70 -0
- lsst/pipe/base/quantum_graph/ingest_graph.py +356 -0
- lsst/pipe/base/quantum_provenance_graph.py +17 -2
- lsst/pipe/base/separable_pipeline_executor.py +5 -6
- lsst/pipe/base/single_quantum_executor.py +6 -6
- lsst/pipe/base/struct.py +4 -0
- lsst/pipe/base/version.py +1 -1
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.400.dist-info}/METADATA +2 -1
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.400.dist-info}/RECORD +40 -39
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.400.dist-info}/WHEEL +1 -1
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.400.dist-info}/entry_points.txt +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.400.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.400.dist-info}/licenses/LICENSE +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.400.dist-info}/licenses/bsd_license.txt +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.400.dist-info}/licenses/gpl-v3.0.txt +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.400.dist-info}/top_level.txt +0 -0
- {lsst_pipe_base-30.2026.300.dist-info → lsst_pipe_base-30.2026.400.dist-info}/zip-safe +0 -0
|
@@ -61,7 +61,7 @@ class Writer:
|
|
|
61
61
|
"""
|
|
62
62
|
|
|
63
63
|
def __post_init__(self) -> None:
|
|
64
|
-
assert self.comms.config.
|
|
64
|
+
assert self.comms.config.is_writing_provenance, "Writer should not be used if writing is disabled."
|
|
65
65
|
self.comms.log.info("Reading predicted quantum graph.")
|
|
66
66
|
with PredictedQuantumGraphReader.open(
|
|
67
67
|
self.predicted_path, import_mode=TaskImportMode.DO_NOT_IMPORT
|
|
@@ -123,7 +123,7 @@ class Writer:
|
|
|
123
123
|
"""
|
|
124
124
|
cdict = self.make_compression_dictionary()
|
|
125
125
|
self.comms.send_compression_dict(cdict.as_bytes())
|
|
126
|
-
assert self.comms.config.output_path is not None
|
|
126
|
+
assert self.comms.config.is_writing_provenance and self.comms.config.output_path is not None
|
|
127
127
|
self.comms.log.info("Opening output files and processing predicted graph.")
|
|
128
128
|
qg_writer = ProvenanceQuantumGraphWriter(
|
|
129
129
|
self.comms.config.output_path,
|
|
@@ -35,9 +35,13 @@ from typing import Any, ClassVar
|
|
|
35
35
|
import pydantic
|
|
36
36
|
|
|
37
37
|
from lsst.daf.butler import FormatterV2
|
|
38
|
+
from lsst.daf.butler.logging import ButlerLogRecords
|
|
39
|
+
from lsst.pex.config import Config
|
|
38
40
|
from lsst.resources import ResourcePath
|
|
39
41
|
from lsst.utils.logging import getLogger
|
|
42
|
+
from lsst.utils.packages import Packages
|
|
40
43
|
|
|
44
|
+
from .._task_metadata import TaskMetadata
|
|
41
45
|
from ..pipeline_graph import TaskImportMode
|
|
42
46
|
from ._provenance import ProvenanceQuantumGraphReader
|
|
43
47
|
|
|
@@ -83,6 +87,17 @@ class ProvenanceFormatter(FormatterV2):
|
|
|
83
87
|
can_read_from_uri: ClassVar[bool] = True
|
|
84
88
|
|
|
85
89
|
def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
|
|
90
|
+
match self._dataset_ref.datasetType.storageClass_name:
|
|
91
|
+
case "TaskMetadata" | "PropertySet":
|
|
92
|
+
return self._read_metadata(uri)
|
|
93
|
+
case "ButlerLogRecords":
|
|
94
|
+
return self._read_log(uri)
|
|
95
|
+
case "Config":
|
|
96
|
+
return self._read_config(uri)
|
|
97
|
+
case "ProvenanceQuantumGraph":
|
|
98
|
+
pass
|
|
99
|
+
case unexpected:
|
|
100
|
+
raise ValueError(f"Unsupported storage class {unexpected!r} for ProvenanceFormatter.")
|
|
86
101
|
parameters = _ProvenanceFormatterParameters.model_validate(self.file_descriptor.parameters or {})
|
|
87
102
|
with ProvenanceQuantumGraphReader.open(uri, import_mode=parameters.import_mode) as reader:
|
|
88
103
|
match component:
|
|
@@ -99,3 +114,58 @@ class ProvenanceFormatter(FormatterV2):
|
|
|
99
114
|
case "packages":
|
|
100
115
|
return reader.fetch_packages()
|
|
101
116
|
raise AssertionError(f"Unexpected component {component!r}.")
|
|
117
|
+
|
|
118
|
+
def _read_metadata(self, uri: ResourcePath) -> TaskMetadata:
|
|
119
|
+
with ProvenanceQuantumGraphReader.open(uri, import_mode=TaskImportMode.DO_NOT_IMPORT) as reader:
|
|
120
|
+
try:
|
|
121
|
+
attempts = reader.fetch_metadata([self._dataset_ref.id])[self._dataset_ref.id]
|
|
122
|
+
except LookupError:
|
|
123
|
+
raise FileNotFoundError(
|
|
124
|
+
f"No dataset with ID {self._dataset_ref.id} present in this graph."
|
|
125
|
+
) from None
|
|
126
|
+
if not attempts:
|
|
127
|
+
raise FileNotFoundError(
|
|
128
|
+
f"No metadata dataset {self._dataset_ref} stored in this graph "
|
|
129
|
+
"(no attempts for this quantum)."
|
|
130
|
+
)
|
|
131
|
+
if attempts[-1] is None:
|
|
132
|
+
raise FileNotFoundError(
|
|
133
|
+
f"No metadata dataset {self._dataset_ref} stored in this graph "
|
|
134
|
+
"(most recent attempt failed and did not write metadata)."
|
|
135
|
+
)
|
|
136
|
+
return attempts[-1]
|
|
137
|
+
|
|
138
|
+
def _read_log(self, uri: ResourcePath) -> ButlerLogRecords:
|
|
139
|
+
with ProvenanceQuantumGraphReader.open(uri, import_mode=TaskImportMode.DO_NOT_IMPORT) as reader:
|
|
140
|
+
try:
|
|
141
|
+
attempts = reader.fetch_logs([self._dataset_ref.id])[self._dataset_ref.id]
|
|
142
|
+
except LookupError:
|
|
143
|
+
raise FileNotFoundError(
|
|
144
|
+
f"No dataset with ID {self._dataset_ref.id} present in this graph."
|
|
145
|
+
) from None
|
|
146
|
+
if not attempts:
|
|
147
|
+
raise FileNotFoundError(
|
|
148
|
+
f"No log dataset {self._dataset_ref} stored in this graph (no attempts for this quantum)."
|
|
149
|
+
)
|
|
150
|
+
if attempts[-1] is None:
|
|
151
|
+
raise FileNotFoundError(
|
|
152
|
+
f"No log dataset {self._dataset_ref} stored in this graph "
|
|
153
|
+
"(most recent attempt failed and did not write logs)."
|
|
154
|
+
)
|
|
155
|
+
return attempts[-1]
|
|
156
|
+
|
|
157
|
+
def _read_packages(self, uri: ResourcePath) -> Packages:
|
|
158
|
+
with ProvenanceQuantumGraphReader.open(uri, import_mode=TaskImportMode.DO_NOT_IMPORT) as reader:
|
|
159
|
+
return reader.fetch_packages()
|
|
160
|
+
|
|
161
|
+
def _read_config(self, uri: ResourcePath) -> Config:
|
|
162
|
+
task_label = self._dataset_ref.datasetType.name.removesuffix("_config")
|
|
163
|
+
with ProvenanceQuantumGraphReader.open(
|
|
164
|
+
uri, import_mode=TaskImportMode.ASSUME_CONSISTENT_EDGES
|
|
165
|
+
) as reader:
|
|
166
|
+
try:
|
|
167
|
+
return reader.pipeline_graph.tasks[task_label].config.copy()
|
|
168
|
+
except KeyError:
|
|
169
|
+
raise FileNotFoundError(
|
|
170
|
+
f"No task with label {task_label!r} found in the pipeline graph."
|
|
171
|
+
) from None
|
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
# This file is part of pipe_base.
|
|
2
|
+
#
|
|
3
|
+
# Developed for the LSST Data Management System.
|
|
4
|
+
# This product includes software developed by the LSST Project
|
|
5
|
+
# (http://www.lsst.org).
|
|
6
|
+
# See the COPYRIGHT file at the top-level directory of this distribution
|
|
7
|
+
# for details of code ownership.
|
|
8
|
+
#
|
|
9
|
+
# This software is dual licensed under the GNU General Public License and also
|
|
10
|
+
# under a 3-clause BSD license. Recipients may choose which of these licenses
|
|
11
|
+
# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
|
|
12
|
+
# respectively. If you choose the GPL option then the following text applies
|
|
13
|
+
# (but note that there is still no warranty even if you opt for BSD instead):
|
|
14
|
+
#
|
|
15
|
+
# This program is free software: you can redistribute it and/or modify
|
|
16
|
+
# it under the terms of the GNU General Public License as published by
|
|
17
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
18
|
+
# (at your option) any later version.
|
|
19
|
+
#
|
|
20
|
+
# This program is distributed in the hope that it will be useful,
|
|
21
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
22
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
23
|
+
# GNU General Public License for more details.
|
|
24
|
+
#
|
|
25
|
+
# You should have received a copy of the GNU General Public License
|
|
26
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
27
|
+
|
|
28
|
+
"""A tool for ingesting provenance quantum graphs (written by the `aggregator`
|
|
29
|
+
module) and [re-]ingesting other datasets (metadata/logs/configs) backed by the
|
|
30
|
+
same file. This "finalizes" the RUN collection, prohibiting (at least
|
|
31
|
+
conceptually) further processing.
|
|
32
|
+
|
|
33
|
+
This always proceeds in three steps, so we can resume efficiently:
|
|
34
|
+
|
|
35
|
+
1. First we ask the butler to "forget" any metadata/log/config datasets that
|
|
36
|
+
exist in the output RUN collection, removing any record of them from the
|
|
37
|
+
butler database while preserving their files.
|
|
38
|
+
|
|
39
|
+
2. Next we ingest the ``run_provenance`` graph dataset itself.
|
|
40
|
+
|
|
41
|
+
3. Finally, in batches of quanta, we use a
|
|
42
|
+
`~lsst.daf.butler.QuantumBackedButler` to delete the original
|
|
43
|
+
metadata/log/config files and ingest new versions of those datasets into the
|
|
44
|
+
butler.
|
|
45
|
+
|
|
46
|
+
Thus, at any point, if the ``run_provenance`` dataset has not been ingested,
|
|
47
|
+
we know any metadata/log/config datasets that have been ingested are backed by
|
|
48
|
+
the original files.
|
|
49
|
+
|
|
50
|
+
Moreover, if the ``run_provenance`` dataset has been ingested, any existing
|
|
51
|
+
metadata/log/config datasets must be backed by the graph file, and the original
|
|
52
|
+
files for those datasets will have been deleted.
|
|
53
|
+
|
|
54
|
+
We also know that at all times the metadata/log/config *content* is safely
|
|
55
|
+
present in either the original files in the butler storage or in an
|
|
56
|
+
already-ingested ``run_provenance`` dataset.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
from __future__ import annotations
|
|
60
|
+
|
|
61
|
+
__all__ = ("ingest_graph",)
|
|
62
|
+
|
|
63
|
+
import dataclasses
|
|
64
|
+
import itertools
|
|
65
|
+
import uuid
|
|
66
|
+
from collections.abc import Iterator
|
|
67
|
+
from contextlib import contextmanager
|
|
68
|
+
|
|
69
|
+
from lsst.daf.butler import (
|
|
70
|
+
Butler,
|
|
71
|
+
Config,
|
|
72
|
+
DataCoordinate,
|
|
73
|
+
DatasetRef,
|
|
74
|
+
DatasetType,
|
|
75
|
+
FileDataset,
|
|
76
|
+
QuantumBackedButler,
|
|
77
|
+
)
|
|
78
|
+
from lsst.daf.butler.registry.sql_registry import SqlRegistry
|
|
79
|
+
from lsst.resources import ResourcePath, ResourcePathExpression
|
|
80
|
+
from lsst.utils.logging import getLogger
|
|
81
|
+
|
|
82
|
+
from ..automatic_connection_constants import PROVENANCE_DATASET_TYPE_NAME, PROVENANCE_STORAGE_CLASS
|
|
83
|
+
from ._provenance import (
|
|
84
|
+
ProvenanceDatasetInfo,
|
|
85
|
+
ProvenanceInitQuantumInfo,
|
|
86
|
+
ProvenanceQuantumGraph,
|
|
87
|
+
ProvenanceQuantumGraphReader,
|
|
88
|
+
ProvenanceQuantumInfo,
|
|
89
|
+
)
|
|
90
|
+
from .formatter import ProvenanceFormatter
|
|
91
|
+
|
|
92
|
+
_LOG = getLogger(__name__)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def ingest_graph(
|
|
96
|
+
butler_config: str | Config,
|
|
97
|
+
uri: ResourcePathExpression | None = None,
|
|
98
|
+
*,
|
|
99
|
+
transfer: str | None = "move",
|
|
100
|
+
batch_size: int = 10000,
|
|
101
|
+
output_run: str | None = None,
|
|
102
|
+
) -> None:
|
|
103
|
+
"""Ingest a provenance graph into a butler repository.
|
|
104
|
+
|
|
105
|
+
Parameters
|
|
106
|
+
----------
|
|
107
|
+
butler_config : `str`
|
|
108
|
+
Path or alias for the butler repository, or a butler repository config
|
|
109
|
+
object.
|
|
110
|
+
uri : convertible to `lsst.resources.ResourcePath` or `None`, optional
|
|
111
|
+
Location of the provenance quantum graph to ingest. `None` indicates
|
|
112
|
+
that the quantum graph has already been ingested, but other ingests
|
|
113
|
+
and/or deletions failed and need to be resumed.
|
|
114
|
+
batch_size : `int`, optional
|
|
115
|
+
Number of datasets to process in each transaction.
|
|
116
|
+
output_run : `str`, optional
|
|
117
|
+
Output `~lsst.daf.butler.CollectionType.RUN` collection name. Only
|
|
118
|
+
needs to be provided if ``uri`` is `None`. If it is provided the
|
|
119
|
+
output run in the graph is checked against it.
|
|
120
|
+
|
|
121
|
+
Notes
|
|
122
|
+
-----
|
|
123
|
+
After this operation, no further processing may be done in the
|
|
124
|
+
`~lsst.daf.butler.CollectionType.RUN` collection.
|
|
125
|
+
|
|
126
|
+
If this process is interrupted, it can pick up where it left off if run
|
|
127
|
+
again (at the cost of some duplicate work to figure out how much progress
|
|
128
|
+
it had made).
|
|
129
|
+
"""
|
|
130
|
+
with _GraphIngester.open(butler_config, uri, output_run) as helper:
|
|
131
|
+
helper.fetch_already_ingested_datasets()
|
|
132
|
+
if not helper.graph_already_ingested:
|
|
133
|
+
assert uri is not None
|
|
134
|
+
helper.forget_ingested_datasets(batch_size=batch_size)
|
|
135
|
+
helper.ingest_graph_dataset(uri, transfer=transfer)
|
|
136
|
+
helper.clean_and_reingest_datasets(batch_size=batch_size)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclasses.dataclass
|
|
140
|
+
class _GraphIngester:
|
|
141
|
+
butler_config: str | Config
|
|
142
|
+
butler: Butler
|
|
143
|
+
graph: ProvenanceQuantumGraph
|
|
144
|
+
graph_already_ingested: bool
|
|
145
|
+
n_datasets: int
|
|
146
|
+
datasets_already_ingested: set[uuid.UUID] = dataclasses.field(default_factory=set)
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def output_run(self) -> str:
|
|
150
|
+
return self.graph.header.output_run
|
|
151
|
+
|
|
152
|
+
@classmethod
|
|
153
|
+
@contextmanager
|
|
154
|
+
def open(
|
|
155
|
+
cls,
|
|
156
|
+
butler_config: str | Config,
|
|
157
|
+
uri: ResourcePathExpression | None,
|
|
158
|
+
output_run: str | None,
|
|
159
|
+
) -> Iterator[_GraphIngester]:
|
|
160
|
+
with Butler.from_config(butler_config, collections=output_run, writeable=True) as butler:
|
|
161
|
+
butler.registry.registerDatasetType(
|
|
162
|
+
DatasetType(PROVENANCE_DATASET_TYPE_NAME, butler.dimensions.empty, PROVENANCE_STORAGE_CLASS)
|
|
163
|
+
)
|
|
164
|
+
graph, graph_already_ingested = cls.read_graph(butler, uri)
|
|
165
|
+
if output_run is not None and graph.header.output_run != output_run:
|
|
166
|
+
raise ValueError(
|
|
167
|
+
f"Given output run {output_run!r} does not match the graph "
|
|
168
|
+
f"header {graph.header.output_run!r}."
|
|
169
|
+
)
|
|
170
|
+
n_datasets = 2 * len(graph.quantum_only_xgraph) + len(graph.init_quanta)
|
|
171
|
+
yield cls(
|
|
172
|
+
butler_config=butler_config,
|
|
173
|
+
butler=butler,
|
|
174
|
+
graph=graph,
|
|
175
|
+
graph_already_ingested=graph_already_ingested,
|
|
176
|
+
n_datasets=n_datasets,
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
@staticmethod
|
|
180
|
+
def read_graph(
|
|
181
|
+
butler: Butler,
|
|
182
|
+
uri: ResourcePathExpression | None,
|
|
183
|
+
) -> tuple[ProvenanceQuantumGraph, bool]:
|
|
184
|
+
if uri is not None:
|
|
185
|
+
_LOG.info("Reading the pre-ingest provenance graph.")
|
|
186
|
+
with ProvenanceQuantumGraphReader.open(uri) as reader:
|
|
187
|
+
reader.read_quanta()
|
|
188
|
+
reader.read_init_quanta()
|
|
189
|
+
graph = reader.graph
|
|
190
|
+
already_ingested = (
|
|
191
|
+
butler.find_dataset(PROVENANCE_DATASET_TYPE_NAME, collections=[graph.header.output_run])
|
|
192
|
+
is not None
|
|
193
|
+
)
|
|
194
|
+
return graph, already_ingested
|
|
195
|
+
else:
|
|
196
|
+
_LOG.info("Reading the already-ingested provenance graph.")
|
|
197
|
+
parameters = {"datasets": [], "read_init_quanta": True}
|
|
198
|
+
return butler.get(PROVENANCE_DATASET_TYPE_NAME, parameters=parameters), True
|
|
199
|
+
|
|
200
|
+
def fetch_already_ingested_datasets(self) -> None:
|
|
201
|
+
_LOG.info("Querying for existing datasets in %r.", self.output_run)
|
|
202
|
+
self.datasets_already_ingested.update(self.butler.registry._fetch_run_dataset_ids(self.output_run))
|
|
203
|
+
|
|
204
|
+
def iter_datasets(self) -> Iterator[tuple[uuid.UUID, ProvenanceDatasetInfo]]:
|
|
205
|
+
xgraph = self.graph.bipartite_xgraph
|
|
206
|
+
for task_label, quanta_for_task in self.graph.quanta_by_task.items():
|
|
207
|
+
_LOG.verbose(
|
|
208
|
+
"Batching up metadata and log datasets from %d %s quanta.", len(quanta_for_task), task_label
|
|
209
|
+
)
|
|
210
|
+
for quantum_id in quanta_for_task.values():
|
|
211
|
+
quantum_info: ProvenanceQuantumInfo = xgraph.nodes[quantum_id]
|
|
212
|
+
metadata_id = quantum_info["metadata_id"]
|
|
213
|
+
yield metadata_id, xgraph.nodes[metadata_id]
|
|
214
|
+
log_id = quantum_info["log_id"]
|
|
215
|
+
yield log_id, xgraph.nodes[log_id]
|
|
216
|
+
_LOG.verbose("Batching up config datasets from %d tasks.", len(self.graph.init_quanta))
|
|
217
|
+
for task_label, quantum_id in self.graph.init_quanta.items():
|
|
218
|
+
init_quantum_info: ProvenanceInitQuantumInfo = xgraph.nodes[quantum_id]
|
|
219
|
+
config_id = init_quantum_info["config_id"]
|
|
220
|
+
yield config_id, xgraph.nodes[config_id]
|
|
221
|
+
|
|
222
|
+
def forget_ingested_datasets(self, batch_size: int) -> None:
|
|
223
|
+
_LOG.info(
|
|
224
|
+
"Dropping database records for metadata/log/config datasets backed by their original files."
|
|
225
|
+
)
|
|
226
|
+
to_forget: list[DatasetRef] = []
|
|
227
|
+
n_forgotten: int = 0
|
|
228
|
+
n_skipped: int = 0
|
|
229
|
+
for dataset_id, dataset_info in self.iter_datasets():
|
|
230
|
+
if dataset_info["produced"] and dataset_id in self.datasets_already_ingested:
|
|
231
|
+
to_forget.append(self._make_ref_from_info(dataset_id, dataset_info))
|
|
232
|
+
self.datasets_already_ingested.remove(dataset_id)
|
|
233
|
+
if len(to_forget) >= batch_size:
|
|
234
|
+
n_forgotten += self._run_forget(to_forget, n_forgotten + n_skipped)
|
|
235
|
+
else:
|
|
236
|
+
n_skipped += 1
|
|
237
|
+
n_forgotten += self._run_forget(to_forget, n_forgotten + n_skipped)
|
|
238
|
+
_LOG.info(
|
|
239
|
+
"Removed database records for %d metadata/log/config datasets, while %d were already absent.",
|
|
240
|
+
n_forgotten,
|
|
241
|
+
n_skipped,
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
def _run_forget(self, to_forget: list[DatasetRef], n_current: int) -> int:
|
|
245
|
+
if to_forget:
|
|
246
|
+
_LOG.verbose(
|
|
247
|
+
"Forgetting a %d-dataset batch; %d/%d forgotten so far or already absent.",
|
|
248
|
+
len(to_forget),
|
|
249
|
+
n_current,
|
|
250
|
+
self.n_datasets,
|
|
251
|
+
)
|
|
252
|
+
with self.butler.registry.transaction():
|
|
253
|
+
self.butler._datastore.forget(to_forget)
|
|
254
|
+
self.butler.registry.removeDatasets(to_forget)
|
|
255
|
+
n = len(to_forget)
|
|
256
|
+
to_forget.clear()
|
|
257
|
+
return n
|
|
258
|
+
|
|
259
|
+
def ingest_graph_dataset(self, uri: ResourcePathExpression, transfer: str | None) -> None:
|
|
260
|
+
_LOG.info("Ingesting the provenance quantum graph.")
|
|
261
|
+
dataset_type = DatasetType(
|
|
262
|
+
PROVENANCE_DATASET_TYPE_NAME, self.butler.dimensions.empty, PROVENANCE_STORAGE_CLASS
|
|
263
|
+
)
|
|
264
|
+
self.butler.registry.registerDatasetType(dataset_type)
|
|
265
|
+
ref = DatasetRef(dataset_type, DataCoordinate.make_empty(self.butler.dimensions), run=self.output_run)
|
|
266
|
+
uri = ResourcePath(uri)
|
|
267
|
+
self.butler.ingest(
|
|
268
|
+
# We use .abspath() since butler assumes paths are relative to the
|
|
269
|
+
# repo root, while users expects them to be relative to the CWD in
|
|
270
|
+
# this context.
|
|
271
|
+
FileDataset(refs=[ref], path=uri.abspath(), formatter=ProvenanceFormatter),
|
|
272
|
+
transfer=transfer,
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
def clean_and_reingest_datasets(self, batch_size: int) -> None:
|
|
276
|
+
_LOG.info(
|
|
277
|
+
"Deleting original metadata/log/config files and re-ingesting them with provenance graph backing."
|
|
278
|
+
)
|
|
279
|
+
direct_uri = self.butler.getURI(PROVENANCE_DATASET_TYPE_NAME, collections=[self.output_run])
|
|
280
|
+
qbb = self.make_qbb()
|
|
281
|
+
to_process: list[DatasetRef] = []
|
|
282
|
+
n_processed: int = 0
|
|
283
|
+
n_skipped: int = 0
|
|
284
|
+
n_not_produced: int = 0
|
|
285
|
+
for dataset_id, dataset_info in self.iter_datasets():
|
|
286
|
+
if not dataset_info["produced"]:
|
|
287
|
+
n_not_produced += 1
|
|
288
|
+
elif dataset_id not in self.datasets_already_ingested:
|
|
289
|
+
to_process.append(self._make_ref_from_info(dataset_id, dataset_info))
|
|
290
|
+
if len(to_process) >= batch_size:
|
|
291
|
+
n_processed += self._run_clean_and_ingest(
|
|
292
|
+
qbb, direct_uri, to_process, n_processed + n_skipped
|
|
293
|
+
)
|
|
294
|
+
else:
|
|
295
|
+
n_skipped += 1
|
|
296
|
+
n_processed += self._run_clean_and_ingest(qbb, direct_uri, to_process, n_processed + n_skipped)
|
|
297
|
+
_LOG.info(
|
|
298
|
+
"Deleted and re-ingested %d metadata/log/config datasets "
|
|
299
|
+
"(%d had already been processed, %d were not produced).",
|
|
300
|
+
n_processed,
|
|
301
|
+
n_skipped,
|
|
302
|
+
n_not_produced,
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
def _run_clean_and_ingest(
|
|
306
|
+
self, qbb: QuantumBackedButler, direct_uri: ResourcePath, to_process: list[DatasetRef], n_current: int
|
|
307
|
+
) -> int:
|
|
308
|
+
if not to_process:
|
|
309
|
+
return 0
|
|
310
|
+
_LOG.verbose(
|
|
311
|
+
"Deleting and deleting a %d-dataset batch; %d/%d complete.",
|
|
312
|
+
len(to_process),
|
|
313
|
+
n_current,
|
|
314
|
+
self.n_datasets,
|
|
315
|
+
)
|
|
316
|
+
sql_registry: SqlRegistry = self.butler._registry # type: ignore[attr-defined]
|
|
317
|
+
expanded_refs = sql_registry.expand_refs(to_process)
|
|
318
|
+
# We need to pass predict=True to keep QBB/FileDatastore from wasting
|
|
319
|
+
# time doing existence checks, since ResourcePath.mremove will ignore
|
|
320
|
+
# nonexistent files anyway.
|
|
321
|
+
original_uris = list(
|
|
322
|
+
itertools.chain.from_iterable(
|
|
323
|
+
ref_uris.iter_all() for ref_uris in qbb.get_many_uris(expanded_refs, predict=True).values()
|
|
324
|
+
)
|
|
325
|
+
)
|
|
326
|
+
removal_status = ResourcePath.mremove(original_uris, do_raise=False)
|
|
327
|
+
for path, status in removal_status.items():
|
|
328
|
+
if not status.success and not isinstance(status.exception, FileNotFoundError):
|
|
329
|
+
assert status.exception is not None, "Exception should be set if success=False."
|
|
330
|
+
status.exception.add_note(f"Attempting to delete original file at {path}.")
|
|
331
|
+
raise status.exception
|
|
332
|
+
file_dataset = FileDataset(refs=expanded_refs, path=direct_uri, formatter=ProvenanceFormatter)
|
|
333
|
+
self.butler.ingest(file_dataset, transfer=None)
|
|
334
|
+
n = len(to_process)
|
|
335
|
+
to_process.clear()
|
|
336
|
+
return n
|
|
337
|
+
|
|
338
|
+
@staticmethod
|
|
339
|
+
def _make_ref_from_info(dataset_id: uuid.UUID, dataset_info: ProvenanceDatasetInfo) -> DatasetRef:
|
|
340
|
+
return DatasetRef(
|
|
341
|
+
dataset_info["pipeline_node"].dataset_type,
|
|
342
|
+
dataset_info["data_id"],
|
|
343
|
+
run=dataset_info["run"],
|
|
344
|
+
id=dataset_id,
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
def make_qbb(self) -> QuantumBackedButler:
|
|
348
|
+
dataset_types = {d.name: d.dataset_type for d in self.graph.pipeline_graph.dataset_types.values()}
|
|
349
|
+
return QuantumBackedButler.from_predicted(
|
|
350
|
+
config=self.butler_config,
|
|
351
|
+
predicted_inputs=(),
|
|
352
|
+
predicted_outputs=(),
|
|
353
|
+
dimensions=self.butler.dimensions,
|
|
354
|
+
datastore_records={},
|
|
355
|
+
dataset_types=dataset_types,
|
|
356
|
+
)
|
|
@@ -79,6 +79,7 @@ from .automatic_connection_constants import (
|
|
|
79
79
|
METADATA_OUTPUT_CONNECTION_NAME,
|
|
80
80
|
METADATA_OUTPUT_STORAGE_CLASS,
|
|
81
81
|
METADATA_OUTPUT_TEMPLATE,
|
|
82
|
+
PROVENANCE_DATASET_TYPE_NAME,
|
|
82
83
|
)
|
|
83
84
|
from .graph import QuantumGraph, QuantumNode
|
|
84
85
|
|
|
@@ -1513,8 +1514,22 @@ class QuantumProvenanceGraph:
|
|
|
1513
1514
|
len(self._datasets.keys()),
|
|
1514
1515
|
)
|
|
1515
1516
|
if use_qbb:
|
|
1516
|
-
|
|
1517
|
-
|
|
1517
|
+
provenance_graph_ref: DatasetRef | None = None
|
|
1518
|
+
try:
|
|
1519
|
+
provenance_graph_ref = butler.find_dataset(
|
|
1520
|
+
PROVENANCE_DATASET_TYPE_NAME, collections=output_run
|
|
1521
|
+
)
|
|
1522
|
+
except MissingDatasetTypeError:
|
|
1523
|
+
pass
|
|
1524
|
+
if provenance_graph_ref is not None:
|
|
1525
|
+
_LOG.warning(
|
|
1526
|
+
"Cannot use QBB for metadata/log reads after provenance has been ingested; "
|
|
1527
|
+
"falling back to full butler."
|
|
1528
|
+
)
|
|
1529
|
+
self._butler_wrappers[output_run] = _ThreadLocalButlerWrapper.wrap_full(butler)
|
|
1530
|
+
else:
|
|
1531
|
+
_LOG.verbose("Using quantum-backed butler for metadata loads.")
|
|
1532
|
+
self._butler_wrappers[output_run] = _ThreadLocalButlerWrapper.wrap_qbb(butler, qgraph)
|
|
1518
1533
|
else:
|
|
1519
1534
|
_LOG.verbose("Using full butler for metadata loads.")
|
|
1520
1535
|
self._butler_wrappers[output_run] = _ThreadLocalButlerWrapper.wrap_full(butler)
|
|
@@ -390,10 +390,11 @@ class SeparablePipelineExecutor:
|
|
|
390
390
|
provenance_dataset_ref : `lsst.daf.butler.DatasetRef`, optional
|
|
391
391
|
Dataset that should be used to save provenance. Provenance is only
|
|
392
392
|
supported when running in a single process (at least for the
|
|
393
|
-
default quantum executor) and
|
|
394
|
-
``skip_existing_in``
|
|
395
|
-
|
|
396
|
-
|
|
393
|
+
default quantum executor), and should not be used with
|
|
394
|
+
``skip_existing_in=[output_run]`` when retrying a previous
|
|
395
|
+
execution attempt. The caller is responsible for registering the
|
|
396
|
+
dataset type and for ensuring that the dimensions of this dataset
|
|
397
|
+
do not lead to uniqueness conflicts.
|
|
397
398
|
"""
|
|
398
399
|
if not graph_executor:
|
|
399
400
|
quantum_executor = SingleQuantumExecutor(
|
|
@@ -415,8 +416,6 @@ class SeparablePipelineExecutor:
|
|
|
415
416
|
self._butler.registry.resetConnectionPool()
|
|
416
417
|
|
|
417
418
|
if provenance_dataset_ref is not None:
|
|
418
|
-
if self._skip_existing_in:
|
|
419
|
-
raise RuntimeError("Provenance writing is not compatible with skip_existing_in=True.")
|
|
420
419
|
with TemporaryForIngest(self._butler, provenance_dataset_ref) as temporary:
|
|
421
420
|
graph_executor.execute(graph, provenance_graph_file=temporary.ospath)
|
|
422
421
|
temporary.ingest()
|
|
@@ -78,12 +78,12 @@ class SingleQuantumExecutor(QuantumExecutor):
|
|
|
78
78
|
Instance of a task factory. Defaults to a new instance of
|
|
79
79
|
`lsst.pipe.base.TaskFactory`.
|
|
80
80
|
skip_existing_in : `str` or `~collections.abc.Iterable` [ `str` ]
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
81
|
+
A collection name or list of collections to search for the existing
|
|
82
|
+
outputs of quanta, which indicates that those quanta should be skipped.
|
|
83
|
+
This class only checks for the presence of butler output run in the
|
|
84
|
+
list of collections. If the output run is present in the list then the
|
|
85
|
+
quanta whose complete outputs exist in the output run will be skipped.
|
|
86
|
+
`None` or empty string/sequence disables skipping.
|
|
87
87
|
clobber_outputs : `bool`, optional
|
|
88
88
|
If `True`, then outputs from a quantum that exist in output run
|
|
89
89
|
collection will be removed prior to executing a quantum. If
|
lsst/pipe/base/struct.py
CHANGED
lsst/pipe/base/version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
__all__ = ["__version__"]
|
|
2
|
-
__version__ = "30.2026.
|
|
2
|
+
__version__ = "30.2026.400"
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lsst-pipe-base
|
|
3
|
-
Version: 30.2026.
|
|
3
|
+
Version: 30.2026.400
|
|
4
4
|
Summary: Pipeline infrastructure for the Rubin Science Pipelines.
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License-Expression: BSD-3-Clause OR GPL-3.0-or-later
|
|
7
7
|
Project-URL: Homepage, https://github.com/lsst/pipe_base
|
|
8
|
+
Project-URL: Source, https://github.com/lsst/pipe_base
|
|
8
9
|
Keywords: lsst
|
|
9
10
|
Classifier: Intended Audience :: Science/Research
|
|
10
11
|
Classifier: Operating System :: OS Independent
|