lsst-pipe-base 30.2026.400__py3-none-any.whl → 30.2026.500__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/pipe/base/_instrument.py +17 -5
- lsst/pipe/base/_quantumContext.py +3 -3
- lsst/pipe/base/_status.py +14 -0
- lsst/pipe/base/automatic_connection_constants.py +11 -0
- lsst/pipe/base/cli/cmd/__init__.py +2 -0
- lsst/pipe/base/cli/cmd/commands.py +108 -1
- lsst/pipe/base/graph/graph.py +9 -8
- lsst/pipe/base/log_capture.py +1 -1
- lsst/pipe/base/pipeline.py +2 -2
- lsst/pipe/base/pipelineIR.py +1 -1
- lsst/pipe/base/pipeline_graph/_dataset_types.py +2 -2
- lsst/pipe/base/pipeline_graph/_edges.py +11 -11
- lsst/pipe/base/pipeline_graph/_pipeline_graph.py +2 -2
- lsst/pipe/base/pipeline_graph/visualization/_dot.py +13 -12
- lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py +7 -0
- lsst/pipe/base/prerequisite_helpers.py +2 -1
- lsst/pipe/base/quantum_graph/_multiblock.py +29 -13
- lsst/pipe/base/quantum_graph/_provenance.py +411 -19
- lsst/pipe/base/quantum_graph/aggregator/__init__.py +0 -1
- lsst/pipe/base/quantum_graph/formatter.py +4 -4
- lsst/pipe/base/quantum_graph/ingest_graph.py +61 -4
- lsst/pipe/base/quantum_graph_builder.py +1 -8
- lsst/pipe/base/quantum_graph_skeleton.py +29 -27
- lsst/pipe/base/quantum_provenance_graph.py +12 -10
- lsst/pipe/base/separable_pipeline_executor.py +1 -1
- lsst/pipe/base/single_quantum_executor.py +1 -1
- lsst/pipe/base/tests/mocks/_storage_class.py +2 -1
- lsst/pipe/base/version.py +1 -1
- {lsst_pipe_base-30.2026.400.dist-info → lsst_pipe_base-30.2026.500.dist-info}/METADATA +1 -1
- {lsst_pipe_base-30.2026.400.dist-info → lsst_pipe_base-30.2026.500.dist-info}/RECORD +38 -38
- {lsst_pipe_base-30.2026.400.dist-info → lsst_pipe_base-30.2026.500.dist-info}/WHEEL +1 -1
- {lsst_pipe_base-30.2026.400.dist-info → lsst_pipe_base-30.2026.500.dist-info}/entry_points.txt +0 -0
- {lsst_pipe_base-30.2026.400.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_pipe_base-30.2026.400.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/LICENSE +0 -0
- {lsst_pipe_base-30.2026.400.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/bsd_license.txt +0 -0
- {lsst_pipe_base-30.2026.400.dist-info → lsst_pipe_base-30.2026.500.dist-info}/licenses/gpl-v3.0.txt +0 -0
- {lsst_pipe_base-30.2026.400.dist-info → lsst_pipe_base-30.2026.500.dist-info}/top_level.txt +0 -0
- {lsst_pipe_base-30.2026.400.dist-info → lsst_pipe_base-30.2026.500.dist-info}/zip-safe +0 -0
lsst/pipe/base/_instrument.py
CHANGED
|
@@ -35,7 +35,7 @@ from abc import ABCMeta, abstractmethod
|
|
|
35
35
|
from collections.abc import Sequence
|
|
36
36
|
from typing import TYPE_CHECKING, Any, Self, cast, final
|
|
37
37
|
|
|
38
|
-
from lsst.daf.butler import DataCoordinate, DataId, DimensionPacker, DimensionRecord, Formatter
|
|
38
|
+
from lsst.daf.butler import DataCoordinate, DataId, DimensionPacker, DimensionRecord, Formatter, FormatterV2
|
|
39
39
|
from lsst.daf.butler.registry import DataIdError
|
|
40
40
|
from lsst.pex.config import Config, RegistryField
|
|
41
41
|
from lsst.resources import ResourcePath, ResourcePathExpression
|
|
@@ -311,7 +311,7 @@ class Instrument(metaclass=ABCMeta):
|
|
|
311
311
|
return instrument_cls(collection_prefix=collection_prefix)
|
|
312
312
|
|
|
313
313
|
@staticmethod
|
|
314
|
-
def importAll(registry: Registry) ->
|
|
314
|
+
def importAll(registry: Registry) -> dict[str, type[Instrument]]:
|
|
315
315
|
"""Import all the instruments known to this registry.
|
|
316
316
|
|
|
317
317
|
This will ensure that all metadata translators have been registered.
|
|
@@ -321,20 +321,31 @@ class Instrument(metaclass=ABCMeta):
|
|
|
321
321
|
registry : `lsst.daf.butler.Registry`
|
|
322
322
|
Butler registry to query to find the information.
|
|
323
323
|
|
|
324
|
+
Returns
|
|
325
|
+
-------
|
|
326
|
+
imported : `dict` [`str`, `type` [`Instrument`]]
|
|
327
|
+
A mapping containing all the instrument classes that were loaded
|
|
328
|
+
successfully, keyed by their butler names.
|
|
329
|
+
|
|
324
330
|
Notes
|
|
325
331
|
-----
|
|
326
332
|
It is allowed for a particular instrument class to fail on import.
|
|
327
333
|
This might simply indicate that a particular obs package has
|
|
328
334
|
not been setup.
|
|
329
335
|
"""
|
|
336
|
+
imported: dict[str, type[Instrument]] = {}
|
|
330
337
|
records = list(registry.queryDimensionRecords("instrument"))
|
|
331
338
|
for record in records:
|
|
332
339
|
cls = record.class_name
|
|
340
|
+
instrument_name: str = cast(str, record.name)
|
|
333
341
|
with contextlib.suppress(Exception):
|
|
334
|
-
doImportType(cls)
|
|
342
|
+
instr = doImportType(cls)
|
|
343
|
+
assert issubclass(instr, Instrument)
|
|
344
|
+
imported[instrument_name] = instr
|
|
345
|
+
return imported
|
|
335
346
|
|
|
336
347
|
@abstractmethod
|
|
337
|
-
def getRawFormatter(self, dataId: DataId) -> type[Formatter]:
|
|
348
|
+
def getRawFormatter(self, dataId: DataId) -> type[Formatter | FormatterV2]:
|
|
338
349
|
"""Return the Formatter class that should be used to read a particular
|
|
339
350
|
raw file.
|
|
340
351
|
|
|
@@ -345,7 +356,8 @@ class Instrument(metaclass=ABCMeta):
|
|
|
345
356
|
|
|
346
357
|
Returns
|
|
347
358
|
-------
|
|
348
|
-
formatter : `
|
|
359
|
+
formatter : `type` \
|
|
360
|
+
[`lsst.daf.butler.Formatter` | `lsst.daf.butler.FormatterV2` ]
|
|
349
361
|
Class to be used that reads the file into the correct
|
|
350
362
|
Python object for the raw data.
|
|
351
363
|
"""
|
|
@@ -380,8 +380,8 @@ class QuantumContext:
|
|
|
380
380
|
if dataset is directly a `list` of `~lsst.daf.butler.DatasetRef`
|
|
381
381
|
or a single `~lsst.daf.butler.DatasetRef`. If ``values.NAME`` is
|
|
382
382
|
None, no output is written.
|
|
383
|
-
dataset : `OutputQuantizedConnection` or `list`
|
|
384
|
-
or `DatasetRef`
|
|
383
|
+
dataset : `OutputQuantizedConnection` or `list` \
|
|
384
|
+
[`lsst.daf.butler.DatasetRef`] or `lsst.daf.butler.DatasetRef`
|
|
385
385
|
This argument may either be an `InputQuantizedConnection` which
|
|
386
386
|
describes all the inputs of a quantum, a list of
|
|
387
387
|
`lsst.daf.butler.DatasetRef`, or a single
|
|
@@ -460,7 +460,7 @@ class QuantumContext:
|
|
|
460
460
|
|
|
461
461
|
Parameters
|
|
462
462
|
----------
|
|
463
|
-
ref : `DatasetRef`
|
|
463
|
+
ref : `lsst.daf.butler.DatasetRef`
|
|
464
464
|
The dataset to attach provenance to. This dataset must have been
|
|
465
465
|
retrieved by this quantum context.
|
|
466
466
|
extra : `dict` [ `str`, `int` | `float` | `str` | `bool` ]
|
lsst/pipe/base/_status.py
CHANGED
|
@@ -338,6 +338,20 @@ class QuantumAttemptStatus(enum.Enum):
|
|
|
338
338
|
"""Whether the log dataset was produced."""
|
|
339
339
|
return self is self.SUCCESSFUL or self is self.FAILED
|
|
340
340
|
|
|
341
|
+
@property
|
|
342
|
+
def title(self) -> str:
|
|
343
|
+
"""A version of this status' name suitable for use as a title in a plot
|
|
344
|
+
or table.
|
|
345
|
+
"""
|
|
346
|
+
return self.name.capitalize().replace("_", " ")
|
|
347
|
+
|
|
348
|
+
@property
|
|
349
|
+
def is_rare(self) -> bool:
|
|
350
|
+
"""Whether this status is rare enough that it should only be listed
|
|
351
|
+
when it actually occurs.
|
|
352
|
+
"""
|
|
353
|
+
return self in (self.ABORTED, self.ABORTED_SUCCESS, self.UNKNOWN)
|
|
354
|
+
|
|
341
355
|
|
|
342
356
|
class GetSetDictMetadataHolder(Protocol):
|
|
343
357
|
"""Protocol for objects that have a ``metadata`` attribute that satisfies
|
|
@@ -45,6 +45,8 @@ __all__ = (
|
|
|
45
45
|
"PACKAGES_INIT_OUTPUT_STORAGE_CLASS",
|
|
46
46
|
"PROVENANCE_DATASET_TYPE_NAME",
|
|
47
47
|
"PROVENANCE_STORAGE_CLASS",
|
|
48
|
+
"RESOURCE_USAGE_STORAGE_CLASS",
|
|
49
|
+
"RESOURCE_USAGE_TEMPLATE",
|
|
48
50
|
)
|
|
49
51
|
|
|
50
52
|
|
|
@@ -99,3 +101,12 @@ PROVENANCE_DATASET_TYPE_NAME: str = "run_provenance"
|
|
|
99
101
|
|
|
100
102
|
PROVENANCE_STORAGE_CLASS: str = "ProvenanceQuantumGraph"
|
|
101
103
|
"""Name of the storage class used to store provenance."""
|
|
104
|
+
|
|
105
|
+
RESOURCE_USAGE_TEMPLATE: str = "{label}_resource_usage"
|
|
106
|
+
"""String template used to form the name of the resource usage dataset type for
|
|
107
|
+
a task.
|
|
108
|
+
"""
|
|
109
|
+
|
|
110
|
+
RESOURCE_USAGE_STORAGE_CLASS: str = "ArrowAstropy"
|
|
111
|
+
"""Storage class of the resource usage dataset type for a task.
|
|
112
|
+
"""
|
|
@@ -32,6 +32,7 @@ __all__ = [
|
|
|
32
32
|
"retrieve_artifacts_for_quanta",
|
|
33
33
|
"aggregate_graph",
|
|
34
34
|
"ingest_graph",
|
|
35
|
+
"provenance_report",
|
|
35
36
|
]
|
|
36
37
|
|
|
37
38
|
from .commands import (
|
|
@@ -41,4 +42,5 @@ from .commands import (
|
|
|
41
42
|
zip_from_graph,
|
|
42
43
|
aggregate_graph,
|
|
43
44
|
ingest_graph,
|
|
45
|
+
provenance_report,
|
|
44
46
|
)
|
|
@@ -25,6 +25,9 @@
|
|
|
25
25
|
# You should have received a copy of the GNU General Public License
|
|
26
26
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
27
27
|
|
|
28
|
+
import functools
|
|
29
|
+
import operator
|
|
30
|
+
from collections.abc import Iterable
|
|
28
31
|
from typing import Any
|
|
29
32
|
|
|
30
33
|
import click
|
|
@@ -40,6 +43,7 @@ from lsst.daf.butler.cli.opt import (
|
|
|
40
43
|
from lsst.daf.butler.cli.utils import ButlerCommand, split_commas, unwrap
|
|
41
44
|
|
|
42
45
|
from ... import script
|
|
46
|
+
from ..._status import QuantumAttemptStatus, QuantumSuccessCaveats
|
|
43
47
|
from ...quantum_graph import aggregator
|
|
44
48
|
from ..opt import instrument_argument, update_output_chain_option
|
|
45
49
|
|
|
@@ -279,7 +283,7 @@ def aggregate_graph(predicted_graph: str, repo: str, **kwargs: Any) -> None:
|
|
|
279
283
|
|
|
280
284
|
|
|
281
285
|
@click.command(
|
|
282
|
-
short_help="Ingest a provenance quantum graph into a butler
|
|
286
|
+
short_help="Ingest a provenance quantum graph into a butler.",
|
|
283
287
|
cls=ButlerCommand,
|
|
284
288
|
)
|
|
285
289
|
@repo_argument(required=True, help="Path or alias for the butler repository.")
|
|
@@ -306,3 +310,106 @@ def ingest_graph(
|
|
|
306
310
|
from ...quantum_graph.ingest_graph import ingest_graph as ingest_graph_py
|
|
307
311
|
|
|
308
312
|
ingest_graph_py(repo, provenance_graph, transfer=transfer, batch_size=batch_size, output_run=output_run)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@click.command(
|
|
316
|
+
short_help="Print and write provenance reports.",
|
|
317
|
+
cls=ButlerCommand,
|
|
318
|
+
)
|
|
319
|
+
@click.argument("repo_or_qg")
|
|
320
|
+
@click.argument("collection", required=False, default=None)
|
|
321
|
+
@click.option(
|
|
322
|
+
"--state",
|
|
323
|
+
multiple=True,
|
|
324
|
+
type=click.Choice(QuantumAttemptStatus),
|
|
325
|
+
help=(
|
|
326
|
+
"Additional quantum state to include in the status report and data ID tables "
|
|
327
|
+
"(FAILED, ABORTED, and ABORTED_SUCCESS are included by default)."
|
|
328
|
+
),
|
|
329
|
+
)
|
|
330
|
+
@click.option(
|
|
331
|
+
"--no-state",
|
|
332
|
+
multiple=True,
|
|
333
|
+
type=str,
|
|
334
|
+
metavar="STATE",
|
|
335
|
+
help="Quantum state to drop from in status report and data ID tables (same options as --state).",
|
|
336
|
+
)
|
|
337
|
+
@click.option(
|
|
338
|
+
"--status-report",
|
|
339
|
+
default=None,
|
|
340
|
+
metavar="URI",
|
|
341
|
+
help="File or URI (.json) for a detailed report (with data IDs) on quanta with certain states.",
|
|
342
|
+
)
|
|
343
|
+
@click.option(
|
|
344
|
+
"--quantum-table/--no-quantum-table",
|
|
345
|
+
default=True,
|
|
346
|
+
help="Whether to print summary of quantum status counts to STDOUT.",
|
|
347
|
+
)
|
|
348
|
+
@click.option(
|
|
349
|
+
"--exception-table/--no-exception-table",
|
|
350
|
+
default=True,
|
|
351
|
+
help="Whether to print summary of exception type counts STDOUT.",
|
|
352
|
+
)
|
|
353
|
+
@click.option(
|
|
354
|
+
"--caveat",
|
|
355
|
+
multiple=True,
|
|
356
|
+
type=click.Choice(QuantumSuccessCaveats),
|
|
357
|
+
help=(
|
|
358
|
+
"Include successful quanta in the status report if they have this caveat. "
|
|
359
|
+
"May be passed multiple times; any matching caveat is included. "
|
|
360
|
+
"Passing this option implicitly adds '--state SUCCESSFUL'."
|
|
361
|
+
),
|
|
362
|
+
)
|
|
363
|
+
@click.option(
|
|
364
|
+
"--data-id-table-dir",
|
|
365
|
+
default=None,
|
|
366
|
+
metavar="URI",
|
|
367
|
+
help=(
|
|
368
|
+
"Directory (may be a URI) for a tree of data ID tables for each "
|
|
369
|
+
"task label, status, and exception type combination in the status report."
|
|
370
|
+
),
|
|
371
|
+
)
|
|
372
|
+
def provenance_report(
|
|
373
|
+
*,
|
|
374
|
+
repo_or_qg: str,
|
|
375
|
+
collection: str | None,
|
|
376
|
+
state: Iterable[QuantumAttemptStatus],
|
|
377
|
+
no_state: Iterable[str],
|
|
378
|
+
status_report: str | None,
|
|
379
|
+
quantum_table: bool = False,
|
|
380
|
+
exception_table: bool = False,
|
|
381
|
+
caveat: Iterable[QuantumSuccessCaveats],
|
|
382
|
+
data_id_table_dir: str | None,
|
|
383
|
+
) -> None:
|
|
384
|
+
"""Read a provenance quantum graph from a butler or file and use it to
|
|
385
|
+
generate reports.
|
|
386
|
+
|
|
387
|
+
REPO_OR_QG is a path or alias for the butler repository (if reading an
|
|
388
|
+
ingested graph, as indicated by passing COLLECTION), or the path to a
|
|
389
|
+
provenance quantum graph file.
|
|
390
|
+
"""
|
|
391
|
+
from ...quantum_graph import ProvenanceQuantumGraph
|
|
392
|
+
|
|
393
|
+
states = set(state)
|
|
394
|
+
states.add(QuantumAttemptStatus.FAILED)
|
|
395
|
+
states.add(QuantumAttemptStatus.ABORTED)
|
|
396
|
+
states.add(QuantumAttemptStatus.ABORTED_SUCCESS)
|
|
397
|
+
for state_name in no_state:
|
|
398
|
+
states.discard(QuantumAttemptStatus.__members__[state_name])
|
|
399
|
+
with_caveats: QuantumSuccessCaveats | None = None
|
|
400
|
+
if caveat:
|
|
401
|
+
states.add(QuantumAttemptStatus.SUCCESSFUL)
|
|
402
|
+
with_caveats = functools.reduce(
|
|
403
|
+
operator.__or__,
|
|
404
|
+
caveat,
|
|
405
|
+
QuantumSuccessCaveats.NO_CAVEATS,
|
|
406
|
+
)
|
|
407
|
+
with ProvenanceQuantumGraph.from_args(repo_or_qg, collection=collection, datasets=()) as (graph, _):
|
|
408
|
+
graph.make_many_reports(
|
|
409
|
+
status_report_file=status_report,
|
|
410
|
+
states=states,
|
|
411
|
+
print_quantum_table=quantum_table,
|
|
412
|
+
print_exception_table=exception_table,
|
|
413
|
+
with_caveats=with_caveats,
|
|
414
|
+
data_id_table_dir=data_id_table_dir,
|
|
415
|
+
)
|
lsst/pipe/base/graph/graph.py
CHANGED
|
@@ -136,13 +136,14 @@ class QuantumGraph:
|
|
|
136
136
|
Maps tasks to their InitOutput dataset refs. Dataset refs can be either
|
|
137
137
|
resolved or non-resolved. For intermediate resolved refs their dataset
|
|
138
138
|
ID must match ``initInputs`` and Quantum ``initInputs``.
|
|
139
|
-
globalInitOutputs :
|
|
139
|
+
globalInitOutputs : `~collections.abc.Iterable` \
|
|
140
|
+
[ `~lsst.daf.butler.DatasetRef` ], optional
|
|
140
141
|
Dataset refs for some global objects produced by pipeline. These
|
|
141
142
|
objects include task configurations and package versions. Typically
|
|
142
143
|
they have an empty DataId, but there is no real restriction on what
|
|
143
144
|
can appear here.
|
|
144
|
-
registryDatasetTypes :
|
|
145
|
-
optional
|
|
145
|
+
registryDatasetTypes : `~collections.abc.Iterable` \
|
|
146
|
+
[ `~lsst.daf.butler.DatasetType` ], optional
|
|
146
147
|
Dataset types which are used by this graph, their definitions must
|
|
147
148
|
match registry. If registry does not define dataset type yet, then
|
|
148
149
|
it should match one that will be created later.
|
|
@@ -488,7 +489,7 @@ class QuantumGraph:
|
|
|
488
489
|
|
|
489
490
|
Returns
|
|
490
491
|
-------
|
|
491
|
-
tasks :
|
|
492
|
+
tasks : `~collections.abc.Iterable` [ `TaskDef` ]
|
|
492
493
|
`TaskDef` objects that have the specified `DatasetTypeName` as an
|
|
493
494
|
input, list will be empty if no tasks use specified
|
|
494
495
|
`DatasetTypeName` as an input.
|
|
@@ -537,7 +538,7 @@ class QuantumGraph:
|
|
|
537
538
|
|
|
538
539
|
Returns
|
|
539
540
|
-------
|
|
540
|
-
result :
|
|
541
|
+
result : `~collections.abc.Iterable` [`TaskDef`]
|
|
541
542
|
`TaskDef` objects that are associated with the specified
|
|
542
543
|
`DatasetTypeName`.
|
|
543
544
|
|
|
@@ -935,7 +936,7 @@ class QuantumGraph:
|
|
|
935
936
|
saved structure. If supplied, the
|
|
936
937
|
`~lsst.daf.butler.DimensionUniverse` from the loaded `QuantumGraph`
|
|
937
938
|
will be validated against the supplied argument for compatibility.
|
|
938
|
-
nodes :
|
|
939
|
+
nodes : `~collections.abc.Iterable` [ `uuid.UUID` | `str` ] or `None`
|
|
939
940
|
UUIDs that correspond to nodes in the graph. If specified, only
|
|
940
941
|
these nodes will be loaded. Defaults to None, in which case all
|
|
941
942
|
nodes will be loaded.
|
|
@@ -1220,7 +1221,7 @@ class QuantumGraph:
|
|
|
1220
1221
|
saved structure. If supplied, the
|
|
1221
1222
|
`~lsst.daf.butler.DimensionUniverse` from the loaded `QuantumGraph`
|
|
1222
1223
|
will be validated against the supplied argument for compatibility.
|
|
1223
|
-
nodes :
|
|
1224
|
+
nodes : `~collections.abc.Iterable` [`uuid.UUID`] or `None`
|
|
1224
1225
|
UUIDs that correspond to nodes in the graph. If specified, only
|
|
1225
1226
|
these nodes will be loaded. Defaults to None, in which case all
|
|
1226
1227
|
nodes will be loaded.
|
|
@@ -1438,7 +1439,7 @@ class QuantumGraph:
|
|
|
1438
1439
|
Returns
|
|
1439
1440
|
-------
|
|
1440
1441
|
summary : `QgraphSummary`
|
|
1441
|
-
|
|
1442
|
+
Summary of QuantumGraph.
|
|
1442
1443
|
"""
|
|
1443
1444
|
inCollection = self.metadata.get("input", None)
|
|
1444
1445
|
if isinstance(inCollection, str):
|
lsst/pipe/base/log_capture.py
CHANGED
|
@@ -103,7 +103,7 @@ class _ExecutionLogRecordsExtra(pydantic.BaseModel):
|
|
|
103
103
|
|
|
104
104
|
Parameters
|
|
105
105
|
----------
|
|
106
|
-
log_records : `ButlerLogRecords`
|
|
106
|
+
log_records : `lsst.daf.butler.ButlerLogRecords`
|
|
107
107
|
Logs from a past attempt to run a quantum.
|
|
108
108
|
"""
|
|
109
109
|
previous = self.model_validate(log_records.extra)
|
lsst/pipe/base/pipeline.py
CHANGED
|
@@ -495,7 +495,7 @@ class Pipeline:
|
|
|
495
495
|
Returns
|
|
496
496
|
-------
|
|
497
497
|
pipeline: `Pipeline`
|
|
498
|
-
|
|
498
|
+
The new pipeline.
|
|
499
499
|
"""
|
|
500
500
|
return cls.fromIR(copy.deepcopy(pipeline._pipelineIR))
|
|
501
501
|
|
|
@@ -605,7 +605,7 @@ class Pipeline:
|
|
|
605
605
|
|
|
606
606
|
@property
|
|
607
607
|
def subsets(self) -> MappingProxyType[str, set]:
|
|
608
|
-
"""Returns a `MappingProxyType` where the keys are the labels of
|
|
608
|
+
"""Returns a `types.MappingProxyType` where the keys are the labels of
|
|
609
609
|
labeled subsets in the `Pipeline` and the values are the set of task
|
|
610
610
|
labels contained within that subset.
|
|
611
611
|
"""
|
lsst/pipe/base/pipelineIR.py
CHANGED
|
@@ -106,8 +106,8 @@ class DatasetTypeNode:
|
|
|
106
106
|
The internal networkx graph.
|
|
107
107
|
get_registered : `~collections.abc.Callable` or `None`
|
|
108
108
|
Callable that takes a dataset type name and returns the
|
|
109
|
-
|
|
110
|
-
not registered.
|
|
109
|
+
`~lsst.daf.butler.DatasetType` registered in the data repository,
|
|
110
|
+
or `None` if it is not registered.
|
|
111
111
|
dimensions : `lsst.daf.butler.DimensionUniverse`
|
|
112
112
|
Definitions of all dimensions.
|
|
113
113
|
previous : `DatasetTypeNode` or `None`
|
|
@@ -480,11 +480,11 @@ class ReadEdge(Edge):
|
|
|
480
480
|
Parameters
|
|
481
481
|
----------
|
|
482
482
|
current : `lsst.daf.butler.DatasetType` or `None`
|
|
483
|
-
The current graph-wide
|
|
484
|
-
be the registry's definition of the parent dataset
|
|
485
|
-
exists. If not, it will be the dataset type
|
|
486
|
-
task in the graph that writes it, if there is
|
|
487
|
-
such task, this will be `None`.
|
|
483
|
+
The current graph-wide `~lsst.daf.butler.DatasetType`, or `None`.
|
|
484
|
+
This will always be the registry's definition of the parent dataset
|
|
485
|
+
type, if one exists. If not, it will be the dataset type
|
|
486
|
+
definition from the task in the graph that writes it, if there is
|
|
487
|
+
one. If there is no such task, this will be `None`.
|
|
488
488
|
is_initial_query_constraint : `bool`
|
|
489
489
|
Whether this dataset type is currently marked as a constraint on
|
|
490
490
|
the initial data ID query in QuantumGraph generation.
|
|
@@ -496,7 +496,7 @@ class ReadEdge(Edge):
|
|
|
496
496
|
producer : `str` or `None`
|
|
497
497
|
The label of the task that produces this dataset type in the
|
|
498
498
|
pipeline, or `None` if it is an overall input.
|
|
499
|
-
consumers :
|
|
499
|
+
consumers : `~collections.abc.Sequence` [ `str` ]
|
|
500
500
|
Labels for other consuming tasks that have already participated in
|
|
501
501
|
this dataset type's resolution.
|
|
502
502
|
is_registered : `bool`
|
|
@@ -512,7 +512,7 @@ class ReadEdge(Edge):
|
|
|
512
512
|
|
|
513
513
|
Returns
|
|
514
514
|
-------
|
|
515
|
-
dataset_type :
|
|
515
|
+
dataset_type : `~lsst.daf.butler.DatasetType`
|
|
516
516
|
The updated graph-wide dataset type. If ``current`` was provided,
|
|
517
517
|
this must be equal to it.
|
|
518
518
|
is_initial_query_constraint : `bool`
|
|
@@ -800,15 +800,15 @@ class WriteEdge(Edge):
|
|
|
800
800
|
Parameters
|
|
801
801
|
----------
|
|
802
802
|
current : `lsst.daf.butler.DatasetType` or `None`
|
|
803
|
-
The current graph-wide
|
|
804
|
-
be the registry's definition of the parent dataset
|
|
805
|
-
exists.
|
|
803
|
+
The current graph-wide `~lsst.daf.butler.DatasetType`, or `None`.
|
|
804
|
+
This will always be the registry's definition of the parent dataset
|
|
805
|
+
type, if one exists.
|
|
806
806
|
universe : `lsst.daf.butler.DimensionUniverse`
|
|
807
807
|
Object that holds all dimension definitions.
|
|
808
808
|
|
|
809
809
|
Returns
|
|
810
810
|
-------
|
|
811
|
-
dataset_type :
|
|
811
|
+
dataset_type : `~lsst.daf.butler.DatasetType`
|
|
812
812
|
A dataset type compatible with this edge. If ``current`` was
|
|
813
813
|
provided, this must be equal to it.
|
|
814
814
|
|
|
@@ -1636,7 +1636,7 @@ class PipelineGraph:
|
|
|
1636
1636
|
|
|
1637
1637
|
Returns
|
|
1638
1638
|
-------
|
|
1639
|
-
subgraphs :
|
|
1639
|
+
subgraphs : `~collections.abc.Iterable` [ `PipelineGraph` ]
|
|
1640
1640
|
An iterable over component subgraphs that could be run
|
|
1641
1641
|
independently (they have only overall inputs in common). May be a
|
|
1642
1642
|
lazy iterator.
|
|
@@ -2236,7 +2236,7 @@ class PipelineGraph:
|
|
|
2236
2236
|
|
|
2237
2237
|
Parameters
|
|
2238
2238
|
----------
|
|
2239
|
-
updates :
|
|
2239
|
+
updates : `~collections.abc.Mapping` [ `str`, `TaskNode` ]
|
|
2240
2240
|
New task nodes with task label keys. All keys must be task labels
|
|
2241
2241
|
that are already present in the graph.
|
|
2242
2242
|
check_edges_unchanged : `bool`, optional
|
|
@@ -66,7 +66,7 @@ def show_dot(
|
|
|
66
66
|
----------
|
|
67
67
|
pipeline_graph : `PipelineGraph`
|
|
68
68
|
Pipeline graph to show.
|
|
69
|
-
stream : `TextIO`, optional
|
|
69
|
+
stream : `io.TextIO`, optional
|
|
70
70
|
Stream to write the DOT representation to.
|
|
71
71
|
label_edge_connections : `bool`, optional
|
|
72
72
|
If `True`, label edges with their connection names.
|
|
@@ -167,21 +167,22 @@ def _render_dataset_type_node(
|
|
|
167
167
|
|
|
168
168
|
Parameters
|
|
169
169
|
----------
|
|
170
|
-
node_key : NodeKey
|
|
171
|
-
The key for the node
|
|
172
|
-
node_data : Mapping[str
|
|
173
|
-
The data associated with the node
|
|
174
|
-
options : NodeAttributeOptions
|
|
175
|
-
Options for rendering the node
|
|
176
|
-
stream : TextIO
|
|
177
|
-
The stream to write the node to
|
|
170
|
+
node_key : `NodeKey`
|
|
171
|
+
The key for the node.
|
|
172
|
+
node_data : `~collections.abc.Mapping` [`str`, `typing.Any`]
|
|
173
|
+
The data associated with the node.
|
|
174
|
+
options : `NodeAttributeOptions`
|
|
175
|
+
Options for rendering the node.
|
|
176
|
+
stream : `io.TextIO`
|
|
177
|
+
The stream to write the node to.
|
|
178
|
+
overflow_ref : `int`, optional
|
|
178
179
|
|
|
179
180
|
Returns
|
|
180
181
|
-------
|
|
181
182
|
overflow_ref : int
|
|
182
|
-
The reference number for the next overflow node
|
|
183
|
+
The reference number for the next overflow node.
|
|
183
184
|
overflow_ids : str | None
|
|
184
|
-
The ID of the overflow node, if any
|
|
185
|
+
The ID of the overflow node, if any.
|
|
185
186
|
"""
|
|
186
187
|
labels, label_extras, common_prefix = _format_label(str(node_key), _LABEL_MAX_LINES_SOFT)
|
|
187
188
|
if len(labels) + len(label_extras) <= _LABEL_MAX_LINES_HARD:
|
|
@@ -271,7 +272,7 @@ def _render_edge(from_node_id: str, to_node_id: str, stream: TextIO, **kwargs: A
|
|
|
271
272
|
The unique ID of the node the edge is going to
|
|
272
273
|
stream : TextIO
|
|
273
274
|
The stream to write the edge to
|
|
274
|
-
kwargs : Any
|
|
275
|
+
**kwargs : Any
|
|
275
276
|
Additional keyword arguments to pass to the edge
|
|
276
277
|
"""
|
|
277
278
|
if kwargs:
|
|
@@ -200,6 +200,13 @@ class QuantumGraphExecutionStatusAnnotator:
|
|
|
200
200
|
"""Annotates a networkx graph with task and dataset status information from
|
|
201
201
|
a quantum graph execution summary, implementing the StatusAnnotator
|
|
202
202
|
protocol to update the graph with status data.
|
|
203
|
+
|
|
204
|
+
Parameters
|
|
205
|
+
----------
|
|
206
|
+
*args : `typing.Any`
|
|
207
|
+
Arbitrary arguments.
|
|
208
|
+
**kwargs : `typing.Any`
|
|
209
|
+
Arbitrary keyword arguments.
|
|
203
210
|
"""
|
|
204
211
|
|
|
205
212
|
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
@@ -252,7 +252,8 @@ class PrerequisiteFinder:
|
|
|
252
252
|
Sequence of collections to search, in order.
|
|
253
253
|
data_id : `lsst.daf.butler.DataCoordinate`
|
|
254
254
|
Data ID for the quantum.
|
|
255
|
-
skypix_bounds :
|
|
255
|
+
skypix_bounds : `~collections.abc.Mapping` \
|
|
256
|
+
[ `str`, `lsst.sphgeom.RangeSet` ]
|
|
256
257
|
The spatial bounds of this quantum in various skypix dimensions.
|
|
257
258
|
Keys are skypix dimension names (a superset of those in
|
|
258
259
|
`dataset_skypix`) and values are sets of integer pixel ID ranges.
|
|
@@ -43,18 +43,15 @@ import dataclasses
|
|
|
43
43
|
import logging
|
|
44
44
|
import tempfile
|
|
45
45
|
import uuid
|
|
46
|
-
|
|
46
|
+
import zipfile
|
|
47
|
+
from collections.abc import Iterator, Set
|
|
47
48
|
from contextlib import contextmanager
|
|
48
49
|
from io import BufferedReader, BytesIO
|
|
49
50
|
from operator import attrgetter
|
|
50
|
-
from typing import IO,
|
|
51
|
+
from typing import IO, Protocol, TypeAlias, TypeVar
|
|
51
52
|
|
|
52
53
|
import pydantic
|
|
53
54
|
|
|
54
|
-
if TYPE_CHECKING:
|
|
55
|
-
import zipfile
|
|
56
|
-
|
|
57
|
-
|
|
58
55
|
_LOG = logging.getLogger(__name__)
|
|
59
56
|
|
|
60
57
|
|
|
@@ -212,7 +209,7 @@ class AddressWriter:
|
|
|
212
209
|
The converse is not true.
|
|
213
210
|
"""
|
|
214
211
|
|
|
215
|
-
def write(self, stream: IO[bytes], int_size: int) -> None:
|
|
212
|
+
def write(self, stream: IO[bytes], int_size: int, all_ids: Set[uuid.UUID] | None = None) -> None:
|
|
216
213
|
"""Write all addresses to a file-like object.
|
|
217
214
|
|
|
218
215
|
Parameters
|
|
@@ -221,15 +218,17 @@ class AddressWriter:
|
|
|
221
218
|
Binary file-like object.
|
|
222
219
|
int_size : `int`
|
|
223
220
|
Number of bytes to use for all integers.
|
|
221
|
+
all_ids : `~collections.abc.Set` [`uuid.UUID`], optional
|
|
222
|
+
Set of the union of all UUIDs in any dictionary from a call to
|
|
223
|
+
`get_all_ids`.
|
|
224
224
|
"""
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
indices.update(address_map.keys())
|
|
225
|
+
if all_ids is None:
|
|
226
|
+
all_ids = self.get_all_ids()
|
|
228
227
|
stream.write(int_size.to_bytes(1))
|
|
229
|
-
stream.write(len(
|
|
228
|
+
stream.write(len(all_ids).to_bytes(int_size))
|
|
230
229
|
stream.write(len(self.addresses).to_bytes(int_size))
|
|
231
230
|
empty_address = Address()
|
|
232
|
-
for n, key in enumerate(sorted(
|
|
231
|
+
for n, key in enumerate(sorted(all_ids, key=attrgetter("int"))):
|
|
233
232
|
row = AddressRow(key, n, [m.get(key, empty_address) for m in self.addresses])
|
|
234
233
|
_LOG.debug("Wrote address %s.", row)
|
|
235
234
|
row.write(stream, int_size)
|
|
@@ -246,9 +245,26 @@ class AddressWriter:
|
|
|
246
245
|
int_size : `int`
|
|
247
246
|
Number of bytes to use for all integers.
|
|
248
247
|
"""
|
|
249
|
-
|
|
248
|
+
all_ids = self.get_all_ids()
|
|
249
|
+
zip_info = zipfile.ZipInfo(f"{name}.addr")
|
|
250
|
+
row_size = AddressReader.compute_row_size(int_size, len(self.addresses))
|
|
251
|
+
zip_info.file_size = AddressReader.compute_header_size(int_size) + len(all_ids) * row_size
|
|
252
|
+
with zf.open(zip_info, mode="w") as stream:
|
|
250
253
|
self.write(stream, int_size=int_size)
|
|
251
254
|
|
|
255
|
+
def get_all_ids(self) -> Set[uuid.UUID]:
|
|
256
|
+
"""Return all IDs used by any address dictionary.
|
|
257
|
+
|
|
258
|
+
Returns
|
|
259
|
+
-------
|
|
260
|
+
all_ids : `~collections.abc.Set` [`uuid.UUID`]
|
|
261
|
+
Set of all IDs.
|
|
262
|
+
"""
|
|
263
|
+
all_ids: set[uuid.UUID] = set()
|
|
264
|
+
for address_map in self.addresses:
|
|
265
|
+
all_ids.update(address_map.keys())
|
|
266
|
+
return all_ids
|
|
267
|
+
|
|
252
268
|
|
|
253
269
|
@dataclasses.dataclass
|
|
254
270
|
class AddressPage:
|