lsst-pipe-base 30.0.0__py3-none-any.whl → 30.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lsst/pipe/base/_instrument.py +5 -6
- lsst/pipe/base/log_capture.py +79 -39
- lsst/pipe/base/mp_graph_executor.py +15 -51
- lsst/pipe/base/quantum_graph/_common.py +3 -4
- lsst/pipe/base/quantum_graph/_multiblock.py +16 -6
- lsst/pipe/base/quantum_graph/_predicted.py +12 -106
- lsst/pipe/base/quantum_graph/_provenance.py +6 -657
- lsst/pipe/base/quantum_graph/aggregator/_communicators.py +50 -18
- lsst/pipe/base/quantum_graph/aggregator/_scanner.py +229 -35
- lsst/pipe/base/quantum_graph/aggregator/_structs.py +113 -3
- lsst/pipe/base/quantum_graph/aggregator/_supervisor.py +5 -10
- lsst/pipe/base/quantum_graph/aggregator/_writer.py +348 -31
- lsst/pipe/base/quantum_graph_builder.py +1 -12
- lsst/pipe/base/quantum_graph_executor.py +13 -116
- lsst/pipe/base/quantum_graph_skeleton.py +7 -1
- lsst/pipe/base/separable_pipeline_executor.py +2 -18
- lsst/pipe/base/single_quantum_executor.py +35 -53
- lsst/pipe/base/version.py +1 -1
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/METADATA +1 -1
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/RECORD +28 -30
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/WHEEL +1 -1
- lsst/pipe/base/log_on_close.py +0 -79
- lsst/pipe/base/quantum_graph/formatter.py +0 -101
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/entry_points.txt +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/COPYRIGHT +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/LICENSE +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/bsd_license.txt +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/gpl-v3.0.txt +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/top_level.txt +0 -0
- {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/zip-safe +0 -0
lsst/pipe/base/log_on_close.py
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
# This file is part of pipe_base.
|
|
2
|
-
#
|
|
3
|
-
# Developed for the LSST Data Management System.
|
|
4
|
-
# This product includes software developed by the LSST Project
|
|
5
|
-
# (http://www.lsst.org).
|
|
6
|
-
# See the COPYRIGHT file at the top-level directory of this distribution
|
|
7
|
-
# for details of code ownership.
|
|
8
|
-
#
|
|
9
|
-
# This software is dual licensed under the GNU General Public License and also
|
|
10
|
-
# under a 3-clause BSD license. Recipients may choose which of these licenses
|
|
11
|
-
# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
|
|
12
|
-
# respectively. If you choose the GPL option then the following text applies
|
|
13
|
-
# (but note that there is still no warranty even if you opt for BSD instead):
|
|
14
|
-
#
|
|
15
|
-
# This program is free software: you can redistribute it and/or modify
|
|
16
|
-
# it under the terms of the GNU General Public License as published by
|
|
17
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
18
|
-
# (at your option) any later version.
|
|
19
|
-
#
|
|
20
|
-
# This program is distributed in the hope that it will be useful,
|
|
21
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
22
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
23
|
-
# GNU General Public License for more details.
|
|
24
|
-
#
|
|
25
|
-
# You should have received a copy of the GNU General Public License
|
|
26
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
27
|
-
|
|
28
|
-
from __future__ import annotations
|
|
29
|
-
|
|
30
|
-
__all__ = ("LogOnClose",)
|
|
31
|
-
|
|
32
|
-
from collections.abc import Callable, Iterator
|
|
33
|
-
from contextlib import AbstractContextManager, contextmanager
|
|
34
|
-
from typing import TypeVar
|
|
35
|
-
|
|
36
|
-
from lsst.utils.logging import VERBOSE
|
|
37
|
-
|
|
38
|
-
_T = TypeVar("_T")
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
class LogOnClose:
|
|
42
|
-
"""A factory for context manager wrappers that emit a log message when
|
|
43
|
-
they are closed.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
log_func : `~collections.abc.Callable` [ `int`, `str` ]
|
|
48
|
-
Callable that takes an integer log level and a string message and emits
|
|
49
|
-
a log message. Note that placeholder formatting is not supported.
|
|
50
|
-
"""
|
|
51
|
-
|
|
52
|
-
def __init__(self, log_func: Callable[[int, str], None]):
|
|
53
|
-
self.log_func = log_func
|
|
54
|
-
|
|
55
|
-
def wrap(
|
|
56
|
-
self,
|
|
57
|
-
cm: AbstractContextManager[_T],
|
|
58
|
-
msg: str,
|
|
59
|
-
level: int = VERBOSE,
|
|
60
|
-
) -> AbstractContextManager[_T]:
|
|
61
|
-
"""Wrap a context manager to log when it is exited.
|
|
62
|
-
|
|
63
|
-
Parameters
|
|
64
|
-
----------
|
|
65
|
-
cm : `contextlib.AbstractContextManager`
|
|
66
|
-
Context manager to wrap.
|
|
67
|
-
msg : `str`
|
|
68
|
-
Log message.
|
|
69
|
-
level : `int`, optional
|
|
70
|
-
Log level.
|
|
71
|
-
"""
|
|
72
|
-
|
|
73
|
-
@contextmanager
|
|
74
|
-
def wrapper() -> Iterator[_T]:
|
|
75
|
-
with cm as result:
|
|
76
|
-
yield result
|
|
77
|
-
self.log_func(level, msg)
|
|
78
|
-
|
|
79
|
-
return wrapper()
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
# This file is part of pipe_base.
|
|
2
|
-
#
|
|
3
|
-
# Developed for the LSST Data Management System.
|
|
4
|
-
# This product includes software developed by the LSST Project
|
|
5
|
-
# (http://www.lsst.org).
|
|
6
|
-
# See the COPYRIGHT file at the top-level directory of this distribution
|
|
7
|
-
# for details of code ownership.
|
|
8
|
-
#
|
|
9
|
-
# This software is dual licensed under the GNU General Public License and also
|
|
10
|
-
# under a 3-clause BSD license. Recipients may choose which of these licenses
|
|
11
|
-
# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
|
|
12
|
-
# respectively. If you choose the GPL option then the following text applies
|
|
13
|
-
# (but note that there is still no warranty even if you opt for BSD instead):
|
|
14
|
-
#
|
|
15
|
-
# This program is free software: you can redistribute it and/or modify
|
|
16
|
-
# it under the terms of the GNU General Public License as published by
|
|
17
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
18
|
-
# (at your option) any later version.
|
|
19
|
-
#
|
|
20
|
-
# This program is distributed in the hope that it will be useful,
|
|
21
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
22
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
23
|
-
# GNU General Public License for more details.
|
|
24
|
-
#
|
|
25
|
-
# You should have received a copy of the GNU General Public License
|
|
26
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
27
|
-
|
|
28
|
-
from __future__ import annotations
|
|
29
|
-
|
|
30
|
-
__all__ = ("ProvenanceFormatter",)
|
|
31
|
-
|
|
32
|
-
import uuid
|
|
33
|
-
from typing import Any, ClassVar
|
|
34
|
-
|
|
35
|
-
import pydantic
|
|
36
|
-
|
|
37
|
-
from lsst.daf.butler import FormatterV2
|
|
38
|
-
from lsst.resources import ResourcePath
|
|
39
|
-
from lsst.utils.logging import getLogger
|
|
40
|
-
|
|
41
|
-
from ..pipeline_graph import TaskImportMode
|
|
42
|
-
from ._provenance import ProvenanceQuantumGraphReader
|
|
43
|
-
|
|
44
|
-
_LOG = getLogger(__file__)
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
class _ProvenanceFormatterParameters(pydantic.BaseModel):
|
|
48
|
-
"""A Pydantic model for validating and applying defaults to the
|
|
49
|
-
read parameters of `ProvenanceFormatter`.
|
|
50
|
-
"""
|
|
51
|
-
|
|
52
|
-
import_mode: TaskImportMode = TaskImportMode.DO_NOT_IMPORT
|
|
53
|
-
quanta: list[uuid.UUID] | None = None
|
|
54
|
-
datasets: list[uuid.UUID] | None = None
|
|
55
|
-
read_init_quanta: bool = True
|
|
56
|
-
|
|
57
|
-
@pydantic.field_validator("quanta", mode="before")
|
|
58
|
-
@classmethod
|
|
59
|
-
def quanta_to_list(cls, v: Any) -> list[uuid.UUID]:
|
|
60
|
-
return list(v)
|
|
61
|
-
|
|
62
|
-
@pydantic.field_validator("datasets", mode="before")
|
|
63
|
-
@classmethod
|
|
64
|
-
def datasets_to_list(cls, v: Any) -> list[uuid.UUID]:
|
|
65
|
-
return list(v)
|
|
66
|
-
|
|
67
|
-
@property
|
|
68
|
-
def nodes(self) -> list[uuid.UUID]:
|
|
69
|
-
if self.quanta is not None:
|
|
70
|
-
if self.datasets is not None:
|
|
71
|
-
return self.quanta + self.datasets
|
|
72
|
-
else:
|
|
73
|
-
return self.quanta
|
|
74
|
-
elif self.datasets is not None:
|
|
75
|
-
return self.datasets
|
|
76
|
-
raise ValueError("'datasets' and/or 'quanta' parameters are required for this component")
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
class ProvenanceFormatter(FormatterV2):
|
|
80
|
-
"""Butler interface for reading `ProvenanceQuantumGraph` objects."""
|
|
81
|
-
|
|
82
|
-
default_extension: ClassVar[str] = ".qg"
|
|
83
|
-
can_read_from_uri: ClassVar[bool] = True
|
|
84
|
-
|
|
85
|
-
def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
|
|
86
|
-
parameters = _ProvenanceFormatterParameters.model_validate(self.file_descriptor.parameters or {})
|
|
87
|
-
with ProvenanceQuantumGraphReader.open(uri, import_mode=parameters.import_mode) as reader:
|
|
88
|
-
match component:
|
|
89
|
-
case None:
|
|
90
|
-
if parameters.read_init_quanta:
|
|
91
|
-
reader.read_init_quanta()
|
|
92
|
-
reader.read_quanta(parameters.quanta)
|
|
93
|
-
reader.read_datasets(parameters.datasets)
|
|
94
|
-
return reader.graph
|
|
95
|
-
case "metadata":
|
|
96
|
-
return reader.fetch_metadata(parameters.nodes)
|
|
97
|
-
case "logs":
|
|
98
|
-
return reader.fetch_logs(parameters.nodes)
|
|
99
|
-
case "packages":
|
|
100
|
-
return reader.fetch_packages()
|
|
101
|
-
raise AssertionError(f"Unexpected component {component!r}.")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/bsd_license.txt
RENAMED
|
File without changes
|
{lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/gpl-v3.0.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|