lsst-pipe-base 30.0.0__py3-none-any.whl → 30.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. lsst/pipe/base/_instrument.py +5 -6
  2. lsst/pipe/base/log_capture.py +79 -39
  3. lsst/pipe/base/mp_graph_executor.py +15 -51
  4. lsst/pipe/base/quantum_graph/_common.py +3 -4
  5. lsst/pipe/base/quantum_graph/_multiblock.py +16 -6
  6. lsst/pipe/base/quantum_graph/_predicted.py +12 -106
  7. lsst/pipe/base/quantum_graph/_provenance.py +6 -657
  8. lsst/pipe/base/quantum_graph/aggregator/_communicators.py +50 -18
  9. lsst/pipe/base/quantum_graph/aggregator/_scanner.py +229 -35
  10. lsst/pipe/base/quantum_graph/aggregator/_structs.py +113 -3
  11. lsst/pipe/base/quantum_graph/aggregator/_supervisor.py +5 -10
  12. lsst/pipe/base/quantum_graph/aggregator/_writer.py +348 -31
  13. lsst/pipe/base/quantum_graph_builder.py +1 -12
  14. lsst/pipe/base/quantum_graph_executor.py +13 -116
  15. lsst/pipe/base/quantum_graph_skeleton.py +7 -1
  16. lsst/pipe/base/separable_pipeline_executor.py +2 -18
  17. lsst/pipe/base/single_quantum_executor.py +35 -53
  18. lsst/pipe/base/version.py +1 -1
  19. {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/METADATA +1 -1
  20. {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/RECORD +28 -30
  21. {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/WHEEL +1 -1
  22. lsst/pipe/base/log_on_close.py +0 -79
  23. lsst/pipe/base/quantum_graph/formatter.py +0 -101
  24. {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/entry_points.txt +0 -0
  25. {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/COPYRIGHT +0 -0
  26. {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/LICENSE +0 -0
  27. {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/bsd_license.txt +0 -0
  28. {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/licenses/gpl-v3.0.txt +0 -0
  29. {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/top_level.txt +0 -0
  30. {lsst_pipe_base-30.0.0.dist-info → lsst_pipe_base-30.0.0rc1.dist-info}/zip-safe +0 -0
@@ -1,79 +0,0 @@
1
- # This file is part of pipe_base.
2
- #
3
- # Developed for the LSST Data Management System.
4
- # This product includes software developed by the LSST Project
5
- # (http://www.lsst.org).
6
- # See the COPYRIGHT file at the top-level directory of this distribution
7
- # for details of code ownership.
8
- #
9
- # This software is dual licensed under the GNU General Public License and also
10
- # under a 3-clause BSD license. Recipients may choose which of these licenses
11
- # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
- # respectively. If you choose the GPL option then the following text applies
13
- # (but note that there is still no warranty even if you opt for BSD instead):
14
- #
15
- # This program is free software: you can redistribute it and/or modify
16
- # it under the terms of the GNU General Public License as published by
17
- # the Free Software Foundation, either version 3 of the License, or
18
- # (at your option) any later version.
19
- #
20
- # This program is distributed in the hope that it will be useful,
21
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
- # GNU General Public License for more details.
24
- #
25
- # You should have received a copy of the GNU General Public License
26
- # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
-
28
- from __future__ import annotations
29
-
30
- __all__ = ("LogOnClose",)
31
-
32
- from collections.abc import Callable, Iterator
33
- from contextlib import AbstractContextManager, contextmanager
34
- from typing import TypeVar
35
-
36
- from lsst.utils.logging import VERBOSE
37
-
38
- _T = TypeVar("_T")
39
-
40
-
41
- class LogOnClose:
42
- """A factory for context manager wrappers that emit a log message when
43
- they are closed.
44
-
45
- Parameters
46
- ----------
47
- log_func : `~collections.abc.Callable` [ `int`, `str` ]
48
- Callable that takes an integer log level and a string message and emits
49
- a log message. Note that placeholder formatting is not supported.
50
- """
51
-
52
- def __init__(self, log_func: Callable[[int, str], None]):
53
- self.log_func = log_func
54
-
55
- def wrap(
56
- self,
57
- cm: AbstractContextManager[_T],
58
- msg: str,
59
- level: int = VERBOSE,
60
- ) -> AbstractContextManager[_T]:
61
- """Wrap a context manager to log when it is exited.
62
-
63
- Parameters
64
- ----------
65
- cm : `contextlib.AbstractContextManager`
66
- Context manager to wrap.
67
- msg : `str`
68
- Log message.
69
- level : `int`, optional
70
- Log level.
71
- """
72
-
73
- @contextmanager
74
- def wrapper() -> Iterator[_T]:
75
- with cm as result:
76
- yield result
77
- self.log_func(level, msg)
78
-
79
- return wrapper()
@@ -1,101 +0,0 @@
1
- # This file is part of pipe_base.
2
- #
3
- # Developed for the LSST Data Management System.
4
- # This product includes software developed by the LSST Project
5
- # (http://www.lsst.org).
6
- # See the COPYRIGHT file at the top-level directory of this distribution
7
- # for details of code ownership.
8
- #
9
- # This software is dual licensed under the GNU General Public License and also
10
- # under a 3-clause BSD license. Recipients may choose which of these licenses
11
- # to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12
- # respectively. If you choose the GPL option then the following text applies
13
- # (but note that there is still no warranty even if you opt for BSD instead):
14
- #
15
- # This program is free software: you can redistribute it and/or modify
16
- # it under the terms of the GNU General Public License as published by
17
- # the Free Software Foundation, either version 3 of the License, or
18
- # (at your option) any later version.
19
- #
20
- # This program is distributed in the hope that it will be useful,
21
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
22
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23
- # GNU General Public License for more details.
24
- #
25
- # You should have received a copy of the GNU General Public License
26
- # along with this program. If not, see <http://www.gnu.org/licenses/>.
27
-
28
- from __future__ import annotations
29
-
30
- __all__ = ("ProvenanceFormatter",)
31
-
32
- import uuid
33
- from typing import Any, ClassVar
34
-
35
- import pydantic
36
-
37
- from lsst.daf.butler import FormatterV2
38
- from lsst.resources import ResourcePath
39
- from lsst.utils.logging import getLogger
40
-
41
- from ..pipeline_graph import TaskImportMode
42
- from ._provenance import ProvenanceQuantumGraphReader
43
-
44
- _LOG = getLogger(__file__)
45
-
46
-
47
- class _ProvenanceFormatterParameters(pydantic.BaseModel):
48
- """A Pydantic model for validating and applying defaults to the
49
- read parameters of `ProvenanceFormatter`.
50
- """
51
-
52
- import_mode: TaskImportMode = TaskImportMode.DO_NOT_IMPORT
53
- quanta: list[uuid.UUID] | None = None
54
- datasets: list[uuid.UUID] | None = None
55
- read_init_quanta: bool = True
56
-
57
- @pydantic.field_validator("quanta", mode="before")
58
- @classmethod
59
- def quanta_to_list(cls, v: Any) -> list[uuid.UUID]:
60
- return list(v)
61
-
62
- @pydantic.field_validator("datasets", mode="before")
63
- @classmethod
64
- def datasets_to_list(cls, v: Any) -> list[uuid.UUID]:
65
- return list(v)
66
-
67
- @property
68
- def nodes(self) -> list[uuid.UUID]:
69
- if self.quanta is not None:
70
- if self.datasets is not None:
71
- return self.quanta + self.datasets
72
- else:
73
- return self.quanta
74
- elif self.datasets is not None:
75
- return self.datasets
76
- raise ValueError("'datasets' and/or 'quanta' parameters are required for this component")
77
-
78
-
79
- class ProvenanceFormatter(FormatterV2):
80
- """Butler interface for reading `ProvenanceQuantumGraph` objects."""
81
-
82
- default_extension: ClassVar[str] = ".qg"
83
- can_read_from_uri: ClassVar[bool] = True
84
-
85
- def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
86
- parameters = _ProvenanceFormatterParameters.model_validate(self.file_descriptor.parameters or {})
87
- with ProvenanceQuantumGraphReader.open(uri, import_mode=parameters.import_mode) as reader:
88
- match component:
89
- case None:
90
- if parameters.read_init_quanta:
91
- reader.read_init_quanta()
92
- reader.read_quanta(parameters.quanta)
93
- reader.read_datasets(parameters.datasets)
94
- return reader.graph
95
- case "metadata":
96
- return reader.fetch_metadata(parameters.nodes)
97
- case "logs":
98
- return reader.fetch_logs(parameters.nodes)
99
- case "packages":
100
- return reader.fetch_packages()
101
- raise AssertionError(f"Unexpected component {component!r}.")