askap-patchwork 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askap_patchwork-0.1.0.dist-info/METADATA +179 -0
- askap_patchwork-0.1.0.dist-info/RECORD +33 -0
- askap_patchwork-0.1.0.dist-info/WHEEL +4 -0
- askap_patchwork-0.1.0.dist-info/entry_points.txt +4 -0
- askap_patchwork-0.1.0.dist-info/licenses/LICENSE +28 -0
- patchwork/__init__.py +11 -0
- patchwork/__main__.py +8 -0
- patchwork/_version.py +24 -0
- patchwork/_version.pyi +2 -0
- patchwork/cli.py +60 -0
- patchwork/combine.py +146 -0
- patchwork/convolve.py +534 -0
- patchwork/coordinates.py +222 -0
- patchwork/frame.py +439 -0
- patchwork/fused.py +618 -0
- patchwork/fused_kernels.py +695 -0
- patchwork/io.py +668 -0
- patchwork/logging.py +62 -0
- patchwork/mosaic.py +546 -0
- patchwork/numba_kernels.py +203 -0
- patchwork/options.py +211 -0
- patchwork/pipeline/__init__.py +3 -0
- patchwork/pipeline/cluster.py +73 -0
- patchwork/pipeline/flow.py +143 -0
- patchwork/plan.py +818 -0
- patchwork/polarisation.py +178 -0
- patchwork/primary_beam.py +636 -0
- patchwork/py.typed +0 -0
- patchwork/resample.py +267 -0
- patchwork/rotate.py +319 -0
- patchwork/tile.py +344 -0
- patchwork/weights.py +175 -0
- patchwork/write.py +502 -0
patchwork/logging.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Logging for patchwork.
|
|
2
|
+
|
|
3
|
+
Importing patchwork only creates its logger, at INFO, so an application that
|
|
4
|
+
uses it (flint, say) decides where its messages go, and sees each once. The
|
|
5
|
+
command line tools call ``setup_logging`` to print them to the terminal.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
from typing import ClassVar
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger("patchwork")
|
|
14
|
+
logger.setLevel(logging.INFO)
|
|
15
|
+
logger.addHandler(logging.NullHandler())
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class CustomFormatter(logging.Formatter):
|
|
19
|
+
"""A custom logger formatter"""
|
|
20
|
+
|
|
21
|
+
grey = "\x1b[38;20m"
|
|
22
|
+
blue = "\x1b[34;20m"
|
|
23
|
+
green = "\x1b[32;20m"
|
|
24
|
+
yellow = "\x1b[33;20m"
|
|
25
|
+
red = "\x1b[31;20m"
|
|
26
|
+
bold_red = "\x1b[31;1m"
|
|
27
|
+
reset = "\x1b[0m"
|
|
28
|
+
format_str = "%(asctime)s.%(msecs)03d %(module)s - %(funcName)s: %(message)s"
|
|
29
|
+
|
|
30
|
+
FORMATS: ClassVar[dict[int, str]] = {
|
|
31
|
+
logging.DEBUG: f"{blue}%(levelname)s{reset} {format_str}",
|
|
32
|
+
logging.INFO: f"{green}%(levelname)s{reset} {format_str}",
|
|
33
|
+
logging.WARNING: f"{yellow}%(levelname)s{reset} {format_str}",
|
|
34
|
+
logging.ERROR: f"{red}%(levelname)s{reset} {format_str}",
|
|
35
|
+
logging.CRITICAL: f"{bold_red}%(levelname)s{reset} {format_str}",
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
39
|
+
log_fmt = self.FORMATS.get(record.levelno)
|
|
40
|
+
formatter = logging.Formatter(log_fmt, "%Y-%m-%d %H:%M:%S")
|
|
41
|
+
return formatter.format(record)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
terminal = logging.StreamHandler()
|
|
45
|
+
"""The handler that prints patchwork's messages once ``setup_logging`` is called"""
|
|
46
|
+
terminal.setFormatter(CustomFormatter())
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def setup_logging(level: int | str = logging.INFO) -> None:
|
|
50
|
+
"""Print patchwork's messages to the terminal, as the command line tools do, along with Python warnings.
|
|
51
|
+
|
|
52
|
+
An application whose root logger already prints messages need not call it,
|
|
53
|
+
as patchwork's messages reach that too. Calling it again only changes the
|
|
54
|
+
level.
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
level (int | str, optional): The lowest level of message printed. Defaults to logging.INFO.
|
|
58
|
+
"""
|
|
59
|
+
logging.captureWarnings(True)
|
|
60
|
+
logger.setLevel(level)
|
|
61
|
+
if terminal not in logger.handlers:
|
|
62
|
+
logger.addHandler(terminal)
|
patchwork/mosaic.py
ADDED
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
"""Linear mosaicking of images and cubes of Stokes I, Q, U and V onto a common frame.
|
|
2
|
+
|
|
3
|
+
Images and cubes are given as one list, with a weight for each. Each one's
|
|
4
|
+
Stokes parameter is read from its STOKES axis, and those on one pixel grid are
|
|
5
|
+
the Stokes parameters of one beam. Each input is resampled onto the frame,
|
|
6
|
+
weighted by its primary beam and noise, and the inputs are
|
|
7
|
+
combined tile by tile, every channel and Stokes parameter in one pass: rotating
|
|
8
|
+
Q and U into the frame of the mosaic, and removing the leakage of Stokes I. The
|
|
9
|
+
inputs can first be convolved to a common beam.
|
|
10
|
+
|
|
11
|
+
The work is handed to a ``concurrent.futures.Executor``: threads or processes
|
|
12
|
+
here, or dask workers through ``patchwork.pipeline.flow``.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import functools
|
|
18
|
+
import logging
|
|
19
|
+
import logging.handlers
|
|
20
|
+
import multiprocessing
|
|
21
|
+
import pickle
|
|
22
|
+
import tempfile
|
|
23
|
+
import time
|
|
24
|
+
import uuid
|
|
25
|
+
from argparse import ArgumentParser, Namespace
|
|
26
|
+
from concurrent.futures import (
|
|
27
|
+
Executor,
|
|
28
|
+
Future,
|
|
29
|
+
ProcessPoolExecutor,
|
|
30
|
+
ThreadPoolExecutor,
|
|
31
|
+
)
|
|
32
|
+
from contextlib import contextmanager
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
35
|
+
|
|
36
|
+
from capn_crunch import add_options_to_parser, create_options_from_parser
|
|
37
|
+
|
|
38
|
+
from patchwork.convolve import channel_target_beams, convolve_cube, convolved_name
|
|
39
|
+
from patchwork.io import BeamShape, check_output_fits, read_image_info
|
|
40
|
+
from patchwork.logging import logger, setup_logging
|
|
41
|
+
from patchwork.options import (
|
|
42
|
+
ConvolveOptions,
|
|
43
|
+
MosaicOptions,
|
|
44
|
+
RunOptions,
|
|
45
|
+
add_mosaic_options_to_parser,
|
|
46
|
+
create_mosaic_options_from_parser,
|
|
47
|
+
)
|
|
48
|
+
from patchwork.plan import (
|
|
49
|
+
STOKES_NAMES,
|
|
50
|
+
MosaicNames,
|
|
51
|
+
check_noise,
|
|
52
|
+
check_spectral_axes,
|
|
53
|
+
group_inputs,
|
|
54
|
+
input_frame_centre,
|
|
55
|
+
needs_image_weight,
|
|
56
|
+
output_names,
|
|
57
|
+
plan_mosaic,
|
|
58
|
+
read_weight,
|
|
59
|
+
rotates,
|
|
60
|
+
)
|
|
61
|
+
from patchwork.polarisation import STOKES_CODES
|
|
62
|
+
from patchwork.primary_beam import beam_model
|
|
63
|
+
from patchwork.resample import available_cpus, tile_threads
|
|
64
|
+
from patchwork.tile import PlaneWeight
|
|
65
|
+
from patchwork.weights import measure_channel_weights
|
|
66
|
+
from patchwork.write import (
|
|
67
|
+
_duration,
|
|
68
|
+
bounded_results,
|
|
69
|
+
create_outputs,
|
|
70
|
+
logged_progress,
|
|
71
|
+
run_tiles,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
if TYPE_CHECKING:
|
|
75
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class _PlanFile:
|
|
79
|
+
"""A mosaic's plan in a file. A process it is sent to reads the file once, however many of its tasks are sent it"""
|
|
80
|
+
|
|
81
|
+
def __init__(self, path: Path) -> None:
|
|
82
|
+
self.path = path
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def write(cls, directory: Path, plan: Any) -> _PlanFile:
|
|
86
|
+
"""Write a plan to a new file in a directory."""
|
|
87
|
+
path = directory / f"plan-{uuid.uuid4().hex}.pickle"
|
|
88
|
+
with path.open("wb") as file:
|
|
89
|
+
pickle.dump(plan, file, protocol=pickle.HIGHEST_PROTOCOL)
|
|
90
|
+
return cls(path)
|
|
91
|
+
|
|
92
|
+
def __reduce__(self) -> tuple[Callable[[str], Any], tuple[str]]:
|
|
93
|
+
# Unpickled in a process, it becomes the plan itself, so tasks need not know it was sent this way
|
|
94
|
+
return _read_plan, (str(self.path),)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@functools.lru_cache(maxsize=1)
|
|
98
|
+
def _read_plan(path: str) -> Any:
|
|
99
|
+
"""The plan in a file, which each process reads once."""
|
|
100
|
+
with Path(path).open("rb") as file:
|
|
101
|
+
return pickle.load(file)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class _Relay(logging.Handler):
|
|
105
|
+
"""Hands the messages of patchwork's processes to the logger they were logged to in this process, as though logged here"""
|
|
106
|
+
|
|
107
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
108
|
+
logging.getLogger(record.name).handle(record)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _start_process(level: int, messages: Any) -> None:
|
|
112
|
+
"""Send patchwork's messages from a new process to a queue the process that started it reads."""
|
|
113
|
+
for handler in list(logger.handlers):
|
|
114
|
+
logger.removeHandler(handler)
|
|
115
|
+
logger.addHandler(logging.handlers.QueueHandler(messages))
|
|
116
|
+
logger.setLevel(level)
|
|
117
|
+
logger.propagate = False
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@contextmanager
|
|
121
|
+
def _own_pool(
|
|
122
|
+
max_workers: int, processes: bool
|
|
123
|
+
) -> Iterator[tuple[Executor, Callable[[Any], Any] | None]]:
|
|
124
|
+
"""A pool of threads, or of processes along with a way of sending the plan to them once.
|
|
125
|
+
|
|
126
|
+
What a process logs is passed to this process, so it is printed, or reaches
|
|
127
|
+
an application's own handlers, as what a thread logs would be.
|
|
128
|
+
|
|
129
|
+
Raises:
|
|
130
|
+
ValueError: Raised if processes are asked for in a daemonic process, which cannot start them
|
|
131
|
+
"""
|
|
132
|
+
if not processes:
|
|
133
|
+
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
|
134
|
+
yield pool, None
|
|
135
|
+
return
|
|
136
|
+
if multiprocessing.current_process().daemon:
|
|
137
|
+
msg = (
|
|
138
|
+
"patchwork cannot start processes from a daemonic process, as a dask worker is. "
|
|
139
|
+
"Use threads, or run patchwork in a subprocess"
|
|
140
|
+
)
|
|
141
|
+
raise ValueError(msg)
|
|
142
|
+
# Spawned, not forked, so a process never inherits a lock some thread of this one held
|
|
143
|
+
context = multiprocessing.get_context("spawn")
|
|
144
|
+
messages = context.Queue()
|
|
145
|
+
listener = logging.handlers.QueueListener(messages, _Relay())
|
|
146
|
+
listener.start()
|
|
147
|
+
try:
|
|
148
|
+
with (
|
|
149
|
+
tempfile.TemporaryDirectory(prefix="patchwork-plan-") as directory,
|
|
150
|
+
ProcessPoolExecutor(
|
|
151
|
+
max_workers=max_workers,
|
|
152
|
+
mp_context=context,
|
|
153
|
+
initializer=_start_process,
|
|
154
|
+
initargs=(logger.getEffectiveLevel(), messages),
|
|
155
|
+
) as pool,
|
|
156
|
+
):
|
|
157
|
+
yield pool, functools.partial(_PlanFile.write, Path(directory))
|
|
158
|
+
finally:
|
|
159
|
+
# Every process has stopped, so this passes on what is left and ends
|
|
160
|
+
listener.stop()
|
|
161
|
+
messages.close()
|
|
162
|
+
messages.join_thread()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def mosaic(
|
|
166
|
+
images: list[Path],
|
|
167
|
+
output_base: str | Path,
|
|
168
|
+
options: MosaicOptions = MosaicOptions(), # noqa: B008
|
|
169
|
+
run_options: RunOptions = RunOptions(), # noqa: B008
|
|
170
|
+
convolve_options: ConvolveOptions = ConvolveOptions(), # noqa: B008
|
|
171
|
+
weights: list[Path] | None = None,
|
|
172
|
+
stokes_i_images: list[Path] | None = None,
|
|
173
|
+
beam: BeamShape | None = None,
|
|
174
|
+
beam_numbers: list[int] | None = None,
|
|
175
|
+
overwrite: bool = False,
|
|
176
|
+
executor: Executor | None = None,
|
|
177
|
+
share: Callable[[Any], Any] | None = None,
|
|
178
|
+
max_workers: int = 1,
|
|
179
|
+
capacity: Callable[[], int] | None = None,
|
|
180
|
+
processes: bool = False,
|
|
181
|
+
) -> MosaicNames:
|
|
182
|
+
"""Mosaic images or cubes of Stokes I, Q, U and V, optionally convolving them to a common beam first.
|
|
183
|
+
|
|
184
|
+
Args:
|
|
185
|
+
images (list[Path]): The images or cubes, in any order: their Stokes parameters come from their headers, and their beams from their grids
|
|
186
|
+
output_base (str | Path): Base name of the output files
|
|
187
|
+
options (MosaicOptions, optional): The mosaicking options. Defaults to MosaicOptions().
|
|
188
|
+
run_options (RunOptions, optional): Which channels are mosaicked, whether the inputs are convolved first, and how the mosaic is written. Defaults to RunOptions().
|
|
189
|
+
convolve_options (ConvolveOptions, optional): How the common beam is found and convolved to. Defaults to ConvolveOptions().
|
|
190
|
+
weights (list[Path] | None, optional): A weights log (``.txt``) or weight image for each image, in the same order. Defaults to None, weighting by the noise of each channel.
|
|
191
|
+
stokes_i_images (list[Path] | None, optional): Stokes I images, matched to the beams by their grid, to remove the leakage with but not mosaic. Defaults to None, using the beams' own.
|
|
192
|
+
beam (BeamShape | None, optional): Restoring beam recorded for every channel of the mosaic. Defaults to None, the beams convolved to, or else those of the first input.
|
|
193
|
+
beam_numbers (list[int] | None, optional): The holography beam of each image. Defaults to None, read from their names.
|
|
194
|
+
overwrite (bool, optional): Replace existing outputs. Defaults to False.
|
|
195
|
+
executor (Executor | None, optional): Runs the tasks. Defaults to None, a pool of ``max_workers`` threads, or processes with ``processes``.
|
|
196
|
+
share (Callable[[Any], Any] | None, optional): Turns what every task needs into what is sent to it, such as dask's ``scatter``. Defaults to None, sending it as it is to threads, and once to each process.
|
|
197
|
+
max_workers (int, optional): Threads or processes of the default executor, or how many tasks a given executor runs at once. Defaults to 1.
|
|
198
|
+
capacity (Callable[[], int] | None, optional): How many tasks the executor runs at once, asked again as tiles are made, such as the threads of a dask cluster as it scales. Defaults to None, ``max_workers``.
|
|
199
|
+
processes (bool, optional): Make the default executor a pool of processes rather than threads. Python code holding the GIL then holds up only its own process, which matters on machines with many cores. What they log is passed to this process. The processes are spawned, so a script that calls this must do so under ``if __name__ == "__main__":``. Defaults to False.
|
|
200
|
+
|
|
201
|
+
Raises:
|
|
202
|
+
ValueError: Raised if the weights or beam numbers do not match the images, the channels are not among theirs, or ``processes`` is given with an executor or in a daemonic process
|
|
203
|
+
|
|
204
|
+
Returns:
|
|
205
|
+
MosaicNames: The files the mosaic was written to
|
|
206
|
+
"""
|
|
207
|
+
for name, given in (("weights", weights), ("beam numbers", beam_numbers)):
|
|
208
|
+
if given is not None and len(given) != len(images):
|
|
209
|
+
msg = f"Got {len(given)} {name} for {len(images)} images"
|
|
210
|
+
raise ValueError(msg)
|
|
211
|
+
if processes and executor is not None:
|
|
212
|
+
msg = "processes makes patchwork's own pool of processes, so cannot be given with an executor"
|
|
213
|
+
raise ValueError(msg)
|
|
214
|
+
if executor is None:
|
|
215
|
+
if options.resample.tile_threads == 0:
|
|
216
|
+
# The threads or processes of the pool each make a tile at once, on this machine
|
|
217
|
+
shared = max(1, available_cpus() // max(max_workers, 1))
|
|
218
|
+
options = options._replace(
|
|
219
|
+
resample=options.resample.model_copy(update={"tile_threads": shared})
|
|
220
|
+
)
|
|
221
|
+
with _own_pool(max_workers, processes) as (pool, pool_share):
|
|
222
|
+
return mosaic(
|
|
223
|
+
images,
|
|
224
|
+
output_base,
|
|
225
|
+
options,
|
|
226
|
+
run_options,
|
|
227
|
+
convolve_options,
|
|
228
|
+
weights,
|
|
229
|
+
stokes_i_images,
|
|
230
|
+
beam,
|
|
231
|
+
beam_numbers,
|
|
232
|
+
overwrite,
|
|
233
|
+
executor=pool,
|
|
234
|
+
share=share or pool_share,
|
|
235
|
+
max_workers=max_workers,
|
|
236
|
+
capacity=capacity,
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
infos = [
|
|
240
|
+
read_image_info(
|
|
241
|
+
image, beam_number=None if beam_numbers is None else beam_numbers[index]
|
|
242
|
+
)
|
|
243
|
+
for index, image in enumerate(images)
|
|
244
|
+
]
|
|
245
|
+
stokes_i = [read_image_info(image) for image in stokes_i_images or []]
|
|
246
|
+
check_spectral_axes(infos + stokes_i)
|
|
247
|
+
n_channels = infos[0].n_channels
|
|
248
|
+
start, stop = run_options.channels or (0, n_channels)
|
|
249
|
+
if not 0 <= start < stop <= n_channels:
|
|
250
|
+
msg = f"Channels {start} to {stop} are not a range within the {n_channels} channels of the inputs"
|
|
251
|
+
raise ValueError(msg)
|
|
252
|
+
channels = slice(start, stop)
|
|
253
|
+
given_beams = None if beam is None else (beam,) * n_channels
|
|
254
|
+
plane_weights = (
|
|
255
|
+
None
|
|
256
|
+
if weights is None
|
|
257
|
+
else [
|
|
258
|
+
read_weight(weight, info)
|
|
259
|
+
for weight, info in zip(weights, infos, strict=True)
|
|
260
|
+
]
|
|
261
|
+
)
|
|
262
|
+
remove_leakage = options.polarisation.remove_leakage
|
|
263
|
+
# Check the images form inputs, whose Q and U frames are known, and nothing will be written over them, before any work
|
|
264
|
+
for item in group_inputs(
|
|
265
|
+
infos, plane_weights or [PlaneWeight()] * len(infos), remove_leakage, stokes_i
|
|
266
|
+
):
|
|
267
|
+
if rotates(item.stokes):
|
|
268
|
+
input_frame_centre(item, options.polarisation)
|
|
269
|
+
# Build each input's primary beam now, so a model that cannot be built fails before any convolving
|
|
270
|
+
leaky = remove_leakage and any(info.stokes != STOKES_CODES["I"] for info in infos)
|
|
271
|
+
if options.weight.weight_type != "weight_image" or leaky:
|
|
272
|
+
for info in infos:
|
|
273
|
+
beam_model(options.primary_beam, info, leakage=leaky)
|
|
274
|
+
names = output_names(
|
|
275
|
+
output_base,
|
|
276
|
+
tuple(sorted({info.stokes or 0 for info in infos})),
|
|
277
|
+
cube=n_channels > 1,
|
|
278
|
+
output_format=run_options.output_format,
|
|
279
|
+
noise=options.weight.noise_image,
|
|
280
|
+
)
|
|
281
|
+
check_noise(options.weight, weighted_by_images=weights is not None)
|
|
282
|
+
originals = [*images, *(weights or []), *(stokes_i_images or [])]
|
|
283
|
+
for output in (*names.images, *names.weights, *names.noises):
|
|
284
|
+
check_output_fits(output, overwrite=overwrite, input_fits=originals)
|
|
285
|
+
|
|
286
|
+
stages: list[tuple[str, float]] = []
|
|
287
|
+
stage_started = time.monotonic()
|
|
288
|
+
beams = None
|
|
289
|
+
convolved: list[Path] = []
|
|
290
|
+
if run_options.convolve != "none":
|
|
291
|
+
beams = channel_target_beams(
|
|
292
|
+
infos + stokes_i, run_options.convolve, convolve_options, channels
|
|
293
|
+
)
|
|
294
|
+
# Runs over different channels each convolve to their own files
|
|
295
|
+
suffix = convolve_options.suffix + (
|
|
296
|
+
f".ch{start}-{stop}" if run_options.channels else ""
|
|
297
|
+
)
|
|
298
|
+
to_convolve = infos + stokes_i
|
|
299
|
+
outputs = [convolved_name(info.path, suffix) for info in to_convolve]
|
|
300
|
+
for output in outputs:
|
|
301
|
+
check_output_fits(output, overwrite=overwrite, input_fits=originals)
|
|
302
|
+
logger.info(
|
|
303
|
+
f"Convolving {len(to_convolve)} inputs to {len(set(beams[channels]))} beams"
|
|
304
|
+
)
|
|
305
|
+
# The outputs are known, so the cubes are taken as they finish, and counted
|
|
306
|
+
calls = [
|
|
307
|
+
(
|
|
308
|
+
_convolve_cube,
|
|
309
|
+
(
|
|
310
|
+
info.path,
|
|
311
|
+
beams,
|
|
312
|
+
output,
|
|
313
|
+
convolve_options.beam_cutoff_arcsec,
|
|
314
|
+
overwrite,
|
|
315
|
+
channels,
|
|
316
|
+
options.resample.tile_threads,
|
|
317
|
+
),
|
|
318
|
+
)
|
|
319
|
+
for info, output in zip(to_convolve, outputs, strict=True)
|
|
320
|
+
]
|
|
321
|
+
list(
|
|
322
|
+
logged_progress(
|
|
323
|
+
bounded_results(executor, calls, lambda: len(calls)),
|
|
324
|
+
len(calls),
|
|
325
|
+
"convolved cubes",
|
|
326
|
+
)
|
|
327
|
+
)
|
|
328
|
+
convolved = list(outputs)
|
|
329
|
+
stages.append(("convolving", time.monotonic() - stage_started))
|
|
330
|
+
stage_started = time.monotonic()
|
|
331
|
+
n_images = len(infos)
|
|
332
|
+
infos = [
|
|
333
|
+
read_image_info(path, beam_number=info.beam_number)
|
|
334
|
+
for path, info in zip(convolved[:n_images], infos, strict=True)
|
|
335
|
+
]
|
|
336
|
+
stokes_i = [read_image_info(path) for path in convolved[n_images:]]
|
|
337
|
+
|
|
338
|
+
if plane_weights is None:
|
|
339
|
+
measure = needs_image_weight(options.weight, None)
|
|
340
|
+
plane_weights = [
|
|
341
|
+
PlaneWeight(values=values)
|
|
342
|
+
for values in _results_in_order(
|
|
343
|
+
executor.submit(
|
|
344
|
+
_measure_weights,
|
|
345
|
+
info.path,
|
|
346
|
+
mode=options.weight.image_weight if measure else "none",
|
|
347
|
+
stride=options.weight.weight_stride,
|
|
348
|
+
channels=channels,
|
|
349
|
+
threads=options.resample.tile_threads,
|
|
350
|
+
)
|
|
351
|
+
for info in infos
|
|
352
|
+
)
|
|
353
|
+
]
|
|
354
|
+
|
|
355
|
+
stages.append(("measuring noise", time.monotonic() - stage_started))
|
|
356
|
+
stage_started = time.monotonic()
|
|
357
|
+
|
|
358
|
+
inputs = group_inputs(infos, plane_weights, remove_leakage, stokes_i)
|
|
359
|
+
logger.info(
|
|
360
|
+
f"Found {len(inputs)} inputs of Stokes {', '.join(STOKES_NAMES.get(code, '?') for code in inputs[0].stokes)}"
|
|
361
|
+
)
|
|
362
|
+
plan = plan_mosaic(
|
|
363
|
+
inputs,
|
|
364
|
+
names,
|
|
365
|
+
options,
|
|
366
|
+
run_options,
|
|
367
|
+
beams=beams if given_beams is None else given_beams,
|
|
368
|
+
)
|
|
369
|
+
create_outputs(plan, run_options)
|
|
370
|
+
run_tiles(
|
|
371
|
+
plan,
|
|
372
|
+
options,
|
|
373
|
+
run_options,
|
|
374
|
+
executor,
|
|
375
|
+
share,
|
|
376
|
+
capacity=capacity or (lambda: max_workers),
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
stages.append(("making the mosaic", time.monotonic() - stage_started))
|
|
380
|
+
|
|
381
|
+
if convolved and not run_options.keep_convolved:
|
|
382
|
+
for path in convolved:
|
|
383
|
+
path.unlink()
|
|
384
|
+
logger.info(f"Removed the {len(convolved)} convolved inputs")
|
|
385
|
+
logger.info(
|
|
386
|
+
f"Took {_duration(sum(seconds for _, seconds in stages))}: "
|
|
387
|
+
+ ", ".join(f"{name} {_duration(seconds)}" for name, seconds in stages)
|
|
388
|
+
)
|
|
389
|
+
logger.info(
|
|
390
|
+
f"Written {', '.join(map(str, (*names.images, *names.weights, *names.noises)))}"
|
|
391
|
+
)
|
|
392
|
+
return names
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _convolve_cube(
|
|
396
|
+
path: Path,
|
|
397
|
+
beams: tuple[BeamShape, ...],
|
|
398
|
+
output: Path,
|
|
399
|
+
cutoff_arcsec: float | None,
|
|
400
|
+
overwrite: bool,
|
|
401
|
+
channels: slice | None,
|
|
402
|
+
threads: int,
|
|
403
|
+
) -> Path:
|
|
404
|
+
"""Convolve a cube in the threads a tile would be made with, resolved where it runs, as ``_measure_weights`` does."""
|
|
405
|
+
return convolve_cube(
|
|
406
|
+
path,
|
|
407
|
+
beams,
|
|
408
|
+
output,
|
|
409
|
+
cutoff_arcsec,
|
|
410
|
+
overwrite,
|
|
411
|
+
channels,
|
|
412
|
+
threads=tile_threads(threads),
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _measure_weights(
|
|
417
|
+
path: Path,
|
|
418
|
+
mode: Literal["mad", "std", "none"],
|
|
419
|
+
stride: int,
|
|
420
|
+
channels: slice | None,
|
|
421
|
+
threads: int,
|
|
422
|
+
) -> tuple[float, ...]:
|
|
423
|
+
"""Measure the weights of a cube's channels in the threads a tile would be made with, with whichever backend.
|
|
424
|
+
|
|
425
|
+
Runs where the tiles are made, so ``threads`` of 0 is resolved there: on a
|
|
426
|
+
dask worker, to its CPUs over its threads.
|
|
427
|
+
"""
|
|
428
|
+
return measure_channel_weights(
|
|
429
|
+
path,
|
|
430
|
+
mode=mode,
|
|
431
|
+
stride=stride,
|
|
432
|
+
channels=channels,
|
|
433
|
+
threads=tile_threads(threads),
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _results_in_order(futures: Iterable[Future[Any]]) -> list[Any]:
|
|
438
|
+
"""The results of futures, in the order they were submitted, cancelling those still waiting should one fail."""
|
|
439
|
+
submitted = list(futures)
|
|
440
|
+
try:
|
|
441
|
+
return [future.result() for future in submitted]
|
|
442
|
+
finally:
|
|
443
|
+
for future in submitted:
|
|
444
|
+
future.cancel()
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def add_mosaic_arguments(parser: ArgumentParser) -> ArgumentParser:
|
|
448
|
+
"""Add the inputs, outputs and options of a mosaic to a parser."""
|
|
449
|
+
parser.add_argument(
|
|
450
|
+
"images",
|
|
451
|
+
nargs="+",
|
|
452
|
+
type=Path,
|
|
453
|
+
help="The images or cubes, in any order. Their Stokes parameters are read from their headers, and those on one pixel grid are taken to be one beam",
|
|
454
|
+
)
|
|
455
|
+
parser.add_argument(
|
|
456
|
+
"--weights",
|
|
457
|
+
nargs="+",
|
|
458
|
+
type=Path,
|
|
459
|
+
default=None,
|
|
460
|
+
help="A weights log (.txt) or weight image for each image, in the same order. If not given each channel is weighted by its noise",
|
|
461
|
+
)
|
|
462
|
+
parser.add_argument(
|
|
463
|
+
"--stokes-i-images",
|
|
464
|
+
nargs="+",
|
|
465
|
+
type=Path,
|
|
466
|
+
default=None,
|
|
467
|
+
help="Stokes I images to remove the leakage with, but not mosaic, matched to the beams by their grid",
|
|
468
|
+
)
|
|
469
|
+
parser.add_argument(
|
|
470
|
+
"--output-base",
|
|
471
|
+
type=Path,
|
|
472
|
+
default=Path("mosaic"),
|
|
473
|
+
help="Base name of the outputs: <base>.<stokes>.mosaic.fits and <base>.<stokes>.weight.fits, or <base>.<stokes>.cube.fits and <base>.<stokes>.weight.cube.fits for cubes (or .zarr)",
|
|
474
|
+
)
|
|
475
|
+
parser.add_argument(
|
|
476
|
+
"--overwrite", action="store_true", help="Replace existing outputs"
|
|
477
|
+
)
|
|
478
|
+
add_mosaic_options_to_parser(parser)
|
|
479
|
+
add_options_to_parser(parser=parser, options_class=RunOptions)
|
|
480
|
+
return add_options_to_parser(parser=parser, options_class=ConvolveOptions)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def options_from_args(
|
|
484
|
+
args: Namespace,
|
|
485
|
+
) -> tuple[MosaicOptions, RunOptions, ConvolveOptions]:
|
|
486
|
+
"""The options of a mosaic from parsed arguments."""
|
|
487
|
+
return (
|
|
488
|
+
create_mosaic_options_from_parser(args),
|
|
489
|
+
create_options_from_parser(parser_namespace=args, options_class=RunOptions),
|
|
490
|
+
create_options_from_parser(
|
|
491
|
+
parser_namespace=args, options_class=ConvolveOptions
|
|
492
|
+
),
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def get_parser(parser: ArgumentParser | None = None) -> ArgumentParser:
|
|
497
|
+
"""The parser of the ``mosaic`` command."""
|
|
498
|
+
if parser is None:
|
|
499
|
+
parser = ArgumentParser(description=__doc__)
|
|
500
|
+
parser.add_argument(
|
|
501
|
+
"--max-workers",
|
|
502
|
+
type=int,
|
|
503
|
+
default=1,
|
|
504
|
+
help="Number of threads, or processes with --processes, that convolve, measure noise and make tiles",
|
|
505
|
+
)
|
|
506
|
+
parser.add_argument(
|
|
507
|
+
"--processes",
|
|
508
|
+
action="store_true",
|
|
509
|
+
help="Work in --max-workers processes rather than threads. Threads share Python's lock (the GIL), so on a machine with many cores they leave some idle",
|
|
510
|
+
)
|
|
511
|
+
return add_log_level_argument(add_mosaic_arguments(parser))
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def add_log_level_argument(parser: ArgumentParser) -> ArgumentParser:
|
|
515
|
+
"""Add ``--log-level`` to a command's parser."""
|
|
516
|
+
parser.add_argument(
|
|
517
|
+
"--log-level",
|
|
518
|
+
default="INFO",
|
|
519
|
+
choices=["DEBUG", "INFO", "WARNING", "ERROR"],
|
|
520
|
+
help="The lowest level of message printed. DEBUG logs where the time of each tile went",
|
|
521
|
+
)
|
|
522
|
+
return parser
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def cli(args: Namespace | None = None) -> None:
|
|
526
|
+
"""Run the ``mosaic`` command."""
|
|
527
|
+
setup_logging()
|
|
528
|
+
args = get_parser().parse_args() if args is None else args
|
|
529
|
+
setup_logging(args.log_level)
|
|
530
|
+
options, run_options, convolve_options = options_from_args(args)
|
|
531
|
+
mosaic(
|
|
532
|
+
images=args.images,
|
|
533
|
+
output_base=args.output_base,
|
|
534
|
+
options=options,
|
|
535
|
+
run_options=run_options,
|
|
536
|
+
convolve_options=convolve_options,
|
|
537
|
+
weights=args.weights,
|
|
538
|
+
stokes_i_images=args.stokes_i_images,
|
|
539
|
+
overwrite=args.overwrite,
|
|
540
|
+
max_workers=args.max_workers,
|
|
541
|
+
processes=args.processes,
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
if __name__ == "__main__":
|
|
546
|
+
cli()
|