askap-patchwork 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
patchwork/fused.py ADDED
@@ -0,0 +1,618 @@
1
+ """The fused backend: a tile made in one compiled pass over each input, on as many threads as it is given.
2
+
3
+ The other backends interpolate an input with a compiled kernel (or numpy),
4
+ then weight and add it with numpy, an array of the whole region at a time.
5
+ That half of the work runs on one core, and holds the GIL. Here all of it is
6
+ done by ``patchwork.fused_kernels``, pixel by pixel, and Python only finds
7
+ where each input falls on the tile and reads it.
8
+
9
+ With more than one thread, a tile is made by three things at once:
10
+
11
+ - threads of this module read the inputs a few ahead of the one being added,
12
+ which is mostly waiting for the file system,
13
+ - numba's threads add each input in turn, sharing the pixels of the tile,
14
+ - and they then turn the sums into the tile.
15
+
16
+ This suits a dask worker that runs one task at a time and has many cores, as
17
+ a cluster set up for a multithreaded imager has: each task then uses all of
18
+ them. See ``tile_threads`` for how many threads a tile is given.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import functools
24
+ import logging
25
+ import threading
26
+ import time
27
+ from collections import deque
28
+ from concurrent.futures import Executor, Future, ThreadPoolExecutor
29
+ from typing import TYPE_CHECKING, NamedTuple
30
+
31
+ import numpy as np
32
+
33
+ from patchwork import fused_kernels as kernels
34
+ from patchwork.combine import TileSums
35
+ from patchwork.coordinates import east_north, map_to_input, tile_coordinates
36
+ from patchwork.frame import tile_region
37
+ from patchwork.io import read_window
38
+ from patchwork.logging import logger
39
+ from patchwork.polarisation import STOKES_CODES
40
+ from patchwork.resample import input_region, kernel_width, tile_threads
41
+ from patchwork.tile import (
42
+ MosaicInput,
43
+ TileResult,
44
+ _add_input,
45
+ group_members,
46
+ weight_groups,
47
+ )
48
+
49
+ if TYPE_CHECKING:
50
+ from collections.abc import Callable, Iterator
51
+
52
+ from numpy.typing import NDArray
53
+
54
+ from patchwork.coordinates import TileCoordinates
55
+ from patchwork.frame import Frame
56
+ from patchwork.io import Slices
57
+ from patchwork.options import MosaicOptions
58
+ from patchwork.primary_beam import BeamModel
59
+
60
+ READ_AHEAD_BYTES = 256 * 2**20
61
+ """The memory each thread of a tile may hold in inputs read ahead of the one being added"""
62
+
63
+ KERNEL_CODES = {"nearest": 0, "linear": 1, "cubic": 2}
64
+ """The code of each interpolation kernel in the compiled kernels, where any other is a Lanczos kernel"""
65
+ COMBINE_CODES = {
66
+ "weighted": kernels.WEIGHTED,
67
+ "average": kernels.AVERAGE,
68
+ "sum": kernels.SUM,
69
+ }
70
+ WEIGHT_TYPE_CODES = {
71
+ "primary_beam": kernels.BY_PRIMARY_BEAM,
72
+ "weight_image": kernels.BY_WEIGHT_IMAGE,
73
+ "combined": kernels.BY_BOTH,
74
+ }
75
+ WEIGHT_STATE_CODES = {
76
+ "corrected": kernels.CORRECTED,
77
+ "inherent": kernels.INHERENT,
78
+ "weighted": kernels.ALREADY_WEIGHTED,
79
+ }
80
+
81
+
82
+ @functools.cache
83
+ def _warn_fewer_threads(requested: int, most: int) -> None:
84
+ """Say, once, that numba has fewer threads than were asked for."""
85
+ logger.warning(
86
+ f"Making tiles with {most} threads, not the {requested} asked for, as numba has no more. "
87
+ "It starts as many as the CPUs the process may run on, or NUMBA_NUM_THREADS"
88
+ )
89
+
90
+
91
+ def kernel_threads(requested: int) -> int:
92
+ """The threads the kernels make a tile with: ``patchwork.resample.tile_threads``, and no more than numba has."""
93
+ threads = tile_threads(requested)
94
+ most = kernels.numba_threads()
95
+ if requested > most:
96
+ _warn_fewer_threads(requested, most)
97
+ return min(threads, most)
98
+
99
+
100
+ KERNEL_SHARE = 5 / 8
101
+ """The share of a tile's threads that run the kernels by default, the rest reading.
102
+
103
+ On a 16-core node with the inputs on a striped Lustre file system the kernels
104
+ took 2.8 s a strip on 12 threads while 4 readers took 6 s to read and map the
105
+ inputs onto it; the two balance at about 10 kernel threads of 16.
106
+ """
107
+
108
+
109
+ def split_threads(threads: int, requested: int) -> tuple[int, int]:
110
+ """How many of a tile's ``threads`` run the kernels, and how many read the inputs.
111
+
112
+ ``requested`` is ``ResampleOptions.kernel_threads``: a count below
113
+ ``threads`` is used as given, and 0 picks ``KERNEL_SHARE`` of the threads
114
+ for the kernels when there are at least four. With fewer, or ``requested``
115
+ at least ``threads``, every thread does both.
116
+ """
117
+ if 0 < requested < threads:
118
+ return requested, threads - requested
119
+ if requested == 0 and threads >= 4:
120
+ in_kernels = max(1, round(threads * KERNEL_SHARE))
121
+ return in_kernels, threads - in_kernels
122
+ return threads, threads
123
+
124
+
125
+ class _Holography(NamedTuple):
126
+ """The holography beams of some channels of an input, as the kernels take them"""
127
+
128
+ pattern: NDArray[np.float64]
129
+ """The beam of each channel, shaped (channel, y, x)"""
130
+ leakage: NDArray[np.float64]
131
+ """The fractional leakage of each channel, shaped (channel, 3, y, x), or empty if it was not read"""
132
+ pattern_scale: NDArray[np.int64]
133
+ """Which of ``scales`` each channel uses"""
134
+ scales: NDArray[np.float64]
135
+ """The distinct frequency scales of the beams"""
136
+ pattern_turn: NDArray[np.float64]
137
+ """The cosine and sine of the angle from sky offsets to the axes of the grid"""
138
+ pattern_pixel: NDArray[np.float64]
139
+ """The signed pixel size of the grid along x and y, in radians"""
140
+ pattern_centre: NDArray[np.float64]
141
+ """The (x, y) pixel of the beam centre"""
142
+ feed_turn: NDArray[np.float64]
143
+ """The cosine and sine of twice the feed rotation"""
144
+ cutoff_scales: NDArray[np.float64]
145
+ """The response a cutoff is a fraction of in each channel"""
146
+
147
+
148
+ NO_HOLOGRAPHY = _Holography(
149
+ pattern=np.zeros((0, 1, 1)),
150
+ leakage=np.zeros((0, 3, 1, 1)),
151
+ pattern_scale=np.zeros(0, dtype=np.int64),
152
+ scales=np.zeros(0),
153
+ pattern_turn=np.zeros(2),
154
+ pattern_pixel=np.ones(2),
155
+ pattern_centre=np.zeros(2),
156
+ feed_turn=np.zeros(2),
157
+ cutoff_scales=np.zeros(0),
158
+ )
159
+ """What the kernels are given for an input without a holography beam"""
160
+
161
+
162
+ @functools.lru_cache(maxsize=64)
163
+ def _holography(model: BeamModel, start: int, stop: int) -> _Holography:
164
+ """The holography beams of channels ``start`` to ``stop`` of an input, stacked.
165
+
166
+ They are kept, as every tile an input covers needs them, and the tiles of a
167
+ run of channels are made one after another.
168
+ """
169
+ beams = model._holography_beams(range(start, stop))
170
+ first = beams[0]
171
+ # As patchwork.primary_beam._shared_taps: channels whose scales agree share their taps
172
+ keys = [round(beam.scale, 9) for beam in beams]
173
+ distinct = list(dict.fromkeys(keys))
174
+ rotation = first.alpha_rad + first.frame_rotation_rad
175
+ return _Holography(
176
+ pattern=np.array([beam.plane for beam in beams], dtype=np.float64),
177
+ leakage=NO_HOLOGRAPHY.leakage
178
+ if first.leakage is None
179
+ else np.array([beam.leakage for beam in beams], dtype=np.float64),
180
+ pattern_scale=np.array([distinct.index(key) for key in keys], dtype=np.int64),
181
+ scales=np.array([beams[keys.index(key)].scale for key in distinct]),
182
+ pattern_turn=np.array([np.cos(rotation), np.sin(rotation)]),
183
+ pattern_pixel=np.array(first.pixel_scale_rad, dtype=np.float64),
184
+ pattern_centre=np.array(first.centre_pixel, dtype=np.float64),
185
+ feed_turn=np.array([np.cos(2 * first.alpha_rad), np.sin(2 * first.alpha_rad)]),
186
+ cutoff_scales=np.array([beam.cutoff_scale for beam in beams]),
187
+ )
188
+
189
+
190
+ class _Read(NamedTuple):
191
+ """An input read over the region of a tile it contributes to"""
192
+
193
+ item: MosaicInput
194
+ """The input"""
195
+ region: Slices
196
+ """The (y, x) region of the tile, relative to the tile"""
197
+ usable: NDArray[np.bool_]
198
+ """Which of the channels hold an image"""
199
+ x: NDArray[np.float64]
200
+ """The x position in the blocks of each pixel of the region"""
201
+ y: NDArray[np.float64]
202
+ """The y position in the blocks of each pixel of the region"""
203
+ blocks: NDArray[np.float32]
204
+ """The part of each image that is interpolated, shaped (image, channel, y, x): the images of the input, then its Stokes I if that is not one of them"""
205
+ stokes_i: int
206
+ """The block of Stokes I, or -1 if its leakage is not removed"""
207
+
208
+
209
+ def channel_runs(usable: NDArray[np.bool_], pieces: int = 1) -> list[slice]:
210
+ """The usable channels as runs of neighbouring channels, none longer than one part in ``pieces`` of them.
211
+
212
+ Each run is one read of a cube, so a cube with enough usable channels is
213
+ read in at least ``pieces`` reads, and any cube in at least one for each of
214
+ its stretches of usable channels.
215
+ """
216
+ channels = np.flatnonzero(usable)
217
+ longest = max(1, len(channels) // max(pieces, 1))
218
+ runs = []
219
+ for stretch in np.split(channels, np.flatnonzero(np.diff(channels) > 1) + 1):
220
+ runs += [
221
+ slice(int(stretch[start]), int(stretch[start : start + longest][-1]) + 1)
222
+ for start in range(0, len(stretch), longest)
223
+ ]
224
+ return runs
225
+
226
+
227
+ def _uses_weight_images(item: MosaicInput) -> bool:
228
+ """Whether an input is weighted by weight images, which the kernels do not interpolate."""
229
+ return any(weight.paths for weight in item.weights)
230
+
231
+
232
+ class _Timing:
233
+ """Where the time of a tile goes, to log once the tile is made.
234
+
235
+ The reading and finding times are summed over the threads that did them, so
236
+ they can exceed the time the tile took.
237
+ """
238
+
239
+ def __init__(self) -> None:
240
+ self.started = time.perf_counter()
241
+ self.finding = 0.0
242
+ """Seconds of thread time finding where inputs fall on the tile"""
243
+ self.reading = 0.0
244
+ """Seconds of thread time reading inputs"""
245
+ self.read_bytes = 0
246
+ """Bytes read from the inputs"""
247
+ self.waiting = 0.0
248
+ """Seconds the tile waited for an input to be read"""
249
+ self.adding = 0.0
250
+ """Seconds the kernels spent adding inputs"""
251
+ self.inputs = 0
252
+ """Inputs added"""
253
+ self.finalising = 0.0
254
+ """Seconds turning the sums into the tile"""
255
+ self._lock = threading.Lock()
256
+
257
+ def add(self, name: str, seconds: float, read_bytes: int = 0) -> None:
258
+ """Add to one of the times, from any thread."""
259
+ with self._lock:
260
+ setattr(self, name, getattr(self, name) + seconds)
261
+ self.read_bytes += read_bytes
262
+
263
+ def log(
264
+ self, tile: Slices, channels: range, threads: int, in_kernels: int, reading: int
265
+ ) -> None:
266
+ """Log the times of the tile, at DEBUG level."""
267
+ if not logger.isEnabledFor(logging.DEBUG):
268
+ return
269
+ total = time.perf_counter() - self.started
270
+ logger.debug(
271
+ f"Tile rows {tile[0].start}-{tile[0].stop}, columns {tile[1].start}-{tile[1].stop}, "
272
+ f"channels {channels.start}-{channels.stop}: {total:.1f} s on {threads} threads, "
273
+ f"{in_kernels} in the kernels and {reading} reading; "
274
+ f"waited {self.waiting:.1f} s for inputs, added {self.inputs} in {self.adding:.1f} s, "
275
+ f"finalised in {self.finalising:.1f} s; the threads spent {self.finding:.1f} s finding inputs "
276
+ f"and {self.reading:.1f} s reading {self.read_bytes / 2**20:.0f} MB of them"
277
+ )
278
+
279
+
280
+ def _read_input(
281
+ item: MosaicInput,
282
+ tile: Slices,
283
+ coordinates: TileCoordinates,
284
+ options: MosaicOptions,
285
+ channels: slice,
286
+ readers: Executor | None = None,
287
+ pieces: int = 1,
288
+ timing: _Timing | None = None,
289
+ ) -> _Read | None:
290
+ """Find where an input falls on a tile and read it there, or None if it does not contribute to the tile.
291
+
292
+ This is the start of ``patchwork.tile._add_input``, and may run in a
293
+ thread while other inputs are read or added. With ``readers`` each of its
294
+ images is read in up to ``pieces`` runs of channels, all at the same time.
295
+ Channels without an image are not read, as nothing is made of them.
296
+ """
297
+ kernel = options.resample.kernel
298
+ channel_range = range(channels.start or 0, channels.stop)
299
+ started = time.perf_counter()
300
+ region = tile_region(tile, item.bounding_box)
301
+ if region is None:
302
+ return None
303
+ usable = np.array(
304
+ [
305
+ item.usable_channels[index] if item.usable_channels else True
306
+ for index in channel_range
307
+ ]
308
+ )
309
+ if not np.any(usable):
310
+ return None
311
+ x, y = map_to_input(coordinates, item.info.wcs, region)
312
+ window = input_region(item.info.shape, x, y, kernel)
313
+ if window is None:
314
+ return None
315
+
316
+ paths = list(item.planes)
317
+ stokes_i = -1
318
+ if item.stokes_i_path is not None and item.primary_beam is not None:
319
+ if item.stokes_i_path not in paths:
320
+ paths.append(item.stokes_i_path)
321
+ stokes_i = paths.index(item.stokes_i_path)
322
+ shape = (
323
+ len(paths),
324
+ len(channel_range),
325
+ window[0].stop - window[0].start,
326
+ window[1].stop - window[1].start,
327
+ )
328
+ blocks = np.empty(shape, dtype=np.float32)
329
+
330
+ first = channel_range.start
331
+ reads = [
332
+ (index, run)
333
+ for run in channel_runs(usable, pieces if readers is not None else 1)
334
+ for index in range(len(paths))
335
+ ]
336
+
337
+ def read(index: int, run: slice) -> None:
338
+ read_window(
339
+ paths[index],
340
+ window,
341
+ slice(first + run.start, first + run.stop),
342
+ out=blocks[index, run],
343
+ )
344
+
345
+ found = time.perf_counter()
346
+ if readers is None:
347
+ for index, run in reads:
348
+ read(index, run)
349
+ else:
350
+ # Consumed, so that a read that fails is raised here
351
+ list(readers.map(read, *zip(*reads, strict=True)))
352
+ if timing is not None:
353
+ timing.add("finding", found - started)
354
+ read_bytes = (
355
+ 4 * shape[2] * shape[3] * sum(run.stop - run.start for _, run in reads)
356
+ )
357
+ timing.add("reading", time.perf_counter() - found, read_bytes)
358
+ return _Read(
359
+ item=item,
360
+ region=region,
361
+ usable=usable,
362
+ x=np.ascontiguousarray(x - window[1].start, dtype=np.float64),
363
+ y=np.ascontiguousarray(y - window[0].start, dtype=np.float64),
364
+ blocks=blocks,
365
+ stokes_i=stokes_i,
366
+ )
367
+
368
+
369
+ def _task(
370
+ read: _Read,
371
+ frame: Frame,
372
+ vectors: NDArray[np.float64],
373
+ groups: tuple[int, ...],
374
+ options: MosaicOptions,
375
+ channels: slice,
376
+ ) -> kernels.Task:
377
+ """What the kernels need to add an input that has been read to a tile.
378
+
379
+ Raises:
380
+ ValueError: Raised if leakage is to be removed from an image that is not of Q, U or V, or with a beam that does not carry it
381
+ """
382
+ item = read.item
383
+ channel_range = range(channels.start or 0, channels.stop)
384
+ n_channels = len(channel_range)
385
+ weight_options, polarisation = options.weight, options.polarisation
386
+ kernel = options.resample.kernel
387
+ model = item.primary_beam
388
+ stokes_code = STOKES_CODES
389
+
390
+ removes_leakage = read.stokes_i >= 0
391
+ holography = NO_HOLOGRAPHY
392
+ if model is not None and (
393
+ removes_leakage or model.options.pb_model == "holography"
394
+ ):
395
+ holography = _holography(model, channel_range.start, channel_range.stop)
396
+ if removes_leakage:
397
+ leaky = tuple(code for code in item.stokes if code != stokes_code["I"])
398
+ if any(code not in (2, 3, 4) for code in leaky):
399
+ msg = f"Leakage is only removed from Q, U and V, not from Stokes codes {leaky}"
400
+ raise ValueError(msg)
401
+ if not len(holography.leakage):
402
+ msg = "The leakage of this holography beam was not read"
403
+ raise ValueError(msg)
404
+
405
+ beam = kernels.NO_BEAM
406
+ fwhm = np.ones(n_channels)
407
+ cutoff = np.full(n_channels, float(options.primary_beam.pb_cutoff))
408
+ if model is not None and weight_options.weight_type != "weight_image":
409
+ if model.options.pb_model == "gaussian":
410
+ beam = kernels.GAUSSIAN
411
+ fwhm = np.array([model.channel(index).fwhm_rad for index in channel_range]) # type: ignore[union-attr]
412
+ else:
413
+ beam = kernels.HOLOGRAPHY
414
+ cutoff = cutoff * holography.cutoff_scales
415
+
416
+ rotates = stokes_code["Q"] in item.stokes and stokes_code["U"] in item.stokes
417
+ to_centre = polarisation.output_frame == "centre"
418
+ centre = (float(frame.wcs.wcs.crval[0]), float(frame.wcs.wcs.crval[1]))
419
+ east, north = east_north(*item.info.beam_centre)
420
+
421
+ # As patchwork.tile._group_weight gives the weight of each group without weight images
422
+ weights = np.empty((max(groups) + 1, n_channels))
423
+ for group, plane_weight in zip(groups, item.weights, strict=True):
424
+ values = np.asarray(plane_weight.values, dtype=np.float64)
425
+ weights[group] = values if len(values) == 1 else values[channels]
426
+
427
+ return kernels.Task(
428
+ blocks=read.blocks,
429
+ x=read.x,
430
+ y=read.y,
431
+ code=KERNEL_CODES.get(kernel, 3),
432
+ width=kernel_width(kernel),
433
+ usable=read.usable,
434
+ plane_block=np.arange(len(item.planes), dtype=np.int64),
435
+ plane_stokes=np.array(item.stokes, dtype=np.int64),
436
+ plane_group=np.array(groups, dtype=np.int64),
437
+ weights=weights,
438
+ stokes_i=read.stokes_i,
439
+ removes_leakage=removes_leakage,
440
+ q=item.stokes.index(stokes_code["Q"]) if rotates else -1,
441
+ u=item.stokes.index(stokes_code["U"]) if rotates else -1,
442
+ row=int(read.region[0].start),
443
+ column=int(read.region[1].start),
444
+ vectors=vectors,
445
+ east=east,
446
+ north=north,
447
+ beam=beam,
448
+ fwhm=fwhm,
449
+ cutoff=cutoff,
450
+ pattern=holography.pattern,
451
+ leakage=holography.leakage,
452
+ pattern_scale=holography.pattern_scale,
453
+ scales=holography.scales,
454
+ pattern_turn=holography.pattern_turn,
455
+ pattern_pixel=holography.pattern_pixel,
456
+ pattern_centre=holography.pattern_centre,
457
+ feed_turn=holography.feed_turn,
458
+ input_north=east_north(*(item.frame_centre or centre))[1],
459
+ has_input_frame=item.frame_centre is not None,
460
+ output_north=east_north(*centre)[1],
461
+ has_output_frame=to_centre,
462
+ weight_type=WEIGHT_TYPE_CODES[weight_options.weight_type],
463
+ weight_state=WEIGHT_STATE_CODES[weight_options.weight_state],
464
+ combine=COMBINE_CODES[options.combine.combine_type],
465
+ )
466
+
467
+
468
+ def _read_ahead(
469
+ inputs: list[MosaicInput],
470
+ read: Callable[[MosaicInput, Executor | None], _Read | None],
471
+ threads: int,
472
+ ahead: int,
473
+ ) -> Iterator[tuple[MosaicInput, _Read | None]]:
474
+ """Each input with what was read of it, in order, reading up to ``ahead`` inputs at once with ``threads`` threads.
475
+
476
+ Reading is mostly waiting for the file system, which serves many reads at
477
+ once, so the images of the next few inputs are all read side by side while
478
+ the kernels add the current one. An input weighted by weight images is not
479
+ read here, and is given with None. With one thread and one input ahead,
480
+ each input is read as it is asked for.
481
+ """
482
+ if threads <= 1 and ahead <= 1:
483
+ for item in inputs:
484
+ yield item, None if _uses_weight_images(item) else read(item, None)
485
+ return
486
+
487
+ with (
488
+ # One finds where each input falls and waits for its images, the other reads them
489
+ ThreadPoolExecutor(ahead, thread_name_prefix="patchwork-input") as finders,
490
+ ThreadPoolExecutor(threads, thread_name_prefix="patchwork-read") as readers,
491
+ ):
492
+ remaining = iter(inputs)
493
+ pending: deque[tuple[MosaicInput, Future[_Read | None] | None]] = deque()
494
+ try:
495
+ while True:
496
+ while len(pending) < ahead:
497
+ following = next(remaining, None)
498
+ if following is None:
499
+ break
500
+ reading = (
501
+ None
502
+ if _uses_weight_images(following)
503
+ else finders.submit(read, following, readers)
504
+ )
505
+ pending.append((following, reading))
506
+ if not pending:
507
+ return
508
+ item, future = pending.popleft()
509
+ yield item, None if future is None else future.result()
510
+ finally:
511
+ for _, future in pending:
512
+ if future is not None:
513
+ future.cancel()
514
+
515
+
516
+ def mosaic_tile_fused(
517
+ tile: Slices,
518
+ frame: Frame,
519
+ inputs: list[MosaicInput],
520
+ options: MosaicOptions,
521
+ channels: slice | None = None,
522
+ ) -> TileResult:
523
+ """Resample, weight and combine every input that covers one tile of the frame, over some channels, as ``patchwork.tile.mosaic_tile`` does, with the compiled kernels.
524
+
525
+ An input weighted by weight images is added as the numba backend adds it,
526
+ as the kernels do not interpolate weights.
527
+
528
+ Args:
529
+ tile (Slices): The (y, x) region of the frame to make
530
+ frame (Frame): The frame of the mosaic
531
+ inputs (list[MosaicInput]): The inputs of the mosaic
532
+ options (MosaicOptions): The mosaicking options
533
+ channels (slice | None, optional): The channels of the inputs to make. Defaults to None, the first channel.
534
+
535
+ Returns:
536
+ TileResult: The finished tile
537
+ """
538
+ combine_type = options.combine.combine_type
539
+ channels = channels or slice(0, 1)
540
+ channel_range = range(channels.start or 0, channels.stop)
541
+ threads = kernel_threads(options.resample.tile_threads)
542
+ in_kernels, reading = split_threads(threads, options.resample.kernel_threads)
543
+ timing = _Timing()
544
+
545
+ coordinates = tile_coordinates(
546
+ frame.wcs, tile, step=options.resample.coordinate_step
547
+ )
548
+ vectors = np.ascontiguousarray(coordinates.vectors, dtype=np.float64)
549
+ shape = (len(channel_range), *vectors.shape[1:])
550
+ stokes = inputs[0].stokes if inputs else (0,)
551
+ groups = weight_groups(stokes)
552
+ members = group_members(groups)
553
+ kept = (len(members), *shape) if combine_type != "weighted" else (0, 0, 0, 0)
554
+ sums = kernels.Sums(
555
+ totals=np.zeros((len(stokes), *shape)),
556
+ weight=np.zeros((len(members), *shape)),
557
+ inverse_weight=np.zeros(kept),
558
+ n_inputs=np.zeros(kept),
559
+ )
560
+ # The same sums as the other backends keep them, for inputs added their way
561
+ tile_sums = [
562
+ TileSums(
563
+ totals=tuple(sums.totals[index] for index in indices),
564
+ weight=sums.weight[group],
565
+ inverse_weight=sums.inverse_weight[group] if kept[0] else None,
566
+ n_inputs=sums.n_inputs[group] if kept[0] else None,
567
+ )
568
+ for group, indices in enumerate(members)
569
+ ]
570
+
571
+ # An input is read over about the area of the tile, a little more where it is rotated
572
+ held = 4 * (len(stokes) + 1) * int(np.prod(shape))
573
+ ahead = max(1, min(threads, threads * READ_AHEAD_BYTES // held))
574
+
575
+ def read_input(item: MosaicInput, readers: Executor | None) -> _Read | None:
576
+ # Enough reads of each image to keep every thread reading while one input is read
577
+ pieces = max(1, reading // (len(stokes) + 1))
578
+ return _read_input(
579
+ item, tile, coordinates, options, channels, readers, pieces, timing
580
+ )
581
+
582
+ waited = time.perf_counter()
583
+ for item, read in _read_ahead(inputs, read_input, reading, ahead):
584
+ started = time.perf_counter()
585
+ timing.add("waiting", started - waited)
586
+ if _uses_weight_images(item):
587
+ _add_input(
588
+ item, tile, frame, coordinates, tile_sums, members, options, channels
589
+ )
590
+ timing.inputs += 1
591
+ elif read is not None:
592
+ kernels.add_input(
593
+ _task(read, frame, vectors, groups, options, channels),
594
+ sums,
595
+ in_kernels,
596
+ )
597
+ timing.inputs += 1
598
+ waited = time.perf_counter()
599
+ timing.add("adding", waited - started)
600
+
601
+ images = np.empty((len(stokes), *shape), dtype=np.float32)
602
+ weights = np.empty((len(members), *shape), dtype=np.float32)
603
+ kernels.finalise(
604
+ sums,
605
+ np.array(groups, dtype=np.int64),
606
+ COMBINE_CODES[combine_type],
607
+ images,
608
+ weights,
609
+ in_kernels,
610
+ )
611
+ timing.add("finalising", time.perf_counter() - waited)
612
+ timing.log(tile, channel_range, threads, in_kernels, reading)
613
+ return TileResult(
614
+ slices=tile,
615
+ channels=slice(channel_range.start, channel_range.stop),
616
+ images=tuple(images),
617
+ weights=tuple(weights),
618
+ )