askap-patchwork 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,695 @@
1
+ """Numba kernels of the fused backend, which make a tile in one compiled pass over each input.
2
+
3
+ ``add_block`` does to the pixels of an input everything that
4
+ ``patchwork.tile._add_input`` does with numpy: it interpolates the input's
5
+ images, evaluates the primary beam, removes the leakage, rotates Q and U,
6
+ weights the images and adds them to the sums of the tile. It works a row of
7
+ one channel at a time, so nothing is held for the whole region between those
8
+ steps, and none of them holds the GIL. ``finalise_rows`` turns the sums into
9
+ the tile.
10
+
11
+ Each has a twin that runs it over many parts of the tile on numba's threads
12
+ (``add_chunks`` and ``finalise_chunks``). The parts share no pixels, so the
13
+ threads never write the same memory. ``add_input`` and ``finalise`` choose
14
+ between the two.
15
+
16
+ They follow the numpy code, and give the same values to round-off.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import functools
22
+ import itertools
23
+ import threading
24
+ from typing import TYPE_CHECKING, NamedTuple
25
+
26
+ import numba
27
+ import numpy as np
28
+
29
+ from patchwork.logging import logger
30
+ from patchwork.numba_kernels import _kernel, _taps
31
+
32
+ if TYPE_CHECKING:
33
+ from numpy.typing import NDArray
34
+
35
+ WEIGHTED, AVERAGE, SUM = 0, 1, 2
36
+ """How the inputs are combined, as ``CombineOptions.combine_type``"""
37
+ BY_PRIMARY_BEAM, BY_WEIGHT_IMAGE, BY_BOTH = 0, 1, 2
38
+ """Where the weights come from, as ``WeightOptions.weight_type``"""
39
+ CORRECTED, INHERENT, ALREADY_WEIGHTED = 0, 1, 2
40
+ """What has been done to the inputs, as ``WeightOptions.weight_state``"""
41
+ NO_BEAM, GAUSSIAN, HOLOGRAPHY = 0, 1, 2
42
+ """The primary beam that weights an input"""
43
+ STOKES_I, STOKES_V = 1, 4
44
+ """FITS codes of Stokes parameters"""
45
+
46
+ _PARALLEL = threading.Lock()
47
+ """Held while numba's threads run. Its default threading layer (workqueue)
48
+ stops the process if two Python threads start parallel kernels at once, so
49
+ Python threads that make tiles at the same time take turns at them."""
50
+
51
+
52
+ class Sums(NamedTuple):
53
+ """The running sums of a tile over its inputs, as ``patchwork.combine.TileSums`` keeps for each group, stacked"""
54
+
55
+ totals: NDArray[np.float64]
56
+ """For each image, the sum of its weighted estimates, shaped (image, channel, y, x)"""
57
+ weight: NDArray[np.float64]
58
+ """For each group of images that share a weight, the sum of the weights, shaped (group, channel, y, x)"""
59
+ inverse_weight: NDArray[np.float64]
60
+ """For each group, the sum of the inverse weights, kept for a plain mean or a sum, and otherwise empty"""
61
+ n_inputs: NDArray[np.float64]
62
+ """For each group, the number of inputs that contributed, kept for a plain mean or a sum, and otherwise empty"""
63
+
64
+
65
+ class Task(NamedTuple):
66
+ """An input over the region of a tile it contributes to, and how it is added to the tile.
67
+
68
+ Every field has one type whatever the input, so the kernels are compiled once.
69
+ """
70
+
71
+ blocks: NDArray[np.float32]
72
+ """The part of each image of the input that is interpolated, shaped (image, channel, y, x)"""
73
+ x: NDArray[np.float64]
74
+ """The x position in the blocks of each pixel of the region"""
75
+ y: NDArray[np.float64]
76
+ """The y position in the blocks of each pixel of the region"""
77
+ code: int
78
+ """The interpolation kernel: 0 nearest, 1 linear, 2 cubic, 3 Lanczos"""
79
+ width: int
80
+ """The number of taps of the kernel along each axis"""
81
+ usable: NDArray[np.bool_]
82
+ """Which channels of the input hold an image"""
83
+ plane_block: NDArray[np.int64]
84
+ """The block of each image of the mosaic"""
85
+ plane_stokes: NDArray[np.int64]
86
+ """The FITS Stokes code of each image of the mosaic"""
87
+ plane_group: NDArray[np.int64]
88
+ """The group, of images that share a weight, of each image of the mosaic"""
89
+ weights: NDArray[np.float64]
90
+ """The weight of the input in each group and channel, shaped (group, channel)"""
91
+ stokes_i: int
92
+ """The block of Stokes I, whose leakage is removed from the other images"""
93
+ removes_leakage: bool
94
+ """Whether the leakage of Stokes I is removed"""
95
+ q: int
96
+ """The image of Stokes Q, which is rotated with ``u``, or -1 if the input does not have both"""
97
+ u: int
98
+ """The image of Stokes U"""
99
+ row: int
100
+ """The first row of the region in the tile"""
101
+ column: int
102
+ """The first column of the region in the tile"""
103
+ vectors: NDArray[np.float64]
104
+ """Unit vectors of every pixel of the tile, shaped (3, y, x)"""
105
+ east: NDArray[np.float64]
106
+ """The unit vector pointing east at the beam centre"""
107
+ north: NDArray[np.float64]
108
+ """The unit vector pointing north at the beam centre"""
109
+ beam: int
110
+ """The primary beam that weights the input: ``NO_BEAM``, ``GAUSSIAN`` or ``HOLOGRAPHY``"""
111
+ fwhm: NDArray[np.float64]
112
+ """The FWHM in radians of the Gaussian beam of each channel"""
113
+ cutoff: NDArray[np.float64]
114
+ """The response of each channel below which the input does not contribute"""
115
+ pattern: NDArray[np.float64]
116
+ """The holography beam of each channel, shaped (channel, y, x)"""
117
+ leakage: NDArray[np.float64]
118
+ """The fractional leakage of Stokes I into (Q, U, V) of each channel, shaped (channel, 3, y, x)"""
119
+ pattern_scale: NDArray[np.int64]
120
+ """Which of ``scales`` each channel uses"""
121
+ scales: NDArray[np.float64]
122
+ """The distinct frequency scales of the holography beams, as ``HolographyBeam.scale``"""
123
+ pattern_turn: NDArray[np.float64]
124
+ """The cosine and sine of the angle from sky offsets to the axes of the holography grid"""
125
+ pattern_pixel: NDArray[np.float64]
126
+ """The signed pixel size of the holography grid along x and y, in radians"""
127
+ pattern_centre: NDArray[np.float64]
128
+ """The (x, y) pixel of the beam centre on the holography grid"""
129
+ feed_turn: NDArray[np.float64]
130
+ """The cosine and sine of twice the feed rotation, which turns the leakage of Q and U"""
131
+ input_north: NDArray[np.float64]
132
+ """The unit vector that is north where the Q and U of the input are referenced to"""
133
+ has_input_frame: bool
134
+ """Whether the Q and U of the input are referenced to ``input_north``, rather than to north at each pixel"""
135
+ output_north: NDArray[np.float64]
136
+ """The unit vector that is north where the Q and U of the mosaic are referenced to"""
137
+ has_output_frame: bool
138
+ """Whether the Q and U of the mosaic are referenced to ``output_north``, rather than to north at each pixel"""
139
+ weight_type: int
140
+ """Where the weights come from"""
141
+ weight_state: int
142
+ """What has been done to the input"""
143
+ combine: int
144
+ """How the inputs are combined"""
145
+
146
+
147
+ _fused = {"error_model": "numpy"}
148
+ """Division follows IEEE arithmetic, as numpy's does, rather than raising"""
149
+
150
+
151
+ def _resample_row(
152
+ window: NDArray[np.float32],
153
+ wanted: NDArray[np.bool_],
154
+ inside: NDArray[np.bool_],
155
+ x_first: NDArray[np.int64],
156
+ y_first: NDArray[np.int64],
157
+ x_weights: NDArray[np.float64],
158
+ y_weights: NDArray[np.float64],
159
+ width: int,
160
+ values: NDArray[np.float64],
161
+ valid: NDArray[np.bool_],
162
+ ) -> None:
163
+ """Interpolate one image at the wanted positions of a row, whose taps are already found, as ``numba_kernels.interpolate_stack`` does."""
164
+ n_rows, n_columns = window.shape
165
+ for index in range(values.shape[0]):
166
+ if not wanted[index]:
167
+ continue
168
+ left, top = x_first[index], y_first[index]
169
+ if inside[index]:
170
+ # Every tap is on the image. Each row of taps is summed, then the rows,
171
+ # which takes fewer multiplications than summing tap by tap and gives
172
+ # the same value to round-off. If no tap is blank the sum is finite,
173
+ # and nothing else needs checking
174
+ total = 0.0
175
+ for j in range(width):
176
+ along = 0.0
177
+ for i in range(width):
178
+ along += x_weights[index, i] * window[top + j, left + i]
179
+ total += y_weights[index, j] * along
180
+ if np.isfinite(total):
181
+ values[index] = total
182
+ valid[index] = True
183
+ continue
184
+
185
+ total = 0.0
186
+ good = True
187
+ for j in range(width):
188
+ row = top + j
189
+ row_outside = row < 0 or row >= n_rows
190
+ row = min(max(row, 0), n_rows - 1)
191
+ for i in range(width):
192
+ column = left + i
193
+ outside = row_outside or column < 0 or column >= n_columns
194
+ column = min(max(column, 0), n_columns - 1)
195
+ tap_weight = y_weights[index, j] * x_weights[index, i]
196
+ value = window[row, column]
197
+ blank = not np.isfinite(value)
198
+ if abs(tap_weight) > 1e-9 and (outside or blank):
199
+ good = False
200
+ total += tap_weight * (0.0 if blank else value)
201
+ values[index] = total if good else 0.0
202
+ valid[index] = good
203
+
204
+
205
+ resample_row = _kernel(_resample_row, **_fused)
206
+
207
+
208
+ def _pattern_taps(
209
+ east: float,
210
+ north: float,
211
+ scale: float,
212
+ turn: NDArray[np.float64],
213
+ pixel: NDArray[np.float64],
214
+ centre: NDArray[np.float64],
215
+ n_rows: int,
216
+ n_columns: int,
217
+ weights: NDArray[np.float64],
218
+ ) -> tuple[int, int]:
219
+ """Where an offset from the beam centre falls on a holography grid: the lower pixel around it, with the weights of the four around it put in ``weights``.
220
+
221
+ As ``HolographyBeam.taps``, which gives no weight to a position off the grid.
222
+ """
223
+ x = east * turn[0] - north * turn[1]
224
+ y = east * turn[1] + north * turn[0]
225
+ radius = np.hypot(x, y)
226
+ offset = np.arcsin(min(radius, 1.0))
227
+ stretch = np.sin(scale * offset) / radius if radius > 0 else scale
228
+ x_pixel = x * stretch / pixel[0] + centre[0]
229
+ y_pixel = y * stretch / pixel[1] + centre[1]
230
+ on_grid = (
231
+ y_pixel >= 0
232
+ and y_pixel <= n_rows - 1
233
+ and x_pixel >= 0
234
+ and x_pixel <= n_columns - 1
235
+ )
236
+ if not on_grid:
237
+ weights[:] = 0.0
238
+ return 0, 0
239
+ row = int(min(np.floor(y_pixel), max(n_rows - 2, 0)))
240
+ column = int(min(np.floor(x_pixel), max(n_columns - 2, 0)))
241
+ y_fraction, x_fraction = y_pixel - row, x_pixel - column
242
+ weights[0] = (1 - y_fraction) * (1 - x_fraction)
243
+ weights[1] = (1 - y_fraction) * x_fraction
244
+ weights[2] = y_fraction * (1 - x_fraction)
245
+ weights[3] = y_fraction * x_fraction
246
+ return row, column
247
+
248
+
249
+ pattern_taps = _kernel(_pattern_taps, **_fused)
250
+
251
+
252
+ def _sample(
253
+ plane: NDArray[np.float64], row: int, column: int, weights: NDArray[np.float64]
254
+ ) -> float:
255
+ """Interpolate a plane of a holography grid with the taps of a position, as ``PatternTaps.sample``."""
256
+ row_step = min(1, plane.shape[0] - 1)
257
+ column_step = min(1, plane.shape[1] - 1)
258
+ value = weights[0] * plane[row, column]
259
+ value += weights[1] * plane[row, column + column_step]
260
+ value += weights[2] * plane[row + row_step, column]
261
+ value += weights[3] * plane[row + row_step, column + column_step]
262
+ return float(value)
263
+
264
+
265
+ sample = _kernel(_sample, **_fused)
266
+
267
+
268
+ def _frame_angle(north: NDArray[np.float64], x: float, y: float, z: float) -> float:
269
+ """The position angle at a position of the direction that is north where ``north`` is, as ``patchwork.polarisation.frame_angle``."""
270
+ along_east = -north[0] * y + north[1] * x
271
+ along_north = -z * (north[0] * x + north[1] * y) + north[2] * (x**2 + y**2)
272
+ return float(np.arctan2(along_east, along_north))
273
+
274
+
275
+ frame_angle = _kernel(_frame_angle, **_fused)
276
+
277
+
278
+ def _add_block(
279
+ first_row: int,
280
+ last_row: int,
281
+ first_column: int,
282
+ last_column: int,
283
+ task: Task,
284
+ sums: Sums,
285
+ ) -> None:
286
+ """Add an input to the sums of a tile, over a block of rows and columns of the region it contributes to.
287
+
288
+ Blocks that share no pixels may be added at the same time.
289
+ """
290
+ blocks = task.blocks
291
+ n_blocks, n_channels = blocks.shape[0], blocks.shape[1]
292
+ window_rows, window_columns = blocks.shape[2], blocks.shape[3]
293
+ width = task.width
294
+ n_columns = last_column - first_column
295
+ n_planes = task.plane_block.shape[0]
296
+ n_groups = task.weights.shape[0]
297
+ n_scales = task.scales.shape[0]
298
+ samples_pattern = task.beam == HOLOGRAPHY or task.removes_leakage
299
+ needs_offsets = task.beam != NO_BEAM or task.removes_leakage
300
+ turns = task.q >= 0 and (task.has_input_frame or task.has_output_frame)
301
+ has_v = False
302
+ for plane in range(n_planes):
303
+ if task.plane_stokes[plane] == STOKES_V:
304
+ has_v = True
305
+
306
+ shifts = np.pi * (np.arange(width) - (width // 2 - 1)) / max(width // 2, 1)
307
+ shift_trig = np.vstack((np.cos(shifts), np.sin(shifts)))
308
+ # What is known of each pixel of a row, whatever the channel
309
+ located = np.zeros(n_columns, dtype=np.bool_)
310
+ inside = np.zeros(n_columns, dtype=np.bool_)
311
+ x_first = np.zeros(n_columns, dtype=np.int64)
312
+ y_first = np.zeros(n_columns, dtype=np.int64)
313
+ x_weights = np.zeros((n_columns, width))
314
+ y_weights = np.zeros((n_columns, width))
315
+ east = np.zeros(n_columns)
316
+ north = np.zeros(n_columns)
317
+ offset_squared = np.zeros(n_columns)
318
+ cos_turn = np.ones(n_columns)
319
+ sin_turn = np.zeros(n_columns)
320
+ pattern_row = np.zeros((n_scales, n_columns), dtype=np.int64)
321
+ pattern_column = np.zeros((n_scales, n_columns), dtype=np.int64)
322
+ pattern_weights = np.zeros((n_scales, n_columns, 4))
323
+ # A row of one channel: where the input reaches, its response there and its
324
+ # images, and then the images of one of its pixels
325
+ wanted = np.zeros(n_columns, dtype=np.bool_)
326
+ response = np.ones(n_columns)
327
+ values = np.zeros((n_blocks, n_columns))
328
+ valid = np.zeros((n_blocks, n_columns), dtype=np.bool_)
329
+ plane_values = np.zeros(n_planes)
330
+ plane_valid = np.zeros(n_planes, dtype=np.bool_)
331
+ leak = np.zeros(3)
332
+
333
+ for row in range(first_row, last_row):
334
+ tile_row = task.row + row
335
+ tile_column = task.column + first_column
336
+ for index in range(n_columns):
337
+ x_position = task.x[row, first_column + index]
338
+ y_position = task.y[row, first_column + index]
339
+ located[index] = np.isfinite(x_position) and np.isfinite(y_position)
340
+ if not located[index]:
341
+ continue
342
+ left = _taps(x_position, task.code, width, x_weights[index], shift_trig)
343
+ top = _taps(y_position, task.code, width, y_weights[index], shift_trig)
344
+ x_first[index], y_first[index] = left, top
345
+ inside[index] = (
346
+ left >= 0
347
+ and left + width <= window_columns
348
+ and top >= 0
349
+ and top + width <= window_rows
350
+ )
351
+
352
+ if needs_offsets or turns:
353
+ for index in range(n_columns):
354
+ x = task.vectors[0, tile_row, tile_column + index]
355
+ y = task.vectors[1, tile_row, tile_column + index]
356
+ z = task.vectors[2, tile_row, tile_column + index]
357
+ if needs_offsets:
358
+ east[index] = task.east[0] * x + task.east[1] * y + task.east[2] * z
359
+ north[index] = (
360
+ task.north[0] * x + task.north[1] * y + task.north[2] * z
361
+ )
362
+ if turns:
363
+ turn = 0.0
364
+ if task.has_input_frame:
365
+ turn = frame_angle(task.input_north, x, y, z)
366
+ if task.has_output_frame:
367
+ turn = turn - frame_angle(task.output_north, x, y, z)
368
+ cos_turn[index] = np.cos(2 * turn)
369
+ sin_turn[index] = np.sin(2 * turn)
370
+ if task.beam == GAUSSIAN:
371
+ for index in range(n_columns):
372
+ offset = np.arcsin(min(np.hypot(east[index], north[index]), 1.0))
373
+ offset_squared[index] = offset**2
374
+ if samples_pattern:
375
+ for scale in range(n_scales):
376
+ for index in range(n_columns):
377
+ pattern_row[scale, index], pattern_column[scale, index] = (
378
+ pattern_taps(
379
+ east[index],
380
+ north[index],
381
+ task.scales[scale],
382
+ task.pattern_turn,
383
+ task.pattern_pixel,
384
+ task.pattern_centre,
385
+ task.pattern.shape[1],
386
+ task.pattern.shape[2],
387
+ pattern_weights[scale, index],
388
+ )
389
+ )
390
+
391
+ for channel in range(n_channels):
392
+ if not task.usable[channel]:
393
+ continue
394
+ # The beam first, so nothing is interpolated where it is below the cutoff
395
+ scale = task.pattern_scale[channel] if samples_pattern else 0
396
+ any_wanted = False
397
+ for index in range(n_columns):
398
+ wanted[index] = located[index]
399
+ if wanted[index] and task.beam != NO_BEAM:
400
+ if task.beam == GAUSSIAN:
401
+ response[index] = np.exp(
402
+ -4
403
+ * np.log(2)
404
+ * offset_squared[index]
405
+ / task.fwhm[channel] ** 2
406
+ )
407
+ else:
408
+ response[index] = sample(
409
+ task.pattern[channel],
410
+ pattern_row[scale, index],
411
+ pattern_column[scale, index],
412
+ pattern_weights[scale, index],
413
+ )
414
+ wanted[index] = response[index] >= task.cutoff[channel]
415
+ any_wanted = any_wanted or wanted[index]
416
+ if not any_wanted:
417
+ continue
418
+ for block in range(n_blocks):
419
+ resample_row(
420
+ blocks[block, channel],
421
+ wanted,
422
+ inside,
423
+ x_first,
424
+ y_first,
425
+ x_weights,
426
+ y_weights,
427
+ width,
428
+ values[block],
429
+ valid[block],
430
+ )
431
+
432
+ for index in range(n_columns):
433
+ if not wanted[index]:
434
+ continue
435
+
436
+ for plane in range(n_planes):
437
+ source = task.plane_block[plane]
438
+ plane_values[plane] = values[source, index]
439
+ plane_valid[plane] = valid[source, index]
440
+
441
+ if task.removes_leakage:
442
+ stokes_i = values[task.stokes_i, index]
443
+ stokes_i_valid = valid[task.stokes_i, index]
444
+ for stokes in range(3 if has_v else 2):
445
+ leak[stokes] = sample(
446
+ task.leakage[channel, stokes],
447
+ pattern_row[scale, index],
448
+ pattern_column[scale, index],
449
+ pattern_weights[scale, index],
450
+ )
451
+ q_leak, u_leak = leak[0], leak[1]
452
+ leak[0] = q_leak * task.feed_turn[0] - u_leak * task.feed_turn[1]
453
+ leak[1] = q_leak * task.feed_turn[1] + u_leak * task.feed_turn[0]
454
+ for plane in range(n_planes):
455
+ code = task.plane_stokes[plane]
456
+ if code != STOKES_I:
457
+ plane_values[plane] -= leak[code - 2] * stokes_i
458
+ plane_valid[plane] = plane_valid[plane] and stokes_i_valid
459
+
460
+ if task.q >= 0:
461
+ q, u = plane_values[task.q], plane_values[task.u]
462
+ plane_values[task.q] = q * cos_turn[index] - u * sin_turn[index]
463
+ plane_values[task.u] = q * sin_turn[index] + u * cos_turn[index]
464
+ both = plane_valid[task.q] and plane_valid[task.u]
465
+ plane_valid[task.q] = plane_valid[task.u] = both
466
+
467
+ for group in range(n_groups):
468
+ contributes = True
469
+ for plane in range(n_planes):
470
+ if task.plane_group[plane] == group and not plane_valid[plane]:
471
+ contributes = False
472
+ if not contributes:
473
+ continue
474
+
475
+ # As patchwork.weights.weighting
476
+ weight = task.weights[group, channel]
477
+ if task.weight_type == BY_WEIGHT_IMAGE:
478
+ if task.weight_state == CORRECTED:
479
+ factor = weight
480
+ elif task.weight_state == INHERENT:
481
+ factor = np.sqrt(weight) if weight > 0 else 0.0
482
+ else:
483
+ factor = 1.0
484
+ else:
485
+ scaling = weight if task.weight_type == BY_BOTH else 1.0
486
+ factor = weight = scaling
487
+ if task.beam != NO_BEAM:
488
+ if task.weight_state != ALREADY_WEIGHTED:
489
+ factor = scaling * response[index]
490
+ if task.weight_state == CORRECTED:
491
+ factor = factor * response[index]
492
+ weight = scaling * response[index] ** 2
493
+ if not (np.isfinite(weight) and weight > 0):
494
+ continue
495
+
496
+ # As patchwork.combine.accumulate
497
+ column = tile_column + index
498
+ if task.combine != WEIGHTED:
499
+ inverse = 1.0 / weight
500
+ factor = factor * inverse
501
+ sums.inverse_weight[group, channel, tile_row, column] += inverse
502
+ sums.n_inputs[group, channel, tile_row, column] += 1.0
503
+ sums.weight[group, channel, tile_row, column] += weight
504
+ for plane in range(n_planes):
505
+ if task.plane_group[plane] == group:
506
+ sums.totals[plane, channel, tile_row, column] += (
507
+ plane_values[plane] * factor
508
+ )
509
+
510
+
511
+ add_block = _kernel(_add_block, **_fused)
512
+
513
+
514
+ def _add_chunks(chunks: NDArray[np.int64], task: Task, sums: Sums) -> None:
515
+ """Add an input to the sums of a tile over many blocks, the rows of ``chunks``, on numba's threads."""
516
+ for index in numba.prange(chunks.shape[0]):
517
+ add_block(
518
+ chunks[index, 0],
519
+ chunks[index, 1],
520
+ chunks[index, 2],
521
+ chunks[index, 3],
522
+ task,
523
+ sums,
524
+ )
525
+
526
+
527
+ add_chunks = _kernel(_add_chunks, parallel=True, **_fused)
528
+
529
+
530
+ def _finalise_rows(
531
+ first: int,
532
+ last: int,
533
+ sums: Sums,
534
+ plane_group: NDArray[np.int64],
535
+ combine: int,
536
+ images: NDArray[np.float32],
537
+ weights: NDArray[np.float32],
538
+ ) -> None:
539
+ """Turn the sums of some rows of a tile into its images and weights, as ``patchwork.combine.finalise``.
540
+
541
+ The rows of every channel are counted together, so row ``index`` is row
542
+ ``index % n_rows`` of channel ``index // n_rows``.
543
+ """
544
+ n_planes, n_groups = images.shape[0], weights.shape[0]
545
+ n_rows, n_columns = images.shape[2], images.shape[3]
546
+ for index in range(first, last):
547
+ channel, row = index // n_rows, index % n_rows
548
+ for group in range(n_groups):
549
+ for column in range(n_columns):
550
+ weight = sums.weight[group, channel, row, column]
551
+ divisor = weight
552
+ if combine == WEIGHTED:
553
+ covered = weight > 0
554
+ else:
555
+ count = sums.n_inputs[group, channel, row, column]
556
+ inverse = sums.inverse_weight[group, channel, row, column]
557
+ covered = count > 0
558
+ if combine == AVERAGE:
559
+ divisor = count
560
+ weight = count**2 / inverse
561
+ else:
562
+ divisor = 1.0
563
+ weight = 1.0 / inverse
564
+ weights[group, channel, row, column] = weight if covered else np.nan
565
+ for plane in range(n_planes):
566
+ if plane_group[plane] == group:
567
+ images[plane, channel, row, column] = (
568
+ sums.totals[plane, channel, row, column] / divisor
569
+ if covered
570
+ else np.nan
571
+ )
572
+
573
+
574
+ finalise_rows = _kernel(_finalise_rows, **_fused)
575
+
576
+
577
+ def _finalise_chunks(
578
+ chunks: NDArray[np.int64],
579
+ sums: Sums,
580
+ plane_group: NDArray[np.int64],
581
+ combine: int,
582
+ images: NDArray[np.float32],
583
+ weights: NDArray[np.float32],
584
+ ) -> None:
585
+ """Turn the sums of a tile into its images and weights over many runs of rows, the rows of ``chunks``, on numba's threads."""
586
+ for index in numba.prange(chunks.shape[0]):
587
+ finalise_rows(
588
+ chunks[index, 0],
589
+ chunks[index, 1],
590
+ sums,
591
+ plane_group,
592
+ combine,
593
+ images,
594
+ weights,
595
+ )
596
+
597
+
598
+ finalise_chunks = _kernel(_finalise_chunks, parallel=True, **_fused)
599
+
600
+
601
+ def spread(chunks: NDArray[np.int64], threads: int) -> NDArray[np.int64]:
602
+ """Reorder chunks so that each thread's share is spread over all of them.
603
+
604
+ numba gives each thread a run of consecutive chunks. Neighbouring chunks
605
+ take about as long as each other, and those far apart may not (an input
606
+ covers one end of a tile and not the other), so every ``threads``'th chunk
607
+ is put in one run.
608
+ """
609
+ order = np.argsort(np.arange(len(chunks)) % threads, kind="stable")
610
+ return np.ascontiguousarray(chunks[order])
611
+
612
+
613
+ def block_chunks(
614
+ n_rows: int, n_columns: int, threads: int, widest: int = 1024
615
+ ) -> NDArray[np.int64]:
616
+ """Split a region into blocks for ``threads`` threads to share, as rows of (first row, last row, first column, last column).
617
+
618
+ There are about four blocks to each thread, so a thread whose blocks are
619
+ quick is not left with nothing to do for long, and none is wider than
620
+ ``widest`` pixels, so what is kept for a row of one stays in the cache. A
621
+ strip of whole rows, which may be only a few rows of a wide mosaic, is
622
+ therefore split along its rows too.
623
+ """
624
+ edges = np.linspace(0, n_columns, -(-n_columns // widest) + 1).astype(np.int64)
625
+ rows = max(1, n_rows * (len(edges) - 1) // (4 * threads))
626
+ chunks = np.array(
627
+ [
628
+ (row, min(row + rows, n_rows), left, right)
629
+ for row in range(0, n_rows, rows)
630
+ for left, right in itertools.pairwise(edges)
631
+ ],
632
+ dtype=np.int64,
633
+ ).reshape(-1, 4)
634
+ return spread(chunks, threads)
635
+
636
+
637
+ def row_chunks(n_rows: int, threads: int) -> NDArray[np.int64]:
638
+ """Split rows into runs for ``threads`` threads to share, as rows of (first, last), about four to each thread."""
639
+ rows = max(1, n_rows // (4 * threads))
640
+ chunks = np.array(
641
+ [(row, min(row + rows, n_rows)) for row in range(0, n_rows, rows)],
642
+ dtype=np.int64,
643
+ ).reshape(-1, 2)
644
+ return spread(chunks, threads)
645
+
646
+
647
+ def numba_threads() -> int:
648
+ """The most threads numba runs a kernel on: as many as the CPUs the process could run on when numba was imported, or ``NUMBA_NUM_THREADS``."""
649
+ return int(numba.config.NUMBA_NUM_THREADS) # type: ignore[attr-defined,unused-ignore]
650
+
651
+
652
+ @functools.cache
653
+ def _log_threads(threads: int) -> None:
654
+ """Say, once in each process, how many threads tiles are made with, and what runs them."""
655
+ layer = numba.threading_layer() # type: ignore[attr-defined,unused-ignore]
656
+ logger.info(
657
+ f"Making tiles with {threads} threads of numba's {layer} threading layer, "
658
+ f"which has {numba_threads()}"
659
+ )
660
+
661
+
662
+ def add_input(task: Task, sums: Sums, threads: int = 1) -> None:
663
+ """Add an input to the sums of a tile, on ``threads`` threads.
664
+
665
+ One thread runs in the calling thread, as any number of Python threads may
666
+ at once. More run on numba's threads, which one Python thread uses at a time.
667
+ """
668
+ n_rows, n_columns = task.x.shape
669
+ if threads <= 1:
670
+ add_block(0, n_rows, 0, n_columns, task, sums)
671
+ return
672
+ chunks = block_chunks(n_rows, n_columns, threads)
673
+ with _PARALLEL:
674
+ numba.set_num_threads(threads)
675
+ add_chunks(chunks, task, sums)
676
+ _log_threads(threads)
677
+
678
+
679
+ def finalise(
680
+ sums: Sums,
681
+ plane_group: NDArray[np.int64],
682
+ combine: int,
683
+ images: NDArray[np.float32],
684
+ weights: NDArray[np.float32],
685
+ threads: int = 1,
686
+ ) -> None:
687
+ """Turn the sums of a tile into its images and weights, on ``threads`` threads, as ``add_input`` uses them."""
688
+ n_rows = images.shape[1] * images.shape[2]
689
+ if threads <= 1:
690
+ finalise_rows(0, n_rows, sums, plane_group, combine, images, weights)
691
+ return
692
+ chunks = row_chunks(n_rows, threads)
693
+ with _PARALLEL:
694
+ numba.set_num_threads(threads)
695
+ finalise_chunks(chunks, sums, plane_group, combine, images, weights)