s4ever 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. s4ever/__init__.py +10 -0
  2. s4ever/_backends/__init__.py +5 -0
  3. s4ever/_backends/metal/__init__.py +64 -0
  4. s4ever/_backends/metal/cauchy.py +111 -0
  5. s4ever/_backends/metal/cauchy_spectrum.metal +431 -0
  6. s4ever/_backends/metal/chunk.metal +553 -0
  7. s4ever/_backends/metal/chunk.py +278 -0
  8. s4ever/_backends/metal/common.metal +25 -0
  9. s4ever/_backends/metal/discretize.metal +76 -0
  10. s4ever/_backends/metal/fft.metal +94 -0
  11. s4ever/_backends/metal/fft.py +425 -0
  12. s4ever/_backends/metal/fft_columns.metal +63 -0
  13. s4ever/_backends/metal/fft_rows.metal +420 -0
  14. s4ever/_backends/metal/msl.py +90 -0
  15. s4ever/_backends/metal/powers.metal +112 -0
  16. s4ever/_backends/metal/powers.py +53 -0
  17. s4ever/_backends/metal/recurrent.metal +162 -0
  18. s4ever/_backends/metal/recurrent.py +136 -0
  19. s4ever/_backends/metal/transpose.metal +23 -0
  20. s4ever/_backends/metal/transpose.py +44 -0
  21. s4ever/_backends/metal/vandermonde.metal +263 -0
  22. s4ever/_backends/metal/vandermonde.py +118 -0
  23. s4ever/_backends/shared.py +111 -0
  24. s4ever/_backends/tilelang/__init__.py +33 -0
  25. s4ever/_backends/tilelang/fftconv.py +632 -0
  26. s4ever/_backends/tilelang/kernels/__init__.py +1 -0
  27. s4ever/_backends/tilelang/kernels/cauchy.py +682 -0
  28. s4ever/_backends/tilelang/kernels/chunk.py +830 -0
  29. s4ever/_backends/tilelang/kernels/discretization.py +93 -0
  30. s4ever/_backends/tilelang/kernels/dsl.py +134 -0
  31. s4ever/_backends/tilelang/kernels/fft.py +611 -0
  32. s4ever/_backends/tilelang/kernels/fftconv.py +1266 -0
  33. s4ever/_backends/tilelang/kernels/final_state.py +143 -0
  34. s4ever/_backends/tilelang/kernels/parameter_transforms.py +161 -0
  35. s4ever/_backends/tilelang/kernels/powers.py +258 -0
  36. s4ever/_backends/tilelang/kernels/probe.py +44 -0
  37. s4ever/_backends/tilelang/kernels/recurrent.py +368 -0
  38. s4ever/_backends/tilelang/kernels/state_inputs.py +104 -0
  39. s4ever/_backends/tilelang/kernels/transpose.py +39 -0
  40. s4ever/_backends/tilelang/kernels/vandermonde.py +360 -0
  41. s4ever/_backends/tilelang/layers.py +256 -0
  42. s4ever/_backends/tilelang/layout.py +13 -0
  43. s4ever/_backends/tilelang/recurrent.py +137 -0
  44. s4ever/_backends/tilelang/runtime.py +374 -0
  45. s4ever/_backends/tilelang/s4.py +264 -0
  46. s4ever/_backends/tilelang/s4d.py +385 -0
  47. s4ever/_backends/tilelang/transpose.py +51 -0
  48. s4ever/_block.py +122 -0
  49. s4ever/_initialization.py +113 -0
  50. s4ever/_layers.py +767 -0
  51. s4ever/ops/__init__.py +5 -0
  52. s4ever/ops/_complex.py +75 -0
  53. s4ever/ops/_fftconv.py +214 -0
  54. s4ever/ops/_s4.py +703 -0
  55. s4ever/ops/_s4d.py +500 -0
  56. s4ever/ops/_shared.py +132 -0
  57. s4ever/py.typed +0 -0
  58. s4ever-0.1.0.dist-info/METADATA +218 -0
  59. s4ever-0.1.0.dist-info/RECORD +61 -0
  60. s4ever-0.1.0.dist-info/WHEEL +4 -0
  61. s4ever-0.1.0.dist-info/licenses/LICENSE +202 -0
s4ever/__init__.py ADDED
@@ -0,0 +1,10 @@
1
+ """S4 and S4D state space sequence layers for PyTorch.
2
+
3
+ The GPU kernels (TileLang on CUDA and ROCm, Metal on MPS) are compiled at first use, so the package
4
+ has no extension to build.
5
+ """
6
+
7
+ from ._block import S4Block
8
+ from ._layers import S4, S4D, param_groups
9
+
10
+ __all__ = ["S4", "S4D", "S4Block", "param_groups"]
@@ -0,0 +1,5 @@
1
+ """The fused kernels of the ops.
2
+
3
+ The TileLang kernels in `tilelang` run on NVIDIA and AMD GPUs, and the Metal kernels in `metal` run
4
+ on Apple GPUs.
5
+ """
@@ -0,0 +1,64 @@
1
+ """Metal kernels for Apple GPUs.
2
+
3
+ `torch.mps.compile_shader` compiles each library on first use. The kernels compute in float32 and
4
+ read and write sequences in the dtype of the input. The chunk kernels write y and du in float32
5
+ instead when the modes form several groups (`chunk.out_dtype`).
6
+ """
7
+
8
+ import functools
9
+ import warnings
10
+
11
+ import torch
12
+ from torch import Tensor
13
+
14
+ from s4ever._backends import shared
15
+
16
+ from . import cauchy, chunk, fft, powers, recurrent, transpose, vandermonde
17
+
18
+
19
+ def enabled(x: Tensor, fused_kernels: bool | None) -> bool:
20
+ return (
21
+ x.device.type == "mps"
22
+ and x.dtype in shared.INPUT_DTYPES
23
+ and shared.wanted(fused_kernels)
24
+ and not unavailable()
25
+ )
26
+
27
+
28
+ @shared.dynamo_constant
29
+ def unavailable() -> str:
30
+ """Return the error message of the probe, or "" if the probe runs.
31
+
32
+ The function is a Dynamo constant because Dynamo cannot trace the probe.
33
+ """
34
+ return probe()
35
+
36
+
37
+ @functools.cache
38
+ def probe() -> str:
39
+ try:
40
+ transpose.transpose(torch.ones(1, 2, 2, device="mps"))
41
+ except Exception as e: # Metal compilers and drivers fail in version-specific ways
42
+ warnings.warn(
43
+ "s4ever: the ops use PyTorch unless fused_kernels=True, because the Metal kernels "
44
+ f"cannot be compiled or run: {e}",
45
+ # The warning names the line in s4ever that first asked whether the kernels are
46
+ # available.
47
+ stacklevel=3,
48
+ )
49
+ # The repr is not empty when the exception has no message.
50
+ return str(e) or repr(e)
51
+ return ""
52
+
53
+
54
+ __all__ = [
55
+ "cauchy",
56
+ "chunk",
57
+ "enabled",
58
+ "fft",
59
+ "powers",
60
+ "recurrent",
61
+ "transpose",
62
+ "unavailable",
63
+ "vandermonde",
64
+ ]
@@ -0,0 +1,111 @@
1
+ """S4's convolution kernel from its parameters (`s4_conv_kernel`) on Metal.
2
+
3
+ The kernels use the method that `_backends.tilelang.kernels.cauchy` describes.
4
+ """
5
+
6
+ import functools
7
+
8
+ import torch
9
+ from torch import Tensor
10
+
11
+ from s4ever._backends import shared
12
+ from s4ever._backends.shared import ceil_div
13
+
14
+ from . import fft, msl
15
+
16
+ BK = 128 # frequencies per threadgroup of `spectrum`, one per thread
17
+
18
+
19
+ @functools.cache
20
+ def spectrum_library(N: int, C: int):
21
+ prelude = f"constant constexpr uint THREADS = {BK}, N = {N}, C = {C};\n"
22
+ names = ("spectrum", "spectrum_grads", "spectrum_finish")
23
+ return msl.library(msl.source(prelude, "cauchy_spectrum.metal"), dict.fromkeys(names, BK))
24
+
25
+
26
+ def fits(N: int) -> bool:
27
+ return N <= shared.MAX_CONV_KERNEL_MODES and builds()
28
+
29
+
30
+ @msl.constant
31
+ def builds() -> bool:
32
+ """Return whether the kernels build for the most modes and channels, and warn if they do not.
33
+
34
+ The kernels use the most registers and threadgroup memory at these sizes.
35
+ """
36
+ args = (shared.MAX_CONV_KERNEL_MODES, shared.MAX_CHANNELS)
37
+ return msl.builds("S4's convolution kernels", spectrum_library, *args)
38
+
39
+
40
+ @shared.register_op(
41
+ "metal_s4_conv_kernel", lambda dt, A, P, B, C, L: C.new_empty(*C.shape[:2], L), "MPS"
42
+ )
43
+ def s4_conv_kernel_op(dt: Tensor, A: Tensor, P: Tensor, B: Tensor, C: Tensor, L: int) -> Tensor:
44
+ Cc, H, N, _ = C.shape
45
+ F = L // 2 + 1
46
+ k_f = C.new_empty(Cc, H, F, 2)
47
+ threads, group = [H * ceil_div(F, BK) * BK, 1, 1], [BK, 1, 1]
48
+ # The kernels run for at most `shared.MAX_CHANNELS` channels at a time, but
49
+ # `ops._s4.state_response` passes each batch row of x0 as a channel, and the batch can have
50
+ # more rows.
51
+ for c in range(0, Cc, shared.MAX_CHANNELS):
52
+ Cs = C[c : c + shared.MAX_CHANNELS]
53
+ lib = spectrum_library(N, len(Cs))
54
+ lib.spectrum(k_f[c : c + len(Cs)], dt, A, P, B, Cs, H, L, threads=threads, group_size=group)
55
+ if fft.real_fits(L):
56
+ return fft.real_inverse(k_f, L)
57
+ return torch.fft.irfft(torch.view_as_complex(k_f), n=L, norm="forward")
58
+
59
+
60
+ def grads_outputs(
61
+ dt: Tensor, A: Tensor, P: Tensor, B: Tensor, C: Tensor, g: Tensor
62
+ ) -> list[Tensor]:
63
+ return [torch.empty_like(x) for x in (dt, A, P, B, C)]
64
+
65
+
66
+ @shared.register_op("metal_s4_conv_kernel_grads", grads_outputs, "MPS")
67
+ def s4_conv_kernel_grads_op(
68
+ dt: Tensor, A: Tensor, P: Tensor, B: Tensor, C: Tensor, g: Tensor
69
+ ) -> list[Tensor]:
70
+ (Cc, H, N, _), L = C.shape, g.shape[-1]
71
+ gf = torch.view_as_real(torch.fft.rfft(g, n=L))
72
+ n_freq = (L + 1) // 2 - 1 # the frequencies 0 < k < L / 2
73
+ # Each threadgroup sums `split_tiles` tiles of BK frequencies, so that the dispatch has about
74
+ # msl.GROUPS threadgroups.
75
+ tiles = ceil_div(n_freq, BK)
76
+ group = [BK, 1, 1]
77
+ split_tiles = ceil_div(tiles, min(tiles, ceil_div(msl.GROUPS, H))) if tiles else 0
78
+ splits = ceil_div(tiles, split_tiles) if tiles else 1
79
+ # Each kernel runs once for all channels. The backward has at most `shared.MAX_CHANNELS`
80
+ # channels, because S4 runs `ops._s4.state_response` only without gradients.
81
+ grads = grads_outputs(dt, A, P, B, C, g)
82
+ lib = spectrum_library(N, Cc)
83
+ rows = 4 * (Cc + 1) + 2 # COEFS + 2 in cauchy_spectrum.metal
84
+ part = C.new_empty(H, splits, rows, N) if tiles else C.new_zeros(H, 1, rows, N)
85
+ args = (dt, A, P, B, C, H, L, splits)
86
+ if tiles:
87
+ lib.spectrum_grads(
88
+ part, gf, *args, split_tiles, threads=[H * splits * BK, 1, 1], group_size=group
89
+ )
90
+ lib.spectrum_finish(*grads, part, gf, *args, threads=[H * BK, 1, 1], group_size=group)
91
+ return grads
92
+
93
+
94
+ class S4ConvKernel(torch.autograd.Function):
95
+ @staticmethod
96
+ def forward(ctx, dt, A, P, B, C, L):
97
+ ctx.save_for_backward(dt, A, P, B, C)
98
+ return s4_conv_kernel_op(dt, A, P, B, C, L)
99
+
100
+ @staticmethod
101
+ def backward(ctx, *grads):
102
+ shared.refuse_double_backward()
103
+ return *s4_conv_kernel_grads_op(*ctx.saved_tensors, grads[0].contiguous()), None
104
+
105
+
106
+ def s4_conv_kernel(dt: Tensor, A: Tensor, P: Tensor, B: Tensor, C: Tensor, L: int) -> Tensor:
107
+ """Return `s4ever.ops.s4_conv_kernel` computed by the Metal kernels.
108
+
109
+ Callers check `fits` first.
110
+ """
111
+ return S4ConvKernel.apply(*shared.contiguous_f32(dt, A, P, B, C), L)
@@ -0,0 +1,431 @@
1
+ // The kernels of cauchy.py, which use the method that `tilelang.kernels.cauchy` describes. They are
2
+ // compiled after the constants THREADS, N and C of `spectrum_library`. In `spectrum` and
3
+ // `spectrum_grads`, each thread handles one frequency of a tile of THREADS frequencies.
4
+
5
+ // The resolvent r_XY of the pair (X_x, Y_y) has index x (C + 1) + y, for X = (B, P),
6
+ // Y = (C..., Q) and Q = conj(P), so r[BQ] is r_BQ and r[PQ] is r_PQ. Each pair has the two Cauchy
7
+ // coefficients coef_0 and coef_1.
8
+ constant constexpr uint PAIRS = 2 * (C + 1), COEFS = 2 * PAIRS, BQ = C, PQ = 2 * C + 1;
9
+ constant constexpr uint SIMDS = THREADS / 32; // SIMD groups per threadgroup
10
+ // The tables of the Cauchy sums hold one chunk of the modes at a time. The modes form CHUNKS chunks
11
+ // of CHUNK_MODES <= THREADS modes, so that each thread loads at most one mode of a chunk.
12
+ constant constexpr uint CHUNKS = (N + THREADS - 1) / THREADS;
13
+ constant constexpr uint CHUNK_MODES = (N + CHUNKS - 1) / CHUNKS;
14
+ // `others` adds the terms of r_XY(0) in blocks of BLOCK_MODES modes, so that it takes about
15
+ // N / BLOCK_MODES + BLOCK_MODES steps per mode instead of N.
16
+ constant constexpr uint BLOCK_MODES = 16, BLOCKS = (N + BLOCK_MODES - 1) / BLOCK_MODES;
17
+
18
+ // Return nu = 2 tan(pi k / L) for 0 <= k <= L / 2. For k > L / 4, the function evaluates
19
+ // 2 cot(pi (L - 2 k) / (2 L)) instead, so that the angle stays at most pi / 4 and each argument of
20
+ // sinpi and cospi is computed with one rounding.
21
+ inline float frequency(long k, long L) {
22
+ float x = float(k) / float(L), z = float(L - 2 * k) / float(2 * L);
23
+ return 4 * k <= L ? 2 * precise::sinpi(x) / precise::cospi(x)
24
+ : 2 * precise::cospi(z) / precise::sinpi(z);
25
+ }
26
+
27
+ // The parameters of one mode, with u[x (C + 1) + y] = X_x Y_y.
28
+ struct Mode {
29
+ float2 lam, X[2], Y[C + 1], u[PAIRS];
30
+ };
31
+
32
+ inline Mode load_mode(device const float2* Lam, device const float2* P, device const float2* B,
33
+ device const float2* Cm, long H, long h, uint n) {
34
+ Mode mode;
35
+ long i = h * N + n;
36
+ mode.lam = Lam[i];
37
+ mode.X[0] = B[i];
38
+ mode.X[1] = P[i];
39
+ for (uint c = 0; c < C; c++) mode.Y[c] = Cm[c * H * N + i];
40
+ mode.Y[C] = conjugate(P[i]);
41
+ for (uint x = 0; x < 2; x++)
42
+ for (uint y = 0; y <= C; y++) mode.u[x * (C + 1) + y] = cmul(mode.X[x], mode.Y[y]);
43
+ return mode;
44
+ }
45
+
46
+ // Load the tables of the chunk that starts at mode m0, whose entry n holds mode i = m0 + n. aw[n]
47
+ // holds a = Re(w) and |w|^2 for the pole w, and cs[n] holds the coefficients.
48
+ // w = dt Lambda
49
+ // v = dt u
50
+ // coef_0 = -2 Re(v conj(w))
51
+ // coef_1 = 2 Re(v)
52
+ // cs[n][p PAIRS + j] = coef_p[j, i]
53
+ // The entries past the last mode have zero coefficients, so they add nothing to the sums.
54
+ inline void tables(threadgroup float (&cs)[CHUNK_MODES][COEFS],
55
+ threadgroup float2 (&aw)[CHUNK_MODES], device const float* dt,
56
+ device const float2* Lam, device const float2* P, device const float2* B,
57
+ device const float2* Cm, long H, long h, uint t, uint m0) {
58
+ for (uint n = t; n < CHUNK_MODES; n += THREADS) {
59
+ uint i = m0 + n;
60
+ Mode mode = load_mode(Lam, P, B, Cm, H, h, min(i, N - 1));
61
+ float dt_h = dt[h];
62
+ float2 w = dt_h * mode.lam;
63
+ aw[n] = float2(w.x, dot(w, w));
64
+ for (uint j = 0; j < PAIRS; j++) {
65
+ float2 v = dt_h * mode.u[j];
66
+ cs[n][j] = i < N ? -2 * dot(v, w) : 0;
67
+ cs[n][PAIRS + j] = i < N ? 2 * v.x : 0;
68
+ }
69
+ }
70
+ threadgroup_barrier(mem_flags::mem_threadgroup);
71
+ }
72
+
73
+ // Return m = 1 / q_n at z = i nu for the table entry aw = (a_n, |w_n|^2).
74
+ // q_n = (|w_n|^2 - nu^2) - 2i nu a_n
75
+ // The forward and the backward both call this function, so that they round |w_n|^2 - nu^2 the same
76
+ // way. Near resonance, |w_n|^2 - nu^2 is small, and a rounding difference between the passes would
77
+ // make the gradients of P, B and C inaccurate.
78
+ inline float2 inverse_q(float nu, float2 aw) {
79
+ float2 q_conj = float2(fma(-nu, nu, aw.y), 2 * nu * aw.x);
80
+ return q_conj * (1 / dot(q_conj, q_conj));
81
+ }
82
+
83
+ // Add sum_n cs[n][j] / q_n over the modes of the tables to S[j] at z = i nu.
84
+ inline void sums(threadgroup const float (&cs)[CHUNK_MODES][COEFS],
85
+ threadgroup const float2 (&aw)[CHUNK_MODES], float nu,
86
+ thread float2 (&S)[COEFS]) {
87
+ for (uint n = 0; n < CHUNK_MODES; n++) {
88
+ float2 m = inverse_q(nu, aw[n]);
89
+ for (uint j = 0; j < COEFS; j++) S[j] += cs[n][j] * m;
90
+ }
91
+ }
92
+
93
+ // Set the resolvents r = S0 + i nu S1 at z = i nu, s = 1 / (1 + r_PQ) and b = r_BQ s.
94
+ inline void resolvents(thread const float2 (&S)[COEFS], float nu, thread float2 (&r)[PAIRS],
95
+ thread float2& s, thread float2& b) {
96
+ for (uint j = 0; j < PAIRS; j++) r[j] = S[j] + nu * float2(-S[PAIRS + j].y, S[PAIRS + j].x);
97
+ float2 den = float2(1 + r[PQ].x, r[PQ].y);
98
+ s = conjugate(den) / dot(den, den);
99
+ b = cmul(r[BQ], s);
100
+ }
101
+
102
+ // Return the term -2 Re(u_j / Lambda) that the mode adds to r_XY(0) for pair j.
103
+ inline float term(Mode mode, uint j) {
104
+ return -2 * dot(mode.u[j], mode.lam) / dot(mode.lam, mode.lam);
105
+ }
106
+
107
+ // Set block_terms[block][j] to the sum of terms[i][j] over the modes i of the block. Thread t sums
108
+ // block t.
109
+ template <uint J>
110
+ inline void block_sums(threadgroup const float (&terms)[N][J],
111
+ threadgroup float (&block_terms)[BLOCKS][J], uint t) {
112
+ if (t < BLOCKS)
113
+ for (uint j = 0; j < J; j++) {
114
+ float sum = 0;
115
+ for (uint i = t * BLOCK_MODES; i < min(N, t * BLOCK_MODES + BLOCK_MODES); i++)
116
+ sum += terms[i][j];
117
+ block_terms[t][j] = sum;
118
+ }
119
+ }
120
+
121
+ // Set o[j] to the sum of terms[i][j] over the modes i other than n, and r0[j] to the sum over all
122
+ // modes. For N > 2 BLOCK_MODES, the sums add the block sums and then the terms of n's block. The
123
+ // sum o never adds the term of mode n, so a large term of mode n does not leave a rounding error
124
+ // in o.
125
+ template <uint J>
126
+ inline void others(threadgroup const float (&terms)[N][J],
127
+ threadgroup const float (&block_terms)[BLOCKS][J], uint n, thread float (&o)[J],
128
+ thread float (&r0)[J]) {
129
+ for (uint j = 0; j < J; j++) o[j] = r0[j] = 0;
130
+ if (N <= 2 * BLOCK_MODES) {
131
+ for (uint i = 0; i < N; i++)
132
+ for (uint j = 0; j < J; j++) {
133
+ o[j] += i == n ? 0 : terms[i][j];
134
+ r0[j] += terms[i][j];
135
+ }
136
+ return;
137
+ }
138
+ for (uint block = 0; block < BLOCKS; block++)
139
+ for (uint j = 0; j < J; j++) {
140
+ o[j] += block == n / BLOCK_MODES ? 0 : block_terms[block][j];
141
+ r0[j] += block_terms[block][j];
142
+ }
143
+ uint i0 = n / BLOCK_MODES * BLOCK_MODES;
144
+ for (uint i = i0; i < min(N, i0 + BLOCK_MODES); i++)
145
+ for (uint j = 0; j < J; j++) o[j] += i == n ? 0 : terms[i][j];
146
+ }
147
+
148
+ // Return entry n of the solution x of (Lambda - W W^*) x = V at z = 0, for (V, W) = (B, P) or
149
+ // (C, Q). The argument o_pq sums the terms of r_PQ(0) over the modes other than n, and o_v sums
150
+ // those of r_BQ(0) for V = B and of r_PC(0) for V = C. The argument den is 1 + r_PQ(0). The
151
+ // function adds the terms of mode n in closed form.
152
+ // x_n = (V_n (1 + o_pq) - W_n o_v - 2i W_n Im(V_n conj(W_n)) / conj(Lambda_n)) / (Lambda_n den)
153
+ inline float2 solve(float2 V, float2 W, float2 lam, float o_pq, float o_v, float den) {
154
+ float scale = 1 / (dot(lam, lam) * den), im = V.y * W.x - V.x * W.y; // Im(V conj(W))
155
+ float2 num = V + (V * o_pq - W * o_v);
156
+ return float2(num.x * lam.x + num.y * lam.y + 2 * im * W.y,
157
+ num.y * lam.x - num.x * lam.y - 2 * im * W.x) * scale;
158
+ }
159
+
160
+ // Write k_f / L (C, H, L / 2 + 1, 2) from dt (H), Lambda, P and B (H, N, 2), and Cm (C, H, N, 2).
161
+ kernel void spectrum(device float2* out, device const float* dt, device const float2* Lam,
162
+ device const float2* P, device const float2* B, device const float2* Cm,
163
+ constant long& H, constant long& L, IDS) {
164
+ threadgroup float cs[CHUNK_MODES][COEFS];
165
+ threadgroup float terms[N][2], block_terms[BLOCKS][2];
166
+ threadgroup float zero_parts[C][CHUNK_MODES], nyquist_parts[C][CHUNK_MODES];
167
+ threadgroup float2 aw[CHUNK_MODES];
168
+ long F = L / 2 + 1;
169
+ uint tiles = uint(F + THREADS - 1) / THREADS, h = g / tiles, tile = g % tiles;
170
+ long k = tile * THREADS + t, K = (L + 1) / 2 - 1; // K is the last bin below L / 2.
171
+ float scale = 1 / float(L);
172
+ tables(cs, aw, dt, Lam, P, B, Cm, H, h, t, 0);
173
+ if (tile == 0) {
174
+ // At k = 0, the terms of r_XY(0) are large for nearly undamped modes, and their rounding
175
+ // errors would dominate r_BC - b r_PC. The kernel computes bin 0 as -2 Re sum_n C_n x_n
176
+ // instead, where x solves (Lambda - P P^*) x = B. Thread t < CHUNK_MODES handles the modes
177
+ // t, t + CHUNK_MODES, ...
178
+ uint first = t < CHUNK_MODES ? t : N;
179
+ for (uint n = first; n < N; n += CHUNK_MODES) {
180
+ Mode mode = load_mode(Lam, P, B, Cm, H, h, n);
181
+ terms[n][0] = term(mode, BQ);
182
+ terms[n][1] = term(mode, PQ);
183
+ }
184
+ threadgroup_barrier(mem_flags::mem_threadgroup);
185
+ if (N > 2 * BLOCK_MODES) {
186
+ block_sums(terms, block_terms, t);
187
+ threadgroup_barrier(mem_flags::mem_threadgroup);
188
+ }
189
+ float zero_sum[C] = {}, nyquist_sum[C] = {};
190
+ for (uint n = first; n < N; n += CHUNK_MODES) {
191
+ Mode mode = load_mode(Lam, P, B, Cm, H, h, n);
192
+ float o[2], r0[2];
193
+ others(terms, block_terms, n, o, r0);
194
+ float2 x = solve(mode.X[0], mode.X[1], mode.lam, o[1], o[0], 1 + r0[1]);
195
+ for (uint c = 0; c < C; c++) {
196
+ zero_sum[c] += mode.Y[c].x * x.x - mode.Y[c].y * x.y;
197
+ nyquist_sum[c] += dt[h] * mode.u[c].x;
198
+ }
199
+ }
200
+ if (t < CHUNK_MODES)
201
+ for (uint c = 0; c < C; c++) {
202
+ zero_parts[c][t] = zero_sum[c];
203
+ nyquist_parts[c][t] = nyquist_sum[c];
204
+ }
205
+ threadgroup_barrier(mem_flags::mem_threadgroup);
206
+ if (t < C) {
207
+ // For even L, the Nyquist bin is the limit sum_n Re v_BC.
208
+ float zero = 0, nyquist = 0;
209
+ for (uint n = 0; n < CHUNK_MODES; n++) {
210
+ zero += zero_parts[t][n];
211
+ nyquist += nyquist_parts[t][n];
212
+ }
213
+ out[(t * H + h) * F] = float2(-2 * zero * scale, 0);
214
+ if (L % 2 == 0) out[(t * H + h) * F + L / 2] = float2(nyquist * scale, 0);
215
+ }
216
+ }
217
+ // With more than one chunk, the threads with k outside 0 < k <= K stay to load the tables of
218
+ // the later chunks.
219
+ bool active = k >= 1 && k <= K;
220
+ if (CHUNKS == 1 && !active) return;
221
+ float nu = frequency(k, L);
222
+ float2 S[COEFS] = {}, r[PAIRS], s, b;
223
+ for (uint m0 = 0; m0 < N; m0 += CHUNK_MODES) {
224
+ if (m0 > 0) {
225
+ threadgroup_barrier(mem_flags::mem_threadgroup);
226
+ tables(cs, aw, dt, Lam, P, B, Cm, H, h, t, m0);
227
+ }
228
+ if (active) sums(cs, aw, nu, S);
229
+ }
230
+ if (!active) return;
231
+ resolvents(S, nu, r, s, b);
232
+ // k_f / L = (1 + z / 2) (r_BC - b r_PC) / L with z = i nu
233
+ for (uint c = 0; c < C; c++) {
234
+ float2 v = r[c] - cmul(b, r[C + 1 + c]);
235
+ out[(c * H + h) * F + k] = (v + nu / 2 * float2(-v.y, v.x)) * scale;
236
+ }
237
+ }
238
+
239
+ // Write the partial sums of the gradients to part (H, splits, COEFS + 2, N). Rows j < COEFS hold
240
+ // the gradients of the Cauchy coefficients, and rows COEFS and COEFS + 1 hold the gradients of a
241
+ // and |w|^2. The input gf (C, H, L / 2 + 1, 2) is the rfft of the kernel's gradient. Threadgroup
242
+ // (h, split) sums over the frequencies 0 < k < L / 2 of its split_tiles tiles. In each tile, the
243
+ // threads first store nu and the gradients GS of the sums S at their frequencies. Then thread
244
+ // t = k0 CHUNK_MODES + n sums the terms of table entry n over the frequencies k0,
245
+ // k0 + ENTRY_THREADS, ... of the tile.
246
+ kernel void spectrum_grads(device float* part, device const float2* gf, device const float* dt,
247
+ device const float2* Lam, device const float2* P,
248
+ device const float2* B, device const float2* Cm, constant long& H,
249
+ constant long& L, constant long& splits, constant long& split_tiles,
250
+ IDS) {
251
+ // ENTRY_THREADS threads share each table entry. With more than one chunk,
252
+ // CHUNK_MODES > THREADS / 2, so ENTRY_THREADS is 1, and the threads with k0 = 0 hold all the
253
+ // sums.
254
+ constexpr uint ENTRY_THREADS = THREADS / CHUNK_MODES;
255
+ threadgroup float cs[CHUNK_MODES][COEFS], nus[THREADS];
256
+ threadgroup float2 aw[CHUNK_MODES], GS[THREADS][COEFS];
257
+ long F = L / 2 + 1, K = (L + 1) / 2 - 1;
258
+ uint h = g / uint(splits), split = g % uint(splits);
259
+ tables(cs, aw, dt, Lam, P, B, Cm, H, h, t, 0);
260
+ uint n = t % CHUNK_MODES, k0 = t / CHUNK_MODES;
261
+ float cn[COEFS], acc[COEFS + 2] = {};
262
+ float2 wn;
263
+ auto load_entry = [&]() {
264
+ for (uint j = 0; j < COEFS; j++) cn[j] = cs[n][j];
265
+ wn = aw[n];
266
+ };
267
+ if (CHUNKS == 1) load_entry();
268
+ for (long i = 0; i < split_tiles; i++) {
269
+ long k = (split * split_tiles + i) * THREADS + t + 1, kc = min(k, K);
270
+ float nu = frequency(kc, L), weight = k <= K ? 2 / float(L) : 0;
271
+ float2 S[COEFS] = {}, r[PAIRS], s, b, gr[PAIRS], gb = 0;
272
+ sums(cs, aw, nu, S);
273
+ for (uint m0 = CHUNK_MODES; m0 < N; m0 += CHUNK_MODES) {
274
+ threadgroup_barrier(mem_flags::mem_threadgroup);
275
+ tables(cs, aw, dt, Lam, P, B, Cm, H, h, t, m0);
276
+ sums(cs, aw, nu, S);
277
+ }
278
+ resolvents(S, nu, r, s, b);
279
+ for (uint c = 0; c < C; c++) {
280
+ float2 gk = gf[(c * H + h) * F + kc] * weight;
281
+ float2 p = gk + nu / 2 * float2(gk.y, -gk.x); // p = conj(1 + i nu / 2) gk
282
+ // k_f = (1 + z/2) (r_BC - b r_PC)
283
+ gr[c] = p;
284
+ gr[C + 1 + c] = -cmulc(p, b);
285
+ gb -= cmulc(p, r[C + 1 + c]);
286
+ }
287
+ // G_BQ = conj(s) G_b and G_PQ = -conj(s^2 r_BQ) G_b.
288
+ gr[BQ] = cmulc(gb, s);
289
+ gr[PQ] = -cmulc(gb, cmul(b, s));
290
+ // r = S0 + i nu S1 gives G_S0 = G_r and G_S1 = -i nu G_r.
291
+ for (uint j = 0; j < PAIRS; j++) {
292
+ GS[t][j] = gr[j];
293
+ GS[t][PAIRS + j] = nu * float2(gr[j].y, -gr[j].x);
294
+ }
295
+ nus[t] = nu;
296
+ threadgroup_barrier(mem_flags::mem_threadgroup);
297
+ // The chunks run in reverse order, because the tables already hold the last chunk.
298
+ for (uint ci = CHUNKS; ci-- > 0;) {
299
+ uint m0 = ci * CHUNK_MODES;
300
+ if (CHUNKS > 1) {
301
+ if (ci < CHUNKS - 1) {
302
+ threadgroup_barrier(mem_flags::mem_threadgroup);
303
+ tables(cs, aw, dt, Lam, P, B, Cm, H, h, t, m0);
304
+ }
305
+ load_entry();
306
+ for (uint j = 0; j < COEFS + 2; j++) acc[j] = 0;
307
+ }
308
+ if (k0 < ENTRY_THREADS)
309
+ for (uint freq = k0; freq < THREADS; freq += ENTRY_THREADS) {
310
+ float nuk = nus[freq];
311
+ float2 m = inverse_q(nuk, wn), gm = 0;
312
+ for (uint j = 0; j < COEFS; j++) {
313
+ gm += cn[j] * GS[freq][j];
314
+ acc[j] += dot(m, GS[freq][j]);
315
+ }
316
+ // G_q = -conj(m^2) G_m, and q_n = (|w|^2 - nu^2) - 2i nu a gives the gradients
317
+ // of a and |w|^2.
318
+ float c2 = m.x * m.x - m.y * m.y, s2 = 2 * m.x * m.y;
319
+ acc[COEFS] += 2 * nuk * (c2 * gm.y - s2 * gm.x);
320
+ acc[COEFS + 1] -= c2 * gm.x + s2 * gm.y;
321
+ }
322
+ if (CHUNKS > 1 && k0 == 0 && m0 + n < N)
323
+ for (uint j = 0; j < COEFS + 2; j++) {
324
+ device float* partial =
325
+ part + ((h * splits + split) * (COEFS + 2) + j) * N + m0 + n;
326
+ *partial = i == 0 ? acc[j] : *partial + acc[j];
327
+ }
328
+ }
329
+ threadgroup_barrier(mem_flags::mem_threadgroup);
330
+ }
331
+ if (CHUNKS > 1) return;
332
+ threadgroup float* red = (threadgroup float*)GS; // (ENTRY_THREADS, N, COEFS + 2)
333
+ if (k0 < ENTRY_THREADS)
334
+ for (uint j = 0; j < COEFS + 2; j++) red[(k0 * N + n) * (COEFS + 2) + j] = acc[j];
335
+ threadgroup_barrier(mem_flags::mem_threadgroup);
336
+ if (t < N)
337
+ for (uint j = 0; j < COEFS + 2; j++) {
338
+ float tot = 0;
339
+ for (uint e = 0; e < ENTRY_THREADS; e++) tot += red[(e * N + t) * (COEFS + 2) + j];
340
+ part[((h * splits + split) * (COEFS + 2) + j) * N + t] = tot;
341
+ }
342
+ }
343
+
344
+ // Write the gradients of dt (H), Lambda, P and B (H, N, 2), and Cm (C, H, N, 2) from the partial
345
+ // sums in `part` and the bins 0 and L / 2 of gf.
346
+ kernel void spectrum_finish(device float* gdt, device float2* gLam, device float2* gP,
347
+ device float2* gB, device float2* gC, device const float* part,
348
+ device const float2* gf, device const float* dt,
349
+ device const float2* Lam, device const float2* P,
350
+ device const float2* B, device const float2* Cm, constant long& H,
351
+ constant long& L, constant long& splits, IDS) {
352
+ // The columns of `terms` are the terms of r_BQ(0), r_PC(0)... and r_PQ(0).
353
+ threadgroup float terms[N][C + 2], block_terms[BLOCKS][C + 2], red[SIMDS];
354
+ long h = g, F = L / 2 + 1;
355
+ for (uint n = t; n < N; n += THREADS) {
356
+ Mode mode = load_mode(Lam, P, B, Cm, H, h, n);
357
+ for (uint j = 0; j < C + 2; j++) terms[n][j] = term(mode, BQ + j);
358
+ }
359
+ threadgroup_barrier(mem_flags::mem_threadgroup);
360
+ if (N > 2 * BLOCK_MODES) {
361
+ block_sums(terms, block_terms, t);
362
+ threadgroup_barrier(mem_flags::mem_threadgroup);
363
+ }
364
+ float gdt_thread = 0;
365
+ for (uint m0 = 0; m0 < N; m0 += THREADS) {
366
+ uint n = min(m0 + t, N - 1);
367
+ Mode mode = load_mode(Lam, P, B, Cm, H, h, n);
368
+ float o[C + 2], r0[C + 2];
369
+ others(terms, block_terms, n, o, r0);
370
+ // M = Lambda - P P^* is a matrix over the modes and their conjugates. The gradient of a
371
+ // parameter is 2 conj(G), where G is the derivative of k_f(0) / L with respect to the
372
+ // parameter.
373
+ // k_f(0) / L = -C^T x / L
374
+ // x = M^-1 B
375
+ // mu = M^-T C
376
+ float den = 1 + r0[C + 1], gs = 0; // gs = sum_c g_c P^T mu_c, P^T mu_c = -r_PC / den
377
+ float2 x = solve(mode.X[0], mode.X[1], mode.lam, o[C + 1], o[0], den), gmu = 0, gc[C];
378
+ for (uint c = 0; c < C; c++) {
379
+ float gk = gf[(c * H + h) * F].x / float(L);
380
+ gmu += gk * solve(mode.Y[c], mode.Y[C], mode.lam, o[C + 1], o[1 + c], den);
381
+ gs -= gk * r0[1 + c] / den;
382
+ gc[c] = -2 * gk * conjugate(x);
383
+ }
384
+ float qx = -r0[0] / den; // Q^T x
385
+ // Bin 0 adds these gradients of Lambda and P.
386
+ float2 zero_gLam = 2 * conjugate(cmul(gmu, x));
387
+ float2 zero_gP = -2 * (qx * conjugate(gmu) + gs * x);
388
+ // Sum the rows of `part` for mode n over the splits.
389
+ float gcoef[COEFS + 2] = {};
390
+ for (long split = 0; split < splits; split++)
391
+ for (uint j = 0; j < COEFS + 2; j++)
392
+ gcoef[j] += part[((h * splits + split) * (COEFS + 2) + j) * N + n];
393
+ float dt_h = dt[h], nyquist = L % 2 == 0 ? 1 / float(L) : 0;
394
+ float2 w = dt_h * mode.lam;
395
+ float2 gw = float2(gcoef[COEFS] + 2 * w.x * gcoef[COEFS + 1], 2 * w.y * gcoef[COEFS + 1]);
396
+ float2 gv[PAIRS];
397
+ for (uint j = 0; j < PAIRS; j++) {
398
+ float g0 = gcoef[j], g1 = gcoef[PAIRS + j];
399
+ gv[j] = float2(-2 * w.x * g0 + 2 * g1, -2 * w.y * g0);
400
+ gw -= 2 * dt_h * mode.u[j] * g0;
401
+ if (j < C) gv[j].x += nyquist * gf[(j * H + h) * F + L / 2].x;
402
+ }
403
+ float gdt_mode = dot(mode.lam, gw);
404
+ // gX and gY start from the gradients that bin 0 gives.
405
+ float2 gX[2] = {-2 * conjugate(gmu), zero_gP}, gY[C + 1];
406
+ for (uint c = 0; c < C; c++) gY[c] = gc[c];
407
+ gY[C] = 0;
408
+ for (uint i = 0; i < 2; i++)
409
+ for (uint c = 0; c <= C; c++) {
410
+ uint j = i * (C + 1) + c;
411
+ gdt_mode += dot(gv[j], mode.u[j]);
412
+ gX[i] += cmulc(dt_h * gv[j], mode.Y[c]);
413
+ gY[c] += cmulc(dt_h * gv[j], mode.X[i]);
414
+ }
415
+ if (m0 + t < N) {
416
+ long i = h * N + n;
417
+ gB[i] = gX[0];
418
+ gP[i] = gX[1] + conjugate(gY[C]);
419
+ gLam[i] = dt_h * gw + zero_gLam;
420
+ for (uint c = 0; c < C; c++) gC[c * H * N + i] = gY[c];
421
+ gdt_thread += gdt_mode;
422
+ }
423
+ }
424
+ gdt_thread = simd_sum(gdt_thread);
425
+ if (lane == 0) red[simd] = gdt_thread;
426
+ threadgroup_barrier(mem_flags::mem_threadgroup);
427
+ if (t == 0) {
428
+ for (uint i = 1; i < SIMDS; i++) gdt_thread += red[i];
429
+ gdt[h] = gdt_thread;
430
+ }
431
+ }