s4ever 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- s4ever/__init__.py +10 -0
- s4ever/_backends/__init__.py +5 -0
- s4ever/_backends/metal/__init__.py +64 -0
- s4ever/_backends/metal/cauchy.py +111 -0
- s4ever/_backends/metal/cauchy_spectrum.metal +431 -0
- s4ever/_backends/metal/chunk.metal +553 -0
- s4ever/_backends/metal/chunk.py +278 -0
- s4ever/_backends/metal/common.metal +25 -0
- s4ever/_backends/metal/discretize.metal +76 -0
- s4ever/_backends/metal/fft.metal +94 -0
- s4ever/_backends/metal/fft.py +425 -0
- s4ever/_backends/metal/fft_columns.metal +63 -0
- s4ever/_backends/metal/fft_rows.metal +420 -0
- s4ever/_backends/metal/msl.py +90 -0
- s4ever/_backends/metal/powers.metal +112 -0
- s4ever/_backends/metal/powers.py +53 -0
- s4ever/_backends/metal/recurrent.metal +162 -0
- s4ever/_backends/metal/recurrent.py +136 -0
- s4ever/_backends/metal/transpose.metal +23 -0
- s4ever/_backends/metal/transpose.py +44 -0
- s4ever/_backends/metal/vandermonde.metal +263 -0
- s4ever/_backends/metal/vandermonde.py +118 -0
- s4ever/_backends/shared.py +111 -0
- s4ever/_backends/tilelang/__init__.py +33 -0
- s4ever/_backends/tilelang/fftconv.py +632 -0
- s4ever/_backends/tilelang/kernels/__init__.py +1 -0
- s4ever/_backends/tilelang/kernels/cauchy.py +682 -0
- s4ever/_backends/tilelang/kernels/chunk.py +830 -0
- s4ever/_backends/tilelang/kernels/discretization.py +93 -0
- s4ever/_backends/tilelang/kernels/dsl.py +134 -0
- s4ever/_backends/tilelang/kernels/fft.py +611 -0
- s4ever/_backends/tilelang/kernels/fftconv.py +1266 -0
- s4ever/_backends/tilelang/kernels/final_state.py +143 -0
- s4ever/_backends/tilelang/kernels/parameter_transforms.py +161 -0
- s4ever/_backends/tilelang/kernels/powers.py +258 -0
- s4ever/_backends/tilelang/kernels/probe.py +44 -0
- s4ever/_backends/tilelang/kernels/recurrent.py +368 -0
- s4ever/_backends/tilelang/kernels/state_inputs.py +104 -0
- s4ever/_backends/tilelang/kernels/transpose.py +39 -0
- s4ever/_backends/tilelang/kernels/vandermonde.py +360 -0
- s4ever/_backends/tilelang/layers.py +256 -0
- s4ever/_backends/tilelang/layout.py +13 -0
- s4ever/_backends/tilelang/recurrent.py +137 -0
- s4ever/_backends/tilelang/runtime.py +374 -0
- s4ever/_backends/tilelang/s4.py +264 -0
- s4ever/_backends/tilelang/s4d.py +385 -0
- s4ever/_backends/tilelang/transpose.py +51 -0
- s4ever/_block.py +122 -0
- s4ever/_initialization.py +113 -0
- s4ever/_layers.py +767 -0
- s4ever/ops/__init__.py +5 -0
- s4ever/ops/_complex.py +75 -0
- s4ever/ops/_fftconv.py +214 -0
- s4ever/ops/_s4.py +703 -0
- s4ever/ops/_s4d.py +500 -0
- s4ever/ops/_shared.py +132 -0
- s4ever/py.typed +0 -0
- s4ever-0.1.0.dist-info/METADATA +218 -0
- s4ever-0.1.0.dist-info/RECORD +61 -0
- s4ever-0.1.0.dist-info/WHEEL +4 -0
- s4ever-0.1.0.dist-info/licenses/LICENSE +202 -0
s4ever/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""S4 and S4D state space sequence layers for PyTorch.
|
|
2
|
+
|
|
3
|
+
The GPU kernels (TileLang on CUDA and ROCm, Metal on MPS) are compiled at first use, so the package
|
|
4
|
+
has no extension to build.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from ._block import S4Block
|
|
8
|
+
from ._layers import S4, S4D, param_groups
|
|
9
|
+
|
|
10
|
+
__all__ = ["S4", "S4D", "S4Block", "param_groups"]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Metal kernels for Apple GPUs.
|
|
2
|
+
|
|
3
|
+
`torch.mps.compile_shader` compiles each library on first use. The kernels compute in float32 and
|
|
4
|
+
read and write sequences in the dtype of the input. The chunk kernels write y and du in float32
|
|
5
|
+
instead when the modes form several groups (`chunk.out_dtype`).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import functools
|
|
9
|
+
import warnings
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
from torch import Tensor
|
|
13
|
+
|
|
14
|
+
from s4ever._backends import shared
|
|
15
|
+
|
|
16
|
+
from . import cauchy, chunk, fft, powers, recurrent, transpose, vandermonde
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def enabled(x: Tensor, fused_kernels: bool | None) -> bool:
|
|
20
|
+
return (
|
|
21
|
+
x.device.type == "mps"
|
|
22
|
+
and x.dtype in shared.INPUT_DTYPES
|
|
23
|
+
and shared.wanted(fused_kernels)
|
|
24
|
+
and not unavailable()
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@shared.dynamo_constant
|
|
29
|
+
def unavailable() -> str:
|
|
30
|
+
"""Return the error message of the probe, or "" if the probe runs.
|
|
31
|
+
|
|
32
|
+
The function is a Dynamo constant because Dynamo cannot trace the probe.
|
|
33
|
+
"""
|
|
34
|
+
return probe()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@functools.cache
|
|
38
|
+
def probe() -> str:
|
|
39
|
+
try:
|
|
40
|
+
transpose.transpose(torch.ones(1, 2, 2, device="mps"))
|
|
41
|
+
except Exception as e: # Metal compilers and drivers fail in version-specific ways
|
|
42
|
+
warnings.warn(
|
|
43
|
+
"s4ever: the ops use PyTorch unless fused_kernels=True, because the Metal kernels "
|
|
44
|
+
f"cannot be compiled or run: {e}",
|
|
45
|
+
# The warning names the line in s4ever that first asked whether the kernels are
|
|
46
|
+
# available.
|
|
47
|
+
stacklevel=3,
|
|
48
|
+
)
|
|
49
|
+
# The repr is not empty when the exception has no message.
|
|
50
|
+
return str(e) or repr(e)
|
|
51
|
+
return ""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
__all__ = [
|
|
55
|
+
"cauchy",
|
|
56
|
+
"chunk",
|
|
57
|
+
"enabled",
|
|
58
|
+
"fft",
|
|
59
|
+
"powers",
|
|
60
|
+
"recurrent",
|
|
61
|
+
"transpose",
|
|
62
|
+
"unavailable",
|
|
63
|
+
"vandermonde",
|
|
64
|
+
]
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""S4's convolution kernel from its parameters (`s4_conv_kernel`) on Metal.
|
|
2
|
+
|
|
3
|
+
The kernels use the method that `_backends.tilelang.kernels.cauchy` describes.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import functools
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
|
|
11
|
+
from s4ever._backends import shared
|
|
12
|
+
from s4ever._backends.shared import ceil_div
|
|
13
|
+
|
|
14
|
+
from . import fft, msl
|
|
15
|
+
|
|
16
|
+
BK = 128 # frequencies per threadgroup of `spectrum`, one per thread
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@functools.cache
|
|
20
|
+
def spectrum_library(N: int, C: int):
|
|
21
|
+
prelude = f"constant constexpr uint THREADS = {BK}, N = {N}, C = {C};\n"
|
|
22
|
+
names = ("spectrum", "spectrum_grads", "spectrum_finish")
|
|
23
|
+
return msl.library(msl.source(prelude, "cauchy_spectrum.metal"), dict.fromkeys(names, BK))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def fits(N: int) -> bool:
|
|
27
|
+
return N <= shared.MAX_CONV_KERNEL_MODES and builds()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@msl.constant
|
|
31
|
+
def builds() -> bool:
|
|
32
|
+
"""Return whether the kernels build for the most modes and channels, and warn if they do not.
|
|
33
|
+
|
|
34
|
+
The kernels use the most registers and threadgroup memory at these sizes.
|
|
35
|
+
"""
|
|
36
|
+
args = (shared.MAX_CONV_KERNEL_MODES, shared.MAX_CHANNELS)
|
|
37
|
+
return msl.builds("S4's convolution kernels", spectrum_library, *args)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@shared.register_op(
|
|
41
|
+
"metal_s4_conv_kernel", lambda dt, A, P, B, C, L: C.new_empty(*C.shape[:2], L), "MPS"
|
|
42
|
+
)
|
|
43
|
+
def s4_conv_kernel_op(dt: Tensor, A: Tensor, P: Tensor, B: Tensor, C: Tensor, L: int) -> Tensor:
|
|
44
|
+
Cc, H, N, _ = C.shape
|
|
45
|
+
F = L // 2 + 1
|
|
46
|
+
k_f = C.new_empty(Cc, H, F, 2)
|
|
47
|
+
threads, group = [H * ceil_div(F, BK) * BK, 1, 1], [BK, 1, 1]
|
|
48
|
+
# The kernels run for at most `shared.MAX_CHANNELS` channels at a time, but
|
|
49
|
+
# `ops._s4.state_response` passes each batch row of x0 as a channel, and the batch can have
|
|
50
|
+
# more rows.
|
|
51
|
+
for c in range(0, Cc, shared.MAX_CHANNELS):
|
|
52
|
+
Cs = C[c : c + shared.MAX_CHANNELS]
|
|
53
|
+
lib = spectrum_library(N, len(Cs))
|
|
54
|
+
lib.spectrum(k_f[c : c + len(Cs)], dt, A, P, B, Cs, H, L, threads=threads, group_size=group)
|
|
55
|
+
if fft.real_fits(L):
|
|
56
|
+
return fft.real_inverse(k_f, L)
|
|
57
|
+
return torch.fft.irfft(torch.view_as_complex(k_f), n=L, norm="forward")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def grads_outputs(
|
|
61
|
+
dt: Tensor, A: Tensor, P: Tensor, B: Tensor, C: Tensor, g: Tensor
|
|
62
|
+
) -> list[Tensor]:
|
|
63
|
+
return [torch.empty_like(x) for x in (dt, A, P, B, C)]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@shared.register_op("metal_s4_conv_kernel_grads", grads_outputs, "MPS")
|
|
67
|
+
def s4_conv_kernel_grads_op(
|
|
68
|
+
dt: Tensor, A: Tensor, P: Tensor, B: Tensor, C: Tensor, g: Tensor
|
|
69
|
+
) -> list[Tensor]:
|
|
70
|
+
(Cc, H, N, _), L = C.shape, g.shape[-1]
|
|
71
|
+
gf = torch.view_as_real(torch.fft.rfft(g, n=L))
|
|
72
|
+
n_freq = (L + 1) // 2 - 1 # the frequencies 0 < k < L / 2
|
|
73
|
+
# Each threadgroup sums `split_tiles` tiles of BK frequencies, so that the dispatch has about
|
|
74
|
+
# msl.GROUPS threadgroups.
|
|
75
|
+
tiles = ceil_div(n_freq, BK)
|
|
76
|
+
group = [BK, 1, 1]
|
|
77
|
+
split_tiles = ceil_div(tiles, min(tiles, ceil_div(msl.GROUPS, H))) if tiles else 0
|
|
78
|
+
splits = ceil_div(tiles, split_tiles) if tiles else 1
|
|
79
|
+
# Each kernel runs once for all channels. The backward has at most `shared.MAX_CHANNELS`
|
|
80
|
+
# channels, because S4 runs `ops._s4.state_response` only without gradients.
|
|
81
|
+
grads = grads_outputs(dt, A, P, B, C, g)
|
|
82
|
+
lib = spectrum_library(N, Cc)
|
|
83
|
+
rows = 4 * (Cc + 1) + 2 # COEFS + 2 in cauchy_spectrum.metal
|
|
84
|
+
part = C.new_empty(H, splits, rows, N) if tiles else C.new_zeros(H, 1, rows, N)
|
|
85
|
+
args = (dt, A, P, B, C, H, L, splits)
|
|
86
|
+
if tiles:
|
|
87
|
+
lib.spectrum_grads(
|
|
88
|
+
part, gf, *args, split_tiles, threads=[H * splits * BK, 1, 1], group_size=group
|
|
89
|
+
)
|
|
90
|
+
lib.spectrum_finish(*grads, part, gf, *args, threads=[H * BK, 1, 1], group_size=group)
|
|
91
|
+
return grads
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class S4ConvKernel(torch.autograd.Function):
|
|
95
|
+
@staticmethod
|
|
96
|
+
def forward(ctx, dt, A, P, B, C, L):
|
|
97
|
+
ctx.save_for_backward(dt, A, P, B, C)
|
|
98
|
+
return s4_conv_kernel_op(dt, A, P, B, C, L)
|
|
99
|
+
|
|
100
|
+
@staticmethod
|
|
101
|
+
def backward(ctx, *grads):
|
|
102
|
+
shared.refuse_double_backward()
|
|
103
|
+
return *s4_conv_kernel_grads_op(*ctx.saved_tensors, grads[0].contiguous()), None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def s4_conv_kernel(dt: Tensor, A: Tensor, P: Tensor, B: Tensor, C: Tensor, L: int) -> Tensor:
|
|
107
|
+
"""Return `s4ever.ops.s4_conv_kernel` computed by the Metal kernels.
|
|
108
|
+
|
|
109
|
+
Callers check `fits` first.
|
|
110
|
+
"""
|
|
111
|
+
return S4ConvKernel.apply(*shared.contiguous_f32(dt, A, P, B, C), L)
|
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
// The kernels of cauchy.py, which use the method that `tilelang.kernels.cauchy` describes. They are
|
|
2
|
+
// compiled after the constants THREADS, N and C of `spectrum_library`. In `spectrum` and
|
|
3
|
+
// `spectrum_grads`, each thread handles one frequency of a tile of THREADS frequencies.
|
|
4
|
+
|
|
5
|
+
// The resolvent r_XY of the pair (X_x, Y_y) has index x (C + 1) + y, for X = (B, P),
|
|
6
|
+
// Y = (C..., Q) and Q = conj(P), so r[BQ] is r_BQ and r[PQ] is r_PQ. Each pair has the two Cauchy
|
|
7
|
+
// coefficients coef_0 and coef_1.
|
|
8
|
+
constant constexpr uint PAIRS = 2 * (C + 1), COEFS = 2 * PAIRS, BQ = C, PQ = 2 * C + 1;
|
|
9
|
+
constant constexpr uint SIMDS = THREADS / 32; // SIMD groups per threadgroup
|
|
10
|
+
// The tables of the Cauchy sums hold one chunk of the modes at a time. The modes form CHUNKS chunks
|
|
11
|
+
// of CHUNK_MODES <= THREADS modes, so that each thread loads at most one mode of a chunk.
|
|
12
|
+
constant constexpr uint CHUNKS = (N + THREADS - 1) / THREADS;
|
|
13
|
+
constant constexpr uint CHUNK_MODES = (N + CHUNKS - 1) / CHUNKS;
|
|
14
|
+
// `others` adds the terms of r_XY(0) in blocks of BLOCK_MODES modes, so that it takes about
|
|
15
|
+
// N / BLOCK_MODES + BLOCK_MODES steps per mode instead of N.
|
|
16
|
+
constant constexpr uint BLOCK_MODES = 16, BLOCKS = (N + BLOCK_MODES - 1) / BLOCK_MODES;
|
|
17
|
+
|
|
18
|
+
// Return nu = 2 tan(pi k / L) for 0 <= k <= L / 2. For k > L / 4, the function evaluates
|
|
19
|
+
// 2 cot(pi (L - 2 k) / (2 L)) instead, so that the angle stays at most pi / 4 and each argument of
|
|
20
|
+
// sinpi and cospi is computed with one rounding.
|
|
21
|
+
inline float frequency(long k, long L) {
|
|
22
|
+
float x = float(k) / float(L), z = float(L - 2 * k) / float(2 * L);
|
|
23
|
+
return 4 * k <= L ? 2 * precise::sinpi(x) / precise::cospi(x)
|
|
24
|
+
: 2 * precise::cospi(z) / precise::sinpi(z);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// The parameters of one mode, with u[x (C + 1) + y] = X_x Y_y.
|
|
28
|
+
struct Mode {
|
|
29
|
+
float2 lam, X[2], Y[C + 1], u[PAIRS];
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
inline Mode load_mode(device const float2* Lam, device const float2* P, device const float2* B,
|
|
33
|
+
device const float2* Cm, long H, long h, uint n) {
|
|
34
|
+
Mode mode;
|
|
35
|
+
long i = h * N + n;
|
|
36
|
+
mode.lam = Lam[i];
|
|
37
|
+
mode.X[0] = B[i];
|
|
38
|
+
mode.X[1] = P[i];
|
|
39
|
+
for (uint c = 0; c < C; c++) mode.Y[c] = Cm[c * H * N + i];
|
|
40
|
+
mode.Y[C] = conjugate(P[i]);
|
|
41
|
+
for (uint x = 0; x < 2; x++)
|
|
42
|
+
for (uint y = 0; y <= C; y++) mode.u[x * (C + 1) + y] = cmul(mode.X[x], mode.Y[y]);
|
|
43
|
+
return mode;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Load the tables of the chunk that starts at mode m0, whose entry n holds mode i = m0 + n. aw[n]
|
|
47
|
+
// holds a = Re(w) and |w|^2 for the pole w, and cs[n] holds the coefficients.
|
|
48
|
+
// w = dt Lambda
|
|
49
|
+
// v = dt u
|
|
50
|
+
// coef_0 = -2 Re(v conj(w))
|
|
51
|
+
// coef_1 = 2 Re(v)
|
|
52
|
+
// cs[n][p PAIRS + j] = coef_p[j, i]
|
|
53
|
+
// The entries past the last mode have zero coefficients, so they add nothing to the sums.
|
|
54
|
+
inline void tables(threadgroup float (&cs)[CHUNK_MODES][COEFS],
|
|
55
|
+
threadgroup float2 (&aw)[CHUNK_MODES], device const float* dt,
|
|
56
|
+
device const float2* Lam, device const float2* P, device const float2* B,
|
|
57
|
+
device const float2* Cm, long H, long h, uint t, uint m0) {
|
|
58
|
+
for (uint n = t; n < CHUNK_MODES; n += THREADS) {
|
|
59
|
+
uint i = m0 + n;
|
|
60
|
+
Mode mode = load_mode(Lam, P, B, Cm, H, h, min(i, N - 1));
|
|
61
|
+
float dt_h = dt[h];
|
|
62
|
+
float2 w = dt_h * mode.lam;
|
|
63
|
+
aw[n] = float2(w.x, dot(w, w));
|
|
64
|
+
for (uint j = 0; j < PAIRS; j++) {
|
|
65
|
+
float2 v = dt_h * mode.u[j];
|
|
66
|
+
cs[n][j] = i < N ? -2 * dot(v, w) : 0;
|
|
67
|
+
cs[n][PAIRS + j] = i < N ? 2 * v.x : 0;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Return m = 1 / q_n at z = i nu for the table entry aw = (a_n, |w_n|^2).
|
|
74
|
+
// q_n = (|w_n|^2 - nu^2) - 2i nu a_n
|
|
75
|
+
// The forward and the backward both call this function, so that they round |w_n|^2 - nu^2 the same
|
|
76
|
+
// way. Near resonance, |w_n|^2 - nu^2 is small, and a rounding difference between the passes would
|
|
77
|
+
// make the gradients of P, B and C inaccurate.
|
|
78
|
+
inline float2 inverse_q(float nu, float2 aw) {
|
|
79
|
+
float2 q_conj = float2(fma(-nu, nu, aw.y), 2 * nu * aw.x);
|
|
80
|
+
return q_conj * (1 / dot(q_conj, q_conj));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Add sum_n cs[n][j] / q_n over the modes of the tables to S[j] at z = i nu.
|
|
84
|
+
inline void sums(threadgroup const float (&cs)[CHUNK_MODES][COEFS],
|
|
85
|
+
threadgroup const float2 (&aw)[CHUNK_MODES], float nu,
|
|
86
|
+
thread float2 (&S)[COEFS]) {
|
|
87
|
+
for (uint n = 0; n < CHUNK_MODES; n++) {
|
|
88
|
+
float2 m = inverse_q(nu, aw[n]);
|
|
89
|
+
for (uint j = 0; j < COEFS; j++) S[j] += cs[n][j] * m;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Set the resolvents r = S0 + i nu S1 at z = i nu, s = 1 / (1 + r_PQ) and b = r_BQ s.
|
|
94
|
+
inline void resolvents(thread const float2 (&S)[COEFS], float nu, thread float2 (&r)[PAIRS],
|
|
95
|
+
thread float2& s, thread float2& b) {
|
|
96
|
+
for (uint j = 0; j < PAIRS; j++) r[j] = S[j] + nu * float2(-S[PAIRS + j].y, S[PAIRS + j].x);
|
|
97
|
+
float2 den = float2(1 + r[PQ].x, r[PQ].y);
|
|
98
|
+
s = conjugate(den) / dot(den, den);
|
|
99
|
+
b = cmul(r[BQ], s);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Return the term -2 Re(u_j / Lambda) that the mode adds to r_XY(0) for pair j.
|
|
103
|
+
inline float term(Mode mode, uint j) {
|
|
104
|
+
return -2 * dot(mode.u[j], mode.lam) / dot(mode.lam, mode.lam);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Set block_terms[block][j] to the sum of terms[i][j] over the modes i of the block. Thread t sums
|
|
108
|
+
// block t.
|
|
109
|
+
template <uint J>
|
|
110
|
+
inline void block_sums(threadgroup const float (&terms)[N][J],
|
|
111
|
+
threadgroup float (&block_terms)[BLOCKS][J], uint t) {
|
|
112
|
+
if (t < BLOCKS)
|
|
113
|
+
for (uint j = 0; j < J; j++) {
|
|
114
|
+
float sum = 0;
|
|
115
|
+
for (uint i = t * BLOCK_MODES; i < min(N, t * BLOCK_MODES + BLOCK_MODES); i++)
|
|
116
|
+
sum += terms[i][j];
|
|
117
|
+
block_terms[t][j] = sum;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Set o[j] to the sum of terms[i][j] over the modes i other than n, and r0[j] to the sum over all
|
|
122
|
+
// modes. For N > 2 BLOCK_MODES, the sums add the block sums and then the terms of n's block. The
|
|
123
|
+
// sum o never adds the term of mode n, so a large term of mode n does not leave a rounding error
|
|
124
|
+
// in o.
|
|
125
|
+
template <uint J>
|
|
126
|
+
inline void others(threadgroup const float (&terms)[N][J],
|
|
127
|
+
threadgroup const float (&block_terms)[BLOCKS][J], uint n, thread float (&o)[J],
|
|
128
|
+
thread float (&r0)[J]) {
|
|
129
|
+
for (uint j = 0; j < J; j++) o[j] = r0[j] = 0;
|
|
130
|
+
if (N <= 2 * BLOCK_MODES) {
|
|
131
|
+
for (uint i = 0; i < N; i++)
|
|
132
|
+
for (uint j = 0; j < J; j++) {
|
|
133
|
+
o[j] += i == n ? 0 : terms[i][j];
|
|
134
|
+
r0[j] += terms[i][j];
|
|
135
|
+
}
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
for (uint block = 0; block < BLOCKS; block++)
|
|
139
|
+
for (uint j = 0; j < J; j++) {
|
|
140
|
+
o[j] += block == n / BLOCK_MODES ? 0 : block_terms[block][j];
|
|
141
|
+
r0[j] += block_terms[block][j];
|
|
142
|
+
}
|
|
143
|
+
uint i0 = n / BLOCK_MODES * BLOCK_MODES;
|
|
144
|
+
for (uint i = i0; i < min(N, i0 + BLOCK_MODES); i++)
|
|
145
|
+
for (uint j = 0; j < J; j++) o[j] += i == n ? 0 : terms[i][j];
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Return entry n of the solution x of (Lambda - W W^*) x = V at z = 0, for (V, W) = (B, P) or
|
|
149
|
+
// (C, Q). The argument o_pq sums the terms of r_PQ(0) over the modes other than n, and o_v sums
|
|
150
|
+
// those of r_BQ(0) for V = B and of r_PC(0) for V = C. The argument den is 1 + r_PQ(0). The
|
|
151
|
+
// function adds the terms of mode n in closed form.
|
|
152
|
+
// x_n = (V_n (1 + o_pq) - W_n o_v - 2i W_n Im(V_n conj(W_n)) / conj(Lambda_n)) / (Lambda_n den)
|
|
153
|
+
inline float2 solve(float2 V, float2 W, float2 lam, float o_pq, float o_v, float den) {
|
|
154
|
+
float scale = 1 / (dot(lam, lam) * den), im = V.y * W.x - V.x * W.y; // Im(V conj(W))
|
|
155
|
+
float2 num = V + (V * o_pq - W * o_v);
|
|
156
|
+
return float2(num.x * lam.x + num.y * lam.y + 2 * im * W.y,
|
|
157
|
+
num.y * lam.x - num.x * lam.y - 2 * im * W.x) * scale;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Write k_f / L (C, H, L / 2 + 1, 2) from dt (H), Lambda, P and B (H, N, 2), and Cm (C, H, N, 2).
|
|
161
|
+
kernel void spectrum(device float2* out, device const float* dt, device const float2* Lam,
|
|
162
|
+
device const float2* P, device const float2* B, device const float2* Cm,
|
|
163
|
+
constant long& H, constant long& L, IDS) {
|
|
164
|
+
threadgroup float cs[CHUNK_MODES][COEFS];
|
|
165
|
+
threadgroup float terms[N][2], block_terms[BLOCKS][2];
|
|
166
|
+
threadgroup float zero_parts[C][CHUNK_MODES], nyquist_parts[C][CHUNK_MODES];
|
|
167
|
+
threadgroup float2 aw[CHUNK_MODES];
|
|
168
|
+
long F = L / 2 + 1;
|
|
169
|
+
uint tiles = uint(F + THREADS - 1) / THREADS, h = g / tiles, tile = g % tiles;
|
|
170
|
+
long k = tile * THREADS + t, K = (L + 1) / 2 - 1; // K is the last bin below L / 2.
|
|
171
|
+
float scale = 1 / float(L);
|
|
172
|
+
tables(cs, aw, dt, Lam, P, B, Cm, H, h, t, 0);
|
|
173
|
+
if (tile == 0) {
|
|
174
|
+
// At k = 0, the terms of r_XY(0) are large for nearly undamped modes, and their rounding
|
|
175
|
+
// errors would dominate r_BC - b r_PC. The kernel computes bin 0 as -2 Re sum_n C_n x_n
|
|
176
|
+
// instead, where x solves (Lambda - P P^*) x = B. Thread t < CHUNK_MODES handles the modes
|
|
177
|
+
// t, t + CHUNK_MODES, ...
|
|
178
|
+
uint first = t < CHUNK_MODES ? t : N;
|
|
179
|
+
for (uint n = first; n < N; n += CHUNK_MODES) {
|
|
180
|
+
Mode mode = load_mode(Lam, P, B, Cm, H, h, n);
|
|
181
|
+
terms[n][0] = term(mode, BQ);
|
|
182
|
+
terms[n][1] = term(mode, PQ);
|
|
183
|
+
}
|
|
184
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
185
|
+
if (N > 2 * BLOCK_MODES) {
|
|
186
|
+
block_sums(terms, block_terms, t);
|
|
187
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
188
|
+
}
|
|
189
|
+
float zero_sum[C] = {}, nyquist_sum[C] = {};
|
|
190
|
+
for (uint n = first; n < N; n += CHUNK_MODES) {
|
|
191
|
+
Mode mode = load_mode(Lam, P, B, Cm, H, h, n);
|
|
192
|
+
float o[2], r0[2];
|
|
193
|
+
others(terms, block_terms, n, o, r0);
|
|
194
|
+
float2 x = solve(mode.X[0], mode.X[1], mode.lam, o[1], o[0], 1 + r0[1]);
|
|
195
|
+
for (uint c = 0; c < C; c++) {
|
|
196
|
+
zero_sum[c] += mode.Y[c].x * x.x - mode.Y[c].y * x.y;
|
|
197
|
+
nyquist_sum[c] += dt[h] * mode.u[c].x;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
if (t < CHUNK_MODES)
|
|
201
|
+
for (uint c = 0; c < C; c++) {
|
|
202
|
+
zero_parts[c][t] = zero_sum[c];
|
|
203
|
+
nyquist_parts[c][t] = nyquist_sum[c];
|
|
204
|
+
}
|
|
205
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
206
|
+
if (t < C) {
|
|
207
|
+
// For even L, the Nyquist bin is the limit sum_n Re v_BC.
|
|
208
|
+
float zero = 0, nyquist = 0;
|
|
209
|
+
for (uint n = 0; n < CHUNK_MODES; n++) {
|
|
210
|
+
zero += zero_parts[t][n];
|
|
211
|
+
nyquist += nyquist_parts[t][n];
|
|
212
|
+
}
|
|
213
|
+
out[(t * H + h) * F] = float2(-2 * zero * scale, 0);
|
|
214
|
+
if (L % 2 == 0) out[(t * H + h) * F + L / 2] = float2(nyquist * scale, 0);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
// With more than one chunk, the threads with k outside 0 < k <= K stay to load the tables of
|
|
218
|
+
// the later chunks.
|
|
219
|
+
bool active = k >= 1 && k <= K;
|
|
220
|
+
if (CHUNKS == 1 && !active) return;
|
|
221
|
+
float nu = frequency(k, L);
|
|
222
|
+
float2 S[COEFS] = {}, r[PAIRS], s, b;
|
|
223
|
+
for (uint m0 = 0; m0 < N; m0 += CHUNK_MODES) {
|
|
224
|
+
if (m0 > 0) {
|
|
225
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
226
|
+
tables(cs, aw, dt, Lam, P, B, Cm, H, h, t, m0);
|
|
227
|
+
}
|
|
228
|
+
if (active) sums(cs, aw, nu, S);
|
|
229
|
+
}
|
|
230
|
+
if (!active) return;
|
|
231
|
+
resolvents(S, nu, r, s, b);
|
|
232
|
+
// k_f / L = (1 + z / 2) (r_BC - b r_PC) / L with z = i nu
|
|
233
|
+
for (uint c = 0; c < C; c++) {
|
|
234
|
+
float2 v = r[c] - cmul(b, r[C + 1 + c]);
|
|
235
|
+
out[(c * H + h) * F + k] = (v + nu / 2 * float2(-v.y, v.x)) * scale;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// Write the partial sums of the gradients to part (H, splits, COEFS + 2, N). Rows j < COEFS hold
|
|
240
|
+
// the gradients of the Cauchy coefficients, and rows COEFS and COEFS + 1 hold the gradients of a
|
|
241
|
+
// and |w|^2. The input gf (C, H, L / 2 + 1, 2) is the rfft of the kernel's gradient. Threadgroup
|
|
242
|
+
// (h, split) sums over the frequencies 0 < k < L / 2 of its split_tiles tiles. In each tile, the
|
|
243
|
+
// threads first store nu and the gradients GS of the sums S at their frequencies. Then thread
|
|
244
|
+
// t = k0 CHUNK_MODES + n sums the terms of table entry n over the frequencies k0,
|
|
245
|
+
// k0 + ENTRY_THREADS, ... of the tile.
|
|
246
|
+
kernel void spectrum_grads(device float* part, device const float2* gf, device const float* dt,
|
|
247
|
+
device const float2* Lam, device const float2* P,
|
|
248
|
+
device const float2* B, device const float2* Cm, constant long& H,
|
|
249
|
+
constant long& L, constant long& splits, constant long& split_tiles,
|
|
250
|
+
IDS) {
|
|
251
|
+
// ENTRY_THREADS threads share each table entry. With more than one chunk,
|
|
252
|
+
// CHUNK_MODES > THREADS / 2, so ENTRY_THREADS is 1, and the threads with k0 = 0 hold all the
|
|
253
|
+
// sums.
|
|
254
|
+
constexpr uint ENTRY_THREADS = THREADS / CHUNK_MODES;
|
|
255
|
+
threadgroup float cs[CHUNK_MODES][COEFS], nus[THREADS];
|
|
256
|
+
threadgroup float2 aw[CHUNK_MODES], GS[THREADS][COEFS];
|
|
257
|
+
long F = L / 2 + 1, K = (L + 1) / 2 - 1;
|
|
258
|
+
uint h = g / uint(splits), split = g % uint(splits);
|
|
259
|
+
tables(cs, aw, dt, Lam, P, B, Cm, H, h, t, 0);
|
|
260
|
+
uint n = t % CHUNK_MODES, k0 = t / CHUNK_MODES;
|
|
261
|
+
float cn[COEFS], acc[COEFS + 2] = {};
|
|
262
|
+
float2 wn;
|
|
263
|
+
auto load_entry = [&]() {
|
|
264
|
+
for (uint j = 0; j < COEFS; j++) cn[j] = cs[n][j];
|
|
265
|
+
wn = aw[n];
|
|
266
|
+
};
|
|
267
|
+
if (CHUNKS == 1) load_entry();
|
|
268
|
+
for (long i = 0; i < split_tiles; i++) {
|
|
269
|
+
long k = (split * split_tiles + i) * THREADS + t + 1, kc = min(k, K);
|
|
270
|
+
float nu = frequency(kc, L), weight = k <= K ? 2 / float(L) : 0;
|
|
271
|
+
float2 S[COEFS] = {}, r[PAIRS], s, b, gr[PAIRS], gb = 0;
|
|
272
|
+
sums(cs, aw, nu, S);
|
|
273
|
+
for (uint m0 = CHUNK_MODES; m0 < N; m0 += CHUNK_MODES) {
|
|
274
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
275
|
+
tables(cs, aw, dt, Lam, P, B, Cm, H, h, t, m0);
|
|
276
|
+
sums(cs, aw, nu, S);
|
|
277
|
+
}
|
|
278
|
+
resolvents(S, nu, r, s, b);
|
|
279
|
+
for (uint c = 0; c < C; c++) {
|
|
280
|
+
float2 gk = gf[(c * H + h) * F + kc] * weight;
|
|
281
|
+
float2 p = gk + nu / 2 * float2(gk.y, -gk.x); // p = conj(1 + i nu / 2) gk
|
|
282
|
+
// k_f = (1 + z/2) (r_BC - b r_PC)
|
|
283
|
+
gr[c] = p;
|
|
284
|
+
gr[C + 1 + c] = -cmulc(p, b);
|
|
285
|
+
gb -= cmulc(p, r[C + 1 + c]);
|
|
286
|
+
}
|
|
287
|
+
// G_BQ = conj(s) G_b and G_PQ = -conj(s^2 r_BQ) G_b.
|
|
288
|
+
gr[BQ] = cmulc(gb, s);
|
|
289
|
+
gr[PQ] = -cmulc(gb, cmul(b, s));
|
|
290
|
+
// r = S0 + i nu S1 gives G_S0 = G_r and G_S1 = -i nu G_r.
|
|
291
|
+
for (uint j = 0; j < PAIRS; j++) {
|
|
292
|
+
GS[t][j] = gr[j];
|
|
293
|
+
GS[t][PAIRS + j] = nu * float2(gr[j].y, -gr[j].x);
|
|
294
|
+
}
|
|
295
|
+
nus[t] = nu;
|
|
296
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
297
|
+
// The chunks run in reverse order, because the tables already hold the last chunk.
|
|
298
|
+
for (uint ci = CHUNKS; ci-- > 0;) {
|
|
299
|
+
uint m0 = ci * CHUNK_MODES;
|
|
300
|
+
if (CHUNKS > 1) {
|
|
301
|
+
if (ci < CHUNKS - 1) {
|
|
302
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
303
|
+
tables(cs, aw, dt, Lam, P, B, Cm, H, h, t, m0);
|
|
304
|
+
}
|
|
305
|
+
load_entry();
|
|
306
|
+
for (uint j = 0; j < COEFS + 2; j++) acc[j] = 0;
|
|
307
|
+
}
|
|
308
|
+
if (k0 < ENTRY_THREADS)
|
|
309
|
+
for (uint freq = k0; freq < THREADS; freq += ENTRY_THREADS) {
|
|
310
|
+
float nuk = nus[freq];
|
|
311
|
+
float2 m = inverse_q(nuk, wn), gm = 0;
|
|
312
|
+
for (uint j = 0; j < COEFS; j++) {
|
|
313
|
+
gm += cn[j] * GS[freq][j];
|
|
314
|
+
acc[j] += dot(m, GS[freq][j]);
|
|
315
|
+
}
|
|
316
|
+
// G_q = -conj(m^2) G_m, and q_n = (|w|^2 - nu^2) - 2i nu a gives the gradients
|
|
317
|
+
// of a and |w|^2.
|
|
318
|
+
float c2 = m.x * m.x - m.y * m.y, s2 = 2 * m.x * m.y;
|
|
319
|
+
acc[COEFS] += 2 * nuk * (c2 * gm.y - s2 * gm.x);
|
|
320
|
+
acc[COEFS + 1] -= c2 * gm.x + s2 * gm.y;
|
|
321
|
+
}
|
|
322
|
+
if (CHUNKS > 1 && k0 == 0 && m0 + n < N)
|
|
323
|
+
for (uint j = 0; j < COEFS + 2; j++) {
|
|
324
|
+
device float* partial =
|
|
325
|
+
part + ((h * splits + split) * (COEFS + 2) + j) * N + m0 + n;
|
|
326
|
+
*partial = i == 0 ? acc[j] : *partial + acc[j];
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
330
|
+
}
|
|
331
|
+
if (CHUNKS > 1) return;
|
|
332
|
+
threadgroup float* red = (threadgroup float*)GS; // (ENTRY_THREADS, N, COEFS + 2)
|
|
333
|
+
if (k0 < ENTRY_THREADS)
|
|
334
|
+
for (uint j = 0; j < COEFS + 2; j++) red[(k0 * N + n) * (COEFS + 2) + j] = acc[j];
|
|
335
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
336
|
+
if (t < N)
|
|
337
|
+
for (uint j = 0; j < COEFS + 2; j++) {
|
|
338
|
+
float tot = 0;
|
|
339
|
+
for (uint e = 0; e < ENTRY_THREADS; e++) tot += red[(e * N + t) * (COEFS + 2) + j];
|
|
340
|
+
part[((h * splits + split) * (COEFS + 2) + j) * N + t] = tot;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// Write the gradients of dt (H), Lambda, P and B (H, N, 2), and Cm (C, H, N, 2) from the partial
|
|
345
|
+
// sums in `part` and the bins 0 and L / 2 of gf.
|
|
346
|
+
kernel void spectrum_finish(device float* gdt, device float2* gLam, device float2* gP,
|
|
347
|
+
device float2* gB, device float2* gC, device const float* part,
|
|
348
|
+
device const float2* gf, device const float* dt,
|
|
349
|
+
device const float2* Lam, device const float2* P,
|
|
350
|
+
device const float2* B, device const float2* Cm, constant long& H,
|
|
351
|
+
constant long& L, constant long& splits, IDS) {
|
|
352
|
+
// The columns of `terms` are the terms of r_BQ(0), r_PC(0)... and r_PQ(0).
|
|
353
|
+
threadgroup float terms[N][C + 2], block_terms[BLOCKS][C + 2], red[SIMDS];
|
|
354
|
+
long h = g, F = L / 2 + 1;
|
|
355
|
+
for (uint n = t; n < N; n += THREADS) {
|
|
356
|
+
Mode mode = load_mode(Lam, P, B, Cm, H, h, n);
|
|
357
|
+
for (uint j = 0; j < C + 2; j++) terms[n][j] = term(mode, BQ + j);
|
|
358
|
+
}
|
|
359
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
360
|
+
if (N > 2 * BLOCK_MODES) {
|
|
361
|
+
block_sums(terms, block_terms, t);
|
|
362
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
363
|
+
}
|
|
364
|
+
float gdt_thread = 0;
|
|
365
|
+
for (uint m0 = 0; m0 < N; m0 += THREADS) {
|
|
366
|
+
uint n = min(m0 + t, N - 1);
|
|
367
|
+
Mode mode = load_mode(Lam, P, B, Cm, H, h, n);
|
|
368
|
+
float o[C + 2], r0[C + 2];
|
|
369
|
+
others(terms, block_terms, n, o, r0);
|
|
370
|
+
// M = Lambda - P P^* is a matrix over the modes and their conjugates. The gradient of a
|
|
371
|
+
// parameter is 2 conj(G), where G is the derivative of k_f(0) / L with respect to the
|
|
372
|
+
// parameter.
|
|
373
|
+
// k_f(0) / L = -C^T x / L
|
|
374
|
+
// x = M^-1 B
|
|
375
|
+
// mu = M^-T C
|
|
376
|
+
float den = 1 + r0[C + 1], gs = 0; // gs = sum_c g_c P^T mu_c, P^T mu_c = -r_PC / den
|
|
377
|
+
float2 x = solve(mode.X[0], mode.X[1], mode.lam, o[C + 1], o[0], den), gmu = 0, gc[C];
|
|
378
|
+
for (uint c = 0; c < C; c++) {
|
|
379
|
+
float gk = gf[(c * H + h) * F].x / float(L);
|
|
380
|
+
gmu += gk * solve(mode.Y[c], mode.Y[C], mode.lam, o[C + 1], o[1 + c], den);
|
|
381
|
+
gs -= gk * r0[1 + c] / den;
|
|
382
|
+
gc[c] = -2 * gk * conjugate(x);
|
|
383
|
+
}
|
|
384
|
+
float qx = -r0[0] / den; // Q^T x
|
|
385
|
+
// Bin 0 adds these gradients of Lambda and P.
|
|
386
|
+
float2 zero_gLam = 2 * conjugate(cmul(gmu, x));
|
|
387
|
+
float2 zero_gP = -2 * (qx * conjugate(gmu) + gs * x);
|
|
388
|
+
// Sum the rows of `part` for mode n over the splits.
|
|
389
|
+
float gcoef[COEFS + 2] = {};
|
|
390
|
+
for (long split = 0; split < splits; split++)
|
|
391
|
+
for (uint j = 0; j < COEFS + 2; j++)
|
|
392
|
+
gcoef[j] += part[((h * splits + split) * (COEFS + 2) + j) * N + n];
|
|
393
|
+
float dt_h = dt[h], nyquist = L % 2 == 0 ? 1 / float(L) : 0;
|
|
394
|
+
float2 w = dt_h * mode.lam;
|
|
395
|
+
float2 gw = float2(gcoef[COEFS] + 2 * w.x * gcoef[COEFS + 1], 2 * w.y * gcoef[COEFS + 1]);
|
|
396
|
+
float2 gv[PAIRS];
|
|
397
|
+
for (uint j = 0; j < PAIRS; j++) {
|
|
398
|
+
float g0 = gcoef[j], g1 = gcoef[PAIRS + j];
|
|
399
|
+
gv[j] = float2(-2 * w.x * g0 + 2 * g1, -2 * w.y * g0);
|
|
400
|
+
gw -= 2 * dt_h * mode.u[j] * g0;
|
|
401
|
+
if (j < C) gv[j].x += nyquist * gf[(j * H + h) * F + L / 2].x;
|
|
402
|
+
}
|
|
403
|
+
float gdt_mode = dot(mode.lam, gw);
|
|
404
|
+
// gX and gY start from the gradients that bin 0 gives.
|
|
405
|
+
float2 gX[2] = {-2 * conjugate(gmu), zero_gP}, gY[C + 1];
|
|
406
|
+
for (uint c = 0; c < C; c++) gY[c] = gc[c];
|
|
407
|
+
gY[C] = 0;
|
|
408
|
+
for (uint i = 0; i < 2; i++)
|
|
409
|
+
for (uint c = 0; c <= C; c++) {
|
|
410
|
+
uint j = i * (C + 1) + c;
|
|
411
|
+
gdt_mode += dot(gv[j], mode.u[j]);
|
|
412
|
+
gX[i] += cmulc(dt_h * gv[j], mode.Y[c]);
|
|
413
|
+
gY[c] += cmulc(dt_h * gv[j], mode.X[i]);
|
|
414
|
+
}
|
|
415
|
+
if (m0 + t < N) {
|
|
416
|
+
long i = h * N + n;
|
|
417
|
+
gB[i] = gX[0];
|
|
418
|
+
gP[i] = gX[1] + conjugate(gY[C]);
|
|
419
|
+
gLam[i] = dt_h * gw + zero_gLam;
|
|
420
|
+
for (uint c = 0; c < C; c++) gC[c * H * N + i] = gY[c];
|
|
421
|
+
gdt_thread += gdt_mode;
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
gdt_thread = simd_sum(gdt_thread);
|
|
425
|
+
if (lane == 0) red[simd] = gdt_thread;
|
|
426
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
427
|
+
if (t == 0) {
|
|
428
|
+
for (uint i = 1; i < SIMDS; i++) gdt_thread += red[i];
|
|
429
|
+
gdt[h] = gdt_thread;
|
|
430
|
+
}
|
|
431
|
+
}
|