witwin-maxwell 0.3.0__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. witwin/maxwell/__init__.py +164 -0
  2. witwin/maxwell/adapters/__init__.py +1 -0
  3. witwin/maxwell/adapters/gds.py +203 -0
  4. witwin/maxwell/adapters/tidy3d.py +1669 -0
  5. witwin/maxwell/adjoint_inputs.py +70 -0
  6. witwin/maxwell/compiler/__init__.py +11 -0
  7. witwin/maxwell/compiler/materials.py +1355 -0
  8. witwin/maxwell/compiler/monitors.py +164 -0
  9. witwin/maxwell/compiler/sources.py +222 -0
  10. witwin/maxwell/fdfd/__init__.py +3 -0
  11. witwin/maxwell/fdfd/adjoint/__init__.py +3 -0
  12. witwin/maxwell/fdfd/adjoint/bridge.py +180 -0
  13. witwin/maxwell/fdfd/ams.py +300 -0
  14. witwin/maxwell/fdfd/krylov.py +280 -0
  15. witwin/maxwell/fdfd/plotting.py +110 -0
  16. witwin/maxwell/fdfd/postprocess.py +33 -0
  17. witwin/maxwell/fdfd/solver.py +1057 -0
  18. witwin/maxwell/fdtd/__init__.py +4 -0
  19. witwin/maxwell/fdtd/adjoint/__init__.py +21 -0
  20. witwin/maxwell/fdtd/adjoint/bridge.py +746 -0
  21. witwin/maxwell/fdtd/adjoint/capabilities.py +29 -0
  22. witwin/maxwell/fdtd/adjoint/core.py +3614 -0
  23. witwin/maxwell/fdtd/adjoint/dispatch.py +684 -0
  24. witwin/maxwell/fdtd/adjoint/native.py +2561 -0
  25. witwin/maxwell/fdtd/adjoint/profiler.py +195 -0
  26. witwin/maxwell/fdtd/adjoint/reverse_common.py +207 -0
  27. witwin/maxwell/fdtd/adjoint/seeds.py +524 -0
  28. witwin/maxwell/fdtd/boundary/__init__.py +37 -0
  29. witwin/maxwell/fdtd/boundary/bloch.py +90 -0
  30. witwin/maxwell/fdtd/boundary/common.py +41 -0
  31. witwin/maxwell/fdtd/boundary/cpml.py +527 -0
  32. witwin/maxwell/fdtd/boundary/runtime.py +209 -0
  33. witwin/maxwell/fdtd/checkpoint.py +227 -0
  34. witwin/maxwell/fdtd/coords.py +32 -0
  35. witwin/maxwell/fdtd/cuda/__init__.py +11 -0
  36. witwin/maxwell/fdtd/cuda/backend.py +3532 -0
  37. witwin/maxwell/fdtd/cuda/build.py +292 -0
  38. witwin/maxwell/fdtd/cuda/extension.cpp +2053 -0
  39. witwin/maxwell/fdtd/cuda/kernels/adjoint.cu +5290 -0
  40. witwin/maxwell/fdtd/cuda/kernels/aniso.cu +1704 -0
  41. witwin/maxwell/fdtd/cuda/kernels/boundary.cu +424 -0
  42. witwin/maxwell/fdtd/cuda/kernels/common.cu +74 -0
  43. witwin/maxwell/fdtd/cuda/kernels/common.cuh +82 -0
  44. witwin/maxwell/fdtd/cuda/kernels/dispersive.cu +782 -0
  45. witwin/maxwell/fdtd/cuda/kernels/electric.cu +3613 -0
  46. witwin/maxwell/fdtd/cuda/kernels/magnetic.cu +781 -0
  47. witwin/maxwell/fdtd/cuda/kernels/observers.cu +347 -0
  48. witwin/maxwell/fdtd/cuda/kernels/projection.cu +211 -0
  49. witwin/maxwell/fdtd/cuda/kernels/sources.cu +1482 -0
  50. witwin/maxwell/fdtd/cuda/kernels/spectral.cu +297 -0
  51. witwin/maxwell/fdtd/cuda/launch.h +23 -0
  52. witwin/maxwell/fdtd/cuda/prebuilt/witwin_maxwell_fdtd_cuda.pyd +0 -0
  53. witwin/maxwell/fdtd/cuda/runtime/__init__.py +6 -0
  54. witwin/maxwell/fdtd/cuda/runtime/dispatcher.py +6 -0
  55. witwin/maxwell/fdtd/cuda/runtime/graph.py +33 -0
  56. witwin/maxwell/fdtd/cuda/tensors.h +35 -0
  57. witwin/maxwell/fdtd/dispersion.py +48 -0
  58. witwin/maxwell/fdtd/evaluation.py +290 -0
  59. witwin/maxwell/fdtd/excitation/__init__.py +27 -0
  60. witwin/maxwell/fdtd/excitation/injection.py +1102 -0
  61. witwin/maxwell/fdtd/excitation/modes.py +1675 -0
  62. witwin/maxwell/fdtd/excitation/spatial.py +469 -0
  63. witwin/maxwell/fdtd/excitation/temporal.py +510 -0
  64. witwin/maxwell/fdtd/excitation/tfsf_apply.py +264 -0
  65. witwin/maxwell/fdtd/excitation/tfsf_common.py +421 -0
  66. witwin/maxwell/fdtd/excitation/tfsf_specs.py +490 -0
  67. witwin/maxwell/fdtd/excitation/tfsf_state.py +778 -0
  68. witwin/maxwell/fdtd/io.py +31 -0
  69. witwin/maxwell/fdtd/material_pullback.py +371 -0
  70. witwin/maxwell/fdtd/meshing/__init__.py +3 -0
  71. witwin/maxwell/fdtd/meshing/autogrid.py +335 -0
  72. witwin/maxwell/fdtd/observers.py +1453 -0
  73. witwin/maxwell/fdtd/plotting.py +121 -0
  74. witwin/maxwell/fdtd/postprocess.py +205 -0
  75. witwin/maxwell/fdtd/runtime/__init__.py +119 -0
  76. witwin/maxwell/fdtd/runtime/initialization.py +236 -0
  77. witwin/maxwell/fdtd/runtime/materials.py +1563 -0
  78. witwin/maxwell/fdtd/runtime/module_cache.py +31 -0
  79. witwin/maxwell/fdtd/runtime/spectral.py +581 -0
  80. witwin/maxwell/fdtd/runtime/stepping.py +1987 -0
  81. witwin/maxwell/fdtd/solver.py +502 -0
  82. witwin/maxwell/geometry/__init__.py +5 -0
  83. witwin/maxwell/geometry/polyslab.py +299 -0
  84. witwin/maxwell/materials.py +3 -0
  85. witwin/maxwell/media.py +1467 -0
  86. witwin/maxwell/monitors.py +743 -0
  87. witwin/maxwell/ports.py +113 -0
  88. witwin/maxwell/postprocess/RCS.py +139 -0
  89. witwin/maxwell/postprocess/__init__.py +42 -0
  90. witwin/maxwell/postprocess/diffraction.py +219 -0
  91. witwin/maxwell/postprocess/directivity.py +152 -0
  92. witwin/maxwell/postprocess/emission.py +67 -0
  93. witwin/maxwell/postprocess/modal.py +324 -0
  94. witwin/maxwell/postprocess/nfft.py +266 -0
  95. witwin/maxwell/postprocess/scattering_parameters.py +358 -0
  96. witwin/maxwell/postprocess/stratton_chu.py +1193 -0
  97. witwin/maxwell/result.py +967 -0
  98. witwin/maxwell/scene.py +1338 -0
  99. witwin/maxwell/simulation.py +659 -0
  100. witwin/maxwell/sources.py +917 -0
  101. witwin/maxwell/visualization/__init__.py +28 -0
  102. witwin/maxwell/visualization/interactive.py +484 -0
  103. witwin/maxwell/visualization/plots.py +568 -0
  104. witwin_maxwell-0.3.0.dist-info/METADATA +184 -0
  105. witwin_maxwell-0.3.0.dist-info/RECORD +107 -0
  106. witwin_maxwell-0.3.0.dist-info/WHEEL +4 -0
  107. witwin_maxwell-0.3.0.dist-info/licenses/LICENSE +13 -0
@@ -0,0 +1,3613 @@
1
+ #include "../launch.h"
2
+ #include "../tensors.h"
3
+ #include "common.cuh"
4
+
5
+ namespace {
6
+
7
+ dim3 field_block3d() {
8
+ return dim3(128, 2, 1);
9
+ }
10
+
11
+ dim3 field_grid3d(int64_t nx, int64_t ny, int64_t nz, dim3 block) {
12
+ return dim3(
13
+ static_cast<unsigned int>((nz + block.x - 1) / block.x),
14
+ static_cast<unsigned int>((ny + block.y - 1) / block.y),
15
+ static_cast<unsigned int>((nx + block.z - 1) / block.z));
16
+ }
17
+
18
+ bool inactive_boundary_pair(int64_t low_mode, int64_t high_mode) {
19
+ return (low_mode == BOUNDARY_NONE || low_mode == BOUNDARY_PML) &&
20
+ (high_mode == BOUNDARY_NONE || high_mode == BOUNDARY_PML);
21
+ }
22
+
23
+ __device__ __forceinline__ float boundary_diff_low(float low_value, float high_value, int low_mode, float inv_delta) {
24
+ if (low_mode == BOUNDARY_PERIODIC) {
25
+ return (low_value - high_value) * inv_delta;
26
+ }
27
+ if (low_mode == BOUNDARY_PMC) {
28
+ return 2.0f * low_value * inv_delta;
29
+ }
30
+ return 0.0f;
31
+ }
32
+
33
+ __device__ __forceinline__ float boundary_diff_high(float low_value, float high_value, int high_mode, float inv_delta) {
34
+ if (high_mode == BOUNDARY_PERIODIC) {
35
+ return (low_value - high_value) * inv_delta;
36
+ }
37
+ if (high_mode == BOUNDARY_PMC) {
38
+ return -2.0f * high_value * inv_delta;
39
+ }
40
+ return 0.0f;
41
+ }
42
+
43
+ __device__ __forceinline__ bool boundary_pec(unsigned int coord, unsigned int size, int low_mode, int high_mode) {
44
+ return (coord == 0 && low_mode == BOUNDARY_PEC) || (coord + 1 == size && high_mode == BOUNDARY_PEC);
45
+ }
46
+
47
+ __device__ __forceinline__ bool boundary_inactive(unsigned int coord, unsigned int size, int low_mode, int high_mode) {
48
+ const bool low = coord == 0 && (low_mode == BOUNDARY_NONE || low_mode == BOUNDARY_PML);
49
+ const bool high = coord + 1 == size && (high_mode == BOUNDARY_NONE || high_mode == BOUNDARY_PML);
50
+ return low || high;
51
+ }
52
+
53
+ template <int Axis>
54
+ __device__ __forceinline__ float backward_diff_axis(
55
+ const float* __restrict__ source,
56
+ unsigned int target_size,
57
+ unsigned int source_y,
58
+ unsigned int source_z,
59
+ unsigned int i,
60
+ unsigned int j,
61
+ unsigned int k,
62
+ int low_mode,
63
+ int high_mode,
64
+ const float* __restrict__ inv_delta) {
65
+ const unsigned int coord = Axis == 0 ? i : (Axis == 1 ? j : k);
66
+ const unsigned int source_size = target_size - 1;
67
+ const float inv = inv_delta[coord];
68
+ const long long low = Axis == 0 ? offset3d(0, j, k, source_y, source_z)
69
+ : (Axis == 1 ? offset3d(i, 0, k, source_y, source_z) : offset3d(i, j, 0, source_y, source_z));
70
+ const long long high = Axis == 0 ? offset3d(source_size - 1, j, k, source_y, source_z)
71
+ : (Axis == 1 ? offset3d(i, source_size - 1, k, source_y, source_z)
72
+ : offset3d(i, j, source_size - 1, source_y, source_z));
73
+ if (coord == 0) {
74
+ const float low_value = source[low];
75
+ const float high_value = source[high];
76
+ return boundary_diff_low(low_value, high_value, low_mode, inv);
77
+ }
78
+ if (coord + 1 == target_size) {
79
+ const float low_value = source[low];
80
+ const float high_value = source[high];
81
+ return boundary_diff_high(low_value, high_value, high_mode, inv);
82
+ }
83
+ const long long current = offset3d(i, j, k, source_y, source_z);
84
+ const long long previous = Axis == 0 ? offset3d(i - 1, j, k, source_y, source_z)
85
+ : (Axis == 1 ? offset3d(i, j - 1, k, source_y, source_z) : offset3d(i, j, k - 1, source_y, source_z));
86
+ return (source[current] - source[previous]) * inv;
87
+ }
88
+
89
+ __device__ __forceinline__ float backward_diff_axis0(
90
+ const float* source, unsigned int target_x, unsigned int source_y, unsigned int source_z,
91
+ unsigned int i, unsigned int j, unsigned int k, int low_mode, int high_mode, const float* inv_delta) {
92
+ return backward_diff_axis<0>(source, target_x, source_y, source_z, i, j, k, low_mode, high_mode, inv_delta);
93
+ }
94
+
95
+ __device__ __forceinline__ float backward_diff_axis1(
96
+ const float* __restrict__ source,
97
+ unsigned int source_y,
98
+ unsigned int source_z,
99
+ unsigned int i,
100
+ unsigned int j,
101
+ unsigned int k,
102
+ int low_mode,
103
+ int high_mode,
104
+ const float* __restrict__ inv_delta) {
105
+ return backward_diff_axis<1>(source, source_y + 1, source_y, source_z, i, j, k, low_mode, high_mode, inv_delta);
106
+ }
107
+
108
+ __device__ __forceinline__ float backward_diff_axis2(
109
+ const float* __restrict__ source,
110
+ unsigned int source_y,
111
+ unsigned int source_z,
112
+ unsigned int i,
113
+ unsigned int j,
114
+ unsigned int k,
115
+ int low_mode,
116
+ int high_mode,
117
+ const float* __restrict__ inv_delta) {
118
+ return backward_diff_axis<2>(source, source_z + 1, source_y, source_z, i, j, k, low_mode, high_mode, inv_delta);
119
+ }
120
+
121
+ struct ComplexValue {
122
+ float real;
123
+ float imag;
124
+ };
125
+
126
+ __device__ __forceinline__ ComplexValue phase_positive(float real, float imag, float phase_cos, float phase_sin) {
127
+ return {phase_cos * real - phase_sin * imag, phase_sin * real + phase_cos * imag};
128
+ }
129
+
130
+ __device__ __forceinline__ ComplexValue phase_negative(float real, float imag, float phase_cos, float phase_sin) {
131
+ return {phase_cos * real + phase_sin * imag, phase_cos * imag - phase_sin * real};
132
+ }
133
+
134
+ template <int Axis>
135
+ __device__ __forceinline__ ComplexValue bloch_backward_diff_axis(
136
+ const float* __restrict__ real,
137
+ const float* __restrict__ imag,
138
+ unsigned int target_size,
139
+ unsigned int source_x,
140
+ unsigned int source_y,
141
+ unsigned int source_z,
142
+ unsigned int i,
143
+ unsigned int j,
144
+ unsigned int k,
145
+ float phase_cos,
146
+ float phase_sin,
147
+ const float* __restrict__ inv_delta) {
148
+ const unsigned int coord = Axis == 0 ? i : (Axis == 1 ? j : k);
149
+ const unsigned int source_size = Axis == 0 ? source_x : (Axis == 1 ? source_y : source_z);
150
+ const float inv = inv_delta[coord];
151
+ const long long low = Axis == 0 ? offset3d(0, j, k, source_y, source_z)
152
+ : (Axis == 1 ? offset3d(i, 0, k, source_y, source_z) : offset3d(i, j, 0, source_y, source_z));
153
+ const long long high = Axis == 0 ? offset3d(source_size - 1, j, k, source_y, source_z)
154
+ : (Axis == 1 ? offset3d(i, source_size - 1, k, source_y, source_z)
155
+ : offset3d(i, j, source_size - 1, source_y, source_z));
156
+ if (coord == 0) {
157
+ const ComplexValue shifted = phase_negative(real[high], imag[high], phase_cos, phase_sin);
158
+ return {(real[low] - shifted.real) * inv, (imag[low] - shifted.imag) * inv};
159
+ }
160
+ if (coord + 1 == target_size) {
161
+ const ComplexValue shifted = phase_positive(real[low], imag[low], phase_cos, phase_sin);
162
+ return {(shifted.real - real[high]) * inv, (shifted.imag - imag[high]) * inv};
163
+ }
164
+ const long long current = offset3d(i, j, k, source_y, source_z);
165
+ const long long previous = Axis == 0 ? offset3d(i - 1, j, k, source_y, source_z)
166
+ : (Axis == 1 ? offset3d(i, j - 1, k, source_y, source_z) : offset3d(i, j, k - 1, source_y, source_z));
167
+ return {(real[current] - real[previous]) * inv, (imag[current] - imag[previous]) * inv};
168
+ }
169
+
170
+ __device__ __forceinline__ ComplexValue bloch_backward_diff_axis0(
171
+ const float* real, const float* imag, unsigned int target_x, unsigned int source_x,
172
+ unsigned int source_y, unsigned int source_z, unsigned int i, unsigned int j, unsigned int k,
173
+ float phase_cos, float phase_sin, const float* inv_delta) {
174
+ return bloch_backward_diff_axis<0>(real, imag, target_x, source_x, source_y, source_z, i, j, k, phase_cos, phase_sin, inv_delta);
175
+ }
176
+
177
+ __device__ __forceinline__ ComplexValue bloch_backward_diff_axis1(
178
+ const float* __restrict__ real,
179
+ const float* __restrict__ imag,
180
+ unsigned int target_y,
181
+ unsigned int source_y,
182
+ unsigned int source_z,
183
+ unsigned int i,
184
+ unsigned int j,
185
+ unsigned int k,
186
+ float phase_cos,
187
+ float phase_sin,
188
+ const float* __restrict__ inv_delta) {
189
+ return bloch_backward_diff_axis<1>(real, imag, target_y, 0, source_y, source_z, i, j, k, phase_cos, phase_sin, inv_delta);
190
+ }
191
+
192
+ __device__ __forceinline__ ComplexValue bloch_backward_diff_axis2(
193
+ const float* __restrict__ real,
194
+ const float* __restrict__ imag,
195
+ unsigned int target_z,
196
+ unsigned int source_y,
197
+ unsigned int source_z,
198
+ unsigned int i,
199
+ unsigned int j,
200
+ unsigned int k,
201
+ float phase_cos,
202
+ float phase_sin,
203
+ const float* __restrict__ inv_delta) {
204
+ return bloch_backward_diff_axis<2>(real, imag, target_z, 0, source_y, source_z, i, j, k, phase_cos, phase_sin, inv_delta);
205
+ }
206
+
207
+ template <int Component>
208
+ __global__ void update_electric_standard_kernel(
209
+ unsigned int nx,
210
+ unsigned int ny,
211
+ unsigned int nz,
212
+ const float* __restrict__ first,
213
+ const float* __restrict__ second,
214
+ const float* __restrict__ decay,
215
+ const float* __restrict__ curl_coeff,
216
+ const float* __restrict__ inv_a,
217
+ const float* __restrict__ inv_b,
218
+ int a_low_mode,
219
+ int a_high_mode,
220
+ int b_low_mode,
221
+ int b_high_mode,
222
+ float* __restrict__ field) {
223
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
224
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
225
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
226
+ if (i >= nx || j >= ny || k >= nz) {
227
+ return;
228
+ }
229
+ const long long linear = offset3d(i, j, k, ny, nz);
230
+ const unsigned int coord_a = Component == 0 ? j : i;
231
+ const unsigned int size_a = Component == 0 ? ny : nx;
232
+ const unsigned int coord_b = Component == 2 ? j : k;
233
+ const unsigned int size_b = Component == 2 ? ny : nz;
234
+ if (boundary_pec(coord_a, size_a, a_low_mode, a_high_mode) ||
235
+ boundary_pec(coord_b, size_b, b_low_mode, b_high_mode)) {
236
+ field[linear] = 0.0f;
237
+ return;
238
+ }
239
+ if (boundary_inactive(coord_a, size_a, a_low_mode, a_high_mode) ||
240
+ boundary_inactive(coord_b, size_b, b_low_mode, b_high_mode)) {
241
+ return;
242
+ }
243
+ float positive;
244
+ float negative;
245
+ if constexpr (Component == 0) {
246
+ positive = backward_diff_axis1(second, ny - 1, nz, i, j, k, a_low_mode, a_high_mode, inv_a);
247
+ negative = backward_diff_axis2(first, ny, nz - 1, i, j, k, b_low_mode, b_high_mode, inv_b);
248
+ } else if constexpr (Component == 1) {
249
+ positive = backward_diff_axis2(first, ny, nz - 1, i, j, k, b_low_mode, b_high_mode, inv_b);
250
+ negative = backward_diff_axis0(second, nx, ny, nz, i, j, k, a_low_mode, a_high_mode, inv_a);
251
+ } else {
252
+ positive = backward_diff_axis0(second, nx, ny, nz, i, j, k, a_low_mode, a_high_mode, inv_a);
253
+ negative = backward_diff_axis1(first, ny - 1, nz, i, j, k, b_low_mode, b_high_mode, inv_b);
254
+ }
255
+ field[linear] = field[linear] * decay[linear] + curl_coeff[linear] * (positive - negative);
256
+ }
257
+
258
+ template <int Component>
259
+ __global__ void update_electric_standard_interior_kernel(
260
+ unsigned int nx,
261
+ unsigned int ny,
262
+ unsigned int nz,
263
+ const float* __restrict__ first,
264
+ const float* __restrict__ second,
265
+ const float* __restrict__ decay,
266
+ const float* __restrict__ curl_coeff,
267
+ const float* __restrict__ inv_a,
268
+ const float* __restrict__ inv_b,
269
+ float* __restrict__ field) {
270
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x + (Component == 2 ? 0 : 1);
271
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y + (Component == 1 ? 0 : 1);
272
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z + (Component == 0 ? 0 : 1);
273
+ if (i + (Component == 0 ? 0 : 1) >= nx || j + (Component == 1 ? 0 : 1) >= ny ||
274
+ k + (Component == 2 ? 0 : 1) >= nz) {
275
+ return;
276
+ }
277
+ const long long linear = offset3d(i, j, k, ny, nz);
278
+ float positive;
279
+ float negative;
280
+ if constexpr (Component == 0) {
281
+ positive = (second[offset3d(i, j, k, ny - 1, nz)] - second[offset3d(i, j - 1, k, ny - 1, nz)]) * inv_a[j];
282
+ negative = (first[offset3d(i, j, k, ny, nz - 1)] - first[offset3d(i, j, k - 1, ny, nz - 1)]) * inv_b[k];
283
+ } else if constexpr (Component == 1) {
284
+ positive = (first[offset3d(i, j, k, ny, nz - 1)] - first[offset3d(i, j, k - 1, ny, nz - 1)]) * inv_b[k];
285
+ negative = (second[offset3d(i, j, k, ny, nz)] - second[offset3d(i - 1, j, k, ny, nz)]) * inv_a[i];
286
+ } else {
287
+ positive = (second[offset3d(i, j, k, ny, nz)] - second[offset3d(i - 1, j, k, ny, nz)]) * inv_a[i];
288
+ negative = (first[offset3d(i, j, k, ny - 1, nz)] - first[offset3d(i, j - 1, k, ny - 1, nz)]) * inv_b[j];
289
+ }
290
+ field[linear] = field[linear] * decay[linear] + curl_coeff[linear] * (positive - negative);
291
+ }
292
+
293
+ // Space-time modulated permittivity: eps(x, t) = eps_static(x) * m(x, t) with
294
+ // m(x, t) = 1 + mod_cos(x) * cos(W*t) - mod_sin(x) * sin(W*t)
295
+ // where mod_cos = amplitude*cos(phase) and mod_sin = amplitude*sin(phase) are
296
+ // compiled once. The conservative update of d(eps E)/dt = curl H gives
297
+ // E_new = decay * (m_prev / m_next) * E_old + (curl_coeff / m_next) * curl(H)
298
+ // which reduces bit-exactly to the static update where the modulation depth is
299
+ // zero (m_prev = m_next = 1). The host passes the (cos, sin) phase pairs of the
300
+ // previous and new E time instants as scalars.
301
+ __device__ __forceinline__ float modulation_factor(
302
+ float mod_cos, float mod_sin, float phase_cos, float phase_sin) {
303
+ return 1.0f + mod_cos * phase_cos - mod_sin * phase_sin;
304
+ }
305
+
306
+ // Per-cell modulation phase. The scene may hold several distinct modulation
307
+ // frequencies; each Yee edge carries its OWN angular frequency ``omega`` (0 where
308
+ // unmodulated), so the phase cos/sin of the previous and new E time instants are
309
+ // evaluated per cell here rather than passed as scene-wide host scalars. This is
310
+ // what lets disjoint structures modulate at independent frequencies in one run.
311
+ __device__ __forceinline__ void modulation_phase_from_omega(
312
+ float omega, const float* __restrict__ modulation_time,
313
+ float& cos_prev, float& sin_prev, float& cos_next, float& sin_next) {
314
+ sincosf(omega * modulation_time[0], &sin_prev, &cos_prev);
315
+ sincosf(omega * modulation_time[1], &sin_next, &cos_next);
316
+ }
317
+
318
+ __global__ void advance_modulation_time_kernel(float dt, float* __restrict__ modulation_time) {
319
+ modulation_time[0] = modulation_time[1];
320
+ modulation_time[1] += dt;
321
+ }
322
+
323
+ __global__ void update_electric_ex_modulated_kernel(
324
+ unsigned int nx,
325
+ unsigned int ny,
326
+ unsigned int nz,
327
+ const float* __restrict__ hy,
328
+ const float* __restrict__ hz,
329
+ const float* __restrict__ decay,
330
+ const float* __restrict__ curl_coeff,
331
+ const float* __restrict__ mod_cos,
332
+ const float* __restrict__ mod_sin,
333
+ const float* __restrict__ mod_omega,
334
+ const float* __restrict__ modulation_time,
335
+ const float* __restrict__ inv_dy,
336
+ const float* __restrict__ inv_dz,
337
+ int y_low_mode,
338
+ int y_high_mode,
339
+ int z_low_mode,
340
+ int z_high_mode,
341
+ float* __restrict__ ex) {
342
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
343
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
344
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
345
+ if (i >= nx || j >= ny || k >= nz) {
346
+ return;
347
+ }
348
+ const long long linear = offset3d(i, j, k, ny, nz);
349
+ if (boundary_pec(j, ny, y_low_mode, y_high_mode) || boundary_pec(k, nz, z_low_mode, z_high_mode)) {
350
+ ex[linear] = 0.0f;
351
+ return;
352
+ }
353
+ if (boundary_inactive(j, ny, y_low_mode, y_high_mode) ||
354
+ boundary_inactive(k, nz, z_low_mode, z_high_mode)) {
355
+ return;
356
+ }
357
+ const float d_y = backward_diff_axis1(hz, ny - 1, nz, i, j, k, y_low_mode, y_high_mode, inv_dy);
358
+ const float d_z = backward_diff_axis2(hy, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
359
+ const float mc = mod_cos[linear];
360
+ const float ms = mod_sin[linear];
361
+ float cos_prev, sin_prev, cos_next, sin_next;
362
+ modulation_phase_from_omega(mod_omega[linear], modulation_time, cos_prev, sin_prev, cos_next, sin_next);
363
+ const float m_prev = modulation_factor(mc, ms, cos_prev, sin_prev);
364
+ const float inv_next = 1.0f / fmaxf(modulation_factor(mc, ms, cos_next, sin_next), 1.0e-6f);
365
+ ex[linear] = ex[linear] * (decay[linear] * m_prev * inv_next) + (curl_coeff[linear] * inv_next) * (d_y - d_z);
366
+ }
367
+
368
+ __global__ void update_electric_ey_modulated_kernel(
369
+ unsigned int nx,
370
+ unsigned int ny,
371
+ unsigned int nz,
372
+ const float* __restrict__ hx,
373
+ const float* __restrict__ hz,
374
+ const float* __restrict__ decay,
375
+ const float* __restrict__ curl_coeff,
376
+ const float* __restrict__ mod_cos,
377
+ const float* __restrict__ mod_sin,
378
+ const float* __restrict__ mod_omega,
379
+ const float* __restrict__ modulation_time,
380
+ const float* __restrict__ inv_dx,
381
+ const float* __restrict__ inv_dz,
382
+ int x_low_mode,
383
+ int x_high_mode,
384
+ int z_low_mode,
385
+ int z_high_mode,
386
+ float* __restrict__ ey) {
387
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
388
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
389
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
390
+ if (i >= nx || j >= ny || k >= nz) {
391
+ return;
392
+ }
393
+ const long long linear = offset3d(i, j, k, ny, nz);
394
+ if (boundary_pec(i, nx, x_low_mode, x_high_mode) || boundary_pec(k, nz, z_low_mode, z_high_mode)) {
395
+ ey[linear] = 0.0f;
396
+ return;
397
+ }
398
+ if (boundary_inactive(i, nx, x_low_mode, x_high_mode) ||
399
+ boundary_inactive(k, nz, z_low_mode, z_high_mode)) {
400
+ return;
401
+ }
402
+ const float d_z = backward_diff_axis2(hx, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
403
+ const float d_x = backward_diff_axis0(hz, nx, ny, nz, i, j, k, x_low_mode, x_high_mode, inv_dx);
404
+ const float mc = mod_cos[linear];
405
+ const float ms = mod_sin[linear];
406
+ float cos_prev, sin_prev, cos_next, sin_next;
407
+ modulation_phase_from_omega(mod_omega[linear], modulation_time, cos_prev, sin_prev, cos_next, sin_next);
408
+ const float m_prev = modulation_factor(mc, ms, cos_prev, sin_prev);
409
+ const float inv_next = 1.0f / fmaxf(modulation_factor(mc, ms, cos_next, sin_next), 1.0e-6f);
410
+ ey[linear] = ey[linear] * (decay[linear] * m_prev * inv_next) + (curl_coeff[linear] * inv_next) * (d_z - d_x);
411
+ }
412
+
413
+ __global__ void update_electric_ez_modulated_kernel(
414
+ unsigned int nx,
415
+ unsigned int ny,
416
+ unsigned int nz,
417
+ const float* __restrict__ hx,
418
+ const float* __restrict__ hy,
419
+ const float* __restrict__ decay,
420
+ const float* __restrict__ curl_coeff,
421
+ const float* __restrict__ mod_cos,
422
+ const float* __restrict__ mod_sin,
423
+ const float* __restrict__ mod_omega,
424
+ const float* __restrict__ modulation_time,
425
+ const float* __restrict__ inv_dx,
426
+ const float* __restrict__ inv_dy,
427
+ int x_low_mode,
428
+ int x_high_mode,
429
+ int y_low_mode,
430
+ int y_high_mode,
431
+ float* __restrict__ ez) {
432
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
433
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
434
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
435
+ if (i >= nx || j >= ny || k >= nz) {
436
+ return;
437
+ }
438
+ const long long linear = offset3d(i, j, k, ny, nz);
439
+ if (boundary_pec(i, nx, x_low_mode, x_high_mode) || boundary_pec(j, ny, y_low_mode, y_high_mode)) {
440
+ ez[linear] = 0.0f;
441
+ return;
442
+ }
443
+ if (boundary_inactive(i, nx, x_low_mode, x_high_mode) ||
444
+ boundary_inactive(j, ny, y_low_mode, y_high_mode)) {
445
+ return;
446
+ }
447
+ const float d_x = backward_diff_axis0(hy, nx, ny, nz, i, j, k, x_low_mode, x_high_mode, inv_dx);
448
+ const float d_y = backward_diff_axis1(hx, ny - 1, nz, i, j, k, y_low_mode, y_high_mode, inv_dy);
449
+ const float mc = mod_cos[linear];
450
+ const float ms = mod_sin[linear];
451
+ float cos_prev, sin_prev, cos_next, sin_next;
452
+ modulation_phase_from_omega(mod_omega[linear], modulation_time, cos_prev, sin_prev, cos_next, sin_next);
453
+ const float m_prev = modulation_factor(mc, ms, cos_prev, sin_prev);
454
+ const float inv_next = 1.0f / fmaxf(modulation_factor(mc, ms, cos_next, sin_next), 1.0e-6f);
455
+ ez[linear] = ez[linear] * (decay[linear] * m_prev * inv_next) + (curl_coeff[linear] * inv_next) * (d_x - d_y);
456
+ }
457
+
458
+ __global__ void update_electric_ex_bloch_kernel(
459
+ unsigned int nx,
460
+ unsigned int ny,
461
+ unsigned int nz,
462
+ const float* __restrict__ hy_real,
463
+ const float* __restrict__ hy_imag,
464
+ const float* __restrict__ hz_real,
465
+ const float* __restrict__ hz_imag,
466
+ const float* __restrict__ decay,
467
+ const float* __restrict__ curl_coeff,
468
+ float phase_cos_y,
469
+ float phase_sin_y,
470
+ float phase_cos_z,
471
+ float phase_sin_z,
472
+ const float* __restrict__ inv_dy,
473
+ const float* __restrict__ inv_dz,
474
+ float* __restrict__ ex_real,
475
+ float* __restrict__ ex_imag) {
476
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
477
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
478
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
479
+ if (i >= nx || j >= ny || k >= nz) {
480
+ return;
481
+ }
482
+ const long long linear = offset3d(i, j, k, ny, nz);
483
+ const ComplexValue d_y = bloch_backward_diff_axis1(
484
+ hz_real, hz_imag, ny, ny - 1, nz, i, j, k, phase_cos_y, phase_sin_y, inv_dy);
485
+ const ComplexValue d_z = bloch_backward_diff_axis2(
486
+ hy_real, hy_imag, nz, ny, nz - 1, i, j, k, phase_cos_z, phase_sin_z, inv_dz);
487
+ ex_real[linear] = ex_real[linear] * decay[linear] + curl_coeff[linear] * (d_y.real - d_z.real);
488
+ ex_imag[linear] = ex_imag[linear] * decay[linear] + curl_coeff[linear] * (d_y.imag - d_z.imag);
489
+ }
490
+
491
+ __global__ void update_electric_ey_bloch_kernel(
492
+ unsigned int nx,
493
+ unsigned int ny,
494
+ unsigned int nz,
495
+ const float* __restrict__ hx_real,
496
+ const float* __restrict__ hx_imag,
497
+ const float* __restrict__ hz_real,
498
+ const float* __restrict__ hz_imag,
499
+ const float* __restrict__ decay,
500
+ const float* __restrict__ curl_coeff,
501
+ float phase_cos_x,
502
+ float phase_sin_x,
503
+ float phase_cos_z,
504
+ float phase_sin_z,
505
+ const float* __restrict__ inv_dx,
506
+ const float* __restrict__ inv_dz,
507
+ float* __restrict__ ey_real,
508
+ float* __restrict__ ey_imag) {
509
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
510
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
511
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
512
+ if (i >= nx || j >= ny || k >= nz) {
513
+ return;
514
+ }
515
+ const long long linear = offset3d(i, j, k, ny, nz);
516
+ const ComplexValue d_z = bloch_backward_diff_axis2(
517
+ hx_real, hx_imag, nz, ny, nz - 1, i, j, k, phase_cos_z, phase_sin_z, inv_dz);
518
+ const ComplexValue d_x = bloch_backward_diff_axis0(
519
+ hz_real, hz_imag, nx, nx - 1, ny, nz, i, j, k, phase_cos_x, phase_sin_x, inv_dx);
520
+ ey_real[linear] = ey_real[linear] * decay[linear] + curl_coeff[linear] * (d_z.real - d_x.real);
521
+ ey_imag[linear] = ey_imag[linear] * decay[linear] + curl_coeff[linear] * (d_z.imag - d_x.imag);
522
+ }
523
+
524
+ __global__ void update_electric_ez_bloch_kernel(
525
+ unsigned int nx,
526
+ unsigned int ny,
527
+ unsigned int nz,
528
+ const float* __restrict__ hx_real,
529
+ const float* __restrict__ hx_imag,
530
+ const float* __restrict__ hy_real,
531
+ const float* __restrict__ hy_imag,
532
+ const float* __restrict__ decay,
533
+ const float* __restrict__ curl_coeff,
534
+ float phase_cos_x,
535
+ float phase_sin_x,
536
+ float phase_cos_y,
537
+ float phase_sin_y,
538
+ const float* __restrict__ inv_dx,
539
+ const float* __restrict__ inv_dy,
540
+ float* __restrict__ ez_real,
541
+ float* __restrict__ ez_imag) {
542
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
543
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
544
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
545
+ if (i >= nx || j >= ny || k >= nz) {
546
+ return;
547
+ }
548
+ const long long linear = offset3d(i, j, k, ny, nz);
549
+ const ComplexValue d_x = bloch_backward_diff_axis0(
550
+ hy_real, hy_imag, nx, nx - 1, ny, nz, i, j, k, phase_cos_x, phase_sin_x, inv_dx);
551
+ const ComplexValue d_y = bloch_backward_diff_axis1(
552
+ hx_real, hx_imag, ny, ny - 1, nz, i, j, k, phase_cos_y, phase_sin_y, inv_dy);
553
+ ez_real[linear] = ez_real[linear] * decay[linear] + curl_coeff[linear] * (d_x.real - d_y.real);
554
+ ez_imag[linear] = ez_imag[linear] * decay[linear] + curl_coeff[linear] * (d_x.imag - d_y.imag);
555
+ }
556
+
557
+ // Dense-CPML electric update, templated over the Yee component. The two curl
558
+ // derivatives are the boundary-mode-aware backward differences of the standard
559
+ // kernel (``second`` feeds the a-axis derivative, ``first`` the b-axis one), and
560
+ // the psi/kappa split-field correction matches the magnetic CPML kernel. Reduces
561
+ // per-component (ex,ey,ez) to a single instantiation.
562
+ template <int Component>
563
+ __global__ void update_electric_cpml_kernel(
564
+ unsigned int nx,
565
+ unsigned int ny,
566
+ unsigned int nz,
567
+ const float* __restrict__ first,
568
+ const float* __restrict__ second,
569
+ const float* __restrict__ decay,
570
+ const float* __restrict__ curl_coeff,
571
+ const float* __restrict__ inv_kappa_a,
572
+ const float* __restrict__ b_a,
573
+ const float* __restrict__ c_a,
574
+ const float* __restrict__ inv_kappa_b,
575
+ const float* __restrict__ b_b,
576
+ const float* __restrict__ c_b,
577
+ const float* __restrict__ inv_a,
578
+ const float* __restrict__ inv_b,
579
+ int a_low_mode,
580
+ int a_high_mode,
581
+ int b_low_mode,
582
+ int b_high_mode,
583
+ float* __restrict__ psi_a,
584
+ float* __restrict__ psi_b,
585
+ float* __restrict__ field) {
586
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
587
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
588
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
589
+ if (i >= nx || j >= ny || k >= nz) {
590
+ return;
591
+ }
592
+ const long long linear = offset3d(i, j, k, ny, nz);
593
+ const unsigned int coord_a = Component == 0 ? j : i;
594
+ const unsigned int size_a = Component == 0 ? ny : nx;
595
+ const unsigned int coord_b = Component == 2 ? j : k;
596
+ const unsigned int size_b = Component == 2 ? ny : nz;
597
+ if (boundary_pec(coord_a, size_a, a_low_mode, a_high_mode) ||
598
+ boundary_pec(coord_b, size_b, b_low_mode, b_high_mode)) {
599
+ field[linear] = 0.0f;
600
+ return;
601
+ }
602
+ if (boundary_inactive(coord_a, size_a, a_low_mode, a_high_mode) ||
603
+ boundary_inactive(coord_b, size_b, b_low_mode, b_high_mode)) {
604
+ return;
605
+ }
606
+ float d_a;
607
+ float d_b;
608
+ if constexpr (Component == 0) {
609
+ d_a = backward_diff_axis1(second, ny - 1, nz, i, j, k, a_low_mode, a_high_mode, inv_a);
610
+ d_b = backward_diff_axis2(first, ny, nz - 1, i, j, k, b_low_mode, b_high_mode, inv_b);
611
+ } else if constexpr (Component == 1) {
612
+ d_a = backward_diff_axis0(second, nx, ny, nz, i, j, k, a_low_mode, a_high_mode, inv_a);
613
+ d_b = backward_diff_axis2(first, ny, nz - 1, i, j, k, b_low_mode, b_high_mode, inv_b);
614
+ } else {
615
+ d_a = backward_diff_axis0(second, nx, ny, nz, i, j, k, a_low_mode, a_high_mode, inv_a);
616
+ d_b = backward_diff_axis1(first, ny - 1, nz, i, j, k, b_low_mode, b_high_mode, inv_b);
617
+ }
618
+ const float psi_a_value = b_a[coord_a] * psi_a[linear] + c_a[coord_a] * d_a;
619
+ const float psi_b_value = b_b[coord_b] * psi_b[linear] + c_b[coord_b] * d_b;
620
+ psi_a[linear] = psi_a_value;
621
+ psi_b[linear] = psi_b_value;
622
+ const float corrected_a = d_a * inv_kappa_a[coord_a] + psi_a_value;
623
+ const float corrected_b = d_b * inv_kappa_b[coord_b] + psi_b_value;
624
+ const float curl = Component == 1 ? corrected_b - corrected_a : corrected_a - corrected_b;
625
+ field[linear] = field[linear] * decay[linear] + curl_coeff[linear] * curl;
626
+ }
627
+
628
+ // Branch-free interior fast path for the dense-CPML electric update, templated
629
+ // over the component. Plain backward differences (no boundary-mode logic); the
630
+ // per-component thread-offset scheme matches update_electric_standard_interior_kernel.
631
+ template <int Component>
632
+ __global__ void update_electric_cpml_interior_kernel(
633
+ unsigned int nx,
634
+ unsigned int ny,
635
+ unsigned int nz,
636
+ const float* __restrict__ first,
637
+ const float* __restrict__ second,
638
+ const float* __restrict__ decay,
639
+ const float* __restrict__ curl_coeff,
640
+ const float* __restrict__ inv_kappa_a,
641
+ const float* __restrict__ b_a,
642
+ const float* __restrict__ c_a,
643
+ const float* __restrict__ inv_kappa_b,
644
+ const float* __restrict__ b_b,
645
+ const float* __restrict__ c_b,
646
+ const float* __restrict__ inv_a,
647
+ const float* __restrict__ inv_b,
648
+ float* __restrict__ psi_a,
649
+ float* __restrict__ psi_b,
650
+ float* __restrict__ field) {
651
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x + (Component == 2 ? 0 : 1);
652
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y + (Component == 1 ? 0 : 1);
653
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z + (Component == 0 ? 0 : 1);
654
+ if (i + (Component == 0 ? 0 : 1) >= nx || j + (Component == 1 ? 0 : 1) >= ny ||
655
+ k + (Component == 2 ? 0 : 1) >= nz) {
656
+ return;
657
+ }
658
+ const long long linear = offset3d(i, j, k, ny, nz);
659
+ const unsigned int coord_a = Component == 0 ? j : i;
660
+ const unsigned int coord_b = Component == 2 ? j : k;
661
+ float d_a;
662
+ float d_b;
663
+ if constexpr (Component == 0) {
664
+ d_a = (second[offset3d(i, j, k, ny - 1, nz)] - second[offset3d(i, j - 1, k, ny - 1, nz)]) * inv_a[coord_a];
665
+ d_b = (first[offset3d(i, j, k, ny, nz - 1)] - first[offset3d(i, j, k - 1, ny, nz - 1)]) * inv_b[coord_b];
666
+ } else if constexpr (Component == 1) {
667
+ d_a = (second[offset3d(i, j, k, ny, nz)] - second[offset3d(i - 1, j, k, ny, nz)]) * inv_a[coord_a];
668
+ d_b = (first[offset3d(i, j, k, ny, nz - 1)] - first[offset3d(i, j, k - 1, ny, nz - 1)]) * inv_b[coord_b];
669
+ } else {
670
+ d_a = (second[offset3d(i, j, k, ny, nz)] - second[offset3d(i - 1, j, k, ny, nz)]) * inv_a[coord_a];
671
+ d_b = (first[offset3d(i, j, k, ny - 1, nz)] - first[offset3d(i, j - 1, k, ny - 1, nz)]) * inv_b[coord_b];
672
+ }
673
+ const float psi_a_value = b_a[coord_a] * psi_a[linear] + c_a[coord_a] * d_a;
674
+ const float psi_b_value = b_b[coord_b] * psi_b[linear] + c_b[coord_b] * d_b;
675
+ psi_a[linear] = psi_a_value;
676
+ psi_b[linear] = psi_b_value;
677
+ const float corrected_a = d_a * inv_kappa_a[coord_a] + psi_a_value;
678
+ const float corrected_b = d_b * inv_kappa_b[coord_b] + psi_b_value;
679
+ const float curl = Component == 1 ? corrected_b - corrected_a : corrected_a - corrected_b;
680
+ field[linear] = field[linear] * decay[linear] + curl_coeff[linear] * curl;
681
+ }
682
+
683
+ // Dense-CPML variants of the modulated electric update. Identical psi/kappa
684
+ // handling to the base CPML kernels with the modulation factor applied to the
685
+ // decay/curl pair (see the modulated standard kernels above for the scheme).
686
+ __global__ void update_electric_ex_cpml_modulated_kernel(
687
+ unsigned int nx,
688
+ unsigned int ny,
689
+ unsigned int nz,
690
+ const float* __restrict__ hy,
691
+ const float* __restrict__ hz,
692
+ const float* __restrict__ decay,
693
+ const float* __restrict__ curl_coeff,
694
+ const float* __restrict__ mod_cos,
695
+ const float* __restrict__ mod_sin,
696
+ const float* __restrict__ mod_omega,
697
+ const float* __restrict__ modulation_time,
698
+ const float* __restrict__ inv_kappa_y,
699
+ const float* __restrict__ b_y,
700
+ const float* __restrict__ c_y,
701
+ const float* __restrict__ inv_kappa_z,
702
+ const float* __restrict__ b_z,
703
+ const float* __restrict__ c_z,
704
+ const float* __restrict__ inv_dy,
705
+ const float* __restrict__ inv_dz,
706
+ int y_low_mode,
707
+ int y_high_mode,
708
+ int z_low_mode,
709
+ int z_high_mode,
710
+ float* __restrict__ psi_y,
711
+ float* __restrict__ psi_z,
712
+ float* __restrict__ ex) {
713
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
714
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
715
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
716
+ if (i >= nx || j >= ny || k >= nz) {
717
+ return;
718
+ }
719
+ const long long linear = offset3d(i, j, k, ny, nz);
720
+ if (boundary_pec(j, ny, y_low_mode, y_high_mode) || boundary_pec(k, nz, z_low_mode, z_high_mode)) {
721
+ ex[linear] = 0.0f;
722
+ return;
723
+ }
724
+ if (boundary_inactive(j, ny, y_low_mode, y_high_mode) ||
725
+ boundary_inactive(k, nz, z_low_mode, z_high_mode)) {
726
+ return;
727
+ }
728
+ const float d_y = backward_diff_axis1(hz, ny - 1, nz, i, j, k, y_low_mode, y_high_mode, inv_dy);
729
+ const float d_z = backward_diff_axis2(hy, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
730
+ const float psi_y_value = b_y[j] * psi_y[linear] + c_y[j] * d_y;
731
+ const float psi_z_value = b_z[k] * psi_z[linear] + c_z[k] * d_z;
732
+ psi_y[linear] = psi_y_value;
733
+ psi_z[linear] = psi_z_value;
734
+ const float curl = d_y * inv_kappa_y[j] + psi_y_value - d_z * inv_kappa_z[k] - psi_z_value;
735
+ const float mc = mod_cos[linear];
736
+ const float ms = mod_sin[linear];
737
+ float cos_prev, sin_prev, cos_next, sin_next;
738
+ modulation_phase_from_omega(mod_omega[linear], modulation_time, cos_prev, sin_prev, cos_next, sin_next);
739
+ const float m_prev = modulation_factor(mc, ms, cos_prev, sin_prev);
740
+ const float inv_next = 1.0f / fmaxf(modulation_factor(mc, ms, cos_next, sin_next), 1.0e-6f);
741
+ ex[linear] = ex[linear] * (decay[linear] * m_prev * inv_next) + (curl_coeff[linear] * inv_next) * curl;
742
+ }
743
+
744
+ __global__ void update_electric_ey_cpml_modulated_kernel(
745
+ unsigned int nx,
746
+ unsigned int ny,
747
+ unsigned int nz,
748
+ const float* __restrict__ hx,
749
+ const float* __restrict__ hz,
750
+ const float* __restrict__ decay,
751
+ const float* __restrict__ curl_coeff,
752
+ const float* __restrict__ mod_cos,
753
+ const float* __restrict__ mod_sin,
754
+ const float* __restrict__ mod_omega,
755
+ const float* __restrict__ modulation_time,
756
+ const float* __restrict__ inv_kappa_x,
757
+ const float* __restrict__ b_x,
758
+ const float* __restrict__ c_x,
759
+ const float* __restrict__ inv_kappa_z,
760
+ const float* __restrict__ b_z,
761
+ const float* __restrict__ c_z,
762
+ const float* __restrict__ inv_dx,
763
+ const float* __restrict__ inv_dz,
764
+ int x_low_mode,
765
+ int x_high_mode,
766
+ int z_low_mode,
767
+ int z_high_mode,
768
+ float* __restrict__ psi_x,
769
+ float* __restrict__ psi_z,
770
+ float* __restrict__ ey) {
771
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
772
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
773
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
774
+ if (i >= nx || j >= ny || k >= nz) {
775
+ return;
776
+ }
777
+ const long long linear = offset3d(i, j, k, ny, nz);
778
+ if (boundary_pec(i, nx, x_low_mode, x_high_mode) || boundary_pec(k, nz, z_low_mode, z_high_mode)) {
779
+ ey[linear] = 0.0f;
780
+ return;
781
+ }
782
+ if (boundary_inactive(i, nx, x_low_mode, x_high_mode) ||
783
+ boundary_inactive(k, nz, z_low_mode, z_high_mode)) {
784
+ return;
785
+ }
786
+ const float d_z = backward_diff_axis2(hx, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
787
+ const float d_x = backward_diff_axis0(hz, nx, ny, nz, i, j, k, x_low_mode, x_high_mode, inv_dx);
788
+ const float psi_x_value = b_x[i] * psi_x[linear] + c_x[i] * d_x;
789
+ const float psi_z_value = b_z[k] * psi_z[linear] + c_z[k] * d_z;
790
+ psi_x[linear] = psi_x_value;
791
+ psi_z[linear] = psi_z_value;
792
+ const float curl = d_z * inv_kappa_z[k] + psi_z_value - d_x * inv_kappa_x[i] - psi_x_value;
793
+ const float mc = mod_cos[linear];
794
+ const float ms = mod_sin[linear];
795
+ float cos_prev, sin_prev, cos_next, sin_next;
796
+ modulation_phase_from_omega(mod_omega[linear], modulation_time, cos_prev, sin_prev, cos_next, sin_next);
797
+ const float m_prev = modulation_factor(mc, ms, cos_prev, sin_prev);
798
+ const float inv_next = 1.0f / fmaxf(modulation_factor(mc, ms, cos_next, sin_next), 1.0e-6f);
799
+ ey[linear] = ey[linear] * (decay[linear] * m_prev * inv_next) + (curl_coeff[linear] * inv_next) * curl;
800
+ }
801
+
802
+ __global__ void update_electric_ez_cpml_modulated_kernel(
803
+ unsigned int nx,
804
+ unsigned int ny,
805
+ unsigned int nz,
806
+ const float* __restrict__ hx,
807
+ const float* __restrict__ hy,
808
+ const float* __restrict__ decay,
809
+ const float* __restrict__ curl_coeff,
810
+ const float* __restrict__ mod_cos,
811
+ const float* __restrict__ mod_sin,
812
+ const float* __restrict__ mod_omega,
813
+ const float* __restrict__ modulation_time,
814
+ const float* __restrict__ inv_kappa_x,
815
+ const float* __restrict__ b_x,
816
+ const float* __restrict__ c_x,
817
+ const float* __restrict__ inv_kappa_y,
818
+ const float* __restrict__ b_y,
819
+ const float* __restrict__ c_y,
820
+ const float* __restrict__ inv_dx,
821
+ const float* __restrict__ inv_dy,
822
+ int x_low_mode,
823
+ int x_high_mode,
824
+ int y_low_mode,
825
+ int y_high_mode,
826
+ float* __restrict__ psi_x,
827
+ float* __restrict__ psi_y,
828
+ float* __restrict__ ez) {
829
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
830
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
831
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
832
+ if (i >= nx || j >= ny || k >= nz) {
833
+ return;
834
+ }
835
+ const long long linear = offset3d(i, j, k, ny, nz);
836
+ if (boundary_pec(i, nx, x_low_mode, x_high_mode) || boundary_pec(j, ny, y_low_mode, y_high_mode)) {
837
+ ez[linear] = 0.0f;
838
+ return;
839
+ }
840
+ if (boundary_inactive(i, nx, x_low_mode, x_high_mode) ||
841
+ boundary_inactive(j, ny, y_low_mode, y_high_mode)) {
842
+ return;
843
+ }
844
+ const float d_x = backward_diff_axis0(hy, nx, ny, nz, i, j, k, x_low_mode, x_high_mode, inv_dx);
845
+ const float d_y = backward_diff_axis1(hx, ny - 1, nz, i, j, k, y_low_mode, y_high_mode, inv_dy);
846
+ const float psi_x_value = b_x[i] * psi_x[linear] + c_x[i] * d_x;
847
+ const float psi_y_value = b_y[j] * psi_y[linear] + c_y[j] * d_y;
848
+ psi_x[linear] = psi_x_value;
849
+ psi_y[linear] = psi_y_value;
850
+ const float curl = d_x * inv_kappa_x[i] + psi_x_value - d_y * inv_kappa_y[j] - psi_y_value;
851
+ const float mc = mod_cos[linear];
852
+ const float ms = mod_sin[linear];
853
+ float cos_prev, sin_prev, cos_next, sin_next;
854
+ modulation_phase_from_omega(mod_omega[linear], modulation_time, cos_prev, sin_prev, cos_next, sin_next);
855
+ const float m_prev = modulation_factor(mc, ms, cos_prev, sin_prev);
856
+ const float inv_next = 1.0f / fmaxf(modulation_factor(mc, ms, cos_next, sin_next), 1.0e-6f);
857
+ ez[linear] = ez[linear] * (decay[linear] * m_prev * inv_next) + (curl_coeff[linear] * inv_next) * curl;
858
+ }
859
+
860
+ template <int Axis>
861
+ __device__ __forceinline__ float update_compact_electric_psi(
862
+ float* __restrict__ psi,
863
+ const float* __restrict__ b,
864
+ const float* __restrict__ c,
865
+ unsigned int i,
866
+ unsigned int j,
867
+ unsigned int k,
868
+ unsigned int size_y,
869
+ unsigned int size_z,
870
+ unsigned int axis_coord,
871
+ int low_length,
872
+ int high_start,
873
+ int high_length,
874
+ float derivative) {
875
+ const int local = compact_local_index(axis_coord, low_length, high_start, high_length);
876
+ if (local < 0) {
877
+ return 0.0f;
878
+ }
879
+ const unsigned int compact_length = static_cast<unsigned int>(low_length + high_length);
880
+ const long long psi_offset = compact_offset3d_axis<Axis>(i, j, k, size_y, size_z, local, compact_length);
881
+ const float value = b[axis_coord] * psi[psi_offset] + c[axis_coord] * derivative;
882
+ psi[psi_offset] = value;
883
+ return value;
884
+ }
885
+
886
+ template <bool UniformDecay, bool UniformCurl>
887
+ __global__ void update_electric_ex_cpml_compressed_kernel(
888
+ unsigned int nx,
889
+ unsigned int ny,
890
+ unsigned int nz,
891
+ const float* __restrict__ hy,
892
+ const float* __restrict__ hz,
893
+ const float* __restrict__ decay,
894
+ const float* __restrict__ curl_coeff,
895
+ float decay_value,
896
+ float curl_value,
897
+ const float* __restrict__ inv_kappa_y,
898
+ const float* __restrict__ b_y,
899
+ const float* __restrict__ c_y,
900
+ const float* __restrict__ inv_kappa_z,
901
+ const float* __restrict__ b_z,
902
+ const float* __restrict__ c_z,
903
+ const float* __restrict__ inv_dy,
904
+ const float* __restrict__ inv_dz,
905
+ int y_low_mode,
906
+ int y_high_mode,
907
+ int z_low_mode,
908
+ int z_high_mode,
909
+ int y_low_length,
910
+ int y_high_start,
911
+ int y_high_length,
912
+ int z_low_length,
913
+ int z_high_start,
914
+ int z_high_length,
915
+ float* __restrict__ psi_y,
916
+ float* __restrict__ psi_z,
917
+ float* __restrict__ ex) {
918
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
919
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
920
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
921
+ if (i >= nx || j >= ny || k >= nz) {
922
+ return;
923
+ }
924
+ const long long linear = offset3d(i, j, k, ny, nz);
925
+ if (boundary_pec(j, ny, y_low_mode, y_high_mode) || boundary_pec(k, nz, z_low_mode, z_high_mode)) {
926
+ ex[linear] = 0.0f;
927
+ return;
928
+ }
929
+ const bool active = !(boundary_inactive(j, ny, y_low_mode, y_high_mode) ||
930
+ boundary_inactive(k, nz, z_low_mode, z_high_mode));
931
+ if (!active) {
932
+ return;
933
+ }
934
+ const float d_y = backward_diff_axis1(hz, ny - 1, nz, i, j, k, y_low_mode, y_high_mode, inv_dy);
935
+ const float d_z = backward_diff_axis2(hy, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
936
+ const float psi_y_value = update_compact_electric_psi<1>(
937
+ psi_y, b_y, c_y, i, j, k, ny, nz, j, y_low_length, y_high_start, y_high_length, d_y);
938
+ const float psi_z_value = update_compact_electric_psi<2>(
939
+ psi_z, b_z, c_z, i, j, k, ny, nz, k, z_low_length, z_high_start, z_high_length, d_z);
940
+ const float curl = d_y * inv_kappa_y[j] + psi_y_value - d_z * inv_kappa_z[k] - psi_z_value;
941
+ const float decay_factor = UniformDecay ? decay_value : decay[linear];
942
+ const float curl_factor = UniformCurl ? curl_value : curl_coeff[linear];
943
+ ex[linear] = ex[linear] * decay_factor + curl_factor * curl;
944
+ }
945
+
946
+ template <bool UniformDecay, bool UniformCurl>
947
+ __global__ void update_electric_ey_cpml_compressed_kernel(
948
+ unsigned int nx,
949
+ unsigned int ny,
950
+ unsigned int nz,
951
+ const float* __restrict__ hx,
952
+ const float* __restrict__ hz,
953
+ const float* __restrict__ decay,
954
+ const float* __restrict__ curl_coeff,
955
+ float decay_value,
956
+ float curl_value,
957
+ const float* __restrict__ inv_kappa_x,
958
+ const float* __restrict__ b_x,
959
+ const float* __restrict__ c_x,
960
+ const float* __restrict__ inv_kappa_z,
961
+ const float* __restrict__ b_z,
962
+ const float* __restrict__ c_z,
963
+ const float* __restrict__ inv_dx,
964
+ const float* __restrict__ inv_dz,
965
+ int x_low_mode,
966
+ int x_high_mode,
967
+ int z_low_mode,
968
+ int z_high_mode,
969
+ int x_low_length,
970
+ int x_high_start,
971
+ int x_high_length,
972
+ int z_low_length,
973
+ int z_high_start,
974
+ int z_high_length,
975
+ float* __restrict__ psi_x,
976
+ float* __restrict__ psi_z,
977
+ float* __restrict__ ey) {
978
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
979
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
980
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
981
+ if (i >= nx || j >= ny || k >= nz) {
982
+ return;
983
+ }
984
+ const long long linear = offset3d(i, j, k, ny, nz);
985
+ if (boundary_pec(i, nx, x_low_mode, x_high_mode) || boundary_pec(k, nz, z_low_mode, z_high_mode)) {
986
+ ey[linear] = 0.0f;
987
+ return;
988
+ }
989
+ const bool active = !(boundary_inactive(i, nx, x_low_mode, x_high_mode) ||
990
+ boundary_inactive(k, nz, z_low_mode, z_high_mode));
991
+ if (!active) {
992
+ return;
993
+ }
994
+ const float d_z = backward_diff_axis2(hx, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
995
+ const float d_x = backward_diff_axis0(hz, nx, ny, nz, i, j, k, x_low_mode, x_high_mode, inv_dx);
996
+ const float psi_x_value = update_compact_electric_psi<0>(
997
+ psi_x, b_x, c_x, i, j, k, ny, nz, i, x_low_length, x_high_start, x_high_length, d_x);
998
+ const float psi_z_value = update_compact_electric_psi<2>(
999
+ psi_z, b_z, c_z, i, j, k, ny, nz, k, z_low_length, z_high_start, z_high_length, d_z);
1000
+ const float curl = d_z * inv_kappa_z[k] + psi_z_value - d_x * inv_kappa_x[i] - psi_x_value;
1001
+ const float decay_factor = UniformDecay ? decay_value : decay[linear];
1002
+ const float curl_factor = UniformCurl ? curl_value : curl_coeff[linear];
1003
+ ey[linear] = ey[linear] * decay_factor + curl_factor * curl;
1004
+ }
1005
+
1006
+ template <bool UniformDecay, bool UniformCurl>
1007
+ __global__ void update_electric_ez_cpml_compressed_kernel(
1008
+ unsigned int nx,
1009
+ unsigned int ny,
1010
+ unsigned int nz,
1011
+ const float* __restrict__ hx,
1012
+ const float* __restrict__ hy,
1013
+ const float* __restrict__ decay,
1014
+ const float* __restrict__ curl_coeff,
1015
+ float decay_value,
1016
+ float curl_value,
1017
+ const float* __restrict__ inv_kappa_x,
1018
+ const float* __restrict__ b_x,
1019
+ const float* __restrict__ c_x,
1020
+ const float* __restrict__ inv_kappa_y,
1021
+ const float* __restrict__ b_y,
1022
+ const float* __restrict__ c_y,
1023
+ const float* __restrict__ inv_dx,
1024
+ const float* __restrict__ inv_dy,
1025
+ int x_low_mode,
1026
+ int x_high_mode,
1027
+ int y_low_mode,
1028
+ int y_high_mode,
1029
+ int x_low_length,
1030
+ int x_high_start,
1031
+ int x_high_length,
1032
+ int y_low_length,
1033
+ int y_high_start,
1034
+ int y_high_length,
1035
+ float* __restrict__ psi_x,
1036
+ float* __restrict__ psi_y,
1037
+ float* __restrict__ ez) {
1038
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
1039
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
1040
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
1041
+ if (i >= nx || j >= ny || k >= nz) {
1042
+ return;
1043
+ }
1044
+ const long long linear = offset3d(i, j, k, ny, nz);
1045
+ if (boundary_pec(i, nx, x_low_mode, x_high_mode) || boundary_pec(j, ny, y_low_mode, y_high_mode)) {
1046
+ ez[linear] = 0.0f;
1047
+ return;
1048
+ }
1049
+ const bool active = !(boundary_inactive(i, nx, x_low_mode, x_high_mode) ||
1050
+ boundary_inactive(j, ny, y_low_mode, y_high_mode));
1051
+ if (!active) {
1052
+ return;
1053
+ }
1054
+ const float d_x = backward_diff_axis0(hy, nx, ny, nz, i, j, k, x_low_mode, x_high_mode, inv_dx);
1055
+ const float d_y = backward_diff_axis1(hx, ny - 1, nz, i, j, k, y_low_mode, y_high_mode, inv_dy);
1056
+ const float psi_x_value = update_compact_electric_psi<0>(
1057
+ psi_x, b_x, c_x, i, j, k, ny, nz, i, x_low_length, x_high_start, x_high_length, d_x);
1058
+ const float psi_y_value = update_compact_electric_psi<1>(
1059
+ psi_y, b_y, c_y, i, j, k, ny, nz, j, y_low_length, y_high_start, y_high_length, d_y);
1060
+ const float curl = d_x * inv_kappa_x[i] + psi_x_value - d_y * inv_kappa_y[j] - psi_y_value;
1061
+ const float decay_factor = UniformDecay ? decay_value : decay[linear];
1062
+ const float curl_factor = UniformCurl ? curl_value : curl_coeff[linear];
1063
+ ez[linear] = ez[linear] * decay_factor + curl_factor * curl;
1064
+ }
1065
+
1066
+ // Compressed-(slab-)CPML variants of the modulated electric update. The compact
1067
+ // psi bookkeeping is identical to the plain compressed CPML kernels; the
1068
+ // modulation factor is applied to the decay/curl pair exactly as in the dense
1069
+ // modulated CPML kernels. The modulated path always carries per-cell decay/curl
1070
+ // coefficients, so these kernels do not take the uniform-scalar fast path.
1071
+ __global__ void update_electric_ex_cpml_modulated_compressed_kernel(
1072
+ unsigned int nx,
1073
+ unsigned int ny,
1074
+ unsigned int nz,
1075
+ const float* __restrict__ hy,
1076
+ const float* __restrict__ hz,
1077
+ const float* __restrict__ decay,
1078
+ const float* __restrict__ curl_coeff,
1079
+ const float* __restrict__ mod_cos,
1080
+ const float* __restrict__ mod_sin,
1081
+ const float* __restrict__ mod_omega,
1082
+ const float* __restrict__ modulation_time,
1083
+ const float* __restrict__ inv_kappa_y,
1084
+ const float* __restrict__ b_y,
1085
+ const float* __restrict__ c_y,
1086
+ const float* __restrict__ inv_kappa_z,
1087
+ const float* __restrict__ b_z,
1088
+ const float* __restrict__ c_z,
1089
+ const float* __restrict__ inv_dy,
1090
+ const float* __restrict__ inv_dz,
1091
+ int y_low_mode,
1092
+ int y_high_mode,
1093
+ int z_low_mode,
1094
+ int z_high_mode,
1095
+ int y_low_length,
1096
+ int y_high_start,
1097
+ int y_high_length,
1098
+ int z_low_length,
1099
+ int z_high_start,
1100
+ int z_high_length,
1101
+ float* __restrict__ psi_y,
1102
+ float* __restrict__ psi_z,
1103
+ float* __restrict__ ex) {
1104
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
1105
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
1106
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
1107
+ if (i >= nx || j >= ny || k >= nz) {
1108
+ return;
1109
+ }
1110
+ const long long linear = offset3d(i, j, k, ny, nz);
1111
+ if (boundary_pec(j, ny, y_low_mode, y_high_mode) || boundary_pec(k, nz, z_low_mode, z_high_mode)) {
1112
+ ex[linear] = 0.0f;
1113
+ return;
1114
+ }
1115
+ const bool active = !(boundary_inactive(j, ny, y_low_mode, y_high_mode) ||
1116
+ boundary_inactive(k, nz, z_low_mode, z_high_mode));
1117
+ if (!active) {
1118
+ return;
1119
+ }
1120
+ const float d_y = backward_diff_axis1(hz, ny - 1, nz, i, j, k, y_low_mode, y_high_mode, inv_dy);
1121
+ const float d_z = backward_diff_axis2(hy, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
1122
+ const float psi_y_value = update_compact_electric_psi<1>(
1123
+ psi_y, b_y, c_y, i, j, k, ny, nz, j, y_low_length, y_high_start, y_high_length, d_y);
1124
+ const float psi_z_value = update_compact_electric_psi<2>(
1125
+ psi_z, b_z, c_z, i, j, k, ny, nz, k, z_low_length, z_high_start, z_high_length, d_z);
1126
+ const float curl = d_y * inv_kappa_y[j] + psi_y_value - d_z * inv_kappa_z[k] - psi_z_value;
1127
+ const float mc = mod_cos[linear];
1128
+ const float ms = mod_sin[linear];
1129
+ float cos_prev, sin_prev, cos_next, sin_next;
1130
+ modulation_phase_from_omega(mod_omega[linear], modulation_time, cos_prev, sin_prev, cos_next, sin_next);
1131
+ const float m_prev = modulation_factor(mc, ms, cos_prev, sin_prev);
1132
+ const float inv_next = 1.0f / fmaxf(modulation_factor(mc, ms, cos_next, sin_next), 1.0e-6f);
1133
+ ex[linear] = ex[linear] * (decay[linear] * m_prev * inv_next) + (curl_coeff[linear] * inv_next) * curl;
1134
+ }
1135
+
1136
+ __global__ void update_electric_ey_cpml_modulated_compressed_kernel(
1137
+ unsigned int nx,
1138
+ unsigned int ny,
1139
+ unsigned int nz,
1140
+ const float* __restrict__ hx,
1141
+ const float* __restrict__ hz,
1142
+ const float* __restrict__ decay,
1143
+ const float* __restrict__ curl_coeff,
1144
+ const float* __restrict__ mod_cos,
1145
+ const float* __restrict__ mod_sin,
1146
+ const float* __restrict__ mod_omega,
1147
+ const float* __restrict__ modulation_time,
1148
+ const float* __restrict__ inv_kappa_x,
1149
+ const float* __restrict__ b_x,
1150
+ const float* __restrict__ c_x,
1151
+ const float* __restrict__ inv_kappa_z,
1152
+ const float* __restrict__ b_z,
1153
+ const float* __restrict__ c_z,
1154
+ const float* __restrict__ inv_dx,
1155
+ const float* __restrict__ inv_dz,
1156
+ int x_low_mode,
1157
+ int x_high_mode,
1158
+ int z_low_mode,
1159
+ int z_high_mode,
1160
+ int x_low_length,
1161
+ int x_high_start,
1162
+ int x_high_length,
1163
+ int z_low_length,
1164
+ int z_high_start,
1165
+ int z_high_length,
1166
+ float* __restrict__ psi_x,
1167
+ float* __restrict__ psi_z,
1168
+ float* __restrict__ ey) {
1169
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
1170
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
1171
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
1172
+ if (i >= nx || j >= ny || k >= nz) {
1173
+ return;
1174
+ }
1175
+ const long long linear = offset3d(i, j, k, ny, nz);
1176
+ if (boundary_pec(i, nx, x_low_mode, x_high_mode) || boundary_pec(k, nz, z_low_mode, z_high_mode)) {
1177
+ ey[linear] = 0.0f;
1178
+ return;
1179
+ }
1180
+ const bool active = !(boundary_inactive(i, nx, x_low_mode, x_high_mode) ||
1181
+ boundary_inactive(k, nz, z_low_mode, z_high_mode));
1182
+ if (!active) {
1183
+ return;
1184
+ }
1185
+ const float d_z = backward_diff_axis2(hx, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
1186
+ const float d_x = backward_diff_axis0(hz, nx, ny, nz, i, j, k, x_low_mode, x_high_mode, inv_dx);
1187
+ const float psi_x_value = update_compact_electric_psi<0>(
1188
+ psi_x, b_x, c_x, i, j, k, ny, nz, i, x_low_length, x_high_start, x_high_length, d_x);
1189
+ const float psi_z_value = update_compact_electric_psi<2>(
1190
+ psi_z, b_z, c_z, i, j, k, ny, nz, k, z_low_length, z_high_start, z_high_length, d_z);
1191
+ const float curl = d_z * inv_kappa_z[k] + psi_z_value - d_x * inv_kappa_x[i] - psi_x_value;
1192
+ const float mc = mod_cos[linear];
1193
+ const float ms = mod_sin[linear];
1194
+ float cos_prev, sin_prev, cos_next, sin_next;
1195
+ modulation_phase_from_omega(mod_omega[linear], modulation_time, cos_prev, sin_prev, cos_next, sin_next);
1196
+ const float m_prev = modulation_factor(mc, ms, cos_prev, sin_prev);
1197
+ const float inv_next = 1.0f / fmaxf(modulation_factor(mc, ms, cos_next, sin_next), 1.0e-6f);
1198
+ ey[linear] = ey[linear] * (decay[linear] * m_prev * inv_next) + (curl_coeff[linear] * inv_next) * curl;
1199
+ }
1200
+
1201
+ __global__ void update_electric_ez_cpml_modulated_compressed_kernel(
1202
+ unsigned int nx,
1203
+ unsigned int ny,
1204
+ unsigned int nz,
1205
+ const float* __restrict__ hx,
1206
+ const float* __restrict__ hy,
1207
+ const float* __restrict__ decay,
1208
+ const float* __restrict__ curl_coeff,
1209
+ const float* __restrict__ mod_cos,
1210
+ const float* __restrict__ mod_sin,
1211
+ const float* __restrict__ mod_omega,
1212
+ const float* __restrict__ modulation_time,
1213
+ const float* __restrict__ inv_kappa_x,
1214
+ const float* __restrict__ b_x,
1215
+ const float* __restrict__ c_x,
1216
+ const float* __restrict__ inv_kappa_y,
1217
+ const float* __restrict__ b_y,
1218
+ const float* __restrict__ c_y,
1219
+ const float* __restrict__ inv_dx,
1220
+ const float* __restrict__ inv_dy,
1221
+ int x_low_mode,
1222
+ int x_high_mode,
1223
+ int y_low_mode,
1224
+ int y_high_mode,
1225
+ int x_low_length,
1226
+ int x_high_start,
1227
+ int x_high_length,
1228
+ int y_low_length,
1229
+ int y_high_start,
1230
+ int y_high_length,
1231
+ float* __restrict__ psi_x,
1232
+ float* __restrict__ psi_y,
1233
+ float* __restrict__ ez) {
1234
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
1235
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
1236
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
1237
+ if (i >= nx || j >= ny || k >= nz) {
1238
+ return;
1239
+ }
1240
+ const long long linear = offset3d(i, j, k, ny, nz);
1241
+ if (boundary_pec(i, nx, x_low_mode, x_high_mode) || boundary_pec(j, ny, y_low_mode, y_high_mode)) {
1242
+ ez[linear] = 0.0f;
1243
+ return;
1244
+ }
1245
+ const bool active = !(boundary_inactive(i, nx, x_low_mode, x_high_mode) ||
1246
+ boundary_inactive(j, ny, y_low_mode, y_high_mode));
1247
+ if (!active) {
1248
+ return;
1249
+ }
1250
+ const float d_x = backward_diff_axis0(hy, nx, ny, nz, i, j, k, x_low_mode, x_high_mode, inv_dx);
1251
+ const float d_y = backward_diff_axis1(hx, ny - 1, nz, i, j, k, y_low_mode, y_high_mode, inv_dy);
1252
+ const float psi_x_value = update_compact_electric_psi<0>(
1253
+ psi_x, b_x, c_x, i, j, k, ny, nz, i, x_low_length, x_high_start, x_high_length, d_x);
1254
+ const float psi_y_value = update_compact_electric_psi<1>(
1255
+ psi_y, b_y, c_y, i, j, k, ny, nz, j, y_low_length, y_high_start, y_high_length, d_y);
1256
+ const float curl = d_x * inv_kappa_x[i] + psi_x_value - d_y * inv_kappa_y[j] - psi_y_value;
1257
+ const float mc = mod_cos[linear];
1258
+ const float ms = mod_sin[linear];
1259
+ float cos_prev, sin_prev, cos_next, sin_next;
1260
+ modulation_phase_from_omega(mod_omega[linear], modulation_time, cos_prev, sin_prev, cos_next, sin_next);
1261
+ const float m_prev = modulation_factor(mc, ms, cos_prev, sin_prev);
1262
+ const float inv_next = 1.0f / fmaxf(modulation_factor(mc, ms, cos_next, sin_next), 1.0e-6f);
1263
+ ez[linear] = ez[linear] * (decay[linear] * m_prev * inv_next) + (curl_coeff[linear] * inv_next) * curl;
1264
+ }
1265
+
1266
+ void check_electric_inputs(
1267
+ const torch::stable::Tensor& field,
1268
+ const torch::stable::Tensor& first,
1269
+ const torch::stable::Tensor& second,
1270
+ const torch::stable::Tensor& decay,
1271
+ const torch::stable::Tensor& curl,
1272
+ const char* name) {
1273
+ check_float32_tensor(field, name);
1274
+ check_float32_tensor(first, "first");
1275
+ check_float32_tensor(second, "second");
1276
+ check_float32_tensor(decay, "decay");
1277
+ check_float32_tensor(curl, "curl");
1278
+ check_same_cuda_device(field, first, "first");
1279
+ check_same_cuda_device(field, second, "second");
1280
+ check_same_cuda_device(field, decay, "decay");
1281
+ check_same_cuda_device(field, curl, "curl");
1282
+ check_contiguous_tensor(field, name);
1283
+ check_contiguous_tensor(first, "first");
1284
+ check_contiguous_tensor(second, "second");
1285
+ check_contiguous_tensor(decay, "decay");
1286
+ check_contiguous_tensor(curl, "curl");
1287
+ STD_TORCH_CHECK(field.dim() == 3, name, " must be rank 3");
1288
+ STD_TORCH_CHECK(first.dim() == 3, "first must be rank 3");
1289
+ STD_TORCH_CHECK(second.dim() == 3, "second must be rank 3");
1290
+ STD_TORCH_CHECK(decay.sizes().equals(field.sizes()), "decay must match field shape");
1291
+ STD_TORCH_CHECK(curl.sizes().equals(field.sizes()), "curl must match field shape");
1292
+ }
1293
+
1294
+ void check_modulation_fields(
1295
+ const torch::stable::Tensor& field,
1296
+ const torch::stable::Tensor& mod_cos,
1297
+ const torch::stable::Tensor& mod_sin,
1298
+ const torch::stable::Tensor& mod_omega) {
1299
+ check_float32_tensor(mod_cos, "mod_cos");
1300
+ check_float32_tensor(mod_sin, "mod_sin");
1301
+ check_float32_tensor(mod_omega, "mod_omega");
1302
+ check_same_cuda_device(field, mod_cos, "mod_cos");
1303
+ check_same_cuda_device(field, mod_sin, "mod_sin");
1304
+ check_same_cuda_device(field, mod_omega, "mod_omega");
1305
+ check_contiguous_tensor(mod_cos, "mod_cos");
1306
+ check_contiguous_tensor(mod_sin, "mod_sin");
1307
+ check_contiguous_tensor(mod_omega, "mod_omega");
1308
+ STD_TORCH_CHECK(mod_cos.sizes().equals(field.sizes()), "mod_cos must match field shape");
1309
+ STD_TORCH_CHECK(mod_sin.sizes().equals(field.sizes()), "mod_sin must match field shape");
1310
+ STD_TORCH_CHECK(mod_omega.sizes().equals(field.sizes()), "mod_omega must match field shape");
1311
+ }
1312
+
1313
+ void check_modulation_time(
1314
+ const torch::stable::Tensor& field,
1315
+ const torch::stable::Tensor& modulation_time) {
1316
+ check_float32_tensor(modulation_time, "modulation_time");
1317
+ check_contiguous_tensor(modulation_time, "modulation_time");
1318
+ check_same_cuda_device(field, modulation_time, "modulation_time");
1319
+ STD_TORCH_CHECK(
1320
+ modulation_time.dim() == 1 && modulation_time.numel() == 2,
1321
+ "modulation_time must be a contiguous float32 CUDA tensor with two elements");
1322
+ }
1323
+
1324
+ void check_rank3_shape(
1325
+ const torch::stable::Tensor& tensor,
1326
+ const char* name,
1327
+ int64_t x,
1328
+ int64_t y,
1329
+ int64_t z) {
1330
+ STD_TORCH_CHECK(tensor.dim() == 3, name, " must be rank 3");
1331
+ STD_TORCH_CHECK(
1332
+ tensor.size(0) == x && tensor.size(1) == y && tensor.size(2) == z,
1333
+ name,
1334
+ " has an incompatible Yee-grid shape");
1335
+ }
1336
+
1337
+ void check_electric_bloch_inputs(
1338
+ const torch::stable::Tensor& field_real,
1339
+ const torch::stable::Tensor& field_imag,
1340
+ const torch::stable::Tensor& first_real,
1341
+ const torch::stable::Tensor& first_imag,
1342
+ const torch::stable::Tensor& second_real,
1343
+ const torch::stable::Tensor& second_imag,
1344
+ const torch::stable::Tensor& decay,
1345
+ const torch::stable::Tensor& curl,
1346
+ const char* name) {
1347
+ check_electric_inputs(field_real, first_real, second_real, decay, curl, name);
1348
+ check_float32_tensor(field_imag, "field_imag");
1349
+ check_float32_tensor(first_imag, "first_imag");
1350
+ check_float32_tensor(second_imag, "second_imag");
1351
+ check_same_cuda_device(field_real, field_imag, "field_imag");
1352
+ check_same_cuda_device(field_real, first_imag, "first_imag");
1353
+ check_same_cuda_device(field_real, second_imag, "second_imag");
1354
+ check_contiguous_tensor(field_imag, "field_imag");
1355
+ check_contiguous_tensor(first_imag, "first_imag");
1356
+ check_contiguous_tensor(second_imag, "second_imag");
1357
+ STD_TORCH_CHECK(field_imag.sizes().equals(field_real.sizes()), "field imaginary tensor must match real shape");
1358
+ STD_TORCH_CHECK(first_imag.sizes().equals(first_real.sizes()), "first imaginary tensor must match real shape");
1359
+ STD_TORCH_CHECK(second_imag.sizes().equals(second_real.sizes()), "second imaginary tensor must match real shape");
1360
+ STD_TORCH_CHECK(first_real.dim() == 3, "first Bloch source tensor must be rank 3");
1361
+ STD_TORCH_CHECK(second_real.dim() == 3, "second Bloch source tensor must be rank 3");
1362
+ }
1363
+
1364
+ void check_vector_input(const torch::stable::Tensor& tensor, int64_t length, const char* name) {
1365
+ check_float32_tensor(tensor, name);
1366
+ check_contiguous_tensor(tensor, name);
1367
+ STD_TORCH_CHECK(tensor.dim() == 1, name, " must be rank 1");
1368
+ STD_TORCH_CHECK(tensor.size(0) == length, name, " length must match CPML field axis");
1369
+ }
1370
+
1371
+ void check_spacing_vector(
1372
+ const torch::stable::Tensor& field,
1373
+ const torch::stable::Tensor& inv_delta,
1374
+ int64_t axis,
1375
+ const char* name) {
1376
+ check_vector_input(inv_delta, field.size(axis), name);
1377
+ check_same_cuda_device(field, inv_delta, name);
1378
+ }
1379
+
1380
+ void check_electric_cpml_inputs(
1381
+ const torch::stable::Tensor& field,
1382
+ const torch::stable::Tensor& first,
1383
+ const torch::stable::Tensor& second,
1384
+ const torch::stable::Tensor& decay,
1385
+ const torch::stable::Tensor& curl,
1386
+ const torch::stable::Tensor& psi_first,
1387
+ const torch::stable::Tensor& psi_second,
1388
+ const torch::stable::Tensor& inv_kappa_first,
1389
+ const torch::stable::Tensor& b_first,
1390
+ const torch::stable::Tensor& c_first,
1391
+ const torch::stable::Tensor& inv_kappa_second,
1392
+ const torch::stable::Tensor& b_second,
1393
+ const torch::stable::Tensor& c_second,
1394
+ int64_t first_axis,
1395
+ int64_t second_axis,
1396
+ const char* name) {
1397
+ check_electric_inputs(field, first, second, decay, curl, name);
1398
+ check_float32_tensor(psi_first, "psi_first");
1399
+ check_float32_tensor(psi_second, "psi_second");
1400
+ check_same_cuda_device(field, psi_first, "psi_first");
1401
+ check_same_cuda_device(field, psi_second, "psi_second");
1402
+ check_contiguous_tensor(psi_first, "psi_first");
1403
+ check_contiguous_tensor(psi_second, "psi_second");
1404
+ STD_TORCH_CHECK(psi_first.sizes().equals(field.sizes()), "psi_first must match field shape");
1405
+ STD_TORCH_CHECK(psi_second.sizes().equals(field.sizes()), "psi_second must match field shape");
1406
+ check_vector_input(inv_kappa_first, field.size(first_axis), "inv_kappa_first");
1407
+ check_vector_input(b_first, field.size(first_axis), "b_first");
1408
+ check_vector_input(c_first, field.size(first_axis), "c_first");
1409
+ check_vector_input(inv_kappa_second, field.size(second_axis), "inv_kappa_second");
1410
+ check_vector_input(b_second, field.size(second_axis), "b_second");
1411
+ check_vector_input(c_second, field.size(second_axis), "c_second");
1412
+ check_same_cuda_device(field, inv_kappa_first, "inv_kappa_first");
1413
+ check_same_cuda_device(field, b_first, "b_first");
1414
+ check_same_cuda_device(field, c_first, "c_first");
1415
+ check_same_cuda_device(field, inv_kappa_second, "inv_kappa_second");
1416
+ check_same_cuda_device(field, b_second, "b_second");
1417
+ check_same_cuda_device(field, c_second, "c_second");
1418
+ }
1419
+
1420
+ void check_compact_psi_input(
1421
+ const torch::stable::Tensor& psi,
1422
+ const torch::stable::Tensor& field,
1423
+ int64_t axis,
1424
+ int64_t low_length,
1425
+ int64_t high_length,
1426
+ const char* name) {
1427
+ check_float32_tensor(psi, name);
1428
+ check_same_cuda_device(field, psi, name);
1429
+ check_contiguous_tensor(psi, name);
1430
+ STD_TORCH_CHECK(psi.dim() == 3, name, " must be rank 3");
1431
+ for (int64_t dim = 0; dim < 3; ++dim) {
1432
+ const int64_t expected = dim == axis ? low_length + high_length : field.size(dim);
1433
+ STD_TORCH_CHECK(psi.size(dim) == expected, name, " shape does not match compact CPML layout");
1434
+ }
1435
+ }
1436
+
1437
+ void check_electric_cpml_compressed_inputs(
1438
+ const torch::stable::Tensor& field,
1439
+ const torch::stable::Tensor& first,
1440
+ const torch::stable::Tensor& second,
1441
+ const torch::stable::Tensor& decay,
1442
+ const torch::stable::Tensor& curl,
1443
+ const torch::stable::Tensor& psi_first,
1444
+ const torch::stable::Tensor& psi_second,
1445
+ const torch::stable::Tensor& inv_kappa_first,
1446
+ const torch::stable::Tensor& b_first,
1447
+ const torch::stable::Tensor& c_first,
1448
+ const torch::stable::Tensor& inv_kappa_second,
1449
+ const torch::stable::Tensor& b_second,
1450
+ const torch::stable::Tensor& c_second,
1451
+ int64_t first_axis,
1452
+ int64_t second_axis,
1453
+ int64_t first_low_length,
1454
+ int64_t first_high_length,
1455
+ int64_t second_low_length,
1456
+ int64_t second_high_length,
1457
+ const char* name) {
1458
+ check_electric_inputs(field, first, second, decay, curl, name);
1459
+ check_compact_psi_input(psi_first, field, first_axis, first_low_length, first_high_length, "psi_first");
1460
+ check_compact_psi_input(psi_second, field, second_axis, second_low_length, second_high_length, "psi_second");
1461
+ check_vector_input(inv_kappa_first, field.size(first_axis), "inv_kappa_first");
1462
+ check_vector_input(b_first, field.size(first_axis), "b_first");
1463
+ check_vector_input(c_first, field.size(first_axis), "c_first");
1464
+ check_vector_input(inv_kappa_second, field.size(second_axis), "inv_kappa_second");
1465
+ check_vector_input(b_second, field.size(second_axis), "b_second");
1466
+ check_vector_input(c_second, field.size(second_axis), "c_second");
1467
+ check_same_cuda_device(field, inv_kappa_first, "inv_kappa_first");
1468
+ check_same_cuda_device(field, b_first, "b_first");
1469
+ check_same_cuda_device(field, c_first, "c_first");
1470
+ check_same_cuda_device(field, inv_kappa_second, "inv_kappa_second");
1471
+ check_same_cuda_device(field, b_second, "b_second");
1472
+ check_same_cuda_device(field, c_second, "c_second");
1473
+ }
1474
+
1475
+ __device__ __forceinline__ bool cpml_correction_active(
1476
+ unsigned int global_index, unsigned int full_size, int low_mode, int high_mode) {
1477
+ const bool low_inactive = (global_index == 0) &&
1478
+ (low_mode == BOUNDARY_NONE || low_mode == BOUNDARY_PEC || low_mode == BOUNDARY_PML);
1479
+ const bool high_inactive = (global_index + 1 == full_size) &&
1480
+ (high_mode == BOUNDARY_NONE || high_mode == BOUNDARY_PEC || high_mode == BOUNDARY_PML);
1481
+ return !(low_inactive || high_inactive);
1482
+ }
1483
+
1484
+ // Mixed Bloch (transverse) + standard (z) electric update for the Bloch+CPML
1485
+ // path. Ex: Bloch backward diff of Hz along y, standard backward diff of Hy
1486
+ // along z.
1487
+ __global__ void update_electric_ex_bloch_y_standard_z_kernel(
1488
+ unsigned int nx,
1489
+ unsigned int ny,
1490
+ unsigned int nz,
1491
+ const float* __restrict__ hy_real,
1492
+ const float* __restrict__ hy_imag,
1493
+ const float* __restrict__ hz_real,
1494
+ const float* __restrict__ hz_imag,
1495
+ const float* __restrict__ decay,
1496
+ const float* __restrict__ curl_coeff,
1497
+ float phase_cos_y,
1498
+ float phase_sin_y,
1499
+ const float* __restrict__ inv_dy,
1500
+ const float* __restrict__ inv_dz,
1501
+ int z_low_mode,
1502
+ int z_high_mode,
1503
+ float* __restrict__ ex_real,
1504
+ float* __restrict__ ex_imag) {
1505
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
1506
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
1507
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
1508
+ if (i >= nx || j >= ny || k >= nz) {
1509
+ return;
1510
+ }
1511
+ const long long linear = offset3d(i, j, k, ny, nz);
1512
+ const ComplexValue d_y = bloch_backward_diff_axis1(
1513
+ hz_real, hz_imag, ny, ny - 1, nz, i, j, k, phase_cos_y, phase_sin_y, inv_dy);
1514
+ const float d_z_real = backward_diff_axis2(hy_real, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
1515
+ const float d_z_imag = backward_diff_axis2(hy_imag, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
1516
+ ex_real[linear] = ex_real[linear] * decay[linear] + curl_coeff[linear] * (d_y.real - d_z_real);
1517
+ ex_imag[linear] = ex_imag[linear] * decay[linear] + curl_coeff[linear] * (d_y.imag - d_z_imag);
1518
+ }
1519
+
1520
+ // Ey: Bloch backward diff of Hz along x, standard backward diff of Hx along z.
1521
+ __global__ void update_electric_ey_bloch_x_standard_z_kernel(
1522
+ unsigned int nx,
1523
+ unsigned int ny,
1524
+ unsigned int nz,
1525
+ const float* __restrict__ hx_real,
1526
+ const float* __restrict__ hx_imag,
1527
+ const float* __restrict__ hz_real,
1528
+ const float* __restrict__ hz_imag,
1529
+ const float* __restrict__ decay,
1530
+ const float* __restrict__ curl_coeff,
1531
+ float phase_cos_x,
1532
+ float phase_sin_x,
1533
+ const float* __restrict__ inv_dx,
1534
+ const float* __restrict__ inv_dz,
1535
+ int z_low_mode,
1536
+ int z_high_mode,
1537
+ float* __restrict__ ey_real,
1538
+ float* __restrict__ ey_imag) {
1539
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
1540
+ const unsigned int j = blockIdx.y * blockDim.y + threadIdx.y;
1541
+ const unsigned int i = blockIdx.z * blockDim.z + threadIdx.z;
1542
+ if (i >= nx || j >= ny || k >= nz) {
1543
+ return;
1544
+ }
1545
+ const long long linear = offset3d(i, j, k, ny, nz);
1546
+ const float d_z_real = backward_diff_axis2(hx_real, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
1547
+ const float d_z_imag = backward_diff_axis2(hx_imag, ny, nz - 1, i, j, k, z_low_mode, z_high_mode, inv_dz);
1548
+ const ComplexValue d_x = bloch_backward_diff_axis0(
1549
+ hz_real, hz_imag, nx, nx - 1, ny, nz, i, j, k, phase_cos_x, phase_sin_x, inv_dx);
1550
+ ey_real[linear] = ey_real[linear] * decay[linear] + curl_coeff[linear] * (d_z_real - d_x.real);
1551
+ ey_imag[linear] = ey_imag[linear] * decay[linear] + curl_coeff[linear] * (d_z_imag - d_x.imag);
1552
+ }
1553
+
1554
+ // CPML psi + kappa correction along z applied over a region narrowed along z,
1555
+ // for the Bloch+CPML mixed electric update. Region-local (x, y, lz) map to
1556
+ // global (offset_i + x, offset_j + y, offset_k + lz); psi/curl/field are
1557
+ // region-shaped, the CPML coefficient arrays are indexed by global z.
1558
+ __global__ void apply_electric_ex_cpml_z_correction_kernel(
1559
+ unsigned int sx,
1560
+ unsigned int sy,
1561
+ unsigned int sz,
1562
+ unsigned int hy_ny,
1563
+ unsigned int hy_nz,
1564
+ const float* __restrict__ hy,
1565
+ const float* __restrict__ curl,
1566
+ const float* __restrict__ inv_kappa_z,
1567
+ const float* __restrict__ b_z,
1568
+ const float* __restrict__ c_z,
1569
+ const float* __restrict__ inv_dz,
1570
+ int offset_i,
1571
+ int offset_j,
1572
+ int offset_k,
1573
+ int y_low_mode,
1574
+ int y_high_mode,
1575
+ unsigned int full_size_y,
1576
+ int local_z_start,
1577
+ int local_z_stop,
1578
+ float* __restrict__ psi_z,
1579
+ float* __restrict__ ex) {
1580
+ const unsigned int lz = blockIdx.x * blockDim.x + threadIdx.x;
1581
+ const unsigned int y = blockIdx.y * blockDim.y + threadIdx.y;
1582
+ const unsigned int x = blockIdx.z * blockDim.z + threadIdx.z;
1583
+ if (x >= sx || y >= sy || lz >= sz) {
1584
+ return;
1585
+ }
1586
+ if (static_cast<int>(lz) < local_z_start || static_cast<int>(lz) >= local_z_stop) {
1587
+ return;
1588
+ }
1589
+ const unsigned int gy = static_cast<unsigned int>(offset_j) + y;
1590
+ if (!cpml_correction_active(gy, full_size_y, y_low_mode, y_high_mode)) {
1591
+ return;
1592
+ }
1593
+ const unsigned int gx = static_cast<unsigned int>(offset_i) + x;
1594
+ const unsigned int gz = static_cast<unsigned int>(offset_k) + lz;
1595
+ const long long region_lin = offset3d(x, y, lz, sy, sz);
1596
+ const long long hy_cur = offset3d(gx, gy, gz, hy_ny, hy_nz);
1597
+ const long long hy_prev = offset3d(gx, gy, gz - 1, hy_ny, hy_nz);
1598
+ const float derivative = (hy[hy_cur] - hy[hy_prev]) * inv_dz[gz];
1599
+ const float psi_new = b_z[gz] * psi_z[region_lin] + c_z[gz] * derivative;
1600
+ psi_z[region_lin] = psi_new;
1601
+ const float correction = derivative * (inv_kappa_z[gz] - 1.0f) + psi_new;
1602
+ ex[region_lin] = ex[region_lin] - curl[region_lin] * correction;
1603
+ }
1604
+
1605
+ __global__ void apply_electric_ey_cpml_z_correction_kernel(
1606
+ unsigned int sx,
1607
+ unsigned int sy,
1608
+ unsigned int sz,
1609
+ unsigned int hx_ny,
1610
+ unsigned int hx_nz,
1611
+ const float* __restrict__ hx,
1612
+ const float* __restrict__ curl,
1613
+ const float* __restrict__ inv_kappa_z,
1614
+ const float* __restrict__ b_z,
1615
+ const float* __restrict__ c_z,
1616
+ const float* __restrict__ inv_dz,
1617
+ int offset_i,
1618
+ int offset_j,
1619
+ int offset_k,
1620
+ int x_low_mode,
1621
+ int x_high_mode,
1622
+ unsigned int full_size_x,
1623
+ int local_z_start,
1624
+ int local_z_stop,
1625
+ float* __restrict__ psi_z,
1626
+ float* __restrict__ ey) {
1627
+ const unsigned int lz = blockIdx.x * blockDim.x + threadIdx.x;
1628
+ const unsigned int y = blockIdx.y * blockDim.y + threadIdx.y;
1629
+ const unsigned int x = blockIdx.z * blockDim.z + threadIdx.z;
1630
+ if (x >= sx || y >= sy || lz >= sz) {
1631
+ return;
1632
+ }
1633
+ if (static_cast<int>(lz) < local_z_start || static_cast<int>(lz) >= local_z_stop) {
1634
+ return;
1635
+ }
1636
+ const unsigned int gx = static_cast<unsigned int>(offset_i) + x;
1637
+ if (!cpml_correction_active(gx, full_size_x, x_low_mode, x_high_mode)) {
1638
+ return;
1639
+ }
1640
+ const unsigned int gy = static_cast<unsigned int>(offset_j) + y;
1641
+ const unsigned int gz = static_cast<unsigned int>(offset_k) + lz;
1642
+ const long long region_lin = offset3d(x, y, lz, sy, sz);
1643
+ const long long hx_cur = offset3d(gx, gy, gz, hx_ny, hx_nz);
1644
+ const long long hx_prev = offset3d(gx, gy, gz - 1, hx_ny, hx_nz);
1645
+ const float derivative = (hx[hx_cur] - hx[hx_prev]) * inv_dz[gz];
1646
+ const float psi_new = b_z[gz] * psi_z[region_lin] + c_z[gz] * derivative;
1647
+ psi_z[region_lin] = psi_new;
1648
+ const float correction = derivative * (inv_kappa_z[gz] - 1.0f) + psi_new;
1649
+ ey[region_lin] = ey[region_lin] + curl[region_lin] * correction;
1650
+ }
1651
+
1652
+ // CPML psi + kappa correction along y (the PML axis) for the general
1653
+ // single-PML-axis Bloch+CPML electric update, where the two Bloch axes carry
1654
+ // the transverse phase in the base full-Bloch kernel. Ex: derivative of Hz
1655
+ // along y; tangent (skip) check is on the x/z Bloch axis that also touches Ex.
1656
+ // The region is narrowed along y (the psi memory axis); region-local
1657
+ // (x, ly, k) map to global (offset_i + x, offset_j + ly, offset_k + k).
1658
+ __global__ void apply_electric_ex_cpml_y_correction_kernel(
1659
+ unsigned int sx,
1660
+ unsigned int sy,
1661
+ unsigned int sz,
1662
+ unsigned int hz_ny,
1663
+ unsigned int hz_nz,
1664
+ const float* __restrict__ hz,
1665
+ const float* __restrict__ curl,
1666
+ const float* __restrict__ inv_kappa_y,
1667
+ const float* __restrict__ b_y,
1668
+ const float* __restrict__ c_y,
1669
+ const float* __restrict__ inv_dy,
1670
+ int offset_i,
1671
+ int offset_j,
1672
+ int offset_k,
1673
+ int z_low_mode,
1674
+ int z_high_mode,
1675
+ unsigned int full_size_z,
1676
+ int local_y_start,
1677
+ int local_y_stop,
1678
+ float* __restrict__ psi_y,
1679
+ float* __restrict__ ex) {
1680
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
1681
+ const unsigned int ly = blockIdx.y * blockDim.y + threadIdx.y;
1682
+ const unsigned int x = blockIdx.z * blockDim.z + threadIdx.z;
1683
+ if (x >= sx || ly >= sy || k >= sz) {
1684
+ return;
1685
+ }
1686
+ if (static_cast<int>(ly) < local_y_start || static_cast<int>(ly) >= local_y_stop) {
1687
+ return;
1688
+ }
1689
+ const unsigned int gz = static_cast<unsigned int>(offset_k) + k;
1690
+ if (!cpml_correction_active(gz, full_size_z, z_low_mode, z_high_mode)) {
1691
+ return;
1692
+ }
1693
+ const unsigned int gx = static_cast<unsigned int>(offset_i) + x;
1694
+ const unsigned int gy = static_cast<unsigned int>(offset_j) + ly;
1695
+ const long long region_lin = offset3d(x, ly, k, sy, sz);
1696
+ const long long hz_cur = offset3d(gx, gy, gz, hz_ny, hz_nz);
1697
+ const long long hz_prev = offset3d(gx, gy - 1, gz, hz_ny, hz_nz);
1698
+ const float derivative = (hz[hz_cur] - hz[hz_prev]) * inv_dy[gy];
1699
+ const float psi_new = b_y[gy] * psi_y[region_lin] + c_y[gy] * derivative;
1700
+ psi_y[region_lin] = psi_new;
1701
+ const float correction = derivative * (inv_kappa_y[gy] - 1.0f) + psi_new;
1702
+ ex[region_lin] = ex[region_lin] + curl[region_lin] * correction;
1703
+ }
1704
+
1705
+ // Ez: derivative of Hx along y (the PML axis); tangent check on x.
1706
+ __global__ void apply_electric_ez_cpml_y_correction_kernel(
1707
+ unsigned int sx,
1708
+ unsigned int sy,
1709
+ unsigned int sz,
1710
+ unsigned int hx_ny,
1711
+ unsigned int hx_nz,
1712
+ const float* __restrict__ hx,
1713
+ const float* __restrict__ curl,
1714
+ const float* __restrict__ inv_kappa_y,
1715
+ const float* __restrict__ b_y,
1716
+ const float* __restrict__ c_y,
1717
+ const float* __restrict__ inv_dy,
1718
+ int offset_i,
1719
+ int offset_j,
1720
+ int offset_k,
1721
+ int x_low_mode,
1722
+ int x_high_mode,
1723
+ unsigned int full_size_x,
1724
+ int local_y_start,
1725
+ int local_y_stop,
1726
+ float* __restrict__ psi_y,
1727
+ float* __restrict__ ez) {
1728
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
1729
+ const unsigned int ly = blockIdx.y * blockDim.y + threadIdx.y;
1730
+ const unsigned int x = blockIdx.z * blockDim.z + threadIdx.z;
1731
+ if (x >= sx || ly >= sy || k >= sz) {
1732
+ return;
1733
+ }
1734
+ if (static_cast<int>(ly) < local_y_start || static_cast<int>(ly) >= local_y_stop) {
1735
+ return;
1736
+ }
1737
+ const unsigned int gx = static_cast<unsigned int>(offset_i) + x;
1738
+ if (!cpml_correction_active(gx, full_size_x, x_low_mode, x_high_mode)) {
1739
+ return;
1740
+ }
1741
+ const unsigned int gy = static_cast<unsigned int>(offset_j) + ly;
1742
+ const unsigned int gz = static_cast<unsigned int>(offset_k) + k;
1743
+ const long long region_lin = offset3d(x, ly, k, sy, sz);
1744
+ const long long hx_cur = offset3d(gx, gy, gz, hx_ny, hx_nz);
1745
+ const long long hx_prev = offset3d(gx, gy - 1, gz, hx_ny, hx_nz);
1746
+ const float derivative = (hx[hx_cur] - hx[hx_prev]) * inv_dy[gy];
1747
+ const float psi_new = b_y[gy] * psi_y[region_lin] + c_y[gy] * derivative;
1748
+ psi_y[region_lin] = psi_new;
1749
+ const float correction = derivative * (inv_kappa_y[gy] - 1.0f) + psi_new;
1750
+ ez[region_lin] = ez[region_lin] - curl[region_lin] * correction;
1751
+ }
1752
+
1753
+ // CPML correction along x (the PML axis). Region narrowed along x; region-local
1754
+ // (lx, y, k) map to global (offset_i + lx, offset_j + y, offset_k + k).
1755
+ // Ey: derivative of Hz along x; tangent check on z.
1756
+ __global__ void apply_electric_ey_cpml_x_correction_kernel(
1757
+ unsigned int sx,
1758
+ unsigned int sy,
1759
+ unsigned int sz,
1760
+ unsigned int hz_ny,
1761
+ unsigned int hz_nz,
1762
+ const float* __restrict__ hz,
1763
+ const float* __restrict__ curl,
1764
+ const float* __restrict__ inv_kappa_x,
1765
+ const float* __restrict__ b_x,
1766
+ const float* __restrict__ c_x,
1767
+ const float* __restrict__ inv_dx,
1768
+ int offset_i,
1769
+ int offset_j,
1770
+ int offset_k,
1771
+ int z_low_mode,
1772
+ int z_high_mode,
1773
+ unsigned int full_size_z,
1774
+ int local_x_start,
1775
+ int local_x_stop,
1776
+ float* __restrict__ psi_x,
1777
+ float* __restrict__ ey) {
1778
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
1779
+ const unsigned int y = blockIdx.y * blockDim.y + threadIdx.y;
1780
+ const unsigned int lx = blockIdx.z * blockDim.z + threadIdx.z;
1781
+ if (lx >= sx || y >= sy || k >= sz) {
1782
+ return;
1783
+ }
1784
+ if (static_cast<int>(lx) < local_x_start || static_cast<int>(lx) >= local_x_stop) {
1785
+ return;
1786
+ }
1787
+ const unsigned int gz = static_cast<unsigned int>(offset_k) + k;
1788
+ if (!cpml_correction_active(gz, full_size_z, z_low_mode, z_high_mode)) {
1789
+ return;
1790
+ }
1791
+ const unsigned int gx = static_cast<unsigned int>(offset_i) + lx;
1792
+ const unsigned int gy = static_cast<unsigned int>(offset_j) + y;
1793
+ const long long region_lin = offset3d(lx, y, k, sy, sz);
1794
+ const long long hz_cur = offset3d(gx, gy, gz, hz_ny, hz_nz);
1795
+ const long long hz_prev = offset3d(gx - 1, gy, gz, hz_ny, hz_nz);
1796
+ const float derivative = (hz[hz_cur] - hz[hz_prev]) * inv_dx[gx];
1797
+ const float psi_new = b_x[gx] * psi_x[region_lin] + c_x[gx] * derivative;
1798
+ psi_x[region_lin] = psi_new;
1799
+ const float correction = derivative * (inv_kappa_x[gx] - 1.0f) + psi_new;
1800
+ ey[region_lin] = ey[region_lin] - curl[region_lin] * correction;
1801
+ }
1802
+
1803
+ // Ez: derivative of Hy along x (the PML axis); tangent check on y.
1804
+ __global__ void apply_electric_ez_cpml_x_correction_kernel(
1805
+ unsigned int sx,
1806
+ unsigned int sy,
1807
+ unsigned int sz,
1808
+ unsigned int hy_ny,
1809
+ unsigned int hy_nz,
1810
+ const float* __restrict__ hy,
1811
+ const float* __restrict__ curl,
1812
+ const float* __restrict__ inv_kappa_x,
1813
+ const float* __restrict__ b_x,
1814
+ const float* __restrict__ c_x,
1815
+ const float* __restrict__ inv_dx,
1816
+ int offset_i,
1817
+ int offset_j,
1818
+ int offset_k,
1819
+ int y_low_mode,
1820
+ int y_high_mode,
1821
+ unsigned int full_size_y,
1822
+ int local_x_start,
1823
+ int local_x_stop,
1824
+ float* __restrict__ psi_x,
1825
+ float* __restrict__ ez) {
1826
+ const unsigned int k = blockIdx.x * blockDim.x + threadIdx.x;
1827
+ const unsigned int y = blockIdx.y * blockDim.y + threadIdx.y;
1828
+ const unsigned int lx = blockIdx.z * blockDim.z + threadIdx.z;
1829
+ if (lx >= sx || y >= sy || k >= sz) {
1830
+ return;
1831
+ }
1832
+ if (static_cast<int>(lx) < local_x_start || static_cast<int>(lx) >= local_x_stop) {
1833
+ return;
1834
+ }
1835
+ const unsigned int gy = static_cast<unsigned int>(offset_j) + y;
1836
+ if (!cpml_correction_active(gy, full_size_y, y_low_mode, y_high_mode)) {
1837
+ return;
1838
+ }
1839
+ const unsigned int gx = static_cast<unsigned int>(offset_i) + lx;
1840
+ const unsigned int gz = static_cast<unsigned int>(offset_k) + k;
1841
+ const long long region_lin = offset3d(lx, y, k, sy, sz);
1842
+ const long long hy_cur = offset3d(gx, gy, gz, hy_ny, hy_nz);
1843
+ const long long hy_prev = offset3d(gx - 1, gy, gz, hy_ny, hy_nz);
1844
+ const float derivative = (hy[hy_cur] - hy[hy_prev]) * inv_dx[gx];
1845
+ const float psi_new = b_x[gx] * psi_x[region_lin] + c_x[gx] * derivative;
1846
+ psi_x[region_lin] = psi_new;
1847
+ const float correction = derivative * (inv_kappa_x[gx] - 1.0f) + psi_new;
1848
+ ez[region_lin] = ez[region_lin] + curl[region_lin] * correction;
1849
+ }
1850
+
1851
+ } // namespace
1852
+
1853
+ void advance_modulation_time_cuda(torch::stable::Tensor modulation_time, double dt) {
1854
+ check_float32_tensor(modulation_time, "modulation_time");
1855
+ check_contiguous_tensor(modulation_time, "modulation_time");
1856
+ STD_TORCH_CHECK(
1857
+ modulation_time.dim() == 1 && modulation_time.numel() == 2,
1858
+ "modulation_time must be a contiguous float32 CUDA tensor with two elements");
1859
+ const torch::stable::accelerator::DeviceGuard guard(modulation_time.get_device_index());
1860
+ advance_modulation_time_kernel<<<1, 1, 0, current_cuda_stream()>>>(
1861
+ static_cast<float>(dt), modulation_time.mutable_data_ptr<float>());
1862
+ WITWIN_CUDA_CHECK();
1863
+ }
1864
+
1865
+ void update_electric_ex_standard_cuda(
1866
+ torch::stable::Tensor ex,
1867
+ const torch::stable::Tensor& hy,
1868
+ const torch::stable::Tensor& hz,
1869
+ const torch::stable::Tensor& decay,
1870
+ const torch::stable::Tensor& curl,
1871
+ const torch::stable::Tensor& inv_dy,
1872
+ const torch::stable::Tensor& inv_dz,
1873
+ int64_t y_low_mode,
1874
+ int64_t y_high_mode,
1875
+ int64_t z_low_mode,
1876
+ int64_t z_high_mode) {
1877
+ check_electric_inputs(ex, hy, hz, decay, curl, "ex");
1878
+ check_rank3_shape(hy, "hy", ex.size(0), ex.size(1), ex.size(2) - 1);
1879
+ check_rank3_shape(hz, "hz", ex.size(0), ex.size(1) - 1, ex.size(2));
1880
+ check_spacing_vector(ex, inv_dy, 1, "inv_dy");
1881
+ check_spacing_vector(ex, inv_dz, 2, "inv_dz");
1882
+ torch::stable::accelerator::DeviceGuard guard(ex.get_device_index());
1883
+ const auto sizes = ex.sizes();
1884
+ const dim3 block = field_block3d();
1885
+ if (inactive_boundary_pair(y_low_mode, y_high_mode) && inactive_boundary_pair(z_low_mode, z_high_mode)) {
1886
+ if (sizes[1] > 2 && sizes[2] > 2) {
1887
+ update_electric_standard_interior_kernel<0><<<field_grid3d(sizes[0], sizes[1] - 2, sizes[2] - 2, block), block, 0, current_cuda_stream()>>>(
1888
+ static_cast<unsigned int>(sizes[0]),
1889
+ static_cast<unsigned int>(sizes[1]),
1890
+ static_cast<unsigned int>(sizes[2]),
1891
+ hy.mutable_data_ptr<float>(),
1892
+ hz.mutable_data_ptr<float>(),
1893
+ decay.mutable_data_ptr<float>(),
1894
+ curl.mutable_data_ptr<float>(),
1895
+ inv_dy.mutable_data_ptr<float>(),
1896
+ inv_dz.mutable_data_ptr<float>(),
1897
+ ex.mutable_data_ptr<float>());
1898
+ }
1899
+ } else {
1900
+ update_electric_standard_kernel<0><<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
1901
+ static_cast<unsigned int>(sizes[0]),
1902
+ static_cast<unsigned int>(sizes[1]),
1903
+ static_cast<unsigned int>(sizes[2]),
1904
+ hy.mutable_data_ptr<float>(),
1905
+ hz.mutable_data_ptr<float>(),
1906
+ decay.mutable_data_ptr<float>(),
1907
+ curl.mutable_data_ptr<float>(),
1908
+ inv_dy.mutable_data_ptr<float>(),
1909
+ inv_dz.mutable_data_ptr<float>(),
1910
+ static_cast<int>(y_low_mode),
1911
+ static_cast<int>(y_high_mode),
1912
+ static_cast<int>(z_low_mode),
1913
+ static_cast<int>(z_high_mode),
1914
+ ex.mutable_data_ptr<float>());
1915
+ }
1916
+ WITWIN_CUDA_CHECK();
1917
+ }
1918
+
1919
+ void update_electric_ey_standard_cuda(
1920
+ torch::stable::Tensor ey,
1921
+ const torch::stable::Tensor& hx,
1922
+ const torch::stable::Tensor& hz,
1923
+ const torch::stable::Tensor& decay,
1924
+ const torch::stable::Tensor& curl,
1925
+ const torch::stable::Tensor& inv_dx,
1926
+ const torch::stable::Tensor& inv_dz,
1927
+ int64_t x_low_mode,
1928
+ int64_t x_high_mode,
1929
+ int64_t z_low_mode,
1930
+ int64_t z_high_mode) {
1931
+ check_electric_inputs(ey, hx, hz, decay, curl, "ey");
1932
+ check_rank3_shape(hx, "hx", ey.size(0), ey.size(1), ey.size(2) - 1);
1933
+ check_rank3_shape(hz, "hz", ey.size(0) - 1, ey.size(1), ey.size(2));
1934
+ check_spacing_vector(ey, inv_dx, 0, "inv_dx");
1935
+ check_spacing_vector(ey, inv_dz, 2, "inv_dz");
1936
+ torch::stable::accelerator::DeviceGuard guard(ey.get_device_index());
1937
+ const auto sizes = ey.sizes();
1938
+ const dim3 block = field_block3d();
1939
+ if (inactive_boundary_pair(x_low_mode, x_high_mode) && inactive_boundary_pair(z_low_mode, z_high_mode)) {
1940
+ if (sizes[0] > 2 && sizes[2] > 2) {
1941
+ update_electric_standard_interior_kernel<1><<<field_grid3d(sizes[0] - 2, sizes[1], sizes[2] - 2, block), block, 0, current_cuda_stream()>>>(
1942
+ static_cast<unsigned int>(sizes[0]),
1943
+ static_cast<unsigned int>(sizes[1]),
1944
+ static_cast<unsigned int>(sizes[2]),
1945
+ hx.mutable_data_ptr<float>(),
1946
+ hz.mutable_data_ptr<float>(),
1947
+ decay.mutable_data_ptr<float>(),
1948
+ curl.mutable_data_ptr<float>(),
1949
+ inv_dx.mutable_data_ptr<float>(),
1950
+ inv_dz.mutable_data_ptr<float>(),
1951
+ ey.mutable_data_ptr<float>());
1952
+ }
1953
+ } else {
1954
+ update_electric_standard_kernel<1><<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
1955
+ static_cast<unsigned int>(sizes[0]),
1956
+ static_cast<unsigned int>(sizes[1]),
1957
+ static_cast<unsigned int>(sizes[2]),
1958
+ hx.mutable_data_ptr<float>(),
1959
+ hz.mutable_data_ptr<float>(),
1960
+ decay.mutable_data_ptr<float>(),
1961
+ curl.mutable_data_ptr<float>(),
1962
+ inv_dx.mutable_data_ptr<float>(),
1963
+ inv_dz.mutable_data_ptr<float>(),
1964
+ static_cast<int>(x_low_mode),
1965
+ static_cast<int>(x_high_mode),
1966
+ static_cast<int>(z_low_mode),
1967
+ static_cast<int>(z_high_mode),
1968
+ ey.mutable_data_ptr<float>());
1969
+ }
1970
+ WITWIN_CUDA_CHECK();
1971
+ }
1972
+
1973
+ void update_electric_ez_standard_cuda(
1974
+ torch::stable::Tensor ez,
1975
+ const torch::stable::Tensor& hx,
1976
+ const torch::stable::Tensor& hy,
1977
+ const torch::stable::Tensor& decay,
1978
+ const torch::stable::Tensor& curl,
1979
+ const torch::stable::Tensor& inv_dx,
1980
+ const torch::stable::Tensor& inv_dy,
1981
+ int64_t x_low_mode,
1982
+ int64_t x_high_mode,
1983
+ int64_t y_low_mode,
1984
+ int64_t y_high_mode) {
1985
+ check_electric_inputs(ez, hx, hy, decay, curl, "ez");
1986
+ check_rank3_shape(hx, "hx", ez.size(0), ez.size(1) - 1, ez.size(2));
1987
+ check_rank3_shape(hy, "hy", ez.size(0) - 1, ez.size(1), ez.size(2));
1988
+ check_spacing_vector(ez, inv_dx, 0, "inv_dx");
1989
+ check_spacing_vector(ez, inv_dy, 1, "inv_dy");
1990
+ torch::stable::accelerator::DeviceGuard guard(ez.get_device_index());
1991
+ const auto sizes = ez.sizes();
1992
+ const dim3 block = field_block3d();
1993
+ if (inactive_boundary_pair(x_low_mode, x_high_mode) && inactive_boundary_pair(y_low_mode, y_high_mode)) {
1994
+ if (sizes[0] > 2 && sizes[1] > 2) {
1995
+ update_electric_standard_interior_kernel<2><<<field_grid3d(sizes[0] - 2, sizes[1] - 2, sizes[2], block), block, 0, current_cuda_stream()>>>(
1996
+ static_cast<unsigned int>(sizes[0]),
1997
+ static_cast<unsigned int>(sizes[1]),
1998
+ static_cast<unsigned int>(sizes[2]),
1999
+ hx.mutable_data_ptr<float>(),
2000
+ hy.mutable_data_ptr<float>(),
2001
+ decay.mutable_data_ptr<float>(),
2002
+ curl.mutable_data_ptr<float>(),
2003
+ inv_dx.mutable_data_ptr<float>(),
2004
+ inv_dy.mutable_data_ptr<float>(),
2005
+ ez.mutable_data_ptr<float>());
2006
+ }
2007
+ } else {
2008
+ update_electric_standard_kernel<2><<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2009
+ static_cast<unsigned int>(sizes[0]),
2010
+ static_cast<unsigned int>(sizes[1]),
2011
+ static_cast<unsigned int>(sizes[2]),
2012
+ hx.mutable_data_ptr<float>(),
2013
+ hy.mutable_data_ptr<float>(),
2014
+ decay.mutable_data_ptr<float>(),
2015
+ curl.mutable_data_ptr<float>(),
2016
+ inv_dx.mutable_data_ptr<float>(),
2017
+ inv_dy.mutable_data_ptr<float>(),
2018
+ static_cast<int>(x_low_mode),
2019
+ static_cast<int>(x_high_mode),
2020
+ static_cast<int>(y_low_mode),
2021
+ static_cast<int>(y_high_mode),
2022
+ ez.mutable_data_ptr<float>());
2023
+ }
2024
+ WITWIN_CUDA_CHECK();
2025
+ }
2026
+
2027
+ void update_electric_ex_bloch_cuda(
2028
+ torch::stable::Tensor ex_real,
2029
+ torch::stable::Tensor ex_imag,
2030
+ const torch::stable::Tensor& hy_real,
2031
+ const torch::stable::Tensor& hy_imag,
2032
+ const torch::stable::Tensor& hz_real,
2033
+ const torch::stable::Tensor& hz_imag,
2034
+ const torch::stable::Tensor& decay,
2035
+ const torch::stable::Tensor& curl,
2036
+ double phase_cos_y,
2037
+ double phase_sin_y,
2038
+ double phase_cos_z,
2039
+ double phase_sin_z,
2040
+ const torch::stable::Tensor& inv_dy,
2041
+ const torch::stable::Tensor& inv_dz) {
2042
+ check_electric_bloch_inputs(ex_real, ex_imag, hy_real, hy_imag, hz_real, hz_imag, decay, curl, "ex_real");
2043
+ check_rank3_shape(hy_real, "hy_real", ex_real.size(0), ex_real.size(1), ex_real.size(2) - 1);
2044
+ check_rank3_shape(hy_imag, "hy_imag", ex_real.size(0), ex_real.size(1), ex_real.size(2) - 1);
2045
+ check_rank3_shape(hz_real, "hz_real", ex_real.size(0), ex_real.size(1) - 1, ex_real.size(2));
2046
+ check_rank3_shape(hz_imag, "hz_imag", ex_real.size(0), ex_real.size(1) - 1, ex_real.size(2));
2047
+ check_spacing_vector(ex_real, inv_dy, 1, "inv_dy");
2048
+ check_spacing_vector(ex_real, inv_dz, 2, "inv_dz");
2049
+ torch::stable::accelerator::DeviceGuard guard(ex_real.get_device_index());
2050
+ const auto sizes = ex_real.sizes();
2051
+ const dim3 block = field_block3d();
2052
+ update_electric_ex_bloch_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2053
+ static_cast<unsigned int>(sizes[0]),
2054
+ static_cast<unsigned int>(sizes[1]),
2055
+ static_cast<unsigned int>(sizes[2]),
2056
+ hy_real.mutable_data_ptr<float>(),
2057
+ hy_imag.mutable_data_ptr<float>(),
2058
+ hz_real.mutable_data_ptr<float>(),
2059
+ hz_imag.mutable_data_ptr<float>(),
2060
+ decay.mutable_data_ptr<float>(),
2061
+ curl.mutable_data_ptr<float>(),
2062
+ static_cast<float>(phase_cos_y),
2063
+ static_cast<float>(phase_sin_y),
2064
+ static_cast<float>(phase_cos_z),
2065
+ static_cast<float>(phase_sin_z),
2066
+ inv_dy.mutable_data_ptr<float>(),
2067
+ inv_dz.mutable_data_ptr<float>(),
2068
+ ex_real.mutable_data_ptr<float>(),
2069
+ ex_imag.mutable_data_ptr<float>());
2070
+ WITWIN_CUDA_CHECK();
2071
+ }
2072
+
2073
+ void update_electric_ey_bloch_cuda(
2074
+ torch::stable::Tensor ey_real,
2075
+ torch::stable::Tensor ey_imag,
2076
+ const torch::stable::Tensor& hx_real,
2077
+ const torch::stable::Tensor& hx_imag,
2078
+ const torch::stable::Tensor& hz_real,
2079
+ const torch::stable::Tensor& hz_imag,
2080
+ const torch::stable::Tensor& decay,
2081
+ const torch::stable::Tensor& curl,
2082
+ double phase_cos_x,
2083
+ double phase_sin_x,
2084
+ double phase_cos_z,
2085
+ double phase_sin_z,
2086
+ const torch::stable::Tensor& inv_dx,
2087
+ const torch::stable::Tensor& inv_dz) {
2088
+ check_electric_bloch_inputs(ey_real, ey_imag, hx_real, hx_imag, hz_real, hz_imag, decay, curl, "ey_real");
2089
+ check_rank3_shape(hx_real, "hx_real", ey_real.size(0), ey_real.size(1), ey_real.size(2) - 1);
2090
+ check_rank3_shape(hx_imag, "hx_imag", ey_real.size(0), ey_real.size(1), ey_real.size(2) - 1);
2091
+ check_rank3_shape(hz_real, "hz_real", ey_real.size(0) - 1, ey_real.size(1), ey_real.size(2));
2092
+ check_rank3_shape(hz_imag, "hz_imag", ey_real.size(0) - 1, ey_real.size(1), ey_real.size(2));
2093
+ check_spacing_vector(ey_real, inv_dx, 0, "inv_dx");
2094
+ check_spacing_vector(ey_real, inv_dz, 2, "inv_dz");
2095
+ torch::stable::accelerator::DeviceGuard guard(ey_real.get_device_index());
2096
+ const auto sizes = ey_real.sizes();
2097
+ const dim3 block = field_block3d();
2098
+ update_electric_ey_bloch_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2099
+ static_cast<unsigned int>(sizes[0]),
2100
+ static_cast<unsigned int>(sizes[1]),
2101
+ static_cast<unsigned int>(sizes[2]),
2102
+ hx_real.mutable_data_ptr<float>(),
2103
+ hx_imag.mutable_data_ptr<float>(),
2104
+ hz_real.mutable_data_ptr<float>(),
2105
+ hz_imag.mutable_data_ptr<float>(),
2106
+ decay.mutable_data_ptr<float>(),
2107
+ curl.mutable_data_ptr<float>(),
2108
+ static_cast<float>(phase_cos_x),
2109
+ static_cast<float>(phase_sin_x),
2110
+ static_cast<float>(phase_cos_z),
2111
+ static_cast<float>(phase_sin_z),
2112
+ inv_dx.mutable_data_ptr<float>(),
2113
+ inv_dz.mutable_data_ptr<float>(),
2114
+ ey_real.mutable_data_ptr<float>(),
2115
+ ey_imag.mutable_data_ptr<float>());
2116
+ WITWIN_CUDA_CHECK();
2117
+ }
2118
+
2119
+ void update_electric_ez_bloch_cuda(
2120
+ torch::stable::Tensor ez_real,
2121
+ torch::stable::Tensor ez_imag,
2122
+ const torch::stable::Tensor& hx_real,
2123
+ const torch::stable::Tensor& hx_imag,
2124
+ const torch::stable::Tensor& hy_real,
2125
+ const torch::stable::Tensor& hy_imag,
2126
+ const torch::stable::Tensor& decay,
2127
+ const torch::stable::Tensor& curl,
2128
+ double phase_cos_x,
2129
+ double phase_sin_x,
2130
+ double phase_cos_y,
2131
+ double phase_sin_y,
2132
+ const torch::stable::Tensor& inv_dx,
2133
+ const torch::stable::Tensor& inv_dy) {
2134
+ check_electric_bloch_inputs(ez_real, ez_imag, hx_real, hx_imag, hy_real, hy_imag, decay, curl, "ez_real");
2135
+ check_rank3_shape(hx_real, "hx_real", ez_real.size(0), ez_real.size(1) - 1, ez_real.size(2));
2136
+ check_rank3_shape(hx_imag, "hx_imag", ez_real.size(0), ez_real.size(1) - 1, ez_real.size(2));
2137
+ check_rank3_shape(hy_real, "hy_real", ez_real.size(0) - 1, ez_real.size(1), ez_real.size(2));
2138
+ check_rank3_shape(hy_imag, "hy_imag", ez_real.size(0) - 1, ez_real.size(1), ez_real.size(2));
2139
+ check_spacing_vector(ez_real, inv_dx, 0, "inv_dx");
2140
+ check_spacing_vector(ez_real, inv_dy, 1, "inv_dy");
2141
+ torch::stable::accelerator::DeviceGuard guard(ez_real.get_device_index());
2142
+ const auto sizes = ez_real.sizes();
2143
+ const dim3 block = field_block3d();
2144
+ update_electric_ez_bloch_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2145
+ static_cast<unsigned int>(sizes[0]),
2146
+ static_cast<unsigned int>(sizes[1]),
2147
+ static_cast<unsigned int>(sizes[2]),
2148
+ hx_real.mutable_data_ptr<float>(),
2149
+ hx_imag.mutable_data_ptr<float>(),
2150
+ hy_real.mutable_data_ptr<float>(),
2151
+ hy_imag.mutable_data_ptr<float>(),
2152
+ decay.mutable_data_ptr<float>(),
2153
+ curl.mutable_data_ptr<float>(),
2154
+ static_cast<float>(phase_cos_x),
2155
+ static_cast<float>(phase_sin_x),
2156
+ static_cast<float>(phase_cos_y),
2157
+ static_cast<float>(phase_sin_y),
2158
+ inv_dx.mutable_data_ptr<float>(),
2159
+ inv_dy.mutable_data_ptr<float>(),
2160
+ ez_real.mutable_data_ptr<float>(),
2161
+ ez_imag.mutable_data_ptr<float>());
2162
+ WITWIN_CUDA_CHECK();
2163
+ }
2164
+
2165
+ void update_electric_ex_cpml_cuda(
2166
+ torch::stable::Tensor ex,
2167
+ const torch::stable::Tensor& hy,
2168
+ const torch::stable::Tensor& hz,
2169
+ const torch::stable::Tensor& decay,
2170
+ const torch::stable::Tensor& curl,
2171
+ torch::stable::Tensor psi_y,
2172
+ torch::stable::Tensor psi_z,
2173
+ const torch::stable::Tensor& inv_kappa_y,
2174
+ const torch::stable::Tensor& b_y,
2175
+ const torch::stable::Tensor& c_y,
2176
+ const torch::stable::Tensor& inv_kappa_z,
2177
+ const torch::stable::Tensor& b_z,
2178
+ const torch::stable::Tensor& c_z,
2179
+ const torch::stable::Tensor& inv_dy,
2180
+ const torch::stable::Tensor& inv_dz,
2181
+ int64_t y_low_mode,
2182
+ int64_t y_high_mode,
2183
+ int64_t z_low_mode,
2184
+ int64_t z_high_mode) {
2185
+ check_electric_cpml_inputs(
2186
+ ex, hy, hz, decay, curl, psi_y, psi_z, inv_kappa_y, b_y, c_y, inv_kappa_z, b_z, c_z, 1, 2, "ex");
2187
+ check_rank3_shape(hy, "hy", ex.size(0), ex.size(1), ex.size(2) - 1);
2188
+ check_rank3_shape(hz, "hz", ex.size(0), ex.size(1) - 1, ex.size(2));
2189
+ check_spacing_vector(ex, inv_dy, 1, "inv_dy");
2190
+ check_spacing_vector(ex, inv_dz, 2, "inv_dz");
2191
+ torch::stable::accelerator::DeviceGuard guard(ex.get_device_index());
2192
+ const auto sizes = ex.sizes();
2193
+ const dim3 block = field_block3d();
2194
+ if (inactive_boundary_pair(y_low_mode, y_high_mode) && inactive_boundary_pair(z_low_mode, z_high_mode)) {
2195
+ if (sizes[1] > 2 && sizes[2] > 2) {
2196
+ update_electric_cpml_interior_kernel<0><<<field_grid3d(sizes[0], sizes[1] - 2, sizes[2] - 2, block), block, 0, current_cuda_stream()>>>(
2197
+ static_cast<unsigned int>(sizes[0]),
2198
+ static_cast<unsigned int>(sizes[1]),
2199
+ static_cast<unsigned int>(sizes[2]),
2200
+ hy.mutable_data_ptr<float>(),
2201
+ hz.mutable_data_ptr<float>(),
2202
+ decay.mutable_data_ptr<float>(),
2203
+ curl.mutable_data_ptr<float>(),
2204
+ inv_kappa_y.mutable_data_ptr<float>(),
2205
+ b_y.mutable_data_ptr<float>(),
2206
+ c_y.mutable_data_ptr<float>(),
2207
+ inv_kappa_z.mutable_data_ptr<float>(),
2208
+ b_z.mutable_data_ptr<float>(),
2209
+ c_z.mutable_data_ptr<float>(),
2210
+ inv_dy.mutable_data_ptr<float>(),
2211
+ inv_dz.mutable_data_ptr<float>(),
2212
+ psi_y.mutable_data_ptr<float>(),
2213
+ psi_z.mutable_data_ptr<float>(),
2214
+ ex.mutable_data_ptr<float>());
2215
+ }
2216
+ } else {
2217
+ update_electric_cpml_kernel<0><<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2218
+ static_cast<unsigned int>(sizes[0]),
2219
+ static_cast<unsigned int>(sizes[1]),
2220
+ static_cast<unsigned int>(sizes[2]),
2221
+ hy.mutable_data_ptr<float>(),
2222
+ hz.mutable_data_ptr<float>(),
2223
+ decay.mutable_data_ptr<float>(),
2224
+ curl.mutable_data_ptr<float>(),
2225
+ inv_kappa_y.mutable_data_ptr<float>(),
2226
+ b_y.mutable_data_ptr<float>(),
2227
+ c_y.mutable_data_ptr<float>(),
2228
+ inv_kappa_z.mutable_data_ptr<float>(),
2229
+ b_z.mutable_data_ptr<float>(),
2230
+ c_z.mutable_data_ptr<float>(),
2231
+ inv_dy.mutable_data_ptr<float>(),
2232
+ inv_dz.mutable_data_ptr<float>(),
2233
+ static_cast<int>(y_low_mode),
2234
+ static_cast<int>(y_high_mode),
2235
+ static_cast<int>(z_low_mode),
2236
+ static_cast<int>(z_high_mode),
2237
+ psi_y.mutable_data_ptr<float>(),
2238
+ psi_z.mutable_data_ptr<float>(),
2239
+ ex.mutable_data_ptr<float>());
2240
+ }
2241
+ WITWIN_CUDA_CHECK();
2242
+ }
2243
+
2244
+ void update_electric_ey_cpml_cuda(
2245
+ torch::stable::Tensor ey,
2246
+ const torch::stable::Tensor& hx,
2247
+ const torch::stable::Tensor& hz,
2248
+ const torch::stable::Tensor& decay,
2249
+ const torch::stable::Tensor& curl,
2250
+ torch::stable::Tensor psi_x,
2251
+ torch::stable::Tensor psi_z,
2252
+ const torch::stable::Tensor& inv_kappa_x,
2253
+ const torch::stable::Tensor& b_x,
2254
+ const torch::stable::Tensor& c_x,
2255
+ const torch::stable::Tensor& inv_kappa_z,
2256
+ const torch::stable::Tensor& b_z,
2257
+ const torch::stable::Tensor& c_z,
2258
+ const torch::stable::Tensor& inv_dx,
2259
+ const torch::stable::Tensor& inv_dz,
2260
+ int64_t x_low_mode,
2261
+ int64_t x_high_mode,
2262
+ int64_t z_low_mode,
2263
+ int64_t z_high_mode) {
2264
+ check_electric_cpml_inputs(
2265
+ ey, hx, hz, decay, curl, psi_x, psi_z, inv_kappa_x, b_x, c_x, inv_kappa_z, b_z, c_z, 0, 2, "ey");
2266
+ check_rank3_shape(hx, "hx", ey.size(0), ey.size(1), ey.size(2) - 1);
2267
+ check_rank3_shape(hz, "hz", ey.size(0) - 1, ey.size(1), ey.size(2));
2268
+ check_spacing_vector(ey, inv_dx, 0, "inv_dx");
2269
+ check_spacing_vector(ey, inv_dz, 2, "inv_dz");
2270
+ torch::stable::accelerator::DeviceGuard guard(ey.get_device_index());
2271
+ const auto sizes = ey.sizes();
2272
+ const dim3 block = field_block3d();
2273
+ if (inactive_boundary_pair(x_low_mode, x_high_mode) && inactive_boundary_pair(z_low_mode, z_high_mode)) {
2274
+ if (sizes[0] > 2 && sizes[2] > 2) {
2275
+ update_electric_cpml_interior_kernel<1><<<field_grid3d(sizes[0] - 2, sizes[1], sizes[2] - 2, block), block, 0, current_cuda_stream()>>>(
2276
+ static_cast<unsigned int>(sizes[0]),
2277
+ static_cast<unsigned int>(sizes[1]),
2278
+ static_cast<unsigned int>(sizes[2]),
2279
+ hx.mutable_data_ptr<float>(),
2280
+ hz.mutable_data_ptr<float>(),
2281
+ decay.mutable_data_ptr<float>(),
2282
+ curl.mutable_data_ptr<float>(),
2283
+ inv_kappa_x.mutable_data_ptr<float>(),
2284
+ b_x.mutable_data_ptr<float>(),
2285
+ c_x.mutable_data_ptr<float>(),
2286
+ inv_kappa_z.mutable_data_ptr<float>(),
2287
+ b_z.mutable_data_ptr<float>(),
2288
+ c_z.mutable_data_ptr<float>(),
2289
+ inv_dx.mutable_data_ptr<float>(),
2290
+ inv_dz.mutable_data_ptr<float>(),
2291
+ psi_x.mutable_data_ptr<float>(),
2292
+ psi_z.mutable_data_ptr<float>(),
2293
+ ey.mutable_data_ptr<float>());
2294
+ }
2295
+ } else {
2296
+ update_electric_cpml_kernel<1><<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2297
+ static_cast<unsigned int>(sizes[0]),
2298
+ static_cast<unsigned int>(sizes[1]),
2299
+ static_cast<unsigned int>(sizes[2]),
2300
+ hx.mutable_data_ptr<float>(),
2301
+ hz.mutable_data_ptr<float>(),
2302
+ decay.mutable_data_ptr<float>(),
2303
+ curl.mutable_data_ptr<float>(),
2304
+ inv_kappa_x.mutable_data_ptr<float>(),
2305
+ b_x.mutable_data_ptr<float>(),
2306
+ c_x.mutable_data_ptr<float>(),
2307
+ inv_kappa_z.mutable_data_ptr<float>(),
2308
+ b_z.mutable_data_ptr<float>(),
2309
+ c_z.mutable_data_ptr<float>(),
2310
+ inv_dx.mutable_data_ptr<float>(),
2311
+ inv_dz.mutable_data_ptr<float>(),
2312
+ static_cast<int>(x_low_mode),
2313
+ static_cast<int>(x_high_mode),
2314
+ static_cast<int>(z_low_mode),
2315
+ static_cast<int>(z_high_mode),
2316
+ psi_x.mutable_data_ptr<float>(),
2317
+ psi_z.mutable_data_ptr<float>(),
2318
+ ey.mutable_data_ptr<float>());
2319
+ }
2320
+ WITWIN_CUDA_CHECK();
2321
+ }
2322
+
2323
+ void update_electric_ez_cpml_cuda(
2324
+ torch::stable::Tensor ez,
2325
+ const torch::stable::Tensor& hx,
2326
+ const torch::stable::Tensor& hy,
2327
+ const torch::stable::Tensor& decay,
2328
+ const torch::stable::Tensor& curl,
2329
+ torch::stable::Tensor psi_x,
2330
+ torch::stable::Tensor psi_y,
2331
+ const torch::stable::Tensor& inv_kappa_x,
2332
+ const torch::stable::Tensor& b_x,
2333
+ const torch::stable::Tensor& c_x,
2334
+ const torch::stable::Tensor& inv_kappa_y,
2335
+ const torch::stable::Tensor& b_y,
2336
+ const torch::stable::Tensor& c_y,
2337
+ const torch::stable::Tensor& inv_dx,
2338
+ const torch::stable::Tensor& inv_dy,
2339
+ int64_t x_low_mode,
2340
+ int64_t x_high_mode,
2341
+ int64_t y_low_mode,
2342
+ int64_t y_high_mode) {
2343
+ check_electric_cpml_inputs(
2344
+ ez, hx, hy, decay, curl, psi_x, psi_y, inv_kappa_x, b_x, c_x, inv_kappa_y, b_y, c_y, 0, 1, "ez");
2345
+ check_rank3_shape(hx, "hx", ez.size(0), ez.size(1) - 1, ez.size(2));
2346
+ check_rank3_shape(hy, "hy", ez.size(0) - 1, ez.size(1), ez.size(2));
2347
+ check_spacing_vector(ez, inv_dx, 0, "inv_dx");
2348
+ check_spacing_vector(ez, inv_dy, 1, "inv_dy");
2349
+ torch::stable::accelerator::DeviceGuard guard(ez.get_device_index());
2350
+ const auto sizes = ez.sizes();
2351
+ const dim3 block = field_block3d();
2352
+ if (inactive_boundary_pair(x_low_mode, x_high_mode) && inactive_boundary_pair(y_low_mode, y_high_mode)) {
2353
+ if (sizes[0] > 2 && sizes[1] > 2) {
2354
+ update_electric_cpml_interior_kernel<2><<<field_grid3d(sizes[0] - 2, sizes[1] - 2, sizes[2], block), block, 0, current_cuda_stream()>>>(
2355
+ static_cast<unsigned int>(sizes[0]),
2356
+ static_cast<unsigned int>(sizes[1]),
2357
+ static_cast<unsigned int>(sizes[2]),
2358
+ hx.mutable_data_ptr<float>(),
2359
+ hy.mutable_data_ptr<float>(),
2360
+ decay.mutable_data_ptr<float>(),
2361
+ curl.mutable_data_ptr<float>(),
2362
+ inv_kappa_x.mutable_data_ptr<float>(),
2363
+ b_x.mutable_data_ptr<float>(),
2364
+ c_x.mutable_data_ptr<float>(),
2365
+ inv_kappa_y.mutable_data_ptr<float>(),
2366
+ b_y.mutable_data_ptr<float>(),
2367
+ c_y.mutable_data_ptr<float>(),
2368
+ inv_dx.mutable_data_ptr<float>(),
2369
+ inv_dy.mutable_data_ptr<float>(),
2370
+ psi_x.mutable_data_ptr<float>(),
2371
+ psi_y.mutable_data_ptr<float>(),
2372
+ ez.mutable_data_ptr<float>());
2373
+ }
2374
+ } else {
2375
+ update_electric_cpml_kernel<2><<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2376
+ static_cast<unsigned int>(sizes[0]),
2377
+ static_cast<unsigned int>(sizes[1]),
2378
+ static_cast<unsigned int>(sizes[2]),
2379
+ hx.mutable_data_ptr<float>(),
2380
+ hy.mutable_data_ptr<float>(),
2381
+ decay.mutable_data_ptr<float>(),
2382
+ curl.mutable_data_ptr<float>(),
2383
+ inv_kappa_x.mutable_data_ptr<float>(),
2384
+ b_x.mutable_data_ptr<float>(),
2385
+ c_x.mutable_data_ptr<float>(),
2386
+ inv_kappa_y.mutable_data_ptr<float>(),
2387
+ b_y.mutable_data_ptr<float>(),
2388
+ c_y.mutable_data_ptr<float>(),
2389
+ inv_dx.mutable_data_ptr<float>(),
2390
+ inv_dy.mutable_data_ptr<float>(),
2391
+ static_cast<int>(x_low_mode),
2392
+ static_cast<int>(x_high_mode),
2393
+ static_cast<int>(y_low_mode),
2394
+ static_cast<int>(y_high_mode),
2395
+ psi_x.mutable_data_ptr<float>(),
2396
+ psi_y.mutable_data_ptr<float>(),
2397
+ ez.mutable_data_ptr<float>());
2398
+ }
2399
+ WITWIN_CUDA_CHECK();
2400
+ }
2401
+
2402
+ void update_electric_ex_cpml_compressed_cuda(
2403
+ torch::stable::Tensor ex,
2404
+ const torch::stable::Tensor& hy,
2405
+ const torch::stable::Tensor& hz,
2406
+ const torch::stable::Tensor& decay,
2407
+ const torch::stable::Tensor& curl,
2408
+ torch::stable::Tensor psi_y,
2409
+ torch::stable::Tensor psi_z,
2410
+ const torch::stable::Tensor& inv_kappa_y,
2411
+ const torch::stable::Tensor& b_y,
2412
+ const torch::stable::Tensor& c_y,
2413
+ const torch::stable::Tensor& inv_kappa_z,
2414
+ const torch::stable::Tensor& b_z,
2415
+ const torch::stable::Tensor& c_z,
2416
+ const torch::stable::Tensor& inv_dy,
2417
+ const torch::stable::Tensor& inv_dz,
2418
+ int64_t y_low_mode,
2419
+ int64_t y_high_mode,
2420
+ int64_t z_low_mode,
2421
+ int64_t z_high_mode,
2422
+ int64_t y_low_length,
2423
+ int64_t y_high_start,
2424
+ int64_t y_high_length,
2425
+ int64_t z_low_length,
2426
+ int64_t z_high_start,
2427
+ int64_t z_high_length,
2428
+ std::optional<double> uniform_decay,
2429
+ std::optional<double> uniform_curl) {
2430
+ check_electric_cpml_compressed_inputs(
2431
+ ex, hy, hz, decay, curl, psi_y, psi_z, inv_kappa_y, b_y, c_y, inv_kappa_z, b_z, c_z,
2432
+ 1, 2, y_low_length, y_high_length, z_low_length, z_high_length, "ex");
2433
+ check_rank3_shape(hy, "hy", ex.size(0), ex.size(1), ex.size(2) - 1);
2434
+ check_rank3_shape(hz, "hz", ex.size(0), ex.size(1) - 1, ex.size(2));
2435
+ check_spacing_vector(ex, inv_dy, 1, "inv_dy");
2436
+ check_spacing_vector(ex, inv_dz, 2, "inv_dz");
2437
+ torch::stable::accelerator::DeviceGuard guard(ex.get_device_index());
2438
+ const auto sizes = ex.sizes();
2439
+ const dim3 block = field_block3d();
2440
+ dispatch_uniform_coefficients(uniform_decay.has_value(), uniform_curl.has_value(), [&](auto u_decay, auto u_curl) {
2441
+ update_electric_ex_cpml_compressed_kernel<decltype(u_decay)::value, decltype(u_curl)::value><<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2442
+ static_cast<unsigned int>(sizes[0]),
2443
+ static_cast<unsigned int>(sizes[1]),
2444
+ static_cast<unsigned int>(sizes[2]),
2445
+ hy.mutable_data_ptr<float>(),
2446
+ hz.mutable_data_ptr<float>(),
2447
+ decay.mutable_data_ptr<float>(),
2448
+ curl.mutable_data_ptr<float>(),
2449
+ static_cast<float>(uniform_decay.value_or(0.0)),
2450
+ static_cast<float>(uniform_curl.value_or(0.0)),
2451
+ inv_kappa_y.mutable_data_ptr<float>(),
2452
+ b_y.mutable_data_ptr<float>(),
2453
+ c_y.mutable_data_ptr<float>(),
2454
+ inv_kappa_z.mutable_data_ptr<float>(),
2455
+ b_z.mutable_data_ptr<float>(),
2456
+ c_z.mutable_data_ptr<float>(),
2457
+ inv_dy.mutable_data_ptr<float>(),
2458
+ inv_dz.mutable_data_ptr<float>(),
2459
+ static_cast<int>(y_low_mode),
2460
+ static_cast<int>(y_high_mode),
2461
+ static_cast<int>(z_low_mode),
2462
+ static_cast<int>(z_high_mode),
2463
+ static_cast<int>(y_low_length),
2464
+ static_cast<int>(y_high_start),
2465
+ static_cast<int>(y_high_length),
2466
+ static_cast<int>(z_low_length),
2467
+ static_cast<int>(z_high_start),
2468
+ static_cast<int>(z_high_length),
2469
+ psi_y.mutable_data_ptr<float>(),
2470
+ psi_z.mutable_data_ptr<float>(),
2471
+ ex.mutable_data_ptr<float>());
2472
+ });
2473
+ WITWIN_CUDA_CHECK();
2474
+ }
2475
+
2476
+ void update_electric_ey_cpml_compressed_cuda(
2477
+ torch::stable::Tensor ey,
2478
+ const torch::stable::Tensor& hx,
2479
+ const torch::stable::Tensor& hz,
2480
+ const torch::stable::Tensor& decay,
2481
+ const torch::stable::Tensor& curl,
2482
+ torch::stable::Tensor psi_x,
2483
+ torch::stable::Tensor psi_z,
2484
+ const torch::stable::Tensor& inv_kappa_x,
2485
+ const torch::stable::Tensor& b_x,
2486
+ const torch::stable::Tensor& c_x,
2487
+ const torch::stable::Tensor& inv_kappa_z,
2488
+ const torch::stable::Tensor& b_z,
2489
+ const torch::stable::Tensor& c_z,
2490
+ const torch::stable::Tensor& inv_dx,
2491
+ const torch::stable::Tensor& inv_dz,
2492
+ int64_t x_low_mode,
2493
+ int64_t x_high_mode,
2494
+ int64_t z_low_mode,
2495
+ int64_t z_high_mode,
2496
+ int64_t x_low_length,
2497
+ int64_t x_high_start,
2498
+ int64_t x_high_length,
2499
+ int64_t z_low_length,
2500
+ int64_t z_high_start,
2501
+ int64_t z_high_length,
2502
+ std::optional<double> uniform_decay,
2503
+ std::optional<double> uniform_curl) {
2504
+ check_electric_cpml_compressed_inputs(
2505
+ ey, hx, hz, decay, curl, psi_x, psi_z, inv_kappa_x, b_x, c_x, inv_kappa_z, b_z, c_z,
2506
+ 0, 2, x_low_length, x_high_length, z_low_length, z_high_length, "ey");
2507
+ check_rank3_shape(hx, "hx", ey.size(0), ey.size(1), ey.size(2) - 1);
2508
+ check_rank3_shape(hz, "hz", ey.size(0) - 1, ey.size(1), ey.size(2));
2509
+ check_spacing_vector(ey, inv_dx, 0, "inv_dx");
2510
+ check_spacing_vector(ey, inv_dz, 2, "inv_dz");
2511
+ torch::stable::accelerator::DeviceGuard guard(ey.get_device_index());
2512
+ const auto sizes = ey.sizes();
2513
+ const dim3 block = field_block3d();
2514
+ dispatch_uniform_coefficients(uniform_decay.has_value(), uniform_curl.has_value(), [&](auto u_decay, auto u_curl) {
2515
+ update_electric_ey_cpml_compressed_kernel<decltype(u_decay)::value, decltype(u_curl)::value><<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2516
+ static_cast<unsigned int>(sizes[0]),
2517
+ static_cast<unsigned int>(sizes[1]),
2518
+ static_cast<unsigned int>(sizes[2]),
2519
+ hx.mutable_data_ptr<float>(),
2520
+ hz.mutable_data_ptr<float>(),
2521
+ decay.mutable_data_ptr<float>(),
2522
+ curl.mutable_data_ptr<float>(),
2523
+ static_cast<float>(uniform_decay.value_or(0.0)),
2524
+ static_cast<float>(uniform_curl.value_or(0.0)),
2525
+ inv_kappa_x.mutable_data_ptr<float>(),
2526
+ b_x.mutable_data_ptr<float>(),
2527
+ c_x.mutable_data_ptr<float>(),
2528
+ inv_kappa_z.mutable_data_ptr<float>(),
2529
+ b_z.mutable_data_ptr<float>(),
2530
+ c_z.mutable_data_ptr<float>(),
2531
+ inv_dx.mutable_data_ptr<float>(),
2532
+ inv_dz.mutable_data_ptr<float>(),
2533
+ static_cast<int>(x_low_mode),
2534
+ static_cast<int>(x_high_mode),
2535
+ static_cast<int>(z_low_mode),
2536
+ static_cast<int>(z_high_mode),
2537
+ static_cast<int>(x_low_length),
2538
+ static_cast<int>(x_high_start),
2539
+ static_cast<int>(x_high_length),
2540
+ static_cast<int>(z_low_length),
2541
+ static_cast<int>(z_high_start),
2542
+ static_cast<int>(z_high_length),
2543
+ psi_x.mutable_data_ptr<float>(),
2544
+ psi_z.mutable_data_ptr<float>(),
2545
+ ey.mutable_data_ptr<float>());
2546
+ });
2547
+ WITWIN_CUDA_CHECK();
2548
+ }
2549
+
2550
+ void update_electric_ez_cpml_compressed_cuda(
2551
+ torch::stable::Tensor ez,
2552
+ const torch::stable::Tensor& hx,
2553
+ const torch::stable::Tensor& hy,
2554
+ const torch::stable::Tensor& decay,
2555
+ const torch::stable::Tensor& curl,
2556
+ torch::stable::Tensor psi_x,
2557
+ torch::stable::Tensor psi_y,
2558
+ const torch::stable::Tensor& inv_kappa_x,
2559
+ const torch::stable::Tensor& b_x,
2560
+ const torch::stable::Tensor& c_x,
2561
+ const torch::stable::Tensor& inv_kappa_y,
2562
+ const torch::stable::Tensor& b_y,
2563
+ const torch::stable::Tensor& c_y,
2564
+ const torch::stable::Tensor& inv_dx,
2565
+ const torch::stable::Tensor& inv_dy,
2566
+ int64_t x_low_mode,
2567
+ int64_t x_high_mode,
2568
+ int64_t y_low_mode,
2569
+ int64_t y_high_mode,
2570
+ int64_t x_low_length,
2571
+ int64_t x_high_start,
2572
+ int64_t x_high_length,
2573
+ int64_t y_low_length,
2574
+ int64_t y_high_start,
2575
+ int64_t y_high_length,
2576
+ std::optional<double> uniform_decay,
2577
+ std::optional<double> uniform_curl) {
2578
+ check_electric_cpml_compressed_inputs(
2579
+ ez, hx, hy, decay, curl, psi_x, psi_y, inv_kappa_x, b_x, c_x, inv_kappa_y, b_y, c_y,
2580
+ 0, 1, x_low_length, x_high_length, y_low_length, y_high_length, "ez");
2581
+ check_rank3_shape(hx, "hx", ez.size(0), ez.size(1) - 1, ez.size(2));
2582
+ check_rank3_shape(hy, "hy", ez.size(0) - 1, ez.size(1), ez.size(2));
2583
+ check_spacing_vector(ez, inv_dx, 0, "inv_dx");
2584
+ check_spacing_vector(ez, inv_dy, 1, "inv_dy");
2585
+ torch::stable::accelerator::DeviceGuard guard(ez.get_device_index());
2586
+ const auto sizes = ez.sizes();
2587
+ const dim3 block = field_block3d();
2588
+ dispatch_uniform_coefficients(uniform_decay.has_value(), uniform_curl.has_value(), [&](auto u_decay, auto u_curl) {
2589
+ update_electric_ez_cpml_compressed_kernel<decltype(u_decay)::value, decltype(u_curl)::value><<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2590
+ static_cast<unsigned int>(sizes[0]),
2591
+ static_cast<unsigned int>(sizes[1]),
2592
+ static_cast<unsigned int>(sizes[2]),
2593
+ hx.mutable_data_ptr<float>(),
2594
+ hy.mutable_data_ptr<float>(),
2595
+ decay.mutable_data_ptr<float>(),
2596
+ curl.mutable_data_ptr<float>(),
2597
+ static_cast<float>(uniform_decay.value_or(0.0)),
2598
+ static_cast<float>(uniform_curl.value_or(0.0)),
2599
+ inv_kappa_x.mutable_data_ptr<float>(),
2600
+ b_x.mutable_data_ptr<float>(),
2601
+ c_x.mutable_data_ptr<float>(),
2602
+ inv_kappa_y.mutable_data_ptr<float>(),
2603
+ b_y.mutable_data_ptr<float>(),
2604
+ c_y.mutable_data_ptr<float>(),
2605
+ inv_dx.mutable_data_ptr<float>(),
2606
+ inv_dy.mutable_data_ptr<float>(),
2607
+ static_cast<int>(x_low_mode),
2608
+ static_cast<int>(x_high_mode),
2609
+ static_cast<int>(y_low_mode),
2610
+ static_cast<int>(y_high_mode),
2611
+ static_cast<int>(x_low_length),
2612
+ static_cast<int>(x_high_start),
2613
+ static_cast<int>(x_high_length),
2614
+ static_cast<int>(y_low_length),
2615
+ static_cast<int>(y_high_start),
2616
+ static_cast<int>(y_high_length),
2617
+ psi_x.mutable_data_ptr<float>(),
2618
+ psi_y.mutable_data_ptr<float>(),
2619
+ ez.mutable_data_ptr<float>());
2620
+ });
2621
+ WITWIN_CUDA_CHECK();
2622
+ }
2623
+
2624
+ void update_electric_ex_cpml_modulated_compressed_cuda(
2625
+ torch::stable::Tensor ex,
2626
+ const torch::stable::Tensor& hy,
2627
+ const torch::stable::Tensor& hz,
2628
+ const torch::stable::Tensor& decay,
2629
+ const torch::stable::Tensor& curl,
2630
+ const torch::stable::Tensor& mod_cos,
2631
+ const torch::stable::Tensor& mod_sin,
2632
+ const torch::stable::Tensor& mod_omega,
2633
+ const torch::stable::Tensor& modulation_time,
2634
+ torch::stable::Tensor psi_y,
2635
+ torch::stable::Tensor psi_z,
2636
+ const torch::stable::Tensor& inv_kappa_y,
2637
+ const torch::stable::Tensor& b_y,
2638
+ const torch::stable::Tensor& c_y,
2639
+ const torch::stable::Tensor& inv_kappa_z,
2640
+ const torch::stable::Tensor& b_z,
2641
+ const torch::stable::Tensor& c_z,
2642
+ const torch::stable::Tensor& inv_dy,
2643
+ const torch::stable::Tensor& inv_dz,
2644
+ int64_t y_low_mode,
2645
+ int64_t y_high_mode,
2646
+ int64_t z_low_mode,
2647
+ int64_t z_high_mode,
2648
+ int64_t y_low_length,
2649
+ int64_t y_high_start,
2650
+ int64_t y_high_length,
2651
+ int64_t z_low_length,
2652
+ int64_t z_high_start,
2653
+ int64_t z_high_length) {
2654
+ check_electric_cpml_compressed_inputs(
2655
+ ex, hy, hz, decay, curl, psi_y, psi_z, inv_kappa_y, b_y, c_y, inv_kappa_z, b_z, c_z,
2656
+ 1, 2, y_low_length, y_high_length, z_low_length, z_high_length, "ex");
2657
+ check_modulation_fields(ex, mod_cos, mod_sin, mod_omega);
2658
+ check_modulation_time(ex, modulation_time);
2659
+ check_rank3_shape(hy, "hy", ex.size(0), ex.size(1), ex.size(2) - 1);
2660
+ check_rank3_shape(hz, "hz", ex.size(0), ex.size(1) - 1, ex.size(2));
2661
+ check_spacing_vector(ex, inv_dy, 1, "inv_dy");
2662
+ check_spacing_vector(ex, inv_dz, 2, "inv_dz");
2663
+ torch::stable::accelerator::DeviceGuard guard(ex.get_device_index());
2664
+ const auto sizes = ex.sizes();
2665
+ const dim3 block = field_block3d();
2666
+ update_electric_ex_cpml_modulated_compressed_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2667
+ static_cast<unsigned int>(sizes[0]),
2668
+ static_cast<unsigned int>(sizes[1]),
2669
+ static_cast<unsigned int>(sizes[2]),
2670
+ hy.mutable_data_ptr<float>(),
2671
+ hz.mutable_data_ptr<float>(),
2672
+ decay.mutable_data_ptr<float>(),
2673
+ curl.mutable_data_ptr<float>(),
2674
+ mod_cos.mutable_data_ptr<float>(),
2675
+ mod_sin.mutable_data_ptr<float>(),
2676
+ mod_omega.mutable_data_ptr<float>(),
2677
+ modulation_time.mutable_data_ptr<float>(),
2678
+ inv_kappa_y.mutable_data_ptr<float>(),
2679
+ b_y.mutable_data_ptr<float>(),
2680
+ c_y.mutable_data_ptr<float>(),
2681
+ inv_kappa_z.mutable_data_ptr<float>(),
2682
+ b_z.mutable_data_ptr<float>(),
2683
+ c_z.mutable_data_ptr<float>(),
2684
+ inv_dy.mutable_data_ptr<float>(),
2685
+ inv_dz.mutable_data_ptr<float>(),
2686
+ static_cast<int>(y_low_mode),
2687
+ static_cast<int>(y_high_mode),
2688
+ static_cast<int>(z_low_mode),
2689
+ static_cast<int>(z_high_mode),
2690
+ static_cast<int>(y_low_length),
2691
+ static_cast<int>(y_high_start),
2692
+ static_cast<int>(y_high_length),
2693
+ static_cast<int>(z_low_length),
2694
+ static_cast<int>(z_high_start),
2695
+ static_cast<int>(z_high_length),
2696
+ psi_y.mutable_data_ptr<float>(),
2697
+ psi_z.mutable_data_ptr<float>(),
2698
+ ex.mutable_data_ptr<float>());
2699
+ WITWIN_CUDA_CHECK();
2700
+ }
2701
+
2702
+ void update_electric_ey_cpml_modulated_compressed_cuda(
2703
+ torch::stable::Tensor ey,
2704
+ const torch::stable::Tensor& hx,
2705
+ const torch::stable::Tensor& hz,
2706
+ const torch::stable::Tensor& decay,
2707
+ const torch::stable::Tensor& curl,
2708
+ const torch::stable::Tensor& mod_cos,
2709
+ const torch::stable::Tensor& mod_sin,
2710
+ const torch::stable::Tensor& mod_omega,
2711
+ const torch::stable::Tensor& modulation_time,
2712
+ torch::stable::Tensor psi_x,
2713
+ torch::stable::Tensor psi_z,
2714
+ const torch::stable::Tensor& inv_kappa_x,
2715
+ const torch::stable::Tensor& b_x,
2716
+ const torch::stable::Tensor& c_x,
2717
+ const torch::stable::Tensor& inv_kappa_z,
2718
+ const torch::stable::Tensor& b_z,
2719
+ const torch::stable::Tensor& c_z,
2720
+ const torch::stable::Tensor& inv_dx,
2721
+ const torch::stable::Tensor& inv_dz,
2722
+ int64_t x_low_mode,
2723
+ int64_t x_high_mode,
2724
+ int64_t z_low_mode,
2725
+ int64_t z_high_mode,
2726
+ int64_t x_low_length,
2727
+ int64_t x_high_start,
2728
+ int64_t x_high_length,
2729
+ int64_t z_low_length,
2730
+ int64_t z_high_start,
2731
+ int64_t z_high_length) {
2732
+ check_electric_cpml_compressed_inputs(
2733
+ ey, hx, hz, decay, curl, psi_x, psi_z, inv_kappa_x, b_x, c_x, inv_kappa_z, b_z, c_z,
2734
+ 0, 2, x_low_length, x_high_length, z_low_length, z_high_length, "ey");
2735
+ check_modulation_fields(ey, mod_cos, mod_sin, mod_omega);
2736
+ check_modulation_time(ey, modulation_time);
2737
+ check_rank3_shape(hx, "hx", ey.size(0), ey.size(1), ey.size(2) - 1);
2738
+ check_rank3_shape(hz, "hz", ey.size(0) - 1, ey.size(1), ey.size(2));
2739
+ check_spacing_vector(ey, inv_dx, 0, "inv_dx");
2740
+ check_spacing_vector(ey, inv_dz, 2, "inv_dz");
2741
+ torch::stable::accelerator::DeviceGuard guard(ey.get_device_index());
2742
+ const auto sizes = ey.sizes();
2743
+ const dim3 block = field_block3d();
2744
+ update_electric_ey_cpml_modulated_compressed_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2745
+ static_cast<unsigned int>(sizes[0]),
2746
+ static_cast<unsigned int>(sizes[1]),
2747
+ static_cast<unsigned int>(sizes[2]),
2748
+ hx.mutable_data_ptr<float>(),
2749
+ hz.mutable_data_ptr<float>(),
2750
+ decay.mutable_data_ptr<float>(),
2751
+ curl.mutable_data_ptr<float>(),
2752
+ mod_cos.mutable_data_ptr<float>(),
2753
+ mod_sin.mutable_data_ptr<float>(),
2754
+ mod_omega.mutable_data_ptr<float>(),
2755
+ modulation_time.mutable_data_ptr<float>(),
2756
+ inv_kappa_x.mutable_data_ptr<float>(),
2757
+ b_x.mutable_data_ptr<float>(),
2758
+ c_x.mutable_data_ptr<float>(),
2759
+ inv_kappa_z.mutable_data_ptr<float>(),
2760
+ b_z.mutable_data_ptr<float>(),
2761
+ c_z.mutable_data_ptr<float>(),
2762
+ inv_dx.mutable_data_ptr<float>(),
2763
+ inv_dz.mutable_data_ptr<float>(),
2764
+ static_cast<int>(x_low_mode),
2765
+ static_cast<int>(x_high_mode),
2766
+ static_cast<int>(z_low_mode),
2767
+ static_cast<int>(z_high_mode),
2768
+ static_cast<int>(x_low_length),
2769
+ static_cast<int>(x_high_start),
2770
+ static_cast<int>(x_high_length),
2771
+ static_cast<int>(z_low_length),
2772
+ static_cast<int>(z_high_start),
2773
+ static_cast<int>(z_high_length),
2774
+ psi_x.mutable_data_ptr<float>(),
2775
+ psi_z.mutable_data_ptr<float>(),
2776
+ ey.mutable_data_ptr<float>());
2777
+ WITWIN_CUDA_CHECK();
2778
+ }
2779
+
2780
+ void update_electric_ez_cpml_modulated_compressed_cuda(
2781
+ torch::stable::Tensor ez,
2782
+ const torch::stable::Tensor& hx,
2783
+ const torch::stable::Tensor& hy,
2784
+ const torch::stable::Tensor& decay,
2785
+ const torch::stable::Tensor& curl,
2786
+ const torch::stable::Tensor& mod_cos,
2787
+ const torch::stable::Tensor& mod_sin,
2788
+ const torch::stable::Tensor& mod_omega,
2789
+ const torch::stable::Tensor& modulation_time,
2790
+ torch::stable::Tensor psi_x,
2791
+ torch::stable::Tensor psi_y,
2792
+ const torch::stable::Tensor& inv_kappa_x,
2793
+ const torch::stable::Tensor& b_x,
2794
+ const torch::stable::Tensor& c_x,
2795
+ const torch::stable::Tensor& inv_kappa_y,
2796
+ const torch::stable::Tensor& b_y,
2797
+ const torch::stable::Tensor& c_y,
2798
+ const torch::stable::Tensor& inv_dx,
2799
+ const torch::stable::Tensor& inv_dy,
2800
+ int64_t x_low_mode,
2801
+ int64_t x_high_mode,
2802
+ int64_t y_low_mode,
2803
+ int64_t y_high_mode,
2804
+ int64_t x_low_length,
2805
+ int64_t x_high_start,
2806
+ int64_t x_high_length,
2807
+ int64_t y_low_length,
2808
+ int64_t y_high_start,
2809
+ int64_t y_high_length) {
2810
+ check_electric_cpml_compressed_inputs(
2811
+ ez, hx, hy, decay, curl, psi_x, psi_y, inv_kappa_x, b_x, c_x, inv_kappa_y, b_y, c_y,
2812
+ 0, 1, x_low_length, x_high_length, y_low_length, y_high_length, "ez");
2813
+ check_modulation_fields(ez, mod_cos, mod_sin, mod_omega);
2814
+ check_modulation_time(ez, modulation_time);
2815
+ check_rank3_shape(hx, "hx", ez.size(0), ez.size(1) - 1, ez.size(2));
2816
+ check_rank3_shape(hy, "hy", ez.size(0) - 1, ez.size(1), ez.size(2));
2817
+ check_spacing_vector(ez, inv_dx, 0, "inv_dx");
2818
+ check_spacing_vector(ez, inv_dy, 1, "inv_dy");
2819
+ torch::stable::accelerator::DeviceGuard guard(ez.get_device_index());
2820
+ const auto sizes = ez.sizes();
2821
+ const dim3 block = field_block3d();
2822
+ update_electric_ez_cpml_modulated_compressed_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2823
+ static_cast<unsigned int>(sizes[0]),
2824
+ static_cast<unsigned int>(sizes[1]),
2825
+ static_cast<unsigned int>(sizes[2]),
2826
+ hx.mutable_data_ptr<float>(),
2827
+ hy.mutable_data_ptr<float>(),
2828
+ decay.mutable_data_ptr<float>(),
2829
+ curl.mutable_data_ptr<float>(),
2830
+ mod_cos.mutable_data_ptr<float>(),
2831
+ mod_sin.mutable_data_ptr<float>(),
2832
+ mod_omega.mutable_data_ptr<float>(),
2833
+ modulation_time.mutable_data_ptr<float>(),
2834
+ inv_kappa_x.mutable_data_ptr<float>(),
2835
+ b_x.mutable_data_ptr<float>(),
2836
+ c_x.mutable_data_ptr<float>(),
2837
+ inv_kappa_y.mutable_data_ptr<float>(),
2838
+ b_y.mutable_data_ptr<float>(),
2839
+ c_y.mutable_data_ptr<float>(),
2840
+ inv_dx.mutable_data_ptr<float>(),
2841
+ inv_dy.mutable_data_ptr<float>(),
2842
+ static_cast<int>(x_low_mode),
2843
+ static_cast<int>(x_high_mode),
2844
+ static_cast<int>(y_low_mode),
2845
+ static_cast<int>(y_high_mode),
2846
+ static_cast<int>(x_low_length),
2847
+ static_cast<int>(x_high_start),
2848
+ static_cast<int>(x_high_length),
2849
+ static_cast<int>(y_low_length),
2850
+ static_cast<int>(y_high_start),
2851
+ static_cast<int>(y_high_length),
2852
+ psi_x.mutable_data_ptr<float>(),
2853
+ psi_y.mutable_data_ptr<float>(),
2854
+ ez.mutable_data_ptr<float>());
2855
+ WITWIN_CUDA_CHECK();
2856
+ }
2857
+
2858
+ void update_electric_ex_bloch_y_standard_z_cuda(
2859
+ torch::stable::Tensor ex_real,
2860
+ torch::stable::Tensor ex_imag,
2861
+ const torch::stable::Tensor& hy_real,
2862
+ const torch::stable::Tensor& hy_imag,
2863
+ const torch::stable::Tensor& hz_real,
2864
+ const torch::stable::Tensor& hz_imag,
2865
+ const torch::stable::Tensor& decay,
2866
+ const torch::stable::Tensor& curl,
2867
+ double phase_cos_y,
2868
+ double phase_sin_y,
2869
+ const torch::stable::Tensor& inv_dy,
2870
+ const torch::stable::Tensor& inv_dz,
2871
+ int64_t z_low_mode,
2872
+ int64_t z_high_mode) {
2873
+ check_spacing_vector(ex_real, inv_dy, 1, "inv_dy");
2874
+ check_spacing_vector(ex_real, inv_dz, 2, "inv_dz");
2875
+ torch::stable::accelerator::DeviceGuard guard(ex_real.get_device_index());
2876
+ const auto sizes = ex_real.sizes();
2877
+ const dim3 block = field_block3d();
2878
+ update_electric_ex_bloch_y_standard_z_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2879
+ static_cast<unsigned int>(sizes[0]),
2880
+ static_cast<unsigned int>(sizes[1]),
2881
+ static_cast<unsigned int>(sizes[2]),
2882
+ hy_real.mutable_data_ptr<float>(),
2883
+ hy_imag.mutable_data_ptr<float>(),
2884
+ hz_real.mutable_data_ptr<float>(),
2885
+ hz_imag.mutable_data_ptr<float>(),
2886
+ decay.mutable_data_ptr<float>(),
2887
+ curl.mutable_data_ptr<float>(),
2888
+ static_cast<float>(phase_cos_y),
2889
+ static_cast<float>(phase_sin_y),
2890
+ inv_dy.mutable_data_ptr<float>(),
2891
+ inv_dz.mutable_data_ptr<float>(),
2892
+ static_cast<int>(z_low_mode),
2893
+ static_cast<int>(z_high_mode),
2894
+ ex_real.mutable_data_ptr<float>(),
2895
+ ex_imag.mutable_data_ptr<float>());
2896
+ WITWIN_CUDA_CHECK();
2897
+ }
2898
+
2899
+ void update_electric_ey_bloch_x_standard_z_cuda(
2900
+ torch::stable::Tensor ey_real,
2901
+ torch::stable::Tensor ey_imag,
2902
+ const torch::stable::Tensor& hx_real,
2903
+ const torch::stable::Tensor& hx_imag,
2904
+ const torch::stable::Tensor& hz_real,
2905
+ const torch::stable::Tensor& hz_imag,
2906
+ const torch::stable::Tensor& decay,
2907
+ const torch::stable::Tensor& curl,
2908
+ double phase_cos_x,
2909
+ double phase_sin_x,
2910
+ const torch::stable::Tensor& inv_dx,
2911
+ const torch::stable::Tensor& inv_dz,
2912
+ int64_t z_low_mode,
2913
+ int64_t z_high_mode) {
2914
+ check_spacing_vector(ey_real, inv_dx, 0, "inv_dx");
2915
+ check_spacing_vector(ey_real, inv_dz, 2, "inv_dz");
2916
+ torch::stable::accelerator::DeviceGuard guard(ey_real.get_device_index());
2917
+ const auto sizes = ey_real.sizes();
2918
+ const dim3 block = field_block3d();
2919
+ update_electric_ey_bloch_x_standard_z_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
2920
+ static_cast<unsigned int>(sizes[0]),
2921
+ static_cast<unsigned int>(sizes[1]),
2922
+ static_cast<unsigned int>(sizes[2]),
2923
+ hx_real.mutable_data_ptr<float>(),
2924
+ hx_imag.mutable_data_ptr<float>(),
2925
+ hz_real.mutable_data_ptr<float>(),
2926
+ hz_imag.mutable_data_ptr<float>(),
2927
+ decay.mutable_data_ptr<float>(),
2928
+ curl.mutable_data_ptr<float>(),
2929
+ static_cast<float>(phase_cos_x),
2930
+ static_cast<float>(phase_sin_x),
2931
+ inv_dx.mutable_data_ptr<float>(),
2932
+ inv_dz.mutable_data_ptr<float>(),
2933
+ static_cast<int>(z_low_mode),
2934
+ static_cast<int>(z_high_mode),
2935
+ ey_real.mutable_data_ptr<float>(),
2936
+ ey_imag.mutable_data_ptr<float>());
2937
+ WITWIN_CUDA_CHECK();
2938
+ }
2939
+
2940
+ void apply_electric_ex_cpml_z_correction_cuda(
2941
+ torch::stable::Tensor ex,
2942
+ const torch::stable::Tensor& hy,
2943
+ const torch::stable::Tensor& curl,
2944
+ torch::stable::Tensor psi_z,
2945
+ const torch::stable::Tensor& inv_kappa_z,
2946
+ const torch::stable::Tensor& b_z,
2947
+ const torch::stable::Tensor& c_z,
2948
+ const torch::stable::Tensor& inv_dz,
2949
+ int64_t offset_i,
2950
+ int64_t offset_j,
2951
+ int64_t offset_k,
2952
+ int64_t y_low_mode,
2953
+ int64_t y_high_mode,
2954
+ int64_t full_size_y,
2955
+ int64_t full_size_z) {
2956
+ const auto sizes = ex.sizes();
2957
+ const int sx = static_cast<int>(sizes[0]);
2958
+ const int sy = static_cast<int>(sizes[1]);
2959
+ const int sz = static_cast<int>(sizes[2]);
2960
+ const int start_candidate = 1 - static_cast<int>(offset_k);
2961
+ const int local_z_start = start_candidate > 0 ? start_candidate : 0;
2962
+ const int stop_candidate = static_cast<int>(full_size_z) - 1 - static_cast<int>(offset_k);
2963
+ const int local_z_stop = sz < stop_candidate ? sz : stop_candidate;
2964
+ if (local_z_stop <= local_z_start) {
2965
+ return;
2966
+ }
2967
+ check_vector_input(inv_dz, full_size_z, "inv_dz");
2968
+ check_same_cuda_device(ex, inv_dz, "inv_dz");
2969
+ torch::stable::accelerator::DeviceGuard guard(ex.get_device_index());
2970
+ const dim3 block = field_block3d();
2971
+ apply_electric_ex_cpml_z_correction_kernel<<<field_grid3d(sx, sy, sz, block), block, 0, current_cuda_stream()>>>(
2972
+ static_cast<unsigned int>(sx),
2973
+ static_cast<unsigned int>(sy),
2974
+ static_cast<unsigned int>(sz),
2975
+ static_cast<unsigned int>(hy.size(1)),
2976
+ static_cast<unsigned int>(hy.size(2)),
2977
+ hy.mutable_data_ptr<float>(),
2978
+ curl.mutable_data_ptr<float>(),
2979
+ inv_kappa_z.mutable_data_ptr<float>(),
2980
+ b_z.mutable_data_ptr<float>(),
2981
+ c_z.mutable_data_ptr<float>(),
2982
+ inv_dz.mutable_data_ptr<float>(),
2983
+ static_cast<int>(offset_i),
2984
+ static_cast<int>(offset_j),
2985
+ static_cast<int>(offset_k),
2986
+ static_cast<int>(y_low_mode),
2987
+ static_cast<int>(y_high_mode),
2988
+ static_cast<unsigned int>(full_size_y),
2989
+ local_z_start,
2990
+ local_z_stop,
2991
+ psi_z.mutable_data_ptr<float>(),
2992
+ ex.mutable_data_ptr<float>());
2993
+ WITWIN_CUDA_CHECK();
2994
+ }
2995
+
2996
+ void apply_electric_ey_cpml_z_correction_cuda(
2997
+ torch::stable::Tensor ey,
2998
+ const torch::stable::Tensor& hx,
2999
+ const torch::stable::Tensor& curl,
3000
+ torch::stable::Tensor psi_z,
3001
+ const torch::stable::Tensor& inv_kappa_z,
3002
+ const torch::stable::Tensor& b_z,
3003
+ const torch::stable::Tensor& c_z,
3004
+ const torch::stable::Tensor& inv_dz,
3005
+ int64_t offset_i,
3006
+ int64_t offset_j,
3007
+ int64_t offset_k,
3008
+ int64_t x_low_mode,
3009
+ int64_t x_high_mode,
3010
+ int64_t full_size_x,
3011
+ int64_t full_size_z) {
3012
+ const auto sizes = ey.sizes();
3013
+ const int sx = static_cast<int>(sizes[0]);
3014
+ const int sy = static_cast<int>(sizes[1]);
3015
+ const int sz = static_cast<int>(sizes[2]);
3016
+ const int start_candidate = 1 - static_cast<int>(offset_k);
3017
+ const int local_z_start = start_candidate > 0 ? start_candidate : 0;
3018
+ const int stop_candidate = static_cast<int>(full_size_z) - 1 - static_cast<int>(offset_k);
3019
+ const int local_z_stop = sz < stop_candidate ? sz : stop_candidate;
3020
+ if (local_z_stop <= local_z_start) {
3021
+ return;
3022
+ }
3023
+ check_vector_input(inv_dz, full_size_z, "inv_dz");
3024
+ check_same_cuda_device(ey, inv_dz, "inv_dz");
3025
+ torch::stable::accelerator::DeviceGuard guard(ey.get_device_index());
3026
+ const dim3 block = field_block3d();
3027
+ apply_electric_ey_cpml_z_correction_kernel<<<field_grid3d(sx, sy, sz, block), block, 0, current_cuda_stream()>>>(
3028
+ static_cast<unsigned int>(sx),
3029
+ static_cast<unsigned int>(sy),
3030
+ static_cast<unsigned int>(sz),
3031
+ static_cast<unsigned int>(hx.size(1)),
3032
+ static_cast<unsigned int>(hx.size(2)),
3033
+ hx.mutable_data_ptr<float>(),
3034
+ curl.mutable_data_ptr<float>(),
3035
+ inv_kappa_z.mutable_data_ptr<float>(),
3036
+ b_z.mutable_data_ptr<float>(),
3037
+ c_z.mutable_data_ptr<float>(),
3038
+ inv_dz.mutable_data_ptr<float>(),
3039
+ static_cast<int>(offset_i),
3040
+ static_cast<int>(offset_j),
3041
+ static_cast<int>(offset_k),
3042
+ static_cast<int>(x_low_mode),
3043
+ static_cast<int>(x_high_mode),
3044
+ static_cast<unsigned int>(full_size_x),
3045
+ local_z_start,
3046
+ local_z_stop,
3047
+ psi_z.mutable_data_ptr<float>(),
3048
+ ey.mutable_data_ptr<float>());
3049
+ WITWIN_CUDA_CHECK();
3050
+ }
3051
+
3052
+ void apply_electric_ex_cpml_y_correction_cuda(
3053
+ torch::stable::Tensor ex,
3054
+ const torch::stable::Tensor& hz,
3055
+ const torch::stable::Tensor& curl,
3056
+ torch::stable::Tensor psi_y,
3057
+ const torch::stable::Tensor& inv_kappa_y,
3058
+ const torch::stable::Tensor& b_y,
3059
+ const torch::stable::Tensor& c_y,
3060
+ const torch::stable::Tensor& inv_dy,
3061
+ int64_t offset_i,
3062
+ int64_t offset_j,
3063
+ int64_t offset_k,
3064
+ int64_t z_low_mode,
3065
+ int64_t z_high_mode,
3066
+ int64_t full_size_z,
3067
+ int64_t full_size_y) {
3068
+ const auto sizes = ex.sizes();
3069
+ const int sx = static_cast<int>(sizes[0]);
3070
+ const int sy = static_cast<int>(sizes[1]);
3071
+ const int sz = static_cast<int>(sizes[2]);
3072
+ const int start_candidate = 1 - static_cast<int>(offset_j);
3073
+ const int local_y_start = start_candidate > 0 ? start_candidate : 0;
3074
+ const int stop_candidate = static_cast<int>(full_size_y) - 1 - static_cast<int>(offset_j);
3075
+ const int local_y_stop = sy < stop_candidate ? sy : stop_candidate;
3076
+ if (local_y_stop <= local_y_start) {
3077
+ return;
3078
+ }
3079
+ check_vector_input(inv_dy, full_size_y, "inv_dy");
3080
+ check_same_cuda_device(ex, inv_dy, "inv_dy");
3081
+ torch::stable::accelerator::DeviceGuard guard(ex.get_device_index());
3082
+ const dim3 block = field_block3d();
3083
+ apply_electric_ex_cpml_y_correction_kernel<<<field_grid3d(sx, sy, sz, block), block, 0, current_cuda_stream()>>>(
3084
+ static_cast<unsigned int>(sx),
3085
+ static_cast<unsigned int>(sy),
3086
+ static_cast<unsigned int>(sz),
3087
+ static_cast<unsigned int>(hz.size(1)),
3088
+ static_cast<unsigned int>(hz.size(2)),
3089
+ hz.mutable_data_ptr<float>(),
3090
+ curl.mutable_data_ptr<float>(),
3091
+ inv_kappa_y.mutable_data_ptr<float>(),
3092
+ b_y.mutable_data_ptr<float>(),
3093
+ c_y.mutable_data_ptr<float>(),
3094
+ inv_dy.mutable_data_ptr<float>(),
3095
+ static_cast<int>(offset_i),
3096
+ static_cast<int>(offset_j),
3097
+ static_cast<int>(offset_k),
3098
+ static_cast<int>(z_low_mode),
3099
+ static_cast<int>(z_high_mode),
3100
+ static_cast<unsigned int>(full_size_z),
3101
+ local_y_start,
3102
+ local_y_stop,
3103
+ psi_y.mutable_data_ptr<float>(),
3104
+ ex.mutable_data_ptr<float>());
3105
+ WITWIN_CUDA_CHECK();
3106
+ }
3107
+
3108
+ void apply_electric_ez_cpml_y_correction_cuda(
3109
+ torch::stable::Tensor ez,
3110
+ const torch::stable::Tensor& hx,
3111
+ const torch::stable::Tensor& curl,
3112
+ torch::stable::Tensor psi_y,
3113
+ const torch::stable::Tensor& inv_kappa_y,
3114
+ const torch::stable::Tensor& b_y,
3115
+ const torch::stable::Tensor& c_y,
3116
+ const torch::stable::Tensor& inv_dy,
3117
+ int64_t offset_i,
3118
+ int64_t offset_j,
3119
+ int64_t offset_k,
3120
+ int64_t x_low_mode,
3121
+ int64_t x_high_mode,
3122
+ int64_t full_size_x,
3123
+ int64_t full_size_y) {
3124
+ const auto sizes = ez.sizes();
3125
+ const int sx = static_cast<int>(sizes[0]);
3126
+ const int sy = static_cast<int>(sizes[1]);
3127
+ const int sz = static_cast<int>(sizes[2]);
3128
+ const int start_candidate = 1 - static_cast<int>(offset_j);
3129
+ const int local_y_start = start_candidate > 0 ? start_candidate : 0;
3130
+ const int stop_candidate = static_cast<int>(full_size_y) - 1 - static_cast<int>(offset_j);
3131
+ const int local_y_stop = sy < stop_candidate ? sy : stop_candidate;
3132
+ if (local_y_stop <= local_y_start) {
3133
+ return;
3134
+ }
3135
+ check_vector_input(inv_dy, full_size_y, "inv_dy");
3136
+ check_same_cuda_device(ez, inv_dy, "inv_dy");
3137
+ torch::stable::accelerator::DeviceGuard guard(ez.get_device_index());
3138
+ const dim3 block = field_block3d();
3139
+ apply_electric_ez_cpml_y_correction_kernel<<<field_grid3d(sx, sy, sz, block), block, 0, current_cuda_stream()>>>(
3140
+ static_cast<unsigned int>(sx),
3141
+ static_cast<unsigned int>(sy),
3142
+ static_cast<unsigned int>(sz),
3143
+ static_cast<unsigned int>(hx.size(1)),
3144
+ static_cast<unsigned int>(hx.size(2)),
3145
+ hx.mutable_data_ptr<float>(),
3146
+ curl.mutable_data_ptr<float>(),
3147
+ inv_kappa_y.mutable_data_ptr<float>(),
3148
+ b_y.mutable_data_ptr<float>(),
3149
+ c_y.mutable_data_ptr<float>(),
3150
+ inv_dy.mutable_data_ptr<float>(),
3151
+ static_cast<int>(offset_i),
3152
+ static_cast<int>(offset_j),
3153
+ static_cast<int>(offset_k),
3154
+ static_cast<int>(x_low_mode),
3155
+ static_cast<int>(x_high_mode),
3156
+ static_cast<unsigned int>(full_size_x),
3157
+ local_y_start,
3158
+ local_y_stop,
3159
+ psi_y.mutable_data_ptr<float>(),
3160
+ ez.mutable_data_ptr<float>());
3161
+ WITWIN_CUDA_CHECK();
3162
+ }
3163
+
3164
+ void apply_electric_ey_cpml_x_correction_cuda(
3165
+ torch::stable::Tensor ey,
3166
+ const torch::stable::Tensor& hz,
3167
+ const torch::stable::Tensor& curl,
3168
+ torch::stable::Tensor psi_x,
3169
+ const torch::stable::Tensor& inv_kappa_x,
3170
+ const torch::stable::Tensor& b_x,
3171
+ const torch::stable::Tensor& c_x,
3172
+ const torch::stable::Tensor& inv_dx,
3173
+ int64_t offset_i,
3174
+ int64_t offset_j,
3175
+ int64_t offset_k,
3176
+ int64_t z_low_mode,
3177
+ int64_t z_high_mode,
3178
+ int64_t full_size_z,
3179
+ int64_t full_size_x) {
3180
+ const auto sizes = ey.sizes();
3181
+ const int sx = static_cast<int>(sizes[0]);
3182
+ const int sy = static_cast<int>(sizes[1]);
3183
+ const int sz = static_cast<int>(sizes[2]);
3184
+ const int start_candidate = 1 - static_cast<int>(offset_i);
3185
+ const int local_x_start = start_candidate > 0 ? start_candidate : 0;
3186
+ const int stop_candidate = static_cast<int>(full_size_x) - 1 - static_cast<int>(offset_i);
3187
+ const int local_x_stop = sx < stop_candidate ? sx : stop_candidate;
3188
+ if (local_x_stop <= local_x_start) {
3189
+ return;
3190
+ }
3191
+ check_vector_input(inv_dx, full_size_x, "inv_dx");
3192
+ check_same_cuda_device(ey, inv_dx, "inv_dx");
3193
+ torch::stable::accelerator::DeviceGuard guard(ey.get_device_index());
3194
+ const dim3 block = field_block3d();
3195
+ apply_electric_ey_cpml_x_correction_kernel<<<field_grid3d(sx, sy, sz, block), block, 0, current_cuda_stream()>>>(
3196
+ static_cast<unsigned int>(sx),
3197
+ static_cast<unsigned int>(sy),
3198
+ static_cast<unsigned int>(sz),
3199
+ static_cast<unsigned int>(hz.size(1)),
3200
+ static_cast<unsigned int>(hz.size(2)),
3201
+ hz.mutable_data_ptr<float>(),
3202
+ curl.mutable_data_ptr<float>(),
3203
+ inv_kappa_x.mutable_data_ptr<float>(),
3204
+ b_x.mutable_data_ptr<float>(),
3205
+ c_x.mutable_data_ptr<float>(),
3206
+ inv_dx.mutable_data_ptr<float>(),
3207
+ static_cast<int>(offset_i),
3208
+ static_cast<int>(offset_j),
3209
+ static_cast<int>(offset_k),
3210
+ static_cast<int>(z_low_mode),
3211
+ static_cast<int>(z_high_mode),
3212
+ static_cast<unsigned int>(full_size_z),
3213
+ local_x_start,
3214
+ local_x_stop,
3215
+ psi_x.mutable_data_ptr<float>(),
3216
+ ey.mutable_data_ptr<float>());
3217
+ WITWIN_CUDA_CHECK();
3218
+ }
3219
+
3220
+ void apply_electric_ez_cpml_x_correction_cuda(
3221
+ torch::stable::Tensor ez,
3222
+ const torch::stable::Tensor& hy,
3223
+ const torch::stable::Tensor& curl,
3224
+ torch::stable::Tensor psi_x,
3225
+ const torch::stable::Tensor& inv_kappa_x,
3226
+ const torch::stable::Tensor& b_x,
3227
+ const torch::stable::Tensor& c_x,
3228
+ const torch::stable::Tensor& inv_dx,
3229
+ int64_t offset_i,
3230
+ int64_t offset_j,
3231
+ int64_t offset_k,
3232
+ int64_t y_low_mode,
3233
+ int64_t y_high_mode,
3234
+ int64_t full_size_y,
3235
+ int64_t full_size_x) {
3236
+ const auto sizes = ez.sizes();
3237
+ const int sx = static_cast<int>(sizes[0]);
3238
+ const int sy = static_cast<int>(sizes[1]);
3239
+ const int sz = static_cast<int>(sizes[2]);
3240
+ const int start_candidate = 1 - static_cast<int>(offset_i);
3241
+ const int local_x_start = start_candidate > 0 ? start_candidate : 0;
3242
+ const int stop_candidate = static_cast<int>(full_size_x) - 1 - static_cast<int>(offset_i);
3243
+ const int local_x_stop = sx < stop_candidate ? sx : stop_candidate;
3244
+ if (local_x_stop <= local_x_start) {
3245
+ return;
3246
+ }
3247
+ check_vector_input(inv_dx, full_size_x, "inv_dx");
3248
+ check_same_cuda_device(ez, inv_dx, "inv_dx");
3249
+ torch::stable::accelerator::DeviceGuard guard(ez.get_device_index());
3250
+ const dim3 block = field_block3d();
3251
+ apply_electric_ez_cpml_x_correction_kernel<<<field_grid3d(sx, sy, sz, block), block, 0, current_cuda_stream()>>>(
3252
+ static_cast<unsigned int>(sx),
3253
+ static_cast<unsigned int>(sy),
3254
+ static_cast<unsigned int>(sz),
3255
+ static_cast<unsigned int>(hy.size(1)),
3256
+ static_cast<unsigned int>(hy.size(2)),
3257
+ hy.mutable_data_ptr<float>(),
3258
+ curl.mutable_data_ptr<float>(),
3259
+ inv_kappa_x.mutable_data_ptr<float>(),
3260
+ b_x.mutable_data_ptr<float>(),
3261
+ c_x.mutable_data_ptr<float>(),
3262
+ inv_dx.mutable_data_ptr<float>(),
3263
+ static_cast<int>(offset_i),
3264
+ static_cast<int>(offset_j),
3265
+ static_cast<int>(offset_k),
3266
+ static_cast<int>(y_low_mode),
3267
+ static_cast<int>(y_high_mode),
3268
+ static_cast<unsigned int>(full_size_y),
3269
+ local_x_start,
3270
+ local_x_stop,
3271
+ psi_x.mutable_data_ptr<float>(),
3272
+ ez.mutable_data_ptr<float>());
3273
+ WITWIN_CUDA_CHECK();
3274
+ }
3275
+
3276
+ void update_electric_ex_modulated_cuda(
3277
+ torch::stable::Tensor ex,
3278
+ const torch::stable::Tensor& hy,
3279
+ const torch::stable::Tensor& hz,
3280
+ const torch::stable::Tensor& decay,
3281
+ const torch::stable::Tensor& curl,
3282
+ const torch::stable::Tensor& mod_cos,
3283
+ const torch::stable::Tensor& mod_sin,
3284
+ const torch::stable::Tensor& mod_omega,
3285
+ const torch::stable::Tensor& modulation_time,
3286
+ const torch::stable::Tensor& inv_dy,
3287
+ const torch::stable::Tensor& inv_dz,
3288
+ int64_t y_low_mode,
3289
+ int64_t y_high_mode,
3290
+ int64_t z_low_mode,
3291
+ int64_t z_high_mode) {
3292
+ check_electric_inputs(ex, hy, hz, decay, curl, "ex");
3293
+ check_modulation_fields(ex, mod_cos, mod_sin, mod_omega);
3294
+ check_modulation_time(ex, modulation_time);
3295
+ check_rank3_shape(hy, "hy", ex.size(0), ex.size(1), ex.size(2) - 1);
3296
+ check_rank3_shape(hz, "hz", ex.size(0), ex.size(1) - 1, ex.size(2));
3297
+ check_spacing_vector(ex, inv_dy, 1, "inv_dy");
3298
+ check_spacing_vector(ex, inv_dz, 2, "inv_dz");
3299
+ torch::stable::accelerator::DeviceGuard guard(ex.get_device_index());
3300
+ const auto sizes = ex.sizes();
3301
+ const dim3 block = field_block3d();
3302
+ update_electric_ex_modulated_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
3303
+ static_cast<unsigned int>(sizes[0]),
3304
+ static_cast<unsigned int>(sizes[1]),
3305
+ static_cast<unsigned int>(sizes[2]),
3306
+ hy.mutable_data_ptr<float>(),
3307
+ hz.mutable_data_ptr<float>(),
3308
+ decay.mutable_data_ptr<float>(),
3309
+ curl.mutable_data_ptr<float>(),
3310
+ mod_cos.mutable_data_ptr<float>(),
3311
+ mod_sin.mutable_data_ptr<float>(),
3312
+ mod_omega.mutable_data_ptr<float>(),
3313
+ modulation_time.mutable_data_ptr<float>(),
3314
+ inv_dy.mutable_data_ptr<float>(),
3315
+ inv_dz.mutable_data_ptr<float>(),
3316
+ static_cast<int>(y_low_mode),
3317
+ static_cast<int>(y_high_mode),
3318
+ static_cast<int>(z_low_mode),
3319
+ static_cast<int>(z_high_mode),
3320
+ ex.mutable_data_ptr<float>());
3321
+ WITWIN_CUDA_CHECK();
3322
+ }
3323
+
3324
+ void update_electric_ey_modulated_cuda(
3325
+ torch::stable::Tensor ey,
3326
+ const torch::stable::Tensor& hx,
3327
+ const torch::stable::Tensor& hz,
3328
+ const torch::stable::Tensor& decay,
3329
+ const torch::stable::Tensor& curl,
3330
+ const torch::stable::Tensor& mod_cos,
3331
+ const torch::stable::Tensor& mod_sin,
3332
+ const torch::stable::Tensor& mod_omega,
3333
+ const torch::stable::Tensor& modulation_time,
3334
+ const torch::stable::Tensor& inv_dx,
3335
+ const torch::stable::Tensor& inv_dz,
3336
+ int64_t x_low_mode,
3337
+ int64_t x_high_mode,
3338
+ int64_t z_low_mode,
3339
+ int64_t z_high_mode) {
3340
+ check_electric_inputs(ey, hx, hz, decay, curl, "ey");
3341
+ check_modulation_fields(ey, mod_cos, mod_sin, mod_omega);
3342
+ check_modulation_time(ey, modulation_time);
3343
+ check_rank3_shape(hx, "hx", ey.size(0), ey.size(1), ey.size(2) - 1);
3344
+ check_rank3_shape(hz, "hz", ey.size(0) - 1, ey.size(1), ey.size(2));
3345
+ check_spacing_vector(ey, inv_dx, 0, "inv_dx");
3346
+ check_spacing_vector(ey, inv_dz, 2, "inv_dz");
3347
+ torch::stable::accelerator::DeviceGuard guard(ey.get_device_index());
3348
+ const auto sizes = ey.sizes();
3349
+ const dim3 block = field_block3d();
3350
+ update_electric_ey_modulated_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
3351
+ static_cast<unsigned int>(sizes[0]),
3352
+ static_cast<unsigned int>(sizes[1]),
3353
+ static_cast<unsigned int>(sizes[2]),
3354
+ hx.mutable_data_ptr<float>(),
3355
+ hz.mutable_data_ptr<float>(),
3356
+ decay.mutable_data_ptr<float>(),
3357
+ curl.mutable_data_ptr<float>(),
3358
+ mod_cos.mutable_data_ptr<float>(),
3359
+ mod_sin.mutable_data_ptr<float>(),
3360
+ mod_omega.mutable_data_ptr<float>(),
3361
+ modulation_time.mutable_data_ptr<float>(),
3362
+ inv_dx.mutable_data_ptr<float>(),
3363
+ inv_dz.mutable_data_ptr<float>(),
3364
+ static_cast<int>(x_low_mode),
3365
+ static_cast<int>(x_high_mode),
3366
+ static_cast<int>(z_low_mode),
3367
+ static_cast<int>(z_high_mode),
3368
+ ey.mutable_data_ptr<float>());
3369
+ WITWIN_CUDA_CHECK();
3370
+ }
3371
+
3372
+ void update_electric_ez_modulated_cuda(
3373
+ torch::stable::Tensor ez,
3374
+ const torch::stable::Tensor& hx,
3375
+ const torch::stable::Tensor& hy,
3376
+ const torch::stable::Tensor& decay,
3377
+ const torch::stable::Tensor& curl,
3378
+ const torch::stable::Tensor& mod_cos,
3379
+ const torch::stable::Tensor& mod_sin,
3380
+ const torch::stable::Tensor& mod_omega,
3381
+ const torch::stable::Tensor& modulation_time,
3382
+ const torch::stable::Tensor& inv_dx,
3383
+ const torch::stable::Tensor& inv_dy,
3384
+ int64_t x_low_mode,
3385
+ int64_t x_high_mode,
3386
+ int64_t y_low_mode,
3387
+ int64_t y_high_mode) {
3388
+ check_electric_inputs(ez, hx, hy, decay, curl, "ez");
3389
+ check_modulation_fields(ez, mod_cos, mod_sin, mod_omega);
3390
+ check_modulation_time(ez, modulation_time);
3391
+ check_rank3_shape(hx, "hx", ez.size(0), ez.size(1) - 1, ez.size(2));
3392
+ check_rank3_shape(hy, "hy", ez.size(0) - 1, ez.size(1), ez.size(2));
3393
+ check_spacing_vector(ez, inv_dx, 0, "inv_dx");
3394
+ check_spacing_vector(ez, inv_dy, 1, "inv_dy");
3395
+ torch::stable::accelerator::DeviceGuard guard(ez.get_device_index());
3396
+ const auto sizes = ez.sizes();
3397
+ const dim3 block = field_block3d();
3398
+ update_electric_ez_modulated_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
3399
+ static_cast<unsigned int>(sizes[0]),
3400
+ static_cast<unsigned int>(sizes[1]),
3401
+ static_cast<unsigned int>(sizes[2]),
3402
+ hx.mutable_data_ptr<float>(),
3403
+ hy.mutable_data_ptr<float>(),
3404
+ decay.mutable_data_ptr<float>(),
3405
+ curl.mutable_data_ptr<float>(),
3406
+ mod_cos.mutable_data_ptr<float>(),
3407
+ mod_sin.mutable_data_ptr<float>(),
3408
+ mod_omega.mutable_data_ptr<float>(),
3409
+ modulation_time.mutable_data_ptr<float>(),
3410
+ inv_dx.mutable_data_ptr<float>(),
3411
+ inv_dy.mutable_data_ptr<float>(),
3412
+ static_cast<int>(x_low_mode),
3413
+ static_cast<int>(x_high_mode),
3414
+ static_cast<int>(y_low_mode),
3415
+ static_cast<int>(y_high_mode),
3416
+ ez.mutable_data_ptr<float>());
3417
+ WITWIN_CUDA_CHECK();
3418
+ }
3419
+
3420
+ void update_electric_ex_cpml_modulated_cuda(
3421
+ torch::stable::Tensor ex,
3422
+ const torch::stable::Tensor& hy,
3423
+ const torch::stable::Tensor& hz,
3424
+ const torch::stable::Tensor& decay,
3425
+ const torch::stable::Tensor& curl,
3426
+ const torch::stable::Tensor& mod_cos,
3427
+ const torch::stable::Tensor& mod_sin,
3428
+ const torch::stable::Tensor& mod_omega,
3429
+ const torch::stable::Tensor& modulation_time,
3430
+ torch::stable::Tensor psi_y,
3431
+ torch::stable::Tensor psi_z,
3432
+ const torch::stable::Tensor& inv_kappa_y,
3433
+ const torch::stable::Tensor& b_y,
3434
+ const torch::stable::Tensor& c_y,
3435
+ const torch::stable::Tensor& inv_kappa_z,
3436
+ const torch::stable::Tensor& b_z,
3437
+ const torch::stable::Tensor& c_z,
3438
+ const torch::stable::Tensor& inv_dy,
3439
+ const torch::stable::Tensor& inv_dz,
3440
+ int64_t y_low_mode,
3441
+ int64_t y_high_mode,
3442
+ int64_t z_low_mode,
3443
+ int64_t z_high_mode) {
3444
+ check_electric_cpml_inputs(
3445
+ ex, hy, hz, decay, curl, psi_y, psi_z, inv_kappa_y, b_y, c_y, inv_kappa_z, b_z, c_z, 1, 2, "ex");
3446
+ check_modulation_fields(ex, mod_cos, mod_sin, mod_omega);
3447
+ check_modulation_time(ex, modulation_time);
3448
+ check_rank3_shape(hy, "hy", ex.size(0), ex.size(1), ex.size(2) - 1);
3449
+ check_rank3_shape(hz, "hz", ex.size(0), ex.size(1) - 1, ex.size(2));
3450
+ check_spacing_vector(ex, inv_dy, 1, "inv_dy");
3451
+ check_spacing_vector(ex, inv_dz, 2, "inv_dz");
3452
+ torch::stable::accelerator::DeviceGuard guard(ex.get_device_index());
3453
+ const auto sizes = ex.sizes();
3454
+ const dim3 block = field_block3d();
3455
+ update_electric_ex_cpml_modulated_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
3456
+ static_cast<unsigned int>(sizes[0]),
3457
+ static_cast<unsigned int>(sizes[1]),
3458
+ static_cast<unsigned int>(sizes[2]),
3459
+ hy.mutable_data_ptr<float>(),
3460
+ hz.mutable_data_ptr<float>(),
3461
+ decay.mutable_data_ptr<float>(),
3462
+ curl.mutable_data_ptr<float>(),
3463
+ mod_cos.mutable_data_ptr<float>(),
3464
+ mod_sin.mutable_data_ptr<float>(),
3465
+ mod_omega.mutable_data_ptr<float>(),
3466
+ modulation_time.mutable_data_ptr<float>(),
3467
+ inv_kappa_y.mutable_data_ptr<float>(),
3468
+ b_y.mutable_data_ptr<float>(),
3469
+ c_y.mutable_data_ptr<float>(),
3470
+ inv_kappa_z.mutable_data_ptr<float>(),
3471
+ b_z.mutable_data_ptr<float>(),
3472
+ c_z.mutable_data_ptr<float>(),
3473
+ inv_dy.mutable_data_ptr<float>(),
3474
+ inv_dz.mutable_data_ptr<float>(),
3475
+ static_cast<int>(y_low_mode),
3476
+ static_cast<int>(y_high_mode),
3477
+ static_cast<int>(z_low_mode),
3478
+ static_cast<int>(z_high_mode),
3479
+ psi_y.mutable_data_ptr<float>(),
3480
+ psi_z.mutable_data_ptr<float>(),
3481
+ ex.mutable_data_ptr<float>());
3482
+ WITWIN_CUDA_CHECK();
3483
+ }
3484
+
3485
+ void update_electric_ey_cpml_modulated_cuda(
3486
+ torch::stable::Tensor ey,
3487
+ const torch::stable::Tensor& hx,
3488
+ const torch::stable::Tensor& hz,
3489
+ const torch::stable::Tensor& decay,
3490
+ const torch::stable::Tensor& curl,
3491
+ const torch::stable::Tensor& mod_cos,
3492
+ const torch::stable::Tensor& mod_sin,
3493
+ const torch::stable::Tensor& mod_omega,
3494
+ const torch::stable::Tensor& modulation_time,
3495
+ torch::stable::Tensor psi_x,
3496
+ torch::stable::Tensor psi_z,
3497
+ const torch::stable::Tensor& inv_kappa_x,
3498
+ const torch::stable::Tensor& b_x,
3499
+ const torch::stable::Tensor& c_x,
3500
+ const torch::stable::Tensor& inv_kappa_z,
3501
+ const torch::stable::Tensor& b_z,
3502
+ const torch::stable::Tensor& c_z,
3503
+ const torch::stable::Tensor& inv_dx,
3504
+ const torch::stable::Tensor& inv_dz,
3505
+ int64_t x_low_mode,
3506
+ int64_t x_high_mode,
3507
+ int64_t z_low_mode,
3508
+ int64_t z_high_mode) {
3509
+ check_electric_cpml_inputs(
3510
+ ey, hx, hz, decay, curl, psi_x, psi_z, inv_kappa_x, b_x, c_x, inv_kappa_z, b_z, c_z, 0, 2, "ey");
3511
+ check_modulation_fields(ey, mod_cos, mod_sin, mod_omega);
3512
+ check_modulation_time(ey, modulation_time);
3513
+ check_rank3_shape(hx, "hx", ey.size(0), ey.size(1), ey.size(2) - 1);
3514
+ check_rank3_shape(hz, "hz", ey.size(0) - 1, ey.size(1), ey.size(2));
3515
+ check_spacing_vector(ey, inv_dx, 0, "inv_dx");
3516
+ check_spacing_vector(ey, inv_dz, 2, "inv_dz");
3517
+ torch::stable::accelerator::DeviceGuard guard(ey.get_device_index());
3518
+ const auto sizes = ey.sizes();
3519
+ const dim3 block = field_block3d();
3520
+ update_electric_ey_cpml_modulated_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
3521
+ static_cast<unsigned int>(sizes[0]),
3522
+ static_cast<unsigned int>(sizes[1]),
3523
+ static_cast<unsigned int>(sizes[2]),
3524
+ hx.mutable_data_ptr<float>(),
3525
+ hz.mutable_data_ptr<float>(),
3526
+ decay.mutable_data_ptr<float>(),
3527
+ curl.mutable_data_ptr<float>(),
3528
+ mod_cos.mutable_data_ptr<float>(),
3529
+ mod_sin.mutable_data_ptr<float>(),
3530
+ mod_omega.mutable_data_ptr<float>(),
3531
+ modulation_time.mutable_data_ptr<float>(),
3532
+ inv_kappa_x.mutable_data_ptr<float>(),
3533
+ b_x.mutable_data_ptr<float>(),
3534
+ c_x.mutable_data_ptr<float>(),
3535
+ inv_kappa_z.mutable_data_ptr<float>(),
3536
+ b_z.mutable_data_ptr<float>(),
3537
+ c_z.mutable_data_ptr<float>(),
3538
+ inv_dx.mutable_data_ptr<float>(),
3539
+ inv_dz.mutable_data_ptr<float>(),
3540
+ static_cast<int>(x_low_mode),
3541
+ static_cast<int>(x_high_mode),
3542
+ static_cast<int>(z_low_mode),
3543
+ static_cast<int>(z_high_mode),
3544
+ psi_x.mutable_data_ptr<float>(),
3545
+ psi_z.mutable_data_ptr<float>(),
3546
+ ey.mutable_data_ptr<float>());
3547
+ WITWIN_CUDA_CHECK();
3548
+ }
3549
+
3550
+ void update_electric_ez_cpml_modulated_cuda(
3551
+ torch::stable::Tensor ez,
3552
+ const torch::stable::Tensor& hx,
3553
+ const torch::stable::Tensor& hy,
3554
+ const torch::stable::Tensor& decay,
3555
+ const torch::stable::Tensor& curl,
3556
+ const torch::stable::Tensor& mod_cos,
3557
+ const torch::stable::Tensor& mod_sin,
3558
+ const torch::stable::Tensor& mod_omega,
3559
+ const torch::stable::Tensor& modulation_time,
3560
+ torch::stable::Tensor psi_x,
3561
+ torch::stable::Tensor psi_y,
3562
+ const torch::stable::Tensor& inv_kappa_x,
3563
+ const torch::stable::Tensor& b_x,
3564
+ const torch::stable::Tensor& c_x,
3565
+ const torch::stable::Tensor& inv_kappa_y,
3566
+ const torch::stable::Tensor& b_y,
3567
+ const torch::stable::Tensor& c_y,
3568
+ const torch::stable::Tensor& inv_dx,
3569
+ const torch::stable::Tensor& inv_dy,
3570
+ int64_t x_low_mode,
3571
+ int64_t x_high_mode,
3572
+ int64_t y_low_mode,
3573
+ int64_t y_high_mode) {
3574
+ check_electric_cpml_inputs(
3575
+ ez, hx, hy, decay, curl, psi_x, psi_y, inv_kappa_x, b_x, c_x, inv_kappa_y, b_y, c_y, 0, 1, "ez");
3576
+ check_modulation_fields(ez, mod_cos, mod_sin, mod_omega);
3577
+ check_modulation_time(ez, modulation_time);
3578
+ check_rank3_shape(hx, "hx", ez.size(0), ez.size(1) - 1, ez.size(2));
3579
+ check_rank3_shape(hy, "hy", ez.size(0) - 1, ez.size(1), ez.size(2));
3580
+ check_spacing_vector(ez, inv_dx, 0, "inv_dx");
3581
+ check_spacing_vector(ez, inv_dy, 1, "inv_dy");
3582
+ torch::stable::accelerator::DeviceGuard guard(ez.get_device_index());
3583
+ const auto sizes = ez.sizes();
3584
+ const dim3 block = field_block3d();
3585
+ update_electric_ez_cpml_modulated_kernel<<<field_grid3d(sizes[0], sizes[1], sizes[2], block), block, 0, current_cuda_stream()>>>(
3586
+ static_cast<unsigned int>(sizes[0]),
3587
+ static_cast<unsigned int>(sizes[1]),
3588
+ static_cast<unsigned int>(sizes[2]),
3589
+ hx.mutable_data_ptr<float>(),
3590
+ hy.mutable_data_ptr<float>(),
3591
+ decay.mutable_data_ptr<float>(),
3592
+ curl.mutable_data_ptr<float>(),
3593
+ mod_cos.mutable_data_ptr<float>(),
3594
+ mod_sin.mutable_data_ptr<float>(),
3595
+ mod_omega.mutable_data_ptr<float>(),
3596
+ modulation_time.mutable_data_ptr<float>(),
3597
+ inv_kappa_x.mutable_data_ptr<float>(),
3598
+ b_x.mutable_data_ptr<float>(),
3599
+ c_x.mutable_data_ptr<float>(),
3600
+ inv_kappa_y.mutable_data_ptr<float>(),
3601
+ b_y.mutable_data_ptr<float>(),
3602
+ c_y.mutable_data_ptr<float>(),
3603
+ inv_dx.mutable_data_ptr<float>(),
3604
+ inv_dy.mutable_data_ptr<float>(),
3605
+ static_cast<int>(x_low_mode),
3606
+ static_cast<int>(x_high_mode),
3607
+ static_cast<int>(y_low_mode),
3608
+ static_cast<int>(y_high_mode),
3609
+ psi_x.mutable_data_ptr<float>(),
3610
+ psi_y.mutable_data_ptr<float>(),
3611
+ ez.mutable_data_ptr<float>());
3612
+ WITWIN_CUDA_CHECK();
3613
+ }