cuwave 0.1.0__tar.gz → 0.3.0__tar.gz

{cuwave-0.1.0 → cuwave-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cuwave
-Version: 0.1.0
+Version: 0.3.0
 Summary: GPU finite-difference wave solver with differentiable adjoints
 Author-email: Leon Herrmann <herrmann.leon@pm.me>
 License-Expression: MIT
@@ -93,31 +93,11 @@ Additional benefits of **CuWave** are
 
 ## Install
 
-Dependencies are kept **lightweight**. Only CuPy is required beyond standard Python library.
-
 ```bash
-pip install cupy-cuda12x        # or cupy-cuda11x, to match your CUDA
-pip install cuwave               # or `pip install -e .` from a checkout
+pip install cuwave
 ```
 
-CuPy must be installed separately because the wheel depends on your CUDA toolkit;
-all remaining dependencies are declared in `pyproject.toml`.
-
-PyTorch is optional for the regularization via neural optimization; see [pytorch](https://pytorch.org/get-started/locally/) for the installation. Otherwise it is not needed.
-
-> [!NOTE]
-> Match PyTorch's CUDA version to CuPy's, or the two runtimes clash at the first kernel launch. 
-> With `cupy-cuda12x`:
-> ```bash
-> pip install torch --index-url https://download.pytorch.org/whl/cu128
-> ```
-
-The tests under `tests/` are `unittest` classes, but `pytest` is the recommended runner:
-
-```bash
-pip install pytest
-python -m pytest tests/ -q  # ~20 s on a GPU, ~3 s without: CUDA and PyTorch tests skip when unavailable
-```
+Requires an NVIDIA GPU and [CuPy](https://docs.cupy.dev/en/stable/install.html) matching your CUDA toolkit (e.g. `pip install cupy-cuda12x`), which is not pulled in automatically. For the full installation, including the optional PyTorch and running the tests, see [docs/install.md](https://github.com/cmpmech/cuwave/blob/main/docs/install.md).
 
 ## References
 

{cuwave-0.1.0 → cuwave-0.3.0}/README.md

@@ -67,31 +67,11 @@ Additional benefits of **CuWave** are
 
 ## Install
 
-Dependencies are kept **lightweight**. Only CuPy is required beyond standard Python library.
-
 ```bash
-pip install cupy-cuda12x        # or cupy-cuda11x, to match your CUDA
-pip install cuwave               # or `pip install -e .` from a checkout
+pip install cuwave
 ```
 
-CuPy must be installed separately because the wheel depends on your CUDA toolkit;
-all remaining dependencies are declared in `pyproject.toml`.
-
-PyTorch is optional for the regularization via neural optimization; see [pytorch](https://pytorch.org/get-started/locally/) for the installation. Otherwise it is not needed.
-
-> [!NOTE]
-> Match PyTorch's CUDA version to CuPy's, or the two runtimes clash at the first kernel launch. 
-> With `cupy-cuda12x`:
-> ```bash
-> pip install torch --index-url https://download.pytorch.org/whl/cu128
-> ```
-
-The tests under `tests/` are `unittest` classes, but `pytest` is the recommended runner:
-
-```bash
-pip install pytest
-python -m pytest tests/ -q  # ~20 s on a GPU, ~3 s without: CUDA and PyTorch tests skip when unavailable
-```
+Requires an NVIDIA GPU and [CuPy](https://docs.cupy.dev/en/stable/install.html) matching your CUDA toolkit (e.g. `pip install cupy-cuda12x`), which is not pulled in automatically. For the full installation, including the optional PyTorch and running the tests, see [docs/install.md](https://github.com/cmpmech/cuwave/blob/main/docs/install.md).
 
 ## References
 

{cuwave-0.1.0 → cuwave-0.3.0}/cuwave/anisotropic.py

@@ -25,7 +25,7 @@ import numpy.typing as npt
 
 from .boundary import Clamped, Traction, faces_with
 from .elastic import voigt
-from .wave import PAIRS, Simulation, apply_cell_weights, grid_block
+from .wave import PAIRS, Simulation, apply_cell_weights, axis_geometry, grid_block
 
 KERNEL_PATH = Path(__file__).parent / "kernels" / "anisotropic.cu"
 SENSITIVITY_PATH = Path(__file__).parent / "kernels" / "anisotropic_sensitivity.cu"
@@ -253,13 +253,6 @@ class AnisotropicElasticWave(Simulation):
         """Source scaling, unscaled since rho0 is already folded into the lumped inertia."""
         return 1.0
 
-    def axis_geometry(self) -> list:
-        """Extents and previous-axis strides, the step kernel's tail without the factors."""
-        geom = [self.Nx[0]]
-        for d in range(1, self.ndim):
-            geom += [self.Nx[d], self.strides[d - 1]]
-        return geom
-
     def gradient_fields(self, mat: dict) -> dict[str, cpt.NDArray]:
         """Nodal accumulators plus the cell one the stiffness density lands in first."""
         grads = {
@@ -285,7 +278,7 @@ class AnisotropicElasticWave(Simulation):
                 grads["material"],
                 grads["design"],
                 self.dtype(1.0 / 2.0**self.ndim),
-                *self.axis_geometry(),
+                *axis_geometry(self),
             ],
         )
         return {"mass": grads["mass"], "stiff": grads["stiff"]}
@@ -302,7 +295,7 @@ class AnisotropicElasticWave(Simulation):
             mat["stencil"],
             mass_factor,
             np.int32(self.comp_stride),
-            *self.axis_geometry(),
+            *axis_geometry(self),
         ]
 
         def gradient_step(u0, u1, u2, l1):
@@ -323,7 +316,7 @@ class AnisotropicElasticWave(Simulation):
             self.dtype(sign * self.density * volume / (2.0 * self.dt) ** 2),
             self.dtype(-sign),
             np.int32(self.comp_stride),
-            *self.axis_geometry(),
+            *axis_geometry(self),
         ]
 
         def frechet_step(u0, u1, u2):

{cuwave-0.1.0 → cuwave-0.3.0}/cuwave/elastic.py

@@ -26,13 +26,12 @@ from .wave import (
     Simulation,
     apply_cell_weights,
     axis_geometry,
-    component_weights,
     grid_block,
     pair_average,
     pair_average_adjoint,
-    pair_weights,
     point_average,
     point_average_adjoint,
+    wall_weights,
 )
 
 KERNEL_PATH = Path(__file__).parent / "kernels" / "elastic.cu"
@@ -160,7 +159,7 @@ class ElasticWave(Simulation):
         for c in range(self.ncomp):
             mass = (
                 self.density
-                * component_weights(self, c)
+                * wall_weights(self, (c,))
                 * point_average(self, gamma, c)
             )
             minv[c] = 1.0 / cp.maximum(mass, cp.finfo(self.dtype).tiny)
@@ -178,7 +177,7 @@ class ElasticWave(Simulation):
         if self.ndim > 1:
             gshear = cp.zeros((self.npairs, *self.Nx_padded), dtype=self.dtype)
             for p, axes in enumerate(PAIRS[self.ndim][self.ndim :]):
-                gshear[p] = pair_weights(self, axes) * pair_average(self, gamma, axes)
+                gshear[p] = wall_weights(self, axes) * pair_average(self, gamma, axes)
             mat["gshear"] = cp.ascontiguousarray(gshear)
         if self.damping is not None:
             mat["damping"] = self.damping
@@ -269,12 +268,12 @@ class ElasticWave(Simulation):
         gamma = grads["design"]
         g_mass = cp.zeros(self.Nx_padded, dtype=self.dtype)
         for c in range(self.ncomp):
-            density = grads["mass"][c] * component_weights(self, c) * self.density
+            density = grads["mass"][c] * wall_weights(self, (c,)) * self.density
             g_mass += point_average_adjoint(self, density, c)
         # the normal density sits on the nodes, so its chain rule is the weight alone
         g_stiff = apply_cell_weights(self, grads["normal"].copy())
         for p, axes in enumerate(PAIRS[self.ndim][self.ndim :]):
-            density = grads["shear"][p] * pair_weights(self, axes)
+            density = grads["shear"][p] * wall_weights(self, axes)
             g_stiff += pair_average_adjoint(self, density, gamma, axes)
         return {"mass": g_mass, "stiff": g_stiff}
 

{cuwave-0.1.0 → cuwave-0.3.0}/cuwave/kernels/common.cuh

@@ -69,6 +69,22 @@ excitation_kernel(real_t *__restrict__ u, const real_t *__restrict__ source,
   }
 }
 
+// ------------------------------------------------------------------------------------
+__global__ void adjoint_excitation_kernel(
+    real_t *__restrict__ l2, const real_t *__restrict__ signal,
+    const int offset, const int *__restrict__ lin_index, const int num_sensors,
+    const real_t *__restrict__ weight, real_t *__restrict__ g_mass,
+    const real_t *__restrict__ u1, const real_t mf) {
+  const int idx = blockIdx.x * blockDim.x + threadIdx.x;
+  if (idx < num_sensors) {
+    const int n = lin_index[idx];
+    const real_t load = weight[idx] * signal[offset + idx];
+    atomicAdd(&l2[n], load);
+    // the injected load is part of the second time difference of lambda
+    atomicAdd(&g_mass[n], -mf * u1[n] * load);
+  }
+}
+
 // ------------------------------------------------------------------------------------
 __global__ void get_signal_kernel(const real_t *__restrict__ u,
                                   real_t *__restrict__ um, const int offset,

{cuwave-0.1.0 → cuwave-0.3.0}/cuwave/kernels/scalar.cu

@@ -2,6 +2,7 @@
 // Compile-time configuration this file responds to:
 //   NDIM = 1 | 2 | 3
 //   USE_DAMPING
+//   USE_DOMAIN
 
 // spatial finite difference stencil for Laplacian
 __device__ __forceinline__ real_t flux_divergence_axis(
@@ -25,6 +26,81 @@ __device__ __forceinline__ real_t flux_divergence_axis(
   return factor * (Dp * gp - Dm * gm); // outer grad (incl. inner grad)
 }
 
+// ----------------------------------- domain helpers
+#ifdef USE_DOMAIN
+#define DIRICHLET 7 // the radius code of a cell open onto a node held at zero
+
+// the same with each cell at its own radius, zero for a cell leaving the domain
+__device__ __forceinline__ real_t flux_divergence_cells(
+    const real_t *__restrict__ u1, const real_t *__restrict__ stiff,
+    const int idx, const int s, const real_t uc, const real_t sc,
+    const real_t factor, const int rp, const int rm) {
+  real_t Dp = 0, Dm = 0; // a closed cell carries no flux
+  if (rp == DIRICHLET)
+    Dp = -uc * sc; // onto a zero halfway across, at the node's own stiffness
+  else if (rp) {
+    const real_t sp = stiff[idx + s];
+    Dp = OP_W(rp, 1) * (u1[idx + s] - uc);
+#pragma unroll
+    for (int k = 2; k <= STENCIL_RADIUS; ++k)
+      if (k <= rp)
+        Dp += OP_W(rp, k) * (u1[idx + k * s] - u1[idx - (k - 1) * s]);
+    Dp *= sc * sp / (sc + sp); // harmonic mean
+  }
+  if (rm == DIRICHLET)
+    Dm = uc * sc;
+  else if (rm) {
+    const real_t sm = stiff[idx - s];
+    Dm = OP_W(rm, 1) * (uc - u1[idx - s]);
+#pragma unroll
+    for (int k = 2; k <= STENCIL_RADIUS; ++k)
+      if (k <= rm)
+        Dm += OP_W(rm, k) * (u1[idx + (k - 1) * s] - u1[idx - k * s]);
+    Dm *= sc * sm / (sc + sm); // harmonic mean
+  }
+  return factor * (Dp - Dm);
+}
+#endif
+
+#ifdef USE_DOMAIN
+#define BLOCK_X (tile & 1023) // one block per domain tile, 10 bits an axis
+#define BLOCK_Y ((tile >> 10) & 1023)
+#define BLOCK_Z ((tile >> 20) & 1023)
+#else
+#define BLOCK_X blockIdx.x
+#define BLOCK_Y blockIdx.y
+#define BLOCK_Z blockIdx.z
+#endif
+
+#ifdef USE_DOMAIN
+#define CELLS(s, f, d)                                                         \
+  flux_divergence_cells(u1, stiff, idx, s, uc, sc, f, (code >> 6 * (d)) & 7,   \
+                        (code >> (6 * (d) + 3)) & 7)
+
+// out of line, so the rare wall node costs the deep ones no registers
+__device__ __noinline__ real_t
+wall_laplacian(const real_t *__restrict__ u1, const real_t *__restrict__ stiff,
+               const int idx, const real_t uc, const real_t sc, const int code,
+               const real_t f0
+#if NDIM >= 2
+               ,
+               const real_t f1, const int s0
+#endif
+#if NDIM >= 3
+               ,
+               const real_t f2, const int s1
+#endif
+) {
+#if NDIM == 1
+  return CELLS(1, f0, 0);
+#elif NDIM == 2
+  return CELLS(s0, f0, 0) + CELLS(1, f1, 1);
+#elif NDIM == 3
+  return CELLS(s0, f0, 0) + CELLS(s1, f1, 1) + CELLS(1, f2, 2);
+#endif
+}
+#endif
+
 // ----------------------------- boundary condition helper
 #if NDIM == 1
 #define BC_PARAMS const int N0
@@ -80,6 +156,9 @@ fd_kernel(const real_t *__restrict__ u0, const real_t *__restrict__ u1,
           const real_t *__restrict__ minv, const int derive_inertia,
 #ifdef USE_DAMPING
           const real_t *__restrict__ damping, const real_t dt,
+#endif
+#ifdef USE_DOMAIN
+          const int *__restrict__ cells, const int *__restrict__ tiles,
 #endif
           const real_t f0, const int N0
 #if NDIM >= 2
@@ -91,23 +170,26 @@ fd_kernel(const real_t *__restrict__ u0, const real_t *__restrict__ u1,
           const real_t f2, const int N2, const int s1
 #endif
 ) {
+#ifdef USE_DOMAIN
+  const int tile = tiles[blockIdx.x];
+#endif
 #if NDIM == 1
-  const int a0 = blockIdx.x * blockDim.x + threadIdx.x;
+  const int a0 = BLOCK_X * blockDim.x + threadIdx.x;
   if (!(a0 > 0 && a0 < N0 - 1))
     return;
   const int idx = a0;
   const int r0 = CLOSURE(a0, N0);
 #elif NDIM == 2
-  const int a1 = blockIdx.x * blockDim.x + threadIdx.x;
-  const int a0 = blockIdx.y * blockDim.y + threadIdx.y;
+  const int a1 = BLOCK_X * blockDim.x + threadIdx.x;
+  const int a0 = BLOCK_Y * blockDim.y + threadIdx.y;
   if (!(a0 > 0 && a0 < N0 - 1 && a1 > 0 && a1 < N1 - 1))
     return;
   const int idx = a0 * s0 + a1;
   const int r0 = CLOSURE(a0, N0), r1 = CLOSURE(a1, N1);
 #elif NDIM == 3
-  const int a2 = blockIdx.x * blockDim.x + threadIdx.x;
-  const int a1 = blockIdx.y * blockDim.y + threadIdx.y;
-  const int a0 = blockIdx.z * blockDim.z + threadIdx.z;
+  const int a2 = BLOCK_X * blockDim.x + threadIdx.x;
+  const int a1 = BLOCK_Y * blockDim.y + threadIdx.y;
+  const int a0 = BLOCK_Z * blockDim.z + threadIdx.z;
   if (!(a0 > 0 && a0 < N0 - 1 && a1 > 0 && a1 < N1 - 1 && a2 > 0 &&
         a2 < N2 - 1))
     return;
@@ -117,7 +199,6 @@ fd_kernel(const real_t *__restrict__ u0, const real_t *__restrict__ u1,
 
   const real_t uc = u1[idx];    // load once
   const real_t sc = stiff[idx]; // load once
-
 #if NDIM == 1
   real_t laplacian = flux_divergence_axis(u1, stiff, idx, 1, uc, sc, f0, r0);
 #elif NDIM == 2
@@ -128,6 +209,25 @@ fd_kernel(const real_t *__restrict__ u0, const real_t *__restrict__ u1,
                      flux_divergence_axis(u1, stiff, idx, s1, uc, sc, f1, r1) +
                      flux_divergence_axis(u1, stiff, idx, 1, uc, sc, f2, r2);
 #endif
+#ifdef USE_DOMAIN
+  // bit 30 marks a tile touching the wall, so the branch is uniform per block
+  if (tile >> 30) {
+    const int code = cells[idx]; // 0 outside, bit 30 deep, else the cell radii
+    if (!code)
+      return; // outside the domain nothing is stepped
+#if NDIM == 1
+    if (!(code >> 30))
+      laplacian = wall_laplacian(u1, stiff, idx, uc, sc, code, f0);
+#elif NDIM == 2
+    if (!(code >> 30))
+      laplacian = wall_laplacian(u1, stiff, idx, uc, sc, code, f0, f1, s0);
+#elif NDIM == 3
+    if (!(code >> 30))
+      laplacian =
+          wall_laplacian(u1, stiff, idx, uc, sc, code, f0, f1, s0, f2, s1);
+#endif
+  }
+#endif
 
   const real_t mi = derive_inertia ? 1.f / sc : minv[idx];
 #ifdef USE_DAMPING

cuwave-0.3.0/cuwave/kernels/scalar_sensitivity.cu

@@ -0,0 +1,302 @@
+// Prepended by wave.compile_kernels: stencils.preamble, then common.cuh.
+// Compile-time configuration this file responds to:
+//   NDIM = 1 | 2 | 3
+//   USE_DAMPING
+
+// ------------------------------- interior guard macros
+#if NDIM == 1
+#define INTERIOR_OR_RETURN                                                     \
+  const int a0 = blockIdx.x * blockDim.x + threadIdx.x;                        \
+  if (!(a0 > 0 && a0 < N0 - 1))                                                \
+    return;                                                                    \
+  const int idx = a0
+#define AXIS_RADII const int r0 = CLOSURE(a0, N0)
+#define AXIS_OFFSETS const int o0 = 1
+#elif NDIM == 2
+#define INTERIOR_OR_RETURN                                                     \
+  const int a1 = blockIdx.x * blockDim.x + threadIdx.x;                        \
+  const int a0 = blockIdx.y * blockDim.y + threadIdx.y;                        \
+  if (!(a0 > 0 && a0 < N0 - 1 && a1 > 0 && a1 < N1 - 1))                       \
+    return;                                                                    \
+  const int idx = a0 * s0 + a1
+#define AXIS_RADII const int r0 = CLOSURE(a0, N0), r1 = CLOSURE(a1, N1)
+#define AXIS_OFFSETS const int o0 = s0, o1 = 1
+#elif NDIM == 3
+#define INTERIOR_OR_RETURN                                                     \
+  const int a2 = blockIdx.x * blockDim.x + threadIdx.x;                        \
+  const int a1 = blockIdx.y * blockDim.y + threadIdx.y;                        \
+  const int a0 = blockIdx.z * blockDim.z + threadIdx.z;                        \
+  if (!(a0 > 0 && a0 < N0 - 1 && a1 > 0 && a1 < N1 - 1 && a2 > 0 &&            \
+        a2 < N2 - 1))                                                          \
+    return;                                                                    \
+  const int idx = a0 * s0 + a1 * s1 + a2
+#define AXIS_RADII                                                             \
+  const int r0 = CLOSURE(a0, N0), r1 = CLOSURE(a1, N1), r2 = CLOSURE(a2, N2)
+#define AXIS_OFFSETS const int o0 = s0, o1 = s1, o2 = 1
+#endif
+
+// ---------------------------------- gradient helpers
+__device__ __forceinline__ real_t stiffness_gradient_axis(
+    const real_t *__restrict__ u1, const real_t *__restrict__ l1,
+    const real_t *__restrict__ stiff, const int idx, const int s,
+    const real_t uc, const real_t lc, const real_t sc, const real_t factor,
+    const int r) {
+  const real_t sp = stiff[idx + s];                     // plus of sc
+  const real_t sm = stiff[idx - s];                     // minus of sc
+  const real_t dgp = sp * sp / ((sc + sp) * (sc + sp)); // d(harmonic mean)/dsc
+  const real_t dgm = sm * sm / ((sc + sm) * (sc + sm)); // d(harmonic mean)/dsc
+  real_t Dp = OP_W(r, 1) * (u1[idx + s] - uc);          // inner grad
+  real_t Dm = OP_W(r, 1) * (uc - u1[idx - s]);          // inner grad
+#pragma unroll
+  for (int k = 2; k <= STENCIL_RADIUS; ++k)
+    if (k <= r) {
+      Dp += OP_W(r, k) * (u1[idx + k * s] - u1[idx - (k - 1) * s]);
+      Dm += OP_W(r, k) * (u1[idx + (k - 1) * s] - u1[idx - k * s]);
+    }
+  return factor * (dgp * Dp * (l1[idx + s] - lc) +
+                   dgm * Dm * (lc - l1[idx - s])); // both cells of the node
+}
+
+// the adjoint step's flux divergence and the stiffness gradient of one axis,
+// sharing the cell stiffnesses
+__device__ __forceinline__ void adjoint_gradient_axis(
+    const real_t *__restrict__ u1, const real_t *__restrict__ l1,
+    const real_t *__restrict__ stiff, const int idx, const int s,
+    const real_t uc, const real_t lc, const real_t sc, const real_t factor,
+    const int r, real_t &div_l, real_t &g) {
+  const real_t sp = stiff[idx + s];                     // plus of sc
+  const real_t sm = stiff[idx - s];                     // minus of sc
+  const real_t gp = sc * sp / (sc + sp);                // harmonic mean
+  const real_t gm = sc * sm / (sc + sm);                // harmonic mean
+  const real_t dgp = sp * sp / ((sc + sp) * (sc + sp)); // d(harmonic mean)/dsc
+  const real_t dgm = sm * sm / ((sc + sm) * (sc + sm)); // d(harmonic mean)/dsc
+  const real_t lp = l1[idx + s];
+  const real_t lm = l1[idx - s];
+  real_t Lp = OP_W(r, 1) * (lp - lc);          // inner grad of lambda
+  real_t Lm = OP_W(r, 1) * (lc - lm);          // inner grad of lambda
+  real_t Dp = OP_W(r, 1) * (u1[idx + s] - uc); // inner grad of u
+  real_t Dm = OP_W(r, 1) * (uc - u1[idx - s]); // inner grad of u
+#pragma unroll
+  for (int k = 2; k <= STENCIL_RADIUS; ++k)
+    if (k <= r) {
+      Lp += OP_W(r, k) * (l1[idx + k * s] - l1[idx - (k - 1) * s]);
+      Lm += OP_W(r, k) * (l1[idx + (k - 1) * s] - l1[idx - k * s]);
+      Dp += OP_W(r, k) * (u1[idx + k * s] - u1[idx - (k - 1) * s]);
+      Dm += OP_W(r, k) * (u1[idx + (k - 1) * s] - u1[idx - k * s]);
+    }
+  div_l += factor * (Lp * gp - Lm * gm); // the step's flux divergence
+  g += factor * (dgp * Dp * (lp - lc) + dgm * Dm * (lc - lm));
+}
+
+// byte-identical to scalar.cu: each file is its own compilation unit
+__device__ __forceinline__ real_t flux_divergence_axis(
+    const real_t *__restrict__ u1, const real_t *__restrict__ stiff,
+    const int idx, const int s, const real_t uc, const real_t sc,
+    const real_t factor, const int r) {
+  const real_t sp = stiff[idx + s];            // plus of sc
+  const real_t sm = stiff[idx - s];            // minus of sc
+  const real_t gp = sc * sp / (sc + sp);       // harmonic mean
+  const real_t gm = sc * sm / (sc + sm);       // harmonic mean
+  real_t Dp = OP_W(r, 1) * (u1[idx + s] - uc); // initialization: inner grad
+  real_t Dm = OP_W(r, 1) * (uc - u1[idx - s]); // initialization: inner grad
+#pragma unroll
+  for (int k = 2; k <= STENCIL_RADIUS; ++k)
+    if (k <= r) {
+      Dp +=
+          OP_W(r, k) * (u1[idx + k * s] - u1[idx - (k - 1) * s]); // inner grad
+      Dm +=
+          OP_W(r, k) * (u1[idx + (k - 1) * s] - u1[idx - k * s]); // inner grad
+    }
+  return factor * (Dp * gp - Dm * gm); // outer grad (incl. inner grad)
+}
+
+// -------------------------------------- kernels
+extern "C" {
+
+// ------------------------------------------------------------------------------------
+__global__ void
+gradient_kernel(real_t *__restrict__ g_mass, real_t *__restrict__ g_stiff,
+                const real_t *__restrict__ u0, const real_t *__restrict__ u1,
+                const real_t *__restrict__ u2, const real_t *__restrict__ l1,
+                const real_t *__restrict__ stiff, const real_t inv_dt2,
+                const real_t F0, const int N0
+#if NDIM >= 2
+                ,
+                const real_t F1, const int N1, const int s0
+#endif
+#if NDIM >= 3
+                ,
+                const real_t F2, const int N2, const int s1
+#endif
+) {
+  INTERIOR_OR_RETURN;
+  AXIS_RADII;
+
+  const real_t uc = u1[idx];
+  const real_t lc = l1[idx];
+
+  // dJ/dmass: no neighbour and no material load
+  g_mass[idx] -= inv_dt2 * lc * (u2[idx] - 2.f * uc + u0[idx]);
+
+  // dJ/dstiff: scalar.cu's harmonic cell mean differentiated in place
+  const real_t sc = stiff[idx];
+#if NDIM == 1
+  const real_t g =
+      stiffness_gradient_axis(u1, l1, stiff, idx, 1, uc, lc, sc, F0, r0);
+#elif NDIM == 2
+  const real_t g =
+      stiffness_gradient_axis(u1, l1, stiff, idx, s0, uc, lc, sc, F0, r0) +
+      stiffness_gradient_axis(u1, l1, stiff, idx, 1, uc, lc, sc, F1, r1);
+#elif NDIM == 3
+  const real_t g =
+      stiffness_gradient_axis(u1, l1, stiff, idx, s0, uc, lc, sc, F0, r0) +
+      stiffness_gradient_axis(u1, l1, stiff, idx, s1, uc, lc, sc, F1, r1) +
+      stiffness_gradient_axis(u1, l1, stiff, idx, 1, uc, lc, sc, F2, r2);
+#endif
+  g_stiff[idx] -= g;
+}
+
+// ------------------------------------------------------------------------------------
+__global__ void frechet_kernel(real_t *__restrict__ acc_mass,
+                               real_t *__restrict__ acc_stiff,
+                               const real_t *__restrict__ u0,
+                               const real_t *__restrict__ u1,
+                               const real_t *__restrict__ u2, const real_t ft,
+                               const real_t f0, const int N0
+#if NDIM >= 2
+                               ,
+                               const real_t f1, const int N1, const int s0
+#endif
+#if NDIM >= 3
+                               ,
+                               const real_t f2, const int N2, const int s1
+#endif
+) {
+  INTERIOR_OR_RETURN;
+  AXIS_OFFSETS;
+
+  const real_t dudt = u2[idx] - u0[idx];
+  acc_mass[idx] += ft * dudt * dudt;
+
+  const real_t g0 = u1[idx + o0] - u1[idx - o0];
+  real_t sum = f0 * g0 * g0;
+#if NDIM >= 2
+  const real_t g1 = u1[idx + o1] - u1[idx - o1];
+  sum += f1 * g1 * g1;
+#endif
+#if NDIM >= 3
+  const real_t g2 = u1[idx + o2] - u1[idx - o2];
+  sum += f2 * g2 * g2;
+#endif
+  acc_stiff[idx] += sum;
+}
+
+// ------------------------------------------------------------------------------------
+__global__ void adjoint_gradient_kernel(
+    real_t *__restrict__ l0, const real_t *__restrict__ l1,
+    real_t *__restrict__ g_mass, real_t *__restrict__ g_stiff,
+    const real_t *__restrict__ u1, const real_t *__restrict__ stiff,
+    const real_t *__restrict__ minv, const int derive_inertia,
+#ifdef USE_DAMPING
+    const real_t *__restrict__ damping, const real_t dt,
+#endif
+    const real_t mf, const real_t f0, const int N0
+#if NDIM >= 2
+    ,
+    const real_t f1, const int N1, const int s0
+#endif
+#if NDIM >= 3
+    ,
+    const real_t f2, const int N2, const int s1
+#endif
+) {
+  INTERIOR_OR_RETURN;
+  AXIS_RADII;
+
+  const real_t uc = u1[idx];
+  const real_t lc = l1[idx];
+  const real_t sc = stiff[idx];
+  real_t div_l = 0, g = 0;
+#if NDIM == 1
+  adjoint_gradient_axis(u1, l1, stiff, idx, 1, uc, lc, sc, f0, r0, div_l, g);
+#elif NDIM == 2
+  adjoint_gradient_axis(u1, l1, stiff, idx, s0, uc, lc, sc, f0, r0, div_l, g);
+  adjoint_gradient_axis(u1, l1, stiff, idx, 1, uc, lc, sc, f1, r1, div_l, g);
+#elif NDIM == 3
+  adjoint_gradient_axis(u1, l1, stiff, idx, s0, uc, lc, sc, f0, r0, div_l, g);
+  adjoint_gradient_axis(u1, l1, stiff, idx, s1, uc, lc, sc, f1, r1, div_l, g);
+  adjoint_gradient_axis(u1, l1, stiff, idx, 1, uc, lc, sc, f2, r2, div_l, g);
+#endif
+
+  const real_t mi = derive_inertia ? 1.f / sc : minv[idx];
+  const real_t lo = l0[idx];
+#ifdef USE_DAMPING
+  const real_t beta = 0.5f * mi * damping[idx] * dt;
+  const real_t ln = (2.f * lc - lo * (1.f - beta) + mi * div_l) / (1.f + beta);
+#else
+  const real_t ln = -lo + 2.f * lc + mi * div_l;
+#endif
+  l0[idx] = ln;
+  // dJ/dmass by parts in time: u against the second difference of lambda
+  g_mass[idx] -= mf * uc * (ln - 2.f * lc + lo);
+  g_stiff[idx] -= mf * g;
+}
+
+// ------------------------------------------------------------------------------------
+__global__ void superposed_kernel(
+    const real_t *__restrict__ u0, const real_t *__restrict__ u1,
+    real_t *__restrict__ u2, real_t *__restrict__ acc_mass,
+    real_t *__restrict__ acc_stiff, const real_t *__restrict__ stiff,
+    const real_t *__restrict__ minv, const int derive_inertia, const real_t ft,
+    const real_t fs, const real_t f0, const int N0
+#if NDIM >= 2
+    ,
+    const real_t f1, const int N1, const int s0
+#endif
+#if NDIM >= 3
+    ,
+    const real_t f2, const int N2, const int s1
+#endif
+) {
+  INTERIOR_OR_RETURN;
+  AXIS_RADII;
+  AXIS_OFFSETS;
+
+  const real_t uc = u1[idx];
+  const real_t sc = stiff[idx];
+#if NDIM == 1
+  const real_t laplacian =
+      flux_divergence_axis(u1, stiff, idx, 1, uc, sc, f0, r0);
+#elif NDIM == 2
+  const real_t laplacian =
+      flux_divergence_axis(u1, stiff, idx, s0, uc, sc, f0, r0) +
+      flux_divergence_axis(u1, stiff, idx, 1, uc, sc, f1, r1);
+#elif NDIM == 3
+  const real_t laplacian =
+      flux_divergence_axis(u1, stiff, idx, s0, uc, sc, f0, r0) +
+      flux_divergence_axis(u1, stiff, idx, s1, uc, sc, f1, r1) +
+      flux_divergence_axis(u1, stiff, idx, 1, uc, sc, f2, r2);
+#endif
+
+  // the mass density of the triplet before, whose oldest slot u2 still holds
+  const real_t dudt = uc - u2[idx];
+  acc_mass[idx] += ft * dudt * dudt;
+
+  // the stiffness density of this triplet, its middle slot being u1
+  const real_t g0 = u1[idx + o0] - u1[idx - o0];
+  real_t sum = f0 * g0 * g0;
+#if NDIM >= 2
+  const real_t g1 = u1[idx + o1] - u1[idx - o1];
+  sum += f1 * g1 * g1;
+#endif
+#if NDIM >= 3
+  const real_t g2 = u1[idx + o2] - u1[idx - o2];
+  sum += f2 * g2 * g2;
+#endif
+  acc_stiff[idx] += fs * sum;
+
+  const real_t mi = derive_inertia ? 1.f / sc : minv[idx];
+  u2[idx] = -u0[idx] + 2.f * uc + mi * laplacian;
+}
+
+} // extern "C"