Braindecode 1.8.1.dev174738705__tar.gz → 1.8.1.dev174812945__tar.gz

{braindecode-1.8.1.dev174738705/braindecode.egg-info → braindecode-1.8.1.dev174812945}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: braindecode
-Version: 1.8.1.dev174738705
+Version: 1.8.1.dev174812945
 Summary: Deep learning software to decode EEG, ECG or MEG signals
 Author-email: Robin Tibor Schirrmeister <robintibor@gmail.com>, Bruno Aristimunha Pinto <b.aristimunha@gmail.com>, Alexandre Gramfort <agramfort@meta.com>
 Maintainer-email: Alexandre Gramfort <agramfort@meta.com>, Bruno Aristimunha Pinto <b.aristimunha@gmail.com>, Robin Tibor Schirrmeister <robintibor@gmail.com>

{braindecode-1.8.1.dev174738705 → braindecode-1.8.1.dev174812945}/braindecode/datasets/base.py

@@ -279,7 +279,7 @@ def _compact_fif_tags(inst):
     return inst
 
 
-def _read_fif_window(raw, start, stop):
+def _read_fif_window(raw, start, stop, willneed=False):
     """Samples ``start:stop`` of a lazy FIF raw, copied from the file in one go.
 
     mne reads, casts and calibrates the FIF data buffers of a range one at a
@@ -290,6 +290,10 @@ def _read_fif_window(raw, start, stop):
     times ``raw._cals``), so the values equal ``raw._getitem``'s. Returns None
     (the caller uses mne) unless ``raw`` is a lazy, single-file, uncompressed
     FIF raw of float32 buffers without projector, compensation or gaps.
+
+    With ``willneed=True`` nothing is read and None is returned: the OS is
+    only asked to start reading the window's bytes into the page cache in the
+    background (``posix_fadvise(POSIX_FADV_WILLNEED)``).
     """
     if (
         not isinstance(raw, mne.io.Raw)
@@ -316,6 +320,18 @@ def _read_fif_window(raw, start, stop):
     if not len(rows) or (rows[:, 1] != FIFF.FIFFT_FLOAT).any():
         return None
     size, pos = rows[:, 2].tolist(), rows[:, 4].tolist()
+    if willneed:  # only a hint: any error is left to the read itself
+        try:
+            with open(fname, "rb") as f:
+                os.posix_fadvise(
+                    f.fileno(),
+                    pos[0],
+                    pos[-1] + 16 + size[-1] - pos[0],
+                    os.POSIX_FADV_WILLNEED,
+                )
+        except OSError:
+            pass
+        return None
     # mmap: no copy through a 4 MB read buffer, and a cold file is read around
     # the window by the kernel (as mne's 4 MB reads did), not window by window
     with open(fname, "rb") as f, mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as m:
@@ -1362,7 +1378,8 @@ class BaseConcatDataset(ConcatDataset, HubDatasetMixin, Generic[T]):
     def __len__(self):
         return self._ensure_cumulative_sizes()[-1]
 
-    def _get_single_item(self, idx: int):
+    def _locate(self, idx: int):
+        """The dataset that holds item ``idx`` and the item's index in it."""
         if idx < 0:
             if -idx > len(self):
                 raise ValueError(
@@ -1376,7 +1393,33 @@ class BaseConcatDataset(ConcatDataset, HubDatasetMixin, Generic[T]):
             sample_idx = idx
         else:
             sample_idx = idx - cumulative_sizes[dataset_idx - 1]
-        return self.datasets[dataset_idx][sample_idx]
+        return self.datasets[dataset_idx], sample_idx
+
+    def _get_single_item(self, idx: int):
+        ds, i = self._locate(idx)
+        return ds[i]
+
+    def __getitems__(self, indices):
+        """Items ``indices``, as ``[self[i] for i in indices]``.
+
+        A :class:`~torch.utils.data.DataLoader` fetches each batch through
+        this. The lazy FIF windows of the batch are first requested from the
+        OS all at once (``posix_fadvise(POSIX_FADV_WILLNEED)``, where it
+        exists), so their reads from disk overlap instead of running one after
+        another: on a network file system a batch read from cold files then
+        costs about one round trip, not one per window.
+        """
+        # ponytail: advises the whole batch at once; chunk it if a batch's
+        # windows ever outgrow the page cache
+        if hasattr(os, "posix_fadvise"):
+            for idx in indices:
+                if not isinstance(idx, (int, np.integer)):
+                    continue  # e.g. a SequenceSampler tuple: read, not advised
+                ds, i = self._locate(idx)
+                if isinstance(ds, EEGWindowsDataset) and ds._zarr_data is None:
+                    _, start, stop = ds.crop_inds[i].tolist()
+                    _read_fif_window(ds.raw, start, stop, willneed=True)
+        return [self[idx] for idx in indices]
 
     def __getitem__(self, idx: int | list):
         """

{braindecode-1.8.1.dev174738705 → braindecode-1.8.1.dev174812945}/braindecode/functional/_real_dft.py

@@ -12,8 +12,10 @@ never downcast, so callers wrap the numerically sensitive parts with
 #
 # License: BSD (3-clause)
 import functools
+from typing import Optional
 
 import torch
+from torch import nn
 
 
 def needs_real_dft(x: torch.Tensor) -> bool:
@@ -136,3 +138,56 @@ def hilbert_freq_real(x: torch.Tensor, forward_fourier: bool) -> torch.Tensor:
         return torch.stack((out_real, out_imag), dim=-1)
 
     return compute(x)
+
+
+@torch.jit.unused
+def fft_conv1d_real(
+    x: torch.Tensor, weight: torch.Tensor, bias: Optional[torch.Tensor]
+) -> torch.Tensor:
+    """Real-valued equivalent of :func:`braindecode.functional.fft_conv1d`.
+
+    Overlap-save: the input is cut into blocks of ``n`` samples (the power of
+    two ``>= 2 * kernel_size``) that advance by ``n - kernel_size + 1`` outputs,
+    each block's DFT is a matmul against the cosine/sine basis, and per
+    frequency the complex product with the kernels, summed over the input
+    channels, is one real matmul.
+    """
+
+    @fp32_island
+    def compute(x, weight):
+        n_times, kernel_size = x.shape[-1], weight.shape[-1]
+        n = 1 << (2 * kernel_size - 1).bit_length()
+        hop = n - kernel_size + 1
+        n_blocks = -(-n_times // hop)
+        n_freqs = n // 2 + 1
+        # "same" alignment: torch pads the extra zero of an even kernel on the right
+        left = (kernel_size - 1) // 2
+        x = nn.functional.pad(x, (left, (n_blocks + 1) * hop - n_times - left))
+        hops = x.unflatten(-1, (n_blocks + 1, hop))
+        # block i = hop i + the first kernel_size - 1 samples of hop i + 1
+        blocks = torch.cat([hops[:, :, :-1], hops[:, :, 1:, : kernel_size - 1]], -1)
+        cosine, sine = real_dft_basis(n, device=x.device, dtype=x.dtype)
+        basis = torch.cat([cosine, -sine]).transpose(0, 1)  # -> [Re | Im] of rfft
+        x_f = blocks @ basis  # (batch, in, blocks, 2 * n_freqs)
+        w_f = nn.functional.pad(weight.to(x), (0, n - kernel_size)) @ basis
+        w_re, w_im = w_f[..., :n_freqs], w_f[..., n_freqs:]
+        # conv1d is a cross-correlation: x_f * conj(w_f), summed over the input
+        # channels. Per frequency [Re x, Im x] @ [[Re w, -Im w], [Im w, Re w]].
+        w_f = torch.cat([torch.cat([w_re, -w_im]), torch.cat([w_im, w_re])], 1)
+        x_f = x_f.unflatten(-1, (2, n_freqs)).permute(4, 0, 2, 3, 1).flatten(1, 2)
+        y_f = torch.bmm(x_f.flatten(2), w_f.permute(2, 1, 0))  # (freq, ., 2 * out)
+        y_f = y_f.unflatten(1, (x.shape[0], n_blocks)).unflatten(-1, (2, -1))
+        y_f = y_f.permute(1, 4, 2, 3, 0).flatten(
+            -2
+        )  # (batch, out, blocks, 2 * n_freqs)
+        # inverse DFT of the first `hop` samples, the ones without wrap-around
+        scale = torch.full((n_freqs, 1), 2.0 / n, dtype=x.dtype, device=x.device)
+        scale[0] = 1.0 / n
+        scale[-1] = 1.0 / n  # Nyquist (n is even)
+        inverse = torch.cat([scale * cosine[:, :hop], -scale * sine[:, :hop]])
+        return (y_f @ inverse).flatten(-2)[..., :n_times]
+
+    y = compute(x, weight)
+    if bias is not None:
+        y = y + bias.to(y).unsqueeze(-1)
+    return y.to(x)

{braindecode-1.8.1.dev174738705 → braindecode-1.8.1.dev174812945}/braindecode/functional/functions.py

@@ -62,9 +62,10 @@ def fft_conv1d(
 
     Equal to the direct convolution up to float rounding: the FFT is long
     enough for a linear (not circular) convolution. Its cost barely grows with
-    the kernel size, so it is faster than the direct one for long kernels on
-    CPU (see :func:`prefer_fft_conv`). float16/bfloat16 inputs are computed in
-    float32 (:func:`spectral_input`) and cast back.
+    the kernel size, so it is faster than the direct one for long kernels
+    (see :func:`prefer_fft_conv`). float16/bfloat16 inputs are computed in
+    float32 (:func:`spectral_input`) and cast back. On Intel Gaudi (HPU, no
+    complex dtype) the DFTs are real matmuls over overlapping blocks.
 
     Parameters
     ----------
@@ -82,6 +83,8 @@ def fft_conv1d(
         ``x``. For an even ``kernel_size`` it is aligned like torch's
         ``padding="same"`` (the extra zero padded on the right).
     """
+    if _real_dft.needs_real_dft(x):
+        return _real_dft.fft_conv1d_real(x, weight, bias)
     n_times, kernel_size = x.shape[-1], weight.shape[-1]
     # >= n_times + kernel_size - 1 (no circular wrap-around), even and
     # 5-smooth: the fast cases of a real FFT
@@ -115,8 +118,11 @@ def prefer_fft_conv(x: torch.Tensor, kernel_size: int) -> bool:
     and from ``batch_size * kernel_size >= 600`` if also
     ``in_channels * kernel_size >= 120``: with one or two input channels a
     float32 direct convolution stays cheaper up to k = 85 / 31 (13 long
-    temporal convolutions of the model zoo, 22 channels x batch 32). GPUs and
-    HPUs (no complex dtype) keep the direct convolution.
+    temporal convolutions of the model zoo, 22 channels x batch 32). On Intel
+    Gaudi (HPU) a direct convolution costs about ``batch_size * n_times *
+    kernel_size`` and the FFT about a fixed few milliseconds in eager mode: the
+    FFT wins from 8 input channels and ``batch_size * n_times * kernel_size >=
+    8e6``. GPUs keep the direct convolution.
 
     Parameters
     ----------
@@ -125,6 +131,8 @@ def prefer_fft_conv(x: torch.Tensor, kernel_size: int) -> bool:
     kernel_size : int
         Kernel length in samples.
     """
+    if x.device.type == "hpu":
+        return x.shape[1] >= 8 and x.shape[0] * x.shape[-1] * kernel_size >= 8e6
     return x.device.type == "cpu" and (
         x.element_size() == 2
         or (x.shape[0] * kernel_size >= 600 and x.shape[1] * kernel_size >= 120)

{braindecode-1.8.1.dev174738705 → braindecode-1.8.1.dev174812945}/braindecode/models/barista.py

@@ -7,13 +7,13 @@ from __future__ import annotations
 from typing import Optional
 
 import torch
-import torch.nn.functional as F
 from einops.layers.torch import Rearrange
 from torch import nn
 
 from braindecode.models.base import EEGModuleMixin
 from braindecode.models.util import extract_channel_locations_from_chs_info
 from braindecode.modules import GatedLinearUnit, PatchTokenizer
+from braindecode.modules.attention import _scaled_dot_product_attention
 
 
 class BaRISTA(EEGModuleMixin, nn.Module, license="other"):
@@ -689,7 +689,7 @@ class _RotarySelfAttention(nn.Module):
 
         # Dropout falls on the attention output rather than on the weights,
         # through dropout_p, because the reference disables the latter outright.
-        attention = F.scaled_dot_product_attention(query, key, value)
+        attention = _scaled_dot_product_attention(query, key, value)
         attention = self.dropout(attention)
         attention = self.merge_heads(attention)
         return self.o_proj(attention)

{braindecode-1.8.1.dev174738705 → braindecode-1.8.1.dev174812945}/braindecode/models/codebrain.py

@@ -15,6 +15,7 @@ from torch import nn
 from torch.nn.utils.parametrizations import weight_norm
 
 from braindecode.functional import spectral_input
+from braindecode.functional.functions import _next_fast_len
 from braindecode.models.base import EEGModuleMixin
 
 
@@ -521,16 +522,21 @@ class _GConv(nn.Module):
             kernel = F.pad(k_fwd, (0, seq_len)) + F.pad(k_bwd.flip(-1), (seq_len, 0))
 
         # FFT-based convolution: O(N log N)
+        # The bidirectional kernel is circular (lags 0..L-1, then -L..-1).
+        # Rolled to lags -L..L-1 it is a linear convolution read from lag 0,
+        # so any FFT length >= 2 * seq_len works: take a 5-smooth (fast) one.
+        start = seq_len if self.bidirectional else 0
+        n_fft = 2 * _next_fast_len(seq_len)
         # kernel_freq: (channels, d_model, freq_bins)
         # spectral_input(x) dtype: x's, at least float32 (the reference's .float()).
         x_in = spectral_input(x)
-        kernel_freq = torch.fft.rfft(kernel.to(x_in), n=2 * seq_len)
+        kernel_freq = torch.fft.rfft(kernel.roll(start, -1).to(x_in), n=n_fft)
         # x_freq: (batch, d_model, freq_bins)
-        x_freq = torch.fft.rfft(x_in, n=2 * seq_len)
+        x_freq = torch.fft.rfft(x_in, n=n_fft)
         # out_freq: (batch, channels, d_model, freq_bins)
         out_freq = torch.einsum("bhl,chl->bchl", x_freq, kernel_freq)
         # out: (batch, channels, d_model, seq_len)
-        out = torch.fft.irfft(out_freq, n=2 * seq_len)[..., :seq_len].to(x)
+        out = torch.fft.irfft(out_freq, n=n_fft)[..., start : start + seq_len].to(x)
 
         # Skip connection via learnable D matrix
         # (batch, channels, d_model, seq_len)
@@ -697,18 +703,15 @@ class _ResidualBlock(nn.Module):
         # Sliding-window attention branch
         # (batch, 2*res_channels, seq_len) -> (batch, seq_len, 2*res_channels)
         h_attn = h_ssm.transpose(1, 2)
-        if (
-            self.swa_window_size == 1
-            and not self.training
-            and not torch.is_grad_enabled()
-            and h_attn.device.type == "cpu"
-        ):
+        if self.swa_window_size == 1 and h_attn.device.type == "cpu":
             # A one-wide window keeps only the diagonal: the softmax weights are
-            # exactly 1 and 0, so the attention returns its value projection.
-            # Same projection calls as nn.MultiheadAttention (on its (seq,
-            # batch, embed) view), so the output is unchanged bit for bit. (In
-            # training the weights carry dropout; on Gaudi the full attention
-            # was faster.)
+            # exactly 1 and 0, so the attention returns its value projection,
+            # times the weights' dropout in training (the same (batch * heads,
+            # L, L) mask nn.MultiheadAttention draws, of which only the
+            # diagonal reaches the output). Same projection calls as
+            # nn.MultiheadAttention (on its (seq, batch, embed) view), so
+            # outputs and gradients are unchanged bit for bit, and so is the
+            # RNG stream. (On Gaudi the full attention was faster.)
             # ponytail: the unused query/key projections are computed too, to
             # keep the rounding; in_proj_weight[2E:] alone is ~2x less GEMM.
             embed_dim = self.attention.embed_dim
@@ -716,6 +719,16 @@ class _ResidualBlock(nn.Module):
             value = F.linear(
                 seq_first, self.attention.in_proj_weight, self.attention.in_proj_bias
             )[..., 2 * embed_dim :]
+            p = self.attention.dropout
+            if self.training and p > 0.0:
+                n_rows = seq_first.shape[1] * self.attention.num_heads
+                keep = (
+                    value.new_empty(n_rows, seq_len, seq_len)
+                    .bernoulli_(1 - p)
+                    .diagonal(dim1=1, dim2=2)
+                    .div(1 - p)
+                )
+                value = value.reshape(seq_len, n_rows, -1) * keep.t().unsqueeze(-1)
             h_attn = (
                 self.attention.out_proj(value.reshape(-1, embed_dim))
                 .view(seq_first.shape)

{braindecode-1.8.1.dev174738705 → braindecode-1.8.1.dev174812945}/braindecode/models/contrawr.py

@@ -4,7 +4,7 @@ import torch
 import torch.nn as nn
 from mne.utils import warn
 
-from braindecode.functional import spectral_input
+from braindecode.functional import _real_dft, spectral_input
 from braindecode.models.base import EEGModuleMixin
 from braindecode.models.util import _disable_batch_norm_training_if_batch_size_one
 
@@ -297,6 +297,8 @@ class _STFTModule(nn.Module):
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         # x: (B, C, T)
+        if _real_dft.needs_real_dft(x) and self.center and self.one_sided:
+            return self._forward_real_dft(x).to(x)
         B, C, T = x.shape
         # flatten batch & channel into one dim
         x_flat = spectral_input(x.reshape(B * C, T))
@@ -319,3 +321,23 @@ class _STFTModule(nn.Module):
         spec = spec_flat.view(B, C, F, L)
 
         return torch.abs(spec).to(x)
+
+    @torch.jit.unused
+    @_real_dft.fp32_island
+    def _forward_real_dft(self, x):
+        # The magnitude above without complex tensors, on the device (Intel
+        # Gaudi / HPU), instead of a host STFT and a host-to-device copy of
+        # the whole spectrogram: reflect-padded frames (gathered, not a
+        # strided unfold) times the one-sided DFT basis. No annotations: with
+        # postponed annotations torch.jit.unused cannot resolve them.
+        pad = self.n_fft // 2
+        x = nn.functional.pad(x, (pad, pad), mode="reflect")
+        n_frames = 1 + (x.shape[-1] - self.n_fft) // self.hop_length
+        starts = torch.arange(n_frames, device=x.device) * self.hop_length
+        index = (starts[:, None] + torch.arange(self.n_fft, device=x.device)).flatten()
+        frames = x.index_select(-1, index).unflatten(-1, (n_frames, self.n_fft))
+        real, imag = _real_dft.real_rfft(frames)
+        magnitude = torch.sqrt(real * real + imag * imag)
+        if self.normalized:
+            magnitude = magnitude / self.n_fft**0.5
+        return magnitude.transpose(-1, -2)

{braindecode-1.8.1.dev174738705 → braindecode-1.8.1.dev174812945}/braindecode/models/csbrain.py

@@ -541,6 +541,7 @@ class _RegionEmbedding(nn.Module):
     """
 
     region_bounds: List[Tuple[int, int]]
+    tap_starts: List[int]
 
     def __init__(
         self,
@@ -574,42 +575,61 @@ class _RegionEmbedding(nn.Module):
                 for region_key in area_config
             }
         )
+        # All regions in one pass: tap j of the widest kernel reads, for every
+        # electrode, the electrode at offset j - half on its region's ring
+        # (circular padding), or nothing (zero padding of a one-electrode
+        # region); kernel k uses the middle k taps. As a 0/1 matrix, so the
+        # gather and its backward are matmuls.
+        half = (max(kernel_sizes) - 1) // 2
+        n_taps = 2 * half + 1
+        self.tap_starts = [half - (k - 1) // 2 for k in kernel_sizes]
+        n_chans = self.region_bounds[-1][1] if self.region_bounds else 0
+        chan_region = []
+        tap_select = torch.zeros(n_chans * n_taps, n_chans)
+        for i, (start, stop) in enumerate(self.region_bounds):
+            n = stop - start
+            for p in range(n):
+                chan_region.append(i)
+                for d in range(-half, half + 1) if n > 1 else [0]:
+                    row = (start + p) * n_taps + d + half
+                    tap_select[row, start + (p + d) % n] = 1.0
+        self.register_buffer(
+            "chan_region", torch.tensor(chan_region, dtype=torch.long), False
+        )
+        self.register_buffer("tap_select", tap_select, False)
 
     def forward(self, x):
         batch, chans, T, dim_in = x.shape
-        output = torch.zeros(
-            (batch, chans, T, self.dim_out), device=x.device, dtype=x.dtype
-        )
-        for i, blocks in enumerate(self.region_blocks.values()):
-            start, stop = self.region_bounds[i]
-            n_electrodes = stop - start
-            x_region = x[:, start:stop, :, :]
-            x_trans = (
-                x_region.permute(0, 2, 1, 3)
-                .reshape(-1, n_electrodes, dim_in)
-                .permute(0, 2, 1)
-                .unsqueeze(-1)
+        if len(self.region_bounds) == 0:
+            return torch.zeros(
+                (batch, chans, T, self.dim_out), device=x.device, dtype=x.dtype
             )
-            fmap_outputs = []
-            for conv in blocks:
-                k = conv.kernel_size[0]
-                pad = (k - 1) // 2
-                if n_electrodes == 1:
-                    x_padded = F.pad(
-                        x_trans, (0, 0, pad, pad), mode="constant", value=0.0
-                    )
-                else:
-                    x_padded = F.pad(x_trans, (0, 0, pad, pad), mode="circular")
-                fmap_outputs.append(conv(x_padded))
-            fmap_cat = torch.cat(fmap_outputs, dim=1)
-            fmap_out = (
-                fmap_cat.squeeze(-1)
-                .permute(0, 2, 1)
-                .reshape(batch, T, n_electrodes, self.dim_out)
-                .permute(0, 2, 1, 3)
+        # (batch, chans, n_taps, T, dim_in): every electrode's ring neighbours
+        taps = torch.matmul(self.tap_select, x.reshape(batch, chans, T * dim_in))
+        taps = taps.reshape(batch, chans, -1, T, dim_in)
+        # Per kernel, one batched matmul over electrodes, each with its
+        # region's weights: (chans, batch * T, k * dim_in) @ (chans, k * dim_in, out)
+        weights: List[List[Tensor]] = [[] for _ in self.tap_starts]
+        biases: List[List[Tensor]] = [[] for _ in self.tap_starts]
+        for blocks in self.region_blocks.values():
+            for j, conv in enumerate(blocks):
+                bias = conv.bias
+                assert bias is not None
+                weights[j].append(conv.weight)
+                biases[j].append(bias)
+        outs = []
+        for j, start in enumerate(self.tap_starts):
+            w = torch.stack(weights[j]).index_select(0, self.chan_region)
+            k = w.shape[3]
+            w = w.squeeze(-1).permute(0, 3, 2, 1).reshape(chans, k * dim_in, -1)
+            t = taps[:, :, start : start + k].permute(1, 0, 3, 2, 4)
+            b = torch.stack(biases[j]).index_select(0, self.chan_region)
+            outs.append(
+                torch.bmm(t.reshape(chans, batch * T, k * dim_in), w) + b.unsqueeze(1)
             )
-            output[:, start:stop, :, :] = fmap_out
-        return output
+        out = torch.cat(outs, dim=-1).view(chans, batch, T, self.dim_out)
+        # contiguous: the residual sum keeps this layout and the next layer views it
+        return out.permute(1, 0, 2, 3).contiguous()
 
 
 class _CSBrainEncoderLayer(nn.Module):
@@ -675,6 +695,17 @@ class _CSBrainEncoderLayer(nn.Module):
             )
         else:
             self.region_attn_mask = None
+        # Region of every electrode and electrodes per region, for the
+        # region means of all regions at once (index_add_ sums in channel
+        # order, as the per-region mean did).
+        sizes = [stop - start for start, stop in self.region_bounds]
+        chan_region = [i for i, n in enumerate(sizes) for _ in range(n)]
+        self.register_buffer(
+            "chan_region", torch.tensor(chan_region, dtype=torch.long), False
+        )
+        self.register_buffer(
+            "region_sizes", torch.tensor(sizes, dtype=torch.long)[:, None], False
+        )
 
     def forward(self, src: Tensor) -> Tensor:
         x = src
@@ -711,11 +742,13 @@ class _CSBrainEncoderLayer(nn.Module):
     def _inter_region_attention(self, x: Tensor) -> Tensor:
         batch, chans, T, Fea = x.shape
         x_flat = x.permute(0, 2, 1, 3).reshape(batch * T, chans, Fea)
-        global_features = torch.zeros_like(x_flat)
-        for start, stop in self.region_bounds:
-            region_global = x[:, start:stop, :, :].mean(dim=1, keepdim=True)
-            region_global = region_global.permute(0, 2, 1, 3).reshape(batch * T, 1, Fea)
-            global_features[:, start:stop, :] = region_global
+        if len(self.region_bounds) == 0:
+            global_features = torch.zeros_like(x_flat)
+        else:
+            region_sum = x_flat.new_zeros(batch * T, len(self.region_bounds), Fea)
+            region_sum.index_add_(1, self.chan_region, x_flat)
+            region_mean = region_sum / self.region_sizes
+            global_features = region_mean.index_select(1, self.chan_region)
         x_enhanced = x_flat + self.global_fc(global_features)
 
         attn_output = self.inter_region_attn(

{braindecode-1.8.1.dev174738705 → braindecode-1.8.1.dev174812945}/braindecode/models/diver1.py

@@ -18,7 +18,7 @@ import torch.nn.functional as F
 from einops.layers.torch import Rearrange
 from torch import nn
 
-from braindecode.functional import rotate_pairs, spectral_input
+from braindecode.functional import _real_dft, rotate_pairs, spectral_input
 from braindecode.models.base import EEGModuleMixin
 from braindecode.models.util import (
     INTRACRANIAL_CH_TYPES,
@@ -397,10 +397,13 @@ class DIVER1(EEGModuleMixin, nn.Module, license="apache-2.0"):
         tokens = self.patch_tokenizer(x)
         tokens = self.patch_cnn(tokens)
         if self.spectral_emb is not None:
-            # At least float32, as the reference's float32 FFT.
-            fft_input = spectral_input(tokens)
-            spectrum = torch.fft.rfft(fft_input, dim=-1, norm="forward")
-            amplitude = spectrum.abs().to(tokens)
+            if _real_dft.needs_real_dft(tokens):
+                amplitude = self._amplitude_real_dft(tokens).to(tokens)
+            else:
+                # At least float32, as the reference's float32 FFT.
+                fft_input = spectral_input(tokens)
+                spectrum = torch.fft.rfft(fft_input, dim=-1, norm="forward")
+                amplitude = spectrum.abs().to(tokens)
             spectral_emb = self.spectral_emb(amplitude)
             tokens = tokens + spectral_emb
         if self.chan_emb is not None:
@@ -439,6 +442,16 @@ class DIVER1(EEGModuleMixin, nn.Module, license="apache-2.0"):
             pooled = tokens.mean(dim=(1, 2))
         return self.final_layer(pooled)
 
+    @torch.jit.unused
+    @_real_dft.fp32_island
+    def _amplitude_real_dft(self, tokens):
+        # |rfft(tokens, norm="forward")| on the device for accelerators without
+        # complex tensors (Intel Gaudi / HPU), instead of a host FFT. The norm
+        # keeps abs()'s zero gradient at an all-zero bin.
+        real, imag = _real_dft.real_rfft(tokens)
+        magnitude = torch.linalg.vector_norm(torch.stack((real, imag)), dim=0)
+        return magnitude / tokens.shape[-1]
+
 
 class _PatchCNN(nn.Sequential):
     """Power-of-two padded patches encoded by Conv2d/GroupNorm/GELU.
@@ -618,28 +631,13 @@ class _STCPE(nn.Module):
             inner_heads -= 1
         self.window = window
         self.n_patches = n_patches
-        # Full-height windows, padded in time to include partial overlaps.
-        self.stride = (1, 1)
-        self.padding = (0, window - 1)
+        # Full-height windows, padded in time to include partial overlaps:
+        # window w holds patches w - (window - 1) .. w. Entry j * n_windows + w
+        # is the padded patch index of offset j in window w (offset outermost,
+        # the order in which F.unfold / F.fold accumulate).
         self.n_windows = n_patches + window - 1
-        # Fold features into batch; each temporal window spans every channel.
-        self.fold_features = Rearrange(
-            "batch chan patch dim -> (batch dim) 1 chan patch"
-        )
-        self.unfold_features = Rearrange(
-            "(batch dim) 1 chan patch -> batch chan patch dim", dim=inner_dim
-        )
-        # One sequence per window for the inner encoder, and back again.
-        self.windows_to_batch = Rearrange(
-            "(batch dim) (chan window) n_win -> (batch n_win) (chan window) dim",
-            dim=inner_dim,
-            window=window,
-        )
-        self.batch_to_windows = Rearrange(
-            "(batch n_win) (chan window) dim -> (batch dim) (chan window) n_win",
-            n_win=self.n_windows,
-            window=window,
-        )
+        offsets = torch.arange(window)[:, None] + torch.arange(self.n_windows)
+        self.register_buffer("window_index", offsets.flatten(), persistent=False)
         self.down = nn.Linear(d_model, inner_dim)
         self.encoder = _AnyVariateEncoder(
             d_model=inner_dim,
@@ -655,49 +653,43 @@ class _STCPE(nn.Module):
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         """Map a token grid to a positional bias of the same shape."""
-        n_chans = x.shape[1]
-        kernel_size = (n_chans, self.window)
-        grid_size = (n_chans, self.n_patches)
+        batch, n_chans, n_patches, _ = x.shape
+        window, n_windows = self.window, self.n_windows
 
         x = self.down(x)
-        flat = self.fold_features(x)
-        unfolded = F.unfold(
-            flat,
-            kernel_size=kernel_size,
-            stride=self.stride,
-            padding=self.padding,
+        dim = x.shape[-1]
+        # Every window gathered at once (a strided unfold and its fold-shaped
+        # backward are slow on some accelerators, e.g. Intel Gaudi).
+        windows = (
+            F.pad(x, (0, 0, window - 1, window - 1))
+            .index_select(2, self.window_index)
+            .view(batch, n_chans, window, n_windows, dim)
+            .permute(0, 3, 1, 2, 4)
+            .reshape(batch * n_windows, n_chans * window, dim)
         )
-        windows = self.windows_to_batch(unfolded)
 
         var_id = torch.arange(n_chans, device=x.device).repeat_interleave(self.window)
         time_id = torch.arange(self.window, device=x.device).repeat(n_chans)
         encoded = self.encoder(windows, var_id=var_id, time_id=time_id)
-        encoded = self.batch_to_windows(encoded)
-
-        folded = F.fold(
-            encoded,
-            output_size=grid_size,
-            kernel_size=kernel_size,
-            stride=self.stride,
-            padding=self.padding,
+        encoded = (
+            encoded.view(batch, n_windows, n_chans, window, dim)
+            .permute(0, 2, 3, 1, 4)
+            .reshape(batch, n_chans, window * n_windows, dim)
+        )
+
+        # Overlap-add back onto the padded patches, then drop the padding.
+        n_padded = n_patches + 2 * (window - 1)
+        folded = encoded.new_zeros(batch, n_chans, n_padded, dim).index_add(
+            2, self.window_index, encoded
         )
-        # Fold in the working dtype: a scalar window divisor changes BF16
+        # Count in the working dtype: a scalar window divisor changes BF16
         # accumulation for large windows (e.g. 259 overlaps accumulate to 256).
-        overlap = F.fold(
-            torch.ones(
-                1,
-                n_chans * self.window,
-                self.n_windows,
-                dtype=x.dtype,
-                device=x.device,
-            ),
-            output_size=grid_size,
-            kernel_size=kernel_size,
-            stride=self.stride,
-            padding=self.padding,
+        overlap = x.new_zeros(n_padded).index_add(
+            0, self.window_index, x.new_ones(window * n_windows)
         )
-        averaged = folded / overlap
-        return self.up(self.unfold_features(averaged))
+        start, stop = window - 1, window - 1 + n_patches
+        averaged = folded[:, :, start:stop] / overlap[start:stop, None]
+        return self.up(averaged)
 
 
 class _AnyVariateEncoder(nn.Module):