rxnn 0.1.36__tar.gz → 0.1.37__tar.gz

{rxnn-0.1.36 → rxnn-0.1.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rxnn
-Version: 0.1.36
+Version: 0.1.37
 Summary: RxNN: Reactive Neural Networks Platform
 License: Apache-2.0
 Keywords: deep-learning,ai,machine-learning

{rxnn-0.1.36 → rxnn-0.1.37}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "rxnn"
-version = "0.1.36"
+version = "0.1.37"
 description = "RxNN: Reactive Neural Networks Platform"
 
 license = "Apache-2.0"

{rxnn-0.1.36 → rxnn-0.1.37}/src/rxnn/transformers/attention.py

@@ -102,8 +102,12 @@ class MultiHeadAttention(nn.Module):
 
     def _flash_attention(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, b: int, t: int, d: int,
                          mask: torch.Tensor = None, enable_gqa: bool = False):
-        with sdpa_kernel(backends=[SDPBackend.FLASH_ATTENTION]):
-            return self._torch_attention(q, k, v, b, t, d, mask=mask, enable_gqa=enable_gqa)
+        # After ~6h of fighthing, PyTorch based is still now working so I decided to use FlashAttention directly
+        # with sdpa_kernel(backends=[SDPBackend.FLASH_ATTENTION]):
+        #     return self._torch_attention(q, k, v, b, t, d, mask=mask, enable_gqa=enable_gqa)
+        from flash_attn import flash_attn_func
+        attn_output = flash_attn_func(q, k, v, dropout_p=self.dropout.p if self.training else 0.0, is_causal=self.is_causal)
+        return self._transpose_output(attn_output, b, t, d)
 
     def _torch_attention(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, b: int, t: int, d: int,
                          mask: torch.Tensor = None, enable_gqa: bool = False):