titans-pytorch 0.0.36__py3-none-any.whl → 0.0.38__py3-none-any.whl

titans_pytorch/mac_transformer.py

@@ -7,7 +7,7 @@ from torch import nn, cat
 import torch.nn.functional as F
 from torch.nn import Module, ModuleList, Linear
 
-from einops import repeat, rearrange
+from einops import repeat, rearrange, pack, unpack
 from einops.layers.torch import Rearrange
 
 from hyper_connections import get_init_and_expand_reduce_stream_functions
@@ -214,7 +214,12 @@ class MemoryAsContextTransformer(Module):
             if layer in neural_memory_layers:
                 assert has_longterm_mems, '`num_longterm_mem_tokens` must be greater than 0'
 
-                mem = NeuralMemory(dim = dim, chunk_size = num_longterm_mem_tokens)
+                mem = NeuralMemory(
+                    dim = dim,
+                    chunk_size = num_longterm_mem_tokens + segment_len,
+                    **neural_memory_kwargs
+                )
+
                 mem = init_hyper_conn(dim = dim, branch = mem)
 
             self.neural_mem_layers.append(mem)
@@ -266,7 +271,7 @@ class MemoryAsContextTransformer(Module):
         x, inverse_segment = pad_and_segment_with_inverse(x, segment_len)
 
         mems = repeat(self.longterm_mems, 'n d -> b n d', b = x.shape[0])
-        x = cat((mems, x), dim = -2)
+        x, mem_ps = pack((x, mems), 'b * d')
 
         x = inverse_segment(x)
 
@@ -283,21 +288,8 @@ class MemoryAsContextTransformer(Module):
         for (attn, ff), maybe_neural_mem in zip(self.layers, self.neural_mem_layers):
 
             if exists(maybe_neural_mem):
-                batch_streams = x.shape[0]
-
-                x, inverse_segment = pad_and_segment_with_inverse(x, total_segment_len)
-
-                longterm_mems, x = x[:, :num_longterm_mem_tokens], x[:, num_longterm_mem_tokens:]
-
-                longterm_mems = rearrange(longterm_mems, '(b w) n d -> b (w n) d', b = batch_streams)
-
-                longterm_mems = maybe_neural_mem(longterm_mems)
-
-                longterm_mems = rearrange(longterm_mems, 'b (w n) d -> (b w) n d', n = num_longterm_mem_tokens)
-
-                x = cat((longterm_mems, x), dim = -2)
+                x = maybe_neural_mem(x)
 
-                x = inverse_segment(x)
 
             x = attn(x)
 
@@ -309,7 +301,7 @@ class MemoryAsContextTransformer(Module):
 
         x, inverse_segment = pad_and_segment_with_inverse(x, total_segment_len)
 
-        x = x[:, num_longterm_mem_tokens:]
+        x, _ = unpack(x, mem_ps, 'b * d')
 
         x = inverse_segment(x)
 

titans_pytorch/titans.py

@@ -27,9 +27,7 @@ n - sequence
 d - feature dimension
 c - intra-chunk
 """
-
-# constants
-
+7
 LinearNoBias = partial(Linear, bias = False)
 
 # functions
@@ -132,7 +130,7 @@ class NeuralMemory(Module):
         max_grad_norm: float | None = None,
         use_accelerated_scan = False,
         default_mlp_kwargs: dict = dict(
-            depth = 4
+            depth = 2
         )
     ):
         super().__init__()
@@ -390,7 +388,10 @@ class NeuralMemory(Module):
 
         padding = next_seq_len - curtailed_seq_len
 
-        seq = pad_at_dim(seq, (0, padding), dim = 1)
+        needs_pad = padding > 0
+
+        if needs_pad:
+            seq = pad_at_dim(seq, (0, padding), dim = 1)
 
         # the parameters of the memory model stores the memories of the key / values
         # when the MLP has only 1 weight matrix, it is equivalent to `kv` fast weight memories from linear attention literature (recall fetching of memories is q @ (kv)) / schmidhuber's paper
@@ -442,7 +443,9 @@ class NeuralMemory(Module):
         empty_memory_embeds = self.init_empty_memory_embed(values.shape[0], chunk_size - 1)
         values = torch.cat((empty_memory_embeds, values), dim = -2)
 
-        values = values[:, :-padding]
+        if needs_pad:
+            values = values[:, :-padding]
+
         return values
 
     def forward(

{titans_pytorch-0.0.36.dist-info → titans_pytorch-0.0.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: titans-pytorch
-Version: 0.0.36
+Version: 0.0.38
 Summary: Titans
 Project-URL: Homepage, https://pypi.org/project/titans-pytorch/
 Project-URL: Repository, https://github.com/lucidrains/titans-pytorch

titans_pytorch-0.0.38.dist-info/RECORD

@@ -0,0 +1,9 @@
+titans_pytorch/__init__.py,sha256=F6pV8BamKCsbJFVo5x2hw69vzfJNLy54SwIKIueMdp4,142
+titans_pytorch/associative_scan.py,sha256=Y-iYqmFuG-NoCKu6kgql1mhowXTeJfyawi3eUIXamp0,2650
+titans_pytorch/mac_transformer.py,sha256=5koIfEulJ841FNrs6URZfW2dp9LMuHzMkaySDrlbuP0,8393
+titans_pytorch/titans.py,sha256=bv2Ceq-_4nNb5FNx4hLd2inC93m5MmJxO2-Mbf6PKK4,14378
+titans_pytorch/titans_attn_memory.py,sha256=Rwx_-riGeISBefZg5Kjic8jzmmRRys-u93D2Kgb7Mos,12691
+titans_pytorch-0.0.38.dist-info/METADATA,sha256=L6tEQTEOXCeAU_BuRLbwUO0-gmnbJE-WQNAZ83BNCWA,3938
+titans_pytorch-0.0.38.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+titans_pytorch-0.0.38.dist-info/licenses/LICENSE,sha256=1yCiA9b5nhslTavxPjsQAO-wpOnwJR9-l8LTVi7GJuk,1066
+titans_pytorch-0.0.38.dist-info/RECORD,,

titans_pytorch-0.0.36.dist-info/RECORD

@@ -1,9 +0,0 @@
-titans_pytorch/__init__.py,sha256=F6pV8BamKCsbJFVo5x2hw69vzfJNLy54SwIKIueMdp4,142
-titans_pytorch/associative_scan.py,sha256=Y-iYqmFuG-NoCKu6kgql1mhowXTeJfyawi3eUIXamp0,2650
-titans_pytorch/mac_transformer.py,sha256=xXQ9GvtvNArYidV1OOhUhCJ0pIxniElTLnL0_eIZtEE,8821
-titans_pytorch/titans.py,sha256=Kx_tl_QkeDccvkMwPZ0xQ_saYjZfbKzDNPTTSHNWYcA,14304
-titans_pytorch/titans_attn_memory.py,sha256=Rwx_-riGeISBefZg5Kjic8jzmmRRys-u93D2Kgb7Mos,12691
-titans_pytorch-0.0.36.dist-info/METADATA,sha256=7Bum0wO6e6BsB7TShLBZWALyStcs1LLaEv5vvnVlQ9c,3938
-titans_pytorch-0.0.36.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-titans_pytorch-0.0.36.dist-info/licenses/LICENSE,sha256=1yCiA9b5nhslTavxPjsQAO-wpOnwJR9-l8LTVi7GJuk,1066
-titans_pytorch-0.0.36.dist-info/RECORD,,