returnn 1.20250902.10950__tar.gz → 1.20250902.133328__tar.gz

{returnn-1.20250902.10950/returnn.egg-info → returnn-1.20250902.133328}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250902.10950
+Version: 1.20250902.133328
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250902.133328/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20250902.133328'
+long_version = '1.20250902.133328+git.75be98d'

{returnn-1.20250902.10950 → returnn-1.20250902.133328}/returnn/frontend/array_.py

@@ -1195,7 +1195,10 @@ def reverse_sequence(tensor: Tensor, *, axis: Dim, handle_dynamic_dims: bool = T
     if not handle_dynamic_dims or not axis.need_masking():
         # noinspection PyProtectedMember
         return tensor._raw_backend.flip_no_mask(tensor, axis=axis)
-    indices = rf.combine_bc(axis.get_size_tensor(), "-", rf.range_over_dim(axis)) - 1
+    indices = (
+        rf.combine_bc(axis.get_size_tensor(device=tensor.device), "-", rf.range_over_dim(axis, device=tensor.device))
+        - 1
+    )
     return rf.gather(tensor, indices=indices, axis=axis, clip_to_valid=True)
 
 

{returnn-1.20250902.10950 → returnn-1.20250902.133328}/returnn/frontend/decoder/transformer.py

@@ -49,6 +49,7 @@ class TransformerDecoder(rf.Module):
         layer_opts: Optional[Dict[str, Any]] = None,
         embed_dim: Optional[Dim] = None,
         share_embedding: bool = None,
+        input_embedding: bool = True,
         input_embedding_scale: float = None,
         input_dropout: float = None,
         logits_with_bias: bool = False,
@@ -72,6 +73,7 @@ class TransformerDecoder(rf.Module):
         :param layer_opts: options for the decoder layer
         :param embed_dim: if given, will first have an embedding [vocab,embed] and then a linear [embed,model].
         :param share_embedding:
+        :param input_embedding: whether to use input embedding. If False, you must provide input of dimension model_dim.
         :param input_embedding_scale:
         :param input_dropout:
         :param logits_with_bias:
@@ -103,7 +105,7 @@ class TransformerDecoder(rf.Module):
 
         # We could make this optional or configurable if we ever need to.
         # Or maybe you would just have another separate implementation of this module then...
-        self.input_embedding = rf.Embedding(vocab_dim, embed_dim or model_dim)
+        self.input_embedding = rf.Embedding(vocab_dim, embed_dim or model_dim) if input_embedding else None
 
         self.input_embedding_proj = None
         if embed_dim:
@@ -121,21 +123,31 @@ class TransformerDecoder(rf.Module):
             raise TypeError(f"unexpected pos_enc type {pos_enc!r}")
         self.pos_enc = pos_enc
         if share_embedding is None:
-            if BehaviorVersion.get() < 20:
-                logging.getLogger("returnn.frontend").warning(
-                    "TransformerDecoder share_embedding default is False"
-                    f" with your behavior version {BehaviorVersion.get()}."
-                    " Explicitly set share_embedding or switch to a new behavior version >= 20."
-                )
-            share_embedding = True if BehaviorVersion.get() >= 20 else False
+            if embed_dim and embed_dim != model_dim:
+                share_embedding = False
+            elif input_embedding:
+                if BehaviorVersion.get() < 20:
+                    logging.getLogger("returnn.frontend").warning(
+                        "TransformerDecoder share_embedding default is False"
+                        f" with your behavior version {BehaviorVersion.get()}."
+                        " Explicitly set share_embedding or switch to a new behavior version >= 20."
+                    )
+                share_embedding = True if BehaviorVersion.get() >= 20 else False
+            else:  # not input_embedding
+                share_embedding = False
         if input_embedding_scale is None:
-            if BehaviorVersion.get() < 20:
-                logging.getLogger("returnn.frontend").warning(
-                    "TransformerDecoder input_embedding_scale default is suboptimal"
-                    f" with your behavior version {BehaviorVersion.get()}."
-                    " Explicitly set input_embedding_scale or switch to a new behavior version >= 20."
-                )
-            input_embedding_scale = model_dim.dimension**0.5 if BehaviorVersion.get() >= 20 else 1.0
+            if input_embedding:
+                if BehaviorVersion.get() < 20:
+                    logging.getLogger("returnn.frontend").warning(
+                        "TransformerDecoder input_embedding_scale default is suboptimal"
+                        f" with your behavior version {BehaviorVersion.get()}."
+                        " Explicitly set input_embedding_scale or switch to a new behavior version >= 20."
+                    )
+                input_embedding_scale = model_dim.dimension**0.5 if BehaviorVersion.get() >= 20 else 1.0
+            elif pos_enc:
+                input_embedding_scale = model_dim.dimension**0.5
+            else:
+                input_embedding_scale = 1.0
         self.input_embedding_scale = input_embedding_scale
         if input_dropout is None:
             if dropout > 0 and BehaviorVersion.get() < 20:
@@ -179,7 +191,9 @@ class TransformerDecoder(rf.Module):
         self.logits = rf.Linear(model_dim, vocab_dim, with_bias=logits_with_bias)
 
         if share_embedding:
-            assert not embed_dim and not logits_with_bias, "not supported together with share_embedding"
+            assert input_embedding, "input_embedding=True required for share_embedding"
+            assert not embed_dim or embed_dim == model_dim, f"{embed_dim=} not supported with share_embedding"
+            assert not logits_with_bias, "logits_with_bias=True expected with share_embedding"
             self.logits.weight = self.input_embedding.weight
 
     def default_initial_state(self, *, batch_dims: Sequence[Dim]) -> rf.State:
@@ -219,7 +233,12 @@ class TransformerDecoder(rf.Module):
         """
         new_state = rf.State()
 
-        decoded = self.input_embedding(source) * self.input_embedding_scale
+        if self.input_embedding is not None:
+            decoded = self.input_embedding(source)
+        else:
+            decoded = source
+        if self.input_embedding_scale != 1:
+            decoded = decoded * self.input_embedding_scale
         if self.pos_enc is not None:
             decoded = decoded + self.pos_enc(spatial_dim=spatial_dim, offset=state.pos)
         decoded = rf.dropout(decoded, self.input_dropout)

{returnn-1.20250902.10950 → returnn-1.20250902.133328/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250902.10950
+Version: 1.20250902.133328
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250902.10950/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20250902.010950'
-long_version = '1.20250902.010950+git.9d5debf'