returnn 1.20250304.113330__py3-none-any.whl → 1.20250305.150759__py3-none-any.whl

returnn/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250304.113330
+Version: 1.20250305.150759
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn/_setup_info_generated.py

@@ -1,2 +1,2 @@
-version = '1.20250304.113330'
-long_version = '1.20250304.113330+git.acf09da'
+version = '1.20250305.150759'
+long_version = '1.20250305.150759+git.fd21be2'

returnn/frontend/loop.py

@@ -128,6 +128,16 @@ def scan(
         like selecting the right beam entries.
     :return: outputs ys, final state, and the new spatial_dim
     """
+    device = None
+    if initial is not None:
+        vs = [v for v in tree.flatten(initial) if isinstance(v, Tensor) and v.device not in (None, "cpu")]
+        if vs:
+            device = vs[0].device
+    if device is None and xs is not None:
+        vs = [v for v in tree.flatten(xs) if isinstance(v, Tensor) and v.device not in (None, "cpu")]
+        if vs:
+            device = vs[0].device
+
     if spatial_dim is None or not spatial_dim.is_dim_known():
         assert cond is not None, f"scan: spatial_dim {spatial_dim} is None/unknown, need to provide `cond`"
         assert cond_dims is not None, f"scan: spatial_dim {spatial_dim} is None/unknown, need to provide `cond_dims`"
@@ -145,12 +155,13 @@ def scan(
         def _body(_s: Tuple[Tensor, Tensor, Tensor, S, Y]) -> Tuple[Tensor, Tensor, Tensor, S, Y]:
             i, seq_len_, prev_cond, s, ys_ = _s
             seq_len_ = seq_len_ + rf.cast(prev_cond, dtype=seq_len_.dtype)
-            y, s = body(None, s)
+            y, s_ = body(None, s)
             tree.assert_same_structure(ys_, y)
             ys_ = tree.map_structure(lambda ys__, y_: ys__.push_back(y_) if ys__ is not None else None, ys_, y)
-            c = cond(None, s)
+            c = cond(None, s_)
             c = rf.logical_and(c, prev_cond)
-            return i + 1, seq_len_, c, s, ys_
+            s_ = rf.nested.mask_nested(s_, mask=c, mask_value=s, allow_dim_extension=False)
+            return i + 1, seq_len_, c, s_, ys_
 
         if cond_before_body:
             initial_cond = cond(None, initial)
@@ -187,10 +198,13 @@ def scan(
 
         def _body(_s: Tuple[Tensor, S, Y]) -> Tuple[Tensor, S, Y]:
             i, s, ys_ = _s
-            y, s = body(tree.map_structure(lambda x: x[i], xs), s)
+            y, s_ = body(tree.map_structure(lambda x: x[i], xs), s)
             tree.assert_same_structure(ys_, y)
             ys_ = tree.map_structure(lambda ys__, y_: ys__.push_back(y_) if ys__ is not None else None, ys_, y)
-            return i + 1, s, ys_
+            s_ = rf.nested.mask_nested(
+                s_, mask=i < spatial_dim.get_size_tensor(device=device), mask_value=s, allow_dim_extension=False
+            )
+            return i + 1, s_, ys_
 
         _, final_s, ys = while_loop(
             _cond,

returnn/frontend/nested.py

@@ -3,7 +3,7 @@ Some utility functions on nested structures.
 """
 
 from __future__ import annotations
-from typing import TypeVar, Optional, Sequence, Tuple, Dict
+from typing import TypeVar, Optional, Union, Sequence, Tuple, Dict
 import functools
 import re
 import tree
@@ -11,12 +11,108 @@ from returnn.tensor import Tensor, Dim
 import returnn.frontend as rf
 
 
-__all__ = ["gather_nested", "masked_select_nested", "masked_scatter_nested"]
+__all__ = ["mask_nested", "gather_nested", "masked_select_nested", "masked_scatter_nested"]
 
 
 T = TypeVar("T")
 
 
+def mask_nested(
+    s: T,
+    *,
+    mask: Tensor,
+    mask_value: Union[T, Tensor, float, None],
+    dim_map: Optional[Dict[Dim, Dim]] = None,
+    allow_dim_extension: bool = True,
+) -> T:
+    """
+    Applies where(mask, s, mask_value) for nested structures.
+
+    :param s:
+    :param mask:
+    :param mask_value:
+    :param dim_map:
+    :param allow_dim_extension:
+    :return: s with masked values
+    """
+    if dim_map is None:
+        dim_map = {}
+    partial_kwargs = dict(mask=mask, dim_map=dim_map, allow_dim_extension=allow_dim_extension)
+    structures = [s]
+    if type(s) is type(mask_value):  # mask_value also same nested structure?
+        tree.assert_same_structure(s, mask_value)
+        structures.append(mask_value)
+    else:
+        partial_kwargs["mask_value"] = mask_value
+    tree.map_structure(functools.partial(_mask_prepare_dims, **partial_kwargs), *structures)
+    return tree.map_structure(functools.partial(_mask, **partial_kwargs), *structures)
+
+
+def _mask_prepare_dims(
+    s: T, mask_value: Union[T, Tensor, float, None], *, mask: Tensor, dim_map: Dict[Dim, Dim], allow_dim_extension: bool
+) -> T:
+    if isinstance(s, Dim):
+        if mask_value is None:
+            return s  # not sure if always correct...
+        assert isinstance(mask_value, Dim)
+        if s == mask_value:
+            return s
+        if not allow_dim_extension:
+            dim_size_dims = set()
+            if s.dyn_size_ext is not None:
+                dim_size_dims.update(s.dyn_size_ext.dims_set)
+            if mask_value.dyn_size_ext is not None:
+                dim_size_dims.update(mask_value.dyn_size_ext.dims_set)
+            if not mask.dims_set.issubset(dim_size_dims):
+                assert not mask.dims_set.intersection(dim_size_dims)  # not sure...
+                return s
+        new_dyn_size = _mask(
+            s.get_size_tensor(),
+            mask=mask,
+            mask_value=mask_value.get_size_tensor(),
+            dim_map=dim_map,
+            allow_dim_extension=allow_dim_extension,
+        )
+        new_dim = Dim(new_dyn_size, name=_extend_dim_name(s.name))
+        dim_map[s] = dim_map[mask_value] = new_dim
+        return new_dim
+    return s
+
+
+def _mask(
+    s: T, mask_value: Union[T, Tensor, float, None], *, mask: Tensor, dim_map: Dict[Dim, Dim], allow_dim_extension: bool
+) -> T:
+    if s is None:
+        return s
+    if isinstance(s, Tensor):
+        if dim_map:
+            for d in s.dims:
+                if d in dim_map:
+                    s = rf.replace_dim_v2(s, in_dim=d, out_dim=dim_map[d])
+            if isinstance(mask_value, Tensor):
+                for d in mask_value.dims:
+                    if d in dim_map:
+                        mask_value = rf.replace_dim_v2(mask_value, in_dim=d, out_dim=dim_map[d])
+        if not allow_dim_extension and isinstance(mask_value, Tensor):
+            if not s.dims_set.issuperset(mask_value.dims_set):
+                return s
+        if not allow_dim_extension or mask_value is None or (isinstance(mask_value, (int, float)) and mask_value == 0):
+            if mask.dims_set.issubset(s.dims_set):
+                return rf.where(mask, s, mask_value)
+            assert not mask.dims_set.intersection(s.dims_set)  # not sure...
+            return s
+        assert isinstance(mask_value, (int, float, Tensor))
+        return rf.where(mask, s, mask_value, allow_broadcast_all_sources=True)
+    if isinstance(s, Dim):
+        if mask_value is None:
+            return s
+        assert isinstance(mask_value, Dim)
+        if s == mask_value:
+            return s
+        return dim_map.get(s, s)
+    raise TypeError(f"_mask: unexpected {s!r} type {type(s).__name__}")
+
+
 def gather_nested(s: T, *, indices: Tensor, dim_map: Optional[Dict[Dim, Dim]] = None) -> T:
     """
     This is like :func:`gather`, but for nested structures.

returnn/frontend/rec.py

@@ -70,6 +70,9 @@ class LSTM(rf.Module):
             out_dim=self.out_dim,
         )
         new_state = LstmState(h=new_state_h, c=new_state_c)
+        result.feature_dim = self.out_dim
+        new_state.h.feature_dim = self.out_dim
+        new_state.c.feature_dim = self.out_dim
 
         return result, new_state
 

{returnn-1.20250304.113330.dist-info → returnn-1.20250305.150759.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250304.113330
+Version: 1.20250305.150759
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250304.113330.dist-info → returnn-1.20250305.150759.dist-info}/RECORD

@@ -1,9 +1,9 @@
-returnn/PKG-INFO,sha256=BmSxZKkRxyL20E4Zsud1muiQ-rth9Ob9PMR-43IrAMw,5215
+returnn/PKG-INFO,sha256=WGNUtpero6ia7jcFsuoDC8KQaPT077SI434pgHtd6y0,5215
 returnn/__init__.py,sha256=biBtRsM0WZ406vShaeH-9WFoqJ8XwTbn6g0EeFJ7l8E,1012
 returnn/__main__.py,sha256=qBFbuB1yN3adgVM5pXt2-Yq9vorjRNchNPL8kDKx44M,31752
 returnn/__old_mod_loader__.py,sha256=nvsNY-xELdS_IPNkv66Q9Rmvg4dbGW0-EBRDcCmctos,7654
 returnn/__setup__.py,sha256=22kQn2fh11iPM0hLb2Fy5sLmoU1JGvmDxXRYuRgQkwU,4659
-returnn/_setup_info_generated.py,sha256=94BElbYUGmjpsoY8BzvfW39RUTXw9Fy3UwlPoEjrkU8,77
+returnn/_setup_info_generated.py,sha256=_BUxCpQI2oODCG2knN-6EoX8imvgddkHg0F7kXRyik0,77
 returnn/config.py,sha256=3tmKhB6FnQZaNdtcYsiB61JnEY--iZ2qmJ4yq0b6tE0,29140
 returnn/forward_iface.py,sha256=A_OJiaXsX4MlXQRzST86ylyxSUZbC402PQL1REcqHjM,911
 returnn/learning_rate_control.py,sha256=ZvWryAn_tv9DhV8sh1LV3eE34Yltl3On3mYZAG4hR9s,34684
@@ -99,19 +99,19 @@ returnn/frontend/hooks.py,sha256=jYPbsb4gy5HORRZvKTEJbLcoJri5hOt5ADbhnTCytQo,550
 returnn/frontend/init.py,sha256=bVB7bpghaY8DI_HL0mkB_9z95onWnIX2zlW4hlMYnRw,7494
 returnn/frontend/label_smoothing.py,sha256=lxmaowNr61sCMzMewqHhu1r0CcklYfhLXlFnBu8DeAU,5676
 returnn/frontend/linear.py,sha256=xRUjnkD3MTWDezSaYATBYJQ2fa1RhKMNrTuhC54hhVs,2252
-returnn/frontend/loop.py,sha256=anIdKjrS0PdHwcqRR5NG9OTPpoydeEGd0QawlG-AF_k,15498
+returnn/frontend/loop.py,sha256=vNfq-9U6Jb0ZW2zM9QmbTaXZBkBFQ5MzC9V0aT1ATCU,16158
 returnn/frontend/loss.py,sha256=r_qRAiFIXgXTnFuLoAhE5jsyAZYMCucRzf55XfWbkC8,7441
 returnn/frontend/math_.py,sha256=KlJxdIib8ENlid7cc4lcwHv5e21tzTjTEV8VgEDAijo,16984
 returnn/frontend/matmul.py,sha256=3QaGiZtSs9PriT40T7Vc3KnYKPgYSN4tCZytYeq9qMA,1945
 returnn/frontend/module.py,sha256=219rh5mE0CD0-NdxXLsKyhv3BNtOI9jSyiI1Rb8MOyU,10700
-returnn/frontend/nested.py,sha256=CT3C0wXkeWGjJcAoF6yebsRXuN8-YpjO2eqgdl1-vaE,11005
+returnn/frontend/nested.py,sha256=oDiqnyTML7ZtCxrufU4ypG0fOZ_WsZPIvfdSn1Phs6M,14698
 returnn/frontend/normalization.py,sha256=QIjXYg0C8BD2g_1lAkVO4Cara729uHC_bsQh99VsWeI,14061
 returnn/frontend/parameter.py,sha256=w6SN-uv87OyeWBt90_3UBbK0h6sftSOCxkqXPg76caY,10375
 returnn/frontend/parametrizations.py,sha256=hVbOlgm1pQAmZnAnNxq8Tk23rykr_iy3-6R1H6CwlMA,2798
 returnn/frontend/parametrize.py,sha256=VhgTEP7ehON950Q4bkCy8rvg9641moEKAXn0XzomK6E,7216
 returnn/frontend/piecewise_linear.py,sha256=TdL6wzop8P1dcIZwkEbJFvSUZSI1cbhS3XKzlWQkEVI,1964
 returnn/frontend/rand.py,sha256=Levgf5VtOOBKDSgz0869Jf3VW4BWxYZuRXsa_fOxNI4,12969
-returnn/frontend/rec.py,sha256=4m20LvsPJ75pRYykVrup6Csj_D7duG-dW28SaJh-sq8,7863
+returnn/frontend/rec.py,sha256=la-VXR_hzvwNzpAgn4Okl-yDx3F4gOW-81EKm-jAAlg,7999
 returnn/frontend/reduce.py,sha256=-Zt-OH6Zbtb9uR6YEzurCyrowH-anIXvuga6Pla2V70,10220
 returnn/frontend/run_ctx.py,sha256=ItcZwuFItkZjYWrg715L1Za2Xg7__MQCrRCAwBeTUxA,21411
 returnn/frontend/signal.py,sha256=XgOBL1iy-cJgulePH5HRPAwp2cScy60q4RItr7xzvGc,4412
@@ -253,8 +253,8 @@ returnn/util/sig_proc.py,sha256=Tjz0VOAVyqu2qDCF5HZ1JjALjcFsHcNkcd96WgZeKfE,7265
 returnn/util/task_system.py,sha256=y4sMVXQ25Qd2z0rx03uOlXlkE-jbCYC1Sjfn-XlraVU,26003
 returnn/util/train_proc_manager.py,sha256=Pjht28k6uz6BNQ47uW6Gf880iyq5q4wx7P_K2tmoAM8,3266
 returnn/util/watch_memory.py,sha256=BR5P2kvBN6UI81cE0_1WAA6Hd1SByLbBaiDxvLhPOew,4213
-returnn-1.20250304.113330.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
-returnn-1.20250304.113330.dist-info/METADATA,sha256=BmSxZKkRxyL20E4Zsud1muiQ-rth9Ob9PMR-43IrAMw,5215
-returnn-1.20250304.113330.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-returnn-1.20250304.113330.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
-returnn-1.20250304.113330.dist-info/RECORD,,
+returnn-1.20250305.150759.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
+returnn-1.20250305.150759.dist-info/METADATA,sha256=WGNUtpero6ia7jcFsuoDC8KQaPT077SI434pgHtd6y0,5215
+returnn-1.20250305.150759.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+returnn-1.20250305.150759.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
+returnn-1.20250305.150759.dist-info/RECORD,,