returnn 1.20250304.101951__tar.gz → 1.20250305.150759__tar.gz

{returnn-1.20250304.101951/returnn.egg-info → returnn-1.20250305.150759}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250304.101951
+Version: 1.20250305.150759
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250305.150759/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20250305.150759'
+long_version = '1.20250305.150759+git.fd21be2'

{returnn-1.20250304.101951 → returnn-1.20250305.150759}/returnn/frontend/loop.py

@@ -128,6 +128,16 @@ def scan(
         like selecting the right beam entries.
     :return: outputs ys, final state, and the new spatial_dim
     """
+    device = None
+    if initial is not None:
+        vs = [v for v in tree.flatten(initial) if isinstance(v, Tensor) and v.device not in (None, "cpu")]
+        if vs:
+            device = vs[0].device
+    if device is None and xs is not None:
+        vs = [v for v in tree.flatten(xs) if isinstance(v, Tensor) and v.device not in (None, "cpu")]
+        if vs:
+            device = vs[0].device
+
     if spatial_dim is None or not spatial_dim.is_dim_known():
         assert cond is not None, f"scan: spatial_dim {spatial_dim} is None/unknown, need to provide `cond`"
         assert cond_dims is not None, f"scan: spatial_dim {spatial_dim} is None/unknown, need to provide `cond_dims`"
@@ -145,12 +155,13 @@ def scan(
         def _body(_s: Tuple[Tensor, Tensor, Tensor, S, Y]) -> Tuple[Tensor, Tensor, Tensor, S, Y]:
             i, seq_len_, prev_cond, s, ys_ = _s
             seq_len_ = seq_len_ + rf.cast(prev_cond, dtype=seq_len_.dtype)
-            y, s = body(None, s)
+            y, s_ = body(None, s)
             tree.assert_same_structure(ys_, y)
             ys_ = tree.map_structure(lambda ys__, y_: ys__.push_back(y_) if ys__ is not None else None, ys_, y)
-            c = cond(None, s)
+            c = cond(None, s_)
             c = rf.logical_and(c, prev_cond)
-            return i + 1, seq_len_, c, s, ys_
+            s_ = rf.nested.mask_nested(s_, mask=c, mask_value=s, allow_dim_extension=False)
+            return i + 1, seq_len_, c, s_, ys_
 
         if cond_before_body:
             initial_cond = cond(None, initial)
@@ -187,10 +198,13 @@ def scan(
 
         def _body(_s: Tuple[Tensor, S, Y]) -> Tuple[Tensor, S, Y]:
             i, s, ys_ = _s
-            y, s = body(tree.map_structure(lambda x: x[i], xs), s)
+            y, s_ = body(tree.map_structure(lambda x: x[i], xs), s)
             tree.assert_same_structure(ys_, y)
             ys_ = tree.map_structure(lambda ys__, y_: ys__.push_back(y_) if ys__ is not None else None, ys_, y)
-            return i + 1, s, ys_
+            s_ = rf.nested.mask_nested(
+                s_, mask=i < spatial_dim.get_size_tensor(device=device), mask_value=s, allow_dim_extension=False
+            )
+            return i + 1, s_, ys_
 
         _, final_s, ys = while_loop(
             _cond,

{returnn-1.20250304.101951 → returnn-1.20250305.150759}/returnn/frontend/nested.py

@@ -3,7 +3,7 @@ Some utility functions on nested structures.
 """
 
 from __future__ import annotations
-from typing import TypeVar, Optional, Sequence, Tuple, Dict
+from typing import TypeVar, Optional, Union, Sequence, Tuple, Dict
 import functools
 import re
 import tree
@@ -11,12 +11,108 @@ from returnn.tensor import Tensor, Dim
 import returnn.frontend as rf
 
 
-__all__ = ["gather_nested", "masked_select_nested", "masked_scatter_nested"]
+__all__ = ["mask_nested", "gather_nested", "masked_select_nested", "masked_scatter_nested"]
 
 
 T = TypeVar("T")
 
 
+def mask_nested(
+    s: T,
+    *,
+    mask: Tensor,
+    mask_value: Union[T, Tensor, float, None],
+    dim_map: Optional[Dict[Dim, Dim]] = None,
+    allow_dim_extension: bool = True,
+) -> T:
+    """
+    Applies where(mask, s, mask_value) for nested structures.
+
+    :param s:
+    :param mask:
+    :param mask_value:
+    :param dim_map:
+    :param allow_dim_extension:
+    :return: s with masked values
+    """
+    if dim_map is None:
+        dim_map = {}
+    partial_kwargs = dict(mask=mask, dim_map=dim_map, allow_dim_extension=allow_dim_extension)
+    structures = [s]
+    if type(s) is type(mask_value):  # mask_value also same nested structure?
+        tree.assert_same_structure(s, mask_value)
+        structures.append(mask_value)
+    else:
+        partial_kwargs["mask_value"] = mask_value
+    tree.map_structure(functools.partial(_mask_prepare_dims, **partial_kwargs), *structures)
+    return tree.map_structure(functools.partial(_mask, **partial_kwargs), *structures)
+
+
+def _mask_prepare_dims(
+    s: T, mask_value: Union[T, Tensor, float, None], *, mask: Tensor, dim_map: Dict[Dim, Dim], allow_dim_extension: bool
+) -> T:
+    if isinstance(s, Dim):
+        if mask_value is None:
+            return s  # not sure if always correct...
+        assert isinstance(mask_value, Dim)
+        if s == mask_value:
+            return s
+        if not allow_dim_extension:
+            dim_size_dims = set()
+            if s.dyn_size_ext is not None:
+                dim_size_dims.update(s.dyn_size_ext.dims_set)
+            if mask_value.dyn_size_ext is not None:
+                dim_size_dims.update(mask_value.dyn_size_ext.dims_set)
+            if not mask.dims_set.issubset(dim_size_dims):
+                assert not mask.dims_set.intersection(dim_size_dims)  # not sure...
+                return s
+        new_dyn_size = _mask(
+            s.get_size_tensor(),
+            mask=mask,
+            mask_value=mask_value.get_size_tensor(),
+            dim_map=dim_map,
+            allow_dim_extension=allow_dim_extension,
+        )
+        new_dim = Dim(new_dyn_size, name=_extend_dim_name(s.name))
+        dim_map[s] = dim_map[mask_value] = new_dim
+        return new_dim
+    return s
+
+
+def _mask(
+    s: T, mask_value: Union[T, Tensor, float, None], *, mask: Tensor, dim_map: Dict[Dim, Dim], allow_dim_extension: bool
+) -> T:
+    if s is None:
+        return s
+    if isinstance(s, Tensor):
+        if dim_map:
+            for d in s.dims:
+                if d in dim_map:
+                    s = rf.replace_dim_v2(s, in_dim=d, out_dim=dim_map[d])
+            if isinstance(mask_value, Tensor):
+                for d in mask_value.dims:
+                    if d in dim_map:
+                        mask_value = rf.replace_dim_v2(mask_value, in_dim=d, out_dim=dim_map[d])
+        if not allow_dim_extension and isinstance(mask_value, Tensor):
+            if not s.dims_set.issuperset(mask_value.dims_set):
+                return s
+        if not allow_dim_extension or mask_value is None or (isinstance(mask_value, (int, float)) and mask_value == 0):
+            if mask.dims_set.issubset(s.dims_set):
+                return rf.where(mask, s, mask_value)
+            assert not mask.dims_set.intersection(s.dims_set)  # not sure...
+            return s
+        assert isinstance(mask_value, (int, float, Tensor))
+        return rf.where(mask, s, mask_value, allow_broadcast_all_sources=True)
+    if isinstance(s, Dim):
+        if mask_value is None:
+            return s
+        assert isinstance(mask_value, Dim)
+        if s == mask_value:
+            return s
+        return dim_map.get(s, s)
+    raise TypeError(f"_mask: unexpected {s!r} type {type(s).__name__}")
+
+
 def gather_nested(s: T, *, indices: Tensor, dim_map: Optional[Dict[Dim, Dim]] = None) -> T:
     """
     This is like :func:`gather`, but for nested structures.

{returnn-1.20250304.101951 → returnn-1.20250305.150759}/returnn/frontend/rec.py

@@ -70,6 +70,9 @@ class LSTM(rf.Module):
             out_dim=self.out_dim,
         )
         new_state = LstmState(h=new_state_h, c=new_state_c)
+        result.feature_dim = self.out_dim
+        new_state.h.feature_dim = self.out_dim
+        new_state.c.feature_dim = self.out_dim
 
         return result, new_state
 

{returnn-1.20250304.101951 → returnn-1.20250305.150759}/returnn/torch/engine.py

@@ -3,7 +3,7 @@ Main engine for PyTorch
 """
 
 from __future__ import annotations
-from typing import Optional, Any, Union, Callable, Dict, Set, Tuple
+from typing import Optional, Any, Union, Callable, Dict, Set
 from contextlib import nullcontext, ExitStack, contextmanager
 
 import gc
@@ -365,8 +365,6 @@ class Engine(EngineBase):
         zero_grad_next_step = True
         cur_count_grad_accum = 0
         extern_data = None
-        num_seqs = None
-        last_seq_idx = 0
 
         total_data_size_packed = NumbersDict()
         total_data_size_padded = NumbersDict()
@@ -400,20 +398,8 @@ class Engine(EngineBase):
                 )
 
                 complete_frac = float(extern_data_raw["complete_frac"])
-                num_seqs, last_seq_idx = _get_num_seqs_last_seq_idx(
-                    report_prefix=report_prefix,
-                    extern_data_raw=extern_data_raw,
-                    step_idx=step_idx,
-                    prev_num_seqs=num_seqs,
-                    prev_last_seq_idx=last_seq_idx,
-                )
-                epoch_continuous = (
-                    self.epoch - 1 + complete_frac
-                    if complete_frac >= 0.0
-                    else (self.epoch - 1 + (last_seq_idx + 1) / num_seqs)
-                    if num_seqs is not None
-                    else None
-                )
+                epoch_continuous = self.epoch - 1 + complete_frac if complete_frac >= 0.0 else None
+                num_seqs = int(extern_data_raw["num_seqs"])
 
                 # clear the gradients when every gradient accumulation loop starts
                 if zero_grad_next_step:
@@ -490,7 +476,7 @@ class Engine(EngineBase):
                     eval_info=dict(eval_info),
                     step_duration=step_duration,
                     start_elapsed=step_end_time - epoch_start_time,
-                    seq_idx=last_seq_idx,
+                    complete_frac=complete_frac,
                     num_seqs=num_seqs,
                     batch_size_info=_get_batch_size_info(extern_data) if self._log_batch_size else None,
                     log_memory_usage_device=self._device if self._log_memory_usage else None,
@@ -629,13 +615,18 @@ class Engine(EngineBase):
             accumulated_losses_dict = NumbersDict()
             accumulated_inv_norm_factors_dict = NumbersDict()
             step_idx = 0
+            eval_start_time = time.monotonic()
 
+            report_prefix = f"ep {self.epoch} {dataset_name} eval"
             with torch.no_grad():
                 for extern_data_raw in data_loader:
                     if self._torch_distributed_ctx and step_idx % 100 == 0:
                         _has_data = torch.tensor([True], device="cpu", dtype=torch.int8)
                         torch.distributed.broadcast(_has_data, src=0)
 
+                    complete_frac = float(extern_data_raw["complete_frac"])
+                    num_seqs = int(extern_data_raw["num_seqs"])
+
                     extern_data = extern_data_util.raw_dict_to_extern_data(
                         extern_data_raw,
                         extern_data_template=self.extern_data,
@@ -644,6 +635,8 @@ class Engine(EngineBase):
                     )
 
                     self._run_step(extern_data, train_func=True)
+                    step_end_time = time.monotonic()
+
                     train_ctx = rf.get_run_ctx()
 
                     losses_dict = NumbersDict(
@@ -664,9 +657,12 @@ class Engine(EngineBase):
                     accumulated_inv_norm_factors_dict += inv_norm_factors_dict
                     eval_info = self._maybe_extend_losses_info(losses_dict / inv_norm_factors_dict)
                     _print_process(
-                        f"ep {self.epoch} {dataset_name} eval",
+                        report_prefix,
                         step=step_idx,
                         eval_info=dict(eval_info),
+                        complete_frac=complete_frac,
+                        num_seqs=num_seqs,
+                        start_elapsed=step_end_time - eval_start_time,
                         log_memory_usage_device=self._device if self._log_memory_usage else None,
                     )
                     step_idx += 1
@@ -1290,8 +1286,6 @@ class Engine(EngineBase):
             new_dim.dyn_size_ext = _get_tensor_wo_batch_numpy(dim.dyn_size_ext)
             return new_dim
 
-        num_seqs = None
-        last_seq_idx = 0
         report_prefix = f"ep {self.epoch} {dataset.name} forward"
         with torch.no_grad():
             callback.init(model=self._orig_model)
@@ -1300,13 +1294,8 @@ class Engine(EngineBase):
             for extern_data_raw in data_loader:
                 step_begin_time = time.monotonic()
 
-                num_seqs, last_seq_idx = _get_num_seqs_last_seq_idx(
-                    report_prefix=report_prefix,
-                    extern_data_raw=extern_data_raw,
-                    step_idx=step_idx,
-                    prev_num_seqs=num_seqs,
-                    prev_last_seq_idx=last_seq_idx,
-                )
+                complete_frac = float(extern_data_raw["complete_frac"])
+                num_seqs = int(extern_data_raw["num_seqs"])
 
                 if self._forward_step_expected_outputs:
                     # Also resets any dyn dims, which might have been set in the prev step.
@@ -1354,7 +1343,7 @@ class Engine(EngineBase):
                     eval_info=None,
                     step_duration=step_duration,
                     start_elapsed=step_end_time - epoch_start_time,
-                    seq_idx=last_seq_idx,
+                    complete_frac=complete_frac,
                     num_seqs=num_seqs,
                     batch_size_info=_get_batch_size_info(extern_data) if self._log_batch_size else None,
                     log_memory_usage_device=self._device if self._log_memory_usage else None,
@@ -1442,7 +1431,7 @@ def _print_process(
     batch_size_info: Optional[Dict[str, Any]] = None,
     step_duration: Optional[float] = None,
     start_elapsed: Optional[float] = None,
-    seq_idx: Optional[int] = None,
+    complete_frac: Optional[float] = None,
     num_seqs: Optional[int] = None,
     log_memory_usage_device: Optional[str] = None,
 ):
@@ -1455,11 +1444,14 @@ def _print_process(
     :param batch_size_info:
     :param step_duration: time elapsed for this step (secs)
     :param start_elapsed: time elapsed since epoch start (secs)
-    :param num_seqs: total number of sequences for this epoch
+    :param complete_frac: how much of the current epoch is already consumed
+    :param num_seqs: total number of seqs this epoch
     :param log_memory_usage_device: if given, will log memory usage (peak allocated memory)
     :return: nothing, will be printed to log
     """
     if log.verbose[5]:  # report every minibatch
+        if step == 0 and num_seqs is not None and num_seqs >= 0:
+            print(f"{report_prefix} num_seqs: {num_seqs}", file=log.v5)
         info = [report_prefix, "step %i" % step]
         if eval_info:  # Such as score.
             info += ["%s %s" % (k, _format_score_value(v)) for k, v in eval_info.items()]
@@ -1475,17 +1467,16 @@ def _print_process(
             info += ["%.3f sec/step" % step_duration]
         if start_elapsed is not None:
             info += ["elapsed %s" % hms(start_elapsed)]
-        if num_seqs is not None:
-            assert seq_idx is not None and start_elapsed is not None  # unexpected combination...
-            complete = (seq_idx + 1) / num_seqs
-            assert 1 >= complete > 0, f"{step} step, {num_seqs} num_seqs"
-            total_time_estimated = start_elapsed / complete
+        if complete_frac is not None:
+            assert 1 >= complete_frac > 0, f"{step} step, {complete_frac} complete_frac"
+            assert start_elapsed is not None
+            total_time_estimated = start_elapsed / complete_frac
             remaining_estimated = total_time_estimated - start_elapsed
             info += [
                 "exp. remaining %s" % hms(remaining_estimated),
-                "complete %.02f%%" % (complete * 100),
+                "complete %.02f%%" % (complete_frac * 100),
             ]
-        if start_elapsed is not None and num_seqs is None:
+        if start_elapsed is not None and complete_frac is None:
             info += ["(unk epoch len)"]
         print(", ".join(filter(None, info)), file=log.v5)
 
@@ -1634,27 +1625,3 @@ def _get_total_grad_norm(model: torch.nn.Module, p: float) -> float:
             p=p,
         ).item()
     )
-
-
-def _get_num_seqs_last_seq_idx(
-    *,
-    report_prefix: str,
-    extern_data_raw: Dict[str, Any],
-    step_idx: int,
-    prev_num_seqs: Optional[int],
-    prev_last_seq_idx: int,
-) -> Tuple[Optional[int], int]:
-    num_seqs = prev_num_seqs
-    num_seqs_ = int(extern_data_raw["num_seqs"]) if extern_data_raw.get("num_seqs", None) is not None else -1
-    # Note: The batches might have been shuffled,
-    # thus we cannot really assert that the seq_idx is always increasing.
-    last_seq_idx = max(int(extern_data_raw["seq_idx"].max()), prev_last_seq_idx)
-    if step_idx == 0:
-        if num_seqs_ >= 0:
-            print(f"{report_prefix} num_seqs: {num_seqs_}", file=log.v5)
-            num_seqs = num_seqs_
-    elif num_seqs_ >= 0:
-        assert num_seqs_ == num_seqs
-    if num_seqs is not None:
-        assert last_seq_idx < num_seqs
-    return num_seqs, last_seq_idx

{returnn-1.20250304.101951 → returnn-1.20250305.150759/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250304.101951
+Version: 1.20250305.150759
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250304.101951/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20250304.101951'
-long_version = '1.20250304.101951+git.0fa434e'