returnn 1.20241105.131828__tar.gz → 1.20241106.124322__tar.gz

{returnn-1.20241105.131828 → returnn-1.20241106.124322}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241105.131828
+Version: 1.20241106.124322
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241106.124322/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20241106.124322'
+long_version = '1.20241106.124322+git.bf0c605'

{returnn-1.20241105.131828 → returnn-1.20241106.124322}/returnn/frontend/array_.py

@@ -36,6 +36,7 @@ __all__ = [
     "masked_scatter",
     "sequence_mask",
     "pack_padded",
+    "pad_packed",
     "gather",
     "scatter",
     "scatter_argmax",
@@ -627,6 +628,8 @@ def pack_padded(
     Packing means to only store the non-padded frames.
     This uses :func:`masked_select` internally based on the mask of non-masked frames.
 
+    See :func:`pad_packed` for the inverse operation.
+
     :param source:
     :param dims: dims in source to pack. the order defines the format. first dim is major, etc.
         if there are no padded frames, e.g. dims=[B,T] would just result in the [B*T,...] reshaped tensor.
@@ -648,6 +651,14 @@ def pack_padded(
     return rf.masked_select(source, mask=mask, dims=dims, out_dim=out_dim)
 
 
+def pad_packed(source: Tensor, *, in_dim: Dim, dims: Sequence[Dim]) -> Tensor:
+    """
+    Inverse of :func:`pack_padded`, i.e. unpack the sequence, i.e. pad it back to the original length.
+    """
+    mask = rf.sequence_mask(dims, device=source.device)
+    return rf.masked_scatter(source, mask=mask, in_dim=in_dim, dims=dims)
+
+
 # noinspection PyUnusedLocal
 def gather(
     source: Tensor,

{returnn-1.20241105.131828 → returnn-1.20241106.124322}/returnn/learning_rate_control.py

@@ -5,7 +5,7 @@ The base class is :class:`LearningRateControl`.
 
 from __future__ import annotations
 
-from typing import Optional, Any, Dict
+from typing import Optional, Union, Any, Dict
 import typing
 import os
 import returnn.util.basic as util
@@ -350,7 +350,7 @@ class LearningRateControl:
                 relative_error /= learning_rate / self.default_learning_rate
         return relative_error
 
-    def set_epoch_error(self, epoch, error):
+    def set_epoch_error(self, epoch: int, error: Dict[str, Union[float, Dict[str, float]]]):
         """
         :type epoch: int
         :type error: dict[str,float|dict[str,float]]

{returnn-1.20241105.131828 → returnn-1.20241106.124322}/returnn/torch/data/returnn_dataset_wrapper.py

@@ -67,7 +67,14 @@ class ReturnnDatasetIterDataPipe(torch.utils.data.IterDataPipe):
 
     def reset(self):
         """
-        :return:
+        This is called by PyTorch DataLoader mechanism once we create a new iterator over the DataLoader.
+        This happens at the beginning of each epoch.
+
+        (Note: The mechanism where ``reset()`` is actually called is very obfuscated in PyTorch.
+        As I understand it, there is a IterDataPipe metaclass (_IterDataPipeMeta)
+        which automatically registers a hook on ``__iter__`` via ``hook_iterator``.
+        Deep inside the complex logic of this hook, it calls ``_set_datapipe_valid_iterator_id``
+        which then calls ``reset()``.)
         """
         self._reset_callback()
 

{returnn-1.20241105.131828 → returnn-1.20241106.124322}/returnn/torch/engine.py

@@ -385,6 +385,7 @@ class Engine(EngineBase):
                 del num_seqs_
                 if num_seqs is not None:
                     assert last_seq_idx < num_seqs
+                epoch_continuous = (self.epoch - 1 + (last_seq_idx + 1) / num_seqs) if num_seqs is not None else None
 
                 # clear the gradients when every gradient accumulation loop starts
                 if zero_grad_next_step:
@@ -417,7 +418,10 @@ class Engine(EngineBase):
 
                 if accum_grad_multiple_step_dyn:
                     accum_grad_multiple_step = accum_grad_multiple_step_dyn(
-                        epoch=self.epoch, global_train_step=self.global_train_step
+                        epoch=self.epoch,
+                        epoch_continuous=epoch_continuous,
+                        global_train_step=self.global_train_step,
+                        **util.get_fwd_compat_kwargs(),
                     )
                 cur_count_grad_accum += 1
                 perform_update_step = cur_count_grad_accum >= accum_grad_multiple_step
@@ -477,9 +481,7 @@ class Engine(EngineBase):
                 step_idx += 1
                 self.global_train_step += 1
                 self._updater.set_current_train_step(
-                    global_train_step=self.global_train_step,
-                    epoch=self.epoch,
-                    epoch_continuous=(self.epoch - 1 + (last_seq_idx + 1) / num_seqs) if num_seqs is not None else None,
+                    global_train_step=self.global_train_step, epoch=self.epoch, epoch_continuous=epoch_continuous
                 )
         except Exception as exc:
             help_on_torch_exception(exc, step_idx=step_idx, model=self._orig_model, extern_data=extern_data)
@@ -488,8 +490,8 @@ class Engine(EngineBase):
         elapsed = time.monotonic() - epoch_start_time
         elapsed_computation_percentage = elapsed_computation_time / elapsed
         print(
-            "Trained %i steps, %s elapsed (%.1f%% computing time)"
-            % (step_idx, hms(elapsed), (elapsed_computation_percentage * 100.0)),
+            "Epoch %i: Trained %i steps, %s elapsed (%.1f%% computing time)"
+            % (self.epoch, step_idx, hms(elapsed), (elapsed_computation_percentage * 100.0)),
             file=log.v3,
         )
 
@@ -509,7 +511,7 @@ class Engine(EngineBase):
         if self._do_save():
             self.learning_rate_control.save()
 
-        print(f"Total train loss:", _format_score(dict(accumulated_losses_dict)), file=log.v3)
+        print(f"Epoch {self.epoch}: Total train loss:", _format_score(dict(accumulated_losses_dict)), file=log.v3)
 
         self._maybe_report_dev_memory_stats()
 
@@ -540,8 +542,6 @@ class Engine(EngineBase):
         self._reset_dev_memory_stats()
 
         eval_dump_str = []
-        score_keys = None
-        error_keys = None
 
         for dataset_name, dataset in self.eval_datasets.items():
             if skip_already_evaluated and self._is_dataset_evaluated(name=dataset_name):
@@ -583,10 +583,6 @@ class Engine(EngineBase):
                     self._run_step(extern_data, train_func=True)
                     train_ctx = rf.get_run_ctx()
 
-                    if score_keys is None:
-                        score_keys = set(name for name, loss in train_ctx.losses.items() if not loss.as_error)
-                        error_keys = set(name for name, loss in train_ctx.losses.items() if loss.as_error)
-
                     losses_dict = NumbersDict(
                         {
                             name: (
@@ -623,14 +619,7 @@ class Engine(EngineBase):
                 self.learning_rate_control.save()
 
             # Same format as the TF engine.
-            eval_dump_str += [
-                "%s: score %s error %s"
-                % (
-                    dataset_name,
-                    _format_score({name: accumulated_losses_dict[name] for name in score_keys}),
-                    _format_score({name: accumulated_losses_dict[name] for name in error_keys}),
-                )
-            ]
+            eval_dump_str += ["%s: %s" % (dataset_name, _format_score(dict(accumulated_losses_dict)))]
 
             if self._torch_distributed_ctx:
                 assert self._torch_distributed_ctx.rank() == 0
@@ -638,7 +627,11 @@ class Engine(EngineBase):
                 torch.distributed.broadcast(_has_data, src=0)
 
         if not self._torch_distributed_ctx or self._torch_distributed_ctx.rank() == 0:
-            print(" ".join(eval_dump_str) if eval_dump_str else "(No evaluations.)", file=log.v1)
+            print(
+                f"Epoch {self.epoch} evaluation:",
+                " ".join(eval_dump_str) if eval_dump_str else "(No evaluations.)",
+                file=log.v1,
+            )
 
         self._maybe_report_dev_memory_stats()
 
@@ -1333,7 +1326,7 @@ def _format_score(score: Dict[str, float]) -> str:
         return "None"
     if len(score) == 1:
         return _format_score_value(list(score.values())[0])
-    return " ".join(["%s %s" % (key.split(":", 2)[-1], _format_score_value(score[key])) for key in score.keys()])
+    return " ".join(["%s %s" % (k, _format_score_value(v)) for k, v in score.items()])
 
 
 def _format_score_value(v: Any) -> str:

{returnn-1.20241105.131828 → returnn-1.20241106.124322}/returnn/util/basic.py

@@ -5,7 +5,7 @@ Various generic utilities, which are shared across different backend engines.
 """
 
 from __future__ import annotations
-from typing import Optional, Union, Any, Generic, TypeVar, Iterable, Tuple, Dict, List, Callable
+from typing import Optional, Union, Any, Generic, TypeVar, Iterable, Tuple, Dict, List, Set, Callable
 
 import subprocess
 from subprocess import CalledProcessError
@@ -1962,9 +1962,9 @@ class NumbersDict:
         self.value = broadcast_value
         self.max = self._max_error
 
-    def copy(self):
+    def copy(self) -> NumbersDict:
         """
-        :rtype: NumbersDict
+        :return: copy
         """
         return NumbersDict(self)
 
@@ -1981,11 +1981,10 @@ class NumbersDict:
             numbers_dict={k: const_number for k in numbers_dict.dict.keys()},
         )
 
-    def copy_like(self, numbers_dict):
+    def copy_like(self, numbers_dict: NumbersDict) -> NumbersDict:
         """
-        :param NumbersDict numbers_dict:
+        :param numbers_dict:
         :return: copy of self with same keys as numbers_dict as far as we have them
-        :rtype: NumbersDict
         """
         if self.value is not None:
             return NumbersDict(
@@ -1998,11 +1997,11 @@ class NumbersDict:
             )
 
     @property
-    def keys_set(self):
+    def keys_set(self) -> Set[str]:
         """
         Also see :func:`keys_union` if you want to have a deterministic order.
 
-        :rtype: set[str]
+        :return: set of keys
         """
         return set(self.dict.keys())
 
@@ -2019,32 +2018,32 @@ class NumbersDict:
                     res.append(key)
         return res
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str):
         if self.value is not None:
             return self.dict.get(key, self.value)
         return self.dict[key]
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: str, value):
         self.dict[key] = value
 
-    def __delitem__(self, key):
+    def __delitem__(self, key: str):
         del self.dict[key]
 
-    def __contains__(self, item):
+    def __contains__(self, item: str):
         return item in self.dict
 
-    def get(self, key, default=None):
+    def get(self, key: str, default=None):
         """
-        :param str key:
+        :param key:
         :param T default:
         :rtype: object|T
         """
         # Keep consistent with self.__getitem__. If self.value is set, this will always be the default value.
         return self.dict.get(key, self.value if self.value is not None else default)
 
-    def pop(self, key, *args):
+    def pop(self, key: str, *args):
         """
-        :param str key:
+        :param key:
         :param T args: default, or not
         :rtype: object|T
         """
@@ -2057,22 +2056,21 @@ class NumbersDict:
         # which would only make sense for our values, not the dict keys.
         raise Exception("%s.__iter__ is undefined" % self.__class__.__name__)
 
-    def keys(self):
+    def keys(self) -> Iterable[str]:
         """
         :rtype: set[str]
         """
         return self.dict.keys()
 
-    def values(self):
+    def values(self) -> List[Any]:
         """
-        :rtype: list[object]
+        :return: values: dict values + self.value
         """
         return list(self.dict.values()) + ([self.value] if self.value is not None else [])
 
-    def items(self):
+    def items(self) -> Iterable[Tuple[str, Any]]:
         """
         :return: dict items. this excludes self.value
-        :rtype: set[(str,object)]
         """
         return self.dict.items()
 
@@ -2082,9 +2080,9 @@ class NumbersDict:
         """
         return self.value is not None or key in self.dict
 
-    def has_values(self):
+    def has_values(self) -> bool:
         """
-        :rtype: bool
+        :return: any values in self.dict or self.value
         """
         return bool(self.dict) or self.value is not None
 
@@ -2188,12 +2186,12 @@ class NumbersDict:
     def __neg__(self):
         return self.unary_op(op=lambda a: -a)
 
-    def __bool__(self):
+    def __bool__(self) -> bool:
         return any(self.values())
 
     __nonzero__ = __bool__  # Python 2
 
-    def elem_eq(self, other, result_with_default=True):
+    def elem_eq(self, other, result_with_default: bool = True) -> NumbersDict:
         """
         Element-wise equality check with other.
         Note about broadcast default value: Consider some key which is neither in self nor in other.
@@ -2204,8 +2202,8 @@ class NumbersDict:
           You can control the behavior via result_with_default.
 
         :param NumbersDict|T other:
-        :param bool result_with_default:
-        :rtype: NumbersDict
+        :param result_with_default:
+        :return: new NumbersDict with bool values
         """
 
         def op(a, b):
@@ -2225,19 +2223,17 @@ class NumbersDict:
             res.value = None
         return res
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         """
         :param NumbersDict|T other:
         :return: whether self == other elemwise. see self.elem_eq
-        :rtype: bool
         """
         return all(self.elem_eq(other).values())
 
-    def __ne__(self, other):
+    def __ne__(self, other) -> bool:
         """
         :param NumbersDict|T other:
         :return: not (self == other)
-        :rtype: bool
         """
         return not (self == other)
 
@@ -2246,11 +2242,10 @@ class NumbersDict:
         # and it would just confuse.
         raise Exception("%s.__cmp__ is undefined" % self.__class__.__name__)
 
-    def any_compare(self, other, cmp):
+    def any_compare(self, other, cmp) -> bool:
         """
         :param NumbersDict other:
         :param ((object,object)->True) cmp:
-        :rtype: True
         """
         for key in self.keys():
             if key in other.keys():
@@ -2283,11 +2278,11 @@ class NumbersDict:
         return min(*args)
 
     @classmethod
-    def max(cls, items):
+    def max(cls, items) -> NumbersDict:
         """
         Element-wise maximum for item in items.
+
         :param list[NumbersDict|int|float] items:
-        :rtype: NumbersDict
         """
         assert items
         if len(items) == 1:
@@ -2297,11 +2292,10 @@ class NumbersDict:
         return cls.max([items[0], cls.max(items[1:])])
 
     @classmethod
-    def min(cls, items):
+    def min(cls, items) -> NumbersDict:
         """
         Element-wise minimum for item in items.
         :param list[NumbersDict|int|float] items:
-        :rtype: NumbersDict
         """
         assert items
         if len(items) == 1:
@@ -2327,7 +2321,7 @@ class NumbersDict:
         """
         return min(self.values())
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         if self.value is None and not self.dict:
             return "%s()" % self.__class__.__name__
         if self.value is None and self.dict:

{returnn-1.20241105.131828 → returnn-1.20241106.124322}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241105.131828
+Version: 1.20241106.124322
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20241105.131828 → returnn-1.20241106.124322}/tests/test_demos.py

@@ -79,6 +79,9 @@ def parse_last_fer(out: str) -> float:
         if not m:
             # example: dev: score 0.03350000149202181 error 0.009919877954075871
             m = re.match("dev: score .* error ([0-9.]+)\\s?", line)
+        if not m:
+            # example: Epoch 2 evaluation: dev: ce 0.034 fer 0.019
+            m = re.match("Epoch [0-9]+ evaluation: dev: .* fer ([0-9.]+)\\s?", line)
         if not m:
             continue
         parsed_fer = float(m.group(1))

returnn-1.20241105.131828/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20241105.131828'
-long_version = '1.20241105.131828+git.0494bcf'