returnn 1.20241105.131828__tar.gz → 1.20241106.173429__tar.gz

{returnn-1.20241105.131828 → returnn-1.20241106.173429}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241105.131828
+Version: 1.20241106.173429
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241106.173429/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20241106.173429'
+long_version = '1.20241106.173429+git.0f87197'

{returnn-1.20241105.131828 → returnn-1.20241106.173429}/returnn/frontend/array_.py

@@ -36,6 +36,7 @@ __all__ = [
     "masked_scatter",
     "sequence_mask",
     "pack_padded",
+    "pad_packed",
     "gather",
     "scatter",
     "scatter_argmax",
@@ -627,6 +628,8 @@ def pack_padded(
     Packing means to only store the non-padded frames.
     This uses :func:`masked_select` internally based on the mask of non-masked frames.
 
+    See :func:`pad_packed` for the inverse operation.
+
     :param source:
     :param dims: dims in source to pack. the order defines the format. first dim is major, etc.
         if there are no padded frames, e.g. dims=[B,T] would just result in the [B*T,...] reshaped tensor.
@@ -648,6 +651,14 @@ def pack_padded(
     return rf.masked_select(source, mask=mask, dims=dims, out_dim=out_dim)
 
 
+def pad_packed(source: Tensor, *, in_dim: Dim, dims: Sequence[Dim]) -> Tensor:
+    """
+    Inverse of :func:`pack_padded`, i.e. unpack the sequence, i.e. pad it back to the original length.
+    """
+    mask = rf.sequence_mask(dims, device=source.device)
+    return rf.masked_scatter(source, mask=mask, in_dim=in_dim, dims=dims)
+
+
 # noinspection PyUnusedLocal
 def gather(
     source: Tensor,

{returnn-1.20241105.131828 → returnn-1.20241106.173429}/returnn/frontend/rand.py

@@ -69,13 +69,14 @@ __all__ = [
 
 def set_random_seed(seed: int):
     """
+    This initializes the random state of the backend
+    and also the step-based random state
+    (see :func:`get_static_step_based_seed`, only used when ``static=True`` in :func:`random`).
+
     Call this at the beginning of the program
     (after the RF backend was selected),
-    or when the model and computation graph is supposed to be reinitialized.
-
-    This initializes the random state of the backend and also the step-based random state.
-
-    This is *not* expected to be called after each epoch or step.
+    or when the model and computation graph is supposed to be reinitialized
+    or at the beginning of each epoch.
 
     :param seed: should depend on epoch or step
     """
@@ -124,6 +125,8 @@ def reset_step_random_state():
 
 def get_static_step_based_seed(*, size=None) -> Union[int, numpy.ndarray]:
     """
+    This is intended as a static seed for :func:`random` when ``static=True`` is used.
+
     :return: from the static step-based random state, get a seed
     """
     return _step_rnd.randint(2**31, size=size)
@@ -180,18 +183,20 @@ def random(
     :param int|float|Tensor|None bound: for uniform, defining the range [-bound, bound)
     :param int|float|Tensor|None minval: for uniform
     :param int|float|Tensor|None maxval: for uniform
-    :param int|list[int]|numpy.ndarray|None seed: If not given, uses self.network.random.randint,
-      i.e. then it is controlled by the global seed setting, and every layer would get its own seed.
-      If you specify it explicitly, make sure every :class:`RandomLayer` uses a different seed,
-      otherwise you would get the same random numbers everywhere.
+    :param int|list[int]|numpy.ndarray|None seed:
+        Only for the case ``static=True``.
+        If not given, uses self.network.random.randint,
+        i.e. then it is controlled by the global seed setting, and every layer would get its own seed.
+        If you specify it explicitly, make sure every :class:`RandomLayer` uses a different seed,
+        otherwise you would get the same random numbers everywhere.
     :param str|tf.random.Algorithm|None algorithm: see :class:`RandomStateInitLayer`
     :param Tensor|None explicit_state: You can pass the state explicitly here.
-      If not given, will be created automatically, and updated automatically.
-      You could pass a :class:`VariableLayer` with initial value via :class:`RandomStateInitLayer`,
-      or directly a :class:`RandomStateInitLayer`.
-      If auto_update_state is True, it must be a variable,
-      and every time a new random number is created, this variable is updated.
-      Otherwise (default), it will not be updated automatically.
+        If not given, will be created automatically, and updated automatically.
+        You could pass a :class:`VariableLayer` with initial value via :class:`RandomStateInitLayer`,
+        or directly a :class:`RandomStateInitLayer`.
+        If auto_update_state is True, it must be a variable,
+        and every time a new random number is created, this variable is updated.
+        Otherwise (default), it will not be updated automatically.
     :param bool|None auto_update_state: only used when you pass an explicit state
     :param bool|None static: if no state at all should be used. it just relies on the seed then.
     :param out: if given, will directly write into it, if possible by backend

{returnn-1.20241105.131828 → returnn-1.20241106.173429}/returnn/learning_rate_control.py

@@ -5,7 +5,7 @@ The base class is :class:`LearningRateControl`.
 
 from __future__ import annotations
 
-from typing import Optional, Any, Dict
+from typing import Optional, Union, Any, Dict
 import typing
 import os
 import returnn.util.basic as util
@@ -350,7 +350,7 @@ class LearningRateControl:
                 relative_error /= learning_rate / self.default_learning_rate
         return relative_error
 
-    def set_epoch_error(self, epoch, error):
+    def set_epoch_error(self, epoch: int, error: Dict[str, Union[float, Dict[str, float]]]):
         """
         :type epoch: int
         :type error: dict[str,float|dict[str,float]]

{returnn-1.20241105.131828 → returnn-1.20241106.173429}/returnn/torch/data/returnn_dataset_wrapper.py

@@ -67,7 +67,14 @@ class ReturnnDatasetIterDataPipe(torch.utils.data.IterDataPipe):
 
     def reset(self):
         """
-        :return:
+        This is called by PyTorch DataLoader mechanism once we create a new iterator over the DataLoader.
+        This happens at the beginning of each epoch.
+
+        (Note: The mechanism where ``reset()`` is actually called is very obfuscated in PyTorch.
+        As I understand it, there is a IterDataPipe metaclass (_IterDataPipeMeta)
+        which automatically registers a hook on ``__iter__`` via ``hook_iterator``.
+        Deep inside the complex logic of this hook, it calls ``_set_datapipe_valid_iterator_id``
+        which then calls ``reset()``.)
         """
         self._reset_callback()
 

{returnn-1.20241105.131828 → returnn-1.20241106.173429}/returnn/torch/engine.py

@@ -34,7 +34,7 @@ from returnn.util import NumbersDict
 from returnn.util.basic import hms, NotSpecified
 from returnn.util.result_with_reason import ResultWithReason
 from returnn.util.debug import debug_shell
-from returnn.util.math import simplify_and_format_number
+from returnn.util.math import simplify_and_format_number, merge_random_seeds
 from returnn.forward_iface import ForwardCallbackIface
 
 from .updater import Updater
@@ -282,6 +282,17 @@ class Engine(EngineBase):
             }
         )
 
+        # Note: The RF/Torch default random number generator influences many things during training,
+        # such as dropout and other random operations inside the model,
+        # but also some potential shuffling in the dataset iterator.
+        # Also see Dataset._get_default_random_seed_offset() and Dataset._get_random_seed_for_epoch().
+        random_seed = self.config.int("random_seed", 42)
+        seed_data = [self.epoch, self.global_train_step, random_seed]
+        if self._torch_distributed_ctx:
+            seed_data.append(self._torch_distributed_ctx.rank())
+        random_seed = merge_random_seeds(seed_data)  # Join all seeds into one int.
+        rf.set_random_seed(random_seed)
+
     def _maybe_reset_dev_memory_caches(self, *, force: bool = False):
         if not force and not self._reset_dev_memory_caches:
             return
@@ -372,10 +383,9 @@ class Engine(EngineBase):
                 num_seqs_ = (
                     int(extern_data_raw["num_seqs"]) if extern_data_raw.get("num_seqs", None) is not None else -1
                 )
-                last_seq_idx_ = extern_data_raw["seq_idx"].max()
-                assert last_seq_idx_ >= last_seq_idx
-                last_seq_idx = int(last_seq_idx_)
-                del last_seq_idx_
+                # Note: The batches might have been shuffled,
+                # thus we cannot really assert that the seq_idx is always increasing.
+                last_seq_idx = max(int(extern_data_raw["seq_idx"].max()), last_seq_idx)
                 if step_idx == 0:
                     if num_seqs_ >= 0:
                         print(f"Epoch {self.epoch} num_seqs: {num_seqs_}", file=log.v5)
@@ -385,6 +395,7 @@ class Engine(EngineBase):
                 del num_seqs_
                 if num_seqs is not None:
                     assert last_seq_idx < num_seqs
+                epoch_continuous = (self.epoch - 1 + (last_seq_idx + 1) / num_seqs) if num_seqs is not None else None
 
                 # clear the gradients when every gradient accumulation loop starts
                 if zero_grad_next_step:
@@ -417,7 +428,10 @@ class Engine(EngineBase):
 
                 if accum_grad_multiple_step_dyn:
                     accum_grad_multiple_step = accum_grad_multiple_step_dyn(
-                        epoch=self.epoch, global_train_step=self.global_train_step
+                        epoch=self.epoch,
+                        epoch_continuous=epoch_continuous,
+                        global_train_step=self.global_train_step,
+                        **util.get_fwd_compat_kwargs(),
                     )
                 cur_count_grad_accum += 1
                 perform_update_step = cur_count_grad_accum >= accum_grad_multiple_step
@@ -477,9 +491,7 @@ class Engine(EngineBase):
                 step_idx += 1
                 self.global_train_step += 1
                 self._updater.set_current_train_step(
-                    global_train_step=self.global_train_step,
-                    epoch=self.epoch,
-                    epoch_continuous=(self.epoch - 1 + (last_seq_idx + 1) / num_seqs) if num_seqs is not None else None,
+                    global_train_step=self.global_train_step, epoch=self.epoch, epoch_continuous=epoch_continuous
                 )
         except Exception as exc:
             help_on_torch_exception(exc, step_idx=step_idx, model=self._orig_model, extern_data=extern_data)
@@ -488,8 +500,8 @@ class Engine(EngineBase):
         elapsed = time.monotonic() - epoch_start_time
         elapsed_computation_percentage = elapsed_computation_time / elapsed
         print(
-            "Trained %i steps, %s elapsed (%.1f%% computing time)"
-            % (step_idx, hms(elapsed), (elapsed_computation_percentage * 100.0)),
+            "Epoch %i: Trained %i steps, %s elapsed (%.1f%% computing time)"
+            % (self.epoch, step_idx, hms(elapsed), (elapsed_computation_percentage * 100.0)),
             file=log.v3,
         )
 
@@ -509,7 +521,7 @@ class Engine(EngineBase):
         if self._do_save():
             self.learning_rate_control.save()
 
-        print(f"Total train loss:", _format_score(dict(accumulated_losses_dict)), file=log.v3)
+        print(f"Epoch {self.epoch}: Total train loss:", _format_score(dict(accumulated_losses_dict)), file=log.v3)
 
         self._maybe_report_dev_memory_stats()
 
@@ -540,8 +552,6 @@ class Engine(EngineBase):
         self._reset_dev_memory_stats()
 
         eval_dump_str = []
-        score_keys = None
-        error_keys = None
 
         for dataset_name, dataset in self.eval_datasets.items():
             if skip_already_evaluated and self._is_dataset_evaluated(name=dataset_name):
@@ -583,10 +593,6 @@ class Engine(EngineBase):
                     self._run_step(extern_data, train_func=True)
                     train_ctx = rf.get_run_ctx()
 
-                    if score_keys is None:
-                        score_keys = set(name for name, loss in train_ctx.losses.items() if not loss.as_error)
-                        error_keys = set(name for name, loss in train_ctx.losses.items() if loss.as_error)
-
                     losses_dict = NumbersDict(
                         {
                             name: (
@@ -623,14 +629,7 @@ class Engine(EngineBase):
                 self.learning_rate_control.save()
 
             # Same format as the TF engine.
-            eval_dump_str += [
-                "%s: score %s error %s"
-                % (
-                    dataset_name,
-                    _format_score({name: accumulated_losses_dict[name] for name in score_keys}),
-                    _format_score({name: accumulated_losses_dict[name] for name in error_keys}),
-                )
-            ]
+            eval_dump_str += ["%s: %s" % (dataset_name, _format_score(dict(accumulated_losses_dict)))]
 
             if self._torch_distributed_ctx:
                 assert self._torch_distributed_ctx.rank() == 0
@@ -638,7 +637,11 @@ class Engine(EngineBase):
                 torch.distributed.broadcast(_has_data, src=0)
 
         if not self._torch_distributed_ctx or self._torch_distributed_ctx.rank() == 0:
-            print(" ".join(eval_dump_str) if eval_dump_str else "(No evaluations.)", file=log.v1)
+            print(
+                f"Epoch {self.epoch} evaluation:",
+                " ".join(eval_dump_str) if eval_dump_str else "(No evaluations.)",
+                file=log.v1,
+            )
 
         self._maybe_report_dev_memory_stats()
 
@@ -662,7 +665,10 @@ class Engine(EngineBase):
             for key, value in losses.items():
                 losses_[key] = value
                 if key in score_keys:
-                    losses_[f"{key}:exp"] = math.exp(value)
+                    try:
+                        losses_[f"{key}:exp"] = math.exp(value)
+                    except OverflowError:
+                        losses_[f"{key}:exp"] = float("inf")
             losses = NumbersDict(losses_)
         return losses
 
@@ -695,6 +701,32 @@ class Engine(EngineBase):
         max_seqs = self.config.typed_value("max_seqs", -1)
         batches_dataset = data_pipeline.BatchingIterDataPipe(wrapped_dataset, batch_size=batch_size, max_seqs=max_seqs)
 
+        online_shuffle_batches = self.config.typed_value("online_shuffle_batches", None)
+        if train and online_shuffle_batches:
+            if isinstance(online_shuffle_batches, int):
+                online_shuffle_batches = {"buffer_size": online_shuffle_batches}
+            elif isinstance(online_shuffle_batches, dict):
+                if "buffer_size" not in online_shuffle_batches:
+                    raise ValueError(
+                        f"config online_shuffle_batches, buffer_size not defined, got {online_shuffle_batches}"
+                    )
+            else:
+                raise TypeError(
+                    f"config online_shuffle_batches, expected int or dict, got {type(online_shuffle_batches)}"
+                )
+            # Note on random seed: This is handled by the PyTorch DataLoader iterator logic and IterDataPipe reset.
+            # Specifically, when we create a new DataLoader iterator,
+            # this will get fetch a new random number (from current Torch RNG state),
+            # use that as seed for the shuffle buffer.
+            # Note: In case of distributed training, it will broadcast the seed from rank 0 to all others.
+            # This is maybe not really what we want?
+            # https://discuss.pytorch.org/t/shuffleriterdatapipe-but-different-random-seed-per-distributed-rank/212612
+            # I currently don't really see a good way to override this behavior.
+            # Also note that we are likely using persistent multiprocessing data loader workers,
+            # so calling torch.utils.data.graph_settings.apply_random_seed here in the main proc
+            # will not have an effect then.
+            batches_dataset = torch.utils.data.datapipes.iter.Shuffler(batches_dataset, **online_shuffle_batches)
+
         loader_opts = self.config.typed_value("torch_dataloader_opts") or {}
         assert isinstance(loader_opts, dict), f"config torch_dataloader_opts, expected dict, got {type(loader_opts)}"
 
@@ -1333,7 +1365,7 @@ def _format_score(score: Dict[str, float]) -> str:
         return "None"
     if len(score) == 1:
         return _format_score_value(list(score.values())[0])
-    return " ".join(["%s %s" % (key.split(":", 2)[-1], _format_score_value(score[key])) for key in score.keys()])
+    return " ".join(["%s %s" % (k, _format_score_value(v)) for k, v in score.items()])
 
 
 def _format_score_value(v: Any) -> str:

{returnn-1.20241105.131828 → returnn-1.20241106.173429}/returnn/util/basic.py

@@ -5,7 +5,7 @@ Various generic utilities, which are shared across different backend engines.
 """
 
 from __future__ import annotations
-from typing import Optional, Union, Any, Generic, TypeVar, Iterable, Tuple, Dict, List, Callable
+from typing import Optional, Union, Any, Generic, TypeVar, Iterable, Tuple, Dict, List, Set, Callable
 
 import subprocess
 from subprocess import CalledProcessError
@@ -1962,9 +1962,9 @@ class NumbersDict:
         self.value = broadcast_value
         self.max = self._max_error
 
-    def copy(self):
+    def copy(self) -> NumbersDict:
         """
-        :rtype: NumbersDict
+        :return: copy
         """
         return NumbersDict(self)
 
@@ -1981,11 +1981,10 @@ class NumbersDict:
             numbers_dict={k: const_number for k in numbers_dict.dict.keys()},
         )
 
-    def copy_like(self, numbers_dict):
+    def copy_like(self, numbers_dict: NumbersDict) -> NumbersDict:
         """
-        :param NumbersDict numbers_dict:
+        :param numbers_dict:
         :return: copy of self with same keys as numbers_dict as far as we have them
-        :rtype: NumbersDict
         """
         if self.value is not None:
             return NumbersDict(
@@ -1998,11 +1997,11 @@ class NumbersDict:
             )
 
     @property
-    def keys_set(self):
+    def keys_set(self) -> Set[str]:
         """
         Also see :func:`keys_union` if you want to have a deterministic order.
 
-        :rtype: set[str]
+        :return: set of keys
         """
         return set(self.dict.keys())
 
@@ -2019,32 +2018,32 @@ class NumbersDict:
                     res.append(key)
         return res
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str):
         if self.value is not None:
             return self.dict.get(key, self.value)
         return self.dict[key]
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: str, value):
         self.dict[key] = value
 
-    def __delitem__(self, key):
+    def __delitem__(self, key: str):
         del self.dict[key]
 
-    def __contains__(self, item):
+    def __contains__(self, item: str):
         return item in self.dict
 
-    def get(self, key, default=None):
+    def get(self, key: str, default=None):
         """
-        :param str key:
+        :param key:
         :param T default:
         :rtype: object|T
         """
         # Keep consistent with self.__getitem__. If self.value is set, this will always be the default value.
         return self.dict.get(key, self.value if self.value is not None else default)
 
-    def pop(self, key, *args):
+    def pop(self, key: str, *args):
         """
-        :param str key:
+        :param key:
         :param T args: default, or not
         :rtype: object|T
         """
@@ -2057,22 +2056,21 @@ class NumbersDict:
         # which would only make sense for our values, not the dict keys.
         raise Exception("%s.__iter__ is undefined" % self.__class__.__name__)
 
-    def keys(self):
+    def keys(self) -> Iterable[str]:
         """
         :rtype: set[str]
         """
         return self.dict.keys()
 
-    def values(self):
+    def values(self) -> List[Any]:
         """
-        :rtype: list[object]
+        :return: values: dict values + self.value
         """
         return list(self.dict.values()) + ([self.value] if self.value is not None else [])
 
-    def items(self):
+    def items(self) -> Iterable[Tuple[str, Any]]:
         """
         :return: dict items. this excludes self.value
-        :rtype: set[(str,object)]
         """
         return self.dict.items()
 
@@ -2082,9 +2080,9 @@ class NumbersDict:
         """
         return self.value is not None or key in self.dict
 
-    def has_values(self):
+    def has_values(self) -> bool:
         """
-        :rtype: bool
+        :return: any values in self.dict or self.value
         """
         return bool(self.dict) or self.value is not None
 
@@ -2188,12 +2186,12 @@ class NumbersDict:
     def __neg__(self):
         return self.unary_op(op=lambda a: -a)
 
-    def __bool__(self):
+    def __bool__(self) -> bool:
         return any(self.values())
 
     __nonzero__ = __bool__  # Python 2
 
-    def elem_eq(self, other, result_with_default=True):
+    def elem_eq(self, other, result_with_default: bool = True) -> NumbersDict:
         """
         Element-wise equality check with other.
         Note about broadcast default value: Consider some key which is neither in self nor in other.
@@ -2204,8 +2202,8 @@ class NumbersDict:
           You can control the behavior via result_with_default.
 
         :param NumbersDict|T other:
-        :param bool result_with_default:
-        :rtype: NumbersDict
+        :param result_with_default:
+        :return: new NumbersDict with bool values
         """
 
         def op(a, b):
@@ -2225,19 +2223,17 @@ class NumbersDict:
             res.value = None
         return res
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         """
         :param NumbersDict|T other:
         :return: whether self == other elemwise. see self.elem_eq
-        :rtype: bool
         """
         return all(self.elem_eq(other).values())
 
-    def __ne__(self, other):
+    def __ne__(self, other) -> bool:
         """
         :param NumbersDict|T other:
         :return: not (self == other)
-        :rtype: bool
         """
         return not (self == other)
 
@@ -2246,11 +2242,10 @@ class NumbersDict:
         # and it would just confuse.
         raise Exception("%s.__cmp__ is undefined" % self.__class__.__name__)
 
-    def any_compare(self, other, cmp):
+    def any_compare(self, other, cmp) -> bool:
         """
         :param NumbersDict other:
         :param ((object,object)->True) cmp:
-        :rtype: True
         """
         for key in self.keys():
             if key in other.keys():
@@ -2283,11 +2278,11 @@ class NumbersDict:
         return min(*args)
 
     @classmethod
-    def max(cls, items):
+    def max(cls, items) -> NumbersDict:
         """
         Element-wise maximum for item in items.
+
         :param list[NumbersDict|int|float] items:
-        :rtype: NumbersDict
         """
         assert items
         if len(items) == 1:
@@ -2297,11 +2292,10 @@ class NumbersDict:
         return cls.max([items[0], cls.max(items[1:])])
 
     @classmethod
-    def min(cls, items):
+    def min(cls, items) -> NumbersDict:
         """
         Element-wise minimum for item in items.
         :param list[NumbersDict|int|float] items:
-        :rtype: NumbersDict
         """
         assert items
         if len(items) == 1:
@@ -2327,7 +2321,7 @@ class NumbersDict:
         """
         return min(self.values())
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         if self.value is None and not self.dict:
             return "%s()" % self.__class__.__name__
         if self.value is None and self.dict:

{returnn-1.20241105.131828 → returnn-1.20241106.173429}/returnn/util/math.py

@@ -3,8 +3,9 @@ Some mathematical functions, in pure NumPy.
 """
 
 from __future__ import annotations
-from typing import Union, Optional, Dict
+from typing import Union, Optional, Sequence, Dict
 import numpy
+import hashlib
 
 
 def ceil_div(a: int, b: int) -> int:
@@ -85,3 +86,19 @@ def simplify_and_format_number(n: Union[int, float]) -> str:
         return str(n).rstrip("0").rstrip(".")
     else:
         raise TypeError(f"Expected int or float, got {n!r} type {type(n)}")
+
+
+def merge_random_seeds(data_sources: Sequence[int], *, num_bytes: int = 4, signed: bool = False) -> int:
+    """
+    :param data_sources: A list of integers. We expect that they are all representable as 64-bit signed integers.
+    :param num_bytes: for the output seed.
+    :param signed: whether the output seed should be signed.
+    :return: A num_bytes*8-bit integer seed, deterministically derived from the input data.
+    """
+    # Convert each integer to bytes and concatenate them
+    combined = b"".join(int(source).to_bytes(8, "big", signed=True) for source in data_sources)
+    # Use SHA-256 to hash the combined bytes
+    hash_digest = hashlib.sha256(combined).digest()
+    # Convert the hash digest to an integer seed
+    seed = int.from_bytes(hash_digest[:num_bytes], "big", signed=signed)
+    return seed

{returnn-1.20241105.131828 → returnn-1.20241106.173429}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241105.131828
+Version: 1.20241106.173429
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20241105.131828 → returnn-1.20241106.173429}/tests/test_demos.py

@@ -79,6 +79,9 @@ def parse_last_fer(out: str) -> float:
         if not m:
             # example: dev: score 0.03350000149202181 error 0.009919877954075871
             m = re.match("dev: score .* error ([0-9.]+)\\s?", line)
+        if not m:
+            # example: Epoch 2 evaluation: dev: ce 0.034 fer 0.019
+            m = re.match("Epoch [0-9]+ evaluation: dev: .* fer ([0-9.]+)\\s?", line)
         if not m:
             continue
         parsed_fer = float(m.group(1))

returnn-1.20241105.131828/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20241105.131828'
-long_version = '1.20241105.131828+git.0494bcf'