returnn 1.20231109.122538__tar.gz → 1.20231109.165622__tar.gz

{returnn-1.20231109.122538/returnn.egg-info → returnn-1.20231109.165622}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231109.122538
+Version: 1.20231109.165622
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20231109.165622/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20231109.165622'
+long_version = '1.20231109.165622+git.0f48d49'

{returnn-1.20231109.122538 → returnn-1.20231109.165622}/returnn/frontend/container.py

@@ -3,7 +3,6 @@ container functions
 """
 
 from __future__ import annotations
-import operator
 import returnn.frontend as rf
 from returnn.tensor import Tensor
 from typing import Optional, TypeVar, Generic, Iterable, Iterator, Union, Tuple, Dict, Callable
@@ -22,9 +21,14 @@ class ModuleList(rf.Module, Generic[__ModT]):
     Module list, getting passed an Iterable of Modules and creates a list of Modules in that order
     """
 
-    def __init__(self, *modules: Union[__ModT, Iterable[__ModT], ModuleList]):
+    def __init__(self, *modules: Union[__ModT, Iterable[__ModT], Dict[str, __ModT], ModuleList]):
         super().__init__()
-        if len(modules) == 1 and isinstance(modules[0], ModuleList):
+        if len(modules) == 1 and isinstance(modules[0], dict):
+            for i, (key, module) in enumerate(modules[0].items()):
+                if _is_int_str(key):
+                    key = str(i)
+                setattr(self, key, _convert_to_module(module))
+        elif len(modules) == 1 and isinstance(modules[0], ModuleList):
             for key, module in modules[0]._get_modules().items():
                 setattr(self, key, _convert_to_module(module))
         elif len(modules) == 1 and _is_iterable(modules[0]):
@@ -35,29 +39,7 @@ class ModuleList(rf.Module, Generic[__ModT]):
                 setattr(self, str(idx), _convert_to_module(module))
 
     def _get_modules(self) -> Dict[str, __ModT]:
-        # Note: Insertion order is relevant here. We use it in __getitem__ for slicing, etc.
-        res = {}
-        i = 0
-        while True:
-            try:
-                res[str(i)] = getattr(self, str(i))
-                i += 1
-            except AttributeError:
-                break
-        return res
-
-    def _get_abs_index(self, idx: int) -> int:
-        """Get the absolute index for the list of modules"""
-        idx = operator.index(idx)
-        if not (-len(self) <= idx < len(self)):
-            raise IndexError("index {} is out of range".format(idx))
-        if idx < 0:
-            idx += len(self)
-        return idx
-
-    def _get_abs_string_index(self, idx: int) -> str:
-        """Get the absolute index for the list of modules"""
-        return str(self._get_abs_index(idx))
+        return {key: value for (key, value) in vars(self).items() if isinstance(value, rf.Module)}
 
     def append(self, module: __ModT) -> ModuleList[__ModT]:
         """
@@ -84,36 +66,43 @@ class ModuleList(rf.Module, Generic[__ModT]):
         """module items"""
         return self._get_modules().items()
 
-    def __getitem__(self, idx) -> Union[ModuleList[__ModT], __ModT]:
-        from builtins import slice
-
+    def __getitem__(self, idx: Union[slice, int]) -> Union[ModuleList[__ModT], __ModT]:
         if isinstance(idx, slice):
-            return self.__class__(list(self._get_modules().values())[idx])
+            return self.__class__(dict(list(self._get_modules().items())[idx]))
         else:
-            key = self._get_abs_string_index(idx)
-            if not hasattr(self, key):
-                raise IndexError("index {} is out of range".format(idx))
-            return getattr(self, key)
-
-    def __setitem__(self, idx: int, module: __ModT) -> None:
-        key = self._get_abs_string_index(idx)
-        if not hasattr(self, key):
-            raise IndexError("index {} is out of range".format(idx))
-        return setattr(self, key, _convert_to_module(module))
-
-    def __delitem__(self, idx: Union[int, slice]) -> None:
-        # To preserve numbering, we reconstruct the list of modules after deletion.
-        modules = list(self._get_modules().values())
-        old_len = len(modules)
-        del modules[idx]
+            return list(self._get_modules().values())[idx]
+
+    def __setitem__(self, idx: Union[slice, int], module: Union[__ModT, Iterable[__ModT]]) -> None:
+        key = list(self._get_modules().keys())[idx]
         if isinstance(idx, slice):
-            min_idx = self._get_abs_index(idx.start)
+            assert not idx.step or idx.step == 1  # not supported
+            mod_items = list(self._get_modules().items())
+            if idx.stop is not None:
+                remaining = mod_items[idx.stop :]
+            else:
+                remaining = []
+            # Delete also remaining, and then re-add them later, such that indices are correct.
+            for k, _ in mod_items[idx.start :]:
+                delattr(self, k)
+            i = idx.start
+            for mod_ in module:
+                assert not hasattr(self, str(i))
+                setattr(self, str(i), _convert_to_module(mod_))
+                i += 1
+            for k, v in remaining:
+                if _is_int_str(k):
+                    k = str(i)
+                assert not hasattr(self, k)
+                setattr(self, k, v)
+                i += 1
+        else:
+            setattr(self, key, _convert_to_module(module))
+
+    def __delitem__(self, key: Union[slice, int]):
+        if isinstance(key, slice):
+            self[key] = []
         else:
-            min_idx = self._get_abs_index(idx)
-        for i in range(min_idx, len(modules)):
-            setattr(self, str(i), modules[i])
-        for i in range(len(modules), old_len):
-            delattr(self, str(i))
+            self[key : key + 1] = []
 
     __call__ = rf.Module.__call__  # stays abstract
 
@@ -165,9 +154,16 @@ class ParameterList(rf.Module):
     Parameter list, getting passed an Iterable of Parameters and creates a list of Parameters in that order
     """
 
-    def __init__(self, *parameters: Union[rf.Parameter, Iterable[rf.Parameter], ParameterList]):
+    def __init__(
+        self, *parameters: Union[rf.Parameter, Iterable[rf.Parameter], Dict[str, rf.Parameter], ParameterList]
+    ):
         super().__init__()
-        if len(parameters) == 1 and isinstance(parameters[0], ParameterList):
+        if len(parameters) == 1 and isinstance(parameters[0], dict):
+            for i, (key, parameter) in enumerate(parameters[0].items()):
+                if _is_int_str(key):
+                    key = str(i)
+                setattr(self, key, parameter)
+        elif len(parameters) == 1 and isinstance(parameters[0], ParameterList):
             for key, parameter in parameters[0]._get_parameters().items():
                 setattr(self, key, parameter)
         elif len(parameters) == 1 and _is_iterable(parameters[0]):
@@ -202,8 +198,6 @@ class ParameterList(rf.Module):
         return iter(self._get_parameters().values())
 
     def __getitem__(self, idx) -> Union[ParameterList, rf.Parameter]:
-        from builtins import slice
-
         if isinstance(idx, slice):
             return self.__class__(dict(list(self._get_parameters().items())[idx]))
         else:
@@ -214,3 +208,11 @@ class ParameterList(rf.Module):
         return setattr(self, key, rf.Parameter)
 
     __call__ = rf.Module.__call__  # stays abstract
+
+
+def _is_int_str(s: str) -> bool:
+    try:
+        int(s)
+        return True
+    except ValueError:
+        return False

{returnn-1.20231109.122538 → returnn-1.20231109.165622}/returnn/learning_rate_control.py

@@ -5,8 +5,10 @@ The base class is :class:`LearningRateControl`.
 
 from __future__ import annotations
 
-import os
+from typing import Optional, Any, Dict
 import typing
+import os
+import returnn.util.basic as util
 from returnn.util.basic import better_repr, simple_obj_repr, ObjAsDict, unicode
 from returnn.log import log
 import numpy
@@ -26,24 +28,53 @@ class LearningRateControl(object):
         such as the individual scores (cv or train; cross-entropy or frame-error or whatever).
         """
 
-        # Need to keep the non-PEP8 name for compatibility, because we store the repr of the object.
-        # noinspection PyPep8Naming
-        def __init__(self, learningRate, error=None):
+        def __init__(
+            self,
+            *,
+            learning_rate: float = None,
+            error: Optional[Dict[str, float]] = None,
+            meta: Optional[Dict[str, Any]] = None,
+            **kwargs,
+        ):
             """
-            :type learningRate: float
-            :type error: dict[str,float] | None
+            :param learning_rate:
+            :param error: scores (loss values) and errors (frame error rates, etc)
+            :param meta: any other extra information (e.g. effective learning rate)
+
+            Note that this is serialized as EpochData(learningRate=..., error=...),
+            and we keep that for compatibility,
+            so that is why we have special handling for kwargs.
             """
-            self.learning_rate = learningRate
+            if learning_rate is None:
+                learning_rate = kwargs.pop("learningRate", None)
+            if not isinstance(learning_rate, float):
+                raise TypeError(f"EpochData: unexpected learning_rate type: {type(learning_rate)}")
+            if kwargs:
+                raise TypeError(f"EpochData: unexpected kwargs: {kwargs}")
+            self.learning_rate = learning_rate
             if isinstance(error, float):  # Old format.
                 error = {"old_format_score": error}
-            if error is None:
+            elif error is None:
                 error = {}
+            if not isinstance(error, dict):
+                raise TypeError(f"EpochData: unexpected error type: {type(error)}")
+            if meta is None:
+                meta = {k[len(":meta:") :]: v for (k, v) in error.items() if k.startswith(":meta:")}
+                error = {k: v for (k, v) in error.items() if not k.startswith(":meta:")}
+            if not isinstance(meta, dict):
+                raise TypeError(f"EpochData: unexpected meta type: {type(meta)}")
+            if any(k for k in error if k.startswith(":meta:")):
+                raise ValueError(f"EpochData: unexpected error keys: {error}")
             self.error = error
+            self.meta = meta
 
         def __repr__(self):
             # This is being used for serialization, and we want some forward/backward compatibility,
             # so we should try to keep this consistent.
-            return "EpochData(learningRate=%s, error=%s)" % (better_repr(self.learning_rate), better_repr(self.error))
+            return "EpochData(learningRate=%s, error=%s)" % (
+                better_repr(self.learning_rate),
+                better_repr(util.dict_joined(self.error, {f":meta:{k}": v for (k, v) in self.meta.items()})),
+            )
 
     @classmethod
     def load_initial_kwargs_from_config(cls, config):
@@ -263,7 +294,7 @@ class LearningRateControl(object):
             if not self.epoch_data[epoch].learning_rate:
                 self.epoch_data[epoch].learning_rate = learning_rate
         else:
-            self.epoch_data[epoch] = self.EpochData(learning_rate)
+            self.epoch_data[epoch] = self.EpochData(learning_rate=learning_rate)
 
     def get_last_epoch(self, epoch):
         """
@@ -829,7 +860,7 @@ def demo():
         if epoch in control.epoch_data:
             control.epoch_data[epoch].learning_rate = learning_rate
         else:
-            control.epoch_data[epoch] = control.EpochData(learningRate=learning_rate)
+            control.epoch_data[epoch] = control.EpochData(learning_rate=learning_rate)
     print("Finished, last stored epoch was %i." % max_epoch)
 
 

{returnn-1.20231109.122538 → returnn-1.20231109.165622}/returnn/tf/util/basic.py

@@ -3697,7 +3697,7 @@ def debug_register_better_repr():
 
 def cond(pred, true_fn, false_fn, name=None):
     """
-    This is a wrapper around tf.control_flow_ops.cond().
+    This is a wrapper around tf.cond().
     This will be a branched execution, i.e. either fn1() or fn2() will be executed,
     or at least the resulting graph will be evaluated.
     If pred can is constant at the call, only the corresponding fn will be called.
@@ -3727,9 +3727,8 @@ def cond(pred, true_fn, false_fn, name=None):
             return true_fn()
         else:
             return false_fn()
-    from tensorflow.python.ops import control_flow_ops
 
-    return control_flow_ops.cond(pred, true_fn, false_fn, name=name)
+    return tf.cond(pred, true_fn, false_fn, name=name)
 
 
 def single_strided_slice(x, axis, begin=None, end=None, step=None):

{returnn-1.20231109.122538 → returnn-1.20231109.165622}/returnn/torch/engine.py

@@ -236,6 +236,13 @@ class Engine(EngineBase):
         self._updater.set_learning_rate(self.learning_rate)
         self._updater.set_current_train_step(global_train_step=self.global_train_step, epoch=self.epoch)
 
+        self.learning_rate_control.epoch_data[self.epoch].meta.update(
+            {
+                "global_train_step": self.global_train_step,
+                "effective_learning_rate": self._updater.get_effective_learning_rate(),
+            }
+        )
+
     def train_epoch(self):
         """
         train one (sub)epoch
@@ -336,6 +343,13 @@ class Engine(EngineBase):
             file=log.v3,
         )
 
+        self.learning_rate_control.epoch_data[self.epoch].meta.update(
+            {
+                "epoch_num_train_steps": step_idx,
+                "epoch_train_time_secs": round(elapsed),
+            }
+        )
+
         accumulated_losses_dict = accumulated_losses_dict / accumulated_inv_norm_factors_dict
         self.learning_rate_control.set_epoch_error(
             self.epoch, {f"train_loss_{k}": v for k, v in accumulated_losses_dict.items()}

{returnn-1.20231109.122538 → returnn-1.20231109.165622/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231109.122538
+Version: 1.20231109.165622
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20231109.122538 → returnn-1.20231109.165622}/tests/test_LearningRateControl.py

@@ -21,7 +21,7 @@ def test_save_load():
     control = LearningRateControl(default_learning_rate=1.0, filename=filename)
     assert 2 not in control.epoch_data
     control.epoch_data[2] = LearningRateControl.EpochData(
-        learningRate=0.0008,
+        learning_rate=0.0008,
         error={
             "dev_error_ctc": 0.22486433815293946,
             "dev_error_decision": 0.0,

{returnn-1.20231109.122538 → returnn-1.20231109.165622}/tests/test_rf_container.py

@@ -76,6 +76,41 @@ def test_sequential_base_case():
     run_model(extern_data, lambda *, epoch, step: _Net(), _forward_step)
 
 
+def test_sequential_named_case():
+    time_dim = Dim(Tensor("time", [batch_dim], dtype="int32"))
+    in_dim = Dim(7, name="in")
+    extern_data = TensorDict(
+        {
+            "data": Tensor("data", [batch_dim, time_dim, in_dim], dtype="float32"),
+        }
+    )
+
+    class _Net(rf.Module):
+        def __init__(self):
+            super().__init__()
+            dims = [Dim(1, name="feat1"), Dim(2, name="feat2"), Dim(3, name="feat3")]
+            self.out_dim = dims[-1]
+            x = OrderedDict()
+            x["one"] = rf.Linear(in_dim, dims[0])
+            x["two"] = rf.Linear(dims[0], dims[1])
+            x["three"] = rf.Linear(dims[1], dims[2])
+            self.seq = rf.Sequential(x)
+
+        def __call__(self, data: Tensor) -> Tensor:
+            """
+            Forward
+            """
+            seq = self.seq(data)
+            return seq
+
+    # noinspection PyShadowingNames
+    def _forward_step(*, model: _Net, extern_data: TensorDict):
+        out = model(extern_data["data"])
+        out.mark_as_default_output(shape=(batch_dim, time_dim, model.out_dim))
+
+    run_model(extern_data, lambda *, epoch, step: _Net(), _forward_step)
+
+
 def test_parameter_list():
     time_dim = Dim(Tensor("time", [batch_dim], dtype="int32"))
     in_dim = Dim(7, name="in")

returnn-1.20231109.122538/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20231109.122538'
-long_version = '1.20231109.122538+git.e5a5d90'