returnn 1.20231108.140626__tar.gz → 1.20231109.165622__tar.gz

{returnn-1.20231108.140626 → returnn-1.20231109.165622}/LICENSE

@@ -159,4 +159,10 @@ Contains code from CPython:
  - Copyright 2001..2021 Python Software Foundation
  - py-to-pickle.cpp
 
+Contains code from PyTorch:
+ - https://github.com/pytorch/pytorch/
+ - BSD 3-Clause License
+ - Copyright 2016..2023 various developers
+ - Various code snippets
+
 Various code snippets from StackOverflow, which are under Creative Commons / Public Domain.

{returnn-1.20231108.140626/returnn.egg-info → returnn-1.20231109.165622}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231108.140626
+Version: 1.20231109.165622
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20231109.165622/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20231109.165622'
+long_version = '1.20231109.165622+git.0f48d49'

{returnn-1.20231108.140626 → returnn-1.20231109.165622}/returnn/frontend/container.py

@@ -24,7 +24,9 @@ class ModuleList(rf.Module, Generic[__ModT]):
     def __init__(self, *modules: Union[__ModT, Iterable[__ModT], Dict[str, __ModT], ModuleList]):
         super().__init__()
         if len(modules) == 1 and isinstance(modules[0], dict):
-            for key, module in modules[0].items():
+            for i, (key, module) in enumerate(modules[0].items()):
+                if _is_int_str(key):
+                    key = str(i)
                 setattr(self, key, _convert_to_module(module))
         elif len(modules) == 1 and isinstance(modules[0], ModuleList):
             for key, module in modules[0]._get_modules().items():
@@ -64,17 +66,43 @@ class ModuleList(rf.Module, Generic[__ModT]):
         """module items"""
         return self._get_modules().items()
 
-    def __getitem__(self, idx) -> Union[ModuleList[__ModT], __ModT]:
-        from builtins import slice
-
+    def __getitem__(self, idx: Union[slice, int]) -> Union[ModuleList[__ModT], __ModT]:
         if isinstance(idx, slice):
             return self.__class__(dict(list(self._get_modules().items())[idx]))
         else:
             return list(self._get_modules().values())[idx]
 
-    def __setitem__(self, idx: int, module: __ModT) -> None:
+    def __setitem__(self, idx: Union[slice, int], module: Union[__ModT, Iterable[__ModT]]) -> None:
         key = list(self._get_modules().keys())[idx]
-        return setattr(self, key, _convert_to_module(module))
+        if isinstance(idx, slice):
+            assert not idx.step or idx.step == 1  # not supported
+            mod_items = list(self._get_modules().items())
+            if idx.stop is not None:
+                remaining = mod_items[idx.stop :]
+            else:
+                remaining = []
+            # Delete also remaining, and then re-add them later, such that indices are correct.
+            for k, _ in mod_items[idx.start :]:
+                delattr(self, k)
+            i = idx.start
+            for mod_ in module:
+                assert not hasattr(self, str(i))
+                setattr(self, str(i), _convert_to_module(mod_))
+                i += 1
+            for k, v in remaining:
+                if _is_int_str(k):
+                    k = str(i)
+                assert not hasattr(self, k)
+                setattr(self, k, v)
+                i += 1
+        else:
+            setattr(self, key, _convert_to_module(module))
+
+    def __delitem__(self, key: Union[slice, int]):
+        if isinstance(key, slice):
+            self[key] = []
+        else:
+            self[key : key + 1] = []
 
     __call__ = rf.Module.__call__  # stays abstract
 
@@ -108,7 +136,7 @@ def _convert_to_module(obj: _ModT) -> rf.Module:
     elif callable(obj):
         return rf.Functional(obj)
     else:
-        raise TypeError(f"did not expect {obj!r}")
+        raise TypeError(f"Expected rf.Module or callable, did not expect {obj!r} ({type(obj)})")
 
 
 def _is_iterable(obj) -> bool:
@@ -126,9 +154,16 @@ class ParameterList(rf.Module):
     Parameter list, getting passed an Iterable of Parameters and creates a list of Parameters in that order
     """
 
-    def __init__(self, *parameters: Union[rf.Parameter, Iterable[rf.Parameter], ParameterList]):
+    def __init__(
+        self, *parameters: Union[rf.Parameter, Iterable[rf.Parameter], Dict[str, rf.Parameter], ParameterList]
+    ):
         super().__init__()
-        if len(parameters) == 1 and isinstance(parameters[0], ParameterList):
+        if len(parameters) == 1 and isinstance(parameters[0], dict):
+            for i, (key, parameter) in enumerate(parameters[0].items()):
+                if _is_int_str(key):
+                    key = str(i)
+                setattr(self, key, parameter)
+        elif len(parameters) == 1 and isinstance(parameters[0], ParameterList):
             for key, parameter in parameters[0]._get_parameters().items():
                 setattr(self, key, parameter)
         elif len(parameters) == 1 and _is_iterable(parameters[0]):
@@ -163,8 +198,6 @@ class ParameterList(rf.Module):
         return iter(self._get_parameters().values())
 
     def __getitem__(self, idx) -> Union[ParameterList, rf.Parameter]:
-        from builtins import slice
-
         if isinstance(idx, slice):
             return self.__class__(dict(list(self._get_parameters().items())[idx]))
         else:
@@ -175,3 +208,11 @@ class ParameterList(rf.Module):
         return setattr(self, key, rf.Parameter)
 
     __call__ = rf.Module.__call__  # stays abstract
+
+
+def _is_int_str(s: str) -> bool:
+    try:
+        int(s)
+        return True
+    except ValueError:
+        return False

{returnn-1.20231108.140626 → returnn-1.20231109.165622}/returnn/learning_rate_control.py

@@ -5,8 +5,10 @@ The base class is :class:`LearningRateControl`.
 
 from __future__ import annotations
 
-import os
+from typing import Optional, Any, Dict
 import typing
+import os
+import returnn.util.basic as util
 from returnn.util.basic import better_repr, simple_obj_repr, ObjAsDict, unicode
 from returnn.log import log
 import numpy
@@ -26,24 +28,53 @@ class LearningRateControl(object):
         such as the individual scores (cv or train; cross-entropy or frame-error or whatever).
         """
 
-        # Need to keep the non-PEP8 name for compatibility, because we store the repr of the object.
-        # noinspection PyPep8Naming
-        def __init__(self, learningRate, error=None):
+        def __init__(
+            self,
+            *,
+            learning_rate: float = None,
+            error: Optional[Dict[str, float]] = None,
+            meta: Optional[Dict[str, Any]] = None,
+            **kwargs,
+        ):
             """
-            :type learningRate: float
-            :type error: dict[str,float] | None
+            :param learning_rate:
+            :param error: scores (loss values) and errors (frame error rates, etc)
+            :param meta: any other extra information (e.g. effective learning rate)
+
+            Note that this is serialized as EpochData(learningRate=..., error=...),
+            and we keep that for compatibility,
+            so that is why we have special handling for kwargs.
             """
-            self.learning_rate = learningRate
+            if learning_rate is None:
+                learning_rate = kwargs.pop("learningRate", None)
+            if not isinstance(learning_rate, float):
+                raise TypeError(f"EpochData: unexpected learning_rate type: {type(learning_rate)}")
+            if kwargs:
+                raise TypeError(f"EpochData: unexpected kwargs: {kwargs}")
+            self.learning_rate = learning_rate
             if isinstance(error, float):  # Old format.
                 error = {"old_format_score": error}
-            if error is None:
+            elif error is None:
                 error = {}
+            if not isinstance(error, dict):
+                raise TypeError(f"EpochData: unexpected error type: {type(error)}")
+            if meta is None:
+                meta = {k[len(":meta:") :]: v for (k, v) in error.items() if k.startswith(":meta:")}
+                error = {k: v for (k, v) in error.items() if not k.startswith(":meta:")}
+            if not isinstance(meta, dict):
+                raise TypeError(f"EpochData: unexpected meta type: {type(meta)}")
+            if any(k for k in error if k.startswith(":meta:")):
+                raise ValueError(f"EpochData: unexpected error keys: {error}")
             self.error = error
+            self.meta = meta
 
         def __repr__(self):
             # This is being used for serialization, and we want some forward/backward compatibility,
             # so we should try to keep this consistent.
-            return "EpochData(learningRate=%s, error=%s)" % (better_repr(self.learning_rate), better_repr(self.error))
+            return "EpochData(learningRate=%s, error=%s)" % (
+                better_repr(self.learning_rate),
+                better_repr(util.dict_joined(self.error, {f":meta:{k}": v for (k, v) in self.meta.items()})),
+            )
 
     @classmethod
     def load_initial_kwargs_from_config(cls, config):
@@ -263,7 +294,7 @@ class LearningRateControl(object):
             if not self.epoch_data[epoch].learning_rate:
                 self.epoch_data[epoch].learning_rate = learning_rate
         else:
-            self.epoch_data[epoch] = self.EpochData(learning_rate)
+            self.epoch_data[epoch] = self.EpochData(learning_rate=learning_rate)
 
     def get_last_epoch(self, epoch):
         """
@@ -829,7 +860,7 @@ def demo():
         if epoch in control.epoch_data:
             control.epoch_data[epoch].learning_rate = learning_rate
         else:
-            control.epoch_data[epoch] = control.EpochData(learningRate=learning_rate)
+            control.epoch_data[epoch] = control.EpochData(learning_rate=learning_rate)
     print("Finished, last stored epoch was %i." % max_epoch)
 
 

{returnn-1.20231108.140626 → returnn-1.20231109.165622}/returnn/tf/util/basic.py

@@ -3697,7 +3697,7 @@ def debug_register_better_repr():
 
 def cond(pred, true_fn, false_fn, name=None):
     """
-    This is a wrapper around tf.control_flow_ops.cond().
+    This is a wrapper around tf.cond().
     This will be a branched execution, i.e. either fn1() or fn2() will be executed,
     or at least the resulting graph will be evaluated.
     If pred can is constant at the call, only the corresponding fn will be called.
@@ -3727,9 +3727,8 @@ def cond(pred, true_fn, false_fn, name=None):
             return true_fn()
         else:
             return false_fn()
-    from tensorflow.python.ops import control_flow_ops
 
-    return control_flow_ops.cond(pred, true_fn, false_fn, name=name)
+    return tf.cond(pred, true_fn, false_fn, name=name)
 
 
 def single_strided_slice(x, axis, begin=None, end=None, step=None):

{returnn-1.20231108.140626 → returnn-1.20231109.165622}/returnn/torch/engine.py

@@ -236,6 +236,13 @@ class Engine(EngineBase):
         self._updater.set_learning_rate(self.learning_rate)
         self._updater.set_current_train_step(global_train_step=self.global_train_step, epoch=self.epoch)
 
+        self.learning_rate_control.epoch_data[self.epoch].meta.update(
+            {
+                "global_train_step": self.global_train_step,
+                "effective_learning_rate": self._updater.get_effective_learning_rate(),
+            }
+        )
+
     def train_epoch(self):
         """
         train one (sub)epoch
@@ -336,6 +343,13 @@ class Engine(EngineBase):
             file=log.v3,
         )
 
+        self.learning_rate_control.epoch_data[self.epoch].meta.update(
+            {
+                "epoch_num_train_steps": step_idx,
+                "epoch_train_time_secs": round(elapsed),
+            }
+        )
+
         accumulated_losses_dict = accumulated_losses_dict / accumulated_inv_norm_factors_dict
         self.learning_rate_control.set_epoch_error(
             self.epoch, {f"train_loss_{k}": v for k, v in accumulated_losses_dict.items()}

{returnn-1.20231108.140626 → returnn-1.20231109.165622/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20231108.140626
+Version: 1.20231109.165622
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20231108.140626 → returnn-1.20231109.165622}/tests/test_LearningRateControl.py

@@ -21,7 +21,7 @@ def test_save_load():
     control = LearningRateControl(default_learning_rate=1.0, filename=filename)
     assert 2 not in control.epoch_data
     control.epoch_data[2] = LearningRateControl.EpochData(
-        learningRate=0.0008,
+        learning_rate=0.0008,
         error={
             "dev_error_ctc": 0.22486433815293946,
             "dev_error_decision": 0.0,

returnn-1.20231108.140626/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20231108.140626'
-long_version = '1.20231108.140626+git.9fe9359'