returnn 1.20250820.171158__py3-none-any.whl → 1.20250826.115240__py3-none-any.whl

returnn/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250820.171158
+Version: 1.20250826.115240
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn/_setup_info_generated.py

@@ -1,2 +1,2 @@
-version = '1.20250820.171158'
-long_version = '1.20250820.171158+git.d60d270'
+version = '1.20250826.115240'
+long_version = '1.20250826.115240+git.2baa52f'

returnn/torch/updater.py

@@ -95,6 +95,8 @@ class Updater:
     Wraps a torch.optim.Optimizer, and extends it by some further functionality.
     """
 
+    _OptimizerParamGroupsExtraOpts = ("learning_rate_multiplier",)
+
     def __init__(self, *, config, network, device, initial_learning_rate=1.0):
         """
         :param returnn.config.Config config: config defining the training conditions.
@@ -131,6 +133,7 @@ class Updater:
 
         self._optimizer_opts: Optional[Dict[str, Any]] = None
         self.optimizer: Optional[torch.optim.Optimizer] = None
+        self._optimizer_param_groups_extra_opts: Optional[List[Dict[str, Any]]] = None
 
         self._grad_clip = self.config.float("gradient_clip", 0.0)
         self._grad_clip_global_norm = self.config.float("gradient_clip_global_norm", 0.0)
@@ -189,8 +192,15 @@ class Updater:
             )
             self._effective_learning_rate = float(lr)
         if self.optimizer:
-            for param_group in self.optimizer.param_groups:
-                param_group["lr"] = self._effective_learning_rate
+            if self._optimizer_param_groups_extra_opts:
+                assert len(self.optimizer.param_groups) == len(self._optimizer_param_groups_extra_opts)
+                lr_multiplies = [
+                    opts.get("learning_rate_multiplier", 1.0) for opts in self._optimizer_param_groups_extra_opts
+                ]
+            else:
+                lr_multiplies = [1.0] * len(self.optimizer.param_groups)
+            for i, param_group in enumerate(self.optimizer.param_groups):
+                param_group["lr"] = self._effective_learning_rate * lr_multiplies[i]
 
     def set_current_train_step(self, *, global_train_step: int, epoch: int, epoch_continuous: Optional[float] = None):
         """
@@ -273,7 +283,7 @@ class Updater:
         if optimizer_opts is None:
             raise ValueError("config field 'optimizer' needs to be set explicitely for the Torch backend")
         self._optimizer_opts = optimizer_opts
-        self.optimizer = self._create_optimizer(optimizer_opts)
+        self.optimizer, self._optimizer_param_groups_extra_opts = self._create_optimizer(optimizer_opts)
 
     def load_optimizer(self, filename):
         """
@@ -421,21 +431,20 @@ class Updater:
         """
         return self.optimizer
 
-    def _create_optimizer(self, optimizer_opts):
+    def _create_optimizer(self, optimizer_opts) -> Tuple[torch.optim.Optimizer, Optional[List[Dict[str, Any]]]]:
         """
         Returns a valid optimizer considering the dictionary given by the user in the config.
 
         :param dict[str]|str optimizer_opts: Optimizer configuration specified by the user.
             If it's a dict, it must contain "class" with the optimizer name or callable.
             If it's a str, it must be the optimizer name.
-        :return: A valid optimizer.
-        :rtype: torch.optim.Optimizer
+        :return: tuple (optimizer, optional optimizer_param_groups_extra_opts).
         """
         lr = self.learning_rate
 
         # If the parameter is already a valid optimizer, return it without further processing
         if isinstance(optimizer_opts, torch.optim.Optimizer):
-            return optimizer_opts
+            return optimizer_opts, None
         elif callable(optimizer_opts):
             optimizer_opts: Dict[str, Any] = {"class": optimizer_opts}
         else:
@@ -461,12 +470,23 @@ class Updater:
         lr = lr * opt_kwargs.pop("learning_rate_multiplier", 1.0)
         opt_kwargs["lr"] = lr
 
-        params_or_param_groups = self._get_optimizer_param_groups(optim_class, opt_kwargs)
-        optimizer = optim_class(params_or_param_groups, **opt_kwargs)
+        param_groups = self._get_optimizer_param_groups(optim_class, opt_kwargs)
+        param_groups = list(param_groups)
+        assert len(param_groups) > 0, "got an empty parameter list?"
+        if not isinstance(param_groups[0], dict):
+            param_groups = [{"params": param_groups}]
+        optimizer_param_groups_extra_opts: Optional[List[Dict[str, Any]]] = None
+        if any(any(key in group for key in self._OptimizerParamGroupsExtraOpts) for group in param_groups):
+            param_groups = [dict(group) for group in param_groups]  # copy to make sure we can modify it
+            optimizer_param_groups_extra_opts = [
+                {key: group.pop(key) for key in self._OptimizerParamGroupsExtraOpts if key in group}
+                for group in param_groups
+            ]
+        optimizer = optim_class(param_groups, **opt_kwargs)
         print("Optimizer: %s" % optimizer, file=log.v1)
         assert isinstance(optimizer, torch.optim.Optimizer)
 
-        return optimizer
+        return optimizer, optimizer_param_groups_extra_opts
 
     def _create_default_optimizer(self):
         """
@@ -551,11 +571,9 @@ class Updater:
         # Parameters without weight decay: biases + LayerNorm/Embedding layers.
         wd_params = set()
         no_wd_params = set()
-        blacklist_wd_modules: Any = optimizer_opts.pop("weight_decay_modules_blacklist", None)
-        if blacklist_wd_modules is None:
-            blacklist_wd_modules = (torch.nn.LayerNorm, torch.nn.Embedding)
-        else:
-            blacklist_wd_modules = _wrap_user_blacklist_wd_modules(blacklist_wd_modules)
+        blacklist_wd_modules = wrap_user_blacklist_wd_modules(
+            optimizer_opts.pop("weight_decay_modules_blacklist", None)
+        )
         custom_include_check = optimizer_opts.pop("weight_decay_custom_include_check", None)
         if custom_include_check:
             assert callable(custom_include_check), f"invalid weight_decay_custom_include_check {custom_include_check!r}"
@@ -602,9 +620,16 @@ class Updater:
         return optim_groups
 
 
-def _wrap_user_blacklist_wd_modules(
-    mods: Sequence[Union[str, Type[rf.Module], Type[torch.nn.Module]]],
+def wrap_user_blacklist_wd_modules(
+    mods: Optional[Sequence[Union[str, Type[rf.Module], Type[torch.nn.Module]]]],
 ) -> Tuple[type, ...]:
+    """
+    Wraps the user-provided blacklist_weight_decay_modules into a tuple of types.
+    This supports both pure PyTorch modules (e.g. "torch.nn.LayerNorm")
+    and RF modules (e.g. "rf.LayerNorm"), which can be specified as strings or types.
+    """
+    if mods is None:
+        return torch.nn.LayerNorm, torch.nn.Embedding
     assert isinstance(mods, (list, tuple)), f"invalid blacklist_weight_decay_modules {mods!r}"
     res = []
     for mod in mods:

returnn/util/file_cache.py

@@ -390,6 +390,10 @@ class FileCache:
                 #   - https://github.com/rwth-i6/returnn/pull/1709
                 os.utime(dst_filename, None)
                 os.utime(info_file_name, None)
+                # Ensure we proactively make space for other users
+                # even in case we have all files ready on disk.
+                # See for discussion: https://github.com/rwth-i6/returnn/pull/1752.
+                self.cleanup(need_at_least_free_space_size=0)
                 return
 
             print(f"FileCache: Copy file {src_filename} to cache")

{returnn-1.20250820.171158.dist-info → returnn-1.20250826.115240.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250820.171158
+Version: 1.20250826.115240
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250820.171158.dist-info → returnn-1.20250826.115240.dist-info}/RECORD

@@ -1,9 +1,9 @@
-returnn/PKG-INFO,sha256=Cbw-LFRDg3cxVzUdgT7yaNpnAUDTjOw5nRYgq8jjH8A,5215
+returnn/PKG-INFO,sha256=FML_TBBHEcTCM4--oW_2wEb8Yk4ybXtAH6CQ7tSums0,5215
 returnn/__init__.py,sha256=biBtRsM0WZ406vShaeH-9WFoqJ8XwTbn6g0EeFJ7l8E,1012
 returnn/__main__.py,sha256=lHyZcu_0yc9f7Vf_Kfdy9PmeU0T76XVXnpalHi5WKro,31740
 returnn/__old_mod_loader__.py,sha256=nvsNY-xELdS_IPNkv66Q9Rmvg4dbGW0-EBRDcCmctos,7654
 returnn/__setup__.py,sha256=22kQn2fh11iPM0hLb2Fy5sLmoU1JGvmDxXRYuRgQkwU,4659
-returnn/_setup_info_generated.py,sha256=OQkFtzn37F7h2FgPXE84vkON338V4uRqbMen7VsZWR8,77
+returnn/_setup_info_generated.py,sha256=BCJ_PJr520UrgVLYRMgDdj07KiDWctntLtC9aS8cTH0,77
 returnn/config.py,sha256=3tmKhB6FnQZaNdtcYsiB61JnEY--iZ2qmJ4yq0b6tE0,29140
 returnn/forward_iface.py,sha256=A_OJiaXsX4MlXQRzST86ylyxSUZbC402PQL1REcqHjM,911
 returnn/learning_rate_control.py,sha256=ZvWryAn_tv9DhV8sh1LV3eE34Yltl3On3mYZAG4hR9s,34684
@@ -208,7 +208,7 @@ returnn/torch/README.md,sha256=jzJ2FpOHW02vxN69yKaV97C9LI-hmvjBglKfdZXIDdc,85
 returnn/torch/__init__.py,sha256=MHEUyNHB20Vy89uKAqZoj6FxJKF1Gq3HW-i6ra1pNcI,24
 returnn/torch/distributed.py,sha256=_lyJR71HIoCHpMi5GztGM7YwrX54Am8zSkjnDkE1Lbk,7524
 returnn/torch/engine.py,sha256=JSsQZZiVs9TxRyFEJuR3iH-YZb9sRw7TzoIAIqmplZY,78275
-returnn/torch/updater.py,sha256=-v_uY-8jDhreXPmjJYR4cgrlW_7ZI4kt2X2xIZdX_DE,30377
+returnn/torch/updater.py,sha256=7lMoA01Yzp18MY5jjIFncsajTjOD713pK38nU6r-jiE,31999
 returnn/torch/data/__init__.py,sha256=6cLNEi8KoGI12PF6akN7mI_mtjlx-0hcQAfMYoExwik,132
 returnn/torch/data/extern_data.py,sha256=5al706ZaYtHWLp5VH2vS-rW69YXP3NHyOFRKY0WY714,7810
 returnn/torch/data/pipeline.py,sha256=HgIL0jQsPcgvh_SPC4wQ6BzclmrnpFja-UiboF_GPN4,29459
@@ -238,7 +238,7 @@ returnn/util/better_exchook.py,sha256=39yvRecluDgYhViwSkaQ8crJ_cBWI63KeEGuK4RKe5
 returnn/util/bpe.py,sha256=LWFhICZsEOnMwNws0lybPNzKRX6rSr8yKCvP65vjl9Y,19656
 returnn/util/debug.py,sha256=wuRzdg9zB84WWCGyTjmRR_zYypu8gXxlc0nZ6si9OC8,28224
 returnn/util/debug_helpers.py,sha256=0EINLK4uLtoSt5_kHs1M2NIFpMd0S7i4c4rx90U4fJk,2914
-returnn/util/file_cache.py,sha256=15p0daE4KmzO5OvkhDr9cA_zJNbJrX98mnL-2fECtmM,27575
+returnn/util/file_cache.py,sha256=ERGz6TEWqetGk4odj1x6cMfecfQ5G5G4e5psSrbx03Y,27852
 returnn/util/fsa.py,sha256=k2lJ8tyf_g44Xk1EPVLwDwpP4spoMTqIigDVOWocQHY,59177
 returnn/util/literal_py_to_pickle.py,sha256=3dnjWPeeiDT2xp4bRDgIf9yddx7b1AG7mOKEn_jiSl8,2173
 returnn/util/lru_cache.py,sha256=7Q5H3a8b07E8e1iB7PA9jCpRnxMJZOFS2KO07cy0gqk,11446
@@ -253,8 +253,8 @@ returnn/util/sig_proc.py,sha256=Tjz0VOAVyqu2qDCF5HZ1JjALjcFsHcNkcd96WgZeKfE,7265
 returnn/util/task_system.py,sha256=y4sMVXQ25Qd2z0rx03uOlXlkE-jbCYC1Sjfn-XlraVU,26003
 returnn/util/train_proc_manager.py,sha256=Pjht28k6uz6BNQ47uW6Gf880iyq5q4wx7P_K2tmoAM8,3266
 returnn/util/watch_memory.py,sha256=BR5P2kvBN6UI81cE0_1WAA6Hd1SByLbBaiDxvLhPOew,4213
-returnn-1.20250820.171158.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
-returnn-1.20250820.171158.dist-info/METADATA,sha256=Cbw-LFRDg3cxVzUdgT7yaNpnAUDTjOw5nRYgq8jjH8A,5215
-returnn-1.20250820.171158.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-returnn-1.20250820.171158.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
-returnn-1.20250820.171158.dist-info/RECORD,,
+returnn-1.20250826.115240.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
+returnn-1.20250826.115240.dist-info/METADATA,sha256=FML_TBBHEcTCM4--oW_2wEb8Yk4ybXtAH6CQ7tSums0,5215
+returnn-1.20250826.115240.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+returnn-1.20250826.115240.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
+returnn-1.20250826.115240.dist-info/RECORD,,