returnn 1.20250820.123936__tar.gz → 1.20250821.93927__tar.gz

{returnn-1.20250820.123936/returnn.egg-info → returnn-1.20250821.93927}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250820.123936
+Version: 1.20250821.93927
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250821.93927/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20250821.093927'
+long_version = '1.20250821.093927+git.ec56958'

{returnn-1.20250820.123936 → returnn-1.20250821.93927}/returnn/torch/updater.py

@@ -95,6 +95,8 @@ class Updater:
     Wraps a torch.optim.Optimizer, and extends it by some further functionality.
     """
 
+    _OptimizerParamGroupsExtraOpts = ("learning_rate_multiplier",)
+
     def __init__(self, *, config, network, device, initial_learning_rate=1.0):
         """
         :param returnn.config.Config config: config defining the training conditions.
@@ -131,6 +133,7 @@ class Updater:
 
         self._optimizer_opts: Optional[Dict[str, Any]] = None
         self.optimizer: Optional[torch.optim.Optimizer] = None
+        self._optimizer_param_groups_extra_opts: Optional[List[Dict[str, Any]]] = None
 
         self._grad_clip = self.config.float("gradient_clip", 0.0)
         self._grad_clip_global_norm = self.config.float("gradient_clip_global_norm", 0.0)
@@ -189,8 +192,15 @@ class Updater:
             )
             self._effective_learning_rate = float(lr)
         if self.optimizer:
-            for param_group in self.optimizer.param_groups:
-                param_group["lr"] = self._effective_learning_rate
+            if self._optimizer_param_groups_extra_opts:
+                assert len(self.optimizer.param_groups) == len(self._optimizer_param_groups_extra_opts)
+                lr_multiplies = [
+                    opts.get("learning_rate_multiplier", 1.0) for opts in self._optimizer_param_groups_extra_opts
+                ]
+            else:
+                lr_multiplies = [1.0] * len(self.optimizer.param_groups)
+            for i, param_group in enumerate(self.optimizer.param_groups):
+                param_group["lr"] = self._effective_learning_rate * lr_multiplies[i]
 
     def set_current_train_step(self, *, global_train_step: int, epoch: int, epoch_continuous: Optional[float] = None):
         """
@@ -273,7 +283,7 @@ class Updater:
         if optimizer_opts is None:
             raise ValueError("config field 'optimizer' needs to be set explicitely for the Torch backend")
         self._optimizer_opts = optimizer_opts
-        self.optimizer = self._create_optimizer(optimizer_opts)
+        self.optimizer, self._optimizer_param_groups_extra_opts = self._create_optimizer(optimizer_opts)
 
     def load_optimizer(self, filename):
         """
@@ -421,21 +431,20 @@ class Updater:
         """
         return self.optimizer
 
-    def _create_optimizer(self, optimizer_opts):
+    def _create_optimizer(self, optimizer_opts) -> Tuple[torch.optim.Optimizer, Optional[List[Dict[str, Any]]]]:
         """
         Returns a valid optimizer considering the dictionary given by the user in the config.
 
         :param dict[str]|str optimizer_opts: Optimizer configuration specified by the user.
             If it's a dict, it must contain "class" with the optimizer name or callable.
             If it's a str, it must be the optimizer name.
-        :return: A valid optimizer.
-        :rtype: torch.optim.Optimizer
+        :return: tuple (optimizer, optional optimizer_param_groups_extra_opts).
         """
         lr = self.learning_rate
 
         # If the parameter is already a valid optimizer, return it without further processing
         if isinstance(optimizer_opts, torch.optim.Optimizer):
-            return optimizer_opts
+            return optimizer_opts, None
         elif callable(optimizer_opts):
             optimizer_opts: Dict[str, Any] = {"class": optimizer_opts}
         else:
@@ -461,12 +470,23 @@ class Updater:
         lr = lr * opt_kwargs.pop("learning_rate_multiplier", 1.0)
         opt_kwargs["lr"] = lr
 
-        params_or_param_groups = self._get_optimizer_param_groups(optim_class, opt_kwargs)
-        optimizer = optim_class(params_or_param_groups, **opt_kwargs)
+        param_groups = self._get_optimizer_param_groups(optim_class, opt_kwargs)
+        param_groups = list(param_groups)
+        assert len(param_groups) > 0, "got an empty parameter list?"
+        if not isinstance(param_groups[0], dict):
+            param_groups = [{"params": param_groups}]
+        optimizer_param_groups_extra_opts: Optional[List[Dict[str, Any]]] = None
+        if any(any(key in group for key in self._OptimizerParamGroupsExtraOpts) for group in param_groups):
+            param_groups = [dict(group) for group in param_groups]  # copy to make sure we can modify it
+            optimizer_param_groups_extra_opts = [
+                {key: group.pop(key) for key in self._OptimizerParamGroupsExtraOpts if key in group}
+                for group in param_groups
+            ]
+        optimizer = optim_class(param_groups, **opt_kwargs)
         print("Optimizer: %s" % optimizer, file=log.v1)
         assert isinstance(optimizer, torch.optim.Optimizer)
 
-        return optimizer
+        return optimizer, optimizer_param_groups_extra_opts
 
     def _create_default_optimizer(self):
         """
@@ -514,7 +534,11 @@ class Updater:
             assert callable(custom_param_groups), f"invalid param_groups_custom {custom_param_groups!r}"
             rf_model = wrapped_pt_module_to_rf_module(self.network)
             custom_param_groups_ = custom_param_groups(
-                model=self.network, rf_model=rf_model, optimizer_class=optim_class, optimizer_opts=optimizer_opts
+                model=self.network,
+                rf_model=rf_model,
+                optimizer_class=optim_class,
+                optimizer_opts=optimizer_opts,
+                **get_fwd_compat_kwargs(),
             )
             assert isinstance(custom_param_groups_, Iterable) and all(
                 isinstance(group, dict) for group in custom_param_groups_
@@ -547,11 +571,9 @@ class Updater:
         # Parameters without weight decay: biases + LayerNorm/Embedding layers.
         wd_params = set()
         no_wd_params = set()
-        blacklist_wd_modules: Any = optimizer_opts.pop("weight_decay_modules_blacklist", None)
-        if blacklist_wd_modules is None:
-            blacklist_wd_modules = (torch.nn.LayerNorm, torch.nn.Embedding)
-        else:
-            blacklist_wd_modules = _wrap_user_blacklist_wd_modules(blacklist_wd_modules)
+        blacklist_wd_modules = wrap_user_blacklist_wd_modules(
+            optimizer_opts.pop("weight_decay_modules_blacklist", None)
+        )
         custom_include_check = optimizer_opts.pop("weight_decay_custom_include_check", None)
         if custom_include_check:
             assert callable(custom_include_check), f"invalid weight_decay_custom_include_check {custom_include_check!r}"
@@ -598,9 +620,16 @@ class Updater:
         return optim_groups
 
 
-def _wrap_user_blacklist_wd_modules(
-    mods: Sequence[Union[str, Type[rf.Module], Type[torch.nn.Module]]],
+def wrap_user_blacklist_wd_modules(
+    mods: Optional[Sequence[Union[str, Type[rf.Module], Type[torch.nn.Module]]]],
 ) -> Tuple[type, ...]:
+    """
+    Wraps the user-provided blacklist_weight_decay_modules into a tuple of types.
+    This supports both pure PyTorch modules (e.g. "torch.nn.LayerNorm")
+    and RF modules (e.g. "rf.LayerNorm"), which can be specified as strings or types.
+    """
+    if mods is None:
+        return torch.nn.LayerNorm, torch.nn.Embedding
     assert isinstance(mods, (list, tuple)), f"invalid blacklist_weight_decay_modules {mods!r}"
     res = []
     for mod in mods:

{returnn-1.20250820.123936 → returnn-1.20250821.93927}/returnn/util/basic.py

@@ -2459,8 +2459,12 @@ class DictRefKeys(Generic[K, V]):
     Like `dict`, but hash and equality of the keys
     """
 
-    def __init__(self):
+    def __init__(self, items: Union[None, Iterable[Tuple[K, V]], Dict[K, V]] = None, /, **kwargs):
         self._d = {}  # type: Dict[RefIdEq[K], V]
+        if items is not None:
+            self.update(items)
+        if kwargs:
+            self.update(kwargs)
 
     def __repr__(self):
         return "DictRefKeys(%s)" % ", ".join(["%r: %r" % (k, v) for (k, v) in self.items()])
@@ -2489,6 +2493,15 @@ class DictRefKeys(Generic[K, V]):
     def __contains__(self, item: K):
         return RefIdEq(item) in self._d
 
+    def update(self, other: Union[Dict[K, V], Iterable[Tuple[K, V]]], /):
+        """
+        :param other: dict or iterable of (key, value) tuples
+        """
+        if isinstance(other, dict):
+            other = other.items()
+        for k, v in other:
+            self[k] = v
+
 
 def make_dll_name(basename):
     """

{returnn-1.20250820.123936 → returnn-1.20250821.93927/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250820.123936
+Version: 1.20250821.93927
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250820.123936 → returnn-1.20250821.93927}/tests/test_torch_engine.py

@@ -582,6 +582,128 @@ def test_load_optimizer_old_format():
         updater.load_optimizer(tmp_dir + "/model.opt.new_format.pt")
 
 
+def test_updater_weight_decay_blacklist():
+    from returnn.util.basic import DictRefKeys
+
+    # Don't specify weight_decay_modules_blacklist, so it should use the default,
+    # which should exclude Embedding and LayerNorm, and all biases.
+    # So this also tests that the default behavior does not change unexpectedly.
+    config = Config(dict(optimizer={"class": "adamw", "weight_decay": 1e-3}))
+    model = torch.nn.Sequential(
+        torch.nn.Embedding(10, 5),
+        torch.nn.LayerNorm(5),
+        torch.nn.Linear(5, 5),
+        torch.nn.ReLU(),
+    )
+    updater = Updater(config=config, network=model, device=torch.device("cpu"))
+    updater.create_optimizer()
+    updater.set_current_train_step(global_train_step=0, epoch=1)
+
+    opt = updater.get_optimizer()
+    assert isinstance(opt, torch.optim.AdamW)
+    assert len(opt.param_groups) == 2
+    groups_by_wd = {pg.get("weight_decay", 0.0): pg for pg in opt.param_groups}
+    assert set(groups_by_wd.keys()) == {0.0, 1e-3}
+    param_to_name = DictRefKeys((param, name) for name, param in model.named_parameters())
+    params_by_wd = {wd: set(map(param_to_name.__getitem__, group["params"])) for wd, group in groups_by_wd.items()}
+    print("params by wd:", params_by_wd)
+    assert params_by_wd[0.0] == {"0.weight", "1.weight", "1.bias", "2.bias"}
+    assert params_by_wd[1e-3] == {"2.weight"}
+
+
+def test_updater_lr_multipliers():
+    from collections import defaultdict
+    from fnmatch import fnmatchcase
+    from typing import Dict, List, Set, Any
+    from returnn.util.basic import DictRefKeys, FrozenDict
+    from returnn.torch.updater import wrap_user_blacklist_wd_modules
+    from returnn.torch.frontend.bridge import wrapped_pt_module_to_rf_module
+
+    # noinspection PyShadowingNames
+    def _param_groups_custom(*, model: torch.nn.Module, optimizer_opts: Dict[str, Any], **_kwargs):
+        default_weight_decay = optimizer_opts.get("weight_decay", 0.0)
+
+        blacklist_wd_modules = wrap_user_blacklist_wd_modules(
+            optimizer_opts.pop("weight_decay_modules_blacklist", None)
+        )
+        lr_multipliers_by_patterns = optimizer_opts.pop("learning_rate_multipliers_by_patterns")
+
+        # Tracker of visited parameters to only add each parameter once, in case two modules share common parameters.
+        # We need the wrapper class RefIdEq because Parameters are compared by value and not by reference.
+        params_by_opts: defaultdict[FrozenDict, List[torch.nn.Parameter]] = defaultdict(list)
+        visited_params = DictRefKeys()
+        for module_name, module in model.named_modules():
+            module_name: str
+            module: torch.nn.Module
+            rf_module = wrapped_pt_module_to_rf_module(module)
+            for param_name, param in module.named_parameters(recurse=False):
+                param_name: str
+                param: torch.nn.Parameter
+                if param in visited_params:
+                    continue
+                visited_params[param] = True
+                full_param_name = "%s.%s" % (module_name, param_name) if module_name else param_name
+
+                opts = {}
+                if (
+                    param_name.endswith("bias")
+                    or isinstance(module, blacklist_wd_modules)
+                    or isinstance(rf_module, blacklist_wd_modules)
+                ):
+                    opts["weight_decay"] = 0.0
+                else:
+                    opts["weight_decay"] = default_weight_decay
+                for pattern, lr_multiplier in lr_multipliers_by_patterns.items():
+                    if fnmatchcase(full_param_name, pattern):
+                        if lr_multiplier != 1.0:
+                            opts["learning_rate_multiplier"] = lr_multiplier
+                        break
+                params_by_opts[FrozenDict(opts)].append(param)
+
+        return [{"params": params, **opts} for opts, params in params_by_opts.items()]
+
+    config = Config(
+        dict(
+            optimizer={
+                "class": "adamw",
+                "weight_decay": 1e-3,
+                "param_groups_custom": _param_groups_custom,
+                "learning_rate_multipliers_by_patterns": {"0.*": 1.0, "1.*": 0.5, "2.*": 0.1},
+            }
+        )
+    )
+    model = torch.nn.Sequential(
+        torch.nn.Embedding(10, 5),
+        torch.nn.LayerNorm(5),
+        torch.nn.Linear(5, 5),
+        torch.nn.ReLU(),
+    )
+    updater = Updater(config=config, network=model, device=torch.device("cpu"))
+    updater.create_optimizer()
+    updater.set_current_train_step(global_train_step=0, epoch=1)
+
+    param_to_name = DictRefKeys((param, name) for name, param in model.named_parameters())
+    opt = updater.get_optimizer()
+    param_names_by_opts: Dict[FrozenDict, Set[str]] = {}
+    for group in opt.param_groups:
+        group_opts = FrozenDict({k: group[k] for k in ["weight_decay", "lr"]})
+        assert group_opts not in param_names_by_opts  # unique
+        param_names_by_opts[group_opts] = {param_to_name[p] for p in group["params"]}
+    assert len(param_names_by_opts) == 4, "Expected 4 param groups"
+    for opts, ref_param_names in [
+        ({"weight_decay": 0.0, "lr": 1.0}, {"0.weight"}),
+        ({"weight_decay": 0.0, "lr": 0.5}, {"1.weight", "1.bias"}),
+        ({"weight_decay": 0.001, "lr": 0.1}, {"2.weight"}),
+        ({"weight_decay": 0.0, "lr": 0.1}, {"2.bias"}),
+    ]:
+        opts = FrozenDict(opts)
+        assert opts in param_names_by_opts, f"Expected param group with opts {opts} not found"
+        param_names = param_names_by_opts[opts]
+        assert param_names == ref_param_names, (
+            f"For opts {opts}, expected param names {ref_param_names} but got {param_names}"
+        )
+
+
 def test_optimizer_convert_aux_param():
     # See rf_module_to_pt_module aux_params_as_buffers option.
     # This causes a change in the optimizer state dict.

returnn-1.20250820.123936/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20250820.123936'
-long_version = '1.20250820.123936+git.9c74169'