returnn 1.20250725.163756__py3-none-any.whl → 1.20250810.211220__py3-none-any.whl

returnn/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250725.163756
+Version: 1.20250810.211220
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn/_setup_info_generated.py

@@ -1,2 +1,2 @@
-version = '1.20250725.163756'
-long_version = '1.20250725.163756+git.9859629'
+version = '1.20250810.211220'
+long_version = '1.20250810.211220+git.49e7afd'

returnn/torch/updater.py

@@ -113,6 +113,7 @@ class Updater:
         self._current_train_step = 0
         self._current_epoch = 1
         self._current_epoch_continuous = 0.0
+        self._num_consec_invalid_gradients_steps = 0
 
         self.learning_rate_function = self.config.typed_value("dynamic_learning_rate", None)
         if self.learning_rate_function is not None:
@@ -134,6 +135,9 @@ class Updater:
 
         self._grad_clip = self.config.float("gradient_clip", 0.0)
         self._grad_clip_global_norm = self.config.float("gradient_clip_global_norm", 0.0)
+        self._num_allowed_consec_invalid_gradient_steps = self.config.typed_value(
+            "num_allowed_consec_invalid_gradient_steps", None
+        )
         self._grad_noise = self.config.float("gradient_noise", 0.0)
 
         # Check other options we have in TF updater, which we might support here later as well,
@@ -208,6 +212,8 @@ class Updater:
             if the number of steps per epoch is known in advance.
         """
         self._current_train_step = global_train_step
+        if self._current_epoch != epoch:
+            self._num_consec_invalid_gradients_steps = 0
         self._current_epoch = epoch
         self._current_epoch_continuous = epoch_continuous
         self._update_effective_learning_rate()
@@ -224,12 +230,40 @@ class Updater:
         if self._grad_clip:
             torch.nn.utils.clip_grad_value_(self.network.parameters(), self._grad_clip)
         if self._grad_clip_global_norm:
-            torch.nn.utils.clip_grad_norm_(self.network.parameters(), self._grad_clip_global_norm)
+            norm = torch.nn.utils.clip_grad_norm_(self.network.parameters(), self._grad_clip_global_norm)
+        else:
+            norm = None
+
+        has_invalid_gradient = False
+        if self._num_allowed_consec_invalid_gradient_steps is not None:
+            if norm is None:
+                norm = torch.nn.utils.get_total_norm(self.network.parameters())
+            has_invalid_gradient = torch.isnan(norm) or torch.isinf(norm)
+            if has_invalid_gradient:
+                self._num_consec_invalid_gradients_steps += 1
+                if self._num_consec_invalid_gradients_steps > self._num_allowed_consec_invalid_gradient_steps:
+                    raise RuntimeError(
+                        f"Got {self._num_consec_invalid_gradients_steps} invalid gradients in succession, "
+                        f"abort training"
+                    )
+                else:
+                    invalid_grads_left = (
+                        self._num_allowed_consec_invalid_gradient_steps - self._num_consec_invalid_gradients_steps
+                    )
+                    print(
+                        f"Invalid gradient in step {self._current_train_step}, skipping. "
+                        f"{invalid_grads_left} subsequent broken steps left until training is aborted.",
+                        file=log.v2,
+                    )
+            else:
+                self._num_consec_invalid_gradients_steps = 0
 
         if grad_scaler is not None:
-            grad_scaler.step(self.optimizer)
+            if not has_invalid_gradient:
+                grad_scaler.step(self.optimizer)
+            # update needs to be called even if we discard the update due to an invalid gradient
             grad_scaler.update()
-        else:
+        elif not has_invalid_gradient:
             self.optimizer.step()
 
     def create_optimizer(self):

{returnn-1.20250725.163756.dist-info → returnn-1.20250810.211220.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250725.163756
+Version: 1.20250810.211220
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250725.163756.dist-info → returnn-1.20250810.211220.dist-info}/RECORD

@@ -1,9 +1,9 @@
-returnn/PKG-INFO,sha256=wx2M7IlY7YIufSJIJvDiBsaXtYwynBc4CmeInkz4dF0,5215
+returnn/PKG-INFO,sha256=os7SMsZDIE8iA3rqu8Yv3il3m8BO4WeHIcMrHhEGMzs,5215
 returnn/__init__.py,sha256=biBtRsM0WZ406vShaeH-9WFoqJ8XwTbn6g0EeFJ7l8E,1012
 returnn/__main__.py,sha256=lHyZcu_0yc9f7Vf_Kfdy9PmeU0T76XVXnpalHi5WKro,31740
 returnn/__old_mod_loader__.py,sha256=nvsNY-xELdS_IPNkv66Q9Rmvg4dbGW0-EBRDcCmctos,7654
 returnn/__setup__.py,sha256=22kQn2fh11iPM0hLb2Fy5sLmoU1JGvmDxXRYuRgQkwU,4659
-returnn/_setup_info_generated.py,sha256=ZvtqI_pqyL2IDUdjKEqtyIjWQIWPXyk41lwRrX4DPSM,77
+returnn/_setup_info_generated.py,sha256=DRGtGuwT8umaoAEnnZhmbwwrp5N9ngkeq-F3eNVVKEI,77
 returnn/config.py,sha256=3tmKhB6FnQZaNdtcYsiB61JnEY--iZ2qmJ4yq0b6tE0,29140
 returnn/forward_iface.py,sha256=A_OJiaXsX4MlXQRzST86ylyxSUZbC402PQL1REcqHjM,911
 returnn/learning_rate_control.py,sha256=ZvWryAn_tv9DhV8sh1LV3eE34Yltl3On3mYZAG4hR9s,34684
@@ -208,7 +208,7 @@ returnn/torch/README.md,sha256=jzJ2FpOHW02vxN69yKaV97C9LI-hmvjBglKfdZXIDdc,85
 returnn/torch/__init__.py,sha256=MHEUyNHB20Vy89uKAqZoj6FxJKF1Gq3HW-i6ra1pNcI,24
 returnn/torch/distributed.py,sha256=_lyJR71HIoCHpMi5GztGM7YwrX54Am8zSkjnDkE1Lbk,7524
 returnn/torch/engine.py,sha256=JSsQZZiVs9TxRyFEJuR3iH-YZb9sRw7TzoIAIqmplZY,78275
-returnn/torch/updater.py,sha256=skKeIJVNVJ9OAQonL61azdOZ3MhDF1JXBALPfWpQgWY,28239
+returnn/torch/updater.py,sha256=Vyh5w6ZFVc1hQvyyoWpeienQdlBVLZ2HYfjFZRQB3cQ,30035
 returnn/torch/data/__init__.py,sha256=6cLNEi8KoGI12PF6akN7mI_mtjlx-0hcQAfMYoExwik,132
 returnn/torch/data/extern_data.py,sha256=5al706ZaYtHWLp5VH2vS-rW69YXP3NHyOFRKY0WY714,7810
 returnn/torch/data/pipeline.py,sha256=HgIL0jQsPcgvh_SPC4wQ6BzclmrnpFja-UiboF_GPN4,29459
@@ -253,8 +253,8 @@ returnn/util/sig_proc.py,sha256=Tjz0VOAVyqu2qDCF5HZ1JjALjcFsHcNkcd96WgZeKfE,7265
 returnn/util/task_system.py,sha256=y4sMVXQ25Qd2z0rx03uOlXlkE-jbCYC1Sjfn-XlraVU,26003
 returnn/util/train_proc_manager.py,sha256=Pjht28k6uz6BNQ47uW6Gf880iyq5q4wx7P_K2tmoAM8,3266
 returnn/util/watch_memory.py,sha256=BR5P2kvBN6UI81cE0_1WAA6Hd1SByLbBaiDxvLhPOew,4213
-returnn-1.20250725.163756.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
-returnn-1.20250725.163756.dist-info/METADATA,sha256=wx2M7IlY7YIufSJIJvDiBsaXtYwynBc4CmeInkz4dF0,5215
-returnn-1.20250725.163756.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-returnn-1.20250725.163756.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
-returnn-1.20250725.163756.dist-info/RECORD,,
+returnn-1.20250810.211220.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
+returnn-1.20250810.211220.dist-info/METADATA,sha256=os7SMsZDIE8iA3rqu8Yv3il3m8BO4WeHIcMrHhEGMzs,5215
+returnn-1.20250810.211220.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+returnn-1.20250810.211220.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
+returnn-1.20250810.211220.dist-info/RECORD,,