returnn 1.20250807.132148__py3-none-any.whl → 1.20250812.202710__py3-none-any.whl

returnn/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250807.132148
+Version: 1.20250812.202710
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn/_setup_info_generated.py

@@ -1,2 +1,2 @@
-version = '1.20250807.132148'
-long_version = '1.20250807.132148+git.8363a10'
+version = '1.20250812.202710'
+long_version = '1.20250812.202710+git.6c611de'

returnn/datasets/basic.py

@@ -564,14 +564,26 @@ class Dataset:
             reverse = -1 if seq_ordering_method == "sorted_reverse" else 1
             seq_lens = [reverse * get_seq_len(i) for i in range(num_seqs)]
             seq_index = numpy.argsort(seq_lens, kind="stable")
-        elif seq_ordering_method.startswith("random"):
-            tmp = seq_ordering_method.split(":")
+        elif seq_ordering_method == "random" or seq_ordering_method.startswith("random:"):
+            tmp = seq_ordering_method.split(":", 1)
             nth = int(tmp[1]) if len(tmp) > 1 else 1
             # Keep this deterministic! Use fixed seed.
             rnd_seed = self._get_random_seed_for_epoch(epoch=epoch, num_epochs_fixed=nth)
             random_generator = numpy.random.RandomState(rnd_seed)
             seq_index = random_generator.permutation(num_seqs)
-        elif seq_ordering_method.startswith("sort_bin_shuffle"):
+        elif seq_ordering_method == "random_sample" or seq_ordering_method.startswith("random_sample:"):
+            tmp = seq_ordering_method.split(":", 1)
+            nth = int(tmp[1]) if len(tmp) > 1 else 1
+            # Keep this deterministic! Use fixed seed.
+            rnd_seed = self._get_random_seed_for_epoch(epoch=epoch, num_epochs_fixed=nth)
+            random_generator = numpy.random.RandomState(rnd_seed)
+            seq_index = random_generator.randint(0, num_seqs, size=num_seqs)
+        elif (
+            seq_ordering_method == "sort_bin_shuffle"
+            or seq_ordering_method.startswith("sort_bin_shuffle:")
+            or seq_ordering_method == "sort_bin_shuffle_x2"
+            or seq_ordering_method.startswith("sort_bin_shuffle_x2:")
+        ):
             # Shuffle seqs, sort by length, and shuffle bins (then shuffle seqs within each bin if sort_bin_shuffle_x2).
             assert get_seq_len
             tmp = seq_ordering_method.split(":")[1:]
@@ -602,7 +614,7 @@ class Dataset:
                     random_generator.shuffle(part)  # Shuffle within the bin.
                 out_index.append(part)
             seq_index = numpy.concatenate(out_index)
-        elif seq_ordering_method.startswith("laplace"):
+        elif seq_ordering_method == "laplace" or seq_ordering_method.startswith("laplace:"):
             assert get_seq_len
             tmp = seq_ordering_method.split(":")[1:]
             if len(tmp) == 0:

returnn/torch/updater.py

@@ -113,6 +113,7 @@ class Updater:
         self._current_train_step = 0
         self._current_epoch = 1
         self._current_epoch_continuous = 0.0
+        self._num_consec_invalid_gradients_steps = 0
 
         self.learning_rate_function = self.config.typed_value("dynamic_learning_rate", None)
         if self.learning_rate_function is not None:
@@ -134,6 +135,9 @@ class Updater:
 
         self._grad_clip = self.config.float("gradient_clip", 0.0)
         self._grad_clip_global_norm = self.config.float("gradient_clip_global_norm", 0.0)
+        self._num_allowed_consec_invalid_gradient_steps = self.config.typed_value(
+            "num_allowed_consec_invalid_gradient_steps", None
+        )
         self._grad_noise = self.config.float("gradient_noise", 0.0)
 
         # Check other options we have in TF updater, which we might support here later as well,
@@ -208,6 +212,8 @@ class Updater:
             if the number of steps per epoch is known in advance.
         """
         self._current_train_step = global_train_step
+        if self._current_epoch != epoch:
+            self._num_consec_invalid_gradients_steps = 0
         self._current_epoch = epoch
         self._current_epoch_continuous = epoch_continuous
         self._update_effective_learning_rate()
@@ -224,12 +230,40 @@ class Updater:
         if self._grad_clip:
             torch.nn.utils.clip_grad_value_(self.network.parameters(), self._grad_clip)
         if self._grad_clip_global_norm:
-            torch.nn.utils.clip_grad_norm_(self.network.parameters(), self._grad_clip_global_norm)
+            norm = torch.nn.utils.clip_grad_norm_(self.network.parameters(), self._grad_clip_global_norm)
+        else:
+            norm = None
+
+        has_invalid_gradient = False
+        if self._num_allowed_consec_invalid_gradient_steps is not None:
+            if norm is None:
+                norm = torch.nn.utils.get_total_norm(self.network.parameters())
+            has_invalid_gradient = torch.isnan(norm) or torch.isinf(norm)
+            if has_invalid_gradient:
+                self._num_consec_invalid_gradients_steps += 1
+                if self._num_consec_invalid_gradients_steps > self._num_allowed_consec_invalid_gradient_steps:
+                    raise RuntimeError(
+                        f"Got {self._num_consec_invalid_gradients_steps} invalid gradients in succession, "
+                        f"abort training"
+                    )
+                else:
+                    invalid_grads_left = (
+                        self._num_allowed_consec_invalid_gradient_steps - self._num_consec_invalid_gradients_steps
+                    )
+                    print(
+                        f"Invalid gradient in step {self._current_train_step}, skipping. "
+                        f"{invalid_grads_left} subsequent broken steps left until training is aborted.",
+                        file=log.v2,
+                    )
+            else:
+                self._num_consec_invalid_gradients_steps = 0
 
         if grad_scaler is not None:
-            grad_scaler.step(self.optimizer)
+            if not has_invalid_gradient:
+                grad_scaler.step(self.optimizer)
+            # update needs to be called even if we discard the update due to an invalid gradient
             grad_scaler.update()
-        else:
+        elif not has_invalid_gradient:
             self.optimizer.step()
 
     def create_optimizer(self):

{returnn-1.20250807.132148.dist-info → returnn-1.20250812.202710.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250807.132148
+Version: 1.20250812.202710
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250807.132148.dist-info → returnn-1.20250812.202710.dist-info}/RECORD

@@ -1,9 +1,9 @@
-returnn/PKG-INFO,sha256=qjaW4g9jnc-m8PIwKo9OuWxjBD8K2fLK_uFz1uREsYU,5215
+returnn/PKG-INFO,sha256=rpwjyEcl0A4JGcT_ukgciH0uSWUHPI8REBpNnhAaPjs,5215
 returnn/__init__.py,sha256=biBtRsM0WZ406vShaeH-9WFoqJ8XwTbn6g0EeFJ7l8E,1012
 returnn/__main__.py,sha256=lHyZcu_0yc9f7Vf_Kfdy9PmeU0T76XVXnpalHi5WKro,31740
 returnn/__old_mod_loader__.py,sha256=nvsNY-xELdS_IPNkv66Q9Rmvg4dbGW0-EBRDcCmctos,7654
 returnn/__setup__.py,sha256=22kQn2fh11iPM0hLb2Fy5sLmoU1JGvmDxXRYuRgQkwU,4659
-returnn/_setup_info_generated.py,sha256=B6IzmcXbF_a6e6x6vv8tpUuytoQXbMyEpV9FDUP3NYM,77
+returnn/_setup_info_generated.py,sha256=GR2tWQ1k_EaJwxITWrntrJVKLZr3eU5Omp7yXewSGGQ,77
 returnn/config.py,sha256=3tmKhB6FnQZaNdtcYsiB61JnEY--iZ2qmJ4yq0b6tE0,29140
 returnn/forward_iface.py,sha256=A_OJiaXsX4MlXQRzST86ylyxSUZbC402PQL1REcqHjM,911
 returnn/learning_rate_control.py,sha256=ZvWryAn_tv9DhV8sh1LV3eE34Yltl3On3mYZAG4hR9s,34684
@@ -13,7 +13,7 @@ returnn/native_op.py,sha256=4_NnvfNxsM8GE_FsD6yOg6PZegqIdtJ3Sl1GdBWmFvg,244424
 returnn/pretrain.py,sha256=MHiXJZqkQFmDVyaYsGpd_Acv20wxl7Pr6s6qJzAT2FI,22648
 returnn/datasets/__init__.py,sha256=PvDlfDOaaopIeUIt0OSvHD2eHZkdkyE-sjMXf35EH5U,390
 returnn/datasets/audio.py,sha256=Gmj7a08dnvYh7Z-G1TNapz42L50AIcDE9JeIZaO1s1M,23334
-returnn/datasets/basic.py,sha256=IJhytVPiQZi7BD8-JVziKKT__PE528FwLQmbeiVQHzc,72303
+returnn/datasets/basic.py,sha256=_42fQztTZq7jNQrWdFBwulB1bNta17LOTyrD8XJ-7_E,73089
 returnn/datasets/bundle_file.py,sha256=KQNrS1MSf-4_idlK0c0KFwON-f5sEK0sWU15WpoMYpE,2380
 returnn/datasets/cached.py,sha256=RyefRjSDdp-HveK-2vLy2C6BIHcpqQ_lNvUKlIa4QAI,25412
 returnn/datasets/cached2.py,sha256=oJOq2lWRQpxm6kyUKW1w5qZBd4kdKEpwM7KY_QnXbq4,11922
@@ -208,7 +208,7 @@ returnn/torch/README.md,sha256=jzJ2FpOHW02vxN69yKaV97C9LI-hmvjBglKfdZXIDdc,85
 returnn/torch/__init__.py,sha256=MHEUyNHB20Vy89uKAqZoj6FxJKF1Gq3HW-i6ra1pNcI,24
 returnn/torch/distributed.py,sha256=_lyJR71HIoCHpMi5GztGM7YwrX54Am8zSkjnDkE1Lbk,7524
 returnn/torch/engine.py,sha256=JSsQZZiVs9TxRyFEJuR3iH-YZb9sRw7TzoIAIqmplZY,78275
-returnn/torch/updater.py,sha256=skKeIJVNVJ9OAQonL61azdOZ3MhDF1JXBALPfWpQgWY,28239
+returnn/torch/updater.py,sha256=Vyh5w6ZFVc1hQvyyoWpeienQdlBVLZ2HYfjFZRQB3cQ,30035
 returnn/torch/data/__init__.py,sha256=6cLNEi8KoGI12PF6akN7mI_mtjlx-0hcQAfMYoExwik,132
 returnn/torch/data/extern_data.py,sha256=5al706ZaYtHWLp5VH2vS-rW69YXP3NHyOFRKY0WY714,7810
 returnn/torch/data/pipeline.py,sha256=HgIL0jQsPcgvh_SPC4wQ6BzclmrnpFja-UiboF_GPN4,29459
@@ -253,8 +253,8 @@ returnn/util/sig_proc.py,sha256=Tjz0VOAVyqu2qDCF5HZ1JjALjcFsHcNkcd96WgZeKfE,7265
 returnn/util/task_system.py,sha256=y4sMVXQ25Qd2z0rx03uOlXlkE-jbCYC1Sjfn-XlraVU,26003
 returnn/util/train_proc_manager.py,sha256=Pjht28k6uz6BNQ47uW6Gf880iyq5q4wx7P_K2tmoAM8,3266
 returnn/util/watch_memory.py,sha256=BR5P2kvBN6UI81cE0_1WAA6Hd1SByLbBaiDxvLhPOew,4213
-returnn-1.20250807.132148.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
-returnn-1.20250807.132148.dist-info/METADATA,sha256=qjaW4g9jnc-m8PIwKo9OuWxjBD8K2fLK_uFz1uREsYU,5215
-returnn-1.20250807.132148.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-returnn-1.20250807.132148.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
-returnn-1.20250807.132148.dist-info/RECORD,,
+returnn-1.20250812.202710.dist-info/LICENSE,sha256=ywBD_U2aD4vpuoIgNAsjIGBYydl0tVKll3De0Z8s77c,11041
+returnn-1.20250812.202710.dist-info/METADATA,sha256=rpwjyEcl0A4JGcT_ukgciH0uSWUHPI8REBpNnhAaPjs,5215
+returnn-1.20250812.202710.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+returnn-1.20250812.202710.dist-info/top_level.txt,sha256=Lsn4WZc5Pbfk0-xDQOgnFCxOoqxL4CyeM3N1TFbJncw,8
+returnn-1.20250812.202710.dist-info/RECORD,,