returnn 1.20250226.104737__tar.gz → 1.20250226.115259__tar.gz

{returnn-1.20250226.104737/returnn.egg-info → returnn-1.20250226.115259}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250226.104737
+Version: 1.20250226.115259
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250226.115259/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20250226.115259'
+long_version = '1.20250226.115259+git.0d32534'

{returnn-1.20250226.104737 → returnn-1.20250226.115259}/returnn/torch/util/debug_inf_nan.py

@@ -39,6 +39,7 @@ from io import TextIOBase
 import traceback
 from types import FrameType
 import torch
+import tree
 
 # noinspection PyProtectedMember
 from torch.utils._python_dispatch import TorchDispatchMode
@@ -96,6 +97,7 @@ def debug_inf_nan(
 
 # For efficiency, and to be less spammy
 _TraceFuncNameBlacklist = {
+    "aten::empty.memory_format",
     "aten::zeros_like",
     "aten::ones_like",
     "aten::full",
@@ -113,6 +115,7 @@ _TraceFuncNameBlacklist = {
     "aten::split_with_sizes",
     "aten::slice.Tensor",
     "aten::select.int",
+    "aten::max_pool2d_with_indices",
 }
 
 
@@ -140,19 +143,20 @@ class _TraceOps(TorchDispatchMode):
         if self.report_every_op_call:
             print(f"--- op {func.name()}", file=self.file)
         out = func(*args, **kwargs)
-        if isinstance(out, torch.Tensor):
-            with no_python_dispatcher():
-                got_nan_inf_t = torch.stack([torch.isnan(out).any(), torch.isinf(out).any()]).cpu()
-                got_nan = got_nan_inf_t[0].item()
-                got_inf = got_nan_inf_t[1].item()
-                if got_nan or got_inf:
-                    s = "/".join([s_ for s_, b in [("nan", got_nan), ("inf", got_inf)] if b])
-                    print(f"--> {s} in {func}: {out}", file=self.file)
-                    traceback.print_list(
-                        _extract_stack_up_to(skip_top_num_frames=1, root_frame=self.root_frame), file=self.file
-                    )
-                    if self.stop_reporting_after_first_inf_nan:
-                        self.enabled = False
+        for out_ in tree.flatten(out):
+            if isinstance(out_, torch.Tensor):
+                with no_python_dispatcher():
+                    got_nan_inf_t = torch.stack([torch.isnan(out_).any(), torch.isinf(out_).any()]).cpu()
+                    got_nan = got_nan_inf_t[0].item()
+                    got_inf = got_nan_inf_t[1].item()
+                    if got_nan or got_inf:
+                        s = "/".join([s_ for s_, b in [("nan", got_nan), ("inf", got_inf)] if b])
+                        print(f"--> {s} in {func}: {out_}", file=self.file)
+                        traceback.print_list(
+                            _extract_stack_up_to(skip_top_num_frames=1, root_frame=self.root_frame), file=self.file
+                        )
+                        if self.stop_reporting_after_first_inf_nan:
+                            self.enabled = False
         return out
 
 

{returnn-1.20250226.104737 → returnn-1.20250226.115259/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250226.104737
+Version: 1.20250226.115259
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250226.104737 → returnn-1.20250226.115259}/tests/rf_utils.py

@@ -17,6 +17,7 @@ from returnn.tensor import Tensor, Dim, TensorDict
 from returnn.tensor.utils import tensor_dict_fill_random_numpy_
 import returnn.torch.frontend as rft
 from returnn.torch.data.tensor_utils import tensor_dict_numpy_to_torch_, tensor_dict_torch_to_numpy_
+from returnn.torch.util.debug_inf_nan import debug_inf_nan as torch_debug_inf_nan
 
 # noinspection PyProtectedMember
 from returnn.frontend._random_journal import RandomJournal
@@ -72,8 +73,17 @@ def run_model(
         out_pt_raw = out_pt.as_raw_tensor_dict(include_const_sizes=True)
 
     if not allow_inf_nan_in_output:
+        non_finite_outputs = {}
         for k, v in out_pt.data.items():
-            assert numpy.isfinite(v.raw_tensor).all(), f"output {k!r} has non-finite values: {v.raw_tensor}"
+            if not numpy.isfinite(v.raw_tensor).all():
+                non_finite_outputs[k] = v
+                print(f"ERROR: output {k!r} has non-finite values:\n{v.raw_tensor}")
+        if non_finite_outputs:
+            torch_debug_inf_nan(
+                lambda: (_run_model_torch(extern_data, get_model, forward_step), None)[-1],
+                stop_reporting_after_first_inf_nan=False,
+            )
+            raise Exception(f"Non-finite values in output: {non_finite_outputs}. See log above.")
 
     if not test_tensorflow:
         return out_pt

returnn-1.20250226.104737/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20250226.104737'
-long_version = '1.20250226.104737+git.5a134ff'