returnn 1.20240713.225734__tar.gz → 1.20240713.233804__tar.gz

{returnn-1.20240713.225734 → returnn-1.20240713.233804}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240713.225734
+Version: 1.20240713.233804
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240713.233804/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240713.233804'
+long_version = '1.20240713.233804+git.78af734'

{returnn-1.20240713.225734 → returnn-1.20240713.233804}/returnn/frontend/_backend.py

@@ -620,6 +620,7 @@ class Backend(Generic[T]):
     def ctc_loss(
         *,
         logits: Tensor,
+        logits_normalized: bool = False,
         targets: Tensor,
         input_spatial_dim: Dim,
         targets_spatial_dim: Dim,

{returnn-1.20240713.225734 → returnn-1.20240713.233804}/returnn/frontend/label_smoothing.py

@@ -114,6 +114,8 @@ def label_smoothed_log_prob_gradient(
                        = softmax(z)[j] - target_prob[j]    # assuming (sum_i target_prob[i]) == 1
 
     """
+    if not isinstance(smoothing, Tensor) and smoothing == 0:
+        return log_prob  # no-op
     if not axis:
         assert log_prob.feature_dim
         axis = log_prob.feature_dim

{returnn-1.20240713.225734 → returnn-1.20240713.233804}/returnn/frontend/loss.py

@@ -57,6 +57,7 @@ def cross_entropy(
 def ctc_loss(
     *,
     logits: Tensor,
+    logits_normalized: bool = False,
     targets: Tensor,
     input_spatial_dim: Dim,
     targets_spatial_dim: Dim,
@@ -72,6 +73,7 @@ def ctc_loss(
     Output is of shape [B].
 
     :param logits: (before softmax). shape [B...,input_spatial,C]
+    :param logits_normalized: whether the logits are already normalized (e.g. via log-softmax)
     :param targets: sparse. shape [B...,targets_spatial] -> C
     :param input_spatial_dim: spatial dim of input logits
     :param targets_spatial_dim: spatial dim of targets
@@ -82,6 +84,7 @@ def ctc_loss(
     # noinspection PyProtectedMember
     return logits._raw_backend.ctc_loss(
         logits=logits,
+        logits_normalized=logits_normalized,
         targets=targets,
         input_spatial_dim=input_spatial_dim,
         targets_spatial_dim=targets_spatial_dim,

{returnn-1.20240713.225734 → returnn-1.20240713.233804}/returnn/tf/frontend_layers/_backend.py

@@ -457,6 +457,7 @@ class ReturnnLayersBackend(Backend[Layer]):
     def ctc_loss(
         *,
         logits: Tensor,
+        logits_normalized: bool = False,
         targets: Tensor,
         input_spatial_dim: Dim,
         targets_spatial_dim: Dim,
@@ -475,6 +476,7 @@ class ReturnnLayersBackend(Backend[Layer]):
             {
                 "class": "ctc_loss",
                 "logits": logits,
+                "logits_normalized": logits_normalized,
                 "targets": targets,
                 "blank_index": blank_index,
                 "max_approx": max_approx,

{returnn-1.20240713.225734 → returnn-1.20240713.233804}/returnn/tf/layers/basic.py

@@ -11454,10 +11454,11 @@ class CtcLossLayer(LayerBase):
     layer_class = "ctc_loss"
     recurrent = True  # order matters
 
-    def __init__(self, logits, targets, blank_index=-1, max_approx=False, **kwargs):
+    def __init__(self, logits, targets, logits_normalized=False, blank_index=-1, max_approx=False, **kwargs):
         """
         :param LayerBase logits: (before softmax). shape [B,T,D]
         :param LayerBase targets: sparse. shape [B,T]
+        :param bool logits_normalized: whether the logits are already normalized (e.g. via log-softmax)
         :param int blank_index: vocab index of the blank symbol
         :param bool max_approx: if True, use max instead of sum over alignments (max approx, Viterbi)
         """
@@ -11477,6 +11478,8 @@ class CtcLossLayer(LayerBase):
             logits=logits.output.copy_as_time_batch_major().placeholder,
             logits_time_major=True,
             logits_seq_lens=logits.output.get_sequence_lengths(),
+            logits_normalize=not logits_normalized,
+            grad_wrt_softmax_in=not logits_normalized,
             targets=targets.output.copy_as_batch_major().placeholder,
             targets_seq_lens=targets.output.get_sequence_lengths(),
             blank_index=blank_index,

{returnn-1.20240713.225734 → returnn-1.20240713.233804}/returnn/torch/frontend/_backend.py

@@ -632,6 +632,7 @@ class TorchBackend(Backend[torch.Tensor]):
     def ctc_loss(
         *,
         logits: Tensor,
+        logits_normalized: bool = False,
         targets: Tensor,
         input_spatial_dim: Dim,
         targets_spatial_dim: Dim,
@@ -651,7 +652,10 @@ class TorchBackend(Backend[torch.Tensor]):
         if len(batch_dims) != 1:
             logits_raw = torch.reshape(logits_raw, logits_raw.shape[:1] + (-1,) + logits_raw.shape[-1:])  # [T, B', C]
             input_lengths = torch.reshape(input_lengths, (-1,))  # [B']
-        log_probs = torch.nn.functional.log_softmax(logits_raw, dim=-1)
+        if logits_normalized:
+            log_probs = logits_raw
+        else:
+            log_probs = torch.nn.functional.log_softmax(logits_raw, dim=-1)
         # PyTorch expects the targets to be of shape (B, S) where S is the targets spatial dim.
         targets = targets.copy_transpose(batch_dims + [targets_spatial_dim])
         targets_raw = targets.raw_tensor  # [B..., S]

{returnn-1.20240713.225734 → returnn-1.20240713.233804}/returnn/torch/util/gradient_checkpoint.py

@@ -26,6 +26,7 @@ from dataclasses import dataclass, field
 import contextlib
 from weakref import ref, WeakSet
 import threading
+import atexit
 
 import torch
 from torch.utils.weak import WeakTensorKeyDictionary  # needs Torch >=2.0.0
@@ -178,6 +179,8 @@ class gradient_checkpoint_scope:
                 self.exit_saved_tensors_hooks_scope()
 
     def __del__(self):
+        if _python_exit:
+            return
         # Note, be very careful what we do in __del__ because it might be called in a different thread!
         # Note that the __del__ will likely be called very late,
         # as the reference to the _Graph is kept alive until we used it for backprop,
@@ -220,6 +223,8 @@ class gradient_checkpoint_scope:
         return x
 
     def _tensor_del_hook(self):
+        if _python_exit:
+            return
         # Some of the relevant tensors got deleted.
         # If we are in the right thread, maybe we can do the cleanup now.
         self._maybe_exit_saved_tensors_hooks_scope()
@@ -601,3 +606,12 @@ def _custom_saved_tensors_hooks_call_callbacks():
             assert not _custom_saved_tensors_hooks_tls_ctx.callbacks and not _custom_saved_tensors_hooks_tls_ctx.stack
     finally:
         _custom_saved_tensors_hooks_tls_ctx.in_callback = False
+
+
+def _python_exit_handler():
+    global _python_exit
+    _python_exit = True
+
+
+_python_exit = False
+atexit.register(_python_exit_handler)

{returnn-1.20240713.225734 → returnn-1.20240713.233804}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240713.225734
+Version: 1.20240713.233804
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240713.225734/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240713.225734'
-long_version = '1.20240713.225734+git.1f00da6'