returnn 1.20240712.164051__tar.gz → 1.20240713.231521__tar.gz

{returnn-1.20240712.164051 → returnn-1.20240713.231521}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240712.164051
+Version: 1.20240713.231521
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240713.231521/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240713.231521'
+long_version = '1.20240713.231521+git.90f56c9'

{returnn-1.20240712.164051 → returnn-1.20240713.231521}/returnn/frontend/_backend.py

@@ -620,6 +620,7 @@ class Backend(Generic[T]):
     def ctc_loss(
         *,
         logits: Tensor,
+        logits_normalized: bool = False,
         targets: Tensor,
         input_spatial_dim: Dim,
         targets_spatial_dim: Dim,

{returnn-1.20240712.164051 → returnn-1.20240713.231521}/returnn/frontend/label_smoothing.py

@@ -63,46 +63,73 @@ def label_smoothed_log_prob_gradient(
     """
     :param log_prob: shape [...,D] (not necessarily the same as loss)
     :param smoothing: smoothing factor, for :func:`label_smoothing`
-    :param axis: label axis. uses feature_dim by default
+    :param axis: label axis in ``log_prob`` (D). uses feature_dim by default
     :param exclude_labels: list of labels to exclude from smoothing (e.g. blank)
+    :return: ``log_prob``, but the gradient is smoothed
 
-    Assume some cross-entropy-like loss:
+    Assume some cross-entropy-like loss::
 
-      loss = - sum_i target_prob[i] * log_prob[i] .
+        loss = - sum_i target_prob[i] * log_prob[i] .
 
     The sum is over the label indices i (corresponding to the ``axis`` argument).
-    Then the gradient of loss w.r.t. log_prob[i] is:
+    Then the gradient of loss w.r.t. log_prob[i] is::
 
-      grad_logprob[i] loss = -target_prob[i] .
+        grad_logprob[i] loss = -target_prob[i] .
 
-    We assume that the negative gradient is a probability distribution, and apply :func:`label_smoothing` on it.
+    We assume that the negative gradient is a probability distribution
+    (potentially scaled by some factor, e.g. when you scale the loss by some factor)
+    and apply :func:`label_smoothing` on it.
     More specifically, we apply the same scale and shift as in the :func:`label_smoothing` function
-    via :func:`scaled_gradient`.
+    via :func:`scaled_gradient_ext`.
 
-    Just as a side remark: assume
+    Note, this is also the case for CTC or RNNT loss,
+    that the negative gradient of the loss w.r.t. the log-probabilities
+    is a probability distribution.
 
-      log_prob = log_softmax(z) .
+    Common usage example::
 
-    The gradient of log_softmax is:
+        # E.g. there was some log_softmax, or anything to get log probs.
+        log_probs = model.get_log_probs(...)
 
-      grad_z[j] log_prob[i] = delta(i==j) - softmax(z)[j] .
+        # Now apply label smoothing on the log prob gradients.
+        log_probs = rf.label_smoothed_log_prob_gradient(log_probs, 0.1)
 
-    Then the gradient w.r.t. z[j] is:
+        # E.g. CE, CTC, or similar, any kind of NLL should work.
+        loss = loss_func(log_probs, targets)
+        loss.sum().backward()
 
-      grad_z[j] loss = sum_i (grad_logprob[i] loss) (grad_z[j] logprob[i])
-                     = sum_i -target_prob[i] delta(i==j) + target_prob[i] softmax(z)[j]
-                     = -target_prob[j] + (sum_i target_prob[i]) softmax(z)[j]
-                     = softmax(z)[j] - target_prob[j]    # assuming (sum_i target_prob[i]) == 1
+    Just as a side remark: assume::
+
+        log_prob = log_softmax(z) .
+
+    The gradient of log_softmax is::
+
+        grad_z[j] log_prob[i] = delta(i==j) - softmax(z)[j] .
+
+    Then the gradient w.r.t. z[j] is::
+
+        grad_z[j] loss = sum_i (grad_logprob[i] loss) (grad_z[j] logprob[i])
+                       = sum_i -target_prob[i] delta(i==j) + target_prob[i] softmax(z)[j]
+                       = -target_prob[j] + (sum_i target_prob[i]) softmax(z)[j]
+                       = softmax(z)[j] - target_prob[j]    # assuming (sum_i target_prob[i]) == 1
 
     """
+    if not isinstance(smoothing, Tensor) and smoothing == 0:
+        return log_prob  # no-op
     if not axis:
         assert log_prob.feature_dim
         axis = log_prob.feature_dim
     # See formula above for label_smoothing.
     dim = axis.dimension
-    floor_prob = smoothing / (dim - 1)
-    factor = 1.0 - dim * floor_prob
-    if exclude_labels:
+    if not exclude_labels:
+        # See formula in code comments in label_smoothing above.
+        floor_prob = smoothing / (dim - 1)
+        factor = 1.0 - dim * floor_prob
+    else:  # have exclude_labels
+        # Sum of prob over included labels does not change with this factor.
+        # For prob[i] == 1, we still have (1 - smoothing) for the smoothed prob.
+        floor_prob = smoothing / (dim - len(exclude_labels) - 1)
+        factor = 1.0 - (dim - len(exclude_labels)) * floor_prob
         indices = rf.range_over_dim(axis)
         mask = True
         for label in exclude_labels:

{returnn-1.20240712.164051 → returnn-1.20240713.231521}/returnn/frontend/loss.py

@@ -57,6 +57,7 @@ def cross_entropy(
 def ctc_loss(
     *,
     logits: Tensor,
+    logits_normalized: bool = False,
     targets: Tensor,
     input_spatial_dim: Dim,
     targets_spatial_dim: Dim,
@@ -72,6 +73,7 @@ def ctc_loss(
     Output is of shape [B].
 
     :param logits: (before softmax). shape [B...,input_spatial,C]
+    :param logits_normalized: whether the logits are already normalized (e.g. via log-softmax)
     :param targets: sparse. shape [B...,targets_spatial] -> C
     :param input_spatial_dim: spatial dim of input logits
     :param targets_spatial_dim: spatial dim of targets
@@ -82,6 +84,7 @@ def ctc_loss(
     # noinspection PyProtectedMember
     return logits._raw_backend.ctc_loss(
         logits=logits,
+        logits_normalized=logits_normalized,
         targets=targets,
         input_spatial_dim=input_spatial_dim,
         targets_spatial_dim=targets_spatial_dim,

{returnn-1.20240712.164051 → returnn-1.20240713.231521}/returnn/tf/frontend_layers/_backend.py

@@ -457,6 +457,7 @@ class ReturnnLayersBackend(Backend[Layer]):
     def ctc_loss(
         *,
         logits: Tensor,
+        logits_normalized: bool = False,
         targets: Tensor,
         input_spatial_dim: Dim,
         targets_spatial_dim: Dim,
@@ -475,6 +476,7 @@ class ReturnnLayersBackend(Backend[Layer]):
             {
                 "class": "ctc_loss",
                 "logits": logits,
+                "logits_normalized": logits_normalized,
                 "targets": targets,
                 "blank_index": blank_index,
                 "max_approx": max_approx,

{returnn-1.20240712.164051 → returnn-1.20240713.231521}/returnn/tf/layers/basic.py

@@ -11454,10 +11454,11 @@ class CtcLossLayer(LayerBase):
     layer_class = "ctc_loss"
     recurrent = True  # order matters
 
-    def __init__(self, logits, targets, blank_index=-1, max_approx=False, **kwargs):
+    def __init__(self, logits, targets, logits_normalized=False, blank_index=-1, max_approx=False, **kwargs):
         """
         :param LayerBase logits: (before softmax). shape [B,T,D]
         :param LayerBase targets: sparse. shape [B,T]
+        :param bool logits_normalized: whether the logits are already normalized (e.g. via log-softmax)
         :param int blank_index: vocab index of the blank symbol
         :param bool max_approx: if True, use max instead of sum over alignments (max approx, Viterbi)
         """
@@ -11477,6 +11478,8 @@ class CtcLossLayer(LayerBase):
             logits=logits.output.copy_as_time_batch_major().placeholder,
             logits_time_major=True,
             logits_seq_lens=logits.output.get_sequence_lengths(),
+            logits_normalize=not logits_normalized,
+            grad_wrt_softmax_in=not logits_normalized,
             targets=targets.output.copy_as_batch_major().placeholder,
             targets_seq_lens=targets.output.get_sequence_lengths(),
             blank_index=blank_index,

{returnn-1.20240712.164051 → returnn-1.20240713.231521}/returnn/torch/frontend/_backend.py

@@ -632,6 +632,7 @@ class TorchBackend(Backend[torch.Tensor]):
     def ctc_loss(
         *,
         logits: Tensor,
+        logits_normalized: bool = False,
         targets: Tensor,
         input_spatial_dim: Dim,
         targets_spatial_dim: Dim,
@@ -651,7 +652,10 @@ class TorchBackend(Backend[torch.Tensor]):
         if len(batch_dims) != 1:
             logits_raw = torch.reshape(logits_raw, logits_raw.shape[:1] + (-1,) + logits_raw.shape[-1:])  # [T, B', C]
             input_lengths = torch.reshape(input_lengths, (-1,))  # [B']
-        log_probs = torch.nn.functional.log_softmax(logits_raw, dim=-1)
+        if logits_normalized:
+            log_probs = logits_raw
+        else:
+            log_probs = torch.nn.functional.log_softmax(logits_raw, dim=-1)
         # PyTorch expects the targets to be of shape (B, S) where S is the targets spatial dim.
         targets = targets.copy_transpose(batch_dims + [targets_spatial_dim])
         targets_raw = targets.raw_tensor  # [B..., S]

{returnn-1.20240712.164051 → returnn-1.20240713.231521}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240712.164051
+Version: 1.20240713.231521
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240712.164051/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240712.164051'
-long_version = '1.20240712.164051+git.d5b954b'