returnn 1.20240327.165809__tar.gz → 1.20240522.175941__tar.gz

{returnn-1.20240327.165809/returnn.egg-info → returnn-1.20240522.175941}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240327.165809
+Version: 1.20240522.175941
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240522.175941/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240522.175941'
+long_version = '1.20240522.175941+git.36de1fe'

{returnn-1.20240327.165809 → returnn-1.20240522.175941}/returnn/datasets/audio.py

@@ -50,6 +50,7 @@ class OggZipDataset(CachedDataset2):
         segment_file=None,
         zip_audio_files_have_name_as_prefix=True,
         fixed_random_subset=None,
+        fixed_random_subset_seed=42,
         epoch_wise_filter=None,
         **kwargs,
     ):
@@ -68,7 +69,8 @@ class OggZipDataset(CachedDataset2):
           Value in [0,1] to specify the fraction, or integer >=1 which specifies number of seqs.
           If given, will use this random subset. This will be applied initially at loading time,
           i.e. not dependent on the epoch.
-          It will use an internally hardcoded fixed random seed, i.e. it's deterministic.
+          It uses the fixed fixed_random_subset_seed as seed, i.e. it's deterministic.
+        :param int fixed_random_subset_seed: Seed for drawing the fixed random subset, default 42
         :param dict|None epoch_wise_filter: see init_seq_order
         """
         import os
@@ -153,6 +155,7 @@ class OggZipDataset(CachedDataset2):
             self.num_outputs["data"] = [0, 2]
         self._data: Optional[List[Dict[str, Any]]] = None  # lazily loaded
         self._fixed_random_subset = fixed_random_subset
+        self._fixed_random_subset_seed = fixed_random_subset_seed
         if epoch_wise_filter is None:
             self.epoch_wise_filter = None  # type: Optional[EpochWiseFilter]
         elif isinstance(epoch_wise_filter, dict):
@@ -249,7 +252,7 @@ class OggZipDataset(CachedDataset2):
             if 0 < fixed_random_subset < 1:
                 fixed_random_subset = int(len(data) * fixed_random_subset)
             assert isinstance(fixed_random_subset, int) and fixed_random_subset > 0
-            rnd = numpy.random.RandomState(42)
+            rnd = numpy.random.RandomState(self._fixed_random_subset_seed)
             rnd.shuffle(data)
             data = data[:fixed_random_subset]
 

{returnn-1.20240327.165809 → returnn-1.20240522.175941}/returnn/datasets/sprint.py

@@ -402,9 +402,9 @@ class SprintDatasetBase(Dataset):
                 self.reduce_target_factor,
             )
         if "speaker_name" in targets:
-            targets["speaker_name"] = targets["speaker_name"].decode("utf8").strip()
+            targets["speaker_name"] = targets["speaker_name"].strip()
         if "orth" in targets:
-            targets["orth"] = targets["orth"].decode("utf8").strip()
+            targets["orth"] = targets["orth"].strip()
         if "orth" in targets and self.orth_post_process:
             targets["orth"] = self.orth_post_process(targets["orth"])
         if self.bpe:
@@ -815,7 +815,7 @@ class ExternSprintDataset(SprintDatasetBase):
 
         try:
             init_signal, (input_dim, output_dim, num_segments) = self._read_next_raw()
-            assert init_signal == b"init"
+            assert init_signal == "init"
             assert isinstance(input_dim, int) and isinstance(output_dim, int)
             # Ignore num_segments. It can be totally different than the real number of sequences.
             self.set_dimensions(input_dim, output_dim)
@@ -922,9 +922,7 @@ class ExternSprintDataset(SprintDatasetBase):
         :return: (data_type, args)
         :rtype: (str, object)
         """
-        # encoding is for converting Python2 strings to Python3.
-        # Cannot use utf8 because Numpy will also encode the data as strings and there we need it as bytes.
-        data_type, args = util.read_pickled_object(self.pipe_c2p[0], encoding="bytes")
+        data_type, args = util.read_pickled_object(self.pipe_c2p[0])
         return data_type, args
 
     def _join_child(self, wait=True, expected_exit_status=None):
@@ -974,20 +972,16 @@ class ExternSprintDataset(SprintDatasetBase):
                     if self.python_exit or not self.child_pid:
                         break
 
-                    if data_type == b"data":
+                    if data_type == "data":
                         seq_count += 1
                         segment_name, features, targets = args
-                        if segment_name is not None:
-                            segment_name = segment_name.decode("utf8")
                         assert isinstance(features, numpy.ndarray)
-                        if isinstance(targets, dict):
-                            targets = {key.decode("utf8"): value for (key, value) in targets.items()}
                         self.add_new_data(
                             numpy_copy_and_set_unused(features),
                             numpy_copy_and_set_unused(targets),
                             segment_name=segment_name,
                         )
-                    elif data_type == b"exit":
+                    elif data_type == "exit":
                         have_seen_the_whole = True
                         break
                     else:
@@ -1148,7 +1142,7 @@ class SprintCacheDataset(CachedDataset2):
             """
             res = self.sprint_cache.read(name, typ=self.type)
             if self.type == "align":
-                for (t, a, s, w) in res:
+                for t, a, s, w in res:
                     assert w == 1, "soft alignment not supported"
                 label_seq = numpy.array(
                     [self.allophone_labeling.get_label_idx(a, s) for (t, a, s, w) in res], dtype=self.dtype
@@ -1156,7 +1150,7 @@ class SprintCacheDataset(CachedDataset2):
                 assert label_seq.shape == (len(res),)
                 return label_seq
             elif self.type == "align_raw":
-                for (t, a, s, w) in res:
+                for t, a, s, w in res:
                     assert w == 1, "soft alignment not supported"
                 label_seq = numpy.array(
                     [self.allophone_labeling.state_tying_by_allo_state_idx[a] for (t, a, s, w) in res], dtype=self.dtype

{returnn-1.20240327.165809 → returnn-1.20240522.175941}/returnn/datasets/util/vocabulary.py

@@ -13,9 +13,9 @@ __all__ = [
     "Utf8ByteTargets",
 ]
 
-import sys
+from typing import Optional, Union, Type, List
 import typing
-
+import sys
 import numpy
 
 from returnn.log import log
@@ -107,13 +107,13 @@ class Vocabulary(object):
             parts.append("pad_label=%r" % self.id_to_label(self.pad_label_id))
         return "%s(%s)" % (self.__class__.__name__, ", ".join(parts))
 
-    def set_random_seed(self, seed):
+    def set_random_seed(self, seed: int):
         """
         This can be called for a new epoch or so.
         Usually it has no effect, as there is no randomness.
         However, some vocab class could introduce some sampling process.
 
-        :param int seed:
+        :param seed:
         """
         pass  # usually there is no randomness, so ignore
 
@@ -205,12 +205,16 @@ class Vocabulary(object):
 
         return init_vocab_var
 
-    def to_id(self, label, default=KeyError, allow_none=False):
+    def to_id(
+        self,
+        label: Union[str, int, None],
+        default: Union[str, Type[KeyError], None] = KeyError,
+        allow_none: bool = False,
+    ) -> Optional[int]:
         """
-        :param str|int|None label:
-        :param str|type[KeyError]|None default:
-        :param bool allow_none: whether label can be None. in this case, None is returned
-        :rtype: int|None
+        :param label:
+        :param default:
+        :param allow_none: whether label can be None. in this case, None is returned
         """
         if isinstance(label, str):
             return self.label_to_id(label, default=default)
@@ -226,21 +230,19 @@ class Vocabulary(object):
             return None
         raise TypeError("invalid label type %r" % type(label))
 
-    def label_to_id(self, label, default=KeyError):
+    def label_to_id(self, label: str, default: Union[int, Type[KeyError], None] = KeyError) -> Optional[int]:
         """
-        :param str label:
-        :param int|type[KeyError]|None default:
-        :rtype: int|None
+        :param label:
+        :param default:
         """
         if default is KeyError:
             return self._vocab[label]
         return self._vocab.get(label, default)
 
-    def id_to_label(self, idx, default=KeyError):
+    def id_to_label(self, idx: int, default: Union[str, Type[KeyError], None] = KeyError) -> Optional[str]:
         """
-        :param int idx:
-        :param str|KeyError|None default:
-        :rtype: str|None
+        :param idx:
+        :param default:
         """
         if self.is_id_valid(idx):
             return self._labels[idx]
@@ -248,43 +250,43 @@ class Vocabulary(object):
             raise KeyError("idx %i out of range" % idx)
         return default
 
-    def is_id_valid(self, idx):
+    def is_id_valid(self, idx: int) -> bool:
         """
-        :param int idx:
-        :rtype: bool
+        :param idx:
         """
         return 0 <= idx < len(self._labels)
 
     @property
-    def labels(self):
-        """
-        :rtype: list[str]
-        """
+    def labels(self) -> List[str]:
+        """list of labels"""
         return self._labels
 
-    def get_seq(self, sentence):
+    def get_seq(self, sentence: str) -> List[int]:
         """
-        :param str sentence: assumed to be seq of vocab entries separated by whitespace
-        :rtype: list[int]
+        :param sentence: assumed to be seq of vocab entries separated by whitespace
+        :return: seq of label indices
         """
         segments = sentence.split()
         return self.get_seq_indices(segments) + self.seq_postfix
 
-    def get_seq_indices(self, seq):
+    def get_seq_indices(self, seq: List[str]) -> List[int]:
         """
-        :param list[str] seq:
-        :rtype: list[int]
+        :param seq: seq of labels (entries in vocab)
+        :return: seq of label indices, returns unknown_label_id if unknown_label is set
         """
         if self.unknown_label is not None:
             return [self._vocab.get(k, self.unknown_label_id) for k in seq]
         return [self._vocab[k] for k in seq]
 
-    def get_seq_labels(self, seq):
+    def get_seq_labels(self, seq: Union[List[int], numpy.ndarray]) -> str:
         """
-        :param list[int]|numpy.ndarray seq: 1D sequence
-        :rtype: str
+        Inverse of :func:`get_seq`.
+
+        :param seq: 1D sequence of label indices
+        :return: serialized sequence string, such that ``get_seq(get_seq_labels(seq)) == seq``
         """
-        return " ".join(map(self._labels.__getitem__, seq))
+        labels = self.labels
+        return " ".join(map(labels.__getitem__, seq))
 
 
 class BytePairEncoding(Vocabulary):
@@ -421,10 +423,8 @@ class SentencePieces(Vocabulary):
         # Do not load labels/vocab here. This is not really needed.
 
     @property
-    def labels(self):
-        """
-        :rtype: list[str]
-        """
+    def labels(self) -> List[str]:
+        """list of labels"""
         if self._cache_key and self._cache_key in self._cache:
             self._vocab, self._labels = self._cache[self._cache_key]
             assert self.num_labels == len(self._vocab) == len(self._labels)
@@ -435,28 +435,25 @@ class SentencePieces(Vocabulary):
                 self._cache[self._cache_key] = (self._vocab, self._labels)
         return self._labels
 
-    def is_id_valid(self, idx):
+    def is_id_valid(self, idx: int) -> bool:
         """
-        :param int idx:
-        :rtype: bool
+        :param idx:
         """
         return not self.sp.IsUnused(idx)
 
-    def id_to_label(self, idx, default=KeyError):
+    def id_to_label(self, idx: int, default: Union[str, Type[KeyError], None] = KeyError) -> Optional[str]:
         """
-        :param int idx:
-        :param str|KeyError|None default:
-        :rtype: str|None
+        :param idx:
+        :param default:
         """
         if default is not KeyError and not self.is_id_valid(idx):
             return default
         return self.sp.IdToPiece(idx)
 
-    def label_to_id(self, label, default=KeyError):
+    def label_to_id(self, label: str, default: Union[int, Type[KeyError], None] = KeyError) -> Optional[int]:
         """
-        :param str label:
-        :param int|type[KeyError]|None default:
-        :rtype: int|None
+        :param label:
+        :param default:
         """
         res = self.sp.PieceToId(label)
         if res == self.unknown_label_id or res < 0 or res is None:
@@ -468,9 +465,9 @@ class SentencePieces(Vocabulary):
             return default
         return res
 
-    def set_random_seed(self, seed):
+    def set_random_seed(self, seed: int):
         """
-        :param int seed:
+        :param seed:
         """
         # Unfortunately, there is only a global seed,
         # and also, it will only be used for new threads
@@ -480,10 +477,9 @@ class SentencePieces(Vocabulary):
 
         spm.set_random_generator_seed(seed)
 
-    def get_seq(self, sentence):
+    def get_seq(self, sentence: str) -> List[int]:
         """
-        :param str sentence: assumed to be seq of vocab entries separated by whitespace
-        :rtype: list[int]
+        :param sentence: assumed to be seq of vocab entries separated by whitespace
         """
         return self.sp.encode(sentence, out_type=int)  # noqa
 

{returnn-1.20240327.165809 → returnn-1.20240522.175941}/returnn/frontend/_backend.py

@@ -475,6 +475,7 @@ class Backend(Generic[T]):
         axes: Sequence[Dim],
         padding: Sequence[Tuple[Union[Dim, int], Union[Dim, int]]],
         out_dims: Sequence[Dim],
+        handle_dynamic_dims: bool,
         mode: str = "constant",
         value: Optional[Union[rf.RawTensorTypes, Tensor]] = None,
     ) -> Tensor:
@@ -483,6 +484,7 @@ class Backend(Generic[T]):
         :param axes:
         :param padding:
         :param out_dims:
+        :param handle_dynamic_dims:
         :param mode:
         :param value:
         :return: padded tensor

{returnn-1.20240327.165809 → returnn-1.20240522.175941}/returnn/frontend/array_.py

@@ -4,6 +4,7 @@ Array (Tensor) functions
 
 from __future__ import annotations
 from typing import Optional, Union, Type, TypeVar, Sequence, Tuple
+import logging
 import numpy
 from returnn.tensor import Tensor, Dim
 import returnn.frontend as rf
@@ -385,6 +386,7 @@ def pad(
     out_dims: Optional[Sequence[Dim]] = None,
     mode: str = "constant",
     value: Optional[Union[rf.RawTensorTypes, Tensor]] = None,
+    handle_dynamic_dims: Optional[bool] = None,
 ) -> Tuple[Tensor, Sequence[Dim]]:
     """
     Pad values left/right in the specified axes.
@@ -392,9 +394,13 @@ def pad(
     :param source:
     :param axes: which axes to add padding to
     :param padding: list of (left, right) padding for each axis
-    :param out_dims: (optional) predefined out dim tags, otherwise will automatically create
+    :param out_dims: (optional) predefined out dims for each padded dim in axes. will automatically create if not given
     :param mode: 'constant', 'reflect', 'replicate' or 'circular'
     :param value: (optional) value to pad with in "constant" mode
+    :param handle_dynamic_dims: True: when doing right padding on a dynamic dim, value will be added after the seq end,
+        not at the end of the dimension. False: value will be added at the end of the dimension.
+        By default, in behavior version >=21, this is True, in older versions, this is False.
+    :return: padded tensor, out_dims. out dims are for each dim in axes
     """
     assert len(axes) == len(padding)
     if not out_dims:
@@ -405,13 +411,62 @@ def pad(
                 assert not right.need_masking(), f"padding {padding} does not support dynamic right padding"
             # Note that even dynamic middle dims is not exactly correct...
         out_dims = [left + middle + right for middle, (left, right) in zip(axes, padding)]
+    if handle_dynamic_dims is None:
+        handle_dynamic_dims = _pad_handle_dynamic_dims_default(axes, padding, mode=mode)
     # noinspection PyProtectedMember
     return (
-        source._raw_backend.pad(source, axes=axes, padding=padding, out_dims=out_dims, mode=mode, value=value),
+        source._raw_backend.pad(
+            source,
+            axes=axes,
+            padding=padding,
+            out_dims=out_dims,
+            handle_dynamic_dims=handle_dynamic_dims,
+            mode=mode,
+            value=value,
+        ),
         out_dims,
     )
 
 
+_pad_handle_dynamic_dims_shown_warning = False
+
+
+def _pad_handle_dynamic_dims_default(
+    pad_axes: Sequence[Dim], padding: Sequence[Tuple[Union[Dim, int], Union[Dim, int]]], *, mode: str
+) -> bool:
+    """
+    :param pad_axes: list of axes to pad
+    :param padding: list of (left, right) padding for each axis
+    :param mode: 'constant', 'reflect', 'replicate' or 'circular'
+    :return: True if dynamic dims should be handled as specified in the default behavior
+    """
+    from returnn.util.basic import BehaviorVersion
+
+    if BehaviorVersion.get() >= 21:
+        return True
+
+    # Check whether not handling the dynamic dims is safe. Print a warning if not safe.
+    global _pad_handle_dynamic_dims_shown_warning
+    if not _pad_handle_dynamic_dims_shown_warning:
+        for middle, (left, right) in zip(pad_axes, padding):
+            middle: Dim
+            if not middle.need_masking() and (isinstance(left, int) or not left.need_masking()):
+                continue
+            if mode != "circular" and isinstance(right, int) and right == 0:
+                continue
+
+            logging.getLogger("returnn.frontend").warning(
+                f"rf.pad applied on dynamic dim {middle} but handle_dynamic_dims=False used by default"
+                f" due to behavior version {BehaviorVersion.get()} < 21."
+                " Set handle_dynamic_dims explicitly to avoid the warning,"
+                " or switch to a new behavior version >= 21."
+                " (This warning is only printed once.)"
+            )
+            _pad_handle_dynamic_dims_shown_warning = True
+            break
+    return False
+
+
 def cum_concat_step(
     source: Tensor, *, prev_accum: Tensor, axis: Dim, out_spatial_dim: Optional[Dim] = None
 ) -> Tuple[Tensor, Dim]:

{returnn-1.20240327.165809 → returnn-1.20240522.175941}/returnn/tf/frontend_layers/_backend.py

@@ -355,6 +355,7 @@ class ReturnnLayersBackend(Backend[Layer]):
         axes: Sequence[Dim],
         padding: Sequence[Tuple[Union[Dim, int], Union[Dim, int]]],
         out_dims: Sequence[Dim],
+        handle_dynamic_dims: bool,
         mode: str = "constant",
         value: Union[rf.RawTensorTypes, Tensor] = None,
     ) -> Tensor:
@@ -367,6 +368,7 @@ class ReturnnLayersBackend(Backend[Layer]):
                 "axes": axes,
                 "padding": padding,
                 "out_dims": out_dims,
+                "handle_dynamic_dims": handle_dynamic_dims,
                 "mode": mode,
                 "value": value,
             },

{returnn-1.20240327.165809 → returnn-1.20240522.175941}/returnn/tf/layers/basic.py

@@ -1981,13 +1981,15 @@ class ScatterNdLayer(_ConcatInputLayer):
         # Now we need to implement a similar logic as `returnn.tf.util.basic.nd_indices`, but more generic.
         idxs = [
             (
-                tf.reshape(
-                    tf.range(pos_shape[i], dtype=pos_v.dtype), [1] * i + [pos_shape[i]] + [1] * (pos_ndim - i - 1)
+                (
+                    tf.reshape(
+                        tf.range(pos_shape[i], dtype=pos_v.dtype), [1] * i + [pos_shape[i]] + [1] * (pos_ndim - i - 1)
+                    )
+                    + tf.zeros_like(pos_v)
                 )
-                + tf.zeros_like(pos_v)
+                if i != replace_common_axis
+                else pos_v
             )
-            if i != replace_common_axis
-            else pos_v
             for i in range(pos_ndim)
         ]
         nd_idxs = tf.stack(idxs, axis=-1)
@@ -4173,13 +4175,28 @@ class PadLayer(_ConcatInputLayer):
 
     layer_class = "pad"
 
-    def __init__(self, axes, padding, out_dims=None, value=0, mode="constant", **kwargs):
+    def __init__(
+        self,
+        *,
+        axes: Union[Dim, str, Sequence[Union[Dim, str]]],
+        padding: Union[int, Tuple[int, int], Sequence[Tuple[int, int]]],
+        out_dims: Optional[Union[Dim, Sequence[Dim]]] = None,
+        handle_dynamic_dims: Optional[bool] = None,
+        value: Union[int, float] = 0,
+        mode: str = "constant",
+        **kwargs,
+    ):
         """
-        :param Dim|str|list[Dim|str] axes: e.g. "F" etc. see :func:`Data.get_axes_from_description`.
-        :param list[(int,int)]|(int,int)|int padding: how much to pad left/right in each axis
-        :param Dim|list[Dim]|None out_dims:
-        :param int|float value: what constant value to pad, with mode=="constant"
-        :param str mode: "constant", "reflect", "symmetric" and "replication"
+        :param axes: e.g. "F" etc. see :func:`Data.get_axes_from_description`.
+        :param padding: how much to pad left/right in each axis
+        :param out_dims:
+        :param handle_dynamic_dims: True: when doing right padding on a dynamic dim,
+            value will be added after the seq end,
+            not at the end of the dimension.
+            False: value will be added at the end of the dimension.
+            By default, in behavior version >=21, this is True, in older versions, this is False.
+        :param value: what constant value to pad, with mode=="constant"
+        :param mode: "constant", "reflect", "symmetric" and "replication"
         """
         out_dims  # noqa  # handled in get_out_data_from_opts
         super(PadLayer, self).__init__(**kwargs)
@@ -4190,15 +4207,47 @@ class PadLayer(_ConcatInputLayer):
         paddings = [(0, 0)] * len(range(self.input_data.batch_ndim))
         for i, a in enumerate(axes):
             paddings[a] = padding[i]
-        mode = mode.upper()
+        mode = mode.lower()
+        if handle_dynamic_dims is None:
+            handle_dynamic_dims = self._handle_dynamic_dims_default(
+                pad_axes=[self.input_data.dims[axis] for axis in axes_],
+                padding=padding,
+                mode=mode,
+            )
         if all(sum(p) == 0 for p in padding):
             self.output.placeholder = self.input_data.placeholder
-        elif mode == "REPLICATION":
+        elif mode == "replication":
             self.output.placeholder = tf_util.pad_replicate(self.input_data.placeholder, axes, padding)
         else:
             self.output.placeholder = tf.pad(
                 self.input_data.placeholder, paddings=paddings, mode=mode, constant_values=value
             )
+        if all(right == 0 for left, right in padding) and mode != "circular":
+            pass  # no masking needed
+        else:
+            import returnn.frontend as rf
+
+            for middle_axis, (left, right) in zip(axes, padding):
+                out_dim: Dim = self.output.dims[middle_axis]
+                middle = self.input_data.dims[middle_axis]
+                if handle_dynamic_dims and middle.need_masking() or (isinstance(left, Dim) and left.need_masking()):
+                    if mode != "constant":
+                        raise NotImplementedError(
+                            f"pad: mode {mode} not implemented with dynamic dims and handle_dynamic_dims=True"
+                        )
+                    if isinstance(right, Dim) or right > 0:
+                        mask = rf.compare_bc(
+                            rf.range_over_dim(out_dim),
+                            "<",
+                            (left + middle)
+                            .get_for_batch_ctx(self.output.batch, self.output.control_flow_ctx)
+                            .dyn_size_ext,
+                        )
+                        self.output.raw_tensor = tf_util.where_bc(
+                            mask.copy_compatible_to(self.output, check_sparse=False, check_dtype=False).raw_tensor,
+                            self.output.raw_tensor,
+                            tf.convert_to_tensor(value, dtype=self.output.dtype),
+                        )
 
     @classmethod
     def _transform_padding(cls, padding, axes):
@@ -4218,6 +4267,45 @@ class PadLayer(_ConcatInputLayer):
             padding = [(padding, padding)] * len(axes)
         return padding
 
+    _handle_dynamic_dims_shown_warning = False
+
+    @classmethod
+    def _handle_dynamic_dims_default(
+        cls, pad_axes: Sequence[Dim], padding: Sequence[Tuple[Union[Dim, int], Union[Dim, int]]], *, mode: str
+    ) -> bool:
+        """
+        :param pad_axes: list of axes to pad
+        :param padding: list of (left, right) padding for each axis
+        :param mode: 'constant', 'reflect', 'replicate' or 'circular'
+        :return: True if dynamic dims should be handled as specified in the default behavior
+        """
+        from returnn.util.basic import BehaviorVersion
+
+        if BehaviorVersion.get() >= 21:
+            return True
+
+        # Check whether not handling the dynamic dims is safe. Print a warning if not safe.
+        if not cls._handle_dynamic_dims_shown_warning:
+            import logging
+
+            for middle, (left, right) in zip(pad_axes, padding):
+                middle: Dim
+                if not middle.need_masking() and (isinstance(left, int) or not left.need_masking()):
+                    continue
+                if mode != "circular" and isinstance(right, int) and right == 0:
+                    continue
+
+                logging.getLogger("returnn.tf").warning(
+                    f"PadLayer applied on dynamic dim {middle} but handle_dynamic_dims=False used by default"
+                    f" due to behavior version {BehaviorVersion.get()} < 21."
+                    " Set handle_dynamic_dims explicitly to avoid the warning,"
+                    " or switch to a new behavior version >= 21."
+                    " (This warning is only printed once.)"
+                )
+                cls._handle_dynamic_dims_shown_warning = True
+                break
+        return False
+
     @classmethod
     def get_out_data_from_opts(cls, name, sources, axes, padding, out_dims=None, **kwargs):
         """
@@ -4929,14 +5017,16 @@ class SplitDimsLayer(_ConcatInputLayer):
                 rem_dim = None
             if not resolved_dims:
                 resolved_dims = tuple(
-                    Dim(
-                        kind=axis_dim_tag.kind if not axis_dim_tag.is_batch_dim() else Dim.Types.Spatial,
-                        description="%s_split_dims%i" % (name, i),
-                        dimension=shape_dim,
-                        auto_generated=True,
+                    (
+                        Dim(
+                            kind=axis_dim_tag.kind if not axis_dim_tag.is_batch_dim() else Dim.Types.Spatial,
+                            description="%s_split_dims%i" % (name, i),
+                            dimension=shape_dim,
+                            auto_generated=True,
+                        )
+                        if rem_dim is None or i != rem_dim_idx
+                        else rem_dim
                     )
-                    if rem_dim is None or i != rem_dim_idx
-                    else rem_dim
                     for i, shape_dim in enumerate(resolved_shape_dims)
                 )
         out_batch = data.batch