returnn 1.20250228.101938__tar.gz → 1.20250304.10039__tar.gz

{returnn-1.20250228.101938/returnn.egg-info → returnn-1.20250304.10039}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250228.101938
+Version: 1.20250304.10039
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250304.10039/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20250304.010039'
+long_version = '1.20250304.010039+git.3e53d74'

{returnn-1.20250228.101938 → returnn-1.20250304.10039}/returnn/tensor/_dim_extra.py

@@ -1271,9 +1271,9 @@ class _DimMixin:
             if self.batch:
                 x_dim = x_dim.get_for_batch_ctx(self.batch, self.control_flow_ctx)
             x_dim.complete_dyn_size(template_only=template_only, _backend=backend)
-            if x_dim.dyn_size_ext is None and not x_dim.dimension:
+            if x_dim.dyn_size_ext is None and x_dim.dimension is None:
                 return
-            y = _bin_op(y, x_dim.dimension or x_dim.dyn_size_ext)
+            y = _bin_op(y, x_dim.dimension if x_dim.dimension is not None else x_dim.dyn_size_ext)
             if not template_only and y.raw_tensor is not None:
                 y_max_value = _bin_op(y_max_value, x_dim.get_dim_value_tensor())
         assert y is not None, f"op {op}?"

{returnn-1.20250228.101938 → returnn-1.20250304.10039}/returnn/tf/layers/basic.py

@@ -6587,6 +6587,7 @@ class ConvLayer(_ConcatInputLayer):
         input_split_feature_dim=None,
         input_add_feature_dim=False,
         use_time_mask=False,
+        mask_value: float = 0.0,
     ):
         """
         :param Data input_data:
@@ -6600,6 +6601,7 @@ class ConvLayer(_ConcatInputLayer):
         :param bool input_add_feature_dim: will add a dim at the end and use input-feature-dim == 1,
           and use the original input feature-dim as a spatial dim.
         :param bool use_time_mask:
+        :param mask_value: when ``use_time_mask`` is used, what value to use for the mask
         :return: (transformed input, num batch dims). all batch dims are at the front
         :rtype: (Data, int)
         """
@@ -6697,7 +6699,7 @@ class ConvLayer(_ConcatInputLayer):
                     continue
                 axis = input_data.get_axis_from_description(dim)
                 mask = input_data.get_sequence_mask_broadcast(axis=axis)
-                x = tf_util.where_bc(mask, x, 0.0)
+                x = tf_util.where_bc(mask, x, mask_value)
 
             input_data.placeholder = x
 
@@ -7061,6 +7063,7 @@ class PoolLayer(_ConcatInputLayer):
             in_dim=in_dim,
             in_spatial_dims=in_spatial_dims,
             use_time_mask=use_time_mask,
+            mask_value={"MAX": float("-inf"), "AVG": 0}[mode],
         )
         # We want to prepare the input data such that the batch-dim(s) is the very first,
         # the feature-dim is the very last ("NHWC" format) or right after batch-dim ("NCHW"),

{returnn-1.20250228.101938 → returnn-1.20250304.10039}/returnn/torch/data/pipeline.py

@@ -337,7 +337,13 @@ class BucketOrderingIterDataPipe(torch.utils.data.IterDataPipe):
     """
 
     def __init__(
-        self, dataset: torch.utils.data.IterableDataset, *, buckets: Sequence[Tuple[int, int]], length_key: str
+        self,
+        dataset: torch.utils.data.IterableDataset,
+        *,
+        buckets: Sequence[Tuple[int, int]],
+        length_key: str,
+        random_bucket_prob: float = 0.0,
+        seed: Optional[int] = None,
     ):
         """
         :param dataset: dataset to apply bucket batching to
@@ -345,9 +351,17 @@ class BucketOrderingIterDataPipe(torch.utils.data.IterDataPipe):
             Segments longer than the largest size limit configured in the buckets are dropped. To avoid dropping
             any segments make sure your largest bucket allows segments larger than your longest training segment.
         :param length_key: data key to take as length measure
+        :param random_bucket_prob: Probability of putting a segment not into the best-fitting bucket, but into
+            a randomly chosen still-fitting bucket.
+            This increases seq length variation within the buckets at the cost of slighly more padding.
+        :param seed: random seed
         """
         self._dataset = dataset
         self._length_key = length_key
+        assert random_bucket_prob >= 0.0
+        self._random_bucket_prob = random_bucket_prob
+        self._rng = numpy.random.RandomState()
+        self._seed = seed % (2**32) if seed is not None else None
 
         assert buckets, "empty bucket batching configuration"
         if not all(size > 0 and max_seqs > 0 for size, max_seqs in buckets):
@@ -367,6 +381,12 @@ class BucketOrderingIterDataPipe(torch.utils.data.IterDataPipe):
             if bucket_idx >= len(self._max_seq_lens):
                 # seg is too long, drop it
                 continue
+            if (
+                self._random_bucket_prob > 0.0
+                and bucket_idx < len(self._max_seq_lens) - 1
+                and self._rng.rand() < self._random_bucket_prob
+            ):
+                bucket_idx = self._rng.randint(bucket_idx, len(self._max_bucket_sizes))
             buckets[bucket_idx].append(data_dict)
             if len(buckets[bucket_idx]) >= self._max_bucket_sizes[bucket_idx]:
                 yield buckets[bucket_idx]
@@ -383,6 +403,21 @@ class BucketOrderingIterDataPipe(torch.utils.data.IterDataPipe):
     def __getitem__(self, index):
         raise Exception(f"{self.__class__.__name__}.__getitem__ is not supported")
 
+    def set_seed(self, seed: int) -> BucketOrderingIterDataPipe:
+        """
+        Sets the seed for the next invocation of ``__iter__``, for compatibility with
+        ``torch.utils.data.graph_settings.apply_random_seed``.
+        """
+        self._seed = seed % (2**32)  # seed must be within [0, 2**32) for seeding RandomState
+        return self
+
+    def reset(self):
+        """resets the internal state of the data pipe"""
+        if self._seed is None:
+            self._seed = int(2**31 + torch.empty((), dtype=torch.int32).random_().item())
+        self._rng.seed(self._seed)
+        self._seed = None
+
 
 def get_batching_iterable_dataset_from_config(
     *, dataset: torch.utils.data.IterableDataset, config: Config, train: bool
@@ -497,7 +532,7 @@ class ShufflingDataPipe(torch.utils.data.IterDataPipe):
         self._buffer_size = buffer_size
         self._monotonic_data_keys = monotonic_data_keys
         self._rng = numpy.random.RandomState()
-        self._seed = seed
+        self._seed = seed % (2**32) if seed is not None else None
 
     def __iter__(self):
         # The implementation is very similar to the PostprocessingDataset's combinator LaplaceOrdering.
@@ -550,7 +585,7 @@ class ShufflingDataPipe(torch.utils.data.IterDataPipe):
         self._buffer.clear()
         self._next_buffer.clear()
         if self._seed is None:
-            self._seed = int(torch.empty((), dtype=torch.int32).random_().item())
+            self._seed = int(2**31 + torch.empty((), dtype=torch.int32).random_().item())
         self._rng.seed(self._seed)
         self._seed = None
 

{returnn-1.20250228.101938 → returnn-1.20250304.10039}/returnn/torch/frontend/_backend.py

@@ -1925,15 +1925,19 @@ class TorchBackend(Backend[torch.Tensor]):
                     stride_ = strides[i] if isinstance(strides, (list, tuple)) else strides
                 else:
                     stride_ = 1
-                # What is the logic here? You might be aware, in case without striding,
+                # What is the logic here? Also see https://github.com/rwth-i6/returnn/issues/1693.
+                # You might be aware, in case without striding,
                 # we simply have pad_left = (s.dimension - 1) // 2,
-                # or the total amount of padding is s.dimension - 1.
+                # or the total amount of padding is s.dimension - 1
+                # (s.dimension is the filter size).
                 # So we add the same amount of padding on both sides (or one less on the left side if odd).
                 # The output seq length in case of "valid" padding is ⌈(in_len - s.dimension + 1) / stride⌉.
                 # The output seq length in case of "same" padding with no striding (stride = 1)
-                # is simply the same as the input length.
+                # is simply the same as the input length (that's why it's called "same").
                 # What is the output seq length in case of "same" padding with striding?
-                # It should be ⌈in_len / stride⌉.
+                # It should be out_len = ⌈in_len / stride⌉ (it should be independent of s.dimension).
+                # We can rewrite out_len as:
+                # out_len = ⌊(in_len + stride - 1) / stride⌋ = (in_len + stride - 1 - (in_len - 1) % stride) / stride
                 # So, then we need to add a total amount of padding of s.dimension - 1.
                 # However, doing it the same way as without striding is incorrect.
                 # Why? Because then the first window might have more padding than the final window.
@@ -1941,8 +1945,6 @@ class TorchBackend(Backend[torch.Tensor]):
                 # or maybe one less on the first window if odd.
                 # How many frames do the windows cover?
                 # in_len_covered = out_len * stride + (s.dimension - stride)
-                # We can rewrite out_len as:
-                # out_len = ⌊(in_len + stride - 1) / stride⌋ = (in_len + stride - 1 - (in_len - 1) % stride) / stride
                 # So we have:
                 # in_len_covered = (in_len + stride - 1 - (in_len - 1) % stride) + (s.dimension - stride)
                 #                = in_len + s.dimension - 1 - (in_len - 1) % stride

{returnn-1.20250228.101938 → returnn-1.20250304.10039/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250228.101938
+Version: 1.20250304.10039
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250228.101938 → returnn-1.20250304.10039}/tests/rf_utils.py

@@ -143,7 +143,10 @@ def run_model(
     for k, v_pt in out_pt_raw.items():
         v_tf = out_tf_raw[k]
         print(f"  comparing {k!r} {_array_repr(v_pt)} PT vs TF")
-        numpy.testing.assert_allclose(v_pt, v_tf, atol=1e-5, rtol=1e-5, err_msg=f"output {k!r} differs")
+        if not numpy.allclose(v_pt, v_tf, atol=1e-5, rtol=1e-5):
+            print(f"  PT:\n{v_pt}")
+            print(f"  TF:\n{v_tf}")
+            raise Exception(f"output {k!r} differs")
     return out_pt
 
 

returnn-1.20250228.101938/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20250228.101938'
-long_version = '1.20250228.101938+git.c053cfd'