returnn 1.20240920.164009__tar.gz → 1.20240925.152757__tar.gz

{returnn-1.20240920.164009 → returnn-1.20240925.152757}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240920.164009
+Version: 1.20240925.152757
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240925.152757/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240925.152757'
+long_version = '1.20240925.152757+git.dc1a941'

{returnn-1.20240920.164009 → returnn-1.20240925.152757}/returnn/datasets/basic.py

@@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Optional, Any, Union, Type, Dict, Sequence, Li
 from returnn.log import log
 from returnn.engine.batch import Batch, BatchSetGenerator
 from returnn.datasets.util.vocabulary import Vocabulary
-from returnn.util.basic import try_run, NumbersDict, OptionalNotImplementedError
+from returnn.util.basic import get_fwd_compat_kwargs, try_run, NumbersDict, OptionalNotImplementedError
 from returnn.util import file_cache
 from returnn.tensor import TensorDict
 
@@ -1050,7 +1050,7 @@ class Dataset(object):
         :rtype: list[(int,NumbersDict,NumbersDict)]
         """
         if self.custom_chunking_func:
-            sentinel_kw = {"__fwd_compatible_random_arg_%i" % int(random() * 100): None}
+            sentinel_kw = get_fwd_compat_kwargs()
             for seq_idx, t_start, t_end in self.custom_chunking_func(
                 dataset=self, seq_idx_start=0, recurrent_net=recurrent_net, used_data_keys=used_data_keys, **sentinel_kw
             ):

{returnn-1.20240920.164009 → returnn-1.20240925.152757}/returnn/datasets/hdf.py

@@ -1322,7 +1322,6 @@ class SimpleHDFWriter:
             # Thus, we flatten all together, and calculate the flattened seq len.
             # (Ignore this if there is only a single time dimension.)
             flat_seq_len = int(numpy.prod([seq_len[axis][i] for axis in range(ndim_with_seq_len)]))
-            assert flat_seq_len > 0
             flat_shape = [flat_seq_len]
             if self.dim and not sparse:
                 flat_shape.append(self.dim)

{returnn-1.20240920.164009 → returnn-1.20240925.152757}/returnn/datasets/postprocessing.py

@@ -13,6 +13,7 @@ from returnn.datasets.util.strings import str_to_numpy_array
 from returnn.datasets.util.vocabulary import Vocabulary
 from returnn.tensor import Tensor, TensorDict
 from returnn.tensor.dim import Dim
+from returnn.util import basic as util
 from .basic import init_dataset
 from .cached2 import CachedDataset2
 
@@ -228,9 +229,7 @@ class PostprocessingDataset(CachedDataset2):
 
         data_iter = self._iterate_dataset()
         if self._map_seq_stream is not None:
-            data_iter = self._map_seq_stream(
-                data_iter, rng=self._rng, **{f"fwd_compatible_random_kwarg_{self._rng.randint(0, 1000)}": None}
-            )
+            data_iter = self._map_seq_stream(data_iter, rng=self._rng, **util.get_fwd_compat_kwargs())
             assert isinstance(
                 data_iter, Iterator
             ), f"map_seq_stream must produce an {Iterator.__name__}, but produced {type(data_iter).__name__}"
@@ -252,9 +251,7 @@ class PostprocessingDataset(CachedDataset2):
             tensor_dict.data["seq_tag"].raw_tensor = str_to_numpy_array(self._dataset.get_tag(seq_index))
 
             if self._map_seq is not None:
-                tensor_dict = self._map_seq(
-                    tensor_dict, rng=self._rng, **{f"fwd_compatible_random_kwarg_{self._rng.randint(0, 1000)}": None}
-                )
+                tensor_dict = self._map_seq(tensor_dict, rng=self._rng, **util.get_fwd_compat_kwargs())
                 assert isinstance(
                     tensor_dict, TensorDict
                 ), f"map_seq must produce a {TensorDict.__name__}, but produced {type(tensor_dict).__name__}"
@@ -307,16 +304,31 @@ class LaplaceOrdering(Callable[[Iterator[TensorDict]], Iterator[TensorDict]]):
         iterator = iter(iterator)
         is_down_phase = False
 
+        seq_buffer = list(islice(iterator, self.num_seqs_per_bin))
         while True:
-            seq_buffer = list(islice(iterator, self.num_seqs_per_bin))
             seq_buffer.sort(key=self._get_seq_len, reverse=is_down_phase)
-            yield from seq_buffer
 
-            is_down_phase = not is_down_phase
+            next_seq_buffer = []
+            has_ended = False
+
+            # Yield items to trainer while gradually pulling more data from PP function.
+            # This optimizes CPU load when multiple workers are used.
+            for item in seq_buffer:
+                yield item
+
+                try:
+                    if not has_ended:
+                        next_seq_buffer.append(next(iterator))
+                except StopIteration:
+                    has_ended = True
 
             if len(seq_buffer) < self.num_seqs_per_bin:
+                assert has_ended and not next_seq_buffer
                 break
 
+            is_down_phase = not is_down_phase
+            seq_buffer = next_seq_buffer
+
     def _get_seq_len(self, tdict: TensorDict) -> int:
         """
         :return: segment length of the segment in `tdict` as measured by `self.length_key` for comparison.

{returnn-1.20240920.164009 → returnn-1.20240925.152757}/returnn/tensor/_dim_extra.py

@@ -2294,7 +2294,15 @@ class _DimMixin:
         return cache, cache_key, cache.get(cache_key, None)
 
     def __sis_state__(self):
-        raise ValueError(f"{self}: currently not expected to be part of the Sisyphus state/hash")
+        # Note: The name (description) is deliberately not part of the state/hash,
+        # as this is quite arbitrary set by the user,
+        # and could easily change.
+        # In general, we keep the relevant state for Sisyphus quite minimal.
+        value = {"dim": self.dimension}
+        if self.kind is not None:
+            value["kind"] = self.kind.name
+        assert self.derived_from_op is None  # not handled yet for hashing...
+        return value
 
 
 def _make_constant_static_dim(value, kind=None):

{returnn-1.20240920.164009 → returnn-1.20240925.152757}/returnn/torch/engine.py

@@ -19,7 +19,6 @@ from torch.nn.parallel import DistributedDataParallel
 from torch.utils.data import DataLoader
 from torch import autocast
 from torch.cuda import amp
-from random import random
 import math
 
 import returnn
@@ -700,7 +699,7 @@ class Engine(EngineBase):
             if self._use_autocast
             else nullcontext()
         ), rf.set_default_device_ctx(self._device):
-            sentinel_kw = {"__fwd_compatible_random_arg_%i" % int(random() * 100): None}
+            sentinel_kw = util.get_fwd_compat_kwargs()
             if train_func:
                 self._train_step_func(model=self._orig_model, extern_data=extern_data, **sentinel_kw)
             else:
@@ -900,7 +899,7 @@ class Engine(EngineBase):
                 if self._use_autocast
                 else nullcontext()
             ), rf.set_default_device_ctx(self._device):
-                sentinel_kw = {"__fwd_compatible_random_arg_%i" % int(random() * 100): None}
+                sentinel_kw = util.get_fwd_compat_kwargs()
                 for hook in load_model_post_hooks:
                     hook(model=self._orig_model, **sentinel_kw)
 
@@ -930,7 +929,7 @@ class Engine(EngineBase):
 
         get_model_func = self.config.typed_value("get_model")
         assert get_model_func, "get_model not defined in config"
-        sentinel_kw = {"__fwd_compatible_random_arg_%i" % int(random() * 100): None}
+        sentinel_kw = util.get_fwd_compat_kwargs()
         model = get_model_func(epoch=epoch, step=step, **sentinel_kw)
         self._orig_model = model
         if isinstance(model, rf.Module):

{returnn-1.20240920.164009 → returnn-1.20240925.152757}/returnn/util/basic.py

@@ -4586,3 +4586,14 @@ def override_env_var(var_name: str, value: str):
             os.environ[var_name] = cur_val
         else:
             os.environ.pop(var_name)
+
+
+fwd_compatibility_rng = np.random.default_rng()
+
+
+def get_fwd_compat_kwargs() -> Dict[str, Any]:
+    """
+    Get randomly named kwargs for ensuring forwards compatibility in user code.
+    """
+    i = fwd_compatibility_rng.integers(0, 100)
+    return {f"__fwd_compat_random_arg_{i:03}": None}

{returnn-1.20240920.164009 → returnn-1.20240925.152757}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240920.164009
+Version: 1.20240925.152757
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20240920.164009 → returnn-1.20240925.152757}/tools/torch_export_to_onnx.py

@@ -35,7 +35,6 @@ import torch
 from typing import Callable, Optional, Dict, List
 import argparse
 import os
-from random import random
 
 import _setup_returnn_env  # noqa
 from returnn.config import Config
@@ -204,7 +203,7 @@ def main():
 
     get_model_func = config.typed_value("get_model")
     assert get_model_func, "get_model() isn't specified in the config passed as a parameter."
-    sentinel_kw = {"__fwd_compatible_random_arg_%i" % int(random() * 100): None}
+    sentinel_kw = util.get_fwd_compat_kwargs()
     model = get_model_func(epoch=epoch, step=step, **sentinel_kw)
 
     is_rf_module = isinstance(model, rf.Module)

returnn-1.20240920.164009/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240920.164009'
-long_version = '1.20240920.164009+git.450689b'