returnn 1.20241126.105536__tar.gz → 1.20241127.143843__tar.gz

{returnn-1.20241126.105536 → returnn-1.20241127.143843}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241126.105536
+Version: 1.20241127.143843
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241127.143843/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20241127.143843'
+long_version = '1.20241127.143843+git.5780574'

{returnn-1.20241126.105536 → returnn-1.20241127.143843}/returnn/datasets/postprocessing.py

@@ -127,6 +127,7 @@ class PostprocessingDataset(CachedDataset2):
         self._map_seq_stream = map_seq_stream
         self._map_outputs = map_outputs
         self._rng = RandomState(self._get_random_seed_for_epoch(0))
+        self._seq_list_for_validation: Optional[List[str]] = None
 
         self._dataset = init_dataset(self._dataset_def, parent_dataset=self)
         if self._map_seq_stream is None:
@@ -169,6 +170,12 @@ class PostprocessingDataset(CachedDataset2):
         """
         super().init_seq_order(epoch=epoch, seq_list=seq_list, seq_order=seq_order)
 
+        if self._map_seq_stream is not None:
+            if seq_list is not None:
+                raise ValueError("map_seq_stream is set, cannot specify custom seq_list")
+            if seq_order is not None:
+                raise ValueError("map_seq_stream is set, cannot specify custom seq_order")
+
         if epoch is None and seq_list is None and seq_order is None:
             self._num_seqs = 0
             return True
@@ -177,6 +184,7 @@ class PostprocessingDataset(CachedDataset2):
         assert self._dataset is not None
         self._dataset.init_seq_order(epoch=epoch, seq_list=seq_list, seq_order=seq_order)
         self._data_iter = enumerate(self._build_mapping_iter())
+        self._seq_list_for_validation = seq_list
         if self._map_seq_stream is None:
             # If we don't have an iterable mapper we know the number of segments exactly
             # equals the number of segments in the wrapped dataset
@@ -241,7 +249,7 @@ class PostprocessingDataset(CachedDataset2):
 
         data_iter = self._iterate_dataset()
         if self._map_seq_stream is not None:
-            data_iter = self._map_seq_stream(data_iter, rng=self._rng, **util.get_fwd_compat_kwargs())
+            data_iter = self._map_seq_stream(data_iter, epoch=self.epoch, rng=self._rng, **util.get_fwd_compat_kwargs())
             assert isinstance(
                 data_iter, Iterator
             ), f"map_seq_stream must produce an {Iterator.__name__}, but produced {type(data_iter).__name__}"
@@ -260,10 +268,13 @@ class PostprocessingDataset(CachedDataset2):
             tensor_dict = self._in_tensor_dict_template.copy_template()
             for data_key in data_keys:
                 tensor_dict.data[data_key].raw_tensor = self._dataset.get_data(seq_index, data_key)
-            tensor_dict.data["seq_tag"].raw_tensor = str_to_numpy_array(self._dataset.get_tag(seq_index))
+            seq_tag_tensor = str_to_numpy_array(self._dataset.get_tag(seq_index))
+            tensor_dict.data["seq_tag"].raw_tensor = seq_tag_tensor
 
             if self._map_seq is not None:
-                tensor_dict = self._map_seq(tensor_dict, rng=self._rng, **util.get_fwd_compat_kwargs())
+                tensor_dict = self._map_seq(
+                    tensor_dict, epoch=self.epoch, seq_idx=seq_index, rng=self._rng, **util.get_fwd_compat_kwargs()
+                )
                 assert isinstance(
                     tensor_dict, TensorDict
                 ), f"map_seq must produce a {TensorDict.__name__}, but produced {type(tensor_dict).__name__}"
@@ -271,7 +282,14 @@ class PostprocessingDataset(CachedDataset2):
                 # Re-adding the seq tag here causes no harm in case it's dropped since we don't
                 # add/drop any segments w/ the non-iterator postprocessing function.
                 if "seq_tag" not in tensor_dict.data:
-                    tensor_dict.data["seq_tag"].raw_tensor = str_to_numpy_array(self._dataset.get_tag(seq_index))
+                    tensor_dict.data["seq_tag"].raw_tensor = seq_tag_tensor
+
+                if self._seq_list_for_validation is not None:
+                    seq_tag = self._seq_list_for_validation[seq_index]
+                    tag_of_seq = tensor_dict.data["seq_tag"].raw_tensor.item()
+                    assert (
+                        tag_of_seq == seq_tag
+                    ), f"seq tag mismath: {tag_of_seq} != {seq_tag} for seq index {seq_index} when seq list is given"
 
             yield tensor_dict
             seq_index += 1

{returnn-1.20241126.105536 → returnn-1.20241127.143843}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241126.105536
+Version: 1.20241127.143843
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241126.105536/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20241126.105536'
-long_version = '1.20241126.105536+git.13640bc'