returnn 1.20240610.115802__tar.gz → 1.20240611.113715__tar.gz

{returnn-1.20240610.115802/returnn.egg-info → returnn-1.20240611.113715}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240610.115802
+Version: 1.20240611.113715
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240611.113715/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240611.113715'
+long_version = '1.20240611.113715+git.8f3d985'

{returnn-1.20240610.115802 → returnn-1.20240611.113715}/returnn/datasets/basic.py

@@ -1367,7 +1367,7 @@ def get_dataset_class(name: Union[str, Type[Dataset]]) -> Optional[Type[Dataset]
         "raw_wav",
         "map",
         "multi_proc",
-        "concat_files",
+        "distrib_files",
     ]
     for mod_name in mod_names:
         mod = import_module("returnn.datasets.%s" % mod_name)

returnn-1.20240610.115802/returnn/datasets/concat_files.py → returnn-1.20240611.113715/returnn/datasets/distrib_files.py

@@ -1,5 +1,5 @@
 """
-:class:`ConcatFilesDataset `
+:class:`DistributeFilesDataset`
 
 https://github.com/rwth-i6/returnn/issues/1519
 """
@@ -23,19 +23,21 @@ from multiprocessing.connection import Connection as mpConnection
 _mp = NonDaemonicSpawnContext(process_pre_init_func=SubProcCopyGlobalConfigPreInitFunc())
 
 
-__all__ = ["ConcatFilesDataset"]
+__all__ = ["DistributeFilesDataset"]
 
 Filename = str
 FileTree = Union[Filename, Tuple["FileTree", ...], Dict[Any, "FileTree"], List["FileTree"]]
 
 
-class ConcatFilesDataset(CachedDataset2):
+class DistributeFilesDataset(CachedDataset2):
     """
-    This is similar to :class:`ConcatDataset`, but instead of concatenating datasets,
-    we distribute files over subepochs,
-    and then create a sub dataset for every sub epoch
-    for a given subset of the files
-    via the given ``get_sub_epoch_dataset``.
+    Dataset that distributes files over subepochs and then creates a
+    sub dataset for every sub epoch for a given (random) subset of the files.
+    The sub dataset is user-defined via a function ``get_sub_epoch_dataset``.
+    Thus, this dataset wraps the sub datasets.
+
+    It is conceptually very similar to :class:`ConcatDataset` in the sense
+    that it concatenates all the sub datasets together to form one larger dataset.
 
     This scheme allows to shuffle over the files,
     which makes shuffling much more efficient over a large dataset
@@ -70,7 +72,7 @@ class ConcatFilesDataset(CachedDataset2):
           }
 
         train = {
-          "class": "ConcatFilesDataset",
+          "class": "DistributeFilesDataset",
           "files": [
             "/nfs/big_data_1.hdf",
             ...
@@ -105,7 +107,7 @@ class ConcatFilesDataset(CachedDataset2):
           }
 
         train = {
-          "class": "ConcatFilesDataset",
+          "class": "DistributeFilesDataset",
           "files": [
             ("/nfs/alignment_1.hdf", "/nfs/features_1.hdf"),
             ...
@@ -321,7 +323,7 @@ class ConcatFilesDataset(CachedDataset2):
         # [[1,1], [78], [120], []] or [[1,1,78], [120], [], []].
         # Or consider [5,5]+[10]*7, partition_epoch=5, which has avg size 16.
         # A simple algorithm could end up with [[5,5,10], [10,10], [10,10], [10,10], []].
-        # See test_ConcatFilesDataset_get_files_per_sub_epochs for some test cases.
+        # See test_DistributeFilesDataset_get_files_per_sub_epochs for some test cases.
         assert len(files_order) >= partition_epoch
         files_per_sub_epochs = [[] for _ in range(partition_epoch)]
         assert len(files_per_sub_epochs) == partition_epoch

{returnn-1.20240610.115802 → returnn-1.20240611.113715}/returnn/datasets/meta.py

@@ -1934,7 +1934,6 @@ class AnythingDataset(Dataset):
         self._input_key = next(iter(data_keys))
         self.num_inputs = data_keys[self._input_key]["dim"]
         self.num_outputs = {k: (v["dim"], len(v["shape"])) for (k, v) in data_keys.items()}
-        self.labels = None
         self._seq_list = None
         self._seq_order = None
 

{returnn-1.20240610.115802 → returnn-1.20240611.113715/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240610.115802
+Version: 1.20240611.113715
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20240610.115802 → returnn-1.20240611.113715}/returnn.egg-info/SOURCES.txt

@@ -99,7 +99,7 @@ returnn/datasets/basic.py
 returnn/datasets/bundle_file.py
 returnn/datasets/cached.py
 returnn/datasets/cached2.py
-returnn/datasets/concat_files.py
+returnn/datasets/distrib_files.py
 returnn/datasets/generating.py
 returnn/datasets/hdf.py
 returnn/datasets/lm.py

{returnn-1.20240610.115802 → returnn-1.20240611.113715}/tests/test_Dataset.py

@@ -623,13 +623,13 @@ def test_MapDatasetWrapper():
     assert res.features["data"].shape == (5, 3)
 
 
-def test_ConcatFilesDataset_get_files_per_sub_epochs():
-    from returnn.datasets.concat_files import ConcatFilesDataset
+def test_DistributeFilesDataset_get_files_per_sub_epochs():
+    from returnn.datasets.distrib_files import DistributeFilesDataset
 
     def _test(sizes: List[int], partition_epoch: int, expected: List[List[int]]):
         files = [f"file-{i}" for i in range(len(sizes))]
         file_sizes = {f: s for f, s in zip(files, sizes)}
-        res = ConcatFilesDataset._get_files_per_sub_epochs(
+        res = DistributeFilesDataset._get_files_per_sub_epochs(
             partition_epoch=partition_epoch, file_sizes=file_sizes, files_order=files
         )
         assert all(res) and len(res) == partition_epoch
@@ -643,8 +643,8 @@ def test_ConcatFilesDataset_get_files_per_sub_epochs():
     _test([5, 5] + [10] * 7, 5, [[5, 5, 10], [10, 10], [10, 10], [10], [10]])
 
 
-def test_ConcatFilesDataset():
-    from returnn.datasets.concat_files import ConcatFilesDataset
+def test_DistributeFilesDataset():
+    from returnn.datasets.distrib_files import DistributeFilesDataset
     from test_HDFDataset import generate_hdf_from_other
 
     # Create a few HDF files such that we can easily verify the data later.
@@ -686,7 +686,7 @@ def test_ConcatFilesDataset():
         total_num_seqs += 1
     assert total_num_seqs == num_hdf_files * num_seqs
 
-    # Test to load via ConcatFilesDataset.
+    # Test to load via DistributeFilesDataset.
 
     def _get_sub_epoch_dataset(files_subepoch: List[str]) -> Dict[str, Any]:
         return {"class": "HDFDataset", "files": files_subepoch, "seq_ordering": "default"}
@@ -695,13 +695,13 @@ def test_ConcatFilesDataset():
     assert num_hdf_files % partition_epoch == 0  # just for easier testing here
     concat_dataset = init_dataset(
         {
-            "class": "ConcatFilesDataset",
+            "class": "DistributeFilesDataset",
             "files": hdf_files,
             "get_sub_epoch_dataset": _get_sub_epoch_dataset,
             "partition_epoch": partition_epoch,
         }
     )
-    assert isinstance(concat_dataset, ConcatFilesDataset)
+    assert isinstance(concat_dataset, DistributeFilesDataset)
     assert concat_dataset.get_data_keys() == ["classes"]
     num_hdfs_per_part = num_hdf_files // partition_epoch
     global_seq_idx = 0

returnn-1.20240610.115802/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240610.115802'
-long_version = '1.20240610.115802+git.dd63c66'