returnn 1.20230823.170645__tar.gz → 1.20230824.648__tar.gz

{returnn-1.20230823.170645/returnn.egg-info → returnn-1.20230824.648}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20230823.170645
+Version: 1.20230824.648
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20230824.648/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20230824.000648'
+long_version = '1.20230824.000648+git.e565223'

{returnn-1.20230823.170645 → returnn-1.20230824.648}/returnn/datasets/basic.py

@@ -209,6 +209,7 @@ class Dataset(object):
         """
         assert issubclass(cls, Dataset)
         ds = cls(**kwargs)
+        assert isinstance(ds, Dataset)
         for attr, value in state.items():
             setattr(ds, attr, value)
         return ds

{returnn-1.20230823.170645 → returnn-1.20230824.648}/returnn/datasets/map.py

@@ -84,7 +84,7 @@ class MapDatasetWrapper(CachedDataset2):
         super(MapDatasetWrapper, self).__init__(**kwargs)
 
         if callable(map_dataset):
-            map_dataset = map_dataset(**kwargs)
+            map_dataset = map_dataset()
         assert isinstance(map_dataset, MapDatasetBase)
 
         self._dataset = map_dataset
@@ -92,6 +92,13 @@ class MapDatasetWrapper(CachedDataset2):
         assert map_dataset.data_types, f"{self}: map_dataset {map_dataset} needs to provide data_types"
         self.num_outputs = {key: _get_num_outputs_entry(key, opts) for key, opts in map_dataset.data_types.items()}
 
+    @property
+    def map_dataset(self) -> MapDatasetBase:  # also important for Dataset.__reduce__
+        """
+        :return: the wrapped MapDataset
+        """
+        return self._dataset
+
     @property
     def num_seqs(self):
         """

{returnn-1.20230823.170645 → returnn-1.20230824.648}/returnn/datasets/multi_proc.py

@@ -52,6 +52,10 @@ class MultiProcDataset(CachedDataset2):
 
     def initialize(self):
         """init"""
+        self._lazy_init()
+        super().initialize()
+
+    def _lazy_init(self):
         if not self._worker_procs:
             # We send the global config to the subprocesses,
             # because some datasets might use custom functions inside the config,
@@ -127,8 +131,6 @@ class MultiProcDataset(CachedDataset2):
             msg, self.labels = self._seq_order_proc_parent_conn.recv()
             assert msg == "labels"
 
-        super().initialize()
-
     def __del__(self):
         if self._seq_order_proc:
             self._seq_order_proc_parent_conn.send(("exit", {}))
@@ -274,8 +276,8 @@ class MultiProcDataset(CachedDataset2):
         :returns whether the order changed (True is always safe to return)
         """
         super().init_seq_order(epoch=epoch, seq_list=seq_list, seq_order=seq_order)
-
         if epoch is not None or seq_list is not None or seq_order is not None:
+            self._lazy_init()
             self._seq_order_proc_parent_conn.send(
                 ("init_seq_order", {"epoch": epoch, "seq_list": seq_list, "seq_order": seq_order})
             )

{returnn-1.20230823.170645 → returnn-1.20230824.648/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20230823.170645
+Version: 1.20230824.648
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20230823.170645 → returnn-1.20230824.648}/tests/test_MultiProcDataset.py

@@ -179,6 +179,37 @@ def test_MultiProcDataset_exception_at_item():
             raise Exception("Expected exception")
 
 
+class _MyCustomDummyMapDataset(MapDatasetBase):
+    def __init__(self):
+        super().__init__(data_types={"data": {"shape": (None, 3)}})
+
+    def __len__(self):
+        return 2
+
+    def __getitem__(self, item):
+        return {"data": numpy.zeros((item * 2 + 5, 3))}
+
+
+def test_MultiProcDataset_pickle():
+    import pickle
+
+    with timeout():
+        mp_dataset = MultiProcDataset(
+            dataset={"class": "MapDatasetWrapper", "map_dataset": _MyCustomDummyMapDataset},
+            num_workers=1,
+            buffer_size=1,
+        )
+        mp_dataset.initialize()
+        mp_dataset_seqs = dummy_iter_dataset(mp_dataset)
+        mp_dataset_serialized = pickle.dumps(mp_dataset)
+
+    with timeout():
+        mp_dataset_ = pickle.loads(mp_dataset_serialized)
+        mp_dataset_seqs_ = dummy_iter_dataset(mp_dataset_)
+
+    compare_dataset_seqs(mp_dataset_seqs, mp_dataset_seqs_)
+
+
 if __name__ == "__main__":
     better_exchook.install()
     if len(sys.argv) <= 1:

{returnn-1.20230823.170645 → returnn-1.20230824.648}/tests/test_PTDataset.py

@@ -85,6 +85,33 @@ def test_HDFDataset():
     assert c == n
 
 
+def test_MultiProcDataset_HDFDataset():
+    from test_HDFDataset import generate_hdf_from_other
+    from test_MultiProcDataset import timeout
+    from returnn.datasets.multi_proc import MultiProcDataset
+
+    hdf_fn = generate_hdf_from_other({"class": "Task12AXDataset", "num_seqs": 23})
+    with timeout(10):
+        mp_dataset = MultiProcDataset(
+            dataset={"class": "HDFDataset", "files": [hdf_fn], "cache_byte_size": 0},
+            num_workers=1,
+            buffer_size=1,
+        )
+        mp_dataset.initialize()
+
+        mp_manager = torch.multiprocessing.Manager()
+        loader = get_loader_from_returnn_dataset(mp_dataset, mp_manager)
+        c = 0
+        n = 3
+        for batch in loader:
+            print(batch)
+            c += 1
+            if c >= n:
+                break
+
+        assert c == n
+
+
 if __name__ == "__main__":
     better_exchook.install()
     if len(sys.argv) <= 1:

returnn-1.20230823.170645/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20230823.170645'
-long_version = '1.20230823.170645+git.68affc0'