returnn 1.20240522.175941__tar.gz → 1.20240605.145641__tar.gz

{returnn-1.20240522.175941 → returnn-1.20240605.145641}/LICENSE

@@ -168,7 +168,7 @@ Contains code from CPython:
  - https://www.python.org/
  - Python Software Foundation License (PSF)
  - Copyright 2001..2021 Python Software Foundation
- - py-to-pickle.cpp
+ - py-to-pickle.cpp, util/task_system.py, util/basic.py
 
 Contains code from PyTorch:
  - https://github.com/pytorch/pytorch/

{returnn-1.20240522.175941/returnn.egg-info → returnn-1.20240605.145641}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240522.175941
+Version: 1.20240605.145641
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240605.145641/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240605.145641'
+long_version = '1.20240605.145641+git.24cc84b'

{returnn-1.20240522.175941 → returnn-1.20240605.145641}/returnn/config.py

@@ -781,3 +781,36 @@ class _GlobalConfigAsPyModuleProxy(_types.ModuleType):
         cfg: Optional[Config] = self._get_config()
         if cfg:
             cfg.typed_dict[key] = value
+
+
+class SubProcCopyGlobalConfigPreInitFunc:
+    """
+    A pre-init hook for a subprocess which copies the global config to the subprocess.
+
+    It can be important that this init function is called even before the unpickling of other data,
+    as that unpickling might depend on the right context, e.g. having the global RETURNN config.
+    Example issue with MultiProcDataset: https://github.com/rwth-i6/returnn/issues/1495
+
+    Example usage::
+
+        NonDaemonicSpawnContext(process_pre_init_func=SubProcCopyGlobalConfigPreInitFunc())
+    """
+
+    def __init__(self):
+        # Get the RETURNN global config here. Allow this to be optional (for use outside of RETURNN).
+        # We store it here such that pickling this worker init func will also pickle the config,
+        # so that we can reset it as global config inside the worker.
+        # Some RETURNN datasets might depend on the config.
+        # https://github.com/rwth-i6/returnn/issues/1495
+        # MultiProcDataset has a similar logic, see https://github.com/rwth-i6/returnn/issues/1384.
+        self.global_config = get_global_config(raise_exception=False)
+
+    def __call__(self):
+        from returnn.util import better_exchook
+        from returnn import __old_mod_loader__
+
+        better_exchook.install()
+        __old_mod_loader__.disable_lazy_mod_loads()
+
+        if self.global_config:
+            set_global_config(self.global_config)

{returnn-1.20240522.175941 → returnn-1.20240605.145641}/returnn/datasets/basic.py

@@ -1343,7 +1343,19 @@ def get_dataset_class(name: Union[str, Type[Dataset]]) -> Optional[Type[Dataset]
 
     # Only those modules which make sense to be loaded by the user,
     # because this function is only used for such cases.
-    mod_names = ["hdf", "sprint", "generating", "numpy_dump", "meta", "lm", "stereo", "raw_wav", "map", "multi_proc"]
+    mod_names = [
+        "hdf",
+        "sprint",
+        "generating",
+        "numpy_dump",
+        "meta",
+        "lm",
+        "stereo",
+        "raw_wav",
+        "map",
+        "multi_proc",
+        "concat_files",
+    ]
     for mod_name in mod_names:
         mod = import_module("returnn.datasets.%s" % mod_name)
         for name_, clazz in vars(mod).items():

{returnn-1.20240522.175941 → returnn-1.20240605.145641}/returnn/datasets/cached.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+from typing import List
 import gc
 import sys
 import time
@@ -621,6 +622,14 @@ class CachedDataset(Dataset):
         seq_len = self.get_seq_length_nd(sorted_seq_idx)[idx]
         return self.targets[target][seq_start : seq_start + seq_len]
 
+    def get_data_keys(self) -> List[str]:
+        """data keys"""
+        res = []
+        if self.num_inputs > 0:
+            res.append("data")
+        res.extend(self.get_target_list())
+        return res
+
     def get_target_list(self):
         return list(self.targets.keys())
 

{returnn-1.20240522.175941 → returnn-1.20240605.145641}/returnn/datasets/cached2.py

@@ -3,6 +3,7 @@ Provides :class:`CachedDataset2`.
 """
 
 from __future__ import annotations
+import numpy
 import typing
 from typing import Optional
 from threading import Condition
@@ -168,6 +169,7 @@ class CachedDataset2(Dataset):
         if self.added_data:
             return
         self.load_seqs(self.expected_load_seq_start, self.expected_load_seq_start + 1)
+        assert self.added_data, f"{self}: _load_something: could not load anything"
 
     def get_seq_length(self, sorted_seq_idx):
         """
@@ -180,33 +182,29 @@ class CachedDataset2(Dataset):
         self.load_seqs(self.expected_load_seq_start, sorted_seq_idx + 1)
         return self._get_seq(sorted_seq_idx).num_frames
 
-    def get_data(self, seq_idx, key):
+    def get_data(self, seq_idx: int, key: str) -> numpy.ndarray:
         """
-        :param int seq_idx:
-        :param str key:
-        :rtype: numpy.ndarray
+        :param seq_idx:
+        :param key:
         """
         return self._get_seq(seq_idx).features[key]
 
-    def get_input_data(self, seq_idx):
+    def get_input_data(self, seq_idx: int) -> numpy.ndarray:
         """
-        :param int seq_idx:
-        :rtype: numpy.ndarray
+        :param seq_idx:
         """
         return self.get_data(seq_idx, "data")
 
-    def get_targets(self, target, seq_idx):
+    def get_targets(self, target: str, seq_idx: int) -> numpy.ndarray:
         """
-        :param str target:
-        :param int seq_idx:
-        :rtype: numpy.ndarray
+        :param target:
+        :param seq_idx:
         """
         return self.get_data(seq_idx, target)
 
-    def get_tag(self, sorted_seq_idx):
+    def get_tag(self, sorted_seq_idx: int) -> str:
         """
-        :param int sorted_seq_idx:
-        :rtype: str
+        :param sorted_seq_idx:
         """
         # get_tag() can be called before the seq is loaded via load_seqs().
         # Thus, we just call load_seqs() ourselves here.