returnn 1.20241114.200746__tar.gz → 1.20241118.95601__tar.gz

{returnn-1.20241114.200746 → returnn-1.20241118.95601}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241114.200746
+Version: 1.20241118.95601
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241118.95601/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20241118.095601'
+long_version = '1.20241118.095601+git.a0d992a'

{returnn-1.20241114.200746 → returnn-1.20241118.95601}/returnn/datasets/audio.py

@@ -3,7 +3,7 @@ Datasets dealing with audio
 """
 
 from __future__ import annotations
-from typing import Optional, Any, List, Tuple, Dict
+from typing import Optional, Union, Any, List, Tuple, Dict
 import numpy
 import typing
 
@@ -42,38 +42,38 @@ class OggZipDataset(CachedDataset2):
 
     def __init__(
         self,
-        path,
-        audio,
-        targets,
+        path: Union[str, List[str]],
+        audio: Optional[Dict[str, Any]],
+        targets: Union[Vocabulary, Dict[str, Any], None],
         targets_post_process=None,
-        use_cache_manager=False,
-        segment_file=None,
-        zip_audio_files_have_name_as_prefix=True,
-        fixed_random_subset=None,
-        fixed_random_subset_seed=42,
-        epoch_wise_filter=None,
+        use_cache_manager: bool = False,
+        segment_file: Optional[str] = None,
+        zip_audio_files_have_name_as_prefix: bool = True,
+        fixed_random_subset: Union[float, int, None] = None,
+        fixed_random_subset_seed: int = 42,
+        epoch_wise_filter: Optional[dict] = None,
         **kwargs,
     ):
         """
-        :param str|list[str] path: filename to zip
-        :param dict[str]|None audio: options for :class:`ExtractAudioFeatures`.
+        :param path: filename to zip
+        :param audio: options for :class:`ExtractAudioFeatures`.
             use {} for default. None means to disable.
-        :param Vocabulary|dict[str]|None targets: options for :func:`Vocabulary.create_vocab`
+        :param targets: options for :func:`Vocabulary.create_vocab`
             (e.g. :class:`BytePairEncoding`)
         :param str|list[str]|((str)->str)|None targets_post_process: :func:`get_post_processor_function`,
             applied on orth
-        :param bool use_cache_manager: uses :func:`returnn.util.basic.cf`
-        :param str|None segment_file: .txt or .gz text file containing sequence tags that will be used as whitelist.
+        :param use_cache_manager: uses :func:`returnn.util.basic.cf`
+        :param segment_file: .txt or .gz text file containing sequence tags that will be used as whitelist.
             Note: This is somewhat deprecated, as we also support ``seq_list_filter_file`` (via the base class),
             which does the same but more universally.
-        :param bool zip_audio_files_have_name_as_prefix:
-        :param float|int|None fixed_random_subset:
+        :param zip_audio_files_have_name_as_prefix:
+        :param fixed_random_subset:
           Value in [0,1] to specify the fraction, or integer >=1 which specifies number of seqs.
           If given, will use this random subset. This will be applied initially at loading time,
           i.e. not dependent on the epoch.
           It uses the fixed fixed_random_subset_seed as seed, i.e. it's deterministic.
-        :param int fixed_random_subset_seed: Seed for drawing the fixed random subset, default 42
-        :param dict|None epoch_wise_filter: see init_seq_order
+        :param fixed_random_subset_seed: Seed for drawing the fixed random subset, default 42
+        :param epoch_wise_filter: see init_seq_order
         """
         import os
         import zipfile

{returnn-1.20241114.200746 → returnn-1.20241118.95601}/returnn/datasets/multi_proc.py

@@ -66,6 +66,7 @@ class MultiProcDataset(CachedDataset2):
             )
         self._sharding_method = sharding_method
         self._data_keys = None
+        self._data_dtypes: Optional[Dict[str, str]] = None
         self._num_seqs = None
         self._total_num_seqs = None
         self._all_tags = None
@@ -83,6 +84,8 @@ class MultiProcDataset(CachedDataset2):
             self.num_outputs = _meta_info_cache["num_outputs"]
             self._total_num_seqs = _meta_info_cache["total_num_seqs"]
             self.labels = _meta_info_cache["labels"]
+            self._data_keys = _meta_info_cache["data_keys"]
+            self._data_dtypes = _meta_info_cache["data_dtypes"]
 
     def initialize(self):
         """init"""
@@ -100,6 +103,8 @@ class MultiProcDataset(CachedDataset2):
             "labels": self.labels,
             "total_num_seqs": self._total_num_seqs,
             "all_tags": self._all_tags,
+            "data_keys": self._data_keys,
+            "data_dtypes": self._data_dtypes,
         }
 
     def _lazy_init(self):
@@ -177,6 +182,26 @@ class MultiProcDataset(CachedDataset2):
         msg, self.labels = self._seq_order_proc_parent_conn.recv()
         assert msg == "labels"
 
+    def _lazy_init_data_keys(self):
+        if self._data_keys is not None:
+            return
+
+        self._lazy_init()  # indempotent, will not run twice if not needed
+
+        self._seq_order_proc_parent_conn.send(("get_data_keys", {}))
+        msg, self._data_keys = self._seq_order_proc_parent_conn.recv()
+        assert msg == "data_keys"
+
+    def _lazy_init_data_dtypes(self):
+        if self._data_dtypes is not None:
+            return
+
+        self._lazy_init()  # indempotent, will not run twice if not needed
+
+        self._seq_order_proc_parent_conn.send(("get_data_dtypes", {}))
+        msg, self._data_dtypes = self._seq_order_proc_parent_conn.recv()
+        assert msg == "data_dtypes"
+
     def __del__(self):
         if self._worker_procs:
             got_exception = False
@@ -298,6 +323,14 @@ class MultiProcDataset(CachedDataset2):
                     except NotImplementedError as exc:
                         all_tags = NotImplementedError(f"{exc} in {dataset}")
                     parent_conn.send(("all_tags", all_tags))
+                elif msg == "get_data_keys":
+                    assert dataset
+                    data_keys = dataset.get_data_keys()
+                    parent_conn.send(("data_keys", data_keys))
+                elif msg == "get_data_dtypes":
+                    assert dataset
+                    data_keys = dataset.get_data_keys()
+                    parent_conn.send(("data_dtypes", {k: dataset.get_data_dtype(k) for k in data_keys}))
                 elif msg == "init_seq_order":
                     if dataset is None:
                         dataset = init_dataset(dataset_dict)
@@ -444,3 +477,24 @@ class MultiProcDataset(CachedDataset2):
         super().finish_epoch(free_resources=free_resources)
         for worker_parent_conn in self._worker_parent_conns:
             worker_parent_conn.send(("finish_epoch", {"free_resources": free_resources}))
+
+    def get_data_keys(self) -> List[str]:
+        """data keys"""
+        if self._data_keys is None:
+            self._lazy_init_data_keys()
+            assert self._data_keys is not None
+        return self._data_keys
+
+    def get_data_dtype(self, key: str) -> str:
+        """:return: dtype of `key`"""
+        if self._data_dtypes is None:
+            self._lazy_init_data_dtypes()
+            assert self._data_dtypes is not None
+        return self._data_dtypes[key]
+
+    def is_data_sparse(self, key: str) -> bool:
+        """:return: whether `key` is sparse"""
+        if self.num_outputs is None:
+            self._lazy_init()
+            assert self.num_outputs is not None
+        return super().is_data_sparse(key)

{returnn-1.20241114.200746 → returnn-1.20241118.95601}/returnn/datasets/util/vocabulary.py

@@ -592,4 +592,10 @@ class Utf8ByteTargets(Vocabulary):
         :param list[int]|numpy.ndarray seq: 1D sequence
         :rtype: str
         """
+        if not isinstance(seq, numpy.ndarray):
+            seq = numpy.array(seq)
+        assert seq.ndim == 1
+        if seq.dtype != numpy.uint8:
+            assert ((seq >= 0) & (seq < 256)).all(), f"invalid byte value, must be within 0-255: {seq}"
+            seq = seq.astype(numpy.uint8)
         return bytearray(seq).decode(encoding="utf8")

{returnn-1.20241114.200746 → returnn-1.20241118.95601}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241114.200746
+Version: 1.20241118.95601
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241114.200746/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20241114.200746'
-long_version = '1.20241114.200746+git.cb79470'