returnn 1.20241219.171259__tar.gz → 1.20241224.5643__tar.gz

{returnn-1.20241219.171259/returnn.egg-info → returnn-1.20241224.5643}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241219.171259
+Version: 1.20241224.5643
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241224.5643/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20241224.005643'
+long_version = '1.20241224.005643+git.d5f6aa4'

{returnn-1.20241219.171259 → returnn-1.20241224.5643}/returnn/datasets/audio.py

@@ -3,7 +3,7 @@ Datasets dealing with audio
 """
 
 from __future__ import annotations
-from typing import Optional, Union, Any, List, Tuple, Dict
+from typing import Optional, Union, Any, Sequence, List, Tuple, Dict
 import typing
 import os
 import numpy
@@ -41,10 +41,13 @@ class OggZipDataset(CachedDataset2):
     however, it does not have to match the real duration in any way.
     """
 
+    _getnewargs_remap = dict(content_name="_names", **CachedDataset2._getnewargs_remap)
+
     def __init__(
         self,
-        path: Union[str, List[str]],
+        path: Union[str, Sequence[str]],
         *,
+        content_name: Optional[Union[str, Sequence[str]]] = None,
         resolve_symlink_for_name: bool = False,
         audio: Optional[Dict[str, Any]],
         targets: Union[Vocabulary, Dict[str, Any], None],
@@ -59,8 +62,11 @@ class OggZipDataset(CachedDataset2):
     ):
         """
         :param path: filename to zip
+        :param content_name: internal name of the dataset, which is used for filenames inside the ZIP.
+            If not given, the internal name is determined as ``os.path.splitext(os.path.basename(path))[0]``.
         :param resolve_symlink_for_name: The internal name of the dataset, which is used for filenames inside the ZIP,
-            is determined as ``os.path.splitext(os.path.basename(path))[0]``.
+            is determined as ``os.path.splitext(os.path.basename(path))[0]``
+            (if ``content_name`` is not given).
             If this is True, we will resolve symlinks for the name first.
         :param audio: options for :class:`ExtractAudioFeatures`.
             use {} for default. None means to disable.
@@ -96,19 +102,29 @@ class OggZipDataset(CachedDataset2):
             and os.path.isdir(path)
             and os.path.isfile(path + ".txt")
         ):
+            assert content_name is None  # not implemented
+            assert not resolve_symlink_for_name  # not implemented
             # Special case (mostly for debugging) to directly access the filesystem, not via zip-file.
             self.paths = [os.path.dirname(path)]
             self._names = [os.path.basename(path)]
             self._use_zip_files = False
             assert not use_cache_manager, "cache manager only for zip file"
         else:
-            if not isinstance(path, (tuple, list)):
+            if isinstance(path, str):
                 path = [path]
+            assert isinstance(path, (tuple, list))
+            if content_name is not None:
+                if isinstance(content_name, str):
+                    content_name = [content_name] * len(path)
+                assert isinstance(content_name, (tuple, list))
+                assert len(content_name) == len(path)
             self.paths = []
             self._names = []
-            for path_ in path:
+            for i, path_ in enumerate(path):
                 assert isinstance(path_, str)
                 name, ext = _get_internal_ogg_zip_name_ext(path_, resolve_symlink=resolve_symlink_for_name)
+                if content_name is not None:
+                    name = content_name[i]
                 if use_cache_manager:
                     path_ = returnn.util.basic.cf(path_)
                 if ext == ".txt.gz":

{returnn-1.20241219.171259 → returnn-1.20241224.5643}/returnn/datasets/lm.py

@@ -67,6 +67,7 @@ class LmDataset(CachedDataset2):
         error_on_invalid_seq=True,
         add_delayed_seq_data=False,
         delayed_seq_data_start_symbol="[START]",
+        dtype: Optional[str] = None,
         **kwargs,
     ):
         """
@@ -117,6 +118,7 @@ class LmDataset(CachedDataset2):
         :param bool add_delayed_seq_data: will add another data-key "delayed" which will have the sequence.
           delayed_seq_data_start_symbol + original_sequence[:-1].
         :param str delayed_seq_data_start_symbol: used for add_delayed_seq_data.
+        :param dtype: explicit dtype. if not given, automatically determined based on the number of labels.
         """
         super(LmDataset, self).__init__(**kwargs)
 
@@ -245,7 +247,9 @@ class LmDataset(CachedDataset2):
             assert not orth_replace_map_file
 
         num_labels = len(self.labels["data"])
-        if num_labels <= 2**7:
+        if dtype:
+            self.dtype = dtype
+        elif num_labels <= 2**7:
             self.dtype = "int8"
         elif num_labels <= 2**8:
             self.dtype = "uint8"
@@ -347,8 +351,9 @@ class LmDataset(CachedDataset2):
                 last_print_time = time.time()
 
         # If a list of files is provided, concatenate all.
-        if not isinstance(corpus_file, list):
+        if isinstance(corpus_file, str):
             corpus_file = [corpus_file]
+        assert isinstance(corpus_file, (tuple, list))
         prev_orth_len = 0
         for file_name in corpus_file:
             if self._use_cache_manager:

{returnn-1.20241219.171259 → returnn-1.20241224.5643}/returnn/datasets/meta.py

@@ -1905,14 +1905,6 @@ class VariableDataset(Dataset):
         """current seq order"""
         return self._dataset.get_current_seq_order()
 
-    def get_all_tags(self) -> List[str]:
-        """all tags"""
-        return self._dataset.get_all_tags()
-
-    def get_total_num_seqs(self, *, fast: bool = False) -> int:
-        """total num seqs"""
-        return self._dataset.get_total_num_seqs(fast=fast)
-
     def get_seq_length(self, sorted_seq_idx: int) -> NumbersDict:
         """seq len"""
         return self._dataset.get_seq_length(sorted_seq_idx)

{returnn-1.20241219.171259 → returnn-1.20241224.5643/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241219.171259
+Version: 1.20241224.5643
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241219.171259/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20241219.171259'
-long_version = '1.20241219.171259+git.3709edd'