returnn 1.20241203.102806__tar.gz → 1.20241204.102546__tar.gz

{returnn-1.20241203.102806 → returnn-1.20241204.102546}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241203.102806
+Version: 1.20241204.102546
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241204.102546/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20241204.102546'
+long_version = '1.20241204.102546+git.2709208'

{returnn-1.20241203.102806 → returnn-1.20241204.102546}/returnn/datasets/audio.py

@@ -4,8 +4,9 @@ Datasets dealing with audio
 
 from __future__ import annotations
 from typing import Optional, Union, Any, List, Tuple, Dict
-import numpy
 import typing
+import os
+import numpy
 
 from .basic import DatasetSeq
 from .cached2 import CachedDataset2
@@ -43,6 +44,8 @@ class OggZipDataset(CachedDataset2):
     def __init__(
         self,
         path: Union[str, List[str]],
+        *,
+        resolve_symlink_for_name: bool = False,
         audio: Optional[Dict[str, Any]],
         targets: Union[Vocabulary, Dict[str, Any], None],
         targets_post_process=None,
@@ -56,6 +59,9 @@ class OggZipDataset(CachedDataset2):
     ):
         """
         :param path: filename to zip
+        :param resolve_symlink_for_name: The internal name of the dataset, which is used for filenames inside the ZIP,
+            is determined as ``os.path.splitext(os.path.basename(path))[0]``.
+            If this is True, we will resolve symlinks for the name first.
         :param audio: options for :class:`ExtractAudioFeatures`.
             use {} for default. None means to disable.
         :param targets: options for :func:`Vocabulary.create_vocab`
@@ -75,13 +81,13 @@ class OggZipDataset(CachedDataset2):
         :param fixed_random_subset_seed: Seed for drawing the fixed random subset, default 42
         :param epoch_wise_filter: see init_seq_order
         """
-        import os
         import zipfile
         import returnn.util.basic
         from .meta import EpochWiseFilter
 
         self._separate_txt_files = {}  # name -> filename
         self._path = path
+        self._resolve_symlink_for_name = resolve_symlink_for_name
         self._use_cache_manager = use_cache_manager
         self._zip_files: Optional[List[zipfile.ZipFile]] = None  # lazily loaded
         if (
@@ -102,9 +108,7 @@ class OggZipDataset(CachedDataset2):
             self._names = []
             for path_ in path:
                 assert isinstance(path_, str)
-                name, ext = os.path.splitext(os.path.basename(path_))
-                if "." in name and ext == ".gz":
-                    name, ext = name[: name.rindex(".")], name[name.rindex(".") :] + ext
+                name, ext = _get_internal_ogg_zip_name_ext(path_, resolve_symlink=resolve_symlink_for_name)
                 if use_cache_manager:
                     path_ = returnn.util.basic.cf(path_)
                 if ext == ".txt.gz":
@@ -538,3 +542,12 @@ class OggZipDataset(CachedDataset2):
             seq_idx=corpus_seq_idx,
             seq_tag=seq_tag,
         )
+
+
+def _get_internal_ogg_zip_name_ext(path: str, *, resolve_symlink: bool = False) -> Tuple[str, str]:
+    if resolve_symlink:
+        path = os.path.realpath(path)
+    name, ext = os.path.splitext(os.path.basename(path))
+    if "." in name and ext == ".gz":
+        name, ext = name[: name.rindex(".")], name[name.rindex(".") :] + ext
+    return name, ext

{returnn-1.20241203.102806 → returnn-1.20241204.102546}/returnn/datasets/basic.py

@@ -331,29 +331,42 @@ class Dataset:
         return chunk_size, chunk_step, None
 
     @staticmethod
-    def _load_seq_list_file(filename, use_cache_manager=False, expect_list=True):
+    def _load_seq_list_file(
+        filename: str, *, use_cache_manager: bool = False, expect_list: bool = True
+    ) -> Union[List[str], Dict[str, List[str]]]:
         """
-        :param str filename:
-        :param bool use_cache_manager:
-        :param bool expect_list:
-        :rtype: list[str]|dict[str,list[str]]
+        :param filename:
+        :param use_cache_manager:
+        :param expect_list:
+        :return: seq list file content. usually a list of seqs.
         """
         if use_cache_manager:
             import returnn.util.basic
 
             filename = returnn.util.basic.cf(filename)
+
+        if filename.endswith(".gz"):
+            import gzip
+
+            filename = filename[: -len(".gz")]
+
+            def _open(mode):
+                return gzip.open(filename + ".gz", mode)
+
+        else:
+
+            def _open(mode):
+                return open(filename, mode)
+
         if filename.endswith(".pkl"):
+            import gzip
             import pickle
 
-            seq_list = pickle.load(open(filename, "rb"))
+            seq_list = pickle.load(_open("rb"))
             if expect_list:
                 assert isinstance(seq_list, list)
-        elif filename.endswith(".gz"):
-            import gzip
-
-            seq_list = gzip.open(filename, "rt").read().splitlines()
         else:
-            seq_list = open(filename).read().splitlines()
+            seq_list = _open("rt").read().splitlines()
         return seq_list
 
     def _sliding_window(self, xr):

{returnn-1.20241203.102806 → returnn-1.20241204.102546}/returnn/datasets/lm.py

@@ -465,6 +465,10 @@ class LmDataset(CachedDataset2):
             self.seq_gen.random_seed(self._get_random_seed_for_epoch(epoch))
         return True
 
+    def get_current_seq_order(self) -> List[int]:
+        """:return: seq order of current epoch"""
+        return self.seq_order
+
     def supports_seq_order_sorting(self) -> bool:
         """supports sorting"""
         return True

{returnn-1.20241203.102806 → returnn-1.20241204.102546}/returnn/datasets/meta.py

@@ -202,7 +202,7 @@ class MetaDataset(CachedDataset2):
         self,
         datasets: Dict[str, Dict[str, Any]],
         data_map: Dict[str, Tuple[str, str]],
-        seq_list_file: Optional[str] = None,
+        seq_list_file: Optional[Union[str, Dict[str, str]]] = None,
         seq_order_control_dataset: Optional[str] = None,
         seq_lens_file: Optional[str] = None,
         data_dims: Optional[Dict[str, Tuple[int, int]]] = None,  # deprecated
@@ -214,7 +214,10 @@ class MetaDataset(CachedDataset2):
         :param datasets: dataset-key -> dataset-kwargs. including keyword 'class' and maybe 'files'
         :param data_map: self-data-key -> (dataset-key, dataset-data-key).
             Should contain 'data' as key. Also defines the target-list, which is all except 'data'.
-        :param seq_list_file: filename. pickle. dict[str,list[str]], dataset-key -> list of sequence tags.
+        :param seq_list_file: filename. pickle (.pkl) or txt (line-based seq tags). optionally gzipped (.gz).
+            If a single file, and pickled, it can directly contain the dict:
+                dict[str,list[str]]: dataset-key -> list of sequence tags.
+            If a dict, expect dataset-key -> filename.
             Can be None if tag format is the same for all datasets.
                 Then the sequence list will be default sequence order of default dataset (``data_map["data"][0]``),
                 or seq_order_control_dataset.
@@ -289,14 +292,12 @@ class MetaDataset(CachedDataset2):
         self.orig_seq_order_is_initialized = False
         self.seq_list_ordered = None  # type: typing.Optional[typing.Dict[str,typing.List[str]]]
 
-    def _load_seq_list(self, seq_list_file: Optional[str] = None) -> Dict[str, List[str]]:
+    def _load_seq_list(self, seq_list_file: Optional[Union[str, Dict[str, str]]] = None) -> Dict[str, List[str]]:
         """
         :param seq_list_file:
         :return: dict: dataset key -> seq list
         """
-        if seq_list_file:
-            seq_list = Dataset._load_seq_list_file(seq_list_file, expect_list=False)
-        else:
+        if not seq_list_file:
             # We create a sequence list from all the sequences of the default dataset
             # and hope that it also applies to the
             # other datasets.
@@ -347,10 +348,24 @@ class MetaDataset(CachedDataset2):
                         )
                         break  # only print one
                 raise Exception("Dataset %r is missing seqs." % key)
+        elif isinstance(seq_list_file, str):
+            seq_list = Dataset._load_seq_list_file(seq_list_file, expect_list=False)
+        elif isinstance(seq_list_file, dict):
+            for key in self.dataset_keys:
+                if key not in seq_list_file:
+                    raise ValueError(f"seq_list_file does not contain all datasets, missing {key}")
+            seq_list = {key: Dataset._load_seq_list_file(seq_list_file[key]) for key in self.dataset_keys}
+        else:
+            raise TypeError(f"unexpected seq_list_file type {type(seq_list_file)}")
 
-        assert isinstance(seq_list, (list, dict))
         if isinstance(seq_list, list):
             seq_list = {key: seq_list for key in self.dataset_keys}
+        elif isinstance(seq_list, dict):
+            for key in self.dataset_keys:
+                if key not in seq_list:
+                    raise ValueError(f"seq_list does not contain all datasets, missing {key}")
+        else:
+            raise TypeError(f"unexpected seq_list type {type(seq_list)}")
 
         return seq_list
 

{returnn-1.20241203.102806 → returnn-1.20241204.102546}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241203.102806
+Version: 1.20241204.102546
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241203.102806/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20241203.102806'
-long_version = '1.20241203.102806+git.8441d07'