returnn 1.20240723.153926__tar.gz → 1.20240723.171654__tar.gz

{returnn-1.20240723.153926 → returnn-1.20240723.171654}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240723.153926
+Version: 1.20240723.171654
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240723.171654/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240723.171654'
+long_version = '1.20240723.171654+git.24e59a7'

{returnn-1.20240723.153926 → returnn-1.20240723.171654}/returnn/datasets/audio.py

@@ -62,7 +62,7 @@ class OggZipDataset(CachedDataset2):
             (e.g. :class:`BytePairEncoding`)
         :param str|list[str]|((str)->str)|None targets_post_process: :func:`get_post_processor_function`,
             applied on orth
-        :param bool use_cache_manager: uses :func:`Util.cf`
+        :param bool use_cache_manager: uses :func:`returnn.util.basic.cf`
         :param str|None segment_file: .txt or .gz text file containing sequence tags that will be used as whitelist
         :param bool zip_audio_files_have_name_as_prefix:
         :param float|int|None fixed_random_subset:

{returnn-1.20240723.153926 → returnn-1.20240723.171654}/returnn/datasets/lm.py

@@ -17,7 +17,7 @@ import xml.etree.ElementTree as ElementTree
 import numpy
 from random import Random
 
-from returnn.util.basic import parse_orthography, parse_orthography_into_symbols, load_json, unicode
+from returnn.util.basic import parse_orthography, parse_orthography_into_symbols, load_json, unicode, cf
 from returnn.util.literal_py_to_pickle import literal_eval
 from returnn.log import log
 
@@ -37,6 +37,7 @@ class LmDataset(CachedDataset2):
     def __init__(
         self,
         corpus_file,
+        use_cache_manager=False,
         skip_empty_lines=True,
         orth_vocab=None,
         orth_symbols_file=None,
@@ -77,6 +78,7 @@ class LmDataset(CachedDataset2):
         mapping from symbol to integer index (in case ``phone_info`` is not set).
 
         :param str|()->str|list[str]|()->list[str] corpus_file: Bliss XML or line-based txt. optionally can be gzip.
+        :param bool use_cache_manager: uses :func:`returnn.util.basic.cf`
         :param bool skip_empty_lines: for line-based txt
         :param dict[str,typing.Any]|Vocabulary orth_vocab:
         :param str|()->str|None orth_symbols_file: a text file containing a list of orthography symbols
@@ -106,14 +108,13 @@ class LmDataset(CachedDataset2):
         super(LmDataset, self).__init__(**kwargs)
 
         self._corpus_file = corpus_file
+        self._use_cache_manager = use_cache_manager
         self._skip_empty_lines = skip_empty_lines
         self._orth_symbols_file = orth_symbols_file
         self._orth_symbols_map_file = orth_symbols_map_file
         self._orth_replace_map_file = orth_replace_map_file
         self._phone_info = phone_info
 
-        if callable(corpus_file):
-            corpus_file = corpus_file()
         if callable(orth_symbols_file):
             orth_symbols_file = orth_symbols_file()
         if callable(orth_symbols_map_file):
@@ -121,8 +122,6 @@ class LmDataset(CachedDataset2):
         if callable(orth_replace_map_file):
             orth_replace_map_file = orth_replace_map_file()
 
-        print("LmDataset, loading file", corpus_file, file=log.v4)
-
         self.word_based = word_based
         self.word_end_symbol = word_end_symbol
         self.seq_end_symbol = seq_end_symbol
@@ -263,20 +262,35 @@ class LmDataset(CachedDataset2):
             self.num_outputs["delayed"] = self.num_outputs["data"]
             self.labels["delayed"] = self.labels["data"]
 
+        self.orths = None  # will be loaded in _lazy_init
+
+        self.next_orth_idx = 0
+        self.next_seq_idx = 0
+        self.num_skipped = 0
+        self.num_unknown = 0
+
+    def _lazy_init(self):
+        if self.orths is not None:
+            return
+
+        corpus_file = self._corpus_file
+        if callable(corpus_file):
+            corpus_file = corpus_file()
+        print("LmDataset, loading file", corpus_file, file=log.v4)
         if isinstance(corpus_file, list):  # If a list of files is provided, concatenate all.
             self.orths = []
             for file_name in corpus_file:
-                self.orths += read_corpus(file_name, skip_empty_lines=skip_empty_lines)
+                if self._use_cache_manager:
+                    file_name = cf(file_name)
+                self.orths += read_corpus(file_name, skip_empty_lines=self._skip_empty_lines)
         else:
-            self.orths = read_corpus(corpus_file, skip_empty_lines=skip_empty_lines)
-        # It's only estimated because we might filter some out or so.
-        self._estimated_num_seqs = len(self.orths) // self.partition_epoch
+            if self._use_cache_manager:
+                corpus_file = cf(corpus_file)
+            self.orths = read_corpus(corpus_file, skip_empty_lines=self._skip_empty_lines)
         print("  done, loaded %i sequences" % len(self.orths), file=log.v4)
 
-        self.next_orth_idx = 0
-        self.next_seq_idx = 0
-        self.num_skipped = 0
-        self.num_unknown = 0
+        # It's only estimated because we might filter some out or so.
+        self._estimated_num_seqs = len(self.orths) // self.partition_epoch
 
     def get_data_keys(self):
         """
@@ -325,7 +339,10 @@ class LmDataset(CachedDataset2):
             self.seq_order = seq_order
         elif seq_list is not None:
             self.seq_order = [int(s[len(self._tag_prefix) :]) for s in seq_list]
+        elif epoch is None:
+            self.seq_order = []
         else:
+            self._lazy_init()
             self.seq_order = self.get_seq_order_for_epoch(
                 epoch=epoch, num_seqs=len(self.orths), get_seq_len=lambda i: len(self.orths[i])
             )
@@ -343,6 +360,7 @@ class LmDataset(CachedDataset2):
 
     def get_total_num_seqs(self) -> int:
         """total num seqs"""
+        self._lazy_init()
         return len(self.orths)
 
     def _reduce_log_skipped_seqs(self):
@@ -373,10 +391,11 @@ class LmDataset(CachedDataset2):
             if self.next_orth_idx >= len(self.seq_order):
                 assert self.next_seq_idx <= seq_idx, "We expect that we iterate through all seqs."
                 if self.num_skipped > 0:
-                    print("LmDataset: reached end, skipped %i sequences" % self.num_skipped)
+                    print("LmDataset: reached end, skipped %i sequences" % self.num_skipped, file=log.v2)
                 return None
             assert self.next_seq_idx == seq_idx, "We expect that we iterate through all seqs."
             true_idx = self.seq_order[self.next_orth_idx]
+            self._lazy_init()
             orth = self.orths[true_idx]  # get sequence for the next index given by seq_order
             seq_tag = self._tag_prefix + str(true_idx)
             self.next_orth_idx += 1

{returnn-1.20240723.153926 → returnn-1.20240723.171654}/returnn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240723.153926
+Version: 1.20240723.171654
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20240723.153926 → returnn-1.20240723.171654}/tests/test_Dataset.py

@@ -722,12 +722,15 @@ def test_LmDataset_pickle():
             }
         )
         assert isinstance(dataset, LmDataset)
+        assert dataset.orths is None  # not yet loaded, will be lazily loaded
 
         s = pickle.dumps(dataset)
         dataset = pickle.loads(s)
         assert isinstance(dataset, LmDataset)
+        assert dataset.orths is None  # not yet loaded, will be lazily loaded
 
         dataset.init_seq_order(epoch=1)
+        assert dataset.orths is not None  # loaded now
         dataset.load_seqs(0, 2)
         orth = dataset.get_data(0, "data")
         assert orth.tolist() == [1, 2]

returnn-1.20240723.153926/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240723.153926'
-long_version = '1.20240723.153926+git.7118c54'