returnn 1.20241223.212239__tar.gz → 1.20241224.5643__tar.gz

{returnn-1.20241223.212239/returnn.egg-info → returnn-1.20241224.5643}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241223.212239
+Version: 1.20241224.5643
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241224.5643/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20241224.005643'
+long_version = '1.20241224.005643+git.d5f6aa4'

{returnn-1.20241223.212239 → returnn-1.20241224.5643}/returnn/datasets/lm.py

@@ -67,6 +67,7 @@ class LmDataset(CachedDataset2):
         error_on_invalid_seq=True,
         add_delayed_seq_data=False,
         delayed_seq_data_start_symbol="[START]",
+        dtype: Optional[str] = None,
         **kwargs,
     ):
         """
@@ -117,6 +118,7 @@ class LmDataset(CachedDataset2):
         :param bool add_delayed_seq_data: will add another data-key "delayed" which will have the sequence.
           delayed_seq_data_start_symbol + original_sequence[:-1].
         :param str delayed_seq_data_start_symbol: used for add_delayed_seq_data.
+        :param dtype: explicit dtype. if not given, automatically determined based on the number of labels.
         """
         super(LmDataset, self).__init__(**kwargs)
 
@@ -245,7 +247,9 @@ class LmDataset(CachedDataset2):
             assert not orth_replace_map_file
 
         num_labels = len(self.labels["data"])
-        if num_labels <= 2**7:
+        if dtype:
+            self.dtype = dtype
+        elif num_labels <= 2**7:
             self.dtype = "int8"
         elif num_labels <= 2**8:
             self.dtype = "uint8"
@@ -347,8 +351,9 @@ class LmDataset(CachedDataset2):
                 last_print_time = time.time()
 
         # If a list of files is provided, concatenate all.
-        if not isinstance(corpus_file, list):
+        if isinstance(corpus_file, str):
             corpus_file = [corpus_file]
+        assert isinstance(corpus_file, (tuple, list))
         prev_orth_len = 0
         for file_name in corpus_file:
             if self._use_cache_manager:

{returnn-1.20241223.212239 → returnn-1.20241224.5643}/returnn/datasets/meta.py

@@ -1905,14 +1905,6 @@ class VariableDataset(Dataset):
         """current seq order"""
         return self._dataset.get_current_seq_order()
 
-    def get_all_tags(self) -> List[str]:
-        """all tags"""
-        return self._dataset.get_all_tags()
-
-    def get_total_num_seqs(self, *, fast: bool = False) -> int:
-        """total num seqs"""
-        return self._dataset.get_total_num_seqs(fast=fast)
-
     def get_seq_length(self, sorted_seq_idx: int) -> NumbersDict:
         """seq len"""
         return self._dataset.get_seq_length(sorted_seq_idx)

{returnn-1.20241223.212239 → returnn-1.20241224.5643/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241223.212239
+Version: 1.20241224.5643
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241223.212239/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20241223.212239'
-long_version = '1.20241223.212239+git.bfd6f5d'