returnn 1.20241223.212239__tar.gz → 1.20241224.13515__tar.gz

{returnn-1.20241223.212239/returnn.egg-info → returnn-1.20241224.13515}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241223.212239
+Version: 1.20241224.13515
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20241224.13515/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20241224.013515'
+long_version = '1.20241224.013515+git.ed5db53'

{returnn-1.20241223.212239 → returnn-1.20241224.13515}/returnn/datasets/lm.py

@@ -67,6 +67,7 @@ class LmDataset(CachedDataset2):
         error_on_invalid_seq=True,
         add_delayed_seq_data=False,
         delayed_seq_data_start_symbol="[START]",
+        dtype: Optional[str] = None,
         **kwargs,
     ):
         """
@@ -117,6 +118,7 @@ class LmDataset(CachedDataset2):
         :param bool add_delayed_seq_data: will add another data-key "delayed" which will have the sequence.
           delayed_seq_data_start_symbol + original_sequence[:-1].
         :param str delayed_seq_data_start_symbol: used for add_delayed_seq_data.
+        :param dtype: explicit dtype. if not given, automatically determined based on the number of labels.
         """
         super(LmDataset, self).__init__(**kwargs)
 
@@ -245,7 +247,9 @@ class LmDataset(CachedDataset2):
             assert not orth_replace_map_file
 
         num_labels = len(self.labels["data"])
-        if num_labels <= 2**7:
+        if dtype:
+            self.dtype = dtype
+        elif num_labels <= 2**7:
             self.dtype = "int8"
         elif num_labels <= 2**8:
             self.dtype = "uint8"
@@ -347,8 +351,9 @@ class LmDataset(CachedDataset2):
                 last_print_time = time.time()
 
         # If a list of files is provided, concatenate all.
-        if not isinstance(corpus_file, list):
+        if isinstance(corpus_file, str):
             corpus_file = [corpus_file]
+        assert isinstance(corpus_file, (tuple, list))
         prev_orth_len = 0
         for file_name in corpus_file:
             if self._use_cache_manager:

{returnn-1.20241223.212239 → returnn-1.20241224.13515}/returnn/datasets/meta.py

@@ -1905,14 +1905,6 @@ class VariableDataset(Dataset):
         """current seq order"""
         return self._dataset.get_current_seq_order()
 
-    def get_all_tags(self) -> List[str]:
-        """all tags"""
-        return self._dataset.get_all_tags()
-
-    def get_total_num_seqs(self, *, fast: bool = False) -> int:
-        """total num seqs"""
-        return self._dataset.get_total_num_seqs(fast=fast)
-
     def get_seq_length(self, sorted_seq_idx: int) -> NumbersDict:
         """seq len"""
         return self._dataset.get_seq_length(sorted_seq_idx)

{returnn-1.20241223.212239 → returnn-1.20241224.13515/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20241223.212239
+Version: 1.20241224.13515
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20241223.212239 → returnn-1.20241224.13515}/tools/dump-dataset.py

@@ -53,7 +53,9 @@ def dump_dataset(options):
         ", ".join(f"{k!r}: {v[:3]}... len {len(v)}" for k, v in dataset.labels.items()) or "None",
         file=log.v3,
     )
-    assert options.key in dataset.get_data_keys()
+    assert (
+        options.key in dataset.get_data_keys()
+    ), f"key {options.key!r} not in {dataset.get_data_keys()} (targets {dataset.get_target_list()})"
     max_seq_length = NumbersDict(options.max_seq_length)
     min_seq_length = NumbersDict(options.min_seq_length)
 

returnn-1.20241223.212239/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20241223.212239'
-long_version = '1.20241223.212239+git.bfd6f5d'