returnn 1.20250318.214311__tar.gz → 1.20250404.233250__tar.gz

{returnn-1.20250318.214311/returnn.egg-info → returnn-1.20250404.233250}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250318.214311
+Version: 1.20250404.233250
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250404.233250/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20250404.233250'
+long_version = '1.20250404.233250+git.1aa10a4'

{returnn-1.20250318.214311 → returnn-1.20250404.233250}/returnn/datasets/generating.py

@@ -2037,7 +2037,7 @@ class LibriSpeechCorpus(CachedDataset2):
         """
         :param str path: dir, should contain "train-*/*/*/{*.flac,*.trans.txt}", or "train-*.zip"
         :param str prefix: "train", "dev", "test", "dev-clean", "dev-other", ...
-        :param str|list[str]|None orth_post_process: :func:`get_post_processor_function`, applied on orth
+        :param str|list[str]|function|None orth_post_process: :func:`get_post_processor_function`, applied on orth
         :param str|dict[str]|None targets: "bpe" or "chars" or None or dict for :func:`Vocabulary.create_vocab`
         :param dict[str]|None audio: options for :class:`ExtractAudioFeatures`
         :param dict[str]|None bpe: options for :class:`BytePairEncoding`

{returnn-1.20250318.214311 → returnn-1.20250404.233250}/returnn/datasets/lm.py

@@ -55,6 +55,7 @@ class LmDataset(CachedDataset2):
         orth_symbols_file=None,
         orth_symbols_map_file=None,
         orth_replace_map_file=None,
+        orth_post_process=None,
         word_based=False,
         word_end_symbol=None,
         seq_end_symbol="[END]",
@@ -101,6 +102,7 @@ class LmDataset(CachedDataset2):
                                                        a python dict with {"<symbol>": <index>, ...}
                                                        or a pickled dictionary
         :param str|()->str|None orth_replace_map_file: JSON file with replacement dict for orth symbols.
+        :param str|list[str]|function|None orth_post_process: :func:`get_post_processor_function`, applied on orth
         :param bool word_based: whether to parse single words, or otherwise will be character based.
         :param str|None word_end_symbol: If provided and if word_based is False (character based modeling),
             token to be used to represent word ends.
@@ -247,6 +249,10 @@ class LmDataset(CachedDataset2):
         else:
             assert not orth_replace_map_file
 
+        self.orth_post_process = None
+        if orth_post_process:
+            self.orth_post_process = get_post_processor_function(orth_post_process)
+
         num_labels = len(self.labels["data"])
         if dtype:
             self.dtype = dtype
@@ -578,6 +584,9 @@ class LmDataset(CachedDataset2):
                 seq_tag = self._seq_list[true_idx]
             self.next_orth_idx += 1
 
+            if self.orth_post_process:
+                orth = self.orth_post_process(orth)
+
             if self.orth_vocab is not None:
                 data = numpy.array(self.orth_vocab.get_seq(orth), dtype=self.dtype)
 
@@ -2421,7 +2430,7 @@ def get_post_processor_function(opts):
     for some normalization / cleanup.
     This function can be used to get such functions.
 
-    :param str|list[str] opts: e.g. "english_cleaners", or "get_remove_chars(',/')"
+    :param str|list[str]|function opts: e.g. "english_cleaners", or "get_remove_chars(',/')"
     :return: function
     :rtype: (str)->str
     """

{returnn-1.20250318.214311 → returnn-1.20250404.233250}/returnn/datasets/util/vocabulary.py

@@ -17,7 +17,6 @@ from typing import Optional, Union, Type, Callable, List, Dict
 import sys
 import numpy
 
-from returnn.log import log
 from returnn.util.basic import NotSpecified
 
 
@@ -157,7 +156,7 @@ class Vocabulary:
 
     def _parse_vocab(self):
         """
-        Sets self.vocab, self.labels, self.num_labels.
+        Sets self._vocab, self._labels, self.num_labels.
         """
         filename = self.vocab_file
         if self._labels is not None:
@@ -167,34 +166,41 @@ class Vocabulary:
             self._vocab, self._labels = self._cache[filename]
             self.num_labels = len(self._labels)
         else:
+            labels_from_idx = None
             if filename.endswith(".pkl"):
                 import pickle
 
-                d = pickle.load(open(filename, "rb"))
+                labels_to_idx = pickle.load(open(filename, "rb"))
             else:
                 if filename.endswith(".gz"):
                     import gzip
 
-                    file_content = gzip.open(filename, "rt").read()
+                    file_content = gzip.open(filename, "rt", encoding="utf8").read()
                 else:
-                    file_content = open(filename, "r").read()
+                    file_content = open(filename, "r", encoding="utf8").read()
                 if file_content.startswith("{"):
-                    d = eval(file_content)
+                    labels_to_idx = eval(file_content)
                 else:
                     # Do line-based parsing.
-                    lines = file_content.splitlines()
-                    d = {line: i for (i, line) in enumerate(lines)}
-            assert isinstance(d, dict), f"{self}: expected dict, got {type(d).__name__} in {filename}"
-            labels = {idx: label for (label, idx) in sorted(d.items())}
-            min_label, max_label, num_labels = min(labels), max(labels), len(labels)
-            assert 0 == min_label
-            if num_labels - 1 < max_label:
-                print("Vocab error: not all indices used? max label: %i" % max_label, file=log.v1)
-                print("unused labels: %r" % ([i for i in range(max_label + 1) if i not in labels],), file=log.v2)
-            assert num_labels - 1 == max_label
-            self.num_labels = len(labels)
-            self._vocab = d
-            self._labels = [label for (idx, label) in sorted(labels.items())]
+                    labels = file_content.splitlines()
+                    labels_from_idx = {i: line for (i, line) in enumerate(labels)}
+                    labels_to_idx = {line: i for (i, line) in enumerate(labels)}
+            assert isinstance(
+                labels_to_idx, dict
+            ), f"{self}: expected dict, got {type(labels_to_idx).__name__} in {filename}"
+            if labels_from_idx is None:
+                labels_from_idx = {idx: label for (label, idx) in sorted(labels_to_idx.items())}
+            min_label, max_label, num_labels = min(labels_from_idx), max(labels_from_idx), len(labels_from_idx)
+            if 0 != min_label or num_labels - 1 != max_label:
+                raise Exception(
+                    f"Vocab error: not all indices used? min label idx {min_label}, max label idx {max_label},"
+                    f" num labels {num_labels}, "
+                    f" unused labels: {[i for i in range(max_label + 1) if i not in labels_from_idx]}."
+                    "There are duplicates in the vocab."
+                )
+            self.num_labels = len(labels_from_idx)
+            self._vocab = labels_to_idx
+            self._labels = [label for (idx, label) in sorted(labels_from_idx.items())]
             self._cache[filename] = (self._vocab, self._labels)
 
     @classmethod

{returnn-1.20250318.214311 → returnn-1.20250404.233250}/returnn/torch/data/extern_data.py

@@ -132,7 +132,7 @@ def raw_dict_split_batch(
             raise TypeError(f"got invalid value of type ({type(v).__name__}) for key {k!r}")
         offset = 0
         for i, split_size in enumerate(splits):
-            res[i][k] = v[offset : offset + split_size]
+            res[i][k] = v[offset : offset + split_size] if v.ndim > 0 else v
             offset += split_size
     for res_ in res:
         res_: Dict[str, Union[torch.Tensor, numpy.ndarray]]

{returnn-1.20250318.214311 → returnn-1.20250404.233250}/returnn/util/basic.py

@@ -590,7 +590,7 @@ class ReportImportedDevModules:
             if path not in self.ignore_sys_path:
                 print("New sys.path entry:", path, file=log.v3)
                 has_changes = True
-        for mod_name, mod in sys.modules.items():
+        for mod_name, mod in list(sys.modules.items()):
             if "." not in mod_name and mod_name not in self.ignore_sys_modules:
                 if hasattr(mod, "__file__") and mod.__file__:
                     # __file__ is e.g. ".../recipe/i6_experiments/__init__.py"

{returnn-1.20250318.214311 → returnn-1.20250404.233250}/returnn/util/lru_cache.py

@@ -300,7 +300,7 @@ class _HashedSeq(list):
     the key multiple times on a cache miss.
     """
 
-    __slots__ = "hashvalue"
+    __slots__ = ("hashvalue",)
 
     def __init__(self, tup, *, _hash=hash):
         super().__init__(tup)

{returnn-1.20250318.214311 → returnn-1.20250404.233250/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250318.214311
+Version: 1.20250404.233250
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250318.214311/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20250318.214311'
-long_version = '1.20250318.214311+git.0d4a307'