returnn 1.20250117.155146__tar.gz → 1.20250117.212735__tar.gz

{returnn-1.20250117.155146/returnn.egg-info → returnn-1.20250117.212735}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250117.155146
+Version: 1.20250117.212735
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20250117.212735/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20250117.212735'
+long_version = '1.20250117.212735+git.7087556'

{returnn-1.20250117.155146 → returnn-1.20250117.212735}/returnn/datasets/util/vocabulary.py

@@ -13,8 +13,7 @@ __all__ = [
     "Utf8ByteTargets",
 ]
 
-from typing import Optional, Union, Type, List
-import typing
+from typing import Optional, Union, Type, Callable, List, Dict
 import sys
 import numpy
 
@@ -48,35 +47,71 @@ class Vocabulary:
 
     def __init__(
         self,
-        vocab_file,
-        seq_postfix=None,
-        unknown_label="UNK",
-        bos_label=None,
-        eos_label=None,
-        pad_label=None,
-        control_symbols=None,
-        user_defined_symbols=None,
-        num_labels=None,
-        labels=None,
+        vocab_file: Optional[str],
+        *,
+        special_symbols_via_file: Optional[str] = None,
+        unknown_label: Optional[Union[str, int]] = NotSpecified,
+        bos_label: Optional[Union[str, int]] = None,
+        eos_label: Optional[Union[str, int]] = None,
+        pad_label: Optional[Union[str, int]] = None,
+        control_symbols: Optional[Dict[str, Union[str, int]]] = None,
+        user_defined_symbols: Optional[Dict[str, Union[str, int]]] = None,
+        num_labels: Optional[int] = None,
+        seq_postfix: Optional[List[int]] = None,
+        labels: Optional[Union[List[str], Callable[[], List[str]]]] = None,
     ):
         """
-        :param str|None vocab_file:
-        :param str|int|None unknown_label: e.g. "UNK" or "<unk>"
-        :param str|int|None bos_label: e.g. "<s>"
-        :param str|int|None eos_label: e.g. "</s>"
-        :param str|int|None pad_label: e.g. "<pad>"
-        :param dict[str,str|int]|None control_symbols:
-          https://github.com/google/sentencepiece/blob/master/doc/special_symbols.md
-        :param dict[str,str|int]|None user_defined_symbols:
-          https://github.com/google/sentencepiece/blob/master/doc/special_symbols.md
-        :param int num_labels: just for verification
-        :param list[int]|None seq_postfix: labels will be added to the seq in self.get_seq
-        :param list[str]|(()->list[str])|None labels:
+        :param vocab_file:
+        :param special_symbols_via_file: if given, the file is supposed to contain a dict
+            with potential keys "unknown_label", "bos_label", "eos_label", "pad_label",
+            "control_symbols", "user_defined_symbols".
+            When label are specified directly as kwargs, those take precedence over any option in the file.
+        :param unknown_label: e.g. "UNK" or "<unk>"
+        :param bos_label: e.g. "<s>"
+        :param eos_label: e.g. "</s>"
+        :param pad_label: e.g. "<pad>"
+        :param control_symbols:
+            https://github.com/google/sentencepiece/blob/master/doc/special_symbols.md
+        :param user_defined_symbols:
+            https://github.com/google/sentencepiece/blob/master/doc/special_symbols.md
+        :param num_labels: just for verification
+        :param seq_postfix: labels will be added to the seq in self.get_seq
+        :param labels:
         """
         self.vocab_file = vocab_file
+
+        if special_symbols_via_file:
+            from ast import literal_eval
+
+            special_symbols = literal_eval(open(special_symbols_via_file).read())
+            assert isinstance(special_symbols, dict), f"expected dict, got {type(special_symbols).__name__}"
+            assert set(special_symbols.keys()).issubset(
+                {"unknown_label", "bos_label", "eos_label", "pad_label", "control_symbols", "user_defined_symbols"}
+            ), f"invalid special symbols {special_symbols}"
+            # When any of these opts is specified directly, that takes precedence over the file.
+            if unknown_label is NotSpecified:
+                unknown_label = special_symbols.get("unknown_label", NotSpecified)
+            if bos_label is None:
+                bos_label = special_symbols.get("bos_label", None)
+            if eos_label is None:
+                eos_label = special_symbols.get("eos_label", None)
+            if pad_label is None:
+                pad_label = special_symbols.get("pad_label", None)
+            if "control_symbols" in special_symbols:
+                control_symbols_ = control_symbols or {}
+                control_symbols: Dict[str, Union[str, int]] = special_symbols["control_symbols"].copy()
+                control_symbols.update(control_symbols_)
+            if "user_defined_symbols" in special_symbols:
+                user_defined_symbols_ = user_defined_symbols or {}
+                user_defined_symbols: Dict[str, Union[str, int]] = special_symbols["user_defined_symbols"].copy()
+                user_defined_symbols.update(user_defined_symbols_)
+
+        if unknown_label is NotSpecified:
+            # Unfortunately, this is the default, and we keep it for backward compatibility.
+            unknown_label = "UNK"
         self.unknown_label = unknown_label
-        self.num_labels = None  # type: typing.Optional[int]  # will be set by _parse_vocab
-        self._vocab = None  # type: typing.Optional[typing.Dict[str,int]]  # label->idx
+        self.num_labels: Optional[int] = None  # will be set by _parse_vocab
+        self._vocab: Optional[Dict[str, int]] = None  # label->idx
         if labels is not None and callable(labels):
             labels = labels()
         if labels is not None:

{returnn-1.20250117.155146 → returnn-1.20250117.212735}/returnn/frontend/parametrize.py

@@ -9,10 +9,9 @@ https://github.com/rwth-i6/returnn/issues/1518
 """
 
 from __future__ import annotations
-from typing import Optional, Union
+from typing import Optional, Union, Protocol
 import copyreg
 import weakref
-from returnn.util.py_compat import Protocol
 import returnn.frontend as rf
 from returnn.tensor import Tensor
 

{returnn-1.20250117.155146 → returnn-1.20250117.212735}/returnn/frontend/types.py

@@ -3,21 +3,12 @@ Types
 """
 
 from __future__ import annotations
-from typing import Union, Type, TYPE_CHECKING, Sequence
+from typing import Union, Type, TYPE_CHECKING, Protocol, Sequence
 import numpy
 from returnn.tensor import Tensor, Dim, TensorDict
 import returnn.frontend as rf
 
 
-try:
-    from typing import Protocol
-except ImportError:
-    try:
-        from typing_extensions import Protocol
-    except ImportError:
-        Protocol = object
-
-
 __all__ = ["RawTensorTypes", "ItemKeyType", "Tensor", "Dim", "get_raw_tensor_type", "GetModelFunc", "StepFunc"]
 
 

{returnn-1.20250117.155146 → returnn-1.20250117.212735}/returnn/tf/network.py

@@ -3,7 +3,7 @@ Defines the :class:`TFNetwork` and :class:`ExternData`.
 """
 
 from __future__ import annotations
-from typing import Optional, Any, List, Tuple, Dict
+from typing import Optional, Any, Protocol, List, Tuple, Dict
 import tensorflow as tf
 import sys
 import re
@@ -18,7 +18,6 @@ import returnn.tf.util.basic as tf_util
 from returnn.tensor import Tensor, Dim, TensorDict
 from returnn.tf.util.data import Data
 from returnn.util import basic as util
-from returnn.util.py_compat import Protocol
 
 
 class DataNotFound(Exception):

{returnn-1.20250117.155146 → returnn-1.20250117.212735/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20250117.155146
+Version: 1.20250117.212735
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

{returnn-1.20250117.155146 → returnn-1.20250117.212735}/returnn.egg-info/SOURCES.txt

@@ -328,7 +328,6 @@ returnn/util/multi_proc_non_daemonic_spawn.py
 returnn/util/native_code_compiler.py
 returnn/util/pprint.py
 returnn/util/py-to-pickle.cpp
-returnn/util/py_compat.py
 returnn/util/py_ext_mod_compiler.py
 returnn/util/result_with_reason.py
 returnn/util/sig_proc.py

returnn-1.20250117.155146/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20250117.155146'
-long_version = '1.20250117.155146+git.a41252b'

returnn-1.20250117.155146/returnn/util/py_compat.py

@@ -1,18 +0,0 @@
-"""
-Python compatibility utils
-"""
-
-from __future__ import annotations
-
-
-try:
-    from typing import Protocol
-except ImportError:
-    try:
-        from typing_extensions import Protocol
-    except ImportError:
-
-        class Protocol:
-            """dummy"""
-
-            pass