returnn 1.20240619.184216__tar.gz → 1.20240619.234159__tar.gz

{returnn-1.20240619.184216/returnn.egg-info → returnn-1.20240619.234159}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240619.184216
+Version: 1.20240619.234159
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240619.234159/_setup_info_generated.py

@@ -0,0 +1,2 @@
+version = '1.20240619.234159'
+long_version = '1.20240619.234159+git.91aabfb'

{returnn-1.20240619.184216 → returnn-1.20240619.234159}/returnn/util/bpe.py

@@ -3,8 +3,7 @@ Provide basic Byte-Pair-Encoding (BPE) utilities.
 """
 
 from __future__ import annotations
-from typing import Dict
-import typing
+from typing import Optional, List, Dict, Callable
 import re
 import numpy
 
@@ -234,10 +233,10 @@ class PrefixTree:
     This class represents both a single node and the tree.
     """
 
-    def __init__(self, prefix="", root=None):
+    def __init__(self, prefix: str = "", root: Optional[PrefixTree] = None):
         """
-        :param str prefix:
-        :param PrefixTree|None root:
+        :param prefix:
+        :param root:
         """
         self.prefix = prefix
         self.arcs: Dict[str, PrefixTree] = {}  # single char (or BpePostMergeSymbol) -> sub tree
@@ -246,11 +245,10 @@ class PrefixTree:
         self.is_root = not root
         self.root = root
 
-    def add(self, postfix, root=None):
+    def add(self, postfix: str, root: Optional[PrefixTree] = None) -> PrefixTree:
         """
-        :param str postfix:
-        :param None|PrefixTree root:
-        :rtype: PrefixTree
+        :param postfix:
+        :param root:
         """
         if not root:
             if self.is_root:
@@ -300,21 +298,21 @@ class CharSyncSearch:
     Covers the search hyps and the search itself.
     """
 
-    def __init__(self, bpe, word, word_pos=0):
+    def __init__(self, bpe: PrefixTree, word: str, word_pos: int = 0):
         """
-        :param PrefixTree bpe:
-        :param str word:
-        :param int word_pos:
+        :param bpe:
+        :param word:
+        :param word_pos:
         """
         self.bpe = bpe
         self.word = word
         self.word_pos = word_pos
-        self.hyps = [Hyp(bpe_sym_history=[], cur_node=bpe)]  # type: typing.List[Hyp]
-        self.final_bpe_seqs = None  # type: typing.Optional[typing.List[typing.List[str]]]
+        self.hyps: List[Hyp] = [Hyp(bpe_sym_history=[], cur_node=bpe)]
+        self.final_bpe_seqs: Optional[List[List[str]]] = None
 
     def _get_finished(self):
         assert self.word_pos == len(self.word)
-        finals = []  # type: typing.List[typing.List[str]]
+        finals: List[List[str]] = []
         for hyp in self.hyps:
             if hyp.cur_node.finished:
                 finals.append(hyp.bpe_sym_history + [hyp.cur_node.prefix])
@@ -323,7 +321,7 @@ class CharSyncSearch:
     def _expand(self):
         assert self.word_pos < len(self.word)
         char = self.word[self.word_pos]
-        new_hyps = []  # type: typing.List[Hyp]
+        new_hyps: List[Hyp] = []
         for hyp in self.hyps:
             if hyp.cur_node.bpe_finished:
                 # Start again from root node.
@@ -340,10 +338,9 @@ class CharSyncSearch:
                 new_hyps.append(Hyp(bpe_sym_history=hyp.bpe_sym_history, cur_node=next_node))
         self.hyps = new_hyps
 
-    def search(self):
+    def search(self) -> List[List[str]]:
         """
         :return: collection of possible BPE symbol seqs
-        :rtype: list[list[str]]
         """
         while self.word_pos < len(self.word):
             self._expand()
@@ -373,23 +370,20 @@ class DepthFirstSearch:
     Depth-first search.
     """
 
-    def __init__(self, bpe, word, sampler=None):
+    def __init__(self, bpe: PrefixTree, word: str, sampler: Optional[Callable[[], bool]] = None):
         """
-        :param PrefixTree bpe:
-        :param str word:
-        :param (()->bool)|None sampler:
+        :param bpe:
+        :param word:
+        :param sampler:
         """
         self.bpe = bpe
         self.word = word
         self.sampler = sampler
-        self.hyps = []  # type: typing.List[HypInPos]
-        self.final_bpe_seq = None  # type: typing.Optional[typing.List[str]]
+        self.hyps: List[HypInPos] = []
+        self.final_bpe_seq: Optional[List[str]] = None
         self._add_hyp(HypInPos(bpe_sym_history=[], cur_node=bpe, pos=0))
 
-    def _add_hyp(self, hyp):
-        """
-        :param HypInPos hyp:
-        """
+    def _add_hyp(self, hyp: HypInPos):
         if hyp.pos >= len(self.word):
             if hyp.cur_node.finished:
                 self.final_bpe_seq = hyp.bpe_sym_history + [hyp.cur_node.prefix]
@@ -401,7 +395,7 @@ class DepthFirstSearch:
 
         # Now check for possible further hyps.
         char = self.word[hyp.pos]
-        new_hyps = []  # type: typing.List[HypInPos]
+        new_hyps: List[HypInPos] = []
         if hyp.cur_node.bpe_finished:
             # Start again from root node.
             next_node = self.bpe.arcs.get(char)
@@ -423,10 +417,9 @@ class DepthFirstSearch:
         for hyp in new_hyps:
             self._add_hyp(hyp)
 
-    def search(self):
+    def search(self) -> Optional[List[str]]:
         """
         :return: BPE symbol seq if one is found
-        :rtype: list[str]|None
         """
         while self.hyps and self.final_bpe_seq is None:
             self._expand()
@@ -459,23 +452,21 @@ class SamplingBytePairEncoder:
             bpe.add(bpe_sym)
         self._bpe_prefix_tree = bpe
 
-    def _sampler(self):
+    def _sampler(self) -> bool:
         # When this returns true, it will differ from depth-first search.
         return self.rnd.random_sample() <= self.breadth_prob
 
-    def get_bpe_split_for_word(self, word):
+    def get_bpe_split_for_word(self, word: str) -> Optional[List[str]]:
         """
-        :param str word:
-        :rtype: list[str]|None
+        :param word:
         """
         return DepthFirstSearch(bpe=self._bpe_prefix_tree, word=word, sampler=self._sampler).search()
 
-    def segment_sentence(self, sentence):
+    def segment_sentence(self, sentence: str) -> List[str]:
         """
         Segment single sentence (whitespace-tokenized string) with BPE encoding.
 
-        :param str sentence:
-        :rtype: list[str]
+        :param sentence:
         """
         output = []
         for word in sentence.split():

{returnn-1.20240619.184216 → returnn-1.20240619.234159/returnn.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: returnn
-Version: 1.20240619.184216
+Version: 1.20240619.234159
 Summary: The RWTH extensible training framework for universal recurrent neural networks
 Home-page: https://github.com/rwth-i6/returnn/
 Author: Albert Zeyer

returnn-1.20240619.184216/_setup_info_generated.py

@@ -1,2 +0,0 @@
-version = '1.20240619.184216'
-long_version = '1.20240619.184216+git.bc0e997'