kitten-text-processing 0.1.0__py3-none-any.whl

kitten_text_processing/__init__.py

@@ -0,0 +1,5 @@
+"""Dependency-free multilingual text normalization for speech synthesis."""
+from .normalizer import Normalizer, PRESERVES_SENTINELS, SUPPORTED_LANGUAGES, normalize_text, warm
+
+__version__ = '0.1.0'
+__all__ = ['Normalizer', 'PRESERVES_SENTINELS', 'SUPPORTED_LANGUAGES', 'normalize_text', 'warm']

kitten_text_processing/__main__.py

@@ -0,0 +1,23 @@
+"""Command-line interface: python -m kitten_text_processing --lang es 'Tengo 2 gatos.'"""
+import argparse
+import sys
+from . import SUPPORTED_LANGUAGES, normalize_text
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Normalize written text for speech; no downloads or dependencies.')
+    parser.add_argument('text', nargs='?', help='Text to normalize; otherwise read stdin line by line')
+    parser.add_argument('--lang', default='en', help='Language code or locale (' + ', '.join(SUPPORTED_LANGUAGES) + ')')
+    args = parser.parse_args()
+    try:
+        if args.text is not None:
+            print(normalize_text(args.text, locale=args.lang))
+        else:
+            for line in sys.stdin:
+                print(normalize_text(line.rstrip('\n'), locale=args.lang))
+    except (ValueError, RuntimeError) as exc:
+        parser.exit(1, f'{exc}\n')
+
+
+if __name__ == '__main__':
+    main()

kitten_text_processing/_detokenize.py

@@ -0,0 +1,62 @@
+"""Moses-compatible punctuation handling for the supported TN languages.
+
+Reimplemented using only re and Unicode character properties. NeMo calls Moses
+with a single string in a list, which is then split on whitespace.
+"""
+import re
+from bisect import bisect_right
+from ._unicode_data import ALPHA_RANGES, CURRENCY_SYMBOLS
+
+_ALPHA_STARTS = tuple(a for a, _ in ALPHA_RANGES)
+
+def _alpha(character):
+    value = ord(character)
+    i = bisect_right(_ALPHA_STARTS, value) - 1
+    return i >= 0 and value <= ALPHA_RANGES[i][1]
+
+_CJK = ((4352, 4607), (11904, 42191), (43072, 43135), (44032, 55215),
+        (63744, 64255), (65072, 65103), (65381, 65500), (94208, 101119),
+        (110592, 110895), (110960, 111359), (131072, 196607))
+
+
+def _cjk(c):
+    return any(a <= ord(c) <= b for a, b in _CJK)
+
+
+def detokenize(text, lang):
+    tokens = re.sub(r' @-@ ', '-', ' ' + text + ' ').split()
+    quotes = {}
+    space, result = ' ', ''
+    for i, token in enumerate(tokens):
+        if _cjk(token[0]) and lang != 'ko':
+            result += ('' if i and _cjk(tokens[i-1][-1]) else space) + token
+            space = ' '
+        elif all(c in CURRENCY_SYMBOLS or c in '([{¿¡' for c in token):
+            result += space + token
+            space = ''
+        elif re.fullmatch(r'[,\.?!:;\\%}\])]+', token):
+            result += (' ' if lang == 'fr' and re.fullmatch(r'[?!:;\\%]', token) else '') + token
+            space = ' '
+        elif lang == 'en' and i and len(token) > 1 and token[0] == "'" and _alpha(token[1]):
+            result += token
+            space = ' '
+        elif (lang in ('fr', 'it') and i+1 < len(tokens) and len(token) > 1
+              and token[-1] == "'" and _alpha(token[-2]) and _alpha(tokens[i+1][0])):
+            result += space + token
+            space = ''
+        elif re.fullmatch('[\'"„“`]+', token):
+            key = '"' if all(c in '„“”' for c in token) else token
+            count = quotes.get(key, 0)
+            if count % 2 or (lang == 'en' and token == "'" and i and tokens[i-1].endswith('s')):
+                result += token
+                space = ' '
+                if count % 2:
+                    quotes[key] = count + 1
+            else:
+                result += space + token
+                space = ''
+                quotes[key] = count + 1
+        else:
+            result += space + token
+            space = ' '
+    return re.sub(' {2,}', ' ', result).strip()

kitten_text_processing/_fst.py

@@ -0,0 +1,304 @@
+"""Byte-level tropical weighted transducers, implemented with the standard library.
+
+Graph data is immutable. A frontier decoder handles unambiguous rewrites; tied
+outputs use a trimmed lattice and OpenFst-compatible topological ordering.
+Weights are rounded to float32 at each addition, as in the source grammars.
+No beam pruning or vocabulary approximations are used.
+"""
+from array import array
+from collections import deque
+import gzip
+import math
+import struct
+import sys
+
+
+class NoPathError(ValueError):
+    """The grammar has no accepting path for this input."""
+
+
+class Graph:
+    def __init__(self, path, *, project_output=False):
+        self.project_output = project_output
+        with gzip.open(path, 'rb') as source:
+            magic, self.start, states, arcs = struct.unpack('<8sIII', source.read(20))
+            if magic != b'KITTEN1\0':
+                raise ValueError(f'Invalid grammar: {path}')
+            for name, kind, size in [('offsets', 'I', states + 1), ('finals', 'f', states),
+                                     ('labels', 'I', arcs), ('targets', 'I', arcs), ('weights', 'f', arcs)]:
+                values = array(kind)
+                values.frombytes(source.read(size * 4))
+                if len(values) != size:
+                    raise ValueError(f'Truncated grammar: {path}')
+                if sys.byteorder != 'little':
+                    values.byteswap()
+                setattr(self, name, values)
+        self._index = {}
+
+    def arcs(self, state):
+        index = self._index.get(state)
+        if index is None:
+            incoming = getattr(self, '_indegree', None)
+            if incoming is None:
+                incoming = bytearray(len(self.finals))
+                for target in self.targets:
+                    if incoming[target] < 2:
+                        incoming[target] += 1
+                self._indegree = incoming
+            index = {}
+            for i in range(self.offsets[state], self.offsets[state + 1]):
+                label = self.labels[i]
+                target = self.targets[i]
+                suffix = bytearray([label >> 8]) if label >> 8 else bytearray()
+                # Contract deterministic zero-cost output-only chains. Do not
+                # merge nonzero weights: float32 addition is not associative.
+                # Never contract joins: their order resolves equal-weight paths.
+                visited = set()
+                while (not getattr(self, 'project_output', False) and incoming[target] == 1
+                       and not math.isfinite(self.finals[target]) and target not in visited):
+                    start, end = self.offsets[target], self.offsets[target + 1]
+                    if end != start + 1 or self.labels[start] & 255 or self.weights[start]:
+                        break
+                    # Keep the predecessor of a join. Contracting it could
+                    # turn separate paths into parallel arcs and reverse the
+                    # equal-weight winner (arc order versus topological order).
+                    if incoming[self.targets[start]] != 1:
+                        break
+                    visited.add(target)
+                    output = self.labels[start] >> 8
+                    if output:
+                        suffix.append(output)
+                    target = self.targets[start]
+                index.setdefault(label & 255, []).append((target, self.weights[i], bytes(suffix)))
+            # Bound auxiliary memory; graph arrays remain the source of truth.
+            if len(self._index) >= 50000:
+                self._index.clear()
+            self._index[state] = index
+        return index
+
+    def rewrite(self, text):
+        # Most inputs have one best output. Keep only the current input frontier
+        # there; build a full lattice only when equal-cost output strings differ.
+        # Pynini compiles NUL bytes as epsilon labels, so they consume no input.
+        text = text.replace('\x00', '')
+        if getattr(self, 'project_output', False):
+            return self._rewrite_ordered(text)
+        active = {self.start: (0.0, b'', False)}
+        pack, unpack = struct.Struct('<f').pack, struct.Struct('<f').unpack
+        for char in (*text.encode('utf-8'), None):
+            queue = deque(active)
+            queued = set(active)
+            while queue:
+                state = queue.popleft()
+                queued.discard(state)
+                cost, output, ambiguous = active[state]
+                for target, weight, suffix in self.arcs(state).get(0, ()):
+                    candidate = unpack(pack(cost + weight))[0] if weight else cost
+                    old = active.get(target)
+                    value = output + suffix
+                    if old is None or candidate < old[0]:
+                        active[target] = (candidate, value, ambiguous)
+                    elif candidate == old[0] and not old[2] and (ambiguous or value != old[1]):
+                        active[target] = (old[0], old[1], True)
+                    else:
+                        continue
+                    if target not in queued:
+                        queue.append(target)
+                        queued.add(target)
+            if char is None:
+                best = None
+                for state, (cost, output, ambiguous) in active.items():
+                    final = self.finals[state]
+                    if not math.isfinite(final):
+                        continue
+                    candidate = unpack(pack(cost + final))[0]
+                    if best is None or candidate < best[0]:
+                        best = (candidate, output, ambiguous)
+                    elif candidate == best[0] and (ambiguous or output != best[1]):
+                        best = (best[0], best[1], True)
+                if best is None:
+                    raise NoPathError(f'No accepting path for {text[:80]!r}')
+                if best[2]:
+                    return self._rewrite_ordered(text)
+                return best[1].decode('utf-8')
+            following = {}
+            for state, (cost, output, ambiguous) in active.items():
+                for target, weight, suffix in self.arcs(state).get(char, ()):
+                    candidate = unpack(pack(cost + weight))[0] if weight else cost
+                    old = following.get(target)
+                    value = output + suffix
+                    if old is None or candidate < old[0]:
+                        following[target] = (candidate, value, ambiguous)
+                    elif candidate == old[0] and not old[2] and (ambiguous or value != old[1]):
+                        following[target] = (old[0], old[1], True)
+            if not following:
+                raise NoPathError(f'No accepting path for {text[:80]!r}')
+            active = following
+
+    def _rewrite_ordered(self, text):
+        data = text.encode('utf-8')
+        length = len(data)
+        start = (0, self.start)
+        adjacency = {}
+
+        def edges(node):
+            position, state = node
+            arcs = self.arcs(state)
+            out = [((position, target), weight, suffix) for target, weight, suffix in arcs.get(0, ())]
+            if position < length:
+                out.extend(((position + 1, target), weight, suffix)
+                           for target, weight, suffix in arcs.get(data[position], ()))
+            adjacency[node] = out
+            return out
+
+        # Reverse DFS finishing order matches OpenFst's topological queue. In
+        # particular, the order of equal-weight readings is not arbitrary.
+        seen = {start}
+        visiting = {start}
+        stack = [(start, iter(edges(start)))]
+        ordered = []
+        live = set()
+        cyclic = False
+        while stack:
+            node, iterator = stack[-1]
+            edge = next(iterator, None)
+            if edge is None:
+                ordered.append(node)
+                if ((node[0] == length and math.isfinite(self.finals[node[1]]))
+                        or any(target in live for target, _, _ in adjacency[node])):
+                    live.add(node)
+                visiting.remove(node)
+                stack.pop()
+                continue
+            target = edge[0]
+            if target in visiting:
+                cyclic = True
+            if target not in seen:
+                seen.add(target)
+                visiting.add(target)
+                stack.append((target, iter(edges(target))))
+        # Composition in OpenFst trims states that cannot reach a final state.
+        # Some grammars contain dead epsilon cycles, so trim before decoding.
+        if cyclic:
+            # Back edges can hide productive exits until their SCC is complete.
+            # Finish liveness by reverse reachability only for cyclic lattices.
+            reverse = {}
+            for node, outgoing in adjacency.items():
+                for target, _, _ in outgoing:
+                    reverse.setdefault(target, []).append(node)
+            pending = list(live)
+            while pending:
+                for parent in reverse.get(pending.pop(), ()):
+                    if parent not in live:
+                        live.add(parent)
+                        pending.append(parent)
+        if start not in live:
+            raise NoPathError(f'No accepting path for {text[:80]!r}')
+        final_weights = {node: self.finals[node[1]] for node in live if node[0] == length}
+        if getattr(self, 'project_output', False):
+            adjacency, final_weights, ordered = self._project_lattice(adjacency, final_weights, ordered, live, start)
+            live = set(ordered)
+        costs = {start: 0.0}
+        parents = {}
+        best, final = float('inf'), None
+        pack, unpack = struct.Struct('<f').pack, struct.Struct('<f').unpack
+        pending = deque(node for node in reversed(ordered) if node in live)
+        queued = set(pending)
+        while pending:
+            node = pending.popleft()
+            queued.remove(node)
+            cost = costs.get(node, float('inf'))
+            if not math.isfinite(cost):
+                continue
+            final_weight = final_weights.get(node, math.inf)
+            if math.isfinite(final_weight):
+                candidate = unpack(pack(cost + final_weight))[0]
+                if candidate < best:
+                    best, final = candidate, node
+            for target, weight, suffix in adjacency[node]:
+                if target not in live:
+                    continue
+                candidate = unpack(pack(cost + weight))[0] if weight else cost
+                if candidate < costs.get(target, float('inf')):
+                    costs[target] = candidate
+                    parents[target] = (node, suffix)
+                    # Usually the lattice is a DAG. Requeue an already visited
+                    # state if a live epsilon cycle improves its distance.
+                    if target not in queued:
+                        pending.append(target)
+                        queued.add(target)
+        if final is None:
+            raise NoPathError(f'No accepting path for {text[:80]!r}')
+        result = []
+        while final != start:
+            final, suffix = parents[final]
+            result.append(suffix)
+        return b''.join(reversed(result)).decode('utf-8')
+
+    @staticmethod
+    def _project_lattice(adjacency, finals, ordered, live, start):
+        """Output projection and epsilon removal used by Pynini top_rewrite.
+
+        The insertion order and reversal of expanded arcs follow OpenFst's
+        RmEpsilon. Calling shortestpath on the unprojected transducer resolves
+        equally weighted punctuation rewrites differently.
+        """
+        pack, unpack = struct.Struct('<f').pack, struct.Struct('<f').unpack
+        def add(a, b):
+            return unpack(pack(a + b))[0] if b else a
+        incoming = {start}
+        for node in live:
+            for target, _, output in adjacency[node]:
+                if output and target in live:
+                    incoming.add(target)
+        for source in ordered:
+            if source not in live or source not in incoming:
+                continue
+            distances = {source: 0.0}
+            pending = deque([source])
+            queued = {source}
+            while pending:
+                node = pending.popleft()
+                queued.remove(node)
+                for target, weight, output in adjacency[node]:
+                    if output or target not in live:
+                        continue
+                    distance = add(distances[node], weight)
+                    if distance < distances.get(target, math.inf):
+                        distances[target] = distance
+                        if target not in queued:
+                            pending.append(target)
+                            queued.add(target)
+            expanded = {}
+            final = math.inf
+            stack, seen = [source], set()
+            while stack:
+                node = stack.pop()
+                if node in seen:
+                    continue
+                seen.add(node)
+                distance = distances[node]
+                final = min(final, add(distance, finals.get(node, math.inf)))
+                for target, weight, output in adjacency[node]:
+                    if target not in live:
+                        continue
+                    if not output:
+                        if target not in seen:
+                            stack.append(target)
+                    else:
+                        key = (target, output)
+                        expanded[key] = min(expanded.get(key, math.inf), add(distance, weight))
+            adjacency[source] = [(target, weight, output) for (target, output), weight in reversed(expanded.items())]
+            finals[source] = final
+        seen, order = {start}, []
+        stack = [(start, iter(adjacency[start]))]
+        while stack:
+            node, arcs = stack[-1]
+            arc = next(arcs, None)
+            if arc is None:
+                order.append(node)
+                stack.pop()
+            elif arc[0] not in seen:
+                seen.add(arc[0])
+                stack.append((arc[0], iter(adjacency[arc[0]])))
+        return adjacency, finals, order

kitten_text_processing/_ordering.py

@@ -0,0 +1,126 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Adapted from NeMo 1.2.0 normalize.py; only token ordering is retained.
+import itertools
+from collections import OrderedDict
+from math import factorial
+from typing import Dict, List, Union
+from ._token_parser import PRESERVE_ORDER_KEY
+
+class TokenOrdering:
+    def _estimate_number_of_permutations_in_nested_dict(
+        self, token_group: Dict[str, Union[OrderedDict, str, bool]]
+    ) -> int:
+        num_perms = 1
+        for k, inner in token_group.items():
+            if isinstance(inner, dict):
+                num_perms *= self._estimate_number_of_permutations_in_nested_dict(inner)
+        num_perms *= factorial(len(token_group))
+        return num_perms
+
+    def _split_tokens_to_reduce_number_of_permutations(self, tokens: List[dict]) -> List[List[dict]]:
+        """
+        Splits a sequence of tokens in a smaller sequences of tokens in a way that maximum number of composite
+        tokens permutations does not exceed ``max_number_of_permutations_per_split``.
+
+        For example,
+
+        .. code-block:: python
+
+            # setup normalizer with self.max_number_of_permutations_per_split=6
+             tokens = [
+                {"tokens": {"date": {"year": "twenty eighteen", "month": "december", "day": "thirty one"}}},
+                {"tokens": {"date": {"year": "twenty eighteen", "month": "january", "day": "eight"}}},
+            ]
+            split = normalizer._split_tokens_to_reduce_number_of_permutations(tokens)
+            assert split == [
+                [{"tokens": {"date": {"year": "twenty eighteen", "month": "december", "day": "thirty one"}}}],
+                [{"tokens": {"date": {"year": "twenty eighteen", "month": "january", "day": "eight"}}}],
+            ]
+
+        Date tokens contain 3 items each which gives 6 permutations for every date. Since there are 2 dates, total
+        number of permutations would be ``6 * 6 == 36``. Parameter ``self.max_number_of_permutations_per_split`` equals 6,
+        so input sequence of tokens is split into 2 smaller sequences.
+
+        Args:
+            tokens: a list of dictionaries, possibly nested.
+
+        Returns:
+            a list of smaller sequences of tokens resulting from ``tokens`` split.
+        """
+        splits = []
+        prev_end_of_split = 0
+        current_number_of_permutations = 1
+        for i, token_group in enumerate(tokens):
+            n = self._estimate_number_of_permutations_in_nested_dict(token_group)
+            if n * current_number_of_permutations > self.max_number_of_permutations_per_split:
+                splits.append(tokens[prev_end_of_split:i])
+                prev_end_of_split = i
+                current_number_of_permutations = 1
+            if n > self.max_number_of_permutations_per_split:
+                raise ValueError(
+                    f"Could not split token list with respect to condition that every split can generate number of "
+                    f"permutations less or equal to "
+                    f"`self.max_number_of_permutations_per_split={self.max_number_of_permutations_per_split}`. "
+                    f"There is an unsplittable token group that generates more than "
+                    f"{self.max_number_of_permutations_per_split} permutations. Try to increase "
+                    f"`--max_number_of_permutations_per_split` parameter."
+                )
+            current_number_of_permutations *= n
+        splits.append(tokens[prev_end_of_split:])
+        assert sum([len(s) for s in splits]) == len(tokens)
+        return splits
+
+    def _permute(self, d: OrderedDict) -> List[str]:
+        """
+        Creates reorderings of dictionary elements and serializes as strings
+
+        Args:
+            d: (nested) dictionary of key value pairs
+
+        Return permutations of different string serializations of key value pairs
+        """
+        l = []
+        if PRESERVE_ORDER_KEY in d.keys():
+            d_permutations = [d.items()]
+        else:
+            d_permutations = itertools.permutations(d.items())
+        for perm in d_permutations:
+            subl = [""]
+            for k, v in perm:
+                if isinstance(v, str):
+                    subl = ["".join(x) for x in itertools.product(subl, [f"{k}: \"{v}\" "])]
+                elif isinstance(v, OrderedDict):
+                    rec = self._permute(v)
+                    subl = ["".join(x) for x in itertools.product(subl, [f" {k} {{ "], rec, [f" }} "])]
+                elif isinstance(v, bool):
+                    subl = ["".join(x) for x in itertools.product(subl, [f"{k}: true "])]
+                else:
+                    raise ValueError("Key: " + str(k) + " Value: " + str(v))
+            l.extend(subl)
+        return l
+
+    def generate_permutations(self, tokens: List[dict]):
+        """
+        Generates permutations of string serializations of list of dictionaries
+
+        Args:
+            tokens: list of dictionaries
+
+        Returns string serialization of list of dictionaries
+        """
+
+        # itertools.product avoids recursion proportional to sentence length.
+        # Each split has a bounded product of options, as in the upstream API.
+        return ("".join(parts) for parts in itertools.product(*(self._permute(t) for t in tokens)))

kitten_text_processing/_punctuation.py

@@ -0,0 +1,112 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Adapted from NeMo 1.2.0 data_loader_utils.py; punctuation helpers only.
+import re
+import string
+import sys
+from unicodedata import category
+import logging
+logger = logging.getLogger(__name__)
+
+def pre_process(text: str) -> str:
+    """
+    Optional text preprocessing before normalization (part of TTS TN pipeline)
+
+    Args:
+        text: string that may include semiotic classes
+
+    Returns: text with spaces around punctuation marks
+    """
+    space_both = '[]'
+    for punct in space_both:
+        text = text.replace(punct, ' ' + punct + ' ')
+
+    # remove extra space
+    text = re.sub(r' +', ' ', text)
+    return text
+
+def post_process_punct(input: str, normalized_text: str, add_unicode_punct: bool = False):
+    """
+    Post-processing of the normalized output to match input in terms of spaces around punctuation marks.
+    After NN normalization, Moses detokenization puts a space after
+    punctuation marks, and attaches an opening quote "'" to the word to the right.
+    E.g., input to the TN NN model is "12 test' example",
+    after normalization and detokenization -> "twelve test 'example" (the quote is considered to be an opening quote,
+    but it doesn't match the input and can cause issues during TTS voice generation.)
+    The current function will match the punctuation and spaces of the normalized text with the input sequence.
+    "12 test' example" -> "twelve test 'example" -> "twelve test' example" (the quote was shifted to match the input).
+
+    Args:
+        input: input text (original input to the NN, before normalization or tokenization)
+        normalized_text: output text (output of the TN NN model)
+        add_unicode_punct: set to True to handle unicode punctuation marks as well as default string.punctuation (increases post processing time)
+    """
+    # in the post-processing WFST graph "``" are repalced with '"" quotes (otherwise single quotes "`" won't be handled correctly)
+    # this function fixes spaces around them based on input sequence, so here we're making the same double quote replacement
+    # to make sure these new double quotes work with this function
+    if "``" in input and "``" not in normalized_text:
+        input = input.replace("``", '"')
+    input = [x for x in input]
+    normalized_text = [x for x in normalized_text]
+    punct_marks = [x for x in string.punctuation if x in input]
+
+    if add_unicode_punct:
+        punct_unicode = [
+            chr(i)
+            for i in range(sys.maxunicode)
+            if category(chr(i)).startswith("P") and chr(i) not in punct_marks and chr(i) in input
+        ]
+        punct_marks = punct_marks.extend(punct_unicode)
+
+    for punct in punct_marks:
+        try:
+            equal = True
+            if input.count(punct) != normalized_text.count(punct):
+                equal = False
+            idx_in, idx_out = 0, 0
+            while punct in input[idx_in:]:
+                idx_out = normalized_text.index(punct, idx_out)
+                idx_in = input.index(punct, idx_in)
+
+                def _is_valid(idx_out, idx_in, normalized_text, input):
+                    """Check if previous or next word match (for cases when punctuation marks are part of
+                    semiotic token, i.e. some punctuation can be missing in the normalized text)"""
+                    return (idx_out > 0 and idx_in > 0 and normalized_text[idx_out - 1] == input[idx_in - 1]) or (
+                        idx_out < len(normalized_text) - 1
+                        and idx_in < len(input) - 1
+                        and normalized_text[idx_out + 1] == input[idx_in + 1]
+                    )
+
+                if not equal and not _is_valid(idx_out, idx_in, normalized_text, input):
+                    idx_in += 1
+                    continue
+                if idx_in > 0 and idx_out > 0:
+                    if normalized_text[idx_out - 1] == " " and input[idx_in - 1] != " ":
+                        normalized_text[idx_out - 1] = ""
+
+                    elif normalized_text[idx_out - 1] != " " and input[idx_in - 1] == " ":
+                        normalized_text[idx_out - 1] += " "
+
+                if idx_in < len(input) - 1 and idx_out < len(normalized_text) - 1:
+                    if normalized_text[idx_out + 1] == " " and input[idx_in + 1] != " ":
+                        normalized_text[idx_out + 1] = ""
+                    elif normalized_text[idx_out + 1] != " " and input[idx_in + 1] == " ":
+                        normalized_text[idx_out] = normalized_text[idx_out] + " "
+                idx_out += 1
+                idx_in += 1
+        except:
+            logger.info(f"Skipping post-processing of {''.join(normalized_text)} for '{punct}'")
+
+    normalized_text = "".join(normalized_text)
+    return re.sub(r' +', ' ', normalized_text)