WeTextProcessing 0.1.12__tar.gz → 0.2.1__tar.gz

{WeTextProcessing-0.1.12 → WeTextProcessing-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: WeTextProcessing
-Version: 0.1.12
+Version: 0.2.1
 Summary: WeTextProcessing, including TN & ITN
 Home-page: https://github.com/wenet-e2e/WeTextProcessing
 Author: Zhendong Peng, Xingchen Song

{WeTextProcessing-0.1.12 → WeTextProcessing-0.2.1}/WeTextProcessing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: WeTextProcessing
-Version: 0.1.12
+Version: 0.2.1
 Summary: WeTextProcessing, including TN & ITN
 Home-page: https://github.com/wenet-e2e/WeTextProcessing
 Author: Zhendong Peng, Xingchen Song

{WeTextProcessing-0.1.12 → WeTextProcessing-0.2.1}/WeTextProcessing.egg-info/SOURCES.txt

@@ -31,9 +31,12 @@ itn/chinese/test/normalizer_test.py
 itn/chinese/test/utils.py
 tn/__init__.py
 tn/__main__.py
+tn/en_tn_tagger.fst
+tn/en_tn_verbalizer.fst
 tn/main.py
 tn/processor.py
 tn/token_parser.py
+tn/utils.py
 tn/zh_tn_tagger.fst
 tn/zh_tn_verbalizer.fst
 tn/chinese/__init__.py
@@ -66,4 +69,49 @@ tn/chinese/test/sport_test.py
 tn/chinese/test/time_test.py
 tn/chinese/test/token_parser_test.py
 tn/chinese/test/utils.py
-tn/chinese/test/whitelist_test.py
+tn/chinese/test/whitelist_test.py
+tn/english/__init__.py
+tn/english/normalizer.py
+tn/english/data/__init__.py
+tn/english/data/address/__init__.py
+tn/english/data/date/__init__.py
+tn/english/data/electronic/__init__.py
+tn/english/data/measure/__init__.py
+tn/english/data/money/__init__.py
+tn/english/data/number/__init__.py
+tn/english/data/ordinal/__init__.py
+tn/english/data/roman/__init__.py
+tn/english/data/telephone/__init__.py
+tn/english/data/time/__init__.py
+tn/english/data/whitelist/__init__.py
+tn/english/rules/__init__.py
+tn/english/rules/cardinal.py
+tn/english/rules/date.py
+tn/english/rules/decimal.py
+tn/english/rules/electronic.py
+tn/english/rules/fraction.py
+tn/english/rules/measure.py
+tn/english/rules/money.py
+tn/english/rules/ordinal.py
+tn/english/rules/punctuation.py
+tn/english/rules/roman.py
+tn/english/rules/telephone.py
+tn/english/rules/time.py
+tn/english/rules/whitelist.py
+tn/english/rules/word.py
+tn/english/test/__init__.py
+tn/english/test/cardinal_test.py
+tn/english/test/date_test.py
+tn/english/test/decimal_test.py
+tn/english/test/electronic_test.py
+tn/english/test/fraction_test.py
+tn/english/test/measure_test.py
+tn/english/test/money_test.py
+tn/english/test/normalizer_test.py
+tn/english/test/ordinal_test.py
+tn/english/test/roman_test.py
+tn/english/test/telephone_test.py
+tn/english/test/time_test.py
+tn/english/test/utils.py
+tn/english/test/whitelist_test.py
+tn/english/test/word_test.py

{WeTextProcessing-0.1.12 → WeTextProcessing-0.2.1}/itn/chinese/rules/cardinal.py

@@ -134,8 +134,9 @@ class Cardinal(Processor):
         cardinal |= (number + dot + digits.plus)
         # cardinal string like 110 or 12306 or 13125617878, used in phone,
         #   340621199806051223, used in ID card
+        idcard_last_char = digits | 'X' | 'x'
         cardinal |= (digits**3 | digits**4 | digits**5 | digits**11
-                     | digits**18)
+                     | (digits**17 + idcard_last_char) | digits**18)
 
         # 4. 特殊格式的数字 + 标准数字
         # cardinal string like 23

{WeTextProcessing-0.1.12 → WeTextProcessing-0.2.1}/itn/chinese/rules/license_plate.py

@@ -27,8 +27,11 @@ class LicensePlate(Processor):
 
     def build_tagger(self):
         digit = string_file('itn/chinese/data/number/digit.tsv')  # 1 ~ 9
+        zero = string_file('itn/chinese/data/number/zero.tsv')  # 0
+        digits = zero | digit
         province = string_file(
             'itn/chinese/data/license_plate/province.tsv')  # 皖
-        license_plate = province + self.ALPHA + (self.ALPHA | digit)**5
+        license_plate = province + self.ALPHA + (self.ALPHA | digits)**5
+        license_plate |= province + self.ALPHA + (self.ALPHA | digits)**6
         tagger = insert('value: "') + license_plate + insert('"')
         self.tagger = self.add_tokens(tagger)

{WeTextProcessing-0.1.12 → WeTextProcessing-0.2.1}/tn/chinese/rules/cardinal.py

@@ -64,11 +64,13 @@ class Cardinal(Processor):
         number @= self.build_rule(
             cross('二百', '两百')
             | cross('二千', '两千')
-            | cross('二万', '两万'))
-        self.number = accep('约').ques + accep('人均').ques + number.optimize()
+            | cross('二万', '两万')).optimize()
+        percent = insert('百分之') + number + delete('%')
+        self.number = accep('约').ques + accep('人均').ques + (number | percent)
 
         # cardinal string like 127.0.0.1, used in ID, IP, etc.
         cardinal = digits.plus + (dot + digits.plus)**3
+        cardinal |= percent
         # xxxx-xxx-xxx
         cardinal |= digits.plus + (delete('-') + digits.plus)**2
         # xxx-xxxxxxxx

{WeTextProcessing-0.1.12 → WeTextProcessing-0.2.1}/tn/chinese/rules/math.py

@@ -15,23 +15,32 @@
 from tn.chinese.rules.cardinal import Cardinal
 from tn.processor import Processor
 
-from pynini import string_file
+from pynini import cross, string_file
 from pynini.lib.pynutil import delete, insert
 
 
 class Math(Processor):
 
     def __init__(self):
-        super().__init__(name='math')
+        super().__init__(name="math")
         self.build_tagger()
         self.build_verbalizer()
 
     def build_tagger(self):
-        operator = string_file('tn/chinese/data/math/operator.tsv')
+        operator = string_file("tn/chinese/data/math/operator.tsv")
+        # When it appears alone, it is treated as punctuation
+        symbols = (
+            cross("~", "到")
+            | cross(":", "比")
+            | cross("<", "小于")
+            | cross(">", "大于")
+        )
 
         number = Cardinal().number
-        tagger = (number + ((delete(' ').ques + operator + delete(' ').ques +
-                             number).plus).ques)
+        tagger = (
+            number
+            + (delete(" ").ques + (operator | symbols) + delete(" ").ques + number).star
+        )
         tagger |= operator
         tagger = insert('value: "') + tagger + insert('"')
         self.tagger = self.add_tokens(tagger)

{WeTextProcessing-0.1.12 → WeTextProcessing-0.2.1}/tn/chinese/rules/measure.py

@@ -35,8 +35,6 @@ class Measure(Processor):
         to = cross('-', '到') | cross('~', '到') | accep('到')
 
         number = Cardinal().number
-        percent = insert('百分之') + number + delete('%')
-
         number @= self.build_rule(cross('二', '两'), '[BOS]', '[EOS]')
         # 1-11个，1个-11个
         prefix = number + (rmspace + units).ques + to
@@ -55,8 +53,7 @@ class Measure(Processor):
         prefix = cardinal + (rmspace + unit).ques + to
         annual = prefix.ques + cardinal + unit
 
-        tagger = insert('value: "') + (measure | percent
-                                       | annual) + insert('"')
+        tagger = insert('value: "') + (measure | annual) + insert('"')
 
         # 10km/h
         rmsign = rmspace + delete('/') + rmspace

WeTextProcessing-0.2.1/tn/english/data/address/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

WeTextProcessing-0.2.1/tn/english/data/measure/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

WeTextProcessing-0.2.1/tn/english/data/number/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

WeTextProcessing-0.2.1/tn/english/data/time/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

WeTextProcessing-0.2.1/tn/english/normalizer.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+# Copyright (c) 2024 Xingchen Song (sxc19@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tn.processor import Processor
+from tn.english.rules.cardinal import Cardinal
+from tn.english.rules.ordinal import Ordinal
+from tn.english.rules.decimal import Decimal
+from tn.english.rules.fraction import Fraction
+from tn.english.rules.word import Word
+from tn.english.rules.date import Date
+from tn.english.rules.time import Time
+from tn.english.rules.measure import Measure
+from tn.english.rules.money import Money
+from tn.english.rules.telephone import Telephone
+from tn.english.rules.electronic import Electronic
+from tn.english.rules.whitelist import WhiteList
+from tn.english.rules.punctuation import Punctuation
+
+from pynini.lib.pynutil import add_weight, delete
+from importlib_resources import files
+
+
+class Normalizer(Processor):
+
+    def __init__(self, cache_dir=None, overwrite_cache=False):
+        super().__init__(name='en_normalizer', ordertype="en_tn")
+        if cache_dir is None:
+            cache_dir = files("tn")
+        self.build_fst('en_tn', cache_dir, overwrite_cache)
+
+    def build_tagger(self):
+        cardinal = add_weight(Cardinal().tagger, 1.0)
+        ordinal = add_weight(Ordinal().tagger, 1.0)
+        decimal = add_weight(Decimal().tagger, 1.0)
+        fraction = add_weight(Fraction().tagger, 1.0)
+        date = add_weight(Date().tagger, 0.99)
+        time = add_weight(Time().tagger, 1.00)
+        measure = add_weight(Measure().tagger, 1.00)
+        money = add_weight(Money().tagger, 1.00)
+        telephone = add_weight(Telephone().tagger, 1.00)
+        electronic = add_weight(Electronic().tagger, 1.00)
+        word = add_weight(Word().tagger, 100)
+        whitelist = add_weight(WhiteList().tagger, 1.00)
+        punct = add_weight(Punctuation().tagger, 2.00)
+        # TODO(xcsong): add roman
+        tagger = punct.star + \
+            (cardinal | ordinal | word
+             | date | decimal | fraction
+             | time | measure | money
+             | telephone | electronic
+             | whitelist
+             | punct).optimize() + (punct.plus | self.DELETE_SPACE)
+        # delete the last space
+        self.tagger = tagger.star @ self.build_rule(delete(' '), r='[EOS]')
+
+    def build_verbalizer(self):
+        cardinal = Cardinal().verbalizer
+        ordinal = Ordinal().verbalizer
+        decimal = Decimal().verbalizer
+        fraction = Fraction().verbalizer
+        word = Word().verbalizer
+        date = Date().verbalizer
+        time = Time().verbalizer
+        measure = Measure().verbalizer
+        money = Money().verbalizer
+        telephone = Telephone().verbalizer
+        electronic = Electronic().verbalizer
+        whitelist = WhiteList().verbalizer
+        punct = Punctuation().verbalizer
+        verbalizer = \
+            (cardinal | ordinal | word
+             | date | decimal
+             | fraction | time
+             | measure | money
+             | telephone
+             | electronic
+             | whitelist
+             | punct).optimize() + punct.ques + self.INSERT_SPACE
+        self.verbalizer = verbalizer.star @ self.build_rule(delete(' '),
+                                                            r='[EOS]')

WeTextProcessing-0.2.1/tn/english/rules/cardinal.py

@@ -0,0 +1,186 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2024, WENET COMMUNITY.  Xingchen Song (sxc19@tsinghua.org.cn).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini.examples import plurals
+from pynini.lib import pynutil
+
+from tn.processor import Processor
+from tn.utils import get_abs_path
+
+
+class Cardinal(Processor):
+
+    def __init__(self, deterministic: bool = False):
+        """
+        Args:
+            deterministic: if True will provide a single transduction option,
+                for False multiple transduction are generated (used for audio-based normalization)
+        """
+        super().__init__('cardinal', ordertype="en_tn")
+        self.deterministic = deterministic
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        """
+        Finite state transducer for classifying cardinals, e.g.
+            -23 -> cardinal { negative: "true" integer: "twenty three" }
+        """
+        # TODO replace to have "oh" as a default for "0"
+        graph = pynini.Far(
+            get_abs_path(
+                "english/data/number/cardinal_number_name.far")).get_fst()
+        graph_au = pynini.Far(
+            get_abs_path(
+                "english/data/number/cardinal_number_name_au.far")).get_fst()
+        self.graph_hundred_component_at_least_one_none_zero_digit = (
+            pynini.closure(self.DIGIT, 2, 3)
+            | pynini.difference(self.DIGIT, pynini.accep("0"))) @ graph
+
+        graph_digit = pynini.string_file(
+            get_abs_path("english/data/number/digit.tsv"))
+        graph_zero = pynini.string_file(
+            get_abs_path("english/data/number/zero.tsv"))
+
+        single_digits_graph = pynini.invert(graph_digit | graph_zero)
+        self.single_digits_graph = single_digits_graph + pynini.closure(
+            self.INSERT_SPACE + single_digits_graph)
+
+        if not self.deterministic:
+            # for a single token allow only the same normalization
+            # "007" -> {"oh oh seven", "zero zero seven"} not {"oh zero seven"}
+            single_digits_graph_zero = pynini.invert(graph_digit | graph_zero)
+            single_digits_graph_oh = pynini.invert(graph_digit) | pynini.cross(
+                "0", "oh")
+
+            self.single_digits_graph = single_digits_graph_zero + pynini.closure(
+                self.INSERT_SPACE + single_digits_graph_zero)
+            self.single_digits_graph |= single_digits_graph_oh + pynini.closure(
+                self.INSERT_SPACE + single_digits_graph_oh)
+
+            single_digits_graph_with_commas = pynini.closure(
+                self.single_digits_graph + self.INSERT_SPACE, 1,
+                3) + pynini.closure(
+                    pynutil.delete(",") + single_digits_graph +
+                    self.INSERT_SPACE + single_digits_graph +
+                    self.INSERT_SPACE + single_digits_graph, 1)
+
+        graph = (pynini.closure(self.DIGIT, 1, 3) +
+                 (pynini.closure(pynutil.delete(",") + self.DIGIT**3)
+                  | pynini.closure(self.DIGIT**3))) @ graph
+
+        self.graph = graph
+        self.graph_with_and = self.add_optional_and(graph)
+
+        if self.deterministic:
+            long_numbers = pynini.compose(self.DIGIT**(5, ...),
+                                          self.single_digits_graph).optimize()
+            self.long_numbers = plurals._priority_union(
+                long_numbers, self.graph_with_and,
+                pynini.closure(self.VCHAR)).optimize()
+            cardinal_with_leading_zeros = pynini.compose(
+                pynini.accep("0") + pynini.closure(self.DIGIT),
+                self.single_digits_graph)
+            final_graph = self.long_numbers | cardinal_with_leading_zeros
+            final_graph |= self.add_optional_and(graph_au)
+        else:
+            leading_zeros = pynini.compose(
+                pynini.closure(pynini.accep("0"), 1), self.single_digits_graph)
+            cardinal_with_leading_zeros = (
+                leading_zeros + self.INSERT_SPACE + pynini.compose(
+                    pynini.closure(self.DIGIT), self.graph_with_and))
+            self.long_numbers = self.graph_with_and | pynutil.add_weight(
+                self.single_digits_graph, 0.0001)
+            # add small weight to non-default graphs to make sure the deterministic option is listed first
+            final_graph = (self.long_numbers
+                           | pynutil.add_weight(
+                               single_digits_graph_with_commas, 0.0001)
+                           | cardinal_with_leading_zeros).optimize()
+
+            one_to_a_replacement_graph = (
+                pynini.cross("one hundred", "a hundred")
+                | pynini.cross("one thousand", "thousand")
+                | pynini.cross("one million", "a million"))
+            final_graph |= pynini.compose(
+                final_graph,
+                one_to_a_replacement_graph.optimize() +
+                pynini.closure(self.VCHAR)).optimize()
+            # remove commas for 4 digits numbers
+            four_digit_comma_graph = (
+                self.DIGIT - "0") + pynutil.delete(",") + self.DIGIT**3
+            final_graph |= pynini.compose(four_digit_comma_graph.optimize(),
+                                          final_graph).optimize()
+
+        self.final_graph = final_graph
+        optional_minus_graph = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0,
+            1)
+        final_graph = optional_minus_graph + pynutil.insert(
+            "integer: \"") + final_graph + pynutil.insert("\"")
+        final_graph = self.add_tokens(final_graph)
+        self.tagger = final_graph.optimize()
+
+    def add_optional_and(self, graph):
+        graph_with_and = graph
+
+        graph_with_and = pynutil.add_weight(graph, 0.00001)
+        not_quote = pynini.closure(self.NOT_QUOTE)
+        no_thousand_million = pynini.difference(
+            not_quote, not_quote + pynini.union("thousand", "million") +
+            not_quote).optimize()
+        integer = (not_quote + pynutil.add_weight(
+            pynini.cross("hundred ", "hundred and ") + no_thousand_million,
+            -0.0001)).optimize()
+
+        no_hundred = pynini.difference(
+            pynini.closure(self.VCHAR),
+            not_quote + pynini.accep("hundred") + not_quote).optimize()
+        integer |= (not_quote + pynutil.add_weight(
+            pynini.cross("thousand ", "thousand and ") + no_hundred,
+            -0.0001)).optimize()
+
+        optional_hundred = pynini.compose((self.DIGIT - "0")**3,
+                                          graph).optimize()
+        optional_hundred = pynini.compose(
+            optional_hundred,
+            pynini.closure(self.VCHAR) + pynini.cross(" hundred", "") +
+            pynini.closure(self.VCHAR))
+        graph_with_and |= pynini.compose(graph, integer).optimize()
+        graph_with_and |= optional_hundred
+        return graph_with_and
+
+    def build_verbalizer(self):
+        """
+        Finite state transducer for verbalizing cardinal, e.g.
+            cardinal { negative: "true" integer: "23" } -> minus twenty three
+        """
+        optional_sign = pynini.cross("negative: \"true\"", "minus ")
+        if not self.deterministic:
+            optional_sign |= pynini.cross("negative: \"true\"", "negative ")
+            optional_sign |= pynini.cross("negative: \"true\"", "dash ")
+
+        self.optional_sign = pynini.closure(optional_sign + self.DELETE_SPACE,
+                                            0, 1)
+
+        integer = pynini.closure(self.NOT_QUOTE)
+
+        self.integer = self.DELETE_SPACE + pynutil.delete(
+            "\"") + integer + pynutil.delete("\"")
+        integer = pynutil.delete("integer:") + self.integer
+
+        self.numbers = self.optional_sign + integer
+        delete_tokens = self.delete_tokens(self.numbers)
+        self.verbalizer = delete_tokens.optimize()