WeTextProcessing 1.0.6.dev19__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- itn/README.md +12 -4
- itn/arabic/__init__.py +0 -0
- itn/arabic/data/measure/measurements.tsv +14 -0
- itn/arabic/data/money/currency_major.tsv +24 -0
- itn/arabic/data/money/currency_minor_plural.tsv +4 -0
- itn/arabic/data/money/currency_minor_singular.tsv +3 -0
- itn/arabic/data/money/local_currency.tsv +20 -0
- itn/arabic/data/number/digit.tsv +9 -0
- itn/arabic/data/number/digit_100.tsv +7 -0
- itn/arabic/data/number/digit_1000.tsv +7 -0
- itn/arabic/data/number/flops.tsv +72 -0
- itn/arabic/data/number/fraction_dual.tsv +9 -0
- itn/arabic/data/number/fraction_plural.tsv +8 -0
- itn/arabic/data/number/fraction_singular.tsv +9 -0
- itn/arabic/data/number/quantities.tsv +6 -0
- itn/arabic/data/number/teens.tsv +9 -0
- itn/arabic/data/number/tens.tsv +10 -0
- itn/arabic/inverse_normalizer.py +60 -0
- itn/arabic/rules/__init__.py +0 -0
- itn/arabic/rules/cardinal.py +66 -0
- itn/arabic/rules/decimal.py +58 -0
- itn/arabic/rules/fraction.py +72 -0
- itn/arabic/rules/measure.py +49 -0
- itn/arabic/rules/money.py +73 -0
- itn/arabic/rules/punctuation.py +29 -0
- itn/arabic/rules/word.py +28 -0
- itn/arabic/test/__init__.py +0 -0
- itn/arabic/test/data/test_cases_cardinal.txt +34 -0
- itn/arabic/test/data/test_cases_decimal.txt +30 -0
- itn/arabic/test/data/test_cases_fraction.txt +20 -0
- itn/arabic/test/data/test_cases_measure.txt +15 -0
- itn/arabic/test/data/test_cases_money.txt +21 -0
- itn/arabic/test/data/test_cases_whitelist.txt +6 -0
- itn/arabic/test/normalizer_test.py +137 -0
- itn/armenian/__init__.py +0 -0
- itn/armenian/data/currency.tsv +53 -0
- itn/armenian/data/measurement_dates.tsv +4 -0
- itn/armenian/data/measurements.tsv +48 -0
- itn/armenian/data/numbers/digit.tsv +11 -0
- itn/armenian/data/numbers/digits_no_one.tsv +10 -0
- itn/armenian/data/numbers/ties.tsv +10 -0
- itn/armenian/data/ordinals/digit.tsv +10 -0
- itn/armenian/data/time/hours.tsv +29 -0
- itn/armenian/data/time/minutes.tsv +65 -0
- itn/armenian/data/time/minutes_to.tsv +59 -0
- itn/armenian/data/time/to_hour.tsv +25 -0
- itn/armenian/data/whitelist.tsv +7 -0
- itn/armenian/inverse_normalizer.py +66 -0
- itn/armenian/rules/__init__.py +0 -0
- itn/armenian/rules/cardinal.py +67 -0
- itn/armenian/rules/decimal.py +67 -0
- itn/armenian/rules/fraction.py +40 -0
- itn/armenian/rules/measure.py +52 -0
- itn/armenian/rules/money.py +38 -0
- itn/armenian/rules/ordinal.py +37 -0
- itn/armenian/rules/punctuation.py +29 -0
- itn/armenian/rules/time.py +59 -0
- itn/armenian/rules/utils.py +58 -0
- itn/armenian/rules/whitelist.py +27 -0
- itn/armenian/rules/word.py +28 -0
- itn/armenian/test/__init__.py +0 -0
- itn/armenian/test/data/test_cases_cardinal.txt +23 -0
- itn/armenian/test/data/test_cases_decimal.txt +14 -0
- itn/armenian/test/data/test_cases_fraction.txt +24 -0
- itn/armenian/test/data/test_cases_measure.txt +12 -0
- itn/armenian/test/data/test_cases_money.txt +10 -0
- itn/armenian/test/data/test_cases_ordinal.txt +11 -0
- itn/armenian/test/data/test_cases_time.txt +9 -0
- itn/armenian/test/data/test_cases_whitelist.txt +7 -0
- itn/armenian/test/data/test_cases_word.txt +50 -0
- itn/armenian/test/normalizer_test.py +158 -0
- itn/chinese/inverse_normalizer.py +49 -33
- itn/chinese/rules/cardinal.py +42 -69
- itn/chinese/rules/date.py +25 -18
- itn/chinese/rules/fraction.py +15 -14
- itn/chinese/rules/license_plate.py +11 -10
- itn/chinese/rules/math.py +7 -7
- itn/chinese/rules/measure.py +27 -42
- itn/chinese/rules/money.py +12 -18
- itn/chinese/rules/postprocessor.py +2 -2
- itn/chinese/rules/time.py +24 -25
- itn/chinese/rules/train_number.py +9 -8
- itn/chinese/rules/whitelist.py +7 -6
- itn/chinese/test/normalizer_test.py +155 -30
- itn/english/__init__.py +0 -0
- itn/english/data/currency.tsv +168 -0
- itn/english/data/date_period.tsv +10 -0
- itn/english/data/electronic/domain.tsv +10 -0
- itn/english/data/electronic/server_name.tsv +17 -0
- itn/english/data/electronic/symbols.tsv +22 -0
- itn/english/data/electronic/url_symbols.tsv +5 -0
- itn/english/data/magnitudes.tsv +4 -0
- itn/english/data/measurements.tsv +149 -0
- itn/english/data/months.tsv +12 -0
- itn/english/data/months_cased.tsv +12 -0
- itn/english/data/numbers/digit.tsv +9 -0
- itn/english/data/numbers/hundred.tsv +1 -0
- itn/english/data/numbers/teen.tsv +10 -0
- itn/english/data/numbers/thousands.tsv +22 -0
- itn/english/data/numbers/ties.tsv +9 -0
- itn/english/data/numbers/zero.tsv +1 -0
- itn/english/data/ordinals/digit.tsv +9 -0
- itn/english/data/ordinals/teen.tsv +1 -0
- itn/english/data/time/minute_to.tsv +59 -0
- itn/english/data/time/time_suffix.tsv +8 -0
- itn/english/data/time/time_suffix_cased.tsv +8 -0
- itn/english/data/time/time_zone.tsv +7 -0
- itn/english/data/time/time_zone_cased.tsv +7 -0
- itn/english/data/time/to_hour.tsv +12 -0
- itn/english/data/whitelist.tsv +417 -0
- itn/english/data/whitelist_tech.tsv +450 -0
- itn/english/data/year_suffix.tsv +6 -0
- itn/english/inverse_normalizer.py +91 -0
- itn/english/rules/__init__.py +0 -0
- itn/english/rules/cardinal.py +111 -0
- itn/english/rules/char.py +29 -0
- itn/english/rules/date.py +163 -0
- itn/english/rules/decimal.py +82 -0
- itn/english/rules/electronic.py +79 -0
- itn/english/rules/measure.py +106 -0
- itn/english/rules/money.py +121 -0
- itn/english/rules/ordinal.py +50 -0
- itn/english/rules/punctuation.py +31 -0
- itn/english/rules/telephone.py +147 -0
- itn/english/rules/time.py +138 -0
- itn/english/rules/whitelist.py +33 -0
- itn/english/rules/word.py +26 -0
- itn/english/test/__init__.py +0 -0
- itn/english/test/data/en_cardinal.txt +28 -0
- itn/english/test/data/en_date.txt +44 -0
- itn/english/test/data/en_decimal.txt +63 -0
- itn/english/test/data/en_electronic.txt +25 -0
- itn/english/test/data/en_measure.txt +114 -0
- itn/english/test/data/en_money.txt +52 -0
- itn/english/test/data/en_ordinal.txt +34 -0
- itn/english/test/data/en_telephone.txt +24 -0
- itn/english/test/data/en_time.txt +36 -0
- itn/english/test/data/en_whitelist.txt +12 -0
- itn/english/test/data/en_word.txt +55 -0
- itn/english/test/normalizer_test.py +130 -0
- tn/chinese/test/preprocessor_test.py → itn/english/test/utils.py +14 -11
- itn/french/__init__.py +0 -0
- itn/french/data/electronic/domain.tsv +25 -0
- itn/french/data/electronic/server_name.tsv +17 -0
- itn/french/data/electronic/symbols.tsv +12 -0
- itn/french/data/fractions.tsv +41 -0
- itn/french/data/measurements/magnitudes.tsv +16 -0
- itn/french/data/measurements/measurements.tsv +27 -0
- itn/french/data/money/currency_major.tsv +40 -0
- itn/french/data/money/currency_minor.tsv +5 -0
- itn/french/data/months.tsv +12 -0
- itn/french/data/numbers/digit.tsv +10 -0
- itn/french/data/numbers/teen.tsv +6 -0
- itn/french/data/numbers/ties.tsv +13 -0
- itn/french/data/numbers/ties_unique.tsv +25 -0
- itn/french/data/numbers/zero.tsv +1 -0
- itn/french/data/ordinals/digits_root_change.tsv +17 -0
- itn/french/data/ordinals/firsts.tsv +4 -0
- itn/french/data/ordinals/key_nouns.tsv +1 -0
- itn/french/data/ordinals/second.tsv +4 -0
- itn/french/data/roman/digits_large.tsv +9 -0
- itn/french/data/roman/hundreds_large.tsv +9 -0
- itn/french/data/roman/ties_large.tsv +9 -0
- itn/french/data/time/hour_to_night.tsv +12 -0
- itn/french/data/time/hours.tsv +26 -0
- itn/french/data/time/hours_to.tsv +25 -0
- itn/french/data/time/minutes.tsv +63 -0
- itn/french/data/time/minutes_to.tsv +59 -0
- itn/french/data/time/time_suffix_am.tsv +1 -0
- itn/french/data/time/time_suffix_pm.tsv +2 -0
- itn/french/data/whitelist.tsv +16 -0
- itn/french/inverse_normalizer.py +74 -0
- itn/french/rules/__init__.py +0 -0
- itn/french/rules/cardinal.py +115 -0
- itn/french/rules/date.py +45 -0
- itn/french/rules/decimal.py +70 -0
- itn/french/rules/electronic.py +67 -0
- itn/french/rules/fraction.py +55 -0
- itn/french/rules/measure.py +55 -0
- itn/french/rules/money.py +67 -0
- itn/french/rules/ordinal.py +63 -0
- itn/french/rules/punctuation.py +29 -0
- itn/french/rules/telephone.py +59 -0
- itn/french/rules/time.py +70 -0
- itn/french/rules/whitelist.py +27 -0
- itn/french/rules/word.py +29 -0
- itn/french/test/__init__.py +0 -0
- itn/french/test/data/test_cases_cardinal.txt +106 -0
- itn/french/test/data/test_cases_date.txt +6 -0
- itn/french/test/data/test_cases_decimal.txt +15 -0
- itn/french/test/data/test_cases_electronic.txt +10 -0
- itn/french/test/data/test_cases_fraction.txt +30 -0
- itn/french/test/data/test_cases_measure.txt +15 -0
- itn/french/test/data/test_cases_money.txt +22 -0
- itn/french/test/data/test_cases_ordinal.txt +23 -0
- itn/french/test/data/test_cases_telephone.txt +5 -0
- itn/french/test/data/test_cases_time.txt +18 -0
- itn/french/test/data/test_cases_whitelist.txt +8 -0
- itn/french/test/data/test_cases_word.txt +49 -0
- itn/french/test/normalizer_test.py +155 -0
- itn/german/__init__.py +0 -0
- itn/german/inverse_normalizer.py +85 -0
- itn/german/rules/__init__.py +0 -0
- itn/german/rules/cardinal.py +48 -0
- itn/german/rules/date.py +63 -0
- itn/german/rules/decimal.py +67 -0
- itn/german/rules/electronic.py +58 -0
- itn/german/rules/fraction.py +75 -0
- itn/german/rules/measure.py +80 -0
- itn/german/rules/money.py +87 -0
- itn/german/rules/ordinal.py +31 -0
- itn/german/rules/punctuation.py +29 -0
- itn/german/rules/telephone.py +45 -0
- itn/german/rules/time.py +102 -0
- itn/german/rules/whitelist.py +30 -0
- itn/german/rules/word.py +33 -0
- itn/german/test/__init__.py +0 -0
- itn/german/test/calendar_test.py +147 -0
- itn/german/test/data/test_cases_cardinal.txt +62 -0
- itn/german/test/data/test_cases_date.txt +22 -0
- itn/german/test/data/test_cases_decimal.txt +10 -0
- itn/german/test/data/test_cases_electronic.txt +9 -0
- itn/german/test/data/test_cases_fraction.txt +34 -0
- itn/german/test/data/test_cases_measure.txt +28 -0
- itn/german/test/data/test_cases_money.txt +23 -0
- itn/german/test/data/test_cases_ordinal.txt +20 -0
- itn/german/test/data/test_cases_telephone.txt +1 -0
- itn/german/test/data/test_cases_time.txt +24 -0
- itn/german/test/data/test_cases_whitelist.txt +6 -0
- itn/german/test/data/test_cases_word.txt +49 -0
- itn/german/test/financial_test.py +284 -0
- itn/german/test/inverse_normalizer_test.py +92 -0
- itn/german/test/numeric_test.py +219 -0
- itn/german/test/text_test.py +281 -0
- itn/german/test/time_test.py +70 -0
- itn/hebrew/__init__.py +0 -0
- itn/hebrew/data/decimals/minutes_exception.tsv +2 -0
- itn/hebrew/data/measurements.tsv +45 -0
- itn/hebrew/data/months.tsv +13 -0
- itn/hebrew/data/months_name2number.tsv +12 -0
- itn/hebrew/data/months_ordinal2number.tsv +12 -0
- itn/hebrew/data/numbers/decimal_fractions.tsv +6 -0
- itn/hebrew/data/numbers/digit.tsv +20 -0
- itn/hebrew/data/numbers/hundreds_exception.tsv +2 -0
- itn/hebrew/data/numbers/millions_exception.tsv +1 -0
- itn/hebrew/data/numbers/teen.tsv +21 -0
- itn/hebrew/data/numbers/thousands.tsv +8 -0
- itn/hebrew/data/numbers/thousands_exception.tsv +2 -0
- itn/hebrew/data/numbers/ties.tsv +8 -0
- itn/hebrew/data/numbers/viable_hours.tsv +15 -0
- itn/hebrew/data/numbers/zero.tsv +1 -0
- itn/hebrew/data/ordinals/digit.tsv +10 -0
- itn/hebrew/data/prefix.tsv +17 -0
- itn/hebrew/data/spaced_measurements.tsv +17 -0
- itn/hebrew/data/time/day_suffix.tsv +2 -0
- itn/hebrew/data/time/evening_suffix.tsv +2 -0
- itn/hebrew/data/time/hour_to_evening.tsv +7 -0
- itn/hebrew/data/time/hour_to_night.tsv +9 -0
- itn/hebrew/data/time/hour_to_noon.tsv +7 -0
- itn/hebrew/data/time/midnight_to_hour.tsv +1 -0
- itn/hebrew/data/time/minute_to.tsv +58 -0
- itn/hebrew/data/time/minute_to_verbose.tsv +6 -0
- itn/hebrew/data/time/minute_verbose.tsv +8 -0
- itn/hebrew/data/time/night_suffix.tsv +1 -0
- itn/hebrew/data/time/noon_suffix.tsv +3 -0
- itn/hebrew/data/time/to_hour.tsv +13 -0
- itn/hebrew/data/whitelist.tsv +20 -0
- itn/hebrew/inverse_normalizer.py +64 -0
- itn/hebrew/rules/__init__.py +0 -0
- itn/hebrew/rules/cardinal.py +94 -0
- itn/hebrew/rules/date.py +62 -0
- itn/hebrew/rules/decimal.py +74 -0
- itn/hebrew/rules/measure.py +53 -0
- itn/hebrew/rules/punctuation.py +28 -0
- itn/hebrew/rules/time.py +101 -0
- itn/hebrew/rules/utils.py +78 -0
- itn/hebrew/rules/whitelist.py +30 -0
- itn/hebrew/rules/word.py +28 -0
- itn/hebrew/test/__init__.py +0 -0
- itn/hebrew/test/data/test_cases_cardinal.txt +138 -0
- itn/hebrew/test/data/test_cases_date.txt +29 -0
- itn/hebrew/test/data/test_cases_decimal.txt +66 -0
- itn/hebrew/test/data/test_cases_measure.txt +10 -0
- itn/hebrew/test/data/test_cases_time.txt +34 -0
- itn/hebrew/test/data/test_cases_whitelist.txt +4 -0
- itn/hebrew/test/data/test_full_sentences.txt +56 -0
- itn/hebrew/test/normalizer_test.py +178 -0
- itn/hindi/__init__.py +0 -0
- itn/hindi/data/address/cities.tsv +36 -0
- itn/hindi/data/address/context_cues.tsv +85 -0
- itn/hindi/data/address/digit_passthrough.tsv +10 -0
- itn/hindi/data/address/ordinals.tsv +167 -0
- itn/hindi/data/address/special_characters.tsv +2 -0
- itn/hindi/data/address/states.tsv +36 -0
- itn/hindi/data/date/century.tsv +5 -0
- itn/hindi/data/date/months.tsv +14 -0
- itn/hindi/data/measure/measurements.tsv +295 -0
- itn/hindi/data/money/currency.tsv +181 -0
- itn/hindi/data/numbers/digit.tsv +13 -0
- itn/hindi/data/numbers/hundred.tsv +1 -0
- itn/hindi/data/numbers/labels_exception.tsv +4 -0
- itn/hindi/data/numbers/paune.tsv +231 -0
- itn/hindi/data/numbers/teens_and_ties.tsv +220 -0
- itn/hindi/data/numbers/thousands.tsv +8 -0
- itn/hindi/data/numbers/zero.tsv +1 -0
- itn/hindi/data/ordinals/digit.tsv +6 -0
- itn/hindi/data/ordinals/digit_fem.tsv +6 -0
- itn/hindi/data/ordinals/hundred_digit.tsv +12 -0
- itn/hindi/data/ordinals/hundred_digit_fem.tsv +12 -0
- itn/hindi/data/ordinals/morph_features.tsv +6 -0
- itn/hindi/data/ordinals/suffix.tsv +4 -0
- itn/hindi/data/ordinals/suffix_fem.tsv +2 -0
- itn/hindi/data/ordinals/teens_and_ties.tsv +1 -0
- itn/hindi/data/ordinals/teens_and_ties_fem.tsv +1 -0
- itn/hindi/data/telephone/context_cues.tsv +15 -0
- itn/hindi/data/telephone/country_codes.tsv +1 -0
- itn/hindi/data/telephone/eng_digit.tsv +9 -0
- itn/hindi/data/telephone/eng_zero.tsv +1 -0
- itn/hindi/data/telephone/teens_and_ties_eng_to_hin.tsv +90 -0
- itn/hindi/data/time/hour_for_paune.tsv +15 -0
- itn/hindi/data/whitelist/whitelist.tsv +7 -0
- itn/hindi/inverse_normalizer.py +78 -0
- itn/hindi/rules/__init__.py +0 -0
- itn/hindi/rules/cardinal.py +93 -0
- itn/hindi/rules/date.py +64 -0
- itn/hindi/rules/decimal.py +57 -0
- itn/hindi/rules/fraction.py +55 -0
- itn/hindi/rules/measure.py +83 -0
- itn/hindi/rules/money.py +81 -0
- itn/hindi/rules/ordinal.py +40 -0
- itn/hindi/rules/punctuation.py +31 -0
- itn/hindi/rules/telephone.py +65 -0
- itn/hindi/rules/time.py +53 -0
- itn/hindi/rules/utils.py +52 -0
- itn/hindi/rules/whitelist.py +32 -0
- itn/hindi/rules/word.py +30 -0
- itn/hindi/test/__init__.py +0 -0
- itn/hindi/test/data/test_cases_address.txt +25 -0
- itn/hindi/test/data/test_cases_cardinal.txt +54 -0
- itn/hindi/test/data/test_cases_date.txt +42 -0
- itn/hindi/test/data/test_cases_decimal.txt +13 -0
- itn/hindi/test/data/test_cases_fraction.txt +31 -0
- itn/hindi/test/data/test_cases_measure.txt +48 -0
- itn/hindi/test/data/test_cases_money.txt +50 -0
- itn/hindi/test/data/test_cases_ordinal.txt +13 -0
- itn/hindi/test/data/test_cases_telephone.txt +28 -0
- itn/hindi/test/data/test_cases_time.txt +25 -0
- itn/hindi/test/data/test_cases_whitelist.txt +8 -0
- itn/hindi/test/data/test_cases_word.txt +15 -0
- itn/hindi/test/normalizer_test.py +127 -0
- itn/hindi_english/__init__.py +0 -0
- itn/hindi_english/data/measurements.tsv +2 -0
- itn/hindi_english/inverse_normalizer.py +93 -0
- itn/hindi_english/rules/__init__.py +0 -0
- itn/hindi_english/rules/english.py +175 -0
- itn/hindi_english/test/__init__.py +0 -0
- itn/hindi_english/test/data/test_cases_address.txt +41 -0
- itn/hindi_english/test/data/test_cases_cardinal.txt +61 -0
- itn/hindi_english/test/data/test_cases_date.txt +37 -0
- itn/hindi_english/test/data/test_cases_date_cased.txt +70 -0
- itn/hindi_english/test/data/test_cases_decimal.txt +25 -0
- itn/hindi_english/test/data/test_cases_electronic.txt +17 -0
- itn/hindi_english/test/data/test_cases_fraction.txt +39 -0
- itn/hindi_english/test/data/test_cases_measure.txt +25 -0
- itn/hindi_english/test/data/test_cases_money.txt +41 -0
- itn/hindi_english/test/data/test_cases_ordinal.txt +26 -0
- itn/hindi_english/test/data/test_cases_telephone.txt +39 -0
- itn/hindi_english/test/data/test_cases_time.txt +29 -0
- itn/hindi_english/test/data/test_cases_whitelist.txt +17 -0
- itn/hindi_english/test/data/test_cases_word.txt +19 -0
- itn/hindi_english/test/normalizer_test.py +211 -0
- itn/hungarian/__init__.py +0 -0
- itn/hungarian/data/dates/days.tsv +31 -0
- itn/hungarian/data/dates/months.tsv +12 -0
- itn/hungarian/data/electronic/domain.tsv +15 -0
- itn/hungarian/data/electronic/server_name.tsv +15 -0
- itn/hungarian/data/electronic/symbols.tsv +23 -0
- itn/hungarian/data/inflection/endings.tsv +51 -0
- itn/hungarian/data/inflection/plural_endings.tsv +39 -0
- itn/hungarian/data/inflection/word_endings.tsv +1 -0
- itn/hungarian/data/measures/measurements.tsv +52 -0
- itn/hungarian/data/money/currency.tsv +11 -0
- itn/hungarian/data/money/currency_minor.tsv +11 -0
- itn/hungarian/data/number/digit.tsv +9 -0
- itn/hungarian/data/number/digit_inline.tsv +9 -0
- itn/hungarian/data/number/quantities.tsv +21 -0
- itn/hungarian/data/number/tens.tsv +9 -0
- itn/hungarian/data/number/tens_inline.tsv +9 -0
- itn/hungarian/data/number/zero.tsv +1 -0
- itn/hungarian/data/ordinals/endings.tsv +23 -0
- itn/hungarian/data/ordinals/exceptional.tsv +4 -0
- itn/hungarian/data/telephone/area_codes.tsv +71 -0
- itn/hungarian/data/telephone/country_codes.tsv +213 -0
- itn/hungarian/data/time/time_zone.tsv +9 -0
- itn/hungarian/data/whitelist.tsv +24 -0
- itn/hungarian/data/whitelist_inflect.tsv +39 -0
- itn/hungarian/data/whitelist_inflect_sg.tsv +14 -0
- itn/hungarian/inverse_normalizer.py +93 -0
- itn/hungarian/rules/__init__.py +0 -0
- itn/hungarian/rules/cardinal.py +99 -0
- itn/hungarian/rules/date.py +66 -0
- itn/hungarian/rules/decimal.py +80 -0
- itn/hungarian/rules/electronic.py +71 -0
- itn/hungarian/rules/fraction.py +41 -0
- itn/hungarian/rules/measure.py +74 -0
- itn/hungarian/rules/money.py +82 -0
- itn/hungarian/rules/ordinal.py +42 -0
- itn/hungarian/rules/punctuation.py +44 -0
- itn/hungarian/rules/telephone.py +68 -0
- itn/hungarian/rules/time.py +99 -0
- itn/hungarian/rules/utils.py +143 -0
- itn/hungarian/rules/whitelist.py +50 -0
- itn/hungarian/rules/word.py +38 -0
- itn/hungarian/test/__init__.py +0 -0
- itn/hungarian/test/normalizer_test.py +214 -0
- itn/italian/__init__.py +0 -0
- itn/italian/data/electronic/domain.tsv +14 -0
- itn/italian/data/electronic/server_name.tsv +11 -0
- itn/italian/data/electronic/symbols.tsv +22 -0
- itn/italian/data/measure/denominators.tsv +4 -0
- itn/italian/data/measure/units.tsv +51 -0
- itn/italian/data/money/currency.tsv +11 -0
- itn/italian/data/numbers/digit.tsv +9 -0
- itn/italian/data/numbers/hundreds.tsv +9 -0
- itn/italian/data/numbers/quantities.tsv +4 -0
- itn/italian/data/numbers/teen.tsv +10 -0
- itn/italian/data/numbers/tens.tsv +8 -0
- itn/italian/data/whitelist.tsv +19 -0
- itn/italian/inverse_normalizer.py +77 -0
- itn/italian/rules/__init__.py +0 -0
- itn/italian/rules/cardinal.py +100 -0
- itn/italian/rules/decimal.py +59 -0
- itn/italian/rules/electronic.py +70 -0
- itn/italian/rules/measure.py +52 -0
- itn/italian/rules/money.py +78 -0
- itn/italian/rules/punctuation.py +27 -0
- itn/italian/rules/time.py +84 -0
- itn/italian/rules/utils.py +49 -0
- itn/italian/rules/whitelist.py +31 -0
- itn/italian/rules/word.py +27 -0
- itn/italian/test/__init__.py +0 -0
- itn/italian/test/normalizer_test.py +277 -0
- itn/japanese/inverse_normalizer.py +62 -47
- itn/japanese/rules/cardinal.py +40 -55
- itn/japanese/rules/char.py +7 -5
- itn/japanese/rules/date.py +22 -25
- itn/japanese/rules/fraction.py +21 -19
- itn/japanese/rules/math.py +9 -8
- itn/japanese/rules/measure.py +14 -13
- itn/japanese/rules/money.py +10 -7
- itn/japanese/rules/ordinal.py +7 -5
- itn/japanese/rules/postprocessor.py +5 -5
- itn/japanese/rules/preprocessor.py +2 -2
- itn/japanese/rules/time.py +16 -17
- itn/japanese/rules/whitelist.py +9 -7
- itn/japanese/test/normalizer_test.py +132 -30
- itn/kinyarwanda/__init__.py +0 -0
- itn/kinyarwanda/data/numbers/digit.tsv +9 -0
- itn/kinyarwanda/data/numbers/digit_bi.tsv +9 -0
- itn/kinyarwanda/data/numbers/digit_e.tsv +9 -0
- itn/kinyarwanda/data/numbers/hundreds.tsv +9 -0
- itn/kinyarwanda/data/numbers/tens.tsv +8 -0
- itn/kinyarwanda/data/numbers/zero.tsv +1 -0
- itn/kinyarwanda/data/time/hours.tsv +12 -0
- itn/kinyarwanda/data/time/minutes.tsv +59 -0
- itn/kinyarwanda/data/whitelist.tsv +18 -0
- itn/kinyarwanda/inverse_normalizer.py +53 -0
- itn/kinyarwanda/rules/__init__.py +0 -0
- itn/kinyarwanda/rules/cardinal.py +131 -0
- itn/kinyarwanda/rules/punctuation.py +28 -0
- itn/kinyarwanda/rules/time.py +39 -0
- itn/kinyarwanda/rules/utils.py +39 -0
- itn/kinyarwanda/rules/whitelist.py +32 -0
- itn/kinyarwanda/rules/word.py +28 -0
- itn/kinyarwanda/test/__init__.py +0 -0
- itn/kinyarwanda/test/normalizer_test.py +194 -0
- itn/korean/__init__.py +3 -0
- itn/korean/data/__init__.py +13 -0
- itn/korean/data/currency.tsv +9 -0
- itn/korean/data/measure_units.tsv +61 -0
- itn/korean/data/months.tsv +12 -0
- itn/korean/data/numbers/__init__.py +13 -0
- itn/korean/data/numbers/digit.tsv +9 -0
- itn/korean/data/numbers/zero.tsv +1 -0
- itn/korean/data/ordinals/__init__.py +13 -0
- itn/korean/data/ordinals/counter_suffix.tsv +17 -0
- itn/korean/data/ordinals/digit.tsv +9 -0
- itn/korean/data/time/__init__.py +13 -0
- itn/korean/data/time/ten_prefix.tsv +4 -0
- itn/korean/data/time/time_hours.tsv +23 -0
- itn/korean/data/whitelist.tsv +35 -0
- itn/korean/inverse_normalizer.py +76 -0
- itn/korean/rules/__init__.py +3 -0
- itn/korean/rules/cardinal.py +44 -0
- itn/korean/rules/date.py +42 -0
- itn/korean/rules/decimal.py +43 -0
- itn/korean/rules/fraction.py +48 -0
- itn/korean/rules/measure.py +52 -0
- itn/korean/rules/money.py +34 -0
- itn/korean/rules/ordinal.py +49 -0
- itn/korean/rules/telephone.py +30 -0
- itn/korean/rules/time.py +61 -0
- itn/korean/rules/utils.py +22 -0
- tn/chinese/rules/preprocessor.py → itn/korean/rules/whitelist.py +10 -13
- itn/korean/rules/word.py +27 -0
- itn/korean/test/__init__.py +0 -0
- itn/korean/test/data/test_cases_cardinal.txt +37 -0
- itn/korean/test/data/test_cases_date.txt +15 -0
- itn/korean/test/data/test_cases_decimal.txt +14 -0
- itn/korean/test/data/test_cases_fraction.txt +29 -0
- itn/korean/test/data/test_cases_measure.txt +46 -0
- itn/korean/test/data/test_cases_money.txt +29 -0
- itn/korean/test/data/test_cases_ordinal.txt +17 -0
- itn/korean/test/data/test_cases_telephone.txt +7 -0
- itn/korean/test/data/test_cases_time.txt +15 -0
- itn/korean/test/test_inverse_normalizer.py +184 -0
- itn/main.py +2 -51
- itn/marathi/__init__.py +0 -0
- itn/marathi/data/date/eras.tsv +2 -0
- itn/marathi/data/date/months.tsv +12 -0
- itn/marathi/data/numbers/digit.tsv +9 -0
- itn/marathi/data/numbers/tens.tsv +90 -0
- itn/marathi/data/numbers/zero.tsv +1 -0
- itn/marathi/inverse_normalizer.py +56 -0
- itn/marathi/rules/__init__.py +0 -0
- itn/marathi/rules/cardinal.py +76 -0
- itn/marathi/rules/date.py +63 -0
- itn/marathi/rules/decimal.py +62 -0
- itn/marathi/rules/punctuation.py +29 -0
- itn/marathi/rules/time.py +69 -0
- itn/marathi/rules/utils.py +62 -0
- itn/marathi/rules/word.py +28 -0
- itn/marathi/test/__init__.py +0 -0
- itn/marathi/test/data/test_cases_cardinal.txt +33 -0
- itn/marathi/test/data/test_cases_date.txt +29 -0
- itn/marathi/test/data/test_cases_decimal.txt +29 -0
- itn/marathi/test/data/test_cases_time.txt +29 -0
- itn/marathi/test/data/test_cases_word.txt +11 -0
- itn/marathi/test/normalizer_test.py +164 -0
- itn/portuguese/__init__.py +0 -0
- itn/portuguese/data/currency_plural.tsv +5 -0
- itn/portuguese/data/currency_singular.tsv +5 -0
- itn/portuguese/data/electronic/domain.tsv +26 -0
- itn/portuguese/data/electronic/server_name.tsv +11 -0
- itn/portuguese/data/electronic/symbols.tsv +6 -0
- itn/portuguese/data/measurements_plural.tsv +56 -0
- itn/portuguese/data/measurements_singular.tsv +55 -0
- itn/portuguese/data/months.tsv +12 -0
- itn/portuguese/data/numbers/digit.tsv +11 -0
- itn/portuguese/data/numbers/hundreds.tsv +17 -0
- itn/portuguese/data/numbers/onehundred.tsv +1 -0
- itn/portuguese/data/numbers/teen.tsv +11 -0
- itn/portuguese/data/numbers/ties.tsv +8 -0
- itn/portuguese/data/numbers/twenties.tsv +9 -0
- itn/portuguese/data/numbers/zero.tsv +1 -0
- itn/portuguese/data/ordinals/digit.tsv +18 -0
- itn/portuguese/data/ordinals/hundreds.tsv +28 -0
- itn/portuguese/data/ordinals/ties.tsv +20 -0
- itn/portuguese/data/time/hours_to.tsv +23 -0
- itn/portuguese/data/time/minutes_to.tsv +59 -0
- itn/portuguese/data/time/time_suffix_am.tsv +2 -0
- itn/portuguese/data/time/time_suffix_pm.tsv +2 -0
- itn/portuguese/data/whitelist.tsv +5 -0
- itn/portuguese/inverse_normalizer.py +69 -0
- itn/portuguese/rules/__init__.py +0 -0
- itn/portuguese/rules/cardinal.py +117 -0
- itn/portuguese/rules/date.py +64 -0
- itn/portuguese/rules/decimal.py +76 -0
- itn/portuguese/rules/electronic.py +69 -0
- itn/portuguese/rules/measure.py +61 -0
- itn/portuguese/rules/money.py +89 -0
- itn/portuguese/rules/ordinal.py +56 -0
- itn/portuguese/rules/punctuation.py +32 -0
- itn/portuguese/rules/telephone.py +56 -0
- itn/portuguese/rules/time.py +107 -0
- itn/portuguese/rules/whitelist.py +32 -0
- itn/portuguese/rules/word.py +31 -0
- itn/portuguese/test/__init__.py +0 -0
- itn/portuguese/test/data/test_cases_cardinal.txt +70 -0
- itn/portuguese/test/data/test_cases_date.txt +15 -0
- itn/portuguese/test/data/test_cases_decimal.txt +26 -0
- itn/portuguese/test/data/test_cases_electronic.txt +13 -0
- itn/portuguese/test/data/test_cases_measure.txt +12 -0
- itn/portuguese/test/data/test_cases_money.txt +25 -0
- itn/portuguese/test/data/test_cases_ordinal.txt +19 -0
- itn/portuguese/test/data/test_cases_telephone.txt +27 -0
- itn/portuguese/test/data/test_cases_time.txt +19 -0
- itn/portuguese/test/data/test_cases_whitelist.txt +6 -0
- itn/portuguese/test/data/test_cases_word.txt +49 -0
- itn/portuguese/test/normalizer_test.py +156 -0
- itn/russian/__init__.py +0 -0
- itn/russian/data/currency/currency_plural.tsv +53 -0
- itn/russian/data/currency/currency_singular.tsv +48 -0
- itn/russian/data/electronic/domain.tsv +9 -0
- itn/russian/data/electronic/server_name.tsv +17 -0
- itn/russian/data/electronic/symbols.tsv +28 -0
- itn/russian/data/latin_to_cyrillic.tsv +38 -0
- itn/russian/data/measurements.tsv +321 -0
- itn/russian/data/months/abbr.tsv +12 -0
- itn/russian/data/months/abbr_to_name.tsv +62 -0
- itn/russian/data/months/numbers.tsv +21 -0
- itn/russian/data/numbers/1_cardinals_nominative.tsv +50 -0
- itn/russian/data/numbers/2_cardinals_genitive.tsv +45 -0
- itn/russian/data/numbers/3_cardinals_dative.tsv +45 -0
- itn/russian/data/numbers/4_cardinals_accusative.tsv +48 -0
- itn/russian/data/numbers/5_cardinals_instrumental.tsv +47 -0
- itn/russian/data/numbers/6_cardinals_prepositional.tsv +46 -0
- itn/russian/data/numbers/cardinal_endings.tsv +6 -0
- itn/russian/data/numbers/cardinals_alternatives.tsv +16 -0
- itn/russian/data/numbers/cardinals_nominative_case.tsv +37 -0
- itn/russian/data/numbers/decimal_delimiter.tsv +15 -0
- itn/russian/data/numbers/decimal_endings.tsv +37 -0
- itn/russian/data/numbers/digits_nominative_case.tsv +10 -0
- itn/russian/data/numbers/ordinal_endings.tsv +40 -0
- itn/russian/data/numbers/ordinals.tsv +752 -0
- itn/russian/data/numbers/quantity.tsv +16 -0
- itn/russian/data/time/increment_hour_cardinal.tsv +24 -0
- itn/russian/data/time/increment_hour_ordinal.tsv +24 -0
- itn/russian/data/time/minutes_to_hour.tsv +25 -0
- itn/russian/data/time/time_convert.tsv +12 -0
- itn/russian/data/whitelist.tsv +37 -0
- itn/russian/inverse_normalizer.py +71 -0
- itn/russian/rules/__init__.py +0 -0
- itn/russian/rules/cardinal.py +57 -0
- itn/russian/rules/date.py +78 -0
- itn/russian/rules/decimal.py +55 -0
- itn/russian/rules/electronic.py +67 -0
- itn/russian/rules/measure.py +45 -0
- itn/russian/rules/money.py +53 -0
- itn/russian/rules/ordinal.py +40 -0
- itn/russian/rules/punctuation.py +40 -0
- itn/russian/rules/telephone.py +45 -0
- itn/russian/rules/time.py +83 -0
- itn/russian/rules/utils.py +151 -0
- itn/russian/rules/whitelist.py +29 -0
- itn/russian/rules/word.py +30 -0
- itn/russian/test/__init__.py +0 -0
- itn/russian/test/data/test_cases_cardinal.txt +30 -0
- itn/russian/test/data/test_cases_date.txt +15 -0
- itn/russian/test/data/test_cases_decimal.txt +16 -0
- itn/russian/test/data/test_cases_electronic.txt +4 -0
- itn/russian/test/data/test_cases_measure.txt +16 -0
- itn/russian/test/data/test_cases_money.txt +9 -0
- itn/russian/test/data/test_cases_ordinal.txt +15 -0
- itn/russian/test/data/test_cases_ordinal_hard.txt +57 -0
- itn/russian/test/data/test_cases_telephone.txt +1 -0
- itn/russian/test/data/test_cases_time.txt +9 -0
- itn/russian/test/data/test_cases_whitelist.txt +4 -0
- itn/russian/test/data/test_cases_word.txt +50 -0
- itn/russian/test/normalizer_test.py +193 -0
- itn/spanish/__init__.py +0 -0
- itn/spanish/data/dates/months.tsv +12 -0
- itn/spanish/data/dates/months_cased.tsv +12 -0
- itn/spanish/data/dates/year_suffix.tsv +11 -0
- itn/spanish/data/dates/year_suffix_cased.tsv +11 -0
- itn/spanish/data/electronic/domain.tsv +27 -0
- itn/spanish/data/electronic/server_name.tsv +17 -0
- itn/spanish/data/electronic/symbols.tsv +22 -0
- itn/spanish/data/fractions/numbers_read_as_ordinals.tsv +28 -0
- itn/spanish/data/fractions/ordinal_exceptions.tsv +2 -0
- itn/spanish/data/measures/math_symbols.tsv +4 -0
- itn/spanish/data/measures/measurements_plural.tsv +34 -0
- itn/spanish/data/measures/measurements_singular.tsv +35 -0
- itn/spanish/data/money/currency_major_plural.tsv +43 -0
- itn/spanish/data/money/currency_major_plural_capitalized.tsv +75 -0
- itn/spanish/data/money/currency_major_plural_ext.tsv +196 -0
- itn/spanish/data/money/currency_major_singular.tsv +43 -0
- itn/spanish/data/money/currency_major_singular_capitalized.tsv +76 -0
- itn/spanish/data/money/currency_major_singular_ext.tsv +196 -0
- itn/spanish/data/money/currency_minor_plural.tsv +5 -0
- itn/spanish/data/money/currency_minor_plural_ext.tsv +62 -0
- itn/spanish/data/money/currency_minor_singular.tsv +5 -0
- itn/spanish/data/money/currency_minor_singular_ext.tsv +64 -0
- itn/spanish/data/months.tsv +12 -0
- itn/spanish/data/numbers/digit.tsv +11 -0
- itn/spanish/data/numbers/hundreds.tsv +18 -0
- itn/spanish/data/numbers/teen.tsv +10 -0
- itn/spanish/data/numbers/ties.tsv +8 -0
- itn/spanish/data/numbers/twenties.tsv +11 -0
- itn/spanish/data/numbers/zero.tsv +1 -0
- itn/spanish/data/ordinals/digit.tsv +22 -0
- itn/spanish/data/ordinals/digit_capitalized.tsv +22 -0
- itn/spanish/data/ordinals/hundreds.tsv +18 -0
- itn/spanish/data/ordinals/hundreds_capitalized.tsv +18 -0
- itn/spanish/data/ordinals/teen.tsv +60 -0
- itn/spanish/data/ordinals/teen_capitalized.tsv +60 -0
- itn/spanish/data/ordinals/ties.tsv +15 -0
- itn/spanish/data/ordinals/ties_capitalized.tsv +15 -0
- itn/spanish/data/ordinals/twenties.tsv +50 -0
- itn/spanish/data/ordinals/twenties_capitalized.tsv +50 -0
- itn/spanish/data/roman/digit.tsv +9 -0
- itn/spanish/data/roman/hundreds.tsv +9 -0
- itn/spanish/data/roman/thousands.tsv +3 -0
- itn/spanish/data/roman/ties.tsv +9 -0
- itn/spanish/data/time/minutes_to.tsv +6 -0
- itn/spanish/data/time/time_suffix.tsv +15 -0
- itn/spanish/data/time/time_suffix_cased.tsv +15 -0
- itn/spanish/data/time/time_to.tsv +25 -0
- itn/spanish/data/time/time_zone.tsv +42 -0
- itn/spanish/data/time/time_zone_cased.tsv +42 -0
- itn/spanish/data/time/time_zone_ext.tsv +105 -0
- itn/spanish/data/whitelist.tsv +47 -0
- itn/spanish/inverse_normalizer.py +77 -0
- itn/spanish/rules/__init__.py +0 -0
- itn/spanish/rules/cardinal.py +98 -0
- itn/spanish/rules/date.py +83 -0
- itn/spanish/rules/decimal.py +64 -0
- itn/spanish/rules/electronic.py +94 -0
- itn/spanish/rules/fraction.py +77 -0
- itn/spanish/rules/measure.py +84 -0
- itn/spanish/rules/money.py +95 -0
- itn/spanish/rules/ordinal.py +53 -0
- itn/spanish/rules/punctuation.py +30 -0
- itn/spanish/rules/telephone.py +70 -0
- itn/spanish/rules/time.py +112 -0
- itn/spanish/rules/utils.py +102 -0
- itn/spanish/rules/whitelist.py +47 -0
- itn/spanish/rules/word.py +32 -0
- itn/spanish/test/__init__.py +0 -0
- itn/spanish/test/data/test_cases_cardinal.txt +51 -0
- itn/spanish/test/data/test_cases_cardinal_cased.txt +30 -0
- itn/spanish/test/data/test_cases_date.txt +8 -0
- itn/spanish/test/data/test_cases_date_cased.txt +8 -0
- itn/spanish/test/data/test_cases_decimal.txt +29 -0
- itn/spanish/test/data/test_cases_decimal_cased.txt +6 -0
- itn/spanish/test/data/test_cases_electronic.txt +16 -0
- itn/spanish/test/data/test_cases_electronic_cased.txt +5 -0
- itn/spanish/test/data/test_cases_fraction.txt +12 -0
- itn/spanish/test/data/test_cases_measure.txt +20 -0
- itn/spanish/test/data/test_cases_measure_cased.txt +11 -0
- itn/spanish/test/data/test_cases_money.txt +24 -0
- itn/spanish/test/data/test_cases_money_cased.txt +6 -0
- itn/spanish/test/data/test_cases_ordinal.txt +30 -0
- itn/spanish/test/data/test_cases_ordinal_cased.txt +11 -0
- itn/spanish/test/data/test_cases_telephone.txt +9 -0
- itn/spanish/test/data/test_cases_telephone_cased.txt +6 -0
- itn/spanish/test/data/test_cases_time.txt +25 -0
- itn/spanish/test/data/test_cases_time_cased.txt +9 -0
- itn/spanish/test/data/test_cases_whitelist.txt +5 -0
- itn/spanish/test/data/test_cases_word.txt +49 -0
- itn/spanish/test/data/test_cases_word_cased.txt +11 -0
- itn/spanish/test/normalizer_test.py +143 -0
- itn/spanish_english/__init__.py +0 -0
- itn/spanish_english/data/en_whitelist.tsv +416 -0
- itn/spanish_english/data/es_whitelist.tsv +16 -0
- itn/spanish_english/data/measurements.tsv +3 -0
- itn/spanish_english/inverse_normalizer.py +99 -0
- itn/spanish_english/rules/__init__.py +0 -0
- itn/spanish_english/rules/english.py +230 -0
- itn/spanish_english/rules/prefixed.py +34 -0
- itn/spanish_english/rules/time.py +39 -0
- itn/spanish_english/rules/whitelist.py +38 -0
- itn/spanish_english/test/__init__.py +0 -0
- itn/spanish_english/test/data/test_cases_cardinal.txt +80 -0
- itn/spanish_english/test/data/test_cases_cardinal_cased.txt +60 -0
- itn/spanish_english/test/data/test_cases_date.txt +44 -0
- itn/spanish_english/test/data/test_cases_date_cased.txt +20 -0
- itn/spanish_english/test/data/test_cases_decimal.txt +92 -0
- itn/spanish_english/test/data/test_cases_decimal_cased.txt +19 -0
- itn/spanish_english/test/data/test_cases_electronic.txt +41 -0
- itn/spanish_english/test/data/test_cases_electronic_cased.txt +14 -0
- itn/spanish_english/test/data/test_cases_fraction.txt +12 -0
- itn/spanish_english/test/data/test_cases_measure.txt +132 -0
- itn/spanish_english/test/data/test_cases_measure_cased.txt +22 -0
- itn/spanish_english/test/data/test_cases_money.txt +76 -0
- itn/spanish_english/test/data/test_cases_money_cased.txt +16 -0
- itn/spanish_english/test/data/test_cases_ordinal.txt +64 -0
- itn/spanish_english/test/data/test_cases_ordinal_cased.txt +21 -0
- itn/spanish_english/test/data/test_cases_telephone.txt +32 -0
- itn/spanish_english/test/data/test_cases_telephone_cased.txt +19 -0
- itn/spanish_english/test/data/test_cases_time.txt +54 -0
- itn/spanish_english/test/data/test_cases_time_cased.txt +19 -0
- itn/spanish_english/test/data/test_cases_whitelist.txt +18 -0
- itn/spanish_english/test/data/test_cases_word.txt +104 -0
- itn/spanish_english/test/data/test_cases_word_cased.txt +17 -0
- itn/spanish_english/test/normalizer_test.py +207 -0
- itn/swedish/__init__.py +0 -0
- itn/swedish/data/dates/era_words.tsv +4 -0
- itn/swedish/data/dates/month_abbr.tsv +17 -0
- itn/swedish/data/dates/months.tsv +12 -0
- itn/swedish/data/electronic/domain.tsv +10 -0
- itn/swedish/data/electronic/server_name.tsv +29 -0
- itn/swedish/data/electronic/symbols.tsv +20 -0
- itn/swedish/data/numbers/digit.tsv +9 -0
- itn/swedish/data/numbers/millions.tsv +88 -0
- itn/swedish/data/numbers/teen.tsv +10 -0
- itn/swedish/data/numbers/ties.tsv +8 -0
- itn/swedish/data/numbers/zero.tsv +1 -0
- itn/swedish/data/ordinals/digit.tsv +9 -0
- itn/swedish/data/ordinals/teen.tsv +10 -0
- itn/swedish/data/ordinals/ties.tsv +8 -0
- itn/swedish/data/time/suffix.tsv +2 -0
- itn/swedish/data/time/time_zone.tsv +7 -0
- itn/swedish/data/whitelist.tsv +4 -0
- itn/swedish/inverse_normalizer.py +67 -0
- itn/swedish/rules/__init__.py +0 -0
- itn/swedish/rules/cardinal.py +88 -0
- itn/swedish/rules/date.py +91 -0
- itn/swedish/rules/decimal.py +68 -0
- itn/swedish/rules/electronic.py +69 -0
- itn/swedish/rules/fraction.py +90 -0
- itn/swedish/rules/ordinal.py +85 -0
- itn/swedish/rules/punctuation.py +29 -0
- itn/swedish/rules/telephone.py +59 -0
- itn/swedish/rules/time.py +99 -0
- itn/swedish/rules/utils.py +55 -0
- itn/swedish/rules/whitelist.py +29 -0
- itn/swedish/rules/word.py +29 -0
- itn/swedish/test/__init__.py +0 -0
- itn/swedish/test/data/test_cases_cardinal.txt +24 -0
- itn/swedish/test/data/test_cases_date.txt +9 -0
- itn/swedish/test/data/test_cases_decimal.txt +15 -0
- itn/swedish/test/data/test_cases_electronic.txt +12 -0
- itn/swedish/test/data/test_cases_fraction.txt +3 -0
- itn/swedish/test/data/test_cases_ordinal.txt +7 -0
- itn/swedish/test/data/test_cases_telephone.txt +2 -0
- itn/swedish/test/data/test_cases_time.txt +14 -0
- itn/swedish/test/data/test_cases_whitelist.txt +1 -0
- itn/swedish/test/data/test_cases_word.txt +48 -0
- itn/swedish/test/normalizer_test.py +122 -0
- itn/test/utils_test.py +17 -0
- itn/utils.py +40 -0
- itn/vietnamese/__init__.py +0 -0
- itn/vietnamese/data/currency.tsv +11 -0
- itn/vietnamese/data/electronic/domain.tsv +11 -0
- itn/vietnamese/data/electronic/server_name.tsv +16 -0
- itn/vietnamese/data/electronic/symbols.tsv +21 -0
- itn/vietnamese/data/electronic/url_symbols.tsv +8 -0
- itn/vietnamese/data/magnitudes.tsv +5 -0
- itn/vietnamese/data/math/symbols.tsv +10 -0
- itn/vietnamese/data/measurements.tsv +185 -0
- itn/vietnamese/data/months.tsv +14 -0
- itn/vietnamese/data/numbers/digit.tsv +10 -0
- itn/vietnamese/data/numbers/digit_special.tsv +3 -0
- itn/vietnamese/data/numbers/hundred.tsv +1 -0
- itn/vietnamese/data/numbers/teen.tsv +11 -0
- itn/vietnamese/data/numbers/thousands.tsv +6 -0
- itn/vietnamese/data/numbers/ties.tsv +9 -0
- itn/vietnamese/data/numbers/zero.tsv +1 -0
- itn/vietnamese/data/ordinals/digit.tsv +12 -0
- itn/vietnamese/data/time/hours.tsv +29 -0
- itn/vietnamese/data/time/hours_to.tsv +25 -0
- itn/vietnamese/data/time/hours_to_night.tsv +12 -0
- itn/vietnamese/data/time/minutes.tsv +118 -0
- itn/vietnamese/data/time/minutes_to.tsv +59 -0
- itn/vietnamese/data/time/time_suffix.tsv +8 -0
- itn/vietnamese/data/time/time_zone.tsv +7 -0
- itn/vietnamese/data/time/zero_prefix.tsv +3 -0
- itn/vietnamese/data/whitelist.tsv +0 -0
- itn/vietnamese/inverse_normalizer.py +73 -0
- itn/vietnamese/rules/__init__.py +0 -0
- itn/vietnamese/rules/cardinal.py +99 -0
- itn/vietnamese/rules/date.py +76 -0
- itn/vietnamese/rules/decimal.py +72 -0
- itn/vietnamese/rules/electronic.py +60 -0
- itn/vietnamese/rules/fraction.py +41 -0
- itn/vietnamese/rules/measure.py +60 -0
- itn/vietnamese/rules/money.py +54 -0
- itn/vietnamese/rules/ordinal.py +32 -0
- itn/vietnamese/rules/punctuation.py +29 -0
- itn/vietnamese/rules/telephone.py +57 -0
- itn/vietnamese/rules/time.py +83 -0
- itn/vietnamese/rules/whitelist.py +30 -0
- itn/vietnamese/rules/word.py +28 -0
- itn/vietnamese/test/__init__.py +0 -0
- itn/vietnamese/test/data/test_cases_cardinal.txt +142 -0
- itn/vietnamese/test/data/test_cases_date.txt +69 -0
- itn/vietnamese/test/data/test_cases_decimal.txt +74 -0
- itn/vietnamese/test/data/test_cases_electronic.txt +18 -0
- itn/vietnamese/test/data/test_cases_fraction.txt +14 -0
- itn/vietnamese/test/data/test_cases_measure.txt +35 -0
- itn/vietnamese/test/data/test_cases_money.txt +15 -0
- itn/vietnamese/test/data/test_cases_ordinal.txt +12 -0
- itn/vietnamese/test/data/test_cases_telephone.txt +25 -0
- itn/vietnamese/test/data/test_cases_time.txt +31 -0
- itn/vietnamese/test/data/test_cases_whitelist.txt +1 -0
- itn/vietnamese/test/data/test_cases_word.txt +17 -0
- itn/vietnamese/test/normalizer_test.py +144 -0
- tn/README.md +20 -8
- tn/__init__.py +3 -0
- tn/_version.py +9 -19
- tn/alignment.py +254 -0
- tn/arabic/__init__.py +0 -0
- tn/arabic/data/measure/measurements.tsv +14 -0
- tn/arabic/data/money/currency_major.tsv +24 -0
- tn/arabic/data/money/currency_minor_plural.tsv +4 -0
- tn/arabic/data/money/currency_minor_singular.tsv +3 -0
- tn/arabic/data/money/local_currency.tsv +20 -0
- tn/arabic/data/number/digit.tsv +9 -0
- tn/arabic/data/number/digit_100.tsv +7 -0
- tn/arabic/data/number/digit_1000.tsv +7 -0
- tn/arabic/data/number/flops.tsv +72 -0
- tn/arabic/data/number/fraction.tsv +18 -0
- tn/arabic/data/number/fraction_dual.tsv +9 -0
- tn/arabic/data/number/fraction_plural.tsv +8 -0
- tn/arabic/data/number/fraction_singular.tsv +9 -0
- tn/arabic/data/number/quantities.tsv +6 -0
- tn/arabic/data/number/teens.tsv +9 -0
- tn/arabic/data/number/tens.tsv +10 -0
- tn/arabic/data/punctuation/symbol.tsv +23 -0
- tn/arabic/normalizer.py +61 -0
- tn/arabic/rules/__init__.py +0 -0
- tn/arabic/rules/cardinal.py +86 -0
- tn/arabic/rules/decimal.py +69 -0
- tn/arabic/rules/fraction.py +79 -0
- tn/arabic/rules/measure.py +80 -0
- tn/arabic/rules/money.py +131 -0
- tn/arabic/rules/punctuation.py +44 -0
- tn/arabic/rules/word.py +32 -0
- tn/arabic/test/__init__.py +0 -0
- tn/arabic/test/data/test_cases_cardinal.txt +27 -0
- tn/arabic/test/data/test_cases_decimal.txt +30 -0
- tn/arabic/test/data/test_cases_fraction.txt +20 -0
- tn/arabic/test/data/test_cases_measure.txt +14 -0
- tn/arabic/test/data/test_cases_money.txt +27 -0
- tn/arabic/test/data/test_cases_whitelist.txt +6 -0
- tn/arabic/test/normalizer_test.py +162 -0
- tn/armenian/__init__.py +0 -0
- tn/armenian/data/currency.tsv +52 -0
- tn/armenian/data/measurement_dates.tsv +8 -0
- tn/armenian/data/measurements.tsv +44 -0
- tn/armenian/data/numbers/digit.tsv +9 -0
- tn/armenian/data/numbers/quantities.tsv +3 -0
- tn/armenian/data/numbers/ties.tsv +8 -0
- tn/armenian/data/time/hours.tsv +26 -0
- tn/armenian/data/time/minutes.tsv +61 -0
- tn/armenian/data/whitelist.tsv +14 -0
- tn/armenian/normalizer.py +65 -0
- tn/armenian/rules/__init__.py +0 -0
- tn/armenian/rules/cardinal.py +75 -0
- tn/armenian/rules/decimal.py +63 -0
- tn/armenian/rules/fraction.py +36 -0
- tn/armenian/rules/measure.py +49 -0
- tn/armenian/rules/money.py +38 -0
- tn/armenian/rules/ordinal.py +59 -0
- tn/armenian/rules/punctuation.py +31 -0
- tn/armenian/rules/time.py +34 -0
- tn/armenian/rules/utils.py +60 -0
- tn/armenian/rules/whitelist.py +31 -0
- tn/armenian/rules/word.py +28 -0
- tn/armenian/test/__init__.py +0 -0
- tn/armenian/test/data/test_cases_cardinal.txt +12 -0
- tn/armenian/test/data/test_cases_decimal.txt +9 -0
- tn/armenian/test/data/test_cases_fraction.txt +10 -0
- tn/armenian/test/data/test_cases_measure.txt +9 -0
- tn/armenian/test/data/test_cases_money.txt +10 -0
- tn/armenian/test/data/test_cases_ordinal.txt +11 -0
- tn/armenian/test/data/test_cases_time.txt +9 -0
- tn/armenian/test/data/test_cases_whitelist.txt +14 -0
- tn/armenian/test/data/test_cases_word.txt +50 -0
- tn/armenian/test/normalizer_test.py +156 -0
- tn/cache.py +822 -0
- tn/chinese/data/cardinal/phone.tsv +28 -0
- tn/chinese/data/math/operator_unambiguous.tsv +6 -0
- tn/chinese/normalizer.py +45 -37
- tn/chinese/rules/cardinal.py +21 -19
- tn/chinese/rules/date.py +26 -18
- tn/chinese/rules/fraction.py +4 -12
- tn/chinese/rules/math.py +10 -5
- tn/chinese/rules/measure.py +18 -10
- tn/chinese/rules/money.py +5 -11
- tn/chinese/rules/postprocessor.py +23 -8
- tn/chinese/rules/range.py +29 -0
- tn/chinese/rules/sport.py +8 -6
- tn/chinese/rules/time.py +22 -19
- tn/chinese/rules/whitelist.py +7 -6
- tn/chinese/test/alignment_test.py +154 -0
- tn/chinese/test/cache_safety_test.py +505 -0
- tn/chinese/test/cache_test.py +192 -0
- tn/chinese/test/data/normalizer.txt +0 -2
- tn/chinese/test/data/normalizer_tag_oov.txt +2 -0
- tn/chinese/test/normalizer_test.py +57 -4
- tn/chinese/test/postprocessor_test.py +16 -0
- tn/chinese/test/processor_test.py +496 -0
- tn/chinese/test/token_parser_test.py +35 -1
- tn/cli.py +278 -0
- tn/english/data/measure/unit_alternatives.tsv +0 -1
- tn/english/normalizer.py +46 -42
- tn/english/rules/cardinal.py +37 -61
- tn/english/rules/date.py +84 -166
- tn/english/rules/decimal.py +32 -79
- tn/english/rules/electronic.py +56 -66
- tn/english/rules/fraction.py +33 -23
- tn/english/rules/measure.py +107 -148
- tn/english/rules/money.py +46 -42
- tn/english/rules/ordinal.py +14 -30
- tn/english/rules/punctuation.py +15 -21
- tn/english/rules/range.py +26 -40
- tn/english/rules/roman.py +27 -45
- tn/english/rules/serial.py +88 -0
- tn/english/rules/telephone.py +44 -80
- tn/english/rules/time.py +54 -102
- tn/english/rules/whitelist.py +9 -19
- tn/english/rules/word.py +9 -0
- tn/english/test/data/date.txt +1 -1
- tn/english/test/data/normalizer.txt +4 -1
- tn/english/test/electronic_test.py +27 -0
- tn/english/test/nbest_test.py +87 -0
- tn/english/test/normalizer_test.py +181 -3
- tn/french/__init__.py +0 -0
- tn/french/data/dates/eras.tsv +8 -0
- tn/french/data/dates/months.tsv +12 -0
- tn/french/data/fractions/irregular_denominators.tsv +7 -0
- tn/french/data/numbers/digit.tsv +9 -0
- tn/french/data/numbers/eighty.tsv +1 -0
- tn/french/data/numbers/ninety.tsv +1 -0
- tn/french/data/numbers/quantities.tsv +10 -0
- tn/french/data/numbers/seventy.tsv +1 -0
- tn/french/data/numbers/teens.tsv +9 -0
- tn/french/data/numbers/ten.tsv +1 -0
- tn/french/data/numbers/tens_simple.tsv +5 -0
- tn/french/data/numbers/zero.tsv +1 -0
- tn/french/data/ordinals/irregular_numbers.tsv +30 -0
- tn/french/data/ordinals/suffixes.tsv +6 -0
- tn/french/data/punctuation/symbol.tsv +23 -0
- tn/french/data/whitelist.tsv +13 -0
- tn/french/normalizer.py +66 -0
- tn/french/rules/__init__.py +0 -0
- tn/french/rules/cardinal.py +110 -0
- tn/french/rules/date.py +68 -0
- tn/french/rules/decimal.py +62 -0
- tn/french/rules/fraction.py +60 -0
- tn/french/rules/ordinal.py +56 -0
- tn/french/rules/punctuation.py +43 -0
- tn/french/rules/whitelist.py +29 -0
- tn/french/rules/word.py +29 -0
- tn/french/test/__init__.py +0 -0
- tn/french/test/data/test_cases_cardinal.txt +14 -0
- tn/french/test/data/test_cases_date.txt +13 -0
- tn/french/test/data/test_cases_decimal.txt +7 -0
- tn/french/test/data/test_cases_fraction.txt +12 -0
- tn/french/test/data/test_cases_ordinal.txt +15 -0
- tn/french/test/data/test_cases_whitelist.txt +7 -0
- tn/french/test/data/test_cases_word.txt +49 -0
- tn/french/test/normalizer_test.py +157 -0
- tn/german/__init__.py +0 -0
- tn/german/data/__init__.py +13 -0
- tn/german/data/electronic/__init__.py +13 -0
- tn/german/data/electronic/domain.tsv +844 -0
- tn/german/data/electronic/server_name.tsv +17 -0
- tn/german/data/electronic/symbols.tsv +20 -0
- tn/german/data/fractions.tsv +30 -0
- tn/german/data/measure/__init__.py +13 -0
- tn/german/data/measure/measurements.tsv +82 -0
- tn/german/data/measure/suppletive.tsv +10 -0
- tn/german/data/money/__init__.py +13 -0
- tn/german/data/money/currency.tsv +25 -0
- tn/german/data/money/currency_minor_plural.tsv +3 -0
- tn/german/data/money/currency_minor_singular.tsv +3 -0
- tn/german/data/months/__init__.py +13 -0
- tn/german/data/months/abbr_to_name.tsv +13 -0
- tn/german/data/months/numbers.tsv +24 -0
- tn/german/data/numbers/__init__.py +13 -0
- tn/german/data/numbers/digit.tsv +8 -0
- tn/german/data/numbers/ones.tsv +3 -0
- tn/german/data/numbers/quantities.tsv +12 -0
- tn/german/data/numbers/teen.tsv +10 -0
- tn/german/data/numbers/ties.tsv +8 -0
- tn/german/data/numbers/zero.tsv +1 -0
- tn/german/data/ordinals/__init__.py +13 -0
- tn/german/data/ordinals/digit.tsv +9 -0
- tn/german/data/ordinals/thousands.tsv +4 -0
- tn/german/data/ordinals/ties.tsv +8 -0
- tn/german/data/time/__init__.py +13 -0
- tn/german/data/time/hour_to.tsv +12 -0
- tn/german/data/time/hour_to_night.tsv +13 -0
- tn/german/data/time/minute_to.tsv +59 -0
- tn/german/data/time/time_zone.tsv +7 -0
- tn/german/data/whitelist.tsv +21 -0
- tn/german/normalizer.py +77 -0
- tn/german/rules/__init__.py +0 -0
- tn/german/rules/cardinal.py +120 -0
- tn/german/rules/date.py +98 -0
- tn/german/rules/decimal.py +71 -0
- tn/german/rules/electronic.py +67 -0
- tn/german/rules/fraction.py +65 -0
- tn/german/rules/measure.py +117 -0
- tn/german/rules/money.py +144 -0
- tn/german/rules/ordinal.py +46 -0
- tn/german/rules/punctuation.py +39 -0
- tn/german/rules/telephone.py +48 -0
- tn/german/rules/time.py +123 -0
- tn/german/rules/whitelist.py +34 -0
- tn/german/rules/word.py +34 -0
- tn/german/test/__init__.py +0 -0
- tn/german/test/calendar_test.py +168 -0
- tn/german/test/data/test_cases_cardinal.txt +50 -0
- tn/german/test/data/test_cases_date.txt +15 -0
- tn/german/test/data/test_cases_decimal.txt +10 -0
- tn/german/test/data/test_cases_electronic.txt +19 -0
- tn/german/test/data/test_cases_fraction.txt +34 -0
- tn/german/test/data/test_cases_measure.txt +29 -0
- tn/german/test/data/test_cases_money.txt +21 -0
- tn/german/test/data/test_cases_normalize_with_audio.txt +131 -0
- tn/german/test/data/test_cases_ordinal.txt +19 -0
- tn/german/test/data/test_cases_telephone.txt +4 -0
- tn/german/test/data/test_cases_time.txt +28 -0
- tn/german/test/data/test_cases_whitelist.txt +6 -0
- tn/german/test/data/test_cases_word.txt +49 -0
- tn/german/test/financial_test.py +167 -0
- tn/german/test/normalizer_test.py +136 -0
- tn/german/test/numeric_test.py +185 -0
- tn/german/test/text_test.py +235 -0
- tn/hebrew/__init__.py +0 -0
- tn/hebrew/data/measurements.tsv +29 -0
- tn/hebrew/data/prefix.tsv +17 -0
- tn/hebrew/data/spaced_measurements.tsv +16 -0
- tn/hebrew/data/whitelist.tsv +9 -0
- tn/hebrew/normalizer.py +58 -0
- tn/hebrew/rules/__init__.py +0 -0
- tn/hebrew/rules/cardinal.py +43 -0
- tn/hebrew/rules/date.py +84 -0
- tn/hebrew/rules/decimal.py +71 -0
- tn/hebrew/rules/measure.py +56 -0
- tn/hebrew/rules/numbers.py +135 -0
- tn/hebrew/rules/punctuation.py +29 -0
- tn/hebrew/rules/time.py +94 -0
- tn/hebrew/rules/utils.py +59 -0
- tn/hebrew/rules/whitelist.py +26 -0
- tn/hebrew/rules/word.py +27 -0
- tn/hebrew/test/__init__.py +0 -0
- tn/hebrew/test/normalizer_test.py +245 -0
- tn/hindi/__init__.py +0 -0
- tn/hindi/data/address/cities.tsv +36 -0
- tn/hindi/data/address/context.tsv +47 -0
- tn/hindi/data/address/letters.tsv +26 -0
- tn/hindi/data/address/special_characters.tsv +2 -0
- tn/hindi/data/address/states.tsv +36 -0
- tn/hindi/data/date/days.tsv +62 -0
- tn/hindi/data/date/months.tsv +24 -0
- tn/hindi/data/date/prefixes.tsv +4 -0
- tn/hindi/data/date/suffixes.tsv +10 -0
- tn/hindi/data/date/year_suffix.tsv +2 -0
- tn/hindi/data/electronic/domain.tsv +24 -0
- tn/hindi/data/electronic/elements.tsv +132 -0
- tn/hindi/data/electronic/file_extensions.tsv +34 -0
- tn/hindi/data/electronic/protocols.tsv +5 -0
- tn/hindi/data/electronic/subscript_digit.tsv +10 -0
- tn/hindi/data/electronic/symbol_classes.tsv +16 -0
- tn/hindi/data/electronic/symbols.tsv +36 -0
- tn/hindi/data/fraction/common_fractions.tsv +10 -0
- tn/hindi/data/measure/quarterly_units_list.tsv +5 -0
- tn/hindi/data/measure/quarterly_units_map.tsv +10 -0
- tn/hindi/data/measure/unit.tsv +146 -0
- tn/hindi/data/measure/unit_year_formal.tsv +1 -0
- tn/hindi/data/money/currency.tsv +9 -0
- tn/hindi/data/money/currency_singular.tsv +9 -0
- tn/hindi/data/money/major_minor_currencies.tsv +10 -0
- tn/hindi/data/numbers/digit.tsv +18 -0
- tn/hindi/data/numbers/hundred.tsv +1 -0
- tn/hindi/data/numbers/teens_and_ties.tsv +90 -0
- tn/hindi/data/numbers/teens_and_ties_en.tsv +90 -0
- tn/hindi/data/numbers/thousands.tsv +8 -0
- tn/hindi/data/numbers/zero.tsv +2 -0
- tn/hindi/data/ordinal/en_to_hi_digit.tsv +10 -0
- tn/hindi/data/ordinal/exceptions.tsv +50 -0
- tn/hindi/data/ordinal/suffixes.tsv +3 -0
- tn/hindi/data/ordinal/suffixes_map.tsv +1 -0
- tn/hindi/data/roman/roman_ordinal_exceptions.tsv +10 -0
- tn/hindi/data/roman/roman_to_spoken.tsv +100 -0
- tn/hindi/data/serial/chars.tsv +68 -0
- tn/hindi/data/serial/power_special.tsv +4 -0
- tn/hindi/data/serial/special_symbols.tsv +4 -0
- tn/hindi/data/telephone/credit_context.tsv +3 -0
- tn/hindi/data/telephone/landline_context.tsv +5 -0
- tn/hindi/data/telephone/mobile_context.tsv +4 -0
- tn/hindi/data/telephone/number.tsv +10 -0
- tn/hindi/data/telephone/pincode_context.tsv +4 -0
- tn/hindi/data/time/hours.tsv +50 -0
- tn/hindi/data/time/minutes.tsv +120 -0
- tn/hindi/data/time/seconds.tsv +120 -0
- tn/hindi/data/whitelist/abbreviations.tsv +7 -0
- tn/hindi/data/whitelist/paune_mappings.tsv +100 -0
- tn/hindi/normalizer.py +95 -0
- tn/hindi/rules/__init__.py +0 -0
- tn/hindi/rules/address.py +97 -0
- tn/hindi/rules/cardinal.py +117 -0
- tn/hindi/rules/date.py +135 -0
- tn/hindi/rules/decimal.py +42 -0
- tn/hindi/rules/electronic.py +145 -0
- tn/hindi/rules/fraction.py +62 -0
- tn/hindi/rules/measure.py +110 -0
- tn/hindi/rules/money.py +104 -0
- tn/hindi/rules/ordinal.py +37 -0
- tn/hindi/rules/punctuation.py +50 -0
- tn/hindi/rules/roman.py +70 -0
- tn/hindi/rules/serial.py +82 -0
- tn/hindi/rules/telephone.py +98 -0
- tn/hindi/rules/time.py +62 -0
- tn/hindi/rules/utils.py +54 -0
- tn/hindi/rules/whitelist.py +37 -0
- tn/hindi/rules/word.py +31 -0
- tn/hindi/test/__init__.py +0 -0
- tn/hindi/test/data/test_cases_address.txt +52 -0
- tn/hindi/test/data/test_cases_cardinal.txt +161 -0
- tn/hindi/test/data/test_cases_date.txt +39 -0
- tn/hindi/test/data/test_cases_decimal.txt +24 -0
- tn/hindi/test/data/test_cases_electronic.txt +65 -0
- tn/hindi/test/data/test_cases_fraction.txt +27 -0
- tn/hindi/test/data/test_cases_measure.txt +69 -0
- tn/hindi/test/data/test_cases_money.txt +144 -0
- tn/hindi/test/data/test_cases_ordinal.txt +72 -0
- tn/hindi/test/data/test_cases_roman.txt +23 -0
- tn/hindi/test/data/test_cases_serial.txt +31 -0
- tn/hindi/test/data/test_cases_telephone.txt +25 -0
- tn/hindi/test/data/test_cases_time.txt +18 -0
- tn/hindi/test/data/test_cases_whitelist.txt +14 -0
- tn/hindi/test/data/test_cases_word.txt +20 -0
- tn/hindi/test/normalizer_test.py +137 -0
- tn/hungarian/__init__.py +0 -0
- tn/hungarian/data/dates/days.tsv +31 -0
- tn/hungarian/data/dates/month_abbr.tsv +12 -0
- tn/hungarian/data/dates/months.tsv +12 -0
- tn/hungarian/data/dates/months_roman.tsv +12 -0
- tn/hungarian/data/electronic/domain.tsv +15 -0
- tn/hungarian/data/electronic/server_name.tsv +15 -0
- tn/hungarian/data/electronic/symbols.tsv +23 -0
- tn/hungarian/data/fractions/fraction_symbols.tsv +19 -0
- tn/hungarian/data/inflection/endings.tsv +51 -0
- tn/hungarian/data/inflection/plural_endings.tsv +39 -0
- tn/hungarian/data/inflection/word_endings.tsv +1 -0
- tn/hungarian/data/measures/measurements.tsv +52 -0
- tn/hungarian/data/money/alphabet.tsv +44 -0
- tn/hungarian/data/money/currency.tsv +11 -0
- tn/hungarian/data/money/currency_minor.tsv +11 -0
- tn/hungarian/data/number/digit.tsv +9 -0
- tn/hungarian/data/number/digit_inline.tsv +9 -0
- tn/hungarian/data/number/quantities.tsv +21 -0
- tn/hungarian/data/number/tens.tsv +9 -0
- tn/hungarian/data/number/tens_inline.tsv +9 -0
- tn/hungarian/data/number/zero.tsv +1 -0
- tn/hungarian/data/ordinals/endings.tsv +23 -0
- tn/hungarian/data/ordinals/exceptional.tsv +4 -0
- tn/hungarian/data/telephone/area_codes.tsv +71 -0
- tn/hungarian/data/telephone/country_codes.tsv +213 -0
- tn/hungarian/data/telephone/telephone_abbr.tsv +3 -0
- tn/hungarian/data/time/time_zone.tsv +9 -0
- tn/hungarian/data/whitelist.tsv +24 -0
- tn/hungarian/data/whitelist_inflect.tsv +39 -0
- tn/hungarian/data/whitelist_inflect_sg.tsv +14 -0
- tn/hungarian/normalizer.py +80 -0
- tn/hungarian/rules/__init__.py +0 -0
- tn/hungarian/rules/cardinal.py +98 -0
- tn/hungarian/rules/date.py +104 -0
- tn/hungarian/rules/decimal.py +85 -0
- tn/hungarian/rules/electronic.py +69 -0
- tn/hungarian/rules/fraction.py +55 -0
- tn/hungarian/rules/measure.py +58 -0
- tn/hungarian/rules/money.py +81 -0
- tn/hungarian/rules/ordinal.py +41 -0
- tn/hungarian/rules/punctuation.py +46 -0
- tn/hungarian/rules/telephone.py +71 -0
- tn/hungarian/rules/time.py +106 -0
- tn/hungarian/rules/utils.py +156 -0
- tn/hungarian/rules/whitelist.py +32 -0
- tn/hungarian/rules/word.py +38 -0
- tn/hungarian/test/__init__.py +0 -0
- tn/hungarian/test/data/test_cases_cardinal.txt +69 -0
- tn/hungarian/test/data/test_cases_date.txt +17 -0
- tn/hungarian/test/data/test_cases_decimal.txt +6 -0
- tn/hungarian/test/data/test_cases_electronic.txt +8 -0
- tn/hungarian/test/data/test_cases_fraction.txt +4 -0
- tn/hungarian/test/data/test_cases_measure.txt +6 -0
- tn/hungarian/test/data/test_cases_money.txt +9 -0
- tn/hungarian/test/data/test_cases_ordinal.txt +5 -0
- tn/hungarian/test/data/test_cases_telephone.txt +9 -0
- tn/hungarian/test/data/test_cases_time.txt +10 -0
- tn/hungarian/test/data/test_cases_whitelist.txt +10 -0
- tn/hungarian/test/data/test_cases_word.txt +52 -0
- tn/hungarian/test/normalizer_test.py +210 -0
- tn/italian/__init__.py +0 -0
- tn/italian/data/electronic/domain.tsv +14 -0
- tn/italian/data/electronic/server_name.tsv +11 -0
- tn/italian/data/electronic/symbols.tsv +22 -0
- tn/italian/data/measure/measurements.tsv +64 -0
- tn/italian/data/measure/suppletive.tsv +11 -0
- tn/italian/data/money/currency_major.tsv +11 -0
- tn/italian/data/money/currency_minor.tsv +11 -0
- tn/italian/data/money/currency_plural_fem.tsv +3 -0
- tn/italian/data/money/currency_plural_masc.tsv +11 -0
- tn/italian/data/numbers/digit.tsv +9 -0
- tn/italian/data/numbers/hundreds.tsv +8 -0
- tn/italian/data/numbers/quantities.tsv +2 -0
- tn/italian/data/numbers/teen.tsv +10 -0
- tn/italian/data/numbers/tens.tsv +8 -0
- tn/italian/data/numbers/tens_eight.tsv +8 -0
- tn/italian/data/numbers/tens_one.tsv +8 -0
- tn/italian/data/numbers/zero.tsv +1 -0
- tn/italian/data/whitelist/whitelist.tsv +65 -0
- tn/italian/normalizer.py +60 -0
- tn/italian/rules/__init__.py +0 -0
- tn/italian/rules/cardinal.py +99 -0
- tn/italian/rules/decimal.py +82 -0
- tn/italian/rules/electronic.py +80 -0
- tn/italian/rules/measure.py +73 -0
- tn/italian/rules/money.py +92 -0
- tn/italian/rules/punctuation.py +30 -0
- tn/italian/rules/time.py +61 -0
- tn/italian/rules/utils.py +74 -0
- tn/italian/rules/whitelist.py +27 -0
- tn/italian/rules/word.py +28 -0
- tn/italian/test/__init__.py +0 -0
- tn/italian/test/data/test_cases_cardinal.txt +13 -0
- tn/italian/test/data/test_cases_decimal.txt +4 -0
- tn/italian/test/data/test_cases_electronic.txt +13 -0
- tn/italian/test/data/test_cases_measure.txt +7 -0
- tn/italian/test/data/test_cases_money.txt +5 -0
- tn/italian/test/data/test_cases_time.txt +6 -0
- tn/italian/test/data/test_cases_whitelist.txt +4 -0
- tn/italian/test/normalizer_test.py +209 -0
- tn/japanese/normalizer.py +56 -48
- tn/japanese/rules/cardinal.py +35 -54
- tn/japanese/rules/date.py +33 -18
- tn/japanese/rules/fraction.py +8 -14
- tn/japanese/rules/math.py +14 -6
- tn/japanese/rules/measure.py +10 -4
- tn/japanese/rules/money.py +9 -14
- tn/japanese/rules/postprocessor.py +5 -1
- tn/japanese/rules/preprocessor.py +2 -2
- tn/japanese/rules/range.py +32 -0
- tn/japanese/rules/sport.py +18 -10
- tn/japanese/rules/time.py +25 -14
- tn/japanese/rules/transliteration.py +12 -5
- tn/japanese/rules/whitelist.py +11 -7
- tn/japanese/test/normalizer_test.py +107 -4
- tn/kinyarwanda/__init__.py +0 -0
- tn/kinyarwanda/data/numbers/digit.tsv +9 -0
- tn/kinyarwanda/data/numbers/digit_bi.tsv +9 -0
- tn/kinyarwanda/data/numbers/digit_e.tsv +9 -0
- tn/kinyarwanda/data/numbers/hundreds.tsv +9 -0
- tn/kinyarwanda/data/numbers/tens.tsv +8 -0
- tn/kinyarwanda/data/numbers/zero.tsv +1 -0
- tn/kinyarwanda/data/time/hours.tsv +12 -0
- tn/kinyarwanda/data/time/minutes.tsv +60 -0
- tn/kinyarwanda/data/whitelist.tsv +175 -0
- tn/kinyarwanda/normalizer.py +56 -0
- tn/kinyarwanda/rules/__init__.py +0 -0
- tn/kinyarwanda/rules/cardinal.py +117 -0
- tn/kinyarwanda/rules/punctuation.py +31 -0
- tn/kinyarwanda/rules/time.py +40 -0
- tn/kinyarwanda/rules/utils.py +43 -0
- tn/kinyarwanda/rules/whitelist.py +28 -0
- tn/kinyarwanda/rules/word.py +29 -0
- tn/kinyarwanda/test/__init__.py +0 -0
- tn/kinyarwanda/test/data/test_cases_cardinal.txt +57 -0
- tn/kinyarwanda/test/data/test_cases_time.txt +14 -0
- tn/kinyarwanda/test/data/test_cases_whitelist.txt +3 -0
- tn/kinyarwanda/test/data/test_cases_word.txt +26 -0
- tn/kinyarwanda/test/normalizer_test.py +198 -0
- tn/korean/__init__.py +16 -0
- tn/korean/data/date/exceptions.tsv +2 -0
- tn/korean/data/date/week.tsv +8 -0
- tn/korean/data/electronic/cc_cues.tsv +11 -0
- tn/korean/data/electronic/domain.tsv +28 -0
- tn/korean/data/electronic/extensions.tsv +6 -0
- tn/korean/data/electronic/symbol.tsv +12 -0
- tn/korean/data/measure/unit.tsv +18 -0
- tn/korean/data/money/currency_major.tsv +22 -0
- tn/korean/data/number/counter_suffix.tsv +16 -0
- tn/korean/data/number/digit.tsv +9 -0
- tn/korean/data/number/native_ones.tsv +9 -0
- tn/korean/data/number/tens.tsv +9 -0
- tn/korean/data/number/zero.tsv +1 -0
- tn/korean/data/ordinal/digit.tsv +8 -0
- tn/korean/data/ordinal/exceptions.tsv +5 -0
- tn/korean/data/ordinal/tens.tsv +2 -0
- tn/korean/data/ordinal/tens_prefix.tsv +3 -0
- tn/korean/data/time/division.tsv +7 -0
- tn/korean/data/time/hour.tsv +12 -0
- tn/korean/data/whitelist.tsv +39 -0
- tn/korean/normalizer.py +79 -0
- tn/korean/rules/__init__.py +3 -0
- tn/korean/rules/cardinal.py +91 -0
- tn/korean/rules/date.py +129 -0
- tn/korean/rules/decimal.py +53 -0
- tn/korean/rules/electronic.py +84 -0
- tn/korean/rules/fraction.py +81 -0
- tn/korean/rules/measure.py +82 -0
- tn/korean/rules/money.py +60 -0
- tn/korean/rules/ordinal.py +62 -0
- tn/korean/rules/punctuation.py +16 -0
- tn/korean/rules/telephone.py +40 -0
- tn/korean/rules/time.py +81 -0
- tn/korean/rules/whitelist.py +17 -0
- tn/korean/rules/word.py +18 -0
- tn/korean/test/__init__.py +3 -0
- tn/korean/test/data/test_cases_cardinal.txt +68 -0
- tn/korean/test/data/test_cases_date.txt +29 -0
- tn/korean/test/data/test_cases_decimal.txt +28 -0
- tn/korean/test/data/test_cases_electronic.txt +20 -0
- tn/korean/test/data/test_cases_fraction.txt +28 -0
- tn/korean/test/data/test_cases_measure.txt +40 -0
- tn/korean/test/data/test_cases_money.txt +64 -0
- tn/korean/test/data/test_cases_ordinal.txt +54 -0
- tn/korean/test/data/test_cases_telephone.txt +39 -0
- tn/korean/test/data/test_cases_time.txt +25 -0
- tn/korean/test/date_test.py +99 -0
- tn/korean/test/decimal_test.py +67 -0
- tn/korean/test/electronic_test.py +58 -0
- tn/korean/test/fraction_test.py +89 -0
- tn/korean/test/measure_test.py +90 -0
- tn/korean/test/money_test.py +92 -0
- tn/korean/test/normalizer_test.py +146 -0
- tn/korean/test/ordinal_test.py +78 -0
- tn/korean/test/telephone_test.py +40 -0
- tn/korean/test/time_test.py +51 -0
- tn/main.py +2 -52
- tn/marathi/__init__.py +0 -0
- tn/marathi/data/date/eras.tsv +2 -0
- tn/marathi/data/date/months.tsv +12 -0
- tn/marathi/data/numbers/digit.tsv +9 -0
- tn/marathi/data/numbers/tens.tsv +90 -0
- tn/marathi/data/numbers/zero.tsv +1 -0
- tn/marathi/normalizer.py +53 -0
- tn/marathi/rules/__init__.py +0 -0
- tn/marathi/rules/cardinal.py +76 -0
- tn/marathi/rules/date.py +70 -0
- tn/marathi/rules/decimal.py +59 -0
- tn/marathi/rules/punctuation.py +30 -0
- tn/marathi/rules/time.py +86 -0
- tn/marathi/rules/utils.py +76 -0
- tn/marathi/rules/word.py +28 -0
- tn/marathi/test/__init__.py +0 -0
- tn/marathi/test/normalizer_test.py +221 -0
- tn/portuguese/__init__.py +0 -0
- tn/portuguese/data/date/months.tsv +21 -0
- tn/portuguese/data/date/numeric_separators.tsv +3 -0
- tn/portuguese/data/date/verbal_phrases.tsv +1 -0
- tn/portuguese/data/electronic/domain.tsv +7 -0
- tn/portuguese/data/electronic/electronic_spoken_unit.tsv +16 -0
- tn/portuguese/data/electronic/server_name.tsv +6 -0
- tn/portuguese/data/electronic/symbols.tsv +22 -0
- tn/portuguese/data/fractions/ordinal_exceptions.tsv +2 -0
- tn/portuguese/data/fractions/powers_of_ten.tsv +1 -0
- tn/portuguese/data/fractions/specials.tsv +5 -0
- tn/portuguese/data/measure/measurements_plural.tsv +64 -0
- tn/portuguese/data/measure/measurements_singular.tsv +65 -0
- tn/portuguese/data/money/currency_major.tsv +5 -0
- tn/portuguese/data/money/currency_major_plural.tsv +5 -0
- tn/portuguese/data/money/currency_minor.tsv +5 -0
- tn/portuguese/data/money/currency_minor_plural.tsv +1 -0
- tn/portuguese/data/numbers/cardinal_specials.tsv +4 -0
- tn/portuguese/data/numbers/decimal_specials.tsv +2 -0
- tn/portuguese/data/numbers/digit.tsv +9 -0
- tn/portuguese/data/numbers/hundreds.tsv +8 -0
- tn/portuguese/data/numbers/quantity_words.tsv +9 -0
- tn/portuguese/data/numbers/scales.tsv +4 -0
- tn/portuguese/data/numbers/teens.tsv +10 -0
- tn/portuguese/data/numbers/tens.tsv +8 -0
- tn/portuguese/data/numbers/zero.tsv +1 -0
- tn/portuguese/data/ordinals/digit.tsv +10 -0
- tn/portuguese/data/ordinals/feminine.tsv +11 -0
- tn/portuguese/data/ordinals/hundreds.tsv +10 -0
- tn/portuguese/data/ordinals/ties.tsv +8 -0
- tn/portuguese/data/telephone/ip_prompt.tsv +2 -0
- tn/portuguese/data/telephone/telephone_prompt.tsv +4 -0
- tn/portuguese/data/time/day_period_suffix.tsv +4 -0
- tn/portuguese/data/whitelist/symbol.tsv +23 -0
- tn/portuguese/normalizer.py +74 -0
- tn/portuguese/rules/__init__.py +0 -0
- tn/portuguese/rules/cardinal.py +170 -0
- tn/portuguese/rules/date.py +69 -0
- tn/portuguese/rules/decimal.py +67 -0
- tn/portuguese/rules/electronic.py +82 -0
- tn/portuguese/rules/fraction.py +79 -0
- tn/portuguese/rules/measure.py +66 -0
- tn/portuguese/rules/money.py +117 -0
- tn/portuguese/rules/ordinal.py +82 -0
- tn/portuguese/rules/punctuation.py +47 -0
- tn/portuguese/rules/telephone.py +71 -0
- tn/portuguese/rules/time.py +101 -0
- tn/portuguese/rules/whitelist.py +37 -0
- tn/portuguese/rules/word.py +35 -0
- tn/portuguese/test/__init__.py +0 -0
- tn/portuguese/test/data/test_cases_cardinal.txt +121 -0
- tn/portuguese/test/data/test_cases_date.txt +26 -0
- tn/portuguese/test/data/test_cases_decimal.txt +58 -0
- tn/portuguese/test/data/test_cases_electronic.txt +13 -0
- tn/portuguese/test/data/test_cases_fraction.txt +22 -0
- tn/portuguese/test/data/test_cases_measure.txt +18 -0
- tn/portuguese/test/data/test_cases_money.txt +25 -0
- tn/portuguese/test/data/test_cases_ordinal.txt +39 -0
- tn/portuguese/test/data/test_cases_telephone.txt +7 -0
- tn/portuguese/test/data/test_cases_time.txt +30 -0
- tn/portuguese/test/normalizer_test.py +166 -0
- tn/processor.py +432 -38
- tn/russian/__init__.py +0 -0
- tn/russian/data/currency/currency_agreement.tsv +10 -0
- tn/russian/data/currency/currency_plural.tsv +53 -0
- tn/russian/data/currency/currency_singular.tsv +48 -0
- tn/russian/data/electronic/domain.tsv +9 -0
- tn/russian/data/electronic/server_name.tsv +17 -0
- tn/russian/data/electronic/symbols.tsv +28 -0
- tn/russian/data/latin_to_cyrillic.tsv +38 -0
- tn/russian/data/measurements.tsv +321 -0
- tn/russian/data/months/abbr.tsv +12 -0
- tn/russian/data/months/abbr_to_name.tsv +62 -0
- tn/russian/data/months/numbers.tsv +21 -0
- tn/russian/data/numbers/1_cardinals_nominative.tsv +50 -0
- tn/russian/data/numbers/2_cardinals_genitive.tsv +45 -0
- tn/russian/data/numbers/3_cardinals_dative.tsv +45 -0
- tn/russian/data/numbers/4_cardinals_accusative.tsv +48 -0
- tn/russian/data/numbers/5_cardinals_instrumental.tsv +47 -0
- tn/russian/data/numbers/6_cardinals_prepositional.tsv +46 -0
- tn/russian/data/numbers/cardinals_alternatives.tsv +16 -0
- tn/russian/data/numbers/cardinals_nominative_case.tsv +37 -0
- tn/russian/data/numbers/decimal_delimiter.tsv +15 -0
- tn/russian/data/numbers/decimal_endings.tsv +37 -0
- tn/russian/data/numbers/digits_nominative_case.tsv +10 -0
- tn/russian/data/numbers/ordinal_endings.tsv +40 -0
- tn/russian/data/numbers/ordinals.tsv +752 -0
- tn/russian/data/numbers/quantity.tsv +16 -0
- tn/russian/data/time/increment_hour_cardinal.tsv +24 -0
- tn/russian/data/time/increment_hour_ordinal.tsv +24 -0
- tn/russian/data/time/minutes_to_hour.tsv +25 -0
- tn/russian/data/time/time_convert.tsv +12 -0
- tn/russian/data/whitelist.tsv +37 -0
- tn/russian/normalizer.py +75 -0
- tn/russian/rules/__init__.py +0 -0
- tn/russian/rules/cardinal.py +92 -0
- tn/russian/rules/date.py +93 -0
- tn/russian/rules/decimal.py +88 -0
- tn/russian/rules/electronic.py +69 -0
- tn/russian/rules/measure.py +70 -0
- tn/russian/rules/money.py +73 -0
- tn/russian/rules/ordinal.py +45 -0
- tn/russian/rules/punctuation.py +46 -0
- tn/russian/rules/telephone.py +44 -0
- tn/russian/rules/time.py +80 -0
- tn/russian/rules/utils.py +293 -0
- tn/russian/rules/whitelist.py +41 -0
- tn/russian/rules/word.py +34 -0
- tn/russian/test/__init__.py +0 -0
- tn/russian/test/data/test_cases_cardinal.txt +28 -0
- tn/russian/test/data/test_cases_cardinal_normalize_with_audio.txt +51 -0
- tn/russian/test/data/test_cases_date.txt +6 -0
- tn/russian/test/data/test_cases_decimal.txt +17 -0
- tn/russian/test/data/test_cases_electronic.txt +4 -0
- tn/russian/test/data/test_cases_measure.txt +13 -0
- tn/russian/test/data/test_cases_money.txt +9 -0
- tn/russian/test/data/test_cases_ordinal.txt +75 -0
- tn/russian/test/data/test_cases_telephone.txt +1 -0
- tn/russian/test/data/test_cases_time.txt +9 -0
- tn/russian/test/data/test_cases_whitelist.txt +4 -0
- tn/russian/test/data/test_cases_word.txt +3 -0
- tn/russian/test/normalizer_test.py +214 -0
- tn/spanish/__init__.py +0 -0
- tn/spanish/data/dates/days_abbr.tsv +27 -0
- tn/spanish/data/dates/months.tsv +12 -0
- tn/spanish/data/dates/months_abbr.tsv +44 -0
- tn/spanish/data/dates/year_suffix.tsv +51 -0
- tn/spanish/data/electronic/domain.tsv +6 -0
- tn/spanish/data/electronic/server_name.tsv +11 -0
- tn/spanish/data/electronic/symbols.tsv +22 -0
- tn/spanish/data/fractions/fraction_symbols.tsv +15 -0
- tn/spanish/data/fractions/ordinal_exceptions.tsv +2 -0
- tn/spanish/data/fractions/powers_of_ten.tsv +9 -0
- tn/spanish/data/fractions/powers_of_ten_fem.tsv +6 -0
- tn/spanish/data/measures/math_symbols.tsv +7 -0
- tn/spanish/data/measures/measurements.tsv +18 -0
- tn/spanish/data/measures/measurements_complex.tsv +5 -0
- tn/spanish/data/measures/measurements_plural_fem.tsv +3 -0
- tn/spanish/data/measures/measurements_plural_masc.tsv +21 -0
- tn/spanish/data/money/currency_major.tsv +64 -0
- tn/spanish/data/money/currency_major_ext.tsv +215 -0
- tn/spanish/data/money/currency_minor.tsv +64 -0
- tn/spanish/data/money/currency_minor_ext.tsv +181 -0
- tn/spanish/data/money/currency_plural_fem.tsv +8 -0
- tn/spanish/data/money/currency_plural_fem_ext.tsv +56 -0
- tn/spanish/data/money/currency_plural_masc.tsv +51 -0
- tn/spanish/data/money/currency_plural_masc_ext.tsv +199 -0
- tn/spanish/data/numbers/digit.tsv +9 -0
- tn/spanish/data/numbers/hundreds.tsv +8 -0
- tn/spanish/data/numbers/quantities.tsv +6 -0
- tn/spanish/data/numbers/teen.tsv +10 -0
- tn/spanish/data/numbers/ties.tsv +7 -0
- tn/spanish/data/numbers/twenties.tsv +10 -0
- tn/spanish/data/numbers/zero.tsv +1 -0
- tn/spanish/data/ordinals/digit.tsv +11 -0
- tn/spanish/data/ordinals/gender_suffix.tsv +20 -0
- tn/spanish/data/ordinals/hundreds.tsv +18 -0
- tn/spanish/data/ordinals/roman_exceptions.tsv +10 -0
- tn/spanish/data/ordinals/teen.tsv +10 -0
- tn/spanish/data/ordinals/ties.tsv +8 -0
- tn/spanish/data/ordinals/twenties.tsv +11 -0
- tn/spanish/data/roman/digit.tsv +9 -0
- tn/spanish/data/roman/hundreds.tsv +9 -0
- tn/spanish/data/roman/thousands.tsv +3 -0
- tn/spanish/data/roman/ties.tsv +9 -0
- tn/spanish/data/telephone/ip_prompt.tsv +2 -0
- tn/spanish/data/telephone/telephone_prompt.tsv +10 -0
- tn/spanish/data/time/afternoon_times.tsv +6 -0
- tn/spanish/data/time/alt_minutes.tsv +2 -0
- tn/spanish/data/time/evening_times.tsv +5 -0
- tn/spanish/data/time/hour_to_12.tsv +12 -0
- tn/spanish/data/time/hour_to_24.tsv +25 -0
- tn/spanish/data/time/hour_to_night.tsv +13 -0
- tn/spanish/data/time/minute_to.tsv +59 -0
- tn/spanish/data/time/morning_times.tsv +12 -0
- tn/spanish/data/time/time_suffix.tsv +20 -0
- tn/spanish/data/time/time_zone.tsv +42 -0
- tn/spanish/data/time/time_zone_ext.tsv +105 -0
- tn/spanish/data/whitelist.tsv +62 -0
- tn/spanish/normalizer.py +79 -0
- tn/spanish/rules/__init__.py +0 -0
- tn/spanish/rules/cardinal.py +125 -0
- tn/spanish/rules/date.py +96 -0
- tn/spanish/rules/decimal.py +122 -0
- tn/spanish/rules/electronic.py +69 -0
- tn/spanish/rules/fraction.py +189 -0
- tn/spanish/rules/measure.py +177 -0
- tn/spanish/rules/money.py +186 -0
- tn/spanish/rules/ordinal.py +119 -0
- tn/spanish/rules/punctuation.py +48 -0
- tn/spanish/rules/telephone.py +93 -0
- tn/spanish/rules/time.py +171 -0
- tn/spanish/rules/utils.py +170 -0
- tn/spanish/rules/whitelist.py +39 -0
- tn/spanish/rules/word.py +39 -0
- tn/spanish/test/__init__.py +0 -0
- tn/spanish/test/data/test_cases_cardinal.txt +69 -0
- tn/spanish/test/data/test_cases_date.txt +21 -0
- tn/spanish/test/data/test_cases_decimal.txt +24 -0
- tn/spanish/test/data/test_cases_electronic.txt +22 -0
- tn/spanish/test/data/test_cases_fraction.txt +75 -0
- tn/spanish/test/data/test_cases_measure.txt +32 -0
- tn/spanish/test/data/test_cases_money.txt +69 -0
- tn/spanish/test/data/test_cases_normalize_with_audio.txt +162 -0
- tn/spanish/test/data/test_cases_ordinal.txt +120 -0
- tn/spanish/test/data/test_cases_telephone.txt +15 -0
- tn/spanish/test/data/test_cases_time.txt +35 -0
- tn/spanish/test/data/test_cases_whitelist.txt +7 -0
- tn/spanish/test/data/test_cases_word.txt +48 -0
- tn/spanish/test/normalizer_test.py +166 -0
- tn/swedish/__init__.py +0 -0
- tn/swedish/data/abbreviations_nondet.tsv +27 -0
- tn/swedish/data/dates/era_suffix.tsv +18 -0
- tn/swedish/data/dates/era_words.tsv +4 -0
- tn/swedish/data/dates/month_abbr.tsv +17 -0
- tn/swedish/data/dates/months.tsv +12 -0
- tn/swedish/data/electronic/domain.tsv +10 -0
- tn/swedish/data/electronic/server_name.tsv +29 -0
- tn/swedish/data/electronic/symbols.tsv +20 -0
- tn/swedish/data/math_operations.tsv +8 -0
- tn/swedish/data/measure/greek_lower.tsv +25 -0
- tn/swedish/data/measure/greek_upper.tsv +25 -0
- tn/swedish/data/measure/unit.tsv +147 -0
- tn/swedish/data/measure/unit_neuter.tsv +13 -0
- tn/swedish/data/measure/unit_plural.tsv +121 -0
- tn/swedish/data/money/currency_major.tsv +75 -0
- tn/swedish/data/money/currency_major_nt.tsv +2 -0
- tn/swedish/data/money/currency_minor_plural.tsv +79 -0
- tn/swedish/data/money/currency_minor_singular.tsv +78 -0
- tn/swedish/data/money/currency_plurals.tsv +44 -0
- tn/swedish/data/numbers/digit.tsv +9 -0
- tn/swedish/data/numbers/fraction.tsv +18 -0
- tn/swedish/data/numbers/millions.tsv +88 -0
- tn/swedish/data/numbers/millions_abbr.tsv +4 -0
- tn/swedish/data/numbers/teen.tsv +10 -0
- tn/swedish/data/numbers/ties.tsv +8 -0
- tn/swedish/data/numbers/zero.tsv +1 -0
- tn/swedish/data/ordinals/digit.tsv +9 -0
- tn/swedish/data/ordinals/teen.tsv +10 -0
- tn/swedish/data/ordinals/ties.tsv +8 -0
- tn/swedish/data/ordinals/zero.tsv +1 -0
- tn/swedish/data/telephone/ip_prompt.tsv +2 -0
- tn/swedish/data/telephone/special_numbers.tsv +4 -0
- tn/swedish/data/telephone/ssn_prompt.tsv +3 -0
- tn/swedish/data/telephone/telephone_abbr.tsv +24 -0
- tn/swedish/data/telephone/telephone_prompt.tsv +5 -0
- tn/swedish/data/time/suffix.tsv +16 -0
- tn/swedish/data/time/time_zone.tsv +7 -0
- tn/swedish/data/whitelist.tsv +58 -0
- tn/swedish/normalizer.py +75 -0
- tn/swedish/rules/__init__.py +0 -0
- tn/swedish/rules/cardinal.py +172 -0
- tn/swedish/rules/date.py +122 -0
- tn/swedish/rules/decimal.py +110 -0
- tn/swedish/rules/electronic.py +75 -0
- tn/swedish/rules/fraction.py +85 -0
- tn/swedish/rules/measure.py +143 -0
- tn/swedish/rules/money.py +127 -0
- tn/swedish/rules/ordinal.py +132 -0
- tn/swedish/rules/punctuation.py +47 -0
- tn/swedish/rules/telephone.py +89 -0
- tn/swedish/rules/time.py +98 -0
- tn/swedish/rules/utils.py +63 -0
- tn/swedish/rules/whitelist.py +38 -0
- tn/swedish/rules/word.py +33 -0
- tn/swedish/test/__init__.py +0 -0
- tn/swedish/test/data/test_cases_cardinal.txt +68 -0
- tn/swedish/test/data/test_cases_date.txt +17 -0
- tn/swedish/test/data/test_cases_decimal.txt +7 -0
- tn/swedish/test/data/test_cases_electronic.txt +12 -0
- tn/swedish/test/data/test_cases_fraction.txt +2 -0
- tn/swedish/test/data/test_cases_measure.txt +7 -0
- tn/swedish/test/data/test_cases_money.txt +65 -0
- tn/swedish/test/data/test_cases_normalize_with_audio.txt +14 -0
- tn/swedish/test/data/test_cases_ordinal.txt +7 -0
- tn/swedish/test/data/test_cases_telephone.txt +7 -0
- tn/swedish/test/data/test_cases_time.txt +12 -0
- tn/swedish/test/data/test_cases_whitelist.txt +1 -0
- tn/swedish/test/data/test_cases_word.txt +48 -0
- tn/swedish/test/normalizer_test.py +195 -0
- tn/test/cli_test.py +337 -0
- tn/test/punctuation_spacing_test.py +22 -0
- tn/test/robustness_test.py +32 -0
- tn/token_parser.py +91 -29
- tn/utils.py +10 -0
- tn/vietnamese/__init__.py +0 -0
- tn/vietnamese/data/date/days.tsv +40 -0
- tn/vietnamese/data/date/months.tsv +21 -0
- tn/vietnamese/data/date/year_suffix.tsv +4 -0
- tn/vietnamese/data/fraction/denominator_exceptions.tsv +1 -0
- tn/vietnamese/data/measure/base_units.tsv +20 -0
- tn/vietnamese/data/measure/measurements_minimal.tsv +33 -0
- tn/vietnamese/data/measure/prefixes.tsv +17 -0
- tn/vietnamese/data/money/currency.tsv +51 -0
- tn/vietnamese/data/money/currency_minor.tsv +7 -0
- tn/vietnamese/data/money/per_unit_bases.tsv +8 -0
- tn/vietnamese/data/money/per_unit_non_metric.tsv +28 -0
- tn/vietnamese/data/money/per_unit_prefixes.tsv +6 -0
- tn/vietnamese/data/numbers/digit.tsv +9 -0
- tn/vietnamese/data/numbers/digit_special.tsv +3 -0
- tn/vietnamese/data/numbers/magnitudes.tsv +8 -0
- tn/vietnamese/data/numbers/quantity_abbr.tsv +7 -0
- tn/vietnamese/data/numbers/teen.tsv +10 -0
- tn/vietnamese/data/numbers/ties.tsv +8 -0
- tn/vietnamese/data/numbers/zero.tsv +1 -0
- tn/vietnamese/data/ordinal/ordinal_exceptions.tsv +2 -0
- tn/vietnamese/data/roman/key_word.tsv +16 -0
- tn/vietnamese/data/roman/roman_numerals.tsv +13 -0
- tn/vietnamese/data/time/time_units.tsv +4 -0
- tn/vietnamese/data/time/time_zones.tsv +18 -0
- tn/vietnamese/data/whitelist.tsv +0 -0
- tn/vietnamese/normalizer.py +106 -0
- tn/vietnamese/rules/__init__.py +0 -0
- tn/vietnamese/rules/cardinal.py +163 -0
- tn/vietnamese/rules/date.py +81 -0
- tn/vietnamese/rules/decimal.py +85 -0
- tn/vietnamese/rules/fraction.py +50 -0
- tn/vietnamese/rules/measure.py +57 -0
- tn/vietnamese/rules/money.py +87 -0
- tn/vietnamese/rules/ordinal.py +34 -0
- tn/vietnamese/rules/punctuation.py +31 -0
- tn/vietnamese/rules/range.py +36 -0
- tn/vietnamese/rules/roman.py +59 -0
- tn/vietnamese/rules/time.py +86 -0
- tn/vietnamese/rules/whitelist.py +29 -0
- tn/vietnamese/rules/word.py +36 -0
- tn/vietnamese/test/__init__.py +0 -0
- tn/vietnamese/test/data/test_cases_cardinal.txt +109 -0
- tn/vietnamese/test/data/test_cases_date.txt +13 -0
- tn/vietnamese/test/data/test_cases_decimal.txt +29 -0
- tn/vietnamese/test/data/test_cases_fraction.txt +13 -0
- tn/vietnamese/test/data/test_cases_measure.txt +63 -0
- tn/vietnamese/test/data/test_cases_money.txt +30 -0
- tn/vietnamese/test/data/test_cases_ordinal.txt +30 -0
- tn/vietnamese/test/data/test_cases_range.txt +11 -0
- tn/vietnamese/test/data/test_cases_roman.txt +57 -0
- tn/vietnamese/test/data/test_cases_time.txt +117 -0
- tn/vietnamese/test/normalizer_test.py +146 -0
- wetextprocessing-1.3.0.dist-info/METADATA +290 -0
- wetextprocessing-1.3.0.dist-info/RECORD +2049 -0
- itn/zh_itn_tagger.fst +0 -0
- itn/zh_itn_verbalizer.fst +0 -0
- tn/en_tn_tagger.fst +0 -0
- tn/en_tn_verbalizer.fst +0 -0
- tn/zh_tn_tagger.fst +0 -0
- tn/zh_tn_verbalizer.fst +0 -0
- wetextprocessing-1.0.6.dev19.dist-info/METADATA +0 -160
- wetextprocessing-1.0.6.dev19.dist-info/RECORD +0 -381
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/WHEEL +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/entry_points.txt +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/licenses/LICENSE +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/top_level.txt +0 -0
itn/README.md
CHANGED
|
@@ -3,15 +3,23 @@
|
|
|
3
3
|
### 1. How To Use
|
|
4
4
|
|
|
5
5
|
``` bash
|
|
6
|
-
$ python
|
|
6
|
+
$ python -m itn --language zh --text "二点五平方电线"
|
|
7
|
+
# Or, after installation:
|
|
8
|
+
$ weitn --language zh --text "二点五平方电线"
|
|
7
9
|
```
|
|
8
10
|
|
|
9
11
|
### 2. ITN Pipeline
|
|
10
12
|
|
|
11
|
-
There are
|
|
13
|
+
There are 3 logical components in the ITN pipeline:
|
|
12
14
|
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
+
* optional input canonicalization
|
|
16
|
+
* classification and raw-field tagging
|
|
17
|
+
* non-standard word verbalization and output post-processing
|
|
18
|
+
|
|
19
|
+
The tagger must preserve the spoken input in its fields. Converting those
|
|
20
|
+
fields to written forms belongs to the verbalizer. See the
|
|
21
|
+
[Python rule architecture and contribution guide](../docs/python-rule-architecture.md)
|
|
22
|
+
before adding or modifying a rule.
|
|
15
23
|
|
|
16
24
|
#### 2.1 Pre-Processing
|
|
17
25
|
|
itn/arabic/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
$ دولار
|
|
2
|
+
$ دولار أمريكي
|
|
3
|
+
US$ دولار أمريكي
|
|
4
|
+
฿ بات تايلندي
|
|
5
|
+
£ جنيه استرليني
|
|
6
|
+
€ يورو
|
|
7
|
+
₩ ين
|
|
8
|
+
nzd دولار نيوزيلندي
|
|
9
|
+
rs روبية
|
|
10
|
+
chf فرنك سويسري
|
|
11
|
+
dkk كرونة دنماركية
|
|
12
|
+
aed درهم إماراتي
|
|
13
|
+
¥ ين ياباني
|
|
14
|
+
mro الأوقية الموريتانية
|
|
15
|
+
pkr روبية باكستانية
|
|
16
|
+
hk$ دولار هونج كونج
|
|
17
|
+
npr روبية نيبالية
|
|
18
|
+
sar ريال سعودي
|
|
19
|
+
dm مارك ألماني
|
|
20
|
+
cad دولار الكندي
|
|
21
|
+
DH درهم
|
|
22
|
+
Dh درهم
|
|
23
|
+
Dhs. درهم
|
|
24
|
+
₺ ليرة تركية
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
ج.م جنيه مصري
|
|
2
|
+
د.ع دينار عراقي
|
|
3
|
+
د.ك دينار كويتي
|
|
4
|
+
أ.م أوقية موريتانية
|
|
5
|
+
د.إ درهم إماراتي
|
|
6
|
+
د.ب دينار بحريني
|
|
7
|
+
د.ج دينار جزائري
|
|
8
|
+
ج.س جنيه سوداني
|
|
9
|
+
د.أ دينار أردني
|
|
10
|
+
ر.س ريال سعودي
|
|
11
|
+
ر ق ريال قطري
|
|
12
|
+
ر.ع ريال عماني
|
|
13
|
+
ل.ل ليرة لبنانية
|
|
14
|
+
د.م درهم مغربي
|
|
15
|
+
د.ل دينار ليبي
|
|
16
|
+
د.ت دينار تونسي
|
|
17
|
+
ل.س ليرة سورية
|
|
18
|
+
ر.ي ريال يمني
|
|
19
|
+
ف.ج فرنك جيبوتي
|
|
20
|
+
ش.ص شلن صومالي
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
21 12
|
|
2
|
+
22 22
|
|
3
|
+
23 32
|
|
4
|
+
24 42
|
|
5
|
+
25 52
|
|
6
|
+
26 62
|
|
7
|
+
27 72
|
|
8
|
+
28 82
|
|
9
|
+
29 92
|
|
10
|
+
31 13
|
|
11
|
+
32 23
|
|
12
|
+
33 33
|
|
13
|
+
34 43
|
|
14
|
+
35 53
|
|
15
|
+
36 63
|
|
16
|
+
37 73
|
|
17
|
+
38 83
|
|
18
|
+
39 93
|
|
19
|
+
41 14
|
|
20
|
+
42 24
|
|
21
|
+
43 34
|
|
22
|
+
45 54
|
|
23
|
+
46 64
|
|
24
|
+
47 74
|
|
25
|
+
48 84
|
|
26
|
+
49 94
|
|
27
|
+
44 44
|
|
28
|
+
51 15
|
|
29
|
+
52 25
|
|
30
|
+
53 35
|
|
31
|
+
54 45
|
|
32
|
+
56 65
|
|
33
|
+
57 75
|
|
34
|
+
58 85
|
|
35
|
+
59 95
|
|
36
|
+
55 55
|
|
37
|
+
61 16
|
|
38
|
+
62 26
|
|
39
|
+
63 36
|
|
40
|
+
64 46
|
|
41
|
+
65 56
|
|
42
|
+
67 76
|
|
43
|
+
68 86
|
|
44
|
+
69 96
|
|
45
|
+
66 66
|
|
46
|
+
71 17
|
|
47
|
+
72 27
|
|
48
|
+
73 37
|
|
49
|
+
74 47
|
|
50
|
+
75 57
|
|
51
|
+
76 67
|
|
52
|
+
78 87
|
|
53
|
+
79 97
|
|
54
|
+
77 77
|
|
55
|
+
81 18
|
|
56
|
+
82 28
|
|
57
|
+
83 38
|
|
58
|
+
84 48
|
|
59
|
+
85 58
|
|
60
|
+
86 68
|
|
61
|
+
87 78
|
|
62
|
+
89 98
|
|
63
|
+
88 88
|
|
64
|
+
91 19
|
|
65
|
+
92 29
|
|
66
|
+
93 39
|
|
67
|
+
94 49
|
|
68
|
+
95 59
|
|
69
|
+
96 69
|
|
70
|
+
97 79
|
|
71
|
+
98 89
|
|
72
|
+
99 99
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
from pynini.lib import pynutil
|
|
17
|
+
|
|
18
|
+
from itn.arabic.rules import money
|
|
19
|
+
from itn.arabic.rules.cardinal import Cardinal
|
|
20
|
+
from itn.arabic.rules.decimal import Decimal
|
|
21
|
+
from itn.arabic.rules.fraction import Fraction
|
|
22
|
+
from itn.arabic.rules.measure import Measure
|
|
23
|
+
from itn.arabic.rules.money import Money
|
|
24
|
+
from itn.arabic.rules.punctuation import Punctuation
|
|
25
|
+
from itn.arabic.rules.word import Word
|
|
26
|
+
from tn.processor import Processor, RuleSpec
|
|
27
|
+
|
|
28
|
+
TOKEN_ORDERS = {}
|
|
29
|
+
for _module in (money, ):
|
|
30
|
+
TOKEN_ORDERS.update(_module.TOKEN_ORDER)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class InverseNormalizer(Processor):
|
|
34
|
+
|
|
35
|
+
def __init__(self, cache_dir=None, overwrite_cache=False):
|
|
36
|
+
super().__init__(name="ar_inverse_normalizer", ordertype="itn", token_orders=TOKEN_ORDERS)
|
|
37
|
+
self.build_fst("ar_itn", cache_dir, overwrite_cache, {})
|
|
38
|
+
|
|
39
|
+
def build_tagger_and_verbalizer(self):
|
|
40
|
+
cardinal = Cardinal()
|
|
41
|
+
decimal = Decimal(cardinal)
|
|
42
|
+
fraction = Fraction(cardinal)
|
|
43
|
+
punctuation = Punctuation()
|
|
44
|
+
word = Word()
|
|
45
|
+
rules = (
|
|
46
|
+
RuleSpec(cardinal, 1.1),
|
|
47
|
+
RuleSpec(decimal, 1.1),
|
|
48
|
+
RuleSpec(fraction, 1.1),
|
|
49
|
+
RuleSpec(Money(cardinal), 1.1),
|
|
50
|
+
RuleSpec(Measure(cardinal, decimal, fraction), 1.1),
|
|
51
|
+
RuleSpec(word, 100),
|
|
52
|
+
RuleSpec(punctuation),
|
|
53
|
+
)
|
|
54
|
+
classify = self.tagger_union(rules)
|
|
55
|
+
punct = pynutil.add_weight(punctuation.tagger, 1.1)
|
|
56
|
+
token = punct.star + classify + punct.star
|
|
57
|
+
self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE
|
|
58
|
+
self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus))
|
|
59
|
+
verbalize = self.verbalizer_union(rules)
|
|
60
|
+
self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star
|
|
File without changes
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
import pynini
|
|
17
|
+
from pynini.lib import byte
|
|
18
|
+
from pynini.lib.pynutil import delete, insert
|
|
19
|
+
|
|
20
|
+
from itn.utils import get_abs_path
|
|
21
|
+
from tn.processor import Processor
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def spell_out():
|
|
25
|
+
"""Written digits (1..9999) -> Arabic number words, one digit per place value."""
|
|
26
|
+
|
|
27
|
+
digit = pynini.string_file(get_abs_path("arabic/data/number/digit.tsv"))
|
|
28
|
+
digit_100 = pynini.string_file(get_abs_path("arabic/data/number/digit_100.tsv"))
|
|
29
|
+
digit_1000 = pynini.string_file(get_abs_path("arabic/data/number/digit_1000.tsv"))
|
|
30
|
+
teens = pynini.string_file(get_abs_path("arabic/data/number/teens.tsv"))
|
|
31
|
+
# tens.tsv also lists 10 -> مئة, which would let "100" pose as two digits.
|
|
32
|
+
tens = byte.DIGIT @ pynini.string_file(get_abs_path("arabic/data/number/tens.tsv"))
|
|
33
|
+
flops = pynini.string_file(get_abs_path("arabic/data/number/flops.tsv"))
|
|
34
|
+
conjunction = insert(" و")
|
|
35
|
+
two_digits = teens | tens + delete("0") | flops @ (digit + conjunction + tens)
|
|
36
|
+
|
|
37
|
+
def tail(width, lower):
|
|
38
|
+
# Round hundreds/thousands cost a little more, so a decimal like
|
|
39
|
+
# "مئة وواحد وخمسون من مئة" keeps the longer integer part (101.50).
|
|
40
|
+
rest = delete("0" * width, weight=0.001)
|
|
41
|
+
for zeros, graph in enumerate(lower):
|
|
42
|
+
rest |= delete("0" * zeros) + conjunction + graph
|
|
43
|
+
return rest
|
|
44
|
+
|
|
45
|
+
hundreds = pynini.union(pynini.cross("1", "مئة"), pynini.cross("2", "مئتان"), digit_100 + insert(" مئة"))
|
|
46
|
+
hundreds = (hundreds + tail(2, [two_digits, digit])).optimize()
|
|
47
|
+
thousands = pynini.union(pynini.cross("1", "ألف"), pynini.cross("2", "ألفان"), digit_1000 + insert(" اَلاف"))
|
|
48
|
+
thousands += tail(3, [hundreds, two_digits, digit])
|
|
49
|
+
return (digit | two_digits | hundreds | thousands).optimize()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class Cardinal(Processor):
|
|
53
|
+
"""Number words up to 9999, e.g. ``سالب تسعة وتسعون`` -> ``-99``.
|
|
54
|
+
|
|
55
|
+
The tagger keeps the spoken words; the verbalizer writes the digits.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
def __init__(self):
|
|
59
|
+
super().__init__(name="cardinal", ordertype="itn")
|
|
60
|
+
self.spoken = spell_out()
|
|
61
|
+
self.graph = pynini.invert(self.spoken).optimize()
|
|
62
|
+
self.negative = pynini.cross("سالب", "-")
|
|
63
|
+
sign = self.tag_field("negative", self.negative) + delete(" ") + insert(" ")
|
|
64
|
+
self.tagger = self.add_tokens(sign.ques + self.tag_field("integer", self.graph))
|
|
65
|
+
sign_out = self.verbalize_field("negative", self.negative) + delete(" ")
|
|
66
|
+
self.verbalizer = self.delete_tokens(sign_out.ques + self.verbalize_field("integer", self.graph))
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
import pynini
|
|
17
|
+
from pynini.lib.pynutil import delete, insert
|
|
18
|
+
|
|
19
|
+
from itn.utils import get_abs_path
|
|
20
|
+
from tn.processor import Processor
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Decimal(Processor):
|
|
24
|
+
"""Decimals, e.g. ``ستة وثمانية وثمانون من مئة`` -> ``6.88``.
|
|
25
|
+
|
|
26
|
+
The spoken denominator (عشرة, مئة, ألف, عشرة آلاف) fixes the number of
|
|
27
|
+
fractional digits. The tagger keeps the words, dropping only the ``و``
|
|
28
|
+
that joins the two parts.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self, cardinal):
|
|
32
|
+
super().__init__(name="decimal", ordertype="itn")
|
|
33
|
+
self.cardinal = cardinal
|
|
34
|
+
padded = delete("0").star + cardinal.spoken
|
|
35
|
+
denominators = ["عشرة", "مئة", "ألف", "عشرة آلاف"]
|
|
36
|
+
fractional = pynini.union(*[
|
|
37
|
+
self.DIGIT**width @ padded + insert(" من " + denominator)
|
|
38
|
+
for width, denominator in enumerate(denominators, 1)
|
|
39
|
+
])
|
|
40
|
+
self.fractional = pynini.invert(fractional).optimize()
|
|
41
|
+
self.quantity = pynini.string_file(get_abs_path("arabic/data/number/quantities.tsv")).optimize()
|
|
42
|
+
self.negative = cardinal.negative
|
|
43
|
+
sign = self.tag_field("negative", self.negative) + self.DELETE_EXTRA_SPACE
|
|
44
|
+
self.tagger = self.add_tokens(sign.ques + self.tag_number())
|
|
45
|
+
sign_out = self.verbalize_field("negative", self.negative) + delete(" ")
|
|
46
|
+
self.verbalizer = self.delete_tokens(sign_out.ques + self.verbalize_number())
|
|
47
|
+
|
|
48
|
+
def tag_number(self):
|
|
49
|
+
"""Raw decimal fields without the sign, shared with measure."""
|
|
50
|
+
|
|
51
|
+
tagger = self.tag_field("integer_part", self.cardinal.graph) + pynini.accep(" ") + delete("و")
|
|
52
|
+
tagger += self.tag_field("fractional_part", self.fractional)
|
|
53
|
+
return tagger + (pynini.accep(" ") + self.tag_field("quantity", self.quantity)).ques
|
|
54
|
+
|
|
55
|
+
def verbalize_number(self):
|
|
56
|
+
verbalizer = self.verbalize_field("integer_part", self.cardinal.graph) + insert(".")
|
|
57
|
+
verbalizer += self.verbalize_field("fractional_part", self.fractional, True)
|
|
58
|
+
return verbalizer + (insert(" ") + self.verbalize_field("quantity", self.quantity, True)).ques
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
import pynini
|
|
17
|
+
from pynini.lib.pynutil import add_weight, delete, insert
|
|
18
|
+
|
|
19
|
+
from itn.utils import get_abs_path
|
|
20
|
+
from tn.processor import Processor
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Fraction(Processor):
|
|
24
|
+
"""Fractions, e.g. ``واحد و نصف`` -> ``1 1/2`` and ``تسعة أثلاث`` -> ``9/3``.
|
|
25
|
+
|
|
26
|
+
A lone singular (``نصف``) or dual (``ثلثين``) fraction noun implies the
|
|
27
|
+
numerator 1 or 2; plural nouns follow a numerator from 3 to 10; other
|
|
28
|
+
fractions are spoken ``X على Y``. The tagger keeps the words, dropping
|
|
29
|
+
only the ``و`` and ``على`` joiners.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(self, cardinal):
|
|
33
|
+
super().__init__(name="fraction", ordertype="itn")
|
|
34
|
+
self.cardinal = cardinal
|
|
35
|
+
number = cardinal.graph
|
|
36
|
+
singular = pynini.string_file(get_abs_path("arabic/data/number/fraction_singular.tsv"))
|
|
37
|
+
dual = pynini.string_file(get_abs_path("arabic/data/number/fraction_dual.tsv"))
|
|
38
|
+
plural = pynini.string_file(get_abs_path("arabic/data/number/fraction_plural.tsv"))
|
|
39
|
+
three_to_ten = pynini.union(*[str(n) for n in range(3, 11)])
|
|
40
|
+
self.number = number
|
|
41
|
+
self.singular = pynini.invert(singular).optimize()
|
|
42
|
+
self.dual = pynini.invert(dual).optimize()
|
|
43
|
+
self.plural = pynini.invert(plural).optimize()
|
|
44
|
+
self.three_to_ten = (number @ three_to_ten).optimize()
|
|
45
|
+
self.negative = cardinal.negative
|
|
46
|
+
sign = self.tag_field("negative", self.negative) + self.DELETE_SPACE + insert(" ")
|
|
47
|
+
self.tagger = self.add_tokens(sign.ques + self.tag_number())
|
|
48
|
+
sign_out = self.verbalize_field("negative", self.negative) + delete(" ")
|
|
49
|
+
self.verbalizer = self.delete_tokens(sign_out.ques + self.verbalize_number())
|
|
50
|
+
|
|
51
|
+
def tag_number(self):
|
|
52
|
+
"""Raw fraction fields without the sign, shared with measure."""
|
|
53
|
+
|
|
54
|
+
optional_space = self.DELETE_ZERO_OR_ONE_SPACE
|
|
55
|
+
integer = self.tag_field("integer_part", self.number) + optional_space + delete("و") + optional_space
|
|
56
|
+
integer += insert(" ")
|
|
57
|
+
lone = self.tag_field("denominator", self.singular | self.dual)
|
|
58
|
+
plural = self.tag_field("numerator", self.three_to_ten) + optional_space + insert(" ")
|
|
59
|
+
plural += self.tag_field("denominator", self.plural)
|
|
60
|
+
ratio = self.tag_field("numerator", self.number) + optional_space + delete("على") + optional_space
|
|
61
|
+
ratio += insert(" ") + self.tag_field("denominator", self.number)
|
|
62
|
+
return integer.ques + (lone | plural | add_weight(ratio, 0.001))
|
|
63
|
+
|
|
64
|
+
def verbalize_number(self):
|
|
65
|
+
integer = self.verbalize_field("integer_part", self.number) + pynini.accep(" ")
|
|
66
|
+
lone = insert("1/") + self.verbalize_field("denominator", self.singular)
|
|
67
|
+
lone |= insert("2/") + self.verbalize_field("denominator", self.dual)
|
|
68
|
+
plural = self.verbalize_field("numerator", self.three_to_ten) + insert("/")
|
|
69
|
+
plural += self.verbalize_field("denominator", self.plural, True)
|
|
70
|
+
ratio = self.verbalize_field("numerator", self.number) + insert("/")
|
|
71
|
+
ratio += self.verbalize_field("denominator", self.number, True)
|
|
72
|
+
return integer.ques + (lone | plural | ratio)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
import pynini
|
|
17
|
+
from pynini.lib.pynutil import delete
|
|
18
|
+
|
|
19
|
+
from itn.utils import get_abs_path
|
|
20
|
+
from tn.processor import Processor
|
|
21
|
+
from tn.utils import load_labels
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Measure(Processor):
|
|
25
|
+
"""Quantities with units, e.g. ``عشرون في المائة`` -> ``20%``.
|
|
26
|
+
|
|
27
|
+
The quantity is a cardinal, decimal or fraction (flat raw fields reusing
|
|
28
|
+
those rules' graphs) followed by a spoken unit; the unit symbol is
|
|
29
|
+
attached without a space.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(self, cardinal, decimal, fraction):
|
|
33
|
+
super().__init__(name="measure", ordertype="itn")
|
|
34
|
+
names = {}
|
|
35
|
+
for symbol, name in load_labels(get_abs_path("arabic/data/measure/measurements.tsv")):
|
|
36
|
+
names.setdefault(name, symbol)
|
|
37
|
+
self.units = pynini.string_map([(pynini.escape(name), pynini.escape(symbol))
|
|
38
|
+
for name, symbol in names.items()]).optimize()
|
|
39
|
+
negative = cardinal.negative
|
|
40
|
+
sign = self.tag_field("negative", negative) + self.DELETE_EXTRA_SPACE
|
|
41
|
+
quantity = self.tag_field("integer", cardinal.graph) | decimal.tag_number() | fraction.tag_number()
|
|
42
|
+
units = self.DELETE_EXTRA_SPACE + self.tag_field("units", self.units)
|
|
43
|
+
self.tagger = self.add_tokens(sign.ques + quantity + units)
|
|
44
|
+
|
|
45
|
+
sign = self.verbalize_field("negative", negative) + delete(" ")
|
|
46
|
+
quantity = self.verbalize_field("integer", cardinal.graph) | decimal.verbalize_number()
|
|
47
|
+
quantity |= fraction.verbalize_number()
|
|
48
|
+
units = self.verbalize_field("units", self.units, True)
|
|
49
|
+
self.verbalizer = self.delete_tokens(sign.ques + quantity + units)
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
import pynini
|
|
17
|
+
from pynini.lib.pynutil import delete, insert
|
|
18
|
+
|
|
19
|
+
from itn.utils import get_abs_path
|
|
20
|
+
from tn.processor import Processor
|
|
21
|
+
from tn.utils import load_labels
|
|
22
|
+
|
|
23
|
+
# The currency symbol is written first: $7.50, د.ك5.
|
|
24
|
+
TOKEN_ORDER = {"money": ["currency", "integer_part", "fractional_part", "currency_min"]}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def symbols(*paths):
|
|
28
|
+
"""Maps each spoken currency name to the first symbol listed for it."""
|
|
29
|
+
|
|
30
|
+
names = {}
|
|
31
|
+
for path in paths:
|
|
32
|
+
for row in load_labels(get_abs_path(path)):
|
|
33
|
+
names.setdefault(row[1], row[0])
|
|
34
|
+
return pynini.string_map([(pynini.escape(name), pynini.escape(symbol)) for name, symbol in names.items()])
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Money(Processor):
|
|
38
|
+
"""Money, e.g. ``سبعة دولار وخمسون سنت`` -> ``$7.50`` and ``خمسة دينار كويتي`` -> ``د.ك5``.
|
|
39
|
+
|
|
40
|
+
The tagger keeps the spoken amount, currency and minor-unit words; the
|
|
41
|
+
verbalizer writes the currency symbol, digits and two-digit cents.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(self, cardinal):
|
|
45
|
+
super().__init__(name="money", ordertype="itn", token_orders=TOKEN_ORDER)
|
|
46
|
+
number = cardinal.graph
|
|
47
|
+
major = "arabic/data/money/currency_major.tsv"
|
|
48
|
+
local = "arabic/data/money/local_currency.tsv"
|
|
49
|
+
# Names listed in both tables (ريال سعودي, درهم إماراتي) take the local symbol.
|
|
50
|
+
self.currency = symbols(local, major).optimize()
|
|
51
|
+
major_names = self.input_projection(symbols(major))
|
|
52
|
+
local_names = self.input_projection(symbols(local))
|
|
53
|
+
minor = [row[1] for name in ("singular", "plural")
|
|
54
|
+
for row in load_labels(get_abs_path("arabic/data/money/currency_minor_{}.tsv".format(name)))]
|
|
55
|
+
minor_names = pynini.union(*[pynini.escape(name) for name in minor]).optimize()
|
|
56
|
+
self.cents = (number @ (self.DIGIT**2 | insert("0") + self.DIGIT)).optimize()
|
|
57
|
+
|
|
58
|
+
integer = self.tag_field("integer_part", number) + self.DELETE_EXTRA_SPACE
|
|
59
|
+
cents = self.tag_field("fractional_part", self.cents) + self.DELETE_SPACE + insert(" ")
|
|
60
|
+
cents += self.tag_field("currency_min", minor_names)
|
|
61
|
+
joined = self.DELETE_SPACE + delete("و") + self.DELETE_ZERO_OR_ONE_SPACE + insert(" ") + cents
|
|
62
|
+
graph = integer + self.tag_field("currency", major_names) + joined.ques
|
|
63
|
+
graph |= integer + self.tag_field("currency", local_names)
|
|
64
|
+
graph |= cents
|
|
65
|
+
self.tagger = self.add_tokens(graph)
|
|
66
|
+
|
|
67
|
+
currency = self.verbalize_field("currency", self.currency)
|
|
68
|
+
amount = self.verbalize_field("integer_part", number, True)
|
|
69
|
+
fractional = insert(".") + self.verbalize_field("fractional_part", self.cents, True)
|
|
70
|
+
minor_unit = self.verbalize_field("currency_min", delete(minor_names), True)
|
|
71
|
+
# Cents on their own are dollars, as in the original grammar.
|
|
72
|
+
alone = insert("$0.") + self.verbalize_field("fractional_part", self.cents) + minor_unit
|
|
73
|
+
self.verbalizer = self.delete_tokens(currency + amount + (fractional + minor_unit).ques | alone)
|