WeTextProcessing 1.0.6.dev19__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- itn/README.md +12 -4
- itn/arabic/__init__.py +0 -0
- itn/arabic/data/measure/measurements.tsv +14 -0
- itn/arabic/data/money/currency_major.tsv +24 -0
- itn/arabic/data/money/currency_minor_plural.tsv +4 -0
- itn/arabic/data/money/currency_minor_singular.tsv +3 -0
- itn/arabic/data/money/local_currency.tsv +20 -0
- itn/arabic/data/number/digit.tsv +9 -0
- itn/arabic/data/number/digit_100.tsv +7 -0
- itn/arabic/data/number/digit_1000.tsv +7 -0
- itn/arabic/data/number/flops.tsv +72 -0
- itn/arabic/data/number/fraction_dual.tsv +9 -0
- itn/arabic/data/number/fraction_plural.tsv +8 -0
- itn/arabic/data/number/fraction_singular.tsv +9 -0
- itn/arabic/data/number/quantities.tsv +6 -0
- itn/arabic/data/number/teens.tsv +9 -0
- itn/arabic/data/number/tens.tsv +10 -0
- itn/arabic/inverse_normalizer.py +60 -0
- itn/arabic/rules/__init__.py +0 -0
- itn/arabic/rules/cardinal.py +66 -0
- itn/arabic/rules/decimal.py +58 -0
- itn/arabic/rules/fraction.py +72 -0
- itn/arabic/rules/measure.py +49 -0
- itn/arabic/rules/money.py +73 -0
- itn/arabic/rules/punctuation.py +29 -0
- itn/arabic/rules/word.py +28 -0
- itn/arabic/test/__init__.py +0 -0
- itn/arabic/test/data/test_cases_cardinal.txt +34 -0
- itn/arabic/test/data/test_cases_decimal.txt +30 -0
- itn/arabic/test/data/test_cases_fraction.txt +20 -0
- itn/arabic/test/data/test_cases_measure.txt +15 -0
- itn/arabic/test/data/test_cases_money.txt +21 -0
- itn/arabic/test/data/test_cases_whitelist.txt +6 -0
- itn/arabic/test/normalizer_test.py +137 -0
- itn/armenian/__init__.py +0 -0
- itn/armenian/data/currency.tsv +53 -0
- itn/armenian/data/measurement_dates.tsv +4 -0
- itn/armenian/data/measurements.tsv +48 -0
- itn/armenian/data/numbers/digit.tsv +11 -0
- itn/armenian/data/numbers/digits_no_one.tsv +10 -0
- itn/armenian/data/numbers/ties.tsv +10 -0
- itn/armenian/data/ordinals/digit.tsv +10 -0
- itn/armenian/data/time/hours.tsv +29 -0
- itn/armenian/data/time/minutes.tsv +65 -0
- itn/armenian/data/time/minutes_to.tsv +59 -0
- itn/armenian/data/time/to_hour.tsv +25 -0
- itn/armenian/data/whitelist.tsv +7 -0
- itn/armenian/inverse_normalizer.py +66 -0
- itn/armenian/rules/__init__.py +0 -0
- itn/armenian/rules/cardinal.py +67 -0
- itn/armenian/rules/decimal.py +67 -0
- itn/armenian/rules/fraction.py +40 -0
- itn/armenian/rules/measure.py +52 -0
- itn/armenian/rules/money.py +38 -0
- itn/armenian/rules/ordinal.py +37 -0
- itn/armenian/rules/punctuation.py +29 -0
- itn/armenian/rules/time.py +59 -0
- itn/armenian/rules/utils.py +58 -0
- itn/armenian/rules/whitelist.py +27 -0
- itn/armenian/rules/word.py +28 -0
- itn/armenian/test/__init__.py +0 -0
- itn/armenian/test/data/test_cases_cardinal.txt +23 -0
- itn/armenian/test/data/test_cases_decimal.txt +14 -0
- itn/armenian/test/data/test_cases_fraction.txt +24 -0
- itn/armenian/test/data/test_cases_measure.txt +12 -0
- itn/armenian/test/data/test_cases_money.txt +10 -0
- itn/armenian/test/data/test_cases_ordinal.txt +11 -0
- itn/armenian/test/data/test_cases_time.txt +9 -0
- itn/armenian/test/data/test_cases_whitelist.txt +7 -0
- itn/armenian/test/data/test_cases_word.txt +50 -0
- itn/armenian/test/normalizer_test.py +158 -0
- itn/chinese/inverse_normalizer.py +49 -33
- itn/chinese/rules/cardinal.py +42 -69
- itn/chinese/rules/date.py +25 -18
- itn/chinese/rules/fraction.py +15 -14
- itn/chinese/rules/license_plate.py +11 -10
- itn/chinese/rules/math.py +7 -7
- itn/chinese/rules/measure.py +27 -42
- itn/chinese/rules/money.py +12 -18
- itn/chinese/rules/postprocessor.py +2 -2
- itn/chinese/rules/time.py +24 -25
- itn/chinese/rules/train_number.py +9 -8
- itn/chinese/rules/whitelist.py +7 -6
- itn/chinese/test/normalizer_test.py +155 -30
- itn/english/__init__.py +0 -0
- itn/english/data/currency.tsv +168 -0
- itn/english/data/date_period.tsv +10 -0
- itn/english/data/electronic/domain.tsv +10 -0
- itn/english/data/electronic/server_name.tsv +17 -0
- itn/english/data/electronic/symbols.tsv +22 -0
- itn/english/data/electronic/url_symbols.tsv +5 -0
- itn/english/data/magnitudes.tsv +4 -0
- itn/english/data/measurements.tsv +149 -0
- itn/english/data/months.tsv +12 -0
- itn/english/data/months_cased.tsv +12 -0
- itn/english/data/numbers/digit.tsv +9 -0
- itn/english/data/numbers/hundred.tsv +1 -0
- itn/english/data/numbers/teen.tsv +10 -0
- itn/english/data/numbers/thousands.tsv +22 -0
- itn/english/data/numbers/ties.tsv +9 -0
- itn/english/data/numbers/zero.tsv +1 -0
- itn/english/data/ordinals/digit.tsv +9 -0
- itn/english/data/ordinals/teen.tsv +1 -0
- itn/english/data/time/minute_to.tsv +59 -0
- itn/english/data/time/time_suffix.tsv +8 -0
- itn/english/data/time/time_suffix_cased.tsv +8 -0
- itn/english/data/time/time_zone.tsv +7 -0
- itn/english/data/time/time_zone_cased.tsv +7 -0
- itn/english/data/time/to_hour.tsv +12 -0
- itn/english/data/whitelist.tsv +417 -0
- itn/english/data/whitelist_tech.tsv +450 -0
- itn/english/data/year_suffix.tsv +6 -0
- itn/english/inverse_normalizer.py +91 -0
- itn/english/rules/__init__.py +0 -0
- itn/english/rules/cardinal.py +111 -0
- itn/english/rules/char.py +29 -0
- itn/english/rules/date.py +163 -0
- itn/english/rules/decimal.py +82 -0
- itn/english/rules/electronic.py +79 -0
- itn/english/rules/measure.py +106 -0
- itn/english/rules/money.py +121 -0
- itn/english/rules/ordinal.py +50 -0
- itn/english/rules/punctuation.py +31 -0
- itn/english/rules/telephone.py +147 -0
- itn/english/rules/time.py +138 -0
- itn/english/rules/whitelist.py +33 -0
- itn/english/rules/word.py +26 -0
- itn/english/test/__init__.py +0 -0
- itn/english/test/data/en_cardinal.txt +28 -0
- itn/english/test/data/en_date.txt +44 -0
- itn/english/test/data/en_decimal.txt +63 -0
- itn/english/test/data/en_electronic.txt +25 -0
- itn/english/test/data/en_measure.txt +114 -0
- itn/english/test/data/en_money.txt +52 -0
- itn/english/test/data/en_ordinal.txt +34 -0
- itn/english/test/data/en_telephone.txt +24 -0
- itn/english/test/data/en_time.txt +36 -0
- itn/english/test/data/en_whitelist.txt +12 -0
- itn/english/test/data/en_word.txt +55 -0
- itn/english/test/normalizer_test.py +130 -0
- tn/chinese/test/preprocessor_test.py → itn/english/test/utils.py +14 -11
- itn/french/__init__.py +0 -0
- itn/french/data/electronic/domain.tsv +25 -0
- itn/french/data/electronic/server_name.tsv +17 -0
- itn/french/data/electronic/symbols.tsv +12 -0
- itn/french/data/fractions.tsv +41 -0
- itn/french/data/measurements/magnitudes.tsv +16 -0
- itn/french/data/measurements/measurements.tsv +27 -0
- itn/french/data/money/currency_major.tsv +40 -0
- itn/french/data/money/currency_minor.tsv +5 -0
- itn/french/data/months.tsv +12 -0
- itn/french/data/numbers/digit.tsv +10 -0
- itn/french/data/numbers/teen.tsv +6 -0
- itn/french/data/numbers/ties.tsv +13 -0
- itn/french/data/numbers/ties_unique.tsv +25 -0
- itn/french/data/numbers/zero.tsv +1 -0
- itn/french/data/ordinals/digits_root_change.tsv +17 -0
- itn/french/data/ordinals/firsts.tsv +4 -0
- itn/french/data/ordinals/key_nouns.tsv +1 -0
- itn/french/data/ordinals/second.tsv +4 -0
- itn/french/data/roman/digits_large.tsv +9 -0
- itn/french/data/roman/hundreds_large.tsv +9 -0
- itn/french/data/roman/ties_large.tsv +9 -0
- itn/french/data/time/hour_to_night.tsv +12 -0
- itn/french/data/time/hours.tsv +26 -0
- itn/french/data/time/hours_to.tsv +25 -0
- itn/french/data/time/minutes.tsv +63 -0
- itn/french/data/time/minutes_to.tsv +59 -0
- itn/french/data/time/time_suffix_am.tsv +1 -0
- itn/french/data/time/time_suffix_pm.tsv +2 -0
- itn/french/data/whitelist.tsv +16 -0
- itn/french/inverse_normalizer.py +74 -0
- itn/french/rules/__init__.py +0 -0
- itn/french/rules/cardinal.py +115 -0
- itn/french/rules/date.py +45 -0
- itn/french/rules/decimal.py +70 -0
- itn/french/rules/electronic.py +67 -0
- itn/french/rules/fraction.py +55 -0
- itn/french/rules/measure.py +55 -0
- itn/french/rules/money.py +67 -0
- itn/french/rules/ordinal.py +63 -0
- itn/french/rules/punctuation.py +29 -0
- itn/french/rules/telephone.py +59 -0
- itn/french/rules/time.py +70 -0
- itn/french/rules/whitelist.py +27 -0
- itn/french/rules/word.py +29 -0
- itn/french/test/__init__.py +0 -0
- itn/french/test/data/test_cases_cardinal.txt +106 -0
- itn/french/test/data/test_cases_date.txt +6 -0
- itn/french/test/data/test_cases_decimal.txt +15 -0
- itn/french/test/data/test_cases_electronic.txt +10 -0
- itn/french/test/data/test_cases_fraction.txt +30 -0
- itn/french/test/data/test_cases_measure.txt +15 -0
- itn/french/test/data/test_cases_money.txt +22 -0
- itn/french/test/data/test_cases_ordinal.txt +23 -0
- itn/french/test/data/test_cases_telephone.txt +5 -0
- itn/french/test/data/test_cases_time.txt +18 -0
- itn/french/test/data/test_cases_whitelist.txt +8 -0
- itn/french/test/data/test_cases_word.txt +49 -0
- itn/french/test/normalizer_test.py +155 -0
- itn/german/__init__.py +0 -0
- itn/german/inverse_normalizer.py +85 -0
- itn/german/rules/__init__.py +0 -0
- itn/german/rules/cardinal.py +48 -0
- itn/german/rules/date.py +63 -0
- itn/german/rules/decimal.py +67 -0
- itn/german/rules/electronic.py +58 -0
- itn/german/rules/fraction.py +75 -0
- itn/german/rules/measure.py +80 -0
- itn/german/rules/money.py +87 -0
- itn/german/rules/ordinal.py +31 -0
- itn/german/rules/punctuation.py +29 -0
- itn/german/rules/telephone.py +45 -0
- itn/german/rules/time.py +102 -0
- itn/german/rules/whitelist.py +30 -0
- itn/german/rules/word.py +33 -0
- itn/german/test/__init__.py +0 -0
- itn/german/test/calendar_test.py +147 -0
- itn/german/test/data/test_cases_cardinal.txt +62 -0
- itn/german/test/data/test_cases_date.txt +22 -0
- itn/german/test/data/test_cases_decimal.txt +10 -0
- itn/german/test/data/test_cases_electronic.txt +9 -0
- itn/german/test/data/test_cases_fraction.txt +34 -0
- itn/german/test/data/test_cases_measure.txt +28 -0
- itn/german/test/data/test_cases_money.txt +23 -0
- itn/german/test/data/test_cases_ordinal.txt +20 -0
- itn/german/test/data/test_cases_telephone.txt +1 -0
- itn/german/test/data/test_cases_time.txt +24 -0
- itn/german/test/data/test_cases_whitelist.txt +6 -0
- itn/german/test/data/test_cases_word.txt +49 -0
- itn/german/test/financial_test.py +284 -0
- itn/german/test/inverse_normalizer_test.py +92 -0
- itn/german/test/numeric_test.py +219 -0
- itn/german/test/text_test.py +281 -0
- itn/german/test/time_test.py +70 -0
- itn/hebrew/__init__.py +0 -0
- itn/hebrew/data/decimals/minutes_exception.tsv +2 -0
- itn/hebrew/data/measurements.tsv +45 -0
- itn/hebrew/data/months.tsv +13 -0
- itn/hebrew/data/months_name2number.tsv +12 -0
- itn/hebrew/data/months_ordinal2number.tsv +12 -0
- itn/hebrew/data/numbers/decimal_fractions.tsv +6 -0
- itn/hebrew/data/numbers/digit.tsv +20 -0
- itn/hebrew/data/numbers/hundreds_exception.tsv +2 -0
- itn/hebrew/data/numbers/millions_exception.tsv +1 -0
- itn/hebrew/data/numbers/teen.tsv +21 -0
- itn/hebrew/data/numbers/thousands.tsv +8 -0
- itn/hebrew/data/numbers/thousands_exception.tsv +2 -0
- itn/hebrew/data/numbers/ties.tsv +8 -0
- itn/hebrew/data/numbers/viable_hours.tsv +15 -0
- itn/hebrew/data/numbers/zero.tsv +1 -0
- itn/hebrew/data/ordinals/digit.tsv +10 -0
- itn/hebrew/data/prefix.tsv +17 -0
- itn/hebrew/data/spaced_measurements.tsv +17 -0
- itn/hebrew/data/time/day_suffix.tsv +2 -0
- itn/hebrew/data/time/evening_suffix.tsv +2 -0
- itn/hebrew/data/time/hour_to_evening.tsv +7 -0
- itn/hebrew/data/time/hour_to_night.tsv +9 -0
- itn/hebrew/data/time/hour_to_noon.tsv +7 -0
- itn/hebrew/data/time/midnight_to_hour.tsv +1 -0
- itn/hebrew/data/time/minute_to.tsv +58 -0
- itn/hebrew/data/time/minute_to_verbose.tsv +6 -0
- itn/hebrew/data/time/minute_verbose.tsv +8 -0
- itn/hebrew/data/time/night_suffix.tsv +1 -0
- itn/hebrew/data/time/noon_suffix.tsv +3 -0
- itn/hebrew/data/time/to_hour.tsv +13 -0
- itn/hebrew/data/whitelist.tsv +20 -0
- itn/hebrew/inverse_normalizer.py +64 -0
- itn/hebrew/rules/__init__.py +0 -0
- itn/hebrew/rules/cardinal.py +94 -0
- itn/hebrew/rules/date.py +62 -0
- itn/hebrew/rules/decimal.py +74 -0
- itn/hebrew/rules/measure.py +53 -0
- itn/hebrew/rules/punctuation.py +28 -0
- itn/hebrew/rules/time.py +101 -0
- itn/hebrew/rules/utils.py +78 -0
- itn/hebrew/rules/whitelist.py +30 -0
- itn/hebrew/rules/word.py +28 -0
- itn/hebrew/test/__init__.py +0 -0
- itn/hebrew/test/data/test_cases_cardinal.txt +138 -0
- itn/hebrew/test/data/test_cases_date.txt +29 -0
- itn/hebrew/test/data/test_cases_decimal.txt +66 -0
- itn/hebrew/test/data/test_cases_measure.txt +10 -0
- itn/hebrew/test/data/test_cases_time.txt +34 -0
- itn/hebrew/test/data/test_cases_whitelist.txt +4 -0
- itn/hebrew/test/data/test_full_sentences.txt +56 -0
- itn/hebrew/test/normalizer_test.py +178 -0
- itn/hindi/__init__.py +0 -0
- itn/hindi/data/address/cities.tsv +36 -0
- itn/hindi/data/address/context_cues.tsv +85 -0
- itn/hindi/data/address/digit_passthrough.tsv +10 -0
- itn/hindi/data/address/ordinals.tsv +167 -0
- itn/hindi/data/address/special_characters.tsv +2 -0
- itn/hindi/data/address/states.tsv +36 -0
- itn/hindi/data/date/century.tsv +5 -0
- itn/hindi/data/date/months.tsv +14 -0
- itn/hindi/data/measure/measurements.tsv +295 -0
- itn/hindi/data/money/currency.tsv +181 -0
- itn/hindi/data/numbers/digit.tsv +13 -0
- itn/hindi/data/numbers/hundred.tsv +1 -0
- itn/hindi/data/numbers/labels_exception.tsv +4 -0
- itn/hindi/data/numbers/paune.tsv +231 -0
- itn/hindi/data/numbers/teens_and_ties.tsv +220 -0
- itn/hindi/data/numbers/thousands.tsv +8 -0
- itn/hindi/data/numbers/zero.tsv +1 -0
- itn/hindi/data/ordinals/digit.tsv +6 -0
- itn/hindi/data/ordinals/digit_fem.tsv +6 -0
- itn/hindi/data/ordinals/hundred_digit.tsv +12 -0
- itn/hindi/data/ordinals/hundred_digit_fem.tsv +12 -0
- itn/hindi/data/ordinals/morph_features.tsv +6 -0
- itn/hindi/data/ordinals/suffix.tsv +4 -0
- itn/hindi/data/ordinals/suffix_fem.tsv +2 -0
- itn/hindi/data/ordinals/teens_and_ties.tsv +1 -0
- itn/hindi/data/ordinals/teens_and_ties_fem.tsv +1 -0
- itn/hindi/data/telephone/context_cues.tsv +15 -0
- itn/hindi/data/telephone/country_codes.tsv +1 -0
- itn/hindi/data/telephone/eng_digit.tsv +9 -0
- itn/hindi/data/telephone/eng_zero.tsv +1 -0
- itn/hindi/data/telephone/teens_and_ties_eng_to_hin.tsv +90 -0
- itn/hindi/data/time/hour_for_paune.tsv +15 -0
- itn/hindi/data/whitelist/whitelist.tsv +7 -0
- itn/hindi/inverse_normalizer.py +78 -0
- itn/hindi/rules/__init__.py +0 -0
- itn/hindi/rules/cardinal.py +93 -0
- itn/hindi/rules/date.py +64 -0
- itn/hindi/rules/decimal.py +57 -0
- itn/hindi/rules/fraction.py +55 -0
- itn/hindi/rules/measure.py +83 -0
- itn/hindi/rules/money.py +81 -0
- itn/hindi/rules/ordinal.py +40 -0
- itn/hindi/rules/punctuation.py +31 -0
- itn/hindi/rules/telephone.py +65 -0
- itn/hindi/rules/time.py +53 -0
- itn/hindi/rules/utils.py +52 -0
- itn/hindi/rules/whitelist.py +32 -0
- itn/hindi/rules/word.py +30 -0
- itn/hindi/test/__init__.py +0 -0
- itn/hindi/test/data/test_cases_address.txt +25 -0
- itn/hindi/test/data/test_cases_cardinal.txt +54 -0
- itn/hindi/test/data/test_cases_date.txt +42 -0
- itn/hindi/test/data/test_cases_decimal.txt +13 -0
- itn/hindi/test/data/test_cases_fraction.txt +31 -0
- itn/hindi/test/data/test_cases_measure.txt +48 -0
- itn/hindi/test/data/test_cases_money.txt +50 -0
- itn/hindi/test/data/test_cases_ordinal.txt +13 -0
- itn/hindi/test/data/test_cases_telephone.txt +28 -0
- itn/hindi/test/data/test_cases_time.txt +25 -0
- itn/hindi/test/data/test_cases_whitelist.txt +8 -0
- itn/hindi/test/data/test_cases_word.txt +15 -0
- itn/hindi/test/normalizer_test.py +127 -0
- itn/hindi_english/__init__.py +0 -0
- itn/hindi_english/data/measurements.tsv +2 -0
- itn/hindi_english/inverse_normalizer.py +93 -0
- itn/hindi_english/rules/__init__.py +0 -0
- itn/hindi_english/rules/english.py +175 -0
- itn/hindi_english/test/__init__.py +0 -0
- itn/hindi_english/test/data/test_cases_address.txt +41 -0
- itn/hindi_english/test/data/test_cases_cardinal.txt +61 -0
- itn/hindi_english/test/data/test_cases_date.txt +37 -0
- itn/hindi_english/test/data/test_cases_date_cased.txt +70 -0
- itn/hindi_english/test/data/test_cases_decimal.txt +25 -0
- itn/hindi_english/test/data/test_cases_electronic.txt +17 -0
- itn/hindi_english/test/data/test_cases_fraction.txt +39 -0
- itn/hindi_english/test/data/test_cases_measure.txt +25 -0
- itn/hindi_english/test/data/test_cases_money.txt +41 -0
- itn/hindi_english/test/data/test_cases_ordinal.txt +26 -0
- itn/hindi_english/test/data/test_cases_telephone.txt +39 -0
- itn/hindi_english/test/data/test_cases_time.txt +29 -0
- itn/hindi_english/test/data/test_cases_whitelist.txt +17 -0
- itn/hindi_english/test/data/test_cases_word.txt +19 -0
- itn/hindi_english/test/normalizer_test.py +211 -0
- itn/hungarian/__init__.py +0 -0
- itn/hungarian/data/dates/days.tsv +31 -0
- itn/hungarian/data/dates/months.tsv +12 -0
- itn/hungarian/data/electronic/domain.tsv +15 -0
- itn/hungarian/data/electronic/server_name.tsv +15 -0
- itn/hungarian/data/electronic/symbols.tsv +23 -0
- itn/hungarian/data/inflection/endings.tsv +51 -0
- itn/hungarian/data/inflection/plural_endings.tsv +39 -0
- itn/hungarian/data/inflection/word_endings.tsv +1 -0
- itn/hungarian/data/measures/measurements.tsv +52 -0
- itn/hungarian/data/money/currency.tsv +11 -0
- itn/hungarian/data/money/currency_minor.tsv +11 -0
- itn/hungarian/data/number/digit.tsv +9 -0
- itn/hungarian/data/number/digit_inline.tsv +9 -0
- itn/hungarian/data/number/quantities.tsv +21 -0
- itn/hungarian/data/number/tens.tsv +9 -0
- itn/hungarian/data/number/tens_inline.tsv +9 -0
- itn/hungarian/data/number/zero.tsv +1 -0
- itn/hungarian/data/ordinals/endings.tsv +23 -0
- itn/hungarian/data/ordinals/exceptional.tsv +4 -0
- itn/hungarian/data/telephone/area_codes.tsv +71 -0
- itn/hungarian/data/telephone/country_codes.tsv +213 -0
- itn/hungarian/data/time/time_zone.tsv +9 -0
- itn/hungarian/data/whitelist.tsv +24 -0
- itn/hungarian/data/whitelist_inflect.tsv +39 -0
- itn/hungarian/data/whitelist_inflect_sg.tsv +14 -0
- itn/hungarian/inverse_normalizer.py +93 -0
- itn/hungarian/rules/__init__.py +0 -0
- itn/hungarian/rules/cardinal.py +99 -0
- itn/hungarian/rules/date.py +66 -0
- itn/hungarian/rules/decimal.py +80 -0
- itn/hungarian/rules/electronic.py +71 -0
- itn/hungarian/rules/fraction.py +41 -0
- itn/hungarian/rules/measure.py +74 -0
- itn/hungarian/rules/money.py +82 -0
- itn/hungarian/rules/ordinal.py +42 -0
- itn/hungarian/rules/punctuation.py +44 -0
- itn/hungarian/rules/telephone.py +68 -0
- itn/hungarian/rules/time.py +99 -0
- itn/hungarian/rules/utils.py +143 -0
- itn/hungarian/rules/whitelist.py +50 -0
- itn/hungarian/rules/word.py +38 -0
- itn/hungarian/test/__init__.py +0 -0
- itn/hungarian/test/normalizer_test.py +214 -0
- itn/italian/__init__.py +0 -0
- itn/italian/data/electronic/domain.tsv +14 -0
- itn/italian/data/electronic/server_name.tsv +11 -0
- itn/italian/data/electronic/symbols.tsv +22 -0
- itn/italian/data/measure/denominators.tsv +4 -0
- itn/italian/data/measure/units.tsv +51 -0
- itn/italian/data/money/currency.tsv +11 -0
- itn/italian/data/numbers/digit.tsv +9 -0
- itn/italian/data/numbers/hundreds.tsv +9 -0
- itn/italian/data/numbers/quantities.tsv +4 -0
- itn/italian/data/numbers/teen.tsv +10 -0
- itn/italian/data/numbers/tens.tsv +8 -0
- itn/italian/data/whitelist.tsv +19 -0
- itn/italian/inverse_normalizer.py +77 -0
- itn/italian/rules/__init__.py +0 -0
- itn/italian/rules/cardinal.py +100 -0
- itn/italian/rules/decimal.py +59 -0
- itn/italian/rules/electronic.py +70 -0
- itn/italian/rules/measure.py +52 -0
- itn/italian/rules/money.py +78 -0
- itn/italian/rules/punctuation.py +27 -0
- itn/italian/rules/time.py +84 -0
- itn/italian/rules/utils.py +49 -0
- itn/italian/rules/whitelist.py +31 -0
- itn/italian/rules/word.py +27 -0
- itn/italian/test/__init__.py +0 -0
- itn/italian/test/normalizer_test.py +277 -0
- itn/japanese/inverse_normalizer.py +62 -47
- itn/japanese/rules/cardinal.py +40 -55
- itn/japanese/rules/char.py +7 -5
- itn/japanese/rules/date.py +22 -25
- itn/japanese/rules/fraction.py +21 -19
- itn/japanese/rules/math.py +9 -8
- itn/japanese/rules/measure.py +14 -13
- itn/japanese/rules/money.py +10 -7
- itn/japanese/rules/ordinal.py +7 -5
- itn/japanese/rules/postprocessor.py +5 -5
- itn/japanese/rules/preprocessor.py +2 -2
- itn/japanese/rules/time.py +16 -17
- itn/japanese/rules/whitelist.py +9 -7
- itn/japanese/test/normalizer_test.py +132 -30
- itn/kinyarwanda/__init__.py +0 -0
- itn/kinyarwanda/data/numbers/digit.tsv +9 -0
- itn/kinyarwanda/data/numbers/digit_bi.tsv +9 -0
- itn/kinyarwanda/data/numbers/digit_e.tsv +9 -0
- itn/kinyarwanda/data/numbers/hundreds.tsv +9 -0
- itn/kinyarwanda/data/numbers/tens.tsv +8 -0
- itn/kinyarwanda/data/numbers/zero.tsv +1 -0
- itn/kinyarwanda/data/time/hours.tsv +12 -0
- itn/kinyarwanda/data/time/minutes.tsv +59 -0
- itn/kinyarwanda/data/whitelist.tsv +18 -0
- itn/kinyarwanda/inverse_normalizer.py +53 -0
- itn/kinyarwanda/rules/__init__.py +0 -0
- itn/kinyarwanda/rules/cardinal.py +131 -0
- itn/kinyarwanda/rules/punctuation.py +28 -0
- itn/kinyarwanda/rules/time.py +39 -0
- itn/kinyarwanda/rules/utils.py +39 -0
- itn/kinyarwanda/rules/whitelist.py +32 -0
- itn/kinyarwanda/rules/word.py +28 -0
- itn/kinyarwanda/test/__init__.py +0 -0
- itn/kinyarwanda/test/normalizer_test.py +194 -0
- itn/korean/__init__.py +3 -0
- itn/korean/data/__init__.py +13 -0
- itn/korean/data/currency.tsv +9 -0
- itn/korean/data/measure_units.tsv +61 -0
- itn/korean/data/months.tsv +12 -0
- itn/korean/data/numbers/__init__.py +13 -0
- itn/korean/data/numbers/digit.tsv +9 -0
- itn/korean/data/numbers/zero.tsv +1 -0
- itn/korean/data/ordinals/__init__.py +13 -0
- itn/korean/data/ordinals/counter_suffix.tsv +17 -0
- itn/korean/data/ordinals/digit.tsv +9 -0
- itn/korean/data/time/__init__.py +13 -0
- itn/korean/data/time/ten_prefix.tsv +4 -0
- itn/korean/data/time/time_hours.tsv +23 -0
- itn/korean/data/whitelist.tsv +35 -0
- itn/korean/inverse_normalizer.py +76 -0
- itn/korean/rules/__init__.py +3 -0
- itn/korean/rules/cardinal.py +44 -0
- itn/korean/rules/date.py +42 -0
- itn/korean/rules/decimal.py +43 -0
- itn/korean/rules/fraction.py +48 -0
- itn/korean/rules/measure.py +52 -0
- itn/korean/rules/money.py +34 -0
- itn/korean/rules/ordinal.py +49 -0
- itn/korean/rules/telephone.py +30 -0
- itn/korean/rules/time.py +61 -0
- itn/korean/rules/utils.py +22 -0
- tn/chinese/rules/preprocessor.py → itn/korean/rules/whitelist.py +10 -13
- itn/korean/rules/word.py +27 -0
- itn/korean/test/__init__.py +0 -0
- itn/korean/test/data/test_cases_cardinal.txt +37 -0
- itn/korean/test/data/test_cases_date.txt +15 -0
- itn/korean/test/data/test_cases_decimal.txt +14 -0
- itn/korean/test/data/test_cases_fraction.txt +29 -0
- itn/korean/test/data/test_cases_measure.txt +46 -0
- itn/korean/test/data/test_cases_money.txt +29 -0
- itn/korean/test/data/test_cases_ordinal.txt +17 -0
- itn/korean/test/data/test_cases_telephone.txt +7 -0
- itn/korean/test/data/test_cases_time.txt +15 -0
- itn/korean/test/test_inverse_normalizer.py +184 -0
- itn/main.py +2 -51
- itn/marathi/__init__.py +0 -0
- itn/marathi/data/date/eras.tsv +2 -0
- itn/marathi/data/date/months.tsv +12 -0
- itn/marathi/data/numbers/digit.tsv +9 -0
- itn/marathi/data/numbers/tens.tsv +90 -0
- itn/marathi/data/numbers/zero.tsv +1 -0
- itn/marathi/inverse_normalizer.py +56 -0
- itn/marathi/rules/__init__.py +0 -0
- itn/marathi/rules/cardinal.py +76 -0
- itn/marathi/rules/date.py +63 -0
- itn/marathi/rules/decimal.py +62 -0
- itn/marathi/rules/punctuation.py +29 -0
- itn/marathi/rules/time.py +69 -0
- itn/marathi/rules/utils.py +62 -0
- itn/marathi/rules/word.py +28 -0
- itn/marathi/test/__init__.py +0 -0
- itn/marathi/test/data/test_cases_cardinal.txt +33 -0
- itn/marathi/test/data/test_cases_date.txt +29 -0
- itn/marathi/test/data/test_cases_decimal.txt +29 -0
- itn/marathi/test/data/test_cases_time.txt +29 -0
- itn/marathi/test/data/test_cases_word.txt +11 -0
- itn/marathi/test/normalizer_test.py +164 -0
- itn/portuguese/__init__.py +0 -0
- itn/portuguese/data/currency_plural.tsv +5 -0
- itn/portuguese/data/currency_singular.tsv +5 -0
- itn/portuguese/data/electronic/domain.tsv +26 -0
- itn/portuguese/data/electronic/server_name.tsv +11 -0
- itn/portuguese/data/electronic/symbols.tsv +6 -0
- itn/portuguese/data/measurements_plural.tsv +56 -0
- itn/portuguese/data/measurements_singular.tsv +55 -0
- itn/portuguese/data/months.tsv +12 -0
- itn/portuguese/data/numbers/digit.tsv +11 -0
- itn/portuguese/data/numbers/hundreds.tsv +17 -0
- itn/portuguese/data/numbers/onehundred.tsv +1 -0
- itn/portuguese/data/numbers/teen.tsv +11 -0
- itn/portuguese/data/numbers/ties.tsv +8 -0
- itn/portuguese/data/numbers/twenties.tsv +9 -0
- itn/portuguese/data/numbers/zero.tsv +1 -0
- itn/portuguese/data/ordinals/digit.tsv +18 -0
- itn/portuguese/data/ordinals/hundreds.tsv +28 -0
- itn/portuguese/data/ordinals/ties.tsv +20 -0
- itn/portuguese/data/time/hours_to.tsv +23 -0
- itn/portuguese/data/time/minutes_to.tsv +59 -0
- itn/portuguese/data/time/time_suffix_am.tsv +2 -0
- itn/portuguese/data/time/time_suffix_pm.tsv +2 -0
- itn/portuguese/data/whitelist.tsv +5 -0
- itn/portuguese/inverse_normalizer.py +69 -0
- itn/portuguese/rules/__init__.py +0 -0
- itn/portuguese/rules/cardinal.py +117 -0
- itn/portuguese/rules/date.py +64 -0
- itn/portuguese/rules/decimal.py +76 -0
- itn/portuguese/rules/electronic.py +69 -0
- itn/portuguese/rules/measure.py +61 -0
- itn/portuguese/rules/money.py +89 -0
- itn/portuguese/rules/ordinal.py +56 -0
- itn/portuguese/rules/punctuation.py +32 -0
- itn/portuguese/rules/telephone.py +56 -0
- itn/portuguese/rules/time.py +107 -0
- itn/portuguese/rules/whitelist.py +32 -0
- itn/portuguese/rules/word.py +31 -0
- itn/portuguese/test/__init__.py +0 -0
- itn/portuguese/test/data/test_cases_cardinal.txt +70 -0
- itn/portuguese/test/data/test_cases_date.txt +15 -0
- itn/portuguese/test/data/test_cases_decimal.txt +26 -0
- itn/portuguese/test/data/test_cases_electronic.txt +13 -0
- itn/portuguese/test/data/test_cases_measure.txt +12 -0
- itn/portuguese/test/data/test_cases_money.txt +25 -0
- itn/portuguese/test/data/test_cases_ordinal.txt +19 -0
- itn/portuguese/test/data/test_cases_telephone.txt +27 -0
- itn/portuguese/test/data/test_cases_time.txt +19 -0
- itn/portuguese/test/data/test_cases_whitelist.txt +6 -0
- itn/portuguese/test/data/test_cases_word.txt +49 -0
- itn/portuguese/test/normalizer_test.py +156 -0
- itn/russian/__init__.py +0 -0
- itn/russian/data/currency/currency_plural.tsv +53 -0
- itn/russian/data/currency/currency_singular.tsv +48 -0
- itn/russian/data/electronic/domain.tsv +9 -0
- itn/russian/data/electronic/server_name.tsv +17 -0
- itn/russian/data/electronic/symbols.tsv +28 -0
- itn/russian/data/latin_to_cyrillic.tsv +38 -0
- itn/russian/data/measurements.tsv +321 -0
- itn/russian/data/months/abbr.tsv +12 -0
- itn/russian/data/months/abbr_to_name.tsv +62 -0
- itn/russian/data/months/numbers.tsv +21 -0
- itn/russian/data/numbers/1_cardinals_nominative.tsv +50 -0
- itn/russian/data/numbers/2_cardinals_genitive.tsv +45 -0
- itn/russian/data/numbers/3_cardinals_dative.tsv +45 -0
- itn/russian/data/numbers/4_cardinals_accusative.tsv +48 -0
- itn/russian/data/numbers/5_cardinals_instrumental.tsv +47 -0
- itn/russian/data/numbers/6_cardinals_prepositional.tsv +46 -0
- itn/russian/data/numbers/cardinal_endings.tsv +6 -0
- itn/russian/data/numbers/cardinals_alternatives.tsv +16 -0
- itn/russian/data/numbers/cardinals_nominative_case.tsv +37 -0
- itn/russian/data/numbers/decimal_delimiter.tsv +15 -0
- itn/russian/data/numbers/decimal_endings.tsv +37 -0
- itn/russian/data/numbers/digits_nominative_case.tsv +10 -0
- itn/russian/data/numbers/ordinal_endings.tsv +40 -0
- itn/russian/data/numbers/ordinals.tsv +752 -0
- itn/russian/data/numbers/quantity.tsv +16 -0
- itn/russian/data/time/increment_hour_cardinal.tsv +24 -0
- itn/russian/data/time/increment_hour_ordinal.tsv +24 -0
- itn/russian/data/time/minutes_to_hour.tsv +25 -0
- itn/russian/data/time/time_convert.tsv +12 -0
- itn/russian/data/whitelist.tsv +37 -0
- itn/russian/inverse_normalizer.py +71 -0
- itn/russian/rules/__init__.py +0 -0
- itn/russian/rules/cardinal.py +57 -0
- itn/russian/rules/date.py +78 -0
- itn/russian/rules/decimal.py +55 -0
- itn/russian/rules/electronic.py +67 -0
- itn/russian/rules/measure.py +45 -0
- itn/russian/rules/money.py +53 -0
- itn/russian/rules/ordinal.py +40 -0
- itn/russian/rules/punctuation.py +40 -0
- itn/russian/rules/telephone.py +45 -0
- itn/russian/rules/time.py +83 -0
- itn/russian/rules/utils.py +151 -0
- itn/russian/rules/whitelist.py +29 -0
- itn/russian/rules/word.py +30 -0
- itn/russian/test/__init__.py +0 -0
- itn/russian/test/data/test_cases_cardinal.txt +30 -0
- itn/russian/test/data/test_cases_date.txt +15 -0
- itn/russian/test/data/test_cases_decimal.txt +16 -0
- itn/russian/test/data/test_cases_electronic.txt +4 -0
- itn/russian/test/data/test_cases_measure.txt +16 -0
- itn/russian/test/data/test_cases_money.txt +9 -0
- itn/russian/test/data/test_cases_ordinal.txt +15 -0
- itn/russian/test/data/test_cases_ordinal_hard.txt +57 -0
- itn/russian/test/data/test_cases_telephone.txt +1 -0
- itn/russian/test/data/test_cases_time.txt +9 -0
- itn/russian/test/data/test_cases_whitelist.txt +4 -0
- itn/russian/test/data/test_cases_word.txt +50 -0
- itn/russian/test/normalizer_test.py +193 -0
- itn/spanish/__init__.py +0 -0
- itn/spanish/data/dates/months.tsv +12 -0
- itn/spanish/data/dates/months_cased.tsv +12 -0
- itn/spanish/data/dates/year_suffix.tsv +11 -0
- itn/spanish/data/dates/year_suffix_cased.tsv +11 -0
- itn/spanish/data/electronic/domain.tsv +27 -0
- itn/spanish/data/electronic/server_name.tsv +17 -0
- itn/spanish/data/electronic/symbols.tsv +22 -0
- itn/spanish/data/fractions/numbers_read_as_ordinals.tsv +28 -0
- itn/spanish/data/fractions/ordinal_exceptions.tsv +2 -0
- itn/spanish/data/measures/math_symbols.tsv +4 -0
- itn/spanish/data/measures/measurements_plural.tsv +34 -0
- itn/spanish/data/measures/measurements_singular.tsv +35 -0
- itn/spanish/data/money/currency_major_plural.tsv +43 -0
- itn/spanish/data/money/currency_major_plural_capitalized.tsv +75 -0
- itn/spanish/data/money/currency_major_plural_ext.tsv +196 -0
- itn/spanish/data/money/currency_major_singular.tsv +43 -0
- itn/spanish/data/money/currency_major_singular_capitalized.tsv +76 -0
- itn/spanish/data/money/currency_major_singular_ext.tsv +196 -0
- itn/spanish/data/money/currency_minor_plural.tsv +5 -0
- itn/spanish/data/money/currency_minor_plural_ext.tsv +62 -0
- itn/spanish/data/money/currency_minor_singular.tsv +5 -0
- itn/spanish/data/money/currency_minor_singular_ext.tsv +64 -0
- itn/spanish/data/months.tsv +12 -0
- itn/spanish/data/numbers/digit.tsv +11 -0
- itn/spanish/data/numbers/hundreds.tsv +18 -0
- itn/spanish/data/numbers/teen.tsv +10 -0
- itn/spanish/data/numbers/ties.tsv +8 -0
- itn/spanish/data/numbers/twenties.tsv +11 -0
- itn/spanish/data/numbers/zero.tsv +1 -0
- itn/spanish/data/ordinals/digit.tsv +22 -0
- itn/spanish/data/ordinals/digit_capitalized.tsv +22 -0
- itn/spanish/data/ordinals/hundreds.tsv +18 -0
- itn/spanish/data/ordinals/hundreds_capitalized.tsv +18 -0
- itn/spanish/data/ordinals/teen.tsv +60 -0
- itn/spanish/data/ordinals/teen_capitalized.tsv +60 -0
- itn/spanish/data/ordinals/ties.tsv +15 -0
- itn/spanish/data/ordinals/ties_capitalized.tsv +15 -0
- itn/spanish/data/ordinals/twenties.tsv +50 -0
- itn/spanish/data/ordinals/twenties_capitalized.tsv +50 -0
- itn/spanish/data/roman/digit.tsv +9 -0
- itn/spanish/data/roman/hundreds.tsv +9 -0
- itn/spanish/data/roman/thousands.tsv +3 -0
- itn/spanish/data/roman/ties.tsv +9 -0
- itn/spanish/data/time/minutes_to.tsv +6 -0
- itn/spanish/data/time/time_suffix.tsv +15 -0
- itn/spanish/data/time/time_suffix_cased.tsv +15 -0
- itn/spanish/data/time/time_to.tsv +25 -0
- itn/spanish/data/time/time_zone.tsv +42 -0
- itn/spanish/data/time/time_zone_cased.tsv +42 -0
- itn/spanish/data/time/time_zone_ext.tsv +105 -0
- itn/spanish/data/whitelist.tsv +47 -0
- itn/spanish/inverse_normalizer.py +77 -0
- itn/spanish/rules/__init__.py +0 -0
- itn/spanish/rules/cardinal.py +98 -0
- itn/spanish/rules/date.py +83 -0
- itn/spanish/rules/decimal.py +64 -0
- itn/spanish/rules/electronic.py +94 -0
- itn/spanish/rules/fraction.py +77 -0
- itn/spanish/rules/measure.py +84 -0
- itn/spanish/rules/money.py +95 -0
- itn/spanish/rules/ordinal.py +53 -0
- itn/spanish/rules/punctuation.py +30 -0
- itn/spanish/rules/telephone.py +70 -0
- itn/spanish/rules/time.py +112 -0
- itn/spanish/rules/utils.py +102 -0
- itn/spanish/rules/whitelist.py +47 -0
- itn/spanish/rules/word.py +32 -0
- itn/spanish/test/__init__.py +0 -0
- itn/spanish/test/data/test_cases_cardinal.txt +51 -0
- itn/spanish/test/data/test_cases_cardinal_cased.txt +30 -0
- itn/spanish/test/data/test_cases_date.txt +8 -0
- itn/spanish/test/data/test_cases_date_cased.txt +8 -0
- itn/spanish/test/data/test_cases_decimal.txt +29 -0
- itn/spanish/test/data/test_cases_decimal_cased.txt +6 -0
- itn/spanish/test/data/test_cases_electronic.txt +16 -0
- itn/spanish/test/data/test_cases_electronic_cased.txt +5 -0
- itn/spanish/test/data/test_cases_fraction.txt +12 -0
- itn/spanish/test/data/test_cases_measure.txt +20 -0
- itn/spanish/test/data/test_cases_measure_cased.txt +11 -0
- itn/spanish/test/data/test_cases_money.txt +24 -0
- itn/spanish/test/data/test_cases_money_cased.txt +6 -0
- itn/spanish/test/data/test_cases_ordinal.txt +30 -0
- itn/spanish/test/data/test_cases_ordinal_cased.txt +11 -0
- itn/spanish/test/data/test_cases_telephone.txt +9 -0
- itn/spanish/test/data/test_cases_telephone_cased.txt +6 -0
- itn/spanish/test/data/test_cases_time.txt +25 -0
- itn/spanish/test/data/test_cases_time_cased.txt +9 -0
- itn/spanish/test/data/test_cases_whitelist.txt +5 -0
- itn/spanish/test/data/test_cases_word.txt +49 -0
- itn/spanish/test/data/test_cases_word_cased.txt +11 -0
- itn/spanish/test/normalizer_test.py +143 -0
- itn/spanish_english/__init__.py +0 -0
- itn/spanish_english/data/en_whitelist.tsv +416 -0
- itn/spanish_english/data/es_whitelist.tsv +16 -0
- itn/spanish_english/data/measurements.tsv +3 -0
- itn/spanish_english/inverse_normalizer.py +99 -0
- itn/spanish_english/rules/__init__.py +0 -0
- itn/spanish_english/rules/english.py +230 -0
- itn/spanish_english/rules/prefixed.py +34 -0
- itn/spanish_english/rules/time.py +39 -0
- itn/spanish_english/rules/whitelist.py +38 -0
- itn/spanish_english/test/__init__.py +0 -0
- itn/spanish_english/test/data/test_cases_cardinal.txt +80 -0
- itn/spanish_english/test/data/test_cases_cardinal_cased.txt +60 -0
- itn/spanish_english/test/data/test_cases_date.txt +44 -0
- itn/spanish_english/test/data/test_cases_date_cased.txt +20 -0
- itn/spanish_english/test/data/test_cases_decimal.txt +92 -0
- itn/spanish_english/test/data/test_cases_decimal_cased.txt +19 -0
- itn/spanish_english/test/data/test_cases_electronic.txt +41 -0
- itn/spanish_english/test/data/test_cases_electronic_cased.txt +14 -0
- itn/spanish_english/test/data/test_cases_fraction.txt +12 -0
- itn/spanish_english/test/data/test_cases_measure.txt +132 -0
- itn/spanish_english/test/data/test_cases_measure_cased.txt +22 -0
- itn/spanish_english/test/data/test_cases_money.txt +76 -0
- itn/spanish_english/test/data/test_cases_money_cased.txt +16 -0
- itn/spanish_english/test/data/test_cases_ordinal.txt +64 -0
- itn/spanish_english/test/data/test_cases_ordinal_cased.txt +21 -0
- itn/spanish_english/test/data/test_cases_telephone.txt +32 -0
- itn/spanish_english/test/data/test_cases_telephone_cased.txt +19 -0
- itn/spanish_english/test/data/test_cases_time.txt +54 -0
- itn/spanish_english/test/data/test_cases_time_cased.txt +19 -0
- itn/spanish_english/test/data/test_cases_whitelist.txt +18 -0
- itn/spanish_english/test/data/test_cases_word.txt +104 -0
- itn/spanish_english/test/data/test_cases_word_cased.txt +17 -0
- itn/spanish_english/test/normalizer_test.py +207 -0
- itn/swedish/__init__.py +0 -0
- itn/swedish/data/dates/era_words.tsv +4 -0
- itn/swedish/data/dates/month_abbr.tsv +17 -0
- itn/swedish/data/dates/months.tsv +12 -0
- itn/swedish/data/electronic/domain.tsv +10 -0
- itn/swedish/data/electronic/server_name.tsv +29 -0
- itn/swedish/data/electronic/symbols.tsv +20 -0
- itn/swedish/data/numbers/digit.tsv +9 -0
- itn/swedish/data/numbers/millions.tsv +88 -0
- itn/swedish/data/numbers/teen.tsv +10 -0
- itn/swedish/data/numbers/ties.tsv +8 -0
- itn/swedish/data/numbers/zero.tsv +1 -0
- itn/swedish/data/ordinals/digit.tsv +9 -0
- itn/swedish/data/ordinals/teen.tsv +10 -0
- itn/swedish/data/ordinals/ties.tsv +8 -0
- itn/swedish/data/time/suffix.tsv +2 -0
- itn/swedish/data/time/time_zone.tsv +7 -0
- itn/swedish/data/whitelist.tsv +4 -0
- itn/swedish/inverse_normalizer.py +67 -0
- itn/swedish/rules/__init__.py +0 -0
- itn/swedish/rules/cardinal.py +88 -0
- itn/swedish/rules/date.py +91 -0
- itn/swedish/rules/decimal.py +68 -0
- itn/swedish/rules/electronic.py +69 -0
- itn/swedish/rules/fraction.py +90 -0
- itn/swedish/rules/ordinal.py +85 -0
- itn/swedish/rules/punctuation.py +29 -0
- itn/swedish/rules/telephone.py +59 -0
- itn/swedish/rules/time.py +99 -0
- itn/swedish/rules/utils.py +55 -0
- itn/swedish/rules/whitelist.py +29 -0
- itn/swedish/rules/word.py +29 -0
- itn/swedish/test/__init__.py +0 -0
- itn/swedish/test/data/test_cases_cardinal.txt +24 -0
- itn/swedish/test/data/test_cases_date.txt +9 -0
- itn/swedish/test/data/test_cases_decimal.txt +15 -0
- itn/swedish/test/data/test_cases_electronic.txt +12 -0
- itn/swedish/test/data/test_cases_fraction.txt +3 -0
- itn/swedish/test/data/test_cases_ordinal.txt +7 -0
- itn/swedish/test/data/test_cases_telephone.txt +2 -0
- itn/swedish/test/data/test_cases_time.txt +14 -0
- itn/swedish/test/data/test_cases_whitelist.txt +1 -0
- itn/swedish/test/data/test_cases_word.txt +48 -0
- itn/swedish/test/normalizer_test.py +122 -0
- itn/test/utils_test.py +17 -0
- itn/utils.py +40 -0
- itn/vietnamese/__init__.py +0 -0
- itn/vietnamese/data/currency.tsv +11 -0
- itn/vietnamese/data/electronic/domain.tsv +11 -0
- itn/vietnamese/data/electronic/server_name.tsv +16 -0
- itn/vietnamese/data/electronic/symbols.tsv +21 -0
- itn/vietnamese/data/electronic/url_symbols.tsv +8 -0
- itn/vietnamese/data/magnitudes.tsv +5 -0
- itn/vietnamese/data/math/symbols.tsv +10 -0
- itn/vietnamese/data/measurements.tsv +185 -0
- itn/vietnamese/data/months.tsv +14 -0
- itn/vietnamese/data/numbers/digit.tsv +10 -0
- itn/vietnamese/data/numbers/digit_special.tsv +3 -0
- itn/vietnamese/data/numbers/hundred.tsv +1 -0
- itn/vietnamese/data/numbers/teen.tsv +11 -0
- itn/vietnamese/data/numbers/thousands.tsv +6 -0
- itn/vietnamese/data/numbers/ties.tsv +9 -0
- itn/vietnamese/data/numbers/zero.tsv +1 -0
- itn/vietnamese/data/ordinals/digit.tsv +12 -0
- itn/vietnamese/data/time/hours.tsv +29 -0
- itn/vietnamese/data/time/hours_to.tsv +25 -0
- itn/vietnamese/data/time/hours_to_night.tsv +12 -0
- itn/vietnamese/data/time/minutes.tsv +118 -0
- itn/vietnamese/data/time/minutes_to.tsv +59 -0
- itn/vietnamese/data/time/time_suffix.tsv +8 -0
- itn/vietnamese/data/time/time_zone.tsv +7 -0
- itn/vietnamese/data/time/zero_prefix.tsv +3 -0
- itn/vietnamese/data/whitelist.tsv +0 -0
- itn/vietnamese/inverse_normalizer.py +73 -0
- itn/vietnamese/rules/__init__.py +0 -0
- itn/vietnamese/rules/cardinal.py +99 -0
- itn/vietnamese/rules/date.py +76 -0
- itn/vietnamese/rules/decimal.py +72 -0
- itn/vietnamese/rules/electronic.py +60 -0
- itn/vietnamese/rules/fraction.py +41 -0
- itn/vietnamese/rules/measure.py +60 -0
- itn/vietnamese/rules/money.py +54 -0
- itn/vietnamese/rules/ordinal.py +32 -0
- itn/vietnamese/rules/punctuation.py +29 -0
- itn/vietnamese/rules/telephone.py +57 -0
- itn/vietnamese/rules/time.py +83 -0
- itn/vietnamese/rules/whitelist.py +30 -0
- itn/vietnamese/rules/word.py +28 -0
- itn/vietnamese/test/__init__.py +0 -0
- itn/vietnamese/test/data/test_cases_cardinal.txt +142 -0
- itn/vietnamese/test/data/test_cases_date.txt +69 -0
- itn/vietnamese/test/data/test_cases_decimal.txt +74 -0
- itn/vietnamese/test/data/test_cases_electronic.txt +18 -0
- itn/vietnamese/test/data/test_cases_fraction.txt +14 -0
- itn/vietnamese/test/data/test_cases_measure.txt +35 -0
- itn/vietnamese/test/data/test_cases_money.txt +15 -0
- itn/vietnamese/test/data/test_cases_ordinal.txt +12 -0
- itn/vietnamese/test/data/test_cases_telephone.txt +25 -0
- itn/vietnamese/test/data/test_cases_time.txt +31 -0
- itn/vietnamese/test/data/test_cases_whitelist.txt +1 -0
- itn/vietnamese/test/data/test_cases_word.txt +17 -0
- itn/vietnamese/test/normalizer_test.py +144 -0
- tn/README.md +20 -8
- tn/__init__.py +3 -0
- tn/_version.py +9 -19
- tn/alignment.py +254 -0
- tn/arabic/__init__.py +0 -0
- tn/arabic/data/measure/measurements.tsv +14 -0
- tn/arabic/data/money/currency_major.tsv +24 -0
- tn/arabic/data/money/currency_minor_plural.tsv +4 -0
- tn/arabic/data/money/currency_minor_singular.tsv +3 -0
- tn/arabic/data/money/local_currency.tsv +20 -0
- tn/arabic/data/number/digit.tsv +9 -0
- tn/arabic/data/number/digit_100.tsv +7 -0
- tn/arabic/data/number/digit_1000.tsv +7 -0
- tn/arabic/data/number/flops.tsv +72 -0
- tn/arabic/data/number/fraction.tsv +18 -0
- tn/arabic/data/number/fraction_dual.tsv +9 -0
- tn/arabic/data/number/fraction_plural.tsv +8 -0
- tn/arabic/data/number/fraction_singular.tsv +9 -0
- tn/arabic/data/number/quantities.tsv +6 -0
- tn/arabic/data/number/teens.tsv +9 -0
- tn/arabic/data/number/tens.tsv +10 -0
- tn/arabic/data/punctuation/symbol.tsv +23 -0
- tn/arabic/normalizer.py +61 -0
- tn/arabic/rules/__init__.py +0 -0
- tn/arabic/rules/cardinal.py +86 -0
- tn/arabic/rules/decimal.py +69 -0
- tn/arabic/rules/fraction.py +79 -0
- tn/arabic/rules/measure.py +80 -0
- tn/arabic/rules/money.py +131 -0
- tn/arabic/rules/punctuation.py +44 -0
- tn/arabic/rules/word.py +32 -0
- tn/arabic/test/__init__.py +0 -0
- tn/arabic/test/data/test_cases_cardinal.txt +27 -0
- tn/arabic/test/data/test_cases_decimal.txt +30 -0
- tn/arabic/test/data/test_cases_fraction.txt +20 -0
- tn/arabic/test/data/test_cases_measure.txt +14 -0
- tn/arabic/test/data/test_cases_money.txt +27 -0
- tn/arabic/test/data/test_cases_whitelist.txt +6 -0
- tn/arabic/test/normalizer_test.py +162 -0
- tn/armenian/__init__.py +0 -0
- tn/armenian/data/currency.tsv +52 -0
- tn/armenian/data/measurement_dates.tsv +8 -0
- tn/armenian/data/measurements.tsv +44 -0
- tn/armenian/data/numbers/digit.tsv +9 -0
- tn/armenian/data/numbers/quantities.tsv +3 -0
- tn/armenian/data/numbers/ties.tsv +8 -0
- tn/armenian/data/time/hours.tsv +26 -0
- tn/armenian/data/time/minutes.tsv +61 -0
- tn/armenian/data/whitelist.tsv +14 -0
- tn/armenian/normalizer.py +65 -0
- tn/armenian/rules/__init__.py +0 -0
- tn/armenian/rules/cardinal.py +75 -0
- tn/armenian/rules/decimal.py +63 -0
- tn/armenian/rules/fraction.py +36 -0
- tn/armenian/rules/measure.py +49 -0
- tn/armenian/rules/money.py +38 -0
- tn/armenian/rules/ordinal.py +59 -0
- tn/armenian/rules/punctuation.py +31 -0
- tn/armenian/rules/time.py +34 -0
- tn/armenian/rules/utils.py +60 -0
- tn/armenian/rules/whitelist.py +31 -0
- tn/armenian/rules/word.py +28 -0
- tn/armenian/test/__init__.py +0 -0
- tn/armenian/test/data/test_cases_cardinal.txt +12 -0
- tn/armenian/test/data/test_cases_decimal.txt +9 -0
- tn/armenian/test/data/test_cases_fraction.txt +10 -0
- tn/armenian/test/data/test_cases_measure.txt +9 -0
- tn/armenian/test/data/test_cases_money.txt +10 -0
- tn/armenian/test/data/test_cases_ordinal.txt +11 -0
- tn/armenian/test/data/test_cases_time.txt +9 -0
- tn/armenian/test/data/test_cases_whitelist.txt +14 -0
- tn/armenian/test/data/test_cases_word.txt +50 -0
- tn/armenian/test/normalizer_test.py +156 -0
- tn/cache.py +822 -0
- tn/chinese/data/cardinal/phone.tsv +28 -0
- tn/chinese/data/math/operator_unambiguous.tsv +6 -0
- tn/chinese/normalizer.py +45 -37
- tn/chinese/rules/cardinal.py +21 -19
- tn/chinese/rules/date.py +26 -18
- tn/chinese/rules/fraction.py +4 -12
- tn/chinese/rules/math.py +10 -5
- tn/chinese/rules/measure.py +18 -10
- tn/chinese/rules/money.py +5 -11
- tn/chinese/rules/postprocessor.py +23 -8
- tn/chinese/rules/range.py +29 -0
- tn/chinese/rules/sport.py +8 -6
- tn/chinese/rules/time.py +22 -19
- tn/chinese/rules/whitelist.py +7 -6
- tn/chinese/test/alignment_test.py +154 -0
- tn/chinese/test/cache_safety_test.py +505 -0
- tn/chinese/test/cache_test.py +192 -0
- tn/chinese/test/data/normalizer.txt +0 -2
- tn/chinese/test/data/normalizer_tag_oov.txt +2 -0
- tn/chinese/test/normalizer_test.py +57 -4
- tn/chinese/test/postprocessor_test.py +16 -0
- tn/chinese/test/processor_test.py +496 -0
- tn/chinese/test/token_parser_test.py +35 -1
- tn/cli.py +278 -0
- tn/english/data/measure/unit_alternatives.tsv +0 -1
- tn/english/normalizer.py +46 -42
- tn/english/rules/cardinal.py +37 -61
- tn/english/rules/date.py +84 -166
- tn/english/rules/decimal.py +32 -79
- tn/english/rules/electronic.py +56 -66
- tn/english/rules/fraction.py +33 -23
- tn/english/rules/measure.py +107 -148
- tn/english/rules/money.py +46 -42
- tn/english/rules/ordinal.py +14 -30
- tn/english/rules/punctuation.py +15 -21
- tn/english/rules/range.py +26 -40
- tn/english/rules/roman.py +27 -45
- tn/english/rules/serial.py +88 -0
- tn/english/rules/telephone.py +44 -80
- tn/english/rules/time.py +54 -102
- tn/english/rules/whitelist.py +9 -19
- tn/english/rules/word.py +9 -0
- tn/english/test/data/date.txt +1 -1
- tn/english/test/data/normalizer.txt +4 -1
- tn/english/test/electronic_test.py +27 -0
- tn/english/test/nbest_test.py +87 -0
- tn/english/test/normalizer_test.py +181 -3
- tn/french/__init__.py +0 -0
- tn/french/data/dates/eras.tsv +8 -0
- tn/french/data/dates/months.tsv +12 -0
- tn/french/data/fractions/irregular_denominators.tsv +7 -0
- tn/french/data/numbers/digit.tsv +9 -0
- tn/french/data/numbers/eighty.tsv +1 -0
- tn/french/data/numbers/ninety.tsv +1 -0
- tn/french/data/numbers/quantities.tsv +10 -0
- tn/french/data/numbers/seventy.tsv +1 -0
- tn/french/data/numbers/teens.tsv +9 -0
- tn/french/data/numbers/ten.tsv +1 -0
- tn/french/data/numbers/tens_simple.tsv +5 -0
- tn/french/data/numbers/zero.tsv +1 -0
- tn/french/data/ordinals/irregular_numbers.tsv +30 -0
- tn/french/data/ordinals/suffixes.tsv +6 -0
- tn/french/data/punctuation/symbol.tsv +23 -0
- tn/french/data/whitelist.tsv +13 -0
- tn/french/normalizer.py +66 -0
- tn/french/rules/__init__.py +0 -0
- tn/french/rules/cardinal.py +110 -0
- tn/french/rules/date.py +68 -0
- tn/french/rules/decimal.py +62 -0
- tn/french/rules/fraction.py +60 -0
- tn/french/rules/ordinal.py +56 -0
- tn/french/rules/punctuation.py +43 -0
- tn/french/rules/whitelist.py +29 -0
- tn/french/rules/word.py +29 -0
- tn/french/test/__init__.py +0 -0
- tn/french/test/data/test_cases_cardinal.txt +14 -0
- tn/french/test/data/test_cases_date.txt +13 -0
- tn/french/test/data/test_cases_decimal.txt +7 -0
- tn/french/test/data/test_cases_fraction.txt +12 -0
- tn/french/test/data/test_cases_ordinal.txt +15 -0
- tn/french/test/data/test_cases_whitelist.txt +7 -0
- tn/french/test/data/test_cases_word.txt +49 -0
- tn/french/test/normalizer_test.py +157 -0
- tn/german/__init__.py +0 -0
- tn/german/data/__init__.py +13 -0
- tn/german/data/electronic/__init__.py +13 -0
- tn/german/data/electronic/domain.tsv +844 -0
- tn/german/data/electronic/server_name.tsv +17 -0
- tn/german/data/electronic/symbols.tsv +20 -0
- tn/german/data/fractions.tsv +30 -0
- tn/german/data/measure/__init__.py +13 -0
- tn/german/data/measure/measurements.tsv +82 -0
- tn/german/data/measure/suppletive.tsv +10 -0
- tn/german/data/money/__init__.py +13 -0
- tn/german/data/money/currency.tsv +25 -0
- tn/german/data/money/currency_minor_plural.tsv +3 -0
- tn/german/data/money/currency_minor_singular.tsv +3 -0
- tn/german/data/months/__init__.py +13 -0
- tn/german/data/months/abbr_to_name.tsv +13 -0
- tn/german/data/months/numbers.tsv +24 -0
- tn/german/data/numbers/__init__.py +13 -0
- tn/german/data/numbers/digit.tsv +8 -0
- tn/german/data/numbers/ones.tsv +3 -0
- tn/german/data/numbers/quantities.tsv +12 -0
- tn/german/data/numbers/teen.tsv +10 -0
- tn/german/data/numbers/ties.tsv +8 -0
- tn/german/data/numbers/zero.tsv +1 -0
- tn/german/data/ordinals/__init__.py +13 -0
- tn/german/data/ordinals/digit.tsv +9 -0
- tn/german/data/ordinals/thousands.tsv +4 -0
- tn/german/data/ordinals/ties.tsv +8 -0
- tn/german/data/time/__init__.py +13 -0
- tn/german/data/time/hour_to.tsv +12 -0
- tn/german/data/time/hour_to_night.tsv +13 -0
- tn/german/data/time/minute_to.tsv +59 -0
- tn/german/data/time/time_zone.tsv +7 -0
- tn/german/data/whitelist.tsv +21 -0
- tn/german/normalizer.py +77 -0
- tn/german/rules/__init__.py +0 -0
- tn/german/rules/cardinal.py +120 -0
- tn/german/rules/date.py +98 -0
- tn/german/rules/decimal.py +71 -0
- tn/german/rules/electronic.py +67 -0
- tn/german/rules/fraction.py +65 -0
- tn/german/rules/measure.py +117 -0
- tn/german/rules/money.py +144 -0
- tn/german/rules/ordinal.py +46 -0
- tn/german/rules/punctuation.py +39 -0
- tn/german/rules/telephone.py +48 -0
- tn/german/rules/time.py +123 -0
- tn/german/rules/whitelist.py +34 -0
- tn/german/rules/word.py +34 -0
- tn/german/test/__init__.py +0 -0
- tn/german/test/calendar_test.py +168 -0
- tn/german/test/data/test_cases_cardinal.txt +50 -0
- tn/german/test/data/test_cases_date.txt +15 -0
- tn/german/test/data/test_cases_decimal.txt +10 -0
- tn/german/test/data/test_cases_electronic.txt +19 -0
- tn/german/test/data/test_cases_fraction.txt +34 -0
- tn/german/test/data/test_cases_measure.txt +29 -0
- tn/german/test/data/test_cases_money.txt +21 -0
- tn/german/test/data/test_cases_normalize_with_audio.txt +131 -0
- tn/german/test/data/test_cases_ordinal.txt +19 -0
- tn/german/test/data/test_cases_telephone.txt +4 -0
- tn/german/test/data/test_cases_time.txt +28 -0
- tn/german/test/data/test_cases_whitelist.txt +6 -0
- tn/german/test/data/test_cases_word.txt +49 -0
- tn/german/test/financial_test.py +167 -0
- tn/german/test/normalizer_test.py +136 -0
- tn/german/test/numeric_test.py +185 -0
- tn/german/test/text_test.py +235 -0
- tn/hebrew/__init__.py +0 -0
- tn/hebrew/data/measurements.tsv +29 -0
- tn/hebrew/data/prefix.tsv +17 -0
- tn/hebrew/data/spaced_measurements.tsv +16 -0
- tn/hebrew/data/whitelist.tsv +9 -0
- tn/hebrew/normalizer.py +58 -0
- tn/hebrew/rules/__init__.py +0 -0
- tn/hebrew/rules/cardinal.py +43 -0
- tn/hebrew/rules/date.py +84 -0
- tn/hebrew/rules/decimal.py +71 -0
- tn/hebrew/rules/measure.py +56 -0
- tn/hebrew/rules/numbers.py +135 -0
- tn/hebrew/rules/punctuation.py +29 -0
- tn/hebrew/rules/time.py +94 -0
- tn/hebrew/rules/utils.py +59 -0
- tn/hebrew/rules/whitelist.py +26 -0
- tn/hebrew/rules/word.py +27 -0
- tn/hebrew/test/__init__.py +0 -0
- tn/hebrew/test/normalizer_test.py +245 -0
- tn/hindi/__init__.py +0 -0
- tn/hindi/data/address/cities.tsv +36 -0
- tn/hindi/data/address/context.tsv +47 -0
- tn/hindi/data/address/letters.tsv +26 -0
- tn/hindi/data/address/special_characters.tsv +2 -0
- tn/hindi/data/address/states.tsv +36 -0
- tn/hindi/data/date/days.tsv +62 -0
- tn/hindi/data/date/months.tsv +24 -0
- tn/hindi/data/date/prefixes.tsv +4 -0
- tn/hindi/data/date/suffixes.tsv +10 -0
- tn/hindi/data/date/year_suffix.tsv +2 -0
- tn/hindi/data/electronic/domain.tsv +24 -0
- tn/hindi/data/electronic/elements.tsv +132 -0
- tn/hindi/data/electronic/file_extensions.tsv +34 -0
- tn/hindi/data/electronic/protocols.tsv +5 -0
- tn/hindi/data/electronic/subscript_digit.tsv +10 -0
- tn/hindi/data/electronic/symbol_classes.tsv +16 -0
- tn/hindi/data/electronic/symbols.tsv +36 -0
- tn/hindi/data/fraction/common_fractions.tsv +10 -0
- tn/hindi/data/measure/quarterly_units_list.tsv +5 -0
- tn/hindi/data/measure/quarterly_units_map.tsv +10 -0
- tn/hindi/data/measure/unit.tsv +146 -0
- tn/hindi/data/measure/unit_year_formal.tsv +1 -0
- tn/hindi/data/money/currency.tsv +9 -0
- tn/hindi/data/money/currency_singular.tsv +9 -0
- tn/hindi/data/money/major_minor_currencies.tsv +10 -0
- tn/hindi/data/numbers/digit.tsv +18 -0
- tn/hindi/data/numbers/hundred.tsv +1 -0
- tn/hindi/data/numbers/teens_and_ties.tsv +90 -0
- tn/hindi/data/numbers/teens_and_ties_en.tsv +90 -0
- tn/hindi/data/numbers/thousands.tsv +8 -0
- tn/hindi/data/numbers/zero.tsv +2 -0
- tn/hindi/data/ordinal/en_to_hi_digit.tsv +10 -0
- tn/hindi/data/ordinal/exceptions.tsv +50 -0
- tn/hindi/data/ordinal/suffixes.tsv +3 -0
- tn/hindi/data/ordinal/suffixes_map.tsv +1 -0
- tn/hindi/data/roman/roman_ordinal_exceptions.tsv +10 -0
- tn/hindi/data/roman/roman_to_spoken.tsv +100 -0
- tn/hindi/data/serial/chars.tsv +68 -0
- tn/hindi/data/serial/power_special.tsv +4 -0
- tn/hindi/data/serial/special_symbols.tsv +4 -0
- tn/hindi/data/telephone/credit_context.tsv +3 -0
- tn/hindi/data/telephone/landline_context.tsv +5 -0
- tn/hindi/data/telephone/mobile_context.tsv +4 -0
- tn/hindi/data/telephone/number.tsv +10 -0
- tn/hindi/data/telephone/pincode_context.tsv +4 -0
- tn/hindi/data/time/hours.tsv +50 -0
- tn/hindi/data/time/minutes.tsv +120 -0
- tn/hindi/data/time/seconds.tsv +120 -0
- tn/hindi/data/whitelist/abbreviations.tsv +7 -0
- tn/hindi/data/whitelist/paune_mappings.tsv +100 -0
- tn/hindi/normalizer.py +95 -0
- tn/hindi/rules/__init__.py +0 -0
- tn/hindi/rules/address.py +97 -0
- tn/hindi/rules/cardinal.py +117 -0
- tn/hindi/rules/date.py +135 -0
- tn/hindi/rules/decimal.py +42 -0
- tn/hindi/rules/electronic.py +145 -0
- tn/hindi/rules/fraction.py +62 -0
- tn/hindi/rules/measure.py +110 -0
- tn/hindi/rules/money.py +104 -0
- tn/hindi/rules/ordinal.py +37 -0
- tn/hindi/rules/punctuation.py +50 -0
- tn/hindi/rules/roman.py +70 -0
- tn/hindi/rules/serial.py +82 -0
- tn/hindi/rules/telephone.py +98 -0
- tn/hindi/rules/time.py +62 -0
- tn/hindi/rules/utils.py +54 -0
- tn/hindi/rules/whitelist.py +37 -0
- tn/hindi/rules/word.py +31 -0
- tn/hindi/test/__init__.py +0 -0
- tn/hindi/test/data/test_cases_address.txt +52 -0
- tn/hindi/test/data/test_cases_cardinal.txt +161 -0
- tn/hindi/test/data/test_cases_date.txt +39 -0
- tn/hindi/test/data/test_cases_decimal.txt +24 -0
- tn/hindi/test/data/test_cases_electronic.txt +65 -0
- tn/hindi/test/data/test_cases_fraction.txt +27 -0
- tn/hindi/test/data/test_cases_measure.txt +69 -0
- tn/hindi/test/data/test_cases_money.txt +144 -0
- tn/hindi/test/data/test_cases_ordinal.txt +72 -0
- tn/hindi/test/data/test_cases_roman.txt +23 -0
- tn/hindi/test/data/test_cases_serial.txt +31 -0
- tn/hindi/test/data/test_cases_telephone.txt +25 -0
- tn/hindi/test/data/test_cases_time.txt +18 -0
- tn/hindi/test/data/test_cases_whitelist.txt +14 -0
- tn/hindi/test/data/test_cases_word.txt +20 -0
- tn/hindi/test/normalizer_test.py +137 -0
- tn/hungarian/__init__.py +0 -0
- tn/hungarian/data/dates/days.tsv +31 -0
- tn/hungarian/data/dates/month_abbr.tsv +12 -0
- tn/hungarian/data/dates/months.tsv +12 -0
- tn/hungarian/data/dates/months_roman.tsv +12 -0
- tn/hungarian/data/electronic/domain.tsv +15 -0
- tn/hungarian/data/electronic/server_name.tsv +15 -0
- tn/hungarian/data/electronic/symbols.tsv +23 -0
- tn/hungarian/data/fractions/fraction_symbols.tsv +19 -0
- tn/hungarian/data/inflection/endings.tsv +51 -0
- tn/hungarian/data/inflection/plural_endings.tsv +39 -0
- tn/hungarian/data/inflection/word_endings.tsv +1 -0
- tn/hungarian/data/measures/measurements.tsv +52 -0
- tn/hungarian/data/money/alphabet.tsv +44 -0
- tn/hungarian/data/money/currency.tsv +11 -0
- tn/hungarian/data/money/currency_minor.tsv +11 -0
- tn/hungarian/data/number/digit.tsv +9 -0
- tn/hungarian/data/number/digit_inline.tsv +9 -0
- tn/hungarian/data/number/quantities.tsv +21 -0
- tn/hungarian/data/number/tens.tsv +9 -0
- tn/hungarian/data/number/tens_inline.tsv +9 -0
- tn/hungarian/data/number/zero.tsv +1 -0
- tn/hungarian/data/ordinals/endings.tsv +23 -0
- tn/hungarian/data/ordinals/exceptional.tsv +4 -0
- tn/hungarian/data/telephone/area_codes.tsv +71 -0
- tn/hungarian/data/telephone/country_codes.tsv +213 -0
- tn/hungarian/data/telephone/telephone_abbr.tsv +3 -0
- tn/hungarian/data/time/time_zone.tsv +9 -0
- tn/hungarian/data/whitelist.tsv +24 -0
- tn/hungarian/data/whitelist_inflect.tsv +39 -0
- tn/hungarian/data/whitelist_inflect_sg.tsv +14 -0
- tn/hungarian/normalizer.py +80 -0
- tn/hungarian/rules/__init__.py +0 -0
- tn/hungarian/rules/cardinal.py +98 -0
- tn/hungarian/rules/date.py +104 -0
- tn/hungarian/rules/decimal.py +85 -0
- tn/hungarian/rules/electronic.py +69 -0
- tn/hungarian/rules/fraction.py +55 -0
- tn/hungarian/rules/measure.py +58 -0
- tn/hungarian/rules/money.py +81 -0
- tn/hungarian/rules/ordinal.py +41 -0
- tn/hungarian/rules/punctuation.py +46 -0
- tn/hungarian/rules/telephone.py +71 -0
- tn/hungarian/rules/time.py +106 -0
- tn/hungarian/rules/utils.py +156 -0
- tn/hungarian/rules/whitelist.py +32 -0
- tn/hungarian/rules/word.py +38 -0
- tn/hungarian/test/__init__.py +0 -0
- tn/hungarian/test/data/test_cases_cardinal.txt +69 -0
- tn/hungarian/test/data/test_cases_date.txt +17 -0
- tn/hungarian/test/data/test_cases_decimal.txt +6 -0
- tn/hungarian/test/data/test_cases_electronic.txt +8 -0
- tn/hungarian/test/data/test_cases_fraction.txt +4 -0
- tn/hungarian/test/data/test_cases_measure.txt +6 -0
- tn/hungarian/test/data/test_cases_money.txt +9 -0
- tn/hungarian/test/data/test_cases_ordinal.txt +5 -0
- tn/hungarian/test/data/test_cases_telephone.txt +9 -0
- tn/hungarian/test/data/test_cases_time.txt +10 -0
- tn/hungarian/test/data/test_cases_whitelist.txt +10 -0
- tn/hungarian/test/data/test_cases_word.txt +52 -0
- tn/hungarian/test/normalizer_test.py +210 -0
- tn/italian/__init__.py +0 -0
- tn/italian/data/electronic/domain.tsv +14 -0
- tn/italian/data/electronic/server_name.tsv +11 -0
- tn/italian/data/electronic/symbols.tsv +22 -0
- tn/italian/data/measure/measurements.tsv +64 -0
- tn/italian/data/measure/suppletive.tsv +11 -0
- tn/italian/data/money/currency_major.tsv +11 -0
- tn/italian/data/money/currency_minor.tsv +11 -0
- tn/italian/data/money/currency_plural_fem.tsv +3 -0
- tn/italian/data/money/currency_plural_masc.tsv +11 -0
- tn/italian/data/numbers/digit.tsv +9 -0
- tn/italian/data/numbers/hundreds.tsv +8 -0
- tn/italian/data/numbers/quantities.tsv +2 -0
- tn/italian/data/numbers/teen.tsv +10 -0
- tn/italian/data/numbers/tens.tsv +8 -0
- tn/italian/data/numbers/tens_eight.tsv +8 -0
- tn/italian/data/numbers/tens_one.tsv +8 -0
- tn/italian/data/numbers/zero.tsv +1 -0
- tn/italian/data/whitelist/whitelist.tsv +65 -0
- tn/italian/normalizer.py +60 -0
- tn/italian/rules/__init__.py +0 -0
- tn/italian/rules/cardinal.py +99 -0
- tn/italian/rules/decimal.py +82 -0
- tn/italian/rules/electronic.py +80 -0
- tn/italian/rules/measure.py +73 -0
- tn/italian/rules/money.py +92 -0
- tn/italian/rules/punctuation.py +30 -0
- tn/italian/rules/time.py +61 -0
- tn/italian/rules/utils.py +74 -0
- tn/italian/rules/whitelist.py +27 -0
- tn/italian/rules/word.py +28 -0
- tn/italian/test/__init__.py +0 -0
- tn/italian/test/data/test_cases_cardinal.txt +13 -0
- tn/italian/test/data/test_cases_decimal.txt +4 -0
- tn/italian/test/data/test_cases_electronic.txt +13 -0
- tn/italian/test/data/test_cases_measure.txt +7 -0
- tn/italian/test/data/test_cases_money.txt +5 -0
- tn/italian/test/data/test_cases_time.txt +6 -0
- tn/italian/test/data/test_cases_whitelist.txt +4 -0
- tn/italian/test/normalizer_test.py +209 -0
- tn/japanese/normalizer.py +56 -48
- tn/japanese/rules/cardinal.py +35 -54
- tn/japanese/rules/date.py +33 -18
- tn/japanese/rules/fraction.py +8 -14
- tn/japanese/rules/math.py +14 -6
- tn/japanese/rules/measure.py +10 -4
- tn/japanese/rules/money.py +9 -14
- tn/japanese/rules/postprocessor.py +5 -1
- tn/japanese/rules/preprocessor.py +2 -2
- tn/japanese/rules/range.py +32 -0
- tn/japanese/rules/sport.py +18 -10
- tn/japanese/rules/time.py +25 -14
- tn/japanese/rules/transliteration.py +12 -5
- tn/japanese/rules/whitelist.py +11 -7
- tn/japanese/test/normalizer_test.py +107 -4
- tn/kinyarwanda/__init__.py +0 -0
- tn/kinyarwanda/data/numbers/digit.tsv +9 -0
- tn/kinyarwanda/data/numbers/digit_bi.tsv +9 -0
- tn/kinyarwanda/data/numbers/digit_e.tsv +9 -0
- tn/kinyarwanda/data/numbers/hundreds.tsv +9 -0
- tn/kinyarwanda/data/numbers/tens.tsv +8 -0
- tn/kinyarwanda/data/numbers/zero.tsv +1 -0
- tn/kinyarwanda/data/time/hours.tsv +12 -0
- tn/kinyarwanda/data/time/minutes.tsv +60 -0
- tn/kinyarwanda/data/whitelist.tsv +175 -0
- tn/kinyarwanda/normalizer.py +56 -0
- tn/kinyarwanda/rules/__init__.py +0 -0
- tn/kinyarwanda/rules/cardinal.py +117 -0
- tn/kinyarwanda/rules/punctuation.py +31 -0
- tn/kinyarwanda/rules/time.py +40 -0
- tn/kinyarwanda/rules/utils.py +43 -0
- tn/kinyarwanda/rules/whitelist.py +28 -0
- tn/kinyarwanda/rules/word.py +29 -0
- tn/kinyarwanda/test/__init__.py +0 -0
- tn/kinyarwanda/test/data/test_cases_cardinal.txt +57 -0
- tn/kinyarwanda/test/data/test_cases_time.txt +14 -0
- tn/kinyarwanda/test/data/test_cases_whitelist.txt +3 -0
- tn/kinyarwanda/test/data/test_cases_word.txt +26 -0
- tn/kinyarwanda/test/normalizer_test.py +198 -0
- tn/korean/__init__.py +16 -0
- tn/korean/data/date/exceptions.tsv +2 -0
- tn/korean/data/date/week.tsv +8 -0
- tn/korean/data/electronic/cc_cues.tsv +11 -0
- tn/korean/data/electronic/domain.tsv +28 -0
- tn/korean/data/electronic/extensions.tsv +6 -0
- tn/korean/data/electronic/symbol.tsv +12 -0
- tn/korean/data/measure/unit.tsv +18 -0
- tn/korean/data/money/currency_major.tsv +22 -0
- tn/korean/data/number/counter_suffix.tsv +16 -0
- tn/korean/data/number/digit.tsv +9 -0
- tn/korean/data/number/native_ones.tsv +9 -0
- tn/korean/data/number/tens.tsv +9 -0
- tn/korean/data/number/zero.tsv +1 -0
- tn/korean/data/ordinal/digit.tsv +8 -0
- tn/korean/data/ordinal/exceptions.tsv +5 -0
- tn/korean/data/ordinal/tens.tsv +2 -0
- tn/korean/data/ordinal/tens_prefix.tsv +3 -0
- tn/korean/data/time/division.tsv +7 -0
- tn/korean/data/time/hour.tsv +12 -0
- tn/korean/data/whitelist.tsv +39 -0
- tn/korean/normalizer.py +79 -0
- tn/korean/rules/__init__.py +3 -0
- tn/korean/rules/cardinal.py +91 -0
- tn/korean/rules/date.py +129 -0
- tn/korean/rules/decimal.py +53 -0
- tn/korean/rules/electronic.py +84 -0
- tn/korean/rules/fraction.py +81 -0
- tn/korean/rules/measure.py +82 -0
- tn/korean/rules/money.py +60 -0
- tn/korean/rules/ordinal.py +62 -0
- tn/korean/rules/punctuation.py +16 -0
- tn/korean/rules/telephone.py +40 -0
- tn/korean/rules/time.py +81 -0
- tn/korean/rules/whitelist.py +17 -0
- tn/korean/rules/word.py +18 -0
- tn/korean/test/__init__.py +3 -0
- tn/korean/test/data/test_cases_cardinal.txt +68 -0
- tn/korean/test/data/test_cases_date.txt +29 -0
- tn/korean/test/data/test_cases_decimal.txt +28 -0
- tn/korean/test/data/test_cases_electronic.txt +20 -0
- tn/korean/test/data/test_cases_fraction.txt +28 -0
- tn/korean/test/data/test_cases_measure.txt +40 -0
- tn/korean/test/data/test_cases_money.txt +64 -0
- tn/korean/test/data/test_cases_ordinal.txt +54 -0
- tn/korean/test/data/test_cases_telephone.txt +39 -0
- tn/korean/test/data/test_cases_time.txt +25 -0
- tn/korean/test/date_test.py +99 -0
- tn/korean/test/decimal_test.py +67 -0
- tn/korean/test/electronic_test.py +58 -0
- tn/korean/test/fraction_test.py +89 -0
- tn/korean/test/measure_test.py +90 -0
- tn/korean/test/money_test.py +92 -0
- tn/korean/test/normalizer_test.py +146 -0
- tn/korean/test/ordinal_test.py +78 -0
- tn/korean/test/telephone_test.py +40 -0
- tn/korean/test/time_test.py +51 -0
- tn/main.py +2 -52
- tn/marathi/__init__.py +0 -0
- tn/marathi/data/date/eras.tsv +2 -0
- tn/marathi/data/date/months.tsv +12 -0
- tn/marathi/data/numbers/digit.tsv +9 -0
- tn/marathi/data/numbers/tens.tsv +90 -0
- tn/marathi/data/numbers/zero.tsv +1 -0
- tn/marathi/normalizer.py +53 -0
- tn/marathi/rules/__init__.py +0 -0
- tn/marathi/rules/cardinal.py +76 -0
- tn/marathi/rules/date.py +70 -0
- tn/marathi/rules/decimal.py +59 -0
- tn/marathi/rules/punctuation.py +30 -0
- tn/marathi/rules/time.py +86 -0
- tn/marathi/rules/utils.py +76 -0
- tn/marathi/rules/word.py +28 -0
- tn/marathi/test/__init__.py +0 -0
- tn/marathi/test/normalizer_test.py +221 -0
- tn/portuguese/__init__.py +0 -0
- tn/portuguese/data/date/months.tsv +21 -0
- tn/portuguese/data/date/numeric_separators.tsv +3 -0
- tn/portuguese/data/date/verbal_phrases.tsv +1 -0
- tn/portuguese/data/electronic/domain.tsv +7 -0
- tn/portuguese/data/electronic/electronic_spoken_unit.tsv +16 -0
- tn/portuguese/data/electronic/server_name.tsv +6 -0
- tn/portuguese/data/electronic/symbols.tsv +22 -0
- tn/portuguese/data/fractions/ordinal_exceptions.tsv +2 -0
- tn/portuguese/data/fractions/powers_of_ten.tsv +1 -0
- tn/portuguese/data/fractions/specials.tsv +5 -0
- tn/portuguese/data/measure/measurements_plural.tsv +64 -0
- tn/portuguese/data/measure/measurements_singular.tsv +65 -0
- tn/portuguese/data/money/currency_major.tsv +5 -0
- tn/portuguese/data/money/currency_major_plural.tsv +5 -0
- tn/portuguese/data/money/currency_minor.tsv +5 -0
- tn/portuguese/data/money/currency_minor_plural.tsv +1 -0
- tn/portuguese/data/numbers/cardinal_specials.tsv +4 -0
- tn/portuguese/data/numbers/decimal_specials.tsv +2 -0
- tn/portuguese/data/numbers/digit.tsv +9 -0
- tn/portuguese/data/numbers/hundreds.tsv +8 -0
- tn/portuguese/data/numbers/quantity_words.tsv +9 -0
- tn/portuguese/data/numbers/scales.tsv +4 -0
- tn/portuguese/data/numbers/teens.tsv +10 -0
- tn/portuguese/data/numbers/tens.tsv +8 -0
- tn/portuguese/data/numbers/zero.tsv +1 -0
- tn/portuguese/data/ordinals/digit.tsv +10 -0
- tn/portuguese/data/ordinals/feminine.tsv +11 -0
- tn/portuguese/data/ordinals/hundreds.tsv +10 -0
- tn/portuguese/data/ordinals/ties.tsv +8 -0
- tn/portuguese/data/telephone/ip_prompt.tsv +2 -0
- tn/portuguese/data/telephone/telephone_prompt.tsv +4 -0
- tn/portuguese/data/time/day_period_suffix.tsv +4 -0
- tn/portuguese/data/whitelist/symbol.tsv +23 -0
- tn/portuguese/normalizer.py +74 -0
- tn/portuguese/rules/__init__.py +0 -0
- tn/portuguese/rules/cardinal.py +170 -0
- tn/portuguese/rules/date.py +69 -0
- tn/portuguese/rules/decimal.py +67 -0
- tn/portuguese/rules/electronic.py +82 -0
- tn/portuguese/rules/fraction.py +79 -0
- tn/portuguese/rules/measure.py +66 -0
- tn/portuguese/rules/money.py +117 -0
- tn/portuguese/rules/ordinal.py +82 -0
- tn/portuguese/rules/punctuation.py +47 -0
- tn/portuguese/rules/telephone.py +71 -0
- tn/portuguese/rules/time.py +101 -0
- tn/portuguese/rules/whitelist.py +37 -0
- tn/portuguese/rules/word.py +35 -0
- tn/portuguese/test/__init__.py +0 -0
- tn/portuguese/test/data/test_cases_cardinal.txt +121 -0
- tn/portuguese/test/data/test_cases_date.txt +26 -0
- tn/portuguese/test/data/test_cases_decimal.txt +58 -0
- tn/portuguese/test/data/test_cases_electronic.txt +13 -0
- tn/portuguese/test/data/test_cases_fraction.txt +22 -0
- tn/portuguese/test/data/test_cases_measure.txt +18 -0
- tn/portuguese/test/data/test_cases_money.txt +25 -0
- tn/portuguese/test/data/test_cases_ordinal.txt +39 -0
- tn/portuguese/test/data/test_cases_telephone.txt +7 -0
- tn/portuguese/test/data/test_cases_time.txt +30 -0
- tn/portuguese/test/normalizer_test.py +166 -0
- tn/processor.py +432 -38
- tn/russian/__init__.py +0 -0
- tn/russian/data/currency/currency_agreement.tsv +10 -0
- tn/russian/data/currency/currency_plural.tsv +53 -0
- tn/russian/data/currency/currency_singular.tsv +48 -0
- tn/russian/data/electronic/domain.tsv +9 -0
- tn/russian/data/electronic/server_name.tsv +17 -0
- tn/russian/data/electronic/symbols.tsv +28 -0
- tn/russian/data/latin_to_cyrillic.tsv +38 -0
- tn/russian/data/measurements.tsv +321 -0
- tn/russian/data/months/abbr.tsv +12 -0
- tn/russian/data/months/abbr_to_name.tsv +62 -0
- tn/russian/data/months/numbers.tsv +21 -0
- tn/russian/data/numbers/1_cardinals_nominative.tsv +50 -0
- tn/russian/data/numbers/2_cardinals_genitive.tsv +45 -0
- tn/russian/data/numbers/3_cardinals_dative.tsv +45 -0
- tn/russian/data/numbers/4_cardinals_accusative.tsv +48 -0
- tn/russian/data/numbers/5_cardinals_instrumental.tsv +47 -0
- tn/russian/data/numbers/6_cardinals_prepositional.tsv +46 -0
- tn/russian/data/numbers/cardinals_alternatives.tsv +16 -0
- tn/russian/data/numbers/cardinals_nominative_case.tsv +37 -0
- tn/russian/data/numbers/decimal_delimiter.tsv +15 -0
- tn/russian/data/numbers/decimal_endings.tsv +37 -0
- tn/russian/data/numbers/digits_nominative_case.tsv +10 -0
- tn/russian/data/numbers/ordinal_endings.tsv +40 -0
- tn/russian/data/numbers/ordinals.tsv +752 -0
- tn/russian/data/numbers/quantity.tsv +16 -0
- tn/russian/data/time/increment_hour_cardinal.tsv +24 -0
- tn/russian/data/time/increment_hour_ordinal.tsv +24 -0
- tn/russian/data/time/minutes_to_hour.tsv +25 -0
- tn/russian/data/time/time_convert.tsv +12 -0
- tn/russian/data/whitelist.tsv +37 -0
- tn/russian/normalizer.py +75 -0
- tn/russian/rules/__init__.py +0 -0
- tn/russian/rules/cardinal.py +92 -0
- tn/russian/rules/date.py +93 -0
- tn/russian/rules/decimal.py +88 -0
- tn/russian/rules/electronic.py +69 -0
- tn/russian/rules/measure.py +70 -0
- tn/russian/rules/money.py +73 -0
- tn/russian/rules/ordinal.py +45 -0
- tn/russian/rules/punctuation.py +46 -0
- tn/russian/rules/telephone.py +44 -0
- tn/russian/rules/time.py +80 -0
- tn/russian/rules/utils.py +293 -0
- tn/russian/rules/whitelist.py +41 -0
- tn/russian/rules/word.py +34 -0
- tn/russian/test/__init__.py +0 -0
- tn/russian/test/data/test_cases_cardinal.txt +28 -0
- tn/russian/test/data/test_cases_cardinal_normalize_with_audio.txt +51 -0
- tn/russian/test/data/test_cases_date.txt +6 -0
- tn/russian/test/data/test_cases_decimal.txt +17 -0
- tn/russian/test/data/test_cases_electronic.txt +4 -0
- tn/russian/test/data/test_cases_measure.txt +13 -0
- tn/russian/test/data/test_cases_money.txt +9 -0
- tn/russian/test/data/test_cases_ordinal.txt +75 -0
- tn/russian/test/data/test_cases_telephone.txt +1 -0
- tn/russian/test/data/test_cases_time.txt +9 -0
- tn/russian/test/data/test_cases_whitelist.txt +4 -0
- tn/russian/test/data/test_cases_word.txt +3 -0
- tn/russian/test/normalizer_test.py +214 -0
- tn/spanish/__init__.py +0 -0
- tn/spanish/data/dates/days_abbr.tsv +27 -0
- tn/spanish/data/dates/months.tsv +12 -0
- tn/spanish/data/dates/months_abbr.tsv +44 -0
- tn/spanish/data/dates/year_suffix.tsv +51 -0
- tn/spanish/data/electronic/domain.tsv +6 -0
- tn/spanish/data/electronic/server_name.tsv +11 -0
- tn/spanish/data/electronic/symbols.tsv +22 -0
- tn/spanish/data/fractions/fraction_symbols.tsv +15 -0
- tn/spanish/data/fractions/ordinal_exceptions.tsv +2 -0
- tn/spanish/data/fractions/powers_of_ten.tsv +9 -0
- tn/spanish/data/fractions/powers_of_ten_fem.tsv +6 -0
- tn/spanish/data/measures/math_symbols.tsv +7 -0
- tn/spanish/data/measures/measurements.tsv +18 -0
- tn/spanish/data/measures/measurements_complex.tsv +5 -0
- tn/spanish/data/measures/measurements_plural_fem.tsv +3 -0
- tn/spanish/data/measures/measurements_plural_masc.tsv +21 -0
- tn/spanish/data/money/currency_major.tsv +64 -0
- tn/spanish/data/money/currency_major_ext.tsv +215 -0
- tn/spanish/data/money/currency_minor.tsv +64 -0
- tn/spanish/data/money/currency_minor_ext.tsv +181 -0
- tn/spanish/data/money/currency_plural_fem.tsv +8 -0
- tn/spanish/data/money/currency_plural_fem_ext.tsv +56 -0
- tn/spanish/data/money/currency_plural_masc.tsv +51 -0
- tn/spanish/data/money/currency_plural_masc_ext.tsv +199 -0
- tn/spanish/data/numbers/digit.tsv +9 -0
- tn/spanish/data/numbers/hundreds.tsv +8 -0
- tn/spanish/data/numbers/quantities.tsv +6 -0
- tn/spanish/data/numbers/teen.tsv +10 -0
- tn/spanish/data/numbers/ties.tsv +7 -0
- tn/spanish/data/numbers/twenties.tsv +10 -0
- tn/spanish/data/numbers/zero.tsv +1 -0
- tn/spanish/data/ordinals/digit.tsv +11 -0
- tn/spanish/data/ordinals/gender_suffix.tsv +20 -0
- tn/spanish/data/ordinals/hundreds.tsv +18 -0
- tn/spanish/data/ordinals/roman_exceptions.tsv +10 -0
- tn/spanish/data/ordinals/teen.tsv +10 -0
- tn/spanish/data/ordinals/ties.tsv +8 -0
- tn/spanish/data/ordinals/twenties.tsv +11 -0
- tn/spanish/data/roman/digit.tsv +9 -0
- tn/spanish/data/roman/hundreds.tsv +9 -0
- tn/spanish/data/roman/thousands.tsv +3 -0
- tn/spanish/data/roman/ties.tsv +9 -0
- tn/spanish/data/telephone/ip_prompt.tsv +2 -0
- tn/spanish/data/telephone/telephone_prompt.tsv +10 -0
- tn/spanish/data/time/afternoon_times.tsv +6 -0
- tn/spanish/data/time/alt_minutes.tsv +2 -0
- tn/spanish/data/time/evening_times.tsv +5 -0
- tn/spanish/data/time/hour_to_12.tsv +12 -0
- tn/spanish/data/time/hour_to_24.tsv +25 -0
- tn/spanish/data/time/hour_to_night.tsv +13 -0
- tn/spanish/data/time/minute_to.tsv +59 -0
- tn/spanish/data/time/morning_times.tsv +12 -0
- tn/spanish/data/time/time_suffix.tsv +20 -0
- tn/spanish/data/time/time_zone.tsv +42 -0
- tn/spanish/data/time/time_zone_ext.tsv +105 -0
- tn/spanish/data/whitelist.tsv +62 -0
- tn/spanish/normalizer.py +79 -0
- tn/spanish/rules/__init__.py +0 -0
- tn/spanish/rules/cardinal.py +125 -0
- tn/spanish/rules/date.py +96 -0
- tn/spanish/rules/decimal.py +122 -0
- tn/spanish/rules/electronic.py +69 -0
- tn/spanish/rules/fraction.py +189 -0
- tn/spanish/rules/measure.py +177 -0
- tn/spanish/rules/money.py +186 -0
- tn/spanish/rules/ordinal.py +119 -0
- tn/spanish/rules/punctuation.py +48 -0
- tn/spanish/rules/telephone.py +93 -0
- tn/spanish/rules/time.py +171 -0
- tn/spanish/rules/utils.py +170 -0
- tn/spanish/rules/whitelist.py +39 -0
- tn/spanish/rules/word.py +39 -0
- tn/spanish/test/__init__.py +0 -0
- tn/spanish/test/data/test_cases_cardinal.txt +69 -0
- tn/spanish/test/data/test_cases_date.txt +21 -0
- tn/spanish/test/data/test_cases_decimal.txt +24 -0
- tn/spanish/test/data/test_cases_electronic.txt +22 -0
- tn/spanish/test/data/test_cases_fraction.txt +75 -0
- tn/spanish/test/data/test_cases_measure.txt +32 -0
- tn/spanish/test/data/test_cases_money.txt +69 -0
- tn/spanish/test/data/test_cases_normalize_with_audio.txt +162 -0
- tn/spanish/test/data/test_cases_ordinal.txt +120 -0
- tn/spanish/test/data/test_cases_telephone.txt +15 -0
- tn/spanish/test/data/test_cases_time.txt +35 -0
- tn/spanish/test/data/test_cases_whitelist.txt +7 -0
- tn/spanish/test/data/test_cases_word.txt +48 -0
- tn/spanish/test/normalizer_test.py +166 -0
- tn/swedish/__init__.py +0 -0
- tn/swedish/data/abbreviations_nondet.tsv +27 -0
- tn/swedish/data/dates/era_suffix.tsv +18 -0
- tn/swedish/data/dates/era_words.tsv +4 -0
- tn/swedish/data/dates/month_abbr.tsv +17 -0
- tn/swedish/data/dates/months.tsv +12 -0
- tn/swedish/data/electronic/domain.tsv +10 -0
- tn/swedish/data/electronic/server_name.tsv +29 -0
- tn/swedish/data/electronic/symbols.tsv +20 -0
- tn/swedish/data/math_operations.tsv +8 -0
- tn/swedish/data/measure/greek_lower.tsv +25 -0
- tn/swedish/data/measure/greek_upper.tsv +25 -0
- tn/swedish/data/measure/unit.tsv +147 -0
- tn/swedish/data/measure/unit_neuter.tsv +13 -0
- tn/swedish/data/measure/unit_plural.tsv +121 -0
- tn/swedish/data/money/currency_major.tsv +75 -0
- tn/swedish/data/money/currency_major_nt.tsv +2 -0
- tn/swedish/data/money/currency_minor_plural.tsv +79 -0
- tn/swedish/data/money/currency_minor_singular.tsv +78 -0
- tn/swedish/data/money/currency_plurals.tsv +44 -0
- tn/swedish/data/numbers/digit.tsv +9 -0
- tn/swedish/data/numbers/fraction.tsv +18 -0
- tn/swedish/data/numbers/millions.tsv +88 -0
- tn/swedish/data/numbers/millions_abbr.tsv +4 -0
- tn/swedish/data/numbers/teen.tsv +10 -0
- tn/swedish/data/numbers/ties.tsv +8 -0
- tn/swedish/data/numbers/zero.tsv +1 -0
- tn/swedish/data/ordinals/digit.tsv +9 -0
- tn/swedish/data/ordinals/teen.tsv +10 -0
- tn/swedish/data/ordinals/ties.tsv +8 -0
- tn/swedish/data/ordinals/zero.tsv +1 -0
- tn/swedish/data/telephone/ip_prompt.tsv +2 -0
- tn/swedish/data/telephone/special_numbers.tsv +4 -0
- tn/swedish/data/telephone/ssn_prompt.tsv +3 -0
- tn/swedish/data/telephone/telephone_abbr.tsv +24 -0
- tn/swedish/data/telephone/telephone_prompt.tsv +5 -0
- tn/swedish/data/time/suffix.tsv +16 -0
- tn/swedish/data/time/time_zone.tsv +7 -0
- tn/swedish/data/whitelist.tsv +58 -0
- tn/swedish/normalizer.py +75 -0
- tn/swedish/rules/__init__.py +0 -0
- tn/swedish/rules/cardinal.py +172 -0
- tn/swedish/rules/date.py +122 -0
- tn/swedish/rules/decimal.py +110 -0
- tn/swedish/rules/electronic.py +75 -0
- tn/swedish/rules/fraction.py +85 -0
- tn/swedish/rules/measure.py +143 -0
- tn/swedish/rules/money.py +127 -0
- tn/swedish/rules/ordinal.py +132 -0
- tn/swedish/rules/punctuation.py +47 -0
- tn/swedish/rules/telephone.py +89 -0
- tn/swedish/rules/time.py +98 -0
- tn/swedish/rules/utils.py +63 -0
- tn/swedish/rules/whitelist.py +38 -0
- tn/swedish/rules/word.py +33 -0
- tn/swedish/test/__init__.py +0 -0
- tn/swedish/test/data/test_cases_cardinal.txt +68 -0
- tn/swedish/test/data/test_cases_date.txt +17 -0
- tn/swedish/test/data/test_cases_decimal.txt +7 -0
- tn/swedish/test/data/test_cases_electronic.txt +12 -0
- tn/swedish/test/data/test_cases_fraction.txt +2 -0
- tn/swedish/test/data/test_cases_measure.txt +7 -0
- tn/swedish/test/data/test_cases_money.txt +65 -0
- tn/swedish/test/data/test_cases_normalize_with_audio.txt +14 -0
- tn/swedish/test/data/test_cases_ordinal.txt +7 -0
- tn/swedish/test/data/test_cases_telephone.txt +7 -0
- tn/swedish/test/data/test_cases_time.txt +12 -0
- tn/swedish/test/data/test_cases_whitelist.txt +1 -0
- tn/swedish/test/data/test_cases_word.txt +48 -0
- tn/swedish/test/normalizer_test.py +195 -0
- tn/test/cli_test.py +337 -0
- tn/test/punctuation_spacing_test.py +22 -0
- tn/test/robustness_test.py +32 -0
- tn/token_parser.py +91 -29
- tn/utils.py +10 -0
- tn/vietnamese/__init__.py +0 -0
- tn/vietnamese/data/date/days.tsv +40 -0
- tn/vietnamese/data/date/months.tsv +21 -0
- tn/vietnamese/data/date/year_suffix.tsv +4 -0
- tn/vietnamese/data/fraction/denominator_exceptions.tsv +1 -0
- tn/vietnamese/data/measure/base_units.tsv +20 -0
- tn/vietnamese/data/measure/measurements_minimal.tsv +33 -0
- tn/vietnamese/data/measure/prefixes.tsv +17 -0
- tn/vietnamese/data/money/currency.tsv +51 -0
- tn/vietnamese/data/money/currency_minor.tsv +7 -0
- tn/vietnamese/data/money/per_unit_bases.tsv +8 -0
- tn/vietnamese/data/money/per_unit_non_metric.tsv +28 -0
- tn/vietnamese/data/money/per_unit_prefixes.tsv +6 -0
- tn/vietnamese/data/numbers/digit.tsv +9 -0
- tn/vietnamese/data/numbers/digit_special.tsv +3 -0
- tn/vietnamese/data/numbers/magnitudes.tsv +8 -0
- tn/vietnamese/data/numbers/quantity_abbr.tsv +7 -0
- tn/vietnamese/data/numbers/teen.tsv +10 -0
- tn/vietnamese/data/numbers/ties.tsv +8 -0
- tn/vietnamese/data/numbers/zero.tsv +1 -0
- tn/vietnamese/data/ordinal/ordinal_exceptions.tsv +2 -0
- tn/vietnamese/data/roman/key_word.tsv +16 -0
- tn/vietnamese/data/roman/roman_numerals.tsv +13 -0
- tn/vietnamese/data/time/time_units.tsv +4 -0
- tn/vietnamese/data/time/time_zones.tsv +18 -0
- tn/vietnamese/data/whitelist.tsv +0 -0
- tn/vietnamese/normalizer.py +106 -0
- tn/vietnamese/rules/__init__.py +0 -0
- tn/vietnamese/rules/cardinal.py +163 -0
- tn/vietnamese/rules/date.py +81 -0
- tn/vietnamese/rules/decimal.py +85 -0
- tn/vietnamese/rules/fraction.py +50 -0
- tn/vietnamese/rules/measure.py +57 -0
- tn/vietnamese/rules/money.py +87 -0
- tn/vietnamese/rules/ordinal.py +34 -0
- tn/vietnamese/rules/punctuation.py +31 -0
- tn/vietnamese/rules/range.py +36 -0
- tn/vietnamese/rules/roman.py +59 -0
- tn/vietnamese/rules/time.py +86 -0
- tn/vietnamese/rules/whitelist.py +29 -0
- tn/vietnamese/rules/word.py +36 -0
- tn/vietnamese/test/__init__.py +0 -0
- tn/vietnamese/test/data/test_cases_cardinal.txt +109 -0
- tn/vietnamese/test/data/test_cases_date.txt +13 -0
- tn/vietnamese/test/data/test_cases_decimal.txt +29 -0
- tn/vietnamese/test/data/test_cases_fraction.txt +13 -0
- tn/vietnamese/test/data/test_cases_measure.txt +63 -0
- tn/vietnamese/test/data/test_cases_money.txt +30 -0
- tn/vietnamese/test/data/test_cases_ordinal.txt +30 -0
- tn/vietnamese/test/data/test_cases_range.txt +11 -0
- tn/vietnamese/test/data/test_cases_roman.txt +57 -0
- tn/vietnamese/test/data/test_cases_time.txt +117 -0
- tn/vietnamese/test/normalizer_test.py +146 -0
- wetextprocessing-1.3.0.dist-info/METADATA +290 -0
- wetextprocessing-1.3.0.dist-info/RECORD +2049 -0
- itn/zh_itn_tagger.fst +0 -0
- itn/zh_itn_verbalizer.fst +0 -0
- tn/en_tn_tagger.fst +0 -0
- tn/en_tn_verbalizer.fst +0 -0
- tn/zh_tn_tagger.fst +0 -0
- tn/zh_tn_verbalizer.fst +0 -0
- wetextprocessing-1.0.6.dev19.dist-info/METADATA +0 -160
- wetextprocessing-1.0.6.dev19.dist-info/RECORD +0 -381
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/WHEEL +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/entry_points.txt +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/licenses/LICENSE +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
import pynini
|
|
17
|
+
|
|
18
|
+
from tn.processor import Processor
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Punctuation(Processor):
|
|
22
|
+
"""Single ASCII punctuation marks, kept verbatim."""
|
|
23
|
+
|
|
24
|
+
def __init__(self):
|
|
25
|
+
super().__init__(name="punctuation", ordertype="itn")
|
|
26
|
+
marks = "!#$%&'()*+,-./:;<=>?@^_`{|}~"
|
|
27
|
+
self.graph = pynini.union(*[pynini.escape(mark) for mark in marks]).optimize()
|
|
28
|
+
self.tagger = self.add_tokens(self.tag_field("value", self.graph))
|
|
29
|
+
self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.graph))
|
itn/arabic/rules/word.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
from pynini.lib.pynutil import delete
|
|
17
|
+
|
|
18
|
+
from tn.processor import Processor
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Word(Processor):
|
|
22
|
+
"""Space-free words; a whitespace-only input verbalizes to nothing."""
|
|
23
|
+
|
|
24
|
+
def __init__(self):
|
|
25
|
+
super().__init__(name="word", ordertype="itn")
|
|
26
|
+
self.tagger = self.add_tokens(self.tag_field("value", self.NOT_SPACE.plus))
|
|
27
|
+
graph = self.NOT_SPACE.plus | delete(self.SPACE.plus)
|
|
28
|
+
self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph))
|
|
File without changes
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
ألفان وتسعة~2009
|
|
2
|
+
ألف~1000
|
|
3
|
+
ثلاثة اَلاف وستة~3006
|
|
4
|
+
ألف وسبعة وسبعون~1077
|
|
5
|
+
تسعة اَلاف وثمان مئة وسبعة وثلاثون~9837
|
|
6
|
+
ثلاثة اَلاف وتسع مئة وسبعة عشر~3917
|
|
7
|
+
ألفان~2000
|
|
8
|
+
ثلاثة اَلاف وستة~3006
|
|
9
|
+
ألف وسبعة وسبعون~1077
|
|
10
|
+
تسعة اَلاف وثمان مئة وسبعة وثلاثون~9837
|
|
11
|
+
ثلاثة اَلاف وتسع مئة وسبعة عشر~3917
|
|
12
|
+
خمسة اَلاف وخمس مئة وعشرون~5520
|
|
13
|
+
ثمانية اَلاف وسبع مئة وواحد وعشرون~8721
|
|
14
|
+
ثمانية اَلاف وتسع مئة~8900
|
|
15
|
+
ثمان مئة وتسعة~809
|
|
16
|
+
مئة وسبعة~107
|
|
17
|
+
ثمان مئة وسبعة وثلاثون~837
|
|
18
|
+
تسع مئة وسبعة عشر~917
|
|
19
|
+
خمس مئة وعشرون~520
|
|
20
|
+
اثنان~2
|
|
21
|
+
ثلاثة وثلاثون~33
|
|
22
|
+
ستون~60
|
|
23
|
+
ثمانية عشر~18
|
|
24
|
+
ستة عشر~16
|
|
25
|
+
اثنا عشر~12
|
|
26
|
+
واحد~1
|
|
27
|
+
عشرون~20
|
|
28
|
+
واحد وعشرون~21
|
|
29
|
+
مئة~100
|
|
30
|
+
أربعة وثلاثون~34
|
|
31
|
+
ثمانية وسبعون~78
|
|
32
|
+
تسعة وتسعون~99
|
|
33
|
+
عشرة~10
|
|
34
|
+
سبعون~70
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
ستة وثمانية وثمانون من مئة~6.88
|
|
2
|
+
سبعة وتسعة وتسعون من مئة~7.99
|
|
3
|
+
أربعة وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف~4.9876
|
|
4
|
+
ستة وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف~6.8944
|
|
5
|
+
ستة اَلاف وسبع مئة وثمانية وتسعون وتسعة من عشرة~6798.9
|
|
6
|
+
ست مئة وثمانية وخمسون وثمان مئة وثمانية وثمانون من ألف~658.888
|
|
7
|
+
اثنان وثمانية من عشرة~2.8
|
|
8
|
+
سبعة وتسعة من عشرة~7.9
|
|
9
|
+
أربعون وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف~40.9876
|
|
10
|
+
مئتان وواحد وسبعة من عشرة~201.7
|
|
11
|
+
مئة وواحد وخمسون من مئة~101.50
|
|
12
|
+
ثلاث مئة وأربعون من مئة~300.40
|
|
13
|
+
سبعة وسبعون وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف~77.8944
|
|
14
|
+
تسعة اَلاف وثمان مئة وستة وسبعون وتسع مئة من ألف~9876.900
|
|
15
|
+
ألفان وسبعة وثمان مئة وثمانية وثمانون من ألف~2007.888
|
|
16
|
+
سبعون وتسع مئة وتسعون من ألف~70.990
|
|
17
|
+
واحد وعشرون وثمان مئة واثنان وثلاثون من ألف~21.832
|
|
18
|
+
سبعة عشر وسبعة عشر من مئة~17.17
|
|
19
|
+
ستة وثمانون واثنا عشر من مئة~86.12
|
|
20
|
+
عشرة ومئتان من ألف~10.200
|
|
21
|
+
سبع مئة وتسعة وستون وثمان مئة من ألف~769.800
|
|
22
|
+
خمسة اَلاف وست مئة وتسعة وسبعون وثمانية اَلاف وتسع مئة وثلاثة من عشرة آلاف~5679.8903
|
|
23
|
+
ستة اَلاف وتسعون من مئة~6000.90
|
|
24
|
+
واحد وثمانية عشر من مئة~1.18
|
|
25
|
+
اثنان واثنان من عشرة~2.2
|
|
26
|
+
ثلاثة وخمسة من عشرة~3.5
|
|
27
|
+
خمسة وأربعة من عشرة~5.4
|
|
28
|
+
اثنان وثمانون وثمان مئة من ألف~82.800
|
|
29
|
+
مئتان وسبعة ومئتان من ألف~207.200
|
|
30
|
+
عشرة وواحد من عشرة~10.1
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
واحد و نصف~1 1/2
|
|
2
|
+
سبعة و عشرة أخماس~7 10/5
|
|
3
|
+
خمسة و ثلثين~5 2/3
|
|
4
|
+
عشرة و نصف~10 1/2
|
|
5
|
+
ثلاثون و تسعة أعشار~30 9/10
|
|
6
|
+
سبعة وسبعون و ثلاثة أسداس~77 3/6
|
|
7
|
+
ستة عشر و سبعة عشر على ثلاثة~16 17/3
|
|
8
|
+
مئتان و واحد على ثلاث مئة وسبعة وثمانون~200 1/387
|
|
9
|
+
عشرون و ثلث~20 1/3
|
|
10
|
+
ربع~1/4
|
|
11
|
+
نصف~1/2
|
|
12
|
+
ثلث~1/3
|
|
13
|
+
تسعة أثلاث~9/3
|
|
14
|
+
ستة و ثلثين~6 2/3
|
|
15
|
+
ثلثين~2/3
|
|
16
|
+
خمس~1/5
|
|
17
|
+
سدس~1/6
|
|
18
|
+
سبع~1/7
|
|
19
|
+
ثمن~1/8
|
|
20
|
+
سبعين~2/7
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
سبعة في المائة~7%
|
|
2
|
+
تسعة فهرنهايت~9f
|
|
3
|
+
اثنا عشر في المائة~12%
|
|
4
|
+
تسعة في المائة~9%
|
|
5
|
+
عشرون في المائة~20%
|
|
6
|
+
ستة وثمانية وثمانون من مئة في المائة~6.88%
|
|
7
|
+
سبعة وتسعة وتسعون من مئة في المائة~7.99%
|
|
8
|
+
أربعة وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف في المائة~4.9876%
|
|
9
|
+
ستة وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف في المائة~6.8944%
|
|
10
|
+
ستة اَلاف وسبع مئة وثمانية وتسعون وتسعة من عشرة في المائة~6798.9%
|
|
11
|
+
ست مئة وثمانية وخمسون وثمان مئة وثمانية وثمانون من ألف في المائة~658.888%
|
|
12
|
+
اثنان وثمانية من عشرة في المائة~2.8%
|
|
13
|
+
سبعة وتسعة من عشرة في المائة~7.9%
|
|
14
|
+
واحد في المائة~1%
|
|
15
|
+
ألفان وتسعة في المائة~2009%
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
سبعة دولار~$7
|
|
2
|
+
سبعة دولار وخمسون سنت~$7.50
|
|
3
|
+
سبعة دولار وخمسة سنتات~$7.05
|
|
4
|
+
سبعة دولار وتسعة وتسعون سنت~$7.99
|
|
5
|
+
سبعة دولار وعشرة سنتات~$7.10
|
|
6
|
+
تسعة وتسعون سنت~$0.99
|
|
7
|
+
خمسة وتسعون سنت~$0.95
|
|
8
|
+
تسعة يورو~€9
|
|
9
|
+
تسعة دولار وتسعة سنتات~$9.09
|
|
10
|
+
خمسة دينار كويتي~د.ك5
|
|
11
|
+
تسعة دينار كويتي~د.ك9
|
|
12
|
+
خمسة ريال سعودي~ر.س5
|
|
13
|
+
سبعون دولار وستون سنت~$70.60
|
|
14
|
+
اثنا عشر دولار~$12
|
|
15
|
+
تسعة دولار~$9
|
|
16
|
+
عشرون دولار~$20
|
|
17
|
+
عشرون دولار وخمسون سنت~$20.50
|
|
18
|
+
عشرون دولار وعشرة سنتات~$20.10
|
|
19
|
+
عشرة سنتات~$0.10
|
|
20
|
+
ثلاثة سنتات~$0.03
|
|
21
|
+
عشرون سنت~$0.20
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
|
2
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
import pytest
|
|
19
|
+
|
|
20
|
+
from itn.arabic.inverse_normalizer import InverseNormalizer
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def original_cases():
|
|
24
|
+
cases = []
|
|
25
|
+
for path in sorted((Path(__file__).parent / "data").glob("test_cases_*.txt")):
|
|
26
|
+
# Universal newlines, as in the original tests.
|
|
27
|
+
with path.open(encoding="utf-8") as stream:
|
|
28
|
+
for index, line in enumerate(stream, 1):
|
|
29
|
+
if line.endswith("\n"):
|
|
30
|
+
line = line[:-1]
|
|
31
|
+
spoken, written = line.split("~")
|
|
32
|
+
cases.append(pytest.param(spoken, written, id="{}-{}".format(path.stem, index)))
|
|
33
|
+
return cases
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
CASES = original_cases()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@pytest.fixture(scope="module")
|
|
40
|
+
def normalizer(shared_pipeline):
|
|
41
|
+
return shared_pipeline(InverseNormalizer)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def parse(normalizer, spoken):
|
|
45
|
+
parser = normalizer.token_parser()
|
|
46
|
+
parser.parse(normalizer.tag(spoken))
|
|
47
|
+
return [(token.name, token.members) for token in parser.tokens]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_original_fixture_count():
|
|
51
|
+
assert len(CASES) == 126
|
|
52
|
+
assert CASES[0].values == ("ألفان وتسعة", "2009")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@pytest.mark.parametrize("spoken, written", CASES)
|
|
56
|
+
def test_original(normalizer, spoken, written):
|
|
57
|
+
assert normalizer.normalize(spoken) == written
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@pytest.mark.parametrize("spoken, written", CASES)
|
|
61
|
+
def test_original_mapping(normalizer, spoken, written):
|
|
62
|
+
result = normalizer.normalize_with_mapping(spoken, include_identity=True)
|
|
63
|
+
assert result.output_text == written
|
|
64
|
+
for mapping in result.mappings:
|
|
65
|
+
assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text
|
|
66
|
+
assert written[mapping.output_start:mapping.output_end] == mapping.output_text
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_tag_then_verbalize_matches_normalize(normalizer):
|
|
70
|
+
for param in CASES:
|
|
71
|
+
spoken, written = param.values
|
|
72
|
+
assert normalizer.verbalize(normalizer.tag(spoken)) == written
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@pytest.mark.parametrize("spoken, expected", [
|
|
76
|
+
("تسعة اَلاف وثمان مئة وسبعة وثلاثون", [("cardinal", {"integer": "تسعة اَلاف وثمان مئة وسبعة وثلاثون"})]),
|
|
77
|
+
("سالب خمسة", [("cardinal", {"negative": "سالب", "integer": "خمسة"})]),
|
|
78
|
+
("سبعة وتسعة وتسعون من مئة", [("decimal", {"integer_part": "سبعة", "fractional_part": "تسعة وتسعون من مئة"})]),
|
|
79
|
+
("واحد و نصف", [("fraction", {"integer_part": "واحد", "denominator": "نصف"})]),
|
|
80
|
+
("ستة عشر و سبعة عشر على ثلاثة", [("fraction", {
|
|
81
|
+
"integer_part": "ستة عشر",
|
|
82
|
+
"numerator": "سبعة عشر",
|
|
83
|
+
"denominator": "ثلاثة"
|
|
84
|
+
})]),
|
|
85
|
+
("سبعة دولار وخمسون سنت", [("money", {
|
|
86
|
+
"integer_part": "سبعة",
|
|
87
|
+
"currency": "دولار",
|
|
88
|
+
"fractional_part": "خمسون",
|
|
89
|
+
"currency_min": "سنت"
|
|
90
|
+
})]),
|
|
91
|
+
("عشرون في المائة", [("measure", {"integer": "عشرون", "units": "في المائة"})]),
|
|
92
|
+
])
|
|
93
|
+
def test_raw_field_tags(normalizer, spoken, expected):
|
|
94
|
+
assert parse(normalizer, spoken) == expected
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@pytest.mark.parametrize("spoken, written", [
|
|
98
|
+
("سالب خمسة", "-5"),
|
|
99
|
+
("ثلاثة اَلاف ومئة", "3100"),
|
|
100
|
+
("لدي ثلاثة كتب", "لدي 3 كتب"),
|
|
101
|
+
("خمسة ريال سعودي", "ر.س5"),
|
|
102
|
+
("سالب ستة وثمانية من عشرة", "-6.8"),
|
|
103
|
+
])
|
|
104
|
+
def test_hand_written(normalizer, spoken, written):
|
|
105
|
+
assert normalizer.normalize(spoken) == written
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@pytest.mark.parametrize("spoken", ["ثلاثة اَلاف وستة", "ستة وثمانية وثمانون من مئة", "واحد و نصف", "سبعة دولار وخمسون سنت"])
|
|
109
|
+
def test_nbest_mapping_synchronization(normalizer, spoken):
|
|
110
|
+
outputs = normalizer.normalize(spoken, nbest=3)
|
|
111
|
+
mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True)
|
|
112
|
+
assert [result.output_text for result in mapped] == outputs
|
|
113
|
+
assert len(outputs) == len(set(outputs))
|
|
114
|
+
assert outputs[0] == normalizer.normalize(spoken)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_escaped_word_round_trips(normalizer):
|
|
118
|
+
spoken = 'مسار\\a"b'
|
|
119
|
+
assert parse(normalizer, spoken) == [("word", {"value": spoken})]
|
|
120
|
+
assert normalizer.normalize(spoken) == spoken
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_empty_input(normalizer):
|
|
124
|
+
assert normalizer.normalize("") == ""
|
|
125
|
+
assert normalizer.normalize("", nbest=3) == [""]
|
|
126
|
+
assert normalizer.normalize_with_mapping("").output_text == ""
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@pytest.mark.parametrize("spoken", [" ", "\t", "\n", " \t\n ", "\u00a0"])
|
|
130
|
+
def test_whitespace_only_input(normalizer, spoken):
|
|
131
|
+
tagged = normalizer.tag(spoken)
|
|
132
|
+
assert parse(normalizer, spoken) == [("word", {"value": spoken})]
|
|
133
|
+
assert normalizer.verbalize(tagged) == ""
|
|
134
|
+
assert normalizer.normalize(spoken, nbest=3) == [""]
|
|
135
|
+
mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True)
|
|
136
|
+
assert [result.output_text for result in mapped] == [""]
|
|
137
|
+
assert all(mapping.output_text == "" for mapping in mapped[0].mappings)
|
itn/armenian/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
aed դիրհամ
|
|
2
|
+
ARS արգենտինական պեսո
|
|
3
|
+
֏ դրամ
|
|
4
|
+
֏ հայկական դրամ
|
|
5
|
+
a$ ավստրալական դոլար
|
|
6
|
+
br բելառուսական ռուբլի
|
|
7
|
+
₿ բիթքոին
|
|
8
|
+
£ ֆունտ
|
|
9
|
+
£ բրիտանական ֆունտ
|
|
10
|
+
ca$ կանադական դոլար
|
|
11
|
+
元 չինական յեն
|
|
12
|
+
元 յեն
|
|
13
|
+
kr դանիական կրոն
|
|
14
|
+
$ դոլար
|
|
15
|
+
€ եվրո
|
|
16
|
+
₾ վրացական լարի
|
|
17
|
+
₾ լարի
|
|
18
|
+
₹ հնդկական ռուփի
|
|
19
|
+
₹ ռուփի
|
|
20
|
+
﷼ պարսկական ռիալ
|
|
21
|
+
﷼ ռիալ
|
|
22
|
+
₪ իսրայելական շեկել
|
|
23
|
+
₪ շեկել
|
|
24
|
+
¥ ճապոնական յեն
|
|
25
|
+
¥ յեն
|
|
26
|
+
₸ ղազախական տենգե
|
|
27
|
+
₸ տենգե
|
|
28
|
+
som ղրղզական սոմ
|
|
29
|
+
som սոմ
|
|
30
|
+
ل.ل լիբանանյան ֆունտ
|
|
31
|
+
ل.ل լիբանանյան լիրա
|
|
32
|
+
$ մեքսիկական պեսո
|
|
33
|
+
nz$ նորզելանդական դոլլր
|
|
34
|
+
kr նորվեգական կրոն
|
|
35
|
+
zł լեհական զլոտի
|
|
36
|
+
zł զլոտի
|
|
37
|
+
£ ֆունտ ստերլինգ
|
|
38
|
+
£ ֆունտ
|
|
39
|
+
₽ ռուսական ռուբլի
|
|
40
|
+
₽ ռուբլի
|
|
41
|
+
rsd սերբական դինար
|
|
42
|
+
s$ սինգապուրի դոլար
|
|
43
|
+
₩ կորեական վոն
|
|
44
|
+
kr շվեդական կրոն
|
|
45
|
+
chf շվեյցարական ֆրանկ
|
|
46
|
+
£s սիրիական ֆունտ
|
|
47
|
+
₺ թուրքական լիրա
|
|
48
|
+
₴ ուկրաինական գրիվնա
|
|
49
|
+
$ ամերիկյան դոլար
|
|
50
|
+
$ ամն դոլար
|
|
51
|
+
som ուզբեկական սոմ
|
|
52
|
+
₩ վոն
|
|
53
|
+
¥ յեն
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
°F ֆարենհայթ
|
|
2
|
+
°C ցելսիուս
|
|
3
|
+
K կելվին
|
|
4
|
+
կմ կիլոմետր
|
|
5
|
+
կմ/ժ կիլոմետր ժամ
|
|
6
|
+
մ մետր
|
|
7
|
+
մ/ժ մետր ժամ
|
|
8
|
+
սմ սանտիմետր
|
|
9
|
+
մմ միլիմետր
|
|
10
|
+
հա հեկտար
|
|
11
|
+
մղն մղոն
|
|
12
|
+
մ² քառակուսի մետր
|
|
13
|
+
մ² մետր քառակուսի
|
|
14
|
+
կմ² քառակուսի կիլոմետր
|
|
15
|
+
կմ² կիլոմետր քառակուսի
|
|
16
|
+
% տոկոս
|
|
17
|
+
% տոկոսադրույք
|
|
18
|
+
Հց հերց
|
|
19
|
+
կՎտ կիլովատտ
|
|
20
|
+
կՎտ կիլո վատտ
|
|
21
|
+
կՎ/Ժ կիլովատտ ժամ
|
|
22
|
+
կՎ/ժ կիլո վատտ ժամ
|
|
23
|
+
Վտ/ժ վատտ ժամ
|
|
24
|
+
Վտ վատտ
|
|
25
|
+
ձ.ու. ձիաուժ
|
|
26
|
+
մգ միլիգրամ
|
|
27
|
+
կգ կիլոգրամ
|
|
28
|
+
Վ վոլտ
|
|
29
|
+
ժ ժամ
|
|
30
|
+
վ վայրկյան
|
|
31
|
+
ր րոպե
|
|
32
|
+
մ³ խորանարդ մետր
|
|
33
|
+
գ գրամ
|
|
34
|
+
տ տոննա
|
|
35
|
+
մբ մեգաբայթ
|
|
36
|
+
կբ կիլոբայթ
|
|
37
|
+
գբ գիգաբայթ
|
|
38
|
+
գբ գեգաբայթ
|
|
39
|
+
տբ տերաբայթ
|
|
40
|
+
տբ տեռաբայթ
|
|
41
|
+
կՎ կիլո վոլտ
|
|
42
|
+
մՎ մեգա վոլտ
|
|
43
|
+
Ա ամպեր
|
|
44
|
+
մԱ միլի ամպեր
|
|
45
|
+
մվ միլի վայրկյան
|
|
46
|
+
դմ դեցիմետր
|
|
47
|
+
սմ² քառակուսի սանտիմետր
|
|
48
|
+
սմ² քառակուսի սանտիմետր
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
զրո 0
|
|
2
|
+
մեկ 1
|
|
3
|
+
երկու 2
|
|
4
|
+
երկուս 2
|
|
5
|
+
երեք 3
|
|
6
|
+
չորս 4
|
|
7
|
+
հինգ 5
|
|
8
|
+
վեց 6
|
|
9
|
+
յոթ 7
|
|
10
|
+
ութ 8
|
|
11
|
+
ինը 9
|
|
12
|
+
տաս 10
|
|
13
|
+
տասն 10
|
|
14
|
+
տասնմեկ 11
|
|
15
|
+
տասներկու 12
|
|
16
|
+
տասներկուս 12
|
|
17
|
+
տասներեք 13
|
|
18
|
+
տասնչորս 14
|
|
19
|
+
տասնհինգ 15
|
|
20
|
+
տասնվեց 16
|
|
21
|
+
տասնյոթ 17
|
|
22
|
+
տասնութ 18
|
|
23
|
+
տասնինը 19
|
|
24
|
+
քսան 20
|
|
25
|
+
քսանմեկ 21
|
|
26
|
+
քսաներկու 22
|
|
27
|
+
քսաներկուս 22
|
|
28
|
+
քսաներեք 23
|
|
29
|
+
քսանչորս 24
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
մեկ 01
|
|
2
|
+
երկու 02
|
|
3
|
+
երկուս 02
|
|
4
|
+
երեք 03
|
|
5
|
+
չորս 04
|
|
6
|
+
հինգ 05
|
|
7
|
+
վեց 06
|
|
8
|
+
յոթ 07
|
|
9
|
+
ութ 08
|
|
10
|
+
ինը 09
|
|
11
|
+
տասը 10
|
|
12
|
+
տասնմեկ 11
|
|
13
|
+
տասներկու 12
|
|
14
|
+
տասներկուս 12
|
|
15
|
+
տասներեք 13
|
|
16
|
+
տասնչորս 14
|
|
17
|
+
տասնհինգ 15
|
|
18
|
+
տասնվեց 16
|
|
19
|
+
տասնյոթ 17
|
|
20
|
+
տասնութ 18
|
|
21
|
+
տասնինը 19
|
|
22
|
+
քսան 20
|
|
23
|
+
քսանմեկ 21
|
|
24
|
+
քսաներկու 22
|
|
25
|
+
քսաներկուս 22
|
|
26
|
+
քսաներեք 23
|
|
27
|
+
քսանչորս 24
|
|
28
|
+
քսանհինգ 25
|
|
29
|
+
քսանվեց 26
|
|
30
|
+
քսանյոթ 27
|
|
31
|
+
քսանութ 28
|
|
32
|
+
քսանինը 29
|
|
33
|
+
երեսուն 30
|
|
34
|
+
երեսունմեկ 31
|
|
35
|
+
երեսուներկու 32
|
|
36
|
+
երեսուներկուս 32
|
|
37
|
+
երեսուներեք 33
|
|
38
|
+
երեսունչորս 34
|
|
39
|
+
երեսունհինգ 35
|
|
40
|
+
երեսունվեց 36
|
|
41
|
+
երեսունյոթ 37
|
|
42
|
+
երեսունութ 38
|
|
43
|
+
երեսունինը 39
|
|
44
|
+
քառասուն 40
|
|
45
|
+
քառասունմեկ 41
|
|
46
|
+
քառասուներկու 42
|
|
47
|
+
քառասուներկուս 42
|
|
48
|
+
քառասուներեք 43
|
|
49
|
+
քառասունչորս 44
|
|
50
|
+
քառասունհինգ 45
|
|
51
|
+
քառասունվեց 46
|
|
52
|
+
քառասունյոթ 47
|
|
53
|
+
քառասունութ 48
|
|
54
|
+
քառասունինը 49
|
|
55
|
+
հիսուն 50
|
|
56
|
+
հիսունմեկ 51
|
|
57
|
+
հիսուներկու 52
|
|
58
|
+
հիսուներկուս 52
|
|
59
|
+
հիսուներեք 53
|
|
60
|
+
հիսունչորս 54
|
|
61
|
+
հիսունհինգ 55
|
|
62
|
+
հիսունվեց 56
|
|
63
|
+
հիսունյոթ 57
|
|
64
|
+
հիսունութ 58
|
|
65
|
+
հիսունինը 59
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
01 59
|
|
2
|
+
02 58
|
|
3
|
+
03 57
|
|
4
|
+
04 56
|
|
5
|
+
05 55
|
|
6
|
+
06 54
|
|
7
|
+
07 53
|
|
8
|
+
08 52
|
|
9
|
+
09 51
|
|
10
|
+
10 50
|
|
11
|
+
11 49
|
|
12
|
+
12 48
|
|
13
|
+
13 47
|
|
14
|
+
14 46
|
|
15
|
+
15 45
|
|
16
|
+
16 44
|
|
17
|
+
17 43
|
|
18
|
+
18 42
|
|
19
|
+
19 41
|
|
20
|
+
20 40
|
|
21
|
+
21 39
|
|
22
|
+
22 38
|
|
23
|
+
23 37
|
|
24
|
+
24 36
|
|
25
|
+
25 35
|
|
26
|
+
26 34
|
|
27
|
+
27 33
|
|
28
|
+
28 32
|
|
29
|
+
29 31
|
|
30
|
+
30 30
|
|
31
|
+
31 29
|
|
32
|
+
32 28
|
|
33
|
+
33 27
|
|
34
|
+
34 26
|
|
35
|
+
35 25
|
|
36
|
+
36 24
|
|
37
|
+
37 23
|
|
38
|
+
38 22
|
|
39
|
+
39 21
|
|
40
|
+
40 20
|
|
41
|
+
41 19
|
|
42
|
+
42 18
|
|
43
|
+
43 17
|
|
44
|
+
44 16
|
|
45
|
+
45 15
|
|
46
|
+
46 14
|
|
47
|
+
47 13
|
|
48
|
+
48 12
|
|
49
|
+
49 11
|
|
50
|
+
50 10
|
|
51
|
+
51 09
|
|
52
|
+
52 08
|
|
53
|
+
53 07
|
|
54
|
+
54 06
|
|
55
|
+
55 05
|
|
56
|
+
56 04
|
|
57
|
+
57 03
|
|
58
|
+
58 02
|
|
59
|
+
59 01
|