WeTextProcessing 1.0.6.dev19__py3-none-any.whl → 1.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1776) hide show
  1. itn/README.md +12 -4
  2. itn/arabic/__init__.py +0 -0
  3. itn/arabic/data/measure/measurements.tsv +14 -0
  4. itn/arabic/data/money/currency_major.tsv +24 -0
  5. itn/arabic/data/money/currency_minor_plural.tsv +4 -0
  6. itn/arabic/data/money/currency_minor_singular.tsv +3 -0
  7. itn/arabic/data/money/local_currency.tsv +20 -0
  8. itn/arabic/data/number/digit.tsv +9 -0
  9. itn/arabic/data/number/digit_100.tsv +7 -0
  10. itn/arabic/data/number/digit_1000.tsv +7 -0
  11. itn/arabic/data/number/flops.tsv +72 -0
  12. itn/arabic/data/number/fraction_dual.tsv +9 -0
  13. itn/arabic/data/number/fraction_plural.tsv +8 -0
  14. itn/arabic/data/number/fraction_singular.tsv +9 -0
  15. itn/arabic/data/number/quantities.tsv +6 -0
  16. itn/arabic/data/number/teens.tsv +9 -0
  17. itn/arabic/data/number/tens.tsv +10 -0
  18. itn/arabic/inverse_normalizer.py +60 -0
  19. itn/arabic/rules/__init__.py +0 -0
  20. itn/arabic/rules/cardinal.py +66 -0
  21. itn/arabic/rules/decimal.py +58 -0
  22. itn/arabic/rules/fraction.py +72 -0
  23. itn/arabic/rules/measure.py +49 -0
  24. itn/arabic/rules/money.py +73 -0
  25. itn/arabic/rules/punctuation.py +29 -0
  26. itn/arabic/rules/word.py +28 -0
  27. itn/arabic/test/__init__.py +0 -0
  28. itn/arabic/test/data/test_cases_cardinal.txt +34 -0
  29. itn/arabic/test/data/test_cases_decimal.txt +30 -0
  30. itn/arabic/test/data/test_cases_fraction.txt +20 -0
  31. itn/arabic/test/data/test_cases_measure.txt +15 -0
  32. itn/arabic/test/data/test_cases_money.txt +21 -0
  33. itn/arabic/test/data/test_cases_whitelist.txt +6 -0
  34. itn/arabic/test/normalizer_test.py +137 -0
  35. itn/armenian/__init__.py +0 -0
  36. itn/armenian/data/currency.tsv +53 -0
  37. itn/armenian/data/measurement_dates.tsv +4 -0
  38. itn/armenian/data/measurements.tsv +48 -0
  39. itn/armenian/data/numbers/digit.tsv +11 -0
  40. itn/armenian/data/numbers/digits_no_one.tsv +10 -0
  41. itn/armenian/data/numbers/ties.tsv +10 -0
  42. itn/armenian/data/ordinals/digit.tsv +10 -0
  43. itn/armenian/data/time/hours.tsv +29 -0
  44. itn/armenian/data/time/minutes.tsv +65 -0
  45. itn/armenian/data/time/minutes_to.tsv +59 -0
  46. itn/armenian/data/time/to_hour.tsv +25 -0
  47. itn/armenian/data/whitelist.tsv +7 -0
  48. itn/armenian/inverse_normalizer.py +66 -0
  49. itn/armenian/rules/__init__.py +0 -0
  50. itn/armenian/rules/cardinal.py +67 -0
  51. itn/armenian/rules/decimal.py +67 -0
  52. itn/armenian/rules/fraction.py +40 -0
  53. itn/armenian/rules/measure.py +52 -0
  54. itn/armenian/rules/money.py +38 -0
  55. itn/armenian/rules/ordinal.py +37 -0
  56. itn/armenian/rules/punctuation.py +29 -0
  57. itn/armenian/rules/time.py +59 -0
  58. itn/armenian/rules/utils.py +58 -0
  59. itn/armenian/rules/whitelist.py +27 -0
  60. itn/armenian/rules/word.py +28 -0
  61. itn/armenian/test/__init__.py +0 -0
  62. itn/armenian/test/data/test_cases_cardinal.txt +23 -0
  63. itn/armenian/test/data/test_cases_decimal.txt +14 -0
  64. itn/armenian/test/data/test_cases_fraction.txt +24 -0
  65. itn/armenian/test/data/test_cases_measure.txt +12 -0
  66. itn/armenian/test/data/test_cases_money.txt +10 -0
  67. itn/armenian/test/data/test_cases_ordinal.txt +11 -0
  68. itn/armenian/test/data/test_cases_time.txt +9 -0
  69. itn/armenian/test/data/test_cases_whitelist.txt +7 -0
  70. itn/armenian/test/data/test_cases_word.txt +50 -0
  71. itn/armenian/test/normalizer_test.py +158 -0
  72. itn/chinese/inverse_normalizer.py +49 -33
  73. itn/chinese/rules/cardinal.py +42 -69
  74. itn/chinese/rules/date.py +25 -18
  75. itn/chinese/rules/fraction.py +15 -14
  76. itn/chinese/rules/license_plate.py +11 -10
  77. itn/chinese/rules/math.py +7 -7
  78. itn/chinese/rules/measure.py +27 -42
  79. itn/chinese/rules/money.py +12 -18
  80. itn/chinese/rules/postprocessor.py +2 -2
  81. itn/chinese/rules/time.py +24 -25
  82. itn/chinese/rules/train_number.py +9 -8
  83. itn/chinese/rules/whitelist.py +7 -6
  84. itn/chinese/test/normalizer_test.py +155 -30
  85. itn/english/__init__.py +0 -0
  86. itn/english/data/currency.tsv +168 -0
  87. itn/english/data/date_period.tsv +10 -0
  88. itn/english/data/electronic/domain.tsv +10 -0
  89. itn/english/data/electronic/server_name.tsv +17 -0
  90. itn/english/data/electronic/symbols.tsv +22 -0
  91. itn/english/data/electronic/url_symbols.tsv +5 -0
  92. itn/english/data/magnitudes.tsv +4 -0
  93. itn/english/data/measurements.tsv +149 -0
  94. itn/english/data/months.tsv +12 -0
  95. itn/english/data/months_cased.tsv +12 -0
  96. itn/english/data/numbers/digit.tsv +9 -0
  97. itn/english/data/numbers/hundred.tsv +1 -0
  98. itn/english/data/numbers/teen.tsv +10 -0
  99. itn/english/data/numbers/thousands.tsv +22 -0
  100. itn/english/data/numbers/ties.tsv +9 -0
  101. itn/english/data/numbers/zero.tsv +1 -0
  102. itn/english/data/ordinals/digit.tsv +9 -0
  103. itn/english/data/ordinals/teen.tsv +1 -0
  104. itn/english/data/time/minute_to.tsv +59 -0
  105. itn/english/data/time/time_suffix.tsv +8 -0
  106. itn/english/data/time/time_suffix_cased.tsv +8 -0
  107. itn/english/data/time/time_zone.tsv +7 -0
  108. itn/english/data/time/time_zone_cased.tsv +7 -0
  109. itn/english/data/time/to_hour.tsv +12 -0
  110. itn/english/data/whitelist.tsv +417 -0
  111. itn/english/data/whitelist_tech.tsv +450 -0
  112. itn/english/data/year_suffix.tsv +6 -0
  113. itn/english/inverse_normalizer.py +91 -0
  114. itn/english/rules/__init__.py +0 -0
  115. itn/english/rules/cardinal.py +111 -0
  116. itn/english/rules/char.py +29 -0
  117. itn/english/rules/date.py +163 -0
  118. itn/english/rules/decimal.py +82 -0
  119. itn/english/rules/electronic.py +79 -0
  120. itn/english/rules/measure.py +106 -0
  121. itn/english/rules/money.py +121 -0
  122. itn/english/rules/ordinal.py +50 -0
  123. itn/english/rules/punctuation.py +31 -0
  124. itn/english/rules/telephone.py +147 -0
  125. itn/english/rules/time.py +138 -0
  126. itn/english/rules/whitelist.py +33 -0
  127. itn/english/rules/word.py +26 -0
  128. itn/english/test/__init__.py +0 -0
  129. itn/english/test/data/en_cardinal.txt +28 -0
  130. itn/english/test/data/en_date.txt +44 -0
  131. itn/english/test/data/en_decimal.txt +63 -0
  132. itn/english/test/data/en_electronic.txt +25 -0
  133. itn/english/test/data/en_measure.txt +114 -0
  134. itn/english/test/data/en_money.txt +52 -0
  135. itn/english/test/data/en_ordinal.txt +34 -0
  136. itn/english/test/data/en_telephone.txt +24 -0
  137. itn/english/test/data/en_time.txt +36 -0
  138. itn/english/test/data/en_whitelist.txt +12 -0
  139. itn/english/test/data/en_word.txt +55 -0
  140. itn/english/test/normalizer_test.py +130 -0
  141. tn/chinese/test/preprocessor_test.py → itn/english/test/utils.py +14 -11
  142. itn/french/__init__.py +0 -0
  143. itn/french/data/electronic/domain.tsv +25 -0
  144. itn/french/data/electronic/server_name.tsv +17 -0
  145. itn/french/data/electronic/symbols.tsv +12 -0
  146. itn/french/data/fractions.tsv +41 -0
  147. itn/french/data/measurements/magnitudes.tsv +16 -0
  148. itn/french/data/measurements/measurements.tsv +27 -0
  149. itn/french/data/money/currency_major.tsv +40 -0
  150. itn/french/data/money/currency_minor.tsv +5 -0
  151. itn/french/data/months.tsv +12 -0
  152. itn/french/data/numbers/digit.tsv +10 -0
  153. itn/french/data/numbers/teen.tsv +6 -0
  154. itn/french/data/numbers/ties.tsv +13 -0
  155. itn/french/data/numbers/ties_unique.tsv +25 -0
  156. itn/french/data/numbers/zero.tsv +1 -0
  157. itn/french/data/ordinals/digits_root_change.tsv +17 -0
  158. itn/french/data/ordinals/firsts.tsv +4 -0
  159. itn/french/data/ordinals/key_nouns.tsv +1 -0
  160. itn/french/data/ordinals/second.tsv +4 -0
  161. itn/french/data/roman/digits_large.tsv +9 -0
  162. itn/french/data/roman/hundreds_large.tsv +9 -0
  163. itn/french/data/roman/ties_large.tsv +9 -0
  164. itn/french/data/time/hour_to_night.tsv +12 -0
  165. itn/french/data/time/hours.tsv +26 -0
  166. itn/french/data/time/hours_to.tsv +25 -0
  167. itn/french/data/time/minutes.tsv +63 -0
  168. itn/french/data/time/minutes_to.tsv +59 -0
  169. itn/french/data/time/time_suffix_am.tsv +1 -0
  170. itn/french/data/time/time_suffix_pm.tsv +2 -0
  171. itn/french/data/whitelist.tsv +16 -0
  172. itn/french/inverse_normalizer.py +74 -0
  173. itn/french/rules/__init__.py +0 -0
  174. itn/french/rules/cardinal.py +115 -0
  175. itn/french/rules/date.py +45 -0
  176. itn/french/rules/decimal.py +70 -0
  177. itn/french/rules/electronic.py +67 -0
  178. itn/french/rules/fraction.py +55 -0
  179. itn/french/rules/measure.py +55 -0
  180. itn/french/rules/money.py +67 -0
  181. itn/french/rules/ordinal.py +63 -0
  182. itn/french/rules/punctuation.py +29 -0
  183. itn/french/rules/telephone.py +59 -0
  184. itn/french/rules/time.py +70 -0
  185. itn/french/rules/whitelist.py +27 -0
  186. itn/french/rules/word.py +29 -0
  187. itn/french/test/__init__.py +0 -0
  188. itn/french/test/data/test_cases_cardinal.txt +106 -0
  189. itn/french/test/data/test_cases_date.txt +6 -0
  190. itn/french/test/data/test_cases_decimal.txt +15 -0
  191. itn/french/test/data/test_cases_electronic.txt +10 -0
  192. itn/french/test/data/test_cases_fraction.txt +30 -0
  193. itn/french/test/data/test_cases_measure.txt +15 -0
  194. itn/french/test/data/test_cases_money.txt +22 -0
  195. itn/french/test/data/test_cases_ordinal.txt +23 -0
  196. itn/french/test/data/test_cases_telephone.txt +5 -0
  197. itn/french/test/data/test_cases_time.txt +18 -0
  198. itn/french/test/data/test_cases_whitelist.txt +8 -0
  199. itn/french/test/data/test_cases_word.txt +49 -0
  200. itn/french/test/normalizer_test.py +155 -0
  201. itn/german/__init__.py +0 -0
  202. itn/german/inverse_normalizer.py +85 -0
  203. itn/german/rules/__init__.py +0 -0
  204. itn/german/rules/cardinal.py +48 -0
  205. itn/german/rules/date.py +63 -0
  206. itn/german/rules/decimal.py +67 -0
  207. itn/german/rules/electronic.py +58 -0
  208. itn/german/rules/fraction.py +75 -0
  209. itn/german/rules/measure.py +80 -0
  210. itn/german/rules/money.py +87 -0
  211. itn/german/rules/ordinal.py +31 -0
  212. itn/german/rules/punctuation.py +29 -0
  213. itn/german/rules/telephone.py +45 -0
  214. itn/german/rules/time.py +102 -0
  215. itn/german/rules/whitelist.py +30 -0
  216. itn/german/rules/word.py +33 -0
  217. itn/german/test/__init__.py +0 -0
  218. itn/german/test/calendar_test.py +147 -0
  219. itn/german/test/data/test_cases_cardinal.txt +62 -0
  220. itn/german/test/data/test_cases_date.txt +22 -0
  221. itn/german/test/data/test_cases_decimal.txt +10 -0
  222. itn/german/test/data/test_cases_electronic.txt +9 -0
  223. itn/german/test/data/test_cases_fraction.txt +34 -0
  224. itn/german/test/data/test_cases_measure.txt +28 -0
  225. itn/german/test/data/test_cases_money.txt +23 -0
  226. itn/german/test/data/test_cases_ordinal.txt +20 -0
  227. itn/german/test/data/test_cases_telephone.txt +1 -0
  228. itn/german/test/data/test_cases_time.txt +24 -0
  229. itn/german/test/data/test_cases_whitelist.txt +6 -0
  230. itn/german/test/data/test_cases_word.txt +49 -0
  231. itn/german/test/financial_test.py +284 -0
  232. itn/german/test/inverse_normalizer_test.py +92 -0
  233. itn/german/test/numeric_test.py +219 -0
  234. itn/german/test/text_test.py +281 -0
  235. itn/german/test/time_test.py +70 -0
  236. itn/hebrew/__init__.py +0 -0
  237. itn/hebrew/data/decimals/minutes_exception.tsv +2 -0
  238. itn/hebrew/data/measurements.tsv +45 -0
  239. itn/hebrew/data/months.tsv +13 -0
  240. itn/hebrew/data/months_name2number.tsv +12 -0
  241. itn/hebrew/data/months_ordinal2number.tsv +12 -0
  242. itn/hebrew/data/numbers/decimal_fractions.tsv +6 -0
  243. itn/hebrew/data/numbers/digit.tsv +20 -0
  244. itn/hebrew/data/numbers/hundreds_exception.tsv +2 -0
  245. itn/hebrew/data/numbers/millions_exception.tsv +1 -0
  246. itn/hebrew/data/numbers/teen.tsv +21 -0
  247. itn/hebrew/data/numbers/thousands.tsv +8 -0
  248. itn/hebrew/data/numbers/thousands_exception.tsv +2 -0
  249. itn/hebrew/data/numbers/ties.tsv +8 -0
  250. itn/hebrew/data/numbers/viable_hours.tsv +15 -0
  251. itn/hebrew/data/numbers/zero.tsv +1 -0
  252. itn/hebrew/data/ordinals/digit.tsv +10 -0
  253. itn/hebrew/data/prefix.tsv +17 -0
  254. itn/hebrew/data/spaced_measurements.tsv +17 -0
  255. itn/hebrew/data/time/day_suffix.tsv +2 -0
  256. itn/hebrew/data/time/evening_suffix.tsv +2 -0
  257. itn/hebrew/data/time/hour_to_evening.tsv +7 -0
  258. itn/hebrew/data/time/hour_to_night.tsv +9 -0
  259. itn/hebrew/data/time/hour_to_noon.tsv +7 -0
  260. itn/hebrew/data/time/midnight_to_hour.tsv +1 -0
  261. itn/hebrew/data/time/minute_to.tsv +58 -0
  262. itn/hebrew/data/time/minute_to_verbose.tsv +6 -0
  263. itn/hebrew/data/time/minute_verbose.tsv +8 -0
  264. itn/hebrew/data/time/night_suffix.tsv +1 -0
  265. itn/hebrew/data/time/noon_suffix.tsv +3 -0
  266. itn/hebrew/data/time/to_hour.tsv +13 -0
  267. itn/hebrew/data/whitelist.tsv +20 -0
  268. itn/hebrew/inverse_normalizer.py +64 -0
  269. itn/hebrew/rules/__init__.py +0 -0
  270. itn/hebrew/rules/cardinal.py +94 -0
  271. itn/hebrew/rules/date.py +62 -0
  272. itn/hebrew/rules/decimal.py +74 -0
  273. itn/hebrew/rules/measure.py +53 -0
  274. itn/hebrew/rules/punctuation.py +28 -0
  275. itn/hebrew/rules/time.py +101 -0
  276. itn/hebrew/rules/utils.py +78 -0
  277. itn/hebrew/rules/whitelist.py +30 -0
  278. itn/hebrew/rules/word.py +28 -0
  279. itn/hebrew/test/__init__.py +0 -0
  280. itn/hebrew/test/data/test_cases_cardinal.txt +138 -0
  281. itn/hebrew/test/data/test_cases_date.txt +29 -0
  282. itn/hebrew/test/data/test_cases_decimal.txt +66 -0
  283. itn/hebrew/test/data/test_cases_measure.txt +10 -0
  284. itn/hebrew/test/data/test_cases_time.txt +34 -0
  285. itn/hebrew/test/data/test_cases_whitelist.txt +4 -0
  286. itn/hebrew/test/data/test_full_sentences.txt +56 -0
  287. itn/hebrew/test/normalizer_test.py +178 -0
  288. itn/hindi/__init__.py +0 -0
  289. itn/hindi/data/address/cities.tsv +36 -0
  290. itn/hindi/data/address/context_cues.tsv +85 -0
  291. itn/hindi/data/address/digit_passthrough.tsv +10 -0
  292. itn/hindi/data/address/ordinals.tsv +167 -0
  293. itn/hindi/data/address/special_characters.tsv +2 -0
  294. itn/hindi/data/address/states.tsv +36 -0
  295. itn/hindi/data/date/century.tsv +5 -0
  296. itn/hindi/data/date/months.tsv +14 -0
  297. itn/hindi/data/measure/measurements.tsv +295 -0
  298. itn/hindi/data/money/currency.tsv +181 -0
  299. itn/hindi/data/numbers/digit.tsv +13 -0
  300. itn/hindi/data/numbers/hundred.tsv +1 -0
  301. itn/hindi/data/numbers/labels_exception.tsv +4 -0
  302. itn/hindi/data/numbers/paune.tsv +231 -0
  303. itn/hindi/data/numbers/teens_and_ties.tsv +220 -0
  304. itn/hindi/data/numbers/thousands.tsv +8 -0
  305. itn/hindi/data/numbers/zero.tsv +1 -0
  306. itn/hindi/data/ordinals/digit.tsv +6 -0
  307. itn/hindi/data/ordinals/digit_fem.tsv +6 -0
  308. itn/hindi/data/ordinals/hundred_digit.tsv +12 -0
  309. itn/hindi/data/ordinals/hundred_digit_fem.tsv +12 -0
  310. itn/hindi/data/ordinals/morph_features.tsv +6 -0
  311. itn/hindi/data/ordinals/suffix.tsv +4 -0
  312. itn/hindi/data/ordinals/suffix_fem.tsv +2 -0
  313. itn/hindi/data/ordinals/teens_and_ties.tsv +1 -0
  314. itn/hindi/data/ordinals/teens_and_ties_fem.tsv +1 -0
  315. itn/hindi/data/telephone/context_cues.tsv +15 -0
  316. itn/hindi/data/telephone/country_codes.tsv +1 -0
  317. itn/hindi/data/telephone/eng_digit.tsv +9 -0
  318. itn/hindi/data/telephone/eng_zero.tsv +1 -0
  319. itn/hindi/data/telephone/teens_and_ties_eng_to_hin.tsv +90 -0
  320. itn/hindi/data/time/hour_for_paune.tsv +15 -0
  321. itn/hindi/data/whitelist/whitelist.tsv +7 -0
  322. itn/hindi/inverse_normalizer.py +78 -0
  323. itn/hindi/rules/__init__.py +0 -0
  324. itn/hindi/rules/cardinal.py +93 -0
  325. itn/hindi/rules/date.py +64 -0
  326. itn/hindi/rules/decimal.py +57 -0
  327. itn/hindi/rules/fraction.py +55 -0
  328. itn/hindi/rules/measure.py +83 -0
  329. itn/hindi/rules/money.py +81 -0
  330. itn/hindi/rules/ordinal.py +40 -0
  331. itn/hindi/rules/punctuation.py +31 -0
  332. itn/hindi/rules/telephone.py +65 -0
  333. itn/hindi/rules/time.py +53 -0
  334. itn/hindi/rules/utils.py +52 -0
  335. itn/hindi/rules/whitelist.py +32 -0
  336. itn/hindi/rules/word.py +30 -0
  337. itn/hindi/test/__init__.py +0 -0
  338. itn/hindi/test/data/test_cases_address.txt +25 -0
  339. itn/hindi/test/data/test_cases_cardinal.txt +54 -0
  340. itn/hindi/test/data/test_cases_date.txt +42 -0
  341. itn/hindi/test/data/test_cases_decimal.txt +13 -0
  342. itn/hindi/test/data/test_cases_fraction.txt +31 -0
  343. itn/hindi/test/data/test_cases_measure.txt +48 -0
  344. itn/hindi/test/data/test_cases_money.txt +50 -0
  345. itn/hindi/test/data/test_cases_ordinal.txt +13 -0
  346. itn/hindi/test/data/test_cases_telephone.txt +28 -0
  347. itn/hindi/test/data/test_cases_time.txt +25 -0
  348. itn/hindi/test/data/test_cases_whitelist.txt +8 -0
  349. itn/hindi/test/data/test_cases_word.txt +15 -0
  350. itn/hindi/test/normalizer_test.py +127 -0
  351. itn/hindi_english/__init__.py +0 -0
  352. itn/hindi_english/data/measurements.tsv +2 -0
  353. itn/hindi_english/inverse_normalizer.py +93 -0
  354. itn/hindi_english/rules/__init__.py +0 -0
  355. itn/hindi_english/rules/english.py +175 -0
  356. itn/hindi_english/test/__init__.py +0 -0
  357. itn/hindi_english/test/data/test_cases_address.txt +41 -0
  358. itn/hindi_english/test/data/test_cases_cardinal.txt +61 -0
  359. itn/hindi_english/test/data/test_cases_date.txt +37 -0
  360. itn/hindi_english/test/data/test_cases_date_cased.txt +70 -0
  361. itn/hindi_english/test/data/test_cases_decimal.txt +25 -0
  362. itn/hindi_english/test/data/test_cases_electronic.txt +17 -0
  363. itn/hindi_english/test/data/test_cases_fraction.txt +39 -0
  364. itn/hindi_english/test/data/test_cases_measure.txt +25 -0
  365. itn/hindi_english/test/data/test_cases_money.txt +41 -0
  366. itn/hindi_english/test/data/test_cases_ordinal.txt +26 -0
  367. itn/hindi_english/test/data/test_cases_telephone.txt +39 -0
  368. itn/hindi_english/test/data/test_cases_time.txt +29 -0
  369. itn/hindi_english/test/data/test_cases_whitelist.txt +17 -0
  370. itn/hindi_english/test/data/test_cases_word.txt +19 -0
  371. itn/hindi_english/test/normalizer_test.py +211 -0
  372. itn/hungarian/__init__.py +0 -0
  373. itn/hungarian/data/dates/days.tsv +31 -0
  374. itn/hungarian/data/dates/months.tsv +12 -0
  375. itn/hungarian/data/electronic/domain.tsv +15 -0
  376. itn/hungarian/data/electronic/server_name.tsv +15 -0
  377. itn/hungarian/data/electronic/symbols.tsv +23 -0
  378. itn/hungarian/data/inflection/endings.tsv +51 -0
  379. itn/hungarian/data/inflection/plural_endings.tsv +39 -0
  380. itn/hungarian/data/inflection/word_endings.tsv +1 -0
  381. itn/hungarian/data/measures/measurements.tsv +52 -0
  382. itn/hungarian/data/money/currency.tsv +11 -0
  383. itn/hungarian/data/money/currency_minor.tsv +11 -0
  384. itn/hungarian/data/number/digit.tsv +9 -0
  385. itn/hungarian/data/number/digit_inline.tsv +9 -0
  386. itn/hungarian/data/number/quantities.tsv +21 -0
  387. itn/hungarian/data/number/tens.tsv +9 -0
  388. itn/hungarian/data/number/tens_inline.tsv +9 -0
  389. itn/hungarian/data/number/zero.tsv +1 -0
  390. itn/hungarian/data/ordinals/endings.tsv +23 -0
  391. itn/hungarian/data/ordinals/exceptional.tsv +4 -0
  392. itn/hungarian/data/telephone/area_codes.tsv +71 -0
  393. itn/hungarian/data/telephone/country_codes.tsv +213 -0
  394. itn/hungarian/data/time/time_zone.tsv +9 -0
  395. itn/hungarian/data/whitelist.tsv +24 -0
  396. itn/hungarian/data/whitelist_inflect.tsv +39 -0
  397. itn/hungarian/data/whitelist_inflect_sg.tsv +14 -0
  398. itn/hungarian/inverse_normalizer.py +93 -0
  399. itn/hungarian/rules/__init__.py +0 -0
  400. itn/hungarian/rules/cardinal.py +99 -0
  401. itn/hungarian/rules/date.py +66 -0
  402. itn/hungarian/rules/decimal.py +80 -0
  403. itn/hungarian/rules/electronic.py +71 -0
  404. itn/hungarian/rules/fraction.py +41 -0
  405. itn/hungarian/rules/measure.py +74 -0
  406. itn/hungarian/rules/money.py +82 -0
  407. itn/hungarian/rules/ordinal.py +42 -0
  408. itn/hungarian/rules/punctuation.py +44 -0
  409. itn/hungarian/rules/telephone.py +68 -0
  410. itn/hungarian/rules/time.py +99 -0
  411. itn/hungarian/rules/utils.py +143 -0
  412. itn/hungarian/rules/whitelist.py +50 -0
  413. itn/hungarian/rules/word.py +38 -0
  414. itn/hungarian/test/__init__.py +0 -0
  415. itn/hungarian/test/normalizer_test.py +214 -0
  416. itn/italian/__init__.py +0 -0
  417. itn/italian/data/electronic/domain.tsv +14 -0
  418. itn/italian/data/electronic/server_name.tsv +11 -0
  419. itn/italian/data/electronic/symbols.tsv +22 -0
  420. itn/italian/data/measure/denominators.tsv +4 -0
  421. itn/italian/data/measure/units.tsv +51 -0
  422. itn/italian/data/money/currency.tsv +11 -0
  423. itn/italian/data/numbers/digit.tsv +9 -0
  424. itn/italian/data/numbers/hundreds.tsv +9 -0
  425. itn/italian/data/numbers/quantities.tsv +4 -0
  426. itn/italian/data/numbers/teen.tsv +10 -0
  427. itn/italian/data/numbers/tens.tsv +8 -0
  428. itn/italian/data/whitelist.tsv +19 -0
  429. itn/italian/inverse_normalizer.py +77 -0
  430. itn/italian/rules/__init__.py +0 -0
  431. itn/italian/rules/cardinal.py +100 -0
  432. itn/italian/rules/decimal.py +59 -0
  433. itn/italian/rules/electronic.py +70 -0
  434. itn/italian/rules/measure.py +52 -0
  435. itn/italian/rules/money.py +78 -0
  436. itn/italian/rules/punctuation.py +27 -0
  437. itn/italian/rules/time.py +84 -0
  438. itn/italian/rules/utils.py +49 -0
  439. itn/italian/rules/whitelist.py +31 -0
  440. itn/italian/rules/word.py +27 -0
  441. itn/italian/test/__init__.py +0 -0
  442. itn/italian/test/normalizer_test.py +277 -0
  443. itn/japanese/inverse_normalizer.py +62 -47
  444. itn/japanese/rules/cardinal.py +40 -55
  445. itn/japanese/rules/char.py +7 -5
  446. itn/japanese/rules/date.py +22 -25
  447. itn/japanese/rules/fraction.py +21 -19
  448. itn/japanese/rules/math.py +9 -8
  449. itn/japanese/rules/measure.py +14 -13
  450. itn/japanese/rules/money.py +10 -7
  451. itn/japanese/rules/ordinal.py +7 -5
  452. itn/japanese/rules/postprocessor.py +5 -5
  453. itn/japanese/rules/preprocessor.py +2 -2
  454. itn/japanese/rules/time.py +16 -17
  455. itn/japanese/rules/whitelist.py +9 -7
  456. itn/japanese/test/normalizer_test.py +132 -30
  457. itn/kinyarwanda/__init__.py +0 -0
  458. itn/kinyarwanda/data/numbers/digit.tsv +9 -0
  459. itn/kinyarwanda/data/numbers/digit_bi.tsv +9 -0
  460. itn/kinyarwanda/data/numbers/digit_e.tsv +9 -0
  461. itn/kinyarwanda/data/numbers/hundreds.tsv +9 -0
  462. itn/kinyarwanda/data/numbers/tens.tsv +8 -0
  463. itn/kinyarwanda/data/numbers/zero.tsv +1 -0
  464. itn/kinyarwanda/data/time/hours.tsv +12 -0
  465. itn/kinyarwanda/data/time/minutes.tsv +59 -0
  466. itn/kinyarwanda/data/whitelist.tsv +18 -0
  467. itn/kinyarwanda/inverse_normalizer.py +53 -0
  468. itn/kinyarwanda/rules/__init__.py +0 -0
  469. itn/kinyarwanda/rules/cardinal.py +131 -0
  470. itn/kinyarwanda/rules/punctuation.py +28 -0
  471. itn/kinyarwanda/rules/time.py +39 -0
  472. itn/kinyarwanda/rules/utils.py +39 -0
  473. itn/kinyarwanda/rules/whitelist.py +32 -0
  474. itn/kinyarwanda/rules/word.py +28 -0
  475. itn/kinyarwanda/test/__init__.py +0 -0
  476. itn/kinyarwanda/test/normalizer_test.py +194 -0
  477. itn/korean/__init__.py +3 -0
  478. itn/korean/data/__init__.py +13 -0
  479. itn/korean/data/currency.tsv +9 -0
  480. itn/korean/data/measure_units.tsv +61 -0
  481. itn/korean/data/months.tsv +12 -0
  482. itn/korean/data/numbers/__init__.py +13 -0
  483. itn/korean/data/numbers/digit.tsv +9 -0
  484. itn/korean/data/numbers/zero.tsv +1 -0
  485. itn/korean/data/ordinals/__init__.py +13 -0
  486. itn/korean/data/ordinals/counter_suffix.tsv +17 -0
  487. itn/korean/data/ordinals/digit.tsv +9 -0
  488. itn/korean/data/time/__init__.py +13 -0
  489. itn/korean/data/time/ten_prefix.tsv +4 -0
  490. itn/korean/data/time/time_hours.tsv +23 -0
  491. itn/korean/data/whitelist.tsv +35 -0
  492. itn/korean/inverse_normalizer.py +76 -0
  493. itn/korean/rules/__init__.py +3 -0
  494. itn/korean/rules/cardinal.py +44 -0
  495. itn/korean/rules/date.py +42 -0
  496. itn/korean/rules/decimal.py +43 -0
  497. itn/korean/rules/fraction.py +48 -0
  498. itn/korean/rules/measure.py +52 -0
  499. itn/korean/rules/money.py +34 -0
  500. itn/korean/rules/ordinal.py +49 -0
  501. itn/korean/rules/telephone.py +30 -0
  502. itn/korean/rules/time.py +61 -0
  503. itn/korean/rules/utils.py +22 -0
  504. tn/chinese/rules/preprocessor.py → itn/korean/rules/whitelist.py +10 -13
  505. itn/korean/rules/word.py +27 -0
  506. itn/korean/test/__init__.py +0 -0
  507. itn/korean/test/data/test_cases_cardinal.txt +37 -0
  508. itn/korean/test/data/test_cases_date.txt +15 -0
  509. itn/korean/test/data/test_cases_decimal.txt +14 -0
  510. itn/korean/test/data/test_cases_fraction.txt +29 -0
  511. itn/korean/test/data/test_cases_measure.txt +46 -0
  512. itn/korean/test/data/test_cases_money.txt +29 -0
  513. itn/korean/test/data/test_cases_ordinal.txt +17 -0
  514. itn/korean/test/data/test_cases_telephone.txt +7 -0
  515. itn/korean/test/data/test_cases_time.txt +15 -0
  516. itn/korean/test/test_inverse_normalizer.py +184 -0
  517. itn/main.py +2 -51
  518. itn/marathi/__init__.py +0 -0
  519. itn/marathi/data/date/eras.tsv +2 -0
  520. itn/marathi/data/date/months.tsv +12 -0
  521. itn/marathi/data/numbers/digit.tsv +9 -0
  522. itn/marathi/data/numbers/tens.tsv +90 -0
  523. itn/marathi/data/numbers/zero.tsv +1 -0
  524. itn/marathi/inverse_normalizer.py +56 -0
  525. itn/marathi/rules/__init__.py +0 -0
  526. itn/marathi/rules/cardinal.py +76 -0
  527. itn/marathi/rules/date.py +63 -0
  528. itn/marathi/rules/decimal.py +62 -0
  529. itn/marathi/rules/punctuation.py +29 -0
  530. itn/marathi/rules/time.py +69 -0
  531. itn/marathi/rules/utils.py +62 -0
  532. itn/marathi/rules/word.py +28 -0
  533. itn/marathi/test/__init__.py +0 -0
  534. itn/marathi/test/data/test_cases_cardinal.txt +33 -0
  535. itn/marathi/test/data/test_cases_date.txt +29 -0
  536. itn/marathi/test/data/test_cases_decimal.txt +29 -0
  537. itn/marathi/test/data/test_cases_time.txt +29 -0
  538. itn/marathi/test/data/test_cases_word.txt +11 -0
  539. itn/marathi/test/normalizer_test.py +164 -0
  540. itn/portuguese/__init__.py +0 -0
  541. itn/portuguese/data/currency_plural.tsv +5 -0
  542. itn/portuguese/data/currency_singular.tsv +5 -0
  543. itn/portuguese/data/electronic/domain.tsv +26 -0
  544. itn/portuguese/data/electronic/server_name.tsv +11 -0
  545. itn/portuguese/data/electronic/symbols.tsv +6 -0
  546. itn/portuguese/data/measurements_plural.tsv +56 -0
  547. itn/portuguese/data/measurements_singular.tsv +55 -0
  548. itn/portuguese/data/months.tsv +12 -0
  549. itn/portuguese/data/numbers/digit.tsv +11 -0
  550. itn/portuguese/data/numbers/hundreds.tsv +17 -0
  551. itn/portuguese/data/numbers/onehundred.tsv +1 -0
  552. itn/portuguese/data/numbers/teen.tsv +11 -0
  553. itn/portuguese/data/numbers/ties.tsv +8 -0
  554. itn/portuguese/data/numbers/twenties.tsv +9 -0
  555. itn/portuguese/data/numbers/zero.tsv +1 -0
  556. itn/portuguese/data/ordinals/digit.tsv +18 -0
  557. itn/portuguese/data/ordinals/hundreds.tsv +28 -0
  558. itn/portuguese/data/ordinals/ties.tsv +20 -0
  559. itn/portuguese/data/time/hours_to.tsv +23 -0
  560. itn/portuguese/data/time/minutes_to.tsv +59 -0
  561. itn/portuguese/data/time/time_suffix_am.tsv +2 -0
  562. itn/portuguese/data/time/time_suffix_pm.tsv +2 -0
  563. itn/portuguese/data/whitelist.tsv +5 -0
  564. itn/portuguese/inverse_normalizer.py +69 -0
  565. itn/portuguese/rules/__init__.py +0 -0
  566. itn/portuguese/rules/cardinal.py +117 -0
  567. itn/portuguese/rules/date.py +64 -0
  568. itn/portuguese/rules/decimal.py +76 -0
  569. itn/portuguese/rules/electronic.py +69 -0
  570. itn/portuguese/rules/measure.py +61 -0
  571. itn/portuguese/rules/money.py +89 -0
  572. itn/portuguese/rules/ordinal.py +56 -0
  573. itn/portuguese/rules/punctuation.py +32 -0
  574. itn/portuguese/rules/telephone.py +56 -0
  575. itn/portuguese/rules/time.py +107 -0
  576. itn/portuguese/rules/whitelist.py +32 -0
  577. itn/portuguese/rules/word.py +31 -0
  578. itn/portuguese/test/__init__.py +0 -0
  579. itn/portuguese/test/data/test_cases_cardinal.txt +70 -0
  580. itn/portuguese/test/data/test_cases_date.txt +15 -0
  581. itn/portuguese/test/data/test_cases_decimal.txt +26 -0
  582. itn/portuguese/test/data/test_cases_electronic.txt +13 -0
  583. itn/portuguese/test/data/test_cases_measure.txt +12 -0
  584. itn/portuguese/test/data/test_cases_money.txt +25 -0
  585. itn/portuguese/test/data/test_cases_ordinal.txt +19 -0
  586. itn/portuguese/test/data/test_cases_telephone.txt +27 -0
  587. itn/portuguese/test/data/test_cases_time.txt +19 -0
  588. itn/portuguese/test/data/test_cases_whitelist.txt +6 -0
  589. itn/portuguese/test/data/test_cases_word.txt +49 -0
  590. itn/portuguese/test/normalizer_test.py +156 -0
  591. itn/russian/__init__.py +0 -0
  592. itn/russian/data/currency/currency_plural.tsv +53 -0
  593. itn/russian/data/currency/currency_singular.tsv +48 -0
  594. itn/russian/data/electronic/domain.tsv +9 -0
  595. itn/russian/data/electronic/server_name.tsv +17 -0
  596. itn/russian/data/electronic/symbols.tsv +28 -0
  597. itn/russian/data/latin_to_cyrillic.tsv +38 -0
  598. itn/russian/data/measurements.tsv +321 -0
  599. itn/russian/data/months/abbr.tsv +12 -0
  600. itn/russian/data/months/abbr_to_name.tsv +62 -0
  601. itn/russian/data/months/numbers.tsv +21 -0
  602. itn/russian/data/numbers/1_cardinals_nominative.tsv +50 -0
  603. itn/russian/data/numbers/2_cardinals_genitive.tsv +45 -0
  604. itn/russian/data/numbers/3_cardinals_dative.tsv +45 -0
  605. itn/russian/data/numbers/4_cardinals_accusative.tsv +48 -0
  606. itn/russian/data/numbers/5_cardinals_instrumental.tsv +47 -0
  607. itn/russian/data/numbers/6_cardinals_prepositional.tsv +46 -0
  608. itn/russian/data/numbers/cardinal_endings.tsv +6 -0
  609. itn/russian/data/numbers/cardinals_alternatives.tsv +16 -0
  610. itn/russian/data/numbers/cardinals_nominative_case.tsv +37 -0
  611. itn/russian/data/numbers/decimal_delimiter.tsv +15 -0
  612. itn/russian/data/numbers/decimal_endings.tsv +37 -0
  613. itn/russian/data/numbers/digits_nominative_case.tsv +10 -0
  614. itn/russian/data/numbers/ordinal_endings.tsv +40 -0
  615. itn/russian/data/numbers/ordinals.tsv +752 -0
  616. itn/russian/data/numbers/quantity.tsv +16 -0
  617. itn/russian/data/time/increment_hour_cardinal.tsv +24 -0
  618. itn/russian/data/time/increment_hour_ordinal.tsv +24 -0
  619. itn/russian/data/time/minutes_to_hour.tsv +25 -0
  620. itn/russian/data/time/time_convert.tsv +12 -0
  621. itn/russian/data/whitelist.tsv +37 -0
  622. itn/russian/inverse_normalizer.py +71 -0
  623. itn/russian/rules/__init__.py +0 -0
  624. itn/russian/rules/cardinal.py +57 -0
  625. itn/russian/rules/date.py +78 -0
  626. itn/russian/rules/decimal.py +55 -0
  627. itn/russian/rules/electronic.py +67 -0
  628. itn/russian/rules/measure.py +45 -0
  629. itn/russian/rules/money.py +53 -0
  630. itn/russian/rules/ordinal.py +40 -0
  631. itn/russian/rules/punctuation.py +40 -0
  632. itn/russian/rules/telephone.py +45 -0
  633. itn/russian/rules/time.py +83 -0
  634. itn/russian/rules/utils.py +151 -0
  635. itn/russian/rules/whitelist.py +29 -0
  636. itn/russian/rules/word.py +30 -0
  637. itn/russian/test/__init__.py +0 -0
  638. itn/russian/test/data/test_cases_cardinal.txt +30 -0
  639. itn/russian/test/data/test_cases_date.txt +15 -0
  640. itn/russian/test/data/test_cases_decimal.txt +16 -0
  641. itn/russian/test/data/test_cases_electronic.txt +4 -0
  642. itn/russian/test/data/test_cases_measure.txt +16 -0
  643. itn/russian/test/data/test_cases_money.txt +9 -0
  644. itn/russian/test/data/test_cases_ordinal.txt +15 -0
  645. itn/russian/test/data/test_cases_ordinal_hard.txt +57 -0
  646. itn/russian/test/data/test_cases_telephone.txt +1 -0
  647. itn/russian/test/data/test_cases_time.txt +9 -0
  648. itn/russian/test/data/test_cases_whitelist.txt +4 -0
  649. itn/russian/test/data/test_cases_word.txt +50 -0
  650. itn/russian/test/normalizer_test.py +193 -0
  651. itn/spanish/__init__.py +0 -0
  652. itn/spanish/data/dates/months.tsv +12 -0
  653. itn/spanish/data/dates/months_cased.tsv +12 -0
  654. itn/spanish/data/dates/year_suffix.tsv +11 -0
  655. itn/spanish/data/dates/year_suffix_cased.tsv +11 -0
  656. itn/spanish/data/electronic/domain.tsv +27 -0
  657. itn/spanish/data/electronic/server_name.tsv +17 -0
  658. itn/spanish/data/electronic/symbols.tsv +22 -0
  659. itn/spanish/data/fractions/numbers_read_as_ordinals.tsv +28 -0
  660. itn/spanish/data/fractions/ordinal_exceptions.tsv +2 -0
  661. itn/spanish/data/measures/math_symbols.tsv +4 -0
  662. itn/spanish/data/measures/measurements_plural.tsv +34 -0
  663. itn/spanish/data/measures/measurements_singular.tsv +35 -0
  664. itn/spanish/data/money/currency_major_plural.tsv +43 -0
  665. itn/spanish/data/money/currency_major_plural_capitalized.tsv +75 -0
  666. itn/spanish/data/money/currency_major_plural_ext.tsv +196 -0
  667. itn/spanish/data/money/currency_major_singular.tsv +43 -0
  668. itn/spanish/data/money/currency_major_singular_capitalized.tsv +76 -0
  669. itn/spanish/data/money/currency_major_singular_ext.tsv +196 -0
  670. itn/spanish/data/money/currency_minor_plural.tsv +5 -0
  671. itn/spanish/data/money/currency_minor_plural_ext.tsv +62 -0
  672. itn/spanish/data/money/currency_minor_singular.tsv +5 -0
  673. itn/spanish/data/money/currency_minor_singular_ext.tsv +64 -0
  674. itn/spanish/data/months.tsv +12 -0
  675. itn/spanish/data/numbers/digit.tsv +11 -0
  676. itn/spanish/data/numbers/hundreds.tsv +18 -0
  677. itn/spanish/data/numbers/teen.tsv +10 -0
  678. itn/spanish/data/numbers/ties.tsv +8 -0
  679. itn/spanish/data/numbers/twenties.tsv +11 -0
  680. itn/spanish/data/numbers/zero.tsv +1 -0
  681. itn/spanish/data/ordinals/digit.tsv +22 -0
  682. itn/spanish/data/ordinals/digit_capitalized.tsv +22 -0
  683. itn/spanish/data/ordinals/hundreds.tsv +18 -0
  684. itn/spanish/data/ordinals/hundreds_capitalized.tsv +18 -0
  685. itn/spanish/data/ordinals/teen.tsv +60 -0
  686. itn/spanish/data/ordinals/teen_capitalized.tsv +60 -0
  687. itn/spanish/data/ordinals/ties.tsv +15 -0
  688. itn/spanish/data/ordinals/ties_capitalized.tsv +15 -0
  689. itn/spanish/data/ordinals/twenties.tsv +50 -0
  690. itn/spanish/data/ordinals/twenties_capitalized.tsv +50 -0
  691. itn/spanish/data/roman/digit.tsv +9 -0
  692. itn/spanish/data/roman/hundreds.tsv +9 -0
  693. itn/spanish/data/roman/thousands.tsv +3 -0
  694. itn/spanish/data/roman/ties.tsv +9 -0
  695. itn/spanish/data/time/minutes_to.tsv +6 -0
  696. itn/spanish/data/time/time_suffix.tsv +15 -0
  697. itn/spanish/data/time/time_suffix_cased.tsv +15 -0
  698. itn/spanish/data/time/time_to.tsv +25 -0
  699. itn/spanish/data/time/time_zone.tsv +42 -0
  700. itn/spanish/data/time/time_zone_cased.tsv +42 -0
  701. itn/spanish/data/time/time_zone_ext.tsv +105 -0
  702. itn/spanish/data/whitelist.tsv +47 -0
  703. itn/spanish/inverse_normalizer.py +77 -0
  704. itn/spanish/rules/__init__.py +0 -0
  705. itn/spanish/rules/cardinal.py +98 -0
  706. itn/spanish/rules/date.py +83 -0
  707. itn/spanish/rules/decimal.py +64 -0
  708. itn/spanish/rules/electronic.py +94 -0
  709. itn/spanish/rules/fraction.py +77 -0
  710. itn/spanish/rules/measure.py +84 -0
  711. itn/spanish/rules/money.py +95 -0
  712. itn/spanish/rules/ordinal.py +53 -0
  713. itn/spanish/rules/punctuation.py +30 -0
  714. itn/spanish/rules/telephone.py +70 -0
  715. itn/spanish/rules/time.py +112 -0
  716. itn/spanish/rules/utils.py +102 -0
  717. itn/spanish/rules/whitelist.py +47 -0
  718. itn/spanish/rules/word.py +32 -0
  719. itn/spanish/test/__init__.py +0 -0
  720. itn/spanish/test/data/test_cases_cardinal.txt +51 -0
  721. itn/spanish/test/data/test_cases_cardinal_cased.txt +30 -0
  722. itn/spanish/test/data/test_cases_date.txt +8 -0
  723. itn/spanish/test/data/test_cases_date_cased.txt +8 -0
  724. itn/spanish/test/data/test_cases_decimal.txt +29 -0
  725. itn/spanish/test/data/test_cases_decimal_cased.txt +6 -0
  726. itn/spanish/test/data/test_cases_electronic.txt +16 -0
  727. itn/spanish/test/data/test_cases_electronic_cased.txt +5 -0
  728. itn/spanish/test/data/test_cases_fraction.txt +12 -0
  729. itn/spanish/test/data/test_cases_measure.txt +20 -0
  730. itn/spanish/test/data/test_cases_measure_cased.txt +11 -0
  731. itn/spanish/test/data/test_cases_money.txt +24 -0
  732. itn/spanish/test/data/test_cases_money_cased.txt +6 -0
  733. itn/spanish/test/data/test_cases_ordinal.txt +30 -0
  734. itn/spanish/test/data/test_cases_ordinal_cased.txt +11 -0
  735. itn/spanish/test/data/test_cases_telephone.txt +9 -0
  736. itn/spanish/test/data/test_cases_telephone_cased.txt +6 -0
  737. itn/spanish/test/data/test_cases_time.txt +25 -0
  738. itn/spanish/test/data/test_cases_time_cased.txt +9 -0
  739. itn/spanish/test/data/test_cases_whitelist.txt +5 -0
  740. itn/spanish/test/data/test_cases_word.txt +49 -0
  741. itn/spanish/test/data/test_cases_word_cased.txt +11 -0
  742. itn/spanish/test/normalizer_test.py +143 -0
  743. itn/spanish_english/__init__.py +0 -0
  744. itn/spanish_english/data/en_whitelist.tsv +416 -0
  745. itn/spanish_english/data/es_whitelist.tsv +16 -0
  746. itn/spanish_english/data/measurements.tsv +3 -0
  747. itn/spanish_english/inverse_normalizer.py +99 -0
  748. itn/spanish_english/rules/__init__.py +0 -0
  749. itn/spanish_english/rules/english.py +230 -0
  750. itn/spanish_english/rules/prefixed.py +34 -0
  751. itn/spanish_english/rules/time.py +39 -0
  752. itn/spanish_english/rules/whitelist.py +38 -0
  753. itn/spanish_english/test/__init__.py +0 -0
  754. itn/spanish_english/test/data/test_cases_cardinal.txt +80 -0
  755. itn/spanish_english/test/data/test_cases_cardinal_cased.txt +60 -0
  756. itn/spanish_english/test/data/test_cases_date.txt +44 -0
  757. itn/spanish_english/test/data/test_cases_date_cased.txt +20 -0
  758. itn/spanish_english/test/data/test_cases_decimal.txt +92 -0
  759. itn/spanish_english/test/data/test_cases_decimal_cased.txt +19 -0
  760. itn/spanish_english/test/data/test_cases_electronic.txt +41 -0
  761. itn/spanish_english/test/data/test_cases_electronic_cased.txt +14 -0
  762. itn/spanish_english/test/data/test_cases_fraction.txt +12 -0
  763. itn/spanish_english/test/data/test_cases_measure.txt +132 -0
  764. itn/spanish_english/test/data/test_cases_measure_cased.txt +22 -0
  765. itn/spanish_english/test/data/test_cases_money.txt +76 -0
  766. itn/spanish_english/test/data/test_cases_money_cased.txt +16 -0
  767. itn/spanish_english/test/data/test_cases_ordinal.txt +64 -0
  768. itn/spanish_english/test/data/test_cases_ordinal_cased.txt +21 -0
  769. itn/spanish_english/test/data/test_cases_telephone.txt +32 -0
  770. itn/spanish_english/test/data/test_cases_telephone_cased.txt +19 -0
  771. itn/spanish_english/test/data/test_cases_time.txt +54 -0
  772. itn/spanish_english/test/data/test_cases_time_cased.txt +19 -0
  773. itn/spanish_english/test/data/test_cases_whitelist.txt +18 -0
  774. itn/spanish_english/test/data/test_cases_word.txt +104 -0
  775. itn/spanish_english/test/data/test_cases_word_cased.txt +17 -0
  776. itn/spanish_english/test/normalizer_test.py +207 -0
  777. itn/swedish/__init__.py +0 -0
  778. itn/swedish/data/dates/era_words.tsv +4 -0
  779. itn/swedish/data/dates/month_abbr.tsv +17 -0
  780. itn/swedish/data/dates/months.tsv +12 -0
  781. itn/swedish/data/electronic/domain.tsv +10 -0
  782. itn/swedish/data/electronic/server_name.tsv +29 -0
  783. itn/swedish/data/electronic/symbols.tsv +20 -0
  784. itn/swedish/data/numbers/digit.tsv +9 -0
  785. itn/swedish/data/numbers/millions.tsv +88 -0
  786. itn/swedish/data/numbers/teen.tsv +10 -0
  787. itn/swedish/data/numbers/ties.tsv +8 -0
  788. itn/swedish/data/numbers/zero.tsv +1 -0
  789. itn/swedish/data/ordinals/digit.tsv +9 -0
  790. itn/swedish/data/ordinals/teen.tsv +10 -0
  791. itn/swedish/data/ordinals/ties.tsv +8 -0
  792. itn/swedish/data/time/suffix.tsv +2 -0
  793. itn/swedish/data/time/time_zone.tsv +7 -0
  794. itn/swedish/data/whitelist.tsv +4 -0
  795. itn/swedish/inverse_normalizer.py +67 -0
  796. itn/swedish/rules/__init__.py +0 -0
  797. itn/swedish/rules/cardinal.py +88 -0
  798. itn/swedish/rules/date.py +91 -0
  799. itn/swedish/rules/decimal.py +68 -0
  800. itn/swedish/rules/electronic.py +69 -0
  801. itn/swedish/rules/fraction.py +90 -0
  802. itn/swedish/rules/ordinal.py +85 -0
  803. itn/swedish/rules/punctuation.py +29 -0
  804. itn/swedish/rules/telephone.py +59 -0
  805. itn/swedish/rules/time.py +99 -0
  806. itn/swedish/rules/utils.py +55 -0
  807. itn/swedish/rules/whitelist.py +29 -0
  808. itn/swedish/rules/word.py +29 -0
  809. itn/swedish/test/__init__.py +0 -0
  810. itn/swedish/test/data/test_cases_cardinal.txt +24 -0
  811. itn/swedish/test/data/test_cases_date.txt +9 -0
  812. itn/swedish/test/data/test_cases_decimal.txt +15 -0
  813. itn/swedish/test/data/test_cases_electronic.txt +12 -0
  814. itn/swedish/test/data/test_cases_fraction.txt +3 -0
  815. itn/swedish/test/data/test_cases_ordinal.txt +7 -0
  816. itn/swedish/test/data/test_cases_telephone.txt +2 -0
  817. itn/swedish/test/data/test_cases_time.txt +14 -0
  818. itn/swedish/test/data/test_cases_whitelist.txt +1 -0
  819. itn/swedish/test/data/test_cases_word.txt +48 -0
  820. itn/swedish/test/normalizer_test.py +122 -0
  821. itn/test/utils_test.py +17 -0
  822. itn/utils.py +40 -0
  823. itn/vietnamese/__init__.py +0 -0
  824. itn/vietnamese/data/currency.tsv +11 -0
  825. itn/vietnamese/data/electronic/domain.tsv +11 -0
  826. itn/vietnamese/data/electronic/server_name.tsv +16 -0
  827. itn/vietnamese/data/electronic/symbols.tsv +21 -0
  828. itn/vietnamese/data/electronic/url_symbols.tsv +8 -0
  829. itn/vietnamese/data/magnitudes.tsv +5 -0
  830. itn/vietnamese/data/math/symbols.tsv +10 -0
  831. itn/vietnamese/data/measurements.tsv +185 -0
  832. itn/vietnamese/data/months.tsv +14 -0
  833. itn/vietnamese/data/numbers/digit.tsv +10 -0
  834. itn/vietnamese/data/numbers/digit_special.tsv +3 -0
  835. itn/vietnamese/data/numbers/hundred.tsv +1 -0
  836. itn/vietnamese/data/numbers/teen.tsv +11 -0
  837. itn/vietnamese/data/numbers/thousands.tsv +6 -0
  838. itn/vietnamese/data/numbers/ties.tsv +9 -0
  839. itn/vietnamese/data/numbers/zero.tsv +1 -0
  840. itn/vietnamese/data/ordinals/digit.tsv +12 -0
  841. itn/vietnamese/data/time/hours.tsv +29 -0
  842. itn/vietnamese/data/time/hours_to.tsv +25 -0
  843. itn/vietnamese/data/time/hours_to_night.tsv +12 -0
  844. itn/vietnamese/data/time/minutes.tsv +118 -0
  845. itn/vietnamese/data/time/minutes_to.tsv +59 -0
  846. itn/vietnamese/data/time/time_suffix.tsv +8 -0
  847. itn/vietnamese/data/time/time_zone.tsv +7 -0
  848. itn/vietnamese/data/time/zero_prefix.tsv +3 -0
  849. itn/vietnamese/data/whitelist.tsv +0 -0
  850. itn/vietnamese/inverse_normalizer.py +73 -0
  851. itn/vietnamese/rules/__init__.py +0 -0
  852. itn/vietnamese/rules/cardinal.py +99 -0
  853. itn/vietnamese/rules/date.py +76 -0
  854. itn/vietnamese/rules/decimal.py +72 -0
  855. itn/vietnamese/rules/electronic.py +60 -0
  856. itn/vietnamese/rules/fraction.py +41 -0
  857. itn/vietnamese/rules/measure.py +60 -0
  858. itn/vietnamese/rules/money.py +54 -0
  859. itn/vietnamese/rules/ordinal.py +32 -0
  860. itn/vietnamese/rules/punctuation.py +29 -0
  861. itn/vietnamese/rules/telephone.py +57 -0
  862. itn/vietnamese/rules/time.py +83 -0
  863. itn/vietnamese/rules/whitelist.py +30 -0
  864. itn/vietnamese/rules/word.py +28 -0
  865. itn/vietnamese/test/__init__.py +0 -0
  866. itn/vietnamese/test/data/test_cases_cardinal.txt +142 -0
  867. itn/vietnamese/test/data/test_cases_date.txt +69 -0
  868. itn/vietnamese/test/data/test_cases_decimal.txt +74 -0
  869. itn/vietnamese/test/data/test_cases_electronic.txt +18 -0
  870. itn/vietnamese/test/data/test_cases_fraction.txt +14 -0
  871. itn/vietnamese/test/data/test_cases_measure.txt +35 -0
  872. itn/vietnamese/test/data/test_cases_money.txt +15 -0
  873. itn/vietnamese/test/data/test_cases_ordinal.txt +12 -0
  874. itn/vietnamese/test/data/test_cases_telephone.txt +25 -0
  875. itn/vietnamese/test/data/test_cases_time.txt +31 -0
  876. itn/vietnamese/test/data/test_cases_whitelist.txt +1 -0
  877. itn/vietnamese/test/data/test_cases_word.txt +17 -0
  878. itn/vietnamese/test/normalizer_test.py +144 -0
  879. tn/README.md +20 -8
  880. tn/__init__.py +3 -0
  881. tn/_version.py +9 -19
  882. tn/alignment.py +254 -0
  883. tn/arabic/__init__.py +0 -0
  884. tn/arabic/data/measure/measurements.tsv +14 -0
  885. tn/arabic/data/money/currency_major.tsv +24 -0
  886. tn/arabic/data/money/currency_minor_plural.tsv +4 -0
  887. tn/arabic/data/money/currency_minor_singular.tsv +3 -0
  888. tn/arabic/data/money/local_currency.tsv +20 -0
  889. tn/arabic/data/number/digit.tsv +9 -0
  890. tn/arabic/data/number/digit_100.tsv +7 -0
  891. tn/arabic/data/number/digit_1000.tsv +7 -0
  892. tn/arabic/data/number/flops.tsv +72 -0
  893. tn/arabic/data/number/fraction.tsv +18 -0
  894. tn/arabic/data/number/fraction_dual.tsv +9 -0
  895. tn/arabic/data/number/fraction_plural.tsv +8 -0
  896. tn/arabic/data/number/fraction_singular.tsv +9 -0
  897. tn/arabic/data/number/quantities.tsv +6 -0
  898. tn/arabic/data/number/teens.tsv +9 -0
  899. tn/arabic/data/number/tens.tsv +10 -0
  900. tn/arabic/data/punctuation/symbol.tsv +23 -0
  901. tn/arabic/normalizer.py +61 -0
  902. tn/arabic/rules/__init__.py +0 -0
  903. tn/arabic/rules/cardinal.py +86 -0
  904. tn/arabic/rules/decimal.py +69 -0
  905. tn/arabic/rules/fraction.py +79 -0
  906. tn/arabic/rules/measure.py +80 -0
  907. tn/arabic/rules/money.py +131 -0
  908. tn/arabic/rules/punctuation.py +44 -0
  909. tn/arabic/rules/word.py +32 -0
  910. tn/arabic/test/__init__.py +0 -0
  911. tn/arabic/test/data/test_cases_cardinal.txt +27 -0
  912. tn/arabic/test/data/test_cases_decimal.txt +30 -0
  913. tn/arabic/test/data/test_cases_fraction.txt +20 -0
  914. tn/arabic/test/data/test_cases_measure.txt +14 -0
  915. tn/arabic/test/data/test_cases_money.txt +27 -0
  916. tn/arabic/test/data/test_cases_whitelist.txt +6 -0
  917. tn/arabic/test/normalizer_test.py +162 -0
  918. tn/armenian/__init__.py +0 -0
  919. tn/armenian/data/currency.tsv +52 -0
  920. tn/armenian/data/measurement_dates.tsv +8 -0
  921. tn/armenian/data/measurements.tsv +44 -0
  922. tn/armenian/data/numbers/digit.tsv +9 -0
  923. tn/armenian/data/numbers/quantities.tsv +3 -0
  924. tn/armenian/data/numbers/ties.tsv +8 -0
  925. tn/armenian/data/time/hours.tsv +26 -0
  926. tn/armenian/data/time/minutes.tsv +61 -0
  927. tn/armenian/data/whitelist.tsv +14 -0
  928. tn/armenian/normalizer.py +65 -0
  929. tn/armenian/rules/__init__.py +0 -0
  930. tn/armenian/rules/cardinal.py +75 -0
  931. tn/armenian/rules/decimal.py +63 -0
  932. tn/armenian/rules/fraction.py +36 -0
  933. tn/armenian/rules/measure.py +49 -0
  934. tn/armenian/rules/money.py +38 -0
  935. tn/armenian/rules/ordinal.py +59 -0
  936. tn/armenian/rules/punctuation.py +31 -0
  937. tn/armenian/rules/time.py +34 -0
  938. tn/armenian/rules/utils.py +60 -0
  939. tn/armenian/rules/whitelist.py +31 -0
  940. tn/armenian/rules/word.py +28 -0
  941. tn/armenian/test/__init__.py +0 -0
  942. tn/armenian/test/data/test_cases_cardinal.txt +12 -0
  943. tn/armenian/test/data/test_cases_decimal.txt +9 -0
  944. tn/armenian/test/data/test_cases_fraction.txt +10 -0
  945. tn/armenian/test/data/test_cases_measure.txt +9 -0
  946. tn/armenian/test/data/test_cases_money.txt +10 -0
  947. tn/armenian/test/data/test_cases_ordinal.txt +11 -0
  948. tn/armenian/test/data/test_cases_time.txt +9 -0
  949. tn/armenian/test/data/test_cases_whitelist.txt +14 -0
  950. tn/armenian/test/data/test_cases_word.txt +50 -0
  951. tn/armenian/test/normalizer_test.py +156 -0
  952. tn/cache.py +822 -0
  953. tn/chinese/data/cardinal/phone.tsv +28 -0
  954. tn/chinese/data/math/operator_unambiguous.tsv +6 -0
  955. tn/chinese/normalizer.py +45 -37
  956. tn/chinese/rules/cardinal.py +21 -19
  957. tn/chinese/rules/date.py +26 -18
  958. tn/chinese/rules/fraction.py +4 -12
  959. tn/chinese/rules/math.py +10 -5
  960. tn/chinese/rules/measure.py +18 -10
  961. tn/chinese/rules/money.py +5 -11
  962. tn/chinese/rules/postprocessor.py +23 -8
  963. tn/chinese/rules/range.py +29 -0
  964. tn/chinese/rules/sport.py +8 -6
  965. tn/chinese/rules/time.py +22 -19
  966. tn/chinese/rules/whitelist.py +7 -6
  967. tn/chinese/test/alignment_test.py +154 -0
  968. tn/chinese/test/cache_safety_test.py +505 -0
  969. tn/chinese/test/cache_test.py +192 -0
  970. tn/chinese/test/data/normalizer.txt +0 -2
  971. tn/chinese/test/data/normalizer_tag_oov.txt +2 -0
  972. tn/chinese/test/normalizer_test.py +57 -4
  973. tn/chinese/test/postprocessor_test.py +16 -0
  974. tn/chinese/test/processor_test.py +496 -0
  975. tn/chinese/test/token_parser_test.py +35 -1
  976. tn/cli.py +278 -0
  977. tn/english/data/measure/unit_alternatives.tsv +0 -1
  978. tn/english/normalizer.py +46 -42
  979. tn/english/rules/cardinal.py +37 -61
  980. tn/english/rules/date.py +84 -166
  981. tn/english/rules/decimal.py +32 -79
  982. tn/english/rules/electronic.py +56 -66
  983. tn/english/rules/fraction.py +33 -23
  984. tn/english/rules/measure.py +107 -148
  985. tn/english/rules/money.py +46 -42
  986. tn/english/rules/ordinal.py +14 -30
  987. tn/english/rules/punctuation.py +15 -21
  988. tn/english/rules/range.py +26 -40
  989. tn/english/rules/roman.py +27 -45
  990. tn/english/rules/serial.py +88 -0
  991. tn/english/rules/telephone.py +44 -80
  992. tn/english/rules/time.py +54 -102
  993. tn/english/rules/whitelist.py +9 -19
  994. tn/english/rules/word.py +9 -0
  995. tn/english/test/data/date.txt +1 -1
  996. tn/english/test/data/normalizer.txt +4 -1
  997. tn/english/test/electronic_test.py +27 -0
  998. tn/english/test/nbest_test.py +87 -0
  999. tn/english/test/normalizer_test.py +181 -3
  1000. tn/french/__init__.py +0 -0
  1001. tn/french/data/dates/eras.tsv +8 -0
  1002. tn/french/data/dates/months.tsv +12 -0
  1003. tn/french/data/fractions/irregular_denominators.tsv +7 -0
  1004. tn/french/data/numbers/digit.tsv +9 -0
  1005. tn/french/data/numbers/eighty.tsv +1 -0
  1006. tn/french/data/numbers/ninety.tsv +1 -0
  1007. tn/french/data/numbers/quantities.tsv +10 -0
  1008. tn/french/data/numbers/seventy.tsv +1 -0
  1009. tn/french/data/numbers/teens.tsv +9 -0
  1010. tn/french/data/numbers/ten.tsv +1 -0
  1011. tn/french/data/numbers/tens_simple.tsv +5 -0
  1012. tn/french/data/numbers/zero.tsv +1 -0
  1013. tn/french/data/ordinals/irregular_numbers.tsv +30 -0
  1014. tn/french/data/ordinals/suffixes.tsv +6 -0
  1015. tn/french/data/punctuation/symbol.tsv +23 -0
  1016. tn/french/data/whitelist.tsv +13 -0
  1017. tn/french/normalizer.py +66 -0
  1018. tn/french/rules/__init__.py +0 -0
  1019. tn/french/rules/cardinal.py +110 -0
  1020. tn/french/rules/date.py +68 -0
  1021. tn/french/rules/decimal.py +62 -0
  1022. tn/french/rules/fraction.py +60 -0
  1023. tn/french/rules/ordinal.py +56 -0
  1024. tn/french/rules/punctuation.py +43 -0
  1025. tn/french/rules/whitelist.py +29 -0
  1026. tn/french/rules/word.py +29 -0
  1027. tn/french/test/__init__.py +0 -0
  1028. tn/french/test/data/test_cases_cardinal.txt +14 -0
  1029. tn/french/test/data/test_cases_date.txt +13 -0
  1030. tn/french/test/data/test_cases_decimal.txt +7 -0
  1031. tn/french/test/data/test_cases_fraction.txt +12 -0
  1032. tn/french/test/data/test_cases_ordinal.txt +15 -0
  1033. tn/french/test/data/test_cases_whitelist.txt +7 -0
  1034. tn/french/test/data/test_cases_word.txt +49 -0
  1035. tn/french/test/normalizer_test.py +157 -0
  1036. tn/german/__init__.py +0 -0
  1037. tn/german/data/__init__.py +13 -0
  1038. tn/german/data/electronic/__init__.py +13 -0
  1039. tn/german/data/electronic/domain.tsv +844 -0
  1040. tn/german/data/electronic/server_name.tsv +17 -0
  1041. tn/german/data/electronic/symbols.tsv +20 -0
  1042. tn/german/data/fractions.tsv +30 -0
  1043. tn/german/data/measure/__init__.py +13 -0
  1044. tn/german/data/measure/measurements.tsv +82 -0
  1045. tn/german/data/measure/suppletive.tsv +10 -0
  1046. tn/german/data/money/__init__.py +13 -0
  1047. tn/german/data/money/currency.tsv +25 -0
  1048. tn/german/data/money/currency_minor_plural.tsv +3 -0
  1049. tn/german/data/money/currency_minor_singular.tsv +3 -0
  1050. tn/german/data/months/__init__.py +13 -0
  1051. tn/german/data/months/abbr_to_name.tsv +13 -0
  1052. tn/german/data/months/numbers.tsv +24 -0
  1053. tn/german/data/numbers/__init__.py +13 -0
  1054. tn/german/data/numbers/digit.tsv +8 -0
  1055. tn/german/data/numbers/ones.tsv +3 -0
  1056. tn/german/data/numbers/quantities.tsv +12 -0
  1057. tn/german/data/numbers/teen.tsv +10 -0
  1058. tn/german/data/numbers/ties.tsv +8 -0
  1059. tn/german/data/numbers/zero.tsv +1 -0
  1060. tn/german/data/ordinals/__init__.py +13 -0
  1061. tn/german/data/ordinals/digit.tsv +9 -0
  1062. tn/german/data/ordinals/thousands.tsv +4 -0
  1063. tn/german/data/ordinals/ties.tsv +8 -0
  1064. tn/german/data/time/__init__.py +13 -0
  1065. tn/german/data/time/hour_to.tsv +12 -0
  1066. tn/german/data/time/hour_to_night.tsv +13 -0
  1067. tn/german/data/time/minute_to.tsv +59 -0
  1068. tn/german/data/time/time_zone.tsv +7 -0
  1069. tn/german/data/whitelist.tsv +21 -0
  1070. tn/german/normalizer.py +77 -0
  1071. tn/german/rules/__init__.py +0 -0
  1072. tn/german/rules/cardinal.py +120 -0
  1073. tn/german/rules/date.py +98 -0
  1074. tn/german/rules/decimal.py +71 -0
  1075. tn/german/rules/electronic.py +67 -0
  1076. tn/german/rules/fraction.py +65 -0
  1077. tn/german/rules/measure.py +117 -0
  1078. tn/german/rules/money.py +144 -0
  1079. tn/german/rules/ordinal.py +46 -0
  1080. tn/german/rules/punctuation.py +39 -0
  1081. tn/german/rules/telephone.py +48 -0
  1082. tn/german/rules/time.py +123 -0
  1083. tn/german/rules/whitelist.py +34 -0
  1084. tn/german/rules/word.py +34 -0
  1085. tn/german/test/__init__.py +0 -0
  1086. tn/german/test/calendar_test.py +168 -0
  1087. tn/german/test/data/test_cases_cardinal.txt +50 -0
  1088. tn/german/test/data/test_cases_date.txt +15 -0
  1089. tn/german/test/data/test_cases_decimal.txt +10 -0
  1090. tn/german/test/data/test_cases_electronic.txt +19 -0
  1091. tn/german/test/data/test_cases_fraction.txt +34 -0
  1092. tn/german/test/data/test_cases_measure.txt +29 -0
  1093. tn/german/test/data/test_cases_money.txt +21 -0
  1094. tn/german/test/data/test_cases_normalize_with_audio.txt +131 -0
  1095. tn/german/test/data/test_cases_ordinal.txt +19 -0
  1096. tn/german/test/data/test_cases_telephone.txt +4 -0
  1097. tn/german/test/data/test_cases_time.txt +28 -0
  1098. tn/german/test/data/test_cases_whitelist.txt +6 -0
  1099. tn/german/test/data/test_cases_word.txt +49 -0
  1100. tn/german/test/financial_test.py +167 -0
  1101. tn/german/test/normalizer_test.py +136 -0
  1102. tn/german/test/numeric_test.py +185 -0
  1103. tn/german/test/text_test.py +235 -0
  1104. tn/hebrew/__init__.py +0 -0
  1105. tn/hebrew/data/measurements.tsv +29 -0
  1106. tn/hebrew/data/prefix.tsv +17 -0
  1107. tn/hebrew/data/spaced_measurements.tsv +16 -0
  1108. tn/hebrew/data/whitelist.tsv +9 -0
  1109. tn/hebrew/normalizer.py +58 -0
  1110. tn/hebrew/rules/__init__.py +0 -0
  1111. tn/hebrew/rules/cardinal.py +43 -0
  1112. tn/hebrew/rules/date.py +84 -0
  1113. tn/hebrew/rules/decimal.py +71 -0
  1114. tn/hebrew/rules/measure.py +56 -0
  1115. tn/hebrew/rules/numbers.py +135 -0
  1116. tn/hebrew/rules/punctuation.py +29 -0
  1117. tn/hebrew/rules/time.py +94 -0
  1118. tn/hebrew/rules/utils.py +59 -0
  1119. tn/hebrew/rules/whitelist.py +26 -0
  1120. tn/hebrew/rules/word.py +27 -0
  1121. tn/hebrew/test/__init__.py +0 -0
  1122. tn/hebrew/test/normalizer_test.py +245 -0
  1123. tn/hindi/__init__.py +0 -0
  1124. tn/hindi/data/address/cities.tsv +36 -0
  1125. tn/hindi/data/address/context.tsv +47 -0
  1126. tn/hindi/data/address/letters.tsv +26 -0
  1127. tn/hindi/data/address/special_characters.tsv +2 -0
  1128. tn/hindi/data/address/states.tsv +36 -0
  1129. tn/hindi/data/date/days.tsv +62 -0
  1130. tn/hindi/data/date/months.tsv +24 -0
  1131. tn/hindi/data/date/prefixes.tsv +4 -0
  1132. tn/hindi/data/date/suffixes.tsv +10 -0
  1133. tn/hindi/data/date/year_suffix.tsv +2 -0
  1134. tn/hindi/data/electronic/domain.tsv +24 -0
  1135. tn/hindi/data/electronic/elements.tsv +132 -0
  1136. tn/hindi/data/electronic/file_extensions.tsv +34 -0
  1137. tn/hindi/data/electronic/protocols.tsv +5 -0
  1138. tn/hindi/data/electronic/subscript_digit.tsv +10 -0
  1139. tn/hindi/data/electronic/symbol_classes.tsv +16 -0
  1140. tn/hindi/data/electronic/symbols.tsv +36 -0
  1141. tn/hindi/data/fraction/common_fractions.tsv +10 -0
  1142. tn/hindi/data/measure/quarterly_units_list.tsv +5 -0
  1143. tn/hindi/data/measure/quarterly_units_map.tsv +10 -0
  1144. tn/hindi/data/measure/unit.tsv +146 -0
  1145. tn/hindi/data/measure/unit_year_formal.tsv +1 -0
  1146. tn/hindi/data/money/currency.tsv +9 -0
  1147. tn/hindi/data/money/currency_singular.tsv +9 -0
  1148. tn/hindi/data/money/major_minor_currencies.tsv +10 -0
  1149. tn/hindi/data/numbers/digit.tsv +18 -0
  1150. tn/hindi/data/numbers/hundred.tsv +1 -0
  1151. tn/hindi/data/numbers/teens_and_ties.tsv +90 -0
  1152. tn/hindi/data/numbers/teens_and_ties_en.tsv +90 -0
  1153. tn/hindi/data/numbers/thousands.tsv +8 -0
  1154. tn/hindi/data/numbers/zero.tsv +2 -0
  1155. tn/hindi/data/ordinal/en_to_hi_digit.tsv +10 -0
  1156. tn/hindi/data/ordinal/exceptions.tsv +50 -0
  1157. tn/hindi/data/ordinal/suffixes.tsv +3 -0
  1158. tn/hindi/data/ordinal/suffixes_map.tsv +1 -0
  1159. tn/hindi/data/roman/roman_ordinal_exceptions.tsv +10 -0
  1160. tn/hindi/data/roman/roman_to_spoken.tsv +100 -0
  1161. tn/hindi/data/serial/chars.tsv +68 -0
  1162. tn/hindi/data/serial/power_special.tsv +4 -0
  1163. tn/hindi/data/serial/special_symbols.tsv +4 -0
  1164. tn/hindi/data/telephone/credit_context.tsv +3 -0
  1165. tn/hindi/data/telephone/landline_context.tsv +5 -0
  1166. tn/hindi/data/telephone/mobile_context.tsv +4 -0
  1167. tn/hindi/data/telephone/number.tsv +10 -0
  1168. tn/hindi/data/telephone/pincode_context.tsv +4 -0
  1169. tn/hindi/data/time/hours.tsv +50 -0
  1170. tn/hindi/data/time/minutes.tsv +120 -0
  1171. tn/hindi/data/time/seconds.tsv +120 -0
  1172. tn/hindi/data/whitelist/abbreviations.tsv +7 -0
  1173. tn/hindi/data/whitelist/paune_mappings.tsv +100 -0
  1174. tn/hindi/normalizer.py +95 -0
  1175. tn/hindi/rules/__init__.py +0 -0
  1176. tn/hindi/rules/address.py +97 -0
  1177. tn/hindi/rules/cardinal.py +117 -0
  1178. tn/hindi/rules/date.py +135 -0
  1179. tn/hindi/rules/decimal.py +42 -0
  1180. tn/hindi/rules/electronic.py +145 -0
  1181. tn/hindi/rules/fraction.py +62 -0
  1182. tn/hindi/rules/measure.py +110 -0
  1183. tn/hindi/rules/money.py +104 -0
  1184. tn/hindi/rules/ordinal.py +37 -0
  1185. tn/hindi/rules/punctuation.py +50 -0
  1186. tn/hindi/rules/roman.py +70 -0
  1187. tn/hindi/rules/serial.py +82 -0
  1188. tn/hindi/rules/telephone.py +98 -0
  1189. tn/hindi/rules/time.py +62 -0
  1190. tn/hindi/rules/utils.py +54 -0
  1191. tn/hindi/rules/whitelist.py +37 -0
  1192. tn/hindi/rules/word.py +31 -0
  1193. tn/hindi/test/__init__.py +0 -0
  1194. tn/hindi/test/data/test_cases_address.txt +52 -0
  1195. tn/hindi/test/data/test_cases_cardinal.txt +161 -0
  1196. tn/hindi/test/data/test_cases_date.txt +39 -0
  1197. tn/hindi/test/data/test_cases_decimal.txt +24 -0
  1198. tn/hindi/test/data/test_cases_electronic.txt +65 -0
  1199. tn/hindi/test/data/test_cases_fraction.txt +27 -0
  1200. tn/hindi/test/data/test_cases_measure.txt +69 -0
  1201. tn/hindi/test/data/test_cases_money.txt +144 -0
  1202. tn/hindi/test/data/test_cases_ordinal.txt +72 -0
  1203. tn/hindi/test/data/test_cases_roman.txt +23 -0
  1204. tn/hindi/test/data/test_cases_serial.txt +31 -0
  1205. tn/hindi/test/data/test_cases_telephone.txt +25 -0
  1206. tn/hindi/test/data/test_cases_time.txt +18 -0
  1207. tn/hindi/test/data/test_cases_whitelist.txt +14 -0
  1208. tn/hindi/test/data/test_cases_word.txt +20 -0
  1209. tn/hindi/test/normalizer_test.py +137 -0
  1210. tn/hungarian/__init__.py +0 -0
  1211. tn/hungarian/data/dates/days.tsv +31 -0
  1212. tn/hungarian/data/dates/month_abbr.tsv +12 -0
  1213. tn/hungarian/data/dates/months.tsv +12 -0
  1214. tn/hungarian/data/dates/months_roman.tsv +12 -0
  1215. tn/hungarian/data/electronic/domain.tsv +15 -0
  1216. tn/hungarian/data/electronic/server_name.tsv +15 -0
  1217. tn/hungarian/data/electronic/symbols.tsv +23 -0
  1218. tn/hungarian/data/fractions/fraction_symbols.tsv +19 -0
  1219. tn/hungarian/data/inflection/endings.tsv +51 -0
  1220. tn/hungarian/data/inflection/plural_endings.tsv +39 -0
  1221. tn/hungarian/data/inflection/word_endings.tsv +1 -0
  1222. tn/hungarian/data/measures/measurements.tsv +52 -0
  1223. tn/hungarian/data/money/alphabet.tsv +44 -0
  1224. tn/hungarian/data/money/currency.tsv +11 -0
  1225. tn/hungarian/data/money/currency_minor.tsv +11 -0
  1226. tn/hungarian/data/number/digit.tsv +9 -0
  1227. tn/hungarian/data/number/digit_inline.tsv +9 -0
  1228. tn/hungarian/data/number/quantities.tsv +21 -0
  1229. tn/hungarian/data/number/tens.tsv +9 -0
  1230. tn/hungarian/data/number/tens_inline.tsv +9 -0
  1231. tn/hungarian/data/number/zero.tsv +1 -0
  1232. tn/hungarian/data/ordinals/endings.tsv +23 -0
  1233. tn/hungarian/data/ordinals/exceptional.tsv +4 -0
  1234. tn/hungarian/data/telephone/area_codes.tsv +71 -0
  1235. tn/hungarian/data/telephone/country_codes.tsv +213 -0
  1236. tn/hungarian/data/telephone/telephone_abbr.tsv +3 -0
  1237. tn/hungarian/data/time/time_zone.tsv +9 -0
  1238. tn/hungarian/data/whitelist.tsv +24 -0
  1239. tn/hungarian/data/whitelist_inflect.tsv +39 -0
  1240. tn/hungarian/data/whitelist_inflect_sg.tsv +14 -0
  1241. tn/hungarian/normalizer.py +80 -0
  1242. tn/hungarian/rules/__init__.py +0 -0
  1243. tn/hungarian/rules/cardinal.py +98 -0
  1244. tn/hungarian/rules/date.py +104 -0
  1245. tn/hungarian/rules/decimal.py +85 -0
  1246. tn/hungarian/rules/electronic.py +69 -0
  1247. tn/hungarian/rules/fraction.py +55 -0
  1248. tn/hungarian/rules/measure.py +58 -0
  1249. tn/hungarian/rules/money.py +81 -0
  1250. tn/hungarian/rules/ordinal.py +41 -0
  1251. tn/hungarian/rules/punctuation.py +46 -0
  1252. tn/hungarian/rules/telephone.py +71 -0
  1253. tn/hungarian/rules/time.py +106 -0
  1254. tn/hungarian/rules/utils.py +156 -0
  1255. tn/hungarian/rules/whitelist.py +32 -0
  1256. tn/hungarian/rules/word.py +38 -0
  1257. tn/hungarian/test/__init__.py +0 -0
  1258. tn/hungarian/test/data/test_cases_cardinal.txt +69 -0
  1259. tn/hungarian/test/data/test_cases_date.txt +17 -0
  1260. tn/hungarian/test/data/test_cases_decimal.txt +6 -0
  1261. tn/hungarian/test/data/test_cases_electronic.txt +8 -0
  1262. tn/hungarian/test/data/test_cases_fraction.txt +4 -0
  1263. tn/hungarian/test/data/test_cases_measure.txt +6 -0
  1264. tn/hungarian/test/data/test_cases_money.txt +9 -0
  1265. tn/hungarian/test/data/test_cases_ordinal.txt +5 -0
  1266. tn/hungarian/test/data/test_cases_telephone.txt +9 -0
  1267. tn/hungarian/test/data/test_cases_time.txt +10 -0
  1268. tn/hungarian/test/data/test_cases_whitelist.txt +10 -0
  1269. tn/hungarian/test/data/test_cases_word.txt +52 -0
  1270. tn/hungarian/test/normalizer_test.py +210 -0
  1271. tn/italian/__init__.py +0 -0
  1272. tn/italian/data/electronic/domain.tsv +14 -0
  1273. tn/italian/data/electronic/server_name.tsv +11 -0
  1274. tn/italian/data/electronic/symbols.tsv +22 -0
  1275. tn/italian/data/measure/measurements.tsv +64 -0
  1276. tn/italian/data/measure/suppletive.tsv +11 -0
  1277. tn/italian/data/money/currency_major.tsv +11 -0
  1278. tn/italian/data/money/currency_minor.tsv +11 -0
  1279. tn/italian/data/money/currency_plural_fem.tsv +3 -0
  1280. tn/italian/data/money/currency_plural_masc.tsv +11 -0
  1281. tn/italian/data/numbers/digit.tsv +9 -0
  1282. tn/italian/data/numbers/hundreds.tsv +8 -0
  1283. tn/italian/data/numbers/quantities.tsv +2 -0
  1284. tn/italian/data/numbers/teen.tsv +10 -0
  1285. tn/italian/data/numbers/tens.tsv +8 -0
  1286. tn/italian/data/numbers/tens_eight.tsv +8 -0
  1287. tn/italian/data/numbers/tens_one.tsv +8 -0
  1288. tn/italian/data/numbers/zero.tsv +1 -0
  1289. tn/italian/data/whitelist/whitelist.tsv +65 -0
  1290. tn/italian/normalizer.py +60 -0
  1291. tn/italian/rules/__init__.py +0 -0
  1292. tn/italian/rules/cardinal.py +99 -0
  1293. tn/italian/rules/decimal.py +82 -0
  1294. tn/italian/rules/electronic.py +80 -0
  1295. tn/italian/rules/measure.py +73 -0
  1296. tn/italian/rules/money.py +92 -0
  1297. tn/italian/rules/punctuation.py +30 -0
  1298. tn/italian/rules/time.py +61 -0
  1299. tn/italian/rules/utils.py +74 -0
  1300. tn/italian/rules/whitelist.py +27 -0
  1301. tn/italian/rules/word.py +28 -0
  1302. tn/italian/test/__init__.py +0 -0
  1303. tn/italian/test/data/test_cases_cardinal.txt +13 -0
  1304. tn/italian/test/data/test_cases_decimal.txt +4 -0
  1305. tn/italian/test/data/test_cases_electronic.txt +13 -0
  1306. tn/italian/test/data/test_cases_measure.txt +7 -0
  1307. tn/italian/test/data/test_cases_money.txt +5 -0
  1308. tn/italian/test/data/test_cases_time.txt +6 -0
  1309. tn/italian/test/data/test_cases_whitelist.txt +4 -0
  1310. tn/italian/test/normalizer_test.py +209 -0
  1311. tn/japanese/normalizer.py +56 -48
  1312. tn/japanese/rules/cardinal.py +35 -54
  1313. tn/japanese/rules/date.py +33 -18
  1314. tn/japanese/rules/fraction.py +8 -14
  1315. tn/japanese/rules/math.py +14 -6
  1316. tn/japanese/rules/measure.py +10 -4
  1317. tn/japanese/rules/money.py +9 -14
  1318. tn/japanese/rules/postprocessor.py +5 -1
  1319. tn/japanese/rules/preprocessor.py +2 -2
  1320. tn/japanese/rules/range.py +32 -0
  1321. tn/japanese/rules/sport.py +18 -10
  1322. tn/japanese/rules/time.py +25 -14
  1323. tn/japanese/rules/transliteration.py +12 -5
  1324. tn/japanese/rules/whitelist.py +11 -7
  1325. tn/japanese/test/normalizer_test.py +107 -4
  1326. tn/kinyarwanda/__init__.py +0 -0
  1327. tn/kinyarwanda/data/numbers/digit.tsv +9 -0
  1328. tn/kinyarwanda/data/numbers/digit_bi.tsv +9 -0
  1329. tn/kinyarwanda/data/numbers/digit_e.tsv +9 -0
  1330. tn/kinyarwanda/data/numbers/hundreds.tsv +9 -0
  1331. tn/kinyarwanda/data/numbers/tens.tsv +8 -0
  1332. tn/kinyarwanda/data/numbers/zero.tsv +1 -0
  1333. tn/kinyarwanda/data/time/hours.tsv +12 -0
  1334. tn/kinyarwanda/data/time/minutes.tsv +60 -0
  1335. tn/kinyarwanda/data/whitelist.tsv +175 -0
  1336. tn/kinyarwanda/normalizer.py +56 -0
  1337. tn/kinyarwanda/rules/__init__.py +0 -0
  1338. tn/kinyarwanda/rules/cardinal.py +117 -0
  1339. tn/kinyarwanda/rules/punctuation.py +31 -0
  1340. tn/kinyarwanda/rules/time.py +40 -0
  1341. tn/kinyarwanda/rules/utils.py +43 -0
  1342. tn/kinyarwanda/rules/whitelist.py +28 -0
  1343. tn/kinyarwanda/rules/word.py +29 -0
  1344. tn/kinyarwanda/test/__init__.py +0 -0
  1345. tn/kinyarwanda/test/data/test_cases_cardinal.txt +57 -0
  1346. tn/kinyarwanda/test/data/test_cases_time.txt +14 -0
  1347. tn/kinyarwanda/test/data/test_cases_whitelist.txt +3 -0
  1348. tn/kinyarwanda/test/data/test_cases_word.txt +26 -0
  1349. tn/kinyarwanda/test/normalizer_test.py +198 -0
  1350. tn/korean/__init__.py +16 -0
  1351. tn/korean/data/date/exceptions.tsv +2 -0
  1352. tn/korean/data/date/week.tsv +8 -0
  1353. tn/korean/data/electronic/cc_cues.tsv +11 -0
  1354. tn/korean/data/electronic/domain.tsv +28 -0
  1355. tn/korean/data/electronic/extensions.tsv +6 -0
  1356. tn/korean/data/electronic/symbol.tsv +12 -0
  1357. tn/korean/data/measure/unit.tsv +18 -0
  1358. tn/korean/data/money/currency_major.tsv +22 -0
  1359. tn/korean/data/number/counter_suffix.tsv +16 -0
  1360. tn/korean/data/number/digit.tsv +9 -0
  1361. tn/korean/data/number/native_ones.tsv +9 -0
  1362. tn/korean/data/number/tens.tsv +9 -0
  1363. tn/korean/data/number/zero.tsv +1 -0
  1364. tn/korean/data/ordinal/digit.tsv +8 -0
  1365. tn/korean/data/ordinal/exceptions.tsv +5 -0
  1366. tn/korean/data/ordinal/tens.tsv +2 -0
  1367. tn/korean/data/ordinal/tens_prefix.tsv +3 -0
  1368. tn/korean/data/time/division.tsv +7 -0
  1369. tn/korean/data/time/hour.tsv +12 -0
  1370. tn/korean/data/whitelist.tsv +39 -0
  1371. tn/korean/normalizer.py +79 -0
  1372. tn/korean/rules/__init__.py +3 -0
  1373. tn/korean/rules/cardinal.py +91 -0
  1374. tn/korean/rules/date.py +129 -0
  1375. tn/korean/rules/decimal.py +53 -0
  1376. tn/korean/rules/electronic.py +84 -0
  1377. tn/korean/rules/fraction.py +81 -0
  1378. tn/korean/rules/measure.py +82 -0
  1379. tn/korean/rules/money.py +60 -0
  1380. tn/korean/rules/ordinal.py +62 -0
  1381. tn/korean/rules/punctuation.py +16 -0
  1382. tn/korean/rules/telephone.py +40 -0
  1383. tn/korean/rules/time.py +81 -0
  1384. tn/korean/rules/whitelist.py +17 -0
  1385. tn/korean/rules/word.py +18 -0
  1386. tn/korean/test/__init__.py +3 -0
  1387. tn/korean/test/data/test_cases_cardinal.txt +68 -0
  1388. tn/korean/test/data/test_cases_date.txt +29 -0
  1389. tn/korean/test/data/test_cases_decimal.txt +28 -0
  1390. tn/korean/test/data/test_cases_electronic.txt +20 -0
  1391. tn/korean/test/data/test_cases_fraction.txt +28 -0
  1392. tn/korean/test/data/test_cases_measure.txt +40 -0
  1393. tn/korean/test/data/test_cases_money.txt +64 -0
  1394. tn/korean/test/data/test_cases_ordinal.txt +54 -0
  1395. tn/korean/test/data/test_cases_telephone.txt +39 -0
  1396. tn/korean/test/data/test_cases_time.txt +25 -0
  1397. tn/korean/test/date_test.py +99 -0
  1398. tn/korean/test/decimal_test.py +67 -0
  1399. tn/korean/test/electronic_test.py +58 -0
  1400. tn/korean/test/fraction_test.py +89 -0
  1401. tn/korean/test/measure_test.py +90 -0
  1402. tn/korean/test/money_test.py +92 -0
  1403. tn/korean/test/normalizer_test.py +146 -0
  1404. tn/korean/test/ordinal_test.py +78 -0
  1405. tn/korean/test/telephone_test.py +40 -0
  1406. tn/korean/test/time_test.py +51 -0
  1407. tn/main.py +2 -52
  1408. tn/marathi/__init__.py +0 -0
  1409. tn/marathi/data/date/eras.tsv +2 -0
  1410. tn/marathi/data/date/months.tsv +12 -0
  1411. tn/marathi/data/numbers/digit.tsv +9 -0
  1412. tn/marathi/data/numbers/tens.tsv +90 -0
  1413. tn/marathi/data/numbers/zero.tsv +1 -0
  1414. tn/marathi/normalizer.py +53 -0
  1415. tn/marathi/rules/__init__.py +0 -0
  1416. tn/marathi/rules/cardinal.py +76 -0
  1417. tn/marathi/rules/date.py +70 -0
  1418. tn/marathi/rules/decimal.py +59 -0
  1419. tn/marathi/rules/punctuation.py +30 -0
  1420. tn/marathi/rules/time.py +86 -0
  1421. tn/marathi/rules/utils.py +76 -0
  1422. tn/marathi/rules/word.py +28 -0
  1423. tn/marathi/test/__init__.py +0 -0
  1424. tn/marathi/test/normalizer_test.py +221 -0
  1425. tn/portuguese/__init__.py +0 -0
  1426. tn/portuguese/data/date/months.tsv +21 -0
  1427. tn/portuguese/data/date/numeric_separators.tsv +3 -0
  1428. tn/portuguese/data/date/verbal_phrases.tsv +1 -0
  1429. tn/portuguese/data/electronic/domain.tsv +7 -0
  1430. tn/portuguese/data/electronic/electronic_spoken_unit.tsv +16 -0
  1431. tn/portuguese/data/electronic/server_name.tsv +6 -0
  1432. tn/portuguese/data/electronic/symbols.tsv +22 -0
  1433. tn/portuguese/data/fractions/ordinal_exceptions.tsv +2 -0
  1434. tn/portuguese/data/fractions/powers_of_ten.tsv +1 -0
  1435. tn/portuguese/data/fractions/specials.tsv +5 -0
  1436. tn/portuguese/data/measure/measurements_plural.tsv +64 -0
  1437. tn/portuguese/data/measure/measurements_singular.tsv +65 -0
  1438. tn/portuguese/data/money/currency_major.tsv +5 -0
  1439. tn/portuguese/data/money/currency_major_plural.tsv +5 -0
  1440. tn/portuguese/data/money/currency_minor.tsv +5 -0
  1441. tn/portuguese/data/money/currency_minor_plural.tsv +1 -0
  1442. tn/portuguese/data/numbers/cardinal_specials.tsv +4 -0
  1443. tn/portuguese/data/numbers/decimal_specials.tsv +2 -0
  1444. tn/portuguese/data/numbers/digit.tsv +9 -0
  1445. tn/portuguese/data/numbers/hundreds.tsv +8 -0
  1446. tn/portuguese/data/numbers/quantity_words.tsv +9 -0
  1447. tn/portuguese/data/numbers/scales.tsv +4 -0
  1448. tn/portuguese/data/numbers/teens.tsv +10 -0
  1449. tn/portuguese/data/numbers/tens.tsv +8 -0
  1450. tn/portuguese/data/numbers/zero.tsv +1 -0
  1451. tn/portuguese/data/ordinals/digit.tsv +10 -0
  1452. tn/portuguese/data/ordinals/feminine.tsv +11 -0
  1453. tn/portuguese/data/ordinals/hundreds.tsv +10 -0
  1454. tn/portuguese/data/ordinals/ties.tsv +8 -0
  1455. tn/portuguese/data/telephone/ip_prompt.tsv +2 -0
  1456. tn/portuguese/data/telephone/telephone_prompt.tsv +4 -0
  1457. tn/portuguese/data/time/day_period_suffix.tsv +4 -0
  1458. tn/portuguese/data/whitelist/symbol.tsv +23 -0
  1459. tn/portuguese/normalizer.py +74 -0
  1460. tn/portuguese/rules/__init__.py +0 -0
  1461. tn/portuguese/rules/cardinal.py +170 -0
  1462. tn/portuguese/rules/date.py +69 -0
  1463. tn/portuguese/rules/decimal.py +67 -0
  1464. tn/portuguese/rules/electronic.py +82 -0
  1465. tn/portuguese/rules/fraction.py +79 -0
  1466. tn/portuguese/rules/measure.py +66 -0
  1467. tn/portuguese/rules/money.py +117 -0
  1468. tn/portuguese/rules/ordinal.py +82 -0
  1469. tn/portuguese/rules/punctuation.py +47 -0
  1470. tn/portuguese/rules/telephone.py +71 -0
  1471. tn/portuguese/rules/time.py +101 -0
  1472. tn/portuguese/rules/whitelist.py +37 -0
  1473. tn/portuguese/rules/word.py +35 -0
  1474. tn/portuguese/test/__init__.py +0 -0
  1475. tn/portuguese/test/data/test_cases_cardinal.txt +121 -0
  1476. tn/portuguese/test/data/test_cases_date.txt +26 -0
  1477. tn/portuguese/test/data/test_cases_decimal.txt +58 -0
  1478. tn/portuguese/test/data/test_cases_electronic.txt +13 -0
  1479. tn/portuguese/test/data/test_cases_fraction.txt +22 -0
  1480. tn/portuguese/test/data/test_cases_measure.txt +18 -0
  1481. tn/portuguese/test/data/test_cases_money.txt +25 -0
  1482. tn/portuguese/test/data/test_cases_ordinal.txt +39 -0
  1483. tn/portuguese/test/data/test_cases_telephone.txt +7 -0
  1484. tn/portuguese/test/data/test_cases_time.txt +30 -0
  1485. tn/portuguese/test/normalizer_test.py +166 -0
  1486. tn/processor.py +432 -38
  1487. tn/russian/__init__.py +0 -0
  1488. tn/russian/data/currency/currency_agreement.tsv +10 -0
  1489. tn/russian/data/currency/currency_plural.tsv +53 -0
  1490. tn/russian/data/currency/currency_singular.tsv +48 -0
  1491. tn/russian/data/electronic/domain.tsv +9 -0
  1492. tn/russian/data/electronic/server_name.tsv +17 -0
  1493. tn/russian/data/electronic/symbols.tsv +28 -0
  1494. tn/russian/data/latin_to_cyrillic.tsv +38 -0
  1495. tn/russian/data/measurements.tsv +321 -0
  1496. tn/russian/data/months/abbr.tsv +12 -0
  1497. tn/russian/data/months/abbr_to_name.tsv +62 -0
  1498. tn/russian/data/months/numbers.tsv +21 -0
  1499. tn/russian/data/numbers/1_cardinals_nominative.tsv +50 -0
  1500. tn/russian/data/numbers/2_cardinals_genitive.tsv +45 -0
  1501. tn/russian/data/numbers/3_cardinals_dative.tsv +45 -0
  1502. tn/russian/data/numbers/4_cardinals_accusative.tsv +48 -0
  1503. tn/russian/data/numbers/5_cardinals_instrumental.tsv +47 -0
  1504. tn/russian/data/numbers/6_cardinals_prepositional.tsv +46 -0
  1505. tn/russian/data/numbers/cardinals_alternatives.tsv +16 -0
  1506. tn/russian/data/numbers/cardinals_nominative_case.tsv +37 -0
  1507. tn/russian/data/numbers/decimal_delimiter.tsv +15 -0
  1508. tn/russian/data/numbers/decimal_endings.tsv +37 -0
  1509. tn/russian/data/numbers/digits_nominative_case.tsv +10 -0
  1510. tn/russian/data/numbers/ordinal_endings.tsv +40 -0
  1511. tn/russian/data/numbers/ordinals.tsv +752 -0
  1512. tn/russian/data/numbers/quantity.tsv +16 -0
  1513. tn/russian/data/time/increment_hour_cardinal.tsv +24 -0
  1514. tn/russian/data/time/increment_hour_ordinal.tsv +24 -0
  1515. tn/russian/data/time/minutes_to_hour.tsv +25 -0
  1516. tn/russian/data/time/time_convert.tsv +12 -0
  1517. tn/russian/data/whitelist.tsv +37 -0
  1518. tn/russian/normalizer.py +75 -0
  1519. tn/russian/rules/__init__.py +0 -0
  1520. tn/russian/rules/cardinal.py +92 -0
  1521. tn/russian/rules/date.py +93 -0
  1522. tn/russian/rules/decimal.py +88 -0
  1523. tn/russian/rules/electronic.py +69 -0
  1524. tn/russian/rules/measure.py +70 -0
  1525. tn/russian/rules/money.py +73 -0
  1526. tn/russian/rules/ordinal.py +45 -0
  1527. tn/russian/rules/punctuation.py +46 -0
  1528. tn/russian/rules/telephone.py +44 -0
  1529. tn/russian/rules/time.py +80 -0
  1530. tn/russian/rules/utils.py +293 -0
  1531. tn/russian/rules/whitelist.py +41 -0
  1532. tn/russian/rules/word.py +34 -0
  1533. tn/russian/test/__init__.py +0 -0
  1534. tn/russian/test/data/test_cases_cardinal.txt +28 -0
  1535. tn/russian/test/data/test_cases_cardinal_normalize_with_audio.txt +51 -0
  1536. tn/russian/test/data/test_cases_date.txt +6 -0
  1537. tn/russian/test/data/test_cases_decimal.txt +17 -0
  1538. tn/russian/test/data/test_cases_electronic.txt +4 -0
  1539. tn/russian/test/data/test_cases_measure.txt +13 -0
  1540. tn/russian/test/data/test_cases_money.txt +9 -0
  1541. tn/russian/test/data/test_cases_ordinal.txt +75 -0
  1542. tn/russian/test/data/test_cases_telephone.txt +1 -0
  1543. tn/russian/test/data/test_cases_time.txt +9 -0
  1544. tn/russian/test/data/test_cases_whitelist.txt +4 -0
  1545. tn/russian/test/data/test_cases_word.txt +3 -0
  1546. tn/russian/test/normalizer_test.py +214 -0
  1547. tn/spanish/__init__.py +0 -0
  1548. tn/spanish/data/dates/days_abbr.tsv +27 -0
  1549. tn/spanish/data/dates/months.tsv +12 -0
  1550. tn/spanish/data/dates/months_abbr.tsv +44 -0
  1551. tn/spanish/data/dates/year_suffix.tsv +51 -0
  1552. tn/spanish/data/electronic/domain.tsv +6 -0
  1553. tn/spanish/data/electronic/server_name.tsv +11 -0
  1554. tn/spanish/data/electronic/symbols.tsv +22 -0
  1555. tn/spanish/data/fractions/fraction_symbols.tsv +15 -0
  1556. tn/spanish/data/fractions/ordinal_exceptions.tsv +2 -0
  1557. tn/spanish/data/fractions/powers_of_ten.tsv +9 -0
  1558. tn/spanish/data/fractions/powers_of_ten_fem.tsv +6 -0
  1559. tn/spanish/data/measures/math_symbols.tsv +7 -0
  1560. tn/spanish/data/measures/measurements.tsv +18 -0
  1561. tn/spanish/data/measures/measurements_complex.tsv +5 -0
  1562. tn/spanish/data/measures/measurements_plural_fem.tsv +3 -0
  1563. tn/spanish/data/measures/measurements_plural_masc.tsv +21 -0
  1564. tn/spanish/data/money/currency_major.tsv +64 -0
  1565. tn/spanish/data/money/currency_major_ext.tsv +215 -0
  1566. tn/spanish/data/money/currency_minor.tsv +64 -0
  1567. tn/spanish/data/money/currency_minor_ext.tsv +181 -0
  1568. tn/spanish/data/money/currency_plural_fem.tsv +8 -0
  1569. tn/spanish/data/money/currency_plural_fem_ext.tsv +56 -0
  1570. tn/spanish/data/money/currency_plural_masc.tsv +51 -0
  1571. tn/spanish/data/money/currency_plural_masc_ext.tsv +199 -0
  1572. tn/spanish/data/numbers/digit.tsv +9 -0
  1573. tn/spanish/data/numbers/hundreds.tsv +8 -0
  1574. tn/spanish/data/numbers/quantities.tsv +6 -0
  1575. tn/spanish/data/numbers/teen.tsv +10 -0
  1576. tn/spanish/data/numbers/ties.tsv +7 -0
  1577. tn/spanish/data/numbers/twenties.tsv +10 -0
  1578. tn/spanish/data/numbers/zero.tsv +1 -0
  1579. tn/spanish/data/ordinals/digit.tsv +11 -0
  1580. tn/spanish/data/ordinals/gender_suffix.tsv +20 -0
  1581. tn/spanish/data/ordinals/hundreds.tsv +18 -0
  1582. tn/spanish/data/ordinals/roman_exceptions.tsv +10 -0
  1583. tn/spanish/data/ordinals/teen.tsv +10 -0
  1584. tn/spanish/data/ordinals/ties.tsv +8 -0
  1585. tn/spanish/data/ordinals/twenties.tsv +11 -0
  1586. tn/spanish/data/roman/digit.tsv +9 -0
  1587. tn/spanish/data/roman/hundreds.tsv +9 -0
  1588. tn/spanish/data/roman/thousands.tsv +3 -0
  1589. tn/spanish/data/roman/ties.tsv +9 -0
  1590. tn/spanish/data/telephone/ip_prompt.tsv +2 -0
  1591. tn/spanish/data/telephone/telephone_prompt.tsv +10 -0
  1592. tn/spanish/data/time/afternoon_times.tsv +6 -0
  1593. tn/spanish/data/time/alt_minutes.tsv +2 -0
  1594. tn/spanish/data/time/evening_times.tsv +5 -0
  1595. tn/spanish/data/time/hour_to_12.tsv +12 -0
  1596. tn/spanish/data/time/hour_to_24.tsv +25 -0
  1597. tn/spanish/data/time/hour_to_night.tsv +13 -0
  1598. tn/spanish/data/time/minute_to.tsv +59 -0
  1599. tn/spanish/data/time/morning_times.tsv +12 -0
  1600. tn/spanish/data/time/time_suffix.tsv +20 -0
  1601. tn/spanish/data/time/time_zone.tsv +42 -0
  1602. tn/spanish/data/time/time_zone_ext.tsv +105 -0
  1603. tn/spanish/data/whitelist.tsv +62 -0
  1604. tn/spanish/normalizer.py +79 -0
  1605. tn/spanish/rules/__init__.py +0 -0
  1606. tn/spanish/rules/cardinal.py +125 -0
  1607. tn/spanish/rules/date.py +96 -0
  1608. tn/spanish/rules/decimal.py +122 -0
  1609. tn/spanish/rules/electronic.py +69 -0
  1610. tn/spanish/rules/fraction.py +189 -0
  1611. tn/spanish/rules/measure.py +177 -0
  1612. tn/spanish/rules/money.py +186 -0
  1613. tn/spanish/rules/ordinal.py +119 -0
  1614. tn/spanish/rules/punctuation.py +48 -0
  1615. tn/spanish/rules/telephone.py +93 -0
  1616. tn/spanish/rules/time.py +171 -0
  1617. tn/spanish/rules/utils.py +170 -0
  1618. tn/spanish/rules/whitelist.py +39 -0
  1619. tn/spanish/rules/word.py +39 -0
  1620. tn/spanish/test/__init__.py +0 -0
  1621. tn/spanish/test/data/test_cases_cardinal.txt +69 -0
  1622. tn/spanish/test/data/test_cases_date.txt +21 -0
  1623. tn/spanish/test/data/test_cases_decimal.txt +24 -0
  1624. tn/spanish/test/data/test_cases_electronic.txt +22 -0
  1625. tn/spanish/test/data/test_cases_fraction.txt +75 -0
  1626. tn/spanish/test/data/test_cases_measure.txt +32 -0
  1627. tn/spanish/test/data/test_cases_money.txt +69 -0
  1628. tn/spanish/test/data/test_cases_normalize_with_audio.txt +162 -0
  1629. tn/spanish/test/data/test_cases_ordinal.txt +120 -0
  1630. tn/spanish/test/data/test_cases_telephone.txt +15 -0
  1631. tn/spanish/test/data/test_cases_time.txt +35 -0
  1632. tn/spanish/test/data/test_cases_whitelist.txt +7 -0
  1633. tn/spanish/test/data/test_cases_word.txt +48 -0
  1634. tn/spanish/test/normalizer_test.py +166 -0
  1635. tn/swedish/__init__.py +0 -0
  1636. tn/swedish/data/abbreviations_nondet.tsv +27 -0
  1637. tn/swedish/data/dates/era_suffix.tsv +18 -0
  1638. tn/swedish/data/dates/era_words.tsv +4 -0
  1639. tn/swedish/data/dates/month_abbr.tsv +17 -0
  1640. tn/swedish/data/dates/months.tsv +12 -0
  1641. tn/swedish/data/electronic/domain.tsv +10 -0
  1642. tn/swedish/data/electronic/server_name.tsv +29 -0
  1643. tn/swedish/data/electronic/symbols.tsv +20 -0
  1644. tn/swedish/data/math_operations.tsv +8 -0
  1645. tn/swedish/data/measure/greek_lower.tsv +25 -0
  1646. tn/swedish/data/measure/greek_upper.tsv +25 -0
  1647. tn/swedish/data/measure/unit.tsv +147 -0
  1648. tn/swedish/data/measure/unit_neuter.tsv +13 -0
  1649. tn/swedish/data/measure/unit_plural.tsv +121 -0
  1650. tn/swedish/data/money/currency_major.tsv +75 -0
  1651. tn/swedish/data/money/currency_major_nt.tsv +2 -0
  1652. tn/swedish/data/money/currency_minor_plural.tsv +79 -0
  1653. tn/swedish/data/money/currency_minor_singular.tsv +78 -0
  1654. tn/swedish/data/money/currency_plurals.tsv +44 -0
  1655. tn/swedish/data/numbers/digit.tsv +9 -0
  1656. tn/swedish/data/numbers/fraction.tsv +18 -0
  1657. tn/swedish/data/numbers/millions.tsv +88 -0
  1658. tn/swedish/data/numbers/millions_abbr.tsv +4 -0
  1659. tn/swedish/data/numbers/teen.tsv +10 -0
  1660. tn/swedish/data/numbers/ties.tsv +8 -0
  1661. tn/swedish/data/numbers/zero.tsv +1 -0
  1662. tn/swedish/data/ordinals/digit.tsv +9 -0
  1663. tn/swedish/data/ordinals/teen.tsv +10 -0
  1664. tn/swedish/data/ordinals/ties.tsv +8 -0
  1665. tn/swedish/data/ordinals/zero.tsv +1 -0
  1666. tn/swedish/data/telephone/ip_prompt.tsv +2 -0
  1667. tn/swedish/data/telephone/special_numbers.tsv +4 -0
  1668. tn/swedish/data/telephone/ssn_prompt.tsv +3 -0
  1669. tn/swedish/data/telephone/telephone_abbr.tsv +24 -0
  1670. tn/swedish/data/telephone/telephone_prompt.tsv +5 -0
  1671. tn/swedish/data/time/suffix.tsv +16 -0
  1672. tn/swedish/data/time/time_zone.tsv +7 -0
  1673. tn/swedish/data/whitelist.tsv +58 -0
  1674. tn/swedish/normalizer.py +75 -0
  1675. tn/swedish/rules/__init__.py +0 -0
  1676. tn/swedish/rules/cardinal.py +172 -0
  1677. tn/swedish/rules/date.py +122 -0
  1678. tn/swedish/rules/decimal.py +110 -0
  1679. tn/swedish/rules/electronic.py +75 -0
  1680. tn/swedish/rules/fraction.py +85 -0
  1681. tn/swedish/rules/measure.py +143 -0
  1682. tn/swedish/rules/money.py +127 -0
  1683. tn/swedish/rules/ordinal.py +132 -0
  1684. tn/swedish/rules/punctuation.py +47 -0
  1685. tn/swedish/rules/telephone.py +89 -0
  1686. tn/swedish/rules/time.py +98 -0
  1687. tn/swedish/rules/utils.py +63 -0
  1688. tn/swedish/rules/whitelist.py +38 -0
  1689. tn/swedish/rules/word.py +33 -0
  1690. tn/swedish/test/__init__.py +0 -0
  1691. tn/swedish/test/data/test_cases_cardinal.txt +68 -0
  1692. tn/swedish/test/data/test_cases_date.txt +17 -0
  1693. tn/swedish/test/data/test_cases_decimal.txt +7 -0
  1694. tn/swedish/test/data/test_cases_electronic.txt +12 -0
  1695. tn/swedish/test/data/test_cases_fraction.txt +2 -0
  1696. tn/swedish/test/data/test_cases_measure.txt +7 -0
  1697. tn/swedish/test/data/test_cases_money.txt +65 -0
  1698. tn/swedish/test/data/test_cases_normalize_with_audio.txt +14 -0
  1699. tn/swedish/test/data/test_cases_ordinal.txt +7 -0
  1700. tn/swedish/test/data/test_cases_telephone.txt +7 -0
  1701. tn/swedish/test/data/test_cases_time.txt +12 -0
  1702. tn/swedish/test/data/test_cases_whitelist.txt +1 -0
  1703. tn/swedish/test/data/test_cases_word.txt +48 -0
  1704. tn/swedish/test/normalizer_test.py +195 -0
  1705. tn/test/cli_test.py +337 -0
  1706. tn/test/punctuation_spacing_test.py +22 -0
  1707. tn/test/robustness_test.py +32 -0
  1708. tn/token_parser.py +91 -29
  1709. tn/utils.py +10 -0
  1710. tn/vietnamese/__init__.py +0 -0
  1711. tn/vietnamese/data/date/days.tsv +40 -0
  1712. tn/vietnamese/data/date/months.tsv +21 -0
  1713. tn/vietnamese/data/date/year_suffix.tsv +4 -0
  1714. tn/vietnamese/data/fraction/denominator_exceptions.tsv +1 -0
  1715. tn/vietnamese/data/measure/base_units.tsv +20 -0
  1716. tn/vietnamese/data/measure/measurements_minimal.tsv +33 -0
  1717. tn/vietnamese/data/measure/prefixes.tsv +17 -0
  1718. tn/vietnamese/data/money/currency.tsv +51 -0
  1719. tn/vietnamese/data/money/currency_minor.tsv +7 -0
  1720. tn/vietnamese/data/money/per_unit_bases.tsv +8 -0
  1721. tn/vietnamese/data/money/per_unit_non_metric.tsv +28 -0
  1722. tn/vietnamese/data/money/per_unit_prefixes.tsv +6 -0
  1723. tn/vietnamese/data/numbers/digit.tsv +9 -0
  1724. tn/vietnamese/data/numbers/digit_special.tsv +3 -0
  1725. tn/vietnamese/data/numbers/magnitudes.tsv +8 -0
  1726. tn/vietnamese/data/numbers/quantity_abbr.tsv +7 -0
  1727. tn/vietnamese/data/numbers/teen.tsv +10 -0
  1728. tn/vietnamese/data/numbers/ties.tsv +8 -0
  1729. tn/vietnamese/data/numbers/zero.tsv +1 -0
  1730. tn/vietnamese/data/ordinal/ordinal_exceptions.tsv +2 -0
  1731. tn/vietnamese/data/roman/key_word.tsv +16 -0
  1732. tn/vietnamese/data/roman/roman_numerals.tsv +13 -0
  1733. tn/vietnamese/data/time/time_units.tsv +4 -0
  1734. tn/vietnamese/data/time/time_zones.tsv +18 -0
  1735. tn/vietnamese/data/whitelist.tsv +0 -0
  1736. tn/vietnamese/normalizer.py +106 -0
  1737. tn/vietnamese/rules/__init__.py +0 -0
  1738. tn/vietnamese/rules/cardinal.py +163 -0
  1739. tn/vietnamese/rules/date.py +81 -0
  1740. tn/vietnamese/rules/decimal.py +85 -0
  1741. tn/vietnamese/rules/fraction.py +50 -0
  1742. tn/vietnamese/rules/measure.py +57 -0
  1743. tn/vietnamese/rules/money.py +87 -0
  1744. tn/vietnamese/rules/ordinal.py +34 -0
  1745. tn/vietnamese/rules/punctuation.py +31 -0
  1746. tn/vietnamese/rules/range.py +36 -0
  1747. tn/vietnamese/rules/roman.py +59 -0
  1748. tn/vietnamese/rules/time.py +86 -0
  1749. tn/vietnamese/rules/whitelist.py +29 -0
  1750. tn/vietnamese/rules/word.py +36 -0
  1751. tn/vietnamese/test/__init__.py +0 -0
  1752. tn/vietnamese/test/data/test_cases_cardinal.txt +109 -0
  1753. tn/vietnamese/test/data/test_cases_date.txt +13 -0
  1754. tn/vietnamese/test/data/test_cases_decimal.txt +29 -0
  1755. tn/vietnamese/test/data/test_cases_fraction.txt +13 -0
  1756. tn/vietnamese/test/data/test_cases_measure.txt +63 -0
  1757. tn/vietnamese/test/data/test_cases_money.txt +30 -0
  1758. tn/vietnamese/test/data/test_cases_ordinal.txt +30 -0
  1759. tn/vietnamese/test/data/test_cases_range.txt +11 -0
  1760. tn/vietnamese/test/data/test_cases_roman.txt +57 -0
  1761. tn/vietnamese/test/data/test_cases_time.txt +117 -0
  1762. tn/vietnamese/test/normalizer_test.py +146 -0
  1763. wetextprocessing-1.3.0.dist-info/METADATA +290 -0
  1764. wetextprocessing-1.3.0.dist-info/RECORD +2049 -0
  1765. itn/zh_itn_tagger.fst +0 -0
  1766. itn/zh_itn_verbalizer.fst +0 -0
  1767. tn/en_tn_tagger.fst +0 -0
  1768. tn/en_tn_verbalizer.fst +0 -0
  1769. tn/zh_tn_tagger.fst +0 -0
  1770. tn/zh_tn_verbalizer.fst +0 -0
  1771. wetextprocessing-1.0.6.dev19.dist-info/METADATA +0 -160
  1772. wetextprocessing-1.0.6.dev19.dist-info/RECORD +0 -381
  1773. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/WHEEL +0 -0
  1774. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/entry_points.txt +0 -0
  1775. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/licenses/LICENSE +0 -0
  1776. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.3.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,29 @@
1
+ # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ import pynini
17
+
18
+ from tn.processor import Processor
19
+
20
+
21
+ class Punctuation(Processor):
22
+ """Single ASCII punctuation marks, kept verbatim."""
23
+
24
+ def __init__(self):
25
+ super().__init__(name="punctuation", ordertype="itn")
26
+ marks = "!#$%&'()*+,-./:;<=>?@^_`{|}~"
27
+ self.graph = pynini.union(*[pynini.escape(mark) for mark in marks]).optimize()
28
+ self.tagger = self.add_tokens(self.tag_field("value", self.graph))
29
+ self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.graph))
@@ -0,0 +1,28 @@
1
+ # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ from pynini.lib.pynutil import delete
17
+
18
+ from tn.processor import Processor
19
+
20
+
21
+ class Word(Processor):
22
+ """Space-free words; a whitespace-only input verbalizes to nothing."""
23
+
24
+ def __init__(self):
25
+ super().__init__(name="word", ordertype="itn")
26
+ self.tagger = self.add_tokens(self.tag_field("value", self.NOT_SPACE.plus))
27
+ graph = self.NOT_SPACE.plus | delete(self.SPACE.plus)
28
+ self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph))
File without changes
@@ -0,0 +1,34 @@
1
+ ألفان وتسعة~2009
2
+ ألف~1000
3
+ ثلاثة اَلاف وستة~3006
4
+ ألف وسبعة وسبعون~1077
5
+ تسعة اَلاف وثمان مئة وسبعة وثلاثون~9837
6
+ ثلاثة اَلاف وتسع مئة وسبعة عشر~3917
7
+ ألفان~2000
8
+ ثلاثة اَلاف وستة~3006
9
+ ألف وسبعة وسبعون~1077
10
+ تسعة اَلاف وثمان مئة وسبعة وثلاثون~9837
11
+ ثلاثة اَلاف وتسع مئة وسبعة عشر~3917
12
+ خمسة اَلاف وخمس مئة وعشرون~5520
13
+ ثمانية اَلاف وسبع مئة وواحد وعشرون~8721
14
+ ثمانية اَلاف وتسع مئة~8900
15
+ ثمان مئة وتسعة~809
16
+ مئة وسبعة~107
17
+ ثمان مئة وسبعة وثلاثون~837
18
+ تسع مئة وسبعة عشر~917
19
+ خمس مئة وعشرون~520
20
+ اثنان~2
21
+ ثلاثة وثلاثون~33
22
+ ستون~60
23
+ ثمانية عشر~18
24
+ ستة عشر~16
25
+ اثنا عشر~12
26
+ واحد~1
27
+ عشرون~20
28
+ واحد وعشرون~21
29
+ مئة~100
30
+ أربعة وثلاثون~34
31
+ ثمانية وسبعون~78
32
+ تسعة وتسعون~99
33
+ عشرة~10
34
+ سبعون~70
@@ -0,0 +1,30 @@
1
+ ستة وثمانية وثمانون من مئة~6.88
2
+ سبعة وتسعة وتسعون من مئة~7.99
3
+ أربعة وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف~4.9876
4
+ ستة وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف~6.8944
5
+ ستة اَلاف وسبع مئة وثمانية وتسعون وتسعة من عشرة~6798.9
6
+ ست مئة وثمانية وخمسون وثمان مئة وثمانية وثمانون من ألف~658.888
7
+ اثنان وثمانية من عشرة~2.8
8
+ سبعة وتسعة من عشرة~7.9
9
+ أربعون وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف~40.9876
10
+ مئتان وواحد وسبعة من عشرة~201.7
11
+ مئة وواحد وخمسون من مئة~101.50
12
+ ثلاث مئة وأربعون من مئة~300.40
13
+ سبعة وسبعون وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف~77.8944
14
+ تسعة اَلاف وثمان مئة وستة وسبعون وتسع مئة من ألف~9876.900
15
+ ألفان وسبعة وثمان مئة وثمانية وثمانون من ألف~2007.888
16
+ سبعون وتسع مئة وتسعون من ألف~70.990
17
+ واحد وعشرون وثمان مئة واثنان وثلاثون من ألف~21.832
18
+ سبعة عشر وسبعة عشر من مئة~17.17
19
+ ستة وثمانون واثنا عشر من مئة~86.12
20
+ عشرة ومئتان من ألف~10.200
21
+ سبع مئة وتسعة وستون وثمان مئة من ألف~769.800
22
+ خمسة اَلاف وست مئة وتسعة وسبعون وثمانية اَلاف وتسع مئة وثلاثة من عشرة آلاف~5679.8903
23
+ ستة اَلاف وتسعون من مئة~6000.90
24
+ واحد وثمانية عشر من مئة~1.18
25
+ اثنان واثنان من عشرة~2.2
26
+ ثلاثة وخمسة من عشرة~3.5
27
+ خمسة وأربعة من عشرة~5.4
28
+ اثنان وثمانون وثمان مئة من ألف~82.800
29
+ مئتان وسبعة ومئتان من ألف~207.200
30
+ عشرة وواحد من عشرة~10.1
@@ -0,0 +1,20 @@
1
+ واحد و نصف~1 1/2
2
+ سبعة و عشرة أخماس~7 10/5
3
+ خمسة و ثلثين~5 2/3
4
+ عشرة و نصف~10 1/2
5
+ ثلاثون و تسعة أعشار~30 9/10
6
+ سبعة وسبعون و ثلاثة أسداس~77 3/6
7
+ ستة عشر و سبعة عشر على ثلاثة~16 17/3
8
+ مئتان و واحد على ثلاث مئة وسبعة وثمانون~200 1/387
9
+ عشرون و ثلث~20 1/3
10
+ ربع~1/4
11
+ نصف~1/2
12
+ ثلث~1/3
13
+ تسعة أثلاث~9/3
14
+ ستة و ثلثين~6 2/3
15
+ ثلثين~2/3
16
+ خمس~1/5
17
+ سدس~1/6
18
+ سبع~1/7
19
+ ثمن~1/8
20
+ سبعين~2/7
@@ -0,0 +1,15 @@
1
+ سبعة في المائة~7%
2
+ تسعة فهرنهايت~9f
3
+ اثنا عشر في المائة~12%
4
+ تسعة في المائة~9%
5
+ عشرون في المائة~20%
6
+ ستة وثمانية وثمانون من مئة في المائة~6.88%
7
+ سبعة وتسعة وتسعون من مئة في المائة~7.99%
8
+ أربعة وتسعة اَلاف وثمان مئة وستة وسبعون من عشرة آلاف في المائة~4.9876%
9
+ ستة وثمانية اَلاف وتسع مئة وأربعة وأربعون من عشرة آلاف في المائة~6.8944%
10
+ ستة اَلاف وسبع مئة وثمانية وتسعون وتسعة من عشرة في المائة~6798.9%
11
+ ست مئة وثمانية وخمسون وثمان مئة وثمانية وثمانون من ألف في المائة~658.888%
12
+ اثنان وثمانية من عشرة في المائة~2.8%
13
+ سبعة وتسعة من عشرة في المائة~7.9%
14
+ واحد في المائة~1%
15
+ ألفان وتسعة في المائة~2009%
@@ -0,0 +1,21 @@
1
+ سبعة دولار~$7
2
+ سبعة دولار وخمسون سنت~$7.50
3
+ سبعة دولار وخمسة سنتات~$7.05
4
+ سبعة دولار وتسعة وتسعون سنت~$7.99
5
+ سبعة دولار وعشرة سنتات~$7.10
6
+ تسعة وتسعون سنت~$0.99
7
+ خمسة وتسعون سنت~$0.95
8
+ تسعة يورو~€9
9
+ تسعة دولار وتسعة سنتات~$9.09
10
+ خمسة دينار كويتي~د.ك5
11
+ تسعة دينار كويتي~د.ك9
12
+ خمسة ريال سعودي~ر.س5
13
+ سبعون دولار وستون سنت~$70.60
14
+ اثنا عشر دولار~$12
15
+ تسعة دولار~$9
16
+ عشرون دولار~$20
17
+ عشرون دولار وخمسون سنت~$20.50
18
+ عشرون دولار وعشرة سنتات~$20.10
19
+ عشرة سنتات~$0.10
20
+ ثلاثة سنتات~$0.03
21
+ عشرون سنت~$0.20
@@ -0,0 +1,6 @@
1
+ السلام عليكم~السلام عليكم
2
+ صباح الخير~صباح الخير
3
+ ليلة سعيدة~ليلة سعيدة
4
+ ما اسمك~ما اسمك
5
+ أهلا وسهلا~أهلا وسهلا
6
+ كيف أصل إلى هناك، من فضلك~كيف أصل إلى هناك، من فضلك
@@ -0,0 +1,137 @@
1
+ # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2
+ # Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ from pathlib import Path
17
+
18
+ import pytest
19
+
20
+ from itn.arabic.inverse_normalizer import InverseNormalizer
21
+
22
+
23
+ def original_cases():
24
+ cases = []
25
+ for path in sorted((Path(__file__).parent / "data").glob("test_cases_*.txt")):
26
+ # Universal newlines, as in the original tests.
27
+ with path.open(encoding="utf-8") as stream:
28
+ for index, line in enumerate(stream, 1):
29
+ if line.endswith("\n"):
30
+ line = line[:-1]
31
+ spoken, written = line.split("~")
32
+ cases.append(pytest.param(spoken, written, id="{}-{}".format(path.stem, index)))
33
+ return cases
34
+
35
+
36
+ CASES = original_cases()
37
+
38
+
39
+ @pytest.fixture(scope="module")
40
+ def normalizer(shared_pipeline):
41
+ return shared_pipeline(InverseNormalizer)
42
+
43
+
44
+ def parse(normalizer, spoken):
45
+ parser = normalizer.token_parser()
46
+ parser.parse(normalizer.tag(spoken))
47
+ return [(token.name, token.members) for token in parser.tokens]
48
+
49
+
50
+ def test_original_fixture_count():
51
+ assert len(CASES) == 126
52
+ assert CASES[0].values == ("ألفان وتسعة", "2009")
53
+
54
+
55
+ @pytest.mark.parametrize("spoken, written", CASES)
56
+ def test_original(normalizer, spoken, written):
57
+ assert normalizer.normalize(spoken) == written
58
+
59
+
60
+ @pytest.mark.parametrize("spoken, written", CASES)
61
+ def test_original_mapping(normalizer, spoken, written):
62
+ result = normalizer.normalize_with_mapping(spoken, include_identity=True)
63
+ assert result.output_text == written
64
+ for mapping in result.mappings:
65
+ assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text
66
+ assert written[mapping.output_start:mapping.output_end] == mapping.output_text
67
+
68
+
69
+ def test_tag_then_verbalize_matches_normalize(normalizer):
70
+ for param in CASES:
71
+ spoken, written = param.values
72
+ assert normalizer.verbalize(normalizer.tag(spoken)) == written
73
+
74
+
75
+ @pytest.mark.parametrize("spoken, expected", [
76
+ ("تسعة اَلاف وثمان مئة وسبعة وثلاثون", [("cardinal", {"integer": "تسعة اَلاف وثمان مئة وسبعة وثلاثون"})]),
77
+ ("سالب خمسة", [("cardinal", {"negative": "سالب", "integer": "خمسة"})]),
78
+ ("سبعة وتسعة وتسعون من مئة", [("decimal", {"integer_part": "سبعة", "fractional_part": "تسعة وتسعون من مئة"})]),
79
+ ("واحد و نصف", [("fraction", {"integer_part": "واحد", "denominator": "نصف"})]),
80
+ ("ستة عشر و سبعة عشر على ثلاثة", [("fraction", {
81
+ "integer_part": "ستة عشر",
82
+ "numerator": "سبعة عشر",
83
+ "denominator": "ثلاثة"
84
+ })]),
85
+ ("سبعة دولار وخمسون سنت", [("money", {
86
+ "integer_part": "سبعة",
87
+ "currency": "دولار",
88
+ "fractional_part": "خمسون",
89
+ "currency_min": "سنت"
90
+ })]),
91
+ ("عشرون في المائة", [("measure", {"integer": "عشرون", "units": "في المائة"})]),
92
+ ])
93
+ def test_raw_field_tags(normalizer, spoken, expected):
94
+ assert parse(normalizer, spoken) == expected
95
+
96
+
97
+ @pytest.mark.parametrize("spoken, written", [
98
+ ("سالب خمسة", "-5"),
99
+ ("ثلاثة اَلاف ومئة", "3100"),
100
+ ("لدي ثلاثة كتب", "لدي 3 كتب"),
101
+ ("خمسة ريال سعودي", "ر.س5"),
102
+ ("سالب ستة وثمانية من عشرة", "-6.8"),
103
+ ])
104
+ def test_hand_written(normalizer, spoken, written):
105
+ assert normalizer.normalize(spoken) == written
106
+
107
+
108
+ @pytest.mark.parametrize("spoken", ["ثلاثة اَلاف وستة", "ستة وثمانية وثمانون من مئة", "واحد و نصف", "سبعة دولار وخمسون سنت"])
109
+ def test_nbest_mapping_synchronization(normalizer, spoken):
110
+ outputs = normalizer.normalize(spoken, nbest=3)
111
+ mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True)
112
+ assert [result.output_text for result in mapped] == outputs
113
+ assert len(outputs) == len(set(outputs))
114
+ assert outputs[0] == normalizer.normalize(spoken)
115
+
116
+
117
+ def test_escaped_word_round_trips(normalizer):
118
+ spoken = 'مسار\\a"b'
119
+ assert parse(normalizer, spoken) == [("word", {"value": spoken})]
120
+ assert normalizer.normalize(spoken) == spoken
121
+
122
+
123
+ def test_empty_input(normalizer):
124
+ assert normalizer.normalize("") == ""
125
+ assert normalizer.normalize("", nbest=3) == [""]
126
+ assert normalizer.normalize_with_mapping("").output_text == ""
127
+
128
+
129
+ @pytest.mark.parametrize("spoken", [" ", "\t", "\n", " \t\n ", "\u00a0"])
130
+ def test_whitespace_only_input(normalizer, spoken):
131
+ tagged = normalizer.tag(spoken)
132
+ assert parse(normalizer, spoken) == [("word", {"value": spoken})]
133
+ assert normalizer.verbalize(tagged) == ""
134
+ assert normalizer.normalize(spoken, nbest=3) == [""]
135
+ mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True)
136
+ assert [result.output_text for result in mapped] == [""]
137
+ assert all(mapping.output_text == "" for mapping in mapped[0].mappings)
File without changes
@@ -0,0 +1,53 @@
1
+ aed դիրհամ
2
+ ARS արգենտինական պեսո
3
+ ֏ դրամ
4
+ ֏ հայկական դրամ
5
+ a$ ավստրալական դոլար
6
+ br բելառուսական ռուբլի
7
+ ₿ բիթքոին
8
+ £ ֆունտ
9
+ £ բրիտանական ֆունտ
10
+ ca$ կանադական դոլար
11
+ 元 չինական յեն
12
+ 元 յեն
13
+ kr դանիական կրոն
14
+ $ դոլար
15
+ € եվրո
16
+ ₾ վրացական լարի
17
+ ₾ լարի
18
+ ₹ հնդկական ռուփի
19
+ ₹ ռուփի
20
+ ﷼ պարսկական ռիալ
21
+ ﷼ ռիալ
22
+ ₪ իսրայելական շեկել
23
+ ₪ շեկել
24
+ ¥ ճապոնական յեն
25
+ ¥ յեն
26
+ ₸ ղազախական տենգե
27
+ ₸ տենգե
28
+ som ղրղզական սոմ
29
+ som սոմ
30
+ ل.ل լիբանանյան ֆունտ
31
+ ل.ل լիբանանյան լիրա
32
+ $ մեքսիկական պեսո
33
+ nz$ նորզելանդական դոլլր
34
+ kr նորվեգական կրոն
35
+ zł լեհական զլոտի
36
+ zł զլոտի
37
+ £ ֆունտ ստերլինգ
38
+ £ ֆունտ
39
+ ₽ ռուսական ռուբլի
40
+ ₽ ռուբլի
41
+ rsd սերբական դինար
42
+ s$ սինգապուրի դոլար
43
+ ₩ կորեական վոն
44
+ kr շվեդական կրոն
45
+ chf շվեյցարական ֆրանկ
46
+ £s սիրիական ֆունտ
47
+ ₺ թուրքական լիրա
48
+ ₴ ուկրաինական գրիվնա
49
+ $ ամերիկյան դոլար
50
+ $ ամն դոլար
51
+ som ուզբեկական սոմ
52
+ ₩ վոն
53
+ ¥ յեն
@@ -0,0 +1,4 @@
1
+ թ. թվական
2
+ թթ. թվականներ
3
+ դ. դար
4
+ դդ. դարեր
@@ -0,0 +1,48 @@
1
+ °F ֆարենհայթ
2
+ °C ցելսիուս
3
+ K կելվին
4
+ կմ կիլոմետր
5
+ կմ/ժ կիլոմետր ժամ
6
+ մ մետր
7
+ մ/ժ մետր ժամ
8
+ սմ սանտիմետր
9
+ մմ միլիմետր
10
+ հա հեկտար
11
+ մղն մղոն
12
+ մ² քառակուսի մետր
13
+ մ² մետր քառակուսի
14
+ կմ² քառակուսի կիլոմետր
15
+ կմ² կիլոմետր քառակուսի
16
+ % տոկոս
17
+ % տոկոսադրույք
18
+ Հց հերց
19
+ կՎտ կիլովատտ
20
+ կՎտ կիլո վատտ
21
+ կՎ/Ժ կիլովատտ ժամ
22
+ կՎ/ժ կիլո վատտ ժամ
23
+ Վտ/ժ վատտ ժամ
24
+ Վտ վատտ
25
+ ձ.ու. ձիաուժ
26
+ մգ միլիգրամ
27
+ կգ կիլոգրամ
28
+ Վ վոլտ
29
+ ժ ժամ
30
+ վ վայրկյան
31
+ ր րոպե
32
+ մ³ խորանարդ մետր
33
+ գ գրամ
34
+ տ տոննա
35
+ մբ մեգաբայթ
36
+ կբ կիլոբայթ
37
+ գբ գիգաբայթ
38
+ գբ գեգաբայթ
39
+ տբ տերաբայթ
40
+ տբ տեռաբայթ
41
+ կՎ կիլո վոլտ
42
+ մՎ մեգա վոլտ
43
+ Ա ամպեր
44
+ մԱ միլի ամպեր
45
+ մվ միլի վայրկյան
46
+ դմ դեցիմետր
47
+ սմ² քառակուսի սանտիմետր
48
+ սմ² քառակուսի սանտիմետր
@@ -0,0 +1,11 @@
1
+ մեկ 1
2
+ երկու 2
3
+ երկուս 2
4
+ երեք 3
5
+ չորս 4
6
+ հինգ 5
7
+ վեց 6
8
+ յոթ 7
9
+ ութ 8
10
+ ինը 9
11
+ ինն 9
@@ -0,0 +1,10 @@
1
+ երկու 2
2
+ երկուս 2
3
+ երեք 3
4
+ չորս 4
5
+ հինգ 5
6
+ վեց 6
7
+ յոթ 7
8
+ ութ 8
9
+ ինը 9
10
+ ինն 9
@@ -0,0 +1,10 @@
1
+ տասը 1
2
+ տասն 1
3
+ քսան 2
4
+ երեսուն 3
5
+ քառասուն 4
6
+ հիսուն 5
7
+ վաթսուն 6
8
+ յոթանասուն 7
9
+ ութսուն 8
10
+ իննսուն 9
@@ -0,0 +1,10 @@
1
+ զրոերորդ զրո
2
+ առաջին մեկ
3
+ երկրորդ երկու
4
+ երրորդ երեք
5
+ չորրորդ չորս
6
+ հինգերորդ հինգ
7
+ վեցերորդ վեց
8
+ յոթերորդ յոթ
9
+ ութերորդ ութ
10
+ իններորդ ինը
@@ -0,0 +1,29 @@
1
+ զրո 0
2
+ մեկ 1
3
+ երկու 2
4
+ երկուս 2
5
+ երեք 3
6
+ չորս 4
7
+ հինգ 5
8
+ վեց 6
9
+ յոթ 7
10
+ ութ 8
11
+ ինը 9
12
+ տաս 10
13
+ տասն 10
14
+ տասնմեկ 11
15
+ տասներկու 12
16
+ տասներկուս 12
17
+ տասներեք 13
18
+ տասնչորս 14
19
+ տասնհինգ 15
20
+ տասնվեց 16
21
+ տասնյոթ 17
22
+ տասնութ 18
23
+ տասնինը 19
24
+ քսան 20
25
+ քսանմեկ 21
26
+ քսաներկու 22
27
+ քսաներկուս 22
28
+ քսաներեք 23
29
+ քսանչորս 24
@@ -0,0 +1,65 @@
1
+ մեկ 01
2
+ երկու 02
3
+ երկուս 02
4
+ երեք 03
5
+ չորս 04
6
+ հինգ 05
7
+ վեց 06
8
+ յոթ 07
9
+ ութ 08
10
+ ինը 09
11
+ տասը 10
12
+ տասնմեկ 11
13
+ տասներկու 12
14
+ տասներկուս 12
15
+ տասներեք 13
16
+ տասնչորս 14
17
+ տասնհինգ 15
18
+ տասնվեց 16
19
+ տասնյոթ 17
20
+ տասնութ 18
21
+ տասնինը 19
22
+ քսան 20
23
+ քսանմեկ 21
24
+ քսաներկու 22
25
+ քսաներկուս 22
26
+ քսաներեք 23
27
+ քսանչորս 24
28
+ քսանհինգ 25
29
+ քսանվեց 26
30
+ քսանյոթ 27
31
+ քսանութ 28
32
+ քսանինը 29
33
+ երեսուն 30
34
+ երեսունմեկ 31
35
+ երեսուներկու 32
36
+ երեսուներկուս 32
37
+ երեսուներեք 33
38
+ երեսունչորս 34
39
+ երեսունհինգ 35
40
+ երեսունվեց 36
41
+ երեսունյոթ 37
42
+ երեսունութ 38
43
+ երեսունինը 39
44
+ քառասուն 40
45
+ քառասունմեկ 41
46
+ քառասուներկու 42
47
+ քառասուներկուս 42
48
+ քառասուներեք 43
49
+ քառասունչորս 44
50
+ քառասունհինգ 45
51
+ քառասունվեց 46
52
+ քառասունյոթ 47
53
+ քառասունութ 48
54
+ քառասունինը 49
55
+ հիսուն 50
56
+ հիսունմեկ 51
57
+ հիսուներկու 52
58
+ հիսուներկուս 52
59
+ հիսուներեք 53
60
+ հիսունչորս 54
61
+ հիսունհինգ 55
62
+ հիսունվեց 56
63
+ հիսունյոթ 57
64
+ հիսունութ 58
65
+ հիսունինը 59
@@ -0,0 +1,59 @@
1
+ 01 59
2
+ 02 58
3
+ 03 57
4
+ 04 56
5
+ 05 55
6
+ 06 54
7
+ 07 53
8
+ 08 52
9
+ 09 51
10
+ 10 50
11
+ 11 49
12
+ 12 48
13
+ 13 47
14
+ 14 46
15
+ 15 45
16
+ 16 44
17
+ 17 43
18
+ 18 42
19
+ 19 41
20
+ 20 40
21
+ 21 39
22
+ 22 38
23
+ 23 37
24
+ 24 36
25
+ 25 35
26
+ 26 34
27
+ 27 33
28
+ 28 32
29
+ 29 31
30
+ 30 30
31
+ 31 29
32
+ 32 28
33
+ 33 27
34
+ 34 26
35
+ 35 25
36
+ 36 24
37
+ 37 23
38
+ 38 22
39
+ 39 21
40
+ 40 20
41
+ 41 19
42
+ 42 18
43
+ 43 17
44
+ 44 16
45
+ 45 15
46
+ 46 14
47
+ 47 13
48
+ 48 12
49
+ 49 11
50
+ 50 10
51
+ 51 09
52
+ 52 08
53
+ 53 07
54
+ 54 06
55
+ 55 05
56
+ 56 04
57
+ 57 03
58
+ 58 02
59
+ 59 01
@@ -0,0 +1,25 @@
1
+ 1 0
2
+ 2 1
3
+ 3 2
4
+ 4 3
5
+ 5 4
6
+ 6 5
7
+ 7 6
8
+ 8 7
9
+ 9 8
10
+ 10 9
11
+ 11 10
12
+ 12 11
13
+ 13 12
14
+ 14 13
15
+ 15 14
16
+ 16 15
17
+ 17 16
18
+ 18 17
19
+ 19 18
20
+ 20 19
21
+ 21 20
22
+ 22 21
23
+ 23 22
24
+ 24 23
25
+ 0 23
@@ -0,0 +1,7 @@
1
+ ս.թ. սույն թվականի
2
+ մ.թ.ա. մեր թվարկությունից առաջ
3
+ մ.թ. մեր թվարկություն
4
+ Ք.ա. քրիստոսից առաջ
5
+ Ք.հ. քրիստոսից հետո
6
+ Ք.ծ.ա. քրիստոսի ծնունդից առաջ
7
+ Ք.ծ.հ. քրիստոսից ծնունդից հետո