WeTextProcessing 1.0.6.dev19__py3-none-any.whl → 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. itn/chinese/inverse_normalizer.py +30 -20
  2. itn/english/__init__.py +0 -0
  3. itn/english/data/currency.tsv +168 -0
  4. itn/english/data/date_period.tsv +10 -0
  5. itn/english/data/electronic/domain.tsv +10 -0
  6. itn/english/data/electronic/server_name.tsv +17 -0
  7. itn/english/data/electronic/symbols.tsv +22 -0
  8. itn/english/data/electronic/url_symbols.tsv +5 -0
  9. itn/english/data/magnitudes.tsv +4 -0
  10. itn/english/data/measurements.tsv +149 -0
  11. itn/english/data/months.tsv +12 -0
  12. itn/english/data/months_cased.tsv +12 -0
  13. itn/english/data/numbers/digit.tsv +9 -0
  14. itn/english/data/numbers/hundred.tsv +1 -0
  15. itn/english/data/numbers/teen.tsv +10 -0
  16. itn/english/data/numbers/thousands.tsv +22 -0
  17. itn/english/data/numbers/ties.tsv +9 -0
  18. itn/english/data/numbers/zero.tsv +1 -0
  19. itn/english/data/ordinals/digit.tsv +9 -0
  20. itn/english/data/ordinals/teen.tsv +1 -0
  21. itn/english/data/time/minute_to.tsv +59 -0
  22. itn/english/data/time/time_suffix.tsv +8 -0
  23. itn/english/data/time/time_suffix_cased.tsv +8 -0
  24. itn/english/data/time/time_zone.tsv +7 -0
  25. itn/english/data/time/time_zone_cased.tsv +7 -0
  26. itn/english/data/time/to_hour.tsv +12 -0
  27. itn/english/data/whitelist.tsv +417 -0
  28. itn/english/data/whitelist_tech.tsv +450 -0
  29. itn/english/data/year_suffix.tsv +6 -0
  30. itn/english/inverse_normalizer.py +96 -0
  31. itn/english/rules/__init__.py +0 -0
  32. itn/english/rules/cardinal.py +136 -0
  33. itn/english/rules/char.py +29 -0
  34. itn/english/rules/date.py +220 -0
  35. itn/english/rules/decimal.py +85 -0
  36. itn/english/rules/electronic.py +76 -0
  37. itn/english/rules/measure.py +115 -0
  38. itn/english/rules/money.py +145 -0
  39. itn/english/rules/ordinal.py +52 -0
  40. itn/english/rules/punctuation.py +31 -0
  41. itn/english/rules/telephone.py +123 -0
  42. itn/english/rules/time.py +123 -0
  43. itn/english/rules/whitelist.py +32 -0
  44. itn/english/rules/word.py +31 -0
  45. itn/english/test/__init__.py +0 -0
  46. itn/english/test/data/en_cardinal.txt +28 -0
  47. itn/english/test/data/en_date.txt +44 -0
  48. itn/english/test/data/en_decimal.txt +63 -0
  49. itn/english/test/data/en_electronic.txt +25 -0
  50. itn/english/test/data/en_measure.txt +114 -0
  51. itn/english/test/data/en_money.txt +52 -0
  52. itn/english/test/data/en_ordinal.txt +34 -0
  53. itn/english/test/data/en_telephone.txt +24 -0
  54. itn/english/test/data/en_time.txt +36 -0
  55. itn/english/test/data/en_whitelist.txt +12 -0
  56. itn/english/test/data/en_word.txt +55 -0
  57. itn/english/test/normalizer_test.py +43 -0
  58. itn/english/test/utils.py +32 -0
  59. itn/main.py +2 -12
  60. tn/_version.py +2 -2
  61. tn/chinese/data/cardinal/phone.tsv +28 -0
  62. tn/chinese/data/math/operator_unambiguous.tsv +6 -0
  63. tn/chinese/normalizer.py +2 -2
  64. tn/chinese/rules/cardinal.py +13 -3
  65. tn/chinese/rules/math.py +2 -1
  66. tn/chinese/rules/measure.py +5 -5
  67. tn/en_tn_tagger.fst +0 -0
  68. tn/en_tn_verbalizer.fst +0 -0
  69. tn/english/normalizer.py +4 -0
  70. tn/english/rules/serial.py +106 -0
  71. tn/japanese/normalizer.py +0 -1
  72. tn/main.py +2 -2
  73. tn/processor.py +18 -10
  74. tn/token_parser.py +6 -4
  75. tn/utils.py +10 -0
  76. tn/zh_tn_tagger.fst +0 -0
  77. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/METADATA +1 -1
  78. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/RECORD +82 -22
  79. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/WHEEL +0 -0
  80. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/entry_points.txt +0 -0
  81. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/licenses/LICENSE +0 -0
  82. {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,220 @@
1
+ # Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import pynini
16
+ from pynini import closure, cross, string_file
17
+ from pynini.lib.pynutil import add_weight, delete, insert
18
+
19
+ from itn.english.rules.cardinal import Cardinal
20
+ from itn.english.rules.ordinal import Ordinal
21
+ from tn.processor import Processor
22
+ from tn.utils import get_abs_path
23
+
24
+
25
+ class Date(Processor):
26
+
27
+ def __init__(self, cardinal=None, ordinal=None):
28
+ super().__init__(name="date", ordertype="itn")
29
+ self.cardinal = cardinal or Cardinal()
30
+ self.ordinal = ordinal or Ordinal(cardinal=self.cardinal)
31
+ self.build_tagger()
32
+ self.build_verbalizer()
33
+
34
+ def build_tagger(self):
35
+ ds = delete(" ")
36
+ month_names = string_file(get_abs_path("../itn/english/data/months.tsv"))
37
+ month = insert('month: "') + month_names + insert('"')
38
+
39
+ # Day: accept ordinal words ("fifth", "twenty first") or cardinal
40
+ # words ("thirty") -- both resolve to a number via the cardinal graph.
41
+ # Restrict to 1-31 range via composition with DIGIT{1,2}.
42
+ day_graph = self.ordinal.graph | self.cardinal.graph
43
+ day_graph = pynini.compose(day_graph, self.DIGIT + closure(self.DIGIT, 0, 1))
44
+ day = insert('day: "') + day_graph + insert('"')
45
+
46
+ # Year graph: handles common spoken year forms
47
+ digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
48
+ teen = string_file(get_abs_path("../itn/english/data/numbers/teen.tsv"))
49
+ ties = string_file(get_abs_path("../itn/english/data/numbers/ties.tsv"))
50
+
51
+ # Two-digit: 10-99
52
+ two_digit = teen | (ties + (ds + digit | insert("0")))
53
+
54
+ # "oh five" / "o five" => 05
55
+ oh_digit = (cross("oh", "0") | cross("o", "0")) + ds + digit
56
+
57
+ # Year as two groups of two digits: "twenty twelve" => 2012
58
+ year_two_parts = (teen | two_digit) + ds + (two_digit | oh_digit | teen)
59
+ # 3-digit year: "seven fifty" => 750
60
+ year_three_digit = digit + ds + (two_digit | oh_digit | teen)
61
+
62
+ # Year as "X thousand Y": "two thousand twelve" => 2012
63
+ # Need zero-padded variants so "two thousand three" => 2003
64
+ hundreds = digit + ds + delete("hundred") + (ds + two_digit | ds + insert("0") + digit | insert("00"))
65
+ up_to_999_padded = hundreds | insert("0") + two_digit | insert("00") + digit
66
+ year_thousands = (
67
+ digit
68
+ + ds
69
+ + delete("thousand")
70
+ + (ds + up_to_999_padded | insert("000"))
71
+ )
72
+
73
+ # Year as hundreds: "nineteen oh five" => 1905
74
+ year_hundreds = (teen | two_digit) + ds + oh_digit
75
+ # Year as "X hundred": "nineteen hundred" => 1900
76
+ year_xx_hundred = (teen | two_digit) + ds + delete("hundred") + insert("00")
77
+
78
+ year_graph = year_two_parts | year_thousands | year_hundreds | year_xx_hundred
79
+
80
+ # Delete optional "and" within year
81
+ delete_and = self.build_rule(delete("and "), " ", self.ALPHA)
82
+ year_graph = (delete_and @ year_graph).optimize()
83
+
84
+ year = insert('year: "') + year_graph + insert('"')
85
+
86
+ # Marker to preserve field order through TokenParser
87
+ po = insert(' preserve_order: "true"')
88
+
89
+ # Format: month day year => "july twenty fifth two thousand twelve"
90
+ graph_mdy = month + ds + insert(" ") + day + ds + insert(" ") + year + po
91
+ # Format: month day (no year) => "january first"
92
+ graph_md = month + ds + insert(" ") + day + po
93
+ # Format: month year (no day) => "july two thousand twelve"
94
+ graph_my = month + ds + insert(" ") + add_weight(year, -0.1) + po
95
+ # Format: "the day of month year" => "the twenty fifth of july twenty twelve"
96
+ graph_dmy = (
97
+ delete("the")
98
+ + ds
99
+ + day
100
+ + ds
101
+ + delete("of")
102
+ + ds
103
+ + insert(" ")
104
+ + month
105
+ + ds
106
+ + insert(" ")
107
+ + year
108
+ + po
109
+ )
110
+ # Format: "the day of month" (no year) => "the fifteenth of january"
111
+ graph_dm = (
112
+ delete("the")
113
+ + ds
114
+ + day
115
+ + ds
116
+ + delete("of")
117
+ + ds
118
+ + insert(" ")
119
+ + month
120
+ + po
121
+ )
122
+ # Year only => "twenty twelve", "two thousand three"
123
+ graph_y = year + po
124
+
125
+ # Decades: "nineteen eighties" => 1980s
126
+ decade_suffix = closure(self.ALPHA, 1) + (cross("ies", "y") | delete("s"))
127
+ decade_word = pynini.compose(decade_suffix, ties | cross("ten", "10"))
128
+ graph_decade = (
129
+ insert('year: "') + (teen | two_digit) + ds + decade_word + insert('0s"') + po
130
+ )
131
+
132
+ # Quarter: "second quarter of twenty twenty two" => Q2 2022
133
+ quarter_num = (
134
+ cross("first", "1") | cross("second", "2")
135
+ | cross("third", "3") | cross("fourth", "4")
136
+ )
137
+ graph_quarter = (
138
+ insert('day: "Q') + quarter_num + insert('"')
139
+ + ds + delete("quarter") + ds + delete("of") + ds
140
+ + insert(' year: "') + year_graph + insert('"') + po
141
+ )
142
+
143
+ # BC/AD/BCE/CE suffix
144
+ bc_ad = ds + (
145
+ cross("b c e", "BCE") | cross("before common era", "BCE")
146
+ | cross("b c", "BC")
147
+ | cross("c e", "CE") | cross("common era", "CE")
148
+ | cross("a d", "AD")
149
+ )
150
+ year_graph_with_3digit = year_graph | year_three_digit
151
+ graph_y_bc = insert('year: "') + year_graph_with_3digit + bc_ad + insert('"') + po
152
+
153
+ # Half: "first half of twenty twenty two" => H1 2022
154
+ half_num = cross("first", "1") | cross("second", "2")
155
+ graph_half = (
156
+ insert('day: "H') + half_num + insert('"')
157
+ + ds + delete("half") + ds + delete("of") + ds
158
+ + insert(' year: "') + year_graph + insert('"') + po
159
+ )
160
+
161
+ # Century: "nineteen hundreds" => 1900s
162
+ graph_century = (
163
+ insert('year: "') + (teen | two_digit) + ds + cross("hundreds", "00s") + insert('"') + po
164
+ )
165
+ # Millennium: "two thousands" => 2000s
166
+ graph_millennium = (
167
+ insert('year: "') + cross("two", "2") + ds + cross("thousands", "000s") + insert('"') + po
168
+ )
169
+
170
+ final_graph = (
171
+ graph_mdy | graph_md | graph_my | graph_dmy | graph_dm | graph_y
172
+ | graph_decade | graph_quarter | graph_half | graph_y_bc
173
+ | graph_century | graph_millennium
174
+ )
175
+ self.tagger = self.add_tokens(final_graph)
176
+
177
+ def build_verbalizer(self):
178
+ month = (
179
+ delete("month:")
180
+ + self.DELETE_SPACE
181
+ + delete('"')
182
+ + self.NOT_QUOTE.plus
183
+ + delete('"')
184
+ )
185
+ day = (
186
+ delete("day:")
187
+ + self.DELETE_SPACE
188
+ + delete('"')
189
+ + self.NOT_QUOTE.plus
190
+ + delete('"')
191
+ )
192
+ year = (
193
+ delete("year:")
194
+ + self.DELETE_SPACE
195
+ + delete('"')
196
+ + self.NOT_QUOTE.plus
197
+ + delete('"')
198
+ )
199
+ delete_po = (
200
+ delete("preserve_order:")
201
+ + self.DELETE_SPACE
202
+ + delete('"')
203
+ + delete("true")
204
+ + delete('"')
205
+ )
206
+
207
+ optional_day = closure(self.DELETE_SPACE + insert(" ") + day, 0, 1)
208
+ optional_year = closure(self.DELETE_SPACE + insert(" ") + year, 0, 1)
209
+
210
+ # month (day) (year)
211
+ graph_mdy = month + optional_day + optional_year
212
+ # day month (year)
213
+ graph_dmy = day + self.DELETE_SPACE + insert(" ") + month + optional_year
214
+ # year only
215
+ graph_y = year
216
+ # day + year (for quarter: Q2 2022)
217
+ graph_dy = day + self.DELETE_SPACE + insert(" ") + year
218
+
219
+ graph = (graph_mdy | graph_dmy | graph_dy | graph_y) + self.DELETE_SPACE + delete_po
220
+ self.verbalizer = self.delete_tokens(graph)
@@ -0,0 +1,85 @@
1
+ # Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from pynini import closure, cross, string_file, union
16
+ from pynini.lib.pynutil import delete, insert
17
+
18
+ from itn.english.rules.cardinal import Cardinal
19
+ from tn.processor import Processor
20
+ from tn.utils import get_abs_path, load_labels
21
+
22
+
23
+ class Decimal(Processor):
24
+
25
+ def __init__(self, cardinal=None):
26
+ super().__init__(name="decimal", ordertype="itn")
27
+ self.cardinal = cardinal or Cardinal()
28
+ self.build_tagger()
29
+ self.build_verbalizer()
30
+
31
+ def build_tagger(self):
32
+ digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
33
+ zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
34
+ ds = delete(" ")
35
+
36
+ # fractional part: digit by digit, "o" => 0
37
+ frac_digit = digit | zero | cross("o", "0")
38
+ frac_graph = closure(frac_digit + ds) + frac_digit
39
+
40
+ optional_negative = closure(
41
+ insert('negative: "true" ') + delete("minus") + ds, 0, 1
42
+ )
43
+ integer_part = insert('integer_part: "') + self.cardinal.graph + insert('"')
44
+ frac_part = insert('fractional_part: "') + frac_graph + insert('"')
45
+ point = delete("point")
46
+
47
+ graph = optional_negative + closure(integer_part + ds, 0, 1) + point + ds + frac_part
48
+
49
+ # quantity: "five point two million" => 5.2 million
50
+ quantities = load_labels(get_abs_path("../itn/english/data/numbers/thousands.tsv"))
51
+ quantity_all = union(*[x[0] for x in quantities])
52
+ quantity_no_thousand = union(*[x[0] for x in quantities if x[0] != "thousand"])
53
+ # decimal + quantity: five point two million, 164.58 thousand
54
+ quantity_graph = (
55
+ optional_negative + integer_part + ds + point + ds + frac_part
56
+ + ds + insert(' quantity: "') + quantity_all + insert('"')
57
+ )
58
+ # cardinal (up to 999) + quantity: four hundred million, five million
59
+ # exclude thousand to let cardinal handle "ten thousand" => 10000
60
+ cardinal_small = self.cardinal.up_to_999
61
+ cardinal_quantity = (
62
+ optional_negative + insert('integer_part: "') + cardinal_small + insert('"')
63
+ + ds + insert(' quantity: "') + quantity_no_thousand + insert('"')
64
+ )
65
+ graph |= quantity_graph | cardinal_quantity
66
+
67
+ self.tagger = self.add_tokens(graph)
68
+
69
+ def build_verbalizer(self):
70
+ optional_sign = closure(cross('negative: "true"', "-") + self.DELETE_SPACE, 0, 1)
71
+ integer = delete('integer_part:') + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
72
+ optional_integer = closure(integer + self.DELETE_SPACE, 0, 1)
73
+ fractional = (
74
+ insert(".") + delete('fractional_part:') + self.DELETE_SPACE
75
+ + delete('"') + self.NOT_QUOTE.plus + delete('"')
76
+ )
77
+ optional_fractional = closure(fractional + self.DELETE_SPACE, 0, 1)
78
+ quantity = (
79
+ insert(" ") + delete('quantity:') + self.DELETE_SPACE
80
+ + delete('"') + self.NOT_QUOTE.plus + delete('"')
81
+ )
82
+ optional_quantity = closure(quantity + self.DELETE_SPACE, 0, 1)
83
+ graph = optional_sign + optional_integer + optional_fractional + optional_quantity
84
+ self.numbers = graph
85
+ self.verbalizer = self.delete_tokens(graph)
@@ -0,0 +1,76 @@
1
+ # Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from pynini import accep, closure, cross, difference, invert, string_file
16
+ from pynini.lib.pynutil import add_weight, delete, insert
17
+
18
+ from tn.processor import Processor
19
+ from tn.utils import get_abs_path
20
+
21
+
22
+ class Electronic(Processor):
23
+
24
+ def __init__(self):
25
+ super().__init__(name="electronic", ordertype="itn")
26
+ self.build_tagger()
27
+ self.build_verbalizer()
28
+
29
+ def build_tagger(self):
30
+ ds = delete(" ")
31
+ digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
32
+ zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
33
+ symbols = invert(string_file(get_abs_path("../itn/english/data/electronic/symbols.tsv")))
34
+
35
+ char = self.ALPHA | digit | zero
36
+ word = add_weight(closure(self.ALPHA, 2), 0.1)
37
+ token = char | symbols | word
38
+ first_token = char | difference(word, accep("dot"))
39
+ component = first_token + closure(ds + token)
40
+
41
+ dot = cross("dot", ".")
42
+ domain = component + (ds + dot + ds + component).plus
43
+
44
+ username = insert('username: "') + component + insert('"')
45
+ domain_field = insert('domain: "') + domain + insert('"')
46
+
47
+ # Email: X at Y dot Z (requires "at" keyword)
48
+ graph_email = username + ds + delete("at") + ds + insert(" ") + domain_field
49
+
50
+ # URL: requires protocol or www prefix
51
+ http = cross("h t t p", "http")
52
+ https = cross("h t t p s", "https")
53
+ protocol = (http | https) + cross(" colon slash slash ", "://")
54
+ www = cross("w w w", "www")
55
+
56
+ # protocol + [www.] + domain
57
+ url_with_protocol = protocol + closure(www + ds + dot + ds, 0, 1) + domain
58
+ # www. + domain (no protocol)
59
+ url_with_www = www + ds + dot + ds + domain
60
+ # domain only (must have dot): nvidia dot com
61
+ url_domain_only = domain
62
+
63
+ graph_url = insert('protocol: "') + (url_with_protocol | url_with_www | url_domain_only) + insert('"')
64
+
65
+ final_graph = graph_email | graph_url
66
+ self.tagger = self.add_tokens(final_graph)
67
+
68
+ def build_verbalizer(self):
69
+ username = delete("username:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
70
+ domain = delete("domain:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
71
+ protocol = delete("protocol:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
72
+
73
+ graph_email = username + self.DELETE_SPACE + insert("@") + domain
74
+ graph_url = protocol
75
+
76
+ self.verbalizer = self.delete_tokens(graph_email | graph_url)
@@ -0,0 +1,115 @@
1
+ # Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import pynini
16
+ from pynini import closure, cross, invert, string_file
17
+ from pynini.lib.pynutil import add_weight, delete, insert
18
+
19
+ from itn.english.rules.cardinal import Cardinal
20
+ from itn.english.rules.decimal import Decimal
21
+ from tn.processor import Processor
22
+ from tn.utils import get_abs_path
23
+
24
+
25
+ class Measure(Processor):
26
+
27
+ def __init__(self, cardinal=None, decimal=None):
28
+ super().__init__(name="measure", ordertype="itn")
29
+ self.cardinal = cardinal or Cardinal()
30
+ self.decimal = decimal or Decimal(cardinal=self.cardinal)
31
+ self.build_tagger()
32
+ self.build_verbalizer()
33
+
34
+ def build_tagger(self):
35
+ ds = delete(" ")
36
+
37
+ # Load measurements: symbol\tname, invert to get name -> symbol
38
+ tsv_path = get_abs_path("../itn/english/data/measurements.tsv")
39
+ units_graph = invert(string_file(tsv_path))
40
+
41
+ # Handle plurals: generate plural->symbol mappings from the singular TSV entries
42
+ # Uses finite string_map instead of cdrewrite to avoid slow runtime compose
43
+ singular_names = {}
44
+ with open(tsv_path, encoding="utf-8") as f:
45
+ for line in f:
46
+ parts = line.strip().split("\t", 1)
47
+ if len(parts) == 2:
48
+ singular_names.setdefault(parts[1], parts[0])
49
+
50
+ plural_pairs = []
51
+ irregular_plurals = {
52
+ "foot": "feet", "inch": "inches",
53
+ "ounce": "ounces",
54
+ }
55
+ for name, symbol in singular_names.items():
56
+ if name in irregular_plurals:
57
+ plural_pairs.append((irregular_plurals[name], symbol))
58
+ elif name.endswith(("s", "z", "sh", "ch", "x")):
59
+ plural_pairs.append((name + "es", symbol))
60
+ elif name.endswith("y") and len(name) > 1 and name[-2] not in "aeiou":
61
+ plural_pairs.append((name[:-1] + "ies", symbol))
62
+ else:
63
+ plural_pairs.append((name + "s", symbol))
64
+
65
+ unit_singular = units_graph
66
+ unit_plural = pynini.string_map(plural_pairs)
67
+
68
+ unit = unit_singular | unit_plural
69
+
70
+ # Handle "per" units: "per hour" -> "/h"
71
+ per_unit = add_weight(insert("/") + delete("per") + ds + unit_singular, 1)
72
+ full_unit = unit + closure(ds + per_unit, 0, 1) | per_unit
73
+
74
+ # Cardinal value
75
+ cardinal_value = self.cardinal.graph
76
+
77
+ # Decimal value (reuse decimal's internal graph for the number)
78
+ decimal_digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
79
+ decimal_zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
80
+ frac_digit = decimal_digit | decimal_zero | cross("o", "0")
81
+ frac_graph = closure(frac_digit + ds) + frac_digit
82
+ decimal_value = cardinal_value + ds + delete("point") + ds + insert(".") + frac_graph
83
+
84
+ # Optional minus/negative prefix
85
+ minus = delete("minus") | delete("negative")
86
+ optional_sign = closure(insert("-") + minus + ds, 0, 1)
87
+
88
+ # "point X" with no integer part
89
+ point_only = delete("point") + ds + insert(".") + frac_graph
90
+
91
+ number = optional_sign + (decimal_value | cardinal_value | point_only)
92
+
93
+ value = insert('value: "') + number + insert('"')
94
+ units = insert('units: "') + full_unit + insert('"')
95
+
96
+ final_graph = value + ds + insert(" ") + units
97
+ self.tagger = self.add_tokens(final_graph)
98
+
99
+ def build_verbalizer(self):
100
+ value = (
101
+ delete("value:")
102
+ + self.DELETE_SPACE
103
+ + delete('"')
104
+ + self.NOT_QUOTE.plus
105
+ + delete('"')
106
+ )
107
+ units = (
108
+ delete("units:")
109
+ + self.DELETE_SPACE
110
+ + delete('"')
111
+ + self.NOT_QUOTE.plus
112
+ + delete('"')
113
+ )
114
+ graph = value + self.DELETE_SPACE + insert(" ") + units
115
+ self.verbalizer = self.delete_tokens(graph)
@@ -0,0 +1,145 @@
1
+ # Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from pynini import accep, closure, compose, cross, string_file, union
16
+ from pynini.lib.pynutil import add_weight, delete, insert
17
+
18
+ from itn.english.rules.cardinal import Cardinal
19
+ from itn.english.rules.decimal import Decimal
20
+ from itn.english.rules.time import _num_to_word
21
+ from tn.processor import Processor
22
+ from tn.utils import get_abs_path, load_labels
23
+
24
+
25
+ class Money(Processor):
26
+
27
+ def __init__(self, cardinal=None, decimal=None):
28
+ super().__init__(name="money", ordertype="itn")
29
+ self.cardinal = cardinal or Cardinal()
30
+ self.decimal = decimal or Decimal(cardinal=self.cardinal)
31
+ self.build_tagger()
32
+ self.build_verbalizer()
33
+
34
+ def build_tagger(self):
35
+ cardinal_graph = self.cardinal.graph
36
+ cardinal_small = self.cardinal.up_to_999
37
+ ds = delete(" ")
38
+
39
+ currency_labels = load_labels(get_abs_path("../itn/english/data/currency.tsv"))
40
+ singular_pairs = [(name, symbol) for symbol, name in currency_labels]
41
+ plural_pairs = []
42
+ for name, symbol in singular_pairs:
43
+ if name.endswith("s"):
44
+ plural_pairs.append((name + "es", symbol))
45
+ else:
46
+ plural_pairs.append((name + "s", symbol))
47
+ currency_singular = union(*[cross(name, symbol) for name, symbol in singular_pairs]).optimize()
48
+ currency_plural = union(*[cross(name, symbol) for name, symbol in singular_pairs + plural_pairs]).optimize()
49
+
50
+ cent = cross("cent", "") | cross("cents", "")
51
+ magnitudes = load_labels(get_abs_path("../itn/english/data/magnitudes.tsv"))
52
+ magnitude = union(*[name for symbol, name in magnitudes if name != "thousand"])
53
+
54
+ # "two dollars"
55
+ # add "one fifty five" => "one hundred fifty five" => 155
56
+ with_hundred = compose(
57
+ closure(self.NOT_SPACE) + accep(" ") + insert("hundred ") + self.VSIGMA,
58
+ compose(cardinal_graph, self.DIGIT ** 3),
59
+ )
60
+ cardinal_with_hundred = cardinal_graph | with_hundred
61
+ not_one = self.DIGIT ** (2, ...) | (self.DIGIT - accep("1"))
62
+ cardinal_plural = compose(cardinal_with_hundred, not_one)
63
+ # "one dollar" (singular) vs "two dollars" (plural)
64
+ one = cross("one", "1")
65
+ integer_graph = (
66
+ insert('value: "') + cardinal_plural + insert('"')
67
+ + ds + insert(' currency: "') + currency_plural + insert('"')
68
+ )
69
+ integer_graph |= (
70
+ insert('value: "') + one + insert('"')
71
+ + ds + insert(' currency: "') + currency_singular + insert('"')
72
+ )
73
+ # "fifty million dollars" / "four hundred billion won"
74
+ quantity_graph = (
75
+ insert('value: "') + cardinal_small + insert('"')
76
+ + ds + insert(' quantity: "') + magnitude + insert('"')
77
+ + ds + insert(' currency: "') + currency_plural + insert('"')
78
+ )
79
+ # "two point five billion dollars"
80
+ digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
81
+ zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
82
+ frac_d = digit | zero | cross("o", "0")
83
+ frac = closure(frac_d + ds) + frac_d
84
+ decimal_quantity_graph = (
85
+ insert('value: "') + cardinal_graph + insert(".")
86
+ + ds + delete("point") + ds + frac + insert('"')
87
+ + ds + insert(' quantity: "') + magnitude + insert('"')
88
+ + ds + insert(' currency: "') + currency_plural + insert('"')
89
+ )
90
+ # "twenty point five o six dollars" (decimal without quantity)
91
+ decimal_graph = (
92
+ insert('value: "') + cardinal_graph + insert(".")
93
+ + ds + delete("point") + ds + frac + insert('"')
94
+ + ds + insert(' currency: "') + currency_plural + insert('"')
95
+ )
96
+ # "point five o six dollars"
97
+ decimal_no_int = (
98
+ insert('value: ".') + delete("point") + ds + frac + insert('"')
99
+ + ds + insert(' currency: "') + currency_plural + insert('"')
100
+ )
101
+ # "one fifty five dollars" => $155 (missing "hundred")
102
+ with_hundred = (
103
+ insert('value: "') + cardinal_small + insert('"')
104
+ + ds + insert(' currency: "') + currency_plural + insert('"')
105
+ )
106
+
107
+ # cents
108
+ cents_graph = union(*[cross(_num_to_word(x), f"{x:02d}") for x in range(1, 100) if _num_to_word(x)])
109
+ with_cents = (
110
+ insert('value: "') + cardinal_graph + insert('"')
111
+ + ds + insert(' currency: "') + currency_plural + insert('"')
112
+ + ds + (delete("and") + ds).ques
113
+ + insert(' decimal: "') + cents_graph + insert('"')
114
+ + ds + cent
115
+ )
116
+ # "seventy five dollars sixty three" (no "cents" word)
117
+ dollars_amount = (
118
+ insert('value: "') + cardinal_graph + insert('"')
119
+ + ds + insert(' currency: "') + currency_plural + insert('"')
120
+ + ds + insert(' decimal: "') + cents_graph + insert('"')
121
+ )
122
+ cents_only = (
123
+ insert('currency: "$" decimal: "') + cents_graph + insert('"')
124
+ + ds + cent
125
+ )
126
+
127
+ graph = (
128
+ integer_graph | add_weight(quantity_graph, -1) | add_weight(decimal_quantity_graph, -1)
129
+ | decimal_graph | decimal_no_int
130
+ | with_cents | dollars_amount | cents_only
131
+ )
132
+ self.tagger = self.add_tokens(graph)
133
+
134
+ def build_verbalizer(self):
135
+ currency = delete('currency: "') + self.NOT_QUOTE.plus + delete('"')
136
+ value = delete(' value: "') + self.NOT_QUOTE.plus + delete('"')
137
+ decimal = delete(' decimal: "') + self.NOT_QUOTE.plus + delete('"')
138
+ quantity = delete(' quantity: "') + self.NOT_QUOTE.plus + delete('"')
139
+
140
+ graph = currency + value
141
+ graph += closure(insert(".") + self.DELETE_SPACE + decimal, 0, 1)
142
+ graph += closure(insert(" ") + self.DELETE_SPACE + quantity, 0, 1)
143
+ graph |= currency + insert("0.") + self.DELETE_SPACE + decimal
144
+
145
+ self.verbalizer = self.delete_tokens(graph)