WeTextProcessing 1.0.6.dev19__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- itn/chinese/inverse_normalizer.py +30 -20
- itn/english/__init__.py +0 -0
- itn/english/data/currency.tsv +168 -0
- itn/english/data/date_period.tsv +10 -0
- itn/english/data/electronic/domain.tsv +10 -0
- itn/english/data/electronic/server_name.tsv +17 -0
- itn/english/data/electronic/symbols.tsv +22 -0
- itn/english/data/electronic/url_symbols.tsv +5 -0
- itn/english/data/magnitudes.tsv +4 -0
- itn/english/data/measurements.tsv +149 -0
- itn/english/data/months.tsv +12 -0
- itn/english/data/months_cased.tsv +12 -0
- itn/english/data/numbers/digit.tsv +9 -0
- itn/english/data/numbers/hundred.tsv +1 -0
- itn/english/data/numbers/teen.tsv +10 -0
- itn/english/data/numbers/thousands.tsv +22 -0
- itn/english/data/numbers/ties.tsv +9 -0
- itn/english/data/numbers/zero.tsv +1 -0
- itn/english/data/ordinals/digit.tsv +9 -0
- itn/english/data/ordinals/teen.tsv +1 -0
- itn/english/data/time/minute_to.tsv +59 -0
- itn/english/data/time/time_suffix.tsv +8 -0
- itn/english/data/time/time_suffix_cased.tsv +8 -0
- itn/english/data/time/time_zone.tsv +7 -0
- itn/english/data/time/time_zone_cased.tsv +7 -0
- itn/english/data/time/to_hour.tsv +12 -0
- itn/english/data/whitelist.tsv +417 -0
- itn/english/data/whitelist_tech.tsv +450 -0
- itn/english/data/year_suffix.tsv +6 -0
- itn/english/inverse_normalizer.py +96 -0
- itn/english/rules/__init__.py +0 -0
- itn/english/rules/cardinal.py +136 -0
- itn/english/rules/char.py +29 -0
- itn/english/rules/date.py +220 -0
- itn/english/rules/decimal.py +85 -0
- itn/english/rules/electronic.py +76 -0
- itn/english/rules/measure.py +115 -0
- itn/english/rules/money.py +145 -0
- itn/english/rules/ordinal.py +52 -0
- itn/english/rules/punctuation.py +31 -0
- itn/english/rules/telephone.py +123 -0
- itn/english/rules/time.py +123 -0
- itn/english/rules/whitelist.py +32 -0
- itn/english/rules/word.py +31 -0
- itn/english/test/__init__.py +0 -0
- itn/english/test/data/en_cardinal.txt +28 -0
- itn/english/test/data/en_date.txt +44 -0
- itn/english/test/data/en_decimal.txt +63 -0
- itn/english/test/data/en_electronic.txt +25 -0
- itn/english/test/data/en_measure.txt +114 -0
- itn/english/test/data/en_money.txt +52 -0
- itn/english/test/data/en_ordinal.txt +34 -0
- itn/english/test/data/en_telephone.txt +24 -0
- itn/english/test/data/en_time.txt +36 -0
- itn/english/test/data/en_whitelist.txt +12 -0
- itn/english/test/data/en_word.txt +55 -0
- itn/english/test/normalizer_test.py +43 -0
- itn/english/test/utils.py +32 -0
- itn/main.py +2 -12
- tn/_version.py +2 -2
- tn/chinese/data/cardinal/phone.tsv +28 -0
- tn/chinese/data/math/operator_unambiguous.tsv +6 -0
- tn/chinese/normalizer.py +2 -2
- tn/chinese/rules/cardinal.py +13 -3
- tn/chinese/rules/math.py +2 -1
- tn/chinese/rules/measure.py +5 -5
- tn/en_tn_tagger.fst +0 -0
- tn/en_tn_verbalizer.fst +0 -0
- tn/english/normalizer.py +4 -0
- tn/english/rules/serial.py +106 -0
- tn/japanese/normalizer.py +0 -1
- tn/main.py +2 -2
- tn/processor.py +18 -10
- tn/token_parser.py +6 -4
- tn/utils.py +10 -0
- tn/zh_tn_tagger.fst +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/METADATA +1 -1
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/RECORD +82 -22
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/WHEEL +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/entry_points.txt +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/licenses/LICENSE +0 -0
- {wetextprocessing-1.0.6.dev19.dist-info → wetextprocessing-1.2.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import pynini
|
|
16
|
+
from pynini import closure, cross, string_file
|
|
17
|
+
from pynini.lib.pynutil import add_weight, delete, insert
|
|
18
|
+
|
|
19
|
+
from itn.english.rules.cardinal import Cardinal
|
|
20
|
+
from itn.english.rules.ordinal import Ordinal
|
|
21
|
+
from tn.processor import Processor
|
|
22
|
+
from tn.utils import get_abs_path
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Date(Processor):
|
|
26
|
+
|
|
27
|
+
def __init__(self, cardinal=None, ordinal=None):
|
|
28
|
+
super().__init__(name="date", ordertype="itn")
|
|
29
|
+
self.cardinal = cardinal or Cardinal()
|
|
30
|
+
self.ordinal = ordinal or Ordinal(cardinal=self.cardinal)
|
|
31
|
+
self.build_tagger()
|
|
32
|
+
self.build_verbalizer()
|
|
33
|
+
|
|
34
|
+
def build_tagger(self):
|
|
35
|
+
ds = delete(" ")
|
|
36
|
+
month_names = string_file(get_abs_path("../itn/english/data/months.tsv"))
|
|
37
|
+
month = insert('month: "') + month_names + insert('"')
|
|
38
|
+
|
|
39
|
+
# Day: accept ordinal words ("fifth", "twenty first") or cardinal
|
|
40
|
+
# words ("thirty") -- both resolve to a number via the cardinal graph.
|
|
41
|
+
# Restrict to 1-31 range via composition with DIGIT{1,2}.
|
|
42
|
+
day_graph = self.ordinal.graph | self.cardinal.graph
|
|
43
|
+
day_graph = pynini.compose(day_graph, self.DIGIT + closure(self.DIGIT, 0, 1))
|
|
44
|
+
day = insert('day: "') + day_graph + insert('"')
|
|
45
|
+
|
|
46
|
+
# Year graph: handles common spoken year forms
|
|
47
|
+
digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
|
|
48
|
+
teen = string_file(get_abs_path("../itn/english/data/numbers/teen.tsv"))
|
|
49
|
+
ties = string_file(get_abs_path("../itn/english/data/numbers/ties.tsv"))
|
|
50
|
+
|
|
51
|
+
# Two-digit: 10-99
|
|
52
|
+
two_digit = teen | (ties + (ds + digit | insert("0")))
|
|
53
|
+
|
|
54
|
+
# "oh five" / "o five" => 05
|
|
55
|
+
oh_digit = (cross("oh", "0") | cross("o", "0")) + ds + digit
|
|
56
|
+
|
|
57
|
+
# Year as two groups of two digits: "twenty twelve" => 2012
|
|
58
|
+
year_two_parts = (teen | two_digit) + ds + (two_digit | oh_digit | teen)
|
|
59
|
+
# 3-digit year: "seven fifty" => 750
|
|
60
|
+
year_three_digit = digit + ds + (two_digit | oh_digit | teen)
|
|
61
|
+
|
|
62
|
+
# Year as "X thousand Y": "two thousand twelve" => 2012
|
|
63
|
+
# Need zero-padded variants so "two thousand three" => 2003
|
|
64
|
+
hundreds = digit + ds + delete("hundred") + (ds + two_digit | ds + insert("0") + digit | insert("00"))
|
|
65
|
+
up_to_999_padded = hundreds | insert("0") + two_digit | insert("00") + digit
|
|
66
|
+
year_thousands = (
|
|
67
|
+
digit
|
|
68
|
+
+ ds
|
|
69
|
+
+ delete("thousand")
|
|
70
|
+
+ (ds + up_to_999_padded | insert("000"))
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
# Year as hundreds: "nineteen oh five" => 1905
|
|
74
|
+
year_hundreds = (teen | two_digit) + ds + oh_digit
|
|
75
|
+
# Year as "X hundred": "nineteen hundred" => 1900
|
|
76
|
+
year_xx_hundred = (teen | two_digit) + ds + delete("hundred") + insert("00")
|
|
77
|
+
|
|
78
|
+
year_graph = year_two_parts | year_thousands | year_hundreds | year_xx_hundred
|
|
79
|
+
|
|
80
|
+
# Delete optional "and" within year
|
|
81
|
+
delete_and = self.build_rule(delete("and "), " ", self.ALPHA)
|
|
82
|
+
year_graph = (delete_and @ year_graph).optimize()
|
|
83
|
+
|
|
84
|
+
year = insert('year: "') + year_graph + insert('"')
|
|
85
|
+
|
|
86
|
+
# Marker to preserve field order through TokenParser
|
|
87
|
+
po = insert(' preserve_order: "true"')
|
|
88
|
+
|
|
89
|
+
# Format: month day year => "july twenty fifth two thousand twelve"
|
|
90
|
+
graph_mdy = month + ds + insert(" ") + day + ds + insert(" ") + year + po
|
|
91
|
+
# Format: month day (no year) => "january first"
|
|
92
|
+
graph_md = month + ds + insert(" ") + day + po
|
|
93
|
+
# Format: month year (no day) => "july two thousand twelve"
|
|
94
|
+
graph_my = month + ds + insert(" ") + add_weight(year, -0.1) + po
|
|
95
|
+
# Format: "the day of month year" => "the twenty fifth of july twenty twelve"
|
|
96
|
+
graph_dmy = (
|
|
97
|
+
delete("the")
|
|
98
|
+
+ ds
|
|
99
|
+
+ day
|
|
100
|
+
+ ds
|
|
101
|
+
+ delete("of")
|
|
102
|
+
+ ds
|
|
103
|
+
+ insert(" ")
|
|
104
|
+
+ month
|
|
105
|
+
+ ds
|
|
106
|
+
+ insert(" ")
|
|
107
|
+
+ year
|
|
108
|
+
+ po
|
|
109
|
+
)
|
|
110
|
+
# Format: "the day of month" (no year) => "the fifteenth of january"
|
|
111
|
+
graph_dm = (
|
|
112
|
+
delete("the")
|
|
113
|
+
+ ds
|
|
114
|
+
+ day
|
|
115
|
+
+ ds
|
|
116
|
+
+ delete("of")
|
|
117
|
+
+ ds
|
|
118
|
+
+ insert(" ")
|
|
119
|
+
+ month
|
|
120
|
+
+ po
|
|
121
|
+
)
|
|
122
|
+
# Year only => "twenty twelve", "two thousand three"
|
|
123
|
+
graph_y = year + po
|
|
124
|
+
|
|
125
|
+
# Decades: "nineteen eighties" => 1980s
|
|
126
|
+
decade_suffix = closure(self.ALPHA, 1) + (cross("ies", "y") | delete("s"))
|
|
127
|
+
decade_word = pynini.compose(decade_suffix, ties | cross("ten", "10"))
|
|
128
|
+
graph_decade = (
|
|
129
|
+
insert('year: "') + (teen | two_digit) + ds + decade_word + insert('0s"') + po
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
# Quarter: "second quarter of twenty twenty two" => Q2 2022
|
|
133
|
+
quarter_num = (
|
|
134
|
+
cross("first", "1") | cross("second", "2")
|
|
135
|
+
| cross("third", "3") | cross("fourth", "4")
|
|
136
|
+
)
|
|
137
|
+
graph_quarter = (
|
|
138
|
+
insert('day: "Q') + quarter_num + insert('"')
|
|
139
|
+
+ ds + delete("quarter") + ds + delete("of") + ds
|
|
140
|
+
+ insert(' year: "') + year_graph + insert('"') + po
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
# BC/AD/BCE/CE suffix
|
|
144
|
+
bc_ad = ds + (
|
|
145
|
+
cross("b c e", "BCE") | cross("before common era", "BCE")
|
|
146
|
+
| cross("b c", "BC")
|
|
147
|
+
| cross("c e", "CE") | cross("common era", "CE")
|
|
148
|
+
| cross("a d", "AD")
|
|
149
|
+
)
|
|
150
|
+
year_graph_with_3digit = year_graph | year_three_digit
|
|
151
|
+
graph_y_bc = insert('year: "') + year_graph_with_3digit + bc_ad + insert('"') + po
|
|
152
|
+
|
|
153
|
+
# Half: "first half of twenty twenty two" => H1 2022
|
|
154
|
+
half_num = cross("first", "1") | cross("second", "2")
|
|
155
|
+
graph_half = (
|
|
156
|
+
insert('day: "H') + half_num + insert('"')
|
|
157
|
+
+ ds + delete("half") + ds + delete("of") + ds
|
|
158
|
+
+ insert(' year: "') + year_graph + insert('"') + po
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# Century: "nineteen hundreds" => 1900s
|
|
162
|
+
graph_century = (
|
|
163
|
+
insert('year: "') + (teen | two_digit) + ds + cross("hundreds", "00s") + insert('"') + po
|
|
164
|
+
)
|
|
165
|
+
# Millennium: "two thousands" => 2000s
|
|
166
|
+
graph_millennium = (
|
|
167
|
+
insert('year: "') + cross("two", "2") + ds + cross("thousands", "000s") + insert('"') + po
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
final_graph = (
|
|
171
|
+
graph_mdy | graph_md | graph_my | graph_dmy | graph_dm | graph_y
|
|
172
|
+
| graph_decade | graph_quarter | graph_half | graph_y_bc
|
|
173
|
+
| graph_century | graph_millennium
|
|
174
|
+
)
|
|
175
|
+
self.tagger = self.add_tokens(final_graph)
|
|
176
|
+
|
|
177
|
+
def build_verbalizer(self):
|
|
178
|
+
month = (
|
|
179
|
+
delete("month:")
|
|
180
|
+
+ self.DELETE_SPACE
|
|
181
|
+
+ delete('"')
|
|
182
|
+
+ self.NOT_QUOTE.plus
|
|
183
|
+
+ delete('"')
|
|
184
|
+
)
|
|
185
|
+
day = (
|
|
186
|
+
delete("day:")
|
|
187
|
+
+ self.DELETE_SPACE
|
|
188
|
+
+ delete('"')
|
|
189
|
+
+ self.NOT_QUOTE.plus
|
|
190
|
+
+ delete('"')
|
|
191
|
+
)
|
|
192
|
+
year = (
|
|
193
|
+
delete("year:")
|
|
194
|
+
+ self.DELETE_SPACE
|
|
195
|
+
+ delete('"')
|
|
196
|
+
+ self.NOT_QUOTE.plus
|
|
197
|
+
+ delete('"')
|
|
198
|
+
)
|
|
199
|
+
delete_po = (
|
|
200
|
+
delete("preserve_order:")
|
|
201
|
+
+ self.DELETE_SPACE
|
|
202
|
+
+ delete('"')
|
|
203
|
+
+ delete("true")
|
|
204
|
+
+ delete('"')
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
optional_day = closure(self.DELETE_SPACE + insert(" ") + day, 0, 1)
|
|
208
|
+
optional_year = closure(self.DELETE_SPACE + insert(" ") + year, 0, 1)
|
|
209
|
+
|
|
210
|
+
# month (day) (year)
|
|
211
|
+
graph_mdy = month + optional_day + optional_year
|
|
212
|
+
# day month (year)
|
|
213
|
+
graph_dmy = day + self.DELETE_SPACE + insert(" ") + month + optional_year
|
|
214
|
+
# year only
|
|
215
|
+
graph_y = year
|
|
216
|
+
# day + year (for quarter: Q2 2022)
|
|
217
|
+
graph_dy = day + self.DELETE_SPACE + insert(" ") + year
|
|
218
|
+
|
|
219
|
+
graph = (graph_mdy | graph_dmy | graph_dy | graph_y) + self.DELETE_SPACE + delete_po
|
|
220
|
+
self.verbalizer = self.delete_tokens(graph)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from pynini import closure, cross, string_file, union
|
|
16
|
+
from pynini.lib.pynutil import delete, insert
|
|
17
|
+
|
|
18
|
+
from itn.english.rules.cardinal import Cardinal
|
|
19
|
+
from tn.processor import Processor
|
|
20
|
+
from tn.utils import get_abs_path, load_labels
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Decimal(Processor):
|
|
24
|
+
|
|
25
|
+
def __init__(self, cardinal=None):
|
|
26
|
+
super().__init__(name="decimal", ordertype="itn")
|
|
27
|
+
self.cardinal = cardinal or Cardinal()
|
|
28
|
+
self.build_tagger()
|
|
29
|
+
self.build_verbalizer()
|
|
30
|
+
|
|
31
|
+
def build_tagger(self):
|
|
32
|
+
digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
|
|
33
|
+
zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
|
|
34
|
+
ds = delete(" ")
|
|
35
|
+
|
|
36
|
+
# fractional part: digit by digit, "o" => 0
|
|
37
|
+
frac_digit = digit | zero | cross("o", "0")
|
|
38
|
+
frac_graph = closure(frac_digit + ds) + frac_digit
|
|
39
|
+
|
|
40
|
+
optional_negative = closure(
|
|
41
|
+
insert('negative: "true" ') + delete("minus") + ds, 0, 1
|
|
42
|
+
)
|
|
43
|
+
integer_part = insert('integer_part: "') + self.cardinal.graph + insert('"')
|
|
44
|
+
frac_part = insert('fractional_part: "') + frac_graph + insert('"')
|
|
45
|
+
point = delete("point")
|
|
46
|
+
|
|
47
|
+
graph = optional_negative + closure(integer_part + ds, 0, 1) + point + ds + frac_part
|
|
48
|
+
|
|
49
|
+
# quantity: "five point two million" => 5.2 million
|
|
50
|
+
quantities = load_labels(get_abs_path("../itn/english/data/numbers/thousands.tsv"))
|
|
51
|
+
quantity_all = union(*[x[0] for x in quantities])
|
|
52
|
+
quantity_no_thousand = union(*[x[0] for x in quantities if x[0] != "thousand"])
|
|
53
|
+
# decimal + quantity: five point two million, 164.58 thousand
|
|
54
|
+
quantity_graph = (
|
|
55
|
+
optional_negative + integer_part + ds + point + ds + frac_part
|
|
56
|
+
+ ds + insert(' quantity: "') + quantity_all + insert('"')
|
|
57
|
+
)
|
|
58
|
+
# cardinal (up to 999) + quantity: four hundred million, five million
|
|
59
|
+
# exclude thousand to let cardinal handle "ten thousand" => 10000
|
|
60
|
+
cardinal_small = self.cardinal.up_to_999
|
|
61
|
+
cardinal_quantity = (
|
|
62
|
+
optional_negative + insert('integer_part: "') + cardinal_small + insert('"')
|
|
63
|
+
+ ds + insert(' quantity: "') + quantity_no_thousand + insert('"')
|
|
64
|
+
)
|
|
65
|
+
graph |= quantity_graph | cardinal_quantity
|
|
66
|
+
|
|
67
|
+
self.tagger = self.add_tokens(graph)
|
|
68
|
+
|
|
69
|
+
def build_verbalizer(self):
|
|
70
|
+
optional_sign = closure(cross('negative: "true"', "-") + self.DELETE_SPACE, 0, 1)
|
|
71
|
+
integer = delete('integer_part:') + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
|
|
72
|
+
optional_integer = closure(integer + self.DELETE_SPACE, 0, 1)
|
|
73
|
+
fractional = (
|
|
74
|
+
insert(".") + delete('fractional_part:') + self.DELETE_SPACE
|
|
75
|
+
+ delete('"') + self.NOT_QUOTE.plus + delete('"')
|
|
76
|
+
)
|
|
77
|
+
optional_fractional = closure(fractional + self.DELETE_SPACE, 0, 1)
|
|
78
|
+
quantity = (
|
|
79
|
+
insert(" ") + delete('quantity:') + self.DELETE_SPACE
|
|
80
|
+
+ delete('"') + self.NOT_QUOTE.plus + delete('"')
|
|
81
|
+
)
|
|
82
|
+
optional_quantity = closure(quantity + self.DELETE_SPACE, 0, 1)
|
|
83
|
+
graph = optional_sign + optional_integer + optional_fractional + optional_quantity
|
|
84
|
+
self.numbers = graph
|
|
85
|
+
self.verbalizer = self.delete_tokens(graph)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from pynini import accep, closure, cross, difference, invert, string_file
|
|
16
|
+
from pynini.lib.pynutil import add_weight, delete, insert
|
|
17
|
+
|
|
18
|
+
from tn.processor import Processor
|
|
19
|
+
from tn.utils import get_abs_path
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Electronic(Processor):
|
|
23
|
+
|
|
24
|
+
def __init__(self):
|
|
25
|
+
super().__init__(name="electronic", ordertype="itn")
|
|
26
|
+
self.build_tagger()
|
|
27
|
+
self.build_verbalizer()
|
|
28
|
+
|
|
29
|
+
def build_tagger(self):
|
|
30
|
+
ds = delete(" ")
|
|
31
|
+
digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
|
|
32
|
+
zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
|
|
33
|
+
symbols = invert(string_file(get_abs_path("../itn/english/data/electronic/symbols.tsv")))
|
|
34
|
+
|
|
35
|
+
char = self.ALPHA | digit | zero
|
|
36
|
+
word = add_weight(closure(self.ALPHA, 2), 0.1)
|
|
37
|
+
token = char | symbols | word
|
|
38
|
+
first_token = char | difference(word, accep("dot"))
|
|
39
|
+
component = first_token + closure(ds + token)
|
|
40
|
+
|
|
41
|
+
dot = cross("dot", ".")
|
|
42
|
+
domain = component + (ds + dot + ds + component).plus
|
|
43
|
+
|
|
44
|
+
username = insert('username: "') + component + insert('"')
|
|
45
|
+
domain_field = insert('domain: "') + domain + insert('"')
|
|
46
|
+
|
|
47
|
+
# Email: X at Y dot Z (requires "at" keyword)
|
|
48
|
+
graph_email = username + ds + delete("at") + ds + insert(" ") + domain_field
|
|
49
|
+
|
|
50
|
+
# URL: requires protocol or www prefix
|
|
51
|
+
http = cross("h t t p", "http")
|
|
52
|
+
https = cross("h t t p s", "https")
|
|
53
|
+
protocol = (http | https) + cross(" colon slash slash ", "://")
|
|
54
|
+
www = cross("w w w", "www")
|
|
55
|
+
|
|
56
|
+
# protocol + [www.] + domain
|
|
57
|
+
url_with_protocol = protocol + closure(www + ds + dot + ds, 0, 1) + domain
|
|
58
|
+
# www. + domain (no protocol)
|
|
59
|
+
url_with_www = www + ds + dot + ds + domain
|
|
60
|
+
# domain only (must have dot): nvidia dot com
|
|
61
|
+
url_domain_only = domain
|
|
62
|
+
|
|
63
|
+
graph_url = insert('protocol: "') + (url_with_protocol | url_with_www | url_domain_only) + insert('"')
|
|
64
|
+
|
|
65
|
+
final_graph = graph_email | graph_url
|
|
66
|
+
self.tagger = self.add_tokens(final_graph)
|
|
67
|
+
|
|
68
|
+
def build_verbalizer(self):
|
|
69
|
+
username = delete("username:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
|
|
70
|
+
domain = delete("domain:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
|
|
71
|
+
protocol = delete("protocol:") + self.DELETE_SPACE + delete('"') + self.NOT_QUOTE.plus + delete('"')
|
|
72
|
+
|
|
73
|
+
graph_email = username + self.DELETE_SPACE + insert("@") + domain
|
|
74
|
+
graph_url = protocol
|
|
75
|
+
|
|
76
|
+
self.verbalizer = self.delete_tokens(graph_email | graph_url)
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import pynini
|
|
16
|
+
from pynini import closure, cross, invert, string_file
|
|
17
|
+
from pynini.lib.pynutil import add_weight, delete, insert
|
|
18
|
+
|
|
19
|
+
from itn.english.rules.cardinal import Cardinal
|
|
20
|
+
from itn.english.rules.decimal import Decimal
|
|
21
|
+
from tn.processor import Processor
|
|
22
|
+
from tn.utils import get_abs_path
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Measure(Processor):
|
|
26
|
+
|
|
27
|
+
def __init__(self, cardinal=None, decimal=None):
|
|
28
|
+
super().__init__(name="measure", ordertype="itn")
|
|
29
|
+
self.cardinal = cardinal or Cardinal()
|
|
30
|
+
self.decimal = decimal or Decimal(cardinal=self.cardinal)
|
|
31
|
+
self.build_tagger()
|
|
32
|
+
self.build_verbalizer()
|
|
33
|
+
|
|
34
|
+
def build_tagger(self):
|
|
35
|
+
ds = delete(" ")
|
|
36
|
+
|
|
37
|
+
# Load measurements: symbol\tname, invert to get name -> symbol
|
|
38
|
+
tsv_path = get_abs_path("../itn/english/data/measurements.tsv")
|
|
39
|
+
units_graph = invert(string_file(tsv_path))
|
|
40
|
+
|
|
41
|
+
# Handle plurals: generate plural->symbol mappings from the singular TSV entries
|
|
42
|
+
# Uses finite string_map instead of cdrewrite to avoid slow runtime compose
|
|
43
|
+
singular_names = {}
|
|
44
|
+
with open(tsv_path, encoding="utf-8") as f:
|
|
45
|
+
for line in f:
|
|
46
|
+
parts = line.strip().split("\t", 1)
|
|
47
|
+
if len(parts) == 2:
|
|
48
|
+
singular_names.setdefault(parts[1], parts[0])
|
|
49
|
+
|
|
50
|
+
plural_pairs = []
|
|
51
|
+
irregular_plurals = {
|
|
52
|
+
"foot": "feet", "inch": "inches",
|
|
53
|
+
"ounce": "ounces",
|
|
54
|
+
}
|
|
55
|
+
for name, symbol in singular_names.items():
|
|
56
|
+
if name in irregular_plurals:
|
|
57
|
+
plural_pairs.append((irregular_plurals[name], symbol))
|
|
58
|
+
elif name.endswith(("s", "z", "sh", "ch", "x")):
|
|
59
|
+
plural_pairs.append((name + "es", symbol))
|
|
60
|
+
elif name.endswith("y") and len(name) > 1 and name[-2] not in "aeiou":
|
|
61
|
+
plural_pairs.append((name[:-1] + "ies", symbol))
|
|
62
|
+
else:
|
|
63
|
+
plural_pairs.append((name + "s", symbol))
|
|
64
|
+
|
|
65
|
+
unit_singular = units_graph
|
|
66
|
+
unit_plural = pynini.string_map(plural_pairs)
|
|
67
|
+
|
|
68
|
+
unit = unit_singular | unit_plural
|
|
69
|
+
|
|
70
|
+
# Handle "per" units: "per hour" -> "/h"
|
|
71
|
+
per_unit = add_weight(insert("/") + delete("per") + ds + unit_singular, 1)
|
|
72
|
+
full_unit = unit + closure(ds + per_unit, 0, 1) | per_unit
|
|
73
|
+
|
|
74
|
+
# Cardinal value
|
|
75
|
+
cardinal_value = self.cardinal.graph
|
|
76
|
+
|
|
77
|
+
# Decimal value (reuse decimal's internal graph for the number)
|
|
78
|
+
decimal_digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
|
|
79
|
+
decimal_zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
|
|
80
|
+
frac_digit = decimal_digit | decimal_zero | cross("o", "0")
|
|
81
|
+
frac_graph = closure(frac_digit + ds) + frac_digit
|
|
82
|
+
decimal_value = cardinal_value + ds + delete("point") + ds + insert(".") + frac_graph
|
|
83
|
+
|
|
84
|
+
# Optional minus/negative prefix
|
|
85
|
+
minus = delete("minus") | delete("negative")
|
|
86
|
+
optional_sign = closure(insert("-") + minus + ds, 0, 1)
|
|
87
|
+
|
|
88
|
+
# "point X" with no integer part
|
|
89
|
+
point_only = delete("point") + ds + insert(".") + frac_graph
|
|
90
|
+
|
|
91
|
+
number = optional_sign + (decimal_value | cardinal_value | point_only)
|
|
92
|
+
|
|
93
|
+
value = insert('value: "') + number + insert('"')
|
|
94
|
+
units = insert('units: "') + full_unit + insert('"')
|
|
95
|
+
|
|
96
|
+
final_graph = value + ds + insert(" ") + units
|
|
97
|
+
self.tagger = self.add_tokens(final_graph)
|
|
98
|
+
|
|
99
|
+
def build_verbalizer(self):
|
|
100
|
+
value = (
|
|
101
|
+
delete("value:")
|
|
102
|
+
+ self.DELETE_SPACE
|
|
103
|
+
+ delete('"')
|
|
104
|
+
+ self.NOT_QUOTE.plus
|
|
105
|
+
+ delete('"')
|
|
106
|
+
)
|
|
107
|
+
units = (
|
|
108
|
+
delete("units:")
|
|
109
|
+
+ self.DELETE_SPACE
|
|
110
|
+
+ delete('"')
|
|
111
|
+
+ self.NOT_QUOTE.plus
|
|
112
|
+
+ delete('"')
|
|
113
|
+
)
|
|
114
|
+
graph = value + self.DELETE_SPACE + insert(" ") + units
|
|
115
|
+
self.verbalizer = self.delete_tokens(graph)
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from pynini import accep, closure, compose, cross, string_file, union
|
|
16
|
+
from pynini.lib.pynutil import add_weight, delete, insert
|
|
17
|
+
|
|
18
|
+
from itn.english.rules.cardinal import Cardinal
|
|
19
|
+
from itn.english.rules.decimal import Decimal
|
|
20
|
+
from itn.english.rules.time import _num_to_word
|
|
21
|
+
from tn.processor import Processor
|
|
22
|
+
from tn.utils import get_abs_path, load_labels
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Money(Processor):
|
|
26
|
+
|
|
27
|
+
def __init__(self, cardinal=None, decimal=None):
|
|
28
|
+
super().__init__(name="money", ordertype="itn")
|
|
29
|
+
self.cardinal = cardinal or Cardinal()
|
|
30
|
+
self.decimal = decimal or Decimal(cardinal=self.cardinal)
|
|
31
|
+
self.build_tagger()
|
|
32
|
+
self.build_verbalizer()
|
|
33
|
+
|
|
34
|
+
def build_tagger(self):
|
|
35
|
+
cardinal_graph = self.cardinal.graph
|
|
36
|
+
cardinal_small = self.cardinal.up_to_999
|
|
37
|
+
ds = delete(" ")
|
|
38
|
+
|
|
39
|
+
currency_labels = load_labels(get_abs_path("../itn/english/data/currency.tsv"))
|
|
40
|
+
singular_pairs = [(name, symbol) for symbol, name in currency_labels]
|
|
41
|
+
plural_pairs = []
|
|
42
|
+
for name, symbol in singular_pairs:
|
|
43
|
+
if name.endswith("s"):
|
|
44
|
+
plural_pairs.append((name + "es", symbol))
|
|
45
|
+
else:
|
|
46
|
+
plural_pairs.append((name + "s", symbol))
|
|
47
|
+
currency_singular = union(*[cross(name, symbol) for name, symbol in singular_pairs]).optimize()
|
|
48
|
+
currency_plural = union(*[cross(name, symbol) for name, symbol in singular_pairs + plural_pairs]).optimize()
|
|
49
|
+
|
|
50
|
+
cent = cross("cent", "") | cross("cents", "")
|
|
51
|
+
magnitudes = load_labels(get_abs_path("../itn/english/data/magnitudes.tsv"))
|
|
52
|
+
magnitude = union(*[name for symbol, name in magnitudes if name != "thousand"])
|
|
53
|
+
|
|
54
|
+
# "two dollars"
|
|
55
|
+
# add "one fifty five" => "one hundred fifty five" => 155
|
|
56
|
+
with_hundred = compose(
|
|
57
|
+
closure(self.NOT_SPACE) + accep(" ") + insert("hundred ") + self.VSIGMA,
|
|
58
|
+
compose(cardinal_graph, self.DIGIT ** 3),
|
|
59
|
+
)
|
|
60
|
+
cardinal_with_hundred = cardinal_graph | with_hundred
|
|
61
|
+
not_one = self.DIGIT ** (2, ...) | (self.DIGIT - accep("1"))
|
|
62
|
+
cardinal_plural = compose(cardinal_with_hundred, not_one)
|
|
63
|
+
# "one dollar" (singular) vs "two dollars" (plural)
|
|
64
|
+
one = cross("one", "1")
|
|
65
|
+
integer_graph = (
|
|
66
|
+
insert('value: "') + cardinal_plural + insert('"')
|
|
67
|
+
+ ds + insert(' currency: "') + currency_plural + insert('"')
|
|
68
|
+
)
|
|
69
|
+
integer_graph |= (
|
|
70
|
+
insert('value: "') + one + insert('"')
|
|
71
|
+
+ ds + insert(' currency: "') + currency_singular + insert('"')
|
|
72
|
+
)
|
|
73
|
+
# "fifty million dollars" / "four hundred billion won"
|
|
74
|
+
quantity_graph = (
|
|
75
|
+
insert('value: "') + cardinal_small + insert('"')
|
|
76
|
+
+ ds + insert(' quantity: "') + magnitude + insert('"')
|
|
77
|
+
+ ds + insert(' currency: "') + currency_plural + insert('"')
|
|
78
|
+
)
|
|
79
|
+
# "two point five billion dollars"
|
|
80
|
+
digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
|
|
81
|
+
zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
|
|
82
|
+
frac_d = digit | zero | cross("o", "0")
|
|
83
|
+
frac = closure(frac_d + ds) + frac_d
|
|
84
|
+
decimal_quantity_graph = (
|
|
85
|
+
insert('value: "') + cardinal_graph + insert(".")
|
|
86
|
+
+ ds + delete("point") + ds + frac + insert('"')
|
|
87
|
+
+ ds + insert(' quantity: "') + magnitude + insert('"')
|
|
88
|
+
+ ds + insert(' currency: "') + currency_plural + insert('"')
|
|
89
|
+
)
|
|
90
|
+
# "twenty point five o six dollars" (decimal without quantity)
|
|
91
|
+
decimal_graph = (
|
|
92
|
+
insert('value: "') + cardinal_graph + insert(".")
|
|
93
|
+
+ ds + delete("point") + ds + frac + insert('"')
|
|
94
|
+
+ ds + insert(' currency: "') + currency_plural + insert('"')
|
|
95
|
+
)
|
|
96
|
+
# "point five o six dollars"
|
|
97
|
+
decimal_no_int = (
|
|
98
|
+
insert('value: ".') + delete("point") + ds + frac + insert('"')
|
|
99
|
+
+ ds + insert(' currency: "') + currency_plural + insert('"')
|
|
100
|
+
)
|
|
101
|
+
# "one fifty five dollars" => $155 (missing "hundred")
|
|
102
|
+
with_hundred = (
|
|
103
|
+
insert('value: "') + cardinal_small + insert('"')
|
|
104
|
+
+ ds + insert(' currency: "') + currency_plural + insert('"')
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# cents
|
|
108
|
+
cents_graph = union(*[cross(_num_to_word(x), f"{x:02d}") for x in range(1, 100) if _num_to_word(x)])
|
|
109
|
+
with_cents = (
|
|
110
|
+
insert('value: "') + cardinal_graph + insert('"')
|
|
111
|
+
+ ds + insert(' currency: "') + currency_plural + insert('"')
|
|
112
|
+
+ ds + (delete("and") + ds).ques
|
|
113
|
+
+ insert(' decimal: "') + cents_graph + insert('"')
|
|
114
|
+
+ ds + cent
|
|
115
|
+
)
|
|
116
|
+
# "seventy five dollars sixty three" (no "cents" word)
|
|
117
|
+
dollars_amount = (
|
|
118
|
+
insert('value: "') + cardinal_graph + insert('"')
|
|
119
|
+
+ ds + insert(' currency: "') + currency_plural + insert('"')
|
|
120
|
+
+ ds + insert(' decimal: "') + cents_graph + insert('"')
|
|
121
|
+
)
|
|
122
|
+
cents_only = (
|
|
123
|
+
insert('currency: "$" decimal: "') + cents_graph + insert('"')
|
|
124
|
+
+ ds + cent
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
graph = (
|
|
128
|
+
integer_graph | add_weight(quantity_graph, -1) | add_weight(decimal_quantity_graph, -1)
|
|
129
|
+
| decimal_graph | decimal_no_int
|
|
130
|
+
| with_cents | dollars_amount | cents_only
|
|
131
|
+
)
|
|
132
|
+
self.tagger = self.add_tokens(graph)
|
|
133
|
+
|
|
134
|
+
def build_verbalizer(self):
|
|
135
|
+
currency = delete('currency: "') + self.NOT_QUOTE.plus + delete('"')
|
|
136
|
+
value = delete(' value: "') + self.NOT_QUOTE.plus + delete('"')
|
|
137
|
+
decimal = delete(' decimal: "') + self.NOT_QUOTE.plus + delete('"')
|
|
138
|
+
quantity = delete(' quantity: "') + self.NOT_QUOTE.plus + delete('"')
|
|
139
|
+
|
|
140
|
+
graph = currency + value
|
|
141
|
+
graph += closure(insert(".") + self.DELETE_SPACE + decimal, 0, 1)
|
|
142
|
+
graph += closure(insert(" ") + self.DELETE_SPACE + quantity, 0, 1)
|
|
143
|
+
graph |= currency + insert("0.") + self.DELETE_SPACE + decimal
|
|
144
|
+
|
|
145
|
+
self.verbalizer = self.delete_tokens(graph)
|