mathparse 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mathparse/__init__.py +1 -1
- mathparse/mathparse.py +265 -11
- mathparse/mathwords.py +1280 -39
- {mathparse-0.2.2.dist-info → mathparse-0.2.4.dist-info}/METADATA +1 -1
- mathparse-0.2.4.dist-info/RECORD +8 -0
- mathparse-0.2.2.dist-info/RECORD +0 -8
- {mathparse-0.2.2.dist-info → mathparse-0.2.4.dist-info}/WHEEL +0 -0
- {mathparse-0.2.2.dist-info → mathparse-0.2.4.dist-info}/licenses/LICENSE +0 -0
- {mathparse-0.2.2.dist-info → mathparse-0.2.4.dist-info}/top_level.txt +0 -0
mathparse/__init__.py
CHANGED
mathparse/mathparse.py
CHANGED
|
@@ -79,6 +79,41 @@ def is_word(word: str, language: str) -> bool:
|
|
|
79
79
|
return word in words
|
|
80
80
|
|
|
81
81
|
|
|
82
|
+
def create_unicode_word_boundary_pattern(word: str) -> str:
|
|
83
|
+
"""
|
|
84
|
+
Create a regex pattern with Unicode-aware word boundaries.
|
|
85
|
+
|
|
86
|
+
Standard regex \\b word boundaries don't work with non-ASCII characters
|
|
87
|
+
(e.g., Devanagari, Arabic, Hebrew, Chinese, Thai, etc.). This function
|
|
88
|
+
creates a pattern that works across all Unicode scripts.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
word (str): The word to create a boundary pattern for
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
str: A regex pattern string with proper Unicode boundaries
|
|
95
|
+
|
|
96
|
+
Examples:
|
|
97
|
+
>>> create_unicode_word_boundary_pattern("two")
|
|
98
|
+
'(?<![\\w])two(?![\\w])'
|
|
99
|
+
>>> create_unicode_word_boundary_pattern("दो") # Hindi "two"
|
|
100
|
+
'(?:^|(?<=[\\s+\\-*/^()]))दो(?:$|(?=[\\s+\\-*/^()]))'
|
|
101
|
+
"""
|
|
102
|
+
# Check if word contains only ASCII alphanumeric characters
|
|
103
|
+
if word.isascii() and word.replace('-', '').replace("'", '').isalnum():
|
|
104
|
+
# For ASCII words, use standard word boundaries
|
|
105
|
+
return r'\b' + re.escape(word) + r'\b'
|
|
106
|
+
else:
|
|
107
|
+
# For non-ASCII (Unicode) words, use space/operator boundaries
|
|
108
|
+
# This pattern matches the word when it's:
|
|
109
|
+
# - At the start of string OR preceded by whitespace/operator
|
|
110
|
+
# - At the end of string OR followed by whitespace/operator
|
|
111
|
+
escaped_word = re.escape(word)
|
|
112
|
+
# Boundary characters: whitespace, operators, parentheses
|
|
113
|
+
boundary = r'[\s+\-*/^()]'
|
|
114
|
+
return f'(?:^|(?<={boundary})){escaped_word}(?:$|(?={boundary}))'
|
|
115
|
+
|
|
116
|
+
|
|
82
117
|
def find_word_groups(string: str, words: list) -> list:
|
|
83
118
|
"""
|
|
84
119
|
Find matches for words in the format "3 thousand 6 hundred 2".
|
|
@@ -98,6 +133,115 @@ def find_word_groups(string: str, words: list) -> list:
|
|
|
98
133
|
return result
|
|
99
134
|
|
|
100
135
|
|
|
136
|
+
def replace_word_tokens_simplified_chinese(
|
|
137
|
+
string, stopwords: set[str] = None
|
|
138
|
+
) -> str:
|
|
139
|
+
"""
|
|
140
|
+
simplified Chinese version:
|
|
141
|
+
Given a string and an ISO 639-2 language code,
|
|
142
|
+
return the string with the words replaced with
|
|
143
|
+
an operational equivalent.
|
|
144
|
+
"""
|
|
145
|
+
words = mathwords.word_groups_for_language('CHI')
|
|
146
|
+
|
|
147
|
+
# Handle special Chinese power construction patterns FIRST
|
|
148
|
+
# In Chinese, '的...次方' forms a power construction where:
|
|
149
|
+
# - '的' acts as a possessive/linking word (provides the ^ operator)
|
|
150
|
+
# - '次方' is a suffix meaning "power" (should be removed, not replaced)
|
|
151
|
+
# - '次幂' is similar to '次方'
|
|
152
|
+
# Example: 二的四次方 = 2's 4th power = 2 ^ 4
|
|
153
|
+
# We need to remove '次方' and '次幂' BEFORE they get replaced as operators
|
|
154
|
+
string = string.replace('次方', ' ')
|
|
155
|
+
string = string.replace('次幂', ' ')
|
|
156
|
+
|
|
157
|
+
# Collect all operators (binary, prefix unary, postfix unary)
|
|
158
|
+
# and process them by length (longest first) to handle cases where
|
|
159
|
+
# a shorter operator is a substring of a longer one
|
|
160
|
+
# Example: '平方' (squared) vs '平方根' (square root)
|
|
161
|
+
all_operators = {}
|
|
162
|
+
|
|
163
|
+
# Add binary operators
|
|
164
|
+
all_operators.update(words['binary_operators'])
|
|
165
|
+
|
|
166
|
+
# Add prefix unary operators
|
|
167
|
+
all_operators.update(words['prefix_unary_operators'])
|
|
168
|
+
|
|
169
|
+
# Add postfix unary operators
|
|
170
|
+
if 'postfix_unary_operators' in words:
|
|
171
|
+
all_operators.update(words['postfix_unary_operators'])
|
|
172
|
+
|
|
173
|
+
# Sort all operators by length (longest first)
|
|
174
|
+
sorted_operators = sorted(all_operators.keys(), key=len, reverse=True)
|
|
175
|
+
|
|
176
|
+
for operator in sorted_operators:
|
|
177
|
+
if operator in string:
|
|
178
|
+
# 中文没有分隔符,后面需要靠分隔符分割式子,每次识别一个符号都将其分开来
|
|
179
|
+
string = string.replace(
|
|
180
|
+
operator, ' ' + all_operators[operator] + ' '
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
# chinese_scales用list的原因是为了保持从大到小的顺序,亿、万、千...
|
|
184
|
+
# Sort scales by their numeric value (largest first) to ensure correct
|
|
185
|
+
# parsing. For example, '千' (1000) should be found before '百' (100)
|
|
186
|
+
digits = set(words['numbers'].keys())
|
|
187
|
+
scales = sorted(
|
|
188
|
+
words['scales'].keys(), key=lambda x: words['scales'][x], reverse=True
|
|
189
|
+
)
|
|
190
|
+
digits_scales = words['numbers'].copy()
|
|
191
|
+
digits_scales.update(words['scales'])
|
|
192
|
+
|
|
193
|
+
# 九千八百万九千八百——> 98009800
|
|
194
|
+
def chinese_string_to_num(str):
|
|
195
|
+
if str == '':
|
|
196
|
+
return 0
|
|
197
|
+
|
|
198
|
+
if str in digits:
|
|
199
|
+
return digits_scales[str]
|
|
200
|
+
|
|
201
|
+
digits_scales.update(words['scales'])
|
|
202
|
+
|
|
203
|
+
for scale in scales:
|
|
204
|
+
index = str.find(scale)
|
|
205
|
+
if index >= 0:
|
|
206
|
+
t1 = chinese_string_to_num(str[:index])
|
|
207
|
+
t2 = digits_scales[scale]
|
|
208
|
+
t3 = chinese_string_to_num(str[index + 1:])
|
|
209
|
+
return t1 * t2 + t3
|
|
210
|
+
else:
|
|
211
|
+
return digits_scales[str]
|
|
212
|
+
|
|
213
|
+
# 扫描看有没有汉字数字,有转化为阿拉伯数字
|
|
214
|
+
index = 0
|
|
215
|
+
start = end = 0
|
|
216
|
+
while True:
|
|
217
|
+
char = string[index]
|
|
218
|
+
|
|
219
|
+
if char in digits_scales:
|
|
220
|
+
end += 1
|
|
221
|
+
index += 1
|
|
222
|
+
else:
|
|
223
|
+
if start < end:
|
|
224
|
+
num = str(chinese_string_to_num(string[start:end]))
|
|
225
|
+
# 需要加多一个分隔符
|
|
226
|
+
num_str = str(num) + ' '
|
|
227
|
+
string = string[:start] + num_str + string[end:]
|
|
228
|
+
index = start + len(num_str)
|
|
229
|
+
start = end = index
|
|
230
|
+
else:
|
|
231
|
+
index += 1
|
|
232
|
+
start = end = index
|
|
233
|
+
|
|
234
|
+
if index >= len(string):
|
|
235
|
+
if start < end:
|
|
236
|
+
num = str(chinese_string_to_num(string[start:end]))
|
|
237
|
+
num_str = str(num) + ' '
|
|
238
|
+
string = string[:start] + num_str + string[end:]
|
|
239
|
+
|
|
240
|
+
break
|
|
241
|
+
|
|
242
|
+
return string
|
|
243
|
+
|
|
244
|
+
|
|
101
245
|
def replace_word_tokens(
|
|
102
246
|
string: str, language: str, stopwords: set[str] = None
|
|
103
247
|
) -> str:
|
|
@@ -107,19 +251,39 @@ def replace_word_tokens(
|
|
|
107
251
|
Given a string and an ISO 639-2 language code,
|
|
108
252
|
return the string with the words replaced with
|
|
109
253
|
an operational equivalent.
|
|
254
|
+
|
|
255
|
+
Args:
|
|
256
|
+
string (str): The input string containing a mathematical expression
|
|
257
|
+
with words.
|
|
258
|
+
|
|
259
|
+
language (str): ISO 639-2 language code for word-based parsing.
|
|
260
|
+
|
|
261
|
+
stopwords (set[str], optional): A set of words to ignore during
|
|
262
|
+
parsing. This can be used to filter out
|
|
263
|
+
non-mathematical words in expressions.
|
|
264
|
+
|
|
265
|
+
Returns:
|
|
266
|
+
str: The input string with word-based mathematical terms replaced
|
|
267
|
+
with their symbolic equivalents.
|
|
110
268
|
"""
|
|
111
269
|
words = mathwords.word_groups_for_language(language)
|
|
112
270
|
|
|
113
|
-
#
|
|
271
|
+
# Process operators by length (longest first) to handle compound operators
|
|
114
272
|
binary_operators = words['binary_operators']
|
|
115
|
-
|
|
273
|
+
sorted_operators = sorted(binary_operators.keys(), key=len, reverse=True)
|
|
274
|
+
for operator in sorted_operators:
|
|
116
275
|
if operator in string:
|
|
117
276
|
string = string.replace(operator, binary_operators[operator])
|
|
118
277
|
|
|
119
278
|
# Handle prefix unary operators (like "square root of")
|
|
120
279
|
if 'prefix_unary_operators' in words:
|
|
121
280
|
prefix_unary_operators = words['prefix_unary_operators']
|
|
122
|
-
|
|
281
|
+
# Sort by length to handle compound operators where a shorter token
|
|
282
|
+
# is a substring of a longer token
|
|
283
|
+
sorted_prefix_operators = sorted(
|
|
284
|
+
prefix_unary_operators.keys(), key=len, reverse=True
|
|
285
|
+
)
|
|
286
|
+
for operator in sorted_prefix_operators:
|
|
123
287
|
if operator in string:
|
|
124
288
|
string = string.replace(
|
|
125
289
|
operator, prefix_unary_operators[operator]
|
|
@@ -128,7 +292,12 @@ def replace_word_tokens(
|
|
|
128
292
|
# Handle postfix unary operators (like "squared", "cubed")
|
|
129
293
|
if 'postfix_unary_operators' in words:
|
|
130
294
|
postfix_unary_operators = words['postfix_unary_operators']
|
|
131
|
-
|
|
295
|
+
# Sort by length to handle compound operators where a shorter token
|
|
296
|
+
# is a substring of a longer token
|
|
297
|
+
sorted_postfix_operators = sorted(
|
|
298
|
+
postfix_unary_operators.keys(), key=len, reverse=True
|
|
299
|
+
)
|
|
300
|
+
for operator in sorted_postfix_operators:
|
|
132
301
|
if operator in string:
|
|
133
302
|
# Captures the number/operand before the unary operator
|
|
134
303
|
pattern = r'(\w+)\s+' + re.escape(operator)
|
|
@@ -168,9 +337,10 @@ def replace_word_tokens(
|
|
|
168
337
|
|
|
169
338
|
# Replace number words with numeric values
|
|
170
339
|
for number in frozenset(numbers.keys()):
|
|
171
|
-
# Use word boundaries to prevent partial matches
|
|
172
|
-
# (e.g., "nine" in "nineteen")
|
|
173
|
-
|
|
340
|
+
# Use Unicode-aware word boundaries to prevent partial matches
|
|
341
|
+
# (e.g., "nine" in "nineteen") and support non-ASCII scripts
|
|
342
|
+
# (e.g., Devanagari, Arabic, Hebrew, Chinese, etc.)
|
|
343
|
+
pattern = create_unicode_word_boundary_pattern(number)
|
|
174
344
|
if re.search(pattern, string):
|
|
175
345
|
string = re.sub(pattern, str(numbers[number]), string)
|
|
176
346
|
|
|
@@ -185,8 +355,7 @@ def replace_word_tokens(
|
|
|
185
355
|
|
|
186
356
|
# Replace scaling multipliers with numeric values
|
|
187
357
|
scales = words['scales']
|
|
188
|
-
end_index_characters = mathwords.BINARY_OPERATORS
|
|
189
|
-
end_index_characters.add('(')
|
|
358
|
+
end_index_characters = mathwords.BINARY_OPERATORS | {'('}
|
|
190
359
|
|
|
191
360
|
word_matches = find_word_groups(string, frozenset(scales.keys()))
|
|
192
361
|
|
|
@@ -223,11 +392,54 @@ def replace_word_tokens(
|
|
|
223
392
|
return string
|
|
224
393
|
|
|
225
394
|
|
|
395
|
+
def preprocess_unary_operators(tokens: list) -> list:
|
|
396
|
+
"""
|
|
397
|
+
Preprocess tokens to convert unary minus to the 'neg' function.
|
|
398
|
+
|
|
399
|
+
A minus sign is considered unary (negative) if it appears:
|
|
400
|
+
* At the beginning of the expression
|
|
401
|
+
* After an opening parenthesis '('
|
|
402
|
+
* After a binary operator `(+, -, *, /, ^)`
|
|
403
|
+
"""
|
|
404
|
+
if not tokens:
|
|
405
|
+
return tokens
|
|
406
|
+
|
|
407
|
+
processed_tokens = []
|
|
408
|
+
|
|
409
|
+
binary_operators = mathwords.BINARY_OPERATORS | {'('}
|
|
410
|
+
|
|
411
|
+
for i, token in enumerate(tokens):
|
|
412
|
+
if token == '-':
|
|
413
|
+
# Check if this minus should be treated as unary
|
|
414
|
+
is_unary_minus = False
|
|
415
|
+
|
|
416
|
+
if i == 0:
|
|
417
|
+
# The first token is unary minus
|
|
418
|
+
is_unary_minus = True
|
|
419
|
+
elif i > 0:
|
|
420
|
+
prev_token = tokens[i - 1]
|
|
421
|
+
# A unary minus after opening parenthesis or binary operators
|
|
422
|
+
if prev_token in binary_operators:
|
|
423
|
+
is_unary_minus = True
|
|
424
|
+
|
|
425
|
+
if is_unary_minus:
|
|
426
|
+
# Convert the unary minus to 'neg' function
|
|
427
|
+
processed_tokens.append('neg')
|
|
428
|
+
else:
|
|
429
|
+
# Keep as binary minus
|
|
430
|
+
processed_tokens.append(token)
|
|
431
|
+
else:
|
|
432
|
+
processed_tokens.append(token)
|
|
433
|
+
|
|
434
|
+
return processed_tokens
|
|
435
|
+
|
|
436
|
+
|
|
226
437
|
def to_postfix(tokens: list) -> list:
|
|
227
438
|
"""
|
|
228
439
|
Convert a list of evaluatable tokens to postfix format.
|
|
229
440
|
"""
|
|
230
441
|
precedence = {
|
|
442
|
+
'.': 5,
|
|
231
443
|
'/': 4,
|
|
232
444
|
'*': 4,
|
|
233
445
|
'+': 3,
|
|
@@ -324,6 +536,17 @@ def evaluate_postfix(tokens: list) -> Union[int, float, str, Decimal]:
|
|
|
324
536
|
total = 'undefined'
|
|
325
537
|
else:
|
|
326
538
|
total = Decimal(str(a)) / Decimal(str(b))
|
|
539
|
+
elif token == '.':
|
|
540
|
+
# Treat decimal points as a binary operator that combines the
|
|
541
|
+
# integer and fractional part of two numbers
|
|
542
|
+
# Example: 53 . 25 = 53.25
|
|
543
|
+
if b == 0:
|
|
544
|
+
total = Decimal(a)
|
|
545
|
+
else:
|
|
546
|
+
# Count the digits in b to determine the divisor
|
|
547
|
+
digits = len(str(int(b)))
|
|
548
|
+
divisor = 10 ** digits
|
|
549
|
+
total = a + (b / divisor)
|
|
327
550
|
else:
|
|
328
551
|
raise PostfixTokenEvaluationException(
|
|
329
552
|
'Unknown token "{}"'.format(token)
|
|
@@ -367,6 +590,31 @@ def tokenize(string: str, language: str = None, escape: str = '___') -> list:
|
|
|
367
590
|
character = string[-1]
|
|
368
591
|
string = string[:-1] + ' ' + character
|
|
369
592
|
|
|
593
|
+
# If language is specified, normalize compound operators by removing
|
|
594
|
+
# spaces between their characters. This handles cases like '乘 以'
|
|
595
|
+
# which should be treated as the single compound operator '乘以'.
|
|
596
|
+
# Process by length (longest first) to avoid partial matches.
|
|
597
|
+
if language:
|
|
598
|
+
words = mathwords.words_for_language(language)
|
|
599
|
+
|
|
600
|
+
# Sort all phrases by length (longest first) to handle cases where
|
|
601
|
+
# a shorter phrase is a substring of a longer one
|
|
602
|
+
phrases_by_length = sorted(words, key=len, reverse=True)
|
|
603
|
+
|
|
604
|
+
for phrase in phrases_by_length:
|
|
605
|
+
# For multi-character phrases, create a spaced version
|
|
606
|
+
# and replace it with the non-spaced version
|
|
607
|
+
if len(phrase) > 1:
|
|
608
|
+
# Create pattern with optional spaces between each character
|
|
609
|
+
# For example, '乘以' could appear as '乘 以' or '乘 以'
|
|
610
|
+
spaced_phrase = ' '.join(phrase)
|
|
611
|
+
# Replace spaced version with non-spaced version
|
|
612
|
+
string = string.replace(spaced_phrase, phrase)
|
|
613
|
+
|
|
614
|
+
# Binary operators must have space around them to be tokenized properly
|
|
615
|
+
for operator in mathwords.BINARY_OPERATORS:
|
|
616
|
+
string = string.replace(operator, f' {operator} ')
|
|
617
|
+
|
|
370
618
|
# Parenthesis must have space around them to be tokenized properly
|
|
371
619
|
string = string.replace('(', ' ( ')
|
|
372
620
|
string = string.replace(')', ' ) ')
|
|
@@ -448,9 +696,15 @@ def parse(
|
|
|
448
696
|
- Division by zero returns 'undefined' instead of raising an exception
|
|
449
697
|
"""
|
|
450
698
|
if language:
|
|
451
|
-
|
|
699
|
+
if language == 'CHI':
|
|
700
|
+
string = replace_word_tokens_simplified_chinese(
|
|
701
|
+
string, stopwords
|
|
702
|
+
)
|
|
703
|
+
else:
|
|
704
|
+
string = replace_word_tokens(string, language, stopwords)
|
|
452
705
|
|
|
453
|
-
tokens = tokenize(string)
|
|
706
|
+
tokens = tokenize(string, language)
|
|
707
|
+
tokens = preprocess_unary_operators(tokens)
|
|
454
708
|
postfix = to_postfix(tokens)
|
|
455
709
|
|
|
456
710
|
return evaluate_postfix(postfix)
|