sinonym 0.2.4__tar.gz → 0.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sinonym-0.2.4 → sinonym-0.2.7}/PKG-INFO +2 -1
- {sinonym-0.2.4 → sinonym-0.2.7}/pyproject.toml +2 -1
- {sinonym-0.2.4 → sinonym-0.2.7}/scripts/README.md +1 -1
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/detector.py +116 -2
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/batch_analysis.py +158 -5
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/formatting.py +21 -1
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/parsing.py +63 -6
- sinonym-0.2.7/sinonym/timo/__init__.py +0 -0
- sinonym-0.2.7/sinonym/timo/config.yaml +12 -0
- sinonym-0.2.7/sinonym/timo/integration_test.py +70 -0
- sinonym-0.2.7/sinonym/timo/interface.py +244 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/utils/string_manipulation.py +79 -4
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_all_chinese_inputs.py +3 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_regression_proposals.py +170 -0
- sinonym-0.2.7/tests/test_timo_interface.py +87 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/.gitignore +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/LICENSE +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/README.md +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/__init__.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/chinese_names_data.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/coretypes/__init__.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/coretypes/config.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/coretypes/results.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/README.md +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/acl_2025_authors.txt +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/chinese_japanese_classifier.joblib +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/chinese_japanese_classifier.skops +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/familyname_orcid.csv +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/givenname_orcid.csv +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/model_features.json +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/ml_model_components.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/patterns/__init__.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/patterns/compiled_patterns.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/patterns/regex_builders.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/resources.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/__init__.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/cache.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/ethnicity.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/initialization.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/normalization.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/process_pool.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/text_processing/__init__.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/text_processing/compound_detector.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/text_processing/text_normalizer.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/text_processing/text_preprocessor.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/utils/__init__.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/utils/thread_cache.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/_fail_log.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/conftest.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_acl.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_basic_chinese_names.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_batch.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_compound_names.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_compound_surname_formats.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_middle_names.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_misc.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_mixed_production_cases.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_mixed_scripts.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_ml_japanese_detection.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_multiprocess_pool.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_name_formatting.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_non_chinese_rejection.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_performance.py +0 -0
- {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_regional_variants.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sinonym
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.7
|
|
4
4
|
Summary: Chinese Name Detection and Normalization Module
|
|
5
5
|
Project-URL: Homepage, https://github.com/allenai/sinonym
|
|
6
6
|
Project-URL: Repository, https://github.com/allenai/sinonym
|
|
@@ -21,6 +21,7 @@ Classifier: Topic :: Text Processing :: Linguistic
|
|
|
21
21
|
Requires-Python: >=3.10
|
|
22
22
|
Requires-Dist: joblib>=1.3
|
|
23
23
|
Requires-Dist: numpy>=1.24
|
|
24
|
+
Requires-Dist: pydantic<2,>=1.10
|
|
24
25
|
Requires-Dist: pypinyin>=0.44.0
|
|
25
26
|
Requires-Dist: requests>=2.31
|
|
26
27
|
Requires-Dist: scikit-learn<2,>=1.2
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "sinonym"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.7"
|
|
8
8
|
description = "Chinese Name Detection and Normalization Module"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -27,6 +27,7 @@ keywords = ["chinese", "names", "nlp", "romanization", "pinyin"]
|
|
|
27
27
|
dependencies = [
|
|
28
28
|
"joblib>=1.3",
|
|
29
29
|
"numpy>=1.24",
|
|
30
|
+
"pydantic>=1.10,<2",
|
|
30
31
|
"pypinyin>=0.44.0",
|
|
31
32
|
"requests>=2.31",
|
|
32
33
|
"scikit-learn>=1.2,<2",
|
|
@@ -5,7 +5,7 @@ Utility scripts for benchmarking, profiling, testing, and model training.
|
|
|
5
5
|
## Active Scripts
|
|
6
6
|
|
|
7
7
|
### `check_test_status.py`
|
|
8
|
-
Runs the full test suite and reports individual test case failures with detailed diagnostics. Runs performance tests separately, then exits 0/1 based on whether the failure count matches the expected baseline
|
|
8
|
+
Runs the full test suite and reports individual test case failures with detailed diagnostics. Runs performance tests separately, then exits 0/1 based on whether the failure count matches the expected baseline configured in `scripts/check_test_status.py`. Improvements (fewer failures) also pass; regressions fail.
|
|
9
9
|
|
|
10
10
|
```bash
|
|
11
11
|
uv run python scripts/check_test_status.py
|
|
@@ -178,6 +178,7 @@ from sinonym.services import (
|
|
|
178
178
|
NameFormattingService,
|
|
179
179
|
NameParsingService,
|
|
180
180
|
NormalizationService,
|
|
181
|
+
NormalizedInput,
|
|
181
182
|
ParseResult,
|
|
182
183
|
PinyinCacheService,
|
|
183
184
|
ServiceContext,
|
|
@@ -244,6 +245,114 @@ class ChineseNameDetector:
|
|
|
244
245
|
# Initialize services
|
|
245
246
|
self._initialize_services()
|
|
246
247
|
|
|
248
|
+
def _is_surname_group(self, tokens: list[str], normalized_cache: dict[str, str]) -> bool:
|
|
249
|
+
"""Return whether a romanized Han token group is a surname component."""
|
|
250
|
+
if len(tokens) == 1:
|
|
251
|
+
token = tokens[0]
|
|
252
|
+
normalized = self._normalizer.get_normalized(token, normalized_cache)
|
|
253
|
+
return self._data.is_surname(token, normalized)
|
|
254
|
+
|
|
255
|
+
normalized_tokens = [self._normalizer.get_normalized(token, normalized_cache) for token in tokens]
|
|
256
|
+
spaced = " ".join(normalized_tokens)
|
|
257
|
+
compact = "".join(normalized_tokens)
|
|
258
|
+
|
|
259
|
+
return (
|
|
260
|
+
spaced in self._data.compound_surnames
|
|
261
|
+
or spaced in self._data.compound_surnames_normalized
|
|
262
|
+
or compact in self._data.compound_original_format_map
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
def _is_compound_surname_group(self, tokens: list[str], normalized_cache: dict[str, str]) -> bool:
|
|
266
|
+
"""Return whether a romanized Han token group is a compound surname."""
|
|
267
|
+
return len(tokens) > 1 and self._is_surname_group(tokens, normalized_cache)
|
|
268
|
+
|
|
269
|
+
def _has_only_cjk_token_groups(self, normalized_input: NormalizedInput) -> bool:
|
|
270
|
+
"""Return whether separator-delimited input tokens are all CJK characters."""
|
|
271
|
+
return len(normalized_input.tokens) > 1 and all(
|
|
272
|
+
token and all(self._config.cjk_pattern.search(char) for char in token)
|
|
273
|
+
for token in normalized_input.tokens
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
def _format_parse_result(
|
|
277
|
+
self,
|
|
278
|
+
surname_tokens: list[str],
|
|
279
|
+
given_tokens: list[str],
|
|
280
|
+
normalized_input: NormalizedInput,
|
|
281
|
+
original_order: list[str],
|
|
282
|
+
) -> ParseResult:
|
|
283
|
+
"""Format parsed components and attach stable structured name fields."""
|
|
284
|
+
formatted_name, given_final, surname_final, surname_str, given_str, middle_tokens = (
|
|
285
|
+
self._formatting_service.format_name_output_with_tokens(
|
|
286
|
+
surname_tokens,
|
|
287
|
+
given_tokens,
|
|
288
|
+
normalized_input.norm_map,
|
|
289
|
+
normalized_input.compound_metadata,
|
|
290
|
+
)
|
|
291
|
+
)
|
|
292
|
+
parsed = ParsedName(
|
|
293
|
+
surname=surname_str,
|
|
294
|
+
given_name=given_str,
|
|
295
|
+
surname_tokens=surname_final,
|
|
296
|
+
given_tokens=given_final,
|
|
297
|
+
middle_name=" ".join(middle_tokens) if middle_tokens else "",
|
|
298
|
+
middle_tokens=middle_tokens,
|
|
299
|
+
order=["given", "middle", "surname"],
|
|
300
|
+
)
|
|
301
|
+
parsed_original_order = ParsedName(
|
|
302
|
+
surname=surname_str,
|
|
303
|
+
given_name=given_str,
|
|
304
|
+
surname_tokens=surname_final,
|
|
305
|
+
given_tokens=given_final,
|
|
306
|
+
middle_name=" ".join(middle_tokens) if middle_tokens else "",
|
|
307
|
+
middle_tokens=middle_tokens,
|
|
308
|
+
order=original_order,
|
|
309
|
+
)
|
|
310
|
+
return ParseResult.success_with_name(
|
|
311
|
+
formatted_name,
|
|
312
|
+
parsed=parsed,
|
|
313
|
+
parsed_original_order=parsed_original_order,
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
def _normalize_spaced_all_chinese_name(self, normalized_input: NormalizedInput) -> ParseResult | None:
|
|
317
|
+
"""Parse all-Han names whose whitespace already separates name components."""
|
|
318
|
+
if (
|
|
319
|
+
len(normalized_input.tokens) != self._config.min_tokens_required
|
|
320
|
+
or len(normalized_input.roman_tokens) <= self._config.min_tokens_required
|
|
321
|
+
):
|
|
322
|
+
return None
|
|
323
|
+
|
|
324
|
+
first_group = self._cache_service.han_to_pinyin_fast(normalized_input.tokens[0])
|
|
325
|
+
last_group = self._cache_service.han_to_pinyin_fast(normalized_input.tokens[1])
|
|
326
|
+
if not first_group or not last_group:
|
|
327
|
+
return None
|
|
328
|
+
|
|
329
|
+
# OCR/noisy spacing can split a compound surname across the group boundary.
|
|
330
|
+
boundary_compound = first_group + last_group[:1] if len(first_group) == 1 and len(last_group) > 1 else []
|
|
331
|
+
if boundary_compound and self._is_surname_group(boundary_compound, normalized_input.norm_map):
|
|
332
|
+
surname_tokens = boundary_compound
|
|
333
|
+
given_tokens = last_group[1:]
|
|
334
|
+
original_order = ["surname", "given"]
|
|
335
|
+
else:
|
|
336
|
+
first_is_surname = self._is_surname_group(first_group, normalized_input.norm_map)
|
|
337
|
+
last_is_surname = self._is_surname_group(last_group, normalized_input.norm_map)
|
|
338
|
+
last_is_compound_surname = self._is_compound_surname_group(last_group, normalized_input.norm_map)
|
|
339
|
+
|
|
340
|
+
if last_is_surname and (not first_is_surname or last_is_compound_surname):
|
|
341
|
+
surname_tokens = last_group
|
|
342
|
+
given_tokens = first_group
|
|
343
|
+
original_order = ["given", "surname"]
|
|
344
|
+
elif first_is_surname:
|
|
345
|
+
surname_tokens = first_group
|
|
346
|
+
given_tokens = last_group
|
|
347
|
+
original_order = ["surname", "given"]
|
|
348
|
+
else:
|
|
349
|
+
return None
|
|
350
|
+
|
|
351
|
+
try:
|
|
352
|
+
return self._format_parse_result(surname_tokens, given_tokens, normalized_input, original_order)
|
|
353
|
+
except ValueError as e:
|
|
354
|
+
return ParseResult.failure(str(e))
|
|
355
|
+
|
|
247
356
|
# Public API methods
|
|
248
357
|
def get_cache_info(self) -> CacheInfo:
|
|
249
358
|
"""Get cache information."""
|
|
@@ -291,6 +400,11 @@ class ChineseNameDetector:
|
|
|
291
400
|
|
|
292
401
|
# Try parsing in both orders - for all-Chinese inputs, choose best scoring parse
|
|
293
402
|
|
|
403
|
+
if self._has_only_cjk_token_groups(normalized_input):
|
|
404
|
+
grouped_result = self._normalize_spaced_all_chinese_name(normalized_input)
|
|
405
|
+
if grouped_result is not None:
|
|
406
|
+
return grouped_result
|
|
407
|
+
|
|
294
408
|
if is_all_chinese and len(normalized_input.roman_tokens) == self._config.min_tokens_required:
|
|
295
409
|
# For all-Chinese 2-token inputs, ALWAYS assume surname-first order
|
|
296
410
|
# Two-character Chinese names are always (surname, given_name)
|
|
@@ -443,8 +557,8 @@ class ChineseNameDetector:
|
|
|
443
557
|
given_tokens,
|
|
444
558
|
original_tokens,
|
|
445
559
|
normalized_input.norm_map,
|
|
446
|
-
False,
|
|
447
|
-
original_compound_surname,
|
|
560
|
+
is_all_chinese=False,
|
|
561
|
+
original_compound_format=original_compound_surname,
|
|
448
562
|
)
|
|
449
563
|
used_original = order_tokens == original_tokens
|
|
450
564
|
|
|
@@ -8,6 +8,7 @@ improve parsing accuracy for ambiguous individual names.
|
|
|
8
8
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
|
+
from statistics import median
|
|
11
12
|
from typing import TYPE_CHECKING
|
|
12
13
|
|
|
13
14
|
from sinonym.coretypes import (
|
|
@@ -20,6 +21,12 @@ from sinonym.coretypes import (
|
|
|
20
21
|
)
|
|
21
22
|
from sinonym.coretypes.results import ParsedName
|
|
22
23
|
|
|
24
|
+
GUARDED_GIVEN_FIRST_BATCH_MIN_SHARE = 0.75
|
|
25
|
+
GIVEN_NAME_SHAPE_MIN_SHARE = 0.5
|
|
26
|
+
ULTRA_LOW_FIRST_SURNAME_MEDIAN_MAX = 130.0
|
|
27
|
+
LONG_GIVEN_NAME_TOKEN_MIN_LENGTH = 6
|
|
28
|
+
TWO_TOKEN_NAME_LENGTH = 2
|
|
29
|
+
|
|
23
30
|
if TYPE_CHECKING:
|
|
24
31
|
from sinonym.services.ethnicity import EthnicityClassificationService
|
|
25
32
|
from sinonym.services.parsing import NameParsingService
|
|
@@ -69,10 +76,15 @@ class BatchAnalysisService:
|
|
|
69
76
|
# Normalize once and reuse
|
|
70
77
|
normalized_input = normalizer.apply(name)
|
|
71
78
|
candidates, best_candidate = self._analyze_individual_name_with_normalized(
|
|
72
|
-
name,
|
|
79
|
+
name,
|
|
80
|
+
normalized_input,
|
|
81
|
+
data,
|
|
82
|
+
allow_guarded_given_first_bonus=False,
|
|
73
83
|
)
|
|
74
84
|
name_candidates.append((name, candidates, best_candidate, normalized_input.compound_metadata))
|
|
75
85
|
|
|
86
|
+
name_candidates = self._promote_guarded_given_first_batch_votes(name_candidates, normalizer, data)
|
|
87
|
+
|
|
76
88
|
# Phase 2: Detect the dominant format pattern
|
|
77
89
|
format_pattern = self._detect_format_pattern(name_candidates)
|
|
78
90
|
|
|
@@ -119,10 +131,15 @@ class BatchAnalysisService:
|
|
|
119
131
|
# Normalize once for format detection (compound_metadata not needed)
|
|
120
132
|
normalized_input = normalizer.apply(name)
|
|
121
133
|
candidates, best_candidate = self._analyze_individual_name_with_normalized(
|
|
122
|
-
name,
|
|
134
|
+
name,
|
|
135
|
+
normalized_input,
|
|
136
|
+
data,
|
|
137
|
+
allow_guarded_given_first_bonus=False,
|
|
123
138
|
)
|
|
124
139
|
name_candidates.append((name, candidates, best_candidate, None)) # No compound_metadata needed for format detection
|
|
125
140
|
|
|
141
|
+
name_candidates = self._promote_guarded_given_first_batch_votes(name_candidates, normalizer, data)
|
|
142
|
+
|
|
126
143
|
return self._detect_format_pattern(name_candidates)
|
|
127
144
|
|
|
128
145
|
def _process_individually(self, names: list[str], normalizer, data, formatting_service) -> BatchParseResult:
|
|
@@ -181,7 +198,12 @@ class BatchAnalysisService:
|
|
|
181
198
|
)
|
|
182
199
|
|
|
183
200
|
def _analyze_individual_name_with_normalized(
|
|
184
|
-
self,
|
|
201
|
+
self,
|
|
202
|
+
name: str,
|
|
203
|
+
normalized_input,
|
|
204
|
+
_data,
|
|
205
|
+
*,
|
|
206
|
+
allow_guarded_given_first_bonus: bool = True,
|
|
185
207
|
) -> tuple[list[ParseCandidate], ParseCandidate | None]:
|
|
186
208
|
"""Analyze a single name using pre-computed normalized input."""
|
|
187
209
|
tokens = list(normalized_input.roman_tokens)
|
|
@@ -218,8 +240,9 @@ class BatchAnalysisService:
|
|
|
218
240
|
given_tokens,
|
|
219
241
|
tokens,
|
|
220
242
|
normalized_input.norm_map,
|
|
221
|
-
False,
|
|
222
|
-
original_compound_format,
|
|
243
|
+
is_all_chinese=False,
|
|
244
|
+
original_compound_format=original_compound_format,
|
|
245
|
+
allow_guarded_given_first_bonus=allow_guarded_given_first_bonus,
|
|
223
246
|
)
|
|
224
247
|
|
|
225
248
|
# Determine the format of this parse
|
|
@@ -240,6 +263,136 @@ class BatchAnalysisService:
|
|
|
240
263
|
best_candidate = candidates[0] if candidates else None
|
|
241
264
|
return candidates, best_candidate
|
|
242
265
|
|
|
266
|
+
def _promote_guarded_given_first_batch_votes(
|
|
267
|
+
self,
|
|
268
|
+
name_candidates: list[tuple[str, list[ParseCandidate], ParseCandidate | None, dict | None]],
|
|
269
|
+
normalizer,
|
|
270
|
+
data,
|
|
271
|
+
) -> list[tuple[str, list[ParseCandidate], ParseCandidate | None, dict | None]]:
|
|
272
|
+
"""Promote contested given-first votes when batch-level shape evidence supports them."""
|
|
273
|
+
participant_count = sum(1 for _name, candidates, best_candidate, _ in name_candidates if candidates and best_candidate)
|
|
274
|
+
if participant_count == 0:
|
|
275
|
+
return name_candidates
|
|
276
|
+
|
|
277
|
+
promoted, first_surname_freqs, given_shape_count = self._collect_guarded_given_first_promotions(
|
|
278
|
+
name_candidates,
|
|
279
|
+
normalizer,
|
|
280
|
+
data,
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
if not self._should_promote_guarded_given_first_votes(
|
|
284
|
+
participant_count,
|
|
285
|
+
promoted,
|
|
286
|
+
first_surname_freqs,
|
|
287
|
+
given_shape_count,
|
|
288
|
+
):
|
|
289
|
+
return name_candidates
|
|
290
|
+
|
|
291
|
+
adjusted = list(name_candidates)
|
|
292
|
+
for index, promoted_candidate in promoted:
|
|
293
|
+
name, candidates, _best_candidate, compound_metadata = adjusted[index]
|
|
294
|
+
adjusted[index] = (name, candidates, promoted_candidate, compound_metadata)
|
|
295
|
+
return adjusted
|
|
296
|
+
|
|
297
|
+
def _collect_guarded_given_first_promotions(
|
|
298
|
+
self,
|
|
299
|
+
name_candidates: list[tuple[str, list[ParseCandidate], ParseCandidate | None, dict | None]],
|
|
300
|
+
normalizer,
|
|
301
|
+
data,
|
|
302
|
+
) -> tuple[list[tuple[int, ParseCandidate]], list[float], int]:
|
|
303
|
+
"""Collect given-first candidates that only become best under individual guarded scoring."""
|
|
304
|
+
promoted: list[tuple[int, ParseCandidate]] = []
|
|
305
|
+
first_surname_freqs: list[float] = []
|
|
306
|
+
given_shape_count = 0
|
|
307
|
+
|
|
308
|
+
for index, (name, candidates, best_candidate, _compound_metadata) in enumerate(name_candidates):
|
|
309
|
+
promotion = self._guarded_given_first_promotion(name, candidates, best_candidate, normalizer, data)
|
|
310
|
+
if promotion is None:
|
|
311
|
+
continue
|
|
312
|
+
|
|
313
|
+
promoted_candidate, first_token, first_norm = promotion
|
|
314
|
+
promoted.append((index, promoted_candidate))
|
|
315
|
+
first_surname_freqs.append(data.get_surname_freq(first_norm))
|
|
316
|
+
if self._has_given_name_shape(first_token):
|
|
317
|
+
given_shape_count += 1
|
|
318
|
+
|
|
319
|
+
return promoted, first_surname_freqs, given_shape_count
|
|
320
|
+
|
|
321
|
+
def _guarded_given_first_promotion(
|
|
322
|
+
self,
|
|
323
|
+
name: str,
|
|
324
|
+
candidates: list[ParseCandidate],
|
|
325
|
+
best_candidate: ParseCandidate | None,
|
|
326
|
+
normalizer,
|
|
327
|
+
data,
|
|
328
|
+
) -> tuple[ParseCandidate, str, str] | None:
|
|
329
|
+
"""Return the promoted given-first candidate and first-token evidence, if any."""
|
|
330
|
+
if not candidates or best_candidate is None or best_candidate.format != NameFormat.SURNAME_FIRST:
|
|
331
|
+
return None
|
|
332
|
+
|
|
333
|
+
normalized_input = normalizer.apply(name)
|
|
334
|
+
tokens = list(normalized_input.roman_tokens)
|
|
335
|
+
if len(tokens) != TWO_TOKEN_NAME_LENGTH:
|
|
336
|
+
return None
|
|
337
|
+
|
|
338
|
+
_individual_candidates, individual_best = self._analyze_individual_name_with_normalized(
|
|
339
|
+
name,
|
|
340
|
+
normalized_input,
|
|
341
|
+
data,
|
|
342
|
+
allow_guarded_given_first_bonus=True,
|
|
343
|
+
)
|
|
344
|
+
if individual_best is None or individual_best.format != NameFormat.GIVEN_FIRST:
|
|
345
|
+
return None
|
|
346
|
+
|
|
347
|
+
promoted_candidate = self._matching_given_first_candidate(candidates, individual_best)
|
|
348
|
+
if promoted_candidate is None:
|
|
349
|
+
return None
|
|
350
|
+
|
|
351
|
+
first_token = tokens[0]
|
|
352
|
+
first_norm = normalizer.get_normalized(first_token, normalized_input.norm_map).replace(" ", "")
|
|
353
|
+
return promoted_candidate, first_token, first_norm
|
|
354
|
+
|
|
355
|
+
@staticmethod
|
|
356
|
+
def _matching_given_first_candidate(
|
|
357
|
+
candidates: list[ParseCandidate],
|
|
358
|
+
individual_best: ParseCandidate,
|
|
359
|
+
) -> ParseCandidate | None:
|
|
360
|
+
"""Find the no-bonus candidate that matches the individual guarded winner."""
|
|
361
|
+
return next(
|
|
362
|
+
(
|
|
363
|
+
candidate
|
|
364
|
+
for candidate in candidates
|
|
365
|
+
if candidate.format == NameFormat.GIVEN_FIRST
|
|
366
|
+
and candidate.surname_tokens == individual_best.surname_tokens
|
|
367
|
+
and candidate.given_tokens == individual_best.given_tokens
|
|
368
|
+
),
|
|
369
|
+
None,
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
@staticmethod
|
|
373
|
+
def _should_promote_guarded_given_first_votes(
|
|
374
|
+
participant_count: int,
|
|
375
|
+
promoted: list[tuple[int, ParseCandidate]],
|
|
376
|
+
first_surname_freqs: list[float],
|
|
377
|
+
given_shape_count: int,
|
|
378
|
+
) -> bool:
|
|
379
|
+
"""Return whether guarded given-first votes have enough batch-level support."""
|
|
380
|
+
if not promoted or len(promoted) / participant_count < GUARDED_GIVEN_FIRST_BATCH_MIN_SHARE:
|
|
381
|
+
return False
|
|
382
|
+
|
|
383
|
+
has_shape_evidence = given_shape_count / len(promoted) >= GIVEN_NAME_SHAPE_MIN_SHARE
|
|
384
|
+
has_ultra_low_first_surname_evidence = (
|
|
385
|
+
bool(first_surname_freqs)
|
|
386
|
+
and median(first_surname_freqs) < ULTRA_LOW_FIRST_SURNAME_MEDIAN_MAX
|
|
387
|
+
)
|
|
388
|
+
return has_shape_evidence or has_ultra_low_first_surname_evidence
|
|
389
|
+
|
|
390
|
+
@staticmethod
|
|
391
|
+
def _has_given_name_shape(token: str) -> bool:
|
|
392
|
+
"""Return whether a token has independent shape evidence for given-name position."""
|
|
393
|
+
raw = token.replace("-", "").replace("'", "")
|
|
394
|
+
return len(raw) >= LONG_GIVEN_NAME_TOKEN_MIN_LENGTH
|
|
395
|
+
|
|
243
396
|
def _determine_parse_format(
|
|
244
397
|
self, surname_tokens: list[str], _given_tokens: list[str], original_tokens: list[str],
|
|
245
398
|
) -> NameFormat:
|
|
@@ -86,6 +86,14 @@ class NameFormattingService:
|
|
|
86
86
|
self._normalizer,
|
|
87
87
|
self._config,
|
|
88
88
|
)
|
|
89
|
+
if not split:
|
|
90
|
+
split = StringManipulationUtils.split_surname_like_given_name(
|
|
91
|
+
token,
|
|
92
|
+
normalized_cache,
|
|
93
|
+
self._data,
|
|
94
|
+
self._normalizer,
|
|
95
|
+
self._config,
|
|
96
|
+
)
|
|
89
97
|
if split:
|
|
90
98
|
parts.extend(split)
|
|
91
99
|
# Final validation: accept if it's a valid Chinese token
|
|
@@ -217,8 +225,20 @@ class NameFormattingService:
|
|
|
217
225
|
continue
|
|
218
226
|
|
|
219
227
|
split = StringManipulationUtils.split_concatenated_name(
|
|
220
|
-
token,
|
|
228
|
+
token,
|
|
229
|
+
normalized_cache,
|
|
230
|
+
self._data,
|
|
231
|
+
self._normalizer,
|
|
232
|
+
self._config,
|
|
221
233
|
)
|
|
234
|
+
if not split:
|
|
235
|
+
split = StringManipulationUtils.split_surname_like_given_name(
|
|
236
|
+
token,
|
|
237
|
+
normalized_cache,
|
|
238
|
+
self._data,
|
|
239
|
+
self._normalizer,
|
|
240
|
+
self._config,
|
|
241
|
+
)
|
|
222
242
|
if split:
|
|
223
243
|
parts.extend(split)
|
|
224
244
|
elif self._normalizer.is_valid_given_name_token(token, normalized_cache):
|
|
@@ -18,6 +18,10 @@ from sinonym.utils.string_manipulation import StringManipulationUtils
|
|
|
18
18
|
if TYPE_CHECKING:
|
|
19
19
|
from sinonym.services.normalization import CompoundMetadata
|
|
20
20
|
|
|
21
|
+
LOW_FREQUENCY_SURNAME_MAX = 500.0
|
|
22
|
+
GIVEN_FIRST_SURNAME_FREQ_RATIO_MIN = 50.0
|
|
23
|
+
GIVEN_FIRST_ORDER_PRESERVATION_BONUS = 4.0
|
|
24
|
+
|
|
21
25
|
|
|
22
26
|
class NameParsingService:
|
|
23
27
|
"""Service for parsing Chinese names into surname and given name components."""
|
|
@@ -110,7 +114,13 @@ class NameParsingService:
|
|
|
110
114
|
given_tokens = order[given_slice]
|
|
111
115
|
if given_tokens:
|
|
112
116
|
# Check if this parse would have a reasonable score
|
|
113
|
-
score = self.calculate_parse_score(
|
|
117
|
+
score = self.calculate_parse_score(
|
|
118
|
+
surname_tokens,
|
|
119
|
+
given_tokens,
|
|
120
|
+
order,
|
|
121
|
+
normalized_cache,
|
|
122
|
+
is_all_chinese=False,
|
|
123
|
+
)
|
|
114
124
|
|
|
115
125
|
# Western name detection pattern
|
|
116
126
|
has_single_letter_given = any(len(token) == 1 for token in given_tokens)
|
|
@@ -159,7 +169,13 @@ class NameParsingService:
|
|
|
159
169
|
surname_tokens = [surname_token]
|
|
160
170
|
given_tokens = order[given_slice]
|
|
161
171
|
if given_tokens:
|
|
162
|
-
score = self.calculate_parse_score(
|
|
172
|
+
score = self.calculate_parse_score(
|
|
173
|
+
surname_tokens,
|
|
174
|
+
given_tokens,
|
|
175
|
+
order,
|
|
176
|
+
normalized_cache,
|
|
177
|
+
is_all_chinese=False,
|
|
178
|
+
)
|
|
163
179
|
|
|
164
180
|
has_single_letter_given = any(len(token) == 1 for token in given_tokens)
|
|
165
181
|
has_multi_syllable_tokens = any(len(token) > 3 for token in order)
|
|
@@ -246,8 +262,8 @@ class NameParsingService:
|
|
|
246
262
|
given_tokens,
|
|
247
263
|
tokens,
|
|
248
264
|
normalized_cache,
|
|
249
|
-
False,
|
|
250
|
-
original_compound_format,
|
|
265
|
+
is_all_chinese=False,
|
|
266
|
+
original_compound_format=original_compound_format,
|
|
251
267
|
score_cache=score_cache,
|
|
252
268
|
)
|
|
253
269
|
|
|
@@ -351,8 +367,8 @@ class NameParsingService:
|
|
|
351
367
|
given_tokens,
|
|
352
368
|
tokens,
|
|
353
369
|
normalized_cache,
|
|
354
|
-
False,
|
|
355
|
-
original_compound_format,
|
|
370
|
+
is_all_chinese=False,
|
|
371
|
+
original_compound_format=original_compound_format,
|
|
356
372
|
score_cache=score_cache,
|
|
357
373
|
)
|
|
358
374
|
|
|
@@ -536,9 +552,11 @@ class NameParsingService:
|
|
|
536
552
|
given_tokens: list[str],
|
|
537
553
|
tokens: list[str],
|
|
538
554
|
normalized_cache: dict[str, str],
|
|
555
|
+
*,
|
|
539
556
|
is_all_chinese: bool = False,
|
|
540
557
|
original_compound_format: str | None = None,
|
|
541
558
|
score_cache: dict[str, dict] | None = None,
|
|
559
|
+
allow_guarded_given_first_bonus: bool = True,
|
|
542
560
|
) -> float:
|
|
543
561
|
"""Calculate unified score for a parse candidate."""
|
|
544
562
|
if not given_tokens:
|
|
@@ -617,6 +635,12 @@ class NameParsingService:
|
|
|
617
635
|
|
|
618
636
|
if is_ambiguous:
|
|
619
637
|
order_preservation_bonus = 1.0 # Strong bonus for preserving original order in ambiguous cases
|
|
638
|
+
elif allow_guarded_given_first_bonus and self._has_guarded_given_first_surname_ratio(
|
|
639
|
+
surname_tokens[0],
|
|
640
|
+
given_tokens[0],
|
|
641
|
+
normalized_cache,
|
|
642
|
+
):
|
|
643
|
+
order_preservation_bonus = GIVEN_FIRST_ORDER_PRESERVATION_BONUS
|
|
620
644
|
if not is_all_chinese and len(tokens) == 3 and len(surname_tokens) == 1 and len(given_tokens) == 2:
|
|
621
645
|
# Narrow extension for hard 3-token given-first cases:
|
|
622
646
|
# only apply when surname-last is materially more plausible than surname-first.
|
|
@@ -724,6 +748,39 @@ class NameParsingService:
|
|
|
724
748
|
freq_ratio = max(surname_freq, given_freq) / min(surname_freq, given_freq)
|
|
725
749
|
return freq_ratio < 5.0 # Ambiguous if frequencies are within 5x of each other
|
|
726
750
|
|
|
751
|
+
def _has_guarded_given_first_surname_ratio(
|
|
752
|
+
self,
|
|
753
|
+
surname_token: str,
|
|
754
|
+
given_token: str,
|
|
755
|
+
normalized_cache: dict[str, str],
|
|
756
|
+
) -> bool:
|
|
757
|
+
"""Return whether surname frequencies strongly support given-first order."""
|
|
758
|
+
if "-" in given_token or self._is_compound_surname_token(given_token, normalized_cache):
|
|
759
|
+
return False
|
|
760
|
+
|
|
761
|
+
surname_key = self._surname_key([surname_token], normalized_cache)
|
|
762
|
+
given_as_surname_key = self._surname_key([given_token], normalized_cache)
|
|
763
|
+
|
|
764
|
+
if not (
|
|
765
|
+
self._data.is_surname(surname_token, surname_key)
|
|
766
|
+
and self._data.is_surname(given_token, given_as_surname_key)
|
|
767
|
+
):
|
|
768
|
+
return False
|
|
769
|
+
|
|
770
|
+
given_surname_freq = self._data.get_surname_freq(given_as_surname_key)
|
|
771
|
+
surname_freq = self._data.get_surname_freq(surname_key)
|
|
772
|
+
if not (0 < given_surname_freq < LOW_FREQUENCY_SURNAME_MAX):
|
|
773
|
+
return False
|
|
774
|
+
|
|
775
|
+
return surname_freq / given_surname_freq > GIVEN_FIRST_SURNAME_FREQ_RATIO_MIN
|
|
776
|
+
|
|
777
|
+
def _is_compound_surname_token(self, token: str, _normalized_cache: dict[str, str]) -> bool:
|
|
778
|
+
"""Return whether a single token is a known compact/camelCase compound surname."""
|
|
779
|
+
token_key = token.strip().lower()
|
|
780
|
+
return token_key in COMPOUND_VARIANTS or (
|
|
781
|
+
"-" in token_key and token_key in self._data.compound_hyphen_map
|
|
782
|
+
)
|
|
783
|
+
|
|
727
784
|
def _surname_key(self, surname_tokens: list[str], normalized_cache: dict[str, str]) -> str:
|
|
728
785
|
"""Convert surname tokens to lookup key, preferring Chinese characters when available."""
|
|
729
786
|
if len(surname_tokens) == 1:
|
|
File without changes
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
config_version: 0.0.1
|
|
2
|
+
model_variants:
|
|
3
|
+
sinonym_v1:
|
|
4
|
+
instance: sinonym.timo.interface.Instance
|
|
5
|
+
prediction: sinonym.timo.interface.Prediction
|
|
6
|
+
predictor: sinonym.timo.interface.Predictor
|
|
7
|
+
predictor_config: sinonym.timo.interface.PredictorConfig
|
|
8
|
+
artifacts_s3_path: null
|
|
9
|
+
python_version: "3.10"
|
|
10
|
+
cuda: False
|
|
11
|
+
integration_test: sinonym.timo.integration_test.TestIntegration
|
|
12
|
+
docker_run_commands: []
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
|
|
3
|
+
from sinonym.timo.interface import Instance, Prediction, Predictor, PredictorConfig
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TestIntegration(unittest.TestCase):
|
|
7
|
+
@classmethod
|
|
8
|
+
def setUpClass(cls):
|
|
9
|
+
cls.predictor = Predictor(config=PredictorConfig(), artifacts_dir=".")
|
|
10
|
+
|
|
11
|
+
def test_chinese_name(self):
|
|
12
|
+
results = self.predictor.predict_batch([Instance(name="Li Wei")])
|
|
13
|
+
self.assertEqual(len(results), 1)
|
|
14
|
+
self.assertIsInstance(results[0], Prediction)
|
|
15
|
+
self.assertTrue(results[0].success)
|
|
16
|
+
self.assertIsNotNone(results[0].given_name)
|
|
17
|
+
self.assertIsNotNone(results[0].surname)
|
|
18
|
+
|
|
19
|
+
def test_non_chinese_name(self):
|
|
20
|
+
results = self.predictor.predict_batch([Instance(name="John Smith")])
|
|
21
|
+
self.assertEqual(len(results), 1)
|
|
22
|
+
self.assertFalse(results[0].success)
|
|
23
|
+
self.assertIsNotNone(results[0].error_message)
|
|
24
|
+
|
|
25
|
+
def test_batch_superset_output(self):
|
|
26
|
+
instances = [Instance(name="Li Wei"), Instance(name="Wang Weiming")]
|
|
27
|
+
results = self.predictor.predict_batch(instances)
|
|
28
|
+
self.assertEqual(len(results), 2)
|
|
29
|
+
for r in results:
|
|
30
|
+
self.assertIsInstance(r, Prediction)
|
|
31
|
+
self.assertTrue(r.success)
|
|
32
|
+
# superset: every Prediction carries confidence + format_pattern
|
|
33
|
+
self.assertIsNotNone(r.confidence)
|
|
34
|
+
self.assertIsNotNone(r.format_pattern)
|
|
35
|
+
# shared batch pattern replicated onto each row
|
|
36
|
+
self.assertEqual(
|
|
37
|
+
results[0].format_pattern.dominant_format,
|
|
38
|
+
results[1].format_pattern.dominant_format,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def test_predict_batch_empty(self):
|
|
42
|
+
self.assertEqual(self.predictor.predict_batch([]), [])
|
|
43
|
+
|
|
44
|
+
def test_score_name_batch(self):
|
|
45
|
+
names = ["Li Wei", "Wang Weiming"]
|
|
46
|
+
summary = self.predictor.score_name_batch(names)
|
|
47
|
+
self.assertEqual(summary.names, names)
|
|
48
|
+
self.assertEqual(len(summary.results), 2)
|
|
49
|
+
self.assertEqual(len(summary.confidences), 2)
|
|
50
|
+
self.assertIsNotNone(summary.format_pattern.dominant_format)
|
|
51
|
+
|
|
52
|
+
def test_score_name_batch_tuned_threshold(self):
|
|
53
|
+
names = ["Li Wei", "Wang Weiming"]
|
|
54
|
+
summary = self.predictor.score_name_batch(names, format_threshold=0.9)
|
|
55
|
+
self.assertEqual(len(summary.results), 2)
|
|
56
|
+
self.assertEqual(
|
|
57
|
+
summary.format_pattern.threshold_met,
|
|
58
|
+
summary.format_pattern.confidence >= 0.9,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
def test_detect_batch_format(self):
|
|
62
|
+
pattern = self.predictor.detect_batch_format(["Zhang Wei", "Li Ming", "Wang Xiaoli"])
|
|
63
|
+
self.assertIn(
|
|
64
|
+
pattern.dominant_format, {"surname_first", "given_first", "mixed"}
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
def test_analyze_name_batch_full(self):
|
|
68
|
+
result = self.predictor.analyze_name_batch(["Li Wei", "Wang Weiming"])
|
|
69
|
+
self.assertEqual(len(result.results), 2)
|
|
70
|
+
self.assertEqual(len(result.individual_analyses), 2)
|