sinonym 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sinonym-0.2.2 → sinonym-0.2.4}/PKG-INFO +38 -4
- {sinonym-0.2.2 → sinonym-0.2.4}/README.md +37 -3
- {sinonym-0.2.2 → sinonym-0.2.4}/pyproject.toml +1 -1
- sinonym-0.2.4/scripts/README.md +65 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/chinese_names_data.py +1 -1
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/coretypes/results.py +2 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/detector.py +202 -100
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/resources.py +7 -5
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/batch_analysis.py +35 -14
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/ethnicity.py +46 -24
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/formatting.py +35 -7
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/initialization.py +55 -23
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/normalization.py +4 -20
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/parsing.py +278 -34
- sinonym-0.2.4/sinonym/services/process_pool.py +166 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/utils/string_manipulation.py +15 -11
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/conftest.py +2 -2
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_middle_names.py +3 -3
- sinonym-0.2.4/tests/test_multiprocess_pool.py +61 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_performance.py +3 -3
- sinonym-0.2.4/tests/test_regression_proposals.py +391 -0
- sinonym-0.2.2/scripts/README.md +0 -126
- {sinonym-0.2.2 → sinonym-0.2.4}/.gitignore +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/LICENSE +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/__init__.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/coretypes/__init__.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/coretypes/config.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/README.md +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/acl_2025_authors.txt +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/chinese_japanese_classifier.joblib +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/chinese_japanese_classifier.skops +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/familyname_orcid.csv +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/givenname_orcid.csv +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/model_features.json +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/ml_model_components.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/patterns/__init__.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/patterns/compiled_patterns.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/patterns/regex_builders.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/__init__.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/cache.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/text_processing/__init__.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/text_processing/compound_detector.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/text_processing/text_normalizer.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/text_processing/text_preprocessor.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/utils/__init__.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/utils/thread_cache.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/_fail_log.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_acl.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_all_chinese_inputs.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_basic_chinese_names.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_batch.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_compound_names.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_compound_surname_formats.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_misc.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_mixed_production_cases.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_mixed_scripts.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_ml_japanese_detection.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_name_formatting.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_non_chinese_rejection.py +0 -0
- {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_regional_variants.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sinonym
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: Chinese Name Detection and Normalization Module
|
|
5
5
|
Project-URL: Homepage, https://github.com/allenai/sinonym
|
|
6
6
|
Project-URL: Repository, https://github.com/allenai/sinonym
|
|
@@ -148,7 +148,7 @@ Formatted Output
|
|
|
148
148
|
### 5. Performance
|
|
149
149
|
|
|
150
150
|
* **High-Performance with Caching**
|
|
151
|
-
* The library is benchmarked to be very fast, capable of processing over
|
|
151
|
+
* The library is benchmarked to be very fast, capable of processing over 3,000 diverse names per second, and uses caching to significantly speed up the processing of repeated names.
|
|
152
152
|
|
|
153
153
|
## How It Works
|
|
154
154
|
|
|
@@ -357,6 +357,36 @@ for result in results:
|
|
|
357
357
|
print(f"Processed: {result.result}")
|
|
358
358
|
```
|
|
359
359
|
|
|
360
|
+
### Persistent Multi-Process Processing
|
|
361
|
+
|
|
362
|
+
For high-throughput workloads, you can keep a persistent process pool alive and
|
|
363
|
+
reuse worker processes across multiple calls. This avoids repeated process
|
|
364
|
+
start-up overhead and works on Windows/macOS/Linux via `spawn`.
|
|
365
|
+
|
|
366
|
+
```python
|
|
367
|
+
from sinonym.detector import ChineseNameDetector
|
|
368
|
+
|
|
369
|
+
def main():
|
|
370
|
+
detector = ChineseNameDetector()
|
|
371
|
+
names_a = ["Li Wei", "Wang Weiming", "Zhang Ming"]
|
|
372
|
+
names_b = ["Xin Liu", "Yang Li", "Chen Huang"]
|
|
373
|
+
|
|
374
|
+
# Reuse workers across many calls
|
|
375
|
+
with detector.create_persistent_multiprocess_pool(max_workers=6, chunk_size=64) as pool:
|
|
376
|
+
results_a = pool.normalize_names(names_a)
|
|
377
|
+
results_b = pool.normalize_names(names_b)
|
|
378
|
+
|
|
379
|
+
# One-off convenience wrapper (creates and closes a temporary pool)
|
|
380
|
+
single_batch = detector.process_name_batch_multiprocess(names_a, max_workers=6, chunk_size=64)
|
|
381
|
+
return results_a, results_b, single_batch
|
|
382
|
+
|
|
383
|
+
if __name__ == "__main__":
|
|
384
|
+
main()
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
Use the `if __name__ == "__main__":` guard in scripts to ensure safe process
|
|
388
|
+
spawning on Windows and macOS.
|
|
389
|
+
|
|
360
390
|
### When to Use Batch Processing
|
|
361
391
|
|
|
362
392
|
* **Academic Papers**: Author lists typically follow consistent formatting
|
|
@@ -431,14 +461,14 @@ If you'd like to contribute to Sinonym, here’s how to set up your development
|
|
|
431
461
|
First, clone the repository:
|
|
432
462
|
|
|
433
463
|
```bash
|
|
434
|
-
git clone https://github.com/
|
|
464
|
+
git clone https://github.com/allenai/sinonym.git
|
|
435
465
|
cd sinonym
|
|
436
466
|
```
|
|
437
467
|
|
|
438
468
|
Then, install the development dependencies:
|
|
439
469
|
|
|
440
470
|
```bash
|
|
441
|
-
uv sync --
|
|
471
|
+
uv sync --active --all-extras --dev
|
|
442
472
|
```
|
|
443
473
|
|
|
444
474
|
### Running Tests
|
|
@@ -459,6 +489,10 @@ uv run ruff check . --fix
|
|
|
459
489
|
uv run ruff format .
|
|
460
490
|
```
|
|
461
491
|
|
|
492
|
+
### Benchmarking & Profiling
|
|
493
|
+
|
|
494
|
+
See [scripts/README.md](scripts/README.md) for benchmark, profiling, and test status scripts.
|
|
495
|
+
|
|
462
496
|
## License
|
|
463
497
|
|
|
464
498
|
Sinonym is licensed under the Apache 2.0 License. See the `LICENSE` file for more details.
|
|
@@ -111,7 +111,7 @@ Formatted Output
|
|
|
111
111
|
### 5. Performance
|
|
112
112
|
|
|
113
113
|
* **High-Performance with Caching**
|
|
114
|
-
* The library is benchmarked to be very fast, capable of processing over
|
|
114
|
+
* The library is benchmarked to be very fast, capable of processing over 3,000 diverse names per second, and uses caching to significantly speed up the processing of repeated names.
|
|
115
115
|
|
|
116
116
|
## How It Works
|
|
117
117
|
|
|
@@ -320,6 +320,36 @@ for result in results:
|
|
|
320
320
|
print(f"Processed: {result.result}")
|
|
321
321
|
```
|
|
322
322
|
|
|
323
|
+
### Persistent Multi-Process Processing
|
|
324
|
+
|
|
325
|
+
For high-throughput workloads, you can keep a persistent process pool alive and
|
|
326
|
+
reuse worker processes across multiple calls. This avoids repeated process
|
|
327
|
+
start-up overhead and works on Windows/macOS/Linux via `spawn`.
|
|
328
|
+
|
|
329
|
+
```python
|
|
330
|
+
from sinonym.detector import ChineseNameDetector
|
|
331
|
+
|
|
332
|
+
def main():
|
|
333
|
+
detector = ChineseNameDetector()
|
|
334
|
+
names_a = ["Li Wei", "Wang Weiming", "Zhang Ming"]
|
|
335
|
+
names_b = ["Xin Liu", "Yang Li", "Chen Huang"]
|
|
336
|
+
|
|
337
|
+
# Reuse workers across many calls
|
|
338
|
+
with detector.create_persistent_multiprocess_pool(max_workers=6, chunk_size=64) as pool:
|
|
339
|
+
results_a = pool.normalize_names(names_a)
|
|
340
|
+
results_b = pool.normalize_names(names_b)
|
|
341
|
+
|
|
342
|
+
# One-off convenience wrapper (creates and closes a temporary pool)
|
|
343
|
+
single_batch = detector.process_name_batch_multiprocess(names_a, max_workers=6, chunk_size=64)
|
|
344
|
+
return results_a, results_b, single_batch
|
|
345
|
+
|
|
346
|
+
if __name__ == "__main__":
|
|
347
|
+
main()
|
|
348
|
+
```
|
|
349
|
+
|
|
350
|
+
Use the `if __name__ == "__main__":` guard in scripts to ensure safe process
|
|
351
|
+
spawning on Windows and macOS.
|
|
352
|
+
|
|
323
353
|
### When to Use Batch Processing
|
|
324
354
|
|
|
325
355
|
* **Academic Papers**: Author lists typically follow consistent formatting
|
|
@@ -394,14 +424,14 @@ If you'd like to contribute to Sinonym, here’s how to set up your development
|
|
|
394
424
|
First, clone the repository:
|
|
395
425
|
|
|
396
426
|
```bash
|
|
397
|
-
git clone https://github.com/
|
|
427
|
+
git clone https://github.com/allenai/sinonym.git
|
|
398
428
|
cd sinonym
|
|
399
429
|
```
|
|
400
430
|
|
|
401
431
|
Then, install the development dependencies:
|
|
402
432
|
|
|
403
433
|
```bash
|
|
404
|
-
uv sync --
|
|
434
|
+
uv sync --active --all-extras --dev
|
|
405
435
|
```
|
|
406
436
|
|
|
407
437
|
### Running Tests
|
|
@@ -422,6 +452,10 @@ uv run ruff check . --fix
|
|
|
422
452
|
uv run ruff format .
|
|
423
453
|
```
|
|
424
454
|
|
|
455
|
+
### Benchmarking & Profiling
|
|
456
|
+
|
|
457
|
+
See [scripts/README.md](scripts/README.md) for benchmark, profiling, and test status scripts.
|
|
458
|
+
|
|
425
459
|
## License
|
|
426
460
|
|
|
427
461
|
Sinonym is licensed under the Apache 2.0 License. See the `LICENSE` file for more details.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Sinonym Scripts
|
|
2
|
+
|
|
3
|
+
Utility scripts for benchmarking, profiling, testing, and model training.
|
|
4
|
+
|
|
5
|
+
## Active Scripts
|
|
6
|
+
|
|
7
|
+
### `check_test_status.py`
|
|
8
|
+
Runs the full test suite and reports individual test case failures with detailed diagnostics. Runs performance tests separately, then exits 0/1 based on whether the failure count matches the expected baseline (`EXPECTED_FAILURES = 52`). Improvements (fewer failures) also pass; regressions fail.
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
uv run python scripts/check_test_status.py
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
### `benchmark_stable.py`
|
|
15
|
+
Median-based performance benchmark gate. Spawns isolated worker subprocesses (fresh process per run) with controlled `PYTHONHASHSEED` and thread environment variables. Reports mean/median/stddev/CV of throughput and supports a `--min-median-names-per-sec` gate that exits non-zero on failure.
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
uv run python scripts/benchmark_stable.py --runs 5 --names 3000 --warmup 3000
|
|
19
|
+
uv run python scripts/benchmark_stable.py --runs 7 --min-median-names-per-sec 5000
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
### `profile_hotspots.py`
|
|
23
|
+
Hotspot time-share profiler. Warms caches on deterministic test names, runs one `cProfile` pass, then reports top functions and modules ranked by internal time (`tottime`) share. Use `--sinonym-only` to filter out third-party/stdlib noise.
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
uv run python scripts/profile_hotspots.py --names 3000 --warmup 3000 --sinonym-only
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
### `profile_run.py`
|
|
30
|
+
Quick single-process profiling script. Generates deterministic test names, warms caches, takes 5 pure timing measurements (no profiling overhead) for accurate throughput stats, then runs one `cProfile` pass for a top-25 function breakdown. Good for a fast sanity check during development.
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
uv run python scripts/profile_run.py
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### `profile_threaded.py`
|
|
37
|
+
Multi-threaded performance and thread-safety validation. Tests `normalize_name` throughput across 1/2/4/8 threads using a shared `ChineseNameDetector` instance, verifies that multi-threaded results are identical to single-threaded results, and reports speedup and CV per thread count.
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
uv run python scripts/profile_threaded.py
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
### `profile_multiprocess.py`
|
|
44
|
+
Persistent multi-process throughput and parity check. Compares single-process throughput to a spawn-based persistent process pool, verifies that outputs are identical for a deterministic workload, and reports median speedup.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
uv run python scripts/profile_multiprocess.py --names 12000 --warmup 3000 --runs 3 --workers 6 --chunk-size 64
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
### `train_ml_classifier_for_chinese_vs_japanese.py`
|
|
51
|
+
Trains the Chinese-vs-Japanese name classifier used in production. Downloads Chinese (~1.2M) and Japanese (~180K) name corpora, trains a scikit-learn pipeline (TF-IDF character n-grams + 20 linguistic heuristic features + logistic regression), and saves the model to `data/chinese_japanese_classifier.skops`.
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
uv run python scripts/train_ml_classifier_for_chinese_vs_japanese.py
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Abandoned Scripts
|
|
58
|
+
|
|
59
|
+
These remain for historical reference but are not used by the library. The rule-based parser in `sinonym.services.parsing` replaced the ML approach.
|
|
60
|
+
|
|
61
|
+
### `generate_chinese_name_corpus_data.py`
|
|
62
|
+
Was intended to generate training data for an ML-based name parsing disambiguation model. Downloads 200K Chinese names, romanizes them, generates all possible surname/given-name parses, and creates labeled training examples. The ML parsing model did not outperform the rule-based system.
|
|
63
|
+
|
|
64
|
+
### `generate_acl_data.py`
|
|
65
|
+
Supplementary data generator for the abandoned ML parsing effort. Processes ACL 2025 conference author names to create additional training examples in a different distribution (romanized, Western ordering).
|
|
@@ -137,7 +137,7 @@ CANTONESE_PHONETIC_RULES = {
|
|
|
137
137
|
"sau": "shou",
|
|
138
138
|
"tau": "tao",
|
|
139
139
|
"wun": "huan",
|
|
140
|
-
"kai": "
|
|
140
|
+
"kai": "kai", # Preserve "kai" by default; avoid forcing it to "qi" without explicit Cantonese context
|
|
141
141
|
"hoi": "hai",
|
|
142
142
|
"ngai": "ai",
|
|
143
143
|
"tsz": "zi",
|
|
@@ -18,6 +18,8 @@ class ParsedName:
|
|
|
18
18
|
parts appear when combined. For normalized output this is typically
|
|
19
19
|
["given", "middle", "surname"], while for original input order it may
|
|
20
20
|
be ["surname", "given", "middle"], etc.
|
|
21
|
+
Component labels remain stable: `surname`/`given_name` always identify
|
|
22
|
+
those components regardless of `order`.
|
|
21
23
|
"""
|
|
22
24
|
surname: str
|
|
23
25
|
given_name: str
|
|
@@ -164,6 +164,7 @@ data structures and the detector can be safely used from multiple threads.
|
|
|
164
164
|
|
|
165
165
|
import logging
|
|
166
166
|
import string
|
|
167
|
+
import threading
|
|
167
168
|
|
|
168
169
|
from sinonym.coretypes import BatchFormatPattern, BatchParseResult
|
|
169
170
|
from sinonym.coretypes.results import ParsedName
|
|
@@ -181,6 +182,7 @@ from sinonym.services import (
|
|
|
181
182
|
PinyinCacheService,
|
|
182
183
|
ServiceContext,
|
|
183
184
|
)
|
|
185
|
+
from sinonym.services.process_pool import PersistentMultiprocessNormalizer, normalize_names_multiprocess
|
|
184
186
|
|
|
185
187
|
# ════════════════════════════════════════════════════════════════════════════════
|
|
186
188
|
# MAIN CHINESE NAME DETECTOR CLASS
|
|
@@ -203,6 +205,7 @@ class ChineseNameDetector:
|
|
|
203
205
|
self._parsing_service: NameParsingService | None = None
|
|
204
206
|
self._formatting_service: NameFormattingService | None = None
|
|
205
207
|
self._batch_analysis_service: BatchAnalysisService | None = None
|
|
208
|
+
self._batch_threshold_lock = threading.Lock()
|
|
206
209
|
|
|
207
210
|
# Initialize data structures
|
|
208
211
|
self._initialize()
|
|
@@ -332,13 +335,13 @@ class ChineseNameDetector:
|
|
|
332
335
|
)
|
|
333
336
|
# Determine original input order and assign components accordingly
|
|
334
337
|
if token1_is_surname:
|
|
335
|
-
# Original: surname-first →
|
|
338
|
+
# Original: surname-first → preserve component labels and annotate order
|
|
336
339
|
order_list = ["surname", "given"]
|
|
337
340
|
parsed_original_order = ParsedName(
|
|
338
|
-
surname=
|
|
339
|
-
given_name=
|
|
340
|
-
surname_tokens=
|
|
341
|
-
given_tokens=
|
|
341
|
+
surname=surname_str,
|
|
342
|
+
given_name=given_str,
|
|
343
|
+
surname_tokens=surname_final,
|
|
344
|
+
given_tokens=given_final,
|
|
342
345
|
middle_name=" ".join(middle_tokens) if middle_tokens else "",
|
|
343
346
|
middle_tokens=middle_tokens,
|
|
344
347
|
order=order_list,
|
|
@@ -368,17 +371,15 @@ class ChineseNameDetector:
|
|
|
368
371
|
|
|
369
372
|
# Try both possibilities and see which one the parsing service accepts
|
|
370
373
|
# Option 1: First two tokens as compound surname + third as given
|
|
371
|
-
compound_parse = self._parsing_service.
|
|
374
|
+
compound_parse = self._parsing_service.parse_name_order_tokens(
|
|
372
375
|
tokens,
|
|
373
376
|
normalized_input.norm_map,
|
|
374
377
|
normalized_input.compound_metadata,
|
|
375
378
|
)
|
|
376
379
|
|
|
377
|
-
if (compound_parse
|
|
378
|
-
len(compound_parse.result[0]) == 2 and
|
|
379
|
-
len(compound_parse.result[1]) == 1):
|
|
380
|
+
if compound_parse is not None and len(compound_parse[0]) == 2 and len(compound_parse[1]) == 1:
|
|
380
381
|
# Parsing service recognized first two as compound surname
|
|
381
|
-
best_result = compound_parse
|
|
382
|
+
best_result = (compound_parse[0], compound_parse[1])
|
|
382
383
|
else:
|
|
383
384
|
# Option 2: First token as single surname + last two as given name
|
|
384
385
|
best_result = ([tokens[0]], tokens[1:])
|
|
@@ -403,12 +404,13 @@ class ChineseNameDetector:
|
|
|
403
404
|
middle_tokens=middle_tokens,
|
|
404
405
|
order=["given", "middle", "surname"],
|
|
405
406
|
)
|
|
406
|
-
# For 3-character all-Chinese, original order is surname-first
|
|
407
|
+
# For 3-character all-Chinese, original order is surname-first
|
|
408
|
+
# Preserve component labels and annotate only the order.
|
|
407
409
|
parsed_original_order = ParsedName(
|
|
408
|
-
surname=
|
|
409
|
-
given_name=
|
|
410
|
-
surname_tokens=
|
|
411
|
-
given_tokens=
|
|
410
|
+
surname=surname_str,
|
|
411
|
+
given_name=given_str,
|
|
412
|
+
surname_tokens=surname_final,
|
|
413
|
+
given_tokens=given_final,
|
|
412
414
|
middle_name=" ".join(middle_tokens) if middle_tokens else "",
|
|
413
415
|
middle_tokens=middle_tokens,
|
|
414
416
|
order=["surname", "given"],
|
|
@@ -421,76 +423,124 @@ class ChineseNameDetector:
|
|
|
421
423
|
except ValueError as e:
|
|
422
424
|
return ParseResult.failure(str(e))
|
|
423
425
|
else:
|
|
424
|
-
#
|
|
426
|
+
# Evaluate both order hypotheses and pick the best-scoring parse
|
|
427
|
+
original_tokens = list(normalized_input.roman_tokens)
|
|
428
|
+
best_candidate = None
|
|
429
|
+
|
|
425
430
|
for order in (normalized_input.roman_tokens, normalized_input.roman_tokens[::-1]):
|
|
426
|
-
|
|
427
|
-
|
|
431
|
+
order_tokens = list(order)
|
|
432
|
+
parse_result = self._parsing_service.parse_name_order_tokens(
|
|
433
|
+
order_tokens,
|
|
428
434
|
normalized_input.norm_map,
|
|
429
435
|
normalized_input.compound_metadata,
|
|
430
436
|
)
|
|
431
|
-
if parse_result
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
437
|
+
if parse_result is None:
|
|
438
|
+
continue
|
|
439
|
+
|
|
440
|
+
surname_tokens, given_tokens, original_compound_surname = parse_result
|
|
441
|
+
score = self._parsing_service.calculate_parse_score(
|
|
442
|
+
surname_tokens,
|
|
443
|
+
given_tokens,
|
|
444
|
+
original_tokens,
|
|
445
|
+
normalized_input.norm_map,
|
|
446
|
+
False,
|
|
447
|
+
original_compound_surname,
|
|
448
|
+
)
|
|
449
|
+
used_original = order_tokens == original_tokens
|
|
450
|
+
|
|
451
|
+
candidate = {
|
|
452
|
+
"surname_tokens": surname_tokens,
|
|
453
|
+
"given_tokens": given_tokens,
|
|
454
|
+
"score": score,
|
|
455
|
+
"order_tokens": order_tokens,
|
|
456
|
+
"used_original": used_original,
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
if (
|
|
460
|
+
best_candidate is None
|
|
461
|
+
or candidate["score"] > best_candidate["score"]
|
|
462
|
+
or (
|
|
463
|
+
candidate["score"] == best_candidate["score"]
|
|
464
|
+
and candidate["used_original"]
|
|
465
|
+
and not best_candidate["used_original"]
|
|
466
|
+
)
|
|
467
|
+
):
|
|
468
|
+
best_candidate = candidate
|
|
469
|
+
|
|
470
|
+
if best_candidate is not None:
|
|
471
|
+
surname_tokens = best_candidate["surname_tokens"]
|
|
472
|
+
given_tokens = best_candidate["given_tokens"]
|
|
473
|
+
order_tokens = best_candidate["order_tokens"]
|
|
474
|
+
used_original = best_candidate["used_original"]
|
|
475
|
+
try:
|
|
476
|
+
formatted_name, given_final, surname_final, surname_str, given_str, middle_tokens = (
|
|
477
|
+
self._formatting_service.format_name_output_with_tokens(
|
|
478
|
+
surname_tokens,
|
|
479
|
+
given_tokens,
|
|
480
|
+
normalized_input.norm_map,
|
|
481
|
+
normalized_input.compound_metadata,
|
|
441
482
|
)
|
|
442
|
-
|
|
483
|
+
)
|
|
484
|
+
parsed = ParsedName(
|
|
485
|
+
surname=surname_str,
|
|
486
|
+
given_name=given_str,
|
|
487
|
+
surname_tokens=surname_final,
|
|
488
|
+
given_tokens=given_final,
|
|
489
|
+
middle_name=" ".join(middle_tokens) if middle_tokens else "",
|
|
490
|
+
middle_tokens=middle_tokens,
|
|
491
|
+
order=["given", "middle", "surname"],
|
|
492
|
+
)
|
|
493
|
+
# Determine original input order relative to detected parse
|
|
494
|
+
k = len(surname_tokens)
|
|
495
|
+
is_surname_first_in_this_order = list(order_tokens[:k]) == surname_tokens
|
|
496
|
+
is_surname_last_in_this_order = list(order_tokens[-k:]) == surname_tokens
|
|
497
|
+
|
|
498
|
+
# Compound surname fallback: surname_tokens may be sub-tokens
|
|
499
|
+
# of a single original token (e.g. ['Ou','yang'] from 'Ouyang')
|
|
500
|
+
if not is_surname_first_in_this_order and not is_surname_last_in_this_order:
|
|
501
|
+
joined_surname = "".join(surname_tokens).lower()
|
|
502
|
+
if order_tokens[0].lower() == joined_surname:
|
|
503
|
+
is_surname_first_in_this_order = True
|
|
504
|
+
elif order_tokens[-1].lower() == joined_surname:
|
|
505
|
+
is_surname_last_in_this_order = True
|
|
506
|
+
|
|
507
|
+
if used_original:
|
|
508
|
+
original_is_given_first = is_surname_last_in_this_order
|
|
509
|
+
else:
|
|
510
|
+
original_is_given_first = is_surname_first_in_this_order
|
|
511
|
+
|
|
512
|
+
if original_is_given_first:
|
|
513
|
+
order_list = ["given"] + (["middle"] if middle_tokens else []) + ["surname"]
|
|
514
|
+
# Keep labels as-is
|
|
515
|
+
parsed_original_order = ParsedName(
|
|
443
516
|
surname=surname_str,
|
|
444
517
|
given_name=given_str,
|
|
445
518
|
surname_tokens=surname_final,
|
|
446
519
|
given_tokens=given_final,
|
|
447
520
|
middle_name=" ".join(middle_tokens) if middle_tokens else "",
|
|
448
521
|
middle_tokens=middle_tokens,
|
|
449
|
-
order=
|
|
522
|
+
order=order_list,
|
|
450
523
|
)
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
if original_is_given_first:
|
|
463
|
-
order_list = ["given"] + (["middle"] if middle_tokens else []) + ["surname"]
|
|
464
|
-
# Keep labels as-is
|
|
465
|
-
parsed_original_order = ParsedName(
|
|
466
|
-
surname=surname_str,
|
|
467
|
-
given_name=given_str,
|
|
468
|
-
surname_tokens=surname_final,
|
|
469
|
-
given_tokens=given_final,
|
|
470
|
-
middle_name=" ".join(middle_tokens) if middle_tokens else "",
|
|
471
|
-
middle_tokens=middle_tokens,
|
|
472
|
-
order=order_list,
|
|
473
|
-
)
|
|
474
|
-
else:
|
|
475
|
-
order_list = ["surname"] + (["middle"] if middle_tokens else []) + ["given"]
|
|
476
|
-
# Original: surname-first → swap labels to reflect "first part" as given
|
|
477
|
-
parsed_original_order = ParsedName(
|
|
478
|
-
surname=given_str,
|
|
479
|
-
given_name=surname_str,
|
|
480
|
-
surname_tokens=given_final,
|
|
481
|
-
given_tokens=surname_final,
|
|
482
|
-
middle_name=" ".join(middle_tokens) if middle_tokens else "",
|
|
483
|
-
middle_tokens=middle_tokens,
|
|
484
|
-
order=order_list,
|
|
485
|
-
)
|
|
486
|
-
|
|
487
|
-
return ParseResult.success_with_name(
|
|
488
|
-
formatted_name,
|
|
489
|
-
parsed=parsed,
|
|
490
|
-
parsed_original_order=parsed_original_order,
|
|
524
|
+
else:
|
|
525
|
+
order_list = ["surname"] + (["middle"] if middle_tokens else []) + ["given"]
|
|
526
|
+
# Original: surname-first → keep labels stable and annotate order only
|
|
527
|
+
parsed_original_order = ParsedName(
|
|
528
|
+
surname=surname_str,
|
|
529
|
+
given_name=given_str,
|
|
530
|
+
surname_tokens=surname_final,
|
|
531
|
+
given_tokens=given_final,
|
|
532
|
+
middle_name=" ".join(middle_tokens) if middle_tokens else "",
|
|
533
|
+
middle_tokens=middle_tokens,
|
|
534
|
+
order=order_list,
|
|
491
535
|
)
|
|
492
|
-
|
|
493
|
-
|
|
536
|
+
|
|
537
|
+
return ParseResult.success_with_name(
|
|
538
|
+
formatted_name,
|
|
539
|
+
parsed=parsed,
|
|
540
|
+
parsed_original_order=parsed_original_order,
|
|
541
|
+
)
|
|
542
|
+
except ValueError as e:
|
|
543
|
+
return ParseResult.failure(str(e))
|
|
494
544
|
|
|
495
545
|
return ParseResult.failure("name not recognised as Chinese")
|
|
496
546
|
|
|
@@ -533,21 +583,22 @@ class ChineseNameDetector:
|
|
|
533
583
|
individual_results = [self.normalize_name(name) for name in names]
|
|
534
584
|
return self._create_fallback_batch_result(names, individual_results)
|
|
535
585
|
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
586
|
+
with self._batch_threshold_lock:
|
|
587
|
+
# Configure threshold for this analysis
|
|
588
|
+
original_threshold = self._batch_analysis_service._format_threshold
|
|
589
|
+
self._batch_analysis_service._format_threshold = format_threshold
|
|
590
|
+
|
|
591
|
+
try:
|
|
592
|
+
return self._batch_analysis_service.analyze_name_batch(
|
|
593
|
+
names,
|
|
594
|
+
self._normalizer,
|
|
595
|
+
self._data,
|
|
596
|
+
self._formatting_service,
|
|
597
|
+
minimum_batch_size,
|
|
598
|
+
)
|
|
599
|
+
finally:
|
|
600
|
+
# Restore original threshold
|
|
601
|
+
self._batch_analysis_service._format_threshold = original_threshold
|
|
551
602
|
|
|
552
603
|
def detect_batch_format(
|
|
553
604
|
self,
|
|
@@ -587,19 +638,20 @@ class ChineseNameDetector:
|
|
|
587
638
|
threshold_met=False,
|
|
588
639
|
)
|
|
589
640
|
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
641
|
+
with self._batch_threshold_lock:
|
|
642
|
+
# Configure threshold for this analysis
|
|
643
|
+
original_threshold = self._batch_analysis_service._format_threshold
|
|
644
|
+
self._batch_analysis_service._format_threshold = format_threshold
|
|
593
645
|
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
646
|
+
try:
|
|
647
|
+
return self._batch_analysis_service.detect_batch_format(
|
|
648
|
+
names,
|
|
649
|
+
self._normalizer,
|
|
650
|
+
self._data,
|
|
651
|
+
)
|
|
652
|
+
finally:
|
|
653
|
+
# Restore original threshold
|
|
654
|
+
self._batch_analysis_service._format_threshold = original_threshold
|
|
603
655
|
|
|
604
656
|
def process_name_batch(
|
|
605
657
|
self,
|
|
@@ -632,8 +684,58 @@ class ChineseNameDetector:
|
|
|
632
684
|
batch_result = self.analyze_name_batch(names, format_threshold, minimum_batch_size)
|
|
633
685
|
return batch_result.results
|
|
634
686
|
|
|
687
|
+
def create_persistent_multiprocess_pool(
|
|
688
|
+
self,
|
|
689
|
+
*,
|
|
690
|
+
max_workers: int | None = None,
|
|
691
|
+
chunk_size: int = 64,
|
|
692
|
+
mp_start_method: str = "spawn",
|
|
693
|
+
) -> PersistentMultiprocessNormalizer:
|
|
694
|
+
"""
|
|
695
|
+
Create a persistent multi-process pool for repeated normalization calls.
|
|
696
|
+
|
|
697
|
+
Notes:
|
|
698
|
+
- Uses one detector instance per worker process.
|
|
699
|
+
- For Windows/macOS scripts, call this behind an
|
|
700
|
+
`if __name__ == "__main__":` guard.
|
|
701
|
+
"""
|
|
702
|
+
self._ensure_initialized()
|
|
703
|
+
return PersistentMultiprocessNormalizer(
|
|
704
|
+
max_workers=max_workers,
|
|
705
|
+
chunk_size=chunk_size,
|
|
706
|
+
mp_start_method=mp_start_method,
|
|
707
|
+
detector_config=self._config,
|
|
708
|
+
detector_weights=self._weights,
|
|
709
|
+
)
|
|
710
|
+
|
|
711
|
+
def process_name_batch_multiprocess(
|
|
712
|
+
self,
|
|
713
|
+
names: list[str],
|
|
714
|
+
*,
|
|
715
|
+
max_workers: int | None = None,
|
|
716
|
+
chunk_size: int = 64,
|
|
717
|
+
mp_start_method: str = "spawn",
|
|
718
|
+
) -> list[ParseResult]:
|
|
719
|
+
"""
|
|
720
|
+
Process one batch with a temporary multi-process pool.
|
|
721
|
+
|
|
722
|
+
For repeated calls, prefer `create_persistent_multiprocess_pool()`
|
|
723
|
+
to avoid repeated process start-up overhead.
|
|
724
|
+
"""
|
|
725
|
+
self._ensure_initialized()
|
|
726
|
+
return normalize_names_multiprocess(
|
|
727
|
+
names,
|
|
728
|
+
max_workers=max_workers,
|
|
729
|
+
chunk_size=chunk_size,
|
|
730
|
+
mp_start_method=mp_start_method,
|
|
731
|
+
detector_config=self._config,
|
|
732
|
+
detector_weights=self._weights,
|
|
733
|
+
)
|
|
734
|
+
|
|
635
735
|
def _create_fallback_batch_result(
|
|
636
|
-
self,
|
|
736
|
+
self,
|
|
737
|
+
names: list[str],
|
|
738
|
+
individual_results: list[ParseResult],
|
|
637
739
|
) -> BatchParseResult:
|
|
638
740
|
"""Create a fallback BatchParseResult when batch analysis is not available."""
|
|
639
741
|
from sinonym.coretypes import BatchFormatPattern, IndividualAnalysis, NameFormat
|