sinonym 0.2.2__tar.gz → 0.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {sinonym-0.2.2 → sinonym-0.2.4}/PKG-INFO +38 -4
  2. {sinonym-0.2.2 → sinonym-0.2.4}/README.md +37 -3
  3. {sinonym-0.2.2 → sinonym-0.2.4}/pyproject.toml +1 -1
  4. sinonym-0.2.4/scripts/README.md +65 -0
  5. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/chinese_names_data.py +1 -1
  6. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/coretypes/results.py +2 -0
  7. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/detector.py +202 -100
  8. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/resources.py +7 -5
  9. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/batch_analysis.py +35 -14
  10. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/ethnicity.py +46 -24
  11. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/formatting.py +35 -7
  12. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/initialization.py +55 -23
  13. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/normalization.py +4 -20
  14. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/parsing.py +278 -34
  15. sinonym-0.2.4/sinonym/services/process_pool.py +166 -0
  16. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/utils/string_manipulation.py +15 -11
  17. {sinonym-0.2.2 → sinonym-0.2.4}/tests/conftest.py +2 -2
  18. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_middle_names.py +3 -3
  19. sinonym-0.2.4/tests/test_multiprocess_pool.py +61 -0
  20. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_performance.py +3 -3
  21. sinonym-0.2.4/tests/test_regression_proposals.py +391 -0
  22. sinonym-0.2.2/scripts/README.md +0 -126
  23. {sinonym-0.2.2 → sinonym-0.2.4}/.gitignore +0 -0
  24. {sinonym-0.2.2 → sinonym-0.2.4}/LICENSE +0 -0
  25. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/__init__.py +0 -0
  26. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/coretypes/__init__.py +0 -0
  27. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/coretypes/config.py +0 -0
  28. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/README.md +0 -0
  29. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/acl_2025_authors.txt +0 -0
  30. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/chinese_japanese_classifier.joblib +0 -0
  31. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/chinese_japanese_classifier.skops +0 -0
  32. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/familyname_orcid.csv +0 -0
  33. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/givenname_orcid.csv +0 -0
  34. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/data/model_features.json +0 -0
  35. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/ml_model_components.py +0 -0
  36. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/patterns/__init__.py +0 -0
  37. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/patterns/compiled_patterns.py +0 -0
  38. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/patterns/regex_builders.py +0 -0
  39. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/__init__.py +0 -0
  40. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/services/cache.py +0 -0
  41. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/text_processing/__init__.py +0 -0
  42. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/text_processing/compound_detector.py +0 -0
  43. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/text_processing/text_normalizer.py +0 -0
  44. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/text_processing/text_preprocessor.py +0 -0
  45. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/utils/__init__.py +0 -0
  46. {sinonym-0.2.2 → sinonym-0.2.4}/sinonym/utils/thread_cache.py +0 -0
  47. {sinonym-0.2.2 → sinonym-0.2.4}/tests/_fail_log.py +0 -0
  48. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_acl.py +0 -0
  49. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_all_chinese_inputs.py +0 -0
  50. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_basic_chinese_names.py +0 -0
  51. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_batch.py +0 -0
  52. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_compound_names.py +0 -0
  53. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_compound_surname_formats.py +0 -0
  54. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_misc.py +0 -0
  55. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_mixed_production_cases.py +0 -0
  56. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_mixed_scripts.py +0 -0
  57. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_ml_japanese_detection.py +0 -0
  58. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_name_formatting.py +0 -0
  59. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_non_chinese_rejection.py +0 -0
  60. {sinonym-0.2.2 → sinonym-0.2.4}/tests/test_regional_variants.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sinonym
3
- Version: 0.2.2
3
+ Version: 0.2.4
4
4
  Summary: Chinese Name Detection and Normalization Module
5
5
  Project-URL: Homepage, https://github.com/allenai/sinonym
6
6
  Project-URL: Repository, https://github.com/allenai/sinonym
@@ -148,7 +148,7 @@ Formatted Output
148
148
  ### 5. Performance
149
149
 
150
150
  * **High-Performance with Caching**
151
- * The library is benchmarked to be very fast, capable of processing over 10,000 diverse names per second, and uses caching to significantly speed up the processing of repeated names.
151
+ * The library is benchmarked to be very fast, capable of processing over 3,000 diverse names per second, and uses caching to significantly speed up the processing of repeated names.
152
152
 
153
153
  ## How It Works
154
154
 
@@ -357,6 +357,36 @@ for result in results:
357
357
  print(f"Processed: {result.result}")
358
358
  ```
359
359
 
360
+ ### Persistent Multi-Process Processing
361
+
362
+ For high-throughput workloads, you can keep a persistent process pool alive and
363
+ reuse worker processes across multiple calls. This avoids repeated process
364
+ start-up overhead and works on Windows/macOS/Linux via `spawn`.
365
+
366
+ ```python
367
+ from sinonym.detector import ChineseNameDetector
368
+
369
+ def main():
370
+ detector = ChineseNameDetector()
371
+ names_a = ["Li Wei", "Wang Weiming", "Zhang Ming"]
372
+ names_b = ["Xin Liu", "Yang Li", "Chen Huang"]
373
+
374
+ # Reuse workers across many calls
375
+ with detector.create_persistent_multiprocess_pool(max_workers=6, chunk_size=64) as pool:
376
+ results_a = pool.normalize_names(names_a)
377
+ results_b = pool.normalize_names(names_b)
378
+
379
+ # One-off convenience wrapper (creates and closes a temporary pool)
380
+ single_batch = detector.process_name_batch_multiprocess(names_a, max_workers=6, chunk_size=64)
381
+ return results_a, results_b, single_batch
382
+
383
+ if __name__ == "__main__":
384
+ main()
385
+ ```
386
+
387
+ Use the `if __name__ == "__main__":` guard in scripts to ensure safe process
388
+ spawning on Windows and macOS.
389
+
360
390
  ### When to Use Batch Processing
361
391
 
362
392
  * **Academic Papers**: Author lists typically follow consistent formatting
@@ -431,14 +461,14 @@ If you'd like to contribute to Sinonym, here’s how to set up your development
431
461
  First, clone the repository:
432
462
 
433
463
  ```bash
434
- git clone https://github.com/yourusername/sinonym.git
464
+ git clone https://github.com/allenai/sinonym.git
435
465
  cd sinonym
436
466
  ```
437
467
 
438
468
  Then, install the development dependencies:
439
469
 
440
470
  ```bash
441
- uv sync --extra dev
471
+ uv sync --active --all-extras --dev
442
472
  ```
443
473
 
444
474
  ### Running Tests
@@ -459,6 +489,10 @@ uv run ruff check . --fix
459
489
  uv run ruff format .
460
490
  ```
461
491
 
492
+ ### Benchmarking & Profiling
493
+
494
+ See [scripts/README.md](scripts/README.md) for benchmark, profiling, and test status scripts.
495
+
462
496
  ## License
463
497
 
464
498
  Sinonym is licensed under the Apache 2.0 License. See the `LICENSE` file for more details.
@@ -111,7 +111,7 @@ Formatted Output
111
111
  ### 5. Performance
112
112
 
113
113
  * **High-Performance with Caching**
114
- * The library is benchmarked to be very fast, capable of processing over 10,000 diverse names per second, and uses caching to significantly speed up the processing of repeated names.
114
+ * The library is benchmarked to be very fast, capable of processing over 3,000 diverse names per second, and uses caching to significantly speed up the processing of repeated names.
115
115
 
116
116
  ## How It Works
117
117
 
@@ -320,6 +320,36 @@ for result in results:
320
320
  print(f"Processed: {result.result}")
321
321
  ```
322
322
 
323
+ ### Persistent Multi-Process Processing
324
+
325
+ For high-throughput workloads, you can keep a persistent process pool alive and
326
+ reuse worker processes across multiple calls. This avoids repeated process
327
+ start-up overhead and works on Windows/macOS/Linux via `spawn`.
328
+
329
+ ```python
330
+ from sinonym.detector import ChineseNameDetector
331
+
332
+ def main():
333
+ detector = ChineseNameDetector()
334
+ names_a = ["Li Wei", "Wang Weiming", "Zhang Ming"]
335
+ names_b = ["Xin Liu", "Yang Li", "Chen Huang"]
336
+
337
+ # Reuse workers across many calls
338
+ with detector.create_persistent_multiprocess_pool(max_workers=6, chunk_size=64) as pool:
339
+ results_a = pool.normalize_names(names_a)
340
+ results_b = pool.normalize_names(names_b)
341
+
342
+ # One-off convenience wrapper (creates and closes a temporary pool)
343
+ single_batch = detector.process_name_batch_multiprocess(names_a, max_workers=6, chunk_size=64)
344
+ return results_a, results_b, single_batch
345
+
346
+ if __name__ == "__main__":
347
+ main()
348
+ ```
349
+
350
+ Use the `if __name__ == "__main__":` guard in scripts to ensure safe process
351
+ spawning on Windows and macOS.
352
+
323
353
  ### When to Use Batch Processing
324
354
 
325
355
  * **Academic Papers**: Author lists typically follow consistent formatting
@@ -394,14 +424,14 @@ If you'd like to contribute to Sinonym, here’s how to set up your development
394
424
  First, clone the repository:
395
425
 
396
426
  ```bash
397
- git clone https://github.com/yourusername/sinonym.git
427
+ git clone https://github.com/allenai/sinonym.git
398
428
  cd sinonym
399
429
  ```
400
430
 
401
431
  Then, install the development dependencies:
402
432
 
403
433
  ```bash
404
- uv sync --extra dev
434
+ uv sync --active --all-extras --dev
405
435
  ```
406
436
 
407
437
  ### Running Tests
@@ -422,6 +452,10 @@ uv run ruff check . --fix
422
452
  uv run ruff format .
423
453
  ```
424
454
 
455
+ ### Benchmarking & Profiling
456
+
457
+ See [scripts/README.md](scripts/README.md) for benchmark, profiling, and test status scripts.
458
+
425
459
  ## License
426
460
 
427
461
  Sinonym is licensed under the Apache 2.0 License. See the `LICENSE` file for more details.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "sinonym"
7
- version = "0.2.2"
7
+ version = "0.2.4"
8
8
  description = "Chinese Name Detection and Normalization Module"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -0,0 +1,65 @@
1
+ # Sinonym Scripts
2
+
3
+ Utility scripts for benchmarking, profiling, testing, and model training.
4
+
5
+ ## Active Scripts
6
+
7
+ ### `check_test_status.py`
8
+ Runs the full test suite and reports individual test case failures with detailed diagnostics. Runs performance tests separately, then exits 0/1 based on whether the failure count matches the expected baseline (`EXPECTED_FAILURES = 52`). Improvements (fewer failures) also pass; regressions fail.
9
+
10
+ ```bash
11
+ uv run python scripts/check_test_status.py
12
+ ```
13
+
14
+ ### `benchmark_stable.py`
15
+ Median-based performance benchmark gate. Spawns isolated worker subprocesses (fresh process per run) with controlled `PYTHONHASHSEED` and thread environment variables. Reports mean/median/stddev/CV of throughput and supports a `--min-median-names-per-sec` gate that exits non-zero on failure.
16
+
17
+ ```bash
18
+ uv run python scripts/benchmark_stable.py --runs 5 --names 3000 --warmup 3000
19
+ uv run python scripts/benchmark_stable.py --runs 7 --min-median-names-per-sec 5000
20
+ ```
21
+
22
+ ### `profile_hotspots.py`
23
+ Hotspot time-share profiler. Warms caches on deterministic test names, runs one `cProfile` pass, then reports top functions and modules ranked by internal time (`tottime`) share. Use `--sinonym-only` to filter out third-party/stdlib noise.
24
+
25
+ ```bash
26
+ uv run python scripts/profile_hotspots.py --names 3000 --warmup 3000 --sinonym-only
27
+ ```
28
+
29
+ ### `profile_run.py`
30
+ Quick single-process profiling script. Generates deterministic test names, warms caches, takes 5 pure timing measurements (no profiling overhead) for accurate throughput stats, then runs one `cProfile` pass for a top-25 function breakdown. Good for a fast sanity check during development.
31
+
32
+ ```bash
33
+ uv run python scripts/profile_run.py
34
+ ```
35
+
36
+ ### `profile_threaded.py`
37
+ Multi-threaded performance and thread-safety validation. Tests `normalize_name` throughput across 1/2/4/8 threads using a shared `ChineseNameDetector` instance, verifies that multi-threaded results are identical to single-threaded results, and reports speedup and CV per thread count.
38
+
39
+ ```bash
40
+ uv run python scripts/profile_threaded.py
41
+ ```
42
+
43
+ ### `profile_multiprocess.py`
44
+ Persistent multi-process throughput and parity check. Compares single-process throughput to a spawn-based persistent process pool, verifies that outputs are identical for a deterministic workload, and reports median speedup.
45
+
46
+ ```bash
47
+ uv run python scripts/profile_multiprocess.py --names 12000 --warmup 3000 --runs 3 --workers 6 --chunk-size 64
48
+ ```
49
+
50
+ ### `train_ml_classifier_for_chinese_vs_japanese.py`
51
+ Trains the Chinese-vs-Japanese name classifier used in production. Downloads Chinese (~1.2M) and Japanese (~180K) name corpora, trains a scikit-learn pipeline (TF-IDF character n-grams + 20 linguistic heuristic features + logistic regression), and saves the model to `data/chinese_japanese_classifier.skops`.
52
+
53
+ ```bash
54
+ uv run python scripts/train_ml_classifier_for_chinese_vs_japanese.py
55
+ ```
56
+
57
+ ## Abandoned Scripts
58
+
59
+ These remain for historical reference but are not used by the library. The rule-based parser in `sinonym.services.parsing` replaced the ML approach.
60
+
61
+ ### `generate_chinese_name_corpus_data.py`
62
+ Was intended to generate training data for an ML-based name parsing disambiguation model. Downloads 200K Chinese names, romanizes them, generates all possible surname/given-name parses, and creates labeled training examples. The ML parsing model did not outperform the rule-based system.
63
+
64
+ ### `generate_acl_data.py`
65
+ Supplementary data generator for the abandoned ML parsing effort. Processes ACL 2025 conference author names to create additional training examples in a different distribution (romanized, Western ordering).
@@ -137,7 +137,7 @@ CANTONESE_PHONETIC_RULES = {
137
137
  "sau": "shou",
138
138
  "tau": "tao",
139
139
  "wun": "huan",
140
- "kai": "qi", # Cantonese mapping (overrides Wade-Giles "gai")
140
+ "kai": "kai", # Preserve "kai" by default; avoid forcing it to "qi" without explicit Cantonese context
141
141
  "hoi": "hai",
142
142
  "ngai": "ai",
143
143
  "tsz": "zi",
@@ -18,6 +18,8 @@ class ParsedName:
18
18
  parts appear when combined. For normalized output this is typically
19
19
  ["given", "middle", "surname"], while for original input order it may
20
20
  be ["surname", "given", "middle"], etc.
21
+ Component labels remain stable: `surname`/`given_name` always identify
22
+ those components regardless of `order`.
21
23
  """
22
24
  surname: str
23
25
  given_name: str
@@ -164,6 +164,7 @@ data structures and the detector can be safely used from multiple threads.
164
164
 
165
165
  import logging
166
166
  import string
167
+ import threading
167
168
 
168
169
  from sinonym.coretypes import BatchFormatPattern, BatchParseResult
169
170
  from sinonym.coretypes.results import ParsedName
@@ -181,6 +182,7 @@ from sinonym.services import (
181
182
  PinyinCacheService,
182
183
  ServiceContext,
183
184
  )
185
+ from sinonym.services.process_pool import PersistentMultiprocessNormalizer, normalize_names_multiprocess
184
186
 
185
187
  # ════════════════════════════════════════════════════════════════════════════════
186
188
  # MAIN CHINESE NAME DETECTOR CLASS
@@ -203,6 +205,7 @@ class ChineseNameDetector:
203
205
  self._parsing_service: NameParsingService | None = None
204
206
  self._formatting_service: NameFormattingService | None = None
205
207
  self._batch_analysis_service: BatchAnalysisService | None = None
208
+ self._batch_threshold_lock = threading.Lock()
206
209
 
207
210
  # Initialize data structures
208
211
  self._initialize()
@@ -332,13 +335,13 @@ class ChineseNameDetector:
332
335
  )
333
336
  # Determine original input order and assign components accordingly
334
337
  if token1_is_surname:
335
- # Original: surname-first → in original-order view, first part is "given"
338
+ # Original: surname-first → preserve component labels and annotate order
336
339
  order_list = ["surname", "given"]
337
340
  parsed_original_order = ParsedName(
338
- surname=given_str,
339
- given_name=surname_str,
340
- surname_tokens=given_final,
341
- given_tokens=surname_final,
341
+ surname=surname_str,
342
+ given_name=given_str,
343
+ surname_tokens=surname_final,
344
+ given_tokens=given_final,
342
345
  middle_name=" ".join(middle_tokens) if middle_tokens else "",
343
346
  middle_tokens=middle_tokens,
344
347
  order=order_list,
@@ -368,17 +371,15 @@ class ChineseNameDetector:
368
371
 
369
372
  # Try both possibilities and see which one the parsing service accepts
370
373
  # Option 1: First two tokens as compound surname + third as given
371
- compound_parse = self._parsing_service.parse_name_order(
374
+ compound_parse = self._parsing_service.parse_name_order_tokens(
372
375
  tokens,
373
376
  normalized_input.norm_map,
374
377
  normalized_input.compound_metadata,
375
378
  )
376
379
 
377
- if (compound_parse.success and
378
- len(compound_parse.result[0]) == 2 and
379
- len(compound_parse.result[1]) == 1):
380
+ if compound_parse is not None and len(compound_parse[0]) == 2 and len(compound_parse[1]) == 1:
380
381
  # Parsing service recognized first two as compound surname
381
- best_result = compound_parse.result
382
+ best_result = (compound_parse[0], compound_parse[1])
382
383
  else:
383
384
  # Option 2: First token as single surname + last two as given name
384
385
  best_result = ([tokens[0]], tokens[1:])
@@ -403,12 +404,13 @@ class ChineseNameDetector:
403
404
  middle_tokens=middle_tokens,
404
405
  order=["given", "middle", "surname"],
405
406
  )
406
- # For 3-character all-Chinese, original order is surname-first → swap labels
407
+ # For 3-character all-Chinese, original order is surname-first
408
+ # Preserve component labels and annotate only the order.
407
409
  parsed_original_order = ParsedName(
408
- surname=given_str,
409
- given_name=surname_str,
410
- surname_tokens=given_final,
411
- given_tokens=surname_final,
410
+ surname=surname_str,
411
+ given_name=given_str,
412
+ surname_tokens=surname_final,
413
+ given_tokens=given_final,
412
414
  middle_name=" ".join(middle_tokens) if middle_tokens else "",
413
415
  middle_tokens=middle_tokens,
414
416
  order=["surname", "given"],
@@ -421,76 +423,124 @@ class ChineseNameDetector:
421
423
  except ValueError as e:
422
424
  return ParseResult.failure(str(e))
423
425
  else:
424
- # Original logic for non-all-Chinese or multi-token inputs
426
+ # Evaluate both order hypotheses and pick the best-scoring parse
427
+ original_tokens = list(normalized_input.roman_tokens)
428
+ best_candidate = None
429
+
425
430
  for order in (normalized_input.roman_tokens, normalized_input.roman_tokens[::-1]):
426
- parse_result = self._parsing_service.parse_name_order(
427
- list(order),
431
+ order_tokens = list(order)
432
+ parse_result = self._parsing_service.parse_name_order_tokens(
433
+ order_tokens,
428
434
  normalized_input.norm_map,
429
435
  normalized_input.compound_metadata,
430
436
  )
431
- if parse_result.success:
432
- surname_tokens, given_tokens = parse_result.result
433
- try:
434
- formatted_name, given_final, surname_final, surname_str, given_str, middle_tokens = (
435
- self._formatting_service.format_name_output_with_tokens(
436
- surname_tokens,
437
- given_tokens,
438
- normalized_input.norm_map,
439
- normalized_input.compound_metadata,
440
- )
437
+ if parse_result is None:
438
+ continue
439
+
440
+ surname_tokens, given_tokens, original_compound_surname = parse_result
441
+ score = self._parsing_service.calculate_parse_score(
442
+ surname_tokens,
443
+ given_tokens,
444
+ original_tokens,
445
+ normalized_input.norm_map,
446
+ False,
447
+ original_compound_surname,
448
+ )
449
+ used_original = order_tokens == original_tokens
450
+
451
+ candidate = {
452
+ "surname_tokens": surname_tokens,
453
+ "given_tokens": given_tokens,
454
+ "score": score,
455
+ "order_tokens": order_tokens,
456
+ "used_original": used_original,
457
+ }
458
+
459
+ if (
460
+ best_candidate is None
461
+ or candidate["score"] > best_candidate["score"]
462
+ or (
463
+ candidate["score"] == best_candidate["score"]
464
+ and candidate["used_original"]
465
+ and not best_candidate["used_original"]
466
+ )
467
+ ):
468
+ best_candidate = candidate
469
+
470
+ if best_candidate is not None:
471
+ surname_tokens = best_candidate["surname_tokens"]
472
+ given_tokens = best_candidate["given_tokens"]
473
+ order_tokens = best_candidate["order_tokens"]
474
+ used_original = best_candidate["used_original"]
475
+ try:
476
+ formatted_name, given_final, surname_final, surname_str, given_str, middle_tokens = (
477
+ self._formatting_service.format_name_output_with_tokens(
478
+ surname_tokens,
479
+ given_tokens,
480
+ normalized_input.norm_map,
481
+ normalized_input.compound_metadata,
441
482
  )
442
- parsed = ParsedName(
483
+ )
484
+ parsed = ParsedName(
485
+ surname=surname_str,
486
+ given_name=given_str,
487
+ surname_tokens=surname_final,
488
+ given_tokens=given_final,
489
+ middle_name=" ".join(middle_tokens) if middle_tokens else "",
490
+ middle_tokens=middle_tokens,
491
+ order=["given", "middle", "surname"],
492
+ )
493
+ # Determine original input order relative to detected parse
494
+ k = len(surname_tokens)
495
+ is_surname_first_in_this_order = list(order_tokens[:k]) == surname_tokens
496
+ is_surname_last_in_this_order = list(order_tokens[-k:]) == surname_tokens
497
+
498
+ # Compound surname fallback: surname_tokens may be sub-tokens
499
+ # of a single original token (e.g. ['Ou','yang'] from 'Ouyang')
500
+ if not is_surname_first_in_this_order and not is_surname_last_in_this_order:
501
+ joined_surname = "".join(surname_tokens).lower()
502
+ if order_tokens[0].lower() == joined_surname:
503
+ is_surname_first_in_this_order = True
504
+ elif order_tokens[-1].lower() == joined_surname:
505
+ is_surname_last_in_this_order = True
506
+
507
+ if used_original:
508
+ original_is_given_first = is_surname_last_in_this_order
509
+ else:
510
+ original_is_given_first = is_surname_first_in_this_order
511
+
512
+ if original_is_given_first:
513
+ order_list = ["given"] + (["middle"] if middle_tokens else []) + ["surname"]
514
+ # Keep labels as-is
515
+ parsed_original_order = ParsedName(
443
516
  surname=surname_str,
444
517
  given_name=given_str,
445
518
  surname_tokens=surname_final,
446
519
  given_tokens=given_final,
447
520
  middle_name=" ".join(middle_tokens) if middle_tokens else "",
448
521
  middle_tokens=middle_tokens,
449
- order=["given", "middle", "surname"],
522
+ order=order_list,
450
523
  )
451
- # Determine original input order relative to detected parse
452
- used_original = order == normalized_input.roman_tokens
453
- k = len(surname_tokens)
454
- is_surname_first_in_this_order = list(order[:k]) == surname_tokens
455
- is_surname_last_in_this_order = list(order[-k:]) == surname_tokens
456
-
457
- if used_original:
458
- original_is_given_first = is_surname_last_in_this_order
459
- else:
460
- original_is_given_first = is_surname_first_in_this_order
461
-
462
- if original_is_given_first:
463
- order_list = ["given"] + (["middle"] if middle_tokens else []) + ["surname"]
464
- # Keep labels as-is
465
- parsed_original_order = ParsedName(
466
- surname=surname_str,
467
- given_name=given_str,
468
- surname_tokens=surname_final,
469
- given_tokens=given_final,
470
- middle_name=" ".join(middle_tokens) if middle_tokens else "",
471
- middle_tokens=middle_tokens,
472
- order=order_list,
473
- )
474
- else:
475
- order_list = ["surname"] + (["middle"] if middle_tokens else []) + ["given"]
476
- # Original: surname-first → swap labels to reflect "first part" as given
477
- parsed_original_order = ParsedName(
478
- surname=given_str,
479
- given_name=surname_str,
480
- surname_tokens=given_final,
481
- given_tokens=surname_final,
482
- middle_name=" ".join(middle_tokens) if middle_tokens else "",
483
- middle_tokens=middle_tokens,
484
- order=order_list,
485
- )
486
-
487
- return ParseResult.success_with_name(
488
- formatted_name,
489
- parsed=parsed,
490
- parsed_original_order=parsed_original_order,
524
+ else:
525
+ order_list = ["surname"] + (["middle"] if middle_tokens else []) + ["given"]
526
+ # Original: surname-first → keep labels stable and annotate order only
527
+ parsed_original_order = ParsedName(
528
+ surname=surname_str,
529
+ given_name=given_str,
530
+ surname_tokens=surname_final,
531
+ given_tokens=given_final,
532
+ middle_name=" ".join(middle_tokens) if middle_tokens else "",
533
+ middle_tokens=middle_tokens,
534
+ order=order_list,
491
535
  )
492
- except ValueError as e:
493
- return ParseResult.failure(str(e))
536
+
537
+ return ParseResult.success_with_name(
538
+ formatted_name,
539
+ parsed=parsed,
540
+ parsed_original_order=parsed_original_order,
541
+ )
542
+ except ValueError as e:
543
+ return ParseResult.failure(str(e))
494
544
 
495
545
  return ParseResult.failure("name not recognised as Chinese")
496
546
 
@@ -533,21 +583,22 @@ class ChineseNameDetector:
533
583
  individual_results = [self.normalize_name(name) for name in names]
534
584
  return self._create_fallback_batch_result(names, individual_results)
535
585
 
536
- # Configure threshold for this analysis
537
- original_threshold = self._batch_analysis_service._format_threshold
538
- self._batch_analysis_service._format_threshold = format_threshold
539
-
540
- try:
541
- return self._batch_analysis_service.analyze_name_batch(
542
- names,
543
- self._normalizer,
544
- self._data,
545
- self._formatting_service,
546
- minimum_batch_size,
547
- )
548
- finally:
549
- # Restore original threshold
550
- self._batch_analysis_service._format_threshold = original_threshold
586
+ with self._batch_threshold_lock:
587
+ # Configure threshold for this analysis
588
+ original_threshold = self._batch_analysis_service._format_threshold
589
+ self._batch_analysis_service._format_threshold = format_threshold
590
+
591
+ try:
592
+ return self._batch_analysis_service.analyze_name_batch(
593
+ names,
594
+ self._normalizer,
595
+ self._data,
596
+ self._formatting_service,
597
+ minimum_batch_size,
598
+ )
599
+ finally:
600
+ # Restore original threshold
601
+ self._batch_analysis_service._format_threshold = original_threshold
551
602
 
552
603
  def detect_batch_format(
553
604
  self,
@@ -587,19 +638,20 @@ class ChineseNameDetector:
587
638
  threshold_met=False,
588
639
  )
589
640
 
590
- # Configure threshold for this analysis
591
- original_threshold = self._batch_analysis_service._format_threshold
592
- self._batch_analysis_service._format_threshold = format_threshold
641
+ with self._batch_threshold_lock:
642
+ # Configure threshold for this analysis
643
+ original_threshold = self._batch_analysis_service._format_threshold
644
+ self._batch_analysis_service._format_threshold = format_threshold
593
645
 
594
- try:
595
- return self._batch_analysis_service.detect_batch_format(
596
- names,
597
- self._normalizer,
598
- self._data,
599
- )
600
- finally:
601
- # Restore original threshold
602
- self._batch_analysis_service._format_threshold = original_threshold
646
+ try:
647
+ return self._batch_analysis_service.detect_batch_format(
648
+ names,
649
+ self._normalizer,
650
+ self._data,
651
+ )
652
+ finally:
653
+ # Restore original threshold
654
+ self._batch_analysis_service._format_threshold = original_threshold
603
655
 
604
656
  def process_name_batch(
605
657
  self,
@@ -632,8 +684,58 @@ class ChineseNameDetector:
632
684
  batch_result = self.analyze_name_batch(names, format_threshold, minimum_batch_size)
633
685
  return batch_result.results
634
686
 
687
+ def create_persistent_multiprocess_pool(
688
+ self,
689
+ *,
690
+ max_workers: int | None = None,
691
+ chunk_size: int = 64,
692
+ mp_start_method: str = "spawn",
693
+ ) -> PersistentMultiprocessNormalizer:
694
+ """
695
+ Create a persistent multi-process pool for repeated normalization calls.
696
+
697
+ Notes:
698
+ - Uses one detector instance per worker process.
699
+ - For Windows/macOS scripts, call this behind an
700
+ `if __name__ == "__main__":` guard.
701
+ """
702
+ self._ensure_initialized()
703
+ return PersistentMultiprocessNormalizer(
704
+ max_workers=max_workers,
705
+ chunk_size=chunk_size,
706
+ mp_start_method=mp_start_method,
707
+ detector_config=self._config,
708
+ detector_weights=self._weights,
709
+ )
710
+
711
+ def process_name_batch_multiprocess(
712
+ self,
713
+ names: list[str],
714
+ *,
715
+ max_workers: int | None = None,
716
+ chunk_size: int = 64,
717
+ mp_start_method: str = "spawn",
718
+ ) -> list[ParseResult]:
719
+ """
720
+ Process one batch with a temporary multi-process pool.
721
+
722
+ For repeated calls, prefer `create_persistent_multiprocess_pool()`
723
+ to avoid repeated process start-up overhead.
724
+ """
725
+ self._ensure_initialized()
726
+ return normalize_names_multiprocess(
727
+ names,
728
+ max_workers=max_workers,
729
+ chunk_size=chunk_size,
730
+ mp_start_method=mp_start_method,
731
+ detector_config=self._config,
732
+ detector_weights=self._weights,
733
+ )
734
+
635
735
  def _create_fallback_batch_result(
636
- self, names: list[str], individual_results: list[ParseResult],
736
+ self,
737
+ names: list[str],
738
+ individual_results: list[ParseResult],
637
739
  ) -> BatchParseResult:
638
740
  """Create a fallback BatchParseResult when batch analysis is not available."""
639
741
  from sinonym.coretypes import BatchFormatPattern, IndividualAnalysis, NameFormat