sinonym 0.2.4__tar.gz → 0.2.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. {sinonym-0.2.4 → sinonym-0.2.7}/PKG-INFO +2 -1
  2. {sinonym-0.2.4 → sinonym-0.2.7}/pyproject.toml +2 -1
  3. {sinonym-0.2.4 → sinonym-0.2.7}/scripts/README.md +1 -1
  4. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/detector.py +116 -2
  5. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/batch_analysis.py +158 -5
  6. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/formatting.py +21 -1
  7. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/parsing.py +63 -6
  8. sinonym-0.2.7/sinonym/timo/__init__.py +0 -0
  9. sinonym-0.2.7/sinonym/timo/config.yaml +12 -0
  10. sinonym-0.2.7/sinonym/timo/integration_test.py +70 -0
  11. sinonym-0.2.7/sinonym/timo/interface.py +244 -0
  12. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/utils/string_manipulation.py +79 -4
  13. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_all_chinese_inputs.py +3 -0
  14. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_regression_proposals.py +170 -0
  15. sinonym-0.2.7/tests/test_timo_interface.py +87 -0
  16. {sinonym-0.2.4 → sinonym-0.2.7}/.gitignore +0 -0
  17. {sinonym-0.2.4 → sinonym-0.2.7}/LICENSE +0 -0
  18. {sinonym-0.2.4 → sinonym-0.2.7}/README.md +0 -0
  19. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/__init__.py +0 -0
  20. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/chinese_names_data.py +0 -0
  21. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/coretypes/__init__.py +0 -0
  22. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/coretypes/config.py +0 -0
  23. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/coretypes/results.py +0 -0
  24. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/README.md +0 -0
  25. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/acl_2025_authors.txt +0 -0
  26. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/chinese_japanese_classifier.joblib +0 -0
  27. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/chinese_japanese_classifier.skops +0 -0
  28. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/familyname_orcid.csv +0 -0
  29. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/givenname_orcid.csv +0 -0
  30. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/data/model_features.json +0 -0
  31. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/ml_model_components.py +0 -0
  32. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/patterns/__init__.py +0 -0
  33. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/patterns/compiled_patterns.py +0 -0
  34. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/patterns/regex_builders.py +0 -0
  35. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/resources.py +0 -0
  36. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/__init__.py +0 -0
  37. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/cache.py +0 -0
  38. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/ethnicity.py +0 -0
  39. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/initialization.py +0 -0
  40. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/normalization.py +0 -0
  41. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/services/process_pool.py +0 -0
  42. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/text_processing/__init__.py +0 -0
  43. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/text_processing/compound_detector.py +0 -0
  44. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/text_processing/text_normalizer.py +0 -0
  45. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/text_processing/text_preprocessor.py +0 -0
  46. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/utils/__init__.py +0 -0
  47. {sinonym-0.2.4 → sinonym-0.2.7}/sinonym/utils/thread_cache.py +0 -0
  48. {sinonym-0.2.4 → sinonym-0.2.7}/tests/_fail_log.py +0 -0
  49. {sinonym-0.2.4 → sinonym-0.2.7}/tests/conftest.py +0 -0
  50. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_acl.py +0 -0
  51. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_basic_chinese_names.py +0 -0
  52. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_batch.py +0 -0
  53. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_compound_names.py +0 -0
  54. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_compound_surname_formats.py +0 -0
  55. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_middle_names.py +0 -0
  56. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_misc.py +0 -0
  57. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_mixed_production_cases.py +0 -0
  58. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_mixed_scripts.py +0 -0
  59. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_ml_japanese_detection.py +0 -0
  60. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_multiprocess_pool.py +0 -0
  61. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_name_formatting.py +0 -0
  62. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_non_chinese_rejection.py +0 -0
  63. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_performance.py +0 -0
  64. {sinonym-0.2.4 → sinonym-0.2.7}/tests/test_regional_variants.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sinonym
3
- Version: 0.2.4
3
+ Version: 0.2.7
4
4
  Summary: Chinese Name Detection and Normalization Module
5
5
  Project-URL: Homepage, https://github.com/allenai/sinonym
6
6
  Project-URL: Repository, https://github.com/allenai/sinonym
@@ -21,6 +21,7 @@ Classifier: Topic :: Text Processing :: Linguistic
21
21
  Requires-Python: >=3.10
22
22
  Requires-Dist: joblib>=1.3
23
23
  Requires-Dist: numpy>=1.24
24
+ Requires-Dist: pydantic<2,>=1.10
24
25
  Requires-Dist: pypinyin>=0.44.0
25
26
  Requires-Dist: requests>=2.31
26
27
  Requires-Dist: scikit-learn<2,>=1.2
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "sinonym"
7
- version = "0.2.4"
7
+ version = "0.2.7"
8
8
  description = "Chinese Name Detection and Normalization Module"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -27,6 +27,7 @@ keywords = ["chinese", "names", "nlp", "romanization", "pinyin"]
27
27
  dependencies = [
28
28
  "joblib>=1.3",
29
29
  "numpy>=1.24",
30
+ "pydantic>=1.10,<2",
30
31
  "pypinyin>=0.44.0",
31
32
  "requests>=2.31",
32
33
  "scikit-learn>=1.2,<2",
@@ -5,7 +5,7 @@ Utility scripts for benchmarking, profiling, testing, and model training.
5
5
  ## Active Scripts
6
6
 
7
7
  ### `check_test_status.py`
8
- Runs the full test suite and reports individual test case failures with detailed diagnostics. Runs performance tests separately, then exits 0/1 based on whether the failure count matches the expected baseline (`EXPECTED_FAILURES = 52`). Improvements (fewer failures) also pass; regressions fail.
8
+ Runs the full test suite and reports individual test case failures with detailed diagnostics. Runs performance tests separately, then exits 0/1 based on whether the failure count matches the expected baseline configured in `scripts/check_test_status.py`. Improvements (fewer failures) also pass; regressions fail.
9
9
 
10
10
  ```bash
11
11
  uv run python scripts/check_test_status.py
@@ -178,6 +178,7 @@ from sinonym.services import (
178
178
  NameFormattingService,
179
179
  NameParsingService,
180
180
  NormalizationService,
181
+ NormalizedInput,
181
182
  ParseResult,
182
183
  PinyinCacheService,
183
184
  ServiceContext,
@@ -244,6 +245,114 @@ class ChineseNameDetector:
244
245
  # Initialize services
245
246
  self._initialize_services()
246
247
 
248
+ def _is_surname_group(self, tokens: list[str], normalized_cache: dict[str, str]) -> bool:
249
+ """Return whether a romanized Han token group is a surname component."""
250
+ if len(tokens) == 1:
251
+ token = tokens[0]
252
+ normalized = self._normalizer.get_normalized(token, normalized_cache)
253
+ return self._data.is_surname(token, normalized)
254
+
255
+ normalized_tokens = [self._normalizer.get_normalized(token, normalized_cache) for token in tokens]
256
+ spaced = " ".join(normalized_tokens)
257
+ compact = "".join(normalized_tokens)
258
+
259
+ return (
260
+ spaced in self._data.compound_surnames
261
+ or spaced in self._data.compound_surnames_normalized
262
+ or compact in self._data.compound_original_format_map
263
+ )
264
+
265
+ def _is_compound_surname_group(self, tokens: list[str], normalized_cache: dict[str, str]) -> bool:
266
+ """Return whether a romanized Han token group is a compound surname."""
267
+ return len(tokens) > 1 and self._is_surname_group(tokens, normalized_cache)
268
+
269
+ def _has_only_cjk_token_groups(self, normalized_input: NormalizedInput) -> bool:
270
+ """Return whether separator-delimited input tokens are all CJK characters."""
271
+ return len(normalized_input.tokens) > 1 and all(
272
+ token and all(self._config.cjk_pattern.search(char) for char in token)
273
+ for token in normalized_input.tokens
274
+ )
275
+
276
+ def _format_parse_result(
277
+ self,
278
+ surname_tokens: list[str],
279
+ given_tokens: list[str],
280
+ normalized_input: NormalizedInput,
281
+ original_order: list[str],
282
+ ) -> ParseResult:
283
+ """Format parsed components and attach stable structured name fields."""
284
+ formatted_name, given_final, surname_final, surname_str, given_str, middle_tokens = (
285
+ self._formatting_service.format_name_output_with_tokens(
286
+ surname_tokens,
287
+ given_tokens,
288
+ normalized_input.norm_map,
289
+ normalized_input.compound_metadata,
290
+ )
291
+ )
292
+ parsed = ParsedName(
293
+ surname=surname_str,
294
+ given_name=given_str,
295
+ surname_tokens=surname_final,
296
+ given_tokens=given_final,
297
+ middle_name=" ".join(middle_tokens) if middle_tokens else "",
298
+ middle_tokens=middle_tokens,
299
+ order=["given", "middle", "surname"],
300
+ )
301
+ parsed_original_order = ParsedName(
302
+ surname=surname_str,
303
+ given_name=given_str,
304
+ surname_tokens=surname_final,
305
+ given_tokens=given_final,
306
+ middle_name=" ".join(middle_tokens) if middle_tokens else "",
307
+ middle_tokens=middle_tokens,
308
+ order=original_order,
309
+ )
310
+ return ParseResult.success_with_name(
311
+ formatted_name,
312
+ parsed=parsed,
313
+ parsed_original_order=parsed_original_order,
314
+ )
315
+
316
+ def _normalize_spaced_all_chinese_name(self, normalized_input: NormalizedInput) -> ParseResult | None:
317
+ """Parse all-Han names whose whitespace already separates name components."""
318
+ if (
319
+ len(normalized_input.tokens) != self._config.min_tokens_required
320
+ or len(normalized_input.roman_tokens) <= self._config.min_tokens_required
321
+ ):
322
+ return None
323
+
324
+ first_group = self._cache_service.han_to_pinyin_fast(normalized_input.tokens[0])
325
+ last_group = self._cache_service.han_to_pinyin_fast(normalized_input.tokens[1])
326
+ if not first_group or not last_group:
327
+ return None
328
+
329
+ # OCR/noisy spacing can split a compound surname across the group boundary.
330
+ boundary_compound = first_group + last_group[:1] if len(first_group) == 1 and len(last_group) > 1 else []
331
+ if boundary_compound and self._is_surname_group(boundary_compound, normalized_input.norm_map):
332
+ surname_tokens = boundary_compound
333
+ given_tokens = last_group[1:]
334
+ original_order = ["surname", "given"]
335
+ else:
336
+ first_is_surname = self._is_surname_group(first_group, normalized_input.norm_map)
337
+ last_is_surname = self._is_surname_group(last_group, normalized_input.norm_map)
338
+ last_is_compound_surname = self._is_compound_surname_group(last_group, normalized_input.norm_map)
339
+
340
+ if last_is_surname and (not first_is_surname or last_is_compound_surname):
341
+ surname_tokens = last_group
342
+ given_tokens = first_group
343
+ original_order = ["given", "surname"]
344
+ elif first_is_surname:
345
+ surname_tokens = first_group
346
+ given_tokens = last_group
347
+ original_order = ["surname", "given"]
348
+ else:
349
+ return None
350
+
351
+ try:
352
+ return self._format_parse_result(surname_tokens, given_tokens, normalized_input, original_order)
353
+ except ValueError as e:
354
+ return ParseResult.failure(str(e))
355
+
247
356
  # Public API methods
248
357
  def get_cache_info(self) -> CacheInfo:
249
358
  """Get cache information."""
@@ -291,6 +400,11 @@ class ChineseNameDetector:
291
400
 
292
401
  # Try parsing in both orders - for all-Chinese inputs, choose best scoring parse
293
402
 
403
+ if self._has_only_cjk_token_groups(normalized_input):
404
+ grouped_result = self._normalize_spaced_all_chinese_name(normalized_input)
405
+ if grouped_result is not None:
406
+ return grouped_result
407
+
294
408
  if is_all_chinese and len(normalized_input.roman_tokens) == self._config.min_tokens_required:
295
409
  # For all-Chinese 2-token inputs, ALWAYS assume surname-first order
296
410
  # Two-character Chinese names are always (surname, given_name)
@@ -443,8 +557,8 @@ class ChineseNameDetector:
443
557
  given_tokens,
444
558
  original_tokens,
445
559
  normalized_input.norm_map,
446
- False,
447
- original_compound_surname,
560
+ is_all_chinese=False,
561
+ original_compound_format=original_compound_surname,
448
562
  )
449
563
  used_original = order_tokens == original_tokens
450
564
 
@@ -8,6 +8,7 @@ improve parsing accuracy for ambiguous individual names.
8
8
 
9
9
  from __future__ import annotations
10
10
 
11
+ from statistics import median
11
12
  from typing import TYPE_CHECKING
12
13
 
13
14
  from sinonym.coretypes import (
@@ -20,6 +21,12 @@ from sinonym.coretypes import (
20
21
  )
21
22
  from sinonym.coretypes.results import ParsedName
22
23
 
24
+ GUARDED_GIVEN_FIRST_BATCH_MIN_SHARE = 0.75
25
+ GIVEN_NAME_SHAPE_MIN_SHARE = 0.5
26
+ ULTRA_LOW_FIRST_SURNAME_MEDIAN_MAX = 130.0
27
+ LONG_GIVEN_NAME_TOKEN_MIN_LENGTH = 6
28
+ TWO_TOKEN_NAME_LENGTH = 2
29
+
23
30
  if TYPE_CHECKING:
24
31
  from sinonym.services.ethnicity import EthnicityClassificationService
25
32
  from sinonym.services.parsing import NameParsingService
@@ -69,10 +76,15 @@ class BatchAnalysisService:
69
76
  # Normalize once and reuse
70
77
  normalized_input = normalizer.apply(name)
71
78
  candidates, best_candidate = self._analyze_individual_name_with_normalized(
72
- name, normalized_input, data,
79
+ name,
80
+ normalized_input,
81
+ data,
82
+ allow_guarded_given_first_bonus=False,
73
83
  )
74
84
  name_candidates.append((name, candidates, best_candidate, normalized_input.compound_metadata))
75
85
 
86
+ name_candidates = self._promote_guarded_given_first_batch_votes(name_candidates, normalizer, data)
87
+
76
88
  # Phase 2: Detect the dominant format pattern
77
89
  format_pattern = self._detect_format_pattern(name_candidates)
78
90
 
@@ -119,10 +131,15 @@ class BatchAnalysisService:
119
131
  # Normalize once for format detection (compound_metadata not needed)
120
132
  normalized_input = normalizer.apply(name)
121
133
  candidates, best_candidate = self._analyze_individual_name_with_normalized(
122
- name, normalized_input, data,
134
+ name,
135
+ normalized_input,
136
+ data,
137
+ allow_guarded_given_first_bonus=False,
123
138
  )
124
139
  name_candidates.append((name, candidates, best_candidate, None)) # No compound_metadata needed for format detection
125
140
 
141
+ name_candidates = self._promote_guarded_given_first_batch_votes(name_candidates, normalizer, data)
142
+
126
143
  return self._detect_format_pattern(name_candidates)
127
144
 
128
145
  def _process_individually(self, names: list[str], normalizer, data, formatting_service) -> BatchParseResult:
@@ -181,7 +198,12 @@ class BatchAnalysisService:
181
198
  )
182
199
 
183
200
  def _analyze_individual_name_with_normalized(
184
- self, name: str, normalized_input, _data,
201
+ self,
202
+ name: str,
203
+ normalized_input,
204
+ _data,
205
+ *,
206
+ allow_guarded_given_first_bonus: bool = True,
185
207
  ) -> tuple[list[ParseCandidate], ParseCandidate | None]:
186
208
  """Analyze a single name using pre-computed normalized input."""
187
209
  tokens = list(normalized_input.roman_tokens)
@@ -218,8 +240,9 @@ class BatchAnalysisService:
218
240
  given_tokens,
219
241
  tokens,
220
242
  normalized_input.norm_map,
221
- False,
222
- original_compound_format,
243
+ is_all_chinese=False,
244
+ original_compound_format=original_compound_format,
245
+ allow_guarded_given_first_bonus=allow_guarded_given_first_bonus,
223
246
  )
224
247
 
225
248
  # Determine the format of this parse
@@ -240,6 +263,136 @@ class BatchAnalysisService:
240
263
  best_candidate = candidates[0] if candidates else None
241
264
  return candidates, best_candidate
242
265
 
266
+ def _promote_guarded_given_first_batch_votes(
267
+ self,
268
+ name_candidates: list[tuple[str, list[ParseCandidate], ParseCandidate | None, dict | None]],
269
+ normalizer,
270
+ data,
271
+ ) -> list[tuple[str, list[ParseCandidate], ParseCandidate | None, dict | None]]:
272
+ """Promote contested given-first votes when batch-level shape evidence supports them."""
273
+ participant_count = sum(1 for _name, candidates, best_candidate, _ in name_candidates if candidates and best_candidate)
274
+ if participant_count == 0:
275
+ return name_candidates
276
+
277
+ promoted, first_surname_freqs, given_shape_count = self._collect_guarded_given_first_promotions(
278
+ name_candidates,
279
+ normalizer,
280
+ data,
281
+ )
282
+
283
+ if not self._should_promote_guarded_given_first_votes(
284
+ participant_count,
285
+ promoted,
286
+ first_surname_freqs,
287
+ given_shape_count,
288
+ ):
289
+ return name_candidates
290
+
291
+ adjusted = list(name_candidates)
292
+ for index, promoted_candidate in promoted:
293
+ name, candidates, _best_candidate, compound_metadata = adjusted[index]
294
+ adjusted[index] = (name, candidates, promoted_candidate, compound_metadata)
295
+ return adjusted
296
+
297
+ def _collect_guarded_given_first_promotions(
298
+ self,
299
+ name_candidates: list[tuple[str, list[ParseCandidate], ParseCandidate | None, dict | None]],
300
+ normalizer,
301
+ data,
302
+ ) -> tuple[list[tuple[int, ParseCandidate]], list[float], int]:
303
+ """Collect given-first candidates that only become best under individual guarded scoring."""
304
+ promoted: list[tuple[int, ParseCandidate]] = []
305
+ first_surname_freqs: list[float] = []
306
+ given_shape_count = 0
307
+
308
+ for index, (name, candidates, best_candidate, _compound_metadata) in enumerate(name_candidates):
309
+ promotion = self._guarded_given_first_promotion(name, candidates, best_candidate, normalizer, data)
310
+ if promotion is None:
311
+ continue
312
+
313
+ promoted_candidate, first_token, first_norm = promotion
314
+ promoted.append((index, promoted_candidate))
315
+ first_surname_freqs.append(data.get_surname_freq(first_norm))
316
+ if self._has_given_name_shape(first_token):
317
+ given_shape_count += 1
318
+
319
+ return promoted, first_surname_freqs, given_shape_count
320
+
321
+ def _guarded_given_first_promotion(
322
+ self,
323
+ name: str,
324
+ candidates: list[ParseCandidate],
325
+ best_candidate: ParseCandidate | None,
326
+ normalizer,
327
+ data,
328
+ ) -> tuple[ParseCandidate, str, str] | None:
329
+ """Return the promoted given-first candidate and first-token evidence, if any."""
330
+ if not candidates or best_candidate is None or best_candidate.format != NameFormat.SURNAME_FIRST:
331
+ return None
332
+
333
+ normalized_input = normalizer.apply(name)
334
+ tokens = list(normalized_input.roman_tokens)
335
+ if len(tokens) != TWO_TOKEN_NAME_LENGTH:
336
+ return None
337
+
338
+ _individual_candidates, individual_best = self._analyze_individual_name_with_normalized(
339
+ name,
340
+ normalized_input,
341
+ data,
342
+ allow_guarded_given_first_bonus=True,
343
+ )
344
+ if individual_best is None or individual_best.format != NameFormat.GIVEN_FIRST:
345
+ return None
346
+
347
+ promoted_candidate = self._matching_given_first_candidate(candidates, individual_best)
348
+ if promoted_candidate is None:
349
+ return None
350
+
351
+ first_token = tokens[0]
352
+ first_norm = normalizer.get_normalized(first_token, normalized_input.norm_map).replace(" ", "")
353
+ return promoted_candidate, first_token, first_norm
354
+
355
+ @staticmethod
356
+ def _matching_given_first_candidate(
357
+ candidates: list[ParseCandidate],
358
+ individual_best: ParseCandidate,
359
+ ) -> ParseCandidate | None:
360
+ """Find the no-bonus candidate that matches the individual guarded winner."""
361
+ return next(
362
+ (
363
+ candidate
364
+ for candidate in candidates
365
+ if candidate.format == NameFormat.GIVEN_FIRST
366
+ and candidate.surname_tokens == individual_best.surname_tokens
367
+ and candidate.given_tokens == individual_best.given_tokens
368
+ ),
369
+ None,
370
+ )
371
+
372
+ @staticmethod
373
+ def _should_promote_guarded_given_first_votes(
374
+ participant_count: int,
375
+ promoted: list[tuple[int, ParseCandidate]],
376
+ first_surname_freqs: list[float],
377
+ given_shape_count: int,
378
+ ) -> bool:
379
+ """Return whether guarded given-first votes have enough batch-level support."""
380
+ if not promoted or len(promoted) / participant_count < GUARDED_GIVEN_FIRST_BATCH_MIN_SHARE:
381
+ return False
382
+
383
+ has_shape_evidence = given_shape_count / len(promoted) >= GIVEN_NAME_SHAPE_MIN_SHARE
384
+ has_ultra_low_first_surname_evidence = (
385
+ bool(first_surname_freqs)
386
+ and median(first_surname_freqs) < ULTRA_LOW_FIRST_SURNAME_MEDIAN_MAX
387
+ )
388
+ return has_shape_evidence or has_ultra_low_first_surname_evidence
389
+
390
+ @staticmethod
391
+ def _has_given_name_shape(token: str) -> bool:
392
+ """Return whether a token has independent shape evidence for given-name position."""
393
+ raw = token.replace("-", "").replace("'", "")
394
+ return len(raw) >= LONG_GIVEN_NAME_TOKEN_MIN_LENGTH
395
+
243
396
  def _determine_parse_format(
244
397
  self, surname_tokens: list[str], _given_tokens: list[str], original_tokens: list[str],
245
398
  ) -> NameFormat:
@@ -86,6 +86,14 @@ class NameFormattingService:
86
86
  self._normalizer,
87
87
  self._config,
88
88
  )
89
+ if not split:
90
+ split = StringManipulationUtils.split_surname_like_given_name(
91
+ token,
92
+ normalized_cache,
93
+ self._data,
94
+ self._normalizer,
95
+ self._config,
96
+ )
89
97
  if split:
90
98
  parts.extend(split)
91
99
  # Final validation: accept if it's a valid Chinese token
@@ -217,8 +225,20 @@ class NameFormattingService:
217
225
  continue
218
226
 
219
227
  split = StringManipulationUtils.split_concatenated_name(
220
- token, normalized_cache, self._data, self._normalizer, self._config,
228
+ token,
229
+ normalized_cache,
230
+ self._data,
231
+ self._normalizer,
232
+ self._config,
221
233
  )
234
+ if not split:
235
+ split = StringManipulationUtils.split_surname_like_given_name(
236
+ token,
237
+ normalized_cache,
238
+ self._data,
239
+ self._normalizer,
240
+ self._config,
241
+ )
222
242
  if split:
223
243
  parts.extend(split)
224
244
  elif self._normalizer.is_valid_given_name_token(token, normalized_cache):
@@ -18,6 +18,10 @@ from sinonym.utils.string_manipulation import StringManipulationUtils
18
18
  if TYPE_CHECKING:
19
19
  from sinonym.services.normalization import CompoundMetadata
20
20
 
21
+ LOW_FREQUENCY_SURNAME_MAX = 500.0
22
+ GIVEN_FIRST_SURNAME_FREQ_RATIO_MIN = 50.0
23
+ GIVEN_FIRST_ORDER_PRESERVATION_BONUS = 4.0
24
+
21
25
 
22
26
  class NameParsingService:
23
27
  """Service for parsing Chinese names into surname and given name components."""
@@ -110,7 +114,13 @@ class NameParsingService:
110
114
  given_tokens = order[given_slice]
111
115
  if given_tokens:
112
116
  # Check if this parse would have a reasonable score
113
- score = self.calculate_parse_score(surname_tokens, given_tokens, order, normalized_cache, False)
117
+ score = self.calculate_parse_score(
118
+ surname_tokens,
119
+ given_tokens,
120
+ order,
121
+ normalized_cache,
122
+ is_all_chinese=False,
123
+ )
114
124
 
115
125
  # Western name detection pattern
116
126
  has_single_letter_given = any(len(token) == 1 for token in given_tokens)
@@ -159,7 +169,13 @@ class NameParsingService:
159
169
  surname_tokens = [surname_token]
160
170
  given_tokens = order[given_slice]
161
171
  if given_tokens:
162
- score = self.calculate_parse_score(surname_tokens, given_tokens, order, normalized_cache, False)
172
+ score = self.calculate_parse_score(
173
+ surname_tokens,
174
+ given_tokens,
175
+ order,
176
+ normalized_cache,
177
+ is_all_chinese=False,
178
+ )
163
179
 
164
180
  has_single_letter_given = any(len(token) == 1 for token in given_tokens)
165
181
  has_multi_syllable_tokens = any(len(token) > 3 for token in order)
@@ -246,8 +262,8 @@ class NameParsingService:
246
262
  given_tokens,
247
263
  tokens,
248
264
  normalized_cache,
249
- False,
250
- original_compound_format,
265
+ is_all_chinese=False,
266
+ original_compound_format=original_compound_format,
251
267
  score_cache=score_cache,
252
268
  )
253
269
 
@@ -351,8 +367,8 @@ class NameParsingService:
351
367
  given_tokens,
352
368
  tokens,
353
369
  normalized_cache,
354
- False,
355
- original_compound_format,
370
+ is_all_chinese=False,
371
+ original_compound_format=original_compound_format,
356
372
  score_cache=score_cache,
357
373
  )
358
374
 
@@ -536,9 +552,11 @@ class NameParsingService:
536
552
  given_tokens: list[str],
537
553
  tokens: list[str],
538
554
  normalized_cache: dict[str, str],
555
+ *,
539
556
  is_all_chinese: bool = False,
540
557
  original_compound_format: str | None = None,
541
558
  score_cache: dict[str, dict] | None = None,
559
+ allow_guarded_given_first_bonus: bool = True,
542
560
  ) -> float:
543
561
  """Calculate unified score for a parse candidate."""
544
562
  if not given_tokens:
@@ -617,6 +635,12 @@ class NameParsingService:
617
635
 
618
636
  if is_ambiguous:
619
637
  order_preservation_bonus = 1.0 # Strong bonus for preserving original order in ambiguous cases
638
+ elif allow_guarded_given_first_bonus and self._has_guarded_given_first_surname_ratio(
639
+ surname_tokens[0],
640
+ given_tokens[0],
641
+ normalized_cache,
642
+ ):
643
+ order_preservation_bonus = GIVEN_FIRST_ORDER_PRESERVATION_BONUS
620
644
  if not is_all_chinese and len(tokens) == 3 and len(surname_tokens) == 1 and len(given_tokens) == 2:
621
645
  # Narrow extension for hard 3-token given-first cases:
622
646
  # only apply when surname-last is materially more plausible than surname-first.
@@ -724,6 +748,39 @@ class NameParsingService:
724
748
  freq_ratio = max(surname_freq, given_freq) / min(surname_freq, given_freq)
725
749
  return freq_ratio < 5.0 # Ambiguous if frequencies are within 5x of each other
726
750
 
751
+ def _has_guarded_given_first_surname_ratio(
752
+ self,
753
+ surname_token: str,
754
+ given_token: str,
755
+ normalized_cache: dict[str, str],
756
+ ) -> bool:
757
+ """Return whether surname frequencies strongly support given-first order."""
758
+ if "-" in given_token or self._is_compound_surname_token(given_token, normalized_cache):
759
+ return False
760
+
761
+ surname_key = self._surname_key([surname_token], normalized_cache)
762
+ given_as_surname_key = self._surname_key([given_token], normalized_cache)
763
+
764
+ if not (
765
+ self._data.is_surname(surname_token, surname_key)
766
+ and self._data.is_surname(given_token, given_as_surname_key)
767
+ ):
768
+ return False
769
+
770
+ given_surname_freq = self._data.get_surname_freq(given_as_surname_key)
771
+ surname_freq = self._data.get_surname_freq(surname_key)
772
+ if not (0 < given_surname_freq < LOW_FREQUENCY_SURNAME_MAX):
773
+ return False
774
+
775
+ return surname_freq / given_surname_freq > GIVEN_FIRST_SURNAME_FREQ_RATIO_MIN
776
+
777
+ def _is_compound_surname_token(self, token: str, _normalized_cache: dict[str, str]) -> bool:
778
+ """Return whether a single token is a known compact/camelCase compound surname."""
779
+ token_key = token.strip().lower()
780
+ return token_key in COMPOUND_VARIANTS or (
781
+ "-" in token_key and token_key in self._data.compound_hyphen_map
782
+ )
783
+
727
784
  def _surname_key(self, surname_tokens: list[str], normalized_cache: dict[str, str]) -> str:
728
785
  """Convert surname tokens to lookup key, preferring Chinese characters when available."""
729
786
  if len(surname_tokens) == 1:
File without changes
@@ -0,0 +1,12 @@
1
+ config_version: 0.0.1
2
+ model_variants:
3
+ sinonym_v1:
4
+ instance: sinonym.timo.interface.Instance
5
+ prediction: sinonym.timo.interface.Prediction
6
+ predictor: sinonym.timo.interface.Predictor
7
+ predictor_config: sinonym.timo.interface.PredictorConfig
8
+ artifacts_s3_path: null
9
+ python_version: "3.10"
10
+ cuda: False
11
+ integration_test: sinonym.timo.integration_test.TestIntegration
12
+ docker_run_commands: []
@@ -0,0 +1,70 @@
1
+ import unittest
2
+
3
+ from sinonym.timo.interface import Instance, Prediction, Predictor, PredictorConfig
4
+
5
+
6
+ class TestIntegration(unittest.TestCase):
7
+ @classmethod
8
+ def setUpClass(cls):
9
+ cls.predictor = Predictor(config=PredictorConfig(), artifacts_dir=".")
10
+
11
+ def test_chinese_name(self):
12
+ results = self.predictor.predict_batch([Instance(name="Li Wei")])
13
+ self.assertEqual(len(results), 1)
14
+ self.assertIsInstance(results[0], Prediction)
15
+ self.assertTrue(results[0].success)
16
+ self.assertIsNotNone(results[0].given_name)
17
+ self.assertIsNotNone(results[0].surname)
18
+
19
+ def test_non_chinese_name(self):
20
+ results = self.predictor.predict_batch([Instance(name="John Smith")])
21
+ self.assertEqual(len(results), 1)
22
+ self.assertFalse(results[0].success)
23
+ self.assertIsNotNone(results[0].error_message)
24
+
25
+ def test_batch_superset_output(self):
26
+ instances = [Instance(name="Li Wei"), Instance(name="Wang Weiming")]
27
+ results = self.predictor.predict_batch(instances)
28
+ self.assertEqual(len(results), 2)
29
+ for r in results:
30
+ self.assertIsInstance(r, Prediction)
31
+ self.assertTrue(r.success)
32
+ # superset: every Prediction carries confidence + format_pattern
33
+ self.assertIsNotNone(r.confidence)
34
+ self.assertIsNotNone(r.format_pattern)
35
+ # shared batch pattern replicated onto each row
36
+ self.assertEqual(
37
+ results[0].format_pattern.dominant_format,
38
+ results[1].format_pattern.dominant_format,
39
+ )
40
+
41
+ def test_predict_batch_empty(self):
42
+ self.assertEqual(self.predictor.predict_batch([]), [])
43
+
44
+ def test_score_name_batch(self):
45
+ names = ["Li Wei", "Wang Weiming"]
46
+ summary = self.predictor.score_name_batch(names)
47
+ self.assertEqual(summary.names, names)
48
+ self.assertEqual(len(summary.results), 2)
49
+ self.assertEqual(len(summary.confidences), 2)
50
+ self.assertIsNotNone(summary.format_pattern.dominant_format)
51
+
52
+ def test_score_name_batch_tuned_threshold(self):
53
+ names = ["Li Wei", "Wang Weiming"]
54
+ summary = self.predictor.score_name_batch(names, format_threshold=0.9)
55
+ self.assertEqual(len(summary.results), 2)
56
+ self.assertEqual(
57
+ summary.format_pattern.threshold_met,
58
+ summary.format_pattern.confidence >= 0.9,
59
+ )
60
+
61
+ def test_detect_batch_format(self):
62
+ pattern = self.predictor.detect_batch_format(["Zhang Wei", "Li Ming", "Wang Xiaoli"])
63
+ self.assertIn(
64
+ pattern.dominant_format, {"surname_first", "given_first", "mixed"}
65
+ )
66
+
67
+ def test_analyze_name_batch_full(self):
68
+ result = self.predictor.analyze_name_batch(["Li Wei", "Wang Weiming"])
69
+ self.assertEqual(len(result.results), 2)
70
+ self.assertEqual(len(result.individual_analyses), 2)