sinonym 0.2.8__tar.gz → 0.3.0__tar.gz

{sinonym-0.2.8 → sinonym-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sinonym
-Version: 0.2.8
+Version: 0.3.0
 Summary: Chinese Name Detection and Normalization Module
 Project-URL: Homepage, https://github.com/allenai/sinonym
 Project-URL: Repository, https://github.com/allenai/sinonym
@@ -24,7 +24,7 @@ Requires-Dist: numpy>=1.24
 Requires-Dist: pydantic<2,>=1.10
 Requires-Dist: pypinyin>=0.44.0
 Requires-Dist: requests>=2.31
-Requires-Dist: scikit-learn<2,>=1.2
+Requires-Dist: scikit-learn==1.7.1
 Requires-Dist: scipy>=1.10
 Requires-Dist: skops>=0.9
 Provides-Extra: dev
@@ -411,15 +411,25 @@ def main():
     detector = ChineseNameDetector()
     names_a = ["Li Wei", "Wang Weiming", "Zhang Ming"]
     names_b = ["Xin Liu", "Yang Li", "Chen Huang"]
+    author_batches = [
+        ["Wang An", "Yan Li", "Wu Gang", "Li Bao"],
+        ["Li Wei", "Wang Weiming", "Zhang Ming"],
+    ]
+
+    # High-level auto wrappers choose local vs process-pool execution.
+    # Each inner author list remains an independent batch-context boundary.
+    normalized = detector.normalize_names(names_a, parallel="auto")
+    batch_results = detector.process_name_batches(author_batches, parallel="auto")
 
     # Reuse workers across many calls
     with detector.create_persistent_multiprocess_pool(max_workers=6, chunk_size=64) as pool:
         results_a = pool.normalize_names(names_a)
         results_b = pool.normalize_names(names_b)
+        persistent_batch_results = pool.process_name_batches(author_batches)
 
-    # One-off compatibility wrapper with full batch-context semantics
+    # One-off temporary pool with process_name_batch batch-context semantics
     single_batch = detector.process_name_batch_multiprocess(names_a, max_workers=6, chunk_size=64)
-    return results_a, results_b, single_batch
+    return normalized, batch_results, results_a, results_b, persistent_batch_results, single_batch
 
 if __name__ == "__main__":
     main()
@@ -428,6 +438,19 @@ if __name__ == "__main__":
 Use the `if __name__ == "__main__":` guard in scripts to ensure safe process
 spawning on Windows and macOS.
 
+`normalize_names()` has per-name `normalize_name()` semantics. `process_name_batches()`
+is the high-level wrapper for many independent author lists: each inner list
+gets normal batch-format correction, and `parallel="auto"` uses conservative
+platform-aware thresholds to avoid one-off process-startup regressions.
+
+For repeated high-throughput calls, keep a persistent pool open. The persistent
+pool exposes `normalize_names()` for independent names and `process_name_batches()`
+for independent author-list batches. The one-off
+`process_name_batch_multiprocess()` method is kept for compatibility and has
+`process_name_batch()` batch-context semantics. For per-name multiprocessing,
+use `normalize_names(..., parallel="always")` or a persistent pool's
+`normalize_names()` method.
+
 ### When to Use Batch Processing
 
 *   **Academic Papers**: Author lists typically follow consistent formatting

{sinonym-0.2.8 → sinonym-0.3.0}/README.md

@@ -373,15 +373,25 @@ def main():
     detector = ChineseNameDetector()
     names_a = ["Li Wei", "Wang Weiming", "Zhang Ming"]
     names_b = ["Xin Liu", "Yang Li", "Chen Huang"]
+    author_batches = [
+        ["Wang An", "Yan Li", "Wu Gang", "Li Bao"],
+        ["Li Wei", "Wang Weiming", "Zhang Ming"],
+    ]
+
+    # High-level auto wrappers choose local vs process-pool execution.
+    # Each inner author list remains an independent batch-context boundary.
+    normalized = detector.normalize_names(names_a, parallel="auto")
+    batch_results = detector.process_name_batches(author_batches, parallel="auto")
 
     # Reuse workers across many calls
     with detector.create_persistent_multiprocess_pool(max_workers=6, chunk_size=64) as pool:
         results_a = pool.normalize_names(names_a)
         results_b = pool.normalize_names(names_b)
+        persistent_batch_results = pool.process_name_batches(author_batches)
 
-    # One-off compatibility wrapper with full batch-context semantics
+    # One-off temporary pool with process_name_batch batch-context semantics
     single_batch = detector.process_name_batch_multiprocess(names_a, max_workers=6, chunk_size=64)
-    return results_a, results_b, single_batch
+    return normalized, batch_results, results_a, results_b, persistent_batch_results, single_batch
 
 if __name__ == "__main__":
     main()
@@ -390,6 +400,19 @@ if __name__ == "__main__":
 Use the `if __name__ == "__main__":` guard in scripts to ensure safe process
 spawning on Windows and macOS.
 
+`normalize_names()` has per-name `normalize_name()` semantics. `process_name_batches()`
+is the high-level wrapper for many independent author lists: each inner list
+gets normal batch-format correction, and `parallel="auto"` uses conservative
+platform-aware thresholds to avoid one-off process-startup regressions.
+
+For repeated high-throughput calls, keep a persistent pool open. The persistent
+pool exposes `normalize_names()` for independent names and `process_name_batches()`
+for independent author-list batches. The one-off
+`process_name_batch_multiprocess()` method is kept for compatibility and has
+`process_name_batch()` batch-context semantics. For per-name multiprocessing,
+use `normalize_names(..., parallel="always")` or a persistent pool's
+`normalize_names()` method.
+
 ### When to Use Batch Processing
 
 *   **Academic Papers**: Author lists typically follow consistent formatting

{sinonym-0.2.8 → sinonym-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sinonym"
-version = "0.2.8"
+version = "0.3.0"
 description = "Chinese Name Detection and Normalization Module"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -30,7 +30,7 @@ dependencies = [
     "pydantic>=1.10,<2",
     "pypinyin>=0.44.0",
     "requests>=2.31",
-    "scikit-learn>=1.2,<2",
+    "scikit-learn==1.7.1",
     "scipy>=1.10",
     "skops>=0.9",
 ]
@@ -61,8 +61,13 @@ include = [
 include = [
     "sinonym",
     "tests",
+    "scripts/README.md",
     "scripts/check_test_status.py",
     "scripts/name_order_routing_rules.py",
+    "scripts/verify_multiprocess.py",
+    "scripts/generate_surname_romanizations.py",
+    "scripts/generate_name_statistics.py",
+    "scripts/train_ml_classifier_for_chinese_vs_japanese.py",
     "README.md",
     "pyproject.toml",
 ]
@@ -82,7 +87,32 @@ select = ["E", "F", "W", "C90", "I", "N", "UP", "YTT", "S", "BLE", "FBT", "B", "
 ignore = ["S101", "T201"]  # Allow assert statements and print statements
 
 [tool.ruff.lint.per-file-ignores]
-"tests/*" = ["S101", "T201", "ARG", "FBT", "INP001"]
+"tests/*" = ["S101", "T201", "ARG", "FBT", "INP001", "PLR2004"]
+"sinonym/detector.py" = ["C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", "SLF001"]
+"sinonym/services/ethnicity.py" = ["C901", "PLR0911", "PLR0912", "PLR0915", "PLR2004", "ARG002", "SLF001"]
+"scripts/generate_acl_data.py" = ["B007", "BLE001", "C901", "F841", "PLC0415", "PLR0912", "PLR0915", "PLR2004", "PLW2901", "PTH123", "SIM102", "SIM108", "SLF001"]
+"scripts/generate_chinese_name_corpus_data.py" = ["ARG001", "BLE001", "C901", "ERA001", "N806", "PLC0415", "PLR0912", "PLR2004", "PTH123", "SLF001", "TRY300"]
+"scripts/profile_multiprocess.py" = ["EM101", "PLR0913", "TRY003"]
+"scripts/profile_run.py" = ["FBT002", "PLR2004"]
+"scripts/profile_threaded.py" = ["E501", "PLR2004", "SIM108"]
+"scripts/train_ml_classifier_for_chinese_vs_japanese.py" = ["N806", "PLC0415"]
+"sinonym/__init__.py" = ["BLE001", "EM102", "PLC0415", "PTH100", "PTH109", "PTH120", "S110", "TRY003"]
+"sinonym/chinese_names_data.py" = ["ERA001"]
+"sinonym/coretypes/config.py" = ["E501", "RUF001", "RUF003"]
+"sinonym/ml_model_components.py" = ["ARG002", "N803", "PLR2004", "UP038"]
+"sinonym/patterns/regex_builders.py" = ["N806", "PLR2004", "RUF001"]
+"sinonym/resources.py" = ["PLC0415", "TC003"]
+"sinonym/services/cache.py" = ["BLE001", "PERF203", "PLC0415", "RUF002", "S112"]
+"sinonym/services/formatting.py" = ["C901", "E741", "EM101", "FIX002", "PLR0911", "PLR0912", "PLR0915", "RUF003", "SIM114", "TD002", "TD003", "TRY003"]
+"sinonym/services/initialization.py" = ["C901", "PLR0911", "PLR0912", "PLR2004", "RUF003", "SIM102", "TC003"]
+"sinonym/services/normalization.py" = ["ARG002", "C901", "FBT001", "FBT002", "FBT003", "PLR0911", "PLR0912", "PLR0915", "PLR2004", "SLF001"]
+"sinonym/services/parsing.py" = ["C901", "COM812", "PLR0912", "PLR0913", "PLR0915", "PLR2004", "RET504", "SIM102", "SIM114"]
+"sinonym/text_processing/compound_detector.py" = ["E501", "PLC0415", "SIM102"]
+"sinonym/text_processing/text_normalizer.py" = ["B019", "RUF001", "S105"]
+"sinonym/text_processing/text_preprocessor.py" = ["ARG002", "PLC0415", "PLR2004", "SLF001"]
+"sinonym/timo/integration_test.py" = ["PT009", "RUF005"]
+"sinonym/utils/string_manipulation.py" = ["C901", "ERA001", "F841", "PLC0415", "PLR0911", "PLR0912", "PLR0915", "PLR2004", "RUF013", "SIM102", "SIM108", "W293"]
+"tests/test_performance.py" = ["C414", "S311", "SIM108", "SLF001"]
 "tests/test_mixed_scripts.py" = ["RUF001"]
 "tests/test_non_chinese_rejection.py" = ["RUF001"]
 

{sinonym-0.2.8 → sinonym-0.3.0}/scripts/README.md

@@ -5,7 +5,7 @@ Utility scripts for benchmarking, profiling, testing, and model training.
 ## Active Scripts
 
 ### `check_test_status.py`
-Runs the full test suite and reports individual test case failures with detailed diagnostics. Runs performance tests separately, then exits 0/1 based on whether the failure count and failure signatures match the expected baseline configured in `scripts/check_test_status.py`. Improvements (fewer failures) also pass when the remaining failures are all from the known baseline; regressions or unexpected failure signatures fail.
+Runs the full test suite and reports individual test case failures with detailed diagnostics. Runs performance tests separately, then exits 0/1 based on whether the failure count and failure signatures exactly match the expected baseline configured in `scripts/check_test_status.py`. Regressions, unexpected failure signatures, or missing expected baseline signatures fail until the configured baseline is intentionally updated.
 
 ```bash
 uv run python scripts/check_test_status.py
@@ -47,6 +47,16 @@ Persistent multi-process throughput and parity check. Compares single-process th
 uv run python scripts/profile_multiprocess.py --names 12000 --warmup 3000 --runs 3 --workers 6 --chunk-size 64
 ```
 
+### `verify_multiprocess.py`
+Cross-platform multiprocessing verifier. Checks correctness parity for local,
+one-shot, auto-wrapper, and persistent-pool paths, then reports throughput for
+flat independent names and independent author-list batches.
+
+```bash
+uv run python scripts/verify_multiprocess.py --json-output scratch/mp_verify_windows.json
+UV_PROJECT_ENVIRONMENT=/tmp/sinonym-wsl-venv uv run python scripts/verify_multiprocess.py --json-output scratch/mp_verify_wsl.json
+```
+
 ### `train_ml_classifier_for_chinese_vs_japanese.py`
 Trains the Chinese-vs-Japanese name classifier used in production. Downloads Chinese (~1.2M) and Japanese (~180K) name corpora, trains a scikit-learn pipeline (TF-IDF character n-grams + 20 linguistic heuristic features + logistic regression), and saves the model to `data/chinese_japanese_classifier.skops`.
 

{sinonym-0.2.8 → sinonym-0.3.0}/scripts/check_test_status.py

@@ -30,36 +30,12 @@ EXPECTED_NORMALIZED_NAME_FAILURES = (
         "Hao Fei",
         "Fei Hao",
     ),
-    (
-        "tests.test_acl::test_acl_order_preservation[Kun Kuang-Kun Kuang]",
-        "Kun Kuang",
-        "Kun Kuang",
-        "Kuang Kun",
-    ),
     (
         "tests.test_acl::test_acl_order_preservation[Yao Shu-Yao Shu]",
         "Yao Shu",
         "Yao Shu",
         "Shu Yao",
     ),
-    (
-        "tests.test_basic_chinese_names::test_basic_chinese_names[Feng Cha-expected4]",
-        "Feng Cha",
-        "Cha Feng",
-        "Feng Cha",
-    ),
-    (
-        "tests.test_basic_chinese_names::test_basic_chinese_names[He Cha-expected7]",
-        "He Cha",
-        "Cha He",
-        "He Cha",
-    ),
-    (
-        "tests.test_basic_chinese_names::test_basic_chinese_names[Hu Cha-expected9]",
-        "Hu Cha",
-        "Cha Hu",
-        "Hu Cha",
-    ),
     (
         "tests.test_basic_chinese_names::test_basic_chinese_names[Li Gong-expected13]",
         "Li Gong",
@@ -72,12 +48,6 @@ EXPECTED_NORMALIZED_NAME_FAILURES = (
         "Wei Gao",
         "Gao Wei",
     ),
-    (
-        "tests.test_basic_chinese_names::test_basic_chinese_names[Kong Kung-expected63]",
-        "Kong Kung",
-        "Kung Kong",
-        "Kong Kung",
-    ),
     (
         "tests.test_basic_chinese_names::test_basic_chinese_names[Lu Xun-expected75]",
         "Lu Xun",
@@ -85,23 +55,11 @@ EXPECTED_NORMALIZED_NAME_FAILURES = (
         "Lu Xun",
     ),
     (
-        "tests.test_basic_chinese_names::test_basic_chinese_names[Qin Shi-expected80]",
-        "Qin Shi",
-        "Shi Qin",
-        "Qin Shi",
-    ),
-    (
-        "tests.test_basic_chinese_names::test_basic_chinese_names[Zhou Xun-expected110]",
+        "tests.test_basic_chinese_names::test_basic_chinese_names[Zhou Xun-expected109]",
         "Zhou Xun",
         "Xun Zhou",
         "Zhou Xun",
     ),
-    (
-        "tests.test_compound_names::test_compound_names[Leung Ka Fai-expected7]",
-        "Leung Ka Fai",
-        "Ka-Fai Leung",
-        "Leung-Ka Fai",
-    ),
     (
         "tests.test_misc::test_misc_chinese_names[Jin Hua-expected13]",
         "Jin Hua",
@@ -120,18 +78,6 @@ EXPECTED_NORMALIZED_NAME_FAILURES = (
         "Miao Yu",
         "Yu Miao",
     ),
-    (
-        "tests.test_misc::test_misc_chinese_names[Wen Jing-expected33]",
-        "Wen Jing",
-        "Jing Wen",
-        "Wen Jing",
-    ),
-    (
-        "tests.test_misc::test_misc_chinese_names[Jing Wen-expected34]",
-        "Jing Wen",
-        "Jing Wen",
-        "Wen Jing",
-    ),
     (
         "tests.test_mixed_production_cases::test_mixed_cases[Gui Rui-expected_result6]",
         "Gui Rui",
@@ -144,24 +90,12 @@ EXPECTED_NORMALIZED_NAME_FAILURES = (
         "Yao Shu",
         "Shu Yao",
     ),
-    (
-        "tests.test_mixed_production_cases::test_mixed_cases[Huang Yu Chang-expected_result19]",
-        "Huang Yu Chang",
-        "Yu-Chang Huang",
-        "Huang-Yu Chang",
-    ),
     (
         "tests.test_mixed_production_cases::test_mixed_cases[Jia Jian Feng-expected_result24]",
         "Jia Jian Feng",
         "Jian-Feng Jia",
         "Jia-Jian Feng",
     ),
-    (
-        "tests.test_mixed_production_cases::test_mixed_cases[Fan Jia Liang-expected_result41]",
-        "Fan Jia Liang",
-        "Jia-Liang Fan",
-        "Fan-Jia Liang",
-    ),
     (
         "tests.test_mixed_production_cases::test_mixed_cases[Wei Wen Xing-expected_result48]",
         "Wei Wen Xing",
@@ -174,18 +108,6 @@ EXPECTED_NORMALIZED_NAME_FAILURES = (
         "Zhao Xi",
         "Xi Zhao",
     ),
-    (
-        "tests.test_mixed_production_cases::test_mixed_cases[xu feng-expected_result120]",
-        "xu feng",
-        "Feng Xu",
-        "Xu Feng",
-    ),
-    (
-        "tests.test_mixed_production_cases::test_mixed_cases[yang guang-expected_result121]",
-        "yang guang",
-        "Guang Yang",
-        "Yang Guang",
-    ),
 )
 EXPECTED_FAILURES = len(EXPECTED_NORMALIZED_NAME_FAILURES)
 EXPECTED_FAILURE_SIGNATURES = tuple(
@@ -379,6 +301,20 @@ def unexpected_failure_signatures(failures: Iterable[FailureDetail]) -> list[Fai
     return sorted(unexpected, key=_failure_signature)
 
 
+def missing_expected_failure_signatures(failures: Iterable[FailureDetail]) -> list[FailureDetail]:
+    """Return known baseline failures that were not present in the current failure list."""
+    observed_counts = Counter(_failure_signature(failure) for failure in failures)
+    missing = []
+    for nodeid, kind, message in EXPECTED_FAILURE_SIGNATURES:
+        signature = (nodeid, kind, message)
+        if observed_counts[signature] > 0:
+            observed_counts[signature] -= 1
+        else:
+            missing.append(FailureDetail(nodeid, kind, message))
+
+    return sorted(missing, key=_failure_signature)
+
+
 def _unexpected_failure_sample(failures: list[FailureDetail]) -> str:
     """Return a concise sample of unexpected failure signatures for the status line."""
     samples = [f"{failure.nodeid} ({failure.kind}: {failure.message})" for failure in failures[:UNEXPECTED_FAILURE_SAMPLE_SIZE]]
@@ -390,6 +326,32 @@ def _unexpected_failure_sample(failures: list[FailureDetail]) -> str:
     return f"{', '.join(samples)}{suffix}"
 
 
+def _execution_status_error(
+    total_failures: int,
+    *,
+    pytest_returncode: int | None,
+    junit_error: str | None,
+) -> str | None:
+    """Return a failure message if the run cannot be scored against the baseline.
+
+    These conditions describe a run that never produced a trustworthy JUnit
+    failure list (crash, non-baseline exit code, or a returncode/total mismatch),
+    so signature-level comparison would be meaningless. Returns None when the run
+    is well-formed enough to be scored.
+    """
+    if pytest_returncode is None:
+        return "Test execution failed before pytest returned a status."
+    if junit_error is not None:
+        return junit_error
+    if pytest_returncode not in PYTEST_BASELINE_RETURN_CODES:
+        return f"Pytest execution failed with exit code {pytest_returncode}."
+    if pytest_returncode == 1 and total_failures == 0:
+        return "Pytest failed without JUnit-reported failures."
+    if pytest_returncode == 0 and total_failures > 0:
+        return "JUnit reported failures even though pytest exited successfully."
+    return None
+
+
 def status_exit_decision(
     total_failures: int,
     *,
@@ -398,40 +360,47 @@ def status_exit_decision(
     junit_error: str | None = None,
     failures: Iterable[FailureDetail] | None = None,
 ) -> tuple[int, str]:
-    """Return process exit code and final status message."""
-    unexpected_signatures = unexpected_failure_signatures(failures or ())
-    if pytest_returncode is None:
-        exit_code = 1
-        status_message = "Test execution failed before pytest returned a status."
-    elif junit_error is not None:
-        exit_code = 1
-        status_message = junit_error
-    elif pytest_returncode not in PYTEST_BASELINE_RETURN_CODES:
-        exit_code = 1
-        status_message = f"Pytest execution failed with exit code {pytest_returncode}."
-    elif pytest_returncode == 1 and total_failures == 0:
-        exit_code = 1
-        status_message = "Pytest failed without JUnit-reported failures."
-    elif pytest_returncode == 0 and total_failures > 0:
-        exit_code = 1
-        status_message = "JUnit reported failures even though pytest exited successfully."
+    """Return process exit code and final status message.
+
+    The exit-0 (baseline / improvement) paths are only reached once the concrete
+    failure list has been validated against ``total_failures`` and cleared of any
+    signature outside the known baseline. A missing or length-inconsistent list is
+    treated as caller misuse and fails loudly rather than being guessed into a
+    false pass.
+    """
+    execution_error = _execution_status_error(
+        total_failures,
+        pytest_returncode=pytest_returncode,
+        junit_error=junit_error,
+    )
+    # Past the execution guard signatures must be checked, so the concrete list is
+    # required and must agree with the reported total before it can be scored.
+    failures_list = None if failures is None else list(failures)
+    length_ok = failures_list is not None and len(failures_list) == total_failures
+    unexpected_signatures = unexpected_failure_signatures(failures_list) if length_ok else []
+    missing_signatures = missing_expected_failure_signatures(failures_list) if length_ok else []
+
+    if execution_error is not None:
+        exit_code, status_message = 1, execution_error
+    elif failures_list is None:
+        exit_code, status_message = 1, "Cannot verify failures against the baseline: no failure list was provided."
+    elif len(failures_list) != total_failures:
+        exit_code, status_message = (
+            1,
+            (f"Failure count mismatch: received {len(failures_list)} JUnit failures but total_failures={total_failures}."),
+        )
+    elif total_failures > EXPECTED_FAILURES:
+        exit_code, status_message = 1, f"REGRESSION! Too many failures ({total_failures} > {EXPECTED_FAILURES})"
     elif unexpected_signatures:
-        exit_code = 1
-        status_message = f"Unexpected pytest failure signatures: {_unexpected_failure_sample(unexpected_signatures)}"
+        sample = _unexpected_failure_sample(unexpected_signatures)
+        exit_code, status_message = 1, f"Unexpected pytest failure signatures: {sample}"
+    elif missing_signatures:
+        sample = _unexpected_failure_sample(missing_signatures)
+        exit_code, status_message = 1, f"Missing expected pytest failure signatures: {sample}"
     elif not perf_passed:
-        exit_code = 1
-        status_message = "Performance tests failed!"
-    elif total_failures > EXPECTED_FAILURES:
-        exit_code = 1
-        status_message = f"REGRESSION! Too many failures ({total_failures} > EXPECTED_FAILURES)"
-    elif total_failures == EXPECTED_FAILURES:
-        exit_code = 0
-        status_message = "Tests are at expected baseline (EXPECTED_FAILURES failures, performance OK)"
+        exit_code, status_message = 1, "Performance tests failed!"
     else:
-        exit_code = 0
-        status_message = (
-            f"IMPROVEMENT! Tests are better than baseline ({total_failures} < EXPECTED_FAILURES failures, performance OK)"
-        )
+        exit_code, status_message = 0, f"Tests are at expected baseline ({EXPECTED_FAILURES} failures, performance OK)"
 
     return exit_code, status_message
 
@@ -503,7 +472,7 @@ def main() -> None:
         junit_error=junit_error,
         failures=failures,
     )
-    print(f"\n{status_message}")
+    print(f"\n{_safe_for_console(status_message)}")
     sys.exit(exit_code)