py-pinyin-split 2.0.0__tar.gz → 3.0.2__tar.gz

{py_pinyin_split-2.0.0 → py_pinyin_split-3.0.2}/.gitignore

@@ -20,6 +20,7 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 cover/
+.benchmarks/
 
 # Virtual environments
 .venv

{py_pinyin_split-2.0.0 → py_pinyin_split-3.0.2}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.3
 Name: py-pinyin-split
-Version: 2.0.0
+Version: 3.0.2
 Summary: Library for splitting Hanyu Pinyin phrases into all valid syllable combinations
-Project-URL: Documentation, https://github.com/lstrobel/pinyinsplit#readme
-Project-URL: Issues, https://github.com/lstrobel/pinyinsplit/issues
-Project-URL: Source, https://github.com/lstrobel/pinyinsplit
+Project-URL: Documentation, https://github.com/lstrobel/py-pinyin-split#readme
+Project-URL: Issues, https://github.com/lstrobel/py-pinyin-split/issues
+Project-URL: Source, https://github.com/lstrobel/py-pinyin-split
 Author-email: lstrobel <mail@lstrobel.com>, Thomas Lee <thomaslee@throput.com>
 Maintainer-email: lstrobel <mail@lstrobel.com>
 License: MIT
@@ -24,8 +24,8 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Topic :: Text Processing :: Linguistic
 Classifier: Topic :: Utilities
 Requires-Python: <3.13,>=3.8
+Requires-Dist: marisa-trie>=1.2.1
 Requires-Dist: nltk==3.9.1
-Requires-Dist: pygtrie>=2.5.0
 Description-Content-Type: text/markdown
 
 # py-pinyin-split
@@ -47,16 +47,12 @@ pip install py-pinyin-split
 
 Instantiate a tokenizer and split away.
 
- The tokenizer expects a clean Hanyu Pinyin word as input - you'll need to preprocess text to:
-- Remove punctuation and whitespace
-- Convert numeric tones (pin1yin1) to tone marks (pīnyīn)
-- Handle sentence boundaries
-
-The tokenizer uses syllable frequency data to resolve ambiguous splits. It currently does not support apostrophes (sorry!) (e.g. "xi'an" will throw an error)
+The tokenizer can handle standard Hanyu Pinyin with whitespaces and punctuation. However, invalid pinyin syllables will raise a `ValueError`
 
+The tokenizer uses syllable frequency data to resolve ambiguous splits.
 
 ```python
-from pinyin_split import PinyinTokenizer
+from py_pinyin_split import PinyinTokenizer
 
 tokenizer = PinyinTokenizer()
 
@@ -64,13 +60,21 @@ tokenizer = PinyinTokenizer()
 tokenizer.tokenize("nǐhǎo")  # ['nǐ', 'hǎo']
 tokenizer.tokenize("Běijīng")  # ['Běi', 'jīng']
 
+# Handles whitespace and punctuation
+tokenizer.tokenize("Nǐ hǎo ma?")  # ['Nǐ', 'hǎo', 'ma', '?']
+tokenizer.tokenize("Wǒ hěn hǎo!")  # ['Wǒ', 'hěn', 'hǎo', '!']
+
 # Handles ambiguous splits using frequency data
 tokenizer.tokenize("xian")  # ['xian'] not ['xi', 'an']
 tokenizer.tokenize("wanan")  # ['wan', 'an'] not ['wa', 'nan']
 
-# Tone marks help resolve ambiguity
+# Tone marks or punctuation help resolve ambiguity
 tokenizer.tokenize("xīān")  # ['xī', 'ān']
 tokenizer.tokenize("xián")  # ['xián']
+tokenizer.tokenize("Xī'ān") # ["Xī", "'", "ān"]
+
+# Raises ValueError for invalid pinyin
+tokenizer.tokenize("hello")  # ValueError
 
 # Optional support for non-standard syllables
 tokenizer = PinyinTokenizer(include_nonstandard=True)
@@ -80,4 +84,4 @@ tokenizer.tokenize("duang")  # ['duang']
 ## Related Projects
 - https://pypi.org/project/pinyintokenizer/
 - https://pypi.org/project/pypinyin/
-- https://github.com/throput/pinyinsplit
+- https://github.com/throput/pinyinsplit

{py_pinyin_split-2.0.0 → py_pinyin_split-3.0.2}/README.md

@@ -17,16 +17,12 @@ pip install py-pinyin-split
 
 Instantiate a tokenizer and split away.
 
- The tokenizer expects a clean Hanyu Pinyin word as input - you'll need to preprocess text to:
-- Remove punctuation and whitespace
-- Convert numeric tones (pin1yin1) to tone marks (pīnyīn)
-- Handle sentence boundaries
-
-The tokenizer uses syllable frequency data to resolve ambiguous splits. It currently does not support apostrophes (sorry!) (e.g. "xi'an" will throw an error)
+The tokenizer can handle standard Hanyu Pinyin with whitespaces and punctuation. However, invalid pinyin syllables will raise a `ValueError`
 
+The tokenizer uses syllable frequency data to resolve ambiguous splits.
 
 ```python
-from pinyin_split import PinyinTokenizer
+from py_pinyin_split import PinyinTokenizer
 
 tokenizer = PinyinTokenizer()
 
@@ -34,13 +30,21 @@ tokenizer = PinyinTokenizer()
 tokenizer.tokenize("nǐhǎo")  # ['nǐ', 'hǎo']
 tokenizer.tokenize("Běijīng")  # ['Běi', 'jīng']
 
+# Handles whitespace and punctuation
+tokenizer.tokenize("Nǐ hǎo ma?")  # ['Nǐ', 'hǎo', 'ma', '?']
+tokenizer.tokenize("Wǒ hěn hǎo!")  # ['Wǒ', 'hěn', 'hǎo', '!']
+
 # Handles ambiguous splits using frequency data
 tokenizer.tokenize("xian")  # ['xian'] not ['xi', 'an']
 tokenizer.tokenize("wanan")  # ['wan', 'an'] not ['wa', 'nan']
 
-# Tone marks help resolve ambiguity
+# Tone marks or punctuation help resolve ambiguity
 tokenizer.tokenize("xīān")  # ['xī', 'ān']
 tokenizer.tokenize("xián")  # ['xián']
+tokenizer.tokenize("Xī'ān") # ["Xī", "'", "ān"]
+
+# Raises ValueError for invalid pinyin
+tokenizer.tokenize("hello")  # ValueError
 
 # Optional support for non-standard syllables
 tokenizer = PinyinTokenizer(include_nonstandard=True)
@@ -50,4 +54,4 @@ tokenizer.tokenize("duang")  # ['duang']
 ## Related Projects
 - https://pypi.org/project/pinyintokenizer/
 - https://pypi.org/project/pypinyin/
-- https://github.com/throput/pinyinsplit
+- https://github.com/throput/pinyinsplit

{py_pinyin_split-2.0.0 → py_pinyin_split-3.0.2}/pyproject.toml

@@ -2,8 +2,8 @@
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
-[tool.hatch.build.targets.wheel]
-packages = ["src/pinyin_split"]
+[tool.hatch.version]
+path = "src/py_pinyin_split/__about__.py"
 
 [project]
 name = "py-pinyin-split"
@@ -34,33 +34,37 @@ classifiers = [
     'Topic :: Text Processing :: Linguistic',
     'Topic :: Utilities',
 ]
-dependencies = ["nltk==3.9.1", "pygtrie>=2.5.0"]
+dependencies = ["marisa-trie>=1.2.1", "nltk==3.9.1"]
 
 [dependency-groups]
-dev = ["ruff>=0.7.4"]
+dev = [
+    "mypy>=1.13.0",
+    "pytest-benchmark>=4.0.0",
+    "pytest>=8.3.4",
+    "ruff>=0.7.4",
+]
 
 [project.urls]
-Documentation = "https://github.com/lstrobel/pinyinsplit#readme"
-Issues = "https://github.com/lstrobel/pinyinsplit/issues"
-Source = "https://github.com/lstrobel/pinyinsplit"
-
-[tool.hatch.version]
-path = "src/pinyin_split/__about__.py"
-
-[tool.hatch.envs.types]
-extra-dependencies = ["mypy>=1.0.0"]
-[tool.hatch.envs.types.scripts]
-check = "mypy --install-types --non-interactive {args:src/pinyin_split tests}"
+Documentation = "https://github.com/lstrobel/py-pinyin-split#readme"
+Issues = "https://github.com/lstrobel/py-pinyin-split/issues"
+Source = "https://github.com/lstrobel/py-pinyin-split"
 
-[tool.coverage.run]
-source_pkgs = ["pinyin_split", "tests"]
-branch = true
-parallel = true
-omit = ["src/pinyin_split/__about__.py"]
 
-[tool.coverage.paths]
-pinyin_split = ["src/pinyin_split", "*/pinyin-split/src/pinyin_split"]
-tests = ["tests", "*/pinyin-split/tests"]
+[tool.ruff.format]
+docstring-code-format = true
 
-[tool.coverage.report]
-exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
+[tool.ruff.lint]
+select = [
+    # pycodestyle
+    "E",
+    # Pyflakes
+    "F",
+    # pyupgrade
+    "UP",
+    # flake8-bugbear
+    "B",
+    # flake8-simplify
+    "SIM",
+    # isort
+    "I",
+]

{py_pinyin_split-2.0.0/src/pinyin_split → py_pinyin_split-3.0.2/src/py_pinyin_split}/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Lukas Strobel <mail@lstrobel.com>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "2.0.0"
+__version__ = "3.0.2"

py_pinyin_split-2.0.0/src/pinyin_split/pinyinsplit.py → py_pinyin_split-3.0.2/src/py_pinyin_split/__init__.py

@@ -1,17 +1,23 @@
+# SPDX-FileCopyrightText: 2024-present Lukas Strobel <mail@lstrobel.com>
+#
+# SPDX-License-Identifier: MIT
+
 import re
-import string
 from typing import Iterator, List, Tuple
-from nltk.tokenize.api import TokenizerI
-from pygtrie import CharTrie
+
+import marisa_trie  # type: ignore
+from nltk.tokenize import WordPunctTokenizer  # type: ignore
+from nltk.tokenize.api import TokenizerI  # type: ignore
 
 
 class PinyinTokenizer(TokenizerI):
     """
-    Splits Hanyu Pinyin words on syllable boundaries. Cannot handle punctuation or whitespace.
+    Splits Hanyu Pinyin words on syllable boundaries.
+    Cannot handle punctuation or whitespace.
 
     Args:
-        include_nonstandard: If True, includes rare/non-standard syllables in the valid set.
-            Defaults to False.
+        include_nonstandard: If True, includes rare/non-standard syllables in the valid
+            set. Defaults to False.
 
     Example:
         >>> tokenizer = PinyinTokenizer()
@@ -83,7 +89,7 @@ class PinyinTokenizer(TokenizerI):
         'chi', 'cha', 'che', 'chai', 'chao', 'chou', 'chan', 'chen', 'chang', 'cheng',
         'chu', 'chua', 'chuo', 'chuai', 'chui', 'chuan', 'chun', 'chuang', 'chong',
 
-        'shi', 'sha', 'she', 'shai', 'shei', 'shao', 'shou', 'shan', 'shen', 'shang', 'sheng',
+        'shi', 'sha', 'she', 'shai', 'shei', 'shao', 'shou', 'shan', 'shen', 'shang', 'sheng',  # noqa: E501
         'shu', 'shua', 'shuo', 'shuai', 'shui', 'shuan', 'shun', 'shuang',
 
         'r', # Include to cover erhua
@@ -533,21 +539,12 @@ class PinyinTokenizer(TokenizerI):
         "zuo": "893830",
     }
 
-    # Build allowed characters pattern
-    # TODO: Handle this more gracefully
-    _ALLOWED_CHARS = set(string.ascii_letters)
-    for vowel, variants in VOWEL_TONE_VARIANTS.items():
-        _ALLOWED_CHARS.update(vowel)  # Add lowercase
-        _ALLOWED_CHARS.update(vowel.upper())  # Add uppercase
-        for tone_char in variants:
-            _ALLOWED_CHARS.update(tone_char)
-            _ALLOWED_CHARS.update(tone_char.upper())
-
-    # Compile regex pattern for invalid character detection once at module level
-    VALID_CHARS_PATTERN = re.compile(f"[^{''.join(sorted(_ALLOWED_CHARS))}]")
+    NON_ALPHABETIC_REGEX = re.compile(r"[^\w\s]+")
 
     def _get_tone_variants(self, syllable: str):
-        """Generate all valid tone variants for a syllable. Assumes syllabe is lowercase"""
+        """
+        Generate all valid tone variants for a syllable. Assumes syllabe is lowercase
+        """
         variants = [syllable]  # Include toneless variation
 
         # Find the vowels in the syllable (both upper and lower case)
@@ -565,7 +562,8 @@ class PinyinTokenizer(TokenizerI):
         elif any(v == "o" for v in vowels):
             tone_vowel = next(v for v in vowels if v == "o")
         else:
-            # If there is no a e or o, the vowels are either 'iu', 'ui', or 'ê', in which case the mark goes on the last vowel
+            # If there is no a e or o, the vowels are either 'iu', 'ui', or 'ê',
+            # in which case the mark goes on the last vowel
             tone_vowel = vowels[-1]
 
         # Generate variants with each tone mark
@@ -578,27 +576,31 @@ class PinyinTokenizer(TokenizerI):
         return variants
 
     def __init__(self, include_nonstandard=False):
-        self.trie = CharTrie()
-        # Add standard syllables
+        self.preprocess_tokenizer = WordPunctTokenizer()
 
+        trie_contents = []
+
+        # Add standard syllables
         for syllable in self.STANDARD_SYLLABLES:
             for variant in self._get_tone_variants(syllable):
-                self.trie[variant] = len(variant)
+                trie_contents.append(variant)
 
         if include_nonstandard:
             for syllable in self.NON_STANDARD_SYLLABLES:
                 for variant in self._get_tone_variants(syllable):
-                    self.trie[variant] = len(variant)
+                    trie_contents.append(variant)
 
-    def span_tokenize(self, s: str) -> Iterator[Tuple[int, int]]:
-        # Check for any invalid characters
-        if self.VALID_CHARS_PATTERN.search(s):
-            raise ValueError("Input string can only contain letters and tone marks")
+        self.trie = marisa_trie.Trie(trie_contents)
 
+    def _get_string_possibilites(self, s) -> List[List[Tuple[int, int]]]:
+        """
+        For a given string, return all possible valid syllable spans of that string,
+        indexed local to the string
+        """
         candidates = []
 
         # Generate all possible splits starting from the beginning
-        to_process = [(0, [])]
+        to_process: List[Tuple[int, List[int]]] = [(0, [])]
         while to_process:
             # Get next position and accumulated split indices to process
             start_pos, split_indices = to_process.pop()
@@ -608,14 +610,15 @@ class PinyinTokenizer(TokenizerI):
             prefix_matches = self.trie.prefixes(remaining_s)
 
             # For each possible syllable length
-            for _, length in prefix_matches:
+            for match in prefix_matches:
                 # Create a new split point list with this syllable's endpoint
                 new_splits = split_indices.copy()
-                new_splits.append(start_pos + length)
+                new_splits.append(start_pos + len(match))
 
-                if start_pos + length < len(s):
-                    # If we haven't reached the end, continue processing from end of this syllable
-                    to_process.append((start_pos + length, new_splits))
+                if start_pos + len(match) < len(s):
+                    # If we haven't reached the end,
+                    # continue processing from end of this syllable
+                    to_process.append((start_pos + len(match), new_splits))
                 else:
                     # We've reached the end - construct the output span tuples
                     spans = []
@@ -625,35 +628,55 @@ class PinyinTokenizer(TokenizerI):
                         prev = pos
                     candidates.append(spans)
 
-        if not candidates:
-            raise ValueError(f"No valid pinyin syllable splits found for '{s}'")
-
-        # Find the shortest candidate(s) by comparing total number of splits
-        min_length = min(len(splits) for splits in candidates)
-        shortest = [c for c in candidates if len(c) == min_length]
+        return candidates
 
-        if len(shortest) > 1:
-            # Use syllable frequencies as tiebreaker
-            max_freq = float("-inf")
-            best_split = None
-
-            for split in shortest:
-                # Get syllables without tones and sum their frequencies
-                syllables = [s[start:end].lower() for start, end in split]
-                total_freq = sum(
-                    int(self.SYLLABLE_FREQUENCIES.get(syl, "0")) for syl in syllables
-                )
-
-                if total_freq > max_freq:
-                    max_freq = total_freq
-                    best_split = split
-
-            spans = best_split
-        else:
-            spans = shortest[0]
-
-        for span in spans:
-            yield span
+    def span_tokenize(self, s: str) -> Iterator[Tuple[int, int]]:
+        # Start by splitting into sub-spans to examine
+        starting_spans = self.preprocess_tokenizer.span_tokenize(s)
+
+        final_spans = []
+        for start, end in starting_spans:
+            subspan_possibilities = self._get_string_possibilites(s[start:end])
+            if not subspan_possibilities:
+                # We were passed a subspan that wasnt valid pinyin
+                # If it is a non-alphabetic span, then we return as-is
+                if self.NON_ALPHABETIC_REGEX.match(s[start:end]):
+                    final_spans.append((start, end))
+                    continue
+                else:
+                    raise ValueError(f"Invalid pinyin at substring: {s[start:end]}")
+
+            # Find the shortest candidate(s) for this span
+            min_length = min(len(splits) for splits in subspan_possibilities)
+            shortest = [c for c in subspan_possibilities if len(c) == min_length]
+
+            if len(shortest) > 1:
+                # Use syllable frequencies as tiebreaker
+                max_freq = float("-inf")
+                best_split = None
+
+                for split in shortest:
+                    # Get syllables without tones and sum their frequencies
+                    syllables = [s[start:end].lower() for start, end in split]
+                    total_freq = sum(
+                        int(self.SYLLABLE_FREQUENCIES.get(syl, "0"))
+                        for syl in syllables
+                    )
+
+                    if total_freq > max_freq:
+                        max_freq = total_freq
+                        best_split = split
+
+                assert best_split is not None
+                chosen = best_split
+            else:
+                chosen = shortest[0]
+
+            for subspan in chosen:
+                # Convert from local span indices to full string indices
+                final_spans.append((start + subspan[0], start + subspan[1]))
+
+        yield from final_spans
 
     def tokenize(self, s: str) -> List[str]:
         return [s[start:end] for start, end in self.span_tokenize(s)]

{py_pinyin_split-2.0.0 → py_pinyin_split-3.0.2}/tests/test_pinyinsplit.py

@@ -1,5 +1,12 @@
-import pytest  # type: ignore
-from pinyin_split.pinyinsplit import PinyinTokenizer
+import pytest
+
+from py_pinyin_split import PinyinTokenizer
+
+
+def test_benchmark_simple(benchmark):
+    tokenizer = PinyinTokenizer()
+    result = benchmark(tokenizer.tokenize, "xīnniánkuàilè")
+    assert result == ["xīn", "nián", "kuài", "lè"]
 
 
 def test_no_tone_splits():
@@ -22,18 +29,10 @@ def test_tone_splits():
     assert tokenizer.tokenize("màn") == ["màn"]
 
 
-def test_edge_cases():
-    """Test edge cases and invalid inputs"""
+def test_invalid_pinyin():
+    """Inputs with invalid pinyin throw ValueErrors"""
     tokenizer = PinyinTokenizer()
 
-    # Empty string should raise ValueError
-    with pytest.raises(ValueError):
-        tokenizer.tokenize("")
-
-    # Invalid characters should raise ValueError
-    with pytest.raises(ValueError):
-        tokenizer.tokenize("hello!")
-
     # Single consonant should raise ValueError
     with pytest.raises(ValueError):
         tokenizer.tokenize("x")
@@ -47,6 +46,30 @@ def test_edge_cases():
         tokenizer.tokenize("ni3hao3")
 
 
+def test_text_with_whitespace():
+    """Test handling of longer text with whitespace"""
+    tokenizer = PinyinTokenizer()
+
+    # Simple whitespace
+    assert tokenizer.tokenize("nǐ hǎo") == ["nǐ", "hǎo"]
+
+    # Multiple words with mixed tones
+    assert tokenizer.tokenize("Wǒ hěn xǐhuān Zhōngguó") == [
+        "Wǒ",
+        "hěn",
+        "xǐ",
+        "huān",
+        "Zhōng",
+        "guó",
+    ]
+
+    # Leading/trailing whitespace
+    assert tokenizer.tokenize("  nǐ hǎo  ") == ["nǐ", "hǎo"]
+
+    # Multiple whitespace characters
+    assert tokenizer.tokenize("nǐ  hǎo\t\nma") == ["nǐ", "hǎo", "ma"]
+
+
 def test_nonstandard_syllables():
     """Test handling of non-standard syllables"""
     standard_tokenizer = PinyinTokenizer(include_nonstandard=False)
@@ -108,3 +131,8 @@ def test_ambiguous_splits():
     # Test that tones help resolve ambiguity
     assert tokenizer.tokenize("xīan") == ["xī", "an"]
     assert tokenizer.tokenize("xián") == ["xián"]
+
+    # Test apostrophe handling
+    assert tokenizer.tokenize("Xī'ān") == ["Xī", "'", "ān"]
+    assert tokenizer.tokenize("yī'er") == ["yī", "'", "er"]
+    assert tokenizer.tokenize("tián'é") == ["tián", "'", "é"]