kokorog2p 0.3.2__tar.gz → 0.3.3__tar.gz

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kokorog2p
-Version: 0.3.2
+Version: 0.3.3
 Summary: A unified G2P (Grapheme-to-Phoneme) library for Kokoro TTS
 Author-email: Holger Nahrstaedt <nahrstaedt@gmail.com>
 License:                                  Apache License

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/kokorog2p/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.3.2'
-__version_tuple__ = version_tuple = (0, 3, 2)
+__version__ = version = '0.3.3'
+__version_tuple__ = version_tuple = (0, 3, 3)
 
-__commit_id__ = commit_id = 'g47241d749'
+__commit_id__ = commit_id = 'g2b66f6ceb'

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/kokorog2p/base.py

@@ -70,12 +70,12 @@ class G2PBase(ABC):
             if token.phonemes:
                 result.append(token.phonemes)
                 if token.whitespace:
-                    result.append(" ")
+                    result.append(token.whitespace)
             elif token.is_punctuation:
                 # Keep punctuation as-is
                 result.append(token.text)
                 if token.whitespace:
-                    result.append(" ")
+                    result.append(token.whitespace)
         return "".join(result).strip()
 
     def word_to_phonemes(self, word: str, tag: str | None = None) -> str | None:

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/kokorog2p/markdown.py

@@ -61,8 +61,8 @@ def preprocess_markdown(text: str) -> tuple[str, list[str], dict[int, str]]:
         # Extract phonemes (group 2)
         phonemes = m.group(2)
 
-        # Check if it's a phoneme annotation (starts with /)
-        if phonemes and phonemes.startswith("/"):
+        # Check if it's a phoneme annotation (starts and ends with /)
+        if phonemes and phonemes.startswith("/") and phonemes.endswith("/"):
             phonemes = phonemes.strip("/")  # Remove leading and trailing slashes
             features[len(tokens)] = phonemes
 
@@ -99,8 +99,9 @@ def apply_markdown_features(
     # This assumes G2P tokenization preserves words from preprocessing
     token_map: dict[int, int] = {}
     for i, orig_word in enumerate(original_tokens):
+        orig_key = orig_word.casefold()
         for j, token in enumerate(tokens):
-            if token.text == orig_word and j not in token_map.values():
+            if token.text.casefold() == orig_key and j not in token_map.values():
                 token_map[i] = j
                 break
 

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/kokorog2p/mixed_language_g2p.py

@@ -165,15 +165,28 @@ class MixedLanguageG2P(G2PBase):
             ImportError: If enable_detection=True but lingua-language-detector
                 is not installed (will warn and disable detection instead).
         """
-        super().__init__(language=primary_language)
-        self.version = version
-
         if allowed_languages is None or len(allowed_languages) == 0:
             raise ValueError(
                 "allowed_languages must be specified and non-empty. "
                 "Example: allowed_languages=['de', 'en-us']"
             )
 
+        normalized_allowed = [
+            lang.lower().replace("_", "-") for lang in allowed_languages
+        ]
+        primary_normalized = primary_language.lower().replace("_", "-")
+        if primary_normalized not in normalized_allowed:
+            raise ValueError(
+                "primary_language must be in allowed_languages. "
+                "Example: primary_language='de', allowed_languages=['de', 'en-us']"
+            )
+
+        primary_language = primary_normalized
+        allowed_languages = normalized_allowed
+
+        super().__init__(language=primary_language)
+        self.version = version
+
         # Check if lingua is available
         if not LINGUA_AVAILABLE:
             if enable_detection:

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/kokorog2p/pipeline/tokenizer.py

@@ -202,10 +202,15 @@ class BaseTokenizer(ABC):
         """
         if self.phoneme_quotes == "ascii":
             # Convert all quote variants to ASCII
-            text = text.replace('"', '"').replace('"', '"')
+            text = text.replace("\u201c", '"').replace("\u201d", '"').replace("`", '"')
         elif self.phoneme_quotes == "none":
             # Strip all quotes
-            text = text.replace('"', "").replace('"', "").replace('"', "")
+            text = (
+                text.replace('"', "")
+                .replace("\u201c", "")
+                .replace("\u201d", "")
+                .replace("`", "")
+            )
         # else: "curly" - keep as-is
 
         return text

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/kokorog2p/token.py

@@ -37,7 +37,20 @@ class GToken:
     @property
     def is_punctuation(self) -> bool:
         """Check if this token is punctuation."""
-        return self.tag in (".", ",", ":", ";", "!", "?", "-", "'", '"', "(", ")")
+        return self.tag in (
+            ".",
+            ",",
+            ":",
+            ";",
+            "!",
+            "?",
+            "-",
+            "'",
+            '"',
+            "(",
+            ")",
+            "PUNCT",
+        )
 
     @property
     def is_word(self) -> bool:
@@ -61,6 +74,7 @@ class GToken:
             phonemes=self.phonemes,
             start_ts=self.start_ts,
             end_ts=self.end_ts,
+            rating=self.rating,
             _=dict(self._),
         )
 

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/kokorog2p.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kokorog2p
-Version: 0.3.2
+Version: 0.3.3
 Summary: A unified G2P (Grapheme-to-Phoneme) library for Kokoro TTS
 Author-email: Holger Nahrstaedt <nahrstaedt@gmail.com>
 License:                                  Apache License

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/kokorog2p.egg-info/SOURCES.txt

@@ -214,6 +214,7 @@ kokorog2p/zh/transcription.py
 tests/__init__.py
 tests/conftest.py
 tests/test_abbreviation_customization.py
+tests/test_base.py
 tests/test_benchmark_validation.py
 tests/test_ci_bug_fix.py
 tests/test_contraction_detection.py

kokorog2p-0.3.3/tests/test_base.py

@@ -0,0 +1,33 @@
+"""Tests for G2PBase utilities."""
+
+from kokorog2p.base import G2PBase
+from kokorog2p.token import GToken
+
+
+class DummyG2P(G2PBase):
+    """Minimal G2P implementation for base tests."""
+
+    def __init__(self, tokens: list[GToken]):
+        super().__init__(language="en-us")
+        self._tokens = tokens
+
+    def __call__(self, text: str) -> list[GToken]:
+        return list(self._tokens)
+
+    def lookup(self, word: str, tag: str | None = None) -> str | None:
+        return None
+
+
+class TestG2PBase:
+    """Tests for G2PBase helpers."""
+
+    def test_phonemize_preserves_whitespace(self):
+        """phonemize should preserve token whitespace exactly."""
+        tokens = [
+            GToken(text="Hello", phonemes="h", whitespace="   "),
+            GToken(text="world", phonemes="w", whitespace=""),
+            GToken(text=".", tag=".", whitespace=""),
+        ]
+        g2p = DummyG2P(tokens)
+
+        assert g2p.phonemize("ignored") == "h   w."

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/tests/test_markdown.py

@@ -52,6 +52,13 @@ class TestPreprocessMarkdown:
         assert "link" in clean
         assert len(features) == 0  # Not a phoneme annotation
 
+    def test_relative_link_is_not_annotation(self):
+        """Relative links should not be treated as annotations."""
+        text = "[link](/docs/page) test"
+        clean, tokens, features = preprocess_markdown(text)
+        assert "link" in clean
+        assert len(features) == 0
+
     def test_empty_annotation(self):
         """Test annotation with empty phonemes."""
         text = "[word](/) test"
@@ -151,6 +158,19 @@ class TestApplyMarkdownFeatures:
         result = apply_markdown_features(tokens, features, orig_tokens)
         assert result[0].phonemes == "hɛloʊ"
 
+    def test_apply_feature_case_insensitive(self):
+        """Features should match regardless of casing differences."""
+        tokens = [
+            GToken(text="hello", phonemes="hɛloʊ"),
+            GToken(text="world", phonemes="wɝld"),
+        ]
+        features = {0: "hˈɛloʊ"}
+        orig_tokens = ["Hello", "world"]
+
+        result = apply_markdown_features(tokens, features, orig_tokens)
+        assert result[0].phonemes == "hˈɛloʊ"
+        assert result[0].get("rating") == 5
+
 
 class TestPhonemizeWithMarkdown:
     """Tests for phonemize_with_markdown function."""

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/tests/test_mixed_language_g2p.py

@@ -46,6 +46,27 @@ class TestMixedLanguageG2PBasic:
         with pytest.raises(ValueError, match="allowed_languages must be specified"):
             MixedLanguageG2P(primary_language="de", allowed_languages=[])
 
+    def test_primary_language_must_be_allowed(self):
+        """Primary language should be part of allowed_languages."""
+        from kokorog2p.mixed_language_g2p import MixedLanguageG2P
+
+        with pytest.raises(
+            ValueError, match="primary_language must be in allowed_languages"
+        ):
+            MixedLanguageG2P(primary_language="de", allowed_languages=["en-us"])
+
+    def test_language_codes_normalized(self):
+        """Language codes should normalize to lowercase with hyphens."""
+        from kokorog2p.mixed_language_g2p import MixedLanguageG2P
+
+        g2p = MixedLanguageG2P(
+            primary_language="DE",
+            allowed_languages=["EN_US", "DE"],
+            enable_detection=False,
+        )
+        assert g2p.primary_language == "de"
+        assert g2p.allowed_languages == ["en-us", "de"]
+
     def test_repr(self):
         """Test string representation."""
         from kokorog2p.mixed_language_g2p import MixedLanguageG2P

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/tests/test_token.py

@@ -52,6 +52,7 @@ class TestGToken:
         assert GToken(text=".", tag=".").is_punctuation is True
         assert GToken(text=",", tag=",").is_punctuation is True
         assert GToken(text="!", tag="!").is_punctuation is True
+        assert GToken(text="!", tag="PUNCT").is_punctuation is True
         assert GToken(text="hello", tag="NN").is_punctuation is False
 
     def test_is_word(self):
@@ -80,6 +81,7 @@ class TestGToken:
             text="hello",
             tag="NN",
             phonemes="hˈɛlO",
+            rating="espeak",
             _={"rating": 4},
         )
 
@@ -89,6 +91,7 @@ class TestGToken:
         assert copy.text == original.text
         assert copy.tag == original.tag
         assert copy.phonemes == original.phonemes
+        assert copy.rating == original.rating
         assert copy._["rating"] == 4
 
         # Check it's a different object

{kokorog2p-0.3.2 → kokorog2p-0.3.3}/tests/test_tokenizer.py

@@ -111,6 +111,18 @@ class TestRegexTokenizer:
         assert quote_tokens[0].quote_depth == 1
         assert quote_tokens[1].quote_depth == 0
 
+    def test_normalize_phoneme_quotes_ascii(self):
+        """Curly quotes should normalize to ASCII when requested."""
+        tokenizer = RegexTokenizer(phoneme_quotes="ascii")
+        text = "\u201cHello\u201d"
+        assert tokenizer.normalize_phoneme_quotes(text) == '"Hello"'
+
+    def test_normalize_phoneme_quotes_none(self):
+        """Quote characters should be stripped when requested."""
+        tokenizer = RegexTokenizer(phoneme_quotes="none")
+        text = "\u201cHello\u201d"
+        assert tokenizer.normalize_phoneme_quotes(text) == "Hello"
+
     def test_empty_string(self, tokenizer):
         """Test tokenizing empty string."""
         tokens = tokenizer.tokenize("")