scriptconv 0.0.4a30__tar.gz → 0.0.4a31__tar.gz

{scriptconv-0.0.4a30/scriptconv.egg-info → scriptconv-0.0.4a31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a30
+Version: 0.0.4a31
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -42,6 +42,9 @@ Requires-Dist: epitran; extra == "epitran"
 Provides-Extra: transphone
 Requires-Dist: scriptconv[phonemizers]; extra == "transphone"
 Requires-Dist: transphone; extra == "transphone"
+Provides-Extra: phonetisaurus
+Requires-Dist: scriptconv[phonemizers]; extra == "phonetisaurus"
+Requires-Dist: phonetisaurus; extra == "phonetisaurus"
 Provides-Extra: misaki
 Requires-Dist: scriptconv[phonemizers]; extra == "misaki"
 Requires-Dist: misaki; extra == "misaki"

{scriptconv-0.0.4a30 → scriptconv-0.0.4a31}/pyproject.toml

@@ -42,6 +42,7 @@ gruut = ["scriptconv[phonemizers]", "gruut"]
 goruut = ["scriptconv[phonemizers]", "pygoruut"]
 epitran = ["scriptconv[phonemizers]", "epitran"]
 transphone = ["scriptconv[phonemizers]", "transphone"]
+phonetisaurus = ["scriptconv[phonemizers]", "phonetisaurus"]
 misaki = ["scriptconv[phonemizers]", "misaki"]
 byt5 = ["scriptconv[phonemizers]", "onnxruntime", "numpy"]
 o2i = ["scriptconv[phonemizers]", "orthography2ipa"]

{scriptconv-0.0.4a30 → scriptconv-0.0.4a31}/scriptconv/phonemizers/enums.py

@@ -52,6 +52,7 @@ class Phonemizer(str, Enum):
     BYT5 = "byt5"
     CHARSIU = "charsiu"  # technically same as byt5, but needs special handling for whitespace
     TRANSPHONE = "transphone"
+    PHONETISAURUS = "phonetisaurus"  # WFST g2p; language and symbols come from the model
     MIRANDESE = "mwl_phonemizer"
 
     DEEPPHONEMIZER = "deepphonemizer" # en

{scriptconv-0.0.4a30 → scriptconv-0.0.4a31}/scriptconv/phonemizers/mul.py

@@ -9,7 +9,7 @@ if TYPE_CHECKING:
     import numpy as np
 
 from scriptconv.phonemizers.enums import Alphabet
-from scriptconv.phonemizers.base import BasePhonemizer
+from scriptconv.phonemizers.base import BasePhonemizer, _check_alphabet
 # minimal ONNX session construction — scriptconv has no provider
 # machinery; consumers wanting CUDA/etc. pass providers= explicitly
 ProviderSpec = str
@@ -1465,3 +1465,101 @@ class TransphonePhonemizer(BasePhonemizer):
             [p if p != "<SPACE>" else " "
              for p in pho.tokenize(text, use_space=True)]
         ).strip()
+
+
+class PhonetisaurusPhonemizer(BasePhonemizer):
+    """WFST grapheme-to-phoneme, through the ``phonetisaurus`` bindings.
+
+    https://github.com/AdolfVonKleist/Phonetisaurus
+
+    Phonetisaurus has no rules and no language list of its own: all of its
+    knowledge is in one trained FST, so the language and the symbol set are
+    properties of the MODEL, not of this wrapper. That has three consequences,
+    and each one is a design decision rather than a limitation to work around:
+
+    * ``model`` is required. scriptconv never downloads a model (the same rule
+      :class:`ByT5Phonemizer` follows). ``MODEL_SOURCES`` names where published
+      models live; a caller resolves one and passes the path.
+    * ``alphabet`` is declared by the caller, because only the caller knows
+      what the model was trained to emit. A CMUdict-trained model emits ARPA,
+      an o2i-trained or espeak-trained one emits IPA. The wrapper cannot read
+      it off the FST, so it does not guess.
+    * There is no language check and no ``LANG_DEFAULTS`` entry. A wrapper that
+      cannot know its own model's language must not be any language's default.
+
+    ``phonetisaurus`` ships the binaries it drives, so no system package is
+    needed. Training a model needs the bundled ``estimate-ngram``, which is not
+    exercised here: this wrapper only applies a model.
+    """
+
+    #: Where published FSTs come from. Documentation, never a download.
+    MODEL_SOURCES = {
+        "rhasspy": "https://github.com/rhasspy/*-g2p (one g2p.fst per language)",
+        "cmudict": "trained from CMUdict with 'phonetisaurus train' (ARPA symbols)",
+    }
+
+    #: Symbol sets a Phonetisaurus model is known to be trained on. The wrapper
+    #: passes the model's symbols through untouched, so this list says which
+    #: declarations are meaningful, not which conversions happen.
+    SUPPORTED_ALPHABETS = [Alphabet.IPA, Alphabet.ARPA, Alphabet.SAMPA,
+                           Alphabet.XSAMPA, Alphabet.GRAPHEMES]
+
+    def __init__(self, model: Optional[str] = None,
+                 alphabet: Alphabet = Alphabet.IPA,
+                 nbest: int = 1,
+                 separator: str = " ",
+                 normalizer=None):
+        """
+        Args:
+            model: path to a trained Phonetisaurus FST. Required.
+            alphabet: the symbol set the model emits. The caller states it.
+            nbest: how many pronunciations the engine ranks per word. The
+                BEST one is returned whatever this is, so a higher value
+                changes the search, not the output shape. ``predict`` prints
+                one line per ranked pronunciation, best first, and yields one
+                pair per line, so the wrapper keeps the FIRST pair it sees for
+                a word and drops the rest.
+            separator: joins the symbols of one word. Models are trained on
+                symbol sequences, so the parts need a separator to stay
+                readable; pass "" for a bare string.
+            normalizer: see :class:`BasePhonemizer`.
+        """
+        _check_alphabet(self, alphabet, self.SUPPORTED_ALPHABETS)
+        super().__init__(alphabet, normalizer=normalizer)
+        try:
+            import phonetisaurus
+        except ImportError as e:
+            raise ImportError(
+                "phonetisaurus is required for the Phonetisaurus phonemizer. "
+                "Install it with 'pip install phonetisaurus' "
+                "(or 'pip install scriptconv[phonetisaurus]')."
+            ) from e
+        if not model or not os.path.isfile(model):
+            raise ValueError(
+                "Phonetisaurus phonemization needs a local trained FST: pass "
+                "model=<path> (scriptconv never downloads — see "
+                "MODEL_SOURCES for where published models live)")
+        self.phonetisaurus = phonetisaurus
+        self.model = model
+        self.nbest = nbest
+        self.separator = separator
+
+    def phonemize_string(self, text: str, lang: str) -> str:
+        """Phonemize ``text``. ``lang`` is accepted and not used: the model
+        decides the language, so a language argument here would claim a
+        routing this wrapper does not do."""
+        words = text.split()
+        if not words:
+            return ""
+        # predict() yields (word, symbols) and may reorder or drop a word it
+        # cannot read, so the result is indexed by word rather than zipped.
+        #
+        # With nbest > 1 it yields one pair PER RANKED PRONUNCIATION, best
+        # first, so a word appears more than once. dict() would keep the last
+        # pair, which is the WORST-ranked reading. Keep the first instead.
+        guesses: Dict[str, List[str]] = {}
+        for word, symbols in self.phonetisaurus.predict(
+                words, self.model, nbest=self.nbest):
+            guesses.setdefault(word, symbols)
+        return " ".join(self.separator.join(guesses[w])
+                        for w in words if w in guesses)

{scriptconv-0.0.4a30 → scriptconv-0.0.4a31}/scriptconv/phonemizers/registry.py

@@ -35,6 +35,7 @@ PHONEMIZER_REGISTRY: Dict[Phonemizer, Tuple[str, str, Optional[str]]] = {
     _P.GORUUT: (f"{_BASE}.mul", "GoruutPhonemizer", "goruut"),
     _P.EPITRAN: (f"{_BASE}.mul", "EpitranPhonemizer", "epitran"),
     _P.TRANSPHONE: (f"{_BASE}.mul", "TransphonePhonemizer", "transphone"),
+    _P.PHONETISAURUS: (f"{_BASE}.mul", "PhonetisaurusPhonemizer", "phonetisaurus"),
     _P.BYT5: (f"{_BASE}.mul", "ByT5Phonemizer", "byt5"),
     _P.CHARSIU: (f"{_BASE}.mul", "CharsiuPhonemizer", "byt5"),
     _P.MISAKI: (f"{_BASE}.mul", "MisakiPhonemizer", "misaki"),

{scriptconv-0.0.4a30 → scriptconv-0.0.4a31}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 30
+VERSION_ALPHA = 31
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a30 → scriptconv-0.0.4a31/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a30
+Version: 0.0.4a31
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -42,6 +42,9 @@ Requires-Dist: epitran; extra == "epitran"
 Provides-Extra: transphone
 Requires-Dist: scriptconv[phonemizers]; extra == "transphone"
 Requires-Dist: transphone; extra == "transphone"
+Provides-Extra: phonetisaurus
+Requires-Dist: scriptconv[phonemizers]; extra == "phonetisaurus"
+Requires-Dist: phonetisaurus; extra == "phonetisaurus"
 Provides-Extra: misaki
 Requires-Dist: scriptconv[phonemizers]; extra == "misaki"
 Requires-Dist: misaki; extra == "misaki"

{scriptconv-0.0.4a30 → scriptconv-0.0.4a31}/scriptconv.egg-info/SOURCES.txt

@@ -511,6 +511,7 @@ tests/test_phonemizers_cjk_ar.py
 tests/test_phonemizers_friendly_import_errors.py
 tests/test_phonemizers_ghana.py
 tests/test_phonemizers_halabi_iqra.py
+tests/test_phonemizers_phonetisaurus.py
 tests/test_phonemizers_pt_tugaphone.py
 tests/test_phonemizers_ru.py
 tests/test_readings.py

{scriptconv-0.0.4a30 → scriptconv-0.0.4a31}/scriptconv.egg-info/requires.txt

@@ -84,6 +84,10 @@ orthography2ipa
 quebra-frases
 langcodes
 
+[phonetisaurus]
+scriptconv[phonemizers]
+phonetisaurus
+
 [pt]
 scriptconv[phonemizers]
 bifonia>=0.1.1

{scriptconv-0.0.4a30 → scriptconv-0.0.4a31}/tests/test_phonemizers_base.py

@@ -26,7 +26,7 @@ class TestEnums(unittest.TestCase):
         self.assertEqual(Alphabet.XSAMPA.value, "x-sampa")
         self.assertEqual(Alphabet.VOSK.value, "vosk")
         self.assertEqual(Phonemizer.GHANA_G2P.value, "ghana_g2p")
-        self.assertEqual(len(list(Phonemizer)), 46)  # +HALABI, +IQRA, +GHANA_G2P
+        self.assertEqual(len(list(Phonemizer)), 47)  # +HALABI, +IQRA, +GHANA_G2P, +PHONETISAURUS
         self.assertEqual(len(list(Alphabet)), 23)  # incl. HALABI, VOSK, AFRICA_G2P
 
 

{scriptconv-0.0.4a30 → scriptconv-0.0.4a31}/tests/test_phonemizers_friendly_import_errors.py

@@ -51,6 +51,7 @@ CASES = {
     Phonemizer.GRUUT: ("gruut", _trigger_gruut),
     Phonemizer.GORUUT: ("pygoruut", _trigger_construct),
     Phonemizer.EPITRAN: ("epitran", _trigger_construct),
+    Phonemizer.PHONETISAURUS: ("phonetisaurus", _trigger_construct),
     Phonemizer.TRANSPHONE: ("transphone", _trigger_construct),
     Phonemizer.DEEPPHONEMIZER: ("dp", _trigger_construct),
     Phonemizer.OPENPHONEMIZER: ("openphonemizer", _trigger_construct),
@@ -68,7 +69,7 @@ class TestLazyBackendImportsAreFriendly(unittest.TestCase):
     def test_all_documented_backends_covered(self):
         # keeps this test list honest if a new Phonemizer member is added
         # backed by one of the wrapper modules touched by this convention
-        self.assertEqual(len(CASES), 15)
+        self.assertEqual(len(CASES), 16)
 
     def _assert_friendly(self, member):
         module_name, trigger = CASES[member]
@@ -97,6 +98,9 @@ class TestLazyBackendImportsAreFriendly(unittest.TestCase):
     def test_transphone(self):
         self._assert_friendly(Phonemizer.TRANSPHONE)
 
+    def test_phonetisaurus(self):
+        self._assert_friendly(Phonemizer.PHONETISAURUS)
+
     def test_deepphonemizer(self):
         self._assert_friendly(Phonemizer.DEEPPHONEMIZER)
 

scriptconv-0.0.4a31/tests/test_phonemizers_phonetisaurus.py

@@ -0,0 +1,201 @@
+"""The Phonetisaurus wrapper: registry wiring, the model contract, and the
+output string.
+
+Phonetisaurus carries no rules of its own. Everything it knows is in one
+trained FST, and no FST is published with the ``phonetisaurus`` package or with
+scriptconv, so a test that phonemizes a real word against a real model would
+have to train one or download one. Training runs the bundled ``estimate-ngram``
+binary, which is a system-level dependency this suite must not need, and a test
+never downloads. So the engine call is stubbed at the one function the wrapper
+uses, ``phonetisaurus.predict``, and the assertions are on the string the
+wrapper builds from the engine's answer. The parts NOT stubbed are the parts
+this wrapper owns: the model contract, the word indexing, the separator and the
+alphabet check.
+"""
+import os
+import sys
+import tempfile
+import unittest
+from types import ModuleType
+from unittest.mock import patch
+
+from scriptconv.phonemizers import Phonemizer, get_phonemizer_class
+from scriptconv.phonemizers.enums import Alphabet
+from scriptconv.phonemizers.registry import PHONEMIZER_REGISTRY
+
+
+def _fake_phonetisaurus(answers, reorder=True):
+    """A stand-in for the ``phonetisaurus`` package, exposing only
+    ``predict``, which is the whole surface the wrapper uses.
+
+    The real ``predict`` reads a word list and yields ``(word, symbols)``
+    pairs. It does not promise the caller's order, and it yields nothing for a
+    word the model cannot read. The stub returns pairs in REVERSE order by
+    default for exactly that reason: a wrapper that consumed the pairs
+    positionally would pass this suite if the stub echoed the request order,
+    and would put the wrong pronunciation on every word in production.
+
+    ``nbest`` is modelled too, because it changes the SHAPE of the answer and
+    not just its length: the engine prints one line per ranked pronunciation,
+    BEST FIRST, and ``predict`` yields one pair per line, so a word appears
+    ``nbest`` times. ``answers`` maps a word to its ranked list, best first.
+
+    The reversal therefore applies to the WORDS and never inside one word's
+    ranked lines. Word order is a freedom the engine has; rank order is a
+    promise it keeps, and a stub that broke it would be testing a different
+    engine.
+    """
+    mod = ModuleType("phonetisaurus")
+
+    def predict(words, model_path, nbest=1, env=None):
+        # one group per word, the group's pairs in rank order
+        groups = [[(w, ranked) for ranked in answers[w][:nbest]]
+                  for w in words if w in answers]
+        if reorder:
+            # the WORDS may come back in any order; the ranked lines of one
+            # word may not, so the groups reverse and their contents do not
+            groups.reverse()
+        for group in groups:
+            for pair in group:
+                yield pair
+
+    mod.predict = predict
+    return mod
+
+
+class TestRegistryWiring(unittest.TestCase):
+    def test_registry_names_the_module_class_and_extra(self):
+        entry = PHONEMIZER_REGISTRY[Phonemizer.PHONETISAURUS]
+        self.assertEqual(
+            entry,
+            ("scriptconv.phonemizers.mul", "PhonetisaurusPhonemizer",
+             "phonetisaurus"))
+
+    def test_the_class_resolves(self):
+        cls = get_phonemizer_class(Phonemizer.PHONETISAURUS)
+        self.assertEqual(cls.__name__, "PhonetisaurusPhonemizer")
+
+    def test_the_enum_value_is_the_wire_string(self):
+        self.assertEqual(Phonemizer.PHONETISAURUS.value, "phonetisaurus")
+
+    def test_the_extra_is_declared_in_pyproject(self):
+        try:
+            import tomllib
+        except ImportError:
+            import tomli as tomllib
+        root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        with open(os.path.join(root, "pyproject.toml"), "rb") as f:
+            extras = tomllib.load(f)["project"]["optional-dependencies"]
+        self.assertIn("phonetisaurus", extras)
+        self.assertIn("phonetisaurus", extras["phonetisaurus"])
+
+
+class TestModelContract(unittest.TestCase):
+    """scriptconv never downloads a model, so the wrapper must refuse to be
+    built without one, and say what to pass."""
+
+    def _cls(self):
+        return get_phonemizer_class(Phonemizer.PHONETISAURUS)
+
+    def test_no_model_is_refused(self):
+        with patch.dict(sys.modules,
+                        {"phonetisaurus": _fake_phonetisaurus({})}):
+            with self.assertRaises(ValueError) as ctx:
+                self._cls()()
+        self.assertIn("model=", str(ctx.exception))
+
+    def test_a_missing_file_is_refused(self):
+        with patch.dict(sys.modules,
+                        {"phonetisaurus": _fake_phonetisaurus({})}):
+            with self.assertRaises(ValueError):
+                self._cls()(model="/nonexistent/g2p.fst")
+
+    def test_an_alphabet_the_wrapper_does_not_declare_is_refused(self):
+        with tempfile.NamedTemporaryFile(suffix=".fst") as fst:
+            with patch.dict(sys.modules,
+                            {"phonetisaurus": _fake_phonetisaurus({})}):
+                with self.assertRaises(ValueError) as ctx:
+                    self._cls()(model=fst.name, alphabet=Alphabet.HANGUL)
+        self.assertIn("hangul", str(ctx.exception))
+
+
+class TestOutputString(unittest.TestCase):
+    """What the wrapper builds out of the engine's answer."""
+
+    def _phonemizer(self, answers, **kw):
+        cls = get_phonemizer_class(Phonemizer.PHONETISAURUS)
+        fst = tempfile.NamedTemporaryFile(suffix=".fst", delete=False)
+        fst.close()
+        self.addCleanup(os.unlink, fst.name)
+        with patch.dict(sys.modules,
+                        {"phonetisaurus": _fake_phonetisaurus(answers)}):
+            return cls(model=fst.name, **kw)
+
+    def test_one_word(self):
+        p = self._phonemizer({"cat": [["k", "a", "t"]]})
+        self.assertEqual(p.phonemize_string("cat", "en"), "k a t")
+
+    def test_two_words_keep_their_order(self):
+        p = self._phonemizer({"cat": [["k", "a", "t"]],
+                              "bat": [["b", "a", "t"]]})
+        self.assertEqual(p.phonemize_string("cat bat", "en"), "k a t b a t")
+
+    def test_an_empty_separator_gives_a_bare_string(self):
+        p = self._phonemizer({"cat": [["k", "a", "t"]]}, separator="")
+        self.assertEqual(p.phonemize_string("cat", "en"), "kat")
+
+    def test_a_word_the_engine_drops_is_skipped(self):
+        """predict() yields nothing for a word the model cannot read. The
+        wrapper drops it rather than raising."""
+        p = self._phonemizer({"cat": [["k", "a", "t"]],
+                              "bat": [["b", "a", "t"]]})
+        self.assertEqual(p.phonemize_string("cat xyzzy bat", "en"),
+                         "k a t b a t")
+
+    def test_the_output_follows_the_INPUT_order_not_the_engine_order(self):
+        """The discriminating case. The stub answers in reverse, so a wrapper
+        that consumed the pairs positionally would return them reversed. The
+        wrapper indexes by word, so the sentence keeps its own order."""
+        p = self._phonemizer({"cat": [["k", "a", "t"]],
+                              "dog": [["d", "o", "g"]],
+                              "bat": [["b", "a", "t"]]})
+        self.assertEqual(p.phonemize_string("cat dog bat", "en"),
+                         "k a t d o g b a t")
+
+    def test_nbest_2_keeps_the_BEST_ranked_pronunciation(self):
+        """With nbest > 1 the engine yields one pair per ranked reading, best
+        first, so a word appears twice. Building a dict over the pairs keeps
+        the LAST one, which is the worst-ranked reading, while the docstring
+        promises the best. The wrapper keeps the first pair per word.
+
+        This is the case that tells the two apart: both readings are present,
+        both are well-formed, and only the ORDER says which is right."""
+        p = self._phonemizer({"cat": [["k", "a", "t"], ["k", "a", "ts"]]},
+                             nbest=2)
+        self.assertEqual(p.phonemize_string("cat", "en"), "k a t")
+
+    def test_nbest_2_over_several_words(self):
+        p = self._phonemizer({"cat": [["k", "a", "t"], ["k", "a", "ts"]],
+                              "bat": [["b", "a", "t"], ["b", "a", "ts"]]},
+                             nbest=2)
+        self.assertEqual(p.phonemize_string("cat bat", "en"), "k a t b a t")
+
+    def test_empty_text(self):
+        p = self._phonemizer({"cat": [["k", "a", "t"]]})
+        self.assertEqual(p.phonemize_string("", "en"), "")
+
+    def test_the_language_argument_does_not_route(self):
+        """The model decides the language. Any tag must give the same answer,
+        because the wrapper claims no routing."""
+        p = self._phonemizer({"cat": [["k", "a", "t"]]})
+        self.assertEqual(p.phonemize_string("cat", "en"),
+                         p.phonemize_string("cat", "oc"))
+
+    def test_the_declared_alphabet_is_kept(self):
+        p = self._phonemizer({"cat": [["K", "AE", "T"]]}, alphabet=Alphabet.ARPA)
+        self.assertEqual(p.alphabet, Alphabet.ARPA)
+        self.assertEqual(p.phonemize_string("cat", "en"), "K AE T")
+
+
+if __name__ == "__main__":
+    unittest.main()