scriptconv 0.0.4a32__tar.gz → 0.0.4a34__tar.gz

{scriptconv-0.0.4a32/scriptconv.egg-info → scriptconv-0.0.4a34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a32
+Version: 0.0.4a34
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -483,6 +483,39 @@ The core installs with zero dependencies. Capabilities opt in:
 | `tashkeel` | Arabic diacritization for the phonemizer pipeline (text2tashkeel) |
 | `stress` | Word-stress restoration for 26 stressonnx language tags: East Slavic; Bulgarian, Macedonian, and Slovene; Latvian; Armenian; Georgian; and Turkic/Caucasian, for the phonemizer pipeline (stressonnx; not yet on PyPI) |
 | `pt` | European-Portuguese heterophonic-homograph sense diacritics for the phonemizer pipeline (bifonia) |
+| `phonetisaurus` | WFST grapheme-to-phoneme through a trained FST the caller supplies (phonetisaurus). Training also needs `libquadmath0` on the host: see below |
+
+### The `phonetisaurus` extra and its libraries
+
+`pip install scriptconv[phonetisaurus]` completes, and the `phonetisaurus`
+0.3.0 wheel bundles its own executables and most of its own shared objects
+(`libfst.so.13`, `libmitlm.so.1`, `libgfortran.so.3` and the OpenFST
+companions) under `phonetisaurus/lib/<machine>`. The binaries find them only
+through the `PATH` and `LD_LIBRARY_PATH` that `phonetisaurus.guess_environment()`
+builds, which is the environment `phonetisaurus.predict` and
+`phonetisaurus.train` pass to every call. A binary started with the plain
+shell environment exits 127 from the dynamic loader; that is a report about
+the caller, not about the box.
+
+The apply path therefore runs out of the box, and this wrapper only applies a
+model. One object is NOT bundled:
+
+| Missing object | Needed by | Where it comes from |
+|---|---|---|
+| `libquadmath.so.0` | `estimate-ngram`, and so `phonetisaurus.train` | the GCC runtime: package `libquadmath0` on Debian and Ubuntu |
+
+So a box without `libquadmath0` can phonemize with a model it is given, and
+cannot train one.
+
+`model=` is required and scriptconv never downloads: see `MODEL_SOURCES` on
+the class, or train a model with `phonetisaurus.train` from a lexicon such as
+CMUdict.
+
+`tests/test_phonemizers_phonetisaurus_engine.py` is the real-engine cell. It
+runs the same word through the engine at `nbest=1` and `nbest=2` and checks
+the wrapper returns the engine's own best-ranked line both times. It skips,
+and names the loader error or the missing model, when the engine cannot run.
+Set `SCRIPTCONV_PHONETISAURUS_MODEL` to a trained FST to run it.
 
 ## Licensing
 

{scriptconv-0.0.4a32 → scriptconv-0.0.4a34}/README.md

@@ -345,6 +345,39 @@ The core installs with zero dependencies. Capabilities opt in:
 | `tashkeel` | Arabic diacritization for the phonemizer pipeline (text2tashkeel) |
 | `stress` | Word-stress restoration for 26 stressonnx language tags: East Slavic; Bulgarian, Macedonian, and Slovene; Latvian; Armenian; Georgian; and Turkic/Caucasian, for the phonemizer pipeline (stressonnx; not yet on PyPI) |
 | `pt` | European-Portuguese heterophonic-homograph sense diacritics for the phonemizer pipeline (bifonia) |
+| `phonetisaurus` | WFST grapheme-to-phoneme through a trained FST the caller supplies (phonetisaurus). Training also needs `libquadmath0` on the host: see below |
+
+### The `phonetisaurus` extra and its libraries
+
+`pip install scriptconv[phonetisaurus]` completes, and the `phonetisaurus`
+0.3.0 wheel bundles its own executables and most of its own shared objects
+(`libfst.so.13`, `libmitlm.so.1`, `libgfortran.so.3` and the OpenFST
+companions) under `phonetisaurus/lib/<machine>`. The binaries find them only
+through the `PATH` and `LD_LIBRARY_PATH` that `phonetisaurus.guess_environment()`
+builds, which is the environment `phonetisaurus.predict` and
+`phonetisaurus.train` pass to every call. A binary started with the plain
+shell environment exits 127 from the dynamic loader; that is a report about
+the caller, not about the box.
+
+The apply path therefore runs out of the box, and this wrapper only applies a
+model. One object is NOT bundled:
+
+| Missing object | Needed by | Where it comes from |
+|---|---|---|
+| `libquadmath.so.0` | `estimate-ngram`, and so `phonetisaurus.train` | the GCC runtime: package `libquadmath0` on Debian and Ubuntu |
+
+So a box without `libquadmath0` can phonemize with a model it is given, and
+cannot train one.
+
+`model=` is required and scriptconv never downloads: see `MODEL_SOURCES` on
+the class, or train a model with `phonetisaurus.train` from a lexicon such as
+CMUdict.
+
+`tests/test_phonemizers_phonetisaurus_engine.py` is the real-engine cell. It
+runs the same word through the engine at `nbest=1` and `nbest=2` and checks
+the wrapper returns the engine's own best-ranked line both times. It skips,
+and names the loader error or the missing model, when the engine cannot run.
+Set `SCRIPTCONV_PHONETISAURUS_MODEL` to a trained FST to run it.
 
 ## Licensing
 

{scriptconv-0.0.4a32 → scriptconv-0.0.4a34}/scriptconv/phonemizers/mul.py

@@ -336,13 +336,19 @@ class EspeakPhonemizer(BasePhonemizer):
             "zh-tw": "cmn", "zh-hant": "cmn", "zh-hant-tw": "cmn",
             "zh-hk": "yue", "zh-mo": "yue",
         }
-        if target_lang.lower() in ZH_ALIASES:
-            return ZH_ALIASES[target_lang.lower()]
-        if target_lang in cls.ESPEAK_LANGS:
-            return target_lang
-        if target_lang.lower().split("-")[0] in cls.ESPEAK_LANGS:
-            return target_lang.lower().split("-")[0]
-        return cls.match_lang(target_lang, cls.ESPEAK_LANGS)
+        # ESPEAK_LANGS is all lowercase, so the tag is lowercased once and
+        # matched as a whole BEFORE the primary subtag. A BCP-47 tag such as
+        # pt-BR or fr-BE used to miss the exact check on case and fall to the
+        # bare pt or fr, which are European Portuguese and metropolitan
+        # French: a Brazilian voice phonemized as European.
+        tag = target_lang.lower().replace("_", "-")
+        if tag in ZH_ALIASES:
+            return ZH_ALIASES[tag]
+        if tag in cls.ESPEAK_LANGS:
+            return tag
+        if tag.split("-")[0] in cls.ESPEAK_LANGS:
+            return tag.split("-")[0]
+        return cls.match_lang(tag, cls.ESPEAK_LANGS)
 
     @staticmethod
     def _run_espeak_command(args: List[str], input_text: str = None, check: bool = True) -> str:
@@ -1487,9 +1493,13 @@ class PhonetisaurusPhonemizer(BasePhonemizer):
     * There is no language check and no ``LANG_DEFAULTS`` entry. A wrapper that
       cannot know its own model's language must not be any language's default.
 
-    ``phonetisaurus`` ships the binaries it drives, so no system package is
-    needed. Training a model needs the bundled ``estimate-ngram``, which is not
-    exercised here: this wrapper only applies a model.
+    ``phonetisaurus`` ships the binaries it drives and the shared objects they
+    link against, and ``predict`` runs them under the ``PATH`` and
+    ``LD_LIBRARY_PATH`` from ``phonetisaurus.guess_environment()``. The apply
+    path needs no system package. Training is the exception: ``estimate-ngram``
+    also needs ``libquadmath.so.0`` from the host GCC runtime (``libquadmath0``
+    on Debian and Ubuntu). Training is not exercised here: this wrapper only
+    applies a model.
     """
 
     #: Where published FSTs come from. Documentation, never a download.

{scriptconv-0.0.4a32 → scriptconv-0.0.4a34}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 32
+VERSION_ALPHA = 34
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a32 → scriptconv-0.0.4a34/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a32
+Version: 0.0.4a34
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -483,6 +483,39 @@ The core installs with zero dependencies. Capabilities opt in:
 | `tashkeel` | Arabic diacritization for the phonemizer pipeline (text2tashkeel) |
 | `stress` | Word-stress restoration for 26 stressonnx language tags: East Slavic; Bulgarian, Macedonian, and Slovene; Latvian; Armenian; Georgian; and Turkic/Caucasian, for the phonemizer pipeline (stressonnx; not yet on PyPI) |
 | `pt` | European-Portuguese heterophonic-homograph sense diacritics for the phonemizer pipeline (bifonia) |
+| `phonetisaurus` | WFST grapheme-to-phoneme through a trained FST the caller supplies (phonetisaurus). Training also needs `libquadmath0` on the host: see below |
+
+### The `phonetisaurus` extra and its libraries
+
+`pip install scriptconv[phonetisaurus]` completes, and the `phonetisaurus`
+0.3.0 wheel bundles its own executables and most of its own shared objects
+(`libfst.so.13`, `libmitlm.so.1`, `libgfortran.so.3` and the OpenFST
+companions) under `phonetisaurus/lib/<machine>`. The binaries find them only
+through the `PATH` and `LD_LIBRARY_PATH` that `phonetisaurus.guess_environment()`
+builds, which is the environment `phonetisaurus.predict` and
+`phonetisaurus.train` pass to every call. A binary started with the plain
+shell environment exits 127 from the dynamic loader; that is a report about
+the caller, not about the box.
+
+The apply path therefore runs out of the box, and this wrapper only applies a
+model. One object is NOT bundled:
+
+| Missing object | Needed by | Where it comes from |
+|---|---|---|
+| `libquadmath.so.0` | `estimate-ngram`, and so `phonetisaurus.train` | the GCC runtime: package `libquadmath0` on Debian and Ubuntu |
+
+So a box without `libquadmath0` can phonemize with a model it is given, and
+cannot train one.
+
+`model=` is required and scriptconv never downloads: see `MODEL_SOURCES` on
+the class, or train a model with `phonetisaurus.train` from a lexicon such as
+CMUdict.
+
+`tests/test_phonemizers_phonetisaurus_engine.py` is the real-engine cell. It
+runs the same word through the engine at `nbest=1` and `nbest=2` and checks
+the wrapper returns the engine's own best-ranked line both times. It skips,
+and names the loader error or the missing model, when the engine cannot run.
+Set `SCRIPTCONV_PHONETISAURUS_MODEL` to a trained FST to run it.
 
 ## Licensing
 

{scriptconv-0.0.4a32 → scriptconv-0.0.4a34}/scriptconv.egg-info/SOURCES.txt

@@ -503,6 +503,7 @@ tests/test_conventions.py
 tests/test_diacritics.py
 tests/test_diacritics_graph.py
 tests/test_errors_policy.py
+tests/test_espeak_get_lang_case.py
 tests/test_examples.py
 tests/test_graph.py
 tests/test_he_extra_declares_phonikud.py
@@ -515,6 +516,7 @@ tests/test_phonemizers_ghana.py
 tests/test_phonemizers_halabi_iqra.py
 tests/test_phonemizers_matoub_kab.py
 tests/test_phonemizers_phonetisaurus.py
+tests/test_phonemizers_phonetisaurus_engine.py
 tests/test_phonemizers_pt_tugaphone.py
 tests/test_phonemizers_ru.py
 tests/test_readings.py

scriptconv-0.0.4a34/tests/test_espeak_get_lang_case.py

@@ -0,0 +1,58 @@
+"""EspeakPhonemizer.get_lang reads a BCP-47 tag whatever its case (T-2783).
+
+``ESPEAK_LANGS`` is all lowercase. The exact-match check used the tag as
+written, so ``pt-BR`` missed it and fell to the bare primary subtag ``pt``,
+which is European Portuguese: a phoonnx Brazilian voice phonemized as
+European. ``fr-BE`` fell to metropolitan ``fr`` the same way.
+
+These tests need no espeak binary: ``get_lang`` is a classmethod over the
+static voice list.
+"""
+import pytest
+
+from scriptconv.phonemizers.mul import EspeakPhonemizer
+
+
+@pytest.mark.parametrize("tag, voice", [
+    ("pt-BR", "pt-br"),
+    ("PT-BR", "pt-br"),
+    ("pt_BR", "pt-br"),
+    ("fr-BE", "fr-be"),
+    ("fr-CH", "fr-ch"),
+    ("ES-419", "es-419"),
+    ("vi-VN-x-central", "vi-vn-x-central"),
+    ("vi-VN-x-south", "vi-vn-x-south"),
+])
+def test_a_region_tag_keeps_its_region_whatever_its_case(tag, voice):
+    assert EspeakPhonemizer.get_lang(tag) == voice
+
+
+@pytest.mark.parametrize("tag, voice", [
+    ("pt-br", "pt-br"),
+    ("fr-be", "fr-be"),
+    ("en-us", "en-us"),
+    ("pt", "pt"),
+    ("fr", "fr"),
+])
+def test_lowercase_tags_are_unchanged(tag, voice):
+    assert EspeakPhonemizer.get_lang(tag) == voice
+
+
+def test_the_bare_language_is_not_silently_substituted_for_a_region():
+    # the defect: pt-BR must never resolve to European pt while pt-br exists
+    assert EspeakPhonemizer.get_lang("pt-BR") != "pt"
+    assert EspeakPhonemizer.get_lang("fr-BE") != "fr"
+
+
+@pytest.mark.parametrize("tag, voice", [
+    ("EN-GB", "en-gb-x-rp"),
+    ("zh-CN", "cmn"),
+    ("ZH-HK", "yue"),
+])
+def test_the_existing_aliases_still_apply_in_any_case(tag, voice):
+    assert EspeakPhonemizer.get_lang(tag) == voice
+
+
+def test_an_unknown_tag_still_raises():
+    with pytest.raises(ValueError):
+        EspeakPhonemizer.get_lang("xx")

scriptconv-0.0.4a34/tests/test_phonemizers_phonetisaurus_engine.py

@@ -0,0 +1,138 @@
+"""The Phonetisaurus wrapper against the REAL engine.
+
+``tests/test_phonemizers_phonetisaurus.py`` stubs ``phonetisaurus.predict``,
+so it proves what the wrapper does with an answer and never that the engine
+gives one. This file is the other half.
+
+The engine runs from inside the wheel: the binaries and their shared objects
+ship with the package, and ``phonetisaurus.guess_environment()`` builds the
+``PATH`` and ``LD_LIBRARY_PATH`` that reach them. This file resolves the
+binaries the same way, because a binary started with the plain shell
+environment exits 127 from the dynamic loader and would make a working box
+read as a broken one.
+
+What is not shipped is a model, and scriptconv never downloads. So the cell
+SKIPS, with the loader error or the missing model named, unless
+``SCRIPTCONV_PHONETISAURUS_MODEL`` points at a trained FST. With one, it
+measures two things the stub cannot:
+
+* the wrapper's answer equals ``phonetisaurus-apply``'s own FIRST output line
+  for the same word and the same model, and
+* ``nbest=2`` returns the same string as ``nbest=1``. ``nbest`` widens the
+  search and prints more ranked lines; the wrapper keeps the best one. A
+  wrapper that kept the last line would differ here, and the stub suite
+  proves that only against a stub.
+"""
+import os
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+MODEL = os.environ.get("SCRIPTCONV_PHONETISAURUS_MODEL")
+WORD = os.environ.get("SCRIPTCONV_PHONETISAURUS_WORD", "hello")
+
+
+def _bin(name, env):
+    """Resolve a bundled executable the way the engine does: from the PATH
+    ``phonetisaurus.guess_environment`` builds, not the caller's PATH. The
+    wheel keeps its binaries inside the package."""
+    return shutil.which(name, path=env["PATH"])
+
+
+def _engine_state():
+    """Return None when the engine can run, or the reason it cannot."""
+    try:
+        import phonetisaurus
+    except ImportError as e:
+        return f"phonetisaurus is not installed: {e}"
+    env = phonetisaurus.guess_environment()
+    if not _bin("phonetisaurus-apply", env):
+        return "phonetisaurus-apply is not in the package bin directory"
+    g2p_bin = _bin("phonetisaurus-g2pfst", env)
+    if not g2p_bin:
+        return "phonetisaurus-g2pfst is not in the package bin directory"
+    # phonetisaurus-apply is a Python script and always starts. The binaries
+    # it drives are the ones that fail to load, so probe one of THEM.
+    proc = subprocess.run([g2p_bin, "--help"], capture_output=True, text=True,
+                          env=env)
+    if proc.returncode == 127:
+        return (f"phonetisaurus-g2pfst exits 127, the dynamic loader says: "
+                f"{proc.stderr.strip() or proc.stdout.strip()}")
+    if not MODEL:
+        return ("no trained FST: set SCRIPTCONV_PHONETISAURUS_MODEL to a "
+                "model path (scriptconv never downloads one)")
+    if not os.path.isfile(MODEL):
+        return f"SCRIPTCONV_PHONETISAURUS_MODEL is not a file: {MODEL}"
+    return None
+
+
+SKIP_REASON = _engine_state()
+
+
+def _apply_first_line(word, nbest):
+    """The best-ranked pronunciation ``phonetisaurus-apply`` prints itself.
+
+    Its output is one tab-separated line per ranked pronunciation, best
+    first. The word is the first field; the rest is the pronunciation.
+    """
+    with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as fh:
+        fh.write(word + "\n")
+        word_list = fh.name
+    try:
+        import phonetisaurus
+        env = phonetisaurus.guess_environment()
+        proc = subprocess.run(
+            [_bin("phonetisaurus-apply", env), "--model", MODEL,
+             "--word_list", word_list, "--nbest", str(nbest)],
+            capture_output=True, text=True, env=env)
+    finally:
+        os.unlink(word_list)
+    if proc.returncode != 0:
+        raise AssertionError(
+            f"phonetisaurus-apply exited {proc.returncode}: {proc.stderr}")
+    lines = [ln for ln in proc.stdout.splitlines() if ln.strip()]
+    assert lines, f"phonetisaurus-apply printed nothing for {word!r}"
+    fields = lines[0].split("\t")
+    return " ".join(fields[1:]).split()
+
+
+@unittest.skipIf(SKIP_REASON is not None, SKIP_REASON or "")
+class TestPhonetisaurusRealEngine(unittest.TestCase):
+
+    def _wrapper(self, nbest):
+        from scriptconv.phonemizers.enums import Alphabet
+        from scriptconv.phonemizers.mul import PhonetisaurusPhonemizer
+        return PhonetisaurusPhonemizer(model=MODEL, alphabet=Alphabet.ARPA,
+                                       nbest=nbest)
+
+    def test_nbest_1_matches_the_engines_own_first_line(self):
+        expected = " ".join(_apply_first_line(WORD, 1))
+        self.assertEqual(self._wrapper(1).phonemize_string(WORD, "en"),
+                         expected)
+
+    def test_nbest_2_returns_the_same_best_answer_as_nbest_1(self):
+        one = self._wrapper(1).phonemize_string(WORD, "en")
+        two = self._wrapper(2).phonemize_string(WORD, "en")
+        self.assertTrue(one)
+        self.assertEqual(one, two)
+
+    def test_nbest_2_makes_the_engine_rank_more_than_one_reading(self):
+        # The proof that nbest reached the engine at all: two ranked lines
+        # where nbest=1 gives one. A model with a single reading for the word
+        # cannot show this, so the cell states that instead of failing.
+        import phonetisaurus
+        pairs = list(phonetisaurus.predict([WORD], MODEL, nbest=2))
+        if len(pairs) < 2:
+            self.skipTest(f"the model ranks one reading only for {WORD!r}")
+        self.assertEqual({p[0] for p in pairs}, {WORD})
+
+
+class TestEngineCellIsOwed(unittest.TestCase):
+    """This one always runs: it records WHY the cell above did not."""
+
+    def test_the_skip_reason_is_reported(self):
+        if SKIP_REASON is None:
+            return
+        self.assertTrue(SKIP_REASON.strip())
+        print(f"\nphonetisaurus real-engine cell skipped: {SKIP_REASON}")