scriptconv 0.0.4a36__tar.gz → 0.0.4a37__tar.gz

{scriptconv-0.0.4a36/scriptconv.egg-info → scriptconv-0.0.4a37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a36
+Version: 0.0.4a37
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -11,13 +11,12 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Text Processing :: Linguistic
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Provides-Extra: ja
@@ -54,7 +53,8 @@ Requires-Dist: onnxruntime; extra == "byt5"
 Requires-Dist: numpy; extra == "byt5"
 Provides-Extra: o2i
 Requires-Dist: scriptconv[phonemizers]; extra == "o2i"
-Requires-Dist: orthography2ipa; extra == "o2i"
+Requires-Dist: orthography2ipa>=7.112.0a1; extra == "o2i"
+Requires-Dist: o2i-lexicons>=0.1.0a1; extra == "o2i"
 Provides-Extra: eu
 Requires-Dist: scriptconv[phonemizers]; extra == "eu"
 Requires-Dist: euskaphone; extra == "eu"
@@ -134,6 +134,8 @@ Requires-Dist: stressonnx>=0.0.3a2; extra == "test"
 Requires-Dist: tugaphone>=1.2.0a1; extra == "test"
 Requires-Dist: ghana-g2p>=0.1.1; extra == "test"
 Requires-Dist: espyak; extra == "test"
+Requires-Dist: orthography2ipa>=7.112.0a1; extra == "test"
+Requires-Dist: o2i-lexicons>=0.1.0a1; extra == "test"
 Requires-Dist: tomli; python_version < "3.11" and extra == "test"
 Dynamic: license-file
 
@@ -391,7 +393,7 @@ g.convert("bom dia", "text", "ipa", lang="pt")     # 'ˈbõ ˈdʒiɐ'
 g.can_convert("text", "arpa")                      # True (chains through IPA)
 ```
 
-Two design points worth knowing:
+Three design points worth knowing:
 
 - **Normalization is injectable.** TTS stacks expand numbers and dates
   before phonemizing, and that needs language resources scriptconv does
@@ -406,6 +408,17 @@ Two design points worth knowing:
   overrides it with a path or callable; the cache location is set via
   `SCRIPTCONV_CACHE` or `XDG_CACHE_HOME`).
 
+- **The orthography2ipa phonemizer uses the packaged lexicons.** The `o2i`
+  extra installs [o2i-lexicons](https://github.com/TigreGotico/o2i-lexicons),
+  which holds the words that the orthography2ipa rules get wrong, such as
+  en-GB ⟨yacht⟩. The key `packaged_lexicons` (default `True`) enables the
+  packaged lexicon for each language that the phonemizer transcribes, when a
+  package serves that language. Set it to `False` to use the rules only:
+  `get_phonemizer(Phonemizer.ORTHOGRAPHY2IPA, packaged_lexicons=False)`.
+  orthography2ipa keeps its lexicons for the whole process. After one caller
+  enables a lexicon, every orthography2ipa caller in the process uses it, and
+  `packaged_lexicons=False` does not disable it again.
+
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)`
 restores information ordinary orthography omits but a G2P needs, before
 phonemization. It covers Hebrew niqqud (phonikud), Arabic tashkeel

{scriptconv-0.0.4a36 → scriptconv-0.0.4a37}/README.md

@@ -252,7 +252,7 @@ g.convert("bom dia", "text", "ipa", lang="pt")     # 'ˈbõ ˈdʒiɐ'
 g.can_convert("text", "arpa")                      # True (chains through IPA)
 ```
 
-Two design points worth knowing:
+Three design points worth knowing:
 
 - **Normalization is injectable.** TTS stacks expand numbers and dates
   before phonemizing, and that needs language resources scriptconv does
@@ -267,6 +267,17 @@ Two design points worth knowing:
   overrides it with a path or callable; the cache location is set via
   `SCRIPTCONV_CACHE` or `XDG_CACHE_HOME`).
 
+- **The orthography2ipa phonemizer uses the packaged lexicons.** The `o2i`
+  extra installs [o2i-lexicons](https://github.com/TigreGotico/o2i-lexicons),
+  which holds the words that the orthography2ipa rules get wrong, such as
+  en-GB ⟨yacht⟩. The key `packaged_lexicons` (default `True`) enables the
+  packaged lexicon for each language that the phonemizer transcribes, when a
+  package serves that language. Set it to `False` to use the rules only:
+  `get_phonemizer(Phonemizer.ORTHOGRAPHY2IPA, packaged_lexicons=False)`.
+  orthography2ipa keeps its lexicons for the whole process. After one caller
+  enables a lexicon, every orthography2ipa caller in the process uses it, and
+  `packaged_lexicons=False` does not disable it again.
+
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)`
 restores information ordinary orthography omits but a G2P needs, before
 phonemization. It covers Hebrew niqqud (phonikud), Arabic tashkeel

{scriptconv-0.0.4a36 → scriptconv-0.0.4a37}/pyproject.toml

@@ -8,14 +8,13 @@ dynamic = ["version", "dependencies"]
 description = "Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana"
 readme = "README.md"
 license = {text = "Apache-2.0"}
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "Topic :: Text Processing :: Linguistic",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -45,7 +44,7 @@ transphone = ["scriptconv[phonemizers]", "transphone"]
 phonetisaurus = ["scriptconv[phonemizers]", "phonetisaurus"]
 misaki = ["scriptconv[phonemizers]", "misaki"]
 byt5 = ["scriptconv[phonemizers]", "onnxruntime", "numpy"]
-o2i = ["scriptconv[phonemizers]", "orthography2ipa"]
+o2i = ["scriptconv[phonemizers]", "orthography2ipa>=7.112.0a1", "o2i-lexicons>=0.1.0a1"]
 eu = ["scriptconv[phonemizers]", "euskaphone", "ahotts_g2p"]
 pt-phonemizers = ["scriptconv[phonemizers]", "tugaphone>=1.2.0a1", "g2p_barranquenho"]
 mwl = ["scriptconv[phonemizers]", "mwl_phonemizer"]
@@ -63,7 +62,7 @@ zh-phonemizers = ["scriptconv[phonemizers]", "jieba", "pypinyin>=0.50,<1", "xpin
 en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer", "requests"]
 ar-phonemizers = ["scriptconv[phonemizers]", "arbtok", "ovos-number-parser"]
 shami = ["scriptconv[phonemizers]", "regex"]
-test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1", "stressonnx>=0.0.3a2", "tugaphone>=1.2.0a1", "ghana-g2p>=0.1.1", "espyak", "tomli; python_version < '3.11'"]
+test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1", "stressonnx>=0.0.3a2", "tugaphone>=1.2.0a1", "ghana-g2p>=0.1.1", "espyak", "orthography2ipa>=7.112.0a1", "o2i-lexicons>=0.1.0a1", "tomli; python_version < '3.11'"]
 
 [project.urls]
 Homepage = "https://github.com/TigreGotico/scriptconv"

{scriptconv-0.0.4a36 → scriptconv-0.0.4a37}/scriptconv/phonemizers/o2ipa.py

@@ -12,6 +12,13 @@ diacritizer on top of this same lattice); for dialect-aware Basque use
 
 ``orthography2ipa`` is imported lazily so importing ``scriptconv`` never requires
 it; install with ``pip install orthography2ipa`` (or ``pip install scriptconv[o2i]``).
+
+With ``packaged_lexicons=True`` (the default) the phonemizer enables the
+installed packaged lexicon (``o2i-lexicons``) for each language it transcribes,
+when a package serves that language. orthography2ipa keeps its lexicons
+process-wide, so an enabled lexicon serves every orthography2ipa caller in the
+process, and ``packaged_lexicons=False`` does not disable one that another
+caller enabled.
 """
 from typing import Dict, List
 
@@ -29,14 +36,18 @@ class Orthography2IPAPhonemizer(BasePhonemizer):
     lazily on first use and cached for the lifetime of the instance.
     """
 
-    def __init__(self):
+    def __init__(self, packaged_lexicons: bool = True):
         super().__init__(alphabet=Alphabet.IPA)
+        self.packaged_lexicons = packaged_lexicons
         self._cache: Dict[str, object] = {}
 
     def _engine(self, resolved_lang: str):
         """Return, lazily creating and caching, the G2P engine for *resolved_lang*."""
         if resolved_lang not in self._cache:
             import orthography2ipa
+            if (self.packaged_lexicons and resolved_lang
+                    in orthography2ipa.packaged_lexicon_sources()):
+                orthography2ipa.use_packaged_lexicons([resolved_lang])
             self._cache[resolved_lang] = orthography2ipa.G2P(resolved_lang)
         return self._cache[resolved_lang]
 

{scriptconv-0.0.4a36 → scriptconv-0.0.4a37}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 4
-VERSION_ALPHA = 36
+VERSION_ALPHA = 37
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.4a36 → scriptconv-0.0.4a37/scriptconv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a36
+Version: 0.0.4a37
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -11,13 +11,12 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Text Processing :: Linguistic
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Provides-Extra: ja
@@ -54,7 +53,8 @@ Requires-Dist: onnxruntime; extra == "byt5"
 Requires-Dist: numpy; extra == "byt5"
 Provides-Extra: o2i
 Requires-Dist: scriptconv[phonemizers]; extra == "o2i"
-Requires-Dist: orthography2ipa; extra == "o2i"
+Requires-Dist: orthography2ipa>=7.112.0a1; extra == "o2i"
+Requires-Dist: o2i-lexicons>=0.1.0a1; extra == "o2i"
 Provides-Extra: eu
 Requires-Dist: scriptconv[phonemizers]; extra == "eu"
 Requires-Dist: euskaphone; extra == "eu"
@@ -134,6 +134,8 @@ Requires-Dist: stressonnx>=0.0.3a2; extra == "test"
 Requires-Dist: tugaphone>=1.2.0a1; extra == "test"
 Requires-Dist: ghana-g2p>=0.1.1; extra == "test"
 Requires-Dist: espyak; extra == "test"
+Requires-Dist: orthography2ipa>=7.112.0a1; extra == "test"
+Requires-Dist: o2i-lexicons>=0.1.0a1; extra == "test"
 Requires-Dist: tomli; python_version < "3.11" and extra == "test"
 Dynamic: license-file
 
@@ -391,7 +393,7 @@ g.convert("bom dia", "text", "ipa", lang="pt")     # 'ˈbõ ˈdʒiɐ'
 g.can_convert("text", "arpa")                      # True (chains through IPA)
 ```
 
-Two design points worth knowing:
+Three design points worth knowing:
 
 - **Normalization is injectable.** TTS stacks expand numbers and dates
   before phonemizing, and that needs language resources scriptconv does
@@ -406,6 +408,17 @@ Two design points worth knowing:
   overrides it with a path or callable; the cache location is set via
   `SCRIPTCONV_CACHE` or `XDG_CACHE_HOME`).
 
+- **The orthography2ipa phonemizer uses the packaged lexicons.** The `o2i`
+  extra installs [o2i-lexicons](https://github.com/TigreGotico/o2i-lexicons),
+  which holds the words that the orthography2ipa rules get wrong, such as
+  en-GB ⟨yacht⟩. The key `packaged_lexicons` (default `True`) enables the
+  packaged lexicon for each language that the phonemizer transcribes, when a
+  package serves that language. Set it to `False` to use the rules only:
+  `get_phonemizer(Phonemizer.ORTHOGRAPHY2IPA, packaged_lexicons=False)`.
+  orthography2ipa keeps its lexicons for the whole process. After one caller
+  enables a lexicon, every orthography2ipa caller in the process uses it, and
+  `packaged_lexicons=False` does not disable it again.
+
 **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)`
 restores information ordinary orthography omits but a G2P needs, before
 phonemization. It covers Hebrew niqqud (phonikud), Arabic tashkeel

{scriptconv-0.0.4a36 → scriptconv-0.0.4a37}/scriptconv.egg-info/SOURCES.txt

@@ -515,6 +515,7 @@ tests/test_phonemizers_friendly_import_errors.py
 tests/test_phonemizers_ghana.py
 tests/test_phonemizers_halabi_iqra.py
 tests/test_phonemizers_matoub_kab.py
+tests/test_phonemizers_o2i_lexicons.py
 tests/test_phonemizers_phonetisaurus.py
 tests/test_phonemizers_phonetisaurus_engine.py
 tests/test_phonemizers_pt_tugaphone.py

{scriptconv-0.0.4a36 → scriptconv-0.0.4a37}/scriptconv.egg-info/requires.txt

@@ -78,7 +78,8 @@ mwl_phonemizer
 
 [o2i]
 scriptconv[phonemizers]
-orthography2ipa
+orthography2ipa>=7.112.0a1
+o2i-lexicons>=0.1.0a1
 
 [phonemizers]
 quebra-frases
@@ -123,6 +124,8 @@ stressonnx>=0.0.3a2
 tugaphone>=1.2.0a1
 ghana-g2p>=0.1.1
 espyak
+orthography2ipa>=7.112.0a1
+o2i-lexicons>=0.1.0a1
 
 [test:python_version < "3.11"]
 tomli

scriptconv-0.0.4a37/tests/test_phonemizers_o2i_lexicons.py

@@ -0,0 +1,68 @@
+"""The orthography2ipa phonemizer enables the packaged o2i-lexicons by default.
+
+The expected IPA is read from the o2i-lexicons data file, not from engine
+output. orthography2ipa keeps lexicons process-wide, so every test starts and
+ends with ``clear_lexicons()``.
+"""
+import unittest
+
+import orthography2ipa
+from o2i_lexicons import DATA_DIR
+
+from scriptconv.phonemizers.o2ipa import Orthography2IPAPhonemizer
+
+STRESS = str.maketrans("", "", "ˈˌ")
+
+
+def lexicon_ipa(code: str, word: str) -> str:
+    """The IPA that the packaged TSV file for *code* lists for *word*."""
+    for path in sorted((DATA_DIR / code).glob("*.tsv")):
+        for line in path.read_text(encoding="utf-8").splitlines():
+            entry, _, ipa = line.partition("\t")
+            if entry == word:
+                return ipa
+    raise LookupError(f"{word!r} is not in the packaged {code} lexicon")
+
+
+class TestPackagedLexicons(unittest.TestCase):
+    def setUp(self):
+        orthography2ipa.clear_lexicons()
+
+    def tearDown(self):
+        orthography2ipa.clear_lexicons()
+
+    def test_default_uses_the_packaged_lexicon(self):
+        expected = lexicon_ipa("en-GB", "yacht")
+        out = Orthography2IPAPhonemizer().phonemize_string("yacht", "en-GB")
+        self.assertEqual(out.translate(STRESS), expected)
+
+    def test_bare_english_tag_uses_the_packaged_lexicon(self):
+        expected = lexicon_ipa("en-GB", "yacht")
+        out = Orthography2IPAPhonemizer().phonemize_string("yacht", "en")
+        self.assertEqual(out.translate(STRESS), expected)
+
+    def test_off_equals_plain_orthography2ipa(self):
+        out = Orthography2IPAPhonemizer(packaged_lexicons=False) \
+            .phonemize_string("yacht", "en-GB")
+        self.assertEqual(out, orthography2ipa.G2P("en-GB").transcribe("yacht"))
+        self.assertNotEqual(out.translate(STRESS), lexicon_ipa("en-GB", "yacht"))
+        self.assertEqual(orthography2ipa.available_lexicon_codes(), [])
+
+    def test_enables_only_the_languages_it_transcribes(self):
+        Orthography2IPAPhonemizer().phonemize_string("yacht", "en-GB")
+        self.assertEqual(orthography2ipa.available_lexicon_codes(), ["en-GB"])
+
+    def test_language_without_a_packaged_lexicon_uses_the_rules(self):
+        out = Orthography2IPAPhonemizer().phonemize_string("Kaixo", "eu")
+        self.assertEqual(out, orthography2ipa.G2P("eu").transcribe("Kaixo"))
+        self.assertEqual(orthography2ipa.available_lexicon_codes(), [])
+
+    def test_registry_forwards_the_key(self):
+        from scriptconv.phonemizers import Phonemizer, get_phonemizer
+        off = get_phonemizer(Phonemizer.ORTHOGRAPHY2IPA, packaged_lexicons=False)
+        self.assertFalse(off.packaged_lexicons)
+        self.assertTrue(get_phonemizer(Phonemizer.ORTHOGRAPHY2IPA).packaged_lexicons)
+
+
+if __name__ == "__main__":
+    unittest.main()