scriptconv 0.0.4a4__tar.gz → 0.0.4a6__tar.gz

{scriptconv-0.0.4a4/scriptconv.egg-info → scriptconv-0.0.4a6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.4a4
+Version: 0.0.4a6
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -105,6 +105,7 @@ Requires-Dist: scriptconv[phonemizers]; extra == "en-phonemizers"
 Requires-Dist: g2p_en; extra == "en-phonemizers"
 Requires-Dist: openphonemizer; extra == "en-phonemizers"
 Requires-Dist: deep-phonemizer; extra == "en-phonemizers"
+Requires-Dist: requests; extra == "en-phonemizers"
 Provides-Extra: ar-phonemizers
 Requires-Dist: scriptconv[phonemizers]; extra == "ar-phonemizers"
 Requires-Dist: arbtok; extra == "ar-phonemizers"

{scriptconv-0.0.4a4 → scriptconv-0.0.4a6}/pyproject.toml

@@ -58,7 +58,7 @@ pt = ["scriptconv[phonemizers]", "bifonia>=0.1.1"]
 ja-phonemizers = ["scriptconv[phonemizers]", "pyopenjtalk", "cutlet", "pykakasi>=2.3,<3"]
 ko = ["scriptconv[phonemizers]", "g2pk", "regex"]
 zh-phonemizers = ["scriptconv[phonemizers]", "jieba", "pypinyin>=0.50,<1", "xpinyin", "g2pM", "pinyin_to_ipa"]
-en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer"]
+en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer", "requests"]
 ar-phonemizers = ["scriptconv[phonemizers]", "arbtok", "ovos-number-parser"]
 shami = ["scriptconv[phonemizers]", "regex"]
 test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1"]

{scriptconv-0.0.4a4 → scriptconv-0.0.4a6}/scriptconv/__init__.py

@@ -69,6 +69,7 @@ from scriptconv.notation import (
     ipa_to_cotovia,
     rfe_to_ipa,
     ipa_to_rfe,
+    mantoq_to_ipa,
     looks_like_ipa,
 )
 from scriptconv.translit import (
@@ -140,6 +141,7 @@ __all__ = [
     "ipa_to_cotovia",
     "rfe_to_ipa",
     "ipa_to_rfe",
+    "mantoq_to_ipa",
     "looks_like_ipa",
     # translit
     "decompose_hangul",

{scriptconv-0.0.4a4 → scriptconv-0.0.4a6}/scriptconv/cangjie.py

@@ -15,7 +15,7 @@ from __future__ import annotations
 
 import gzip
 from importlib.resources import files
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 __all__ = ["cangjie_code", "to_cangjie"]
 
@@ -49,15 +49,16 @@ def to_cangjie(text: str, sep: str = " ") -> str:
     """
     table = _table()
     tokens = []
-    last_raw = False
+    raw_run: List[str] = []
     for ch in text:
         code = table.get(ch)
         if code is not None:
+            if raw_run:
+                tokens.append("".join(raw_run))
+                raw_run = []
             tokens.append(code)
-            last_raw = False
-        elif last_raw:
-            tokens[-1] += ch
         else:
-            tokens.append(ch)
-            last_raw = True
+            raw_run.append(ch)
+    if raw_run:
+        tokens.append("".join(raw_run))
     return sep.join(tokens)

scriptconv-0.0.4a6/scriptconv/diacritics.py

@@ -0,0 +1,319 @@
+"""Diacritization: mechanism, dispatch, and graph extension.
+
+This module OWNS diacritization — lang→backend routing, lazy/cached backend
+loading (phonikud, text2tashkeel, stressonnx, bifonia), and the overlay-strip
+helpers. :mod:`scriptconv.phonemizers` (``BasePhonemizer`` and subclasses)
+knows nothing about diacritics: phonemization (orthography → sound) and
+diacritization (a text → text graph transform disambiguating pronunciation
+before G2P) are separate concerns.
+
+Diacritization (Arabic tashkeel, Hebrew niqqud, East-Slavic/Turkic/Caucasian
+word stress, European-Portuguese homograph sense marks) is, architecturally,
+just another transform between text representations: it maps the ``"text"``
+node to a lang-contextual ``"text-diacritized"`` node.  Like
+:func:`scriptconv.phonemizers.register` it is opt-in — ``DEFAULT_GRAPH`` stays
+free of it until a caller extends a graph::
+
+    from scriptconv.graph import DEFAULT_GRAPH
+    from scriptconv import diacritics, phonemizers
+    g = DEFAULT_GRAPH.extend(diacritics.register).extend(phonemizers.register)
+
+    g.convert("замок стоит на горе", "text", "text-diacritized", lang="ru")
+    # 'за́мок сто́ит на горе́'    — just the diacritized text
+
+    g.convert("Tenho muita sede hoje.", "text", "text-diacritized", lang="pt")
+    # 'Tenho muita sêde hoje.'
+
+Routing ``"text" -> "ipa"`` still takes the direct phonemization edge by
+default: the diacritization edge is model-based (``lossless=False``), so it
+never out-prices a direct phonemization, and enabling this extension does not
+silently change phonemization output.  The diacritized route is taken only
+when a caller asks for ``"text-diacritized"`` explicitly.
+
+A future per-engine *stance* would push this further into topology rather than
+a flag: an engine that *requires* vocalized input would carry only a
+``"text-diacritized" -> "ipa"`` edge (forcing the detour), one that *forbids*
+diacritics would carry only ``"text" -> "ipa"`` (making the detour
+unroutable), and a tolerant engine (e.g. arbtok, which self-vocalizes) carries
+both.  The strip direction (``"text-diacritized" -> "text"``) is registered too, but
+gated to languages whose marks are removable *overlay* diacritics —
+Arabic/Hebrew vocalization and East-Slavic/Turkic/Caucasian stress. Strip
+removes only the specific overlay codepoints each backend adds (combining
+acute/grave for stress, tashkeel for Arabic, niqqud for Hebrew) — never a
+blanket combining-mark filter — so precomposed native letters survive
+(Cyrillic й/ё, Latvian macrons, Azerbaijani ç/ö, Arabic hamza carriers أ إ آ ؤ
+ئ). The gate uses exact primary-subtag matching, so Aragonese (``arg``),
+Herero (``her``), Mapudungun (``arn``) etc. are never misread as Arabic/Hebrew.
+Languages whose diacritics are native orthography (European Portuguese, via
+bifonia) refuse the strip with :class:`ValueError` instead of silently
+corrupting the spelling (``café`` must never become ``cafe``). This split
+tracks which backend/model produced the marks: overlay backends (phonikud,
+tashkeel, stressonnx) are strippable; spelling-integral backends (bifonia) are
+not.
+"""
+from typing import Optional
+
+from scriptconv.graph import Edge
+from scriptconv.phonemizers.base import _primary_subtag
+
+#: The lang-contextual node produced by diacritization.  Like ``"text"`` it is
+#: meaningful only with ``lang=`` context and exists only in opted-in graphs.
+DIACRITIZED = "text-diacritized"
+
+# Across East Slavic, Bulgarian/Macedonian/Slovene, Latvian, Armenian,
+# Georgian, and several Turkic/Caucasian languages, lexical word stress is
+# free (not fixed to a syllable) and ordinary orthography leaves it unwritten
+# or under-marked. The clearest case is East Slavic: stress is also mobile
+# (it shifts between forms of the same word) and unstressed vowels *reduce*
+# — Russian unstressed "о" surfaces as [ɐ] or [ə] depending on distance from
+# the stress, not [o] — so a wrong or missing mark there corrupts the vowel
+# quality of the whole word, not just its prosody. Other families in this set
+# don't necessarily reduce vowels, but still need the mark for correct stress
+# placement and prosody. stressonnx restores it as a combining acute (U+0301)
+# after the stressed vowel, covering 26 BCP-47 tags across these families
+# (24 primary subtags; Azerbaijani and Uzbek each have Cyrillic/Latin script
+# variants routed by the full tag).
+STRESS_LANGS = {
+    "az", "ba", "be", "bg", "cv", "hy", "ka", "kbd", "kjh", "kk", "ky", "lv",
+    "mdf", "mk", "myv", "ru", "sah", "sl", "tg", "tt", "udm", "uk", "uz", "xal",
+}
+
+
+def _is_european_portuguese(lang: str) -> bool:
+    """True for European Portuguese (``pt`` or a ``pt-PT`` region tag).
+
+    False for Brazilian Portuguese (``pt-BR``) and everything else — the two
+    varieties' vowel systems differ, and bifonia's open/closed diacritics are
+    only valid for European Portuguese phonology.
+    """
+    norm = lang.lower().replace("_", "-")
+    return norm == "pt" or norm == "pt-pt"
+
+
+def _diacritizer_family(lang: str) -> Optional[str]:
+    """Which diacritization backend family handles *lang*, or ``None``.
+
+    Single source of truth for lang→backend routing: both the forward
+    dispatch (:func:`diacritize`) and the strip direction (:func:`_overlay_marks`)
+    resolve through this, so the two can never disagree about which language
+    uses which backend. Returns one of ``"he"`` (niqqud), ``"ar"`` (tashkeel),
+    ``"stress"`` (stressonnx), ``"pt"`` (bifonia sense diacritics), or
+    ``None``. Uses exact primary-subtag matching (never ``startswith``), so
+    Aragonese (``arg``), Herero (``her``) and Mapudungun (``arn``) are never
+    misread as Arabic/Hebrew.
+    """
+    p = _primary_subtag(lang)
+    if p == "he":
+        return "he"
+    if p == "ar":
+        return "ar"
+    if p in STRESS_LANGS:
+        return "stress"
+    if _is_european_portuguese(lang):
+        return "pt"
+    return None
+
+
+_DEFAULT_DIACRITIZER_MODEL = "rawi-ensemble"
+
+_PHONIKUD_CACHE: dict = {}   # resolved model path/string -> Phonikud instance
+_TASHKEEL_CACHE: dict = {}  # model name -> text2tashkeel Diacritizer
+
+
+def _phonikud(phonikud_model):
+    """Lazily build (and cache) the phonikud Phonikud instance used for Hebrew.
+
+    ``phonikud_model`` is a local path to a phonikud ONNX model, or a
+    zero-arg callable resolving one lazily; scriptconv never downloads —
+    the consumer resolves the file. Install with ``pip install
+    scriptconv[he]`` (or ``pip install phonikud-onnx``)."""
+    model = phonikud_model() if callable(phonikud_model) else phonikud_model
+    if not model:
+        raise ValueError(
+            "Hebrew diacritization needs a local phonikud ONNX model: "
+            "pass phonikud_model=<path> (scriptconv never downloads "
+            "models; obtain one from the phonikud-onnx release)")
+    if model not in _PHONIKUD_CACHE:
+        try:
+            from phonikud_onnx import Phonikud
+        except ImportError:
+            raise ImportError(
+                "Hebrew diacritization needs phonikud-onnx — install "
+                "with `pip install scriptconv[he]`") from None
+        _PHONIKUD_CACHE[model] = Phonikud(model)
+    return _PHONIKUD_CACHE[model]
+
+
+def _tashkeel(model: Optional[str] = None):
+    """Lazily build (and cache) the text2tashkeel Diacritizer used for Arabic.
+
+    text2tashkeel is a dependency of the ``[ar]`` extra; it restores hamza and the
+    dagger alef in addition to the standard marks. Install with
+    ``pip install scriptconv[tashkeel]`` (or ``pip install text2tashkeel``)."""
+    model = model or _DEFAULT_DIACRITIZER_MODEL
+    if model not in _TASHKEEL_CACHE:
+        try:
+            from text2tashkeel import Diacritizer
+        except ImportError as e:
+            raise ImportError(
+                "Arabic diacritization requires the text2tashkeel package: "
+                "pip install scriptconv[tashkeel]  (or pip install text2tashkeel)"
+            ) from e
+        _TASHKEEL_CACHE[model] = Diacritizer(model)
+    return _TASHKEEL_CACHE[model]
+
+
+def _stress(text: str, lang: str, model: Optional[str] = None) -> str:
+    """Word-stress restoration via stressonnx, for the 26 language tags
+    it covers (see ``STRESS_LANGS``) — East Slavic, Bulgarian/Macedonian/
+    Slovene, Latvian, Armenian, Georgian, and Turkic/Caucasian languages.
+
+    stressonnx is not on PyPI yet; install straight from source. Install
+    with ``pip install scriptconv[stress]`` (or ``pip install
+    stressonnx``)."""
+    try:
+        from stressonnx import stress
+    except ImportError as e:
+        raise ImportError(
+            "stress restoration requires the stressonnx package: "
+            "pip install scriptconv[stress]  (or pip install stressonnx)"
+        ) from e
+    return stress(text, lang, model=model)
+
+
+def _sense_diacritics_pt(text: str) -> str:
+    """European-Portuguese heterophonic-homograph sense diacritics via bifonia.
+
+    Rewrites homographs whose pronunciation depends on meaning (e.g.
+    "sede" thirst/closed vs. seat/open) with an explicit open/closed
+    vowel diacritic. These are ordinary Portuguese orthographic marks,
+    chosen so any downstream G2P — rule-based, neural, or espeak —
+    reads them correctly. Install with ``pip install scriptconv[pt]``
+    (or ``pip install bifonia``)."""
+    try:
+        from bifonia import add_extra_diacritics
+    except ImportError as e:
+        raise ImportError(
+            "European-Portuguese sense diacritics require the bifonia package: "
+            "pip install scriptconv[pt]  (or pip install bifonia)"
+        ) from e
+    return add_extra_diacritics(text)
+
+
+def diacritize(text: str, lang: str = "und", model=None,
+               phonikud_model=None, diacritizer_model=None, **_) -> str:
+    """Add pronunciation-disambiguating diacritics to *text* for *lang*.
+
+    Four backends, each restoring information ordinary orthography omits but
+    downstream G2P needs:
+
+    - Hebrew (``he``) — niqqud via phonikud (``phonikud_model=``).
+    - Arabic (``ar``) — tashkeel via text2tashkeel (``[tashkeel]``,
+      ``model=``/``diacritizer_model=``).
+    - East Slavic, Bulgarian/Macedonian/Slovene, Latvian, Armenian, Georgian,
+      and Turkic/Caucasian languages (``STRESS_LANGS``, 26 stressonnx tags) —
+      word stress via stressonnx (``[stress]``); stress is unwritten or
+      under-marked in these languages, and in East Slavic unstressed vowels
+      also reduce, so a missing mark can corrupt more than prosody.
+    - European Portuguese (``pt``/``pt-PT``, never ``pt-BR``) —
+      heterophonic-homograph sense diacritics via bifonia (``[pt]``);
+      ordinary Portuguese orthographic marks that any downstream G2P reads
+      correctly.
+
+    Unrecognized languages are returned unchanged. Each backend raises
+    ``ImportError`` naming its extra when the optional dependency is missing
+    — scriptconv never installs anything on the caller's behalf.
+    """
+    family = _diacritizer_family(lang)
+    if family == "he":
+        return _phonikud(phonikud_model).add_diacritics(text)
+    if family == "ar":
+        return _tashkeel(model or diacritizer_model).diacritize(text)
+    if family == "stress":
+        return _stress(text, lang, model)
+    if family == "pt":
+        return _sense_diacritics_pt(text)
+    return text
+
+
+# Combining marks each diacritization backend overlays onto the bare text.
+# Strip removes ONLY these codepoints — never a blanket combining-mark filter,
+# which (via NFD) would decompose and destroy precomposed native letters:
+# Cyrillic й/ё, Latvian ī, Azerbaijani ç/ö, and Arabic hamza carriers أ إ آ ؤ ئ.
+_STRESS_MARKS = frozenset({0x0300, 0x0301})            # combining grave / acute
+# U+0300 is included defensively alongside the U+0301 that stressonnx
+# actually emits, in case any backend/locale marks secondary stress with a
+# grave instead of an acute; harmless to strip since native precomposed
+# letters (e.g. Cyrillic й/ё) are unaffected either way.
+_ARABIC_MARKS = frozenset(range(0x064B, 0x0660)) | frozenset({0x0670})  # tashkeel + dagger alef
+_HEBREW_MARKS = (frozenset(range(0x05B0, 0x05BE))
+                 | frozenset({0x05BF, 0x05C1, 0x05C2, 0x05C4, 0x05C5, 0x05C7}))  # niqqud
+
+
+def _overlay_marks(lang: str) -> "frozenset | None":
+    """The overlay codepoints for *lang*'s diacritization backend, or None.
+
+    Resolves the backend family through :func:`_diacritizer_family` (the shared
+    lang→backend routing), so this can never disagree with
+    :func:`diacritize` about which language a backend owns.
+    Only *overlay* families are strippable: ``"pt"`` (bifonia sense marks are
+    native orthography) and ``None`` both return None.
+    """
+    family = _diacritizer_family(lang)
+    if family == "stress":
+        return _STRESS_MARKS
+    if family == "ar":
+        return _ARABIC_MARKS
+    if family == "he":
+        return _HEBREW_MARKS
+    return None
+
+
+def _supports_strip(lang: str) -> bool:
+    """True for languages whose diacritics are removable overlays (stress marks,
+    Arabic tashkeel, Hebrew niqqud) rather than native orthography."""
+    return _overlay_marks(lang) is not None
+
+
+def strip_diacritics(text: str, lang: str = "und", **_) -> str:
+    """Remove the overlay diacritics *lang*'s backend adds, recovering the bare
+    text WITHOUT touching native letters.
+
+    Removes only the specific overlay codepoints (combining acute/grave for
+    stress, tashkeel for Arabic, niqqud for Hebrew) — precomposed native
+    letters (Cyrillic й/ё, Latvian ī, Azerbaijani ç, Arabic hamza carriers) are
+    left intact. Raises :class:`ValueError` for languages whose diacritics are
+    part of the native orthography (e.g. European Portuguese), where any removal
+    would corrupt the spelling.
+    """
+    marks = _overlay_marks(lang)
+    if marks is None:
+        raise ValueError(
+            f"cannot strip diacritics for lang={lang!r}: its diacritics are "
+            "part of the native orthography, so removal would corrupt the "
+            "spelling. strip is defined only for overlay diacritics "
+            "(Arabic/Hebrew vocalization, East-Slavic/Turkic stress).")
+    return "".join(c for c in text if ord(c) not in marks)
+
+
+def register(graph) -> None:
+    """Opt-in graph integration: add the diacritize/strip edge pair.
+
+    ``text -> text-diacritized`` is model-based (``lossless=False``), so a
+    direct ``text -> ipa`` phonemization always out-prices the detour —
+    enabling this extension is safe and non-invasive.  Pair with
+    :func:`scriptconv.phonemizers.register` to make
+    ``"text-diacritized" -> "ipa"`` reachable.
+
+    ``text-diacritized -> text`` (:func:`strip_diacritics`) is lossless and
+    cheap, but gated: it raises for languages whose diacritics are native
+    orthography rather than removable overlays.
+    """
+    graph.register(
+        Edge("text", DIACRITIZED,
+             lambda text, lang="und", model=None, **c: diacritize(text, lang, model, **c),
+             lossless=False))
+    graph.register(
+        Edge(DIACRITIZED, "text",
+             lambda text, lang="und", **_: strip_diacritics(text, lang),
+             lossless=True))

{scriptconv-0.0.4a4 → scriptconv-0.0.4a6}/scriptconv/graph.py

@@ -38,6 +38,11 @@ from typing import Callable, Dict, List, Optional, Tuple
 __all__ = ["Representation", "Edge", "ConversionGraph", "DEFAULT_GRAPH",
            "REPRESENTATIONS"]
 
+# Chosen so a single lossy hop never wins over a lossless path unless the
+# lossless alternative is at least this many hops longer. A 10-hop lossless
+# chain (cost 10, unit edges) ties a 1-hop lossy edge (cost 10) exactly, and
+# heapq's stable tie-break then decides — a non-issue in practice since real
+# scriptconv graphs are shallow (a handful of hops at most).
 _LOSSY_COST = 10.0
 
 
@@ -56,7 +61,10 @@ class Edge:
     """One registered transform between two representations.
 
     ``fn`` is called as ``fn(text, **context)``; context keys (``lang``,
-    engine-specific options…) pass through the router opaquely.  ``requires``
+    engine-specific options…) pass through the router opaquely.  Every
+    registered ``fn`` MUST accept ``**kwargs`` for this reason — a callable
+    that only takes ``text`` raises :class:`TypeError` the moment routing
+    passes through any context. ``requires``
     names an optional extra the transform needs — metadata only; the
     transform itself raises :class:`ImportError` with an install hint.
     ``cost`` defaults from ``lossless`` so routing prefers lossless paths.

{scriptconv-0.0.4a4 → scriptconv-0.0.4a6}/scriptconv/notation.py

@@ -41,6 +41,7 @@ __all__ = [
     "ipa_to_cotovia",
     "rfe_to_ipa",
     "ipa_to_rfe",
+    "mantoq_to_ipa",
     "looks_like_ipa",
 ]
 
@@ -1053,7 +1054,7 @@ def _tokenize_mantoq(text: str) -> list[str]:
     return tokens
 
 
-def mantoq_to_ipa(mantoq, errors: str = "pass") -> str:
+def mantoq_to_ipa(mantoq: str | list[str], errors: str = "pass") -> str:
     """Convert a Mantoq phoneme string to IPA.
 
     ``_dbl_`` lengthens/geminates the preceding symbol (``ː``), ``_+_``
@@ -1235,15 +1236,19 @@ def can_convert(src: str | Notation, dst: str | Notation) -> bool:
     src = Notation(src)
     dst = Notation(dst)
     if src == dst:
-        # historical contract: identity is not a "conversion"
+        # historical contract: identity is not a "conversion" here, unlike
+        # graph.ConversionGraph.can_convert (which deliberately returns True
+        # for identity — an empty route is a valid, zero-cost conversion there)
         return False
     from scriptconv.graph import DEFAULT_GRAPH
-    if not DEFAULT_GRAPH.can_convert(src.value, dst.value):
+    try:
+        route = DEFAULT_GRAPH.route(src.value, dst.value)
+    except ValueError:
         return False
     # only notation-to-notation reachability counts here; guard against paths
     # that would leave the notation node set (none exist today, cheap to keep)
     return all(e.src in _NOTATION_VALUES and e.dst in _NOTATION_VALUES
-               for e in DEFAULT_GRAPH.route(src.value, dst.value))
+               for e in route)
 
 
 # ---------------------------------------------------------------------------

{scriptconv-0.0.4a4 → scriptconv-0.0.4a6}/scriptconv/phonemizers/_thirdparty/hangul2ipa.py

@@ -2,9 +2,10 @@
 import csv
 import math
 import os.path
+import threading
 from base64 import b64decode
 from pathlib import Path
-from typing import Union, List, Dict
+from typing import Union, List, Dict, Optional
 
 import regex as re
 
@@ -311,21 +312,46 @@ OBSTRUENTS = ()
 SONORANTS = ()
 
 
+_initialized_tables_dir: Optional[Path] = None
+# guards the one-time table population below. The rule functions read the
+# module globals (CT_*, CONSONANTS, VOWELS, ...) without locking, which is safe
+# only because those globals are written exactly once and never mutated
+# afterwards; this lock serialises concurrent first-time initialisation so no
+# thread can observe a half-populated set of tables.
+_init_lock = threading.Lock()
+
+
 def initialize_conversion_tables(tables_dir: Path):
-    global CT_double_codas, CT_neutral, CT_tensification, CT_assimilation, CT_aspiration, CT_convention
-    CT_double_codas = ConversionTable('double_coda', tables_dir)
-    CT_neutral = ConversionTable('neutralization', tables_dir)
-    CT_tensification = ConversionTable('tensification', tables_dir)
-    CT_assimilation = ConversionTable('assimilation', tables_dir)
-    CT_aspiration = ConversionTable('aspiration', tables_dir)
-    CT_convention = ConversionTable('ipa', tables_dir)
-
-    global CONSONANTS, VOWELS, OBSTRUENTS, SONORANTS
-    CONSONANTS = tuple(
-        list(CT_convention.C)[:-2])  # from the C column of the IPA table, remove special characters # and $
-    VOWELS = tuple(list(CT_convention.V))  # from the V column of the IPA table
-    OBSTRUENTS = tuple(set(CONSONANTS) - set(C_SONORANTS))
-    SONORANTS = VOWELS + C_SONORANTS
+    global _initialized_tables_dir
+    # the CSV tables never change at runtime, so once a given tables_dir has
+    # been parsed there is no need to re-read and re-parse it on every call.
+    # Fast path is lock-free: _initialized_tables_dir is published (assigned)
+    # last, after every table global is fully populated, so any thread that
+    # sees it set also sees the complete tables.
+    if _initialized_tables_dir == tables_dir:
+        return
+    with _init_lock:
+        # re-check under the lock: another thread may have initialised while we
+        # waited (double-checked locking)
+        if _initialized_tables_dir == tables_dir:
+            return
+        global CT_double_codas, CT_neutral, CT_tensification, CT_assimilation, CT_aspiration, CT_convention
+        CT_double_codas = ConversionTable('double_coda', tables_dir)
+        CT_neutral = ConversionTable('neutralization', tables_dir)
+        CT_tensification = ConversionTable('tensification', tables_dir)
+        CT_assimilation = ConversionTable('assimilation', tables_dir)
+        CT_aspiration = ConversionTable('aspiration', tables_dir)
+        CT_convention = ConversionTable('ipa', tables_dir)
+
+        global CONSONANTS, VOWELS, OBSTRUENTS, SONORANTS
+        CONSONANTS = tuple(
+            list(CT_convention.C)[:-2])  # from the C column of the IPA table, remove special characters # and $
+        VOWELS = tuple(list(CT_convention.V))  # from the V column of the IPA table
+        OBSTRUENTS = tuple(set(CONSONANTS) - set(C_SONORANTS))
+        SONORANTS = VOWELS + C_SONORANTS
+
+        # published last: any thread seeing this set also sees all tables above
+        _initialized_tables_dir = tables_dir
 
 
 def get_substring_ind(string: str, pattern: str) -> List[int]:

{scriptconv-0.0.4a4 → scriptconv-0.0.4a6}/scriptconv/phonemizers/_vendored/kog2p/__init__.py

@@ -33,6 +33,7 @@ import re
 import math
 import sys
 import optparse
+from functools import lru_cache
 
 # Option
 # Import-time CLI parsing removed for library use (the upstream script
@@ -366,10 +367,19 @@ kog2p_to_hangul = {
 }
 
 
+@lru_cache(maxsize=None)
+def _cachedReadRules(pver, rule_book):
+    # rulebook.txt never changes at runtime, so parse it once per path
+    # instead of re-opening and re-parsing it on every runKoG2P call
+    rule_in, rule_out = readRules(pver, rule_book)
+    return tuple(rule_in), tuple(rule_out)
+
+
 def runKoG2P(graph, rulebook=None):
     if not rulebook:
         rulebook = f"{os.path.dirname(__file__)}/rulebook.txt"
-    [rule_in, rule_out] = readRules(ver_info[0], rulebook)
+    rule_in, rule_out = _cachedReadRules(ver_info[0], rulebook)
+    rule_in, rule_out = list(rule_in), list(rule_out)
     words = graph.split()
     phonemized = []
     for w in words: