scriptconv 0.0.3a8__tar.gz → 0.0.3a10__tar.gz

{scriptconv-0.0.3a8 → scriptconv-0.0.3a10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a8
+Version: 0.0.3a10
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -199,6 +199,15 @@ Transcoding faithfulness depends on the target notation's inventory. IPA is the
 notation↔notation goes through IPA. The table below states, for each notation, whether a
 round-trip is exact and what happens to a symbol the table does not know.
 
+Every converter (and `convert()`) accepts a codecs-style `errors=` policy for
+symbols outside its table: `"pass"` keeps the symbol (the default everywhere
+except `ipa_to_arpa`), `"replace"` substitutes the notation's placeholder
+(`?`; the historical `ipa_to_arpa` default, tunable via `unknown=`),
+`"ignore"` drops it, and `"strict"` raises `UnknownSymbolError` naming the
+symbol, its position, and the notation. The "Unknown-token behaviour" column
+below describes the per-converter default.
+
+
 | Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
 |----------|----------------------------------|----------------------------------|-------------------------|
 | **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |

{scriptconv-0.0.3a8 → scriptconv-0.0.3a10}/README.md

@@ -166,6 +166,15 @@ Transcoding faithfulness depends on the target notation's inventory. IPA is the
 notation↔notation goes through IPA. The table below states, for each notation, whether a
 round-trip is exact and what happens to a symbol the table does not know.
 
+Every converter (and `convert()`) accepts a codecs-style `errors=` policy for
+symbols outside its table: `"pass"` keeps the symbol (the default everywhere
+except `ipa_to_arpa`), `"replace"` substitutes the notation's placeholder
+(`?`; the historical `ipa_to_arpa` default, tunable via `unknown=`),
+`"ignore"` drops it, and `"strict"` raises `UnknownSymbolError` naming the
+symbol, its position, and the notation. The "Unknown-token behaviour" column
+below describes the per-converter default.
+
+
 | Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
 |----------|----------------------------------|----------------------------------|-------------------------|
 | **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |

{scriptconv-0.0.3a8 → scriptconv-0.0.3a10}/scriptconv/__init__.py

@@ -49,6 +49,7 @@ from scriptconv.scripts import (
 )
 from scriptconv.notation import (
     Notation,
+    UnknownSymbolError,
     NotationInfo,
     NOTATION_INFO,
     convert,
@@ -119,6 +120,7 @@ __all__ = [
     "normalize_script_tag",
     # notation
     "Notation",
+    "UnknownSymbolError",
     "NotationInfo",
     "NOTATION_INFO",
     "convert",

{scriptconv-0.0.3a8 → scriptconv-0.0.3a10}/scriptconv/graph.py

@@ -197,17 +197,23 @@ def _notation_edges() -> List[Edge]:
     from scriptconv.notation import (_TO_IPA, _FROM_IPA, _DIRECT_PAIRS,
                                      NOTATION_INFO)
     edges = []
+    def _wrap(fn):
+        # forward only the errors= policy from routing context; other context
+        # keys (lang, engine options) are not the notation layer's business
+        def _edge(text, errors=None, **_):
+            return fn(text, errors=errors) if errors is not None else fn(text)
+        return _edge
+
     for notation, fn in _TO_IPA.items():
         info = NOTATION_INFO[notation]
-        edges.append(Edge(notation.value, "ipa", (lambda f: lambda t, **_: f(t))(fn),
+        edges.append(Edge(notation.value, "ipa", _wrap(fn),
                           lossless=info.lossless_to_ipa))
     for notation, fn in _FROM_IPA.items():
         info = NOTATION_INFO[notation]
-        edges.append(Edge("ipa", notation.value, (lambda f: lambda t, **_: f(t))(fn),
+        edges.append(Edge("ipa", notation.value, _wrap(fn),
                           lossless=info.lossless_from_ipa))
     for (src, dst), fn in _DIRECT_PAIRS.items():
-        edges.append(Edge(src.value, dst.value, (lambda f: lambda t, **_: f(t))(fn),
-                          lossless=True))
+        edges.append(Edge(src.value, dst.value, _wrap(fn), lossless=True))
     return edges
 
 

{scriptconv-0.0.3a8 → scriptconv-0.0.3a10}/scriptconv/notation.py

@@ -158,10 +158,41 @@ _IPA_ARPA_KEYS_SORTED = sorted(_IPA_TO_ARPA.keys(), key=len, reverse=True)
 _IPA_ARPA_RE = re.compile("|".join(re.escape(s) for s in _IPA_ARPA_KEYS_SORTED))
 
 
-def arpa_to_ipa(arpa_sequence: str) -> str:
+
+class UnknownSymbolError(ValueError):
+    """A symbol has no mapping in the requested notation (``errors="strict"``)."""
+
+    def __init__(self, symbol: str, position: int, notation: str):
+        self.symbol = symbol
+        self.position = position
+        self.notation = notation
+        super().__init__(
+            f"no {notation} mapping for {symbol!r} at position {position}")
+
+
+_ERROR_POLICIES = ("pass", "replace", "strict", "ignore")
+
+
+def _unknown(symbol: str, position: int, notation: str, errors: str,
+             replacement: str = "?") -> str:
+    """Resolve one unmapped symbol according to the codecs-style policy."""
+    if errors == "pass":
+        return symbol
+    if errors == "replace":
+        return replacement
+    if errors == "ignore":
+        return ""
+    if errors == "strict":
+        raise UnknownSymbolError(symbol, position, notation)
+    raise ValueError(f"errors must be one of {_ERROR_POLICIES}, not {errors!r}")
+
+
+def arpa_to_ipa(arpa_sequence: str, errors: str = "pass") -> str:
     """Convert a space-separated ARPABET sequence to an IPA string.
 
-    Stress digits are stripped.  Unknown tokens are passed through.
+    Stress digits are stripped.  ``errors`` selects the unknown-token policy:
+    ``"pass"`` (default) keeps the token, ``"replace"`` substitutes ``?``,
+    ``"ignore"`` drops it, ``"strict"`` raises :class:`UnknownSymbolError`.
 
     Examples
     --------
@@ -170,22 +201,27 @@ def arpa_to_ipa(arpa_sequence: str) -> str:
     """
     tokens = arpa_sequence.strip().split()
     result = []
-    for tok in tokens:
+    for i, tok in enumerate(tokens):
         ipa = _ARPA_TO_IPA.get(tok)
         if ipa is None:
             # strip trailing digit and retry
             stripped = tok.rstrip("012")
-            ipa = _ARPA_TO_IPA.get(stripped, tok)
+            ipa = _ARPA_TO_IPA.get(stripped)
+        if ipa is None:
+            ipa = _unknown(tok, i, "arpa", errors)
         result.append(ipa)
     return "".join(result)
 
 
-def ipa_to_arpa(ipa_string: str, unknown: str = "?") -> str:
+def ipa_to_arpa(ipa_string: str, unknown: str = "?",
+                errors: str = "replace") -> str:
     """Convert an IPA string to a space-separated ARPABET sequence.
 
-    Matches longest IPA symbol first.  Characters outside the ARPABET table
-    are replaced by *unknown* (default ``"?"``); pass ``unknown=""`` to drop
-    them silently.
+    Matches longest IPA symbol first.  ``errors`` selects the unknown-symbol
+    policy (default ``"replace"``, preserving the historical behavior of
+    substituting *unknown*, ``"?"``); ``unknown=""`` still drops silently.
+    Combining marks and modifier letters qualify the preceding phoneme and
+    are always dropped rather than treated as unknowns.
 
     Examples
     --------
@@ -207,8 +243,11 @@ def ipa_to_arpa(ipa_string: str, unknown: str = "?") -> str:
             # modifier letters) have no ARPABET equivalent; they qualify the
             # preceding phoneme rather than standing alone, so drop them
             # instead of emitting a spurious *unknown* token.
-            if unknown and unicodedata.category(ch) not in ("Mn", "Mc", "Me", "Lm", "Sk"):
-                tokens.append(unknown)
+            if unicodedata.category(ch) not in ("Mn", "Mc", "Me", "Lm", "Sk"):
+                effective = "ignore" if (errors == "replace" and not unknown) else errors
+                out = _unknown(ch, pos, "arpa", effective, replacement=unknown)
+                if out:
+                    tokens.append(out)
             pos += 1
     return " ".join(tokens)
 
@@ -338,7 +377,7 @@ _XS_RE = re.compile("|".join(re.escape(k) for k in _XS_KEYS_SORTED))
 _IPA_XS_RE = re.compile("|".join(re.escape(k) for k in _IPA_KEYS_SORTED))
 
 
-def xsampa_to_ipa(xsampa: str) -> str:
+def xsampa_to_ipa(xsampa: str, errors: str = "pass") -> str:
     """Convert an X-SAMPA string to IPA.
 
     Multi-character symbols are matched longest-first.  Unknown characters
@@ -359,12 +398,14 @@ def xsampa_to_ipa(xsampa: str) -> str:
             result.append(_XSAMPA_TO_IPA[m.group(0)])
             pos = m.end()
         else:
-            result.append(xsampa[pos])
+            out = _unknown(xsampa[pos], pos, "x-sampa", errors)
+            if out:
+                result.append(out)
             pos += 1
     return "".join(result)
 
 
-def ipa_to_xsampa(ipa: str) -> str:
+def ipa_to_xsampa(ipa: str, errors: str = "pass") -> str:
     """Convert an IPA string to X-SAMPA.
 
     Multi-character IPA symbols are matched longest-first.  Unknown
@@ -385,7 +426,9 @@ def ipa_to_xsampa(ipa: str) -> str:
             result.append(_IPA_TO_XSAMPA[m.group(0)])
             pos = m.end()
         else:
-            result.append(ipa[pos])
+            out = _unknown(ipa[pos], pos, "x-sampa", errors)
+            if out:
+                result.append(out)
             pos += 1
     return "".join(result)
 
@@ -461,7 +504,7 @@ _ARABIC_TO_BW["\uFEF8"] = "l|"   # لآ  lam + alef madda
 _ARABIC_TO_BW["\u0651"] = "~"   # shadda -> canonical "~", not the "^" alias
 
 
-def buckwalter_to_arabic(bw: str) -> str:
+def buckwalter_to_arabic(bw: str, errors: str = "pass") -> str:
     """Convert a Buckwalter-encoded string to Arabic Unicode.
 
     Unknown characters are passed through unchanged.
@@ -471,10 +514,12 @@ def buckwalter_to_arabic(bw: str) -> str:
     >>> buckwalter_to_arabic("mrHbA")
     'مرحبا'
     """
-    return "".join(_BW_TO_ARABIC.get(ch, ch) for ch in bw)
+    return "".join(_BW_TO_ARABIC[ch] if ch in _BW_TO_ARABIC
+                   else _unknown(ch, i, "buckwalter", errors)
+                   for i, ch in enumerate(bw))
 
 
-def arabic_to_buckwalter(arabic: str) -> str:
+def arabic_to_buckwalter(arabic: str, errors: str = "pass") -> str:
     """Convert an Arabic Unicode string to Buckwalter transliteration.
 
     Unknown characters are passed through unchanged.
@@ -484,7 +529,9 @@ def arabic_to_buckwalter(arabic: str) -> str:
     >>> arabic_to_buckwalter("مرحبا")
     'mrHbA'
     """
-    return "".join(_ARABIC_TO_BW.get(ch, ch) for ch in arabic)
+    return "".join(_ARABIC_TO_BW[ch] if ch in _ARABIC_TO_BW
+                   else _unknown(ch, i, "arabic", errors)
+                   for i, ch in enumerate(arabic))
 
 
 # ---------------------------------------------------------------------------
@@ -560,7 +607,7 @@ _IPA_FOR_LX_SORTED = sorted(_IPA_TO_LEXIQUE.keys(), key=len, reverse=True)
 _IPA_LX_RE = re.compile("|".join(re.escape(s) for s in _IPA_FOR_LX_SORTED))
 
 
-def lexique_to_ipa(lexique: str) -> str:
+def lexique_to_ipa(lexique: str, errors: str = "pass") -> str:
     """Convert a Lexique phoneme-code string to IPA.
 
     Each Lexique phoneme is exactly one character; they are read left-to-right
@@ -576,12 +623,14 @@ def lexique_to_ipa(lexique: str) -> str:
     'vɛ̃'
     """
     result = []
-    for ch in lexique:
-        result.append(_LEXIQUE_TO_IPA.get(ch, ch))
+    for i, ch in enumerate(lexique):
+        mapped = _LEXIQUE_TO_IPA.get(ch)
+        result.append(mapped if mapped is not None
+                      else _unknown(ch, i, "lexique", errors))
     return "".join(result)
 
 
-def ipa_to_lexique(ipa: str) -> str:
+def ipa_to_lexique(ipa: str, errors: str = "pass") -> str:
     """Convert an IPA string to Lexique phoneme codes.
 
     Matches longest IPA symbol first.  Unknown characters are passed through
@@ -605,7 +654,9 @@ def ipa_to_lexique(ipa: str) -> str:
             result.append(_IPA_TO_LEXIQUE[m.group(0)])
             pos = m.end()
         else:
-            result.append(ipa[pos])
+            out = _unknown(ipa[pos], pos, "lexique", errors)
+            if out:
+                result.append(out)
             pos += 1
     return "".join(result)
 
@@ -640,7 +691,7 @@ for _k, _ip in _KIRSHENBAUM_TO_IPA.items():
     _IPA_TO_KIRSHENBAUM.setdefault(_ip, _k)
 
 
-def kirshenbaum_to_ipa(kirshenbaum: str) -> str:
+def kirshenbaum_to_ipa(kirshenbaum: str, errors: str = "pass") -> str:
     """Convert a Kirshenbaum (ASCII-IPA) string to IPA.
 
     Each character is mapped independently. Characters outside the table
@@ -653,10 +704,12 @@ def kirshenbaum_to_ipa(kirshenbaum: str) -> str:
     >>> kirshenbaum_to_ipa("N")
     'ŋ'
     """
-    return "".join(_KIRSHENBAUM_TO_IPA.get(ch, ch) for ch in kirshenbaum)
+    return "".join(_KIRSHENBAUM_TO_IPA[ch] if ch in _KIRSHENBAUM_TO_IPA
+                   else _unknown(ch, i, "kirshenbaum", errors)
+                   for i, ch in enumerate(kirshenbaum))
 
 
-def ipa_to_kirshenbaum(ipa: str) -> str:
+def ipa_to_kirshenbaum(ipa: str, errors: str = "pass") -> str:
     """Convert an IPA string to Kirshenbaum (ASCII-IPA).
 
     Characters outside the table pass through unchanged.
@@ -668,7 +721,9 @@ def ipa_to_kirshenbaum(ipa: str) -> str:
     >>> ipa_to_kirshenbaum("ŋ")
     'N'
     """
-    return "".join(_IPA_TO_KIRSHENBAUM.get(ch, ch) for ch in ipa)
+    return "".join(_IPA_TO_KIRSHENBAUM[ch] if ch in _IPA_TO_KIRSHENBAUM
+                   else _unknown(ch, i, "kirshenbaum", errors)
+                   for i, ch in enumerate(ipa))
 
 
 # ---------------------------------------------------------------------------
@@ -737,7 +792,7 @@ _IPA_CV_KEYS_SORTED = sorted(_IPA_TO_COTOVIA.keys(), key=len, reverse=True)
 _IPA_CV_RE = re.compile("|".join(re.escape(s) for s in _IPA_CV_KEYS_SORTED))
 
 
-def cotovia_to_ipa(cotovia: str) -> str:
+def cotovia_to_ipa(cotovia: str, errors: str = "pass") -> str:
     """Convert a Cotovía phoneme string to IPA.
 
     Multi-character symbols are matched longest-first. Characters outside the
@@ -758,12 +813,14 @@ def cotovia_to_ipa(cotovia: str) -> str:
             result.append(_COTOVIA_TO_IPA[m.group(0)])
             pos = m.end()
         else:
-            result.append(cotovia[pos])
+            out = _unknown(cotovia[pos], pos, "cotovia", errors)
+            if out:
+                result.append(out)
             pos += 1
     return "".join(result)
 
 
-def ipa_to_cotovia(ipa: str) -> str:
+def ipa_to_cotovia(ipa: str, errors: str = "pass") -> str:
     """Convert an IPA string to Cotovía notation.
 
     Multi-character IPA symbols are matched longest-first. Characters outside
@@ -784,7 +841,9 @@ def ipa_to_cotovia(ipa: str) -> str:
             result.append(_IPA_TO_COTOVIA[m.group(0)])
             pos = m.end()
         else:
-            result.append(ipa[pos])
+            out = _unknown(ipa[pos], pos, "cotovia", errors)
+            if out:
+                result.append(out)
             pos += 1
     return "".join(result)
 
@@ -857,7 +916,7 @@ _IPA_RFE_KEYS_SORTED = sorted(_IPA_TO_RFE.keys(), key=len, reverse=True)
 _IPA_RFE_RE = re.compile("|".join(re.escape(x) for x in _IPA_RFE_KEYS_SORTED))
 
 
-def rfe_to_ipa(rfe: str) -> str:
+def rfe_to_ipa(rfe: str, errors: str = "pass") -> str:
     """Convert an RFE (Revista de Filología Española) string to IPA.
 
     Multi-character symbols are matched longest-first. Characters outside the
@@ -878,12 +937,14 @@ def rfe_to_ipa(rfe: str) -> str:
             result.append(_RFE_TO_IPA[m.group(0)])
             pos = m.end()
         else:
-            result.append(rfe[pos])
+            out = _unknown(rfe[pos], pos, "rfe", errors)
+            if out:
+                result.append(out)
             pos += 1
     return "".join(result)
 
 
-def ipa_to_rfe(ipa: str) -> str:
+def ipa_to_rfe(ipa: str, errors: str = "pass") -> str:
     """Convert an IPA string to RFE (Revista de Filología Española) notation.
 
     Multi-character IPA symbols are matched longest-first. Characters outside
@@ -904,7 +965,9 @@ def ipa_to_rfe(ipa: str) -> str:
             result.append(_IPA_TO_RFE[m.group(0)])
             pos = m.end()
         else:
-            result.append(ipa[pos])
+            out = _unknown(ipa[pos], pos, "rfe", errors)
+            if out:
+                result.append(out)
             pos += 1
     return "".join(result)
 
@@ -913,7 +976,8 @@ def ipa_to_rfe(ipa: str) -> str:
 # convert — facade routing through IPA where no direct map exists
 # ---------------------------------------------------------------------------
 
-def convert(text: str, src: str | Notation, dst: str | Notation) -> str:
+def convert(text: str, src: str | Notation, dst: str | Notation,
+            errors: str = "pass") -> str:
     """Convert *text* from *src* notation to *dst* notation.
 
     Supported pairs (direct):
@@ -956,7 +1020,9 @@ def convert(text: str, src: str | Notation, dst: str | Notation) -> str:
     # node, Buckwalter <-> Arabic is a direct edge).
     from scriptconv.graph import DEFAULT_GRAPH
     try:
-        return DEFAULT_GRAPH.convert(text, src.value, dst.value)
+        return DEFAULT_GRAPH.convert(text, src.value, dst.value, errors=errors)
+    except UnknownSymbolError:
+        raise
     except ValueError:
         raise ValueError(f"Unsupported conversion: {src!r} → {dst!r}") from None
 

{scriptconv-0.0.3a8 → scriptconv-0.0.3a10}/scriptconv/scripts.py

@@ -652,6 +652,34 @@ _LANG_TO_SCRIPT: dict[str, str] = {
     "rw": "Latn",   # Kinyarwanda
     "tk": "Latn",   # Turkmen (Latin since 1993)
     "eo": "Latn",   # Esperanto
+    # Languages with no ISO 639-1 code (keyed by their 639-3 code directly)
+    "kjh": "Cyrl",  # Khakas
+    "xal": "Cyrl",  # Kalmyk (Cyrillic since 1938)
+    "kbd": "Cyrl",  # Kabardian
+    "erz": "Cyrl",  # Erzya
+    "mdf": "Cyrl",  # Moksha
+}
+
+# ISO 639-2/3 (incl. bibliographic variants) -> the 639-1 key used above.
+# Reference: Library of Congress ISO 639-2 code list.
+_ISO639_ALIASES: dict[str, str] = {
+    "rus": "ru", "ukr": "uk", "bel": "be", "bul": "bg", "srp": "sr",
+    "mkd": "mk", "mac": "mk", "kaz": "kk", "kir": "ky", "tgk": "tg",
+    "mon": "mn", "tat": "tt", "bak": "ba", "chv": "cv", "oss": "os",
+    "che": "ce", "hye": "hy", "arm": "hy", "kat": "ka", "geo": "ka",
+    "aze": "az", "uzb": "uz", "tuk": "tk", "eng": "en", "deu": "de",
+    "ger": "de", "fra": "fr", "fre": "fr", "spa": "es", "por": "pt",
+    "ita": "it", "nld": "nl", "dut": "nl", "ell": "el", "gre": "el",
+    "ara": "ar", "heb": "he", "zho": "zh", "chi": "zh", "jpn": "ja",
+    "kor": "ko", "tur": "tr", "fas": "fa", "per": "fa", "urd": "ur",
+    "hin": "hi", "tha": "th", "vie": "vi", "pol": "pl", "ces": "cs",
+    "cze": "cs", "slk": "sk", "slo": "sk", "ron": "ro", "rum": "ro",
+    "hun": "hu", "fin": "fi", "swe": "sv", "nor": "no", "dan": "da",
+    "isl": "is", "ice": "is", "mya": "my", "bur": "my", "khm": "km",
+    "lao": "lo", "amh": "am", "tam": "ta", "tel": "te", "ben": "bn",
+    "guj": "gu", "kan": "kn", "mal": "ml", "mar": "mr", "pan": "pa",
+    "sin": "si", "nep": "ne", "eus": "eu", "baq": "eu", "cat": "ca",
+    "glg": "gl", "cym": "cy", "wel": "cy", "gle": "ga", "epo": "eo",
 }
 
 
@@ -674,14 +702,21 @@ def lang_to_script(lang: str) -> Optional[str]:
     'Latn'
     """
     subtags = lang.replace("_", "-").split("-")
-    # An explicit script subtag (4 letters, e.g. "Latn") overrides the
-    # language's default script.
+    # An explicit script subtag overrides the language's default script:
+    # 4-letter ISO-15924 codes ("sr-Latn"), and the widespread informal
+    # 3-letter forms "cyr"/"lat" ("uzb_cyr", "aze_lat").
     for sub in subtags[1:]:
+        low = sub.lower()
         if len(sub) == 4 and sub.isalpha():
             resolved = normalize_script_tag(sub)
             if resolved is not None:
                 return resolved
-    return _LANG_TO_SCRIPT.get(subtags[0].lower())
+        elif low in ("cyr", "lat"):
+            return "Cyrl" if low == "cyr" else "Latn"
+    key = subtags[0].lower()
+    if key in _LANG_TO_SCRIPT:
+        return _LANG_TO_SCRIPT[key]
+    return _LANG_TO_SCRIPT.get(_ISO639_ALIASES.get(key, ""))
 
 
 # ---------------------------------------------------------------------------

{scriptconv-0.0.3a8 → scriptconv-0.0.3a10}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 3
-VERSION_ALPHA = 8
+VERSION_ALPHA = 10
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.3a8 → scriptconv-0.0.3a10}/scriptconv.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a8
+Version: 0.0.3a10
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -199,6 +199,15 @@ Transcoding faithfulness depends on the target notation's inventory. IPA is the
 notation↔notation goes through IPA. The table below states, for each notation, whether a
 round-trip is exact and what happens to a symbol the table does not know.
 
+Every converter (and `convert()`) accepts a codecs-style `errors=` policy for
+symbols outside its table: `"pass"` keeps the symbol (the default everywhere
+except `ipa_to_arpa`), `"replace"` substitutes the notation's placeholder
+(`?`; the historical `ipa_to_arpa` default, tunable via `unknown=`),
+`"ignore"` drops it, and `"strict"` raises `UnknownSymbolError` naming the
+symbol, its position, and the notation. The "Unknown-token behaviour" column
+below describes the per-converter default.
+
+
 | Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
 |----------|----------------------------------|----------------------------------|-------------------------|
 | **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |

{scriptconv-0.0.3a8 → scriptconv-0.0.3a10}/scriptconv.egg-info/SOURCES.txt

@@ -23,10 +23,12 @@ scriptconv/data/cangjie5_tc.tsv.gz
 tests/test_cangjie.py
 tests/test_cli.py
 tests/test_conventions.py
+tests/test_errors_policy.py
 tests/test_examples.py
 tests/test_graph.py
 tests/test_notation.py
 tests/test_readings.py
 tests/test_readings_zh.py
 tests/test_scripts.py
+tests/test_scripts_stressonnx_compat.py
 tests/test_translit.py

scriptconv-0.0.3a10/tests/test_errors_policy.py

@@ -0,0 +1,102 @@
+import json
+import pathlib
+import unittest
+
+from scriptconv import UnknownSymbolError, convert
+from scriptconv import notation as N
+
+FIXTURE = json.loads(
+    (pathlib.Path(__file__).parent / "data_errors_fixture.json").read_text())
+
+# (converter name, a sample containing an unknown symbol, the unknown symbol)
+UNKNOWN_SAMPLES = [
+    ("arpa_to_ipa", "ZZZ", "ZZZ"),
+    ("ipa_to_arpa", "χ", "χ"),
+    ("xsampa_to_ipa", "¤", "¤"),
+    ("ipa_to_xsampa", "¤", "¤"),
+    ("lexique_to_ipa", "¤", "¤"),
+    ("ipa_to_lexique", "χ", "χ"),
+    ("kirshenbaum_to_ipa", "¤", "¤"),
+    ("ipa_to_kirshenbaum", "ꟼ", "ꟼ"),
+    ("cotovia_to_ipa", "¤", "¤"),
+    ("ipa_to_cotovia", "χ", "χ"),
+    ("rfe_to_ipa", "¤", "¤"),
+    ("ipa_to_rfe", "χ", "χ"),
+    ("buckwalter_to_arabic", "#", "#"),
+    ("arabic_to_buckwalter", "x", "x"),
+]
+
+
+class TestDefaultsUnchanged(unittest.TestCase):
+    """Pre-change fixture battery: no-kwargs output is byte-identical."""
+
+    def test_fixture_battery(self):
+        for fn_name, cases in FIXTURE.items():
+            fn = getattr(N, fn_name)
+            for inp, expected in cases.items():
+                self.assertEqual(fn(inp), expected, f"{fn_name}({inp!r})")
+
+
+class TestPolicies(unittest.TestCase):
+    def test_all_four_policies_per_converter(self):
+        for fn_name, sample, symbol in UNKNOWN_SAMPLES:
+            fn = getattr(N, fn_name)
+            passed = fn(sample, errors="pass")
+            self.assertIn(symbol, passed, fn_name)
+            replaced = fn(sample, errors="replace")
+            self.assertNotIn(symbol, replaced, fn_name)
+            self.assertIn("?", replaced, fn_name)
+            ignored = fn(sample, errors="ignore")
+            self.assertNotIn(symbol, ignored, fn_name)
+            self.assertNotIn("?", ignored, fn_name)
+            with self.assertRaises(UnknownSymbolError, msg=fn_name):
+                fn(sample, errors="strict")
+
+    def test_strict_error_names_symbol_position_notation(self):
+        with self.assertRaises(UnknownSymbolError) as ctx:
+            N.xsampa_to_ipa("ab¤", errors="strict")
+        e = ctx.exception
+        self.assertEqual(e.symbol, "¤")
+        self.assertEqual(e.position, 2)
+        self.assertEqual(e.notation, "x-sampa")
+        self.assertIn("'¤'", str(e))
+        self.assertIn("2", str(e))
+        self.assertIn("x-sampa", str(e))
+
+    def test_invalid_policy_raises_value_error(self):
+        with self.assertRaises(ValueError):
+            N.xsampa_to_ipa("¤", errors="explode")
+
+    def test_unknown_symbol_error_is_value_error(self):
+        self.assertTrue(issubclass(UnknownSymbolError, ValueError))
+
+
+class TestIpaToArpaCompat(unittest.TestCase):
+    def test_default_still_replaces_with_question_mark(self):
+        self.assertEqual(N.ipa_to_arpa("χ"), "?")
+
+    def test_unknown_empty_still_drops(self):
+        self.assertEqual(N.ipa_to_arpa("χkæt", unknown=""), "K AE T")
+
+    def test_custom_unknown_token(self):
+        self.assertEqual(N.ipa_to_arpa("χ", unknown="<UNK>"), "<UNK>")
+
+    def test_diacritics_never_treated_as_unknown(self):
+        # combining marks qualify the preceding phoneme even under strict
+        self.assertEqual(N.ipa_to_arpa("kæ̃t", errors="strict"), "K AE T")
+
+
+class TestConvertFacade(unittest.TestCase):
+    def test_errors_threads_through_routing(self):
+        with self.assertRaises(UnknownSymbolError):
+            convert("¤", "x-sampa", "arpa", errors="strict")
+        self.assertEqual(convert("h@loU", "x-sampa", "arpa"), "HH AX L OW")
+
+    def test_errors_ignore_through_two_hops(self):
+        out = convert("k{t¤", "x-sampa", "arpa", errors="ignore")
+        self.assertNotIn("¤", out)
+        self.assertNotIn("?", out)
+
+
+if __name__ == "__main__":
+    unittest.main()

scriptconv-0.0.3a10/tests/test_scripts_stressonnx_compat.py

@@ -0,0 +1,47 @@
+"""Downstream compat: the script tags scriptconv returns are a wire format.
+
+stressonnx (accentor.py) maps its supported languages to a private script
+taxonomy that mirrors phoonnx's Alphabet enum.  These fixtures pin, key by
+key, that scriptconv resolves every one of those language codes to the
+expected ISO-15924 tag — so stressonnx's migration to scriptconv is a pure
+deletion, and future edits here cannot silently break it.
+"""
+import pytest
+
+from scriptconv import lang_to_script
+
+# stressonnx LANG_SCRIPT, verbatim keys -> the ISO-15924 tag its Script
+# enum value corresponds to (cyrillic->Cyrl, latin->Latn, armenian->Armn,
+# georgian->Geor).
+STRESSONNX_LANGS = {
+    "ru": "Cyrl", "ukr": "Cyrl", "bel": "Cyrl", "bel_simple": "Cyrl",
+    "kaz": "Cyrl", "tat": "Cyrl", "bak": "Cyrl", "chv": "Cyrl",
+    "sah": "Cyrl", "kir": "Cyrl", "kjh": "Cyrl", "tgk": "Cyrl",
+    "udm": "Cyrl", "xal": "Cyrl", "kbd": "Cyrl", "erz": "Cyrl",
+    "mdf": "Cyrl", "uzb_cyr": "Cyrl", "aze_cyr": "Cyrl",
+    "aze_lat": "Latn", "uzb_lat": "Latn",
+    "hye": "Armn",
+    "kat": "Geor",
+}
+
+
+@pytest.mark.parametrize("lang,expected", sorted(STRESSONNX_LANGS.items()))
+def test_stressonnx_language_resolves(lang, expected):
+    assert lang_to_script(lang) == expected
+
+
+def test_iso639_bibliographic_variants():
+    assert lang_to_script("ger") == lang_to_script("deu") == "Latn"
+    assert lang_to_script("gre") == lang_to_script("ell") == "Grek"
+    assert lang_to_script("arm") == lang_to_script("hye") == "Armn"
+
+
+def test_explicit_script_subtag_still_authoritative():
+    assert lang_to_script("uzb-Latn") == "Latn"
+    assert lang_to_script("sr_lat") == "Latn"
+    assert lang_to_script("sr") == "Cyrl"
+
+
+def test_unknown_language_stays_none():
+    assert lang_to_script("zzz") is None
+    assert lang_to_script("qqq_cyr") == "Cyrl"  # variant subtag wins even unknown-lang