scriptconv 0.0.3a5__tar.gz → 0.0.3a7__tar.gz

{scriptconv-0.0.3a5 → scriptconv-0.0.3a7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a5
+Version: 0.0.3a7
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv

{scriptconv-0.0.3a5 → scriptconv-0.0.3a7}/scriptconv/__init__.py

@@ -22,6 +22,10 @@ readings:
     extra; Chinese hanzi → pinyin/bopomofo via the ``zh`` extra).
 cangjie:
     Hanzi → Cangjie5 input codes (shape decomposition, vendored table).
+conventions:
+    Orthographic conventions — script-scoped decorations orthogonal to
+    identity (tashkeel, niqqud, wakachigaki, pinyin tone spelling…).
+    Scripts are nodes; conventions are parameters, never nodes.
 
 Zero required runtime dependencies (stdlib only); the readings module
 needs the optional ``ja``/``zh`` extras.
@@ -68,11 +72,22 @@ from scriptconv.translit import (
     kana_to_hira,
 )
 from scriptconv.readings import (
+    ReadingToken,
+    tokens,
     to_hiragana,
     to_katakana,
     to_pinyin,
     to_bopomofo,
 )
+from scriptconv.conventions import (
+    Convention,
+    CONVENTION_REGISTRY,
+    conventions_for,
+    restyle,
+    strip,
+    apply,
+    detect as detect_convention,
+)
 from scriptconv.cangjie import (
     cangjie_code,
     to_cangjie,
@@ -117,8 +132,18 @@ __all__ = [
     "hira_to_kana",
     "kana_to_hira",
     # readings (dictionary-backed, needs scriptconv[ja])
+    "ReadingToken",
+    "tokens",
     "to_hiragana",
     "to_katakana",
+    # conventions (orthographic decorations, orthogonal to script identity)
+    "Convention",
+    "CONVENTION_REGISTRY",
+    "conventions_for",
+    "restyle",
+    "strip",
+    "apply",
+    "detect_convention",
     # cangjie (vendored Cangjie5 table)
     "cangjie_code",
     "to_cangjie",

{scriptconv-0.0.3a5 → scriptconv-0.0.3a7}/scriptconv/__main__.py

@@ -8,6 +8,9 @@ Usage
     python -m scriptconv direction <text>
     python -m scriptconv decompose <text>
     python -m scriptconv lang <code>
+    python -m scriptconv strip <convention> <text>
+    python -m scriptconv restyle <convention> <style> <text>
+    python -m scriptconv conventions [--script CODE]
 
 Examples
 --------
@@ -48,6 +51,19 @@ def main(argv: list[str] | None = None) -> int:
     lang = sub.add_parser("lang", help="Map language code to script")
     lang.add_argument("code", help="BCP-47 or ISO 639 language code")
 
+    st = sub.add_parser("strip", help="Remove an orthographic convention's decoration")
+    st.add_argument("convention", help="Convention id (see `conventions`)")
+    st.add_argument("text", help="Text to strip")
+
+    rs = sub.add_parser("restyle", help="Rewrite text between a convention's styles")
+    rs.add_argument("convention", help="Convention id (see `conventions`)")
+    rs.add_argument("to", help="Target style")
+    rs.add_argument("text", help="Text to restyle")
+    rs.add_argument("--from", dest="frm", default=None, help="Source style")
+
+    cv = sub.add_parser("conventions", help="List orthographic conventions")
+    cv.add_argument("--script", default=None, help="Filter by ISO-15924 script code")
+
     args = p.parse_args(argv)
 
     if args.command is None:
@@ -61,6 +77,9 @@ def main(argv: list[str] | None = None) -> int:
         "direction": lambda: _do_direction(args),
         "decompose": lambda: _do_decompose(args),
         "lang": lambda: _do_lang(args),
+        "strip": lambda: _do_strip(args),
+        "restyle": lambda: _do_restyle(args),
+        "conventions": lambda: _do_conventions(args),
     }
     dispatch[args.command]()
     return 0
@@ -100,6 +119,34 @@ def _do_decompose(args: argparse.Namespace) -> None:
     print(decompose_hangul(args.text))
 
 
+def _do_strip(args: argparse.Namespace) -> None:
+    from scriptconv.conventions import strip
+    try:
+        print(strip(args.text, args.convention))
+    except ValueError as e:
+        raise SystemExit(f"error: {e}")
+
+
+def _do_restyle(args: argparse.Namespace) -> None:
+    from scriptconv.conventions import restyle
+    try:
+        print(restyle(args.text, args.convention, args.to, frm=args.frm))
+    except ValueError as e:
+        raise SystemExit(f"error: {e}")
+
+
+def _do_conventions(args: argparse.Namespace) -> None:
+    from scriptconv.conventions import CONVENTION_REGISTRY, conventions_for
+    convs = (conventions_for(args.script) if args.script
+             else list(CONVENTION_REGISTRY.values()))
+    if not convs:
+        print("(no conventions registered for that script)")
+    for c in convs:
+        system = f" [{c.system}]" if c.system else ""
+        print(f"  {c.id}{system}  scripts={','.join(c.scripts)}  "
+              f"styles={','.join(c.styles)}  — {c.name}")
+
+
 def _do_lang(args: argparse.Namespace) -> None:
     from scriptconv.scripts import lang_to_script
     print(lang_to_script(args.code) or "(unknown)")

scriptconv-0.0.3a7/scriptconv/conventions.py

@@ -0,0 +1,495 @@
+"""Orthographic conventions — script-scoped decorations, orthogonal to identity.
+
+Scripts are text *identity*: character-evident, ISO-15924-coded, the nodes of
+scriptconv's conversion space (:mod:`scriptconv.scripts`).  Conventions are
+*decorations* a script's orthography can carry or omit — optional vocalization
+(Arabic tashkeel, Hebrew niqqud), justification (kashida), spacing modes
+(Japanese wakachigaki), or mutually-exclusive spellings of the same
+information (pinyin tone marks vs digits).  A convention never participates in
+script detection or conversion routing; it is always a parameter, never a
+node.
+
+Each convention declares its *styles* — the mutually exclusive states its text
+can be in.  Binary decorations have styles ``("marked", "none")``; variant
+conventions name their spellings (``("mark", "number", "none")``).  The one
+real operation is :func:`restyle`, a partial transition matrix between styles;
+:func:`strip` and :func:`apply` are sugar for transitions to and from
+``"none"``.
+
+Which transitions exist follows the same boundary the rest of scriptconv
+draws:
+
+* stripping and deterministic re-spelling are pure orthography — always
+  available, zero-dependency;
+* transitions that need a *dictionary lookup* (wakachigaki application =
+  word segmentation) are orthography too, but live behind the optional
+  extras, like :mod:`scriptconv.readings`;
+* transitions that need contextual *prediction* (restoring tashkeel or
+  niqqud, i.e. diacritization) are linguistic inference, out of scope for the
+  same reason grapheme-to-phoneme is.  Their absence is queryable data, not a
+  runtime surprise.
+
+What is **not** a convention: marks that are letter identity rather than
+decoration.  Vietnamese tone diacritics spell different words, Arabic
+hamza/madda (U+0653–U+0655) distinguish letters, and stripping them corrupts
+text — none are registered.  Casing on bicameral scripts *is* a convention in
+the sense used here, but the stdlib (``str.lower``/``str.casefold``) already
+models it correctly, so it is deliberately not duplicated.
+
+``jamo-form`` is the registry's one documented impurity: compatibility vs
+conjoining jamo is a Unicode *representation* convention rather than an
+orthographic one, registered because free text genuinely occurs in both
+repertoires and the restyle is deterministic.
+"""
+from __future__ import annotations
+
+import re
+import unicodedata
+from dataclasses import dataclass
+from typing import Callable, Dict, List, Optional, Tuple
+
+__all__ = [
+    "Convention",
+    "CONVENTION_REGISTRY",
+    "conventions_for",
+    "detect",
+    "restyle",
+    "strip",
+    "apply",
+]
+
+
+@dataclass(frozen=True)
+class Convention:
+    """Metadata for one orthographic convention (NotationInfo-style: data, not prose)."""
+    id: str
+    name: str
+    scripts: Tuple[str, ...]           # ISO-15924 codes whose text it decorates
+    system: Optional[str]              # sub-script system qualifier ("pinyin", "japanese")
+    styles: Tuple[str, ...]            # mutually exclusive states; includes "none" if strippable
+    default_style: str
+    detectable: bool                   # current style is character-evident
+    strip_lossless: bool               # can the stripped information be recovered?
+    apply_requires: Optional[str]      # optional-extra name needed to leave "none", if any
+    ranges: Tuple[Tuple[int, int], ...]  # decoration codepoints (empty for spacing/variant conventions)
+    reference: str
+
+
+# ---------------------------------------------------------------------------
+# Registry
+# ---------------------------------------------------------------------------
+# Tashkeel deliberately EXCLUDES U+0653–0655 (madda above, hamza above, hamza
+# below): in NFD text آ/أ/إ decompose to alef + those marks, which are letter
+# identity — stripping them would corrupt the consonantal skeleton.
+_TASHKEEL_RANGES = ((0x064B, 0x0652), (0x0656, 0x065F), (0x0670, 0x0670),
+                    (0x08F0, 0x08F2))
+_KASHIDA_RANGES = ((0x0640, 0x0640),)
+# 06DD/06DE (end of ayah, rub el hizb) are structural signs and 06E5/06E6 are
+# small letters — kept; 06E7-06E9 (small high marks, place of sajdah) and
+# 06EA-06ED are annotation marks — stripped.
+_QURANIC_RANGES = ((0x06D6, 0x06DC), (0x06DF, 0x06E4), (0x06E7, 0x06ED))
+# Niqqud includes dagesh/mapiq U+05BC, meteg U+05BD (a masoretic stress mark,
+# but traditionally stripped with the points), rafe U+05BF and the shin/sin
+# dots U+05C1/05C2 — stripping loses the שׁ/שׂ distinction, exactly as
+# unpointed Hebrew does.  Maqaf U+05BE, sof pasuq U+05C3 and nun hafukha
+# U+05C6 are punctuation and survive.
+_NIQQUD_RANGES = ((0x05B0, 0x05BD), (0x05BF, 0x05BF), (0x05C1, 0x05C2),
+                  (0x05C7, 0x05C7))
+_TEAMIM_RANGES = ((0x0591, 0x05AF), (0x05C0, 0x05C0), (0x05C4, 0x05C5))
+
+CONVENTION_REGISTRY: Dict[str, Convention] = {c.id: c for c in (
+    Convention("tashkeel", "Arabic vocalization (harakat/tanwin/shadda/sukun)",
+               ("Arab",), None, ("marked", "none"), "marked",
+               detectable=True, strip_lossless=False, apply_requires=None,
+               ranges=_TASHKEEL_RANGES,
+               reference="Unicode 15 ch. 9.2 (Arabic); excludes letter-identity U+0653-0655"),
+    Convention("kashida", "Arabic justification elongation (tatweel)",
+               ("Arab",), None, ("marked", "none"), "none",
+               detectable=True, strip_lossless=False, apply_requires=None,
+               ranges=_KASHIDA_RANGES, reference="Unicode U+0640 ARABIC TATWEEL"),
+    Convention("quranic-marks", "Quranic annotation signs",
+               ("Arab",), None, ("marked", "none"), "none",
+               detectable=True, strip_lossless=False, apply_requires=None,
+               ranges=_QURANIC_RANGES, reference="Unicode U+06D6-06ED Quranic annotation signs"),
+    Convention("niqqud", "Hebrew vocalization points",
+               ("Hebr",), None, ("marked", "none"), "marked",
+               detectable=True, strip_lossless=False, apply_requires=None,
+               ranges=_NIQQUD_RANGES,
+               reference="Unicode 15 ch. 9.1 (Hebrew); strip loses the shin/sin dot distinction"),
+    Convention("teamim", "Hebrew cantillation marks",
+               ("Hebr",), None, ("marked", "none"), "none",
+               detectable=True, strip_lossless=False, apply_requires=None,
+               ranges=_TEAMIM_RANGES, reference="Unicode U+0591-05AF Hebrew accents (masoretic)"),
+    Convention("wakachigaki", "Japanese spaced writing (分かち書き)",
+               ("Hira", "Kana", "Hani"), "japanese", ("marked", "none"), "none",
+               detectable=False, strip_lossless=False, apply_requires="ja",
+               ranges=(),
+               reference="Spaced orthographic mode of Japanese (children's books, games)"),
+    Convention("pinyin-tone", "Pinyin tone spelling",
+               ("Latn",), "pinyin", ("mark", "number", "none"), "mark",
+               detectable=False, strip_lossless=False, apply_requires=None,
+               ranges=(),
+               reference="GB/T 16159 Basic rules of Hanyu Pinyin orthography"),
+    Convention("jamo-form", "Hangul jamo repertoire (compatibility vs conjoining)",
+               ("Hang",), None, ("compatibility", "conjoining"), "compatibility",
+               detectable=True, strip_lossless=True, apply_requires=None,
+               ranges=(),
+               reference="Unicode Hangul Compatibility Jamo U+3130 vs Hangul Jamo U+1100"),
+)}
+
+
+def conventions_for(script: str) -> List[Convention]:
+    """Conventions that decorate *script* (ISO-15924 code), registry order."""
+    return [c for c in CONVENTION_REGISTRY.values() if script in c.scripts]
+
+
+def _conv(convention_id: str) -> Convention:
+    try:
+        return CONVENTION_REGISTRY[convention_id]
+    except KeyError:
+        raise ValueError(
+            f"unknown convention {convention_id!r}; known: "
+            f"{', '.join(sorted(CONVENTION_REGISTRY))}") from None
+
+
+def _in_ranges(cp: int, ranges: Tuple[Tuple[int, int], ...]) -> bool:
+    return any(lo <= cp <= hi for lo, hi in ranges)
+
+
+def _range_strip(ranges: Tuple[Tuple[int, int], ...]) -> Callable[[str], str]:
+    def _strip(text: str) -> str:
+        return "".join(c for c in text if not _in_ranges(ord(c), ranges))
+    return _strip
+
+
+# ---------------------------------------------------------------------------
+# Wakachigaki
+# ---------------------------------------------------------------------------
+# Strip removes space runs only when BOTH flanks are Japanese-class: spaces
+# next to Latin, digits or boundaries carry information ("きょうは good day",
+# "第 3 章") and must survive.
+
+def _is_japanese_flank(ch: str) -> bool:
+    cp = ord(ch)
+    return (0x3041 <= cp <= 0x309F or          # hiragana
+            0x30A0 <= cp <= 0x30FF or          # katakana incl. ー
+            0x31F0 <= cp <= 0x31FF or          # katakana phonetic extensions
+            0xFF66 <= cp <= 0xFF9F or          # halfwidth katakana
+            0x4E00 <= cp <= 0x9FFF or          # CJK unified
+            0x3400 <= cp <= 0x4DBF or          # CJK ext A
+            cp in (0x3005, 0x3006, 0x303B))    # 々 〆 repeat marks
+
+_WAKACHI_SPACE = re.compile(r"[ 　]+")
+
+
+def _wakachigaki_strip(text: str) -> str:
+    out = []
+    i = 0
+    while i < len(text):
+        m = _WAKACHI_SPACE.match(text, i)
+        if m:
+            before = out[-1] if out else ""
+            after = text[m.end():m.end() + 1]
+            if not (before and after and _is_japanese_flank(before[-1])
+                    and _is_japanese_flank(after)):
+                out.append(m.group(0))
+            i = m.end()
+        else:
+            out.append(text[i])
+            i += 1
+    return "".join(out)
+
+
+def _wakachigaki_apply(text: str) -> str:
+    try:
+        from scriptconv.readings import tokens
+    except ImportError:
+        raise ImportError(
+            "applying wakachigaki needs dictionary word segmentation — "
+            "install with `pip install scriptconv[ja]`") from None
+    parts: List[str] = []
+    for tok in tokens(text):
+        if parts and _is_japanese_flank(parts[-1][-1]) and _is_japanese_flank(tok.orig[0]):
+            parts.append(" ")
+        parts.append(tok.orig)
+    return "".join(parts)
+
+
+# ---------------------------------------------------------------------------
+# Pinyin tone spelling
+# ---------------------------------------------------------------------------
+# number→mark is deterministic.  mark→number is deterministic for GB/T
+# 16159-compliant (apostrophized) pinyin; on non-compliant input where several
+# marked vowels share one letter run, syllables are split best-effort before
+# the longest valid onset preceding each subsequent marked vowel.
+
+_TONE_COMBINING = {1: "̄", 2: "́", 3: "̌", 4: "̀"}
+_COMBINING_TONE = {v: k for k, v in _TONE_COMBINING.items()}
+_VOWELS = "aeiouü"
+_ONSETS = {"b", "p", "m", "f", "d", "t", "n", "l", "g", "k", "h",
+           "j", "q", "x", "r", "z", "c", "s", "y", "w", "zh", "ch", "sh"}
+
+
+def _mark_syllable(syl: str, tone: int) -> str:
+    """Place the tone mark of *syl* (tone digit already removed)."""
+    syl = syl.replace("v", "ü").replace("V", "Ü").replace("u:", "ü").replace("U:", "Ü")
+    if tone in (0, 5):
+        return syl
+    low = syl.lower()
+    pos = -1
+    if "a" in low:
+        pos = low.index("a")
+    elif "e" in low:
+        pos = low.index("e")
+    elif "ou" in low:
+        pos = low.index("o")
+    else:
+        for i in range(len(low) - 1, -1, -1):
+            if low[i] in _VOWELS:
+                pos = i
+                break
+    if pos == -1:  # syllabic nasal (m, n, ng, hm, hng): mark the first nasal
+        for i, c in enumerate(low):
+            if c in "mn":
+                pos = i
+                break
+    if pos == -1:
+        return syl
+    marked = unicodedata.normalize(
+        "NFC", syl[:pos + 1] + _TONE_COMBINING[tone] + syl[pos + 1:])
+    return marked
+
+
+_NUM_SYL = re.compile(r"([a-zA-ZüÜ:]+)([0-5])")
+
+
+def _pinyin_number_to_mark(text: str) -> str:
+    return _NUM_SYL.sub(lambda m: _mark_syllable(m.group(1), int(m.group(2))), text)
+
+
+def _split_run(run: str) -> List[str]:
+    """Split one letter run into syllables, one tone mark each (best effort)."""
+    decomp = unicodedata.normalize("NFD", run)
+    mark_positions = [i for i, c in enumerate(decomp) if c in _COMBINING_TONE]
+    if len(mark_positions) <= 1:
+        return [run]
+    cuts = []
+    for mp in mark_positions[1:]:
+        # walk back over the marked vowel's preceding consonant cluster and
+        # cut before the longest valid onset
+        j = mp - 1
+        while j >= 0 and decomp[j] in _COMBINING_TONE:
+            j -= 1
+        # j is the vowel carrying the mark; find start of consonant cluster
+        k = j
+        while k > 0 and decomp[k - 1].lower() not in _VOWELS \
+                and decomp[k - 1] not in _COMBINING_TONE:
+            k -= 1
+        cluster = decomp[k:j]
+        cut = j
+        for size in (2, 1):
+            if len(cluster) >= size and cluster[-size:].lower() in _ONSETS:
+                cut = j - size
+                break
+        cuts.append(cut)
+    pieces, prev = [], 0
+    for cut in cuts:
+        pieces.append(unicodedata.normalize("NFC", decomp[prev:cut]))
+        prev = cut
+    pieces.append(unicodedata.normalize("NFC", decomp[prev:]))
+    return [p for p in pieces if p]
+
+
+_LETTER_RUN = re.compile(r"[^\W\d_]+", re.UNICODE)
+
+
+def _pinyin_mark_to_number(text: str) -> str:
+    def _convert_run(m: re.Match) -> str:
+        out = []
+        for syl in _split_run(m.group(0)):
+            decomp = unicodedata.normalize("NFD", syl)
+            tone = 5
+            kept = []
+            for c in decomp:
+                if c in _COMBINING_TONE:
+                    tone = _COMBINING_TONE[c]
+                else:
+                    kept.append(c)
+            out.append(unicodedata.normalize("NFC", "".join(kept)) + str(tone))
+        return "".join(out)
+    return _LETTER_RUN.sub(_convert_run, text)
+
+
+def _pinyin_strip(text: str) -> str:
+    def _strip_run(m: re.Match) -> str:
+        decomp = unicodedata.normalize("NFD", m.group(0))
+        kept = "".join(c for c in decomp if c not in _COMBINING_TONE)
+        return unicodedata.normalize("NFC", kept)
+    # digits are removed only when attached to a letter (tone digits), so
+    # ordinary numerals in surrounding text survive
+    return re.sub(r"(?<=[^\W\d_])[0-5]", "", _LETTER_RUN.sub(_strip_run, text))
+
+
+# ---------------------------------------------------------------------------
+# Jamo form
+# ---------------------------------------------------------------------------
+# Conjoining→compatibility is a direct table.  Compatibility→conjoining needs
+# position: compatibility jamo do not distinguish onset from coda (both ㄱ are
+# U+3131), so a consonant is treated as an onset when a vowel follows,
+# otherwise as a coda — deterministic over well-formed jamo sequences.
+
+def _jamo_tables():
+    from scriptconv.translit import _ONSETS as _CO, _VOWELS as _CV, _CODAS as _CT
+    conj_to_compat = {}
+    compat_onset = {}
+    compat_vowel = {}
+    compat_coda = {}
+    for i, ch in enumerate(_CO):
+        conj_to_compat[chr(0x1100 + i)] = ch
+        compat_onset[ch] = chr(0x1100 + i)
+    for i, ch in enumerate(_CV):
+        conj_to_compat[chr(0x1161 + i)] = ch
+        compat_vowel[ch] = chr(0x1161 + i)
+    for i, ch in enumerate(_CT):
+        if ch:
+            conj_to_compat[chr(0x11A8 + i - 1)] = ch
+            compat_coda[ch] = chr(0x11A8 + i - 1)
+    return conj_to_compat, compat_onset, compat_vowel, compat_coda
+
+
+_JAMO_TABLES = None
+
+
+def _jamo_conjoining_to_compat(text: str) -> str:
+    global _JAMO_TABLES
+    if _JAMO_TABLES is None:
+        _JAMO_TABLES = _jamo_tables()
+    table = _JAMO_TABLES[0]
+    return "".join(table.get(c, c) for c in text)
+
+
+def _jamo_compat_to_conjoining(text: str) -> str:
+    global _JAMO_TABLES
+    if _JAMO_TABLES is None:
+        _JAMO_TABLES = _jamo_tables()
+    _, onset, vowel, coda = _JAMO_TABLES
+    out = []
+    for i, c in enumerate(text):
+        if c in vowel:
+            out.append(vowel[c])
+        elif c in onset or c in coda:
+            nxt = text[i + 1] if i + 1 < len(text) else ""
+            if nxt in vowel and c in onset:
+                out.append(onset[c])
+            elif c in coda:
+                out.append(coda[c])
+            else:
+                out.append(onset[c])
+        else:
+            out.append(c)
+    return "".join(out)
+
+
+def _jamo_detect(text: str) -> Optional[str]:
+    for c in text:
+        cp = ord(c)
+        if 0x1100 <= cp <= 0x11FF:
+            return "conjoining"
+        if 0x3130 <= cp <= 0x318F:
+            return "compatibility"
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Transition matrix and operations
+# ---------------------------------------------------------------------------
+
+_TRANSITIONS: Dict[Tuple[str, str, str], Callable[[str], str]] = {
+    ("tashkeel", "marked", "none"): _range_strip(_TASHKEEL_RANGES),
+    ("kashida", "marked", "none"): _range_strip(_KASHIDA_RANGES),
+    ("quranic-marks", "marked", "none"): _range_strip(_QURANIC_RANGES),
+    ("niqqud", "marked", "none"): _range_strip(_NIQQUD_RANGES),
+    ("teamim", "marked", "none"): _range_strip(_TEAMIM_RANGES),
+    ("wakachigaki", "marked", "none"): _wakachigaki_strip,
+    ("wakachigaki", "none", "marked"): _wakachigaki_apply,
+    ("pinyin-tone", "mark", "number"): _pinyin_mark_to_number,
+    ("pinyin-tone", "number", "mark"): _pinyin_number_to_mark,
+    ("pinyin-tone", "mark", "none"): _pinyin_strip,
+    ("pinyin-tone", "number", "none"): _pinyin_strip,
+    ("jamo-form", "conjoining", "compatibility"): _jamo_conjoining_to_compat,
+    ("jamo-form", "compatibility", "conjoining"): _jamo_compat_to_conjoining,
+}
+
+
+def restyle(text: str, convention_id: str, to: str,
+            frm: Optional[str] = None) -> str:
+    """Rewrite *text* from one style of a convention to another.
+
+    ``frm`` defaults to the convention's marked/default style for strips and
+    is required where the source style is not inferable.  Unsupported
+    transitions raise :class:`ValueError`; transitions that need a missing
+    optional dependency raise :class:`ImportError` with an install hint.
+    """
+    conv = _conv(convention_id)
+    if to not in conv.styles and to != "none":
+        raise ValueError(f"{convention_id!r} has styles {conv.styles}, not {to!r}")
+    if frm is None:
+        if conv.id == "jamo-form":
+            frm = _jamo_detect(text) or conv.default_style
+        else:
+            sources = sorted({f for (c, f, t) in _TRANSITIONS
+                              if c == convention_id and t == to})
+            if len(sources) == 1:
+                frm = sources[0]
+            elif conv.default_style != to:
+                frm = conv.default_style
+            else:
+                raise ValueError(
+                    f"{convention_id!r}: source style for ->{to!r} is ambiguous "
+                    f"({', '.join(sources)}); pass frm=")
+    if frm == to:
+        return text
+    fn = _TRANSITIONS.get((convention_id, frm, to))
+    if fn is None:
+        supported = sorted(f"{f}->{t}" for (c, f, t) in _TRANSITIONS
+                           if c == convention_id)
+        raise ValueError(
+            f"{convention_id!r} does not define {frm!r}->{to!r}; "
+            f"supported: {', '.join(supported)}")
+    return fn(text)
+
+
+def strip(text: str, convention_id: str) -> str:
+    """Remove a convention's decoration (restyle to ``"none"``)."""
+    conv = _conv(convention_id)
+    if "none" not in conv.styles:
+        raise ValueError(f"{convention_id!r} has no unmarked style; "
+                         f"styles: {conv.styles}")
+    frm = "mark" if conv.id == "pinyin-tone" else "marked"
+    if conv.id == "pinyin-tone":
+        # digits and marks are both handled by the strip transition
+        return _pinyin_strip(text)
+    return restyle(text, convention_id, "none", frm=frm)
+
+
+def apply(text: str, convention_id: str, style: Optional[str] = None) -> str:
+    """Add a convention's decoration to unmarked text (restyle from ``"none"``)."""
+    conv = _conv(convention_id)
+    if style is None:
+        marked = [s for s in conv.styles if s != "none"]
+        style = conv.default_style if conv.default_style != "none" else marked[0]
+    return restyle(text, convention_id, style, frm="none")
+
+
+def detect(text: str, convention_id: str) -> Optional[str]:
+    """Current style of *text* under a convention, or None when not character-evident.
+
+    For binary decorations this means "any mark present" — a single shadda in
+    a paragraph reports ``"marked"``.
+    """
+    conv = _conv(convention_id)
+    if not conv.detectable:
+        return None
+    if conv.id == "jamo-form":
+        return _jamo_detect(text)
+    return "marked" if any(_in_ranges(ord(c), conv.ranges) for c in text) else "none"

{scriptconv-0.0.3a5 → scriptconv-0.0.3a7}/scriptconv/readings.py

@@ -22,7 +22,17 @@ unconverted text.
 """
 from __future__ import annotations
 
-__all__ = ["to_hiragana", "to_katakana", "to_pinyin", "to_bopomofo"]
+from typing import Iterator, NamedTuple
+
+__all__ = ["ReadingToken", "tokens", "to_hiragana", "to_katakana",
+           "to_pinyin", "to_bopomofo"]
+
+
+class ReadingToken(NamedTuple):
+    """One dictionary-segmented token of Japanese text and its kana readings."""
+    orig: str
+    hira: str
+    kana: str
 
 _kakasi = None
 
@@ -44,32 +54,53 @@ def _converter():
     return _kakasi
 
 
-def to_hiragana(text: str, keep_katakana: bool = False) -> str:
+def tokens(text: str) -> Iterator[ReadingToken]:
+    """Iterate the dictionary's segmentation of *text* with per-token readings.
+
+    Each token is a :class:`ReadingToken` with the original surface form and
+    its hiragana and katakana readings.  This is the primitive under
+    :func:`to_hiragana`/:func:`to_katakana`; use it directly when the consumer
+    needs token boundaries — e.g. word segmentation (wakachigaki) or
+    reading-dependent post-processing — rather than a joined string.
+    """
+    for t in _converter().convert(text):
+        if t["orig"]:
+            yield ReadingToken(t["orig"], t["hira"], t["kana"])
+
+
+def to_hiragana(text: str, keep_katakana: bool = False,
+                segment: bool = False) -> str:
     """Respell Japanese text in hiragana.
 
     Kanji are resolved to their dictionary reading; existing kana and any
     non-Japanese characters pass through.  With ``keep_katakana=True``,
     katakana tokens keep their original script (useful when katakana marks
-    loanwords deliberately) instead of being folded into hiragana.
+    loanwords deliberately) instead of being folded into hiragana.  With
+    ``segment=True``, dictionary tokens are joined with spaces (wakachigaki,
+    the spaced orthographic mode) instead of concatenated — equivalent to
+    ``scriptconv.apply(..., "wakachigaki")`` over this token stream
+    (:mod:`scriptconv.conventions`).
     """
     out = []
-    for token in _converter().convert(text):
-        orig = token["orig"]
-        if keep_katakana and orig and all(ord(c) in _KANA or not _is_japanese(c)
-                                          for c in orig):
-            out.append(orig)
+    for tok in tokens(text):
+        if keep_katakana and tok.orig and all(
+                ord(c) in _KANA or not _is_japanese(c) for c in tok.orig):
+            out.append(tok.orig)
         else:
-            out.append(token["hira"])
-    return "".join(out)
+            out.append(tok.hira)
+    return " ".join(" ".join(out).split()) if segment else "".join(out)
 
 
-def to_katakana(text: str) -> str:
+def to_katakana(text: str, segment: bool = False) -> str:
     """Respell Japanese text in katakana.
 
     Kanji are resolved to their dictionary reading; existing kana are
     transposed to katakana and any non-Japanese characters pass through.
+    With ``segment=True``, dictionary tokens are joined with spaces
+    (wakachigaki) instead of concatenated.
     """
-    return "".join(token["kana"] for token in _converter().convert(text))
+    kana = [tok.kana for tok in tokens(text)]
+    return " ".join(" ".join(kana).split()) if segment else "".join(kana)
 
 
 def _is_japanese(ch: str) -> bool:
@@ -105,7 +136,10 @@ def to_pinyin(text: str, tone: str = "mark") -> str:
     Hanzi are resolved to their dictionary reading (heteronyms decided by
     pypinyin's phrase dictionary); non-Chinese characters pass through.
     ``tone`` selects the tone spelling: ``"mark"`` (``zhōng``, default),
-    ``"number"`` (``zhong1``) or ``"none"`` (``zhong``).
+    ``"number"`` (``zhong1``) or ``"none"`` (``zhong``) — the styles of the
+    ``pinyin-tone`` convention (:mod:`scriptconv.conventions`); use
+    :func:`scriptconv.restyle` to convert existing pinyin between them
+    without the ``zh`` extra.
     """
     return _pinyin(text, tone)
 

{scriptconv-0.0.3a5 → scriptconv-0.0.3a7}/scriptconv/translit.py

@@ -52,6 +52,8 @@ def decompose_hangul(
     text:
         Input string.
     form:
+        A style of the ``jamo-form`` convention
+        (:mod:`scriptconv.conventions`);
         ``"compatibility"`` (default) emits Hangul Compatibility Jamo
         (U+3130 block, e.g. ``ㄱ``); ``"conjoining"`` emits conjoining
         jamo (U+1100/1161/11A8 blocks) that recombine into syllables.

{scriptconv-0.0.3a5 → scriptconv-0.0.3a7}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 3
-VERSION_ALPHA = 5
+VERSION_ALPHA = 7
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.3a5 → scriptconv-0.0.3a7}/scriptconv.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a5
+Version: 0.0.3a7
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv

{scriptconv-0.0.3a5 → scriptconv-0.0.3a7}/scriptconv.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ requirements.txt
 scriptconv/__init__.py
 scriptconv/__main__.py
 scriptconv/cangjie.py
+scriptconv/conventions.py
 scriptconv/notation.py
 scriptconv/py.typed
 scriptconv/readings.py
@@ -20,6 +21,7 @@ scriptconv/data/__init__.py
 scriptconv/data/cangjie5_tc.tsv.gz
 tests/test_cangjie.py
 tests/test_cli.py
+tests/test_conventions.py
 tests/test_examples.py
 tests/test_notation.py
 tests/test_readings.py

{scriptconv-0.0.3a5 → scriptconv-0.0.3a7}/tests/test_cli.py

@@ -86,3 +86,25 @@ def test_convert_bad_notation_clean_error(capsys):
     msg = str(exc.value)
     assert msg.startswith("error:")
     assert "valid notations:" in msg
+
+
+def test_strip_tashkeel(capsys):
+    assert main(["strip", "tashkeel", "مُحَمَّد"]) == 0
+    assert capsys.readouterr().out.strip() == "محمد"
+
+
+def test_restyle_pinyin(capsys):
+    assert main(["restyle", "pinyin-tone", "mark", "zhong1 guo2"]) == 0
+    assert capsys.readouterr().out.strip() == "zhōng guó"
+
+
+def test_conventions_listing_filtered(capsys):
+    assert main(["conventions", "--script", "Arab"]) == 0
+    out = capsys.readouterr().out
+    assert "tashkeel" in out and "kashida" in out and "niqqud" not in out
+
+
+def test_strip_unknown_convention_clean_error():
+    with pytest.raises(SystemExit) as exc:
+        main(["strip", "bogus", "text"])
+    assert str(exc.value).startswith("error:")

scriptconv-0.0.3a7/tests/test_conventions.py

@@ -0,0 +1,210 @@
+import unicodedata
+import unittest
+
+from scriptconv import (
+    CONVENTION_REGISTRY,
+    apply,
+    conventions_for,
+    decompose_hangul,
+    detect_convention,
+    restyle,
+    strip,
+)
+
+
+class TestRegistryInvariants(unittest.TestCase):
+    def test_default_style_is_a_declared_style(self):
+        for c in CONVENTION_REGISTRY.values():
+            self.assertIn(c.default_style, c.styles, c.id)
+
+    def test_sibling_conventions_on_same_script_have_disjoint_ranges(self):
+        convs = list(CONVENTION_REGISTRY.values())
+        for i, a in enumerate(convs):
+            for b in convs[i + 1:]:
+                if not set(a.scripts) & set(b.scripts):
+                    continue
+                a_cps = {cp for lo, hi in a.ranges for cp in range(lo, hi + 1)}
+                b_cps = {cp for lo, hi in b.ranges for cp in range(lo, hi + 1)}
+                self.assertFalse(a_cps & b_cps, f"{a.id} overlaps {b.id}")
+
+    def test_strippable_conventions_declare_none_style(self):
+        for c in CONVENTION_REGISTRY.values():
+            if c.id == "jamo-form":
+                self.assertNotIn("none", c.styles)  # no unmarked state exists
+            else:
+                self.assertIn("none", c.styles, c.id)
+
+    def test_conventions_for_surfaces_system(self):
+        latn = conventions_for("Latn")
+        self.assertEqual([c.id for c in latn], ["pinyin-tone"])
+        self.assertEqual(latn[0].system, "pinyin")
+
+    def test_unknown_convention_raises_with_known_ids(self):
+        with self.assertRaises(ValueError) as ctx:
+            strip("x", "vowel-harmony")
+        self.assertIn("tashkeel", str(ctx.exception))
+
+
+class TestTashkeel(unittest.TestCase):
+    def test_strips_full_vocalization(self):
+        self.assertEqual(strip("مُحَمَّدٌ", "tashkeel"), "محمد")
+
+    def test_nfd_hamza_and_madda_survive(self):
+        # آ أ إ decomposed to alef + combining madda/hamza — letter identity,
+        # must NOT be stripped
+        nfd = unicodedata.normalize("NFD", "آمين أب إن")
+        out = strip(nfd, "tashkeel")
+        self.assertEqual(unicodedata.normalize("NFC", out), "آمين أب إن")
+
+    def test_dagger_alif_stripped(self):
+        self.assertEqual(strip("رحمٰن", "tashkeel"), "رحمن")
+
+    def test_lone_shadda_detects_marked(self):
+        self.assertEqual(detect_convention("plain text مدّ here", "tashkeel"), "marked")
+        self.assertEqual(detect_convention("محمد", "tashkeel"), "none")
+
+    def test_no_apply_transition(self):
+        with self.assertRaises(ValueError):
+            apply("محمد", "tashkeel")
+
+
+class TestKashidaAndQuranic(unittest.TestCase):
+    def test_kashida_stripped_independently_of_tashkeel(self):
+        self.assertEqual(strip("محمـــد", "kashida"), "محمد")
+        # tashkeel strip leaves tatweel alone
+        self.assertEqual(strip("محمـــد", "tashkeel"), "محمـــد")
+
+    def test_quranic_sajdah_sign_stripped(self):
+        text = "قرأ۩"
+        self.assertEqual(strip(text, "quranic-marks"), "قرأ")
+
+
+class TestHebrew(unittest.TestCase):
+    def test_niqqud_stripped_points_and_dagesh(self):
+        self.assertEqual(strip("שָׁלוֹם", "niqqud"), "שלום")
+
+    def test_maqaf_and_sof_pasuq_survive_both_strips(self):
+        text = "כָּל־הָעוֹלָם׃"
+        for conv in ("niqqud", "teamim"):
+            out = strip(text, conv)
+            self.assertIn("־", out)
+            self.assertIn("׃", out)
+
+    def test_teamim_stripped_niqqud_kept(self):
+        # etnahta U+0591 removed, qamats U+05B8 kept
+        text = "וַ֑יֹּאמֶר"
+        out = strip(text, "teamim")
+        self.assertNotIn("֑", out)
+        self.assertIn("ַ", unicodedata.normalize("NFD", out))
+
+    def test_detect_distinguishes_layers(self):
+        pointed_only = "שָׁלוֹם"
+        self.assertEqual(detect_convention(pointed_only, "niqqud"), "marked")
+        self.assertEqual(detect_convention(pointed_only, "teamim"), "none")
+
+
+class TestWakachigaki(unittest.TestCase):
+    def test_strip_removes_spaces_between_japanese(self):
+        self.assertEqual(strip("わたし は がくせい です", "wakachigaki"),
+                         "わたしはがくせいです")
+
+    def test_ideographic_space_stripped(self):
+        self.assertEqual(strip("東京　大阪", "wakachigaki"), "東京大阪")
+
+    def test_spaces_next_to_latin_survive(self):
+        self.assertEqual(strip("きょうは good day", "wakachigaki"),
+                         "きょうは good day")
+
+    def test_spaces_next_to_digits_survive(self):
+        self.assertEqual(strip("第 3 章", "wakachigaki"), "第 3 章")
+
+    def test_not_detectable(self):
+        self.assertIsNone(detect_convention("わたし は", "wakachigaki"))
+
+    def test_apply_strip_round_trip_pure_japanese(self):
+        text = "私は学生です"
+        spaced = apply(text, "wakachigaki")
+        self.assertIn(" ", spaced)
+        self.assertEqual(strip(spaced, "wakachigaki"), text)
+
+
+class TestPinyinTone(unittest.TestCase):
+    def test_number_to_mark_basic(self):
+        self.assertEqual(restyle("zhong1 guo2 ren2", "pinyin-tone", "mark"),
+                         "zhōng guó rén")
+
+    def test_umlaut_v_and_colon_variants(self):
+        self.assertEqual(restyle("lv4 nu:3", "pinyin-tone", "mark"), "lǜ nǚ")
+
+    def test_capitals_and_apostrophe(self):
+        self.assertEqual(restyle("Xi1'an1", "pinyin-tone", "mark"), "Xī'ān")
+
+    def test_erhua_and_syllabic_nasal(self):
+        self.assertEqual(restyle("huar2", "pinyin-tone", "mark"), "huár")
+        self.assertEqual(restyle("m2", "pinyin-tone", "mark"), "ḿ")
+
+    def test_ou_placement_rule(self):
+        self.assertEqual(restyle("gou3", "pinyin-tone", "mark"), "gǒu")
+        self.assertEqual(restyle("guo2", "pinyin-tone", "mark"), "guó")
+        self.assertEqual(restyle("jiu3", "pinyin-tone", "mark"), "jiǔ")
+
+    def test_neutral_tone_unmarked(self):
+        self.assertEqual(restyle("ma5 de0", "pinyin-tone", "mark"), "ma de")
+
+    def test_mark_to_number_compliant(self):
+        self.assertEqual(restyle("zhōng guó", "pinyin-tone", "number", frm="mark"),
+                         "zhong1 guo2")
+        self.assertEqual(restyle("Xī'ān", "pinyin-tone", "number", frm="mark"),
+                         "Xi1'an1")
+
+    def test_mark_to_number_unapostrophized_best_effort(self):
+        # documented best-effort split: cut before the longest valid onset
+        self.assertEqual(restyle("fāngān", "pinyin-tone", "number", frm="mark"),
+                         "fan1gan1")
+
+    def test_strip_handles_both_styles_and_spares_ordinary_digits(self):
+        self.assertEqual(strip("Nǐ hǎo ma5", "pinyin-tone"), "Ni hao ma")
+        self.assertEqual(strip("di4 3 ceng2", "pinyin-tone"), "di 3 ceng")
+
+    def test_restyle_to_mark_infers_number_source(self):
+        self.assertEqual(restyle("hao3", "pinyin-tone", "mark"), "hǎo")
+
+
+class TestJamoForm(unittest.TestCase):
+    def test_round_trip_byte_identical(self):
+        compat = decompose_hangul("한국말")
+        conj = restyle(compat, "jamo-form", "conjoining")
+        self.assertEqual(restyle(conj, "jamo-form", "compatibility"), compat)
+
+    def test_agrees_with_decompose_hangul(self):
+        for word in ("안녕", "값", "한국말"):
+            self.assertEqual(
+                restyle(decompose_hangul(word), "jamo-form", "conjoining"),
+                decompose_hangul(word, form="conjoining"))
+
+    def test_detect_repertoire(self):
+        self.assertEqual(detect_convention("ㄱㅏ", "jamo-form"), "compatibility")
+        self.assertEqual(detect_convention("가", "jamo-form"), "conjoining")
+        self.assertIsNone(detect_convention("hangul-free", "jamo-form"))
+
+    def test_no_none_style(self):
+        with self.assertRaises(ValueError):
+            strip("ㄱㅏ", "jamo-form")
+
+
+class TestRestyleContract(unittest.TestCase):
+    def test_unsupported_transition_names_supported_ones(self):
+        with self.assertRaises(ValueError) as ctx:
+            restyle("x", "tashkeel", "marked", frm="none")
+        self.assertIn("marked->none", str(ctx.exception))
+
+    def test_same_style_is_identity(self):
+        self.assertEqual(restyle("abc", "pinyin-tone", "mark", frm="mark"), "abc")
+
+    def test_invalid_target_style_raises(self):
+        with self.assertRaises(ValueError):
+            restyle("abc", "pinyin-tone", "cyrillic")
+
+
+if __name__ == "__main__":
+    unittest.main()

{scriptconv-0.0.3a5 → scriptconv-0.0.3a7}/tests/test_readings.py

@@ -71,3 +71,45 @@ class TestDependencyBoundary(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class TestTokens(unittest.TestCase):
+    def test_yields_reading_tokens_with_all_forms(self):
+        from scriptconv import ReadingToken, tokens
+        toks = list(tokens("東京タワー"))
+        self.assertTrue(all(isinstance(t, ReadingToken) for t in toks))
+        origs = [t.orig for t in toks]
+        self.assertIn("東京", origs)
+        tokyo = toks[origs.index("東京")]
+        self.assertEqual(tokyo.hira, "とうきょう")
+        self.assertEqual(tokyo.kana, "トウキョウ")
+
+    def test_tokens_cover_input_exactly(self):
+        from scriptconv import tokens
+        text = "私はcoffeeが好き"
+        self.assertEqual("".join(t.orig for t in tokens(text)), text)
+
+    def test_empty_string_yields_nothing(self):
+        from scriptconv import tokens
+        self.assertEqual(list(tokens("")), [])
+
+
+class TestSegmentFlag(unittest.TestCase):
+    def test_hiragana_wakachigaki(self):
+        from scriptconv import to_hiragana
+        self.assertEqual(to_hiragana("私は学生です", segment=True),
+                         "わたし は がくせい です")
+
+    def test_katakana_wakachigaki(self):
+        from scriptconv import to_katakana
+        self.assertEqual(to_katakana("私は学生です", segment=True),
+                         "ワタシ ハ ガクセイ デス")
+
+    def test_segment_false_is_default_concatenation(self):
+        from scriptconv import to_hiragana
+        self.assertEqual(to_hiragana("私は学生です"), "わたしはがくせいです")
+
+    def test_segment_collapses_original_whitespace(self):
+        from scriptconv import to_hiragana
+        out = to_hiragana("東京 大阪", segment=True)
+        self.assertNotIn("  ", out)