scriptconv 0.0.3a2__tar.gz → 0.0.3a4__tar.gz

{scriptconv-0.0.3a2 → scriptconv-0.0.3a4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a2
+Version: 0.0.3a4
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -22,10 +22,13 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Provides-Extra: ja
 Requires-Dist: pykakasi<3,>=2.3; extra == "ja"
+Provides-Extra: zh
+Requires-Dist: pypinyin<1,>=0.50; extra == "zh"
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-timeout; extra == "test"
 Requires-Dist: pykakasi<3,>=2.3; extra == "test"
+Requires-Dist: pypinyin<1,>=0.50; extra == "test"
 Dynamic: license-file
 
 # scriptconv

{scriptconv-0.0.3a2 → scriptconv-0.0.3a4}/pyproject.toml

@@ -34,7 +34,8 @@ dependencies = {file = ["requirements.txt"]}
 
 [project.optional-dependencies]
 ja = ["pykakasi>=2.3,<3"]
-test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3"]
+zh = ["pypinyin>=0.50,<1"]
+test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1"]
 
 [project.urls]
 Homepage = "https://github.com/TigreGotico/scriptconv"

{scriptconv-0.0.3a2 → scriptconv-0.0.3a4}/scriptconv/__init__.py

@@ -18,11 +18,13 @@ translit:
     Script-level decomposition and transliteration (Hangul → jamo,
     Hiragana ↔ Katakana).
 readings:
-    Dictionary-backed respelling (Japanese kanji → kana).  Needs the
-    optional ``pykakasi`` dependency (``pip install scriptconv[ja]``).
+    Dictionary-backed respelling (Japanese kanji → kana via the ``ja``
+    extra; Chinese hanzi → pinyin/bopomofo via the ``zh`` extra).
+cangjie:
+    Hanzi → Cangjie5 input codes (shape decomposition, vendored table).
 
 Zero required runtime dependencies (stdlib only); the readings module
-needs the optional ``ja`` extra.
+needs the optional ``ja``/``zh`` extras.
 """
 
 from scriptconv.scripts import (
@@ -68,6 +70,12 @@ from scriptconv.translit import (
 from scriptconv.readings import (
     to_hiragana,
     to_katakana,
+    to_pinyin,
+    to_bopomofo,
+)
+from scriptconv.cangjie import (
+    cangjie_code,
+    to_cangjie,
 )
 
 __all__ = [
@@ -111,4 +119,9 @@ __all__ = [
     # readings (dictionary-backed, needs scriptconv[ja])
     "to_hiragana",
     "to_katakana",
+    # cangjie (vendored Cangjie5 table)
+    "cangjie_code",
+    "to_cangjie",
+    "to_pinyin",
+    "to_bopomofo",
 ]

scriptconv-0.0.3a4/scriptconv/cangjie.py

@@ -0,0 +1,63 @@
+"""Hanzi → Cangjie input codes (fifth generation, traditional table).
+
+Cangjie codes encode a glyph's SHAPE — its decomposition into geometric
+radicals — not its reading: 日 is ``a`` because of what it looks like, however
+it is pronounced.  That makes this a deterministic, table-driven orthographic
+transcode like the rest of scriptconv's core: no reading dictionary, no
+ambiguity, no phonology.
+
+The table is vendored (gzipped, loaded lazily on first use) from the Cangjie5
+project — https://github.com/Jackchows/Cangjie5, MIT License — so no runtime
+download or extra dependency is needed.  Where the upstream table lists
+alternative codes for a glyph, the primary (first) code is kept.
+"""
+from __future__ import annotations
+
+import gzip
+from importlib.resources import files
+from typing import Dict, Optional
+
+__all__ = ["cangjie_code", "to_cangjie"]
+
+_TABLE: Optional[Dict[str, str]] = None
+
+
+def _table() -> Dict[str, str]:
+    global _TABLE
+    if _TABLE is None:
+        _TABLE = {}
+        raw = files("scriptconv.data").joinpath("cangjie5_tc.tsv.gz").read_bytes()
+        for line in gzip.decompress(raw).decode("utf-8").splitlines():
+            if line.startswith("#"):
+                continue
+            ch, _, code = line.partition("\t")
+            _TABLE[ch] = code
+    return _TABLE
+
+
+def cangjie_code(char: str) -> Optional[str]:
+    """Return the Cangjie5 code for a single glyph, or None if unmapped."""
+    return _table().get(char)
+
+
+def to_cangjie(text: str, sep: str = " ") -> str:
+    """Transcode hanzi in *text* to their Cangjie5 codes.
+
+    Mapped glyphs become their code, joined by *sep*; runs of unmapped
+    characters (Latin, digits, punctuation, rare glyphs) pass through
+    unchanged as single tokens.
+    """
+    table = _table()
+    tokens = []
+    last_raw = False
+    for ch in text:
+        code = table.get(ch)
+        if code is not None:
+            tokens.append(code)
+            last_raw = False
+        elif last_raw:
+            tokens[-1] += ch
+        else:
+            tokens.append(ch)
+            last_raw = True
+    return sep.join(tokens)

{scriptconv-0.0.3a2 → scriptconv-0.0.3a4}/scriptconv/readings.py

@@ -1,4 +1,4 @@
-"""Dictionary-backed script respelling (Japanese kanji → kana).
+"""Dictionary-backed script respelling (Japanese kanji → kana, Chinese hanzi → pinyin/bopomofo).
 
 Unlike the table-driven modules (:mod:`scriptconv.translit`,
 :mod:`scriptconv.notation`), converting kanji to kana is not a codepoint
@@ -8,15 +8,21 @@ dictionary's segmentation.  This is still orthography — the output is how the
 word is written in kana, exactly as an IME or furigana would render it — not
 phonemization; no sound rules are applied.
 
-Because it needs a reading dictionary, this module requires the optional
-``pykakasi`` dependency (``pip install scriptconv[ja]``).  Importing the
+The same reasoning covers Chinese: a hanzi's pinyin or bopomofo spelling is a
+lexical property with genuinely ambiguous readings (heteronyms such as 行
+xíng/háng) resolved by a phrase dictionary — standard orthographic respelling,
+not phonemization.
+
+Because it needs reading dictionaries, this module requires optional
+dependencies: ``pykakasi`` for Japanese (``pip install scriptconv[ja]``) and
+``pypinyin`` for Chinese (``pip install scriptconv[zh]``).  Importing the
 module is always safe; the dependency is resolved on first conversion and a
 missing dictionary raises :class:`ImportError` rather than silently returning
 unconverted text.
 """
 from __future__ import annotations
 
-__all__ = ["to_hiragana", "to_katakana"]
+__all__ = ["to_hiragana", "to_katakana", "to_pinyin", "to_bopomofo"]
 
 _kakasi = None
 
@@ -69,3 +75,45 @@ def to_katakana(text: str) -> str:
 def _is_japanese(ch: str) -> bool:
     cp = ord(ch)
     return cp in _HIRA or cp in _KANA or 0x4E00 <= cp <= 0x9FFF
+
+
+_PINYIN_STYLES = None
+
+
+def _pinyin(text: str, style_name: str) -> str:
+    global _PINYIN_STYLES
+    if _PINYIN_STYLES is None:
+        try:
+            from pypinyin import Style, lazy_pinyin
+        except ImportError:
+            raise ImportError(
+                "hanzi→pinyin/bopomofo conversion needs the phrase dictionary "
+                "from pypinyin — install with `pip install scriptconv[zh]`"
+            ) from None
+        _PINYIN_STYLES = {"mark": Style.TONE, "number": Style.TONE3,
+                          "none": Style.NORMAL, "bopomofo": Style.BOPOMOFO,
+                          "_fn": lazy_pinyin}
+    if style_name not in _PINYIN_STYLES:
+        raise ValueError(f"tone must be one of 'mark', 'number', 'none' — got {style_name!r}")
+    tokens = _PINYIN_STYLES["_fn"](text, style=_PINYIN_STYLES[style_name])
+    return " ".join(" ".join(tokens).split())
+
+
+def to_pinyin(text: str, tone: str = "mark") -> str:
+    """Respell Chinese text in pinyin, syllables space-separated.
+
+    Hanzi are resolved to their dictionary reading (heteronyms decided by
+    pypinyin's phrase dictionary); non-Chinese characters pass through.
+    ``tone`` selects the tone spelling: ``"mark"`` (``zhōng``, default),
+    ``"number"`` (``zhong1``) or ``"none"`` (``zhong``).
+    """
+    return _pinyin(text, tone)
+
+
+def to_bopomofo(text: str) -> str:
+    """Respell Chinese text in bopomofo (zhuyin), syllables space-separated.
+
+    Hanzi are resolved to their dictionary reading; non-Chinese characters
+    pass through.  Tone marks follow zhuyin convention (first tone unmarked).
+    """
+    return _pinyin(text, "bopomofo")

{scriptconv-0.0.3a2 → scriptconv-0.0.3a4}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 3
-VERSION_ALPHA = 2
+VERSION_ALPHA = 4
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.3a2 → scriptconv-0.0.3a4}/scriptconv.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a2
+Version: 0.0.3a4
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -22,10 +22,13 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Provides-Extra: ja
 Requires-Dist: pykakasi<3,>=2.3; extra == "ja"
+Provides-Extra: zh
+Requires-Dist: pypinyin<1,>=0.50; extra == "zh"
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-timeout; extra == "test"
 Requires-Dist: pykakasi<3,>=2.3; extra == "test"
+Requires-Dist: pypinyin<1,>=0.50; extra == "test"
 Dynamic: license-file
 
 # scriptconv

{scriptconv-0.0.3a2 → scriptconv-0.0.3a4}/scriptconv.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ pyproject.toml
 requirements.txt
 scriptconv/__init__.py
 scriptconv/__main__.py
+scriptconv/cangjie.py
 scriptconv/notation.py
 scriptconv/py.typed
 scriptconv/readings.py
@@ -15,9 +16,12 @@ scriptconv.egg-info/SOURCES.txt
 scriptconv.egg-info/dependency_links.txt
 scriptconv.egg-info/requires.txt
 scriptconv.egg-info/top_level.txt
+scriptconv/data/__init__.py
+tests/test_cangjie.py
 tests/test_cli.py
 tests/test_examples.py
 tests/test_notation.py
 tests/test_readings.py
+tests/test_readings_zh.py
 tests/test_scripts.py
 tests/test_translit.py

{scriptconv-0.0.3a2 → scriptconv-0.0.3a4}/scriptconv.egg-info/requires.txt

@@ -6,3 +6,7 @@ pykakasi<3,>=2.3
 pytest
 pytest-timeout
 pykakasi<3,>=2.3
+pypinyin<1,>=0.50
+
+[zh]
+pypinyin<1,>=0.50

scriptconv-0.0.3a4/tests/test_cangjie.py

@@ -0,0 +1,51 @@
+import unittest
+
+from scriptconv import cangjie_code, to_cangjie
+from scriptconv import cangjie
+
+
+class TestCangjieCode(unittest.TestCase):
+    def test_official_single_radical_glyphs(self):
+        self.assertEqual(cangjie_code("日"), "a")
+        self.assertEqual(cangjie_code("木"), "d")
+
+    def test_multi_radical_glyph(self):
+        self.assertEqual(cangjie_code("昌"), "aa")
+
+    def test_unmapped_returns_none(self):
+        self.assertIsNone(cangjie_code("X"))
+        self.assertIsNone(cangjie_code("1"))
+        self.assertIsNone(cangjie_code(" "))
+
+    def test_codes_are_lowercase_alpha(self):
+        table = cangjie._table()
+        self.assertGreater(len(table), 100000)
+        sample = list(table.values())[:5000]
+        self.assertTrue(all(c.isalpha() and c.islower() for c in sample))
+
+
+class TestToCangjie(unittest.TestCase):
+    def test_all_hanzi_sentence(self):
+        self.assertEqual(to_cangjie("倉頡"), "oiar grmbc")
+
+    def test_custom_separator(self):
+        self.assertEqual(to_cangjie("倉頡", sep="|"), "oiar|grmbc")
+
+    def test_unmapped_runs_stay_contiguous(self):
+        self.assertEqual(to_cangjie("我有ABC 123!"), "hqi kb ABC 123!")
+
+    def test_non_chinese_only_passes_through_verbatim(self):
+        self.assertEqual(to_cangjie("OVOS 123!"), "OVOS 123!")
+
+    def test_empty_string(self):
+        self.assertEqual(to_cangjie(""), "")
+
+    def test_table_loaded_once(self):
+        to_cangjie("日")
+        first = cangjie._TABLE
+        cangjie_code("木")
+        self.assertIs(cangjie._TABLE, first)
+
+
+if __name__ == "__main__":
+    unittest.main()

scriptconv-0.0.3a4/tests/test_readings_zh.py

@@ -0,0 +1,78 @@
+import sys
+import unittest
+from unittest import mock
+
+from scriptconv import to_pinyin, to_bopomofo
+from scriptconv import readings
+
+
+class TestToPinyin(unittest.TestCase):
+    def test_hanzi_resolved_with_tone_marks(self):
+        self.assertEqual(to_pinyin("中国人"), "zhōng guó rén")
+
+    def test_tone_number_style(self):
+        self.assertEqual(to_pinyin("中国人", tone="number"), "zhong1 guo2 ren2")
+
+    def test_tone_none_style(self):
+        self.assertEqual(to_pinyin("中国人", tone="none"), "zhong guo ren")
+
+    def test_heteronym_resolved_by_phrase_context(self):
+        # 行 reads háng in 银行 (bank) but xíng in 行走 (to walk)
+        self.assertIn("háng", to_pinyin("银行"))
+        self.assertIn("xíng", to_pinyin("行走"))
+
+    def test_non_chinese_passes_through(self):
+        self.assertEqual(to_pinyin("OVOS 123!"), "OVOS 123!")
+
+    def test_mixed_script_sentence(self):
+        out = to_pinyin("我有3个apple")
+        self.assertIn("wǒ", out)
+        self.assertIn("apple", out)
+
+    def test_invalid_tone_raises_value_error(self):
+        with self.assertRaises(ValueError):
+            to_pinyin("中", tone="fancy")
+
+    def test_empty_string(self):
+        self.assertEqual(to_pinyin(""), "")
+
+
+class TestToBopomofo(unittest.TestCase):
+    def test_hanzi_resolved_to_zhuyin(self):
+        self.assertEqual(to_bopomofo("中国"), "ㄓㄨㄥ ㄍㄨㄛˊ")
+
+    def test_first_tone_unmarked_second_tone_marked(self):
+        out = to_bopomofo("中国")
+        self.assertNotIn("ˉ", out)
+        self.assertIn("ˊ", out)
+
+    def test_non_chinese_passes_through(self):
+        self.assertEqual(to_bopomofo("OVOS 123!"), "OVOS 123!")
+
+    def test_empty_string(self):
+        self.assertEqual(to_bopomofo(""), "")
+
+
+class TestDependencyBoundary(unittest.TestCase):
+    def test_missing_pypinyin_raises_with_install_hint(self):
+        with mock.patch.object(readings, "_PINYIN_STYLES", None), \
+                mock.patch.dict(sys.modules, {"pypinyin": None}):
+            with self.assertRaises(ImportError) as ctx:
+                to_pinyin("中国")
+            self.assertIn("scriptconv[zh]", str(ctx.exception))
+
+    def test_style_table_is_cached(self):
+        to_pinyin("中国")
+        first = readings._PINYIN_STYLES
+        to_bopomofo("中国")
+        self.assertIs(readings._PINYIN_STYLES, first)
+
+    def test_chinese_and_japanese_backends_independent(self):
+        # pypinyin absence must not break Japanese conversion
+        with mock.patch.object(readings, "_PINYIN_STYLES", None), \
+                mock.patch.dict(sys.modules, {"pypinyin": None}):
+            self.assertEqual(readings.to_hiragana("日本語"), "にほんご")
+
+
+if __name__ == "__main__":
+    unittest.main()