scriptconv 0.0.3a4__tar.gz → 0.0.3a6__tar.gz

{scriptconv-0.0.3a4 → scriptconv-0.0.3a6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a4
+Version: 0.0.3a6
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv

{scriptconv-0.0.3a4 → scriptconv-0.0.3a6}/pyproject.toml

@@ -46,6 +46,7 @@ include = ["scriptconv*"]
 
 [tool.setuptools.package-data]
 scriptconv = ["py.typed"]
+"scriptconv.data" = ["*.tsv.gz"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

{scriptconv-0.0.3a4 → scriptconv-0.0.3a6}/scriptconv/__init__.py

@@ -68,6 +68,8 @@ from scriptconv.translit import (
     kana_to_hira,
 )
 from scriptconv.readings import (
+    ReadingToken,
+    tokens,
     to_hiragana,
     to_katakana,
     to_pinyin,
@@ -117,6 +119,8 @@ __all__ = [
     "hira_to_kana",
     "kana_to_hira",
     # readings (dictionary-backed, needs scriptconv[ja])
+    "ReadingToken",
+    "tokens",
     "to_hiragana",
     "to_katakana",
     # cangjie (vendored Cangjie5 table)

{scriptconv-0.0.3a4 → scriptconv-0.0.3a6}/scriptconv/readings.py

@@ -22,7 +22,17 @@ unconverted text.
 """
 from __future__ import annotations
 
-__all__ = ["to_hiragana", "to_katakana", "to_pinyin", "to_bopomofo"]
+from typing import Iterator, NamedTuple
+
+__all__ = ["ReadingToken", "tokens", "to_hiragana", "to_katakana",
+           "to_pinyin", "to_bopomofo"]
+
+
+class ReadingToken(NamedTuple):
+    """One dictionary-segmented token of Japanese text and its kana readings."""
+    orig: str
+    hira: str
+    kana: str
 
 _kakasi = None
 
@@ -44,32 +54,51 @@ def _converter():
     return _kakasi
 
 
-def to_hiragana(text: str, keep_katakana: bool = False) -> str:
+def tokens(text: str) -> Iterator[ReadingToken]:
+    """Iterate the dictionary's segmentation of *text* with per-token readings.
+
+    Each token is a :class:`ReadingToken` with the original surface form and
+    its hiragana and katakana readings.  This is the primitive under
+    :func:`to_hiragana`/:func:`to_katakana`; use it directly when the consumer
+    needs token boundaries — e.g. word segmentation (wakachigaki) or
+    reading-dependent post-processing — rather than a joined string.
+    """
+    for t in _converter().convert(text):
+        if t["orig"]:
+            yield ReadingToken(t["orig"], t["hira"], t["kana"])
+
+
+def to_hiragana(text: str, keep_katakana: bool = False,
+                segment: bool = False) -> str:
     """Respell Japanese text in hiragana.
 
     Kanji are resolved to their dictionary reading; existing kana and any
     non-Japanese characters pass through.  With ``keep_katakana=True``,
     katakana tokens keep their original script (useful when katakana marks
-    loanwords deliberately) instead of being folded into hiragana.
+    loanwords deliberately) instead of being folded into hiragana.  With
+    ``segment=True``, dictionary tokens are joined with spaces (wakachigaki,
+    the spaced orthographic mode) instead of concatenated.
     """
     out = []
-    for token in _converter().convert(text):
-        orig = token["orig"]
-        if keep_katakana and orig and all(ord(c) in _KANA or not _is_japanese(c)
-                                          for c in orig):
-            out.append(orig)
+    for tok in tokens(text):
+        if keep_katakana and tok.orig and all(
+                ord(c) in _KANA or not _is_japanese(c) for c in tok.orig):
+            out.append(tok.orig)
         else:
-            out.append(token["hira"])
-    return "".join(out)
+            out.append(tok.hira)
+    return " ".join(" ".join(out).split()) if segment else "".join(out)
 
 
-def to_katakana(text: str) -> str:
+def to_katakana(text: str, segment: bool = False) -> str:
     """Respell Japanese text in katakana.
 
     Kanji are resolved to their dictionary reading; existing kana are
     transposed to katakana and any non-Japanese characters pass through.
+    With ``segment=True``, dictionary tokens are joined with spaces
+    (wakachigaki) instead of concatenated.
     """
-    return "".join(token["kana"] for token in _converter().convert(text))
+    kana = [tok.kana for tok in tokens(text)]
+    return " ".join(" ".join(kana).split()) if segment else "".join(kana)
 
 
 def _is_japanese(ch: str) -> bool:

{scriptconv-0.0.3a4 → scriptconv-0.0.3a6}/scriptconv/version.py

@@ -2,7 +2,7 @@
 VERSION_MAJOR = 0
 VERSION_MINOR = 0
 VERSION_BUILD = 3
-VERSION_ALPHA = 4
+VERSION_ALPHA = 6
 # END_VERSION_BLOCK
 
 VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"

{scriptconv-0.0.3a4 → scriptconv-0.0.3a6}/scriptconv.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptconv
-Version: 0.0.3a4
+Version: 0.0.3a6
 Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/TigreGotico/scriptconv

{scriptconv-0.0.3a4 → scriptconv-0.0.3a6}/scriptconv.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ scriptconv.egg-info/dependency_links.txt
 scriptconv.egg-info/requires.txt
 scriptconv.egg-info/top_level.txt
 scriptconv/data/__init__.py
+scriptconv/data/cangjie5_tc.tsv.gz
 tests/test_cangjie.py
 tests/test_cli.py
 tests/test_examples.py

{scriptconv-0.0.3a4 → scriptconv-0.0.3a6}/tests/test_cangjie.py

@@ -24,6 +24,13 @@ class TestCangjieCode(unittest.TestCase):
         self.assertTrue(all(c.isalpha() and c.islower() for c in sample))
 
 
+class TestTableIsPackaged(unittest.TestCase):
+    def test_data_file_resolvable_via_importlib_resources(self):
+        from importlib.resources import files
+        raw = files("scriptconv.data").joinpath("cangjie5_tc.tsv.gz").read_bytes()
+        self.assertGreater(len(raw), 100000)
+
+
 class TestToCangjie(unittest.TestCase):
     def test_all_hanzi_sentence(self):
         self.assertEqual(to_cangjie("倉頡"), "oiar grmbc")

{scriptconv-0.0.3a4 → scriptconv-0.0.3a6}/tests/test_readings.py

@@ -71,3 +71,45 @@ class TestDependencyBoundary(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class TestTokens(unittest.TestCase):
+    def test_yields_reading_tokens_with_all_forms(self):
+        from scriptconv import ReadingToken, tokens
+        toks = list(tokens("東京タワー"))
+        self.assertTrue(all(isinstance(t, ReadingToken) for t in toks))
+        origs = [t.orig for t in toks]
+        self.assertIn("東京", origs)
+        tokyo = toks[origs.index("東京")]
+        self.assertEqual(tokyo.hira, "とうきょう")
+        self.assertEqual(tokyo.kana, "トウキョウ")
+
+    def test_tokens_cover_input_exactly(self):
+        from scriptconv import tokens
+        text = "私はcoffeeが好き"
+        self.assertEqual("".join(t.orig for t in tokens(text)), text)
+
+    def test_empty_string_yields_nothing(self):
+        from scriptconv import tokens
+        self.assertEqual(list(tokens("")), [])
+
+
+class TestSegmentFlag(unittest.TestCase):
+    def test_hiragana_wakachigaki(self):
+        from scriptconv import to_hiragana
+        self.assertEqual(to_hiragana("私は学生です", segment=True),
+                         "わたし は がくせい です")
+
+    def test_katakana_wakachigaki(self):
+        from scriptconv import to_katakana
+        self.assertEqual(to_katakana("私は学生です", segment=True),
+                         "ワタシ ハ ガクセイ デス")
+
+    def test_segment_false_is_default_concatenation(self):
+        from scriptconv import to_hiragana
+        self.assertEqual(to_hiragana("私は学生です"), "わたしはがくせいです")
+
+    def test_segment_collapses_original_whitespace(self):
+        from scriptconv import to_hiragana
+        out = to_hiragana("東京 大阪", segment=True)
+        self.assertNotIn("  ", out)