scriptconv 0.0.3a7__tar.gz → 0.0.3a8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/PKG-INFO +1 -1
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/__init__.py +19 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/__main__.py +29 -1
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/conventions.py +205 -164
- scriptconv-0.0.3a8/scriptconv/graph.py +259 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/notation.py +19 -28
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/version.py +1 -1
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv.egg-info/PKG-INFO +1 -1
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv.egg-info/SOURCES.txt +2 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/tests/test_cli.py +17 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/tests/test_conventions.py +28 -0
- scriptconv-0.0.3a8/tests/test_graph.py +132 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/LICENSE +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/README.md +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/pyproject.toml +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/requirements.txt +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/cangjie.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/data/__init__.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/data/cangjie5_tc.tsv.gz +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/py.typed +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/readings.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/scripts.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv/translit.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv.egg-info/dependency_links.txt +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv.egg-info/requires.txt +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/scriptconv.egg-info/top_level.txt +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/setup.cfg +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/tests/test_cangjie.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/tests/test_examples.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/tests/test_notation.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/tests/test_readings.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/tests/test_readings_zh.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/tests/test_scripts.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a8}/tests/test_translit.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptconv
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.3a8
|
|
4
4
|
Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
|
|
@@ -22,6 +22,10 @@ readings:
|
|
|
22
22
|
extra; Chinese hanzi → pinyin/bopomofo via the ``zh`` extra).
|
|
23
23
|
cangjie:
|
|
24
24
|
Hanzi → Cangjie5 input codes (shape decomposition, vendored table).
|
|
25
|
+
graph:
|
|
26
|
+
Conversion-graph engine: representations (notations and orthographies)
|
|
27
|
+
as nodes, registered transforms as edges, lossless-preferring routing.
|
|
28
|
+
External packages extend graph instances explicitly.
|
|
25
29
|
conventions:
|
|
26
30
|
Orthographic conventions — script-scoped decorations orthogonal to
|
|
27
31
|
identity (tashkeel, niqqud, wakachigaki, pinyin tone spelling…).
|
|
@@ -81,6 +85,7 @@ from scriptconv.readings import (
|
|
|
81
85
|
)
|
|
82
86
|
from scriptconv.conventions import (
|
|
83
87
|
Convention,
|
|
88
|
+
Transition,
|
|
84
89
|
CONVENTION_REGISTRY,
|
|
85
90
|
conventions_for,
|
|
86
91
|
restyle,
|
|
@@ -88,6 +93,13 @@ from scriptconv.conventions import (
|
|
|
88
93
|
apply,
|
|
89
94
|
detect as detect_convention,
|
|
90
95
|
)
|
|
96
|
+
from scriptconv.graph import (
|
|
97
|
+
Representation,
|
|
98
|
+
Edge,
|
|
99
|
+
ConversionGraph,
|
|
100
|
+
DEFAULT_GRAPH,
|
|
101
|
+
REPRESENTATIONS,
|
|
102
|
+
)
|
|
91
103
|
from scriptconv.cangjie import (
|
|
92
104
|
cangjie_code,
|
|
93
105
|
to_cangjie,
|
|
@@ -138,12 +150,19 @@ __all__ = [
|
|
|
138
150
|
"to_katakana",
|
|
139
151
|
# conventions (orthographic decorations, orthogonal to script identity)
|
|
140
152
|
"Convention",
|
|
153
|
+
"Transition",
|
|
141
154
|
"CONVENTION_REGISTRY",
|
|
142
155
|
"conventions_for",
|
|
143
156
|
"restyle",
|
|
144
157
|
"strip",
|
|
145
158
|
"apply",
|
|
146
159
|
"detect_convention",
|
|
160
|
+
# graph (conversion routing engine over representations)
|
|
161
|
+
"Representation",
|
|
162
|
+
"Edge",
|
|
163
|
+
"ConversionGraph",
|
|
164
|
+
"DEFAULT_GRAPH",
|
|
165
|
+
"REPRESENTATIONS",
|
|
147
166
|
# cangjie (vendored Cangjie5 table)
|
|
148
167
|
"cangjie_code",
|
|
149
168
|
"to_cangjie",
|
|
@@ -11,6 +11,7 @@ Usage
|
|
|
11
11
|
python -m scriptconv strip <convention> <text>
|
|
12
12
|
python -m scriptconv restyle <convention> <style> <text>
|
|
13
13
|
python -m scriptconv conventions [--script CODE]
|
|
14
|
+
python -m scriptconv route <src> <dst>
|
|
14
15
|
|
|
15
16
|
Examples
|
|
16
17
|
--------
|
|
@@ -64,6 +65,10 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
64
65
|
cv = sub.add_parser("conventions", help="List orthographic conventions")
|
|
65
66
|
cv.add_argument("--script", default=None, help="Filter by ISO-15924 script code")
|
|
66
67
|
|
|
68
|
+
rt = sub.add_parser("route", help="Show the conversion path between representations")
|
|
69
|
+
rt.add_argument("src", help="Source representation id")
|
|
70
|
+
rt.add_argument("dst", help="Target representation id")
|
|
71
|
+
|
|
67
72
|
args = p.parse_args(argv)
|
|
68
73
|
|
|
69
74
|
if args.command is None:
|
|
@@ -80,6 +85,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
80
85
|
"strip": lambda: _do_strip(args),
|
|
81
86
|
"restyle": lambda: _do_restyle(args),
|
|
82
87
|
"conventions": lambda: _do_conventions(args),
|
|
88
|
+
"route": lambda: _do_route(args),
|
|
83
89
|
}
|
|
84
90
|
dispatch[args.command]()
|
|
85
91
|
return 0
|
|
@@ -89,9 +95,31 @@ def _do_convert(args: argparse.Namespace) -> None:
|
|
|
89
95
|
from scriptconv.notation import Notation, convert
|
|
90
96
|
try:
|
|
91
97
|
print(convert(args.text, args.src, args.dst))
|
|
98
|
+
return
|
|
99
|
+
except ValueError:
|
|
100
|
+
pass
|
|
101
|
+
# not a notation pair — try any representation pair on the default graph
|
|
102
|
+
from scriptconv.graph import DEFAULT_GRAPH
|
|
103
|
+
try:
|
|
104
|
+
print(DEFAULT_GRAPH.convert(args.text, args.src, args.dst))
|
|
92
105
|
except ValueError as e:
|
|
93
106
|
valid = ", ".join(n.value for n in Notation)
|
|
94
|
-
raise SystemExit(f"error: {e} (valid notations: {valid}
|
|
107
|
+
raise SystemExit(f"error: {e} (valid notations: {valid}; "
|
|
108
|
+
f"graph nodes: {', '.join(DEFAULT_GRAPH.nodes)})")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _do_route(args: argparse.Namespace) -> None:
|
|
112
|
+
from scriptconv.graph import DEFAULT_GRAPH
|
|
113
|
+
try:
|
|
114
|
+
path = DEFAULT_GRAPH.route(args.src, args.dst)
|
|
115
|
+
except ValueError as e:
|
|
116
|
+
raise SystemExit(f"error: {e}")
|
|
117
|
+
if not path:
|
|
118
|
+
print("(identity)")
|
|
119
|
+
for e in path:
|
|
120
|
+
flags = "lossless" if e.lossless else "lossy"
|
|
121
|
+
extra = f", needs scriptconv[{e.requires}]" if e.requires else ""
|
|
122
|
+
print(f" {e.src} -> {e.dst} ({flags}, cost {e.cost:g}{extra})")
|
|
95
123
|
|
|
96
124
|
|
|
97
125
|
def _do_detect(args: argparse.Namespace) -> None:
|
|
@@ -43,6 +43,7 @@ repertoires and the restyle is deterministic.
|
|
|
43
43
|
"""
|
|
44
44
|
from __future__ import annotations
|
|
45
45
|
|
|
46
|
+
import functools
|
|
46
47
|
import re
|
|
47
48
|
import unicodedata
|
|
48
49
|
from dataclasses import dataclass
|
|
@@ -50,6 +51,7 @@ from typing import Callable, Dict, List, Optional, Tuple
|
|
|
50
51
|
|
|
51
52
|
__all__ = [
|
|
52
53
|
"Convention",
|
|
54
|
+
"Transition",
|
|
53
55
|
"CONVENTION_REGISTRY",
|
|
54
56
|
"conventions_for",
|
|
55
57
|
"detect",
|
|
@@ -59,98 +61,118 @@ __all__ = [
|
|
|
59
61
|
]
|
|
60
62
|
|
|
61
63
|
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class Transition:
|
|
66
|
+
"""One rewrite between two styles of a convention.
|
|
67
|
+
|
|
68
|
+
``fn`` is a pure ``str -> str`` transform. ``requires`` names the
|
|
69
|
+
optional extra the transform needs, if any (queryable before calling —
|
|
70
|
+
the transform itself raises :class:`ImportError` with an install hint
|
|
71
|
+
when the dependency is missing).
|
|
72
|
+
"""
|
|
73
|
+
frm: str
|
|
74
|
+
to: str
|
|
75
|
+
fn: Callable[[str], str]
|
|
76
|
+
lossless: bool = False
|
|
77
|
+
requires: Optional[str] = None
|
|
78
|
+
|
|
79
|
+
|
|
62
80
|
@dataclass(frozen=True)
|
|
63
81
|
class Convention:
|
|
64
|
-
"""
|
|
82
|
+
"""One orthographic convention: metadata plus the behavior that owns it.
|
|
83
|
+
|
|
84
|
+
Transitions and the optional detector live on the convention itself, so
|
|
85
|
+
every operation is generic — nothing in this module branches on a
|
|
86
|
+
convention's identity.
|
|
87
|
+
"""
|
|
65
88
|
id: str
|
|
66
89
|
name: str
|
|
67
90
|
scripts: Tuple[str, ...] # ISO-15924 codes whose text it decorates
|
|
68
91
|
system: Optional[str] # sub-script system qualifier ("pinyin", "japanese")
|
|
69
92
|
styles: Tuple[str, ...] # mutually exclusive states; includes "none" if strippable
|
|
70
93
|
default_style: str
|
|
71
|
-
detectable: bool # current style is character-evident
|
|
72
|
-
strip_lossless: bool # can the stripped information be recovered?
|
|
73
|
-
apply_requires: Optional[str] # optional-extra name needed to leave "none", if any
|
|
74
94
|
ranges: Tuple[Tuple[int, int], ...] # decoration codepoints (empty for spacing/variant conventions)
|
|
75
95
|
reference: str
|
|
96
|
+
transitions: Tuple[Transition, ...] = ()
|
|
97
|
+
detector: Optional[Callable[[str], Optional[str]]] = None
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def detectable(self) -> bool:
|
|
101
|
+
return self.detector is not None
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def strip_lossless(self) -> bool:
|
|
105
|
+
stripping = [t for t in self.transitions if t.to == "none"]
|
|
106
|
+
return bool(stripping) and all(t.lossless for t in stripping)
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def apply_requires(self) -> Optional[str]:
|
|
110
|
+
return next((t.requires for t in self.transitions if t.frm == "none"), None)
|
|
111
|
+
|
|
112
|
+
def transition(self, frm: str, to: str) -> Optional[Transition]:
|
|
113
|
+
return next((t for t in self.transitions
|
|
114
|
+
if t.frm == frm and t.to == to), None)
|
|
115
|
+
|
|
116
|
+
def restyle(self, text: str, to: str, frm: Optional[str] = None) -> str:
|
|
117
|
+
"""Rewrite *text* from one style of this convention to another."""
|
|
118
|
+
if to not in self.styles and to != "none":
|
|
119
|
+
raise ValueError(f"{self.id!r} has styles {self.styles}, not {to!r}")
|
|
120
|
+
if frm is None:
|
|
121
|
+
frm = self._infer_source(text, to)
|
|
122
|
+
if frm == to:
|
|
123
|
+
return text
|
|
124
|
+
t = self.transition(frm, to)
|
|
125
|
+
if t is None:
|
|
126
|
+
supported = sorted(f"{x.frm}->{x.to}" for x in self.transitions)
|
|
127
|
+
raise ValueError(f"{self.id!r} does not define {frm!r}->{to!r}; "
|
|
128
|
+
f"supported: {', '.join(supported)}")
|
|
129
|
+
return t.fn(text)
|
|
130
|
+
|
|
131
|
+
def _infer_source(self, text: str, to: str) -> str:
|
|
132
|
+
if self.detector is not None:
|
|
133
|
+
detected = self.detector(text)
|
|
134
|
+
if detected is not None and (detected == to
|
|
135
|
+
or self.transition(detected, to)):
|
|
136
|
+
return detected
|
|
137
|
+
sources = sorted({t.frm for t in self.transitions if t.to == to})
|
|
138
|
+
if len(sources) == 1:
|
|
139
|
+
return sources[0]
|
|
140
|
+
if self.default_style != to:
|
|
141
|
+
return self.default_style
|
|
142
|
+
raise ValueError(f"{self.id!r}: source style for ->{to!r} is ambiguous "
|
|
143
|
+
f"({', '.join(sources)}); pass frm=")
|
|
144
|
+
|
|
145
|
+
def strip(self, text: str) -> str:
|
|
146
|
+
"""Remove this convention's decoration (restyle to ``"none"``)."""
|
|
147
|
+
stripping = [t for t in self.transitions if t.to == "none"]
|
|
148
|
+
if not stripping:
|
|
149
|
+
raise ValueError(f"{self.id!r} has no unmarked style; "
|
|
150
|
+
f"styles: {self.styles}")
|
|
151
|
+
# when every stripping transition is the same transform (e.g. one
|
|
152
|
+
# strip handles both pinyin tone spellings), the source is irrelevant
|
|
153
|
+
if len({t.fn for t in stripping}) == 1:
|
|
154
|
+
return stripping[0].fn(text)
|
|
155
|
+
return self.restyle(text, "none")
|
|
156
|
+
|
|
157
|
+
def apply(self, text: str, style: Optional[str] = None) -> str:
|
|
158
|
+
"""Add this convention's decoration to unmarked text."""
|
|
159
|
+
if style is None:
|
|
160
|
+
marked = [s for s in self.styles if s != "none"]
|
|
161
|
+
style = self.default_style if self.default_style != "none" else marked[0]
|
|
162
|
+
return self.restyle(text, style, frm="none")
|
|
163
|
+
|
|
164
|
+
def detect(self, text: str) -> Optional[str]:
|
|
165
|
+
"""Current style of *text*, or None when not character-evident.
|
|
166
|
+
|
|
167
|
+
For binary decorations this means "any mark present" — a single
|
|
168
|
+
shadda in a paragraph reports ``"marked"``.
|
|
169
|
+
"""
|
|
170
|
+
return self.detector(text) if self.detector is not None else None
|
|
76
171
|
|
|
77
172
|
|
|
78
173
|
# ---------------------------------------------------------------------------
|
|
79
|
-
#
|
|
174
|
+
# Shared helpers
|
|
80
175
|
# ---------------------------------------------------------------------------
|
|
81
|
-
# Tashkeel deliberately EXCLUDES U+0653–0655 (madda above, hamza above, hamza
|
|
82
|
-
# below): in NFD text آ/أ/إ decompose to alef + those marks, which are letter
|
|
83
|
-
# identity — stripping them would corrupt the consonantal skeleton.
|
|
84
|
-
_TASHKEEL_RANGES = ((0x064B, 0x0652), (0x0656, 0x065F), (0x0670, 0x0670),
|
|
85
|
-
(0x08F0, 0x08F2))
|
|
86
|
-
_KASHIDA_RANGES = ((0x0640, 0x0640),)
|
|
87
|
-
# 06DD/06DE (end of ayah, rub el hizb) are structural signs and 06E5/06E6 are
|
|
88
|
-
# small letters — kept; 06E7-06E9 (small high marks, place of sajdah) and
|
|
89
|
-
# 06EA-06ED are annotation marks — stripped.
|
|
90
|
-
_QURANIC_RANGES = ((0x06D6, 0x06DC), (0x06DF, 0x06E4), (0x06E7, 0x06ED))
|
|
91
|
-
# Niqqud includes dagesh/mapiq U+05BC, meteg U+05BD (a masoretic stress mark,
|
|
92
|
-
# but traditionally stripped with the points), rafe U+05BF and the shin/sin
|
|
93
|
-
# dots U+05C1/05C2 — stripping loses the שׁ/שׂ distinction, exactly as
|
|
94
|
-
# unpointed Hebrew does. Maqaf U+05BE, sof pasuq U+05C3 and nun hafukha
|
|
95
|
-
# U+05C6 are punctuation and survive.
|
|
96
|
-
_NIQQUD_RANGES = ((0x05B0, 0x05BD), (0x05BF, 0x05BF), (0x05C1, 0x05C2),
|
|
97
|
-
(0x05C7, 0x05C7))
|
|
98
|
-
_TEAMIM_RANGES = ((0x0591, 0x05AF), (0x05C0, 0x05C0), (0x05C4, 0x05C5))
|
|
99
|
-
|
|
100
|
-
CONVENTION_REGISTRY: Dict[str, Convention] = {c.id: c for c in (
|
|
101
|
-
Convention("tashkeel", "Arabic vocalization (harakat/tanwin/shadda/sukun)",
|
|
102
|
-
("Arab",), None, ("marked", "none"), "marked",
|
|
103
|
-
detectable=True, strip_lossless=False, apply_requires=None,
|
|
104
|
-
ranges=_TASHKEEL_RANGES,
|
|
105
|
-
reference="Unicode 15 ch. 9.2 (Arabic); excludes letter-identity U+0653-0655"),
|
|
106
|
-
Convention("kashida", "Arabic justification elongation (tatweel)",
|
|
107
|
-
("Arab",), None, ("marked", "none"), "none",
|
|
108
|
-
detectable=True, strip_lossless=False, apply_requires=None,
|
|
109
|
-
ranges=_KASHIDA_RANGES, reference="Unicode U+0640 ARABIC TATWEEL"),
|
|
110
|
-
Convention("quranic-marks", "Quranic annotation signs",
|
|
111
|
-
("Arab",), None, ("marked", "none"), "none",
|
|
112
|
-
detectable=True, strip_lossless=False, apply_requires=None,
|
|
113
|
-
ranges=_QURANIC_RANGES, reference="Unicode U+06D6-06ED Quranic annotation signs"),
|
|
114
|
-
Convention("niqqud", "Hebrew vocalization points",
|
|
115
|
-
("Hebr",), None, ("marked", "none"), "marked",
|
|
116
|
-
detectable=True, strip_lossless=False, apply_requires=None,
|
|
117
|
-
ranges=_NIQQUD_RANGES,
|
|
118
|
-
reference="Unicode 15 ch. 9.1 (Hebrew); strip loses the shin/sin dot distinction"),
|
|
119
|
-
Convention("teamim", "Hebrew cantillation marks",
|
|
120
|
-
("Hebr",), None, ("marked", "none"), "none",
|
|
121
|
-
detectable=True, strip_lossless=False, apply_requires=None,
|
|
122
|
-
ranges=_TEAMIM_RANGES, reference="Unicode U+0591-05AF Hebrew accents (masoretic)"),
|
|
123
|
-
Convention("wakachigaki", "Japanese spaced writing (分かち書き)",
|
|
124
|
-
("Hira", "Kana", "Hani"), "japanese", ("marked", "none"), "none",
|
|
125
|
-
detectable=False, strip_lossless=False, apply_requires="ja",
|
|
126
|
-
ranges=(),
|
|
127
|
-
reference="Spaced orthographic mode of Japanese (children's books, games)"),
|
|
128
|
-
Convention("pinyin-tone", "Pinyin tone spelling",
|
|
129
|
-
("Latn",), "pinyin", ("mark", "number", "none"), "mark",
|
|
130
|
-
detectable=False, strip_lossless=False, apply_requires=None,
|
|
131
|
-
ranges=(),
|
|
132
|
-
reference="GB/T 16159 Basic rules of Hanyu Pinyin orthography"),
|
|
133
|
-
Convention("jamo-form", "Hangul jamo repertoire (compatibility vs conjoining)",
|
|
134
|
-
("Hang",), None, ("compatibility", "conjoining"), "compatibility",
|
|
135
|
-
detectable=True, strip_lossless=True, apply_requires=None,
|
|
136
|
-
ranges=(),
|
|
137
|
-
reference="Unicode Hangul Compatibility Jamo U+3130 vs Hangul Jamo U+1100"),
|
|
138
|
-
)}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
def conventions_for(script: str) -> List[Convention]:
|
|
142
|
-
"""Conventions that decorate *script* (ISO-15924 code), registry order."""
|
|
143
|
-
return [c for c in CONVENTION_REGISTRY.values() if script in c.scripts]
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
def _conv(convention_id: str) -> Convention:
|
|
147
|
-
try:
|
|
148
|
-
return CONVENTION_REGISTRY[convention_id]
|
|
149
|
-
except KeyError:
|
|
150
|
-
raise ValueError(
|
|
151
|
-
f"unknown convention {convention_id!r}; known: "
|
|
152
|
-
f"{', '.join(sorted(CONVENTION_REGISTRY))}") from None
|
|
153
|
-
|
|
154
176
|
|
|
155
177
|
def _in_ranges(cp: int, ranges: Tuple[Tuple[int, int], ...]) -> bool:
|
|
156
178
|
return any(lo <= cp <= hi for lo, hi in ranges)
|
|
@@ -162,6 +184,12 @@ def _range_strip(ranges: Tuple[Tuple[int, int], ...]) -> Callable[[str], str]:
|
|
|
162
184
|
return _strip
|
|
163
185
|
|
|
164
186
|
|
|
187
|
+
def _range_detector(ranges: Tuple[Tuple[int, int], ...]) -> Callable[[str], Optional[str]]:
|
|
188
|
+
def _detect(text: str) -> Optional[str]:
|
|
189
|
+
return "marked" if any(_in_ranges(ord(c), ranges) for c in text) else "none"
|
|
190
|
+
return _detect
|
|
191
|
+
|
|
192
|
+
|
|
165
193
|
# ---------------------------------------------------------------------------
|
|
166
194
|
# Wakachigaki
|
|
167
195
|
# ---------------------------------------------------------------------------
|
|
@@ -338,6 +366,7 @@ def _pinyin_strip(text: str) -> str:
|
|
|
338
366
|
# U+3131), so a consonant is treated as an onset when a vowel follows,
|
|
339
367
|
# otherwise as a coda — deterministic over well-formed jamo sequences.
|
|
340
368
|
|
|
369
|
+
@functools.lru_cache(maxsize=1)
|
|
341
370
|
def _jamo_tables():
|
|
342
371
|
from scriptconv.translit import _ONSETS as _CO, _VOWELS as _CV, _CODAS as _CT
|
|
343
372
|
conj_to_compat = {}
|
|
@@ -357,22 +386,13 @@ def _jamo_tables():
|
|
|
357
386
|
return conj_to_compat, compat_onset, compat_vowel, compat_coda
|
|
358
387
|
|
|
359
388
|
|
|
360
|
-
_JAMO_TABLES = None
|
|
361
|
-
|
|
362
|
-
|
|
363
389
|
def _jamo_conjoining_to_compat(text: str) -> str:
|
|
364
|
-
|
|
365
|
-
if _JAMO_TABLES is None:
|
|
366
|
-
_JAMO_TABLES = _jamo_tables()
|
|
367
|
-
table = _JAMO_TABLES[0]
|
|
390
|
+
table = _jamo_tables()[0]
|
|
368
391
|
return "".join(table.get(c, c) for c in text)
|
|
369
392
|
|
|
370
393
|
|
|
371
394
|
def _jamo_compat_to_conjoining(text: str) -> str:
|
|
372
|
-
|
|
373
|
-
if _JAMO_TABLES is None:
|
|
374
|
-
_JAMO_TABLES = _jamo_tables()
|
|
375
|
-
_, onset, vowel, coda = _JAMO_TABLES
|
|
395
|
+
_, onset, vowel, coda = _jamo_tables()
|
|
376
396
|
out = []
|
|
377
397
|
for i, c in enumerate(text):
|
|
378
398
|
if c in vowel:
|
|
@@ -401,95 +421,116 @@ def _jamo_detect(text: str) -> Optional[str]:
|
|
|
401
421
|
|
|
402
422
|
|
|
403
423
|
# ---------------------------------------------------------------------------
|
|
404
|
-
#
|
|
424
|
+
# Registry
|
|
405
425
|
# ---------------------------------------------------------------------------
|
|
406
426
|
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
427
|
+
# Tashkeel deliberately EXCLUDES U+0653–0655 (madda above, hamza above, hamza
|
|
428
|
+
# below): in NFD text آ/أ/إ decompose to alef + those marks, which are letter
|
|
429
|
+
# identity — stripping them would corrupt the consonantal skeleton.
|
|
430
|
+
_TASHKEEL_RANGES = ((0x064B, 0x0652), (0x0656, 0x065F), (0x0670, 0x0670),
|
|
431
|
+
(0x08F0, 0x08F2))
|
|
432
|
+
_KASHIDA_RANGES = ((0x0640, 0x0640),)
|
|
433
|
+
# 06DD/06DE (end of ayah, rub el hizb) are structural signs and 06E5/06E6 are
|
|
434
|
+
# small letters — kept; 06E7-06E9 (small high marks, place of sajdah) and
|
|
435
|
+
# 06EA-06ED are annotation marks — stripped.
|
|
436
|
+
_QURANIC_RANGES = ((0x06D6, 0x06DC), (0x06DF, 0x06E4), (0x06E7, 0x06ED))
|
|
437
|
+
# Niqqud includes dagesh/mapiq U+05BC, meteg U+05BD (a masoretic stress mark,
|
|
438
|
+
# but traditionally stripped with the points), rafe U+05BF and the shin/sin
|
|
439
|
+
# dots U+05C1/05C2 — stripping loses the שׁ/שׂ distinction, exactly as
|
|
440
|
+
# unpointed Hebrew does. Maqaf U+05BE, sof pasuq U+05C3 and nun hafukha
|
|
441
|
+
# U+05C6 are punctuation and survive.
|
|
442
|
+
_NIQQUD_RANGES = ((0x05B0, 0x05BD), (0x05BF, 0x05BF), (0x05C1, 0x05C2),
|
|
443
|
+
(0x05C7, 0x05C7))
|
|
444
|
+
_TEAMIM_RANGES = ((0x0591, 0x05AF), (0x05C0, 0x05C0), (0x05C4, 0x05C5))
|
|
422
445
|
|
|
423
446
|
|
|
424
|
-
def
|
|
425
|
-
|
|
426
|
-
|
|
447
|
+
def _marks(id: str, name: str, scripts: Tuple[str, ...],
|
|
448
|
+
ranges: Tuple[Tuple[int, int], ...], reference: str,
|
|
449
|
+
*, default: str = "marked") -> Convention:
|
|
450
|
+
"""A binary decoration: strippable marks with a range-based detector."""
|
|
451
|
+
return Convention(id, name, scripts, None, ("marked", "none"), default,
|
|
452
|
+
ranges, reference,
|
|
453
|
+
transitions=(Transition("marked", "none", _range_strip(ranges)),),
|
|
454
|
+
detector=_range_detector(ranges))
|
|
427
455
|
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
+
|
|
457
|
+
CONVENTION_REGISTRY: Dict[str, Convention] = {c.id: c for c in (
|
|
458
|
+
_marks("tashkeel", "Arabic vocalization (harakat/tanwin/shadda/sukun)",
|
|
459
|
+
("Arab",), _TASHKEEL_RANGES,
|
|
460
|
+
"Unicode 15 ch. 9.2 (Arabic); excludes letter-identity U+0653-0655"),
|
|
461
|
+
_marks("kashida", "Arabic justification elongation (tatweel)",
|
|
462
|
+
("Arab",), _KASHIDA_RANGES, "Unicode U+0640 ARABIC TATWEEL",
|
|
463
|
+
default="none"),
|
|
464
|
+
_marks("quranic-marks", "Quranic annotation signs",
|
|
465
|
+
("Arab",), _QURANIC_RANGES,
|
|
466
|
+
"Unicode U+06D6-06ED Quranic annotation signs", default="none"),
|
|
467
|
+
_marks("niqqud", "Hebrew vocalization points",
|
|
468
|
+
("Hebr",), _NIQQUD_RANGES,
|
|
469
|
+
"Unicode 15 ch. 9.1 (Hebrew); strip loses the shin/sin dot distinction"),
|
|
470
|
+
_marks("teamim", "Hebrew cantillation marks",
|
|
471
|
+
("Hebr",), _TEAMIM_RANGES,
|
|
472
|
+
"Unicode U+0591-05AF Hebrew accents (masoretic)", default="none"),
|
|
473
|
+
Convention("wakachigaki", "Japanese spaced writing (分かち書き)",
|
|
474
|
+
("Hira", "Kana", "Hani"), "japanese", ("marked", "none"), "none",
|
|
475
|
+
(), "Spaced orthographic mode of Japanese (children's books, games)",
|
|
476
|
+
transitions=(
|
|
477
|
+
Transition("marked", "none", _wakachigaki_strip),
|
|
478
|
+
Transition("none", "marked", _wakachigaki_apply, requires="ja"),
|
|
479
|
+
)),
|
|
480
|
+
Convention("pinyin-tone", "Pinyin tone spelling",
|
|
481
|
+
("Latn",), "pinyin", ("mark", "number", "none"), "mark",
|
|
482
|
+
(), "GB/T 16159 Basic rules of Hanyu Pinyin orthography",
|
|
483
|
+
transitions=(
|
|
484
|
+
Transition("mark", "number", _pinyin_mark_to_number),
|
|
485
|
+
Transition("number", "mark", _pinyin_number_to_mark),
|
|
486
|
+
Transition("mark", "none", _pinyin_strip),
|
|
487
|
+
Transition("number", "none", _pinyin_strip),
|
|
488
|
+
)),
|
|
489
|
+
Convention("jamo-form", "Hangul jamo repertoire (compatibility vs conjoining)",
|
|
490
|
+
("Hang",), None, ("compatibility", "conjoining"), "compatibility",
|
|
491
|
+
(), "Unicode Hangul Compatibility Jamo U+3130 vs Hangul Jamo U+1100",
|
|
492
|
+
transitions=(
|
|
493
|
+
Transition("conjoining", "compatibility",
|
|
494
|
+
_jamo_conjoining_to_compat, lossless=True),
|
|
495
|
+
Transition("compatibility", "conjoining",
|
|
496
|
+
_jamo_compat_to_conjoining, lossless=True),
|
|
497
|
+
),
|
|
498
|
+
detector=_jamo_detect),
|
|
499
|
+
)}
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def conventions_for(script: str) -> List[Convention]:
|
|
503
|
+
"""Conventions that decorate *script* (ISO-15924 code), registry order."""
|
|
504
|
+
return [c for c in CONVENTION_REGISTRY.values() if script in c.scripts]
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _conv(convention_id: str) -> Convention:
|
|
508
|
+
try:
|
|
509
|
+
return CONVENTION_REGISTRY[convention_id]
|
|
510
|
+
except KeyError:
|
|
456
511
|
raise ValueError(
|
|
457
|
-
f"
|
|
458
|
-
f"
|
|
459
|
-
|
|
512
|
+
f"unknown convention {convention_id!r}; known: "
|
|
513
|
+
f"{', '.join(sorted(CONVENTION_REGISTRY))}") from None
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
# Module-level facades over the conventions' own behavior — the published API.
|
|
517
|
+
|
|
518
|
+
def restyle(text: str, convention_id: str, to: str,
|
|
519
|
+
frm: Optional[str] = None) -> str:
|
|
520
|
+
"""Rewrite *text* from one style of a convention to another."""
|
|
521
|
+
return _conv(convention_id).restyle(text, to, frm=frm)
|
|
460
522
|
|
|
461
523
|
|
|
462
524
|
def strip(text: str, convention_id: str) -> str:
|
|
463
525
|
"""Remove a convention's decoration (restyle to ``"none"``)."""
|
|
464
|
-
|
|
465
|
-
if "none" not in conv.styles:
|
|
466
|
-
raise ValueError(f"{convention_id!r} has no unmarked style; "
|
|
467
|
-
f"styles: {conv.styles}")
|
|
468
|
-
frm = "mark" if conv.id == "pinyin-tone" else "marked"
|
|
469
|
-
if conv.id == "pinyin-tone":
|
|
470
|
-
# digits and marks are both handled by the strip transition
|
|
471
|
-
return _pinyin_strip(text)
|
|
472
|
-
return restyle(text, convention_id, "none", frm=frm)
|
|
526
|
+
return _conv(convention_id).strip(text)
|
|
473
527
|
|
|
474
528
|
|
|
475
529
|
def apply(text: str, convention_id: str, style: Optional[str] = None) -> str:
|
|
476
530
|
"""Add a convention's decoration to unmarked text (restyle from ``"none"``)."""
|
|
477
|
-
|
|
478
|
-
if style is None:
|
|
479
|
-
marked = [s for s in conv.styles if s != "none"]
|
|
480
|
-
style = conv.default_style if conv.default_style != "none" else marked[0]
|
|
481
|
-
return restyle(text, convention_id, style, frm="none")
|
|
531
|
+
return _conv(convention_id).apply(text, style)
|
|
482
532
|
|
|
483
533
|
|
|
484
534
|
def detect(text: str, convention_id: str) -> Optional[str]:
|
|
485
|
-
"""Current style of *text* under a convention, or None when not character-evident.
|
|
486
|
-
|
|
487
|
-
For binary decorations this means "any mark present" — a single shadda in
|
|
488
|
-
a paragraph reports ``"marked"``.
|
|
489
|
-
"""
|
|
490
|
-
conv = _conv(convention_id)
|
|
491
|
-
if not conv.detectable:
|
|
492
|
-
return None
|
|
493
|
-
if conv.id == "jamo-form":
|
|
494
|
-
return _jamo_detect(text)
|
|
495
|
-
return "marked" if any(_in_ranges(ord(c), conv.ranges) for c in text) else "none"
|
|
535
|
+
"""Current style of *text* under a convention, or None when not character-evident."""
|
|
536
|
+
return _conv(convention_id).detect(text)
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""Conversion graph — one routing engine for every text representation.
|
|
2
|
+
|
|
3
|
+
Nodes are *representations*: phoneme notations ("ipa", "arpa") and
|
|
4
|
+
orthographies ("hira", "hanzi", "cangjie") are peers in one flat id
|
|
5
|
+
namespace, described by :class:`Representation` metadata. Edges are
|
|
6
|
+
registered transforms between two representations. Routing finds the
|
|
7
|
+
cheapest edge path, preferring lossless edges — silent lossy multi-hop is
|
|
8
|
+
the classic failure mode of conversion graphs, so losing information costs
|
|
9
|
+
extra by construction.
|
|
10
|
+
|
|
11
|
+
The engine is generic and dependency-free. :data:`DEFAULT_GRAPH` comes
|
|
12
|
+
pre-registered with scriptconv's own edges: every notation pair (the IPA-hub
|
|
13
|
+
routing that :func:`scriptconv.notation.convert` exposes) and the
|
|
14
|
+
orthographic conversions from :mod:`translit`, :mod:`cangjie` and
|
|
15
|
+
:mod:`readings` (the latter behind their optional extras — the edge exists,
|
|
16
|
+
and raises :class:`ImportError` with an install hint when traversed without
|
|
17
|
+
the dictionary installed).
|
|
18
|
+
|
|
19
|
+
External packages extend graphs explicitly — ``pkg.register(graph)`` on a
|
|
20
|
+
:meth:`ConversionGraph.copy` or the default graph; there is no entry-point
|
|
21
|
+
autoloading, so a graph's contents never depend on what happens to be
|
|
22
|
+
installed. This is also the purity boundary: scriptconv's own edges are
|
|
23
|
+
orthography only, but a client (e.g. a TTS stack) may register phonemization
|
|
24
|
+
edges into *its* graph instance — the engine routes them without knowing.
|
|
25
|
+
|
|
26
|
+
Two deliberate non-nodes: "graphemes" (a per-language contextual alias the
|
|
27
|
+
caller must resolve — which script a language ordinarily writes in is not a
|
|
28
|
+
property of text) and convention styles (:mod:`scriptconv.conventions` —
|
|
29
|
+
decorations are parameters of a representation, never nodes, or the graph
|
|
30
|
+
would double per style).
|
|
31
|
+
"""
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import heapq
|
|
35
|
+
from dataclasses import dataclass, field
|
|
36
|
+
from typing import Callable, Dict, List, Optional, Tuple
|
|
37
|
+
|
|
38
|
+
__all__ = ["Representation", "Edge", "ConversionGraph", "DEFAULT_GRAPH",
|
|
39
|
+
"REPRESENTATIONS"]
|
|
40
|
+
|
|
41
|
+
_LOSSY_COST = 10.0
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class Representation:
|
|
46
|
+
"""Metadata for one node of the conversion space."""
|
|
47
|
+
id: str
|
|
48
|
+
kind: str # "notation" | "orthography"
|
|
49
|
+
script: Optional[str] = None # ISO-15924, when script-bound
|
|
50
|
+
system: Optional[str] = None # sub-script system ("pinyin", "japanese")
|
|
51
|
+
reference: str = ""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class Edge:
|
|
56
|
+
"""One registered transform between two representations.
|
|
57
|
+
|
|
58
|
+
``fn`` is called as ``fn(text, **context)``; context keys (``lang``,
|
|
59
|
+
engine-specific options…) pass through the router opaquely. ``requires``
|
|
60
|
+
names an optional extra the transform needs — metadata only; the
|
|
61
|
+
transform itself raises :class:`ImportError` with an install hint.
|
|
62
|
+
``cost`` defaults from ``lossless`` so routing prefers lossless paths.
|
|
63
|
+
"""
|
|
64
|
+
src: str
|
|
65
|
+
dst: str
|
|
66
|
+
fn: Callable[..., str]
|
|
67
|
+
lossless: bool = False
|
|
68
|
+
requires: Optional[str] = None
|
|
69
|
+
cost: float = field(default=-1.0)
|
|
70
|
+
|
|
71
|
+
def __post_init__(self):
|
|
72
|
+
if self.cost < 0:
|
|
73
|
+
object.__setattr__(self, "cost",
|
|
74
|
+
1.0 if self.lossless else _LOSSY_COST)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class ConversionGraph:
|
|
78
|
+
"""A registry of edges plus cheapest-path routing between representations."""
|
|
79
|
+
|
|
80
|
+
def __init__(self, edges: Optional[List[Edge]] = None):
|
|
81
|
+
self._edges: Dict[str, List[Edge]] = {}
|
|
82
|
+
for e in edges or []:
|
|
83
|
+
self.register(e)
|
|
84
|
+
|
|
85
|
+
def register(self, edge: Edge) -> None:
|
|
86
|
+
"""Add an edge. A later edge for the same (src, dst) pair wins routing
|
|
87
|
+
only if cheaper; both remain queryable via :attr:`edges`."""
|
|
88
|
+
self._edges.setdefault(edge.src, []).append(edge)
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def edges(self) -> List[Edge]:
|
|
92
|
+
return [e for outs in self._edges.values() for e in outs]
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def nodes(self) -> List[str]:
|
|
96
|
+
seen = {e.src for e in self.edges} | {e.dst for e in self.edges}
|
|
97
|
+
return sorted(seen)
|
|
98
|
+
|
|
99
|
+
def copy(self) -> "ConversionGraph":
|
|
100
|
+
"""Independent copy — registrations on it never affect this graph."""
|
|
101
|
+
return ConversionGraph(self.edges)
|
|
102
|
+
|
|
103
|
+
def extend(self, register_fn: Callable[["ConversionGraph"], None]) -> "ConversionGraph":
|
|
104
|
+
"""Return a copy extended by *register_fn* (explicit, no autoloading)."""
|
|
105
|
+
g = self.copy()
|
|
106
|
+
register_fn(g)
|
|
107
|
+
return g
|
|
108
|
+
|
|
109
|
+
def route(self, src: str, dst: str) -> List[Edge]:
|
|
110
|
+
"""Cheapest edge path from *src* to *dst* (empty when src == dst).
|
|
111
|
+
|
|
112
|
+
Raises :class:`ValueError` when unroutable, naming the reachable
|
|
113
|
+
targets so callers can self-diagnose.
|
|
114
|
+
"""
|
|
115
|
+
if src == dst:
|
|
116
|
+
return []
|
|
117
|
+
best: Dict[str, float] = {src: 0.0}
|
|
118
|
+
back: Dict[str, Edge] = {}
|
|
119
|
+
queue: List[Tuple[float, str]] = [(0.0, src)]
|
|
120
|
+
while queue:
|
|
121
|
+
cost, node = heapq.heappop(queue)
|
|
122
|
+
if node == dst:
|
|
123
|
+
break
|
|
124
|
+
if cost > best.get(node, float("inf")):
|
|
125
|
+
continue
|
|
126
|
+
for e in self._edges.get(node, ()):
|
|
127
|
+
nxt = cost + e.cost
|
|
128
|
+
if nxt < best.get(e.dst, float("inf")):
|
|
129
|
+
best[e.dst] = nxt
|
|
130
|
+
back[e.dst] = e
|
|
131
|
+
heapq.heappush(queue, (nxt, e.dst))
|
|
132
|
+
if dst not in back:
|
|
133
|
+
reachable = sorted(n for n in best if n != src)
|
|
134
|
+
raise ValueError(
|
|
135
|
+
f"no conversion path {src!r} -> {dst!r}; "
|
|
136
|
+
f"reachable from {src!r}: {', '.join(reachable) or '(nothing)'}")
|
|
137
|
+
path: List[Edge] = []
|
|
138
|
+
node = dst
|
|
139
|
+
while node != src:
|
|
140
|
+
edge = back[node]
|
|
141
|
+
path.append(edge)
|
|
142
|
+
node = edge.src
|
|
143
|
+
return list(reversed(path))
|
|
144
|
+
|
|
145
|
+
def can_convert(self, src: str, dst: str) -> bool:
|
|
146
|
+
if src == dst:
|
|
147
|
+
return True
|
|
148
|
+
try:
|
|
149
|
+
self.route(src, dst)
|
|
150
|
+
return True
|
|
151
|
+
except ValueError:
|
|
152
|
+
return False
|
|
153
|
+
|
|
154
|
+
def convert(self, text: str, src: str, dst: str, **context) -> str:
|
|
155
|
+
"""Convert *text* along the cheapest path; context passes through."""
|
|
156
|
+
for edge in self.route(src, dst):
|
|
157
|
+
text = edge.fn(text, **context)
|
|
158
|
+
return text
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# ---------------------------------------------------------------------------
|
|
162
|
+
# Representation registry and the default graph
|
|
163
|
+
# ---------------------------------------------------------------------------
|
|
164
|
+
|
|
165
|
+
REPRESENTATIONS: Dict[str, Representation] = {r.id: r for r in (
|
|
166
|
+
Representation("ipa", "notation", reference="International Phonetic Alphabet"),
|
|
167
|
+
Representation("arpa", "notation", reference="ARPABET"),
|
|
168
|
+
Representation("x-sampa", "notation", reference="X-SAMPA"),
|
|
169
|
+
Representation("lexique", "notation", reference="Lexique 383"),
|
|
170
|
+
Representation("kirshenbaum", "notation", reference="Kirshenbaum ASCII-IPA"),
|
|
171
|
+
Representation("cotovia", "notation", reference="Cotovía (Galician TTS)"),
|
|
172
|
+
Representation("rfe", "notation", reference="Revista de Filología Española"),
|
|
173
|
+
Representation("buckwalter", "notation", script="Latn",
|
|
174
|
+
reference="Buckwalter Arabic transliteration"),
|
|
175
|
+
Representation("arabic", "orthography", script="Arab",
|
|
176
|
+
reference="Arabic script"),
|
|
177
|
+
Representation("japanese", "orthography", script="Jpan", system="japanese",
|
|
178
|
+
reference="ordinary mixed Japanese orthography (kanji+kana)"),
|
|
179
|
+
Representation("hira", "orthography", script="Hira", reference="Hiragana"),
|
|
180
|
+
Representation("kana", "orthography", script="Kana", reference="Katakana"),
|
|
181
|
+
Representation("hangul", "orthography", script="Hang",
|
|
182
|
+
reference="Hangul syllable blocks"),
|
|
183
|
+
Representation("jamo", "orthography", script="Hang",
|
|
184
|
+
reference="decomposed jamo (form is the jamo-form convention)"),
|
|
185
|
+
Representation("hanzi", "orthography", script="Hani",
|
|
186
|
+
reference="Chinese characters"),
|
|
187
|
+
Representation("cangjie", "orthography", script="Hani",
|
|
188
|
+
reference="Cangjie5 input codes (glyph shape)"),
|
|
189
|
+
Representation("pinyin", "orthography", script="Latn", system="pinyin",
|
|
190
|
+
reference="Hanyu Pinyin romanization"),
|
|
191
|
+
Representation("bopomofo", "orthography", script="Bopo",
|
|
192
|
+
reference="Zhuyin fuhao"),
|
|
193
|
+
)}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _notation_edges() -> List[Edge]:
|
|
197
|
+
from scriptconv.notation import (_TO_IPA, _FROM_IPA, _DIRECT_PAIRS,
|
|
198
|
+
NOTATION_INFO)
|
|
199
|
+
edges = []
|
|
200
|
+
for notation, fn in _TO_IPA.items():
|
|
201
|
+
info = NOTATION_INFO[notation]
|
|
202
|
+
edges.append(Edge(notation.value, "ipa", (lambda f: lambda t, **_: f(t))(fn),
|
|
203
|
+
lossless=info.lossless_to_ipa))
|
|
204
|
+
for notation, fn in _FROM_IPA.items():
|
|
205
|
+
info = NOTATION_INFO[notation]
|
|
206
|
+
edges.append(Edge("ipa", notation.value, (lambda f: lambda t, **_: f(t))(fn),
|
|
207
|
+
lossless=info.lossless_from_ipa))
|
|
208
|
+
for (src, dst), fn in _DIRECT_PAIRS.items():
|
|
209
|
+
edges.append(Edge(src.value, dst.value, (lambda f: lambda t, **_: f(t))(fn),
|
|
210
|
+
lossless=True))
|
|
211
|
+
return edges
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _orthography_edges() -> List[Edge]:
|
|
215
|
+
def hira_to_kana(text, **_):
|
|
216
|
+
from scriptconv.translit import hira_to_kana as f
|
|
217
|
+
return f(text)
|
|
218
|
+
|
|
219
|
+
def kana_to_hira(text, **_):
|
|
220
|
+
from scriptconv.translit import kana_to_hira as f
|
|
221
|
+
return f(text)
|
|
222
|
+
|
|
223
|
+
def hangul_to_jamo(text, **_):
|
|
224
|
+
from scriptconv.translit import decompose_hangul
|
|
225
|
+
return decompose_hangul(text)
|
|
226
|
+
|
|
227
|
+
def japanese_to_hira(text, **_):
|
|
228
|
+
from scriptconv.readings import to_hiragana
|
|
229
|
+
return to_hiragana(text)
|
|
230
|
+
|
|
231
|
+
def japanese_to_kana(text, **_):
|
|
232
|
+
from scriptconv.readings import to_katakana
|
|
233
|
+
return to_katakana(text)
|
|
234
|
+
|
|
235
|
+
def hanzi_to_pinyin(text, **_):
|
|
236
|
+
from scriptconv.readings import to_pinyin
|
|
237
|
+
return to_pinyin(text)
|
|
238
|
+
|
|
239
|
+
def hanzi_to_bopomofo(text, **_):
|
|
240
|
+
from scriptconv.readings import to_bopomofo
|
|
241
|
+
return to_bopomofo(text)
|
|
242
|
+
|
|
243
|
+
def hanzi_to_cangjie(text, **_):
|
|
244
|
+
from scriptconv.cangjie import to_cangjie
|
|
245
|
+
return to_cangjie(text)
|
|
246
|
+
|
|
247
|
+
return [
|
|
248
|
+
Edge("hira", "kana", hira_to_kana, lossless=True),
|
|
249
|
+
Edge("kana", "hira", kana_to_hira, lossless=True),
|
|
250
|
+
Edge("hangul", "jamo", hangul_to_jamo, lossless=False),
|
|
251
|
+
Edge("japanese", "hira", japanese_to_hira, requires="ja"),
|
|
252
|
+
Edge("japanese", "kana", japanese_to_kana, requires="ja"),
|
|
253
|
+
Edge("hanzi", "pinyin", hanzi_to_pinyin, requires="zh"),
|
|
254
|
+
Edge("hanzi", "bopomofo", hanzi_to_bopomofo, requires="zh"),
|
|
255
|
+
Edge("hanzi", "cangjie", hanzi_to_cangjie),
|
|
256
|
+
]
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
DEFAULT_GRAPH = ConversionGraph(_notation_edges() + _orthography_edges())
|
|
@@ -951,19 +951,14 @@ def convert(text: str, src: str | Notation, dst: str | Notation) -> str:
|
|
|
951
951
|
if src == dst:
|
|
952
952
|
return text
|
|
953
953
|
|
|
954
|
-
#
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
# Buckwalter ↔ Arabic script is a direct pair (not IPA-routable).
|
|
963
|
-
if (src, dst) in _DIRECT_PAIRS:
|
|
964
|
-
return _DIRECT_PAIRS[(src, dst)](text)
|
|
965
|
-
|
|
966
|
-
raise ValueError(f"Unsupported conversion: {src!r} → {dst!r}")
|
|
954
|
+
# Routing lives on the shared conversion graph; the notation tables below
|
|
955
|
+
# are registered there as edges (IPA-pivot pairs route through the "ipa"
|
|
956
|
+
# node, Buckwalter <-> Arabic is a direct edge).
|
|
957
|
+
from scriptconv.graph import DEFAULT_GRAPH
|
|
958
|
+
try:
|
|
959
|
+
return DEFAULT_GRAPH.convert(text, src.value, dst.value)
|
|
960
|
+
except ValueError:
|
|
961
|
+
raise ValueError(f"Unsupported conversion: {src!r} → {dst!r}") from None
|
|
967
962
|
|
|
968
963
|
|
|
969
964
|
# ---------------------------------------------------------------------------
|
|
@@ -1000,20 +995,7 @@ _DIRECT_PAIRS: dict[tuple[Notation, Notation], "callable"] = {
|
|
|
1000
995
|
# can_convert — predicate for supported conversion paths
|
|
1001
996
|
# ---------------------------------------------------------------------------
|
|
1002
997
|
|
|
1003
|
-
|
|
1004
|
-
pairs: set[tuple[Notation, Notation]] = set(_DIRECT_PAIRS)
|
|
1005
|
-
for _s in _TO_IPA:
|
|
1006
|
-
pairs.add((_s, Notation.IPA))
|
|
1007
|
-
for _d in _FROM_IPA:
|
|
1008
|
-
pairs.add((Notation.IPA, _d))
|
|
1009
|
-
for _s in _TO_IPA:
|
|
1010
|
-
for _d in _FROM_IPA:
|
|
1011
|
-
if _s != _d:
|
|
1012
|
-
pairs.add((_s, _d))
|
|
1013
|
-
return pairs
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
_SUPPORTED_PAIRS: set[tuple[Notation, Notation]] = _build_supported_pairs()
|
|
998
|
+
_NOTATION_VALUES = {n.value for n in Notation}
|
|
1017
999
|
|
|
1018
1000
|
|
|
1019
1001
|
def can_convert(src: str | Notation, dst: str | Notation) -> bool:
|
|
@@ -1037,7 +1019,16 @@ def can_convert(src: str | Notation, dst: str | Notation) -> bool:
|
|
|
1037
1019
|
"""
|
|
1038
1020
|
src = Notation(src)
|
|
1039
1021
|
dst = Notation(dst)
|
|
1040
|
-
|
|
1022
|
+
if src == dst:
|
|
1023
|
+
# historical contract: identity is not a "conversion"
|
|
1024
|
+
return False
|
|
1025
|
+
from scriptconv.graph import DEFAULT_GRAPH
|
|
1026
|
+
if not DEFAULT_GRAPH.can_convert(src.value, dst.value):
|
|
1027
|
+
return False
|
|
1028
|
+
# only notation-to-notation reachability counts here; guard against paths
|
|
1029
|
+
# that would leave the notation node set (none exist today, cheap to keep)
|
|
1030
|
+
return all(e.src in _NOTATION_VALUES and e.dst in _NOTATION_VALUES
|
|
1031
|
+
for e in DEFAULT_GRAPH.route(src.value, dst.value))
|
|
1041
1032
|
|
|
1042
1033
|
|
|
1043
1034
|
# ---------------------------------------------------------------------------
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptconv
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.3a8
|
|
4
4
|
Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
|
|
@@ -6,6 +6,7 @@ scriptconv/__init__.py
|
|
|
6
6
|
scriptconv/__main__.py
|
|
7
7
|
scriptconv/cangjie.py
|
|
8
8
|
scriptconv/conventions.py
|
|
9
|
+
scriptconv/graph.py
|
|
9
10
|
scriptconv/notation.py
|
|
10
11
|
scriptconv/py.typed
|
|
11
12
|
scriptconv/readings.py
|
|
@@ -23,6 +24,7 @@ tests/test_cangjie.py
|
|
|
23
24
|
tests/test_cli.py
|
|
24
25
|
tests/test_conventions.py
|
|
25
26
|
tests/test_examples.py
|
|
27
|
+
tests/test_graph.py
|
|
26
28
|
tests/test_notation.py
|
|
27
29
|
tests/test_readings.py
|
|
28
30
|
tests/test_readings_zh.py
|
|
@@ -108,3 +108,20 @@ def test_strip_unknown_convention_clean_error():
|
|
|
108
108
|
with pytest.raises(SystemExit) as exc:
|
|
109
109
|
main(["strip", "bogus", "text"])
|
|
110
110
|
assert str(exc.value).startswith("error:")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def test_route_notation_pair(capsys):
|
|
114
|
+
assert main(["route", "arpa", "x-sampa"]) == 0
|
|
115
|
+
out = capsys.readouterr().out
|
|
116
|
+
assert "arpa -> ipa" in out and "ipa -> x-sampa" in out
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def test_convert_orthography_pair_via_graph(capsys):
|
|
120
|
+
assert main(["convert", "hira", "kana", "こんにちは"]) == 0
|
|
121
|
+
assert capsys.readouterr().out.strip() == "コンニチハ"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def test_route_unroutable_clean_error():
|
|
125
|
+
with pytest.raises(SystemExit) as exc:
|
|
126
|
+
main(["route", "ipa", "hangul"])
|
|
127
|
+
assert str(exc.value).startswith("error:")
|
|
@@ -208,3 +208,31 @@ class TestRestyleContract(unittest.TestCase):
|
|
|
208
208
|
|
|
209
209
|
if __name__ == "__main__":
|
|
210
210
|
unittest.main()
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
class TestEncapsulation(unittest.TestCase):
|
|
214
|
+
"""Operations are fully driven by a convention's own transitions/detector."""
|
|
215
|
+
|
|
216
|
+
def test_ad_hoc_convention_works_through_generic_operations(self):
|
|
217
|
+
from scriptconv.conventions import Convention, Transition
|
|
218
|
+
rot = Convention(
|
|
219
|
+
"x-demo", "demo", ("Latn",), None, ("upper", "none"), "none", (),
|
|
220
|
+
"test fixture",
|
|
221
|
+
transitions=(Transition("none", "upper", str.upper),
|
|
222
|
+
Transition("upper", "none", str.lower)),
|
|
223
|
+
detector=lambda t: "upper" if t.isupper() else "none")
|
|
224
|
+
self.assertEqual(rot.apply("abc"), "ABC")
|
|
225
|
+
self.assertEqual(rot.strip("ABC"), "abc")
|
|
226
|
+
self.assertEqual(rot.detect("ABC"), "upper")
|
|
227
|
+
self.assertEqual(rot.restyle("ABC", "none"), "abc")
|
|
228
|
+
|
|
229
|
+
def test_wakachigaki_apply_requires_is_queryable(self):
|
|
230
|
+
conv = CONVENTION_REGISTRY["wakachigaki"]
|
|
231
|
+
self.assertEqual(conv.apply_requires, "ja")
|
|
232
|
+
self.assertIsNone(CONVENTION_REGISTRY["tashkeel"].apply_requires)
|
|
233
|
+
|
|
234
|
+
def test_jamo_form_strip_lossless_derived_from_transitions(self):
|
|
235
|
+
self.assertFalse(CONVENTION_REGISTRY["tashkeel"].strip_lossless)
|
|
236
|
+
self.assertFalse(CONVENTION_REGISTRY["jamo-form"].strip_lossless) # no "none" transitions
|
|
237
|
+
self.assertTrue(all(t.lossless for t in
|
|
238
|
+
CONVENTION_REGISTRY["jamo-form"].transitions))
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import unittest
|
|
3
|
+
from unittest import mock
|
|
4
|
+
|
|
5
|
+
from scriptconv.graph import (
|
|
6
|
+
DEFAULT_GRAPH,
|
|
7
|
+
REPRESENTATIONS,
|
|
8
|
+
ConversionGraph,
|
|
9
|
+
Edge,
|
|
10
|
+
Representation,
|
|
11
|
+
)
|
|
12
|
+
from scriptconv.notation import Notation, can_convert, convert
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TestEngine(unittest.TestCase):
|
|
16
|
+
def test_identity_is_empty_route(self):
|
|
17
|
+
g = ConversionGraph([Edge("a", "b", lambda t, **_: t + "b")])
|
|
18
|
+
self.assertEqual(g.route("a", "a"), [])
|
|
19
|
+
self.assertEqual(g.convert("x", "a", "a"), "x")
|
|
20
|
+
|
|
21
|
+
def test_multi_hop_route_and_convert(self):
|
|
22
|
+
g = ConversionGraph([
|
|
23
|
+
Edge("a", "b", lambda t, **_: t + ">b", lossless=True),
|
|
24
|
+
Edge("b", "c", lambda t, **_: t + ">c", lossless=True),
|
|
25
|
+
])
|
|
26
|
+
self.assertEqual([e.dst for e in g.route("a", "c")], ["b", "c"])
|
|
27
|
+
self.assertEqual(g.convert("x", "a", "c"), "x>b>c")
|
|
28
|
+
|
|
29
|
+
def test_lossless_two_hop_beats_lossy_direct(self):
|
|
30
|
+
g = ConversionGraph([
|
|
31
|
+
Edge("a", "c", lambda t, **_: "LOSSY"),
|
|
32
|
+
Edge("a", "b", lambda t, **_: t + ">b", lossless=True),
|
|
33
|
+
Edge("b", "c", lambda t, **_: t + ">c", lossless=True),
|
|
34
|
+
])
|
|
35
|
+
self.assertEqual(g.convert("x", "a", "c"), "x>b>c")
|
|
36
|
+
|
|
37
|
+
def test_unroutable_names_reachable_targets(self):
|
|
38
|
+
g = ConversionGraph([Edge("a", "b", lambda t, **_: t, lossless=True)])
|
|
39
|
+
with self.assertRaises(ValueError) as ctx:
|
|
40
|
+
g.route("a", "z")
|
|
41
|
+
self.assertIn("reachable from 'a': b", str(ctx.exception))
|
|
42
|
+
|
|
43
|
+
def test_context_passes_through_opaquely(self):
|
|
44
|
+
seen = {}
|
|
45
|
+
|
|
46
|
+
def edge_fn(text, **context):
|
|
47
|
+
seen.update(context)
|
|
48
|
+
return text
|
|
49
|
+
|
|
50
|
+
g = ConversionGraph([Edge("a", "b", edge_fn)])
|
|
51
|
+
g.convert("x", "a", "b", lang="pt-PT", phoneme_type="espeak")
|
|
52
|
+
self.assertEqual(seen, {"lang": "pt-PT", "phoneme_type": "espeak"})
|
|
53
|
+
|
|
54
|
+
def test_copy_and_extend_are_isolated(self):
|
|
55
|
+
g = ConversionGraph([Edge("a", "b", lambda t, **_: t)])
|
|
56
|
+
g2 = g.extend(lambda gg: gg.register(Edge("b", "c", lambda t, **_: t)))
|
|
57
|
+
self.assertTrue(g2.can_convert("a", "c"))
|
|
58
|
+
self.assertFalse(g.can_convert("a", "c"))
|
|
59
|
+
|
|
60
|
+
def test_cost_defaults_from_lossless(self):
|
|
61
|
+
self.assertEqual(Edge("a", "b", lambda t, **_: t, lossless=True).cost, 1.0)
|
|
62
|
+
self.assertGreater(Edge("a", "b", lambda t, **_: t).cost, 1.0)
|
|
63
|
+
self.assertEqual(Edge("a", "b", lambda t, **_: t, cost=3.5).cost, 3.5)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class TestNotationParity(unittest.TestCase):
|
|
67
|
+
def test_every_supported_notation_pair_routes_and_matches(self):
|
|
68
|
+
samples = {"ipa": "həˈloʊ", "arpa": "HH AH0 L OW1"}
|
|
69
|
+
for src in Notation:
|
|
70
|
+
for dst in Notation:
|
|
71
|
+
if src == dst:
|
|
72
|
+
continue
|
|
73
|
+
graph_ok = DEFAULT_GRAPH.can_convert(src.value, dst.value)
|
|
74
|
+
self.assertEqual(can_convert(src, dst), graph_ok,
|
|
75
|
+
f"{src}->{dst}")
|
|
76
|
+
if graph_ok and src.value in samples:
|
|
77
|
+
self.assertEqual(
|
|
78
|
+
convert(samples[src.value], src, dst),
|
|
79
|
+
DEFAULT_GRAPH.convert(samples[src.value],
|
|
80
|
+
src.value, dst.value))
|
|
81
|
+
|
|
82
|
+
def test_buckwalter_not_ipa_routable(self):
|
|
83
|
+
self.assertFalse(DEFAULT_GRAPH.can_convert("buckwalter", "ipa"))
|
|
84
|
+
self.assertTrue(DEFAULT_GRAPH.can_convert("buckwalter", "arabic"))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class TestDefaultGraphOrthography(unittest.TestCase):
|
|
88
|
+
def test_hira_kana_edges(self):
|
|
89
|
+
self.assertEqual(DEFAULT_GRAPH.convert("こんにちは", "hira", "kana"),
|
|
90
|
+
"コンニチハ")
|
|
91
|
+
self.assertEqual(DEFAULT_GRAPH.convert("コーヒー", "kana", "hira"),
|
|
92
|
+
"こーひー")
|
|
93
|
+
|
|
94
|
+
def test_hangul_to_jamo(self):
|
|
95
|
+
self.assertEqual(DEFAULT_GRAPH.convert("가", "hangul", "jamo"), "ㄱㅏ")
|
|
96
|
+
|
|
97
|
+
def test_hanzi_to_cangjie(self):
|
|
98
|
+
self.assertEqual(DEFAULT_GRAPH.convert("日", "hanzi", "cangjie"), "a")
|
|
99
|
+
|
|
100
|
+
def test_japanese_readings_edges(self):
|
|
101
|
+
self.assertEqual(DEFAULT_GRAPH.convert("東京", "japanese", "hira"),
|
|
102
|
+
"とうきょう")
|
|
103
|
+
self.assertEqual(DEFAULT_GRAPH.convert("中国", "hanzi", "pinyin"),
|
|
104
|
+
"zhōng guó")
|
|
105
|
+
|
|
106
|
+
def test_requires_metadata_and_import_hint(self):
|
|
107
|
+
ja_edges = [e for e in DEFAULT_GRAPH.edges
|
|
108
|
+
if e.src == "japanese" and e.dst == "hira"]
|
|
109
|
+
self.assertEqual(ja_edges[0].requires, "ja")
|
|
110
|
+
import scriptconv.readings as readings
|
|
111
|
+
with mock.patch.object(readings, "_kakasi", None), \
|
|
112
|
+
mock.patch.dict(sys.modules, {"pykakasi": None}):
|
|
113
|
+
with self.assertRaises(ImportError) as ctx:
|
|
114
|
+
DEFAULT_GRAPH.convert("東京", "japanese", "hira")
|
|
115
|
+
self.assertIn("scriptconv[ja]", str(ctx.exception))
|
|
116
|
+
|
|
117
|
+
def test_representations_cover_all_default_nodes(self):
|
|
118
|
+
for node in DEFAULT_GRAPH.nodes:
|
|
119
|
+
self.assertIn(node, REPRESENTATIONS, node)
|
|
120
|
+
|
|
121
|
+
def test_no_graphemes_node(self):
|
|
122
|
+
self.assertNotIn("graphemes", REPRESENTATIONS)
|
|
123
|
+
self.assertNotIn("graphemes", DEFAULT_GRAPH.nodes)
|
|
124
|
+
|
|
125
|
+
def test_representation_kinds(self):
|
|
126
|
+
self.assertEqual(REPRESENTATIONS["ipa"].kind, "notation")
|
|
127
|
+
self.assertEqual(REPRESENTATIONS["pinyin"].system, "pinyin")
|
|
128
|
+
self.assertEqual(REPRESENTATIONS["cangjie"].script, "Hani")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
if __name__ == "__main__":
|
|
132
|
+
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|