scriptconv 0.0.3a7__tar.gz → 0.0.3a9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/PKG-INFO +10 -1
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/README.md +9 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/__init__.py +21 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/__main__.py +29 -1
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/conventions.py +205 -164
- scriptconv-0.0.3a9/scriptconv/graph.py +265 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/notation.py +121 -64
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/version.py +1 -1
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv.egg-info/PKG-INFO +10 -1
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv.egg-info/SOURCES.txt +3 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_cli.py +17 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_conventions.py +28 -0
- scriptconv-0.0.3a9/tests/test_errors_policy.py +102 -0
- scriptconv-0.0.3a9/tests/test_graph.py +132 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/LICENSE +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/pyproject.toml +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/requirements.txt +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/cangjie.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/data/__init__.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/data/cangjie5_tc.tsv.gz +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/py.typed +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/readings.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/scripts.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/translit.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv.egg-info/dependency_links.txt +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv.egg-info/requires.txt +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv.egg-info/top_level.txt +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/setup.cfg +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_cangjie.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_examples.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_notation.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_readings.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_readings_zh.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_scripts.py +0 -0
- {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_translit.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptconv
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.3a9
|
|
4
4
|
Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
|
|
@@ -199,6 +199,15 @@ Transcoding faithfulness depends on the target notation's inventory. IPA is the
|
|
|
199
199
|
notation↔notation goes through IPA. The table below states, for each notation, whether a
|
|
200
200
|
round-trip is exact and what happens to a symbol the table does not know.
|
|
201
201
|
|
|
202
|
+
Every converter (and `convert()`) accepts a codecs-style `errors=` policy for
|
|
203
|
+
symbols outside its table: `"pass"` keeps the symbol (the default everywhere
|
|
204
|
+
except `ipa_to_arpa`), `"replace"` substitutes the notation's placeholder
|
|
205
|
+
(`?`; the historical `ipa_to_arpa` default, tunable via `unknown=`),
|
|
206
|
+
`"ignore"` drops it, and `"strict"` raises `UnknownSymbolError` naming the
|
|
207
|
+
symbol, its position, and the notation. The "Unknown-token behaviour" column
|
|
208
|
+
below describes the per-converter default.
|
|
209
|
+
|
|
210
|
+
|
|
202
211
|
| Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
|
|
203
212
|
|----------|----------------------------------|----------------------------------|-------------------------|
|
|
204
213
|
| **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |
|
|
@@ -166,6 +166,15 @@ Transcoding faithfulness depends on the target notation's inventory. IPA is the
|
|
|
166
166
|
notation↔notation goes through IPA. The table below states, for each notation, whether a
|
|
167
167
|
round-trip is exact and what happens to a symbol the table does not know.
|
|
168
168
|
|
|
169
|
+
Every converter (and `convert()`) accepts a codecs-style `errors=` policy for
|
|
170
|
+
symbols outside its table: `"pass"` keeps the symbol (the default everywhere
|
|
171
|
+
except `ipa_to_arpa`), `"replace"` substitutes the notation's placeholder
|
|
172
|
+
(`?`; the historical `ipa_to_arpa` default, tunable via `unknown=`),
|
|
173
|
+
`"ignore"` drops it, and `"strict"` raises `UnknownSymbolError` naming the
|
|
174
|
+
symbol, its position, and the notation. The "Unknown-token behaviour" column
|
|
175
|
+
below describes the per-converter default.
|
|
176
|
+
|
|
177
|
+
|
|
169
178
|
| Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
|
|
170
179
|
|----------|----------------------------------|----------------------------------|-------------------------|
|
|
171
180
|
| **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |
|
|
@@ -22,6 +22,10 @@ readings:
|
|
|
22
22
|
extra; Chinese hanzi → pinyin/bopomofo via the ``zh`` extra).
|
|
23
23
|
cangjie:
|
|
24
24
|
Hanzi → Cangjie5 input codes (shape decomposition, vendored table).
|
|
25
|
+
graph:
|
|
26
|
+
Conversion-graph engine: representations (notations and orthographies)
|
|
27
|
+
as nodes, registered transforms as edges, lossless-preferring routing.
|
|
28
|
+
External packages extend graph instances explicitly.
|
|
25
29
|
conventions:
|
|
26
30
|
Orthographic conventions — script-scoped decorations orthogonal to
|
|
27
31
|
identity (tashkeel, niqqud, wakachigaki, pinyin tone spelling…).
|
|
@@ -45,6 +49,7 @@ from scriptconv.scripts import (
|
|
|
45
49
|
)
|
|
46
50
|
from scriptconv.notation import (
|
|
47
51
|
Notation,
|
|
52
|
+
UnknownSymbolError,
|
|
48
53
|
NotationInfo,
|
|
49
54
|
NOTATION_INFO,
|
|
50
55
|
convert,
|
|
@@ -81,6 +86,7 @@ from scriptconv.readings import (
|
|
|
81
86
|
)
|
|
82
87
|
from scriptconv.conventions import (
|
|
83
88
|
Convention,
|
|
89
|
+
Transition,
|
|
84
90
|
CONVENTION_REGISTRY,
|
|
85
91
|
conventions_for,
|
|
86
92
|
restyle,
|
|
@@ -88,6 +94,13 @@ from scriptconv.conventions import (
|
|
|
88
94
|
apply,
|
|
89
95
|
detect as detect_convention,
|
|
90
96
|
)
|
|
97
|
+
from scriptconv.graph import (
|
|
98
|
+
Representation,
|
|
99
|
+
Edge,
|
|
100
|
+
ConversionGraph,
|
|
101
|
+
DEFAULT_GRAPH,
|
|
102
|
+
REPRESENTATIONS,
|
|
103
|
+
)
|
|
91
104
|
from scriptconv.cangjie import (
|
|
92
105
|
cangjie_code,
|
|
93
106
|
to_cangjie,
|
|
@@ -107,6 +120,7 @@ __all__ = [
|
|
|
107
120
|
"normalize_script_tag",
|
|
108
121
|
# notation
|
|
109
122
|
"Notation",
|
|
123
|
+
"UnknownSymbolError",
|
|
110
124
|
"NotationInfo",
|
|
111
125
|
"NOTATION_INFO",
|
|
112
126
|
"convert",
|
|
@@ -138,12 +152,19 @@ __all__ = [
|
|
|
138
152
|
"to_katakana",
|
|
139
153
|
# conventions (orthographic decorations, orthogonal to script identity)
|
|
140
154
|
"Convention",
|
|
155
|
+
"Transition",
|
|
141
156
|
"CONVENTION_REGISTRY",
|
|
142
157
|
"conventions_for",
|
|
143
158
|
"restyle",
|
|
144
159
|
"strip",
|
|
145
160
|
"apply",
|
|
146
161
|
"detect_convention",
|
|
162
|
+
# graph (conversion routing engine over representations)
|
|
163
|
+
"Representation",
|
|
164
|
+
"Edge",
|
|
165
|
+
"ConversionGraph",
|
|
166
|
+
"DEFAULT_GRAPH",
|
|
167
|
+
"REPRESENTATIONS",
|
|
147
168
|
# cangjie (vendored Cangjie5 table)
|
|
148
169
|
"cangjie_code",
|
|
149
170
|
"to_cangjie",
|
|
@@ -11,6 +11,7 @@ Usage
|
|
|
11
11
|
python -m scriptconv strip <convention> <text>
|
|
12
12
|
python -m scriptconv restyle <convention> <style> <text>
|
|
13
13
|
python -m scriptconv conventions [--script CODE]
|
|
14
|
+
python -m scriptconv route <src> <dst>
|
|
14
15
|
|
|
15
16
|
Examples
|
|
16
17
|
--------
|
|
@@ -64,6 +65,10 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
64
65
|
cv = sub.add_parser("conventions", help="List orthographic conventions")
|
|
65
66
|
cv.add_argument("--script", default=None, help="Filter by ISO-15924 script code")
|
|
66
67
|
|
|
68
|
+
rt = sub.add_parser("route", help="Show the conversion path between representations")
|
|
69
|
+
rt.add_argument("src", help="Source representation id")
|
|
70
|
+
rt.add_argument("dst", help="Target representation id")
|
|
71
|
+
|
|
67
72
|
args = p.parse_args(argv)
|
|
68
73
|
|
|
69
74
|
if args.command is None:
|
|
@@ -80,6 +85,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
80
85
|
"strip": lambda: _do_strip(args),
|
|
81
86
|
"restyle": lambda: _do_restyle(args),
|
|
82
87
|
"conventions": lambda: _do_conventions(args),
|
|
88
|
+
"route": lambda: _do_route(args),
|
|
83
89
|
}
|
|
84
90
|
dispatch[args.command]()
|
|
85
91
|
return 0
|
|
@@ -89,9 +95,31 @@ def _do_convert(args: argparse.Namespace) -> None:
|
|
|
89
95
|
from scriptconv.notation import Notation, convert
|
|
90
96
|
try:
|
|
91
97
|
print(convert(args.text, args.src, args.dst))
|
|
98
|
+
return
|
|
99
|
+
except ValueError:
|
|
100
|
+
pass
|
|
101
|
+
# not a notation pair — try any representation pair on the default graph
|
|
102
|
+
from scriptconv.graph import DEFAULT_GRAPH
|
|
103
|
+
try:
|
|
104
|
+
print(DEFAULT_GRAPH.convert(args.text, args.src, args.dst))
|
|
92
105
|
except ValueError as e:
|
|
93
106
|
valid = ", ".join(n.value for n in Notation)
|
|
94
|
-
raise SystemExit(f"error: {e} (valid notations: {valid}
|
|
107
|
+
raise SystemExit(f"error: {e} (valid notations: {valid}; "
|
|
108
|
+
f"graph nodes: {', '.join(DEFAULT_GRAPH.nodes)})")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _do_route(args: argparse.Namespace) -> None:
|
|
112
|
+
from scriptconv.graph import DEFAULT_GRAPH
|
|
113
|
+
try:
|
|
114
|
+
path = DEFAULT_GRAPH.route(args.src, args.dst)
|
|
115
|
+
except ValueError as e:
|
|
116
|
+
raise SystemExit(f"error: {e}")
|
|
117
|
+
if not path:
|
|
118
|
+
print("(identity)")
|
|
119
|
+
for e in path:
|
|
120
|
+
flags = "lossless" if e.lossless else "lossy"
|
|
121
|
+
extra = f", needs scriptconv[{e.requires}]" if e.requires else ""
|
|
122
|
+
print(f" {e.src} -> {e.dst} ({flags}, cost {e.cost:g}{extra})")
|
|
95
123
|
|
|
96
124
|
|
|
97
125
|
def _do_detect(args: argparse.Namespace) -> None:
|
|
@@ -43,6 +43,7 @@ repertoires and the restyle is deterministic.
|
|
|
43
43
|
"""
|
|
44
44
|
from __future__ import annotations
|
|
45
45
|
|
|
46
|
+
import functools
|
|
46
47
|
import re
|
|
47
48
|
import unicodedata
|
|
48
49
|
from dataclasses import dataclass
|
|
@@ -50,6 +51,7 @@ from typing import Callable, Dict, List, Optional, Tuple
|
|
|
50
51
|
|
|
51
52
|
__all__ = [
|
|
52
53
|
"Convention",
|
|
54
|
+
"Transition",
|
|
53
55
|
"CONVENTION_REGISTRY",
|
|
54
56
|
"conventions_for",
|
|
55
57
|
"detect",
|
|
@@ -59,98 +61,118 @@ __all__ = [
|
|
|
59
61
|
]
|
|
60
62
|
|
|
61
63
|
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class Transition:
|
|
66
|
+
"""One rewrite between two styles of a convention.
|
|
67
|
+
|
|
68
|
+
``fn`` is a pure ``str -> str`` transform. ``requires`` names the
|
|
69
|
+
optional extra the transform needs, if any (queryable before calling —
|
|
70
|
+
the transform itself raises :class:`ImportError` with an install hint
|
|
71
|
+
when the dependency is missing).
|
|
72
|
+
"""
|
|
73
|
+
frm: str
|
|
74
|
+
to: str
|
|
75
|
+
fn: Callable[[str], str]
|
|
76
|
+
lossless: bool = False
|
|
77
|
+
requires: Optional[str] = None
|
|
78
|
+
|
|
79
|
+
|
|
62
80
|
@dataclass(frozen=True)
|
|
63
81
|
class Convention:
|
|
64
|
-
"""
|
|
82
|
+
"""One orthographic convention: metadata plus the behavior that owns it.
|
|
83
|
+
|
|
84
|
+
Transitions and the optional detector live on the convention itself, so
|
|
85
|
+
every operation is generic — nothing in this module branches on a
|
|
86
|
+
convention's identity.
|
|
87
|
+
"""
|
|
65
88
|
id: str
|
|
66
89
|
name: str
|
|
67
90
|
scripts: Tuple[str, ...] # ISO-15924 codes whose text it decorates
|
|
68
91
|
system: Optional[str] # sub-script system qualifier ("pinyin", "japanese")
|
|
69
92
|
styles: Tuple[str, ...] # mutually exclusive states; includes "none" if strippable
|
|
70
93
|
default_style: str
|
|
71
|
-
detectable: bool # current style is character-evident
|
|
72
|
-
strip_lossless: bool # can the stripped information be recovered?
|
|
73
|
-
apply_requires: Optional[str] # optional-extra name needed to leave "none", if any
|
|
74
94
|
ranges: Tuple[Tuple[int, int], ...] # decoration codepoints (empty for spacing/variant conventions)
|
|
75
95
|
reference: str
|
|
96
|
+
transitions: Tuple[Transition, ...] = ()
|
|
97
|
+
detector: Optional[Callable[[str], Optional[str]]] = None
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def detectable(self) -> bool:
|
|
101
|
+
return self.detector is not None
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def strip_lossless(self) -> bool:
|
|
105
|
+
stripping = [t for t in self.transitions if t.to == "none"]
|
|
106
|
+
return bool(stripping) and all(t.lossless for t in stripping)
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def apply_requires(self) -> Optional[str]:
|
|
110
|
+
return next((t.requires for t in self.transitions if t.frm == "none"), None)
|
|
111
|
+
|
|
112
|
+
def transition(self, frm: str, to: str) -> Optional[Transition]:
|
|
113
|
+
return next((t for t in self.transitions
|
|
114
|
+
if t.frm == frm and t.to == to), None)
|
|
115
|
+
|
|
116
|
+
def restyle(self, text: str, to: str, frm: Optional[str] = None) -> str:
|
|
117
|
+
"""Rewrite *text* from one style of this convention to another."""
|
|
118
|
+
if to not in self.styles and to != "none":
|
|
119
|
+
raise ValueError(f"{self.id!r} has styles {self.styles}, not {to!r}")
|
|
120
|
+
if frm is None:
|
|
121
|
+
frm = self._infer_source(text, to)
|
|
122
|
+
if frm == to:
|
|
123
|
+
return text
|
|
124
|
+
t = self.transition(frm, to)
|
|
125
|
+
if t is None:
|
|
126
|
+
supported = sorted(f"{x.frm}->{x.to}" for x in self.transitions)
|
|
127
|
+
raise ValueError(f"{self.id!r} does not define {frm!r}->{to!r}; "
|
|
128
|
+
f"supported: {', '.join(supported)}")
|
|
129
|
+
return t.fn(text)
|
|
130
|
+
|
|
131
|
+
def _infer_source(self, text: str, to: str) -> str:
|
|
132
|
+
if self.detector is not None:
|
|
133
|
+
detected = self.detector(text)
|
|
134
|
+
if detected is not None and (detected == to
|
|
135
|
+
or self.transition(detected, to)):
|
|
136
|
+
return detected
|
|
137
|
+
sources = sorted({t.frm for t in self.transitions if t.to == to})
|
|
138
|
+
if len(sources) == 1:
|
|
139
|
+
return sources[0]
|
|
140
|
+
if self.default_style != to:
|
|
141
|
+
return self.default_style
|
|
142
|
+
raise ValueError(f"{self.id!r}: source style for ->{to!r} is ambiguous "
|
|
143
|
+
f"({', '.join(sources)}); pass frm=")
|
|
144
|
+
|
|
145
|
+
def strip(self, text: str) -> str:
|
|
146
|
+
"""Remove this convention's decoration (restyle to ``"none"``)."""
|
|
147
|
+
stripping = [t for t in self.transitions if t.to == "none"]
|
|
148
|
+
if not stripping:
|
|
149
|
+
raise ValueError(f"{self.id!r} has no unmarked style; "
|
|
150
|
+
f"styles: {self.styles}")
|
|
151
|
+
# when every stripping transition is the same transform (e.g. one
|
|
152
|
+
# strip handles both pinyin tone spellings), the source is irrelevant
|
|
153
|
+
if len({t.fn for t in stripping}) == 1:
|
|
154
|
+
return stripping[0].fn(text)
|
|
155
|
+
return self.restyle(text, "none")
|
|
156
|
+
|
|
157
|
+
def apply(self, text: str, style: Optional[str] = None) -> str:
|
|
158
|
+
"""Add this convention's decoration to unmarked text."""
|
|
159
|
+
if style is None:
|
|
160
|
+
marked = [s for s in self.styles if s != "none"]
|
|
161
|
+
style = self.default_style if self.default_style != "none" else marked[0]
|
|
162
|
+
return self.restyle(text, style, frm="none")
|
|
163
|
+
|
|
164
|
+
def detect(self, text: str) -> Optional[str]:
|
|
165
|
+
"""Current style of *text*, or None when not character-evident.
|
|
166
|
+
|
|
167
|
+
For binary decorations this means "any mark present" — a single
|
|
168
|
+
shadda in a paragraph reports ``"marked"``.
|
|
169
|
+
"""
|
|
170
|
+
return self.detector(text) if self.detector is not None else None
|
|
76
171
|
|
|
77
172
|
|
|
78
173
|
# ---------------------------------------------------------------------------
|
|
79
|
-
#
|
|
174
|
+
# Shared helpers
|
|
80
175
|
# ---------------------------------------------------------------------------
|
|
81
|
-
# Tashkeel deliberately EXCLUDES U+0653–0655 (madda above, hamza above, hamza
|
|
82
|
-
# below): in NFD text آ/أ/إ decompose to alef + those marks, which are letter
|
|
83
|
-
# identity — stripping them would corrupt the consonantal skeleton.
|
|
84
|
-
_TASHKEEL_RANGES = ((0x064B, 0x0652), (0x0656, 0x065F), (0x0670, 0x0670),
|
|
85
|
-
(0x08F0, 0x08F2))
|
|
86
|
-
_KASHIDA_RANGES = ((0x0640, 0x0640),)
|
|
87
|
-
# 06DD/06DE (end of ayah, rub el hizb) are structural signs and 06E5/06E6 are
|
|
88
|
-
# small letters — kept; 06E7-06E9 (small high marks, place of sajdah) and
|
|
89
|
-
# 06EA-06ED are annotation marks — stripped.
|
|
90
|
-
_QURANIC_RANGES = ((0x06D6, 0x06DC), (0x06DF, 0x06E4), (0x06E7, 0x06ED))
|
|
91
|
-
# Niqqud includes dagesh/mapiq U+05BC, meteg U+05BD (a masoretic stress mark,
|
|
92
|
-
# but traditionally stripped with the points), rafe U+05BF and the shin/sin
|
|
93
|
-
# dots U+05C1/05C2 — stripping loses the שׁ/שׂ distinction, exactly as
|
|
94
|
-
# unpointed Hebrew does. Maqaf U+05BE, sof pasuq U+05C3 and nun hafukha
|
|
95
|
-
# U+05C6 are punctuation and survive.
|
|
96
|
-
_NIQQUD_RANGES = ((0x05B0, 0x05BD), (0x05BF, 0x05BF), (0x05C1, 0x05C2),
|
|
97
|
-
(0x05C7, 0x05C7))
|
|
98
|
-
_TEAMIM_RANGES = ((0x0591, 0x05AF), (0x05C0, 0x05C0), (0x05C4, 0x05C5))
|
|
99
|
-
|
|
100
|
-
CONVENTION_REGISTRY: Dict[str, Convention] = {c.id: c for c in (
|
|
101
|
-
Convention("tashkeel", "Arabic vocalization (harakat/tanwin/shadda/sukun)",
|
|
102
|
-
("Arab",), None, ("marked", "none"), "marked",
|
|
103
|
-
detectable=True, strip_lossless=False, apply_requires=None,
|
|
104
|
-
ranges=_TASHKEEL_RANGES,
|
|
105
|
-
reference="Unicode 15 ch. 9.2 (Arabic); excludes letter-identity U+0653-0655"),
|
|
106
|
-
Convention("kashida", "Arabic justification elongation (tatweel)",
|
|
107
|
-
("Arab",), None, ("marked", "none"), "none",
|
|
108
|
-
detectable=True, strip_lossless=False, apply_requires=None,
|
|
109
|
-
ranges=_KASHIDA_RANGES, reference="Unicode U+0640 ARABIC TATWEEL"),
|
|
110
|
-
Convention("quranic-marks", "Quranic annotation signs",
|
|
111
|
-
("Arab",), None, ("marked", "none"), "none",
|
|
112
|
-
detectable=True, strip_lossless=False, apply_requires=None,
|
|
113
|
-
ranges=_QURANIC_RANGES, reference="Unicode U+06D6-06ED Quranic annotation signs"),
|
|
114
|
-
Convention("niqqud", "Hebrew vocalization points",
|
|
115
|
-
("Hebr",), None, ("marked", "none"), "marked",
|
|
116
|
-
detectable=True, strip_lossless=False, apply_requires=None,
|
|
117
|
-
ranges=_NIQQUD_RANGES,
|
|
118
|
-
reference="Unicode 15 ch. 9.1 (Hebrew); strip loses the shin/sin dot distinction"),
|
|
119
|
-
Convention("teamim", "Hebrew cantillation marks",
|
|
120
|
-
("Hebr",), None, ("marked", "none"), "none",
|
|
121
|
-
detectable=True, strip_lossless=False, apply_requires=None,
|
|
122
|
-
ranges=_TEAMIM_RANGES, reference="Unicode U+0591-05AF Hebrew accents (masoretic)"),
|
|
123
|
-
Convention("wakachigaki", "Japanese spaced writing (分かち書き)",
|
|
124
|
-
("Hira", "Kana", "Hani"), "japanese", ("marked", "none"), "none",
|
|
125
|
-
detectable=False, strip_lossless=False, apply_requires="ja",
|
|
126
|
-
ranges=(),
|
|
127
|
-
reference="Spaced orthographic mode of Japanese (children's books, games)"),
|
|
128
|
-
Convention("pinyin-tone", "Pinyin tone spelling",
|
|
129
|
-
("Latn",), "pinyin", ("mark", "number", "none"), "mark",
|
|
130
|
-
detectable=False, strip_lossless=False, apply_requires=None,
|
|
131
|
-
ranges=(),
|
|
132
|
-
reference="GB/T 16159 Basic rules of Hanyu Pinyin orthography"),
|
|
133
|
-
Convention("jamo-form", "Hangul jamo repertoire (compatibility vs conjoining)",
|
|
134
|
-
("Hang",), None, ("compatibility", "conjoining"), "compatibility",
|
|
135
|
-
detectable=True, strip_lossless=True, apply_requires=None,
|
|
136
|
-
ranges=(),
|
|
137
|
-
reference="Unicode Hangul Compatibility Jamo U+3130 vs Hangul Jamo U+1100"),
|
|
138
|
-
)}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
def conventions_for(script: str) -> List[Convention]:
|
|
142
|
-
"""Conventions that decorate *script* (ISO-15924 code), registry order."""
|
|
143
|
-
return [c for c in CONVENTION_REGISTRY.values() if script in c.scripts]
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
def _conv(convention_id: str) -> Convention:
|
|
147
|
-
try:
|
|
148
|
-
return CONVENTION_REGISTRY[convention_id]
|
|
149
|
-
except KeyError:
|
|
150
|
-
raise ValueError(
|
|
151
|
-
f"unknown convention {convention_id!r}; known: "
|
|
152
|
-
f"{', '.join(sorted(CONVENTION_REGISTRY))}") from None
|
|
153
|
-
|
|
154
176
|
|
|
155
177
|
def _in_ranges(cp: int, ranges: Tuple[Tuple[int, int], ...]) -> bool:
|
|
156
178
|
return any(lo <= cp <= hi for lo, hi in ranges)
|
|
@@ -162,6 +184,12 @@ def _range_strip(ranges: Tuple[Tuple[int, int], ...]) -> Callable[[str], str]:
|
|
|
162
184
|
return _strip
|
|
163
185
|
|
|
164
186
|
|
|
187
|
+
def _range_detector(ranges: Tuple[Tuple[int, int], ...]) -> Callable[[str], Optional[str]]:
|
|
188
|
+
def _detect(text: str) -> Optional[str]:
|
|
189
|
+
return "marked" if any(_in_ranges(ord(c), ranges) for c in text) else "none"
|
|
190
|
+
return _detect
|
|
191
|
+
|
|
192
|
+
|
|
165
193
|
# ---------------------------------------------------------------------------
|
|
166
194
|
# Wakachigaki
|
|
167
195
|
# ---------------------------------------------------------------------------
|
|
@@ -338,6 +366,7 @@ def _pinyin_strip(text: str) -> str:
|
|
|
338
366
|
# U+3131), so a consonant is treated as an onset when a vowel follows,
|
|
339
367
|
# otherwise as a coda — deterministic over well-formed jamo sequences.
|
|
340
368
|
|
|
369
|
+
@functools.lru_cache(maxsize=1)
|
|
341
370
|
def _jamo_tables():
|
|
342
371
|
from scriptconv.translit import _ONSETS as _CO, _VOWELS as _CV, _CODAS as _CT
|
|
343
372
|
conj_to_compat = {}
|
|
@@ -357,22 +386,13 @@ def _jamo_tables():
|
|
|
357
386
|
return conj_to_compat, compat_onset, compat_vowel, compat_coda
|
|
358
387
|
|
|
359
388
|
|
|
360
|
-
_JAMO_TABLES = None
|
|
361
|
-
|
|
362
|
-
|
|
363
389
|
def _jamo_conjoining_to_compat(text: str) -> str:
|
|
364
|
-
|
|
365
|
-
if _JAMO_TABLES is None:
|
|
366
|
-
_JAMO_TABLES = _jamo_tables()
|
|
367
|
-
table = _JAMO_TABLES[0]
|
|
390
|
+
table = _jamo_tables()[0]
|
|
368
391
|
return "".join(table.get(c, c) for c in text)
|
|
369
392
|
|
|
370
393
|
|
|
371
394
|
def _jamo_compat_to_conjoining(text: str) -> str:
|
|
372
|
-
|
|
373
|
-
if _JAMO_TABLES is None:
|
|
374
|
-
_JAMO_TABLES = _jamo_tables()
|
|
375
|
-
_, onset, vowel, coda = _JAMO_TABLES
|
|
395
|
+
_, onset, vowel, coda = _jamo_tables()
|
|
376
396
|
out = []
|
|
377
397
|
for i, c in enumerate(text):
|
|
378
398
|
if c in vowel:
|
|
@@ -401,95 +421,116 @@ def _jamo_detect(text: str) -> Optional[str]:
|
|
|
401
421
|
|
|
402
422
|
|
|
403
423
|
# ---------------------------------------------------------------------------
|
|
404
|
-
#
|
|
424
|
+
# Registry
|
|
405
425
|
# ---------------------------------------------------------------------------
|
|
406
426
|
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
427
|
+
# Tashkeel deliberately EXCLUDES U+0653–0655 (madda above, hamza above, hamza
|
|
428
|
+
# below): in NFD text آ/أ/إ decompose to alef + those marks, which are letter
|
|
429
|
+
# identity — stripping them would corrupt the consonantal skeleton.
|
|
430
|
+
_TASHKEEL_RANGES = ((0x064B, 0x0652), (0x0656, 0x065F), (0x0670, 0x0670),
|
|
431
|
+
(0x08F0, 0x08F2))
|
|
432
|
+
_KASHIDA_RANGES = ((0x0640, 0x0640),)
|
|
433
|
+
# 06DD/06DE (end of ayah, rub el hizb) are structural signs and 06E5/06E6 are
|
|
434
|
+
# small letters — kept; 06E7-06E9 (small high marks, place of sajdah) and
|
|
435
|
+
# 06EA-06ED are annotation marks — stripped.
|
|
436
|
+
_QURANIC_RANGES = ((0x06D6, 0x06DC), (0x06DF, 0x06E4), (0x06E7, 0x06ED))
|
|
437
|
+
# Niqqud includes dagesh/mapiq U+05BC, meteg U+05BD (a masoretic stress mark,
|
|
438
|
+
# but traditionally stripped with the points), rafe U+05BF and the shin/sin
|
|
439
|
+
# dots U+05C1/05C2 — stripping loses the שׁ/שׂ distinction, exactly as
|
|
440
|
+
# unpointed Hebrew does. Maqaf U+05BE, sof pasuq U+05C3 and nun hafukha
|
|
441
|
+
# U+05C6 are punctuation and survive.
|
|
442
|
+
_NIQQUD_RANGES = ((0x05B0, 0x05BD), (0x05BF, 0x05BF), (0x05C1, 0x05C2),
|
|
443
|
+
(0x05C7, 0x05C7))
|
|
444
|
+
_TEAMIM_RANGES = ((0x0591, 0x05AF), (0x05C0, 0x05C0), (0x05C4, 0x05C5))
|
|
422
445
|
|
|
423
446
|
|
|
424
|
-
def
|
|
425
|
-
|
|
426
|
-
|
|
447
|
+
def _marks(id: str, name: str, scripts: Tuple[str, ...],
|
|
448
|
+
ranges: Tuple[Tuple[int, int], ...], reference: str,
|
|
449
|
+
*, default: str = "marked") -> Convention:
|
|
450
|
+
"""A binary decoration: strippable marks with a range-based detector."""
|
|
451
|
+
return Convention(id, name, scripts, None, ("marked", "none"), default,
|
|
452
|
+
ranges, reference,
|
|
453
|
+
transitions=(Transition("marked", "none", _range_strip(ranges)),),
|
|
454
|
+
detector=_range_detector(ranges))
|
|
427
455
|
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
+
|
|
457
|
+
CONVENTION_REGISTRY: Dict[str, Convention] = {c.id: c for c in (
|
|
458
|
+
_marks("tashkeel", "Arabic vocalization (harakat/tanwin/shadda/sukun)",
|
|
459
|
+
("Arab",), _TASHKEEL_RANGES,
|
|
460
|
+
"Unicode 15 ch. 9.2 (Arabic); excludes letter-identity U+0653-0655"),
|
|
461
|
+
_marks("kashida", "Arabic justification elongation (tatweel)",
|
|
462
|
+
("Arab",), _KASHIDA_RANGES, "Unicode U+0640 ARABIC TATWEEL",
|
|
463
|
+
default="none"),
|
|
464
|
+
_marks("quranic-marks", "Quranic annotation signs",
|
|
465
|
+
("Arab",), _QURANIC_RANGES,
|
|
466
|
+
"Unicode U+06D6-06ED Quranic annotation signs", default="none"),
|
|
467
|
+
_marks("niqqud", "Hebrew vocalization points",
|
|
468
|
+
("Hebr",), _NIQQUD_RANGES,
|
|
469
|
+
"Unicode 15 ch. 9.1 (Hebrew); strip loses the shin/sin dot distinction"),
|
|
470
|
+
_marks("teamim", "Hebrew cantillation marks",
|
|
471
|
+
("Hebr",), _TEAMIM_RANGES,
|
|
472
|
+
"Unicode U+0591-05AF Hebrew accents (masoretic)", default="none"),
|
|
473
|
+
Convention("wakachigaki", "Japanese spaced writing (分かち書き)",
|
|
474
|
+
("Hira", "Kana", "Hani"), "japanese", ("marked", "none"), "none",
|
|
475
|
+
(), "Spaced orthographic mode of Japanese (children's books, games)",
|
|
476
|
+
transitions=(
|
|
477
|
+
Transition("marked", "none", _wakachigaki_strip),
|
|
478
|
+
Transition("none", "marked", _wakachigaki_apply, requires="ja"),
|
|
479
|
+
)),
|
|
480
|
+
Convention("pinyin-tone", "Pinyin tone spelling",
|
|
481
|
+
("Latn",), "pinyin", ("mark", "number", "none"), "mark",
|
|
482
|
+
(), "GB/T 16159 Basic rules of Hanyu Pinyin orthography",
|
|
483
|
+
transitions=(
|
|
484
|
+
Transition("mark", "number", _pinyin_mark_to_number),
|
|
485
|
+
Transition("number", "mark", _pinyin_number_to_mark),
|
|
486
|
+
Transition("mark", "none", _pinyin_strip),
|
|
487
|
+
Transition("number", "none", _pinyin_strip),
|
|
488
|
+
)),
|
|
489
|
+
Convention("jamo-form", "Hangul jamo repertoire (compatibility vs conjoining)",
|
|
490
|
+
("Hang",), None, ("compatibility", "conjoining"), "compatibility",
|
|
491
|
+
(), "Unicode Hangul Compatibility Jamo U+3130 vs Hangul Jamo U+1100",
|
|
492
|
+
transitions=(
|
|
493
|
+
Transition("conjoining", "compatibility",
|
|
494
|
+
_jamo_conjoining_to_compat, lossless=True),
|
|
495
|
+
Transition("compatibility", "conjoining",
|
|
496
|
+
_jamo_compat_to_conjoining, lossless=True),
|
|
497
|
+
),
|
|
498
|
+
detector=_jamo_detect),
|
|
499
|
+
)}
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def conventions_for(script: str) -> List[Convention]:
|
|
503
|
+
"""Conventions that decorate *script* (ISO-15924 code), registry order."""
|
|
504
|
+
return [c for c in CONVENTION_REGISTRY.values() if script in c.scripts]
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _conv(convention_id: str) -> Convention:
|
|
508
|
+
try:
|
|
509
|
+
return CONVENTION_REGISTRY[convention_id]
|
|
510
|
+
except KeyError:
|
|
456
511
|
raise ValueError(
|
|
457
|
-
f"
|
|
458
|
-
f"
|
|
459
|
-
|
|
512
|
+
f"unknown convention {convention_id!r}; known: "
|
|
513
|
+
f"{', '.join(sorted(CONVENTION_REGISTRY))}") from None
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
# Module-level facades over the conventions' own behavior — the published API.
|
|
517
|
+
|
|
518
|
+
def restyle(text: str, convention_id: str, to: str,
|
|
519
|
+
frm: Optional[str] = None) -> str:
|
|
520
|
+
"""Rewrite *text* from one style of a convention to another."""
|
|
521
|
+
return _conv(convention_id).restyle(text, to, frm=frm)
|
|
460
522
|
|
|
461
523
|
|
|
462
524
|
def strip(text: str, convention_id: str) -> str:
|
|
463
525
|
"""Remove a convention's decoration (restyle to ``"none"``)."""
|
|
464
|
-
|
|
465
|
-
if "none" not in conv.styles:
|
|
466
|
-
raise ValueError(f"{convention_id!r} has no unmarked style; "
|
|
467
|
-
f"styles: {conv.styles}")
|
|
468
|
-
frm = "mark" if conv.id == "pinyin-tone" else "marked"
|
|
469
|
-
if conv.id == "pinyin-tone":
|
|
470
|
-
# digits and marks are both handled by the strip transition
|
|
471
|
-
return _pinyin_strip(text)
|
|
472
|
-
return restyle(text, convention_id, "none", frm=frm)
|
|
526
|
+
return _conv(convention_id).strip(text)
|
|
473
527
|
|
|
474
528
|
|
|
475
529
|
def apply(text: str, convention_id: str, style: Optional[str] = None) -> str:
|
|
476
530
|
"""Add a convention's decoration to unmarked text (restyle from ``"none"``)."""
|
|
477
|
-
|
|
478
|
-
if style is None:
|
|
479
|
-
marked = [s for s in conv.styles if s != "none"]
|
|
480
|
-
style = conv.default_style if conv.default_style != "none" else marked[0]
|
|
481
|
-
return restyle(text, convention_id, style, frm="none")
|
|
531
|
+
return _conv(convention_id).apply(text, style)
|
|
482
532
|
|
|
483
533
|
|
|
484
534
|
def detect(text: str, convention_id: str) -> Optional[str]:
|
|
485
|
-
"""Current style of *text* under a convention, or None when not character-evident.
|
|
486
|
-
|
|
487
|
-
For binary decorations this means "any mark present" — a single shadda in
|
|
488
|
-
a paragraph reports ``"marked"``.
|
|
489
|
-
"""
|
|
490
|
-
conv = _conv(convention_id)
|
|
491
|
-
if not conv.detectable:
|
|
492
|
-
return None
|
|
493
|
-
if conv.id == "jamo-form":
|
|
494
|
-
return _jamo_detect(text)
|
|
495
|
-
return "marked" if any(_in_ranges(ord(c), conv.ranges) for c in text) else "none"
|
|
535
|
+
"""Current style of *text* under a convention, or None when not character-evident."""
|
|
536
|
+
return _conv(convention_id).detect(text)
|