scriptconv 0.0.3a7__tar.gz → 0.0.3a9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/PKG-INFO +10 -1
  2. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/README.md +9 -0
  3. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/__init__.py +21 -0
  4. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/__main__.py +29 -1
  5. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/conventions.py +205 -164
  6. scriptconv-0.0.3a9/scriptconv/graph.py +265 -0
  7. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/notation.py +121 -64
  8. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/version.py +1 -1
  9. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv.egg-info/PKG-INFO +10 -1
  10. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv.egg-info/SOURCES.txt +3 -0
  11. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_cli.py +17 -0
  12. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_conventions.py +28 -0
  13. scriptconv-0.0.3a9/tests/test_errors_policy.py +102 -0
  14. scriptconv-0.0.3a9/tests/test_graph.py +132 -0
  15. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/LICENSE +0 -0
  16. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/pyproject.toml +0 -0
  17. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/requirements.txt +0 -0
  18. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/cangjie.py +0 -0
  19. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/data/__init__.py +0 -0
  20. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/data/cangjie5_tc.tsv.gz +0 -0
  21. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/py.typed +0 -0
  22. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/readings.py +0 -0
  23. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/scripts.py +0 -0
  24. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv/translit.py +0 -0
  25. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv.egg-info/dependency_links.txt +0 -0
  26. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv.egg-info/requires.txt +0 -0
  27. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/scriptconv.egg-info/top_level.txt +0 -0
  28. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/setup.cfg +0 -0
  29. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_cangjie.py +0 -0
  30. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_examples.py +0 -0
  31. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_notation.py +0 -0
  32. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_readings.py +0 -0
  33. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_readings_zh.py +0 -0
  34. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_scripts.py +0 -0
  35. {scriptconv-0.0.3a7 → scriptconv-0.0.3a9}/tests/test_translit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptconv
3
- Version: 0.0.3a7
3
+ Version: 0.0.3a9
4
4
  Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
5
5
  License: Apache-2.0
6
6
  Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -199,6 +199,15 @@ Transcoding faithfulness depends on the target notation's inventory. IPA is the
199
199
  notation↔notation goes through IPA. The table below states, for each notation, whether a
200
200
  round-trip is exact and what happens to a symbol the table does not know.
201
201
 
202
+ Every converter (and `convert()`) accepts a codecs-style `errors=` policy for
203
+ symbols outside its table: `"pass"` keeps the symbol (the default everywhere
204
+ except `ipa_to_arpa`), `"replace"` substitutes the notation's placeholder
205
+ (`?`; the historical `ipa_to_arpa` default, tunable via `unknown=`),
206
+ `"ignore"` drops it, and `"strict"` raises `UnknownSymbolError` naming the
207
+ symbol, its position, and the notation. The "Unknown-token behaviour" column
208
+ below describes the per-converter default.
209
+
210
+
202
211
  | Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
203
212
  |----------|----------------------------------|----------------------------------|-------------------------|
204
213
  | **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |
@@ -166,6 +166,15 @@ Transcoding faithfulness depends on the target notation's inventory. IPA is the
166
166
  notation↔notation goes through IPA. The table below states, for each notation, whether a
167
167
  round-trip is exact and what happens to a symbol the table does not know.
168
168
 
169
+ Every converter (and `convert()`) accepts a codecs-style `errors=` policy for
170
+ symbols outside its table: `"pass"` keeps the symbol (the default everywhere
171
+ except `ipa_to_arpa`), `"replace"` substitutes the notation's placeholder
172
+ (`?`; the historical `ipa_to_arpa` default, tunable via `unknown=`),
173
+ `"ignore"` drops it, and `"strict"` raises `UnknownSymbolError` naming the
174
+ symbol, its position, and the notation. The "Unknown-token behaviour" column
175
+ below describes the per-converter default.
176
+
177
+
169
178
  | Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
170
179
  |----------|----------------------------------|----------------------------------|-------------------------|
171
180
  | **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |
@@ -22,6 +22,10 @@ readings:
22
22
  extra; Chinese hanzi → pinyin/bopomofo via the ``zh`` extra).
23
23
  cangjie:
24
24
  Hanzi → Cangjie5 input codes (shape decomposition, vendored table).
25
+ graph:
26
+ Conversion-graph engine: representations (notations and orthographies)
27
+ as nodes, registered transforms as edges, lossless-preferring routing.
28
+ External packages extend graph instances explicitly.
25
29
  conventions:
26
30
  Orthographic conventions — script-scoped decorations orthogonal to
27
31
  identity (tashkeel, niqqud, wakachigaki, pinyin tone spelling…).
@@ -45,6 +49,7 @@ from scriptconv.scripts import (
45
49
  )
46
50
  from scriptconv.notation import (
47
51
  Notation,
52
+ UnknownSymbolError,
48
53
  NotationInfo,
49
54
  NOTATION_INFO,
50
55
  convert,
@@ -81,6 +86,7 @@ from scriptconv.readings import (
81
86
  )
82
87
  from scriptconv.conventions import (
83
88
  Convention,
89
+ Transition,
84
90
  CONVENTION_REGISTRY,
85
91
  conventions_for,
86
92
  restyle,
@@ -88,6 +94,13 @@ from scriptconv.conventions import (
88
94
  apply,
89
95
  detect as detect_convention,
90
96
  )
97
+ from scriptconv.graph import (
98
+ Representation,
99
+ Edge,
100
+ ConversionGraph,
101
+ DEFAULT_GRAPH,
102
+ REPRESENTATIONS,
103
+ )
91
104
  from scriptconv.cangjie import (
92
105
  cangjie_code,
93
106
  to_cangjie,
@@ -107,6 +120,7 @@ __all__ = [
107
120
  "normalize_script_tag",
108
121
  # notation
109
122
  "Notation",
123
+ "UnknownSymbolError",
110
124
  "NotationInfo",
111
125
  "NOTATION_INFO",
112
126
  "convert",
@@ -138,12 +152,19 @@ __all__ = [
138
152
  "to_katakana",
139
153
  # conventions (orthographic decorations, orthogonal to script identity)
140
154
  "Convention",
155
+ "Transition",
141
156
  "CONVENTION_REGISTRY",
142
157
  "conventions_for",
143
158
  "restyle",
144
159
  "strip",
145
160
  "apply",
146
161
  "detect_convention",
162
+ # graph (conversion routing engine over representations)
163
+ "Representation",
164
+ "Edge",
165
+ "ConversionGraph",
166
+ "DEFAULT_GRAPH",
167
+ "REPRESENTATIONS",
147
168
  # cangjie (vendored Cangjie5 table)
148
169
  "cangjie_code",
149
170
  "to_cangjie",
@@ -11,6 +11,7 @@ Usage
11
11
  python -m scriptconv strip <convention> <text>
12
12
  python -m scriptconv restyle <convention> <style> <text>
13
13
  python -m scriptconv conventions [--script CODE]
14
+ python -m scriptconv route <src> <dst>
14
15
 
15
16
  Examples
16
17
  --------
@@ -64,6 +65,10 @@ def main(argv: list[str] | None = None) -> int:
64
65
  cv = sub.add_parser("conventions", help="List orthographic conventions")
65
66
  cv.add_argument("--script", default=None, help="Filter by ISO-15924 script code")
66
67
 
68
+ rt = sub.add_parser("route", help="Show the conversion path between representations")
69
+ rt.add_argument("src", help="Source representation id")
70
+ rt.add_argument("dst", help="Target representation id")
71
+
67
72
  args = p.parse_args(argv)
68
73
 
69
74
  if args.command is None:
@@ -80,6 +85,7 @@ def main(argv: list[str] | None = None) -> int:
80
85
  "strip": lambda: _do_strip(args),
81
86
  "restyle": lambda: _do_restyle(args),
82
87
  "conventions": lambda: _do_conventions(args),
88
+ "route": lambda: _do_route(args),
83
89
  }
84
90
  dispatch[args.command]()
85
91
  return 0
@@ -89,9 +95,31 @@ def _do_convert(args: argparse.Namespace) -> None:
89
95
  from scriptconv.notation import Notation, convert
90
96
  try:
91
97
  print(convert(args.text, args.src, args.dst))
98
+ return
99
+ except ValueError:
100
+ pass
101
+ # not a notation pair — try any representation pair on the default graph
102
+ from scriptconv.graph import DEFAULT_GRAPH
103
+ try:
104
+ print(DEFAULT_GRAPH.convert(args.text, args.src, args.dst))
92
105
  except ValueError as e:
93
106
  valid = ", ".join(n.value for n in Notation)
94
- raise SystemExit(f"error: {e} (valid notations: {valid})")
107
+ raise SystemExit(f"error: {e} (valid notations: {valid}; "
108
+ f"graph nodes: {', '.join(DEFAULT_GRAPH.nodes)})")
109
+
110
+
111
+ def _do_route(args: argparse.Namespace) -> None:
112
+ from scriptconv.graph import DEFAULT_GRAPH
113
+ try:
114
+ path = DEFAULT_GRAPH.route(args.src, args.dst)
115
+ except ValueError as e:
116
+ raise SystemExit(f"error: {e}")
117
+ if not path:
118
+ print("(identity)")
119
+ for e in path:
120
+ flags = "lossless" if e.lossless else "lossy"
121
+ extra = f", needs scriptconv[{e.requires}]" if e.requires else ""
122
+ print(f" {e.src} -> {e.dst} ({flags}, cost {e.cost:g}{extra})")
95
123
 
96
124
 
97
125
  def _do_detect(args: argparse.Namespace) -> None:
@@ -43,6 +43,7 @@ repertoires and the restyle is deterministic.
43
43
  """
44
44
  from __future__ import annotations
45
45
 
46
+ import functools
46
47
  import re
47
48
  import unicodedata
48
49
  from dataclasses import dataclass
@@ -50,6 +51,7 @@ from typing import Callable, Dict, List, Optional, Tuple
50
51
 
51
52
  __all__ = [
52
53
  "Convention",
54
+ "Transition",
53
55
  "CONVENTION_REGISTRY",
54
56
  "conventions_for",
55
57
  "detect",
@@ -59,98 +61,118 @@ __all__ = [
59
61
  ]
60
62
 
61
63
 
64
+ @dataclass(frozen=True)
65
+ class Transition:
66
+ """One rewrite between two styles of a convention.
67
+
68
+ ``fn`` is a pure ``str -> str`` transform. ``requires`` names the
69
+ optional extra the transform needs, if any (queryable before calling —
70
+ the transform itself raises :class:`ImportError` with an install hint
71
+ when the dependency is missing).
72
+ """
73
+ frm: str
74
+ to: str
75
+ fn: Callable[[str], str]
76
+ lossless: bool = False
77
+ requires: Optional[str] = None
78
+
79
+
62
80
  @dataclass(frozen=True)
63
81
  class Convention:
64
- """Metadata for one orthographic convention (NotationInfo-style: data, not prose)."""
82
+ """One orthographic convention: metadata plus the behavior that owns it.
83
+
84
+ Transitions and the optional detector live on the convention itself, so
85
+ every operation is generic — nothing in this module branches on a
86
+ convention's identity.
87
+ """
65
88
  id: str
66
89
  name: str
67
90
  scripts: Tuple[str, ...] # ISO-15924 codes whose text it decorates
68
91
  system: Optional[str] # sub-script system qualifier ("pinyin", "japanese")
69
92
  styles: Tuple[str, ...] # mutually exclusive states; includes "none" if strippable
70
93
  default_style: str
71
- detectable: bool # current style is character-evident
72
- strip_lossless: bool # can the stripped information be recovered?
73
- apply_requires: Optional[str] # optional-extra name needed to leave "none", if any
74
94
  ranges: Tuple[Tuple[int, int], ...] # decoration codepoints (empty for spacing/variant conventions)
75
95
  reference: str
96
+ transitions: Tuple[Transition, ...] = ()
97
+ detector: Optional[Callable[[str], Optional[str]]] = None
98
+
99
+ @property
100
+ def detectable(self) -> bool:
101
+ return self.detector is not None
102
+
103
+ @property
104
+ def strip_lossless(self) -> bool:
105
+ stripping = [t for t in self.transitions if t.to == "none"]
106
+ return bool(stripping) and all(t.lossless for t in stripping)
107
+
108
+ @property
109
+ def apply_requires(self) -> Optional[str]:
110
+ return next((t.requires for t in self.transitions if t.frm == "none"), None)
111
+
112
+ def transition(self, frm: str, to: str) -> Optional[Transition]:
113
+ return next((t for t in self.transitions
114
+ if t.frm == frm and t.to == to), None)
115
+
116
+ def restyle(self, text: str, to: str, frm: Optional[str] = None) -> str:
117
+ """Rewrite *text* from one style of this convention to another."""
118
+ if to not in self.styles and to != "none":
119
+ raise ValueError(f"{self.id!r} has styles {self.styles}, not {to!r}")
120
+ if frm is None:
121
+ frm = self._infer_source(text, to)
122
+ if frm == to:
123
+ return text
124
+ t = self.transition(frm, to)
125
+ if t is None:
126
+ supported = sorted(f"{x.frm}->{x.to}" for x in self.transitions)
127
+ raise ValueError(f"{self.id!r} does not define {frm!r}->{to!r}; "
128
+ f"supported: {', '.join(supported)}")
129
+ return t.fn(text)
130
+
131
+ def _infer_source(self, text: str, to: str) -> str:
132
+ if self.detector is not None:
133
+ detected = self.detector(text)
134
+ if detected is not None and (detected == to
135
+ or self.transition(detected, to)):
136
+ return detected
137
+ sources = sorted({t.frm for t in self.transitions if t.to == to})
138
+ if len(sources) == 1:
139
+ return sources[0]
140
+ if self.default_style != to:
141
+ return self.default_style
142
+ raise ValueError(f"{self.id!r}: source style for ->{to!r} is ambiguous "
143
+ f"({', '.join(sources)}); pass frm=")
144
+
145
+ def strip(self, text: str) -> str:
146
+ """Remove this convention's decoration (restyle to ``"none"``)."""
147
+ stripping = [t for t in self.transitions if t.to == "none"]
148
+ if not stripping:
149
+ raise ValueError(f"{self.id!r} has no unmarked style; "
150
+ f"styles: {self.styles}")
151
+ # when every stripping transition is the same transform (e.g. one
152
+ # strip handles both pinyin tone spellings), the source is irrelevant
153
+ if len({t.fn for t in stripping}) == 1:
154
+ return stripping[0].fn(text)
155
+ return self.restyle(text, "none")
156
+
157
+ def apply(self, text: str, style: Optional[str] = None) -> str:
158
+ """Add this convention's decoration to unmarked text."""
159
+ if style is None:
160
+ marked = [s for s in self.styles if s != "none"]
161
+ style = self.default_style if self.default_style != "none" else marked[0]
162
+ return self.restyle(text, style, frm="none")
163
+
164
+ def detect(self, text: str) -> Optional[str]:
165
+ """Current style of *text*, or None when not character-evident.
166
+
167
+ For binary decorations this means "any mark present" — a single
168
+ shadda in a paragraph reports ``"marked"``.
169
+ """
170
+ return self.detector(text) if self.detector is not None else None
76
171
 
77
172
 
78
173
  # ---------------------------------------------------------------------------
79
- # Registry
174
+ # Shared helpers
80
175
  # ---------------------------------------------------------------------------
81
- # Tashkeel deliberately EXCLUDES U+0653–0655 (madda above, hamza above, hamza
82
- # below): in NFD text آ/أ/إ decompose to alef + those marks, which are letter
83
- # identity — stripping them would corrupt the consonantal skeleton.
84
- _TASHKEEL_RANGES = ((0x064B, 0x0652), (0x0656, 0x065F), (0x0670, 0x0670),
85
- (0x08F0, 0x08F2))
86
- _KASHIDA_RANGES = ((0x0640, 0x0640),)
87
- # 06DD/06DE (end of ayah, rub el hizb) are structural signs and 06E5/06E6 are
88
- # small letters — kept; 06E7-06E9 (small high marks, place of sajdah) and
89
- # 06EA-06ED are annotation marks — stripped.
90
- _QURANIC_RANGES = ((0x06D6, 0x06DC), (0x06DF, 0x06E4), (0x06E7, 0x06ED))
91
- # Niqqud includes dagesh/mapiq U+05BC, meteg U+05BD (a masoretic stress mark,
92
- # but traditionally stripped with the points), rafe U+05BF and the shin/sin
93
- # dots U+05C1/05C2 — stripping loses the שׁ/שׂ distinction, exactly as
94
- # unpointed Hebrew does. Maqaf U+05BE, sof pasuq U+05C3 and nun hafukha
95
- # U+05C6 are punctuation and survive.
96
- _NIQQUD_RANGES = ((0x05B0, 0x05BD), (0x05BF, 0x05BF), (0x05C1, 0x05C2),
97
- (0x05C7, 0x05C7))
98
- _TEAMIM_RANGES = ((0x0591, 0x05AF), (0x05C0, 0x05C0), (0x05C4, 0x05C5))
99
-
100
- CONVENTION_REGISTRY: Dict[str, Convention] = {c.id: c for c in (
101
- Convention("tashkeel", "Arabic vocalization (harakat/tanwin/shadda/sukun)",
102
- ("Arab",), None, ("marked", "none"), "marked",
103
- detectable=True, strip_lossless=False, apply_requires=None,
104
- ranges=_TASHKEEL_RANGES,
105
- reference="Unicode 15 ch. 9.2 (Arabic); excludes letter-identity U+0653-0655"),
106
- Convention("kashida", "Arabic justification elongation (tatweel)",
107
- ("Arab",), None, ("marked", "none"), "none",
108
- detectable=True, strip_lossless=False, apply_requires=None,
109
- ranges=_KASHIDA_RANGES, reference="Unicode U+0640 ARABIC TATWEEL"),
110
- Convention("quranic-marks", "Quranic annotation signs",
111
- ("Arab",), None, ("marked", "none"), "none",
112
- detectable=True, strip_lossless=False, apply_requires=None,
113
- ranges=_QURANIC_RANGES, reference="Unicode U+06D6-06ED Quranic annotation signs"),
114
- Convention("niqqud", "Hebrew vocalization points",
115
- ("Hebr",), None, ("marked", "none"), "marked",
116
- detectable=True, strip_lossless=False, apply_requires=None,
117
- ranges=_NIQQUD_RANGES,
118
- reference="Unicode 15 ch. 9.1 (Hebrew); strip loses the shin/sin dot distinction"),
119
- Convention("teamim", "Hebrew cantillation marks",
120
- ("Hebr",), None, ("marked", "none"), "none",
121
- detectable=True, strip_lossless=False, apply_requires=None,
122
- ranges=_TEAMIM_RANGES, reference="Unicode U+0591-05AF Hebrew accents (masoretic)"),
123
- Convention("wakachigaki", "Japanese spaced writing (分かち書き)",
124
- ("Hira", "Kana", "Hani"), "japanese", ("marked", "none"), "none",
125
- detectable=False, strip_lossless=False, apply_requires="ja",
126
- ranges=(),
127
- reference="Spaced orthographic mode of Japanese (children's books, games)"),
128
- Convention("pinyin-tone", "Pinyin tone spelling",
129
- ("Latn",), "pinyin", ("mark", "number", "none"), "mark",
130
- detectable=False, strip_lossless=False, apply_requires=None,
131
- ranges=(),
132
- reference="GB/T 16159 Basic rules of Hanyu Pinyin orthography"),
133
- Convention("jamo-form", "Hangul jamo repertoire (compatibility vs conjoining)",
134
- ("Hang",), None, ("compatibility", "conjoining"), "compatibility",
135
- detectable=True, strip_lossless=True, apply_requires=None,
136
- ranges=(),
137
- reference="Unicode Hangul Compatibility Jamo U+3130 vs Hangul Jamo U+1100"),
138
- )}
139
-
140
-
141
- def conventions_for(script: str) -> List[Convention]:
142
- """Conventions that decorate *script* (ISO-15924 code), registry order."""
143
- return [c for c in CONVENTION_REGISTRY.values() if script in c.scripts]
144
-
145
-
146
- def _conv(convention_id: str) -> Convention:
147
- try:
148
- return CONVENTION_REGISTRY[convention_id]
149
- except KeyError:
150
- raise ValueError(
151
- f"unknown convention {convention_id!r}; known: "
152
- f"{', '.join(sorted(CONVENTION_REGISTRY))}") from None
153
-
154
176
 
155
177
  def _in_ranges(cp: int, ranges: Tuple[Tuple[int, int], ...]) -> bool:
156
178
  return any(lo <= cp <= hi for lo, hi in ranges)
@@ -162,6 +184,12 @@ def _range_strip(ranges: Tuple[Tuple[int, int], ...]) -> Callable[[str], str]:
162
184
  return _strip
163
185
 
164
186
 
187
+ def _range_detector(ranges: Tuple[Tuple[int, int], ...]) -> Callable[[str], Optional[str]]:
188
+ def _detect(text: str) -> Optional[str]:
189
+ return "marked" if any(_in_ranges(ord(c), ranges) for c in text) else "none"
190
+ return _detect
191
+
192
+
165
193
  # ---------------------------------------------------------------------------
166
194
  # Wakachigaki
167
195
  # ---------------------------------------------------------------------------
@@ -338,6 +366,7 @@ def _pinyin_strip(text: str) -> str:
338
366
  # U+3131), so a consonant is treated as an onset when a vowel follows,
339
367
  # otherwise as a coda — deterministic over well-formed jamo sequences.
340
368
 
369
+ @functools.lru_cache(maxsize=1)
341
370
  def _jamo_tables():
342
371
  from scriptconv.translit import _ONSETS as _CO, _VOWELS as _CV, _CODAS as _CT
343
372
  conj_to_compat = {}
@@ -357,22 +386,13 @@ def _jamo_tables():
357
386
  return conj_to_compat, compat_onset, compat_vowel, compat_coda
358
387
 
359
388
 
360
- _JAMO_TABLES = None
361
-
362
-
363
389
  def _jamo_conjoining_to_compat(text: str) -> str:
364
- global _JAMO_TABLES
365
- if _JAMO_TABLES is None:
366
- _JAMO_TABLES = _jamo_tables()
367
- table = _JAMO_TABLES[0]
390
+ table = _jamo_tables()[0]
368
391
  return "".join(table.get(c, c) for c in text)
369
392
 
370
393
 
371
394
  def _jamo_compat_to_conjoining(text: str) -> str:
372
- global _JAMO_TABLES
373
- if _JAMO_TABLES is None:
374
- _JAMO_TABLES = _jamo_tables()
375
- _, onset, vowel, coda = _JAMO_TABLES
395
+ _, onset, vowel, coda = _jamo_tables()
376
396
  out = []
377
397
  for i, c in enumerate(text):
378
398
  if c in vowel:
@@ -401,95 +421,116 @@ def _jamo_detect(text: str) -> Optional[str]:
401
421
 
402
422
 
403
423
  # ---------------------------------------------------------------------------
404
- # Transition matrix and operations
424
+ # Registry
405
425
  # ---------------------------------------------------------------------------
406
426
 
407
- _TRANSITIONS: Dict[Tuple[str, str, str], Callable[[str], str]] = {
408
- ("tashkeel", "marked", "none"): _range_strip(_TASHKEEL_RANGES),
409
- ("kashida", "marked", "none"): _range_strip(_KASHIDA_RANGES),
410
- ("quranic-marks", "marked", "none"): _range_strip(_QURANIC_RANGES),
411
- ("niqqud", "marked", "none"): _range_strip(_NIQQUD_RANGES),
412
- ("teamim", "marked", "none"): _range_strip(_TEAMIM_RANGES),
413
- ("wakachigaki", "marked", "none"): _wakachigaki_strip,
414
- ("wakachigaki", "none", "marked"): _wakachigaki_apply,
415
- ("pinyin-tone", "mark", "number"): _pinyin_mark_to_number,
416
- ("pinyin-tone", "number", "mark"): _pinyin_number_to_mark,
417
- ("pinyin-tone", "mark", "none"): _pinyin_strip,
418
- ("pinyin-tone", "number", "none"): _pinyin_strip,
419
- ("jamo-form", "conjoining", "compatibility"): _jamo_conjoining_to_compat,
420
- ("jamo-form", "compatibility", "conjoining"): _jamo_compat_to_conjoining,
421
- }
427
+ # Tashkeel deliberately EXCLUDES U+0653–0655 (madda above, hamza above, hamza
428
+ # below): in NFD text آ/أ/إ decompose to alef + those marks, which are letter
429
+ # identity stripping them would corrupt the consonantal skeleton.
430
+ _TASHKEEL_RANGES = ((0x064B, 0x0652), (0x0656, 0x065F), (0x0670, 0x0670),
431
+ (0x08F0, 0x08F2))
432
+ _KASHIDA_RANGES = ((0x0640, 0x0640),)
433
+ # 06DD/06DE (end of ayah, rub el hizb) are structural signs and 06E5/06E6 are
434
+ # small letters — kept; 06E7-06E9 (small high marks, place of sajdah) and
435
+ # 06EA-06ED are annotation marks — stripped.
436
+ _QURANIC_RANGES = ((0x06D6, 0x06DC), (0x06DF, 0x06E4), (0x06E7, 0x06ED))
437
+ # Niqqud includes dagesh/mapiq U+05BC, meteg U+05BD (a masoretic stress mark,
438
+ # but traditionally stripped with the points), rafe U+05BF and the shin/sin
439
+ # dots U+05C1/05C2 — stripping loses the שׁ/שׂ distinction, exactly as
440
+ # unpointed Hebrew does. Maqaf U+05BE, sof pasuq U+05C3 and nun hafukha
441
+ # U+05C6 are punctuation and survive.
442
+ _NIQQUD_RANGES = ((0x05B0, 0x05BD), (0x05BF, 0x05BF), (0x05C1, 0x05C2),
443
+ (0x05C7, 0x05C7))
444
+ _TEAMIM_RANGES = ((0x0591, 0x05AF), (0x05C0, 0x05C0), (0x05C4, 0x05C5))
422
445
 
423
446
 
424
- def restyle(text: str, convention_id: str, to: str,
425
- frm: Optional[str] = None) -> str:
426
- """Rewrite *text* from one style of a convention to another.
447
+ def _marks(id: str, name: str, scripts: Tuple[str, ...],
448
+ ranges: Tuple[Tuple[int, int], ...], reference: str,
449
+ *, default: str = "marked") -> Convention:
450
+ """A binary decoration: strippable marks with a range-based detector."""
451
+ return Convention(id, name, scripts, None, ("marked", "none"), default,
452
+ ranges, reference,
453
+ transitions=(Transition("marked", "none", _range_strip(ranges)),),
454
+ detector=_range_detector(ranges))
427
455
 
428
- ``frm`` defaults to the convention's marked/default style for strips and
429
- is required where the source style is not inferable. Unsupported
430
- transitions raise :class:`ValueError`; transitions that need a missing
431
- optional dependency raise :class:`ImportError` with an install hint.
432
- """
433
- conv = _conv(convention_id)
434
- if to not in conv.styles and to != "none":
435
- raise ValueError(f"{convention_id!r} has styles {conv.styles}, not {to!r}")
436
- if frm is None:
437
- if conv.id == "jamo-form":
438
- frm = _jamo_detect(text) or conv.default_style
439
- else:
440
- sources = sorted({f for (c, f, t) in _TRANSITIONS
441
- if c == convention_id and t == to})
442
- if len(sources) == 1:
443
- frm = sources[0]
444
- elif conv.default_style != to:
445
- frm = conv.default_style
446
- else:
447
- raise ValueError(
448
- f"{convention_id!r}: source style for ->{to!r} is ambiguous "
449
- f"({', '.join(sources)}); pass frm=")
450
- if frm == to:
451
- return text
452
- fn = _TRANSITIONS.get((convention_id, frm, to))
453
- if fn is None:
454
- supported = sorted(f"{f}->{t}" for (c, f, t) in _TRANSITIONS
455
- if c == convention_id)
456
+
457
+ CONVENTION_REGISTRY: Dict[str, Convention] = {c.id: c for c in (
458
+ _marks("tashkeel", "Arabic vocalization (harakat/tanwin/shadda/sukun)",
459
+ ("Arab",), _TASHKEEL_RANGES,
460
+ "Unicode 15 ch. 9.2 (Arabic); excludes letter-identity U+0653-0655"),
461
+ _marks("kashida", "Arabic justification elongation (tatweel)",
462
+ ("Arab",), _KASHIDA_RANGES, "Unicode U+0640 ARABIC TATWEEL",
463
+ default="none"),
464
+ _marks("quranic-marks", "Quranic annotation signs",
465
+ ("Arab",), _QURANIC_RANGES,
466
+ "Unicode U+06D6-06ED Quranic annotation signs", default="none"),
467
+ _marks("niqqud", "Hebrew vocalization points",
468
+ ("Hebr",), _NIQQUD_RANGES,
469
+ "Unicode 15 ch. 9.1 (Hebrew); strip loses the shin/sin dot distinction"),
470
+ _marks("teamim", "Hebrew cantillation marks",
471
+ ("Hebr",), _TEAMIM_RANGES,
472
+ "Unicode U+0591-05AF Hebrew accents (masoretic)", default="none"),
473
+ Convention("wakachigaki", "Japanese spaced writing (分かち書き)",
474
+ ("Hira", "Kana", "Hani"), "japanese", ("marked", "none"), "none",
475
+ (), "Spaced orthographic mode of Japanese (children's books, games)",
476
+ transitions=(
477
+ Transition("marked", "none", _wakachigaki_strip),
478
+ Transition("none", "marked", _wakachigaki_apply, requires="ja"),
479
+ )),
480
+ Convention("pinyin-tone", "Pinyin tone spelling",
481
+ ("Latn",), "pinyin", ("mark", "number", "none"), "mark",
482
+ (), "GB/T 16159 Basic rules of Hanyu Pinyin orthography",
483
+ transitions=(
484
+ Transition("mark", "number", _pinyin_mark_to_number),
485
+ Transition("number", "mark", _pinyin_number_to_mark),
486
+ Transition("mark", "none", _pinyin_strip),
487
+ Transition("number", "none", _pinyin_strip),
488
+ )),
489
+ Convention("jamo-form", "Hangul jamo repertoire (compatibility vs conjoining)",
490
+ ("Hang",), None, ("compatibility", "conjoining"), "compatibility",
491
+ (), "Unicode Hangul Compatibility Jamo U+3130 vs Hangul Jamo U+1100",
492
+ transitions=(
493
+ Transition("conjoining", "compatibility",
494
+ _jamo_conjoining_to_compat, lossless=True),
495
+ Transition("compatibility", "conjoining",
496
+ _jamo_compat_to_conjoining, lossless=True),
497
+ ),
498
+ detector=_jamo_detect),
499
+ )}
500
+
501
+
502
+ def conventions_for(script: str) -> List[Convention]:
503
+ """Conventions that decorate *script* (ISO-15924 code), registry order."""
504
+ return [c for c in CONVENTION_REGISTRY.values() if script in c.scripts]
505
+
506
+
507
+ def _conv(convention_id: str) -> Convention:
508
+ try:
509
+ return CONVENTION_REGISTRY[convention_id]
510
+ except KeyError:
456
511
  raise ValueError(
457
- f"{convention_id!r} does not define {frm!r}->{to!r}; "
458
- f"supported: {', '.join(supported)}")
459
- return fn(text)
512
+ f"unknown convention {convention_id!r}; known: "
513
+ f"{', '.join(sorted(CONVENTION_REGISTRY))}") from None
514
+
515
+
516
+ # Module-level facades over the conventions' own behavior — the published API.
517
+
518
+ def restyle(text: str, convention_id: str, to: str,
519
+ frm: Optional[str] = None) -> str:
520
+ """Rewrite *text* from one style of a convention to another."""
521
+ return _conv(convention_id).restyle(text, to, frm=frm)
460
522
 
461
523
 
462
524
  def strip(text: str, convention_id: str) -> str:
463
525
  """Remove a convention's decoration (restyle to ``"none"``)."""
464
- conv = _conv(convention_id)
465
- if "none" not in conv.styles:
466
- raise ValueError(f"{convention_id!r} has no unmarked style; "
467
- f"styles: {conv.styles}")
468
- frm = "mark" if conv.id == "pinyin-tone" else "marked"
469
- if conv.id == "pinyin-tone":
470
- # digits and marks are both handled by the strip transition
471
- return _pinyin_strip(text)
472
- return restyle(text, convention_id, "none", frm=frm)
526
+ return _conv(convention_id).strip(text)
473
527
 
474
528
 
475
529
  def apply(text: str, convention_id: str, style: Optional[str] = None) -> str:
476
530
  """Add a convention's decoration to unmarked text (restyle from ``"none"``)."""
477
- conv = _conv(convention_id)
478
- if style is None:
479
- marked = [s for s in conv.styles if s != "none"]
480
- style = conv.default_style if conv.default_style != "none" else marked[0]
481
- return restyle(text, convention_id, style, frm="none")
531
+ return _conv(convention_id).apply(text, style)
482
532
 
483
533
 
484
534
  def detect(text: str, convention_id: str) -> Optional[str]:
485
- """Current style of *text* under a convention, or None when not character-evident.
486
-
487
- For binary decorations this means "any mark present" — a single shadda in
488
- a paragraph reports ``"marked"``.
489
- """
490
- conv = _conv(convention_id)
491
- if not conv.detectable:
492
- return None
493
- if conv.id == "jamo-form":
494
- return _jamo_detect(text)
495
- return "marked" if any(_in_ranges(ord(c), conv.ranges) for c in text) else "none"
535
+ """Current style of *text* under a convention, or None when not character-evident."""
536
+ return _conv(convention_id).detect(text)