scriptconv 0.0.3a8__tar.gz → 0.0.3a9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/PKG-INFO +10 -1
  2. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/README.md +9 -0
  3. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/__init__.py +2 -0
  4. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/graph.py +10 -4
  5. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/notation.py +103 -37
  6. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/version.py +1 -1
  7. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv.egg-info/PKG-INFO +10 -1
  8. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv.egg-info/SOURCES.txt +1 -0
  9. scriptconv-0.0.3a9/tests/test_errors_policy.py +102 -0
  10. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/LICENSE +0 -0
  11. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/pyproject.toml +0 -0
  12. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/requirements.txt +0 -0
  13. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/__main__.py +0 -0
  14. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/cangjie.py +0 -0
  15. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/conventions.py +0 -0
  16. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/data/__init__.py +0 -0
  17. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/data/cangjie5_tc.tsv.gz +0 -0
  18. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/py.typed +0 -0
  19. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/readings.py +0 -0
  20. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/scripts.py +0 -0
  21. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv/translit.py +0 -0
  22. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv.egg-info/dependency_links.txt +0 -0
  23. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv.egg-info/requires.txt +0 -0
  24. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/scriptconv.egg-info/top_level.txt +0 -0
  25. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/setup.cfg +0 -0
  26. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/tests/test_cangjie.py +0 -0
  27. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/tests/test_cli.py +0 -0
  28. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/tests/test_conventions.py +0 -0
  29. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/tests/test_examples.py +0 -0
  30. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/tests/test_graph.py +0 -0
  31. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/tests/test_notation.py +0 -0
  32. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/tests/test_readings.py +0 -0
  33. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/tests/test_readings_zh.py +0 -0
  34. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/tests/test_scripts.py +0 -0
  35. {scriptconv-0.0.3a8 → scriptconv-0.0.3a9}/tests/test_translit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptconv
3
- Version: 0.0.3a8
3
+ Version: 0.0.3a9
4
4
  Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
5
5
  License: Apache-2.0
6
6
  Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -199,6 +199,15 @@ Transcoding faithfulness depends on the target notation's inventory. IPA is the
199
199
  notation↔notation goes through IPA. The table below states, for each notation, whether a
200
200
  round-trip is exact and what happens to a symbol the table does not know.
201
201
 
202
+ Every converter (and `convert()`) accepts a codecs-style `errors=` policy for
203
+ symbols outside its table: `"pass"` keeps the symbol (the default everywhere
204
+ except `ipa_to_arpa`), `"replace"` substitutes the notation's placeholder
205
+ (`?`; the historical `ipa_to_arpa` default, tunable via `unknown=`),
206
+ `"ignore"` drops it, and `"strict"` raises `UnknownSymbolError` naming the
207
+ symbol, its position, and the notation. The "Unknown-token behaviour" column
208
+ below describes the per-converter default.
209
+
210
+
202
211
  | Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
203
212
  |----------|----------------------------------|----------------------------------|-------------------------|
204
213
  | **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |
@@ -166,6 +166,15 @@ Transcoding faithfulness depends on the target notation's inventory. IPA is the
166
166
  notation↔notation goes through IPA. The table below states, for each notation, whether a
167
167
  round-trip is exact and what happens to a symbol the table does not know.
168
168
 
169
+ Every converter (and `convert()`) accepts a codecs-style `errors=` policy for
170
+ symbols outside its table: `"pass"` keeps the symbol (the default everywhere
171
+ except `ipa_to_arpa`), `"replace"` substitutes the notation's placeholder
172
+ (`?`; the historical `ipa_to_arpa` default, tunable via `unknown=`),
173
+ `"ignore"` drops it, and `"strict"` raises `UnknownSymbolError` naming the
174
+ symbol, its position, and the notation. The "Unknown-token behaviour" column
175
+ below describes the per-converter default.
176
+
177
+
169
178
  | Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
170
179
  |----------|----------------------------------|----------------------------------|-------------------------|
171
180
  | **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |
@@ -49,6 +49,7 @@ from scriptconv.scripts import (
49
49
  )
50
50
  from scriptconv.notation import (
51
51
  Notation,
52
+ UnknownSymbolError,
52
53
  NotationInfo,
53
54
  NOTATION_INFO,
54
55
  convert,
@@ -119,6 +120,7 @@ __all__ = [
119
120
  "normalize_script_tag",
120
121
  # notation
121
122
  "Notation",
123
+ "UnknownSymbolError",
122
124
  "NotationInfo",
123
125
  "NOTATION_INFO",
124
126
  "convert",
@@ -197,17 +197,23 @@ def _notation_edges() -> List[Edge]:
197
197
  from scriptconv.notation import (_TO_IPA, _FROM_IPA, _DIRECT_PAIRS,
198
198
  NOTATION_INFO)
199
199
  edges = []
200
+ def _wrap(fn):
201
+ # forward only the errors= policy from routing context; other context
202
+ # keys (lang, engine options) are not the notation layer's business
203
+ def _edge(text, errors=None, **_):
204
+ return fn(text, errors=errors) if errors is not None else fn(text)
205
+ return _edge
206
+
200
207
  for notation, fn in _TO_IPA.items():
201
208
  info = NOTATION_INFO[notation]
202
- edges.append(Edge(notation.value, "ipa", (lambda f: lambda t, **_: f(t))(fn),
209
+ edges.append(Edge(notation.value, "ipa", _wrap(fn),
203
210
  lossless=info.lossless_to_ipa))
204
211
  for notation, fn in _FROM_IPA.items():
205
212
  info = NOTATION_INFO[notation]
206
- edges.append(Edge("ipa", notation.value, (lambda f: lambda t, **_: f(t))(fn),
213
+ edges.append(Edge("ipa", notation.value, _wrap(fn),
207
214
  lossless=info.lossless_from_ipa))
208
215
  for (src, dst), fn in _DIRECT_PAIRS.items():
209
- edges.append(Edge(src.value, dst.value, (lambda f: lambda t, **_: f(t))(fn),
210
- lossless=True))
216
+ edges.append(Edge(src.value, dst.value, _wrap(fn), lossless=True))
211
217
  return edges
212
218
 
213
219
 
@@ -158,10 +158,41 @@ _IPA_ARPA_KEYS_SORTED = sorted(_IPA_TO_ARPA.keys(), key=len, reverse=True)
158
158
  _IPA_ARPA_RE = re.compile("|".join(re.escape(s) for s in _IPA_ARPA_KEYS_SORTED))
159
159
 
160
160
 
161
- def arpa_to_ipa(arpa_sequence: str) -> str:
161
+
162
+ class UnknownSymbolError(ValueError):
163
+ """A symbol has no mapping in the requested notation (``errors="strict"``)."""
164
+
165
+ def __init__(self, symbol: str, position: int, notation: str):
166
+ self.symbol = symbol
167
+ self.position = position
168
+ self.notation = notation
169
+ super().__init__(
170
+ f"no {notation} mapping for {symbol!r} at position {position}")
171
+
172
+
173
+ _ERROR_POLICIES = ("pass", "replace", "strict", "ignore")
174
+
175
+
176
+ def _unknown(symbol: str, position: int, notation: str, errors: str,
177
+ replacement: str = "?") -> str:
178
+ """Resolve one unmapped symbol according to the codecs-style policy."""
179
+ if errors == "pass":
180
+ return symbol
181
+ if errors == "replace":
182
+ return replacement
183
+ if errors == "ignore":
184
+ return ""
185
+ if errors == "strict":
186
+ raise UnknownSymbolError(symbol, position, notation)
187
+ raise ValueError(f"errors must be one of {_ERROR_POLICIES}, not {errors!r}")
188
+
189
+
190
+ def arpa_to_ipa(arpa_sequence: str, errors: str = "pass") -> str:
162
191
  """Convert a space-separated ARPABET sequence to an IPA string.
163
192
 
164
- Stress digits are stripped. Unknown tokens are passed through.
193
+ Stress digits are stripped. ``errors`` selects the unknown-token policy:
194
+ ``"pass"`` (default) keeps the token, ``"replace"`` substitutes ``?``,
195
+ ``"ignore"`` drops it, ``"strict"`` raises :class:`UnknownSymbolError`.
165
196
 
166
197
  Examples
167
198
  --------
@@ -170,22 +201,27 @@ def arpa_to_ipa(arpa_sequence: str) -> str:
170
201
  """
171
202
  tokens = arpa_sequence.strip().split()
172
203
  result = []
173
- for tok in tokens:
204
+ for i, tok in enumerate(tokens):
174
205
  ipa = _ARPA_TO_IPA.get(tok)
175
206
  if ipa is None:
176
207
  # strip trailing digit and retry
177
208
  stripped = tok.rstrip("012")
178
- ipa = _ARPA_TO_IPA.get(stripped, tok)
209
+ ipa = _ARPA_TO_IPA.get(stripped)
210
+ if ipa is None:
211
+ ipa = _unknown(tok, i, "arpa", errors)
179
212
  result.append(ipa)
180
213
  return "".join(result)
181
214
 
182
215
 
183
- def ipa_to_arpa(ipa_string: str, unknown: str = "?") -> str:
216
+ def ipa_to_arpa(ipa_string: str, unknown: str = "?",
217
+ errors: str = "replace") -> str:
184
218
  """Convert an IPA string to a space-separated ARPABET sequence.
185
219
 
186
- Matches longest IPA symbol first. Characters outside the ARPABET table
187
- are replaced by *unknown* (default ``"?"``); pass ``unknown=""`` to drop
188
- them silently.
220
+ Matches longest IPA symbol first. ``errors`` selects the unknown-symbol
221
+ policy (default ``"replace"``, preserving the historical behavior of
222
+ substituting *unknown*, ``"?"``); ``unknown=""`` still drops silently.
223
+ Combining marks and modifier letters qualify the preceding phoneme and
224
+ are always dropped rather than treated as unknowns.
189
225
 
190
226
  Examples
191
227
  --------
@@ -207,8 +243,11 @@ def ipa_to_arpa(ipa_string: str, unknown: str = "?") -> str:
207
243
  # modifier letters) have no ARPABET equivalent; they qualify the
208
244
  # preceding phoneme rather than standing alone, so drop them
209
245
  # instead of emitting a spurious *unknown* token.
210
- if unknown and unicodedata.category(ch) not in ("Mn", "Mc", "Me", "Lm", "Sk"):
211
- tokens.append(unknown)
246
+ if unicodedata.category(ch) not in ("Mn", "Mc", "Me", "Lm", "Sk"):
247
+ effective = "ignore" if (errors == "replace" and not unknown) else errors
248
+ out = _unknown(ch, pos, "arpa", effective, replacement=unknown)
249
+ if out:
250
+ tokens.append(out)
212
251
  pos += 1
213
252
  return " ".join(tokens)
214
253
 
@@ -338,7 +377,7 @@ _XS_RE = re.compile("|".join(re.escape(k) for k in _XS_KEYS_SORTED))
338
377
  _IPA_XS_RE = re.compile("|".join(re.escape(k) for k in _IPA_KEYS_SORTED))
339
378
 
340
379
 
341
- def xsampa_to_ipa(xsampa: str) -> str:
380
+ def xsampa_to_ipa(xsampa: str, errors: str = "pass") -> str:
342
381
  """Convert an X-SAMPA string to IPA.
343
382
 
344
383
  Multi-character symbols are matched longest-first. Unknown characters
@@ -359,12 +398,14 @@ def xsampa_to_ipa(xsampa: str) -> str:
359
398
  result.append(_XSAMPA_TO_IPA[m.group(0)])
360
399
  pos = m.end()
361
400
  else:
362
- result.append(xsampa[pos])
401
+ out = _unknown(xsampa[pos], pos, "x-sampa", errors)
402
+ if out:
403
+ result.append(out)
363
404
  pos += 1
364
405
  return "".join(result)
365
406
 
366
407
 
367
- def ipa_to_xsampa(ipa: str) -> str:
408
+ def ipa_to_xsampa(ipa: str, errors: str = "pass") -> str:
368
409
  """Convert an IPA string to X-SAMPA.
369
410
 
370
411
  Multi-character IPA symbols are matched longest-first. Unknown
@@ -385,7 +426,9 @@ def ipa_to_xsampa(ipa: str) -> str:
385
426
  result.append(_IPA_TO_XSAMPA[m.group(0)])
386
427
  pos = m.end()
387
428
  else:
388
- result.append(ipa[pos])
429
+ out = _unknown(ipa[pos], pos, "x-sampa", errors)
430
+ if out:
431
+ result.append(out)
389
432
  pos += 1
390
433
  return "".join(result)
391
434
 
@@ -461,7 +504,7 @@ _ARABIC_TO_BW["\uFEF8"] = "l|" # لآ lam + alef madda
461
504
  _ARABIC_TO_BW["\u0651"] = "~" # shadda -> canonical "~", not the "^" alias
462
505
 
463
506
 
464
- def buckwalter_to_arabic(bw: str) -> str:
507
+ def buckwalter_to_arabic(bw: str, errors: str = "pass") -> str:
465
508
  """Convert a Buckwalter-encoded string to Arabic Unicode.
466
509
 
467
510
  Unknown characters are passed through unchanged.
@@ -471,10 +514,12 @@ def buckwalter_to_arabic(bw: str) -> str:
471
514
  >>> buckwalter_to_arabic("mrHbA")
472
515
  'مرحبا'
473
516
  """
474
- return "".join(_BW_TO_ARABIC.get(ch, ch) for ch in bw)
517
+ return "".join(_BW_TO_ARABIC[ch] if ch in _BW_TO_ARABIC
518
+ else _unknown(ch, i, "buckwalter", errors)
519
+ for i, ch in enumerate(bw))
475
520
 
476
521
 
477
- def arabic_to_buckwalter(arabic: str) -> str:
522
+ def arabic_to_buckwalter(arabic: str, errors: str = "pass") -> str:
478
523
  """Convert an Arabic Unicode string to Buckwalter transliteration.
479
524
 
480
525
  Unknown characters are passed through unchanged.
@@ -484,7 +529,9 @@ def arabic_to_buckwalter(arabic: str) -> str:
484
529
  >>> arabic_to_buckwalter("مرحبا")
485
530
  'mrHbA'
486
531
  """
487
- return "".join(_ARABIC_TO_BW.get(ch, ch) for ch in arabic)
532
+ return "".join(_ARABIC_TO_BW[ch] if ch in _ARABIC_TO_BW
533
+ else _unknown(ch, i, "arabic", errors)
534
+ for i, ch in enumerate(arabic))
488
535
 
489
536
 
490
537
  # ---------------------------------------------------------------------------
@@ -560,7 +607,7 @@ _IPA_FOR_LX_SORTED = sorted(_IPA_TO_LEXIQUE.keys(), key=len, reverse=True)
560
607
  _IPA_LX_RE = re.compile("|".join(re.escape(s) for s in _IPA_FOR_LX_SORTED))
561
608
 
562
609
 
563
- def lexique_to_ipa(lexique: str) -> str:
610
+ def lexique_to_ipa(lexique: str, errors: str = "pass") -> str:
564
611
  """Convert a Lexique phoneme-code string to IPA.
565
612
 
566
613
  Each Lexique phoneme is exactly one character; they are read left-to-right
@@ -576,12 +623,14 @@ def lexique_to_ipa(lexique: str) -> str:
576
623
  'vɛ̃'
577
624
  """
578
625
  result = []
579
- for ch in lexique:
580
- result.append(_LEXIQUE_TO_IPA.get(ch, ch))
626
+ for i, ch in enumerate(lexique):
627
+ mapped = _LEXIQUE_TO_IPA.get(ch)
628
+ result.append(mapped if mapped is not None
629
+ else _unknown(ch, i, "lexique", errors))
581
630
  return "".join(result)
582
631
 
583
632
 
584
- def ipa_to_lexique(ipa: str) -> str:
633
+ def ipa_to_lexique(ipa: str, errors: str = "pass") -> str:
585
634
  """Convert an IPA string to Lexique phoneme codes.
586
635
 
587
636
  Matches longest IPA symbol first. Unknown characters are passed through
@@ -605,7 +654,9 @@ def ipa_to_lexique(ipa: str) -> str:
605
654
  result.append(_IPA_TO_LEXIQUE[m.group(0)])
606
655
  pos = m.end()
607
656
  else:
608
- result.append(ipa[pos])
657
+ out = _unknown(ipa[pos], pos, "lexique", errors)
658
+ if out:
659
+ result.append(out)
609
660
  pos += 1
610
661
  return "".join(result)
611
662
 
@@ -640,7 +691,7 @@ for _k, _ip in _KIRSHENBAUM_TO_IPA.items():
640
691
  _IPA_TO_KIRSHENBAUM.setdefault(_ip, _k)
641
692
 
642
693
 
643
- def kirshenbaum_to_ipa(kirshenbaum: str) -> str:
694
+ def kirshenbaum_to_ipa(kirshenbaum: str, errors: str = "pass") -> str:
644
695
  """Convert a Kirshenbaum (ASCII-IPA) string to IPA.
645
696
 
646
697
  Each character is mapped independently. Characters outside the table
@@ -653,10 +704,12 @@ def kirshenbaum_to_ipa(kirshenbaum: str) -> str:
653
704
  >>> kirshenbaum_to_ipa("N")
654
705
  'ŋ'
655
706
  """
656
- return "".join(_KIRSHENBAUM_TO_IPA.get(ch, ch) for ch in kirshenbaum)
707
+ return "".join(_KIRSHENBAUM_TO_IPA[ch] if ch in _KIRSHENBAUM_TO_IPA
708
+ else _unknown(ch, i, "kirshenbaum", errors)
709
+ for i, ch in enumerate(kirshenbaum))
657
710
 
658
711
 
659
- def ipa_to_kirshenbaum(ipa: str) -> str:
712
+ def ipa_to_kirshenbaum(ipa: str, errors: str = "pass") -> str:
660
713
  """Convert an IPA string to Kirshenbaum (ASCII-IPA).
661
714
 
662
715
  Characters outside the table pass through unchanged.
@@ -668,7 +721,9 @@ def ipa_to_kirshenbaum(ipa: str) -> str:
668
721
  >>> ipa_to_kirshenbaum("ŋ")
669
722
  'N'
670
723
  """
671
- return "".join(_IPA_TO_KIRSHENBAUM.get(ch, ch) for ch in ipa)
724
+ return "".join(_IPA_TO_KIRSHENBAUM[ch] if ch in _IPA_TO_KIRSHENBAUM
725
+ else _unknown(ch, i, "kirshenbaum", errors)
726
+ for i, ch in enumerate(ipa))
672
727
 
673
728
 
674
729
  # ---------------------------------------------------------------------------
@@ -737,7 +792,7 @@ _IPA_CV_KEYS_SORTED = sorted(_IPA_TO_COTOVIA.keys(), key=len, reverse=True)
737
792
  _IPA_CV_RE = re.compile("|".join(re.escape(s) for s in _IPA_CV_KEYS_SORTED))
738
793
 
739
794
 
740
- def cotovia_to_ipa(cotovia: str) -> str:
795
+ def cotovia_to_ipa(cotovia: str, errors: str = "pass") -> str:
741
796
  """Convert a Cotovía phoneme string to IPA.
742
797
 
743
798
  Multi-character symbols are matched longest-first. Characters outside the
@@ -758,12 +813,14 @@ def cotovia_to_ipa(cotovia: str) -> str:
758
813
  result.append(_COTOVIA_TO_IPA[m.group(0)])
759
814
  pos = m.end()
760
815
  else:
761
- result.append(cotovia[pos])
816
+ out = _unknown(cotovia[pos], pos, "cotovia", errors)
817
+ if out:
818
+ result.append(out)
762
819
  pos += 1
763
820
  return "".join(result)
764
821
 
765
822
 
766
- def ipa_to_cotovia(ipa: str) -> str:
823
+ def ipa_to_cotovia(ipa: str, errors: str = "pass") -> str:
767
824
  """Convert an IPA string to Cotovía notation.
768
825
 
769
826
  Multi-character IPA symbols are matched longest-first. Characters outside
@@ -784,7 +841,9 @@ def ipa_to_cotovia(ipa: str) -> str:
784
841
  result.append(_IPA_TO_COTOVIA[m.group(0)])
785
842
  pos = m.end()
786
843
  else:
787
- result.append(ipa[pos])
844
+ out = _unknown(ipa[pos], pos, "cotovia", errors)
845
+ if out:
846
+ result.append(out)
788
847
  pos += 1
789
848
  return "".join(result)
790
849
 
@@ -857,7 +916,7 @@ _IPA_RFE_KEYS_SORTED = sorted(_IPA_TO_RFE.keys(), key=len, reverse=True)
857
916
  _IPA_RFE_RE = re.compile("|".join(re.escape(x) for x in _IPA_RFE_KEYS_SORTED))
858
917
 
859
918
 
860
- def rfe_to_ipa(rfe: str) -> str:
919
+ def rfe_to_ipa(rfe: str, errors: str = "pass") -> str:
861
920
  """Convert an RFE (Revista de Filología Española) string to IPA.
862
921
 
863
922
  Multi-character symbols are matched longest-first. Characters outside the
@@ -878,12 +937,14 @@ def rfe_to_ipa(rfe: str) -> str:
878
937
  result.append(_RFE_TO_IPA[m.group(0)])
879
938
  pos = m.end()
880
939
  else:
881
- result.append(rfe[pos])
940
+ out = _unknown(rfe[pos], pos, "rfe", errors)
941
+ if out:
942
+ result.append(out)
882
943
  pos += 1
883
944
  return "".join(result)
884
945
 
885
946
 
886
- def ipa_to_rfe(ipa: str) -> str:
947
+ def ipa_to_rfe(ipa: str, errors: str = "pass") -> str:
887
948
  """Convert an IPA string to RFE (Revista de Filología Española) notation.
888
949
 
889
950
  Multi-character IPA symbols are matched longest-first. Characters outside
@@ -904,7 +965,9 @@ def ipa_to_rfe(ipa: str) -> str:
904
965
  result.append(_IPA_TO_RFE[m.group(0)])
905
966
  pos = m.end()
906
967
  else:
907
- result.append(ipa[pos])
968
+ out = _unknown(ipa[pos], pos, "rfe", errors)
969
+ if out:
970
+ result.append(out)
908
971
  pos += 1
909
972
  return "".join(result)
910
973
 
@@ -913,7 +976,8 @@ def ipa_to_rfe(ipa: str) -> str:
913
976
  # convert — facade routing through IPA where no direct map exists
914
977
  # ---------------------------------------------------------------------------
915
978
 
916
- def convert(text: str, src: str | Notation, dst: str | Notation) -> str:
979
+ def convert(text: str, src: str | Notation, dst: str | Notation,
980
+ errors: str = "pass") -> str:
917
981
  """Convert *text* from *src* notation to *dst* notation.
918
982
 
919
983
  Supported pairs (direct):
@@ -956,7 +1020,9 @@ def convert(text: str, src: str | Notation, dst: str | Notation) -> str:
956
1020
  # node, Buckwalter <-> Arabic is a direct edge).
957
1021
  from scriptconv.graph import DEFAULT_GRAPH
958
1022
  try:
959
- return DEFAULT_GRAPH.convert(text, src.value, dst.value)
1023
+ return DEFAULT_GRAPH.convert(text, src.value, dst.value, errors=errors)
1024
+ except UnknownSymbolError:
1025
+ raise
960
1026
  except ValueError:
961
1027
  raise ValueError(f"Unsupported conversion: {src!r} → {dst!r}") from None
962
1028
 
@@ -2,7 +2,7 @@
2
2
  VERSION_MAJOR = 0
3
3
  VERSION_MINOR = 0
4
4
  VERSION_BUILD = 3
5
- VERSION_ALPHA = 8
5
+ VERSION_ALPHA = 9
6
6
  # END_VERSION_BLOCK
7
7
 
8
8
  VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptconv
3
- Version: 0.0.3a8
3
+ Version: 0.0.3a9
4
4
  Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
5
5
  License: Apache-2.0
6
6
  Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -199,6 +199,15 @@ Transcoding faithfulness depends on the target notation's inventory. IPA is the
199
199
  notation↔notation goes through IPA. The table below states, for each notation, whether a
200
200
  round-trip is exact and what happens to a symbol the table does not know.
201
201
 
202
+ Every converter (and `convert()`) accepts a codecs-style `errors=` policy for
203
+ symbols outside its table: `"pass"` keeps the symbol (the default everywhere
204
+ except `ipa_to_arpa`), `"replace"` substitutes the notation's placeholder
205
+ (`?`; the historical `ipa_to_arpa` default, tunable via `unknown=`),
206
+ `"ignore"` drops it, and `"strict"` raises `UnknownSymbolError` naming the
207
+ symbol, its position, and the notation. The "Unknown-token behaviour" column
208
+ below describes the per-converter default.
209
+
210
+
202
211
  | Notation | `to_ipa` → `from_ipa` round-trip | `from_ipa` → `to_ipa` round-trip | Unknown-token behaviour |
203
212
  |----------|----------------------------------|----------------------------------|-------------------------|
204
213
  | **ARPABET** | Exact for base symbols; **stress digits are dropped** and `AH0`↔`AX` (schwa) is not distinguished | **Lossy** — ARPABET is an English-only inventory, so any IPA symbol outside it becomes the *unknown* placeholder | `arpa_to_ipa`: passed through unchanged. `ipa_to_arpa`: diacritics and suprasegmentals (combining marks, length/stress modifiers) are dropped; other out-of-inventory symbols are replaced with `?` by default (`unknown=` param; `unknown=""` drops) |
@@ -23,6 +23,7 @@ scriptconv/data/cangjie5_tc.tsv.gz
23
23
  tests/test_cangjie.py
24
24
  tests/test_cli.py
25
25
  tests/test_conventions.py
26
+ tests/test_errors_policy.py
26
27
  tests/test_examples.py
27
28
  tests/test_graph.py
28
29
  tests/test_notation.py
@@ -0,0 +1,102 @@
1
+ import json
2
+ import pathlib
3
+ import unittest
4
+
5
+ from scriptconv import UnknownSymbolError, convert
6
+ from scriptconv import notation as N
7
+
8
+ FIXTURE = json.loads(
9
+ (pathlib.Path(__file__).parent / "data_errors_fixture.json").read_text())
10
+
11
+ # (converter name, a sample containing an unknown symbol, the unknown symbol)
12
+ UNKNOWN_SAMPLES = [
13
+ ("arpa_to_ipa", "ZZZ", "ZZZ"),
14
+ ("ipa_to_arpa", "χ", "χ"),
15
+ ("xsampa_to_ipa", "¤", "¤"),
16
+ ("ipa_to_xsampa", "¤", "¤"),
17
+ ("lexique_to_ipa", "¤", "¤"),
18
+ ("ipa_to_lexique", "χ", "χ"),
19
+ ("kirshenbaum_to_ipa", "¤", "¤"),
20
+ ("ipa_to_kirshenbaum", "ꟼ", "ꟼ"),
21
+ ("cotovia_to_ipa", "¤", "¤"),
22
+ ("ipa_to_cotovia", "χ", "χ"),
23
+ ("rfe_to_ipa", "¤", "¤"),
24
+ ("ipa_to_rfe", "χ", "χ"),
25
+ ("buckwalter_to_arabic", "#", "#"),
26
+ ("arabic_to_buckwalter", "x", "x"),
27
+ ]
28
+
29
+
30
+ class TestDefaultsUnchanged(unittest.TestCase):
31
+ """Pre-change fixture battery: no-kwargs output is byte-identical."""
32
+
33
+ def test_fixture_battery(self):
34
+ for fn_name, cases in FIXTURE.items():
35
+ fn = getattr(N, fn_name)
36
+ for inp, expected in cases.items():
37
+ self.assertEqual(fn(inp), expected, f"{fn_name}({inp!r})")
38
+
39
+
40
+ class TestPolicies(unittest.TestCase):
41
+ def test_all_four_policies_per_converter(self):
42
+ for fn_name, sample, symbol in UNKNOWN_SAMPLES:
43
+ fn = getattr(N, fn_name)
44
+ passed = fn(sample, errors="pass")
45
+ self.assertIn(symbol, passed, fn_name)
46
+ replaced = fn(sample, errors="replace")
47
+ self.assertNotIn(symbol, replaced, fn_name)
48
+ self.assertIn("?", replaced, fn_name)
49
+ ignored = fn(sample, errors="ignore")
50
+ self.assertNotIn(symbol, ignored, fn_name)
51
+ self.assertNotIn("?", ignored, fn_name)
52
+ with self.assertRaises(UnknownSymbolError, msg=fn_name):
53
+ fn(sample, errors="strict")
54
+
55
+ def test_strict_error_names_symbol_position_notation(self):
56
+ with self.assertRaises(UnknownSymbolError) as ctx:
57
+ N.xsampa_to_ipa("ab¤", errors="strict")
58
+ e = ctx.exception
59
+ self.assertEqual(e.symbol, "¤")
60
+ self.assertEqual(e.position, 2)
61
+ self.assertEqual(e.notation, "x-sampa")
62
+ self.assertIn("'¤'", str(e))
63
+ self.assertIn("2", str(e))
64
+ self.assertIn("x-sampa", str(e))
65
+
66
+ def test_invalid_policy_raises_value_error(self):
67
+ with self.assertRaises(ValueError):
68
+ N.xsampa_to_ipa("¤", errors="explode")
69
+
70
+ def test_unknown_symbol_error_is_value_error(self):
71
+ self.assertTrue(issubclass(UnknownSymbolError, ValueError))
72
+
73
+
74
+ class TestIpaToArpaCompat(unittest.TestCase):
75
+ def test_default_still_replaces_with_question_mark(self):
76
+ self.assertEqual(N.ipa_to_arpa("χ"), "?")
77
+
78
+ def test_unknown_empty_still_drops(self):
79
+ self.assertEqual(N.ipa_to_arpa("χkæt", unknown=""), "K AE T")
80
+
81
+ def test_custom_unknown_token(self):
82
+ self.assertEqual(N.ipa_to_arpa("χ", unknown="<UNK>"), "<UNK>")
83
+
84
+ def test_diacritics_never_treated_as_unknown(self):
85
+ # combining marks qualify the preceding phoneme even under strict
86
+ self.assertEqual(N.ipa_to_arpa("kæ̃t", errors="strict"), "K AE T")
87
+
88
+
89
+ class TestConvertFacade(unittest.TestCase):
90
+ def test_errors_threads_through_routing(self):
91
+ with self.assertRaises(UnknownSymbolError):
92
+ convert("¤", "x-sampa", "arpa", errors="strict")
93
+ self.assertEqual(convert("h@loU", "x-sampa", "arpa"), "HH AX L OW")
94
+
95
+ def test_errors_ignore_through_two_hops(self):
96
+ out = convert("k{t¤", "x-sampa", "arpa", errors="ignore")
97
+ self.assertNotIn("¤", out)
98
+ self.assertNotIn("?", out)
99
+
100
+
101
+ if __name__ == "__main__":
102
+ unittest.main()
File without changes
File without changes