scriptconv 0.0.4a2__tar.gz → 0.0.4a3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {scriptconv-0.0.4a2/scriptconv.egg-info → scriptconv-0.0.4a3}/PKG-INFO +40 -22
  2. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/README.md +32 -21
  3. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/pyproject.toml +3 -1
  4. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/base.py +100 -0
  5. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/version.py +1 -1
  6. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3/scriptconv.egg-info}/PKG-INFO +40 -22
  7. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv.egg-info/requires.txt +9 -0
  8. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_phonemizers_base.py +82 -0
  9. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/LICENSE +0 -0
  10. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/requirements.txt +0 -0
  11. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/__init__.py +0 -0
  12. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/__main__.py +0 -0
  13. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/cangjie.py +0 -0
  14. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/conventions.py +0 -0
  15. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/data/__init__.py +0 -0
  16. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/data/cangjie5_tc.tsv.gz +0 -0
  17. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/graph.py +0 -0
  18. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/notation.py +0 -0
  19. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/__init__.py +0 -0
  20. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/__init__.py +0 -0
  21. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/bw2ipa.py +0 -0
  22. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/hangul2ipa.py +0 -0
  23. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/ko_tables/aspiration.csv +0 -0
  24. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/ko_tables/assimilation.csv +0 -0
  25. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/ko_tables/double_coda.csv +0 -0
  26. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/ko_tables/hanja.tsv +0 -0
  27. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/ko_tables/ipa.csv +0 -0
  28. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/ko_tables/neutralization.csv +0 -0
  29. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/ko_tables/tensification.csv +0 -0
  30. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/ko_tables/yale.csv +0 -0
  31. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/shami/__init__.py +0 -0
  32. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/shami/codeswitch.py +0 -0
  33. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/shami/diacritize.py +0 -0
  34. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/shami/dialectal.py +0 -0
  35. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/shami/english_g2p.py +0 -0
  36. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/shami/espeak.py +0 -0
  37. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/shami/frontend.py +0 -0
  38. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/shami/levantine_g2p.py +0 -0
  39. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/shami/normalize.py +0 -0
  40. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/shami/phoneme_inventory.py +0 -0
  41. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_thirdparty/zh_num.py +0 -0
  42. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/__init__.py +0 -0
  43. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/kog2p/LICENSE.md +0 -0
  44. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/kog2p/__init__.py +0 -0
  45. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/kog2p/rulebook.txt +0 -0
  46. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/mantoq/LICENSE.md +0 -0
  47. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/mantoq/__init__.py +0 -0
  48. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/mantoq/buck/__init__.py +0 -0
  49. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/mantoq/buck/phonetise_buckwalter.py +0 -0
  50. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/mantoq/buck/symbols.py +0 -0
  51. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/mantoq/buck/tokenization.py +0 -0
  52. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/mantoq/num2words.py +0 -0
  53. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/_vendored/mantoq/unicode_symbol2label.py +0 -0
  54. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/ar.py +0 -0
  55. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/en.py +0 -0
  56. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/enums.py +0 -0
  57. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/eu.py +0 -0
  58. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/fa.py +0 -0
  59. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/gl.py +0 -0
  60. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/he.py +0 -0
  61. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/ja.py +0 -0
  62. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/ko.py +0 -0
  63. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/mul.py +0 -0
  64. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/mwl.py +0 -0
  65. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/o2ipa.py +0 -0
  66. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/pt.py +0 -0
  67. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/registry.py +0 -0
  68. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/shami.py +0 -0
  69. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/vi.py +0 -0
  70. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/phonemizers/zh.py +0 -0
  71. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/py.typed +0 -0
  72. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/readings.py +0 -0
  73. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/scripts.py +0 -0
  74. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv/translit.py +0 -0
  75. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv.egg-info/SOURCES.txt +0 -0
  76. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv.egg-info/dependency_links.txt +0 -0
  77. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/scriptconv.egg-info/top_level.txt +0 -0
  78. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/setup.cfg +0 -0
  79. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_arpa_stress.py +0 -0
  80. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_cangjie.py +0 -0
  81. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_cli.py +0 -0
  82. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_conventions.py +0 -0
  83. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_errors_policy.py +0 -0
  84. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_examples.py +0 -0
  85. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_graph.py +0 -0
  86. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_notation.py +0 -0
  87. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_phonemizers_cjk_ar.py +0 -0
  88. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_readings.py +0 -0
  89. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_readings_zh.py +0 -0
  90. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_scripts.py +0 -0
  91. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_scripts_stressonnx_compat.py +0 -0
  92. {scriptconv-0.0.4a2 → scriptconv-0.0.4a3}/tests/test_translit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptconv
3
- Version: 0.0.4a2
3
+ Version: 0.0.4a3
4
4
  Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
5
5
  License: Apache-2.0
6
6
  Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -78,6 +78,12 @@ Requires-Dist: viphoneme; extra == "vi"
78
78
  Provides-Extra: tashkeel
79
79
  Requires-Dist: scriptconv[phonemizers]; extra == "tashkeel"
80
80
  Requires-Dist: text2tashkeel; extra == "tashkeel"
81
+ Provides-Extra: stress
82
+ Requires-Dist: scriptconv[phonemizers]; extra == "stress"
83
+ Requires-Dist: stressonnx>=0.0.2; extra == "stress"
84
+ Provides-Extra: pt
85
+ Requires-Dist: scriptconv[phonemizers]; extra == "pt"
86
+ Requires-Dist: bifonia>=0.1.1; extra == "pt"
81
87
  Provides-Extra: ja-phonemizers
82
88
  Requires-Dist: scriptconv[phonemizers]; extra == "ja-phonemizers"
83
89
  Requires-Dist: pyopenjtalk; extra == "ja-phonemizers"
@@ -115,6 +121,7 @@ Requires-Dist: quebra-frases; extra == "test"
115
121
  Requires-Dist: langcodes; extra == "test"
116
122
  Requires-Dist: regex; extra == "test"
117
123
  Requires-Dist: ovos-number-parser; extra == "test"
124
+ Requires-Dist: bifonia>=0.1.1; extra == "test"
118
125
  Dynamic: license-file
119
126
 
120
127
  <div align="center">
@@ -369,6 +376,27 @@ Two design points worth knowing:
369
376
  local `model=` path; resolving and caching model files is the caller's
370
377
  concern.
371
378
 
379
+ **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)` restores
380
+ information ordinary orthography omits but a G2P needs, before phonemization:
381
+ Hebrew niqqud (phonikud), Arabic tashkeel (text2tashkeel), word stress across
382
+ 26 stressonnx language tags — East Slavic, Bulgarian/Macedonian/Slovene,
383
+ Latvian, Armenian, Georgian, and Turkic/Caucasian languages (`scriptconv[stress]`),
384
+ and European-Portuguese heterophonic-homograph sense diacritics (bifonia —
385
+ `scriptconv[pt]`, never applied to `pt-BR`, whose vowel system differs):
386
+
387
+ ```python
388
+ from scriptconv.phonemizers import GraphemePhonemizer
389
+
390
+ p = GraphemePhonemizer()
391
+ p.add_diacritics("Tenho muita sede hoje.", "pt") # 'Tenho muita sêde hoje.' (thirst)
392
+ ```
393
+
394
+ Stress is unwritten or under-marked in all 26 covered languages; East Slavic
395
+ is the clearest case, where unstressed vowels also reduce (e.g. Russian
396
+ о→[ɐ]/[ə]), so a missing mark corrupts more than prosody there. stressonnx is
397
+ optional (not yet on PyPI) and emits the standard combining acute (U+0301)
398
+ after the stressed vowel.
399
+
372
400
  Details: [docs/phonemizers.md](docs/phonemizers.md).
373
401
 
374
402
  ## Fidelity guarantees
@@ -412,6 +440,8 @@ The core installs with zero dependencies. Capabilities opt in:
412
440
  | `espeak`, `gruut`, `goruut`, `epitran`, `transphone`, `misaki`, `byt5` | Multilingual phonemizer backends |
413
441
  | `en-phonemizers`, `ja-phonemizers`, `zh-phonemizers`, `ko`, `ar-phonemizers`, `eu`, `pt-phonemizers`, `gl`, `he`, `fa`, `vi`, `mwl`, `shami`, `o2i` | Per-language phonemizer backends |
414
442
  | `tashkeel` | Arabic diacritization for the phonemizer pipeline (text2tashkeel) |
443
+ | `stress` | Word-stress restoration for 26 stressonnx language tags — East Slavic, Bulgarian/Macedonian/Slovene, Latvian, Armenian, Georgian, Turkic/Caucasian — for the phonemizer pipeline (stressonnx; not yet on PyPI) |
444
+ | `pt` | European-Portuguese heterophonic-homograph sense diacritics for the phonemizer pipeline (bifonia) |
415
445
 
416
446
  ## Licensing
417
447
 
@@ -433,29 +463,17 @@ build on it:
433
463
  text-to-speech; consumes scriptconv for scripts, notation, conventions and
434
464
  the whole phonemizer layer.
435
465
  - [orthography2ipa](https://github.com/TigreGotico/orthography2ipa) —
436
- data-driven orthography→IPA engine; the usual per-language phonemizer
437
- default.
438
- - [arbtok](https://github.com/TigreGotico/arbtok) — Arabic tokenization and
439
- phonemization; the Arabic default.
466
+ data-driven orthography→IPA engine; the usual per-language phonemizer default.
467
+ - [arbtok](https://github.com/TigreGotico/arbtok) — Arabic phonemizer (dialect-aware).
468
+ - [euskaphone](https://github.com/TigreGotico/euskaphone) — Basque phonemizer (dialect-aware).
469
+ - [tugaphone](https://github.com/TigreGotico/tugaphone) Portuguese phonemizer (dialect-aware).
470
+ - [mwl_phonemizer](https://github.com/TigreGotico/mwl_phonemizer) — Mirandese phonemizer.
471
+ - [g2p_barranquenho](https://github.com/TigreGotico/g2p_barranquenho) — Barranquenho phonemizer.
472
+ - [pycotovia](https://github.com/TigreGotico/pycotovia) — Pure-Python phonemizer port
473
+ of the Cotovía Galician TTS engine, whose notation scriptconv transcodes.
440
474
  - [espyak](https://github.com/TigreGotico/espyak) — pure-Python port of
441
475
  espeak-ng's G2P; the espeak fallback when the binary is absent.
442
- - [euskaphone](https://github.com/TigreGotico/euskaphone) — Basque
443
- phonemizer (dialect-aware).
444
- - [tugaphone](https://github.com/TigreGotico/tugaphone) — European
445
- Portuguese phonemizer.
446
- - [mwl_phonemizer](https://github.com/TigreGotico/mwl_phonemizer) —
447
- Mirandese phonemizer.
448
- - [g2p_barranquenho](https://github.com/TigreGotico/g2p_barranquenho) —
449
- Barranquenho phonemizer.
450
- - [pycotovia](https://github.com/TigreGotico/pycotovia) — Python bindings
451
- for the Cotovía Galician TTS engine, whose notation scriptconv transcodes.
452
- - [stressonnx](https://github.com/TigreGotico/stressonnx) — East Slavic
453
- stress annotation; shares scriptconv's script tags.
454
- - [text2tashkeel](https://github.com/TigreGotico/text2tashkeel) — Arabic
455
- diacritization models (the prediction side of the `tashkeel` convention).
456
- - [text2nikkud](https://github.com/TigreGotico/text2nikkud) — Hebrew
457
- niqqud restoration (the prediction side of the `niqqud` convention).
458
-
476
+
459
477
  ## Development
460
478
 
461
479
  ```bash
@@ -250,6 +250,27 @@ Two design points worth knowing:
250
250
  local `model=` path; resolving and caching model files is the caller's
251
251
  concern.
252
252
 
253
+ **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)` restores
254
+ information ordinary orthography omits but a G2P needs, before phonemization:
255
+ Hebrew niqqud (phonikud), Arabic tashkeel (text2tashkeel), word stress across
256
+ 26 stressonnx language tags — East Slavic, Bulgarian/Macedonian/Slovene,
257
+ Latvian, Armenian, Georgian, and Turkic/Caucasian languages (`scriptconv[stress]`),
258
+ and European-Portuguese heterophonic-homograph sense diacritics (bifonia —
259
+ `scriptconv[pt]`, never applied to `pt-BR`, whose vowel system differs):
260
+
261
+ ```python
262
+ from scriptconv.phonemizers import GraphemePhonemizer
263
+
264
+ p = GraphemePhonemizer()
265
+ p.add_diacritics("Tenho muita sede hoje.", "pt") # 'Tenho muita sêde hoje.' (thirst)
266
+ ```
267
+
268
+ Stress is unwritten or under-marked in all 26 covered languages; East Slavic
269
+ is the clearest case, where unstressed vowels also reduce (e.g. Russian
270
+ о→[ɐ]/[ə]), so a missing mark corrupts more than prosody there. stressonnx is
271
+ optional (not yet on PyPI) and emits the standard combining acute (U+0301)
272
+ after the stressed vowel.
273
+
253
274
  Details: [docs/phonemizers.md](docs/phonemizers.md).
254
275
 
255
276
  ## Fidelity guarantees
@@ -293,6 +314,8 @@ The core installs with zero dependencies. Capabilities opt in:
293
314
  | `espeak`, `gruut`, `goruut`, `epitran`, `transphone`, `misaki`, `byt5` | Multilingual phonemizer backends |
294
315
  | `en-phonemizers`, `ja-phonemizers`, `zh-phonemizers`, `ko`, `ar-phonemizers`, `eu`, `pt-phonemizers`, `gl`, `he`, `fa`, `vi`, `mwl`, `shami`, `o2i` | Per-language phonemizer backends |
295
316
  | `tashkeel` | Arabic diacritization for the phonemizer pipeline (text2tashkeel) |
317
+ | `stress` | Word-stress restoration for 26 stressonnx language tags — East Slavic, Bulgarian/Macedonian/Slovene, Latvian, Armenian, Georgian, Turkic/Caucasian — for the phonemizer pipeline (stressonnx; not yet on PyPI) |
318
+ | `pt` | European-Portuguese heterophonic-homograph sense diacritics for the phonemizer pipeline (bifonia) |
296
319
 
297
320
  ## Licensing
298
321
 
@@ -314,29 +337,17 @@ build on it:
314
337
  text-to-speech; consumes scriptconv for scripts, notation, conventions and
315
338
  the whole phonemizer layer.
316
339
  - [orthography2ipa](https://github.com/TigreGotico/orthography2ipa) —
317
- data-driven orthography→IPA engine; the usual per-language phonemizer
318
- default.
319
- - [arbtok](https://github.com/TigreGotico/arbtok) — Arabic tokenization and
320
- phonemization; the Arabic default.
340
+ data-driven orthography→IPA engine; the usual per-language phonemizer default.
341
+ - [arbtok](https://github.com/TigreGotico/arbtok) — Arabic phonemizer (dialect-aware).
342
+ - [euskaphone](https://github.com/TigreGotico/euskaphone) — Basque phonemizer (dialect-aware).
343
+ - [tugaphone](https://github.com/TigreGotico/tugaphone) Portuguese phonemizer (dialect-aware).
344
+ - [mwl_phonemizer](https://github.com/TigreGotico/mwl_phonemizer) — Mirandese phonemizer.
345
+ - [g2p_barranquenho](https://github.com/TigreGotico/g2p_barranquenho) — Barranquenho phonemizer.
346
+ - [pycotovia](https://github.com/TigreGotico/pycotovia) — Pure-Python phonemizer port
347
+ of the Cotovía Galician TTS engine, whose notation scriptconv transcodes.
321
348
  - [espyak](https://github.com/TigreGotico/espyak) — pure-Python port of
322
349
  espeak-ng's G2P; the espeak fallback when the binary is absent.
323
- - [euskaphone](https://github.com/TigreGotico/euskaphone) — Basque
324
- phonemizer (dialect-aware).
325
- - [tugaphone](https://github.com/TigreGotico/tugaphone) — European
326
- Portuguese phonemizer.
327
- - [mwl_phonemizer](https://github.com/TigreGotico/mwl_phonemizer) —
328
- Mirandese phonemizer.
329
- - [g2p_barranquenho](https://github.com/TigreGotico/g2p_barranquenho) —
330
- Barranquenho phonemizer.
331
- - [pycotovia](https://github.com/TigreGotico/pycotovia) — Python bindings
332
- for the Cotovía Galician TTS engine, whose notation scriptconv transcodes.
333
- - [stressonnx](https://github.com/TigreGotico/stressonnx) — East Slavic
334
- stress annotation; shares scriptconv's script tags.
335
- - [text2tashkeel](https://github.com/TigreGotico/text2tashkeel) — Arabic
336
- diacritization models (the prediction side of the `tashkeel` convention).
337
- - [text2nikkud](https://github.com/TigreGotico/text2nikkud) — Hebrew
338
- niqqud restoration (the prediction side of the `niqqud` convention).
339
-
350
+
340
351
  ## Development
341
352
 
342
353
  ```bash
@@ -53,13 +53,15 @@ he = ["scriptconv[phonemizers]", "phonikud-onnx"]
53
53
  fa = ["scriptconv[phonemizers]", "persian_phonemizer"]
54
54
  vi = ["scriptconv[phonemizers]", "viphoneme"]
55
55
  tashkeel = ["scriptconv[phonemizers]", "text2tashkeel"]
56
+ stress = ["scriptconv[phonemizers]", "stressonnx>=0.0.2"]
57
+ pt = ["scriptconv[phonemizers]", "bifonia>=0.1.1"]
56
58
  ja-phonemizers = ["scriptconv[phonemizers]", "pyopenjtalk", "cutlet", "pykakasi>=2.3,<3"]
57
59
  ko = ["scriptconv[phonemizers]", "g2pk", "regex"]
58
60
  zh-phonemizers = ["scriptconv[phonemizers]", "jieba", "pypinyin>=0.50,<1", "xpinyin", "g2pM", "pinyin_to_ipa"]
59
61
  en-phonemizers = ["scriptconv[phonemizers]", "g2p_en", "openphonemizer", "deep-phonemizer"]
60
62
  ar-phonemizers = ["scriptconv[phonemizers]", "arbtok", "ovos-number-parser"]
61
63
  shami = ["scriptconv[phonemizers]", "regex"]
62
- test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser"]
64
+ test = ["pytest", "pytest-timeout", "pykakasi>=2.3,<3", "pypinyin>=0.50,<1", "quebra-frases", "langcodes", "regex", "ovos-number-parser", "bifonia>=0.1.1"]
63
65
 
64
66
  [project.urls]
65
67
  Homepage = "https://github.com/TigreGotico/scriptconv"
@@ -34,6 +34,45 @@ RawPhonemizedChunks = List[Tuple[str, str, bool]]
34
34
 
35
35
  PhonemizedChunks = list[list[str]]
36
36
 
37
+ # Across East Slavic, Bulgarian/Macedonian/Slovene, Latvian, Armenian,
38
+ # Georgian, and several Turkic/Caucasian languages, lexical word stress is
39
+ # free (not fixed to a syllable) and ordinary orthography leaves it unwritten
40
+ # or under-marked. The clearest case is East Slavic: stress is also mobile
41
+ # (it shifts between forms of the same word) and unstressed vowels *reduce*
42
+ # — Russian unstressed "о" surfaces as [ɐ] or [ə] depending on distance from
43
+ # the stress, not [o] — so a wrong or missing mark there corrupts the vowel
44
+ # quality of the whole word, not just its prosody. Other families in this set
45
+ # don't necessarily reduce vowels, but still need the mark for correct stress
46
+ # placement and prosody. stressonnx restores it as a combining acute (U+0301)
47
+ # after the stressed vowel, covering 26 BCP-47 tags across these families
48
+ # (24 primary subtags; Azerbaijani and Uzbek each have Cyrillic/Latin script
49
+ # variants routed by the full tag).
50
+ STRESS_LANGS = {
51
+ "az", "ba", "be", "bg", "cv", "hy", "ka", "kbd", "kjh", "kk", "ky", "lv",
52
+ "mdf", "mk", "myv", "ru", "sah", "sl", "tg", "tt", "udm", "uk", "uz", "xal",
53
+ }
54
+
55
+
56
+ def _primary_subtag(lang: str) -> str:
57
+ """Lowercase, ``_``→``-`` normalized primary language subtag.
58
+
59
+ Used for exact-match routing (``STRESS_LANGS`` membership) rather than
60
+ ``str.startswith``, so e.g. Berber (``ber``) never false-matches
61
+ Belarusian (``be``).
62
+ """
63
+ return lang.lower().replace("_", "-").split("-")[0]
64
+
65
+
66
+ def _is_european_portuguese(lang: str) -> bool:
67
+ """True for European Portuguese (``pt`` or a ``pt-PT`` region tag).
68
+
69
+ False for Brazilian Portuguese (``pt-BR``) and everything else — the two
70
+ varieties' vowel systems differ, and bifonia's open/closed diacritics are
71
+ only valid for European Portuguese phonology.
72
+ """
73
+ norm = lang.lower().replace("_", "-")
74
+ return norm == "pt" or norm == "pt-pt"
75
+
37
76
 
38
77
  class BasePhonemizer(metaclass=abc.ABCMeta):
39
78
  def __init__(self, alphabet: Alphabet = Alphabet.UNICODE,
@@ -94,6 +133,41 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
94
133
  self._tashkeel[model] = Diacritizer(model)
95
134
  return self._tashkeel[model]
96
135
 
136
+ def _stress(self, text: str, lang: str, model: Optional[str] = None) -> str:
137
+ """Word-stress restoration via stressonnx, for the 26 language tags
138
+ it covers (see ``STRESS_LANGS``) — East Slavic, Bulgarian/Macedonian/
139
+ Slovene, Latvian, Armenian, Georgian, and Turkic/Caucasian languages.
140
+
141
+ stressonnx is not on PyPI yet; install straight from source. Install
142
+ with ``pip install scriptconv[stress]`` (or ``pip install
143
+ stressonnx``)."""
144
+ try:
145
+ from stressonnx import stress
146
+ except ImportError as e:
147
+ raise ImportError(
148
+ "stress restoration requires the stressonnx package: "
149
+ "pip install scriptconv[stress] (or pip install stressonnx)"
150
+ ) from e
151
+ return stress(text, lang, model=model)
152
+
153
+ def _sense_diacritics_pt(self, text: str) -> str:
154
+ """European-Portuguese heterophonic-homograph sense diacritics via bifonia.
155
+
156
+ Rewrites homographs whose pronunciation depends on meaning (e.g.
157
+ "sede" thirst/closed vs. seat/open) with an explicit open/closed
158
+ vowel diacritic. These are ordinary Portuguese orthographic marks,
159
+ chosen so any downstream G2P — rule-based, neural, or espeak —
160
+ reads them correctly. Install with ``pip install scriptconv[pt]``
161
+ (or ``pip install bifonia``)."""
162
+ try:
163
+ from bifonia import add_extra_diacritics
164
+ except ImportError as e:
165
+ raise ImportError(
166
+ "European-Portuguese sense diacritics require the bifonia package: "
167
+ "pip install scriptconv[pt] (or pip install bifonia)"
168
+ ) from e
169
+ return add_extra_diacritics(text)
170
+
97
171
  @abc.abstractmethod
98
172
  def phonemize_string(self, text: str, lang: str) -> str:
99
173
  raise NotImplementedError
@@ -103,10 +177,36 @@ class BasePhonemizer(metaclass=abc.ABCMeta):
103
177
 
104
178
  def add_diacritics(self, text: str, lang: str,
105
179
  model: Optional[str] = None) -> str:
180
+ """Disambiguate pronunciation before G2P by adding diacritics.
181
+
182
+ Four backends, each restoring information ordinary orthography
183
+ omits but downstream G2P needs:
184
+
185
+ - Hebrew (``he``) — niqqud via phonikud (``phonikud_model=``).
186
+ - Arabic (``ar``) — tashkeel via text2tashkeel (``[tashkeel]``).
187
+ - East Slavic, Bulgarian/Macedonian/Slovene, Latvian, Armenian,
188
+ Georgian, and Turkic/Caucasian languages (``STRESS_LANGS``, 26
189
+ stressonnx tags) — word stress via stressonnx (``[stress]``);
190
+ stress is unwritten or under-marked in these languages, and in
191
+ East Slavic unstressed vowels also reduce, so a missing mark can
192
+ corrupt more than prosody.
193
+ - European Portuguese (``pt``/``pt-PT``, never ``pt-BR``) —
194
+ heterophonic-homograph sense diacritics via bifonia (``[pt]``);
195
+ ordinary Portuguese orthographic marks that any downstream G2P
196
+ reads correctly.
197
+
198
+ Unrecognized languages are returned unchanged. Each backend raises
199
+ ``ImportError`` naming its extra when the optional dependency is
200
+ missing — scriptconv never installs anything on the caller's behalf.
201
+ """
106
202
  if lang.startswith("he"):
107
203
  return self.phonikud.add_diacritics(text)
108
204
  elif lang.startswith("ar"):
109
205
  return self.tashkeel(model).diacritize(text)
206
+ elif _primary_subtag(lang) in STRESS_LANGS:
207
+ return self._stress(text, lang, model)
208
+ elif _is_european_portuguese(lang):
209
+ return self._sense_diacritics_pt(text)
110
210
  return text
111
211
 
112
212
  def phonemize(self, text: str, lang: str) -> PhonemizedChunks:
@@ -2,7 +2,7 @@
2
2
  VERSION_MAJOR = 0
3
3
  VERSION_MINOR = 0
4
4
  VERSION_BUILD = 4
5
- VERSION_ALPHA = 2
5
+ VERSION_ALPHA = 3
6
6
  # END_VERSION_BLOCK
7
7
 
8
8
  VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptconv
3
- Version: 0.0.4a2
3
+ Version: 0.0.4a3
4
4
  Summary: Zero-dependency script & phoneme-notation core — ISO-15924 detection & metadata, IPA↔ARPABET/X-SAMPA/Lexique/Kirshenbaum/Cotovía/RFE, Buckwalter↔Arabic, Hangul→jamo, kana
5
5
  License: Apache-2.0
6
6
  Project-URL: Homepage, https://github.com/TigreGotico/scriptconv
@@ -78,6 +78,12 @@ Requires-Dist: viphoneme; extra == "vi"
78
78
  Provides-Extra: tashkeel
79
79
  Requires-Dist: scriptconv[phonemizers]; extra == "tashkeel"
80
80
  Requires-Dist: text2tashkeel; extra == "tashkeel"
81
+ Provides-Extra: stress
82
+ Requires-Dist: scriptconv[phonemizers]; extra == "stress"
83
+ Requires-Dist: stressonnx>=0.0.2; extra == "stress"
84
+ Provides-Extra: pt
85
+ Requires-Dist: scriptconv[phonemizers]; extra == "pt"
86
+ Requires-Dist: bifonia>=0.1.1; extra == "pt"
81
87
  Provides-Extra: ja-phonemizers
82
88
  Requires-Dist: scriptconv[phonemizers]; extra == "ja-phonemizers"
83
89
  Requires-Dist: pyopenjtalk; extra == "ja-phonemizers"
@@ -115,6 +121,7 @@ Requires-Dist: quebra-frases; extra == "test"
115
121
  Requires-Dist: langcodes; extra == "test"
116
122
  Requires-Dist: regex; extra == "test"
117
123
  Requires-Dist: ovos-number-parser; extra == "test"
124
+ Requires-Dist: bifonia>=0.1.1; extra == "test"
118
125
  Dynamic: license-file
119
126
 
120
127
  <div align="center">
@@ -369,6 +376,27 @@ Two design points worth knowing:
369
376
  local `model=` path; resolving and caching model files is the caller's
370
377
  concern.
371
378
 
379
+ **Pre-G2P disambiguation.** `add_diacritics(text, lang, model=None)` restores
380
+ information ordinary orthography omits but a G2P needs, before phonemization:
381
+ Hebrew niqqud (phonikud), Arabic tashkeel (text2tashkeel), word stress across
382
+ 26 stressonnx language tags — East Slavic, Bulgarian/Macedonian/Slovene,
383
+ Latvian, Armenian, Georgian, and Turkic/Caucasian languages (`scriptconv[stress]`),
384
+ and European-Portuguese heterophonic-homograph sense diacritics (bifonia —
385
+ `scriptconv[pt]`, never applied to `pt-BR`, whose vowel system differs):
386
+
387
+ ```python
388
+ from scriptconv.phonemizers import GraphemePhonemizer
389
+
390
+ p = GraphemePhonemizer()
391
+ p.add_diacritics("Tenho muita sede hoje.", "pt") # 'Tenho muita sêde hoje.' (thirst)
392
+ ```
393
+
394
+ Stress is unwritten or under-marked in all 26 covered languages; East Slavic
395
+ is the clearest case, where unstressed vowels also reduce (e.g. Russian
396
+ о→[ɐ]/[ə]), so a missing mark corrupts more than prosody there. stressonnx is
397
+ optional (not yet on PyPI) and emits the standard combining acute (U+0301)
398
+ after the stressed vowel.
399
+
372
400
  Details: [docs/phonemizers.md](docs/phonemizers.md).
373
401
 
374
402
  ## Fidelity guarantees
@@ -412,6 +440,8 @@ The core installs with zero dependencies. Capabilities opt in:
412
440
  | `espeak`, `gruut`, `goruut`, `epitran`, `transphone`, `misaki`, `byt5` | Multilingual phonemizer backends |
413
441
  | `en-phonemizers`, `ja-phonemizers`, `zh-phonemizers`, `ko`, `ar-phonemizers`, `eu`, `pt-phonemizers`, `gl`, `he`, `fa`, `vi`, `mwl`, `shami`, `o2i` | Per-language phonemizer backends |
414
442
  | `tashkeel` | Arabic diacritization for the phonemizer pipeline (text2tashkeel) |
443
+ | `stress` | Word-stress restoration for 26 stressonnx language tags — East Slavic, Bulgarian/Macedonian/Slovene, Latvian, Armenian, Georgian, Turkic/Caucasian — for the phonemizer pipeline (stressonnx; not yet on PyPI) |
444
+ | `pt` | European-Portuguese heterophonic-homograph sense diacritics for the phonemizer pipeline (bifonia) |
415
445
 
416
446
  ## Licensing
417
447
 
@@ -433,29 +463,17 @@ build on it:
433
463
  text-to-speech; consumes scriptconv for scripts, notation, conventions and
434
464
  the whole phonemizer layer.
435
465
  - [orthography2ipa](https://github.com/TigreGotico/orthography2ipa) —
436
- data-driven orthography→IPA engine; the usual per-language phonemizer
437
- default.
438
- - [arbtok](https://github.com/TigreGotico/arbtok) — Arabic tokenization and
439
- phonemization; the Arabic default.
466
+ data-driven orthography→IPA engine; the usual per-language phonemizer default.
467
+ - [arbtok](https://github.com/TigreGotico/arbtok) — Arabic phonemizer (dialect-aware).
468
+ - [euskaphone](https://github.com/TigreGotico/euskaphone) — Basque phonemizer (dialect-aware).
469
+ - [tugaphone](https://github.com/TigreGotico/tugaphone) Portuguese phonemizer (dialect-aware).
470
+ - [mwl_phonemizer](https://github.com/TigreGotico/mwl_phonemizer) — Mirandese phonemizer.
471
+ - [g2p_barranquenho](https://github.com/TigreGotico/g2p_barranquenho) — Barranquenho phonemizer.
472
+ - [pycotovia](https://github.com/TigreGotico/pycotovia) — Pure-Python phonemizer port
473
+ of the Cotovía Galician TTS engine, whose notation scriptconv transcodes.
440
474
  - [espyak](https://github.com/TigreGotico/espyak) — pure-Python port of
441
475
  espeak-ng's G2P; the espeak fallback when the binary is absent.
442
- - [euskaphone](https://github.com/TigreGotico/euskaphone) — Basque
443
- phonemizer (dialect-aware).
444
- - [tugaphone](https://github.com/TigreGotico/tugaphone) — European
445
- Portuguese phonemizer.
446
- - [mwl_phonemizer](https://github.com/TigreGotico/mwl_phonemizer) —
447
- Mirandese phonemizer.
448
- - [g2p_barranquenho](https://github.com/TigreGotico/g2p_barranquenho) —
449
- Barranquenho phonemizer.
450
- - [pycotovia](https://github.com/TigreGotico/pycotovia) — Python bindings
451
- for the Cotovía Galician TTS engine, whose notation scriptconv transcodes.
452
- - [stressonnx](https://github.com/TigreGotico/stressonnx) — East Slavic
453
- stress annotation; shares scriptconv's script tags.
454
- - [text2tashkeel](https://github.com/TigreGotico/text2tashkeel) — Arabic
455
- diacritization models (the prediction side of the `tashkeel` convention).
456
- - [text2nikkud](https://github.com/TigreGotico/text2nikkud) — Hebrew
457
- niqqud restoration (the prediction side of the `niqqud` convention).
458
-
476
+
459
477
  ## Development
460
478
 
461
479
  ```bash
@@ -78,6 +78,10 @@ orthography2ipa
78
78
  quebra-frases
79
79
  langcodes
80
80
 
81
+ [pt]
82
+ scriptconv[phonemizers]
83
+ bifonia>=0.1.1
84
+
81
85
  [pt-phonemizers]
82
86
  scriptconv[phonemizers]
83
87
  tugaphone
@@ -87,6 +91,10 @@ g2p_barranquenho
87
91
  scriptconv[phonemizers]
88
92
  regex
89
93
 
94
+ [stress]
95
+ scriptconv[phonemizers]
96
+ stressonnx>=0.0.2
97
+
90
98
  [tashkeel]
91
99
  scriptconv[phonemizers]
92
100
  text2tashkeel
@@ -100,6 +108,7 @@ quebra-frases
100
108
  langcodes
101
109
  regex
102
110
  ovos-number-parser
111
+ bifonia>=0.1.1
103
112
 
104
113
  [transphone]
105
114
  scriptconv[phonemizers]
@@ -221,3 +221,85 @@ class TestModelForwarding(unittest.TestCase):
221
221
  def test_model_none_forwards_nothing(self):
222
222
  g = get_phonemizer(Phonemizer.GRAPHEMES, model=None)
223
223
  self.assertIsInstance(g, GraphemePhonemizer)
224
+
225
+
226
+ class TestEuropeanPortugueseSenseDiacritics(unittest.TestCase):
227
+ """Real bifonia — it is on PyPI, so this exercises the actual backend."""
228
+
229
+ def test_thirst_sense_gets_closed_vowel(self):
230
+ out = GraphemePhonemizer().add_diacritics(
231
+ "Tenho muita sede hoje.", "pt")
232
+ self.assertEqual(out, "Tenho muita sêde hoje.")
233
+
234
+ def test_seat_sense_gets_open_vowel(self):
235
+ out = GraphemePhonemizer().add_diacritics(
236
+ "A sede da empresa fica em Lisboa.", "pt")
237
+ self.assertEqual(out, "A séde da empresa fica em Lisboa.")
238
+
239
+ def test_no_homograph_unchanged(self):
240
+ text = "O cão correu no jardim."
241
+ self.assertEqual(GraphemePhonemizer().add_diacritics(text, "pt"), text)
242
+
243
+ def test_brazilian_portuguese_excluded(self):
244
+ text = "Tenho muita sede hoje."
245
+ self.assertEqual(GraphemePhonemizer().add_diacritics(text, "pt-BR"), text)
246
+
247
+
248
+ class TestEastSlavicStressRouting(unittest.TestCase):
249
+ """stressonnx is not yet on PyPI — routing is verified against a stub."""
250
+
251
+ def _stub(self, calls):
252
+ import types
253
+ mod = types.ModuleType("stressonnx")
254
+
255
+ def stress(text, lang, model=None):
256
+ calls.append((text, lang, model))
257
+ return "STRESSED"
258
+
259
+ mod.stress = stress
260
+ return mod
261
+
262
+ def test_russian_routes_to_stress_backend(self):
263
+ import sys
264
+ from unittest import mock
265
+ calls = []
266
+ with mock.patch.dict(sys.modules, {"stressonnx": self._stub(calls)}):
267
+ out = GraphemePhonemizer().add_diacritics(
268
+ "замок стоит", "ru", model="silero")
269
+ self.assertEqual(out, "STRESSED")
270
+ self.assertEqual(calls, [("замок стоит", "ru", "silero")])
271
+
272
+ def test_ukrainian_and_belarusian_route_too(self):
273
+ import sys
274
+ from unittest import mock
275
+ for lang in ("uk", "be"):
276
+ calls = []
277
+ with mock.patch.dict(sys.modules, {"stressonnx": self._stub(calls)}):
278
+ GraphemePhonemizer().add_diacritics("текст", lang, model="ruaccent")
279
+ self.assertEqual(calls, [("текст", lang, "ruaccent")], lang)
280
+
281
+ def test_additional_stressonnx_languages_route_too(self):
282
+ import sys
283
+ from unittest import mock
284
+ for lang in ("kk", "hy", "az-Latn"):
285
+ calls = []
286
+ with mock.patch.dict(sys.modules, {"stressonnx": self._stub(calls)}):
287
+ GraphemePhonemizer().add_diacritics("text", lang, model="simple")
288
+ self.assertEqual(calls, [("text", lang, "simple")], lang)
289
+
290
+ def test_berber_does_not_false_match_belarusian(self):
291
+ import sys
292
+ from unittest import mock
293
+ calls = []
294
+ with mock.patch.dict(sys.modules, {"stressonnx": self._stub(calls)}):
295
+ out = GraphemePhonemizer().add_diacritics("azul", "ber")
296
+ self.assertEqual(out, "azul")
297
+ self.assertEqual(calls, [])
298
+
299
+ def test_missing_stressonnx_raises_named_importerror(self):
300
+ import sys
301
+ from unittest import mock
302
+ with mock.patch.dict(sys.modules, {"stressonnx": None}):
303
+ with self.assertRaises(ImportError) as ctx:
304
+ GraphemePhonemizer().add_diacritics("замок", "ru")
305
+ self.assertIn("stressonnx", str(ctx.exception))
File without changes
File without changes