mwl-phonemizer 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mwl_phonemizer/__init__.py +70 -0
- mwl_phonemizer/base.py +115 -0
- mwl_phonemizer/central.json +147 -0
- mwl_phonemizer/crf_mwl.py +258 -0
- mwl_phonemizer/epitran_mwl.py +411 -0
- mwl_phonemizer/espeak_mwl.py +470 -0
- mwl_phonemizer/g2p.json +102 -0
- mwl_phonemizer/ngram_mwl.py +364 -0
- mwl_phonemizer/orthography_hand_rules.py +576 -0
- mwl_phonemizer/raiano.json +4 -0
- mwl_phonemizer/sendinese.json +13 -0
- mwl_phonemizer/version.py +10 -0
- mwl_phonemizer-0.0.2.dist-info/METADATA +7 -0
- mwl_phonemizer-0.0.2.dist-info/RECORD +16 -0
- mwl_phonemizer-0.0.2.dist-info/WHEEL +5 -0
- mwl_phonemizer-0.0.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
from mwl_phonemizer.crf_mwl import CRFPhonemizer
|
|
2
|
+
from mwl_phonemizer.epitran_mwl import EpitranMWL
|
|
3
|
+
from mwl_phonemizer.espeak_mwl import EspeakMWL
|
|
4
|
+
from mwl_phonemizer.ngram_mwl import NgramMWLPhonemizer
|
|
5
|
+
from mwl_phonemizer.orthography_hand_rules import OrthographyRulesMWL
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
if __name__ == "__main__":
|
|
9
|
+
sample_texts = [
|
|
10
|
+
"Muitas lhénguas ténen proua de ls sous pergaminos antigos, de la lhiteratura screbida hai cientos d'anhos i de scritores hai muito afamados, hoije bandeiras dessas lhénguas. Mas outras hai que nun puoden tener proua de nada desso, cumo ye l causo de la lhéngua mirandesa.",
|
|
11
|
+
"Todos ls seres houmanos nácen lhibres i eiguales an honra i an dreitos. Dotados de rezon i de cuncéncia, dében de se dar bien uns culs outros i cumo armano",
|
|
12
|
+
"Hai más fuogo alhá, i ye deimingo!",
|
|
13
|
+
"""Quien dirie qu'antre ls matos eiriçados
|
|
14
|
+
Las ourriêtas i ls rius d'esta tiêrra,
|
|
15
|
+
Bibie, cumo l chaugarço de la siêrra,
|
|
16
|
+
Ua lhéngua de sons tan bariados?
|
|
17
|
+
|
|
18
|
+
Mostre-se i fale-s' essa lhéngua filha
|
|
19
|
+
D'un pobo que ten neilha l choro i l canto!
|
|
20
|
+
Nada por ciêrto mos cautiba tanto
|
|
21
|
+
Cumo la form' an que l'eideia brilha.
|
|
22
|
+
|
|
23
|
+
Zgraçiado d'aquel, qu'abandonando
|
|
24
|
+
La patri' an que naciu, la casa i l huôrto.
|
|
25
|
+
Tamien se squeçe de la fala! Quando
|
|
26
|
+
L furdes ber, talbéç que stéia muôrto!"""
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
phonemizer = EpitranMWL()
|
|
30
|
+
for text in sample_texts:
|
|
31
|
+
print(f"Original: {text}")
|
|
32
|
+
print(f"Phonemized: {phonemizer.phonemize_sentence(text)}\n")
|
|
33
|
+
|
|
34
|
+
# Original: Muitas lhénguas ténen proua de ls sous pergaminos antigos, de la lhiteratura screbida hai cientos d'anhos i de scritores hai muito afamados, hoije bandeiras dessas lhénguas. Mas outras hai que nun puoden tener proua de nada desso, cumo ye l causo de la lhéngua mirandesa.
|
|
35
|
+
# Phonemized: mujtɐs̺ ʎenɡɐs̺ tenɛ̃ pɾowuɐ dɛ ls̺ sowus̺ pɛɾɡɐminos̺ ɐntiɡos̺, dɛ ʎɐ ʎitɛɾɐtuɾɐ s̺kɾβdɐ aj s̻iɛntos̻ d'ɐnos̺ i dɛ s̺kɾitoɾɛs̺ aj mujtu ɐfɐmðs̺, owiʒɛ bɐndɛjɾɐs̺ dɛʃsɐs̺ ʎenɡɐs̺. mɐs̺ owutrɐs̺ aj kʷɛ nũ puð̃ tɨˈneɾ pɾowuɐ dɛ nð dɛʃsu, kumu ˈje l̩ kawzu dɛ ʎɐ ˈʎɛ̃ɡwɐ miɾɐndɛzɐ.
|
|
36
|
+
#
|
|
37
|
+
# Original: Todos ls seres houmanos nácen lhibres i eiguales an honra i an dreitos. Dotados de rezon i de cuncéncia, dében de se dar bien uns culs outros i cumo armano
|
|
38
|
+
# Phonemized: tðs̺ ls̺ sɛɾɛs̺ owumɐnos̺ nazɛ̃ ʎibrɛs̺ i ɛjɡɐlɛs̺ ɐ̃ onrɐ i ɐ̃ dɾɛjtos̺. dotðs̺ dɛ rɛzõ i dɛ kuns̻ens̻iɐ, dβ̃ dɛ sɛ dɐɾ biɛ̃ uns̺ kuls̺ owutros̺ i kumu ɐɾmɐnu
|
|
39
|
+
#
|
|
40
|
+
# Original: Hai más fuogo alhá, i ye deimingo!
|
|
41
|
+
# Phonemized: aj mas̺ fwoɣʊ ɐˈʎa, i ˈje dejˈmĩgʊ!
|
|
42
|
+
#
|
|
43
|
+
# Original: Quien dirie qu'antre ls matos eiriçados
|
|
44
|
+
# Las ourriêtas i ls rius d'esta tiêrra,
|
|
45
|
+
# Bibie, cumo l chaugarço de la siêrra,
|
|
46
|
+
# Ua lhéngua de sons tan bariados?
|
|
47
|
+
#
|
|
48
|
+
# Mostre-se i fale-s' essa lhéngua filha
|
|
49
|
+
# D'un pobo que ten neilha l choro i l canto!
|
|
50
|
+
# Nada por ciêrto mos cautiba tanto
|
|
51
|
+
# Cumo la form' an que l'eideia brilha.
|
|
52
|
+
#
|
|
53
|
+
# Zgraçiado d'aquel, qu'abandonando
|
|
54
|
+
# La patri' an que naciu, la casa i l huôrto.
|
|
55
|
+
# Tamien se squeçe de la fala! Quando
|
|
56
|
+
# L furdes ber, talbéç que stéia muôrto!
|
|
57
|
+
# Phonemized: kʷiɛ̃ diɾiɛ k'ɐntɾɛ ls̺ mɐtos̺ ɛjɾisðs̻
|
|
58
|
+
# ʎɐs̺ owurietɐs̺ i ls̺ riws̺ d'ɛʃtɐ tierɐ,
|
|
59
|
+
# bβɛ, kumu l̩ kawɡɐɾsu dɛ ʎɐ s̻ierɐ,
|
|
60
|
+
# ˈũŋɐ ˈʎɛ̃ɡwɐ dɛ sons̺ tɐ̃ bɐɾiðs̺?
|
|
61
|
+
#
|
|
62
|
+
# moʃtɾɛ sɛ i fɐlɛ s̺̺' ɛʃsɐ ˈʎɛ̃ɡwɐ filɐ
|
|
63
|
+
# d'ũ pβ kʷɛ tɛ̃ nɛjlɐ l̩ koɾu i l̩ kɐntu!
|
|
64
|
+
# nð poɾ s̻ieɾtu mos̺ kawtβ tɐntu
|
|
65
|
+
# kumu ʎɐ foɾ' ɐ̃ kʷɛ l̩'ɛjdɛjɐ bɾilɐ.
|
|
66
|
+
#
|
|
67
|
+
# zɡɾɐs̻ið d'ɐkʷɛl, k'βndonɐndu
|
|
68
|
+
# ʎɐ pɐtri' ɐ̃ kʷɛ nɐziw, ʎɐ kɐzɐ i l̩ uoɾtu.
|
|
69
|
+
# tɐˈmjẽ sɛ s̻kʷɛsɛ dɛ ʎɐ fɐlɐ! ˈkwɐ̃du
|
|
70
|
+
# l̩ fuɾdɛs̺ bɛɾ, tɐlbes kʷɛ s̺tejɐ muoɾtu!
|
mwl_phonemizer/base.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
import json
|
|
3
|
+
import re
|
|
4
|
+
import os
|
|
5
|
+
from collections import Counter
|
|
6
|
+
import editdistance
|
|
7
|
+
from enum import Enum
|
|
8
|
+
|
|
9
|
+
class Dialects(str, Enum):
|
|
10
|
+
CENTRAL = "central"
|
|
11
|
+
RAIANO = "raiano"
|
|
12
|
+
SENDINESE = "sendinese"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class MirandesePhonemizer:
|
|
16
|
+
def __init__(self,
|
|
17
|
+
gold_dict: str | None = None,
|
|
18
|
+
raiano_dict: str | None = None, # dialect exceptions
|
|
19
|
+
sendinese_dict: str | None = None, # dialect exceptions
|
|
20
|
+
dialect: Dialects = Dialects.CENTRAL):
|
|
21
|
+
|
|
22
|
+
self.dialect = dialect
|
|
23
|
+
|
|
24
|
+
gold_dict = gold_dict or f"{os.path.dirname(__file__)}/central.json"
|
|
25
|
+
raiano_dict = raiano_dict or f"{os.path.dirname(__file__)}/raiano.json"
|
|
26
|
+
sendinese_dict = sendinese_dict or f"{os.path.dirname(__file__)}/sendinese.json"
|
|
27
|
+
|
|
28
|
+
with open(gold_dict, "r", encoding="utf-8") as f:
|
|
29
|
+
self.GOLD = {k: self.strip_markers(v) for k, v in json.load(f).items()}
|
|
30
|
+
with open(raiano_dict, "r", encoding="utf-8") as f:
|
|
31
|
+
self.RAIANO_GOLD = {k: self.strip_markers(v) for k, v in json.load(f).items()}
|
|
32
|
+
with open(sendinese_dict, "r", encoding="utf-8") as f:
|
|
33
|
+
self.SENDINESE_GOLD = {k: self.strip_markers(v) for k, v in json.load(f).items()}
|
|
34
|
+
|
|
35
|
+
def phonemize(self, word: str, lookup_word: bool = True) -> str:
|
|
36
|
+
if lookup_word and word.lower() in self.GOLD:
|
|
37
|
+
return self.GOLD[word.lower()]
|
|
38
|
+
raise ValueError(f"unknown word: '{word}'")
|
|
39
|
+
|
|
40
|
+
def phonemize_sentence(self,
|
|
41
|
+
text: str, lookup_word: bool = True):
|
|
42
|
+
text = text.replace("-", " ")
|
|
43
|
+
words = re.findall(r"\b\w+\b|[\W_]+", text) # Split by words and keep punctuation/spaces
|
|
44
|
+
phonemized_parts = []
|
|
45
|
+
for word_or_punc in words:
|
|
46
|
+
if word_or_punc.isalpha():
|
|
47
|
+
phonemized_parts.append(self.phonemize(word_or_punc, lookup_word=lookup_word))
|
|
48
|
+
else:
|
|
49
|
+
phonemized_parts.append(word_or_punc) # Keep punctuation and spaces as is
|
|
50
|
+
return "".join(phonemized_parts)
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def strip_markers(ipa: str) -> str:
|
|
54
|
+
return ipa.replace(".", "").replace("(", "").replace(")", "") # drop syllable/optional markers
|
|
55
|
+
|
|
56
|
+
@staticmethod
|
|
57
|
+
def strip_stress(ipa: str) -> str:
|
|
58
|
+
"""Removes the primary stress marker 'ˈ' for stress-agnostic comparison."""
|
|
59
|
+
return ipa.replace("ˈ", "").replace("ˌ", "")
|
|
60
|
+
|
|
61
|
+
@staticmethod
|
|
62
|
+
def word_edit_distance(a: str, b: str) -> int:
|
|
63
|
+
return editdistance.eval(a, b)
|
|
64
|
+
|
|
65
|
+
def evaluate_on_gold(self, limit=None, detailed=False, show_changes=False):
|
|
66
|
+
pairs = list(self.GOLD.items())
|
|
67
|
+
if limit:
|
|
68
|
+
pairs = pairs[:limit]
|
|
69
|
+
|
|
70
|
+
total_ed_after = 0
|
|
71
|
+
cnt = 0
|
|
72
|
+
improvements = Counter()
|
|
73
|
+
details = []
|
|
74
|
+
|
|
75
|
+
# Stress-agnostic metrics
|
|
76
|
+
total_ed_no_stress_after = 0
|
|
77
|
+
|
|
78
|
+
for ortho, gold_ipa in pairs:
|
|
79
|
+
phonemes = self.phonemize(ortho, lookup_word=False)
|
|
80
|
+
|
|
81
|
+
# Standard metrics (includes stress)
|
|
82
|
+
ed_after = self.word_edit_distance(phonemes, gold_ipa)
|
|
83
|
+
total_ed_after += ed_after
|
|
84
|
+
cnt += 1
|
|
85
|
+
|
|
86
|
+
# Stress-agnostic metrics (ignores stress)
|
|
87
|
+
corrected_no_stress = self.strip_stress(phonemes)
|
|
88
|
+
gold_ipa_no_stress = self.strip_stress(gold_ipa)
|
|
89
|
+
|
|
90
|
+
ed_after_no_stress = self.word_edit_distance(corrected_no_stress, gold_ipa_no_stress)
|
|
91
|
+
|
|
92
|
+
total_ed_no_stress_after += ed_after_no_stress
|
|
93
|
+
|
|
94
|
+
# Only append to details if the final corrected IPA does not match the gold (ED > 0)
|
|
95
|
+
if ed_after > 0:
|
|
96
|
+
details.append({
|
|
97
|
+
"word": ortho,
|
|
98
|
+
"phonemes": phonemes,
|
|
99
|
+
"gold": gold_ipa,
|
|
100
|
+
"ed": ed_after,
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
result = {
|
|
104
|
+
# Standard Metrics
|
|
105
|
+
"avg_edit_distance": total_ed_after / cnt if cnt else 0,
|
|
106
|
+
|
|
107
|
+
# Stress-Agnostic Metrics
|
|
108
|
+
"avg_edit_distance_no_stress": total_ed_no_stress_after / cnt if cnt else 0,
|
|
109
|
+
|
|
110
|
+
"counts": cnt,
|
|
111
|
+
"improvements": improvements,
|
|
112
|
+
"details": details
|
|
113
|
+
}
|
|
114
|
+
return result
|
|
115
|
+
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
{
|
|
2
|
+
"hai": "aj",
|
|
3
|
+
"más": "mas̺",
|
|
4
|
+
"mais": "majs̺",
|
|
5
|
+
"alhá": "ɐˈʎa",
|
|
6
|
+
"deimingo": "dejˈmĩ.gʊ",
|
|
7
|
+
"abandono": "a.bɐ̃ˈdo.nu",
|
|
8
|
+
"adbertido": "ɐ.dbɨɾˈti.du",
|
|
9
|
+
"adulto": "ɐˈdul.tu",
|
|
10
|
+
"afamado": "ɐ.fɐˈma.du",
|
|
11
|
+
"afeito": "ɐˈfej.tʊ",
|
|
12
|
+
"afelhado": "ɐ.fɨˈʎa.du",
|
|
13
|
+
"alternatibo": "al.tɨɾ.nɐˈti.bu",
|
|
14
|
+
"amarielho": "ɐ.mɐˈɾjɛ.ʎu",
|
|
15
|
+
"ambesible": "ɐ̃.bɨˈs̺i.blɨ",
|
|
16
|
+
"amouchado": "amowˈtʃaðu",
|
|
17
|
+
"amportante": "ɐ̃.puɾˈtɐ̃.tɨ",
|
|
18
|
+
"ampossible": "ɐ̃.puˈsi.blɨ",
|
|
19
|
+
"ampressionante": "ɐ̃.pɾɨ.sjuˈnɐ̃.tɨ",
|
|
20
|
+
"anchir": "ɐ̃.ˈtʃiɾ",
|
|
21
|
+
"antender": "ɐ̃.tɨ̃.ˈdeɾ",
|
|
22
|
+
"arena": "ɐˈɾenɐ",
|
|
23
|
+
"açpuis": "ɐsˈpujs̺",
|
|
24
|
+
"berde": "ˈveɾ.dɨ",
|
|
25
|
+
"besible": "bɨˈz̺i.blɨ",
|
|
26
|
+
"bexanar": "bɨ.ʃɐ.ˈnaɾ",
|
|
27
|
+
"bibal": "bi.ˈβaɫ",
|
|
28
|
+
"bielho": "bjɛʎu",
|
|
29
|
+
"biolento": "bjuˈlẽ.tu",
|
|
30
|
+
"biúba": "biˈuβɐ",
|
|
31
|
+
"brabo": "bɾa.bu",
|
|
32
|
+
"branco": "bɾɐ̃.ku",
|
|
33
|
+
"buono": "bwo.nu",
|
|
34
|
+
"burmeilho": "buɾˈmɐj.ʎu",
|
|
35
|
+
"bíblico": "bi.bli.ku",
|
|
36
|
+
"cabresto": "kɐˈbɾeʃ.tu",
|
|
37
|
+
"canhona": "kɐˈɲo.nɐ",
|
|
38
|
+
"cheno": "ˈtʃe.nu",
|
|
39
|
+
"chober": "tʃuˈβeɾ",
|
|
40
|
+
"ciguonha": "s̻i.ˈɣwo.ɲɐ",
|
|
41
|
+
"cul": "kul",
|
|
42
|
+
"dafeito": "ðɐˈfej.tʊ",
|
|
43
|
+
"defrente": "dɨˈfɾẽ.tɨ",
|
|
44
|
+
"defícel": "dɨˈfi.sɛl",
|
|
45
|
+
"drento": "ˈdɾẽtu",
|
|
46
|
+
"eigual": "ɐjˈɡwal",
|
|
47
|
+
"era": "ˈɛ.ɾɐ",
|
|
48
|
+
"eras": "ˈɛ.ɾɐs̺",
|
|
49
|
+
"feliç": "fɨˈlis̻",
|
|
50
|
+
"fierro": "ˈfjɛ.ru",
|
|
51
|
+
"francesa": "fɾɐ̃ˈsɛ.zɐ",
|
|
52
|
+
"francesas": "fɾɐ̃ˈsɛ.zɐs̺",
|
|
53
|
+
"franceses": "fɾɐ̃ˈsɛ.zɨs̺",
|
|
54
|
+
"francés": "fɾɐ̃ˈsɛs̺",
|
|
55
|
+
"fui": "fuj",
|
|
56
|
+
"fumos": "ˈfu.mus̺",
|
|
57
|
+
"fuogo": "fwo.ɣʊ",
|
|
58
|
+
"fuonte": "ˈfwõ.tɨ",
|
|
59
|
+
"fuorte": "ˈfwɔɾ.tɨ",
|
|
60
|
+
"fuortemente": "fwɔɾ.tɨˈmẽ.tɨ",
|
|
61
|
+
"fuorça": "ˈfwɔɾ.s̻ɐ",
|
|
62
|
+
"fuste": "ˈfus̺.tɨ",
|
|
63
|
+
"fácele": "ˈfa.sɨ.lɨ",
|
|
64
|
+
"guapo": "ˈɡwa.pu",
|
|
65
|
+
"haber": "ɐˈβeɾ",
|
|
66
|
+
"houmano": "o(w)ˈmɐ.nu",
|
|
67
|
+
"i": "i",
|
|
68
|
+
"l": "l̩",
|
|
69
|
+
"lhabrar": "ʎɐˈbɾaɾ(i)",
|
|
70
|
+
"lhimpo": "ˈʎĩ.pʊ",
|
|
71
|
+
"lhobo": "ˈʎo.bʊ",
|
|
72
|
+
"lhuç": "ˈʎus̻",
|
|
73
|
+
"lhéngua": "ˈʎɛ̃ɡwɐ",
|
|
74
|
+
"luç": "ˈʎus̻",
|
|
75
|
+
"macado": "mɐˈka.du",
|
|
76
|
+
"maias": "ˈmajɐs̺",
|
|
77
|
+
"mirandés": "mi.ɾɐ̃ˈdes̺",
|
|
78
|
+
"molineiro": "mʊ.li.ˈnei̯.rʊ",
|
|
79
|
+
"molino": "muˈlinu",
|
|
80
|
+
"muola": "ˈmu̯olɐ",
|
|
81
|
+
"ne l": "nɨl",
|
|
82
|
+
"neçairo": "nɨˈsaj.ɾu",
|
|
83
|
+
"nuobo": "ˈnwo.βʊ",
|
|
84
|
+
"nó": "ˈnɔ",
|
|
85
|
+
"onte": "ˈõ.tɨ",
|
|
86
|
+
"oucidental": "ow.s̻i.dẽˈtal",
|
|
87
|
+
"oufecialmente": "o(w).fɨˌsjalˈmẽ.tɨ",
|
|
88
|
+
"ourdenhar": "ou̯ɾdɨˈɲaɾ",
|
|
89
|
+
"oureginal": "ow.ɾɨ.ʒiˈnal",
|
|
90
|
+
"ourganizaçon": "ou̯r.ɡɐ.ni.zɐ.ˈsõ",
|
|
91
|
+
"ouropeu": "ow.ɾuˈpew",
|
|
92
|
+
"ourriêta": "ˈowrjetɐ",
|
|
93
|
+
"paxarina": "pɐʃɐˈɾinɐ",
|
|
94
|
+
"pequeinho": "pɨˈkɐi.ɲu",
|
|
95
|
+
"piranha": "piˈra.ɲɐ",
|
|
96
|
+
"puis": "ˈpujs̺",
|
|
97
|
+
"pul": "ˈpul",
|
|
98
|
+
"puorta": "ˈpwoɾtɐ",
|
|
99
|
+
"purmeiro": "puɾˈmɐj.ɾu",
|
|
100
|
+
"quaije": "ˈkwaj.ʒɨ",
|
|
101
|
+
"quando": "ˈkwɐ̃.du",
|
|
102
|
+
"quelobrinas": "kɨluˈbrinas̺",
|
|
103
|
+
"quemun": "kɨˈmun",
|
|
104
|
+
"rabielho": "rɐ.ˈβje.ʎu",
|
|
105
|
+
"rico": "ˈri.ku",
|
|
106
|
+
"salir": "s̺ɐˈliɾ",
|
|
107
|
+
"screbir": "s̺krɨˈβiɾ",
|
|
108
|
+
"segar": "s̺ɨˈɣaɾ",
|
|
109
|
+
"sendo": "ˈsẽ.du",
|
|
110
|
+
"ser": "ˈseɾ",
|
|
111
|
+
"sida": "ˈsi.dɐ",
|
|
112
|
+
"sidas": "ˈsi.dɐs̺",
|
|
113
|
+
"sido": "ˈsi.du",
|
|
114
|
+
"sidos": "ˈsi.dus̺",
|
|
115
|
+
"simple": "ˈs̺ĩ.plɨ",
|
|
116
|
+
"sobrino": "s̺uˈbɾinu",
|
|
117
|
+
"sodes": "ˈso.dɨs̺",
|
|
118
|
+
"somos": "ˈso.mus̺",
|
|
119
|
+
"son": "ˈsõ",
|
|
120
|
+
"sou": "ˈso(w)",
|
|
121
|
+
"spanha": "ˈs̺pɐ.ɲɐ",
|
|
122
|
+
"squierdo": "ˈs̺kjeɾ.du",
|
|
123
|
+
"sós": "ˈs̺ɔs̺",
|
|
124
|
+
"talbeç": "talˈbes",
|
|
125
|
+
"tamien": "tɐˈmjẽ",
|
|
126
|
+
"tascar": "tɐs̺.ˈkaɾ",
|
|
127
|
+
"tener": "tɨˈneɾ",
|
|
128
|
+
"trasdonte": "ˈtɾɐz̺dõtɨ",
|
|
129
|
+
"trasdontonte": "ˈtɾɐz̺dõtõtɨ",
|
|
130
|
+
"ye": "ˈje",
|
|
131
|
+
"you": "jow",
|
|
132
|
+
"yê": "ˈje",
|
|
133
|
+
"zastre": "ˈzas̺tɾɨ",
|
|
134
|
+
"zeigual": "zɐjˈɡwal",
|
|
135
|
+
"zenhar": "zɨˈɲaɾ",
|
|
136
|
+
"áfrica": "ˈafɾikɐ",
|
|
137
|
+
"çcansar": "skɐ̃ˈs̺aɾ",
|
|
138
|
+
"çcrebir": "skɾɨˈβiɾ",
|
|
139
|
+
"çcriçon": "skɾiˈsõ",
|
|
140
|
+
"çtinto": "ˈstĩ.tu",
|
|
141
|
+
"érades": "ˈɛ.ɾɐ.dɨs̺",
|
|
142
|
+
"éramos": "ˈɛ.ɾɐ.mus̺",
|
|
143
|
+
"éran": "ˈɛ.ɾɐn",
|
|
144
|
+
"ũ": "ˈũ",
|
|
145
|
+
"ũa": "ˈũ.ŋɐ",
|
|
146
|
+
"ua": "ˈũ.ŋɐ"
|
|
147
|
+
}
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
from mwl_phonemizer.base import MirandesePhonemizer, Dialects
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class CRFPhonemizer(MirandesePhonemizer):
|
|
5
|
+
def __init__(self, crf_model_path: str | None = None, *args, **kwargs):
|
|
6
|
+
super().__init__(*args, **kwargs)
|
|
7
|
+
self.crf_model_path = crf_model_path
|
|
8
|
+
self.model = None
|
|
9
|
+
if crf_model_path and os.path.exists(crf_model_path):
|
|
10
|
+
self.load_model(crf_model_path)
|
|
11
|
+
else:
|
|
12
|
+
# Prepare training data from GOLD dictionary
|
|
13
|
+
train_data = list(self.GOLD.items()) # [(word, ipa), ...]
|
|
14
|
+
# Train CRF
|
|
15
|
+
self.train_crf(train_data)
|
|
16
|
+
|
|
17
|
+
def _word_to_features(self, word):
|
|
18
|
+
# Simple character-level features for CRF
|
|
19
|
+
features = []
|
|
20
|
+
for i, char in enumerate(word):
|
|
21
|
+
feats = {
|
|
22
|
+
'char': char,
|
|
23
|
+
'is_first': i == 0,
|
|
24
|
+
'is_last': i == len(word) - 1,
|
|
25
|
+
'prev_char': '' if i == 0 else word[i - 1],
|
|
26
|
+
'next_char': '' if i == len(word) - 1 else word[i + 1],
|
|
27
|
+
'is_upper': char.isupper(),
|
|
28
|
+
'is_digit': char.isdigit()
|
|
29
|
+
}
|
|
30
|
+
features.append(feats)
|
|
31
|
+
return features
|
|
32
|
+
|
|
33
|
+
def train_crf(self, train_data):
|
|
34
|
+
# train_data: list of (word, ipa) pairs
|
|
35
|
+
X, y = [], []
|
|
36
|
+
for word, ipa in train_data:
|
|
37
|
+
if len(word) != len(ipa):
|
|
38
|
+
# If word and IPA lengths differ, use character-level alignment with padding
|
|
39
|
+
# This is a simple heuristic: repeat last IPA to match word length
|
|
40
|
+
ipa_aligned = list(ipa)
|
|
41
|
+
while len(ipa_aligned) < len(word):
|
|
42
|
+
ipa_aligned.append(ipa_aligned[-1])
|
|
43
|
+
ipa_aligned = ipa_aligned[:len(word)]
|
|
44
|
+
else:
|
|
45
|
+
ipa_aligned = list(ipa)
|
|
46
|
+
X.append(self._word_to_features(word))
|
|
47
|
+
y.append(ipa_aligned)
|
|
48
|
+
|
|
49
|
+
import sklearn_crfsuite # imported here so it's optional
|
|
50
|
+
self.model = sklearn_crfsuite.CRF(
|
|
51
|
+
algorithm='lbfgs',
|
|
52
|
+
c1=0.1,
|
|
53
|
+
c2=0.1,
|
|
54
|
+
max_iterations=100,
|
|
55
|
+
all_possible_transitions=True
|
|
56
|
+
)
|
|
57
|
+
self.model.fit(X, y)
|
|
58
|
+
|
|
59
|
+
if self.crf_model_path:
|
|
60
|
+
self.save_model(self.crf_model_path)
|
|
61
|
+
|
|
62
|
+
def phonemize(self, word: str, lookup_word: bool = True) -> str:
|
|
63
|
+
if lookup_word and word.lower() in self.GOLD:
|
|
64
|
+
return self.GOLD[word.lower()]
|
|
65
|
+
if not self.model:
|
|
66
|
+
raise ValueError("CRF model is not trained or loaded.")
|
|
67
|
+
features = self._word_to_features(word)
|
|
68
|
+
pred = self.model.predict_single(features)
|
|
69
|
+
return ''.join(pred)
|
|
70
|
+
|
|
71
|
+
def save_model(self, path: str):
|
|
72
|
+
import joblib
|
|
73
|
+
joblib.dump(self.model, path)
|
|
74
|
+
|
|
75
|
+
def load_model(self, path: str):
|
|
76
|
+
import joblib
|
|
77
|
+
self.model = joblib.load(path)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
if __name__ == "__main__":
|
|
81
|
+
phonemizer = CRFPhonemizer(dialect=Dialects.CENTRAL)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# Evaluate on the same data (overfitting expected due to small dataset)
|
|
86
|
+
stats = phonemizer.evaluate_on_gold(limit=None, detailed=False, show_changes=False)
|
|
87
|
+
|
|
88
|
+
# --- Compute PER (Phoneme Error Rate) --- # TODO - move this to evaluate_on_gold
|
|
89
|
+
total_ref_len_stress = sum(len(v) for v in phonemizer.GOLD.values())
|
|
90
|
+
total_ref_len_no_stress = sum(len(phonemizer.strip_stress(v)) for v in phonemizer.GOLD.values())
|
|
91
|
+
|
|
92
|
+
per = stats['avg_edit_distance'] * stats['counts'] / total_ref_len_stress
|
|
93
|
+
|
|
94
|
+
per_no_stress = stats['avg_edit_distance_no_stress'] * stats['counts'] / total_ref_len_no_stress
|
|
95
|
+
|
|
96
|
+
# --- Print Summary Metrics ---
|
|
97
|
+
print("\n" + "=" * 50)
|
|
98
|
+
print(" Mirandese Phonemizer Rule Evaluation")
|
|
99
|
+
print("=" * 50)
|
|
100
|
+
print(f"Total Words Evaluated: {stats['counts']}\n")
|
|
101
|
+
|
|
102
|
+
print("## Phoneme Error Rate (PER, Full IPA Match, includes stress)")
|
|
103
|
+
print(f"PER: {per:.2%}")
|
|
104
|
+
|
|
105
|
+
print("\n## Phoneme Error Rate (PER, Stress-Agnostic)")
|
|
106
|
+
print(f"PER: {per_no_stress:.2%}")
|
|
107
|
+
|
|
108
|
+
# --- Print only 'wrong' words (ED > 0) ---
|
|
109
|
+
print("\n--- Incorrectly Phonemized Words (Full IPA Match ED > 0) ---")
|
|
110
|
+
wrong_words = stats.get("details", [])
|
|
111
|
+
|
|
112
|
+
if wrong_words:
|
|
113
|
+
print(f"Total Incorrect: {len(wrong_words)} words\n")
|
|
114
|
+
|
|
115
|
+
# Print a header for the detailed list
|
|
116
|
+
print(f"{'Word':<20} | {'Gold':<15} | {'Phonemized':<15} | {'ED After':<8}")
|
|
117
|
+
print("-" * 75)
|
|
118
|
+
|
|
119
|
+
# Print the detailed list
|
|
120
|
+
for d in wrong_words:
|
|
121
|
+
print(
|
|
122
|
+
f"{d['word']:<20} | {d['gold']:<15} | {d['phonemes']:<15} | {d['ed']:<8}")
|
|
123
|
+
else:
|
|
124
|
+
print("All words achieved an exact match (100% Accuracy)!")
|
|
125
|
+
|
|
126
|
+
# ==================================================
|
|
127
|
+
# Mirandese Phonemizer Rule Evaluation
|
|
128
|
+
# ==================================================
|
|
129
|
+
# Total Words Evaluated: 145
|
|
130
|
+
#
|
|
131
|
+
# ## Phoneme Error Rate (PER, Full IPA Match, includes stress)
|
|
132
|
+
# PER: 20.25%
|
|
133
|
+
#
|
|
134
|
+
# ## Phoneme Error Rate (PER, Stress-Agnostic)
|
|
135
|
+
# PER: 20.76%
|
|
136
|
+
#
|
|
137
|
+
# --- Incorrectly Phonemized Words (Full IPA Match ED > 0) ---
|
|
138
|
+
# Total Incorrect: 117 words
|
|
139
|
+
#
|
|
140
|
+
# Word | Gold | Phonemized | ED After
|
|
141
|
+
# ---------------------------------------------------------------------------
|
|
142
|
+
# hai | aj | ɐˈj | 2
|
|
143
|
+
# más | mas̺ | mas | 1
|
|
144
|
+
# mais | majs̺ | ˈmaj | 3
|
|
145
|
+
# deimingo | dejˈmĩgʊ | ˈdɨˈmĩgʊ | 3
|
|
146
|
+
# abandono | abɐ̃ˈdonu | abɐ̃ˈdon | 1
|
|
147
|
+
# adbertido | ɐdbɨɾˈtidu | ɐdbɨɾˈtid | 1
|
|
148
|
+
# adulto | ɐˈdultu | ɐˈdult | 1
|
|
149
|
+
# afamado | ɐfɐˈmadu | ɐfɐˈmad | 1
|
|
150
|
+
# afeito | ɐˈfejtʊ | ɐˈfejt | 1
|
|
151
|
+
# alternatibo | altɨɾnɐˈtibu | altɨˈnɐˈtib | 2
|
|
152
|
+
# ambesible | ɐ̃bɨˈs̺iblɨ | ɐ̃bɨˈs̺ib | 2
|
|
153
|
+
# amouchado | amowˈtʃaðu | ɐmowˈtʃad | 3
|
|
154
|
+
# amportante | ɐ̃puɾˈtɐ̃tɨ | ɐ̃puɾtɐ̃tɨ | 1
|
|
155
|
+
# ampressionante | ɐ̃pɾɨsjuˈnɐ̃tɨ | ɐ̃pɾɨˈs̺inɐ̃tɨ | 4
|
|
156
|
+
# anchir | ɐ̃ˈtʃiɾ | ɐ̃ˈtʃi | 1
|
|
157
|
+
# antender | ɐ̃tɨ̃ˈdeɾ | ɐ̃tɨ̃ˈde | 1
|
|
158
|
+
# arena | ɐˈɾenɐ | ɐˈɾen | 1
|
|
159
|
+
# açpuis | ɐsˈpujs̺ | ɐ̃ˈpuj | 3
|
|
160
|
+
# berde | ˈveɾdɨ | ˈβeɾd | 2
|
|
161
|
+
# besible | bɨˈz̺iblɨ | bɨˈs̺ib | 3
|
|
162
|
+
# bexanar | bɨʃɐˈnaɾ | bɨʃɐˈna | 1
|
|
163
|
+
# bibal | biˈβaɫ | biˈβa | 1
|
|
164
|
+
# bielho | bjɛʎu | ˈβjɛʎu | 2
|
|
165
|
+
# biúba | biˈuβɐ | biˈbɐ | 2
|
|
166
|
+
# burmeilho | buɾˈmɐjʎu | buɾˈmɐˈʎu | 1
|
|
167
|
+
# cabresto | kɐˈbɾeʃtu | kɐˈbɾeʃt | 1
|
|
168
|
+
# cheno | ˈtʃenu | ˈtʃen | 1
|
|
169
|
+
# chober | tʃuˈβeɾ | tʃuˈβe | 1
|
|
170
|
+
# ciguonha | s̻iˈɣwoɲɐ | s̻iˈɣwoɲ | 1
|
|
171
|
+
# dafeito | ðɐˈfejtʊ | ðɐˈfejt | 1
|
|
172
|
+
# defícel | dɨˈfisɛl | dɨˈfisɛ | 1
|
|
173
|
+
# eigual | ɐjˈɡwal | ɐjˈɡwa | 1
|
|
174
|
+
# era | ˈɛɾɐ | ˈɛɾ | 1
|
|
175
|
+
# eras | ˈɛɾɐs̺ | ˈɛɾɐ | 2
|
|
176
|
+
# feliç | fɨˈlis̻ | fɨˈli | 2
|
|
177
|
+
# fierro | ˈfjɛru | ˈfjeɾu | 2
|
|
178
|
+
# francesa | fɾɐ̃ˈsɛzɐ | fɾɐ̃ˈsɛz | 1
|
|
179
|
+
# francesas | fɾɐ̃ˈsɛzɐs̺ | fɾɐ̃ˈsɛzɐ | 2
|
|
180
|
+
# franceses | fɾɐ̃ˈsɛzɨs̺ | fɾɐ̃ˈsɛzɨ | 2
|
|
181
|
+
# francés | fɾɐ̃ˈsɛs̺ | fɾɐ̃ˈsɛ | 2
|
|
182
|
+
# fui | fuj | ˈfi | 3
|
|
183
|
+
# fumos | ˈfumus̺ | ˈfumu | 2
|
|
184
|
+
# fuorte | ˈfwɔɾtɨ | ˈfwɔɾt | 1
|
|
185
|
+
# fuortemente | fwɔɾtɨˈmẽtɨ | ˈfuɾtɨˈmẽtɨ | 3
|
|
186
|
+
# fuorça | ˈfwɔɾs̻ɐ | ˈfwɔɾɐ | 2
|
|
187
|
+
# fuste | ˈfus̺tɨ | ˈfus̺ | 2
|
|
188
|
+
# fácele | ˈfasɨlɨ | ˈfasɨl | 1
|
|
189
|
+
# guapo | ˈɡwapu | ˈɡwap | 1
|
|
190
|
+
# haber | ɐˈβeɾ | ɐˈbɨɾ | 2
|
|
191
|
+
# l | l̩ | l | 1
|
|
192
|
+
# lhabrar | ʎɐˈbɾaɾi | ˈʎabɾaɾ | 4
|
|
193
|
+
# lhimpo | ˈʎĩpʊ | ˈʎĩpʊʊ | 1
|
|
194
|
+
# lhobo | ˈʎobʊ | ˈʎobu | 1
|
|
195
|
+
# lhuç | ˈʎus̻ | ˈʎus | 1
|
|
196
|
+
# luç | ˈʎus̻ | ˈʎu | 2
|
|
197
|
+
# macado | mɐˈkadu | mɐˈkad | 1
|
|
198
|
+
# maias | ˈmajɐs̺ | ˈmajɐ | 2
|
|
199
|
+
# mirandés | miɾɐ̃ˈdes̺ | miɾɐ̃ˈdu | 3
|
|
200
|
+
# molineiro | mʊliˈnei̯rʊ | mʊliˈnejɾ | 4
|
|
201
|
+
# molino | muˈlinu | muˈlin | 1
|
|
202
|
+
# muola | ˈmu̯olɐ | ˈmuˈl | 3
|
|
203
|
+
# ne l | nɨl | nɨll | 1
|
|
204
|
+
# neçairo | nɨˈsajɾu | nɨˈsajɾ | 1
|
|
205
|
+
# nuobo | ˈnwoβʊ | ˈnwoβ | 1
|
|
206
|
+
# nó | ˈnɔ | ˈn | 1
|
|
207
|
+
# oucidental | ows̻idẽˈtal | ows̻idɨˈta | 2
|
|
208
|
+
# oufecialmente | owfɨˌsjalˈmẽtɨ | owfɨˈs̺ɐˈmẽtɨ | 4
|
|
209
|
+
# ourdenhar | ou̯ɾdɨˈɲaɾ | owɔɾdɨˈɲa | 3
|
|
210
|
+
# oureginal | owɾɨʒiˈnal | owɾɨʒiˈna | 1
|
|
211
|
+
# ourganizaçon | ou̯rɡɐnizɐˈsõ | ou̯rɡɐnizɐˈn | 2
|
|
212
|
+
# ouropeu | owɾuˈpew | owɾuˈpe | 1
|
|
213
|
+
# ourriêta | ˈowrjetɐ | owˈrjetɐ | 2
|
|
214
|
+
# paxarina | pɐʃɐˈɾinɐ | pɐʃɐˈɾin | 1
|
|
215
|
+
# pequeinho | pɨˈkɐiɲu | pɨˈkɐjˈɲu | 2
|
|
216
|
+
# piranha | piˈraɲɐ | piˈɾɐˈɲ | 4
|
|
217
|
+
# puis | ˈpujs̺ | ˈpuj | 2
|
|
218
|
+
# pul | ˈpul | ˈpu | 1
|
|
219
|
+
# puorta | ˈpwoɾtɐ | ˈpwɔɾt | 2
|
|
220
|
+
# purmeiro | puɾˈmɐjɾu | puɾˈmɐjɾ | 1
|
|
221
|
+
# quaije | ˈkwajʒɨ | ˈkwajʒ | 1
|
|
222
|
+
# quando | ˈkwɐ̃du | ˈkɐ̃ˈd | 3
|
|
223
|
+
# quelobrinas | kɨluˈbrinas̺ | kɨluˈbɾiˈnɐ | 5
|
|
224
|
+
# rabielho | rɐˈβjeʎu | ɾɐˈβjɛʎu | 2
|
|
225
|
+
# rico | ˈriku | ˈrik | 1
|
|
226
|
+
# salir | s̺ɐˈliɾ | ˈsali | 5
|
|
227
|
+
# screbir | s̺krɨˈβiɾ | s̺kɨˈβi | 2
|
|
228
|
+
# segar | s̺ɨˈɣaɾ | s̺ɨˈɣ | 2
|
|
229
|
+
# ser | ˈseɾ | ˈse | 1
|
|
230
|
+
# sida | ˈsidɐ | ˈsid | 1
|
|
231
|
+
# sidas | ˈsidɐs̺ | ˈsidɐ | 2
|
|
232
|
+
# sido | ˈsidu | ˈsid | 1
|
|
233
|
+
# sidos | ˈsidus̺ | ˈsidu | 2
|
|
234
|
+
# simple | ˈs̺ĩplɨ | ˈs̺ĩpl | 1
|
|
235
|
+
# sobrino | s̺uˈbɾinu | s̺uˈɾin | 2
|
|
236
|
+
# sodes | ˈsodɨs̺ | ˈsodɨ | 2
|
|
237
|
+
# somos | ˈsomus̺ | ˈsomu | 2
|
|
238
|
+
# sou | ˈsow | ˈso | 1
|
|
239
|
+
# spanha | ˈs̺pɐɲɐ | ˈs̺ɐˈɲ | 3
|
|
240
|
+
# squierdo | ˈs̺kjeɾdu | ˈs̺kjeɾd | 1
|
|
241
|
+
# sós | ˈs̺ɔs̺ | ˈs̺ | 3
|
|
242
|
+
# talbeç | talˈbes | talˈbe | 1
|
|
243
|
+
# tascar | tɐs̺ˈkaɾ | tas̺ka | 3
|
|
244
|
+
# tener | tɨˈneɾ | tɨˈne | 1
|
|
245
|
+
# trasdonte | ˈtɾɐz̺dõtɨ | ˈtɾɐz̺dõt | 1
|
|
246
|
+
# ye | ˈje | ˈj | 1
|
|
247
|
+
# yê | ˈje | ˈj | 1
|
|
248
|
+
# zastre | ˈzas̺tɾɨ | ˈzɐs̺ɨ | 3
|
|
249
|
+
# zeigual | zɐjˈɡwal | zɐjˈɡwa | 1
|
|
250
|
+
# áfrica | ˈafɾikɐ | ˈafɾik | 1
|
|
251
|
+
# çcansar | skɐ̃ˈs̺aɾ | skɐ̃ˈs̺ | 2
|
|
252
|
+
# çcrebir | skɾɨˈβiɾ | skɾɨˈβi | 1
|
|
253
|
+
# érades | ˈɛɾɐdɨs̺ | ˈɛɾɐdɨ | 2
|
|
254
|
+
# éramos | ˈɛɾɐmus̺ | ˈɛɾɐmu | 2
|
|
255
|
+
# éran | ˈɛɾɐn | ˈɛɾɐ | 1
|
|
256
|
+
# ũ | ˈũ | ˈ | 1
|
|
257
|
+
# ũa | ˈũŋɐ | ˈũ | 2
|
|
258
|
+
# ua | ˈũŋɐ | ˈũ | 2
|