phonematcher 0.0.0a2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phonematcher-0.0.0a2/LICENSE +22 -0
- phonematcher-0.0.0a2/PKG-INFO +67 -0
- phonematcher-0.0.0a2/README.md +53 -0
- phonematcher-0.0.0a2/phonematcher/__init__.py +3 -0
- phonematcher-0.0.0a2/phonematcher/clustering.py +135 -0
- phonematcher-0.0.0a2/phonematcher/data/features.json +6878 -0
- phonematcher-0.0.0a2/phonematcher/data/overrides.json +329 -0
- phonematcher-0.0.0a2/phonematcher/distance.py +87 -0
- phonematcher-0.0.0a2/phonematcher/features.py +145 -0
- phonematcher-0.0.0a2/phonematcher/phonex.py +15 -0
- phonematcher-0.0.0a2/phonematcher/scales.py +47 -0
- phonematcher-0.0.0a2/phonematcher/version.py +10 -0
- phonematcher-0.0.0a2/phonematcher.egg-info/PKG-INFO +67 -0
- phonematcher-0.0.0a2/phonematcher.egg-info/SOURCES.txt +21 -0
- phonematcher-0.0.0a2/phonematcher.egg-info/dependency_links.txt +1 -0
- phonematcher-0.0.0a2/phonematcher.egg-info/requires.txt +3 -0
- phonematcher-0.0.0a2/phonematcher.egg-info/top_level.txt +1 -0
- phonematcher-0.0.0a2/pyproject.toml +29 -0
- phonematcher-0.0.0a2/setup.cfg +4 -0
- phonematcher-0.0.0a2/tests/test_distance.py +52 -0
- phonematcher-0.0.0a2/tests/test_distance_ordering.py +155 -0
- phonematcher-0.0.0a2/tests/test_features.py +360 -0
- phonematcher-0.0.0a2/tests/test_search.py +66 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) TigreGotico
|
|
4
|
+
Portions copyright 2017 Ling Zhang (pyphone, fast_fuzzy_search)
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
SOFTWARE.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: phonematcher
|
|
3
|
+
Version: 0.0.0a2
|
|
4
|
+
Summary: Distinctive-feature distances between IPA phones, and phonetic fuzzy search over IPA
|
|
5
|
+
Author-email: JarbasAi <jarbasai@mailfence.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Source, https://github.com/TigreGotico/phonematcher
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Provides-Extra: test
|
|
12
|
+
Requires-Dist: pytest>=8; extra == "test"
|
|
13
|
+
Dynamic: license-file
|
|
14
|
+
|
|
15
|
+
# phonematcher
|
|
16
|
+
|
|
17
|
+
phonematcher measures how far apart two IPA sounds are, and finds terms that sound like a query. It works on IPA only. To get IPA from text, use a grapheme-to-IPA library such as [orthography2ipa](https://github.com/TigreGotico/orthography2ipa).
|
|
18
|
+
|
|
19
|
+
Every distance comes from distinctive features. The feature table starts from PanPhon (Mortensen et al. 2016), takes the vowels from the IPA chart, and lists every other change with its source. [docs/features.md](docs/features.md) explains the table.
|
|
20
|
+
|
|
21
|
+
## Install
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
uv pip install phonematcher
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
phonematcher has no runtime dependencies.
|
|
28
|
+
|
|
29
|
+
## Distance between sounds
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from phonematcher import phonetic_distance, ipa_distance, segments
|
|
33
|
+
|
|
34
|
+
phonetic_distance("ʁ", "χ") # 0.013: uvular fricatives that differ only in voice
|
|
35
|
+
phonetic_distance("ʁ", "k") # 0.118: a different manner and place
|
|
36
|
+
segments("ˈt͡ʃɐ̃w̃") # ['t͡ʃ', 'ɐ̃', 'w̃']
|
|
37
|
+
ipa_distance("ˈpojʃ", "ˈpojs") # 0.039: one near substitution
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
`phonetic_distance` compares two segments, from 0 to 1. `ipa_distance` compares two IPA strings: an edit distance where a substitution costs the feature distance and an insertion or deletion costs 1.
|
|
41
|
+
|
|
42
|
+
## Search by sound
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from orthography2ipa import transcribe
|
|
46
|
+
from phonematcher import PhoneticFuzzySearch
|
|
47
|
+
|
|
48
|
+
search = PhoneticFuzzySearch()
|
|
49
|
+
for word in ["pois", "retirada", "carro", "caro", "casa", "dia"]:
|
|
50
|
+
search.add_term(transcribe(word, "pt-PT"), word)
|
|
51
|
+
|
|
52
|
+
query = transcribe("carro", "pt-BR-x-rj") # 'ˈkaχu'
|
|
53
|
+
best = search.search(query)[0]
|
|
54
|
+
best.id, best.term, round(best.score, 3) # ('carro', ('ˈkaʁu',), 0.013)
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
A term may have several readings: `search.add_term(["ˈpojʃ", "ˈpojs"], "pois")`.
|
|
58
|
+
|
|
59
|
+
## Documentation
|
|
60
|
+
|
|
61
|
+
- [docs/quickstart.md](docs/quickstart.md): first steps.
|
|
62
|
+
- [docs/features.md](docs/features.md): the feature table, its sources and the distance.
|
|
63
|
+
- [docs/api.md](docs/api.md): every public name.
|
|
64
|
+
|
|
65
|
+
## Licence
|
|
66
|
+
|
|
67
|
+
MIT. The feature table derives from PanPhon (MIT) and from Hayes (2009) through the CSV conversion by Kyle Gorman (Apache-2.0).
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# phonematcher
|
|
2
|
+
|
|
3
|
+
phonematcher measures how far apart two IPA sounds are, and finds terms that sound like a query. It works on IPA only. To get IPA from text, use a grapheme-to-IPA library such as [orthography2ipa](https://github.com/TigreGotico/orthography2ipa).
|
|
4
|
+
|
|
5
|
+
Every distance comes from distinctive features. The feature table starts from PanPhon (Mortensen et al. 2016), takes the vowels from the IPA chart, and lists every other change with its source. [docs/features.md](docs/features.md) explains the table.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
uv pip install phonematcher
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
phonematcher has no runtime dependencies.
|
|
14
|
+
|
|
15
|
+
## Distance between sounds
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from phonematcher import phonetic_distance, ipa_distance, segments
|
|
19
|
+
|
|
20
|
+
phonetic_distance("ʁ", "χ") # 0.013: uvular fricatives that differ only in voice
|
|
21
|
+
phonetic_distance("ʁ", "k") # 0.118: a different manner and place
|
|
22
|
+
segments("ˈt͡ʃɐ̃w̃") # ['t͡ʃ', 'ɐ̃', 'w̃']
|
|
23
|
+
ipa_distance("ˈpojʃ", "ˈpojs") # 0.039: one near substitution
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
`phonetic_distance` compares two segments, from 0 to 1. `ipa_distance` compares two IPA strings: an edit distance where a substitution costs the feature distance and an insertion or deletion costs 1.
|
|
27
|
+
|
|
28
|
+
## Search by sound
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from orthography2ipa import transcribe
|
|
32
|
+
from phonematcher import PhoneticFuzzySearch
|
|
33
|
+
|
|
34
|
+
search = PhoneticFuzzySearch()
|
|
35
|
+
for word in ["pois", "retirada", "carro", "caro", "casa", "dia"]:
|
|
36
|
+
search.add_term(transcribe(word, "pt-PT"), word)
|
|
37
|
+
|
|
38
|
+
query = transcribe("carro", "pt-BR-x-rj") # 'ˈkaχu'
|
|
39
|
+
best = search.search(query)[0]
|
|
40
|
+
best.id, best.term, round(best.score, 3) # ('carro', ('ˈkaʁu',), 0.013)
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
A term may have several readings: `search.add_term(["ˈpojʃ", "ˈpojs"], "pois")`.
|
|
44
|
+
|
|
45
|
+
## Documentation
|
|
46
|
+
|
|
47
|
+
- [docs/quickstart.md](docs/quickstart.md): first steps.
|
|
48
|
+
- [docs/features.md](docs/features.md): the feature table, its sources and the distance.
|
|
49
|
+
- [docs/api.md](docs/api.md): every public name.
|
|
50
|
+
|
|
51
|
+
## Licence
|
|
52
|
+
|
|
53
|
+
MIT. The feature table derives from PanPhon (MIT) and from Hayes (2009) through the CSV conversion by Kyle Gorman (Apache-2.0).
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Phonetic fuzzy search over IPA.
|
|
2
|
+
|
|
3
|
+
Phones are grouped into clusters of near sounds (UPGMA average linkage over
|
|
4
|
+
:func:`~phonematcher.features.phonetic_distance`). Each word of a term is indexed by its cluster
|
|
5
|
+
sequence and by every sequence with one or two clusters deleted, so a query finds terms that differ
|
|
6
|
+
by near sounds or by a dropped sound. Candidates are ranked by the feature-weighted edit distance
|
|
7
|
+
of :mod:`phonematcher.distance`.
|
|
8
|
+
|
|
9
|
+
Terms and queries are IPA. Use a grapheme-to-IPA library such as orthography2ipa to get IPA from
|
|
10
|
+
text.
|
|
11
|
+
|
|
12
|
+
Adapted from pyphone (https://github.com/lingz/pyphone) and fast_fuzzy_search
|
|
13
|
+
(https://github.com/lingz/fast_fuzzy_search), MIT License.
|
|
14
|
+
"""
|
|
15
|
+
import unicodedata
|
|
16
|
+
from collections import namedtuple
|
|
17
|
+
from functools import lru_cache
|
|
18
|
+
from typing import Dict, Hashable, Iterable, List, Sequence, Set, Tuple, Union
|
|
19
|
+
|
|
20
|
+
from phonematcher.distance import edit_distance, segments
|
|
21
|
+
from phonematcher.features import phone_features, phonetic_distance
|
|
22
|
+
from phonematcher.phonex import phonex
|
|
23
|
+
|
|
24
|
+
#: A search result: the term's id, its readings, and its edit distance to the query.
|
|
25
|
+
Match = namedtuple("Match", ["id", "term", "score"])
|
|
26
|
+
|
|
27
|
+
Readings = Union[str, Sequence[str]]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def upgma(labels: Sequence[str], threshold: float) -> Dict[str, int]:
|
|
31
|
+
"""Group *labels* by UPGMA average linkage, merging while the closest pair of groups is at most
|
|
32
|
+
*threshold* apart. Returns ``label -> cluster id``."""
|
|
33
|
+
dist = {(a, b): phonetic_distance(a, b) for a in labels for b in labels if a < b}
|
|
34
|
+
|
|
35
|
+
def between(c1: Set[str], c2: Set[str]) -> float:
|
|
36
|
+
return sum(dist[min(a, b), max(a, b)] for a in c1 for b in c2) / (len(c1) * len(c2))
|
|
37
|
+
|
|
38
|
+
clusters = [{label} for label in labels]
|
|
39
|
+
while len(clusters) > 1:
|
|
40
|
+
d, i, j = min((between(clusters[i], clusters[j]), i, j)
|
|
41
|
+
for i in range(len(clusters)) for j in range(i + 1, len(clusters)))
|
|
42
|
+
if d > threshold:
|
|
43
|
+
break
|
|
44
|
+
clusters[i] |= clusters.pop(j)
|
|
45
|
+
return {label: cid for cid, cluster in enumerate(clusters, start=1) for label in cluster}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@lru_cache(maxsize=8)
|
|
49
|
+
def phone_clusters(threshold: float) -> Dict[str, int]:
|
|
50
|
+
"""The clusters of the IPA letters and tied affricates in the table at *threshold*.
|
|
51
|
+
|
|
52
|
+
Any other segment, such as a letter with diacritics, takes the cluster of its nearest phone.
|
|
53
|
+
"""
|
|
54
|
+
phones = [p for p in phone_features
|
|
55
|
+
if (len(p) == 1 and unicodedata.category(p).startswith("L"))
|
|
56
|
+
or (len(p) == 3 and p[1] in "\u0361\u035c")]
|
|
57
|
+
return upgma(sorted(phones), threshold)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _as_readings(term: Readings) -> Tuple[str, ...]:
|
|
61
|
+
return (term,) if isinstance(term, str) else tuple(term)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class PhoneticFuzzySearch:
|
|
65
|
+
"""An index of IPA terms, searched by sound.
|
|
66
|
+
|
|
67
|
+
A term is one IPA string or several readings of the same item; a match on any reading counts.
|
|
68
|
+
*cluster_sensitivity* is the UPGMA merge threshold: higher values put more phones in one
|
|
69
|
+
cluster, which finds more candidates.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
def __init__(self, cluster_sensitivity: float = 0.1):
|
|
73
|
+
self.cluster_sensitivity = cluster_sensitivity
|
|
74
|
+
self.clusters = phone_clusters(cluster_sensitivity)
|
|
75
|
+
self.index: Dict[Tuple[int, ...], Set[Hashable]] = {}
|
|
76
|
+
self.library: Dict[Hashable, Tuple[str, ...]] = {}
|
|
77
|
+
self._cluster_memo: Dict[str, int] = {}
|
|
78
|
+
|
|
79
|
+
def cluster_of(self, segment: str) -> int:
|
|
80
|
+
"""The cluster of *segment*; a segment outside the table takes its nearest phone's."""
|
|
81
|
+
if segment in self.clusters:
|
|
82
|
+
return self.clusters[segment]
|
|
83
|
+
if segment not in self._cluster_memo:
|
|
84
|
+
nearest = min(self.clusters, key=lambda p: (phonetic_distance(segment, p), p))
|
|
85
|
+
self._cluster_memo[segment] = self.clusters[nearest]
|
|
86
|
+
return self._cluster_memo[segment]
|
|
87
|
+
|
|
88
|
+
def _words(self, ipa: str) -> List[List[str]]:
|
|
89
|
+
words: List[List[str]] = [[]]
|
|
90
|
+
for seg in segments(ipa):
|
|
91
|
+
if seg == " ":
|
|
92
|
+
words.append([])
|
|
93
|
+
else:
|
|
94
|
+
words[-1].append(seg)
|
|
95
|
+
return [w for w in words if w]
|
|
96
|
+
|
|
97
|
+
def _keys(self, word: Sequence[str]) -> Set[Tuple[int, ...]]:
|
|
98
|
+
key = phonex(word, self.cluster_of)
|
|
99
|
+
one = {key[:i] + key[i + 1:] for i in range(len(key))}
|
|
100
|
+
two = {k[:i] + k[i + 1:] for k in one for i in range(len(k))}
|
|
101
|
+
return {k for k in {key} | one | two if k}
|
|
102
|
+
|
|
103
|
+
def add_term(self, term: Readings, id: Hashable) -> None:
|
|
104
|
+
"""Index *term*, an IPA string or a list of IPA readings, under *id*."""
|
|
105
|
+
readings = _as_readings(term)
|
|
106
|
+
self.library[id] = readings
|
|
107
|
+
for reading in readings:
|
|
108
|
+
for word in self._words(reading):
|
|
109
|
+
for key in self._keys(word):
|
|
110
|
+
self.index.setdefault(key, set()).add(id)
|
|
111
|
+
|
|
112
|
+
def _ids_for_word(self, word: Sequence[str]) -> Set[Hashable]:
|
|
113
|
+
found: Set[Hashable] = set()
|
|
114
|
+
for key in self._keys(word):
|
|
115
|
+
found |= self.index.get(key, set())
|
|
116
|
+
return found
|
|
117
|
+
|
|
118
|
+
def search(self, query: Readings, n_results: int = 10) -> List[Match]:
|
|
119
|
+
"""Terms that sound like *query*, nearest first.
|
|
120
|
+
|
|
121
|
+
A candidate must match every word of at least one query reading. Its score is the smallest
|
|
122
|
+
feature-weighted edit distance between a query reading and a term reading.
|
|
123
|
+
"""
|
|
124
|
+
queries = _as_readings(query)
|
|
125
|
+
candidates: Set[Hashable] = set()
|
|
126
|
+
for q in queries:
|
|
127
|
+
per_word: Iterable[Set[Hashable]] = (self._ids_for_word(w) for w in self._words(q))
|
|
128
|
+
candidates |= set.intersection(*per_word) if self._words(q) else set()
|
|
129
|
+
q_segs = [segments(q) for q in queries]
|
|
130
|
+
matches = []
|
|
131
|
+
for id in candidates:
|
|
132
|
+
score = min(edit_distance(q, segments(t)) for q in q_segs for t in self.library[id])
|
|
133
|
+
matches.append(Match(id, self.library[id], score))
|
|
134
|
+
matches.sort(key=lambda m: (m.score, str(m.id)))
|
|
135
|
+
return matches[:n_results]
|