phonematcher 0.0.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phonematcher/__init__.py +3 -0
- phonematcher/clustering.py +135 -0
- phonematcher/data/features.json +6878 -0
- phonematcher/data/overrides.json +329 -0
- phonematcher/distance.py +87 -0
- phonematcher/features.py +145 -0
- phonematcher/phonex.py +15 -0
- phonematcher/scales.py +47 -0
- phonematcher/version.py +10 -0
- phonematcher-0.0.0a2.dist-info/METADATA +67 -0
- phonematcher-0.0.0a2.dist-info/RECORD +14 -0
- phonematcher-0.0.0a2.dist-info/WHEEL +5 -0
- phonematcher-0.0.0a2.dist-info/licenses/LICENSE +22 -0
- phonematcher-0.0.0a2.dist-info/top_level.txt +1 -0
phonematcher/__init__.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Phonetic fuzzy search over IPA.
|
|
2
|
+
|
|
3
|
+
Phones are grouped into clusters of near sounds (UPGMA average linkage over
|
|
4
|
+
:func:`~phonematcher.features.phonetic_distance`). Each word of a term is indexed by its cluster
|
|
5
|
+
sequence and by every sequence with one or two clusters deleted, so a query finds terms that differ
|
|
6
|
+
by near sounds or by a dropped sound. Candidates are ranked by the feature-weighted edit distance
|
|
7
|
+
of :mod:`phonematcher.distance`.
|
|
8
|
+
|
|
9
|
+
Terms and queries are IPA. Use a grapheme-to-IPA library such as orthography2ipa to get IPA from
|
|
10
|
+
text.
|
|
11
|
+
|
|
12
|
+
Adapted from pyphone (https://github.com/lingz/pyphone) and fast_fuzzy_search
|
|
13
|
+
(https://github.com/lingz/fast_fuzzy_search), MIT License.
|
|
14
|
+
"""
|
|
15
|
+
import unicodedata
|
|
16
|
+
from collections import namedtuple
|
|
17
|
+
from functools import lru_cache
|
|
18
|
+
from typing import Dict, Hashable, Iterable, List, Sequence, Set, Tuple, Union
|
|
19
|
+
|
|
20
|
+
from phonematcher.distance import edit_distance, segments
|
|
21
|
+
from phonematcher.features import phone_features, phonetic_distance
|
|
22
|
+
from phonematcher.phonex import phonex
|
|
23
|
+
|
|
24
|
+
#: A search result: the term's id, its readings, and its edit distance to the query.
|
|
25
|
+
Match = namedtuple("Match", ["id", "term", "score"])
|
|
26
|
+
|
|
27
|
+
Readings = Union[str, Sequence[str]]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def upgma(labels: Sequence[str], threshold: float) -> Dict[str, int]:
|
|
31
|
+
"""Group *labels* by UPGMA average linkage, merging while the closest pair of groups is at most
|
|
32
|
+
*threshold* apart. Returns ``label -> cluster id``."""
|
|
33
|
+
dist = {(a, b): phonetic_distance(a, b) for a in labels for b in labels if a < b}
|
|
34
|
+
|
|
35
|
+
def between(c1: Set[str], c2: Set[str]) -> float:
|
|
36
|
+
return sum(dist[min(a, b), max(a, b)] for a in c1 for b in c2) / (len(c1) * len(c2))
|
|
37
|
+
|
|
38
|
+
clusters = [{label} for label in labels]
|
|
39
|
+
while len(clusters) > 1:
|
|
40
|
+
d, i, j = min((between(clusters[i], clusters[j]), i, j)
|
|
41
|
+
for i in range(len(clusters)) for j in range(i + 1, len(clusters)))
|
|
42
|
+
if d > threshold:
|
|
43
|
+
break
|
|
44
|
+
clusters[i] |= clusters.pop(j)
|
|
45
|
+
return {label: cid for cid, cluster in enumerate(clusters, start=1) for label in cluster}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@lru_cache(maxsize=8)
|
|
49
|
+
def phone_clusters(threshold: float) -> Dict[str, int]:
|
|
50
|
+
"""The clusters of the IPA letters and tied affricates in the table at *threshold*.
|
|
51
|
+
|
|
52
|
+
Any other segment, such as a letter with diacritics, takes the cluster of its nearest phone.
|
|
53
|
+
"""
|
|
54
|
+
phones = [p for p in phone_features
|
|
55
|
+
if (len(p) == 1 and unicodedata.category(p).startswith("L"))
|
|
56
|
+
or (len(p) == 3 and p[1] in "\u0361\u035c")]
|
|
57
|
+
return upgma(sorted(phones), threshold)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _as_readings(term: Readings) -> Tuple[str, ...]:
|
|
61
|
+
return (term,) if isinstance(term, str) else tuple(term)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class PhoneticFuzzySearch:
|
|
65
|
+
"""An index of IPA terms, searched by sound.
|
|
66
|
+
|
|
67
|
+
A term is one IPA string or several readings of the same item; a match on any reading counts.
|
|
68
|
+
*cluster_sensitivity* is the UPGMA merge threshold: higher values put more phones in one
|
|
69
|
+
cluster, which finds more candidates.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
def __init__(self, cluster_sensitivity: float = 0.1):
|
|
73
|
+
self.cluster_sensitivity = cluster_sensitivity
|
|
74
|
+
self.clusters = phone_clusters(cluster_sensitivity)
|
|
75
|
+
self.index: Dict[Tuple[int, ...], Set[Hashable]] = {}
|
|
76
|
+
self.library: Dict[Hashable, Tuple[str, ...]] = {}
|
|
77
|
+
self._cluster_memo: Dict[str, int] = {}
|
|
78
|
+
|
|
79
|
+
def cluster_of(self, segment: str) -> int:
|
|
80
|
+
"""The cluster of *segment*; a segment outside the table takes its nearest phone's."""
|
|
81
|
+
if segment in self.clusters:
|
|
82
|
+
return self.clusters[segment]
|
|
83
|
+
if segment not in self._cluster_memo:
|
|
84
|
+
nearest = min(self.clusters, key=lambda p: (phonetic_distance(segment, p), p))
|
|
85
|
+
self._cluster_memo[segment] = self.clusters[nearest]
|
|
86
|
+
return self._cluster_memo[segment]
|
|
87
|
+
|
|
88
|
+
def _words(self, ipa: str) -> List[List[str]]:
|
|
89
|
+
words: List[List[str]] = [[]]
|
|
90
|
+
for seg in segments(ipa):
|
|
91
|
+
if seg == " ":
|
|
92
|
+
words.append([])
|
|
93
|
+
else:
|
|
94
|
+
words[-1].append(seg)
|
|
95
|
+
return [w for w in words if w]
|
|
96
|
+
|
|
97
|
+
def _keys(self, word: Sequence[str]) -> Set[Tuple[int, ...]]:
|
|
98
|
+
key = phonex(word, self.cluster_of)
|
|
99
|
+
one = {key[:i] + key[i + 1:] for i in range(len(key))}
|
|
100
|
+
two = {k[:i] + k[i + 1:] for k in one for i in range(len(k))}
|
|
101
|
+
return {k for k in {key} | one | two if k}
|
|
102
|
+
|
|
103
|
+
def add_term(self, term: Readings, id: Hashable) -> None:
|
|
104
|
+
"""Index *term*, an IPA string or a list of IPA readings, under *id*."""
|
|
105
|
+
readings = _as_readings(term)
|
|
106
|
+
self.library[id] = readings
|
|
107
|
+
for reading in readings:
|
|
108
|
+
for word in self._words(reading):
|
|
109
|
+
for key in self._keys(word):
|
|
110
|
+
self.index.setdefault(key, set()).add(id)
|
|
111
|
+
|
|
112
|
+
def _ids_for_word(self, word: Sequence[str]) -> Set[Hashable]:
|
|
113
|
+
found: Set[Hashable] = set()
|
|
114
|
+
for key in self._keys(word):
|
|
115
|
+
found |= self.index.get(key, set())
|
|
116
|
+
return found
|
|
117
|
+
|
|
118
|
+
def search(self, query: Readings, n_results: int = 10) -> List[Match]:
|
|
119
|
+
"""Terms that sound like *query*, nearest first.
|
|
120
|
+
|
|
121
|
+
A candidate must match every word of at least one query reading. Its score is the smallest
|
|
122
|
+
feature-weighted edit distance between a query reading and a term reading.
|
|
123
|
+
"""
|
|
124
|
+
queries = _as_readings(query)
|
|
125
|
+
candidates: Set[Hashable] = set()
|
|
126
|
+
for q in queries:
|
|
127
|
+
per_word: Iterable[Set[Hashable]] = (self._ids_for_word(w) for w in self._words(q))
|
|
128
|
+
candidates |= set.intersection(*per_word) if self._words(q) else set()
|
|
129
|
+
q_segs = [segments(q) for q in queries]
|
|
130
|
+
matches = []
|
|
131
|
+
for id in candidates:
|
|
132
|
+
score = min(edit_distance(q, segments(t)) for q in q_segs for t in self.library[id])
|
|
133
|
+
matches.append(Match(id, self.library[id], score))
|
|
134
|
+
matches.sort(key=lambda m: (m.score, str(m.id)))
|
|
135
|
+
return matches[:n_results]
|