phonematcher 0.0.0a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ from phonematcher.clustering import Match, PhoneticFuzzySearch
2
+ from phonematcher.distance import edit_distance, ipa_distance, segments
3
+ from phonematcher.features import phonetic_distance, vectorize_phones
@@ -0,0 +1,135 @@
1
+ """Phonetic fuzzy search over IPA.
2
+
3
+ Phones are grouped into clusters of near sounds (UPGMA average linkage over
4
+ :func:`~phonematcher.features.phonetic_distance`). Each word of a term is indexed by its cluster
5
+ sequence and by every sequence with one or two clusters deleted, so a query finds terms that differ
6
+ by near sounds or by a dropped sound. Candidates are ranked by the feature-weighted edit distance
7
+ of :mod:`phonematcher.distance`.
8
+
9
+ Terms and queries are IPA. Use a grapheme-to-IPA library such as orthography2ipa to get IPA from
10
+ text.
11
+
12
+ Adapted from pyphone (https://github.com/lingz/pyphone) and fast_fuzzy_search
13
+ (https://github.com/lingz/fast_fuzzy_search), MIT License.
14
+ """
15
+ import unicodedata
16
+ from collections import namedtuple
17
+ from functools import lru_cache
18
+ from typing import Dict, Hashable, Iterable, List, Sequence, Set, Tuple, Union
19
+
20
+ from phonematcher.distance import edit_distance, segments
21
+ from phonematcher.features import phone_features, phonetic_distance
22
+ from phonematcher.phonex import phonex
23
+
24
+ #: A search result: the term's id, its readings, and its edit distance to the query.
25
+ Match = namedtuple("Match", ["id", "term", "score"])
26
+
27
+ Readings = Union[str, Sequence[str]]
28
+
29
+
30
+ def upgma(labels: Sequence[str], threshold: float) -> Dict[str, int]:
31
+ """Group *labels* by UPGMA average linkage, merging while the closest pair of groups is at most
32
+ *threshold* apart. Returns ``label -> cluster id``."""
33
+ dist = {(a, b): phonetic_distance(a, b) for a in labels for b in labels if a < b}
34
+
35
+ def between(c1: Set[str], c2: Set[str]) -> float:
36
+ return sum(dist[min(a, b), max(a, b)] for a in c1 for b in c2) / (len(c1) * len(c2))
37
+
38
+ clusters = [{label} for label in labels]
39
+ while len(clusters) > 1:
40
+ d, i, j = min((between(clusters[i], clusters[j]), i, j)
41
+ for i in range(len(clusters)) for j in range(i + 1, len(clusters)))
42
+ if d > threshold:
43
+ break
44
+ clusters[i] |= clusters.pop(j)
45
+ return {label: cid for cid, cluster in enumerate(clusters, start=1) for label in cluster}
46
+
47
+
48
+ @lru_cache(maxsize=8)
49
+ def phone_clusters(threshold: float) -> Dict[str, int]:
50
+ """The clusters of the IPA letters and tied affricates in the table at *threshold*.
51
+
52
+ Any other segment, such as a letter with diacritics, takes the cluster of its nearest phone.
53
+ """
54
+ phones = [p for p in phone_features
55
+ if (len(p) == 1 and unicodedata.category(p).startswith("L"))
56
+ or (len(p) == 3 and p[1] in "\u0361\u035c")]
57
+ return upgma(sorted(phones), threshold)
58
+
59
+
60
+ def _as_readings(term: Readings) -> Tuple[str, ...]:
61
+ return (term,) if isinstance(term, str) else tuple(term)
62
+
63
+
64
+ class PhoneticFuzzySearch:
65
+ """An index of IPA terms, searched by sound.
66
+
67
+ A term is one IPA string or several readings of the same item; a match on any reading counts.
68
+ *cluster_sensitivity* is the UPGMA merge threshold: higher values put more phones in one
69
+ cluster, which finds more candidates.
70
+ """
71
+
72
+ def __init__(self, cluster_sensitivity: float = 0.1):
73
+ self.cluster_sensitivity = cluster_sensitivity
74
+ self.clusters = phone_clusters(cluster_sensitivity)
75
+ self.index: Dict[Tuple[int, ...], Set[Hashable]] = {}
76
+ self.library: Dict[Hashable, Tuple[str, ...]] = {}
77
+ self._cluster_memo: Dict[str, int] = {}
78
+
79
+ def cluster_of(self, segment: str) -> int:
80
+ """The cluster of *segment*; a segment outside the table takes its nearest phone's."""
81
+ if segment in self.clusters:
82
+ return self.clusters[segment]
83
+ if segment not in self._cluster_memo:
84
+ nearest = min(self.clusters, key=lambda p: (phonetic_distance(segment, p), p))
85
+ self._cluster_memo[segment] = self.clusters[nearest]
86
+ return self._cluster_memo[segment]
87
+
88
+ def _words(self, ipa: str) -> List[List[str]]:
89
+ words: List[List[str]] = [[]]
90
+ for seg in segments(ipa):
91
+ if seg == " ":
92
+ words.append([])
93
+ else:
94
+ words[-1].append(seg)
95
+ return [w for w in words if w]
96
+
97
+ def _keys(self, word: Sequence[str]) -> Set[Tuple[int, ...]]:
98
+ key = phonex(word, self.cluster_of)
99
+ one = {key[:i] + key[i + 1:] for i in range(len(key))}
100
+ two = {k[:i] + k[i + 1:] for k in one for i in range(len(k))}
101
+ return {k for k in {key} | one | two if k}
102
+
103
+ def add_term(self, term: Readings, id: Hashable) -> None:
104
+ """Index *term*, an IPA string or a list of IPA readings, under *id*."""
105
+ readings = _as_readings(term)
106
+ self.library[id] = readings
107
+ for reading in readings:
108
+ for word in self._words(reading):
109
+ for key in self._keys(word):
110
+ self.index.setdefault(key, set()).add(id)
111
+
112
+ def _ids_for_word(self, word: Sequence[str]) -> Set[Hashable]:
113
+ found: Set[Hashable] = set()
114
+ for key in self._keys(word):
115
+ found |= self.index.get(key, set())
116
+ return found
117
+
118
+ def search(self, query: Readings, n_results: int = 10) -> List[Match]:
119
+ """Terms that sound like *query*, nearest first.
120
+
121
+ A candidate must match every word of at least one query reading. Its score is the smallest
122
+ feature-weighted edit distance between a query reading and a term reading.
123
+ """
124
+ queries = _as_readings(query)
125
+ candidates: Set[Hashable] = set()
126
+ for q in queries:
127
+ per_word: Iterable[Set[Hashable]] = (self._ids_for_word(w) for w in self._words(q))
128
+ candidates |= set.intersection(*per_word) if self._words(q) else set()
129
+ q_segs = [segments(q) for q in queries]
130
+ matches = []
131
+ for id in candidates:
132
+ score = min(edit_distance(q, segments(t)) for q in q_segs for t in self.library[id])
133
+ matches.append(Match(id, self.library[id], score))
134
+ matches.sort(key=lambda m: (m.score, str(m.id)))
135
+ return matches[:n_results]