phonematcher 0.0.0a2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,22 @@
1
+ MIT License
2
+
3
+ Copyright (c) TigreGotico
4
+ Portions copyright 2017 Ling Zhang (pyphone, fast_fuzzy_search)
5
+
6
+ Permission is hereby granted, free of charge, to any person obtaining a copy
7
+ of this software and associated documentation files (the "Software"), to deal
8
+ in the Software without restriction, including without limitation the rights
9
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
+ copies of the Software, and to permit persons to whom the Software is
11
+ furnished to do so, subject to the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be included in all
14
+ copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22
+ SOFTWARE.
@@ -0,0 +1,67 @@
1
+ Metadata-Version: 2.4
2
+ Name: phonematcher
3
+ Version: 0.0.0a2
4
+ Summary: Distinctive-feature distances between IPA phones, and phonetic fuzzy search over IPA
5
+ Author-email: JarbasAi <jarbasai@mailfence.com>
6
+ License-Expression: MIT
7
+ Project-URL: Source, https://github.com/TigreGotico/phonematcher
8
+ Requires-Python: >=3.11
9
+ Description-Content-Type: text/markdown
10
+ License-File: LICENSE
11
+ Provides-Extra: test
12
+ Requires-Dist: pytest>=8; extra == "test"
13
+ Dynamic: license-file
14
+
15
+ # phonematcher
16
+
17
+ phonematcher measures how far apart two IPA sounds are, and finds terms that sound like a query. It works on IPA only. To get IPA from text, use a grapheme-to-IPA library such as [orthography2ipa](https://github.com/TigreGotico/orthography2ipa).
18
+
19
+ Every distance comes from distinctive features. The feature table starts from PanPhon (Mortensen et al. 2016), takes the vowels from the IPA chart, and lists every other change with its source. [docs/features.md](docs/features.md) explains the table.
20
+
21
+ ## Install
22
+
23
+ ```
24
+ uv pip install phonematcher
25
+ ```
26
+
27
+ phonematcher has no runtime dependencies.
28
+
29
+ ## Distance between sounds
30
+
31
+ ```python
32
+ from phonematcher import phonetic_distance, ipa_distance, segments
33
+
34
+ phonetic_distance("ʁ", "χ") # 0.013: uvular fricatives that differ only in voice
35
+ phonetic_distance("ʁ", "k") # 0.118: a different manner and place
36
+ segments("ˈt͡ʃɐ̃w̃") # ['t͡ʃ', 'ɐ̃', 'w̃']
37
+ ipa_distance("ˈpojʃ", "ˈpojs") # 0.039: one near substitution
38
+ ```
39
+
40
+ `phonetic_distance` compares two segments, from 0 to 1. `ipa_distance` compares two IPA strings: an edit distance where a substitution costs the feature distance and an insertion or deletion costs 1.
41
+
42
+ ## Search by sound
43
+
44
+ ```python
45
+ from orthography2ipa import transcribe
46
+ from phonematcher import PhoneticFuzzySearch
47
+
48
+ search = PhoneticFuzzySearch()
49
+ for word in ["pois", "retirada", "carro", "caro", "casa", "dia"]:
50
+ search.add_term(transcribe(word, "pt-PT"), word)
51
+
52
+ query = transcribe("carro", "pt-BR-x-rj") # 'ˈkaχu'
53
+ best = search.search(query)[0]
54
+ best.id, best.term, round(best.score, 3) # ('carro', ('ˈkaʁu',), 0.013)
55
+ ```
56
+
57
+ A term may have several readings: `search.add_term(["ˈpojʃ", "ˈpojs"], "pois")`.
58
+
59
+ ## Documentation
60
+
61
+ - [docs/quickstart.md](docs/quickstart.md): first steps.
62
+ - [docs/features.md](docs/features.md): the feature table, its sources and the distance.
63
+ - [docs/api.md](docs/api.md): every public name.
64
+
65
+ ## Licence
66
+
67
+ MIT. The feature table derives from PanPhon (MIT) and from Hayes (2009) through the CSV conversion by Kyle Gorman (Apache-2.0).
@@ -0,0 +1,53 @@
1
+ # phonematcher
2
+
3
+ phonematcher measures how far apart two IPA sounds are, and finds terms that sound like a query. It works on IPA only. To get IPA from text, use a grapheme-to-IPA library such as [orthography2ipa](https://github.com/TigreGotico/orthography2ipa).
4
+
5
+ Every distance comes from distinctive features. The feature table starts from PanPhon (Mortensen et al. 2016), takes the vowels from the IPA chart, and lists every other change with its source. [docs/features.md](docs/features.md) explains the table.
6
+
7
+ ## Install
8
+
9
+ ```
10
+ uv pip install phonematcher
11
+ ```
12
+
13
+ phonematcher has no runtime dependencies.
14
+
15
+ ## Distance between sounds
16
+
17
+ ```python
18
+ from phonematcher import phonetic_distance, ipa_distance, segments
19
+
20
+ phonetic_distance("ʁ", "χ") # 0.013: uvular fricatives that differ only in voice
21
+ phonetic_distance("ʁ", "k") # 0.118: a different manner and place
22
+ segments("ˈt͡ʃɐ̃w̃") # ['t͡ʃ', 'ɐ̃', 'w̃']
23
+ ipa_distance("ˈpojʃ", "ˈpojs") # 0.039: one near substitution
24
+ ```
25
+
26
+ `phonetic_distance` compares two segments, from 0 to 1. `ipa_distance` compares two IPA strings: an edit distance where a substitution costs the feature distance and an insertion or deletion costs 1.
27
+
28
+ ## Search by sound
29
+
30
+ ```python
31
+ from orthography2ipa import transcribe
32
+ from phonematcher import PhoneticFuzzySearch
33
+
34
+ search = PhoneticFuzzySearch()
35
+ for word in ["pois", "retirada", "carro", "caro", "casa", "dia"]:
36
+ search.add_term(transcribe(word, "pt-PT"), word)
37
+
38
+ query = transcribe("carro", "pt-BR-x-rj") # 'ˈkaχu'
39
+ best = search.search(query)[0]
40
+ best.id, best.term, round(best.score, 3) # ('carro', ('ˈkaʁu',), 0.013)
41
+ ```
42
+
43
+ A term may have several readings: `search.add_term(["ˈpojʃ", "ˈpojs"], "pois")`.
44
+
45
+ ## Documentation
46
+
47
+ - [docs/quickstart.md](docs/quickstart.md): first steps.
48
+ - [docs/features.md](docs/features.md): the feature table, its sources and the distance.
49
+ - [docs/api.md](docs/api.md): every public name.
50
+
51
+ ## Licence
52
+
53
+ MIT. The feature table derives from PanPhon (MIT) and from Hayes (2009) through the CSV conversion by Kyle Gorman (Apache-2.0).
@@ -0,0 +1,3 @@
1
+ from phonematcher.clustering import Match, PhoneticFuzzySearch
2
+ from phonematcher.distance import edit_distance, ipa_distance, segments
3
+ from phonematcher.features import phonetic_distance, vectorize_phones
@@ -0,0 +1,135 @@
1
+ """Phonetic fuzzy search over IPA.
2
+
3
+ Phones are grouped into clusters of near sounds (UPGMA average linkage over
4
+ :func:`~phonematcher.features.phonetic_distance`). Each word of a term is indexed by its cluster
5
+ sequence and by every sequence with one or two clusters deleted, so a query finds terms that differ
6
+ by near sounds or by a dropped sound. Candidates are ranked by the feature-weighted edit distance
7
+ of :mod:`phonematcher.distance`.
8
+
9
+ Terms and queries are IPA. Use a grapheme-to-IPA library such as orthography2ipa to get IPA from
10
+ text.
11
+
12
+ Adapted from pyphone (https://github.com/lingz/pyphone) and fast_fuzzy_search
13
+ (https://github.com/lingz/fast_fuzzy_search), MIT License.
14
+ """
15
+ import unicodedata
16
+ from collections import namedtuple
17
+ from functools import lru_cache
18
+ from typing import Dict, Hashable, Iterable, List, Sequence, Set, Tuple, Union
19
+
20
+ from phonematcher.distance import edit_distance, segments
21
+ from phonematcher.features import phone_features, phonetic_distance
22
+ from phonematcher.phonex import phonex
23
+
24
+ #: A search result: the term's id, its readings, and its edit distance to the query.
25
+ Match = namedtuple("Match", ["id", "term", "score"])
26
+
27
+ Readings = Union[str, Sequence[str]]
28
+
29
+
30
+ def upgma(labels: Sequence[str], threshold: float) -> Dict[str, int]:
31
+ """Group *labels* by UPGMA average linkage, merging while the closest pair of groups is at most
32
+ *threshold* apart. Returns ``label -> cluster id``."""
33
+ dist = {(a, b): phonetic_distance(a, b) for a in labels for b in labels if a < b}
34
+
35
+ def between(c1: Set[str], c2: Set[str]) -> float:
36
+ return sum(dist[min(a, b), max(a, b)] for a in c1 for b in c2) / (len(c1) * len(c2))
37
+
38
+ clusters = [{label} for label in labels]
39
+ while len(clusters) > 1:
40
+ d, i, j = min((between(clusters[i], clusters[j]), i, j)
41
+ for i in range(len(clusters)) for j in range(i + 1, len(clusters)))
42
+ if d > threshold:
43
+ break
44
+ clusters[i] |= clusters.pop(j)
45
+ return {label: cid for cid, cluster in enumerate(clusters, start=1) for label in cluster}
46
+
47
+
48
+ @lru_cache(maxsize=8)
49
+ def phone_clusters(threshold: float) -> Dict[str, int]:
50
+ """The clusters of the IPA letters and tied affricates in the table at *threshold*.
51
+
52
+ Any other segment, such as a letter with diacritics, takes the cluster of its nearest phone.
53
+ """
54
+ phones = [p for p in phone_features
55
+ if (len(p) == 1 and unicodedata.category(p).startswith("L"))
56
+ or (len(p) == 3 and p[1] in "\u0361\u035c")]
57
+ return upgma(sorted(phones), threshold)
58
+
59
+
60
+ def _as_readings(term: Readings) -> Tuple[str, ...]:
61
+ return (term,) if isinstance(term, str) else tuple(term)
62
+
63
+
64
+ class PhoneticFuzzySearch:
65
+ """An index of IPA terms, searched by sound.
66
+
67
+ A term is one IPA string or several readings of the same item; a match on any reading counts.
68
+ *cluster_sensitivity* is the UPGMA merge threshold: higher values put more phones in one
69
+ cluster, which finds more candidates.
70
+ """
71
+
72
+ def __init__(self, cluster_sensitivity: float = 0.1):
73
+ self.cluster_sensitivity = cluster_sensitivity
74
+ self.clusters = phone_clusters(cluster_sensitivity)
75
+ self.index: Dict[Tuple[int, ...], Set[Hashable]] = {}
76
+ self.library: Dict[Hashable, Tuple[str, ...]] = {}
77
+ self._cluster_memo: Dict[str, int] = {}
78
+
79
+ def cluster_of(self, segment: str) -> int:
80
+ """The cluster of *segment*; a segment outside the table takes its nearest phone's."""
81
+ if segment in self.clusters:
82
+ return self.clusters[segment]
83
+ if segment not in self._cluster_memo:
84
+ nearest = min(self.clusters, key=lambda p: (phonetic_distance(segment, p), p))
85
+ self._cluster_memo[segment] = self.clusters[nearest]
86
+ return self._cluster_memo[segment]
87
+
88
+ def _words(self, ipa: str) -> List[List[str]]:
89
+ words: List[List[str]] = [[]]
90
+ for seg in segments(ipa):
91
+ if seg == " ":
92
+ words.append([])
93
+ else:
94
+ words[-1].append(seg)
95
+ return [w for w in words if w]
96
+
97
+ def _keys(self, word: Sequence[str]) -> Set[Tuple[int, ...]]:
98
+ key = phonex(word, self.cluster_of)
99
+ one = {key[:i] + key[i + 1:] for i in range(len(key))}
100
+ two = {k[:i] + k[i + 1:] for k in one for i in range(len(k))}
101
+ return {k for k in {key} | one | two if k}
102
+
103
+ def add_term(self, term: Readings, id: Hashable) -> None:
104
+ """Index *term*, an IPA string or a list of IPA readings, under *id*."""
105
+ readings = _as_readings(term)
106
+ self.library[id] = readings
107
+ for reading in readings:
108
+ for word in self._words(reading):
109
+ for key in self._keys(word):
110
+ self.index.setdefault(key, set()).add(id)
111
+
112
+ def _ids_for_word(self, word: Sequence[str]) -> Set[Hashable]:
113
+ found: Set[Hashable] = set()
114
+ for key in self._keys(word):
115
+ found |= self.index.get(key, set())
116
+ return found
117
+
118
+ def search(self, query: Readings, n_results: int = 10) -> List[Match]:
119
+ """Terms that sound like *query*, nearest first.
120
+
121
+ A candidate must match every word of at least one query reading. Its score is the smallest
122
+ feature-weighted edit distance between a query reading and a term reading.
123
+ """
124
+ queries = _as_readings(query)
125
+ candidates: Set[Hashable] = set()
126
+ for q in queries:
127
+ per_word: Iterable[Set[Hashable]] = (self._ids_for_word(w) for w in self._words(q))
128
+ candidates |= set.intersection(*per_word) if self._words(q) else set()
129
+ q_segs = [segments(q) for q in queries]
130
+ matches = []
131
+ for id in candidates:
132
+ score = min(edit_distance(q, segments(t)) for q in q_segs for t in self.library[id])
133
+ matches.append(Match(id, self.library[id], score))
134
+ matches.sort(key=lambda m: (m.score, str(m.id)))
135
+ return matches[:n_results]