lexsync 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
lexsync/generation.py ADDED
@@ -0,0 +1,280 @@
1
+ """Deterministic, orthographically-controlled pseudoword generation.
2
+
3
+ For each real base word a pseudoword is produced by *constrained letter
4
+ substitution*: change the fewest letters so that (i) every resulting letter
5
+ bigram is attested in the corpus (orthographic legality), (ii) the form is not a
6
+ real word, and (iii) the length is preserved exactly. Among the legal non-word
7
+ neighbours the most bigram-plausible is chosen, with byte-order tie-breaks, so
8
+ the R and Python engines generate the identical pseudoword from the identical
9
+ corpus (see R_workflow/R/generation.R). This is a deterministic orthographic
10
+ cousin of Wuggy (Keuleers & Brysbaert, 2010); it trades Wuggy's subsyllabic model
11
+ for exact length matching and cross-engine reproducibility.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import numpy as np
16
+ import pandas as pd
17
+
18
+ from .querying import _VOWELS
19
+
20
+ _LETTERS = "abcdefghijklmnopqrstuvwxyz"
21
+ # Order in which subsyllabic constituents are considered for substitution: codas
22
+ # and nuclei vary more freely than onsets, which carry the most identifying
23
+ # orthography, so they are changed first.
24
+ _ROLE_ORDER = {"coda": 0, "nucleus": 1, "onset": 2}
25
+
26
+
27
+ def bigram_counts(words) -> dict:
28
+ """Integer counts of adjacent letter bigrams across a word list."""
29
+ counts: dict[str, int] = {}
30
+ for w in words:
31
+ w = str(w)
32
+ for i in range(len(w) - 1):
33
+ bg = w[i:i + 2]
34
+ counts[bg] = counts.get(bg, 0) + 1
35
+ return counts
36
+
37
+
38
+ def _legal(cand: str, bigrams: dict) -> bool:
39
+ return all(cand[i:i + 2] in bigrams for i in range(len(cand) - 1))
40
+
41
+
42
+ def _score(cand: str, bigrams: dict) -> int:
43
+ return sum(bigrams.get(cand[i:i + 2], 0) for i in range(len(cand) - 1))
44
+
45
+
46
+ def make_pseudoword(word: str, bigrams: dict, lexicon: set, used: set) -> str | None:
47
+ """The most bigram-plausible legal non-word at the smallest edit distance.
48
+
49
+ Searches single-letter substitutions first, then two-letter substitutions;
50
+ candidates are ranked by summed bigram frequency with a byte-order tie-break,
51
+ so the choice is deterministic and identical across engines.
52
+ """
53
+ word = str(word)
54
+ L = len(word)
55
+ # Distance 1: one substituted position.
56
+ cands = []
57
+ for pos in range(L):
58
+ for c in _LETTERS:
59
+ if c == word[pos]:
60
+ continue
61
+ cand = word[:pos] + c + word[pos + 1:]
62
+ if cand in lexicon or cand in used:
63
+ continue
64
+ if _legal(cand, bigrams):
65
+ cands.append(cand)
66
+ if not cands:
67
+ # Distance 2: two substituted positions (rare fallback).
68
+ for i in range(L):
69
+ for j in range(i + 1, L):
70
+ for ci in _LETTERS:
71
+ if ci == word[i]:
72
+ continue
73
+ for cj in _LETTERS:
74
+ if cj == word[j]:
75
+ continue
76
+ cand = word[:i] + ci + word[i + 1:j] + cj + word[j + 1:]
77
+ if cand in lexicon or cand in used:
78
+ continue
79
+ if _legal(cand, bigrams):
80
+ cands.append(cand)
81
+ if not cands:
82
+ return None
83
+ return min(cands, key=lambda c: (-_score(c, bigrams), c.encode("utf-8")))
84
+
85
+
86
+ def generate_pseudowords(base_words, reference_words) -> pd.DataFrame:
87
+ """A length-matched pseudoword for each base word.
88
+
89
+ Base words are processed in byte order so the ``used`` set evolves identically
90
+ across engines. Returns a frame with ``base_word`` and ``pseudoword``.
91
+ """
92
+ base = [str(w) for w in base_words]
93
+ lexicon = set(str(w) for w in reference_words)
94
+ bigrams = bigram_counts(reference_words)
95
+ order = sorted(range(len(base)), key=lambda i: base[i].encode("utf-8"))
96
+ used: set = set()
97
+ pseudo = [None] * len(base)
98
+ for i in order:
99
+ pw = make_pseudoword(base[i], bigrams, lexicon, used)
100
+ if pw is None:
101
+ raise ValueError(f"lexsync: could not generate a pseudoword for '{base[i]}'.")
102
+ used.add(pw)
103
+ pseudo[i] = pw
104
+ return pd.DataFrame({"base_word": base, "pseudoword": pseudo})
105
+
106
+
107
+ def segment_subsyllabic(word) -> list:
108
+ """Split a word into ordered ``(role, text)`` subsyllabic constituents.
109
+
110
+ Nuclei are the maximal vowel runs (the same regex used for syllable counting);
111
+ the consonants before the first nucleus form the first onset, those after the
112
+ last nucleus the final coda, and a consonant run *between* two nuclei is split
113
+ at its midpoint (``floor(m/2)`` to the left coda, the rest to the right onset).
114
+ Segmentation is on character indices, so accented multi-byte vowels are handled
115
+ identically in R and Python. A word with no vowel run returns ``[]`` (its caller
116
+ falls back to letter substitution). This is a deterministic orthographic
117
+ approximation of Wuggy's subsyllabic model (Keuleers & Brysbaert, 2010), not
118
+ phonological syllabification.
119
+ """
120
+ w = str(word).lower()
121
+ # Subsyllabic segmentation is an orthographic model for Latin a-z words; any
122
+ # word with a character outside a-z (accented, hyphenated, digit) returns [] so
123
+ # the caller falls back to letter substitution, which keeps the two engines in
124
+ # step and avoids multi-byte-string edge cases.
125
+ if not (w.isascii() and w.isalpha()):
126
+ return []
127
+ spans = [(m.start(), m.end()) for m in _VOWELS.finditer(w)]
128
+ if not spans:
129
+ return []
130
+ out = []
131
+ if spans[0][0] > 0:
132
+ out.append(("onset", w[:spans[0][0]]))
133
+ for si, (s, e) in enumerate(spans):
134
+ out.append(("nucleus", w[s:e]))
135
+ run = w[e:(spans[si + 1][0] if si + 1 < len(spans) else len(w))]
136
+ m = len(run)
137
+ if si + 1 < len(spans):
138
+ cut = m // 2
139
+ if cut:
140
+ out.append(("coda", run[:cut]))
141
+ if m - cut:
142
+ out.append(("onset", run[cut:]))
143
+ elif m:
144
+ out.append(("coda", run))
145
+ return out
146
+
147
+
148
+ def build_constituent_inventory(reference_words) -> dict:
149
+ """Attested subsyllabic constituents keyed by ``(role, length)`` with counts.
150
+
151
+ Integer counts, order-independent, so the inventory is identical however the
152
+ reference lexicon is ordered.
153
+ """
154
+ inv: dict = {}
155
+ for w in reference_words:
156
+ for role, text in segment_subsyllabic(w):
157
+ key = (role, len(text))
158
+ bucket = inv.setdefault(key, {})
159
+ bucket[text] = bucket.get(text, 0) + 1
160
+ return inv
161
+
162
+
163
+ def make_subsyllabic_pseudoword(word, inv, bigrams, lexicon, used):
164
+ """A pseudoword built by swapping whole subsyllabic constituents.
165
+
166
+ Up to ``ceil(2k/3)`` of the word's ``k`` constituents (codas and nuclei before
167
+ onsets) are each replaced, one at a time, by an attested constituent of the
168
+ same role and length, keeping every letter bigram legal and the form a novel
169
+ non-word. Each swap preserves length, so the whole form does. Returns ``None``
170
+ if no legal swap is possible (the caller then falls back to letter
171
+ substitution), so structure is respected without ever failing to produce a
172
+ pseudoword. The winning replacement is the most bigram-plausible with a
173
+ byte-order tie-break, so the two engines agree.
174
+ """
175
+ segs = segment_subsyllabic(word)
176
+ if not any(role == "nucleus" for role, _ in segs):
177
+ return None
178
+ k = len(segs)
179
+ target = -(-2 * k // 3) # ceil(2k/3)
180
+ texts = [t for _, t in segs]
181
+ eligible = [i for i, (role, t) in enumerate(segs)
182
+ if len(inv.get((role, len(t)), {})) >= 2]
183
+ eligible.sort(key=lambda i: (_ROLE_ORDER[segs[i][0]], i))
184
+ changed = 0
185
+ for i in eligible[:target]:
186
+ role, orig = segs[i][0], texts[i]
187
+ best = None
188
+ for alt in inv[(role, len(orig))]:
189
+ if alt == orig:
190
+ continue
191
+ trial = texts[:]
192
+ trial[i] = alt
193
+ cand = "".join(trial)
194
+ if cand in lexicon or cand in used or not _legal(cand, bigrams):
195
+ continue
196
+ key = (-_score(cand, bigrams), cand.encode("utf-8"))
197
+ if best is None or key < best[0]:
198
+ best = (key, trial)
199
+ if best is not None:
200
+ texts = best[1]
201
+ changed += 1
202
+ if changed == 0:
203
+ return None
204
+ final = "".join(texts)
205
+ if final in lexicon or final in used or not _legal(final, bigrams):
206
+ return None
207
+ return final
208
+
209
+
210
+ def generate_pseudowords_subsyllabic(base_words, reference_words) -> pd.DataFrame:
211
+ """A subsyllabic pseudoword for each base word (with letter-substitution fallback).
212
+
213
+ Base words are processed in byte order so the ``used`` set evolves identically
214
+ across engines; a base word with no legal subsyllabic swap falls back to the
215
+ letter-substitution generator, so every word yields a pseudoword.
216
+ """
217
+ base = [str(w) for w in base_words]
218
+ lexicon = set(str(w) for w in reference_words)
219
+ bigrams = bigram_counts(reference_words)
220
+ inv = build_constituent_inventory(reference_words)
221
+ order = sorted(range(len(base)), key=lambda i: base[i].encode("utf-8"))
222
+ used: set = set()
223
+ pseudo = [None] * len(base)
224
+ for i in order:
225
+ pw = make_subsyllabic_pseudoword(base[i], inv, bigrams, lexicon, used)
226
+ if pw is None:
227
+ pw = make_pseudoword(base[i], bigrams, lexicon, used)
228
+ if pw is None:
229
+ raise ValueError(f"lexsync: could not generate a pseudoword for '{base[i]}'.")
230
+ used.add(pw)
231
+ pseudo[i] = pw
232
+ return pd.DataFrame({"base_word": base, "pseudoword": pseudo})
233
+
234
+
235
+ def build_lexdec_stimuli(pool: pd.DataFrame, n: int, reference_words=None,
236
+ method: str = "letter_substitution") -> pd.DataFrame:
237
+ """Assemble a word-vs-pseudoword lexical-decision set from a candidate pool.
238
+
239
+ Real words are drawn by an even spread across the byte-ordered pool (the same
240
+ deterministic device as the matcher's anchor), then a length-matched
241
+ pseudoword is generated for each. The pool is first filtered to lower-case
242
+ a-z forms, the only ones the pseudoword generators are defined for, so the
243
+ eligible pool can be smaller than the request; the pipeline's shortfall
244
+ policy then decides whether that errors. ``reference_words`` (the full lexicon)
245
+ supplies the bigram statistics and the real-word list a pseudoword must avoid;
246
+ it falls back to the pool when not given. The presented string is the
247
+ ``target`` column; conditions are ``word`` and ``pseudoword`` and ``set``
248
+ pairs them.
249
+ """
250
+ pool = (pool.assign(_k=pool["word"].map(lambda w: w.encode("utf-8")))
251
+ .sort_values("_k").drop(columns="_k").reset_index(drop=True))
252
+ # The pseudoword generators are orthographic and defined for Latin a-z words;
253
+ # any word with another character is skipped, keeping the two engines in step.
254
+ pool = pool[pool["word"].astype(str).str.fullmatch(r"[a-z]+")].reset_index(drop=True)
255
+ if len(pool) == 0:
256
+ raise ValueError("lexsync: lexical-decision pool has no a-z words.")
257
+ n_take = min(n, len(pool))
258
+ idx = np.unique(np.round(np.linspace(1, len(pool), n_take)).astype(int)) - 1
259
+ words = pool.iloc[idx].reset_index(drop=True)
260
+ ref = list(reference_words) if reference_words is not None else pool["word"].tolist()
261
+ if method == "subsyllabic":
262
+ gen = generate_pseudowords_subsyllabic(words["word"].tolist(), ref)
263
+ elif method == "letter_substitution":
264
+ gen = generate_pseudowords(words["word"].tolist(), ref)
265
+ else:
266
+ raise ValueError(f"lexsync: unknown pseudoword generation method '{method}'.")
267
+ pw_map = dict(zip(gen["base_word"], gen["pseudoword"], strict=True))
268
+
269
+ real = pd.DataFrame({
270
+ "target": words["word"].tolist(), "word": words["word"].tolist(),
271
+ "condition": "word", "length": words["length"].tolist(),
272
+ "set": list(range(1, len(words) + 1)),
273
+ })
274
+ pseudo = pd.DataFrame({
275
+ "target": [pw_map[w] for w in words["word"]], "word": [pw_map[w] for w in words["word"]],
276
+ "condition": "pseudoword", "length": words["length"].tolist(),
277
+ "set": list(range(1, len(words) + 1)),
278
+ })
279
+ out = pd.concat([real, pseudo], ignore_index=True)
280
+ return out