lexsync 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lexsync/__init__.py +53 -0
- lexsync/__main__.py +5 -0
- lexsync/blocks.py +149 -0
- lexsync/cli.py +85 -0
- lexsync/corpora.py +331 -0
- lexsync/counterbalancing.py +399 -0
- lexsync/data/en_example.csv +3001 -0
- lexsync/data/es_example.csv +3001 -0
- lexsync/data/registry.yaml +149 -0
- lexsync/data/schema.yaml +193 -0
- lexsync/data/zh_example.csv +3001 -0
- lexsync/datasheet.py +752 -0
- lexsync/generation.py +280 -0
- lexsync/io_utils.py +609 -0
- lexsync/logging.py +88 -0
- lexsync/matching.py +614 -0
- lexsync/pairs.py +177 -0
- lexsync/paradigms.py +163 -0
- lexsync/py.typed +0 -0
- lexsync/querying.py +481 -0
- lexsync/run_pipeline.py +425 -0
- lexsync/scripting.py +845 -0
- lexsync/templates/jspsych/experiment_template.html +223 -0
- lexsync/templates/opensesame/README.md +36 -0
- lexsync/templates/opensesame/inline_send_triggers.py +18 -0
- lexsync/templates/opensesame/inline_trigger_setup.py +53 -0
- lexsync/templates/psychopy/mock_port.py +23 -0
- lexsync/templates/psychopy/trial_runner_template.py +300 -0
- lexsync/validation.py +267 -0
- lexsync-0.1.1.dist-info/METADATA +234 -0
- lexsync-0.1.1.dist-info/RECORD +35 -0
- lexsync-0.1.1.dist-info/WHEEL +4 -0
- lexsync-0.1.1.dist-info/entry_points.txt +2 -0
- lexsync-0.1.1.dist-info/licenses/LICENSE +33 -0
- lexsync-0.1.1.dist-info/licenses/LICENSE.note +108 -0
lexsync/generation.py
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""Deterministic, orthographically-controlled pseudoword generation.
|
|
2
|
+
|
|
3
|
+
For each real base word a pseudoword is produced by *constrained letter
|
|
4
|
+
substitution*: change the fewest letters so that (i) every resulting letter
|
|
5
|
+
bigram is attested in the corpus (orthographic legality), (ii) the form is not a
|
|
6
|
+
real word, and (iii) the length is preserved exactly. Among the legal non-word
|
|
7
|
+
neighbours the most bigram-plausible is chosen, with byte-order tie-breaks, so
|
|
8
|
+
the R and Python engines generate the identical pseudoword from the identical
|
|
9
|
+
corpus (see R_workflow/R/generation.R). This is a deterministic orthographic
|
|
10
|
+
cousin of Wuggy (Keuleers & Brysbaert, 2010); it trades Wuggy's subsyllabic model
|
|
11
|
+
for exact length matching and cross-engine reproducibility.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
from .querying import _VOWELS
|
|
19
|
+
|
|
20
|
+
_LETTERS = "abcdefghijklmnopqrstuvwxyz"
|
|
21
|
+
# Order in which subsyllabic constituents are considered for substitution: codas
|
|
22
|
+
# and nuclei vary more freely than onsets, which carry the most identifying
|
|
23
|
+
# orthography, so they are changed first.
|
|
24
|
+
_ROLE_ORDER = {"coda": 0, "nucleus": 1, "onset": 2}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def bigram_counts(words) -> dict:
|
|
28
|
+
"""Integer counts of adjacent letter bigrams across a word list."""
|
|
29
|
+
counts: dict[str, int] = {}
|
|
30
|
+
for w in words:
|
|
31
|
+
w = str(w)
|
|
32
|
+
for i in range(len(w) - 1):
|
|
33
|
+
bg = w[i:i + 2]
|
|
34
|
+
counts[bg] = counts.get(bg, 0) + 1
|
|
35
|
+
return counts
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _legal(cand: str, bigrams: dict) -> bool:
|
|
39
|
+
return all(cand[i:i + 2] in bigrams for i in range(len(cand) - 1))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _score(cand: str, bigrams: dict) -> int:
|
|
43
|
+
return sum(bigrams.get(cand[i:i + 2], 0) for i in range(len(cand) - 1))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def make_pseudoword(word: str, bigrams: dict, lexicon: set, used: set) -> str | None:
|
|
47
|
+
"""The most bigram-plausible legal non-word at the smallest edit distance.
|
|
48
|
+
|
|
49
|
+
Searches single-letter substitutions first, then two-letter substitutions;
|
|
50
|
+
candidates are ranked by summed bigram frequency with a byte-order tie-break,
|
|
51
|
+
so the choice is deterministic and identical across engines.
|
|
52
|
+
"""
|
|
53
|
+
word = str(word)
|
|
54
|
+
L = len(word)
|
|
55
|
+
# Distance 1: one substituted position.
|
|
56
|
+
cands = []
|
|
57
|
+
for pos in range(L):
|
|
58
|
+
for c in _LETTERS:
|
|
59
|
+
if c == word[pos]:
|
|
60
|
+
continue
|
|
61
|
+
cand = word[:pos] + c + word[pos + 1:]
|
|
62
|
+
if cand in lexicon or cand in used:
|
|
63
|
+
continue
|
|
64
|
+
if _legal(cand, bigrams):
|
|
65
|
+
cands.append(cand)
|
|
66
|
+
if not cands:
|
|
67
|
+
# Distance 2: two substituted positions (rare fallback).
|
|
68
|
+
for i in range(L):
|
|
69
|
+
for j in range(i + 1, L):
|
|
70
|
+
for ci in _LETTERS:
|
|
71
|
+
if ci == word[i]:
|
|
72
|
+
continue
|
|
73
|
+
for cj in _LETTERS:
|
|
74
|
+
if cj == word[j]:
|
|
75
|
+
continue
|
|
76
|
+
cand = word[:i] + ci + word[i + 1:j] + cj + word[j + 1:]
|
|
77
|
+
if cand in lexicon or cand in used:
|
|
78
|
+
continue
|
|
79
|
+
if _legal(cand, bigrams):
|
|
80
|
+
cands.append(cand)
|
|
81
|
+
if not cands:
|
|
82
|
+
return None
|
|
83
|
+
return min(cands, key=lambda c: (-_score(c, bigrams), c.encode("utf-8")))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def generate_pseudowords(base_words, reference_words) -> pd.DataFrame:
|
|
87
|
+
"""A length-matched pseudoword for each base word.
|
|
88
|
+
|
|
89
|
+
Base words are processed in byte order so the ``used`` set evolves identically
|
|
90
|
+
across engines. Returns a frame with ``base_word`` and ``pseudoword``.
|
|
91
|
+
"""
|
|
92
|
+
base = [str(w) for w in base_words]
|
|
93
|
+
lexicon = set(str(w) for w in reference_words)
|
|
94
|
+
bigrams = bigram_counts(reference_words)
|
|
95
|
+
order = sorted(range(len(base)), key=lambda i: base[i].encode("utf-8"))
|
|
96
|
+
used: set = set()
|
|
97
|
+
pseudo = [None] * len(base)
|
|
98
|
+
for i in order:
|
|
99
|
+
pw = make_pseudoword(base[i], bigrams, lexicon, used)
|
|
100
|
+
if pw is None:
|
|
101
|
+
raise ValueError(f"lexsync: could not generate a pseudoword for '{base[i]}'.")
|
|
102
|
+
used.add(pw)
|
|
103
|
+
pseudo[i] = pw
|
|
104
|
+
return pd.DataFrame({"base_word": base, "pseudoword": pseudo})
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def segment_subsyllabic(word) -> list:
|
|
108
|
+
"""Split a word into ordered ``(role, text)`` subsyllabic constituents.
|
|
109
|
+
|
|
110
|
+
Nuclei are the maximal vowel runs (the same regex used for syllable counting);
|
|
111
|
+
the consonants before the first nucleus form the first onset, those after the
|
|
112
|
+
last nucleus the final coda, and a consonant run *between* two nuclei is split
|
|
113
|
+
at its midpoint (``floor(m/2)`` to the left coda, the rest to the right onset).
|
|
114
|
+
Segmentation is on character indices, so accented multi-byte vowels are handled
|
|
115
|
+
identically in R and Python. A word with no vowel run returns ``[]`` (its caller
|
|
116
|
+
falls back to letter substitution). This is a deterministic orthographic
|
|
117
|
+
approximation of Wuggy's subsyllabic model (Keuleers & Brysbaert, 2010), not
|
|
118
|
+
phonological syllabification.
|
|
119
|
+
"""
|
|
120
|
+
w = str(word).lower()
|
|
121
|
+
# Subsyllabic segmentation is an orthographic model for Latin a-z words; any
|
|
122
|
+
# word with a character outside a-z (accented, hyphenated, digit) returns [] so
|
|
123
|
+
# the caller falls back to letter substitution, which keeps the two engines in
|
|
124
|
+
# step and avoids multi-byte-string edge cases.
|
|
125
|
+
if not (w.isascii() and w.isalpha()):
|
|
126
|
+
return []
|
|
127
|
+
spans = [(m.start(), m.end()) for m in _VOWELS.finditer(w)]
|
|
128
|
+
if not spans:
|
|
129
|
+
return []
|
|
130
|
+
out = []
|
|
131
|
+
if spans[0][0] > 0:
|
|
132
|
+
out.append(("onset", w[:spans[0][0]]))
|
|
133
|
+
for si, (s, e) in enumerate(spans):
|
|
134
|
+
out.append(("nucleus", w[s:e]))
|
|
135
|
+
run = w[e:(spans[si + 1][0] if si + 1 < len(spans) else len(w))]
|
|
136
|
+
m = len(run)
|
|
137
|
+
if si + 1 < len(spans):
|
|
138
|
+
cut = m // 2
|
|
139
|
+
if cut:
|
|
140
|
+
out.append(("coda", run[:cut]))
|
|
141
|
+
if m - cut:
|
|
142
|
+
out.append(("onset", run[cut:]))
|
|
143
|
+
elif m:
|
|
144
|
+
out.append(("coda", run))
|
|
145
|
+
return out
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def build_constituent_inventory(reference_words) -> dict:
|
|
149
|
+
"""Attested subsyllabic constituents keyed by ``(role, length)`` with counts.
|
|
150
|
+
|
|
151
|
+
Integer counts, order-independent, so the inventory is identical however the
|
|
152
|
+
reference lexicon is ordered.
|
|
153
|
+
"""
|
|
154
|
+
inv: dict = {}
|
|
155
|
+
for w in reference_words:
|
|
156
|
+
for role, text in segment_subsyllabic(w):
|
|
157
|
+
key = (role, len(text))
|
|
158
|
+
bucket = inv.setdefault(key, {})
|
|
159
|
+
bucket[text] = bucket.get(text, 0) + 1
|
|
160
|
+
return inv
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def make_subsyllabic_pseudoword(word, inv, bigrams, lexicon, used):
|
|
164
|
+
"""A pseudoword built by swapping whole subsyllabic constituents.
|
|
165
|
+
|
|
166
|
+
Up to ``ceil(2k/3)`` of the word's ``k`` constituents (codas and nuclei before
|
|
167
|
+
onsets) are each replaced, one at a time, by an attested constituent of the
|
|
168
|
+
same role and length, keeping every letter bigram legal and the form a novel
|
|
169
|
+
non-word. Each swap preserves length, so the whole form does. Returns ``None``
|
|
170
|
+
if no legal swap is possible (the caller then falls back to letter
|
|
171
|
+
substitution), so structure is respected without ever failing to produce a
|
|
172
|
+
pseudoword. The winning replacement is the most bigram-plausible with a
|
|
173
|
+
byte-order tie-break, so the two engines agree.
|
|
174
|
+
"""
|
|
175
|
+
segs = segment_subsyllabic(word)
|
|
176
|
+
if not any(role == "nucleus" for role, _ in segs):
|
|
177
|
+
return None
|
|
178
|
+
k = len(segs)
|
|
179
|
+
target = -(-2 * k // 3) # ceil(2k/3)
|
|
180
|
+
texts = [t for _, t in segs]
|
|
181
|
+
eligible = [i for i, (role, t) in enumerate(segs)
|
|
182
|
+
if len(inv.get((role, len(t)), {})) >= 2]
|
|
183
|
+
eligible.sort(key=lambda i: (_ROLE_ORDER[segs[i][0]], i))
|
|
184
|
+
changed = 0
|
|
185
|
+
for i in eligible[:target]:
|
|
186
|
+
role, orig = segs[i][0], texts[i]
|
|
187
|
+
best = None
|
|
188
|
+
for alt in inv[(role, len(orig))]:
|
|
189
|
+
if alt == orig:
|
|
190
|
+
continue
|
|
191
|
+
trial = texts[:]
|
|
192
|
+
trial[i] = alt
|
|
193
|
+
cand = "".join(trial)
|
|
194
|
+
if cand in lexicon or cand in used or not _legal(cand, bigrams):
|
|
195
|
+
continue
|
|
196
|
+
key = (-_score(cand, bigrams), cand.encode("utf-8"))
|
|
197
|
+
if best is None or key < best[0]:
|
|
198
|
+
best = (key, trial)
|
|
199
|
+
if best is not None:
|
|
200
|
+
texts = best[1]
|
|
201
|
+
changed += 1
|
|
202
|
+
if changed == 0:
|
|
203
|
+
return None
|
|
204
|
+
final = "".join(texts)
|
|
205
|
+
if final in lexicon or final in used or not _legal(final, bigrams):
|
|
206
|
+
return None
|
|
207
|
+
return final
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def generate_pseudowords_subsyllabic(base_words, reference_words) -> pd.DataFrame:
|
|
211
|
+
"""A subsyllabic pseudoword for each base word (with letter-substitution fallback).
|
|
212
|
+
|
|
213
|
+
Base words are processed in byte order so the ``used`` set evolves identically
|
|
214
|
+
across engines; a base word with no legal subsyllabic swap falls back to the
|
|
215
|
+
letter-substitution generator, so every word yields a pseudoword.
|
|
216
|
+
"""
|
|
217
|
+
base = [str(w) for w in base_words]
|
|
218
|
+
lexicon = set(str(w) for w in reference_words)
|
|
219
|
+
bigrams = bigram_counts(reference_words)
|
|
220
|
+
inv = build_constituent_inventory(reference_words)
|
|
221
|
+
order = sorted(range(len(base)), key=lambda i: base[i].encode("utf-8"))
|
|
222
|
+
used: set = set()
|
|
223
|
+
pseudo = [None] * len(base)
|
|
224
|
+
for i in order:
|
|
225
|
+
pw = make_subsyllabic_pseudoword(base[i], inv, bigrams, lexicon, used)
|
|
226
|
+
if pw is None:
|
|
227
|
+
pw = make_pseudoword(base[i], bigrams, lexicon, used)
|
|
228
|
+
if pw is None:
|
|
229
|
+
raise ValueError(f"lexsync: could not generate a pseudoword for '{base[i]}'.")
|
|
230
|
+
used.add(pw)
|
|
231
|
+
pseudo[i] = pw
|
|
232
|
+
return pd.DataFrame({"base_word": base, "pseudoword": pseudo})
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def build_lexdec_stimuli(pool: pd.DataFrame, n: int, reference_words=None,
|
|
236
|
+
method: str = "letter_substitution") -> pd.DataFrame:
|
|
237
|
+
"""Assemble a word-vs-pseudoword lexical-decision set from a candidate pool.
|
|
238
|
+
|
|
239
|
+
Real words are drawn by an even spread across the byte-ordered pool (the same
|
|
240
|
+
deterministic device as the matcher's anchor), then a length-matched
|
|
241
|
+
pseudoword is generated for each. The pool is first filtered to lower-case
|
|
242
|
+
a-z forms, the only ones the pseudoword generators are defined for, so the
|
|
243
|
+
eligible pool can be smaller than the request; the pipeline's shortfall
|
|
244
|
+
policy then decides whether that errors. ``reference_words`` (the full lexicon)
|
|
245
|
+
supplies the bigram statistics and the real-word list a pseudoword must avoid;
|
|
246
|
+
it falls back to the pool when not given. The presented string is the
|
|
247
|
+
``target`` column; conditions are ``word`` and ``pseudoword`` and ``set``
|
|
248
|
+
pairs them.
|
|
249
|
+
"""
|
|
250
|
+
pool = (pool.assign(_k=pool["word"].map(lambda w: w.encode("utf-8")))
|
|
251
|
+
.sort_values("_k").drop(columns="_k").reset_index(drop=True))
|
|
252
|
+
# The pseudoword generators are orthographic and defined for Latin a-z words;
|
|
253
|
+
# any word with another character is skipped, keeping the two engines in step.
|
|
254
|
+
pool = pool[pool["word"].astype(str).str.fullmatch(r"[a-z]+")].reset_index(drop=True)
|
|
255
|
+
if len(pool) == 0:
|
|
256
|
+
raise ValueError("lexsync: lexical-decision pool has no a-z words.")
|
|
257
|
+
n_take = min(n, len(pool))
|
|
258
|
+
idx = np.unique(np.round(np.linspace(1, len(pool), n_take)).astype(int)) - 1
|
|
259
|
+
words = pool.iloc[idx].reset_index(drop=True)
|
|
260
|
+
ref = list(reference_words) if reference_words is not None else pool["word"].tolist()
|
|
261
|
+
if method == "subsyllabic":
|
|
262
|
+
gen = generate_pseudowords_subsyllabic(words["word"].tolist(), ref)
|
|
263
|
+
elif method == "letter_substitution":
|
|
264
|
+
gen = generate_pseudowords(words["word"].tolist(), ref)
|
|
265
|
+
else:
|
|
266
|
+
raise ValueError(f"lexsync: unknown pseudoword generation method '{method}'.")
|
|
267
|
+
pw_map = dict(zip(gen["base_word"], gen["pseudoword"], strict=True))
|
|
268
|
+
|
|
269
|
+
real = pd.DataFrame({
|
|
270
|
+
"target": words["word"].tolist(), "word": words["word"].tolist(),
|
|
271
|
+
"condition": "word", "length": words["length"].tolist(),
|
|
272
|
+
"set": list(range(1, len(words) + 1)),
|
|
273
|
+
})
|
|
274
|
+
pseudo = pd.DataFrame({
|
|
275
|
+
"target": [pw_map[w] for w in words["word"]], "word": [pw_map[w] for w in words["word"]],
|
|
276
|
+
"condition": "pseudoword", "length": words["length"].tolist(),
|
|
277
|
+
"set": list(range(1, len(words) + 1)),
|
|
278
|
+
})
|
|
279
|
+
out = pd.concat([real, pseudo], ignore_index=True)
|
|
280
|
+
return out
|