lexsync 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,399 @@
1
+ """Assign matched stimuli to lists and trial order, and build participant tables.
2
+
3
+ Mirrors R_workflow/R/counterbalancing.R. Two recipes are provided: the original
4
+ ``factorial`` one (every matched item is shown, lists split the matched sets) and
5
+ ``latin_square_target`` for paired/sentence paradigms (each item appears once per
6
+ list in one rotated condition, so a target is never repeated within a list). Trial
7
+ order within a list comes from a seeded, keyed-hash shuffle, so the R and Python
8
+ engines produce the same order byte for byte.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import hashlib
13
+ import math
14
+ from itertools import product
15
+
16
+ import numpy as np
17
+ import pandas as pd
18
+
19
+ from .io_utils import _key_part
20
+ from .paradigms import get_paradigm
21
+
22
+
23
+ def _recipe(design: dict) -> str:
24
+ if design.get("events"):
25
+ return "factorial"
26
+ name = design.get("paradigm", "factorial")
27
+ try:
28
+ return get_paradigm(name).get("counterbalance", "factorial")
29
+ except ValueError:
30
+ return "factorial"
31
+
32
+
33
+ def _shuffle_deterministic(df: pd.DataFrame, seed) -> pd.DataFrame:
34
+ """Order the trials of one list by a keyed hash, not a random number generator.
35
+
36
+ Each row is ranked by the SHA-256 digest of "seed|replicate|list|set|condition",
37
+ a tuple that identifies the trial uniquely under either recipe. Distinct inputs
38
+ to SHA-256 behave as independent uniform draws, so ordering by the digest
39
+ realises a seeded random permutation, but as a pure function of the design: the
40
+ same bytes from the R and Python engines on any platform, and a different order
41
+ for every seed. R's sample() and numpy's PCG64 could never agree on a
42
+ permutation, which used to be the one engine-specific artefact in an otherwise
43
+ byte-identical pipeline. The replicate term keeps independently counterbalanced
44
+ item sets from sharing one permutation pattern.
45
+ """
46
+ rep = df["replicate"] if "replicate" in df.columns else [0] * len(df)
47
+ # Every component is formatted explicitly rather than interpolated. Default
48
+ # stringification is where the two engines silently part company: R renders
49
+ # the double 42 as "42" and Python as "42.0", and a pandas column is promoted
50
+ # to float64 by a single missing value, which would change every digest, every
51
+ # trial order and every generated artefact with nothing to signal it. The
52
+ # values are integers today, so this changes no output; it removes a trap.
53
+ ranks = [hashlib.sha256("|".join(
54
+ [_key_part(seed), _key_part(r), _key_part(l), _key_part(s), _key_part(c)]
55
+ ).encode("utf-8")).hexdigest()
56
+ for r, l, s, c in zip(rep, df["list"], df["set"], df["condition"], strict=True)]
57
+ df = df.iloc[np.argsort(ranks, kind="stable")].copy()
58
+ df["trial"] = np.arange(1, len(df) + 1)
59
+ return df
60
+
61
+
62
+ # ---- Balance-aware list assignment (counterbalance.optimise) ----------------
63
+ #
64
+ # The factorial recipe deals item sets to lists by rank: set 1 to list 1, set 2 to
65
+ # list 2, and so on. That is reproducible but arbitrary with respect to the items
66
+ # themselves, so one list can end up holding systematically shorter or more frequent
67
+ # words than another. Where lists are given to different participants, that difference
68
+ # is confounded with the between-subjects factor.
69
+ #
70
+ # ``counterbalance.optimise: true`` searches for an assignment whose lists match on
71
+ # the declared dimensions instead. It is off by default: switching it on changes which
72
+ # items a participant sees, so it must be an explicit choice rather than something
73
+ # that happens to a design on upgrade.
74
+ #
75
+ # The search obeys the package's two hard rules, and both shape the code below more
76
+ # than is obvious.
77
+ #
78
+ # No RNG. The order of the search and every tie in it is decided by the seeded keyed
79
+ # hash already used for trial order, so the result is a pure function of the design.
80
+ #
81
+ # An ALL-INTEGER objective. This is not fastidiousness. Cohen's d computed in the two
82
+ # engines was measured to differ by about 3e-16 on average, which at nine decimal
83
+ # places leaves roughly a one-in-three chance over a full run that a comparison of two
84
+ # candidate swaps resolves differently in Python than in R -- and the failure is
85
+ # silent and total, different words rather than different last bits. The values are
86
+ # quantised to integers ONCE, and the search then uses only +, -, * and comparison.
87
+ # There is no division anywhere in the objective.
88
+ #
89
+ # Mirrors R_workflow/R/counterbalancing.R.
90
+
91
+ # Milli-units of a dimension's own mean, the scale every dimension is quantised to.
92
+ # Expressing all dimensions in one unit is what lets their imbalances be added without
93
+ # a per-dimension float weight: a word's length and its Zipf frequency differ by
94
+ # orders of magnitude in raw units, and summing raw deviations would silently optimise
95
+ # almost entirely for whichever dimension has the larger numbers.
96
+ #
97
+ # The constant is 1000 times the 100 that `unit` already carries (see _quantise_dim),
98
+ # so a value at the dimension's mean quantises to 1000. That factor is not cosmetic: at
99
+ # 1000 here, a value at the mean quantised to 10, which left only about ten
100
+ # distinguishable levels across a dimension's whole range, and imbalances smaller than
101
+ # a tenth of the mean were invisible to the objective. The search then traded away a
102
+ # dimension it could not see in order to improve one it could.
103
+ BALANCE_UNIT_SCALE = 100000
104
+
105
+ # Guard on the objective's magnitude. Python ints are arbitrary precision, but the R
106
+ # engine holds these as doubles and is exact only below 2^53; the bound is shared so
107
+ # both engines refuse the same designs rather than one of them silently rounding.
108
+ # Nothing near this is reachable with realistic designs.
109
+ BALANCE_MAX_MAGNITUDE = 2 ** 50
110
+
111
+
112
+ def _balance_dims(design: dict) -> list:
113
+ cb = design.get("counterbalance") or {}
114
+ dims = list(cb.get("balance_on") or [])
115
+ if not dims:
116
+ dims = list(design.get("match_on") or [])
117
+ if not dims and design.get("continuous"):
118
+ dims = [design["continuous"]["predictor"]] + list(
119
+ design["continuous"].get("controls") or [])
120
+ seen, out = set(), []
121
+ for d in dims:
122
+ if str(d) not in seen:
123
+ seen.add(str(d))
124
+ out.append(str(d))
125
+ return out
126
+
127
+
128
+ def _quantise_dim(values, name: str) -> list:
129
+ """Quantise one dimension to integers in milli-units of its own mean.
130
+
131
+ The only floating-point steps in the whole optimiser are here, and each is a
132
+ single IEEE operation that the standard requires to be correctly rounded, so both
133
+ engines hold the same double before it is truncated. Truncation of a double is
134
+ exact, so no rounding mode is involved. ``round`` would have agreed on every case
135
+ measured, but truncation removes the question rather than answering it.
136
+
137
+ The mean is computed from INTEGER counts, never as a float sum: summing 20000
138
+ doubles was measured to give three different answers across R's sum(),
139
+ math.fsum, numpy's pairwise sum and a naive loop, so a float mean here could put
140
+ the two engines on different search paths from the first step.
141
+ """
142
+ vals = [float(v) for v in values]
143
+ if any(v != v for v in vals):
144
+ raise ValueError(
145
+ "lexsync: dimension '%s' has missing values, so it cannot be balanced "
146
+ "across lists. Fill or drop those items." % name)
147
+ # math.trunc, guarded on finiteness, rather than a bare int(): int(inf) raises
148
+ # OverflowError with a message of Python's own, where R's trunc(Inf) is Inf and
149
+ # reaches the size refusal below. Truncating BEFORE the comparison, as the R twin
150
+ # does, so the two engines refuse exactly the same values.
151
+ scaled = [math.trunc(s) if math.isfinite(s) else s
152
+ for s in (abs(v) * 100 for v in vals)]
153
+ if max(scaled, default=0) > 2 ** 31 - 1:
154
+ raise ValueError(
155
+ "lexsync: dimension '%s' has values too large to balance on." % name)
156
+ unit = max(1, sum(scaled) // len(scaled))
157
+ return [int(v * BALANCE_UNIT_SCALE / unit) for v in vals]
158
+
159
+
160
+ def _balance_values(stimuli: pd.DataFrame, dims: list) -> dict:
161
+ """One integer per set per dimension: the set's total, because a list receives
162
+ whole sets. Rows are grouped by set in sorted order so both engines build the
163
+ same matrix from the same frame."""
164
+ sets = sorted(stimuli["set"].unique())
165
+ V = {}
166
+ for d in dims:
167
+ q = _quantise_dim(stimuli[d], d)
168
+ by_set = {s: 0 for s in sets}
169
+ for s, qv in zip(stimuli["set"], q, strict=True):
170
+ by_set[s] += qv # exact integer sum
171
+ V[d] = [by_set[s] for s in sets]
172
+ return V
173
+
174
+
175
+ def _balance_cost(V: dict, assign: list, n_lists: int, n_sets: int) -> int:
176
+ """Total absolute deviation of each list's dimension total from its fair share,
177
+ scaled by the number of sets so the fair share stays an integer. Every term is an
178
+ exact integer; there is no division."""
179
+ cost = 0
180
+ for vals in V.values():
181
+ total = sum(vals)
182
+ for l in range(1, n_lists + 1):
183
+ s = sum(v for v, a in zip(vals, assign, strict=True) if a == l)
184
+ n_in = sum(1 for a in assign if a == l)
185
+ cost += abs(s * n_sets - total * n_in)
186
+ return cost
187
+
188
+
189
+ def balance_lists(stimuli: pd.DataFrame, design: dict, schema: dict) -> dict:
190
+ """Assign item sets to lists so the lists match on the item dimensions.
191
+
192
+ The factorial recipe's default deal is by set rank, which balances nothing. This
193
+ searches instead for an assignment whose lists have near-equal totals on each
194
+ declared dimension, by steepest-descent pairwise swaps between lists. List sizes
195
+ are preserved, because a swap exchanges one set for another.
196
+
197
+ The search is deterministic and identical in the R and Python engines: the
198
+ objective is all-integer (see the notes in this module), the descent takes the
199
+ single best swap each pass, and ties are broken by the seeded keyed hash rather
200
+ than by position, so no list is favoured by being numbered first. Because the cost
201
+ is a non-negative integer that strictly decreases, the search terminates;
202
+ ``max_passes`` bounds it anyway and the report says whether the bound was reached.
203
+
204
+ Five situations raise ``ValueError`` rather than being answered with an assignment
205
+ that would mislead. A Latin-square design is refused because every item already
206
+ appears in every list there, so there is nothing left to equate, and fewer than two
207
+ lists leaves no pair of lists to exchange sets between. A design with no resolvable
208
+ balance dimension is refused, as is one naming a dimension the stimuli do not carry,
209
+ and the message names the columns. The last refusal is arithmetic: the search stops
210
+ if the integer objective would leave the range a double represents exactly, since
211
+ past that point the two engines could disagree.
212
+
213
+ Returns ``{"list_of_set": {set: list}, "report": {...}}``.
214
+ """
215
+ cb = design.get("counterbalance") or {}
216
+ n_lists = int(cb.get("lists", 1))
217
+ seed = schema.get("seed", 1)
218
+ if _recipe(design) == "latin_square_target":
219
+ raise ValueError(
220
+ "lexsync: counterbalance.optimise does not apply to a Latin-square "
221
+ "design. Every item already appears in every list there, so the lists are "
222
+ "balanced on the items by construction; the rotation decides only which "
223
+ "condition each item takes.")
224
+ if n_lists < 2:
225
+ raise ValueError("lexsync: counterbalance.optimise needs "
226
+ "counterbalance.lists to be 2 or more.")
227
+ dims = _balance_dims(design)
228
+ if not dims:
229
+ raise ValueError(
230
+ "lexsync: counterbalance.optimise has no dimensions to balance. Name them "
231
+ "in counterbalance.balance_on, or give the design a match_on.")
232
+ absent = sorted((d for d in dims if d not in stimuli.columns),
233
+ key=lambda s: s.encode("utf-8"))
234
+ if absent:
235
+ raise ValueError("lexsync: cannot balance on column(s) the stimuli do not "
236
+ "have: %s." % ", ".join("'%s'" % a for a in absent))
237
+
238
+ sets = sorted(stimuli["set"].unique())
239
+ n_sets = len(sets)
240
+ V = _balance_values(stimuli, dims)
241
+ totals = {d: sum(v) for d, v in V.items()}
242
+ if max((abs(t) for t in totals.values()), default=0) * n_sets > BALANCE_MAX_MAGNITUDE:
243
+ raise ValueError(
244
+ "lexsync: the balance objective would exceed the exact-integer range.")
245
+
246
+ # Start from the default deal, so the search improves on the shipped behaviour
247
+ # rather than starting somewhere unrelated to it.
248
+ assign = [(i % n_lists) + 1 for i in range(n_sets)]
249
+ cost0 = _balance_cost(V, assign, n_lists, n_sets)
250
+
251
+ pairs = [(i, j) for i in range(n_sets - 1) for j in range(i + 1, n_sets)]
252
+ max_passes = int(cb.get("max_passes", 500))
253
+ n_swaps = 0
254
+ cost = cost0
255
+ hit_bound = False
256
+ for pass_i in range(1, max(0, max_passes) + 1):
257
+ cand = [(i, j) for i, j in pairs if assign[i] != assign[j]]
258
+ if not cand:
259
+ break
260
+ n_in = [sum(1 for a in assign if a == l) for l in range(1, n_lists + 1)]
261
+ S = {d: [sum(v for v, a in zip(V[d], assign, strict=True) if a == l)
262
+ for l in range(1, n_lists + 1)] for d in dims}
263
+ best = 0
264
+ tied = []
265
+ for i, j in cand:
266
+ la, lb = assign[i], assign[j]
267
+ delta = 0
268
+ for d in dims:
269
+ va, vb = V[d][i], V[d][j]
270
+ Sa, Sb = S[d][la - 1], S[d][lb - 1]
271
+ sha = totals[d] * n_in[la - 1]
272
+ shb = totals[d] * n_in[lb - 1]
273
+ delta += (abs((Sa - va + vb) * n_sets - sha)
274
+ + abs((Sb - vb + va) * n_sets - shb)
275
+ - abs(Sa * n_sets - sha)
276
+ - abs(Sb * n_sets - shb))
277
+ if delta < best:
278
+ best, tied = delta, [(i, j)]
279
+ elif delta == best and tied:
280
+ tied.append((i, j))
281
+ if best >= 0 or not tied:
282
+ break
283
+ if len(tied) > 1:
284
+ # Hash tie-break, not position: taking the first tied pair would
285
+ # systematically prefer low-numbered sets, and the digest is the package's
286
+ # established way of choosing without a generator.
287
+ def _h(pair):
288
+ key = "|".join([_key_part(seed), "balance",
289
+ _key_part(sets[pair[0]]), _key_part(sets[pair[1]])])
290
+ return hashlib.sha256(key.encode("utf-8")).hexdigest()
291
+ pick = sorted(tied, key=_h)[0]
292
+ else:
293
+ pick = tied[0]
294
+ i, j = pick
295
+ assign[i], assign[j] = assign[j], assign[i]
296
+ cost += best
297
+ n_swaps += 1
298
+ if pass_i == max_passes:
299
+ hit_bound = True
300
+
301
+ return {
302
+ "list_of_set": {s: int(a) for s, a in zip(sets, assign, strict=True)},
303
+ "report": {
304
+ "dimensions": list(dims), "cost_before": int(cost0),
305
+ "cost_after": int(cost), "n_swaps": int(n_swaps),
306
+ "max_passes_reached": hit_bound,
307
+ "cost_unit": ("summed absolute deviation of each list's dimension total "
308
+ "from its fair share, in milli-units of the dimension's "
309
+ "mean, scaled by the number of item sets"),
310
+ },
311
+ }
312
+
313
+
314
+ def counterbalance(stimuli: pd.DataFrame, design: dict, schema: dict,
315
+ list_of_set=None) -> pd.DataFrame:
316
+ # Resampled designs counterbalance each replicate (an independent item set)
317
+ # on its own, so trial order is numbered within each replicate.
318
+ if "replicate" in stimuli.columns and stimuli["replicate"].nunique() > 1:
319
+ parts = [_counterbalance_one(g.reset_index(drop=True), design, schema, list_of_set)
320
+ for _, g in stimuli.groupby("replicate", sort=True)]
321
+ return pd.concat(parts, ignore_index=True)
322
+ return _counterbalance_one(stimuli, design, schema, list_of_set)
323
+
324
+
325
+ def _counterbalance_one(stimuli: pd.DataFrame, design: dict, schema: dict,
326
+ list_of_set=None) -> pd.DataFrame:
327
+ if _recipe(design) == "latin_square_target":
328
+ return counterbalance_latin_square(stimuli, design, schema)
329
+ return counterbalance_factorial(stimuli, design, schema, list_of_set)
330
+
331
+
332
+ def counterbalance_factorial(stimuli: pd.DataFrame, design: dict, schema: dict,
333
+ list_of_set=None) -> pd.DataFrame:
334
+ n_lists = (design.get("counterbalance") or {}).get("lists", 1)
335
+ seed = schema.get("seed", 1)
336
+ stimuli = stimuli.copy()
337
+ stimuli["list"] = 1
338
+ if n_lists > 1:
339
+ sets = sorted(stimuli["set"].unique())
340
+ if list_of_set is not None:
341
+ # A balanced assignment from balance_lists(). Looked up by set VALUE, since
342
+ # the map is keyed by it; a set the map does not name is a caller error
343
+ # rather than a silent fall-back to list 1.
344
+ missing = sorted(set(stimuli["set"]) - set(list_of_set),
345
+ key=lambda s: str(s))
346
+ if missing:
347
+ raise ValueError(
348
+ "lexsync: the balanced list assignment does not cover set(s) %s."
349
+ % ", ".join(str(m) for m in missing))
350
+ mapping = dict(list_of_set)
351
+ else:
352
+ # Deal by set RANK, not by set value, so the deal does not depend on `set`
353
+ # being contiguous and starting at 1 (R's seq_along does the same).
354
+ mapping = {s: (i % n_lists) + 1 for i, s in enumerate(sets)}
355
+ stimuli["list"] = stimuli["set"].map(mapping)
356
+ parts = [_shuffle_deterministic(df, seed)
357
+ for _, df in stimuli.groupby("list", sort=True)]
358
+ return pd.concat(parts, ignore_index=True)
359
+
360
+
361
+ def counterbalance_latin_square(stimuli: pd.DataFrame, design: dict, schema: dict) -> pd.DataFrame:
362
+ """One row per item per list, condition rotated across lists (Latin square).
363
+
364
+ Each item (``set``) contributes exactly one trial to a list, so its target is
365
+ never repeated within a list; conditions are balanced because items rotate
366
+ through them. With ``lists`` unset the number of lists equals the number of
367
+ conditions, the canonical fully-counterbalanced design.
368
+ """
369
+ seed = schema.get("seed", 1)
370
+ conds = sorted(stimuli["condition"].unique(), key=lambda s: str(s).encode("utf-8"))
371
+ n_cond = len(conds)
372
+ n_lists = (design.get("counterbalance") or {}).get("lists", n_cond)
373
+ sets = sorted(stimuli["set"].unique())
374
+ parts = []
375
+ for li in range(n_lists):
376
+ rows = []
377
+ for si, s in enumerate(sets):
378
+ cond = conds[(si + li) % n_cond]
379
+ row = stimuli[(stimuli["set"] == s) & (stimuli["condition"] == cond)]
380
+ if len(row) == 0:
381
+ raise ValueError(f"lexsync: item set {s} has no row for condition '{cond}'.")
382
+ rows.append(row.iloc[0])
383
+ df = pd.DataFrame(rows).reset_index(drop=True)
384
+ df["list"] = li + 1
385
+ df = _shuffle_deterministic(df, seed).reset_index(drop=True)
386
+ parts.append(df)
387
+ return pd.concat(parts, ignore_index=True)
388
+
389
+
390
+ def participant_table(factors: dict, n_participants: int) -> pd.DataFrame:
391
+ keys = list(factors.keys())
392
+ # R's expand.grid() varies the first factor fastest, itertools.product the
393
+ # last; cross the reversed keys and unreverse each cell so a participant
394
+ # number is allocated the same cell by either engine.
395
+ grid = [cell[::-1] for cell in product(*[factors[k] for k in reversed(keys)])]
396
+ rows = [dict(zip(keys, grid[i % len(grid)], strict=True)) for i in range(n_participants)]
397
+ df = pd.DataFrame(rows)
398
+ df["participant"] = np.arange(1, n_participants + 1)
399
+ return df