lexsync 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,149 @@
1
+ # lexsync corpus registry
2
+ # ---------------------------------------------------------------------------
3
+ # Each entry documents an accessible, reusable and citable lexical corpus.
4
+ # status: validated -> bundled example slice + demonstrated end to end
5
+ # status: supported -> fetched on demand into the user cache (not redistributed)
6
+ # status: manual -> citation, licence and landing page verified live, but the
7
+ # publisher offers no delimited file, so a human retrieves it
8
+ # status: listed -> citation/licence/URL pending live verification
9
+ # A corpus is promoted to 'supported'/'validated' only after its citation has
10
+ # been verified against Crossref or the official record, and its licence checked.
11
+ #
12
+ # The two URL keys are not interchangeable:
13
+ # url -> the corpus's human-facing landing page (citation/provenance only)
14
+ # openlexicon -> a directly downloadable delimited file; fetch_corpus() requires this
15
+ #
16
+ # 'connector' selects the ingestion path:
17
+ # openlexicon -> a SUBTLEX-family/openlexicon delimited file (Connector A)
18
+ # wordfreq -> the wordfreq Python package, ~40 languages (Connector B)
19
+ # The 'columns' map names source columns onto the lexsync schema; 'freq_zipf'
20
+ # is computed from 'freq_permillion' when the source lacks a Zipf column
21
+ # (Zipf = log10(freq_per_million) + 3).
22
+
23
+ defaults:
24
+ licence: 'CC BY-SA 4.0'
25
+ connector: openlexicon
26
+ attribution_note: >-
27
+ SUBTLEX resources are made freely available by their authors for any
28
+ purpose subject to attribution (terms equivalent to CC BY-SA); openlexicon
29
+ redistributes them under CC BY-SA 4.0.
30
+
31
+ corpora:
32
+
33
+ subtlex_uk:
34
+ language: { name: 'English (British)', iso: en-GB }
35
+ connector: openlexicon
36
+ status: manual
37
+ url: 'https://psychology.nottingham.ac.uk/subtlex-uk/'
38
+ licence: 'CC BY-SA 4.0 (Brysbaert terms)'
39
+ citation_key: vanheuven2014subtlexuk
40
+ citation: 'van Heuven, Mandera, Keuleers & Brysbaert (2014). SUBTLEX-UK. Quarterly Journal of Experimental Psychology, 67(6), 1176-1190. doi:10.1080/17470218.2013.850521'
41
+ columns: { word: 'Spelling', freq_zipf: 'LogFreq(Zipf)', pos: 'DomPoS' }
42
+ notes: >-
43
+ Published only as zipped xlsx/txt archives, which the delimited-file
44
+ connector cannot ingest, so no 'openlexicon' key: download
45
+ SUBTLEX-UK.txt.zip from the landing page, unzip it, and pass the file to
46
+ load_lexicon(). The 'columns' map above names its columns.
47
+
48
+ subtlex_esp:
49
+ language: { name: 'Spanish', iso: es }
50
+ connector: openlexicon
51
+ status: listed
52
+ url: 'https://osf.io/xp6sz/'
53
+ licence: 'CC BY-SA 4.0 (Brysbaert terms)'
54
+ citation_key: cuetos2011subtlexesp
55
+ citation: 'Cuetos, González-Nosti, Barbón & Brysbaert (2011). SUBTLEX-ESP. Psicológica, 32(2), 133-143.'
56
+ columns: { word: 'Word', freq_permillion: 'FreqPerMillion', pos: 'pos' }
57
+
58
+ subtlex_us:
59
+ language: { name: 'English (American)', iso: en-US }
60
+ connector: openlexicon
61
+ status: listed
62
+ url: 'https://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexus'
63
+ licence: 'CC BY-SA 4.0 (Brysbaert terms)'
64
+ citation_key: brysbaert2009subtlexus
65
+ citation: 'Brysbaert & New (2009). Behavior Research Methods, 41(4), 977-990. doi:10.3758/BRM.41.4.977'
66
+ columns: { word: 'Word', freq_zipf: 'Lg10WF', pos: 'Dom_PoS_SUBTLEX' }
67
+
68
+ subtlex_nl:
69
+ language: { name: 'Dutch', iso: nl }
70
+ connector: openlexicon
71
+ status: listed
72
+ licence: 'CC BY-SA 4.0 (Brysbaert terms)'
73
+ citation_key: keuleers2010subtlexnl
74
+ citation: 'Keuleers, Brysbaert & New (2010). SUBTLEX-NL. Behavior Research Methods, 42(3), 643-650. doi:10.3758/BRM.42.3.643'
75
+ columns: { word: 'Word', freq_zipf: 'Zipf' }
76
+
77
+ subtlex_de:
78
+ language: { name: 'German', iso: de }
79
+ connector: openlexicon
80
+ status: listed
81
+ licence: 'CC BY-SA 4.0 (Brysbaert terms)'
82
+ citation_key: brysbaert2011subtlexde
83
+ citation: 'Brysbaert et al. (2011). SUBTLEX-DE. Experimental Psychology, 58(5), 412-424. doi:10.1027/1618-3169/a000123'
84
+ columns: { word: 'Word', freq_zipf: 'lgSUBTLEX' }
85
+
86
+ lexique_fr:
87
+ language: { name: 'French', iso: fr }
88
+ connector: openlexicon
89
+ status: listed
90
+ licence: 'CC BY-SA 4.0'
91
+ citation_key: new2004lexique
92
+ citation: 'New, Pallier, Brysbaert & Ferrand (2004). Lexique 2. Behavior Research Methods, Instruments, & Computers, 36(3), 516-524. doi:10.3758/BF03195598'
93
+ columns: { word: 'ortho', freq_permillion: 'freqfilms2' }
94
+
95
+ subtlex_it:
96
+ language: { name: 'Italian', iso: it }
97
+ connector: openlexicon
98
+ status: listed
99
+ licence: 'CC BY-SA 4.0 (Brysbaert terms)'
100
+ citation_key: crepaldi2015subtlexit
101
+ citation: 'Crepaldi, Amenta, Mandera, Keuleers & Brysbaert (2015). SUBTLEX-IT. (verify on ingestion).'
102
+ columns: { word: 'spelling', freq_zipf: 'zipf' }
103
+
104
+ subtlex_ch:
105
+ language: { name: 'Chinese (Mandarin)', iso: zh }
106
+ connector: openlexicon
107
+ status: listed
108
+ licence: 'CC BY-SA 4.0 (Brysbaert terms)'
109
+ citation_key: cai2010subtlexch
110
+ citation: 'Cai & Brysbaert (2010). SUBTLEX-CH. PLoS ONE, 5(6), e10729. doi:10.1371/journal.pone.0010729'
111
+ columns: { word: 'Word', freq_zipf: 'WCpmw' }
112
+ notes: 'Character-based; length/N/OLD20 semantics differ for logographic scripts.'
113
+
114
+ subtlex_pl:
115
+ language: { name: 'Polish', iso: pl }
116
+ connector: openlexicon
117
+ status: listed
118
+ licence: 'CC BY-SA 4.0 (Brysbaert terms)'
119
+ citation_key: mandera2015subtlexpl
120
+ citation: 'Mandera, Keuleers, Wodniecka & Brysbaert (2015). SUBTLEX-PL. Behavior Research Methods, 47(2), 471-483. doi:10.3758/s13428-014-0489-4'
121
+ columns: { word: 'spelling', freq_zipf: 'zipf' }
122
+
123
+ subtlex_pt:
124
+ language: { name: 'Portuguese (European)', iso: pt }
125
+ connector: openlexicon
126
+ status: listed
127
+ licence: 'CC BY-SA 4.0 (Brysbaert terms)'
128
+ citation_key: soares2015subtlexpt
129
+ citation: 'Soares, Machado, Costa, Iriarte, Simoes, de Almeida, Comesana & Perea (2015). On the advantages of word frequency and contextual diversity measures extracted from subtitles: The case of Portuguese. Quarterly Journal of Experimental Psychology, 68(4), 680-696. doi:10.1080/17470218.2014.964271'
130
+ columns: { word: 'word', freq_zipf: 'zipf' }
131
+
132
+ # Connector B: one dependency, many languages.
133
+ wordfreq_connector:
134
+ connector: wordfreq
135
+ status: supported
136
+ description: 'Programmatic word frequencies for ~40 languages via the wordfreq package; lexsync derives length, N and OLD20 itself.'
137
+ # Two different licences, and the data one is what matters for redistribution:
138
+ # wordfreq's code is Apache-2.0 (it was MIT up to 3.0.2 and relicensed at
139
+ # 3.0.3, which the >=3.0,<4 pin spans), while its data files are
140
+ # redistributable under CC BY-SA 4.0. The full record, including the SUBTLEX
141
+ # credit that wordfreq's terms require of anything derived from it, is at the
142
+ # link below. It is spelled out in full because a bare filename resolves only
143
+ # for someone reading the repository, and this file also ships inside the
144
+ # installed package, where no LICENSE-DATA sits beside it.
145
+ # https://github.com/pablobernabeu/lexsync/blob/main/LICENSE-DATA
146
+ licence: 'Apache-2.0 (package code); CC BY-SA 4.0 (data files)'
147
+ citation_key: speer2022wordfreq
148
+ citation: 'Speer (2022). rspeer/wordfreq v3.0. Zenodo. doi:10.5281/zenodo.7199437'
149
+ languages: [en, es, fr, de, nl, it, pt, pl, el, ru, uk, ar, fa, he, hi, bn, zh, ja, ko, id, ms, tr, sv, nb, da, fi, cs, ro, hu, ca]
@@ -0,0 +1,193 @@
1
+ # lexsync global schema and defaults
2
+ # ---------------------------------------------------------------------------
3
+ # Every numeric constant lives here (and in the per-design configs), so adding a
4
+ # language or a design needs a configuration file, not a change to the code.
5
+ # This file is read identically by the R package and the Python package.
6
+
7
+ seed: 2026
8
+
9
+ # ---- Lexicon column contract ----------------------------------------------
10
+ # Derived lexica (corpora/derived/*.csv and the bundled example slices) must
11
+ # expose the 'required' columns; 'optional' columns are used when present.
12
+ lexicon_schema:
13
+ required:
14
+ - word # orthographic form, UTF-8, lower-case
15
+ - freq_zipf # Zipf frequency (van Heuven et al., 2014); roughly 1-7
16
+ optional:
17
+ - freq_permillion # frequency per million tokens
18
+ - pos # part of speech (source tag)
19
+ - language # ISO 639-1 / BCP-47 code
20
+ - source # corpus identifier (matches corpora/registry.yaml)
21
+
22
+ # ---- Lexical dimensions ----------------------------------------------------
23
+ # 'column' dimensions are read straight from the lexicon; 'derived' dimensions
24
+ # are computed at load time from the orthographic form, identically in R and
25
+ # Python (Coltheart's N and OLD20 from Levenshtein distances).
26
+ dimensions:
27
+ length: { type: derived, compute: n_letters, unit: 'characters' }
28
+ frequency: { type: column, column: freq_zipf, unit: 'Zipf' }
29
+ n_density: { type: derived, compute: coltheart_n, unit: 'neighbours' }
30
+ old20: { type: derived, compute: old20, unit: 'mean Levenshtein distance' }
31
+ n_syllables: { type: derived, compute: vowel_groups, unit: 'syllables (orthographic estimate)' }
32
+ bigram_freq: { type: derived, compute: mean_bigram_probability, unit: 'mean bigram probability (type-based, non-positional)' }
33
+
34
+ # ---- Semantic and other external dimensions --------------------------------
35
+ # The dimensions above are computed from the orthographic form or read from the
36
+ # lexicon. A dimension that cannot be derived from the form -- concreteness, age of
37
+ # acquisition, valence, imageability -- comes from a published norm table, which is
38
+ # joined per design rather than declared here, because which norms a design needs
39
+ # depends on what it manipulates and because their licences differ:
40
+ #
41
+ # norms:
42
+ # - path: norms/en_concreteness.csv
43
+ # on: word # optional join key, default 'word'
44
+ # columns: [concreteness] # optional; default every column but the key
45
+ #
46
+ # The tables are joined onto the lexicon before the candidate pool is built, so
47
+ # `pool_filters`, `match_on` and a continuous `predictor` may all name a norm column.
48
+ # The join is case-insensitive and whitespace-insensitive on both sides, keeps the
49
+ # lexicon's own row and column order, and refuses a norm column whose name already
50
+ # exists on the lexicon rather than silently renaming it.
51
+ #
52
+ # lexsync does not bundle norm data. Fetch it from the source, whose licence and
53
+ # citation are the user's to honour, and record the file: every joined table is
54
+ # written into the materials datasheet with its sha256 and its per-column coverage,
55
+ # because a norm table can carry the very variable a design manipulates, and a
56
+ # selection over columns of unstated origin is not reproducible. Coverage matters
57
+ # too -- a word the table does not cover gets a missing value, and the tolerance
58
+ # windows then drop it from the pool.
59
+
60
+ # ---- Default matching configuration ---------------------------------------
61
+ # Pseudoword generation (the lexical_decision paradigm) is chosen per design under
62
+ # items.generation.method:
63
+ # letter_substitution change the fewest single letters, keeping bigrams attested (default)
64
+ # subsyllabic swap whole onset/nucleus/coda constituents (Wuggy-style; Keuleers
65
+ # & Brysbaert, 2010)
66
+ # Both preserve length and select byte-identical stimuli across the R and Python engines.
67
+
68
+ matching:
69
+ method: standardised_euclidean
70
+ # method options:
71
+ # standardised_euclidean greedy nearest-neighbour on z-scored dimensions (default)
72
+ # joint optimal-greedy pairing for a two-condition design
73
+ # mahalanobis covariance-aware distance; down-weights correlated dimensions
74
+ # optimal globally optimal (assignment) pairing for two conditions
75
+ # standardised_euclidean and joint are byte-identical across the R and Python
76
+ # engines; mahalanobis and optimal use a covariance inverse / assignment solver,
77
+ # so the two engines agree closely but not byte-for-byte.
78
+ # Stage 1, tolerance pre-filter: keep candidates within mean +/- k * SD of the
79
+ # target condition on each dimension. This generalises the original workflow's
80
+ # mean +/- SD/9 (frequency) and mean +/- 2*SD (letters) windows.
81
+ tolerance_k:
82
+ length: 2.0
83
+ frequency: 1.0
84
+ n_density: 2.0
85
+ old20: 2.0
86
+ # Selection-size policies; a design may override either under its own matching:
87
+ # block. shortfall errors by default because the datasheet and the generated
88
+ # methods paragraph state the requested n, so a silently smaller set would
89
+ # leave the record misstating the materials.
90
+ shortfall: error # what to do when fewer sets than n_per_condition can be selected: error | allow
91
+ on_insufficient_tolerance: relax # when a condition has too few candidates inside the tolerance window: relax | error
92
+ # Stage 2 ranks candidates by standardised multivariate distance; ties are
93
+ # broken by (distance, word bytes, id), fixed in matching.py / matching.R.
94
+ # That order is not configurable: the byte-order tie-break is what makes the
95
+ # two engines select identical stimuli without an RNG.
96
+
97
+ # ---- Counterbalancing list assignment --------------------------------------
98
+ # The factorial recipe deals item sets to lists by rank: set 1 to list 1, set 2 to
99
+ # list 2, and so on. That is reproducible but arbitrary with respect to the items, so
100
+ # one list can end up holding systematically shorter or more frequent words than
101
+ # another, which is confounded with the group of participants that receives it.
102
+ #
103
+ # A design may ask for a balanced assignment instead:
104
+ #
105
+ # counterbalance:
106
+ # lists: 4
107
+ # optimise: true
108
+ # balance_on: [length, n_density, old20] # optional; defaults to match_on
109
+ # max_passes: 500 # optional bound on the search
110
+ #
111
+ # It is OFF by default, and deliberately so: switching it on changes which items a
112
+ # participant sees, which is a design decision and not something a package upgrade
113
+ # should do to a study that is already running.
114
+ #
115
+ # The search exchanges pairs of item sets between lists, taking the single best
116
+ # exchange each pass until none improves the balance. It is a steepest descent to a
117
+ # local optimum, not a global search, and the datasheet records the imbalance before
118
+ # and after so the result is checkable rather than asserted. List sizes are preserved.
119
+ #
120
+ # There is no random number generator and no floating-point score anywhere in it. The
121
+ # dimensions are quantised to integers once, in milli-units of each dimension's own
122
+ # mean so that dimensions on different scales carry comparable weight, and the search
123
+ # then uses only integer arithmetic and comparison; ties are broken by the same seeded
124
+ # keyed hash that fixes trial order. A float objective was measured to differ between
125
+ # the engines by around 3e-16, which is enough to send them down different search
126
+ # paths and select different words.
127
+ #
128
+ # It does not apply to the Latin-square recipe and is refused there: every item
129
+ # appears in every list already, so the lists are balanced on the items by
130
+ # construction and the rotation decides only which condition each item takes.
131
+
132
+ # ---- Equivalence testing (reported in the descriptives) -------------------
133
+ # Equivalence is assessed by two one-sided tests (TOST), fixed in validation.py /
134
+ # validation.R; only the bound and alpha below are configurable.
135
+ equivalence:
136
+ bound_d: 0.5 # smallest effect size of interest, Cohen's d (Lakens, 2017)
137
+ alpha: 0.05
138
+
139
+ # ---- Hardware triggers (defaults; overridable per design) -----------------
140
+ # Triggers are reset to 0 after reset_after_frames; the reset value is fixed in
141
+ # the emitted PsychoPy and OpenSesame code (scripting.py / scripting.R).
142
+ triggers:
143
+ parallel_address: '0x0378' # typical LPT1 base address
144
+ # How long the onset code is held before it is reset to 0, in milliseconds.
145
+ #
146
+ # This was `reset_after_frames: 2`, documented as '~33 ms at 60 Hz'. It was
147
+ # neither: the reset is queued on one flip and executes on the next, so the code
148
+ # was actually held across three flip intervals, 50 ms at 60 Hz. And being a
149
+ # frame count it scaled with the display, falling to 12.5 ms at 240 Hz and 8.3 ms
150
+ # at 360 Hz, below the 10 ms minimum most recorders need to register a pulse.
151
+ #
152
+ # 50 ms preserves exactly what the package has always done at 60 Hz, now stated
153
+ # honestly and in a unit that does not move. The generated script converts it to
154
+ # whole flips against the refresh it measures, never fewer than one flip and
155
+ # never less than 10 ms. `reset_after_frames` is still accepted and converted at
156
+ # presentation.assumed_refresh_hz.
157
+ trigger_hold_ms: 50
158
+ inter_trigger_ms: 10 # spacing of trailing markers (matches the original)
159
+
160
+ # ---- Presentation defaults (overridable per design) -----------------------
161
+ # The word font used by the generated PsychoPy and OpenSesame experiments. The
162
+ # default is a fixed-width Latin font; set a glyph-complete font per design for
163
+ # non-Latin scripts (for example 'SimHei' for Chinese; see design_zh_*.yaml).
164
+ presentation:
165
+ font: 'Courier New'
166
+ opensesame_font: 'mono' # OpenSesame font_family token (space-free)
167
+ # Event durations are canonically MILLISECONDS. A design may still declare a
168
+ # duration in frames (`duration_frames`, and `fixation_frames` / `word_frames` /
169
+ # `isi_frames` in a design's `timing:` block), which is converted to
170
+ # milliseconds at load time using the rate below, so designs written before the
171
+ # millisecond migration keep working unchanged. Set it to the rate those frame
172
+ # counts were authored for.
173
+ #
174
+ # This is NOT the rate the experiment runs at. The generated PsychoPy script
175
+ # measures the display's actual refresh at start-up and converts milliseconds
176
+ # back into whole flips, so a design presents its intended duration on a 60,
177
+ # 120 or 144 Hz monitor alike while the trigger stays bound to the flip. The
178
+ # script falls back to this value only if that measurement fails.
179
+ assumed_refresh_hz: 60
180
+ #
181
+ # An event's duration may also vary from trial to trial, declared per event in
182
+ # a design rather than here:
183
+ #
184
+ # duration: {from_column: soa_ms} read per trial from the items
185
+ # duration: {jitter: [400, 800], as: iti_ms} drawn per trial, uniform, in ms
186
+ #
187
+ # A drawn value is not a random number. It is a uniform integer keyed on the
188
+ # seed, the column name, the list, the set and the condition, so both engines
189
+ # realise the same milliseconds and a rerun reproduces them. Naming the column
190
+ # in the key is what makes two jittered events draw independently rather than
191
+ # sharing a value. Either form writes the realised milliseconds into the
192
+ # stimuli table, so timing that varies is available to the analysis as a
193
+ # regressor instead of being undocumented noise. See design_en_priming_jitter.