lexsync 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
lexsync/__init__.py ADDED
@@ -0,0 +1,53 @@
1
+ """lexsync: lexical optimisation and hardware-timed experiment generation."""
2
+ from .corpora import fetch_corpus, list_corpora
3
+ from .counterbalancing import balance_lists, counterbalance, participant_table
4
+ from .datasheet import build_datasheet, methods_paragraph, write_datasheet
5
+ from .generation import build_lexdec_stimuli, generate_pseudowords, make_pseudoword
6
+ from .matching import match_stimuli, resample_stimuli, select_continuous_stimuli
7
+ from .paradigms import PARADIGMS, required_fields, resolve_events
8
+ from .querying import (
9
+ add_bigram_frequency,
10
+ add_neighbourhood,
11
+ add_pair_overlap,
12
+ build_pool,
13
+ count_syllables,
14
+ load_items,
15
+ load_lexicon,
16
+ load_pool,
17
+ merge_norms,
18
+ )
19
+ from .run_pipeline import run_all, run_pipeline
20
+ from .scripting import (
21
+ export_experiments,
22
+ export_jspsych,
23
+ export_opensesame,
24
+ export_psychopy,
25
+ resolve_trial_timing,
26
+ )
27
+ from .validation import (
28
+ balance_check,
29
+ cohens_d,
30
+ cohens_d_ci,
31
+ describe_stimuli,
32
+ match_report,
33
+ match_report_continuous,
34
+ tost_equiv,
35
+ variance_ratio,
36
+ )
37
+
38
+ __all__ = [
39
+ "load_lexicon", "load_items", "load_pool", "add_neighbourhood", "add_bigram_frequency",
40
+ "count_syllables", "merge_norms", "build_pool", "match_stimuli",
41
+ "resample_stimuli", "select_continuous_stimuli", "make_pseudoword",
42
+ "generate_pseudowords", "build_lexdec_stimuli",
43
+ "PARADIGMS", "resolve_events", "required_fields",
44
+ "counterbalance", "balance_lists", "participant_table", "describe_stimuli", "cohens_d",
45
+ "cohens_d_ci", "tost_equiv", "variance_ratio", "balance_check", "match_report",
46
+ "match_report_continuous", "export_experiments",
47
+ "export_psychopy", "export_opensesame", "export_jspsych", "resolve_trial_timing",
48
+ "add_pair_overlap",
49
+ "list_corpora", "fetch_corpus",
50
+ "build_datasheet", "methods_paragraph", "write_datasheet",
51
+ "run_pipeline", "run_all",
52
+ ]
53
+ __version__ = "0.1.1"
lexsync/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ """Enable ``python -m lexsync ...`` as an alias for the ``lexsync`` console script."""
2
+ from .cli import main
3
+
4
+ if __name__ == "__main__":
5
+ main()
lexsync/blocks.py ADDED
@@ -0,0 +1,149 @@
1
+ """Practice and filler trials: rows that are PRESENTED but not ANALYSED.
2
+
3
+ Everything else in the package treats one frame as both the materials record and the
4
+ thing the experiment runs. That works only while the two are the same set of trials, and
5
+ they usually are not. A practice block exists to settle the participant into the task and
6
+ is discarded before analysis; fillers exist to dilute the manipulation so it is less
7
+ guessable, and are likewise not analysed. Both have to reach the generated experiment and
8
+ neither belongs in the stimuli file, the descriptives or the realised control.
9
+
10
+ The pipeline therefore splits after counterbalancing: the analysis artefacts are
11
+ written from the main rows, and the experiment is generated from the presented rows. A ``block``
12
+ column marks which is which, and it appears ONLY when a design declares one of these
13
+ blocks -- a design without them keeps exactly the columns it had.
14
+
15
+ Where each block goes in the sequence is a methodological choice, not a convenience:
16
+
17
+ Practice comes first, as its own run of trials, because its purpose is to precede the
18
+ task. It is shuffled within itself so participants do not all meet the practice items in
19
+ one order.
20
+
21
+ Fillers are INTERLEAVED with the main trials, not appended, because a block of fillers at
22
+ the end is not a filler -- it is a second block that participants can tell apart. They
23
+ are merged into each list before the order is drawn, so one deterministic shuffle mixes
24
+ them through. That does renumber the main trials, which is correct: adding fillers
25
+ changes the sequence, and the stimuli file records where each item actually appeared.
26
+
27
+ Both blocks appear in EVERY list. They are not counterbalanced, because they carry no
28
+ manipulation to rotate; every participant should get the same practice.
29
+
30
+ Mirrors R_workflow/R/blocks.R.
31
+ """
32
+ from __future__ import annotations
33
+
34
+ import pandas as pd
35
+
36
+ from .counterbalancing import _shuffle_deterministic
37
+ from .io_utils import sha256_file
38
+ from .paradigms import required_fields
39
+ from .querying import load_items
40
+
41
+ BLOCK_MAIN = "main"
42
+
43
+
44
+ def _concat_blocks(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame:
45
+ """Concatenate two block frames that need not have the same columns.
46
+
47
+ A filler table carries no ``frequency`` or ``old20``, and the main rows carry no
48
+ filler-specific field; the union is taken and the gaps left missing, which is honest
49
+ -- a filler has no matched frequency because it was never matched.
50
+
51
+ The key columns are put back to their original dtypes afterwards. This is not
52
+ cosmetic: pandas promotes an integer column to float the moment a missing value
53
+ enters it, and the trial-order shuffle formats an integer `set` as "3" but a float
54
+ one as "3.0", which would change every digest and every trial order with nothing to
55
+ signal it.
56
+ """
57
+ cols = list(a.columns) + [c for c in b.columns if c not in a.columns]
58
+ dtypes = {c: a[c].dtype for c in a.columns if str(a[c].dtype).startswith("int")}
59
+ dtypes.update({c: b[c].dtype for c in b.columns
60
+ if c not in dtypes and str(b[c].dtype).startswith("int")})
61
+ out = pd.concat([a.reindex(columns=cols), b.reindex(columns=cols)],
62
+ ignore_index=True)
63
+ for c, dt in dtypes.items():
64
+ if c in out.columns and out[c].notna().all():
65
+ out[c] = out[c].astype(dt)
66
+ return out
67
+
68
+
69
+ def _load_block(cfg: dict, design: dict, label: str, set_offset: int) -> pd.DataFrame:
70
+ """Read one block's item table and give it a block label and a non-colliding ``set``.
71
+
72
+ The offset matters more than it looks. ``load_items`` numbers sets from 1 within
73
+ whatever table it is given, so practice item 1 and main item 1 would both be set 1 --
74
+ and ``set`` is part of the key the trial-order shuffle hashes. Two rows sharing a key
75
+ would be ordered by a coin the package does not own.
76
+ """
77
+ path = cfg.get("path")
78
+ if not path:
79
+ raise ValueError(
80
+ "lexsync: the `%s:` block needs a `path` to an item table." % label)
81
+ df = load_items(path, required_fields(design)).copy()
82
+ df["block"] = label
83
+ df["set"] = df["set"].astype(int) + int(set_offset)
84
+ return df.reset_index(drop=True)
85
+
86
+
87
+ def add_blocks(stimuli: pd.DataFrame, design: dict, schema: dict) -> dict:
88
+ """Assemble the presented trial sequence from the main, filler and practice blocks.
89
+
90
+ Returns ``{"presented": DataFrame, "report": dict | None}``: every trial the
91
+ experiment runs, in order, with a ``block`` column when more than one block exists;
92
+ and the per-block counts and item-table checksums, or None when the design declares
93
+ no extra block.
94
+ """
95
+ practice_cfg = design.get("practice")
96
+ filler_cfg = design.get("fillers")
97
+ if not practice_cfg and not filler_cfg:
98
+ # No block column at all: a design that declares none must keep the columns it
99
+ # had, so adding this feature moves no existing artefact.
100
+ return {"presented": stimuli, "report": None}
101
+
102
+ seed = schema.get("seed", 1)
103
+ stimuli = stimuli.copy()
104
+ stimuli["block"] = BLOCK_MAIN
105
+ if "list" not in stimuli.columns:
106
+ stimuli["list"] = 1
107
+ lists = sorted(stimuli["list"].unique())
108
+ offset = int(max([0] + [int(s) for s in stimuli["set"]]))
109
+
110
+ fillers = None
111
+ if filler_cfg:
112
+ fillers = _load_block(filler_cfg, design, "filler", offset)
113
+ offset = max(offset, int(fillers["set"].max()))
114
+ practice = None
115
+ if practice_cfg:
116
+ practice = _load_block(practice_cfg, design, "practice", offset)
117
+
118
+ parts = []
119
+ for li in lists:
120
+ body = stimuli[stimuli["list"] == li].reset_index(drop=True)
121
+ if fillers is not None:
122
+ f = fillers.copy()
123
+ f["list"] = li
124
+ body = _concat_blocks(body, f)
125
+ # One shuffle over main and fillers together is what interleaves them.
126
+ body = _shuffle_deterministic(body, seed).reset_index(drop=True)
127
+ if practice is not None:
128
+ p = practice.copy()
129
+ p["list"] = li
130
+ p = _shuffle_deterministic(p, seed).reset_index(drop=True)
131
+ body = _concat_blocks(p, body)
132
+ body["trial"] = range(1, len(body) + 1)
133
+ parts.append(body)
134
+ out = pd.concat(parts, ignore_index=True)
135
+
136
+ n_main = int((stimuli["list"] == lists[0]).sum())
137
+ report = {"blocks": [{"block": BLOCK_MAIN, "n_per_list": n_main}]}
138
+ if fillers is not None:
139
+ report["blocks"].append({
140
+ "block": "filler", "n_per_list": int(len(fillers)),
141
+ "path": filler_cfg["path"], "sha256": sha256_file(filler_cfg["path"]),
142
+ "placement": "interleaved with the main trials by the seeded order"})
143
+ if practice is not None:
144
+ report["blocks"].append({
145
+ "block": "practice", "n_per_list": int(len(practice)),
146
+ "path": practice_cfg["path"], "sha256": sha256_file(practice_cfg["path"]),
147
+ "placement": "before the main trials"})
148
+ report["analysed"] = BLOCK_MAIN
149
+ return {"presented": out, "report": report}
lexsync/cli.py ADDED
@@ -0,0 +1,85 @@
1
+ """Command-line interface: ``lexsync run | corpora list | fetch``."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import os
6
+
7
+ from .corpora import fetch_corpus, list_corpora
8
+ from .run_pipeline import run_all, run_pipeline
9
+
10
+ # _resolve_schema treats this exact value as "the user asked for nothing in
11
+ # particular", which is what licenses the bundled-copy fallback; an explicit
12
+ # --schema is honoured or refused, never substituted.
13
+ DEFAULT_SCHEMA = "config/schema.yaml"
14
+
15
+
16
+ def _version() -> str:
17
+ # importlib.metadata knows only installed distributions, so a source-tree
18
+ # invocation (sys.path pointing at src/) has no version record to read;
19
+ # "unknown" keeps --version working there instead of crashing.
20
+ from importlib.metadata import PackageNotFoundError, version
21
+ try:
22
+ return version("lexsync")
23
+ except PackageNotFoundError:
24
+ return "unknown"
25
+
26
+
27
+ def _resolve_schema(schema_path: str) -> str:
28
+ """Resolve --schema for the installed CLI as well as for a checkout.
29
+
30
+ The default is checkout-relative, so on its own it strands an installed CLI
31
+ run from anywhere else. A byte-identical copy of config/schema.yaml ships
32
+ inside the package (the identity is test-enforced), and the default falls
33
+ back to it when the checkout-relative file is absent. An explicit path
34
+ still errors: silently substituting the bundled schema there would run a
35
+ different file from the one the user named.
36
+ """
37
+ if os.path.exists(schema_path):
38
+ return schema_path
39
+ if schema_path == DEFAULT_SCHEMA:
40
+ from importlib.resources import files
41
+ bundled = files("lexsync") / "data" / "schema.yaml"
42
+ if bundled.is_file():
43
+ return str(bundled)
44
+ raise FileNotFoundError(
45
+ f"lexsync: schema not found: '{schema_path}'. "
46
+ "Run from a checkout root or pass --schema.")
47
+
48
+
49
+ def main(argv=None) -> None:
50
+ parser = argparse.ArgumentParser(
51
+ prog="lexsync",
52
+ description="lexsync: lexical optimisation and hardware-timed experiment generation",
53
+ )
54
+ parser.add_argument("--version", action="version", version="lexsync " + _version())
55
+ sub = parser.add_subparsers(dest="command")
56
+
57
+ run = sub.add_parser("run", help="run one design, or all designs in --config-dir")
58
+ run.add_argument("design", nargs="?", help="path to a design YAML (omit to run all)")
59
+ run.add_argument("--schema", default=DEFAULT_SCHEMA)
60
+ run.add_argument("--config-dir", default="config")
61
+ run.add_argument("--outdir", default="output")
62
+
63
+ corpora = sub.add_parser("corpora", help="corpus registry operations")
64
+ corpora.add_argument("action", choices=["list"])
65
+
66
+ fetch = sub.add_parser("fetch", help="fetch a corpus by registry name or language code")
67
+ fetch.add_argument("name")
68
+
69
+ args = parser.parse_args(argv)
70
+ if args.command == "run":
71
+ schema = _resolve_schema(args.schema)
72
+ if args.design:
73
+ run_pipeline(args.design, schema, args.outdir)
74
+ else:
75
+ run_all(args.config_dir, schema, args.outdir)
76
+ elif args.command == "corpora":
77
+ print(list_corpora().to_string(index=False))
78
+ elif args.command == "fetch":
79
+ print(fetch_corpus(args.name))
80
+ else:
81
+ parser.print_help()
82
+
83
+
84
+ if __name__ == "__main__":
85
+ main()
lexsync/corpora.py ADDED
@@ -0,0 +1,331 @@
1
+ """Access to the many-language corpus registry.
2
+
3
+ Mirrors R_workflow/R/corpora.R, and adds the wordfreq connector (Connector B,
4
+ ~40 languages). The demonstrations read the bundled, pre-derived lexica; new
5
+ languages are fetched on demand into a user cache.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import contextlib
10
+ import os
11
+ import re
12
+ import shutil
13
+ import time
14
+
15
+ import numpy as np
16
+ import pandas as pd
17
+ import yaml
18
+
19
+ from .io_utils import sha256_file
20
+
21
+
22
+ def _registry_path(registry_path: str | None = None) -> str:
23
+ candidates = [
24
+ registry_path,
25
+ os.environ.get("LEXSYNC_REGISTRY"),
26
+ os.path.join("corpora", "registry.yaml"),
27
+ os.path.join("..", "corpora", "registry.yaml"),
28
+ os.path.join(os.path.dirname(__file__), "data", "registry.yaml"),
29
+ ]
30
+ for cand in candidates:
31
+ if cand and os.path.exists(cand):
32
+ return cand
33
+ raise FileNotFoundError("lexsync: could not locate 'registry.yaml'; set LEXSYNC_REGISTRY.")
34
+
35
+
36
+ def list_corpora(registry_path: str | None = None) -> pd.DataFrame:
37
+ with open(_registry_path(registry_path), encoding="utf-8") as handle:
38
+ reg = yaml.safe_load(handle)
39
+ rows = []
40
+ for name, entry in (reg.get("corpora") or {}).items():
41
+ lang = entry.get("language") or {}
42
+ rows.append(dict(
43
+ name=name, language=lang.get("name"), iso=lang.get("iso"),
44
+ status=entry.get("status"), connector=entry.get("connector", "openlexicon"),
45
+ citation=entry.get("citation"),
46
+ ))
47
+ return pd.DataFrame(rows)
48
+
49
+
50
+ def cache_dir() -> str:
51
+ """Per-user cache directory for fetched corpora.
52
+
53
+ Where :func:`fetch_corpus` puts what it fetches, and the only place the
54
+ package writes to without being handed a path. This function only reports
55
+ the path. The directory is created by the first fetch into it, so until then
56
+ it does not exist.
57
+
58
+ The cache persists between sessions. A registered corpus is a delimited word
59
+ list, and a download is refused above 200 MB, so a cache holding several
60
+ large corpora can reach a few hundred megabytes. Each fetch into the cache
61
+ first deletes any partial download that an interrupted transfer left there
62
+ more than a day earlier, and fetching a corpus again replaces the earlier
63
+ copy. Fetched corpora otherwise stay until :func:`cache_clear` removes them,
64
+ one corpus at a time or all together. Nothing kept there is irreplaceable, so
65
+ the next call downloads afresh. The R twin documents the same contract in
66
+ lexsync_cache_dir.Rd; only the location differs, since R uses
67
+ tools::R_user_dir.
68
+ """
69
+ return os.path.join(os.path.expanduser("~"), ".lexsync", "cache")
70
+
71
+
72
+ def cache_clear(name: str | None = None) -> list[str]:
73
+ """Remove fetched corpora from the cache.
74
+
75
+ Deletes what :func:`fetch_corpus` put in :func:`cache_dir`. Given a `name`,
76
+ that is the corpus's ``<name>.csv``, any ``<name>.csv.part`` that an
77
+ interrupted download left behind and the ``<name>_wordfreq.csv`` lexicon the
78
+ wordfreq connector builds. With `name` left as None it is the whole cache
79
+ directory, and the ``~/.lexsync`` directory above it once nothing else is left
80
+ there. A corpus needed again is fetched afresh by the next
81
+ :func:`fetch_corpus` call. Mirrors lexsync_cache_clear() in
82
+ R_workflow/R/corpora.R, whose engine has no wordfreq connector and so no
83
+ ``_wordfreq.csv`` to remove.
84
+
85
+ Returns the paths of the files removed, empty when there was nothing to
86
+ remove. Raises OSError, naming them, when files survive the attempt (one held
87
+ open on Windows, say).
88
+ """
89
+ cache = cache_dir()
90
+ if name is None:
91
+ if not os.path.isdir(cache):
92
+ return []
93
+ removed = sorted(os.path.join(root, fname)
94
+ for root, _dirs, files in os.walk(cache) for fname in files)
95
+ targets = [cache]
96
+ # Keep going past a file that cannot be deleted, as R's unlink() does, and
97
+ # report what survives below.
98
+ shutil.rmtree(cache, ignore_errors=True)
99
+ else:
100
+ # The name becomes a file name inside the cache, so a path separator, or
101
+ # the colon of a Windows drive such as 'D:', would let it reach a file
102
+ # other than the corpus's own: os.path.join() drops the cache directory
103
+ # before a drive. All three characters are refused on every platform, so
104
+ # the two engines refuse the same names.
105
+ if not isinstance(name, str) or not name or re.search(r"[/\\:]", name):
106
+ raise ValueError("lexsync: 'name' must be a single corpus name, not a path.")
107
+ targets = [os.path.join(cache, f"{name}{suffix}")
108
+ for suffix in (".csv", ".csv.part", "_wordfreq.csv")]
109
+ removed = [path for path in targets if os.path.isfile(path)]
110
+ for path in removed:
111
+ with contextlib.suppress(OSError):
112
+ os.remove(path)
113
+ left = [path for path in (targets if name is None else removed)
114
+ if os.path.exists(path)]
115
+ if left:
116
+ raise OSError(f"lexsync: could not remove {', '.join(left)} from the cache; "
117
+ f"check that no other process is using it.")
118
+ if name is None:
119
+ # ~/.lexsync holds nothing but the cache, so an emptied one would be left
120
+ # behind in the user's home. rmdir() removes it only when it is empty. R's
121
+ # twin has no such step: the parent of its cache directory is shared by
122
+ # every R package's cache.
123
+ with contextlib.suppress(OSError):
124
+ os.rmdir(os.path.dirname(cache))
125
+ return removed
126
+
127
+
128
+ def _prepare_cache() -> str:
129
+ """Ready the cache for a fetch into it, and return its path.
130
+
131
+ Mirrors .prepare_cache() in R_workflow/R/corpora.R. A '.part' sidecar
132
+ outlives its fetch only when the transfer was cut short in a way
133
+ fetch_corpus()'s handlers never see: a KeyboardInterrupt derives from
134
+ BaseException, which neither except clause catches, and a process that dies
135
+ runs no handler at all. A sidecar can also belong to a download still running
136
+ in another process, which writes to it as it goes and gives up on a stalled
137
+ server after 60 s. A day without a write therefore marks a sidecar as
138
+ abandoned, and only those are deleted. A younger one is left to its download,
139
+ and a retry of the same corpus truncates its own.
140
+ """
141
+ cache = cache_dir()
142
+ os.makedirs(cache, exist_ok=True)
143
+ cutoff = time.time() - _STALE_PART_SECONDS
144
+ for fname in os.listdir(cache):
145
+ if fname.endswith(".part"):
146
+ path = os.path.join(cache, fname)
147
+ # Best effort, as R's unlink() is: a sidecar that cannot be deleted
148
+ # (held open on Windows, say), or that its download renamed away
149
+ # meanwhile, must not stop the fetch that follows.
150
+ with contextlib.suppress(OSError):
151
+ if os.path.getmtime(path) < cutoff:
152
+ os.remove(path)
153
+ return cache
154
+
155
+
156
+ # How long a sidecar may go unwritten before the sweep treats it as abandoned.
157
+ # Mirrors .stale_part_seconds() in R_workflow/R/corpora.R.
158
+ _STALE_PART_SECONDS = 24 * 60 * 60
159
+
160
+
161
+ def build_wordfreq_lexicon(language: str, n_words: int = 10000,
162
+ min_len: int = 2, max_len: int = 15) -> pd.DataFrame:
163
+ """Build a (word, freq_zipf) lexicon for any wordfreq language.
164
+
165
+ Raises ModuleNotFoundError if the optional [corpora] extra is not installed.
166
+ """
167
+ # Deferred because wordfreq belongs to the [corpora] extra, so importing it at
168
+ # module scope would break `import lexsync` on a default install. Guarded because
169
+ # this is what `lexsync fetch fr` reaches, and the README advertises that command:
170
+ # unguarded, a default install answers it with a bare ModuleNotFoundError nine
171
+ # frames down, which names the missing module and no way to get it. Everything
172
+ # else in this module answers a failure with the remedy, and so does this now.
173
+ # The R twin has no counterpart here: Connector B is Python-only, and R/corpora.R
174
+ # reads its pre-derived output as an ordinary lexicon.
175
+ try:
176
+ import wordfreq
177
+ except ImportError as exc:
178
+ raise ModuleNotFoundError(
179
+ f"lexsync: building a lexicon for '{language}' needs the wordfreq connector, "
180
+ f"which ships in the optional [corpora] extra and is not installed. Install "
181
+ 'it with: pip install "wordfreq>=3.0,<4".'
182
+ ) from exc
183
+
184
+ rx = re.compile(r"^[^\W\d_]+$", re.UNICODE)
185
+ words = []
186
+ for w in wordfreq.top_n_list(language, n_words * 3):
187
+ w = w.lower()
188
+ if rx.match(w) and min_len <= len(w) <= max_len:
189
+ words.append(w)
190
+ if len(words) >= n_words:
191
+ break
192
+ zipf = [wordfreq.zipf_frequency(w, language) for w in words]
193
+ df = pd.DataFrame({"word": words, "freq_zipf": np.round(zipf, 3),
194
+ "language": language, "source": "wordfreq"})
195
+ return df[df.freq_zipf > 0].sort_values("word").reset_index(drop=True)
196
+
197
+
198
+ _BOM = b"\xef\xbb\xbf"
199
+ _LEADING_WS = b" \t\n\r\v\f"
200
+
201
+
202
+ def _starts_with_markup(path: str) -> bool:
203
+ """Does the file open with '<', i.e. an HTML or XML document?
204
+
205
+ Mirrors .starts_with_markup() in R_workflow/R/corpora.R. A delimited file
206
+ opens on data; a login wall, a redirect stub or a 404 page served with a 200
207
+ status opens on a tag, and would otherwise be cached as <name>.csv and
208
+ resurface much later as an unintelligible schema error.
209
+ """
210
+ with open(path, "rb") as handle:
211
+ head = handle.read(512)
212
+ if head.startswith(_BOM):
213
+ head = head[len(_BOM):]
214
+ return head.lstrip(_LEADING_WS).startswith(b"<")
215
+
216
+
217
+ def _max_download_bytes() -> int:
218
+ """Hard cap on a corpus download, in bytes.
219
+
220
+ Mirrors .max_download_bytes() in R_workflow/R/corpora.R. A function rather
221
+ than a bare constant so the twin tests can lower it without writing 200 MB
222
+ to disk; the message below names the real limit either way.
223
+ """
224
+ return 200 * 1024 * 1024
225
+
226
+
227
+ def fetch_corpus(name: str, registry_path: str | None = None, n_words: int = 10000) -> str:
228
+ """Fetch a registered corpus into the cache.
229
+
230
+ If `name` is a language code supported by the wordfreq connector, a lexicon
231
+ is built with wordfreq. Otherwise the corpus's registered URL is downloaded,
232
+ once its scheme has been checked; the transfer lands in a sidecar file that
233
+ is renamed into the cache only after the size cap, the markup sniff and any
234
+ registered sha256 have all passed.
235
+
236
+ The file lands in :func:`cache_dir`. The cache directory is created only once
237
+ the registry entry and its URL have been accepted, or the wordfreq lexicon
238
+ built, so a refused call writes nothing. Each fetch into the cache begins by
239
+ deleting any ``.part`` sidecar that an interrupted transfer left there more
240
+ than a day earlier. The cache persists between sessions, and one corpus may
241
+ reach the 200 MB download cap, so several of them add up. :func:`cache_clear`
242
+ removes one corpus or the whole cache, and the next call fetches the corpus
243
+ again.
244
+ """
245
+ with open(_registry_path(registry_path), encoding="utf-8") as handle:
246
+ reg = yaml.safe_load(handle)
247
+ wf = reg.get("wordfreq_connector") or {}
248
+ if name in (wf.get("languages") or []):
249
+ df = build_wordfreq_lexicon(name, n_words)
250
+ dest = os.path.join(_prepare_cache(), f"{name}_wordfreq.csv")
251
+ # LF explicitly: pandas otherwise follows os.linesep, and the cached
252
+ # lexicon's bytes (and any checksum of them) must not depend on the OS.
253
+ df.to_csv(dest, index=False, encoding="utf-8", lineterminator="\n")
254
+ print(f"lexsync: built '{name}' via wordfreq. Please cite: {wf.get('citation', '')}")
255
+ return dest
256
+ entry = (reg.get("corpora") or {}).get(name)
257
+ if not entry:
258
+ raise ValueError(f"lexsync: corpus '{name}' is not in the registry.")
259
+ # Only 'openlexicon' names a delimited file; 'url' is the landing page, and
260
+ # downloading that would silently cache an HTML document as <name>.csv.
261
+ url = entry.get("openlexicon")
262
+ if not url:
263
+ raise ValueError(
264
+ f"lexsync: corpus '{name}' registers only a landing page "
265
+ f"({entry.get('url', 'see registry.yaml')}); lexsync cannot download it "
266
+ f"automatically. Retrieve the delimited file manually and pass it to "
267
+ f"load_lexicon()."
268
+ )
269
+ # A registry is editable, and fetch_corpus() writes wherever it points, so a
270
+ # 'file://' or 'ftp://' entry would read a local path under the guise of a
271
+ # download. Only the two schemes a corpus is published over are honoured.
272
+ if not re.match(r"https?://", url, re.IGNORECASE):
273
+ raise ValueError(
274
+ f"lexsync: corpus '{name}' registers a non-http(s) URL ({url}); "
275
+ f"refusing to download."
276
+ )
277
+ import urllib.error
278
+ import urllib.request
279
+ # Only now, with every refusal behind it, does the call touch the disk.
280
+ dest = os.path.join(_prepare_cache(), f"{name}.csv")
281
+ # The transfer lands in a sidecar and is renamed over `dest` only after every
282
+ # check below has passed, so a truncated or unverified body can never sit at
283
+ # the cache path, where a later run would trust it.
284
+ part = dest + ".part"
285
+ try:
286
+ # 60 s guards against a stalled server, which urlretrieve() would wait on
287
+ # forever; the R engine's download.file() honours options(timeout).
288
+ with urllib.request.urlopen(url, timeout=60) as response, \
289
+ open(part, "wb") as handle:
290
+ received = 0
291
+ for chunk in iter(lambda: response.read(65536), b""):
292
+ received += len(chunk)
293
+ if received > _max_download_bytes():
294
+ raise ValueError(
295
+ "lexsync: corpus download exceeded the 200 MB size limit. "
296
+ "Retrieve the delimited file manually and pass it to "
297
+ "load_lexicon()."
298
+ )
299
+ handle.write(chunk)
300
+ except (urllib.error.URLError, OSError) as exc:
301
+ if os.path.exists(part):
302
+ os.remove(part)
303
+ raise RuntimeError(
304
+ f"lexsync: could not download corpus '{name}' from {url} ({exc}). "
305
+ f"Check the URL in registry.yaml, or download the file manually and "
306
+ f"pass it to load_lexicon()."
307
+ ) from exc
308
+ except ValueError:
309
+ if os.path.exists(part):
310
+ os.remove(part)
311
+ raise
312
+ if _starts_with_markup(part):
313
+ os.remove(part)
314
+ raise ValueError(
315
+ f"lexsync: corpus '{name}' returned an HTML page, not a delimited file "
316
+ f"({url}); the registry URL may have rotted. Retrieve the delimited file "
317
+ f"manually and pass it to load_lexicon()."
318
+ )
319
+ # 'sha256' is optional per registry entry; when present the download must
320
+ # match it before it may enter the cache.
321
+ expected = entry.get("sha256")
322
+ if expected and sha256_file(part) != expected:
323
+ os.remove(part)
324
+ raise ValueError(
325
+ f"lexsync: checksum mismatch for corpus '{name}'; the download does "
326
+ f"not match the registry's sha256. Retry the download, or verify the "
327
+ f"sha256 recorded in registry.yaml."
328
+ )
329
+ os.replace(part, dest)
330
+ print(f"lexsync: downloaded '{name}'. Please cite: {entry.get('citation', '(see registry)')}")
331
+ return dest