lexsync 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lexsync/__init__.py +53 -0
- lexsync/__main__.py +5 -0
- lexsync/blocks.py +149 -0
- lexsync/cli.py +85 -0
- lexsync/corpora.py +331 -0
- lexsync/counterbalancing.py +399 -0
- lexsync/data/en_example.csv +3001 -0
- lexsync/data/es_example.csv +3001 -0
- lexsync/data/registry.yaml +149 -0
- lexsync/data/schema.yaml +193 -0
- lexsync/data/zh_example.csv +3001 -0
- lexsync/datasheet.py +752 -0
- lexsync/generation.py +280 -0
- lexsync/io_utils.py +609 -0
- lexsync/logging.py +88 -0
- lexsync/matching.py +614 -0
- lexsync/pairs.py +177 -0
- lexsync/paradigms.py +163 -0
- lexsync/py.typed +0 -0
- lexsync/querying.py +481 -0
- lexsync/run_pipeline.py +425 -0
- lexsync/scripting.py +845 -0
- lexsync/templates/jspsych/experiment_template.html +223 -0
- lexsync/templates/opensesame/README.md +36 -0
- lexsync/templates/opensesame/inline_send_triggers.py +18 -0
- lexsync/templates/opensesame/inline_trigger_setup.py +53 -0
- lexsync/templates/psychopy/mock_port.py +23 -0
- lexsync/templates/psychopy/trial_runner_template.py +300 -0
- lexsync/validation.py +267 -0
- lexsync-0.1.1.dist-info/METADATA +234 -0
- lexsync-0.1.1.dist-info/RECORD +35 -0
- lexsync-0.1.1.dist-info/WHEEL +4 -0
- lexsync-0.1.1.dist-info/entry_points.txt +2 -0
- lexsync-0.1.1.dist-info/licenses/LICENSE +33 -0
- lexsync-0.1.1.dist-info/licenses/LICENSE.note +108 -0
lexsync/__init__.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""lexsync: lexical optimisation and hardware-timed experiment generation."""
|
|
2
|
+
from .corpora import fetch_corpus, list_corpora
|
|
3
|
+
from .counterbalancing import balance_lists, counterbalance, participant_table
|
|
4
|
+
from .datasheet import build_datasheet, methods_paragraph, write_datasheet
|
|
5
|
+
from .generation import build_lexdec_stimuli, generate_pseudowords, make_pseudoword
|
|
6
|
+
from .matching import match_stimuli, resample_stimuli, select_continuous_stimuli
|
|
7
|
+
from .paradigms import PARADIGMS, required_fields, resolve_events
|
|
8
|
+
from .querying import (
|
|
9
|
+
add_bigram_frequency,
|
|
10
|
+
add_neighbourhood,
|
|
11
|
+
add_pair_overlap,
|
|
12
|
+
build_pool,
|
|
13
|
+
count_syllables,
|
|
14
|
+
load_items,
|
|
15
|
+
load_lexicon,
|
|
16
|
+
load_pool,
|
|
17
|
+
merge_norms,
|
|
18
|
+
)
|
|
19
|
+
from .run_pipeline import run_all, run_pipeline
|
|
20
|
+
from .scripting import (
|
|
21
|
+
export_experiments,
|
|
22
|
+
export_jspsych,
|
|
23
|
+
export_opensesame,
|
|
24
|
+
export_psychopy,
|
|
25
|
+
resolve_trial_timing,
|
|
26
|
+
)
|
|
27
|
+
from .validation import (
|
|
28
|
+
balance_check,
|
|
29
|
+
cohens_d,
|
|
30
|
+
cohens_d_ci,
|
|
31
|
+
describe_stimuli,
|
|
32
|
+
match_report,
|
|
33
|
+
match_report_continuous,
|
|
34
|
+
tost_equiv,
|
|
35
|
+
variance_ratio,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
"load_lexicon", "load_items", "load_pool", "add_neighbourhood", "add_bigram_frequency",
|
|
40
|
+
"count_syllables", "merge_norms", "build_pool", "match_stimuli",
|
|
41
|
+
"resample_stimuli", "select_continuous_stimuli", "make_pseudoword",
|
|
42
|
+
"generate_pseudowords", "build_lexdec_stimuli",
|
|
43
|
+
"PARADIGMS", "resolve_events", "required_fields",
|
|
44
|
+
"counterbalance", "balance_lists", "participant_table", "describe_stimuli", "cohens_d",
|
|
45
|
+
"cohens_d_ci", "tost_equiv", "variance_ratio", "balance_check", "match_report",
|
|
46
|
+
"match_report_continuous", "export_experiments",
|
|
47
|
+
"export_psychopy", "export_opensesame", "export_jspsych", "resolve_trial_timing",
|
|
48
|
+
"add_pair_overlap",
|
|
49
|
+
"list_corpora", "fetch_corpus",
|
|
50
|
+
"build_datasheet", "methods_paragraph", "write_datasheet",
|
|
51
|
+
"run_pipeline", "run_all",
|
|
52
|
+
]
|
|
53
|
+
__version__ = "0.1.1"
|
lexsync/__main__.py
ADDED
lexsync/blocks.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Practice and filler trials: rows that are PRESENTED but not ANALYSED.
|
|
2
|
+
|
|
3
|
+
Everything else in the package treats one frame as both the materials record and the
|
|
4
|
+
thing the experiment runs. That works only while the two are the same set of trials, and
|
|
5
|
+
they usually are not. A practice block exists to settle the participant into the task and
|
|
6
|
+
is discarded before analysis; fillers exist to dilute the manipulation so it is less
|
|
7
|
+
guessable, and are likewise not analysed. Both have to reach the generated experiment and
|
|
8
|
+
neither belongs in the stimuli file, the descriptives or the realised control.
|
|
9
|
+
|
|
10
|
+
The pipeline therefore splits after counterbalancing: the analysis artefacts are
|
|
11
|
+
written from the main rows, and the experiment is generated from the presented rows. A ``block``
|
|
12
|
+
column marks which is which, and it appears ONLY when a design declares one of these
|
|
13
|
+
blocks -- a design without them keeps exactly the columns it had.
|
|
14
|
+
|
|
15
|
+
Where each block goes in the sequence is a methodological choice, not a convenience:
|
|
16
|
+
|
|
17
|
+
Practice comes first, as its own run of trials, because its purpose is to precede the
|
|
18
|
+
task. It is shuffled within itself so participants do not all meet the practice items in
|
|
19
|
+
one order.
|
|
20
|
+
|
|
21
|
+
Fillers are INTERLEAVED with the main trials, not appended, because a block of fillers at
|
|
22
|
+
the end is not a filler -- it is a second block that participants can tell apart. They
|
|
23
|
+
are merged into each list before the order is drawn, so one deterministic shuffle mixes
|
|
24
|
+
them through. That does renumber the main trials, which is correct: adding fillers
|
|
25
|
+
changes the sequence, and the stimuli file records where each item actually appeared.
|
|
26
|
+
|
|
27
|
+
Both blocks appear in EVERY list. They are not counterbalanced, because they carry no
|
|
28
|
+
manipulation to rotate; every participant should get the same practice.
|
|
29
|
+
|
|
30
|
+
Mirrors R_workflow/R/blocks.R.
|
|
31
|
+
"""
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import pandas as pd
|
|
35
|
+
|
|
36
|
+
from .counterbalancing import _shuffle_deterministic
|
|
37
|
+
from .io_utils import sha256_file
|
|
38
|
+
from .paradigms import required_fields
|
|
39
|
+
from .querying import load_items
|
|
40
|
+
|
|
41
|
+
BLOCK_MAIN = "main"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _concat_blocks(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame:
|
|
45
|
+
"""Concatenate two block frames that need not have the same columns.
|
|
46
|
+
|
|
47
|
+
A filler table carries no ``frequency`` or ``old20``, and the main rows carry no
|
|
48
|
+
filler-specific field; the union is taken and the gaps left missing, which is honest
|
|
49
|
+
-- a filler has no matched frequency because it was never matched.
|
|
50
|
+
|
|
51
|
+
The key columns are put back to their original dtypes afterwards. This is not
|
|
52
|
+
cosmetic: pandas promotes an integer column to float the moment a missing value
|
|
53
|
+
enters it, and the trial-order shuffle formats an integer `set` as "3" but a float
|
|
54
|
+
one as "3.0", which would change every digest and every trial order with nothing to
|
|
55
|
+
signal it.
|
|
56
|
+
"""
|
|
57
|
+
cols = list(a.columns) + [c for c in b.columns if c not in a.columns]
|
|
58
|
+
dtypes = {c: a[c].dtype for c in a.columns if str(a[c].dtype).startswith("int")}
|
|
59
|
+
dtypes.update({c: b[c].dtype for c in b.columns
|
|
60
|
+
if c not in dtypes and str(b[c].dtype).startswith("int")})
|
|
61
|
+
out = pd.concat([a.reindex(columns=cols), b.reindex(columns=cols)],
|
|
62
|
+
ignore_index=True)
|
|
63
|
+
for c, dt in dtypes.items():
|
|
64
|
+
if c in out.columns and out[c].notna().all():
|
|
65
|
+
out[c] = out[c].astype(dt)
|
|
66
|
+
return out
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _load_block(cfg: dict, design: dict, label: str, set_offset: int) -> pd.DataFrame:
|
|
70
|
+
"""Read one block's item table and give it a block label and a non-colliding ``set``.
|
|
71
|
+
|
|
72
|
+
The offset matters more than it looks. ``load_items`` numbers sets from 1 within
|
|
73
|
+
whatever table it is given, so practice item 1 and main item 1 would both be set 1 --
|
|
74
|
+
and ``set`` is part of the key the trial-order shuffle hashes. Two rows sharing a key
|
|
75
|
+
would be ordered by a coin the package does not own.
|
|
76
|
+
"""
|
|
77
|
+
path = cfg.get("path")
|
|
78
|
+
if not path:
|
|
79
|
+
raise ValueError(
|
|
80
|
+
"lexsync: the `%s:` block needs a `path` to an item table." % label)
|
|
81
|
+
df = load_items(path, required_fields(design)).copy()
|
|
82
|
+
df["block"] = label
|
|
83
|
+
df["set"] = df["set"].astype(int) + int(set_offset)
|
|
84
|
+
return df.reset_index(drop=True)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def add_blocks(stimuli: pd.DataFrame, design: dict, schema: dict) -> dict:
|
|
88
|
+
"""Assemble the presented trial sequence from the main, filler and practice blocks.
|
|
89
|
+
|
|
90
|
+
Returns ``{"presented": DataFrame, "report": dict | None}``: every trial the
|
|
91
|
+
experiment runs, in order, with a ``block`` column when more than one block exists;
|
|
92
|
+
and the per-block counts and item-table checksums, or None when the design declares
|
|
93
|
+
no extra block.
|
|
94
|
+
"""
|
|
95
|
+
practice_cfg = design.get("practice")
|
|
96
|
+
filler_cfg = design.get("fillers")
|
|
97
|
+
if not practice_cfg and not filler_cfg:
|
|
98
|
+
# No block column at all: a design that declares none must keep the columns it
|
|
99
|
+
# had, so adding this feature moves no existing artefact.
|
|
100
|
+
return {"presented": stimuli, "report": None}
|
|
101
|
+
|
|
102
|
+
seed = schema.get("seed", 1)
|
|
103
|
+
stimuli = stimuli.copy()
|
|
104
|
+
stimuli["block"] = BLOCK_MAIN
|
|
105
|
+
if "list" not in stimuli.columns:
|
|
106
|
+
stimuli["list"] = 1
|
|
107
|
+
lists = sorted(stimuli["list"].unique())
|
|
108
|
+
offset = int(max([0] + [int(s) for s in stimuli["set"]]))
|
|
109
|
+
|
|
110
|
+
fillers = None
|
|
111
|
+
if filler_cfg:
|
|
112
|
+
fillers = _load_block(filler_cfg, design, "filler", offset)
|
|
113
|
+
offset = max(offset, int(fillers["set"].max()))
|
|
114
|
+
practice = None
|
|
115
|
+
if practice_cfg:
|
|
116
|
+
practice = _load_block(practice_cfg, design, "practice", offset)
|
|
117
|
+
|
|
118
|
+
parts = []
|
|
119
|
+
for li in lists:
|
|
120
|
+
body = stimuli[stimuli["list"] == li].reset_index(drop=True)
|
|
121
|
+
if fillers is not None:
|
|
122
|
+
f = fillers.copy()
|
|
123
|
+
f["list"] = li
|
|
124
|
+
body = _concat_blocks(body, f)
|
|
125
|
+
# One shuffle over main and fillers together is what interleaves them.
|
|
126
|
+
body = _shuffle_deterministic(body, seed).reset_index(drop=True)
|
|
127
|
+
if practice is not None:
|
|
128
|
+
p = practice.copy()
|
|
129
|
+
p["list"] = li
|
|
130
|
+
p = _shuffle_deterministic(p, seed).reset_index(drop=True)
|
|
131
|
+
body = _concat_blocks(p, body)
|
|
132
|
+
body["trial"] = range(1, len(body) + 1)
|
|
133
|
+
parts.append(body)
|
|
134
|
+
out = pd.concat(parts, ignore_index=True)
|
|
135
|
+
|
|
136
|
+
n_main = int((stimuli["list"] == lists[0]).sum())
|
|
137
|
+
report = {"blocks": [{"block": BLOCK_MAIN, "n_per_list": n_main}]}
|
|
138
|
+
if fillers is not None:
|
|
139
|
+
report["blocks"].append({
|
|
140
|
+
"block": "filler", "n_per_list": int(len(fillers)),
|
|
141
|
+
"path": filler_cfg["path"], "sha256": sha256_file(filler_cfg["path"]),
|
|
142
|
+
"placement": "interleaved with the main trials by the seeded order"})
|
|
143
|
+
if practice is not None:
|
|
144
|
+
report["blocks"].append({
|
|
145
|
+
"block": "practice", "n_per_list": int(len(practice)),
|
|
146
|
+
"path": practice_cfg["path"], "sha256": sha256_file(practice_cfg["path"]),
|
|
147
|
+
"placement": "before the main trials"})
|
|
148
|
+
report["analysed"] = BLOCK_MAIN
|
|
149
|
+
return {"presented": out, "report": report}
|
lexsync/cli.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Command-line interface: ``lexsync run | corpora list | fetch``."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
from .corpora import fetch_corpus, list_corpora
|
|
8
|
+
from .run_pipeline import run_all, run_pipeline
|
|
9
|
+
|
|
10
|
+
# _resolve_schema treats this exact value as "the user asked for nothing in
|
|
11
|
+
# particular", which is what licenses the bundled-copy fallback; an explicit
|
|
12
|
+
# --schema is honoured or refused, never substituted.
|
|
13
|
+
DEFAULT_SCHEMA = "config/schema.yaml"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _version() -> str:
|
|
17
|
+
# importlib.metadata knows only installed distributions, so a source-tree
|
|
18
|
+
# invocation (sys.path pointing at src/) has no version record to read;
|
|
19
|
+
# "unknown" keeps --version working there instead of crashing.
|
|
20
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
21
|
+
try:
|
|
22
|
+
return version("lexsync")
|
|
23
|
+
except PackageNotFoundError:
|
|
24
|
+
return "unknown"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _resolve_schema(schema_path: str) -> str:
|
|
28
|
+
"""Resolve --schema for the installed CLI as well as for a checkout.
|
|
29
|
+
|
|
30
|
+
The default is checkout-relative, so on its own it strands an installed CLI
|
|
31
|
+
run from anywhere else. A byte-identical copy of config/schema.yaml ships
|
|
32
|
+
inside the package (the identity is test-enforced), and the default falls
|
|
33
|
+
back to it when the checkout-relative file is absent. An explicit path
|
|
34
|
+
still errors: silently substituting the bundled schema there would run a
|
|
35
|
+
different file from the one the user named.
|
|
36
|
+
"""
|
|
37
|
+
if os.path.exists(schema_path):
|
|
38
|
+
return schema_path
|
|
39
|
+
if schema_path == DEFAULT_SCHEMA:
|
|
40
|
+
from importlib.resources import files
|
|
41
|
+
bundled = files("lexsync") / "data" / "schema.yaml"
|
|
42
|
+
if bundled.is_file():
|
|
43
|
+
return str(bundled)
|
|
44
|
+
raise FileNotFoundError(
|
|
45
|
+
f"lexsync: schema not found: '{schema_path}'. "
|
|
46
|
+
"Run from a checkout root or pass --schema.")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def main(argv=None) -> None:
|
|
50
|
+
parser = argparse.ArgumentParser(
|
|
51
|
+
prog="lexsync",
|
|
52
|
+
description="lexsync: lexical optimisation and hardware-timed experiment generation",
|
|
53
|
+
)
|
|
54
|
+
parser.add_argument("--version", action="version", version="lexsync " + _version())
|
|
55
|
+
sub = parser.add_subparsers(dest="command")
|
|
56
|
+
|
|
57
|
+
run = sub.add_parser("run", help="run one design, or all designs in --config-dir")
|
|
58
|
+
run.add_argument("design", nargs="?", help="path to a design YAML (omit to run all)")
|
|
59
|
+
run.add_argument("--schema", default=DEFAULT_SCHEMA)
|
|
60
|
+
run.add_argument("--config-dir", default="config")
|
|
61
|
+
run.add_argument("--outdir", default="output")
|
|
62
|
+
|
|
63
|
+
corpora = sub.add_parser("corpora", help="corpus registry operations")
|
|
64
|
+
corpora.add_argument("action", choices=["list"])
|
|
65
|
+
|
|
66
|
+
fetch = sub.add_parser("fetch", help="fetch a corpus by registry name or language code")
|
|
67
|
+
fetch.add_argument("name")
|
|
68
|
+
|
|
69
|
+
args = parser.parse_args(argv)
|
|
70
|
+
if args.command == "run":
|
|
71
|
+
schema = _resolve_schema(args.schema)
|
|
72
|
+
if args.design:
|
|
73
|
+
run_pipeline(args.design, schema, args.outdir)
|
|
74
|
+
else:
|
|
75
|
+
run_all(args.config_dir, schema, args.outdir)
|
|
76
|
+
elif args.command == "corpora":
|
|
77
|
+
print(list_corpora().to_string(index=False))
|
|
78
|
+
elif args.command == "fetch":
|
|
79
|
+
print(fetch_corpus(args.name))
|
|
80
|
+
else:
|
|
81
|
+
parser.print_help()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
if __name__ == "__main__":
|
|
85
|
+
main()
|
lexsync/corpora.py
ADDED
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
"""Access to the many-language corpus registry.
|
|
2
|
+
|
|
3
|
+
Mirrors R_workflow/R/corpora.R, and adds the wordfreq connector (Connector B,
|
|
4
|
+
~40 languages). The demonstrations read the bundled, pre-derived lexica; new
|
|
5
|
+
languages are fetched on demand into a user cache.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import contextlib
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import shutil
|
|
13
|
+
import time
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pandas as pd
|
|
17
|
+
import yaml
|
|
18
|
+
|
|
19
|
+
from .io_utils import sha256_file
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _registry_path(registry_path: str | None = None) -> str:
|
|
23
|
+
candidates = [
|
|
24
|
+
registry_path,
|
|
25
|
+
os.environ.get("LEXSYNC_REGISTRY"),
|
|
26
|
+
os.path.join("corpora", "registry.yaml"),
|
|
27
|
+
os.path.join("..", "corpora", "registry.yaml"),
|
|
28
|
+
os.path.join(os.path.dirname(__file__), "data", "registry.yaml"),
|
|
29
|
+
]
|
|
30
|
+
for cand in candidates:
|
|
31
|
+
if cand and os.path.exists(cand):
|
|
32
|
+
return cand
|
|
33
|
+
raise FileNotFoundError("lexsync: could not locate 'registry.yaml'; set LEXSYNC_REGISTRY.")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def list_corpora(registry_path: str | None = None) -> pd.DataFrame:
|
|
37
|
+
with open(_registry_path(registry_path), encoding="utf-8") as handle:
|
|
38
|
+
reg = yaml.safe_load(handle)
|
|
39
|
+
rows = []
|
|
40
|
+
for name, entry in (reg.get("corpora") or {}).items():
|
|
41
|
+
lang = entry.get("language") or {}
|
|
42
|
+
rows.append(dict(
|
|
43
|
+
name=name, language=lang.get("name"), iso=lang.get("iso"),
|
|
44
|
+
status=entry.get("status"), connector=entry.get("connector", "openlexicon"),
|
|
45
|
+
citation=entry.get("citation"),
|
|
46
|
+
))
|
|
47
|
+
return pd.DataFrame(rows)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def cache_dir() -> str:
|
|
51
|
+
"""Per-user cache directory for fetched corpora.
|
|
52
|
+
|
|
53
|
+
Where :func:`fetch_corpus` puts what it fetches, and the only place the
|
|
54
|
+
package writes to without being handed a path. This function only reports
|
|
55
|
+
the path. The directory is created by the first fetch into it, so until then
|
|
56
|
+
it does not exist.
|
|
57
|
+
|
|
58
|
+
The cache persists between sessions. A registered corpus is a delimited word
|
|
59
|
+
list, and a download is refused above 200 MB, so a cache holding several
|
|
60
|
+
large corpora can reach a few hundred megabytes. Each fetch into the cache
|
|
61
|
+
first deletes any partial download that an interrupted transfer left there
|
|
62
|
+
more than a day earlier, and fetching a corpus again replaces the earlier
|
|
63
|
+
copy. Fetched corpora otherwise stay until :func:`cache_clear` removes them,
|
|
64
|
+
one corpus at a time or all together. Nothing kept there is irreplaceable, so
|
|
65
|
+
the next call downloads afresh. The R twin documents the same contract in
|
|
66
|
+
lexsync_cache_dir.Rd; only the location differs, since R uses
|
|
67
|
+
tools::R_user_dir.
|
|
68
|
+
"""
|
|
69
|
+
return os.path.join(os.path.expanduser("~"), ".lexsync", "cache")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def cache_clear(name: str | None = None) -> list[str]:
|
|
73
|
+
"""Remove fetched corpora from the cache.
|
|
74
|
+
|
|
75
|
+
Deletes what :func:`fetch_corpus` put in :func:`cache_dir`. Given a `name`,
|
|
76
|
+
that is the corpus's ``<name>.csv``, any ``<name>.csv.part`` that an
|
|
77
|
+
interrupted download left behind and the ``<name>_wordfreq.csv`` lexicon the
|
|
78
|
+
wordfreq connector builds. With `name` left as None it is the whole cache
|
|
79
|
+
directory, and the ``~/.lexsync`` directory above it once nothing else is left
|
|
80
|
+
there. A corpus needed again is fetched afresh by the next
|
|
81
|
+
:func:`fetch_corpus` call. Mirrors lexsync_cache_clear() in
|
|
82
|
+
R_workflow/R/corpora.R, whose engine has no wordfreq connector and so no
|
|
83
|
+
``_wordfreq.csv`` to remove.
|
|
84
|
+
|
|
85
|
+
Returns the paths of the files removed, empty when there was nothing to
|
|
86
|
+
remove. Raises OSError, naming them, when files survive the attempt (one held
|
|
87
|
+
open on Windows, say).
|
|
88
|
+
"""
|
|
89
|
+
cache = cache_dir()
|
|
90
|
+
if name is None:
|
|
91
|
+
if not os.path.isdir(cache):
|
|
92
|
+
return []
|
|
93
|
+
removed = sorted(os.path.join(root, fname)
|
|
94
|
+
for root, _dirs, files in os.walk(cache) for fname in files)
|
|
95
|
+
targets = [cache]
|
|
96
|
+
# Keep going past a file that cannot be deleted, as R's unlink() does, and
|
|
97
|
+
# report what survives below.
|
|
98
|
+
shutil.rmtree(cache, ignore_errors=True)
|
|
99
|
+
else:
|
|
100
|
+
# The name becomes a file name inside the cache, so a path separator, or
|
|
101
|
+
# the colon of a Windows drive such as 'D:', would let it reach a file
|
|
102
|
+
# other than the corpus's own: os.path.join() drops the cache directory
|
|
103
|
+
# before a drive. All three characters are refused on every platform, so
|
|
104
|
+
# the two engines refuse the same names.
|
|
105
|
+
if not isinstance(name, str) or not name or re.search(r"[/\\:]", name):
|
|
106
|
+
raise ValueError("lexsync: 'name' must be a single corpus name, not a path.")
|
|
107
|
+
targets = [os.path.join(cache, f"{name}{suffix}")
|
|
108
|
+
for suffix in (".csv", ".csv.part", "_wordfreq.csv")]
|
|
109
|
+
removed = [path for path in targets if os.path.isfile(path)]
|
|
110
|
+
for path in removed:
|
|
111
|
+
with contextlib.suppress(OSError):
|
|
112
|
+
os.remove(path)
|
|
113
|
+
left = [path for path in (targets if name is None else removed)
|
|
114
|
+
if os.path.exists(path)]
|
|
115
|
+
if left:
|
|
116
|
+
raise OSError(f"lexsync: could not remove {', '.join(left)} from the cache; "
|
|
117
|
+
f"check that no other process is using it.")
|
|
118
|
+
if name is None:
|
|
119
|
+
# ~/.lexsync holds nothing but the cache, so an emptied one would be left
|
|
120
|
+
# behind in the user's home. rmdir() removes it only when it is empty. R's
|
|
121
|
+
# twin has no such step: the parent of its cache directory is shared by
|
|
122
|
+
# every R package's cache.
|
|
123
|
+
with contextlib.suppress(OSError):
|
|
124
|
+
os.rmdir(os.path.dirname(cache))
|
|
125
|
+
return removed
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _prepare_cache() -> str:
|
|
129
|
+
"""Ready the cache for a fetch into it, and return its path.
|
|
130
|
+
|
|
131
|
+
Mirrors .prepare_cache() in R_workflow/R/corpora.R. A '.part' sidecar
|
|
132
|
+
outlives its fetch only when the transfer was cut short in a way
|
|
133
|
+
fetch_corpus()'s handlers never see: a KeyboardInterrupt derives from
|
|
134
|
+
BaseException, which neither except clause catches, and a process that dies
|
|
135
|
+
runs no handler at all. A sidecar can also belong to a download still running
|
|
136
|
+
in another process, which writes to it as it goes and gives up on a stalled
|
|
137
|
+
server after 60 s. A day without a write therefore marks a sidecar as
|
|
138
|
+
abandoned, and only those are deleted. A younger one is left to its download,
|
|
139
|
+
and a retry of the same corpus truncates its own.
|
|
140
|
+
"""
|
|
141
|
+
cache = cache_dir()
|
|
142
|
+
os.makedirs(cache, exist_ok=True)
|
|
143
|
+
cutoff = time.time() - _STALE_PART_SECONDS
|
|
144
|
+
for fname in os.listdir(cache):
|
|
145
|
+
if fname.endswith(".part"):
|
|
146
|
+
path = os.path.join(cache, fname)
|
|
147
|
+
# Best effort, as R's unlink() is: a sidecar that cannot be deleted
|
|
148
|
+
# (held open on Windows, say), or that its download renamed away
|
|
149
|
+
# meanwhile, must not stop the fetch that follows.
|
|
150
|
+
with contextlib.suppress(OSError):
|
|
151
|
+
if os.path.getmtime(path) < cutoff:
|
|
152
|
+
os.remove(path)
|
|
153
|
+
return cache
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
# How long a sidecar may go unwritten before the sweep treats it as abandoned.
|
|
157
|
+
# Mirrors .stale_part_seconds() in R_workflow/R/corpora.R.
|
|
158
|
+
_STALE_PART_SECONDS = 24 * 60 * 60
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def build_wordfreq_lexicon(language: str, n_words: int = 10000,
|
|
162
|
+
min_len: int = 2, max_len: int = 15) -> pd.DataFrame:
|
|
163
|
+
"""Build a (word, freq_zipf) lexicon for any wordfreq language.
|
|
164
|
+
|
|
165
|
+
Raises ModuleNotFoundError if the optional [corpora] extra is not installed.
|
|
166
|
+
"""
|
|
167
|
+
# Deferred because wordfreq belongs to the [corpora] extra, so importing it at
|
|
168
|
+
# module scope would break `import lexsync` on a default install. Guarded because
|
|
169
|
+
# this is what `lexsync fetch fr` reaches, and the README advertises that command:
|
|
170
|
+
# unguarded, a default install answers it with a bare ModuleNotFoundError nine
|
|
171
|
+
# frames down, which names the missing module and no way to get it. Everything
|
|
172
|
+
# else in this module answers a failure with the remedy, and so does this now.
|
|
173
|
+
# The R twin has no counterpart here: Connector B is Python-only, and R/corpora.R
|
|
174
|
+
# reads its pre-derived output as an ordinary lexicon.
|
|
175
|
+
try:
|
|
176
|
+
import wordfreq
|
|
177
|
+
except ImportError as exc:
|
|
178
|
+
raise ModuleNotFoundError(
|
|
179
|
+
f"lexsync: building a lexicon for '{language}' needs the wordfreq connector, "
|
|
180
|
+
f"which ships in the optional [corpora] extra and is not installed. Install "
|
|
181
|
+
'it with: pip install "wordfreq>=3.0,<4".'
|
|
182
|
+
) from exc
|
|
183
|
+
|
|
184
|
+
rx = re.compile(r"^[^\W\d_]+$", re.UNICODE)
|
|
185
|
+
words = []
|
|
186
|
+
for w in wordfreq.top_n_list(language, n_words * 3):
|
|
187
|
+
w = w.lower()
|
|
188
|
+
if rx.match(w) and min_len <= len(w) <= max_len:
|
|
189
|
+
words.append(w)
|
|
190
|
+
if len(words) >= n_words:
|
|
191
|
+
break
|
|
192
|
+
zipf = [wordfreq.zipf_frequency(w, language) for w in words]
|
|
193
|
+
df = pd.DataFrame({"word": words, "freq_zipf": np.round(zipf, 3),
|
|
194
|
+
"language": language, "source": "wordfreq"})
|
|
195
|
+
return df[df.freq_zipf > 0].sort_values("word").reset_index(drop=True)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
_BOM = b"\xef\xbb\xbf"
|
|
199
|
+
_LEADING_WS = b" \t\n\r\v\f"
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _starts_with_markup(path: str) -> bool:
|
|
203
|
+
"""Does the file open with '<', i.e. an HTML or XML document?
|
|
204
|
+
|
|
205
|
+
Mirrors .starts_with_markup() in R_workflow/R/corpora.R. A delimited file
|
|
206
|
+
opens on data; a login wall, a redirect stub or a 404 page served with a 200
|
|
207
|
+
status opens on a tag, and would otherwise be cached as <name>.csv and
|
|
208
|
+
resurface much later as an unintelligible schema error.
|
|
209
|
+
"""
|
|
210
|
+
with open(path, "rb") as handle:
|
|
211
|
+
head = handle.read(512)
|
|
212
|
+
if head.startswith(_BOM):
|
|
213
|
+
head = head[len(_BOM):]
|
|
214
|
+
return head.lstrip(_LEADING_WS).startswith(b"<")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _max_download_bytes() -> int:
|
|
218
|
+
"""Hard cap on a corpus download, in bytes.
|
|
219
|
+
|
|
220
|
+
Mirrors .max_download_bytes() in R_workflow/R/corpora.R. A function rather
|
|
221
|
+
than a bare constant so the twin tests can lower it without writing 200 MB
|
|
222
|
+
to disk; the message below names the real limit either way.
|
|
223
|
+
"""
|
|
224
|
+
return 200 * 1024 * 1024
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def fetch_corpus(name: str, registry_path: str | None = None, n_words: int = 10000) -> str:
|
|
228
|
+
"""Fetch a registered corpus into the cache.
|
|
229
|
+
|
|
230
|
+
If `name` is a language code supported by the wordfreq connector, a lexicon
|
|
231
|
+
is built with wordfreq. Otherwise the corpus's registered URL is downloaded,
|
|
232
|
+
once its scheme has been checked; the transfer lands in a sidecar file that
|
|
233
|
+
is renamed into the cache only after the size cap, the markup sniff and any
|
|
234
|
+
registered sha256 have all passed.
|
|
235
|
+
|
|
236
|
+
The file lands in :func:`cache_dir`. The cache directory is created only once
|
|
237
|
+
the registry entry and its URL have been accepted, or the wordfreq lexicon
|
|
238
|
+
built, so a refused call writes nothing. Each fetch into the cache begins by
|
|
239
|
+
deleting any ``.part`` sidecar that an interrupted transfer left there more
|
|
240
|
+
than a day earlier. The cache persists between sessions, and one corpus may
|
|
241
|
+
reach the 200 MB download cap, so several of them add up. :func:`cache_clear`
|
|
242
|
+
removes one corpus or the whole cache, and the next call fetches the corpus
|
|
243
|
+
again.
|
|
244
|
+
"""
|
|
245
|
+
with open(_registry_path(registry_path), encoding="utf-8") as handle:
|
|
246
|
+
reg = yaml.safe_load(handle)
|
|
247
|
+
wf = reg.get("wordfreq_connector") or {}
|
|
248
|
+
if name in (wf.get("languages") or []):
|
|
249
|
+
df = build_wordfreq_lexicon(name, n_words)
|
|
250
|
+
dest = os.path.join(_prepare_cache(), f"{name}_wordfreq.csv")
|
|
251
|
+
# LF explicitly: pandas otherwise follows os.linesep, and the cached
|
|
252
|
+
# lexicon's bytes (and any checksum of them) must not depend on the OS.
|
|
253
|
+
df.to_csv(dest, index=False, encoding="utf-8", lineterminator="\n")
|
|
254
|
+
print(f"lexsync: built '{name}' via wordfreq. Please cite: {wf.get('citation', '')}")
|
|
255
|
+
return dest
|
|
256
|
+
entry = (reg.get("corpora") or {}).get(name)
|
|
257
|
+
if not entry:
|
|
258
|
+
raise ValueError(f"lexsync: corpus '{name}' is not in the registry.")
|
|
259
|
+
# Only 'openlexicon' names a delimited file; 'url' is the landing page, and
|
|
260
|
+
# downloading that would silently cache an HTML document as <name>.csv.
|
|
261
|
+
url = entry.get("openlexicon")
|
|
262
|
+
if not url:
|
|
263
|
+
raise ValueError(
|
|
264
|
+
f"lexsync: corpus '{name}' registers only a landing page "
|
|
265
|
+
f"({entry.get('url', 'see registry.yaml')}); lexsync cannot download it "
|
|
266
|
+
f"automatically. Retrieve the delimited file manually and pass it to "
|
|
267
|
+
f"load_lexicon()."
|
|
268
|
+
)
|
|
269
|
+
# A registry is editable, and fetch_corpus() writes wherever it points, so a
|
|
270
|
+
# 'file://' or 'ftp://' entry would read a local path under the guise of a
|
|
271
|
+
# download. Only the two schemes a corpus is published over are honoured.
|
|
272
|
+
if not re.match(r"https?://", url, re.IGNORECASE):
|
|
273
|
+
raise ValueError(
|
|
274
|
+
f"lexsync: corpus '{name}' registers a non-http(s) URL ({url}); "
|
|
275
|
+
f"refusing to download."
|
|
276
|
+
)
|
|
277
|
+
import urllib.error
|
|
278
|
+
import urllib.request
|
|
279
|
+
# Only now, with every refusal behind it, does the call touch the disk.
|
|
280
|
+
dest = os.path.join(_prepare_cache(), f"{name}.csv")
|
|
281
|
+
# The transfer lands in a sidecar and is renamed over `dest` only after every
|
|
282
|
+
# check below has passed, so a truncated or unverified body can never sit at
|
|
283
|
+
# the cache path, where a later run would trust it.
|
|
284
|
+
part = dest + ".part"
|
|
285
|
+
try:
|
|
286
|
+
# 60 s guards against a stalled server, which urlretrieve() would wait on
|
|
287
|
+
# forever; the R engine's download.file() honours options(timeout).
|
|
288
|
+
with urllib.request.urlopen(url, timeout=60) as response, \
|
|
289
|
+
open(part, "wb") as handle:
|
|
290
|
+
received = 0
|
|
291
|
+
for chunk in iter(lambda: response.read(65536), b""):
|
|
292
|
+
received += len(chunk)
|
|
293
|
+
if received > _max_download_bytes():
|
|
294
|
+
raise ValueError(
|
|
295
|
+
"lexsync: corpus download exceeded the 200 MB size limit. "
|
|
296
|
+
"Retrieve the delimited file manually and pass it to "
|
|
297
|
+
"load_lexicon()."
|
|
298
|
+
)
|
|
299
|
+
handle.write(chunk)
|
|
300
|
+
except (urllib.error.URLError, OSError) as exc:
|
|
301
|
+
if os.path.exists(part):
|
|
302
|
+
os.remove(part)
|
|
303
|
+
raise RuntimeError(
|
|
304
|
+
f"lexsync: could not download corpus '{name}' from {url} ({exc}). "
|
|
305
|
+
f"Check the URL in registry.yaml, or download the file manually and "
|
|
306
|
+
f"pass it to load_lexicon()."
|
|
307
|
+
) from exc
|
|
308
|
+
except ValueError:
|
|
309
|
+
if os.path.exists(part):
|
|
310
|
+
os.remove(part)
|
|
311
|
+
raise
|
|
312
|
+
if _starts_with_markup(part):
|
|
313
|
+
os.remove(part)
|
|
314
|
+
raise ValueError(
|
|
315
|
+
f"lexsync: corpus '{name}' returned an HTML page, not a delimited file "
|
|
316
|
+
f"({url}); the registry URL may have rotted. Retrieve the delimited file "
|
|
317
|
+
f"manually and pass it to load_lexicon()."
|
|
318
|
+
)
|
|
319
|
+
# 'sha256' is optional per registry entry; when present the download must
|
|
320
|
+
# match it before it may enter the cache.
|
|
321
|
+
expected = entry.get("sha256")
|
|
322
|
+
if expected and sha256_file(part) != expected:
|
|
323
|
+
os.remove(part)
|
|
324
|
+
raise ValueError(
|
|
325
|
+
f"lexsync: checksum mismatch for corpus '{name}'; the download does "
|
|
326
|
+
f"not match the registry's sha256. Retry the download, or verify the "
|
|
327
|
+
f"sha256 recorded in registry.yaml."
|
|
328
|
+
)
|
|
329
|
+
os.replace(part, dest)
|
|
330
|
+
print(f"lexsync: downloaded '{name}'. Please cite: {entry.get('citation', '(see registry)')}")
|
|
331
|
+
return dest
|