uavocab 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uavocab/__init__.py +225 -0
- uavocab/_store.py +187 -0
- uavocab/data/b1_aspect.parquet +0 -0
- uavocab/data/b1_collocations.parquet +0 -0
- uavocab/data/b1_government.parquet +0 -0
- uavocab/data/borodin_a1.parquet +0 -0
- uavocab/data/borodin_a2.parquet +0 -0
- uavocab/data/crosswalk.parquet +0 -0
- uavocab/data/frequency.parquet +0 -0
- uavocab/data/manifest.json +278 -0
- uavocab/data/stress.parquet +0 -0
- uavocab/data/validated_cefr.parquet +0 -0
- uavocab-0.5.0.dist-info/METADATA +133 -0
- uavocab-0.5.0.dist-info/RECORD +17 -0
- uavocab-0.5.0.dist-info/WHEEL +5 -0
- uavocab-0.5.0.dist-info/licenses/LICENSES.md +50 -0
- uavocab-0.5.0.dist-info/top_level.txt +1 -0
uavocab/__init__.py
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""uavocab: the open Ukrainian vocabulary data behind uavocab.org, offline.
|
|
2
|
+
|
|
3
|
+
The data is the Ukrainian Frequency study at https://uavocab.org/research.
|
|
4
|
+
This package was published as `ukvocab` up to 0.4.1; that name now installs
|
|
5
|
+
this one.
|
|
6
|
+
|
|
7
|
+
Four functions, kept small on purpose:
|
|
8
|
+
|
|
9
|
+
lookup(lemma, pos=None) one lemma: level, its source, rank, enrichment
|
|
10
|
+
levels(lemmas) batch: lemma -> level, with the source
|
|
11
|
+
coverage(lemmas, known_set) the running-text coverage computation
|
|
12
|
+
sources() version, snapshot and license per dataset
|
|
13
|
+
|
|
14
|
+
Every return value carries the dataset, version and snapshot it came from.
|
|
15
|
+
The package does not lemmatize: pass lemmas in the VESUM / UberText
|
|
16
|
+
convention (the infinitive for verbs, the nominative singular for nouns).
|
|
17
|
+
Predicted CEFR levels are deliberately not in this package.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from collections import Counter
|
|
23
|
+
from typing import Dict, Iterable, List, Optional, Union
|
|
24
|
+
|
|
25
|
+
from . import _store
|
|
26
|
+
from ._store import LEVELS, norm
|
|
27
|
+
|
|
28
|
+
__all__ = ["lookup", "levels", "coverage", "sources", "LEVELS", "norm", "__version__"]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def __getattr__(name):
|
|
32
|
+
if name == "__version__":
|
|
33
|
+
return _store.manifest()["packageVersion"]
|
|
34
|
+
raise AttributeError(name)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
LEVEL_SOURCE_VALIDATED = "PULS validated level, POS-aware join onto the frequency list (validated_cefr)"
|
|
38
|
+
LEVEL_SOURCE_CROSSWALK = "PULS validated level via the crosswalk: lemma-string join, no part of speech (crosswalk)"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _reading(row: dict) -> dict:
|
|
42
|
+
lv = _store.level_by_rank().get(row["rank"])
|
|
43
|
+
return {
|
|
44
|
+
"rank": row["rank"],
|
|
45
|
+
"pos": row["pos"],
|
|
46
|
+
"count": row["count"],
|
|
47
|
+
"permille": row["permille"],
|
|
48
|
+
"cumulative_coverage": row["cumulativeCoverage"],
|
|
49
|
+
"freq_band": row["freqBand"],
|
|
50
|
+
"level": lv["level"] if lv else None,
|
|
51
|
+
"join": lv["join"] if lv else None,
|
|
52
|
+
"stress": _store.stress_by_rank().get(row["rank"]),
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _enrichment(key: str) -> dict:
|
|
57
|
+
return {
|
|
58
|
+
"aspect": [dict(r) for r in _store.aspect_by_verb().get(key, [])],
|
|
59
|
+
"government": [dict(r) for r in _store.government_by_verb().get(key, [])],
|
|
60
|
+
"collocations": [dict(r) for r in _store.collocations_by_lemma().get(key, [])],
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def lookup(lemma: str, pos: Optional[str] = None) -> Optional[dict]:
|
|
65
|
+
"""One lemma. Returns None when no table knows the string at all.
|
|
66
|
+
|
|
67
|
+
With `pos` (a UD tag such as NOUN, VERB, ADJ, DET) the reading with that
|
|
68
|
+
part of speech is returned; without it, the most frequent reading, and
|
|
69
|
+
every other reading is listed under `readings` so a homograph is never
|
|
70
|
+
silently collapsed. The level comes from the validated overlay when the
|
|
71
|
+
lemma is on the frequency list (POS-aware); otherwise from the crosswalk,
|
|
72
|
+
which joins by lemma string only, and `level_source` says which.
|
|
73
|
+
"""
|
|
74
|
+
key = norm(lemma)
|
|
75
|
+
readings = _store.frequency_by_key().get(key, [])
|
|
76
|
+
if pos is not None:
|
|
77
|
+
readings = [r for r in readings if r["pos"] == pos]
|
|
78
|
+
primary = _reading(readings[0]) if readings else None
|
|
79
|
+
xw = _store.crosswalk_by_key().get(key)
|
|
80
|
+
if primary is None and xw is None:
|
|
81
|
+
return None
|
|
82
|
+
|
|
83
|
+
level = primary["level"] if primary else None
|
|
84
|
+
level_source = LEVEL_SOURCE_VALIDATED if level else None
|
|
85
|
+
if level is None and xw and xw.get("validatedLevel"):
|
|
86
|
+
level = xw["validatedLevel"]
|
|
87
|
+
level_source = LEVEL_SOURCE_CROSSWALK
|
|
88
|
+
|
|
89
|
+
used = ["frequency", "validated_cefr", "crosswalk", "borodin_a1", "borodin_a2"]
|
|
90
|
+
if primary is not None:
|
|
91
|
+
used.append("stress")
|
|
92
|
+
enrichment = _enrichment(key)
|
|
93
|
+
if any(enrichment.values()):
|
|
94
|
+
used += ["b1_aspect", "b1_government", "b1_collocations"]
|
|
95
|
+
|
|
96
|
+
return {
|
|
97
|
+
"query": lemma,
|
|
98
|
+
"lemma": readings[0]["lemma"] if readings else (xw["lemma"] if xw else key),
|
|
99
|
+
"on_list": primary is not None,
|
|
100
|
+
"rank": primary["rank"] if primary else None,
|
|
101
|
+
"pos": primary["pos"] if primary else (xw.get("pos") if xw else None),
|
|
102
|
+
"count": primary["count"] if primary else None,
|
|
103
|
+
"permille": primary["permille"] if primary else None,
|
|
104
|
+
"freq_band": primary["freq_band"] if primary else None,
|
|
105
|
+
"level": level,
|
|
106
|
+
"level_source": level_source,
|
|
107
|
+
"join": primary["join"] if primary else None,
|
|
108
|
+
"stress": primary["stress"] if primary else None,
|
|
109
|
+
"readings": [_reading(r) for r in _store.frequency_by_key().get(key, [])],
|
|
110
|
+
"minimums": {
|
|
111
|
+
"a1": key in _store.crosswalk_volume_keys("A1"),
|
|
112
|
+
"a2": key in _store.crosswalk_volume_keys("A2"),
|
|
113
|
+
"a1_entries": [r["entry"] for r in _store.borodin_keys("a1").get(key, [])],
|
|
114
|
+
"a2_entries": [r["entry"] for r in _store.borodin_keys("a2").get(key, [])],
|
|
115
|
+
},
|
|
116
|
+
"crosswalk": {
|
|
117
|
+
"class": xw["class"],
|
|
118
|
+
"volume": xw.get("volume"),
|
|
119
|
+
"validated_level": xw.get("validatedLevel"),
|
|
120
|
+
"relative": xw.get("relative"),
|
|
121
|
+
"relative_via": xw.get("relativeVia"),
|
|
122
|
+
} if xw else None,
|
|
123
|
+
"enrichment": enrichment,
|
|
124
|
+
"sources": {name: _store.source_of(name) for name in used},
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def levels(lemmas: Iterable[str]) -> Dict[str, dict]:
|
|
129
|
+
"""Batch lookup: input string -> {level, level_source, rank, pos, source}.
|
|
130
|
+
A lemma no table knows gets level None and an empty source."""
|
|
131
|
+
out: Dict[str, dict] = {}
|
|
132
|
+
for lemma in lemmas:
|
|
133
|
+
hit = lookup(lemma)
|
|
134
|
+
if hit is None:
|
|
135
|
+
out[lemma] = {"level": None, "level_source": None, "rank": None, "pos": None, "source": None}
|
|
136
|
+
continue
|
|
137
|
+
src_name = "validated_cefr" if hit["level_source"] == LEVEL_SOURCE_VALIDATED else ("crosswalk" if hit["level"] else None)
|
|
138
|
+
out[lemma] = {
|
|
139
|
+
"level": hit["level"],
|
|
140
|
+
"level_source": hit["level_source"],
|
|
141
|
+
"rank": hit["rank"],
|
|
142
|
+
"pos": hit["pos"],
|
|
143
|
+
"source": _store.source_of(src_name) if src_name else None,
|
|
144
|
+
}
|
|
145
|
+
return out
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
KnownSet = Union[str, int, Iterable[str]]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _resolve_known(known_set: KnownSet):
|
|
152
|
+
"""A known set is an iterable of lemma strings, a level name (every
|
|
153
|
+
validated lemma at or below it), 'borodin-a1' / 'borodin-a2' (the
|
|
154
|
+
minimum, on and off the list, A2 including A1), or an int N (the top N of
|
|
155
|
+
the frequency list)."""
|
|
156
|
+
if isinstance(known_set, int) and not isinstance(known_set, bool):
|
|
157
|
+
return _store.top_n_set(known_set), f"top {known_set} of the frequency list", ["frequency"]
|
|
158
|
+
if isinstance(known_set, str):
|
|
159
|
+
tag = known_set.strip()
|
|
160
|
+
if tag.upper() in LEVELS:
|
|
161
|
+
return _store.level_set(tag.upper()), f"every validated lemma at or below {tag.upper()}", ["validated_cefr", "frequency"]
|
|
162
|
+
if tag.lower() in ("borodin-a1", "a1-minimum"):
|
|
163
|
+
return _store.crosswalk_volume_keys("A1"), "the Borodin A1 minimum (every headword, on or off the list)", ["crosswalk", "borodin_a1"]
|
|
164
|
+
if tag.lower() in ("borodin-a2", "a2-minimum"):
|
|
165
|
+
return _store.crosswalk_volume_keys("A2"), "the Borodin A1 and A2 minimums (every headword, on or off the list)", ["crosswalk", "borodin_a1", "borodin_a2"]
|
|
166
|
+
raise ValueError(f"unknown known_set name {known_set!r}: a level (A1..C1), 'borodin-a1', 'borodin-a2', an int, or an iterable of lemmas")
|
|
167
|
+
return frozenset(norm(s) for s in known_set), "a caller-supplied set of lemmas", ["frequency"]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def coverage(lemmas: Iterable[str], known_set: KnownSet, top_unknown: int = 20) -> dict:
|
|
171
|
+
"""Running-text coverage of a lemmatized text against a known set.
|
|
172
|
+
|
|
173
|
+
`lemmas` is the text as a sequence of lemmas, one per token, repeats
|
|
174
|
+
included (coverage is token coverage, not type coverage). Tokens are
|
|
175
|
+
counted as known when their normalized lemma is in the known set; the
|
|
176
|
+
package does not decide what a word is, so pass only the tokens you want
|
|
177
|
+
counted (a caller usually drops punctuation, numbers and proper names).
|
|
178
|
+
Also reports how many tokens are on the frequency list at all, and the
|
|
179
|
+
most frequent unknown lemmas, so a gap is a list rather than a number.
|
|
180
|
+
"""
|
|
181
|
+
known, description, used = _resolve_known(known_set)
|
|
182
|
+
counts = Counter(norm(t) for t in lemmas)
|
|
183
|
+
total = sum(counts.values())
|
|
184
|
+
known_tokens = sum(n for k, n in counts.items() if k in known)
|
|
185
|
+
on_list = sum(n for k, n in counts.items() if k in _store.frequency_by_key())
|
|
186
|
+
unknown = Counter({k: n for k, n in counts.items() if k not in known})
|
|
187
|
+
by_level: Dict[str, int] = {lvl: 0 for lvl in LEVELS}
|
|
188
|
+
unleveled = 0
|
|
189
|
+
for k, n in counts.items():
|
|
190
|
+
readings = _store.frequency_by_key().get(k)
|
|
191
|
+
lv = _store.level_by_rank().get(readings[0]["rank"]) if readings else None
|
|
192
|
+
if lv:
|
|
193
|
+
by_level[lv["level"]] += n
|
|
194
|
+
else:
|
|
195
|
+
unleveled += n
|
|
196
|
+
return {
|
|
197
|
+
"tokens": total,
|
|
198
|
+
"types": len(counts),
|
|
199
|
+
"known": known_tokens,
|
|
200
|
+
"unknown": total - known_tokens,
|
|
201
|
+
"coverage": (known_tokens / total) if total else 0.0,
|
|
202
|
+
"on_list": on_list,
|
|
203
|
+
"on_list_share": (on_list / total) if total else 0.0,
|
|
204
|
+
"by_validated_level": by_level,
|
|
205
|
+
"unleveled": unleveled,
|
|
206
|
+
"known_set": description,
|
|
207
|
+
"known_set_size": len(known),
|
|
208
|
+
"top_unknown": unknown.most_common(top_unknown),
|
|
209
|
+
"sources": {name: _store.source_of(name) for name in used},
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def sources() -> dict:
|
|
214
|
+
"""Version, snapshot, license, attribution, citation and checksum per
|
|
215
|
+
dataset, plus the combined license and why, straight from the manifest."""
|
|
216
|
+
m = _store.manifest()
|
|
217
|
+
return {
|
|
218
|
+
"package": m["package"],
|
|
219
|
+
"package_version": m["packageVersion"],
|
|
220
|
+
"site": m["site"],
|
|
221
|
+
"combined_license": m["combinedLicense"],
|
|
222
|
+
"combined_license_why": m["combinedLicenseWhy"],
|
|
223
|
+
"not_shipped": list(m["notShipped"]),
|
|
224
|
+
"datasets": {name: dict(d) for name, d in m["datasets"].items()},
|
|
225
|
+
}
|
uavocab/_store.py
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Lazy, read-only access to the package's Parquet data tier.
|
|
2
|
+
|
|
3
|
+
Nothing is read at import time. Each table loads on first use through
|
|
4
|
+
pyarrow, and the lookup indexes are built once per process. The manifest
|
|
5
|
+
(manifest.json, generated by scripts/build-package.py in
|
|
6
|
+
https://github.com/jake0lawrence/UA-Vocab from the published release files) is the only description of the data; this module
|
|
7
|
+
never hard-codes a row count, a license or a snapshot date.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import unicodedata
|
|
15
|
+
from functools import lru_cache
|
|
16
|
+
from typing import Dict, List, Optional
|
|
17
|
+
|
|
18
|
+
DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
|
|
19
|
+
|
|
20
|
+
LEVELS = ("A1", "A2", "B1", "B2", "C1")
|
|
21
|
+
LEVEL_INDEX = {lvl: i for i, lvl in enumerate(LEVELS)}
|
|
22
|
+
|
|
23
|
+
_APOSTROPHES = ("'", "ʼ", "’", "‘", "ʹ", "`", "´")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def norm(s: str) -> str:
|
|
27
|
+
"""The join key used across every table: NFC, the pedagogical stress mark
|
|
28
|
+
(U+0301) stripped, every apostrophe glyph folded to U+2019 (the frequency
|
|
29
|
+
list's glyph), whitespace trimmed, lowercased."""
|
|
30
|
+
s = unicodedata.normalize("NFD", s or "")
|
|
31
|
+
s = "".join(ch for ch in s if ch != "́")
|
|
32
|
+
s = unicodedata.normalize("NFC", s)
|
|
33
|
+
for glyph in _APOSTROPHES:
|
|
34
|
+
s = s.replace(glyph, "’")
|
|
35
|
+
return s.strip().lower()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@lru_cache(maxsize=None)
|
|
39
|
+
def manifest() -> dict:
|
|
40
|
+
with open(os.path.join(DATA_DIR, "manifest.json"), encoding="utf-8") as fh:
|
|
41
|
+
return json.load(fh)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@lru_cache(maxsize=None)
|
|
45
|
+
def table(name: str):
|
|
46
|
+
"""The pyarrow Table for a dataset name from the manifest."""
|
|
47
|
+
import pyarrow.parquet as pq
|
|
48
|
+
|
|
49
|
+
meta = manifest()["datasets"][name]
|
|
50
|
+
return pq.read_table(os.path.join(DATA_DIR, meta["file"]))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@lru_cache(maxsize=None)
|
|
54
|
+
def rows(name: str) -> List[dict]:
|
|
55
|
+
"""The dataset as a list of plain dicts (nulls as None)."""
|
|
56
|
+
return table(name).to_pylist()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def source_of(name: str) -> dict:
|
|
60
|
+
"""The provenance stamp every return value carries for a dataset."""
|
|
61
|
+
m = manifest()["datasets"][name]
|
|
62
|
+
return {
|
|
63
|
+
"dataset": name,
|
|
64
|
+
"version": m["version"],
|
|
65
|
+
"snapshot": m["snapshot"],
|
|
66
|
+
"license": m["license"],
|
|
67
|
+
"rows": m["rows"],
|
|
68
|
+
"sha256": m["sha256"],
|
|
69
|
+
"conceptDoi": m.get("conceptDoi"),
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# ---------------------------------------------------------------- indexes
|
|
74
|
+
@lru_cache(maxsize=None)
|
|
75
|
+
def frequency_by_key() -> Dict[str, List[dict]]:
|
|
76
|
+
"""norm(lemma) -> frequency rows (rank order); a spelling can have several
|
|
77
|
+
readings because the corpus tagger split it by part of speech."""
|
|
78
|
+
out: Dict[str, List[dict]] = {}
|
|
79
|
+
for r in rows("frequency"):
|
|
80
|
+
out.setdefault(norm(r["lemma"]), []).append(r)
|
|
81
|
+
for v in out.values():
|
|
82
|
+
v.sort(key=lambda r: r["rank"])
|
|
83
|
+
return out
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@lru_cache(maxsize=None)
|
|
87
|
+
def frequency_by_rank() -> Dict[int, dict]:
|
|
88
|
+
return {r["rank"]: r for r in rows("frequency")}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@lru_cache(maxsize=None)
|
|
92
|
+
def level_by_rank() -> Dict[int, dict]:
|
|
93
|
+
"""rank -> validated overlay row (level, join)."""
|
|
94
|
+
return {r["rank"]: r for r in rows("validated_cefr")}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@lru_cache(maxsize=None)
|
|
98
|
+
def stress_by_rank() -> Dict[int, dict]:
|
|
99
|
+
"""rank -> the stress row (stress, variants as a list, status, note, source)."""
|
|
100
|
+
out: Dict[int, dict] = {}
|
|
101
|
+
for r in rows("stress"):
|
|
102
|
+
out[r["rank"]] = {
|
|
103
|
+
"stress": r["stress"] or None,
|
|
104
|
+
"variants": [v for v in (r["stress_variants"] or "").split("|") if v],
|
|
105
|
+
"status": r["stress_status"],
|
|
106
|
+
"note": r["note"] or None,
|
|
107
|
+
"source": r["source"],
|
|
108
|
+
}
|
|
109
|
+
return out
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@lru_cache(maxsize=None)
|
|
113
|
+
def crosswalk_by_key() -> Dict[str, dict]:
|
|
114
|
+
return {norm(r["lemma"]): r for r in rows("crosswalk")}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@lru_cache(maxsize=None)
|
|
118
|
+
def borodin_keys(volume: str) -> Dict[str, List[dict]]:
|
|
119
|
+
"""norm(headword lemma) -> the matched entries of a Borodin volume. Only
|
|
120
|
+
matched entries carry a lemma; off-list headwords are reached through the
|
|
121
|
+
crosswalk's volume column instead."""
|
|
122
|
+
name = {"a1": "borodin_a1", "a2": "borodin_a2"}[volume]
|
|
123
|
+
out: Dict[str, List[dict]] = {}
|
|
124
|
+
for r in rows(name):
|
|
125
|
+
if r.get("matched") and r.get("lemma"):
|
|
126
|
+
out.setdefault(norm(r["lemma"]), []).append(r)
|
|
127
|
+
return out
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@lru_cache(maxsize=None)
|
|
131
|
+
def crosswalk_volume_keys(volume: str) -> frozenset:
|
|
132
|
+
"""Every lemma string a Borodin volume carries, on or off the frequency
|
|
133
|
+
list, as the crosswalk records it (a word in both volumes counts as A1)."""
|
|
134
|
+
keys = set()
|
|
135
|
+
for r in rows("crosswalk"):
|
|
136
|
+
v = r.get("volume")
|
|
137
|
+
if v == volume or (volume == "A2" and v == "A1"):
|
|
138
|
+
keys.add(norm(r["lemma"]))
|
|
139
|
+
return frozenset(keys)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@lru_cache(maxsize=None)
|
|
143
|
+
def aspect_by_verb() -> Dict[str, List[dict]]:
|
|
144
|
+
out: Dict[str, List[dict]] = {}
|
|
145
|
+
for r in rows("b1_aspect"):
|
|
146
|
+
out.setdefault(norm(r["b1_verb"]), []).append(r)
|
|
147
|
+
return out
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@lru_cache(maxsize=None)
|
|
151
|
+
def government_by_verb() -> Dict[str, List[dict]]:
|
|
152
|
+
out: Dict[str, List[dict]] = {}
|
|
153
|
+
for r in rows("b1_government"):
|
|
154
|
+
out.setdefault(norm(r["verb"]), []).append(r)
|
|
155
|
+
return out
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
@lru_cache(maxsize=None)
|
|
159
|
+
def collocations_by_lemma() -> Dict[str, List[dict]]:
|
|
160
|
+
out: Dict[str, List[dict]] = {}
|
|
161
|
+
for r in rows("b1_collocations"):
|
|
162
|
+
out.setdefault(norm(r["lemma"]), []).append(r)
|
|
163
|
+
return out
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
@lru_cache(maxsize=None)
|
|
167
|
+
def level_set(level: str) -> frozenset:
|
|
168
|
+
"""norm(lemma) of every validated lemma at or below a level."""
|
|
169
|
+
if level not in LEVEL_INDEX:
|
|
170
|
+
raise ValueError(f"unknown level {level!r}; one of {LEVELS}")
|
|
171
|
+
top = LEVEL_INDEX[level]
|
|
172
|
+
by_rank = frequency_by_rank()
|
|
173
|
+
return frozenset(norm(by_rank[r["rank"]]["lemma"]) for r in rows("validated_cefr") if LEVEL_INDEX[r["level"]] <= top)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@lru_cache(maxsize=None)
|
|
177
|
+
def top_n_set(n: int) -> frozenset:
|
|
178
|
+
return frozenset(norm(r["lemma"]) for r in rows("frequency") if r["rank"] <= n)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def clear_cache() -> None:
|
|
182
|
+
"""Drop every loaded table and index (tests, or long processes that want
|
|
183
|
+
the memory back)."""
|
|
184
|
+
for fn in (manifest, table, rows, frequency_by_key, frequency_by_rank, level_by_rank, crosswalk_by_key, stress_by_rank,
|
|
185
|
+
borodin_keys, crosswalk_volume_keys, aspect_by_verb, government_by_verb, collocations_by_lemma,
|
|
186
|
+
level_set, top_n_set):
|
|
187
|
+
fn.cache_clear()
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
{
|
|
2
|
+
"package": "uavocab",
|
|
3
|
+
"packageVersion": "0.5.0",
|
|
4
|
+
"generatedBy": "build-package.py (scripts/build-package.py in https://github.com/jake0lawrence/UA-Vocab), from the release files under downloads/ukrainian-frequency",
|
|
5
|
+
"site": "https://uavocab.org/research",
|
|
6
|
+
"combinedLicense": "CC-BY-NC-SA-4.0",
|
|
7
|
+
"combinedLicenseWhy": "The PULS levels (the validated CEFR overlay, the crosswalk, the related-verb levels in the aspect table and the puls_level column of the two Borodin tables) are CC BY-NC-SA 4.0 at the authors' decision (O. Synchak, 2026-09-24), and NonCommercial and share-alike propagate to the combined work. They are the one NonCommercial component, redistributed with the authors' permission; nothing else NonCommercial is shipped. The frequency list and the two Borodin minimums are CC-BY 4.0 on their own and stay so, and the government and collocation tables are CC-BY-SA 4.0; each dataset's own license is recorded beside it, so a downstream user who drops the PULS level columns owes attribution and share-alike only. The stress column is CC-BY-SA 4.0 (Wiktionary-derived rows) and stays its own table, joined on rank.",
|
|
8
|
+
"notShipped": [
|
|
9
|
+
"predicted CEFR levels (the gap-fill model's file on the site; predictions are never merged into the validated overlay)",
|
|
10
|
+
"VESUM-derived lexicon or wordform data (CC BY-NC-SA)",
|
|
11
|
+
"UniMorph derivational families (CC BY-NC 4.0; a modeling input, not data)",
|
|
12
|
+
"Wikipedia example sentences (b1-extension/examples.csv)",
|
|
13
|
+
"the Wiktionary paradigm tables (the profiler bundle)"
|
|
14
|
+
],
|
|
15
|
+
"datasets": {
|
|
16
|
+
"frequency": {
|
|
17
|
+
"file": "frequency.parquet",
|
|
18
|
+
"rows": 12000,
|
|
19
|
+
"columns": [
|
|
20
|
+
"rank",
|
|
21
|
+
"lemma",
|
|
22
|
+
"pos",
|
|
23
|
+
"count",
|
|
24
|
+
"permille",
|
|
25
|
+
"cumulativeCoverage",
|
|
26
|
+
"freqBand"
|
|
27
|
+
],
|
|
28
|
+
"source": {
|
|
29
|
+
"path": "public/downloads/ukrainian-frequency/v0.2.0/lemmas.csv",
|
|
30
|
+
"sha256": "185f108882d024e880f4432bcaeac1bcaab7abd0596121b8581609945cf6a0b2",
|
|
31
|
+
"rows": 12000
|
|
32
|
+
},
|
|
33
|
+
"sha256": "d2e51f1041ba554bfbeb562b26c78b6489eab9f118b49ea8bd22c3406f521d39",
|
|
34
|
+
"version": "v0.2.0",
|
|
35
|
+
"snapshot": "2026-07-14",
|
|
36
|
+
"license": "CC-BY-4.0",
|
|
37
|
+
"title": "Ukrainian Frequency: the top-12,000 lemma list",
|
|
38
|
+
"description": "Lemma, UD part of speech, raw occurrence and document counts, per-mille share, cumulative running-text coverage and thousand-rank band, over the UberText 2.0 lemma frequency dictionary (derived aggregate counts, not corpus text).",
|
|
39
|
+
"attribution": "Chaplynskyi, D. (2023). Introducing UberText 2.0: A Corpus of Modern Ukrainian at Scale. Proc. 2nd Ukrainian NLP Workshop (UNLP), ACL. lang-uk, CC-BY 4.0.",
|
|
40
|
+
"citation": "Lawrence, J. (2026). Ukrainian Frequency: a lemma-based, coverage-graded open frequency list (v0.2.0). uavocab.org/research. Derived from UberText 2.0 (Chaplynskyi, 2023).",
|
|
41
|
+
"url": "https://uavocab.org/research#list",
|
|
42
|
+
"conceptDoi": "10.5281/zenodo.22823134"
|
|
43
|
+
},
|
|
44
|
+
"validated_cefr": {
|
|
45
|
+
"file": "validated_cefr.parquet",
|
|
46
|
+
"rows": 6403,
|
|
47
|
+
"columns": [
|
|
48
|
+
"rank",
|
|
49
|
+
"lemma",
|
|
50
|
+
"pos",
|
|
51
|
+
"level",
|
|
52
|
+
"confidence",
|
|
53
|
+
"join"
|
|
54
|
+
],
|
|
55
|
+
"source": {
|
|
56
|
+
"path": "public/downloads/ukrainian-frequency/validated-cefr/lemmas.csv",
|
|
57
|
+
"sha256": "07247fcd79b08475ecc98ace56e45d263046b093574fa96e64781688c30ab5ef",
|
|
58
|
+
"rows": 6403
|
|
59
|
+
},
|
|
60
|
+
"sha256": "40c1b9d4d717c1f7875126351d006c2a8cd38669bc9ec6e273c94ad3c9e2d3a2",
|
|
61
|
+
"version": "2026-09-23",
|
|
62
|
+
"snapshot": "2026-07-23",
|
|
63
|
+
"license": "CC-BY-NC-SA-4.0",
|
|
64
|
+
"title": "The PULS CEFR overlay: validated A1 to C1 levels on the top-12,000 list",
|
|
65
|
+
"description": "Per lemma and part of speech, the onset level from the Ukrainian Vocabulary Profile (PULS), joined POS-aware onto the frequency list; the join column records how each row matched (pos-family, lemma-only, or one of ten declared cross-tag rules). Validated levels only; no predicted level is in this package.",
|
|
66
|
+
"attribution": "Ukrainian Vocabulary Profile (PULS), Synchak, O., Starko, V., Burak, M., Svystun, M., https://puls.peremova.org/ (eLex 2025). Per-lemma levels redistributed with the authors' permission (V. Starko, 2026-07), CC BY-NC-SA 4.0 at the authors' decision (O. Synchak, 2026-09-24). Cite as: Профіль української лексики (ПУЛЬС) / О. Синчак, В. Старко, М. Бурак, М. Свистун та ін. Львів: Школа української мови та культури УКУ, 2026. Snapshot of the published profile taken 2026-07-23; the profile is not final.",
|
|
67
|
+
"citation": "Synchak, O., Starko, V., Burak, M. and Svystun, M. (2025). Corpus-Based Vocabulary Profiling for Ukrainian: From Lexical Analysis to CEFR. eLex 2025. Joined onto the Ukrainian Frequency list (Lawrence, 2026).",
|
|
68
|
+
"url": "https://uavocab.org/research#cefr-validated",
|
|
69
|
+
"conceptDoi": null
|
|
70
|
+
},
|
|
71
|
+
"borodin_a1": {
|
|
72
|
+
"file": "borodin_a1.parquet",
|
|
73
|
+
"rows": 929,
|
|
74
|
+
"columns": [
|
|
75
|
+
"entry",
|
|
76
|
+
"kind",
|
|
77
|
+
"matched",
|
|
78
|
+
"rank",
|
|
79
|
+
"lemma",
|
|
80
|
+
"pos",
|
|
81
|
+
"puls_level"
|
|
82
|
+
],
|
|
83
|
+
"source": {
|
|
84
|
+
"path": "public/downloads/ukrainian-frequency/borodin-a1/lemmas.csv",
|
|
85
|
+
"sha256": "9b3d054dc9611c8fe3b2c62ddf7edb513586e42faf39e23687e8bab9c5334aab",
|
|
86
|
+
"rows": 929
|
|
87
|
+
},
|
|
88
|
+
"sha256": "9069450ba7dbfb32dc957c11f5a09d4a83ed11eb60a8673ef7951e3aaa2c7862",
|
|
89
|
+
"version": "2026-09-23",
|
|
90
|
+
"snapshot": "2026-07-27",
|
|
91
|
+
"license": "CC-BY-4.0",
|
|
92
|
+
"title": "1000 i 1 slovo, A1: the lexical minimum joined onto the frequency list",
|
|
93
|
+
"description": "One row per published entry, verbatim, with the frequency rank, lemma, POS and validated PULS level where a headword matches the list. The puls_level column is CC-BY-NC-SA (PULS); the entries and ranks are CC-BY.",
|
|
94
|
+
"attribution": "Borodin, K. and Turkevych, O. 1000 i 1 slovo. Leksychnyi minimum z ukrainskoi movy yak inozemnoi. Riven A1. Slavic Language Education 3, LANGUAGE-LAB, Humboldt-Universitat zu Berlin. https://doi.org/10.18452/28236. CC-BY 4.0. Machine-readable list provided by the authors, 2026-07-27.",
|
|
95
|
+
"citation": "Borodin, K. and Turkevych, O. (2023). 1000 i 1 slovo: Lexical Minimum in Ukrainian as a Foreign Language, Level A1. https://doi.org/10.18452/28236",
|
|
96
|
+
"url": "https://uavocab.org/research#cefr-borodin-a1",
|
|
97
|
+
"conceptDoi": null
|
|
98
|
+
},
|
|
99
|
+
"borodin_a2": {
|
|
100
|
+
"file": "borodin_a2.parquet",
|
|
101
|
+
"rows": 1000,
|
|
102
|
+
"columns": [
|
|
103
|
+
"entry",
|
|
104
|
+
"kind",
|
|
105
|
+
"in_a1",
|
|
106
|
+
"matched",
|
|
107
|
+
"rank",
|
|
108
|
+
"lemma",
|
|
109
|
+
"pos",
|
|
110
|
+
"puls_level"
|
|
111
|
+
],
|
|
112
|
+
"source": {
|
|
113
|
+
"path": "public/downloads/ukrainian-frequency/borodin-a2/lemmas.csv",
|
|
114
|
+
"sha256": "904190e6e0a76acbf6e5c0f41fa1f171d71245bf616903e010055280703d17f8",
|
|
115
|
+
"rows": 1000
|
|
116
|
+
},
|
|
117
|
+
"sha256": "6c6c55cdbf2fe23c297ccadee34597c02ef30ea0207f4b3331211b2450be71ac",
|
|
118
|
+
"version": "2026-09-23",
|
|
119
|
+
"snapshot": "2026-07-28",
|
|
120
|
+
"license": "CC-BY-4.0",
|
|
121
|
+
"title": "1000 i 1 slovo, A2: the lexical minimum joined onto the frequency list",
|
|
122
|
+
"description": "One row per published register entry, verbatim, with whether the A1 volume already carried it, and the frequency rank, lemma, POS and validated PULS level where a headword matches the list. The puls_level column is CC-BY-NC-SA (PULS); the entries and ranks are CC-BY.",
|
|
123
|
+
"attribution": "Borodin, K. and Lazarenko, O. (2026). 1000 i 1 slovo. Leksychnyi minimum z ukrainskoi movy yak inozemnoi. Riven A2. Europa-Universitat Viadrina Frankfurt (Oder). https://doi.org/10.11584/opus4-1448. CC-BY 4.0.",
|
|
124
|
+
"citation": "Borodin, K. and Lazarenko, O. (2026). 1000 i 1 slovo: Lexical Minimum in Ukrainian as a Foreign Language, Level A2. https://doi.org/10.11584/opus4-1448",
|
|
125
|
+
"url": "https://uavocab.org/research#cefr-borodin-a2",
|
|
126
|
+
"conceptDoi": null
|
|
127
|
+
},
|
|
128
|
+
"crosswalk": {
|
|
129
|
+
"file": "crosswalk.parquet",
|
|
130
|
+
"rows": 10366,
|
|
131
|
+
"columns": [
|
|
132
|
+
"lemma",
|
|
133
|
+
"rank",
|
|
134
|
+
"pos",
|
|
135
|
+
"volume",
|
|
136
|
+
"entry",
|
|
137
|
+
"validatedLevel",
|
|
138
|
+
"class",
|
|
139
|
+
"relative",
|
|
140
|
+
"relativeVia"
|
|
141
|
+
],
|
|
142
|
+
"source": {
|
|
143
|
+
"path": "public/downloads/ukrainian-frequency/minimums-crosswalk/v1.2.0/crosswalk.csv",
|
|
144
|
+
"sha256": "deddcb787c459e699e8bed4a234cb00e7d189cfb9b075d1a7d3bdeb2c6a7adc3",
|
|
145
|
+
"rows": 10366
|
|
146
|
+
},
|
|
147
|
+
"sha256": "1fd1770c9d461218640a97012f78542a7d66b52516db0d111a384d5d42e0a2ff",
|
|
148
|
+
"version": "v1.2.0",
|
|
149
|
+
"snapshot": "2026-09-13",
|
|
150
|
+
"license": "CC-BY-NC-SA-4.0",
|
|
151
|
+
"title": "The minimums crosswalk: every lemma in the union of the two minimums and the validated profile, classified",
|
|
152
|
+
"description": "One row per lemma string in the union of the Borodin A1 and A2 minimums and the whole published PULS profile, joined by lemma string (no POS on the minimum side), with the class of each agreement or disagreement: agree, earlier, later, absent, beyond, unprofiled.",
|
|
153
|
+
"attribution": "Borodin and Turkevych (A1, https://doi.org/10.18452/28236) and Borodin and Lazarenko (A2, https://doi.org/10.11584/opus4-1448), CC-BY 4.0; Ukrainian Vocabulary Profile (PULS), Synchak, O., Starko, V., Burak, M., Svystun, M., by permission, CC BY-NC-SA 4.0; ranks from UberText 2.0 (lang-uk), CC-BY 4.0.",
|
|
154
|
+
"citation": "Lawrence, J. (2026). A crosswalk between the \"1000 i 1 slovo\" A1 and A2 lexical minimums and the validated Ukrainian Vocabulary Profile (v1.2.0). uavocab.org/research.",
|
|
155
|
+
"url": "https://uavocab.org/research/crosswalk",
|
|
156
|
+
"conceptDoi": null
|
|
157
|
+
},
|
|
158
|
+
"b1_aspect": {
|
|
159
|
+
"file": "b1_aspect.parquet",
|
|
160
|
+
"rows": 1028,
|
|
161
|
+
"columns": [
|
|
162
|
+
"b1_verb",
|
|
163
|
+
"b1_rank",
|
|
164
|
+
"b1_aspect",
|
|
165
|
+
"related",
|
|
166
|
+
"kind",
|
|
167
|
+
"related_aspect",
|
|
168
|
+
"related_level",
|
|
169
|
+
"related_validated",
|
|
170
|
+
"related_rank",
|
|
171
|
+
"ubertext_count",
|
|
172
|
+
"attested",
|
|
173
|
+
"split_level"
|
|
174
|
+
],
|
|
175
|
+
"source": {
|
|
176
|
+
"path": "public/downloads/ukrainian-frequency/b1-extension/aspect-relations.csv",
|
|
177
|
+
"sha256": "fb40340e6829dba542a43679ff7c286c783a23fb950deee4bae338eebdd60a37",
|
|
178
|
+
"rows": 1028
|
|
179
|
+
},
|
|
180
|
+
"sha256": "7eca2f9fe7aae444c4e08afca163b1e354ecf1477c16f2d595d295a58a059171",
|
|
181
|
+
"version": "2026-09-23",
|
|
182
|
+
"snapshot": "2026-09-11",
|
|
183
|
+
"license": "CC-BY-NC-SA-4.0",
|
|
184
|
+
"title": "B1 aspect relations: each validated-B1 verb with its aspect partner and its attested prefixed derivatives",
|
|
185
|
+
"description": "One row per (B1 verb, related verb). kind is partner (Wiktionary-listed or hand-curated suppletive) or prefixed (a corpus-attested prefix derivative); each related verb keeps its own PULS level, so split_level marks a pair the profile places at two levels.",
|
|
186
|
+
"attribution": "Levels: PULS (Synchak, O., Starko, V., Burak, M., Svystun, M.), by permission, CC BY-NC-SA 4.0. Aspect partners: Wiktionary via kaikki.org, CC-BY-SA 3.0. Counts: UberText 2.0 (lang-uk), CC-BY 4.0. No VESUM data is bundled.",
|
|
187
|
+
"citation": "Lawrence, J. (2026). Ukrainian Frequency: the B1 extension, aspect relations. uavocab.org/research.",
|
|
188
|
+
"url": "https://uavocab.org/research#b1-aspect",
|
|
189
|
+
"conceptDoi": null
|
|
190
|
+
},
|
|
191
|
+
"b1_government": {
|
|
192
|
+
"file": "b1_government.parquet",
|
|
193
|
+
"rows": 3157,
|
|
194
|
+
"columns": [
|
|
195
|
+
"verb",
|
|
196
|
+
"rank",
|
|
197
|
+
"total_dependents",
|
|
198
|
+
"preposition",
|
|
199
|
+
"case",
|
|
200
|
+
"n",
|
|
201
|
+
"share",
|
|
202
|
+
"shown"
|
|
203
|
+
],
|
|
204
|
+
"source": {
|
|
205
|
+
"path": "public/downloads/ukrainian-frequency/b1-extension/government.csv",
|
|
206
|
+
"sha256": "2df2019cd92a74b441c9304bf77b08a90aa6444a2c40b766d43fc9f10b262d62",
|
|
207
|
+
"rows": 3157
|
|
208
|
+
},
|
|
209
|
+
"sha256": "ba5484257b308d7597efcd9ec449e6ebbece054208070f729081e1e331ab816b",
|
|
210
|
+
"version": "2026-09-23",
|
|
211
|
+
"snapshot": "2026-09-11",
|
|
212
|
+
"license": "CC-BY-SA-4.0",
|
|
213
|
+
"title": "B1 case government: verb + preposition + case counts",
|
|
214
|
+
"description": "Per validated-B1 verb, each preposition + case pattern with its count and share of the verb's nominal dependents, over 2,880,835 parsed tokens of the UberText 2.0 Wikipedia subcorpus (spaCy uk_core_news_sm 3.8.0). shown marks patterns above the display threshold; every counted pattern is present. The corpus slice was cut on the 1,585-lemma B1 set of 2026-09-11.",
|
|
215
|
+
"attribution": "Counts over UberText 2.0, Wikipedia subcorpus (lang-uk), CC-BY 4.0; derived statistics CC-BY-SA 4.0.",
|
|
216
|
+
"citation": "Lawrence, J. (2026). Ukrainian Frequency: the B1 extension, case government. uavocab.org/research.",
|
|
217
|
+
"url": "https://uavocab.org/research#b1-usage",
|
|
218
|
+
"conceptDoi": null
|
|
219
|
+
},
|
|
220
|
+
"b1_collocations": {
|
|
221
|
+
"file": "b1_collocations.parquet",
|
|
222
|
+
"rows": 6219,
|
|
223
|
+
"columns": [
|
|
224
|
+
"lemma",
|
|
225
|
+
"relation",
|
|
226
|
+
"side",
|
|
227
|
+
"collocate",
|
|
228
|
+
"collocate_pos",
|
|
229
|
+
"n",
|
|
230
|
+
"logdice"
|
|
231
|
+
],
|
|
232
|
+
"source": {
|
|
233
|
+
"path": "public/downloads/ukrainian-frequency/b1-extension/collocations.csv",
|
|
234
|
+
"sha256": "bc8178f7a6b2fcc20f4cad7629232c63ec4a796d3b538d03cf896cddfd0b1d92",
|
|
235
|
+
"rows": 6219
|
|
236
|
+
},
|
|
237
|
+
"sha256": "18656b18631400ba12730b39a66004467b48e097c9f8b18acefa7b6ef16db1b0",
|
|
238
|
+
"version": "2026-09-23",
|
|
239
|
+
"snapshot": "2026-09-11",
|
|
240
|
+
"license": "CC-BY-SA-4.0",
|
|
241
|
+
"title": "B1 collocations: dependency-linked lemma pairs ranked by logDice",
|
|
242
|
+
"description": "Per validated-B1 lemma, the strongest dependency-linked collocates by logDice (14 + log2(2 f_xy / (f_x + f_y))), at least three co-occurrences, up to eight per lemma; relation is the UD dependency label and side says whether the B1 lemma is the head or the child. Same corpus slice as the government table.",
|
|
243
|
+
"attribution": "Counts over UberText 2.0, Wikipedia subcorpus (lang-uk), CC-BY 4.0; derived statistics CC-BY-SA 4.0.",
|
|
244
|
+
"citation": "Lawrence, J. (2026). Ukrainian Frequency: the B1 extension, collocations. uavocab.org/research.",
|
|
245
|
+
"url": "https://uavocab.org/research#b1-usage",
|
|
246
|
+
"conceptDoi": null
|
|
247
|
+
},
|
|
248
|
+
"stress": {
|
|
249
|
+
"file": "stress.parquet",
|
|
250
|
+
"rows": 12000,
|
|
251
|
+
"columns": [
|
|
252
|
+
"rank",
|
|
253
|
+
"lemma",
|
|
254
|
+
"pos",
|
|
255
|
+
"stress",
|
|
256
|
+
"stress_variants",
|
|
257
|
+
"stress_status",
|
|
258
|
+
"note",
|
|
259
|
+
"source"
|
|
260
|
+
],
|
|
261
|
+
"source": {
|
|
262
|
+
"path": "public/downloads/ukrainian-frequency/stress/v1.0.2/stress.csv",
|
|
263
|
+
"sha256": "5df3863cad89877048c6ddf937ba5d939c2f21fc574df0028065c174e2c2b92e",
|
|
264
|
+
"rows": 12000
|
|
265
|
+
},
|
|
266
|
+
"sha256": "407c98fddf9d76b6b007c000145b555a755392b598da1a711b6dafc0b60ea70c",
|
|
267
|
+
"version": "v1.0.2",
|
|
268
|
+
"snapshot": "2026-10-01",
|
|
269
|
+
"license": "CC-BY-SA-4.0",
|
|
270
|
+
"title": "Word stress for every lemma of the top-12,000 list",
|
|
271
|
+
"description": "Per rank, the primary stressed form (combining acute U+0301), the pipe-separated accepted variants with the primary first, the resolution status (one confident stress, a labeled free variant, homograph, dictionary-only doublet or variable name, a monosyllable, or missing), a note (homograph glosses, a compound's secondary stress, the standard spelling of a nonstandard lemma) and the provenance of the chosen stress. Keyed on the v0.2.0 ranks; the lemma and pos columns are the list's own.",
|
|
272
|
+
"attribution": "ukrainian-word-stress 2.1.0 (lang-uk: Oleksiy Syvokon, maintained by Dmytro Chaplynskyi), MIT, https://github.com/lang-uk/ukrainian-word-stress; en.wiktionary via kaikki.org and uk.wiktionary (the dump), CC-BY-SA 3.0; uk.wikipedia lead sentences for 116 names, CC-BY-SA 4.0; goroh.pp.ua consulted for 19 tiebreaks (attested stress only, no content copied); ranks from UberText 2.0 (lang-uk), CC-BY 4.0.",
|
|
273
|
+
"citation": "Lawrence, J. (2026). Ukrainian Frequency: a per-lemma word-stress column for the top-12,000 list (v1.0.2). uavocab.org/research. Over ukrainian-word-stress (lang-uk), en.wiktionary and uk.wiktionary.",
|
|
274
|
+
"url": "https://uavocab.org/research#stress",
|
|
275
|
+
"conceptDoi": "10.5281/zenodo.23092388"
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: uavocab
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: The open Ukrainian vocabulary data behind uavocab.org, offline: the top-12,000 lemma frequency list, the validated PULS CEFR overlay, the Borodin A1 and A2 lexical minimums, their crosswalk, the B1 enrichment tables, and a per-lemma word-stress column.
|
|
5
|
+
Author: Jake Lawrence
|
|
6
|
+
License-Expression: CC-BY-NC-SA-4.0
|
|
7
|
+
Project-URL: Homepage, https://uavocab.org/research
|
|
8
|
+
Project-URL: Documentation, https://uavocab.org/research#release
|
|
9
|
+
Project-URL: Data, https://uavocab.org/data
|
|
10
|
+
Project-URL: Source, https://github.com/jake0lawrence/UA-Vocab
|
|
11
|
+
Project-URL: Issues, https://github.com/jake0lawrence/UA-Vocab/issues
|
|
12
|
+
Project-URL: Changelog, https://github.com/jake0lawrence/UA-Vocab/blob/main/python/uavocab/CHANGELOG.md
|
|
13
|
+
Project-URL: Release files, https://github.com/jake0lawrence/UA-Vocab/tree/main/downloads/ukrainian-frequency
|
|
14
|
+
Project-URL: Zenodo: frequency list, https://doi.org/10.5281/zenodo.22823134
|
|
15
|
+
Project-URL: Zenodo: word stress, https://doi.org/10.5281/zenodo.23092388
|
|
16
|
+
Keywords: ukrainian,uavocab,vocabulary,frequency list,CEFR,lexical minimum,language learning,corpus
|
|
17
|
+
Classifier: Development Status :: 3 - Alpha
|
|
18
|
+
Classifier: Intended Audience :: Education
|
|
19
|
+
Classifier: Intended Audience :: Science/Research
|
|
20
|
+
Classifier: Natural Language :: Ukrainian
|
|
21
|
+
Classifier: Programming Language :: Python :: 3
|
|
22
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
23
|
+
Classifier: Topic :: Education
|
|
24
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSES.md
|
|
28
|
+
Requires-Dist: pyarrow>=14
|
|
29
|
+
Provides-Extra: test
|
|
30
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# uavocab
|
|
34
|
+
|
|
35
|
+
The open Ukrainian vocabulary data behind [uavocab.org](https://uavocab.org),
|
|
36
|
+
installable and offline: the top-12,000 lemma frequency list over UberText 2.0,
|
|
37
|
+
the validated PULS CEFR overlay (A1 to C1), the Borodin A1 and A2 lexical
|
|
38
|
+
minimums, the crosswalk between the minimums and the profile, the B1
|
|
39
|
+
enrichment tables (aspect relations, case government, collocations), and a
|
|
40
|
+
per-lemma word-stress column. The study the data comes from, with its methods,
|
|
41
|
+
findings and tools, is [uavocab.org/research](https://uavocab.org/research).
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
pip install uavocab
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
import uavocab
|
|
49
|
+
uavocab.lookup("п'ять")["level"] # 'A1', with rank, source and enrichment beside it
|
|
50
|
+
uavocab.lookup("орган")["stress"] # {'stress': 'о́рган', 'variants': ['о́рган', 'орга́н'], 'status': 'homograph', ...}
|
|
51
|
+
uavocab.coverage(["в", "школа", "йти"], "A2")["coverage"] # share of tokens a validated-A2 learner knows
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
This package was published as `ukvocab` up to 0.4.1. It was renamed to match
|
|
55
|
+
the site it belongs to; `pip install ukvocab` still works and now installs
|
|
56
|
+
this package, and `import ukvocab` keeps working with a warning that names
|
|
57
|
+
the new import.
|
|
58
|
+
|
|
59
|
+
## Where everything is
|
|
60
|
+
|
|
61
|
+
| | |
|
|
62
|
+
|---|---|
|
|
63
|
+
| The study (methods, findings, tools) | https://uavocab.org/research |
|
|
64
|
+
| Every dataset, its license and how to cite it | https://uavocab.org/data |
|
|
65
|
+
| The release files (CSV, JSON, Anki, with SHA-256 sums) | https://uavocab.org/research#release |
|
|
66
|
+
| Every release file, the datasheets, and this package's source (fork it) | https://github.com/jake0lawrence/UA-Vocab |
|
|
67
|
+
| Archived copies with DOIs (Zenodo) | frequency list [10.5281/zenodo.22823134](https://doi.org/10.5281/zenodo.22823134), word stress [10.5281/zenodo.23092388](https://doi.org/10.5281/zenodo.23092388) |
|
|
68
|
+
| The lemmatizer matched to this list | https://uavocab.org/research/lemmatizer |
|
|
69
|
+
| PyPI | https://pypi.org/project/uavocab/ |
|
|
70
|
+
|
|
71
|
+
## What you get
|
|
72
|
+
|
|
73
|
+
| function | returns |
|
|
74
|
+
|---|---|
|
|
75
|
+
| `lookup(lemma, pos=None)` | one lemma: `level`, `level_source`, `rank`, the frequency figures, the `stress` (primary stressed form, accepted variants, resolution status and note), every other reading of the spelling, minimum membership, the crosswalk class, and the B1 `enrichment` (aspect partners, government, collocations); `None` when no table knows the string |
|
|
76
|
+
| `levels(lemmas)` | batch: input string to `{level, level_source, rank, pos, source}` |
|
|
77
|
+
| `coverage(lemmas, known_set)` | token coverage of a lemmatized text against a known set: a level (`"A2"` means every validated lemma at or below A2), `"borodin-a1"` / `"borodin-a2"`, an int N (top N of the list), or your own set of lemmas; reports the unknowns by frequency |
|
|
78
|
+
| `sources()` | version, snapshot date, license, attribution, citation and SHA-256 per dataset, the combined license and why, and what is deliberately not shipped |
|
|
79
|
+
|
|
80
|
+
Every return value carries the dataset, version and snapshot it came from.
|
|
81
|
+
Data is Parquet, loaded lazily on first use, never on import. The only
|
|
82
|
+
dependency is `pyarrow`.
|
|
83
|
+
|
|
84
|
+
Two things the package does not do, on purpose. It does not lemmatize: pass
|
|
85
|
+
lemmas in the UberText / VESUM convention (infinitive for verbs, nominative
|
|
86
|
+
singular for nouns); the site's `POST /api/lemmatize` is the hosted lemmatizer
|
|
87
|
+
matched to this list ([its contract](https://uavocab.org/research/lemmatizer)).
|
|
88
|
+
And it ships no predicted CEFR levels: only the profile's
|
|
89
|
+
validated levels are here, and a lemma the profile does not cover gets
|
|
90
|
+
`level: None` rather than a guess.
|
|
91
|
+
|
|
92
|
+
## Provenance and honesty
|
|
93
|
+
|
|
94
|
+
The data tier is generated from the site's published release files by
|
|
95
|
+
`scripts/build-package.py` in [the open repository](https://github.com/jake0lawrence/UA-Vocab),
|
|
96
|
+
which holds every release file too, and is never edited by hand; the tests assert that every table's row count and hash
|
|
97
|
+
match the published file exactly. `sources()` is the manifest that build
|
|
98
|
+
wrote. The PULS overlay is a dated snapshot (2026-07-23) of a profile that is
|
|
99
|
+
still being extended, redistributed with its authors' permission. The join
|
|
100
|
+
onto the frequency list is part-of-speech aware; ten declared cross-tag rules
|
|
101
|
+
recover the words the two tag conventions merely label differently, and each
|
|
102
|
+
row's `join` field says how it matched, so the strict join is one filter away.
|
|
103
|
+
|
|
104
|
+
## License
|
|
105
|
+
|
|
106
|
+
The combined package is **CC-BY-NC-SA 4.0**, because the PULS levels are
|
|
107
|
+
NonCommercial and share-alike at the authors' decision (2026-09-24) and they run
|
|
108
|
+
through most tables. The frequency list and the two Borodin minimums are CC-BY
|
|
109
|
+
4.0 on their own, and the government, collocation and stress tables CC-BY-SA
|
|
110
|
+
4.0; drop the PULS level columns and NonCommercial does not apply. `LICENSES.md` has the
|
|
111
|
+
per-dataset terms and every attribution.
|
|
112
|
+
|
|
113
|
+
## Citation
|
|
114
|
+
|
|
115
|
+
```
|
|
116
|
+
Lawrence, J. (2026). uavocab: the Ukrainian Frequency datasets (top-12,000 lemma
|
|
117
|
+
list, validated PULS CEFR overlay, Borodin A1/A2 minimums, crosswalk, B1
|
|
118
|
+
enrichment tables, word stress), version 0.5.0. https://uavocab.org/research
|
|
119
|
+
|
|
120
|
+
Upstream: Chaplynskyi, D. (2023). Introducing UberText 2.0: A Corpus of Modern
|
|
121
|
+
Ukrainian at Scale. Proc. 2nd UNLP, ACL. | Synchak, O., Starko, V., Burak, M.,
|
|
122
|
+
Svystun, M. (2025). Corpus-Based Vocabulary Profiling for Ukrainian: From
|
|
123
|
+
Lexical Analysis to CEFR. eLex 2025; PULS, puls.peremova.org. | Borodin, K. and
|
|
124
|
+
Turkevych, O. 1000 i 1 slovo, A1. https://doi.org/10.18452/28236 | Borodin, K.
|
|
125
|
+
and Lazarenko, O. (2026). 1000 i 1 slovo, A2. https://doi.org/10.11584/opus4-1448
|
|
126
|
+
| Syvokon, O. and Chaplynskyi, D. (lang-uk). ukrainian-word-stress, MIT.
|
|
127
|
+
https://github.com/lang-uk/ukrainian-word-stress
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Archived, DOI-minted copies of each dataset are listed in `sources()` under
|
|
131
|
+
`conceptDoi` once they exist (today the frequency list and the stress column;
|
|
132
|
+
the PULS-derived datasets wait on the profile's published license notice).
|
|
133
|
+
Cite the concept DOI for the dataset you used.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
uavocab/__init__.py,sha256=08GmuPSDK96xIDMV0g2WvEM3Cdiye9wly3LLocPvXR0,9926
|
|
2
|
+
uavocab/_store.py,sha256=MsXeyAqHmoEPUMGv8SGIi0QqffD-P93I4JfEtflTdwY,6326
|
|
3
|
+
uavocab/data/b1_aspect.parquet,sha256=fsovn-eq5ETE4Ir8oWOx41Ts8Ud8FvLVldKVpYoFkXE,21883
|
|
4
|
+
uavocab/data/b1_collocations.parquet,sha256=GGVrGGMUALoScws5pmAERntI4JfJ-LGKzvp7bvFtsbA,63207
|
|
5
|
+
uavocab/data/b1_government.parquet,sha256=ulSEJXswjXWX782exEnm677OBUIIBw9ykIHh4zGrgWs,21028
|
|
6
|
+
uavocab/data/borodin_a1.parquet,sha256=kGlFC6fb-zLclXwR9aCdSoPtEetgqGc-95UeOqoseGI,19610
|
|
7
|
+
uavocab/data/borodin_a2.parquet,sha256=bGxVzb8v4jwpfMre40WXwC7zDqAgf0szMSEbJFC-caw,21960
|
|
8
|
+
uavocab/data/crosswalk.parquet,sha256=H9F3DJ1GEhhkCpcBL3hUKn1mtSUW2w0RGjhNXULgov8,136731
|
|
9
|
+
uavocab/data/frequency.parquet,sha256=0uUfEEG6VUv761YrJseLZInqufEYtJ6ovSLDQG9SHTk,298977
|
|
10
|
+
uavocab/data/manifest.json,sha256=SBmkkGns4QXQrJTTlYX-YH6biotYZD5KsfeVPk1boXY,16134
|
|
11
|
+
uavocab/data/stress.parquet,sha256=QHyY_d-ddrawB8AAFFtVWnVTkrWY2hpxG22vwLYOpww,277712
|
|
12
|
+
uavocab/data/validated_cefr.parquet,sha256=QMG51NcXwfeHUSY1HQBsKozThmm8nsbic8lK08ni06I,75616
|
|
13
|
+
uavocab-0.5.0.dist-info/licenses/LICENSES.md,sha256=fYqgdY_rvSR9wKh_CKItUqJknqDngHHZYDe1obxSZOk,4536
|
|
14
|
+
uavocab-0.5.0.dist-info/METADATA,sha256=Ujj_CyMxghlqR9wlbXSPnzHKrtH0WbrIdsj13qQpTIE,7623
|
|
15
|
+
uavocab-0.5.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
16
|
+
uavocab-0.5.0.dist-info/top_level.txt,sha256=gz_3mGM_XN6ih6dmQairuZTHevwhMDntr6o8E3fWEDQ,8
|
|
17
|
+
uavocab-0.5.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Licenses
|
|
2
|
+
|
|
3
|
+
## The combined work: CC-BY-NC-SA 4.0
|
|
4
|
+
|
|
5
|
+
This package as a whole is distributed under the Creative Commons
|
|
6
|
+
Attribution-NonCommercial-ShareAlike 4.0 International license
|
|
7
|
+
(https://creativecommons.org/licenses/by-nc-sa/4.0/).
|
|
8
|
+
|
|
9
|
+
Why NonCommercial and share-alike: the per-lemma CEFR levels come from the
|
|
10
|
+
Ukrainian Vocabulary Profile (PULS), which its authors license CC BY-NC-SA 4.0
|
|
11
|
+
(decided 2026-09-24) and which is redistributed here with their permission.
|
|
12
|
+
Those levels are in the validated overlay, in every row of the crosswalk, in
|
|
13
|
+
the `puls_level` column of the two Borodin tables, and in the `related_level`
|
|
14
|
+
column of the aspect table. NonCommercial and share-alike propagate to a
|
|
15
|
+
combined work, so the package carries both. They are the one NonCommercial
|
|
16
|
+
component; nothing else NonCommercial is shipped. The frequency list and the
|
|
17
|
+
two Borodin minimums are CC-BY 4.0 on their own and remain so, and the
|
|
18
|
+
government and collocation tables and the stress column are CC-BY-SA 4.0; each
|
|
19
|
+
dataset's own license is recorded in `uavocab.sources()` and below, so a downstream user who drops
|
|
20
|
+
the PULS level columns owes attribution (and, for the corpus tables,
|
|
21
|
+
share-alike) and nothing more.
|
|
22
|
+
|
|
23
|
+
Cite the profile as its authors ask: Профіль української лексики (ПУЛЬС) / О. Синчак, В. Старко, М. Бурак, М. Свистун та ін. Львів: Школа української мови та культури УКУ, 2026.
|
|
24
|
+
|
|
25
|
+
## Per dataset
|
|
26
|
+
|
|
27
|
+
| Dataset | License | Attribution |
|
|
28
|
+
|---|---|---|
|
|
29
|
+
| `frequency` (top-12,000 lemma list) | CC-BY 4.0 | Derived aggregate counts from the UberText 2.0 lemma frequency dictionary, lang-uk. Chaplynskyi, D. (2023). Introducing UberText 2.0: A Corpus of Modern Ukrainian at Scale. Proc. 2nd UNLP, ACL. https://lang.org.ua/en/ubertext/ |
|
|
30
|
+
| `validated_cefr` (PULS CEFR overlay) | CC-BY-NC-SA 4.0 | Ukrainian Vocabulary Profile (PULS): Synchak, O., Starko, V., Burak, M., Svystun, M. https://puls.peremova.org/ (eLex 2025). Per-lemma levels redistributed with the authors' permission (V. Starko, July 2026); CC BY-NC-SA 4.0 at the authors' decision (O. Synchak, 2026-09-24), attribution required. Snapshot 2026-07-23; the profile is not final. |
|
|
31
|
+
| `borodin_a1` (A1 lexical minimum) | CC-BY 4.0 | Borodin, K. and Turkevych, O. 1000 i 1 slovo. Leksychnyi minimum z ukrainskoi movy yak inozemnoi. Riven A1. Slavic Language Education 3, LANGUAGE-LAB, Humboldt-Universitat zu Berlin. https://doi.org/10.18452/28236. Machine-readable list provided by the authors, 2026-07-27. The `puls_level` column is CC-BY-NC-SA (PULS). |
|
|
32
|
+
| `borodin_a2` (A2 lexical minimum) | CC-BY 4.0 | Borodin, K. and Lazarenko, O. (2026). 1000 i 1 slovo. Riven A2. Europa-Universitat Viadrina Frankfurt (Oder). https://doi.org/10.11584/opus4-1448. The `puls_level` column is CC-BY-NC-SA (PULS). |
|
|
33
|
+
| `crosswalk` (minimums against the profile) | CC-BY-NC-SA 4.0 | Lawrence, J. (2026), over the two Borodin volumes (CC-BY 4.0), the PULS profile (CC BY-NC-SA 4.0, by permission) and UberText 2.0 ranks (CC-BY 4.0). |
|
|
34
|
+
| `b1_aspect` (aspect relations) | CC-BY-NC-SA 4.0 | Levels: PULS, by permission, CC BY-NC-SA 4.0. Aspect partners: Wiktionary via kaikki.org, CC-BY-SA 3.0. Counts: UberText 2.0, CC-BY 4.0. |
|
|
35
|
+
| `b1_government` (case government) | CC-BY-SA 4.0 | Counts over the UberText 2.0 Wikipedia subcorpus (lang-uk, CC-BY 4.0); derived statistics CC-BY-SA 4.0. |
|
|
36
|
+
| `b1_collocations` (logDice collocations) | CC-BY-SA 4.0 | Same corpus and terms as `b1_government`. |
|
|
37
|
+
| `stress` (word stress per lemma) | CC-BY-SA 4.0 | ukrainian-word-stress 2.1.0 (lang-uk: Oleksiy Syvokon, maintained by Dmytro Chaplynskyi), MIT, https://github.com/lang-uk/ukrainian-word-stress; en.wiktionary via kaikki.org and uk.wiktionary, CC-BY-SA 3.0; uk.wikipedia lead sentences for a few names, CC-BY-SA 4.0; goroh.pp.ua consulted for 19 tiebreaks (attested stress only). Share-alike from the Wiktionary rows; its own table, joined on rank, so the list stays CC-BY. |
|
|
38
|
+
|
|
39
|
+
## Not in this package
|
|
40
|
+
|
|
41
|
+
Predicted CEFR levels (the site's gap-fill model; predictions are never merged
|
|
42
|
+
into the validated overlay), any VESUM-derived lexicon or wordform data
|
|
43
|
+
(CC BY-NC-SA), the UniMorph derivational families (CC BY-NC 4.0), the
|
|
44
|
+
Wikipedia example sentences, and the Wiktionary paradigm tables. If you need
|
|
45
|
+
any of them, they are on the site with their own terms.
|
|
46
|
+
|
|
47
|
+
## The code
|
|
48
|
+
|
|
49
|
+
The Python code in this package (`src/uavocab/*.py`) is released under the
|
|
50
|
+
same CC-BY-NC-SA 4.0 terms as the data, to keep one license on the wheel.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
uavocab
|