udlex 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. udlex-1.0.0/LICENSE +46 -0
  2. udlex-1.0.0/PKG-INFO +262 -0
  3. udlex-1.0.0/README.md +229 -0
  4. udlex-1.0.0/pyproject.toml +153 -0
  5. udlex-1.0.0/rust/Cargo.lock +708 -0
  6. udlex-1.0.0/rust/Cargo.toml +107 -0
  7. udlex-1.0.0/rust/README.md +229 -0
  8. udlex-1.0.0/rust/src/builder/compounds.rs +186 -0
  9. udlex-1.0.0/rust/src/builder/context_keys.rs +219 -0
  10. udlex-1.0.0/rust/src/builder/corpus.rs +330 -0
  11. udlex-1.0.0/rust/src/builder/corpus_policy_tests.rs +339 -0
  12. udlex-1.0.0/rust/src/builder/input.rs +161 -0
  13. udlex-1.0.0/rust/src/builder/lemma_adapter.rs +272 -0
  14. udlex-1.0.0/rust/src/builder/metadata.rs +149 -0
  15. udlex-1.0.0/rust/src/builder/normalized.rs +132 -0
  16. udlex-1.0.0/rust/src/builder/source_table.rs +483 -0
  17. udlex-1.0.0/rust/src/builder/sources.rs +515 -0
  18. udlex-1.0.0/rust/src/builder/tests.rs +2323 -0
  19. udlex-1.0.0/rust/src/builder.rs +630 -0
  20. udlex-1.0.0/rust/src/format.rs +1127 -0
  21. udlex-1.0.0/rust/src/lexicon/catalog.rs +515 -0
  22. udlex-1.0.0/rust/src/lexicon/codec.rs +59 -0
  23. udlex-1.0.0/rust/src/lexicon/generation/compiler.rs +925 -0
  24. udlex-1.0.0/rust/src/lexicon/generation/identity.rs +140 -0
  25. udlex-1.0.0/rust/src/lexicon/generation/lookup.rs +437 -0
  26. udlex-1.0.0/rust/src/lexicon/generation/source.rs +679 -0
  27. udlex-1.0.0/rust/src/lexicon/generation/writer.rs +308 -0
  28. udlex-1.0.0/rust/src/lexicon/generation.rs +16 -0
  29. udlex-1.0.0/rust/src/lexicon/layered.rs +483 -0
  30. udlex-1.0.0/rust/src/lexicon/metadata.rs +528 -0
  31. udlex-1.0.0/rust/src/lexicon/runtime.rs +1095 -0
  32. udlex-1.0.0/rust/src/lexicon/table/hybrid.rs +631 -0
  33. udlex-1.0.0/rust/src/lexicon/table/inline_codec.rs +162 -0
  34. udlex-1.0.0/rust/src/lexicon/table/map.rs +145 -0
  35. udlex-1.0.0/rust/src/lexicon/table/validation.rs +225 -0
  36. udlex-1.0.0/rust/src/lexicon/table.rs +983 -0
  37. udlex-1.0.0/rust/src/lexicon/tests.rs +1942 -0
  38. udlex-1.0.0/rust/src/lexicon.rs +226 -0
  39. udlex-1.0.0/rust/src/lib.rs +91 -0
  40. udlex-1.0.0/rust/src/normalization.rs +586 -0
  41. udlex-1.0.0/rust/src/probe/optimization_tests.rs +277 -0
  42. udlex-1.0.0/rust/src/probe/prepared.rs +149 -0
  43. udlex-1.0.0/rust/src/probe/token_case_tests.rs +167 -0
  44. udlex-1.0.0/rust/src/probe.rs +1151 -0
  45. udlex-1.0.0/rust/src/python/_udlex/catalog.rs +174 -0
  46. udlex-1.0.0/rust/src/python/_udlex/runtime/config.rs +119 -0
  47. udlex-1.0.0/rust/src/python/_udlex/runtime/tests.rs +97 -0
  48. udlex-1.0.0/rust/src/python/_udlex/runtime.rs +1112 -0
  49. udlex-1.0.0/rust/src/python/_udlex/support.rs +396 -0
  50. udlex-1.0.0/rust/src/python/_udlex/training.rs +592 -0
  51. udlex-1.0.0/rust/src/python/_udlex/user_dictionary.rs +197 -0
  52. udlex-1.0.0/rust/src/python.rs +99 -0
  53. udlex-1.0.0/rust/src/rule.rs +274 -0
  54. udlex-1.0.0/rust/src/secure_fs.rs +250 -0
  55. udlex-1.0.0/rust/src/span.rs +129 -0
  56. udlex-1.0.0/rust/src/train/building.rs +496 -0
  57. udlex-1.0.0/rust/src/train/evaluation.rs +755 -0
  58. udlex-1.0.0/rust/src/train/model.rs +102 -0
  59. udlex-1.0.0/rust/src/train/tests.rs +1018 -0
  60. udlex-1.0.0/rust/src/train/view.rs +151 -0
  61. udlex-1.0.0/rust/src/train/writing.rs +359 -0
  62. udlex-1.0.0/rust/src/train.rs +50 -0
  63. udlex-1.0.0/rust/src/user_dictionary/writer.rs +1320 -0
  64. udlex-1.0.0/rust/src/user_dictionary.rs +1618 -0
  65. udlex-1.0.0/rust/tests/data/de_tiny/meta.json +34 -0
  66. udlex-1.0.0/rust/tests/data/user_dictionary/duplicate_after_lower.tsv +2 -0
  67. udlex-1.0.0/rust/tests/data/user_dictionary/empty_field.tsv +1 -0
  68. udlex-1.0.0/rust/tests/data/user_dictionary/extra_column.tsv +1 -0
  69. udlex-1.0.0/rust/tests/data/user_dictionary/incompatible_script.tsv +1 -0
  70. udlex-1.0.0/rust/tests/data/user_dictionary/non_nfc.tsv +1 -0
  71. udlex-1.0.0/rust/tests/data/user_dictionary/unknown_mode.tsv +1 -0
  72. udlex-1.0.0/rust/tests/data/user_dictionary/valid.tsv +2 -0
  73. udlex-1.0.0/rust-toolchain.toml +4 -0
  74. udlex-1.0.0/src/udlex/__init__.py +398 -0
  75. udlex-1.0.0/src/udlex/__main__.py +5 -0
  76. udlex-1.0.0/src/udlex/_cli_commands.py +427 -0
  77. udlex-1.0.0/src/udlex/_cli_user_dictionary.py +260 -0
  78. udlex-1.0.0/src/udlex/_io.py +978 -0
  79. udlex-1.0.0/src/udlex/_provenance.py +54 -0
  80. udlex-1.0.0/src/udlex/_udlex.pyi +1081 -0
  81. udlex-1.0.0/src/udlex/bundle.py +559 -0
  82. udlex-1.0.0/src/udlex/catalog.py +258 -0
  83. udlex-1.0.0/src/udlex/cli.py +252 -0
  84. udlex-1.0.0/src/udlex/language_codes.py +126 -0
  85. udlex-1.0.0/src/udlex/py.typed +0 -0
  86. udlex-1.0.0/src/udlex/user_dictionary.py +228 -0
udlex-1.0.0/LICENSE ADDED
@@ -0,0 +1,46 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 gulldan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
22
+
23
+ ---
24
+
25
+ The dictionary build in dictionaries/ downloads third-party data that is
26
+ distributed under its own licenses and is not covered by the license above.
27
+ Every input is pinned by URL and SHA-256 in dictionaries/sources.json together
28
+ with its SPDX license; dictionaries/policy.json says which licenses a package
29
+ may take. The inputs include:
30
+
31
+ * Universal Dependencies 2.19 - per treebank, mostly CC BY-SA 4.0, some
32
+ non-commercial;
33
+ * UniMorph - per language, see the repositories at github.com/unimorph;
34
+ * Wiktionary through kaikki.org - CC BY-SA 4.0;
35
+ * national form-to-lemma dictionaries (for example SGJP and PoliMorf for
36
+ Polish, BSD-2-Clause; OpenCorpora for Russian, CC BY-SA 3.0), each under the
37
+ license recorded in sources.json.
38
+
39
+ Each dictionary package (udlex-dict-<iso>, wheel or archive) takes the
40
+ strictest license of its parts; packages named -nc are released under
41
+ CC BY-NC-SA 4.0 for non-commercial use only. Every package carries the
42
+ treebank README and LICENSE, a NOTICE with the attribution of every input and
43
+ SOURCES.json with the license of each part.
44
+
45
+ When you redistribute a package, or a bundle built from these sources, you
46
+ redistribute that data as well: keep the corresponding notices.
udlex-1.0.0/PKG-INFO ADDED
@@ -0,0 +1,262 @@
1
+ Metadata-Version: 2.4
2
+ Name: udlex
3
+ Version: 1.0.0
4
+ Classifier: Development Status :: 5 - Production/Stable
5
+ Classifier: Intended Audience :: Developers
6
+ Classifier: Intended Audience :: Science/Research
7
+ Classifier: Operating System :: MacOS
8
+ Classifier: Operating System :: POSIX :: Linux
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3 :: Only
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
14
+ Classifier: Programming Language :: Python :: 3.14
15
+ Classifier: Programming Language :: Python :: Implementation :: CPython
16
+ Classifier: Programming Language :: Rust
17
+ Classifier: Topic :: Text Processing :: Linguistic
18
+ Classifier: Typing :: Typed
19
+ License-File: LICENSE
20
+ Summary: Fast dictionary lemmatizer with a Rust core
21
+ Keywords: lemmatization,lemmatizer,lemma,morphology,nlp,universal-dependencies,multilingual,rust
22
+ Author: Maksim Kleshchenok
23
+ License-Expression: MIT
24
+ Requires-Python: >=3.11
25
+ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
26
+ Project-URL: Changelog, https://github.com/gulldan/udlex/blob/main/CHANGELOG.md
27
+ Project-URL: Dictionaries, https://github.com/gulldan/udlex/releases/tag/v1.0.0
28
+ Project-URL: Documentation, https://github.com/gulldan/udlex/tree/main/docs
29
+ Project-URL: Homepage, https://github.com/gulldan/udlex
30
+ Project-URL: Issues, https://github.com/gulldan/udlex/issues
31
+ Project-URL: Repository, https://github.com/gulldan/udlex
32
+
33
+ # udlex
34
+
35
+ **English** · [Русский](https://github.com/gulldan/udlex/blob/main/README.ru.md) · [简体中文](https://github.com/gulldan/udlex/blob/main/README.zh-CN.md)
36
+
37
+ udlex is a dictionary lemmatizer. It looks up each token in compiled tables
38
+ learned from Universal Dependencies treebanks and open dictionaries, and guesses
39
+ unknown words from suffix rules. The core is Rust; the package adds Python
40
+ bindings and a command line, and the same core builds as a C library. There is
41
+ no neural network: nothing to run on a GPU, and the answers are deterministic.
42
+
43
+ ## Install
44
+
45
+ With [uv](https://docs.astral.sh/uv/), add the engine and a dictionary to your
46
+ project:
47
+
48
+ ```sh
49
+ uv add udlex
50
+ uv add "udlex-dict-eng-nc @ https://github.com/gulldan/udlex/releases/download/v1.0.0/udlex_dict_eng_nc-1.0.0-py3-none-any.whl"
51
+ uv run udlex lemma eng mice were running
52
+ ```
53
+
54
+ Python 3.11 or newer. Wheels are built for Linux (x86_64, aarch64), macOS
55
+ (arm64, x86_64) and Windows (x86_64); on other platforms uv builds from source
56
+ and needs a Rust toolchain.
57
+
58
+ The engine is on PyPI; the dictionaries are not. Each language is a separate
59
+ wheel attached to the [v1.0.0 release](https://github.com/gulldan/udlex/releases/tag/v1.0.0),
60
+ and `uv add` with the wheel URL records it in `uv.lock` like any dependency.
61
+ `uv run udlex install --dry-run fin rus` prints the URLs for any languages.
62
+ `udlex install <iso> ...` installs them into the current environment with
63
+ `uv pip` (pip when uv is missing); inside a uv project it prints the `uv add`
64
+ command to run, because `uv sync` removes packages the lock does not list.
65
+
66
+ Wheels are named `udlex_dict_<iso>-1.0.0-py3-none-any.whl`, with `_nc` after
67
+ the code for [non-commercial](#licenses) packages such as English. They install
68
+ into the `udlex_bundle` namespace, where `udlex.load` and the command line find
69
+ them. Every package is also published as `udlex-dict-<iso>-1.0.0-bundle.tar.gz`
70
+ (`-nc` likewise), which unpacks into `bundle/<iso>`. Unpack several into the same
71
+ place; udlex reads `./bundle` when no dictionary wheel is installed, or the
72
+ directory in `UDLEX_BUNDLE`, or one you pass as `udlex.load(directory, iso)`.
73
+
74
+ ## Use
75
+
76
+ ```py
77
+ import udlex
78
+
79
+ lexicon = udlex.load("eng") # load once per process; safe to share across threads
80
+ print(lexicon.lemmatize(["The", "mice", "were", "running"]))
81
+ ```
82
+
83
+ The input is a list of tokens: udlex has no tokenizer. Languages are named by
84
+ ISO 639-3 code. From the shell:
85
+
86
+ ```sh
87
+ uv run udlex lemma eng mice were running
88
+ uv run udlex languages
89
+ ```
90
+
91
+ ### Try it without downloading anything
92
+
93
+ This builds a two-word dictionary from a `form<TAB>lemma` table and runs it.
94
+ Save it as `quickstart.py` and run `uv run quickstart.py`.
95
+
96
+ ```python
97
+ from pathlib import Path
98
+
99
+ import udlex
100
+
101
+ bundle = Path("example-bundle")
102
+ bundle.mkdir()
103
+ words = bundle / "lemmas.tsv"
104
+ words.write_text("cats\tcat\nmice\tmouse\n", encoding="utf-8")
105
+
106
+ udlex.build_lexicon(
107
+ str(bundle / "eng"),
108
+ train=None,
109
+ dev=None,
110
+ test=None,
111
+ paradigms=[("example", str(words), False, False)],
112
+ name="eng",
113
+ language="English",
114
+ iso639_3="eng",
115
+ )
116
+ lexicon = udlex.load(bundle, "eng")
117
+ print(lexicon.lemmatize(["cats", "mice", "unknown"]))
118
+ # ['cat', 'mouse', 'unknown']
119
+ ```
120
+
121
+ ```sh
122
+ uv run udlex --bundle example-bundle lemma -q eng cats mice
123
+ ```
124
+
125
+ ## Documentation
126
+
127
+ - [Quick start](https://github.com/gulldan/udlex/blob/main/docs/quickstart.md)
128
+ - [API: Python, command line, C](https://github.com/gulldan/udlex/blob/main/docs/api.md)
129
+ - [Languages and accuracy](https://github.com/gulldan/udlex/blob/main/docs/languages.md)
130
+ - [User dictionary: your own overrides](https://github.com/gulldan/udlex/blob/main/docs/user-dictionary.md)
131
+ - [Building dictionaries](https://github.com/gulldan/udlex/blob/main/docs/building-dictionaries.md)
132
+
133
+ ## Languages
134
+
135
+ The v1.0.0 release has dictionaries for 224 languages, one package per
136
+ language. The [language table](https://github.com/gulldan/udlex/blob/main/docs/languages.md)
137
+ lists each package with its license, how it was scored and its accuracy.
138
+
139
+ ## Accuracy
140
+
141
+ Exact lemma match on the gold tokens of Universal Dependencies 2.19
142
+ ([`evaluation.json`](https://github.com/gulldan/udlex/blob/main/dictionaries/catalog/evaluation.json)).
143
+ *Copy* returns every token unchanged.
144
+
145
+ | Languages | Scored on | udlex, macro % | Copy, % |
146
+ | --- | --- | ---: | ---: |
147
+ | 103 | official test split | 89.85 | 58.41 |
148
+ | 109 | two-fold estimate (no training split) | 73.87 | 57.54 |
149
+ | 12 | no gold lemmas: unmeasured | — | — |
150
+
151
+ Against other lemmatizers on identical test words (from
152
+ [`benchmarks/generated/numbers.tex`](https://github.com/gulldan/udlex/blob/main/benchmarks/generated/numbers.tex)).
153
+ Each row averages over the languages both systems cover, after dropping
154
+ languages whose model saw our test sentences, so rows are not comparable with
155
+ each other.
156
+
157
+ | System | Languages | System, % | udlex, % |
158
+ | --- | ---: | ---: | ---: |
159
+ | Stanza | 65 | 95.32 | 93.81 |
160
+ | UDPipe 2 | 77 | 94.48 | 93.47 |
161
+ | Trankit | 46 | 91.32 | 94.44 |
162
+ | UDPipe 1 | 44 | 91.36 | 95.16 |
163
+ | simplemma | 33 | 90.75 | 95.44 |
164
+ | spaCy sm | 17 | 89.55 | 95.77 |
165
+ | spaCy lg | 17 | 90.63 | 95.77 |
166
+ | spaCy trf | 7 | 93.98 | 97.14 |
167
+ | LemmaGen3 | 16 | 82.27 | 96.33 |
168
+ | CoreNLP | 1 | 96.37 | 97.87 |
169
+
170
+ Stanza and UDPipe 2 are more accurate than udlex by about 1 to 1.5 points:
171
+ they are neural models that see the whole sentence and its part-of-speech
172
+ tags. udlex is ahead of the other systems on average.
173
+
174
+ ## Speed and memory
175
+
176
+ All systems were timed on one AMD Ryzen 9 7950X (16 cores) with an RTX 4090,
177
+ on the pre-tokenized UD 2.19 test sentences of every language each system
178
+ supports. Rates are medians over languages. *udlex faster* is the geometric
179
+ mean, over the languages a system was timed on, of how many times faster the
180
+ Rust core of udlex is on one core. *Parallel* is the better of 16 processes and
181
+ 16 threads, or one GPU for neural systems. *Peak memory* is the resident set of
182
+ the timing process on one core, which also holds the test text: that system's
183
+ median, then udlex's on the same languages.
184
+
185
+ | System | Languages | Words/s, one core | udlex faster, × | Words/s, parallel | Peak memory, MB |
186
+ | --- | ---: | ---: | ---: | ---: | ---: |
187
+ | **udlex, Rust** | 169 | 54.4M | — | 759M | 147 |
188
+ | **udlex, Python** | 169 | 18.0M | — | 142M | 1,172 |
189
+ | simplemma | 37 | 6.6M | 5.4 | 57.8M | 324 / 110 |
190
+ | LemmaGen3 | 17 | 3.5M | 9.5 | 32.5M | 1,445 / 118 |
191
+ | CoreNLP | 1 | 27k | 1,965 | 336k | 1,600 / 111 |
192
+ | spaCy sm | 20 | 27k | 1,422 | 218k | 773 / 114 |
193
+ | spaCy lg | 20 | 25k | 1,568 | 165k | 1,546 / 114 |
194
+ | UDPipe 1 | 50 | 15k | 2,177 | 151k | 153 / 99 |
195
+ | Stanza | 90 | 4.5k | 8,160 | 16k (GPU) | 1,240 / 99 |
196
+ | spaCy trf | 8 | 278 | 143,271 | 20k (GPU) | 2,840 / 111 |
197
+ | Trankit | 53 | 74 | 433,457 | 3.9k (GPU) | 4,665 / 94 |
198
+
199
+ On one core udlex ranges from 6.2 million to 858 million words per second
200
+ depending on the language. simplemma, the fastest other system, is faster only
201
+ on Georgian, by 1 %. The Python binding gains 1.46 times on 16 threads and 8.9
202
+ times on 16 processes, and its memory includes the Python token lists of the
203
+ test text. UDPipe 2 was not timed. Numbers depend on the CPU and on the token
204
+ mix of the test texts. The values come from
205
+ [`numbers.tex`](https://github.com/gulldan/udlex/blob/main/benchmarks/generated/numbers.tex);
206
+ the receipts are in
207
+ [`benchmarks/evidence/speed`](https://github.com/gulldan/udlex/tree/main/benchmarks/evidence/speed).
208
+
209
+ ## Limitations
210
+
211
+ - **Tokens in, lemmas out.** There is no tokenizer, sentence splitter or
212
+ tagger. Split the text first.
213
+ - **No context.** Each token gets one lemma, whatever the sentence. A form
214
+ with several possible lemmas (English *saw*: *see* or *saw*) always gets the
215
+ same one. This is the main gap to Stanza and UDPipe 2. The `upos=` and
216
+ `context=` arguments of `lemmatize` only matter for dictionaries built with
217
+ context keys, and the 1.0.0 packages are built without them.
218
+ - **Unknown words** are guessed from suffix rules learned from the treebank.
219
+ On words the dictionary has never seen, this is weaker than a neural model.
220
+ Pass `use_rules=False` (`--lookup-only` on the command line) to return the
221
+ token unchanged instead of guessing.
222
+ - **Accuracy varies by language**, from near 100 % to under 50 %; check the
223
+ [language table](https://github.com/gulldan/udlex/blob/main/docs/languages.md)
224
+ before relying on a language. For the 109 languages without a training split
225
+ the accuracy is a two-fold estimate on small treebanks, and 12 languages have
226
+ no gold lemmas at all.
227
+ - **Partly exploratory scores.** For languages without an official dev split,
228
+ the dev stand-in was introduced after the first test evaluation.
229
+ - **No machine-labelled lemmas.** The released dictionaries contain no lemmas
230
+ labelled by a language model; each package lists its sources in
231
+ `SOURCES.json`.
232
+ - **Non-commercial treebanks.** 39 packages, among them English, Czech,
233
+ Arabic and Italian, learned from a treebank under a non-commercial license
234
+ and may only be used non-commercially.
235
+
236
+ ## Other interfaces
237
+
238
+ - **C library**: `cargo build --locked --release --manifest-path rust/Cargo.toml -p udlex-capi`
239
+ builds `libudlex_capi` with the header
240
+ [`rust/capi/include/udlex.h`](https://github.com/gulldan/udlex/blob/main/rust/capi/include/udlex.h);
241
+ see the [C API](https://github.com/gulldan/udlex/blob/main/docs/api.md#c-api).
242
+ - **Go**: [gulldan/udlex-go](https://github.com/gulldan/udlex-go).
243
+
244
+ ## Licenses
245
+
246
+ The code is [MIT](https://github.com/gulldan/udlex/blob/main/LICENSE).
247
+
248
+ Each dictionary package has its own license, which follows from its sources:
249
+ in v1.0.0, 184 packages are CC BY-SA (182 version 4.0, 2 version 3.0), 39 are
250
+ CC BY-NC-SA 4.0 and 1 is CC BY 4.0. Packages whose name ends in `-nc` are for
251
+ non-commercial use only. Inside a package every table keeps the license of its
252
+ source, and `SOURCES.json` lists each source with its URL, version, author and
253
+ license, as spaCy's `meta.json` does. If you ship a dictionary, keep its
254
+ `NOTICE` and `SOURCES.json` and follow its license. All sources and package
255
+ licenses are listed in
256
+ [DICTIONARY_LICENSES.md](https://github.com/gulldan/udlex/blob/main/DICTIONARY_LICENSES.md).
257
+
258
+ ## Citation
259
+
260
+ See [CITATION.cff](https://github.com/gulldan/udlex/blob/main/CITATION.cff).
261
+ Contributions: [CONTRIBUTING.md](https://github.com/gulldan/udlex/blob/main/CONTRIBUTING.md).
262
+
udlex-1.0.0/README.md ADDED
@@ -0,0 +1,229 @@
1
+ # udlex
2
+
3
+ **English** · [Русский](https://github.com/gulldan/udlex/blob/main/README.ru.md) · [简体中文](https://github.com/gulldan/udlex/blob/main/README.zh-CN.md)
4
+
5
+ udlex is a dictionary lemmatizer. It looks up each token in compiled tables
6
+ learned from Universal Dependencies treebanks and open dictionaries, and guesses
7
+ unknown words from suffix rules. The core is Rust; the package adds Python
8
+ bindings and a command line, and the same core builds as a C library. There is
9
+ no neural network: nothing to run on a GPU, and the answers are deterministic.
10
+
11
+ ## Install
12
+
13
+ With [uv](https://docs.astral.sh/uv/), add the engine and a dictionary to your
14
+ project:
15
+
16
+ ```sh
17
+ uv add udlex
18
+ uv add "udlex-dict-eng-nc @ https://github.com/gulldan/udlex/releases/download/v1.0.0/udlex_dict_eng_nc-1.0.0-py3-none-any.whl"
19
+ uv run udlex lemma eng mice were running
20
+ ```
21
+
22
+ Python 3.11 or newer. Wheels are built for Linux (x86_64, aarch64), macOS
23
+ (arm64, x86_64) and Windows (x86_64); on other platforms uv builds from source
24
+ and needs a Rust toolchain.
25
+
26
+ The engine is on PyPI; the dictionaries are not. Each language is a separate
27
+ wheel attached to the [v1.0.0 release](https://github.com/gulldan/udlex/releases/tag/v1.0.0),
28
+ and `uv add` with the wheel URL records it in `uv.lock` like any dependency.
29
+ `uv run udlex install --dry-run fin rus` prints the URLs for any languages.
30
+ `udlex install <iso> ...` installs them into the current environment with
31
+ `uv pip` (pip when uv is missing); inside a uv project it prints the `uv add`
32
+ command to run, because `uv sync` removes packages the lock does not list.
33
+
34
+ Wheels are named `udlex_dict_<iso>-1.0.0-py3-none-any.whl`, with `_nc` after
35
+ the code for [non-commercial](#licenses) packages such as English. They install
36
+ into the `udlex_bundle` namespace, where `udlex.load` and the command line find
37
+ them. Every package is also published as `udlex-dict-<iso>-1.0.0-bundle.tar.gz`
38
+ (`-nc` likewise), which unpacks into `bundle/<iso>`. Unpack several into the same
39
+ place; udlex reads `./bundle` when no dictionary wheel is installed, or the
40
+ directory in `UDLEX_BUNDLE`, or one you pass as `udlex.load(directory, iso)`.
41
+
42
+ ## Use
43
+
44
+ ```py
45
+ import udlex
46
+
47
+ lexicon = udlex.load("eng") # load once per process; safe to share across threads
48
+ print(lexicon.lemmatize(["The", "mice", "were", "running"]))
49
+ ```
50
+
51
+ The input is a list of tokens: udlex has no tokenizer. Languages are named by
52
+ ISO 639-3 code. From the shell:
53
+
54
+ ```sh
55
+ uv run udlex lemma eng mice were running
56
+ uv run udlex languages
57
+ ```
58
+
59
+ ### Try it without downloading anything
60
+
61
+ This builds a two-word dictionary from a `form<TAB>lemma` table and runs it.
62
+ Save it as `quickstart.py` and run `uv run quickstart.py`.
63
+
64
+ ```python
65
+ from pathlib import Path
66
+
67
+ import udlex
68
+
69
+ bundle = Path("example-bundle")
70
+ bundle.mkdir()
71
+ words = bundle / "lemmas.tsv"
72
+ words.write_text("cats\tcat\nmice\tmouse\n", encoding="utf-8")
73
+
74
+ udlex.build_lexicon(
75
+ str(bundle / "eng"),
76
+ train=None,
77
+ dev=None,
78
+ test=None,
79
+ paradigms=[("example", str(words), False, False)],
80
+ name="eng",
81
+ language="English",
82
+ iso639_3="eng",
83
+ )
84
+ lexicon = udlex.load(bundle, "eng")
85
+ print(lexicon.lemmatize(["cats", "mice", "unknown"]))
86
+ # ['cat', 'mouse', 'unknown']
87
+ ```
88
+
89
+ ```sh
90
+ uv run udlex --bundle example-bundle lemma -q eng cats mice
91
+ ```
92
+
93
+ ## Documentation
94
+
95
+ - [Quick start](https://github.com/gulldan/udlex/blob/main/docs/quickstart.md)
96
+ - [API: Python, command line, C](https://github.com/gulldan/udlex/blob/main/docs/api.md)
97
+ - [Languages and accuracy](https://github.com/gulldan/udlex/blob/main/docs/languages.md)
98
+ - [User dictionary: your own overrides](https://github.com/gulldan/udlex/blob/main/docs/user-dictionary.md)
99
+ - [Building dictionaries](https://github.com/gulldan/udlex/blob/main/docs/building-dictionaries.md)
100
+
101
+ ## Languages
102
+
103
+ The v1.0.0 release has dictionaries for 224 languages, one package per
104
+ language. The [language table](https://github.com/gulldan/udlex/blob/main/docs/languages.md)
105
+ lists each package with its license, how it was scored and its accuracy.
106
+
107
+ ## Accuracy
108
+
109
+ Exact lemma match on the gold tokens of Universal Dependencies 2.19
110
+ ([`evaluation.json`](https://github.com/gulldan/udlex/blob/main/dictionaries/catalog/evaluation.json)).
111
+ *Copy* returns every token unchanged.
112
+
113
+ | Languages | Scored on | udlex, macro % | Copy, % |
114
+ | --- | --- | ---: | ---: |
115
+ | 103 | official test split | 89.85 | 58.41 |
116
+ | 109 | two-fold estimate (no training split) | 73.87 | 57.54 |
117
+ | 12 | no gold lemmas: unmeasured | — | — |
118
+
119
+ Against other lemmatizers on identical test words (from
120
+ [`benchmarks/generated/numbers.tex`](https://github.com/gulldan/udlex/blob/main/benchmarks/generated/numbers.tex)).
121
+ Each row averages over the languages both systems cover, after dropping
122
+ languages whose model saw our test sentences, so rows are not comparable with
123
+ each other.
124
+
125
+ | System | Languages | System, % | udlex, % |
126
+ | --- | ---: | ---: | ---: |
127
+ | Stanza | 65 | 95.32 | 93.81 |
128
+ | UDPipe 2 | 77 | 94.48 | 93.47 |
129
+ | Trankit | 46 | 91.32 | 94.44 |
130
+ | UDPipe 1 | 44 | 91.36 | 95.16 |
131
+ | simplemma | 33 | 90.75 | 95.44 |
132
+ | spaCy sm | 17 | 89.55 | 95.77 |
133
+ | spaCy lg | 17 | 90.63 | 95.77 |
134
+ | spaCy trf | 7 | 93.98 | 97.14 |
135
+ | LemmaGen3 | 16 | 82.27 | 96.33 |
136
+ | CoreNLP | 1 | 96.37 | 97.87 |
137
+
138
+ Stanza and UDPipe 2 are more accurate than udlex by about 1 to 1.5 points:
139
+ they are neural models that see the whole sentence and its part-of-speech
140
+ tags. udlex is ahead of the other systems on average.
141
+
142
+ ## Speed and memory
143
+
144
+ All systems were timed on one AMD Ryzen 9 7950X (16 cores) with an RTX 4090,
145
+ on the pre-tokenized UD 2.19 test sentences of every language each system
146
+ supports. Rates are medians over languages. *udlex faster* is the geometric
147
+ mean, over the languages a system was timed on, of how many times faster the
148
+ Rust core of udlex is on one core. *Parallel* is the better of 16 processes and
149
+ 16 threads, or one GPU for neural systems. *Peak memory* is the resident set of
150
+ the timing process on one core, which also holds the test text: that system's
151
+ median, then udlex's on the same languages.
152
+
153
+ | System | Languages | Words/s, one core | udlex faster, × | Words/s, parallel | Peak memory, MB |
154
+ | --- | ---: | ---: | ---: | ---: | ---: |
155
+ | **udlex, Rust** | 169 | 54.4M | — | 759M | 147 |
156
+ | **udlex, Python** | 169 | 18.0M | — | 142M | 1,172 |
157
+ | simplemma | 37 | 6.6M | 5.4 | 57.8M | 324 / 110 |
158
+ | LemmaGen3 | 17 | 3.5M | 9.5 | 32.5M | 1,445 / 118 |
159
+ | CoreNLP | 1 | 27k | 1,965 | 336k | 1,600 / 111 |
160
+ | spaCy sm | 20 | 27k | 1,422 | 218k | 773 / 114 |
161
+ | spaCy lg | 20 | 25k | 1,568 | 165k | 1,546 / 114 |
162
+ | UDPipe 1 | 50 | 15k | 2,177 | 151k | 153 / 99 |
163
+ | Stanza | 90 | 4.5k | 8,160 | 16k (GPU) | 1,240 / 99 |
164
+ | spaCy trf | 8 | 278 | 143,271 | 20k (GPU) | 2,840 / 111 |
165
+ | Trankit | 53 | 74 | 433,457 | 3.9k (GPU) | 4,665 / 94 |
166
+
167
+ On one core udlex ranges from 6.2 million to 858 million words per second
168
+ depending on the language. simplemma, the fastest other system, is faster only
169
+ on Georgian, by 1 %. The Python binding gains 1.46 times on 16 threads and 8.9
170
+ times on 16 processes, and its memory includes the Python token lists of the
171
+ test text. UDPipe 2 was not timed. Numbers depend on the CPU and on the token
172
+ mix of the test texts. The values come from
173
+ [`numbers.tex`](https://github.com/gulldan/udlex/blob/main/benchmarks/generated/numbers.tex);
174
+ the receipts are in
175
+ [`benchmarks/evidence/speed`](https://github.com/gulldan/udlex/tree/main/benchmarks/evidence/speed).
176
+
177
+ ## Limitations
178
+
179
+ - **Tokens in, lemmas out.** There is no tokenizer, sentence splitter or
180
+ tagger. Split the text first.
181
+ - **No context.** Each token gets one lemma, whatever the sentence. A form
182
+ with several possible lemmas (English *saw*: *see* or *saw*) always gets the
183
+ same one. This is the main gap to Stanza and UDPipe 2. The `upos=` and
184
+ `context=` arguments of `lemmatize` only matter for dictionaries built with
185
+ context keys, and the 1.0.0 packages are built without them.
186
+ - **Unknown words** are guessed from suffix rules learned from the treebank.
187
+ On words the dictionary has never seen, this is weaker than a neural model.
188
+ Pass `use_rules=False` (`--lookup-only` on the command line) to return the
189
+ token unchanged instead of guessing.
190
+ - **Accuracy varies by language**, from near 100 % to under 50 %; check the
191
+ [language table](https://github.com/gulldan/udlex/blob/main/docs/languages.md)
192
+ before relying on a language. For the 109 languages without a training split
193
+ the accuracy is a two-fold estimate on small treebanks, and 12 languages have
194
+ no gold lemmas at all.
195
+ - **Partly exploratory scores.** For languages without an official dev split,
196
+ the dev stand-in was introduced after the first test evaluation.
197
+ - **No machine-labelled lemmas.** The released dictionaries contain no lemmas
198
+ labelled by a language model; each package lists its sources in
199
+ `SOURCES.json`.
200
+ - **Non-commercial treebanks.** 39 packages, among them English, Czech,
201
+ Arabic and Italian, learned from a treebank under a non-commercial license
202
+ and may only be used non-commercially.
203
+
204
+ ## Other interfaces
205
+
206
+ - **C library**: `cargo build --locked --release --manifest-path rust/Cargo.toml -p udlex-capi`
207
+ builds `libudlex_capi` with the header
208
+ [`rust/capi/include/udlex.h`](https://github.com/gulldan/udlex/blob/main/rust/capi/include/udlex.h);
209
+ see the [C API](https://github.com/gulldan/udlex/blob/main/docs/api.md#c-api).
210
+ - **Go**: [gulldan/udlex-go](https://github.com/gulldan/udlex-go).
211
+
212
+ ## Licenses
213
+
214
+ The code is [MIT](https://github.com/gulldan/udlex/blob/main/LICENSE).
215
+
216
+ Each dictionary package has its own license, which follows from its sources:
217
+ in v1.0.0, 184 packages are CC BY-SA (182 version 4.0, 2 version 3.0), 39 are
218
+ CC BY-NC-SA 4.0 and 1 is CC BY 4.0. Packages whose name ends in `-nc` are for
219
+ non-commercial use only. Inside a package every table keeps the license of its
220
+ source, and `SOURCES.json` lists each source with its URL, version, author and
221
+ license, as spaCy's `meta.json` does. If you ship a dictionary, keep its
222
+ `NOTICE` and `SOURCES.json` and follow its license. All sources and package
223
+ licenses are listed in
224
+ [DICTIONARY_LICENSES.md](https://github.com/gulldan/udlex/blob/main/DICTIONARY_LICENSES.md).
225
+
226
+ ## Citation
227
+
228
+ See [CITATION.cff](https://github.com/gulldan/udlex/blob/main/CITATION.cff).
229
+ Contributions: [CONTRIBUTING.md](https://github.com/gulldan/udlex/blob/main/CONTRIBUTING.md).