radixor-c 4.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radixor_c-4.1.3/LICENSE +28 -0
- radixor_c-4.1.3/PKG-INFO +123 -0
- radixor_c-4.1.3/README.md +105 -0
- radixor_c-4.1.3/pyproject.toml +51 -0
- radixor_c-4.1.3/radixor_c/__init__.py +308 -0
- radixor_c-4.1.3/radixor_c/py.typed +0 -0
- radixor_c-4.1.3/radixor_c.egg-info/PKG-INFO +123 -0
- radixor_c-4.1.3/radixor_c.egg-info/SOURCES.txt +12 -0
- radixor_c-4.1.3/radixor_c.egg-info/dependency_links.txt +1 -0
- radixor_c-4.1.3/radixor_c.egg-info/requires.txt +1 -0
- radixor_c-4.1.3/radixor_c.egg-info/top_level.txt +1 -0
- radixor_c-4.1.3/setup.cfg +4 -0
- radixor_c-4.1.3/setup.py +23 -0
- radixor_c-4.1.3/src/_radixor_c.c +1821 -0
radixor_c-4.1.3/LICENSE
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Copyright (C) 2026, Leo Galambos
|
|
2
|
+
All rights reserved.
|
|
3
|
+
|
|
4
|
+
Redistribution and use in source and binary forms, with or without
|
|
5
|
+
modification, are permitted provided that the following conditions are met:
|
|
6
|
+
|
|
7
|
+
1. Redistributions of source code must retain the above copyright notice,
|
|
8
|
+
this list of conditions and the following disclaimer.
|
|
9
|
+
|
|
10
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
11
|
+
this list of conditions and the following disclaimer in the documentation
|
|
12
|
+
and/or other materials provided with the distribution.
|
|
13
|
+
|
|
14
|
+
3. Neither the name of the copyright holder nor the names of its contributors
|
|
15
|
+
may be used to endorse or promote products derived from this software
|
|
16
|
+
without specific prior written permission.
|
|
17
|
+
|
|
18
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
19
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
20
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
21
|
+
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
22
|
+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
23
|
+
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
24
|
+
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
25
|
+
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
26
|
+
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
27
|
+
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
28
|
+
POSSIBILITY OF SUCH DAMAGE.
|
radixor_c-4.1.3/PKG-INFO
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: radixor-c
|
|
3
|
+
Version: 4.1.3
|
|
4
|
+
Summary: Radixor stemmer — C extension backend
|
|
5
|
+
License-Expression: BSD-3-Clause
|
|
6
|
+
Keywords: stemming,nlp,c-extension
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: radixor-models-standard<2.0,>=1.0
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# radixor-c — Fast Scalar Stemming for Python
|
|
20
|
+
|
|
21
|
+
**radixor-c** is the C-backed Python runtime for
|
|
22
|
+
[Radixor](https://github.com/leogalambos/Radixor), a dictionary-trained
|
|
23
|
+
transformation stemmer supporting 20 languages. Its native engine is implemented
|
|
24
|
+
directly against the CPython C API to keep the overhead of stemming individual
|
|
25
|
+
words low.
|
|
26
|
+
|
|
27
|
+
It uses the same precompiled models and stemming semantics as the Java flagship
|
|
28
|
+
and the Python (PyO3) package, distributed as `radixor`. The distinction is scope: radixor-c focuses
|
|
29
|
+
on basic runtime stemming from prepared models. Text dictionary compilation,
|
|
30
|
+
trie modification and the broader model toolchain belong to Java and, as it
|
|
31
|
+
converges toward Java, the `radixor` Python package.
|
|
32
|
+
|
|
33
|
+
## Which Radixor package should I use?
|
|
34
|
+
|
|
35
|
+
| Package | Choose it for | Model operations |
|
|
36
|
+
|---|---|---|
|
|
37
|
+
| **Python-C** (`radixor-c`) | Fast calls for individual Python words; simple deployment | Load standard or [compiled Radixor models](https://leogalambos.github.io/Radixor/data-formats/) |
|
|
38
|
+
| **Python (PyO3)** (`radixor`) | High-throughput batch processing and Python-side compilation | Load compiled models and compile textual dictionaries; broader capabilities are added here first |
|
|
39
|
+
| **Java Radixor** | The complete, flagship API and model development | Full construction, reduction, extension and persistence |
|
|
40
|
+
|
|
41
|
+
Radixor-c is not a lower-quality stemmer. Given the same compiled trie it
|
|
42
|
+
produces the same results; it currently exposes fewer ways to create or modify
|
|
43
|
+
that trie.
|
|
44
|
+
|
|
45
|
+
## Installation
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
python -m pip install --only-binary=:all: radixor-c
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Published wheels support CPython 3.10–3.14 on Linux, macOS and Windows. The
|
|
52
|
+
installation also resolves `radixor-models-standard`, the shared package of 20
|
|
53
|
+
precompiled standard models.
|
|
54
|
+
|
|
55
|
+
## Quick start
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from radixor_c import Stemmer
|
|
59
|
+
|
|
60
|
+
stemmer = Stemmer("en")
|
|
61
|
+
|
|
62
|
+
stemmer.stem("running") # "run"
|
|
63
|
+
stemmer.stem("unknown_word") # None
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Construct the stemmer once and reuse it. Scalar calls are the primary reason to
|
|
67
|
+
select radixor-c, but batch and PyStemmer-compatible methods are also available:
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
words = ["running", "studies", "cars"]
|
|
71
|
+
|
|
72
|
+
stemmer.stem_batch(words)
|
|
73
|
+
stemmer.stemWord("running")
|
|
74
|
+
stemmer.stemWords(words)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
`stem()` and `stem_batch()` return `None` where no patch applies.
|
|
78
|
+
`stemWord()` and `stemWords()` instead return unmatched input unchanged.
|
|
79
|
+
|
|
80
|
+
## Custom models
|
|
81
|
+
|
|
82
|
+
Radixor-c loads [compiled Radixor models](https://leogalambos.github.io/Radixor/data-formats/):
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
custom = Stemmer(compiled="models/domain-english.rxc")
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
It deliberately does not compile a text dictionary. Prepare the interoperable
|
|
89
|
+
`.rxc` file with Java or the `radixor` package:
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
import radixor
|
|
93
|
+
|
|
94
|
+
radixor.compile("domain.tsv.gz", "models/domain-english.rxc", language="en")
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
The resulting model can be loaded by all three implementations.
|
|
98
|
+
|
|
99
|
+
## API at a glance
|
|
100
|
+
|
|
101
|
+
| API | Purpose |
|
|
102
|
+
|---|---|
|
|
103
|
+
| `Stemmer(language)` | Load a bundled model by alias or model ID |
|
|
104
|
+
| `Stemmer(compiled=path)` | Load an application-owned compiled Radixor model |
|
|
105
|
+
| `stem(word)` | Return the dominant stem or `None` |
|
|
106
|
+
| `stem_batch(words)` | Stem a list with positional `None` results |
|
|
107
|
+
| `stemWord(word)` / `stemWords(words)` | PyStemmer-compatible unmatched-word fallback |
|
|
108
|
+
| `stem_all(word)` / `stem_all_batch(words)` | Return ranked alternative stems |
|
|
109
|
+
| `algorithms()` / `version()` | Compatibility and package information |
|
|
110
|
+
|
|
111
|
+
The default bounded cache holds 10,000 results. Use `cache_size=0` to disable
|
|
112
|
+
it, or configure the PyStemmer-compatible `maxCacheSize` property.
|
|
113
|
+
|
|
114
|
+
## Documentation
|
|
115
|
+
|
|
116
|
+
- [Choose a Radixor runtime](https://leogalambos.github.io/Radixor/getting-started/)
|
|
117
|
+
- [Python-C quick start](https://leogalambos.github.io/Radixor/python-c/quick-start/)
|
|
118
|
+
- [Python-C usage and API](https://leogalambos.github.io/Radixor/python-c/usage/)
|
|
119
|
+
- [Shared Python benchmark](https://leogalambos.github.io/Radixor/python/performance/)
|
|
120
|
+
- [Shared data formats](https://leogalambos.github.io/Radixor/data-formats/)
|
|
121
|
+
|
|
122
|
+
Radixor software is available under the BSD 3-Clause License. Model data keeps
|
|
123
|
+
its separately documented provenance and licensing.
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# radixor-c — Fast Scalar Stemming for Python
|
|
2
|
+
|
|
3
|
+
**radixor-c** is the C-backed Python runtime for
|
|
4
|
+
[Radixor](https://github.com/leogalambos/Radixor), a dictionary-trained
|
|
5
|
+
transformation stemmer supporting 20 languages. Its native engine is implemented
|
|
6
|
+
directly against the CPython C API to keep the overhead of stemming individual
|
|
7
|
+
words low.
|
|
8
|
+
|
|
9
|
+
It uses the same precompiled models and stemming semantics as the Java flagship
|
|
10
|
+
and the Python (PyO3) package, distributed as `radixor`. The distinction is scope: radixor-c focuses
|
|
11
|
+
on basic runtime stemming from prepared models. Text dictionary compilation,
|
|
12
|
+
trie modification and the broader model toolchain belong to Java and, as it
|
|
13
|
+
converges toward Java, the `radixor` Python package.
|
|
14
|
+
|
|
15
|
+
## Which Radixor package should I use?
|
|
16
|
+
|
|
17
|
+
| Package | Choose it for | Model operations |
|
|
18
|
+
|---|---|---|
|
|
19
|
+
| **Python-C** (`radixor-c`) | Fast calls for individual Python words; simple deployment | Load standard or [compiled Radixor models](https://leogalambos.github.io/Radixor/data-formats/) |
|
|
20
|
+
| **Python (PyO3)** (`radixor`) | High-throughput batch processing and Python-side compilation | Load compiled models and compile textual dictionaries; broader capabilities are added here first |
|
|
21
|
+
| **Java Radixor** | The complete, flagship API and model development | Full construction, reduction, extension and persistence |
|
|
22
|
+
|
|
23
|
+
Radixor-c is not a lower-quality stemmer. Given the same compiled trie it
|
|
24
|
+
produces the same results; it currently exposes fewer ways to create or modify
|
|
25
|
+
that trie.
|
|
26
|
+
|
|
27
|
+
## Installation
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
python -m pip install --only-binary=:all: radixor-c
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Published wheels support CPython 3.10–3.14 on Linux, macOS and Windows. The
|
|
34
|
+
installation also resolves `radixor-models-standard`, the shared package of 20
|
|
35
|
+
precompiled standard models.
|
|
36
|
+
|
|
37
|
+
## Quick start
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from radixor_c import Stemmer
|
|
41
|
+
|
|
42
|
+
stemmer = Stemmer("en")
|
|
43
|
+
|
|
44
|
+
stemmer.stem("running") # "run"
|
|
45
|
+
stemmer.stem("unknown_word") # None
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Construct the stemmer once and reuse it. Scalar calls are the primary reason to
|
|
49
|
+
select radixor-c, but batch and PyStemmer-compatible methods are also available:
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
words = ["running", "studies", "cars"]
|
|
53
|
+
|
|
54
|
+
stemmer.stem_batch(words)
|
|
55
|
+
stemmer.stemWord("running")
|
|
56
|
+
stemmer.stemWords(words)
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
`stem()` and `stem_batch()` return `None` where no patch applies.
|
|
60
|
+
`stemWord()` and `stemWords()` instead return unmatched input unchanged.
|
|
61
|
+
|
|
62
|
+
## Custom models
|
|
63
|
+
|
|
64
|
+
Radixor-c loads [compiled Radixor models](https://leogalambos.github.io/Radixor/data-formats/):
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
custom = Stemmer(compiled="models/domain-english.rxc")
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
It deliberately does not compile a text dictionary. Prepare the interoperable
|
|
71
|
+
`.rxc` file with Java or the `radixor` package:
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
import radixor
|
|
75
|
+
|
|
76
|
+
radixor.compile("domain.tsv.gz", "models/domain-english.rxc", language="en")
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
The resulting model can be loaded by all three implementations.
|
|
80
|
+
|
|
81
|
+
## API at a glance
|
|
82
|
+
|
|
83
|
+
| API | Purpose |
|
|
84
|
+
|---|---|
|
|
85
|
+
| `Stemmer(language)` | Load a bundled model by alias or model ID |
|
|
86
|
+
| `Stemmer(compiled=path)` | Load an application-owned compiled Radixor model |
|
|
87
|
+
| `stem(word)` | Return the dominant stem or `None` |
|
|
88
|
+
| `stem_batch(words)` | Stem a list with positional `None` results |
|
|
89
|
+
| `stemWord(word)` / `stemWords(words)` | PyStemmer-compatible unmatched-word fallback |
|
|
90
|
+
| `stem_all(word)` / `stem_all_batch(words)` | Return ranked alternative stems |
|
|
91
|
+
| `algorithms()` / `version()` | Compatibility and package information |
|
|
92
|
+
|
|
93
|
+
The default bounded cache holds 10,000 results. Use `cache_size=0` to disable
|
|
94
|
+
it, or configure the PyStemmer-compatible `maxCacheSize` property.
|
|
95
|
+
|
|
96
|
+
## Documentation
|
|
97
|
+
|
|
98
|
+
- [Choose a Radixor runtime](https://leogalambos.github.io/Radixor/getting-started/)
|
|
99
|
+
- [Python-C quick start](https://leogalambos.github.io/Radixor/python-c/quick-start/)
|
|
100
|
+
- [Python-C usage and API](https://leogalambos.github.io/Radixor/python-c/usage/)
|
|
101
|
+
- [Shared Python benchmark](https://leogalambos.github.io/Radixor/python/performance/)
|
|
102
|
+
- [Shared data formats](https://leogalambos.github.io/Radixor/data-formats/)
|
|
103
|
+
|
|
104
|
+
Radixor software is available under the BSD 3-Clause License. Model data keeps
|
|
105
|
+
its separately documented provenance and licensing.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "radixor-c"
|
|
7
|
+
version = "4.1.3"
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
description = "Radixor stemmer — C extension backend"
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
license = "BSD-3-Clause"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
dependencies = ["radixor-models-standard>=1.0,<2.0"]
|
|
14
|
+
keywords = ["stemming", "nlp", "c-extension"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3.10",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Programming Language :: Python :: 3.14",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[tool.setuptools.packages.find]
|
|
25
|
+
where = ["."]
|
|
26
|
+
include = ["radixor_c*"]
|
|
27
|
+
|
|
28
|
+
[tool.setuptools.package-data]
|
|
29
|
+
radixor_c = ["py.typed"]
|
|
30
|
+
|
|
31
|
+
[tool.cibuildwheel]
|
|
32
|
+
build = "cp310-* cp311-* cp312-* cp313-* cp314-*"
|
|
33
|
+
test-command = 'python -c "import radixor_c, radixor_c._radixor_c"'
|
|
34
|
+
# musllinux is fine — we no longer use towlower (locale-dependent); lowercasing
|
|
35
|
+
# goes through Python's str.lower() which is locale-independent.
|
|
36
|
+
skip = "*-win32"
|
|
37
|
+
|
|
38
|
+
[tool.cibuildwheel.linux]
|
|
39
|
+
archs = ["x86_64", "aarch64"]
|
|
40
|
+
manylinux-x86_64-image = "manylinux2014"
|
|
41
|
+
manylinux-aarch64-image = "manylinux2014"
|
|
42
|
+
|
|
43
|
+
[tool.cibuildwheel.macos]
|
|
44
|
+
archs = ["universal2"]
|
|
45
|
+
|
|
46
|
+
[tool.cibuildwheel.windows]
|
|
47
|
+
archs = ["AMD64"]
|
|
48
|
+
|
|
49
|
+
[tool.ruff]
|
|
50
|
+
line-length = 88
|
|
51
|
+
target-version = "py310"
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
"""Python API for the C-backed Radixor stemmer.
|
|
2
|
+
|
|
3
|
+
Drop-in replacement for ``radixor`` with identical public interface.
|
|
4
|
+
The underlying trie engine is a pure CPython C extension instead of Rust/PyO3.
|
|
5
|
+
|
|
6
|
+
Only pre-compiled ``.rxc`` trie files are supported.
|
|
7
|
+
Use ``radixor.compile()`` to compile a TSV dictionary.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import gzip
|
|
13
|
+
import hashlib
|
|
14
|
+
import importlib.resources
|
|
15
|
+
import importlib.metadata as metadata
|
|
16
|
+
import json
|
|
17
|
+
import re
|
|
18
|
+
from contextlib import contextmanager
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, Iterable, Iterator, Optional, overload
|
|
21
|
+
|
|
22
|
+
from radixor_c._radixor_c import StemmerCore
|
|
23
|
+
|
|
24
|
+
_PYSTEMMER_MODEL_MAP: tuple[tuple[str, bool, tuple[str, ...], tuple[str, ...]], ...] = (
|
|
25
|
+
("cs-cz-default", True, ("czech", "cs", "ces", "cze"), ()),
|
|
26
|
+
("da-dk-default", True, ("danish", "da", "dan"), ()),
|
|
27
|
+
(
|
|
28
|
+
"nl-nl-default",
|
|
29
|
+
True,
|
|
30
|
+
("dutch", "nl", "dut", "nld", "kraaij_pohlmann"),
|
|
31
|
+
("dutch",),
|
|
32
|
+
),
|
|
33
|
+
("us-uk-default", True, ("english", "en", "eng"), ()),
|
|
34
|
+
("fi-fi-default", True, ("finnish", "fi", "fin"), ()),
|
|
35
|
+
("fr-fr-default", True, ("french", "fr", "fre", "fra"), ()),
|
|
36
|
+
("de-de-default", True, ("german", "de", "ger", "deu"), ()),
|
|
37
|
+
("hu-hu-default", True, ("hungarian", "hu", "hun"), ()),
|
|
38
|
+
("it-it-default", True, ("italian", "it", "ita"), ()),
|
|
39
|
+
("nb-no-default", True, ("norwegian", "no", "nor"), ("nb",)),
|
|
40
|
+
("nn-no-default", False, tuple(), ("nn",)),
|
|
41
|
+
("fa-ir-default", True, ("persian", "fa", "fas", "pers"), ()),
|
|
42
|
+
("pl-pl-unimorph", True, ("polish", "pl", "pol"), ()),
|
|
43
|
+
("pt-pt-default", True, ("portuguese", "pt", "por"), ()),
|
|
44
|
+
("ru-ru-default", True, ("russian", "ru", "rus"), ()),
|
|
45
|
+
("es-es-default", True, ("spanish", "es", "esl", "spa"), ()),
|
|
46
|
+
("sv-se-default", True, ("swedish", "sv", "swe"), ()),
|
|
47
|
+
("yi-default", True, ("yiddish", "yi", "yid"), ()),
|
|
48
|
+
("he-il-default", False, tuple(), ("he", "hebrew")),
|
|
49
|
+
("uk-ua-default", False, tuple(), ("uk", "ukrainian")),
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
_LANGUAGE_ALIASES: dict[str, str] = {mid: mid for mid, *_ in _PYSTEMMER_MODEL_MAP}
|
|
53
|
+
_SUPPORTED_PYSTEMMER_ALGORITHMS: list[str] = []
|
|
54
|
+
_SUPPORTED_PYSTEMMER_ALIASES: list[str] = []
|
|
55
|
+
for _mid, _supported, _aliases, _native in _PYSTEMMER_MODEL_MAP:
|
|
56
|
+
for _a in _aliases:
|
|
57
|
+
_LANGUAGE_ALIASES[_a] = _mid
|
|
58
|
+
for _a in _native:
|
|
59
|
+
_LANGUAGE_ALIASES[_a] = _mid
|
|
60
|
+
if _supported:
|
|
61
|
+
_SUPPORTED_PYSTEMMER_ALGORITHMS.append(_aliases[0])
|
|
62
|
+
_SUPPORTED_PYSTEMMER_ALIASES.extend(_aliases)
|
|
63
|
+
|
|
64
|
+
_SUPPORTED_PYSTEMMER_ALIASES = list(dict.fromkeys(_SUPPORTED_PYSTEMMER_ALIASES))
|
|
65
|
+
_SUPPORTED_PYSTEMMER_MODEL_IDS = frozenset(mid for mid, *_ in _PYSTEMMER_MODEL_MAP)
|
|
66
|
+
_RIGHT_TO_LEFT_MODELS: frozenset[str] = frozenset({"fa-ir-default", "he-il-default", "yi-default"})
|
|
67
|
+
|
|
68
|
+
_STANDARD_PACKAGE = "radixor_models_standard"
|
|
69
|
+
_STANDARD_CATALOG_VERSION = "2026.1"
|
|
70
|
+
_STANDARD_DISTRIBUTION_VERSION = re.compile(
|
|
71
|
+
r"(?:0\.0\.0|1\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))\Z"
|
|
72
|
+
)
|
|
73
|
+
_MODEL_ID = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*\Z")
|
|
74
|
+
_SHA256 = re.compile(r"[0-9a-f]{64}\Z")
|
|
75
|
+
_V7_MAGIC = b"EGTR"
|
|
76
|
+
_V7_VERSION = 7
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def algorithms(aliases: bool = False) -> list[str]:
|
|
80
|
+
if aliases:
|
|
81
|
+
return list(_SUPPORTED_PYSTEMMER_ALIASES)
|
|
82
|
+
return list(_SUPPORTED_PYSTEMMER_ALGORITHMS)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def version() -> str:
|
|
86
|
+
try:
|
|
87
|
+
return metadata.version("radixor-c")
|
|
88
|
+
except metadata.PackageNotFoundError:
|
|
89
|
+
return "0.0.0"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _load_standard_manifest() -> dict[str, Any]:
|
|
93
|
+
try:
|
|
94
|
+
ref = importlib.resources.files(_STANDARD_PACKAGE).joinpath("manifest.json")
|
|
95
|
+
except (ModuleNotFoundError, TypeError) as exc:
|
|
96
|
+
raise ModuleNotFoundError(
|
|
97
|
+
"The standard Radixor model package is not installed. "
|
|
98
|
+
"Install with 'pip install radixor-models-standard>=1.0,<2.0'."
|
|
99
|
+
) from exc
|
|
100
|
+
try:
|
|
101
|
+
manifest = json.loads(ref.read_text(encoding="utf-8"))
|
|
102
|
+
except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
103
|
+
raise RuntimeError("radixor-models-standard manifest is missing or corrupt.") from exc
|
|
104
|
+
|
|
105
|
+
try:
|
|
106
|
+
models = manifest["models"]
|
|
107
|
+
format_info = manifest["format"]
|
|
108
|
+
if manifest["schema_version"] != 1:
|
|
109
|
+
raise ValueError("unsupported schema_version")
|
|
110
|
+
if manifest["catalog_version"] != _STANDARD_CATALOG_VERSION:
|
|
111
|
+
raise ValueError(f"catalog version mismatch")
|
|
112
|
+
dv = manifest["distribution_version"]
|
|
113
|
+
if not isinstance(dv, str) or _STANDARD_DISTRIBUTION_VERSION.fullmatch(dv) is None:
|
|
114
|
+
raise ValueError("incompatible distribution_version")
|
|
115
|
+
if format_info != {"compression": "gzip", "magic": "EGTR", "version": 7}:
|
|
116
|
+
raise ValueError("unsupported compiled model format")
|
|
117
|
+
if not isinstance(models, list) or not models:
|
|
118
|
+
raise ValueError("models must be a non-empty list")
|
|
119
|
+
seen: set[str] = set()
|
|
120
|
+
for model in models:
|
|
121
|
+
mid = model["id"]
|
|
122
|
+
if (
|
|
123
|
+
not isinstance(mid, str)
|
|
124
|
+
or _MODEL_ID.fullmatch(mid) is None
|
|
125
|
+
or mid in seen
|
|
126
|
+
or model["file"] != f"models/{mid}.rxc"
|
|
127
|
+
or not isinstance(model["version"], str)
|
|
128
|
+
or _SHA256.fullmatch(model["sha256"]) is None
|
|
129
|
+
):
|
|
130
|
+
raise ValueError("invalid model entry")
|
|
131
|
+
seen.add(mid)
|
|
132
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
133
|
+
raise RuntimeError(f"radixor-models-standard manifest incompatible: {exc}.") from exc
|
|
134
|
+
return manifest
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _manifest_model(model_id: str) -> dict[str, Any]:
|
|
138
|
+
if not isinstance(model_id, str) or _MODEL_ID.fullmatch(model_id) is None:
|
|
139
|
+
raise ValueError(f"Invalid model ID {model_id!r}.")
|
|
140
|
+
manifest = _load_standard_manifest()
|
|
141
|
+
for model in manifest["models"]:
|
|
142
|
+
if model["id"] == model_id:
|
|
143
|
+
return model
|
|
144
|
+
raise FileNotFoundError(f"Model '{model_id}' not in standard catalog.")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _validate_standard_model(path: Path, model: dict[str, Any]) -> None:
|
|
148
|
+
try:
|
|
149
|
+
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
150
|
+
except OSError as exc:
|
|
151
|
+
raise RuntimeError(f"Cannot read standard model '{model['id']}'.") from exc
|
|
152
|
+
if digest != model["sha256"]:
|
|
153
|
+
raise RuntimeError(f"Standard model '{model['id']}' failed SHA-256 validation.")
|
|
154
|
+
try:
|
|
155
|
+
with gzip.open(path, "rb") as stream:
|
|
156
|
+
header = stream.read(8)
|
|
157
|
+
except (OSError, EOFError) as exc:
|
|
158
|
+
raise RuntimeError(f"Standard model '{model['id']}' is not valid gzip.") from exc
|
|
159
|
+
if header[:4] != _V7_MAGIC or len(header) != 8:
|
|
160
|
+
raise RuntimeError(f"Standard model '{model['id']}' missing EGTR marker.")
|
|
161
|
+
version_int = int.from_bytes(header[4:8], "big", signed=True)
|
|
162
|
+
if version_int != _V7_VERSION:
|
|
163
|
+
raise RuntimeError(f"Unsupported model format v{version_int}.")
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
@contextmanager
|
|
167
|
+
def _standard_model_path(model_id: str) -> Iterator[Path]:
|
|
168
|
+
model = _manifest_model(model_id)
|
|
169
|
+
ref = (
|
|
170
|
+
importlib.resources.files(_STANDARD_PACKAGE)
|
|
171
|
+
.joinpath("models")
|
|
172
|
+
.joinpath(f"{model_id}.rxc")
|
|
173
|
+
)
|
|
174
|
+
try:
|
|
175
|
+
with importlib.resources.as_file(ref) as path:
|
|
176
|
+
if not path.is_file():
|
|
177
|
+
raise FileNotFoundError
|
|
178
|
+
_validate_standard_model(path, model)
|
|
179
|
+
yield path
|
|
180
|
+
except FileNotFoundError as exc:
|
|
181
|
+
raise FileNotFoundError(
|
|
182
|
+
f"Standard model '{model_id}' missing; reinstall radixor-models-standard."
|
|
183
|
+
) from exc
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _is_backward(model_id: str) -> bool:
|
|
187
|
+
return model_id not in _RIGHT_TO_LEFT_MODELS
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class Stemmer:
|
|
191
|
+
"""Thread-safe Radixor stemmer backed by the C extension.
|
|
192
|
+
|
|
193
|
+
Identical interface to ``radixor.Stemmer``.
|
|
194
|
+
"""
|
|
195
|
+
|
|
196
|
+
def __init__(
|
|
197
|
+
self,
|
|
198
|
+
language: Optional[str] = None,
|
|
199
|
+
maxCacheSize: Optional[int] = None,
|
|
200
|
+
*,
|
|
201
|
+
path: Optional[str] = None,
|
|
202
|
+
compiled: Optional[str] = None,
|
|
203
|
+
backward: Optional[bool] = None,
|
|
204
|
+
store_original: bool = True,
|
|
205
|
+
lowercase: bool = True,
|
|
206
|
+
cache_size: int = 10_000,
|
|
207
|
+
) -> None:
|
|
208
|
+
source = path if path is not None else compiled
|
|
209
|
+
if maxCacheSize is not None:
|
|
210
|
+
if not isinstance(maxCacheSize, int):
|
|
211
|
+
raise TypeError("maxCacheSize must be an int")
|
|
212
|
+
if maxCacheSize < 0:
|
|
213
|
+
raise ValueError("maxCacheSize must be non-negative")
|
|
214
|
+
cache_size = maxCacheSize
|
|
215
|
+
elif cache_size < 0:
|
|
216
|
+
raise ValueError("cache_size must be non-negative")
|
|
217
|
+
|
|
218
|
+
if source is not None:
|
|
219
|
+
model_path = source
|
|
220
|
+
is_backward = True if backward is None else backward
|
|
221
|
+
model_id_val = None
|
|
222
|
+
elif language is not None:
|
|
223
|
+
if language in _LANGUAGE_ALIASES:
|
|
224
|
+
model_id_val = _LANGUAGE_ALIASES[language]
|
|
225
|
+
elif language in _SUPPORTED_PYSTEMMER_MODEL_IDS:
|
|
226
|
+
model_id_val = language
|
|
227
|
+
elif "-" in language and _MODEL_ID.fullmatch(language) is not None:
|
|
228
|
+
model_id_val = language
|
|
229
|
+
elif ".." in language or "/" in language or "\\" in language:
|
|
230
|
+
raise ValueError(f"Invalid model ID {language!r}.")
|
|
231
|
+
else:
|
|
232
|
+
raise KeyError(language)
|
|
233
|
+
is_backward = _is_backward(model_id_val) if backward is None else backward
|
|
234
|
+
model_path = None
|
|
235
|
+
else:
|
|
236
|
+
raise ValueError("Provide 'language', 'path', or 'compiled'.")
|
|
237
|
+
|
|
238
|
+
self._backward = is_backward
|
|
239
|
+
self._store_original = store_original
|
|
240
|
+
self._lowercase = lowercase
|
|
241
|
+
self._cache_size = cache_size
|
|
242
|
+
self._source_path = model_path
|
|
243
|
+
self._model_id = model_id_val
|
|
244
|
+
self._core = self._create_core(cache_size)
|
|
245
|
+
|
|
246
|
+
def _create_core(self, cache_size: int) -> StemmerCore:
|
|
247
|
+
if self._source_path is not None:
|
|
248
|
+
return StemmerCore(
|
|
249
|
+
self._source_path,
|
|
250
|
+
self._backward,
|
|
251
|
+
self._store_original,
|
|
252
|
+
self._lowercase,
|
|
253
|
+
cache_size,
|
|
254
|
+
)
|
|
255
|
+
with _standard_model_path(self._model_id or "") as model_path:
|
|
256
|
+
return StemmerCore(
|
|
257
|
+
str(model_path),
|
|
258
|
+
self._backward,
|
|
259
|
+
self._store_original,
|
|
260
|
+
self._lowercase,
|
|
261
|
+
cache_size,
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
@staticmethod
|
|
265
|
+
def version() -> str:
|
|
266
|
+
return version()
|
|
267
|
+
|
|
268
|
+
@property
|
|
269
|
+
def maxCacheSize(self) -> int:
|
|
270
|
+
return self._cache_size
|
|
271
|
+
|
|
272
|
+
@maxCacheSize.setter
|
|
273
|
+
def maxCacheSize(self, size: int) -> None:
|
|
274
|
+
if not isinstance(size, int):
|
|
275
|
+
raise TypeError("maxCacheSize must be an int")
|
|
276
|
+
if size < 0:
|
|
277
|
+
raise ValueError("maxCacheSize must be non-negative")
|
|
278
|
+
if size == self._cache_size:
|
|
279
|
+
return
|
|
280
|
+
self._cache_size = size
|
|
281
|
+
self._core = self._create_core(size)
|
|
282
|
+
|
|
283
|
+
def stem(self, word: str) -> Optional[str]:
|
|
284
|
+
return self._core.stem(word)
|
|
285
|
+
|
|
286
|
+
def stem_batch(self, words: list[str]) -> list[Optional[str]]:
|
|
287
|
+
return self._core.stem_batch(words)
|
|
288
|
+
|
|
289
|
+
@overload
|
|
290
|
+
def stemWord(self, word: str) -> str: ...
|
|
291
|
+
|
|
292
|
+
@overload
|
|
293
|
+
def stemWord(self, word: bytes) -> bytes: ...
|
|
294
|
+
|
|
295
|
+
def stemWord(self, word: str | bytes) -> str | bytes:
|
|
296
|
+
return self._core.stemWord(word)
|
|
297
|
+
|
|
298
|
+
def stemWords(self, words: Iterable[str | bytes]) -> list[str | bytes]:
|
|
299
|
+
return self._core.stemWords(words)
|
|
300
|
+
|
|
301
|
+
def stem_all(self, word: str) -> list[str]:
|
|
302
|
+
return self._core.stem_all(word)
|
|
303
|
+
|
|
304
|
+
def stem_all_batch(self, words: list[str]) -> list[list[str]]:
|
|
305
|
+
return self._core.stem_all_batch(words)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
__all__ = ["Stemmer", "StemmerCore", "algorithms", "version"]
|
|
File without changes
|