radixor 4.1.0__tar.gz

radixor-4.1.0/Cargo.lock

@@ -0,0 +1,254 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "crc32fast"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "flate2"
+version = "1.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
+
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "portable-atomic"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "radixor"
+version = "4.1.0"
+dependencies = [
+ "flate2",
+ "pyo3",
+ "unicode-general-category",
+ "unicode-normalization",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
+
+[[package]]
+name = "simd-adler32"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.12.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+
+[[package]]
+name = "tinyvec"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
+[[package]]
+name = "unicode-general-category"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2281c8c1d221438e373249e065ca4989c4c36952c211ff21a0ee91c44a3869e7"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "unicode-normalization"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
+dependencies = [
+ "tinyvec",
+]
+
+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"

radixor-4.1.0/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "radixor"
+version = "4.1.0"
+edition = "2021"
+readme = "README.md"
+
+[lib]
+name = "_radixor"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "0.22", features = ["extension-module", "abi3-py39"] }
+flate2 = "1.0"
+unicode-normalization = "0.1"
+unicode-general-category = "0.6"
+
+[profile.release]
+lto = true
+codegen-units = 1
+opt-level = 3
+strip = true

radixor-4.1.0/LICENSE

@@ -0,0 +1,28 @@
+Copyright (C) 2026, Leo Galambos
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

radixor-4.1.0/PKG-INFO

@@ -0,0 +1,298 @@
+Metadata-Version: 2.4
+Name: radixor
+Version: 4.1.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: Czech
+Classifier: Natural Language :: Danish
+Classifier: Natural Language :: Dutch
+Classifier: Natural Language :: English
+Classifier: Natural Language :: Finnish
+Classifier: Natural Language :: French
+Classifier: Natural Language :: German
+Classifier: Natural Language :: Hebrew
+Classifier: Natural Language :: Hungarian
+Classifier: Natural Language :: Italian
+Classifier: Natural Language :: Norwegian
+Classifier: Natural Language :: Persian
+Classifier: Natural Language :: Polish
+Classifier: Natural Language :: Portuguese
+Classifier: Natural Language :: Russian
+Classifier: Natural Language :: Spanish
+Classifier: Natural Language :: Swedish
+Classifier: Natural Language :: Ukrainian
+Classifier: Natural Language :: Yiddish
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Rust
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Dist: radixor-models-standard>=1.0,<2.0
+License-File: LICENSE
+Summary: Radixor stemmer – fastest stemming for Python, backed by Rust
+Keywords: python,information retrieval,language processing,morphology,stemming algorithms,stemmers,nlp,rust
+License-Expression: BSD-3-Clause
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+
+# radixor — Fastest Stemming for Python
+
+**radixor** is a Python extension for the [Radixor](https://github.com/leogalambos/Radixor) stemmer library, built on a Rust core via [PyO3](https://pyo3.rs/). It provides sub-microsecond per-word stemming with a batch API that amortises the Python↔Rust bridge overhead across thousands of words at once.
+
+## Why radixor?
+
+| Library | Approach | Batch API |
+|---|---|---|
+| **radixor** | Compiled patch-command trie in Rust | ✅ `stem_batch()` |
+| PyStemmer (Snowball) | C extension (`libstemmer`) | ✅ `stemWords()` |
+| snowballstemmer | Pure-Python Snowball | ✅ (Python loop) |
+| NLTK Porter / CISTEM | Pure Python | ❌ |
+
+**Performance.** On the shared UniMorph gold-standard corpus, measuring runtime
+stemming only (construction excluded) and with a fair, cache-disabled,
+same-input methodology, radixor won all **18 / 18** direct comparisons with
+PyStemmer 3.1.0 (Snowball's C `libstemmer`) in the published 2026-08-08 run.
+At batch size 100, the geometric-mean speedup was **1.67×**. The complete
+machine metadata and current results are in the [Python performance
+documentation](../docs/python/performance.md); benchmark implementation and
+fairness notes are in [`benchmarks/`](benchmarks/README.md).
+
+## Installation
+
+From PyPI, once publication is enabled:
+
+```bash
+python -m pip install --only-binary=:all: radixor
+```
+
+The GitHub Releases-backed index is the independent alternative:
+
+```bash
+python -m pip install --only-binary=:all: \
+  --index-url https://leogalambos.github.io/Radixor/python/simple/ radixor
+```
+
+The GitHub command becomes usable after the first model and native releases
+populate that index. See the [installation guide](../docs/python/installation.md)
+for current availability and source-checkout builds.
+
+Wheels are provided for Linux, macOS, and Windows (Python 3.9+). The install
+also resolves the mandatory pure `radixor-models-standard` dependency
+with 20 precompiled standard models. Building the native source distribution
+requires Rust ≥ 1.75 and [maturin](https://www.maturin.rs/).
+
+## Quick start
+
+```python
+from radixor import Stemmer
+
+s = Stemmer("en")          # English (us-uk-default model)
+s.stem("running")          # → "run"
+s.stem("cats")             # → "cat"
+s.stem("unknown_word")     # → None
+```
+
+## Batch API — the fast path
+
+```python
+words = ["running", "cats", "stemming", "quickly"]
+
+# Amortises the Python→Rust bridge cost across all words at once
+stems = s.stem_batch(words)
+# → ["run", "cat", "stem", "quick"]
+```
+
+For large corpora (tens of thousands of words) the batch call is the recommended interface. It avoids per-call Python frame overhead and keeps the hot loop entirely inside Rust.
+
+## Migrating from PyStemmer
+
+Radixor provides PyStemmer's `stemWord` and `stemWords` method names. These
+compatibility methods also follow PyStemmer's fallback behavior: when the trie
+has no patch command, they return the original word instead of `None`.
+
+```python
+# PyStemmer: import Stemmer
+import radixor as Stemmer
+
+stemmer = Stemmer.Stemmer("english")
+stemmer.stemWord("running")                 # → "run"
+stemmer.stemWord("unknown_word")            # → "unknown_word"
+stemmer.stemWords(["running", "unknown"])  # → ["run", "unknown"]
+```
+
+The original Radixor methods remain unchanged: `stem` and `stem_batch` return
+`None` for words without a matching patch command. Radixor accepts PyStemmer's
+full language names for the languages represented by its bundled models, as
+well as its existing two-letter codes and model IDs.
+
+## Supported languages
+
+| Code | Language | Model ID |
+|---|---|---|
+| `cs` | Czech | `cs-cz-default` |
+| `da` | Danish | `da-dk-default` |
+| `de` | German | `de-de-default` |
+| `en` | English | `us-uk-default` |
+| `es` | Spanish | `es-es-default` |
+| `fa` | Persian | `fa-ir-default` |
+| `fi` | Finnish | `fi-fi-default` |
+| `fr` | French | `fr-fr-default` |
+| `he` | Hebrew | `he-il-default` |
+| `hu` | Hungarian | `hu-hu-default` |
+| `it` | Italian | `it-it-default` |
+| `nb` | Norwegian Bokmål | `nb-no-default` |
+| `nl` | Dutch | `nl-nl-default` |
+| `nn` | Norwegian Nynorsk | `nn-no-default` |
+| `pl` | Polish | `pl-pl-unimorph` |
+| `pt` | Portuguese | `pt-pt-default` |
+| `ru` | Russian | `ru-ru-default` |
+| `sv` | Swedish | `sv-se-default` |
+| `uk` | Ukrainian | `uk-ua-default` |
+| `yi` | Yiddish | `yi-default` |
+
+## API reference
+
+### `Stemmer(language=None, *, path=None, compiled=None, backward=None, store_original=True, lowercase=True, cache_size=10_000)`
+
+Create a stemmer for the given language code, model ID, custom textual
+dictionary, or previously compiled version 7 trie. Textual dictionaries are
+compiled in Rust; `compiled=` loads a prepared binary directly.
+
+```python
+s = Stemmer("de")                           # by language code
+s = Stemmer("de-de-default")               # by model ID
+s = Stemmer(path="/data/custom.gz")        # custom gzipped dictionary
+s = Stemmer(compiled="/data/custom.rxc")   # prepared v7 binary
+```
+
+`backward` selects the traversal direction; when left as `None` it is derived
+from the language (BACKWARD, except right-to-left `fa`/`he`/`yi` which use
+FORWARD). `store_original` (default `True`) maps each canonical stem to a no-op
+patch so the stem itself is recognised. `lowercase=False` skips runtime
+lowercasing for already-normalized input, and `cache_size` enables the bounded
+result cache. The default holds up to 10,000 entries, matching PyStemmer;
+`cache_size=0` disables it. One cache is shared by `stem()`, `stemWord()`,
+`stem_batch()`, and `stemWords()`; the `stem_all*()` methods are not cached.
+
+### `stem(word: str) → str | None`
+
+Return the stem, or `None` when the compiled trie finds no applicable patch
+command. This does not mean that lookup is restricted to exact training words.
+
+### `stem_batch(words: list[str]) → list[str | None]`
+
+Stem an entire list. Preferred for large inputs.
+
+### `stemWord(word: str) → str`
+
+PyStemmer-compatible scalar method. Return the original word if it cannot be
+stemmed.
+
+### `stemWords(words: list[str]) → list[str]`
+
+PyStemmer-compatible batch method. Return each unrecognized word unchanged.
+
+### `stem_all(word: str) → list[str]`
+
+Return all stems ordered by descending corpus frequency. Useful when multiple valid stems exist.
+
+### `stem_all_batch(words: list[str]) → list[list[str]]`
+
+Return all stems for each word in a batch.
+
+## Compiling a model (compile once, load instantly)
+
+Compiling the trie from a textual dictionary takes time for large languages
+(seconds). You can compile it **once** to Radixor's binary format and then load
+it near-instantly — the same workflow Java users have:
+
+```python
+import radixor
+radixor.compile("stemmer.gz", "en.rxc", language="en")   # or backward=True/False
+s = radixor.Stemmer(compiled="en.rxc")                    # instant load, no re-compile
+```
+
+The compiled file uses Radixor's **v7 trie format and is byte-compatible with
+the Java `StemmerPatchTrieBinaryIO`** (the inner stream is identical), so a file
+compiled by Java can be loaded by Python and vice versa. `Stemmer(path=...)`
+auto-detects whether it was given a compiled trie or a textual dictionary.
+
+## Using a custom model
+
+Provide your own gzipped source dictionary (tab-separated
+`stem<TAB>variant1<TAB>variant2…` per line, `#` / `//` line remarks allowed) and
+load it directly:
+
+```python
+s = Stemmer(path="my_dictionary.gz")            # BACKWARD by default
+s = Stemmer(path="my_rtl_dictionary.gz", backward=False)   # right-to-left
+```
+
+The dictionary is compiled to a patch-command trie in Rust at construction time.
+
+## Building from source
+
+```bash
+cd Radixor/
+pip install maturin build setuptools wheel pytest
+./gradlew pythonBuildStandardModels
+pip install --no-deps build/python/dist/standard/radixor_models_standard-0.0.0-py3-none-any.whl
+cd python/
+maturin develop --release    # editable install with release optimisations
+```
+
+From the repository root, Gradle builds the native wheel/sdist and pure
+standard-model wheel/sdist without installing them globally:
+
+```bash
+./gradlew pythonBuild
+```
+
+The convenience tasks `pythonBuildLinux`, `pythonBuildWindows`, and
+`pythonBuildMacos` use the host build when the requested platform matches the
+current system. Other platforms are cross-compiled with the corresponding Rust
+target and therefore require that target and its linker/SDK to be installed.
+Override a default target with, for example,
+`-PpythonWindowsTarget=x86_64-pc-windows-gnu`. Build artifacts are written below
+`build/python/dist/`.
+
+The complete batch benchmark runs Radixor for all bundled languages and every
+available comparison engine for the languages it supports, using batch sizes
+10, 20, 50, and 100:
+
+Comparison engines are auto-detected in the environment of `pythonExecutable`.
+Install `python/benchmarks/requirements-bench.txt` there to enable the complete
+comparison set.
+
+```bash
+./gradlew pythonBenchmarkAllLanguagesBatch
+```
+
+Use `pythonBenchmarkWords`, `pythonBenchmarkRepeats`, and
+`pythonBenchmarkWarmup` Gradle properties to tune the run. CSV and JSON reports
+are written below `build/reports/python-benchmarks/`.
+
+Neither runtime distribution contains textual dictionaries. The standard data
+sdist contains build-ready gzip v7 `.rxc` files, a checksummed provenance
+manifest, and per-model CC BY-SA 3.0 notices. They are generated below `build/`
+from canonical `models/*/src/modelInput/stemmer.gz` inputs and are never stored
+in Git. `./gradlew regeneratePythonStandardModels` performs this deterministic
+generation; repository topology selects the 20 defaults and excludes optional
+`pl-pl-polimorf`.
+
+`radixor` requires `radixor-models-standard>=1.0,<2.0`. The Python distribution
+version is independent of its `2026.1` Java model-catalog identity and of the
+individual model versions recorded in the manifest.
+
+## License
+
+The native/API package is BSD-3-Clause — see [LICENSE](LICENSE). Model data
+is separately licensed under CC BY-SA 3.0 in its packaged notices.
+