noentenc 0.1.0__tar.gz

noentenc-0.1.0/PKG-INFO

@@ -0,0 +1,199 @@
+Metadata-Version: 2.3
+Name: noentenc
+Version: 0.1.0
+Summary: Lightweight, CPU-only language detection and translation for Python. Extra detection backends are optional extras.
+Author: marti-jorda-roca
+Author-email: marti-jorda-roca <mjorda98@gmail.com>
+Requires-Dist: huggingface-hub>=1.33.0
+Requires-Dist: numpy>=2.0
+Requires-Dist: onnxruntime>=1.30.0
+Requires-Dist: tokenizers>=0.23.2
+Requires-Dist: tqdm>=4.70.1
+Requires-Dist: noentenc[pandas,polars,lingua,cld3,heliport] ; extra == 'all'
+Requires-Dist: cld3-py>=3.1 ; extra == 'cld3'
+Requires-Dist: heliport>=1.0.1 ; sys_platform != 'win32' and extra == 'heliport'
+Requires-Dist: lingua-language-detector>=2.2 ; extra == 'lingua'
+Requires-Dist: pandas>=3.0.6 ; extra == 'pandas'
+Requires-Dist: polars>=1.44.2 ; extra == 'polars'
+Requires-Python: >=3.14
+Provides-Extra: all
+Provides-Extra: cld3
+Provides-Extra: heliport
+Provides-Extra: lingua
+Provides-Extra: pandas
+Provides-Extra: polars
+Description-Content-Type: text/markdown
+
+<div align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="docs/assets/logo-dark.svg">
+    <source media="(prefers-color-scheme: light)" srcset="docs/assets/logo-light.svg">
+    <img src="docs/assets/logo-light.svg" alt="noentenc" width="520">
+  </picture>
+  <p>Language detection and machine translation for Python, on CPU, without torch.</p>
+</div>
+
+```python
+from noentenc import Language
+from noentenc.language_detection import LanguageDetector
+from noentenc.translation import Translator
+
+LanguageDetector().detect("Bon dia! Com estàs?")
+# 'cat'
+
+Translator().translate("The weather is nice today.", Language.SPANISH, Language.ENGLISH)
+# 'El tiempo es bueno hoy.'
+```
+
+- **Fast.** The default detector labels about 350,000 texts per second on a laptop CPU, and a direct Opus-MT translation takes about 60 ms per sentence. See [benchmarks](docs/benchmarks.md).
+
+  | Task | Model | Single input | Batched |
+  |---|---|---:|---:|
+  | Detection | heliport | 4 µs | 872k texts/s |
+  | Detection | **lid176 (default)** | 17 µs | 356k texts/s |
+  | Detection | cld3 | 16 µs | 66k texts/s |
+  | Detection | lingua | 0.44 ms | 9k texts/s |
+  | Detection | bert-openlid | 0.35 ms | 3.5k texts/s |
+  | Translation (en→es) | **Opus-MT (default for direct pairs)** | 63 ms | 100 sent/s |
+  | Translation (en→es) | **SMaLL-100 (default otherwise)** | 52 ms | 75 sent/s |
+  | Translation (en→es) | M2M100 418M | 231 ms | 15 sent/s |
+  | Translation (en→es) | NLLB-200 600M (int8) | 770 ms | 10 sent/s |
+
+  Apple M3, batches of 256 texts for detection and 32 sentences for translation.
+
+- **Small.** The default detection model is 0.9 MB. noentenc has no torch, transformers or GPU dependency, only numpy, onnxruntime, tokenizers, huggingface-hub and tqdm.
+- **One API, many models.** 12 detection models and 4 translation model families sit behind the same two classes. Swapping one is a one-line change, and every detector returns the same ISO 639-3 labels.
+- **Built for datasets.** You can pass a single string, a list or a pandas or polars column.
+
+## Install
+
+noentenc uses [uv](https://docs.astral.sh/uv/) to manage dependencies.
+
+```bash
+uv add noentenc                  # fastText, ONNX and langid detection + every translation model
+uv add 'noentenc[lingua,cld3]'   # extra detection backends, see the table below
+uv add 'noentenc[all]'
+```
+
+Weights download on first use to `~/.cache/noentenc` (detection) and the Hugging Face cache (translation). Nothing is bundled in the wheel.
+
+## Detect a language
+
+```python
+from noentenc.language_detection import LanguageDetector
+
+detector = LanguageDetector()  # fastText lid.176: 176 languages, 0.9 MB
+
+detector.detect("Bon dia! Com estàs?")
+# 'cat'
+detector.detect("Bon dia! Com estàs?", with_score=True, top_k=2)
+# {'cat': 0.88, 'por': 0.11}
+detector.detect_batch(["Hello there", "Hola, ¿qué tal?", "你好", ""])
+# ['eng', 'spa', 'zho', 'und']
+
+# Add a "lang" column to a pandas or polars DataFrame.
+detector.detect_dataset(df, "text", "lang")
+```
+
+## Translate
+
+```python
+from noentenc import Language
+from noentenc.translation import Translator
+
+translator = Translator()
+
+translator.translate_batch(
+    ["Where is the station?", "I love this city."], Language.SPANISH, Language.ENGLISH
+)
+# ['¿Dónde está la estación?', 'Me encanta esta ciudad.']
+
+# The source language is optional.
+translator.translate("Bon dia a tothom!", Language.ENGLISH)
+# 'Good day to everyone!'
+
+translator.translate_dataset(df, "review", "review_en", Language.ENGLISH)
+```
+
+`Translator()` picks the lightest model for each pair. It uses a dedicated Opus-MT model when one exists for the direction (66 directions, about 75M parameters each). For any other pair it uses SMaLL-100, which covers 100 languages. Languages are always `Language` enum members, so a typo fails at the call site, and a model that can't handle a pair raises `UnsupportedLanguageError`.
+
+## Examples
+
+| Example | Shows how to |
+|---|---|
+| [detect_dataset.py](docs/examples/detect_dataset.py) | Tag a DataFrame column and keep only confident English rows. |
+| [translate_to_english.py](docs/examples/translate_to_english.py) | Detect each message's language, then batch-translate everything into English. |
+| [choose_detection_backend.py](docs/examples/choose_detection_backend.py) | Swap backends, restrict candidate languages, collapse macrolanguages. |
+| [choose_translation_model.py](docs/examples/choose_translation_model.py) | Pick a model, precision and thread count, and run offline. |
+| [custom_models.py](docs/examples/custom_models.py) | Plug your own detector and translator into the same API. |
+
+## Models
+
+### Language detection
+
+| Backend | `model=` | Install | Languages | Size | Licence (weights) |
+|---|---|---|---|---|---|
+| `FastTextModel` | `lid176` (default) | core | 176 | 0.9 MB | CC-BY-SA-3.0 |
+| | `lid176-bin` | core | 176 | 126 MB | CC-BY-SA-3.0 |
+| | `openlid-v2` | core | 200 | 1.2 GB | GPL-3.0 |
+| | `openlid-v3` | core | 195 | 1.2 GB | GPL-3.0 |
+| | `glotlid` | core | 2102 | 1.7 GB | Apache-2.0 |
+| | `nllb-lid218e` | core | 218 | 1.2 GB | CC-BY-NC-4.0 |
+| | path to a `.bin`/`.ftz` | core | | | |
+| `OnnxClassifierModel` | `bert-openlid` (int8) | core | 201 | 25 MB | MIT |
+| | `xlm-roberta-lid` (int8) | core | 20 | 279 MB | MIT |
+| `LangidModel` | | core | 97 | 1.9 MB | BSD-2-Clause |
+| `LinguaModel` | | `noentenc[lingua]` | 75 | ~300 MB wheel | Apache-2.0 |
+| `Cld3Model` | | `noentenc[cld3]` | 107 | 1 MB | Apache-2.0 |
+| `HeliportModel` | | `noentenc[heliport]` (no Windows) | 220 | ~130 MB wheel | GPL-3.0 |
+
+```python
+from noentenc.language_detection import FastTextModel, LanguageDetector, LinguaModel
+
+LanguageDetector(FastTextModel("glotlid"))  # 2102 languages
+LanguageDetector(LinguaModel(languages=["cat", "spa", "eng"]))  # only these candidates
+```
+
+- Labels are ISO 639-3 codes (`eng`, `cat`, `zho`). Empty and whitespace-only texts return `und`.
+  - `normalize_labels=False` returns each model's native codes instead.
+  - `collapse_macrolanguages=True` folds individual languages into their macrolanguage (`arb` → `ara`, `cmn` → `zho`), so results from different models line up.
+- `FastTextModel` is our own numpy implementation of fastText inference. It needs neither the `fasttext` package nor onnxruntime, and it matches `fasttext`'s output to within 1e-6. Large `.bin` models are memory-mapped, so they open instantly.
+- `LangidModel` is our own numpy implementation of [langid.py](https://github.com/saffsd/langid.py). It reads the weights from the langid 1.1.6 source release on PyPI, without installing or importing the `langid` package, and matches its probabilities to within 1e-9.
+
+### Translation
+
+| Model | Languages | Download (default precision) | Licence (weights) |
+|---|---|---|---|
+| `OpusMTModel.from_pair(src, tgt)` | 1 direction each, 66 available (`OPUS_MT_PAIRS`) | 287 MB (q4), 107 MB (int8) | CC-BY-4.0 or Apache-2.0, per pair |
+| `SMaLL100Model` | any source → 100 targets | 595 MB (int8) | MIT |
+| `M2M100Model` | 100 ↔ 100 | 1.2 GB (q4), 603 MB (int8) | MIT |
+| `NLLBModel` | 196 ↔ 196 | 860 MB (int8) | CC-BY-NC-4.0, never picked by default |
+
+Every translation model takes `precision=` (`fp32`, `int8`, `q4`, where the export has it), `num_threads=` and a Hugging Face repo id or local directory as `model=`.
+
+### Weights and licences
+
+Weights are pinned to a revision and, where we download them ourselves, checked against a sha256. Set `NOENTENC_CACHE` to change the detection cache location. With `only_local_files=True`, a model that isn't cached raises instead of downloading, which is what you want on an air-gapped server.
+
+The weights' licences apply to your use of the weights. They don't affect this package's licence.
+
+## Bring your own model
+
+Load your own fastText, ONNX classifier or seq2seq weights into an existing backend, or wrap any detector or translator by subclassing a base class with a few methods. See [docs/custom-models.md](docs/custom-models.md).
+
+```python
+from noentenc.language_detection import FastTextModel, LanguageDetector
+
+LanguageDetector(FastTextModel("models/my-domain-lid.ftz"))
+```
+
+## Development
+
+```bash
+make install             # uv sync with every extra + dev tools
+make lint
+make unit-tests
+make integration-tests   # tests/integrations; downloads real weights
+```
+
+`scripts/benchmark_lid.py` and `scripts/benchmark_translation.py` reproduce the speed numbers. `scripts/make_fasttext_fixtures.py` regenerates the fastText parity fixtures with the reference `fasttext` package. That package needs Python 3.12, and the script's docstring has the command. `scripts/make_langid_fixtures.py` does the same for langid with the reference `langid` package.

noentenc-0.1.0/README.md

@@ -0,0 +1,173 @@
+<div align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="docs/assets/logo-dark.svg">
+    <source media="(prefers-color-scheme: light)" srcset="docs/assets/logo-light.svg">
+    <img src="docs/assets/logo-light.svg" alt="noentenc" width="520">
+  </picture>
+  <p>Language detection and machine translation for Python, on CPU, without torch.</p>
+</div>
+
+```python
+from noentenc import Language
+from noentenc.language_detection import LanguageDetector
+from noentenc.translation import Translator
+
+LanguageDetector().detect("Bon dia! Com estàs?")
+# 'cat'
+
+Translator().translate("The weather is nice today.", Language.SPANISH, Language.ENGLISH)
+# 'El tiempo es bueno hoy.'
+```
+
+- **Fast.** The default detector labels about 350,000 texts per second on a laptop CPU, and a direct Opus-MT translation takes about 60 ms per sentence. See [benchmarks](docs/benchmarks.md).
+
+  | Task | Model | Single input | Batched |
+  |---|---|---:|---:|
+  | Detection | heliport | 4 µs | 872k texts/s |
+  | Detection | **lid176 (default)** | 17 µs | 356k texts/s |
+  | Detection | cld3 | 16 µs | 66k texts/s |
+  | Detection | lingua | 0.44 ms | 9k texts/s |
+  | Detection | bert-openlid | 0.35 ms | 3.5k texts/s |
+  | Translation (en→es) | **Opus-MT (default for direct pairs)** | 63 ms | 100 sent/s |
+  | Translation (en→es) | **SMaLL-100 (default otherwise)** | 52 ms | 75 sent/s |
+  | Translation (en→es) | M2M100 418M | 231 ms | 15 sent/s |
+  | Translation (en→es) | NLLB-200 600M (int8) | 770 ms | 10 sent/s |
+
+  Apple M3, batches of 256 texts for detection and 32 sentences for translation.
+
+- **Small.** The default detection model is 0.9 MB. noentenc has no torch, transformers or GPU dependency, only numpy, onnxruntime, tokenizers, huggingface-hub and tqdm.
+- **One API, many models.** 12 detection models and 4 translation model families sit behind the same two classes. Swapping one is a one-line change, and every detector returns the same ISO 639-3 labels.
+- **Built for datasets.** You can pass a single string, a list or a pandas or polars column.
+
+## Install
+
+noentenc uses [uv](https://docs.astral.sh/uv/) to manage dependencies.
+
+```bash
+uv add noentenc                  # fastText, ONNX and langid detection + every translation model
+uv add 'noentenc[lingua,cld3]'   # extra detection backends, see the table below
+uv add 'noentenc[all]'
+```
+
+Weights download on first use to `~/.cache/noentenc` (detection) and the Hugging Face cache (translation). Nothing is bundled in the wheel.
+
+## Detect a language
+
+```python
+from noentenc.language_detection import LanguageDetector
+
+detector = LanguageDetector()  # fastText lid.176: 176 languages, 0.9 MB
+
+detector.detect("Bon dia! Com estàs?")
+# 'cat'
+detector.detect("Bon dia! Com estàs?", with_score=True, top_k=2)
+# {'cat': 0.88, 'por': 0.11}
+detector.detect_batch(["Hello there", "Hola, ¿qué tal?", "你好", ""])
+# ['eng', 'spa', 'zho', 'und']
+
+# Add a "lang" column to a pandas or polars DataFrame.
+detector.detect_dataset(df, "text", "lang")
+```
+
+## Translate
+
+```python
+from noentenc import Language
+from noentenc.translation import Translator
+
+translator = Translator()
+
+translator.translate_batch(
+    ["Where is the station?", "I love this city."], Language.SPANISH, Language.ENGLISH
+)
+# ['¿Dónde está la estación?', 'Me encanta esta ciudad.']
+
+# The source language is optional.
+translator.translate("Bon dia a tothom!", Language.ENGLISH)
+# 'Good day to everyone!'
+
+translator.translate_dataset(df, "review", "review_en", Language.ENGLISH)
+```
+
+`Translator()` picks the lightest model for each pair. It uses a dedicated Opus-MT model when one exists for the direction (66 directions, about 75M parameters each). For any other pair it uses SMaLL-100, which covers 100 languages. Languages are always `Language` enum members, so a typo fails at the call site, and a model that can't handle a pair raises `UnsupportedLanguageError`.
+
+## Examples
+
+| Example | Shows how to |
+|---|---|
+| [detect_dataset.py](docs/examples/detect_dataset.py) | Tag a DataFrame column and keep only confident English rows. |
+| [translate_to_english.py](docs/examples/translate_to_english.py) | Detect each message's language, then batch-translate everything into English. |
+| [choose_detection_backend.py](docs/examples/choose_detection_backend.py) | Swap backends, restrict candidate languages, collapse macrolanguages. |
+| [choose_translation_model.py](docs/examples/choose_translation_model.py) | Pick a model, precision and thread count, and run offline. |
+| [custom_models.py](docs/examples/custom_models.py) | Plug your own detector and translator into the same API. |
+
+## Models
+
+### Language detection
+
+| Backend | `model=` | Install | Languages | Size | Licence (weights) |
+|---|---|---|---|---|---|
+| `FastTextModel` | `lid176` (default) | core | 176 | 0.9 MB | CC-BY-SA-3.0 |
+| | `lid176-bin` | core | 176 | 126 MB | CC-BY-SA-3.0 |
+| | `openlid-v2` | core | 200 | 1.2 GB | GPL-3.0 |
+| | `openlid-v3` | core | 195 | 1.2 GB | GPL-3.0 |
+| | `glotlid` | core | 2102 | 1.7 GB | Apache-2.0 |
+| | `nllb-lid218e` | core | 218 | 1.2 GB | CC-BY-NC-4.0 |
+| | path to a `.bin`/`.ftz` | core | | | |
+| `OnnxClassifierModel` | `bert-openlid` (int8) | core | 201 | 25 MB | MIT |
+| | `xlm-roberta-lid` (int8) | core | 20 | 279 MB | MIT |
+| `LangidModel` | | core | 97 | 1.9 MB | BSD-2-Clause |
+| `LinguaModel` | | `noentenc[lingua]` | 75 | ~300 MB wheel | Apache-2.0 |
+| `Cld3Model` | | `noentenc[cld3]` | 107 | 1 MB | Apache-2.0 |
+| `HeliportModel` | | `noentenc[heliport]` (no Windows) | 220 | ~130 MB wheel | GPL-3.0 |
+
+```python
+from noentenc.language_detection import FastTextModel, LanguageDetector, LinguaModel
+
+LanguageDetector(FastTextModel("glotlid"))  # 2102 languages
+LanguageDetector(LinguaModel(languages=["cat", "spa", "eng"]))  # only these candidates
+```
+
+- Labels are ISO 639-3 codes (`eng`, `cat`, `zho`). Empty and whitespace-only texts return `und`.
+  - `normalize_labels=False` returns each model's native codes instead.
+  - `collapse_macrolanguages=True` folds individual languages into their macrolanguage (`arb` → `ara`, `cmn` → `zho`), so results from different models line up.
+- `FastTextModel` is our own numpy implementation of fastText inference. It needs neither the `fasttext` package nor onnxruntime, and it matches `fasttext`'s output to within 1e-6. Large `.bin` models are memory-mapped, so they open instantly.
+- `LangidModel` is our own numpy implementation of [langid.py](https://github.com/saffsd/langid.py). It reads the weights from the langid 1.1.6 source release on PyPI, without installing or importing the `langid` package, and matches its probabilities to within 1e-9.
+
+### Translation
+
+| Model | Languages | Download (default precision) | Licence (weights) |
+|---|---|---|---|
+| `OpusMTModel.from_pair(src, tgt)` | 1 direction each, 66 available (`OPUS_MT_PAIRS`) | 287 MB (q4), 107 MB (int8) | CC-BY-4.0 or Apache-2.0, per pair |
+| `SMaLL100Model` | any source → 100 targets | 595 MB (int8) | MIT |
+| `M2M100Model` | 100 ↔ 100 | 1.2 GB (q4), 603 MB (int8) | MIT |
+| `NLLBModel` | 196 ↔ 196 | 860 MB (int8) | CC-BY-NC-4.0, never picked by default |
+
+Every translation model takes `precision=` (`fp32`, `int8`, `q4`, where the export has it), `num_threads=` and a Hugging Face repo id or local directory as `model=`.
+
+### Weights and licences
+
+Weights are pinned to a revision and, where we download them ourselves, checked against a sha256. Set `NOENTENC_CACHE` to change the detection cache location. With `only_local_files=True`, a model that isn't cached raises instead of downloading, which is what you want on an air-gapped server.
+
+The weights' licences apply to your use of the weights. They don't affect this package's licence.
+
+## Bring your own model
+
+Load your own fastText, ONNX classifier or seq2seq weights into an existing backend, or wrap any detector or translator by subclassing a base class with a few methods. See [docs/custom-models.md](docs/custom-models.md).
+
+```python
+from noentenc.language_detection import FastTextModel, LanguageDetector
+
+LanguageDetector(FastTextModel("models/my-domain-lid.ftz"))
+```
+
+## Development
+
+```bash
+make install             # uv sync with every extra + dev tools
+make lint
+make unit-tests
+make integration-tests   # tests/integrations; downloads real weights
+```
+
+`scripts/benchmark_lid.py` and `scripts/benchmark_translation.py` reproduce the speed numbers. `scripts/make_fasttext_fixtures.py` regenerates the fastText parity fixtures with the reference `fasttext` package. That package needs Python 3.12, and the script's docstring has the command. `scripts/make_langid_fixtures.py` does the same for langid with the reference `langid` package.

noentenc-0.1.0/pyproject.toml

@@ -0,0 +1,115 @@
+[project]
+name = "noentenc"
+version = "0.1.0"
+description = "Lightweight, CPU-only language detection and translation for Python. Extra detection backends are optional extras."
+readme = "README.md"
+requires-python = ">=3.14"
+dependencies = [
+    "huggingface-hub>=1.33.0",
+    "numpy>=2.0",
+    "onnxruntime>=1.30.0",
+    "tokenizers>=0.23.2",
+    "tqdm>=4.70.1",
+]
+
+[[project.authors]]
+name = "marti-jorda-roca"
+email = "mjorda98@gmail.com"
+
+[project.optional-dependencies]
+pandas = ["pandas>=3.0.6"]
+polars = ["polars>=1.44.2"]
+lingua = ["lingua-language-detector>=2.2"]
+cld3 = ["cld3-py>=3.1"]
+heliport = ["heliport>=1.0.1; sys_platform != 'win32'"]
+all = ["noentenc[pandas,polars,lingua,cld3,heliport]"]
+
+[build-system]
+requires = ["uv_build>=0.12.22,<0.13.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "bandit[toml]>=1.9.4",
+    "coverage>=7.16.2",
+    "library-skills>=0.0.19",
+    "pandas>=3.0.6",
+    "polars>=1.44.2",
+    "prek>=0.5.4",
+    "pytest>=9.1.1",
+    "ruff>=0.16.10",
+    "ty>=0.0.84",
+]
+
+[tool.ruff]
+target-version = "py314"
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "W",
+    "F",
+    "I",
+    "B",
+    "C4",
+    "UP",
+    "ARG001",
+    "T201",
+    "C90",
+    "PLR",
+    "ERA",
+    "SIM",
+    "RET",
+    "PIE",
+    "B",
+    "ANN",
+]
+ignore = [
+    "E501",
+    "B008",
+    "W191",
+    "B904",
+    "PLR0913",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"docs/examples/**/*.py" = [
+    "PLR2004",
+    "T201",
+    "ERA001",
+]
+"scripts/**/*.py" = [
+    "PLR2004",
+    "T201",
+]
+"tests/**/*.py" = ["PLR2004"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.mccabe]
+max-complexity = 10
+
+[tool.ruff.lint.pylint]
+max-branches = 12
+max-returns = 6
+max-statements = 50
+max-args = 8
+max-nested-blocks = 5
+
+[tool.ty.terminal]
+error-on-warning = true
+
+[tool.ty.src]
+include = [
+    "src",
+    "tests",
+    "docs",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["."]
+addopts = "--import-mode=importlib"
+consider_namespace_packages = true
+markers = ["integration: needs real model weights (downloads them); run with NOENTENC_RUN_INTEGRATION=1"]

noentenc-0.1.0/pyproject.toml.orig

@@ -0,0 +1,111 @@
+[project]
+name = "noentenc"
+version = "0.1.0"
+description = "Lightweight, CPU-only language detection and translation for Python. Extra detection backends are optional extras."
+readme = "README.md"
+authors = [
+    { name = "marti-jorda-roca", email = "mjorda98@gmail.com" }
+]
+requires-python = ">=3.14"
+dependencies = [
+    "huggingface-hub>=1.33.0",
+    "numpy>=2.0",
+    "onnxruntime>=1.30.0",
+    "tokenizers>=0.23.2",
+    "tqdm>=4.70.1",
+]
+
+[project.optional-dependencies]
+pandas = ["pandas>=3.0.6"]
+polars = ["polars>=1.44.2"]
+lingua = ["lingua-language-detector>=2.2"]
+cld3 = ["cld3-py>=3.1"]
+heliport = ["heliport>=1.0.1; sys_platform != 'win32'"]
+all = ["noentenc[pandas,polars,lingua,cld3,heliport]"]
+
+[build-system]
+requires = ["uv_build>=0.12.22,<0.13.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "bandit[toml]>=1.9.4",
+    "coverage>=7.16.2",
+    "library-skills>=0.0.19",
+    "pandas>=3.0.6",
+    "polars>=1.44.2",
+    "prek>=0.5.4",
+    "pytest>=9.1.1",
+    "ruff>=0.16.10",
+    "ty>=0.0.84",
+]
+
+
+[tool.ruff]
+target-version = "py314"
+
+[tool.ruff.lint]
+select = [
+    "E",  # pycodestyle errors
+    "W",  # pycodestyle warnings
+    "F",  # pyflakes
+    "I",  # isort
+    "B",  # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "UP",  # pyupgrade
+    "ARG001", # unused arguments in functions
+    "T201",   # print statements are not allowed,
+    "C90", # cyclomatic complexity
+    "PLR", # pylint-style size limits
+    "ERA",   # Commented-out code
+    "SIM",   # Simplifiable/redundant constructs
+    "RET",   # Redundant return patterns
+    "PIE",   # Miscellaneous unnecessary code
+    "B",     # Bugbear, including duplicate/redundant constructs
+    "ANN",   # Missing annotations and explicit Any checks
+]
+ignore = [
+    "E501",  # line too long, handled by black
+    "B008",  # do not perform function calls in argument defaults
+    "W191",  # indentation contains tabs
+    "B904",  # Allow raising exceptions without from e
+    "PLR0913", # Too many arguments
+]
+
+[tool.ruff.lint.per-file-ignores]
+# Magic values are the point in tests and one-off scripts; only library code must name them.
+# Examples print results and show expected output in comments.
+"docs/examples/**/*.py" = ["PLR2004", "T201", "ERA001"]
+"scripts/**/*.py" = ["PLR2004", "T201"]
+"tests/**/*.py" = ["PLR2004"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.mccabe]
+max-complexity = 10
+
+[tool.ruff.lint.pylint]
+max-branches = 12
+max-returns = 6
+max-statements = 50
+max-args = 8
+max-nested-blocks = 5
+
+[tool.ty.terminal]
+error-on-warning = true
+
+[tool.ty.src]
+include = ["src", "tests", "docs"]
+
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["."]
+# Projects share test module basenames (e.g. test_train_model.py); importlib mode
+# gives each file a unique module name instead of colliding on sys.modules.
+addopts = "--import-mode=importlib"
+consider_namespace_packages = true
+markers = [
+    "integration: needs real model weights (downloads them); run with NOENTENC_RUN_INTEGRATION=1",
+]

noentenc-0.1.0/src/noentenc/__init__.py

@@ -0,0 +1,13 @@
+from noentenc.languages import (
+    ANY_LANGUAGE,
+    Language,
+    LanguageSchema,
+    UnsupportedLanguageError,
+)
+
+__all__ = [
+    "ANY_LANGUAGE",
+    "Language",
+    "LanguageSchema",
+    "UnsupportedLanguageError",
+]

noentenc-0.1.0/src/noentenc/_dataframe.py

@@ -0,0 +1,41 @@
+from typing import TYPE_CHECKING, Any, cast
+
+if TYPE_CHECKING:
+    import pandas as pd
+    import polars as pl
+
+
+def column_values(dataset: pl.DataFrame | pd.DataFrame, column: str) -> list[Any]:
+    _frame_module(dataset)
+    return dataset[column].to_list()
+
+
+def with_column(
+    dataset: pl.DataFrame | pd.DataFrame,
+    column: str,
+    values: list[Any],
+    *,
+    strings: bool = False,
+) -> pl.DataFrame | pd.DataFrame:
+    """Return `dataset` with `column` set to `values`.
+
+    `strings` types a polars column as String even when every value is null.
+    """
+    if _frame_module(dataset) == "polars":
+        # The caller passed a polars frame, so polars is installed.
+        import polars
+
+        frame = cast("pl.DataFrame", dataset)
+        return frame.with_columns(
+            polars.Series(column, values, polars.String if strings else None)
+        )
+    return cast("pd.DataFrame", dataset).assign(**{column: values})
+
+
+def _frame_module(dataset: object) -> str:
+    module = type(dataset).__module__.split(".", 1)[0]
+    if module not in {"polars", "pandas"}:
+        raise TypeError(
+            f"expected a polars or pandas DataFrame, got {type(dataset).__name__}"
+        )
+    return module

noentenc-0.1.0/src/noentenc/_onnx.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+import numpy as np
+import onnxruntime as ort
+
+
+def create_session(path: Path, num_threads: int | None) -> ort.InferenceSession:
+    options = ort.SessionOptions()
+    options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+    options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
+    options.inter_op_num_threads = 1
+    if num_threads is not None:
+        options.intra_op_num_threads = num_threads
+    return ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"])
+
+
+def pad(sequences: list[list[int]], pad_id: int) -> tuple[np.ndarray, np.ndarray]:
+    """Right-pad token id sequences into `(input_ids, attention_mask)`."""
+    width = max(len(s) for s in sequences)
+    input_ids = np.full((len(sequences), width), pad_id, dtype=np.int64)
+    attention_mask = np.zeros((len(sequences), width), dtype=np.int64)
+    for row, sequence in enumerate(sequences):
+        input_ids[row, : len(sequence)] = sequence
+        attention_mask[row, : len(sequence)] = 1
+    return input_ids, attention_mask