banglatok 0.2.0__tar.gz

banglatok-0.2.0/CHANGELOG.md

@@ -0,0 +1,33 @@
+# Changelog
+
+## 0.2.0 — 2026-11-13
+
+First public release.
+
+### Added
+
+- Two pretrained models bundled in the wheel and loaded offline with `Tokenizer.from_pretrained()`. `bn-bpe-58k` (default) and `bn-bpe-28k` are space-fused BPE vocabularies, documented in `docs/MODELS.md` and licensed CC BY-SA 4.0.
+- `tokenize`, `count_tokens`, `decode_batch`, `id_to_token`, `token_to_id`, `get_vocab`, the `algorithm` property, `available_models()` and the control-ID constants.
+- `tokenizer(texts, add_special_tokens=..., max_length=..., padding=...)` returns encoder-ready `input_ids` and `attention_mask`. `decode(..., skip_special_tokens=True)` drops the control IDs again.
+- A `banglatok` command and `python -m banglatok`.
+- `benchmarks/compare_popular.py` and `benchmarks/POPULAR_COMPARISON.md` compare the models with SentencePiece/BNLP, tiktoken, BanglaBERT, MuRIL, mBERT, XLM-R and Qwen3 on unseen and in-domain text.
+- `benchmarks/build_fused_model.py` reproducibly builds fused models from a base BPE model.
+- `Tokenizer.save_huggingface(directory)` writes `tokenizer.json` and `tokenizer_config.json` for Hugging Face `tokenizers` and `transformers`, with the same IDs as the native tokenizer (checked on 24,799 validation, review and stress texts for both models). `tools/publish_huggingface.py` uploads both models to the Hub.
+- Linux aarch64 and Windows x86_64 wheels in the release workflow.
+- `benchmarks/train_sentencepiece_baselines.py` trains SentencePiece Unigram and BPE baselines with the same vocabulary sizes as the bundled models; the comparison now includes them, and a Linux x86_64 run.
+
+### Changed
+
+- The project is now `banglatok` (was `bn-tokenizers-embedding`, never published to PyPI). The import is `banglatok` and the command is `banglatok`.
+- Encoding is about 2× faster with identical IDs. Normalization skips work for text that is already NFC. BPE merges recompute pair ranks only around merged tokens. Single texts are encoded straight into a reusable per-thread buffer, and batch results come back in one aligned buffer. Every model and about 94,000 validation, test and stress texts produce the same IDs as before.
+- Go: `Tokenizer.AppendEncode`, `Tokenizer.Vocabulary`, `Tokenizer.Algorithm` and `Tokenizer.Merges`.
+- The `banglatok` command reads and writes UTF-8 regardless of the console code page.
+- Research reports, experiments and scripts moved under `research/`.
+- License expression: `MIT AND BSD-3-Clause AND CC-BY-SA-4.0`.
+
+## 0.1.0a1 — unreleased preview
+
+- Prepared the typed Python/Go tokenizer API for a public alpha, with an offline toy vocabulary and a self-contained training example.
+- Added MIT licensing, retained BSD-3-Clause native notices, installed-distribution CI, archive validation and opt-in platform-wheel/PyPI workflows.
+
+The earlier `0.1.0` artifacts were local development builds and were never published to PyPI.

banglatok-0.2.0/CITATION.cff

@@ -0,0 +1,19 @@
+cff-version: 1.2.0
+message: "If you use this software or its bundled models, please cite it as below."
+title: "banglatok: fast, lossless Bangla, English and Banglish tokenizers"
+type: software
+authors:
+  - family-names: Islam
+    given-names: Aber
+    email: aber.islam.dev@gmail.com
+repository-code: "https://github.com/iamaber/banglatok"
+license: MIT
+version: 0.2.0
+date-released: 2026-11-13
+keywords:
+  - Bengali
+  - Bangla
+  - Banglish
+  - tokenizer
+  - BPE
+  - code-mixing

banglatok-0.2.0/CONTRIBUTING.md

@@ -0,0 +1,34 @@
+# Contributing
+
+Small, reproducible changes are welcome. Discuss API changes in an issue first. Keep tokenizer behavior, model formats and research claims explicit; a speed improvement must preserve IDs unless a new model version is deliberate.
+
+## Development
+
+Use Python 3.10+ and Go 1.24+ with a C compiler. A supported, maintained Python version is preferable for development.
+
+```sh
+python -m venv .venv-dev
+source .venv-dev/bin/activate
+python -m pip install build twine ruff
+python -m pip install .
+python -m unittest discover -s tests -v
+go test -race ./...
+go vet ./...
+ruff check src tests tools examples setup.py
+```
+
+Tests exercise the installed Go shared library. After changing Go or Python package code, reinstall with `python -m pip install --force-reinstall .` before testing. Running against an old wheel can test an obsolete native runtime.
+
+Format changed Go files with `gofmt`. Prefer the standard library and the existing small API over new dependencies or generic abstractions. Add a focused regression test for behavior changes, and test actual native behavior rather than mocked return values.
+
+## Package changes
+
+Follow [docs/RELEASING.md](docs/RELEASING.md) to build from a source archive, inspect package contents and test installed wheels. Do not add corpora, private text, credentials or research checkpoints to distributions. The only bundled models are the documented ones in `src/banglatok/pretrained/` (see [docs/MODELS.md](docs/MODELS.md)). Performance changes must keep their IDs byte-for-byte identical; `go test ./tokenizer -bench Validation` with `BNTOK_MODEL`/`BNTOK_CORPUS` measures the native path.
+
+## Research changes
+
+Research environments are separate from the zero-dependency tokenizer API; see [research/experiments/retrieval/README.md](research/experiments/retrieval/README.md). Preserve frozen datasets and decisions. Report annotation origin, actual compute/data budgets, uncertainty and failed gates. Agent judgments must not be described as independent human review.
+
+## Pull requests
+
+Explain the problem, resulting behavior and verification. Include OS/architecture and Python/Go versions for native changes. Use a minimal hand-written reproduction instead of uploading private data. Keep discussion respectful and specific; review the code and evidence rather than the person.

banglatok-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Aber Islam
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

banglatok-0.2.0/MANIFEST.in

@@ -0,0 +1,13 @@
+include go.mod go.sum
+recursive-include tokenizer *.go
+recursive-include internal/native *.go
+include cmd/bntok-shared/main.go
+recursive-include src/banglatok/pretrained *.json
+recursive-include tests *.py
+recursive-include examples *.py *.jsonl
+include tools/check_dist.py
+include CHANGELOG.md CONTRIBUTING.md SECURITY.md CITATION.cff MODEL_LICENSE.md docs/MODELS.md
+prune data
+prune models
+prune research
+prune .tools

banglatok-0.2.0/MODEL_LICENSE.md

@@ -0,0 +1,13 @@
+# Bundled model files
+
+The files in `src/banglatok/pretrained/` (`bn-bpe-58k.json` and `bn-bpe-28k.json`) are licensed under the Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0): https://creativecommons.org/licenses/by-sa/4.0/legalcode
+
+They contain subword vocabularies, merge rules and frequent-word lists learned from:
+
+- Bengali Wikipedia (dump `20231101.bn`, via the `wikimedia/wikipedia` dataset). © Wikipedia contributors, CC BY-SA 3.0 / GFDL.
+- Tatoeba sentences, via the ManyThings.org Anki exports. © Tatoeba contributors, CC BY 2.0.
+- BanglaTLit, https://github.com/farhanishmam/BanglaTLit. MIT License.
+
+Changes: the source text was NFC- and whitespace-normalized, filtered and split as described in `data/CORPUS.md`. The models were then learned from the training split with BPE training, space fusion and word-frequency counting. See `docs/MODELS.md`.
+
+Everything else in this repository and in the distributions, including the tokenizer software and the hand-authored demo vocabulary, is covered by `LICENSE` (MIT) and `THIRD_PARTY_NOTICES.md`.

banglatok-0.2.0/PKG-INFO

@@ -0,0 +1,204 @@
+Metadata-Version: 2.4
+Name: banglatok
+Version: 0.2.0
+Summary: Fast, lossless Bangla, English and Banglish subword tokenizers with bundled models and a native Go engine
+Author-email: Aber Islam <aber.islam.dev@gmail.com>
+License-Expression: MIT AND BSD-3-Clause AND CC-BY-SA-4.0
+Project-URL: Homepage, https://github.com/iamaber/banglatok
+Project-URL: Repository, https://github.com/iamaber/banglatok
+Project-URL: Documentation, https://github.com/iamaber/banglatok#readme
+Project-URL: Issues, https://github.com/iamaber/banglatok/issues
+Project-URL: Changelog, https://github.com/iamaber/banglatok/blob/main/CHANGELOG.md
+Keywords: bengali,bangla,banglish,romanized-bengali,code-mixed,tokenizer,subword,bpe,unigram,nlp,llm
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Natural Language :: Bengali
+Classifier: Natural Language :: English
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Programming Language :: Go
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: MODEL_LICENSE.md
+License-File: THIRD_PARTY_NOTICES.md
+License-File: third_party/licenses/GO-LICENSE
+License-File: third_party/licenses/X-TEXT-LICENSE
+License-File: third_party/licenses/X-TEXT-PATENTS
+Dynamic: license-file
+
+# banglatok
+
+**Fast, lossless subword tokenizers for Bangla, English and Banglish (romanized Bengali), including code-mixed text.**
+
+[![CI](https://github.com/iamaber/banglatok/actions/workflows/ci.yml/badge.svg)](https://github.com/iamaber/banglatok/actions/workflows/ci.yml)
+![Python 3.10–3.15](https://img.shields.io/badge/python-3.10%E2%80%933.15-blue)
+![Runtime dependencies: none](https://img.shields.io/badge/runtime%20deps-none-brightgreen)
+[![License: MIT (code), CC BY-SA 4.0 (models)](https://img.shields.io/badge/license-MIT%20%7C%20CC%20BY--SA%204.0-lightgrey)](#license)
+
+```python
+from banglatok import Tokenizer
+
+tok = Tokenizer.from_pretrained()          # bundled model; no download, no network
+tok.tokenize("meetingটা ১০টায়, don't be late 🙂")
+# ['meeting', 'টা', ' ১০', 'টায়', ',', " don't", ' be', ' late', ' 🙂']
+```
+
+- **Short sequences.** On unseen product reviews `bn-bpe-58k` produced 22% fewer tokens than BanglaBERT and GPT-4o's `o200k_base`, 21% fewer than XLM-R/mE5/BGE-M3, 8% fewer than MuRIL, and 62% fewer than Qwen3. Most of that comes from training on Bangla and Banglish text at this vocabulary size: SentencePiece trained on the same data with the same vocabulary size comes within about 2% (see [Benchmarks](#benchmarks)).
+- **Lossless.** `decode(encode(text))` gives back the normalized text exactly, for every input. There is no `[UNK]` and no lowercasing, and Bengali marks, ZWJ/ZWNJ, emoji and mixed scripts are kept.
+- **Fast.** Single-threaded, on unseen text, it was 1.1–1.7× faster than SentencePiece Unigram and 2.6–4.4× faster than SentencePiece BPE at the same vocabulary size, 1.3–5.9× faster than tiktoken, and 5–12× faster than Hugging Face `tokenizers`, on both an Apple M5 and a Linux x86_64 server CPU.
+- **Tiny footprint.** The engine is native Go behind `ctypes`, with zero Python runtime dependencies and typed APIs. It is thread-safe, two pretrained models ship inside the wheel, and they export to Hugging Face `tokenizers`/`transformers` with identical IDs.
+
+The vocabularies are meant for training or adapting encoders and LLMs on Bengali text, and for counting or segmenting such text.
+
+## Benchmarks
+
+Here are 10,000 naturally written e-commerce reviews from [BanglishRev](https://huggingface.co/datasets/BanglishRev/bangla-english-and-code-mixed-ecommerce-review-dataset), in Bengali, English, Banglish and mixed script. This source was never seen in training. Every tokenizer runs **single-threaded** (`RAYON_NUM_THREADS=1`, `num_threads=1`), on an Apple M5 (macOS) and on an AMD EPYC 9V74 GitHub-hosted Linux x86_64 VM, Python 3.12. Speed is MB/s for single calls / batches of 64.
+
+| Tokenizer | Vocab | Tokens / word ↓ | Lossless round trip ↑ | Texts with UNK ↓ | MB/s, Apple M5 ↑ | MB/s, Linux x86_64 ↑ |
+|---|---:|---:|---:|---:|---:|---:|
+| **bn-bpe-58k (this package)** | 57,925 | **1.379** | **100%** | **0** | 58.5 / 71.6 | 24.4 / 29.7 |
+| **bn-bpe-28k (this package)** | 27,760 | 1.587 | **100%** | **0** | **61.6 / 72.9** | **26.1 / 33.6** |
+| SentencePiece Unigram, same data and size as 58k | 57,925 | 1.410 | 100% | 0 | 42.4 / 41.7 | 20.5 / 20.9 |
+| SentencePiece BPE, same data and size as 58k | 57,925 | 1.407 | 100% | 0 | 17.7 / 17.4 | 9.5 / 9.8 |
+| SentencePiece Unigram, same data and size as 28k | 27,760 | 1.572 | 100% | 0 | 46.4 / 43.2 | 22.8 / 23.4 |
+| SentencePiece BPE, same data and size as 28k | 27,760 | 1.578 | 100% | 0 | 16.6 / 16.4 | 9.8 / 9.8 |
+| BNLP `bn_spm` (SentencePiece) | 50,000 | 1.825 | 80.1% | 1,932 | 40.1 / 39.2 | 19.3 / 19.8 |
+| tiktoken `o200k_base` (GPT-4o) | 200,019 | 1.780 | 100% | 0 | 37.3 / 12.1 | 19.4 / 5.4 |
+| BanglaBERT (csebuetnlp) | 32,000 | 1.771 | 51.1% | 1,110 | 9.1 / 9.2 | 4.8 / 4.6 |
+| MuRIL (Google) | 197,258 | 1.498 | 47.9% | 1,617 | 8.8 / 8.9 | 4.6 / 4.4 |
+| XLM-R / multilingual-E5 / BGE-M3 | 250,002 | 1.740 | 98.2% | 109 | 6.5 / 6.4 | 3.7 / 3.5 |
+| mBERT cased | 119,547 | 2.122 | 48.0% | 1,706 | 8.1 / 8.1 | 4.2 / 4.1 |
+| Qwen3 | 151,669 | 3.588 | 100% | 0 | 6.0 / 6.0 | 3.3 / 3.1 |
+
+What this shows:
+
+- **Against SentencePiece with the same data and vocabulary size, compression is a tie.** `bn-bpe-58k` is about 2% shorter than both SentencePiece models; `bn-bpe-28k` is about 1% longer. On the in-domain validation split, SentencePiece at the 28k size is 3–4.5% shorter. The gap to BanglaBERT, XLM-R and tiktoken comes from the training data and the vocabulary size, not from a better algorithm.
+- **The speed lead is real but depends on the CPU.** Against SentencePiece Unigram it is 1.3–1.7× on the M5 and 1.1–1.4× on the Linux VM. Hugging Face `tokenizers` was limited to one thread like everything else; its batch API can use all cores, which this table does not measure.
+- BanglaBERT is about 3% shorter on Bengali-only text but lossy. Among lossless tokenizers, `bn-bpe-58k` is the most compact on Bengali, Latin and mixed script.
+
+[Full report](benchmarks/POPULAR_COMPARISON.md): per-script breakdown, the in-domain validation split, more tokenizers, method, limitations, and the commands to reproduce it. Shorter sequences cut compute and context use, but they do not by themselves make a downstream model more accurate.
+
+## Install
+
+```sh
+pip install banglatok
+```
+
+Prebuilt wheels target Linux (x86_64, aarch64), macOS (x86_64, arm64) and Windows (x86_64) for CPython 3.10–3.15, and need no compiler. On other platforms pip builds from source, which needs Go 1.24+ and a C compiler:
+
+```sh
+pip install git+https://github.com/iamaber/banglatok
+```
+
+## Usage
+
+```python
+from banglatok import Tokenizer
+
+with Tokenizer.from_pretrained("bn-bpe-58k") as tok:
+    ids = tok.encode("আমি বাংলায় গান গাই। ami banglay gaan gai")
+    tok.decode(ids)                       # 'আমি বাংলায় গান গাই। ami banglay gaan gai'
+    tok.encode_batch(["আমি ভালো আছি", "kemon acho?"])   # one native call
+    tok.count_tokens("অনেক লম্বা লেখা…")    # fast length, no list built
+
+    # Encoder-ready inputs: <s> … </s>, truncation and padding with <pad>.
+    batch = tok(["আমি ভালো আছি", "hi"], max_length=128, padding=True)
+    batch["input_ids"], batch["attention_mask"]
+```
+
+Load a tokenizer once and reuse it. Calls from many threads at once are safe, because the native call releases the GIL. In worker processes, create a fresh tokenizer inside each worker.
+
+### Hugging Face
+
+```python
+tok.save_huggingface("bn-bpe-58k-hf")      # tokenizer.json + tokenizer_config.json
+
+from transformers import AutoTokenizer
+hf = AutoTokenizer.from_pretrained("bn-bpe-58k-hf")   # same IDs as tok.encode / tok(...)
+```
+
+The exported models are also on the Hub as [`iamaber/banglatok-bn-bpe-58k`](https://huggingface.co/iamaber/banglatok-bn-bpe-58k) and [`iamaber/banglatok-bn-bpe-28k`](https://huggingface.co/iamaber/banglatok-bn-bpe-28k). Only BPE models can be exported. With the `tokenizers` library alone, set `encode_special_tokens = True` so that literal `<s>` in text stays text; `transformers` does this from the saved configuration.
+
+### Command line
+
+```sh
+$ banglatok "আমি বাংলা বলি"
+আমি |  বাংলা |  বলি
+$ echo "ami bangla boli" | banglatok --ids
+4405 33616 34749
+$ banglatok --json "ami valo"
+{"ids": [4405, 32047], "tokens": ["ami", " valo"]}
+$ banglatok --count --model bn-bpe-28k < lines.txt    # one count per line
+```
+
+### API
+
+| Member | Description |
+|---|---|
+| `Tokenizer.from_pretrained(name="bn-bpe-58k")` | Load a bundled model, offline |
+| `Tokenizer(path)` | Load your own JSON model, e.g. one trained with `bntok train` |
+| `encode(text)` / `encode_batch(texts)` | Token IDs, without control tokens |
+| `decode(ids, skip_special_tokens=False)` / `decode_batch(batch)` | Normalized text |
+| `tokenize(text)` | Token strings: fused IDs start with `" "`, fallback bytes look like `"<0xF0>"` |
+| `count_tokens(text)` | `len(encode(text))`, without building the list |
+| `save_huggingface(directory)` | Write a Hugging Face `tokenizer.json` with identical IDs (BPE models) |
+| `tok(texts, add_special_tokens=True, max_length=None, padding=False)` | `{"input_ids", "attention_mask"}` |
+| `id_to_token`, `token_to_id`, `get_vocab()`, `vocab_size`, `algorithm` | Vocabulary access |
+| `normalize(text)` | The shared normalization: NFC plus whitespace collapse and trim |
+| `available_models()`, `PAD_ID=0`, `UNK_ID=1`, `BOS_ID=2`, `EOS_ID=3` | Constants |
+
+## Models
+
+| Name | Vocab | Best for |
+|---|---:|---|
+| `bn-bpe-58k` (default) | 57,925 | Shortest sequences |
+| `bn-bpe-28k` | 27,760 | Smaller embedding tables, same speed |
+
+Both are BPE vocabularies with *space fusion*: a space and the token after it can share a single ID. That removes most standalone space tokens without overloading a literal marker such as `▁`. They were learned from 563k records of Bengali Wikipedia, Tatoeba and BanglaTLit. [docs/MODELS.md](docs/MODELS.md) covers the build commands, checksums, behavior and limitations. They are not SentencePiece or Hugging Face vocabularies, so pairing one with a pretrained encoder means adapting that encoder's embeddings.
+
+## Train your own
+
+The Go CLI trains Unigram or BPE models on your JSONL corpus, and `Tokenizer(path)` loads the result:
+
+```sh
+go run ./cmd/bntok train -algorithm bpe -vocab 384 \
+  -input examples/training-records.jsonl -out models/toy-bpe.json
+go run ./cmd/bntok encode -model models/toy-bpe.json -text 'আমি বাংলা বলি। ami bangla boli'
+python benchmarks/build_fused_model.py --base models/my-bpe.json --budget 64000 \
+  --output models/my-fused.json        # optional: add space fusion
+```
+
+See [docs/TRAINING_AND_RESEARCH.md](docs/TRAINING_AND_RESEARCH.md) for the corpus format, the full research pipeline and the model contract.
+
+## Research
+
+This repository also holds the research behind the models, under [research/](research/). [data/CORPUS.md](data/CORPUS.md) documents the corpus construction, provenance and leakage controls. [research/reports/RESULTS.md](research/reports/RESULTS.md) has the intrinsic evaluation. The retrieval experiments are in [research/experiments/retrieval/](research/experiments/retrieval/README.md), and [research/reports/ROUND2_ORIGINAL_COMPARISON.md](research/reports/ROUND2_ORIGINAL_COMPARISON.md) has the encoder-adaptation results. In those experiments, swapping a custom vocabulary into multilingual-E5 did not beat the unchanged model. Treat these vocabularies as a strong starting point for training, not as a drop-in accuracy gain for existing encoders.
+
+## Contributing
+
+[CONTRIBUTING.md](CONTRIBUTING.md) explains how to work on the project, [SECURITY.md](SECURITY.md) how to report vulnerabilities, and [docs/RELEASING.md](docs/RELEASING.md) how releases are made. Platform reports, regression cases and independent benchmark runs are especially welcome.
+
+## Citation
+
+If you use this library or its models, please cite it with the metadata in [CITATION.cff](CITATION.cff).
+
+## License
+
+- Code, documentation and the demo vocabulary: [MIT](LICENSE).
+- Bundled model files (`bn-bpe-58k`, `bn-bpe-28k`): [CC BY-SA 4.0](MODEL_LICENSE.md), because they are learned from Wikipedia and Tatoeba text.
+- Native wheels include BSD-3-Clause Go components ([notices](THIRD_PARTY_NOTICES.md)).
+
+The SPDX expression is `MIT AND BSD-3-Clause AND CC-BY-SA-4.0`. Research corpora keep their own terms and are not distributed.

banglatok-0.2.0/README.md

@@ -0,0 +1,163 @@
+# banglatok
+
+**Fast, lossless subword tokenizers for Bangla, English and Banglish (romanized Bengali), including code-mixed text.**
+
+[![CI](https://github.com/iamaber/banglatok/actions/workflows/ci.yml/badge.svg)](https://github.com/iamaber/banglatok/actions/workflows/ci.yml)
+![Python 3.10–3.15](https://img.shields.io/badge/python-3.10%E2%80%933.15-blue)
+![Runtime dependencies: none](https://img.shields.io/badge/runtime%20deps-none-brightgreen)
+[![License: MIT (code), CC BY-SA 4.0 (models)](https://img.shields.io/badge/license-MIT%20%7C%20CC%20BY--SA%204.0-lightgrey)](#license)
+
+```python
+from banglatok import Tokenizer
+
+tok = Tokenizer.from_pretrained()          # bundled model; no download, no network
+tok.tokenize("meetingটা ১০টায়, don't be late 🙂")
+# ['meeting', 'টা', ' ১০', 'টায়', ',', " don't", ' be', ' late', ' 🙂']
+```
+
+- **Short sequences.** On unseen product reviews `bn-bpe-58k` produced 22% fewer tokens than BanglaBERT and GPT-4o's `o200k_base`, 21% fewer than XLM-R/mE5/BGE-M3, 8% fewer than MuRIL, and 62% fewer than Qwen3. Most of that comes from training on Bangla and Banglish text at this vocabulary size: SentencePiece trained on the same data with the same vocabulary size comes within about 2% (see [Benchmarks](#benchmarks)).
+- **Lossless.** `decode(encode(text))` gives back the normalized text exactly, for every input. There is no `[UNK]` and no lowercasing, and Bengali marks, ZWJ/ZWNJ, emoji and mixed scripts are kept.
+- **Fast.** Single-threaded, on unseen text, it was 1.1–1.7× faster than SentencePiece Unigram and 2.6–4.4× faster than SentencePiece BPE at the same vocabulary size, 1.3–5.9× faster than tiktoken, and 5–12× faster than Hugging Face `tokenizers`, on both an Apple M5 and a Linux x86_64 server CPU.
+- **Tiny footprint.** The engine is native Go behind `ctypes`, with zero Python runtime dependencies and typed APIs. It is thread-safe, two pretrained models ship inside the wheel, and they export to Hugging Face `tokenizers`/`transformers` with identical IDs.
+
+The vocabularies are meant for training or adapting encoders and LLMs on Bengali text, and for counting or segmenting such text.
+
+## Benchmarks
+
+Here are 10,000 naturally written e-commerce reviews from [BanglishRev](https://huggingface.co/datasets/BanglishRev/bangla-english-and-code-mixed-ecommerce-review-dataset), in Bengali, English, Banglish and mixed script. This source was never seen in training. Every tokenizer runs **single-threaded** (`RAYON_NUM_THREADS=1`, `num_threads=1`), on an Apple M5 (macOS) and on an AMD EPYC 9V74 GitHub-hosted Linux x86_64 VM, Python 3.12. Speed is MB/s for single calls / batches of 64.
+
+| Tokenizer | Vocab | Tokens / word ↓ | Lossless round trip ↑ | Texts with UNK ↓ | MB/s, Apple M5 ↑ | MB/s, Linux x86_64 ↑ |
+|---|---:|---:|---:|---:|---:|---:|
+| **bn-bpe-58k (this package)** | 57,925 | **1.379** | **100%** | **0** | 58.5 / 71.6 | 24.4 / 29.7 |
+| **bn-bpe-28k (this package)** | 27,760 | 1.587 | **100%** | **0** | **61.6 / 72.9** | **26.1 / 33.6** |
+| SentencePiece Unigram, same data and size as 58k | 57,925 | 1.410 | 100% | 0 | 42.4 / 41.7 | 20.5 / 20.9 |
+| SentencePiece BPE, same data and size as 58k | 57,925 | 1.407 | 100% | 0 | 17.7 / 17.4 | 9.5 / 9.8 |
+| SentencePiece Unigram, same data and size as 28k | 27,760 | 1.572 | 100% | 0 | 46.4 / 43.2 | 22.8 / 23.4 |
+| SentencePiece BPE, same data and size as 28k | 27,760 | 1.578 | 100% | 0 | 16.6 / 16.4 | 9.8 / 9.8 |
+| BNLP `bn_spm` (SentencePiece) | 50,000 | 1.825 | 80.1% | 1,932 | 40.1 / 39.2 | 19.3 / 19.8 |
+| tiktoken `o200k_base` (GPT-4o) | 200,019 | 1.780 | 100% | 0 | 37.3 / 12.1 | 19.4 / 5.4 |
+| BanglaBERT (csebuetnlp) | 32,000 | 1.771 | 51.1% | 1,110 | 9.1 / 9.2 | 4.8 / 4.6 |
+| MuRIL (Google) | 197,258 | 1.498 | 47.9% | 1,617 | 8.8 / 8.9 | 4.6 / 4.4 |
+| XLM-R / multilingual-E5 / BGE-M3 | 250,002 | 1.740 | 98.2% | 109 | 6.5 / 6.4 | 3.7 / 3.5 |
+| mBERT cased | 119,547 | 2.122 | 48.0% | 1,706 | 8.1 / 8.1 | 4.2 / 4.1 |
+| Qwen3 | 151,669 | 3.588 | 100% | 0 | 6.0 / 6.0 | 3.3 / 3.1 |
+
+What this shows:
+
+- **Against SentencePiece with the same data and vocabulary size, compression is a tie.** `bn-bpe-58k` is about 2% shorter than both SentencePiece models; `bn-bpe-28k` is about 1% longer. On the in-domain validation split, SentencePiece at the 28k size is 3–4.5% shorter. The gap to BanglaBERT, XLM-R and tiktoken comes from the training data and the vocabulary size, not from a better algorithm.
+- **The speed lead is real but depends on the CPU.** Against SentencePiece Unigram it is 1.3–1.7× on the M5 and 1.1–1.4× on the Linux VM. Hugging Face `tokenizers` was limited to one thread like everything else; its batch API can use all cores, which this table does not measure.
+- BanglaBERT is about 3% shorter on Bengali-only text but lossy. Among lossless tokenizers, `bn-bpe-58k` is the most compact on Bengali, Latin and mixed script.
+
+[Full report](benchmarks/POPULAR_COMPARISON.md): per-script breakdown, the in-domain validation split, more tokenizers, method, limitations, and the commands to reproduce it. Shorter sequences cut compute and context use, but they do not by themselves make a downstream model more accurate.
+
+## Install
+
+```sh
+pip install banglatok
+```
+
+Prebuilt wheels target Linux (x86_64, aarch64), macOS (x86_64, arm64) and Windows (x86_64) for CPython 3.10–3.15, and need no compiler. On other platforms pip builds from source, which needs Go 1.24+ and a C compiler:
+
+```sh
+pip install git+https://github.com/iamaber/banglatok
+```
+
+## Usage
+
+```python
+from banglatok import Tokenizer
+
+with Tokenizer.from_pretrained("bn-bpe-58k") as tok:
+    ids = tok.encode("আমি বাংলায় গান গাই। ami banglay gaan gai")
+    tok.decode(ids)                       # 'আমি বাংলায় গান গাই। ami banglay gaan gai'
+    tok.encode_batch(["আমি ভালো আছি", "kemon acho?"])   # one native call
+    tok.count_tokens("অনেক লম্বা লেখা…")    # fast length, no list built
+
+    # Encoder-ready inputs: <s> … </s>, truncation and padding with <pad>.
+    batch = tok(["আমি ভালো আছি", "hi"], max_length=128, padding=True)
+    batch["input_ids"], batch["attention_mask"]
+```
+
+Load a tokenizer once and reuse it. Calls from many threads at once are safe, because the native call releases the GIL. In worker processes, create a fresh tokenizer inside each worker.
+
+### Hugging Face
+
+```python
+tok.save_huggingface("bn-bpe-58k-hf")      # tokenizer.json + tokenizer_config.json
+
+from transformers import AutoTokenizer
+hf = AutoTokenizer.from_pretrained("bn-bpe-58k-hf")   # same IDs as tok.encode / tok(...)
+```
+
+The exported models are also on the Hub as [`iamaber/banglatok-bn-bpe-58k`](https://huggingface.co/iamaber/banglatok-bn-bpe-58k) and [`iamaber/banglatok-bn-bpe-28k`](https://huggingface.co/iamaber/banglatok-bn-bpe-28k). Only BPE models can be exported. With the `tokenizers` library alone, set `encode_special_tokens = True` so that literal `<s>` in text stays text; `transformers` does this from the saved configuration.
+
+### Command line
+
+```sh
+$ banglatok "আমি বাংলা বলি"
+আমি |  বাংলা |  বলি
+$ echo "ami bangla boli" | banglatok --ids
+4405 33616 34749
+$ banglatok --json "ami valo"
+{"ids": [4405, 32047], "tokens": ["ami", " valo"]}
+$ banglatok --count --model bn-bpe-28k < lines.txt    # one count per line
+```
+
+### API
+
+| Member | Description |
+|---|---|
+| `Tokenizer.from_pretrained(name="bn-bpe-58k")` | Load a bundled model, offline |
+| `Tokenizer(path)` | Load your own JSON model, e.g. one trained with `bntok train` |
+| `encode(text)` / `encode_batch(texts)` | Token IDs, without control tokens |
+| `decode(ids, skip_special_tokens=False)` / `decode_batch(batch)` | Normalized text |
+| `tokenize(text)` | Token strings: fused IDs start with `" "`, fallback bytes look like `"<0xF0>"` |
+| `count_tokens(text)` | `len(encode(text))`, without building the list |
+| `save_huggingface(directory)` | Write a Hugging Face `tokenizer.json` with identical IDs (BPE models) |
+| `tok(texts, add_special_tokens=True, max_length=None, padding=False)` | `{"input_ids", "attention_mask"}` |
+| `id_to_token`, `token_to_id`, `get_vocab()`, `vocab_size`, `algorithm` | Vocabulary access |
+| `normalize(text)` | The shared normalization: NFC plus whitespace collapse and trim |
+| `available_models()`, `PAD_ID=0`, `UNK_ID=1`, `BOS_ID=2`, `EOS_ID=3` | Constants |
+
+## Models
+
+| Name | Vocab | Best for |
+|---|---:|---|
+| `bn-bpe-58k` (default) | 57,925 | Shortest sequences |
+| `bn-bpe-28k` | 27,760 | Smaller embedding tables, same speed |
+
+Both are BPE vocabularies with *space fusion*: a space and the token after it can share a single ID. That removes most standalone space tokens without overloading a literal marker such as `▁`. They were learned from 563k records of Bengali Wikipedia, Tatoeba and BanglaTLit. [docs/MODELS.md](docs/MODELS.md) covers the build commands, checksums, behavior and limitations. They are not SentencePiece or Hugging Face vocabularies, so pairing one with a pretrained encoder means adapting that encoder's embeddings.
+
+## Train your own
+
+The Go CLI trains Unigram or BPE models on your JSONL corpus, and `Tokenizer(path)` loads the result:
+
+```sh
+go run ./cmd/bntok train -algorithm bpe -vocab 384 \
+  -input examples/training-records.jsonl -out models/toy-bpe.json
+go run ./cmd/bntok encode -model models/toy-bpe.json -text 'আমি বাংলা বলি। ami bangla boli'
+python benchmarks/build_fused_model.py --base models/my-bpe.json --budget 64000 \
+  --output models/my-fused.json        # optional: add space fusion
+```
+
+See [docs/TRAINING_AND_RESEARCH.md](docs/TRAINING_AND_RESEARCH.md) for the corpus format, the full research pipeline and the model contract.
+
+## Research
+
+This repository also holds the research behind the models, under [research/](research/). [data/CORPUS.md](data/CORPUS.md) documents the corpus construction, provenance and leakage controls. [research/reports/RESULTS.md](research/reports/RESULTS.md) has the intrinsic evaluation. The retrieval experiments are in [research/experiments/retrieval/](research/experiments/retrieval/README.md), and [research/reports/ROUND2_ORIGINAL_COMPARISON.md](research/reports/ROUND2_ORIGINAL_COMPARISON.md) has the encoder-adaptation results. In those experiments, swapping a custom vocabulary into multilingual-E5 did not beat the unchanged model. Treat these vocabularies as a strong starting point for training, not as a drop-in accuracy gain for existing encoders.
+
+## Contributing
+
+[CONTRIBUTING.md](CONTRIBUTING.md) explains how to work on the project, [SECURITY.md](SECURITY.md) how to report vulnerabilities, and [docs/RELEASING.md](docs/RELEASING.md) how releases are made. Platform reports, regression cases and independent benchmark runs are especially welcome.
+
+## Citation
+
+If you use this library or its models, please cite it with the metadata in [CITATION.cff](CITATION.cff).
+
+## License
+
+- Code, documentation and the demo vocabulary: [MIT](LICENSE).
+- Bundled model files (`bn-bpe-58k`, `bn-bpe-28k`): [CC BY-SA 4.0](MODEL_LICENSE.md), because they are learned from Wikipedia and Tatoeba text.
+- Native wheels include BSD-3-Clause Go components ([notices](THIRD_PARTY_NOTICES.md)).
+
+The SPDX expression is `MIT AND BSD-3-Clause AND CC-BY-SA-4.0`. Research corpora keep their own terms and are not distributed.

banglatok-0.2.0/SECURITY.md

@@ -0,0 +1,9 @@
+# Security reporting
+
+Report vulnerabilities privately to **aber.islam.dev@gmail.com**, or use the repository's private vulnerability reporting feature if enabled. Include the affected version, OS/architecture, a minimal reproduction and expected impact. Do not post credentials, private corpus text or exploit details in a public issue before coordination.
+
+This project is in beta. Security fixes target current development and the latest published release; there is no supported older-release series yet. Prefer maintained Python and Go versions for deployment and source builds.
+
+The native library runs inside the Python process. Report native crashes, memory-safety problems, unsafe archive/build behavior and unexpected network activity. Model loading uses JSON; the package does not automatically fetch models or execute model-supplied Python code.
+
+Contributors should keep build/publish jobs separate, retain least-privilege workflow permissions and preserve third-party notices. Only the protected publishing job receives an OIDC token permission; ordinary pushes and pull requests do not publish packages.

banglatok-0.2.0/THIRD_PARTY_NOTICES.md

@@ -0,0 +1,25 @@
+# Third-party notices and scope
+
+Original software, documentation and the hand-authored demo vocabulary are
+MIT-licensed; see LICENSE. Native wheels also contain Go runtime/standard-library
+code and golang.org/x/text v0.28.0, each under BSD-3-Clause. Their notices are
+included in third_party/licenses and in the distribution's license directory.
+The bundled pretrained vocabularies in src/banglatok/pretrained
+are learned from Wikipedia, Tatoeba and BanglaTLit text and are distributed under
+CC BY-SA 4.0 with attribution; see MODEL_LICENSE.md. The distribution's SPDX
+expression is MIT AND BSD-3-Clause AND CC-BY-SA-4.0 to reflect all three.
+
+- Go: https://go.dev/ — third_party/licenses/GO-LICENSE
+- golang.org/x/text: https://pkg.go.dev/golang.org/x/text —
+  third_party/licenses/X-TEXT-LICENSE and X-TEXT-PATENTS
+
+The Go corpus-preparation CLI additionally depends on modules listed in go.mod
+and go.sum. Those modules are fetched separately when building the CLI; the
+Python wheel's native bridge imports only the tokenizer and its Unicode dependency.
+Dependencies retain their own licenses.
+
+The software license does not grant rights to third-party corpus text, source
+snapshots or external pretrained models. data/CORPUS.md and the research reports
+record source-specific terms and limitations. Raw corpora and research checkpoints
+are excluded from Python distributions. Do not treat their presence in a local
+research checkout as permission to redistribute them.

banglatok-0.2.0/cmd/bntok-shared/main.go

@@ -0,0 +1,68 @@
+// Package main exposes the Go tokenizer through owned C buffers.
+package main
+
+/*
+#include <stdint.h>
+#include <stdlib.h>
+*/
+import "C"
+
+import (
+	"math"
+	"unsafe"
+
+	"github.com/iamaber/banglatok/internal/native"
+)
+
+var registry native.Registry
+
+// Status codes returned by bntok_encode_into in place of an ID count.
+const (
+	encodeClosed  = -1
+	encodeInvalid = -2
+)
+
+// bntok_encode_into encodes one UTF-8 text into a caller-owned uint32 buffer.
+// It returns the ID count; a count above capacity means retry with that size.
+// Input and output memory are only used during the call.
+//
+//export bntok_encode_into
+func bntok_encode_into(handle C.ulonglong, data *C.char, length C.size_t, out *C.uint32_t, capacity C.size_t) C.longlong {
+	if uint64(length) > math.MaxInt32 || uint64(capacity) > math.MaxInt32 || (data == nil && length != 0) || (out == nil && capacity != 0) {
+		return encodeInvalid
+	}
+	text := unsafe.String((*byte)(unsafe.Pointer(data)), int(length))
+	ids := unsafe.Slice((*uint32)(unsafe.Pointer(out)), int(capacity))
+	n, err := registry.EncodeInto(uint64(handle), text, ids)
+	if err != nil {
+		return encodeClosed
+	}
+	return C.longlong(n)
+}
+
+//export bntok_encode
+func bntok_encode(handle C.ulonglong, data *C.char, length C.size_t, outputLength *C.size_t) *C.char {
+	if outputLength == nil {
+		return nil
+	}
+	var response []byte
+	if uint64(length) > math.MaxInt32 || (data == nil && length != 0) {
+		response = native.EncodeError("invalid encode input size")
+	} else {
+		response = registry.EncodeBinary(uint64(handle), unsafe.Slice((*byte)(unsafe.Pointer(data)), int(length)))
+	}
+	*outputLength = C.size_t(len(response))
+	return (*C.char)(C.CBytes(response))
+}
+
+//export bntok_call
+func bntok_call(request *C.char) *C.char {
+	return C.CString(string(registry.Call([]byte(C.GoString(request)))))
+}
+
+//export bntok_free
+func bntok_free(response *C.char) {
+	C.free(unsafe.Pointer(response))
+}
+
+func main() {}