laghu 0.1.0__tar.gz

laghu-0.1.0/.gitignore

@@ -0,0 +1,158 @@
+_docs/
+_proc/
+
+*.bak
+.gitattributes
+.last_checked
+.gitconfig
+*.bak
+*.log
+*~
+~*
+_tmp*
+tmp*
+tags
+*.pkg
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+conda/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+.vscode
+*.swp
+
+# osx generated files
+.DS_Store
+.DS_Store?
+.Trashes
+ehthumbs.db
+Thumbs.db
+.idea
+
+# pytest
+.pytest_cache
+
+# tools/trust-doc-nbs
+docs_src/.last_checked
+
+# symlinks to fastai
+docs_src/fastai
+tools/fastai
+
+# link checker
+checklink/cookies.txt
+
+# .gitconfig is now autogenerated
+.gitconfig
+
+# Quarto installer
+.deb
+.pkg
+
+# Quarto
+.quarto
+
+.venv/
+.kosha/
+_docs/
+_proc/
+nbs/_fixtures/

laghu-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Karthik
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

laghu-0.1.0/PKG-INFO

@@ -0,0 +1,142 @@
+Metadata-Version: 2.5
+Name: laghu
+Version: 0.1.0
+Summary: model2vec static models in a fraction of the memory, with the same vectors
+Project-URL: Repository, https://github.com/vedicreader/laghu
+Project-URL: Documentation, https://vedicreader.github.io/laghu/
+Author-email: Karthik <karthik.rajgopal@hotmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: embeddings,mmap,model2vec,nbdev,static-embeddings,tokenizers,unigram
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: fastcore>=1.8
+Requires-Dist: huggingface-hub>=0.24
+Requires-Dist: numpy>=1.24
+Requires-Dist: tokenizers>=0.20
+Provides-Extra: fallback
+Requires-Dist: model2vec>=0.9; extra == 'fallback'
+Description-Content-Type: text/markdown
+
+# laghu
+
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
+
+laghu (Sanskrit for light) loads a [model2vec](https://github.com/MinishLab/model2vec) static embedding model with `encode`, `encode_as_sequence` and `tokenize`. The output matches model2vec’s bits, apart from the empty-input difference documented below.
+
+model2vec reads the embedding table into each process. For potion-multilingual-128M, the table takes 512 MB. Its Unigram tokenizer adds a character trie over 500,353 pieces, about another 610 MB. After encoding, the process uses about 1.3 GB of private memory.
+
+laghu maps the table read-only from `model.safetensors`. Processes share the mapped pages through the operating system’s page cache. The operating system can reclaim clean pages under memory pressure. For supported Unigram models, laghu replaces the trie with Python Viterbi over a dict of pieces. The normalizer and pre-tokenizer come from `tokenizers`, built from the same tokenizer.json with the vocabulary removed. The same model then takes about 135 MB of private memory per process.
+
+WordPiece, BPE and word-level models still use `tokenizers`. laghu maps their embedding tables but does not replace their tokenizers. The memory saving depends on the table’s size relative to the tokenizer.
+
+## Install
+
+``` sh
+uv add laghu        # or: pip install laghu
+```
+
+model2vec itself is not a dependency. For the fallback described below, and for [`same_as_model2vec`](https://vedicreader.github.io/laghu/encoder.html#same_as_model2vec), install the extra:
+`uv add 'laghu[fallback]'`.
+
+## Use
+
+``` python
+from laghu import load
+
+m = load('minishlab/potion-multilingual-128M')      # a folder, or a Hugging Face repo id (the local cache is tried first)
+v = m.encode(['The quick brown fox.', 'कर्मण्येवाधिकारस्ते'])   # float32, (2, 256), unit rows: the same array model2vec returns
+m.tokenize(['hello world'])
+m.encode_as_sequence('hello world').shape          # (n_tokens, 256)
+```
+
+    (2, 256)
+
+[`load`](https://vedicreader.github.io/laghu/encoder.html#load) returns a [`LeanModel`](https://vedicreader.github.io/laghu/encoder.html#leanmodel). If loading fails, it logs the cause and tries model2vec’s `StaticModel`. This fallback needs `laghu[fallback]`. If the fallback also fails, [`load`](https://vedicreader.github.io/laghu/encoder.html#load) raises the original laghu error. `load(..., fallback=False)` raises without trying model2vec. Both loaders refuse bfloat16 and float8 tables because numpy cannot hold those dtypes.
+
+`same_as_model2vec(path)` loads a model both ways and compares them on the built-in corpus, or on texts of your own:
+
+``` python
+from laghu import same_as_model2vec
+same_as_model2vec('minishlab/potion-base-8M', ['my', 'own texts'])
+```
+
+    /Users/71293/code/personal/orgs/laghu/.venv/lib/python3.13/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
+      from .autonotebook import tqdm as notebook_tqdm
+
+    {'lean': False,
+     'n': 2,
+     'ids': True,
+     'dtype': True,
+     'f32': True,
+     'f16': True,
+     'exact': True,
+     'seq': True,
+     'median': True,
+     'unk': True,
+     'tokens': True}
+
+It checks the token ids, the vectors cast to float32 and to float16, the exact dtype and bits, `encode_as_sequence`,
+`median_token_length`, the unknown token id and the token list.
+
+## What matches, and what it costs
+
+Every model below matches model2vec’s ids and vectors, bit for bit, on 4,726 texts. The corpus includes prose in 36 languages and random strings from 30 Unicode blocks. It also tests control characters, special tokens, emoji sequences, combining marks, bidi controls and a 200,000-character string. Each memory measurement uses a fresh process. These macOS measurements use `phys_footprint` after encoding and `lifetime_max_phys_footprint` for peak. On Linux, the helper reports `RssAnon` and peak resident memory (`VmHWM`) instead. Speed is `encode` over 3,226 texts on an M3 Max.
+
+| model | tokenizer | laghu tokenizes with | vocab × dims | dtype | norm | identical | private MB, model2vec → laghu | peak MB, model2vec → laghu | texts/s, model2vec → laghu |
+|----|----|----|----|----|----|----|----|----|----|
+| potion-base-2M | WordPiece | tokenizers | 29,528 × 64 | float32 | on | yes | 81 → 58 | 82 → 58 | 84,510 → 92,982 |
+| potion-base-4M | WordPiece | tokenizers | 29,528 × 128 | float32 | on | yes | 94 → 62 | 95 → 62 | 87,248 → 90,242 |
+| potion-base-8M | WordPiece | tokenizers | 29,528 × 256 | float32 | on | yes | 114 → 70 | 115 → 70 | 79,415 → 81,810 |
+| potion-base-32M | WordPiece | tokenizers | 63,091 × 512 | float32 | on | yes | 244 → 97 | 244 → 97 | 75,009 → 81,379 |
+| potion-retrieval-32M | WordPiece | tokenizers | 63,091 × 512 | float32 | on | yes | 244 → 97 | 244 → 97 | 70,799 → 79,717 |
+| potion-science-8M | WordPiece | tokenizers | 29,528 × 256 | float32 | on | yes | 113 → 70 | 114 → 70 | 81,008 → 84,532 |
+| potion-code-16M | WordPiece | tokenizers | 61,826 × 256 | float32 +map +weights | on | yes | 147 → 110 | 211 → 117 | 10,674 → 10,425 |
+| potion-code-16M-v2 | WordPiece | tokenizers | 63,457 × 256 | float16 | on | yes | 108 → 101 | 186 → 119 | 21,819 → 22,164 |
+| potion-multilingual-128M | Unigram | Python Viterbi | 500,353 × 256 | float32 | on | yes | 1,348 → 135 | 1,348 → 135 | 39,105 → 12,795 |
+| M2V_base_output | WordPiece | tokenizers | 29,528 × 256 | float32 | off | yes | 112 → 69 | 113 → 69 | 65,460 → 87,100 |
+| M2V_multilingual_output | WordPiece | tokenizers | 501,054 × 256 | float32 | off | yes | 834 → 198 | 842 → 295 | 42,565 → 90,283 |
+| M2V_base_glove | WordLevel | tokenizers | 400,002 × 256 | float32 | off | yes | 656 → 186 | 656 → 194 | 67,028 → 85,058 |
+| M2V_base_glove_subword | WordPiece | tokenizers | 400,779 × 256 | float32 | off | yes | 647 → 178 | 649 → 195 | 66,256 → 85,430 |
+| jina-embeddings-v3-separation-distilled | Unigram | Python Viterbi | 250,002 × 256 | float32 | off | yes | 722 → 88 | 722 → 88 | 51,836 → 20,840 |
+| multilingual-e5-large-m2v | Unigram | Python Viterbi | 250,002 × 512 | float32 | on | yes | 981 → 109 | 981 → 109 | 24,139 → 19,670 |
+| m2v-gemma-embedding-300m | Unigram | Python Viterbi | 255,732 × 256 | float16 +weights | on | yes | 542 → 93 | 542 → 93 | 20,319 → 12,134 |
+| potion-multilingual-128M-i8-tokenade | Unigram | Python Viterbi | 500,353 × 256 | int8 | on | yes | 985 → 138 | 985 → 138 | 27,676 → 12,090 |
+| m2v-qwen3-embedding-0.6b-1024d | Unigram | Python Viterbi | 151,644 × 1024 | float16 +weights | on | yes | 602 → 132 | 602 → 132 | 16,792 → 7,168 |
+| ovos-m2v-intents-es-ES-bne-v5 | Unigram | Python Viterbi | 50,259 × 64 | float32 | on | yes | 169 → 52 | 169 → 52 | 11,786 → 11,265 |
+| m2v-gte-256-edu | Unigram | tokenizers | 313,845 × 256 | float32 | on | yes | 1,028 → 663 | 1,028 → 675 | 25,363 → 24,981 |
+| bge-m3-m2v-256 | Unigram | Python Viterbi | 250,002 × 256 | float32 | off | yes | 725 → 89 | 725 → 89 | 34,518 → 20,670 |
+| potion-mxbai-micro | WordPiece | tokenizers | 2,000 × 256 | int8 +map +weights | on | yes | 90 → 72 | 91 → 72 | 68,385 → 70,998 |
+| NIFE-mxbai-embed-large-v1_model2vec | WordPiece | tokenizers | 107,962 × 1024 | float32 | off | yes | 558 → 106 | 572 → 116 | 41,042 → 26,756 |
+| ovos-m2v-intents-pt-PT-albertina-v5 | BPE | tokenizers | 50,262 × 256 | float32 | on | yes | 162 → 90 | 162 → 90 | 46,343 → 22,813 |
+| F2LLM-v2-330M-model2vec | BPE | tokenizers | 151,644 × 512 | float16 | on | yes | 356 → 171 | 356 → 171 | 28,795 → 40,132 |
+| static-retrieval-multilingual-69m-v1 | BPE | tokenizers | 179,936 × 384 | float32 | off | yes | 537 → 229 | 541 → 263 | 32,235 → 31,848 |
+
+Unigram throughput ranges from about a third of model2vec’s rate to about the same. `tokenizers` parallelizes batches across cores. Python Viterbi runs on one core. These batch measurements do not establish latency for one query.
+
+The word cache helps when the pre-tokenizer splits words, as in jina-v3, bge-m3 and multilingual-e5. Natural text runs 1.6 to 1.9 times faster on a first pass and 3 to 5 times faster on repeated text. The `bench` notebook records these measurements and the reasons for rejecting tokie and gigatoken.
+
+Other models use the same `tokenizers` code in both loaders. The timing samples last about 0.1 seconds and vary by up to half on repeated runs. They do not establish a speed advantage.
+
+## What is not supported
+
+- Quantizing or truncating on load (`quantize_to`, `dimensionality`, `vocabulary_quantization`): each would copy the table, which is
+  what laghu exists to avoid. Load with model2vec for those.
+- `single_word` added tokens on a Unigram model fall back to `tokenizers` for tokenizing (the table is still mapped). Rust decides a
+  word boundary by Unicode’s Alphabetic property, which Python’s `str.isalpha` does not match.
+- `use_multiprocessing` is accepted and ignored; the result is the same.
+- `encode([])` returns an empty `(0, dim)` array. model2vec raises.
+
+## Developing
+
+nbdev 3 on hatchling: the notebooks in `nbs/` are the source. `uv run nbdev-prepare` exports, tests and rebuilds this README;
+`uv run nbdev-test --flags slow` also runs the bench, which downloads about 5 GB of models.
+
+## Licence
+
+MIT.

laghu-0.1.0/README.md

@@ -0,0 +1,118 @@
+# laghu
+
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
+
+laghu (Sanskrit for light) loads a [model2vec](https://github.com/MinishLab/model2vec) static embedding model with `encode`, `encode_as_sequence` and `tokenize`. The output matches model2vec’s bits, apart from the empty-input difference documented below.
+
+model2vec reads the embedding table into each process. For potion-multilingual-128M, the table takes 512 MB. Its Unigram tokenizer adds a character trie over 500,353 pieces, about another 610 MB. After encoding, the process uses about 1.3 GB of private memory.
+
+laghu maps the table read-only from `model.safetensors`. Processes share the mapped pages through the operating system’s page cache. The operating system can reclaim clean pages under memory pressure. For supported Unigram models, laghu replaces the trie with Python Viterbi over a dict of pieces. The normalizer and pre-tokenizer come from `tokenizers`, built from the same tokenizer.json with the vocabulary removed. The same model then takes about 135 MB of private memory per process.
+
+WordPiece, BPE and word-level models still use `tokenizers`. laghu maps their embedding tables but does not replace their tokenizers. The memory saving depends on the table’s size relative to the tokenizer.
+
+## Install
+
+``` sh
+uv add laghu        # or: pip install laghu
+```
+
+model2vec itself is not a dependency. For the fallback described below, and for [`same_as_model2vec`](https://vedicreader.github.io/laghu/encoder.html#same_as_model2vec), install the extra:
+`uv add 'laghu[fallback]'`.
+
+## Use
+
+``` python
+from laghu import load
+
+m = load('minishlab/potion-multilingual-128M')      # a folder, or a Hugging Face repo id (the local cache is tried first)
+v = m.encode(['The quick brown fox.', 'कर्मण्येवाधिकारस्ते'])   # float32, (2, 256), unit rows: the same array model2vec returns
+m.tokenize(['hello world'])
+m.encode_as_sequence('hello world').shape          # (n_tokens, 256)
+```
+
+    (2, 256)
+
+[`load`](https://vedicreader.github.io/laghu/encoder.html#load) returns a [`LeanModel`](https://vedicreader.github.io/laghu/encoder.html#leanmodel). If loading fails, it logs the cause and tries model2vec’s `StaticModel`. This fallback needs `laghu[fallback]`. If the fallback also fails, [`load`](https://vedicreader.github.io/laghu/encoder.html#load) raises the original laghu error. `load(..., fallback=False)` raises without trying model2vec. Both loaders refuse bfloat16 and float8 tables because numpy cannot hold those dtypes.
+
+`same_as_model2vec(path)` loads a model both ways and compares them on the built-in corpus, or on texts of your own:
+
+``` python
+from laghu import same_as_model2vec
+same_as_model2vec('minishlab/potion-base-8M', ['my', 'own texts'])
+```
+
+    /Users/71293/code/personal/orgs/laghu/.venv/lib/python3.13/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
+      from .autonotebook import tqdm as notebook_tqdm
+
+    {'lean': False,
+     'n': 2,
+     'ids': True,
+     'dtype': True,
+     'f32': True,
+     'f16': True,
+     'exact': True,
+     'seq': True,
+     'median': True,
+     'unk': True,
+     'tokens': True}
+
+It checks the token ids, the vectors cast to float32 and to float16, the exact dtype and bits, `encode_as_sequence`,
+`median_token_length`, the unknown token id and the token list.
+
+## What matches, and what it costs
+
+Every model below matches model2vec’s ids and vectors, bit for bit, on 4,726 texts. The corpus includes prose in 36 languages and random strings from 30 Unicode blocks. It also tests control characters, special tokens, emoji sequences, combining marks, bidi controls and a 200,000-character string. Each memory measurement uses a fresh process. These macOS measurements use `phys_footprint` after encoding and `lifetime_max_phys_footprint` for peak. On Linux, the helper reports `RssAnon` and peak resident memory (`VmHWM`) instead. Speed is `encode` over 3,226 texts on an M3 Max.
+
+| model | tokenizer | laghu tokenizes with | vocab × dims | dtype | norm | identical | private MB, model2vec → laghu | peak MB, model2vec → laghu | texts/s, model2vec → laghu |
+|----|----|----|----|----|----|----|----|----|----|
+| potion-base-2M | WordPiece | tokenizers | 29,528 × 64 | float32 | on | yes | 81 → 58 | 82 → 58 | 84,510 → 92,982 |
+| potion-base-4M | WordPiece | tokenizers | 29,528 × 128 | float32 | on | yes | 94 → 62 | 95 → 62 | 87,248 → 90,242 |
+| potion-base-8M | WordPiece | tokenizers | 29,528 × 256 | float32 | on | yes | 114 → 70 | 115 → 70 | 79,415 → 81,810 |
+| potion-base-32M | WordPiece | tokenizers | 63,091 × 512 | float32 | on | yes | 244 → 97 | 244 → 97 | 75,009 → 81,379 |
+| potion-retrieval-32M | WordPiece | tokenizers | 63,091 × 512 | float32 | on | yes | 244 → 97 | 244 → 97 | 70,799 → 79,717 |
+| potion-science-8M | WordPiece | tokenizers | 29,528 × 256 | float32 | on | yes | 113 → 70 | 114 → 70 | 81,008 → 84,532 |
+| potion-code-16M | WordPiece | tokenizers | 61,826 × 256 | float32 +map +weights | on | yes | 147 → 110 | 211 → 117 | 10,674 → 10,425 |
+| potion-code-16M-v2 | WordPiece | tokenizers | 63,457 × 256 | float16 | on | yes | 108 → 101 | 186 → 119 | 21,819 → 22,164 |
+| potion-multilingual-128M | Unigram | Python Viterbi | 500,353 × 256 | float32 | on | yes | 1,348 → 135 | 1,348 → 135 | 39,105 → 12,795 |
+| M2V_base_output | WordPiece | tokenizers | 29,528 × 256 | float32 | off | yes | 112 → 69 | 113 → 69 | 65,460 → 87,100 |
+| M2V_multilingual_output | WordPiece | tokenizers | 501,054 × 256 | float32 | off | yes | 834 → 198 | 842 → 295 | 42,565 → 90,283 |
+| M2V_base_glove | WordLevel | tokenizers | 400,002 × 256 | float32 | off | yes | 656 → 186 | 656 → 194 | 67,028 → 85,058 |
+| M2V_base_glove_subword | WordPiece | tokenizers | 400,779 × 256 | float32 | off | yes | 647 → 178 | 649 → 195 | 66,256 → 85,430 |
+| jina-embeddings-v3-separation-distilled | Unigram | Python Viterbi | 250,002 × 256 | float32 | off | yes | 722 → 88 | 722 → 88 | 51,836 → 20,840 |
+| multilingual-e5-large-m2v | Unigram | Python Viterbi | 250,002 × 512 | float32 | on | yes | 981 → 109 | 981 → 109 | 24,139 → 19,670 |
+| m2v-gemma-embedding-300m | Unigram | Python Viterbi | 255,732 × 256 | float16 +weights | on | yes | 542 → 93 | 542 → 93 | 20,319 → 12,134 |
+| potion-multilingual-128M-i8-tokenade | Unigram | Python Viterbi | 500,353 × 256 | int8 | on | yes | 985 → 138 | 985 → 138 | 27,676 → 12,090 |
+| m2v-qwen3-embedding-0.6b-1024d | Unigram | Python Viterbi | 151,644 × 1024 | float16 +weights | on | yes | 602 → 132 | 602 → 132 | 16,792 → 7,168 |
+| ovos-m2v-intents-es-ES-bne-v5 | Unigram | Python Viterbi | 50,259 × 64 | float32 | on | yes | 169 → 52 | 169 → 52 | 11,786 → 11,265 |
+| m2v-gte-256-edu | Unigram | tokenizers | 313,845 × 256 | float32 | on | yes | 1,028 → 663 | 1,028 → 675 | 25,363 → 24,981 |
+| bge-m3-m2v-256 | Unigram | Python Viterbi | 250,002 × 256 | float32 | off | yes | 725 → 89 | 725 → 89 | 34,518 → 20,670 |
+| potion-mxbai-micro | WordPiece | tokenizers | 2,000 × 256 | int8 +map +weights | on | yes | 90 → 72 | 91 → 72 | 68,385 → 70,998 |
+| NIFE-mxbai-embed-large-v1_model2vec | WordPiece | tokenizers | 107,962 × 1024 | float32 | off | yes | 558 → 106 | 572 → 116 | 41,042 → 26,756 |
+| ovos-m2v-intents-pt-PT-albertina-v5 | BPE | tokenizers | 50,262 × 256 | float32 | on | yes | 162 → 90 | 162 → 90 | 46,343 → 22,813 |
+| F2LLM-v2-330M-model2vec | BPE | tokenizers | 151,644 × 512 | float16 | on | yes | 356 → 171 | 356 → 171 | 28,795 → 40,132 |
+| static-retrieval-multilingual-69m-v1 | BPE | tokenizers | 179,936 × 384 | float32 | off | yes | 537 → 229 | 541 → 263 | 32,235 → 31,848 |
+
+Unigram throughput ranges from about a third of model2vec’s rate to about the same. `tokenizers` parallelizes batches across cores. Python Viterbi runs on one core. These batch measurements do not establish latency for one query.
+
+The word cache helps when the pre-tokenizer splits words, as in jina-v3, bge-m3 and multilingual-e5. Natural text runs 1.6 to 1.9 times faster on a first pass and 3 to 5 times faster on repeated text. The `bench` notebook records these measurements and the reasons for rejecting tokie and gigatoken.
+
+Other models use the same `tokenizers` code in both loaders. The timing samples last about 0.1 seconds and vary by up to half on repeated runs. They do not establish a speed advantage.
+
+## What is not supported
+
+- Quantizing or truncating on load (`quantize_to`, `dimensionality`, `vocabulary_quantization`): each would copy the table, which is
+  what laghu exists to avoid. Load with model2vec for those.
+- `single_word` added tokens on a Unigram model fall back to `tokenizers` for tokenizing (the table is still mapped). Rust decides a
+  word boundary by Unicode’s Alphabetic property, which Python’s `str.isalpha` does not match.
+- `use_multiprocessing` is accepted and ignored; the result is the same.
+- `encode([])` returns an empty `(0, dim)` array. model2vec raises.
+
+## Developing
+
+nbdev 3 on hatchling: the notebooks in `nbs/` are the source. `uv run nbdev-prepare` exports, tests and rebuilds this README;
+`uv run nbdev-test --flags slow` also runs the bench, which downloads about 5 GB of models.
+
+## Licence
+
+MIT.

laghu-0.1.0/laghu/__init__.py

@@ -0,0 +1,3 @@
+__version__ = "0.1.0"
+from .core import *
+from .encoder import LeanModel, load, same_as_model2vec

laghu-0.1.0/laghu/_modidx.py

@@ -0,0 +1,68 @@
+# Autogenerated by nbdev
+
+d = { 'settings': { 'branch': 'main',
+                'doc_baseurl': '/laghu',
+                'doc_host': 'https://vedicreader.github.io',
+                'git_url': 'https://github.com/vedicreader/laghu',
+                'lib_path': 'laghu'},
+  'syms': { 'laghu.bench': { 'laghu.bench._run': ('bench.html#_run', 'laghu/bench.py'),
+                             'laghu.bench.check': ('bench.html#check', 'laghu/bench.py'),
+                             'laghu.bench.footprint': ('bench.html#footprint', 'laghu/bench.py'),
+                             'laghu.bench.measure': ('bench.html#measure', 'laghu/bench.py'),
+                             'laghu.bench.row': ('bench.html#row', 'laghu/bench.py'),
+                             'laghu.bench.same': ('bench.html#same', 'laghu/bench.py'),
+                             'laghu.bench.table': ('bench.html#table', 'laghu/bench.py')},
+            'laghu.core': { 'laghu.core.UnsupportedDtype': ('core.html#unsupporteddtype', 'laghu/core.py'),
+                            'laghu.core.cached': ('core.html#cached', 'laghu/core.py'),
+                            'laghu.core.corpus': ('core.html#corpus', 'laghu/core.py'),
+                            'laghu.core.find_vocab': ('core.html#find_vocab', 'laghu/core.py'),
+                            'laghu.core.is_unigram': ('core.html#is_unigram', 'laghu/core.py'),
+                            'laghu.core.layout': ('core.html#layout', 'laghu/core.py'),
+                            'laghu.core.model_files': ('core.html#model_files', 'laghu/core.py'),
+                            'laghu.core.noise': ('core.html#noise', 'laghu/core.py'),
+                            'laghu.core.resolve': ('core.html#resolve', 'laghu/core.py'),
+                            'laghu.core.st_header': ('core.html#st_header', 'laghu/core.py'),
+                            'laghu.core.st_map': ('core.html#st_map', 'laghu/core.py'),
+                            'laghu.core.st_write': ('core.html#st_write', 'laghu/core.py'),
+                            'laghu.core.try_resolve': ('core.html#try_resolve', 'laghu/core.py')},
+            'laghu.encoder': { 'laghu.encoder.LeanModel': ('encoder.html#leanmodel', 'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel.__init__': ('encoder.html#leanmodel.__init__', 'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel.__repr__': ('encoder.html#leanmodel.__repr__', 'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel._encode_batch': ('encoder.html#leanmodel._encode_batch', 'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel._rows': ('encoder.html#leanmodel._rows', 'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel.dim': ('encoder.html#leanmodel.dim', 'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel.embedding_dtype': ('encoder.html#leanmodel.embedding_dtype', 'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel.encode': ('encoder.html#leanmodel.encode', 'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel.encode_as_sequence': ( 'encoder.html#leanmodel.encode_as_sequence',
+                                                                               'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel.from_pretrained': ('encoder.html#leanmodel.from_pretrained', 'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel.lean': ('encoder.html#leanmodel.lean', 'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel.tokenize': ('encoder.html#leanmodel.tokenize', 'laghu/encoder.py'),
+                               'laghu.encoder.LeanModel.tokens': ('encoder.html#leanmodel.tokens', 'laghu/encoder.py'),
+                               'laghu.encoder._batches': ('encoder.html#_batches', 'laghu/encoder.py'),
+                               'laghu.encoder.load': ('encoder.html#load', 'laghu/encoder.py'),
+                               'laghu.encoder.same_as_model2vec': ('encoder.html#same_as_model2vec', 'laghu/encoder.py')},
+            'laghu.unigram': { 'laghu.unigram.Added': ('unigram.html#added', 'laghu/unigram.py'),
+                               'laghu.unigram.Added.__init__': ('unigram.html#added.__init__', 'laghu/unigram.py'),
+                               'laghu.unigram.Added.split': ('unigram.html#added.split', 'laghu/unigram.py'),
+                               'laghu.unigram.HFTok': ('unigram.html#hftok', 'laghu/unigram.py'),
+                               'laghu.unigram.HFTok.__call__': ('unigram.html#hftok.__call__', 'laghu/unigram.py'),
+                               'laghu.unigram.HFTok.__init__': ('unigram.html#hftok.__init__', 'laghu/unigram.py'),
+                               'laghu.unigram.HFTok.vocab': ('unigram.html#hftok.vocab', 'laghu/unigram.py'),
+                               'laghu.unigram.Unigram': ('unigram.html#unigram', 'laghu/unigram.py'),
+                               'laghu.unigram.Unigram.__call__': ('unigram.html#unigram.__call__', 'laghu/unigram.py'),
+                               'laghu.unigram.Unigram.__init__': ('unigram.html#unigram.__init__', 'laghu/unigram.py'),
+                               'laghu.unigram.Unigram._first': ('unigram.html#unigram._first', 'laghu/unigram.py'),
+                               'laghu.unigram.Unigram._pieces': ('unigram.html#unigram._pieces', 'laghu/unigram.py'),
+                               'laghu.unigram.Unigram._seg': ('unigram.html#unigram._seg', 'laghu/unigram.py'),
+                               'laghu.unigram.Unigram._viterbi': ('unigram.html#unigram._viterbi', 'laghu/unigram.py'),
+                               'laghu.unigram.Unigram.encode': ('unigram.html#unigram.encode', 'laghu/unigram.py'),
+                               'laghu.unigram.Unigram.unk_token_id': ('unigram.html#unigram.unk_token_id', 'laghu/unigram.py'),
+                               'laghu.unigram.Unigram.vocab': ('unigram.html#unigram.vocab', 'laghu/unigram.py'),
+                               'laghu.unigram.Unigram.word': ('unigram.html#unigram.word', 'laghu/unigram.py'),
+                               'laghu.unigram._metaspace': ('unigram.html#_metaspace', 'laghu/unigram.py'),
+                               'laghu.unigram._pipeline': ('unigram.html#_pipeline', 'laghu/unigram.py'),
+                               'laghu.unigram._unk': ('unigram.html#_unk', 'laghu/unigram.py'),
+                               'laghu.unigram.read_vocab': ('unigram.html#read_vocab', 'laghu/unigram.py'),
+                               'laghu.unigram.tokenizer': ('unigram.html#tokenizer', 'laghu/unigram.py'),
+                               'laghu.unigram.viterbi': ('unigram.html#viterbi', 'laghu/unigram.py')}}}

laghu-0.1.0/laghu/bench.py

@@ -0,0 +1,80 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03_bench.ipynb.
+
+# %% auto #0
+__all__ = ['footprint', 'measure', 'check', 'row', 'same', 'table']
+
+# %% ../nbs/03_bench.ipynb #630431fe
+import json, sys, os, ctypes, platform, subprocess
+from fastcore.all import AttrDict
+
+# %% ../nbs/03_bench.ipynb #ad5d5182
+def footprint():
+    "This process's resident, private and peak memory in MB."
+    if platform.system() == 'Darwin':
+        buf = (ctypes.c_uint64 * 64)()   # rusage_info_v4 as uint64s: a 16-byte uuid, then resident_size at 8, phys_footprint 9, lifetime_max_phys_footprint 30
+        ctypes.CDLL(None).proc_pid_rusage(ctypes.c_int(os.getpid()), 4, buf)
+        return AttrDict(rss=buf[8] / 2**20, private=buf[9] / 2**20, peak=buf[30] / 2**20)
+    kv = dict(l.split(':', 1) for l in open('/proc/self/status'))
+    mb = lambda k: int(kv[k].split()[0]) / 1024
+    return AttrDict(rss=mb('VmRSS'), private=mb('RssAnon'), peak=mb('VmHWM'))
+
+# %% ../nbs/03_bench.ipynb #b4cc8766
+_PROBE = '''
+import json, time
+from laghu.bench import footprint
+from laghu.core import corpus, resolve
+t = time.time()
+if {how!r} == 'model2vec':
+    from model2vec import StaticModel
+    m = StaticModel.from_pretrained(str(resolve({path!r})), force_download=False)
+else:
+    from laghu import load
+    m = load({path!r}, fallback=False)
+tl = time.time() - t; f1 = footprint()
+txts = corpus({n}); t = time.time(); m.encode(txts); te = time.time() - t; f2 = footprint()
+print(json.dumps(dict(how={how!r}, load_s=tl, encode_s=te, texts=len(txts), private_load=f1.private, private=f2.private, rss=f2.rss, peak=f2.peak)))
+'''
+
+_CHECK = '''
+import json
+from laghu import load, same_as_model2vec
+from laghu.core import corpus, model_files, resolve, find_vocab
+m = load({path!r}, fallback=False); f = model_files(m.folder)
+kind = json.loads(f.tok.read_text())['model'].get('type') or 'Unigram'
+r = same_as_model2vec({path!r}, corpus({n}))
+print(json.dumps(dict(r, kind=kind, vocab=len(m.embedding), dim=m.dim, dtype=m.embedding_dtype, normalize=bool(m.normalize),
+    quantized=m.token_mapping is not None, weighted=m.weights is not None)))
+'''
+
+def _run(code):
+    r = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True)
+    if r.returncode: raise RuntimeError(r.stderr[-3000:])
+    return AttrDict(json.loads(r.stdout.strip().splitlines()[-1]))
+
+def measure(path, how='laghu', n=2000):
+    "Memory (MB) and time (s) of loading `path` with `how` ('laghu' or 'model2vec') and encoding `corpus(n)`, in a new process."
+    return _run(_PROBE.format(path=str(path), how=how, n=n))
+
+def check(path, n=3000):
+    "`same_as_model2vec(path, corpus(n))` and the model's shape, in a new process."
+    return _run(_CHECK.format(path=str(path), n=n))
+
+def row(path, n_check=3000, n_speed=2000):
+    "One model's line: what it is, whether it matches, and both ways' memory and speed."
+    r = check(path, n_check); r.model = str(path)
+    r.m2v, r.lg = measure(path, 'model2vec', n_speed), measure(path, 'laghu', n_speed)
+    return r
+
+# %% ../nbs/03_bench.ipynb #33c0bdb7
+def same(r): return all(r[k] for k in ('ids', 'dtype', 'f32', 'f16', 'exact', 'seq', 'median', 'unk', 'tokens'))
+
+def table(rows):
+    "Rows of `row` as a markdown table."
+    hd = ('| model | tokenizer | laghu tokenizes with | vocab × dims | dtype | norm | identical | private MB, model2vec → laghu | peak MB, model2vec → laghu | texts/s, model2vec → laghu |\n'
+          '|---|---|---|---|---|---|---|---|---|---|\n')
+    def ln(r):
+        extra = ' +map' * r.quantized + ' +weights' * r.weighted
+        return (f"| {r.model.split('/')[-1]} | {r.kind} | {'Python Viterbi' if r.lean else 'tokenizers'} | {r.vocab:,} × {r.dim} | {r.dtype}{extra} | "
+                f"{'on' if r.normalize else 'off'} | {'yes' if same(r) else 'NO'} | {r.m2v.private:,.0f} → {r.lg.private:,.0f} | {r.m2v.peak:,.0f} → {r.lg.peak:,.0f} | "
+                f"{r.m2v.texts / r.m2v.encode_s:,.0f} → {r.lg.texts / r.lg.encode_s:,.0f} |")
+    return hd + '\n'.join(map(ln, rows))

laghu-0.1.0/laghu/core.py

@@ -0,0 +1,147 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
+
+# %% auto #0
+__all__ = ['log', 'LAYOUTS', 'ST_DTYPES', 'SENTENCES', 'ODD', 'cached', 'resolve', 'layout', 'model_files', 'UnsupportedDtype',
+           'st_header', 'st_map', 'st_write', 'find_vocab', 'is_unigram', 'try_resolve', 'noise', 'corpus']
+
+# %% ../nbs/00_core.ipynb #fd476c3c
+import json, re, mmap, logging, numpy as np
+from pathlib import Path
+from fastcore.all import AttrDict, ifnone
+
+log = logging.getLogger('laghu')
+
+# %% ../nbs/00_core.ipynb #9073650b
+def cached(repo):
+    "The newest cached snapshot of Hugging Face repo `repo`, or None."
+    if repo.count('/') != 1: return None
+    from huggingface_hub.constants import HF_HUB_CACHE
+    d = Path(HF_HUB_CACHE)/f"models--{repo.replace('/', '--')}"/'snapshots'
+    snaps = [o for o in d.iterdir() if o.is_dir()] if d.exists() else []
+    return max(snaps, key=lambda o: o.stat().st_mtime) if snaps else None
+
+def resolve(path, token=None, subfolder=None, force_download=False):
+    "The local folder for `path`: a folder as given, else a complete snapshot of repo id `path` from the cache, else downloaded."
+    p = Path(path)
+    if p.exists(): return p/subfolder if subfolder else p
+    p = None if force_download else cached(str(path))
+    if p is None or layout(p/subfolder if subfolder else p) is None:   # a snapshot holding only some files (one hf_hub_download) is not the model
+        from huggingface_hub import snapshot_download
+        p = Path(snapshot_download(Path(path).as_posix(), repo_type='model', token=token))
+    return p/subfolder if subfolder else p
+
+# %% ../nbs/00_core.ipynb #b57979e0
+def layout(folder):
+    "The config, table and tokenizer files of the first layout `folder` has, and the table's tensor name, or None."
+    folder = Path(folder)
+    for c, e, t, nm in LAYOUTS:
+        if all((folder/o).exists() for o in (c, e, t)): return AttrDict(cfg=folder/c, emb=folder/e, tok=folder/t, name=nm)
+
+LAYOUTS = (('config.json', 'model.safetensors', 'tokenizer.json', 'embeddings'),
+           ('config_sentence_transformers.json', 'model.safetensors', 'tokenizer.json', 'embedding.weight'),
+           ('config_sentence_transformers.json', '0_StaticEmbedding/model.safetensors', '0_StaticEmbedding/tokenizer.json', 'embedding.weight'))
+
+def model_files(folder):
+    "`layout(folder)`, raising when there is none."
+    if f := layout(folder): return f
+    raise FileNotFoundError(f'{folder}: no model2vec, sentence-transformers or 0_StaticEmbedding layout')
+
+# %% ../nbs/00_core.ipynb #86a0c03a
+ST_DTYPES = dict(F64=np.float64, F32=np.float32, F16=np.float16, I64=np.int64, I32=np.int32, I16=np.int16, I8=np.int8, U8=np.uint8)
+
+class UnsupportedDtype(ValueError): "A tensor in a dtype numpy cannot hold."
+
+def st_header(fn):
+    "Safetensors file `fn`'s header and the offset its data starts at."
+    with open(fn, 'rb') as f: n = int.from_bytes(f.read(8), 'little'); return json.loads(f.read(n)), 8 + n
+
+def st_map(fn, names):
+    "Tensors `names` of safetensors file `fn`, memory-mapped read-only; a name the file lacks is None."
+    hdr, at = st_header(fn)
+    def _m(k):
+        if k not in hdr: return None
+        t = hdr[k]; s, e = t['data_offsets']; dt = ST_DTYPES.get(t['dtype']); shape = tuple(t['shape'])
+        if dt is None: raise UnsupportedDtype(f"{fn}: tensor {k!r} is {t['dtype']}; laghu reads {', '.join(ST_DTYPES)}")
+        if e == s: return np.zeros(shape, dt)   # np.memmap refuses an empty map
+        m = np.memmap(fn, dtype=dt, mode='r', offset=at + s, shape=shape)
+        if m.nbytes != e - s: raise ValueError(f'{fn}: tensor {k!r} has {e - s} bytes, its shape and dtype make {m.nbytes}')
+        return m
+    return [_m(k) for k in names]
+
+# %% ../nbs/00_core.ipynb #4ea0cd84
+_NP2ST = {np.dtype(v): k for k, v in ST_DTYPES.items()}
+
+def st_write(fn, tensors):
+    "Write dict `tensors` of numpy arrays to safetensors file `fn`."
+    hdr, at, bufs = {}, 0, []
+    for k, a in tensors.items():
+        b = np.ascontiguousarray(a).astype(a.dtype.newbyteorder('<'), copy=False).tobytes()
+        hdr[k] = dict(dtype=_NP2ST[a.dtype], shape=list(a.shape), data_offsets=[at, at + len(b)]); at += len(b); bufs.append(b)
+    h = json.dumps(hdr, separators=(',', ':')).encode(); h += b' ' * (-len(h) % 8)
+    Path(fn).parent.mkdir(parents=True, exist_ok=True)
+    with open(fn, 'wb') as f: f.write(len(h).to_bytes(8, 'little')); f.write(h); [f.write(b) for b in bufs]
+
+# %% ../nbs/00_core.ipynb #ad1cbbef
+_MODEL = re.compile(rb'"model"\s*:\s*\{')
+_VOCAB = re.compile(rb'"vocab"\s*:\s*([\[{])')
+
+def find_vocab(t):
+    "Where the model's vocabulary opens in tokenizer.json bytes `t`, and its bracket."
+    ms = list(_MODEL.finditer(t))
+    if not ms: return None, None
+    v = _VOCAB.search(t, ms[-1].end())
+    return (v.end(), v.group(1)) if v else (None, None)
+
+def is_unigram(fn):
+    "Whether tokenizer.json `fn` holds a Unigram model."
+    with open(fn, 'rb') as f, mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as t: return find_vocab(t)[1] == b'['
+
+# %% ../nbs/00_core.ipynb #13167816
+def try_resolve(repo, **kw):
+    "`resolve(repo)`, or None (logged) when it cannot be had, e.g. offline."
+    try: return resolve(repo, **kw)
+    except Exception as e: log.warning(f'laghu: {repo} unavailable: {e!r}'); return None
+
+# %% ../nbs/00_core.ipynb #b4cc1c2d
+SENTENCES = [
+    'The quick brown fox jumps over the lazy dog.', "It's 3:45 p.m. on 2026-10-05; the price is $1,299.99 (incl. 18% GST).",
+    'Les élèves ont étudié l’histoire de la Révolution française.', 'Größere Äpfel schmecken süßer als kleinere, sagt Jürgen.',
+    'El niño comió paella en la playa de Valencia.', 'Il gatto dorme sul divano mentre piove.', 'O pássaro voou sobre a floresta amazônica.',
+    'Москва — столица России, крупнейший город Европы.', 'Η Αθήνα είναι η πρωτεύουσα της Ελλάδας.', 'Zażółć gęślą jaźń.',
+    'Příliš žluťoučký kůň úpěl ďábelské ódy.', 'Árvíztűrő tükörfúrógép.', 'İstanbul’da çay içip simit yedik.', 'Hà Nội là thủ đô của Việt Nam.',
+    'مرحبا بالعالم، كيف حالك اليوم؟', 'שלום עולם, מה שלומך?', 'سلام دنیا، امروز هوا خوب است.', 'नमस्ते दुनिया, आज मौसम अच्छा है।',
+    'कर्मण्येवाधिकारस्ते मा फलेषु कदाचन ।', 'অামি বাংলায় গান গাই।', 'வணக்கம் உலகம், இன்று வானிலை நன்றாக உள்ளது.',
+    'నమస్కారం ప్రపంచం.', 'ನಮಸ್ಕಾರ ಜಗತ್ತು.', 'നമസ്കാരം ലോകം.', 'ਸਤਿ ਸ੍ਰੀ ਅਕਾਲ ਦੁਨੀਆ।', 'નમસ્તે દુનિયા.', 'සුභ උදෑසනක්.',
+    'สวัสดีชาวโลก วันนี้อากาศดี', 'ສະບາຍດີ', 'မင်္ဂလာပါ', 'ជំរាបសួរ', '你好，世界！今天天气很好。', '日本語の文章を正しく分割できますか？',
+    '안녕하세요, 세계! 오늘 날씨가 좋네요.', 'გამარჯობა მსოფლიო', 'Բարեւ աշխարհ', 'ሰላም ልዑል', 'Habari ya dunia, leo ni siku nzuri.',
+    'def f(x): return x**2  # squares\nprint(f(3))', 'SELECT * FROM users WHERE id = 42;', 'https://example.com/a/b?c=d&e=f#g',
+    'user@example.org, +1 (555) 010-9999', 'E = mc², ∑ᵢ xᵢ² ≤ ∞, ∀x∈ℝ', '¯\\_(ツ)_/¯ :-) <3 ^_^']
+
+ODD = ['', ' ', '   ', '\t\n', '\n\n\n', '\x00', '\x00\x01\x1f\x7f', '\x85

', 'tab\tsep\x0bvt\x0cff', 'CRLF\r\nline\rcr',
+    '[UNK]', '[PAD]', '[CLS] [SEP] [MASK]', 'a[PAD]b [UNK] c', '[PAD][PAD]', '[UNK]x[UNK]', ' [PAD]', '<unk>', '<pad></s>', '<s>x</s>',
+    '<extra_id_0>x<extra_id_99>', '<mask>', '▁', '▁▁x', '  leading', 'trailing  ', 'a  b   c', ' nbsp　ideographic thin',
+    '​‌‍', 'bom', '\U0001F600 emoji', '👩‍👩‍👧‍👦 👍🏽 🇮🇳 🏳️‍🌈', 'fi fl Ⅻ ① ㎏ ㍿', 'é é ñ ñ', 'à́̂̃̄',
+    'z̷̢̛a̴l̸g̵o̶', 'ＦＵＬＬＷＩＤＴＨ ｶﾀｶﾅ', '𝔘𝔫𝔦𝔠𝔬𝔡𝔢 𝟙𝟚𝟛', '𑌗𑍍𑌰𑌨𑍍𑌥', ' private', 'RTL ‮evil‬ ‏',
+    '\U000E0041\U000E007F tags', '\U0010FFFF', 'x' * 5000, 'x ' * 3000, ' '.join(['word'] * 600), 'ab' * 100_000, 'नम' * 100_000]
+
+_BLOCKS = [(0x20, 0x7e), (0xa0, 0xff), (0x100, 0x24f), (0x300, 0x36f), (0x370, 0x3ff), (0x400, 0x4ff), (0x530, 0x58f), (0x590, 0x5ff),
+    (0x600, 0x6ff), (0x900, 0x97f), (0x980, 0x9ff), (0xb80, 0xbff), (0xc00, 0xc7f), (0xd00, 0xd7f), (0xe00, 0xe7f), (0x10a0, 0x10ff),
+    (0x1100, 0x11ff), (0x2000, 0x206f), (0x2100, 0x214f), (0x2190, 0x22ff), (0x3000, 0x303f), (0x3040, 0x30ff), (0x4e00, 0x4fff),
+    (0xac00, 0xadff), (0xfb00, 0xfb4f), (0xff00, 0xffef), (0x1d400, 0x1d4ff), (0x1f300, 0x1f6ff), (0x0, 0x1f), (0x11300, 0x1137f)]
+
+def noise(n=2000, seed=0):
+    "`n` seeded random strings, each from one to three Unicode blocks, with spaces between runs."
+    import random
+    r = random.Random(seed); out = []
+    for _ in range(n):
+        bs = r.sample(_BLOCKS, r.randint(1, 3))
+        out.append(''.join(chr(r.randint(*r.choice(bs))) if r.random() > 0.15 else ' ' for _ in range(r.randint(0, 60))))
+    return out
+
+def corpus(n=2000, seed=0):
+    "SENTENCES, ODD, `noise(n, seed)` and the sentences recombined: about `2.4 * n` texts."
+    import random
+    r = random.Random(seed); ws = ' '.join(SENTENCES).split()
+    mix = [' '.join(r.sample(ws, r.randint(1, 20))) for _ in range(n // 2)]
+    mix += [o.upper() for o in SENTENCES] + [o.replace(' ', '') for o in SENTENCES] + [s[:r.randint(0, len(s))] for s in SENTENCES]
+    return SENTENCES + ODD + noise(n, seed) + mix + ['\n'.join(SENTENCES), ' '.join(SENTENCES) * 20]

laghu-0.1.0/laghu/encoder.py

@@ -0,0 +1,112 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_encoder.ipynb.
+
+# %% auto #0
+__all__ = ['LeanModel', 'load', 'same_as_model2vec']
+
+# %% ../nbs/02_encoder.ipynb #07e60f33
+import json, numpy as np
+from pathlib import Path
+from fastcore.all import ifnone, patch
+from .core import log, resolve, model_files, st_map
+from .unigram import tokenizer, Unigram
+
+# %% ../nbs/02_encoder.ipynb #bf662691
+class LeanModel:
+    "model2vec's `StaticModel` for reading a model, with its table mapped from the file and, for Unigram, no trie."
+    def __init__(self, folder, normalize=None, lean=True):
+        f = model_files(folder); self.folder = Path(folder)
+        self.config = json.loads(f.cfg.read_text())
+        self.embedding, self.weights, self.token_mapping = st_map(f.emb, [f.name, 'weights', 'mapping'])
+        if self.embedding is None: raise KeyError(f'{f.emb}: no tensor {f.name!r}')
+        self.tokenizer = tokenizer(f.tok, lean)
+        if self.token_mapping is None and self.tokenizer.n != len(self.embedding):
+            raise ValueError(f'Number of tokens ({self.tokenizer.n}) does not match number of vectors ({len(self.embedding)}).')
+        self.median_token_length, self.unk_token_id = self.tokenizer.median, self.tokenizer.unk_token_id
+        self.normalize = ifnone(normalize, self.config.get('normalize', False))
+        self.config['normalize'] = self.normalize
+
+    @classmethod
+    def from_pretrained(cls, path, token=None, normalize=None, subfolder=None, force_download=False, lean=True):
+        "A `LeanModel` for a folder or Hugging Face repo id `path`; a cached repo loads without the network."
+        return cls(resolve(path, token=token, subfolder=subfolder, force_download=force_download), normalize=normalize, lean=lean)
+
+    @property
+    def dim(self): return self.embedding.shape[1]
+    @property
+    def embedding_dtype(self): return np.dtype(self.embedding.dtype).name
+    @property
+    def lean(self): return isinstance(self.tokenizer, Unigram)
+    @property
+    def tokens(self):
+        "Every token, in id order, as `StaticModel.tokens`."
+        if not hasattr(self, '_tokens'): v = self.tokenizer.vocab(); self._tokens = tuple(sorted(v, key=v.get))
+        return self._tokens
+
+    def __repr__(self):
+        return f"{type(self).__name__}({str(self.folder)!r}, {len(self.embedding)}×{self.dim} {self.embedding_dtype}, {'unigram' if self.lean else 'tokenizers'})"
+
+# %% ../nbs/02_encoder.ipynb #6fb7680d
+def _batches(xs, bs): return (xs[i:i + bs] for i in range(0, len(xs), bs))
+
+@patch
+def tokenize(self:LeanModel, sentences, max_length=None):
+    "The ids of each of `sentences`, as `StaticModel.tokenize`."
+    if max_length is not None: m = max_length * self.median_token_length; sentences = [s[:m] for s in sentences]
+    ids = self.tokenizer(sentences)
+    if self.unk_token_id is not None: ids = [[k for k in o if k != self.unk_token_id] for o in ids]
+    return [o[:max_length] for o in ids] if max_length is not None else ids
+
+@patch
+def _rows(self:LeanModel, ids):
+    "The vectors of `ids`, remapped and weighted when the model is vocabulary-quantized."
+    emb = self.embedding[ids if self.token_mapping is None else self.token_mapping[ids]]
+    return emb * self.weights[ids][:, None] if self.weights is not None else emb
+
+@patch
+def _encode_batch(self:LeanModel, sentences, max_length):
+    "One batch: model2vec stacks the means, so an empty text's float64 zero row makes its whole batch float64."
+    out = np.stack([self._rows(o).mean(axis=0) if o else np.zeros(self.dim) for o in self.tokenize(sentences, max_length)])
+    return out / (np.linalg.norm(out, axis=1, keepdims=True) + 1e-32) if self.normalize else out
+
+@patch
+def encode(self:LeanModel, sentences, show_progress_bar=False, max_length=512, batch_size=1024, use_multiprocessing=True, multiprocessing_threshold=10_000, **kwargs):
+    "Each sentence's mean token vector (unit length when `normalize`), as `StaticModel.encode`; one string gives one vector."
+    one = isinstance(sentences, str)
+    xs = [sentences] if one else list(sentences)
+    if not xs: return np.zeros((0, self.dim), self.embedding.dtype)
+    out = np.concatenate([self._encode_batch(b, max_length) for b in _batches(xs, batch_size)], axis=0)
+    return out[0] if one else out
+
+@patch
+def encode_as_sequence(self:LeanModel, sentences, max_length=None, batch_size=1024, show_progress_bar=False, use_multiprocessing=True, multiprocessing_threshold=10_000):
+    "Each sentence's token vectors, `(n_tokens, dim)`, as `StaticModel.encode_as_sequence`."
+    one = isinstance(sentences, str)
+    xs = [sentences] if one else list(sentences)
+    out = [self._rows(o) if o else np.zeros((0, self.dim)) for b in _batches(xs, batch_size) for o in self.tokenize(b, max_length)]
+    return out[0] if one else out
+
+# %% ../nbs/02_encoder.ipynb #e91068cb
+def load(path, normalize=None, token=None, subfolder=None, force_download=False, lean=True, fallback=True):
+    "A `LeanModel` for `path`, else model2vec's `StaticModel` with a logged warning; `fallback=False` raises instead."
+    try: return LeanModel.from_pretrained(path, token=token, normalize=normalize, subfolder=subfolder, force_download=force_download, lean=lean)
+    except Exception as e:
+        if not fallback: raise
+        log.warning(f'laghu: cannot read {path} ({e!r}); loading it with model2vec instead')
+        try:
+            from model2vec import StaticModel
+            return StaticModel.from_pretrained(path, token=token, normalize=normalize, subfolder=subfolder, force_download=force_download)
+        except Exception as e2: raise e from e2
+
+# %% ../nbs/02_encoder.ipynb #c807791f
+def same_as_model2vec(path, txts=None, max_length=512, seq=200):
+    "Whether `load(path)` matches model2vec on `txts` (default `corpus()`): a dict of the checks."
+    from model2vec import StaticModel
+    from laghu.core import corpus
+    txts = ifnone(txts, corpus())
+    lm, sm = load(path, fallback=False), StaticModel.from_pretrained(str(resolve(path)), force_download=False)
+    a, b = lm.encode(txts, max_length=max_length), sm.encode(txts, max_length=max_length)
+    sa, sb = lm.encode_as_sequence(txts[:seq]), sm.encode_as_sequence(txts[:seq])
+    return dict(lean=lm.lean, n=len(txts), ids=lm.tokenize(txts, max_length) == sm.tokenize(txts, max_length), dtype=a.dtype == b.dtype,
+        f32=np.array_equal(a.astype(np.float32), b.astype(np.float32)), f16=np.array_equal(a.astype(np.float16), b.astype(np.float16)),
+        exact=a.dtype == b.dtype and np.array_equal(a, b), seq=all(x.dtype == y.dtype and np.array_equal(x, y) for x, y in zip(sa, sb)),
+        median=lm.median_token_length == sm.median_token_length, unk=lm.unk_token_id == sm.unk_token_id, tokens=lm.tokens == tuple(sm.tokens))

laghu-0.1.0/laghu/unigram.py

@@ -0,0 +1,214 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_unigram.ipynb.
+
+# %% auto #0
+__all__ = ['UNK_PENALTY', 'WORD_CACHE', 'WORD_LEN', 'WS', 'read_vocab', 'viterbi', 'Added', 'Unigram', 'HFTok', 'tokenizer']
+
+# %% ../nbs/01_unigram.ipynb #d9193c51
+import json, re, mmap, numpy as np
+from array import array
+from functools import lru_cache
+from tokenizers import Tokenizer, PreTokenizedString
+from fastcore.all import ifnone
+from .core import log, find_vocab, is_unigram
+
+# %% ../nbs/01_unigram.ipynb #60cb7473
+_ENTRY = re.compile(rb'\s*\[\s*"((?:[^"\\]|\\.)*)"\s*,\s*([^\],\s]+)\s*\]\s*(,?)')
+_CLOSE = re.compile(rb'\s*\]')
+
+def read_vocab(fn):
+    "Unigram tokenizer.json `fn` as (piece → id, scores, the JSON with the vocabulary emptied); a repeated piece keeps its last id, as in `tokenizers`."
+    with open(fn, 'rb') as f, mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as t:
+        v0, br = find_vocab(t)
+        if br != b'[': raise NotImplementedError(f'{fn}: not a Unigram vocabulary')
+        ids, scores, at, i = {}, array('d'), v0, 0
+        while m := _ENTRY.match(t, at):
+            p = m.group(1); ids[json.loads(b'"' + p + b'"') if b'\\' in p else p.decode()] = i; scores.append(float(m.group(2))); i += 1; at = m.end()
+            if not m.group(3): break
+        if not (c := _CLOSE.match(t, at)): raise ValueError(f'{fn}: vocabulary not read to its end (stopped at byte {at})')
+        return ids, scores, json.loads(t[:v0] + t[c.start():])
+
+# %% ../nbs/01_unigram.ipynb #e3c950b2
+UNK_PENALTY = 10.0
+WORD_CACHE, WORD_LEN = 2**16, 32   # pre-tokens cached per Unigram, and the longest one cached
+
+def _unk(p, ids, unk_id, byte_fallback):
+    "The ids of an unknown run `p`: a piece if it spells one, else its UTF-8 bytes as `<0xXX>` pieces under byte fallback, else the unknown id."
+    if (k := ids.get(p)) is not None: return [k]
+    if byte_fallback and None not in (bs := [ids.get(f'<0x{b:02X}>') for b in p.encode()]): return bs
+    if unk_id < 0: raise ValueError(f'Encountered an unknown token but `unk_id` is missing: {p!r}')   # tokenizers' error, and model2vec's
+    return [unk_id]
+
+def viterbi(s, ids, scores, reach, unk_id, unk_score, byte_fallback=False):
+    "The ids of the best segmentation of `s` into pieces of `ids`, unknown runs fused."
+    n = len(s)
+    best, start, tid = [0.0] + [None] * n, [0] * (n + 1), [0] * (n + 1)
+    for i in range(n):
+        b, single = best[i], False
+        for j in range(i + 1, min(n, i + reach.get(s[i:i + 2], 1)) + 1):
+            k = ids.get(s[i:j])
+            if k is None: continue
+            c = scores[k] + b
+            if best[j] is None or c > best[j]: best[j], start[j], tid[j] = c, i, k
+            if j == i + 1: single = True
+        if not single:
+            c = unk_score + b
+            if best[i + 1] is None or c > best[i + 1]: best[i + 1], start[i + 1], tid[i + 1] = c, i, unk_id
+    out, e, run = [], n, None
+    while e > 0:
+        if tid[e] == unk_id: run = run or e
+        else:
+            if run: out += _unk(s[e:run], ids, unk_id, byte_fallback)[::-1]; run = None
+            out.append(tid[e])
+        e = start[e]
+        if run and e == 0: out += _unk(s[:run], ids, unk_id, byte_fallback)[::-1]
+    return out[::-1]
+
+# %% ../nbs/01_unigram.ipynb #a88855f9
+WS = frozenset('\t\n\x0b\x0c\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000')   # Rust's char::is_whitespace
+
+def _metaspace(pt, scheme):
+    "Pre-tokenizer JSON `pt` with every Metaspace's `prepend_scheme: first` set to `scheme`."
+    if isinstance(pt, list): return [_metaspace(o, scheme) for o in pt]
+    if not isinstance(pt, dict): return pt
+    pt = {k: _metaspace(v, scheme) for k, v in pt.items()}
+    if pt.get('type') == 'Metaspace' and pt.get('prepend_scheme') == 'first': pt['prepend_scheme'] = scheme
+    return pt
+
+def _pipeline(d, pre):
+    "The normalizer and pre-tokenizer of tokenizer JSON `d`, with pre-tokenizer `pre`, and nothing else of it."
+    d = dict(d, pre_tokenizer=pre, added_tokens=[], post_processor=None, decoder=None, truncation=None, padding=None)
+    d['model'] = dict(d['model'], vocab=[['<unk>', 0.0]], unk_id=0, byte_fallback=False)
+    t = Tokenizer.from_str(json.dumps(d))
+    return t.normalizer, t.pre_tokenizer, t
+
+class Added:
+    "A set of added tokens found in a string as `tokenizers`' `AddedVocabulary.find_matches` finds them."
+    def __init__(self, toks, norm=None):
+        "`toks` are matched as written, or as `norm` normalizes them when it is given (`normalized` tokens)."
+        self.toks = {(norm.normalize_str(o['content']) if norm else o['content']): (o['id'], o.get('lstrip'), o.get('rstrip')) for o in toks}
+        self.toks.pop('', None)
+        self.rx = re.compile('|'.join(map(re.escape, sorted(self.toks, key=len, reverse=True)))) if self.toks else None
+
+    def split(self, s):
+        "`s` cut into `(text, None)` and `(content, id)` parts, in order, empty text dropped."
+        if not self.rx: return [(s, None)] if s else []
+        out, prev = [], 0
+        for m in self.rx.finditer(s):
+            st, sp = m.span(); k, ls, rs = self.toks[m.group()]
+            if ls:
+                while st > prev and s[st - 1] in WS: st -= 1
+            if rs:
+                while sp < len(s) and s[sp] in WS: sp += 1
+            if st > prev: out.append((s[prev:st], None))
+            out.append((s[st:sp], k)); prev = sp
+        if prev < len(s): out.append((s[prev:], None))
+        return out
+
+class Unigram:
+    "A Unigram tokenizer.json's `encode_batch(texts, add_special_tokens=False)` ids, with no trie."
+    def __init__(self, fn):
+        self.ids, self.scores, d = read_vocab(fn)
+        mdl = d['model']
+        if mdl.get('type', 'Unigram') != 'Unigram': raise NotImplementedError(f"{fn}: model is {mdl.get('type')}")
+        added = d.get('added_tokens', [])
+        if any(o.get('single_word') for o in added): raise NotImplementedError(f'{fn}: single_word added tokens')
+        self.byte_fallback, self.unk_id = bool(mdl.get('byte_fallback')), ifnone(mdl.get('unk_id'), -1)   # -1: none, an error if met
+        self.trunc, self.pad = d.get('truncation'), d.get('padding')
+        self.unk_score = min(self.scores) - UNK_PENALTY
+        self.reach = {}   # a piece's first two letters → the longest piece that starts so: no longer substring is tried
+        for w in self.ids:
+            if len(w) > 1 and len(w) > self.reach.get(w[:2], 0): self.reach[w[:2]] = len(w)
+        self.added = {o['content']: o['id'] for o in added}
+        self.norm, self.pre, t = _pipeline(d, d.get('pre_tokenizer'))
+        self.raw, self.nrm = Added([o for o in added if not o.get('normalized')]), Added([o for o in added if o.get('normalized')], self.norm)
+        later = _metaspace(d.get('pre_tokenizer'), 'never')
+        self.pre_later = self.pre if later == d.get('pre_tokenizer') else _pipeline(d, later)[1]
+        if self.nrm.rx and self.pre_later is not self.pre: raise NotImplementedError(f'{fn}: normalized added tokens under prepend_scheme first')
+        self.model_unk = getattr(t.model, 'unk_token', None)   # model2vec drops this token's id; a Unigram model has none
+        extra = [k for k in self.added if k not in self.ids]
+        self.n = len(self.ids) + len(extra)   # the size of `Tokenizer.get_vocab()`
+        lens = np.fromiter((len(k) for k in [*self.ids, *extra]), np.int32, self.n)
+        self.median = int(np.median(lens)) if self.n else 0
+        self._cached = lru_cache(WORD_CACHE)(lambda q: tuple(self._viterbi(q)))
+
+    def _viterbi(self, q): return viterbi(q, self.ids, self.scores, self.reach, self.unk_id, self.unk_score, self.byte_fallback)
+
+    def word(self, q):
+        "The ids of pre-token `q`; one of at most `WORD_LEN` characters, as natural text repeats them, from a cache of `WORD_CACHE`."
+        return self._cached(q) if len(q) <= WORD_LEN else self._viterbi(q)
+
+    def _first(self, s):
+        "The pre-tokens of the stretch opening the text, offsets kept, so `first` prepends only where the normalized text starts at 0."
+        pts = PreTokenizedString(s)
+        if self.norm: pts.normalize(self.norm.normalize)
+        pts.split(lambda i, ns: [ns[0:len(ns.normalized)]] if ns.normalized else [])   # as tokenizers re-slices after its added-token pass
+        self.pre.pre_tokenize(pts)
+        return [p for p, *_ in pts.get_splits()]
+
+    def _pieces(self, nz, pt):
+        out = []
+        for p, k in self.nrm.split(nz):
+            if k is not None: out.append(k); continue
+            for q in ([q for q, _ in pt.pre_tokenize_str(p)] if pt else [p]):
+                out += self.word(q)
+        return out
+
+    def _seg(self, s, first):
+        if first and self.pre_later is not self.pre:
+            return [k for p in self._first(s) for k in self.word(p)]
+        nz = self.norm.normalize_str(s) if self.norm else s
+        return self._pieces(nz, self.pre if first else self.pre_later) if nz else []
+
+    def encode(self, s):
+        "The ids of text `s`, before truncation and padding."
+        out, at = [], 0
+        for p, k in self.raw.split(s):
+            out += self._seg(p, at == 0) if k is None else [k]; at += len(p)
+        return out
+
+    def __call__(self, txts):
+        "The ids of each of `txts`, as one batch: truncated and padded when tokenizer.json says so."
+        out = [self.encode(t) for t in txts]
+        if t := self.trunc:
+            L, left = t['max_length'], t.get('direction') == 'Left'
+            out = [(o[-L:] if L else []) if left else o[:L] for o in out]
+        if (p := self.pad) and out:
+            st = p.get('strategy')
+            n = max(map(len, out)) if st == 'BatchLongest' else (st or {}).get('Fixed', 0)
+            if (m := p.get('pad_to_multiple_of')) and n % m: n += m - n % m
+            pid, left = p.get('pad_id', 0), p.get('direction') == 'Left'
+            out = [([pid] * (n - len(o)) + o if left else o + [pid] * (n - len(o))) if len(o) < n else o for o in out]
+        return out
+
+    @property
+    def unk_token_id(self):
+        "The id model2vec drops from every encoding, or None."
+        return None if self.model_unk is None else self.vocab()[self.model_unk]
+
+    def vocab(self):
+        "`Tokenizer.get_vocab()`: piece → id, the added tokens over the model's."
+        return {**self.ids, **self.added}
+
+# %% ../nbs/01_unigram.ipynb #4946b03a
+class HFTok:
+    "`tokenizers`' own encoding, as model2vec calls it."
+    def __init__(self, fn):
+        self.tok = Tokenizer.from_file(str(fn))
+        v = self.tok.get_vocab()
+        self.n, self.median = len(v), int(np.median([len(o) for o in v])) if v else 0
+        unk = getattr(self.tok.model, 'unk_token', None)
+        self.unk_token_id = v[unk] if unk is not None else None
+        self._enc = getattr(self.tok, 'encode_batch_fast', self.tok.encode_batch)
+
+    def __call__(self, txts):
+        "The ids of each of `txts`, one batch as `tokenizers` encodes it (padding, if the file asks for it, is per batch)."
+        return [e.ids for e in self._enc(list(txts), add_special_tokens=False)]
+
+    def vocab(self): return self.tok.get_vocab()
+
+def tokenizer(fn, lean=True):
+    "A `Unigram` for a Unigram tokenizer.json `fn` when `lean` and it can be done, else an `HFTok`."
+    if lean and is_unigram(fn):
+        try: return Unigram(fn)
+        except NotImplementedError as e: log.info(f'laghu: {e}; tokenizing with tokenizers')
+    return HFTok(fn)

laghu-0.1.0/pyproject.toml

@@ -0,0 +1,57 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "laghu"
+dynamic = ["version"]
+description = "model2vec static models in a fraction of the memory, with the same vectors"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{name = "Karthik", email = "karthik.rajgopal@hotmail.com"}]
+keywords = ['nbdev', 'model2vec', 'embeddings', 'static-embeddings', 'mmap', 'tokenizers', 'unigram']
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "fastcore>=1.8",
+    "numpy>=1.24",
+    "tokenizers>=0.20",
+    "huggingface-hub>=0.24",
+]
+
+[project.optional-dependencies]
+fallback = ["model2vec>=0.9"]
+
+[project.urls]
+Repository = "https://github.com/vedicreader/laghu"
+Documentation = "https://vedicreader.github.io/laghu/"
+
+[project.entry-points.nbdev]
+laghu = "laghu._modidx:d"
+
+[tool.nbdev]
+tst_flags = "slow"
+
+[tool.hatch.build.targets.wheel]
+packages = ["laghu"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/laghu", "/README.md", "/pyproject.toml"]
+
+[tool.hatch.version]
+path = "laghu/__init__.py"
+
+[dependency-groups]
+dev = [
+    "ipykernel>=7.3.0",
+    "model2vec>=0.9",
+    "nbdev>=3.3.14",
+    "notebook>=7.6.3",
+]