toke-tokenizer 0.1.0__tar.gz

toke_tokenizer-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Matt Watt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

toke_tokenizer-0.1.0/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: toke-tokenizer
+Version: 0.1.0
+Summary: BPE tokenizer for the toke programming language (16K vocab)
+Author: Matt Watt
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/karwalski/toke-tokenizer
+Project-URL: Repository, https://github.com/karwalski/toke-tokenizer
+Keywords: tokenizer,bpe,toke,programming-language
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Compilers
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Dynamic: license-file
+
+# toke-tokenizer
+
+A pure Python BPE tokenizer for the [toke programming language](https://github.com/karwalski/toke). Trained on normalised toke source code with a 16,384-token vocabulary.
+
+Achieves approximately 52% token reduction compared to OpenAI's cl100k_base tokenizer on toke source code.
+
+## Installation
+
+```bash
+pip install toke-tokenizer
+```
+
+## Usage
+
+```python
+from toke_tokenizer import encode, decode, count_tokens
+
+# Tokenize toke source code
+text = "let x:int = 42"
+tokens = encode(text)
+print(tokens)        # list of token IDs
+
+# Decode back to text
+original = decode(tokens)
+print(original)      # "let x:int = 42"
+
+# Count tokens
+n = count_tokens(text)
+print(f"{n} tokens")
+```
+
+### String normalisation
+
+Toke source often contains string literals that are not useful for structural tokenization. You can normalise strings before counting:
+
+```python
+from toke_tokenizer import count_tokens
+
+# Without normalisation
+count_tokens('let msg:str = "hello world"')
+
+# With normalisation (replaces string contents with "_")
+count_tokens('let msg:str = "hello world"', normalise_strings=True)
+# Equivalent to counting: let msg:str = "_"
+```
+
+## API
+
+- `encode(text: str) -> list[int]` — Tokenize text into a list of token IDs.
+- `decode(ids: list[int]) -> str` — Convert token IDs back to text.
+- `count_tokens(text: str, normalise_strings: bool = False) -> int` — Count the number of tokens in text.
+
+## Details
+
+- Vocabulary: 16,384 tokens
+- Algorithm: Byte-Pair Encoding (BPE)
+- Pre-tokenization: splits on newlines (each newline is a separate token)
+- Special tokens: `<|endoftext|>` (0), `<pad>` (1), `<newline>` (2)
+
+## License
+
+MIT

toke_tokenizer-0.1.0/README.md

@@ -0,0 +1,62 @@
+# toke-tokenizer
+
+A pure Python BPE tokenizer for the [toke programming language](https://github.com/karwalski/toke). Trained on normalised toke source code with a 16,384-token vocabulary.
+
+Achieves approximately 52% token reduction compared to OpenAI's cl100k_base tokenizer on toke source code.
+
+## Installation
+
+```bash
+pip install toke-tokenizer
+```
+
+## Usage
+
+```python
+from toke_tokenizer import encode, decode, count_tokens
+
+# Tokenize toke source code
+text = "let x:int = 42"
+tokens = encode(text)
+print(tokens)        # list of token IDs
+
+# Decode back to text
+original = decode(tokens)
+print(original)      # "let x:int = 42"
+
+# Count tokens
+n = count_tokens(text)
+print(f"{n} tokens")
+```
+
+### String normalisation
+
+Toke source often contains string literals that are not useful for structural tokenization. You can normalise strings before counting:
+
+```python
+from toke_tokenizer import count_tokens
+
+# Without normalisation
+count_tokens('let msg:str = "hello world"')
+
+# With normalisation (replaces string contents with "_")
+count_tokens('let msg:str = "hello world"', normalise_strings=True)
+# Equivalent to counting: let msg:str = "_"
+```
+
+## API
+
+- `encode(text: str) -> list[int]` — Tokenize text into a list of token IDs.
+- `decode(ids: list[int]) -> str` — Convert token IDs back to text.
+- `count_tokens(text: str, normalise_strings: bool = False) -> int` — Count the number of tokens in text.
+
+## Details
+
+- Vocabulary: 16,384 tokens
+- Algorithm: Byte-Pair Encoding (BPE)
+- Pre-tokenization: splits on newlines (each newline is a separate token)
+- Special tokens: `<|endoftext|>` (0), `<pad>` (1), `<newline>` (2)
+
+## License
+
+MIT

toke_tokenizer-0.1.0/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "toke-tokenizer"
+version = "0.1.0"
+description = "BPE tokenizer for the toke programming language (16K vocab)"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [{name = "Matt Watt"}]
+keywords = ["tokenizer", "bpe", "toke", "programming-language"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Compilers",
+    "Topic :: Text Processing",
+]
+
+[project.urls]
+Homepage = "https://github.com/karwalski/toke-tokenizer"
+Repository = "https://github.com/karwalski/toke-tokenizer"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7",
+    "ruff>=0.4",
+]
+
+[tool.setuptools.packages.find]
+include = ["toke_tokenizer*"]
+
+[tool.setuptools.package-data]
+toke_tokenizer = ["data/*.json"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"

toke_tokenizer-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

toke_tokenizer-0.1.0/toke_tokenizer/__init__.py

@@ -0,0 +1,6 @@
+"""toke-tokenizer: BPE tokenizer for the toke programming language."""
+
+from toke_tokenizer.tokenizer import decode, encode, count_tokens, normalise_strings
+
+__all__ = ["encode", "decode", "count_tokens", "normalise_strings"]
+__version__ = "0.1.0"