tiktoken-fast 0.1.3__tar.gz

tiktoken_fast-0.1.3/Cargo.lock

@@ -0,0 +1,232 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
+
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
+[[package]]
+name = "bit-set"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
+dependencies = [
+ "bit-vec",
+]
+
+[[package]]
+name = "bit-vec"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
+
+[[package]]
+name = "bstr"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
+dependencies = [
+ "memchr",
+ "regex-automata",
+ "serde_core",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
+
+[[package]]
+name = "either"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
+
+[[package]]
+name = "fancy-regex"
+version = "0.19.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912"
+dependencies = [
+ "bit-set",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "regex"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
+[[package]]
+name = "rustc-hash"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "tiktoken-fast"
+version = "0.1.3"
+dependencies = [
+ "base64",
+ "rayon",
+ "regex-syntax",
+ "tiktoken-rs",
+]
+
+[[package]]
+name = "tiktoken-rs"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2aeff724640cfe13037336ddf35befdffd2909cbdb65cf041cc8a4cf8c584cfa"
+dependencies = [
+ "anyhow",
+ "base64",
+ "bstr",
+ "fancy-regex",
+ "regex",
+ "rustc-hash",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"

tiktoken_fast-0.1.3/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "tiktoken-fast"
+version = "0.1.3"
+edition = "2021"
+license = "MIT"
+description = "Byte-pair encoder for OpenAI's tiktoken vocabularies: the token ids of the reference core, several times faster"
+repository = "https://github.com/s3bastiankoch/tiktoken-fast"
+readme = "README.md"
+keywords = ["tokenizer", "tiktoken", "bpe", "openai", "llm"]
+categories = ["text-processing", "encoding"]
+include = ["/src/**", "/assets/**", "/Cargo.toml", "/README.md", "/LICENSE", "/NOTICE"]
+
+[lib]
+name = "tiktoken_fast"
+
+[dependencies]
+base64 = "0.22"
+regex-syntax = "0.8"
+rayon = "1.10"
+
+[dev-dependencies]
+tiktoken-rs = "0.12.1"
+
+[profile.release]
+opt-level = 3
+lto = "fat"
+codegen-units = 1
+panic = "abort"

tiktoken_fast-0.1.3/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sebastian Koch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tiktoken_fast-0.1.3/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sebastian Koch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tiktoken_fast-0.1.3/NOTICE

@@ -0,0 +1,25 @@
+The vocabulary files under assets/ are OpenAI's tiktoken vocabularies
+(cl100k_base, o200k_base, p50k_base, r50k_base), copied from the tiktoken-rs
+crate (MIT), which redistributes them. OpenAI serves these files for its
+MIT-licensed tiktoken and states no separate licence for the files themselves. The texts under tests/corpora/
+are the public-domain and open-licence corpora of the Go benchmark in go/bench
+(Moby-Dick, Go standard library source under the Go BSD licence, 紅樓夢, and
+the Universal Declaration of Human Rights in three languages); the Go licence
+is in tests/corpora/GO-LICENSE.
+
+src/backtrack.rs follows the backtracking byte-pair encoder of the bpe crate
+(github.com/github/rust-gems, MIT licence): the split table, the pair-validity
+test and the backtracking search are its design; the longest-token search here
+uses length masks over the crate's own vocabulary table instead of an
+Aho-Corasick automaton.
+
+go/ is a fork of pkoukk/tiktoken-go (MIT licence); its own LICENSE and NOTICE
+list the upstream copyright and every change from the fork point.
+
+bench/fonts/ holds Inter (github.com/rsms/inter, SIL Open Font License 1.1),
+used only to render the README figures; its licence is bench/fonts/OFL.txt.
+
+The Unicode character classes in src/ are built from the tables of the
+regex-syntax crate (MIT or Apache-2.0), which derive from the Unicode
+Character Database, Copyright (c) 1991-2024 Unicode, Inc., used under the
+Unicode License v3 (https://www.unicode.org/license.txt).

tiktoken_fast-0.1.3/NOTICE.txt

@@ -0,0 +1,25 @@
+The vocabulary files under assets/ are OpenAI's tiktoken vocabularies
+(cl100k_base, o200k_base, p50k_base, r50k_base), copied from the tiktoken-rs
+crate (MIT), which redistributes them. OpenAI serves these files for its
+MIT-licensed tiktoken and states no separate licence for the files themselves. The texts under tests/corpora/
+are the public-domain and open-licence corpora of the Go benchmark in go/bench
+(Moby-Dick, Go standard library source under the Go BSD licence, 紅樓夢, and
+the Universal Declaration of Human Rights in three languages); the Go licence
+is in tests/corpora/GO-LICENSE.
+
+src/backtrack.rs follows the backtracking byte-pair encoder of the bpe crate
+(github.com/github/rust-gems, MIT licence): the split table, the pair-validity
+test and the backtracking search are its design; the longest-token search here
+uses length masks over the crate's own vocabulary table instead of an
+Aho-Corasick automaton.
+
+go/ is a fork of pkoukk/tiktoken-go (MIT licence); its own LICENSE and NOTICE
+list the upstream copyright and every change from the fork point.
+
+bench/fonts/ holds Inter (github.com/rsms/inter, SIL Open Font License 1.1),
+used only to render the README figures; its licence is bench/fonts/OFL.txt.
+
+The Unicode character classes in src/ are built from the tables of the
+regex-syntax crate (MIT or Apache-2.0), which derive from the Unicode
+Character Database, Copyright (c) 1991-2024 Unicode, Inc., used under the
+Unicode License v3 (https://www.unicode.org/license.txt).

tiktoken_fast-0.1.3/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.4
+Name: tiktoken-fast
+Version: 0.1.3
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Rust
+Classifier: License :: OSI Approved :: MIT License
+Requires-Dist: tiktoken>=0.7
+License-File: LICENSE.txt
+License-File: NOTICE.txt
+Summary: A faster drop-in for tiktoken: the same token ids, encoded by a new Rust core
+License-Expression: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Repository, https://github.com/s3bastiankoch/tiktoken-fast
+
+# tiktoken-fast for Python
+
+A drop-in for [tiktoken](https://github.com/openai/tiktoken) that returns the
+same token ids, faster. It keeps tiktoken's Python layer, its encoding files and
+model names, and replaces only the core that turns text into tokens.
+
+```python
+import tiktoken_fast as tiktoken
+
+enc = tiktoken.get_encoding("o200k_base")
+enc.encode("hello world")
+enc.encode_batch(texts, num_threads=8)
+```
+
+Or, for code that already imports tiktoken:
+
+```python
+import tiktoken_fast
+tiktoken_fast.patch()   # tiktoken.get_encoding and encoding_for_model now return fast encodings
+```
+
+Two additions that tiktoken does not have:
+
+```python
+enc.count_tokens(text)          # len(enc.encode(text)) without building the list
+```
+
+```sh
+python -m tiktoken_fast.bench   # compare with tiktoken on this machine, about 15 s
+```
+
+The bench uses text every Python installation has (the standard library's
+source and docstrings) unless you pass `--file`. It checks that every call
+returns tiktoken's tokens before it times anything.
+
+Every encoding tiktoken ships (`r50k_base`, `p50k_base`, `p50k_edit`,
+`cl100k_base`, `o200k_base`, `o200k_harmony`) runs on the new core. An
+`Encoding` built with a split pattern the core does not know keeps tiktoken's
+core. `tiktoken_fast.Encoding` is a subclass of `tiktoken.Encoding`, so type
+checks and pickling work as before.
+
+## Speed
+
+Against tiktoken 0.14 through the same Python calls, one core of a Xeon
+Platinum 8481C, Python 3.11, 2026-10-26. Geometric means over Wikipedia in
+eleven languages, 512-byte chat chunks, code and two books, under cl100k_base
+and o200k_base; slowest workload in brackets. No document's tokens differed.
+On AMD EPYC 9B14 and Google Axion `encode_ordinary()` is 3.78x and 3.50x
+([all three machines](../README.md#numbers)).
+
+| call | new text | fresh caches per document |
+|---|---|---|
+| `encode()` | **3.21x** [1.84x] | **2.64x** |
+| `encode_ordinary()` | **3.28x** [1.80x] | **2.72x** |
+
+Short texts gain more, because tiktoken spends most of a short call in Python:
+`encode()` on 16-byte strings is 3.3x faster, and a batch of 16-byte strings
+on four vCPUs 20x to 24x faster. Raw output is in
+[`bench/results/2026-10-26-python`](../bench/results/2026-10-26-python).
+
+## Build
+
+```sh
+pip install maturin
+cd python && maturin build --release && pip install target/wheels/*.whl
+python -m pytest tests
+```
+
+Needs Python 3.10 or later and a Rust toolchain. The tests compare every public
+method with tiktoken.
+