turbobpe 0.1.0__tar.gz

turbobpe-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Amrendra Gupta
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

turbobpe-0.1.0/MANIFEST.in

@@ -0,0 +1,18 @@
+include src/turbobpe/utils.c
+
+include README.md
+include LICENSE
+include pyproject.toml
+include setup.py
+
+exclude src/turbobpe/utils.pyx
+
+# Exclude build artefacts and dev tooling
+exclude src/turbobpe/*.so
+exclude src/turbobpe/*.pyd
+prune build
+prune dist
+prune *.egg-info
+global-exclude __pycache__
+global-exclude *.py[cod]
+global-exclude .DS_Store

turbobpe-0.1.0/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: turbobpe
+Version: 0.1.0
+Summary: A fast BPE tokenizer with Cython-accelerated core
+Author: Amrendra Gupta
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Amrendra-gupta/turbobpe
+Project-URL: Repository, https://github.com/Amrendra-gupta/turbobpe
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: regex>=2023.0
+Dynamic: license-file

turbobpe-0.1.0/pyproject.toml

@@ -0,0 +1,53 @@
+[build-system]
+requires = ["setuptools>=68", "wheel", "packaging>=24.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "turbobpe"
+version = "0.1.0"
+description = "A fast BPE tokenizer with Cython-accelerated core"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.9"
+dependencies = ["regex>=2023.0"]
+authors = [{ name = "Amrendra Gupta"}]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Operating System :: OS Independent",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+[project.urls]
+Homepage = "https://github.com/Amrendra-gupta/turbobpe"
+Repository = "https://github.com/Amrendra-gupta/turbobpe"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+turbobpe = ["*.c"]
+
+[tool.cibuildwheel]
+# Build for CPython 3.9–3.14, skip PyPy and 32-bit Windows
+build = "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*"
+skip = "*-win32 *-musllinux_i686 pp*"
+
+test-command = "python -c \"from turbobpe import Tokenizer, RegexTokenizer; print('OK')\""
+
+[tool.cibuildwheel.linux]
+manylinux-x86_64-image    = "manylinux2014"
+manylinux-aarch64-image   = "manylinux2014"
+archs = ["x86_64", "aarch64"]
+
+[tool.cibuildwheel.macos]
+archs = ["x86_64", "arm64"]
+
+[tool.cibuildwheel.windows]
+archs = ["AMD64"]

turbobpe-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

turbobpe-0.1.0/setup.py

@@ -0,0 +1,14 @@
+from setuptools import setup, Extension
+
+extensions = [
+    Extension(
+        "turbobpe.utils",
+        sources=["src/turbobpe/utils.c"],
+		extra_compile_args=["-O3"],
+    )
+]
+
+setup(
+    ext_modules=extensions,
+    package_dir={"": "src"},
+)

turbobpe-0.1.0/src/turbobpe/__init__.py

@@ -0,0 +1,5 @@
+from .base import Tokenizer
+from .regex import RegexTokenizer
+
+__version__ = "0.1.0"
+__all__ = ["Tokenizer", "RegexTokenizer"]

turbobpe-0.1.0/src/turbobpe/base.py

@@ -0,0 +1,237 @@
+"""
+Contains the base Tokenizer class and a few common helper functions.
+The base class also contains the (common) save/load functionality.
+It would be possible to be a lot more strict about the interface and
+e.g. isolating all regex/pattern parts to the RegexTokenizer, but
+some concessions are made for simplicity.
+"""
+import unicodedata
+
+# -----------------------------------------------------------------------------
+# a few helper functions useful for both BasicTokenizer and RegexTokenizer
+
+def get_stats(ids, counts=None):
+    """
+    Given a list of integers, return a dictionary of counts of consecutive pairs
+    Example: [1, 2, 3, 1, 2] -> {(1, 2): 2, (2, 3): 1, (3, 1): 1}
+    Optionally allows to update an existing dictionary of counts
+    """
+    counts = {} if counts is None else counts
+    for pair in zip(ids, ids[1:]): # iterate consecutive elements
+        counts[pair] = counts.get(pair, 0) + 1
+    return counts
+
+
+def merge(ids, pair, idx):
+    """
+    In the list of integers (ids), replace all consecutive occurrences
+    of pair with the new integer token idx
+    Example: ids=[1, 2, 3, 1, 2], pair=(1, 2), idx=4 -> [4, 3, 4]
+    """
+    newids = []
+    i = 0
+    while i < len(ids):
+        # if not at the very last position AND the pair matches, replace it
+        if ids[i] == pair[0] and i < len(ids) - 1 and ids[i+1] == pair[1]:
+            newids.append(idx)
+            i += 2
+        else:
+            newids.append(ids[i])
+            i += 1
+    return newids
+    
+
+def find_overlapping_cases(coordinates):
+    """
+    Given a list of (A, B) pairs, return all pairs that form a chain overlap.
+    A chain overlap occurs when the right token of one pair is the left token of another: (A, B) and (B, C).
+    Merging both in the same batch would be unsafe as the first merge affects the second.
+    Example: [(1, 2), (2, 3), (4, 5)] -> [(1, 2), (2, 3)]  # (4, 5) is safe, no chain
+    """
+    overlapping_cases = []
+    coordinate_dict = {}
+    
+    for coord in coordinates:
+        if coord[0] not in coordinate_dict:
+            coordinate_dict[coord[0]] = []
+        coordinate_dict[coord[0]].append(coord[1])
+    
+    # Check for (A, B) and (B, C) pairs
+    for a, b_list in coordinate_dict.items():
+        for b in b_list:
+            if b in coordinate_dict:  # Check if B is also a key
+                for c in coordinate_dict[b]:
+                    overlapping_cases.append((a, b))
+                    overlapping_cases.append((b, c))
+                    #print(f"(({vocab[a] + vocab[b]}) -- ({vocab[b] + vocab[c]}))")
+    
+    #print(f"Found {len(overlapping_cases)} overlapping cases.")
+    return overlapping_cases
+
+def filter_top_pairs(top_pairs, overlaps):
+    """
+    Given a sorted (descending by frequency) list of top pairs and their overlapping cases,
+    return a safe subset for batch merging by dropping chained overlaps.
+    The first pair in an overlap chain is kept (highest count, safe to merge), the second is dropped.
+    Example: top_pairs=[(1,2),(2,3),(4,5)], overlaps=[(1,2),(2,3)] -> [(1,2),(4,5)]
+    """
+    common_count = 0  # Counter for common elements
+    result = []  # Store filtered elements
+    for pair in top_pairs:
+        # Check if the pair is in overlaps
+        if pair in overlaps:
+            common_count += 1
+        # Stop when the second common element is found
+        if common_count == 2:
+            break
+        result.append(pair)  # Always add the current element    
+    return result
+
+def batch_merge(ids, pairs, indices):
+    """
+    Sequentially applies a batch of non-overlapping pair merges 
+    to a single ID sequence.
+    """
+    newids = list(ids)
+    n = len(pairs)
+    for i in range(n):
+        p0, p1 = pairs[i]
+        idx = indices[i]
+        j = 0
+        limit = len(newids) - 1  # cache once per pair
+        while j < limit:
+            if newids[j] == p0 and newids[j + 1] == p1:
+                newids[j] = idx
+                del newids[j + 1]
+                limit -= 1  # list shrank, update limit
+                j += 1
+            else:
+                j += 1
+        if limit == 0:   # list is now length 1
+            return None
+    return newids
+
+
+# first two helper functions...
+def replace_control_characters(s: str) -> str:
+    # we don't want to print control characters
+    # which distort the output (e.g. \n or much worse)
+    # https://stackoverflow.com/questions/4324790/removing-control-characters-from-a-string-in-python/19016117#19016117
+    # http://www.unicode.org/reports/tr44/#GC_Values_Table
+    chars = []
+    for ch in s:
+        if unicodedata.category(ch)[0] != "C":
+            chars.append(ch) # this character is ok
+        else:
+            chars.append(f"\\u{ord(ch):04x}") # escape
+    return "".join(chars)
+
+def render_token(t: bytes) -> str:
+    # pretty print a token, escaping control characters
+    s = t.decode('utf-8', errors='replace')
+    s = replace_control_characters(s)
+    return s
+
+# -----------------------------------------------------------------------------
+# the base Tokenizer class
+
+class Tokenizer:
+    """Base class for Tokenizers"""
+
+    def __init__(self):
+        # default: vocab size of 256 (all bytes), no merges, no patterns
+        self.merges = {} # (int, int) -> int
+        self.pattern = "" # str
+        self.special_tokens = {} # str -> int, e.g. {'<|endoftext|>': 100257}
+        self.vocab = self._build_vocab() # int -> bytes
+
+    def train(self, text, vocab_size, verbose=False):
+        # Tokenizer can train a vocabulary of size vocab_size from text
+        raise NotImplementedError
+
+    def encode(self, text):
+        # Tokenizer can encode a string into a list of integers
+        raise NotImplementedError
+
+    def decode(self, ids):
+        # Tokenizer can decode a list of integers into a string
+        raise NotImplementedError
+
+    def _build_vocab(self):
+        # vocab is simply and deterministically derived from merges
+        vocab = {idx: bytes([idx]) for idx in range(256)}
+        for (p0, p1), idx in self.merges.items():
+            vocab[idx] = vocab[p0] + vocab[p1]
+        for special, idx in self.special_tokens.items():
+            vocab[idx] = special.encode("utf-8")
+        return vocab
+
+    def save(self, file_prefix):
+        """
+        Saves two files: file_prefix.vocab and file_prefix.model
+        This is inspired (but not equivalent to!) sentencepiece's model saving:
+        - model file is the critical one, intended for load()
+        - vocab file is just a pretty printed version for human inspection only
+        """
+        # write the model: to be used in load() later
+        model_file = file_prefix + ".model"
+        with open(model_file, 'w') as f:
+            # write the version, pattern and merges, that's all that's needed
+            f.write("minbpe v1\n")
+            f.write(f"{self.pattern}\n")
+            # write the special tokens, first the number of them, then each one
+            f.write(f"{len(self.special_tokens)}\n")
+            for special, idx in self.special_tokens.items():
+                f.write(f"{special} {idx}\n")
+            # the merges dict
+            for idx1, idx2 in self.merges:
+                f.write(f"{idx1} {idx2}\n")
+        # write the vocab: for the human to look at
+        vocab_file = file_prefix + ".vocab"
+        inverted_merges = {idx: pair for pair, idx in self.merges.items()}
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            for idx, token in self.vocab.items():
+                # note: many tokens may be partial utf-8 sequences
+                # and cannot be decoded into valid strings. Here we're using
+                # errors='replace' to replace them with the replacement char �.
+                # this also means that we couldn't possibly use .vocab in load()
+                # because decoding in this way is a lossy operation!
+                s = render_token(token)
+                # find the children of this token, if any
+                if idx in inverted_merges:
+                    # if this token has children, render it nicely as a merge
+                    idx0, idx1 = inverted_merges[idx]
+                    s0 = render_token(self.vocab[idx0])
+                    s1 = render_token(self.vocab[idx1])
+                    f.write(f"[{s0}][{s1}] -> [{s}] {idx}\n")
+                else:
+                    # otherwise this is leaf token, just print it
+                    # (this should just be the first 256 tokens, the bytes)
+                    f.write(f"[{s}] {idx}\n")
+
+    def load(self, model_file):
+        """Inverse of save() but only for the model file"""
+        assert model_file.endswith(".model")
+        # read the model file
+        merges = {}
+        special_tokens = {}
+        idx = 256
+        with open(model_file, 'r', encoding="utf-8") as f:
+            # read the version
+            version = f.readline().strip()
+            assert version == "minbpe v1"
+            # read the pattern
+            self.pattern = f.readline().strip()
+            # read the special tokens
+            num_special = int(f.readline().strip())
+            for _ in range(num_special):
+                special, special_idx = f.readline().strip().split()
+                special_tokens[special] = int(special_idx)
+            # read the merges
+            for line in f:
+                idx1, idx2 = map(int, line.split())
+                merges[(idx1, idx2)] = idx
+                idx += 1
+        self.merges = merges
+        self.special_tokens = special_tokens
+        self.vocab = self._build_vocab()

turbobpe-0.1.0/src/turbobpe/regex.py

@@ -0,0 +1,185 @@
+"""
+Minimal (byte-level) Byte Pair Encoding tokenizer.
+
+Algorithmically follows along the GPT tokenizer:
+https://github.com/openai/gpt-2/blob/master/src/encoder.py
+
+Unlike BasicTokenizer:
+- RegexTokenizer handles an optional regex splitting pattern.
+- RegexTokenizer handles optional special tokens.
+"""
+
+import regex as re
+from .base import Tokenizer, find_overlapping_cases, filter_top_pairs
+try:
+    from turbobpe.utils import get_stats, merge, batch_merge
+    print("Tokenizer initialized (Acceleration: Enabled)")
+except ImportError:
+    from .base import get_stats, merge, batch_merge
+    print("Tokenizer initialized (Acceleration: Disabled)")
+    print("Warning: C-accelerated backend not found. Performance may be degraded.\nEnsure 'turbobpe' is installed for optimal performance.")
+
+
+# the main GPT text split patterns, see
+# https://github.com/openai/tiktoken/blob/main/tiktoken_ext/openai_public.py
+GPT4_SPLIT_PATTERN = r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+"""
+
+
+class RegexTokenizer(Tokenizer):
+
+    def __init__(self, pattern=None):
+        """
+        - pattern: optional string to override the default (GPT-4 split pattern)
+        - special_tokens: str -> int dictionary of special tokens
+          example: {'<|endoftext|>': 100257}
+        """
+        super().__init__()
+        self.pattern = GPT4_SPLIT_PATTERN
+        self.compiled_pattern = re.compile(self.pattern)
+        self.special_tokens = {}
+        self.inverse_special_tokens = {}
+
+    def train(self, text, vocab_size, batch_size = 10, verbose=False):
+        assert vocab_size >= 256
+        num_merges = vocab_size - 256
+
+        # split the text up into text chunks
+        text_chunks = re.findall(self.compiled_pattern, text)
+
+        # input text preprocessing
+        ids = [list(ch.encode("utf-8")) for ch in text_chunks]
+
+        # iteratively merge the most common pairs to create new tokens
+        merges = {} # (int, int) -> int
+        vocab = {idx: bytes([idx]) for idx in range(256)} # idx -> bytes
+        s = 256
+        while s - 256 < num_merges:
+            # count the number of times every consecutive pair appears
+            stats = {}
+            for chunk_ids in ids:
+                # passing in stats will update it in place, adding up counts
+                get_stats(chunk_ids, stats)
+            # pairs we attempt to merge at once (e.g., 10) with highest occurance
+            top_pairs = sorted(stats.items(), key=lambda x: x[1], reverse=True)[:batch_size]
+            top_pairs = [pair[0] for pair in top_pairs]
+            # Filter out structural chain overlaps
+            overlaps = find_overlapping_cases(top_pairs)
+            if len(overlaps)>0:
+                top_pairs = filter_top_pairs(top_pairs, overlaps)
+                
+            # Truncate top_pairs if it would overshoot the exact num_merges limit
+            remaining_slots = num_merges - (s - 256)
+            if len(top_pairs) > remaining_slots:
+                top_pairs = top_pairs[:remaining_slots]
+                
+            # mint a new token: assign it the next available id
+            idx = [s + j for j in range(len(top_pairs))]
+            s += len(idx) 
+            # replace all occurrences of pair in ids with idx
+            ids = [res for chunk_ids in ids if (res := batch_merge(chunk_ids, top_pairs, idx)) is not None]   
+            
+            # save the merge and print details
+            for pair, new_idx in zip(top_pairs, idx):
+                merges[pair] = new_idx
+                vocab[new_idx] = vocab[pair[0]] + vocab[pair[1]]
+                
+                if verbose:
+                    print(f"merge {new_idx-256}: {pair} -> {new_idx} ({vocab[new_idx]}) had {stats[pair]} occurrences")
+                
+        # save class variables
+        self.merges = merges # used in encode()
+        self.vocab = vocab   # used in decode()
+
+    def register_special_tokens(self, special_tokens):
+        # special_tokens is a dictionary of str -> int
+        # example: {"<|endoftext|>": 100257}
+        self.special_tokens = special_tokens
+        self.inverse_special_tokens = {v: k for k, v in special_tokens.items()}
+
+    def decode(self, ids):
+        # given ids (list of integers), return Python string
+        part_bytes = []
+        for idx in ids:
+            if idx in self.vocab:
+                part_bytes.append(self.vocab[idx])
+            elif idx in self.inverse_special_tokens:
+                part_bytes.append(self.inverse_special_tokens[idx].encode("utf-8"))
+            else:
+                raise ValueError(f"invalid token id: {idx}")
+        text_bytes = b"".join(part_bytes)
+        text = text_bytes.decode("utf-8", errors="replace")
+        return text
+
+    def _encode_chunk(self, text_bytes):
+        # return the token ids
+        # let's begin. first, convert all bytes to integers in range 0..255
+        ids = list(text_bytes)
+        while len(ids) >= 2:
+            # find the pair with the lowest merge index
+            stats = get_stats(ids)
+            pair = min(stats, key=lambda p: self.merges.get(p, float("inf")))
+            # subtle: if there are no more merges available, the key will
+            # result in an inf for every single pair, and the min will be
+            # just the first pair in the list, arbitrarily
+            # we can detect this terminating case by a membership check
+            if pair not in self.merges:
+                break # nothing else can be merged anymore
+            # otherwise let's merge the best pair (lowest merge index)
+            idx = self.merges[pair]
+            ids = merge(ids, pair, idx)
+        return ids
+
+    def encode_ordinary(self, text):
+        """Encoding that ignores any special tokens."""
+        # split text into chunks of text by categories defined in regex pattern
+        text_chunks = re.findall(self.compiled_pattern, text)
+        # all chunks of text are encoded separately, then results are joined
+        ids = []
+        for chunk in text_chunks:
+            chunk_bytes = chunk.encode("utf-8") # raw bytes
+            chunk_ids = self._encode_chunk(chunk_bytes)
+            ids.extend(chunk_ids)
+        return ids
+
+    def encode(self, text, allowed_special="none_raise"):
+        """
+        Unlike encode_ordinary, this function handles special tokens.
+        allowed_special: can be "all"|"none"|"none_raise" or a custom set of special tokens
+        if none_raise, then an error is raised if any special token is encountered in text
+        this is the default tiktoken behavior right now as well
+        any other behavior is either annoying, or a major footgun
+        """
+        # decode the user desire w.r.t. handling of special tokens
+        special = None
+        if allowed_special == "all":
+            special = self.special_tokens
+        elif allowed_special == "none":
+            special = {}
+        elif allowed_special == "none_raise":
+            special = {}
+            assert all(token not in text for token in self.special_tokens)
+        elif isinstance(allowed_special, set):
+            special = {k: v for k, v in self.special_tokens.items() if k in allowed_special}
+        else:
+            raise ValueError(f"allowed_special={allowed_special} not understood")
+        if not special:
+            # shortcut: if no special tokens, just use the ordinary encoding
+            return self.encode_ordinary(text)
+        # otherwise, we have to be careful with potential special tokens in text
+        # we handle special tokens by splitting the text
+        # based on the occurrence of any exact match with any of the special tokens
+        # we can use re.split for this. note that surrounding the pattern with ()
+        # makes it into a capturing group, so the special tokens will be included
+        special_pattern = "(" + "|".join(re.escape(k) for k in special) + ")"
+        special_chunks = re.split(special_pattern, text)
+        # now all the special characters are separated from the rest of the text
+        # all chunks of text are encoded separately, then results are joined
+        ids = []
+        for part in special_chunks:
+            if part in special:
+                # this is a special token, encode it separately as a special case
+                ids.append(special[part])
+            else:
+                # this is an ordinary sequence, encode it normally
+                ids.extend(self.encode_ordinary(part))
+        return ids