Sujit-Tokenizer 1.0.0__py3-none-any.whl

Sujit_Tokenizer/__init__.py

@@ -0,0 +1,3 @@
+from .tokenizer import CustomByteLevelBPETokenizer
+
+__all__ = ["CustomByteLevelBPETokenizer"]

Sujit_Tokenizer/tokenizer.py

@@ -0,0 +1,374 @@
+from __future__ import annotations
+
+import pickle
+from collections import defaultdict
+from typing import Dict, List, Tuple
+
+
+class CustomByteLevelBPETokenizer:
+    """
+    Custom Byte-Level BPE Tokenizer.
+
+    Features:
+    - Byte-level tokenization
+    - BPE training
+    - Encoding / Decoding
+    - Save / Load model
+    """
+
+    def __init__(self, vocab_size: int = 1000):
+
+        if vocab_size < 260:
+            raise ValueError(
+                "vocab_size must be >= 260 "
+                "(256 bytes + 4 special tokens)"
+            )
+
+        self.vocab_size = vocab_size
+
+        self.special_tokens = {
+            "<PAD>": 0,
+            "<UNK>": 1,
+            "<BOS>": 2,
+            "<EOS>": 3,
+        }
+
+        self.vocab: Dict[bytes, int] = {}
+
+        # Initialize byte vocabulary
+        for i in range(256):
+            self.vocab[bytes([i])] = (
+                i + len(self.special_tokens)
+            )
+
+        # Add special tokens
+        for token, token_id in self.special_tokens.items():
+            self.vocab[token.encode()] = token_id
+
+        self.inverse_vocab: Dict[int, bytes] = {
+            idx: token
+            for token, idx in self.vocab.items()
+        }
+
+        self.merges: List[
+            Tuple[bytes, bytes]
+        ] = []
+
+    def text_to_bytes(
+        self,
+        text: str
+    ) -> List[int]:
+
+        if not isinstance(text, str):
+            raise TypeError(
+                "text must be a string"
+            )
+
+        return list(text.encode("utf-8"))
+
+    def get_pair_frequencies(
+        self,
+        corpus: List[List[bytes]]
+    ) -> Dict[Tuple[bytes, bytes], int]:
+
+        pair_freq = defaultdict(int)
+
+        for tokens in corpus:
+            for i in range(len(tokens) - 1):
+                pair = (
+                    tokens[i],
+                    tokens[i + 1]
+                )
+                pair_freq[pair] += 1
+
+        return pair_freq
+
+    def merge_pair(
+        self,
+        pair: Tuple[bytes, bytes],
+        corpus: List[List[bytes]]
+    ) -> List[List[bytes]]:
+
+        updated_corpus = []
+
+        for tokens in corpus:
+
+            merged_tokens = []
+            i = 0
+
+            while i < len(tokens):
+
+                if (
+                    i < len(tokens) - 1
+                    and tokens[i] == pair[0]
+                    and tokens[i + 1] == pair[1]
+                ):
+                    merged_tokens.append(
+                        tokens[i] + tokens[i + 1]
+                    )
+                    i += 2
+
+                else:
+                    merged_tokens.append(tokens[i])
+                    i += 1
+
+            updated_corpus.append(
+                merged_tokens
+            )
+
+        return updated_corpus
+
+    def train(
+        self,
+        corpus: List[str]
+    ) -> None:
+
+        if not corpus:
+            raise ValueError(
+                "Corpus cannot be empty"
+            )
+
+        tokenized_corpus = []
+
+        for text in corpus:
+            tokens = [
+                bytes([b])
+                for b in self.text_to_bytes(text)
+            ]
+            tokenized_corpus.append(tokens)
+
+        num_merges = (
+            self.vocab_size -
+            len(self.vocab)
+        )
+
+        print("=" * 50)
+        print("Training Byte-Level BPE")
+        print("=" * 50)
+
+        for step in range(
+            max(0, num_merges)
+        ):
+
+            pair_freq = (
+                self.get_pair_frequencies(
+                    tokenized_corpus
+                )
+            )
+
+            if not pair_freq:
+                break
+
+            best_pair = max(
+                pair_freq,
+                key=pair_freq.get
+            )
+
+            merged_token = (
+                best_pair[0]
+                + best_pair[1]
+            )
+
+            self.merges.append(
+                best_pair
+            )
+
+            if (
+                merged_token
+                not in self.vocab
+            ):
+                token_id = len(
+                    self.vocab
+                )
+
+                self.vocab[
+                    merged_token
+                ] = token_id
+
+                self.inverse_vocab[
+                    token_id
+                ] = merged_token
+
+            tokenized_corpus = (
+                self.merge_pair(
+                    best_pair,
+                    tokenized_corpus
+                )
+            )
+
+            if (
+                step + 1
+            ) % 100 == 0:
+                print(
+                    f"Completed "
+                    f"{step + 1} merges"
+                )
+
+        print(
+            f"Training Complete. "
+            f"Vocabulary Size: "
+            f"{len(self.vocab)}"
+        )
+
+    def apply_bpe(
+        self,
+        byte_tokens: List[int]
+    ) -> List[bytes]:
+
+        tokens = [
+            bytes([b])
+            for b in byte_tokens
+        ]
+
+        for pair in self.merges:
+
+            merged_tokens = []
+            i = 0
+
+            while i < len(tokens):
+
+                if (
+                    i < len(tokens) - 1
+                    and tokens[i] == pair[0]
+                    and tokens[i + 1] == pair[1]
+                ):
+                    merged_tokens.append(
+                        tokens[i]
+                        + tokens[i + 1]
+                    )
+                    i += 2
+
+                else:
+                    merged_tokens.append(
+                        tokens[i]
+                    )
+                    i += 1
+
+            tokens = merged_tokens
+
+        return tokens
+
+    def encode(
+        self,
+        text: str
+    ) -> List[int]:
+
+        byte_tokens = (
+            self.text_to_bytes(text)
+        )
+
+        bpe_tokens = self.apply_bpe(
+            byte_tokens
+        )
+
+        token_ids = [
+            self.special_tokens[
+                "<BOS>"
+            ]
+        ]
+
+        for token in bpe_tokens:
+
+            token_ids.append(
+                self.vocab.get(
+                    token,
+                    self.special_tokens[
+                        "<UNK>"
+                    ]
+                )
+            )
+
+        token_ids.append(
+            self.special_tokens[
+                "<EOS>"
+            ]
+        )
+
+        return token_ids
+
+    def decode(
+        self,
+        token_ids: List[int]
+    ) -> str:
+
+        byte_stream = b""
+
+        for token_id in token_ids:
+
+            token = (
+                self.inverse_vocab.get(
+                    token_id
+                )
+            )
+
+            if token is None:
+                continue
+
+            if token in {
+                b"<PAD>",
+                b"<UNK>",
+                b"<BOS>",
+                b"<EOS>",
+            }:
+                continue
+
+            byte_stream += token
+
+        return byte_stream.decode(
+            "utf-8",
+            errors="replace"
+        )
+
+    def save_model(
+        self,
+        path: str = "tokenizer.model"
+    ) -> None:
+
+        model_data = {
+            "vocab": self.vocab,
+            "inverse_vocab":
+                self.inverse_vocab,
+            "merges": self.merges,
+            "special_tokens":
+                self.special_tokens,
+            "vocab_size":
+                self.vocab_size,
+        }
+
+        with open(path, "wb") as f:
+            pickle.dump(
+                model_data,
+                f
+            )
+
+        print(
+            f"Tokenizer saved to "
+            f"{path}"
+        )
+
+    def load_model(
+        self,
+        path: str = "tokenizer.model"
+    ) -> None:
+
+        with open(path, "rb") as f:
+            model_data = pickle.load(f)
+
+        self.vocab = model_data["vocab"]
+        self.inverse_vocab = (
+            model_data["inverse_vocab"]
+        )
+        self.merges = model_data["merges"]
+        self.special_tokens = (
+            model_data[
+                "special_tokens"
+            ]
+        )
+        self.vocab_size = (
+            model_data["vocab_size"]
+        )
+
+        print(
+            f"Tokenizer loaded from "
+            f"{path}"
+        )

sujit_tokenizer-1.0.0.dist-info/METADATA

@@ -0,0 +1,233 @@
+Metadata-Version: 2.4
+Name: Sujit-Tokenizer
+Version: 1.0.0
+Summary: Custom Byte-Level BPE Tokenizer built from scratch in Python
+Home-page: https://github.com/iamsspm07/Sujit-Tokenizer
+Author: Sujit Maity
+Author-email: Sujit Maity <sujitmaity.in@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/iamsspm07/Sujit-Tokenizer
+Project-URL: Repository, https://github.com/iamsspm07/Sujit-Tokenizer
+Project-URL: Issues, https://github.com/iamsspm07/Sujit-Tokenizer/issues
+Keywords: tokenizer,bpe,byte-pair-encoding,nlp,llm,machine-learning,artificial-intelligence
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Dynamic: author
+Dynamic: home-page
+Dynamic: requires-python
+
+# Sujit-Tokenizer
+
+A custom Byte-Level BPE (Byte Pair Encoding) Tokenizer implemented from scratch in Python.
+
+## Features
+
+* Byte-level tokenization
+* Custom BPE training
+* Text encoding and decoding
+* Save and load tokenizer models
+* UTF-8 support
+* Lightweight and dependency-free
+
+## Installation
+
+### Clone Repository
+
+```bash
+git clone https://github.com/your-username/Sujit-Tokenizer.git
+cd Sujit-Tokenizer
+```
+
+### Install Package
+
+```bash
+pip install -e .
+```
+
+## Project Structure
+
+```text
+Sujit-Tokenizer/
+│
+├── Sujit_Tokenizer/
+│   ├── __init__.py
+│   └── tokenizer.py
+│
+├── corpus.txt
+├── train_and_test.py
+│
+├── README.md
+├── setup.py
+├── pyproject.toml
+└── LICENSE
+```
+
+## Quick Start
+
+### Import Tokenizer
+
+```python
+from Sujit_Tokenizer import CustomByteLevelBPETokenizer
+```
+
+### Train Tokenizer
+
+```python
+corpus = [
+    "Transformers are amazing.",
+    "Machine Learning is powerful.",
+    "Python is widely used in AI."
+]
+
+tokenizer = CustomByteLevelBPETokenizer(
+    vocab_size=1000
+)
+
+tokenizer.train(corpus)
+```
+
+### Save Model
+
+```python
+tokenizer.save_model(
+    "tokenizer.model"
+)
+```
+
+### Load Model
+
+```python
+tokenizer = CustomByteLevelBPETokenizer()
+
+tokenizer.load_model(
+    "tokenizer.model"
+)
+```
+
+### Encode Text
+
+```python
+encoded = tokenizer.encode(
+    "Transformers use attention."
+)
+
+print(encoded)
+```
+
+Example Output:
+
+```python
+[2, 451, 723, 812, 3]
+```
+
+### Decode Text
+
+```python
+decoded = tokenizer.decode(
+    encoded
+)
+
+print(decoded)
+```
+
+Output:
+
+```text
+Transformers use attention.
+```
+
+## Training Workflow
+
+```text
+Corpus
+   ↓
+Byte Conversion
+   ↓
+Pair Frequency Counting
+   ↓
+BPE Merging
+   ↓
+Vocabulary Construction
+   ↓
+Tokenizer Model
+```
+
+## Special Tokens
+
+| Token | ID |
+| ----- | -- |
+| <PAD> | 0  |
+| <UNK> | 1  |
+| <BOS> | 2  |
+| <EOS> | 3  |
+
+## Example
+
+```python
+from Sujit_Tokenizer import CustomByteLevelBPETokenizer
+
+tokenizer = CustomByteLevelBPETokenizer(
+    vocab_size=1000
+)
+
+corpus = [
+    "Artificial Intelligence",
+    "Machine Learning",
+    "Deep Learning"
+]
+
+tokenizer.train(corpus)
+
+tokenizer.save_model(
+    "tokenizer.model"
+)
+
+tokenizer.load_model(
+    "tokenizer.model"
+)
+
+text = "Machine Learning"
+
+encoded = tokenizer.encode(text)
+print(encoded)
+
+decoded = tokenizer.decode(encoded)
+print(decoded)
+```
+
+## Use Cases
+
+* NLP experiments
+* Tokenization research
+* Educational projects
+* Understanding BPE internals
+* Building custom language models
+* Learning tokenizer architecture
+
+## Future Enhancements
+
+* Faster BPE training
+* WordPiece tokenizer
+* SentencePiece tokenizer
+* Parallel processing
+* Vocabulary statistics
+* Token frequency analysis
+* Hugging Face compatibility
+
+## Author
+
+Sujit Maity
+
+## License
+
+MIT License

sujit_tokenizer-1.0.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+Sujit_Tokenizer/__init__.py,sha256=jtOimMzgoBHmBIKSgYkr2clMNcEFKVSOfuC9UbAERxA,95
+Sujit_Tokenizer/tokenizer.py,sha256=MiitxxpwKPCDxCMj8ZbD0lYbUCgs9DkNflXOiTXvP3A,8422
+sujit_tokenizer-1.0.0.dist-info/METADATA,sha256=zI5sQveoSMHBq-laF_Dei05nVStt4qeSHKQl4MA0TSI,4208
+sujit_tokenizer-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+sujit_tokenizer-1.0.0.dist-info/top_level.txt,sha256=hCBMZDYnvCgIsMrvPaAJNEHh17ktggMIHpVe0Z_sgnc,16
+sujit_tokenizer-1.0.0.dist-info/RECORD,,

sujit_tokenizer-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sujit_tokenizer-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+Sujit_Tokenizer