py-tokenizer-ansh 0.0.3__tar.gz

py_tokenizer_ansh-0.0.3/.github/workflows/build.yml

@@ -0,0 +1,38 @@
+name: Build
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false  # <-- Ye sabse zaroori hai, isse jobs achanak cancel nahi honge
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          # C++ extension build karne ke liye zaroori tools add kiye
+          pip install build setuptools wheel pybind11 cmake
+
+      - name: Build wheel and sdist
+        run: python -m build
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: py_tokenizer-${{ matrix.os }}
+          path: dist/*

py_tokenizer_ansh-0.0.3/.github/workflows/publish.yml

@@ -0,0 +1,32 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+
+    # Note: permissions block hata diya hai kyunki hum API Token use kar rahe hain
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build scikit-build-core pybind11 cmake ninja
+
+      - name: Build package
+        run: python -m build --sdist
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

py_tokenizer_ansh-0.0.3/.gitignore

@@ -0,0 +1,47 @@
+# Build directories
+build/
+cmake-build-*/
+
+# CMake
+CMakeCache.txt
+CMakeFiles/
+cmake_install.cmake
+Makefile
+
+# Compiled libraries
+*.o
+*.obj
+*.a
+*.so
+*.dll
+*.dylib
+*.lib
+*.exe
+
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+
+# Distribution
+dist/
+*.egg-info/
+.eggs/
+wheelhouse/
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# IDE
+.vscode/
+.idea/
+
+# OS files
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log

py_tokenizer_ansh-0.0.3/CMakeLists.txt

@@ -0,0 +1,64 @@
+cmake_minimum_required(VERSION 3.15)
+
+project(py_tokenizer LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+include(FetchContent)
+
+FetchContent_Declare(
+    pybind11
+    GIT_REPOSITORY https://github.com/pybind/pybind11.git
+    GIT_TAG v3.0.1
+)
+
+FetchContent_MakeAvailable(pybind11)
+
+# ==========================
+# Common library
+# ==========================
+add_library(tokenizer_core STATIC
+    src/provider.cpp
+    src/splitter.cpp
+    src/loader.cpp
+    src/merger.cpp
+    src/freq_finder.cpp
+    engine/engine.cpp
+    engine/basic_engine.cpp
+)
+
+target_include_directories(tokenizer_core
+    PUBLIC
+        ${CMAKE_SOURCE_DIR}
+        ${CMAKE_SOURCE_DIR}/include
+)
+
+target_compile_features(tokenizer_core PUBLIC cxx_std_17)
+
+# ==========================
+# Executable
+# ==========================
+add_executable(tokenizer
+    main.cpp
+)
+
+target_link_libraries(tokenizer
+    PRIVATE tokenizer_core
+)
+
+# ==========================
+# Python module
+# ==========================
+pybind11_add_module(py_tokenizer
+    src/binding.cpp
+)
+
+target_link_libraries(py_tokenizer
+    PRIVATE tokenizer_core
+)
+
+install(TARGETS py_tokenizer
+    LIBRARY DESTINATION .
+)

py_tokenizer_ansh-0.0.3/MANIFEST.in

@@ -0,0 +1,19 @@
+include README.md
+include LICENSE
+include CMakeLists.txt
+
+recursive-include include *
+recursive-include src *
+recursive-include engine *
+
+include vocabulary.json
+include file_info.json
+
+global-exclude *.pyc
+global-exclude __pycache__
+global-exclude *.o
+global-exclude *.a
+global-exclude *.so
+global-exclude *.obj
+global-exclude *.dll
+global-exclude *.dylib

py_tokenizer_ansh-0.0.3/MIT

@@ -0,0 +1,17 @@
+MIT License
+Copyright (c) 2026 Ansh Raj
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

py_tokenizer_ansh-0.0.3/PKG-INFO

@@ -0,0 +1,187 @@
+Metadata-Version: 2.2
+Name: py_tokenizer_ansh
+Version: 0.0.3
+Summary: High-performance Byte Pair Encoding (BPE) tokenizer written in modern C++ with Python bindings.
+Keywords: tokenizer,bpe,byte-pair-encoding,nlp,llm,transformer,ai,machine-learning,cpp,pybind11,tokenization
+Author-Email: Ansh Raj <anshraj0000000001@gmail.com>
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: C++
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Project-URL: Homepage, https://anshstudios.pages.dev/portfolio
+Project-URL: Documentation, https://anshstudios.pages.dev/py_tokenizer
+Project-URL: Repository, https://github.com/ANSH-ins/py_tokenizer
+Project-URL: Issues, https://github.com/ANSH-ins/py_tokenizer/issues
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+<div align="center">
+<img src="https://res.cloudinary.com/dbmcddwjd/image/upload/v1783410838/file_00000000a2b471fba91bcdf2f3a398fb_t6gln5.png" alt="py_tokenizer Logo" width="250"/>
+# py_tokenizer
+**A High-Performance Byte Pair Encoding (BPE) Tokenizer for Modern AI Workflows.**
+Version
+Language
+Python
+Build
+Platform
+License
+</div>
+## Introduction
+py_tokenizer is a high-performance Byte Pair Encoding (BPE) tokenizer engineered entirely in modern C++17. It provides seamless Python bindings via pybind11, bridging the gap between low-level performance and high-level scripting convenience. Designed to be fast, memory-efficient, and highly modular, py_tokenizer is built to integrate effortlessly into advanced Artificial Intelligence and Machine Learning pipelines, specifically catering to Large Language Models (LLMs).
+## Key Features
+
+| Feature | Description |
+| :--- | :--- |
+| **High Performance** | Optimized core written in C++17 for maximum throughput and minimal latency. |
+| **Multi-threading** | Built-in support for concurrent processing to handle massive text corpora rapidly. |
+| **Memory Efficient** | Strict resource management ensures low memory overhead during training and encoding. |
+| **Python Integration** | Native Python bindings utilizing pybind11 for seamless ML framework integration. |
+| **Modular Architecture** | Clean separation of concerns allows for easy extension and customization of tokenization rules. |
+| **AI/LLM Ready** | Designed specifically for the stringent requirements of modern transformer architectures. |
+
+## Why py_tokenizer?
+Tokenization is the critical first step in natural language processing. While many tokenizers exist, py_tokenizer differentiates itself by focusing on raw compute efficiency without sacrificing usability. It is developed natively on mobile/embedded environments (Android via Termux), which enforces strict memory constraints and results in a highly optimized, lean codebase. Whether you are deploying on resource-constrained devices or scaling up on Linux servers, py_tokenizer delivers consistent, robust performance.
+## Project Architecture
+The project follows a standard wrapper architecture:
+ 1. **Core Library (C++)**: Handles all intensive computations, string manipulations, frequency counting, and merging operations.
+ 2. **Binding Layer (pybind11)**: Exposes the C++ classes and methods to Python, handling memory safety and type conversions natively.
+ 3. **Python API**: Provides a Pythonic interface, allowing users to import the tokenizer just like any native Python module.
+## Installation & Building from Source
+Currently, py_tokenizer supports Linux and Android (via Termux). Windows and macOS support are planned for future releases.
+### Prerequisites
+ * CMake (Version 3.10 or higher)
+ * C++ Compiler (GCC or Clang supporting C++17)
+ * Python 3.x
+ * pybind11 headers
+### Build Instructions
+ 1. Clone the repository:
+```bash
+git clone [https://github.com/ANSH-ins/py_tokenizer.git](https://github.com/ANSH-ins/py_tokenizer.git)
+cd py_tokenizer
+```
+ 2. Create a build directory and compile:
+```bash
+mkdir build
+cd build
+cmake ..
+make -j$(nproc)
+```
+ 3. Install the Python module:
+```bash
+python setup.py install
+```
+## Usage Examples
+### Python Integration
+```python
+import py_tokenizer
+# Initialize the BPE Tokenizer
+tokenizer = py_tokenizer.BPETokenizer()
+# Train the tokenizer on your dataset
+tokenizer.train(
+    file_path="dataset.txt", 
+    vocab_size=10000, 
+    special_tokens=["<PAD>", "<UNK>", "<BOS>", "<EOS>"]
+)
+# Encode text to tokens
+text = "Artificial intelligence relies on efficient tokenization."
+encoded_ids = tokenizer.encode(text)
+print("Encoded IDs:", encoded_ids)
+# Decode tokens back to text
+decoded_text = tokenizer.decode(encoded_ids)
+print("Decoded Text:", decoded_text)
+```
+### C++ Integration
+```cpp
+#include "py_tokenizer/bpe_tokenizer.hpp"
+#include <iostream>
+#include <vector>
+#include <string>
+int main() {
+    // Initialize
+    py_tokenizer::BPETokenizer tokenizer;
+    // Train
+    std::vector<std::string> special_tokens = {"<PAD>", "<UNK>", "<BOS>", "<EOS>"};
+    tokenizer.train("dataset.txt", 10000, special_tokens);
+    // Encode
+    std::string text = "Artificial intelligence relies on efficient tokenization.";
+    std::vector<int> encoded_ids = tokenizer.encode(text);
+    // Decode
+    std::string decoded_text = tokenizer.decode(encoded_ids);
+    std::cout << "Decoded: " << decoded_text << std::endl;
+    return 0;
+}
+```
+## Constructor Arguments
+When initializing or training the BPETokenizer, you can configure its behavior using the following parameters:
+
+| Argument | Type | Default | Description |
+| :--- | :--- | :--- | :--- |
+| vocab_size | int | Required | The target size of the vocabulary to be generated. |
+| file_path | string | Required | Path to the text corpus for training the BPE model. |
+| special_tokens | list / vector | [] | List of special structural tokens (e.g., <UNK>). |
+| num_threads | int | 1 | Number of threads to utilize during the training phase. |
+| casing | enum | PRESERVE | Rules for handling text capitalization (LOWERCASE, PRESERVE). |
+
+<details>
+<summary><b>View Project Folder Structure</b> (Click to Expand)</summary>
+```text
+py_tokenizer/
+├── CMakeLists.txt         # Build configuration
+├── README.md              # Project documentation
+├── setup.py               # Python installation script
+├── include/               # C++ Header files
+│   └── py_tokenizer/
+│       ├── bpe_tokenizer.hpp
+│       └── utils.hpp
+├── src/                   # C++ Source files
+│   ├── bpe_tokenizer.cpp
+│   └── utils.cpp
+├── bindings/              # pybind11 wrapper code
+│   └── bindings.cpp
+├── tests/                 # Unit tests (C++ and Python)
+│   ├── test_core.cpp
+│   └── test_tokenizer.py
+└── examples/              # Usage examples
+    ├── example.cpp
+    └── example.py
+```
+</details>
+<details>
+<summary><b>Roadmap & Future Plans</b> (Click to Expand)</summary>
+ * **Version 0.03**: Introduce streaming text tokenization for continuous data feeds.
+ * **Platform Expansion**: Official build support and CI/CD pipelines for Windows and macOS.
+ * **Advanced Normalization**: Implement customizable pre-tokenization steps (Unicode normalization, regex splitting).
+ * **GPU Acceleration**: Research feasibility of offloading frequency counts to CUDA/OpenCL.
+ * **Hugging Face Hub**: Native export formats to integrate directly with the transformers library.
+</details>
+## Performance Goals
+ 1. **Zero-Copy Serialization**: Minimizing memory duplication when passing standard library containers between C++ and Python.
+ 2. **Cache Locality**: Utilizing contiguous memory structures in C++ to improve CPU cache hit rates during byte-pair merges.
+ 3. **Scalability**: Ensuring the training complexity remains sub-quadratic relative to the vocabulary size.
+## Documentation & Website
+ * **Official Documentation**: py_tokenizer Docs
+ * **Developer Portfolio**: Ansh Studios
+## Developer Information
+
+| Detail | Information |
+| :--- | :--- |
+| **Developer Name** | Ansh Raj |
+| **Location** | India |
+| **Education** | Class 9th |
+| **Core Competency** | C++ |
+| **Languages** | C++, Python, JavaScript, HTML, CSS, Kotlin, Go |
+| **Environments** | Termux, Cxxdroid, Pydroid, Acode |
+
+## Contact & Links
+ * **Email**: anshraj0000000001@gmail.com
+ * **GitHub**: ANSH-ins
+## Contributing
+Contributions to py_tokenizer are highly encouraged. Please follow standard open-source workflows: fork the repository, create a feature branch, commit your changes with clear messages, and open a Pull Request. Ensure that all tests pass and that your code adheres to modern C++ conventions.
+## Acknowledgements
+Special thanks to the open-source community, the maintainers of pybind11, and the developers behind Android coding environments like Termux and Cxxdroid that made the development of this project possible on mobile devices.
+## License
+This project is licensed under the **MIT License**. See the LICENSE file in the repository for full details. You are free to use, modify, and distribute this software in both open-source and commercial projects.

py_tokenizer_ansh-0.0.3/README.md

@@ -0,0 +1,166 @@
+<div align="center">
+<img src="https://res.cloudinary.com/dbmcddwjd/image/upload/v1783410838/file_00000000a2b471fba91bcdf2f3a398fb_t6gln5.png" alt="py_tokenizer Logo" width="250"/>
+# py_tokenizer
+**A High-Performance Byte Pair Encoding (BPE) Tokenizer for Modern AI Workflows.**
+Version
+Language
+Python
+Build
+Platform
+License
+</div>
+## Introduction
+py_tokenizer is a high-performance Byte Pair Encoding (BPE) tokenizer engineered entirely in modern C++17. It provides seamless Python bindings via pybind11, bridging the gap between low-level performance and high-level scripting convenience. Designed to be fast, memory-efficient, and highly modular, py_tokenizer is built to integrate effortlessly into advanced Artificial Intelligence and Machine Learning pipelines, specifically catering to Large Language Models (LLMs).
+## Key Features
+
+| Feature | Description |
+| :--- | :--- |
+| **High Performance** | Optimized core written in C++17 for maximum throughput and minimal latency. |
+| **Multi-threading** | Built-in support for concurrent processing to handle massive text corpora rapidly. |
+| **Memory Efficient** | Strict resource management ensures low memory overhead during training and encoding. |
+| **Python Integration** | Native Python bindings utilizing pybind11 for seamless ML framework integration. |
+| **Modular Architecture** | Clean separation of concerns allows for easy extension and customization of tokenization rules. |
+| **AI/LLM Ready** | Designed specifically for the stringent requirements of modern transformer architectures. |
+
+## Why py_tokenizer?
+Tokenization is the critical first step in natural language processing. While many tokenizers exist, py_tokenizer differentiates itself by focusing on raw compute efficiency without sacrificing usability. It is developed natively on mobile/embedded environments (Android via Termux), which enforces strict memory constraints and results in a highly optimized, lean codebase. Whether you are deploying on resource-constrained devices or scaling up on Linux servers, py_tokenizer delivers consistent, robust performance.
+## Project Architecture
+The project follows a standard wrapper architecture:
+ 1. **Core Library (C++)**: Handles all intensive computations, string manipulations, frequency counting, and merging operations.
+ 2. **Binding Layer (pybind11)**: Exposes the C++ classes and methods to Python, handling memory safety and type conversions natively.
+ 3. **Python API**: Provides a Pythonic interface, allowing users to import the tokenizer just like any native Python module.
+## Installation & Building from Source
+Currently, py_tokenizer supports Linux and Android (via Termux). Windows and macOS support are planned for future releases.
+### Prerequisites
+ * CMake (Version 3.10 or higher)
+ * C++ Compiler (GCC or Clang supporting C++17)
+ * Python 3.x
+ * pybind11 headers
+### Build Instructions
+ 1. Clone the repository:
+```bash
+git clone [https://github.com/ANSH-ins/py_tokenizer.git](https://github.com/ANSH-ins/py_tokenizer.git)
+cd py_tokenizer
+```
+ 2. Create a build directory and compile:
+```bash
+mkdir build
+cd build
+cmake ..
+make -j$(nproc)
+```
+ 3. Install the Python module:
+```bash
+python setup.py install
+```
+## Usage Examples
+### Python Integration
+```python
+import py_tokenizer
+# Initialize the BPE Tokenizer
+tokenizer = py_tokenizer.BPETokenizer()
+# Train the tokenizer on your dataset
+tokenizer.train(
+    file_path="dataset.txt", 
+    vocab_size=10000, 
+    special_tokens=["<PAD>", "<UNK>", "<BOS>", "<EOS>"]
+)
+# Encode text to tokens
+text = "Artificial intelligence relies on efficient tokenization."
+encoded_ids = tokenizer.encode(text)
+print("Encoded IDs:", encoded_ids)
+# Decode tokens back to text
+decoded_text = tokenizer.decode(encoded_ids)
+print("Decoded Text:", decoded_text)
+```
+### C++ Integration
+```cpp
+#include "py_tokenizer/bpe_tokenizer.hpp"
+#include <iostream>
+#include <vector>
+#include <string>
+int main() {
+    // Initialize
+    py_tokenizer::BPETokenizer tokenizer;
+    // Train
+    std::vector<std::string> special_tokens = {"<PAD>", "<UNK>", "<BOS>", "<EOS>"};
+    tokenizer.train("dataset.txt", 10000, special_tokens);
+    // Encode
+    std::string text = "Artificial intelligence relies on efficient tokenization.";
+    std::vector<int> encoded_ids = tokenizer.encode(text);
+    // Decode
+    std::string decoded_text = tokenizer.decode(encoded_ids);
+    std::cout << "Decoded: " << decoded_text << std::endl;
+    return 0;
+}
+```
+## Constructor Arguments
+When initializing or training the BPETokenizer, you can configure its behavior using the following parameters:
+
+| Argument | Type | Default | Description |
+| :--- | :--- | :--- | :--- |
+| vocab_size | int | Required | The target size of the vocabulary to be generated. |
+| file_path | string | Required | Path to the text corpus for training the BPE model. |
+| special_tokens | list / vector | [] | List of special structural tokens (e.g., <UNK>). |
+| num_threads | int | 1 | Number of threads to utilize during the training phase. |
+| casing | enum | PRESERVE | Rules for handling text capitalization (LOWERCASE, PRESERVE). |
+
+<details>
+<summary><b>View Project Folder Structure</b> (Click to Expand)</summary>
+```text
+py_tokenizer/
+├── CMakeLists.txt         # Build configuration
+├── README.md              # Project documentation
+├── setup.py               # Python installation script
+├── include/               # C++ Header files
+│   └── py_tokenizer/
+│       ├── bpe_tokenizer.hpp
+│       └── utils.hpp
+├── src/                   # C++ Source files
+│   ├── bpe_tokenizer.cpp
+│   └── utils.cpp
+├── bindings/              # pybind11 wrapper code
+│   └── bindings.cpp
+├── tests/                 # Unit tests (C++ and Python)
+│   ├── test_core.cpp
+│   └── test_tokenizer.py
+└── examples/              # Usage examples
+    ├── example.cpp
+    └── example.py
+```
+</details>
+<details>
+<summary><b>Roadmap & Future Plans</b> (Click to Expand)</summary>
+ * **Version 0.03**: Introduce streaming text tokenization for continuous data feeds.
+ * **Platform Expansion**: Official build support and CI/CD pipelines for Windows and macOS.
+ * **Advanced Normalization**: Implement customizable pre-tokenization steps (Unicode normalization, regex splitting).
+ * **GPU Acceleration**: Research feasibility of offloading frequency counts to CUDA/OpenCL.
+ * **Hugging Face Hub**: Native export formats to integrate directly with the transformers library.
+</details>
+## Performance Goals
+ 1. **Zero-Copy Serialization**: Minimizing memory duplication when passing standard library containers between C++ and Python.
+ 2. **Cache Locality**: Utilizing contiguous memory structures in C++ to improve CPU cache hit rates during byte-pair merges.
+ 3. **Scalability**: Ensuring the training complexity remains sub-quadratic relative to the vocabulary size.
+## Documentation & Website
+ * **Official Documentation**: py_tokenizer Docs
+ * **Developer Portfolio**: Ansh Studios
+## Developer Information
+
+| Detail | Information |
+| :--- | :--- |
+| **Developer Name** | Ansh Raj |
+| **Location** | India |
+| **Education** | Class 9th |
+| **Core Competency** | C++ |
+| **Languages** | C++, Python, JavaScript, HTML, CSS, Kotlin, Go |
+| **Environments** | Termux, Cxxdroid, Pydroid, Acode |
+
+## Contact & Links
+ * **Email**: anshraj0000000001@gmail.com
+ * **GitHub**: ANSH-ins
+## Contributing
+Contributions to py_tokenizer are highly encouraged. Please follow standard open-source workflows: fork the repository, create a feature branch, commit your changes with clear messages, and open a Pull Request. Ensure that all tests pass and that your code adheres to modern C++ conventions.
+## Acknowledgements
+Special thanks to the open-source community, the maintainers of pybind11, and the developers behind Android coding environments like Termux and Cxxdroid that made the development of this project possible on mobile devices.
+## License
+This project is licensed under the **MIT License**. See the LICENSE file in the repository for full details. You are free to use, modify, and distribute this software in both open-source and commercial projects.

py_tokenizer_ansh-0.0.3/engine/basic_engine.cpp

@@ -0,0 +1,33 @@
+#include "basic_engine.h"
+#include <iostream>
+
+basic_engine::basic_engine(const std::string& sentance, bool normilize, size_t epoch): sentance(sentance),normilize(normilize), epoch(epoch){
+    
+    splitter SPLITTER(sentance, normilize, false);
+    std::vector<std::vector<std::string>> tokens = SPLITTER.get_splited_tokens();
+    
+    size_t vocab = 0;
+    std::unordered_map<std::pair<std::string, std::string>,uint64_t, Pairhash> best_freq_pair;
+    while(epoch != 0){
+        
+        finder FINDER(tokens);
+        
+        best_freq_pair = FINDER.get_freq_pair();
+        
+        merger MERGER(best_freq_pair, tokens);
+        tokens.clear();
+        tokens = MERGER.get_merged_tokens();
+        vocab = MERGER.get_vocab();
+       
+        epoch--;
+    }
+    std::cout<<"\nvocab size: "<<vocab<<"\n"<<std::endl;
+    size_t id = 0;
+    for(const auto& vector_box : tokens){
+        for(const auto& word : vector_box){
+            std::cout<<word<<",";
+        }
+        std::cout<<" : "<<id<<std::endl;
+        id++;
+    }
+}

py_tokenizer_ansh-0.0.3/engine/basic_engine.h

@@ -0,0 +1,18 @@
+#pragma once
+#include "tokenizer/splitter.h"
+#include "tokenizer/merger.h"
+#include "tokenizer/freq_finder.h"
+#include <string>
+#include <vector>
+#include <unordered_map>
+
+
+class basic_engine{
+    private:
+    std::string sentance;
+    bool normilize;
+    size_t epoch;
+    
+    public:
+    basic_engine(const std::string& sentance, bool normilize = false, size_t epoch = 5);
+};

py_tokenizer_ansh-0.0.3/engine/engine.cpp

@@ -0,0 +1,42 @@
+#include "engine.h"
+#include <iostream>
+#include <cstdint>
+#include <unordered_map>
+#include <utility>
+
+engine::engine(const std::string& file_path, const size_t chunk_size, bool get_file_info, bool normilize, size_t epoch, size_t using_cpu, bool use_thread): file_path(file_path),chunk_size(chunk_size), get_file_info(get_file_info), normilize(normilize), epoch(epoch), using_cpu(using_cpu), use_thread(use_thread){
+    
+    std::vector<std::vector<std::string>> tokens;
+    
+    std::unordered_map<std::pair<std::string, std::string>, uint64_t, Pairhash> best_freq_pair;
+    
+    std::string sentance;
+    
+    loader DataLoader(file_path, chunk_size, normilize);
+    
+    sentance = DataLoader.get_sentance();
+    
+    splitter SPLITTER(sentance, using_cpu,use_thread);
+    
+    if(use_thread){
+        SPLITTER.thread_runner();
+    }
+    tokens = SPLITTER.get_splited_tokens();
+    
+    std::vector<file_details> details = DataLoader.detail_provider();
+    
+    while(epoch>0){
+        finder FINDER(tokens);
+        best_freq_pair = FINDER.get_freq_pair();
+        
+        merger MERGER(best_freq_pair, tokens);
+        
+        
+        tokens = MERGER.get_merged_tokens();
+        
+        vocab = MERGER.get_vocab();
+        epoch--;
+    }
+    provider PROVIDER(tokens, details, get_file_info, vocab);
+    
+};