skl-mindforge 0.1.0__tar.gz

skl_mindforge-0.1.0/PKG-INFO

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: skl_mindforge
+Version: 0.1.0
+Requires-Dist: tokenizers
+Dynamic: requires-dist

skl_mindforge-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

skl_mindforge-0.1.0/setup.py

@@ -0,0 +1,11 @@
+
+from setuptools import setup, find_packages
+
+setup(
+    name="skl_mindforge",
+    version="0.1.0",
+    packages=find_packages(),
+    include_package_data=True,
+    package_data={'skl_mindforge': ['*.json']},
+    install_requires=['tokenizers'],
+)

skl_mindforge-0.1.0/skl_mindforge/__init__.py

@@ -0,0 +1,32 @@
+
+import os
+from tokenizers import Tokenizer
+from tokenizers.processors import TemplateProcessing
+
+class ZenithTokenizer:
+    def __init__(self, model_filename="private_vocab_40k.json"):
+        # Locates the file inside the package folder
+        current_dir = os.path.dirname(__file__)
+        model_path = os.path.join(current_dir, model_filename)
+        
+        if not os.path.exists(model_path):
+            raise FileNotFoundError(f"Missing {model_filename} at {model_path}")
+            
+        self.tokenizer = Tokenizer.from_file(model_path)
+        
+        # Post-processor for the Chat/Assistant format
+        self.tokenizer.post_processor = TemplateProcessing(
+            single="<s> $A </s>",
+            pair="<s> $A </s> <s> $B </s>",
+            special_tokens=[("<s>", 1), ("</s>", 2)],
+        )
+        
+        self.vocab_size = self.tokenizer.get_vocab_size()
+
+    def encode(self, text):
+        return self.tokenizer.encode(text).ids
+
+    def decode(self, ids, skip_special_tokens=True):
+        return self.tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)
+
+zenith_tokenizer = ZenithTokenizer