igbot-base 0.0.19__tar.gz → 0.0.21__tar.gz

{igbot_base-0.0.19 → igbot_base-0.0.21}/PKG-INFO

@@ -1,8 +1,9 @@
 Metadata-Version: 2.4
 Name: igbot_base
-Version: 0.0.19
+Version: 0.0.21
 Summary: Base classes for igbot
 Author-email: Igor Kopeć <igor.kopec95@gmail.com>
 License: LGPL-3.0-or-later
 Requires-Python: >=3.12
 Requires-Dist: openai
+Requires-Dist: tiktoken

igbot_base-0.0.21/igbot_base/models.py

@@ -0,0 +1,36 @@
+from enum import Enum
+from openai import OpenAI
+
+from igbot_base.tokenizer import BaseTokenizer, OpenAiTokenizer
+
+
+class ModelInfo:
+
+    def __init__(self, name, client, tokenizer: BaseTokenizer, max_tokens):
+        self.__name = name
+        self.__client = client
+        self.__tokenizer = tokenizer
+        self.__max_tokens = max_tokens
+
+    def get_name(self):
+        return self.__name
+
+    def get_client(self):
+        return self.__client()
+
+    def get_tokenizer(self) -> BaseTokenizer:
+        return self.__tokenizer
+
+    def get_max_tokens(self):
+        return self.__max_tokens
+
+
+class Model(Enum):
+    OLLAMA_3_2_LOCAL = ModelInfo("llama3.2", lambda: OpenAI(base_url="http://localhost:11434/v1", api_key='ollama'),
+                                 OpenAiTokenizer("gpt-4o"), 128_000)
+    OPENAI_GPT_4o_MINI = ModelInfo("gpt-4o-mini", lambda: OpenAI(),
+                                   OpenAiTokenizer("gpt-4o"), 128_000)
+    OPENAI_GPT_4o = ModelInfo("gpt-4o", lambda: OpenAI(),
+                              OpenAiTokenizer("gpt-4o"), 128_000)
+    OPENAI_GPT_4o_MINI_JSON = ModelInfo("gpt-4o-mini-2024-07-18", lambda: OpenAI(),
+                                        OpenAiTokenizer("gpt-4o"), 128_000)

igbot_base-0.0.21/igbot_base/tokenizer.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+import tiktoken
+
+
+class BaseTokenizer(ABC):
+    @abstractmethod
+    def count_tokens(self, text: str):
+        pass
+
+
+class OpenAiTokenizer(BaseTokenizer):
+    def __init__(self, model_name):
+        self.__tokenizer = tiktoken.encoding_for_model(model_name)
+
+    def count_tokens(self, text: str):
+        return len(self.__tokenizer.encode(text))

{igbot_base-0.0.19 → igbot_base-0.0.21}/igbot_base.egg-info/PKG-INFO

@@ -1,8 +1,9 @@
 Metadata-Version: 2.4
 Name: igbot_base
-Version: 0.0.19
+Version: 0.0.21
 Summary: Base classes for igbot
 Author-email: Igor Kopeć <igor.kopec95@gmail.com>
 License: LGPL-3.0-or-later
 Requires-Python: >=3.12
 Requires-Dist: openai
+Requires-Dist: tiktoken

{igbot_base-0.0.19 → igbot_base-0.0.21}/igbot_base.egg-info/SOURCES.txt

@@ -12,6 +12,7 @@ igbot_base/persistable_memory.py
 igbot_base/prompt_template.py
 igbot_base/response_formats.py
 igbot_base/retriever.py
+igbot_base/tokenizer.py
 igbot_base/tool.py
 igbot_base/vectorstore.py
 igbot_base.egg-info/PKG-INFO

igbot_base-0.0.21/igbot_base.egg-info/requires.txt

@@ -0,0 +1,2 @@
+openai
+tiktoken

{igbot_base-0.0.19 → igbot_base-0.0.21}/pyproject.toml

@@ -4,13 +4,14 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "igbot_base"
-version = "0.0.19"
+version = "0.0.21"
 description = "Base classes for igbot"
 authors = [{name = "Igor Kopeć", email = "igor.kopec95@gmail.com"}]
 license={text="LGPL-3.0-or-later"}
 requires-python = ">=3.12"
 dependencies = [
-    "openai"
+    "openai",
+    "tiktoken"
 ]
 
 [tool.setuptools.packages.find]

igbot_base-0.0.19/igbot_base/models.py

@@ -1,22 +0,0 @@
-from enum import Enum
-from openai import OpenAI
-
-
-class ModelInfo:
-
-    def __init__(self, name, client):
-        self.__name = name
-        self.__client = client
-
-    def get_name(self):
-        return self.__name
-
-    def get_client(self):
-        return self.__client()
-
-
-class Model(Enum):
-    OLLAMA_3_2_LOCAL = ModelInfo("llama3.2", lambda: OpenAI(base_url="http://localhost:11434/v1", api_key='ollama'))
-    OPENAI_GPT_4o_MINI = ModelInfo("gpt-4o-mini", lambda: OpenAI())
-    OPENAI_GPT_4o = ModelInfo("gpt-4o", lambda: OpenAI())
-    OPENAI_GPT_4o_MINI_JSON = ModelInfo("gpt-4o-mini-2024-07-18", lambda: OpenAI())

igbot_base-0.0.19/igbot_base.egg-info/requires.txt

@@ -1 +0,0 @@
-openai