igbot-base 0.0.20__tar.gz → 0.0.22__tar.gz

{igbot_base-0.0.20 → igbot_base-0.0.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: igbot_base
-Version: 0.0.20
+Version: 0.0.22
 Summary: Base classes for igbot
 Author-email: Igor Kopeć <igor.kopec95@gmail.com>
 License: LGPL-3.0-or-later

{igbot_base-0.0.20 → igbot_base-0.0.22}/igbot_base/llmmemory.py

@@ -24,7 +24,7 @@ class LlmMemory(ABC):
         pass
 
     @abstractmethod
-    def append_tool(self, tool_call_id: str, content: str):
+    def append_tool_response(self, tool_call_id: str, content: str):
         pass
 
     @abstractmethod

{igbot_base-0.0.20 → igbot_base-0.0.22}/igbot_base/models.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from openai import OpenAI
 
-from tokenizer import BaseTokenizer, OpenAiTokenizer
+from igbot_base.tokenizer import BaseTokenizer, OpenAiTokenizer
 
 
 class ModelInfo:

igbot_base-0.0.22/igbot_base/tokenizer.py

@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+from typing import Union, Dict
+import json
+import tiktoken
+
+
+class BaseTokenizer(ABC):
+    @abstractmethod
+    def count_tokens(self, data: Union[str, Dict, None]) -> int:
+        pass
+
+
+class OpenAiTokenizer(BaseTokenizer):
+
+    def __init__(self, model_name):
+        self.__tokenizer = tiktoken.encoding_for_model(model_name)
+
+    def count_tokens(self, data: Union[str: Dict, None]) -> int:
+        if data is None:
+            return 0
+        elif isinstance(data, str):
+            return len(self.__tokenizer.encode(data))
+        elif isinstance(data, dict):
+            dict_as_str = json.dumps(data, separators=(",", ":"))
+            return len(self.__tokenizer.encode(dict_as_str))
+        else:
+            raise TypeError("Expected either a string, a dictionary, or None.")

{igbot_base-0.0.20 → igbot_base-0.0.22}/igbot_base.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: igbot_base
-Version: 0.0.20
+Version: 0.0.22
 Summary: Base classes for igbot
 Author-email: Igor Kopeć <igor.kopec95@gmail.com>
 License: LGPL-3.0-or-later

{igbot_base-0.0.20 → igbot_base-0.0.22}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "igbot_base"
-version = "0.0.20"
+version = "0.0.22"
 description = "Base classes for igbot"
 authors = [{name = "Igor Kopeć", email = "igor.kopec95@gmail.com"}]
 license={text="LGPL-3.0-or-later"}

igbot_base-0.0.20/igbot_base/tokenizer.py

@@ -1,16 +0,0 @@
-from abc import ABC, abstractmethod
-import tiktoken
-
-
-class BaseTokenizer(ABC):
-    @abstractmethod
-    def count_tokens(self, text: str):
-        pass
-
-
-class OpenAiTokenizer(BaseTokenizer):
-    def __init__(self, model_name):
-        self.__tokenizer = tiktoken.encoding_for_model(model_name)
-
-    def count_tokens(self, text: str):
-        return len(self.__tokenizer.encode(text))