igbot-base 0.0.22__tar.gz → 0.0.23__tar.gz

{igbot_base-0.0.22 → igbot_base-0.0.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: igbot_base
-Version: 0.0.22
+Version: 0.0.23
 Summary: Base classes for igbot
 Author-email: Igor Kopeć <igor.kopec95@gmail.com>
 License: LGPL-3.0-or-later

{igbot_base-0.0.22 → igbot_base-0.0.23}/igbot_base/tokenizer.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Union, Dict
+from typing import Union, Dict, List
 import json
 import tiktoken
 
@@ -15,7 +15,7 @@ class OpenAiTokenizer(BaseTokenizer):
     def __init__(self, model_name):
         self.__tokenizer = tiktoken.encoding_for_model(model_name)
 
-    def count_tokens(self, data: Union[str: Dict, None]) -> int:
+    def count_tokens(self, data: Union[str: Dict, List, None]) -> int:
         if data is None:
             return 0
         elif isinstance(data, str):
@@ -23,5 +23,10 @@ class OpenAiTokenizer(BaseTokenizer):
         elif isinstance(data, dict):
             dict_as_str = json.dumps(data, separators=(",", ":"))
             return len(self.__tokenizer.encode(dict_as_str))
+        elif isinstance(data, list):
+            token_number = 0
+            for datum in data:
+                token_number += self.count_tokens(datum)
+            return token_number
         else:
-            raise TypeError("Expected either a string, a dictionary, or None.")
+            raise TypeError("Expected either a string, a dictionary, a list,  or None.")

{igbot_base-0.0.22 → igbot_base-0.0.23}/igbot_base.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: igbot_base
-Version: 0.0.22
+Version: 0.0.23
 Summary: Base classes for igbot
 Author-email: Igor Kopeć <igor.kopec95@gmail.com>
 License: LGPL-3.0-or-later

{igbot_base-0.0.22 → igbot_base-0.0.23}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "igbot_base"
-version = "0.0.22"
+version = "0.0.23"
 description = "Base classes for igbot"
 authors = [{name = "Igor Kopeć", email = "igor.kopec95@gmail.com"}]
 license={text="LGPL-3.0-or-later"}