pygpt-net 2.4.41__py3-none-any.whl → 2.4.44__py3-none-any.whl

pygpt_net/provider/loaders/web_google_drive.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.08.19 20:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,12 +30,17 @@ class Loader(BaseLoader):
                     "args": {
                         "folder_id": {
                             "type": "str",
+                            "label": "Folder ID",
                         },
                         "file_ids": {
                             "type": "list",
+                            "label": "File IDs",
+                            "description": "List of file ids, separated by comma (,)",
                         },
                         "mime_types": {
                             "type": "list",
+                            "label": "Mime Types",
+                            "description": "List of mime types, separated by comma (,)",
                         },
                     },
                 }
@@ -98,8 +103,12 @@ class Loader(BaseLoader):
         if "file_ids" in kwargs and kwargs.get("file_ids"):
             if isinstance(kwargs.get("file_ids"), list):
                 args["file_ids"] = kwargs.get("file_ids")  # list of file ids
+            elif isinstance(kwargs.get("file_ids"), str):
+                args["file_ids"] = self.explode(kwargs.get("file_ids"))
 
         if "mime_types" in kwargs and kwargs.get("mime_types"):
             if isinstance(kwargs.get("mime_types"), list):
                 args["mime_types"] = kwargs.get("mime_types")  # list of mime types
+            elif isinstance(kwargs.get("mime_types"), str):
+                args["mime_types"] = self.explode(kwargs.get("mime_types"))
         return args

pygpt_net/provider/loaders/web_google_gmail.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,7 @@ class Loader(BaseLoader):
                     "args": {
                         "query": {
                             "type": "str",
+                            "label": "Query",
                         },
                     },
                 }

pygpt_net/provider/loaders/web_google_keep.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,8 @@ class Loader(BaseLoader):
                     "args": {
                         "document_ids": {
                             "type": "list",
+                            "label": "Document IDs",
+                            "description": "List of document ids, separated by comma (,)",
                         },
                     },
                 }
@@ -74,4 +76,6 @@ class Loader(BaseLoader):
         if "document_ids" in kwargs and kwargs.get("document_ids"):
             if isinstance(kwargs.get("document_ids"), list):
                 args["document_ids"] = kwargs.get("document_ids")  # list of document ids
+            elif isinstance(kwargs.get("document_ids"), str):
+                args["document_ids"] = self.explode(kwargs.get("document_ids"))
         return args

pygpt_net/provider/loaders/web_google_sheets.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,8 @@ class Loader(BaseLoader):
                     "args": {
                         "spreadsheet_ids": {
                             "type": "list",
+                            "label": "Spreadsheet IDs",
+                            "description": "List of spreadsheet ids, separated by comma (,)",
                         },
                     },
                 }
@@ -78,4 +80,6 @@ class Loader(BaseLoader):
         if "spreadsheet_ids" in kwargs and kwargs.get("spreadsheet_ids"):
             if isinstance(kwargs.get("spreadsheet_ids"), list):
                 args["spreadsheet_ids"] = kwargs.get("spreadsheet_ids")  # spreadsheet ids
+            elif isinstance(kwargs.get("spreadsheet_ids"), str):
+                args["spreadsheet_ids"] = self.explode(kwargs.get("spreadsheet_ids"))
         return args

pygpt_net/provider/loaders/web_microsoft_onedrive.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,18 +30,26 @@ class Loader(BaseLoader):
                     "args": {
                         "folder_id": {
                             "type": "str",
+                            "label": "Folder ID",
                         },
                         "file_ids": {
                             "type": "list",
+                            "label": "File IDs",
+                            "description": "List of file ids, separated by comma (,)",
                         },
                         "mime_types": {
                             "type": "list",
+                            "label": "Mime Types",
+                            "description": "List of mime types, separated by comma (,)",
                         },
                         "folder_path": {
                             "type": "str",
+                            "label": "Folder Path",
                         },
                         "file_paths": {
                             "type": "list",
+                            "label": "File Paths",
+                            "description": "List of file paths, separated by comma (,)",
                         },
                     },
                 }
@@ -98,10 +106,14 @@ class Loader(BaseLoader):
         if "file_ids" in kwargs and kwargs.get("file_ids"):
             if isinstance(kwargs.get("file_ids"), list):
                 args["file_ids"] = kwargs.get("file_ids")  # list of file ids
+            elif isinstance(kwargs.get("file_ids"), str):
+                args["file_ids"] = self.explode(kwargs.get("file_ids"))
 
         if "mime_types" in kwargs and kwargs.get("mime_types"):
             if isinstance(kwargs.get("mime_types"), list):
                 args["mime_types"] = kwargs.get("mime_types")  # list of mime types
+            elif isinstance(kwargs.get("mime_types"), str):
+                args["mime_types"] = self.explode(kwargs.get("mime_types"))
 
         if "folder_path" in kwargs and kwargs.get("folder_path"):
             if isinstance(kwargs.get("folder_path"), str):
@@ -110,4 +122,6 @@ class Loader(BaseLoader):
         if "file_paths" in kwargs and kwargs.get("file_paths"):
             if isinstance(kwargs.get("file_paths"), list):
                 args["file_paths"] = kwargs.get("file_paths")  # list of file paths
+            elif isinstance(kwargs.get("file_paths"), str):
+                args["file_paths"] = self.explode(kwargs.get("file_paths"))
         return args

pygpt_net/provider/loaders/web_page.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.12 06:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 from llama_index.core.readers.base import BaseReader
@@ -19,7 +19,7 @@ class Loader(BaseLoader):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.id = "webpage"
-        self.name = "Webpages"
+        self.name = "Webpage"
         self.type = ["web"]
         self.instructions = [
             {
@@ -28,6 +28,8 @@ class Loader(BaseLoader):
                     "args": {
                         "url": {
                             "type": "str",
+                            "label": "URL",
+                            "description": "URL of the webpage to index, e.g. https://www.example.com",
                         },
                     },
                 }

pygpt_net/provider/loaders/web_rss.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.12 06:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 from llama_index.core.readers.base import BaseReader
@@ -28,6 +28,7 @@ class Loader(BaseLoader):
                     "args": {
                         "url": {
                             "type": "str",
+                            "label": "URL",
                         },
                     },
                 }

pygpt_net/provider/loaders/web_sitemap.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 from llama_index.core.readers.base import BaseReader
@@ -28,6 +28,7 @@ class Loader(BaseLoader):
                     "args": {
                         "url": {
                             "type": "str",
+                            "label": "URL",
                         },
                     },
                 }

pygpt_net/provider/loaders/web_twitter.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,9 +30,11 @@ class Loader(BaseLoader):
                     "args": {
                         "users": {
                             "type": "list",
+                            "label": "Twitter/X usernames",
                         },
                         "max_tweets": {
                             "type": "int",
+                            "label": "Max tweets",
                         },
                     },
                 }
@@ -80,7 +82,7 @@ class Loader(BaseLoader):
             if isinstance(kwargs.get("users"), list):
                 args["twitterhandles"] = kwargs.get("users")  # usernames
             elif isinstance(kwargs.get("users"), str):
-                args["twitterhandles"] = [kwargs.get("users")]
+                args["twitterhandles"] = self.explode(kwargs.get("users"))
 
         if "max_tweets" in kwargs and kwargs.get("max_tweets"):
             if isinstance(kwargs.get("max_tweets"), int):

pygpt_net/provider/loaders/web_yt.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.26 04:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 from llama_index.core.readers.base import BaseReader
@@ -28,11 +28,25 @@ class Loader(BaseLoader):
                     "args": {
                         "url": {
                             "type": "str",
+                            "label": "Youtube URL",
+                            "description": "URL of the YouTube video, e.g. https://www.youtube.com/watch?v=CRRlbK5w8AE",
                         },
                     },
                 }
             }
         ]
+        self.init_args = {
+            "languages": ["en"],
+        }
+        self.init_args_types = {
+            "languages": "list",
+        }
+        self.init_args_labels = {
+            "languages": "Languages",
+        }
+        self.init_args_desc = {
+            "languages": "List of languages to extract from the video, separated by comma (,), e.g. 'en,de,fr'. Default is 'en'",
+        }
 
     def get(self) -> BaseReader:
         """
@@ -40,7 +54,8 @@ class Loader(BaseLoader):
 
         :return: Data reader instance
         """
-        return YoutubeTranscriptReader()
+        args = self.get_args()
+        return YoutubeTranscriptReader(**args)
 
     def prepare_args(self, **kwargs) -> dict:
         """

pygpt_net/provider/vector_stores/__init__.py

@@ -6,11 +6,13 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 import hashlib
+from typing import Optional, Tuple, List
 
+from llama_index.core.indices.service_context import ServiceContext
 from llama_index.core.indices.base import BaseIndex
 from llama_index.core.indices.vector_store.base import VectorStoreIndex
 
@@ -31,7 +33,7 @@ class Storage:
         self.indexes = {}
         self.tmp_storage = TempProvider(window=window)
 
-    def get_storage(self) -> BaseStore or None:
+    def get_storage(self) -> Optional[BaseStore]:
         """
         Get current vector store provider
 
@@ -44,7 +46,7 @@ class Storage:
             return None
         return self.storages[current]
 
-    def get_tmp_storage(self) -> BaseStore or None:
+    def get_tmp_storage(self) -> Optional[TempProvider]:
         """
         Get temp vector store provider
 
@@ -52,16 +54,19 @@ class Storage:
         """
         return self.tmp_storage
 
-    def get_ctx_idx_storage(self, path: str) -> BaseStore or None:
+    def get_ctx_idx_storage(self, path: str) -> CtxAttachmentProvider:
         """
         Get temp vector store provider
         
         :param path: Path to index on disk
         :return: vector store provider instance
         """
-        return CtxAttachmentProvider(window=self.window, path=path)
+        return CtxAttachmentProvider(
+            window=self.window,
+            path=path
+        )
 
-    def register(self, name: str, storage=None):
+    def register(self, name: str, storage: BaseStore):
         """
         Register vector store provider
 
@@ -71,7 +76,7 @@ class Storage:
         storage.attach(window=self.window)
         self.storages[name] = storage
 
-    def get_ids(self) -> list:
+    def get_ids(self) -> List[str]:
         """
         Return all vector store providers IDs
 
@@ -79,7 +84,7 @@ class Storage:
         """
         return list(self.storages.keys())
 
-    def exists(self, id: str = None) -> bool:
+    def exists(self, id: Optional[str] = None) -> bool:
         """
         Check if index exists
 
@@ -102,7 +107,11 @@ class Storage:
             raise Exception('Storage engine not found!')
         storage.create(id)
 
-    def get(self, id: str, service_context=None) -> BaseIndex:
+    def get(
+            self,
+            id: str,
+            service_context: Optional[ServiceContext]
+    ) -> BaseIndex:
         """
         Get index instance
 
@@ -118,7 +127,11 @@ class Storage:
             service_context=service_context,
         )
 
-    def store(self, id: str, index: BaseIndex = None):
+    def store(
+            self,
+            id: str,
+            index: Optional[BaseIndex] = None
+    ):
         """
         Store index
 
@@ -173,7 +186,11 @@ class Storage:
             doc_id=doc_id,
         )
 
-    def get_tmp(self, identifier: str, service_context=None) -> (str, BaseIndex):
+    def get_tmp(
+            self,
+            identifier: str,
+            service_context: Optional[ServiceContext] = None
+    ) -> Tuple[str, BaseIndex]:
         """
         Get tmp index instance
 
@@ -191,7 +208,11 @@ class Storage:
             service_context=service_context,
         )
 
-    def store_tmp(self, id: str, index: BaseIndex = None):
+    def store_tmp(
+            self,
+            id: str,
+            index: Optional[BaseIndex] = None
+    ):
         """
         Store index
 
@@ -228,7 +249,11 @@ class Storage:
             raise Exception('Storage engine not found!')
         storage.clean(id)
 
-    def get_ctx_idx(self, path: str, service_context=None) -> (str, BaseIndex):
+    def get_ctx_idx(
+            self,
+            path: str,
+            service_context: Optional[ServiceContext] = None
+    ) -> BaseIndex:
         """
         Get context index instance
 
@@ -241,10 +266,15 @@ class Storage:
         if storage is None:
             raise Exception('Storage engine not found!')
         return storage.get(
+            id="",
             service_context=service_context,
         )
 
-    def store_ctx_idx(self, path: str, index: BaseIndex = None):
+    def store_ctx_idx(
+            self,
+            path: str,
+            index: Optional[BaseIndex] = None
+    ):
         """
         Store context index
 
@@ -255,6 +285,7 @@ class Storage:
         if storage is None:
             raise Exception('Storage engine not found!')
         storage.store(
+            id="",
             index=index,
         )
 

pygpt_net/provider/vector_stores/base.py

@@ -6,11 +6,12 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.08.20 19:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 import os
 import shutil
+from typing import Optional
 
 from llama_index.core.indices.base import BaseIndex
 from llama_index.core.indices.service_context import ServiceContext
@@ -31,7 +32,12 @@ class BaseStore:
         self.prefix = ""  # prefix for index directory
         self.indexes = {}
 
-    def index_from_store(self, vector_store, storage_context: StorageContext, service_context: ServiceContext = None):
+    def index_from_store(
+            self,
+            vector_store,
+            storage_context: StorageContext,
+            service_context: Optional[ServiceContext] = None
+    ):
         """
         Get index instance
 
@@ -74,7 +80,10 @@ class BaseStore:
             self.prefix + id,
         )
 
-    def exists(self, id: str = None) -> bool:
+    def exists(
+            self,
+            id: Optional[str] = None
+    ) -> bool:
         """
         Check if index with id exists
 
@@ -94,7 +103,11 @@ class BaseStore:
         """
         pass
 
-    def get(self, id: str, service_context: ServiceContext = None) -> BaseIndex:
+    def get(
+            self,
+            id: str,
+            service_context: Optional[ServiceContext] = None
+    ) -> BaseIndex:
         """
         Get index instance
 
@@ -104,7 +117,11 @@ class BaseStore:
         """
         pass
 
-    def store(self, id: str, index: BaseIndex = None):
+    def store(
+            self,
+            id: str,
+            index: Optional[BaseIndex] = None
+    ):
         """
         Store/persist index
 
@@ -113,7 +130,10 @@ class BaseStore:
         """
         pass
 
-    def remove(self, id: str) -> bool:
+    def remove(
+            self,
+            id: str
+    ) -> bool:
         """
         Clear index
 
@@ -127,7 +147,10 @@ class BaseStore:
             shutil.rmtree(path)
         return True
 
-    def truncate(self, id: str) -> bool:
+    def truncate(
+            self,
+            id: str
+    ) -> bool:
         """
         Truncate index
 
@@ -136,7 +159,11 @@ class BaseStore:
         """
         return self.remove(id)
 
-    def remove_document(self, id: str, doc_id: str) -> bool:
+    def remove_document(
+            self,
+            id: str,
+            doc_id: str
+    ) -> bool:
         """
         Remove document from index
 

pygpt_net/provider/vector_stores/chroma.py

@@ -6,10 +6,12 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.14 05:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 import os.path
+from typing import Optional
+
 import chromadb
 from chromadb.config import Settings
 
@@ -64,7 +66,11 @@ class ChromaProvider(BaseStore):
                 index=index,
             )
 
-    def get(self, id: str, service_context: ServiceContext = None) -> BaseIndex:
+    def get(
+            self,
+            id: str,
+            service_context: Optional[ServiceContext] = None
+    ) -> BaseIndex:
         """
         Get index
 
@@ -86,7 +92,11 @@ class ChromaProvider(BaseStore):
         self.indexes[id] = self.index_from_store(vector_store, storage_context, service_context)
         return self.indexes[id]
 
-    def store(self, id: str, index: BaseIndex = None):
+    def store(
+            self,
+            id: str,
+            index: Optional[BaseIndex] = None
+    ):
         """
         Store index