pygpt-net 2.4.41__py3-none-any.whl → 2.4.44__py3-none-any.whl

pygpt_net/provider/llms/azure_openai.py

@@ -6,9 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
+from typing import Optional, List, Dict
+
 from langchain_openai import AzureOpenAI
 from langchain_openai import AzureChatOpenAI
 
@@ -39,7 +41,12 @@ class AzureOpenAILLM(BaseLLM):
         self.id = "azure_openai"
         self.type = [MODE_LANGCHAIN, MODE_LLAMA_INDEX, "embeddings"]
 
-    def completion(self, window, model: ModelItem, stream: bool = False):
+    def completion(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ):
         """
         Return LLM provider instance for completion
 
@@ -51,7 +58,12 @@ class AzureOpenAILLM(BaseLLM):
         args = self.parse_args(model.langchain)
         return AzureOpenAI(**args)
 
-    def chat(self, window, model: ModelItem, stream: bool = False):
+    def chat(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ):
         """
         Return LLM provider instance for chat
 
@@ -63,7 +75,12 @@ class AzureOpenAILLM(BaseLLM):
         args = self.parse_args(model.langchain)
         return AzureChatOpenAI(**args)
 
-    def llama(self, window, model: ModelItem, stream: bool = False) -> LlamaBaseLLM:
+    def llama(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ) -> LlamaBaseLLM:
         """
         Return LLM provider instance for llama
 
@@ -75,7 +92,11 @@ class AzureOpenAILLM(BaseLLM):
         args = self.parse_args(model.llama_index)
         return LlamaAzureOpenAI(**args)
 
-    def get_embeddings_model(self, window, config: list = None) -> BaseEmbedding:
+    def get_embeddings_model(
+            self,
+            window,
+            config: Optional[List[Dict]] = None
+    ) -> BaseEmbedding:
         """
         Return provider instance for embeddings
 

pygpt_net/provider/llms/base.py

@@ -6,10 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 import os
+from typing import Optional, List, Dict
 
 from llama_index.core.base.embeddings.base import BaseEmbedding
 from llama_index.core.llms.llm import BaseLLM as LlamaBaseLLM
@@ -30,7 +31,13 @@ class BaseLLM:
         self.type = []  # langchain, llama_index, embeddings
         self.description = ""
 
-    def init(self, window, model: ModelItem, mode: str, sub_mode: str = None):
+    def init(
+            self,
+            window,
+            model: ModelItem,
+            mode: str,
+            sub_mode: str = None
+    ):
         """
         Initialize provider
 
@@ -53,7 +60,11 @@ class BaseLLM:
                 except Exception as e:
                     pass
 
-    def init_embeddings(self, window, env: list):
+    def init_embeddings(
+            self,
+            window,
+            env: Optional[List[Dict]] = None
+    ):
         """
         Initialize embeddings provider
 
@@ -69,7 +80,10 @@ class BaseLLM:
                 except Exception as e:
                     pass
 
-    def parse_args(self, options: dict) -> dict:
+    def parse_args(
+            self,
+            options: dict
+    ) -> dict:
         """
         Parse extra args
 
@@ -82,7 +96,12 @@ class BaseLLM:
                 args = parse_args(options['args'])
         return args
 
-    def completion(self, window, model: ModelItem, stream: bool = False) -> any:
+    def completion(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ) -> any:
         """
         Return LLM provider instance for completion in langchain mode
 
@@ -93,7 +112,12 @@ class BaseLLM:
         """
         pass
 
-    def chat(self, window, model: ModelItem, stream: bool = False) -> any:
+    def chat(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ) -> any:
         """
         Return LLM provider instance for chat in langchain mode
 
@@ -104,7 +128,12 @@ class BaseLLM:
         """
         pass
 
-    def llama(self, window, model: ModelItem, stream: bool = False) -> LlamaBaseLLM:
+    def llama(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ) -> LlamaBaseLLM:
         """
         Return LLM provider instance for llama index query and chat
 
@@ -115,7 +144,12 @@ class BaseLLM:
         """
         pass
 
-    def llama_multimodal(self, window, model: ModelItem, stream: bool = False) -> LlamaMultiModalLLM:
+    def llama_multimodal(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ) -> LlamaMultiModalLLM:
         """
         Return multimodal LLM provider instance for llama
 
@@ -126,7 +160,11 @@ class BaseLLM:
         """
         pass
 
-    def get_embeddings_model(self, window, config: list = None) -> BaseEmbedding:
+    def get_embeddings_model(
+            self,
+            window,
+            config: Optional[List[Dict]] = None
+    ) -> BaseEmbedding:
         """
         Return provider instance for embeddings
 

pygpt_net/provider/llms/google.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 from llama_index.llms.gemini import Gemini
@@ -32,7 +32,12 @@ class GoogleLLM(BaseLLM):
         self.id = "google"
         self.type = [MODE_LLAMA_INDEX]
 
-    def llama(self, window, model: ModelItem, stream: bool = False) -> LlamaBaseLLM:
+    def llama(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ) -> LlamaBaseLLM:
         """
         Return LLM provider instance for llama
 

pygpt_net/provider/llms/hugging_face.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 from langchain_community.llms import HuggingFaceHub
@@ -24,7 +24,12 @@ class HuggingFaceLLM(BaseLLM):
         self.id = "huggingface"
         self.type = [MODE_LANGCHAIN]
 
-    def completion(self, window, model: ModelItem, stream: bool = False):
+    def completion(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ):
         """
         Return LLM provider instance for completion
 
@@ -36,7 +41,12 @@ class HuggingFaceLLM(BaseLLM):
         args = self.parse_args(model.langchain)
         return HuggingFaceHub(**args)
 
-    def chat(self, window, model: ModelItem, stream: bool = False):
+    def chat(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ):
         """
         Return LLM provider instance for chat
 

pygpt_net/provider/llms/hugging_face_api.py

@@ -6,10 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 import os
+from typing import Optional, List, Dict
 
 from pygpt_net.core.types import (
     MODE_LLAMA_INDEX,
@@ -28,7 +29,12 @@ class HuggingFaceApiLLM(BaseLLM):
         self.id = "huggingface_api"
         self.type = [MODE_LLAMA_INDEX, "embeddings"]
 
-    def llama(self, window, model: ModelItem, stream: bool = False) -> LlamaBaseLLM:
+    def llama(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ) -> LlamaBaseLLM:
         """
         Return LLM provider instance for llama
 
@@ -40,7 +46,11 @@ class HuggingFaceApiLLM(BaseLLM):
         args = self.parse_args(model.llama_index)
         return HuggingFaceInferenceAPI(**args)
 
-    def get_embeddings_model(self, window, config: list = None) -> BaseEmbedding:
+    def get_embeddings_model(
+            self,
+            window,
+            config: Optional[List[Dict]] = None
+    ) -> BaseEmbedding:
         """
         Return provider instance for embeddings
 
@@ -55,7 +65,11 @@ class HuggingFaceApiLLM(BaseLLM):
             })
         return HuggingFaceAPIEmbedding(**args)
 
-    def init_embeddings(self, window, env: list):
+    def init_embeddings(
+            self,
+            window,
+            env: Optional[List[Dict]] = None
+    ):
         """
         Initialize embeddings provider
 

pygpt_net/provider/llms/local.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 from llama_index.llms.openai_like import OpenAILike
@@ -25,7 +25,12 @@ class LocalLLM(BaseLLM):
         self.id = "local_ai"
         self.type = [MODE_LLAMA_INDEX]
 
-    def llama(self, window, model: ModelItem, stream: bool = False) -> LlamaBaseLLM:
+    def llama(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ) -> LlamaBaseLLM:
         """
         Return LLM provider instance for llama
 

pygpt_net/provider/llms/ollama.py

@@ -6,10 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 import os
+from typing import Optional, List, Dict
 
 from langchain_community.chat_models import ChatOllama
 
@@ -33,7 +34,12 @@ class OllamaLLM(BaseLLM):
         self.id = "ollama"
         self.type = [MODE_LANGCHAIN, MODE_LLAMA_INDEX, "embeddings"]
 
-    def completion(self, window, model: ModelItem, stream: bool = False):
+    def completion(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ):
         """
         Return LLM provider instance for completion
 
@@ -44,7 +50,12 @@ class OllamaLLM(BaseLLM):
         """
         return None
 
-    def chat(self, window, model: ModelItem, stream: bool = False):
+    def chat(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ):
         """
         Return LLM provider instance for chat
 
@@ -56,7 +67,12 @@ class OllamaLLM(BaseLLM):
         args = self.parse_args(model.langchain)
         return ChatOllama(**args)
 
-    def llama(self, window, model: ModelItem, stream: bool = False) -> LlamaBaseLLM:
+    def llama(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ) -> LlamaBaseLLM:
         """
         Return LLM provider instance for llama
 
@@ -69,7 +85,11 @@ class OllamaLLM(BaseLLM):
         args = self.parse_args(model.llama_index)
         return Ollama(**args)
 
-    def get_embeddings_model(self, window, config: list = None) -> BaseEmbedding:
+    def get_embeddings_model(
+            self,
+            window,
+            config: Optional[List[Dict]] = None
+    ) -> BaseEmbedding:
         """
         Return provider instance for embeddings
 
@@ -84,7 +104,11 @@ class OllamaLLM(BaseLLM):
             })
         return OllamaEmbedding(**args)
 
-    def init_embeddings(self, window, env: list):
+    def init_embeddings(
+            self,
+            window,
+            env: Optional[List[Dict]] = None
+    ):
         """
         Initialize embeddings provider
 

pygpt_net/provider/llms/openai.py

@@ -6,9 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
+from typing import Optional, List, Dict
+
 from langchain_openai import OpenAI
 from langchain_openai import ChatOpenAI
 
@@ -33,7 +35,12 @@ class OpenAILLM(BaseLLM):
         self.id = "openai"
         self.type = [MODE_LANGCHAIN, MODE_LLAMA_INDEX, "embeddings"]
 
-    def completion(self, window, model: ModelItem, stream: bool = False):
+    def completion(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ):
         """
         Return LLM provider instance for completion
 
@@ -45,7 +52,12 @@ class OpenAILLM(BaseLLM):
         args = self.parse_args(model.langchain)
         return OpenAI(**args)
 
-    def chat(self, window, model: ModelItem, stream: bool = False):
+    def chat(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ):
         """
         Return LLM provider instance for chat
 
@@ -57,7 +69,12 @@ class OpenAILLM(BaseLLM):
         args = self.parse_args(model.langchain)
         return ChatOpenAI(**args)
 
-    def llama(self, window, model: ModelItem, stream: bool = False) -> LlamaBaseLLM:
+    def llama(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ) -> LlamaBaseLLM:
         """
         Return LLM provider instance for llama
 
@@ -69,7 +86,12 @@ class OpenAILLM(BaseLLM):
         args = self.parse_args(model.llama_index)
         return LlamaOpenAI(**args)
 
-    def llama_multimodal(self, window, model: ModelItem, stream: bool = False) -> LlamaMultiModalLLM:
+    def llama_multimodal(
+            self,
+            window,
+            model: ModelItem,
+            stream: bool = False
+    ) -> LlamaMultiModalLLM:
         """
         Return multimodal LLM provider instance for llama
 
@@ -81,7 +103,11 @@ class OpenAILLM(BaseLLM):
         args = self.parse_args(model.llama_index)
         return LlamaOpenAIMultiModal(**args)
 
-    def get_embeddings_model(self, window, config: list = None) -> BaseEmbedding:
+    def get_embeddings_model(
+            self,
+            window,
+            config: Optional[List[Dict]] = None
+    ) -> BaseEmbedding:
         """
         Return provider instance for embeddings
 

pygpt_net/provider/loaders/base.py

@@ -22,7 +22,9 @@ class BaseLoader:
         self.instructions = []  # list of instructions for 'web_index' command for how to handle this type
         self.args = {}  # custom keyword arguments
         self.init_args = {}  # initial keyword arguments
+        self.init_args_labels = {}
         self.init_args_types = {}
+        self.init_args_desc = {}
         self.allow_compiled = True  # allow in compiled and Snap versions
         # This is required due to some readers may require Python environment to install additional packages
 
@@ -42,6 +44,18 @@ class BaseLoader:
         """
         self.args = args
 
+    def explode(self, value: str) -> list:
+        """
+        Explode list from string
+
+        :param value: value string
+        :return: list
+        """
+        if value:
+            items = value.split(",")
+            return [item.strip() for item in items]
+        return []
+
     def get_args(self):
         """
         Prepare keyword arguments for reader init method

pygpt_net/provider/loaders/hub/yt/base.py

@@ -15,6 +15,7 @@ class YoutubeTranscriptReader(BasePydanticReader):
     """Youtube Transcript reader."""
 
     is_remote: bool = True
+    languages: List[str] = ["en"]
 
     @classmethod
     def class_name(cls) -> str:
@@ -34,6 +35,10 @@ class YoutubeTranscriptReader(BasePydanticReader):
                 for which transcripts are to be read.
 
         """
+        languages = self.languages
+        if not languages:
+            languages = ["en"]
+
         results = []
         for link in ytlinks:
             video_id = self._extract_video_id(link)

pygpt_net/provider/loaders/web_database.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,14 +30,16 @@ class Loader(BaseLoader):
                     "args": {
                         "query": {
                             "type": "str",
+                            "label": "SQL query",
+                            "description": "SQL query to read data from database, e.g. SELECT * FROM table",
                         },
                     },
                 }
             }
         ]
         self.init_args = {
-            "sql_database": None,
-            "engine": None,
+            # "sql_database": None,
+            # "engine": None,
             "uri": None,
             "scheme": None,
             "host": None,
@@ -47,8 +49,8 @@ class Loader(BaseLoader):
             "dbname": None,
         }
         self.init_args_types = {
-            "sql_database": "str",
-            "engine": "str",
+            # "sql_database": "str",
+            # "engine": "str",
             "uri": "str",
             "scheme": "str",
             "host": "str",
@@ -57,6 +59,12 @@ class Loader(BaseLoader):
             "password": "str",
             "dbname": "str",
         }
+        self.init_args_desc = {
+            # "sql_database": "str",
+            # "engine": "str",
+            "uri": "You can provide a single URI in the form of: {scheme}://{user}:{password}@{host}:{port}/{dbname}, "
+                   "or you can provide each field manually:",
+        }
 
     def get(self) -> BaseReader:
         """

pygpt_net/provider/loaders/web_github_issues.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -103,8 +103,12 @@ class Loader(BaseLoader):
         if "label_filters_include" in kwargs and kwargs.get("label_filters_include"):
             if isinstance(kwargs.get("label_filters_include"), list):
                 args["label_filters_include"] = kwargs.get("label_filters_include")
+            elif isinstance(kwargs.get("label_filters_include"), str):
+                args["label_filters_include"] = self.explode(kwargs.get("label_filters_include"))
         if "label_filters_exclude" in kwargs and kwargs.get("label_filters_exclude"):
             if isinstance(kwargs.get("label_filters_exclude"), list):
                 args["label_filters_exclude"] = kwargs.get("label_filters_exclude")
+            elif isinstance(kwargs.get("label_filters_exclude"), str):
+                args["label_filters_exclude"] = self.explode(kwargs.get("label_filters_exclude"))
 
         return args

pygpt_net/provider/loaders/web_google_calendar.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,9 +30,13 @@ class Loader(BaseLoader):
                     "args": {
                         "number_of_results": {
                             "type": "int",
+                            "label": "Number of results",
+                            "description": "Number of results to fetch, default: 100",
                         },
                         "start_date": {
                             "type": "str",
+                            "label": "Start date",
+                            "description": "Start date for events, ISO format: YYYY-MM-DD, default: today",
                         },
                     },
                 }
@@ -79,8 +83,12 @@ class Loader(BaseLoader):
         if "number_of_results" in kwargs and kwargs.get("number_of_results"):
             if isinstance(kwargs.get("number_of_results"), int):
                 args["number_of_results"] = kwargs.get("number_of_results")  # number of results
+                if args["number_of_results"] == 0:
+                    args["number_of_results"] = None
 
         if "start_date" in kwargs and kwargs.get("start_date"):
             if isinstance(kwargs.get("start_date"), str):
                 args["start_date"] = kwargs.get("start_date")  # start date
+                if args["start_date"] == "today" or args["start_date"] == "":
+                    args["start_date"] = None
         return args

pygpt_net/provider/loaders/web_google_docs.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,9 @@ class Loader(BaseLoader):
                     "args": {
                         "document_ids": {
                             "type": "list",
+                            "label": "Document IDs",
+                            "description": "List of document IDs to index, separated by comma (,)",
+                            "required": True,
                         },
                     },
                 }
@@ -76,4 +79,6 @@ class Loader(BaseLoader):
         if "document_ids" in kwargs and kwargs.get("document_ids"):
             if isinstance(kwargs.get("document_ids"), list):
                 args["document_ids"] = kwargs.get("document_ids")  # list of document ids
+            elif isinstance(kwargs.get("document_ids"), str):
+                args["document_ids"] = self.explode(kwargs.get("document_ids"))
         return args