pygpt-net 2.4.37__py3-none-any.whl → 2.4.42__py3-none-any.whl

pygpt_net/core/idx/chat.py

@@ -6,10 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 21:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 import json
+from typing import Optional, Dict, Any, List
 
 from llama_index.core.llms import ChatMessage, MessageRole
 from llama_index.core.prompts import ChatPromptTemplate
@@ -37,7 +38,11 @@ class Chat:
         self.storage = storage
         self.context = Context(window)
 
-    def call(self, context: BridgeContext, extra: dict = None) -> bool:
+    def call(
+            self,
+            context: BridgeContext,
+            extra: Optional[Dict[str, Any]] = None
+    ) -> bool:
         """
         Call chat, complete or query mode
 
@@ -69,7 +74,11 @@ class Chat:
                 extra=extra,
             )  # if not, use query mode
 
-    def raw_query(self, context: BridgeContext, extra: dict = None) -> bool:
+    def raw_query(
+            self,
+            context: BridgeContext,
+            extra: Optional[Dict[str, Any]] = None
+    ) -> bool:
         """
         Raw query mode
 
@@ -82,7 +91,11 @@ class Chat:
             extra=extra,
         )
 
-    def query(self, context: BridgeContext, extra: dict = None) -> bool:
+    def query(
+            self,
+            context: BridgeContext,
+            extra: Optional[Dict[str, Any]] = None
+    ) -> bool:
         """
         Query index mode (no chat, only single query) and append results to context
 
@@ -150,7 +163,11 @@ class Chat:
             return True
         return False
 
-    def chat(self, context: BridgeContext, extra: dict = None) -> bool:
+    def chat(
+            self,
+            context: BridgeContext,
+            extra: Optional[Dict[str, Any]] = None
+    ) -> bool:
         """
         Chat mode (conversation, using context from index) and append result to the context
 
@@ -291,7 +308,7 @@ class Chat:
             ctx: CtxItem,
             path: str,
             query: str,
-            model: ModelItem = None
+            model: Optional[ModelItem] = None
     ) -> str:
         """
         Query file using temp index (created on the fly)
@@ -343,9 +360,9 @@ class Chat:
             ctx: CtxItem,
             type: str,
             url: str,
-            args: dict,
+            args: Dict[str, Any],
             query: str,
-            model: ModelItem = None
+            model: Optional[ModelItem] = None
     ) -> str:
         """
         Query web using temp index (created on the fly)
@@ -405,8 +422,8 @@ class Chat:
             self,
             query: str,
             path: str,
-            model: ModelItem = None,
-            history: list = None,
+            model: Optional[ModelItem] = None,
+            history: Optional[List[CtxItem]] = None,
             verbose: bool = False,
     ) -> str:
         """
@@ -461,7 +478,7 @@ class Chat:
             self,
             query: str,
             idx: str,
-            model: ModelItem = None
+            model: Optional[ModelItem] = None
     ) -> str:
         """
         Query attachment
@@ -486,7 +503,11 @@ class Chat:
             output = str(response)
         return output
 
-    def get_memory_buffer(self, history: list, llm = None) -> ChatMemoryBuffer:
+    def get_memory_buffer(
+            self,
+            history: List[ChatMessage],
+            llm = None
+    ) -> ChatMemoryBuffer:
         """
         Get memory buffer
 
@@ -499,7 +520,10 @@ class Chat:
             llm=llm,
         )
 
-    def get_custom_prompt(self, prompt: str = None) -> ChatPromptTemplate or None:
+    def get_custom_prompt(
+            self,
+            prompt: Optional[str] = None
+    ) -> Optional[ChatPromptTemplate]:
         """
         Get custom prompt template if sys prompt is not empty
 
@@ -528,12 +552,16 @@ class Chat:
         ]
         return ChatPromptTemplate(qa_msgs)
 
-    def get_index(self, idx, model):
+    def get_index(
+            self,
+            idx: str,
+            model: ModelItem
+    ):
         """
         Get index instance
 
         :param idx: idx name (id)
-        :param model: model name
+        :param model: model instance
         :return:
         """
         # check if index exists
@@ -549,7 +577,10 @@ class Chat:
         index = self.storage.get(idx, service_context=service_context)  # get index
         return index, service_context
 
-    def get_metadata(self, source_nodes: list):
+    def get_metadata(
+            self,
+            source_nodes: Optional[list]
+    ) -> Dict[str, Any]:
         """
         Get metadata from source nodes
 
@@ -577,7 +608,10 @@ class Chat:
                             break
         return metadata
 
-    def parse_metadata(self, metadata: dict):
+    def parse_metadata(
+            self,
+            metadata: Optional[Dict]
+    ) -> Dict[str, Any]:
         """
         Parse metadata
 

pygpt_net/core/idx/context.py

@@ -6,11 +6,15 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.12.14 08:00:00                  #
 # ================================================== #
 
+from typing import Optional, List
+
 from llama_index.core.llms import ChatMessage, MessageRole
 
+from pygpt_net.item.ctx import CtxItem
+
 
 class Context:
     def __init__(self, window=None):
@@ -25,7 +29,7 @@ class Context:
             self,
             input_prompt: str,
             system_prompt: str,
-            history: list = None,
+            history: Optional[List[CtxItem]] = None,
             multimodal: bool = False
     ):
         """

pygpt_net/core/idx/indexing.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.26 04:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 import datetime
@@ -14,6 +14,8 @@ import os.path
 import time
 
 from pathlib import Path
+from typing import Optional, Tuple, List, Dict, Any
+
 from sqlalchemy import text
 
 from llama_index.core.indices.base import BaseIndex
@@ -99,7 +101,11 @@ class Indexing:
                 self.window.core.debug.log(msg)
                 self.window.core.debug.log(e)
 
-    def update_loader_args(self, loader: str, args: dict):
+    def update_loader_args(
+            self,
+            loader: str,
+            args: Dict[str, Any]
+    ):
         """
         Update loader arguments
 
@@ -142,7 +148,7 @@ class Indexing:
             self.register_loader(loader)
         self.window.core.idx.log("Data loaders reloaded.")
 
-    def get_external_instructions(self) -> dict:
+    def get_external_instructions(self) -> Dict[str, Any]:
         """
         Get external instructions
 
@@ -150,7 +156,7 @@ class Indexing:
         """
         return self.external_instructions
 
-    def get_external_config(self) -> dict:
+    def get_external_config(self) -> Dict[str, Any]:
         """
         Get external config
 
@@ -177,7 +183,7 @@ class Indexing:
             if ext in extensions:
                 return loader["loader"]
 
-    def get_data_providers(self) -> dict:
+    def get_data_providers(self) -> Dict[str, BaseLoader]:
         """
         Get data providers
 
@@ -185,7 +191,11 @@ class Indexing:
         """
         return self.data_providers
 
-    def get_loader_arguments(self, id: str, type: str = "file") -> dict:
+    def get_loader_arguments(
+            self,
+            id: str,
+            type: str = "file"
+    ) -> Dict[str, Any]:
         """
         Get keyword arguments for loader
 
@@ -267,8 +277,8 @@ class Indexing:
             path: str,
             force: bool = False,
             silent: bool = False,
-            loader_kwargs: dict = None,
-    ) -> list[Document]:
+            loader_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> List[Document]:
         """
         Get documents from path using data loaders
 
@@ -330,7 +340,11 @@ class Indexing:
         self.window.core.idx.metadata.append_file_metadata(documents, path)
         return documents
 
-    def read_text_content(self, path: str, loader_kwargs: dict = None) -> str:
+    def read_text_content(
+            self,
+            path: str,
+            loader_kwargs: Optional[Dict[str, Any]] = None
+    ) -> str:
         """
         Get content from file using loaders
 
@@ -353,7 +367,7 @@ class Indexing:
             self,
             url: str,
             type: str = "webpage",
-            extra_args: dict = None
+            extra_args: Optional[Dict[str, Any]] = None
     ) -> str:
         """
         Get content from external resource
@@ -373,8 +387,8 @@ class Indexing:
             self,
             url: str,
             type: str = "webpage",
-            extra_args: dict = None,
-    ) -> list[Document]:
+            extra_args: Optional[Dict[str, Any]] = None,
+    ) -> List[Document]:
         """
         Read data from external resource
 
@@ -423,11 +437,11 @@ class Indexing:
             self,
             idx: str,
             index: BaseIndex,
-            path: str = None,
+            path: Optional[str] = None,
             is_tmp: bool = False,
-            replace: bool = None,
-            recursive: bool = None
-    ) -> tuple:
+            replace: Optional[bool] = None,
+            recursive: Optional[bool] = None
+    ) -> Tuple[dict, list]:
         """
         Index all files in directory
 
@@ -494,10 +508,10 @@ class Indexing:
             self,
             idx: str,
             index: BaseIndex,
-            path: str = None,
+            path: Optional[str] = None,
             is_tmp: bool = False,
-            replace: bool = None
-    ) -> tuple:
+            replace: Optional[bool] = None
+    ) -> Tuple[dict, list]:
         """
         Index all files in directory and subdirectories recursively.
 
@@ -586,7 +600,10 @@ class Indexing:
 
         return indexed, errors
 
-    def get_db_data_from_ts(self, updated_ts: int = 0) -> list:
+    def get_db_data_from_ts(
+            self,
+            updated_ts: int = 0
+    ) -> List[Document]:
         """
         Get data from database from timestamp
 
@@ -626,7 +643,10 @@ class Indexing:
                 documents.append(doc)
         return documents
 
-    def get_db_meta_ids_from_ts(self, updated_ts: int = 0) -> list:
+    def get_db_meta_ids_from_ts(
+            self,
+            updated_ts: int = 0
+    ) -> List[int]:
         """
         Get IDs of meta from database from timestamp
 
@@ -650,7 +670,11 @@ class Indexing:
                 ids.append(data["id"])
         return ids
 
-    def get_db_data_by_id(self, id: int = 0, updated_ts: int = 0) -> list:
+    def get_db_data_by_id(
+            self,
+            id: int = 0,
+            updated_ts: int = 0
+    ) -> List[Document]:
         """
         Get data from database by meta id
 
@@ -688,7 +712,13 @@ class Indexing:
                 documents.append(doc)
         return documents
 
-    def index_db_by_meta_id(self, idx: str, index: BaseIndex, id: int = 0, from_ts: int = 0) -> (int, list):
+    def index_db_by_meta_id(
+            self,
+            idx: str,
+            index: BaseIndex,
+            id: int = 0,
+            from_ts: int = 0
+    ) -> Tuple[int, List[str]]:
         """
         Index data from database by meta id
 
@@ -706,7 +736,8 @@ class Indexing:
                 self.window.core.idx.log("Indexing documents from database by meta id: {}".format(id))
                 self.remove_old_meta_id(idx, id)
             elif from_ts > 0:
-                self.window.core.idx.log("Indexing documents from database by meta id: {} from timestamp: {}".format(id, from_ts))
+                self.window.core.idx.log("Indexing documents from database by meta id: {} from timestamp: {}".
+                                         format(id, from_ts))
 
             # get items from database
             documents = self.get_db_data_by_id(id, from_ts)
@@ -716,7 +747,8 @@ class Indexing:
 
                 self.index_document(index, d)
                 doc_id = d.id_
-                self.window.core.idx.log("Inserted ctx DB document: {} / {}, id: {}, metadata: {}".format(n+1, len(documents), d.id_, d.metadata))
+                self.window.core.idx.log("Inserted ctx DB document: {} / {}, id: {}, metadata: {}".
+                                         format(n+1, len(documents), d.id_, d.metadata))
                 self.window.core.ctx.idx.set_meta_as_indexed(id, idx, doc_id)  # update ctx
                 n += 1
         except Exception as e:
@@ -724,7 +756,12 @@ class Indexing:
             self.window.core.debug.log(e)
         return n, errors
 
-    def index_db_from_updated_ts(self, idx: str, index: BaseIndex, from_ts: int = 0) -> (int, list):
+    def index_db_from_updated_ts(
+            self,
+            idx: str,
+            index: BaseIndex,
+            from_ts: int = 0
+    ) -> Tuple[int, List[str]]:
         """
         Index data from database from timestamp
 
@@ -752,10 +789,10 @@ class Indexing:
             index: BaseIndex,
             url: str,
             type="webpage",
-            extra_args: dict = None,
+            extra_args: Optional[Dict[str, Any]] = None,
             is_tmp: bool = False,
-            replace: bool = None
-    ) -> (int, list):
+            replace: Optional[bool] = None
+    ) -> Tuple[int, List[str]]:
         """
         Index data from external (remote) resource
 
@@ -825,7 +862,8 @@ class Indexing:
                         idx=idx,
                         doc_id=doc_id,
                     )  # update external index
-                self.window.core.idx.log("Inserted web document: {} / {}, id: {}, metadata: {}".format(n+1, len(documents), d.id_, d.metadata))
+                self.window.core.idx.log("Inserted web document: {} / {}, id: {}, metadata: {}".
+                                         format(n+1, len(documents), d.id_, d.metadata))
                 n += 1
         except Exception as e:
             errors.append(str(e))
@@ -838,9 +876,9 @@ class Indexing:
             index: BaseIndex,
             urls: list,
             type="webpage",
-            extra_args: dict = None,
+            extra_args: Optional[Dict[str, Any]] = None,
             is_tmp: bool = False
-    ) -> (int, list):
+    ) -> Tuple[int, List[str]]:
         """
         Index data from URLs
 
@@ -878,7 +916,12 @@ class Indexing:
             errors.extend(errs)
         return n, errors
 
-    def remove_old_meta_id(self, idx: str, id: int = 0, force: bool = False) -> bool:
+    def remove_old_meta_id(
+            self,
+            idx: str,
+            id: int = 0,
+            force: bool = False
+    ) -> bool:
         """
         Remove old meta id from index
 
@@ -906,7 +949,12 @@ class Indexing:
                 return True
         return False
 
-    def remove_old_file(self, idx: str, file_id: str, force: bool = False):
+    def remove_old_file(
+            self,
+            idx: str,
+            file_id: str,
+            force: bool = False
+    ) -> bool:
         """
         Remove old file from index
 
@@ -934,7 +982,13 @@ class Indexing:
                 return True
         return False
 
-    def remove_old_external(self, idx: str, content: str, type: str, force: bool = False):
+    def remove_old_external(
+            self,
+            idx: str,
+            content: str,
+            type: str,
+            force: bool = False
+    ) -> bool:
         """
         Remove old file from index
 
@@ -963,7 +1017,11 @@ class Indexing:
                 return True
         return False
 
-    def index_document(self, index: BaseIndex, doc: Document):
+    def index_document(
+            self,
+            index: BaseIndex,
+            doc: Document
+    ):
         """
         Index document
 
@@ -983,8 +1041,8 @@ class Indexing:
             self,
             file_path: str,
             index_path: str,
-            model: ModelItem = None,
-            documents: list = None,
+            model: Optional[ModelItem] = None,
+            documents: Optional[List[Document]] = None,
     ) -> list:
         """
         Index context attachment
@@ -999,7 +1057,10 @@ class Indexing:
             model = self.window.core.models.from_defaults()
 
         service_context = self.window.core.idx.llm.get_service_context(model=model)
-        index = self.window.core.idx.storage.get_ctx_idx(index_path, service_context=service_context)  # get or create ctx index
+        index = self.window.core.idx.storage.get_ctx_idx(
+            index_path,
+            service_context=service_context
+        )  # get or create ctx index
 
         idx = "tmp:{}".format(index_path)  # tmp index id
         self.window.core.idx.log("Indexing to context attachment index: {}...".format(idx))
@@ -1021,8 +1082,8 @@ class Indexing:
             self,
             url: str,
             index_path: str,
-            model: ModelItem = None,
-            documents: list = None,
+            model: Optional[ModelItem] = None,
+            documents: Optional[List[Document]] = None,
     ) -> list:
         """
         Index context attachment
@@ -1077,7 +1138,11 @@ class Indexing:
         print("Selected web data loader: {}".format(type))
         return type
 
-    def remove_attachment(self, index_path: str, doc_id: str) -> bool:
+    def remove_attachment(
+            self,
+            index_path: str,
+            doc_id: str
+    ) -> bool:
         """
         Remove document from index
 

pygpt_net/core/idx/llm.py

@@ -6,10 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.15 00:00:00                  #
+# Updated Date: 2024.12.14 08:00:00                  #
 # ================================================== #
 
 import os.path
+from typing import Optional, Union
 
 from llama_index.core.llms.llm import BaseLLM
 from llama_index.core.multi_modal_llms import MultiModalLLM
@@ -41,7 +42,11 @@ class Llm:
         os.environ['OPENAI_API_BASE'] = str(self.window.core.config.get('api_endpoint'))
         os.environ['OPENAI_ORGANIZATION'] = str(self.window.core.config.get('organization_key'))
 
-    def get(self, model: ModelItem = None, multimodal: bool = False) -> BaseLLM or MultiModalLLM:
+    def get(
+            self,
+            model: Optional[ModelItem] = None,
+            multimodal: bool = False
+    ) -> Union[BaseLLM, MultiModalLLM]:
         """
         Get LLM provider
 
@@ -117,7 +122,10 @@ class Llm:
             config=args,
         )
 
-    def get_service_context(self, model: ModelItem = None) -> ServiceContext:
+    def get_service_context(
+            self,
+            model: Optional[ModelItem] = None
+    ) -> ServiceContext:
         """
         Get service context + embeddings provider
 

pygpt_net/core/idx/metadata.py

@@ -6,11 +6,12 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.13 01:00:00                  #
+# Updated Date: 2024.12.14 08:00:00                  #
 # ================================================== #
 
 import datetime
 import os.path
+from typing import List, Dict, Any
 
 from llama_index.core.schema import Document
 
@@ -24,7 +25,11 @@ class Metadata:
         """
         self.window = window
 
-    def append_file_metadata(self, docs: list[Document], path: str):
+    def append_file_metadata(
+            self,
+            docs: List[Document],
+            path: str
+    ):
         """
         Append custom meta to documents (file)
 
@@ -81,7 +86,12 @@ class Metadata:
                             self.window.core.debug.log(e)
                 doc.metadata = meta
 
-    def append_web_metadata(self, docs: list[Document], type: str, args: dict):
+    def append_web_metadata(
+            self,
+            docs: List[Document],
+            type: str,
+            args: Dict[str, Any]
+    ):
         """
         Append custom meta to documents (web)