pygpt-net 2.1.37__py3-none-any.whl → 2.4.48__py3-none-any.whl

pygpt_net/core/idx/indexing.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.19 01:00:00                  #
+# Updated Date: 2025.01.16 01:00:00                  #
 # ================================================== #
 
 import datetime
@@ -14,14 +14,17 @@ import os.path
 import time
 
 from pathlib import Path
+from typing import Optional, Tuple, List, Dict, Any
+
 from sqlalchemy import text
 
 from llama_index.core.indices.base import BaseIndex
 from llama_index.core.schema import Document
 from llama_index.core import SimpleDirectoryReader
 
+from pygpt_net.item.model import ModelItem
 from pygpt_net.provider.loaders.base import BaseLoader
-from pygpt_net.utils import parse_args
+from pygpt_net.utils import parse_args, pack_arg
 
 
 class Indexing:
@@ -38,6 +41,7 @@ class Indexing:
         }
         self.data_providers = {}  # data providers (loaders)
         self.external_instructions = {}
+        self.external_config = {}
         self.last_call = None
 
     def register_loader(self, loader: BaseLoader):
@@ -56,6 +60,7 @@ class Indexing:
         self.data_providers[loader.id] = loader  # cache loader
         extensions = loader.extensions  # available extensions
         types = loader.type  # available types
+
         if "file" in types:
             loader.set_args(self.get_loader_arguments(loader.id, "file"))  # set reader arguments
             try:
@@ -66,6 +71,7 @@ class Indexing:
                 msg = "Error while registering data loader: " + loader.id + " - " + str(e)
                 self.window.core.debug.log(msg)
                 self.window.core.debug.log(e)
+
         if "web" in types:
             loader.set_args(self.get_loader_arguments(loader.id, "web"))  # set reader arguments
             try:
@@ -75,11 +81,83 @@ class Indexing:
                     for item in loader.instructions:
                         cmd = list(item.keys())[0]
                         self.external_instructions[cmd] = item[cmd]
+                if loader.init_args:
+                    for key in loader.init_args:
+                        if loader.id not in self.external_config:
+                            self.external_config[loader.id] = {}
+                        self.external_config[loader.id][key] = {
+                            "key": key,
+                            "value": loader.init_args[key],
+                            "type": "str",  # default = str
+                            "label": key,
+                            "description": None,
+                        }
+                        # from config
+                        if key in loader.args:
+                            self.external_config[loader.id][key]["value"] = loader.args[key]
+                        if key in loader.init_args_types:
+                            self.external_config[loader.id][key]["type"] = loader.init_args_types[key]
+                        if key in loader.init_args_labels:
+                            self.external_config[loader.id][key]["label"] = loader.init_args_labels[key]
+                        if key in loader.init_args_desc:
+                            self.external_config[loader.id][key]["description"] = loader.init_args_desc[key]
+
             except ImportError as e:
                 msg = "Error while registering data loader: " + loader.id + " - " + str(e)
                 self.window.core.debug.log(msg)
                 self.window.core.debug.log(e)
 
+    def get_loader(self, loader: str) -> Optional[BaseLoader]:
+        """
+        Get data loader by id
+
+        :param loader: loader id
+        :return: data loader instance
+        """
+        if loader in self.data_providers:
+            return self.data_providers[loader]
+        return None
+
+    def update_loader_args(
+            self,
+            loader: str,
+            args: Dict[str, Any]
+    ):
+        """
+        Update loader arguments
+
+        :param loader: loader id
+        :param args: keyword arguments
+        """
+        if loader in self.data_providers:
+            self.data_providers[loader].set_args(args)
+            reader = self.data_providers[loader].get()  # get data reader instance
+            self.loaders["web"][loader] = reader  # update reader instance
+
+            # update in config
+            config = self.window.core.config.get("llama.hub.loaders.args")
+            if config is None:
+                config = []
+            loader_key = "web_" + loader
+            for arg in args:
+                found = False
+                for item in config:
+                    if item["loader"] == loader_key and item["name"] == arg:
+                        item["value"] = args[arg]
+                        found = True
+                if not found:
+                    type = "str"
+                    if arg in self.data_providers[loader].init_args_types:
+                        type = self.data_providers[loader].init_args_types[arg]
+                    # pack value
+                    value = pack_arg(args[arg], type)
+                    config.append({
+                        "loader": loader_key,
+                        "name": arg,
+                        "value": value,
+                        "type": type
+                    })
+
     def reload_loaders(self):
         """Reload loaders (update arguments)"""
         self.window.core.idx.log("Reloading data loaders...")
@@ -87,7 +165,7 @@ class Indexing:
             self.register_loader(loader)
         self.window.core.idx.log("Data loaders reloaded.")
 
-    def get_external_instructions(self) -> dict:
+    def get_external_instructions(self) -> Dict[str, Any]:
         """
         Get external instructions
 
@@ -95,6 +173,14 @@ class Indexing:
         """
         return self.external_instructions
 
+    def get_external_config(self) -> Dict[str, Any]:
+        """
+        Get external config
+
+        :return: dict of external config
+        """
+        return self.external_config
+
     def get_online_loader(self, ext: str):
         """
         Get online loader by extension (deprecated)
@@ -114,7 +200,7 @@ class Indexing:
             if ext in extensions:
                 return loader["loader"]
 
-    def get_data_providers(self) -> dict:
+    def get_data_providers(self) -> Dict[str, BaseLoader]:
         """
         Get data providers
 
@@ -122,7 +208,11 @@ class Indexing:
         """
         return self.data_providers
 
-    def get_loader_arguments(self, id: str, type: str = "file") -> dict:
+    def get_loader_arguments(
+            self,
+            id: str,
+            type: str = "file"
+    ) -> Dict[str, Any]:
         """
         Get keyword arguments for loader
 
@@ -160,6 +250,26 @@ class Indexing:
                 return True
         return False
 
+    def is_excluded_path(self, path: str) -> bool:
+        """
+        Check if path is excluded
+
+        :param path: file path
+        :return: True if excluded
+        """
+        data_dir = self.window.core.config.get_user_dir("data")
+        # interpreter files
+        excluded = [
+            os.path.join(data_dir, ".interpreter.output.py"),
+            os.path.join(data_dir, ".interpreter.input.py"),
+            os.path.join(data_dir, ".interpreter.current.py"),
+            os.path.join(data_dir, ".interpreter.kernel.json"),
+            os.path.join(data_dir, ".canvas.html"),
+        ]
+        if path in excluded:
+            return True
+        return False
+
     def is_allowed(self, path: str) -> bool:
         """
         Check if path is allowed for indexing
@@ -171,20 +281,33 @@ class Indexing:
             return True
         ext = os.path.splitext(path)[1][1:]  # get extension
         ext = ext.lower()
+        if self.is_excluded_path(path):
+            return False
         if ext in self.loaders["file"] and not self.window.core.config.get("llama.idx.excluded.force"):
             return True
         if self.is_excluded(ext):
             return False
         return True
 
-    def get_documents(self, path: str) -> list[Document]:
+    def get_documents(
+            self,
+            path: str,
+            force: bool = False,
+            silent: bool = False,
+            loader_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> List[Document]:
         """
-        Get documents from path
+        Get documents from path using data loaders
 
         :param path: path to data
+        :param force: force reading
+        :param silent: disable logging
+        :param loader_kwargs: additional keyword arguments for loader
         :return: list of documents
         """
-        self.window.core.idx.log("Reading documents from path: {}".format(path))
+        # TODO: if .zip then unpack here, and return path to /tmp
+        if not silent:
+            self.window.core.idx.log("Reading documents from path: {}".format(path))
         if os.path.isdir(path):
             reader = SimpleDirectoryReader(
                 input_dir=path,
@@ -196,17 +319,37 @@ class Indexing:
             # get extension
             ext = os.path.splitext(path)[1][1:].lower()
 
-            # check if not excluded
-            if self.is_excluded(ext):
-                self.window.core.idx.log("Ignoring excluded extension: {}".format(ext))
+            # check if not excluded extension
+            if self.is_excluded(ext) and not force:
+                if not silent:
+                    self.window.core.idx.log("Ignoring excluded extension: {}".format(ext))
+                return []
+
+            # check if not excluded path
+            if self.is_excluded_path(path) and not force:
+                if not silent:
+                    self.window.core.idx.log("Ignoring excluded path: {}".format(path))
                 return []
 
+            # check if archive (zip, tar)
+            if self.window.core.filesystem.packer.is_archive(path):
+                tmp_path = self.window.core.filesystem.packer.unpack(path)
+                if tmp_path:
+                    return self.get_documents(tmp_path, force=force, silent=silent, loader_kwargs=loader_kwargs)
+
             if ext in self.loaders["file"]:
-                self.window.core.idx.log("Using loader for: {}".format(ext))
+                if not silent:
+                    self.window.core.idx.log("Using loader for: {}".format(ext))
                 reader = self.loaders["file"][ext]
-                documents = reader.load_data(file=Path(path))
+
+                # use custom loader method if available
+                if hasattr(reader, "load_data_custom") and loader_kwargs:
+                    documents = reader.load_data_custom(file=Path(path), **loader_kwargs)
+                else:
+                    documents = reader.load_data(file=Path(path))
             else:
-                self.window.core.idx.log("Using default SimpleDirectoryReader for: {}".format(ext))
+                if not silent:
+                    self.window.core.idx.log("Using default SimpleDirectoryReader for: {}".format(ext))
                 reader = SimpleDirectoryReader(input_files=[path])
                 documents = reader.load_data()
 
@@ -214,18 +357,87 @@ class Indexing:
         self.window.core.idx.metadata.append_file_metadata(documents, path)
         return documents
 
-    def read_text_content(self, path: str) -> str:
+    def read_text_content(
+            self,
+            path: str,
+            loader_kwargs: Optional[Dict[str, Any]] = None
+    ) -> Tuple[str, List[Document]]:
         """
         Get content from file using loaders
 
         :param path: path to file
-        :return: file content
+        :param loader_kwargs: additional keyword arguments for data loader
+        :return: text content, list of documents
+        """
+        docs = self.get_documents(
+            path,
+            force=True,  # allow excluded extensions
+            silent=True,
+            loader_kwargs=loader_kwargs,
+        )
+        data = []
+        for doc in docs:
+            data.append(doc.text)
+        return "\n".join(data), docs
+
+    def read_web_content(
+            self,
+            url: str,
+            type: str = "webpage",
+            extra_args: Optional[Dict[str, Any]] = None
+    ) -> Tuple[str, List[Document]]:
+        """
+        Get content from external resource
+
+        :param url: external url to index
+        :param type: type of URL (webpage, feed, etc.)
+        :param extra_args: extra arguments for loader
+        :return: text content, list of documents
         """
-        docs = self.get_documents(path)
+        docs = self.read_web(url, type, extra_args)
         data = []
         for doc in docs:
             data.append(doc.text)
-        return "\n".join(data)
+        return "\n".join(data), docs
+
+    def read_web(
+            self,
+            url: str,
+            type: str = "webpage",
+            extra_args: Optional[Dict[str, Any]] = None,
+    ) -> List[Document]:
+        """
+        Read data from external resource
+
+        :param url: external url to index
+        :param type: type of URL (webpage, feed, etc.)
+        :param extra_args: extra arguments for loader
+        :return: list of documents
+        """
+        documents = []
+
+        # check if web loader for defined type exists
+        if type not in self.loaders["web"]:
+            raise ValueError("No web loader for type: {}".format(type))
+
+        try:
+            if "url" not in extra_args:
+                extra_args["url"] = url
+
+            # get unique external content identifier
+            unique_id = self.data_providers[type].get_external_id(extra_args)
+            self.window.core.idx.log("Loading web documents from: {}".format(unique_id))
+            self.window.core.idx.log("Using web loader for type: {}".format(type))
+
+            args = self.data_providers[type].prepare_args(**extra_args)
+
+            # get documents from external resource
+            documents = self.loaders["web"][type].load_data(
+                **args
+            )
+        except Exception as e:
+            self.window.core.debug.log(e)
+        return documents
 
     def prepare_document(self, doc: Document):
         """
@@ -234,11 +446,19 @@ class Indexing:
         :param doc: Document
         """
         # fix empty date in Pinecode
-        if "last_accessed_date" in doc.extra_info and doc.extra_info["last_accessed_date"] is None:
-            if "creation_date" in doc.extra_info:
-                doc.extra_info["last_accessed_date"] = doc.extra_info["creation_date"]
+        if "last_accessed_date" in doc.metadata and doc.metadata["last_accessed_date"] is None:
+            if "creation_date" in doc.metadata:
+                doc.metadata["last_accessed_date"] = doc.metadata["creation_date"]
 
-    def index_files(self, idx: str, index: BaseIndex, path: str = None, is_tmp: bool = False) -> tuple:
+    def index_files(
+            self,
+            idx: str,
+            index: BaseIndex,
+            path: Optional[str] = None,
+            is_tmp: bool = False,
+            replace: Optional[bool] = None,
+            recursive: Optional[bool] = None
+    ) -> Tuple[dict, list]:
         """
         Index all files in directory
 
@@ -246,10 +466,16 @@ class Indexing:
         :param index: index instance
         :param path: path to file or directory
         :param is_tmp: True if temporary index
+        :param replace: True if replace old document
+        :param recursive: True if recursive indexing
         :return: dict with indexed files, errors
         """
-        if self.window.core.config.get("llama.idx.recursive"):
-            return self.index_files_recursive(idx, index, path, is_tmp)
+        if recursive is not None:
+            if recursive:
+                return self.index_files_recursive(idx, index, path, is_tmp, replace)
+        else:
+            if self.window.core.config.get("llama.idx.recursive"):
+                return self.index_files_recursive(idx, index, path, is_tmp, replace)
 
         indexed = {}
         errors = []
@@ -262,15 +488,26 @@ class Indexing:
 
         for file in files:   # per file to allow use of multiple loaders
             try:
-                # remove old file from index if exists
-                file_id = self.window.core.idx.files.get_id(file)
-
-                if not is_tmp:
-                    self.remove_old_file(idx, file_id)
+                if self.is_stopped():  # force stop
+                    break
+
+                # force replace or not old document
+                if replace is not None:
+                    if replace:
+                        file_id = self.window.core.idx.files.get_id(file)
+                        self.remove_old_file(idx, file_id, force=True)
+                else:
+                    # if auto, only replace if not temporary
+                    if not is_tmp:
+                        file_id = self.window.core.idx.files.get_id(file)
+                        self.remove_old_file(idx, file_id)
 
                 # index new version of file
                 documents = self.get_documents(file)
                 for d in documents:
+                    if self.is_stopped():  # force stop
+                        break
+
                     self.prepare_document(d)
                     self.index_document(index, d)
                     indexed[file] = d.id_  # add to index
@@ -279,11 +516,19 @@ class Indexing:
                 errors.append(str(e))
                 print("Error while indexing file: " + file)
                 self.window.core.debug.log(e)
-                continue
+                if self.stop_enabled():
+                    break  # break loop if error
 
         return indexed, errors
 
-    def index_files_recursive(self, idx: str, index: BaseIndex, path: str = None, is_tmp: bool = False) -> tuple:
+    def index_files_recursive(
+            self,
+            idx: str,
+            index: BaseIndex,
+            path: Optional[str] = None,
+            is_tmp: bool = False,
+            replace: Optional[bool] = None
+    ) -> Tuple[dict, list]:
         """
         Index all files in directory and subdirectories recursively.
 
@@ -291,10 +536,12 @@ class Indexing:
         :param index: index instance
         :param path: path to file or directory
         :param is_tmp: True if temporary index
+        :param replace: True if replace old document
         :return: dict with indexed files, errors
         """
         indexed = {}
         errors = []
+        is_break = False
 
         # directory
         if os.path.isdir(path):
@@ -303,14 +550,26 @@ class Indexing:
                     file_path = os.path.join(root, file)
                     try:
                         # remove old file from index if exists
-                        file_id = self.window.core.idx.files.get_id(path)
+                        file_id = self.window.core.idx.files.get_id(file_path)
+
+                        if self.is_stopped():  # force stop
+                            break
 
-                        if not is_tmp:
-                            self.remove_old_file(idx, file_id)
+                        # force replace or not old document
+                        if replace is not None:
+                            if replace:
+                                self.remove_old_file(idx, file_id, force=True)
+                        else:
+                            # if auto, only replace if not temporary
+                            if not is_tmp:
+                                self.remove_old_file(idx, file_id)
 
                         # index new version of file
                         documents = self.get_documents(file_path)
                         for d in documents:
+                            if self.is_stopped():  # force stop
+                                break
+
                             self.prepare_document(d)
                             self.index_document(index, d)
                             indexed[file_path] = d.id_  # add to index
@@ -319,7 +578,12 @@ class Indexing:
                         errors.append(str(e))
                         print("Error while indexing file: " + file_path)
                         self.window.core.debug.log(e)
-                        continue
+                        if self.stop_enabled():
+                            is_break = True
+                            break  # break loop if error
+
+                if is_break or self.is_stopped():
+                    break  # stop os.walk if error or forced stop
 
         # file
         elif os.path.isfile(path):
@@ -327,12 +591,21 @@ class Indexing:
                 # remove old file from index if exists
                 file_id = self.window.core.idx.files.get_id(path)
 
-                if not is_tmp:
-                    self.remove_old_file(idx, file_id)
+                # force replace or not old document
+                if replace is not None:
+                    if replace:
+                        self.remove_old_file(idx, file_id, force=True)
+                else:
+                    # if auto, only replace if not temporary
+                    if not is_tmp:
+                        self.remove_old_file(idx, file_id)
 
                 # index new version of file
                 documents = self.get_documents(path)
                 for d in documents:
+                    if self.is_stopped():  # force stop
+                        break
+
                     self.prepare_document(d)
                     self.index_document(index, d)
                     indexed[path] = d.id_  # add to index
@@ -344,7 +617,10 @@ class Indexing:
 
         return indexed, errors
 
-    def get_db_data_from_ts(self, updated_ts: int = 0) -> list:
+    def get_db_data_from_ts(
+            self,
+            updated_ts: int = 0
+    ) -> List[Document]:
         """
         Get data from database from timestamp
 
@@ -384,7 +660,10 @@ class Indexing:
                 documents.append(doc)
         return documents
 
-    def get_db_meta_ids_from_ts(self, updated_ts: int = 0) -> list:
+    def get_db_meta_ids_from_ts(
+            self,
+            updated_ts: int = 0
+    ) -> List[int]:
         """
         Get IDs of meta from database from timestamp
 
@@ -408,7 +687,11 @@ class Indexing:
                 ids.append(data["id"])
         return ids
 
-    def get_db_data_by_id(self, id: int = 0, updated_ts: int = 0) -> list:
+    def get_db_data_by_id(
+            self,
+            id: int = 0,
+            updated_ts: int = 0
+    ) -> List[Document]:
         """
         Get data from database by meta id
 
@@ -446,7 +729,13 @@ class Indexing:
                 documents.append(doc)
         return documents
 
-    def index_db_by_meta_id(self, idx: str, index: BaseIndex, id: int = 0, from_ts: int = 0) -> (int, list):
+    def index_db_by_meta_id(
+            self,
+            idx: str,
+            index: BaseIndex,
+            id: int = 0,
+            from_ts: int = 0
+    ) -> Tuple[int, List[str]]:
         """
         Index data from database by meta id
 
@@ -464,14 +753,19 @@ class Indexing:
                 self.window.core.idx.log("Indexing documents from database by meta id: {}".format(id))
                 self.remove_old_meta_id(idx, id)
             elif from_ts > 0:
-                self.window.core.idx.log("Indexing documents from database by meta id: {} from timestamp: {}".format(id, from_ts))
+                self.window.core.idx.log("Indexing documents from database by meta id: {} from timestamp: {}".
+                                         format(id, from_ts))
 
             # get items from database
             documents = self.get_db_data_by_id(id, from_ts)
             for d in documents:
+                if self.is_stopped():  # force stop
+                    break
+
                 self.index_document(index, d)
                 doc_id = d.id_
-                self.window.core.idx.log("Inserted ctx DB document: {} / {}, id: {}, metadata: {}".format(n+1, len(documents), d.id_, d.metadata))
+                self.window.core.idx.log("Inserted ctx DB document: {} / {}, id: {}, metadata: {}".
+                                         format(n+1, len(documents), d.id_, d.metadata))
                 self.window.core.ctx.idx.set_meta_as_indexed(id, idx, doc_id)  # update ctx
                 n += 1
         except Exception as e:
@@ -479,7 +773,12 @@ class Indexing:
             self.window.core.debug.log(e)
         return n, errors
 
-    def index_db_from_updated_ts(self, idx: str, index: BaseIndex, from_ts: int = 0) -> (int, list):
+    def index_db_from_updated_ts(
+            self,
+            idx: str,
+            index: BaseIndex,
+            from_ts: int = 0
+    ) -> Tuple[int, List[str]]:
         """
         Index data from database from timestamp
 
@@ -493,6 +792,9 @@ class Indexing:
         n = 0
         ids = self.get_db_meta_ids_from_ts(from_ts)
         for id in ids:
+            if self.is_stopped():  # force stop
+                break
+
             indexed, errs = self.index_db_by_meta_id(idx, index, id, from_ts)
             n += indexed
             errors.extend(errs)
@@ -504,9 +806,10 @@ class Indexing:
             index: BaseIndex,
             url: str,
             type="webpage",
-            extra_args: dict = None,
-            is_tmp: bool = False
-    ) -> (int, list):
+            extra_args: Optional[Dict[str, Any]] = None,
+            is_tmp: bool = False,
+            replace: Optional[bool] = None
+    ) -> Tuple[int, List[str]]:
         """
         Index data from external (remote) resource
 
@@ -516,6 +819,7 @@ class Indexing:
         :param type: type of URL (webpage, feed, etc.)
         :param extra_args: extra arguments for loader
         :param is_tmp: True if temporary index
+        :param replace: True if force replace old document
         :return: number of indexed documents, errors
         """
         errors = []
@@ -541,8 +845,13 @@ class Indexing:
             unique_id = self.data_providers[type].get_external_id(extra_args)
 
             # remove old document from index
-            if not is_tmp:
-                self.remove_old_external(idx, unique_id, type)
+            if replace is not None:
+                if replace:
+                    self.remove_old_external(idx, unique_id, type, force=True)
+            else:
+                # if auto, only replace if not temporary
+                if not is_tmp:
+                    self.remove_old_external(idx, unique_id, type)
 
             self.window.core.idx.log("Loading web documents from: {}".format(unique_id))
             self.window.core.idx.log("Using web loader for type: {}".format(type))
@@ -558,6 +867,9 @@ class Indexing:
             self.window.core.idx.metadata.append_web_metadata(documents, type, args)
 
             for d in documents:
+                if self.is_stopped():  # force stop
+                    break
+
                 self.index_document(index, d)
                 doc_id = d.id_  # URL is used as document ID
                 if not is_tmp:
@@ -567,7 +879,8 @@ class Indexing:
                         idx=idx,
                         doc_id=doc_id,
                     )  # update external index
-                self.window.core.idx.log("Inserted web document: {} / {}, id: {}, metadata: {}".format(n+1, len(documents), d.id_, d.metadata))
+                self.window.core.idx.log("Inserted web document: {} / {}, id: {}, metadata: {}".
+                                         format(n+1, len(documents), d.id_, d.metadata))
                 n += 1
         except Exception as e:
             errors.append(str(e))
@@ -580,9 +893,9 @@ class Indexing:
             index: BaseIndex,
             urls: list,
             type="webpage",
-            extra_args: dict = None,
+            extra_args: Optional[Dict[str, Any]] = None,
             is_tmp: bool = False
-    ) -> (int, list):
+    ) -> Tuple[int, List[str]]:
         """
         Index data from URLs
 
@@ -605,6 +918,9 @@ class Indexing:
             return n, errors
 
         for url in urls:
+            if self.is_stopped():  # force stop
+                break
+
             indexed, errs = self.index_url(
                 idx=idx,
                 index=index,
@@ -617,16 +933,22 @@ class Indexing:
             errors.extend(errs)
         return n, errors
 
-    def remove_old_meta_id(self, idx: str, id: int = 0) -> bool:
+    def remove_old_meta_id(
+            self,
+            idx: str,
+            id: int = 0,
+            force: bool = False
+    ) -> bool:
         """
         Remove old meta id from index
 
         :param idx: index name
         :param id: ctx meta id
+        :param force: force remove
         :return: True if removed, False if not
         """
         # abort if not configured to replace old documents
-        if not self.window.core.config.get("llama.idx.replace_old"):
+        if not self.window.core.config.get("llama.idx.replace_old") and not force:
             return False
 
         store = self.window.core.idx.get_current_store()
@@ -640,20 +962,26 @@ class Indexing:
                         doc_id=doc_id,
                     )
                 except Exception as e:
-                    pass
+                    self.window.core.debug.log(e)
                 return True
         return False
 
-    def remove_old_file(self, idx: str, file_id: str):
+    def remove_old_file(
+            self,
+            idx: str,
+            file_id: str,
+            force: bool = False
+    ) -> bool:
         """
         Remove old file from index
 
         :param idx: index name
         :param file_id: file id
+        :param force: force remove
         :return: True if removed, False if not
         """
         # abort if not configured to replace old documents
-        if not self.window.core.config.get("llama.idx.replace_old"):
+        if not self.window.core.config.get("llama.idx.replace_old") and not force:
             return False
 
         store = self.window.core.idx.get_current_store()
@@ -667,21 +995,28 @@ class Indexing:
                         doc_id=doc_id,
                     )
                 except Exception as e:
-                    pass
+                    self.window.core.debug.log(e)
                 return True
         return False
 
-    def remove_old_external(self, idx: str, content: str, type: str):
+    def remove_old_external(
+            self,
+            idx: str,
+            content: str,
+            type: str,
+            force: bool = False
+    ) -> bool:
         """
         Remove old file from index
 
         :param idx: index name
         :param content: content
         :param type: type
+        :param force: force remove
         :return: True if removed, False if not
         """
         # abort if not configured to replace old documents
-        if not self.window.core.config.get("llama.idx.replace_old"):
+        if not self.window.core.config.get("llama.idx.replace_old") and not force:
             return False
 
         store = self.window.core.idx.get_current_store()
@@ -695,11 +1030,15 @@ class Indexing:
                         doc_id=doc_id,
                     )
                 except Exception as e:
-                    pass
+                    self.window.core.debug.log(e)
                 return True
         return False
 
-    def index_document(self, index: BaseIndex, doc: Document):
+    def index_document(
+            self,
+            index: BaseIndex,
+            doc: Document
+    ):
         """
         Index document
 
@@ -707,8 +1046,137 @@ class Indexing:
         :param doc: document
         """
         self.apply_rate_limit()  # apply RPM limit
+        """
+        try:
+            # display embedding model info
+            if index._embed_model is not None:
+                self.window.core.idx.log("Embedding model: {}".format(index.service_context.embed_model.model_name))
+        except Exception as e:
+            self.window.core.debug.log(e)
+        """
         index.insert(document=doc)
 
+    def index_attachment(
+            self,
+            file_path: str,
+            index_path: str,
+            model: Optional[ModelItem] = None,
+            documents: Optional[List[Document]] = None,
+    ) -> list:
+        """
+        Index context attachment
+
+        :param file_path: path to file to index
+        :param index_path: index path
+        :param model: model
+        :param documents: list of documents (optional)
+        :return: response
+        """
+        if model is None:
+            model = self.window.core.models.from_defaults()
+
+        llm, embed_model = self.window.core.idx.llm.get_service_context(model=model)
+        index = self.window.core.idx.storage.get_ctx_idx(
+            index_path,
+            llm=llm,
+            embed_model=embed_model,
+        )  # get or create ctx index
+
+        idx = "tmp:{}".format(index_path)  # tmp index id
+        self.window.core.idx.log("Indexing to context attachment index: {}...".format(idx))
+
+        doc_ids = []
+        if documents is None:
+            documents = self.get_documents(file_path)
+        for d in documents:
+            if self.is_stopped():  # force stop
+                break
+            self.prepare_document(d)
+            self.index_document(index, d)
+            doc_ids.append(d.id_)  # add to index
+
+        self.window.core.idx.storage.store_ctx_idx(index_path, index)
+        return doc_ids
+
+    def index_attachment_web(
+            self,
+            url: str,
+            index_path: str,
+            model: Optional[ModelItem] = None,
+            documents: Optional[List[Document]] = None,
+    ) -> list:
+        """
+        Index context attachment
+
+        :param url: URL to index
+        :param index_path: index path
+        :param model: model
+        :param documents: list of documents (optional)
+        :return: response
+        """
+        if model is None:
+            model = self.window.core.models.from_defaults()
+
+        llm, embed_model = self.window.core.idx.llm.get_service_context(model=model)
+        index = self.window.core.idx.storage.get_ctx_idx(index_path, llm, embed_model)  # get or create ctx index
+
+        idx = "tmp:{}".format(index_path)  # tmp index id
+        self.window.core.idx.log("Indexing to context attachment index: {}...".format(idx))
+
+        web_type = self.get_webtype(url)
+        doc_ids = []
+        if documents is None:
+            documents = self.read_web(
+                url=url,
+                type=web_type,
+                extra_args={},
+            )
+        for d in documents:
+            if self.is_stopped():  # force stop
+                break
+            self.prepare_document(d)
+            self.index_document(index, d)
+            doc_ids.append(d.id_)  # add to index
+
+        self.window.core.idx.storage.store_ctx_idx(index_path, index)
+        return doc_ids
+
+    def get_webtype(self, url: str) -> str:
+        """
+        Get web loader type by URL
+
+        :param url: URL
+        :return: web loader type
+        """
+        type = "webpage"  # default
+        for id in self.data_providers:
+            loader = self.data_providers[id]
+            if hasattr(loader, "is_supported_attachment"):
+                if loader.is_supported_attachment(url):
+                    type = id
+                    break
+        print("Selected web data loader: {}".format(type))
+        return type
+
+    def remove_attachment(
+            self,
+            index_path: str,
+            doc_id: str
+    ) -> bool:
+        """
+        Remove document from index
+
+        :param index_path: index path
+        :param doc_id: document ID
+        :return: True if success
+        """
+        model = self.window.core.models.from_defaults()
+        llm, embed_model = self.window.core.idx.llm.get_service_context(model=model)
+        index = self.window.core.idx.storage.get_ctx_idx(index_path, llm, embed_model)  # get or create ctx index
+        index.delete_ref_doc(doc_id)
+        self.window.core.idx.storage.store_ctx_idx(index_path, index)
+        return True
+
     def apply_rate_limit(self):
         """Apply API calls RPM limit"""
         max_per_minute = 60
@@ -725,3 +1193,19 @@ class Indexing:
                 self.window.core.idx.log("RPM limit: sleep for {} seconds".format(sleep_time))
                 time.sleep(sleep_time)
         self.last_call = now
+
+    def stop_enabled(self) -> bool:
+        """
+        Check if stop on error is enabled
+
+        :return: True if enabled
+        """
+        return self.window.core.config.get('llama.idx.stop.error')
+
+    def is_stopped(self) -> bool:
+        """
+        Check if indexing is stopped
+
+        :return: True if stopped
+        """
+        return self.window.controller.idx.is_stopped()