ws-bom-robot-app 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

ws_bom_robot_app/config.py

@@ -53,7 +53,6 @@ class Settings(BaseSettings):
                 pass
             return 1
         debug: bool
-        loader_strategy: str
         loader_show_progress: bool
         loader_silent_errors: bool
         number_of_workers: int = _get_number_of_workers()
@@ -68,11 +67,11 @@ class Settings(BaseSettings):
             the loader options is usefull to minimizing sytem requirements/dependencies for local development
       """
       if self.robot_env == "local":
-        return self.RuntimeOptions(debug=True,loader_strategy="auto",loader_show_progress=True, loader_silent_errors=True)
+        return self.RuntimeOptions(debug=True,loader_show_progress=True, loader_silent_errors=True)
       elif self.robot_env == "development":
-        return self.RuntimeOptions(debug=True,loader_strategy="",loader_show_progress=True, loader_silent_errors=False)
+        return self.RuntimeOptions(debug=True,loader_show_progress=True, loader_silent_errors=False)
       else:
-        return self.RuntimeOptions(debug=False,loader_strategy="",loader_show_progress=False, loader_silent_errors=True)
+        return self.RuntimeOptions(debug=False,loader_show_progress=False, loader_silent_errors=True)
 
 # global instance
 config = Settings()

ws_bom_robot_app/llm/main.py

@@ -25,6 +25,7 @@ async def invoke(rq: InvokeRequest) -> str:
 
 async def __stream(rq: StreamRequest,queue: Queue,formatted: bool = True) -> None:
   await rq.initialize()
+  #os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
   if formatted:
     agent_handler = AgentHandler(queue,rq.thread_id)
   else:

ws_bom_robot_app/llm/utils/chunker.py

@@ -0,0 +1,15 @@
+from langchain_core.documents import Document
+from langchain_text_splitters import CharacterTextSplitter
+
+class DocumentChunker:
+  @staticmethod
+  def chunk(documents: list[Document]) -> list[Document]:
+      text_splitter = CharacterTextSplitter(chunk_size=10_000, chunk_overlap=500)
+      chunked_documents = []
+      for doc in documents:
+          chunks = text_splitter.split_text(doc.page_content)
+          for chunk in chunks:
+              chunked_documents.append(
+                  Document(page_content=chunk, metadata=doc.metadata)
+              )
+      return chunked_documents

ws_bom_robot_app/llm/utils/faiss_helper.py

@@ -4,17 +4,24 @@ from langchain_core.vectorstores.base import VectorStoreRetriever
 from langchain_openai import OpenAIEmbeddings
 from typing import Any
 import asyncio, gc, logging
+from langchain_text_splitters import CharacterTextSplitter
 from pydantic import SecretStr
 
+from ws_bom_robot_app.llm.utils.chunker import DocumentChunker
+
 class FaissHelper():
     _embedding_model = "text-embedding-3-small"
     _CACHE: dict[str, FAISS] = {}
+
     @staticmethod
     #@timer
     async def create(documents: list[Document], folder_path: str, api_key: SecretStr, return_folder_path:bool = False) -> str | None:
       try:
         embeddings = OpenAIEmbeddings(api_key=api_key, model=FaissHelper._embedding_model)
-        faiss_instance = await asyncio.to_thread(FAISS.from_documents, documents, embeddings)
+        faiss_instance = await asyncio.to_thread(
+           FAISS.from_documents,
+           DocumentChunker.chunk(documents),
+           embeddings)
         await asyncio.to_thread(faiss_instance.save_local, folder_path)
         del faiss_instance, embeddings
         gc.collect()

ws_bom_robot_app/llm/vector_store/generator.py

@@ -1,4 +1,4 @@
-import os, gc, shutil, traceback
+import os, gc, shutil, logging, traceback
 import asyncio, aiofiles, aiofiles.os
 from fastapi import HTTPException
 from fastapi.responses import StreamingResponse
@@ -67,7 +67,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
         documents.extend(await loaders.load())
       except Exception as e:
         tb = traceback.format_exc()
-        return VectorDbResponse(success = False, error = f"File loader failure: {e} | {tb}")
+        _error = f"File loader failure: {e} | {tb}"
+        logging.warning(_error)
+        return VectorDbResponse(success = False, error = _error)
     except Exception as e:
       await _cleanup_directory(working_path)
       return VectorDbResponse(success = False, error = f"Failed to download file {e}")
@@ -78,7 +80,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
     except Exception as e:
       await _cleanup_directory(working_path)
       tb = traceback.format_exc()
-      return VectorDbResponse(success = False, error = f"Endpoint failure: {e} | {tb}")
+      _error = f"Endpoint failure: {e} | {tb}"
+      logging.warning(_error)
+      return VectorDbResponse(success = False, error = _error)
 
   if rq.integrations:
     tasks = []
@@ -95,7 +99,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
     except Exception as e:
       await _cleanup_directory(working_path)
       tb = traceback.format_exc()
-      return VectorDbResponse(success=False, error=f"Integration failure: {e} | {tb}")
+      _error = f"Integration failure: {e} | {tb}"
+      logging.warning(_error)
+      return VectorDbResponse(success=False, error=_error)
 
   #cleanup
   await _cleanup_directory(working_path)
@@ -116,7 +122,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
       del documents
       gc.collect()
   else:
-    return VectorDbResponse(success = False, error = "No documents found in the knowledgebase folder")
+    _error = "No documents found in the knowledgebase folder"
+    logging.warning(_error)
+    return VectorDbResponse(success = False, error = _error)
 
 async def kb_stream_file(filename: str):
     file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)

ws_bom_robot_app/llm/vector_store/integration/dropbox.py

@@ -12,7 +12,7 @@ class DropboxParams(BaseModel):
   Attributes:
     remote_url (str): The URL of the remote Dropbox location, e.g. 'dropbox://demo-directory' or 'dropbox://demo-directory/sub-directory'.
     token (str): The authentication token for accessing Dropbox.
-      create app: https://www.dropbox.com/developers, with file.content.read permission, and generate token.
+      create app: https://www.dropbox.com/developers, with file.content.read permission, and generate token, or use existing app: https://www.dropbox.com/account/connected_apps / https://www.dropbox.com/developers/apps?_tk=pilot_lp&_ad=topbar4&_camp=myapps
     recursive (bool, optional): A flag indicating whether to search directories recursively. Defaults to False.
     extension (list[str], optional): A list of file extensions to filter by. Defaults to None, e.g. ['.pdf', '.docx'].
   """

ws_bom_robot_app/llm/vector_store/integration/manager.py

@@ -9,6 +9,7 @@ from ws_bom_robot_app.llm.vector_store.integration.googledrive import GoogleDriv
 from ws_bom_robot_app.llm.vector_store.integration.jira import Jira
 from ws_bom_robot_app.llm.vector_store.integration.s3 import S3
 from ws_bom_robot_app.llm.vector_store.integration.sftp import Sftp
+from ws_bom_robot_app.llm.vector_store.integration.sharepoint import Sharepoint
 from ws_bom_robot_app.llm.vector_store.integration.sitemap import Sitemap
 from ws_bom_robot_app.llm.vector_store.integration.slack import Slack
 
@@ -23,6 +24,7 @@ class IntegrationManager:
     "llmkbjira": Jira,
     "llmkbs3": S3,
     "llmkbsftp": Sftp,
+    "llmkbsharepoint": Sharepoint,
     "llmkbsitemap": Sitemap,
     "llmkbslack": Slack,
 

ws_bom_robot_app/llm/vector_store/integration/sharepoint.py

@@ -1,12 +1,27 @@
-import asyncio
+import asyncio, logging, traceback
+from dataclasses import dataclass
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.sharepoint  import SharepointIndexerConfig, SharepointDownloaderConfig, SharepointConnectionConfig, SharepointAccessConfig
+from unstructured_ingest.v2.processes.connectors.sharepoint  import SharepointIndexerConfig, SharepointIndexer, SharepointDownloaderConfig, SharepointConnectionConfig, SharepointAccessConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional
 from pydantic import BaseModel, Field, AliasChoices
 
 class SharepointParams(BaseModel):
+  """
+  SharepointParams is a Pydantic model that defines the parameters required to connect to a SharePoint site.
+
+  Attributes:
+    client_id (str): The client ID for SharePoint authentication.
+    client_secret (str): The client secret for SharePoint authentication.
+    site_url (str): The URL of the SharePoint site. i.e. site collection level: https://<tenant>.sharepoint.com/sites/<site-collection-name>, or root site: https://<tenant>.sharepoint.com
+    site_path (str, optional): TThe path in the SharePoint site from which to start parsing files, for example "Shared Documents". Defaults to None.
+    recursive (bool, optional): Whether to recursively access subdirectories. Defaults to False.
+    omit_files (bool, optional): Whether to omit files from the results. Defaults to False.
+    omit_pages (bool, optional): Whether to omit pages from the results. Defaults to False.
+    omit_lists (bool, optional): Whether to omit lists from the results. Defaults to False.
+    extension (list[str], optional): A list of file extensions to include, i.e. [".pdf"]  Defaults to None.
+  """
   client_id : str = Field(validation_alias=AliasChoices("clientId","client_id"))
   client_secret : str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
   site_url: str = Field(validation_alias=AliasChoices("siteUrl","site_url"))
@@ -40,12 +55,52 @@ class Sharepoint(IntegrationStrategy):
       site=self.__data.site_url,
       permissions_config=None
     )
-    self.__unstructured_ingest.pipeline(
+    pipeline = self.__unstructured_ingest.pipeline(
       indexer_config,
       downloader_config,
       connection_config,
-      extension=self.__data.extension).run()
+      extension=self.__data.extension)
+    current_indexer_process = pipeline.indexer_step.process
+    pipeline.indexer_step.process = CustomSharepointIndexer(**vars(current_indexer_process))
+    pipeline.run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)
       return await Loader(self.working_directory).load()
+
+@dataclass
+class CustomSharepointIndexer(SharepointIndexer):
+  def __init__(self, **kwargs):
+      # Initialize all attributes from the base indexer
+      for key, value in kwargs.items():
+          setattr(self, key, value)
+  def list_files(self, folder, recursive):
+      try:
+        _files = super().list_files(folder, recursive)
+        return _files
+      except Exception as e:
+        tb = traceback.format_exc()
+        logging.error(f"Error listing sharepoint files: {e} \n {tb}")
+        return []
+  def file_to_file_data(self, client, file):
+    try:
+      return super().file_to_file_data(client, file)
+    except Exception as e:
+      tb = traceback.format_exc()
+      logging.error(f"Error converting sharepoint file {file} to data: {e} \n {tb}")
+      return None
+  def list_pages(self, client):
+    try:
+      _pages = super().list_pages(client)
+      _allowed_content_type = None
+      for page in _pages:
+        # determine the allowed content type from the first page (Home.aspx)
+        if not _allowed_content_type:
+          _allowed_content_type = page.content_type_id
+        if not page.content_type_id == _allowed_content_type:
+          _pages.remove_child(page)
+      return _pages
+    except Exception as e:
+      tb = traceback.format_exc()
+      logging.error(f"Error listing sharepoint pages: {e} \n {tb}")
+      return []

ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -1,21 +1,18 @@
-
 import asyncio, gc, logging, os, traceback
 from typing import Any, Optional
 from langchain_community.document_loaders import DirectoryLoader
 from langchain_community.document_loaders.base import BaseLoader
 from langchain_community.document_loaders.merge import MergedDataLoader
 from langchain_core.documents import Document
-from langchain_unstructured import UnstructuredLoader
 from pydantic import BaseModel
 from ws_bom_robot_app.config import config
 from ws_bom_robot_app.llm.vector_store.loader.json_loader import JsonLoader
+from ws_bom_robot_app.llm.vector_store.loader.docling import DoclingLoader
+from langchain_community.document_loaders import CSVLoader, UnstructuredPowerPointLoader, UnstructuredEmailLoader, UnstructuredXMLLoader, TextLoader, UnstructuredHTMLLoader
 
 class LoaderConfig(BaseModel):
   loader: type[BaseLoader]
-  kwargs: Optional[dict[str, Any]] = {
-       'chunking_strategy': 'basic',
-       'max_characters': 10_000
-       }
+  kwargs: Optional[dict[str, Any]] = {}
   #post_processors: Optional[list[Callable[[str], str]]] = None
 
 class Loader():
@@ -25,39 +22,36 @@ class Loader():
 
   _list: dict[str, LoaderConfig | None] = {
     '.json': LoaderConfig(loader=JsonLoader),
-    '.csv': LoaderConfig(loader=UnstructuredLoader),
-    '.xls': LoaderConfig(loader=UnstructuredLoader),
-    '.xlsx': LoaderConfig(loader=UnstructuredLoader),
-    '.eml': LoaderConfig(loader=UnstructuredLoader),
-    '.msg': LoaderConfig(loader=UnstructuredLoader),
+    '.csv': LoaderConfig(loader=CSVLoader),
+    '.xls': None,
+    '.xlsx': LoaderConfig(loader=DoclingLoader),
+    '.eml': LoaderConfig(loader=UnstructuredEmailLoader,kwargs={"strategy":"auto", "process_attachments": False}),
+    '.msg': LoaderConfig(loader=UnstructuredEmailLoader,kwargs={"strategy":"auto", "process_attachments": False}),
     '.epub': None,
-    '.md': LoaderConfig(loader=UnstructuredLoader),
+    '.md': LoaderConfig(loader=TextLoader),
     '.org': None,
     '.odt': None,
     '.ppt': None,
-    '.pptx': LoaderConfig(loader=UnstructuredLoader),
-    '.txt': LoaderConfig(loader=UnstructuredLoader),
+    '.pptx': LoaderConfig(loader=UnstructuredPowerPointLoader,kwargs={"strategy":"auto"}), #docling issue with WMF https://github.com/DS4SD/docling/issues/594
+    '.txt': LoaderConfig(loader=TextLoader),
     '.rst': None,
     '.rtf': None,
     '.tsv': None,
     '.text': None,
     '.log': None,
-    '.htm': LoaderConfig(loader=UnstructuredLoader),
-    '.html': LoaderConfig(loader=UnstructuredLoader),
-    '.pdf': LoaderConfig(loader=UnstructuredLoader,kwargs={
-       'strategy':'ocr_only', #https://docs.unstructured.io/open-source/core-functionality/partitioning auto,ocr_only,hi_res
-       'split_pdf_page': False,
-       'chunking_strategy': 'basic',
-       'max_characters': 10_000,
-       'include_page_breaks': True,
-       'include_orig_elements': False}),
-    '.png': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
-    '.jpg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
-    '.jpeg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
+    '.htm': LoaderConfig(loader=UnstructuredHTMLLoader,kwargs={"strategy":"auto"}),
+    '.html': LoaderConfig(loader=UnstructuredHTMLLoader,kwargs={"strategy":"auto"}),
+    ".pdf": LoaderConfig(loader=DoclingLoader),
+    '.png': LoaderConfig(loader=DoclingLoader),
+    '.jpg': LoaderConfig(loader=DoclingLoader),
+    '.jpeg': LoaderConfig(loader=DoclingLoader),
+    '.gif': None,
+    ".emf": None,
+    ".wmf": None,
     '.tiff': None,
     '.doc': None, #see liberoffice dependency
-    '.docx': LoaderConfig(loader=UnstructuredLoader),
-    '.xml': LoaderConfig(loader=UnstructuredLoader),
+    '.docx': LoaderConfig(loader=DoclingLoader),
+    '.xml': LoaderConfig(loader=UnstructuredXMLLoader,kwargs={"strategy":"auto"}),
     '.js': None,
     '.py': None,
     '.c': None,
@@ -81,8 +75,6 @@ class Loader():
     loader_configs = {}
     for ext, loader_config in Loader._list.items():
         if loader_config:
-            if all([self._runtime_options.loader_strategy != "",loader_config.kwargs,"strategy" in loader_config.kwargs]): # type: ignore
-                loader_config.kwargs["strategy"] = self._runtime_options.loader_strategy # type: ignore
             loader_key = (loader_config.loader, tuple(loader_config.kwargs.items())) # type: ignore
             if loader_key not in loader_configs:
                 loader_configs[loader_key] = {
@@ -122,7 +114,7 @@ class Loader():
           return _documents
         except Exception as e:
           logging.warning(f"Attempt {attempt+1} load document  failed: {e}")
-          await asyncio.sleep(1)
+          await asyncio.sleep(2)
           if attempt == MAX_RETRIES - 1:
             tb = traceback.format_exc()
             logging.error(f"Failed to load documents: {e} | {tb}")

ws_bom_robot_app/llm/vector_store/loader/docling.py

@@ -0,0 +1,37 @@
+import os, logging, traceback
+from typing import Iterator, AsyncIterator, Optional
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+from langchain_core.runnables import run_in_executor
+from docling.document_converter import DocumentConverter, ConversionResult, ConversionStatus
+
+class DoclingLoader(BaseLoader):
+  def __init__(self, file_path: str | list[str]) -> None:
+      self._file_paths = file_path if isinstance(file_path, list) else [file_path]
+      self._converter = DocumentConverter()
+  def load(self) -> list[Document]:
+      """Load data into Document objects."""
+      return list(self.lazy_load())
+  async def aload(self) -> list[Document]:
+      """Load data into Document objects."""
+      return [document async for document in self.alazy_load()]
+  async def alazy_load(self) -> AsyncIterator[Document]:
+      """A lazy loader for Documents."""
+      iterator = await run_in_executor(None, self.lazy_load)
+      done = object()
+      while True:
+          doc = await run_in_executor(None, next, iterator, done)  # type: ignore[call-arg, arg-type]
+          if doc is done:
+              break
+          yield doc  # type: ignore[misc]
+  def lazy_load(self) -> Iterator[Document]:
+      for source in self._file_paths:
+          try:
+            _result = self._converter.convert(
+               os.path.abspath(source),
+               raises_on_error=True)
+            doc = _result.document
+            text = doc.export_to_markdown(image_placeholder="")
+            yield Document(page_content=text, metadata={"source": source})
+          except Exception as e:
+            logging.warning(f"Failed to load document from {source}: {e} | {traceback.format_exc()}")

{ws_bom_robot_app-0.0.24.dist-info → ws_bom_robot_app-0.0.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.24
+Version: 0.0.26
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -21,10 +21,8 @@ Requires-Dist: langchain-openai==0.2.10
 Requires-Dist: langchain-community==0.3.8
 Requires-Dist: langchain-core==0.3.21
 Requires-Dist: faiss-cpu==1.9.0
-Requires-Dist: python-magic==0.4.27
-Requires-Dist: opencv-python-headless==4.10.0.84
-Requires-Dist: unstructured[all-docs]==0.16.11
-Requires-Dist: langchain_unstructured==0.1.5
+Requires-Dist: unstructured==0.16.11
+Requires-Dist: unstructured[image]
 Requires-Dist: unstructured-ingest==0.3.8
 Requires-Dist: unstructured-ingest[azure]
 Requires-Dist: unstructured-ingest[confluence]
@@ -35,10 +33,11 @@ Requires-Dist: unstructured-ingest[google_drive]
 Requires-Dist: unstructured-ingest[jira]
 Requires-Dist: unstructured-ingest[s3]
 Requires-Dist: unstructured-ingest[sftp]
+Requires-Dist: unstructured-ingest[sharepoint]
 Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
-Requires-Dist: nebuly==0.3.33
+Requires-Dist: nebuly==0.3.35
 
 # 🤖 ws-bom-robot-app
 
@@ -214,11 +213,22 @@ launch debugger
 streamlit run debugger.py --server.port 6002
 ```
 
+dockerize base image
+
+```pwsh
+<# cpu #>
+docker build -f Dockerfile-robot-base-cpu -t ghcr.io/websolutespa/ws-bom-robot-base:cpu .
+docker push ghcr.io/websolutespa/ws-bom-robot-base:cpu
+<# gpu #>
+docker build -f Dockerfile-robot-base-gpu -t ghcr.io/websolutespa/ws-bom-robot-base:gpu .
+docker push ghcr.io/websolutespa/ws-bom-robot-base:gpu
+```
+
 dockerize app from src
 
 ```pwsh
 docker build -f Dockerfile-src -t ws-bom-robot-app:src .
-docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -p 6001:6001 ws-bom-robot-app:src
+docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tests:/app/tests" -p 6001:6001 ws-bom-robot-app:src
 ```
 
 ### ✈️ publish

{ws_bom_robot_app-0.0.24.dist-info → ws_bom_robot_app-0.0.26.dist-info}/RECORD

@@ -1,6 +1,6 @@
 ws_bom_robot_app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/auth.py,sha256=84nIbmJsMrNs0sxIQGEHbjsjc2P6ZrZZGSn8dkiL6is,895
-ws_bom_robot_app/config.py,sha256=V5ZrX_JnzpsN32hTTezTfOvEZYkIQBy0lxIQ9JFHdFE,3170
+ws_bom_robot_app/config.py,sha256=7ZTi5GblQFexYB8zt9oAsNmXh193Yz1vMQYq5A65DcI,3079
 ws_bom_robot_app/cron_manager.py,sha256=0Yt5AMTPGlXZ_M5ck0SKMX8wvzoPsseEezg_s0Q3HKY,9224
 ws_bom_robot_app/main.py,sha256=vChP8vfmOCbs51TPUsaaxX8FvoFXuURMkOgmgx0Xi_4,6121
 ws_bom_robot_app/task_manager.py,sha256=Zedzs2R3O-wNSQOqs4jorgFwPRi-ji_0TN4mGfk-VvE,15958
@@ -11,7 +11,7 @@ ws_bom_robot_app/llm/agent_handler.py,sha256=Qz3h1eZdA6pkurEbr8sQwl-0FdjugaO5Q9s
 ws_bom_robot_app/llm/agent_lcel.py,sha256=jkSLMy6y_ZFvWT8bhBBYHY5CO-ea8oMSPMXMahFUBFc,2666
 ws_bom_robot_app/llm/api.py,sha256=vBu_TFTlBjp7e3J-WmlZbXn_TbB550x-NpQN4YsO7To,3004
 ws_bom_robot_app/llm/defaut_prompt.py,sha256=pn5a4lNLWE1NngHYjA_7tD8GasePMgsgude5fIJxsW0,756
-ws_bom_robot_app/llm/main.py,sha256=-JG5WxJfvMzsfasRRRv_AO5u7Tk9p_9P_nGWYyytesk,3500
+ws_bom_robot_app/llm/main.py,sha256=_uW3Iy9iPJbxDfpyoReu3mbYY8a9dS1V6tZU-z6BELo,3547
 ws_bom_robot_app/llm/settings.py,sha256=EkFGCppORenStH9W4e6_dYvQ-5p6xiEMpmUHBqNqG9M,117
 ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/models/api.py,sha256=KlVUbApyz6uuWefAN9K4B_vWDSps5hLW6hNg1Eo3TBQ,6996
@@ -25,32 +25,34 @@ ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 ws_bom_robot_app/llm/tools/models/main.py,sha256=LsOJ7vkcSzYLoE1oa3TG0Rs0pr9J5VS_e4li6aDx_fw,260
 ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/utils/agent_utils.py,sha256=LEfAKQwFrwmIdJL0o54iuGrir9uLcJhkciLL3R6Xqwo,814
+ws_bom_robot_app/llm/utils/chunker.py,sha256=N7570xBYlObneg-fsvDhPAJ-Pv8C8OaYZOBK6q7LmMI,607
 ws_bom_robot_app/llm/utils/download.py,sha256=iAUxH_NiCpTPtGzhC4hBtxotd2HPFt2MBhttslIxqiI,3194
-ws_bom_robot_app/llm/utils/faiss_helper.py,sha256=69juxptz1gidgxVOrqNvJajRl40p5-ugHqyEBDtnSKo,5036
+ws_bom_robot_app/llm/utils/faiss_helper.py,sha256=VikpopCpEzV1lN5JISDabpHcIUkNDACNL52KliB4Hxs,5224
 ws_bom_robot_app/llm/utils/kb.py,sha256=jja45WCbNI7SGEgqDS99nErlwB5eY8Ga7BMnhdMHZ90,1279
 ws_bom_robot_app/llm/utils/print.py,sha256=ZonoLPcfM6Cpw4_Ec455LiCovExOwvnIgvw1QORSCBY,799
 ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
 ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/generator.py,sha256=aVUG08rcsDvtVxfYfUalO1CbKGkLazNyP555IsQQZso,5975
+ws_bom_robot_app/llm/vector_store/generator.py,sha256=Xg-srcH_03lqPHkMn1EXP56GbY1CYa2zIbjvNfcQqyM,6192
 ws_bom_robot_app/llm/vector_store/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/integration/azure.py,sha256=R37TaPQP-HJJJiaKE9rmMc9kpeXeRvdebbTY_982om0,3392
 ws_bom_robot_app/llm/vector_store/integration/base.py,sha256=IvIu8RkISuurrVKr2YPG96fsOI2kqhaEGyTGzjB-jCI,1550
 ws_bom_robot_app/llm/vector_store/integration/confluence.py,sha256=4fiRHB3J-SHZZxNGHwVkCrT-xSPbc91z4WrDE9fy6xU,2505
-ws_bom_robot_app/llm/vector_store/integration/dropbox.py,sha256=Am0jfh54OFUTQihqmkwgKIkcEmjehDh_0NQzBuxTdGY,2481
+ws_bom_robot_app/llm/vector_store/integration/dropbox.py,sha256=yhGvHTN0TEpUfhdvvV7RX5MxBwTUyddAX95Fgqp3mCg,2629
 ws_bom_robot_app/llm/vector_store/integration/gcs.py,sha256=fFDVDUR6eNB7FVTzDSEpMHFEWMgG16GLnpSf_mqGDdE,3184
 ws_bom_robot_app/llm/vector_store/integration/github.py,sha256=18PO30AZcgTn6PHhid3MwImVAdmKBNkr0kmAPgOetGw,2663
 ws_bom_robot_app/llm/vector_store/integration/googledrive.py,sha256=R6hr8iEgrR3QMOzIj5jY6w1x8pZ1LGdh4xM_q7g_ttc,3738
 ws_bom_robot_app/llm/vector_store/integration/jira.py,sha256=o5iINIblp_yNszp54nf7fW97aqjs0A5G89N8sYrd1ds,2771
-ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=_aNiy3w_fnWOwaGkp_X_hDNstnPbxtCuojikBzRSArE,1583
+ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=5Fl3XML6f1wmgraigpUwIFIXh7QFPX0RI0YFgFxBAvg,1700
 ws_bom_robot_app/llm/vector_store/integration/s3.py,sha256=3kh-VmH84IW7DdSLvOk6td1VBJ9aohlVJsk5F3cYj0U,3320
 ws_bom_robot_app/llm/vector_store/integration/sftp.py,sha256=WNzjjS1EUykgFB-8e7QkecSa1r1jTJqKyGzR25uJCtM,2848
-ws_bom_robot_app/llm/vector_store/integration/sharepoint.py,sha256=fIUYudNXB46oaIkPR7o_xUctJTZhq6ugKtIjCc0vDyo,2619
+ws_bom_robot_app/llm/vector_store/integration/sharepoint.py,sha256=zqqn-6qPrK50Phch4nZHJTgaPyPkGe7W2InGL_Ru6vE,5376
 ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=nPbIywp-ZwWbWStvjvYVgHqqejyYFr8eZhBc8ycTuaU,4206
 ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=FMjESXm2QetFXI6i8epze7Kbbu22fV8CVaxb71AHnJ8,2572
 ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=f_CfvmwrOfoPnIkFpjcfYtmuYKo3vYJcDGKgtxsMyfU,5216
+ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=GmicqcmEF965Wnd2ysB4ZR_VOxZ0Ys0aNdubssb3_ZE,4963
+ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=dns_A--Wb-oIGYcbrW5RQ6-ALR5cCmySioo389K0GK0,1775
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
-ws_bom_robot_app-0.0.24.dist-info/METADATA,sha256=DHdXDglEMlG9ubAtlCjrViNVynKu5BmzQ6a85iAh-B8,7152
-ws_bom_robot_app-0.0.24.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ws_bom_robot_app-0.0.24.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.24.dist-info/RECORD,,
+ws_bom_robot_app-0.0.26.dist-info/METADATA,sha256=bu2IDziI3DNb_rLBTWnrRXe7S2QVmmqWzP6NOLbNQFU,7507
+ws_bom_robot_app-0.0.26.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ws_bom_robot_app-0.0.26.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.26.dist-info/RECORD,,