ws-bom-robot-app 0.0.23__py3-none-any.whl → 0.0.25__py3-none-any.whl

ws_bom_robot_app/llm/agent_handler.py

@@ -1,3 +1,4 @@
+from asyncio import Queue
 from langchain_core.agents import AgentFinish
 from langchain_core.outputs import ChatGenerationChunk, GenerationChunk
 from langchain.callbacks.base import AsyncCallbackHandler
@@ -16,13 +17,13 @@ import json
 
 class AgentHandler(AsyncCallbackHandler):
 
-    def __init__(self, threadId) -> None:
+    def __init__(self, queue: Queue, threadId: str = None) -> None:
         super().__init__()
         self._threadId = threadId
-        self.json_cards = None
         self.json_block = ""
         self.is_json_block = False
         self.backtick_count = 0  # Conteggio dei backticks per il controllo accurato
+        self.queue = queue
 
     async def on_llm_start(
         self,
@@ -39,7 +40,7 @@ class AgentHandler(AsyncCallbackHandler):
             "type": "info",
             "threadId": self._threadId,
         }
-        printJson(firstChunk)
+        await self.queue.put(printString(firstChunk))
 
     """async def on_chat_model_start(self, serialized: Dict[str, Any], messages: List[List[BaseMessage]], *, run_id: UUID = None, parent_run_id = None, tags = None, metadata = None, **kwargs: Any) -> Any:
         pass"""
@@ -75,7 +76,7 @@ class AgentHandler(AsyncCallbackHandler):
             elif self.is_json_block:
                 self.json_block += token
             else:
-                printString(token)
+                await self.queue.put(printString(token))
         pass
 
     async def on_agent_finish(
@@ -92,12 +93,9 @@ class AgentHandler(AsyncCallbackHandler):
                 AIMessage(content=finish.return_values["output"]),
             ]
         )
-        if self.json_cards:
-          for card in self.json_cards:
-            printJson(card)
-          self.json_cards = None
         finalChunk = {"type": "end"}
-        printJson(finalChunk)
+        await self.queue.put(printJson(finalChunk))
+        await self.queue.put(None)
 
     async def process_json_block(self, json_block: str):
       """Processa il blocco JSON completo."""
@@ -108,15 +106,16 @@ class AgentHandler(AsyncCallbackHandler):
           try:
               # Prova a fare il parsing del JSON
               parsed_json = json.loads(json_block_clean)
-              printJson(parsed_json)
+              await self.queue.put(printJson(parsed_json))
           except json.JSONDecodeError as e:
               # Se il JSON è malformato, logga l'errore
               raise e
 
 class RawAgentHandler(AsyncCallbackHandler):
 
-    def __init__(self) -> None:
+    def __init__(self,queue: Queue) -> None:
         super().__init__()
+        self.queue = queue
 
     async def on_llm_start(
         self,
@@ -147,10 +146,9 @@ class RawAgentHandler(AsyncCallbackHandler):
         tags: Optional[List[str]] = None,
         **kwargs: Any,
     ) -> None:
-        """Gestisce i nuovi token durante lo streaming."""
-        if token != "":
-            print(token)
-        pass
+        """Handles new tokens during streaming."""
+        if token:  # Only process non-empty tokens
+            await self.queue.put(token)
 
     async def on_agent_finish(
         self,
@@ -166,3 +164,4 @@ class RawAgentHandler(AsyncCallbackHandler):
                 AIMessage(content=finish.return_values["output"]),
             ]
         )
+        await self.queue.put(None)

ws_bom_robot_app/llm/api.py

@@ -25,18 +25,10 @@ async def _invoke(rq: InvokeRequest):
 async def _stream(rq: StreamRequest) -> StreamingResponse:
     return StreamingResponse(stream(rq), media_type="application/json")
 
-@router.post("/stream/none")
-async def _stream_none(rq: StreamRequest) -> None:
-    await stream_none(rq)
-
 @router.post("/stream/raw")
 async def _stream_raw(rq: StreamRequest) -> StreamingResponse:
     return StreamingResponse(stream(rq, formatted=False), media_type="application/json")
 
-@router.post("/stream/raw/none")
-async def _stream_raw_none(rq: StreamRequest) -> None:
-     await stream_none(rq, formatted=False)
-
 @router.post("/kb")
 async def _kb(rq: KbRequest) -> VectorDbResponse:
     return await kb(rq)

ws_bom_robot_app/llm/main.py

@@ -10,6 +10,7 @@ from nebuly.providers.langchain import LangChainTrackingHandler
 from langchain_core.callbacks.base import AsyncCallbackHandler
 import warnings, asyncio, os, io, sys, json
 from typing import List
+from asyncio import Queue
 
 async def invoke(rq: InvokeRequest) -> str:
   await rq.initialize()
@@ -22,12 +23,13 @@ async def invoke(rq: InvokeRequest) -> str:
   result: AIMessage = await processor.run_agent(_msg)
   return {"result": result.content}
 
-async def __stream(rq: StreamRequest,formatted: bool = True) -> None:
+async def __stream(rq: StreamRequest,queue: Queue,formatted: bool = True) -> None:
   await rq.initialize()
+  #os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
   if formatted:
-    agent_handler = AgentHandler(rq.thread_id)
+    agent_handler = AgentHandler(queue,rq.thread_id)
   else:
-    agent_handler = RawAgentHandler()
+    agent_handler = RawAgentHandler(queue)
   os.environ["AGENT_HANDLER_FORMATTED"] = str(formatted)
   callbacks: List[AsyncCallbackHandler] = [agent_handler]
   settings.init()
@@ -53,7 +55,7 @@ async def __stream(rq: StreamRequest,formatted: bool = True) -> None:
   processor = AgentLcel(
       openai_config={"api_key": rq.secrets["openAIApiKey"], "openai_model": rq.model, "temperature": rq.temperature},
       sys_message=rq.system_message,
-      tools=get_structured_tools(tools=rq.app_tools, api_key=rq.secrets["openAIApiKey"], callbacks=[callbacks[0]]),
+      tools=get_structured_tools(tools=rq.app_tools, api_key=rq.secrets["openAIApiKey"], callbacks=[callbacks[0]], queue=queue),
       rules=rq.rules
   )
 
@@ -71,25 +73,20 @@ async def __stream(rq: StreamRequest,formatted: bool = True) -> None:
         {"callbacks": callbacks},
     )
 
+  # Signal the end of streaming
+  await queue.put(None)
+
 async def stream(rq: StreamRequest,formatted:bool = True) -> AsyncGenerator[str, None]:
-    sys_stdout = io.StringIO()
-    original_stdout = sys.stdout
-    sys.stdout = sys_stdout
+    queue = Queue()
+    task = asyncio.create_task(__stream(rq, queue, formatted))
     try:
-        task = asyncio.create_task(__stream(rq,formatted))
-        while not task.done():
-            await asyncio.sleep(0.1)  # yield control to avoid blocking
-            output = sys_stdout.getvalue()
-            if output:
-                yield output
-                sys_stdout.truncate(0)
-                sys_stdout.seek(0)
-        # capture any remaining output after the task completes
-        output = sys_stdout.getvalue()
-        if output:
-            yield output
+        while True:
+            token = await queue.get()
+            if token is None:  # None indicates the end of streaming
+                break
+            yield token
     finally:
-        sys.stdout = original_stdout
+        await task
 
 async def stream_none(rq: StreamRequest, formatted: bool = True) -> None:
   await __stream(rq, formatted)

ws_bom_robot_app/llm/tools/tool_builder.py

@@ -1,12 +1,13 @@
+from asyncio import Queue
 from langchain.tools import StructuredTool
 from ws_bom_robot_app.llm.models.api import LlmAppTool
 from ws_bom_robot_app.llm.tools.tool_manager import ToolManager
 
-def get_structured_tools(tools: list[LlmAppTool], api_key:str, callbacks:list) -> list[StructuredTool]:
+def get_structured_tools(tools: list[LlmAppTool], api_key:str, callbacks:list, queue: Queue) -> list[StructuredTool]:
   _structured_tools :list[StructuredTool] = []
   for tool in [tool for tool in tools if tool.is_active]:
     if _tool_config := ToolManager._list.get(tool.function_name):
-      _tool_instance = ToolManager(tool, api_key, callbacks)
+      _tool_instance = ToolManager(tool, api_key, callbacks, queue)
       _structured_tool = StructuredTool.from_function(
         coroutine=_tool_instance.get_coroutine(),
         name=tool.function_id,

ws_bom_robot_app/llm/tools/tool_manager.py

@@ -1,3 +1,4 @@
+from asyncio import Queue
 from typing import  Optional, Type, Callable
 from ws_bom_robot_app.llm.models.api import LlmAppTool
 from ws_bom_robot_app.llm.utils.faiss_helper import FaissHelper
@@ -33,10 +34,12 @@ class ToolManager:
         app_tool: LlmAppTool,
         api_key: str,
         callbacks: list,
+        queue: Optional[Queue] = None
     ):
         self.app_tool = app_tool
         self.api_key = api_key
         self.callbacks = callbacks
+        self.queue = queue
 
 
     #region functions
@@ -64,7 +67,8 @@ class ToolManager:
                 else:
                     search_type = "mixed"
                     search_kwargs = {"k": search_settings.search_k if search_settings.search_k else 4}
-            getRandomWaitingMessage(self.app_tool.waiting_message, traduction=False)
+            if self.queue:
+              await self.queue.put(getRandomWaitingMessage(self.app_tool.waiting_message, traduction=False))
             return await FaissHelper.invoke(self.app_tool.vector_db, self.api_key, query, search_type, search_kwargs)
         return []
         #raise ValueError(f"Invalid configuration for {self.settings.name} tool of type {self.settings.type}. Must be a function or vector db not found.")

ws_bom_robot_app/llm/tools/utils.py

@@ -3,8 +3,8 @@ from langchain_openai import ChatOpenAI
 from langchain_core.prompts import PromptTemplate
 from ws_bom_robot_app.llm.utils.print import printString
 
-def __print_output(data: str) -> None:
-  printString(data) if os.environ.get("AGENT_HANDLER_FORMATTED") == str(True) else print(f"{data} ")
+def __print_output(data: str) -> str:
+  return printString(data) if os.environ.get("AGENT_HANDLER_FORMATTED") == str(True) else f"{data} "
 
 def getRandomWaitingMessage(waiting_messages: str, traduction: bool = True) -> str:
   if not waiting_messages: return ""
@@ -12,13 +12,12 @@ def getRandomWaitingMessage(waiting_messages: str, traduction: bool = True) -> s
   if not messages: return ""
   chosen_message = random.choice(messages) + "\n"
   if not traduction:
-      __print_output(chosen_message)
+      return __print_output(chosen_message)
   return chosen_message
 
 async def translate_text(api_key, language, text: str, callbacks: list) -> str:
   if language == "it":
-      __print_output(text)
-      return
+      return __print_output(text)
   llm = ChatOpenAI(api_key=api_key, model="gpt-3.5-turbo-0125", streaming=True)
   sys_message = """Il tuo compito è di tradurre il testo_da_tradure nella seguente lingua: \n\n lingua: {language}\n\n testo_da_tradure: {testo_da_tradure} \n\nTraduci il testo_da_tradure nella lingua {language} senza aggiungere altro:"""
   prompt = PromptTemplate.from_template(sys_message)

ws_bom_robot_app/llm/utils/print.py

@@ -14,16 +14,16 @@ class HiddenPrints:
         sys.stdout = self._original_stdout
         sys.stderr = self._original_stderr
 
-def printJson(data) -> None:
-    print(json.dumps(data, indent=2, sort_keys=True), end=",", flush=True)
+def printJson(data) -> str:
+    return f"{json.dumps(data, indent=2, sort_keys=True)},"
 
 def printSingleJson(data) -> str:
-    print(json.dumps(data, indent=2, sort_keys=True), end="", flush=True)
+    return f"{json.dumps(data, indent=2, sort_keys=True)}"
 
-def printString(data: str) -> None:
+def printString(data: str) -> str:
     if data != "":
-        printJson(data)
+        return printJson(data)
 
-def printSingleString(data: str) -> None:
+def printSingleString(data: str) -> str:
     if data != "":
-        printSingleJson(data)
+        return printSingleJson(data)

ws_bom_robot_app/llm/vector_store/generator.py

@@ -1,4 +1,4 @@
-import os, gc, shutil, traceback
+import os, gc, shutil, logging, traceback
 import asyncio, aiofiles, aiofiles.os
 from fastapi import HTTPException
 from fastapi.responses import StreamingResponse
@@ -67,7 +67,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
         documents.extend(await loaders.load())
       except Exception as e:
         tb = traceback.format_exc()
-        return VectorDbResponse(success = False, error = f"File loader failure: {e} | {tb}")
+        _error = f"File loader failure: {e} | {tb}"
+        logging.warning(_error)
+        return VectorDbResponse(success = False, error = _error)
     except Exception as e:
       await _cleanup_directory(working_path)
       return VectorDbResponse(success = False, error = f"Failed to download file {e}")
@@ -78,7 +80,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
     except Exception as e:
       await _cleanup_directory(working_path)
       tb = traceback.format_exc()
-      return VectorDbResponse(success = False, error = f"Endpoint failure: {e} | {tb}")
+      _error = f"Endpoint failure: {e} | {tb}"
+      logging.warning(_error)
+      return VectorDbResponse(success = False, error = _error)
 
   if rq.integrations:
     tasks = []
@@ -95,7 +99,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
     except Exception as e:
       await _cleanup_directory(working_path)
       tb = traceback.format_exc()
-      return VectorDbResponse(success=False, error=f"Integration failure: {e} | {tb}")
+      _error = f"Integration failure: {e} | {tb}"
+      logging.warning(_error)
+      return VectorDbResponse(success=False, error=_error)
 
   #cleanup
   await _cleanup_directory(working_path)
@@ -116,7 +122,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
       del documents
       gc.collect()
   else:
-    return VectorDbResponse(success = False, error = "No documents found in the knowledgebase folder")
+    _error = "No documents found in the knowledgebase folder"
+    logging.warning(_error)
+    return VectorDbResponse(success = False, error = _error)
 
 async def kb_stream_file(filename: str):
     file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)

ws_bom_robot_app/llm/vector_store/integration/dropbox.py

@@ -12,7 +12,7 @@ class DropboxParams(BaseModel):
   Attributes:
     remote_url (str): The URL of the remote Dropbox location, e.g. 'dropbox://demo-directory' or 'dropbox://demo-directory/sub-directory'.
     token (str): The authentication token for accessing Dropbox.
-      create app: https://www.dropbox.com/developers, with file.content.read permission, and generate token.
+      create app: https://www.dropbox.com/developers, with file.content.read permission, and generate token, or use existing app: https://www.dropbox.com/account/connected_apps / https://www.dropbox.com/developers/apps?_tk=pilot_lp&_ad=topbar4&_camp=myapps
     recursive (bool, optional): A flag indicating whether to search directories recursively. Defaults to False.
     extension (list[str], optional): A list of file extensions to filter by. Defaults to None, e.g. ['.pdf', '.docx'].
   """

ws_bom_robot_app/llm/vector_store/integration/manager.py

@@ -9,6 +9,7 @@ from ws_bom_robot_app.llm.vector_store.integration.googledrive import GoogleDriv
 from ws_bom_robot_app.llm.vector_store.integration.jira import Jira
 from ws_bom_robot_app.llm.vector_store.integration.s3 import S3
 from ws_bom_robot_app.llm.vector_store.integration.sftp import Sftp
+from ws_bom_robot_app.llm.vector_store.integration.sharepoint import Sharepoint
 from ws_bom_robot_app.llm.vector_store.integration.sitemap import Sitemap
 from ws_bom_robot_app.llm.vector_store.integration.slack import Slack
 
@@ -23,6 +24,7 @@ class IntegrationManager:
     "llmkbjira": Jira,
     "llmkbs3": S3,
     "llmkbsftp": Sftp,
+    "llmkbsharepoint": Sharepoint,
     "llmkbsitemap": Sitemap,
     "llmkbslack": Slack,
 

ws_bom_robot_app/llm/vector_store/integration/sharepoint.py

@@ -0,0 +1,106 @@
+import asyncio, logging, traceback
+from dataclasses import dataclass
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.v2.processes.connectors.sharepoint  import SharepointIndexerConfig, SharepointIndexer, SharepointDownloaderConfig, SharepointConnectionConfig, SharepointAccessConfig
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import Union, Optional
+from pydantic import BaseModel, Field, AliasChoices
+
+class SharepointParams(BaseModel):
+  """
+  SharepointParams is a Pydantic model that defines the parameters required to connect to a SharePoint site.
+
+  Attributes:
+    client_id (str): The client ID for SharePoint authentication.
+    client_secret (str): The client secret for SharePoint authentication.
+    site_url (str): The URL of the SharePoint site. i.e. site collection level: https://<tenant>.sharepoint.com/sites/<site-collection-name>, or root site: https://<tenant>.sharepoint.com
+    site_path (str, optional): TThe path in the SharePoint site from which to start parsing files, for example "Shared Documents". Defaults to None.
+    recursive (bool, optional): Whether to recursively access subdirectories. Defaults to False.
+    omit_files (bool, optional): Whether to omit files from the results. Defaults to False.
+    omit_pages (bool, optional): Whether to omit pages from the results. Defaults to False.
+    omit_lists (bool, optional): Whether to omit lists from the results. Defaults to False.
+    extension (list[str], optional): A list of file extensions to include, i.e. [".pdf"]  Defaults to None.
+  """
+  client_id : str = Field(validation_alias=AliasChoices("clientId","client_id"))
+  client_secret : str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
+  site_url: str = Field(validation_alias=AliasChoices("siteUrl","site_url"))
+  site_path: str = Field(default=None,validation_alias=AliasChoices("sitePath","site_path"))
+  recursive: bool = Field(default=False)
+  omit_files: bool = Field(default=False, validation_alias=AliasChoices("omitFiles","omit_files")),
+  omit_pages: bool = Field(default=False, validation_alias=AliasChoices("omitPages","omit_pages")),
+  omit_lists: bool = Field(default=False, validation_alias=AliasChoices("omitLists","omit_lists")),
+  extension: list[str] = Field(default=None)
+class Sharepoint(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = SharepointParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'sharepoint'
+  def run(self) -> None:
+    indexer_config = SharepointIndexerConfig(
+      path=self.__data.site_path,
+      recursive=self.__data.recursive,
+      omit_files=self.__data.omit_files,
+      omit_pages=self.__data.omit_pages,
+      omit_lists=self.__data.omit_lists
+    )
+    downloader_config = SharepointDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = SharepointConnectionConfig(
+      access_config=SharepointAccessConfig(client_cred=self.__data.client_secret),
+      client_id=self.__data.client_id,
+      site=self.__data.site_url,
+      permissions_config=None
+    )
+    pipeline = self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.extension)
+    current_indexer_process = pipeline.indexer_step.process
+    pipeline.indexer_step.process = CustomSharepointIndexer(**vars(current_indexer_process))
+    pipeline.run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()
+
+@dataclass
+class CustomSharepointIndexer(SharepointIndexer):
+  def __init__(self, **kwargs):
+      # Initialize all attributes from the base indexer
+      for key, value in kwargs.items():
+          setattr(self, key, value)
+  def list_files(self, folder, recursive):
+      try:
+        _files = super().list_files(folder, recursive)
+        return _files
+      except Exception as e:
+        tb = traceback.format_exc()
+        logging.error(f"Error listing sharepoint files: {e} \n {tb}")
+        return []
+  def file_to_file_data(self, client, file):
+    try:
+      return super().file_to_file_data(client, file)
+    except Exception as e:
+      tb = traceback.format_exc()
+      logging.error(f"Error converting sharepoint file {file} to data: {e} \n {tb}")
+      return None
+  def list_pages(self, client):
+    try:
+      _pages = super().list_pages(client)
+      _allowed_content_type = None
+      for page in _pages:
+        # determine the allowed content type from the first page (Home.aspx)
+        if not _allowed_content_type:
+          _allowed_content_type = page.content_type_id
+        if not page.content_type_id == _allowed_content_type:
+          _pages.remove_child(page)
+      return _pages
+    except Exception as e:
+      tb = traceback.format_exc()
+      logging.error(f"Error listing sharepoint pages: {e} \n {tb}")
+      return []

ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -1,14 +1,14 @@
-
 import asyncio, gc, logging, os, traceback
 from typing import Any, Optional
 from langchain_community.document_loaders import DirectoryLoader
 from langchain_community.document_loaders.base import BaseLoader
 from langchain_community.document_loaders.merge import MergedDataLoader
 from langchain_core.documents import Document
-from langchain_unstructured import UnstructuredLoader
 from pydantic import BaseModel
 from ws_bom_robot_app.config import config
 from ws_bom_robot_app.llm.vector_store.loader.json_loader import JsonLoader
+from ws_bom_robot_app.llm.vector_store.loader.docling import DoclingLoader
+from langchain_community.document_loaders import CSVLoader, UnstructuredPowerPointLoader, UnstructuredEmailLoader, UnstructuredXMLLoader, TextLoader, UnstructuredHTMLLoader
 
 class LoaderConfig(BaseModel):
   loader: type[BaseLoader]
@@ -22,39 +22,46 @@ class Loader():
 
   _list: dict[str, LoaderConfig | None] = {
     '.json': LoaderConfig(loader=JsonLoader),
-    '.csv': LoaderConfig(loader=UnstructuredLoader),
-    '.xls': LoaderConfig(loader=UnstructuredLoader),
-    '.xlsx': LoaderConfig(loader=UnstructuredLoader),
-    '.eml': LoaderConfig(loader=UnstructuredLoader),
-    '.msg': LoaderConfig(loader=UnstructuredLoader),
+    '.csv': LoaderConfig(loader=CSVLoader),
+    '.xls': None,
+    '.xlsx': LoaderConfig(loader=DoclingLoader),
+    '.eml': LoaderConfig(loader=UnstructuredEmailLoader,kwargs={"strategy":"auto", "process_attachments": False}),
+    '.msg': LoaderConfig(loader=UnstructuredEmailLoader,kwargs={"strategy":"auto", "process_attachments": False}),
     '.epub': None,
-    '.md': LoaderConfig(loader=UnstructuredLoader),
+    '.md': LoaderConfig(loader=TextLoader),
     '.org': None,
     '.odt': None,
     '.ppt': None,
-    '.pptx': LoaderConfig(loader=UnstructuredLoader),
-    '.txt': LoaderConfig(loader=UnstructuredLoader),
+    '.pptx': LoaderConfig(loader=UnstructuredPowerPointLoader,kwargs={"strategy":"auto"}), #docling issue with WMF https://github.com/DS4SD/docling/issues/594
+    '.txt': LoaderConfig(loader=TextLoader),
     '.rst': None,
     '.rtf': None,
     '.tsv': None,
     '.text': None,
     '.log': None,
-    '.htm': LoaderConfig(loader=UnstructuredLoader),
-    '.html': LoaderConfig(loader=UnstructuredLoader),
-    '.pdf': LoaderConfig(loader=UnstructuredLoader,kwargs={
-       'strategy':'ocr_only', #https://docs.unstructured.io/open-source/core-functionality/partitioning auto,ocr_only,hi_res
-       'split_pdf_page': False,
-       'chunking_strategy': 'basic',
-       'max_characters': 1000000,
-       'include_page_breaks': True,
-       'include_orig_elements': False}),
-    '.png': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
-    '.jpg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
-    '.jpeg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
+    '.htm': LoaderConfig(loader=UnstructuredHTMLLoader,kwargs={"strategy":"auto"}),
+    '.html': LoaderConfig(loader=UnstructuredHTMLLoader,kwargs={"strategy":"auto"}),
+    ".pdf": LoaderConfig(loader=DoclingLoader),
+    #'.pdf': LoaderConfig(loader=UnstructuredLoader,kwargs={
+    #  'strategy':'ocr_only', #https://docs.unstructured.io/open-source/core-functionality/partitioning auto,ocr_only,hi_res
+    #   'split_pdf_page': False,
+    #   'chunking_strategy': 'basic',
+    #   'max_characters': 10_000,
+    #   'include_page_breaks': True,
+    #   'include_orig_elements': False}),
+    #'.png': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
+    #'.jpg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
+    #'.jpeg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
+    '.png': LoaderConfig(loader=DoclingLoader),
+    '.jpg': LoaderConfig(loader=DoclingLoader),
+    '.jpeg': LoaderConfig(loader=DoclingLoader),
+    '.gif': None,
+    ".emf": None,
+    ".wmf": None,
     '.tiff': None,
     '.doc': None, #see liberoffice dependency
-    '.docx': LoaderConfig(loader=UnstructuredLoader),
-    '.xml': LoaderConfig(loader=UnstructuredLoader),
+    '.docx': LoaderConfig(loader=DoclingLoader),
+    '.xml': LoaderConfig(loader=UnstructuredXMLLoader,kwargs={"strategy":"auto"}),
     '.js': None,
     '.py': None,
     '.c': None,
@@ -78,7 +85,7 @@ class Loader():
     loader_configs = {}
     for ext, loader_config in Loader._list.items():
         if loader_config:
-            if all([self._runtime_options.loader_strategy != "",loader_config.kwargs,"strategy" in loader_config.kwargs]): # type: ignore
+            if all([self._runtime_options.loader_strategy != "",loader_config.kwargs and "strategy" in loader_config.kwargs]): # type: ignore
                 loader_config.kwargs["strategy"] = self._runtime_options.loader_strategy # type: ignore
             loader_key = (loader_config.loader, tuple(loader_config.kwargs.items())) # type: ignore
             if loader_key not in loader_configs:
@@ -119,7 +126,7 @@ class Loader():
           return _documents
         except Exception as e:
           logging.warning(f"Attempt {attempt+1} load document  failed: {e}")
-          await asyncio.sleep(1)
+          await asyncio.sleep(2)
           if attempt == MAX_RETRIES - 1:
             tb = traceback.format_exc()
             logging.error(f"Failed to load documents: {e} | {tb}")

ws_bom_robot_app/llm/vector_store/loader/docling.py

@@ -0,0 +1,35 @@
+import os, logging, traceback
+from typing import Iterator, AsyncIterator, Optional
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+from langchain_core.runnables import run_in_executor
+from docling.document_converter import DocumentConverter, ConversionResult, ConversionStatus
+
+class DoclingLoader(BaseLoader):
+  def __init__(self, file_path: str | list[str]) -> None:
+      self._file_paths = file_path if isinstance(file_path, list) else [file_path]
+      self._converter = DocumentConverter()
+  def load(self) -> list[Document]:
+      """Load data into Document objects."""
+      return list(self.lazy_load())
+  async def aload(self) -> list[Document]:
+      """Load data into Document objects."""
+      return [document async for document in self.alazy_load()]
+  async def alazy_load(self) -> AsyncIterator[Document]:
+      """A lazy loader for Documents."""
+      iterator = await run_in_executor(None, self.lazy_load)
+      done = object()
+      while True:
+          doc = await run_in_executor(None, next, iterator, done)  # type: ignore[call-arg, arg-type]
+          if doc is done:
+              break
+          yield doc  # type: ignore[misc]
+  def lazy_load(self) -> Iterator[Document]:
+      for source in self._file_paths:
+          try:
+            _result = self._converter.convert(os.path.abspath(source),raises_on_error=True)
+            doc = _result.document
+            text = doc.export_to_markdown()
+            yield Document(page_content=text, metadata={"source": source})
+          except Exception as e:
+            logging.warning(f"Failed to load document from {source}: {e} | {traceback.format_exc()}")

{ws_bom_robot_app-0.0.23.dist-info → ws_bom_robot_app-0.0.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.23
+Version: 0.0.25
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -21,21 +21,20 @@ Requires-Dist: langchain-openai==0.2.10
 Requires-Dist: langchain-community==0.3.8
 Requires-Dist: langchain-core==0.3.21
 Requires-Dist: faiss-cpu==1.9.0
-Requires-Dist: python-magic==0.4.27
-Requires-Dist: opencv-python-headless==4.10.0.84
-Requires-Dist: unstructured[all-docs]==0.16.11
-Requires-Dist: langchain_unstructured==0.1.5
+Requires-Dist: unstructured==0.16.11
+Requires-Dist: unstructured[image]
 Requires-Dist: unstructured-ingest==0.3.8
 Requires-Dist: unstructured-ingest[azure]
 Requires-Dist: unstructured-ingest[confluence]
 Requires-Dist: unstructured-ingest[dropbox]
 Requires-Dist: unstructured-ingest[gcs]
-Requires-Dist: unstructured-ingest[google_drive]
 Requires-Dist: unstructured-ingest[github]
+Requires-Dist: unstructured-ingest[google_drive]
 Requires-Dist: unstructured-ingest[jira]
 Requires-Dist: unstructured-ingest[s3]
-Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: unstructured-ingest[sftp]
+Requires-Dist: unstructured-ingest[sharepoint]
+Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
 Requires-Dist: nebuly==0.3.33
@@ -214,11 +213,22 @@ launch debugger
 streamlit run debugger.py --server.port 6002
 ```
 
+dockerize base image
+
+```pwsh
+<# cpu #>
+docker build -f Dockerfile-robot-base-cpu -t ghcr.io/websolutespa/ws-bom-robot-base:cpu .
+docker push ghcr.io/websolutespa/ws-bom-robot-base:cpu
+<# gpu #>
+docker build -f Dockerfile-robot-base-gpu -t ghcr.io/websolutespa/ws-bom-robot-base:gpu .
+docker push ghcr.io/websolutespa/ws-bom-robot-base:gpu
+```
+
 dockerize app from src
 
 ```pwsh
 docker build -f Dockerfile-src -t ws-bom-robot-app:src .
-docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -p 6001:6001 ws-bom-robot-app:src
+docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app:src
 ```
 
 ### ✈️ publish

{ws_bom_robot_app-0.0.23.dist-info → ws_bom_robot_app-0.0.25.dist-info}/RECORD

@@ -7,20 +7,20 @@ ws_bom_robot_app/task_manager.py,sha256=Zedzs2R3O-wNSQOqs4jorgFwPRi-ji_0TN4mGfk-
 ws_bom_robot_app/util.py,sha256=b49ItlZgh2Wzw-6K8k5Wa44eVgjQ0JmWQwJnEaQBVGw,3502
 ws_bom_robot_app/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/agent_description.py,sha256=SDJYMmwfdMxEK3a_HDEQ19bfNKmwMSFf5hqU0VSCCIE,4705
-ws_bom_robot_app/llm/agent_handler.py,sha256=hbOf9i-ynDL3bcClqtUG-yWY8ohbCxONfT5ek9Cv0gY,5667
+ws_bom_robot_app/llm/agent_handler.py,sha256=Qz3h1eZdA6pkurEbr8sQwl-0FdjugaO5Q9sB8f7iD9I,5808
 ws_bom_robot_app/llm/agent_lcel.py,sha256=jkSLMy6y_ZFvWT8bhBBYHY5CO-ea8oMSPMXMahFUBFc,2666
-ws_bom_robot_app/llm/api.py,sha256=5cO49yhU5EXvl20zJORmrZZPc1G_nlvftdcF7cyn4Qc,3252
+ws_bom_robot_app/llm/api.py,sha256=vBu_TFTlBjp7e3J-WmlZbXn_TbB550x-NpQN4YsO7To,3004
 ws_bom_robot_app/llm/defaut_prompt.py,sha256=pn5a4lNLWE1NngHYjA_7tD8GasePMgsgude5fIJxsW0,756
-ws_bom_robot_app/llm/main.py,sha256=oD8dPvoEYD2MK8dU8tjdqBmPxxBcILSSPkQzNlMGayk,3712
+ws_bom_robot_app/llm/main.py,sha256=_uW3Iy9iPJbxDfpyoReu3mbYY8a9dS1V6tZU-z6BELo,3547
 ws_bom_robot_app/llm/settings.py,sha256=EkFGCppORenStH9W4e6_dYvQ-5p6xiEMpmUHBqNqG9M,117
 ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/models/api.py,sha256=KlVUbApyz6uuWefAN9K4B_vWDSps5hLW6hNg1Eo3TBQ,6996
 ws_bom_robot_app/llm/models/base.py,sha256=1TqxuTK3rjJEALn7lvgoen_1ba3R2brAgGx6EDTtDZo,152
 ws_bom_robot_app/llm/models/kb.py,sha256=9zqwDlVULVrWE48wo5AivzWoOtnjA57k9rsw8KNnyDk,8935
 ws_bom_robot_app/llm/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/tools/tool_builder.py,sha256=rkYu0PrXV84PMi7INjCSWlrWMykUCI8aeF-QjZgLysM,854
-ws_bom_robot_app/llm/tools/tool_manager.py,sha256=ZRUzSIrHKrJANc7hrp7st8qRZiy_S-_aN8fIgk7g0U8,4353
-ws_bom_robot_app/llm/tools/utils.py,sha256=1uOJGcFKiIDOipLCokEMdlxJLRQpgpOavm1g5-FLjMU,1307
+ws_bom_robot_app/llm/tools/tool_builder.py,sha256=z9SdwD6dJaJbLByHIGUaIbqbNm33an9agNnm5njSb6Q,901
+ws_bom_robot_app/llm/tools/tool_manager.py,sha256=DzJLQCLBb2jesOx2rR56_z3KyWqwJpvUGD16ImxOj34,4495
+ws_bom_robot_app/llm/tools/utils.py,sha256=yT8dJ2pywCJb-6VlgltVPEa4-b3XT8UYWUqW9m1cKWo,1307
 ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/tools/models/main.py,sha256=LsOJ7vkcSzYLoE1oa3TG0Rs0pr9J5VS_e4li6aDx_fw,260
 ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -28,28 +28,30 @@ ws_bom_robot_app/llm/utils/agent_utils.py,sha256=LEfAKQwFrwmIdJL0o54iuGrir9uLcJh
 ws_bom_robot_app/llm/utils/download.py,sha256=iAUxH_NiCpTPtGzhC4hBtxotd2HPFt2MBhttslIxqiI,3194
 ws_bom_robot_app/llm/utils/faiss_helper.py,sha256=69juxptz1gidgxVOrqNvJajRl40p5-ugHqyEBDtnSKo,5036
 ws_bom_robot_app/llm/utils/kb.py,sha256=jja45WCbNI7SGEgqDS99nErlwB5eY8Ga7BMnhdMHZ90,1279
-ws_bom_robot_app/llm/utils/print.py,sha256=bpLWY0KHXe7x7PWcWy8NS54ZWzHY8b4jrLRkpnDl108,818
+ws_bom_robot_app/llm/utils/print.py,sha256=ZonoLPcfM6Cpw4_Ec455LiCovExOwvnIgvw1QORSCBY,799
 ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
 ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/generator.py,sha256=aVUG08rcsDvtVxfYfUalO1CbKGkLazNyP555IsQQZso,5975
+ws_bom_robot_app/llm/vector_store/generator.py,sha256=Xg-srcH_03lqPHkMn1EXP56GbY1CYa2zIbjvNfcQqyM,6192
 ws_bom_robot_app/llm/vector_store/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/integration/azure.py,sha256=R37TaPQP-HJJJiaKE9rmMc9kpeXeRvdebbTY_982om0,3392
 ws_bom_robot_app/llm/vector_store/integration/base.py,sha256=IvIu8RkISuurrVKr2YPG96fsOI2kqhaEGyTGzjB-jCI,1550
 ws_bom_robot_app/llm/vector_store/integration/confluence.py,sha256=4fiRHB3J-SHZZxNGHwVkCrT-xSPbc91z4WrDE9fy6xU,2505
-ws_bom_robot_app/llm/vector_store/integration/dropbox.py,sha256=Am0jfh54OFUTQihqmkwgKIkcEmjehDh_0NQzBuxTdGY,2481
+ws_bom_robot_app/llm/vector_store/integration/dropbox.py,sha256=yhGvHTN0TEpUfhdvvV7RX5MxBwTUyddAX95Fgqp3mCg,2629
 ws_bom_robot_app/llm/vector_store/integration/gcs.py,sha256=fFDVDUR6eNB7FVTzDSEpMHFEWMgG16GLnpSf_mqGDdE,3184
 ws_bom_robot_app/llm/vector_store/integration/github.py,sha256=18PO30AZcgTn6PHhid3MwImVAdmKBNkr0kmAPgOetGw,2663
 ws_bom_robot_app/llm/vector_store/integration/googledrive.py,sha256=R6hr8iEgrR3QMOzIj5jY6w1x8pZ1LGdh4xM_q7g_ttc,3738
 ws_bom_robot_app/llm/vector_store/integration/jira.py,sha256=o5iINIblp_yNszp54nf7fW97aqjs0A5G89N8sYrd1ds,2771
-ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=_aNiy3w_fnWOwaGkp_X_hDNstnPbxtCuojikBzRSArE,1583
+ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=5Fl3XML6f1wmgraigpUwIFIXh7QFPX0RI0YFgFxBAvg,1700
 ws_bom_robot_app/llm/vector_store/integration/s3.py,sha256=3kh-VmH84IW7DdSLvOk6td1VBJ9aohlVJsk5F3cYj0U,3320
 ws_bom_robot_app/llm/vector_store/integration/sftp.py,sha256=WNzjjS1EUykgFB-8e7QkecSa1r1jTJqKyGzR25uJCtM,2848
+ws_bom_robot_app/llm/vector_store/integration/sharepoint.py,sha256=zqqn-6qPrK50Phch4nZHJTgaPyPkGe7W2InGL_Ru6vE,5376
 ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=nPbIywp-ZwWbWStvjvYVgHqqejyYFr8eZhBc8ycTuaU,4206
 ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=FMjESXm2QetFXI6i8epze7Kbbu22fV8CVaxb71AHnJ8,2572
 ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=ZvcyUPOoQ44gyfl80Jf0y9vbcj8uTUb-lYkb_m2pt1I,5137
+ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=SWV7T6BcsV8nvnUAHHZ9Q2oFUEnfwM33jpJCry5vbIA,5847
+ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=aHHfMf2JsZo0o6jrRDlImY0Oi9NFhVQk8Wg5ePAPa50,1721
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
-ws_bom_robot_app-0.0.23.dist-info/METADATA,sha256=tWzTgX6aPIvqV-chg-GyxPQCogt14pzEPKM3DJLR0yw,7152
-ws_bom_robot_app-0.0.23.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ws_bom_robot_app-0.0.23.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.23.dist-info/RECORD,,
+ws_bom_robot_app-0.0.25.dist-info/METADATA,sha256=TObdL0LhroQrJaqOUTwLEY9gqyk_ct-yDPJzcOWps_w,7478
+ws_bom_robot_app-0.0.25.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ws_bom_robot_app-0.0.25.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.25.dist-info/RECORD,,