ws-bom-robot-app 0.0.22__tar.gz → 0.0.24__tar.gz

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.22
+Version: 0.0.24
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -26,9 +26,16 @@ Requires-Dist: opencv-python-headless==4.10.0.84
 Requires-Dist: unstructured[all-docs]==0.16.11
 Requires-Dist: langchain_unstructured==0.1.5
 Requires-Dist: unstructured-ingest==0.3.8
+Requires-Dist: unstructured-ingest[azure]
 Requires-Dist: unstructured-ingest[confluence]
+Requires-Dist: unstructured-ingest[dropbox]
+Requires-Dist: unstructured-ingest[gcs]
 Requires-Dist: unstructured-ingest[github]
+Requires-Dist: unstructured-ingest[google_drive]
 Requires-Dist: unstructured-ingest[jira]
+Requires-Dist: unstructured-ingest[s3]
+Requires-Dist: unstructured-ingest[sftp]
+Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
 Requires-Dist: nebuly==0.3.33

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.22",
+    version="0.0.24",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/agent_handler.py

@@ -1,3 +1,4 @@
+from asyncio import Queue
 from langchain_core.agents import AgentFinish
 from langchain_core.outputs import ChatGenerationChunk, GenerationChunk
 from langchain.callbacks.base import AsyncCallbackHandler
@@ -16,13 +17,13 @@ import json
 
 class AgentHandler(AsyncCallbackHandler):
 
-    def __init__(self, threadId) -> None:
+    def __init__(self, queue: Queue, threadId: str = None) -> None:
         super().__init__()
         self._threadId = threadId
-        self.json_cards = None
         self.json_block = ""
         self.is_json_block = False
         self.backtick_count = 0  # Conteggio dei backticks per il controllo accurato
+        self.queue = queue
 
     async def on_llm_start(
         self,
@@ -39,7 +40,7 @@ class AgentHandler(AsyncCallbackHandler):
             "type": "info",
             "threadId": self._threadId,
         }
-        printJson(firstChunk)
+        await self.queue.put(printString(firstChunk))
 
     """async def on_chat_model_start(self, serialized: Dict[str, Any], messages: List[List[BaseMessage]], *, run_id: UUID = None, parent_run_id = None, tags = None, metadata = None, **kwargs: Any) -> Any:
         pass"""
@@ -75,7 +76,7 @@ class AgentHandler(AsyncCallbackHandler):
             elif self.is_json_block:
                 self.json_block += token
             else:
-                printString(token)
+                await self.queue.put(printString(token))
         pass
 
     async def on_agent_finish(
@@ -92,12 +93,9 @@ class AgentHandler(AsyncCallbackHandler):
                 AIMessage(content=finish.return_values["output"]),
             ]
         )
-        if self.json_cards:
-          for card in self.json_cards:
-            printJson(card)
-          self.json_cards = None
         finalChunk = {"type": "end"}
-        printJson(finalChunk)
+        await self.queue.put(printJson(finalChunk))
+        await self.queue.put(None)
 
     async def process_json_block(self, json_block: str):
       """Processa il blocco JSON completo."""
@@ -108,15 +106,16 @@ class AgentHandler(AsyncCallbackHandler):
           try:
               # Prova a fare il parsing del JSON
               parsed_json = json.loads(json_block_clean)
-              printJson(parsed_json)
+              await self.queue.put(printJson(parsed_json))
           except json.JSONDecodeError as e:
               # Se il JSON è malformato, logga l'errore
               raise e
 
 class RawAgentHandler(AsyncCallbackHandler):
 
-    def __init__(self) -> None:
+    def __init__(self,queue: Queue) -> None:
         super().__init__()
+        self.queue = queue
 
     async def on_llm_start(
         self,
@@ -147,10 +146,9 @@ class RawAgentHandler(AsyncCallbackHandler):
         tags: Optional[List[str]] = None,
         **kwargs: Any,
     ) -> None:
-        """Gestisce i nuovi token durante lo streaming."""
-        if token != "":
-            print(token)
-        pass
+        """Handles new tokens during streaming."""
+        if token:  # Only process non-empty tokens
+            await self.queue.put(token)
 
     async def on_agent_finish(
         self,
@@ -166,3 +164,4 @@ class RawAgentHandler(AsyncCallbackHandler):
                 AIMessage(content=finish.return_values["output"]),
             ]
         )
+        await self.queue.put(None)

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/api.py

@@ -25,18 +25,10 @@ async def _invoke(rq: InvokeRequest):
 async def _stream(rq: StreamRequest) -> StreamingResponse:
     return StreamingResponse(stream(rq), media_type="application/json")
 
-@router.post("/stream/none")
-async def _stream_none(rq: StreamRequest) -> None:
-    await stream_none(rq)
-
 @router.post("/stream/raw")
 async def _stream_raw(rq: StreamRequest) -> StreamingResponse:
     return StreamingResponse(stream(rq, formatted=False), media_type="application/json")
 
-@router.post("/stream/raw/none")
-async def _stream_raw_none(rq: StreamRequest) -> None:
-     await stream_none(rq, formatted=False)
-
 @router.post("/kb")
 async def _kb(rq: KbRequest) -> VectorDbResponse:
     return await kb(rq)

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/main.py

@@ -10,6 +10,7 @@ from nebuly.providers.langchain import LangChainTrackingHandler
 from langchain_core.callbacks.base import AsyncCallbackHandler
 import warnings, asyncio, os, io, sys, json
 from typing import List
+from asyncio import Queue
 
 async def invoke(rq: InvokeRequest) -> str:
   await rq.initialize()
@@ -22,12 +23,12 @@ async def invoke(rq: InvokeRequest) -> str:
   result: AIMessage = await processor.run_agent(_msg)
   return {"result": result.content}
 
-async def __stream(rq: StreamRequest,formatted: bool = True) -> None:
+async def __stream(rq: StreamRequest,queue: Queue,formatted: bool = True) -> None:
   await rq.initialize()
   if formatted:
-    agent_handler = AgentHandler(rq.thread_id)
+    agent_handler = AgentHandler(queue,rq.thread_id)
   else:
-    agent_handler = RawAgentHandler()
+    agent_handler = RawAgentHandler(queue)
   os.environ["AGENT_HANDLER_FORMATTED"] = str(formatted)
   callbacks: List[AsyncCallbackHandler] = [agent_handler]
   settings.init()
@@ -53,7 +54,7 @@ async def __stream(rq: StreamRequest,formatted: bool = True) -> None:
   processor = AgentLcel(
       openai_config={"api_key": rq.secrets["openAIApiKey"], "openai_model": rq.model, "temperature": rq.temperature},
       sys_message=rq.system_message,
-      tools=get_structured_tools(tools=rq.app_tools, api_key=rq.secrets["openAIApiKey"], callbacks=[callbacks[0]]),
+      tools=get_structured_tools(tools=rq.app_tools, api_key=rq.secrets["openAIApiKey"], callbacks=[callbacks[0]], queue=queue),
       rules=rq.rules
   )
 
@@ -71,25 +72,20 @@ async def __stream(rq: StreamRequest,formatted: bool = True) -> None:
         {"callbacks": callbacks},
     )
 
+  # Signal the end of streaming
+  await queue.put(None)
+
 async def stream(rq: StreamRequest,formatted:bool = True) -> AsyncGenerator[str, None]:
-    sys_stdout = io.StringIO()
-    original_stdout = sys.stdout
-    sys.stdout = sys_stdout
+    queue = Queue()
+    task = asyncio.create_task(__stream(rq, queue, formatted))
     try:
-        task = asyncio.create_task(__stream(rq,formatted))
-        while not task.done():
-            await asyncio.sleep(0.1)  # yield control to avoid blocking
-            output = sys_stdout.getvalue()
-            if output:
-                yield output
-                sys_stdout.truncate(0)
-                sys_stdout.seek(0)
-        # capture any remaining output after the task completes
-        output = sys_stdout.getvalue()
-        if output:
-            yield output
+        while True:
+            token = await queue.get()
+            if token is None:  # None indicates the end of streaming
+                break
+            yield token
     finally:
-        sys.stdout = original_stdout
+        await task
 
 async def stream_none(rq: StreamRequest, formatted: bool = True) -> None:
   await __stream(rq, formatted)

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/tools/tool_builder.py

@@ -1,12 +1,13 @@
+from asyncio import Queue
 from langchain.tools import StructuredTool
 from ws_bom_robot_app.llm.models.api import LlmAppTool
 from ws_bom_robot_app.llm.tools.tool_manager import ToolManager
 
-def get_structured_tools(tools: list[LlmAppTool], api_key:str, callbacks:list) -> list[StructuredTool]:
+def get_structured_tools(tools: list[LlmAppTool], api_key:str, callbacks:list, queue: Queue) -> list[StructuredTool]:
   _structured_tools :list[StructuredTool] = []
   for tool in [tool for tool in tools if tool.is_active]:
     if _tool_config := ToolManager._list.get(tool.function_name):
-      _tool_instance = ToolManager(tool, api_key, callbacks)
+      _tool_instance = ToolManager(tool, api_key, callbacks, queue)
       _structured_tool = StructuredTool.from_function(
         coroutine=_tool_instance.get_coroutine(),
         name=tool.function_id,

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/tools/tool_manager.py

@@ -1,3 +1,4 @@
+from asyncio import Queue
 from typing import  Optional, Type, Callable
 from ws_bom_robot_app.llm.models.api import LlmAppTool
 from ws_bom_robot_app.llm.utils.faiss_helper import FaissHelper
@@ -33,10 +34,12 @@ class ToolManager:
         app_tool: LlmAppTool,
         api_key: str,
         callbacks: list,
+        queue: Optional[Queue] = None
     ):
         self.app_tool = app_tool
         self.api_key = api_key
         self.callbacks = callbacks
+        self.queue = queue
 
 
     #region functions
@@ -64,7 +67,8 @@ class ToolManager:
                 else:
                     search_type = "mixed"
                     search_kwargs = {"k": search_settings.search_k if search_settings.search_k else 4}
-            getRandomWaitingMessage(self.app_tool.waiting_message, traduction=False)
+            if self.queue:
+              await self.queue.put(getRandomWaitingMessage(self.app_tool.waiting_message, traduction=False))
             return await FaissHelper.invoke(self.app_tool.vector_db, self.api_key, query, search_type, search_kwargs)
         return []
         #raise ValueError(f"Invalid configuration for {self.settings.name} tool of type {self.settings.type}. Must be a function or vector db not found.")

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/tools/utils.py

@@ -3,8 +3,8 @@ from langchain_openai import ChatOpenAI
 from langchain_core.prompts import PromptTemplate
 from ws_bom_robot_app.llm.utils.print import printString
 
-def __print_output(data: str) -> None:
-  printString(data) if os.environ.get("AGENT_HANDLER_FORMATTED") == str(True) else print(f"{data} ")
+def __print_output(data: str) -> str:
+  return printString(data) if os.environ.get("AGENT_HANDLER_FORMATTED") == str(True) else f"{data} "
 
 def getRandomWaitingMessage(waiting_messages: str, traduction: bool = True) -> str:
   if not waiting_messages: return ""
@@ -12,13 +12,12 @@ def getRandomWaitingMessage(waiting_messages: str, traduction: bool = True) -> s
   if not messages: return ""
   chosen_message = random.choice(messages) + "\n"
   if not traduction:
-      __print_output(chosen_message)
+      return __print_output(chosen_message)
   return chosen_message
 
 async def translate_text(api_key, language, text: str, callbacks: list) -> str:
   if language == "it":
-      __print_output(text)
-      return
+      return __print_output(text)
   llm = ChatOpenAI(api_key=api_key, model="gpt-3.5-turbo-0125", streaming=True)
   sys_message = """Il tuo compito è di tradurre il testo_da_tradure nella seguente lingua: \n\n lingua: {language}\n\n testo_da_tradure: {testo_da_tradure} \n\nTraduci il testo_da_tradure nella lingua {language} senza aggiungere altro:"""
   prompt = PromptTemplate.from_template(sys_message)

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/utils/print.py

@@ -14,16 +14,16 @@ class HiddenPrints:
         sys.stdout = self._original_stdout
         sys.stderr = self._original_stderr
 
-def printJson(data) -> None:
-    print(json.dumps(data, indent=2, sort_keys=True), end=",", flush=True)
+def printJson(data) -> str:
+    return f"{json.dumps(data, indent=2, sort_keys=True)},"
 
 def printSingleJson(data) -> str:
-    print(json.dumps(data, indent=2, sort_keys=True), end="", flush=True)
+    return f"{json.dumps(data, indent=2, sort_keys=True)}"
 
-def printString(data: str) -> None:
+def printString(data: str) -> str:
     if data != "":
-        printJson(data)
+        return printJson(data)
 
-def printSingleString(data: str) -> None:
+def printSingleString(data: str) -> str:
     if data != "":
-        printSingleJson(data)
+        return printSingleJson(data)

ws_bom_robot_app-0.0.24/ws_bom_robot_app/llm/vector_store/integration/azure.py

@@ -0,0 +1,62 @@
+import asyncio
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.v2.processes.connectors.fsspec.azure import AzureConnectionConfig, AzureAccessConfig, AzureDownloaderConfig, AzureIndexerConfig
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import Union, Optional
+from pydantic import BaseModel, Field, AliasChoices
+class AzureParams(BaseModel):
+  """
+  AzureParams is a model that holds configuration parameters for connecting to Azure services.
+
+  Attributes:
+    remote_url (str): The URL of the remote Azure service, in the form az://<container> or az://<container>/<path> for sub-folders.
+    account_name (str): The name of the Azure storage account.
+    \nProvide one of the following:
+      - account_key (Optional[str]): The key for the Azure storage account. Default is None.
+      - connection_string (Optional[str]): The connection string for the Azure storage account. Default is None.
+      - sas_token (Optional[str]): The Shared Access Signature token for the Azure storage account. Default is None. Detail: https://learn.microsoft.com/en-us/azure/ai-services/translator/document-translation/how-to-guides/create-sas-tokens?tabs=Containers
+    recursive (bool): Indicates whether the operation should be recursive. Default is False.
+    extension (list[str]): A list of file extensions to filter the files. Default is None.
+  """
+  remote_url: str = Field(validation_alias=AliasChoices("remoteUrl","remote_url"))
+  account_name: str = Field(validation_alias=AliasChoices("accountName","account_name"))
+  account_key: Optional[str] = Field(default=None,validation_alias=AliasChoices("accountKey","account_key"))
+  connection_string: Optional[str]  = Field(default=None,validation_alias=AliasChoices("connectionString","connection_string"))
+  sas_token: Optional[str]  = Field(default=None,validation_alias=AliasChoices("sasToken","sas_token"))
+  recursive: bool = False
+  extension: list[str] = Field(default=None)
+class Azure(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = AzureParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'azure'
+  def run(self) -> None:
+    indexer_config = AzureIndexerConfig(
+      remote_url=self.__data.remote_url,
+      recursive=self.__data.recursive,
+      #sample_n_files=1
+    )
+    downloader_config = AzureDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = AzureConnectionConfig(
+      access_config=AzureAccessConfig(
+        account_name=self.__data.account_name,
+        account_key=self.__data.account_key,
+        connection_string=self.__data.connection_string,
+        sas_token=self.__data.sas_token
+        )
+    )
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.extension).run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()
+

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/vector_store/integration/base.py

@@ -10,7 +10,7 @@ class IntegrationStrategy(ABC):
     self.knowledgebase_path = knowledgebase_path
     self.data = data
     self.working_directory = os.path.join(self.knowledgebase_path,self.working_subdirectory())
-    os.makedirs(self.working_directory, mode=666, exist_ok=True)
+    os.makedirs(self.working_directory, exist_ok=True)
   @property
   @abstractmethod
   def working_subdirectory(self) -> str:
@@ -23,7 +23,7 @@ class IntegrationStrategy(ABC):
 class UnstructuredIngest():
   def __init__(self, working_directory: str):
     self.working_directory = working_directory
-  def pipeline(self,indexer,downloader,connection) -> Pipeline:
+  def pipeline(self,indexer,downloader,connection,extension: list[str] = None) -> Pipeline:
     return Pipeline.from_configs(
       context=ProcessorConfig(
         reprocess=False,
@@ -38,6 +38,6 @@ class UnstructuredIngest():
       downloader_config=downloader,
       source_connection_config=connection,
       partitioner_config=PartitionerConfig(),
-      filterer_config=FiltererConfig()
+      filterer_config=FiltererConfig(file_glob=[f"**/*{ext}" for ext in extension] if extension else None)
     )
 

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/vector_store/integration/confluence.py

@@ -7,10 +7,21 @@ from typing import Union
 from pydantic import BaseModel, Field, AliasChoices
 
 class ConfluenceParams(BaseModel):
+  """
+  ConfluenceParams is a data model for storing Confluence integration parameters.
+
+  Attributes:
+    url (str): The URL of the Confluence instance, e.g., 'https://example.atlassian.net'.
+    access_token (str): The access token for authenticating with Confluence, e.g., 'AT....'
+    user_email (str): The email address of the Confluence user
+    spaces (list[str]): A list of Confluence spaces to interact with, e.g., ['SPACE1', 'SPACE2'].
+    extension (list[str], optional): A list of file extensions to filter by. Defaults to None, e.g., ['.pdf', '.docx'].
+  """
   url: str
   access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
   user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"))
   spaces: list[str] = []
+  extension: list[str] = Field(default=None)
 class Confluence(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
@@ -33,7 +44,8 @@ class Confluence(IntegrationStrategy):
     self.__unstructured_ingest.pipeline(
       indexer_config,
       downloader_config,
-      connection_config).run()
+      connection_config,
+      extension=self.__data.extension).run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)

ws_bom_robot_app-0.0.24/ws_bom_robot_app/llm/vector_store/integration/dropbox.py

@@ -0,0 +1,53 @@
+import asyncio
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import DropboxConnectionConfig, DropboxAccessConfig, DropboxDownloaderConfig, DropboxIndexerConfig
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import Union
+from pydantic import BaseModel, Field, AliasChoices
+class DropboxParams(BaseModel):
+  """
+  DropboxParams is a model for storing parameters required to interact with Dropbox.
+
+  Attributes:
+    remote_url (str): The URL of the remote Dropbox location, e.g. 'dropbox://demo-directory' or 'dropbox://demo-directory/sub-directory'.
+    token (str): The authentication token for accessing Dropbox.
+      create app: https://www.dropbox.com/developers, with file.content.read permission, and generate token.
+    recursive (bool, optional): A flag indicating whether to search directories recursively. Defaults to False.
+    extension (list[str], optional): A list of file extensions to filter by. Defaults to None, e.g. ['.pdf', '.docx'].
+  """
+  remote_url: str = Field(validation_alias=AliasChoices("remoteUrl","remote_url"))
+  token: str
+  recursive: bool = False
+  extension: list[str] = Field(default=None)
+class Dropbox(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = DropboxParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'dropbox'
+  def run(self) -> None:
+    indexer_config = DropboxIndexerConfig(
+      remote_url=self.__data.remote_url,
+      recursive=self.__data.recursive,
+      #sample_n_files=1
+    )
+    downloader_config = DropboxDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = DropboxConnectionConfig(
+      access_config=DropboxAccessConfig(
+        token=self.__data.token
+        )
+    )
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.extension).run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()
+

ws_bom_robot_app-0.0.24/ws_bom_robot_app/llm/vector_store/integration/gcs.py

@@ -0,0 +1,62 @@
+import asyncio
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.v2.processes.connectors.fsspec.gcs import GcsIndexerConfig, GcsConnectionConfig, GcsAccessConfig, GcsDownloaderConfig
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import Union, Optional
+from pydantic import BaseModel, Field, AliasChoices
+class GcsParams(BaseModel):
+  """
+  GcsParams is a model that defines the parameters required for Google Cloud Storage (GCS) integration.
+  Documentation:
+    - create service account: https://cloud.google.com/iam/docs/service-accounts-create?hl=en#console
+    - create key: https://cloud.google.com/iam/docs/keys-create-delete?hl=en#creating
+    - export key in a single line\n
+    ```pwsh
+    (Get-Content -Path "<path-to-downloaded-key-file>" -Raw).Replace("`r`n", "").Replace("`n", "")
+    ```
+    - create bucket with 'Storage Object Viewer' permission: https://cloud.google.com/storage/docs/creating-buckets?hl=en#console
+    - add principal to bucket: https://cloud.google.com/storage/docs/access-control/using-iam-permissions?hl=en#console
+    - manage IAM policies: https://cloud.google.com/storage/docs/access-control/using-iam-permissions?hl=en
+
+  Attributes:
+    remote_url (str): The URL of the remote GCS bucket, e.g. 'gcs://demo-bucket' or 'gcs://demo-bucket/sub-directory'.
+    service_account_key (str): The service account key for accessing the GCS bucket.
+    recursive (bool): A flag indicating whether to recursively access the GCS bucket. Defaults to False.
+    extension (list[str]): A list of file extensions to filter the files in the GCS bucket. Defaults to None.
+  """
+  remote_url: str = Field(validation_alias=AliasChoices("remoteUrl","remote_url"))
+  service_account_key: str = Field(validation_alias=AliasChoices("serviceAccountKey","service_account_key"))
+  recursive: bool = False
+  extension: list[str] = Field(default=None)
+class Gcs(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = GcsParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'gcs'
+  def run(self) -> None:
+    indexer_config = GcsIndexerConfig(
+      remote_url=self.__data.remote_url,
+      recursive=self.__data.recursive,
+      #sample_n_files=1
+    )
+    downloader_config = GcsDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = GcsConnectionConfig(
+      access_config=GcsAccessConfig(
+        service_account_key=self.__data.service_account_key
+        )
+    )
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.extension).run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()
+

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/vector_store/integration/github.py

@@ -10,6 +10,15 @@ from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from pydantic import BaseModel, Field, AliasChoices
 
 class GithubParams(BaseModel):
+  """
+  GithubParams is a model for storing parameters required to interact with a GitHub repository.
+
+  Attributes:
+    repo (str): The name of the GitHub repository, e.g., 'companyname/reponame'
+    access_token (Optional[str]): The access token for authenticating with GitHub, e.g., 'ghp_1234567890'.
+    branch (Optional[str]): The branch of the repository to interact with. Defaults to 'main'.
+    file_ext (Optional[list[str]]): A list of file extensions to filter by, e.g. ['.md', '.pdf']. Defaults to an empty list.
+  """
   repo: str
   access_token: Optional[str] | None = Field(None,validation_alias=AliasChoices("accessToken","access_token"))
   branch: Optional[str] = 'main'

ws_bom_robot_app-0.0.24/ws_bom_robot_app/llm/vector_store/integration/googledrive.py

@@ -0,0 +1,69 @@
+import asyncio
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.v2.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import Union
+from pydantic import BaseModel, Field, AliasChoices
+class GoogleDriveParams(BaseModel):
+  """
+  GoogleDriveParams is a model that holds parameters for Google Drive integration.
+
+  Attributes:
+    service_account_key (dict): The service account key for Google Drive API authentication \n
+      - detail: https://developers.google.com/workspace/guides/create-credentials#service-accountc \n
+      - create a service account key, download the JSON file, and pass the content of the JSON file as a dictionary \n
+      - e.g., {
+        "type": "service_account",
+        "project_id": "demo-project-123456",
+        "private_key_id": "**********",
+        "private_key": "-----BEGIN PRIVATE KEY-----...----END PRIVATE KEY-----",
+        "client_email": "demo-client@demo-project-123456.iam.gserviceaccount.com",
+        "client_id": "123456",
+        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+        "token_uri": "https://oauth2.googleapis.com/token",
+        "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+        "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/demo-client%40demo-project-123456.iam.gserviceaccount.com",
+        "universe_domain": "googleapis.com"
+      }
+      - enable Google Drive API: https://console.cloud.google.com/marketplace/product/google/drive.googleapis.com
+      - copy email address of the service account and share the Google Drive with the email address: https://www.youtube.com/watch?v=ykJQzEe_2dM&t=2s
+
+    drive_id (str): The {folder_id} of the Google Drive to interact with, e.g., https://drive.google.com/drive/folders/{folder_id}
+    extensions (list[str]): A list of file extensions to filter the files in the Google Drive, e.g., ['.pdf', '.docx'].
+    recursive (bool): A flag indicating whether to search files recursively in the Google Drive.
+  """
+  service_account_key: dict = Field(validation_alias=AliasChoices("serviceAccountKey","service_account_key"))
+  drive_id: str = Field(validation_alias=AliasChoices("driveId","drive_id"))
+  extensions: list[str] = []
+  recursive: bool = False
+class GoogleDrive(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = GoogleDriveParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'googledrive'
+  def run(self) -> None:
+    indexer_config = GoogleDriveIndexerConfig(
+      extensions=self.__data.extensions,
+      recursive=self.__data.recursive
+    )
+    downloader_config = GoogleDriveDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = GoogleDriveConnectionConfig(
+      access_config=GoogleDriveAccessConfig(
+        service_account_key=self.__data.service_account_key
+        ),
+      drive_id=self.__data.drive_id
+    )
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config).run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()
+

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/vector_store/integration/jira.py

@@ -9,6 +9,17 @@ from pydantic import BaseModel, Field, AliasChoices
 from typing import Optional, Union
 
 class JiraParams(BaseModel):
+  """
+  JiraParams is a Pydantic model that represents the parameters required to interact with a Jira instance.
+
+  Attributes:
+    url (str): The URL of the Jira instance, e.g., 'https://example.atlassian.net'.
+    access_token (str): The access token for authenticating with the Jira API.
+    user_email (str): The email address of the Jira user.
+    projects (list[str]): A list of project keys or IDs to interact with, e.g., ['SCRUM', 'PROJ1'].
+    boards (Optional[list[str]]): An optional list of board IDs to interact with. Defaults to None, e.g., ['1', '2'].
+    issues (Optional[list[str]]): An optional list of issue keys or IDs to interact with. Defaults to None, e.g., ['SCRUM-1', 'PROJ1-1'].
+  """
   url: str
   access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
   user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"))

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/vector_store/integration/manager.py

@@ -1,16 +1,31 @@
 from typing import Type
+from ws_bom_robot_app.llm.vector_store.integration.azure import Azure
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
 from ws_bom_robot_app.llm.vector_store.integration.confluence import Confluence
+from ws_bom_robot_app.llm.vector_store.integration.dropbox import Dropbox
+from ws_bom_robot_app.llm.vector_store.integration.gcs import Gcs
 from ws_bom_robot_app.llm.vector_store.integration.github import Github
+from ws_bom_robot_app.llm.vector_store.integration.googledrive import GoogleDrive
 from ws_bom_robot_app.llm.vector_store.integration.jira import Jira
+from ws_bom_robot_app.llm.vector_store.integration.s3 import S3
+from ws_bom_robot_app.llm.vector_store.integration.sftp import Sftp
 from ws_bom_robot_app.llm.vector_store.integration.sitemap import Sitemap
+from ws_bom_robot_app.llm.vector_store.integration.slack import Slack
 
 class IntegrationManager:
   _list: dict[str, Type[IntegrationStrategy]] = {
-    "llmkbsitemap": Sitemap,
+    "llmkbazure": Azure,
+    "llmkbconfluence": Confluence,
+    "llmkbdropbox": Dropbox,
     "llmkbgithub": Github,
+    "llmkbgcs": Gcs,
+    "llmkbgoogledrive": GoogleDrive,
     "llmkbjira": Jira,
-    "llmkbconfluence": Confluence,
+    "llmkbs3": S3,
+    "llmkbsftp": Sftp,
+    "llmkbsitemap": Sitemap,
+    "llmkbslack": Slack,
+
   }
   @classmethod
   def get_strategy(cls, name: str, knowledgebase_path: str, data: dict[str, str]) -> IntegrationStrategy:

ws_bom_robot_app-0.0.24/ws_bom_robot_app/llm/vector_store/integration/s3.py

@@ -0,0 +1,64 @@
+import asyncio
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.v2.processes.connectors.fsspec.s3 import S3ConnectionConfig, S3AccessConfig, S3DownloaderConfig, S3IndexerConfig
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import Union, Optional
+from pydantic import BaseModel, Field, AliasChoices
+class S3Params(BaseModel):
+  """
+  S3Params is a data model for storing parameters required to interact with an S3 bucket.
+  Documentation:
+    - ceate S3 bucket: https://docs.aws.amazon.com/AmazonS3/latest/userguide/GetStartedWithS3.html#creating-bucket
+    - enable authenticated bucket access: https://docs.aws.amazon.com/AmazonS3/latest/userguide/walkthrough1.html
+    - set policies s3:ListBucket and s3:GetObject: https://docs.aws.amazon.com/AmazonS3/latest/userguide/example-policies-s3.html
+    - generate key/secret: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey
+    - optionally create STS token: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_getsessiontoken
+
+  Attributes:
+    remote_url (str): The URL of the remote S3 bucket, e.g., 's3://demo-bucket' or 's3://demo-bucket/sub-directory'.
+    key (Optional[str]): The AWS access key ID for the authenticated AWS IAM user, e.g., 'AKIAIOSFODNN7EXAMPLE'.
+    secret (Optional[str]): The corresponding AWS secret access key, e.g., 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'.
+    token (Optional[str]):  If required, the AWS STS session token for temporary access. Default is None.
+    recursive (bool): A flag indicating whether to perform operations recursively. Default is False.
+    extension (list[str]): A list of file extensions to filter the files. Default is None. e.g., ['.pdf', '.docx'].
+  """
+  remote_url: str = Field(validation_alias=AliasChoices("remoteUrl","remote_url"))
+  key: str
+  secret: str
+  token: Optional[str]  = None
+  recursive: bool = False
+  extension: list[str] = Field(default=None)
+class S3(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = S3Params.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 's3'
+  def run(self) -> None:
+    indexer_config = S3IndexerConfig(
+      remote_url=self.__data.remote_url,
+      recursive=self.__data.recursive,
+      #sample_n_files=1
+    )
+    downloader_config = S3DownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = S3ConnectionConfig(
+      access_config=S3AccessConfig(
+        key=self.__data.key,
+        secret=self.__data.secret,
+        token=self.__data.token
+        )
+    )
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.extension).run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()
+

ws_bom_robot_app-0.0.24/ws_bom_robot_app/llm/vector_store/integration/sftp.py

@@ -0,0 +1,64 @@
+import asyncio
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.v2.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import Union, Optional
+from pydantic import BaseModel, Field, AliasChoices
+class SftpParams(BaseModel):
+  """
+  SftpParams is a model that defines the parameters required for SFTP integration.
+
+  Attributes:
+    remote_url (str): The URL of the remote SFTP server, e.g. 'sftp://example.com' or 'sftp://example.com/directory'.
+    host (Optional[str]): The hostname or IP address of the SFTP server. Defaults to None and inferred from remote_url
+    port (Optional[int]): The port number to connect to on the SFTP server. Defaults to 22.
+    username (str): The username to authenticate with the SFTP server.
+    password (str): The password to authenticate with the SFTP server.
+    recursive (bool): Whether to perform recursive operations. Defaults to False.
+    extension (list[str]): A list of file extensions to filter by. Defaults to None, e.g. ['.pdf', '.docx'].
+  """
+  remote_url: str = Field(validation_alias=AliasChoices("remoteUrl","remote_url"))
+  host: Optional[str] = None
+  port: Optional[int] = 22
+  username: str
+  password: str
+  recursive: bool = False
+  extension: list[str] = Field(default=None)
+class Sftp(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = SftpParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'sftp'
+  def run(self) -> None:
+    indexer_config = SftpIndexerConfig(
+      remote_url=self.__data.remote_url,
+      recursive=self.__data.recursive,
+      #sample_n_files=1
+    )
+    downloader_config = SftpDownloaderConfig(
+      download_dir=self.working_directory,
+      remote_url=self.__data.remote_url
+    )
+    connection_config = SftpConnectionConfig(
+      access_config=SftpAccessConfig(
+        password=self.__data.password
+        ),
+      username=self.__data.username,
+      host=self.__data.host,
+      port=self.__data.port,
+      look_for_keys=False,
+      allow_agent=False
+    )
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.extension).run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()
+

ws_bom_robot_app-0.0.24/ws_bom_robot_app/llm/vector_store/integration/sharepoint.py

@@ -0,0 +1,51 @@
+import asyncio
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.v2.processes.connectors.sharepoint  import SharepointIndexerConfig, SharepointDownloaderConfig, SharepointConnectionConfig, SharepointAccessConfig
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import Union, Optional
+from pydantic import BaseModel, Field, AliasChoices
+
+class SharepointParams(BaseModel):
+  client_id : str = Field(validation_alias=AliasChoices("clientId","client_id"))
+  client_secret : str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
+  site_url: str = Field(validation_alias=AliasChoices("siteUrl","site_url"))
+  site_path: str = Field(default=None,validation_alias=AliasChoices("sitePath","site_path"))
+  recursive: bool = Field(default=False)
+  omit_files: bool = Field(default=False, validation_alias=AliasChoices("omitFiles","omit_files")),
+  omit_pages: bool = Field(default=False, validation_alias=AliasChoices("omitPages","omit_pages")),
+  omit_lists: bool = Field(default=False, validation_alias=AliasChoices("omitLists","omit_lists")),
+  extension: list[str] = Field(default=None)
+class Sharepoint(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = SharepointParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'sharepoint'
+  def run(self) -> None:
+    indexer_config = SharepointIndexerConfig(
+      path=self.__data.site_path,
+      recursive=self.__data.recursive,
+      omit_files=self.__data.omit_files,
+      omit_pages=self.__data.omit_pages,
+      omit_lists=self.__data.omit_lists
+    )
+    downloader_config = SharepointDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = SharepointConnectionConfig(
+      access_config=SharepointAccessConfig(client_cred=self.__data.client_secret),
+      client_id=self.__data.client_id,
+      site=self.__data.site_url,
+      permissions_config=None
+    )
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.extension).run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()

ws_bom_robot_app-0.0.24/ws_bom_robot_app/llm/vector_store/integration/slack.py

@@ -0,0 +1,57 @@
+import asyncio
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.v2.processes.connectors.slack import SlackIndexerConfig, SlackDownloaderConfig, SlackConnectionConfig, SlackAccessConfig
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import Union
+from pydantic import BaseModel, Field, AliasChoices
+from datetime import datetime, timedelta
+
+class SlackParams(BaseModel):
+  """
+  SlackParams is a data model for storing Slack integration parameters.
+  Documentation:
+    - create slack app: https://api.slack.com/quickstart#creating
+    - set channels:history scope: https://api.slack.com/quickstart#scopes
+    - installing app/get token: https://api.slack.com/quickstart#installing
+    - add app to channel/s
+
+  Attributes:
+    token (str): The authentication token for accessing the Slack API.
+    channels (list[str]): A list of Slack channel IDs, e.g. ['C01B2PZQX1V'].
+    num_days (int, optional): The number of days to retrieve messages from. Defaults to 7.
+    extension (list[str], optional): A list of file extensions to filter messages by, e.g. [".xml"]. Defaults to None.
+  """
+  token: str
+  channels: list[str]
+  num_days: int = Field(default=7,validation_alias=AliasChoices("numDays","num_days"))
+  extension: list[str] = Field(default=None)
+class Slack(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = SlackParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'slack'
+  def run(self) -> None:
+    indexer_config = SlackIndexerConfig(
+      channels=self.__data.channels,
+      start_date=datetime.now() - timedelta(days=self.__data.num_days),
+      end_date=datetime.now()
+    )
+    downloader_config = SlackDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = SlackConnectionConfig(
+      access_config=SlackAccessConfig(token=self.__data.token)
+    )
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.extension).run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()
+

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -12,7 +12,10 @@ from ws_bom_robot_app.llm.vector_store.loader.json_loader import JsonLoader
 
 class LoaderConfig(BaseModel):
   loader: type[BaseLoader]
-  kwargs: Optional[dict[str, Any]] = {}
+  kwargs: Optional[dict[str, Any]] = {
+       'chunking_strategy': 'basic',
+       'max_characters': 10_000
+       }
   #post_processors: Optional[list[Callable[[str], str]]] = None
 
 class Loader():
@@ -45,7 +48,7 @@ class Loader():
        'strategy':'ocr_only', #https://docs.unstructured.io/open-source/core-functionality/partitioning auto,ocr_only,hi_res
        'split_pdf_page': False,
        'chunking_strategy': 'basic',
-       'max_characters': 1000000,
+       'max_characters': 10_000,
        'include_page_breaks': True,
        'include_orig_elements': False}),
     '.png': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/requirements.txt

@@ -23,9 +23,16 @@ opencv-python-headless==4.10.0.84 #docker specs
 unstructured[all-docs]==0.16.11
 langchain_unstructured==0.1.5
 unstructured-ingest==0.3.8
+unstructured-ingest[azure]
 unstructured-ingest[confluence]
+unstructured-ingest[dropbox]
+unstructured-ingest[gcs]
 unstructured-ingest[github]
+unstructured-ingest[google_drive]
 unstructured-ingest[jira]
+unstructured-ingest[s3]
+unstructured-ingest[sftp]
+unstructured-ingest[slack]
 html5lib==1.1 #beautifulsoup4 parser
 
 #integrations

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app/task_manager.py

@@ -170,7 +170,7 @@ class TaskManagerStrategy(ABC):
         pass
 
     def task_cleanup_rule(self, task: TaskEntry) -> bool:
-        return task.status.status in {"completed", "failure"} and datetime.fromisoformat(task.status.metadata.end_at) < datetime.now() - timedelta(days=config.robot_task_retention_days)
+        return task.status.metadata.start_at and datetime.fromisoformat(task.status.metadata.start_at) < datetime.now() - timedelta(days=config.robot_task_retention_days)
 
     def task_done_callback(self, task_entry: TaskEntry, headers: TaskHeader | None = None) -> Callable:
         def callback(task: asyncio.Task):

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.22
+Version: 0.0.24
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -26,9 +26,16 @@ Requires-Dist: opencv-python-headless==4.10.0.84
 Requires-Dist: unstructured[all-docs]==0.16.11
 Requires-Dist: langchain_unstructured==0.1.5
 Requires-Dist: unstructured-ingest==0.3.8
+Requires-Dist: unstructured-ingest[azure]
 Requires-Dist: unstructured-ingest[confluence]
+Requires-Dist: unstructured-ingest[dropbox]
+Requires-Dist: unstructured-ingest[gcs]
 Requires-Dist: unstructured-ingest[github]
+Requires-Dist: unstructured-ingest[google_drive]
 Requires-Dist: unstructured-ingest[jira]
+Requires-Dist: unstructured-ingest[s3]
+Requires-Dist: unstructured-ingest[sftp]
+Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
 Requires-Dist: nebuly==0.3.33

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app.egg-info/SOURCES.txt

@@ -42,12 +42,20 @@ ws_bom_robot_app/llm/utils/webhooks.py
 ws_bom_robot_app/llm/vector_store/__init__.py
 ws_bom_robot_app/llm/vector_store/generator.py
 ws_bom_robot_app/llm/vector_store/integration/__init__.py
+ws_bom_robot_app/llm/vector_store/integration/azure.py
 ws_bom_robot_app/llm/vector_store/integration/base.py
 ws_bom_robot_app/llm/vector_store/integration/confluence.py
+ws_bom_robot_app/llm/vector_store/integration/dropbox.py
+ws_bom_robot_app/llm/vector_store/integration/gcs.py
 ws_bom_robot_app/llm/vector_store/integration/github.py
+ws_bom_robot_app/llm/vector_store/integration/googledrive.py
 ws_bom_robot_app/llm/vector_store/integration/jira.py
 ws_bom_robot_app/llm/vector_store/integration/manager.py
+ws_bom_robot_app/llm/vector_store/integration/s3.py
+ws_bom_robot_app/llm/vector_store/integration/sftp.py
+ws_bom_robot_app/llm/vector_store/integration/sharepoint.py
 ws_bom_robot_app/llm/vector_store/integration/sitemap.py
+ws_bom_robot_app/llm/vector_store/integration/slack.py
 ws_bom_robot_app/llm/vector_store/loader/__init__.py
 ws_bom_robot_app/llm/vector_store/loader/base.py
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py

{ws_bom_robot_app-0.0.22 → ws_bom_robot_app-0.0.24}/ws_bom_robot_app.egg-info/requires.txt

@@ -14,9 +14,16 @@ opencv-python-headless==4.10.0.84
 unstructured[all-docs]==0.16.11
 langchain_unstructured==0.1.5
 unstructured-ingest==0.3.8
+unstructured-ingest[azure]
 unstructured-ingest[confluence]
+unstructured-ingest[dropbox]
+unstructured-ingest[gcs]
 unstructured-ingest[github]
+unstructured-ingest[google_drive]
 unstructured-ingest[jira]
+unstructured-ingest[s3]
+unstructured-ingest[sftp]
+unstructured-ingest[slack]
 html5lib==1.1
 markdownify==0.14.1
 nebuly==0.3.33