ws-bom-robot-app 0.0.63__py3-none-any.whl → 0.0.103__py3-none-any.whl

ws_bom_robot_app/llm/providers/llm_manager.py

@@ -1,9 +1,10 @@
-import json
 from typing import Optional
+from urllib.parse import urlparse
 from langchain_core.embeddings import Embeddings
 from langchain_core.language_models import BaseChatModel
 from pydantic import BaseModel, ConfigDict, Field
 import os
+from ws_bom_robot_app.llm.utils.download import Base64File
 
 class LlmConfig(BaseModel):
     api_url: Optional[str] = None
@@ -36,6 +37,76 @@ class LlmInterface:
     def get_parser(self):
         from langchain.agents.output_parsers.openai_tools import OpenAIToolsAgentOutputParser
         return OpenAIToolsAgentOutputParser()
+    async def _format_multimodal_image_message(self, message: dict) -> dict:
+        return {
+            "type": "image_url",
+            "image_url": {
+                "url": message.get("url")
+            }
+        }
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        _file = file or await Base64File.from_url(message.get("url"))
+        return {"type": "text", "text": f"Here's a file attachment named `{_file.name}` of type `{_file.mime_type}` in base64: `{_file.base64_content}`"}
+    async def format_multimodal_content(self, content: list) -> list:
+        _content = []
+        for message in content:
+            if isinstance(message, dict):
+                if message.get("type") == "image" and "url" in message:
+                    _content.append(await self._format_multimodal_image_message(message))
+                elif message.get("type") == "file" and "url" in message:
+                    _content.append(await self._format_multimodal_file_message(message))
+                else:
+                    # pass through text or other formats unchanged
+                    _content.append(message)
+            else:
+                _content.append(message)
+        return _content
+
+class Anthropic(LlmInterface):
+    def get_llm(self):
+        from langchain_anthropic import ChatAnthropic
+        return ChatAnthropic(
+            api_key=self.config.api_key or os.getenv("ANTHROPIC_API_KEY"),
+            model=self.config.model,
+            temperature=self.config.temperature,
+            max_tokens=8192,
+            streaming=True,
+            #betas=["files-api-2025-04-14"] #https://docs.anthropic.com/en/docs/build-with-claude/files
+        )
+
+    """
+    def get_embeddings(self):
+        from langchain_voyageai import VoyageAIEmbeddings
+        return VoyageAIEmbeddings(
+            api_key=self.config.embedding_api_key, #voyage api key
+            model="voyage-3")
+    """
+
+    def get_models(self):
+        import anthropic
+        client = anthropic.Client(api_key=self.config.api_key or os.getenv("ANTHROPIC_API_KEY"))
+        response = client.models.list()
+        return response.data
+
+    """
+    async def _format_multimodal_image_message(self, message: dict) -> dict:
+        file = await Base64File.from_url(message.get("url"))
+        return { "type": "image_url", "image_url": { "url": file.base64_url }}
+    """
+
+    #https://python.langchain.com/docs/integrations/chat/anthropic/
+    #https://python.langchain.com/docs/how_to/multimodal_inputs/
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        _url = str(message.get("url", ""))
+        _url_lower = _url.lower()
+        if _url_lower.startswith("http") and any(urlparse(_url_lower).path.endswith(ext) for ext in [".pdf"]):
+            return {"type": "file", "source_type": "url", "url": _url}
+        else:
+          _file = file or await Base64File.from_url(_url)
+          if _file.extension in ["pdf"]:
+            return {"type": "document", "source": {"type": "base64", "media_type": _file.mime_type, "data": _file.base64_content}}
+          else:
+            return await super()._format_multimodal_file_message(message, _file)
 
 class OpenAI(LlmInterface):
     def __init__(self, config: LlmConfig):
@@ -47,8 +118,9 @@ class OpenAI(LlmInterface):
         chat = ChatOpenAI(
             api_key=self.config.api_key or os.getenv("OPENAI_API_KEY"),
             model=self.config.model,
-            stream_usage=True)
-        if not (any(self.config.model.startswith(prefix) for prefix in ["o1", "o3"]) or "search" in self.config.model):
+            streaming=True
+        )
+        if not (any(self.config.model.startswith(prefix) for prefix in ["gpt-5", "o1", "o3"]) or "search" in self.config.model):
             chat.temperature = self.config.temperature
             chat.streaming = True
         return chat
@@ -59,6 +131,13 @@ class OpenAI(LlmInterface):
         response = openai.models.list()
         return response.data
 
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        _file = file or await Base64File.from_url(message.get("url"))
+        if _file.extension in ["pdf"]:
+            return {"type": "file", "file": { "source_type": "base64", "file_data": _file.base64_url, "mime_type": _file.mime_type, "filename": _file.name}}
+        else:
+          return await super()._format_multimodal_file_message(message, _file)
+
 class DeepSeek(LlmInterface):
     def get_llm(self):
         from langchain_openai import ChatOpenAI
@@ -68,8 +147,7 @@ class DeepSeek(LlmInterface):
             base_url="https://api.deepseek.com",
             max_tokens=8192,
             temperature=self.config.temperature,
-            streaming=True,
-            stream_usage=True,
+            streaming=True
         )
 
     def get_models(self):
@@ -79,36 +157,50 @@ class DeepSeek(LlmInterface):
         response = openai.models.list()
         return response.data
 
+    async def _format_multimodal_image_message(self, message: dict) -> dict:
+        print(f"{DeepSeek.__name__} does not support image messages")
+        return None
+
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        print(f"{DeepSeek.__name__} does not support file messages")
+        return None
+
 class Google(LlmInterface):
-  def get_llm(self):
-    from langchain_google_genai.chat_models import ChatGoogleGenerativeAI
-    return ChatGoogleGenerativeAI(
-      name="chat",
-      api_key=self.config.api_key or os.getenv("GOOGLE_API_KEY"),
-      model=self.config.model,
-      temperature=self.config.temperature,
-      disable_streaming=False,
-    )
-
-  def get_embeddings(self):
-    from langchain_google_genai.embeddings import GoogleGenerativeAIEmbeddings
-    return GoogleGenerativeAIEmbeddings(
-      google_api_key=self.config.api_key,
-      model="models/text-embedding-005")
-
-  def get_models(self):
-    import google.generativeai as genai
-    genai.configure(api_key=self.config.api_key or os.getenv("GOOGLE_API_KEY"))
-    response = genai.list_models()
-    return [{
-      "id": model.name,
-      "name": model.display_name,
-      "description": model.description,
-      "input_token_limit": model.input_token_limit,
-      "output_token_limit": model.output_token_limit
-    } for model in response if "gemini" in model.name.lower()]
-
-class Gvertex(LlmInterface):
+    def get_llm(self):
+      from langchain_google_genai.chat_models import ChatGoogleGenerativeAI
+      return ChatGoogleGenerativeAI(
+        model=self.config.model,
+        google_api_key=self.config.api_key or os.getenv("GOOGLE_API_KEY"),
+        temperature=self.config.temperature,
+        disable_streaming=False,
+      )
+
+    def get_embeddings(self):
+      from langchain_google_genai.embeddings import GoogleGenerativeAIEmbeddings
+      return GoogleGenerativeAIEmbeddings(
+        google_api_key=self.config.api_key or os.getenv("GOOGLE_API_KEY"),
+        model="models/gemini-embedding-001")
+
+    def get_models(self):
+      import google.generativeai as genai
+      genai.configure(api_key=self.config.api_key or os.getenv("GOOGLE_API_KEY"))
+      response = genai.list_models()
+      return [{
+        "id": model.name,
+        "name": model.display_name,
+        "description": model.description,
+        "input_token_limit": model.input_token_limit,
+        "output_token_limit": model.output_token_limit
+      } for model in response if "gemini" in model.name.lower()]
+
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        _file = file or await Base64File.from_url(message.get("url"))
+        if _file.extension in ["pdf", "csv"]:
+            return {"type": "media", "mime_type": _file.mime_type, "data": _file.base64_content }
+        else:
+          return await super()._format_multimodal_file_message(message, _file)
+
+class GoogleVertex(LlmInterface):
     def get_llm(self):
         from langchain_google_vertexai  import ChatVertexAI
         return ChatVertexAI(
@@ -116,47 +208,37 @@ class Gvertex(LlmInterface):
             temperature=self.config.temperature
         )
     def get_embeddings(self):
-        from langchain_google_vertexai import VertexAIEmbeddings
-        return VertexAIEmbeddings(model_name="text-embedding-005")
+        from langchain_google_vertexai.embeddings import VertexAIEmbeddings
+        embeddings = VertexAIEmbeddings(model_name="gemini-embedding-001")
+        return embeddings
     def get_models(self):
-        #from google.cloud import aiplatform
-        #aiplatform.init()
-        #models = aiplatform.Model.list()
+        _models = [
+                {"id":"gemini-2.5-pro"},
+                {"id":"gemini-2.5-flash"},
+                {"id":"gemini-2.0-flash"},
+                {"id":"gemini-2.0-flash-lite"}
+              ]
+        try:
+          from google.cloud import aiplatform
+          aiplatform.init()
+          _list = aiplatform.Model.list()
+          if _list:
+            _models = list([{"id": model.name} for model in _list])
         # removed due issue: https://github.com/langchain-ai/langchain-google/issues/733
         # Message type "google.cloud.aiplatform.v1beta1.GenerateContentResponse" has no field named "createTime" at "GenerateContentResponse".  Available Fields(except extensions): "['candidates', 'modelVersion', 'promptFeedback', 'usageMetadata']"
+        except Exception as e:
+          print(f"Error fetching models from Gvertex: {e}")
+          # fallback to hardcoded models
+          #see https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations#united-states for available models
+        finally:
+          return _models
 
-        #see https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations#united-states for available models
-        return [
-              {"id":"gemini-2.5-pro-preview-05-06"},
-              {"id":"gemini-2.0-flash"},
-              {"id":"gemini-2.0-flash-lite"},
-              {"id":"gemini-1.5-pro-002"}
-            ]
-
-class Anthropic(LlmInterface):
-    def get_llm(self):
-        from langchain_anthropic import ChatAnthropic
-        return ChatAnthropic(
-            api_key=self.config.api_key or os.getenv("ANTHROPIC_API_KEY"),
-            model=self.config.model,
-            temperature=self.config.temperature,
-            streaming=True,
-            stream_usage=True
-        )
-
-    """
-    def get_embeddings(self):
-        from langchain_voyageai import VoyageAIEmbeddings
-        return VoyageAIEmbeddings(
-            api_key=self.config.embedding_api_key, #voyage api key
-            model="voyage-3")
-    """
-
-    def get_models(self):
-        import anthropic
-        client = anthropic.Client(api_key=self.config.api_key or os.getenv("ANTHROPIC_API_KEY"))
-        response = client.models.list()
-        return response.data
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        _file = file or await Base64File.from_url(message.get("url"))
+        if _file.extension in ["pdf", "csv"]:
+            return {"type": "media", "mime_type": _file.mime_type, "data": _file.base64_content }
+        else:
+          return await super()._format_multimodal_file_message(message, _file)
 
 class Groq(LlmInterface):
     def get_llm(self):
@@ -179,10 +261,74 @@ class Groq(LlmInterface):
         response = requests.get(url, headers=headers)
         return response.json().get("data", [])
 
+class IBM(LlmInterface):
+    def __init__(self, config: LlmConfig):
+        from ibm_watsonx_ai import APIClient,Credentials
+        super().__init__(config)
+        self.__base_url = self.config.api_url or os.getenv("WATSONX_URL") or "https://us-south.ml.cloud.ibm.com"
+        self.__api_key = self.config.api_key or os.getenv("WATSONX_APIKEY")
+        self.__client = APIClient(
+            credentials=Credentials(url=self.__base_url,api_key=self.__api_key),
+            project_id=os.getenv("WATSONX_PROJECTID") or "default"
+        )
+    def get_llm(self):
+        from langchain_ibm import ChatWatsonx
+        return ChatWatsonx(
+            model_id=self.config.model,
+            watsonx_client=self.__client
+        )
+    def get_models(self):
+        import requests
+        from datetime import date
+        try:
+          # https://cloud.ibm.com/apidocs/watsonx-ai#list-foundation-model-specs
+          today = date.today().strftime("%Y-%m-%d")
+          url = f"{self.__base_url}/ml/v1/foundation_model_specs?version={today}&filters=task_generation,task_summarization:and"
+          headers = {
+              "Authorization": f"Bearer {self.__api_key}",
+              "Content-Type": "application/json"
+          }
+          response = requests.get(url, headers=headers)
+          models = response.json().get("resources", [])
+          return [{
+                "id": model['model_id'],
+                "provider": model['provider'],
+                "tasks": model['task_ids'],
+                "limits": model.get('model_limits', {}),
+              } for model in models]
+        except Exception as e:
+          print(f"Error fetching models from IBM WatsonX: {e}")
+          # https://www.ibm.com/products/watsonx-ai/foundation-models
+          return [
+            {"id":"ibm/granite-13b-instruct-v2"},
+            {"id":"ibm/granite-3-2b-instruct"},
+            {"id":"ibm/granite-3-8b-instruct"},
+            {"id":"meta-llama/llama-2-13b-chat"},
+            {"id":"meta-llama/llama-3-3-70b-instruct"},
+            {"id":"meta-llama/llama-4-maverick-17b-128e-instruct-fp8"},
+            {"id":"mistralai/mistral-large"},
+            {"id":"mistralai/mixtral-8x7b-instruct-v01"},
+            {"id":"mistralai/pixtral-12b"}
+          ]
+
+    def get_embeddings(self):
+        from langchain_ibm import WatsonxEmbeddings
+        from ibm_watsonx_ai.metanames import EmbedTextParamsMetaNames
+        # https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx&audience=wdp#embed
+        embed_params = {
+            EmbedTextParamsMetaNames.TRUNCATE_INPUT_TOKENS: 512,
+            #EmbedTextParamsMetaNames.RETURN_OPTIONS: {"input_text": True},
+        }
+        return WatsonxEmbeddings(
+            model_id="ibm/granite-embedding-107m-multilingual", #https://www.ibm.com/products/watsonx-ai/foundation-models
+            watsonx_client=self.__client,
+            params=embed_params
+        )
+
 class Ollama(LlmInterface):
     def __init__(self, config: LlmConfig):
         super().__init__(config)
-        self.__base_url = self.config.api_url or os.getenv("OLLAMA_API_URL")
+        self.__base_url = self.config.api_url or os.getenv("OLLAMA_API_URL") or "http://localhost:11434"
     def get_llm(self):
         from langchain_ollama.chat_models import ChatOllama
         return ChatOllama(
@@ -195,7 +341,7 @@ class Ollama(LlmInterface):
         from langchain_ollama.embeddings import OllamaEmbeddings
         return OllamaEmbeddings(
             base_url=self.__base_url,
-            model="nomic-embed-text" #mxbai-embed-large
+            model="mxbai-embed-large" #nomic-embed-text
         )
     def get_models(self):
         import requests
@@ -212,15 +358,27 @@ class Ollama(LlmInterface):
               "details": model['details']
             } for model in models]
 
+    async def _format_multimodal_image_message(self, message: dict) -> dict:
+        file = await Base64File.from_url(message.get("url"))
+        return { "type": "image_url", "image_url": { "url": file.base64_url }}
+
 class LlmManager:
+    """
+    Expose the available LLM providers.
+    Names are aligned with the LangChain documentation:
+    https://python.langchain.com/api_reference/langchain/chat_models/langchain.chat_models.base.init_chat_model.html
+    """
 
     #class variables (static)
     _list: dict[str,LlmInterface] = {
         "anthropic": Anthropic,
         "deepseek": DeepSeek,
-        "google": Google,
-        "gvertex": Gvertex,
+        "google": Google, #deprecated
+        "google_genai": Google,
+        "gvertex": GoogleVertex,#deprecated
+        "google_vertexai": GoogleVertex,
         "groq": Groq,
+        "ibm": IBM,
         "openai": OpenAI,
         "ollama": Ollama
     }

ws_bom_robot_app/llm/tools/tool_builder.py

@@ -18,7 +18,7 @@ async def __process_proxy_tool(proxy_tool: LlmAppTool) -> LlmAppTool | None:
         if not app:
             raise ValueError(f"App with id {app_id} not found.")
         tool_id = secrets.get("toolId")
-        tool = next((t for t in app.app_tools if t.id == tool_id), None)
+        tool = next((t for t in app.rq.app_tools if app.rq.app_tools and t.id == tool_id), None)
         if not tool:
             raise ValueError(f"Tool with function_id {tool_id} not found in app {app.name}.")
         #override derived tool with proxy tool props
@@ -61,5 +61,8 @@ def get_structured_tools(llm: LlmInterface, tools: list[LlmAppTool], callbacks:l
         #error_on_invalid_docstring=True
       )
       _structured_tool.tags = [tool.function_id if tool.function_id else tool.function_name]
+      secrets = tool.secrets_to_dict()
+      if secrets and secrets.get("stream") == "true":
+        _structured_tool.tags.append("stream")
       _structured_tools.append(_structured_tool)
   return _structured_tools

ws_bom_robot_app/llm/tools/tool_manager.py

@@ -1,5 +1,5 @@
 from asyncio import Queue
-import aiohttp
+import aiohttp, re
 from typing import  Optional, Type, Callable
 from ws_bom_robot_app.config import config
 from ws_bom_robot_app.llm.models.api import LlmApp,LlmAppTool
@@ -128,7 +128,9 @@ class ToolManager:
           from langchain_core.prompts import ChatPromptTemplate
           from langchain_core.output_parsers import StrOutputParser, JsonOutputParser
           from pydantic import create_model
-          system_message = self.app_tool.llm_chain_settings.prompt
+          system_message = self.app_tool.llm_chain_settings.prompt.format(
+             thread_id = self.app_tool.thread_id if self.app_tool.thread_id else "no-thread-id",
+          )
           context = []
           if self.app_tool.data_source == "knowledgebase":
             context = await self.__extract_documents(input, self.app_tool)
@@ -166,6 +168,7 @@ class ToolManager:
           return result
 
     async def proxy_app_chat(self, query: str) -> str | None:
+        from ws_bom_robot_app.llm.models.api import LlmMessage
         secrets = self.app_tool.secrets_to_dict()
         app_id = secrets.get("appId")
         if not app_id:
@@ -173,26 +176,49 @@ class ToolManager:
         app: CmsApp = await get_app_by_id(app_id)
         if not app:
             raise ValueError(f"App with id {app_id} not found.")
-        url = f"{config.robot_cms_host}/api/llm/message?locale=en&raw=true"
-        auth = config.robot_cms_auth
-        headers = {"Authorization": auth} if auth else {}
-        async with aiohttp.ClientSession() as session:
-            data = {
-              "appKey": app.credentials.app_key,
-              "apiKey": app.credentials.api_key,
-              "messages": [
-                {
-                  "role": "user",
-                  "content": query
-                }
-              ]
-            }
-            async with session.post(url, json=data, headers=headers) as response:
-                if response.status == 200:
-                  return await response.text()
-                else:
-                  raise ValueError(f"Error fetching chat response: {response.status}")
-        return None
+        # message
+        app.rq.messages.append(LlmMessage(role="user", content=query))
+        # tracing
+        if str(secrets.get("disable_tracing", False)).lower() in ['1','true','yes']:
+          app.rq.lang_chain_tracing = False
+          app.rq.lang_chain_project = ''
+          app.rq.secrets['nebulyApiKey'] = ''
+        # http: for debugging purposes
+        if str(secrets.get("use_http", False)).lower() in ['1','true','yes']:
+          import base64
+          url = f"http://localhost:{config.runtime_options().tcp_port}/api/llm/stream/raw"
+          auth = f"Basic {base64.b64encode((config.robot_user + ':' + config.robot_password).encode('utf-8')).decode('utf-8')}"
+          headers = {"Authorization": auth} if auth else {}
+          async with aiohttp.ClientSession() as session:
+              _data = app.rq.model_dump(mode='json',by_alias=True,exclude_unset=True,exclude_none=True, exclude_defaults=True)
+              async with session.post(url, json=_data, headers=headers) as response:
+                  if response.status == 200:
+                    return await response.text()
+                  else:
+                    raise ValueError(f"Error fetching chat response: {response.status}")
+          return None
+        else:  # default
+          try:
+            from ws_bom_robot_app.llm.main import stream
+            import json
+            chunks = []
+            async for chunk in stream(rq=app.rq, ctx=None, formatted=False):
+                chunks.append(chunk)
+            rs = ''.join(chunks) if chunks else None
+
+            # if the app has output_structure, parse the JSON and return dict
+            if rs and app.rq.output_structure:
+                try:
+                    cleaned_rs = re.sub(r'^```(?:json)?\s*\n?', '', rs.strip())
+                    cleaned_rs = re.sub(r'\n?```\s*$', '', cleaned_rs)
+                    return json.loads(cleaned_rs)
+                except json.JSONDecodeError:
+                    print(f"[!] Failed to parse JSON output from proxy_app_chat: {rs}")
+                    return rs
+            return rs
+          except Exception as e:
+            print(f"[!] Error in proxy_app_chat: {e}")
+            return None
 
     async def proxy_app_tool(self) -> None:
         return None

ws_bom_robot_app/llm/utils/chunker.py

@@ -1,12 +1,17 @@
 from langchain_core.documents import Document
 from langchain_text_splitters import CharacterTextSplitter
+import logging
 
 class DocumentChunker:
+  _MAX_CHUNK_SIZE = 10_000
   @staticmethod
   def chunk(documents: list[Document]) -> list[Document]:
-      text_splitter = CharacterTextSplitter(chunk_size=10_000, chunk_overlap=500)
+      text_splitter = CharacterTextSplitter(chunk_size=DocumentChunker._MAX_CHUNK_SIZE, chunk_overlap=int(DocumentChunker._MAX_CHUNK_SIZE * 0.02))
       chunked_documents = []
       for doc in documents:
+          if len(doc.page_content) <= DocumentChunker._MAX_CHUNK_SIZE:
+              chunked_documents.append(doc)
+              continue
           chunks = text_splitter.split_text(doc.page_content)
           for chunk in chunks:
               chunked_documents.append(

ws_bom_robot_app/llm/utils/cleanup.py

@@ -0,0 +1,81 @@
+import os, logging
+from ws_bom_robot_app.config import config
+from datetime import datetime, timedelta
+
+def _cleanup_data_file(folders: list[str], retention: float) -> dict:
+    """
+    clean up old data files in the specified folder
+
+    Returns:
+    - Dictionary with cleanup statistics
+    """
+    _deleted_files = []
+    _deleted_dirs = []
+    _freed_space = 0
+
+    for folder in folders:
+        if not os.path.exists(folder):
+            logging.warning(f"Folder does not exist: {folder}")
+            continue
+
+        # delete old files
+        for root, dirs, files in os.walk(folder, topdown=False):
+            for file in files:
+                file_path = os.path.join(root, file)
+                try:
+                    file_stat = os.stat(file_path)
+                    file_creation_time = datetime.fromtimestamp(file_stat.st_mtime)
+                    if file_creation_time < datetime.now() - timedelta(days=retention):
+                        _freed_space += file_stat.st_size
+                        os.remove(file_path)
+                        _deleted_files.append(file_path)
+                except (OSError, IOError) as e:
+                    logging.error(f"Error deleting file {file_path}: {e}")
+
+        # clean up empty directories (bottom-up)
+        for root, dirs, files in os.walk(folder, topdown=False):
+            # skip the root folder itself
+            if root == folder:
+                continue
+            try:
+                # check if directory is empty
+                if not os.listdir(root):
+                    os.rmdir(root)
+                    _deleted_dirs.append(root)
+            except OSError as e:
+                logging.debug(f"Could not remove directory {root}: {e}")
+    logging.info(f"Deleted {len(_deleted_files)} files; Freed space: {_freed_space / (1024 * 1024):.2f} MB")
+
+    return {
+        "deleted_files_count": len(_deleted_files),
+        "deleted_dirs_count": len(_deleted_dirs),
+        "freed_space_mb": _freed_space / (1024 * 1024)
+    }
+
+def kb_cleanup_data_file() -> dict:
+    """
+    clean up vector db data files
+    """
+
+    folders = [
+        os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out),
+        os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_store),
+        os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_src)
+    ]
+    return _cleanup_data_file(folders, config.robot_data_db_retention_days)
+
+def chat_cleanup_attachment() -> dict:
+    """
+    clean up chat attachment files
+    """
+    folders = [
+        os.path.join(config.robot_data_folder, config.robot_data_attachment_folder)
+        ]
+    return _cleanup_data_file(folders, config.robot_data_attachment_retention_days)
+
+def task_cleanup_history() -> None:
+    """
+    clean up task queue
+    """
+    from ws_bom_robot_app.task_manager import task_manager
+    task_manager.cleanup_task()