ws-bom-robot-app 0.0.37__py3-none-any.whl → 0.0.103__py3-none-any.whl

ws_bom_robot_app/llm/providers/llm_manager.py

@@ -1,9 +1,10 @@
-import json
 from typing import Optional
+from urllib.parse import urlparse
 from langchain_core.embeddings import Embeddings
 from langchain_core.language_models import BaseChatModel
 from pydantic import BaseModel, ConfigDict, Field
 import os
+from ws_bom_robot_app.llm.utils.download import Base64File
 
 class LlmConfig(BaseModel):
     api_url: Optional[str] = None
@@ -36,6 +37,76 @@ class LlmInterface:
     def get_parser(self):
         from langchain.agents.output_parsers.openai_tools import OpenAIToolsAgentOutputParser
         return OpenAIToolsAgentOutputParser()
+    async def _format_multimodal_image_message(self, message: dict) -> dict:
+        return {
+            "type": "image_url",
+            "image_url": {
+                "url": message.get("url")
+            }
+        }
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        _file = file or await Base64File.from_url(message.get("url"))
+        return {"type": "text", "text": f"Here's a file attachment named `{_file.name}` of type `{_file.mime_type}` in base64: `{_file.base64_content}`"}
+    async def format_multimodal_content(self, content: list) -> list:
+        _content = []
+        for message in content:
+            if isinstance(message, dict):
+                if message.get("type") == "image" and "url" in message:
+                    _content.append(await self._format_multimodal_image_message(message))
+                elif message.get("type") == "file" and "url" in message:
+                    _content.append(await self._format_multimodal_file_message(message))
+                else:
+                    # pass through text or other formats unchanged
+                    _content.append(message)
+            else:
+                _content.append(message)
+        return _content
+
+class Anthropic(LlmInterface):
+    def get_llm(self):
+        from langchain_anthropic import ChatAnthropic
+        return ChatAnthropic(
+            api_key=self.config.api_key or os.getenv("ANTHROPIC_API_KEY"),
+            model=self.config.model,
+            temperature=self.config.temperature,
+            max_tokens=8192,
+            streaming=True,
+            #betas=["files-api-2025-04-14"] #https://docs.anthropic.com/en/docs/build-with-claude/files
+        )
+
+    """
+    def get_embeddings(self):
+        from langchain_voyageai import VoyageAIEmbeddings
+        return VoyageAIEmbeddings(
+            api_key=self.config.embedding_api_key, #voyage api key
+            model="voyage-3")
+    """
+
+    def get_models(self):
+        import anthropic
+        client = anthropic.Client(api_key=self.config.api_key or os.getenv("ANTHROPIC_API_KEY"))
+        response = client.models.list()
+        return response.data
+
+    """
+    async def _format_multimodal_image_message(self, message: dict) -> dict:
+        file = await Base64File.from_url(message.get("url"))
+        return { "type": "image_url", "image_url": { "url": file.base64_url }}
+    """
+
+    #https://python.langchain.com/docs/integrations/chat/anthropic/
+    #https://python.langchain.com/docs/how_to/multimodal_inputs/
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        _url = str(message.get("url", ""))
+        _url_lower = _url.lower()
+        if _url_lower.startswith("http") and any(urlparse(_url_lower).path.endswith(ext) for ext in [".pdf"]):
+            return {"type": "file", "source_type": "url", "url": _url}
+        else:
+          _file = file or await Base64File.from_url(_url)
+          if _file.extension in ["pdf"]:
+            return {"type": "document", "source": {"type": "base64", "media_type": _file.mime_type, "data": _file.base64_content}}
+          else:
+            return await super()._format_multimodal_file_message(message, _file)
 
 class OpenAI(LlmInterface):
     def __init__(self, config: LlmConfig):
@@ -44,8 +115,12 @@ class OpenAI(LlmInterface):
 
     def get_llm(self):
         from langchain_openai import ChatOpenAI
-        chat = ChatOpenAI(api_key=self.config.api_key, model=self.config.model)
-        if not any(self.config.model.startswith(prefix) for prefix in ["o1", "o3"]):
+        chat = ChatOpenAI(
+            api_key=self.config.api_key or os.getenv("OPENAI_API_KEY"),
+            model=self.config.model,
+            streaming=True
+        )
+        if not (any(self.config.model.startswith(prefix) for prefix in ["gpt-5", "o1", "o3"]) or "search" in self.config.model):
             chat.temperature = self.config.temperature
             chat.streaming = True
         return chat
@@ -56,54 +131,76 @@ class OpenAI(LlmInterface):
         response = openai.models.list()
         return response.data
 
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        _file = file or await Base64File.from_url(message.get("url"))
+        if _file.extension in ["pdf"]:
+            return {"type": "file", "file": { "source_type": "base64", "file_data": _file.base64_url, "mime_type": _file.mime_type, "filename": _file.name}}
+        else:
+          return await super()._format_multimodal_file_message(message, _file)
+
 class DeepSeek(LlmInterface):
     def get_llm(self):
         from langchain_openai import ChatOpenAI
         return ChatOpenAI(
-            api_key=self.config.api_key,
+            api_key=self.config.api_key or os.getenv("DEEPSEEK_API_KEY"),
             model=self.config.model,
-            base_url="https://api.deepseek.com/v1",
+            base_url="https://api.deepseek.com",
             max_tokens=8192,
             temperature=self.config.temperature,
-            streaming=True,
+            streaming=True
         )
 
     def get_models(self):
-        return [
-              {"id":"deepseek-chat"},
-              {"id":"deepseek-reasoner"}
-            ]
+        import openai
+        openai.api_key = self.config.api_key or os.getenv("DEEPSEEK_API_KEY")
+        openai.base_url = "https://api.deepseek.com"
+        response = openai.models.list()
+        return response.data
+
+    async def _format_multimodal_image_message(self, message: dict) -> dict:
+        print(f"{DeepSeek.__name__} does not support image messages")
+        return None
+
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        print(f"{DeepSeek.__name__} does not support file messages")
+        return None
 
 class Google(LlmInterface):
-  def get_llm(self):
-    from langchain_google_genai.chat_models import ChatGoogleGenerativeAI
-    return ChatGoogleGenerativeAI(
-      name="chat",
-      api_key=self.config.api_key,
-      model=self.config.model,
-      temperature=self.config.temperature,
-      disable_streaming=False
-    )
-
-  def get_embeddings(self):
-    from langchain_google_genai.embeddings import GoogleGenerativeAIEmbeddings
-    return GoogleGenerativeAIEmbeddings(
-      google_api_key=self.config.api_key,
-      model="models/text-embedding-004")
-
-  def get_models(self):
-    import google.generativeai as genai
-    genai.configure(api_key=self.config.api_key or os.getenv("GOOGLE_API_KEY"))
-    response = genai.list_models()
-    return [{
-      "id": model.name,
-      "name": model.display_name,
-      "description": model.description,
-      "input_token_limit": model.input_token_limit,
-      "output_token_limit": model.output_token_limit
-    } for model in response if "gemini" in model.name.lower()]
-
-class Gvertex(LlmInterface):
+    def get_llm(self):
+      from langchain_google_genai.chat_models import ChatGoogleGenerativeAI
+      return ChatGoogleGenerativeAI(
+        model=self.config.model,
+        google_api_key=self.config.api_key or os.getenv("GOOGLE_API_KEY"),
+        temperature=self.config.temperature,
+        disable_streaming=False,
+      )
+
+    def get_embeddings(self):
+      from langchain_google_genai.embeddings import GoogleGenerativeAIEmbeddings
+      return GoogleGenerativeAIEmbeddings(
+        google_api_key=self.config.api_key or os.getenv("GOOGLE_API_KEY"),
+        model="models/gemini-embedding-001")
+
+    def get_models(self):
+      import google.generativeai as genai
+      genai.configure(api_key=self.config.api_key or os.getenv("GOOGLE_API_KEY"))
+      response = genai.list_models()
+      return [{
+        "id": model.name,
+        "name": model.display_name,
+        "description": model.description,
+        "input_token_limit": model.input_token_limit,
+        "output_token_limit": model.output_token_limit
+      } for model in response if "gemini" in model.name.lower()]
+
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        _file = file or await Base64File.from_url(message.get("url"))
+        if _file.extension in ["pdf", "csv"]:
+            return {"type": "media", "mime_type": _file.mime_type, "data": _file.base64_content }
+        else:
+          return await super()._format_multimodal_file_message(message, _file)
+
+class GoogleVertex(LlmInterface):
     def get_llm(self):
         from langchain_google_vertexai  import ChatVertexAI
         return ChatVertexAI(
@@ -111,52 +208,43 @@ class Gvertex(LlmInterface):
             temperature=self.config.temperature
         )
     def get_embeddings(self):
-        from langchain_google_vertexai import VertexAIEmbeddings
-        return VertexAIEmbeddings(model_name="text-embedding-004")
+        from langchain_google_vertexai.embeddings import VertexAIEmbeddings
+        embeddings = VertexAIEmbeddings(model_name="gemini-embedding-001")
+        return embeddings
     def get_models(self):
-        #from google.cloud import aiplatform
-        #aiplatform.init()
-        #models = aiplatform.Model.list()
+        _models = [
+                {"id":"gemini-2.5-pro"},
+                {"id":"gemini-2.5-flash"},
+                {"id":"gemini-2.0-flash"},
+                {"id":"gemini-2.0-flash-lite"}
+              ]
+        try:
+          from google.cloud import aiplatform
+          aiplatform.init()
+          _list = aiplatform.Model.list()
+          if _list:
+            _models = list([{"id": model.name} for model in _list])
         # removed due issue: https://github.com/langchain-ai/langchain-google/issues/733
         # Message type "google.cloud.aiplatform.v1beta1.GenerateContentResponse" has no field named "createTime" at "GenerateContentResponse".  Available Fields(except extensions): "['candidates', 'modelVersion', 'promptFeedback', 'usageMetadata']"
+        except Exception as e:
+          print(f"Error fetching models from Gvertex: {e}")
+          # fallback to hardcoded models
+          #see https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations#united-states for available models
+        finally:
+          return _models
 
-        #see https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations#united-states for available models
-        return [
-              {"id":"gemini-2.0-flash-001"},
-              {"id":"gemini-1.5-pro-001"},
-              {"id":"gemini-1.5-pro-002"}
-            ]
-
-class Anthropic(LlmInterface):
-    def get_llm(self):
-        from langchain_anthropic import ChatAnthropic
-        return ChatAnthropic(
-            api_key=self.config.api_key,
-            model=self.config.model,
-            temperature=self.config.temperature,
-            streaming=True,
-            stream_usage=False
-        )
-
-    """
-    def get_embeddings(self):
-        from langchain_voyageai import VoyageAIEmbeddings
-        return VoyageAIEmbeddings(
-            api_key=self.config.embedding_api_key, #voyage api key
-            model="voyage-3")
-    """
-
-    def get_models(self):
-        import anthropic
-        client = anthropic.Client(api_key=self.config.api_key or os.getenv("ANTHROPIC_API_KEY"))
-        response = client.models.list()
-        return response.data
+    async def _format_multimodal_file_message(self, message: dict, file: Base64File = None) -> dict:
+        _file = file or await Base64File.from_url(message.get("url"))
+        if _file.extension in ["pdf", "csv"]:
+            return {"type": "media", "mime_type": _file.mime_type, "data": _file.base64_content }
+        else:
+          return await super()._format_multimodal_file_message(message, _file)
 
 class Groq(LlmInterface):
     def get_llm(self):
         from langchain_groq import ChatGroq
         return ChatGroq(
-            api_key=self.config.api_key,
+            api_key=self.config.api_key or os.getenv("GROQ_API_KEY"),
             model=self.config.model,
             #max_tokens=8192,
             temperature=self.config.temperature,
@@ -173,10 +261,74 @@ class Groq(LlmInterface):
         response = requests.get(url, headers=headers)
         return response.json().get("data", [])
 
+class IBM(LlmInterface):
+    def __init__(self, config: LlmConfig):
+        from ibm_watsonx_ai import APIClient,Credentials
+        super().__init__(config)
+        self.__base_url = self.config.api_url or os.getenv("WATSONX_URL") or "https://us-south.ml.cloud.ibm.com"
+        self.__api_key = self.config.api_key or os.getenv("WATSONX_APIKEY")
+        self.__client = APIClient(
+            credentials=Credentials(url=self.__base_url,api_key=self.__api_key),
+            project_id=os.getenv("WATSONX_PROJECTID") or "default"
+        )
+    def get_llm(self):
+        from langchain_ibm import ChatWatsonx
+        return ChatWatsonx(
+            model_id=self.config.model,
+            watsonx_client=self.__client
+        )
+    def get_models(self):
+        import requests
+        from datetime import date
+        try:
+          # https://cloud.ibm.com/apidocs/watsonx-ai#list-foundation-model-specs
+          today = date.today().strftime("%Y-%m-%d")
+          url = f"{self.__base_url}/ml/v1/foundation_model_specs?version={today}&filters=task_generation,task_summarization:and"
+          headers = {
+              "Authorization": f"Bearer {self.__api_key}",
+              "Content-Type": "application/json"
+          }
+          response = requests.get(url, headers=headers)
+          models = response.json().get("resources", [])
+          return [{
+                "id": model['model_id'],
+                "provider": model['provider'],
+                "tasks": model['task_ids'],
+                "limits": model.get('model_limits', {}),
+              } for model in models]
+        except Exception as e:
+          print(f"Error fetching models from IBM WatsonX: {e}")
+          # https://www.ibm.com/products/watsonx-ai/foundation-models
+          return [
+            {"id":"ibm/granite-13b-instruct-v2"},
+            {"id":"ibm/granite-3-2b-instruct"},
+            {"id":"ibm/granite-3-8b-instruct"},
+            {"id":"meta-llama/llama-2-13b-chat"},
+            {"id":"meta-llama/llama-3-3-70b-instruct"},
+            {"id":"meta-llama/llama-4-maverick-17b-128e-instruct-fp8"},
+            {"id":"mistralai/mistral-large"},
+            {"id":"mistralai/mixtral-8x7b-instruct-v01"},
+            {"id":"mistralai/pixtral-12b"}
+          ]
+
+    def get_embeddings(self):
+        from langchain_ibm import WatsonxEmbeddings
+        from ibm_watsonx_ai.metanames import EmbedTextParamsMetaNames
+        # https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx&audience=wdp#embed
+        embed_params = {
+            EmbedTextParamsMetaNames.TRUNCATE_INPUT_TOKENS: 512,
+            #EmbedTextParamsMetaNames.RETURN_OPTIONS: {"input_text": True},
+        }
+        return WatsonxEmbeddings(
+            model_id="ibm/granite-embedding-107m-multilingual", #https://www.ibm.com/products/watsonx-ai/foundation-models
+            watsonx_client=self.__client,
+            params=embed_params
+        )
+
 class Ollama(LlmInterface):
     def __init__(self, config: LlmConfig):
         super().__init__(config)
-        self.__base_url = self.config.api_url or os.getenv("OLLAMA_API_URL")
+        self.__base_url = self.config.api_url or os.getenv("OLLAMA_API_URL") or "http://localhost:11434"
     def get_llm(self):
         from langchain_ollama.chat_models import ChatOllama
         return ChatOllama(
@@ -189,7 +341,7 @@ class Ollama(LlmInterface):
         from langchain_ollama.embeddings import OllamaEmbeddings
         return OllamaEmbeddings(
             base_url=self.__base_url,
-            model="nomic-embed-text" #mxbai-embed-large
+            model="mxbai-embed-large" #nomic-embed-text
         )
     def get_models(self):
         import requests
@@ -206,15 +358,27 @@ class Ollama(LlmInterface):
               "details": model['details']
             } for model in models]
 
+    async def _format_multimodal_image_message(self, message: dict) -> dict:
+        file = await Base64File.from_url(message.get("url"))
+        return { "type": "image_url", "image_url": { "url": file.base64_url }}
+
 class LlmManager:
+    """
+    Expose the available LLM providers.
+    Names are aligned with the LangChain documentation:
+    https://python.langchain.com/api_reference/langchain/chat_models/langchain.chat_models.base.init_chat_model.html
+    """
 
     #class variables (static)
     _list: dict[str,LlmInterface] = {
         "anthropic": Anthropic,
         "deepseek": DeepSeek,
-        "google": Google,
-        "gvertex": Gvertex,
+        "google": Google, #deprecated
+        "google_genai": Google,
+        "gvertex": GoogleVertex,#deprecated
+        "google_vertexai": GoogleVertex,
         "groq": Groq,
+        "ibm": IBM,
         "openai": OpenAI,
         "ollama": Ollama
     }

ws_bom_robot_app/llm/tools/models/main.py

@@ -7,3 +7,11 @@ class DocumentRetrieverInput(BaseModel):
 class ImageGeneratorInput(BaseModel):
   query: str = Field(description="description of the image to generate.")
   language: str = Field(description="Language of the query. Default is 'it'", default="it")
+class LlmChainInput(BaseModel):
+  input: str = Field(description="Input to the LLM chain")
+class SearchOnlineInput(BaseModel):
+  query: str = Field(description="The search query string")
+class EmailSenderInput(BaseModel):
+  email_subject: str = Field(description="The subject of the email to send")
+  body: str = Field(description="The body of the email to send")
+  to_email: str = Field(description="The recipient email address")

ws_bom_robot_app/llm/tools/tool_builder.py

@@ -1,23 +1,68 @@
-from asyncio import Queue
-from langchain.tools import Tool, StructuredTool
-from ws_bom_robot_app.llm.models.api import LlmAppTool
-from ws_bom_robot_app.llm.tools.tool_manager import ToolManager
-from ws_bom_robot_app.llm.providers.llm_manager import LlmInterface
-
-def get_structured_tools(llm: LlmInterface, tools: list[LlmAppTool], callbacks:list, queue: Queue) -> list[StructuredTool]:
-  _structured_tools :list[StructuredTool] = []
-  for tool in [tool for tool in tools if tool.is_active]:
-    if _tool_config := ToolManager._list.get(tool.function_name):
-      _tool_instance = ToolManager(llm, tool, callbacks, queue)
-      _structured_tool = StructuredTool.from_function(
-        coroutine=_tool_instance.get_coroutine(),
-        name=tool.function_id if tool.function_id else tool.function_name,
-        description=tool.function_description,
-        args_schema=_tool_config.model
-        #infer_schema=True,
-        #parse_docstring=True,
-        #error_on_invalid_docstring=True
-      )
-      _structured_tool.tags = [tool.function_id if tool.function_id else tool.function_name]
-      _structured_tools.append(_structured_tool)
-  return _structured_tools
+import asyncio
+from asyncio import Queue
+from langchain.tools import StructuredTool
+from ws_bom_robot_app.llm.models.api import LlmAppTool
+from ws_bom_robot_app.llm.tools.tool_manager import ToolManager
+from ws_bom_robot_app.llm.providers.llm_manager import LlmInterface
+
+async def __process_proxy_tool(proxy_tool: LlmAppTool) -> LlmAppTool | None:
+    import os
+    from ws_bom_robot_app.llm.utils.cms import CmsApp, get_app_by_id
+    from ws_bom_robot_app.config import config
+    try:
+        secrets = proxy_tool.secrets_to_dict()
+        app_id = secrets.get("appId")
+        if not app_id:
+            raise ValueError("Tool configuration is invalid. 'appId' is required.")
+        app: CmsApp = await get_app_by_id(app_id)
+        if not app:
+            raise ValueError(f"App with id {app_id} not found.")
+        tool_id = secrets.get("toolId")
+        tool = next((t for t in app.rq.app_tools if app.rq.app_tools and t.id == tool_id), None)
+        if not tool:
+            raise ValueError(f"Tool with function_id {tool_id} not found in app {app.name}.")
+        #override derived tool with proxy tool props
+        tool.name = proxy_tool.name if proxy_tool.name else tool.name
+        tool.description = proxy_tool.description if proxy_tool.description else tool.description
+        tool.function_id = proxy_tool.function_id if proxy_tool.function_id else tool.function_id
+        tool.function_description = proxy_tool.function_description if proxy_tool.function_description else tool.function_description
+        #normalize vector_db
+        if tool.vector_db:
+            tool.vector_db = os.path.join(
+               os.path.join(config.robot_data_folder,config.robot_data_db_folder,config.robot_data_db_folder_store),
+               os.path.splitext(os.path.basename(tool.vector_db))[0]) if tool.vector_db else None
+        return tool
+    except Exception as e:
+        print(f"[!] Error in proxy_app_tool: {e}")
+        return None
+
+def get_structured_tools(llm: LlmInterface, tools: list[LlmAppTool], callbacks:list, queue: Queue) -> list[StructuredTool]:
+  _structured_tools :list[StructuredTool] = []
+  for tool in [tool for tool in tools if tool.is_active]:
+    if tool.function_name == "proxy_app_tool":
+        # override the tool
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+          import nest_asyncio
+          nest_asyncio.apply()
+        processed_tool = loop.run_until_complete(__process_proxy_tool(tool))
+        if processed_tool is None:
+            continue
+        tool = processed_tool
+    if _tool_config := ToolManager._list.get(tool.function_name):
+      _tool_instance = ToolManager(llm, tool, callbacks, queue)
+      _structured_tool = StructuredTool.from_function(
+        coroutine=_tool_instance.get_coroutine(),
+        name=tool.function_id if tool.function_id else tool.function_name,
+        description=tool.function_description,
+        args_schema=_tool_config.model
+        #infer_schema=True,
+        #parse_docstring=True,
+        #error_on_invalid_docstring=True
+      )
+      _structured_tool.tags = [tool.function_id if tool.function_id else tool.function_name]
+      secrets = tool.secrets_to_dict()
+      if secrets and secrets.get("stream") == "true":
+        _structured_tool.tags.append("stream")
+      _structured_tools.append(_structured_tool)
+  return _structured_tools