ws-bom-robot-app 0.0.37__py3-none-any.whl → 0.0.103__py3-none-any.whl

ws_bom_robot_app/llm/main.py

@@ -1,110 +1,159 @@
-from asyncio import Queue
-import asyncio, json, logging, os, traceback
-from fastapi import Request
-from langchain.callbacks.tracers import LangChainTracer
-from langchain_core.callbacks.base import AsyncCallbackHandler
-from langchain_core.messages import AIMessage, HumanMessage
-from langsmith import Client as LangSmithClient
-from nebuly.providers.langchain import LangChainTrackingHandler
-from typing import AsyncGenerator, List
-from ws_bom_robot_app.config import config
-from ws_bom_robot_app.llm.agent_description import AgentDescriptor
-from ws_bom_robot_app.llm.agent_handler import AgentHandler, RawAgentHandler
-from ws_bom_robot_app.llm.agent_lcel import AgentLcel
-from ws_bom_robot_app.llm.models.api import InvokeRequest, StreamRequest
-from ws_bom_robot_app.llm.providers.llm_manager import LlmInterface
-from ws_bom_robot_app.llm.tools.tool_builder import get_structured_tools
-import ws_bom_robot_app.llm.settings as settings
-
-async def invoke(rq: InvokeRequest) -> str:
-  await rq.initialize()
-  _msg: str = rq.messages[-1].content
-  processor = AgentDescriptor(
-      llm=rq.get_llm(),
-      prompt=rq.system_message,
-      mode = rq.mode,
-      rules=rq.rules if rq.rules else None
-  )
-  result: AIMessage = await processor.run_agent(_msg)
-  return {"result": result.content}
-
-async def __stream(rq: StreamRequest, ctx: Request, queue: Queue,formatted: bool = True) -> None:
-  await rq.initialize()
-  #os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
-  if formatted:
-    agent_handler = AgentHandler(queue,rq.provider,rq.thread_id)
-  else:
-    agent_handler = RawAgentHandler(queue,rq.provider)
-  os.environ["AGENT_HANDLER_FORMATTED"] = str(formatted)
-  callbacks: List[AsyncCallbackHandler] = [agent_handler]
-  settings.init()
-
-  #CREATION OF CHAT HISTORY FOR AGENT
-  for message in rq.messages:
-      if message.role == "user":
-          settings.chat_history.append(HumanMessage(content=message.content))
-      elif message.role == "assistant":
-          message_content = ""
-          if formatted and '{\"type\":\"text\"' in message.content:
-            try:
-              json_msg = json.loads('[' + message.content[:-1] + ']')
-              for msg in json_msg:
-                if msg.get("content"):
-                  message_content += msg["content"]
-            except:
-              message_content = message.content
-          else:
-            message_content = message.content
-          settings.chat_history.append(AIMessage(content=message_content))
-
-  if rq.lang_chain_tracing:
-    client = LangSmithClient(
-      api_key= rq.secrets.get("langChainApiKey", "")
-    )
-    trace = LangChainTracer(project_name=rq.lang_chain_project,client=client,tags=[str(ctx.base_url)])
-    callbacks.append(trace)
-
-  __llm: LlmInterface =rq.get_llm()
-  processor = AgentLcel(
-      llm=__llm,
-      sys_message=rq.system_message,
-      tools=get_structured_tools(__llm, tools=rq.app_tools, callbacks=[callbacks], queue=queue),
-      rules=rq.rules
-  )
-  if rq.secrets.get("nebulyApiKey","") != "":
-    nebuly_callback = LangChainTrackingHandler(
-            api_key= rq.secrets.get("nebulyApiKey"),
-            user_id=rq.thread_id,
-            nebuly_tags={"project": rq.lang_chain_project},
-        )
-    callbacks.append(nebuly_callback)
-
-  #with warnings.catch_warnings():
-  #  warnings.simplefilter("ignore", UserWarning)
-  try:
-    await processor.executor.ainvoke(
-        {"input": rq.messages[-1], "chat_history": settings.chat_history},
-        {"callbacks": callbacks},
-    )
-  except Exception as e:
-    _error = f"Agent invoke ex: {e}"
-    logging.warning(_error)
-    if config.runtime_options().debug:
-      _error += f" | {traceback.format_exc()}"
-      await queue.put(_error)
-    await queue.put(None)
-
-  # Signal the end of streaming
-  await queue.put(None)
-
-async def stream(rq: StreamRequest, ctx: Request, formatted: bool = True) -> AsyncGenerator[str, None]:
-    queue = Queue()
-    task = asyncio.create_task(__stream(rq, ctx, queue, formatted))
-    try:
-        while True:
-            token = await queue.get()
-            if token is None:  # None indicates the end of streaming
-                break
-            yield token
-    finally:
-        await task
+from asyncio import Queue
+import asyncio, json, logging, os, traceback, re
+from fastapi import Request
+from langchain.callbacks.tracers import LangChainTracer
+from langchain_core.callbacks.base import AsyncCallbackHandler
+from langchain_core.messages import BaseMessage, AIMessage, HumanMessage
+from langsmith import Client as LangSmithClient
+from typing import AsyncGenerator, List
+from ws_bom_robot_app.config import config
+from ws_bom_robot_app.llm.agent_description import AgentDescriptor
+from ws_bom_robot_app.llm.agent_handler import AgentHandler, RawAgentHandler
+from ws_bom_robot_app.llm.agent_lcel import AgentLcel
+from ws_bom_robot_app.llm.models.api import InvokeRequest, StreamRequest
+from ws_bom_robot_app.llm.providers.llm_manager import LlmInterface
+from ws_bom_robot_app.llm.tools.tool_builder import get_structured_tools
+from ws_bom_robot_app.llm.nebuly_handler import NebulyHandler
+
+async def invoke(rq: InvokeRequest) -> str:
+  await rq.initialize()
+  _msg: str = rq.messages[-1].content
+  processor = AgentDescriptor(
+      llm=rq.get_llm(),
+      prompt=rq.system_message,
+      mode = rq.mode,
+      rules=rq.rules if rq.rules else None
+  )
+  result: AIMessage = await processor.run_agent(_msg)
+  return {"result": result.content}
+
+def _parse_formatted_message(message: str) -> str:
+  try:
+    text_fragments = []
+    quoted_strings = re.findall(r'"([^"\\]*(?:\\.[^"\\]*)*)"', message)
+    for string in quoted_strings:
+      if not string.startswith(('threadId', 'type')) and len(string) > 1:
+          text_fragments.append(string)
+    result = ''.join(text_fragments)
+    result = result.replace('\\n', '\n')
+  except:
+    result = message
+  return result
+
+async def __stream(rq: StreamRequest, ctx: Request, queue: Queue, formatted: bool = True) -> None:
+  #os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
+
+  # rq initialization
+  await rq.initialize()
+  for tool in rq.app_tools:
+    tool.thread_id = rq.thread_id
+
+  #llm
+  __llm: LlmInterface = rq.get_llm()
+
+  #chat history
+  chat_history: list[BaseMessage] = []
+  for message in rq.messages:
+    if message.role in ["human","user"]:
+      _content = message.content
+      # multimodal content parsing
+      if isinstance(_content, list):
+        try:
+          _content = await __llm.format_multimodal_content(_content)
+        except Exception as e:
+          logging.warning(f"Error parsing multimodal content {_content[:100]}: {e}")
+      chat_history.append(HumanMessage(content=_content))
+    elif message.role in ["ai","assistant"]:
+      message_content = ""
+      if formatted:
+        if '{\"type\":\"string\"' in message.content:
+          try:
+            json_msg = json.loads('[' + message.content[:-1] + ']')
+            for msg in json_msg:
+              if msg.get("content"):
+                message_content += msg["content"]
+          except:
+            message_content = _parse_formatted_message(message.content)
+        elif '{\"type\":\"text\"' in message.content:
+          try:
+            json_msg = json.loads('[' + message.content[:-1] + ']')
+            for msg in json_msg:
+              if msg.get("text"):
+                message_content += msg["text"]
+          except:
+            message_content = _parse_formatted_message(message.content)
+        else:
+          message_content = _parse_formatted_message(message.content)
+      else:
+        message_content = message.content
+      if message_content:
+        chat_history.append(AIMessage(content=message_content))
+
+
+  #agent handler
+  if formatted:
+    agent_handler = AgentHandler(queue, rq.provider, rq.thread_id)
+  else:
+    agent_handler = RawAgentHandler(queue, rq.provider)
+  #TODO: move from os.environ to rq
+  os.environ["AGENT_HANDLER_FORMATTED"] = str(formatted)
+
+  #callbacks
+  ## agent
+  callbacks: List[AsyncCallbackHandler] = [agent_handler]
+  ## langchain tracing
+  if rq.lang_chain_tracing:
+    client = LangSmithClient(
+      api_key= rq.secrets.get("langChainApiKey", "")
+    )
+    trace = LangChainTracer(project_name=rq.lang_chain_project,client=client,tags=[str(ctx.base_url) if ctx else ''])
+    callbacks.append(trace)
+  ## nebuly tracing
+  if rq.secrets.get("nebulyApiKey","") != "":
+    user_id = rq.system_context.user.id if rq.system_context and rq.system_context.user and rq.system_context.user.id else None
+    nebuly_callback = NebulyHandler(
+        llm_model=__llm.config.model,
+        threadId=rq.thread_id,
+        chat_history=chat_history,
+        url=config.NEBULY_API_URL,
+        api_key=rq.secrets.get("nebulyApiKey", None),
+        user_id=user_id
+      )
+    callbacks.append(nebuly_callback)
+
+  # chain
+  processor = AgentLcel(
+      llm=__llm,
+      sys_message=rq.system_message,
+      sys_context=rq.system_context,
+      tools=get_structured_tools(__llm, tools=rq.app_tools, callbacks=[callbacks], queue=queue),
+      rules=rq.rules,
+      json_schema=rq.output_structure.get("outputFormat") if rq.output_structure and rq.output_structure.get("outputType") == "json" else None
+  )
+  try:
+    await processor.executor.ainvoke(
+        {"chat_history": chat_history},
+        {"callbacks": callbacks},
+    )
+  except Exception as e:
+    _error = f"Agent invoke ex: {e}"
+    logging.warning(_error)
+    if config.runtime_options().debug:
+      _error += f" | {traceback.format_exc()}"
+      await queue.put(_error)
+    await queue.put(None)
+
+  # signal the end of streaming
+  await queue.put(None)
+
+async def stream(rq: StreamRequest, ctx: Request, formatted: bool = True) -> AsyncGenerator[str, None]:
+    queue = Queue()
+    task = asyncio.create_task(__stream(rq, ctx, queue, formatted))
+    try:
+        while True:
+            token = await queue.get()
+            if token is None:  # None indicates the end of streaming
+                break
+            yield token
+    finally:
+        await task

ws_bom_robot_app/llm/models/api.py

@@ -1,8 +1,9 @@
-from typing import List, Dict, Optional, Tuple, Union
+from typing import List, Dict, Optional, Tuple, Union, Any
 from datetime import datetime
 from pydantic import AliasChoices, BaseModel, Field, ConfigDict
 from langchain_core.embeddings import Embeddings
 from langchain.chains.query_constructor.schema import AttributeInfo
+from ws_bom_robot_app.llm.agent_context import AgentContext
 from ws_bom_robot_app.llm.models.kb import LlmKbEndpoint, LlmKbIntegration
 from ws_bom_robot_app.llm.providers.llm_manager import LlmManager, LlmConfig, LlmInterface
 from ws_bom_robot_app.llm.utils.download import download_file
@@ -10,6 +11,39 @@ import os, shutil, uuid
 from ws_bom_robot_app.config import Settings, config
 
 class LlmMessage(BaseModel):
+  """
+  💬 multimodal chat
+
+  The multimodal message allows users to interact with the application using both text and media files.
+  `robot` accept multimodal input in a uniform way, regarding the llm provider used.
+
+  - simple message
+
+  ```json
+  {
+    "role": "user",
+    "content": "What is the capital of France?"
+  }
+  ```
+
+  - multimodal message
+
+  ```jsonc
+  {
+    "role": "user",
+    "content": [
+      { "type": "text", "text": "Read carefully all the attachments, analize the content and provide a summary for each one:" },
+      { "type": "image", "url": "https://www.example.com/image/foo.jpg" },
+      { "type": "file", "url": "https://www.example.com/pdf/bar.pdf" },
+      { "type": "file", "url": "data:plain/text;base64,CiAgICAgIF9fX19fCiAgICAgLyAgIC..." }, // base64 encoded file
+      { "type": "media", "mime_type": "plain/text", "data": "CiAgICAgIF9fX19fCiAgICAgLyAgIC..." } // google/gemini specific input format
+    ]
+  }
+  ```
+
+  > 💡 `url` can be a remote url or a base64 representation of the file: [rfc 2397](https://datatracker.ietf.org/doc/html/rfc2397).
+  Can also be used the llm/model specific input format.
+  """
   role: str
   content: Union[str, list]
 
@@ -28,12 +62,14 @@ class LlmAppToolChainSettings(BaseModel):
   provider: Optional[str] = "openai"
   model: Optional[str] = None
   temperature: Optional[float] = 0
+  outputStructure: Optional[dict] = None
 
 class LlmAppToolDbSettings(BaseModel):
   connection_string: Optional[str] = Field(None, validation_alias=AliasChoices("connectionString","connection_string"))
 
 class LlmAppTool(BaseModel):
   id: Optional[str] = None
+  thread_id: Optional[str] = Field(None, validation_alias=AliasChoices("threadId","thread_id"))
   name: str
   description: Optional[str] = None
   type: str
@@ -51,6 +87,11 @@ class LlmAppTool(BaseModel):
   vector_type: Optional[str] = Field('faiss', validation_alias=AliasChoices("vectorDbType","vector_type"))
   vector_db: Optional[str] = Field(None, validation_alias=AliasChoices("vectorDbFile","vector_db"))
   is_active: Optional[bool] = Field(True, validation_alias=AliasChoices("isActive","is_active"))
+  def secrets_to_dict(self) -> Dict[str, str]:
+      _secrets = {}
+      for d in self.secrets or []:
+        _secrets[d.get("secretId")] = d.get("secretValue")
+      return _secrets
   def get_vector_filtering(self) -> Optional[Tuple[str, List[AttributeInfo]]]:
       _description = None
       _metadata = None
@@ -74,11 +115,33 @@ class LlmAppTool(BaseModel):
       extra='allow'
   )
 
+class NebulyInteraction(BaseModel):
+  conversation_id: str = Field(..., description="Unique identifier for grouping related interactions")
+  input: str = Field(..., description="User input text in the interaction")
+  output: str = Field(..., description="LLM response shown to the user")
+  time_start: str = Field(..., description="ISO 8601 formatted start time of the LLM call")
+  time_end: str = Field(..., description="ISO 8601 formatted end time of the LLM call")
+  end_user: str = Field(..., description="Unique identifier for the end user (recommended: hashed username/email or thread_id)")
+  tags: Optional[Dict[str, str]] = Field(default=None, description="Custom key-value pairs for tagging interactions")
+
+class NebulyLLMTrace(BaseModel):
+  model: str = Field(..., description="The name of the LLM model used for the interaction")
+  messages: List[LlmMessage] = Field(..., description="List of messages exchanged during the interaction")
+  output: str = Field(..., description="The final output generated by the LLM")
+  input_tokens: Optional[int] = Field(..., description="Number of tokens in the input messages")
+  output_tokens: Optional[int] = Field(..., description="Number of tokens in the output message")
+
+class NebulyRetrievalTrace(BaseModel):
+  source: Union[str, None] = Field(..., description="The source of the retrieved documents")
+  input: str = Field(..., description="The input query used for retrieval")
+  outputs: List[str] = Field(..., description="List of retrieved document contents")
+
 #region llm public endpoints
 
 #region api
 class LlmApp(BaseModel):
   system_message: str = Field(..., validation_alias=AliasChoices("systemMessage","system_message"))
+  system_context: Optional[AgentContext] = Field(AgentContext(), validation_alias=AliasChoices("systemContext","system_context"))
   messages: List[LlmMessage]
   provider: Optional[str] = "openai"
   model: Optional[str] = None
@@ -91,6 +154,7 @@ class LlmApp(BaseModel):
   fine_tuned_model: Optional[str] = Field(None, validation_alias=AliasChoices("fineTunedModel","fine_tuned_model"))
   lang_chain_tracing: Optional[bool] = Field(False, validation_alias=AliasChoices("langChainTracing","lang_chain_tracing"))
   lang_chain_project: Optional[str] = Field(None, validation_alias=AliasChoices("langChainProject","lang_chain_project"))
+  output_structure: Optional[Dict[str, Any]] = Field(None, validation_alias=AliasChoices("outputStructure","output_structure"))
   model_config = ConfigDict(
       extra='allow'
   )
@@ -100,7 +164,7 @@ class LlmApp(BaseModel):
       return list(set(
           os.path.basename(db) for db in [self.vector_db] +
           ([self.rules.vector_db] if self.rules and self.rules.vector_db else []) +
-          [db for tool in (self.app_tools or []) for db in [tool.vector_db]]
+          [db for tool in (self.app_tools or []) for db in [tool.vector_db] if tool.is_active]
           if db is not None
       ))
   def __decompress_zip(self,zip_file_path, extract_to):
@@ -124,7 +188,7 @@ class LlmApp(BaseModel):
     for tool in self.app_tools or []:
       tool.vector_db = os.path.join(_vector_db_folder, os.path.splitext(os.path.basename(tool.vector_db))[0]) if tool.vector_db else None
   def api_key(self):
-    return self.secrets.get("openAIApiKey", self.secrets.get("apiKey", ""))
+    return self.secrets.get("apiKey", "")
   def get_llm(self) -> LlmInterface:
     return LlmManager._list[self.provider](LlmConfig(
       api_key=self.api_key(),
@@ -139,7 +203,8 @@ class InvokeRequest(LlmApp):
   mode: str
 
 class StreamRequest(LlmApp):
-  thread_id: Optional[str] = Field(None, validation_alias=AliasChoices("threadId","thread_id"))
+  thread_id: Optional[str] = Field(default=str(uuid.uuid4()), validation_alias=AliasChoices("threadId","thread_id"))
+  msg_id: Optional[str] = Field(default=str(uuid.uuid4()), validation_alias=AliasChoices("msgId","msg_id"))
 #endregion
 
 #region vector_db
@@ -159,7 +224,7 @@ class VectorDbRequest(BaseModel):
   def config(self) -> Settings:
     return config
   def api_key(self):
-    return self.secrets.get("openAIApiKey", self.secrets.get("apiKey", ""))
+    return self.secrets.get("apiKey", "")
   def out_name(self):
     if self.vector_db:
       return ".".join(self.vector_db.split(".")[:-1]) if self.vector_db.endswith(".zip") else self.vector_db

ws_bom_robot_app/llm/models/feedback.py

@@ -0,0 +1,30 @@
+from pydantic import BaseModel, Field
+
+class NebulyFeedbackAction(BaseModel):
+    """
+    FeedbackAction is a model that represents the action taken by the user
+    in response to the feedback provided by the LLM.
+    """
+    slug: str = Field("rating", description="A string identifier for the feedback action",
+             enum=["thumbs_up", "thumbs_down", "copy_input", "copy_output", "paste", "rating"])
+    text: str = Field(..., description="The text content of the feedback")
+    value: int = Field(..., description="A numeric value associated with the feedback")
+
+class NebulyFeedbackMetadata(BaseModel):
+  """
+  FeedbackMetadata is a model that represents the metadata associated with user feedback.
+  This includes information about the interaction and the user who provided feedback.
+  """
+  input: str = Field(None, description="The input of the interactions to which the action refers to")
+  output: str = Field(None, description="The output of the interactions to which the action refers to")
+  end_user: str = Field(..., description="The identifier used for the end-user")
+  timestamp: str = Field(..., description="The timestamp of the action event")
+  anonymize: bool = Field(False, description="Boolean flag. If set to true, PII will be removed from the text field")
+
+class NebulyFeedbackPayload(BaseModel):
+  """
+  NebulyFeedback is a model that combines feedback action and metadata.
+  It represents a complete feedback entry from a user interaction with the LLM.
+  """
+  action: NebulyFeedbackAction = Field(..., description="The action taken by the user as feedback")
+  metadata: NebulyFeedbackMetadata = Field(..., description="Metadata associated with the feedback")

ws_bom_robot_app/llm/nebuly_handler.py

@@ -0,0 +1,185 @@
+from typing import Union
+from ws_bom_robot_app.llm.models.api import NebulyInteraction, NebulyLLMTrace, NebulyRetrievalTrace
+from datetime import datetime, timezone
+from langchain_core.callbacks.base import AsyncCallbackHandler
+from langchain_core.messages import BaseMessage, HumanMessage, AIMessage
+from langchain_core.outputs import ChatGenerationChunk, GenerationChunk
+
+class NebulyHandler(AsyncCallbackHandler):
+    def __init__(self, llm_model: str | None, threadId: str = None, chat_history: list[BaseMessage] = [],  url: str = None, api_key: str = None, user_id: str | None = None):
+        super().__init__()
+        self.__started: bool = False
+        self.__url: str = url
+        self.__api_key: str = api_key
+        self.chat_history = chat_history
+        self.interaction = NebulyInteraction(
+            conversation_id=threadId,
+            input="",
+            output="",
+            time_start="",
+            time_end="",
+            end_user= user_id if user_id and user_id != "" else threadId,
+            tags={"model": llm_model},
+        )
+        self.llm_trace = NebulyLLMTrace(
+            model=llm_model,
+            messages=[],
+            output="",
+            input_tokens=0,
+            output_tokens=0,
+        )
+        self.__response_with_rag: str = "false" # Flag to check if the AI used some retrieval tools
+        self.__retrieval_query: str = ""
+        self.retrieval_traces: list[NebulyRetrievalTrace] = []
+
+    async def on_chat_model_start(self, serialized, messages, *, run_id, parent_run_id = None, tags = None, metadata = None, **kwargs):
+        # Initialize the interaction with the input message
+        if not self.__started:
+          message_list = self.__flat_messages(messages)
+          if isinstance(message_list[-1], HumanMessage):
+            if isinstance(message_list[-1].content, list):
+              self.interaction.input = self.__parse_multimodal_input(message_list[-1].content)
+            else:
+              self.interaction.input = message_list[-1].content
+              self.interaction.tags["generated"] = self.__is_message_generated(message_list)
+          else:
+            raise ValueError("Last message is not a HumanMessage")
+          self.interaction.time_start = datetime.now().astimezone().isoformat()
+          self.__started = True
+
+    async def on_llm_end(self, response, *, run_id, parent_run_id = None, tags = None, **kwargs):
+        generation: Union[ChatGenerationChunk, GenerationChunk] = response.generations[0]
+        usage_metadata: dict = generation[0].message.usage_metadata
+        self.llm_trace.input_tokens = usage_metadata.get("input_tokens", 0)
+        self.llm_trace.output_tokens = usage_metadata.get("output_tokens", 0)
+
+    async def on_retriever_start(self, serialized, query, *, run_id, parent_run_id = None, tags = None, metadata = None, **kwargs):
+        self.__retrieval_query = query
+
+
+    async def on_retriever_end(self, documents, *, run_id, parent_run_id = None, tags = None, **kwargs):
+        # pass the document source because of the large amount of data in the document content
+        for doc in documents:
+          self.retrieval_traces.append(
+            NebulyRetrievalTrace(
+              source=doc.metadata.get("source", "content unavailable"),
+              input=self.__retrieval_query,
+              outputs=[doc.metadata.get("source", "content unavailable")]
+            )
+          )
+
+    async def on_tool_start(self, serialized, input_str, *, run_id, parent_run_id = None, tags = None, metadata = None, inputs = None, **kwargs):
+        self.__response_with_rag = "true"  # Set the flag to true when the retriever starts
+
+    async def on_agent_finish(self, finish, *, run_id, parent_run_id = None, tags = None, **kwargs):
+        # Interaction
+        self.interaction.output = finish.return_values["output"]
+        # Trace
+        self.llm_trace.output = finish.return_values["output"]
+        message_history =  self._convert_to_json_format(self.chat_history)
+        self.llm_trace.messages = self.__parse_multimodal_history(message_history)
+        await self.__send_interaction()
+
+    def __flat_messages(self, messages: list[list[BaseMessage]], to_json: bool = False) -> list[BaseMessage]:
+      """
+      Maps the messages to the format expected by the LLM.
+      Flattens the nested list structure of messages.
+      """
+      # Flatten the nested list structure
+      flattened_messages = []
+      for message_list in messages:
+        flattened_messages.extend(message_list)
+      # Store JSON format in LLM trace
+      if to_json:
+        return self._convert_to_json_format(flattened_messages)
+      return flattened_messages
+
+    def _convert_to_json_format(self, messages: list[BaseMessage]) -> list[dict]:
+      """Converts BaseMessage objects to JSON format with role and content."""
+      result = []
+      for message in messages:
+        if isinstance(message, HumanMessage):
+          role = "user"
+        elif isinstance(message, AIMessage):
+          role = "assistant"
+        else:
+          role = "system"
+
+        result.append({
+          "role": role,
+          "content": message.content
+        })
+      return result
+
+    async def __send_interaction(self):
+        # Send the interaction to the server
+        from urllib.parse import urljoin
+        import requests
+
+        payload = self.__prepare_payload()
+        endpoint = urljoin(self.__url, "event-ingestion/api/v2/events/trace_interaction")
+        # Prepare headers with authentication
+        headers = {"Content-Type": "application/json"}
+        if self.__api_key:
+          headers["Authorization"] = f"Bearer {self.__api_key}"
+        response = requests.post(
+          url=endpoint,
+          json=payload,
+          headers=headers
+        )
+        if response.status_code != 200:
+          print(f"Failed to send interaction: {response.status_code} {response.text}")
+
+    def __prepare_payload(self):
+        self.interaction.time_end = datetime.now().astimezone().isoformat()
+        self.interaction.tags["response_with_rag"] = self.__response_with_rag
+        payload = {
+            "interaction": self.interaction.__dict__,
+            "traces": [
+              self.llm_trace.__dict__,
+            ]
+        }
+        for trace in self.retrieval_traces:
+          if trace.source:
+              payload["traces"].append(trace.__dict__)
+        return payload
+
+    def __parse_multimodal_input(self, input: list[dict]) -> str:
+      """Parse multimodal input and return a string representation."""
+      type_mapping = {
+        "text": lambda item: item.get("text", ""),
+        "image": lambda _: " <image>",
+        "image_url": lambda _: " <image>",
+        "file": lambda _: " <file>",
+        "media": lambda _: " <file>",
+        "document": lambda _: " <file>",
+      }
+
+      return "".join(
+        type_mapping.get(item.get("type", ""), lambda item: f" <{item.get('type', '')}>")
+        (item) for item in input
+      )
+
+    def __parse_multimodal_history(self, messages: list[dict]) -> list[dict]:
+        # Parse the multimodal history and return a list of dictionaries
+        parsed_history = []
+        for message in messages:
+            if isinstance(message["content"], list):
+              parsed_content = self.__parse_multimodal_input(message["content"])
+            else:
+              parsed_content = message["content"]
+            parsed_history.append({
+          "role": message["role"],
+          "content": parsed_content
+            })
+        return parsed_history
+
+    def __is_message_generated(self, messages: list[BaseMessage]) -> bool:
+        # Check if the last message is generated by the model
+        if len(messages) == 0:
+            return False
+        last_user_message = f'<div class="llm__pill">{messages[-1].content}</div>'
+        last_ai_message = messages[-2].content
+        if last_user_message in last_ai_message:
+            return "true"
+        return "false"