ws-bom-robot-app 0.0.63__py3-none-any.whl → 0.0.103__py3-none-any.whl

ws_bom_robot_app/llm/evaluator.py

@@ -0,0 +1,319 @@
+from uuid import UUID
+import requests, base64
+from typing import Iterator, Optional, List, Union
+from enum import Enum
+from ws_bom_robot_app.config import config
+from ws_bom_robot_app.llm.models.api import LlmMessage, StreamRequest
+from langsmith import Client, traceable
+from langsmith.schemas import Dataset, Example, Feedback, Run
+from openevals.llm import create_llm_as_judge
+from openevals.prompts import CORRECTNESS_PROMPT, RAG_HELPFULNESS_PROMPT, CONCISENESS_PROMPT, RAG_GROUNDEDNESS_PROMPT, HALLUCINATION_PROMPT
+from pydantic import BaseModel
+
+ls_client = Client()
+
+class EvaluatorType(Enum):
+    """Available evaluator types"""
+    CORRECTNESS = "correctness"
+    HELPFULNESS = "helpfulness"
+    CONCISENESS = "conciseness"
+    RAG_GROUNDEDNESS = "rag_groundedness"
+    RAG_HALLUCINATION = "rag_hallucination"
+
+    @classmethod
+    def all(cls) -> List['EvaluatorType']:
+        """Get all available evaluator types"""
+        return list(cls)
+
+    @classmethod
+    def default(cls) -> List['EvaluatorType']:
+        """Get default evaluator types"""
+        return [cls.CORRECTNESS]
+
+class EvaluatorDataSets:
+
+    @classmethod
+    def all(cls) -> List[Dataset]:
+        return list(ls_client.list_datasets())
+    @classmethod
+    def find(cls, name: str) -> List[Dataset]:
+        return [d for d in cls.all() if d.name.lower().__contains__(name.lower())]
+    @classmethod
+    def get(cls, id: Union[str, UUID]) -> Optional[Dataset]:
+        return next((d for d in cls.all() if str(d.id) == str(id)), None)
+    @classmethod
+    def create(cls, name: str) -> Dataset:
+        return ls_client.create_dataset(name=name)
+    @classmethod
+    def delete(cls, id: str) -> None:
+        ls_client.delete_dataset(id=id)
+    @classmethod
+    def example(cls, id: str) -> List[Example]:
+        return list(ls_client.list_examples(dataset_id=id, include_attachments=True))
+    @classmethod
+    def add_example(cls, dataset_id: str, inputs: dict, outputs: dict) -> Example:
+        """Add an example to the dataset.
+        Args:
+            inputs (dict): The input data for the example.
+            outputs (dict): The output data for the example.
+        Sample:
+            - inputs: {"question": "What is the capital of France?"}
+              outputs: {"answer": "Paris"}
+        """
+        return ls_client.create_example(dataset_id=dataset_id, inputs=inputs, outputs=outputs)
+    @classmethod
+    def feedback(cls, experiment_name: str) -> Iterator[Feedback]:
+        return ls_client.list_feedback(
+            run_ids=[r.id for r in ls_client.list_runs(project_name=experiment_name)]
+        )
+
+class Evaluator:
+    def __init__(self, rq: StreamRequest, data: Union[Dataset,List[Example]], judge_model: Optional[str] = None):
+        """Evaluator class for assessing model performance.
+
+        Args:
+            rq (StreamRequest): The request object containing input data.
+            data (Union[Dataset, List[Example]]): The dataset to use for evaluation or a list of examples.
+            judge_model (Optional[str], optional): The model to use for evaluation, defaults to "openai:o4-mini".
+              For a list of available models, see the LangChain documentation:
+              https://python.langchain.com/api_reference/langchain/chat_models/langchain.chat_models.base.init_chat_model.html
+        """
+        self.judge_model: str = judge_model or "openai:o4-mini"
+        self.data = data
+        self.rq: StreamRequest = rq
+
+    #region evaluators
+
+    def _get_evaluator_function(self, evaluator_type: EvaluatorType):
+        """Get the evaluator function for a given type"""
+        evaluator_map = {
+            EvaluatorType.CORRECTNESS: self.correctness_evaluator,
+            EvaluatorType.HELPFULNESS: self.helpfulness_evaluator,
+            EvaluatorType.CONCISENESS: self.conciseness_evaluator,
+            EvaluatorType.RAG_GROUNDEDNESS: self.rag_groundedness_evaluator,
+            EvaluatorType.RAG_HALLUCINATION: self.rag_hallucination_evaluator,
+        }
+        return evaluator_map.get(evaluator_type)
+
+    def correctness_evaluator(self, inputs: dict, outputs: dict, reference_outputs: dict):
+        evaluator = create_llm_as_judge(
+          prompt=CORRECTNESS_PROMPT,
+          feedback_key="correctness",
+          model=self.judge_model,
+          continuous=True,
+          choices=[i/10 for i in range(11)]
+        )
+        return evaluator(
+            inputs=inputs,
+            outputs=outputs,
+            reference_outputs=reference_outputs
+        )
+
+    def helpfulness_evaluator(self, inputs: dict, outputs: dict):
+        evaluator = create_llm_as_judge(
+            prompt=RAG_HELPFULNESS_PROMPT,
+            feedback_key="helpfulness",
+            model=self.judge_model,
+            continuous=True,
+            choices=[i/10 for i in range(11)]
+        )
+        return evaluator(
+            inputs=inputs,
+            outputs=outputs,
+        )
+
+    def conciseness_evaluator(self, inputs: dict, outputs: dict, reference_outputs: dict):
+        evaluator = create_llm_as_judge(
+            prompt=CONCISENESS_PROMPT,
+            feedback_key="conciseness",
+            model=self.judge_model,
+            continuous=True,
+            choices=[i/10 for i in range(11)]
+        )
+        return evaluator(
+            inputs=inputs,
+            outputs=outputs,
+            reference_outputs=reference_outputs
+        )
+
+    def _find_retrievers(self, run: Run) -> List[Run]:
+      retrievers = []
+      for child in getattr(run, "child_runs", []):
+        if child.run_type == "retriever":
+          retrievers.append(child)
+        retrievers.extend(self._find_retrievers(child))
+      return retrievers
+
+    def _retriever_documents(self, retrievers_run: List[Run]) -> str:
+      unique_contents = set()
+      for r in retrievers_run:
+        for doc in r.outputs.get("documents", []):
+          unique_contents.add(doc.page_content)
+      return "\n\n".join(unique_contents)
+
+    def rag_groundedness_evaluator(self, run: Run):
+        evaluator = create_llm_as_judge(
+            prompt=RAG_GROUNDEDNESS_PROMPT,
+            feedback_key="rag_groundedness",
+            model=self.judge_model,
+            continuous=True,
+            choices=[i/10 for i in range(11)]
+        )
+        retrievers_run = self._find_retrievers(run)
+        if retrievers_run:
+            try:
+                return evaluator(
+                    outputs=run.outputs["answer"],
+                    context=self._retriever_documents(retrievers_run)
+                )
+            except Exception as e:
+                return 0.0
+        else:
+            return 0.0
+
+    def rag_hallucination_evaluator(self, inputs: dict, outputs: dict, reference_outputs: dict, run: Run):
+        evaluator = create_llm_as_judge(
+            prompt=HALLUCINATION_PROMPT,
+            feedback_key="rag_hallucination",
+            model=self.judge_model,
+            continuous=True,
+            choices=[i/10 for i in range(11)]
+        )
+        retrievers_run = self._find_retrievers(run)
+        if retrievers_run:
+            try:
+                return evaluator(
+                    inputs=inputs['question'],
+                    outputs=outputs['answer'],
+                    reference_outputs=reference_outputs['answer'],
+                    context=self._retriever_documents(retrievers_run)
+                )
+            except Exception as e:
+                return 0.0
+        else:
+            return 0.0
+
+    #endregion evaluators
+
+    #region target
+    def _parse_rq(self, inputs: dict, attachments: dict) -> StreamRequest:
+        _rq = self.rq.__deepcopy__()
+        if not attachments is None and len(attachments) > 0:
+            _content = []
+            _content.append({"type": "text", "text": inputs["question"]})
+            for k,v in attachments.items():
+                if isinstance(v, dict):
+                    _content.append({"type": ("image" if "image" in v.get("mime_type","") else "file"), "url": v.get("presigned_url","")})
+            _rq.messages = [LlmMessage(role="user", content=_content)]
+        else:
+            _rq.messages = [LlmMessage(role="user", content=inputs["question"])]
+        return _rq
+
+    @traceable(run_type="chain",name="stream_internal")
+    async def target_internal(self,inputs: dict, attachments: dict) -> dict:
+      from ws_bom_robot_app.llm.main import stream
+      from unittest.mock import Mock
+      from fastapi import Request
+      _ctx = Mock(spec=Request)
+      _ctx.base_url.return_value = "http://evaluator"
+      _rq = self._parse_rq(inputs, attachments)
+      _chunks = []
+      async for chunk in stream(rq=_rq, ctx=_ctx, formatted=False):
+          _chunks.append(chunk)
+      _content = ''.join(_chunks) if _chunks else ""
+      del _rq, _chunks
+      return { "answer": _content.strip() }
+
+    @traceable(run_type="chain",name="stream_http")
+    async def target_http(self,inputs: dict, attachments: dict) -> dict:
+      _rq = self._parse_rq(inputs, attachments)
+      _host= "http://localhost:6001"
+      _endpoint = f"{_host}/api/llm/stream/raw"
+      _robot_auth =f"Basic {base64.b64encode((config.robot_user + ':' + config.robot_password).encode('utf-8')).decode('utf-8')}"
+      _rs = requests.post(_endpoint, data=_rq.model_dump_json(), stream=True, headers={"Authorization": _robot_auth}, verify=True)
+      _content = ''.join([chunk.decode('utf-8') for chunk in _rs.iter_content(chunk_size=1024, decode_unicode=False)])
+      del _rq, _rs
+      return { "answer": _content.strip() }
+    #endregion target
+
+    async def run(self,
+                  evaluators: Optional[List[EvaluatorType]] = None,
+                  target_method: str = "target_internal") -> dict:
+        """Run evaluation with specified evaluators
+
+        Args:
+            evaluators: List of evaluator types to use. If None, uses default (correctness only)
+            target_method: Method to use for target evaluation ("target_internal" or "target")
+
+        Returns:
+            dict: Evaluation results with scores
+
+        Usage:
+          ```
+          await evaluator.run()  # Uses default (correctness only)
+          await evaluator.run([EvaluatorType.CORRECTNESS, EvaluatorType.HELPFULNESS])
+          await evaluator.run(EvaluatorType.all())  # Uses all available evaluators
+          ```
+        """
+        try:
+          # evaluator functions
+          evaluator_functions = []
+          if evaluators is None:
+              evaluators = EvaluatorType.default()
+          for eval_type in evaluators:
+              func = self._get_evaluator_function(eval_type)
+              if func:
+                  evaluator_functions.append(func)
+              else:
+                  print(f"Warning: Unknown evaluator type: {eval_type}")
+          if not evaluator_functions:
+              print("No valid evaluators provided, using default (correctness)")
+              evaluator_functions = [self.correctness_evaluator]
+
+          # target method
+          target_func = getattr(self, target_method, self.target_internal)
+
+          # run
+          _dataset: Dataset = self.data if isinstance(self.data, Dataset) else EvaluatorDataSets.get(self.data[0].dataset_id)
+          experiment = await ls_client.aevaluate(
+              target_func,
+              data=_dataset.name if isinstance(self.data, Dataset) else self.data,
+              evaluators=evaluator_functions,
+              experiment_prefix=_dataset.name,
+              upload_results=True,
+              max_concurrency=4,
+              metadata={
+                  "app": _dataset.name,
+                  "model": f"{self.rq.provider}:{self.rq.model}",
+                  "judge": self.judge_model,
+                  "evaluators": [e.value for e in evaluators]
+              }
+          )
+          feedback = list(EvaluatorDataSets.feedback(experiment.experiment_name))
+          scores = [f.score for f in feedback]
+          url = f"{ls_client._host_url}/o/{ls_client._tenant_id}/datasets/{_dataset.id}/compare?selectedSessions={feedback[0].session_id}"
+
+          # group scores by evaluator type
+          evaluator_scores = {}
+          for i, eval_type in enumerate(evaluators):
+              eval_scores = [f.score for f in feedback if f.key.lower() == eval_type.value.lower()]
+              if eval_scores:
+                  evaluator_scores[eval_type.value] = sum(eval_scores) / len(eval_scores)
+
+          return {
+              "experiment": {"name": experiment.experiment_name, "url": url},
+              "overall_score": sum(scores) / len(scores) if scores else 0,
+              "evaluator_scores": evaluator_scores
+          }
+        except Exception as e:
+            from traceback import print_exc
+            print(f"Error occurred during evaluation: {e}")
+            print_exc()
+            return {"error": str(e)}
+
+class EvaluatorRunRequest(BaseModel):
+    dataset: dict
+    rq: StreamRequest
+    example: Optional[List[dict]] = None
+    evaluators: Optional[List[str]] = None
+    judge: Optional[str] = None

ws_bom_robot_app/llm/main.py

@@ -3,7 +3,7 @@ import asyncio, json, logging, os, traceback, re
 from fastapi import Request
 from langchain.callbacks.tracers import LangChainTracer
 from langchain_core.callbacks.base import AsyncCallbackHandler
-from langchain_core.messages import AIMessage, HumanMessage
+from langchain_core.messages import BaseMessage, AIMessage, HumanMessage
 from langsmith import Client as LangSmithClient
 from typing import AsyncGenerator, List
 from ws_bom_robot_app.config import config
@@ -14,7 +14,6 @@ from ws_bom_robot_app.llm.models.api import InvokeRequest, StreamRequest
 from ws_bom_robot_app.llm.providers.llm_manager import LlmInterface
 from ws_bom_robot_app.llm.tools.tool_builder import get_structured_tools
 from ws_bom_robot_app.llm.nebuly_handler import NebulyHandler
-import ws_bom_robot_app.llm.settings as settings
 
 async def invoke(rq: InvokeRequest) -> str:
   await rq.initialize()
@@ -40,21 +39,30 @@ def _parse_formatted_message(message: str) -> str:
   except:
     result = message
   return result
-async def __stream(rq: StreamRequest, ctx: Request, queue: Queue,formatted: bool = True) -> None:
-  await rq.initialize()
+
+async def __stream(rq: StreamRequest, ctx: Request, queue: Queue, formatted: bool = True) -> None:
   #os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
-  if formatted:
-    agent_handler = AgentHandler(queue,rq.provider,rq.thread_id)
-  else:
-    agent_handler = RawAgentHandler(queue,rq.provider)
-  os.environ["AGENT_HANDLER_FORMATTED"] = str(formatted)
-  callbacks: List[AsyncCallbackHandler] = [agent_handler]
-  settings.init()
 
-  #CREATION OF CHAT HISTORY FOR AGENT
+  # rq initialization
+  await rq.initialize()
+  for tool in rq.app_tools:
+    tool.thread_id = rq.thread_id
+
+  #llm
+  __llm: LlmInterface = rq.get_llm()
+
+  #chat history
+  chat_history: list[BaseMessage] = []
   for message in rq.messages:
     if message.role in ["human","user"]:
-      settings.chat_history.append(HumanMessage(content=message.content))
+      _content = message.content
+      # multimodal content parsing
+      if isinstance(_content, list):
+        try:
+          _content = await __llm.format_multimodal_content(_content)
+        except Exception as e:
+          logging.warning(f"Error parsing multimodal content {_content[:100]}: {e}")
+      chat_history.append(HumanMessage(content=_content))
     elif message.role in ["ai","assistant"]:
       message_content = ""
       if formatted:
@@ -79,37 +87,52 @@ async def __stream(rq: StreamRequest, ctx: Request, queue: Queue,formatted: bool
       else:
         message_content = message.content
       if message_content:
-        settings.chat_history.append(AIMessage(content=message_content))
+        chat_history.append(AIMessage(content=message_content))
+
+
+  #agent handler
+  if formatted:
+    agent_handler = AgentHandler(queue, rq.provider, rq.thread_id)
+  else:
+    agent_handler = RawAgentHandler(queue, rq.provider)
+  #TODO: move from os.environ to rq
+  os.environ["AGENT_HANDLER_FORMATTED"] = str(formatted)
 
+  #callbacks
+  ## agent
+  callbacks: List[AsyncCallbackHandler] = [agent_handler]
+  ## langchain tracing
   if rq.lang_chain_tracing:
     client = LangSmithClient(
       api_key= rq.secrets.get("langChainApiKey", "")
     )
-    trace = LangChainTracer(project_name=rq.lang_chain_project,client=client,tags=[str(ctx.base_url)])
+    trace = LangChainTracer(project_name=rq.lang_chain_project,client=client,tags=[str(ctx.base_url) if ctx else ''])
     callbacks.append(trace)
-
-  __llm: LlmInterface =rq.get_llm()
-  for tool in rq.app_tools:
-    tool.thread_id = rq.thread_id
-  processor = AgentLcel(
-      llm=__llm,
-      sys_message=rq.system_message,
-      sys_context=rq.system_context,
-      tools=get_structured_tools(__llm, tools=rq.app_tools, callbacks=[callbacks], queue=queue),
-      rules=rq.rules
-  )
+  ## nebuly tracing
   if rq.secrets.get("nebulyApiKey","") != "":
+    user_id = rq.system_context.user.id if rq.system_context and rq.system_context.user and rq.system_context.user.id else None
     nebuly_callback = NebulyHandler(
         llm_model=__llm.config.model,
         threadId=rq.thread_id,
+        chat_history=chat_history,
         url=config.NEBULY_API_URL,
         api_key=rq.secrets.get("nebulyApiKey", None),
+        user_id=user_id
       )
     callbacks.append(nebuly_callback)
 
+  # chain
+  processor = AgentLcel(
+      llm=__llm,
+      sys_message=rq.system_message,
+      sys_context=rq.system_context,
+      tools=get_structured_tools(__llm, tools=rq.app_tools, callbacks=[callbacks], queue=queue),
+      rules=rq.rules,
+      json_schema=rq.output_structure.get("outputFormat") if rq.output_structure and rq.output_structure.get("outputType") == "json" else None
+  )
   try:
     await processor.executor.ainvoke(
-        {"chat_history": settings.chat_history},
+        {"chat_history": chat_history},
         {"callbacks": callbacks},
     )
   except Exception as e:
@@ -120,7 +143,7 @@ async def __stream(rq: StreamRequest, ctx: Request, queue: Queue,formatted: bool
       await queue.put(_error)
     await queue.put(None)
 
-  # Signal the end of streaming
+  # signal the end of streaming
   await queue.put(None)
 
 async def stream(rq: StreamRequest, ctx: Request, formatted: bool = True) -> AsyncGenerator[str, None]:

ws_bom_robot_app/llm/models/api.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Optional, Tuple, Union
+from typing import List, Dict, Optional, Tuple, Union, Any
 from datetime import datetime
 from pydantic import AliasChoices, BaseModel, Field, ConfigDict
 from langchain_core.embeddings import Embeddings
@@ -11,6 +11,39 @@ import os, shutil, uuid
 from ws_bom_robot_app.config import Settings, config
 
 class LlmMessage(BaseModel):
+  """
+  💬 multimodal chat
+
+  The multimodal message allows users to interact with the application using both text and media files.
+  `robot` accept multimodal input in a uniform way, regarding the llm provider used.
+
+  - simple message
+
+  ```json
+  {
+    "role": "user",
+    "content": "What is the capital of France?"
+  }
+  ```
+
+  - multimodal message
+
+  ```jsonc
+  {
+    "role": "user",
+    "content": [
+      { "type": "text", "text": "Read carefully all the attachments, analize the content and provide a summary for each one:" },
+      { "type": "image", "url": "https://www.example.com/image/foo.jpg" },
+      { "type": "file", "url": "https://www.example.com/pdf/bar.pdf" },
+      { "type": "file", "url": "data:plain/text;base64,CiAgICAgIF9fX19fCiAgICAgLyAgIC..." }, // base64 encoded file
+      { "type": "media", "mime_type": "plain/text", "data": "CiAgICAgIF9fX19fCiAgICAgLyAgIC..." } // google/gemini specific input format
+    ]
+  }
+  ```
+
+  > 💡 `url` can be a remote url or a base64 representation of the file: [rfc 2397](https://datatracker.ietf.org/doc/html/rfc2397).
+  Can also be used the llm/model specific input format.
+  """
   role: str
   content: Union[str, list]
 
@@ -121,6 +154,7 @@ class LlmApp(BaseModel):
   fine_tuned_model: Optional[str] = Field(None, validation_alias=AliasChoices("fineTunedModel","fine_tuned_model"))
   lang_chain_tracing: Optional[bool] = Field(False, validation_alias=AliasChoices("langChainTracing","lang_chain_tracing"))
   lang_chain_project: Optional[str] = Field(None, validation_alias=AliasChoices("langChainProject","lang_chain_project"))
+  output_structure: Optional[Dict[str, Any]] = Field(None, validation_alias=AliasChoices("outputStructure","output_structure"))
   model_config = ConfigDict(
       extra='allow'
   )
@@ -130,7 +164,7 @@ class LlmApp(BaseModel):
       return list(set(
           os.path.basename(db) for db in [self.vector_db] +
           ([self.rules.vector_db] if self.rules and self.rules.vector_db else []) +
-          [db for tool in (self.app_tools or []) for db in [tool.vector_db]]
+          [db for tool in (self.app_tools or []) for db in [tool.vector_db] if tool.is_active]
           if db is not None
       ))
   def __decompress_zip(self,zip_file_path, extract_to):
@@ -154,7 +188,7 @@ class LlmApp(BaseModel):
     for tool in self.app_tools or []:
       tool.vector_db = os.path.join(_vector_db_folder, os.path.splitext(os.path.basename(tool.vector_db))[0]) if tool.vector_db else None
   def api_key(self):
-    return self.secrets.get("openAIApiKey", self.secrets.get("apiKey", ""))
+    return self.secrets.get("apiKey", "")
   def get_llm(self) -> LlmInterface:
     return LlmManager._list[self.provider](LlmConfig(
       api_key=self.api_key(),
@@ -169,8 +203,8 @@ class InvokeRequest(LlmApp):
   mode: str
 
 class StreamRequest(LlmApp):
-  thread_id: Optional[str] = Field(None, validation_alias=AliasChoices("threadId","thread_id"))
-  msg_id: Optional[str] = Field(None, validation_alias=AliasChoices("msgId","msg_id"))
+  thread_id: Optional[str] = Field(default=str(uuid.uuid4()), validation_alias=AliasChoices("threadId","thread_id"))
+  msg_id: Optional[str] = Field(default=str(uuid.uuid4()), validation_alias=AliasChoices("msgId","msg_id"))
 #endregion
 
 #region vector_db
@@ -190,7 +224,7 @@ class VectorDbRequest(BaseModel):
   def config(self) -> Settings:
     return config
   def api_key(self):
-    return self.secrets.get("openAIApiKey", self.secrets.get("apiKey", ""))
+    return self.secrets.get("apiKey", "")
   def out_name(self):
     if self.vector_db:
       return ".".join(self.vector_db.split(".")[:-1]) if self.vector_db.endswith(".zip") else self.vector_db

ws_bom_robot_app/llm/nebuly_handler.py

@@ -2,24 +2,23 @@ from typing import Union
 from ws_bom_robot_app.llm.models.api import NebulyInteraction, NebulyLLMTrace, NebulyRetrievalTrace
 from datetime import datetime, timezone
 from langchain_core.callbacks.base import AsyncCallbackHandler
-import ws_bom_robot_app.llm.settings as settings
 from langchain_core.messages import BaseMessage, HumanMessage, AIMessage
 from langchain_core.outputs import ChatGenerationChunk, GenerationChunk
-from uuid import UUID
 
 class NebulyHandler(AsyncCallbackHandler):
-    def __init__(self, llm_model: str | None, threadId: str = None, url: str = None, api_key: str = None):
+    def __init__(self, llm_model: str | None, threadId: str = None, chat_history: list[BaseMessage] = [],  url: str = None, api_key: str = None, user_id: str | None = None):
         super().__init__()
         self.__started: bool = False
         self.__url: str = url
         self.__api_key: str = api_key
+        self.chat_history = chat_history
         self.interaction = NebulyInteraction(
             conversation_id=threadId,
             input="",
             output="",
             time_start="",
             time_end="",
-            end_user=threadId,
+            end_user= user_id if user_id and user_id != "" else threadId,
             tags={"model": llm_model},
         )
         self.llm_trace = NebulyLLMTrace(
@@ -77,7 +76,7 @@ class NebulyHandler(AsyncCallbackHandler):
         self.interaction.output = finish.return_values["output"]
         # Trace
         self.llm_trace.output = finish.return_values["output"]
-        message_history =  self._convert_to_json_format(settings.chat_history)[:-1]
+        message_history =  self._convert_to_json_format(self.chat_history)
         self.llm_trace.messages = self.__parse_multimodal_history(message_history)
         await self.__send_interaction()
 
@@ -146,16 +145,20 @@ class NebulyHandler(AsyncCallbackHandler):
         return payload
 
     def __parse_multimodal_input(self, input: list[dict]) -> str:
-        # Parse the multimodal input and return a string representation
-        # This is a placeholder implementation, you can customize it as needed
-        parsed_input = ""
-        for item in input:
-            if item.get("type") == "text":
-                parsed_input += item.get("text", "")
-            elif item.get("type") == "image_url":
-                parsed_input += " <image>"
-        print(parsed_input)
-        return parsed_input
+      """Parse multimodal input and return a string representation."""
+      type_mapping = {
+        "text": lambda item: item.get("text", ""),
+        "image": lambda _: " <image>",
+        "image_url": lambda _: " <image>",
+        "file": lambda _: " <file>",
+        "media": lambda _: " <file>",
+        "document": lambda _: " <file>",
+      }
+
+      return "".join(
+        type_mapping.get(item.get("type", ""), lambda item: f" <{item.get('type', '')}>")
+        (item) for item in input
+      )
 
     def __parse_multimodal_history(self, messages: list[dict]) -> list[dict]:
         # Parse the multimodal history and return a list of dictionaries