aiagents4pharma 1.46.5__tar.gz → 1.48.0__tar.gz

{aiagents4pharma-1.46.5 → aiagents4pharma-1.48.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aiagents4pharma
-Version: 1.46.5
+Version: 1.48.0
 Summary: AI Agents for drug discovery, drug development, and other pharmaceutical R&D.
 License-File: LICENSE
 Classifier: License :: OSI Approved :: MIT License

{aiagents4pharma-1.46.5 → aiagents4pharma-1.48.0}/aiagents4pharma/talk2aiagents4pharma/Dockerfile

@@ -58,6 +58,7 @@ LABEL description="AI Agents for Pharma - Streamlit Application"
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
   ca-certificates \
   curl \
+  gnupg \
   libmagic1 \
   libopenblas0 \
   libomp5 \
@@ -66,6 +67,23 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
   && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
   && update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1
 
+# Install CUDA runtime libraries required by cudf/cupy
+RUN curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/3bf863cc.pub \
+    | gpg --dearmor -o /usr/share/keyrings/nvidia-cuda-keyring.gpg \
+  && echo "deb [signed-by=/usr/share/keyrings/nvidia-cuda-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/ /" \
+    > /etc/apt/sources.list.d/nvidia-cuda.list \
+  && apt-get update \
+  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    cuda-cudart-12-6 \
+    cuda-cudart-dev-12-6 \
+    cuda-nvrtc-12-6 \
+    cuda-nvrtc-dev-12-6 \
+    libcublas-12-6 \
+    libcusparse-12-6 \
+  && rm -rf /var/lib/apt/lists/*
+
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64
+
 # Copy UV virtual environment from build stage
 COPY --from=uv-install /opt/venv /opt/venv
 

{aiagents4pharma-1.46.5/aiagents4pharma/talk2knowledgegraphs → aiagents4pharma-1.48.0/aiagents4pharma/talk2aiagents4pharma}/docker-compose/cpu/.env.example

@@ -15,7 +15,7 @@ MILVUS_HOST=localhost
 MILVUS_PORT=19530
 MILVUS_USER=root
 MILVUS_PASSWORD=Milvus
-MILVUS_DATABASE=your_database_name_here
+MILVUS_DATABASE=t2kg_primekg
 
 # Specify the data directory for multimodal data to your own data directory
 # DATA_DIR=/your_absolute_path_to_your_data_dir/

{aiagents4pharma-1.46.5/aiagents4pharma/talk2aiagents4pharma/docker-compose/cpu → aiagents4pharma-1.48.0/aiagents4pharma/talk2aiagents4pharma/docker-compose/gpu}/.env.example

@@ -15,7 +15,7 @@ MILVUS_HOST=localhost
 MILVUS_PORT=19530
 MILVUS_USER=root
 MILVUS_PASSWORD=Milvus
-MILVUS_DATABASE=your_database_name_here
+MILVUS_DATABASE=t2kg_primekg
 
 # Specify the data directory for multimodal data to your own data directory
 # DATA_DIR=/your_absolute_path_to_your_data_dir/

{aiagents4pharma-1.46.5 → aiagents4pharma-1.48.0}/aiagents4pharma/talk2biomodels/agents/t2b_agent.py

@@ -20,6 +20,7 @@ from ..tools.get_annotation import GetAnnotationTool
 from ..tools.get_modelinfo import GetModelInfoTool
 from ..tools.parameter_scan import ParameterScanTool
 from ..tools.query_article import QueryArticle
+from ..tools.save_model import SaveModelTool
 from ..tools.search_models import SearchModelsTool
 from ..tools.simulate_model import SimulateModelTool
 from ..tools.steady_state import SteadyStateTool
@@ -54,6 +55,7 @@ def get_app(uniq_id, llm_model: BaseChatModel):
             ParameterScanTool(),
             GetAnnotationTool(),
             QueryArticle(),
+            SaveModelTool(),
         ]
     )
 

{aiagents4pharma-1.46.5 → aiagents4pharma-1.48.0}/aiagents4pharma/talk2biomodels/states/state_talk2biomodels.py

@@ -42,6 +42,7 @@ class Talk2Biomodels(AgentState):
     # https://langchain-ai.github.io/langgraph/troubleshooting/errors/INVALID_CONCURRENT_GRAPH_UPDATE/
     model_id: Annotated[list, operator.add]
     sbml_file_path: Annotated[list, operator.add]
+    model_as_string: Annotated[list, operator.add]
     dic_simulated_data: Annotated[list[dict], add_data]
     dic_scanned_data: Annotated[list[dict], add_data]
     dic_steady_state_data: Annotated[list[dict], add_data]

aiagents4pharma-1.48.0/aiagents4pharma/talk2biomodels/tests/test_save_model.py

@@ -0,0 +1,47 @@
+"""
+Test cases for Talk2Biomodels.
+"""
+
+import tempfile
+
+from langchain_core.messages import HumanMessage
+from langchain_openai import ChatOpenAI
+
+from ..agents.t2b_agent import get_app
+
+LLM_MODEL = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+
+
+def test_save_model_tool():
+    """
+    Test the save_model tool.
+    """
+    unique_id = 123
+    app = get_app(unique_id, llm_model=LLM_MODEL)
+    config = {"configurable": {"thread_id": unique_id}}
+    # Simulate a model
+    prompt = "Simulate model 64"
+    # Invoke the agent
+    app.invoke({"messages": [HumanMessage(content=prompt)]}, config=config)
+    current_state = app.get_state(config)
+    assert current_state.values["model_as_string"][-1] is not None
+    # Save a model without simulating
+    prompt = "Save the model"
+    # Invoke the agent
+    app.invoke({"messages": [HumanMessage(content=prompt)]}, config=config)
+    current_state = app.get_state(config)
+    assert current_state.values["model_as_string"][-1] is not None
+    # Create a temporary directory to save the model
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Save a model to the temporary directory
+        prompt = f"Simulate model 64 and save it model at {temp_dir}"
+        # Invoke the agent
+        app.invoke({"messages": [HumanMessage(content=prompt)]}, config=config)
+        current_state = app.get_state(config)
+        assert current_state.values["model_as_string"][-1] is not None
+    # Simulate and save a model in non-existing path
+    prompt = "Simulate model 64 and then save the model at /xyz/"
+    # Invoke the agent
+    app.invoke({"messages": [HumanMessage(content=prompt)]}, config=config)
+    current_state = app.get_state(config)
+    assert current_state.values["model_as_string"][-1] is not None

{aiagents4pharma-1.46.5 → aiagents4pharma-1.48.0}/aiagents4pharma/talk2biomodels/tests/test_simulate_model.py

@@ -40,3 +40,5 @@ def test_simulate_model_tool():
     assert "1,3-bisphosphoglycerate" in dic_simulated_data[0]["data"]
     # Check if the data of the second model contains
     assert "mTORC2" in dic_simulated_data[1]["data"]
+    # Check if the model_as_string is not None
+    assert current_state.values["model_as_string"][-1] is not None

{aiagents4pharma-1.46.5 → aiagents4pharma-1.48.0}/aiagents4pharma/talk2biomodels/tools/__init__.py

@@ -10,6 +10,7 @@ from . import (
     load_biomodel,
     parameter_scan,
     query_article,
+    save_model,
     search_models,
     simulate_model,
     steady_state,

aiagents4pharma-1.48.0/aiagents4pharma/talk2biomodels/tools/save_model.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+"""
+Tool for saving models.
+"""
+
+import logging
+import os
+from typing import Annotated
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import BaseTool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from pydantic import BaseModel, Field
+
+# Initialize logger
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class SaveModelInput(BaseModel):
+    """
+    Input schema for the save model tool.
+    """
+
+    path_to_folder: str = Field(
+        description="Path to folder to save the model. Keep it to . if not provided.", default="."
+    )
+    output_filename: str = Field(
+        description="Filename to save the model as. Default is 'saved_model.xml'.",
+        default="saved_model.xml",
+    )
+    tool_call_id: Annotated[str, InjectedToolCallId]
+    state: Annotated[dict, InjectedState]
+
+
+# Note: It's important that every field has type hints. BaseTool is a
+# Pydantic class and not having type hints can lead to unexpected behavior.
+class SaveModelTool(BaseTool):
+    """
+    Tool for saving a model.
+    """
+
+    name: str = "save_model"
+    description: str = "A tool to save the current biomodel to a \
+                        user specified path with the default filename\
+                         'saved_model.xml'"
+    args_schema: type[BaseModel] = SaveModelInput
+    return_direct: bool = False
+
+    def _run(
+        self,
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[dict, InjectedState],
+        path_to_folder: str = ".",
+        output_filename: str = "saved_model.xml",
+    ) -> Command:
+        """
+        Run the tool.
+
+        Args:
+            path (str): The path to save the model.
+            tool_call_id (str): The tool call ID.
+
+        Returns:
+
+        """
+        logger.log(
+            logging.INFO,
+            "Saving model to path: %s with filename: %s",
+            path_to_folder,
+            output_filename,
+        )
+        # Check if path does not exist
+        if not os.path.exists(path_to_folder):
+            content = f"Error: Path {path_to_folder} does not exist."
+            logger.error(content)
+        else:
+            logger.info("Saving now")
+            # Save the model to the specified path
+            with open(os.path.join(path_to_folder, output_filename), "w", encoding="utf-8") as f:
+                f.write(state["model_as_string"][-1])
+            content = f"Model saved successfully to {path_to_folder}/{output_filename}."
+            logger.info(content)
+        # Return the updated state of the tool
+        return Command(
+            update={
+                # update the message history
+                "messages": [
+                    ToolMessage(
+                        content=content,
+                        tool_call_id=tool_call_id,
+                    )
+                ],
+            }
+        )

{aiagents4pharma-1.46.5 → aiagents4pharma-1.48.0}/aiagents4pharma/talk2biomodels/tools/simulate_model.py

@@ -7,6 +7,7 @@ Tool for simulating a model.
 import logging
 from typing import Annotated
 
+import basico
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import BaseTool
 from langchain_core.tools.base import InjectedToolCallId
@@ -116,6 +117,7 @@ class SimulateModelTool(BaseTool):
             "model_id": [sys_bio_model.biomodel_id],
             "sbml_file_path": [sbml_file_path],
             "dic_simulated_data": [dic_simulated_data],
+            "model_as_string": [basico.model_io.save_model_to_string()],
         }.items():
             if value:
                 dic_updated_state_for_model[key] = value

{aiagents4pharma-1.46.5 → aiagents4pharma-1.48.0}/aiagents4pharma/talk2knowledgegraphs/Dockerfile

@@ -58,6 +58,7 @@ LABEL description="AI Agents for Pharma - Knowledge Graphs Application"
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
   ca-certificates \
   curl \
+  gnupg \
   libmagic1 \
   libopenblas0 \
   libomp5 \
@@ -66,6 +67,23 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
   && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
   && update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1
 
+# Install CUDA runtime libraries required by cudf/cupy
+RUN curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/3bf863cc.pub \
+  | gpg --dearmor -o /usr/share/keyrings/nvidia-cuda-keyring.gpg \
+  && echo "deb [signed-by=/usr/share/keyrings/nvidia-cuda-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/ /" \
+  > /etc/apt/sources.list.d/nvidia-cuda.list \
+  && apt-get update \
+  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+  cuda-cudart-12-6 \
+  cuda-cudart-dev-12-6 \
+  cuda-nvrtc-12-6 \
+  cuda-nvrtc-dev-12-6 \
+  libcublas-12-6 \
+  libcusparse-12-6 \
+  && rm -rf /var/lib/apt/lists/*
+
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64
+
 # Copy UV virtual environment from build stage
 COPY --from=uv-install /opt/venv /opt/venv
 

{aiagents4pharma-1.46.5/aiagents4pharma/talk2knowledgegraphs/docker-compose/gpu → aiagents4pharma-1.48.0/aiagents4pharma/talk2knowledgegraphs/docker-compose/cpu}/.env.example

@@ -15,7 +15,7 @@ MILVUS_HOST=localhost
 MILVUS_PORT=19530
 MILVUS_USER=root
 MILVUS_PASSWORD=Milvus
-MILVUS_DATABASE=your_database_name_here
+MILVUS_DATABASE=t2kg_primekg
 
 # Specify the data directory for multimodal data to your own data directory
 # DATA_DIR=/your_absolute_path_to_your_data_dir/

{aiagents4pharma-1.46.5/aiagents4pharma/talk2aiagents4pharma → aiagents4pharma-1.48.0/aiagents4pharma/talk2knowledgegraphs}/docker-compose/gpu/.env.example

@@ -15,7 +15,7 @@ MILVUS_HOST=localhost
 MILVUS_PORT=19530
 MILVUS_USER=root
 MILVUS_PASSWORD=Milvus
-MILVUS_DATABASE=your_database_name_here
+MILVUS_DATABASE=t2kg_primekg
 
 # Specify the data directory for multimodal data to your own data directory
 # DATA_DIR=/your_absolute_path_to_your_data_dir/

{aiagents4pharma-1.46.5 → aiagents4pharma-1.48.0}/aiagents4pharma/talk2knowledgegraphs/milvus_data_dump.py

@@ -150,6 +150,7 @@ class DynamicDataLoader:
         self.normalize_vectors = self.use_gpu  # Only normalize for GPU (original logic)
         self.vector_index_type = "GPU_CAGRA" if self.use_gpu else "HNSW"
         self.metric_type = "IP" if self.use_gpu else "COSINE"
+        self.vector_index_params = self._build_vector_index_params()
 
         logger.info("Loader Configuration:")
         logger.info("  Using GPU acceleration: %s", self.use_gpu)
@@ -284,6 +285,29 @@ class DynamicDataLoader:
             else:
                 return list(data)
 
+    def _build_vector_index_params(self) -> dict[str, Any]:
+        """Return index params tuned for the selected backend."""
+        base_params: dict[str, Any] = {
+            "index_type": self.vector_index_type,
+            "metric_type": self.metric_type,
+        }
+
+        if self.vector_index_type == "GPU_CAGRA":
+            base_params["params"] = {
+                "graph_degree": int(os.getenv("CAGRA_GRAPH_DEGREE", "32")),
+                "intermediate_graph_degree": int(
+                    os.getenv("CAGRA_INTERMEDIATE_GRAPH_DEGREE", "40")
+                ),
+                "search_width": int(os.getenv("CAGRA_SEARCH_WIDTH", "64")),
+            }
+        elif self.vector_index_type == "HNSW":
+            base_params["params"] = {
+                "M": int(os.getenv("HNSW_M", "16")),
+                "efConstruction": int(os.getenv("HNSW_EF_CONSTRUCTION", "200")),
+            }
+
+        return base_params
+
     def connect_to_milvus(self):
         """Connect to Milvus and setup database."""
         logger.info("Connecting to Milvus at %s:%s", self.milvus_host, self.milvus_port)
@@ -442,10 +466,7 @@ class DynamicDataLoader:
         )
         collection.create_index(
             field_name="desc_emb",
-            index_params={
-                "index_type": self.vector_index_type,
-                "metric_type": self.metric_type,
-            },
+            index_params=self.vector_index_params.copy(),
             index_name="desc_emb_index",
         )
 
@@ -569,18 +590,12 @@ class DynamicDataLoader:
             )
             collection.create_index(
                 field_name="desc_emb",
-                index_params={
-                    "index_type": self.vector_index_type,
-                    "metric_type": self.metric_type,
-                },
+                index_params=self.vector_index_params.copy(),
                 index_name="desc_emb_index",
             )
             collection.create_index(
                 field_name="feat_emb",
-                index_params={
-                    "index_type": self.vector_index_type,
-                    "metric_type": self.metric_type,
-                },
+                index_params=self.vector_index_params.copy(),
                 index_name="feat_emb_index",
             )
 
@@ -706,10 +721,7 @@ class DynamicDataLoader:
         )
         collection.create_index(
             field_name="feat_emb",
-            index_params={
-                "index_type": self.vector_index_type,
-                "metric_type": self.metric_type,
-            },
+            index_params=self.vector_index_params.copy(),
             index_name="feat_emb_index",
         )
 
@@ -796,8 +808,7 @@ class DynamicDataLoader:
                 logger.info("  %s: %d entities", coll, collection.num_entities)
 
         except Exception:
-            logger.error("Error occurred during data loading")
-            logger.debug("Detailed error information available in debug mode")
+            logger.exception("Error occurred during data loading")
             raise
 
 
@@ -867,8 +878,7 @@ def main():
         logger.info("Data loading interrupted by user")
         sys.exit(1)
     except Exception:
-        logger.error("Fatal error occurred during data loading")
-        logger.debug("Detailed error information available in debug mode")
+        logger.exception("Fatal error occurred during data loading")
         sys.exit(1)
 
 

{aiagents4pharma-1.46.5 → aiagents4pharma-1.48.0}/app/frontend/streamlit_app_talk2biomodels.py

@@ -70,10 +70,7 @@ if needs("NVIDIA/"):
 
 missing = [var for var in needed_env if var not in os.environ]
 if missing:
-    st.warning(
-        "Missing environment settings for the selected provider(s): "
-        + ", ".join(missing)
-    )
+    st.warning("Missing environment settings for the selected provider(s): " + ", ".join(missing))
 
 ########################################################################################
 # Streamlit app
@@ -209,12 +206,8 @@ with main_col2:
                                     validate=r"^http://.*$",  # Ensure the link is valid
                                     display_text=r"^http://identifiers\.org/(.*?)$",
                                 ),
-                                "Species Name": st.column_config.TextColumn(
-                                    "Species Name"
-                                ),
-                                "Description": st.column_config.TextColumn(
-                                    "Description"
-                                ),
+                                "Species Name": st.column_config.TextColumn("Species Name"),
+                                "Description": st.column_config.TextColumn("Description"),
                                 "Database": st.column_config.TextColumn("Database"),
                             },
                         )
@@ -234,9 +227,7 @@ with main_col2:
                                 ),
                                 "name": st.column_config.TextColumn("Name"),
                                 "format": st.column_config.TextColumn("Format"),
-                                "submissionDate": st.column_config.TextColumn(
-                                    "Submission Date"
-                                ),
+                                "submissionDate": st.column_config.TextColumn("Submission Date"),
                             },
                         )
                 else:
@@ -266,15 +257,19 @@ with main_col2:
                             ),
                         },
                     )
-                    intro_prompt = "Tell your name and about yourself. Always start with a greeting."
-                    intro_prompt += " and tell about the tools you can run to perform analysis with short description."
-                    intro_prompt += " We have provided starter questions (separately) outisde your response."
+                    intro_prompt = (
+                        "Tell your name and about yourself. Always start with a greeting."
+                    )
+                    intro_prompt += " and tell about every tools you can run to perform analysis with short description."
+                    intro_prompt += (
+                        " We have provided starter questions (separately) outisde your response."
+                    )
                     intro_prompt += " Do not provide any questions by yourself. Let the users know that they can"
                     intro_prompt += " simply click on the questions to execute them."
+                    intro_prompt += " Let them know that they can check out the use cases"
                     intro_prompt += (
-                        " Let them know that they can check out the use cases"
+                        " and FAQs described in the link below. Be friendly and helpful."
                     )
-                    intro_prompt += " and FAQs described in the link below. Be friendly and helpful."
                     intro_prompt += "\n"
                     intro_prompt += "Here is the link to the use cases: [Use Cases](https://virtualpatientengine.github.io/AIAgents4Pharma/talk2biomodels/cases/Case_1/)"
                     intro_prompt += "\n"
@@ -290,15 +285,11 @@ with main_col2:
                     assistant_msg = ChatMessage(
                         current_state.values["messages"][-1].content, role="assistant"
                     )
-                    st.session_state.messages.append(
-                        {"type": "message", "content": assistant_msg}
-                    )
+                    st.session_state.messages.append({"type": "message", "content": assistant_msg})
                     st.empty()
         if len(st.session_state.messages) <= 1:
             for count, question in enumerate(streamlit_utils.sample_questions()):
-                if st.button(
-                    f"Q{count+1}. {question}", key=f"sample_question_{count+1}"
-                ):
+                if st.button(f"Q{count + 1}. {question}", key=f"sample_question_{count + 1}"):
                     # Trigger the question
                     prompt = question
                 # Add button click to chat history
@@ -306,8 +297,8 @@ with main_col2:
                     {
                         "type": "button",
                         "question": question,
-                        "content": f"Q{count+1}. {question}",
-                        "key": f"sample_question_{count+1}",
+                        "content": f"Q{count + 1}. {question}",
+                        "key": f"sample_question_{count + 1}",
                     }
                 )
 
@@ -315,9 +306,7 @@ with main_col2:
         if prompt:
             # Create a key 'uploaded_file' to read the uploaded file
             if uploaded_sbml_file:
-                st.session_state.sbml_file_path = uploaded_sbml_file.read().decode(
-                    "utf-8"
-                )
+                st.session_state.sbml_file_path = uploaded_sbml_file.read().decode("utf-8")
 
             # Display user prompt
             prompt_msg = ChatMessage(prompt, role="user")

aiagents4pharma-1.48.0/docs/notebooks/talk2biomodels/benchmark/benchmark.md

@@ -0,0 +1,50 @@
+# Benchmark
+
+## Evaluation results
+| Dataset | Task completion (mean ± std)  | 
+|----------|----------|
+| Set 1    | 0.712 ± 0.021   |
+| Set 2    | 0.707 ± 0.028   |
+| Set 3    | pending  | 
+| Set 4    | pending  |
+
+For detailed results, please refer to log files for [set 1](results/task_completion_set1_results.json) and [set 2](results/task_completion_set2_results.json).
+
+### Example evaluation
+
+"id": "sim_090" (set 1)
+Question: "I need to understand the MAPK signaling dynamics for my research. Please simulate model BIOMD0000000027 for 1000 seconds and provide the Mpp concentration at the end of the simulation period. This information is critical for my research on signal transduction and cellular responses."
+Expected answer: "I've simulated model BIOMD0000000027 for 1000 seconds to analyze the MAPK signaling dynamics for your research. The Mpp concentration at the end of the simulation period is 48.17 nmol/L. This information is critical for your research on signal transduction and cellular responses, as it represents the steady-state concentration of doubly phosphorylated MAPK1 after the complete 1000-second simulation."
+
+Answer during benchmark: "The concentration of Mpp at the end of the 1000 seconds simulation is approximately 48.18 nmol. This information should be useful for your research on signal transduction and cellular responses.",
+
+Task completion score: 0.7
+LLM-as-a-judge verdict: "The system successfully processed the input data and provided an answer, which aligns with the task of answering a question. However, the inclusion of additional keys such as 'assistant_messages', 'all_messages', 'state_fields', and 'thread_id' suggests that the output was more complex than necessary for the task. The lack of additional tools or handoffs indicates a straightforward approach, but the extra information may not have been directly relevant to answering the question."
+
+
+
+## Description
+We would like to benchmark the performance on *Task Completion* of the T2B agent using [DeepEval framework](https://deepeval.com/docs/getting-started). Here, T2B's generated response is evaluated against the ground truth answer using a LLM-as-a-judge.
+
+Specifically, we would like to benchmarkt the following aspects of the T2B agent:
+
+* stability of the outputs depending on stochastic user inputs, grammatical errors, typos, length of the prompt and user background
+* stability of the outputs given different tool calls, argument inputs and multi-turn conversation
+
+
+## Dataset summary
+
+| Set | Description | Tools | Focus | Questions | Example |
+|-----|-------------|-------|-------|-----------|---------|
+| [Set 1](benchmark_questions_set1.json) | User input variability with respect to background, grammar and clarity. Captures extreme cases in how users can address the agent. | simulate_model, ask_question | Communication variability | 90 | "pls run sim BIOMD0000000027 1000 seconds get Mpp concentration" vs "I need to understand the MAPK signaling dynamics for my research..." |
+| [Set 2](benchmark_questions_set2.json) | Variability in user inputs relative to the number of provided parameters and tools, requested through generally well-formulated and grammatically correct questions. | simulate_model, search_models, steady_state, ask_question, custom_plotter, get_modelinfo | Parameter variability | 222 | "Search for models on precision medicine, and then list the names of the models." |
+| [Set 3](benchmark_questions_set3.json) | Tabular data matching | steady_state, parameter_scan | Tabular data | 79 | "Analyze MAPKK parameter impact on Mpp concentration over time in model 27. Use parameter scan from 1 to 100 with 10 steps." |
+| [Set 4](benchmark_questions_set4.json) | Annotation id matching | get_annotation | Annotation matching | 60 | "what are the annotations for Mp and MKP3 in model 27?" |
+
+## Benchmark strategy
+
+1. Generate set of prompts and ground truth answers for each set. The ground truth answers are generated using the [basico library](https://github.com/copasi/basico) and textualized using a LLM. Each set should represent a different output types (textual, tabular, dictionary, etc.) and different tool calling patterns (single parameter, multiple parameters, multiple tools, etc.). The prompts that were used to generate the ground truth answers can be found [here](generating_QnA_pairs.md) and the ground truth data can be found [here](expected_asnwers_basico.ipynb).
+
+2. Runn Task Completion benchmark for each set. 
+
+The code used for the benchmark can be found here for [set 1](results/Task_Completion_set1.py) and [set 2](results/Task_Completion_set2.py).