aiqtoolkit 1.2.0a20250612__py3-none-any.whl → 1.2.0a20250613__py3-none-any.whl

aiq/eval/dataset_handler/dataset_handler.py

@@ -81,6 +81,7 @@ class DatasetHandler:
                 output_obj=row.get(self.generated_answer_key, "") if structured else "",
                 trajectory=row.get(self.trajectory_key, []) if structured else [],
                 expected_trajectory=row.get(self.expected_trajectory_key, []) if structured else [],
+                full_dataset_entry=row.to_dict(),
             )
 
         # if input dataframe is empty return an empty list

aiq/eval/evaluator/base_evaluator.py

@@ -0,0 +1,73 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+from abc import ABC
+from abc import abstractmethod
+
+from tqdm import tqdm
+
+from aiq.eval.evaluator.evaluator_model import EvalInput
+from aiq.eval.evaluator.evaluator_model import EvalInputItem
+from aiq.eval.evaluator.evaluator_model import EvalOutput
+from aiq.eval.evaluator.evaluator_model import EvalOutputItem
+from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry
+
+
+class BaseEvaluator(ABC):
+    """
+    Base class for custom evaluators.
+
+    Each custom evaluator must implement the `evaluate_item` method which is used to evaluate a
+    single EvalInputItem.
+    """
+
+    def __init__(self, max_concurrency: int = 4, tqdm_desc: str = "Evaluating"):
+        self.max_concurrency = max_concurrency
+        self.semaphore = asyncio.Semaphore(max_concurrency)
+        self.tqdm_desc = tqdm_desc
+
+    @abstractmethod
+    async def evaluate_item(self, item: EvalInputItem) -> EvalOutputItem:
+        """Each evaluator must implement this for item-level evaluation"""
+        pass
+
+    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
+        pbar = None
+        try:
+            tqdm_position = TqdmPositionRegistry.claim()
+            pbar = tqdm(total=len(eval_input.eval_input_items), desc=self.tqdm_desc, position=tqdm_position)
+
+            async def wrapped(item):
+                async with self.semaphore:
+                    try:
+                        output_item = await self.evaluate_item(item)
+                        pbar.update(1)
+                        return output_item
+                    except Exception as e:
+                        # If the evaluator fails, return an error item with a score of 0.0
+                        pbar.update(1)
+                        return EvalOutputItem(id=item.id, score=0.0, reasoning={"error": f"Evaluator error: {str(e)}"})
+
+            output_items = await asyncio.gather(*[wrapped(item) for item in eval_input.eval_input_items])
+        finally:
+            pbar.close()
+            TqdmPositionRegistry.release(tqdm_position)
+
+        # Compute average if possible
+        numeric_scores = [item.score for item in output_items if isinstance(item.score, (int, float))]
+        avg_score = round(sum(numeric_scores) / len(numeric_scores), 2) if numeric_scores else None
+
+        return EvalOutput(average_score=avg_score, eval_output_items=output_items)

aiq/eval/evaluator/evaluator_model.py

@@ -27,6 +27,7 @@ class EvalInputItem(BaseModel):
     output_obj: typing.Any
     expected_trajectory: list[IntermediateStep]
     trajectory: list[IntermediateStep]
+    full_dataset_entry: typing.Any
 
 
 class EvalInput(BaseModel):

aiq/eval/trajectory_evaluator/evaluate.py

@@ -13,24 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import asyncio
 import logging
 
 from langchain.evaluation import TrajectoryEvalChain
 from langchain_core.language_models import BaseChatModel
 from langchain_core.tools import BaseTool
-from tqdm import tqdm
 
-from aiq.eval.evaluator.evaluator_model import EvalInput
+from aiq.eval.evaluator.base_evaluator import BaseEvaluator
 from aiq.eval.evaluator.evaluator_model import EvalInputItem
-from aiq.eval.evaluator.evaluator_model import EvalOutput
 from aiq.eval.evaluator.evaluator_model import EvalOutputItem
-from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry
 
 logger = logging.getLogger(__name__)
 
 
-class TrajectoryEvaluator:
+class TrajectoryEvaluator(BaseEvaluator):
 
     def __init__(
         self,
@@ -38,11 +34,9 @@ class TrajectoryEvaluator:
         tools: list[BaseTool] | None = None,
         max_concurrency: int = 8,
     ):
-
+        super().__init__(max_concurrency=max_concurrency, tqdm_desc="Evaluating Trajectory")
         self.llm = llm
         self.tools = tools
-        self.max_concurrency = max_concurrency
-        self.semaphore = asyncio.Semaphore(self.max_concurrency)
         # Initialize trajectory evaluation chain
         self.traj_eval_chain = TrajectoryEvalChain.from_llm(llm=self.llm,
                                                             tools=self.tools,
@@ -50,69 +44,32 @@ class TrajectoryEvaluator:
                                                             requires_reference=True)
         logger.debug("Trajectory evaluation chain initialized.")
 
-    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
+    async def evaluate_item(self, item: EvalInputItem) -> EvalOutputItem:
         """
-        Evaluates the agent trajectories using trajectory evaluation chain.
+        Evaluate a single EvalInputItem and return an EvalOutputItem.
         """
-
-        num_records = len(eval_input.eval_input_items)
-        logger.info("Running trajectory evaluation with %d records", num_records)
         from aiq.data_models.intermediate_step import IntermediateStepType
         from aiq.eval.intermediate_step_adapter import IntermediateStepAdapter
 
         intermediate_step_adapter = IntermediateStepAdapter()
         event_filter = [IntermediateStepType.LLM_END, IntermediateStepType.TOOL_END]
 
-        async def process_item(item: EvalInputItem) -> tuple[float, dict]:
-            """
-            Evaluate a single EvalInputItem asynchronously and return a tuple of-
-            1. score
-            2. reasoning for the score
-            """
-            question = item.input_obj
-            generated_answer = item.output_obj
-            agent_trajectory = intermediate_step_adapter.get_agent_actions(item.trajectory, event_filter)
-            try:
-                eval_result = await self.traj_eval_chain.aevaluate_agent_trajectory(
-                    input=question,
-                    agent_trajectory=agent_trajectory,
-                    prediction=generated_answer,
-                )
-            except Exception as e:
-                logger.exception("Error evaluating trajectory for question: %s, Error: %s", question, e, exc_info=True)
-                return 0.0, f"Error evaluating trajectory: {e}"
-
-            reasoning = {
-                "reasoning": eval_result["reasoning"],
-                "trajectory": [(action.model_dump(), output) for (action, output) in agent_trajectory]
-            }
-            return eval_result["score"], reasoning
-
-        async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]:
-            async with self.semaphore:
-                result = await process_item(item)
-                pbar.update(1)
-                return result
+        question = item.input_obj
+        generated_answer = item.output_obj
+        agent_trajectory = intermediate_step_adapter.get_agent_actions(item.trajectory, event_filter)
 
-        # Execute all evaluations asynchronously
         try:
-            tqdm_position = TqdmPositionRegistry.claim()
-            pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Trajectory", position=tqdm_position)
-            results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items])
-        finally:
-            pbar.close()
-            TqdmPositionRegistry.release(tqdm_position)
-
-        # Extract scores and reasonings
-        sample_scores, sample_reasonings = zip(*results) if results else ([], [])
-
-        # Compute average score
-        avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0
-
-        # Construct EvalOutputItems
-        eval_output_items = [
-            EvalOutputItem(id=item.id, score=score, reasoning=reasoning)
-            for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings)
-        ]
-
-        return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items)
+            eval_result = await self.traj_eval_chain.aevaluate_agent_trajectory(
+                input=question,
+                agent_trajectory=agent_trajectory,
+                prediction=generated_answer,
+            )
+        except Exception as e:
+            logger.exception("Error evaluating trajectory for question: %s, Error: %s", question, e, exc_info=True)
+            return EvalOutputItem(id=item.id, score=0.0, reasoning=f"Error evaluating trajectory: {e}")
+
+        reasoning = {
+            "reasoning": eval_result["reasoning"],
+            "trajectory": [(action.model_dump(), output) for (action, output) in agent_trajectory]
+        }
+        return EvalOutputItem(id=item.id, score=eval_result["score"], reasoning=reasoning)

aiq/eval/tunable_rag_evaluator/evaluate.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import asyncio
 import logging
 
 from langchain.output_parsers import ResponseSchema
@@ -21,13 +20,10 @@ from langchain.output_parsers import StructuredOutputParser
 from langchain.schema import HumanMessage
 from langchain.schema import SystemMessage
 from langchain_core.language_models import BaseChatModel
-from tqdm import tqdm
 
-from aiq.eval.evaluator.evaluator_model import EvalInput
+from aiq.eval.evaluator.base_evaluator import BaseEvaluator
 from aiq.eval.evaluator.evaluator_model import EvalInputItem
-from aiq.eval.evaluator.evaluator_model import EvalOutput
 from aiq.eval.evaluator.evaluator_model import EvalOutputItem
-from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry
 
 logger = logging.getLogger(__name__)
 
@@ -69,7 +65,7 @@ def evaluation_prompt(judge_llm_prompt: str,
     return EVAL_PROMPT if not default_scoring else DEFAULT_EVAL_PROMPT
 
 
-class TunableRagEvaluator:
+class TunableRagEvaluator(BaseEvaluator):
     '''Tunable RAG evaluator class with customizable LLM prompt for scoring.'''
 
     def __init__(self,
@@ -78,186 +74,141 @@ class TunableRagEvaluator:
                  max_concurrency: int,
                  default_scoring: bool,
                  default_score_weights: dict):
+        super().__init__(max_concurrency=max_concurrency, tqdm_desc="Evaluating RAG")
         self.llm = llm
-        self.max_concurrency = max_concurrency
         self.judge_llm_prompt = judge_llm_prompt
-        self.semaphore = asyncio.Semaphore(self.max_concurrency)
         self.default_scoring = default_scoring
         # Use user-provided weights if available; otherwise, set equal weights for each score
         self.default_score_weights = default_score_weights if default_score_weights else {
             "coverage": 1 / 3, "correctness": 1 / 3, "relevance": 1 / 3
         }
 
-    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
-        '''Evaluate function'''
-
-        async def process_item(item):
-            """Compute RAG evaluation for an individual item"""
-            question = item.input_obj
-            answer_description = item.expected_output_obj
-            generated_answer = item.output_obj
-
-            # Call judge LLM to generate score
-            score = 0.0
-
-            default_evaluation_schema = [
-                ResponseSchema(
-                    name="coverage_score",
-                    description=
-                    "Score for the coverage of all critical aspects mentioned in the expected answer. Ex. 0.5",
-                    type="float"),
-                ResponseSchema(
-                    name="correctness_score",
-                    description=
-                    "Score for the accuracy of the generated answer compared to the expected answer. Ex. 0.5",
-                    type="float"),
-                ResponseSchema(name="relevance_score",
-                               description="Score for the relevance of the generated answer to the question. Ex. 0.5",
-                               type="float"),
-                ResponseSchema(
-                    name="reasoning",
-                    description=
-                    "1-2 summarized sentences of reasoning for the scores. Ex. 'The generated answer covers all critical aspects mentioned in the expected answer, is correct, and is relevant to the question.'",
-                    type="string"),
-            ]
-
-            custom_evaluation_schema = [
-                ResponseSchema(name="score", description="Score for the generated answer. Ex. 0.5", type="float"),
-                ResponseSchema(
-                    name="reasoning",
-                    description=
-                    "1-2 sentence reasoning for the score. Ex. 'The generated answer is exactly the same as the description of the expected answer.'",
-                    type="string"),
-            ]
+    async def evaluate_item(self, item: EvalInputItem) -> EvalOutputItem:
+        '''Evaluate a single item'''
+        question = item.input_obj
+        answer_description = item.expected_output_obj
+        generated_answer = item.output_obj
+
+        # Call judge LLM to generate score
+        score = 0.0
+
+        default_evaluation_schema = [
+            ResponseSchema(
+                name="coverage_score",
+                description="Score for the coverage of all critical aspects mentioned in the expected answer. Ex. 0.5",
+                type="float"),
+            ResponseSchema(
+                name="correctness_score",
+                description="Score for the accuracy of the generated answer compared to the expected answer. Ex. 0.5",
+                type="float"),
+            ResponseSchema(name="relevance_score",
+                           description="Score for the relevance of the generated answer to the question. Ex. 0.5",
+                           type="float"),
+            ResponseSchema(
+                name="reasoning",
+                description=
+                "1-2 summarized sentences of reasoning for the scores. Ex. 'The generated answer covers all critical aspects mentioned in the expected answer, is correct, and is relevant to the question.'",
+                type="string"),
+        ]
 
-            if self.default_scoring:
-                evaluation_schema = default_evaluation_schema
-            else:
-                evaluation_schema = custom_evaluation_schema
+        custom_evaluation_schema = [
+            ResponseSchema(name="score", description="Score for the generated answer. Ex. 0.5", type="float"),
+            ResponseSchema(
+                name="reasoning",
+                description=
+                "1-2 sentence reasoning for the score. Ex. 'The generated answer is exactly the same as the description of the expected answer.'",
+                type="string"),
+        ]
 
-            llm_input_response_parser = StructuredOutputParser.from_response_schemas(evaluation_schema)
-            format_instructions = llm_input_response_parser.get_format_instructions()
+        if self.default_scoring:
+            evaluation_schema = default_evaluation_schema
+        else:
+            evaluation_schema = custom_evaluation_schema
 
-            eval_prompt = evaluation_prompt(judge_llm_prompt=self.judge_llm_prompt,
-                                            question=question,
-                                            answer_description=answer_description,
-                                            generated_answer=generated_answer,
-                                            format_instructions=format_instructions,
-                                            default_scoring=self.default_scoring)
+        llm_input_response_parser = StructuredOutputParser.from_response_schemas(evaluation_schema)
+        format_instructions = llm_input_response_parser.get_format_instructions()
 
-            messages = [
-                SystemMessage(content="You must respond only in JSON format."), HumanMessage(content=eval_prompt)
-            ]
+        eval_prompt = evaluation_prompt(judge_llm_prompt=self.judge_llm_prompt,
+                                        question=question,
+                                        answer_description=answer_description,
+                                        generated_answer=generated_answer,
+                                        format_instructions=format_instructions,
+                                        default_scoring=self.default_scoring)
 
-            response = await self.llm.ainvoke(messages)
+        messages = [SystemMessage(content="You must respond only in JSON format."), HumanMessage(content=eval_prompt)]
 
-            # Initialize default values to handle service errors
-            coverage_score = 0.0
-            correctness_score = 0.0
-            relevance_score = 0.0
-            reasoning = "Error in evaluator from parsing judge LLM response."
+        response = await self.llm.ainvoke(messages)
 
-            try:
-                parsed_response = llm_input_response_parser.parse(response.content)
-                if self.default_scoring:
-                    try:
-                        coverage_score = parsed_response["coverage_score"]
-                        correctness_score = parsed_response["correctness_score"]
-                        relevance_score = parsed_response["relevance_score"]
-                        reasoning = parsed_response["reasoning"]
-                    except KeyError as e:
-                        logger.error("Missing required keys in default scoring response: %s",
-                                     ", ".join(str(arg) for arg in e.args))
-                        reasoning = f"Error in evaluator from parsing judge LLM response. Missing required key(s): {', '.join(str(arg) for arg in e.args)}"
-
-                    coverage_weight = self.default_score_weights.get("coverage", 1 / 3)
-                    correctness_weight = self.default_score_weights.get("correctness", 1 / 3)
-                    relevance_weight = self.default_score_weights.get("relevance", 1 / 3)
-
-                    # Calculate score
-                    total_weight = coverage_weight + correctness_weight + relevance_weight
-                    coverage_weight = coverage_weight / total_weight
-                    correctness_weight = correctness_weight / total_weight
-                    relevance_weight = relevance_weight / total_weight
-
-                    if round(coverage_weight + correctness_weight + relevance_weight, 2) != 1:
-                        logger.warning("The sum of the default score weights is not 1. The weights will be normalized.")
-                        coverage_weight = coverage_weight / (coverage_weight + correctness_weight + relevance_weight)
-                        correctness_weight = correctness_weight / (coverage_weight + correctness_weight +
-                                                                   relevance_weight)
-                        relevance_weight = relevance_weight / (coverage_weight + correctness_weight + relevance_weight)
-
-                    score = (coverage_weight * coverage_score + correctness_weight * correctness_score +
-                             relevance_weight * relevance_score)
-
-                else:
-                    try:
-                        score = parsed_response["score"]
-                        reasoning = parsed_response["reasoning"]
-                    except KeyError as e:
-                        logger.error("Missing required keys in custom scoring response: %s",
-                                     ", ".join(str(arg) for arg in e.args))
-                        reasoning = f"Error in evaluator from parsing judge LLM response. Missing required key(s): {', '.join(str(arg) for arg in e.args)}"
-                        raise
-            except (KeyError, ValueError) as e:
-                logger.error("Error parsing judge LLM response: %s", e)
-                score = 0.0
-                reasoning = "Error in evaluator from parsing judge LLM response."
+        # Initialize default values to handle service errors
+        coverage_score = 0.0
+        correctness_score = 0.0
+        relevance_score = 0.0
+        reasoning = "Error in evaluator from parsing judge LLM response."
 
+        try:
+            parsed_response = llm_input_response_parser.parse(response.content)
             if self.default_scoring:
-                reasoning = {
-                    "question": question,
-                    "answer_description": answer_description,
-                    "generated_answer": generated_answer,
-                    "score_breakdown": {
-                        "coverage_score": coverage_score,
-                        "correctness_score": correctness_score,
-                        "relevance_score": relevance_score,
-                    },
-                    "reasoning": reasoning,
-                }
-            else:
-                reasoning = {
-                    "question": question,
-                    "answer_description": answer_description,
-                    "generated_answer": generated_answer,
-                    "reasoning": reasoning
-                }
-
-            return score, reasoning
-
-        async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]:
-            """
-            Process an item asynchronously and update the progress bar.
-            Use the semaphore to limit the number of concurrent items.
-            """
-            async with self.semaphore:
-                result = await process_item(item)
-                # Update the progress bar
-                pbar.update(1)
-                return result
+                try:
+                    coverage_score = parsed_response["coverage_score"]
+                    correctness_score = parsed_response["correctness_score"]
+                    relevance_score = parsed_response["relevance_score"]
+                    reasoning = parsed_response["reasoning"]
+                except KeyError as e:
+                    logger.error("Missing required keys in default scoring response: %s",
+                                 ", ".join(str(arg) for arg in e.args))
+                    reasoning = f"Error in evaluator from parsing judge LLM response. Missing required key(s): {', '.join(str(arg) for arg in e.args)}"
+
+                coverage_weight = self.default_score_weights.get("coverage", 1 / 3)
+                correctness_weight = self.default_score_weights.get("correctness", 1 / 3)
+                relevance_weight = self.default_score_weights.get("relevance", 1 / 3)
+
+                # Calculate score
+                total_weight = coverage_weight + correctness_weight + relevance_weight
+                coverage_weight = coverage_weight / total_weight
+                correctness_weight = correctness_weight / total_weight
+                relevance_weight = relevance_weight / total_weight
+
+                if round(coverage_weight + correctness_weight + relevance_weight, 2) != 1:
+                    logger.warning("The sum of the default score weights is not 1. The weights will be normalized.")
+                    coverage_weight = coverage_weight / (coverage_weight + correctness_weight + relevance_weight)
+                    correctness_weight = correctness_weight / (coverage_weight + correctness_weight + relevance_weight)
+                    relevance_weight = relevance_weight / (coverage_weight + correctness_weight + relevance_weight)
+
+                score = (coverage_weight * coverage_score + correctness_weight * correctness_score +
+                         relevance_weight * relevance_score)
 
-        try:
-            # Claim a tqdm position to display the progress bar
-            tqdm_position = TqdmPositionRegistry.claim()
-            # Create a progress bar
-            pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating RAG", position=tqdm_position)
-            # Process items concurrently with a limit on concurrency
-            results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items])
-        finally:
-            pbar.close()
-            TqdmPositionRegistry.release(tqdm_position)
-
-        # Extract scores and reasonings
-        sample_scores, sample_reasonings = zip(*results) if results else ([], [])
-
-        # Compute average score
-        avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0
-
-        # Construct EvalOutputItems
-        eval_output_items = [
-            EvalOutputItem(id=item.id, score=score, reasoning=reasoning)
-            for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings)
-        ]
+            else:
+                try:
+                    score = parsed_response["score"]
+                    reasoning = parsed_response["reasoning"]
+                except KeyError as e:
+                    logger.error("Missing required keys in custom scoring response: %s",
+                                 ", ".join(str(arg) for arg in e.args))
+                    reasoning = f"Error in evaluator from parsing judge LLM response. Missing required key(s): {', '.join(str(arg) for arg in e.args)}"
+                    raise
+        except (KeyError, ValueError) as e:
+            logger.error("Error parsing judge LLM response: %s", e)
+            score = 0.0
+            reasoning = "Error in evaluator from parsing judge LLM response."
 
-        return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items)
+        if self.default_scoring:
+            reasoning = {
+                "question": question,
+                "answer_description": answer_description,
+                "generated_answer": generated_answer,
+                "score_breakdown": {
+                    "coverage_score": coverage_score,
+                    "correctness_score": correctness_score,
+                    "relevance_score": relevance_score,
+                },
+                "reasoning": reasoning,
+            }
+        else:
+            reasoning = {
+                "question": question,
+                "answer_description": answer_description,
+                "generated_answer": generated_answer,
+                "reasoning": reasoning
+            }
+
+        return EvalOutputItem(id=item.id, score=score, reasoning=reasoning)

aiq/observability/register.py

@@ -154,3 +154,27 @@ async def file_logging_method(config: FileLoggingMethod, builder: Builder):
     handler = logging.FileHandler(filename=config.path, mode="a", encoding="utf-8")
     handler.setLevel(level)
     yield handler
+
+
+class PatronusTelemetryExporter(TelemetryExporterBaseConfig, name="patronus"):
+    """A telemetry exporter to transmit traces to Patronus service."""
+
+    endpoint: str = Field(description="The Patronus OTEL endpoint")
+    api_key: str = Field(description="The Patronus API key", default="")
+    project: str = Field(description="The project name to group the telemetry traces.")
+
+
+@register_telemetry_exporter(config_type=PatronusTelemetryExporter)
+async def patronus_telemetry_exporter(config: PatronusTelemetryExporter, builder: Builder):
+    """Create a Patronus telemetry exporter."""
+    trace_exporter = telemetry_optional_import("opentelemetry.exporter.otlp.proto.grpc.trace_exporter")
+
+    api_key = config.api_key or os.environ.get("PATRONUS_API_KEY")
+    if not api_key:
+        raise ValueError("API key is required for Patronus")
+
+    headers = {
+        "x-api-key": api_key,
+        "pat-project-name": config.project,
+    }
+    yield trace_exporter.OTLPSpanExporter(endpoint=config.endpoint, headers=headers)

aiq/tool/mcp/mcp_client.py

@@ -45,6 +45,7 @@ def model_from_mcp_schema(name: str, mcp_input_schema: dict) -> type[BaseModel]:
     }
 
     properties = mcp_input_schema.get("properties", {})
+    required_fields = set(mcp_input_schema.get("required", []))
     schema_dict = {}
 
     def _generate_valid_classname(class_name: str):
@@ -70,7 +71,17 @@ def model_from_mcp_schema(name: str, mcp_input_schema: dict) -> type[BaseModel]:
         else:
             field_type = _type_map.get(json_type, Any)
 
-        default_value = field_properties.get("default", ...)
+        # Determine the default value based on whether the field is required
+        if field_name in required_fields:
+            # Field is required - use explicit default if provided, otherwise make it required
+            default_value = field_properties.get("default", ...)
+        else:
+            # Field is optional - use explicit default if provided, otherwise None
+            default_value = field_properties.get("default", None)
+            # Make the type optional if no default was provided
+            if "default" not in field_properties:
+                field_type = field_type | None
+
         nullable = field_properties.get("nullable", False)
         description = field_properties.get("description", "")
 

{aiqtoolkit-1.2.0a20250612.dist-info → aiqtoolkit-1.2.0a20250613.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aiqtoolkit
-Version: 1.2.0a20250612
+Version: 1.2.0a20250613
 Summary: NVIDIA Agent Intelligence toolkit
 Author: NVIDIA Corporation
 Maintainer: NVIDIA Corporation

{aiqtoolkit-1.2.0a20250612.dist-info → aiqtoolkit-1.2.0a20250613.dist-info}/RECORD

@@ -115,9 +115,10 @@ aiq/eval/runtime_event_subscriber.py,sha256=2VM8MqmPc_EWPxxrDDR9naiioZirkJUfGwzb
 aiq/eval/dataset_handler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiq/eval/dataset_handler/dataset_downloader.py,sha256=Zvfbd-fPOhB9n8ZiCBaBKW0y-5v97mQAy3dkBL0OFZ0,4553
 aiq/eval/dataset_handler/dataset_filter.py,sha256=mop6wa4P_QtQ5QkfXv-hVBm3EMerfNECSTJGGDB1YWE,2115
-aiq/eval/dataset_handler/dataset_handler.py,sha256=cqdGVgHm6tsKk3TwFcFhptxAvcHVlZTOh4bXuBsfCYs,7797
+aiq/eval/dataset_handler/dataset_handler.py,sha256=z4trKYPnqSrLvsKctU9d5WrQW7ddbZZx0zOrYVLqbAA,7847
 aiq/eval/evaluator/__init__.py,sha256=GUJrgGtpvyMUCjUBvR3faAdv-tZzbU9W-izgx9aMEQg,680
-aiq/eval/evaluator/evaluator_model.py,sha256=alO8bVoGmvej1LpN5wZ5HG29TSrL4IMWdVcMew8IOzM,1405
+aiq/eval/evaluator/base_evaluator.py,sha256=5kqOcTYNecnh9us_XvV58pj5tZI82NGkVN4tg9-R_ZE,3040
+aiq/eval/evaluator/evaluator_model.py,sha256=5cxe3mqznlNGzv29v_VseYU7OzoT1eTf7hgSPQxytsM,1440
 aiq/eval/rag_evaluator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiq/eval/rag_evaluator/evaluate.py,sha256=lEjXKiuELAHyWckz-bM91dZ6AT2J6NC7SfvtedR-Qdk,6548
 aiq/eval/rag_evaluator/register.py,sha256=2NzxkgqyoZ4wC8ARj3tiVoE8ENCmplBCIKrNOFh6_VI,5642
@@ -125,10 +126,10 @@ aiq/eval/swe_bench_evaluator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 aiq/eval/swe_bench_evaluator/evaluate.py,sha256=kNukRruq1EM1RsGLvpVuC22xcP0gpn9acF3edGak9vY,9858
 aiq/eval/swe_bench_evaluator/register.py,sha256=sTb74F7w4iuI0ROsEJ4bV13Nt1GEWQn7UvO2O0HXwXk,1537
 aiq/eval/trajectory_evaluator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aiq/eval/trajectory_evaluator/evaluate.py,sha256=pfcrFGMmunHS8lG13Rdi0Vf4dw7cTwY0uUN5eOXAA1s,5064
+aiq/eval/trajectory_evaluator/evaluate.py,sha256=Y51KMhJ9t8AoYWrQlrwipc2CtgIXA9IUGZTbKegtsnw,3257
 aiq/eval/trajectory_evaluator/register.py,sha256=kktT4fu5_1Cou-iohD3YhQevsWiR3TA5NpFSweVz0eQ,1709
 aiq/eval/tunable_rag_evaluator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aiq/eval/tunable_rag_evaluator/evaluate.py,sha256=XsQK8BPTWLkolRUd5wZtQpUITukLCmE_bOlY2tjFvGs,13844
+aiq/eval/tunable_rag_evaluator/evaluate.py,sha256=lZxQDhvcAu0JR1RApkbs-G3T9pUOSfh822TYGp7vrQw,11440
 aiq/eval/tunable_rag_evaluator/register.py,sha256=uV36xONVxQW8qBO_bsvbvZk4-J4IhowxiRKErnYsbzA,2369
 aiq/eval/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiq/eval/utils/output_uploader.py,sha256=SaQbZPkw-Q0H7t5yG60Kh-p1cflR7gPklVkilC4uPbU,5141
@@ -175,7 +176,7 @@ aiq/meta/module_to_distro.json,sha256=1XV7edobFrdDKvsSoynfodXg_hczUWpDrQzGkW9qqE
 aiq/meta/pypi.md,sha256=N1fvWaio3KhnAw9yigeM-oWaLuT5i_C7U_2UVzyPbks,4386
 aiq/observability/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiq/observability/async_otel_listener.py,sha256=2Ye9bkHfAssuxFS_ECyRyl-bTa73yYvsPyO4BaK5Beg,19662
-aiq/observability/register.py,sha256=CoYr6-rt7Go3fhJZHlQg52SfPqHqySaexBxlv4xtRwA,6619
+aiq/observability/register.py,sha256=mejMBVr3dHHfShIiyn1fIbA0Gb6z9Ayg8WRMgB0wf5E,7646
 aiq/plugins/.namespace,sha256=Gace0pOC3ETEJf-TBVuNw0TQV6J_KtOPpEiSzMH-odo,215
 aiq/profiler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiq/profiler/data_frame_row.py,sha256=vudqk1ZzZtlZln2Ir43mPl3nwNc0pQlhwbtdY9oSKtI,1755
@@ -277,7 +278,7 @@ aiq/tool/github_tools/get_github_issue.py,sha256=vwLNkNOszLlymkQju0cR8BNvfdH4Enm
 aiq/tool/github_tools/get_github_pr.py,sha256=b7eCOqrVoejGjRwmUVdU45uF07ihbY8lRacMYOSgMrY,9716
 aiq/tool/github_tools/update_github_issue.py,sha256=TUElxUuzjZr_QldL_48RcqSx0A9b23NB_lA82QwFjkM,4103
 aiq/tool/mcp/__init__.py,sha256=GUJrgGtpvyMUCjUBvR3faAdv-tZzbU9W-izgx9aMEQg,680
-aiq/tool/mcp/mcp_client.py,sha256=HWyYsbpA5IysWWdB3XipWzqCfYUio7cJWymt4TbQKyM,7496
+aiq/tool/mcp/mcp_client.py,sha256=lYbf669ATqGKkL0jjd76r0aAtAFnWeruWw-lOPsmYu8,8103
 aiq/tool/mcp/mcp_tool.py,sha256=rQQcaCT-GHQcDmG5weX-2Y-HxBPX-0cC73LjL1u0FUU,4009
 aiq/tool/memory_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiq/tool/memory_tools/add_memory_tool.py,sha256=9EjB3DpYhxwasz7o3O8Rq__Ys5986fciv44ahC6mVCo,3349
@@ -308,10 +309,10 @@ aiq/utils/reactive/base/observer_base.py,sha256=UAlyAY_ky4q2t0P81RVFo2Bs_R7z5Nde
 aiq/utils/reactive/base/subject_base.py,sha256=Ed-AC6P7cT3qkW1EXjzbd5M9WpVoeN_9KCe3OM3FLU4,2521
 aiq/utils/settings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiq/utils/settings/global_settings.py,sha256=U9TCLdoZsKq5qOVGjREipGVv9e-FlStzqy5zv82_VYk,7454
-aiqtoolkit-1.2.0a20250612.dist-info/licenses/LICENSE-3rd-party.txt,sha256=8o7aySJa9CBvFshPcsRdJbczzdNyDGJ8b0J67WRUQ2k,183936
-aiqtoolkit-1.2.0a20250612.dist-info/licenses/LICENSE.md,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-aiqtoolkit-1.2.0a20250612.dist-info/METADATA,sha256=1SStcP6v0If09VOfwJk8W0EHi7waeCGhJ8KVMW2Q5-o,20274
-aiqtoolkit-1.2.0a20250612.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-aiqtoolkit-1.2.0a20250612.dist-info/entry_points.txt,sha256=gRlPfR5g21t328WNEQ4CcEz80S1sJNS8A7rMDYnzl4A,452
-aiqtoolkit-1.2.0a20250612.dist-info/top_level.txt,sha256=fo7AzYcNhZ_tRWrhGumtxwnxMew4xrT1iwouDy_f0Kc,4
-aiqtoolkit-1.2.0a20250612.dist-info/RECORD,,
+aiqtoolkit-1.2.0a20250613.dist-info/licenses/LICENSE-3rd-party.txt,sha256=8o7aySJa9CBvFshPcsRdJbczzdNyDGJ8b0J67WRUQ2k,183936
+aiqtoolkit-1.2.0a20250613.dist-info/licenses/LICENSE.md,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+aiqtoolkit-1.2.0a20250613.dist-info/METADATA,sha256=0LV-fg4UXDznF9C1ojoVD1qrvT1Spoc0w7duaBn_QVI,20274
+aiqtoolkit-1.2.0a20250613.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+aiqtoolkit-1.2.0a20250613.dist-info/entry_points.txt,sha256=gRlPfR5g21t328WNEQ4CcEz80S1sJNS8A7rMDYnzl4A,452
+aiqtoolkit-1.2.0a20250613.dist-info/top_level.txt,sha256=fo7AzYcNhZ_tRWrhGumtxwnxMew4xrT1iwouDy_f0Kc,4
+aiqtoolkit-1.2.0a20250613.dist-info/RECORD,,