azure-ai-evaluation 1.11.1__tar.gz → 1.11.2__tar.gz

{azure_ai_evaluation-1.11.1 → azure_ai_evaluation-1.11.2}/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release History
 
+## 1.11.2 (2025-10-09)
+
+### Bugs Fixed
+
+- **kwargs in an evaluator signature receives input columns that are not otherwise named in the evaluator's signature
+
 ## 1.11.1 (2025-09-17)
 
 ### Bugs Fixed

{azure_ai_evaluation-1.11.1/azure_ai_evaluation.egg-info → azure_ai_evaluation-1.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: azure-ai-evaluation
-Version: 1.11.1
+Version: 1.11.2
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -413,6 +413,12 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 
 # Release History
 
+## 1.11.2 (2025-10-09)
+
+### Bugs Fixed
+
+- **kwargs in an evaluator signature receives input columns that are not otherwise named in the evaluator's signature
+
 ## 1.11.1 (2025-09-17)
 
 ### Bugs Fixed

{azure_ai_evaluation-1.11.1 → azure_ai_evaluation-1.11.2}/azure/ai/evaluation/_common/utils.py

@@ -659,6 +659,74 @@ def reformat_tool_definitions(tool_definitions, logger=None):
         return tool_definitions
 
 
+def simplify_messages(messages, drop_system=True, drop_tool_calls=False, logger=None):
+    """
+    Simplify a list of conversation messages by keeping only role and content.
+    Optionally filter out system messages and/or tool calls.
+
+    :param messages: List of message dicts (e.g., from query or response)
+    :param drop_system: If True, remove system role messages
+    :param drop_tool_calls: If True, remove tool_call items from assistant content
+    :return: New simplified list of messages
+    """
+    if isinstance(messages, str):
+        return messages
+    try:
+        # Validate input is a list
+        if not isinstance(messages, list):
+            return messages
+
+        simplified_msgs = []
+        for msg in messages:
+            # Ensure msg is a dict
+            if not isinstance(msg, dict):
+                simplified_msgs.append(msg)
+                continue
+
+            role = msg.get("role")
+            content = msg.get("content", [])
+
+            # Drop system message (if should)
+            if drop_system and role == "system":
+                continue
+
+            # Simplify user messages
+            if role == "user":
+                simplified_msg = {
+                    "role": role,
+                    "content": _extract_text_from_content(content),
+                }
+                simplified_msgs.append(simplified_msg)
+                continue
+
+            # Drop tool results (if should)
+            if drop_tool_calls and role == "tool":
+                continue
+
+            # Simplify assistant messages
+            if role == "assistant":
+                simplified_content = _extract_text_from_content(content)
+                # Check if message has content
+                if simplified_content:
+                    simplified_msg = {"role": role, "content": simplified_content}
+                    simplified_msgs.append(simplified_msg)
+                    continue
+
+                # Drop tool calls (if should)
+                if drop_tool_calls and any(c.get("type") == "tool_call" for c in content if isinstance(c, dict)):
+                    continue
+
+            # If we reach here, it means we want to keep the message
+            simplified_msgs.append(msg)
+
+        return simplified_msgs
+
+    except Exception as ex:
+        if logger:
+            logger.debug(f"Error simplifying messages: {str(ex)}. Returning original messages.")
+        return messages
+
+
 def upload(path: str, container_client: ContainerClient, logger=None):
     """Upload files or directories to Azure Blob Storage using a container client.
 

{azure_ai_evaluation-1.11.1 → azure_ai_evaluation-1.11.2}/azure/ai/evaluation/_evaluators/_common/_base_eval.py

@@ -37,6 +37,8 @@ from azure.ai.evaluation._common._experimental import experimental
 
 from ._conversation_aggregators import GetAggregator, GetAggregatorType
 
+import copy
+
 P = ParamSpec("P")
 T = TypeVar("T")
 T_EvalValue = TypeVar("T_EvalValue")
@@ -486,8 +488,12 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
         """
         tool_calls = []
         tool_results_map = {}
-        if isinstance(response, list):
-            for message in response:
+
+        # Work on a deep copy to avoid modifying the original object
+        response_copy = copy.deepcopy(response)
+
+        if isinstance(response_copy, list):
+            for message in response_copy:
                 # Extract tool calls from assistant messages
                 if message.get("role") == "assistant" and isinstance(message.get("content"), list):
                     for content_item in message.get("content"):
@@ -519,7 +525,11 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
         :rtype: Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]
         """
         # Convert inputs into list of evaluable inputs.
-        eval_input_list = self._convert_kwargs_to_eval_input(**kwargs)
+        try:
+            eval_input_list = self._convert_kwargs_to_eval_input(**kwargs)
+        except Exception as e:
+            print(f"Error converting kwargs to eval_input_list: {e}")
+            raise e
         per_turn_results = []
         # Evaluate all inputs.
         for eval_input in eval_input_list:

{azure_ai_evaluation-1.11.1 → azure_ai_evaluation-1.11.2}/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py

@@ -2,7 +2,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 import os, logging
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union, Any, Tuple
 
 from typing_extensions import overload, override
 from azure.ai.evaluation._legacy._adapters._flows import AsyncPrompty
@@ -16,6 +16,7 @@ from ..._common.utils import (
     ErrorCategory,
     construct_prompty_model_config,
     validate_model_config,
+    simplify_messages,
 )
 
 try:
@@ -207,6 +208,42 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
 
         return super().__call__(*args, **kwargs)
 
+    def _has_context(self, eval_input: dict) -> bool:
+        """
+        Return True if eval_input contains a non-empty 'context' field.
+        Treats None, empty strings, empty lists, and lists of empty strings as no context.
+        """
+        context = eval_input.get("context", None)
+        if not context:
+            return False
+        if context == "<>":  # Special marker for no context
+            return False
+        if isinstance(context, list):
+            return any(str(c).strip() for c in context)
+        if isinstance(context, str):
+            return bool(context.strip())
+        return True
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:
+        if "query" not in eval_input:
+            return await super()._do_eval(eval_input)
+
+        contains_context = self._has_context(eval_input)
+
+        simplified_query = simplify_messages(eval_input["query"], drop_tool_calls=contains_context)
+        simplified_response = simplify_messages(eval_input["response"], drop_tool_calls=False)
+
+        # Build simplified input
+        simplified_eval_input = {
+            "query": simplified_query,
+            "response": simplified_response,
+            "context": eval_input["context"],
+        }
+
+        # Replace and call the parent method
+        return await super()._do_eval(simplified_eval_input)
+
     async def _real_call(self, **kwargs):
         """The asynchronous call where real end-to-end evaluation logic is performed.
 
@@ -230,57 +267,73 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
                 raise ex
 
     def _convert_kwargs_to_eval_input(self, **kwargs):
-        if "context" in kwargs or "conversation" in kwargs:
+        if kwargs.get("context") or kwargs.get("conversation"):
             return super()._convert_kwargs_to_eval_input(**kwargs)
-
         query = kwargs.get("query")
         response = kwargs.get("response")
         tool_definitions = kwargs.get("tool_definitions")
 
-        if not query or not response or not tool_definitions:
-            msg = f"{type(self).__name__}: Either 'conversation' or individual inputs must be provided. For Agent groundedness 'query', 'response' and 'tool_definitions' are required."
+        if (not query) or (not response):  # or not tool_definitions:
+            msg = f"{type(self).__name__}: Either 'conversation' or individual inputs must be provided. For Agent groundedness 'query' and 'response' are required."
             raise EvaluationException(
                 message=msg,
                 blame=ErrorBlame.USER_ERROR,
                 category=ErrorCategory.INVALID_VALUE,
                 target=ErrorTarget.GROUNDEDNESS_EVALUATOR,
             )
-
         context = self._get_context_from_agent_response(response, tool_definitions)
-        if not context:
-            raise EvaluationException(
-                message=f"Context could not be extracted from agent response. Supported tools for groundedness are {self._SUPPORTED_TOOLS}. If supported tools are not used groundedness is not calculated.",
-                blame=ErrorBlame.USER_ERROR,
-                category=ErrorCategory.NOT_APPLICABLE,
-                target=ErrorTarget.GROUNDEDNESS_EVALUATOR,
-            )
 
-        return super()._convert_kwargs_to_eval_input(response=response[-1], context=context, query=query)
+        filtered_response = self._filter_file_search_results(response)
+        return super()._convert_kwargs_to_eval_input(response=filtered_response, context=context, query=query)
+
+    def _filter_file_search_results(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Filter out file_search tool results from the messages."""
+        file_search_ids = self._get_file_search_tool_call_ids(messages)
+        return [
+            msg for msg in messages if not (msg.get("role") == "tool" and msg.get("tool_call_id") in file_search_ids)
+        ]
 
     def _get_context_from_agent_response(self, response, tool_definitions):
+        """Extract context text from file_search tool results in the agent response."""
+        NO_CONTEXT = "<>"
         context = ""
         try:
             logger.debug("Extracting context from response")
             tool_calls = self._parse_tools_from_response(response=response)
-            logger.debug(f"Tool Calls parsed successfully : {tool_calls}")
-            if tool_calls:
-                for tool_call in tool_calls:
-                    if isinstance(tool_call, dict) and tool_call.get("type") == "tool_call":
-                        tool_name = tool_call.get("name")
-                        for tool in tool_definitions:
-                            if tool.get("name") == tool_name and tool.get("type") in self._SUPPORTED_TOOLS:
-                                if tool_name == "file_search":
-                                    tool_result = tool_call.get("tool_result")
-                                    if tool_result:
-                                        for result in tool_result:
-                                            content_list = result.get("content")
-                                            if content_list:
-                                                for content in content_list:
-                                                    text = content.get("text")
-                                                    if text:
-                                                        context = context + "\n" + str(text)
+            logger.debug(f"Tool Calls parsed successfully: {tool_calls}")
+
+            if not tool_calls:
+                return NO_CONTEXT
+
+            context_lines = []
+            for tool_call in tool_calls:
+                if not isinstance(tool_call, dict) or tool_call.get("type") != "tool_call":
+                    continue
+
+                tool_name = tool_call.get("name")
+                if tool_name != "file_search":
+                    continue
+
+                # Extract tool results
+                for result in tool_call.get("tool_result", []):
+                    results = result if isinstance(result, list) else [result]
+                    for r in results:
+                        file_name = r.get("file_name", "Unknown file name")
+                        for content in r.get("content", []):
+                            text = content.get("text")
+                            if text:
+                                context_lines.append(f"{file_name}:\n- {text}---\n\n")
+
+            context = "\n".join(context_lines) if len(context_lines) > 0 else None
+
         except Exception as ex:
             logger.debug(f"Error extracting context from agent response : {str(ex)}")
-            context = ""
+            context = None
+
+        context = context if context else NO_CONTEXT
+        return context
 
-        return context if context else None
+    def _get_file_search_tool_call_ids(self, query_or_response):
+        """Return a list of tool_call_ids for file search tool calls."""
+        tool_calls = self._parse_tools_from_response(query_or_response)
+        return [tc.get("tool_call_id") for tc in tool_calls if tc.get("name") == "file_search"]

{azure_ai_evaluation-1.11.1 → azure_ai_evaluation-1.11.2}/azure/ai/evaluation/_evaluators/_groundedness/groundedness_with_query.prompty

@@ -32,52 +32,53 @@ system:
 
 user:
 # Definition
-**Groundedness** refers to how well an answer is anchored in the provided context, evaluating its relevance, accuracy, and completeness based exclusively on that context. It assesses the extent to which the answer directly and fully addresses the question without introducing unrelated or incorrect information. The scale ranges from 1 to 5, with higher numbers indicating greater groundedness.
+**Groundedness** refers to how well an answer is anchored in the provided context, evaluating its relevance, accuracy, and completeness based exclusively on that context. It assesses the extent to which the answer directly and fully addresses the question without introducing unrelated or incorrect information.
+
+> Context is the source of truth for evaluating the response. If it's empty, rely on the tool results in the response and query.
+> Evaluate the groundedness of the response message, not the chat history.
 
 # Ratings
 ## [Groundedness: 1] (Completely Unrelated Response)
-**Definition:** An answer that does not relate to the question or the context in any way. It fails to address the topic, provides irrelevant information, or introduces completely unrelated subjects.
+**Definition:** An answer that does not relate to the question or the context in any way.
+  - Does not relate to the question or context at all.
+  - Talks about the general topic but does not respond to the query.
 
 **Examples:**
   **Context:** The company's annual meeting will be held next Thursday.
   **Query:** When is the company's annual meeting?
   **Response:** I enjoy hiking in the mountains during summer.
 
-  **Context:** The new policy aims to reduce carbon emissions by 20% over the next five years.
-  **Query:** What is the goal of the new policy?
-  **Response:** My favorite color is blue.
-
-## [Groundedness: 2] (Related Topic but Does Not Respond to the Query)
-**Definition:** An answer that relates to the general topic of the context but does not answer the specific question asked. It may mention concepts from the context but fails to provide a direct or relevant response.
-
-**Examples:**
   **Context:** The museum will exhibit modern art pieces from various local artists.
   **Query:** What kind of art will be exhibited at the museum?
   **Response:** Museums are important cultural institutions.
 
-  **Context:** The new software update improves battery life and performance.
-  **Query:** What does the new software update improve?
-  **Response:** Software updates can sometimes fix bugs.
-
-## [Groundedness: 3] (Attempts to Respond but Contains Incorrect Information)
-**Definition:** An answer that attempts to respond to the question but includes incorrect information not supported by the context. It may misstate facts, misinterpret the context, or provide erroneous details.
+## [Groundedness: 2] (Attempts to Respond but Contains Incorrect Information)
+**Definition:** An answer that attempts to respond to the question but includes incorrect information not supported by the context. It may misstate facts, misinterpret the context, or provide erroneous details. Even if some points are correct, the presence of inaccuracies makes the response unreliable.
 
 **Examples:**
-  **Context:** The festival starts on June 5th and features international musicians.
+  **Context:** - The festival starts on June 5th and features international musicians.
   **Query:** When does the festival start?
   **Response:** The festival starts on July 5th and features local artists.
 
-  **Context:** The recipe requires two eggs and one cup of milk.
-  **Query:** How many eggs are needed for the recipe?
-  **Response:** You need three eggs for the recipe.
+  **Context:** bakery_menu.txt: - Croissant au Beurre — flaky, buttery croissant
+  **Query:** [{"role":"user","content":"Are there croissants?"}]
+  **Response:** [{"role":"assistant","content":"Yes, Croissant au Beurre is on the menu, served with jam."}]
+
+## [Groundedness: 3] (Nothing to be Grounded)
+Definition: An answer that does not provide any information that can be evaluated against the context. This includes responses that are asking for clarification, providing polite fillers, or follow-up questions.
+
+**Examples:**
+  **Context:**
+  **Query:** [{"role":"user","content":"How many eggs are needed for the recipe?"}, {"role":"tool","content":"tool_result": [{"file_name": "recipe.txt", "content": "The recipe requires two eggs and one cup of milk."}]}, {"role":"assistant","content":"You need three eggs for the recipe."}, {"role":"user","content":"Thank you."}]
+  **Response:** [{"role":"assistant","content":"You're welcome, anything else I can help with?"}]
 
 ## [Groundedness: 4] (Partially Correct Response)
 **Definition:** An answer that provides a correct response to the question but is incomplete or lacks specific details mentioned in the context. It captures some of the necessary information but omits key elements needed for a full understanding.
 
 **Examples:**
-  **Context:** The bookstore offers a 15% discount to students and a 10% discount to senior citizens.
-  **Query:** What discount does the bookstore offer to students?
-  **Response:** Students get a discount at the bookstore.
+  **Context:** - store_details.txt: The bookstore offers a 15% discount to students and a 10% discount to senior citizens.
+  **Query:** [{"role":"user","content":"What discount does the bookstore offer to students, if any?"}]
+  **Response:** [{"role":"assistant","content":"Yes, students get a discount at the bookstore."}]
 
   **Context:** The company's headquarters are located in Berlin, Germany.
   **Query:** Where are the company's headquarters?
@@ -87,13 +88,13 @@ user:
 **Definition:** An answer that thoroughly and accurately responds to the question, including all relevant details from the context. It directly addresses the question with precise information, demonstrating complete understanding without adding extraneous information.
 
 **Examples:**
-  **Context:** The author released her latest novel, 'The Silent Echo', on September 1st.
-  **Query:** When was 'The Silent Echo' released?
-  **Response:** 'The Silent Echo' was released on September 1st.
+  **CONTEXT:** The author released her latest novel, 'The Silent Echo', on September 1st.
+  **QUERY:** [{"role":"user","content":"When was 'The Silent Echo' released?"}]
+  **RESPONSE:** [{"role":"assistant","content":"The 'Silent Echo' was released on September 1st."}]
 
-  **Context:** Participants must register by May 31st to be eligible for early bird pricing.
+  **Context:** 
   **Query:** By what date must participants register to receive early bird pricing?
-  **Response:** Participants must register by May 31st to receive early bird pricing.
+  **Response:** [{"role":"tool","content":"tool_result": [{"file_name": "store_guidelines.txt", "content": "Participants registering before and including May 31st will be eligible for early bird pricing."}]}, {"role":"assistant","content":"Participants must register by May 31st to receive early bird pricing."}]
 
 
 # Data
@@ -103,7 +104,7 @@ RESPONSE: {{response}}
 
 
 # Tasks
-## Please provide your assessment Score for the previous RESPONSE in relation to the CONTEXT and QUERY based on the Definitions above. Your output should include the following information:
+## Please provide your assessment Score for the previous RESPONSE message in relation to the CONTEXT, QUERY and RESPONSE tools based on the Definitions above. Your output should include the following information:
 - **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
 - **Explanation**: a very short explanation of why you think the input Data should get that Score.
 - **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.

{azure_ai_evaluation-1.11.1 → azure_ai_evaluation-1.11.2}/azure/ai/evaluation/_legacy/_batch_engine/_engine.py

@@ -344,8 +344,13 @@ class BatchEngine:
 
         func_params = inspect.signature(self._func).parameters
 
-        filtered_params = {key: value for key, value in inputs.items() if key in func_params}
-        return filtered_params
+        has_kwargs = any(p.kind == p.VAR_KEYWORD for p in func_params.values())
+
+        if has_kwargs:
+            return inputs
+        else:
+            filtered_params = {key: value for key, value in inputs.items() if key in func_params}
+            return filtered_params
 
     async def _exec_line_async(
         self,

{azure_ai_evaluation-1.11.1 → azure_ai_evaluation-1.11.2}/azure/ai/evaluation/_version.py

@@ -3,4 +3,4 @@
 # ---------------------------------------------------------
 # represents upcoming version
 
-VERSION = "1.11.1"
+VERSION = "1.11.2"

{azure_ai_evaluation-1.11.1 → azure_ai_evaluation-1.11.2/azure_ai_evaluation.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: azure-ai-evaluation
-Version: 1.11.1
+Version: 1.11.2
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -413,6 +413,12 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 
 # Release History
 
+## 1.11.2 (2025-10-09)
+
+### Bugs Fixed
+
+- **kwargs in an evaluator signature receives input columns that are not otherwise named in the evaluator's signature
+
 ## 1.11.1 (2025-09-17)
 
 ### Bugs Fixed

{azure_ai_evaluation-1.11.1 → azure_ai_evaluation-1.11.2}/tests/e2etests/test_builtin_evaluators.py

@@ -1233,7 +1233,6 @@ class TestBuiltInEvaluators:
         "evaluator_cls",
         [
             RelevanceEvaluator,
-            GroundednessEvaluator,
             FluencyEvaluator,
             SimilarityEvaluator,
             CoherenceEvaluator,

{azure_ai_evaluation-1.11.1 → azure_ai_evaluation-1.11.2}/tests/unittests/test_built_in_evaluator.py

@@ -201,27 +201,6 @@ class TestBuiltInEvaluators:
         assert result["groundedness"] == result["gpt_groundedness"] == 1
         assert "groundedness_reason" in result
 
-    def test_groundedness_evaluator_no_supported_tools(self, mock_model_config):
-        """Test GroundednessEvaluator when no supported tools are used"""
-        groundedness_eval = GroundednessEvaluator(model_config=mock_model_config)
-        groundedness_eval._flow = MagicMock(return_value=quality_response_async_mock())
-
-        result = groundedness_eval(
-            query="What is the capital of Japan?",
-            response=[
-                {"role": "user", "content": "What is the capital of Japan?"},
-                {"role": "assistant", "content": "The capital of Japan is Tokyo."},
-            ],
-            tool_definitions=[
-                {"name": "unsupported_tool", "type": "unsupported", "description": "An unsupported tool"}
-            ],
-        )
-
-        # When no supported tools are used, it should return "not applicable" result
-        assert result["groundedness"] == "not applicable"
-        assert result["groundedness_result"] == "pass"
-        assert "Supported tools for groundedness are" in result["groundedness_reason"]
-
     def test_groundedness_evaluator_with_context(self, mock_model_config):
         """Test GroundednessEvaluator with direct context (traditional use)"""
         groundedness_eval = GroundednessEvaluator(model_config=mock_model_config)
@@ -244,11 +223,10 @@ class TestBuiltInEvaluators:
         with pytest.raises(EvaluationException) as exc_info:
             groundedness_eval(
                 query="What is the capital of Japan?",
-                response=[{"role": "assistant", "content": "The capital of Japan is Tokyo."}],
-                # Missing tool_definitions
+                # Missing response
             )
 
         assert (
-            "Either 'conversation' or individual inputs must be provided. For Agent groundedness 'query', 'response' and 'tool_definitions' are required."
+            "Either 'conversation' or individual inputs must be provided. For Agent groundedness 'query' and 'response' are required."
             in exc_info.value.args[0]
         )

{azure_ai_evaluation-1.11.1 → azure_ai_evaluation-1.11.2}/tests/unittests/test_evaluate.py

@@ -975,6 +975,82 @@ class TestEvaluate:
         assert result[EvaluationRunProperties.NAME_MAP_LENGTH] == -1
         assert len(result) == 1
 
+    def test_evaluate_evaluator_only_kwargs_param(self, evaluate_test_data_jsonl_file):
+        """Validate that an evaluator with only an **kwargs param receives all input in kwargs."""
+
+        def evaluator(**kwargs):
+            return locals()
+
+        result = evaluate(data=evaluate_test_data_jsonl_file, evaluators={"test": evaluator})
+
+        assert len(result["rows"]) == 3
+
+        assert {"query", "response", "ground_truth", "context"}.issubset(result["rows"][0]["outputs.test.kwargs"])
+        assert {"query", "response", "ground_truth", "context"}.issubset(result["rows"][1]["outputs.test.kwargs"])
+        assert {"query", "response", "ground_truth", "context"}.issubset(result["rows"][2]["outputs.test.kwargs"])
+
+    def test_evaluate_evaluator_kwargs_param(self, evaluate_test_data_jsonl_file):
+        """Validate that an evaluator with named parameters and **kwargs obeys python function call semantics."""
+
+        def evaluator(query, response, *, bar=None, **kwargs):
+            return locals()
+
+        result = evaluate(data=evaluate_test_data_jsonl_file, evaluators={"test": evaluator})
+
+        assert len(result["rows"]) == 3
+
+        row1_kwargs = result["rows"][0]["outputs.test.kwargs"]
+        row2_kwargs = result["rows"][1]["outputs.test.kwargs"]
+        row3_kwargs = result["rows"][2]["outputs.test.kwargs"]
+
+        assert {"ground_truth", "context"}.issubset(row1_kwargs), "Unnamed parameters should be in kwargs"
+        assert {"query", "response", "bar"}.isdisjoint(row1_kwargs), "Named parameters should not be in kwargs"
+
+        assert {"ground_truth", "context"}.issubset(row2_kwargs), "Unnamed parameters should be in kwargs"
+        assert {"query", "response", "bar"}.isdisjoint(row2_kwargs), "Named parameters should not be in kwargs"
+
+        assert {"ground_truth", "context"}.issubset(row3_kwargs), "Unnamed parameters should be in kwargs"
+        assert {"query", "response", "bar"}.isdisjoint(row3_kwargs), "Named parameters should not be in kwargs"
+
+    def test_evaluate_evaluator_kwargs_param_column_mapping(self, evaluate_test_data_jsonl_file):
+        """Validate that an evaluator with kwargs can receive column mapped values."""
+
+        def evaluator(query, response, *, bar=None, **kwargs):
+            return locals()
+
+        result = evaluate(
+            data=evaluate_test_data_jsonl_file,
+            evaluators={"test": evaluator},
+            evaluator_config={
+                "default": {
+                    "column_mapping": {
+                        "query": "${data.query}",
+                        "response": "${data.response}",
+                        "foo": "${data.context}",
+                        "bar": "${data.ground_truth}",
+                    }
+                }
+            },
+        )
+
+        assert len(result["rows"]) == 3
+
+        row1_kwargs = result["rows"][0]["outputs.test.kwargs"]
+        row2_kwargs = result["rows"][1]["outputs.test.kwargs"]
+        row3_kwargs = result["rows"][2]["outputs.test.kwargs"]
+
+        assert {"ground_truth", "context"}.issubset(row1_kwargs), "Unnamed parameters should be in kwargs"
+        assert "foo" in row1_kwargs, "Making a column mapping to an unnamed parameter should appear in kwargs"
+        assert {"query", "response", "bar"}.isdisjoint(row1_kwargs), "Named parameters should not be in kwargs"
+
+        assert {"ground_truth", "context"}.issubset(row2_kwargs), "Unnamed parameters should be in kwargs"
+        assert "foo" in row2_kwargs, "Making a column mapping to an unnamed parameter should appear in kwargs"
+        assert {"query", "response", "bar"}.isdisjoint(row2_kwargs), "Named parameters should not be in kwargs"
+
+        assert {"ground_truth", "context"}.issubset(row3_kwargs), "Unnamed parameters should be in kwargs"
+        assert "foo" in row3_kwargs, "Making a column mapping to an unnamed parameter should appear in kwargs"
+        assert {"query", "response", "bar"}.isdisjoint(row3_kwargs), "Named parameters should not be in kwargs"
+
 
 @pytest.mark.unittest
 class TestTagsInLoggingFunctions: