ag2 0.9.1__py3-none-any.whl → 0.9.1.post0__py3-none-any.whl

autogen/agentchat/assistant_agent.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from  https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from typing import Any, Callable, Literal, Optional, Union
+
+from ..doc_utils import export_module
+from ..llm_config import LLMConfig
+from ..runtime_logging import log_new_agent, logging_enabled
+from .conversable_agent import ConversableAgent
+
+
+@export_module("autogen")
+class AssistantAgent(ConversableAgent):
+    """(In preview) Assistant agent, designed to solve a task with LLM.
+
+    AssistantAgent is a subclass of ConversableAgent configured with a default system message.
+    The default system message is designed to solve a task with LLM,
+    including suggesting python code blocks and debugging.
+    `human_input_mode` is default to "NEVER"
+    and `code_execution_config` is default to False.
+    This agent doesn't execute code by default, and expects the user to execute the code.
+    """
+
+    DEFAULT_SYSTEM_MESSAGE = """You are a helpful AI assistant.
+Solve tasks using your coding and language skills.
+In the following cases, suggest python code (in a python coding block) or shell script (in a sh coding block) for the user to execute.
+    1. When you need to collect info, use the code to output the info you need, for example, browse or search the web, download/read a file, print the content of a webpage or a file, get the current date/time, check the operating system. After sufficient info is printed and the task is ready to be solved based on your language skill, you can solve the task by yourself.
+    2. When you need to perform some task with code, use the code to perform the task and output the result. Finish the task smartly.
+Solve the task step by step if you need to. If a plan is not provided, explain your plan first. Be clear which step uses code, and which step uses your language skill.
+When using code, you must indicate the script type in the code block. The user cannot provide any other feedback or perform any other action beyond executing the code you suggest. The user can't modify your code. So do not suggest incomplete code which requires users to modify. Don't use a code block if it's not intended to be executed by the user.
+If you want the user to save the code in a file before executing it, put # filename: <filename> inside the code block as the first line. Don't include multiple code blocks in one response. Do not ask users to copy and paste the result. Instead, use 'print' function for the output when relevant. Check the execution result returned by the user.
+If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try.
+When you find an answer, verify the answer carefully. Include verifiable evidence in your response if possible.
+Reply "TERMINATE" in the end when everything is done.
+    """
+
+    DEFAULT_DESCRIPTION = "A helpful and general-purpose AI assistant that has strong language skills, Python skills, and Linux command line skills."
+
+    def __init__(
+        self,
+        name: str,
+        system_message: Optional[str] = DEFAULT_SYSTEM_MESSAGE,
+        llm_config: Optional[Union[LLMConfig, dict[str, Any], Literal[False]]] = None,
+        is_termination_msg: Optional[Callable[[dict[str, Any]], bool]] = None,
+        max_consecutive_auto_reply: Optional[int] = None,
+        human_input_mode: Literal["ALWAYS", "NEVER", "TERMINATE"] = "NEVER",
+        description: Optional[str] = None,
+        **kwargs: Any,
+    ):
+        """Args:
+        name (str): agent name.
+        system_message (str): system message for the ChatCompletion inference.
+            Please override this attribute if you want to reprogram the agent.
+        llm_config (dict or False or None): llm inference configuration.
+            Please refer to [OpenAIWrapper.create](https://docs.ag2.ai/latest/docs/api-reference/autogen/OpenAIWrapper/#autogen.OpenAIWrapper.create)
+            for available options.
+        is_termination_msg (function): a function that takes a message in the form of a dictionary
+            and returns a boolean value indicating if this received message is a termination message.
+            The dict can contain the following keys: "content", "role", "name", "function_call".
+        max_consecutive_auto_reply (int): the maximum number of consecutive auto replies.
+            default to None (no limit provided, class attribute MAX_CONSECUTIVE_AUTO_REPLY will be used as the limit in this case).
+            The limit only plays a role when human_input_mode is not "ALWAYS".
+        **kwargs (dict): Please refer to other kwargs in
+            [ConversableAgent](https://docs.ag2.ai/latest/docs/api-reference/autogen/ConversableAgent).
+        """
+        super().__init__(
+            name,
+            system_message,
+            is_termination_msg,
+            max_consecutive_auto_reply,
+            human_input_mode,
+            llm_config=llm_config,
+            description=description,
+            **kwargs,
+        )
+        if logging_enabled():
+            log_new_agent(self, locals())
+
+        # Update the provided description if None, and we are using the default system_message,
+        # then use the default description.
+        if description is None and system_message == self.DEFAULT_SYSTEM_MESSAGE:
+            self.description = self.DEFAULT_DESCRIPTION

autogen/agentchat/chat.py

@@ -0,0 +1,309 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from  https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+import asyncio
+import datetime
+import logging
+import warnings
+from collections import defaultdict
+from dataclasses import dataclass
+from functools import partial
+from typing import Any
+
+from ..doc_utils import export_module
+from ..events.agent_events import PostCarryoverProcessingEvent
+from ..io.base import IOStream
+from .utils import consolidate_chat_info
+
+logger = logging.getLogger(__name__)
+Prerequisite = tuple[int, int]
+
+__all__ = ["ChatResult", "a_initiate_chats", "initiate_chats"]
+
+
+@dataclass
+@export_module("autogen")
+class ChatResult:
+    """(Experimental) The result of a chat. Almost certain to be changed."""
+
+    chat_id: int = None
+    """chat id"""
+    chat_history: list[dict[str, Any]] = None
+    """The chat history."""
+    summary: str = None
+    """A summary obtained from the chat."""
+    cost: dict[str, dict[str, Any]] = (
+        None  # keys: "usage_including_cached_inference", "usage_excluding_cached_inference"
+    )
+    """The cost of the chat.
+       The value for each usage type is a dictionary containing cost information for that specific type.
+           - "usage_including_cached_inference": Cost information on the total usage, including the tokens in cached inference.
+           - "usage_excluding_cached_inference": Cost information on the usage of tokens, excluding the tokens in cache. No larger than "usage_including_cached_inference".
+    """
+    human_input: list[str] = None
+    """A list of human input solicited during the chat."""
+
+
+def _validate_recipients(chat_queue: list[dict[str, Any]]) -> None:
+    """Validate recipients exits and warn repetitive recipients."""
+    receipts_set = set()
+    for chat_info in chat_queue:
+        assert "recipient" in chat_info, "recipient must be provided."
+        receipts_set.add(chat_info["recipient"])
+    if len(receipts_set) < len(chat_queue):
+        warnings.warn(
+            "Repetitive recipients detected: The chat history will be cleared by default if a recipient appears more than once. To retain the chat history, please set 'clear_history=False' in the configuration of the repeating agent.",
+            UserWarning,
+        )
+
+
+def __create_async_prerequisites(chat_queue: list[dict[str, Any]]) -> list[Prerequisite]:
+    """Create list of Prerequisite (prerequisite_chat_id, chat_id)"""
+    prerequisites = []
+    for chat_info in chat_queue:
+        if "chat_id" not in chat_info:
+            raise ValueError("Each chat must have a unique id for async multi-chat execution.")
+        chat_id = chat_info["chat_id"]
+        pre_chats = chat_info.get("prerequisites", [])
+        for pre_chat_id in pre_chats:
+            if not isinstance(pre_chat_id, int):
+                raise ValueError("Prerequisite chat id is not int.")
+            prerequisites.append((chat_id, pre_chat_id))
+    return prerequisites
+
+
+def __find_async_chat_order(chat_ids: set[int], prerequisites: list[Prerequisite]) -> list[int]:
+    """Find chat order for async execution based on the prerequisite chats
+
+    Args:
+        chat_ids: A set of all chat IDs that need to be scheduled
+        prerequisites: A list of tuples (chat_id, prerequisite_chat_id) where each tuple indicates that chat_id depends on prerequisite_chat_id
+
+    Returns:
+        list: a list of chat_id in order.
+    """
+    edges = defaultdict(set)
+    indegree = defaultdict(int)
+    for pair in prerequisites:
+        chat, pre = pair[0], pair[1]
+        if chat not in edges[pre]:
+            indegree[chat] += 1
+            edges[pre].add(chat)
+    bfs = [i for i in chat_ids if i not in indegree]
+    chat_order = []
+    steps = len(indegree)
+    for _ in range(steps + 1):
+        if not bfs:
+            break
+        chat_order.extend(bfs)
+        nxt = []
+        for node in bfs:
+            if node in edges:
+                for course in edges[node]:
+                    indegree[course] -= 1
+                    if indegree[course] == 0:
+                        nxt.append(course)
+                        indegree.pop(course)
+                edges.pop(node)
+        bfs = nxt
+
+    if indegree:
+        return []
+    return chat_order
+
+
+def _post_process_carryover_item(carryover_item):
+    if isinstance(carryover_item, str):
+        return carryover_item
+    elif isinstance(carryover_item, dict) and "content" in carryover_item:
+        return str(carryover_item["content"])
+    else:
+        return str(carryover_item)
+
+
+def __post_carryover_processing(chat_info: dict[str, Any]) -> None:
+    iostream = IOStream.get_default()
+
+    if "message" not in chat_info:
+        warnings.warn(
+            "message is not provided in a chat_queue entry. input() will be called to get the initial message.",
+            UserWarning,
+        )
+
+    iostream.send(PostCarryoverProcessingEvent(chat_info=chat_info))
+
+
+@export_module("autogen")
+def initiate_chats(chat_queue: list[dict[str, Any]]) -> list[ChatResult]:
+    """Initiate a list of chats.
+
+    Args:
+        chat_queue (List[Dict]): A list of dictionaries containing the information about the chats.
+
+            Each dictionary should contain the input arguments for
+            [`ConversableAgent.initiate_chat`](../ConversableAgent#initiate-chat).
+            For example:
+                - `"sender"` - the sender agent.
+                - `"recipient"` - the recipient agent.
+                - `"clear_history"` (bool) - whether to clear the chat history with the agent.
+                Default is True.
+                - `"silent"` (bool or None) - (Experimental) whether to print the messages in this
+                conversation. Default is False.
+                - `"cache"` (Cache or None) - the cache client to use for this conversation.
+                Default is None.
+                - `"max_turns"` (int or None) - maximum number of turns for the chat. If None, the chat
+                will continue until a termination condition is met. Default is None.
+                - `"summary_method"` (str or callable) - a string or callable specifying the method to get
+                a summary from the chat. Default is DEFAULT_summary_method, i.e., "last_msg".
+                - `"summary_args"` (dict) - a dictionary of arguments to be passed to the summary_method.
+                Default is {}.
+                - `"message"` (str, callable or None) - if None, input() will be called to get the
+                initial message.
+                - `**context` - additional context information to be passed to the chat.
+                - `"carryover"` - It can be used to specify the carryover information to be passed
+                to this chat. If provided, we will combine this carryover with the "message" content when
+                generating the initial chat message in `generate_init_message`.
+                - `"finished_chat_indexes_to_exclude_from_carryover"` - It can be used by specifying a list of indexes of the finished_chats list,
+                from which to exclude the summaries for carryover. If 'finished_chat_indexes_to_exclude_from_carryover' is not provided or an empty list,
+                then summary from all the finished chats will be taken.
+
+    Returns:
+        (list): a list of ChatResult objects corresponding to the finished chats in the chat_queue.
+    """
+    consolidate_chat_info(chat_queue)
+    _validate_recipients(chat_queue)
+    current_chat_queue = chat_queue.copy()
+    finished_chats = []
+    while current_chat_queue:
+        chat_info = current_chat_queue.pop(0)
+        _chat_carryover = chat_info.get("carryover", [])
+        finished_chat_indexes_to_exclude_from_carryover = chat_info.get(
+            "finished_chat_indexes_to_exclude_from_carryover", []
+        )
+
+        if isinstance(_chat_carryover, str):
+            _chat_carryover = [_chat_carryover]
+        chat_info["carryover"] = _chat_carryover + [
+            r.summary for i, r in enumerate(finished_chats) if i not in finished_chat_indexes_to_exclude_from_carryover
+        ]
+
+        if not chat_info.get("silent", False):
+            __post_carryover_processing(chat_info)
+
+        sender = chat_info["sender"]
+        chat_res = sender.initiate_chat(**chat_info)
+        finished_chats.append(chat_res)
+    return finished_chats
+
+
+def __system_now_str():
+    ct = datetime.datetime.now()
+    return f" System time at {ct}. "
+
+
+def _on_chat_future_done(chat_future: asyncio.Future, chat_id: int):
+    """Update ChatResult when async Task for Chat is completed."""
+    logger.debug(f"Update chat {chat_id} result on task completion." + __system_now_str())
+    chat_result = chat_future.result()
+    chat_result.chat_id = chat_id
+
+
+async def _dependent_chat_future(
+    chat_id: int, chat_info: dict[str, Any], prerequisite_chat_futures: dict[int, asyncio.Future]
+) -> asyncio.Task:
+    """Create an async Task for each chat."""
+    logger.debug(f"Create Task for chat {chat_id}." + __system_now_str())
+    _chat_carryover = chat_info.get("carryover", [])
+    finished_chat_indexes_to_exclude_from_carryover = chat_info.get(
+        "finished_chat_indexes_to_exclude_from_carryover", []
+    )
+    finished_chats = dict()
+    for chat in prerequisite_chat_futures:
+        chat_future = prerequisite_chat_futures[chat]
+        if chat_future.cancelled():
+            raise RuntimeError(f"Chat {chat} is cancelled.")
+
+        # wait for prerequisite chat results for the new chat carryover
+        finished_chats[chat] = await chat_future
+
+    if isinstance(_chat_carryover, str):
+        _chat_carryover = [_chat_carryover]
+    data = [
+        chat_result.summary
+        for chat_id, chat_result in finished_chats.items()
+        if chat_id not in finished_chat_indexes_to_exclude_from_carryover
+    ]
+    chat_info["carryover"] = _chat_carryover + data
+    if not chat_info.get("silent", False):
+        __post_carryover_processing(chat_info)
+
+    sender = chat_info["sender"]
+    chat_res_future = asyncio.create_task(sender.a_initiate_chat(**chat_info))
+    call_back_with_args = partial(_on_chat_future_done, chat_id=chat_id)
+    chat_res_future.add_done_callback(call_back_with_args)
+    logger.debug(f"Task for chat {chat_id} created." + __system_now_str())
+    return chat_res_future
+
+
+async def a_initiate_chats(chat_queue: list[dict[str, Any]]) -> dict[int, ChatResult]:
+    """(async) Initiate a list of chats.
+
+    Args:
+        chat_queue (List[Dict]): A list of dictionaries containing the information about the chats.
+
+            Each dictionary should contain the input arguments for
+            [`ConversableAgent.initiate_chat`](../../../ConversableAgent#initiate-chat).
+            For example:
+                - `"sender"` - the sender agent.
+                - `"recipient"` - the recipient agent.
+                - `"clear_history"` (bool) - whether to clear the chat history with the agent.
+                Default is True.
+                - `"silent"` (bool or None) - (Experimental) whether to print the messages in this
+                conversation. Default is False.
+                - `"cache"` (Cache or None) - the cache client to use for this conversation.
+                Default is None.
+                - `"max_turns"` (int or None) - maximum number of turns for the chat. If None, the chat
+                will continue until a termination condition is met. Default is None.
+                - `"summary_method"` (str or callable) - a string or callable specifying the method to get
+                a summary from the chat. Default is DEFAULT_summary_method, i.e., "last_msg".
+                - `"summary_args"` (dict) - a dictionary of arguments to be passed to the summary_method.
+                Default is {}.
+                - `"message"` (str, callable or None) - if None, input() will be called to get the
+                initial message.
+                - `**context` - additional context information to be passed to the chat.
+                - `"carryover"` - It can be used to specify the carryover information to be passed
+                to this chat. If provided, we will combine this carryover with the "message" content when
+                generating the initial chat message in `generate_init_message`.
+                - `"finished_chat_indexes_to_exclude_from_carryover"` - It can be used by specifying a list of indexes of the finished_chats list,
+                from which to exclude the summaries for carryover. If 'finished_chat_indexes_to_exclude_from_carryover' is not provided or an empty list,
+                then summary from all the finished chats will be taken.
+
+
+    Returns:
+        - (Dict): a dict of ChatId: ChatResult corresponding to the finished chats in the chat_queue.
+    """
+    consolidate_chat_info(chat_queue)
+    _validate_recipients(chat_queue)
+    chat_book = {chat_info["chat_id"]: chat_info for chat_info in chat_queue}
+    num_chats = chat_book.keys()
+    prerequisites = __create_async_prerequisites(chat_queue)
+    chat_order_by_id = __find_async_chat_order(num_chats, prerequisites)
+    finished_chat_futures = dict()
+    for chat_id in chat_order_by_id:
+        chat_info = chat_book[chat_id]
+        prerequisite_chat_ids = chat_info.get("prerequisites", [])
+        pre_chat_futures = dict()
+        for pre_chat_id in prerequisite_chat_ids:
+            pre_chat_future = finished_chat_futures[pre_chat_id]
+            pre_chat_futures[pre_chat_id] = pre_chat_future
+        current_chat_future = await _dependent_chat_future(chat_id, chat_info, pre_chat_futures)
+        finished_chat_futures[chat_id] = current_chat_future
+    await asyncio.gather(*list(finished_chat_futures.values()))
+    finished_chats = dict()
+    for chat in finished_chat_futures:
+        chat_result = finished_chat_futures[chat].result()
+        finished_chats[chat] = chat_result
+    return finished_chats

autogen/agentchat/contrib/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+__all__: list[str] = []

autogen/agentchat/contrib/agent_eval/README.md

@@ -0,0 +1,7 @@
+Agents for running the [AgentEval](https://docs.ag2.ai/latest/docs/blog/2023/11/20/AgentEval/) pipeline.
+
+AgentEval is a process for evaluating a LLM-based system's performance on a given task.
+
+When given a task to evaluate and a few example runs, the critic and subcritic agents create evaluation criteria for evaluating a system's solution. Once the criteria have been created, the quantifier agent can evaluate subsequent task solutions based on the generated criteria.
+
+See our [blog post](https://docs.ag2.ai/blog/2024-06-21-AgentEval) for usage examples and general explanations.

autogen/agentchat/contrib/agent_eval/agent_eval.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from  https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from typing import Any, Literal, Optional, Union
+
+from .... import GroupChat, GroupChatManager, UserProxyAgent
+from ....llm_config import LLMConfig
+from .criterion import Criterion
+from .critic_agent import CriticAgent
+from .quantifier_agent import QuantifierAgent
+from .subcritic_agent import SubCriticAgent
+from .task import Task
+
+
+def generate_criteria(
+    llm_config: Optional[Union[LLMConfig, dict[str, Any], Literal[False]]] = None,
+    task: Task = None,
+    additional_instructions: str = "",
+    max_round=2,
+    use_subcritic: bool = False,
+):
+    """Creates a list of criteria for evaluating the utility of a given task.
+
+    Args:
+        llm_config (LLMConfig or dict or bool): llm inference configuration.
+        task (Task): The task to evaluate.
+        additional_instructions (str): Additional instructions for the criteria agent.
+        max_round (int): The maximum number of rounds to run the conversation.
+        use_subcritic (bool): Whether to use the subcritic agent to generate subcriteria.
+
+    Returns:
+        list: A list of Criterion objects for evaluating the utility of the given task.
+    """
+    critic = CriticAgent(
+        system_message=CriticAgent.DEFAULT_SYSTEM_MESSAGE + "\n" + additional_instructions,
+        llm_config=llm_config,
+    )
+
+    critic_user = UserProxyAgent(
+        name="critic_user",
+        max_consecutive_auto_reply=0,  # terminate without auto-reply
+        human_input_mode="NEVER",
+        code_execution_config={"use_docker": False},
+    )
+
+    agents = [critic_user, critic]
+
+    if use_subcritic:
+        subcritic = SubCriticAgent(
+            llm_config=llm_config,
+        )
+        agents.append(subcritic)
+
+    groupchat = GroupChat(agents=agents, messages=[], max_round=max_round, speaker_selection_method="round_robin")
+    critic_manager = GroupChatManager(groupchat=groupchat, llm_config=llm_config)
+
+    critic_user.initiate_chat(critic_manager, message=task.get_sys_message())
+    criteria = critic_user.last_message()
+    content = criteria["content"]
+    # need to strip out any extra code around the returned json
+    content = content[content.find("[") : content.rfind("]") + 1]
+    criteria = Criterion.parse_json_str(content)
+    return criteria
+
+
+def quantify_criteria(
+    llm_config: Optional[Union[LLMConfig, dict[str, Any], Literal[False]]] = None,
+    criteria: list[Criterion] = None,
+    task: Task = None,
+    test_case: str = "",
+    ground_truth: str = "",
+):
+    """Quantifies the performance of a system using the provided criteria.
+
+    Args:
+        llm_config (LLMConfig or dict or bool): llm inference configuration.
+        criteria ([Criterion]): A list of criteria for evaluating the utility of a given task.
+        task (Task): The task to evaluate.
+        test_case (str): The test case to evaluate.
+        ground_truth (str): The ground truth for the test case.
+
+    Returns:
+        dict: A dictionary where the keys are the criteria and the values are the assessed performance based on accepted values for each criteria.
+    """
+    quantifier = QuantifierAgent(
+        llm_config=llm_config,
+    )
+
+    quantifier_user = UserProxyAgent(
+        name="quantifier_user",
+        max_consecutive_auto_reply=0,  # terminate without auto-reply
+        human_input_mode="NEVER",
+        code_execution_config={"use_docker": False},
+    )
+
+    quantifier_user.initiate_chat(
+        quantifier,
+        message=task.get_sys_message()
+        + "Evaluation dictionary: "
+        + Criterion.write_json(criteria)
+        + "actual test case to evaluate: "
+        + test_case,
+    )
+    quantified_results = quantifier_user.last_message()
+    return {"actual_success": ground_truth, "estimated_performance": quantified_results["content"]}

autogen/agentchat/contrib/agent_eval/criterion.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from  https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from __future__ import annotations
+
+import json
+
+from pydantic import BaseModel
+
+
+class Criterion(BaseModel):
+    """A class that represents a criterion for agent evaluation."""
+
+    name: str
+    description: str
+    accepted_values: list[str]
+    sub_criteria: list[Criterion] = list()
+
+    @staticmethod
+    def parse_json_str(criteria: str):
+        """Create a list of Criterion objects from a json string.
+
+        Args:
+            criteria (str): Json string that represents the criteria
+        returns:
+            [Criterion]: A list of Criterion objects that represents the json criteria information.
+        """
+        return [Criterion(**crit) for crit in json.loads(criteria)]
+
+    @staticmethod
+    def write_json(criteria):
+        """Create a json string from a list of Criterion objects.
+
+        Args:
+            criteria ([Criterion]): A list of Criterion objects.
+
+        Returns:
+            str: A json string that represents the list of Criterion objects.
+        """
+        return json.dumps([crit.model_dump() for crit in criteria], indent=2)

autogen/agentchat/contrib/agent_eval/critic_agent.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from  https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from typing import Any, Optional
+
+from ...conversable_agent import ConversableAgent
+
+
+class CriticAgent(ConversableAgent):
+    """An agent for creating list of criteria for evaluating the utility of a given task."""
+
+    DEFAULT_SYSTEM_MESSAGE = """You are a helpful assistant. You suggest criteria for evaluating different tasks. They should be distinguishable, quantifiable and not redundant.
+    Convert the evaluation criteria into a list where each item is a criteria which consists of the following dictionary as follows
+    {"name": name of the criterion, "description": criteria description , "accepted_values": possible accepted inputs for this key}
+    Make sure "accepted_values" include the acceptable inputs for each key that are fine-grained and preferably multi-graded levels and "description" includes the criterion description.
+    Output just the criteria string you have created, no code.
+    """
+
+    DEFAULT_DESCRIPTION = "An AI agent for creating list criteria for evaluating the utility of a given task."
+
+    def __init__(
+        self,
+        name="critic",
+        system_message: Optional[str] = DEFAULT_SYSTEM_MESSAGE,
+        description: Optional[str] = DEFAULT_DESCRIPTION,
+        **kwargs: Any,
+    ):
+        """Args:
+        name (str): agent name.
+        system_message (str): system message for the ChatCompletion inference.
+            Please override this attribute if you want to reprogram the agent.
+        description (str): The description of the agent.
+        **kwargs (dict): Please refer to other kwargs in
+            [ConversableAgent](/docs/api-reference/autogen/ConversableAgent#conversableagent).
+        """
+        super().__init__(
+            name=name,
+            system_message=system_message,
+            description=description,
+            **kwargs,
+        )

autogen/agentchat/contrib/agent_eval/quantifier_agent.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from  https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from typing import Any, Optional
+
+from ...conversable_agent import ConversableAgent
+
+
+class QuantifierAgent(ConversableAgent):
+    """An agent for quantifying the performance of a system using the provided criteria."""
+
+    DEFAULT_SYSTEM_MESSAGE = """"You are a helpful assistant. You quantify the output of different tasks based on the given criteria.
+    The criterion is given in a json list format where each element is a distinct criteria.
+    The each element is a dictionary as follows {"name": name of the criterion, "description": criteria description , "accepted_values": possible accepted inputs for this key}
+    You are going to quantify each of the crieria for a given task based on the task description.
+    Return a dictionary where the keys are the criteria and the values are the assessed performance based on accepted values for each criteria.
+    Return only the dictionary, no code."""
+
+    DEFAULT_DESCRIPTION = "An AI agent for quantifing the performance of a system using the provided criteria."
+
+    def __init__(
+        self,
+        name="quantifier",
+        system_message: Optional[str] = DEFAULT_SYSTEM_MESSAGE,
+        description: Optional[str] = DEFAULT_DESCRIPTION,
+        **kwargs: Any,
+    ):
+        """Args:
+        name (str): agent name.
+        system_message (str): system message for the ChatCompletion inference.
+            Please override this attribute if you want to reprogram the agent.
+        description (str): The description of the agent.
+        **kwargs (dict): Please refer to other kwargs in
+            [ConversableAgent](/docs/api-reference/autogen/ConversableAgent#conversableagent).
+        """
+        super().__init__(name=name, system_message=system_message, description=description, **kwargs)