ag2 0.9.1a1__py3-none-any.whl → 0.9.1.post0__py3-none-any.whl

autogen/agentchat/contrib/llava_agent.py

@@ -0,0 +1,187 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from  https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+import json
+import logging
+from typing import Any, Optional, Union
+
+import requests
+
+from ...code_utils import content_str
+from ...formatting_utils import colored
+from ...import_utils import optional_import_block, require_optional_import
+from ...llm_config import LLMConfig
+from ..agent import Agent
+from .img_utils import get_image_data, llava_formatter
+from .multimodal_conversable_agent import MultimodalConversableAgent
+
+with optional_import_block():
+    import replicate
+
+logger = logging.getLogger(__name__)
+
+# we will override the following variables later.
+SEP = "###"
+
+DEFAULT_LLAVA_SYS_MSG = "You are an AI agent and you can view images."
+
+
+class LLaVAAgent(MultimodalConversableAgent):
+    def __init__(
+        self,
+        name: str,
+        system_message: Optional[tuple[str, list]] = DEFAULT_LLAVA_SYS_MSG,
+        *args,
+        **kwargs: Any,
+    ):
+        """Args:
+        name (str): agent name.
+        system_message (str): system message for the ChatCompletion inference.
+            Please override this attribute if you want to reprogram the agent.
+        **kwargs (dict): Please refer to other kwargs in
+            [ConversableAgent](/docs/api-reference/autogen/ConversableAgent#conversableagent).
+        """
+        super().__init__(
+            name,
+            system_message=system_message,
+            *args,
+            **kwargs,
+        )
+
+        assert self.llm_config is not None, "llm_config must be provided."
+        self.register_reply([Agent, None], reply_func=LLaVAAgent._image_reply, position=2)
+
+    def _image_reply(self, messages=None, sender=None, config=None):
+        # Note: we did not use "llm_config" yet.
+
+        if all((messages is None, sender is None)):
+            error_msg = f"Either {messages=} or {sender=} must be provided."
+            logger.error(error_msg)
+            raise AssertionError(error_msg)
+
+        if messages is None:
+            messages = self._oai_messages[sender]
+
+        # The formats for LLaVA and GPT are different. So, we manually handle them here.
+        images = []
+        prompt = content_str(self.system_message) + "\n"
+        for msg in messages:
+            role = "Human" if msg["role"] == "user" else "Assistant"
+            # pdb.set_trace()
+            images += [d["image_url"]["url"] for d in msg["content"] if d["type"] == "image_url"]
+            content_prompt = content_str(msg["content"])
+            prompt += f"{SEP}{role}: {content_prompt}\n"
+        prompt += "\n" + SEP + "Assistant: "
+
+        # TODO: PIL to base64
+        images = [get_image_data(im) for im in images]
+        print(colored(prompt, "blue"))
+
+        out = ""
+        retry = 10
+        while len(out) == 0 and retry > 0:
+            # image names will be inferred automatically from llava_call
+            out = llava_call_binary(
+                prompt=prompt,
+                images=images,
+                config_list=self.llm_config["config_list"],
+                temperature=self.llm_config.get("temperature", 0.5),
+                max_new_tokens=self.llm_config.get("max_new_tokens", 2000),
+            )
+            retry -= 1
+
+        assert out != "", "Empty response from LLaVA."
+
+        return True, out
+
+
+@require_optional_import("replicate", "lmm")
+def _llava_call_binary_with_config(
+    prompt: str,
+    images: list[Any],
+    config: dict[str, Any],
+    max_new_tokens: int = 1000,
+    temperature: float = 0.5,
+    seed: int = 1,
+):
+    if config["base_url"].find("0.0.0.0") >= 0 or config["base_url"].find("localhost") >= 0:
+        llava_mode = "local"
+    else:
+        llava_mode = "remote"
+
+    if llava_mode == "local":
+        headers = {"User-Agent": "LLaVA Client"}
+        pload = {
+            "model": config["model"],
+            "prompt": prompt,
+            "max_new_tokens": max_new_tokens,
+            "temperature": temperature,
+            "stop": SEP,
+            "images": images,
+        }
+
+        response = requests.post(
+            config["base_url"].rstrip("/") + "/worker_generate_stream", headers=headers, json=pload, stream=False
+        )
+
+        for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
+            if chunk:
+                data = json.loads(chunk.decode("utf-8"))
+                output = data["text"].split(SEP)[-1]
+    elif llava_mode == "remote":
+        # The Replicate version of the model only support 1 image for now.
+        img = "data:image/jpeg;base64," + images[0]
+        response = replicate.run(
+            config["base_url"], input={"image": img, "prompt": prompt.replace("<image>", " "), "seed": seed}
+        )
+        # The yorickvp/llava-13b model can stream output as it's running.
+        # The predict method returns an iterator, and you can iterate over that output.
+        output = ""
+        for item in response:
+            # https://replicate.com/yorickvp/llava-13b/versions/2facb4a474a0462c15041b78b1ad70952ea46b5ec6ad29583c0b29dbd4249591/api#output-schema
+            output += item
+
+    # Remove the prompt and the space.
+    output = output.replace(prompt, "").strip().rstrip()
+    return output
+
+
+@require_optional_import("replicate", "lmm")
+def llava_call_binary(
+    prompt: str,
+    images: list[Any],
+    config_list: list[dict[str, Any]],
+    max_new_tokens: int = 1000,
+    temperature: float = 0.5,
+    seed: int = 1,
+):
+    # TODO 1: add caching around the LLaVA call to save compute and cost
+    # TODO 2: add `seed` to ensure reproducibility. The seed is not working now.
+
+    for config in config_list:
+        try:
+            return _llava_call_binary_with_config(prompt, images, config, max_new_tokens, temperature, seed)
+        except Exception as e:
+            print(f"Error: {e}")
+            continue
+
+
+def llava_call(prompt: str, llm_config: Union[LLMConfig, dict]) -> str:
+    """Makes a call to the LLaVA service to generate text based on a given prompt"""
+    prompt, images = llava_formatter(prompt, order_image_tokens=False)
+
+    for im in images:
+        if len(im) == 0:
+            raise RuntimeError("An image is empty!")
+
+    return llava_call_binary(
+        prompt,
+        images,
+        config_list=llm_config["config_list"],
+        max_new_tokens=llm_config.get("max_new_tokens", 2000),
+        temperature=llm_config.get("temperature", 0.5),
+        seed=llm_config.get("seed"),
+    )

autogen/agentchat/contrib/math_user_proxy_agent.py

@@ -0,0 +1,464 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from  https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+import os
+import re
+from time import sleep
+from typing import Any, Callable, Dict, Literal, Optional, Union
+
+from pydantic import BaseModel, root_validator
+
+from ...code_utils import UNKNOWN, execute_code, extract_code, infer_lang
+from ...import_utils import optional_import_block, require_optional_import
+from ...math_utils import get_answer
+from .. import Agent, UserProxyAgent
+
+with optional_import_block() as result:
+    import wolframalpha
+
+PROMPTS = {
+    # default
+    "default": """Let's use Python to solve a math problem.
+
+Query requirements:
+You should always use the 'print' function for the output and use fractions/radical forms instead of decimals.
+You can use packages like sympy to help you.
+You must follow the formats below to write your code:
+```python
+# your code
+```
+
+First state the key idea to solve the problem. You may choose from three ways to solve the problem:
+Case 1: If the problem can be solved with Python code directly, please write a program to solve it. You can enumerate all possible arrangements if needed.
+Case 2: If the problem is mostly reasoning, you can solve it by yourself directly.
+Case 3: If the problem cannot be handled in the above two ways, please follow this process:
+1. Solve the problem step by step (do not over-divide the steps).
+2. Take out any queries that can be asked through Python (for example, any calculations or equations that can be calculated).
+3. Wait for me to give the results.
+4. Continue if you think the result is correct. If the result is invalid or unexpected, please correct your query or reasoning.
+
+After all the queries are run and you get the answer, put the answer in \\boxed{}.
+
+Problem:
+""",
+    # select python or wolfram
+    "two_tools": """Let's use two tools (Python and Wolfram alpha) to solve a math problem.
+
+Query requirements:
+You must follow the formats below to write your query:
+For Wolfram Alpha:
+```wolfram
+# one wolfram query
+```
+For Python:
+```python
+# your code
+```
+When using Python, you should always use the 'print' function for the output and use fractions/radical forms instead of decimals. You can use packages like sympy to help you.
+When using wolfram, give one query in each code block.
+
+Please follow this process:
+1. Solve the problem step by step (do not over-divide the steps).
+2. Take out any queries that can be asked through Python or Wolfram Alpha, select the most suitable tool to be used (for example, any calculations or equations that can be calculated).
+3. Wait for me to give the results.
+4. Continue if you think the result is correct. If the result is invalid or unexpected, please correct your query or reasoning.
+
+After all the queries are run and you get the answer, put the final answer in \\boxed{}.
+
+Problem: """,
+    # use python step by step
+    "python": """Let's use Python to solve a math problem.
+
+Query requirements:
+You should always use the 'print' function for the output and use fractions/radical forms instead of decimals.
+You can use packages like sympy to help you.
+You must follow the formats below to write your code:
+```python
+# your code
+```
+
+Please follow this process:
+1. Solve the problem step by step (do not over-divide the steps).
+2. Take out any queries that can be asked through Python (for example, any calculations or equations that can be calculated).
+3. Wait for me to give the results.
+4. Continue if you think the result is correct. If the result is invalid or unexpected, please correct your query or reasoning.
+
+After all the queries are run and you get the answer, put the answer in \\boxed{}.
+
+Problem: """,
+}
+
+
+def _is_termination_msg_mathchat(message):
+    """Check if a message is a termination message."""
+    if isinstance(message, dict):
+        message = message.get("content")
+        if message is None:
+            return False
+    cb = extract_code(message)
+    contain_code = False
+    for c in cb:
+        if c[0] == "python" or c[0] == "wolfram":
+            contain_code = True
+            break
+    return not contain_code and get_answer(message) is not None and get_answer(message) != ""
+
+
+def _add_print_to_last_line(code):
+    """Add print() to the last line of a string."""
+    # 1. check if there is already a print statement
+    if "print(" in code:
+        return code
+    # 2. extract the last line, enclose it in print() and return the new string
+    lines = code.splitlines()
+    last_line = lines[-1]
+    if "\t" in last_line or "=" in last_line:
+        return code
+    if "=" in last_line:
+        last_line = "print(" + last_line.split(" = ")[0] + ")"
+        lines.append(last_line)
+    else:
+        lines[-1] = "print(" + last_line + ")"
+    # 3. join the lines back together
+    return "\n".join(lines)
+
+
+def _remove_print(code):
+    """Remove all print statements from a string."""
+    lines = code.splitlines()
+    lines = [line for line in lines if not line.startswith("print(")]
+    return "\n".join(lines)
+
+
+class MathUserProxyAgent(UserProxyAgent):
+    """(Experimental) A MathChat agent that can handle math problems."""
+
+    MAX_CONSECUTIVE_AUTO_REPLY = 15  # maximum number of consecutive auto replies (subject to future change)
+    DEFAULT_REPLY = "Continue. Please keep solving the problem until you need to query. (If you get to the answer, put it in \\boxed{}.)"
+
+    def __init__(
+        self,
+        name: Optional[str] = "MathChatAgent",  # default set to MathChatAgent
+        is_termination_msg: Optional[
+            Callable[[Dict[str, Any]], bool]
+        ] = _is_termination_msg_mathchat,  # terminate if \boxed{} in message
+        human_input_mode: Literal["ALWAYS", "NEVER", "TERMINATE"] = "NEVER",  # Fully automated
+        default_auto_reply: Optional[Union[str, dict[str, Any]]] = DEFAULT_REPLY,
+        max_invalid_q_per_step=3,  # a parameter needed in MathChat
+        **kwargs: Any,
+    ):
+        """Args:
+        name (str): name of the agent
+        is_termination_msg (function): a function that takes a message in the form of a dictionary and returns a boolean value indicating if this received message is a termination message.
+            The dict can contain the following keys: "content", "role", "name", "function_call".
+        human_input_mode (str): whether to ask for human inputs every time a message is received.
+            Possible values are "ALWAYS", "TERMINATE", "NEVER".
+            (1) When "ALWAYS", the agent prompts for human input every time a message is received.
+                Under this mode, the conversation stops when the human input is "exit",
+                or when is_termination_msg is True and there is no human input.
+            (2) When "TERMINATE", the agent only prompts for human input only when a termination message is received or
+                the number of auto reply reaches the max_consecutive_auto_reply.
+            (3) (Default) When "NEVER", the agent will never prompt for human input. Under this mode, the conversation stops
+                when the number of auto reply reaches the max_consecutive_auto_reply or when is_termination_msg is True.
+        default_auto_reply (str or dict or None): the default auto reply message when no code execution or llm based reply is generated.
+        max_invalid_q_per_step (int): (ADDED) the maximum number of invalid queries per step.
+        **kwargs (dict): other kwargs in [UserProxyAgent](/docs/api-reference/autogen/UserProxyAgent#userproxyagent).
+        """
+        super().__init__(
+            name=name,
+            is_termination_msg=is_termination_msg,
+            human_input_mode=human_input_mode,
+            default_auto_reply=default_auto_reply,
+            **kwargs,
+        )
+        self.register_reply([Agent, None], MathUserProxyAgent._generate_math_reply, position=2)
+        # fixed var
+        self._max_invalid_q_per_step = max_invalid_q_per_step
+
+        # mutable
+        self._valid_q_count = 0
+        self._total_q_count = 0
+        self._accum_invalid_q_per_step = 0
+        self._previous_code = ""
+        self.last_reply = None
+
+    @staticmethod
+    def message_generator(sender, recipient, context):
+        """Generate a prompt for the assistant agent with the given problem and prompt.
+
+        Args:
+            sender (Agent): the sender of the message.
+            recipient (Agent): the recipient of the message.
+            context (dict): a dictionary with the following fields:
+                problem (str): the problem to be solved.
+                prompt_type (str, Optional): the type of the prompt. Possible values are "default", "python", "wolfram".
+                    (1) "default": the prompt that allows the agent to choose between 3 ways to solve a problem:
+                        1. write a python program to solve it directly.
+                        2. solve it directly without python.
+                        3. solve it step by step with python.
+                    (2) "python":
+                        a simplified prompt from the third way of the "default" prompt, that asks the assistant
+                        to solve the problem step by step with python.
+                    (3) "two_tools":
+                        a simplified prompt similar to the "python" prompt, but allows the model to choose between
+                        Python and Wolfram Alpha to solve the problem.
+                customized_prompt (str, Optional): a customized prompt to be used. If it is not None, the prompt_type will be ignored.
+
+        Returns:
+            str: the generated prompt ready to be sent to the assistant agent.
+        """
+        sender._reset()
+        problem = context.get("problem")
+        prompt_type = context.get("prompt_type", "default")
+        customized_prompt = context.get("customized_prompt", None)
+        if customized_prompt is not None:
+            return customized_prompt + problem
+        return PROMPTS[prompt_type] + problem
+
+    def _reset(self):
+        # super().reset()
+        self._valid_q_count = 0
+        self._total_q_count = 0
+        self._accum_invalid_q_per_step = 0
+        self._previous_code = ""
+        self.last_reply = None
+
+    def execute_one_python_code(self, pycode):
+        """Execute python code blocks.
+
+        Previous python code will be saved and executed together with the new code.
+        the "print" function will also be added to the last line of the code if needed
+        """
+        # Need to replace all "; " with "\n" to avoid syntax error when adding `print` to the last line
+        pycode = pycode.replace("; ", "\n").replace(";", "\n")
+        pycode = self._previous_code + _add_print_to_last_line(pycode)
+
+        return_code, output, _ = execute_code(pycode, **self._code_execution_config, timeout=5)
+        is_success = return_code == 0
+
+        if not is_success:
+            # Remove the file information from the error string
+            pattern = r'File "/[^"]+\.py", line \d+, in .+\n'
+            if isinstance(output, str):
+                output = re.sub(pattern, "", output)
+            output = "Error: " + output
+        elif output == "":
+            # Check if there is any print statement
+            if "print" not in pycode:
+                output = "No output found. Make sure you print the results."
+                is_success = False
+            else:
+                output = "No output found."
+                is_success = True
+
+        if len(output) > 2000:
+            output = "Your requested query response is too long. You might have made a mistake. Please revise your reasoning and query."
+            is_success = False
+
+        if is_success:
+            # remove print and check if it still works
+            tmp = self._previous_code + "\n" + _remove_print(pycode) + "\n"
+            rcode, _, _ = execute_code(tmp, **self._code_execution_config)
+        else:
+            # only add imports and check if it works
+            tmp = self._previous_code + "\n"
+            for line in pycode.split("\n"):
+                if "import" in line:
+                    tmp += line + "\n"
+            rcode, _, _ = execute_code(tmp, **self._code_execution_config)
+
+        if rcode == 0:
+            self._previous_code = tmp
+        return output, is_success
+
+    def execute_one_wolfram_query(self, query: str):
+        """Run one wolfram query and return the output.
+
+        Args:
+            query: string of the query.
+
+        Returns:
+            output: string with the output of the query.
+            is_success: boolean indicating whether the query was successful.
+        """
+        # wolfram query handler
+        wolfram = WolframAlphaAPIWrapper()
+        output, is_success = wolfram.run(query)
+        if output == "":
+            output = "Error: The wolfram query is invalid."
+            is_success = False
+        return output, is_success
+
+    def _generate_math_reply(
+        self,
+        messages: Optional[list[dict]] = None,
+        sender: Optional[Agent] = None,
+        config: Optional[Any] = None,
+    ):
+        """Generate an auto reply."""
+        if messages is None:
+            messages = self._oai_messages[sender]
+        message = messages[-1]
+        message = message.get("content", "")
+        code_blocks = extract_code(message)
+
+        if len(code_blocks) == 1 and code_blocks[0][0] == UNKNOWN:
+            # no code block is found, lang should be `UNKNOWN``
+            return True, self._default_auto_reply
+        is_success, all_success = True, True
+        reply = ""
+        for code_block in code_blocks:
+            lang, code = code_block
+            if not lang:
+                lang = infer_lang(code)
+            if lang == "python":
+                output, is_success = self.execute_one_python_code(code)
+            elif lang == "wolfram":
+                output, is_success = self.execute_one_wolfram_query(code)
+            else:
+                output = "Error: Unknown language."
+                is_success = False
+
+            reply += output + "\n"
+            if not is_success:
+                all_success = False
+                self._valid_q_count -= 1  # count invalid queries
+
+        reply = reply.strip()
+
+        if self.last_reply == reply:
+            return True, reply + "\nYour query or result is same from the last, please try a new approach."
+        self.last_reply = reply
+
+        if not all_success:
+            self._accum_invalid_q_per_step += 1
+            if self._accum_invalid_q_per_step > self._max_invalid_q_per_step:
+                self._accum_invalid_q_per_step = 0
+                reply = "Please revisit the problem statement and your reasoning. If you think this step is correct, solve it yourself and continue the next step. Otherwise, correct this step."
+
+        return True, reply
+
+
+# Modified based on langchain. Langchain is licensed under MIT License:
+# The MIT License
+
+# Copyright (c) Harrison Chase
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+def get_from_dict_or_env(data: dict[str, Any], key: str, env_key: str, default: Optional[str] = None) -> str:
+    """Get a value from a dictionary or an environment variable."""
+    if data.get(key):
+        return data[key]
+    elif os.environ.get(env_key):
+        return os.environ[env_key]
+    elif default is not None:
+        return default
+    else:
+        raise ValueError(
+            f"Did not find {key}, please add an environment variable"
+            f" `{env_key}` which contains it, or pass"
+            f"  `{key}` as a named parameter."
+        )
+
+
+class WolframAlphaAPIWrapper(BaseModel):
+    """Wrapper for Wolfram Alpha.
+
+    Docs for using:
+
+    1. Go to wolfram alpha and sign up for a developer account
+    2. Create an app and get your APP ID
+    3. Save your APP ID into WOLFRAM_ALPHA_APPID env variable
+    4. pip install wolframalpha
+
+    """
+
+    wolfram_client: Any  #: :meta private:
+    wolfram_alpha_appid: Optional[str] = None
+
+    @root_validator(skip_on_failure=True)
+    @classmethod
+    @require_optional_import("wolframalpha", "mathchat")
+    def validate_environment(cls, values: dict) -> dict:
+        """Validate that api key and python package exists in environment."""
+        wolfram_alpha_appid = get_from_dict_or_env(values, "wolfram_alpha_appid", "WOLFRAM_ALPHA_APPID")
+        values["wolfram_alpha_appid"] = wolfram_alpha_appid
+
+        client = wolframalpha.Client(wolfram_alpha_appid)
+        values["wolfram_client"] = client
+
+        return values
+
+    def run(self, query: str) -> tuple[str, bool]:
+        """Run query through WolframAlpha and parse result."""
+        from urllib.error import HTTPError
+
+        is_success = False  # added
+        res = None
+        for _ in range(20):
+            try:
+                res = self.wolfram_client.query(query)
+                break
+            except HTTPError:
+                sleep(1)
+            except Exception:
+                return (
+                    "Wolfram Alpha wasn't able to answer it. Please try a new query for wolfram or use python.",
+                    is_success,
+                )
+        if res is None:
+            return (
+                "Wolfram Alpha wasn't able to answer it (may due to web error), you can try again or use python.",
+                is_success,
+            )
+
+        try:
+            if not res["@success"]:
+                return (
+                    "Your Wolfram query is invalid. Please try a new query for wolfram or use python.",
+                    is_success,
+                )
+            assumption = next(res.pods).text
+            answer = ""
+            for result in res["pod"]:
+                if result["@title"] == "Solution":
+                    answer = result["subpod"]["plaintext"]
+                if result["@title"] == "Results" or result["@title"] == "Solutions":
+                    for i, sub in enumerate(result["subpod"]):
+                        answer += f"ans {i}: " + sub["plaintext"] + "\n"
+                    break
+            if answer == "":
+                answer = next(res.results).text
+
+        except Exception:
+            return (
+                "Wolfram Alpha wasn't able to answer it. Please try a new query for wolfram or use python.",
+                is_success,
+            )
+
+        if answer is None or answer == "":
+            # We don't want to return the assumption alone if answer is empty
+            return "No good Wolfram Alpha Result was found", is_success
+        is_success = True
+        return f"Assumption: {assumption} \nAnswer: {answer}", is_success