vision-agent 0.2.10__py3-none-any.whl → 0.2.22__py3-none-any.whl

vision_agent/agent/__init__.py

@@ -1,4 +1,6 @@
 from .agent import Agent
+from .agent_coder import AgentCoder
 from .easytool import EasyTool
 from .reflexion import Reflexion
 from .vision_agent import VisionAgent
+from .vision_agent_v2 import VisionAgentV2

vision_agent/agent/agent_coder.py

@@ -0,0 +1,196 @@
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+from vision_agent.agent import Agent
+from vision_agent.agent.agent_coder_prompts import (
+    DEBUG,
+    FIX_BUG,
+    PROGRAM,
+    TEST,
+    VISUAL_TEST,
+)
+from vision_agent.llm import LLM, OpenAILLM
+from vision_agent.lmm import LMM, OpenAILMM
+from vision_agent.tools.tools_v2 import TOOL_DOCSTRING, UTILITIES_DOCSTRING
+from vision_agent.utils import Execute
+
+IMPORT_HELPER = """
+import math
+import re
+import sys
+import copy
+import datetime
+import itertools
+import collections
+import heapq
+import statistics
+import functools
+import hashlib
+import numpy
+import numpy as np
+import string
+from typing import *
+from collections import *
+from vision_agent.tools.tools_v2 import *
+"""
+logging.basicConfig(stream=sys.stdout)
+_LOGGER = logging.getLogger(__name__)
+_EXECUTE = Execute()
+
+
+def write_tests(question: str, code: str, model: LLM) -> str:
+    prompt = TEST.format(
+        question=question,
+        code=code,
+    )
+    completion = model(prompt)
+    return preprocess_data(completion)
+
+
+def preprocess_data(code: str) -> str:
+    if "```python" in code:
+        code = code[code.find("```python") + len("```python") :]
+        code = code[: code.find("```")]
+    return code
+
+
+def parse_file_name(s: str) -> str:
+    # We only output png files
+    return "".join([p for p in s.split(" ") if p.endswith(".png")])
+
+
+def write_program(question: str, feedback: str, model: LLM) -> str:
+    prompt = PROGRAM.format(
+        docstring=TOOL_DOCSTRING, question=question, feedback=feedback
+    )
+    completion = model(prompt)
+    return preprocess_data(completion)
+
+
+def write_debug(question: str, code: str, feedback: str, model: LLM) -> str:
+    prompt = DEBUG.format(
+        docstring=UTILITIES_DOCSTRING,
+        code=code,
+        question=question,
+        feedback=feedback,
+    )
+    completion = model(prompt)
+    return preprocess_data(completion)
+
+
+def execute_tests(code: str, tests: str) -> Dict[str, Union[str, bool]]:
+    full_code = f"{IMPORT_HELPER}\n{code}\n{tests}"
+    success, result = _EXECUTE.run_isolation(full_code)
+    return {"code": code, "result": result, "passed": success}
+
+
+def run_visual_tests(
+    question: str, code: str, viz_file: str, feedback: str, model: LMM
+) -> Dict[str, Union[str, bool]]:
+    prompt = VISUAL_TEST.format(
+        docstring=TOOL_DOCSTRING,
+        code=code,
+        question=question,
+        feedback=feedback,
+    )
+    completion = model(prompt, images=[viz_file])
+    # type is from the prompt
+    return json.loads(completion)  # type: ignore
+
+
+def fix_bugs(code: str, tests: str, result: str, feedback: str, model: LLM) -> str:
+    prompt = FIX_BUG.format(completion=code, test_case=tests, result=result)
+    completion = model(prompt)
+    return preprocess_data(completion)
+
+
+class AgentCoder(Agent):
+    """AgentCoder is based off of the AgentCoder paper https://arxiv.org/abs/2312.13010
+    and it's open source code https://github.com/huangd1999/AgentCoder with some key
+    differences. AgentCoder comprises of 3 components: a coder agent, a tester agent,
+    and an executor. The tester agents writes code to test the code written by the coder
+    agent, but in our case because we are solving a vision task it's difficult to write
+    testing code. We instead have the tester agent write code to visualize the output
+    of the code written by the coder agent. If the code fails, we pass it back to the
+    coder agent to fix the bug, if it succeeds we pass it to a visual tester agent, which
+    is an LMM model like GPT4V, to visually inspect the output and make sure it looks
+    good."""
+
+    def __init__(
+        self,
+        coder_agent: Optional[LLM] = None,
+        tester_agent: Optional[LLM] = None,
+        visual_tester_agent: Optional[LMM] = None,
+        verbose: bool = False,
+    ) -> None:
+        self.coder_agent = (
+            OpenAILLM(temperature=0.1) if coder_agent is None else coder_agent
+        )
+        self.tester_agent = (
+            OpenAILLM(temperature=0.1) if tester_agent is None else tester_agent
+        )
+        self.visual_tester_agent = (
+            OpenAILMM(temperature=0.1, json_mode=True)
+            if visual_tester_agent is None
+            else visual_tester_agent
+        )
+        self.max_turns = 3
+        if verbose:
+            _LOGGER.setLevel(logging.INFO)
+
+    def __call__(
+        self,
+        input: Union[List[Dict[str, str]], str],
+        image: Optional[Union[str, Path]] = None,
+    ) -> str:
+        if isinstance(input, str):
+            input = [{"role": "user", "content": input}]
+        return self.chat(input, image)
+
+    def chat(
+        self,
+        input: List[Dict[str, str]],
+        image: Optional[Union[str, Path]] = None,
+    ) -> str:
+        question = input[0]["content"]
+        if image:
+            question += f" Input file path: {os.path.abspath(image)}"
+
+        code = ""
+        feedback = ""
+        for _ in range(self.max_turns):
+            code = write_program(question, feedback, self.coder_agent)
+            _LOGGER.info(f"code:\n{code}")
+            debug = write_debug(question, code, feedback, self.tester_agent)
+            _LOGGER.info(f"debug:\n{debug}")
+            results = execute_tests(code, debug)
+            _LOGGER.info(
+                f"execution results: passed: {results['passed']}\n{results['result']}"
+            )
+
+            if not results["passed"]:
+                code = fix_bugs(
+                    code, debug, results["result"].strip(), feedback, self.coder_agent  # type: ignore
+                )
+                _LOGGER.info(f"fixed code:\n{code}")
+            else:
+                # TODO: Sometimes it prints nothing, so we need to handle that case
+                # TODO: The visual agent reflection does not work very well, needs more testing
+                # viz_test_results = run_visual_tests(
+                #     question, code, parse_file_name(results["result"].strip()), feedback, self.visual_tester_agent
+                # )
+                # _LOGGER.info(f"visual test results:\n{viz_test_results}")
+                # if viz_test_results["finished"]:
+                #     return f"{IMPORT_HELPER}\n{code}"
+                # feedback += f"\n{viz_test_results['feedback']}"
+
+                return f"{IMPORT_HELPER}\n{code}"
+
+        return f"{IMPORT_HELPER}\n{code}"
+
+    def log_progress(self, description: str) -> None:
+        _LOGGER.info(description)

vision_agent/agent/agent_coder_prompts.py

@@ -0,0 +1,135 @@
+PROGRAM = """
+**Role**: You are a software programmer.
+
+**Task**: As a programmer, you are required to complete the function. Use a Chain-of-Thought approach to break down the problem, create pseudocode, and then write the code in Python language. Ensure that your code is efficient, readable, and well-commented. Return the requested information from the function you create.
+
+**Documentation**:
+This is the documentation for the functions you have access to. You may call any of these functions to help you complete the task, you do not need to worry about defining them or importing them and can assume they are available to you.
+{docstring}
+
+**Input Code Snippet**:
+```python
+def execute(image_path: str):
+    # Your code here
+```
+
+**User Instructions**:
+{question}
+
+**Previous Feedback**:
+{feedback}
+
+**Instructions**:
+1. **Understand and Clarify**: Make sure you understand the task.
+2. **Algorithm/Method Selection**: Decide on the most efficient way.
+3. **Pseudocode Creation**: Write down the steps you will follow in pseudocode.
+4. **Code Generation**: Translate your pseudocode into executable Python code.
+"""
+
+DEBUG = """
+**Role**: You are a software programmer.
+
+**Task**: Your task is to run the `execute` function and either print the output or print a file name containing visualized output for another agent to examine. The other agent will then use your output, either the printed return value of the function or the visualized output as a file, to determine if `execute` is functioning correctly.
+
+**Documentation**
+This is the documentation for the functions you have access to. You may call any of these functions to help you complete the task, you do not need to worry about defining them or importing them and can assume they are available to you.
+{docstring}
+
+**Input Code Snippet**:
+```python
+### Please decided how would you want to generate test cases. Based on incomplete code or completed version.
+{code}
+```
+
+**User Instructions**:
+{question}
+
+**Previous Feedback**:
+{feedback}
+
+**Instructions**:
+1. **Understand and Clarify**: Make sure you understand the task.
+2. **Code Execution**: Run the `execute` function with the given input from the user instructions.
+3. **Output Generation**: Print the output or save it as a file for visualization utilizing the functions you have access to.
+"""
+
+VISUAL_TEST = """
+**Role**: You are a machine vision expert.
+
+**Task**: Your task is to visually inspect the output of the `execute` function and determine if the visualization of the function output looks correct given the user's instructions. If not, you can provide suggestions to improve the `execute` function to imporve it.
+
+**Documentation**:
+This is the documentation for the functions you have access to. You may call any of these functions to help you complete the task, you do not need to worry about defining them or importing them and can assume they are available to you.
+{docstring}
+
+
+**Input Code Snippet**:
+This is the code that
+```python
+{code}
+```
+
+**User Instructions**:
+{question}
+
+**Previous Feedback**:
+{feedback}
+
+**Instructions**:
+1. **Visual Inspection**: Examine the visual output of the `execute` function.
+2. **Evaluation**: Determine if the visualization is correct based on the user's instructions.
+3. **Feedback**: Provide feedback on the visualization and suggest improvements if necessary.
+4. **Clear Concrete Instructions**: Provide clear concrete instructions to improve the results. You can only make coding suggestions based on the either the input code snippet or the documented code provided. For example, do not say the threshold needs to be adjust, instead provide an exact value for adjusting the threshold.
+
+Provide output in JSON format {{"finished": boolean, "feedback": "your feedback"}} where "finished" is True if the output is correct and False if not and "feedback" is your feedback.
+"""
+
+FIX_BUG = """
+Please re-complete the code to fix the error message. Here is the previous version:
+```python
+{code}
+```
+
+When we run this code:
+```python
+{tests}
+```
+
+It raises this error:
+```python
+{result}
+```
+
+This is previous feedback provided on the code:
+{feedback}
+
+Please fix the bug by follow the error information and only return python code. You do not need return the test cases. The re-completion code should in triple backticks format(i.e., in ```python ```).
+"""
+
+TEST = """
+**Role**: As a tester, your task is to create comprehensive test cases for the incomplete `execute` function. These test cases should encompass Basic, Edge, and Large Scale scenarios to ensure the code's robustness, reliability, and scalability.
+
+**User Instructions**:
+{question}
+
+**Input Code Snippet**:
+```python
+### Please decided how would you want to generate test cases. Based on incomplete code or completed version.
+{code}
+```
+
+**1. Basic Test Cases**:
+- **Objective**: To verify the fundamental functionality of the `has_close_elements` function under normal conditions.
+
+**2. Edge Test Cases**:
+- **Objective**: To evaluate the function's behavior under extreme or unusual conditions.
+
+**3. Large Scale Test Cases**:
+- **Objective**: To assess the function’s performance and scalability with large data samples.
+
+**Instructions**:
+- Implement a comprehensive set of test cases following the guidelines above.
+- Ensure each test case is well-documented with comments explaining the scenario it covers.
+- Pay special attention to edge cases as they often reveal hidden bugs.
+- For large-scale tests, focus on the function's efficiency and performance under heavy loads.
+"""

vision_agent/agent/vision_agent.py

@@ -8,18 +8,8 @@ from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
 from PIL import Image
 from tabulate import tabulate
 
-from vision_agent.image_utils import (
-    convert_to_b64,
-    overlay_bboxes,
-    overlay_heat_map,
-    overlay_masks,
-)
-from vision_agent.llm import LLM, OpenAILLM
-from vision_agent.lmm import LMM, OpenAILMM
-from vision_agent.tools import TOOLS
-
-from .agent import Agent
-from .easytool_prompts import (
+from vision_agent.agent.agent import Agent
+from vision_agent.agent.easytool_prompts import (
     ANSWER_GENERATE,
     ANSWER_SUMMARIZE,
     CHOOSE_PARAMETER,
@@ -27,7 +17,7 @@ from .easytool_prompts import (
     TASK_DECOMPOSE,
     TASK_TOPOLOGY,
 )
-from .vision_agent_prompts import (
+from vision_agent.agent.vision_agent_prompts import (
     ANSWER_GENERATE_DEPENDS,
     ANSWER_SUMMARIZE_DEPENDS,
     CHOOSE_PARAMETER_DEPENDS,
@@ -35,6 +25,15 @@ from .vision_agent_prompts import (
     TASK_DECOMPOSE_DEPENDS,
     VISION_AGENT_REFLECTION,
 )
+from vision_agent.llm import LLM, OpenAILLM
+from vision_agent.lmm import LMM, OpenAILMM
+from vision_agent.tools import TOOLS
+from vision_agent.utils.image_utils import (
+    convert_to_b64,
+    overlay_bboxes,
+    overlay_heat_map,
+    overlay_masks,
+)
 
 logging.basicConfig(stream=sys.stdout)
 _LOGGER = logging.getLogger(__name__)
@@ -309,7 +308,7 @@ def _handle_extract_frames(
     # any following processing
     for video_file_output in tool_result["call_results"]:
         # When the video tool is run with wrong parameters, exit the loop
-        if len(video_file_output) < 2:
+        if not isinstance(video_file_output, tuple) or len(video_file_output) < 2:
             break
         for frame, _ in video_file_output:
             image = frame
@@ -465,15 +464,17 @@ class VisionAgent(Agent):
             report_progress_callback: a callback to report the progress of the agent. This is useful for streaming logs in a web application where multiple VisionAgent instances are running in parallel. This callback ensures that the progress are not mixed up.
         """
         self.task_model = (
-            OpenAILLM(json_mode=True, temperature=0.1)
+            OpenAILLM(model_name="gpt-4-turbo", json_mode=True, temperature=0.0)
             if task_model is None
             else task_model
         )
         self.answer_model = (
-            OpenAILLM(temperature=0.1) if answer_model is None else answer_model
+            OpenAILLM(model_name="gpt-4-turbo", temperature=0.0)
+            if answer_model is None
+            else answer_model
         )
         self.reflect_model = (
-            OpenAILMM(json_mode=True, temperature=0.1)
+            OpenAILMM(model_name="gpt-4-turbo", json_mode=True, temperature=0.0)
             if reflect_model is None
             else reflect_model
         )
@@ -489,6 +490,7 @@ class VisionAgent(Agent):
         image: Optional[Union[str, Path]] = None,
         reference_data: Optional[Dict[str, str]] = None,
         visualize_output: Optional[bool] = False,
+        self_reflection: Optional[bool] = True,
     ) -> str:
         """Invoke the vision agent.
 
@@ -501,6 +503,7 @@ class VisionAgent(Agent):
                 {"image": "image.jpg", "mask": "mask.jpg", "bbox": [0.1, 0.2, 0.1, 0.2]}
                 where the bounding box coordinates are normalized.
             visualize_output: Whether to visualize the output.
+            self_reflection: boolean to enable and disable self reflection.
 
         Returns:
             The result of the vision agent in text.
@@ -512,6 +515,7 @@ class VisionAgent(Agent):
             image=image,
             visualize_output=visualize_output,
             reference_data=reference_data,
+            self_reflection=self_reflection,
         )
 
     def log_progress(self, description: str) -> None:
@@ -538,6 +542,7 @@ class VisionAgent(Agent):
         image: Optional[Union[str, Path]] = None,
         reference_data: Optional[Dict[str, str]] = None,
         visualize_output: Optional[bool] = False,
+        self_reflection: Optional[bool] = True,
     ) -> Tuple[str, List[Dict]]:
         """Chat with the vision agent and return the final answer and all tool results.
 
@@ -550,12 +555,16 @@ class VisionAgent(Agent):
                 {"image": "image.jpg", "mask": "mask.jpg", "bbox": [0.1, 0.2, 0.1, 0.2]}
                 where the bounding box coordinates are normalized.
             visualize_output: Whether to visualize the output.
+            self_reflection: boolean to enable and disable self reflection.
 
         Returns:
             A tuple where the first item is the final answer and the second item is a
             list of all the tool results. The last item in the tool results also
             contains the visualized output.
         """
+        if len(chat) == 0:
+            raise ValueError("Input cannot be empty.")
+
         question = chat[0]["content"]
         if image:
             question += f" Image name: {image}"
@@ -625,20 +634,25 @@ class VisionAgent(Agent):
                 reflection_images = [image]
             else:
                 reflection_images = None
-            reflection = self_reflect(
-                self.reflect_model,
-                question,
-                self.tools,
-                all_tool_results,
-                final_answer,
-                reflection_images,
-            )
-            self.log_progress(f"Reflection: {reflection}")
-            parsed_reflection = parse_reflect(reflection)
-            if parsed_reflection["Finish"]:
-                break
+
+            if self_reflection:
+                reflection = self_reflect(
+                    self.reflect_model,
+                    question,
+                    self.tools,
+                    all_tool_results,
+                    final_answer,
+                    reflection_images,
+                )
+                self.log_progress(f"Reflection: {reflection}")
+                parsed_reflection = parse_reflect(reflection)
+                if parsed_reflection["Finish"]:
+                    break
+                else:
+                    reflections += "\n" + parsed_reflection["Reflection"]
             else:
-                reflections += "\n" + parsed_reflection["Reflection"]
+                self.log_progress("Self Reflection skipped based on user request.")
+                break
         # '<ANSWER>' is a symbol to indicate the end of the chat, which is useful for streaming logs.
         self.log_progress(
             f"The Vision Agent has concluded this chat. <ANSWER>{final_answer}</ANSWER>"
@@ -660,12 +674,14 @@ class VisionAgent(Agent):
         image: Optional[Union[str, Path]] = None,
         reference_data: Optional[Dict[str, str]] = None,
         visualize_output: Optional[bool] = False,
+        self_reflection: Optional[bool] = True,
     ) -> str:
         answer, _ = self.chat_with_workflow(
             chat,
             image=image,
             visualize_output=visualize_output,
             reference_data=reference_data,
+            self_reflection=self_reflection,
         )
         return answer
 

vision_agent/agent/vision_agent_prompts.py

@@ -70,7 +70,7 @@ These are the tools you can select to solve the question:
 
 Please note that:
 1. You should only choose one tool from the Tool List to solve this question and it should have maximum chance of solving the question.
-2. You should only choose the tool whose parameters are most relevant to the user's question and are availale as part of the question.
+2. You should only choose the tool whose parameters are most relevant to the user's question and are available as part of the question.
 3. You should choose the tool whose return type is most relevant to the answer of the user's question.
 4. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like:
 
@@ -88,7 +88,7 @@ This is a reflection from a previous failed attempt:
 
 Please note that:
 1. You should only choose one tool from the Tool List to solve this question and it should have maximum chance of solving the question.
-2. You should only choose the tool whose parameters are most relevant to the user's question and are availale as part of the question.
+2. You should only choose the tool whose parameters are most relevant to the user's question and are available as part of the question.
 3. You should choose the tool whose return type is most relevant to the answer of the user's question.
 4. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like:
 
@@ -100,7 +100,7 @@ Output: """
 CHOOSE_PARAMETER_DEPENDS = """Given a user's question and an API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.
 Please note that:
 1. The Example in the API tool documentation can help you better understand the use of the API. Pay attention to the examples which show how to parse the question and extract tool parameters such as prompts and visual inputs.
-2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If there are no paremters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}}
+2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If there are no parameters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}}
 3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.
 4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers for your reference.
 5. If you need to use this API multiple times, please set "Parameters" to a list.