vision-agent 0.2.56__py3-none-any.whl → 0.2.58__py3-none-any.whl

vision_agent/tools/__init__.py

@@ -11,6 +11,7 @@ from .tools import (
     closest_box_distance,
     closest_mask_distance,
     extract_frames,
+    get_tool_documentation,
     grounding_dino,
     grounding_sam,
     image_caption,
@@ -37,7 +38,7 @@ def register_tool(imports: Optional[List] = None) -> Callable:
     def decorator(tool: Callable) -> Callable:
         import inspect
 
-        from .tools import get_tool_descriptions, get_tool_documentation, get_tools_df
+        from .tools import get_tool_descriptions, get_tools_df
 
         global TOOLS, TOOLS_DF, TOOL_DESCRIPTIONS, TOOL_DOCSTRING
 

vision_agent/tools/tools.py

@@ -316,14 +316,14 @@ def visual_prompt_counting(
     return resp_data
 
 
-def image_question_answering(image: np.ndarray, prompt: str) -> str:
+def image_question_answering(prompt: str, image: np.ndarray) -> str:
     """'image_question_answering_' is a tool that can answer questions about the visual
     contents of an image given a question and an image. It returns an answer to the
     question
 
     Parameters:
-        image (np.ndarray): The reference image used for the question
         prompt (str): The question about the image
+        image (np.ndarray): The reference image used for the question
 
     Returns:
         str: A string which is the answer to the given prompt. E.g. {'text': 'This
@@ -331,7 +331,7 @@ def image_question_answering(image: np.ndarray, prompt: str) -> str:
 
     Example
     -------
-        >>> image_question_answering(image, 'What is the cat doing ?')
+        >>> image_question_answering('What is the cat doing ?', image)
         'drinking milk'
     """
 

{vision_agent-0.2.56.dist-info → vision_agent-0.2.58.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.56
+Version: 0.2.58
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -77,6 +77,7 @@ export OPENAI_API_KEY="your-api-key"
 ```
 
 ### Vision Agent
+#### Basic Usage
 You can interact with the agent as you would with any LLM or LMM model:
 
 ```python
@@ -125,10 +126,12 @@ mode by passing in the verbose argument:
 >>> agent = VisionAgent(verbose=2)
 ```
 
-You can also have it return more information by calling `chat_with_workflow`:
+#### Detailed Usage
+You can also have it return more information by calling `chat_with_workflow`. The format
+of the input is a list of dictionaries with the keys `role`, `content`, and `media`:
 
 ```python
->>> results = agent.chat_with_workflow([{"role": "user", "content": "What percentage of the area of the jar is filled with coffee beans?"}], media="jar.jpg")
+>>> results = agent.chat_with_workflow([{"role": "user", "content": "What percentage of the area of the jar is filled with coffee beans?", "media": ["jar.jpg"]}])
 >>> print(results)
 {
     "code": "from vision_agent.tools import ..."
@@ -139,19 +142,45 @@ You can also have it return more information by calling `chat_with_workflow`:
 }
 ```
 
-With this you can examine more detailed information such as the etesting code, testing
+With this you can examine more detailed information such as the testing code, testing
 results, plan or working memory it used to complete the task.
 
+#### Multi-turn conversations
+You can have multi-turn conversations with vision-agent as well, giving it feedback on
+the code and having it update. You just need to add the code as a response from the
+assistant:
+
+```python
+agent = va.agent.VisionAgent(verbosity=2)
+conv = [
+    {
+        "role": "user",
+        "content": "Are these workers wearing safety gear? Output only a True or False value.",
+        "media": ["workers.png"],
+    }
+]
+result = agent.chat_with_workflow(conv)
+code = result["code"]
+conv.append({"role": "assistant", "content": code})
+conv.append(
+    {
+        "role": "user",
+        "content": "Can you also return the number of workers wearing safety gear?",
+    }
+)
+result = agent.chat_with_workflow(conv)
+```
+
 ### Tools
 There are a variety of tools for the model or the user to use. Some are executed locally
-while others are hosted for you. You can also ask an LLM directly to build a tool for
+while others are hosted for you. You can also ask an LMM directly to build a tool for
 you. For example:
 
 ```python
 >>> import vision_agent as va
->>> llm = va.llm.OpenAILLM()
+>>> llm = va.llm.OpenAILMM()
 >>> detector = llm.generate_detector("Can you build a jar detector for me?")
->>> detector("jar.jpg")
+>>> detector(va.tools.load_image("jar.jpg"))
 [{"labels": ["jar",],
   "scores": [0.99],
   "bboxes": [

vision_agent-0.2.58.dist-info/RECORD

@@ -0,0 +1,23 @@
+vision_agent/__init__.py,sha256=EAb4-f9iyuEYkBrX4ag1syM8Syx8118_t0R6_C34M9w,57
+vision_agent/agent/__init__.py,sha256=IUwfbPMcT8X_rnXMLmI8gJ4ltsHy_XSs9eLiKURJxeY,81
+vision_agent/agent/agent.py,sha256=ZK-5lOtd9-eD9aWcXssJpnOyvZuO7_5hAmnb-6sWVe8,569
+vision_agent/agent/vision_agent.py,sha256=QK3G8YdT5vwW5OIh1PN-2gJECFkVNkey-MdK4YdXo-c,24394
+vision_agent/agent/vision_agent_prompts.py,sha256=bMXdZYf6kbikHn__tCGrYE1QvXC88EmpMpM_97V6szA,8472
+vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
+vision_agent/lmm/__init__.py,sha256=3ro5lCIoS3DgEghOy0SPFrEhYvFnWZpVC5S5kSnIx6A,57
+vision_agent/lmm/lmm.py,sha256=XqixNLuLNYu4-xXA8IOEdlcfgktds1ly6Ov7PiFLdsY,8706
+vision_agent/tools/__init__.py,sha256=inKVLRUATQA9oi83l0NluC8Gm-LJU2-AjA6rL1j12Q8,1532
+vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
+vision_agent/tools/tool_utils.py,sha256=wzRacbUpqk9hhfX_Y08rL8qP0XCN2w-8IZoYLi3Upn4,869
+vision_agent/tools/tools.py,sha256=o9ojTfhu8KCSXfW4UPUNOhmki6A-l3jtVi0rPEnELjc,26944
+vision_agent/utils/__init__.py,sha256=CW84HnhqI6XQVuxf2KifkLnSuO7EOhmuL09-gAymAak,219
+vision_agent/utils/execute.py,sha256=GqoAodxtwTPBr1nujPTsWiZO2rBGvWVXTe8lgxY4d_g,20603
+vision_agent/utils/image_utils.py,sha256=_cdiS5YrLzqkq_ZgFUO897m5M4_SCIThwUy4lOklfB8,7700
+vision_agent/utils/sim.py,sha256=rGRGnjsy91IOn8qzt7k04PIRj5jyiaQyYAQl7ossPt8,4195
+vision_agent/utils/type_defs.py,sha256=BlI8ywWHAplC7kYWLvt4AOdnKpEW3qWEFm-GEOSkrFQ,1792
+vision_agent/utils/video.py,sha256=rNmU9KEIkZB5-EztZNlUiKYN0mm_55A_2VGUM0QpqLA,8779
+vision_agent-0.2.58.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.58.dist-info/METADATA,sha256=aC5l1nZmxjqdpZXyF4qy1gRGgdhwYazJl1kK8aqCU6o,7632
+vision_agent-0.2.58.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.58.dist-info/RECORD,,

vision_agent/agent/agent_coder.py

@@ -1,216 +0,0 @@
-import json
-import logging
-import os
-import sys
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
-
-from rich.console import Console
-from rich.syntax import Syntax
-
-from vision_agent.agent import Agent
-from vision_agent.agent.agent_coder_prompts import (
-    DEBUG,
-    FIX_BUG,
-    PROGRAM,
-    TEST,
-    VISUAL_TEST,
-)
-from vision_agent.llm import LLM, OpenAILLM
-from vision_agent.lmm import LMM, OpenAILMM
-from vision_agent.tools import TOOL_DOCSTRING, UTILITIES_DOCSTRING
-from vision_agent.utils import CodeInterpreterFactory
-
-IMPORT_HELPER = """
-import math
-import re
-import sys
-import copy
-import datetime
-import itertools
-import collections
-import heapq
-import statistics
-import functools
-import hashlib
-import numpy
-import numpy as np
-import string
-from typing import *
-from collections import *
-from vision_agent.tools import *
-"""
-logging.basicConfig(stream=sys.stdout)
-_LOGGER = logging.getLogger(__name__)
-_EXECUTE = CodeInterpreterFactory.get_default_instance()
-_CONSOLE = Console()
-
-
-def write_tests(question: str, code: str, model: LLM) -> str:
-    prompt = TEST.format(
-        question=question,
-        code=code,
-    )
-    completion = model(prompt)
-    return preprocess_data(completion)
-
-
-def preprocess_data(code: str) -> str:
-    if "```python" in code:
-        code = code[code.find("```python") + len("```python") :]
-        code = code[: code.find("```")]
-    return code
-
-
-def parse_file_name(s: str) -> str:
-    # We only output png files
-    return "".join([p for p in s.split(" ") if p.endswith(".png")])
-
-
-def write_program(
-    question: str, feedback: str, model: LLM, media: Optional[Union[str, Path]] = None
-) -> str:
-    prompt = PROGRAM.format(
-        docstring=TOOL_DOCSTRING, question=question, feedback=feedback
-    )
-    if isinstance(model, OpenAILMM):
-        completion = model(prompt, images=[media] if media else None)
-    else:
-        completion = model(prompt)
-
-    return preprocess_data(completion)
-
-
-def write_debug(question: str, code: str, feedback: str, model: LLM) -> str:
-    prompt = DEBUG.format(
-        docstring=UTILITIES_DOCSTRING,
-        code=code,
-        question=question,
-        feedback=feedback,
-    )
-    completion = model(prompt)
-    return preprocess_data(completion)
-
-
-def execute_tests(code: str, tests: str) -> Dict[str, Union[str, bool]]:
-    full_code = f"{IMPORT_HELPER}\n{code}\n{tests}"
-    result = _EXECUTE.exec_isolation(full_code)
-    return {"code": code, "result": result.text(), "passed": result.success}
-
-
-def run_visual_tests(
-    question: str, code: str, viz_file: str, feedback: str, model: LMM
-) -> Dict[str, Union[str, bool]]:
-    prompt = VISUAL_TEST.format(
-        docstring=TOOL_DOCSTRING,
-        code=code,
-        question=question,
-        feedback=feedback,
-    )
-    completion = model(prompt, images=[viz_file])
-    # type is from the prompt
-    return json.loads(completion)  # type: ignore
-
-
-def fix_bugs(code: str, tests: str, result: str, feedback: str, model: LLM) -> str:
-    prompt = FIX_BUG.format(code=code, tests=tests, result=result, feedback=feedback)
-    completion = model(prompt)
-    return preprocess_data(completion)
-
-
-class AgentCoder(Agent):
-    """AgentCoder is based off of the AgentCoder paper https://arxiv.org/abs/2312.13010
-    and it's open source code https://github.com/huangd1999/AgentCoder with some key
-    differences. AgentCoder comprises of 3 components: a coder agent, a tester agent,
-    and an executor. The tester agents writes code to test the code written by the coder
-    agent, but in our case because we are solving a vision task it's difficult to write
-    testing code. We instead have the tester agent write code to visualize the output
-    of the code written by the coder agent. If the code fails, we pass it back to the
-    coder agent to fix the bug, if it succeeds we pass it to a visual tester agent, which
-    is an LMM model like GPT4V, to visually inspect the output and make sure it looks
-    good."""
-
-    def __init__(
-        self,
-        coder_agent: Optional[LLM] = None,
-        tester_agent: Optional[LLM] = None,
-        visual_tester_agent: Optional[LMM] = None,
-        verbose: bool = False,
-    ) -> None:
-        self.coder_agent = (
-            OpenAILLM(temperature=0.1) if coder_agent is None else coder_agent
-        )
-        self.tester_agent = (
-            OpenAILLM(temperature=0.1) if tester_agent is None else tester_agent
-        )
-        self.visual_tester_agent = (
-            OpenAILMM(temperature=0.1, json_mode=True)
-            if visual_tester_agent is None
-            else visual_tester_agent
-        )
-        self.max_turns = 3
-        self.verbose = verbose
-        if self.verbose:
-            _LOGGER.setLevel(logging.INFO)
-
-    def __call__(
-        self,
-        input: Union[List[Dict[str, str]], str],
-        media: Optional[Union[str, Path]] = None,
-    ) -> str:
-        if isinstance(input, str):
-            input = [{"role": "user", "content": input}]
-        return self.chat(input, media)
-
-    def chat(
-        self,
-        input: List[Dict[str, str]],
-        media: Optional[Union[str, Path]] = None,
-    ) -> str:
-        question = input[0]["content"]
-        if media:
-            question += f" Input file path: {os.path.abspath(media)}"
-
-        code = ""
-        feedback = ""
-        for _ in range(self.max_turns):
-            code = write_program(question, feedback, self.coder_agent, media=media)
-            if self.verbose:
-                _CONSOLE.print(
-                    Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)
-                )
-            debug = write_debug(question, code, feedback, self.tester_agent)
-            if self.verbose:
-                _CONSOLE.print(
-                    Syntax(debug, "python", theme="gruvbox-dark", line_numbers=True)
-                )
-            results = execute_tests(code, debug)
-            _LOGGER.info(
-                f"execution results: passed: {results['passed']}\n{results['result']}"
-            )
-
-            if not results["passed"]:
-                code = fix_bugs(
-                    code, debug, results["result"].strip(), feedback, self.coder_agent  # type: ignore
-                )
-                if self.verbose:
-                    _CONSOLE.print(
-                        Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)
-                    )
-            else:
-                # TODO: Sometimes it prints nothing, so we need to handle that case
-                # TODO: The visual agent reflection does not work very well, needs more testing
-                # viz_test_results = run_visual_tests(
-                #     question, code, parse_file_name(results["result"].strip()), feedback, self.visual_tester_agent
-                # )
-                # _LOGGER.info(f"visual test results:\n{viz_test_results}")
-                # if viz_test_results["finished"]:
-                #     return f"{IMPORT_HELPER}\n{code}"
-                # feedback += f"\n{viz_test_results['feedback']}"
-
-                return f"{IMPORT_HELPER}\n{code}"
-
-        return f"{IMPORT_HELPER}\n{code}"
-
-    def log_progress(self, data: Dict[str, Any]) -> None:
-        _LOGGER.info(data)

vision_agent/agent/agent_coder_prompts.py

@@ -1,135 +0,0 @@
-PROGRAM = """
-**Role**: You are a software programmer.
-
-**Task**: As a programmer, you are required to complete the function. Use a Chain-of-Thought approach to break down the problem, create pseudocode, and then write the code in Python language. Ensure that your code is efficient, readable, and well-commented. Return the requested information from the function you create.
-
-**Documentation**:
-This is the documentation for the functions you have access to. You may call any of these functions to help you complete the task, you do not need to worry about defining them or importing them and can assume they are available to you.
-{docstring}
-
-**Input Code Snippet**:
-```python
-def execute(image_path: str):
-    # Your code here
-```
-
-**User Instructions**:
-{question}
-
-**Previous Feedback**:
-{feedback}
-
-**Instructions**:
-1. **Understand and Clarify**: Make sure you understand the task.
-2. **Algorithm/Method Selection**: Decide on the most efficient way.
-3. **Pseudocode Creation**: Write down the steps you will follow in pseudocode.
-4. **Code Generation**: Translate your pseudocode into executable Python code.
-"""
-
-DEBUG = """
-**Role**: You are a software programmer.
-
-**Task**: Your task is to run the `execute` function and either print the output or print a file name containing visualized output for another agent to examine. The other agent will then use your output, either the printed return value of the function or the visualized output as a file, to determine if `execute` is functioning correctly.
-
-**Documentation**
-This is the documentation for the functions you have access to. You may call any of these functions to help you complete the task, you do not need to worry about defining them or importing them and can assume they are available to you.
-{docstring}
-
-**Input Code Snippet**:
-```python
-### Please decided how would you want to generate test cases. Based on incomplete code or completed version.
-{code}
-```
-
-**User Instructions**:
-{question}
-
-**Previous Feedback**:
-{feedback}
-
-**Instructions**:
-1. **Understand and Clarify**: Make sure you understand the task.
-2. **Code Execution**: Run the `execute` function with the given input from the user instructions.
-3. **Output Generation**: Print the output or save it as a file for visualization utilizing the functions you have access to.
-"""
-
-VISUAL_TEST = """
-**Role**: You are a machine vision expert.
-
-**Task**: Your task is to visually inspect the output of the `execute` function and determine if the visualization of the function output looks correct given the user's instructions. If not, you can provide suggestions to improve the `execute` function to imporve it.
-
-**Documentation**:
-This is the documentation for the functions you have access to. You may call any of these functions to help you complete the task, you do not need to worry about defining them or importing them and can assume they are available to you.
-{docstring}
-
-
-**Input Code Snippet**:
-This is the code that
-```python
-{code}
-```
-
-**User Instructions**:
-{question}
-
-**Previous Feedback**:
-{feedback}
-
-**Instructions**:
-1. **Visual Inspection**: Examine the visual output of the `execute` function.
-2. **Evaluation**: Determine if the visualization is correct based on the user's instructions.
-3. **Feedback**: Provide feedback on the visualization and suggest improvements if necessary.
-4. **Clear Concrete Instructions**: Provide clear concrete instructions to improve the results. You can only make coding suggestions based on the either the input code snippet or the documented code provided. For example, do not say the threshold needs to be adjust, instead provide an exact value for adjusting the threshold.
-
-Provide output in JSON format {{"finished": boolean, "feedback": "your feedback"}} where "finished" is True if the output is correct and False if not and "feedback" is your feedback.
-"""
-
-FIX_BUG = """
-Please re-complete the code to fix the error message. Here is the previous version:
-```python
-{code}
-```
-
-When we run this code:
-```python
-{tests}
-```
-
-It raises this error:
-```python
-{result}
-```
-
-This is previous feedback provided on the code:
-{feedback}
-
-Please fix the bug by follow the error information and only return python code. You do not need return the test cases. The re-completion code should in triple backticks format(i.e., in ```python ```).
-"""
-
-TEST = """
-**Role**: As a tester, your task is to create comprehensive test cases for the incomplete `execute` function. These test cases should encompass Basic, Edge, and Large Scale scenarios to ensure the code's robustness, reliability, and scalability.
-
-**User Instructions**:
-{question}
-
-**Input Code Snippet**:
-```python
-### Please decided how would you want to generate test cases. Based on incomplete code or completed version.
-{code}
-```
-
-**1. Basic Test Cases**:
-- **Objective**: To verify the fundamental functionality of the `has_close_elements` function under normal conditions.
-
-**2. Edge Test Cases**:
-- **Objective**: To evaluate the function's behavior under extreme or unusual conditions.
-
-**3. Large Scale Test Cases**:
-- **Objective**: To assess the function’s performance and scalability with large data samples.
-
-**Instructions**:
-- Implement a comprehensive set of test cases following the guidelines above.
-- Ensure each test case is well-documented with comments explaining the scenario it covers.
-- Pay special attention to edge cases as they often reveal hidden bugs.
-- For large-scale tests, focus on the function's efficiency and performance under heavy loads.
-"""