vision-agent 0.2.27__tar.gz → 0.2.29__tar.gz

{vision_agent-0.2.27 → vision_agent-0.2.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.27
+Version: 0.2.29
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.27 → vision_agent-0.2.29}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "vision-agent"
-version = "0.2.27"
+version = "0.2.29"
 description = "Toolset for Vision Agent"
 authors = ["Landing AI <dev@landing.ai>"]
 readme = "README.md"

{vision_agent-0.2.27 → vision_agent-0.2.29}/vision_agent/agent/agent.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union, Any
 
 
 class Agent(ABC):
@@ -13,7 +13,7 @@ class Agent(ABC):
         pass
 
     @abstractmethod
-    def log_progress(self, description: str) -> None:
+    def log_progress(self, data: Dict[str, Any]) -> None:
         """Log the progress of the agent.
         This is a hook that is intended for reporting the progress of the agent.
         """

{vision_agent-0.2.27 → vision_agent-0.2.29}/vision_agent/agent/agent_coder.py

@@ -3,7 +3,7 @@ import logging
 import os
 import sys
 from pathlib import Path
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union, Any
 
 from rich.console import Console
 from rich.syntax import Syntax
@@ -206,5 +206,5 @@ class AgentCoder(Agent):
 
         return f"{IMPORT_HELPER}\n{code}"
 
-    def log_progress(self, description: str) -> None:
-        _LOGGER.info(description)
+    def log_progress(self, data: Dict[str, Any]) -> None:
+        _LOGGER.info(data)

{vision_agent-0.2.27 → vision_agent-0.2.29}/vision_agent/agent/vision_agent.py

@@ -451,7 +451,7 @@ class VisionAgent(Agent):
         reflect_model: Optional[Union[LLM, LMM]] = None,
         max_retries: int = 2,
         verbose: bool = False,
-        report_progress_callback: Optional[Callable[[str], None]] = None,
+        report_progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
     ):
         """VisionAgent constructor.
 
@@ -518,23 +518,23 @@ class VisionAgent(Agent):
             self_reflection=self_reflection,
         )
 
-    def log_progress(self, description: str) -> None:
-        _LOGGER.info(description)
+    def log_progress(self, data: Dict[str, Any]) -> None:
+        _LOGGER.info(data)
         if self.report_progress_callback:
-            self.report_progress_callback(description)
+            self.report_progress_callback(data)
 
     def _report_visualization_via_callback(
         self, images: Sequence[Union[str, Path]]
     ) -> None:
         """This is intended for streaming the visualization images via the callback to the client side."""
         if self.report_progress_callback:
-            self.report_progress_callback("<VIZ>")
+            self.report_progress_callback({"log": "<VIZ>"})
             if images:
                 for img in images:
                     self.report_progress_callback(
-                        f"<IMG>base:64{convert_to_b64(img)}</IMG>"
+                        {"log": f"<IMG>base:64{convert_to_b64(img)}</IMG>"}
                     )
-            self.report_progress_callback("</VIZ>")
+            self.report_progress_callback({"log": "</VIZ>"})
 
     def chat_with_workflow(
         self,
@@ -618,8 +618,8 @@ class VisionAgent(Agent):
                 tool_results["answer"] = answer
                 all_tool_results.append(tool_results)
 
-                self.log_progress(f"\tCall Result: {call_results}")
-                self.log_progress(f"\tAnswer: {answer}")
+                self.log_progress({"log": f"\tCall Result: {call_results}"})
+                self.log_progress({"log": f"\tAnswer: {answer}"})
                 answers.append({"task": task_str, "answer": answer})
                 task_depend[task["id"]]["answer"] = answer  # type: ignore
                 task_depend[task["id"]]["call_result"] = call_results  # type: ignore
@@ -644,18 +644,22 @@ class VisionAgent(Agent):
                     final_answer,
                     reflection_images,
                 )
-                self.log_progress(f"Reflection: {reflection}")
+                self.log_progress({"log": f"Reflection: {reflection}"})
                 parsed_reflection = parse_reflect(reflection)
                 if parsed_reflection["Finish"]:
                     break
                 else:
                     reflections += "\n" + parsed_reflection["Reflection"]
             else:
-                self.log_progress("Self Reflection skipped based on user request.")
+                self.log_progress(
+                    {"log": "Self Reflection skipped based on user request."}
+                )
                 break
         # '<ANSWER>' is a symbol to indicate the end of the chat, which is useful for streaming logs.
         self.log_progress(
-            f"The Vision Agent has concluded this chat. <ANSWER>{final_answer}</ANSWER>"
+            {
+                "log": f"The Vision Agent has concluded this chat. <ANSWER>{final_answer}</ANSWER>"
+            }
         )
 
         if visualize_output:
@@ -718,8 +722,10 @@ class VisionAgent(Agent):
         }
 
         self.log_progress(
-            f"""Going to run the following tool(s) in sequence:
+            {
+                "log": f"""Going to run the following tool(s) in sequence:
 {tabulate(tabular_data=[tool_results], headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"""
+            }
         )
 
         def parse_tool_results(result: Dict[str, Union[Dict, List]]) -> Any:
@@ -764,7 +770,9 @@ class VisionAgent(Agent):
         else:
             task_list = []
         self.log_progress(
-            f"""Planned tasks:
-{tabulate(task_list, headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"""
+            {
+                "log": "Planned tasks:",
+                "plan": task_list,
+            }
         )
         return task_list

{vision_agent-0.2.27 → vision_agent-0.2.29}/vision_agent/agent/vision_agent_v2.py

@@ -165,7 +165,7 @@ def write_and_exec_code(
     tool_info: str,
     exec: Execute,
     retrieved_ltm: str,
-    log_progress: Callable[..., str],
+    log_progress: Callable[[Dict[str, Any]], None],
     max_retry: int = 3,
     verbosity: int = 0,
 ) -> Tuple[bool, str, str, Dict[str, List[str]]]:
@@ -179,7 +179,23 @@ def write_and_exec_code(
     success, result = exec.run_isolation(code)
     if verbosity == 2:
         _CONSOLE.print(Syntax(code, "python", theme="gruvbox-dark", line_numbers=True))
-        log_progress(f"\tCode success: {success}\n\tResult: {str(result)}", code)
+        log_progress(
+            {
+                "log": f"Code success: {success}",
+            }
+        )
+        log_progress(
+            {
+                "log": "Code:",
+                "code": code,
+            }
+        )
+        log_progress(
+            {
+                "log": "Result:",
+                "result": str(result),
+            }
+        )
         _LOGGER.info(f"\tCode success: {success}, result: {str(result)}")
     working_memory: Dict[str, List[str]] = {}
     while not success and counter < max_retry:
@@ -206,7 +222,18 @@ def write_and_exec_code(
             _CONSOLE.print(
                 Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)
             )
-            log_progress(f"\tDebugging reflection: {reflection}\n\tResult: {result}")
+            log_progress(
+                {
+                    "log": "Debugging reflection:",
+                    "reflection": reflection,
+                }
+            )
+            log_progress(
+                {
+                    "log": "Result:",
+                    "result": result,
+                }
+            )
             _LOGGER.info(f"\tDebugging reflection: {reflection}, result: {result}")
 
         if success:
@@ -227,7 +254,7 @@ def run_plan(
     exec: Execute,
     code: str,
     tool_recommender: Sim,
-    log_progress: Callable[..., str],
+    log_progress: Callable[[Dict[str, Any]], None],
     long_term_memory: Optional[Sim] = None,
     verbosity: int = 0,
 ) -> Tuple[str, str, List[Dict[str, Any]], Dict[str, List[str]]]:
@@ -239,8 +266,7 @@ def run_plan(
 
     for task in active_plan:
         log_progress(
-            f"""Going to run the following task(s) in sequence:
-{tabulate(tabular_data=[task], headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"""
+            {"log": "Going to run the following task(s) in sequence:", "task": task}
         )
         _LOGGER.info(
             f"""
@@ -250,7 +276,7 @@ def run_plan(
         tool_info = "\n".join([e["doc"] for e in tools])
 
         if verbosity == 2:
-            log_progress(f"Tools retrieved: {[e['desc'] for e in tools]}")
+            log_progress({"log": f"Tools retrieved: {[e['desc'] for e in tools]}"})
             _LOGGER.info(f"Tools retrieved: {[e['desc'] for e in tools]}")
 
         if long_term_memory is not None:
@@ -282,7 +308,17 @@ def run_plan(
                 Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)
             )
 
-        log_progress(f"\tCode success: {success}\n\tResult: {str(result)}")
+        log_progress(
+            {
+                "log": f"Code success: {success}",
+            }
+        )
+        log_progress(
+            {
+                "log": "Result:",
+                "result": str(result),
+            }
+        )
         _LOGGER.info(f"\tCode success: {success} result: {str(result)}")
 
         task["success"] = success
@@ -320,7 +356,7 @@ class VisionAgentV2(Agent):
         tool_recommender: Optional[Sim] = None,
         long_term_memory: Optional[Sim] = None,
         verbosity: int = 0,
-        report_progress_callback: Optional[Callable[..., Any]] = None,
+        report_progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
     ) -> None:
         self.planner = OpenAILLM(temperature=0.0, json_mode=True)
         self.coder = OpenAILLM(temperature=0.0)
@@ -376,8 +412,10 @@ class VisionAgentV2(Agent):
 
         user_req, plan = write_plan(chat, plan, TOOL_DESCRIPTIONS, self.planner)
         self.log_progress(
-            f"""Plan:
-{tabulate(tabular_data=plan, headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"""
+            {
+                "log": "Plans:",
+                "plan": plan,
+            }
         )
         _LOGGER.info(
             f"""Plan:
@@ -412,8 +450,12 @@ class VisionAgentV2(Agent):
 
             retries += 1
 
-        self.log_progress("The Vision Agent V2 has concluded this chat.")
-        self.log_progress(f"<ANSWER>Plan success: {success}</ANSWER>")
+        self.log_progress(
+            {
+                "log": f"The Vision Agent V2 has concluded this chat.\nSuccess: {success}",
+                "finished": True,
+            }
+        )
 
         return {
             "code": working_code,
@@ -423,7 +465,7 @@ class VisionAgentV2(Agent):
             "plan": plan,
         }
 
-    def log_progress(self, description: str, code: Optional[str] = "") -> None:
+    def log_progress(self, data: Dict[str, Any]) -> None:
         if self.report_progress_callback is not None:
-            self.report_progress_callback(description, code)
+            self.report_progress_callback(data)
         pass

{vision_agent-0.2.27 → vision_agent-0.2.29}/vision_agent/agent/vision_agent_v3.py

@@ -3,7 +3,7 @@ import json
 import logging
 import sys
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union, cast
+from typing import Any, Dict, List, Optional, Union, cast, Callable, no_type_check
 
 from rich.console import Console
 from rich.syntax import Syntax
@@ -114,8 +114,10 @@ def write_and_test_code(
     coder: LLM,
     tester: LLM,
     debugger: LLM,
+    log_progress: Callable[[Dict[str, Any]], None],
     verbosity: int = 0,
     max_retries: int = 3,
+    input_media: Optional[Union[str, Path]] = None,
 ) -> Dict[str, Any]:
     code = extract_code(
         coder(CODE.format(docstring=tool_info, question=task, feedback=working_memory))
@@ -123,18 +125,40 @@ def write_and_test_code(
     test = extract_code(
         tester(
             SIMPLE_TEST.format(
-                docstring=tool_utils, question=task, code=code, feedback=working_memory
+                docstring=tool_utils,
+                question=task,
+                code=code,
+                feedback=working_memory,
+                media=input_media,
             )
         )
     )
 
     success, result = _EXECUTE.run_isolation(f"{code}\n{test}")
     if verbosity == 2:
-        _LOGGER.info("First code and tests:")
+        _LOGGER.info("Initial code and tests:")
+        log_progress(
+            {
+                "log": "Code:",
+                "code": code,
+            }
+        )
+        log_progress(
+            {
+                "log": "Test:",
+                "code": test,
+            }
+        )
         _CONSOLE.print(
             Syntax(f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True)
         )
-        _LOGGER.info(f"First result: {result}")
+        log_progress(
+            {
+                "log": "Result:",
+                "result": result,
+            }
+        )
+        _LOGGER.info(f"Initial result: {result}")
 
     count = 0
     new_working_memory = []
@@ -156,6 +180,12 @@ def write_and_test_code(
 
         success, result = _EXECUTE.run_isolation(f"{code}\n{test}")
         if verbosity == 2:
+            log_progress(
+                {
+                    "log": f"Debug attempt {count + 1}, reflection:",
+                    "result": fixed_code_and_test["reflections"],
+                }
+            )
             _LOGGER.info(
                 f"Debug attempt {count + 1}, reflection: {fixed_code_and_test['reflections']}"
             )
@@ -164,25 +194,36 @@ def write_and_test_code(
                     f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True
                 )
             )
+            log_progress(
+                {
+                    "log": "Debug result:",
+                    "result": result,
+                }
+            )
             _LOGGER.info(f"Debug result: {result}")
         count += 1
 
-    if verbosity == 1:
+    if verbosity >= 1:
+        _LOGGER.info("Final code and tests:")
         _CONSOLE.print(
             Syntax(f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True)
         )
-        _LOGGER.info(f"Result: {result}")
+        _LOGGER.info(f"Final Result: {result}")
 
     return {
         "code": code,
         "test": test,
         "success": success,
+        "test_result": result,
         "working_memory": new_working_memory,
     }
 
 
 def retrieve_tools(
-    plan: List[Dict[str, str]], tool_recommender: Sim, verbosity: int = 0
+    plan: List[Dict[str, str]],
+    tool_recommender: Sim,
+    log_progress: Callable[[Dict[str, Any]], None],
+    verbosity: int = 0,
 ) -> str:
     tool_info = []
     tool_desc = []
@@ -191,6 +232,12 @@ def retrieve_tools(
         tool_info.extend([e["doc"] for e in tools])
         tool_desc.extend([e["desc"] for e in tools])
     if verbosity == 2:
+        log_progress(
+            {
+                "log": "Retrieved tools:",
+                "tools": tool_desc,
+            }
+        )
         _LOGGER.info(f"Tools: {tool_desc}")
     tool_info_set = set(tool_info)
     return "\n\n".join(tool_info_set)
@@ -206,6 +253,7 @@ class VisionAgentV3(Agent):
         debugger: Optional[LLM] = None,
         tool_recommender: Optional[Sim] = None,
         verbosity: int = 0,
+        report_progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
     ) -> None:
         self.planner = (
             OpenAILLM(temperature=0.0, json_mode=True) if planner is None else planner
@@ -222,22 +270,26 @@ class VisionAgentV3(Agent):
             else tool_recommender
         )
         self.verbosity = verbosity
-        self.max_retries = 3
+        self.max_retries = 2
+        self.report_progress_callback = report_progress_callback
 
+    @no_type_check
     def __call__(
         self,
         input: Union[List[Dict[str, str]], str],
         image: Optional[Union[str, Path]] = None,
-    ) -> str:
+    ) -> Dict[str, Any]:
         if isinstance(input, str):
             input = [{"role": "user", "content": input}]
         results = self.chat_with_workflow(input, image)
-        return results["code"]  # type: ignore
+        results.pop("working_memory")
+        return results
 
     def chat_with_workflow(
         self,
         chat: List[Dict[str, str]],
         image: Optional[Union[str, Path]] = None,
+        self_reflection: bool = False,
     ) -> Dict[str, Any]:
         if len(chat) == 0:
             raise ValueError("Chat cannot be empty.")
@@ -260,7 +312,14 @@ class VisionAgentV3(Agent):
                 chat, TOOL_DESCRIPTIONS, format_memory(working_memory), self.planner
             )
             plan_i_str = "\n-".join([e["instructions"] for e in plan_i])
-            if self.verbosity == 1 or self.verbosity == 2:
+            if self.verbosity >= 1:
+                self.log_progress(
+                    {
+                        "log": "Going to run the following plan(s) in sequence:\n",
+                        "plan": plan_i,
+                    }
+                )
+
                 _LOGGER.info(
                     f"""
 {tabulate(tabular_data=plan_i, headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"""
@@ -269,6 +328,7 @@ class VisionAgentV3(Agent):
             tool_info = retrieve_tools(
                 plan_i,
                 self.tool_recommender,
+                self.log_progress,
                 self.verbosity,
             )
             results = write_and_test_code(
@@ -279,7 +339,9 @@ class VisionAgentV3(Agent):
                 self.coder,
                 self.tester,
                 self.debugger,
+                self.log_progress,
                 verbosity=self.verbosity,
+                input_media=image,
             )
             success = cast(bool, results["success"])
             code = cast(str, results["code"])
@@ -287,19 +349,38 @@ class VisionAgentV3(Agent):
             working_memory.extend(results["working_memory"])  # type: ignore
             plan.append({"code": code, "test": test, "plan": plan_i})
 
-            reflection = reflect(chat, plan_i_str, code, self.planner)
-            if self.verbosity > 0:
-                _LOGGER.info(f"Reflection: {reflection}")
-            feedback = cast(str, reflection["feedback"])
-            success = cast(bool, reflection["success"])
-            working_memory.append({"code": f"{code}\n{test}", "feedback": feedback})
+            if self_reflection:
+                reflection = reflect(chat, plan_i_str, code, self.planner)
+                if self.verbosity > 0:
+                    self.log_progress(
+                        {
+                            "log": "Reflection:",
+                            "reflection": reflection,
+                        }
+                    )
+                    _LOGGER.info(f"Reflection: {reflection}")
+                feedback = cast(str, reflection["feedback"])
+                success = cast(bool, reflection["success"])
+                working_memory.append({"code": f"{code}\n{test}", "feedback": feedback})
+
+            retries += 1
+
+        self.log_progress(
+            {
+                "log": f"The Vision Agent V3 has concluded this chat.\nSuccess: {success}",
+                "finished": True,
+            }
+        )
 
         return {
             "code": code,
             "test": test,
+            "test_result": results["test_result"],
             "plan": plan,
             "working_memory": working_memory,
         }
 
-    def log_progress(self, description: str) -> None:
+    def log_progress(self, data: Dict[str, Any]) -> None:
+        if self.report_progress_callback is not None:
+            self.report_progress_callback(data)
         pass

{vision_agent-0.2.27 → vision_agent-0.2.29}/vision_agent/agent/vision_agent_v3_prompts.py

@@ -61,6 +61,7 @@ This is the documentation for the functions you have access to. You may call any
 2. **Algorithm/Method Selection**: Decide on the most efficient way.
 3. **Pseudocode Creation**: Write down the steps you will follow in pseudocode.
 4. **Code Generation**: Translate your pseudocode into executable Python code.
+5. **Logging**: Log the output of the custom functions that were provided to you from `from vision_agent.tools.tools_v2 import *`. Use a debug flag in the function parameters to toggle logging on and off.
 """
 
 TEST = """
@@ -149,7 +150,7 @@ This is the documentation for the functions you have access to. You may call any
 
 **Input Code Snippet**:
 ```python
-### Please decided how would you want to generate test cases. Based on incomplete code or completed version.
+### Please decide how would you want to generate test cases. Based on incomplete code or completed version.
 {code}
 ```
 
@@ -159,8 +160,12 @@ This is the documentation for the functions you have access to. You may call any
 **Instructions**:
 1. Verify the fundamental functionality under normal conditions.
 2. Ensure each test case is well-documented with comments explaining the scenario it covers.
-3. DO NOT use any files that are not provided by the user's instructions, your test must be run and will crash if it tries to load a non-existent file.
-4. DO NOT mock any functions, you must test their functionality as is.
+3. Your test case MUST run only on the given image which is {media}
+4. DO NOT use any non-existent or dummy image or video files that are not provided by the user's instructions.
+5. DO NOT mock any functions, you must test their functionality as is.
+6. DO NOT assert the output value, run the code and verify it runs without any errors and assert only the output format or data structure.
+7. DO NOT import the testing function as it will available in the testing environment.
+8. Print the output of the function that is being tested.
 """
 
 

{vision_agent-0.2.27 → vision_agent-0.2.29}/vision_agent/tools/tools_v2.py

@@ -416,12 +416,15 @@ def closest_mask_distance(mask1: np.ndarray, mask2: np.ndarray) -> float:
     return cast(float, np.min(dist_matrix))
 
 
-def closest_box_distance(box1: List[float], box2: List[float]) -> float:
+def closest_box_distance(
+    box1: List[float], box2: List[float], image_size: Tuple[int, int]
+) -> float:
     """'closest_box_distance' calculates the closest distance between two bounding boxes.
 
     Parameters:
         box1 (List[float]): The first bounding box.
         box2 (List[float]): The second bounding box.
+        image_size (Tuple[int, int]): The size of the image given as (height, width).
 
     Returns:
         float: The closest distance between the two bounding boxes.
@@ -432,8 +435,8 @@ def closest_box_distance(box1: List[float], box2: List[float]) -> float:
     141.42
     """
 
-    x11, y11, x12, y12 = box1
-    x21, y21, x22, y22 = box2
+    x11, y11, x12, y12 = denormalize_bbox(box1, image_size)
+    x21, y21, x22, y22 = denormalize_bbox(box2, image_size)
 
     horizontal_distance = np.max([0, x21 - x12, x11 - x22])
     vertical_distance = np.max([0, y21 - y12, y11 - y22])