vision-agent 0.2.28__py3-none-any.whl → 0.2.30__py3-none-any.whl

vision_agent/agent/agent.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Dict, List, Optional, Union, Any
+from typing import Any, Dict, List, Optional, Union
 
 
 class Agent(ABC):

vision_agent/agent/agent_coder.py

@@ -3,7 +3,7 @@ import logging
 import os
 import sys
 from pathlib import Path
-from typing import Dict, List, Optional, Union, Any
+from typing import Any, Dict, List, Optional, Union
 
 from rich.console import Console
 from rich.syntax import Syntax

vision_agent/agent/vision_agent_v3.py

@@ -3,7 +3,7 @@ import json
 import logging
 import sys
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union, cast, Callable
+from typing import Any, Callable, Dict, List, Optional, Union, cast, no_type_check
 
 from rich.console import Console
 from rich.syntax import Syntax
@@ -14,6 +14,7 @@ from vision_agent.agent.vision_agent_v3_prompts import (
     CODE,
     FEEDBACK,
     FIX_BUG,
+    FULL_TASK,
     PLAN,
     REFLECT,
     SIMPLE_TEST,
@@ -117,6 +118,7 @@ def write_and_test_code(
     log_progress: Callable[[Dict[str, Any]], None],
     verbosity: int = 0,
     max_retries: int = 3,
+    input_media: Optional[Union[str, Path]] = None,
 ) -> Dict[str, Any]:
     code = extract_code(
         coder(CODE.format(docstring=tool_info, question=task, feedback=working_memory))
@@ -124,14 +126,18 @@ def write_and_test_code(
     test = extract_code(
         tester(
             SIMPLE_TEST.format(
-                docstring=tool_utils, question=task, code=code, feedback=working_memory
+                docstring=tool_utils,
+                question=task,
+                code=code,
+                feedback=working_memory,
+                media=input_media,
             )
         )
     )
 
     success, result = _EXECUTE.run_isolation(f"{code}\n{test}")
     if verbosity == 2:
-        _LOGGER.info("First code and tests:")
+        _LOGGER.info("Initial code and tests:")
         log_progress(
             {
                 "log": "Code:",
@@ -153,7 +159,7 @@ def write_and_test_code(
                 "result": result,
             }
         )
-        _LOGGER.info(f"First result: {result}")
+        _LOGGER.info(f"Initial result: {result}")
 
     count = 0
     new_working_memory = []
@@ -198,16 +204,18 @@ def write_and_test_code(
             _LOGGER.info(f"Debug result: {result}")
         count += 1
 
-    if verbosity == 1:
+    if verbosity >= 1:
+        _LOGGER.info("Final code and tests:")
         _CONSOLE.print(
             Syntax(f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True)
         )
-        _LOGGER.info(f"Result: {result}")
+        _LOGGER.info(f"Final Result: {result}")
 
     return {
         "code": code,
         "test": test,
         "success": success,
+        "test_result": result,
         "working_memory": new_working_memory,
     }
 
@@ -263,23 +271,26 @@ class VisionAgentV3(Agent):
             else tool_recommender
         )
         self.verbosity = verbosity
-        self.max_retries = 3
+        self.max_retries = 2
         self.report_progress_callback = report_progress_callback
 
+    @no_type_check
     def __call__(
         self,
         input: Union[List[Dict[str, str]], str],
         image: Optional[Union[str, Path]] = None,
-    ) -> str:
+    ) -> Dict[str, Any]:
         if isinstance(input, str):
             input = [{"role": "user", "content": input}]
         results = self.chat_with_workflow(input, image)
-        return results["code"]  # type: ignore
+        results.pop("working_memory")
+        return results
 
     def chat_with_workflow(
         self,
         chat: List[Dict[str, str]],
         image: Optional[Union[str, Path]] = None,
+        self_reflection: bool = False,
     ) -> Dict[str, Any]:
         if len(chat) == 0:
             raise ValueError("Chat cannot be empty.")
@@ -302,13 +313,14 @@ class VisionAgentV3(Agent):
                 chat, TOOL_DESCRIPTIONS, format_memory(working_memory), self.planner
             )
             plan_i_str = "\n-".join([e["instructions"] for e in plan_i])
-            if self.verbosity == 1 or self.verbosity == 2:
+            if self.verbosity >= 1:
                 self.log_progress(
                     {
                         "log": "Going to run the following plan(s) in sequence:\n",
                         "plan": plan_i,
                     }
                 )
+
                 _LOGGER.info(
                     f"""
 {tabulate(tabular_data=plan_i, headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"""
@@ -321,7 +333,7 @@ class VisionAgentV3(Agent):
                 self.verbosity,
             )
             results = write_and_test_code(
-                plan_i_str,
+                FULL_TASK.format(user_request=chat[0]["content"], subtasks=plan_i_str),
                 tool_info,
                 UTILITIES_DOCSTRING,
                 format_memory(working_memory),
@@ -330,6 +342,7 @@ class VisionAgentV3(Agent):
                 self.debugger,
                 self.log_progress,
                 verbosity=self.verbosity,
+                input_media=image,
             )
             success = cast(bool, results["success"])
             code = cast(str, results["code"])
@@ -337,18 +350,28 @@ class VisionAgentV3(Agent):
             working_memory.extend(results["working_memory"])  # type: ignore
             plan.append({"code": code, "test": test, "plan": plan_i})
 
-            reflection = reflect(chat, plan_i_str, code, self.planner)
-            if self.verbosity > 0:
-                self.log_progress(
-                    {
-                        "log": "Reflection:",
-                        "reflection": reflection,
-                    }
+            if self_reflection:
+                reflection = reflect(
+                    chat,
+                    FULL_TASK.format(
+                        user_request=chat[0]["content"], subtasks=plan_i_str
+                    ),
+                    code,
+                    self.planner,
                 )
-                _LOGGER.info(f"Reflection: {reflection}")
-            feedback = cast(str, reflection["feedback"])
-            success = cast(bool, reflection["success"])
-            working_memory.append({"code": f"{code}\n{test}", "feedback": feedback})
+                if self.verbosity > 0:
+                    self.log_progress(
+                        {
+                            "log": "Reflection:",
+                            "reflection": reflection,
+                        }
+                    )
+                    _LOGGER.info(f"Reflection: {reflection}")
+                feedback = cast(str, reflection["feedback"])
+                success = cast(bool, reflection["success"])
+                working_memory.append({"code": f"{code}\n{test}", "feedback": feedback})
+
+            retries += 1
 
         self.log_progress(
             {
@@ -360,6 +383,7 @@ class VisionAgentV3(Agent):
         return {
             "code": code,
             "test": test,
+            "test_result": results["test_result"],
             "plan": plan,
             "working_memory": working_memory,
         }

vision_agent/agent/vision_agent_v3_prompts.py

@@ -3,6 +3,14 @@ USER_REQ = """
 {user_request}
 """
 
+FULL_TASK = """
+## User Request
+{user_request}
+
+## Subtasks
+{subtasks}
+"""
+
 FEEDBACK = """
 ## This contains code and feedback from previous runs and is used for providing context so you do not make the same mistake again.
 
@@ -61,6 +69,7 @@ This is the documentation for the functions you have access to. You may call any
 2. **Algorithm/Method Selection**: Decide on the most efficient way.
 3. **Pseudocode Creation**: Write down the steps you will follow in pseudocode.
 4. **Code Generation**: Translate your pseudocode into executable Python code.
+5. **Logging**: Log the output of the custom functions that were provided to you from `from vision_agent.tools.tools_v2 import *`. Use a debug flag in the function parameters to toggle logging on and off.
 """
 
 TEST = """
@@ -149,7 +158,7 @@ This is the documentation for the functions you have access to. You may call any
 
 **Input Code Snippet**:
 ```python
-### Please decided how would you want to generate test cases. Based on incomplete code or completed version.
+### Please decide how would you want to generate test cases. Based on incomplete code or completed version.
 {code}
 ```
 
@@ -159,13 +168,17 @@ This is the documentation for the functions you have access to. You may call any
 **Instructions**:
 1. Verify the fundamental functionality under normal conditions.
 2. Ensure each test case is well-documented with comments explaining the scenario it covers.
-3. DO NOT use any files that are not provided by the user's instructions, your test must be run and will crash if it tries to load a non-existent file.
-4. DO NOT mock any functions, you must test their functionality as is.
+3. Your test case MUST run only on the given image which is {media}
+4. DO NOT use any non-existent or dummy image or video files that are not provided by the user's instructions.
+5. DO NOT mock any functions, you must test their functionality as is.
+6. DO NOT assert the output value, run the code and verify it runs without any errors and assert only the output format or data structure.
+7. DO NOT import the testing function as it will available in the testing environment.
+8. Print the output of the function that is being tested.
 """
 
 
 FIX_BUG = """
-**Role** As a coder, your job is to find the error in the code and fix it. You are running in a notebook setting so feel free to run !pip install to install missing packages.
+**Role** As a coder, your job is to find the error in the code and fix it. You are running in a notebook setting so you can run !pip install to install missing packages.
 
 **Instructions**:
 Please re-complete the code to fix the error message. Here is the previous version:

vision_agent/tools/tools_v2.py

@@ -416,12 +416,15 @@ def closest_mask_distance(mask1: np.ndarray, mask2: np.ndarray) -> float:
     return cast(float, np.min(dist_matrix))
 
 
-def closest_box_distance(box1: List[float], box2: List[float]) -> float:
+def closest_box_distance(
+    box1: List[float], box2: List[float], image_size: Tuple[int, int]
+) -> float:
     """'closest_box_distance' calculates the closest distance between two bounding boxes.
 
     Parameters:
         box1 (List[float]): The first bounding box.
         box2 (List[float]): The second bounding box.
+        image_size (Tuple[int, int]): The size of the image given as (height, width).
 
     Returns:
         float: The closest distance between the two bounding boxes.
@@ -432,8 +435,8 @@ def closest_box_distance(box1: List[float], box2: List[float]) -> float:
     141.42
     """
 
-    x11, y11, x12, y12 = box1
-    x21, y21, x22, y22 = box2
+    x11, y11, x12, y12 = denormalize_bbox(box1, image_size)
+    x21, y21, x22, y22 = denormalize_bbox(box2, image_size)
 
     horizontal_distance = np.max([0, x21 - x12, x11 - x22])
     vertical_distance = np.max([0, y21 - y12, y11 - y22])

{vision_agent-0.2.28.dist-info → vision_agent-0.2.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.28
+Version: 0.2.30
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.28.dist-info → vision_agent-0.2.30.dist-info}/RECORD

@@ -1,7 +1,7 @@
 vision_agent/__init__.py,sha256=GVLHCeK_R-zgldpbcPmOzJat-BkadvkuRCMxDvTIcXs,108
 vision_agent/agent/__init__.py,sha256=jpmL6z5e4PFfQM21JbSsRwcERRXn58XFmURAMwWeoRM,249
-vision_agent/agent/agent.py,sha256=-HblUCrTZdxsP-MlSBzSfCTm5vELLnfb40l2L-bkVHw,538
-vision_agent/agent/agent_coder.py,sha256=d4N0I7QMKOdGoI-0ZRQ3Fp2QusAHbbJNe8AXg1IyRk4,7006
+vision_agent/agent/agent.py,sha256=4buKL_7PA6q_Ktlf26FxfX0JxRGrL-swYk0xJuYNVz4,538
+vision_agent/agent/agent_coder.py,sha256=4Neo6qM9-J8sJ-PKqSaUHr28SYm43IjEvhDK8BfDosE,7006
 vision_agent/agent/agent_coder_prompts.py,sha256=CJe3v7xvHQ32u3RQAXQga_Tk_4UgU64RBAMHZ3S70KY,5538
 vision_agent/agent/easytool.py,sha256=oMHnBg7YBtIPgqQUNcZgq7uMgpPThs99_UnO7ERkMVg,11511
 vision_agent/agent/easytool_prompts.py,sha256=Bikw-PPLkm78dwywTlnv32Y1Tw6JMeC-R7oCnXWLcTk,4656
@@ -11,8 +11,8 @@ vision_agent/agent/vision_agent.py,sha256=Rs7O0PXc2J9FlrpBa3UGs5NjqQT51Y507klQf9
 vision_agent/agent/vision_agent_prompts.py,sha256=MZSIwovYgB-f-kdJ6btaNDVXptJn47bfOL3-Zn6NiC0,8573
 vision_agent/agent/vision_agent_v2.py,sha256=t2D1mMUYEv1dFeMrkEUVbDEdArunb7F1ZeYB8qijU2w,15109
 vision_agent/agent/vision_agent_v2_prompts.py,sha256=b_0BMq6GrbGfl09MHrv4mj-mqyE1FxMl3Xq44qD4S1E,6161
-vision_agent/agent/vision_agent_v3.py,sha256=y6BB2Mv9vj5OT4b829GgVmbiHX24MGMsz0gncgVy-4g,11632
-vision_agent/agent/vision_agent_v3_prompts.py,sha256=LRZBKObeb0Bs48vo7vtB2M8loPO1lQzruH-3IiMS5ts,7484
+vision_agent/agent/vision_agent_v3.py,sha256=jPU__NueKQwFzIoJd0zzg6z9q7IDQa9QPaxt8Qlca98,12403
+vision_agent/agent/vision_agent_v3_prompts.py,sha256=ejedMNDluVYZjHOIXKN98LzX-pOHin2DJhCyZUWULNE,8070
 vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
 vision_agent/llm/__init__.py,sha256=BoUm_zSAKnLlE8s-gKTSQugXDqVZKPqYlWwlTLdhcz4,48
@@ -23,14 +23,14 @@ vision_agent/tools/__init__.py,sha256=dRHXGpjhItXZRQs0r_l3Z3bQIreaZaYP0CJrl8mOJx
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
 vision_agent/tools/tool_utils.py,sha256=wzRacbUpqk9hhfX_Y08rL8qP0XCN2w-8IZoYLi3Upn4,869
 vision_agent/tools/tools.py,sha256=pZc5dQlYINlV4nYbbzsDi3-wauA-fCeD2iGmJUMoUfE,47373
-vision_agent/tools/tools_v2.py,sha256=3Bv1xuZFoPjaCb-VixF5Vl3uoyac03571FXUzBI8FBQ,21404
+vision_agent/tools/tools_v2.py,sha256=mio0A1l5QcyRC5IgaD4Trfqg7hFTZ8rOjx1dYivwb4Q,21585
 vision_agent/utils/__init__.py,sha256=xsHFyJSDbLdonB9Dh74cwZnVTiT__2OQF3Brd3Nmglc,116
 vision_agent/utils/execute.py,sha256=8_SfK-IkHH4lXF0JVyV7sDFszZn9HKsh1bFITKGCJ1g,3881
 vision_agent/utils/image_utils.py,sha256=_cdiS5YrLzqkq_ZgFUO897m5M4_SCIThwUy4lOklfB8,7700
 vision_agent/utils/sim.py,sha256=oUZ-6eu8Io-UNt9GXJ0XRKtP-Wc0sPWVzYGVpB2yDFk,3001
 vision_agent/utils/type_defs.py,sha256=BlI8ywWHAplC7kYWLvt4AOdnKpEW3qWEFm-GEOSkrFQ,1792
 vision_agent/utils/video.py,sha256=xTElFSFp1Jw4ulOMnk81Vxsh-9dTxcWUO6P9fzEi3AM,7653
-vision_agent-0.2.28.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.28.dist-info/METADATA,sha256=Vnz8N9UhaUvp6ljocoVpXIIuW26Q2Jf1vFQpfdXIXgg,9212
-vision_agent-0.2.28.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.28.dist-info/RECORD,,
+vision_agent-0.2.30.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.30.dist-info/METADATA,sha256=uVj7XfG4Hat1Bed9FYM2dipIseooN4AHY-Tl4rSPOIg,9212
+vision_agent-0.2.30.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.30.dist-info/RECORD,,