vision-agent 0.2.10__py3-none-any.whl → 0.2.12__py3-none-any.whl

vision_agent/agent/vision_agent.py

@@ -489,6 +489,7 @@ class VisionAgent(Agent):
         image: Optional[Union[str, Path]] = None,
         reference_data: Optional[Dict[str, str]] = None,
         visualize_output: Optional[bool] = False,
+        self_reflection: Optional[bool] = True,
     ) -> str:
         """Invoke the vision agent.
 
@@ -501,6 +502,7 @@ class VisionAgent(Agent):
                 {"image": "image.jpg", "mask": "mask.jpg", "bbox": [0.1, 0.2, 0.1, 0.2]}
                 where the bounding box coordinates are normalized.
             visualize_output: Whether to visualize the output.
+            self_reflection: boolean to enable and disable self reflection.
 
         Returns:
             The result of the vision agent in text.
@@ -512,6 +514,7 @@ class VisionAgent(Agent):
             image=image,
             visualize_output=visualize_output,
             reference_data=reference_data,
+            self_reflection=self_reflection,
         )
 
     def log_progress(self, description: str) -> None:
@@ -538,6 +541,7 @@ class VisionAgent(Agent):
         image: Optional[Union[str, Path]] = None,
         reference_data: Optional[Dict[str, str]] = None,
         visualize_output: Optional[bool] = False,
+        self_reflection: Optional[bool] = True,
     ) -> Tuple[str, List[Dict]]:
         """Chat with the vision agent and return the final answer and all tool results.
 
@@ -550,6 +554,7 @@ class VisionAgent(Agent):
                 {"image": "image.jpg", "mask": "mask.jpg", "bbox": [0.1, 0.2, 0.1, 0.2]}
                 where the bounding box coordinates are normalized.
             visualize_output: Whether to visualize the output.
+            self_reflection: boolean to enable and disable self reflection.
 
         Returns:
             A tuple where the first item is the final answer and the second item is a
@@ -625,20 +630,25 @@ class VisionAgent(Agent):
                 reflection_images = [image]
             else:
                 reflection_images = None
-            reflection = self_reflect(
-                self.reflect_model,
-                question,
-                self.tools,
-                all_tool_results,
-                final_answer,
-                reflection_images,
-            )
-            self.log_progress(f"Reflection: {reflection}")
-            parsed_reflection = parse_reflect(reflection)
-            if parsed_reflection["Finish"]:
-                break
+
+            if self_reflection:
+                reflection = self_reflect(
+                    self.reflect_model,
+                    question,
+                    self.tools,
+                    all_tool_results,
+                    final_answer,
+                    reflection_images,
+                )
+                self.log_progress(f"Reflection: {reflection}")
+                parsed_reflection = parse_reflect(reflection)
+                if parsed_reflection["Finish"]:
+                    break
+                else:
+                    reflections += "\n" + parsed_reflection["Reflection"]
             else:
-                reflections += "\n" + parsed_reflection["Reflection"]
+                self.log_progress("Self Reflection skipped based on user request.")
+                break
         # '<ANSWER>' is a symbol to indicate the end of the chat, which is useful for streaming logs.
         self.log_progress(
             f"The Vision Agent has concluded this chat. <ANSWER>{final_answer}</ANSWER>"
@@ -660,12 +670,14 @@ class VisionAgent(Agent):
         image: Optional[Union[str, Path]] = None,
         reference_data: Optional[Dict[str, str]] = None,
         visualize_output: Optional[bool] = False,
+        self_reflection: Optional[bool] = True,
     ) -> str:
         answer, _ = self.chat_with_workflow(
             chat,
             image=image,
             visualize_output=visualize_output,
             reference_data=reference_data,
+            self_reflection=self_reflection,
         )
         return answer
 

vision_agent/tools/__init__.py

@@ -5,7 +5,9 @@ from .tools import (  # Counter,
     TOOLS,
     BboxArea,
     BboxIoU,
+    ObjectDistance,
     BoxDistance,
+    MaskDistance,
     Crop,
     DINOv,
     ExtractFrames,

vision_agent/tools/tools.py

@@ -9,6 +9,7 @@ import numpy as np
 import requests
 from PIL import Image
 from PIL.Image import Image as ImageType
+from scipy.spatial import distance  # type: ignore
 
 from vision_agent.image_utils import (
     b64_to_pil,
@@ -544,7 +545,7 @@ class VisualPromptCounting(Tool):
     -------
         >>> import vision_agent as va
         >>> prompt_count = va.tools.VisualPromptCounting()
-        >>> prompt_count(image="image1.jpg", prompt="0.1, 0.1, 0.4, 0.42")
+        >>> prompt_count(image="image1.jpg", prompt={"bbox": [0.1, 0.1, 0.4, 0.42]})
         {'count': 23}
     """
 
@@ -554,52 +555,60 @@ class VisualPromptCounting(Tool):
     usage = {
         "required_parameters": [
             {"name": "image", "type": "str"},
-            {"name": "prompt", "type": "str"},
+            {"name": "prompt", "type": "Dict[str, List[float]"},
         ],
         "examples": [
             {
                 "scenario": "Here is an example of a lid '0.1, 0.1, 0.14, 0.2', Can you count the items in the image ? Image name: lids.jpg",
-                "parameters": {"image": "lids.jpg", "prompt": "0.1, 0.1, 0.14, 0.2"},
+                "parameters": {
+                    "image": "lids.jpg",
+                    "prompt": {"bbox": [0.1, 0.1, 0.14, 0.2]},
+                },
             },
             {
-                "scenario": "Can you count the total number of objects in this image ? Image name: tray.jpg",
-                "parameters": {"image": "tray.jpg", "prompt": "0.1, 0.1, 0.2, 0.25"},
+                "scenario": "Can you count the total number of objects in this image ? Image name: tray.jpg, reference_data: {'bbox': [0.1, 0.1, 0.2, 0.25]}",
+                "parameters": {
+                    "image": "tray.jpg",
+                    "prompt": {"bbox": [0.1, 0.1, 0.2, 0.25]},
+                },
             },
             {
-                "scenario": "Can you count this item based on an example, reference_data: '0.1, 0.15, 0.2, 0.2' ? Image name: shirts.jpg",
+                "scenario": "Can you count this item based on an example, reference_data: {'bbox': [100, 115, 200, 200]} ? Image name: shirts.jpg",
                 "parameters": {
                     "image": "shirts.jpg",
-                    "prompt": "0.1, 0.15, 0.2, 0.2",
+                    "prompt": {"bbox": [100, 115, 200, 200]},
                 },
             },
             {
-                "scenario": "Can you build me a counting tool based on an example prompt ? Image name: shoes.jpg",
+                "scenario": "Can you build me a counting tool based on an example prompt ? Image name: shoes.jpg, reference_data: {'bbox': [0.1, 0.1, 0.6, 0.65]}",
                 "parameters": {
                     "image": "shoes.jpg",
-                    "prompt": "0.1, 0.1, 0.6, 0.65",
+                    "prompt": {"bbox": [0.1, 0.1, 0.6, 0.65]},
                 },
             },
         ],
     }
 
-    # TODO: Add support for input multiple images, which aligns with the output type.
-    def __call__(self, image: Union[str, ImageType], prompt: str) -> Dict:
+    def __call__(
+        self, image: Union[str, ImageType], prompt: Dict[str, List[float]]
+    ) -> Dict:
         """Invoke the few shot counting model.
 
         Parameters:
             image: the input image.
+            prompt: the visual prompt which is a bounding box describing the object.
 
         Returns:
             A dictionary containing the key 'count' and the count as value. E.g. {count: 12}
         """
         image_size = get_image_size(image)
-        bbox = [float(x) for x in prompt.split(",")]
-        prompt = ", ".join(map(str, denormalize_bbox(bbox, image_size)))
+        bbox = prompt["bbox"]
+        bbox_str = ", ".join(map(str, denormalize_bbox(bbox, image_size)))
         image_b64 = convert_to_b64(image)
 
         data = {
             "image": image_b64,
-            "prompt": prompt,
+            "prompt": bbox_str,
             "tool": "few_shot_counting",
         }
         resp_data = _send_inference_request(data, "tools")
@@ -878,7 +887,7 @@ class SegIoU(Tool):
         ],
         "examples": [
             {
-                "scenario": "If you want to calculate the intersection over union of the segmentation masks for mask_file1.jpg and mask_file2.jpg",
+                "scenario": "Calculate the intersection over union of the segmentation masks for mask_file1.jpg and mask_file2.jpg",
                 "parameters": {"mask1": "mask_file1.png", "mask2": "mask_file2.png"},
             }
         ],
@@ -947,6 +956,46 @@ class BboxContains(Tool):
         }
 
 
+class ObjectDistance(Tool):
+    name = "object_distance_"
+    description = "'object_distance_' calculates the distance between two objects in an image. It returns the minimum distance between the two objects."
+    usage = {
+        "required_parameters": [
+            {"name": "object1", "type": "Dict[str, Any]"},
+            {"name": "object2", "type": "Dict[str, Any]"},
+        ],
+        "examples": [
+            {
+                "scenario": "Calculate the distance between these two objects {bboxes: [0.2, 0.21, 0.34, 0.42], masks: 'mask_file1.png'}, {bboxes: [0.3, 0.31, 0.44, 0.52], masks: 'mask_file2.png'}",
+                "parameters": {
+                    "object1": {
+                        "bboxes": [0.2, 0.21, 0.34, 0.42],
+                        "scores": 0.54,
+                        "masks": "mask_file1.png",
+                    },
+                    "object2": {
+                        "bboxes": [0.3, 0.31, 0.44, 0.52],
+                        "scores": 0.66,
+                        "masks": "mask_file2.png",
+                    },
+                },
+            }
+        ],
+    }
+
+    def __call__(self, object1: Dict[str, Any], object2: Dict[str, Any]) -> float:
+        if "masks" in object1 and "masks" in object2:
+            mask1 = object1["masks"]
+            mask2 = object2["masks"]
+            return MaskDistance()(mask1, mask2)
+        elif "bboxes" in object1 and "bboxes" in object2:
+            bbox1 = object1["bboxes"]
+            bbox2 = object2["bboxes"]
+            return BoxDistance()(bbox1, bbox2)
+        else:
+            raise ValueError("Either of the objects should have masks or bboxes")
+
+
 class BoxDistance(Tool):
     name = "box_distance_"
     description = "'box_distance_' calculates distance between two bounding boxes. It returns the minumum distance between the given bounding boxes"
@@ -957,7 +1006,7 @@ class BoxDistance(Tool):
         ],
         "examples": [
             {
-                "scenario": "Calculate the distance between the bounding boxes [0.2, 0.21, 0.34, 0.42] and [0.3, 0.31, 0.44, 0.52]",
+                "scenario": "Calculate the distance between these two bounding boxes [0.2, 0.21, 0.34, 0.42] and [0.3, 0.31, 0.44, 0.52]",
                 "parameters": {
                     "bbox1": [0.2, 0.21, 0.34, 0.42],
                     "bbox2": [0.3, 0.31, 0.44, 0.52],
@@ -976,6 +1025,34 @@ class BoxDistance(Tool):
         return cast(float, round(np.sqrt(horizontal_dist**2 + vertical_dist**2), 2))
 
 
+class MaskDistance(Tool):
+    name = "mask_distance_"
+    description = "'mask_distance_' calculates distance between two masks. It is helpful in checking proximity of two objects. It returns the minumum distance between the given masks"
+    usage = {
+        "required_parameters": [
+            {"name": "mask1", "type": "str"},
+            {"name": "mask2", "type": "str"},
+        ],
+        "examples": [
+            {
+                "scenario": "Calculate the distance between the segmentation masks for mask_file1.jpg and mask_file2.jpg",
+                "parameters": {"mask1": "mask_file1.png", "mask2": "mask_file2.png"},
+            }
+        ],
+    }
+
+    def __call__(self, mask1: Union[str, Path], mask2: Union[str, Path]) -> float:
+        pil_mask1 = Image.open(str(mask1))
+        pil_mask2 = Image.open(str(mask2))
+        np_mask1 = np.clip(np.array(pil_mask1), 0, 1)
+        np_mask2 = np.clip(np.array(pil_mask2), 0, 1)
+
+        mask1_points = np.transpose(np.nonzero(np_mask1))
+        mask2_points = np.transpose(np.nonzero(np_mask2))
+        dist_matrix = distance.cdist(mask1_points, mask2_points, "euclidean")
+        return cast(float, np.round(np.min(dist_matrix), 2))
+
+
 class ExtractFrames(Tool):
     r"""Extract frames from a video."""
 
@@ -1110,10 +1187,9 @@ TOOLS = {
             Crop,
             BboxArea,
             SegArea,
-            BboxIoU,
-            SegIoU,
+            ObjectDistance,
             BboxContains,
-            BoxDistance,
+            SegIoU,
             OCR,
             Calculator,
         ]

{vision_agent-0.2.10.dist-info → vision_agent-0.2.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.10
+Version: 0.2.12
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -17,6 +17,7 @@ Requires-Dist: pandas (>=2.0.0,<3.0.0)
 Requires-Dist: pillow (>=10.0.0,<11.0.0)
 Requires-Dist: pydantic-settings (>=2.2.1,<3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
+Requires-Dist: scipy (>=1.13.0,<1.14.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
 Requires-Dist: typing_extensions (>=4.0.0,<5.0.0)
@@ -149,7 +150,7 @@ you. For example:
 
 #### Custom Tools
 You can also add your own custom tools for your vision agent to use:
-    
+
 ```python
 from vision_agent.tools import Tool, register_tool
 @register_tool
@@ -187,6 +188,7 @@ find an example that creates a custom tool for template matching [here](examples
 | BboxIoU | BboxIoU returns the intersection over union of two bounding boxes normalized to 2 decimal places. |
 | SegIoU | SegIoU returns the intersection over union of two segmentation masks normalized to 2 decimal places. |
 | BoxDistance | BoxDistance returns the minimum distance between two bounding boxes normalized to 2 decimal places. |
+| MaskDistance | MaskDistance returns the minimum distance between two segmentation masks in pixel units |
 | BboxContains | BboxContains returns the intersection of two boxes over the target box area. It is good for check if one box is contained within another box. |
 | ExtractFrames | ExtractFrames extracts frames with motion from a video. |
 | ZeroShotCounting | ZeroShotCounting returns the total number of objects belonging to a single class in a given image. |

{vision_agent-0.2.10.dist-info → vision_agent-0.2.12.dist-info}/RECORD

@@ -5,7 +5,7 @@ vision_agent/agent/easytool.py,sha256=oMHnBg7YBtIPgqQUNcZgq7uMgpPThs99_UnO7ERkMV
 vision_agent/agent/easytool_prompts.py,sha256=Bikw-PPLkm78dwywTlnv32Y1Tw6JMeC-R7oCnXWLcTk,4656
 vision_agent/agent/reflexion.py,sha256=4gz30BuFMeGxSsTzoDV4p91yE0R8LISXp28IaOI6wdM,10506
 vision_agent/agent/reflexion_prompts.py,sha256=G7UAeNz_g2qCb2yN6OaIC7bQVUkda4m3z42EG8wAyfE,9342
-vision_agent/agent/vision_agent.py,sha256=DVcvT02GjY85mCjhHgJGrhI_dpUvjZhoYzYik9bkHQA,26243
+vision_agent/agent/vision_agent.py,sha256=5W5Xr_h4yDMsFvIk2JWcfMlYoPYmTv3JZnrDDumuZgM,26842
 vision_agent/agent/vision_agent_prompts.py,sha256=moihXFhEzFw8xnf2sUSgd_k9eoxQam3T6XUkB0fyp5o,8570
 vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
@@ -14,12 +14,12 @@ vision_agent/llm/__init__.py,sha256=BoUm_zSAKnLlE8s-gKTSQugXDqVZKPqYlWwlTLdhcz4,
 vision_agent/llm/llm.py,sha256=1BkrSVBWEClyqLc0Rmyw4heLhi_ZVm6JO7-i1wd1ziw,5383
 vision_agent/lmm/__init__.py,sha256=nnNeKD1k7q_4vLb1x51O_EUTYaBgGfeiCx5F433gr3M,67
 vision_agent/lmm/lmm.py,sha256=gK90vMxh0OcGSuIZQikBkDXm4pfkdFk1R2y7rtWDl84,10539
-vision_agent/tools/__init__.py,sha256=HfUr0JQUwk0Kyieen93df9lMbbdpVf9Q6CcVFmKv_q4,413
+vision_agent/tools/__init__.py,sha256=uWySwcIeQMH57PVN6lVIknTx-SFmN_J0mvn_HbGlXcQ,451
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
-vision_agent/tools/tools.py,sha256=EvNDLUxe-Ed8-meHInTIiX3aySLUXFBsAWwL0Is5S1o,43823
+vision_agent/tools/tools.py,sha256=kqwmKPbuSAGOWjzv2LCjsvUAp2mfRk8X5a1DrP2B4i8,47007
 vision_agent/tools/video.py,sha256=xTElFSFp1Jw4ulOMnk81Vxsh-9dTxcWUO6P9fzEi3AM,7653
 vision_agent/type_defs.py,sha256=4LTnTL4HNsfYqCrDn9Ppjg9bSG2ZGcoKSSd9YeQf4Bw,1792
-vision_agent-0.2.10.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.10.dist-info/METADATA,sha256=2uCVxAWBCbaFvxFnd6xoRoPNSo1UXaTLkeZ5qVOSM84,8930
-vision_agent-0.2.10.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.10.dist-info/RECORD,,
+vision_agent-0.2.12.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.12.dist-info/METADATA,sha256=IWJjflG4JW4ZuMzyTw1Rq6IHK-YuO_YCfp_nJ-J0LiY,9073
+vision_agent-0.2.12.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.12.dist-info/RECORD,,