vision-agent 0.2.7__py3-none-any.whl → 0.2.9__py3-none-any.whl

vision_agent/agent/easytool_prompts.py

@@ -44,7 +44,7 @@ Output: """
 
 CHOOSE_PARAMETER = """Given a user's question and an API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.
 Please note that:
-1. The Example in the API tool documentation can help you better understand the use of the API.
+1. The Example in the API tool documentation can help you better understand the use of the API. Pay attention to the examples which show how to parse the question and extract tool parameters such as prompts and visual inputs.
 2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If there are no paremters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}}
 3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.
 4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers for your reference.

vision_agent/agent/vision_agent.py

@@ -308,6 +308,9 @@ def _handle_extract_frames(
     # handle extract_frames_ case, useful if it extracts frames but doesn't do
     # any following processing
     for video_file_output in tool_result["call_results"]:
+        # When the video tool is run with wrong parameters, exit the loop
+        if len(video_file_output) < 2:
+            break
         for frame, _ in video_file_output:
             image = frame
             if image not in image_to_data:
@@ -346,7 +349,9 @@ def _handle_viz_tools(
         # 2. return a dictionary but not have the necessary keys
 
         if not isinstance(call_result, dict) or (
-            "bboxes" not in call_result and "heat_map" not in call_result
+            "bboxes" not in call_result
+            and "mask" not in call_result
+            and "heat_map" not in call_result
         ):
             return image_to_data
 
@@ -366,7 +371,8 @@ def _handle_viz_tools(
         image_to_data[image]["scores"].extend(call_result.get("scores", []))
         image_to_data[image]["masks"].extend(call_result.get("masks", []))
         # only single heatmap is returned
-        image_to_data[image]["heat_map"].append(call_result.get("heat_map", []))
+        if "heat_map" in call_result:
+            image_to_data[image]["heat_map"].append(call_result["heat_map"])
         if "mask_shape" in call_result:
             image_to_data[image]["mask_shape"] = call_result["mask_shape"]
 
@@ -444,7 +450,7 @@ class VisionAgent(Agent):
         task_model: Optional[Union[LLM, LMM]] = None,
         answer_model: Optional[Union[LLM, LMM]] = None,
         reflect_model: Optional[Union[LLM, LMM]] = None,
-        max_retries: int = 3,
+        max_retries: int = 2,
         verbose: bool = False,
         report_progress_callback: Optional[Callable[[str], None]] = None,
     ):

vision_agent/agent/vision_agent_prompts.py

@@ -26,22 +26,24 @@ Final answer:
 
 Reflection: """
 
-TASK_DECOMPOSE = """You need to decompose a user's complex question into some simple subtasks and let the model execute it step by step.
+TASK_DECOMPOSE = """You need to decompose a user's complex question into one or more simple subtasks and let the model execute it step by step.
 This is the user's question: {question}
 This is the tool list:
 {tools}
 
 Please note that:
-1. You should only decompose this user's complex question into some simple subtasks which can be executed easily by using one single tool in the tool list.
-2. If one subtask needs the results from another subtask, you should write clearly. For example:
+1. If the given task is simple and the answer can be provided by executing one tool, you should only use that tool to provide the answer.
+2. If the given task is complex, You should decompose this user's complex question into simple subtasks which can only be executed easily by using one single tool in the tool list.
+3. You should try to decompose the complex question into least number of subtasks.
+4. If one subtask needs the results from another subtask, you should write clearly. For example:
 {{"Tasks": ["Convert 23 km/h to X km/min by 'divide_'", "Multiply X km/min by 45 min to get Y by 'multiply_'"]}}
-3. You must ONLY output in a parsible JSON format. An example output looks like:
+5. You must ONLY output in a parsible JSON format. An example output looks like:
 
 {{"Tasks": ["Task 1", "Task 2", ...]}}
 
 Output: """
 
-TASK_DECOMPOSE_DEPENDS = """You need to decompose a user's complex question into some simple subtasks and let the model execute it step by step.
+TASK_DECOMPOSE_DEPENDS = """You need to decompose a user's complex question into one or more simple subtasks and let the model execute it step by step.
 This is the user's question: {question}
 
 This is the tool list:
@@ -51,10 +53,12 @@ This is a reflection from a previous failed attempt:
 {reflections}
 
 Please note that:
-1. You should only decompose this user's complex question into some simple subtasks which can be executed easily by using one single tool in the tool list.
-2. If one subtask needs the results from another subtask, you should write clearly. For example:
+1. If the given task is simple and the answer can be provided by executing one tool, you should only use that tool to provide the answer.
+2. If the given task is complex, You should decompose this user's complex question into simple subtasks which can only be executed easily by using one single tool in the tool list.
+3. You should try to decompose the complex question into least number of subtasks.
+4. If one subtask needs the results from another subtask, you should write clearly. For example:
 {{"Tasks": ["Convert 23 km/h to X km/min by 'divide_'", "Multiply X km/min by 45 min to get Y by 'multiply_'"]}}
-3. You must ONLY output in a parsible JSON format. An example output looks like:
+5. You must ONLY output in a parsible JSON format. An example output looks like:
 
 {{"Tasks": ["Task 1", "Task 2", ...]}}
 
@@ -65,8 +69,10 @@ These are the tools you can select to solve the question:
 {tools}
 
 Please note that:
-1. You should only choose one tool from the Tool List to solve this question.
-2. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like:
+1. You should only choose one tool from the Tool List to solve this question and it should have maximum chance of solving the question.
+2. You should only choose the tool whose parameters are most relevant to the user's question and are availale as part of the question.
+3. You should choose the tool whose return type is most relevant to the answer of the user's question.
+4. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like:
 
 Example 1: {{"ID": 1}}
 Example 2: {{"ID": 2}}
@@ -81,8 +87,10 @@ This is a reflection from a previous failed attempt:
 {reflections}
 
 Please note that:
-1. You should only choose one tool from the Tool List to solve this question.
-2. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like:
+1. You should only choose one tool from the Tool List to solve this question and it should have maximum chance of solving the question.
+2. You should only choose the tool whose parameters are most relevant to the user's question and are availale as part of the question.
+3. You should choose the tool whose return type is most relevant to the answer of the user's question.
+4. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like:
 
 Example 1: {{"ID": 1}}
 Example 2: {{"ID": 2}}
@@ -91,7 +99,7 @@ Output: """
 
 CHOOSE_PARAMETER_DEPENDS = """Given a user's question and an API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.
 Please note that:
-1. The Example in the API tool documentation can help you better understand the use of the API.
+1. The Example in the API tool documentation can help you better understand the use of the API. Pay attention to the examples which show how to parse the question and extract tool parameters such as prompts and visual inputs.
 2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If there are no paremters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}}
 3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.
 4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers for your reference.

vision_agent/image_utils.py

@@ -238,7 +238,7 @@ def overlay_heat_map(
     elif isinstance(image, np.ndarray):
         image = Image.fromarray(image)
 
-    if "heat_map" not in heat_map:
+    if "heat_map" not in heat_map or len(heat_map["heat_map"]) == 0:
         return image.convert("RGB")
 
     image = image.convert("L")

vision_agent/tools/tools.py

@@ -175,7 +175,7 @@ class GroundingDINO(Tool):
     """
 
     name = "grounding_dino_"
-    description = "'grounding_dino_' is a tool that can detect arbitrary objects with inputs such as category names or referring expressions. It returns a list of bounding boxes, label names and associated probability scores."
+    description = "'grounding_dino_' is a tool that can detect and count objects given a text prompt such as category names or referring expressions. It returns a list and count of bounding boxes, label names and associated probability scores."
     usage = {
         "required_parameters": [
             {"name": "prompt", "type": "str"},
@@ -186,6 +186,13 @@ class GroundingDINO(Tool):
             {"name": "iou_threshold", "type": "float"},
         ],
         "examples": [
+            {
+                "scenario": "Can you detect and count the giraffes and zebras in this image? Image name: animal.jpg",
+                "parameters": {
+                    "prompt": "giraffe. zebra",
+                    "image": "person.jpg",
+                },
+            },
             {
                 "scenario": "Can you build me a car detector?",
                 "parameters": {"prompt": "car", "image": ""},
@@ -198,7 +205,7 @@ class GroundingDINO(Tool):
                 },
             },
             {
-                "scenario": "Detect the red shirts and green shirst. Image name: shirts.jpg",
+                "scenario": "Detect the red shirts and green shirt. Image name: shirts.jpg",
                 "parameters": {
                     "prompt": "red shirt. green shirt",
                     "image": "shirts.jpg",
@@ -271,7 +278,7 @@ class GroundingSAM(Tool):
     """
 
     name = "grounding_sam_"
-    description = "'grounding_sam_' is a tool that can detect arbitrary objects with inputs such as category names or referring expressions. It returns a list of bounding boxes, label names and masks file names and associated probability scores."
+    description = "'grounding_sam_' is a tool that can detect and segment objects given a text prompt such as category names or referring expressions. It returns a list of bounding boxes, label names and masks file names and associated probability scores."
     usage = {
         "required_parameters": [
             {"name": "prompt", "type": "str"},
@@ -282,6 +289,13 @@ class GroundingSAM(Tool):
             {"name": "iou_threshold", "type": "float"},
         ],
         "examples": [
+            {
+                "scenario": "Can you segment the apples and grapes in this image? Image name: fruits.jpg",
+                "parameters": {
+                    "prompt": "apple. grape",
+                    "image": "fruits.jpg",
+                },
+            },
             {
                 "scenario": "Can you build me a car segmentor?",
                 "parameters": {"prompt": "car", "image": ""},
@@ -478,7 +492,7 @@ class ZeroShotCounting(Tool):
     """
 
     name = "zero_shot_counting_"
-    description = "'zero_shot_counting_' is a tool that counts and returns the total number of instances of an object present in an image belonging to the same class without a text or visual prompt."
+    description = "'zero_shot_counting_' is a tool that counts foreground items given only an image and no other information. It returns only the count of the objects in the image"
 
     usage = {
         "required_parameters": [
@@ -486,7 +500,7 @@ class ZeroShotCounting(Tool):
         ],
         "examples": [
             {
-                "scenario": "Can you count the lids in the image? Image name: lids.jpg",
+                "scenario": "Can you count the items in the image? Image name: lids.jpg",
                 "parameters": {"image": "lids.jpg"},
             },
             {
@@ -535,7 +549,7 @@ class VisualPromptCounting(Tool):
     """
 
     name = "visual_prompt_counting_"
-    description = "'visual_prompt_counting_' is a tool that can count and return total number of instances of an object present in an image belonging to the same class given an example bounding box."
+    description = "'visual_prompt_counting_' is a tool that counts foreground items in an image given a visual prompt which is a bounding box describing the object. It returns only the count of the objects in the image."
 
     usage = {
         "required_parameters": [
@@ -544,7 +558,7 @@ class VisualPromptCounting(Tool):
         ],
         "examples": [
             {
-                "scenario": "Here is an example of a lid '0.1, 0.1, 0.14, 0.2', Can you count the lids in the image ? Image name: lids.jpg",
+                "scenario": "Here is an example of a lid '0.1, 0.1, 0.14, 0.2', Can you count the items in the image ? Image name: lids.jpg",
                 "parameters": {"image": "lids.jpg", "prompt": "0.1, 0.1, 0.14, 0.2"},
             },
             {
@@ -552,7 +566,7 @@ class VisualPromptCounting(Tool):
                 "parameters": {"image": "tray.jpg", "prompt": "0.1, 0.1, 0.2, 0.25"},
             },
             {
-                "scenario": "Can you build me a few shot object counting tool ? Image name: shirts.jpg",
+                "scenario": "Can you count this item based on an example, reference_data: '0.1, 0.15, 0.2, 0.2' ? Image name: shirts.jpg",
                 "parameters": {
                     "image": "shirts.jpg",
                     "prompt": "0.1, 0.15, 0.2, 0.2",
@@ -605,7 +619,7 @@ class VisualQuestionAnswering(Tool):
     """
 
     name = "visual_question_answering_"
-    description = "'visual_question_answering_' is a tool that can describe the contents of the image and it can also answer basic questions about the image."
+    description = "'visual_question_answering_' is a tool that can answer basic questions about the image given a question and an image. It returns a text describing the image and the answer to the question"
 
     usage = {
         "required_parameters": [
@@ -672,7 +686,7 @@ class ImageQuestionAnswering(Tool):
     """
 
     name = "image_question_answering_"
-    description = "'image_question_answering_' is a tool that can describe the contents of the image and it can also answer basic questions about the image."
+    description = "'image_question_answering_' is a tool that can answer basic questions about the image given a question and an image. It returns a text describing the image and the answer to the question"
 
     usage = {
         "required_parameters": [
@@ -773,7 +787,7 @@ class BboxArea(Tool):
     r"""BboxArea returns the area of the bounding box in pixels normalized to 2 decimal places."""
 
     name = "bbox_area_"
-    description = "'bbox_area_' returns the area of the bounding box in pixels normalized to 2 decimal places."
+    description = "'bbox_area_' returns the area of the given bounding box in pixels normalized to 2 decimal places."
     usage = {
         "required_parameters": [{"name": "bboxes", "type": "List[int]"}],
         "examples": [
@@ -803,7 +817,7 @@ class SegArea(Tool):
     r"""SegArea returns the area of the segmentation mask in pixels normalized to 2 decimal places."""
 
     name = "seg_area_"
-    description = "'seg_area_' returns the area of the segmentation mask in pixels normalized to 2 decimal places."
+    description = "'seg_area_' returns the area of the given segmentation mask in pixels normalized to 2 decimal places."
     usage = {
         "required_parameters": [{"name": "masks", "type": "str"}],
         "examples": [
@@ -883,7 +897,7 @@ class SegIoU(Tool):
 
 class BboxContains(Tool):
     name = "bbox_contains_"
-    description = "Given two bounding boxes, a target bounding box and a region bounding box, 'bbox_contains_' returns the intersection of the two bounding boxes over the target bounding box, reflects the percentage area of the target bounding box overlaps with the region bounding box. This is a good tool for determining if the region object contains the target object."
+    description = "Given two bounding boxes, a target bounding box and a region bounding box, 'bbox_contains_' returns the intersection of the two bounding boxes which is the percentage area of the target bounding box overlaps with the region bounding box. This is a good tool for determining if the region object contains the target object."
     usage = {
         "required_parameters": [
             {"name": "target", "type": "List[int]"},
@@ -935,9 +949,7 @@ class BboxContains(Tool):
 
 class BoxDistance(Tool):
     name = "box_distance_"
-    description = (
-        "'box_distance_' returns the minimum distance between two bounding boxes."
-    )
+    description = "'box_distance_' calculates distance between two bounding boxes. It returns the minumum distance between the given bounding boxes"
     usage = {
         "required_parameters": [
             {"name": "bbox1", "type": "List[int]"},
@@ -945,7 +957,7 @@ class BoxDistance(Tool):
         ],
         "examples": [
             {
-                "scenario": "If you want to calculate the distance between the bounding boxes [0.2, 0.21, 0.34, 0.42] and [0.3, 0.31, 0.44, 0.52]",
+                "scenario": "Calculate the distance between the bounding boxes [0.2, 0.21, 0.34, 0.42] and [0.3, 0.31, 0.44, 0.52]",
                 "parameters": {
                     "bbox1": [0.2, 0.21, 0.34, 0.42],
                     "bbox2": [0.3, 0.31, 0.44, 0.52],
@@ -1008,7 +1020,7 @@ class ExtractFrames(Tool):
 
 class OCR(Tool):
     name = "ocr_"
-    description = "'ocr_' extracts text from an image."
+    description = "'ocr_' extracts text from an image. It returns a list of detected text, bounding boxes, and confidence scores."
     usage = {
         "required_parameters": [
             {"name": "image", "type": "str"},

{vision_agent-0.2.7.dist-info → vision_agent-0.2.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.7
+Version: 0.2.9
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.7.dist-info → vision_agent-0.2.9.dist-info}/RECORD

@@ -2,24 +2,24 @@ vision_agent/__init__.py,sha256=GVLHCeK_R-zgldpbcPmOzJat-BkadvkuRCMxDvTIcXs,108
 vision_agent/agent/__init__.py,sha256=B4JVrbY4IRVCJfjmrgvcp7h1mTUEk8MZvL0Zmej4Ka0,127
 vision_agent/agent/agent.py,sha256=X7kON-g9ePUKumCDaYfQNBX_MEFE-ax5PnRp7-Cc5Wo,529
 vision_agent/agent/easytool.py,sha256=oMHnBg7YBtIPgqQUNcZgq7uMgpPThs99_UnO7ERkMVg,11511
-vision_agent/agent/easytool_prompts.py,sha256=zdQQw6WpXOmvwOMtlBlNKY5a3WNlr65dbUvMIGiqdeo,4526
+vision_agent/agent/easytool_prompts.py,sha256=Bikw-PPLkm78dwywTlnv32Y1Tw6JMeC-R7oCnXWLcTk,4656
 vision_agent/agent/reflexion.py,sha256=4gz30BuFMeGxSsTzoDV4p91yE0R8LISXp28IaOI6wdM,10506
 vision_agent/agent/reflexion_prompts.py,sha256=G7UAeNz_g2qCb2yN6OaIC7bQVUkda4m3z42EG8wAyfE,9342
-vision_agent/agent/vision_agent.py,sha256=Hp-w0l_Xzja2iqsOHuBBVgHhcaEN6Ue2zElUizgfom4,26024
-vision_agent/agent/vision_agent_prompts.py,sha256=W3Z72FpUt71UIJSkjAcgtQqxeMqkYuATqHAN5fYY26c,7342
+vision_agent/agent/vision_agent.py,sha256=PyAtzDl5h1Uasd-Fjzdl-NK9gdZ2ARxoF9y3tvap7PU,26243
+vision_agent/agent/vision_agent_prompts.py,sha256=moihXFhEzFw8xnf2sUSgd_k9eoxQam3T6XUkB0fyp5o,8570
 vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
-vision_agent/image_utils.py,sha256=BfUueXZKOvjdI_J7vi2vE57FfpaHOkxrTY0d3aQ1zgI,7552
+vision_agent/image_utils.py,sha256=1dggPBhW8_hUXDItCRLa23h-hdBwS50cjL4v1hsoUbg,7586
 vision_agent/llm/__init__.py,sha256=BoUm_zSAKnLlE8s-gKTSQugXDqVZKPqYlWwlTLdhcz4,48
 vision_agent/llm/llm.py,sha256=1BkrSVBWEClyqLc0Rmyw4heLhi_ZVm6JO7-i1wd1ziw,5383
 vision_agent/lmm/__init__.py,sha256=nnNeKD1k7q_4vLb1x51O_EUTYaBgGfeiCx5F433gr3M,67
 vision_agent/lmm/lmm.py,sha256=gK90vMxh0OcGSuIZQikBkDXm4pfkdFk1R2y7rtWDl84,10539
 vision_agent/tools/__init__.py,sha256=HfUr0JQUwk0Kyieen93df9lMbbdpVf9Q6CcVFmKv_q4,413
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
-vision_agent/tools/tools.py,sha256=DZX-w17XWttR4j8bQKY90QxLOs3-ZD5qOHA_53LL7Dk,43013
+vision_agent/tools/tools.py,sha256=EvNDLUxe-Ed8-meHInTIiX3aySLUXFBsAWwL0Is5S1o,43823
 vision_agent/tools/video.py,sha256=xTElFSFp1Jw4ulOMnk81Vxsh-9dTxcWUO6P9fzEi3AM,7653
 vision_agent/type_defs.py,sha256=4LTnTL4HNsfYqCrDn9Ppjg9bSG2ZGcoKSSd9YeQf4Bw,1792
-vision_agent-0.2.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.7.dist-info/METADATA,sha256=69stJrRINrUv19y3j84wGtn6h0XsAM5VY7vtDTiga0M,7697
-vision_agent-0.2.7.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.7.dist-info/RECORD,,
+vision_agent-0.2.9.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.9.dist-info/METADATA,sha256=jyfAwSfDnObeILoLyfB8ijuLLpZUWd-Fvg-xncEMCYc,7697
+vision_agent-0.2.9.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.9.dist-info/RECORD,,