vision-agent 0.2.147__py3-none-any.whl → 0.2.149__py3-none-any.whl

vision_agent/agent/vision_agent.py

@@ -87,7 +87,7 @@ def run_conversation(orch: LMM, chat: List[Message]) -> Dict[str, Any]:
     return extract_json(orch([message], stream=False))  # type: ignore
 
 
-def run_code_action(
+def execute_code_action(
     code: str, code_interpreter: CodeInterpreter, artifact_remote_path: str
 ) -> Tuple[Execution, str]:
     result = code_interpreter.exec_isolation(
@@ -106,19 +106,53 @@ def parse_execution(
     customed_tool_names: Optional[List[str]] = None,
 ) -> Optional[str]:
     code = None
-    if "<execute_python>" in response:
-        code = response[response.find("<execute_python>") + len("<execute_python>") :]
-        code = code[: code.find("</execute_python>")]
+    remaining = response
+    all_code = []
+    while "<execute_python>" in remaining:
+        code_i = remaining[
+            remaining.find("<execute_python>") + len("<execute_python>") :
+        ]
+        code_i = code_i[: code_i.find("</execute_python>")]
+        remaining = remaining[
+            remaining.find("</execute_python>") + len("</execute_python>") :
+        ]
+        all_code.append(code_i)
+
+    if len(all_code) > 0:
+        code = "\n".join(all_code)
 
     if code is not None:
         code = use_extra_vision_agent_args(code, test_multi_plan, customed_tool_names)
     return code
 
 
+def execute_user_code_action(
+    last_user_message: Message,
+    code_interpreter: CodeInterpreter,
+    artifact_remote_path: str,
+) -> Tuple[Optional[Execution], Optional[str]]:
+    user_result = None
+    user_obs = None
+
+    if last_user_message["role"] != "user":
+        return user_result, user_obs
+
+    last_user_content = cast(str, last_user_message.get("content", ""))
+
+    user_code_action = parse_execution(last_user_content, False)
+    if user_code_action is not None:
+        user_result, user_obs = execute_code_action(
+            user_code_action, code_interpreter, artifact_remote_path
+        )
+        if user_result.error:
+            user_obs += f"\n{user_result.error}"
+    return user_result, user_obs
+
+
 class VisionAgent(Agent):
     """Vision Agent is an agent that can chat with the user and call tools or other
     agents to generate code for it. Vision Agent uses python code to execute actions
-    for the user. Vision Agent is inspired by by OpenDev
+    for the user. Vision Agent is inspired by by OpenDevin
     https://github.com/OpenDevin/OpenDevin and CodeAct https://arxiv.org/abs/2402.01030
 
     Example
@@ -278,9 +312,24 @@ class VisionAgent(Agent):
             orig_chat.append({"role": "observation", "content": artifacts_loaded})
             self.streaming_message({"role": "observation", "content": artifacts_loaded})
 
-            finished = self.execute_user_code_action(
-                last_user_message, code_interpreter, remote_artifacts_path
+            user_result, user_obs = execute_user_code_action(
+                last_user_message, code_interpreter, str(remote_artifacts_path)
             )
+            finished = user_result is not None and user_obs is not None
+            if user_result is not None and user_obs is not None:
+                # be sure to update the chat with user execution results
+                chat_elt: Message = {"role": "observation", "content": user_obs}
+                int_chat.append(chat_elt)
+                chat_elt["execution"] = user_result
+                orig_chat.append(chat_elt)
+                self.streaming_message(
+                    {
+                        "role": "observation",
+                        "content": user_obs,
+                        "execution": user_result,
+                        "finished": finished,
+                    }
+                )
 
             while not finished and iterations < self.max_iterations:
                 response = run_conversation(self.agent, int_chat)
@@ -322,7 +371,7 @@ class VisionAgent(Agent):
                     )
 
                 if code_action is not None:
-                    result, obs = run_code_action(
+                    result, obs = execute_code_action(
                         code_action, code_interpreter, str(remote_artifacts_path)
                     )
 
@@ -331,17 +380,17 @@ class VisionAgent(Agent):
                     if self.verbosity >= 1:
                         _LOGGER.info(obs)
 
-                    chat_elt: Message = {"role": "observation", "content": obs}
+                    obs_chat_elt: Message = {"role": "observation", "content": obs}
                     if media_obs and result.success:
-                        chat_elt["media"] = [
+                        obs_chat_elt["media"] = [
                             Path(code_interpreter.remote_path) / media_ob
                             for media_ob in media_obs
                         ]
 
                     # don't add execution results to internal chat
-                    int_chat.append(chat_elt)
-                    chat_elt["execution"] = result
-                    orig_chat.append(chat_elt)
+                    int_chat.append(obs_chat_elt)
+                    obs_chat_elt["execution"] = result
+                    orig_chat.append(obs_chat_elt)
                     self.streaming_message(
                         {
                             "role": "observation",
@@ -362,34 +411,6 @@ class VisionAgent(Agent):
             artifacts.save()
         return orig_chat, artifacts
 
-    def execute_user_code_action(
-        self,
-        last_user_message: Message,
-        code_interpreter: CodeInterpreter,
-        remote_artifacts_path: Path,
-    ) -> bool:
-        if last_user_message["role"] != "user":
-            return False
-        user_code_action = parse_execution(
-            cast(str, last_user_message.get("content", "")), False
-        )
-        if user_code_action is not None:
-            user_result, user_obs = run_code_action(
-                user_code_action, code_interpreter, str(remote_artifacts_path)
-            )
-            if self.verbosity >= 1:
-                _LOGGER.info(user_obs)
-            self.streaming_message(
-                {
-                    "role": "observation",
-                    "content": user_obs,
-                    "execution": user_result,
-                    "finished": True,
-                }
-            )
-            return True
-        return False
-
     def streaming_message(self, message: Dict[str, Any]) -> None:
         if self.callback_message:
             self.callback_message(message)

vision_agent/agent/vision_agent_coder.py

@@ -691,7 +691,7 @@ class VisionAgentCoder(Agent):
         chat: List[Message],
         test_multi_plan: bool = True,
         display_visualization: bool = False,
-        customized_tool_names: Optional[List[str]] = None,
+        custom_tool_names: Optional[List[str]] = None,
     ) -> Dict[str, Any]:
         """Chat with VisionAgentCoder and return intermediate information regarding the
         task.
@@ -707,8 +707,8 @@ class VisionAgentCoder(Agent):
                 with the first plan.
             display_visualization (bool): If True, it opens a new window locally to
                 show the image(s) created by visualization code (if there is any).
-            customized_tool_names (List[str]): A list of customized tools for agent to pick and use.
-                If not provided, default to full tool set from vision_agent.tools.
+            custom_tool_names (List[str]): A list of custom tools for the agent to pick
+                and use. If not provided, default to full tool set from vision_agent.tools.
 
         Returns:
             Dict[str, Any]: A dictionary containing the code, test, test result, plan,
@@ -760,7 +760,7 @@ class VisionAgentCoder(Agent):
             success = False
 
             plans = self._create_plans(
-                int_chat, customized_tool_names, working_memory, self.planner
+                int_chat, custom_tool_names, working_memory, self.planner
             )
 
             if test_multi_plan:

vision_agent/agent/vision_agent_coder_prompts.py

@@ -67,14 +67,7 @@ This is the documentation for the functions you have access to. You may call any
 **Previous Attempts**:
 {previous_attempts}
 
-**Instructions**:
-1. Write a program to load the media and call each tool and print it's output along with other relevant information.
-2. Create a dictionary where the keys are the tool name and the values are the tool outputs. Remove numpy arrays from the printed dictionary.
-3. Your test case MUST run only on the given images which are {media}
-4. Print this final dictionary.
-5. For video input, sample at 1 FPS and use the first 10 frames only to reduce processing time.
-
-**Example**:
+**Examples**:
 --- EXAMPLE1 ---
 plan1:
 - Load the image from the provided file path 'image.jpg'.
@@ -100,6 +93,7 @@ cgd_out = countgd_counting(image)
 
 final_out = {{"owl_v2_image": owl_v2_out, "florence2_sam2_image": f2s2, "countgd_counting": cgd_out}}
 print(final_out)
+--- END EXAMPLE1 ---
 
 --- EXAMPLE2 ---
 plan1:
@@ -173,6 +167,14 @@ print(final_out)
 print(labels_and_scores)
 print(counts)
 ```
+--- END EXAMPLE2 ---
+
+**Instructions**:
+1. Write a program to load the media and call each tool and print it's output along with other relevant information.
+2. Create a dictionary where the keys are the tool name and the values are the tool outputs. Remove numpy arrays from the printed dictionary.
+3. Your test case MUST run only on the given images which are {media}
+4. Print this final dictionary.
+5. For video input, sample at 1 FPS and use the first 10 frames only to reduce processing time.
 """
 
 
@@ -224,11 +226,6 @@ This is the documentation for the functions you have access to. You may call any
 
 {docstring}
 
-**Input Code Snippet**:
-```python
-# Your code here
-```
-
 **User Instructions**:
 {question}
 
@@ -241,11 +238,90 @@ This is the documentation for the functions you have access to. You may call any
 **Previous Feedback**:
 {feedback}
 
+**Examples**:
+--- EXAMPLE1 ---
+**User Instructions**:
+
+## User Request
+Can you write a program to check if each person is wearing a helmet? First detect all the people in the image, then detect the helmets, check whether or not a person is wearing a helmet if the helmet is on the worker. Return a dictionary with the count of people with helments and people without helmets. Media name worker_helmets.webp
+
+## Subtasks
+
+This plan uses the owl_v2_image tool to detect both people and helmets in a single pass, which should be efficient and accurate. We can then compare the detections to determine if each person is wearing a helmet.
+-Use owl_v2_image with prompt 'person, helmet' to detect both people and helmets in the image
+-Process the detections to match helmets with people based on bounding box proximity
+-Count people with and without helmets based on the matching results
+-Return a dictionary with the counts
+
+
+**Tool Tests and Outputs**:
+After examining the image, I can see 4 workers in total, with 3 wearing yellow safety helmets and 1 not wearing a helmet. Plan 1 using owl_v2_image seems to be the most accurate in detecting both people and helmets. However, it needs some modifications to improve accuracy. We should increase the confidence threshold to 0.15 to filter out the lowest confidence box, and implement logic to associate helmets with people based on their bounding box positions. Plan 2 and Plan 3 seem less reliable given the tool outputs, as they either failed to distinguish between people with and without helmets or misclassified all workers as not wearing helmets.
+
+**Tool Output Thoughts**:
+```python
+...
+```
+----- stdout -----
+Plan 1 - owl_v2_image:
+
+[{{'label': 'helmet', 'score': 0.15, 'bbox': [0.85, 0.41, 0.87, 0.45]}}, {{'label': 'helmet', 'score': 0.3, 'bbox': [0.8, 0.43, 0.81, 0.46]}}, {{'label': 'helmet', 'score': 0.31, 'bbox': [0.85, 0.45, 0.86, 0.46]}}, {{'label': 'person', 'score': 0.31, 'bbox': [0.84, 0.45, 0.88, 0.58]}}, {{'label': 'person', 'score': 0.31, 'bbox': [0.78, 0.43, 0.82, 0.57]}}, {{'label': 'helmet', 'score': 0.33, 'bbox': [0.3, 0.65, 0.32, 0.67]}}, {{'label': 'person', 'score': 0.29, 'bbox': [0.28, 0.65, 0.36, 0.84]}}, {{'label': 'helmet', 'score': 0.29, 'bbox': [0.13, 0.82, 0.15, 0.85]}}, {{'label': 'person', 'score': 0.3, 'bbox': [0.1, 0.82, 0.24, 1.0]}}]
+
+...
+
+**Input Code Snippet**:
+```python
+from vision_agent.tools import load_image, owl_v2_image
+
+def check_helmets(image_path):
+    image = load_image(image_path)
+    # Detect people and helmets, filter out the lowest confidence helmet score of 0.15
+    detections = owl_v2_image("person, helmet", image, box_threshold=0.15)
+    height, width = image.shape[:2]
+
+    # Separate people and helmets
+    people = [d for d in detections if d['label'] == 'person']
+    helmets = [d for d in detections if d['label'] == 'helmet']
+
+    people_with_helmets = 0
+    people_without_helmets = 0
+
+    for person in people:
+        person_x = (person['bbox'][0] + person['bbox'][2]) / 2
+        person_y = person['bbox'][1]  # Top of the bounding box
+
+        helmet_found = False
+        for helmet in helmets:
+            helmet_x = (helmet['bbox'][0] + helmet['bbox'][2]) / 2
+            helmet_y = (helmet['bbox'][1] + helmet['bbox'][3]) / 2
+
+            # Check if the helmet is within 20 pixels of the person's head. Unnormalize
+            # the coordinates so we can better compare them.
+            if (abs((helmet_x - person_x) * width) < 20 and
+                -5 < ((helmet_y - person_y) * height) < 20):
+                helmet_found = True
+                break
+
+        if helmet_found:
+            people_with_helmets += 1
+        else:
+            people_without_helmets += 1
+
+    return {{
+        "people_with_helmets": people_with_helmets,
+        "people_without_helmets": people_without_helmets
+    }}
+```
+--- END EXAMPLE1 ---
+
 **Instructions**:
 1. **Understand and Clarify**: Make sure you understand the task.
 2. **Algorithm/Method Selection**: Decide on the most efficient method, use the tool outputs and tool thoughts to guide you.
 3. **Pseudocode Creation**: Write down the steps you will follow in pseudocode.
-4. **Code Generation**: Translate your pseudocode into executable Python code. Ensure you use correct arguments, remember coordinates are always returned normalized from `vision_agent.tools`. All images from `vision_agent.tools` are in RGB format, red is (255, 0, 0) and blue is (0, 0, 255).
+4. **Code Generation**: Translate your pseudocode into executable Python code.
+    4.1. Take in the media path as an argument and load with either `load_image` or `extract_frames_and_timestamps`.
+    4.2. Coordinates are always returned normalized from `vision_agent.tools`.
+    4.3. Do not create dummy input or functions, the code must be usable if the user provides new media.
+    4.4. Use unnormalized coordinates when comparing bounding boxes.
 """
 
 TEST = """

vision_agent/agent/vision_agent_prompts.py

@@ -26,7 +26,9 @@ Here is the current conversation so far:
 
 **Instructions**:
 1. **Understand and Clarify**: Make sure you understand the task, ask clarifying questions if the task is not clear.
-2. **Output in JSON**: Respond in the following format in JSON:
+2. **Code Generation**: Only use code provided in the Documentation in your <execute_python> tags. Only use `edit_vision_code` to modify code written by `generate_vision_code`.
+3. **Execute**: Do only what the user asked you to do and no more. If you need to ask the user a question, set `let_user_respond` to `true`.
+4. **Output in JSON**: Respond in the following format in JSON:
 
 ```json
 {{"thoughts": <your thoughts>, "response": <your response to the user>, "let_user_respond": <a boolean whether or not to let the user respond>}}.
@@ -149,7 +151,7 @@ OBSERVATION:
 16|    return count
 [End of artifact]
 
-AGENT: {"thoughts": "I have generated the code to count the workers with helmets in the image, I must now run the code and print the output and write the visualization to the artifacts so I can see the result and the user can see the visaulization.", "response": "<execute_python>from code import count_workers_with_helmets\n    print(count_workers_with_helmets('/path/to/images/workers.png', 'workers_viz.png'))\n    write_media_artifact(artifacts, 'workers_viz.png')</execute_python>", "let_user_respond": false}
+AGENT: {"thoughts": "I have generated the code to count the workers with helmets in the image, I must now run the code and print the output and write the visualization to the artifacts so I can see the result and the user can see the visaulization.", "response": "<execute_python>from code import count_workers_with_helmets\n    print(count_workers_with_helmets('/path/to/images/workers.png', 'workers_viz.png'))\n    write_media_artifact(artifacts, 'workers_viz.png', 'workers_viz.png')</execute_python>", "let_user_respond": false}
 
 OBSERVATION:
 ----- stdout -----
@@ -164,7 +166,6 @@ AGENT: {"thoughts": "Because the user has supplied me with labels I can call obj
 OBSERVATION:
 [Fine tuning id: 23b3b022-5ebf-4798-9373-20ef36429abf]
 
-
 AGENT: {"thoughts": "The model has finished fine tuning, I will now replace the original florence2_phrase_grounding call with the fine tuning id.", "response": "I will not update the code to use the fine tuned model. <execute_python>use_object_detection_fine_tuning(artifacts, "code.py", "23b3b022-5ebf-4798-9373-20ef36429abf")</execute_python>", "let_user_respond": false}
 
 OBSERVATION:
@@ -182,8 +183,7 @@ OBSERVATION:
      count = 0
 [End of edits]
 
-
-AGENT: {"thoughts": "I have updated the code to use the fine tuned model, I must now run the code and print the output and write the visualization to the artifacts so I can see the result and the user can see the visaulization.", "response": "<execute_python>from code import count_workers_with_helmets\n    print(count_workers_with_helmets('/path/to/images/workers.png', 'workers_viz.png'))\n    write_media_artifact(artifacts, 'workers_viz.png')</execute_python>", "let_user_respond": false}
+AGENT: {"thoughts": "I have updated the code to use the fine tuned model, I must now run the code and print the output and write the visualization to the artifacts so I can see the result and the user can see the visaulization.", "response": "<execute_python>from code import count_workers_with_helmets\n    print(count_workers_with_helmets('/path/to/images/workers.png', 'workers_viz.png'))\n    write_media_artifact(artifacts, 'workers_viz.png', 'workers_viz.png')</execute_python>", "let_user_respond": false}
 
 OBSERVATION:
 ----- stdout -----

vision_agent/tools/meta_tools.py

@@ -8,6 +8,7 @@ import tempfile
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 
+import numpy as np
 from IPython.display import display
 
 import vision_agent as va
@@ -17,7 +18,8 @@ from vision_agent.tools.tool_utils import get_tool_documentation
 from vision_agent.tools.tools import TOOL_DESCRIPTIONS
 from vision_agent.tools.tools_types import BboxInput, BboxInputBase64, PromptTask
 from vision_agent.utils.execute import Execution, MimeType
-from vision_agent.utils.image_utils import convert_to_b64
+from vision_agent.utils.image_utils import convert_to_b64, numpy_to_bytes
+from vision_agent.utils.video import frames_to_bytes
 
 # These tools are adapted from SWE-Agent https://github.com/princeton-nlp/SWE-agent
 
@@ -328,7 +330,7 @@ def generate_vision_code(
     chat: str,
     media: List[str],
     test_multi_plan: bool = True,
-    customized_tool_names: Optional[List[str]] = None,
+    custom_tool_names: Optional[List[str]] = None,
 ) -> str:
     """Generates python code to solve vision based tasks.
 
@@ -338,7 +340,7 @@ def generate_vision_code(
         chat (str): The chat message from the user.
         media (List[str]): The media files to use.
         test_multi_plan (bool): Do not change this parameter.
-        customized_tool_names (Optional[List[str]]): Do not change this parameter.
+        custom_tool_names (Optional[List[str]]): Do not change this parameter.
 
     Returns:
         str: The generated code.
@@ -366,7 +368,7 @@ def generate_vision_code(
     response = agent.chat_with_workflow(
         fixed_chat,
         test_multi_plan=test_multi_plan,
-        customized_tool_names=customized_tool_names,
+        custom_tool_names=custom_tool_names,
     )
     redisplay_results(response["test_result"])
     code = response["code"]
@@ -432,19 +434,21 @@ def edit_vision_code(
 
     # Append latest code to second to last message from assistant
     fixed_chat_history: List[Message] = []
+    user_message = "Previous user requests:"
     for i, chat in enumerate(chat_history):
-        if i == 0:
-            fixed_chat_history.append({"role": "user", "content": chat, "media": media})
-        elif i > 0 and i < len(chat_history) - 1:
-            fixed_chat_history.append({"role": "user", "content": chat})
-        elif i == len(chat_history) - 1:
+        if i < len(chat_history) - 1:
+            user_message += " " + chat
+        else:
+            fixed_chat_history.append(
+                {"role": "user", "content": user_message, "media": media}
+            )
             fixed_chat_history.append({"role": "assistant", "content": code})
             fixed_chat_history.append({"role": "user", "content": chat})
 
     response = agent.chat_with_workflow(
         fixed_chat_history,
         test_multi_plan=False,
-        customized_tool_names=customized_tool_names,
+        custom_tool_names=customized_tool_names,
     )
     redisplay_results(response["test_result"])
     code = response["code"]
@@ -467,17 +471,34 @@ def edit_vision_code(
     return view_lines(code_lines, 0, total_lines, name, total_lines)
 
 
-def write_media_artifact(artifacts: Artifacts, local_path: str) -> str:
+def write_media_artifact(
+    artifacts: Artifacts,
+    name: str,
+    media: Union[str, np.ndarray, List[np.ndarray]],
+    fps: Optional[float] = None,
+) -> str:
     """Writes a media file to the artifacts object.
 
     Parameters:
         artifacts (Artifacts): The artifacts object to save the media to.
-        local_path (str): The local path to the media file.
+        name (str): The name of the media artifact to save.
+        media (Union[str, np.ndarray, List[np.ndarray]]): The media to save, can either
+            be a file path, single image or list of frames for a video.
+        fps (Optional[float]): The frames per second if you are writing a video.
     """
-    with open(local_path, "rb") as f:
-        media = f.read()
-    artifacts[Path(local_path).name] = media
-    return f"[Media {Path(local_path).name} saved]"
+    if isinstance(media, str):
+        with open(media, "rb") as f:
+            media_bytes = f.read()
+    elif isinstance(media, list):
+        media_bytes = frames_to_bytes(media, fps=fps if fps is not None else 1.0)
+    elif isinstance(media, np.ndarray):
+        media_bytes = numpy_to_bytes(media)
+    else:
+        print(f"[Invalid media type {type(media)}]")
+        return f"[Invalid media type {type(media)}]"
+    artifacts[name] = media_bytes
+    print(f"[Media {name} saved]")
+    return f"[Media {name} saved]"
 
 
 def list_artifacts(artifacts: Artifacts) -> str:
@@ -491,16 +512,14 @@ def check_and_load_image(code: str) -> List[str]:
     if not code.strip():
         return []
 
-    pattern = r"show_media_artifact\(\s*([^\)]+),\s*['\"]([^\)]+)['\"]\s*\)"
-    match = re.search(pattern, code)
-    if match:
-        name = match.group(2)
-        return [name]
-    return []
+    pattern = r"view_media_artifact\(\s*([^\)]+),\s*['\"]([^\)]+)['\"]\s*\)"
+    matches = re.findall(pattern, code)
+    return [match[1] for match in matches]
 
 
 def view_media_artifact(artifacts: Artifacts, name: str) -> str:
-    """Views the image artifact with the given name.
+    """Allows you to view the media artifact with the given name. This does not show
+    the media to the user, the user can already see all media saved in the artifacts.
 
     Parameters:
         artifacts (Artifacts): The artifacts object to show the image from.
@@ -598,7 +617,7 @@ def use_extra_vision_agent_args(
         arg = match.group(1)
         out_str = f"generate_vision_code({arg}, test_multi_plan={test_multi_plan}"
         if customized_tool_names is not None:
-            out_str += f", customized_tool_names={customized_tool_names})"
+            out_str += f", custom_tool_names={customized_tool_names})"
         else:
             out_str += ")"
         return out_str
@@ -609,7 +628,7 @@ def use_extra_vision_agent_args(
         arg = match.group(1)
         out_str = f"edit_vision_code({arg}"
         if customized_tool_names is not None:
-            out_str += f", customized_tool_names={customized_tool_names})"
+            out_str += f", custom_tool_names={customized_tool_names})"
         else:
             out_str += ")"
         return out_str
@@ -646,50 +665,28 @@ def use_object_detection_fine_tuning(
 
     patterns_with_fine_tune_id = [
         (
-            r'florence2_phrase_grounding\(\s*"([^"]+)"\s*,\s*([^,]+)(?:,\s*"[^"]+")?\s*\)',
+            r'florence2_phrase_grounding\(\s*["\']([^"\']+)["\']\s*,\s*([^,]+)(?:,\s*["\'][^"\']+["\'])?\s*\)',
             lambda match: f'florence2_phrase_grounding("{match.group(1)}", {match.group(2)}, "{fine_tune_id}")',
         ),
         (
-            r'owl_v2_image\(\s*"([^"]+)"\s*,\s*([^,]+)(?:,\s*"[^"]+")?\s*\)',
+            r'owl_v2_image\(\s*["\']([^"\']+)["\']\s*,\s*([^,]+)(?:,\s*["\'][^"\']+["\'])?\s*\)',
             lambda match: f'owl_v2_image("{match.group(1)}", {match.group(2)}, "{fine_tune_id}")',
         ),
         (
-            r'florence2_sam2_image\(\s*"([^"]+)"\s*,\s*([^,]+)(?:,\s*"[^"]+")?\s*\)',
+            r'florence2_sam2_image\(\s*["\']([^"\']+)["\']\s*,\s*([^,]+)(?:,\s*["\'][^"\']+["\'])?\s*\)',
             lambda match: f'florence2_sam2_image("{match.group(1)}", {match.group(2)}, "{fine_tune_id}")',
         ),
     ]
 
-    patterns_without_fine_tune_id = [
-        (
-            r"florence2_phrase_grounding\(\s*([^\)]+)\s*\)",
-            lambda match: f'florence2_phrase_grounding({match.group(1)}, "{fine_tune_id}")',
-        ),
-        (
-            r"owl_v2_image\(\s*([^\)]+)\s*\)",
-            lambda match: f'owl_v2_image({match.group(1)}, "{fine_tune_id}")',
-        ),
-        (
-            r"florence2_sam2_image\(\s*([^\)]+)\s*\)",
-            lambda match: f'florence2_sam2_image({match.group(1)}, "{fine_tune_id}")',
-        ),
-    ]
-
     new_code = code
-
-    for index, (pattern_with_fine_tune_id, replacer_with_fine_tune_id) in enumerate(
-        patterns_with_fine_tune_id
-    ):
+    for (
+        pattern_with_fine_tune_id,
+        replacer_with_fine_tune_id,
+    ) in patterns_with_fine_tune_id:
         if re.search(pattern_with_fine_tune_id, new_code):
             new_code = re.sub(
                 pattern_with_fine_tune_id, replacer_with_fine_tune_id, new_code
             )
-        else:
-            (pattern_without_fine_tune_id, replacer_without_fine_tune_id) = (
-                patterns_without_fine_tune_id[index]
-            )
-            new_code = re.sub(
-                pattern_without_fine_tune_id, replacer_without_fine_tune_id, new_code
-            )
 
     if new_code == code:
         output_str = (

{vision_agent-0.2.147.dist-info → vision_agent-0.2.149.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.147
+Version: 0.2.149
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.147.dist-info → vision_agent-0.2.149.dist-info}/RECORD

@@ -2,10 +2,10 @@ vision_agent/__init__.py,sha256=EAb4-f9iyuEYkBrX4ag1syM8Syx8118_t0R6_C34M9w,57
 vision_agent/agent/__init__.py,sha256=NF2LABqHixLvbsOIO-fe-VKZ7awvShLtcT0oQT4eWtI,235
 vision_agent/agent/agent.py,sha256=2cjIOxEuSJrqbfPXYoV0qER5ihXsPFCoEFJa4jpqan0,597
 vision_agent/agent/agent_utils.py,sha256=PEUHqvnHmFL4np_TeFmKMwr5s_dWfdfJz6TF_ogd1dU,2353
-vision_agent/agent/vision_agent.py,sha256=Fp2uSbroRzGrxEwbb9srGdl0h31awkzDFm2tTfn28GI,17587
-vision_agent/agent/vision_agent_coder.py,sha256=4bbebV1sKE10vsxcZR-R8P54X2HjLeU9lDt7ylIZAT4,38429
-vision_agent/agent/vision_agent_coder_prompts.py,sha256=YWK4C--YRS1Kuab11Gn-AXBzar1j_GNnTnxi_nnaPRY,14901
-vision_agent/agent/vision_agent_prompts.py,sha256=e_ASPeRFU1yZsQhCkK_bIBG-eyIWyWXmN64lFk-r7e0,10897
+vision_agent/agent/vision_agent.py,sha256=MDXIM5md1V6y62-chyGiDg_138Rns6KKOO2wMqb6vD8,18431
+vision_agent/agent/vision_agent_coder.py,sha256=9BT4gaXsqH5pvxo8WGwJN9MTvP1V3TgoJHBpjtlKP9I,38417
+vision_agent/agent/vision_agent_coder_prompts.py,sha256=BmbTMhth4v1qLexuoSeyo47QQ0kPQvL1pLbCJHMsWDw,18910
+vision_agent/agent/vision_agent_prompts.py,sha256=3n92aF-jpUyyrAy06izdHIMPEMZPKD1JV0wfQvt-PD8,11251
 vision_agent/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/clients/http.py,sha256=k883i6M_4nl7zwwHSI-yP5sAgQZIDPM1nrKD6YFJ3Xs,2009
 vision_agent/clients/landing_public_api.py,sha256=lU2ev6E8NICmR8DMUljuGcVFy5VNJQ4WQkWC8WnnJEc,1503
@@ -15,7 +15,7 @@ vision_agent/lmm/__init__.py,sha256=jyY1sJb_tYKg5-Wzs3p1lvwFkc-aUNZfMcLy3TOC4Zg,
 vision_agent/lmm/lmm.py,sha256=B5ClgwvbybVCWkf9opDMLjTtJZemUU4KUkQoRxGh43I,16787
 vision_agent/lmm/types.py,sha256=ZEXR_ptBL0ZwDMTDYkgxUCmSZFmBYPQd2jreNzr_8UY,221
 vision_agent/tools/__init__.py,sha256=zUv3aVPN1MXfyQiQi5To4rkQGtG7mxLQ1NjLI3pxM80,2412
-vision_agent/tools/meta_tools.py,sha256=skY4nHSH7PFXNBAEKGYI6XZskv8zJ2UX1KbMY2gd_bo,24687
+vision_agent/tools/meta_tools.py,sha256=rudM9heiuTfNjp741ZNcUGRJdpfDZ38BuellA1IPuIo,24747
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
 vision_agent/tools/tool_utils.py,sha256=5ukuDMxbEH4iKetYR9I7twzsA8ECyP4tVwYXQq54mxI,8020
 vision_agent/tools/tools.py,sha256=c7SjtZD7YfxhEAGYYe-ExVCBA4NDXmRwerBIbd-XEH8,74557
@@ -27,7 +27,7 @@ vision_agent/utils/image_utils.py,sha256=rm9GfXvD4JrjnqKrP_f2gfq4SzmqYC0IdC1kKwd
 vision_agent/utils/sim.py,sha256=ebE9Cs00pVEDI1HMjAzUBk88tQQmc2U-yAzIDinnekU,5572
 vision_agent/utils/type_defs.py,sha256=BE12s3JNQy36QvauXHjwyeffVh5enfcvd4vTzSwvEZI,1384
 vision_agent/utils/video.py,sha256=xbMEoRk13l4fHeQlbvMQhLCn8RNndYmsDhUf01TUeR8,4781
-vision_agent-0.2.147.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.147.dist-info/METADATA,sha256=juNWGPjUPW3pxyGKPEBV_CwDllMnTj8B6c9Z7hevCyM,13758
-vision_agent-0.2.147.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.147.dist-info/RECORD,,
+vision_agent-0.2.149.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.149.dist-info/METADATA,sha256=kdaPDsxWxc0gwBoPFzN0EiQ79NCDBeJzkHzEnh7mRjM,13758
+vision_agent-0.2.149.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.149.dist-info/RECORD,,