vision-agent 0.2.15__py3-none-any.whl → 0.2.22__py3-none-any.whl

vision_agent/agent/vision_agent.py

@@ -308,7 +308,7 @@ def _handle_extract_frames(
     # any following processing
     for video_file_output in tool_result["call_results"]:
         # When the video tool is run with wrong parameters, exit the loop
-        if len(video_file_output) < 2:
+        if not isinstance(video_file_output, tuple) or len(video_file_output) < 2:
             break
         for frame, _ in video_file_output:
             image = frame
@@ -464,15 +464,17 @@ class VisionAgent(Agent):
             report_progress_callback: a callback to report the progress of the agent. This is useful for streaming logs in a web application where multiple VisionAgent instances are running in parallel. This callback ensures that the progress are not mixed up.
         """
         self.task_model = (
-            OpenAILLM(json_mode=True, temperature=0.1)
+            OpenAILLM(model_name="gpt-4-turbo", json_mode=True, temperature=0.0)
             if task_model is None
             else task_model
         )
         self.answer_model = (
-            OpenAILLM(temperature=0.1) if answer_model is None else answer_model
+            OpenAILLM(model_name="gpt-4-turbo", temperature=0.0)
+            if answer_model is None
+            else answer_model
         )
         self.reflect_model = (
-            OpenAILMM(json_mode=True, temperature=0.1)
+            OpenAILMM(model_name="gpt-4-turbo", json_mode=True, temperature=0.0)
             if reflect_model is None
             else reflect_model
         )

vision_agent/agent/vision_agent_v2.py

@@ -1,8 +1,9 @@
 import json
 import logging
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union
 
+import pandas as pd
 from rich.console import Console
 from rich.syntax import Syntax
 from tabulate import tabulate
@@ -20,6 +21,7 @@ from vision_agent.agent.vision_agent_v2_prompt import (
     TEST,
     USER_REQ_CONTEXT,
     USER_REQ_SUBTASK_CONTEXT,
+    USER_REQ_SUBTASK_WM_CONTEXT,
 )
 from vision_agent.llm import LLM, OpenAILLM
 from vision_agent.tools.tools_v2 import TOOL_DESCRIPTIONS, TOOLS_DF
@@ -31,28 +33,68 @@ _MAX_TABULATE_COL_WIDTH = 80
 _CONSOLE = Console()
 
 
+def build_working_memory(working_memory: Mapping[str, List[str]]) -> Sim:
+    data: Mapping[str, List[str]] = {"desc": [], "doc": []}
+    for key, value in working_memory.items():
+        data["desc"].append(key)
+        data["doc"].append("\n".join(value))
+    df = pd.DataFrame(data)  # type: ignore
+    return Sim(df, sim_key="desc")
+
+
 def extract_code(code: str) -> str:
     if "```python" in code:
         code = code[code.find("```python") + len("```python") :]
         code = code[: code.find("```")]
+    if code.startswith("python\n"):
+        code = code[len("python\n") :]
     return code
 
 
+def extract_json(json_str: str) -> Dict[str, Any]:
+    try:
+        json_dict = json.loads(json_str)
+    except json.JSONDecodeError:
+        if "```json" in json_str:
+            json_str = json_str[json_str.find("```json") + len("```json") :]
+            json_str = json_str[: json_str.find("```")]
+        elif "```" in json_str:
+            json_str = json_str[json_str.find("```") + len("```") :]
+            # get the last ``` not one from an intermediate string
+            json_str = json_str[: json_str.find("}```")]
+        json_dict = json.loads(json_str)
+    return json_dict  # type: ignore
+
+
 def write_plan(
-    user_requirements: str, tool_desc: str, model: LLM
-) -> List[Dict[str, Any]]:
+    chat: List[Dict[str, str]],
+    plan: Optional[List[Dict[str, Any]]],
+    tool_desc: str,
+    model: LLM,
+) -> Tuple[str, List[Dict[str, Any]]]:
+    # Get last user request
+    if chat[-1]["role"] != "user":
+        raise ValueError("Last chat message must be from the user.")
+    user_requirements = chat[-1]["content"]
+
     context = USER_REQ_CONTEXT.format(user_requirement=user_requirements)
-    prompt = PLAN.format(context=context, plan="", tool_desc=tool_desc)
-    plan = json.loads(model(prompt).replace("```", "").strip())
-    return plan["plan"]  # type: ignore
+    prompt = PLAN.format(context=context, plan=str(plan), tool_desc=tool_desc)
+    chat[-1]["content"] = prompt
+    new_plan = extract_json(model.chat(chat))
+    return new_plan["user_req"], new_plan["plan"]
 
 
 def write_code(
-    user_req: str, subtask: str, tool_info: str, code: str, model: LLM
+    user_req: str,
+    subtask: str,
+    working_memory: str,
+    tool_info: str,
+    code: str,
+    model: LLM,
 ) -> str:
     prompt = CODE.format(
-        context=USER_REQ_SUBTASK_CONTEXT.format(
-            user_requirement=user_req, subtask=subtask
+        context=USER_REQ_SUBTASK_WM_CONTEXT.format(
+            user_requirement=user_req, working_memory=working_memory, subtask=subtask
         ),
         tool_info=tool_info,
         code=code,
@@ -66,7 +108,7 @@ def write_code(
 
 
 def write_test(
-    user_req: str, subtask: str, tool_info: str, code: str, model: LLM
+    user_req: str, subtask: str, tool_info: str, _: str, code: str, model: LLM
 ) -> str:
     prompt = TEST.format(
         context=USER_REQ_SUBTASK_CONTEXT.format(
@@ -83,20 +125,30 @@ def write_test(
     return extract_code(code)
 
 
-def debug_code(sub_task: str, working_memory: List[str], model: LLM) -> Tuple[str, str]:
+def debug_code(
+    user_req: str,
+    subtask: str,
+    retrieved_ltm: str,
+    working_memory: str,
+    model: LLM,
+) -> Tuple[str, str]:
     # Make debug model output JSON
     if hasattr(model, "kwargs"):
         model.kwargs["response_format"] = {"type": "json_object"}
     prompt = DEBUG.format(
         debug_example=DEBUG_EXAMPLE,
-        context=USER_REQ_CONTEXT.format(user_requirement=sub_task),
-        previous_impl="\n".join(working_memory),
+        context=USER_REQ_SUBTASK_WM_CONTEXT.format(
+            user_requirement=user_req,
+            subtask=subtask,
+            working_memory=retrieved_ltm,
+        ),
+        previous_impl=working_memory,
     )
     messages = [
         {"role": "system", "content": DEBUG_SYS_MSG},
         {"role": "user", "content": prompt},
     ]
-    code_and_ref = json.loads(model.chat(messages).replace("```", "").strip())
+    code_and_ref = extract_json(model.chat(messages))
     if hasattr(model, "kwargs"):
         del model.kwargs["response_format"]
     return extract_code(code_and_ref["improved_impl"]), code_and_ref["reflection"]
@@ -106,20 +158,25 @@ def write_and_exec_code(
     user_req: str,
     subtask: str,
     orig_code: str,
-    code_writer_call: Callable,
+    code_writer_call: Callable[..., str],
     model: LLM,
     tool_info: str,
     exec: Execute,
+    retrieved_ltm: str,
     max_retry: int = 3,
-    verbose: bool = False,
+    verbosity: int = 0,
 ) -> Tuple[bool, str, str, Dict[str, List[str]]]:
     success = False
     counter = 0
     reflection = ""
 
-    # TODO: add working memory to code_writer_call and debug_code
-    code = code_writer_call(user_req, subtask, tool_info, orig_code, model)
+    code = code_writer_call(
+        user_req, subtask, retrieved_ltm, tool_info, orig_code, model
+    )
     success, result = exec.run_isolation(code)
+    if verbosity == 2:
+        _CONSOLE.print(Syntax(code, "python", theme="gruvbox-dark", line_numbers=True))
+        _LOGGER.info(f"\tCode success: {success}, result: {str(result)}")
     working_memory: Dict[str, List[str]] = {}
     while not success and counter < max_retry:
         if subtask not in working_memory:
@@ -136,19 +193,21 @@ def write_and_exec_code(
                 PREV_CODE_CONTEXT.format(code=code, result=result)
             )
 
-        code, reflection = debug_code(subtask, working_memory[subtask], model)
+        code, reflection = debug_code(
+            user_req, subtask, retrieved_ltm, "\n".join(working_memory[subtask]), model
+        )
         success, result = exec.run_isolation(code)
         counter += 1
-        if verbose:
+        if verbosity == 2:
             _CONSOLE.print(
                 Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)
             )
-        _LOGGER.info(f"\tDebugging reflection, result: {reflection}, {result}")
+            _LOGGER.info(f"\tDebugging reflection: {reflection}, result: {result}")
 
         if success:
             working_memory[subtask].append(
                 PREV_CODE_CONTEXT_WITH_REFLECTION.format(
-                    code=code, result=result, reflection=reflection
+                    reflection=reflection, code=code, result=result
                 )
             )
 
@@ -162,12 +221,15 @@ def run_plan(
     exec: Execute,
     code: str,
     tool_recommender: Sim,
-    verbose: bool = False,
+    long_term_memory: Optional[Sim] = None,
+    verbosity: int = 0,
 ) -> Tuple[str, str, List[Dict[str, Any]], Dict[str, List[str]]]:
     active_plan = [e for e in plan if "success" not in e or not e["success"]]
-    working_memory: Dict[str, List[str]] = {}
     current_code = code
     current_test = ""
+    retrieved_ltm = ""
+    working_memory: Dict[str, List[str]] = {}
+
     for task in active_plan:
         _LOGGER.info(
             f"""
@@ -176,7 +238,13 @@ def run_plan(
         tool_info = "\n".join(
             [e["doc"] for e in tool_recommender.top_k(task["instruction"])]
         )
-        success, code, result, task_memory = write_and_exec_code(
+
+        if long_term_memory is not None:
+            retrieved_ltm = "\n".join(
+                [e["doc"] for e in long_term_memory.top_k(task["instruction"], 1)]
+            )
+
+        success, code, result, working_memory_i = write_and_exec_code(
             user_req,
             task["instruction"],
             current_code,
@@ -184,20 +252,21 @@ def run_plan(
             coder,
             tool_info,
             exec,
-            verbose,
+            retrieved_ltm,
+            verbosity=verbosity,
         )
         if task["type"] == "code":
             current_code = code
         else:
             current_test = code
 
-        working_memory.update(task_memory)
+        working_memory.update(working_memory_i)
 
-        if verbose:
+        if verbosity == 1:
             _CONSOLE.print(
                 Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)
             )
-        _LOGGER.info(f"\tCode success, result: {success}, {str(result)}")
+        _LOGGER.info(f"\tCode success: {success} result: {str(result)}")
 
         task["success"] = success
         task["result"] = result
@@ -231,53 +300,71 @@ class VisionAgentV2(Agent):
         self,
         timeout: int = 600,
         tool_recommender: Optional[Sim] = None,
-        verbose: bool = False,
+        long_term_memory: Optional[Sim] = None,
+        verbosity: int = 0,
     ) -> None:
-        self.planner = OpenAILLM(temperature=0.1, json_mode=True)
-        self.coder = OpenAILLM(temperature=0.1)
+        self.planner = OpenAILLM(temperature=0.0, json_mode=True)
+        self.coder = OpenAILLM(temperature=0.0)
         self.exec = Execute(timeout=timeout)
         if tool_recommender is None:
             self.tool_recommender = Sim(TOOLS_DF, sim_key="desc")
         else:
             self.tool_recommender = tool_recommender
-        self.verbose = verbose
-        if self.verbose:
+        self.verbosity = verbosity
+        self._working_memory: Dict[str, List[str]] = {}
+        if long_term_memory is not None:
+            if "doc" not in long_term_memory.df.columns:
+                raise ValueError("Long term memory must have a 'doc' column.")
+        self.long_term_memory = long_term_memory
+        self.max_retries = 3
+        if self.verbosity:
             _LOGGER.setLevel(logging.INFO)
 
     def __call__(
         self,
         input: Union[List[Dict[str, str]], str],
         image: Optional[Union[str, Path]] = None,
+        plan: Optional[List[Dict[str, Any]]] = None,
     ) -> str:
         if isinstance(input, str):
             input = [{"role": "user", "content": input}]
-        code, _ = self.chat_with_tests(input, image)
-        return code
+        results = self.chat_with_workflow(input, image, plan)
+        return results["code"]  # type: ignore
 
-    def chat_with_tests(
+    def chat_with_workflow(
         self,
         chat: List[Dict[str, str]],
         image: Optional[Union[str, Path]] = None,
-    ) -> Tuple[str, str]:
+        plan: Optional[List[Dict[str, Any]]] = None,
+    ) -> Dict[str, Any]:
         if len(chat) == 0:
             raise ValueError("Input cannot be empty.")
 
-        user_req = chat[0]["content"]
         if image is not None:
-            user_req += f" Image name {image}"
+            # append file names to all user messages
+            for chat_i in chat:
+                if chat_i["role"] == "user":
+                    chat_i["content"] += f" Image name {image}"
+
+        working_code = ""
+        if plan is not None:
+            # grab the latest working code from a previous plan
+            for task in plan:
+                if "success" in task and "code" in task and task["success"]:
+                    working_code = task["code"]
 
-        plan = write_plan(user_req, TOOL_DESCRIPTIONS, self.planner)
+        user_req, plan = write_plan(chat, plan, TOOL_DESCRIPTIONS, self.planner)
         _LOGGER.info(
             f"""Plan:
 {tabulate(tabular_data=plan, headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"""
         )
-        working_memory: Dict[str, List[str]] = {}
 
-        working_code = ""
         working_test = ""
+        working_memory: Dict[str, List[str]] = {}
         success = False
+        retries = 0
 
-        while not success:
+        while not success and retries < self.max_retries:
             working_code, working_test, plan, working_memory_i = run_plan(
                 user_req,
                 plan,
@@ -285,16 +372,25 @@ class VisionAgentV2(Agent):
                 self.exec,
                 working_code,
                 self.tool_recommender,
-                self.verbose,
+                self.long_term_memory,
+                self.verbosity,
             )
             success = all(task["success"] for task in plan)
             working_memory.update(working_memory_i)
 
             if not success:
-                # TODO: ask for feedback and replan
+                # return to user and request feedback
                 break
 
-        return working_code, working_test
+            retries += 1
+
+        return {
+            "code": working_code,
+            "test": working_test,
+            "success": success,
+            "working_memory": build_working_memory(working_memory),
+            "plan": plan,
+        }
 
     def log_progress(self, description: str) -> None:
         pass

vision_agent/agent/vision_agent_v2_prompt.py

@@ -1,3 +1,8 @@
+USER_REQ_CONTEXT = """
+## User Requirement
+{user_requirement}
+"""
+
 USER_REQ_SUBTASK_CONTEXT = """
 ## User Requirement
 {user_requirement}
@@ -6,11 +11,16 @@ USER_REQ_SUBTASK_CONTEXT = """
 {subtask}
 """
 
-USER_REQ_CONTEXT = """
+USER_REQ_SUBTASK_WM_CONTEXT = """
 ## User Requirement
 {user_requirement}
-"""
 
+## Current Subtask
+{subtask}
+
+## Previous Task
+{working_memory}
+"""
 
 PLAN = """
 # Context
@@ -27,11 +37,13 @@ Based on the context and the tools you have available, write a plan of subtasks
 - For each subtask, you should provide a short instruction on what to do. Ensure the subtasks are large enough to be meaningful, encompassing multiple lines of code.
 - You do not need to have the agent rewrite any tool functionality you already have, you should instead instruct it to utilize one or more of those tools in each subtask.
 - You can have agents either write coding tasks, to code some functionality or testing tasks to test previous functionality.
+- If a current plan exists, examine each item in the plan to determine if it was successful. If there was an item that failed, i.e. 'success': False, then you should rewrite that item and all subsequent items to ensure that the rewritten plan is successful.
 
 Output a list of jsons in the following format:
 
 ```json
 {{
+    "user_req": str, # "a summarized version of the user requirement"
     "plan":
         [
             {{
@@ -61,8 +73,9 @@ CODE = """
 {code}
 
 # Constraints
-- Write a function that accomplishes the User Requirement. You are supplied code from a previous task, feel free to copy over that code into your own implementation if you need it.
-- Always prioritize using pre-defined tools or code for the same functionality. You have access to all these tools through the `from vision_agent.tools.tools_v2 import *` import.
+- Write a function that accomplishes the 'User Requirement'. You are supplied code from a previous task under 'Previous Code', feel free to copy over that code into your own implementation if you need it.
+- Always prioritize using pre-defined tools or code for the same functionality from 'Tool Info for Current Subtask'. You have access to all these tools through the `from vision_agent.tools.tools_v2 import *` import.
+- You may recieve previous trials and errors under 'Previous Task', this is code, output and reflections from previous tasks. You can use these to avoid running in to the same issues when writing your code.
 - Write clean, readable, and well-documented code.
 
 # Output
@@ -102,6 +115,7 @@ def add(a: int, b: int) -> int:
 
 
 PREV_CODE_CONTEXT = """
+[previous impl]
 ```python
 {code}
 ```
@@ -112,18 +126,20 @@ PREV_CODE_CONTEXT = """
 
 
 PREV_CODE_CONTEXT_WITH_REFLECTION = """
+[reflection on previous impl]
+{reflection}
+
+[new impl]
 ```python
 {code}
 ```
 
-[previous output]
+[new output]
 {result}
 
-[reflection on previous impl]
-{reflection}
 """
 
-
+# don't need [previous impl] because it will come from PREV_CODE_CONTEXT or PREV_CODE_CONTEXT_WITH_REFLECTION
 DEBUG = """
 [example]
 Here is an example of debugging with reflection.
@@ -133,7 +149,6 @@ Here is an example of debugging with reflection.
 [context]
 {context}
 
-[previous impl]
 {previous_impl}
 
 [instruction]
@@ -158,7 +173,7 @@ TEST = """
 {code}
 
 # Constraints
-- Write code to test the functionality of the provided code according to the Current Subtask. If you cannot test the code, then write code to visualize the result by calling the code.
+- Write code to test the functionality of the provided code according to the 'Current Subtask'. If you cannot test the code, then write code to visualize the result by calling the code.
 - Always prioritize using pre-defined tools for the same functionality.
 - Write clean, readable, and well-documented code.
 

vision_agent/llm/llm.py

@@ -34,7 +34,7 @@ class OpenAILLM(LLM):
 
     def __init__(
         self,
-        model_name: str = "gpt-4-turbo",
+        model_name: str = "gpt-4o",
         api_key: Optional[str] = None,
         json_mode: bool = False,
         system_prompt: Optional[str] = None,

vision_agent/tools/__init__.py

@@ -3,7 +3,7 @@ from .tools import (  # Counter,
     CLIP,
     OCR,
     TOOLS,
-    BboxArea,
+    BboxStats,
     BboxIoU,
     BoxDistance,
     Crop,

vision_agent/tools/tool_utils.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from typing import Any, Dict
 
 import requests
@@ -13,6 +14,8 @@ _LND_API_URL = "https://api.dev.landing.ai/v1/agent"
 def _send_inference_request(
     payload: Dict[str, Any], endpoint_name: str
 ) -> Dict[str, Any]:
+    if runtime_tag := os.environ.get("RUNTIME_TAG", ""):
+        payload["runtime_tag"] = runtime_tag
     res = requests.post(
         f"{_LND_API_URL}/model/{endpoint_name}",
         headers={

vision_agent/tools/tools.py

@@ -174,15 +174,15 @@ class GroundingDINO(Tool):
     """
 
     name = "grounding_dino_"
-    description = "'grounding_dino_' is a tool that can detect and count objects given a text prompt such as category names or referring expressions. It returns a list and count of bounding boxes, label names and associated probability scores."
+    description = "'grounding_dino_' is a tool that can detect and count multiple objects given a text prompt such as category names or referring expressions. It returns a list and count of bounding boxes, label names and associated probability scores."
     usage = {
         "required_parameters": [
             {"name": "prompt", "type": "str"},
             {"name": "image", "type": "str"},
         ],
         "optional_parameters": [
-            {"name": "box_threshold", "type": "float"},
-            {"name": "iou_threshold", "type": "float"},
+            {"name": "box_threshold", "type": "float", "min": 0.1, "max": 0.5},
+            {"name": "iou_threshold", "type": "float", "min": 0.01, "max": 0.99},
         ],
         "examples": [
             {
@@ -209,7 +209,7 @@ class GroundingDINO(Tool):
                     "prompt": "red shirt. green shirt",
                     "image": "shirts.jpg",
                     "box_threshold": 0.20,
-                    "iou_threshold": 0.75,
+                    "iou_threshold": 0.20,
                 },
             },
         ],
@@ -221,7 +221,7 @@ class GroundingDINO(Tool):
         prompt: str,
         image: Union[str, Path, ImageType],
         box_threshold: float = 0.20,
-        iou_threshold: float = 0.75,
+        iou_threshold: float = 0.20,
     ) -> Dict:
         """Invoke the Grounding DINO model.
 
@@ -249,7 +249,7 @@ class GroundingDINO(Tool):
             data["scores"] = [round(score, 2) for score in data["scores"]]
         if "labels" in data:
             data["labels"] = list(data["labels"])
-        data["size"] = (image_size[1], image_size[0])
+        data["image_size"] = image_size
         return data
 
 
@@ -277,15 +277,15 @@ class GroundingSAM(Tool):
     """
 
     name = "grounding_sam_"
-    description = "'grounding_sam_' is a tool that can detect and segment objects given a text prompt such as category names or referring expressions. It returns a list of bounding boxes, label names and masks file names and associated probability scores."
+    description = "'grounding_sam_' is a tool that can detect and segment multiple objects given a text prompt such as category names or referring expressions. It returns a list of bounding boxes, label names and masks file names and associated probability scores."
     usage = {
         "required_parameters": [
             {"name": "prompt", "type": "str"},
             {"name": "image", "type": "str"},
         ],
         "optional_parameters": [
-            {"name": "box_threshold", "type": "float"},
-            {"name": "iou_threshold", "type": "float"},
+            {"name": "box_threshold", "type": "float", "min": 0.1, "max": 0.5},
+            {"name": "iou_threshold", "type": "float", "min": 0.01, "max": 0.99},
         ],
         "examples": [
             {
@@ -312,7 +312,7 @@ class GroundingSAM(Tool):
                     "prompt": "red shirt, green shirt",
                     "image": "shirts.jpg",
                     "box_threshold": 0.20,
-                    "iou_threshold": 0.75,
+                    "iou_threshold": 0.20,
                 },
             },
         ],
@@ -324,7 +324,7 @@ class GroundingSAM(Tool):
         prompt: str,
         image: Union[str, ImageType],
         box_threshold: float = 0.2,
-        iou_threshold: float = 0.75,
+        iou_threshold: float = 0.2,
     ) -> Dict:
         """Invoke the Grounding SAM model.
 
@@ -353,6 +353,7 @@ class GroundingSAM(Tool):
                 rle_decode(mask_rle=mask, shape=data["mask_shape"])
                 for mask in data["masks"]
             ]
+        data["image_size"] = image_size
         data.pop("mask_shape", None)
         return data
 
@@ -434,6 +435,8 @@ class DINOv(Tool):
                 for mask in data["masks"]
             ]
         data["labels"] = ["visual prompt" for _ in range(len(data["masks"]))]
+        mask_shape = data.pop("mask_shape", None)
+        data["image_size"] = (mask_shape[0], mask_shape[1]) if mask_shape else None
         return data
 
 
@@ -789,33 +792,49 @@ class Crop(Tool):
         return {"image": tmp.name}
 
 
-class BboxArea(Tool):
-    r"""BboxArea returns the area of the bounding box in pixels normalized to 2 decimal places."""
+class BboxStats(Tool):
+    r"""BboxStats returns the height, width and area of the bounding box in pixels to 2 decimal places."""
 
-    name = "bbox_area_"
-    description = "'bbox_area_' returns the area of the given bounding box in pixels normalized to 2 decimal places."
+    name = "bbox_stats_"
+    description = "'bbox_stats_' returns the height, width and area of the given bounding box in pixels to 2 decimal places."
     usage = {
-        "required_parameters": [{"name": "bboxes", "type": "List[int]"}],
+        "required_parameters": [
+            {"name": "bboxes", "type": "List[int]"},
+            {"name": "image_size", "type": "Tuple[int]"},
+        ],
         "examples": [
             {
-                "scenario": "If you want to calculate the area of the bounding box [0.2, 0.21, 0.34, 0.42]",
-                "parameters": {"bboxes": [0.2, 0.21, 0.34, 0.42]},
-            }
+                "scenario": "Calculate the width and height of the bounding box [0.2, 0.21, 0.34, 0.42]",
+                "parameters": {
+                    "bboxes": [[0.2, 0.21, 0.34, 0.42]],
+                    "image_size": (500, 1200),
+                },
+            },
+            {
+                "scenario": "Calculate the area of the bounding box [0.2, 0.21, 0.34, 0.42]",
+                "parameters": {
+                    "bboxes": [[0.2, 0.21, 0.34, 0.42]],
+                    "image_size": (640, 480),
+                },
+            },
         ],
     }
 
-    def __call__(self, bboxes: List[Dict]) -> List[Dict]:
+    def __call__(
+        self, bboxes: List[List[int]], image_size: Tuple[int, int]
+    ) -> List[Dict]:
         areas = []
-        for elt in bboxes:
-            height, width = elt["size"]
-            for label, bbox in zip(elt["labels"], elt["bboxes"]):
-                x1, y1, x2, y2 = bbox
-                areas.append(
-                    {
-                        "area": round((x2 - x1) * (y2 - y1) * width * height, 2),
-                        "label": label,
-                    }
-                )
+        height, width = image_size
+        for bbox in bboxes:
+            x1, y1, x2, y2 = bbox
+            areas.append(
+                {
+                    "width": round((x2 - x1) * width, 2),
+                    "height": round((y2 - y1) * height, 2),
+                    "area": round((x2 - x1) * (y2 - y1) * width * height, 2),
+                }
+            )
+
         return areas
 
 
@@ -1054,22 +1073,25 @@ class ExtractFrames(Tool):
     r"""Extract frames from a video."""
 
     name = "extract_frames_"
-    description = "'extract_frames_' extracts frames from a video, returns a list of tuples (frame, timestamp), where timestamp is the relative time in seconds where the frame was captured. The frame is a local image file path."
+    description = "'extract_frames_' extracts frames from a video every 2 seconds, returns a list of tuples (frame, timestamp), where timestamp is the relative time in seconds where the frame was captured. The frame is a local image file path."
     usage = {
         "required_parameters": [{"name": "video_uri", "type": "str"}],
+        "optional_parameters": [{"name": "frames_every", "type": "float"}],
         "examples": [
             {
                 "scenario": "Can you extract the frames from this video? Video: www.foobar.com/video?name=test.mp4",
                 "parameters": {"video_uri": "www.foobar.com/video?name=test.mp4"},
             },
             {
-                "scenario": "Can you extract the images from this video file? Video path: tests/data/test.mp4",
-                "parameters": {"video_uri": "tests/data/test.mp4"},
+                "scenario": "Can you extract the images from this video file at every 2 seconds ? Video path: tests/data/test.mp4",
+                "parameters": {"video_uri": "tests/data/test.mp4", "frames_every": 2},
             },
         ],
     }
 
-    def __call__(self, video_uri: str) -> List[Tuple[str, float]]:
+    def __call__(
+        self, video_uri: str, frames_every: float = 2
+    ) -> List[Tuple[str, float]]:
         """Extract frames from a video.
 
 
@@ -1079,7 +1101,7 @@ class ExtractFrames(Tool):
         Returns:
             a list of tuples containing the extracted frame and the timestamp in seconds. E.g. [(path_to_frame1, 0.0), (path_to_frame2, 0.5), ...]. The timestamp is the time in seconds from the start of the video. E.g. 12.125 means 12.125 seconds from the start of the video. The frames are sorted by the timestamp in ascending order.
         """
-        frames = extract_frames_from_video(video_uri)
+        frames = extract_frames_from_video(video_uri, fps=round(1 / frames_every, 2))
         result = []
         _LOGGER.info(
             f"Extracted {len(frames)} frames from video {video_uri}. Temporarily saving them as images to disk for downstream tasks."
@@ -1182,7 +1204,7 @@ TOOLS = {
             AgentDINOv,
             ExtractFrames,
             Crop,
-            BboxArea,
+            BboxStats,
             SegArea,
             ObjectDistance,
             BboxContains,

vision_agent/utils/__init__.py

@@ -1,3 +1,3 @@
 from .execute import Execute
-from .sim import Sim
+from .sim import Sim, load_sim, merge_sim
 from .video import extract_frames_from_video

vision_agent/utils/sim.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 from typing import Dict, List, Optional, Sequence, Union
 
+import numpy as np
 import pandas as pd
 from openai import Client
 from scipy.spatial.distance import cosine  # type: ignore
@@ -46,7 +47,14 @@ class Sim:
             )
 
     def save(self, sim_file: Union[str, Path]) -> None:
-        self.df.to_csv(sim_file, index=False)
+        sim_file = Path(sim_file)
+        sim_file.mkdir(parents=True, exist_ok=True)
+
+        df = self.df.copy()
+        embs = np.array(df.embs.tolist())
+        np.save(sim_file / "embs.npy", embs)
+        df = df.drop("embs", axis=1)
+        df.to_csv(sim_file / "df.csv", index=False)
 
     def top_k(self, query: str, k: int = 5) -> Sequence[Dict]:
         """Returns the top k most similar items to the query.
@@ -65,6 +73,13 @@ class Sim:
         return res[[c for c in res.columns if c != "embs"]].to_dict(orient="records")
 
 
+def merge_sim(sim1: Sim, sim2: Sim) -> Sim:
+    return Sim(pd.concat([sim1.df, sim2.df], ignore_index=True))
+
+
 def load_sim(sim_file: Union[str, Path]) -> Sim:
-    df = pd.read_csv(sim_file)
+    sim_file = Path(sim_file)
+    df = pd.read_csv(sim_file / "df.csv")
+    embs = np.load(sim_file / "embs.npy")
+    df["embs"] = list(embs)
     return Sim(df)

{vision_agent-0.2.15.dist-info → vision_agent-0.2.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.15
+Version: 0.2.22
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.15.dist-info → vision_agent-0.2.22.dist-info}/RECORD

@@ -7,28 +7,28 @@ vision_agent/agent/easytool.py,sha256=oMHnBg7YBtIPgqQUNcZgq7uMgpPThs99_UnO7ERkMV
 vision_agent/agent/easytool_prompts.py,sha256=Bikw-PPLkm78dwywTlnv32Y1Tw6JMeC-R7oCnXWLcTk,4656
 vision_agent/agent/reflexion.py,sha256=4gz30BuFMeGxSsTzoDV4p91yE0R8LISXp28IaOI6wdM,10506
 vision_agent/agent/reflexion_prompts.py,sha256=G7UAeNz_g2qCb2yN6OaIC7bQVUkda4m3z42EG8wAyfE,9342
-vision_agent/agent/vision_agent.py,sha256=4-GjEX8ZmLhvLebqNRRTSSu1kSaFYVR_wFsrjXgKdYI,26984
+vision_agent/agent/vision_agent.py,sha256=pnx7gtTPazR7Dck5_kfZC3S3QWKu4e28YVigzOicOX0,27130
 vision_agent/agent/vision_agent_prompts.py,sha256=MZSIwovYgB-f-kdJ6btaNDVXptJn47bfOL3-Zn6NiC0,8573
-vision_agent/agent/vision_agent_v2.py,sha256=CDgGBSoa2LoMS0b4JhyDkoS3PJJNmCCPfxIGUc4RfQg,9658
-vision_agent/agent/vision_agent_v2_prompt.py,sha256=-90Hlbtqb5Fp7OVjGabpTdgr-yCr8AYKIfiMRfoL4SY,5141
+vision_agent/agent/vision_agent_v2.py,sha256=pAOYfNxBVZwnNxyYfv_Bk5dklFr4ougA52ib4q8O4Uo,12942
+vision_agent/agent/vision_agent_v2_prompt.py,sha256=dd9m9Vqp91r4dpsKMDwXr54jG_GTBdJNDzpgR115S8Q,5997
 vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
 vision_agent/llm/__init__.py,sha256=BoUm_zSAKnLlE8s-gKTSQugXDqVZKPqYlWwlTLdhcz4,48
-vision_agent/llm/llm.py,sha256=qWDBpJolGLWNwDjpEXu1NrjlJbo7Fj9efJYkSfVn6oE,5784
+vision_agent/llm/llm.py,sha256=A-gN0vMb79fSxhSK1qBs6PTu1fba9Gvy6pitOyjW2gM,5779
 vision_agent/lmm/__init__.py,sha256=nnNeKD1k7q_4vLb1x51O_EUTYaBgGfeiCx5F433gr3M,67
 vision_agent/lmm/lmm.py,sha256=gK90vMxh0OcGSuIZQikBkDXm4pfkdFk1R2y7rtWDl84,10539
-vision_agent/tools/__init__.py,sha256=WiEjXzXyaBq7IQMKOMbFAK3FKvLNfzZ3dd7CPN-d7B8,451
+vision_agent/tools/__init__.py,sha256=p5SM0YhThSVO_jRF9O-OjH2fYDPv-iMjexDX9xPPb7M,452
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
-vision_agent/tools/tool_utils.py,sha256=moR7X4hkLKQzC56axdojo_OcIuVOv45bKcHPUVZrPvk,753
-vision_agent/tools/tools.py,sha256=WrNu_L5n2cEpe7e1oy8S1o3dy4JJ4AUxTHcjAdX64_g,46398
+vision_agent/tools/tool_utils.py,sha256=mK6QfbYr6oo9ci979-_6R1DrxU2i8HGhwosADyvciI0,865
+vision_agent/tools/tools.py,sha256=sVxN7SpDkz_XTc_SKwkoRF4EwaMTuHvTsCHwtR942Fc,47373
 vision_agent/tools/tools_v2.py,sha256=1Y_ZbYJyuo2eZZkq7jY3YfuKWC82C-GFCZMLYH-I5ew,13800
-vision_agent/utils/__init__.py,sha256=AKXf1QVOpO6MnqU8RSaFLQ_4us4DcKf8ibgEbhuHjvI,95
+vision_agent/utils/__init__.py,sha256=xsHFyJSDbLdonB9Dh74cwZnVTiT__2OQF3Brd3Nmglc,116
 vision_agent/utils/execute.py,sha256=RC_jKrm2kOWwzNe9xKuA2xJcbsNcD0Hb95_o3_Le0_E,3820
 vision_agent/utils/image_utils.py,sha256=1dggPBhW8_hUXDItCRLa23h-hdBwS50cjL4v1hsoUbg,7586
-vision_agent/utils/sim.py,sha256=FaD16kKL1-JR2aSCmznF9KkJux9u3_Nr9tF4smBeoK0,2327
+vision_agent/utils/sim.py,sha256=SO4-pj2Fjs3yr-KT8S0nuUd66lf7m7XvMAp7_ecvKuQ,2813
 vision_agent/utils/type_defs.py,sha256=4LTnTL4HNsfYqCrDn9Ppjg9bSG2ZGcoKSSd9YeQf4Bw,1792
 vision_agent/utils/video.py,sha256=xTElFSFp1Jw4ulOMnk81Vxsh-9dTxcWUO6P9fzEi3AM,7653
-vision_agent-0.2.15.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.15.dist-info/METADATA,sha256=qK9rIVOI_IL0dcUcIqtgoRCxuk5GZuQ5HHSrdsuVLKs,9121
-vision_agent-0.2.15.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.15.dist-info/RECORD,,
+vision_agent-0.2.22.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.22.dist-info/METADATA,sha256=hOVbcYSPue2CEdagEkIiX3dGtjip9p1GgKilmPYj-gU,9121
+vision_agent-0.2.22.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.22.dist-info/RECORD,,