vision-agent 0.2.21__py3-none-any.whl → 0.2.23__py3-none-any.whl

vision_agent/agent/vision_agent_v2.py

@@ -51,6 +51,21 @@ def extract_code(code: str) -> str:
     return code
 
 
+def extract_json(json_str: str) -> Dict[str, Any]:
+    try:
+        json_dict = json.loads(json_str)
+    except json.JSONDecodeError:
+        if "```json" in json_str:
+            json_str = json_str[json_str.find("```json") + len("```json") :]
+            json_str = json_str[: json_str.find("```")]
+        elif "```" in json_str:
+            json_str = json_str[json_str.find("```") + len("```") :]
+            # get the last ``` not one from an intermediate string
+            json_str = json_str[: json_str.find("}```")]
+        json_dict = json.loads(json_str)
+    return json_dict  # type: ignore
+
+
 def write_plan(
     chat: List[Dict[str, str]],
     plan: Optional[List[Dict[str, Any]]],
@@ -65,8 +80,8 @@ def write_plan(
     context = USER_REQ_CONTEXT.format(user_requirement=user_requirements)
     prompt = PLAN.format(context=context, plan=str(plan), tool_desc=tool_desc)
     chat[-1]["content"] = prompt
-    plan = json.loads(model.chat(chat).replace("```", "").strip())
-    return plan["user_req"], plan["plan"]  # type: ignore
+    new_plan = extract_json(model.chat(chat))
+    return new_plan["user_req"], new_plan["plan"]
 
 
 def write_code(
@@ -133,7 +148,7 @@ def debug_code(
         {"role": "system", "content": DEBUG_SYS_MSG},
         {"role": "user", "content": prompt},
     ]
-    code_and_ref = json.loads(model.chat(messages).replace("```", "").strip())
+    code_and_ref = extract_json(model.chat(messages))
     if hasattr(model, "kwargs"):
         del model.kwargs["response_format"]
     return extract_code(code_and_ref["improved_impl"]), code_and_ref["reflection"]
@@ -149,7 +164,7 @@ def write_and_exec_code(
     exec: Execute,
     retrieved_ltm: str,
     max_retry: int = 3,
-    verbose: bool = False,
+    verbosity: int = 0,
 ) -> Tuple[bool, str, str, Dict[str, List[str]]]:
     success = False
     counter = 0
@@ -159,6 +174,9 @@ def write_and_exec_code(
         user_req, subtask, retrieved_ltm, tool_info, orig_code, model
     )
     success, result = exec.run_isolation(code)
+    if verbosity == 2:
+        _CONSOLE.print(Syntax(code, "python", theme="gruvbox-dark", line_numbers=True))
+        _LOGGER.info(f"\tCode success: {success}, result: {str(result)}")
     working_memory: Dict[str, List[str]] = {}
     while not success and counter < max_retry:
         if subtask not in working_memory:
@@ -180,11 +198,11 @@ def write_and_exec_code(
         )
         success, result = exec.run_isolation(code)
         counter += 1
-        if verbose:
+        if verbosity == 2:
             _CONSOLE.print(
                 Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)
             )
-        _LOGGER.info(f"\tDebugging reflection, result: {reflection}, {result}")
+            _LOGGER.info(f"\tDebugging reflection: {reflection}, result: {result}")
 
         if success:
             working_memory[subtask].append(
@@ -204,7 +222,7 @@ def run_plan(
     code: str,
     tool_recommender: Sim,
     long_term_memory: Optional[Sim] = None,
-    verbose: bool = False,
+    verbosity: int = 0,
 ) -> Tuple[str, str, List[Dict[str, Any]], Dict[str, List[str]]]:
     active_plan = [e for e in plan if "success" not in e or not e["success"]]
     current_code = code
@@ -217,9 +235,11 @@ def run_plan(
             f"""
 {tabulate(tabular_data=[task], headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"""
         )
-        tool_info = "\n".join(
-            [e["doc"] for e in tool_recommender.top_k(task["instruction"])]
-        )
+        tools = tool_recommender.top_k(task["instruction"])
+        tool_info = "\n".join([e["doc"] for e in tools])
+
+        if verbosity == 2:
+            _LOGGER.info(f"Tools retrieved: {[e['desc'] for e in tools]}")
 
         if long_term_memory is not None:
             retrieved_ltm = "\n".join(
@@ -235,7 +255,7 @@ def run_plan(
             tool_info,
             exec,
             retrieved_ltm,
-            verbose=verbose,
+            verbosity=verbosity,
         )
         if task["type"] == "code":
             current_code = code
@@ -244,11 +264,11 @@ def run_plan(
 
         working_memory.update(working_memory_i)
 
-        if verbose:
+        if verbosity == 1:
             _CONSOLE.print(
                 Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)
             )
-        _LOGGER.info(f"\tCode success, result: {success}, {str(result)}")
+        _LOGGER.info(f"\tCode success: {success} result: {str(result)}")
 
         task["success"] = success
         task["result"] = result
@@ -283,23 +303,23 @@ class VisionAgentV2(Agent):
         timeout: int = 600,
         tool_recommender: Optional[Sim] = None,
         long_term_memory: Optional[Sim] = None,
-        verbose: bool = False,
+        verbosity: int = 0,
     ) -> None:
-        self.planner = OpenAILLM(temperature=0.1, json_mode=True)
-        self.coder = OpenAILLM(temperature=0.1)
+        self.planner = OpenAILLM(temperature=0.0, json_mode=True)
+        self.coder = OpenAILLM(temperature=0.0)
         self.exec = Execute(timeout=timeout)
         if tool_recommender is None:
             self.tool_recommender = Sim(TOOLS_DF, sim_key="desc")
         else:
             self.tool_recommender = tool_recommender
-        self.verbose = verbose
+        self.verbosity = verbosity
         self._working_memory: Dict[str, List[str]] = {}
         if long_term_memory is not None:
             if "doc" not in long_term_memory.df.columns:
                 raise ValueError("Long term memory must have a 'doc' column.")
         self.long_term_memory = long_term_memory
         self.max_retries = 3
-        if self.verbose:
+        if self.verbosity:
             _LOGGER.setLevel(logging.INFO)
 
     def __call__(
@@ -355,7 +375,7 @@ class VisionAgentV2(Agent):
                 working_code,
                 self.tool_recommender,
                 self.long_term_memory,
-                self.verbose,
+                self.verbosity,
             )
             success = all(task["success"] for task in plan)
             working_memory.update(working_memory_i)

vision_agent/tools/__init__.py

@@ -3,8 +3,8 @@ from .tools import (  # Counter,
     CLIP,
     OCR,
     TOOLS,
-    BboxStats,
     BboxIoU,
+    BboxStats,
     BoxDistance,
     Crop,
     DINOv,

vision_agent/tools/tools_v2.py

@@ -4,12 +4,13 @@ import logging
 import tempfile
 from importlib import resources
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Tuple, Union
+from typing import Any, Callable, Dict, List, Tuple, Union, cast
 
 import numpy as np
 import pandas as pd
 import requests
 from PIL import Image, ImageDraw, ImageFont
+from scipy.spatial import distance  # type: ignore
 
 from vision_agent.tools.tool_utils import _send_inference_request
 from vision_agent.utils import extract_frames_from_video
@@ -233,6 +234,54 @@ def ocr(image: np.ndarray) -> List[Dict[str, Any]]:
     return output
 
 
+def closest_mask_distance(mask1: np.ndarray, mask2: np.ndarray) -> float:
+    """'closest_mask_distance' calculates the closest distance between two masks.
+
+    Parameters:
+        mask1 (np.ndarray): The first mask.
+        mask2 (np.ndarray): The second mask.
+
+    Returns:
+        float: The closest distance between the two masks.
+
+    Example
+    -------
+    >>> closest_mask_distance(mask1, mask2)
+    0.5
+    """
+
+    mask1 = np.clip(mask1, 0, 1)
+    mask2 = np.clip(mask2, 0, 1)
+    mask1_points = np.transpose(np.nonzero(mask1))
+    mask2_points = np.transpose(np.nonzero(mask2))
+    dist_matrix = distance.cdist(mask1_points, mask2_points, "euclidean")
+    return cast(float, np.min(dist_matrix))
+
+
+def closest_box_distance(box1: List[float], box2: List[float]) -> float:
+    """'closest_box_distance' calculates the closest distance between two bounding boxes.
+
+    Parameters:
+        box1 (List[float]): The first bounding box.
+        box2 (List[float]): The second bounding box.
+
+    Returns:
+        float: The closest distance between the two bounding boxes.
+
+    Example
+    -------
+    >>> closest_box_distance([100, 100, 200, 200], [300, 300, 400, 400])
+    141.42
+    """
+
+    x11, y11, x12, y12 = box1
+    x21, y21, x22, y22 = box2
+
+    horizontal_distance = np.max([0, x21 - x12, x11 - x22])
+    vertical_distance = np.max([0, y21 - y12, y11 - y22])
+    return cast(float, np.sqrt(horizontal_distance**2 + vertical_distance**2))
+
+
 # Utility and visualization functions
 
 
@@ -429,6 +478,8 @@ TOOLS = [
     grounding_sam,
     extract_frames,
     ocr,
+    closest_mask_distance,
+    closest_box_distance,
     load_image,
     save_image,
     overlay_bounding_boxes,

{vision_agent-0.2.21.dist-info → vision_agent-0.2.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.21
+Version: 0.2.23
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.21.dist-info → vision_agent-0.2.23.dist-info}/RECORD

@@ -9,7 +9,7 @@ vision_agent/agent/reflexion.py,sha256=4gz30BuFMeGxSsTzoDV4p91yE0R8LISXp28IaOI6w
 vision_agent/agent/reflexion_prompts.py,sha256=G7UAeNz_g2qCb2yN6OaIC7bQVUkda4m3z42EG8wAyfE,9342
 vision_agent/agent/vision_agent.py,sha256=pnx7gtTPazR7Dck5_kfZC3S3QWKu4e28YVigzOicOX0,27130
 vision_agent/agent/vision_agent_prompts.py,sha256=MZSIwovYgB-f-kdJ6btaNDVXptJn47bfOL3-Zn6NiC0,8573
-vision_agent/agent/vision_agent_v2.py,sha256=K-zJ0utlvgWpR0TlP9M3yO0O9t9L37FcBs31OfOC4C0,12185
+vision_agent/agent/vision_agent_v2.py,sha256=0-bJH_KiYB9fdfN5rbutnyJgQr1XYeszNYqmR69IxZc,13045
 vision_agent/agent/vision_agent_v2_prompt.py,sha256=dd9m9Vqp91r4dpsKMDwXr54jG_GTBdJNDzpgR115S8Q,5997
 vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
@@ -17,18 +17,18 @@ vision_agent/llm/__init__.py,sha256=BoUm_zSAKnLlE8s-gKTSQugXDqVZKPqYlWwlTLdhcz4,
 vision_agent/llm/llm.py,sha256=A-gN0vMb79fSxhSK1qBs6PTu1fba9Gvy6pitOyjW2gM,5779
 vision_agent/lmm/__init__.py,sha256=nnNeKD1k7q_4vLb1x51O_EUTYaBgGfeiCx5F433gr3M,67
 vision_agent/lmm/lmm.py,sha256=gK90vMxh0OcGSuIZQikBkDXm4pfkdFk1R2y7rtWDl84,10539
-vision_agent/tools/__init__.py,sha256=p5SM0YhThSVO_jRF9O-OjH2fYDPv-iMjexDX9xPPb7M,452
+vision_agent/tools/__init__.py,sha256=dRHXGpjhItXZRQs0r_l3Z3bQIreaZaYP0CJrl8mOJxM,452
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
 vision_agent/tools/tool_utils.py,sha256=mK6QfbYr6oo9ci979-_6R1DrxU2i8HGhwosADyvciI0,865
 vision_agent/tools/tools.py,sha256=sVxN7SpDkz_XTc_SKwkoRF4EwaMTuHvTsCHwtR942Fc,47373
-vision_agent/tools/tools_v2.py,sha256=1Y_ZbYJyuo2eZZkq7jY3YfuKWC82C-GFCZMLYH-I5ew,13800
+vision_agent/tools/tools_v2.py,sha256=Dh5Rs1iaEs5ijRDwVI3Na9ylC7eOjtrIqtYOZSredH8,15364
 vision_agent/utils/__init__.py,sha256=xsHFyJSDbLdonB9Dh74cwZnVTiT__2OQF3Brd3Nmglc,116
 vision_agent/utils/execute.py,sha256=RC_jKrm2kOWwzNe9xKuA2xJcbsNcD0Hb95_o3_Le0_E,3820
 vision_agent/utils/image_utils.py,sha256=1dggPBhW8_hUXDItCRLa23h-hdBwS50cjL4v1hsoUbg,7586
 vision_agent/utils/sim.py,sha256=SO4-pj2Fjs3yr-KT8S0nuUd66lf7m7XvMAp7_ecvKuQ,2813
 vision_agent/utils/type_defs.py,sha256=4LTnTL4HNsfYqCrDn9Ppjg9bSG2ZGcoKSSd9YeQf4Bw,1792
 vision_agent/utils/video.py,sha256=xTElFSFp1Jw4ulOMnk81Vxsh-9dTxcWUO6P9fzEi3AM,7653
-vision_agent-0.2.21.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.21.dist-info/METADATA,sha256=zth6S82TWHpBoFToYIVQYRyOWBxovRci731LcIHb8Bw,9121
-vision_agent-0.2.21.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.21.dist-info/RECORD,,
+vision_agent-0.2.23.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.23.dist-info/METADATA,sha256=r3JWwYu2mKPjViXrm50ZS_9juGciOrYfEyz2YhPeczQ,9121
+vision_agent-0.2.23.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.23.dist-info/RECORD,,