vision-agent 0.2.22__tar.gz → 0.2.24__tar.gz

{vision_agent-0.2.22 → vision_agent-0.2.24}/PKG-INFO

@@ -1,14 +1,16 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.22
+Version: 0.2.24
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
-Requires-Python: >=3.9
+Requires-Python: >=3.9,<4.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: ipykernel (>=6.29.4,<7.0.0)
+Requires-Dist: langsmith (>=0.1.58,<0.2.0)
 Requires-Dist: moviepy (>=1.0.0,<2.0.0)
 Requires-Dist: nbclient (>=0.10.0,<0.11.0)
 Requires-Dist: nbformat (>=5.10.4,<6.0.0)

{vision_agent-0.2.22 → vision_agent-0.2.24}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "vision-agent"
-version = "0.2.22"
+version = "0.2.24"
 description = "Toolset for Vision Agent"
 authors = ["Landing AI <dev@landing.ai>"]
 readme = "README.md"
@@ -16,7 +16,7 @@ packages = [{include = "vision_agent"}]
 "documentation" = "https://github.com/landing-ai/vision-agent"
 
 [tool.poetry.dependencies]  # main dependency group
-python = ">=3.9"
+python = ">=3.9,<4.0"
 numpy = ">=1.21.0,<2.0.0"
 pillow = "10.*"
 requests = "2.*"
@@ -32,6 +32,8 @@ scipy = "1.13.*"
 nbclient = "^0.10.0"
 nbformat = "^5.10.4"
 rich = "^13.7.1"
+langsmith = "^0.1.58"
+ipykernel = "^6.29.4"
 
 [tool.poetry.group.dev.dependencies]
 autoflake = "1.*"

{vision_agent-0.2.22 → vision_agent-0.2.24}/vision_agent/agent/agent_coder.py

@@ -5,6 +5,9 @@ import sys
 from pathlib import Path
 from typing import Dict, List, Optional, Union
 
+from rich.console import Console
+from rich.syntax import Syntax
+
 from vision_agent.agent import Agent
 from vision_agent.agent.agent_coder_prompts import (
     DEBUG,
@@ -40,6 +43,7 @@ from vision_agent.tools.tools_v2 import *
 logging.basicConfig(stream=sys.stdout)
 _LOGGER = logging.getLogger(__name__)
 _EXECUTE = Execute()
+_CONSOLE = Console()
 
 
 def write_tests(question: str, code: str, model: LLM) -> str:
@@ -103,7 +107,7 @@ def run_visual_tests(
 
 
 def fix_bugs(code: str, tests: str, result: str, feedback: str, model: LLM) -> str:
-    prompt = FIX_BUG.format(completion=code, test_case=tests, result=result)
+    prompt = FIX_BUG.format(code=code, tests=tests, result=result, feedback=feedback)
     completion = model(prompt)
     return preprocess_data(completion)
 
@@ -139,7 +143,8 @@ class AgentCoder(Agent):
             else visual_tester_agent
         )
         self.max_turns = 3
-        if verbose:
+        self.verbose = verbose
+        if self.verbose:
             _LOGGER.setLevel(logging.INFO)
 
     def __call__(
@@ -164,9 +169,15 @@ class AgentCoder(Agent):
         feedback = ""
         for _ in range(self.max_turns):
             code = write_program(question, feedback, self.coder_agent)
-            _LOGGER.info(f"code:\n{code}")
+            if self.verbose:
+                _CONSOLE.print(
+                    Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)
+                )
             debug = write_debug(question, code, feedback, self.tester_agent)
-            _LOGGER.info(f"debug:\n{debug}")
+            if self.verbose:
+                _CONSOLE.print(
+                    Syntax(debug, "python", theme="gruvbox-dark", line_numbers=True)
+                )
             results = execute_tests(code, debug)
             _LOGGER.info(
                 f"execution results: passed: {results['passed']}\n{results['result']}"
@@ -176,7 +187,10 @@ class AgentCoder(Agent):
                 code = fix_bugs(
                     code, debug, results["result"].strip(), feedback, self.coder_agent  # type: ignore
                 )
-                _LOGGER.info(f"fixed code:\n{code}")
+                if self.verbose:
+                    _CONSOLE.print(
+                        Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)
+                    )
             else:
                 # TODO: Sometimes it prints nothing, so we need to handle that case
                 # TODO: The visual agent reflection does not work very well, needs more testing

{vision_agent-0.2.22 → vision_agent-0.2.24}/vision_agent/agent/vision_agent_v2.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union
 
 import pandas as pd
+from langsmith import traceable
 from rich.console import Console
 from rich.syntax import Syntax
 from tabulate import tabulate
@@ -66,6 +67,7 @@ def extract_json(json_str: str) -> Dict[str, Any]:
     return json_dict  # type: ignore
 
 
+@traceable(name="planning")
 def write_plan(
     chat: List[Dict[str, str]],
     plan: Optional[List[Dict[str, Any]]],
@@ -214,6 +216,7 @@ def write_and_exec_code(
     return success, code, result, working_memory
 
 
+@traceable(name="plan execution")
 def run_plan(
     user_req: str,
     plan: List[Dict[str, Any]],
@@ -235,9 +238,11 @@ def run_plan(
             f"""
 {tabulate(tabular_data=[task], headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"""
         )
-        tool_info = "\n".join(
-            [e["doc"] for e in tool_recommender.top_k(task["instruction"])]
-        )
+        tools = tool_recommender.top_k(task["instruction"], thresh=0.3)
+        tool_info = "\n".join([e["doc"] for e in tools])
+
+        if verbosity == 2:
+            _LOGGER.info(f"Tools retrieved: {[e['desc'] for e in tools]}")
 
         if long_term_memory is not None:
             retrieved_ltm = "\n".join(
@@ -283,6 +288,7 @@ class VisionAgentV2(Agent):
     solve vision tasks. It is inspired by MetaGPT's Data Interpreter
     https://arxiv.org/abs/2402.18679. Vision Agent has several key features to help it
     generate code:
+
     - A planner to generate a plan of tasks to solve a user requirement. The planner
     can output code tasks or test tasks, where test tasks are used to verify the code.
     - Automatic debugging, if a task fails, the agent will attempt to debug the code
@@ -331,6 +337,7 @@ class VisionAgentV2(Agent):
         results = self.chat_with_workflow(input, image, plan)
         return results["code"]  # type: ignore
 
+    @traceable
     def chat_with_workflow(
         self,
         chat: List[Dict[str, str]],
@@ -375,7 +382,9 @@ class VisionAgentV2(Agent):
                 self.long_term_memory,
                 self.verbosity,
             )
-            success = all(task["success"] for task in plan)
+            success = all(
+                task["success"] if "success" in task else False for task in plan
+            )
             working_memory.update(working_memory_i)
 
             if not success:

{vision_agent-0.2.22 → vision_agent-0.2.24}/vision_agent/agent/vision_agent_v2_prompt.py

@@ -34,7 +34,7 @@ PLAN = """
 
 # Task:
 Based on the context and the tools you have available, write a plan of subtasks to achieve the user request that adhere to the following requirements:
-- For each subtask, you should provide a short instruction on what to do. Ensure the subtasks are large enough to be meaningful, encompassing multiple lines of code.
+- For each subtask, you should provide instructions on what to do. Write detailed subtasks, ensure they are large enough to be meaningful, encompassing multiple lines of code.
 - You do not need to have the agent rewrite any tool functionality you already have, you should instead instruct it to utilize one or more of those tools in each subtask.
 - You can have agents either write coding tasks, to code some functionality or testing tasks to test previous functionality.
 - If a current plan exists, examine each item in the plan to determine if it was successful. If there was an item that failed, i.e. 'success': False, then you should rewrite that item and all subsequent items to ensure that the rewritten plan is successful.
@@ -73,9 +73,10 @@ CODE = """
 {code}
 
 # Constraints
-- Write a function that accomplishes the 'User Requirement'. You are supplied code from a previous task under 'Previous Code', feel free to copy over that code into your own implementation if you need it.
-- Always prioritize using pre-defined tools or code for the same functionality from 'Tool Info for Current Subtask'. You have access to all these tools through the `from vision_agent.tools.tools_v2 import *` import.
+- Write a function that accomplishes the 'Current Subtask'. You are supplied code from a previous task under 'Previous Code', do not delete or change previous code unless it contains a bug or it is necessary to complete the 'Current Subtask'.
+- Always prioritize using pre-defined tools or code for the same functionality from 'Tool Info' when working on 'Current Subtask'. You have access to all these tools through the `from vision_agent.tools.tools_v2 import *` import.
 - You may recieve previous trials and errors under 'Previous Task', this is code, output and reflections from previous tasks. You can use these to avoid running in to the same issues when writing your code.
+- Use the `save_json` function from `vision_agent.tools.tools_v2` to save your output as a json file.
 - Write clean, readable, and well-documented code.
 
 # Output

{vision_agent-0.2.22 → vision_agent-0.2.24}/vision_agent/llm/llm.py

@@ -3,6 +3,7 @@ import os
 from abc import ABC, abstractmethod
 from typing import Any, Callable, Dict, List, Mapping, Optional, Union, cast
 
+from langsmith.wrappers import wrap_openai
 from openai import AzureOpenAI, OpenAI
 
 from vision_agent.tools import (
@@ -41,9 +42,9 @@ class OpenAILLM(LLM):
         **kwargs: Any
     ):
         if not api_key:
-            self.client = OpenAI()
+            self.client = wrap_openai(OpenAI())
         else:
-            self.client = OpenAI(api_key=api_key)
+            self.client = wrap_openai(OpenAI(api_key=api_key))
 
         self.model_name = model_name
         self.system_prompt = system_prompt
@@ -165,8 +166,10 @@ class AzureOpenAILLM(OpenAILLM):
         if not azure_endpoint:
             raise ValueError("Azure OpenAI endpoint is required.")
 
-        self.client = AzureOpenAI(
-            api_key=api_key, api_version=api_version, azure_endpoint=azure_endpoint
+        self.client = wrap_openai(
+            AzureOpenAI(
+                api_key=api_key, api_version=api_version, azure_endpoint=azure_endpoint
+            )
         )
         self.model_name = model_name
         self.kwargs = kwargs

{vision_agent-0.2.22 → vision_agent-0.2.24}/vision_agent/tools/tools_v2.py

@@ -1,15 +1,17 @@
 import inspect
 import io
+import json
 import logging
 import tempfile
 from importlib import resources
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Tuple, Union
+from typing import Any, Callable, Dict, List, Tuple, Union, cast
 
 import numpy as np
 import pandas as pd
 import requests
 from PIL import Image, ImageDraw, ImageFont
+from scipy.spatial import distance  # type: ignore
 
 from vision_agent.tools.tool_utils import _send_inference_request
 from vision_agent.utils import extract_frames_from_video
@@ -233,9 +235,82 @@ def ocr(image: np.ndarray) -> List[Dict[str, Any]]:
     return output
 
 
+def closest_mask_distance(mask1: np.ndarray, mask2: np.ndarray) -> float:
+    """'closest_mask_distance' calculates the closest distance between two masks.
+
+    Parameters:
+        mask1 (np.ndarray): The first mask.
+        mask2 (np.ndarray): The second mask.
+
+    Returns:
+        float: The closest distance between the two masks.
+
+    Example
+    -------
+    >>> closest_mask_distance(mask1, mask2)
+    0.5
+    """
+
+    mask1 = np.clip(mask1, 0, 1)
+    mask2 = np.clip(mask2, 0, 1)
+    mask1_points = np.transpose(np.nonzero(mask1))
+    mask2_points = np.transpose(np.nonzero(mask2))
+    dist_matrix = distance.cdist(mask1_points, mask2_points, "euclidean")
+    return cast(float, np.min(dist_matrix))
+
+
+def closest_box_distance(box1: List[float], box2: List[float]) -> float:
+    """'closest_box_distance' calculates the closest distance between two bounding boxes.
+
+    Parameters:
+        box1 (List[float]): The first bounding box.
+        box2 (List[float]): The second bounding box.
+
+    Returns:
+        float: The closest distance between the two bounding boxes.
+
+    Example
+    -------
+    >>> closest_box_distance([100, 100, 200, 200], [300, 300, 400, 400])
+    141.42
+    """
+
+    x11, y11, x12, y12 = box1
+    x21, y21, x22, y22 = box2
+
+    horizontal_distance = np.max([0, x21 - x12, x11 - x22])
+    vertical_distance = np.max([0, y21 - y12, y11 - y22])
+    return cast(float, np.sqrt(horizontal_distance**2 + vertical_distance**2))
+
+
 # Utility and visualization functions
 
 
+def save_json(data: Any, file_path: str) -> None:
+    """'save_json' is a utility function that saves data as a JSON file. It is helpful
+    for saving data that contains NumPy arrays which are not JSON serializable.
+
+    Parameters:
+        data (Any): The data to save.
+        file_path (str): The path to save the JSON file.
+
+    Example
+    -------
+    >>> save_json(data, "path/to/file.json")
+    """
+
+    class NumpyEncoder(json.JSONEncoder):
+        def default(self, obj: Any):  # type: ignore
+            if isinstance(obj, np.ndarray):
+                return obj.tolist()
+            elif isinstance(obj, np.bool_):
+                return bool(obj)
+            return json.JSONEncoder.default(self, obj)
+
+    with open(file_path, "w") as f:
+        json.dump(data, f, cls=NumpyEncoder)
+
+
 def load_image(image_path: str) -> np.ndarray:
     """'load_image' is a utility function that loads an image from the given path.
 
@@ -429,6 +504,9 @@ TOOLS = [
     grounding_sam,
     extract_frames,
     ocr,
+    closest_mask_distance,
+    closest_box_distance,
+    save_json,
     load_image,
     save_image,
     overlay_bounding_boxes,
@@ -438,5 +516,5 @@ TOOLS_DF = get_tools_df(TOOLS)  # type: ignore
 TOOL_DESCRIPTIONS = get_tool_descriptions(TOOLS)  # type: ignore
 TOOL_DOCSTRING = get_tool_documentation(TOOLS)  # type: ignore
 UTILITIES_DOCSTRING = get_tool_documentation(
-    [load_image, save_image, overlay_bounding_boxes]
+    [save_json, load_image, save_image, overlay_bounding_boxes]
 )

{vision_agent-0.2.22 → vision_agent-0.2.24}/vision_agent/utils/sim.py

@@ -56,12 +56,15 @@ class Sim:
         df = df.drop("embs", axis=1)
         df.to_csv(sim_file / "df.csv", index=False)
 
-    def top_k(self, query: str, k: int = 5) -> Sequence[Dict]:
+    def top_k(
+        self, query: str, k: int = 5, thresh: Optional[float] = None
+    ) -> Sequence[Dict]:
         """Returns the top k most similar items to the query.
 
         Parameters:
             query: str: The query to compare to.
             k: int: The number of items to return.
+            thresh: Optional[float]: The minimum similarity threshold.
 
         Returns:
             Sequence[Dict]: The top k most similar items.
@@ -70,6 +73,8 @@ class Sim:
         embedding = get_embedding(self.client, query, model=self.model)
         self.df["sim"] = self.df.embs.apply(lambda x: 1 - cosine(x, embedding))
         res = self.df.sort_values("sim", ascending=False).head(k)
+        if thresh is not None:
+            res = res[res.sim > thresh]
         return res[[c for c in res.columns if c != "embs"]].to_dict(orient="records")
 
 

{vision_agent-0.2.22 → vision_agent-0.2.24}/vision_agent/utils/type_defs.py

@@ -12,7 +12,7 @@ class LandingaiAPIKey(BaseSettings):
     """
 
     api_key: str = Field(
-        default="land_sk_hw34v3tyEc35OAhP8F7hnGnrDv2C8hD2ycMyq0aMkVS1H40D22",
+        default="land_sk_PCRPYKqB3cq0JWGY83hjEk33SWSDOwdNoyUjTgCDMZO4NxeCXW",
         alias="LANDINGAI_API_KEY",
         description="The API key of LandingAI.",
     )

{vision_agent-0.2.22 → vision_agent-0.2.24}/vision_agent/tools/__init__.py

@@ -3,8 +3,8 @@ from .tools import (  # Counter,
     CLIP,
     OCR,
     TOOLS,
-    BboxStats,
     BboxIoU,
+    BboxStats,
     BoxDistance,
     Crop,
     DINOv,