vision-agent 0.2.51__py3-none-any.whl → 0.2.53__py3-none-any.whl

vision_agent/agent/vision_agent_prompts.py

@@ -71,7 +71,7 @@ This is the documentation for the functions you have access to. You may call any
 1. **Understand and Clarify**: Make sure you understand the task.
 2. **Algorithm/Method Selection**: Decide on the most efficient way.
 3. **Pseudocode Creation**: Write down the steps you will follow in pseudocode.
-4. **Code Generation**: Translate your pseudocode into executable Python code. Ensure you use correct arguments, remember coordinates are always returned normalized from `vision_agent.tools`.
+4. **Code Generation**: Translate your pseudocode into executable Python code. Ensure you use correct arguments, remember coordinates are always returned normalized from `vision_agent.tools`. All images from `vision_agent.tools` are in RGB format, red is (255, 0, 0) and blue is (0, 0, 255).
 5. **Logging**: Log the output of the custom functions that were provided to you from `from vision_agent.tools import *`. Use a debug flag in the function parameters to toggle logging on and off.
 """
 

vision_agent/llm/llm.py

@@ -148,7 +148,7 @@ class OpenAILLM(LLM):
 class AzureOpenAILLM(OpenAILLM):
     def __init__(
         self,
-        model_name: str = "gpt-4-turbo-preview",
+        model_name: str = "gpt-4o",
         api_key: Optional[str] = None,
         api_version: str = "2024-02-01",
         azure_endpoint: Optional[str] = None,

vision_agent/lmm/lmm.py

@@ -286,11 +286,12 @@ class OpenAILMM(LMM):
 class AzureOpenAILMM(OpenAILMM):
     def __init__(
         self,
-        model_name: str = "gpt-4-vision-preview",
+        model_name: str = "gpt-4o",
         api_key: Optional[str] = None,
         api_version: str = "2024-02-01",
         azure_endpoint: Optional[str] = None,
         max_tokens: int = 1024,
+        json_mode: bool = False,
         **kwargs: Any,
     ):
         if not api_key:
@@ -307,7 +308,11 @@ class AzureOpenAILMM(OpenAILMM):
             api_key=api_key, api_version=api_version, azure_endpoint=azure_endpoint
         )
         self.model_name = model_name
-        self.max_tokens = max_tokens
+
+        if "max_tokens" not in kwargs:
+            kwargs["max_tokens"] = max_tokens
+        if json_mode:
+            kwargs["response_format"] = {"type": "json_object"}
         self.kwargs = kwargs
 
 

vision_agent/tools/tools.py

@@ -179,7 +179,7 @@ def extract_frames(
 ) -> List[Tuple[np.ndarray, float]]:
     """'extract_frames' extracts frames from a video, returns a list of tuples (frame,
     timestamp), where timestamp is the relative time in seconds where the frame was
-    captured. The frame is a local image file path.
+    captured. The frame is a numpy array.
 
     Parameters:
         video_uri (Union[str, Path]): The path to the video file.
@@ -530,27 +530,22 @@ def load_image(image_path: str) -> np.ndarray:
     return np.array(image)
 
 
-def save_image(image: np.ndarray) -> str:
-    """'save_image' is a utility function that saves an image as a temporary file.
+def save_image(image: np.ndarray, file_path: str) -> None:
+    """'save_image' is a utility function that saves an image to a file path.
 
     Parameters:
         image (np.ndarray): The image to save.
-
-    Returns:
-        str: The path to the saved image.
+        file_path (str): The path to save the image file.
 
     Example
     -------
         >>> save_image(image)
-        "/tmp/tmpabc123.png"
     """
     from IPython.display import display
 
     pil_image = Image.fromarray(image.astype(np.uint8))
     display(pil_image)
-    with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f:
-        pil_image.save(f, "PNG")
-    return f.name
+    pil_image.save(file_path)
 
 
 def save_video(

vision_agent/utils/__init__.py

@@ -6,5 +6,5 @@ from .execute import (
     Logs,
     Result,
 )
-from .sim import Sim, load_sim, merge_sim
+from .sim import AzureSim, Sim, load_sim, merge_sim
 from .video import extract_frames_from_video

vision_agent/utils/sim.py

@@ -1,9 +1,10 @@
+import os
 from pathlib import Path
 from typing import Dict, List, Optional, Sequence, Union
 
 import numpy as np
 import pandas as pd
-from openai import Client
+from openai import AzureOpenAI, Client, OpenAI
 from scipy.spatial.distance import cosine  # type: ignore
 
 
@@ -33,9 +34,9 @@ class Sim:
         """
         self.df = df
         if not api_key:
-            self.client = Client()
+            self.client = OpenAI()
         else:
-            self.client = Client(api_key=api_key)
+            self.client = OpenAI(api_key=api_key)
 
         self.model = model
         if "embs" not in df.columns and sim_key is None:
@@ -78,6 +79,41 @@ class Sim:
         return res[[c for c in res.columns if c != "embs"]].to_dict(orient="records")
 
 
+class AzureSim(Sim):
+    def __init__(
+        self,
+        df: pd.DataFrame,
+        sim_key: Optional[str] = None,
+        api_key: Optional[str] = None,
+        api_version: str = "2024-02-01",
+        azure_endpoint: Optional[str] = None,
+        model: str = "text-embedding-3-small",
+    ) -> None:
+        if not api_key:
+            api_key = os.getenv("AZURE_OPENAI_API_KEY")
+        if not azure_endpoint:
+            azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
+
+        if not api_key:
+            raise ValueError("Azure OpenAI API key is required.")
+        if not azure_endpoint:
+            raise ValueError("Azure OpenAI endpoint is required.")
+
+        self.df = df
+        self.client = AzureOpenAI(
+            api_key=api_key, api_version=api_version, azure_endpoint=azure_endpoint
+        )
+
+        self.model = model
+        if "embs" not in df.columns and sim_key is None:
+            raise ValueError("key is required if no column 'embs' is present.")
+
+        if sim_key is not None:
+            self.df["embs"] = self.df[sim_key].apply(
+                lambda x: get_embedding(self.client, x, model=self.model)
+            )
+
+
 def merge_sim(sim1: Sim, sim2: Sim) -> Sim:
     return Sim(pd.concat([sim1.df, sim2.df], ignore_index=True))
 

{vision_agent-0.2.51.dist-info → vision_agent-0.2.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.51
+Version: 0.2.53
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -195,13 +195,14 @@ export AZURE_OPENAI_ENDPOINT="your-endpoint"
 You can then run Vision Agent using the Azure OpenAI models:
 
 ```python
->>> import vision_agent as va
->>> agent = va.agent.VisionAgent(
->>>     planner=va.llm.AzureOpenAILLM(),
->>>     coder=va.lmm.AzureOpenAILMM(),
->>>     tester=va.lmm.AzureOpenAILMM(),
->>>     debugger=va.lmm.AzureOpenAILMM(),
->>> )
+import vision_agent as va
+import vision_agent.tools as T
+agent = va.agent.VisionAgent(
+    planner=va.llm.AzureOpenAILLM(),
+    coder=va.lmm.AzureOpenAILLM(),
+    tester=va.lmm.AzureOpenAILLM(),
+    debugger=va.lmm.AzureOpenAILLM(),
+    tool_recommender=va.utils.AzureSim(T.TOOLS_DF, sim_key="desc"),
+)
 ```
 
-

{vision_agent-0.2.51.dist-info → vision_agent-0.2.53.dist-info}/RECORD

@@ -12,25 +12,25 @@ vision_agent/agent/easytool_v2_prompts.py,sha256=MZSIwovYgB-f-kdJ6btaNDVXptJn47b
 vision_agent/agent/reflexion.py,sha256=scck3YcME6DhX5Vs4Wr1rYb8S4wkBUkN9UksyazfrZg,10506
 vision_agent/agent/reflexion_prompts.py,sha256=G7UAeNz_g2qCb2yN6OaIC7bQVUkda4m3z42EG8wAyfE,9342
 vision_agent/agent/vision_agent.py,sha256=wGGISg6pDVNseF2fIAN1jH66OX2qZk2nDhuobeSNGHk,20957
-vision_agent/agent/vision_agent_prompts.py,sha256=hgnTlaYp2HMBHLi3e4faPb-DI5jQL9jfhKq9jyEUEgY,8370
+vision_agent/agent/vision_agent_prompts.py,sha256=9QVQA-YTSHhYHYbxiqCWFVBHIa6uV4WF0z6599mV_Oc,8470
 vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
 vision_agent/llm/__init__.py,sha256=BoUm_zSAKnLlE8s-gKTSQugXDqVZKPqYlWwlTLdhcz4,48
-vision_agent/llm/llm.py,sha256=UZ73GqQHE-NKOJWsrOTWfmdHYsbCBkJ5rZ7dhcSCHHw,5951
+vision_agent/llm/llm.py,sha256=jElloDcvNp00e4LJp3jelwa34CwVFzQ_SfOcGr0omK8,5938
 vision_agent/lmm/__init__.py,sha256=nnNeKD1k7q_4vLb1x51O_EUTYaBgGfeiCx5F433gr3M,67
-vision_agent/lmm/lmm.py,sha256=NwcZYLTzi95LSMAk0sTtw7G_zBLa9lU-DHM5GUUCiK4,10622
+vision_agent/lmm/lmm.py,sha256=tvBkG3Ot9l4O7lysRLlh7PyYg_p5ufMcEC0bm_UrnUM,10776
 vision_agent/tools/__init__.py,sha256=Sng6dChynJJCYWjraXXM0tep_VPdnYl3L9vb0HMy_Pc,1528
 vision_agent/tools/easytool_tools.py,sha256=pZc5dQlYINlV4nYbbzsDi3-wauA-fCeD2iGmJUMoUfE,47373
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
 vision_agent/tools/tool_utils.py,sha256=wzRacbUpqk9hhfX_Y08rL8qP0XCN2w-8IZoYLi3Upn4,869
-vision_agent/tools/tools.py,sha256=L1_umAVxk_BlrDYEmV2eyu2cJnpieTW-Ipb03VwKqWU,27062
-vision_agent/utils/__init__.py,sha256=Ce4yPhoWanRsnTy3X7YzZNBYYRJsrJeT7N59WUf8GZM,209
+vision_agent/tools/tools.py,sha256=aEph9ikQklqKzz18jgjO7eC77VqmkJCYEZ8DqtpihYg,26944
+vision_agent/utils/__init__.py,sha256=CW84HnhqI6XQVuxf2KifkLnSuO7EOhmuL09-gAymAak,219
 vision_agent/utils/execute.py,sha256=GqoAodxtwTPBr1nujPTsWiZO2rBGvWVXTe8lgxY4d_g,20603
 vision_agent/utils/image_utils.py,sha256=_cdiS5YrLzqkq_ZgFUO897m5M4_SCIThwUy4lOklfB8,7700
-vision_agent/utils/sim.py,sha256=oUZ-6eu8Io-UNt9GXJ0XRKtP-Wc0sPWVzYGVpB2yDFk,3001
+vision_agent/utils/sim.py,sha256=rGRGnjsy91IOn8qzt7k04PIRj5jyiaQyYAQl7ossPt8,4195
 vision_agent/utils/type_defs.py,sha256=BlI8ywWHAplC7kYWLvt4AOdnKpEW3qWEFm-GEOSkrFQ,1792
 vision_agent/utils/video.py,sha256=rNmU9KEIkZB5-EztZNlUiKYN0mm_55A_2VGUM0QpqLA,8779
-vision_agent-0.2.51.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.51.dist-info/METADATA,sha256=xUYxi6YH3U4QTlYNWZ51YI365ER6NANcYBiVeXN4egQ,6817
-vision_agent-0.2.51.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.51.dist-info/RECORD,,
+vision_agent-0.2.53.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.53.dist-info/METADATA,sha256=0p9P33esnEQAWlqOsXC9YLxcR1R5E0oKURtWZ40hf_U,6887
+vision_agent-0.2.53.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.53.dist-info/RECORD,,