vision-agent 0.2.161__py3-none-any.whl → 0.2.162__py3-none-any.whl

vision_agent/agent/vision_agent_planner_prompts.py

@@ -0,0 +1,199 @@
+USER_REQ = """
+## User Request
+{user_request}
+"""
+
+PLAN = """
+**Context**:
+{context}
+
+**Tools Available**:
+{tool_desc}
+
+**Previous Feedback**:
+{feedback}
+
+**Instructions**:
+1. Based on the context and tools you have available, create a plan of subtasks to achieve the user request.
+2. For each subtask, be sure to include the tool(s) you want to use to accomplish that subtask.
+3. Output three different plans each utilize a different strategy or set of tools ordering them from most likely to least likely to succeed.
+
+Output a list of jsons in the following format:
+
+```json
+{{
+    "plan1":
+        {{
+            "thoughts": str # your thought process for choosing this plan
+            "instructions": [
+                str # what you should do in this task associated with a tool
+            ]
+        }},
+    "plan2": ...,
+    "plan3": ...
+}}
+```
+"""
+
+TEST_PLANS = """
+**Role**: You are a software programmer responsible for testing different tools.
+
+**Task**: Your responsibility is to take a set of several plans and test the different tools for each plan.
+
+**Documentation**:
+This is the documentation for the functions you have access to. You may call any of these functions to help you complete the task. They are available through importing `from vision_agent.tools import *`.
+
+{docstring}
+
+**Plans**:
+{plans}
+
+**Previous Attempts**:
+{previous_attempts}
+
+**Examples**:
+--- EXAMPLE1 ---
+plan1:
+- Load the image from the provided file path 'image.jpg'.
+- Use the 'owl_v2_image' tool with the prompt 'person' to detect and count the number of people in the image.
+plan2:
+- Load the image from the provided file path 'image.jpg'.
+- Use the 'florence2_sam2_image' tool with the prompt 'person' to detect and count the number of people in the image.
+- Count the number of detected objects labeled as 'person'.
+plan3:
+- Load the image from the provided file path 'image.jpg'.
+- Use the 'countgd_counting' tool to count the dominant foreground object, which in this case is people.
+
+```python
+from vision_agent.tools import load_image, owl_v2_image, florence2_sam2_image, countgd_counting
+image = load_image("image.jpg")
+owl_v2_out = owl_v2_image("person", image)
+
+f2s2_out = florence2_sam2_image("person", image)
+# strip out the masks from the output becuase they don't provide useful information when printed
+f2s2_out = [{{k: v for k, v in o.items() if k != "mask"}} for o in f2s2_out]
+
+cgd_out = countgd_counting(image)
+
+final_out = {{"owl_v2_image": owl_v2_out, "florence2_sam2_image": f2s2, "countgd_counting": cgd_out}}
+print(final_out)
+--- END EXAMPLE1 ---
+
+--- EXAMPLE2 ---
+plan1:
+- Extract frames from 'video.mp4' at 10 FPS using the 'extract_frames_and_timestamps' tool.
+- Use the 'owl_v2_video' tool with the prompt 'person' to detect where the people are in the video.
+plan2:
+- Extract frames from 'video.mp4' at 10 FPS using the 'extract_frames_and_timestamps' tool.
+- Use the 'florence2_phrase_grounding' tool with the prompt 'person' to detect where the people are in the video.
+plan3:
+- Extract frames from 'video.mp4' at 10 FPS using the 'extract_frames_and_timestamps' tool.
+- Use the 'florence2_sam2_video_tracking' tool with the prompt 'person' to detect where the people are in the video.
+
+
+```python
+import numpy as np
+from vision_agent.tools import extract_frames_and_timestamps, owl_v2_video, florence2_phrase_grounding, florence2_sam2_video_tracking
+
+# sample at 1 FPS and use the first 10 frames to reduce processing time
+frames = extract_frames_and_timestamps("video.mp4", 1)
+frames = [f["frame"] for f in frames][:10]
+
+# strip arrays from the output to make it easier to read
+def remove_arrays(o):
+    if isinstance(o, list):
+        return [remove_arrays(e) for e in o]
+    elif isinstance(o, dict):
+        return {{k: remove_arrays(v) for k, v in o.items()}}
+    elif isinstance(o, np.ndarray):
+        return "array: " + str(o.shape)
+    else:
+        return o
+
+# return the counts of each label per frame to help determine the stability of the model results
+def get_counts(preds):
+    counts = {{}}
+    for i, pred_frame in enumerate(preds):
+        counts_i = {{}}
+        for pred in pred_frame:
+            label = pred["label"].split(":")[1] if ":" in pred["label"] else pred["label"]
+            counts_i[label] = counts_i.get(label, 0) + 1
+        counts[f"frame_{{i}}"] = counts_i
+    return counts
+
+
+# plan1
+owl_v2_out = owl_v2_video("person", frames)
+owl_v2_counts = get_counts(owl_v2_out)
+
+# plan2
+florence2_out = [florence2_phrase_grounding("person", f) for f in frames]
+florence2_counts = get_counts(florence2_out)
+
+# plan3
+f2s2_tracking_out = florence2_sam2_video_tracking("person", frames)
+remove_arrays(f2s2_tracking_out)
+f2s2_counts = get_counts(f2s2_tracking_out)
+
+final_out = {{
+    "owl_v2_video": owl_v2_out,
+    "florence2_phrase_grounding": florence2_out,
+    "florence2_sam2_video_tracking": f2s2_out,
+}}
+
+counts = {{
+    "owl_v2_video": owl_v2_counts,
+    "florence2_phrase_grounding": florence2_counts,
+    "florence2_sam2_video_tracking": f2s2_counts,
+}}
+
+print(final_out)
+print(labels_and_scores)
+print(counts)
+```
+--- END EXAMPLE2 ---
+
+**Instructions**:
+1. Write a program to load the media and call each tool and print it's output along with other relevant information.
+2. Create a dictionary where the keys are the tool name and the values are the tool outputs. Remove numpy arrays from the printed dictionary.
+3. Your test case MUST run only on the given images which are {media}
+4. Print this final dictionary.
+5. For video input, sample at 1 FPS and use the first 10 frames only to reduce processing time.
+"""
+
+PREVIOUS_FAILED = """
+**Previous Failed Attempts**:
+You previously ran this code:
+```python
+{code}
+```
+
+But got the following error or no stdout:
+{error}
+"""
+
+PICK_PLAN = """
+**Role**: You are an advanced AI model that can understand the user request and construct plans to accomplish it.
+
+**Task**: Your responsibility is to pick the best plan from the three plans provided.
+
+**Context**:
+{context}
+
+**Plans**:
+{plans}
+
+**Tool Output**:
+{tool_output}
+
+**Instructions**:
+1. Re-read the user request, plans, tool outputs and examine the image.
+2. Solve the problem yourself given the image and pick the most accurate plan that matches your solution the best.
+3. Add modifications to improve the plan including: changing a tool, adding thresholds, string matching.
+3. Output a JSON object with the following format:
+{{
+    "predicted_answer": str # the answer you would expect from the best plan
+    "thoughts": str # your thought process for choosing the best plan over other plans and any modifications you made
+    "best_plan": str # the best plan you have chosen, must be `plan1`, `plan2`, or `plan3`
+}}
+"""

vision_agent/tools/__init__.py

@@ -1,6 +1,5 @@
 from typing import Callable, List, Optional
 
-from .meta_tools import META_TOOL_DOCSTRING, Artifacts
 from .prompts import CHOOSE_PARAMS, SYSTEM_PROMPT
 from .tool_utils import get_tool_descriptions_by_names
 from .tools import (

vision_agent/tools/meta_tools.py

@@ -13,7 +13,9 @@ import numpy as np
 from IPython.display import display
 
 import vision_agent as va
+from vision_agent.agent.agent_utils import extract_json
 from vision_agent.clients.landing_public_api import LandingPublicAPI
+from vision_agent.lmm import AnthropicLMM
 from vision_agent.lmm.types import Message
 from vision_agent.tools.tool_utils import get_tool_documentation
 from vision_agent.tools.tools import TOOL_DESCRIPTIONS
@@ -338,6 +340,85 @@ def edit_code_artifact(
     return open_code_artifact(artifacts, name, cur_line)
 
 
+def generate_vision_plan(
+    artifacts: Artifacts,
+    name: str,
+    chat: str,
+    media: List[str],
+    test_multi_plan: bool = True,
+    custom_tool_names: Optional[List[str]] = None,
+) -> str:
+    """Generates a plan to solve vision based tasks.
+
+    Parameters:
+        artifacts (Artifacts): The artifacts object to save the plan to.
+        name (str): The name of the artifact to save the plan context to.
+        chat (str): The chat message from the user.
+        media (List[str]): The media files to use.
+        test_multi_plan (bool): Do not change this parameter.
+        custom_tool_names (Optional[List[str]]): Do not change this parameter.
+
+    Returns:
+        str: The generated plan.
+
+    Examples
+    --------
+        >>> generate_vision_plan(artifacts, "plan.json", "Can you detect the dogs in this image?", ["image.jpg"])
+        [Start Plan Context]
+        plan1: This is a plan to detect dogs in an image
+        -load image
+        -detect dogs
+        -return detections
+        [End Plan Context]
+    """
+
+    if ZMQ_PORT is not None:
+        agent = va.agent.VisionAgentPlanner(
+            report_progress_callback=lambda inp: report_progress_callback(
+                int(ZMQ_PORT), inp
+            )
+        )
+    else:
+        agent = va.agent.VisionAgentPlanner()
+
+    fixed_chat: List[Message] = [{"role": "user", "content": chat, "media": media}]
+    response = agent.generate_plan(
+        fixed_chat,
+        test_multi_plan=test_multi_plan,
+        custom_tool_names=custom_tool_names,
+    )
+    if response.test_results is not None:
+        redisplay_results(response.test_results)
+    response.test_results = None
+    artifacts[name] = response.model_dump_json()
+    media_names = extract_json(
+        AnthropicLMM()(  # type: ignore
+            f"""Extract any media file names from this output in the following JSON format:
+{{"media": ["image1.jpg", "image2.jpg"]}}
+
+{artifacts[name]}"""
+        )
+    )
+    if "media" in media_names and isinstance(media_names, dict):
+        for media in media_names["media"]:
+            if isinstance(media, str):
+                with open(media, "rb") as f:
+                    artifacts[media] = f.read()
+
+    output_str = f"[Start Plan Context, saved at {name}]"
+    for plan in response.plans.keys():
+        output_str += f"\n{plan}: {response.plans[plan]['thoughts'].strip()}\n"  # type: ignore
+        output_str += "    -" + "\n    -".join(
+            e.strip() for e in response.plans[plan]["instructions"]
+        )
+
+    output_str += f"\nbest plan: {response.best_plan}\n"
+    output_str += "thoughts: " + response.plan_thoughts.strip() + "\n"
+    output_str += "[End Plan Context]"
+    print(output_str)
+    return output_str
+
+
 def generate_vision_code(
     artifacts: Artifacts,
     name: str,
@@ -368,7 +449,6 @@ def generate_vision_code(
             dogs = owl_v2("dog", image)
             return dogs
     """
-
     if ZMQ_PORT is not None:
         agent = va.agent.VisionAgentCoder(
             report_progress_callback=lambda inp: report_progress_callback(
@@ -379,7 +459,7 @@ def generate_vision_code(
         agent = va.agent.VisionAgentCoder(verbosity=int(VERBOSITY))
 
     fixed_chat: List[Message] = [{"role": "user", "content": chat, "media": media}]
-    response = agent.chat_with_workflow(
+    response = agent.generate_code(
         fixed_chat,
         test_multi_plan=test_multi_plan,
         custom_tool_names=custom_tool_names,
@@ -459,7 +539,7 @@ def edit_vision_code(
             fixed_chat_history.append({"role": "assistant", "content": code})
             fixed_chat_history.append({"role": "user", "content": chat})
 
-    response = agent.chat_with_workflow(
+    response = agent.generate_code(
         fixed_chat_history,
         test_multi_plan=False,
         custom_tool_names=customized_tool_names,
@@ -748,6 +828,7 @@ META_TOOL_DOCSTRING = get_tool_documentation(
         open_code_artifact,
         create_code_artifact,
         edit_code_artifact,
+        generate_vision_plan,
         generate_vision_code,
         edit_vision_code,
         write_media_artifact,

{vision_agent-0.2.161.dist-info → vision_agent-0.2.162.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.161
+Version: 0.2.162
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -142,7 +142,7 @@ continuing, for example it may want to execute code and look at the output befor
 letting the user respond.
 
 ### Chatting and Artifacts
-If you run `chat_with_code` you will also notice an `Artifact` object. `Artifact`'s
+If you run `chat_with_artifacts` you will also notice an `Artifact` object. `Artifact`'s
 are a way to sync files between local and remote environments. The agent will read and
 write to the artifact object, which is just a pickle object, when it wants to save or
 load files.
@@ -159,7 +159,7 @@ with open("image.png", "rb") as f:
     artifacts["image.png"] = f.read()
 
 agent = va.agent.VisionAgent()
-response, artifacts = agent.chat_with_code(
+response, artifacts = agent.chat_with_artifacts(
     [
         {
             "role": "user",
@@ -339,11 +339,11 @@ mode by passing in the verbose argument:
 ```
 
 ### Detailed Usage
-You can also have it return more information by calling `chat_with_workflow`. The format
+You can also have it return more information by calling `generate_code`. The format
 of the input is a list of dictionaries with the keys `role`, `content`, and `media`:
 
 ```python
->>> results = agent.chat_with_workflow([{"role": "user", "content": "What percentage of the area of the jar is filled with coffee beans?", "media": ["jar.jpg"]}])
+>>> results = agent.generate_code([{"role": "user", "content": "What percentage of the area of the jar is filled with coffee beans?", "media": ["jar.jpg"]}])
 >>> print(results)
 {
     "code": "from vision_agent.tools import ..."
@@ -372,7 +372,7 @@ conv = [
         "media": ["workers.png"],
     }
 ]
-result = agent.chat_with_workflow(conv)
+result = agent.generate_code(conv)
 code = result["code"]
 conv.append({"role": "assistant", "content": code})
 conv.append(
@@ -381,7 +381,7 @@ conv.append(
         "content": "Can you also return the number of workers wearing safety gear?",
     }
 )
-result = agent.chat_with_workflow(conv)
+result = agent.generate_code(conv)
 ```
 
 

{vision_agent-0.2.161.dist-info → vision_agent-0.2.162.dist-info}/RECORD

@@ -1,10 +1,12 @@
 vision_agent/__init__.py,sha256=EAb4-f9iyuEYkBrX4ag1syM8Syx8118_t0R6_C34M9w,57
-vision_agent/agent/__init__.py,sha256=NF2LABqHixLvbsOIO-fe-VKZ7awvShLtcT0oQT4eWtI,235
+vision_agent/agent/__init__.py,sha256=RRMPhH8mgm_pCtEKiVFSjJyDi4lCr4F7k05AhK01xlM,436
 vision_agent/agent/agent.py,sha256=2cjIOxEuSJrqbfPXYoV0qER5ihXsPFCoEFJa4jpqan0,597
-vision_agent/agent/agent_utils.py,sha256=eIpLz2NunEqEsBBrECJaD34-2uY0bsFNnW-XKfqqohs,2518
-vision_agent/agent/vision_agent.py,sha256=x0oSuQk-rcERUUdZp29FB77Ua-eyQD3fLi_UfmsBTV0,20761
-vision_agent/agent/vision_agent_coder.py,sha256=2ZoGikn2nakGDfs20XRshZjQUyvbw6l47UhExJAYkqI,38515
-vision_agent/agent/vision_agent_coder_prompts.py,sha256=BmbTMhth4v1qLexuoSeyo47QQ0kPQvL1pLbCJHMsWDw,18910
+vision_agent/agent/agent_utils.py,sha256=AAIqi8U3Cc58RH_AJ6grxIHpgdu9AjctmapokLp88pQ,4766
+vision_agent/agent/vision_agent.py,sha256=ycPuFzUh043ltdR9F3oNeTcud5-lO0ydrsVzuIAKWcY,22567
+vision_agent/agent/vision_agent_coder.py,sha256=ISw85eqVRpFdBMs974Rg8VqjTfbcn2XiwBfPe-u9pNI,28214
+vision_agent/agent/vision_agent_coder_prompts.py,sha256=zQJ2PxX7xWA0RsZrCT8F59s-5F4WpPAqMRdmUA5_UW4,12215
+vision_agent/agent/vision_agent_planner.py,sha256=xsE9W2DJsUygmyD_aIYp-4o4LSzfe8mRVGrPTe0b3CI,17975
+vision_agent/agent/vision_agent_planner_prompts.py,sha256=JDARUzko2HZdxkBtcy6wuP9DCCmbqhK_gnVgrjr6l1k,6691
 vision_agent/agent/vision_agent_prompts.py,sha256=LZ9Bnx7ZFkqbNOMqwfdiWZU4niND9Z1ArcFHNSn_jzA,11187
 vision_agent/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/clients/http.py,sha256=k883i6M_4nl7zwwHSI-yP5sAgQZIDPM1nrKD6YFJ3Xs,2009
@@ -14,8 +16,8 @@ vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1r
 vision_agent/lmm/__init__.py,sha256=jyY1sJb_tYKg5-Wzs3p1lvwFkc-aUNZfMcLy3TOC4Zg,100
 vision_agent/lmm/lmm.py,sha256=B5ClgwvbybVCWkf9opDMLjTtJZemUU4KUkQoRxGh43I,16787
 vision_agent/lmm/types.py,sha256=ZEXR_ptBL0ZwDMTDYkgxUCmSZFmBYPQd2jreNzr_8UY,221
-vision_agent/tools/__init__.py,sha256=PLVbfTMjKxQlHIRWnq9b785W9a52AXQS_tOa0tkQ0ZY,2420
-vision_agent/tools/meta_tools.py,sha256=BF5-fVshLhhpck5lJErKxnfPu9YxudBhR7ar_qA9Mjo,25889
+vision_agent/tools/__init__.py,sha256=50wwisjudmZn7_SEwigTiiDxQ0HXbSIhVI4O8kvE9Es,2365
+vision_agent/tools/meta_tools.py,sha256=T6M-O0uymeVltOggC-Qr1EzSpSnXmy0HTD7j2ZElY6s,28732
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
 vision_agent/tools/tool_utils.py,sha256=VPGqGJ2ZYEJA6AW7K9X7hQv6vRlMtAQcybE4izdToCw,8196
 vision_agent/tools/tools.py,sha256=vS1yCk3Fza9eYOTHPFwwroo_ULdw2ztMQMb81x1U5f8,78524
@@ -27,7 +29,7 @@ vision_agent/utils/image_utils.py,sha256=rm9GfXvD4JrjnqKrP_f2gfq4SzmqYC0IdC1kKwd
 vision_agent/utils/sim.py,sha256=ZuSS07TUXFGjipmiQoY8TKRmSes7XXCdtU9PI8PC1sw,5609
 vision_agent/utils/type_defs.py,sha256=BE12s3JNQy36QvauXHjwyeffVh5enfcvd4vTzSwvEZI,1384
 vision_agent/utils/video.py,sha256=xbMEoRk13l4fHeQlbvMQhLCn8RNndYmsDhUf01TUeR8,4781
-vision_agent-0.2.161.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.161.dist-info/METADATA,sha256=81mQ74IJal478wgLMjkneAVg0kE89VVjX9Wa_hL0lMo,17753
-vision_agent-0.2.161.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.161.dist-info/RECORD,,
+vision_agent-0.2.162.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.162.dist-info/METADATA,sha256=lc7obdExLHxIWS6zTG_wUQq_1PmVpwrlMkR4-fVj75M,17743
+vision_agent-0.2.162.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.162.dist-info/RECORD,,